[Pkg-ceph-commits] [ceph] 04/10: Imported Upstream version 0.78
James Downing Page
jamespage at moszumanska.debian.org
Mon Mar 24 09:31:19 UTC 2014
This is an automated email from the git hooks/post-receive script.
jamespage pushed a commit to branch experimental
in repository ceph.
commit 3b8df304177ae1e56cb56599e1cb5af78c402fd4
Author: James Page <jamespage at debian.org>
Date: Sat Mar 22 18:27:33 2014 +0000
Imported Upstream version 0.78
---
README | 8 +-
ceph.spec | 16 +-
ceph.spec.in | 14 +-
configure | 144 +-
configure.ac | 52 +-
src/.git_version | 4 +-
src/Makefile-env.am | 1 +
src/Makefile.am | 5 +-
src/Makefile.in | 3134 +++---
src/acconfig.h.in | 3 +
src/auth/cephx/CephxSessionHandler.cc | 17 +-
src/brag/Makefile.am | 3 +
src/brag/README.md | 184 +
src/brag/client/ceph-brag | 349 +
src/brag/server/MANIFEST.in | 1 +
src/brag/server/app.wsgi | 5 +
src/brag/server/ceph_brag.egg-info/PKG-INFO | 10 +
src/brag/server/ceph_brag.egg-info/SOURCES.txt | 21 +
.../server/ceph_brag.egg-info/dependency_links.txt | 1 +
src/brag/server/ceph_brag.egg-info/not-zip-safe | 1 +
src/brag/server/ceph_brag.egg-info/requires.txt | 1 +
src/brag/server/ceph_brag.egg-info/top_level.txt | 1 +
src/brag/server/ceph_brag/__init__.py | 0
src/brag/server/ceph_brag/app.py | 19 +
src/brag/server/ceph_brag/controllers/__init__.py | 0
src/brag/server/ceph_brag/controllers/root.py | 73 +
src/brag/server/ceph_brag/json.py | 114 +
src/brag/server/ceph_brag/model/__init__.py | 29 +
src/brag/server/ceph_brag/model/db.py | 282 +
src/brag/server/ceph_brag/tests/__init__.py | 22 +
src/brag/server/ceph_brag/tests/config.py | 54 +
src/brag/server/ceph_brag/tests/test_functional.py | 68 +
src/brag/server/ceph_brag/tests/test_units.py | 7 +
src/brag/server/config.py | 52 +
src/brag/server/sample.json | 98 +
src/brag/server/setup.cfg | 6 +
src/brag/server/setup.py | 22 +
src/ceph-disk | 96 +-
src/ceph.in | 7 +
src/ceph_mon.cc | 45 +-
src/client/Client.cc | 858 +-
src/client/Client.h | 113 +-
src/client/Dentry.h | 10 +-
src/client/Fh.h | 2 +-
src/client/Inode.cc | 6 +-
src/client/Inode.h | 14 +-
src/client/SyntheticClient.cc | 150 +-
src/client/fuse_ll.cc | 200 +-
src/cls/rgw/cls_rgw.cc | 25 +-
src/cls/rgw/cls_rgw_client.cc | 3 +-
src/cls/rgw/cls_rgw_client.h | 2 +-
src/cls/rgw/cls_rgw_ops.cc | 1 +
src/cls/rgw/cls_rgw_ops.h | 11 +-
src/cls/user/cls_user.cc | 24 +-
src/common/Makefile.am | 3 +
src/common/PrioritizedQueue.h | 4 +
src/common/Throttle.cc | 67 +-
src/common/TrackedOp.cc | 22 +-
src/common/TrackedOp.h | 8 +-
src/common/admin_socket.cc | 5 +
src/common/buffer.cc | 6 +-
src/common/ceph_argparse.cc | 42 +-
src/common/ceph_context.cc | 2 +
src/common/ceph_strings.cc | 4 +
src/common/config.cc | 6 +-
src/common/config.h | 2 +
src/common/config_opts.h | 59 +-
src/common/histogram.cc | 58 +
src/{include => common}/histogram.h | 67 +-
src/common/hobject.cc | 3 +-
src/common/hobject.h | 6 +-
src/common/obj_bencher.cc | 3 +-
src/common/shared_cache.hpp | 22 +-
src/common/str_map.cc | 2 +-
src/crush/CrushCompiler.cc | 15 +
src/crush/CrushWrapper.cc | 151 +-
src/crush/CrushWrapper.h | 47 +-
src/crush/builder.c | 1 +
src/crush/crush.h | 7 +
src/crush/grammar.h | 4 +
src/crush/mapper.c | 35 +-
src/{osd => erasure-code}/ErasureCodeInterface.h | 20 +
src/{osd => erasure-code}/ErasureCodePlugin.cc | 0
src/{osd => erasure-code}/ErasureCodePlugin.h | 0
src/erasure-code/Makefile.am | 17 +
.../jerasure}/ErasureCodeJerasure.cc | 40 +-
.../jerasure}/ErasureCodeJerasure.h | 12 +-
.../jerasure}/ErasureCodePluginJerasure.cc | 2 +-
src/erasure-code/jerasure/Makefile.am | 28 +
.../jerasure}/cauchy.c | 0
.../jerasure}/cauchy.h | 0
.../jerasure}/galois.c | 0
.../jerasure}/galois.h | 0
.../jerasure}/jerasure.c | 0
.../jerasure}/jerasure.h | 0
.../jerasure}/liberation.c | 0
.../jerasure}/liberation.h | 0
.../jerasure}/reed_sol.c | 0
.../jerasure}/reed_sol.h | 0
.../jerasure}/vectorop.h | 0
src/global/global_init.cc | 17 +-
src/global/global_init.h | 12 +-
src/include/Makefile.am | 1 -
src/include/buffer.h | 1 +
src/include/ceph_features.h | 5 +
src/include/ceph_fs.h | 13 +-
src/include/cephfs/libcephfs.h | 184 +-
src/include/cmp.h | 72 +-
src/include/encoding.h | 6 +
src/include/rados.h | 12 +-
src/include/rados/buffer.h | 1 +
src/include/rados/librados.h | 494 +-
src/include/rados/librados.hpp | 35 +-
src/init-ceph.in | 15 +-
src/libcephfs.cc | 323 +-
src/librados/IoCtxImpl.cc | 22 +-
src/librados/IoCtxImpl.h | 8 +-
src/librados/RadosClient.cc | 81 +-
src/librados/RadosClient.h | 4 +-
src/librados/librados.cc | 472 +-
src/librbd/AioRequest.cc | 8 +-
src/librbd/AioRequest.h | 5 +-
src/logrotate.conf | 18 +-
src/mds/CDir.cc | 45 +-
src/mds/CDir.h | 14 +-
src/mds/CInode.cc | 179 +-
src/mds/CInode.h | 22 +-
src/mds/Dumper.cc | 3 +-
src/mds/Locker.cc | 174 +-
src/mds/Locker.h | 6 +-
src/mds/MDBalancer.cc | 3 +-
src/mds/MDCache.cc | 849 +-
src/mds/MDCache.h | 35 +-
src/mds/MDS.cc | 45 +-
src/mds/MDS.h | 2 +-
src/mds/Migrator.cc | 297 +-
src/mds/Migrator.h | 55 +-
src/mds/Mutation.h | 8 +
src/mds/Resetter.cc | 3 +-
src/mds/ScatterLock.h | 23 +
src/mds/Server.cc | 162 +-
src/mds/Server.h | 3 +-
src/mds/SimpleLock.h | 9 +-
src/mds/events/EFragment.h | 4 +-
src/mds/events/EMetaBlob.h | 16 +-
src/mds/flock.cc | 16 +-
src/mds/flock.h | 22 +-
src/mds/journal.cc | 32 +-
src/mds/mdstypes.cc | 8 +-
src/mds/mdstypes.h | 4 +-
src/messages/MBackfillReserve.h | 16 +-
src/messages/MClientRequest.h | 2 +-
src/messages/MExportDirPrepAck.h | 10 +-
src/messages/MMDSCacheRejoin.h | 7 +-
src/messages/MMDSFragmentNotify.h | 4 +-
src/messages/MMDSSlaveRequest.h | 12 +-
src/messages/{MOSDPGPull.h => MOSDECSubOpRead.h} | 45 +-
.../{MOSDPGPull.h => MOSDECSubOpReadReply.h} | 45 +-
src/messages/{MOSDPGPull.h => MOSDECSubOpWrite.h} | 53 +-
.../{MOSDPGPull.h => MOSDECSubOpWriteReply.h} | 45 +-
src/messages/MOSDOp.h | 5 +
src/messages/MOSDOpReply.h | 9 +-
src/messages/MOSDPGBackfill.h | 16 +-
src/messages/MOSDPGInfo.h | 20 +-
src/messages/MOSDPGLog.h | 28 +-
src/messages/MOSDPGNotify.h | 20 +-
src/messages/MOSDPGPull.h | 18 +-
src/messages/MOSDPGPush.h | 18 +-
src/messages/MOSDPGPushReply.h | 19 +-
src/messages/MOSDPGQuery.h | 39 +-
src/messages/MOSDPGRemove.h | 42 +-
src/messages/MOSDPGScan.h | 31 +-
src/messages/MOSDPGTrim.h | 19 +-
src/messages/MOSDRepScrub.h | 19 +-
src/messages/MOSDSubOp.h | 27 +-
src/messages/MOSDSubOpReply.h | 31 +-
src/messages/MRecoveryReserve.h | 15 +-
src/messages/Makefile.am | 4 +
src/mon/Elector.cc | 5 +-
src/mon/MDSMonitor.cc | 62 +-
src/mon/Makefile.am | 2 +-
src/mon/MonClient.cc | 57 +-
src/mon/MonClient.h | 50 +-
src/mon/MonCommands.h | 37 +-
src/mon/Monitor.cc | 85 +-
src/mon/Monitor.h | 9 +-
src/mon/OSDMonitor.cc | 866 +-
src/mon/OSDMonitor.h | 26 +-
src/mon/PGMap.cc | 21 +-
src/mon/PGMap.h | 8 +
src/mon/PGMonitor.cc | 102 +-
src/mon/PGMonitor.h | 2 +-
src/msg/Accepter.cc | 2 +-
src/msg/Message.cc | 17 +
src/msg/Message.h | 8 +
src/msg/Pipe.cc | 6 +-
src/msg/SimpleMessenger.cc | 5 +-
src/os/DBObjectMap.cc | 11 +-
src/os/FileJournal.cc | 12 +-
src/os/FileJournal.h | 2 +-
src/os/FileStore.cc | 97 +-
src/os/FileStore.h | 34 +-
src/os/GenericFileStoreBackend.h | 1 +
src/os/GenericObjectMap.cc | 51 +-
src/os/GenericObjectMap.h | 36 +-
src/os/Journal.h | 2 +-
src/os/JournalingObjectStore.cc | 4 +-
src/os/KeyValueStore.cc | 833 +-
src/os/KeyValueStore.h | 91 +-
src/os/LFNIndex.cc | 12 +-
src/os/Makefile.am | 5 +
src/os/MemStore.cc | 4 +
src/os/ObjectStore.cc | 30 +-
src/os/ObjectStore.h | 55 +-
src/os/XfsFileStoreBackend.cc | 124 +
src/os/XfsFileStoreBackend.h | 33 +
src/osd/ECBackend.cc | 1765 ++++
src/osd/ECBackend.h | 476 +
src/osd/ECMsgTypes.cc | 333 +
src/osd/ECMsgTypes.h | 108 +
src/osd/ECTransaction.cc | 283 +
src/osd/ECTransaction.h | 207 +
src/osd/ECUtil.cc | 196 +
src/osd/ECUtil.h | 154 +
src/osd/ErasureCodePluginJerasure/Makefile.am | 28 -
src/osd/HitSet.h | 2 +
src/osd/Makefile.am | 20 +-
src/osd/OSD.cc | 923 +-
src/osd/OSD.h | 328 +-
src/osd/OSDMap.cc | 219 +-
src/osd/OSDMap.h | 70 +-
src/osd/PG.cc | 1817 ++--
src/osd/PG.h | 518 +-
src/osd/PGBackend.cc | 498 +
src/osd/PGBackend.h | 291 +-
src/osd/PGLog.cc | 388 +-
src/osd/PGLog.h | 96 +-
src/osd/ReplicatedBackend.cc | 572 +-
src/osd/ReplicatedBackend.h | 211 +-
src/osd/ReplicatedPG.cc | 2231 ++--
src/osd/ReplicatedPG.h | 261 +-
src/osd/SnapMapper.cc | 4 +-
src/osd/SnapMapper.h | 28 +-
src/osd/TierAgentState.h | 112 +
src/osd/osd_types.cc | 328 +-
src/osd/osd_types.h | 283 +-
src/osdc/ObjectCacher.cc | 27 +-
src/osdc/ObjectCacher.h | 6 +-
src/osdc/Objecter.cc | 287 +-
src/osdc/Objecter.h | 118 +-
src/pybind/rbd.py | 6 +-
src/rbd_fuse/rbd-fuse.c | 34 +-
src/rgw/logrotate.conf | 15 +-
src/rgw/rgw_admin.cc | 12 +-
src/rgw/rgw_common.cc | 3 +-
src/rgw/rgw_common.h | 5 +
src/rgw/rgw_dencoder.cc | 10 -
src/rgw/rgw_gc.cc | 6 +-
src/rgw/rgw_gc.h | 2 +-
src/rgw/rgw_json_enc.cc | 15 +
src/rgw/rgw_log.cc | 3 +
src/rgw/rgw_multi.cc | 3 +-
src/rgw/rgw_op.cc | 77 +-
src/rgw/rgw_op.h | 8 +-
src/rgw/rgw_rados.cc | 637 +-
src/rgw/rgw_rados.h | 370 +-
src/rgw/rgw_rest_swift.cc | 104 +-
src/rgw/rgw_swift.cc | 5 +-
src/test/Makefile.am | 331 +-
src/test/admin_socket.cc | 32 +
src/test/bufferlist.cc | 2 +-
src/test/ceph_argparse.cc | 109 +
src/test/cli/crushtool/help.t | 2 +
src/test/cli/crushtool/test-map-firefly-tunables.t | 10259 +++++++++++++++++++
src/test/cli/crushtool/test-map-vary-r-0.t | 3081 ++++++
src/test/cli/crushtool/test-map-vary-r-1.t | 3078 ++++++
src/test/cli/crushtool/test-map-vary-r-2.t | 3078 ++++++
src/test/cli/crushtool/test-map-vary-r-3.t | 3078 ++++++
src/test/cli/crushtool/test-map-vary-r-4.t | 3078 ++++++
src/test/cli/crushtool/test-map-vary-r.crushmap | Bin 0 -> 3892 bytes
src/test/cli/osdmaptool/clobber.t | 12 +-
src/test/cli/osdmaptool/create-print.t | 12 +-
src/test/cli/osdmaptool/create-racks.t | 12 +-
src/test/cli/osdmaptool/crush.t | 10 +
src/test/cli/osdmaptool/help.t | 4 +
.../osdmaptool/{simple.t => missing-argument.t} | 4 +
src/test/cli/osdmaptool/pool.t | 54 +
src/test/cli/osdmaptool/test-map-pgs.t | 52 +
src/test/cli/radosgw-admin/help.t | 3 +-
src/test/cls_rgw/test_cls_rgw.cc | 12 +-
src/test/common/histogram.cc | 126 +
src/test/common/test_context.cc | 64 +
src/test/crush/TestCrushWrapper.cc | 158 +-
src/test/encoding/types.h | 11 +-
.../{osd => erasure-code}/ErasureCodeExample.h | 12 +-
.../ErasureCodePluginExample.cc | 2 +-
.../ErasureCodePluginFailToInitialize.cc | 2 +-
.../ErasureCodePluginFailToRegister.cc | 2 +-
.../ErasureCodePluginHangs.cc | 2 +-
.../ErasureCodePluginMissingEntryPoint.cc | 0
src/test/erasure-code/Makefile.am | 89 +
.../TestErasureCodeExample.cc | 32 +
.../TestErasureCodeJerasure.cc | 87 +-
.../{osd => erasure-code}/TestErasureCodePlugin.cc | 2 +-
.../TestErasureCodePluginJerasure.cc | 2 +-
src/test/erasure-code/ceph_erasure_code.cc | 166 +
.../ceph_erasure_code_benchmark.cc | 8 +-
.../ceph_erasure_code_benchmark.h | 2 +-
src/test/libcephfs/test.cc | 39 +
src/test/librados/TestCase.cc | 93 +
src/test/librados/TestCase.h | 56 +
src/test/librados/aio.cc | 4 +-
src/test/librados/c_read_operations.cc | 539 +
src/test/librados/c_write_operations.cc | 41 +-
src/test/librados/cmd.cc | 31 +-
src/test/librados/io.cc | 308 +-
src/test/librados/list.cc | 161 +-
src/test/librados/lock.cc | 148 +-
src/test/librados/misc.cc | 182 +-
src/test/librados/snapshots.cc | 195 +-
src/test/librados/stat.cc | 74 +-
src/test/librados/test.cc | 66 +-
src/test/librados/test.h | 1 +
src/test/librados/tier.cc | 735 +-
src/test/librados/watch_notify.cc | 38 +-
src/test/mon/PGMap.cc | 96 +
src/test/mon/mon-test-helpers.sh | 82 +
src/test/mon/test_mon_workloadgen.cc | 2 +-
.../DeterministicOpSequence.cc | 6 +-
.../DeterministicOpSequence.h | 8 +-
.../{filestore => objectstore}/FileStoreDiff.cc | 0
.../{filestore => objectstore}/FileStoreDiff.h | 0
.../{filestore => objectstore}/FileStoreTracker.cc | 0
.../{filestore => objectstore}/FileStoreTracker.h | 0
.../TestObjectStoreState.cc} | 44 +-
.../TestObjectStoreState.h} | 18 +-
src/test/{filestore => objectstore}/chain_xattr.cc | 0
src/test/{filestore => objectstore}/store_test.cc | 214 +-
.../{filestore => objectstore}/test_idempotent.cc | 0
.../test_idempotent_sequence.cc | 0
.../workload_generator.cc | 18 +-
.../workload_generator.h | 12 +-
src/test/osd/Object.cc | 8 +-
src/test/osd/Object.h | 25 +-
src/test/osd/RadosModel.h | 65 +-
src/test/osd/TestECBackend.cc | 60 +
src/test/osd/TestOSDMap.cc | 109 +
src/test/osd/TestPGLog.cc | 599 +-
src/test/osd/TestRados.cc | 2 +-
src/test/osd/osd-test-helpers.sh | 53 +
src/test/{test_osd_types.cc => osd/types.cc} | 159 +-
src/test/rgw/test_rgw_manifest.cc | 227 +
src/test/system/rados_watch_notify.cc | 8 +-
src/test/system/st_rados_watch.cc | 12 +-
src/test/system/st_rados_watch.h | 2 +
src/test/test_crushwrapper.cc | 107 -
src/test/test_snap_mapper.cc | 3 +-
src/tools/ceph-filestore-dump.cc | 41 +-
src/tools/ceph-filestore-tool.cc | 4 +-
src/tools/crushtool.cc | 10 +
src/tools/osdmaptool.cc | 148 +-
src/tools/psim.cc | 23 +-
src/tools/rados/rados.cc | 54 +-
src/tools/rados/rados_sync.cc | 2 +-
src/tools/rados/rados_sync.h | 2 +-
src/upstart/rbdmap.conf | 2 +-
src/vstart.sh | 2 +
367 files changed, 51544 insertions(+), 9399 deletions(-)
diff --git a/README b/README
index 197a7bf..cde4bbc 100644
--- a/README
+++ b/README
@@ -58,7 +58,7 @@ Building packages
You can build packages for Debian or Debian-derived (e.g., Ubuntu)
systems with
-$ sudo apt-get dpkg-dev
+$ sudo apt-get install dpkg-dev
$ dpkg-checkbuilddeps # make sure we have all dependencies
$ dpkg-buildpackage
@@ -115,6 +115,7 @@ To build the source code, you must install the following:
- libtool
- libfcgi
- libfcgi-dev
+- xfslibs-dev
- libfuse-dev
- linux-kernel-headers
- libcrypto++-dev
@@ -135,7 +136,7 @@ To build the source code, you must install the following:
For example:
- $ apt-get install automake autoconf pkg-config gcc g++ make libboost-dev libedit-dev libssl-dev libtool libfcgi libfcgi-dev libfuse-dev linux-kernel-headers libcrypto++-dev libaio-dev libgoogle-perftools-dev libkeyutils-dev uuid-dev libblkid-dev libatomic-ops-dev libboost-program-options-dev libboost-thread-dev libexpat1-dev libleveldb-dev libsnappy-dev libcurl4-gnutls-dev python-argparse python-flask
+ $ apt-get install automake autoconf pkg-config gcc g++ make libboost-dev libedit-dev libssl-dev libtool libfcgi libfcgi-dev xfslibs-dev libfuse-dev linux-kernel-headers libcrypto++-dev libaio-dev libgoogle-perftools-dev libkeyutils-dev uuid-dev libblkid-dev libatomic-ops-dev libboost-program-options-dev libboost-thread-dev libexpat1-dev libleveldb-dev libsnappy-dev libcurl4-gnutls-dev python-argparse python-flask
rpm-based
---------
@@ -157,6 +158,7 @@ These are the rpm packages needed to install in an rpm-based OS:
fcgi-devel
expat-devel
libcurl-devel
+ xfsprogs-devel
fuse-devel
gperftools-devel
libedit-devel
@@ -168,5 +170,5 @@ These are the rpm packages needed to install in an rpm-based OS:
For example:
- $ yum install autoconf automake gcc gcc-c++ make libtool python-argparse python-flask libuuid-devel libblkid-devel keyutils-libs-devel cryptopp-devel nss-devel fcgi-devel expat-devel libcurl-devel fuse-devel gperftools-devel libedit-devel libatomic_ops-devel snappy-devel leveldb-devel libaio-devel boost-devel
+ $ yum install autoconf automake gcc gcc-c++ make libtool python-argparse python-flask libuuid-devel libblkid-devel keyutils-libs-devel cryptopp-devel nss-devel fcgi-devel expat-devel libcurl-devel xfsprogs-devel fuse-devel gperftools-devel libedit-devel libatomic_ops-devel snappy-devel leveldb-devel libaio-devel boost-devel
diff --git a/ceph.spec b/ceph.spec
index 31dc0b8..66eb92a 100644
--- a/ceph.spec
+++ b/ceph.spec
@@ -9,7 +9,7 @@
# common
#################################################################################
Name: ceph
-Version: 0.77
+Version: 0.78
Release: 0%{?dist}
Summary: User space components of the Ceph file system
License: GPL-2.0
@@ -22,6 +22,7 @@ Requires: libcephfs1 = %{version}-%{release}
Requires: python
Requires: python-argparse
Requires: python-ceph
+Requires: python-requests
Requires: xfsprogs
Requires: cryptsetup
Requires: parted
@@ -46,6 +47,7 @@ BuildRequires: libxml2-devel
BuildRequires: libuuid-devel
BuildRequires: libblkid-devel >= 2.17
BuildRequires: leveldb-devel > 1.2
+BuildRequires: xfsprogs-devel
BuildRequires: yasm
%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora}
BuildRequires: snappy-devel
@@ -196,7 +198,6 @@ Requires: librados2 = %{version}-%{release}
Requires: librbd1 = %{version}-%{release}
Requires: libcephfs1 = %{version}-%{release}
Requires: python-flask
-Requires: python-requests
%if 0%{defined suse_version}
%py_requires
%endif
@@ -394,6 +395,7 @@ fi
%{_bindir}/ceph-authtool
%{_bindir}/ceph-syn
%{_bindir}/ceph-post-file
+%{_bindir}/ceph-brag
%{_bindir}/ceph-crush-location
%{_bindir}/ceph-run
%{_bindir}/ceph-mon
@@ -416,7 +418,8 @@ fi
%{_sbindir}/rcceph
/sbin/mkcephfs
/sbin/mount.ceph
-%{_libdir}/ceph
+%dir %{_libdir}/ceph
+%{_libdir}/ceph/ceph_common.sh
%dir %{_libdir}/rados-classes
%{_libdir}/rados-classes/libcls_rbd.so*
%{_libdir}/rados-classes/libcls_hello.so*
@@ -616,6 +619,7 @@ fi
%{_bindir}/ceph_dupstore
%{_bindir}/ceph_kvstorebench
%{_bindir}/ceph_multi_stress_watch
+%{_bindir}/ceph_erasure_code
%{_bindir}/ceph_erasure_code_benchmark
%{_bindir}/ceph_omapbench
%{_bindir}/ceph_psim
@@ -647,10 +651,10 @@ fi
%{_bindir}/ceph_test_cls_version
%{_bindir}/ceph_test_cors
%{_bindir}/ceph_test_filejournal
-%{_bindir}/ceph_test_filestore
+%{_bindir}/ceph_test_objectstore
%{_bindir}/ceph_test_filestore_idempotent
%{_bindir}/ceph_test_filestore_idempotent_sequence
-%{_bindir}/ceph_test_filestore_workloadgen
+%{_bindir}/ceph_test_objectstore_workloadgen
%{_bindir}/ceph_test_get_blkdev_size
%{_bindir}/ceph_test_ioctls
%{_bindir}/ceph_test_keyvaluedb_atomicity
@@ -664,6 +668,8 @@ fi
%{_bindir}/ceph_test_objectcacher_stress
%{_bindir}/ceph_test_rados_api_aio
%{_bindir}/ceph_test_rados_api_cls
+%{_bindir}/ceph_test_rados_api_c_read_operations
+%{_bindir}/ceph_test_rados_api_c_write_operations
%{_bindir}/ceph_test_rados_api_cmd
%{_bindir}/ceph_test_rados_api_io
%{_bindir}/ceph_test_rados_api_list
diff --git a/ceph.spec.in b/ceph.spec.in
index d18aa4f..2f2216f 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -22,6 +22,7 @@ Requires: libcephfs1 = %{version}-%{release}
Requires: python
Requires: python-argparse
Requires: python-ceph
+Requires: python-requests
Requires: xfsprogs
Requires: cryptsetup
Requires: parted
@@ -46,6 +47,7 @@ BuildRequires: libxml2-devel
BuildRequires: libuuid-devel
BuildRequires: libblkid-devel >= 2.17
BuildRequires: leveldb-devel > 1.2
+BuildRequires: xfsprogs-devel
BuildRequires: yasm
%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora}
BuildRequires: snappy-devel
@@ -196,7 +198,6 @@ Requires: librados2 = %{version}-%{release}
Requires: librbd1 = %{version}-%{release}
Requires: libcephfs1 = %{version}-%{release}
Requires: python-flask
-Requires: python-requests
%if 0%{defined suse_version}
%py_requires
%endif
@@ -394,6 +395,7 @@ fi
%{_bindir}/ceph-authtool
%{_bindir}/ceph-syn
%{_bindir}/ceph-post-file
+%{_bindir}/ceph-brag
%{_bindir}/ceph-crush-location
%{_bindir}/ceph-run
%{_bindir}/ceph-mon
@@ -416,7 +418,8 @@ fi
%{_sbindir}/rcceph
/sbin/mkcephfs
/sbin/mount.ceph
-%{_libdir}/ceph
+%dir %{_libdir}/ceph
+%{_libdir}/ceph/ceph_common.sh
%dir %{_libdir}/rados-classes
%{_libdir}/rados-classes/libcls_rbd.so*
%{_libdir}/rados-classes/libcls_hello.so*
@@ -616,6 +619,7 @@ fi
%{_bindir}/ceph_dupstore
%{_bindir}/ceph_kvstorebench
%{_bindir}/ceph_multi_stress_watch
+%{_bindir}/ceph_erasure_code
%{_bindir}/ceph_erasure_code_benchmark
%{_bindir}/ceph_omapbench
%{_bindir}/ceph_psim
@@ -647,10 +651,10 @@ fi
%{_bindir}/ceph_test_cls_version
%{_bindir}/ceph_test_cors
%{_bindir}/ceph_test_filejournal
-%{_bindir}/ceph_test_filestore
+%{_bindir}/ceph_test_objectstore
%{_bindir}/ceph_test_filestore_idempotent
%{_bindir}/ceph_test_filestore_idempotent_sequence
-%{_bindir}/ceph_test_filestore_workloadgen
+%{_bindir}/ceph_test_objectstore_workloadgen
%{_bindir}/ceph_test_get_blkdev_size
%{_bindir}/ceph_test_ioctls
%{_bindir}/ceph_test_keyvaluedb_atomicity
@@ -664,6 +668,8 @@ fi
%{_bindir}/ceph_test_objectcacher_stress
%{_bindir}/ceph_test_rados_api_aio
%{_bindir}/ceph_test_rados_api_cls
+%{_bindir}/ceph_test_rados_api_c_read_operations
+%{_bindir}/ceph_test_rados_api_c_write_operations
%{_bindir}/ceph_test_rados_api_cmd
%{_bindir}/ceph_test_rados_api_io
%{_bindir}/ceph_test_rados_api_list
diff --git a/configure b/configure
index 2f27917..10821fa 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.68 for ceph 0.77.
+# Generated by GNU Autoconf 2.68 for ceph 0.78.
#
# Report bugs to <ceph-devel at vger.kernel.org>.
#
@@ -570,8 +570,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='ceph'
PACKAGE_TARNAME='ceph'
-PACKAGE_VERSION='0.77'
-PACKAGE_STRING='ceph 0.77'
+PACKAGE_VERSION='0.78'
+PACKAGE_STRING='ceph 0.78'
PACKAGE_BUGREPORT='ceph-devel at vger.kernel.org'
PACKAGE_URL=''
@@ -634,6 +634,8 @@ WITH_LIBZFS_FALSE
WITH_LIBZFS_TRUE
LIBZFS_LIBS
LIBZFS_CFLAGS
+WITH_LIBXFS_FALSE
+WITH_LIBXFS_TRUE
WITH_LIBAIO_FALSE
WITH_LIBAIO_TRUE
WITH_REST_BENCH_FALSE
@@ -859,6 +861,7 @@ with_ocf
with_system_libs3
with_rest_bench
with_libaio
+with_libxfs
with_libzfs
'
ac_precious_vars='build_alias
@@ -1430,7 +1433,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures ceph 0.77 to adapt to many kinds of systems.
+\`configure' configures ceph 0.78 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1501,7 +1504,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of ceph 0.77:";;
+ short | recursive ) echo "Configuration of ceph 0.78:";;
esac
cat <<\_ACEOF
@@ -1543,6 +1546,7 @@ Optional Packages:
--with-system-libs3 use system libs3
--with-rest-bench enables rest-bench
--without-libaio disable libaio use by journal
+ --without-libxfs disable libxfs use by FileStore
--with-libzfs build ZFS support
Some influential environment variables:
@@ -1645,7 +1649,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-ceph configure 0.77
+ceph configure 0.78
generated by GNU Autoconf 2.68
Copyright (C) 2010 Free Software Foundation, Inc.
@@ -2337,7 +2341,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by ceph $as_me 0.77, which was
+It was created by ceph $as_me 0.78, which was
generated by GNU Autoconf 2.68. Invocation command line was
$ $0 $@
@@ -4337,7 +4341,7 @@ fi
# Define the identity of the package.
PACKAGE='ceph'
- VERSION='0.77'
+ VERSION='0.78'
cat >>confdefs.h <<_ACEOF
@@ -12315,7 +12319,7 @@ fi
# Define the identity of the package.
PACKAGE='ceph'
- VERSION='0.77'
+ VERSION='0.78'
cat >>confdefs.h <<_ACEOF
@@ -18739,7 +18743,7 @@ else
JAVA_TEST=Test.java
CLASS_TEST=Test.class
cat << \EOF > $JAVA_TEST
-/* #line 18742 "configure" */
+/* #line 18746 "configure" */
public class Test {
}
EOF
@@ -19456,6 +19460,66 @@ else
fi
+# use libxfs?
+
+# Check whether --with-libxfs was given.
+if test "${with_libxfs+set}" = set; then :
+ withval=$with_libxfs;
+else
+ with_libxfs=yes
+fi
+
+if test "x$with_libxfs" != "xno"; then :
+
+ # xfs/xfs.h presence and XFS_XFLAG_EXTSIZE define
+ ac_fn_c_check_header_mongrel "$LINENO" "xfs/xfs.h" "ac_cv_header_xfs_xfs_h" "$ac_includes_default"
+if test "x$ac_cv_header_xfs_xfs_h" = xyes; then :
+
+else
+ as_fn_error $? "xfs/xfs.h not found (--without-libxfs to disable)" "$LINENO" 5
+fi
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for XFS_XFLAG_EXTSIZE in xfs/xfs.h" >&5
+$as_echo_n "checking for XFS_XFLAG_EXTSIZE in xfs/xfs.h... " >&6; }
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+ #include <xfs/xfs.h>
+ #ifdef XFS_XFLAG_EXTSIZE
+ yes_have_xfs_xflag_extsize
+ #endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+ $EGREP "yes_have_xfs_xflag_extsize" >/dev/null 2>&1; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+$as_echo "#define HAVE_LIBXFS 1" >>confdefs.h
+
+
+else
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "XFS_XFLAG_EXTSIZE not found (--without-libxfs to disable)" "$LINENO" 5
+
+fi
+rm -f conftest*
+
+
+fi
+ if test "x$with_libxfs" != "xno"; then
+ WITH_LIBXFS_TRUE=
+ WITH_LIBXFS_FALSE='#'
+else
+ WITH_LIBXFS_TRUE='#'
+ WITH_LIBXFS_FALSE=
+fi
+
+
# use libzfs
# Check whether --with-libzfs was given.
@@ -20219,6 +20283,7 @@ fi
for ac_header in \
arpa/inet.h \
+ arpa/nameser_compat.h \
linux/version.h \
netdb.h \
netinet/in.h \
@@ -20294,47 +20359,38 @@ $as_echo "#define CEPH_HAVE_SPLICE /**/" >>confdefs.h
fi
-
-for ac_header in arpa/nameser_compat.h
-do :
- ac_fn_c_check_header_mongrel "$LINENO" "arpa/nameser_compat.h" "ac_cv_header_arpa_nameser_compat_h" "$ac_includes_default"
-if test "x$ac_cv_header_arpa_nameser_compat_h" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_ARPA_NAMESER_COMPAT_H 1
-_ACEOF
-
-fi
-
-done
-
-
+# F_SETPIPE_SZ in fcntl.h
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for F_SETPIPE_SZ in fcntl.h" >&5
+$as_echo_n "checking for F_SETPIPE_SZ in fcntl.h... " >&6; }
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <fcntl.h>
-F_SETPIPE_SZ
+
+ #define _GNU_SOURCE
+ #include <fcntl.h>
+ #ifdef F_SETPIPE_SZ
+ yes_have_f_setpipe_sz
+ #endif
+
_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+ $EGREP "yes_have_f_setpipe_sz" >/dev/null 2>&1; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
$as_echo "#define CEPH_HAVE_SETPIPE_SZ /**/" >>confdefs.h
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: \"F_SETPIPE_SZ not found, zero-copy may be less efficent\"" >&5
-$as_echo "$as_me: \"F_SETPIPE_SZ not found, zero-copy may be less efficent\"" >&6;}
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+else
-for ac_header in arpa/nameser_compat.h
-do :
- ac_fn_c_check_header_mongrel "$LINENO" "arpa/nameser_compat.h" "ac_cv_header_arpa_nameser_compat_h" "$ac_includes_default"
-if test "x$ac_cv_header_arpa_nameser_compat_h" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_ARPA_NAMESER_COMPAT_H 1
-_ACEOF
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: F_SETPIPE_SZ not found, zero-copy may be less efficent" >&5
+$as_echo "$as_me: F_SETPIPE_SZ not found, zero-copy may be less efficent" >&6;}
fi
+rm -f conftest*
-done
for ac_func in posix_fallocate
do :
@@ -21185,6 +21241,10 @@ if test -z "${WITH_LIBAIO_TRUE}" && test -z "${WITH_LIBAIO_FALSE}"; then
as_fn_error $? "conditional \"WITH_LIBAIO\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
+if test -z "${WITH_LIBXFS_TRUE}" && test -z "${WITH_LIBXFS_FALSE}"; then
+ as_fn_error $? "conditional \"WITH_LIBXFS\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
if test -z "${WITH_LIBZFS_TRUE}" && test -z "${WITH_LIBZFS_FALSE}"; then
as_fn_error $? "conditional \"WITH_LIBZFS\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
@@ -21606,7 +21666,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by ceph $as_me 0.77, which was
+This file was extended by ceph $as_me 0.78, which was
generated by GNU Autoconf 2.68. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -21672,7 +21732,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-ceph config.status 0.77
+ceph config.status 0.78
configured by $0, generated by GNU Autoconf 2.68,
with options \\"\$ac_cs_config\\"
diff --git a/configure.ac b/configure.ac
index 5da65fc..4e5ebf6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -8,7 +8,7 @@ AC_PREREQ(2.59)
# VERSION define is not used by the code. It gets a version string
# from 'git describe'; see src/ceph_ver.[ch]
-AC_INIT([ceph], [0.77], [ceph-devel at vger.kernel.org])
+AC_INIT([ceph], [0.78], [ceph-devel at vger.kernel.org])
# Create release string. Used with VERSION for RPMs.
RPM_RELEASE=0
@@ -529,6 +529,31 @@ AS_IF([test "$with_libaio" = "yes"],
[AC_DEFINE([HAVE_LIBAIO], [1], [Defined if you don't have atomic_ops])])
AM_CONDITIONAL(WITH_LIBAIO, [ test "$with_libaio" = "yes" ])
+# use libxfs?
+AC_ARG_WITH([libxfs],
+ [AS_HELP_STRING([--without-libxfs], [disable libxfs use by FileStore])],
+ [],
+ [with_libxfs=yes])
+AS_IF([test "x$with_libxfs" != "xno"], [
+ # xfs/xfs.h presence and XFS_XFLAG_EXTSIZE define
+ AC_CHECK_HEADER([xfs/xfs.h], [], AC_MSG_ERROR(
+ [xfs/xfs.h not found (--without-libxfs to disable)]))
+ AC_MSG_CHECKING([for XFS_XFLAG_EXTSIZE in xfs/xfs.h])
+ AC_EGREP_CPP([yes_have_xfs_xflag_extsize], [
+ #include <xfs/xfs.h>
+ #ifdef XFS_XFLAG_EXTSIZE
+ yes_have_xfs_xflag_extsize
+ #endif
+ ], [
+ AC_MSG_RESULT([yes])
+ AC_DEFINE([HAVE_LIBXFS], [1], [Define to 1 if you have libxfs])
+ ], [
+ AC_MSG_RESULT([no])
+ AC_MSG_ERROR([XFS_XFLAG_EXTSIZE not found (--without-libxfs to disable)])
+ ])
+])
+AM_CONDITIONAL(WITH_LIBXFS, [test "x$with_libxfs" != "xno"])
+
# use libzfs
AC_ARG_WITH([libzfs],
[AS_HELP_STRING([--with-libzfs], [build ZFS support])],
@@ -596,6 +621,7 @@ AC_CHECK_MEMBER([struct fiemap_extent.fe_logical],
AC_CHECK_HEADERS([ \
arpa/inet.h \
+ arpa/nameser_compat.h \
linux/version.h \
netdb.h \
netinet/in.h \
@@ -639,16 +665,22 @@ AC_CHECK_FUNC([splice],
[AC_DEFINE([CEPH_HAVE_SPLICE], [], [splice(2) is supported])],
[])
+# F_SETPIPE_SZ in fcntl.h
+AC_MSG_CHECKING([for F_SETPIPE_SZ in fcntl.h])
+AC_EGREP_CPP([yes_have_f_setpipe_sz], [
+ #define _GNU_SOURCE
+ #include <fcntl.h>
+ #ifdef F_SETPIPE_SZ
+ yes_have_f_setpipe_sz
+ #endif
+], [
+ AC_MSG_RESULT([yes])
+ AC_DEFINE([CEPH_HAVE_SETPIPE_SZ], [], [F_SETPIPE_SZ is supported])
+], [
+ AC_MSG_RESULT([no])
+ AC_MSG_NOTICE([F_SETPIPE_SZ not found, zero-copy may be less efficent])
+])
-AC_CHECK_HEADERS([arpa/nameser_compat.h])
-
-AC_COMPILE_IFELSE([AC_LANG_SOURCE([[#include <fcntl.h>
-F_SETPIPE_SZ]])],
- [AC_DEFINE([CEPH_HAVE_SETPIPE_SZ], [], [F_SETPIPE_SZ is supported])],
- [AC_MSG_NOTICE(["F_SETPIPE_SZ not found, zero-copy may be less efficent"])])
-
-
-AC_CHECK_HEADERS([arpa/nameser_compat.h])
AC_CHECK_FUNCS([posix_fallocate])
AC_CHECK_HEADERS([sys/prctl.h])
AC_CHECK_FUNCS([prctl])
diff --git a/src/.git_version b/src/.git_version
index 95f3115..2676f1e 100644
--- a/src/.git_version
+++ b/src/.git_version
@@ -1,2 +1,2 @@
-1bca9c5c412b3af722d5250f07fd562a23cf35ff
-v0.77
+f6c746c314d7b87b8419b6e584c94bfe4511dbd4
+v0.78
diff --git a/src/Makefile-env.am b/src/Makefile-env.am
index f637eff..95ac36e 100644
--- a/src/Makefile-env.am
+++ b/src/Makefile-env.am
@@ -149,6 +149,7 @@ LIBRADOS = librados.la
LIBRGW = librgw.la
LIBRBD = librbd.la
LIBCEPHFS = libcephfs.la
+LIBERASURE_CODE = liberasure_code.la
if WITH_LIBAIO
LIBOS += -laio
diff --git a/src/Makefile.am b/src/Makefile.am
index 24f8fa9..599cb57 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -8,11 +8,13 @@ DIST_SUBDIRS += gtest ocf libs3 java
include arch/Makefile.am
include auth/Makefile.am
+include brag/Makefile.am
include crush/Makefile.am
include mon/Makefile.am
include mds/Makefile.am
include os/Makefile.am
include osd/Makefile.am
+include erasure-code/Makefile.am
include osdc/Makefile.am
include client/Makefile.am
include global/Makefile.am
@@ -40,9 +42,6 @@ bin_PROGRAMS += ceph-mon
ceph_osd_SOURCES = ceph_osd.cc
ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON)
-if LINUX
-ceph_osd_LDADD += -ldl
-endif # LINUX
bin_PROGRAMS += ceph-osd
ceph_mds_SOURCES = ceph_mds.cc
diff --git a/src/Makefile.in b/src/Makefile.in
index 32d31ac..d1fb014 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -44,36 +44,41 @@ DIST_COMMON = README $(am__noinst_HEADERS_DIST) $(dist_bin_SCRIPTS) \
$(python_PYTHON) $(srcdir)/Makefile-env.am \
$(srcdir)/Makefile.am $(srcdir)/Makefile.in \
$(srcdir)/acconfig.h.in $(srcdir)/arch/Makefile.am \
- $(srcdir)/auth/Makefile.am $(srcdir)/client/Makefile.am \
- $(srcdir)/cls/Makefile.am $(srcdir)/common/Makefile.am \
- $(srcdir)/crush/Makefile.am $(srcdir)/global/Makefile.am \
- $(srcdir)/include/Makefile.am \
+ $(srcdir)/auth/Makefile.am $(srcdir)/brag/Makefile.am \
+ $(srcdir)/client/Makefile.am $(srcdir)/cls/Makefile.am \
+ $(srcdir)/common/Makefile.am $(srcdir)/crush/Makefile.am \
+ $(srcdir)/erasure-code/Makefile.am \
+ $(srcdir)/erasure-code/jerasure/Makefile.am \
+ $(srcdir)/global/Makefile.am $(srcdir)/include/Makefile.am \
$(srcdir)/json_spirit/Makefile.am \
$(srcdir)/key_value_store/Makefile.am \
$(srcdir)/librados/Makefile.am $(srcdir)/librbd/Makefile.am \
$(srcdir)/log/Makefile.am $(srcdir)/mds/Makefile.am \
$(srcdir)/messages/Makefile.am $(srcdir)/mon/Makefile.am \
$(srcdir)/msg/Makefile.am $(srcdir)/os/Makefile.am \
- $(srcdir)/osd/ErasureCodePluginJerasure/Makefile.am \
$(srcdir)/osd/Makefile.am $(srcdir)/osdc/Makefile.am \
$(srcdir)/perfglue/Makefile.am $(srcdir)/rgw/Makefile.am \
- $(srcdir)/test/Makefile.am $(srcdir)/tools/Makefile.am TODO
-bin_PROGRAMS = $(am__EXEEXT_10) $(am__EXEEXT_11) \
- ceph-dencoder$(EXEEXT) ceph_filestore_tool$(EXEEXT) \
- ceph_filestore_dump$(EXEEXT) monmaptool$(EXEEXT) \
- crushtool$(EXEEXT) osdmaptool$(EXEEXT) rados$(EXEEXT) \
- $(am__EXEEXT_12) ceph-conf$(EXEEXT) ceph-authtool$(EXEEXT) \
- ceph_mon_store_converter$(EXEEXT) ceph-mon$(EXEEXT) \
- ceph-osd$(EXEEXT) ceph-mds$(EXEEXT) cephfs$(EXEEXT) \
- librados-config$(EXEEXT) ceph-syn$(EXEEXT) $(am__EXEEXT_13) \
- $(am__EXEEXT_14)
+ $(srcdir)/test/Makefile.am \
+ $(srcdir)/test/erasure-code/Makefile.am \
+ $(srcdir)/tools/Makefile.am TODO
+bin_PROGRAMS = $(am__EXEEXT_9) $(am__EXEEXT_10) ceph-dencoder$(EXEEXT) \
+ ceph_filestore_tool$(EXEEXT) ceph_filestore_dump$(EXEEXT) \
+ monmaptool$(EXEEXT) crushtool$(EXEEXT) osdmaptool$(EXEEXT) \
+ rados$(EXEEXT) $(am__EXEEXT_11) ceph-conf$(EXEEXT) \
+ ceph-authtool$(EXEEXT) ceph_mon_store_converter$(EXEEXT) \
+ ceph-mon$(EXEEXT) ceph-osd$(EXEEXT) ceph-mds$(EXEEXT) \
+ cephfs$(EXEEXT) librados-config$(EXEEXT) ceph-syn$(EXEEXT) \
+ $(am__EXEEXT_12) $(am__EXEEXT_13)
noinst_PROGRAMS = get_command_descriptions$(EXEEXT)
sbin_PROGRAMS =
-su_sbin_PROGRAMS = $(am__EXEEXT_15)
-check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
- unittest_bloom_filter$(EXEEXT) unittest_str_map$(EXEEXT) \
- unittest_crushwrapper$(EXEEXT) \
- unittest_sharedptr_registry$(EXEEXT) \
+su_sbin_PROGRAMS = $(am__EXEEXT_14)
+check_PROGRAMS = unittest_erasure_code_plugin$(EXEEXT) \
+ unittest_erasure_code_jerasure$(EXEEXT) \
+ unittest_erasure_code_plugin_jerasure$(EXEEXT) \
+ unittest_erasure_code_example$(EXEEXT) \
+ unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
+ unittest_bloom_filter$(EXEEXT) unittest_histogram$(EXEEXT) \
+ unittest_str_map$(EXEEXT) unittest_sharedptr_registry$(EXEEXT) \
unittest_sloppy_crc_map$(EXEEXT) unittest_util$(EXEEXT) \
unittest_crush_indep$(EXEEXT) unittest_osdmap$(EXEEXT) \
unittest_workqueue$(EXEEXT) unittest_striper$(EXEEXT) \
@@ -81,12 +86,8 @@ check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
unittest_str_list$(EXEEXT) unittest_log$(EXEEXT) \
unittest_throttle$(EXEEXT) unittest_crush_wrapper$(EXEEXT) \
unittest_base64$(EXEEXT) unittest_ceph_argparse$(EXEEXT) \
- unittest_ceph_compatset$(EXEEXT) \
- unittest_erasure_code_plugin$(EXEEXT) \
- unittest_erasure_code_jerasure$(EXEEXT) \
- unittest_erasure_code_plugin_jerasure$(EXEEXT) \
- unittest_erasure_code_example$(EXEEXT) \
- unittest_osd_types$(EXEEXT) unittest_pglog$(EXEEXT) \
+ unittest_ceph_compatset$(EXEEXT) unittest_osd_types$(EXEEXT) \
+ unittest_pglog$(EXEEXT) unittest_ecbackend$(EXEEXT) \
unittest_hitset$(EXEEXT) unittest_gather$(EXEEXT) \
unittest_run_cmd$(EXEEXT) unittest_signals$(EXEEXT) \
unittest_simple_spin$(EXEEXT) unittest_librados$(EXEEXT) \
@@ -97,12 +98,14 @@ check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
unittest_mime$(EXEEXT) unittest_escape$(EXEEXT) \
unittest_chain_xattr$(EXEEXT) unittest_flatindex$(EXEEXT) \
unittest_strtol$(EXEEXT) unittest_confutils$(EXEEXT) \
- unittest_config$(EXEEXT) unittest_heartbeatmap$(EXEEXT) \
- unittest_formatter$(EXEEXT) unittest_libcephfs_config$(EXEEXT) \
- unittest_lfnindex$(EXEEXT) unittest_librados_config$(EXEEXT) \
+ unittest_config$(EXEEXT) unittest_context$(EXEEXT) \
+ unittest_heartbeatmap$(EXEEXT) unittest_formatter$(EXEEXT) \
+ unittest_libcephfs_config$(EXEEXT) unittest_lfnindex$(EXEEXT) \
+ unittest_librados_config$(EXEEXT) \
unittest_daemon_config$(EXEEXT) unittest_osd_osdcap$(EXEEXT) \
- unittest_mon_moncap$(EXEEXT) unittest_ipaddr$(EXEEXT) \
- unittest_texttable$(EXEEXT) unittest_on_exit$(EXEEXT)
+ unittest_mon_moncap$(EXEEXT) unittest_mon_pgmap$(EXEEXT) \
+ unittest_ipaddr$(EXEEXT) unittest_texttable$(EXEEXT) \
+ unittest_on_exit$(EXEEXT)
# when doing a debug build, make sure to make the targets
@WITH_DEBUG_TRUE at am__append_1 = $(bin_DEBUGPROGRAMS)
@@ -119,27 +122,29 @@ check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
@WITH_TCMALLOC_TRUE at am__append_12 = -ltcmalloc
@ENABLE_COVERAGE_TRUE at am__append_13 = -lgcov
@LINUX_TRUE at am__append_14 = os/BtrfsFileStoreBackend.cc
- at WITH_LIBZFS_TRUE@am__append_15 = os/ZFSFileStoreBackend.cc
- at WITH_LIBZFS_TRUE@am__append_16 = libos_zfs.a
- at WITH_LIBZFS_TRUE@am__append_17 = os/ZFS.h
- at LINUX_TRUE@am__append_18 = -export-symbols-regex '.*__erasure_code_.*'
- at WITH_FUSE_TRUE@am__append_19 = libclient_fuse.la
- at WITH_FUSE_TRUE@am__append_20 = client/fuse_ll.h
- at WITH_TCMALLOC_TRUE@am__append_21 = perfglue/heap_profiler.cc
- at WITH_TCMALLOC_TRUE@am__append_22 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
- at WITH_TCMALLOC_TRUE@am__append_23 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
- at WITH_TCMALLOC_FALSE@am__append_24 = perfglue/disabled_heap_profiler.cc
- at WITH_PROFILER_TRUE@am__append_25 = perfglue/cpu_profiler.cc
- at WITH_PROFILER_FALSE@am__append_26 = perfglue/disabled_stubs.cc
- at LINUX_TRUE@am__append_27 = \
+ at WITH_LIBXFS_TRUE@am__append_15 = os/XfsFileStoreBackend.cc
+ at WITH_LIBZFS_TRUE@am__append_16 = os/ZFSFileStoreBackend.cc
+ at WITH_LIBZFS_TRUE@am__append_17 = libos_zfs.a
+ at WITH_LIBZFS_TRUE@am__append_18 = os/ZFS.h
+ at LINUX_TRUE@am__append_19 = -export-symbols-regex '.*__erasure_code_.*'
+ at LINUX_TRUE@am__append_20 = -ldl
+ at WITH_FUSE_TRUE@am__append_21 = libclient_fuse.la
+ at WITH_FUSE_TRUE@am__append_22 = client/fuse_ll.h
+ at WITH_TCMALLOC_TRUE@am__append_23 = perfglue/heap_profiler.cc
+ at WITH_TCMALLOC_TRUE@am__append_24 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
+ at WITH_TCMALLOC_TRUE@am__append_25 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
+ at WITH_TCMALLOC_FALSE@am__append_26 = perfglue/disabled_heap_profiler.cc
+ at WITH_PROFILER_TRUE@am__append_27 = perfglue/cpu_profiler.cc
+ at WITH_PROFILER_FALSE@am__append_28 = perfglue/disabled_stubs.cc
+ at LINUX_TRUE@am__append_29 = \
@LINUX_TRUE@ common/secret.c
- at WITH_GOOD_YASM_ELF64_TRUE@am__append_28 = common/crc32c_intel_fast_asm.S common/crc32c_intel_fast_zero_asm.S
- at LINUX_TRUE@am__append_29 = -lrt
- at LINUX_TRUE@am__append_30 = -export-symbols-regex '^rados_.*'
- at LINUX_TRUE@am__append_31 = -export-symbols-regex '^rbd_.*'
- at WITH_RADOSGW_TRUE@am__append_32 = librgw.la
- at WITH_RADOSGW_TRUE@am__append_33 = \
+ at WITH_GOOD_YASM_ELF64_TRUE@am__append_30 = common/crc32c_intel_fast_asm.S common/crc32c_intel_fast_zero_asm.S
+ at LINUX_TRUE@am__append_31 = -lrt
+ at LINUX_TRUE@am__append_32 = -export-symbols-regex '^rados_.*'
+ at LINUX_TRUE@am__append_33 = -export-symbols-regex '^rbd_.*'
+ at WITH_RADOSGW_TRUE@am__append_34 = librgw.la
+ at WITH_RADOSGW_TRUE@am__append_35 = \
@WITH_RADOSGW_TRUE@ $(LIBRADOS) \
@WITH_RADOSGW_TRUE@ libcls_rgw_client.la \
@WITH_RADOSGW_TRUE@ libcls_log_client.a \
@@ -155,53 +160,55 @@ check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
@WITH_RADOSGW_TRUE@ -lfcgi \
@WITH_RADOSGW_TRUE@ -ldl
- at WITH_RADOSGW_TRUE@am__append_34 = radosgw radosgw-admin
- at WITH_RADOSGW_TRUE@am__append_35 = ceph_rgw_multiparser \
+ at WITH_RADOSGW_TRUE@am__append_36 = radosgw radosgw-admin
+ at WITH_RADOSGW_TRUE@am__append_37 = ceph_rgw_multiparser \
@WITH_RADOSGW_TRUE@ ceph_rgw_jsonparser
# inject rgw stuff in the decoder testcase
- at WITH_RADOSGW_TRUE@am__append_36 = \
+ at WITH_RADOSGW_TRUE@am__append_38 = \
@WITH_RADOSGW_TRUE@ rgw/rgw_dencoder.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_acl.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_common.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_env.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.cc
- at LINUX_TRUE@am__append_37 = libcls_kvs.la
- at COMPILER_HAS_VTA_TRUE@am__append_38 = -fno-var-tracking-assignments
- at COMPILER_HAS_VTA_TRUE@am__append_39 = -fno-var-tracking-assignments
- at WITH_BUILD_TESTS_TRUE@am__append_40 = test_build_libcommon \
+ at LINUX_TRUE@am__append_39 = libcls_kvs.la
+ at LINUX_TRUE@am__append_40 = -ldl
+ at LINUX_TRUE@am__append_41 = -ldl
+ at LINUX_TRUE@am__append_42 = -ldl
+ at LINUX_TRUE@am__append_43 = -ldl
+ at LINUX_TRUE@am__append_44 = -ldl
+ at WITH_RADOSGW_TRUE@am__append_45 = $(LIBRGW) $(LIBRGW_DEPS)
+ at COMPILER_HAS_VTA_TRUE@am__append_46 = -fno-var-tracking-assignments
+ at COMPILER_HAS_VTA_TRUE@am__append_47 = -fno-var-tracking-assignments
+ at WITH_BUILD_TESTS_TRUE@am__append_48 = test_build_libcommon \
@WITH_BUILD_TESTS_TRUE@ test_build_librados test_build_librgw \
@WITH_BUILD_TESTS_TRUE@ test_build_libcephfs
- at LINUX_TRUE@am__append_41 = ceph_kvstorebench
- at LINUX_TRUE@am__append_42 = -ldl
- at LINUX_TRUE@am__append_43 = libsystest.la
- at LINUX_TRUE@am__append_44 = ceph_test_rados_list_parallel \
+ at LINUX_TRUE@am__append_49 = ceph_kvstorebench \
+ at LINUX_TRUE@ ceph_test_rados_list_parallel \
@LINUX_TRUE@ ceph_test_rados_open_pools_parallel \
@LINUX_TRUE@ ceph_test_rados_delete_pools_parallel \
@LINUX_TRUE@ ceph_test_rados_watch_notify
- at LINUX_TRUE@am__append_45 = -ldl
- at LINUX_TRUE@am__append_46 = -ldl
- at LINUX_TRUE@am__append_47 = -ldl
- at LINUX_TRUE@am__append_48 = -ldl
- at WITH_RADOSGW_TRUE@am__append_49 = ceph_test_cors \
+ at LINUX_TRUE@am__append_50 = libsystest.la
+ at LINUX_TRUE@am__append_51 = -ldl
+ at WITH_RADOSGW_TRUE@am__append_52 = ceph_test_cors \
+ at WITH_RADOSGW_TRUE@ ceph_test_rgw_manifest \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_meta \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_log \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_opstate
- at LINUX_TRUE@am__append_50 = ceph_test_librbd_fsx
- at WITH_RADOSGW_TRUE@am__append_51 = ceph_test_cls_rgw
- at LINUX_TRUE@am__append_52 = ceph_test_filestore
- at LINUX_TRUE@am__append_53 = -ldl
- at LINUX_TRUE@am__append_54 = -ldl
- at WITH_REST_BENCH_TRUE@am__append_55 = rest-bench
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_TRUE at am__append_56 = -ls3
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_57 = libs3/build/lib/libs3.a -lcurl -lxml2
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_58 = libs3
- at LINUX_TRUE@am__append_59 = -ldl
- at LINUX_TRUE@am__append_60 = mount.ceph
- at LINUX_TRUE@am__append_61 = rbd
- at WITH_FUSE_TRUE@am__append_62 = ceph-fuse rbd-fuse
- at ENABLE_CEPHFS_JAVA_TRUE@am__append_63 = libcephfs_jni.la
+ at LINUX_TRUE@am__append_53 = ceph_test_librbd_fsx
+ at WITH_RADOSGW_TRUE@am__append_54 = ceph_test_cls_rgw
+ at LINUX_TRUE@am__append_55 = ceph_test_objectstore
+ at LINUX_TRUE@am__append_56 = -ldl
+ at LINUX_TRUE@am__append_57 = -ldl
+ at WITH_REST_BENCH_TRUE@am__append_58 = rest-bench
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_TRUE at am__append_59 = -ls3
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_60 = libs3/build/lib/libs3.a -lcurl -lxml2
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_61 = libs3
+ at LINUX_TRUE@am__append_62 = mount.ceph
+ at LINUX_TRUE@am__append_63 = rbd
+ at WITH_FUSE_TRUE@am__append_64 = ceph-fuse rbd-fuse
+ at ENABLE_CEPHFS_JAVA_TRUE@am__append_65 = libcephfs_jni.la
subdir = src
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
@@ -492,10 +499,10 @@ am__libcommon_la_SOURCES_DIST = ceph_ver.c common/DecayCounter.cc \
common/errno.cc common/RefCountedObj.cc common/blkdev.cc \
common/common_init.cc common/pipe.c common/ceph_argparse.cc \
common/ceph_context.cc common/buffer.cc \
- common/code_environment.cc common/dout.cc common/signal.cc \
- common/simple_spin.cc common/Thread.cc common/Formatter.cc \
- common/HeartbeatMap.cc common/config.cc common/utf8.c \
- common/mime.c common/strtol.cc common/page.cc \
+ common/code_environment.cc common/dout.cc common/histogram.cc \
+ common/signal.cc common/simple_spin.cc common/Thread.cc \
+ common/Formatter.cc common/HeartbeatMap.cc common/config.cc \
+ common/utf8.c common/mime.c common/strtol.cc common/page.cc \
common/lockdep.cc common/version.cc common/hex.cc \
common/entity_name.cc common/ceph_crypto.cc \
common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \
@@ -504,7 +511,7 @@ am__libcommon_la_SOURCES_DIST = ceph_ver.c common/DecayCounter.cc \
common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \
common/bloom_filter.cc common/linux_version.c common/secret.c \
mon/MonCap.cc mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc \
- osd/osd_types.cc osd/HitSet.cc mds/MDSMap.cc \
+ osd/osd_types.cc osd/ECMsgTypes.cc osd/HitSet.cc mds/MDSMap.cc \
mds/inode_backtrace.cc mds/mdstypes.cc
@LINUX_TRUE at am__objects_1 = common/secret.lo
am_libcommon_la_OBJECTS = ceph_ver.lo common/DecayCounter.lo \
@@ -522,10 +529,10 @@ am_libcommon_la_OBJECTS = ceph_ver.lo common/DecayCounter.lo \
common/errno.lo common/RefCountedObj.lo common/blkdev.lo \
common/common_init.lo common/pipe.lo common/ceph_argparse.lo \
common/ceph_context.lo common/buffer.lo \
- common/code_environment.lo common/dout.lo common/signal.lo \
- common/simple_spin.lo common/Thread.lo common/Formatter.lo \
- common/HeartbeatMap.lo common/config.lo common/utf8.lo \
- common/mime.lo common/strtol.lo common/page.lo \
+ common/code_environment.lo common/dout.lo common/histogram.lo \
+ common/signal.lo common/simple_spin.lo common/Thread.lo \
+ common/Formatter.lo common/HeartbeatMap.lo common/config.lo \
+ common/utf8.lo common/mime.lo common/strtol.lo common/page.lo \
common/lockdep.lo common/version.lo common/hex.lo \
common/entity_name.lo common/ceph_crypto.lo \
common/ceph_crypto_cms.lo common/ceph_json.lo common/ipaddr.lo \
@@ -534,8 +541,8 @@ am_libcommon_la_OBJECTS = ceph_ver.lo common/DecayCounter.lo \
common/ceph_frag.lo common/addr_parsing.lo common/hobject.lo \
common/bloom_filter.lo common/linux_version.lo \
$(am__objects_1) mon/MonCap.lo mon/MonClient.lo mon/MonMap.lo \
- osd/OSDMap.lo osd/osd_types.lo osd/HitSet.lo mds/MDSMap.lo \
- mds/inode_backtrace.lo mds/mdstypes.lo
+ osd/OSDMap.lo osd/osd_types.lo osd/ECMsgTypes.lo osd/HitSet.lo \
+ mds/MDSMap.lo mds/inode_backtrace.lo mds/mdstypes.lo
libcommon_la_OBJECTS = $(am_libcommon_la_OBJECTS)
libcommon_crc_la_LIBADD =
am__libcommon_crc_la_SOURCES_DIST = common/sctp_crc32.c \
@@ -558,10 +565,9 @@ am_libcrush_la_OBJECTS = crush/builder.lo crush/mapper.lo \
crush/crush.lo crush/hash.lo crush/CrushWrapper.lo \
crush/CrushCompiler.lo crush/CrushTester.lo
libcrush_la_OBJECTS = $(am_libcrush_la_OBJECTS)
-libec_example_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
+libec_example_la_DEPENDENCIES = $(LIBCRUSH) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
-am_libec_example_la_OBJECTS = \
- test/osd/libec_example_la-ErasureCodePluginExample.lo
+am_libec_example_la_OBJECTS = test/erasure-code/libec_example_la-ErasureCodePluginExample.lo
libec_example_la_OBJECTS = $(am_libec_example_la_OBJECTS)
libec_example_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -569,7 +575,7 @@ libec_example_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(libec_example_la_LDFLAGS) $(LDFLAGS) -o $@
libec_fail_to_initialize_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
-am_libec_fail_to_initialize_la_OBJECTS = test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo
+am_libec_fail_to_initialize_la_OBJECTS = test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo
libec_fail_to_initialize_la_OBJECTS = \
$(am_libec_fail_to_initialize_la_OBJECTS)
libec_fail_to_initialize_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
@@ -578,7 +584,7 @@ libec_fail_to_initialize_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(libec_fail_to_initialize_la_LDFLAGS) $(LDFLAGS) -o $@
libec_fail_to_register_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
-am_libec_fail_to_register_la_OBJECTS = test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo
+am_libec_fail_to_register_la_OBJECTS = test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo
libec_fail_to_register_la_OBJECTS = \
$(am_libec_fail_to_register_la_OBJECTS)
libec_fail_to_register_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
@@ -588,21 +594,21 @@ libec_fail_to_register_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
libec_hangs_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
am_libec_hangs_la_OBJECTS = \
- test/osd/libec_hangs_la-ErasureCodePluginHangs.lo
+ test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo
libec_hangs_la_OBJECTS = $(am_libec_hangs_la_OBJECTS)
libec_hangs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) \
$(libec_hangs_la_LDFLAGS) $(LDFLAGS) -o $@
-libec_jerasure_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
+libec_jerasure_la_DEPENDENCIES = $(LIBCRUSH) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
-am_libec_jerasure_la_OBJECTS = osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo \
- osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo \
- osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo \
- osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo \
- osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo \
- osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo \
- osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo
+am_libec_jerasure_la_OBJECTS = erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo \
+ erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo \
+ erasure-code/jerasure/libec_jerasure_la-cauchy.lo \
+ erasure-code/jerasure/libec_jerasure_la-galois.lo \
+ erasure-code/jerasure/libec_jerasure_la-jerasure.lo \
+ erasure-code/jerasure/libec_jerasure_la-liberation.lo \
+ erasure-code/jerasure/libec_jerasure_la-reed_sol.lo
libec_jerasure_la_OBJECTS = $(am_libec_jerasure_la_OBJECTS)
libec_jerasure_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -610,13 +616,20 @@ libec_jerasure_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(libec_jerasure_la_LDFLAGS) $(LDFLAGS) -o $@
libec_missing_entry_point_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
-am_libec_missing_entry_point_la_OBJECTS = test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo
+am_libec_missing_entry_point_la_OBJECTS = test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo
libec_missing_entry_point_la_OBJECTS = \
$(am_libec_missing_entry_point_la_OBJECTS)
libec_missing_entry_point_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) \
$(libec_missing_entry_point_la_LDFLAGS) $(LDFLAGS) -o $@
+ at WITH_LIBZFS_TRUE@am__DEPENDENCIES_4 = libos_zfs.a
+am__DEPENDENCIES_5 = libos.la $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_4)
+liberasure_code_la_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_5) \
+ $(am__DEPENDENCIES_1)
+am_liberasure_code_la_OBJECTS = erasure-code/ErasureCodePlugin.lo
+liberasure_code_la_OBJECTS = $(am_liberasure_code_la_OBJECTS)
libglobal_la_DEPENDENCIES = $(LIBCOMMON)
am_libglobal_la_OBJECTS = global/global_context.lo \
global/global_init.lo global/pidfile.lo \
@@ -639,10 +652,8 @@ am_libmds_la_OBJECTS = mds/Anchor.lo mds/Capability.lo mds/Dumper.lo \
mds/AnchorServer.lo mds/AnchorClient.lo mds/SnapRealm.lo \
mds/SnapServer.lo mds/snap.lo mds/SessionMap.lo mds/MDLog.lo
libmds_la_OBJECTS = $(am_libmds_la_OBJECTS)
- at WITH_LIBZFS_TRUE@am__DEPENDENCIES_4 = libos_zfs.a
-am__DEPENDENCIES_5 = libos.la $(am__DEPENDENCIES_1) \
- $(am__DEPENDENCIES_4)
-libmon_la_DEPENDENCIES = $(LIBAUTH) $(LIBCOMMON) $(am__DEPENDENCIES_5)
+libmon_la_DEPENDENCIES = $(LIBAUTH) $(LIBCOMMON) $(am__DEPENDENCIES_5) \
+ $(LIBERASURE_CODE)
am_libmon_la_OBJECTS = mon/Monitor.lo mon/Paxos.lo mon/PaxosService.lo \
mon/OSDMonitor.lo mon/MDSMonitor.lo mon/MonmapMonitor.lo \
mon/PGMonitor.lo mon/PGMap.lo mon/LogMonitor.lo \
@@ -663,23 +674,27 @@ am__libos_la_SOURCES_DIST = os/chain_xattr.cc os/DBObjectMap.cc \
os/LevelDBStore.cc os/LFNIndex.cc os/MemStore.cc \
os/KeyValueStore.cc os/ObjectStore.cc os/WBThrottle.cc \
common/TrackedOp.cc os/BtrfsFileStoreBackend.cc \
- os/ZFSFileStoreBackend.cc
+ os/XfsFileStoreBackend.cc os/ZFSFileStoreBackend.cc
@LINUX_TRUE at am__objects_3 = os/BtrfsFileStoreBackend.lo
- at WITH_LIBZFS_TRUE@am__objects_4 = os/ZFSFileStoreBackend.lo
+ at WITH_LIBXFS_TRUE@am__objects_4 = os/XfsFileStoreBackend.lo
+ at WITH_LIBZFS_TRUE@am__objects_5 = os/ZFSFileStoreBackend.lo
am_libos_la_OBJECTS = os/chain_xattr.lo os/DBObjectMap.lo \
os/GenericObjectMap.lo os/FileJournal.lo os/FileStore.lo \
os/FlatIndex.lo os/GenericFileStoreBackend.lo os/HashIndex.lo \
os/IndexManager.lo os/JournalingObjectStore.lo \
os/LevelDBStore.lo os/LFNIndex.lo os/MemStore.lo \
os/KeyValueStore.lo os/ObjectStore.lo os/WBThrottle.lo \
- common/TrackedOp.lo $(am__objects_3) $(am__objects_4)
+ common/TrackedOp.lo $(am__objects_3) $(am__objects_4) \
+ $(am__objects_5)
libos_la_OBJECTS = $(am_libos_la_OBJECTS)
-libosd_la_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_5)
-am_libosd_la_OBJECTS = osd/ErasureCodePlugin.lo osd/PG.lo osd/PGLog.lo \
- osd/ReplicatedPG.lo osd/ReplicatedBackend.lo osd/PGBackend.lo \
- osd/Ager.lo osd/HitSet.lo osd/OSD.lo osd/OSDCap.lo \
- osd/Watch.lo osd/ClassHandler.lo osd/OpRequest.lo \
- common/TrackedOp.lo osd/SnapMapper.lo osd/osd_types.lo \
+libosd_la_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_5) \
+ $(LIBERASURE_CODE)
+am_libosd_la_OBJECTS = osd/PG.lo osd/PGLog.lo osd/ReplicatedPG.lo \
+ osd/ReplicatedBackend.lo osd/ECBackend.lo osd/ECMsgTypes.lo \
+ osd/ECTransaction.lo osd/PGBackend.lo osd/Ager.lo \
+ osd/HitSet.lo osd/OSD.lo osd/OSDCap.lo osd/Watch.lo \
+ osd/ClassHandler.lo osd/OpRequest.lo common/TrackedOp.lo \
+ osd/SnapMapper.lo osd/osd_types.lo osd/ECUtil.lo \
objclass/class_api.lo
libosd_la_OBJECTS = $(am_libosd_la_OBJECTS)
libosdc_la_LIBADD =
@@ -690,13 +705,13 @@ libperfglue_la_DEPENDENCIES =
am__libperfglue_la_SOURCES_DIST = perfglue/heap_profiler.cc \
perfglue/disabled_heap_profiler.cc perfglue/cpu_profiler.cc \
perfglue/disabled_stubs.cc
- at WITH_TCMALLOC_TRUE@am__objects_5 = perfglue/heap_profiler.lo
- at WITH_TCMALLOC_FALSE@am__objects_6 = \
+ at WITH_TCMALLOC_TRUE@am__objects_6 = perfglue/heap_profiler.lo
+ at WITH_TCMALLOC_FALSE@am__objects_7 = \
@WITH_TCMALLOC_FALSE@ perfglue/disabled_heap_profiler.lo
- at WITH_PROFILER_TRUE@am__objects_7 = perfglue/cpu_profiler.lo
- at WITH_PROFILER_FALSE@am__objects_8 = perfglue/disabled_stubs.lo
-am_libperfglue_la_OBJECTS = $(am__objects_5) $(am__objects_6) \
- $(am__objects_7) $(am__objects_8)
+ at WITH_PROFILER_TRUE@am__objects_8 = perfglue/cpu_profiler.lo
+ at WITH_PROFILER_FALSE@am__objects_9 = perfglue/disabled_stubs.lo
+am_libperfglue_la_OBJECTS = $(am__objects_6) $(am__objects_7) \
+ $(am__objects_8) $(am__objects_9)
libperfglue_la_OBJECTS = $(am_libperfglue_la_OBJECTS)
librados_la_DEPENDENCIES = $(LIBRADOS_DEPS) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2)
@@ -708,6 +723,14 @@ librados_la_OBJECTS = $(am_librados_la_OBJECTS)
librados_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(librados_la_CXXFLAGS) \
$(CXXFLAGS) $(librados_la_LDFLAGS) $(LDFLAGS) -o $@
+libradostest_la_LIBADD =
+am_libradostest_la_OBJECTS = test/librados/libradostest_la-test.lo \
+ test/librados/libradostest_la-TestCase.lo
+libradostest_la_OBJECTS = $(am_libradostest_la_OBJECTS)
+libradostest_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(libradostest_la_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
librbd_la_DEPENDENCIES = $(LIBRADOS) $(LIBOSDC) libcls_rbd_client.la \
libcls_lock_client.la $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
@@ -787,41 +810,42 @@ libsystest_la_OBJECTS = $(am_libsystest_la_OBJECTS)
@WITH_BUILD_TESTS_TRUE@ test_build_librados$(EXEEXT) \
@WITH_BUILD_TESTS_TRUE@ test_build_librgw$(EXEEXT) \
@WITH_BUILD_TESTS_TRUE@ test_build_libcephfs$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_3 = ceph_kvstorebench$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_4 = ceph_test_rados_list_parallel$(EXEEXT) \
+ at LINUX_TRUE@am__EXEEXT_3 = ceph_kvstorebench$(EXEEXT) \
+ at LINUX_TRUE@ ceph_test_rados_list_parallel$(EXEEXT) \
@LINUX_TRUE@ ceph_test_rados_open_pools_parallel$(EXEEXT) \
@LINUX_TRUE@ ceph_test_rados_delete_pools_parallel$(EXEEXT) \
@LINUX_TRUE@ ceph_test_rados_watch_notify$(EXEEXT)
- at WITH_RADOSGW_TRUE@am__EXEEXT_5 = ceph_test_cors$(EXEEXT) \
+ at WITH_RADOSGW_TRUE@am__EXEEXT_4 = ceph_test_cors$(EXEEXT) \
+ at WITH_RADOSGW_TRUE@ ceph_test_rgw_manifest$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_meta$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_log$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_opstate$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_6 = ceph_test_librbd_fsx$(EXEEXT)
- at WITH_RADOSGW_TRUE@am__EXEEXT_7 = ceph_test_cls_rgw$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_8 = ceph_test_filestore$(EXEEXT)
-am__EXEEXT_9 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
- ceph_test_timers$(EXEEXT) ceph_test_signal_handlers$(EXEEXT) \
- ceph_test_rados$(EXEEXT) ceph_test_mutate$(EXEEXT) \
- ceph_test_rewrite_latency$(EXEEXT) ceph_test_msgr$(EXEEXT) \
- ceph_streamtest$(EXEEXT) ceph_test_trans$(EXEEXT) \
- ceph_test_crypto$(EXEEXT) ceph_test_keys$(EXEEXT) \
- $(am__EXEEXT_2) ceph_smalliobench$(EXEEXT) \
- ceph_smalliobenchfs$(EXEEXT) ceph_smalliobenchdumb$(EXEEXT) \
- ceph_smalliobenchrbd$(EXEEXT) ceph_tpbench$(EXEEXT) \
- ceph_omapbench$(EXEEXT) $(am__EXEEXT_3) \
- ceph_multi_stress_watch$(EXEEXT) \
- ceph_erasure_code_benchmark$(EXEEXT) $(am__EXEEXT_4) \
- ceph_bench_log$(EXEEXT) $(am__EXEEXT_5) \
- ceph_test_librbd$(EXEEXT) $(am__EXEEXT_6) \
- ceph_test_cls_rbd$(EXEEXT) ceph_test_cls_refcount$(EXEEXT) \
- ceph_test_cls_version$(EXEEXT) ceph_test_cls_log$(EXEEXT) \
- ceph_test_cls_statelog$(EXEEXT) \
+ at LINUX_TRUE@am__EXEEXT_5 = ceph_test_librbd_fsx$(EXEEXT)
+ at WITH_RADOSGW_TRUE@am__EXEEXT_6 = ceph_test_cls_rgw$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_7 = ceph_test_objectstore$(EXEEXT)
+am__EXEEXT_8 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
+ ceph_erasure_code_benchmark$(EXEEXT) \
+ ceph_erasure_code$(EXEEXT) ceph_test_timers$(EXEEXT) \
+ ceph_test_signal_handlers$(EXEEXT) ceph_test_rados$(EXEEXT) \
+ ceph_test_mutate$(EXEEXT) ceph_test_rewrite_latency$(EXEEXT) \
+ ceph_test_msgr$(EXEEXT) ceph_streamtest$(EXEEXT) \
+ ceph_test_trans$(EXEEXT) ceph_test_crypto$(EXEEXT) \
+ ceph_test_keys$(EXEEXT) $(am__EXEEXT_2) \
+ ceph_smalliobench$(EXEEXT) ceph_smalliobenchfs$(EXEEXT) \
+ ceph_smalliobenchdumb$(EXEEXT) ceph_smalliobenchrbd$(EXEEXT) \
+ ceph_tpbench$(EXEEXT) ceph_omapbench$(EXEEXT) $(am__EXEEXT_3) \
+ ceph_bench_log$(EXEEXT) $(am__EXEEXT_4) \
+ ceph_multi_stress_watch$(EXEEXT) ceph_test_librbd$(EXEEXT) \
+ $(am__EXEEXT_5) ceph_test_cls_rbd$(EXEEXT) \
+ ceph_test_cls_refcount$(EXEEXT) ceph_test_cls_version$(EXEEXT) \
+ ceph_test_cls_log$(EXEEXT) ceph_test_cls_statelog$(EXEEXT) \
ceph_test_cls_replica_log$(EXEEXT) ceph_test_cls_lock$(EXEEXT) \
- ceph_test_cls_hello$(EXEEXT) $(am__EXEEXT_7) \
+ ceph_test_cls_hello$(EXEEXT) $(am__EXEEXT_6) \
ceph_test_mon_workloadgen$(EXEEXT) \
ceph_test_rados_api_cmd$(EXEEXT) \
ceph_test_rados_api_io$(EXEEXT) \
ceph_test_rados_api_c_write_operations$(EXEEXT) \
+ ceph_test_rados_api_c_read_operations$(EXEEXT) \
ceph_test_rados_api_aio$(EXEEXT) \
ceph_test_rados_api_list$(EXEEXT) \
ceph_test_rados_api_pool$(EXEEXT) \
@@ -832,7 +856,7 @@ am__EXEEXT_9 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
ceph_test_rados_api_misc$(EXEEXT) \
ceph_test_rados_api_tier$(EXEEXT) \
ceph_test_rados_api_lock$(EXEEXT) ceph_test_libcephfs$(EXEEXT) \
- $(am__EXEEXT_8) ceph_test_filestore_workloadgen$(EXEEXT) \
+ $(am__EXEEXT_7) ceph_test_objectstore_workloadgen$(EXEEXT) \
ceph_test_filestore_idempotent$(EXEEXT) \
ceph_test_filestore_idempotent_sequence$(EXEEXT) \
ceph_xattr_bench$(EXEEXT) ceph_test_filejournal$(EXEEXT) \
@@ -848,13 +872,13 @@ am__EXEEXT_9 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
ceph_scratchtool$(EXEEXT) ceph_scratchtoolpp$(EXEEXT) \
ceph_psim$(EXEEXT) ceph_dupstore$(EXEEXT) \
ceph_radosacl$(EXEEXT)
- at WITH_DEBUG_TRUE@am__EXEEXT_10 = $(am__EXEEXT_9)
- at WITH_RADOSGW_TRUE@am__EXEEXT_11 = radosgw$(EXEEXT) \
+ at WITH_DEBUG_TRUE@am__EXEEXT_9 = $(am__EXEEXT_8)
+ at WITH_RADOSGW_TRUE@am__EXEEXT_10 = radosgw$(EXEEXT) \
@WITH_RADOSGW_TRUE@ radosgw-admin$(EXEEXT)
- at WITH_REST_BENCH_TRUE@am__EXEEXT_12 = rest-bench$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_13 = rbd$(EXEEXT)
- at WITH_FUSE_TRUE@am__EXEEXT_14 = ceph-fuse$(EXEEXT) rbd-fuse$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_15 = mount.ceph$(EXEEXT)
+ at WITH_REST_BENCH_TRUE@am__EXEEXT_11 = rest-bench$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_12 = rbd$(EXEEXT)
+ at WITH_FUSE_TRUE@am__EXEEXT_13 = ceph-fuse$(EXEEXT) rbd-fuse$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_14 = mount.ceph$(EXEEXT)
PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) $(sbin_PROGRAMS) \
$(su_sbin_PROGRAMS)
am_ceph_authtool_OBJECTS = tools/ceph_authtool.$(OBJEXT)
@@ -866,25 +890,36 @@ ceph_conf_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON)
am__ceph_dencoder_SOURCES_DIST = test/encoding/ceph_dencoder.cc \
rgw/rgw_dencoder.cc rgw/rgw_acl.cc rgw/rgw_common.cc \
rgw/rgw_env.cc rgw/rgw_json_enc.cc
- at WITH_RADOSGW_TRUE@am__objects_9 = \
+ at WITH_RADOSGW_TRUE@am__objects_10 = \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_dencoder.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_acl.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_common.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_env.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_json_enc.$(OBJEXT)
-am__objects_10 = $(am__objects_9)
+am__objects_11 = $(am__objects_10)
am_ceph_dencoder_OBJECTS = \
test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT) \
- $(am__objects_10)
+ $(am__objects_11)
ceph_dencoder_OBJECTS = $(am_ceph_dencoder_OBJECTS)
am__DEPENDENCIES_7 = libperfglue.la $(am__DEPENDENCIES_1)
am__DEPENDENCIES_8 = libosd.la $(LIBOSDC) $(am__DEPENDENCIES_5) \
$(am__DEPENDENCIES_7)
am__DEPENDENCIES_9 = libmds.la $(am__DEPENDENCIES_7)
am__DEPENDENCIES_10 = libmon.la $(am__DEPENDENCIES_7)
+ at WITH_RADOSGW_TRUE@am__DEPENDENCIES_11 = $(LIBRADOS) \
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_log_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_user_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_replica_log_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_lock_client.la \
+ at WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
+ at WITH_RADOSGW_TRUE@ libcls_version_client.a
+am__DEPENDENCIES_12 = $(am__DEPENDENCIES_11)
+ at WITH_RADOSGW_TRUE@am__DEPENDENCIES_13 = $(LIBRGW) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12)
ceph_dencoder_DEPENDENCIES = $(am__DEPENDENCIES_8) \
$(am__DEPENDENCIES_9) $(am__DEPENDENCIES_10) $(DENCODER_DEPS) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_6) $(am__DEPENDENCIES_13)
ceph_dencoder_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \
@@ -918,7 +953,7 @@ ceph_monstore_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \
am_ceph_osd_OBJECTS = ceph_osd.$(OBJEXT)
ceph_osd_OBJECTS = $(am_ceph_osd_OBJECTS)
ceph_osd_DEPENDENCIES = $(am__DEPENDENCIES_8) $(am__DEPENDENCIES_6) \
- $(LIBCOMMON) $(am__DEPENDENCIES_1)
+ $(LIBCOMMON)
am_ceph_osdomap_tool_OBJECTS = tools/ceph-osdomap-tool.$(OBJEXT)
ceph_osdomap_tool_OBJECTS = $(am_ceph_osdomap_tool_OBJECTS)
ceph_osdomap_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \
@@ -934,8 +969,14 @@ am_ceph_dupstore_OBJECTS = tools/dupstore.$(OBJEXT)
ceph_dupstore_OBJECTS = $(am_ceph_dupstore_OBJECTS)
ceph_dupstore_DEPENDENCIES = $(am__DEPENDENCIES_5) \
$(am__DEPENDENCIES_6)
+am_ceph_erasure_code_OBJECTS = \
+ test/erasure-code/ceph_erasure_code.$(OBJEXT)
+ceph_erasure_code_OBJECTS = $(am_ceph_erasure_code_OBJECTS)
+ceph_erasure_code_DEPENDENCIES = $(am__DEPENDENCIES_8) $(LIBCOMMON) \
+ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) \
+ $(am__DEPENDENCIES_1)
am_ceph_erasure_code_benchmark_OBJECTS = \
- test/osd/ceph_erasure_code_benchmark.$(OBJEXT)
+ test/erasure-code/ceph_erasure_code_benchmark.$(OBJEXT)
ceph_erasure_code_benchmark_OBJECTS = \
$(am_ceph_erasure_code_benchmark_OBJECTS)
ceph_erasure_code_benchmark_DEPENDENCIES = $(am__DEPENDENCIES_8) \
@@ -966,11 +1007,11 @@ ceph_mon_store_converter_OBJECTS = \
ceph_mon_store_converter_DEPENDENCIES = $(am__DEPENDENCIES_10) \
$(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6)
am_ceph_multi_stress_watch_OBJECTS = \
- test/multi_stress_watch.$(OBJEXT) test/librados/test.$(OBJEXT)
+ test/multi_stress_watch.$(OBJEXT)
ceph_multi_stress_watch_OBJECTS = \
$(am_ceph_multi_stress_watch_OBJECTS)
ceph_multi_stress_watch_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_6) $(RADOS_TEST_LDADD)
am_ceph_omapbench_OBJECTS = test/omap_bench.$(OBJEXT)
ceph_omapbench_OBJECTS = $(am_ceph_omapbench_OBJECTS)
ceph_omapbench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6)
@@ -988,15 +1029,6 @@ am__ceph_rgw_jsonparser_SOURCES_DIST = rgw/rgw_jsonparser.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_env.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.$(OBJEXT)
ceph_rgw_jsonparser_OBJECTS = $(am_ceph_rgw_jsonparser_OBJECTS)
- at WITH_RADOSGW_TRUE@am__DEPENDENCIES_11 = $(LIBRADOS) \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_log_client.a \
- at WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
- at WITH_RADOSGW_TRUE@ libcls_user_client.a \
- at WITH_RADOSGW_TRUE@ libcls_replica_log_client.a \
- at WITH_RADOSGW_TRUE@ libcls_lock_client.la \
- at WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_version_client.a
-am__DEPENDENCIES_12 = $(am__DEPENDENCIES_11)
@WITH_RADOSGW_TRUE at ceph_rgw_jsonparser_DEPENDENCIES = $(LIBRGW) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6)
@@ -1057,75 +1089,72 @@ ceph_test_cfuse_cache_invalidate_OBJECTS = \
$(am_ceph_test_cfuse_cache_invalidate_OBJECTS)
ceph_test_cfuse_cache_invalidate_LDADD = $(LDADD)
am_ceph_test_cls_hello_OBJECTS = \
- test/cls_hello/ceph_test_cls_hello-test_cls_hello.$(OBJEXT) \
- test/librados/ceph_test_cls_hello-test.$(OBJEXT)
+ test/cls_hello/ceph_test_cls_hello-test_cls_hello.$(OBJEXT)
ceph_test_cls_hello_OBJECTS = $(am_ceph_test_cls_hello_OBJECTS)
-am__DEPENDENCIES_13 = $(top_builddir)/src/gtest/lib/libgtest.a \
+am__DEPENDENCIES_14 = $(top_builddir)/src/gtest/lib/libgtest.a \
$(top_builddir)/src/gtest/lib/libgtest_main.a \
$(am__DEPENDENCIES_1)
ceph_test_cls_hello_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
+ $(RADOS_TEST_LDADD)
ceph_test_cls_hello_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_cls_lock_OBJECTS = \
- test/cls_lock/ceph_test_cls_lock-test_cls_lock.$(OBJEXT) \
- test/librados/ceph_test_cls_lock-test.$(OBJEXT)
+ test/cls_lock/ceph_test_cls_lock-test_cls_lock.$(OBJEXT)
ceph_test_cls_lock_OBJECTS = $(am_ceph_test_cls_lock_OBJECTS)
ceph_test_cls_lock_DEPENDENCIES = $(LIBRADOS) libcls_lock_client.la \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_cls_lock_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_cls_log_OBJECTS = \
- test/cls_log/ceph_test_cls_log-test_cls_log.$(OBJEXT) \
- test/librados/ceph_test_cls_log-test.$(OBJEXT)
+ test/cls_log/ceph_test_cls_log-test_cls_log.$(OBJEXT)
ceph_test_cls_log_OBJECTS = $(am_ceph_test_cls_log_OBJECTS)
ceph_test_cls_log_DEPENDENCIES = $(LIBRADOS) libcls_log_client.a \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
+ $(RADOS_TEST_LDADD)
ceph_test_cls_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_cls_rbd_OBJECTS = \
- test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.$(OBJEXT) \
- test/librados/ceph_test_cls_rbd-test.$(OBJEXT)
+ test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.$(OBJEXT)
ceph_test_cls_rbd_OBJECTS = $(am_ceph_test_cls_rbd_OBJECTS)
ceph_test_cls_rbd_DEPENDENCIES = $(LIBRADOS) libcls_rbd_client.la \
- libcls_lock_client.la $(am__DEPENDENCIES_13)
+ libcls_lock_client.la $(am__DEPENDENCIES_14) \
+ $(RADOS_TEST_LDADD)
ceph_test_cls_rbd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_cls_refcount_OBJECTS = test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.$(OBJEXT) \
- test/librados/ceph_test_cls_refcount-test.$(OBJEXT)
+am_ceph_test_cls_refcount_OBJECTS = test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.$(OBJEXT)
ceph_test_cls_refcount_OBJECTS = $(am_ceph_test_cls_refcount_OBJECTS)
ceph_test_cls_refcount_DEPENDENCIES = $(LIBRADOS) \
- libcls_refcount_client.la $(am__DEPENDENCIES_13)
+ libcls_refcount_client.la $(am__DEPENDENCIES_14) \
+ $(RADOS_TEST_LDADD)
ceph_test_cls_refcount_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_cls_replica_log_OBJECTS = test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.$(OBJEXT) \
- test/librados/ceph_test_cls_replica_log-test.$(OBJEXT)
+am_ceph_test_cls_replica_log_OBJECTS = test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.$(OBJEXT)
ceph_test_cls_replica_log_OBJECTS = \
$(am_ceph_test_cls_replica_log_OBJECTS)
ceph_test_cls_replica_log_DEPENDENCIES = $(LIBRADOS) \
- libcls_replica_log_client.a $(am__DEPENDENCIES_13) \
- $(am__DEPENDENCIES_6)
+ libcls_replica_log_client.a $(am__DEPENDENCIES_14) \
+ $(am__DEPENDENCIES_6) $(RADOS_TEST_LDADD)
ceph_test_cls_replica_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
-am__ceph_test_cls_rgw_SOURCES_DIST = test/cls_rgw/test_cls_rgw.cc \
- test/librados/test.cc
- at WITH_RADOSGW_TRUE@am_ceph_test_cls_rgw_OBJECTS = test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.$(OBJEXT) \
- at WITH_RADOSGW_TRUE@ test/librados/ceph_test_cls_rgw-test.$(OBJEXT)
+am__ceph_test_cls_rgw_SOURCES_DIST = test/cls_rgw/test_cls_rgw.cc
+ at WITH_RADOSGW_TRUE@am_ceph_test_cls_rgw_OBJECTS = test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.$(OBJEXT)
ceph_test_cls_rgw_OBJECTS = $(am_ceph_test_cls_rgw_OBJECTS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_DEPENDENCIES = $(LIBRADOS) \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la $(am__DEPENDENCIES_13)
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la $(am__DEPENDENCIES_14) \
+ at WITH_RADOSGW_TRUE@ $(RADOS_TEST_LDADD)
ceph_test_cls_rgw_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1135,7 +1164,7 @@ am__ceph_test_cls_rgw_log_SOURCES_DIST = test/test_rgw_admin_log.cc
ceph_test_cls_rgw_log_OBJECTS = $(am_ceph_test_cls_rgw_log_OBJECTS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_log_DEPENDENCIES = $(LIBRADOS) \
@WITH_RADOSGW_TRUE@ $(LIBRGW) $(am__DEPENDENCIES_6) \
- at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_14) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
@@ -1151,7 +1180,7 @@ am__ceph_test_cls_rgw_meta_SOURCES_DIST = test/test_rgw_admin_meta.cc
ceph_test_cls_rgw_meta_OBJECTS = $(am_ceph_test_cls_rgw_meta_OBJECTS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_meta_DEPENDENCIES = $(LIBRADOS) \
@WITH_RADOSGW_TRUE@ $(LIBRGW) $(am__DEPENDENCIES_6) \
- at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_14) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
@@ -1169,7 +1198,7 @@ ceph_test_cls_rgw_opstate_OBJECTS = \
$(am_ceph_test_cls_rgw_opstate_OBJECTS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_opstate_DEPENDENCIES = \
@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(am__DEPENDENCIES_6) \
- at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_14) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
@@ -1180,21 +1209,20 @@ ceph_test_cls_rgw_opstate_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
-am_ceph_test_cls_statelog_OBJECTS = test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.$(OBJEXT) \
- test/librados/ceph_test_cls_statelog-test.$(OBJEXT)
+am_ceph_test_cls_statelog_OBJECTS = test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.$(OBJEXT)
ceph_test_cls_statelog_OBJECTS = $(am_ceph_test_cls_statelog_OBJECTS)
ceph_test_cls_statelog_DEPENDENCIES = $(LIBRADOS) \
- libcls_statelog_client.a $(am__DEPENDENCIES_13) \
- $(am__DEPENDENCIES_6)
+ libcls_statelog_client.a $(am__DEPENDENCIES_14) \
+ $(am__DEPENDENCIES_6) $(RADOS_TEST_LDADD)
ceph_test_cls_statelog_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_cls_version_OBJECTS = test/cls_version/ceph_test_cls_version-test_cls_version.$(OBJEXT) \
- test/librados/ceph_test_cls_version-test.$(OBJEXT)
+am_ceph_test_cls_version_OBJECTS = test/cls_version/ceph_test_cls_version-test_cls_version.$(OBJEXT)
ceph_test_cls_version_OBJECTS = $(am_ceph_test_cls_version_OBJECTS)
ceph_test_cls_version_DEPENDENCIES = $(LIBRADOS) \
- libcls_version_client.a $(am__DEPENDENCIES_13)
+ libcls_version_client.a $(am__DEPENDENCIES_14) \
+ $(RADOS_TEST_LDADD)
ceph_test_cls_version_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1205,7 +1233,7 @@ am__ceph_test_cors_SOURCES_DIST = test/test_cors.cc
ceph_test_cors_OBJECTS = $(am_ceph_test_cors_OBJECTS)
@WITH_RADOSGW_TRUE at ceph_test_cors_DEPENDENCIES = $(LIBRADOS) $(LIBRGW) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) \
- at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13)
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_14)
ceph_test_cors_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1217,44 +1245,28 @@ am_ceph_test_filejournal_OBJECTS = \
test/ceph_test_filejournal-test_filejournal.$(OBJEXT)
ceph_test_filejournal_OBJECTS = $(am_ceph_test_filejournal_OBJECTS)
ceph_test_filejournal_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
ceph_test_filejournal_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am__ceph_test_filestore_SOURCES_DIST = test/filestore/store_test.cc
- at LINUX_TRUE@am_ceph_test_filestore_OBJECTS = test/filestore/ceph_test_filestore-store_test.$(OBJEXT)
-ceph_test_filestore_OBJECTS = $(am_ceph_test_filestore_OBJECTS)
- at LINUX_TRUE@ceph_test_filestore_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- at LINUX_TRUE@ $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
-ceph_test_filestore_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
- $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
- $(ceph_test_filestore_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
- $(LDFLAGS) -o $@
am_ceph_test_filestore_idempotent_OBJECTS = \
- test/filestore/test_idempotent.$(OBJEXT) \
- test/filestore/FileStoreTracker.$(OBJEXT) \
+ test/objectstore/test_idempotent.$(OBJEXT) \
+ test/objectstore/FileStoreTracker.$(OBJEXT) \
test/common/ObjectContents.$(OBJEXT)
ceph_test_filestore_idempotent_OBJECTS = \
$(am_ceph_test_filestore_idempotent_OBJECTS)
ceph_test_filestore_idempotent_DEPENDENCIES = $(am__DEPENDENCIES_5) \
$(am__DEPENDENCIES_6)
am_ceph_test_filestore_idempotent_sequence_OBJECTS = \
- test/filestore/test_idempotent_sequence.$(OBJEXT) \
- test/filestore/DeterministicOpSequence.$(OBJEXT) \
- test/filestore/TestFileStoreState.$(OBJEXT) \
- test/filestore/FileStoreDiff.$(OBJEXT)
+ test/objectstore/test_idempotent_sequence.$(OBJEXT) \
+ test/objectstore/DeterministicOpSequence.$(OBJEXT) \
+ test/objectstore/TestObjectStoreState.$(OBJEXT) \
+ test/objectstore/FileStoreDiff.$(OBJEXT)
ceph_test_filestore_idempotent_sequence_OBJECTS = \
$(am_ceph_test_filestore_idempotent_sequence_OBJECTS)
ceph_test_filestore_idempotent_sequence_DEPENDENCIES = \
$(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6)
-am_ceph_test_filestore_workloadgen_OBJECTS = \
- test/filestore/workload_generator.$(OBJEXT) \
- test/filestore/TestFileStoreState.$(OBJEXT)
-ceph_test_filestore_workloadgen_OBJECTS = \
- $(am_ceph_test_filestore_workloadgen_OBJECTS)
-ceph_test_filestore_workloadgen_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
am_ceph_test_get_blkdev_size_OBJECTS = \
test/test_get_blkdev_size.$(OBJEXT)
ceph_test_get_blkdev_size_OBJECTS = \
@@ -1271,7 +1283,7 @@ am_ceph_test_keyvaluedb_atomicity_OBJECTS = test/ObjectMap/ceph_test_keyvaluedb_
ceph_test_keyvaluedb_atomicity_OBJECTS = \
$(am_ceph_test_keyvaluedb_atomicity_OBJECTS)
ceph_test_keyvaluedb_atomicity_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
ceph_test_keyvaluedb_atomicity_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) \
@@ -1281,7 +1293,7 @@ am_ceph_test_keyvaluedb_iterators_OBJECTS = test/ObjectMap/ceph_test_keyvaluedb_
ceph_test_keyvaluedb_iterators_OBJECTS = \
$(am_ceph_test_keyvaluedb_iterators_OBJECTS)
ceph_test_keyvaluedb_iterators_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
ceph_test_keyvaluedb_iterators_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) \
@@ -1292,17 +1304,17 @@ am_ceph_test_libcephfs_OBJECTS = \
test/libcephfs/ceph_test_libcephfs-caps.$(OBJEXT) \
test/libcephfs/ceph_test_libcephfs-multiclient.$(OBJEXT)
ceph_test_libcephfs_OBJECTS = $(am_ceph_test_libcephfs_OBJECTS)
-ceph_test_libcephfs_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13)
+ceph_test_libcephfs_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_14)
ceph_test_libcephfs_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_librbd_OBJECTS = \
- test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT) \
- test/librados/ceph_test_librbd-test.$(OBJEXT)
+ test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT)
ceph_test_librbd_OBJECTS = $(am_ceph_test_librbd_OBJECTS)
ceph_test_librbd_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
+ $(RADOS_TEST_LDADD)
ceph_test_librbd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1333,7 +1345,7 @@ am_ceph_test_object_map_OBJECTS = \
test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.$(OBJEXT)
ceph_test_object_map_OBJECTS = $(am_ceph_test_object_map_OBJECTS)
ceph_test_object_map_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
ceph_test_object_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1345,147 +1357,163 @@ ceph_test_objectcacher_stress_OBJECTS = \
$(am_ceph_test_objectcacher_stress_OBJECTS)
ceph_test_objectcacher_stress_DEPENDENCIES = $(LIBOSDC) \
$(am__DEPENDENCIES_6)
+am__ceph_test_objectstore_SOURCES_DIST = \
+ test/objectstore/store_test.cc
+ at LINUX_TRUE@am_ceph_test_objectstore_OBJECTS = test/objectstore/ceph_test_objectstore-store_test.$(OBJEXT)
+ceph_test_objectstore_OBJECTS = $(am_ceph_test_objectstore_OBJECTS)
+ at LINUX_TRUE@ceph_test_objectstore_DEPENDENCIES = \
+ at LINUX_TRUE@ $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_14) \
+ at LINUX_TRUE@ $(am__DEPENDENCIES_6)
+ceph_test_objectstore_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
+am_ceph_test_objectstore_workloadgen_OBJECTS = \
+ test/objectstore/workload_generator.$(OBJEXT) \
+ test/objectstore/TestObjectStoreState.$(OBJEXT)
+ceph_test_objectstore_workloadgen_OBJECTS = \
+ $(am_ceph_test_objectstore_workloadgen_OBJECTS)
+ceph_test_objectstore_workloadgen_DEPENDENCIES = \
+ $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6)
am_ceph_test_rados_OBJECTS = test/osd/TestRados.$(OBJEXT) \
test/osd/TestOpStat.$(OBJEXT) test/osd/Object.$(OBJEXT) \
test/osd/RadosModel.$(OBJEXT)
ceph_test_rados_OBJECTS = $(am_ceph_test_rados_OBJECTS)
ceph_test_rados_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6)
am_ceph_test_rados_api_aio_OBJECTS = \
- test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT) \
- test/librados/ceph_test_rados_api_aio-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT)
ceph_test_rados_api_aio_OBJECTS = \
$(am_ceph_test_rados_api_aio_OBJECTS)
ceph_test_rados_api_aio_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_aio_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_rados_api_c_write_operations_OBJECTS = test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT) \
- test/librados/ceph_test_rados_api_c_write_operations-test.$(OBJEXT)
+am_ceph_test_rados_api_c_read_operations_OBJECTS = test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.$(OBJEXT)
+ceph_test_rados_api_c_read_operations_OBJECTS = \
+ $(am_ceph_test_rados_api_c_read_operations_OBJECTS)
+ceph_test_rados_api_c_read_operations_DEPENDENCIES = $(LIBRADOS) \
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_c_read_operations_LINK = $(LIBTOOL) $(AM_V_lt) \
+ --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
+ $(CXXLD) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) \
+ $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+am_ceph_test_rados_api_c_write_operations_OBJECTS = test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT)
ceph_test_rados_api_c_write_operations_OBJECTS = \
$(am_ceph_test_rados_api_c_write_operations_OBJECTS)
ceph_test_rados_api_c_write_operations_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_c_write_operations_LINK = $(LIBTOOL) $(AM_V_lt) \
--tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
$(CXXLD) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
am_ceph_test_rados_api_cls_OBJECTS = \
- test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT) \
- test/librados/ceph_test_rados_api_cls-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT)
ceph_test_rados_api_cls_OBJECTS = \
$(am_ceph_test_rados_api_cls_OBJECTS)
ceph_test_rados_api_cls_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_cls_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_cmd_OBJECTS = \
- test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT) \
- test/librados/ceph_test_rados_api_cmd-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT)
ceph_test_rados_api_cmd_OBJECTS = \
$(am_ceph_test_rados_api_cmd_OBJECTS)
ceph_test_rados_api_cmd_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_cmd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_io_OBJECTS = \
- test/librados/ceph_test_rados_api_io-io.$(OBJEXT) \
- test/librados/ceph_test_rados_api_io-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_io-io.$(OBJEXT)
ceph_test_rados_api_io_OBJECTS = $(am_ceph_test_rados_api_io_OBJECTS)
ceph_test_rados_api_io_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_io_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_list_OBJECTS = \
- test/librados/ceph_test_rados_api_list-list.$(OBJEXT) \
- test/librados/ceph_test_rados_api_list-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_list-list.$(OBJEXT)
ceph_test_rados_api_list_OBJECTS = \
$(am_ceph_test_rados_api_list_OBJECTS)
ceph_test_rados_api_list_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_list_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_lock_OBJECTS = \
- test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT) \
- test/librados/ceph_test_rados_api_lock-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT)
ceph_test_rados_api_lock_OBJECTS = \
$(am_ceph_test_rados_api_lock_OBJECTS)
ceph_test_rados_api_lock_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_lock_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_misc_OBJECTS = \
- test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT) \
- test/librados/ceph_test_rados_api_misc-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT)
ceph_test_rados_api_misc_OBJECTS = \
$(am_ceph_test_rados_api_misc_OBJECTS)
ceph_test_rados_api_misc_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
+ $(RADOS_TEST_LDADD)
ceph_test_rados_api_misc_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_pool_OBJECTS = \
- test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT) \
- test/librados/ceph_test_rados_api_pool-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT)
ceph_test_rados_api_pool_OBJECTS = \
$(am_ceph_test_rados_api_pool_OBJECTS)
ceph_test_rados_api_pool_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_pool_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_rados_api_snapshots_OBJECTS = test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT) \
- test/librados/ceph_test_rados_api_snapshots-test.$(OBJEXT)
+am_ceph_test_rados_api_snapshots_OBJECTS = test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT)
ceph_test_rados_api_snapshots_OBJECTS = \
$(am_ceph_test_rados_api_snapshots_OBJECTS)
ceph_test_rados_api_snapshots_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_snapshots_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
am_ceph_test_rados_api_stat_OBJECTS = \
- test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT) \
- test/librados/ceph_test_rados_api_stat-test.$(OBJEXT)
+ test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT)
ceph_test_rados_api_stat_OBJECTS = \
$(am_ceph_test_rados_api_stat_OBJECTS)
ceph_test_rados_api_stat_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_stat_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_rados_api_tier_OBJECTS = \
test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT) \
- test/librados/ceph_test_rados_api_tier-test.$(OBJEXT) \
osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT)
ceph_test_rados_api_tier_OBJECTS = \
$(am_ceph_test_rados_api_tier_OBJECTS)
ceph_test_rados_api_tier_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
+ $(RADOS_TEST_LDADD)
ceph_test_rados_api_tier_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_rados_api_watch_notify_OBJECTS = test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT) \
- test/librados/ceph_test_rados_api_watch_notify-test.$(OBJEXT)
+am_ceph_test_rados_api_watch_notify_OBJECTS = test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT)
ceph_test_rados_api_watch_notify_OBJECTS = \
$(am_ceph_test_rados_api_watch_notify_OBJECTS)
ceph_test_rados_api_watch_notify_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_rados_api_watch_notify_LINK = $(LIBTOOL) $(AM_V_lt) \
--tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
$(CXXLD) $(ceph_test_rados_api_watch_notify_CXXFLAGS) \
@@ -1550,6 +1578,19 @@ ceph_test_rewrite_latency_OBJECTS = \
ceph_test_rewrite_latency_DEPENDENCIES = $(LIBCOMMON) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
+am__ceph_test_rgw_manifest_SOURCES_DIST = \
+ test/rgw/test_rgw_manifest.cc
+ at WITH_RADOSGW_TRUE@am_ceph_test_rgw_manifest_OBJECTS = test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.$(OBJEXT)
+ceph_test_rgw_manifest_OBJECTS = $(am_ceph_test_rgw_manifest_OBJECTS)
+ at WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_DEPENDENCIES = $(LIBRADOS) \
+ at WITH_RADOSGW_TRUE@ $(LIBRGW) $(am__DEPENDENCIES_12) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_14) \
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1)
+ceph_test_rgw_manifest_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_ceph_test_signal_handlers_OBJECTS = \
test/TestSignalHandlers.$(OBJEXT)
ceph_test_signal_handlers_OBJECTS = \
@@ -1559,17 +1600,16 @@ am_ceph_test_snap_mapper_OBJECTS = \
test/ceph_test_snap_mapper-test_snap_mapper.$(OBJEXT)
ceph_test_snap_mapper_OBJECTS = $(am_ceph_test_snap_mapper_OBJECTS)
ceph_test_snap_mapper_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
ceph_test_snap_mapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_ceph_test_stress_watch_OBJECTS = \
- test/ceph_test_stress_watch-test_stress_watch.$(OBJEXT) \
- test/librados/ceph_test_stress_watch-test.$(OBJEXT)
+ test/ceph_test_stress_watch-test_stress_watch.$(OBJEXT)
ceph_test_stress_watch_OBJECTS = $(am_ceph_test_stress_watch_OBJECTS)
ceph_test_stress_watch_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14) $(RADOS_TEST_LDADD)
ceph_test_stress_watch_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1590,7 +1630,7 @@ am_ceph_xattr_bench_OBJECTS = \
test/ceph_xattr_bench-xattr_bench.$(OBJEXT)
ceph_xattr_bench_OBJECTS = $(am_ceph_xattr_bench_OBJECTS)
ceph_xattr_bench_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
ceph_xattr_bench_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1679,23 +1719,23 @@ am__rest_bench_SOURCES_DIST = tools/rest_bench.cc \
@WITH_REST_BENCH_TRUE@ tools/rest_bench-rest_bench.$(OBJEXT) \
@WITH_REST_BENCH_TRUE@ common/rest_bench-obj_bencher.$(OBJEXT)
rest_bench_OBJECTS = $(am_rest_bench_OBJECTS)
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__DEPENDENCIES_14 = libs3/build/lib/libs3.a
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__DEPENDENCIES_15 = libs3/build/lib/libs3.a
@WITH_REST_BENCH_TRUE at rest_bench_DEPENDENCIES = $(am__DEPENDENCIES_6) \
@WITH_REST_BENCH_TRUE@ $(am__DEPENDENCIES_1) \
- at WITH_REST_BENCH_TRUE@ $(am__DEPENDENCIES_14)
+ at WITH_REST_BENCH_TRUE@ $(am__DEPENDENCIES_15)
rest_bench_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(rest_bench_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
am__test_build_libcephfs_SOURCES_DIST = test/buildtest_skeleton.cc \
osdc/Objecter.cc osdc/ObjectCacher.cc osdc/Filer.cc \
osdc/Striper.cc osdc/Journaler.cc
-am__objects_11 = osdc/test_build_libcephfs-Objecter.$(OBJEXT) \
+am__objects_12 = osdc/test_build_libcephfs-Objecter.$(OBJEXT) \
osdc/test_build_libcephfs-ObjectCacher.$(OBJEXT) \
osdc/test_build_libcephfs-Filer.$(OBJEXT) \
osdc/test_build_libcephfs-Striper.$(OBJEXT) \
osdc/test_build_libcephfs-Journaler.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_libcephfs_OBJECTS = test/test_build_libcephfs-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_11)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_12)
test_build_libcephfs_OBJECTS = $(am_test_build_libcephfs_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_libcephfs_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(LIBCEPHFS) $(am__DEPENDENCIES_1) \
@@ -1721,10 +1761,10 @@ am__test_build_libcommon_SOURCES_DIST = test/buildtest_skeleton.cc \
common/errno.cc common/RefCountedObj.cc common/blkdev.cc \
common/common_init.cc common/pipe.c common/ceph_argparse.cc \
common/ceph_context.cc common/buffer.cc \
- common/code_environment.cc common/dout.cc common/signal.cc \
- common/simple_spin.cc common/Thread.cc common/Formatter.cc \
- common/HeartbeatMap.cc common/config.cc common/utf8.c \
- common/mime.c common/strtol.cc common/page.cc \
+ common/code_environment.cc common/dout.cc common/histogram.cc \
+ common/signal.cc common/simple_spin.cc common/Thread.cc \
+ common/Formatter.cc common/HeartbeatMap.cc common/config.cc \
+ common/utf8.c common/mime.c common/strtol.cc common/page.cc \
common/lockdep.cc common/version.cc common/hex.cc \
common/entity_name.cc common/ceph_crypto.cc \
common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \
@@ -1733,11 +1773,11 @@ am__test_build_libcommon_SOURCES_DIST = test/buildtest_skeleton.cc \
common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \
common/bloom_filter.cc common/linux_version.c common/secret.c \
mon/MonCap.cc mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc \
- osd/osd_types.cc osd/HitSet.cc mds/MDSMap.cc \
+ osd/osd_types.cc osd/ECMsgTypes.cc osd/HitSet.cc mds/MDSMap.cc \
mds/inode_backtrace.cc mds/mdstypes.cc
- at LINUX_TRUE@am__objects_12 = \
+ at LINUX_TRUE@am__objects_13 = \
@LINUX_TRUE@ common/test_build_libcommon-secret.$(OBJEXT)
-am__objects_13 = test_build_libcommon-ceph_ver.$(OBJEXT) \
+am__objects_14 = test_build_libcommon-ceph_ver.$(OBJEXT) \
common/test_build_libcommon-DecayCounter.$(OBJEXT) \
common/test_build_libcommon-LogClient.$(OBJEXT) \
common/test_build_libcommon-LogEntry.$(OBJEXT) \
@@ -1778,6 +1818,7 @@ am__objects_13 = test_build_libcommon-ceph_ver.$(OBJEXT) \
common/test_build_libcommon-buffer.$(OBJEXT) \
common/test_build_libcommon-code_environment.$(OBJEXT) \
common/test_build_libcommon-dout.$(OBJEXT) \
+ common/test_build_libcommon-histogram.$(OBJEXT) \
common/test_build_libcommon-signal.$(OBJEXT) \
common/test_build_libcommon-simple_spin.$(OBJEXT) \
common/test_build_libcommon-Thread.$(OBJEXT) \
@@ -1807,17 +1848,18 @@ am__objects_13 = test_build_libcommon-ceph_ver.$(OBJEXT) \
common/test_build_libcommon-hobject.$(OBJEXT) \
common/test_build_libcommon-bloom_filter.$(OBJEXT) \
common/test_build_libcommon-linux_version.$(OBJEXT) \
- $(am__objects_12) mon/test_build_libcommon-MonCap.$(OBJEXT) \
+ $(am__objects_13) mon/test_build_libcommon-MonCap.$(OBJEXT) \
mon/test_build_libcommon-MonClient.$(OBJEXT) \
mon/test_build_libcommon-MonMap.$(OBJEXT) \
osd/test_build_libcommon-OSDMap.$(OBJEXT) \
osd/test_build_libcommon-osd_types.$(OBJEXT) \
+ osd/test_build_libcommon-ECMsgTypes.$(OBJEXT) \
osd/test_build_libcommon-HitSet.$(OBJEXT) \
mds/test_build_libcommon-MDSMap.$(OBJEXT) \
mds/test_build_libcommon-inode_backtrace.$(OBJEXT) \
mds/test_build_libcommon-mdstypes.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_libcommon_OBJECTS = test/test_build_libcommon-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_13)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_14)
test_build_libcommon_OBJECTS = $(am_test_build_libcommon_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_libcommon_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_3) \
@@ -1831,12 +1873,12 @@ test_build_libcommon_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am__test_build_librados_SOURCES_DIST = test/buildtest_skeleton.cc \
librados/librados.cc librados/RadosClient.cc \
librados/IoCtxImpl.cc librados/snap_set_diff.cc
-am__objects_14 = librados/test_build_librados-librados.$(OBJEXT) \
+am__objects_15 = librados/test_build_librados-librados.$(OBJEXT) \
librados/test_build_librados-RadosClient.$(OBJEXT) \
librados/test_build_librados-IoCtxImpl.$(OBJEXT) \
librados/test_build_librados-snap_set_diff.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_librados_OBJECTS = test/test_build_librados-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_14)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_15)
test_build_librados_OBJECTS = $(am_test_build_librados_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_librados_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(LIBRADOS_DEPS) $(am__DEPENDENCIES_1) \
@@ -1858,7 +1900,7 @@ am__test_build_librgw_SOURCES_DIST = test/buildtest_skeleton.cc \
rgw/rgw_multi_del.cc rgw/rgw_env.cc rgw/rgw_cors.cc \
rgw/rgw_cors_s3.cc rgw/rgw_auth_s3.cc rgw/rgw_metadata.cc \
rgw/rgw_replica_log.cc rgw/rgw_keystone.cc rgw/rgw_quota.cc
- at WITH_RADOSGW_TRUE@am__objects_15 = \
+ at WITH_RADOSGW_TRUE@am__objects_16 = \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-librgw.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl_s3.$(OBJEXT) \
@@ -1893,7 +1935,7 @@ am__test_build_librgw_SOURCES_DIST = test/buildtest_skeleton.cc \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_keystone.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_quota.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_librgw_OBJECTS = test/test_build_librgw-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_15)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_16)
test_build_librgw_OBJECTS = $(am_test_build_librgw_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_librgw_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_12) \
@@ -1907,7 +1949,7 @@ test_build_librgw_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(test_build_librgw_LDFLAGS) $(LDFLAGS) -o $@
am_unittest_addrs_OBJECTS = test/unittest_addrs-test_addrs.$(OBJEXT)
unittest_addrs_OBJECTS = $(am_unittest_addrs_OBJECTS)
-unittest_addrs_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_addrs_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_addrs_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1916,7 +1958,7 @@ unittest_addrs_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_admin_socket_OBJECTS = \
test/unittest_admin_socket-admin_socket.$(OBJEXT)
unittest_admin_socket_OBJECTS = $(am_unittest_admin_socket_OBJECTS)
-unittest_admin_socket_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_admin_socket_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_admin_socket_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1924,7 +1966,7 @@ unittest_admin_socket_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(LDFLAGS) -o $@
am_unittest_arch_OBJECTS = test/unittest_arch-test_arch.$(OBJEXT)
unittest_arch_OBJECTS = $(am_unittest_arch_OBJECTS)
-unittest_arch_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_arch_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_arch_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1932,7 +1974,7 @@ unittest_arch_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
-o $@
am_unittest_base64_OBJECTS = test/unittest_base64-base64.$(OBJEXT)
unittest_base64_OBJECTS = $(am_unittest_base64_OBJECTS)
-unittest_base64_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13)
+unittest_base64_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_14)
unittest_base64_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_base64_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1940,7 +1982,7 @@ unittest_base64_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_bloom_filter_OBJECTS = \
test/common/unittest_bloom_filter-test_bloom_filter.$(OBJEXT)
unittest_bloom_filter_OBJECTS = $(am_unittest_bloom_filter_OBJECTS)
-unittest_bloom_filter_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_bloom_filter_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_bloom_filter_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1949,7 +1991,7 @@ unittest_bloom_filter_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_bufferlist_OBJECTS = \
test/unittest_bufferlist-bufferlist.$(OBJEXT)
unittest_bufferlist_OBJECTS = $(am_unittest_bufferlist_OBJECTS)
-unittest_bufferlist_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_bufferlist_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_bufferlist_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1958,7 +2000,7 @@ unittest_bufferlist_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_ceph_argparse_OBJECTS = \
test/unittest_ceph_argparse-ceph_argparse.$(OBJEXT)
unittest_ceph_argparse_OBJECTS = $(am_unittest_ceph_argparse_OBJECTS)
-unittest_ceph_argparse_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_ceph_argparse_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_ceph_argparse_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1968,7 +2010,7 @@ am_unittest_ceph_compatset_OBJECTS = \
test/unittest_ceph_compatset-ceph_compatset.$(OBJEXT)
unittest_ceph_compatset_OBJECTS = \
$(am_unittest_ceph_compatset_OBJECTS)
-unittest_ceph_compatset_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_ceph_compatset_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_ceph_compatset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -1977,17 +2019,17 @@ unittest_ceph_compatset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_ceph_crypto_OBJECTS = \
test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT)
unittest_ceph_crypto_OBJECTS = $(am_unittest_ceph_crypto_OBJECTS)
-unittest_ceph_crypto_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_ceph_crypto_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_ceph_crypto_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_unittest_chain_xattr_OBJECTS = \
- test/filestore/unittest_chain_xattr-chain_xattr.$(OBJEXT)
+ test/objectstore/unittest_chain_xattr-chain_xattr.$(OBJEXT)
unittest_chain_xattr_OBJECTS = $(am_unittest_chain_xattr_OBJECTS)
unittest_chain_xattr_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_chain_xattr_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1995,7 +2037,7 @@ unittest_chain_xattr_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_config_OBJECTS = \
test/common/unittest_config-test_config.$(OBJEXT)
unittest_config_OBJECTS = $(am_unittest_config_OBJECTS)
-unittest_config_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_config_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2004,16 +2046,25 @@ unittest_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_confutils_OBJECTS = \
test/unittest_confutils-confutils.$(OBJEXT)
unittest_confutils_OBJECTS = $(am_unittest_confutils_OBJECTS)
-unittest_confutils_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_confutils_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_confutils_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_confutils_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_context_OBJECTS = \
+ test/common/unittest_context-test_context.$(OBJEXT)
+unittest_context_OBJECTS = $(am_unittest_context_OBJECTS)
+unittest_context_DEPENDENCIES = $(am__DEPENDENCIES_14) \
+ $(am__DEPENDENCIES_6)
+unittest_context_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_context_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_crc32c_OBJECTS = \
test/common/unittest_crc32c-test_crc32c.$(OBJEXT)
unittest_crc32c_OBJECTS = $(am_unittest_crc32c_OBJECTS)
-unittest_crc32c_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_crc32c_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_crc32c_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2023,7 +2074,7 @@ am_unittest_crush_indep_OBJECTS = \
test/crush/unittest_crush_indep-indep.$(OBJEXT)
unittest_crush_indep_OBJECTS = $(am_unittest_crush_indep_OBJECTS)
unittest_crush_indep_DEPENDENCIES = $(LIBCOMMON) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_2) \
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_2) \
$(am__DEPENDENCIES_6)
unittest_crush_indep_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2032,24 +2083,15 @@ unittest_crush_indep_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_crush_wrapper_OBJECTS = \
test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT)
unittest_crush_wrapper_OBJECTS = $(am_unittest_crush_wrapper_OBJECTS)
-unittest_crush_wrapper_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_crush_wrapper_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6) $(LIBCRUSH)
unittest_crush_wrapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_unittest_crushwrapper_OBJECTS = \
- test/unittest_crushwrapper-test_crushwrapper.$(OBJEXT)
-unittest_crushwrapper_OBJECTS = $(am_unittest_crushwrapper_OBJECTS)
-unittest_crushwrapper_DEPENDENCIES = $(am__DEPENDENCIES_13) \
- $(am__DEPENDENCIES_6) $(LIBCRUSH)
-unittest_crushwrapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
- $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
- $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
- $(LDFLAGS) -o $@
am_unittest_crypto_OBJECTS = test/unittest_crypto-crypto.$(OBJEXT)
unittest_crypto_OBJECTS = $(am_unittest_crypto_OBJECTS)
-unittest_crypto_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_crypto_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_crypto_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2058,63 +2100,72 @@ unittest_crypto_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_daemon_config_OBJECTS = \
test/unittest_daemon_config-daemon_config.$(OBJEXT)
unittest_daemon_config_OBJECTS = $(am_unittest_daemon_config_OBJECTS)
-unittest_daemon_config_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_daemon_config_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_daemon_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_ecbackend_OBJECTS = \
+ test/osd/unittest_ecbackend-TestECBackend.$(OBJEXT)
+unittest_ecbackend_OBJECTS = $(am_unittest_ecbackend_OBJECTS)
+unittest_ecbackend_DEPENDENCIES = $(am__DEPENDENCIES_8) \
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
+unittest_ecbackend_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_encoding_OBJECTS = \
test/unittest_encoding-encoding.$(OBJEXT)
unittest_encoding_OBJECTS = $(am_unittest_encoding_OBJECTS)
unittest_encoding_DEPENDENCIES = $(LIBCEPHFS) $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14)
unittest_encoding_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_encoding_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_unittest_erasure_code_example_OBJECTS = test/osd/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT)
+am_unittest_erasure_code_example_OBJECTS = test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT)
unittest_erasure_code_example_OBJECTS = \
$(am_unittest_erasure_code_example_OBJECTS)
unittest_erasure_code_example_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(LIBCOMMON) $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(LIBCOMMON) $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_erasure_code_example_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
-am__objects_16 = osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT) \
- osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT) \
- osd/ErasureCodePluginJerasure/cauchy.$(OBJEXT) \
- osd/ErasureCodePluginJerasure/galois.$(OBJEXT) \
- osd/ErasureCodePluginJerasure/jerasure.$(OBJEXT) \
- osd/ErasureCodePluginJerasure/liberation.$(OBJEXT) \
- osd/ErasureCodePluginJerasure/reed_sol.$(OBJEXT)
-am_unittest_erasure_code_jerasure_OBJECTS = test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT) \
- $(am__objects_16)
+am__objects_17 = erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT) \
+ erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT) \
+ erasure-code/jerasure/cauchy.$(OBJEXT) \
+ erasure-code/jerasure/galois.$(OBJEXT) \
+ erasure-code/jerasure/jerasure.$(OBJEXT) \
+ erasure-code/jerasure/liberation.$(OBJEXT) \
+ erasure-code/jerasure/reed_sol.$(OBJEXT)
+am_unittest_erasure_code_jerasure_OBJECTS = test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT) \
+ $(am__objects_17)
unittest_erasure_code_jerasure_OBJECTS = \
$(am_unittest_erasure_code_jerasure_OBJECTS)
unittest_erasure_code_jerasure_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(LIBCOMMON) $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \
+ $(LIBCOMMON) $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
$(am__DEPENDENCIES_1)
unittest_erasure_code_jerasure_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
-am_unittest_erasure_code_plugin_OBJECTS = test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT)
+am_unittest_erasure_code_plugin_OBJECTS = test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT)
unittest_erasure_code_plugin_OBJECTS = \
$(am_unittest_erasure_code_plugin_OBJECTS)
unittest_erasure_code_plugin_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(LIBCOMMON) $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \
+ $(LIBCOMMON) $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
$(am__DEPENDENCIES_1)
unittest_erasure_code_plugin_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
-am_unittest_erasure_code_plugin_jerasure_OBJECTS = test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT)
+am_unittest_erasure_code_plugin_jerasure_OBJECTS = test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT)
unittest_erasure_code_plugin_jerasure_OBJECTS = \
$(am_unittest_erasure_code_plugin_jerasure_OBJECTS)
unittest_erasure_code_plugin_jerasure_DEPENDENCIES = \
- $(am__DEPENDENCIES_8) $(LIBCOMMON) $(am__DEPENDENCIES_13) \
+ $(am__DEPENDENCIES_8) $(LIBCOMMON) $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6) $(am__DEPENDENCIES_1)
unittest_erasure_code_plugin_jerasure_LINK = $(LIBTOOL) $(AM_V_lt) \
--tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
@@ -2122,7 +2173,7 @@ unittest_erasure_code_plugin_jerasure_LINK = $(LIBTOOL) $(AM_V_lt) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
am_unittest_escape_OBJECTS = test/unittest_escape-escape.$(OBJEXT)
unittest_escape_OBJECTS = $(am_unittest_escape_OBJECTS)
-unittest_escape_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_escape_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_escape_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2132,7 +2183,7 @@ am_unittest_flatindex_OBJECTS = \
test/os/unittest_flatindex-TestFlatIndex.$(OBJEXT)
unittest_flatindex_OBJECTS = $(am_unittest_flatindex_OBJECTS)
unittest_flatindex_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_flatindex_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_flatindex_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2141,7 +2192,7 @@ am_unittest_formatter_OBJECTS = \
test/unittest_formatter-formatter.$(OBJEXT) \
rgw/unittest_formatter-rgw_formats.$(OBJEXT)
unittest_formatter_OBJECTS = $(am_unittest_formatter_OBJECTS)
-unittest_formatter_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_formatter_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_formatter_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2149,7 +2200,7 @@ unittest_formatter_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(LDFLAGS) -o $@
am_unittest_gather_OBJECTS = test/unittest_gather-gather.$(OBJEXT)
unittest_gather_OBJECTS = $(am_unittest_gather_OBJECTS)
-unittest_gather_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_gather_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_gather_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2159,16 +2210,25 @@ am_unittest_heartbeatmap_OBJECTS = \
test/unittest_heartbeatmap-heartbeat_map.$(OBJEXT)
unittest_heartbeatmap_OBJECTS = $(am_unittest_heartbeatmap_OBJECTS)
unittest_heartbeatmap_DEPENDENCIES = $(LIBCOMMON) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_heartbeatmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_histogram_OBJECTS = \
+ test/common/unittest_histogram-histogram.$(OBJEXT)
+unittest_histogram_OBJECTS = $(am_unittest_histogram_OBJECTS)
+unittest_histogram_DEPENDENCIES = $(am__DEPENDENCIES_14) \
+ $(am__DEPENDENCIES_6)
+unittest_histogram_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_hitset_OBJECTS = \
test/osd/unittest_hitset-hitset.$(OBJEXT)
unittest_hitset_OBJECTS = $(am_unittest_hitset_OBJECTS)
unittest_hitset_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_hitset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_hitset_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2176,7 +2236,7 @@ unittest_hitset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_ipaddr_OBJECTS = \
test/unittest_ipaddr-test_ipaddr.$(OBJEXT)
unittest_ipaddr_OBJECTS = $(am_unittest_ipaddr_OBJECTS)
-unittest_ipaddr_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_ipaddr_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_ipaddr_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2186,7 +2246,7 @@ am_unittest_lfnindex_OBJECTS = \
test/os/unittest_lfnindex-TestLFNIndex.$(OBJEXT)
unittest_lfnindex_OBJECTS = $(am_unittest_lfnindex_OBJECTS)
unittest_lfnindex_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_lfnindex_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_lfnindex_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2196,7 +2256,7 @@ am_unittest_libcephfs_config_OBJECTS = \
unittest_libcephfs_config_OBJECTS = \
$(am_unittest_libcephfs_config_OBJECTS)
unittest_libcephfs_config_DEPENDENCIES = $(LIBCEPHFS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14)
unittest_libcephfs_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_libcephfs_config_CXXFLAGS) $(CXXFLAGS) \
@@ -2204,7 +2264,7 @@ unittest_libcephfs_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_librados_OBJECTS = \
test/librados/unittest_librados-librados.$(OBJEXT)
unittest_librados_OBJECTS = $(am_unittest_librados_OBJECTS)
-unittest_librados_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_13)
+unittest_librados_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_14)
unittest_librados_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_librados_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2213,20 +2273,20 @@ am_unittest_librados_config_OBJECTS = test/librados/unittest_librados_config-lib
unittest_librados_config_OBJECTS = \
$(am_unittest_librados_config_OBJECTS)
unittest_librados_config_DEPENDENCIES = $(LIBRADOS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14)
unittest_librados_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_librados_config_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_unittest_log_OBJECTS = log/unittest_log-test.$(OBJEXT)
unittest_log_OBJECTS = $(am_unittest_log_OBJECTS)
-unittest_log_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_13)
+unittest_log_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_14)
unittest_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(unittest_log_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
am_unittest_mime_OBJECTS = test/unittest_mime-mime.$(OBJEXT)
unittest_mime_OBJECTS = $(am_unittest_mime_OBJECTS)
-unittest_mime_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_mime_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_mime_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2236,11 +2296,20 @@ am_unittest_mon_moncap_OBJECTS = \
test/mon/unittest_mon_moncap-moncap.$(OBJEXT)
unittest_mon_moncap_OBJECTS = $(am_unittest_mon_moncap_OBJECTS)
unittest_mon_moncap_DEPENDENCIES = $(am__DEPENDENCIES_10) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_mon_moncap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_mon_pgmap_OBJECTS = \
+ test/mon/unittest_mon_pgmap-PGMap.$(OBJEXT)
+unittest_mon_pgmap_OBJECTS = $(am_unittest_mon_pgmap_OBJECTS)
+unittest_mon_pgmap_DEPENDENCIES = $(am__DEPENDENCIES_10) \
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
+unittest_mon_pgmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_on_exit_OBJECTS = test/on_exit.$(OBJEXT)
unittest_on_exit_OBJECTS = $(am_unittest_on_exit_OBJECTS)
unittest_on_exit_DEPENDENCIES = $(am__DEPENDENCIES_1)
@@ -2248,15 +2317,15 @@ am_unittest_osd_osdcap_OBJECTS = \
test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT)
unittest_osd_osdcap_OBJECTS = $(am_unittest_osd_osdcap_OBJECTS)
unittest_osd_osdcap_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6)
unittest_osd_osdcap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
am_unittest_osd_types_OBJECTS = \
- test/unittest_osd_types-test_osd_types.$(OBJEXT)
+ test/osd/unittest_osd_types-types.$(OBJEXT)
unittest_osd_types_OBJECTS = $(am_unittest_osd_types_OBJECTS)
-unittest_osd_types_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_osd_types_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_osd_types_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2265,7 +2334,7 @@ unittest_osd_types_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_osdmap_OBJECTS = \
test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT)
unittest_osdmap_OBJECTS = $(am_unittest_osdmap_OBJECTS)
-unittest_osdmap_DEPENDENCIES = $(am__DEPENDENCIES_13) $(LIBCOMMON) \
+unittest_osdmap_DEPENDENCIES = $(am__DEPENDENCIES_14) $(LIBCOMMON) \
$(am__DEPENDENCIES_6)
unittest_osdmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2274,7 +2343,7 @@ unittest_osdmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_perf_counters_OBJECTS = \
test/unittest_perf_counters-perf_counters.$(OBJEXT)
unittest_perf_counters_OBJECTS = $(am_unittest_perf_counters_OBJECTS)
-unittest_perf_counters_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_perf_counters_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_perf_counters_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2284,7 +2353,7 @@ am_unittest_pglog_OBJECTS = \
test/osd/unittest_pglog-TestPGLog.$(OBJEXT)
unittest_pglog_OBJECTS = $(am_unittest_pglog_OBJECTS)
unittest_pglog_DEPENDENCIES = $(am__DEPENDENCIES_8) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_6) \
$(am__DEPENDENCIES_1)
unittest_pglog_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2294,14 +2363,14 @@ am_unittest_prebufferedstreambuf_OBJECTS = test/unittest_prebufferedstreambuf-te
unittest_prebufferedstreambuf_OBJECTS = \
$(am_unittest_prebufferedstreambuf_OBJECTS)
unittest_prebufferedstreambuf_DEPENDENCIES = $(LIBCOMMON) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_2)
+ $(am__DEPENDENCIES_14) $(am__DEPENDENCIES_2)
unittest_prebufferedstreambuf_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_prebufferedstreambuf_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
am_unittest_run_cmd_OBJECTS = test/unittest_run_cmd-run_cmd.$(OBJEXT)
unittest_run_cmd_OBJECTS = $(am_unittest_run_cmd_OBJECTS)
-unittest_run_cmd_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13)
+unittest_run_cmd_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_14)
unittest_run_cmd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_run_cmd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2309,7 +2378,7 @@ unittest_run_cmd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_sharedptr_registry_OBJECTS = test/common/unittest_sharedptr_registry-test_sharedptr_registry.$(OBJEXT)
unittest_sharedptr_registry_OBJECTS = \
$(am_unittest_sharedptr_registry_OBJECTS)
-unittest_sharedptr_registry_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_sharedptr_registry_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_sharedptr_registry_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2317,7 +2386,7 @@ unittest_sharedptr_registry_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
am_unittest_signals_OBJECTS = test/unittest_signals-signals.$(OBJEXT)
unittest_signals_OBJECTS = $(am_unittest_signals_OBJECTS)
-unittest_signals_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_signals_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_signals_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2327,7 +2396,7 @@ am_unittest_simple_spin_OBJECTS = \
test/unittest_simple_spin-simple_spin.$(OBJEXT)
unittest_simple_spin_OBJECTS = $(am_unittest_simple_spin_OBJECTS)
unittest_simple_spin_DEPENDENCIES = $(LIBCEPHFS) \
- $(am__DEPENDENCIES_13)
+ $(am__DEPENDENCIES_14)
unittest_simple_spin_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_simple_spin_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2335,7 +2404,7 @@ unittest_simple_spin_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_sloppy_crc_map_OBJECTS = test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.$(OBJEXT)
unittest_sloppy_crc_map_OBJECTS = \
$(am_unittest_sloppy_crc_map_OBJECTS)
-unittest_sloppy_crc_map_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_sloppy_crc_map_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_sloppy_crc_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2344,7 +2413,7 @@ unittest_sloppy_crc_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_str_list_OBJECTS = \
test/unittest_str_list-test_str_list.$(OBJEXT)
unittest_str_list_OBJECTS = $(am_unittest_str_list_OBJECTS)
-unittest_str_list_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_str_list_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_str_list_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2353,7 +2422,7 @@ unittest_str_list_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_str_map_OBJECTS = \
test/common/unittest_str_map-test_str_map.$(OBJEXT)
unittest_str_map_OBJECTS = $(am_unittest_str_map_OBJECTS)
-unittest_str_map_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_str_map_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_str_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2362,7 +2431,7 @@ unittest_str_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_striper_OBJECTS = \
test/unittest_striper-test_striper.$(OBJEXT)
unittest_striper_OBJECTS = $(am_unittest_striper_OBJECTS)
-unittest_striper_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_13) \
+unittest_striper_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_striper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2370,7 +2439,7 @@ unittest_striper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(LDFLAGS) -o $@
am_unittest_strtol_OBJECTS = test/unittest_strtol-strtol.$(OBJEXT)
unittest_strtol_OBJECTS = $(am_unittest_strtol_OBJECTS)
-unittest_strtol_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_strtol_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_strtol_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2379,7 +2448,7 @@ unittest_strtol_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_texttable_OBJECTS = \
test/unittest_texttable-test_texttable.$(OBJEXT)
unittest_texttable_OBJECTS = $(am_unittest_texttable_OBJECTS)
-unittest_texttable_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_13)
+unittest_texttable_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_14)
unittest_texttable_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_texttable_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -2387,7 +2456,7 @@ unittest_texttable_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_throttle_OBJECTS = \
test/common/unittest_throttle-Throttle.$(OBJEXT)
unittest_throttle_OBJECTS = $(am_unittest_throttle_OBJECTS)
-unittest_throttle_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_throttle_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_throttle_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2395,7 +2464,7 @@ unittest_throttle_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(LDFLAGS) -o $@
am_unittest_utf8_OBJECTS = test/unittest_utf8-utf8.$(OBJEXT)
unittest_utf8_OBJECTS = $(am_unittest_utf8_OBJECTS)
-unittest_utf8_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_utf8_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_utf8_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2404,7 +2473,7 @@ unittest_utf8_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_util_OBJECTS = \
test/common/unittest_util-test_util.$(OBJEXT)
unittest_util_OBJECTS = $(am_unittest_util_OBJECTS)
-unittest_util_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_13) \
+unittest_util_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2)
unittest_util_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2413,7 +2482,7 @@ unittest_util_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_unittest_workqueue_OBJECTS = \
test/unittest_workqueue-test_workqueue.$(OBJEXT)
unittest_workqueue_OBJECTS = $(am_unittest_workqueue_OBJECTS)
-unittest_workqueue_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+unittest_workqueue_DEPENDENCIES = $(am__DEPENDENCIES_14) \
$(am__DEPENDENCIES_6)
unittest_workqueue_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@@ -2491,11 +2560,12 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(libec_fail_to_register_la_SOURCES) $(libec_hangs_la_SOURCES) \
$(libec_jerasure_la_SOURCES) \
$(libec_missing_entry_point_la_SOURCES) \
- $(libglobal_la_SOURCES) $(libjson_spirit_la_SOURCES) \
- $(liblog_la_SOURCES) $(libmds_la_SOURCES) $(libmon_la_SOURCES) \
- $(libmsg_la_SOURCES) $(libos_la_SOURCES) $(libosd_la_SOURCES) \
- $(libosdc_la_SOURCES) $(libperfglue_la_SOURCES) \
- $(librados_la_SOURCES) $(librbd_la_SOURCES) \
+ $(liberasure_code_la_SOURCES) $(libglobal_la_SOURCES) \
+ $(libjson_spirit_la_SOURCES) $(liblog_la_SOURCES) \
+ $(libmds_la_SOURCES) $(libmon_la_SOURCES) $(libmsg_la_SOURCES) \
+ $(libos_la_SOURCES) $(libosd_la_SOURCES) $(libosdc_la_SOURCES) \
+ $(libperfglue_la_SOURCES) $(librados_la_SOURCES) \
+ $(libradostest_la_SOURCES) $(librbd_la_SOURCES) \
$(librgw_la_SOURCES) $(libsystest_la_SOURCES) \
$(ceph_authtool_SOURCES) $(ceph_conf_SOURCES) \
$(ceph_dencoder_SOURCES) $(ceph_fuse_SOURCES) \
@@ -2503,7 +2573,7 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_mon_SOURCES) $(ceph_monstore_tool_SOURCES) \
$(ceph_osd_SOURCES) $(ceph_osdomap_tool_SOURCES) \
$(ceph_syn_SOURCES) $(ceph_bench_log_SOURCES) \
- $(ceph_dupstore_SOURCES) \
+ $(ceph_dupstore_SOURCES) $(ceph_erasure_code_SOURCES) \
$(ceph_erasure_code_benchmark_SOURCES) \
$(ceph_filestore_dump_SOURCES) $(ceph_filestore_tool_SOURCES) \
$(ceph_kvstorebench_SOURCES) \
@@ -2526,10 +2596,8 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_cls_statelog_SOURCES) \
$(ceph_test_cls_version_SOURCES) $(ceph_test_cors_SOURCES) \
$(ceph_test_crypto_SOURCES) $(ceph_test_filejournal_SOURCES) \
- $(ceph_test_filestore_SOURCES) \
$(ceph_test_filestore_idempotent_SOURCES) \
$(ceph_test_filestore_idempotent_sequence_SOURCES) \
- $(ceph_test_filestore_workloadgen_SOURCES) \
$(ceph_test_get_blkdev_size_SOURCES) \
$(ceph_test_ioctls_SOURCES) $(ceph_test_keys_SOURCES) \
$(ceph_test_keyvaluedb_atomicity_SOURCES) \
@@ -2539,7 +2607,10 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_mon_workloadgen_SOURCES) $(ceph_test_msgr_SOURCES) \
$(ceph_test_mutate_SOURCES) $(ceph_test_object_map_SOURCES) \
$(ceph_test_objectcacher_stress_SOURCES) \
+ $(ceph_test_objectstore_SOURCES) \
+ $(ceph_test_objectstore_workloadgen_SOURCES) \
$(ceph_test_rados_SOURCES) $(ceph_test_rados_api_aio_SOURCES) \
+ $(ceph_test_rados_api_c_read_operations_SOURCES) \
$(ceph_test_rados_api_c_write_operations_SOURCES) \
$(ceph_test_rados_api_cls_SOURCES) \
$(ceph_test_rados_api_cmd_SOURCES) \
@@ -2557,6 +2628,7 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_rados_open_pools_parallel_SOURCES) \
$(ceph_test_rados_watch_notify_SOURCES) \
$(ceph_test_rewrite_latency_SOURCES) \
+ $(ceph_test_rgw_manifest_SOURCES) \
$(ceph_test_signal_handlers_SOURCES) \
$(ceph_test_snap_mapper_SOURCES) \
$(ceph_test_stress_watch_SOURCES) $(ceph_test_timers_SOURCES) \
@@ -2577,26 +2649,28 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(unittest_ceph_compatset_SOURCES) \
$(unittest_ceph_crypto_SOURCES) \
$(unittest_chain_xattr_SOURCES) $(unittest_config_SOURCES) \
- $(unittest_confutils_SOURCES) $(unittest_crc32c_SOURCES) \
- $(unittest_crush_indep_SOURCES) \
- $(unittest_crush_wrapper_SOURCES) \
- $(unittest_crushwrapper_SOURCES) $(unittest_crypto_SOURCES) \
- $(unittest_daemon_config_SOURCES) $(unittest_encoding_SOURCES) \
+ $(unittest_confutils_SOURCES) $(unittest_context_SOURCES) \
+ $(unittest_crc32c_SOURCES) $(unittest_crush_indep_SOURCES) \
+ $(unittest_crush_wrapper_SOURCES) $(unittest_crypto_SOURCES) \
+ $(unittest_daemon_config_SOURCES) \
+ $(unittest_ecbackend_SOURCES) $(unittest_encoding_SOURCES) \
$(unittest_erasure_code_example_SOURCES) \
$(unittest_erasure_code_jerasure_SOURCES) \
$(unittest_erasure_code_plugin_SOURCES) \
$(unittest_erasure_code_plugin_jerasure_SOURCES) \
$(unittest_escape_SOURCES) $(unittest_flatindex_SOURCES) \
$(unittest_formatter_SOURCES) $(unittest_gather_SOURCES) \
- $(unittest_heartbeatmap_SOURCES) $(unittest_hitset_SOURCES) \
- $(unittest_ipaddr_SOURCES) $(unittest_lfnindex_SOURCES) \
+ $(unittest_heartbeatmap_SOURCES) $(unittest_histogram_SOURCES) \
+ $(unittest_hitset_SOURCES) $(unittest_ipaddr_SOURCES) \
+ $(unittest_lfnindex_SOURCES) \
$(unittest_libcephfs_config_SOURCES) \
$(unittest_librados_SOURCES) \
$(unittest_librados_config_SOURCES) $(unittest_log_SOURCES) \
$(unittest_mime_SOURCES) $(unittest_mon_moncap_SOURCES) \
- $(unittest_on_exit_SOURCES) $(unittest_osd_osdcap_SOURCES) \
- $(unittest_osd_types_SOURCES) $(unittest_osdmap_SOURCES) \
- $(unittest_perf_counters_SOURCES) $(unittest_pglog_SOURCES) \
+ $(unittest_mon_pgmap_SOURCES) $(unittest_on_exit_SOURCES) \
+ $(unittest_osd_osdcap_SOURCES) $(unittest_osd_types_SOURCES) \
+ $(unittest_osdmap_SOURCES) $(unittest_perf_counters_SOURCES) \
+ $(unittest_pglog_SOURCES) \
$(unittest_prebufferedstreambuf_SOURCES) \
$(unittest_run_cmd_SOURCES) \
$(unittest_sharedptr_registry_SOURCES) \
@@ -2631,11 +2705,12 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(libec_fail_to_register_la_SOURCES) $(libec_hangs_la_SOURCES) \
$(libec_jerasure_la_SOURCES) \
$(libec_missing_entry_point_la_SOURCES) \
- $(libglobal_la_SOURCES) $(libjson_spirit_la_SOURCES) \
- $(liblog_la_SOURCES) $(libmds_la_SOURCES) $(libmon_la_SOURCES) \
- $(libmsg_la_SOURCES) $(am__libos_la_SOURCES_DIST) \
- $(libosd_la_SOURCES) $(libosdc_la_SOURCES) \
- $(am__libperfglue_la_SOURCES_DIST) $(librados_la_SOURCES) \
+ $(liberasure_code_la_SOURCES) $(libglobal_la_SOURCES) \
+ $(libjson_spirit_la_SOURCES) $(liblog_la_SOURCES) \
+ $(libmds_la_SOURCES) $(libmon_la_SOURCES) $(libmsg_la_SOURCES) \
+ $(am__libos_la_SOURCES_DIST) $(libosd_la_SOURCES) \
+ $(libosdc_la_SOURCES) $(am__libperfglue_la_SOURCES_DIST) \
+ $(librados_la_SOURCES) $(libradostest_la_SOURCES) \
$(librbd_la_SOURCES) $(am__librgw_la_SOURCES_DIST) \
$(am__libsystest_la_SOURCES_DIST) $(ceph_authtool_SOURCES) \
$(ceph_conf_SOURCES) $(am__ceph_dencoder_SOURCES_DIST) \
@@ -2644,6 +2719,7 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_monstore_tool_SOURCES) $(ceph_osd_SOURCES) \
$(ceph_osdomap_tool_SOURCES) $(ceph_syn_SOURCES) \
$(ceph_bench_log_SOURCES) $(ceph_dupstore_SOURCES) \
+ $(ceph_erasure_code_SOURCES) \
$(ceph_erasure_code_benchmark_SOURCES) \
$(ceph_filestore_dump_SOURCES) $(ceph_filestore_tool_SOURCES) \
$(am__ceph_kvstorebench_SOURCES_DIST) \
@@ -2669,10 +2745,8 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_cls_version_SOURCES) \
$(am__ceph_test_cors_SOURCES_DIST) $(ceph_test_crypto_SOURCES) \
$(ceph_test_filejournal_SOURCES) \
- $(am__ceph_test_filestore_SOURCES_DIST) \
$(ceph_test_filestore_idempotent_SOURCES) \
$(ceph_test_filestore_idempotent_sequence_SOURCES) \
- $(ceph_test_filestore_workloadgen_SOURCES) \
$(ceph_test_get_blkdev_size_SOURCES) \
$(ceph_test_ioctls_SOURCES) $(ceph_test_keys_SOURCES) \
$(ceph_test_keyvaluedb_atomicity_SOURCES) \
@@ -2682,7 +2756,10 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_mon_workloadgen_SOURCES) $(ceph_test_msgr_SOURCES) \
$(ceph_test_mutate_SOURCES) $(ceph_test_object_map_SOURCES) \
$(ceph_test_objectcacher_stress_SOURCES) \
+ $(am__ceph_test_objectstore_SOURCES_DIST) \
+ $(ceph_test_objectstore_workloadgen_SOURCES) \
$(ceph_test_rados_SOURCES) $(ceph_test_rados_api_aio_SOURCES) \
+ $(ceph_test_rados_api_c_read_operations_SOURCES) \
$(ceph_test_rados_api_c_write_operations_SOURCES) \
$(ceph_test_rados_api_cls_SOURCES) \
$(ceph_test_rados_api_cmd_SOURCES) \
@@ -2700,6 +2777,7 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(am__ceph_test_rados_open_pools_parallel_SOURCES_DIST) \
$(am__ceph_test_rados_watch_notify_SOURCES_DIST) \
$(ceph_test_rewrite_latency_SOURCES) \
+ $(am__ceph_test_rgw_manifest_SOURCES_DIST) \
$(ceph_test_signal_handlers_SOURCES) \
$(ceph_test_snap_mapper_SOURCES) \
$(ceph_test_stress_watch_SOURCES) $(ceph_test_timers_SOURCES) \
@@ -2723,26 +2801,28 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(unittest_ceph_compatset_SOURCES) \
$(unittest_ceph_crypto_SOURCES) \
$(unittest_chain_xattr_SOURCES) $(unittest_config_SOURCES) \
- $(unittest_confutils_SOURCES) $(unittest_crc32c_SOURCES) \
- $(unittest_crush_indep_SOURCES) \
- $(unittest_crush_wrapper_SOURCES) \
- $(unittest_crushwrapper_SOURCES) $(unittest_crypto_SOURCES) \
- $(unittest_daemon_config_SOURCES) $(unittest_encoding_SOURCES) \
+ $(unittest_confutils_SOURCES) $(unittest_context_SOURCES) \
+ $(unittest_crc32c_SOURCES) $(unittest_crush_indep_SOURCES) \
+ $(unittest_crush_wrapper_SOURCES) $(unittest_crypto_SOURCES) \
+ $(unittest_daemon_config_SOURCES) \
+ $(unittest_ecbackend_SOURCES) $(unittest_encoding_SOURCES) \
$(unittest_erasure_code_example_SOURCES) \
$(unittest_erasure_code_jerasure_SOURCES) \
$(unittest_erasure_code_plugin_SOURCES) \
$(unittest_erasure_code_plugin_jerasure_SOURCES) \
$(unittest_escape_SOURCES) $(unittest_flatindex_SOURCES) \
$(unittest_formatter_SOURCES) $(unittest_gather_SOURCES) \
- $(unittest_heartbeatmap_SOURCES) $(unittest_hitset_SOURCES) \
- $(unittest_ipaddr_SOURCES) $(unittest_lfnindex_SOURCES) \
+ $(unittest_heartbeatmap_SOURCES) $(unittest_histogram_SOURCES) \
+ $(unittest_hitset_SOURCES) $(unittest_ipaddr_SOURCES) \
+ $(unittest_lfnindex_SOURCES) \
$(unittest_libcephfs_config_SOURCES) \
$(unittest_librados_SOURCES) \
$(unittest_librados_config_SOURCES) $(unittest_log_SOURCES) \
$(unittest_mime_SOURCES) $(unittest_mon_moncap_SOURCES) \
- $(unittest_on_exit_SOURCES) $(unittest_osd_osdcap_SOURCES) \
- $(unittest_osd_types_SOURCES) $(unittest_osdmap_SOURCES) \
- $(unittest_perf_counters_SOURCES) $(unittest_pglog_SOURCES) \
+ $(unittest_mon_pgmap_SOURCES) $(unittest_on_exit_SOURCES) \
+ $(unittest_osd_osdcap_SOURCES) $(unittest_osd_types_SOURCES) \
+ $(unittest_osdmap_SOURCES) $(unittest_perf_counters_SOURCES) \
+ $(unittest_pglog_SOURCES) \
$(unittest_prebufferedstreambuf_SOURCES) \
$(unittest_run_cmd_SOURCES) \
$(unittest_sharedptr_registry_SOURCES) \
@@ -2820,28 +2900,30 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
os/LevelDBStore.h os/LFNIndex.h os/MemStore.h \
os/KeyValueStore.h os/ObjectMap.h os/ObjectStore.h \
os/SequencerPosition.h os/WBThrottle.h \
- os/ZFSFileStoreBackend.h os/ZFS.h \
- osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \
- osd/ErasureCodePluginJerasure/cauchy.h \
- osd/ErasureCodePluginJerasure/galois.h \
- osd/ErasureCodePluginJerasure/jerasure.h \
- osd/ErasureCodePluginJerasure/liberation.h \
- osd/ErasureCodePluginJerasure/reed_sol.h \
- osd/ErasureCodePluginJerasure/vectorop.h osd/Ager.h \
- osd/ClassHandler.h osd/ErasureCodeInterface.h \
- osd/ErasureCodePlugin.h osd/HitSet.h osd/OSD.h osd/OSDCap.h \
- osd/OSDMap.h osd/ObjectVersioner.h osd/OpRequest.h \
- osd/SnapMapper.h osd/PG.h osd/PGLog.h osd/ReplicatedPG.h \
- osd/PGBackend.h osd/ReplicatedBackend.h osd/Watch.h \
- osd/osd_types.h osdc/Blinker.h osdc/Filer.h osdc/Journaler.h \
- osdc/ObjectCacher.h osdc/Objecter.h osdc/Striper.h \
- osdc/WritebackHandler.h client/Client.h client/Dentry.h \
- client/Dir.h client/Fh.h client/Inode.h client/MetaRequest.h \
- client/MetaSession.h client/ClientSnapRealm.h \
- client/SyntheticClient.h client/Trace.h client/ioctl.h \
- client/ObjecterWriteback.h client/fuse_ll.h global/pidfile.h \
- global/global_init.h global/global_context.h \
- global/signal_handler.h json_spirit/json_spirit.h \
+ os/XfsFileStoreBackend.h os/ZFSFileStoreBackend.h os/ZFS.h \
+ osd/Ager.h osd/ClassHandler.h osd/HitSet.h osd/OSD.h \
+ osd/OSDCap.h osd/OSDMap.h osd/ObjectVersioner.h \
+ osd/OpRequest.h osd/SnapMapper.h osd/PG.h osd/PGLog.h \
+ osd/ReplicatedPG.h osd/PGBackend.h osd/ReplicatedBackend.h \
+ osd/TierAgentState.h osd/ECBackend.h osd/ECUtil.h \
+ osd/ECMsgTypes.h osd/ECTransaction.h osd/Watch.h \
+ osd/osd_types.h erasure-code/jerasure/ErasureCodeJerasure.h \
+ erasure-code/jerasure/cauchy.h erasure-code/jerasure/galois.h \
+ erasure-code/jerasure/jerasure.h \
+ erasure-code/jerasure/liberation.h \
+ erasure-code/jerasure/reed_sol.h \
+ erasure-code/jerasure/vectorop.h \
+ erasure-code/ErasureCodeInterface.h \
+ erasure-code/ErasureCodePlugin.h osdc/Blinker.h osdc/Filer.h \
+ osdc/Journaler.h osdc/ObjectCacher.h osdc/Objecter.h \
+ osdc/Striper.h osdc/WritebackHandler.h client/Client.h \
+ client/Dentry.h client/Dir.h client/Fh.h client/Inode.h \
+ client/MetaRequest.h client/MetaSession.h \
+ client/ClientSnapRealm.h client/SyntheticClient.h \
+ client/Trace.h client/ioctl.h client/ObjecterWriteback.h \
+ client/fuse_ll.h global/pidfile.h global/global_init.h \
+ global/global_context.h global/signal_handler.h \
+ json_spirit/json_spirit.h \
json_spirit/json_spirit_error_position.h \
json_spirit/json_spirit_reader.h \
json_spirit/json_spirit_reader_template.h \
@@ -2862,11 +2944,11 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
common/ceph_context.h common/xattr.h common/blkdev.h \
common/compiler_extensions.h common/debug.h common/dout.h \
common/escape.h common/fd.h common/version.h common/hex.h \
- common/entity_name.h common/errno.h common/environment.h \
- common/likely.h common/lockdep.h common/obj_bencher.h \
- common/snap_types.h common/Clock.h common/Cond.h \
- common/ConfUtils.h common/DecayCounter.h common/Finisher.h \
- common/Formatter.h common/perf_counters.h \
+ common/histogram.h common/entity_name.h common/errno.h \
+ common/environment.h common/likely.h common/lockdep.h \
+ common/obj_bencher.h common/snap_types.h common/Clock.h \
+ common/Cond.h common/ConfUtils.h common/DecayCounter.h \
+ common/Finisher.h common/Formatter.h common/perf_counters.h \
common/OutputDataSocket.h common/admin_socket.h \
common/admin_socket_client.h common/shared_cache.hpp \
common/tracked_int_ptr.hpp common/simple_cache.hpp \
@@ -2928,6 +3010,8 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
messages/MOSDPGLog.h messages/MOSDPGMissing.h \
messages/MOSDPGNotify.h messages/MOSDPGQuery.h \
messages/MOSDPGRemove.h messages/MOSDPGScan.h \
+ messages/MOSDECSubOpWrite.h messages/MOSDECSubOpWriteReply.h \
+ messages/MOSDECSubOpRead.h messages/MOSDECSubOpReadReply.h \
messages/MBackfillReserve.h messages/MRecoveryReserve.h \
messages/MMonQuorumService.h messages/MOSDPGTemp.h \
messages/MOSDPGTrim.h messages/MOSDPing.h \
@@ -2947,32 +3031,31 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
include/ceph_hash.h include/cmp.h include/color.h \
include/compat.h include/crc32c.h include/encoding.h \
include/err.h include/error.h include/filepath.h \
- include/frag.h include/hash.h include/histogram.h \
- include/intarith.h include/interval_set.h include/int_types.h \
- include/ipaddr.h include/linux_fiemap.h include/lru.h \
- include/msgr.h include/object.h include/page.h \
- include/rangeset.h include/rados.h include/rbd_types.h \
- include/statlite.h include/str_list.h include/str_map.h \
- include/stringify.h include/triple.h include/types.h \
- include/utime.h include/dlist.h include/elist.h include/uuid.h \
- include/xlist.h include/rados/librados.h \
- include/rados/rados_types.h include/rados/rados_types.hpp \
- include/rados/librados.hpp include/rados/librgw.h \
- include/rados/page.h include/rados/crc32c.h \
- include/rados/buffer.h include/rbd/features.h \
- include/rbd/librbd.h include/rbd/librbd.hpp include/util.h \
- include/stat.h include/on_exit.h include/memory.h \
- include/rados/memory.h include/hash_namespace.h \
- include/unordered_set.h include/unordered_map.h \
- librados/snap_set_diff.h librados/AioCompletionImpl.h \
- librados/IoCtxImpl.h librados/PoolAsyncCompletionImpl.h \
- librados/RadosClient.h librbd/AioCompletion.h \
- librbd/AioRequest.h librbd/ImageCtx.h librbd/internal.h \
- librbd/LibrbdWriteback.h librbd/parent_types.h \
- librbd/SnapInfo.h librbd/WatchCtx.h rgw/logrotate.conf \
- rgw/rgw_acl.h rgw/rgw_acl_s3.h rgw/rgw_acl_swift.h \
- rgw/rgw_client_io.h rgw/rgw_fcgi.h rgw/rgw_xml.h \
- rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
+ include/frag.h include/hash.h include/intarith.h \
+ include/interval_set.h include/int_types.h include/ipaddr.h \
+ include/linux_fiemap.h include/lru.h include/msgr.h \
+ include/object.h include/page.h include/rangeset.h \
+ include/rados.h include/rbd_types.h include/statlite.h \
+ include/str_list.h include/str_map.h include/stringify.h \
+ include/triple.h include/types.h include/utime.h \
+ include/dlist.h include/elist.h include/uuid.h include/xlist.h \
+ include/rados/librados.h include/rados/rados_types.h \
+ include/rados/rados_types.hpp include/rados/librados.hpp \
+ include/rados/librgw.h include/rados/page.h \
+ include/rados/crc32c.h include/rados/buffer.h \
+ include/rbd/features.h include/rbd/librbd.h \
+ include/rbd/librbd.hpp include/util.h include/stat.h \
+ include/on_exit.h include/memory.h include/rados/memory.h \
+ include/hash_namespace.h include/unordered_set.h \
+ include/unordered_map.h librados/snap_set_diff.h \
+ librados/AioCompletionImpl.h librados/IoCtxImpl.h \
+ librados/PoolAsyncCompletionImpl.h librados/RadosClient.h \
+ librbd/AioCompletion.h librbd/AioRequest.h librbd/ImageCtx.h \
+ librbd/internal.h librbd/LibrbdWriteback.h \
+ librbd/parent_types.h librbd/SnapInfo.h librbd/WatchCtx.h \
+ rgw/logrotate.conf rgw/rgw_acl.h rgw/rgw_acl_s3.h \
+ rgw/rgw_acl_swift.h rgw/rgw_client_io.h rgw/rgw_fcgi.h \
+ rgw/rgw_xml.h rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
rgw/rgw_cors_s3.h rgw/rgw_cors_swift.h rgw/rgw_string.h \
rgw/rgw_formats.h rgw/rgw_http_errors.h rgw/rgw_log.h \
rgw/rgw_loadgen.h rgw/rgw_multi.h rgw/rgw_policy_s3.h \
@@ -3007,20 +3090,22 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
cls/user/cls_user_ops.h cls/user/cls_user_types.h \
key_value_store/key_value_structure.h \
key_value_store/kv_flat_btree_async.h \
- key_value_store/kvs_arg_types.h test/osd/ErasureCodeExample.h \
+ key_value_store/kvs_arg_types.h \
+ test/erasure-code/ErasureCodeExample.h \
+ test/erasure-code/ceph_erasure_code_benchmark.h \
test/bench/backend.h test/bench/bencher.h \
test/bench/detailed_stat_collector.h test/bench/distribution.h \
test/bench/dumb_backend.h test/bench/rados_backend.h \
test/bench/rbd_backend.h test/bench/stat_collector.h \
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h test/encoding/types.h \
- test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreDiff.h \
- test/filestore/FileStoreTracker.h \
- test/filestore/TestFileStoreState.h \
- test/filestore/workload_generator.h test/kv_store_bench.h \
- test/librados/test.h test/ObjectMap/KeyValueDBMemory.h \
- test/omap_bench.h test/osd/ceph_erasure_code_benchmark.h \
+ test/objectstore/DeterministicOpSequence.h \
+ test/objectstore/FileStoreDiff.h \
+ test/objectstore/FileStoreTracker.h \
+ test/objectstore/TestObjectStoreState.h \
+ test/objectstore/workload_generator.h test/kv_store_bench.h \
+ test/librados/test.h test/librados/TestCase.h \
+ test/ObjectMap/KeyValueDBMemory.h test/omap_bench.h \
test/osdc/FakeWriteback.h test/osd/Object.h \
test/osd/RadosModel.h test/osd/TestOpStat.h \
test/system/cross_process_sem.h \
@@ -3075,7 +3160,7 @@ ACLOCAL = @ACLOCAL@
AMTAR = @AMTAR@
AM_CXXFLAGS = @AM_CXXFLAGS@ $(AM_COMMON_CFLAGS) -ftemplate-depth-1024 \
-Wnon-virtual-dtor -Wno-invalid-offsetof $(am__append_3) \
- $(am__append_6) $(am__append_23)
+ $(am__append_6) $(am__append_25)
AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
AR = @AR@
AUTOCONF = @AUTOCONF@
@@ -3243,12 +3328,15 @@ top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
AUTOMAKE_OPTIONS = gnu subdir-objects
-SUBDIRS = ocf java $(am__append_58)
+SUBDIRS = ocf java $(am__append_61)
DIST_SUBDIRS = gtest ocf libs3 java
BUILT_SOURCES = init-ceph
# extra bits
-EXTRA_DIST = $(srcdir)/$(shell_scripts:%=%.in) \
+EXTRA_DIST = brag/server brag/README.md brag/client \
+ $(srcdir)/test/mon/mon-test-helpers.sh \
+ $(srcdir)/test/osd/osd-test-helpers.sh \
+ $(srcdir)/$(shell_scripts:%=%.in) \
$(srcdir)/verify-mds-journal.sh $(srcdir)/vstart.sh \
$(srcdir)/stop.sh ceph-run $(srcdir)/ceph_common.sh \
$(srcdir)/init-radosgw $(srcdir)/init-radosgw.sysv \
@@ -3340,28 +3428,30 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
os/LevelDBStore.h os/LFNIndex.h os/MemStore.h \
os/KeyValueStore.h os/ObjectMap.h os/ObjectStore.h \
os/SequencerPosition.h os/WBThrottle.h \
- os/ZFSFileStoreBackend.h $(am__append_17) \
- osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \
- osd/ErasureCodePluginJerasure/cauchy.h \
- osd/ErasureCodePluginJerasure/galois.h \
- osd/ErasureCodePluginJerasure/jerasure.h \
- osd/ErasureCodePluginJerasure/liberation.h \
- osd/ErasureCodePluginJerasure/reed_sol.h \
- osd/ErasureCodePluginJerasure/vectorop.h osd/Ager.h \
- osd/ClassHandler.h osd/ErasureCodeInterface.h \
- osd/ErasureCodePlugin.h osd/HitSet.h osd/OSD.h osd/OSDCap.h \
- osd/OSDMap.h osd/ObjectVersioner.h osd/OpRequest.h \
- osd/SnapMapper.h osd/PG.h osd/PGLog.h osd/ReplicatedPG.h \
- osd/PGBackend.h osd/ReplicatedBackend.h osd/Watch.h \
- osd/osd_types.h osdc/Blinker.h osdc/Filer.h osdc/Journaler.h \
- osdc/ObjectCacher.h osdc/Objecter.h osdc/Striper.h \
- osdc/WritebackHandler.h client/Client.h client/Dentry.h \
- client/Dir.h client/Fh.h client/Inode.h client/MetaRequest.h \
- client/MetaSession.h client/ClientSnapRealm.h \
- client/SyntheticClient.h client/Trace.h client/ioctl.h \
- client/ObjecterWriteback.h $(am__append_20) global/pidfile.h \
- global/global_init.h global/global_context.h \
- global/signal_handler.h json_spirit/json_spirit.h \
+ os/XfsFileStoreBackend.h os/ZFSFileStoreBackend.h \
+ $(am__append_18) osd/Ager.h osd/ClassHandler.h osd/HitSet.h \
+ osd/OSD.h osd/OSDCap.h osd/OSDMap.h osd/ObjectVersioner.h \
+ osd/OpRequest.h osd/SnapMapper.h osd/PG.h osd/PGLog.h \
+ osd/ReplicatedPG.h osd/PGBackend.h osd/ReplicatedBackend.h \
+ osd/TierAgentState.h osd/ECBackend.h osd/ECUtil.h \
+ osd/ECMsgTypes.h osd/ECTransaction.h osd/Watch.h \
+ osd/osd_types.h erasure-code/jerasure/ErasureCodeJerasure.h \
+ erasure-code/jerasure/cauchy.h erasure-code/jerasure/galois.h \
+ erasure-code/jerasure/jerasure.h \
+ erasure-code/jerasure/liberation.h \
+ erasure-code/jerasure/reed_sol.h \
+ erasure-code/jerasure/vectorop.h \
+ erasure-code/ErasureCodeInterface.h \
+ erasure-code/ErasureCodePlugin.h osdc/Blinker.h osdc/Filer.h \
+ osdc/Journaler.h osdc/ObjectCacher.h osdc/Objecter.h \
+ osdc/Striper.h osdc/WritebackHandler.h client/Client.h \
+ client/Dentry.h client/Dir.h client/Fh.h client/Inode.h \
+ client/MetaRequest.h client/MetaSession.h \
+ client/ClientSnapRealm.h client/SyntheticClient.h \
+ client/Trace.h client/ioctl.h client/ObjecterWriteback.h \
+ $(am__append_22) global/pidfile.h global/global_init.h \
+ global/global_context.h global/signal_handler.h \
+ json_spirit/json_spirit.h \
json_spirit/json_spirit_error_position.h \
json_spirit/json_spirit_reader.h \
json_spirit/json_spirit_reader_template.h \
@@ -3382,11 +3472,11 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
common/ceph_context.h common/xattr.h common/blkdev.h \
common/compiler_extensions.h common/debug.h common/dout.h \
common/escape.h common/fd.h common/version.h common/hex.h \
- common/entity_name.h common/errno.h common/environment.h \
- common/likely.h common/lockdep.h common/obj_bencher.h \
- common/snap_types.h common/Clock.h common/Cond.h \
- common/ConfUtils.h common/DecayCounter.h common/Finisher.h \
- common/Formatter.h common/perf_counters.h \
+ common/histogram.h common/entity_name.h common/errno.h \
+ common/environment.h common/likely.h common/lockdep.h \
+ common/obj_bencher.h common/snap_types.h common/Clock.h \
+ common/Cond.h common/ConfUtils.h common/DecayCounter.h \
+ common/Finisher.h common/Formatter.h common/perf_counters.h \
common/OutputDataSocket.h common/admin_socket.h \
common/admin_socket_client.h common/shared_cache.hpp \
common/tracked_int_ptr.hpp common/simple_cache.hpp \
@@ -3448,6 +3538,8 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
messages/MOSDPGLog.h messages/MOSDPGMissing.h \
messages/MOSDPGNotify.h messages/MOSDPGQuery.h \
messages/MOSDPGRemove.h messages/MOSDPGScan.h \
+ messages/MOSDECSubOpWrite.h messages/MOSDECSubOpWriteReply.h \
+ messages/MOSDECSubOpRead.h messages/MOSDECSubOpReadReply.h \
messages/MBackfillReserve.h messages/MRecoveryReserve.h \
messages/MMonQuorumService.h messages/MOSDPGTemp.h \
messages/MOSDPGTrim.h messages/MOSDPing.h \
@@ -3467,32 +3559,31 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
include/ceph_hash.h include/cmp.h include/color.h \
include/compat.h include/crc32c.h include/encoding.h \
include/err.h include/error.h include/filepath.h \
- include/frag.h include/hash.h include/histogram.h \
- include/intarith.h include/interval_set.h include/int_types.h \
- include/ipaddr.h include/linux_fiemap.h include/lru.h \
- include/msgr.h include/object.h include/page.h \
- include/rangeset.h include/rados.h include/rbd_types.h \
- include/statlite.h include/str_list.h include/str_map.h \
- include/stringify.h include/triple.h include/types.h \
- include/utime.h include/dlist.h include/elist.h include/uuid.h \
- include/xlist.h include/rados/librados.h \
- include/rados/rados_types.h include/rados/rados_types.hpp \
- include/rados/librados.hpp include/rados/librgw.h \
- include/rados/page.h include/rados/crc32c.h \
- include/rados/buffer.h include/rbd/features.h \
- include/rbd/librbd.h include/rbd/librbd.hpp include/util.h \
- include/stat.h include/on_exit.h include/memory.h \
- include/rados/memory.h include/hash_namespace.h \
- include/unordered_set.h include/unordered_map.h \
- librados/snap_set_diff.h librados/AioCompletionImpl.h \
- librados/IoCtxImpl.h librados/PoolAsyncCompletionImpl.h \
- librados/RadosClient.h librbd/AioCompletion.h \
- librbd/AioRequest.h librbd/ImageCtx.h librbd/internal.h \
- librbd/LibrbdWriteback.h librbd/parent_types.h \
- librbd/SnapInfo.h librbd/WatchCtx.h rgw/logrotate.conf \
- rgw/rgw_acl.h rgw/rgw_acl_s3.h rgw/rgw_acl_swift.h \
- rgw/rgw_client_io.h rgw/rgw_fcgi.h rgw/rgw_xml.h \
- rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
+ include/frag.h include/hash.h include/intarith.h \
+ include/interval_set.h include/int_types.h include/ipaddr.h \
+ include/linux_fiemap.h include/lru.h include/msgr.h \
+ include/object.h include/page.h include/rangeset.h \
+ include/rados.h include/rbd_types.h include/statlite.h \
+ include/str_list.h include/str_map.h include/stringify.h \
+ include/triple.h include/types.h include/utime.h \
+ include/dlist.h include/elist.h include/uuid.h include/xlist.h \
+ include/rados/librados.h include/rados/rados_types.h \
+ include/rados/rados_types.hpp include/rados/librados.hpp \
+ include/rados/librgw.h include/rados/page.h \
+ include/rados/crc32c.h include/rados/buffer.h \
+ include/rbd/features.h include/rbd/librbd.h \
+ include/rbd/librbd.hpp include/util.h include/stat.h \
+ include/on_exit.h include/memory.h include/rados/memory.h \
+ include/hash_namespace.h include/unordered_set.h \
+ include/unordered_map.h librados/snap_set_diff.h \
+ librados/AioCompletionImpl.h librados/IoCtxImpl.h \
+ librados/PoolAsyncCompletionImpl.h librados/RadosClient.h \
+ librbd/AioCompletion.h librbd/AioRequest.h librbd/ImageCtx.h \
+ librbd/internal.h librbd/LibrbdWriteback.h \
+ librbd/parent_types.h librbd/SnapInfo.h librbd/WatchCtx.h \
+ rgw/logrotate.conf rgw/rgw_acl.h rgw/rgw_acl_s3.h \
+ rgw/rgw_acl_swift.h rgw/rgw_client_io.h rgw/rgw_fcgi.h \
+ rgw/rgw_xml.h rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
rgw/rgw_cors_s3.h rgw/rgw_cors_swift.h rgw/rgw_string.h \
rgw/rgw_formats.h rgw/rgw_http_errors.h rgw/rgw_log.h \
rgw/rgw_loadgen.h rgw/rgw_multi.h rgw/rgw_policy_s3.h \
@@ -3527,20 +3618,22 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
cls/user/cls_user_ops.h cls/user/cls_user_types.h \
key_value_store/key_value_structure.h \
key_value_store/kv_flat_btree_async.h \
- key_value_store/kvs_arg_types.h test/osd/ErasureCodeExample.h \
+ key_value_store/kvs_arg_types.h \
+ test/erasure-code/ErasureCodeExample.h \
+ test/erasure-code/ceph_erasure_code_benchmark.h \
test/bench/backend.h test/bench/bencher.h \
test/bench/detailed_stat_collector.h test/bench/distribution.h \
test/bench/dumb_backend.h test/bench/rados_backend.h \
test/bench/rbd_backend.h test/bench/stat_collector.h \
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h test/encoding/types.h \
- test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreDiff.h \
- test/filestore/FileStoreTracker.h \
- test/filestore/TestFileStoreState.h \
- test/filestore/workload_generator.h test/kv_store_bench.h \
- test/librados/test.h test/ObjectMap/KeyValueDBMemory.h \
- test/omap_bench.h test/osd/ceph_erasure_code_benchmark.h \
+ test/objectstore/DeterministicOpSequence.h \
+ test/objectstore/FileStoreDiff.h \
+ test/objectstore/FileStoreTracker.h \
+ test/objectstore/TestObjectStoreState.h \
+ test/objectstore/workload_generator.h test/kv_store_bench.h \
+ test/librados/test.h test/librados/TestCase.h \
+ test/ObjectMap/KeyValueDBMemory.h test/omap_bench.h \
test/osdc/FakeWriteback.h test/osd/Object.h \
test/osd/RadosModel.h test/osd/TestOpStat.h \
test/system/cross_process_sem.h \
@@ -3555,49 +3648,52 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
bash_completion/ceph bash_completion/rados bash_completion/rbd \
bash_completion/radosgw-admin mount/canonicalize.c \
mount/mtab.c objclass/objclass.h
-bin_SCRIPTS = ceph ceph-run ceph-rest-api ceph-clsinfo ceph-debugpack \
- ceph-rbdnamer ceph-post-file ceph-crush-location ceph-coverage
+bin_SCRIPTS = brag/client/ceph-brag ceph ceph-run ceph-rest-api \
+ ceph-clsinfo ceph-debugpack ceph-rbdnamer ceph-post-file \
+ ceph-crush-location ceph-coverage
sbin_SCRIPTS =
su_sbin_SCRIPTS = mount.fuse.ceph mkcephfs
dist_bin_SCRIPTS =
-lib_LTLIBRARIES = librados.la librbd.la libcephfs.la $(am__append_63)
+lib_LTLIBRARIES = librados.la librbd.la libcephfs.la $(am__append_65)
noinst_LTLIBRARIES = libarch.la libauth.la libcrush.la libmon.la \
- libmds.la libos.la libosd.la libosdc.la libclient.la \
- $(am__append_19) libglobal.la libjson_spirit.la liblog.la \
- libperfglue.la libcommon_crc.la libcommon.la libmsg.la \
- $(am__append_32) libcls_lock_client.la \
+ libmds.la libos.la libosd.la liberasure_code.la libosdc.la \
+ libclient.la $(am__append_21) libglobal.la libjson_spirit.la \
+ liblog.la libperfglue.la libcommon_crc.la libcommon.la \
+ libmsg.la $(am__append_34) libcls_lock_client.la \
libcls_refcount_client.la libcls_rgw_client.la \
- libcls_rbd_client.la $(am__append_43)
-noinst_LIBRARIES = $(am__append_16) libcls_version_client.a \
+ libcls_rbd_client.la $(am__append_50) libradostest.la
+noinst_LIBRARIES = $(am__append_17) libcls_version_client.a \
libcls_log_client.a libcls_statelog_client.a \
libcls_replica_log_client.a libcls_user_client.a
radoslib_LTLIBRARIES = libcls_hello.la libcls_rbd.la libcls_lock.la \
libcls_refcount.la libcls_version.la libcls_log.la \
libcls_statelog.la libcls_replica_log.la libcls_user.la \
- libcls_rgw.la $(am__append_37)
+ libcls_rgw.la $(am__append_39)
# like bin_PROGRAMS, but these targets are only built for debug builds
-bin_DEBUGPROGRAMS = ceph_test_ioctls $(am__append_35) ceph_test_timers \
+bin_DEBUGPROGRAMS = ceph_test_ioctls $(am__append_37) \
+ ceph_erasure_code_benchmark ceph_erasure_code ceph_test_timers \
ceph_test_signal_handlers ceph_test_rados ceph_test_mutate \
ceph_test_rewrite_latency ceph_test_msgr ceph_streamtest \
ceph_test_trans ceph_test_crypto ceph_test_keys \
- $(am__append_40) ceph_smalliobench ceph_smalliobenchfs \
+ $(am__append_48) ceph_smalliobench ceph_smalliobenchfs \
ceph_smalliobenchdumb ceph_smalliobenchrbd ceph_tpbench \
- ceph_omapbench $(am__append_41) ceph_multi_stress_watch \
- ceph_erasure_code_benchmark $(am__append_44) ceph_bench_log \
- $(am__append_49) ceph_test_librbd $(am__append_50) \
- ceph_test_cls_rbd ceph_test_cls_refcount ceph_test_cls_version \
- ceph_test_cls_log ceph_test_cls_statelog \
+ ceph_omapbench $(am__append_49) ceph_bench_log \
+ $(am__append_52) ceph_multi_stress_watch ceph_test_librbd \
+ $(am__append_53) ceph_test_cls_rbd ceph_test_cls_refcount \
+ ceph_test_cls_version ceph_test_cls_log ceph_test_cls_statelog \
ceph_test_cls_replica_log ceph_test_cls_lock \
- ceph_test_cls_hello $(am__append_51) ceph_test_mon_workloadgen \
+ ceph_test_cls_hello $(am__append_54) ceph_test_mon_workloadgen \
ceph_test_rados_api_cmd ceph_test_rados_api_io \
- ceph_test_rados_api_c_write_operations ceph_test_rados_api_aio \
+ ceph_test_rados_api_c_write_operations \
+ ceph_test_rados_api_c_read_operations ceph_test_rados_api_aio \
ceph_test_rados_api_list ceph_test_rados_api_pool \
ceph_test_rados_api_stat ceph_test_rados_api_watch_notify \
ceph_test_rados_api_snapshots ceph_test_rados_api_cls \
ceph_test_rados_api_misc ceph_test_rados_api_tier \
- ceph_test_rados_api_lock ceph_test_libcephfs $(am__append_52) \
- ceph_test_filestore_workloadgen ceph_test_filestore_idempotent \
+ ceph_test_rados_api_lock ceph_test_libcephfs $(am__append_55) \
+ ceph_test_objectstore_workloadgen \
+ ceph_test_filestore_idempotent \
ceph_test_filestore_idempotent_sequence ceph_xattr_bench \
ceph_test_filejournal ceph_test_stress_watch \
ceph_test_objectcacher_stress ceph_test_snap_mapper \
@@ -3615,9 +3711,10 @@ ceph_sbindir = $(sbindir)
su_sbindir = /sbin
# tests scripts will be appended to this
-check_SCRIPTS = unittest_bufferlist.sh \
- test/encoding/check-generated.sh test/mon/osd-pool-create.sh \
- test/mon/mkfs.sh test/ceph-disk.sh \
+check_SCRIPTS = test/erasure-code/test-erasure-code.sh \
+ unittest_bufferlist.sh test/encoding/check-generated.sh \
+ test/mon/osd-pool-create.sh test/mon/misc.sh \
+ test/mon/osd-crush.sh test/mon/mkfs.sh test/ceph-disk.sh \
test/mon/mon-handle-forward.sh test/vstart_wrapped_tests.sh \
test/pybind/test_ceph_argparse.py
@@ -3643,7 +3740,7 @@ AM_COMMON_CFLAGS = \
-fno-strict-aliasing \
-fsigned-char
-AM_CFLAGS = $(AM_COMMON_CFLAGS) $(am__append_5) $(am__append_22)
+AM_CFLAGS = $(AM_COMMON_CFLAGS) $(am__append_5) $(am__append_24)
AM_CPPFLAGS = $(AM_COMMON_CPPFLAGS)
# note: this is position dependant, it affects the -l options that
@@ -3686,6 +3783,7 @@ LIBRADOS = librados.la
LIBRGW = librgw.la
LIBRBD = librbd.la
LIBCEPHFS = libcephfs.la
+LIBERASURE_CODE = liberasure_code.la
# Use this for binaries requiring libglobal
CEPH_GLOBAL = $(LIBGLOBAL) $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS)
@@ -3695,12 +3793,12 @@ CEPH_GLOBAL = $(LIBGLOBAL) $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXT
# important; libmsg before libauth!
LIBCOMMON_DEPS = libcommon_crc.la $(LIBMSG) $(LIBAUTH) $(LIBCRUSH) \
$(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) $(KEYUTILS_LIB) \
- $(am__append_29)
+ $(am__append_31)
LIBRADOS_DEPS = libcls_lock_client.la $(LIBOSDC) $(LIBCOMMON)
-LIBRGW_DEPS = $(am__append_33)
+LIBRGW_DEPS = $(am__append_35)
# This is used by the dencoder test
-DENCODER_SOURCES = $(am__append_36)
+DENCODER_SOURCES = $(am__append_38)
DENCODER_DEPS = libcls_lock_client.la libcls_refcount_client.la \
libcls_replica_log_client.a libcls_rgw_client.la \
libcls_user_client.a
@@ -3754,7 +3852,7 @@ libmon_la_SOURCES = \
mon/DataHealthService.cc \
mon/ConfigKeyService.cc
-libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS)
+libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS) $(LIBERASURE_CODE)
libmds_la_SOURCES = \
mds/Anchor.cc \
mds/Capability.cc \
@@ -3793,35 +3891,18 @@ libos_la_SOURCES = os/chain_xattr.cc os/DBObjectMap.cc \
os/IndexManager.cc os/JournalingObjectStore.cc \
os/LevelDBStore.cc os/LFNIndex.cc os/MemStore.cc \
os/KeyValueStore.cc os/ObjectStore.cc os/WBThrottle.cc \
- common/TrackedOp.cc $(am__append_14) $(am__append_15)
+ common/TrackedOp.cc $(am__append_14) $(am__append_15) \
+ $(am__append_16)
@WITH_LIBZFS_TRUE at libos_zfs_a_SOURCES = os/ZFS.cc
@WITH_LIBZFS_TRUE at libos_zfs_a_CXXFLAGS = ${AM_CXXFLAGS} ${LIBZFS_CFLAGS}
-erasure_codelibdir = $(pkglibdir)/erasure-code
-erasure_codelib_LTLIBRARIES = libec_jerasure.la libec_example.la \
- libec_missing_entry_point.la libec_hangs.la \
- libec_fail_to_initialize.la libec_fail_to_register.la
-
-# jerasure plugin
-libec_jerasure_la_SOURCES = \
- osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc \
- osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc \
- osd/ErasureCodePluginJerasure/cauchy.c \
- osd/ErasureCodePluginJerasure/galois.c \
- osd/ErasureCodePluginJerasure/jerasure.c \
- osd/ErasureCodePluginJerasure/liberation.c \
- osd/ErasureCodePluginJerasure/reed_sol.c
-
-libec_jerasure_la_CFLAGS = ${AM_CFLAGS}
-libec_jerasure_la_CXXFLAGS = ${AM_CXXFLAGS}
-libec_jerasure_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 \
- $(am__append_18)
libosd_la_SOURCES = \
- osd/ErasureCodePlugin.cc \
osd/PG.cc \
osd/PGLog.cc \
osd/ReplicatedPG.cc \
osd/ReplicatedBackend.cc \
+ osd/ECBackend.cc \
+ osd/ECMsgTypes.cc \
+ osd/ECTransaction.cc \
osd/PGBackend.cc \
osd/Ager.cc \
osd/HitSet.cc \
@@ -3833,9 +3914,34 @@ libosd_la_SOURCES = \
common/TrackedOp.cc \
osd/SnapMapper.cc \
osd/osd_types.cc \
+ osd/ECUtil.cc \
objclass/class_api.cc
-libosd_la_LIBADD = $(LIBOSDC) $(LIBOS)
+libosd_la_LIBADD = $(LIBOSDC) $(LIBOS) $(LIBERASURE_CODE)
+erasure_codelibdir = $(pkglibdir)/erasure-code
+erasure_codelib_LTLIBRARIES = libec_jerasure.la libec_example.la \
+ libec_missing_entry_point.la libec_hangs.la \
+ libec_fail_to_initialize.la libec_fail_to_register.la
+
+# jerasure plugin
+libec_jerasure_la_SOURCES = \
+ erasure-code/jerasure/ErasureCodePluginJerasure.cc \
+ erasure-code/jerasure/ErasureCodeJerasure.cc \
+ erasure-code/jerasure/cauchy.c \
+ erasure-code/jerasure/galois.c \
+ erasure-code/jerasure/jerasure.c \
+ erasure-code/jerasure/liberation.c \
+ erasure-code/jerasure/reed_sol.c
+
+libec_jerasure_la_CFLAGS = ${AM_CFLAGS}
+libec_jerasure_la_CXXFLAGS = ${AM_CXXFLAGS}
+libec_jerasure_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 \
+ $(am__append_19)
+liberasure_code_la_SOURCES = \
+ erasure-code/ErasureCodePlugin.cc
+
+liberasure_code_la_LIBADD = $(LIBOSDC) $(LIBOS) $(am__append_20)
libosdc_la_SOURCES = \
osdc/Objecter.cc \
osdc/ObjectCacher.cc \
@@ -3871,8 +3977,8 @@ liblog_la_SOURCES = \
log/Log.cc \
log/SubsystemMap.cc
-libperfglue_la_SOURCES = $(am__append_21) $(am__append_24) \
- $(am__append_25) $(am__append_26)
+libperfglue_la_SOURCES = $(am__append_23) $(am__append_26) \
+ $(am__append_27) $(am__append_28)
@WITH_TCMALLOC_TRUE at libperfglue_la_LIBADD = -ltcmalloc
# these should go out of libcommon
@@ -3891,25 +3997,25 @@ libcommon_la_SOURCES = ceph_ver.c common/DecayCounter.cc \
common/errno.cc common/RefCountedObj.cc common/blkdev.cc \
common/common_init.cc common/pipe.c common/ceph_argparse.cc \
common/ceph_context.cc common/buffer.cc \
- common/code_environment.cc common/dout.cc common/signal.cc \
- common/simple_spin.cc common/Thread.cc common/Formatter.cc \
- common/HeartbeatMap.cc common/config.cc common/utf8.c \
- common/mime.c common/strtol.cc common/page.cc \
+ common/code_environment.cc common/dout.cc common/histogram.cc \
+ common/signal.cc common/simple_spin.cc common/Thread.cc \
+ common/Formatter.cc common/HeartbeatMap.cc common/config.cc \
+ common/utf8.c common/mime.c common/strtol.cc common/page.cc \
common/lockdep.cc common/version.cc common/hex.cc \
common/entity_name.cc common/ceph_crypto.cc \
common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \
common/pick_address.cc common/util.cc common/TextTable.cc \
common/ceph_fs.cc common/ceph_hash.cc common/ceph_strings.cc \
common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \
- common/bloom_filter.cc common/linux_version.c $(am__append_27) \
+ common/bloom_filter.cc common/linux_version.c $(am__append_29) \
mon/MonCap.cc mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc \
- osd/osd_types.cc osd/HitSet.cc mds/MDSMap.cc \
+ osd/osd_types.cc osd/ECMsgTypes.cc osd/HitSet.cc mds/MDSMap.cc \
mds/inode_backtrace.cc mds/mdstypes.cc
# inject crc in common
libcommon_crc_la_SOURCES = common/sctp_crc32.c common/crc32c.cc \
common/crc32c_intel_baseline.c common/crc32c_intel_fast.c \
- $(am__append_28)
+ $(am__append_30)
@WITH_GOOD_YASM_ELF64_TRUE at libcommon_crc_la_LIBTOOLFLAGS = --tag=CC
libcommon_la_LIBADD = $(LIBCOMMON_DEPS)
libmsg_la_SOURCES = \
@@ -3951,7 +4057,7 @@ librados_la_SOURCES = \
librados_la_CXXFLAGS = ${AM_CXXFLAGS}
librados_la_LIBADD = $(LIBRADOS_DEPS) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS)
librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \
- $(am__append_30)
+ $(am__append_32)
librbd_la_SOURCES = \
librbd/librbd.cc \
librbd/AioCompletion.cc \
@@ -3966,7 +4072,7 @@ librbd_la_LIBADD = \
libcls_rbd_client.la libcls_lock_client.la \
$(PTHREAD_LIBS) $(EXTRALIBS)
-librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 $(am__append_31)
+librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 $(am__append_33)
@WITH_RADOSGW_TRUE at librgw_la_SOURCES = \
@WITH_RADOSGW_TRUE@ rgw/librgw.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_acl.cc \
@@ -4110,6 +4216,61 @@ libcls_user_client_a_SOURCES = cls/user/cls_user_client.cc \
@LINUX_TRUE at libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc
@LINUX_TRUE at libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
@LINUX_TRUE at libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
+ceph_erasure_code_benchmark_SOURCES = \
+ test/erasure-code/ceph_erasure_code_benchmark.cc
+
+ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) \
+ $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) $(am__append_40)
+ceph_erasure_code_SOURCES = \
+ test/erasure-code/ceph_erasure_code.cc
+
+ceph_erasure_code_LDADD = $(LIBOSD) $(LIBCOMMON) \
+ $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) $(am__append_41)
+libec_example_la_SOURCES = test/erasure-code/ErasureCodePluginExample.cc
+libec_example_la_CFLAGS = ${AM_CFLAGS}
+libec_example_la_CXXFLAGS = ${AM_CXXFLAGS}
+libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+libec_missing_entry_point_la_SOURCES = test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
+libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
+libec_missing_entry_point_la_CXXFLAGS = ${AM_CXXFLAGS}
+libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+libec_hangs_la_SOURCES = test/erasure-code/ErasureCodePluginHangs.cc
+libec_hangs_la_CFLAGS = ${AM_CFLAGS}
+libec_hangs_la_CXXFLAGS = ${AM_CXXFLAGS}
+libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+libec_fail_to_initialize_la_SOURCES = test/erasure-code/ErasureCodePluginFailToInitialize.cc
+libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_initialize_la_CXXFLAGS = ${AM_CXXFLAGS}
+libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+libec_fail_to_register_la_SOURCES = test/erasure-code/ErasureCodePluginFailToRegister.cc
+libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_register_la_CXXFLAGS = ${AM_CXXFLAGS}
+libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+unittest_erasure_code_plugin_SOURCES = test/erasure-code/TestErasureCodePlugin.cc
+unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) \
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_42)
+unittest_erasure_code_jerasure_SOURCES = \
+ test/erasure-code/TestErasureCodeJerasure.cc \
+ $(libec_jerasure_la_SOURCES)
+
+unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_43)
+unittest_erasure_code_plugin_jerasure_SOURCES = \
+ test/erasure-code/TestErasureCodePluginJerasure.cc
+
+unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_44)
+unittest_erasure_code_example_SOURCES = test/erasure-code/TestErasureCodeExample.cc
+unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_timers_SOURCES = test/TestTimers.cc
ceph_test_timers_LDADD = $(CEPH_GLOBAL)
ceph_test_signal_handlers_SOURCES = test/TestSignalHandlers.cc
@@ -4139,14 +4300,12 @@ ceph_dencoder_SOURCES = \
test/encoding/ceph_dencoder.cc \
$(DENCODER_SOURCES)
-ceph_dencoder_LDADD = \
- $(LIBOSD) $(LIBMDS) $(LIBMON) \
- $(DENCODER_DEPS) $(CEPH_GLOBAL)
-
+ceph_dencoder_LDADD = $(LIBOSD) $(LIBMDS) $(LIBMON) $(DENCODER_DEPS) \
+ $(CEPH_GLOBAL) $(am__append_45)
# These should always use explicit _CFLAGS/_CXXFLAGS so avoid basename conflicts
-ceph_dencoder_CFLAGS = ${AM_CFLAGS} $(am__append_38)
-ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} $(am__append_39)
+ceph_dencoder_CFLAGS = ${AM_CFLAGS} $(am__append_46)
+ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} $(am__append_47)
get_command_descriptions_SOURCES = test/common/get_command_descriptions.cc
get_command_descriptions_LDADD = $(LIBMON) $(LIBCOMMON) $(CEPH_GLOBAL)
@@ -4238,16 +4397,6 @@ ceph_omapbench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
@LINUX_TRUE@ key_value_store/kv_flat_btree_async.cc
@LINUX_TRUE at ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
-ceph_multi_stress_watch_SOURCES = \
- test/multi_stress_watch.cc \
- test/librados/test.cc
-
-ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
-ceph_erasure_code_benchmark_SOURCES = \
- test/osd/ceph_erasure_code_benchmark.cc
-
-ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) $(am__append_42)
@LINUX_TRUE at libsystest_la_SOURCES = \
@LINUX_TRUE@ test/system/cross_process_sem.cc \
@LINUX_TRUE@ test/system/systest_runnable.cc \
@@ -4302,12 +4451,12 @@ unittest_addrs_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_bloom_filter_SOURCES = test/common/test_bloom_filter.cc
unittest_bloom_filter_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_bloom_filter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_histogram_SOURCES = test/common/histogram.cc
+unittest_histogram_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_histogram_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_str_map_SOURCES = test/common/test_str_map.cc
unittest_str_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_str_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-unittest_crushwrapper_SOURCES = test/test_crushwrapper.cc
-unittest_crushwrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_crushwrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc
unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -4353,58 +4502,16 @@ unittest_ceph_argparse_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_ceph_compatset_SOURCES = test/ceph_compatset.cc
unittest_ceph_compatset_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_ceph_compatset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-libec_example_la_SOURCES = test/osd/ErasureCodePluginExample.cc
-libec_example_la_CFLAGS = ${AM_CFLAGS}
-libec_example_la_CXXFLAGS = ${AM_CXXFLAGS}
-libec_example_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-libec_missing_entry_point_la_SOURCES = test/osd/ErasureCodePluginMissingEntryPoint.cc
-libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
-libec_missing_entry_point_la_CXXFLAGS = ${AM_CXXFLAGS}
-libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-libec_hangs_la_SOURCES = test/osd/ErasureCodePluginHangs.cc
-libec_hangs_la_CFLAGS = ${AM_CFLAGS}
-libec_hangs_la_CXXFLAGS = ${AM_CXXFLAGS}
-libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-libec_fail_to_initialize_la_SOURCES = test/osd/ErasureCodePluginFailToInitialize.cc
-libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
-libec_fail_to_initialize_la_CXXFLAGS = ${AM_CXXFLAGS}
-libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-libec_fail_to_register_la_SOURCES = test/osd/ErasureCodePluginFailToRegister.cc
-libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
-libec_fail_to_register_la_CXXFLAGS = ${AM_CXXFLAGS}
-libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-unittest_erasure_code_plugin_SOURCES = test/osd/TestErasureCodePlugin.cc
-unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_45)
-unittest_erasure_code_jerasure_SOURCES = \
- test/osd/TestErasureCodeJerasure.cc \
- $(libec_jerasure_la_SOURCES)
-
-unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_46)
-unittest_erasure_code_plugin_jerasure_SOURCES = \
- test/osd/TestErasureCodePluginJerasure.cc
-
-unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
-unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_47)
-unittest_erasure_code_example_SOURCES = test/osd/TestErasureCodeExample.cc
-unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-unittest_osd_types_SOURCES = test/test_osd_types.cc
+unittest_osd_types_SOURCES = test/osd/types.cc
unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_pglog_SOURCES = test/osd/TestPGLog.cc
unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) \
- $(am__append_48)
+ $(am__append_51)
+unittest_ecbackend_SOURCES = test/osd/TestECBackend.cc
+unittest_ecbackend_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_ecbackend_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_hitset_SOURCES = test/osd/hitset.cc
unittest_hitset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_hitset_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -4453,7 +4560,7 @@ unittest_mime_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_escape_SOURCES = test/escape.cc
unittest_escape_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_escape_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_chain_xattr_SOURCES = test/filestore/chain_xattr.cc
+unittest_chain_xattr_SOURCES = test/objectstore/chain_xattr.cc
unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc
@@ -4468,6 +4575,9 @@ unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_config_SOURCES = test/common/test_config.cc
unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_context_SOURCES = test/common/test_context.cc
+unittest_context_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_context_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc
unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -4503,6 +4613,9 @@ unittest_osd_osdcap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_mon_moncap_SOURCES = test/mon/moncap.cc
unittest_mon_moncap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_mon_moncap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_mon_pgmap_SOURCES = test/mon/PGMap.cc
+unittest_mon_pgmap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_mon_pgmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
#if WITH_RADOSGW
#unittest_librgw_SOURCES = test/librgw.cc
@@ -4526,6 +4639,13 @@ unittest_on_exit_LDADD = $(PTHREAD_LIBS)
@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat
@WITH_RADOSGW_TRUE at ceph_test_cors_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+ at WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_SOURCES = test/rgw/test_rgw_manifest.cc
+ at WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_LDADD = \
+ at WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) \
+ at WITH_RADOSGW_TRUE@ $(UNITTEST_LDADD) $(CRYPTO_LIBS) \
+ at WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat
+
+ at WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_meta_LDADD = \
@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
@@ -4556,155 +4676,103 @@ unittest_on_exit_LDADD = $(PTHREAD_LIBS)
@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_librbd_SOURCES = \
- test/librbd/test_librbd.cc \
- test/librados/test.cc
-
-ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+libradostest_la_SOURCES = \
+ test/librados/test.cc \
+ test/librados/TestCase.cc
+
+libradostest_la_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+RADOS_TEST_LDADD = libradostest.la
+ceph_multi_stress_watch_SOURCES = test/multi_stress_watch.cc
+ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_librbd_SOURCES = test/librbd/test_librbd.cc
+ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@LINUX_TRUE at ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c
@LINUX_TRUE at ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm
@LINUX_TRUE at ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format
-ceph_test_cls_rbd_SOURCES = \
- test/cls_rbd/test_cls_rbd.cc \
- test/librados/test.cc
-
-ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD)
+ceph_test_cls_rbd_SOURCES = test/cls_rbd/test_cls_rbd.cc
+ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_rbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_refcount_SOURCES = \
- test/cls_refcount/test_cls_refcount.cc \
- test/librados/test.cc
-
-ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD)
+ceph_test_cls_refcount_SOURCES = test/cls_refcount/test_cls_refcount.cc
+ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_refcount_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_version_SOURCES = \
- test/cls_version/test_cls_version.cc \
- test/librados/test.cc
-
-ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD)
+ceph_test_cls_version_SOURCES = test/cls_version/test_cls_version.cc
+ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_version_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_log_SOURCES = \
- test/cls_log/test_cls_log.cc \
- test/librados/test.cc
-
-ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_cls_log_SOURCES = test/cls_log/test_cls_log.cc
+ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_statelog_SOURCES = \
- test/cls_statelog/test_cls_statelog.cc \
- test/librados/test.cc
-
-ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_cls_statelog_SOURCES = test/cls_statelog/test_cls_statelog.cc
+ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_statelog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_replica_log_SOURCES = \
- test/cls_replica_log/test_cls_replica_log.cc \
- test/librados/test.cc
-
+ceph_test_cls_replica_log_SOURCES = test/cls_replica_log/test_cls_replica_log.cc
ceph_test_cls_replica_log_LDADD = \
$(LIBRADOS) libcls_replica_log_client.a \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_replica_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_lock_SOURCES = \
- test/cls_lock/test_cls_lock.cc \
- test/librados/test.cc
-
-ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD)
+ceph_test_cls_lock_SOURCES = test/cls_lock/test_cls_lock.cc
+ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_cls_hello_SOURCES = \
- test/cls_hello/test_cls_hello.cc \
- test/librados/test.cc
-
+ceph_test_cls_hello_SOURCES = test/cls_hello/test_cls_hello.cc
ceph_test_cls_hello_LDADD = \
$(LIBRADOS) $(CRYPTO_LIBS) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_hello_CXXFLAGS = $(UNITTEST_CXXFLAGS)
- at WITH_RADOSGW_TRUE@ceph_test_cls_rgw_SOURCES = \
- at WITH_RADOSGW_TRUE@ test/cls_rgw/test_cls_rgw.cc \
- at WITH_RADOSGW_TRUE@ test/librados/test.cc
-
- at WITH_RADOSGW_TRUE@ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD)
+ at WITH_RADOSGW_TRUE@ceph_test_cls_rgw_SOURCES = test/cls_rgw/test_cls_rgw.cc
+ at WITH_RADOSGW_TRUE@ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_mon_workloadgen_SOURCES = test/mon/test_mon_workloadgen.cc
ceph_test_mon_workloadgen_LDADD = $(LIBOS) $(LIBOSDC) $(CEPH_GLOBAL)
-ceph_test_rados_api_cmd_SOURCES = \
- test/librados/cmd.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_cmd_SOURCES = test/librados/cmd.cc
+ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_io_SOURCES = \
- test/librados/io.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_io_SOURCES = test/librados/io.cc
+ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_rados_api_c_write_operations_SOURCES = \
- test/librados/c_write_operations.cc \
- test/librados/test.cc
+ test/librados/c_write_operations.cc
-ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_c_write_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_aio_SOURCES = \
- test/librados/aio.cc \
- test/librados/test.cc
+ceph_test_rados_api_c_read_operations_SOURCES = \
+ test/librados/c_read_operations.cc
-ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_c_read_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_c_read_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+ceph_test_rados_api_aio_SOURCES = test/librados/aio.cc
+ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_aio_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_list_SOURCES = \
- test/librados/list.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_list_SOURCES = test/librados/list.cc
+ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_list_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_pool_SOURCES = \
- test/librados/pool.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_pool_SOURCES = test/librados/pool.cc
+ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_pool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_stat_SOURCES = \
- test/librados/stat.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_stat_SOURCES = test/librados/stat.cc
+ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_stat_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_watch_notify_SOURCES = \
- test/librados/watch_notify.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_watch_notify_SOURCES = test/librados/watch_notify.cc
+ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_watch_notify_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_snapshots_SOURCES = \
- test/librados/snapshots.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_snapshots_SOURCES = test/librados/snapshots.cc
+ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_snapshots_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_cls_SOURCES = \
- test/librados/cls.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_cls_SOURCES = test/librados/cls.cc
+ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_cls_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_misc_SOURCES = \
- test/librados/misc.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_rados_api_misc_SOURCES = test/librados/misc.cc
+ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_rados_api_tier_SOURCES = \
test/librados/tier.cc \
- test/librados/test.cc \
osd/HitSet.cc
-ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_rados_api_tier_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_rados_api_lock_SOURCES = \
- test/librados/lock.cc \
- test/librados/test.cc
-
-ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_lock_SOURCES = test/librados/lock.cc
+ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_libcephfs_SOURCES = \
test/libcephfs/test.cc \
@@ -4714,25 +4782,25 @@ ceph_test_libcephfs_SOURCES = \
ceph_test_libcephfs_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS)
- at LINUX_TRUE@ceph_test_filestore_SOURCES = test/filestore/store_test.cc
- at LINUX_TRUE@ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
- at LINUX_TRUE@ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_filestore_workloadgen_SOURCES = \
- test/filestore/workload_generator.cc \
- test/filestore/TestFileStoreState.cc
-
-ceph_test_filestore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+ at LINUX_TRUE@ceph_test_objectstore_SOURCES = test/objectstore/store_test.cc
+ at LINUX_TRUE@ceph_test_objectstore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ at LINUX_TRUE@ceph_test_objectstore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+ceph_test_objectstore_workloadgen_SOURCES = \
+ test/objectstore/workload_generator.cc \
+ test/objectstore/TestObjectStoreState.cc
+
+ceph_test_objectstore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
ceph_test_filestore_idempotent_SOURCES = \
- test/filestore/test_idempotent.cc \
- test/filestore/FileStoreTracker.cc \
+ test/objectstore/test_idempotent.cc \
+ test/objectstore/FileStoreTracker.cc \
test/common/ObjectContents.cc
ceph_test_filestore_idempotent_LDADD = $(LIBOS) $(CEPH_GLOBAL)
ceph_test_filestore_idempotent_sequence_SOURCES = \
- test/filestore/test_idempotent_sequence.cc \
- test/filestore/DeterministicOpSequence.cc \
- test/filestore/TestFileStoreState.cc \
- test/filestore/FileStoreDiff.cc
+ test/objectstore/test_idempotent_sequence.cc \
+ test/objectstore/DeterministicOpSequence.cc \
+ test/objectstore/TestObjectStoreState.cc \
+ test/objectstore/FileStoreDiff.cc
ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS) $(CEPH_GLOBAL)
ceph_xattr_bench_SOURCES = test/xattr_bench.cc
@@ -4741,11 +4809,8 @@ ceph_xattr_bench_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_filejournal_SOURCES = test/test_filejournal.cc
ceph_test_filejournal_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_filejournal_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_stress_watch_SOURCES = \
- test/test_stress_watch.cc \
- test/librados/test.cc
-
-ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_stress_watch_SOURCES = test/test_stress_watch.cc
+ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_stress_watch_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_objectcacher_stress_SOURCES = \
test/osdc/object_cacher_stress.cc \
@@ -4784,10 +4849,10 @@ ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL)
ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_filestore_tool_SOURCES = tools/ceph-filestore-tool.cc
ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) \
- -lboost_program_options $(am__append_53)
+ -lboost_program_options $(am__append_56)
ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) \
- $(BOOST_PROGRAM_OPTIONS_LIBS) $(am__append_54)
+ $(BOOST_PROGRAM_OPTIONS_LIBS) $(am__append_57)
monmaptool_SOURCES = tools/monmaptool.cc
monmaptool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
crushtool_SOURCES = tools/crushtool.cc
@@ -4813,7 +4878,7 @@ rados_LDADD = libcls_lock_client.la $(LIBRADOS) $(CEPH_GLOBAL)
@WITH_REST_BENCH_TRUE@ common/obj_bencher.cc # needs cleanup so \
@WITH_REST_BENCH_TRUE@ it can go in libcommon.la
@WITH_REST_BENCH_TRUE at rest_bench_LDADD = $(CEPH_GLOBAL) \
- at WITH_REST_BENCH_TRUE@ $(am__append_56) $(am__append_57)
+ at WITH_REST_BENCH_TRUE@ $(am__append_59) $(am__append_60)
@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at rest_bench_CXXFLAGS = ${AM_CXXFLAGS} -I$(top_srcdir)/src/libs3/inc
ceph_conf_SOURCES = tools/ceph_conf.cc
ceph_conf_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
@@ -4828,8 +4893,7 @@ ceph_mon_store_converter_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL)
ceph_mon_SOURCES = ceph_mon.cc
ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) $(LIBCOMMON)
ceph_osd_SOURCES = ceph_osd.cc
-ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON) \
- $(am__append_59)
+ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON)
ceph_mds_SOURCES = ceph_mds.cc
ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL) $(LIBCOMMON)
@@ -4926,7 +4990,7 @@ all: $(BUILT_SOURCES) acconfig.h
.SUFFIXES:
.SUFFIXES: .S .c .cc .cpp .lo .o .obj
-$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/Makefile-env.am $(srcdir)/arch/Makefile.am $(srcdir)/auth/Makefile.am $(srcdir)/crush/Makefile.am $(srcdir)/mon/Makefile.am $(srcdir)/mds/Makefile.am $(srcdir)/os/Makefile.am $(srcdir)/osd/Makefile.am $(srcdir)/osd/ErasureCodePluginJerasure/Makefile.am $(srcdir)/osdc/Makefile.am $(srcdir)/client/Makefile.am $(srcdir)/global/Makefile.am $(srcdir)/json_spirit/Makefile.am $(srcdir)/log/Makefile.am $(srcdir)/perfglue/Makefile.am $(srcdi [...]
+$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/Makefile-env.am $(srcdir)/arch/Makefile.am $(srcdir)/auth/Makefile.am $(srcdir)/brag/Makefile.am $(srcdir)/crush/Makefile.am $(srcdir)/mon/Makefile.am $(srcdir)/mds/Makefile.am $(srcdir)/os/Makefile.am $(srcdir)/osd/Makefile.am $(srcdir)/erasure-code/Makefile.am $(srcdir)/erasure-code/jerasure/Makefile.am $(srcdir)/osdc/Makefile.am $(srcdir)/client/Makefile.am $(srcdir)/global/Makefile.am $(srcdir)/json_spirit/Makefile.am $(srcdir)/ [...]
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
@@ -4947,7 +5011,7 @@ Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
esac;
-$(srcdir)/Makefile-env.am $(srcdir)/arch/Makefile.am $(srcdir)/auth/Makefile.am $(srcdir)/crush/Makefile.am $(srcdir)/mon/Makefile.am $(srcdir)/mds/Makefile.am $(srcdir)/os/Makefile.am $(srcdir)/osd/Makefile.am $(srcdir)/osd/ErasureCodePluginJerasure/Makefile.am $(srcdir)/osdc/Makefile.am $(srcdir)/client/Makefile.am $(srcdir)/global/Makefile.am $(srcdir)/json_spirit/Makefile.am $(srcdir)/log/Makefile.am $(srcdir)/perfglue/Makefile.am $(srcdir)/common/Makefile.am $(srcdir)/msg/Makefile.a [...]
+$(srcdir)/Makefile-env.am $(srcdir)/arch/Makefile.am $(srcdir)/auth/Makefile.am $(srcdir)/brag/Makefile.am $(srcdir)/crush/Makefile.am $(srcdir)/mon/Makefile.am $(srcdir)/mds/Makefile.am $(srcdir)/os/Makefile.am $(srcdir)/osd/Makefile.am $(srcdir)/erasure-code/Makefile.am $(srcdir)/erasure-code/jerasure/Makefile.am $(srcdir)/osdc/Makefile.am $(srcdir)/client/Makefile.am $(srcdir)/global/Makefile.am $(srcdir)/json_spirit/Makefile.am $(srcdir)/log/Makefile.am $(srcdir)/perfglue/Makefile.am [...]
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
@@ -5466,6 +5530,8 @@ common/code_environment.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/dout.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
+common/histogram.lo: common/$(am__dirstamp) \
+ common/$(DEPDIR)/$(am__dirstamp)
common/signal.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/simple_spin.lo: common/$(am__dirstamp) \
@@ -5540,6 +5606,7 @@ osd/$(DEPDIR)/$(am__dirstamp):
@: > osd/$(DEPDIR)/$(am__dirstamp)
osd/OSDMap.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/osd_types.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
+osd/ECMsgTypes.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/HitSet.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
mds/$(am__dirstamp):
@$(MKDIR_P) mds
@@ -5586,61 +5653,76 @@ crush/CrushTester.lo: crush/$(am__dirstamp) \
crush/$(DEPDIR)/$(am__dirstamp)
libcrush.la: $(libcrush_la_OBJECTS) $(libcrush_la_DEPENDENCIES) $(EXTRA_libcrush_la_DEPENDENCIES)
$(AM_V_CXXLD)$(CXXLINK) $(libcrush_la_OBJECTS) $(libcrush_la_LIBADD) $(LIBS)
-test/osd/$(am__dirstamp):
- @$(MKDIR_P) test/osd
- @: > test/osd/$(am__dirstamp)
-test/osd/$(DEPDIR)/$(am__dirstamp):
- @$(MKDIR_P) test/osd/$(DEPDIR)
- @: > test/osd/$(DEPDIR)/$(am__dirstamp)
-test/osd/libec_example_la-ErasureCodePluginExample.lo: \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/$(am__dirstamp):
+ @$(MKDIR_P) test/erasure-code
+ @: > test/erasure-code/$(am__dirstamp)
+test/erasure-code/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) test/erasure-code/$(DEPDIR)
+ @: > test/erasure-code/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/libec_example_la-ErasureCodePluginExample.lo: \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
libec_example.la: $(libec_example_la_OBJECTS) $(libec_example_la_DEPENDENCIES) $(EXTRA_libec_example_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libec_example_la_LINK) -rpath $(erasure_codelibdir) $(libec_example_la_OBJECTS) $(libec_example_la_LIBADD) $(LIBS)
-test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo: \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo: \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
libec_fail_to_initialize.la: $(libec_fail_to_initialize_la_OBJECTS) $(libec_fail_to_initialize_la_DEPENDENCIES) $(EXTRA_libec_fail_to_initialize_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libec_fail_to_initialize_la_LINK) -rpath $(erasure_codelibdir) $(libec_fail_to_initialize_la_OBJECTS) $(libec_fail_to_initialize_la_LIBADD) $(LIBS)
-test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo: \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo: \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
libec_fail_to_register.la: $(libec_fail_to_register_la_OBJECTS) $(libec_fail_to_register_la_DEPENDENCIES) $(EXTRA_libec_fail_to_register_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libec_fail_to_register_la_LINK) -rpath $(erasure_codelibdir) $(libec_fail_to_register_la_OBJECTS) $(libec_fail_to_register_la_LIBADD) $(LIBS)
-test/osd/libec_hangs_la-ErasureCodePluginHangs.lo: \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo: \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
libec_hangs.la: $(libec_hangs_la_OBJECTS) $(libec_hangs_la_DEPENDENCIES) $(EXTRA_libec_hangs_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libec_hangs_la_LINK) -rpath $(erasure_codelibdir) $(libec_hangs_la_OBJECTS) $(libec_hangs_la_LIBADD) $(LIBS)
-osd/ErasureCodePluginJerasure/$(am__dirstamp):
- @$(MKDIR_P) osd/ErasureCodePluginJerasure
- @: > osd/ErasureCodePluginJerasure/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp):
- @$(MKDIR_P) osd/ErasureCodePluginJerasure/$(DEPDIR)
- @: > osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo: \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/$(am__dirstamp):
+ @$(MKDIR_P) erasure-code/jerasure
+ @: > erasure-code/jerasure/$(am__dirstamp)
+erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) erasure-code/jerasure/$(DEPDIR)
+ @: > erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-cauchy.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-galois.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-jerasure.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-liberation.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/libec_jerasure_la-reed_sol.lo: \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
libec_jerasure.la: $(libec_jerasure_la_OBJECTS) $(libec_jerasure_la_DEPENDENCIES) $(EXTRA_libec_jerasure_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libec_jerasure_la_LINK) -rpath $(erasure_codelibdir) $(libec_jerasure_la_OBJECTS) $(libec_jerasure_la_LIBADD) $(LIBS)
-test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
libec_missing_entry_point.la: $(libec_missing_entry_point_la_OBJECTS) $(libec_missing_entry_point_la_DEPENDENCIES) $(EXTRA_libec_missing_entry_point_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libec_missing_entry_point_la_LINK) -rpath $(erasure_codelibdir) $(libec_missing_entry_point_la_OBJECTS) $(libec_missing_entry_point_la_LIBADD) $(LIBS)
+erasure-code/$(am__dirstamp):
+ @$(MKDIR_P) erasure-code
+ @: > erasure-code/$(am__dirstamp)
+erasure-code/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) erasure-code/$(DEPDIR)
+ @: > erasure-code/$(DEPDIR)/$(am__dirstamp)
+erasure-code/ErasureCodePlugin.lo: erasure-code/$(am__dirstamp) \
+ erasure-code/$(DEPDIR)/$(am__dirstamp)
+liberasure_code.la: $(liberasure_code_la_OBJECTS) $(liberasure_code_la_DEPENDENCIES) $(EXTRA_liberasure_code_la_DEPENDENCIES)
+ $(AM_V_CXXLD)$(CXXLINK) $(liberasure_code_la_OBJECTS) $(liberasure_code_la_LIBADD) $(LIBS)
global/$(am__dirstamp):
@$(MKDIR_P) global
@: > global/$(am__dirstamp)
@@ -5773,17 +5855,20 @@ common/TrackedOp.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
os/BtrfsFileStoreBackend.lo: os/$(am__dirstamp) \
os/$(DEPDIR)/$(am__dirstamp)
+os/XfsFileStoreBackend.lo: os/$(am__dirstamp) \
+ os/$(DEPDIR)/$(am__dirstamp)
os/ZFSFileStoreBackend.lo: os/$(am__dirstamp) \
os/$(DEPDIR)/$(am__dirstamp)
libos.la: $(libos_la_OBJECTS) $(libos_la_DEPENDENCIES) $(EXTRA_libos_la_DEPENDENCIES)
$(AM_V_CXXLD)$(CXXLINK) $(libos_la_OBJECTS) $(libos_la_LIBADD) $(LIBS)
-osd/ErasureCodePlugin.lo: osd/$(am__dirstamp) \
- osd/$(DEPDIR)/$(am__dirstamp)
osd/PG.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/PGLog.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/ReplicatedPG.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/ReplicatedBackend.lo: osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
+osd/ECBackend.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
+osd/ECTransaction.lo: osd/$(am__dirstamp) \
+ osd/$(DEPDIR)/$(am__dirstamp)
osd/PGBackend.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/Ager.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/OSD.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
@@ -5792,6 +5877,7 @@ osd/Watch.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/ClassHandler.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/OpRequest.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/SnapMapper.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
+osd/ECUtil.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
objclass/$(am__dirstamp):
@$(MKDIR_P) objclass
@: > objclass/$(am__dirstamp)
@@ -5848,6 +5934,19 @@ librados/librados_la-snap_set_diff.lo: librados/$(am__dirstamp) \
librados/$(DEPDIR)/$(am__dirstamp)
librados.la: $(librados_la_OBJECTS) $(librados_la_DEPENDENCIES) $(EXTRA_librados_la_DEPENDENCIES)
$(AM_V_CXXLD)$(librados_la_LINK) -rpath $(libdir) $(librados_la_OBJECTS) $(librados_la_LIBADD) $(LIBS)
+test/librados/$(am__dirstamp):
+ @$(MKDIR_P) test/librados
+ @: > test/librados/$(am__dirstamp)
+test/librados/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) test/librados/$(DEPDIR)
+ @: > test/librados/$(DEPDIR)/$(am__dirstamp)
+test/librados/libradostest_la-test.lo: test/librados/$(am__dirstamp) \
+ test/librados/$(DEPDIR)/$(am__dirstamp)
+test/librados/libradostest_la-TestCase.lo: \
+ test/librados/$(am__dirstamp) \
+ test/librados/$(DEPDIR)/$(am__dirstamp)
+libradostest.la: $(libradostest_la_OBJECTS) $(libradostest_la_DEPENDENCIES) $(EXTRA_libradostest_la_DEPENDENCIES)
+ $(AM_V_CXXLD)$(libradostest_la_LINK) $(libradostest_la_OBJECTS) $(libradostest_la_LIBADD) $(LIBS)
librbd/$(am__dirstamp):
@$(MKDIR_P) librbd
@: > librbd/$(am__dirstamp)
@@ -6191,8 +6290,15 @@ tools/dupstore.$(OBJEXT): tools/$(am__dirstamp) \
ceph_dupstore$(EXEEXT): $(ceph_dupstore_OBJECTS) $(ceph_dupstore_DEPENDENCIES) $(EXTRA_ceph_dupstore_DEPENDENCIES)
@rm -f ceph_dupstore$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_dupstore_OBJECTS) $(ceph_dupstore_LDADD) $(LIBS)
-test/osd/ceph_erasure_code_benchmark.$(OBJEXT): \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/ceph_erasure_code.$(OBJEXT): \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
+ceph_erasure_code$(EXEEXT): $(ceph_erasure_code_OBJECTS) $(ceph_erasure_code_DEPENDENCIES) $(EXTRA_ceph_erasure_code_DEPENDENCIES)
+ @rm -f ceph_erasure_code$(EXEEXT)
+ $(AM_V_CXXLD)$(CXXLINK) $(ceph_erasure_code_OBJECTS) $(ceph_erasure_code_LDADD) $(LIBS)
+test/erasure-code/ceph_erasure_code_benchmark.$(OBJEXT): \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
ceph_erasure_code_benchmark$(EXEEXT): $(ceph_erasure_code_benchmark_OBJECTS) $(ceph_erasure_code_benchmark_DEPENDENCIES) $(EXTRA_ceph_erasure_code_benchmark_DEPENDENCIES)
@rm -f ceph_erasure_code_benchmark$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_erasure_code_benchmark_OBJECTS) $(ceph_erasure_code_benchmark_LDADD) $(LIBS)
@@ -6221,14 +6327,6 @@ ceph_mon_store_converter$(EXEEXT): $(ceph_mon_store_converter_OBJECTS) $(ceph_mo
$(AM_V_CXXLD)$(CXXLINK) $(ceph_mon_store_converter_OBJECTS) $(ceph_mon_store_converter_LDADD) $(LIBS)
test/multi_stress_watch.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
-test/librados/$(am__dirstamp):
- @$(MKDIR_P) test/librados
- @: > test/librados/$(am__dirstamp)
-test/librados/$(DEPDIR)/$(am__dirstamp):
- @$(MKDIR_P) test/librados/$(DEPDIR)
- @: > test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/test.$(OBJEXT): test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_multi_stress_watch$(EXEEXT): $(ceph_multi_stress_watch_OBJECTS) $(ceph_multi_stress_watch_DEPENDENCIES) $(EXTRA_ceph_multi_stress_watch_DEPENDENCIES)
@rm -f ceph_multi_stress_watch$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_multi_stress_watch_OBJECTS) $(ceph_multi_stress_watch_LDADD) $(LIBS)
@@ -6337,9 +6435,6 @@ test/cls_hello/$(DEPDIR)/$(am__dirstamp):
test/cls_hello/ceph_test_cls_hello-test_cls_hello.$(OBJEXT): \
test/cls_hello/$(am__dirstamp) \
test/cls_hello/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_hello-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_hello$(EXEEXT): $(ceph_test_cls_hello_OBJECTS) $(ceph_test_cls_hello_DEPENDENCIES) $(EXTRA_ceph_test_cls_hello_DEPENDENCIES)
@rm -f ceph_test_cls_hello$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_hello_LINK) $(ceph_test_cls_hello_OBJECTS) $(ceph_test_cls_hello_LDADD) $(LIBS)
@@ -6352,9 +6447,6 @@ test/cls_lock/$(DEPDIR)/$(am__dirstamp):
test/cls_lock/ceph_test_cls_lock-test_cls_lock.$(OBJEXT): \
test/cls_lock/$(am__dirstamp) \
test/cls_lock/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_lock-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_lock$(EXEEXT): $(ceph_test_cls_lock_OBJECTS) $(ceph_test_cls_lock_DEPENDENCIES) $(EXTRA_ceph_test_cls_lock_DEPENDENCIES)
@rm -f ceph_test_cls_lock$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_lock_LINK) $(ceph_test_cls_lock_OBJECTS) $(ceph_test_cls_lock_LDADD) $(LIBS)
@@ -6367,9 +6459,6 @@ test/cls_log/$(DEPDIR)/$(am__dirstamp):
test/cls_log/ceph_test_cls_log-test_cls_log.$(OBJEXT): \
test/cls_log/$(am__dirstamp) \
test/cls_log/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_log-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_log$(EXEEXT): $(ceph_test_cls_log_OBJECTS) $(ceph_test_cls_log_DEPENDENCIES) $(EXTRA_ceph_test_cls_log_DEPENDENCIES)
@rm -f ceph_test_cls_log$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_log_LINK) $(ceph_test_cls_log_OBJECTS) $(ceph_test_cls_log_LDADD) $(LIBS)
@@ -6382,9 +6471,6 @@ test/cls_rbd/$(DEPDIR)/$(am__dirstamp):
test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.$(OBJEXT): \
test/cls_rbd/$(am__dirstamp) \
test/cls_rbd/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_rbd-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_rbd$(EXEEXT): $(ceph_test_cls_rbd_OBJECTS) $(ceph_test_cls_rbd_DEPENDENCIES) $(EXTRA_ceph_test_cls_rbd_DEPENDENCIES)
@rm -f ceph_test_cls_rbd$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_rbd_LINK) $(ceph_test_cls_rbd_OBJECTS) $(ceph_test_cls_rbd_LDADD) $(LIBS)
@@ -6397,9 +6483,6 @@ test/cls_refcount/$(DEPDIR)/$(am__dirstamp):
test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.$(OBJEXT): \
test/cls_refcount/$(am__dirstamp) \
test/cls_refcount/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_refcount-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_refcount$(EXEEXT): $(ceph_test_cls_refcount_OBJECTS) $(ceph_test_cls_refcount_DEPENDENCIES) $(EXTRA_ceph_test_cls_refcount_DEPENDENCIES)
@rm -f ceph_test_cls_refcount$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_refcount_LINK) $(ceph_test_cls_refcount_OBJECTS) $(ceph_test_cls_refcount_LDADD) $(LIBS)
@@ -6412,9 +6495,6 @@ test/cls_replica_log/$(DEPDIR)/$(am__dirstamp):
test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.$(OBJEXT): \
test/cls_replica_log/$(am__dirstamp) \
test/cls_replica_log/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_replica_log-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_replica_log$(EXEEXT): $(ceph_test_cls_replica_log_OBJECTS) $(ceph_test_cls_replica_log_DEPENDENCIES) $(EXTRA_ceph_test_cls_replica_log_DEPENDENCIES)
@rm -f ceph_test_cls_replica_log$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_replica_log_LINK) $(ceph_test_cls_replica_log_OBJECTS) $(ceph_test_cls_replica_log_LDADD) $(LIBS)
@@ -6427,9 +6507,6 @@ test/cls_rgw/$(DEPDIR)/$(am__dirstamp):
test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.$(OBJEXT): \
test/cls_rgw/$(am__dirstamp) \
test/cls_rgw/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_rgw-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_rgw$(EXEEXT): $(ceph_test_cls_rgw_OBJECTS) $(ceph_test_cls_rgw_DEPENDENCIES) $(EXTRA_ceph_test_cls_rgw_DEPENDENCIES)
@rm -f ceph_test_cls_rgw$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_rgw_LINK) $(ceph_test_cls_rgw_OBJECTS) $(ceph_test_cls_rgw_LDADD) $(LIBS)
@@ -6457,9 +6534,6 @@ test/cls_statelog/$(DEPDIR)/$(am__dirstamp):
test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.$(OBJEXT): \
test/cls_statelog/$(am__dirstamp) \
test/cls_statelog/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_statelog-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_statelog$(EXEEXT): $(ceph_test_cls_statelog_OBJECTS) $(ceph_test_cls_statelog_DEPENDENCIES) $(EXTRA_ceph_test_cls_statelog_DEPENDENCIES)
@rm -f ceph_test_cls_statelog$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_statelog_LINK) $(ceph_test_cls_statelog_OBJECTS) $(ceph_test_cls_statelog_LDADD) $(LIBS)
@@ -6472,9 +6546,6 @@ test/cls_version/$(DEPDIR)/$(am__dirstamp):
test/cls_version/ceph_test_cls_version-test_cls_version.$(OBJEXT): \
test/cls_version/$(am__dirstamp) \
test/cls_version/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_cls_version-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_cls_version$(EXEEXT): $(ceph_test_cls_version_OBJECTS) $(ceph_test_cls_version_DEPENDENCIES) $(EXTRA_ceph_test_cls_version_DEPENDENCIES)
@rm -f ceph_test_cls_version$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_cls_version_LINK) $(ceph_test_cls_version_OBJECTS) $(ceph_test_cls_version_LDADD) $(LIBS)
@@ -6493,24 +6564,18 @@ test/ceph_test_filejournal-test_filejournal.$(OBJEXT): \
ceph_test_filejournal$(EXEEXT): $(ceph_test_filejournal_OBJECTS) $(ceph_test_filejournal_DEPENDENCIES) $(EXTRA_ceph_test_filejournal_DEPENDENCIES)
@rm -f ceph_test_filejournal$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_filejournal_LINK) $(ceph_test_filejournal_OBJECTS) $(ceph_test_filejournal_LDADD) $(LIBS)
-test/filestore/$(am__dirstamp):
- @$(MKDIR_P) test/filestore
- @: > test/filestore/$(am__dirstamp)
-test/filestore/$(DEPDIR)/$(am__dirstamp):
- @$(MKDIR_P) test/filestore/$(DEPDIR)
- @: > test/filestore/$(DEPDIR)/$(am__dirstamp)
-test/filestore/ceph_test_filestore-store_test.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
-ceph_test_filestore$(EXEEXT): $(ceph_test_filestore_OBJECTS) $(ceph_test_filestore_DEPENDENCIES) $(EXTRA_ceph_test_filestore_DEPENDENCIES)
- @rm -f ceph_test_filestore$(EXEEXT)
- $(AM_V_CXXLD)$(ceph_test_filestore_LINK) $(ceph_test_filestore_OBJECTS) $(ceph_test_filestore_LDADD) $(LIBS)
-test/filestore/test_idempotent.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
-test/filestore/FileStoreTracker.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/$(am__dirstamp):
+ @$(MKDIR_P) test/objectstore
+ @: > test/objectstore/$(am__dirstamp)
+test/objectstore/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) test/objectstore/$(DEPDIR)
+ @: > test/objectstore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/test_idempotent.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/FileStoreTracker.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
test/common/$(am__dirstamp):
@$(MKDIR_P) test/common
@: > test/common/$(am__dirstamp)
@@ -6522,27 +6587,21 @@ test/common/ObjectContents.$(OBJEXT): test/common/$(am__dirstamp) \
ceph_test_filestore_idempotent$(EXEEXT): $(ceph_test_filestore_idempotent_OBJECTS) $(ceph_test_filestore_idempotent_DEPENDENCIES) $(EXTRA_ceph_test_filestore_idempotent_DEPENDENCIES)
@rm -f ceph_test_filestore_idempotent$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_test_filestore_idempotent_OBJECTS) $(ceph_test_filestore_idempotent_LDADD) $(LIBS)
-test/filestore/test_idempotent_sequence.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
-test/filestore/DeterministicOpSequence.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
-test/filestore/TestFileStoreState.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
-test/filestore/FileStoreDiff.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/test_idempotent_sequence.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/DeterministicOpSequence.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/TestObjectStoreState.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/FileStoreDiff.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
ceph_test_filestore_idempotent_sequence$(EXEEXT): $(ceph_test_filestore_idempotent_sequence_OBJECTS) $(ceph_test_filestore_idempotent_sequence_DEPENDENCIES) $(EXTRA_ceph_test_filestore_idempotent_sequence_DEPENDENCIES)
@rm -f ceph_test_filestore_idempotent_sequence$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_test_filestore_idempotent_sequence_OBJECTS) $(ceph_test_filestore_idempotent_sequence_LDADD) $(LIBS)
-test/filestore/workload_generator.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
-ceph_test_filestore_workloadgen$(EXEEXT): $(ceph_test_filestore_workloadgen_OBJECTS) $(ceph_test_filestore_workloadgen_DEPENDENCIES) $(EXTRA_ceph_test_filestore_workloadgen_DEPENDENCIES)
- @rm -f ceph_test_filestore_workloadgen$(EXEEXT)
- $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_filestore_workloadgen_OBJECTS) $(ceph_test_filestore_workloadgen_LDADD) $(LIBS)
test/test_get_blkdev_size.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
ceph_test_get_blkdev_size$(EXEEXT): $(ceph_test_get_blkdev_size_OBJECTS) $(ceph_test_get_blkdev_size_DEPENDENCIES) $(EXTRA_ceph_test_get_blkdev_size_DEPENDENCIES)
@@ -6609,9 +6668,6 @@ test/librbd/$(DEPDIR)/$(am__dirstamp):
test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT): \
test/librbd/$(am__dirstamp) \
test/librbd/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_librbd-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_librbd$(EXEEXT): $(ceph_test_librbd_OBJECTS) $(ceph_test_librbd_DEPENDENCIES) $(EXTRA_ceph_test_librbd_DEPENDENCIES)
@rm -f ceph_test_librbd$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_librbd_LINK) $(ceph_test_librbd_OBJECTS) $(ceph_test_librbd_LDADD) $(LIBS)
@@ -6664,6 +6720,24 @@ test/osdc/FakeWriteback.$(OBJEXT): test/osdc/$(am__dirstamp) \
ceph_test_objectcacher_stress$(EXEEXT): $(ceph_test_objectcacher_stress_OBJECTS) $(ceph_test_objectcacher_stress_DEPENDENCIES) $(EXTRA_ceph_test_objectcacher_stress_DEPENDENCIES)
@rm -f ceph_test_objectcacher_stress$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_test_objectcacher_stress_OBJECTS) $(ceph_test_objectcacher_stress_LDADD) $(LIBS)
+test/objectstore/ceph_test_objectstore-store_test.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
+ceph_test_objectstore$(EXEEXT): $(ceph_test_objectstore_OBJECTS) $(ceph_test_objectstore_DEPENDENCIES) $(EXTRA_ceph_test_objectstore_DEPENDENCIES)
+ @rm -f ceph_test_objectstore$(EXEEXT)
+ $(AM_V_CXXLD)$(ceph_test_objectstore_LINK) $(ceph_test_objectstore_OBJECTS) $(ceph_test_objectstore_LDADD) $(LIBS)
+test/objectstore/workload_generator.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
+ceph_test_objectstore_workloadgen$(EXEEXT): $(ceph_test_objectstore_workloadgen_OBJECTS) $(ceph_test_objectstore_workloadgen_DEPENDENCIES) $(EXTRA_ceph_test_objectstore_workloadgen_DEPENDENCIES)
+ @rm -f ceph_test_objectstore_workloadgen$(EXEEXT)
+ $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_objectstore_workloadgen_OBJECTS) $(ceph_test_objectstore_workloadgen_LDADD) $(LIBS)
+test/osd/$(am__dirstamp):
+ @$(MKDIR_P) test/osd
+ @: > test/osd/$(am__dirstamp)
+test/osd/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) test/osd/$(DEPDIR)
+ @: > test/osd/$(DEPDIR)/$(am__dirstamp)
test/osd/TestRados.$(OBJEXT): test/osd/$(am__dirstamp) \
test/osd/$(DEPDIR)/$(am__dirstamp)
test/osd/TestOpStat.$(OBJEXT): test/osd/$(am__dirstamp) \
@@ -6678,16 +6752,16 @@ ceph_test_rados$(EXEEXT): $(ceph_test_rados_OBJECTS) $(ceph_test_rados_DEPENDENC
test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_aio-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_aio$(EXEEXT): $(ceph_test_rados_api_aio_OBJECTS) $(ceph_test_rados_api_aio_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_aio_DEPENDENCIES)
@rm -f ceph_test_rados_api_aio$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_aio_LINK) $(ceph_test_rados_api_aio_OBJECTS) $(ceph_test_rados_api_aio_LDADD) $(LIBS)
-test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT): \
+test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_c_write_operations-test.$(OBJEXT): \
+ceph_test_rados_api_c_read_operations$(EXEEXT): $(ceph_test_rados_api_c_read_operations_OBJECTS) $(ceph_test_rados_api_c_read_operations_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_c_read_operations_DEPENDENCIES)
+ @rm -f ceph_test_rados_api_c_read_operations$(EXEEXT)
+ $(AM_V_CXXLD)$(ceph_test_rados_api_c_read_operations_LINK) $(ceph_test_rados_api_c_read_operations_OBJECTS) $(ceph_test_rados_api_c_read_operations_LDADD) $(LIBS)
+test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_c_write_operations$(EXEEXT): $(ceph_test_rados_api_c_write_operations_OBJECTS) $(ceph_test_rados_api_c_write_operations_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_c_write_operations_DEPENDENCIES)
@@ -6696,90 +6770,60 @@ ceph_test_rados_api_c_write_operations$(EXEEXT): $(ceph_test_rados_api_c_write_o
test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_cls-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_cls$(EXEEXT): $(ceph_test_rados_api_cls_OBJECTS) $(ceph_test_rados_api_cls_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_cls_DEPENDENCIES)
@rm -f ceph_test_rados_api_cls$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_cls_LINK) $(ceph_test_rados_api_cls_OBJECTS) $(ceph_test_rados_api_cls_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_cmd-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_cmd$(EXEEXT): $(ceph_test_rados_api_cmd_OBJECTS) $(ceph_test_rados_api_cmd_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_cmd_DEPENDENCIES)
@rm -f ceph_test_rados_api_cmd$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_cmd_LINK) $(ceph_test_rados_api_cmd_OBJECTS) $(ceph_test_rados_api_cmd_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_io-io.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_io-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_io$(EXEEXT): $(ceph_test_rados_api_io_OBJECTS) $(ceph_test_rados_api_io_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_io_DEPENDENCIES)
@rm -f ceph_test_rados_api_io$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_io_LINK) $(ceph_test_rados_api_io_OBJECTS) $(ceph_test_rados_api_io_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_list-list.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_list-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_list$(EXEEXT): $(ceph_test_rados_api_list_OBJECTS) $(ceph_test_rados_api_list_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_list_DEPENDENCIES)
@rm -f ceph_test_rados_api_list$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_list_LINK) $(ceph_test_rados_api_list_OBJECTS) $(ceph_test_rados_api_list_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_lock-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_lock$(EXEEXT): $(ceph_test_rados_api_lock_OBJECTS) $(ceph_test_rados_api_lock_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_lock_DEPENDENCIES)
@rm -f ceph_test_rados_api_lock$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_lock_LINK) $(ceph_test_rados_api_lock_OBJECTS) $(ceph_test_rados_api_lock_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_misc-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_misc$(EXEEXT): $(ceph_test_rados_api_misc_OBJECTS) $(ceph_test_rados_api_misc_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_misc_DEPENDENCIES)
@rm -f ceph_test_rados_api_misc$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_misc_LINK) $(ceph_test_rados_api_misc_OBJECTS) $(ceph_test_rados_api_misc_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_pool-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_pool$(EXEEXT): $(ceph_test_rados_api_pool_OBJECTS) $(ceph_test_rados_api_pool_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_pool_DEPENDENCIES)
@rm -f ceph_test_rados_api_pool$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_pool_LINK) $(ceph_test_rados_api_pool_OBJECTS) $(ceph_test_rados_api_pool_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_snapshots-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_snapshots$(EXEEXT): $(ceph_test_rados_api_snapshots_OBJECTS) $(ceph_test_rados_api_snapshots_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_snapshots_DEPENDENCIES)
@rm -f ceph_test_rados_api_snapshots$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_snapshots_LINK) $(ceph_test_rados_api_snapshots_OBJECTS) $(ceph_test_rados_api_snapshots_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_stat-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_stat$(EXEEXT): $(ceph_test_rados_api_stat_OBJECTS) $(ceph_test_rados_api_stat_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_stat_DEPENDENCIES)
@rm -f ceph_test_rados_api_stat$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_stat_LINK) $(ceph_test_rados_api_stat_OBJECTS) $(ceph_test_rados_api_stat_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_tier-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT): osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_tier$(EXEEXT): $(ceph_test_rados_api_tier_OBJECTS) $(ceph_test_rados_api_tier_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_tier_DEPENDENCIES)
@@ -6788,9 +6832,6 @@ ceph_test_rados_api_tier$(EXEEXT): $(ceph_test_rados_api_tier_OBJECTS) $(ceph_te
test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_rados_api_watch_notify-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_rados_api_watch_notify$(EXEEXT): $(ceph_test_rados_api_watch_notify_OBJECTS) $(ceph_test_rados_api_watch_notify_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_watch_notify_DEPENDENCIES)
@rm -f ceph_test_rados_api_watch_notify$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_watch_notify_LINK) $(ceph_test_rados_api_watch_notify_OBJECTS) $(ceph_test_rados_api_watch_notify_LDADD) $(LIBS)
@@ -6838,6 +6879,17 @@ test/test_rewrite_latency.$(OBJEXT): test/$(am__dirstamp) \
ceph_test_rewrite_latency$(EXEEXT): $(ceph_test_rewrite_latency_OBJECTS) $(ceph_test_rewrite_latency_DEPENDENCIES) $(EXTRA_ceph_test_rewrite_latency_DEPENDENCIES)
@rm -f ceph_test_rewrite_latency$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rewrite_latency_OBJECTS) $(ceph_test_rewrite_latency_LDADD) $(LIBS)
+test/rgw/$(am__dirstamp):
+ @$(MKDIR_P) test/rgw
+ @: > test/rgw/$(am__dirstamp)
+test/rgw/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) test/rgw/$(DEPDIR)
+ @: > test/rgw/$(DEPDIR)/$(am__dirstamp)
+test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.$(OBJEXT): \
+ test/rgw/$(am__dirstamp) test/rgw/$(DEPDIR)/$(am__dirstamp)
+ceph_test_rgw_manifest$(EXEEXT): $(ceph_test_rgw_manifest_OBJECTS) $(ceph_test_rgw_manifest_DEPENDENCIES) $(EXTRA_ceph_test_rgw_manifest_DEPENDENCIES)
+ @rm -f ceph_test_rgw_manifest$(EXEEXT)
+ $(AM_V_CXXLD)$(ceph_test_rgw_manifest_LINK) $(ceph_test_rgw_manifest_OBJECTS) $(ceph_test_rgw_manifest_LDADD) $(LIBS)
test/TestSignalHandlers.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
ceph_test_signal_handlers$(EXEEXT): $(ceph_test_signal_handlers_OBJECTS) $(ceph_test_signal_handlers_DEPENDENCIES) $(EXTRA_ceph_test_signal_handlers_DEPENDENCIES)
@@ -6850,9 +6902,6 @@ ceph_test_snap_mapper$(EXEEXT): $(ceph_test_snap_mapper_OBJECTS) $(ceph_test_sna
$(AM_V_CXXLD)$(ceph_test_snap_mapper_LINK) $(ceph_test_snap_mapper_OBJECTS) $(ceph_test_snap_mapper_LDADD) $(LIBS)
test/ceph_test_stress_watch-test_stress_watch.$(OBJEXT): \
test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp)
-test/librados/ceph_test_stress_watch-test.$(OBJEXT): \
- test/librados/$(am__dirstamp) \
- test/librados/$(DEPDIR)/$(am__dirstamp)
ceph_test_stress_watch$(EXEEXT): $(ceph_test_stress_watch_OBJECTS) $(ceph_test_stress_watch_DEPENDENCIES) $(EXTRA_ceph_test_stress_watch_DEPENDENCIES)
@rm -f ceph_test_stress_watch$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_stress_watch_LINK) $(ceph_test_stress_watch_OBJECTS) $(ceph_test_stress_watch_LDADD) $(LIBS)
@@ -7105,6 +7154,8 @@ common/test_build_libcommon-code_environment.$(OBJEXT): \
common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-dout.$(OBJEXT): common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
+common/test_build_libcommon-histogram.$(OBJEXT): \
+ common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-signal.$(OBJEXT): common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-simple_spin.$(OBJEXT): \
@@ -7175,6 +7226,8 @@ osd/test_build_libcommon-OSDMap.$(OBJEXT): osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
osd/test_build_libcommon-osd_types.$(OBJEXT): osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
+osd/test_build_libcommon-ECMsgTypes.$(OBJEXT): osd/$(am__dirstamp) \
+ osd/$(DEPDIR)/$(am__dirstamp)
osd/test_build_libcommon-HitSet.$(OBJEXT): osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
mds/test_build_libcommon-MDSMap.$(OBJEXT): mds/$(am__dirstamp) \
@@ -7316,9 +7369,9 @@ test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT): test/$(am__dirstamp) \
unittest_ceph_crypto$(EXEEXT): $(unittest_ceph_crypto_OBJECTS) $(unittest_ceph_crypto_DEPENDENCIES) $(EXTRA_unittest_ceph_crypto_DEPENDENCIES)
@rm -f unittest_ceph_crypto$(EXEEXT)
$(AM_V_CXXLD)$(unittest_ceph_crypto_LINK) $(unittest_ceph_crypto_OBJECTS) $(unittest_ceph_crypto_LDADD) $(LIBS)
-test/filestore/unittest_chain_xattr-chain_xattr.$(OBJEXT): \
- test/filestore/$(am__dirstamp) \
- test/filestore/$(DEPDIR)/$(am__dirstamp)
+test/objectstore/unittest_chain_xattr-chain_xattr.$(OBJEXT): \
+ test/objectstore/$(am__dirstamp) \
+ test/objectstore/$(DEPDIR)/$(am__dirstamp)
unittest_chain_xattr$(EXEEXT): $(unittest_chain_xattr_OBJECTS) $(unittest_chain_xattr_DEPENDENCIES) $(EXTRA_unittest_chain_xattr_DEPENDENCIES)
@rm -f unittest_chain_xattr$(EXEEXT)
$(AM_V_CXXLD)$(unittest_chain_xattr_LINK) $(unittest_chain_xattr_OBJECTS) $(unittest_chain_xattr_LDADD) $(LIBS)
@@ -7333,6 +7386,12 @@ test/unittest_confutils-confutils.$(OBJEXT): test/$(am__dirstamp) \
unittest_confutils$(EXEEXT): $(unittest_confutils_OBJECTS) $(unittest_confutils_DEPENDENCIES) $(EXTRA_unittest_confutils_DEPENDENCIES)
@rm -f unittest_confutils$(EXEEXT)
$(AM_V_CXXLD)$(unittest_confutils_LINK) $(unittest_confutils_OBJECTS) $(unittest_confutils_LDADD) $(LIBS)
+test/common/unittest_context-test_context.$(OBJEXT): \
+ test/common/$(am__dirstamp) \
+ test/common/$(DEPDIR)/$(am__dirstamp)
+unittest_context$(EXEEXT): $(unittest_context_OBJECTS) $(unittest_context_DEPENDENCIES) $(EXTRA_unittest_context_DEPENDENCIES)
+ @rm -f unittest_context$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_context_LINK) $(unittest_context_OBJECTS) $(unittest_context_LDADD) $(LIBS)
test/common/unittest_crc32c-test_crc32c.$(OBJEXT): \
test/common/$(am__dirstamp) \
test/common/$(DEPDIR)/$(am__dirstamp)
@@ -7357,11 +7416,6 @@ test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT): \
unittest_crush_wrapper$(EXEEXT): $(unittest_crush_wrapper_OBJECTS) $(unittest_crush_wrapper_DEPENDENCIES) $(EXTRA_unittest_crush_wrapper_DEPENDENCIES)
@rm -f unittest_crush_wrapper$(EXEEXT)
$(AM_V_CXXLD)$(unittest_crush_wrapper_LINK) $(unittest_crush_wrapper_OBJECTS) $(unittest_crush_wrapper_LDADD) $(LIBS)
-test/unittest_crushwrapper-test_crushwrapper.$(OBJEXT): \
- test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp)
-unittest_crushwrapper$(EXEEXT): $(unittest_crushwrapper_OBJECTS) $(unittest_crushwrapper_DEPENDENCIES) $(EXTRA_unittest_crushwrapper_DEPENDENCIES)
- @rm -f unittest_crushwrapper$(EXEEXT)
- $(AM_V_CXXLD)$(unittest_crushwrapper_LINK) $(unittest_crushwrapper_OBJECTS) $(unittest_crushwrapper_LDADD) $(LIBS)
test/unittest_crypto-crypto.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_crypto$(EXEEXT): $(unittest_crypto_OBJECTS) $(unittest_crypto_DEPENDENCIES) $(EXTRA_unittest_crypto_DEPENDENCIES)
@@ -7372,49 +7426,58 @@ test/unittest_daemon_config-daemon_config.$(OBJEXT): \
unittest_daemon_config$(EXEEXT): $(unittest_daemon_config_OBJECTS) $(unittest_daemon_config_DEPENDENCIES) $(EXTRA_unittest_daemon_config_DEPENDENCIES)
@rm -f unittest_daemon_config$(EXEEXT)
$(AM_V_CXXLD)$(unittest_daemon_config_LINK) $(unittest_daemon_config_OBJECTS) $(unittest_daemon_config_LDADD) $(LIBS)
+test/osd/unittest_ecbackend-TestECBackend.$(OBJEXT): \
+ test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+unittest_ecbackend$(EXEEXT): $(unittest_ecbackend_OBJECTS) $(unittest_ecbackend_DEPENDENCIES) $(EXTRA_unittest_ecbackend_DEPENDENCIES)
+ @rm -f unittest_ecbackend$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_ecbackend_LINK) $(unittest_ecbackend_OBJECTS) $(unittest_ecbackend_LDADD) $(LIBS)
test/unittest_encoding-encoding.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_encoding$(EXEEXT): $(unittest_encoding_OBJECTS) $(unittest_encoding_DEPENDENCIES) $(EXTRA_unittest_encoding_DEPENDENCIES)
@rm -f unittest_encoding$(EXEEXT)
$(AM_V_CXXLD)$(unittest_encoding_LINK) $(unittest_encoding_OBJECTS) $(unittest_encoding_LDADD) $(LIBS)
-test/osd/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT): \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT): \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
unittest_erasure_code_example$(EXEEXT): $(unittest_erasure_code_example_OBJECTS) $(unittest_erasure_code_example_DEPENDENCIES) $(EXTRA_unittest_erasure_code_example_DEPENDENCIES)
@rm -f unittest_erasure_code_example$(EXEEXT)
$(AM_V_CXXLD)$(unittest_erasure_code_example_LINK) $(unittest_erasure_code_example_OBJECTS) $(unittest_erasure_code_example_LDADD) $(LIBS)
-test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT): \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/cauchy.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/galois.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/jerasure.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/liberation.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
-osd/ErasureCodePluginJerasure/reed_sol.$(OBJEXT): \
- osd/ErasureCodePluginJerasure/$(am__dirstamp) \
- osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT): \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/cauchy.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/galois.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/jerasure.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/liberation.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+erasure-code/jerasure/reed_sol.$(OBJEXT): \
+ erasure-code/jerasure/$(am__dirstamp) \
+ erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
unittest_erasure_code_jerasure$(EXEEXT): $(unittest_erasure_code_jerasure_OBJECTS) $(unittest_erasure_code_jerasure_DEPENDENCIES) $(EXTRA_unittest_erasure_code_jerasure_DEPENDENCIES)
@rm -f unittest_erasure_code_jerasure$(EXEEXT)
$(AM_V_CXXLD)$(unittest_erasure_code_jerasure_LINK) $(unittest_erasure_code_jerasure_OBJECTS) $(unittest_erasure_code_jerasure_LDADD) $(LIBS)
-test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT): \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT): \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
unittest_erasure_code_plugin$(EXEEXT): $(unittest_erasure_code_plugin_OBJECTS) $(unittest_erasure_code_plugin_DEPENDENCIES) $(EXTRA_unittest_erasure_code_plugin_DEPENDENCIES)
@rm -f unittest_erasure_code_plugin$(EXEEXT)
$(AM_V_CXXLD)$(unittest_erasure_code_plugin_LINK) $(unittest_erasure_code_plugin_OBJECTS) $(unittest_erasure_code_plugin_LDADD) $(LIBS)
-test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT): \
- test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT): \
+ test/erasure-code/$(am__dirstamp) \
+ test/erasure-code/$(DEPDIR)/$(am__dirstamp)
unittest_erasure_code_plugin_jerasure$(EXEEXT): $(unittest_erasure_code_plugin_jerasure_OBJECTS) $(unittest_erasure_code_plugin_jerasure_DEPENDENCIES) $(EXTRA_unittest_erasure_code_plugin_jerasure_DEPENDENCIES)
@rm -f unittest_erasure_code_plugin_jerasure$(EXEEXT)
$(AM_V_CXXLD)$(unittest_erasure_code_plugin_jerasure_LINK) $(unittest_erasure_code_plugin_jerasure_OBJECTS) $(unittest_erasure_code_plugin_jerasure_LDADD) $(LIBS)
@@ -7451,6 +7514,12 @@ test/unittest_heartbeatmap-heartbeat_map.$(OBJEXT): \
unittest_heartbeatmap$(EXEEXT): $(unittest_heartbeatmap_OBJECTS) $(unittest_heartbeatmap_DEPENDENCIES) $(EXTRA_unittest_heartbeatmap_DEPENDENCIES)
@rm -f unittest_heartbeatmap$(EXEEXT)
$(AM_V_CXXLD)$(unittest_heartbeatmap_LINK) $(unittest_heartbeatmap_OBJECTS) $(unittest_heartbeatmap_LDADD) $(LIBS)
+test/common/unittest_histogram-histogram.$(OBJEXT): \
+ test/common/$(am__dirstamp) \
+ test/common/$(DEPDIR)/$(am__dirstamp)
+unittest_histogram$(EXEEXT): $(unittest_histogram_OBJECTS) $(unittest_histogram_DEPENDENCIES) $(EXTRA_unittest_histogram_DEPENDENCIES)
+ @rm -f unittest_histogram$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_histogram_LINK) $(unittest_histogram_OBJECTS) $(unittest_histogram_LDADD) $(LIBS)
test/osd/unittest_hitset-hitset.$(OBJEXT): test/osd/$(am__dirstamp) \
test/osd/$(DEPDIR)/$(am__dirstamp)
unittest_hitset$(EXEEXT): $(unittest_hitset_OBJECTS) $(unittest_hitset_DEPENDENCIES) $(EXTRA_unittest_hitset_DEPENDENCIES)
@@ -7498,6 +7567,11 @@ test/mon/unittest_mon_moncap-moncap.$(OBJEXT): \
unittest_mon_moncap$(EXEEXT): $(unittest_mon_moncap_OBJECTS) $(unittest_mon_moncap_DEPENDENCIES) $(EXTRA_unittest_mon_moncap_DEPENDENCIES)
@rm -f unittest_mon_moncap$(EXEEXT)
$(AM_V_CXXLD)$(unittest_mon_moncap_LINK) $(unittest_mon_moncap_OBJECTS) $(unittest_mon_moncap_LDADD) $(LIBS)
+test/mon/unittest_mon_pgmap-PGMap.$(OBJEXT): test/mon/$(am__dirstamp) \
+ test/mon/$(DEPDIR)/$(am__dirstamp)
+unittest_mon_pgmap$(EXEEXT): $(unittest_mon_pgmap_OBJECTS) $(unittest_mon_pgmap_DEPENDENCIES) $(EXTRA_unittest_mon_pgmap_DEPENDENCIES)
+ @rm -f unittest_mon_pgmap$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_mon_pgmap_LINK) $(unittest_mon_pgmap_OBJECTS) $(unittest_mon_pgmap_LDADD) $(LIBS)
test/on_exit.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_on_exit$(EXEEXT): $(unittest_on_exit_OBJECTS) $(unittest_on_exit_DEPENDENCIES) $(EXTRA_unittest_on_exit_DEPENDENCIES)
@@ -7508,8 +7582,8 @@ test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT): \
unittest_osd_osdcap$(EXEEXT): $(unittest_osd_osdcap_OBJECTS) $(unittest_osd_osdcap_DEPENDENCIES) $(EXTRA_unittest_osd_osdcap_DEPENDENCIES)
@rm -f unittest_osd_osdcap$(EXEEXT)
$(AM_V_CXXLD)$(unittest_osd_osdcap_LINK) $(unittest_osd_osdcap_OBJECTS) $(unittest_osd_osdcap_LDADD) $(LIBS)
-test/unittest_osd_types-test_osd_types.$(OBJEXT): \
- test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp)
+test/osd/unittest_osd_types-types.$(OBJEXT): test/osd/$(am__dirstamp) \
+ test/osd/$(DEPDIR)/$(am__dirstamp)
unittest_osd_types$(EXEEXT): $(unittest_osd_types_OBJECTS) $(unittest_osd_types_DEPENDENCIES) $(EXTRA_unittest_osd_types_DEPENDENCIES)
@rm -f unittest_osd_types$(EXEEXT)
$(AM_V_CXXLD)$(unittest_osd_types_LINK) $(unittest_osd_types_OBJECTS) $(unittest_osd_types_LDADD) $(LIBS)
@@ -8006,6 +8080,8 @@ mostlyclean-compile:
-rm -f common/fd.lo
-rm -f common/hex.$(OBJEXT)
-rm -f common/hex.lo
+ -rm -f common/histogram.$(OBJEXT)
+ -rm -f common/histogram.lo
-rm -f common/hobject.$(OBJEXT)
-rm -f common/hobject.lo
-rm -f common/ipaddr.$(OBJEXT)
@@ -8104,6 +8180,7 @@ mostlyclean-compile:
-rm -f common/test_build_libcommon-escape.$(OBJEXT)
-rm -f common/test_build_libcommon-fd.$(OBJEXT)
-rm -f common/test_build_libcommon-hex.$(OBJEXT)
+ -rm -f common/test_build_libcommon-histogram.$(OBJEXT)
-rm -f common/test_build_libcommon-hobject.$(OBJEXT)
-rm -f common/test_build_libcommon-ipaddr.$(OBJEXT)
-rm -f common/test_build_libcommon-linux_version.$(OBJEXT)
@@ -8148,6 +8225,29 @@ mostlyclean-compile:
-rm -f crush/hash.lo
-rm -f crush/mapper.$(OBJEXT)
-rm -f crush/mapper.lo
+ -rm -f erasure-code/ErasureCodePlugin.$(OBJEXT)
+ -rm -f erasure-code/ErasureCodePlugin.lo
+ -rm -f erasure-code/jerasure/cauchy.$(OBJEXT)
+ -rm -f erasure-code/jerasure/galois.$(OBJEXT)
+ -rm -f erasure-code/jerasure/jerasure.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo
+ -rm -f erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo
+ -rm -f erasure-code/jerasure/libec_jerasure_la-cauchy.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-cauchy.lo
+ -rm -f erasure-code/jerasure/libec_jerasure_la-galois.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-galois.lo
+ -rm -f erasure-code/jerasure/libec_jerasure_la-jerasure.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-jerasure.lo
+ -rm -f erasure-code/jerasure/libec_jerasure_la-liberation.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-liberation.lo
+ -rm -f erasure-code/jerasure/libec_jerasure_la-reed_sol.$(OBJEXT)
+ -rm -f erasure-code/jerasure/libec_jerasure_la-reed_sol.lo
+ -rm -f erasure-code/jerasure/liberation.$(OBJEXT)
+ -rm -f erasure-code/jerasure/reed_sol.$(OBJEXT)
+ -rm -f erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT)
+ -rm -f erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT)
-rm -f global/global_context.$(OBJEXT)
-rm -f global/global_context.lo
-rm -f global/global_init.$(OBJEXT)
@@ -8353,6 +8453,8 @@ mostlyclean-compile:
-rm -f os/ObjectStore.lo
-rm -f os/WBThrottle.$(OBJEXT)
-rm -f os/WBThrottle.lo
+ -rm -f os/XfsFileStoreBackend.$(OBJEXT)
+ -rm -f os/XfsFileStoreBackend.lo
-rm -f os/ZFSFileStoreBackend.$(OBJEXT)
-rm -f os/ZFSFileStoreBackend.lo
-rm -f os/chain_xattr.$(OBJEXT)
@@ -8362,29 +8464,14 @@ mostlyclean-compile:
-rm -f osd/Ager.lo
-rm -f osd/ClassHandler.$(OBJEXT)
-rm -f osd/ClassHandler.lo
- -rm -f osd/ErasureCodePlugin.$(OBJEXT)
- -rm -f osd/ErasureCodePlugin.lo
- -rm -f osd/ErasureCodePluginJerasure/cauchy.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/galois.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/jerasure.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo
- -rm -f osd/ErasureCodePluginJerasure/liberation.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/reed_sol.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT)
- -rm -f osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT)
+ -rm -f osd/ECBackend.$(OBJEXT)
+ -rm -f osd/ECBackend.lo
+ -rm -f osd/ECMsgTypes.$(OBJEXT)
+ -rm -f osd/ECMsgTypes.lo
+ -rm -f osd/ECTransaction.$(OBJEXT)
+ -rm -f osd/ECTransaction.lo
+ -rm -f osd/ECUtil.$(OBJEXT)
+ -rm -f osd/ECUtil.lo
-rm -f osd/HitSet.$(OBJEXT)
-rm -f osd/HitSet.lo
-rm -f osd/OSD.$(OBJEXT)
@@ -8412,6 +8499,7 @@ mostlyclean-compile:
-rm -f osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT)
-rm -f osd/osd_types.$(OBJEXT)
-rm -f osd/osd_types.lo
+ -rm -f osd/test_build_libcommon-ECMsgTypes.$(OBJEXT)
-rm -f osd/test_build_libcommon-HitSet.$(OBJEXT)
-rm -f osd/test_build_libcommon-OSDMap.$(OBJEXT)
-rm -f osd/test_build_libcommon-osd_types.$(OBJEXT)
@@ -8609,7 +8697,9 @@ mostlyclean-compile:
-rm -f test/common/get_command_descriptions.$(OBJEXT)
-rm -f test/common/unittest_bloom_filter-test_bloom_filter.$(OBJEXT)
-rm -f test/common/unittest_config-test_config.$(OBJEXT)
+ -rm -f test/common/unittest_context-test_context.$(OBJEXT)
-rm -f test/common/unittest_crc32c-test_crc32c.$(OBJEXT)
+ -rm -f test/common/unittest_histogram-histogram.$(OBJEXT)
-rm -f test/common/unittest_sharedptr_registry-test_sharedptr_registry.$(OBJEXT)
-rm -f test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.$(OBJEXT)
-rm -f test/common/unittest_str_map-test_str_map.$(OBJEXT)
@@ -8618,65 +8708,62 @@ mostlyclean-compile:
-rm -f test/crush/unittest_crush_indep-indep.$(OBJEXT)
-rm -f test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT)
-rm -f test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT)
- -rm -f test/filestore/DeterministicOpSequence.$(OBJEXT)
- -rm -f test/filestore/FileStoreDiff.$(OBJEXT)
- -rm -f test/filestore/FileStoreTracker.$(OBJEXT)
- -rm -f test/filestore/TestFileStoreState.$(OBJEXT)
- -rm -f test/filestore/ceph_test_filestore-store_test.$(OBJEXT)
- -rm -f test/filestore/test_idempotent.$(OBJEXT)
- -rm -f test/filestore/test_idempotent_sequence.$(OBJEXT)
- -rm -f test/filestore/unittest_chain_xattr-chain_xattr.$(OBJEXT)
- -rm -f test/filestore/workload_generator.$(OBJEXT)
+ -rm -f test/erasure-code/ceph_erasure_code.$(OBJEXT)
+ -rm -f test/erasure-code/ceph_erasure_code_benchmark.$(OBJEXT)
+ -rm -f test/erasure-code/libec_example_la-ErasureCodePluginExample.$(OBJEXT)
+ -rm -f test/erasure-code/libec_example_la-ErasureCodePluginExample.lo
+ -rm -f test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.$(OBJEXT)
+ -rm -f test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo
+ -rm -f test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.$(OBJEXT)
+ -rm -f test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo
+ -rm -f test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.$(OBJEXT)
+ -rm -f test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo
+ -rm -f test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.$(OBJEXT)
+ -rm -f test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo
+ -rm -f test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT)
+ -rm -f test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT)
+ -rm -f test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT)
+ -rm -f test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT)
-rm -f test/kv_store_bench.$(OBJEXT)
-rm -f test/libcephfs/ceph_test_libcephfs-caps.$(OBJEXT)
-rm -f test/libcephfs/ceph_test_libcephfs-multiclient.$(OBJEXT)
-rm -f test/libcephfs/ceph_test_libcephfs-readdir_r_cb.$(OBJEXT)
-rm -f test/libcephfs/ceph_test_libcephfs-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_hello-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_lock-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_log-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_rbd-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_refcount-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_replica_log-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_rgw-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_statelog-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_cls_version-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_librbd-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_aio-test.$(OBJEXT)
+ -rm -f test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_c_write_operations-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_cls-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_cmd-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_io-io.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_io-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_list-list.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_list-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_lock-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_misc-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_pool-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_snapshots-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_stat-test.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_tier-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT)
- -rm -f test/librados/ceph_test_rados_api_watch_notify-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT)
- -rm -f test/librados/ceph_test_stress_watch-test.$(OBJEXT)
- -rm -f test/librados/test.$(OBJEXT)
+ -rm -f test/librados/libradostest_la-TestCase.$(OBJEXT)
+ -rm -f test/librados/libradostest_la-TestCase.lo
+ -rm -f test/librados/libradostest_la-test.$(OBJEXT)
+ -rm -f test/librados/libradostest_la-test.lo
-rm -f test/librados/unittest_librados-librados.$(OBJEXT)
-rm -f test/librados/unittest_librados_config-librados_config.$(OBJEXT)
-rm -f test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT)
-rm -f test/librbd/ceph_test_librbd_fsx-fsx.$(OBJEXT)
-rm -f test/mon/test_mon_workloadgen.$(OBJEXT)
-rm -f test/mon/unittest_mon_moncap-moncap.$(OBJEXT)
+ -rm -f test/mon/unittest_mon_pgmap-PGMap.$(OBJEXT)
-rm -f test/multi_stress_watch.$(OBJEXT)
+ -rm -f test/objectstore/DeterministicOpSequence.$(OBJEXT)
+ -rm -f test/objectstore/FileStoreDiff.$(OBJEXT)
+ -rm -f test/objectstore/FileStoreTracker.$(OBJEXT)
+ -rm -f test/objectstore/TestObjectStoreState.$(OBJEXT)
+ -rm -f test/objectstore/ceph_test_objectstore-store_test.$(OBJEXT)
+ -rm -f test/objectstore/test_idempotent.$(OBJEXT)
+ -rm -f test/objectstore/test_idempotent_sequence.$(OBJEXT)
+ -rm -f test/objectstore/unittest_chain_xattr-chain_xattr.$(OBJEXT)
+ -rm -f test/objectstore/workload_generator.$(OBJEXT)
-rm -f test/omap_bench.$(OBJEXT)
-rm -f test/on_exit.$(OBJEXT)
-rm -f test/os/unittest_flatindex-TestFlatIndex.$(OBJEXT)
@@ -8685,27 +8772,15 @@ mostlyclean-compile:
-rm -f test/osd/RadosModel.$(OBJEXT)
-rm -f test/osd/TestOpStat.$(OBJEXT)
-rm -f test/osd/TestRados.$(OBJEXT)
- -rm -f test/osd/ceph_erasure_code_benchmark.$(OBJEXT)
- -rm -f test/osd/libec_example_la-ErasureCodePluginExample.$(OBJEXT)
- -rm -f test/osd/libec_example_la-ErasureCodePluginExample.lo
- -rm -f test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.$(OBJEXT)
- -rm -f test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo
- -rm -f test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.$(OBJEXT)
- -rm -f test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo
- -rm -f test/osd/libec_hangs_la-ErasureCodePluginHangs.$(OBJEXT)
- -rm -f test/osd/libec_hangs_la-ErasureCodePluginHangs.lo
- -rm -f test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.$(OBJEXT)
- -rm -f test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo
- -rm -f test/osd/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT)
- -rm -f test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT)
- -rm -f test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT)
- -rm -f test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT)
+ -rm -f test/osd/unittest_ecbackend-TestECBackend.$(OBJEXT)
-rm -f test/osd/unittest_hitset-hitset.$(OBJEXT)
-rm -f test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT)
+ -rm -f test/osd/unittest_osd_types-types.$(OBJEXT)
-rm -f test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT)
-rm -f test/osd/unittest_pglog-TestPGLog.$(OBJEXT)
-rm -f test/osdc/FakeWriteback.$(OBJEXT)
-rm -f test/osdc/object_cacher_stress.$(OBJEXT)
+ -rm -f test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.$(OBJEXT)
-rm -f test/streamtest.$(OBJEXT)
-rm -f test/system/cross_process_sem.$(OBJEXT)
-rm -f test/system/cross_process_sem.lo
@@ -8745,7 +8820,6 @@ mostlyclean-compile:
-rm -f test/unittest_ceph_compatset-ceph_compatset.$(OBJEXT)
-rm -f test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT)
-rm -f test/unittest_confutils-confutils.$(OBJEXT)
- -rm -f test/unittest_crushwrapper-test_crushwrapper.$(OBJEXT)
-rm -f test/unittest_crypto-crypto.$(OBJEXT)
-rm -f test/unittest_daemon_config-daemon_config.$(OBJEXT)
-rm -f test/unittest_encoding-encoding.$(OBJEXT)
@@ -8756,7 +8830,6 @@ mostlyclean-compile:
-rm -f test/unittest_ipaddr-test_ipaddr.$(OBJEXT)
-rm -f test/unittest_libcephfs_config-libcephfs_config.$(OBJEXT)
-rm -f test/unittest_mime-mime.$(OBJEXT)
- -rm -f test/unittest_osd_types-test_osd_types.$(OBJEXT)
-rm -f test/unittest_perf_counters-perf_counters.$(OBJEXT)
-rm -f test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.$(OBJEXT)
-rm -f test/unittest_run_cmd-run_cmd.$(OBJEXT)
@@ -8912,6 +8985,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/escape.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/fd.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/hex.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/histogram.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/hobject.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/ipaddr.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/libcommon_crc_la-crc32c.Plo at am__quote@
@@ -8986,6 +9060,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-escape.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-fd.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-hex.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-histogram.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-hobject.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-ipaddr.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-linux_version.Po at am__quote@
@@ -9019,6 +9094,21 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at crush/$(DEPDIR)/crush.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at crush/$(DEPDIR)/hash.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at crush/$(DEPDIR)/mapper.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/$(DEPDIR)/ErasureCodePlugin.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/cauchy.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/galois.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/jerasure.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-galois.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-liberation.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/liberation.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/reed_sol.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at global/$(DEPDIR)/global_context.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at global/$(DEPDIR)/global_init.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at global/$(DEPDIR)/pidfile.Plo at am__quote@
@@ -9128,12 +9218,16 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/MemStore.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/ObjectStore.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/WBThrottle.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/XfsFileStoreBackend.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/ZFSFileStoreBackend.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/chain_xattr.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/libos_zfs_a-ZFS.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/Ager.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ClassHandler.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ErasureCodePlugin.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ECBackend.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ECMsgTypes.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ECTransaction.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ECUtil.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/HitSet.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/OSD.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/OSDCap.Plo at am__quote@
@@ -9148,23 +9242,10 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/Watch.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/osd_types.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-HitSet.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-OSDMap.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-osd_types.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/cauchy.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/galois.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/jerasure.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-galois.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-liberation.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/liberation.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/reed_sol.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osdc/$(DEPDIR)/Filer.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osdc/$(DEPDIR)/Journaler.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osdc/$(DEPDIR)/ObjectCacher.Plo at am__quote@
@@ -9315,7 +9396,6 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_confutils-confutils.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_crypto-crypto.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_daemon_config-daemon_config.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_encoding-encoding.Po at am__quote@
@@ -9326,7 +9406,6 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_mime-mime.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_osd_types-test_osd_types.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_perf_counters-perf_counters.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_run_cmd-run_cmd.Po at am__quote@
@@ -9367,7 +9446,9 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/get_command_descriptions.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_config-test_config.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_context-test_context.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_histogram-histogram.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po at am__quote@
@@ -9376,85 +9457,68 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/DeterministicOpSequence.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/FileStoreDiff.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/FileStoreTracker.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/TestFileStoreState.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/test_idempotent.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/test_idempotent_sequence.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/workload_generator.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/ceph_erasure_code.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/ceph_erasure_code_benchmark.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_log-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_cls_version-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_librbd-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/test.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/libradostest_la-TestCase.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/libradostest_la-test.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/unittest_librados-librados.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/mon/$(DEPDIR)/test_mon_workloadgen.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/DeterministicOpSequence.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/FileStoreDiff.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/FileStoreTracker.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/TestObjectStoreState.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/test_idempotent.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/test_idempotent_sequence.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/objectstore/$(DEPDIR)/workload_generator.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/Object.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/RadosModel.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/TestOpStat.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/TestRados.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/ceph_erasure_code_benchmark.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_hitset-hitset.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_osd_types-types.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osdc/$(DEPDIR)/FakeWriteback.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osdc/$(DEPDIR)/object_cacher_stress.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/system/$(DEPDIR)/cross_process_sem.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/system/$(DEPDIR)/rados_delete_pools_parallel.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/system/$(DEPDIR)/rados_list_parallel.Po at am__quote@
@@ -9573,40 +9637,40 @@ common/libcommon_crc_la-crc32c_intel_fast.lo: common/crc32c_intel_fast.c
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o common/libcommon_crc_la-crc32c_intel_fast.lo `test -f 'common/crc32c_intel_fast.c' || echo '$(srcdir)/'`common/crc32c_intel_fast.c
-osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo: osd/ErasureCodePluginJerasure/cauchy.c
- at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo `test -f 'osd/ErasureCodePluginJerasure/cauchy.c' || echo '$(s [...]
- at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Plo
- at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='osd/ErasureCodePluginJerasure/cauchy.c' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-cauchy.lo: erasure-code/jerasure/cauchy.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-cauchy.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-cauchy.lo `test -f 'erasure-code/jerasure/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/cauchy.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-cauchy.Plo
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/cauchy.c' object='erasure-code/jerasure/libec_jerasure_la-cauchy.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-cauchy.lo `test -f 'osd/ErasureCodePluginJerasure/cauchy.c' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/cauchy.c
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-cauchy.lo `test -f 'erasure-code/jerasure/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/cauchy.c
-osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo: osd/ErasureCodePluginJerasure/galois.c
- at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-galois.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo `test -f 'osd/ErasureCodePluginJerasure/galois.c' || echo '$(s [...]
- at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-galois.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-galois.Plo
- at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='osd/ErasureCodePluginJerasure/galois.c' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-galois.lo: erasure-code/jerasure/galois.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-galois.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-galois.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-galois.lo `test -f 'erasure-code/jerasure/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/galois.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-galois.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-galois.Plo
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/galois.c' object='erasure-code/jerasure/libec_jerasure_la-galois.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-galois.lo `test -f 'osd/ErasureCodePluginJerasure/galois.c' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/galois.c
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-galois.lo `test -f 'erasure-code/jerasure/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/galois.c
-osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo: osd/ErasureCodePluginJerasure/jerasure.c
- at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo `test -f 'osd/ErasureCodePluginJerasure/jerasure.c' || e [...]
- at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Plo
- at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='osd/ErasureCodePluginJerasure/jerasure.c' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-jerasure.lo: erasure-code/jerasure/jerasure.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-jerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure.c' || echo '$(srcdir)/'`erasure-code/je [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-jerasure.Plo
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure.c' object='erasure-code/jerasure/libec_jerasure_la-jerasure.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-jerasure.lo `test -f 'osd/ErasureCodePluginJerasure/jerasure.c' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/jerasure.c
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure.c
-osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo: osd/ErasureCodePluginJerasure/liberation.c
- at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-liberation.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo `test -f 'osd/ErasureCodePluginJerasure/liberation [...]
- at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-liberation.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-liberation.Plo
- at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='osd/ErasureCodePluginJerasure/liberation.c' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-liberation.lo: erasure-code/jerasure/liberation.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-liberation.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-liberation.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-liberation.lo `test -f 'erasure-code/jerasure/liberation.c' || echo '$(srcdir)/'`erasure [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-liberation.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-liberation.Plo
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/liberation.c' object='erasure-code/jerasure/libec_jerasure_la-liberation.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-liberation.lo `test -f 'osd/ErasureCodePluginJerasure/liberation.c' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/liberation.c
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-liberation.lo `test -f 'erasure-code/jerasure/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/liberation.c
-osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo: osd/ErasureCodePluginJerasure/reed_sol.c
- at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo `test -f 'osd/ErasureCodePluginJerasure/reed_sol.c' || e [...]
- at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Plo
- at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='osd/ErasureCodePluginJerasure/reed_sol.c' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-reed_sol.lo: erasure-code/jerasure/reed_sol.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-reed_sol.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-reed_sol.lo `test -f 'erasure-code/jerasure/reed_sol.c' || echo '$(srcdir)/'`erasure-code/je [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-reed_sol.Plo
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/reed_sol.c' object='erasure-code/jerasure/libec_jerasure_la-reed_sol.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-reed_sol.lo `test -f 'osd/ErasureCodePluginJerasure/reed_sol.c' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/reed_sol.c
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-reed_sol.lo `test -f 'erasure-code/jerasure/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/reed_sol.c
test/librbd/ceph_test_librbd_fsx-fsx.o: test/librbd/fsx.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) -MT test/librbd/ceph_test_librbd_fsx-fsx.o -MD -MP -MF test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Tpo -c -o test/librbd/ceph_test_librbd_fsx-fsx.o `test -f 'test/librbd/fsx.c' || echo '$(srcdir)/'`test/librbd/fsx.c
@@ -9849,54 +9913,54 @@ common/libcommon_crc_la-crc32c.lo: common/crc32c.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o common/libcommon_crc_la-crc32c.lo `test -f 'common/crc32c.cc' || echo '$(srcdir)/'`common/crc32c.cc
-test/osd/libec_example_la-ErasureCodePluginExample.lo: test/osd/ErasureCodePluginExample.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_example_la_CXXFLAGS) $(CXXFLAGS) -MT test/osd/libec_example_la-ErasureCodePluginExample.lo -MD -MP -MF test/osd/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Tpo -c -o test/osd/libec_example_la-ErasureCodePluginExample.lo `test -f 'test/osd/ErasureCodePluginExample.cc' || echo '$(srcdir)/ [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Tpo test/osd/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/ErasureCodePluginExample.cc' object='test/osd/libec_example_la-ErasureCodePluginExample.lo' libtool=yes @AMDEPBACKSLASH@
+test/erasure-code/libec_example_la-ErasureCodePluginExample.lo: test/erasure-code/ErasureCodePluginExample.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_example_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_example_la-ErasureCodePluginExample.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Tpo -c -o test/erasure-code/libec_example_la-ErasureCodePluginExample.lo `test -f 'test/erasure-code/ErasureCodeP [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Tpo test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginExample.cc' object='test/erasure-code/libec_example_la-ErasureCodePluginExample.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_example_la_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/libec_example_la-ErasureCodePluginExample.lo `test -f 'test/osd/ErasureCodePluginExample.cc' || echo '$(srcdir)/'`test/osd/ErasureCodePluginExample.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_example_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_example_la-ErasureCodePluginExample.lo `test -f 'test/erasure-code/ErasureCodePluginExample.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginExample.cc
-test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo: test/osd/ErasureCodePluginFailToInitialize.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) -MT test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo -MD -MP -MF test/osd/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Tpo -c -o test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Tpo test/osd/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/ErasureCodePluginFailToInitialize.cc' object='test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo' libtool=yes @AMDEPBACKSLASH@
+test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo: test/erasure-code/ErasureCodePluginFailToInitialize.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Tpo -c -o test/erasure-code/libec_fail_to_initialize_la-Erasur [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Tpo test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginFailToInitialize.cc' object='test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo `test -f 'test/osd/ErasureCodePluginFailToInitialize.cc' || echo '$(srcdir)/'`test/osd/ErasureCodePluginFailToInitialize.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo `test -f 'test/erasure-code/ErasureCodePluginFailToInitialize.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginFailToInitialize.cc
-test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo: test/osd/ErasureCodePluginFailToRegister.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) -MT test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo -MD -MP -MF test/osd/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Tpo -c -o test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo `test -f ' [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Tpo test/osd/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/ErasureCodePluginFailToRegister.cc' object='test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo' libtool=yes @AMDEPBACKSLASH@
+test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo: test/erasure-code/ErasureCodePluginFailToRegister.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Tpo -c -o test/erasure-code/libec_fail_to_register_la-ErasureCodePluginF [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Tpo test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginFailToRegister.cc' object='test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo `test -f 'test/osd/ErasureCodePluginFailToRegister.cc' || echo '$(srcdir)/'`test/osd/ErasureCodePluginFailToRegister.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo `test -f 'test/erasure-code/ErasureCodePluginFailToRegister.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginFailToRegister.cc
-test/osd/libec_hangs_la-ErasureCodePluginHangs.lo: test/osd/ErasureCodePluginHangs.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) -MT test/osd/libec_hangs_la-ErasureCodePluginHangs.lo -MD -MP -MF test/osd/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Tpo -c -o test/osd/libec_hangs_la-ErasureCodePluginHangs.lo `test -f 'test/osd/ErasureCodePluginHangs.cc' || echo '$(srcdir)/'`test/osd/Erasu [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Tpo test/osd/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/ErasureCodePluginHangs.cc' object='test/osd/libec_hangs_la-ErasureCodePluginHangs.lo' libtool=yes @AMDEPBACKSLASH@
+test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo: test/erasure-code/ErasureCodePluginHangs.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Tpo -c -o test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo `test -f 'test/erasure-code/ErasureCodePluginHangs.cc' [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Tpo test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginHangs.cc' object='test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/libec_hangs_la-ErasureCodePluginHangs.lo `test -f 'test/osd/ErasureCodePluginHangs.cc' || echo '$(srcdir)/'`test/osd/ErasureCodePluginHangs.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo `test -f 'test/erasure-code/ErasureCodePluginHangs.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginHangs.cc
-osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo: osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure. [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo: erasure-code/jerasure/ErasureCodePluginJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo `test -f 'erasure-cod [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginJerasure.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo `test -f 'osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc
-osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo: osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -MT osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Tpo -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo `test -f 'osd/E [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc' object='osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo' libtool=yes @AMDEPBACKSLASH@
+erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo: erasure-code/jerasure/ErasureCodeJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/Erasure [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodeJerasure.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/ErasureCodePluginJerasure/libec_jerasure_la-ErasureCodeJerasure.lo `test -f 'osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc
-test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: test/osd/ErasureCodePluginMissingEntryPoint.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -MT test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo -MD -MP -MF test/osd/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Tpo -c -o test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEnt [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Tpo test/osd/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Plo
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/ErasureCodePluginMissingEntryPoint.cc' object='test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo' libtool=yes @AMDEPBACKSLASH@
+test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Tpo -c -o test/erasure-code/libec_missing_entry_point_la- [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Tpo test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginMissingEntryPoint.cc' object='test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo `test -f 'test/osd/ErasureCodePluginMissingEntryPoint.cc' || echo '$(srcdir)/'`test/osd/ErasureCodePluginMissingEntryPoint.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo `test -f 'test/erasure-code/ErasureCodePluginMissingEntryPoint.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
librados/librados_la-librados.lo: librados/librados.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-librados.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-librados.Tpo -c -o librados/librados_la-librados.lo `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc
@@ -9926,6 +9990,20 @@ librados/librados_la-snap_set_diff.lo: librados/snap_set_diff.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-snap_set_diff.lo `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc
+test/librados/libradostest_la-test.lo: test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -MT test/librados/libradostest_la-test.lo -MD -MP -MF test/librados/$(DEPDIR)/libradostest_la-test.Tpo -c -o test/librados/libradostest_la-test.lo `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/libradostest_la-test.Tpo test/librados/$(DEPDIR)/libradostest_la-test.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/libradostest_la-test.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/libradostest_la-test.lo `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+
+test/librados/libradostest_la-TestCase.lo: test/librados/TestCase.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -MT test/librados/libradostest_la-TestCase.lo -MD -MP -MF test/librados/$(DEPDIR)/libradostest_la-TestCase.Tpo -c -o test/librados/libradostest_la-TestCase.lo `test -f 'test/librados/TestCase.cc' || echo '$(srcdir)/'`test/librados/TestCase.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/libradostest_la-TestCase.Tpo test/librados/$(DEPDIR)/libradostest_la-TestCase.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/TestCase.cc' object='test/librados/libradostest_la-TestCase.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/libradostest_la-TestCase.lo `test -f 'test/librados/TestCase.cc' || echo '$(srcdir)/'`test/librados/TestCase.cc
+
rgw/librgw_la-librgw.lo: rgw/librgw.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-librgw.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-librgw.Tpo -c -o rgw/librgw_la-librgw.lo `test -f 'rgw/librgw.cc' || echo '$(srcdir)/'`rgw/librgw.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-librgw.Tpo rgw/$(DEPDIR)/librgw_la-librgw.Plo
@@ -10269,20 +10347,6 @@ test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj: test/cls_hello/test_cls_h
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj `if test -f 'test/cls_hello/test_cls_hello.cc'; then $(CYGPATH_W) 'test/cls_hello/test_cls_hello.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_hello/test_cls_hello.cc'; fi`
-test/librados/ceph_test_cls_hello-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_hello-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Tpo -c -o test/librados/ceph_test_cls_hello-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_hello-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_hello-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_hello-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_hello-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Tpo -c -o test/librados/ceph_test_cls_hello-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_hello-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_hello-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_hello-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_lock/ceph_test_cls_lock-test_cls_lock.o: test/cls_lock/test_cls_lock.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -MT test/cls_lock/ceph_test_cls_lock-test_cls_lock.o -MD -MP -MF test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Tpo -c -o test/cls_lock/ceph_test_cls_lock-test_cls_lock.o `test -f 'test/cls_lock/test_cls_lock.cc' || echo '$(srcdir)/'`test/cls_lock/test_cls_lock.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Tpo test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Po
@@ -10297,20 +10361,6 @@ test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj: test/cls_lock/test_cls_lock.
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj `if test -f 'test/cls_lock/test_cls_lock.cc'; then $(CYGPATH_W) 'test/cls_lock/test_cls_lock.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_lock/test_cls_lock.cc'; fi`
-test/librados/ceph_test_cls_lock-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_lock-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Tpo -c -o test/librados/ceph_test_cls_lock-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_lock-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_lock-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_lock-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_lock-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Tpo -c -o test/librados/ceph_test_cls_lock-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_lock-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_lock-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_lock-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_log/ceph_test_cls_log-test_cls_log.o: test/cls_log/test_cls_log.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -MT test/cls_log/ceph_test_cls_log-test_cls_log.o -MD -MP -MF test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Tpo -c -o test/cls_log/ceph_test_cls_log-test_cls_log.o `test -f 'test/cls_log/test_cls_log.cc' || echo '$(srcdir)/'`test/cls_log/test_cls_log.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Tpo test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Po
@@ -10325,20 +10375,6 @@ test/cls_log/ceph_test_cls_log-test_cls_log.obj: test/cls_log/test_cls_log.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_log/ceph_test_cls_log-test_cls_log.obj `if test -f 'test/cls_log/test_cls_log.cc'; then $(CYGPATH_W) 'test/cls_log/test_cls_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_log/test_cls_log.cc'; fi`
-test/librados/ceph_test_cls_log-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_log-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_log-test.Tpo -c -o test/librados/ceph_test_cls_log-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_log-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_log-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_log-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_log-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_log-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_log-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_log-test.Tpo -c -o test/librados/ceph_test_cls_log-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_log-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_log-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_log-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_log-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o: test/cls_rbd/test_cls_rbd.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -MT test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o -MD -MP -MF test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Tpo -c -o test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o `test -f 'test/cls_rbd/test_cls_rbd.cc' || echo '$(srcdir)/'`test/cls_rbd/test_cls_rbd.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Tpo test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Po
@@ -10353,20 +10389,6 @@ test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj: test/cls_rbd/test_cls_rbd.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj `if test -f 'test/cls_rbd/test_cls_rbd.cc'; then $(CYGPATH_W) 'test/cls_rbd/test_cls_rbd.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_rbd/test_cls_rbd.cc'; fi`
-test/librados/ceph_test_cls_rbd-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_rbd-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Tpo -c -o test/librados/ceph_test_cls_rbd-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_rbd-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_rbd-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_rbd-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_rbd-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Tpo -c -o test/librados/ceph_test_cls_rbd-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_rbd-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_rbd-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_rbd-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o: test/cls_refcount/test_cls_refcount.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -MT test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o -MD -MP -MF test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Tpo -c -o test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o `test -f 'test/cls_refcount/test_cls_refcount.cc' || echo '$(srcdir)/'`test/cls_refcount/test_cls_refcount.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Tpo test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Po
@@ -10381,20 +10403,6 @@ test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj: test/cls_refcoun
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj `if test -f 'test/cls_refcount/test_cls_refcount.cc'; then $(CYGPATH_W) 'test/cls_refcount/test_cls_refcount.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_refcount/test_cls_refcount.cc'; fi`
-test/librados/ceph_test_cls_refcount-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_refcount-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Tpo -c -o test/librados/ceph_test_cls_refcount-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_refcount-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_refcount-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_refcount-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_refcount-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Tpo -c -o test/librados/ceph_test_cls_refcount-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_refcount-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_refcount-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_refcount-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o: test/cls_replica_log/test_cls_replica_log.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -MT test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o -MD -MP -MF test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Tpo -c -o test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o `test -f 'test/cls_replica_log/test_cls_replica_log.cc' || echo '$(srcdir)/'`test/cls_repli [...]
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Tpo test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Po
@@ -10409,20 +10417,6 @@ test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj: test/cl
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj `if test -f 'test/cls_replica_log/test_cls_replica_log.cc'; then $(CYGPATH_W) 'test/cls_replica_log/test_cls_replica_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_replica_log/test_cls_replica_log.cc'; fi`
-test/librados/ceph_test_cls_replica_log-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_replica_log-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Tpo -c -o test/librados/ceph_test_cls_replica_log-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_replica_log-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_replica_log-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_replica_log-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_replica_log-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Tpo -c -o test/librados/ceph_test_cls_replica_log-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_replica_log-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_replica_log-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_replica_log-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o: test/cls_rgw/test_cls_rgw.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -MT test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o -MD -MP -MF test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Tpo -c -o test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o `test -f 'test/cls_rgw/test_cls_rgw.cc' || echo '$(srcdir)/'`test/cls_rgw/test_cls_rgw.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Tpo test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Po
@@ -10437,20 +10431,6 @@ test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj: test/cls_rgw/test_cls_rgw.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj `if test -f 'test/cls_rgw/test_cls_rgw.cc'; then $(CYGPATH_W) 'test/cls_rgw/test_cls_rgw.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_rgw/test_cls_rgw.cc'; fi`
-test/librados/ceph_test_cls_rgw-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_rgw-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Tpo -c -o test/librados/ceph_test_cls_rgw-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_rgw-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_rgw-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_rgw-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_rgw-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Tpo -c -o test/librados/ceph_test_cls_rgw-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_rgw-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_rgw-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_rgw-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/ceph_test_cls_rgw_log-test_rgw_admin_log.o: test/test_rgw_admin_log.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_log-test_rgw_admin_log.o -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Tpo -c -o test/ceph_test_cls_rgw_log-test_rgw_admin_log.o `test -f 'test/test_rgw_admin_log.cc' || echo '$(srcdir)/'`test/test_rgw_admin_log.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Po
@@ -10507,20 +10487,6 @@ test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj: test/cls_statelo
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj `if test -f 'test/cls_statelog/test_cls_statelog.cc'; then $(CYGPATH_W) 'test/cls_statelog/test_cls_statelog.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_statelog/test_cls_statelog.cc'; fi`
-test/librados/ceph_test_cls_statelog-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_statelog-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Tpo -c -o test/librados/ceph_test_cls_statelog-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_statelog-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_statelog-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_statelog-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_statelog-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Tpo -c -o test/librados/ceph_test_cls_statelog-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_statelog-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_statelog-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_statelog-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/cls_version/ceph_test_cls_version-test_cls_version.o: test/cls_version/test_cls_version.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -MT test/cls_version/ceph_test_cls_version-test_cls_version.o -MD -MP -MF test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Tpo -c -o test/cls_version/ceph_test_cls_version-test_cls_version.o `test -f 'test/cls_version/test_cls_version.cc' || echo '$(srcdir)/'`test/cls_version/test_cls_version.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Tpo test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Po
@@ -10535,20 +10501,6 @@ test/cls_version/ceph_test_cls_version-test_cls_version.obj: test/cls_version/te
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_version/ceph_test_cls_version-test_cls_version.obj `if test -f 'test/cls_version/test_cls_version.cc'; then $(CYGPATH_W) 'test/cls_version/test_cls_version.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_version/test_cls_version.cc'; fi`
-test/librados/ceph_test_cls_version-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_version-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_version-test.Tpo -c -o test/librados/ceph_test_cls_version-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_version-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_version-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_version-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_version-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_cls_version-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_cls_version-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_cls_version-test.Tpo -c -o test/librados/ceph_test_cls_version-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_cls_version-test.Tpo test/librados/$(DEPDIR)/ceph_test_cls_version-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_cls_version-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_cls_version-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/ceph_test_cors-test_cors.o: test/test_cors.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cors-test_cors.o -MD -MP -MF test/$(DEPDIR)/ceph_test_cors-test_cors.Tpo -c -o test/ceph_test_cors-test_cors.o `test -f 'test/test_cors.cc' || echo '$(srcdir)/'`test/test_cors.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cors-test_cors.Tpo test/$(DEPDIR)/ceph_test_cors-test_cors.Po
@@ -10577,20 +10529,6 @@ test/ceph_test_filejournal-test_filejournal.obj: test/test_filejournal.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_filejournal-test_filejournal.obj `if test -f 'test/test_filejournal.cc'; then $(CYGPATH_W) 'test/test_filejournal.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_filejournal.cc'; fi`
-test/filestore/ceph_test_filestore-store_test.o: test/filestore/store_test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filestore_CXXFLAGS) $(CXXFLAGS) -MT test/filestore/ceph_test_filestore-store_test.o -MD -MP -MF test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Tpo -c -o test/filestore/ceph_test_filestore-store_test.o `test -f 'test/filestore/store_test.cc' || echo '$(srcdir)/'`test/filestore/store_test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Tpo test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/filestore/store_test.cc' object='test/filestore/ceph_test_filestore-store_test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filestore_CXXFLAGS) $(CXXFLAGS) -c -o test/filestore/ceph_test_filestore-store_test.o `test -f 'test/filestore/store_test.cc' || echo '$(srcdir)/'`test/filestore/store_test.cc
-
-test/filestore/ceph_test_filestore-store_test.obj: test/filestore/store_test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filestore_CXXFLAGS) $(CXXFLAGS) -MT test/filestore/ceph_test_filestore-store_test.obj -MD -MP -MF test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Tpo -c -o test/filestore/ceph_test_filestore-store_test.obj `if test -f 'test/filestore/store_test.cc'; then $(CYGPATH_W) 'test/filestore/store_test.cc'; else $(CYGPATH_W) '$(srcdir)/test/filestore/store_test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Tpo test/filestore/$(DEPDIR)/ceph_test_filestore-store_test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/filestore/store_test.cc' object='test/filestore/ceph_test_filestore-store_test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filestore_CXXFLAGS) $(CXXFLAGS) -c -o test/filestore/ceph_test_filestore-store_test.obj `if test -f 'test/filestore/store_test.cc'; then $(CYGPATH_W) 'test/filestore/store_test.cc'; else $(CYGPATH_W) '$(srcdir)/test/filestore/store_test.cc'; fi`
-
test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o: test/ObjectMap/test_keyvaluedb_atomicity.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o `test -f 'test/ObjectMap/test_keyvaluedb_atomicity.cc' || echo '$(srcdir)/ [...]
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Po
@@ -10703,20 +10641,6 @@ test/librbd/ceph_test_librbd-test_librbd.obj: test/librbd/test_librbd.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librbd/ceph_test_librbd-test_librbd.obj `if test -f 'test/librbd/test_librbd.cc'; then $(CYGPATH_W) 'test/librbd/test_librbd.cc'; else $(CYGPATH_W) '$(srcdir)/test/librbd/test_librbd.cc'; fi`
-test/librados/ceph_test_librbd-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_librbd-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_librbd-test.Tpo -c -o test/librados/ceph_test_librbd-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_librbd-test.Tpo test/librados/$(DEPDIR)/ceph_test_librbd-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_librbd-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_librbd-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_librbd-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_librbd-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_librbd-test.Tpo -c -o test/librados/ceph_test_librbd-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_librbd-test.Tpo test/librados/$(DEPDIR)/ceph_test_librbd-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_librbd-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_librbd-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/ObjectMap/ceph_test_object_map-test_object_map.o: test/ObjectMap/test_object_map.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_object_map-test_object_map.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Tpo -c -o test/ObjectMap/ceph_test_object_map-test_object_map.o `test -f 'test/ObjectMap/test_object_map.cc' || echo '$(srcdir)/'`test/ObjectMap/test_object_map.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Po
@@ -10745,6 +10669,20 @@ test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj: test/ObjectMap/KeyValu
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj `if test -f 'test/ObjectMap/KeyValueDBMemory.cc'; then $(CYGPATH_W) 'test/ObjectMap/KeyValueDBMemory.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/KeyValueDBMemory.cc'; fi`
+test/objectstore/ceph_test_objectstore-store_test.o: test/objectstore/store_test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/ceph_test_objectstore-store_test.o -MD -MP -MF test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo -c -o test/objectstore/ceph_test_objectstore-store_test.o `test -f 'test/objectstore/store_test.cc' || echo '$(srcdir)/'`test/objectstore/store_test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/store_test.cc' object='test/objectstore/ceph_test_objectstore-store_test.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/ceph_test_objectstore-store_test.o `test -f 'test/objectstore/store_test.cc' || echo '$(srcdir)/'`test/objectstore/store_test.cc
+
+test/objectstore/ceph_test_objectstore-store_test.obj: test/objectstore/store_test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/ceph_test_objectstore-store_test.obj -MD -MP -MF test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo -c -o test/objectstore/ceph_test_objectstore-store_test.obj `if test -f 'test/objectstore/store_test.cc'; then $(CYGPATH_W) 'test/objectstore/store_test.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/s [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/store_test.cc' object='test/objectstore/ceph_test_objectstore-store_test.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/ceph_test_objectstore-store_test.obj `if test -f 'test/objectstore/store_test.cc'; then $(CYGPATH_W) 'test/objectstore/store_test.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/store_test.cc'; fi`
+
test/librados/ceph_test_rados_api_aio-aio.o: test/librados/aio.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_aio-aio.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Tpo -c -o test/librados/ceph_test_rados_api_aio-aio.o `test -f 'test/librados/aio.cc' || echo '$(srcdir)/'`test/librados/aio.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Po
@@ -10759,19 +10697,19 @@ test/librados/ceph_test_rados_api_aio-aio.obj: test/librados/aio.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_aio-aio.obj `if test -f 'test/librados/aio.cc'; then $(CYGPATH_W) 'test/librados/aio.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/aio.cc'; fi`
-test/librados/ceph_test_rados_api_aio-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_aio-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Tpo -c -o test/librados/ceph_test_rados_api_aio-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_aio-test.o' libtool=no @AMDEPBACKSLASH@
+test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o: test/librados/c_read_operations.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o `test -f 'test/librados/c_read_operations.cc' || echo '$(srcdir)/'`test/l [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_read_operations.cc' object='test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_aio-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o `test -f 'test/librados/c_read_operations.cc' || echo '$(srcdir)/'`test/librados/c_read_operations.cc
-test/librados/ceph_test_rados_api_aio-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_aio-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Tpo -c -o test/librados/ceph_test_rados_api_aio-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_aio-test.obj' libtool=no @AMDEPBACKSLASH@
+test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj: test/librados/c_read_operations.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj `if test -f 'test/librados/c_read_operations.cc'; then $(CYGPATH_W) ' [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_read_operations.cc' object='test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_aio-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj `if test -f 'test/librados/c_read_operations.cc'; then $(CYGPATH_W) 'test/librados/c_read_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_read_operations.cc'; fi`
test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o: test/librados/c_write_operations.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o `test -f 'test/librados/c_write_operations.cc' || echo '$(srcdir)/ [...]
@@ -10787,20 +10725,6 @@ test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj: tes
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj `if test -f 'test/librados/c_write_operations.cc'; then $(CYGPATH_W) 'test/librados/c_write_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_write_operations.cc'; fi`
-test/librados/ceph_test_rados_api_c_write_operations-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_c_write_operations-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_c_write_operations-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(src [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_c_write_operations-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_cls-cls.o: test/librados/cls.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cls-cls.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo -c -o test/librados/ceph_test_rados_api_cls-cls.o `test -f 'test/librados/cls.cc' || echo '$(srcdir)/'`test/librados/cls.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po
@@ -10815,20 +10739,6 @@ test/librados/ceph_test_rados_api_cls-cls.obj: test/librados/cls.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cls-cls.obj `if test -f 'test/librados/cls.cc'; then $(CYGPATH_W) 'test/librados/cls.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/cls.cc'; fi`
-test/librados/ceph_test_rados_api_cls-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cls-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Tpo -c -o test/librados/ceph_test_rados_api_cls-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_cls-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cls-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_cls-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cls-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Tpo -c -o test/librados/ceph_test_rados_api_cls-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_cls-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cls-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_cmd-cmd.o: test/librados/cmd.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cmd-cmd.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Tpo -c -o test/librados/ceph_test_rados_api_cmd-cmd.o `test -f 'test/librados/cmd.cc' || echo '$(srcdir)/'`test/librados/cmd.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Po
@@ -10843,20 +10753,6 @@ test/librados/ceph_test_rados_api_cmd-cmd.obj: test/librados/cmd.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cmd-cmd.obj `if test -f 'test/librados/cmd.cc'; then $(CYGPATH_W) 'test/librados/cmd.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/cmd.cc'; fi`
-test/librados/ceph_test_rados_api_cmd-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cmd-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Tpo -c -o test/librados/ceph_test_rados_api_cmd-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_cmd-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cmd-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_cmd-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cmd-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Tpo -c -o test/librados/ceph_test_rados_api_cmd-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_cmd-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cmd-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_io-io.o: test/librados/io.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_io-io.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Tpo -c -o test/librados/ceph_test_rados_api_io-io.o `test -f 'test/librados/io.cc' || echo '$(srcdir)/'`test/librados/io.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Po
@@ -10871,20 +10767,6 @@ test/librados/ceph_test_rados_api_io-io.obj: test/librados/io.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_io-io.obj `if test -f 'test/librados/io.cc'; then $(CYGPATH_W) 'test/librados/io.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/io.cc'; fi`
-test/librados/ceph_test_rados_api_io-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_io-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Tpo -c -o test/librados/ceph_test_rados_api_io-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_io-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_io-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_io-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_io-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Tpo -c -o test/librados/ceph_test_rados_api_io-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_io-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_io-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_io-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_list-list.o: test/librados/list.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_list-list.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Tpo -c -o test/librados/ceph_test_rados_api_list-list.o `test -f 'test/librados/list.cc' || echo '$(srcdir)/'`test/librados/list.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Po
@@ -10899,20 +10781,6 @@ test/librados/ceph_test_rados_api_list-list.obj: test/librados/list.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_list-list.obj `if test -f 'test/librados/list.cc'; then $(CYGPATH_W) 'test/librados/list.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/list.cc'; fi`
-test/librados/ceph_test_rados_api_list-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_list-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Tpo -c -o test/librados/ceph_test_rados_api_list-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_list-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_list-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_list-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_list-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Tpo -c -o test/librados/ceph_test_rados_api_list-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_list-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_list-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_list-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_lock-lock.o: test/librados/lock.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_lock-lock.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Tpo -c -o test/librados/ceph_test_rados_api_lock-lock.o `test -f 'test/librados/lock.cc' || echo '$(srcdir)/'`test/librados/lock.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Po
@@ -10927,20 +10795,6 @@ test/librados/ceph_test_rados_api_lock-lock.obj: test/librados/lock.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_lock-lock.obj `if test -f 'test/librados/lock.cc'; then $(CYGPATH_W) 'test/librados/lock.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/lock.cc'; fi`
-test/librados/ceph_test_rados_api_lock-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_lock-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Tpo -c -o test/librados/ceph_test_rados_api_lock-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_lock-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_lock-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_lock-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_lock-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Tpo -c -o test/librados/ceph_test_rados_api_lock-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_lock-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_lock-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_lock-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_misc-misc.o: test/librados/misc.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_misc-misc.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Tpo -c -o test/librados/ceph_test_rados_api_misc-misc.o `test -f 'test/librados/misc.cc' || echo '$(srcdir)/'`test/librados/misc.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Po
@@ -10955,20 +10809,6 @@ test/librados/ceph_test_rados_api_misc-misc.obj: test/librados/misc.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_misc-misc.obj `if test -f 'test/librados/misc.cc'; then $(CYGPATH_W) 'test/librados/misc.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/misc.cc'; fi`
-test/librados/ceph_test_rados_api_misc-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_misc-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Tpo -c -o test/librados/ceph_test_rados_api_misc-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_misc-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_misc-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_misc-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_misc-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Tpo -c -o test/librados/ceph_test_rados_api_misc-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_misc-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_misc-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_misc-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_pool-pool.o: test/librados/pool.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_pool-pool.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Tpo -c -o test/librados/ceph_test_rados_api_pool-pool.o `test -f 'test/librados/pool.cc' || echo '$(srcdir)/'`test/librados/pool.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Po
@@ -10983,20 +10823,6 @@ test/librados/ceph_test_rados_api_pool-pool.obj: test/librados/pool.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_pool-pool.obj `if test -f 'test/librados/pool.cc'; then $(CYGPATH_W) 'test/librados/pool.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/pool.cc'; fi`
-test/librados/ceph_test_rados_api_pool-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_pool-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Tpo -c -o test/librados/ceph_test_rados_api_pool-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_pool-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_pool-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_pool-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_pool-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Tpo -c -o test/librados/ceph_test_rados_api_pool-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_pool-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_pool-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_pool-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_snapshots-snapshots.o: test/librados/snapshots.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_snapshots-snapshots.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Tpo -c -o test/librados/ceph_test_rados_api_snapshots-snapshots.o `test -f 'test/librados/snapshots.cc' || echo '$(srcdir)/'`test/librados/snapshots.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Po
@@ -11011,20 +10837,6 @@ test/librados/ceph_test_rados_api_snapshots-snapshots.obj: test/librados/snapsho
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_snapshots-snapshots.obj `if test -f 'test/librados/snapshots.cc'; then $(CYGPATH_W) 'test/librados/snapshots.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/snapshots.cc'; fi`
-test/librados/ceph_test_rados_api_snapshots-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_snapshots-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Tpo -c -o test/librados/ceph_test_rados_api_snapshots-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_snapshots-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_snapshots-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_snapshots-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_snapshots-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Tpo -c -o test/librados/ceph_test_rados_api_snapshots-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_snapshots-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_snapshots-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_stat-stat.o: test/librados/stat.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_stat-stat.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Tpo -c -o test/librados/ceph_test_rados_api_stat-stat.o `test -f 'test/librados/stat.cc' || echo '$(srcdir)/'`test/librados/stat.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Po
@@ -11039,20 +10851,6 @@ test/librados/ceph_test_rados_api_stat-stat.obj: test/librados/stat.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_stat-stat.obj `if test -f 'test/librados/stat.cc'; then $(CYGPATH_W) 'test/librados/stat.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/stat.cc'; fi`
-test/librados/ceph_test_rados_api_stat-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_stat-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Tpo -c -o test/librados/ceph_test_rados_api_stat-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_stat-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_stat-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_stat-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_stat-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Tpo -c -o test/librados/ceph_test_rados_api_stat-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_stat-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_stat-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/librados/ceph_test_rados_api_tier-tier.o: test/librados/tier.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-tier.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo -c -o test/librados/ceph_test_rados_api_tier-tier.o `test -f 'test/librados/tier.cc' || echo '$(srcdir)/'`test/librados/tier.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po
@@ -11067,20 +10865,6 @@ test/librados/ceph_test_rados_api_tier-tier.obj: test/librados/tier.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-tier.obj `if test -f 'test/librados/tier.cc'; then $(CYGPATH_W) 'test/librados/tier.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/tier.cc'; fi`
-test/librados/ceph_test_rados_api_tier-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo -c -o test/librados/ceph_test_rados_api_tier-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_tier-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_rados_api_tier-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo -c -o test/librados/ceph_test_rados_api_tier-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_tier-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
osd/ceph_test_rados_api_tier-HitSet.o: osd/HitSet.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT osd/ceph_test_rados_api_tier-HitSet.o -MD -MP -MF osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo -c -o osd/ceph_test_rados_api_tier-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po
@@ -11109,19 +10893,19 @@ test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj: test/librados/w
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj `if test -f 'test/librados/watch_notify.cc'; then $(CYGPATH_W) 'test/librados/watch_notify.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/watch_notify.cc'; fi`
-test/librados/ceph_test_rados_api_watch_notify-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_watch_notify-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Tpo -c -o test/librados/ceph_test_rados_api_watch_notify-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_watch_notify-test.o' libtool=no @AMDEPBACKSLASH@
+test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o: test/rgw/test_rgw_manifest.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -MT test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o -MD -MP -MF test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o `test -f 'test/rgw/test_rgw_manifest.cc' || echo '$(srcdir)/'`test/rgw/test_rgw_manifest.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/rgw/test_rgw_manifest.cc' object='test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_watch_notify-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o `test -f 'test/rgw/test_rgw_manifest.cc' || echo '$(srcdir)/'`test/rgw/test_rgw_manifest.cc
-test/librados/ceph_test_rados_api_watch_notify-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_watch_notify-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Tpo -c -o test/librados/ceph_test_rados_api_watch_notify-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test. [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_watch_notify-test.obj' libtool=no @AMDEPBACKSLASH@
+test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj: test/rgw/test_rgw_manifest.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -MT test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj -MD -MP -MF test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj `if test -f 'test/rgw/test_rgw_manifest.cc'; then $(CYGPATH_W) 'test/rgw/test_rgw_manifest.cc'; else $(CYGPATH_W) '$(srcdir)/test/rgw/test_rgw_m [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/rgw/test_rgw_manifest.cc' object='test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_watch_notify-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj `if test -f 'test/rgw/test_rgw_manifest.cc'; then $(CYGPATH_W) 'test/rgw/test_rgw_manifest.cc'; else $(CYGPATH_W) '$(srcdir)/test/rgw/test_rgw_manifest.cc'; fi`
test/ceph_test_snap_mapper-test_snap_mapper.o: test/test_snap_mapper.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_snap_mapper-test_snap_mapper.o -MD -MP -MF test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Tpo -c -o test/ceph_test_snap_mapper-test_snap_mapper.o `test -f 'test/test_snap_mapper.cc' || echo '$(srcdir)/'`test/test_snap_mapper.cc
@@ -11151,20 +10935,6 @@ test/ceph_test_stress_watch-test_stress_watch.obj: test/test_stress_watch.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_stress_watch-test_stress_watch.obj `if test -f 'test/test_stress_watch.cc'; then $(CYGPATH_W) 'test/test_stress_watch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_stress_watch.cc'; fi`
-test/librados/ceph_test_stress_watch-test.o: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_stress_watch-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Tpo -c -o test/librados/ceph_test_stress_watch-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Tpo test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_stress_watch-test.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_stress_watch-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
-
-test/librados/ceph_test_stress_watch-test.obj: test/librados/test.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_stress_watch-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Tpo -c -o test/librados/ceph_test_stress_watch-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Tpo test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_stress_watch-test.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_stress_watch-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
-
test/ceph_xattr_bench-xattr_bench.o: test/xattr_bench.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_xattr_bench-xattr_bench.o -MD -MP -MF test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Tpo -c -o test/ceph_xattr_bench-xattr_bench.o `test -f 'test/xattr_bench.cc' || echo '$(srcdir)/'`test/xattr_bench.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Tpo test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Po
@@ -11795,6 +11565,20 @@ common/test_build_libcommon-dout.obj: common/dout.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-dout.obj `if test -f 'common/dout.cc'; then $(CYGPATH_W) 'common/dout.cc'; else $(CYGPATH_W) '$(srcdir)/common/dout.cc'; fi`
+common/test_build_libcommon-histogram.o: common/histogram.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-histogram.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-histogram.Tpo -c -o common/test_build_libcommon-histogram.o `test -f 'common/histogram.cc' || echo '$(srcdir)/'`common/histogram.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-histogram.Tpo common/$(DEPDIR)/test_build_libcommon-histogram.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/histogram.cc' object='common/test_build_libcommon-histogram.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-histogram.o `test -f 'common/histogram.cc' || echo '$(srcdir)/'`common/histogram.cc
+
+common/test_build_libcommon-histogram.obj: common/histogram.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-histogram.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-histogram.Tpo -c -o common/test_build_libcommon-histogram.obj `if test -f 'common/histogram.cc'; then $(CYGPATH_W) 'common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/common/histogram.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-histogram.Tpo common/$(DEPDIR)/test_build_libcommon-histogram.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/histogram.cc' object='common/test_build_libcommon-histogram.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-histogram.obj `if test -f 'common/histogram.cc'; then $(CYGPATH_W) 'common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/common/histogram.cc'; fi`
+
common/test_build_libcommon-signal.o: common/signal.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-signal.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-signal.Tpo -c -o common/test_build_libcommon-signal.o `test -f 'common/signal.cc' || echo '$(srcdir)/'`common/signal.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-signal.Tpo common/$(DEPDIR)/test_build_libcommon-signal.Po
@@ -12215,6 +11999,20 @@ osd/test_build_libcommon-osd_types.obj: osd/osd_types.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-osd_types.obj `if test -f 'osd/osd_types.cc'; then $(CYGPATH_W) 'osd/osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/osd/osd_types.cc'; fi`
+osd/test_build_libcommon-ECMsgTypes.o: osd/ECMsgTypes.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-ECMsgTypes.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo -c -o osd/test_build_libcommon-ECMsgTypes.o `test -f 'osd/ECMsgTypes.cc' || echo '$(srcdir)/'`osd/ECMsgTypes.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECMsgTypes.cc' object='osd/test_build_libcommon-ECMsgTypes.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-ECMsgTypes.o `test -f 'osd/ECMsgTypes.cc' || echo '$(srcdir)/'`osd/ECMsgTypes.cc
+
+osd/test_build_libcommon-ECMsgTypes.obj: osd/ECMsgTypes.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-ECMsgTypes.obj -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo -c -o osd/test_build_libcommon-ECMsgTypes.obj `if test -f 'osd/ECMsgTypes.cc'; then $(CYGPATH_W) 'osd/ECMsgTypes.cc'; else $(CYGPATH_W) '$(srcdir)/osd/ECMsgTypes.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECMsgTypes.cc' object='osd/test_build_libcommon-ECMsgTypes.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-ECMsgTypes.obj `if test -f 'osd/ECMsgTypes.cc'; then $(CYGPATH_W) 'osd/ECMsgTypes.cc'; else $(CYGPATH_W) '$(srcdir)/osd/ECMsgTypes.cc'; fi`
+
osd/test_build_libcommon-HitSet.o: osd/HitSet.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-HitSet.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo -c -o osd/test_build_libcommon-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo osd/$(DEPDIR)/test_build_libcommon-HitSet.Po
@@ -12943,19 +12741,19 @@ test/unittest_ceph_crypto-ceph_crypto.obj: test/ceph_crypto.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_crypto-ceph_crypto.obj `if test -f 'test/ceph_crypto.cc'; then $(CYGPATH_W) 'test/ceph_crypto.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_crypto.cc'; fi`
-test/filestore/unittest_chain_xattr-chain_xattr.o: test/filestore/chain_xattr.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -MT test/filestore/unittest_chain_xattr-chain_xattr.o -MD -MP -MF test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo -c -o test/filestore/unittest_chain_xattr-chain_xattr.o `test -f 'test/filestore/chain_xattr.cc' || echo '$(srcdir)/'`test/filestore/chain_xattr.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/filestore/chain_xattr.cc' object='test/filestore/unittest_chain_xattr-chain_xattr.o' libtool=no @AMDEPBACKSLASH@
+test/objectstore/unittest_chain_xattr-chain_xattr.o: test/objectstore/chain_xattr.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/unittest_chain_xattr-chain_xattr.o -MD -MP -MF test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo -c -o test/objectstore/unittest_chain_xattr-chain_xattr.o `test -f 'test/objectstore/chain_xattr.cc' || echo '$(srcdir)/'`test/objectstore/chain_xattr.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/chain_xattr.cc' object='test/objectstore/unittest_chain_xattr-chain_xattr.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/filestore/unittest_chain_xattr-chain_xattr.o `test -f 'test/filestore/chain_xattr.cc' || echo '$(srcdir)/'`test/filestore/chain_xattr.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/unittest_chain_xattr-chain_xattr.o `test -f 'test/objectstore/chain_xattr.cc' || echo '$(srcdir)/'`test/objectstore/chain_xattr.cc
-test/filestore/unittest_chain_xattr-chain_xattr.obj: test/filestore/chain_xattr.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -MT test/filestore/unittest_chain_xattr-chain_xattr.obj -MD -MP -MF test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo -c -o test/filestore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/filestore/chain_xattr.cc'; then $(CYGPATH_W) 'test/filestore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/filestore/chain_xattr. [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo test/filestore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/filestore/chain_xattr.cc' object='test/filestore/unittest_chain_xattr-chain_xattr.obj' libtool=no @AMDEPBACKSLASH@
+test/objectstore/unittest_chain_xattr-chain_xattr.obj: test/objectstore/chain_xattr.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/unittest_chain_xattr-chain_xattr.obj -MD -MP -MF test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo -c -o test/objectstore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/objectstore/chain_xattr.cc'; then $(CYGPATH_W) 'test/objectstore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/ [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/chain_xattr.cc' object='test/objectstore/unittest_chain_xattr-chain_xattr.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/filestore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/filestore/chain_xattr.cc'; then $(CYGPATH_W) 'test/filestore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/filestore/chain_xattr.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/objectstore/chain_xattr.cc'; then $(CYGPATH_W) 'test/objectstore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/chain_xattr.cc'; fi`
test/common/unittest_config-test_config.o: test/common/test_config.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_config-test_config.o -MD -MP -MF test/common/$(DEPDIR)/unittest_config-test_config.Tpo -c -o test/common/unittest_config-test_config.o `test -f 'test/common/test_config.cc' || echo '$(srcdir)/'`test/common/test_config.cc
@@ -12985,6 +12783,20 @@ test/unittest_confutils-confutils.obj: test/confutils.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_confutils-confutils.obj `if test -f 'test/confutils.cc'; then $(CYGPATH_W) 'test/confutils.cc'; else $(CYGPATH_W) '$(srcdir)/test/confutils.cc'; fi`
+test/common/unittest_context-test_context.o: test/common/test_context.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_context-test_context.o -MD -MP -MF test/common/$(DEPDIR)/unittest_context-test_context.Tpo -c -o test/common/unittest_context-test_context.o `test -f 'test/common/test_context.cc' || echo '$(srcdir)/'`test/common/test_context.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_context-test_context.Tpo test/common/$(DEPDIR)/unittest_context-test_context.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_context.cc' object='test/common/unittest_context-test_context.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_context-test_context.o `test -f 'test/common/test_context.cc' || echo '$(srcdir)/'`test/common/test_context.cc
+
+test/common/unittest_context-test_context.obj: test/common/test_context.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_context-test_context.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_context-test_context.Tpo -c -o test/common/unittest_context-test_context.obj `if test -f 'test/common/test_context.cc'; then $(CYGPATH_W) 'test/common/test_context.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_context.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_context-test_context.Tpo test/common/$(DEPDIR)/unittest_context-test_context.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_context.cc' object='test/common/unittest_context-test_context.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_context-test_context.obj `if test -f 'test/common/test_context.cc'; then $(CYGPATH_W) 'test/common/test_context.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_context.cc'; fi`
+
test/common/unittest_crc32c-test_crc32c.o: test/common/test_crc32c.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_crc32c-test_crc32c.o -MD -MP -MF test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Tpo -c -o test/common/unittest_crc32c-test_crc32c.o `test -f 'test/common/test_crc32c.cc' || echo '$(srcdir)/'`test/common/test_crc32c.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Tpo test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Po
@@ -13027,20 +12839,6 @@ test/crush/unittest_crush_wrapper-TestCrushWrapper.obj: test/crush/TestCrushWrap
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.obj `if test -f 'test/crush/TestCrushWrapper.cc'; then $(CYGPATH_W) 'test/crush/TestCrushWrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/TestCrushWrapper.cc'; fi`
-test/unittest_crushwrapper-test_crushwrapper.o: test/test_crushwrapper.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crushwrapper-test_crushwrapper.o -MD -MP -MF test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo -c -o test/unittest_crushwrapper-test_crushwrapper.o `test -f 'test/test_crushwrapper.cc' || echo '$(srcdir)/'`test/test_crushwrapper.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_crushwrapper.cc' object='test/unittest_crushwrapper-test_crushwrapper.o' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crushwrapper-test_crushwrapper.o `test -f 'test/test_crushwrapper.cc' || echo '$(srcdir)/'`test/test_crushwrapper.cc
-
-test/unittest_crushwrapper-test_crushwrapper.obj: test/test_crushwrapper.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crushwrapper-test_crushwrapper.obj -MD -MP -MF test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo -c -o test/unittest_crushwrapper-test_crushwrapper.obj `if test -f 'test/test_crushwrapper.cc'; then $(CYGPATH_W) 'test/test_crushwrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_crushwrapper.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_crushwrapper.cc' object='test/unittest_crushwrapper-test_crushwrapper.obj' libtool=no @AMDEPBACKSLASH@
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crushwrapper-test_crushwrapper.obj `if test -f 'test/test_crushwrapper.cc'; then $(CYGPATH_W) 'test/test_crushwrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_crushwrapper.cc'; fi`
-
test/unittest_crypto-crypto.o: test/crypto.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crypto-crypto.o -MD -MP -MF test/$(DEPDIR)/unittest_crypto-crypto.Tpo -c -o test/unittest_crypto-crypto.o `test -f 'test/crypto.cc' || echo '$(srcdir)/'`test/crypto.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crypto-crypto.Tpo test/$(DEPDIR)/unittest_crypto-crypto.Po
@@ -13069,6 +12867,20 @@ test/unittest_daemon_config-daemon_config.obj: test/daemon_config.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_daemon_config-daemon_config.obj `if test -f 'test/daemon_config.cc'; then $(CYGPATH_W) 'test/daemon_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/daemon_config.cc'; fi`
+test/osd/unittest_ecbackend-TestECBackend.o: test/osd/TestECBackend.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_ecbackend-TestECBackend.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo -c -o test/osd/unittest_ecbackend-TestECBackend.o `test -f 'test/osd/TestECBackend.cc' || echo '$(srcdir)/'`test/osd/TestECBackend.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestECBackend.cc' object='test/osd/unittest_ecbackend-TestECBackend.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_ecbackend-TestECBackend.o `test -f 'test/osd/TestECBackend.cc' || echo '$(srcdir)/'`test/osd/TestECBackend.cc
+
+test/osd/unittest_ecbackend-TestECBackend.obj: test/osd/TestECBackend.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_ecbackend-TestECBackend.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo -c -o test/osd/unittest_ecbackend-TestECBackend.obj `if test -f 'test/osd/TestECBackend.cc'; then $(CYGPATH_W) 'test/osd/TestECBackend.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestECBackend.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestECBackend.cc' object='test/osd/unittest_ecbackend-TestECBackend.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_ecbackend-TestECBackend.obj `if test -f 'test/osd/TestECBackend.cc'; then $(CYGPATH_W) 'test/osd/TestECBackend.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestECBackend.cc'; fi`
+
test/unittest_encoding-encoding.o: test/encoding.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_encoding-encoding.o -MD -MP -MF test/$(DEPDIR)/unittest_encoding-encoding.Tpo -c -o test/unittest_encoding-encoding.o `test -f 'test/encoding.cc' || echo '$(srcdir)/'`test/encoding.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_encoding-encoding.Tpo test/$(DEPDIR)/unittest_encoding-encoding.Po
@@ -13083,89 +12895,89 @@ test/unittest_encoding-encoding.obj: test/encoding.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_encoding-encoding.obj `if test -f 'test/encoding.cc'; then $(CYGPATH_W) 'test/encoding.cc'; else $(CYGPATH_W) '$(srcdir)/test/encoding.cc'; fi`
-test/osd/unittest_erasure_code_example-TestErasureCodeExample.o: test/osd/TestErasureCodeExample.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_example-TestErasureCodeExample.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo -c -o test/osd/unittest_erasure_code_example-TestErasureCodeExample.o `test -f 'test/osd/TestErasureCodeExample.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodeExample.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodeExample.cc' object='test/osd/unittest_erasure_code_example-TestErasureCodeExample.o' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o: test/erasure-code/TestErasureCodeExample.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o `test -f 'test/erasure-code/TestErasureCodeExample.cc' || echo '$(srcdir)/'`te [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeExample.cc' object='test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_example-TestErasureCodeExample.o `test -f 'test/osd/TestErasureCodeExample.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodeExample.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o `test -f 'test/erasure-code/TestErasureCodeExample.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodeExample.cc
-test/osd/unittest_erasure_code_example-TestErasureCodeExample.obj: test/osd/TestErasureCodeExample.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_example-TestErasureCodeExample.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo -c -o test/osd/unittest_erasure_code_example-TestErasureCodeExample.obj `if test -f 'test/osd/TestErasureCodeExample.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCodeExample. [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodeExample.cc' object='test/osd/unittest_erasure_code_example-TestErasureCodeExample.obj' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj: test/erasure-code/TestErasureCodeExample.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj `if test -f 'test/erasure-code/TestErasureCodeExample.cc'; then $(CYGPATH_ [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeExample.cc' object='test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_example-TestErasureCodeExample.obj `if test -f 'test/osd/TestErasureCodeExample.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCodeExample.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestErasureCodeExample.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj `if test -f 'test/erasure-code/TestErasureCodeExample.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodeExample.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodeExample.cc'; fi`
-test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o: test/osd/TestErasureCodeJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo -c -o test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o `test -f 'test/osd/TestErasureCodeJerasure.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodeJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodeJerasure.cc' object='test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o: test/erasure-code/TestErasureCodeJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o `test -f 'test/erasure-code/TestErasureCodeJerasure.cc' || echo '$(srcd [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeJerasure.cc' object='test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o `test -f 'test/osd/TestErasureCodeJerasure.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodeJerasure.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o `test -f 'test/erasure-code/TestErasureCodeJerasure.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodeJerasure.cc
-test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj: test/osd/TestErasureCodeJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo -c -o test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj `if test -f 'test/osd/TestErasureCodeJerasure.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCode [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodeJerasure.cc' object='test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj: test/erasure-code/TestErasureCodeJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj `if test -f 'test/erasure-code/TestErasureCodeJerasure.cc'; then $( [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeJerasure.cc' object='test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj `if test -f 'test/osd/TestErasureCodeJerasure.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestErasureCodeJerasure.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj `if test -f 'test/erasure-code/TestErasureCodeJerasure.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodeJerasure.cc'; fi`
-osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o: osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o `test -f 'osd/ErasureCodePlug [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc' object='osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o' libtool=no @AMDEPBACKSLASH@
+erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o: erasure-code/jerasure/ErasureCodePluginJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o `test -f 'erasure-code/jerasure/ErasureCodePluginJera [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o `test -f 'osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc
-osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj: osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj `if test -f 'osd/ErasureC [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc' object='osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj' libtool=no @AMDEPBACKSLASH@
+erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj: erasure-code/jerasure/ErasureCodePluginJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodePlu [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj `if test -f 'osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/osd/ErasureCodePluginJerasure/ErasureC [...]
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'erasure-code/jerasure/ErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/ErasureCodePluginJerasure.cc'; fi`
-osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o: osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o `test -f 'osd/ErasureCodePluginJerasure/Erasure [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc' object='osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o' libtool=no @AMDEPBACKSLASH@
+erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o: erasure-code/jerasure/ErasureCodeJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcd [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o `test -f 'osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc
-osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj: osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj -MD -MP -MF osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj `if test -f 'osd/ErasureCodePluginJerasure/ [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo osd/ErasureCodePluginJerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc' object='osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj' libtool=no @AMDEPBACKSLASH@
+erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj: erasure-code/jerasure/ErasureCodeJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc'; then $( [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj `if test -f 'osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc'; then $(CYGPATH_W) 'osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc'; then $(CYGPATH_W) 'erasure-code/jerasure/ErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/ErasureCodeJerasure.cc'; fi`
-test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.o: test/osd/TestErasureCodePlugin.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo -c -o test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.o `test -f 'test/osd/TestErasureCodePlugin.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodePlugin.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodePlugin.cc' object='test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.o' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o: test/erasure-code/TestErasureCodePlugin.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o `test -f 'test/erasure-code/TestErasureCodePlugin.cc' || echo '$(srcdir)/'`test/erasu [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePlugin.cc' object='test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.o `test -f 'test/osd/TestErasureCodePlugin.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodePlugin.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o `test -f 'test/erasure-code/TestErasureCodePlugin.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodePlugin.cc
-test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.obj: test/osd/TestErasureCodePlugin.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo -c -o test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.obj `if test -f 'test/osd/TestErasureCodePlugin.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCodePlugin.cc'; else [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodePlugin.cc' object='test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.obj' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj: test/erasure-code/TestErasureCodePlugin.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj `if test -f 'test/erasure-code/TestErasureCodePlugin.cc'; then $(CYGPATH_W) 'test [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePlugin.cc' object='test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.obj `if test -f 'test/osd/TestErasureCodePlugin.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCodePlugin.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestErasureCodePlugin.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj `if test -f 'test/erasure-code/TestErasureCodePlugin.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodePlugin.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodePlugin.cc'; fi`
-test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o: test/osd/TestErasureCodePluginJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo -c -o test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o `test -f 'test/osd/TestErasureCodePluginJerasure.cc' [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodePluginJerasure.cc' object='test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o: test/erasure-code/TestErasureCodePluginJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o `test -f 'test/erasure-co [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePluginJerasure.cc' object='test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o `test -f 'test/osd/TestErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`test/osd/TestErasureCodePluginJerasure.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o `test -f 'test/erasure-code/TestErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodePluginJerasure.cc
-test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj: test/osd/TestErasureCodePluginJerasure.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo -c -o test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj `if test -f 'test/osd/TestErasureCodePluginJeras [...]
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestErasureCodePluginJerasure.cc' object='test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj' libtool=no @AMDEPBACKSLASH@
+test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj: test/erasure-code/TestErasureCodePluginJerasure.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj `if test -f 'test/era [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePluginJerasure.cc' object='test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj `if test -f 'test/osd/TestErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'test/osd/TestErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestErasureCodePluginJerasure.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj `if test -f 'test/erasure-code/TestErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodePluginJerasure. [...]
test/unittest_escape-escape.o: test/escape.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_escape_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_escape-escape.o -MD -MP -MF test/$(DEPDIR)/unittest_escape-escape.Tpo -c -o test/unittest_escape-escape.o `test -f 'test/escape.cc' || echo '$(srcdir)/'`test/escape.cc
@@ -13251,6 +13063,20 @@ test/unittest_heartbeatmap-heartbeat_map.obj: test/heartbeat_map.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_heartbeatmap-heartbeat_map.obj `if test -f 'test/heartbeat_map.cc'; then $(CYGPATH_W) 'test/heartbeat_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/heartbeat_map.cc'; fi`
+test/common/unittest_histogram-histogram.o: test/common/histogram.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_histogram-histogram.o -MD -MP -MF test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo -c -o test/common/unittest_histogram-histogram.o `test -f 'test/common/histogram.cc' || echo '$(srcdir)/'`test/common/histogram.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo test/common/$(DEPDIR)/unittest_histogram-histogram.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/histogram.cc' object='test/common/unittest_histogram-histogram.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_histogram-histogram.o `test -f 'test/common/histogram.cc' || echo '$(srcdir)/'`test/common/histogram.cc
+
+test/common/unittest_histogram-histogram.obj: test/common/histogram.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_histogram-histogram.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo -c -o test/common/unittest_histogram-histogram.obj `if test -f 'test/common/histogram.cc'; then $(CYGPATH_W) 'test/common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/histogram.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo test/common/$(DEPDIR)/unittest_histogram-histogram.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/histogram.cc' object='test/common/unittest_histogram-histogram.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_histogram-histogram.obj `if test -f 'test/common/histogram.cc'; then $(CYGPATH_W) 'test/common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/histogram.cc'; fi`
+
test/osd/unittest_hitset-hitset.o: test/osd/hitset.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_hitset-hitset.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo -c -o test/osd/unittest_hitset-hitset.o `test -f 'test/osd/hitset.cc' || echo '$(srcdir)/'`test/osd/hitset.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo test/osd/$(DEPDIR)/unittest_hitset-hitset.Po
@@ -13377,6 +13203,20 @@ test/mon/unittest_mon_moncap-moncap.obj: test/mon/moncap.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_moncap-moncap.obj `if test -f 'test/mon/moncap.cc'; then $(CYGPATH_W) 'test/mon/moncap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/moncap.cc'; fi`
+test/mon/unittest_mon_pgmap-PGMap.o: test/mon/PGMap.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -MT test/mon/unittest_mon_pgmap-PGMap.o -MD -MP -MF test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo -c -o test/mon/unittest_mon_pgmap-PGMap.o `test -f 'test/mon/PGMap.cc' || echo '$(srcdir)/'`test/mon/PGMap.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mon/PGMap.cc' object='test/mon/unittest_mon_pgmap-PGMap.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_pgmap-PGMap.o `test -f 'test/mon/PGMap.cc' || echo '$(srcdir)/'`test/mon/PGMap.cc
+
+test/mon/unittest_mon_pgmap-PGMap.obj: test/mon/PGMap.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -MT test/mon/unittest_mon_pgmap-PGMap.obj -MD -MP -MF test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo -c -o test/mon/unittest_mon_pgmap-PGMap.obj `if test -f 'test/mon/PGMap.cc'; then $(CYGPATH_W) 'test/mon/PGMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/PGMap.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mon/PGMap.cc' object='test/mon/unittest_mon_pgmap-PGMap.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_pgmap-PGMap.obj `if test -f 'test/mon/PGMap.cc'; then $(CYGPATH_W) 'test/mon/PGMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/PGMap.cc'; fi`
+
test/osd/unittest_osd_osdcap-osdcap.o: test/osd/osdcap.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_osdcap-osdcap.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Tpo -c -o test/osd/unittest_osd_osdcap-osdcap.o `test -f 'test/osd/osdcap.cc' || echo '$(srcdir)/'`test/osd/osdcap.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Tpo test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Po
@@ -13391,19 +13231,19 @@ test/osd/unittest_osd_osdcap-osdcap.obj: test/osd/osdcap.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_osdcap-osdcap.obj `if test -f 'test/osd/osdcap.cc'; then $(CYGPATH_W) 'test/osd/osdcap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/osdcap.cc'; fi`
-test/unittest_osd_types-test_osd_types.o: test/test_osd_types.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_osd_types-test_osd_types.o -MD -MP -MF test/$(DEPDIR)/unittest_osd_types-test_osd_types.Tpo -c -o test/unittest_osd_types-test_osd_types.o `test -f 'test/test_osd_types.cc' || echo '$(srcdir)/'`test/test_osd_types.cc
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_osd_types-test_osd_types.Tpo test/$(DEPDIR)/unittest_osd_types-test_osd_types.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_osd_types.cc' object='test/unittest_osd_types-test_osd_types.o' libtool=no @AMDEPBACKSLASH@
+test/osd/unittest_osd_types-types.o: test/osd/types.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_types-types.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo -c -o test/osd/unittest_osd_types-types.o `test -f 'test/osd/types.cc' || echo '$(srcdir)/'`test/osd/types.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo test/osd/$(DEPDIR)/unittest_osd_types-types.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/types.cc' object='test/osd/unittest_osd_types-types.o' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_osd_types-test_osd_types.o `test -f 'test/test_osd_types.cc' || echo '$(srcdir)/'`test/test_osd_types.cc
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_types-types.o `test -f 'test/osd/types.cc' || echo '$(srcdir)/'`test/osd/types.cc
-test/unittest_osd_types-test_osd_types.obj: test/test_osd_types.cc
- at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_osd_types-test_osd_types.obj -MD -MP -MF test/$(DEPDIR)/unittest_osd_types-test_osd_types.Tpo -c -o test/unittest_osd_types-test_osd_types.obj `if test -f 'test/test_osd_types.cc'; then $(CYGPATH_W) 'test/test_osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_osd_types.cc'; fi`
- at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_osd_types-test_osd_types.Tpo test/$(DEPDIR)/unittest_osd_types-test_osd_types.Po
- at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_osd_types.cc' object='test/unittest_osd_types-test_osd_types.obj' libtool=no @AMDEPBACKSLASH@
+test/osd/unittest_osd_types-types.obj: test/osd/types.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_types-types.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo -c -o test/osd/unittest_osd_types-types.obj `if test -f 'test/osd/types.cc'; then $(CYGPATH_W) 'test/osd/types.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/types.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo test/osd/$(DEPDIR)/unittest_osd_types-types.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/types.cc' object='test/osd/unittest_osd_types-types.obj' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
- at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_osd_types-test_osd_types.obj `if test -f 'test/test_osd_types.cc'; then $(CYGPATH_W) 'test/test_osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_osd_types.cc'; fi`
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_types-types.obj `if test -f 'test/osd/types.cc'; then $(CYGPATH_W) 'test/osd/types.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/types.cc'; fi`
test/osd/unittest_osdmap-TestOSDMap.o: test/osd/TestOSDMap.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osdmap-TestOSDMap.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo -c -o test/osd/unittest_osdmap-TestOSDMap.o `test -f 'test/osd/TestOSDMap.cc' || echo '$(srcdir)/'`test/osd/TestOSDMap.cc
@@ -13704,6 +13544,8 @@ clean-libtool:
-rm -rf cls/version/.libs cls/version/_libs
-rm -rf common/.libs common/_libs
-rm -rf crush/.libs crush/_libs
+ -rm -rf erasure-code/.libs erasure-code/_libs
+ -rm -rf erasure-code/jerasure/.libs erasure-code/jerasure/_libs
-rm -rf global/.libs global/_libs
-rm -rf java/native/.libs java/native/_libs
-rm -rf json_spirit/.libs json_spirit/_libs
@@ -13717,11 +13559,11 @@ clean-libtool:
-rm -rf objclass/.libs objclass/_libs
-rm -rf os/.libs os/_libs
-rm -rf osd/.libs osd/_libs
- -rm -rf osd/ErasureCodePluginJerasure/.libs osd/ErasureCodePluginJerasure/_libs
-rm -rf osdc/.libs osdc/_libs
-rm -rf perfglue/.libs perfglue/_libs
-rm -rf rgw/.libs rgw/_libs
- -rm -rf test/osd/.libs test/osd/_libs
+ -rm -rf test/erasure-code/.libs test/erasure-code/_libs
+ -rm -rf test/librados/.libs test/librados/_libs
-rm -rf test/system/.libs test/system/_libs
install-pythonPYTHON: $(python_PYTHON)
@$(NORMAL_INSTALL)
@@ -14218,6 +14060,10 @@ distclean-generic:
-rm -f common/$(am__dirstamp)
-rm -f crush/$(DEPDIR)/$(am__dirstamp)
-rm -f crush/$(am__dirstamp)
+ -rm -f erasure-code/$(DEPDIR)/$(am__dirstamp)
+ -rm -f erasure-code/$(am__dirstamp)
+ -rm -f erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp)
+ -rm -f erasure-code/jerasure/$(am__dirstamp)
-rm -f global/$(DEPDIR)/$(am__dirstamp)
-rm -f global/$(am__dirstamp)
-rm -f java/native/$(DEPDIR)/$(am__dirstamp)
@@ -14246,8 +14092,6 @@ distclean-generic:
-rm -f os/$(am__dirstamp)
-rm -f osd/$(DEPDIR)/$(am__dirstamp)
-rm -f osd/$(am__dirstamp)
- -rm -f osd/ErasureCodePluginJerasure/$(DEPDIR)/$(am__dirstamp)
- -rm -f osd/ErasureCodePluginJerasure/$(am__dirstamp)
-rm -f osdc/$(DEPDIR)/$(am__dirstamp)
-rm -f osdc/$(am__dirstamp)
-rm -f perfglue/$(DEPDIR)/$(am__dirstamp)
@@ -14286,8 +14130,8 @@ distclean-generic:
-rm -f test/crush/$(am__dirstamp)
-rm -f test/encoding/$(DEPDIR)/$(am__dirstamp)
-rm -f test/encoding/$(am__dirstamp)
- -rm -f test/filestore/$(DEPDIR)/$(am__dirstamp)
- -rm -f test/filestore/$(am__dirstamp)
+ -rm -f test/erasure-code/$(DEPDIR)/$(am__dirstamp)
+ -rm -f test/erasure-code/$(am__dirstamp)
-rm -f test/libcephfs/$(DEPDIR)/$(am__dirstamp)
-rm -f test/libcephfs/$(am__dirstamp)
-rm -f test/librados/$(DEPDIR)/$(am__dirstamp)
@@ -14296,12 +14140,16 @@ distclean-generic:
-rm -f test/librbd/$(am__dirstamp)
-rm -f test/mon/$(DEPDIR)/$(am__dirstamp)
-rm -f test/mon/$(am__dirstamp)
+ -rm -f test/objectstore/$(DEPDIR)/$(am__dirstamp)
+ -rm -f test/objectstore/$(am__dirstamp)
-rm -f test/os/$(DEPDIR)/$(am__dirstamp)
-rm -f test/os/$(am__dirstamp)
-rm -f test/osd/$(DEPDIR)/$(am__dirstamp)
-rm -f test/osd/$(am__dirstamp)
-rm -f test/osdc/$(DEPDIR)/$(am__dirstamp)
-rm -f test/osdc/$(am__dirstamp)
+ -rm -f test/rgw/$(DEPDIR)/$(am__dirstamp)
+ -rm -f test/rgw/$(am__dirstamp)
-rm -f test/system/$(DEPDIR)/$(am__dirstamp)
-rm -f test/system/$(am__dirstamp)
-rm -f tools/$(DEPDIR)/$(am__dirstamp)
@@ -14323,7 +14171,7 @@ clean-am: clean-binPROGRAMS clean-checkPROGRAMS \
clean-sbinPROGRAMS clean-su_sbinPROGRAMS mostlyclean-am
distclean: distclean-recursive
- -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$( [...]
+ -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) erasure-code/$(DEPDIR) erasure-code/jerasure/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json [...]
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-hdr distclean-tags
@@ -14377,7 +14225,7 @@ install-ps-am:
installcheck-am:
maintainer-clean: maintainer-clean-recursive
- -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$( [...]
+ -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) erasure-code/$(DEPDIR) erasure-code/jerasure/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json [...]
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
diff --git a/src/acconfig.h.in b/src/acconfig.h.in
index 43e8c30..c3c132f 100644
--- a/src/acconfig.h.in
+++ b/src/acconfig.h.in
@@ -113,6 +113,9 @@
/* Define if you have tcmalloc */
#undef HAVE_LIBTCMALLOC
+/* Define to 1 if you have libxfs */
+#undef HAVE_LIBXFS
+
/* Defined if you have libzfs enabled */
#undef HAVE_LIBZFS
diff --git a/src/auth/cephx/CephxSessionHandler.cc b/src/auth/cephx/CephxSessionHandler.cc
index 460e252..b2d402d 100644
--- a/src/auth/cephx/CephxSessionHandler.cc
+++ b/src/auth/cephx/CephxSessionHandler.cc
@@ -26,14 +26,13 @@
int CephxSessionHandler::sign_message(Message *m)
{
- bufferlist bl_plaintext, bl_encrypted;
- ceph_msg_header header = m->get_header();
- std::string error;
-
// If runtime signing option is off, just return success without signing.
if (!cct->_conf->cephx_sign_messages) {
return 0;
}
+ bufferlist bl_plaintext, bl_encrypted;
+ ceph_msg_header header = m->get_header();
+ std::string error;
ceph_msg_footer& en_footer = m->get_footer();
@@ -71,16 +70,16 @@ int CephxSessionHandler::sign_message(Message *m)
int CephxSessionHandler::check_message_signature(Message *m)
{
- bufferlist bl_plaintext, bl_ciphertext;
- std::string sig_error;
- ceph_msg_header& header = m->get_header();
- ceph_msg_footer& footer = m->get_footer();
-
// If runtime signing option is off, just return success without checking signature.
if (!cct->_conf->cephx_sign_messages) {
return 0;
}
+ bufferlist bl_plaintext, bl_ciphertext;
+ std::string sig_error;
+ ceph_msg_header& header = m->get_header();
+ ceph_msg_footer& footer = m->get_footer();
+
if ((features & CEPH_FEATURE_MSG_AUTH) == 0) {
// it's fine, we didn't negotiate this feature.
return 0;
diff --git a/src/brag/Makefile.am b/src/brag/Makefile.am
new file mode 100644
index 0000000..35c735b
--- /dev/null
+++ b/src/brag/Makefile.am
@@ -0,0 +1,3 @@
+
+bin_SCRIPTS += brag/client/ceph-brag
+EXTRA_DIST += brag/server brag/README.md brag/client
diff --git a/src/brag/README.md b/src/brag/README.md
new file mode 100644
index 0000000..55af44f
--- /dev/null
+++ b/src/brag/README.md
@@ -0,0 +1,184 @@
+# Ceph-brag
+
+`ceph-brag` is going to be an anonymized cluster reporting tool designed to collect a "registry" of Ceph clusters for community knowledge.
+This data will be displayed on a public web page using UUID by default, but users can claim their cluster and publish information about ownership if they so desire.
+
+For more information please visit:
+
+* [Blueprint](http://wiki.ceph.com/Planning/Blueprints/Firefly/Ceph-Brag)
+* [CDS Etherpad](http://pad.ceph.com/p/cdsfirefly-ceph-brag)
+
+# Client
+
+## How to use:
+
+### Pre-requisites:
+ceph-brag uses 'ceph' python script. Hence, before executing ceph-brag script ensure that ceph services are all running and 'ceph' script is in 'PATH' environment
+
+### Runtime instructions:
+Run 'ceph-brag -h' to get the usage information of this tool.
+
+### Sample output:
+
+ {
+ "cluster_creation_date": "2014-01-16 13:38:41.928551",
+ "uuid": "20679d0e-04b1-4004-8ee9-45ac271510e9",
+ "components_count": {
+ "num_bytes": 0,
+ "num_osds": 1,
+ "num_objects": 0,
+ "num_pgs": 192,
+ "num_pools": 3,
+ "num_mdss": 1,
+ "num_mons": 1
+ },
+ "crush_types": [
+ {
+ "type": "osd"
+ "count": 2,
+ },
+ {
+ "type": "rack"
+ "count": 1,
+ },
+ {
+ "type": "host"
+ "count": 1,
+ },
+ {
+ "type": "root"
+ "count": 1,
+ }
+ ],
+ "ownership": {
+ "organization": "eNovance",
+ "description": "Use case1",
+ "email": "mail at enovance.com",
+ "name": "Cluster1"
+ },
+ "pool_metadata": [
+ {
+ "size": 3,
+ "id": 0,
+ "type": 1
+ },
+ {
+ "size": 3,
+ "id": 1,
+ "type": 1
+ },
+ {
+ "size": 3,
+ "id": 2,
+ "name": 1
+ }
+ ],
+ "sysinfo": {
+ "kernel_types": [
+ {
+ "count": 1,
+ "type": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012"
+ }
+ ],
+ "cpu_archs": [
+ {
+ "count": 1,
+ "arch": "x86_64"
+ }
+ ],
+ "cpus": [
+ {
+ "count": 1,
+ "cpu": "Intel Xeon E312xx (Sandy Bridge)"
+ }
+ ],
+ "kernel_versions": [
+ {
+ "count": 1,
+ "version": "3.2.0-23-virtual"
+ }
+ ],
+ "ceph_versions": [
+ {
+ "count": 1,
+ "version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)"
+ }
+ ],
+ "os_info": [
+ {
+ "count": 1,
+ "os": "Linux"
+ }
+ ],
+ "distros": [
+ {
+ "count": 1,
+ "distro": "Ubuntu 12.04 precise (Ubuntu 12.04 LTS)"
+ }
+ ]
+ }
+ }
+
+
+# Server
+
+## Info
+The ceph-brag server code is a python based web application.
+
+## How to use
+
+### Prerequisites
+* [pecan](http://pecanpy.org) is the web framework that is used by this application.
+* [sqlalchemy](www.sqlalchemy.org) is the ORM that is used by this application
+
+### How to deploy
+* [Common recipes to deploy](http://pecan.readthedocs.org/en/latest/deployment.html#common-recipes)
+* Modify server/config.py:sqlalchemy['url'] to point the correct database connection
+
+## URLs
+Following are the REST urls that are implemented with 'url-prefix' being the mount point for the WSGI script
+
+### GET
+
+##### * GET /url-prefix/
+Returns the list of clusters that are registered so far.
+Outputs - On success application/json of the following format is returned
+
+ [
+ {
+ "num_versions": 3,
+ "cluster_creation_date": "2014-01-16 13:38:41.928551",
+ "uuid": "20679d0e-04b1-4004-8ee9-45ac271510e9",
+ "cluster_name": "Cluster1",
+ "organization": "eNovance",
+ "email": "mail at enovance.com"
+ },
+ ...
+ ]
+
+##### * GET /url-prefix/UUID
+Returns the list of version information for a particular UUID.
+Outputs - On success application/json of the following format is returned
+
+ [
+ {
+ "version_number": 1,
+ "version_date": "2014-02-10 10:17:56.283499"
+ },
+ ...
+ ]
+
+##### * GET /url-prefix/UUID/version\_number
+Returns the entire brag report as mentioned in client's sample output for a particular version of a UUID
+
+### PUT
+
+##### * PUT /url-prefix
+Uploads the brag report and creates a new version for the UUID mentioned in the payload
+
+### DELETE
+
+##### * DELETE /url-prefix?uuid=xxxx
+Deletes all the versions of a cluster whose UUID is sent as a parameter
+
+
diff --git a/src/brag/client/ceph-brag b/src/brag/client/ceph-brag
new file mode 100755
index 0000000..e07ad01
--- /dev/null
+++ b/src/brag/client/ceph-brag
@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+
+import subprocess
+import uuid
+import re
+import json
+import sys
+import ast
+import requests
+from collections import Counter
+
+CLUSTER_UUID_NAME='cluster-uuid'
+CLUSTER_OWNERSHIP_NAME='cluster-ownership'
+
+def run_command(cmd):
+ child = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ (o, e) = child.communicate()
+ return (child.returncode, o, e)
+
+def get_uuid():
+ (rc,uid,e) = run_command(['ceph', 'config-key', 'get', CLUSTER_UUID_NAME])
+ if rc is not 0:
+ #uuid is not yet set.
+ uid = str(uuid.uuid4())
+ (rc, o, e) = run_command(['ceph', 'config-key', 'put',
+ CLUSTER_UUID_NAME, uid])
+ if rc is not 0:
+ raise RuntimeError("\'ceph config-key put\' failed -" + e)
+
+ return uid
+
+def get_cluster_creation_date():
+ (rc, o, e) = run_command(['ceph', 'mon', 'dump', '-f', 'json'])
+ if rc is not 0:
+ raise RuntimeError("\'ceph mon dump\' failed - " + e)
+
+ oj = json.loads(o)
+ return oj['created']
+
+def bytes_pretty_to_raw(byte_count, byte_scale):
+ if byte_scale == 'kB':
+ return byte_count >> 10
+ if byte_scale == 'MB':
+ return byte_count >> 20
+ if byte_scale == 'GB':
+ return byte_count >> 30
+ if byte_scale == 'TB':
+ return byte_count >> 40
+ if byte_scale == 'PB':
+ return byte_count >> 50
+ if byte_scale == 'EB':
+ return byte_count >> 60
+
+ return byte_count
+
+def get_nums():
+ (rc, o, e) = run_command(['ceph', '-s', '-f', 'json'])
+ if rc is not 0:
+ raise RuntimeError("\'ceph -s\' failed - " + e)
+
+ oj = json.loads(o)
+ num_mons = len(oj['monmap']['mons'])
+ num_osds = int(oj['osdmap']['osdmap']['num_in_osds'])
+ num_mdss = oj['mdsmap']['in']
+
+ pgmap = oj['pgmap']
+ num_pgs = pgmap['num_pgs']
+ num_bytes = pgmap['data_bytes']
+
+ (rc, o, e) = run_command(['ceph', 'pg', 'dump', 'pools', '-f', 'json-pretty'])
+ if rc is not 0:
+ raise RuntimeError("\'ceph pg dump pools\' failed - " + e)
+
+ pools = json.loads(o)
+ num_pools = len(pools)
+ num_objs = 0
+ for p in pools:
+ num_objs += p['stat_sum']['num_objects']
+
+ nums = {'num_mons':num_mons,
+ 'num_osds':num_osds,
+ 'num_mdss':num_mdss,
+ 'num_pgs':num_pgs,
+ 'num_bytes':num_bytes,
+ 'num_pools':num_pools,
+ 'num_objects':num_objs}
+ return nums
+
+def get_crush_types():
+ (rc, o, e) = run_command(['ceph', 'osd', 'crush', 'dump'])
+ if rc is not 0:
+ raise RuntimeError("\'ceph osd crush dump\' failed - " + e)
+
+ crush_dump = json.loads(o)
+ if crush_dump['types'] is None:
+ raise RuntimeError("\'types\' item missing in \'ceph osd crush dump\'")
+
+ crush_types = {}
+ for t in crush_dump['types']:
+ crush_types[t['type_id']] = t['name']
+
+ buckets = {}
+ items_list = []
+ for bucket in crush_dump['buckets']:
+ buckets[bucket['id']] = bucket['type_id']
+ for item in bucket['items']:
+ items_list.append(item['id'])
+
+ crush_map = []
+ counter = Counter(items_list)
+ append = lambda t,c: crush_map.append({'type':t, 'count':c})
+ for id,count in counter.items():
+ if id in buckets:
+ append(crush_types[buckets[id]],
+ count)
+ del buckets[id]
+ else:
+ append(crush_types[id], count)
+
+ #the root item
+ for id,type_id in buckets.items():
+ append(crush_types[type_id], 1)
+
+ return crush_map
+
+def get_pool_metadata():
+ (rc, o, e) = run_command(['ceph', 'osd', 'dump', '-f', 'json'])
+ if rc is not 0:
+ raise RuntimeError("\'ceph osd dump\' failed - " + e)
+
+ pool_meta = []
+ oj = json.loads(o)
+ proc = lambda x: {'id':x['pool'], 'type':x['type'], 'size':x['size']}
+ for p in oj['pools']:
+ pool_meta.append(proc(p))
+
+ return pool_meta
+
+def get_sysinfo(max_osds):
+ count = 0
+ osd_metadata_available = False
+
+ os = {}
+ kern_version = {}
+ kern_description = {}
+ distro = {}
+ cpu = {}
+ arch = {}
+ ceph_version = {}
+
+ incr = lambda a,k: 1 if k not in a else a[k]+1
+ while count < max_osds:
+ meta = {'id':count}
+ (rc, o, e) = run_command(['ceph', 'osd', 'metadata', str(count)])
+ if rc is 0:
+ if osd_metadata_available is False:
+ osd_metadata_available = True
+
+ jmeta = json.loads(o)
+
+ version = jmeta['ceph_version'].split()
+ cv = version[2]
+ if (len(version) > 3):
+ cv += version[3]
+
+ ceph_version[cv] = incr(ceph_version, cv)
+ os[jmeta['os']] = incr(os, jmeta['os'])
+ kern_version[jmeta['kernel_version']] = \
+ incr(kern_version, jmeta['kernel_version'])
+ kern_description[jmeta['kernel_description']] = \
+ incr(kern_description, jmeta['kernel_description'])
+
+ try:
+ dstr = jmeta['distro'] + ' '
+ dstr += jmeta['distro_version'] + ' '
+ dstr += jmeta['distro_codename'] + ' ('
+ dstr += jmeta['distro_description'] + ')'
+ distro[dstr] = incr(distro, dstr)
+ except KeyError as ke:
+ pass
+
+ cpu[jmeta['cpu']] = incr(cpu, jmeta['cpu'])
+ arch[jmeta['arch']] = incr(arch, jmeta['arch'])
+
+ count = count + 1
+
+ sysinfo = {}
+ if osd_metadata_available is False:
+ print >> sys.stderr, "'ceph osd metadata' is not available at all"
+ return sysinfo
+
+ def jsonify(type_count, name, type_name):
+ tmp = []
+ for k, v in type_count.items():
+ tmp.append({type_name:k, 'count':v})
+ sysinfo[name] = tmp
+
+ jsonify(os, 'os_info', 'os')
+ jsonify(kern_version, 'kernel_versions', 'version')
+ jsonify(kern_description, 'kernel_types', 'type')
+ jsonify(distro, 'distros', 'distro')
+ jsonify(cpu, 'cpus', 'cpu')
+ jsonify(arch, 'cpu_archs', 'arch')
+ jsonify(ceph_version, 'ceph_versions', 'version')
+ return sysinfo
+
+def get_ownership_info():
+ (rc, o, e) = run_command(['ceph', 'config-key', 'get',
+ CLUSTER_OWNERSHIP_NAME])
+ if rc is not 0:
+ return {}
+
+ return ast.literal_eval(o)
+
+def output_json():
+ out = {}
+ url = None
+
+ out['uuid'] = get_uuid()
+ out['cluster_creation_date'] = get_cluster_creation_date()
+ nums = get_nums()
+ num_osds = int(nums['num_osds'])
+ out['components_count'] = nums
+ out['crush_types'] = get_crush_types()
+ out['pool_metadata'] = get_pool_metadata()
+ out['sysinfo'] = get_sysinfo(num_osds)
+
+ owner = get_ownership_info()
+ if owner is not None:
+ out['ownership'] = owner
+ if 'url' in owner:
+ url = owner.pop('url')
+
+ return json.dumps(out, indent=2, separators=(',', ': ')), url
+
+def describe_usage():
+ print >> sys.stderr, "Usage:"
+ print >> sys.stderr, "======\n"
+
+ print >> sys.stderr, sys.argv[0] + " <commands> [command-options]\n"
+ print >> sys.stderr, "commands:"
+ print >> sys.stderr, "publish - publish the brag report to the server"
+ print >> sys.stderr, "update-metadata <update-metadata-options> - Update"
+ print >> sys.stderr, " ownership information for bragging"
+ print >> sys.stderr, "clear-metadata - Clear information set by update-metadata"
+ print >> sys.stderr, "unpublish --yes-i-am-shy - delete the brag report from the server"
+ print >> sys.stderr, ""
+
+ print >> sys.stderr, "update-metadata options:"
+ print >> sys.stderr, "--name= - Name of the cluster"
+ print >> sys.stderr, "--organization= - Name of the organization"
+ print >> sys.stderr, "--email= - Email contact address"
+ print >> sys.stderr, "--description= - Reporting use-case"
+ print >> sys.stderr, "--url= - The URL that is used to publish and unpublish"
+ print >> sys.stderr, ""
+
+def update_metadata():
+ info = {}
+ possibles = ['name', 'organization', 'email', 'description', 'url']
+
+ #get the existing values
+ info = get_ownership_info();
+
+ for index in range(2, len(sys.argv)):
+ mo = re.search("--(\S+)=(.*)", sys.argv[index])
+ if not mo:
+ describe_usage()
+ return 22
+
+ k = mo.group(1)
+ v = mo.group(2)
+
+ if k in possibles:
+ info[k] = v
+ else:
+ print >> sys.stderr, "Unexpect option --" + k
+ describe_usage()
+ return 22
+
+ (rc, o, e) = run_command(['ceph', 'config-key', 'put',
+ CLUSTER_OWNERSHIP_NAME, str(info)])
+ return rc
+
+def clear_metadata():
+ (rc, o, e) = run_command(['ceph', 'config-key', 'del',
+ CLUSTER_OWNERSHIP_NAME])
+ return rc
+
+def publish():
+ data, url = output_json()
+ if url is None:
+ print >> sys.stderr, "Cannot publish until a URL is set using update-metadata"
+ return 1
+
+ req = requests.put(url, data=data)
+ if req.status_code is not 201:
+ print >> sys.stderr, "Failed to publish, server responded with code " + str(req.status_code)
+ print >> sys.stderr, req.text
+ return 1
+
+ return 0
+
+def unpublish():
+ if len(sys.argv) <= 2 or sys.argv[2] != '--yes-i-am-shy':
+ print >> sys.stderr, "unpublish should be followed by --yes-i-am-shy"
+ return 22
+
+ fail = False
+ owner = get_ownership_info()
+ if owner is None:
+ fail = True
+ try:
+ url = owner['url']
+ except KeyError as e:
+ fail = True
+
+ if fail:
+ print >> sys.stderr, "URL is not updated yet"
+ return 1
+
+ uuid = get_uuid()
+
+ params = {'uuid':uuid}
+ req = requests.delete(url, params=params)
+ if req.status_code is not 200:
+ print >> sys.stderr, "Failed to unpublish, server responsed with code " + str(req.status_code)
+ return 1
+
+ return 0
+
+def main():
+ if len(sys.argv) is 1:
+ print output_json()[0]
+ return 0
+ elif sys.argv[1] == 'update-metadata':
+ return update_metadata()
+ elif sys.argv[1] == 'clear-metadata':
+ return clear_metadata()
+ elif sys.argv[1] == 'publish':
+ return publish()
+ elif sys.argv[1] == 'unpublish':
+ return unpublish()
+ else:
+ describe_usage()
+ return 22
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/src/brag/server/MANIFEST.in b/src/brag/server/MANIFEST.in
new file mode 100644
index 0000000..c922f11
--- /dev/null
+++ b/src/brag/server/MANIFEST.in
@@ -0,0 +1 @@
+recursive-include public *
diff --git a/src/brag/server/app.wsgi b/src/brag/server/app.wsgi
new file mode 100644
index 0000000..01dfc72
--- /dev/null
+++ b/src/brag/server/app.wsgi
@@ -0,0 +1,5 @@
+import os
+from pecan.deploy import deploy
+
+cur_path = os.path.dirname(__file__)
+application = deploy(cur_path + '/config.py')
diff --git a/src/brag/server/ceph_brag.egg-info/PKG-INFO b/src/brag/server/ceph_brag.egg-info/PKG-INFO
new file mode 100644
index 0000000..eeb2b1b
--- /dev/null
+++ b/src/brag/server/ceph_brag.egg-info/PKG-INFO
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: ceph-brag
+Version: 0.1
+Summary: UNKNOWN
+Home-page: UNKNOWN
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/src/brag/server/ceph_brag.egg-info/SOURCES.txt b/src/brag/server/ceph_brag.egg-info/SOURCES.txt
new file mode 100644
index 0000000..fc3b81d
--- /dev/null
+++ b/src/brag/server/ceph_brag.egg-info/SOURCES.txt
@@ -0,0 +1,21 @@
+MANIFEST.in
+config.py
+setup.cfg
+setup.py
+ceph_brag/__init__.py
+ceph_brag/app.py
+ceph_brag/json.py
+ceph_brag.egg-info/PKG-INFO
+ceph_brag.egg-info/SOURCES.txt
+ceph_brag.egg-info/dependency_links.txt
+ceph_brag.egg-info/not-zip-safe
+ceph_brag.egg-info/requires.txt
+ceph_brag.egg-info/top_level.txt
+ceph_brag/controllers/__init__.py
+ceph_brag/controllers/root.py
+ceph_brag/model/__init__.py
+ceph_brag/model/db.py
+ceph_brag/tests/__init__.py
+ceph_brag/tests/config.py
+ceph_brag/tests/test_functional.py
+ceph_brag/tests/test_units.py
\ No newline at end of file
diff --git a/src/brag/server/ceph_brag.egg-info/dependency_links.txt b/src/brag/server/ceph_brag.egg-info/dependency_links.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/src/brag/server/ceph_brag.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/src/brag/server/ceph_brag.egg-info/not-zip-safe b/src/brag/server/ceph_brag.egg-info/not-zip-safe
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/src/brag/server/ceph_brag.egg-info/not-zip-safe
@@ -0,0 +1 @@
+
diff --git a/src/brag/server/ceph_brag.egg-info/requires.txt b/src/brag/server/ceph_brag.egg-info/requires.txt
new file mode 100644
index 0000000..ea2fd7b
--- /dev/null
+++ b/src/brag/server/ceph_brag.egg-info/requires.txt
@@ -0,0 +1 @@
+pecan
\ No newline at end of file
diff --git a/src/brag/server/ceph_brag.egg-info/top_level.txt b/src/brag/server/ceph_brag.egg-info/top_level.txt
new file mode 100644
index 0000000..9e1c53c
--- /dev/null
+++ b/src/brag/server/ceph_brag.egg-info/top_level.txt
@@ -0,0 +1 @@
+ceph_brag
diff --git a/src/brag/server/ceph_brag/__init__.py b/src/brag/server/ceph_brag/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/brag/server/ceph_brag/app.py b/src/brag/server/ceph_brag/app.py
new file mode 100644
index 0000000..340c60d
--- /dev/null
+++ b/src/brag/server/ceph_brag/app.py
@@ -0,0 +1,19 @@
+from pecan import make_app
+from ceph_brag import model, json
+from pecan.hooks import TransactionHook
+
+def setup_app(config):
+
+ model.init_model()
+ app_conf = dict(config.app)
+
+ return make_app(
+ app_conf.pop('root'),
+ logging=getattr(config, 'logging', {}),
+ hooks=[TransactionHook(model.start,
+ model.start,
+ model.commit,
+ model.rollback,
+ model.clear)],
+ **app_conf
+ )
diff --git a/src/brag/server/ceph_brag/controllers/__init__.py b/src/brag/server/ceph_brag/controllers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/brag/server/ceph_brag/controllers/root.py b/src/brag/server/ceph_brag/controllers/root.py
new file mode 100644
index 0000000..254ab86
--- /dev/null
+++ b/src/brag/server/ceph_brag/controllers/root.py
@@ -0,0 +1,73 @@
+from pecan import expose, request, abort, response
+from webob import exc
+from pecan.rest import RestController
+from ceph_brag.model import db
+import sys, traceback
+
+class RootController(RestController):
+ def fail(self, status_code=200, msg="OK"):
+ response.status = status_code
+ return msg
+
+ @expose('json')
+ def get(self, *args, **kwargs):
+ if len(args) is 0:
+ #return the list of uuids
+ try:
+ result = db.get_uuids()
+ except Exception as e:
+ return self.fail(500, msg="Internal Server Error")
+ elif len(args) is 1 or len(args) is 2 and args[1] == '':
+ #/uuid
+ try:
+ result = db.get_versions(args[0])
+ except Exception as e:
+ return self.fail(status_code=500, msg="Internal Server Error")
+
+ if result is None:
+ return self.fail(400, msg="Invalid UUID")
+ elif len(args) is 2 or len(args) is 3 and args[2] == '':
+ #/uuid/version_number
+ try:
+ result = db.get_brag(args[0], args[1])
+ except Exception as e:
+ return self.fail(status_code=500, msg="Internal Server Error")
+
+ if result is None:
+ return self.fail(status_code=400, msg="Invalid UUID,version combination")
+ else:
+ return self.fail(status_code=400, msg="Invalid args")
+
+ return result
+
+ @expose(content_type='application/json')
+ def put(self, *args, **kwargs):
+ try:
+ db.put_new_version(request.body)
+ except ValueError as ve:
+ return self.fail(status_code=422, msg="Improper payload")
+ except KeyError as ke:
+ msg = "Payload not as expected, some keys are missing"
+ return self.fail(status_code=422, msg=msg)
+ except Exception as e:
+ return self.fail(status_code=500, msg="Internal Server Error")
+
+ response.status = 201
+ return "CREATED"
+
+ @expose()
+ def delete(self, *args, **kwargs):
+ if 'uuid' not in kwargs:
+ return self.fail(status_code=400, msg="Required uuid parameter")
+
+ uuid = kwargs['uuid']
+ try:
+ status = db.delete_uuid(uuid)
+ except Exception as e:
+ return self.fail(status_code=500, msg="Internal Server Error")
+
+ if status is not None:
+ return self.fail(status_code=status['status'], msg=status['msg'])
+
+ response.status=200
+ return "DELETED"
diff --git a/src/brag/server/ceph_brag/json.py b/src/brag/server/ceph_brag/json.py
new file mode 100644
index 0000000..bc46702
--- /dev/null
+++ b/src/brag/server/ceph_brag/json.py
@@ -0,0 +1,114 @@
+from pecan.jsonify import jsonify
+from ceph_brag.model import db
+
+ at jsonify.register(db.version_info)
+def jsonify_version(vi):
+ return dict(
+ version_number=vi.version_number,
+ version_date=str(vi.version_date)
+ )
+
+ at jsonify.register(db.cluster_info)
+def jsonify_cluster_info(ci):
+ return dict(
+ uuid=ci.uuid,
+ organization=ci.organization,
+ email=ci.contact_email,
+ cluster_name=ci.cluster_name,
+ cluster_creation_date=str(ci.cluster_creation_date),
+ num_versions=ci.num_versions
+ )
+
+ at jsonify.register(db.components_info)
+def jsonify_components_info(comps):
+ return dict(
+ num_bytes=comps.num_bytes,
+ num_osds=comps.num_osds,
+ num_objects=comps.num_objects,
+ num_pgs=comps.num_pgs,
+ num_pools=comps.num_pools,
+ num_mdss=comps.num_mdss,
+ num_mons=comps.num_mons)
+
+ at jsonify.register(db.crush_types)
+def jsonify_crush_types(crush):
+ return dict(type=crush.crush_type,
+ count=crush.crush_count)
+
+ at jsonify.register(db.pools_info)
+def jsonify_pools_info(pool):
+ return dict(size=pool.pool_rep_size,
+ type=pool.pool_type,
+ id=pool.pool_id)
+
+ at jsonify.register(db.os_info)
+def jsonify_os_info(value):
+ return dict(os=value.os,
+ count=value.count)
+
+ at jsonify.register(db.kernel_versions)
+def jsonify_kernel_versions(value):
+ return dict(version=value.version,
+ count=value.count)
+
+ at jsonify.register(db.kernel_types)
+def jsonify_kernel_types(value):
+ return dict(type=value.type,
+ count=value.count)
+
+ at jsonify.register(db.distros)
+def jsonify_distros(value):
+ return dict(distro=value.distro,
+ count=value.count)
+
+ at jsonify.register(db.cpus)
+def jsonify_cpus(value):
+ return dict(cpu=value.cpu,
+ count=value.count)
+
+ at jsonify.register(db.cpu_archs)
+def jsonify_cpu_archs(value):
+ return dict(arch=value.arch,
+ count=value.count)
+
+ at jsonify.register(db.ceph_versions)
+def jsonify_ceph_versions(value):
+ return dict(version=value.version,
+ count=value.count)
+
+ at jsonify.register(db.sysinfo)
+def jsonify_sysinfo(value):
+ retval = {}
+
+ if value.os:
+ retval['os_info'] = value.os
+ if value.kern_vers:
+ retval['kernel_versions'] = value.kern_vers
+ if value.kern_types:
+ retval['kernel_types'] = value.kern_types
+ if value.distros:
+ retval['distros'] = value.distros
+ if value.cpus:
+ retval['cpus'] = value.cpus
+ if value.cpu_archs:
+ retval['cpu_archs'] = value.cpu_archs
+ if value.ceph_vers:
+ retval['ceph_versions'] = value.ceph_vers
+
+ return retval
+
+ at jsonify.register(db.brag)
+def jsonify_brag(b):
+ ownership = {'organization':b.ci.organization,
+ 'description':b.ci.description,
+ 'email':b.ci.contact_email,
+ 'name':b.ci.cluster_name
+ }
+ return dict(uuid=b.ci.uuid,
+ cluster_creation_date=str(b.ci.cluster_creation_date),
+ components_count=b.comps,
+ crush_types=b.crush,
+ ownership=ownership,
+ pool_metadata=b.pools,
+ sysinfo=b.sysinfo
+ )
diff --git a/src/brag/server/ceph_brag/model/__init__.py b/src/brag/server/ceph_brag/model/__init__.py
new file mode 100644
index 0000000..664427d
--- /dev/null
+++ b/src/brag/server/ceph_brag/model/__init__.py
@@ -0,0 +1,29 @@
+from sqlalchemy import create_engine
+from pecan import conf # noqa
+from db import Session, Base
+import sys
+
+def create_from_conf():
+ configs = dict(conf.sqlalchemy)
+ url = configs.pop('url')
+ return create_engine(url, **configs)
+
+def init_model():
+ engine = create_from_conf()
+ conf.sqlalchemy.engine = engine
+ engine.connect()
+ #create the tables if not existing
+ Base.metadata.create_all(engine)
+
+def start():
+ Session.bind = conf.sqlalchemy.engine
+
+def commit():
+ Session.commit()
+
+def rollback():
+ Session.rollback()
+
+def clear():
+ Session.remove()
+
diff --git a/src/brag/server/ceph_brag/model/db.py b/src/brag/server/ceph_brag/model/db.py
new file mode 100644
index 0000000..94d98ff
--- /dev/null
+++ b/src/brag/server/ceph_brag/model/db.py
@@ -0,0 +1,282 @@
+import json
+from datetime import datetime
+from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy import Column, Integer, String, \
+ DateTime, ForeignKey, BigInteger
+from sqlalchemy import PrimaryKeyConstraint
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.declarative import declared_attr
+
+Base = declarative_base()
+Session = scoped_session(sessionmaker())
+
+class cluster_info(Base):
+ __tablename__ = 'cluster_info'
+
+ index = Column(Integer, primary_key=True)
+ uuid = Column(String(36), unique=True)
+ organization = Column(String(64))
+ contact_email = Column(String(32))
+ cluster_name = Column(String(32))
+ cluster_creation_date = Column(DateTime)
+ description = Column(String(32))
+ num_versions = Column(Integer)
+
+class version_info(Base):
+ __tablename__ = 'version_info'
+
+ index = Column(Integer, primary_key=True)
+ cluster_id = Column(ForeignKey('cluster_info.index'))
+ version_number = Column(Integer)
+ version_date = Column(DateTime)
+
+class components_info(Base):
+ __tablename__ = 'components_info'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ num_bytes = Column(BigInteger)
+ num_osds = Column(Integer)
+ num_objects = Column(Integer)
+ num_pgs = Column(Integer)
+ num_pools = Column(Integer)
+ num_mdss = Column(Integer)
+ num_mons = Column(Integer)
+
+class crush_types(Base):
+ __tablename__ = 'crush_types'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ crush_type = Column(String(16))
+ crush_count = Column(Integer)
+
+class pools_info(Base):
+ __tablename__ = 'pools_info'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ pool_id = Column(Integer)
+ pool_type = Column(Integer)
+ pool_rep_size = Column(Integer)
+
+class os_info(Base):
+ __tablename__ = 'os_info'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ os = Column(String(16))
+ count = Column(Integer)
+
+class kernel_versions(Base):
+ __tablename__ = 'kernel_versions'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ version = Column(String(16))
+ count = Column(Integer)
+
+class kernel_types(Base):
+ __tablename__ = 'kernel_types'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ type = Column(String(64))
+ count = Column(Integer)
+
+class distros(Base):
+ __tablename__ = 'distros'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ distro = Column(String(64))
+ count = Column(Integer)
+
+class cpus(Base):
+ __tablename__ = 'cpus'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ cpu = Column(String(16))
+ count = Column(Integer)
+
+class cpu_archs(Base):
+ __tablename__ = 'cpu_archs'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ arch = Column(String(16))
+ count = Column(Integer)
+
+class ceph_versions(Base):
+ __tablename__ = 'ceph_versions'
+
+ index = Column(Integer, primary_key=True)
+ vid = Column(ForeignKey('version_info.index'))
+ version = Column(String(16))
+ count = Column(Integer)
+
+class sysinfo(object):
+ def __init__(self, vindex):
+ self.os = Session.query(os_info).filter_by(vid=vindex).all()
+ self.kern_vers = Session.query(kernel_versions).filter_by(vid=vindex).all()
+ self.kern_types = Session.query(kernel_types).filter_by(vid=vindex).all()
+ self.distros = Session.query(distros).filter_by(vid=vindex).all()
+ self.cpus = Session.query(cpus).filter_by(vid=vindex).all()
+ self.cpu_archs = Session.query(cpu_archs).filter_by(vid=vindex).all()
+ self.ceph_vers = Session.query(ceph_versions).filter_by(vid=vindex).all()
+
+class brag(object):
+ def __init__(self, uuid, version_number):
+ self.ci = Session.query(cluster_info).filter_by(uuid=uuid).first()
+ if self.ci is not None:
+ self.vi = Session.query(version_info).filter_by(cluster_id=self.ci.index, version_number=version_number).first()
+
+ if self.ci is not None and self.vi is not None:
+ self.comps = Session.query(components_info).filter_by(vid=self.vi.index).first()
+ self.crush = Session.query(crush_types).filter_by(vid=self.vi.index).all()
+ self.pools = Session.query(pools_info).filter_by(vid=self.vi.index).all()
+ self.sysinfo = sysinfo(self.vi.index)
+
+def put_new_version(data):
+ info = json.loads(data)
+ def add_cluster_info():
+ ci = Session.query(cluster_info).filter_by(uuid=info['uuid']).first()
+ if ci is None:
+ dt = datetime.strptime(info['cluster_creation_date'], "%Y-%m-%d %H:%M:%S.%f")
+ ci = cluster_info(uuid=info['uuid'],
+ organization=info['ownership']['organization'],
+ contact_email=info['ownership']['email'],
+ cluster_name=info['ownership']['name'],
+ description=info['ownership']['description'],
+ cluster_creation_date=dt,
+ num_versions=1)
+ Session.add(ci)
+ Session.commit()
+ else:
+ ci.num_versions += 1
+
+ return ci
+
+ def add_version_info(ci):
+ vi = version_info(cluster_id=ci.index,
+ version_number=ci.num_versions,
+ version_date=datetime.now())
+ Session.add(vi)
+ return vi
+
+ def add_components_info(vi):
+ comps_count= info['components_count']
+ comps_info = components_info(vid=vi.index,
+ num_bytes=comps_count['num_bytes'],
+ num_osds=comps_count['num_osds'],
+ num_objects=comps_count['num_objects'],
+ num_pgs=comps_count['num_pgs'],
+ num_pools=comps_count['num_pools'],
+ num_mdss=comps_count['num_mdss'],
+ num_mons=comps_count['num_mons'])
+ Session.add(comps_info)
+
+ def add_crush_types(vi):
+ for c in info['crush_types']:
+ Session.add(crush_types(vid=vi.index,
+ crush_type=c['type'],
+ crush_count=c['count']))
+
+ def add_pools_info(vi):
+ pools = info['pool_metadata']
+ for p in pools:
+ Session.add(pools_info(vid=vi.index,
+ pool_id=p['id'],
+ pool_type=p['type'],
+ pool_rep_size=p['size']))
+
+ def add_sys_info(vi):
+ si = info['sysinfo']
+ while si:
+ k,v = si.popitem()
+ if k == 'os_info':
+ for o in v:
+ Session.add(os_info(vid=vi.index,
+ os=o['os'],
+ count=o['count']))
+ elif k == 'kernel_versions':
+ for k in v:
+ Session.add(kernel_versions(vid=vi.index,
+ version=k['version'],
+ count=k['count']))
+ elif k == 'kernel_types':
+ for k in v:
+ Session.add(kernel_types(vid=vi.index,
+ type=k['type'],
+ count=k['count']))
+ elif k == 'distros':
+ for d in v:
+ Session.add(distros(vid=vi.index,
+ distro=d['distro'],
+ count=d['count']))
+ elif k == 'cpus':
+ for c in v:
+ Session.add(cpus(vid=vi.index,
+ cpu=c['cpu'],
+ count=c['count']))
+ elif k == 'cpu_archs':
+ for c in v:
+ Session.add(cpu_archs(vid=vi.index,
+ arch=c['arch'],
+ count=c['count']))
+ elif k == 'ceph_versions':
+ for c in v:
+ Session.add(ceph_versions(vid=vi.index,
+ version=c['version'],
+ count=c['count']))
+
+ ci = add_cluster_info()
+ add_version_info(ci)
+ vi = Session.query(version_info).filter_by(cluster_id=ci.index,
+ version_number=ci.num_versions).first()
+ add_components_info(vi)
+ add_crush_types(vi)
+ add_pools_info(vi)
+ add_sys_info(vi)
+
+def delete_uuid(uuid):
+ ci = Session.query(cluster_info).filter_by(uuid=uuid).first()
+ if ci is None:
+ return {'status':400, 'msg':'No information for this UUID'}
+
+ for v in Session.query(version_info).filter_by(cluster_id=ci.index).all():
+ Session.query(components_info).filter_by(vid=v.index).delete()
+ Session.query(crush_types).filter_by(vid=v.index).delete()
+ Session.query(pools_info).filter_by(vid=v.index).delete()
+ Session.query(os_info).filter_by(vid=v.index).delete()
+ Session.query(kernel_versions).filter_by(vid=v.index).delete()
+ Session.query(kernel_types).filter_by(vid=v.index).delete()
+ Session.query(distros).filter_by(vid=v.index).delete()
+ Session.query(cpus).filter_by(vid=v.index).delete()
+ Session.query(cpu_archs).filter_by(vid=v.index).delete()
+ Session.query(ceph_versions).filter_by(vid=v.index).delete()
+
+ Session.flush()
+ Session.delete(v)
+ Session.flush()
+
+ Session.delete(ci)
+ return None
+
+def get_uuids():
+ return Session.query(cluster_info).all()
+
+def get_versions(uuid):
+ ci = Session.query(cluster_info).filter_by(uuid=uuid).first()
+ if ci is None:
+ return None
+
+ return Session.query(version_info).filter_by(cluster_id=ci.index).all()
+
+def get_brag(uuid, version_id):
+ b = brag(uuid, version_id)
+ if b.ci is None or b.vi is None:
+ return None
+
+ return b
diff --git a/src/brag/server/ceph_brag/tests/__init__.py b/src/brag/server/ceph_brag/tests/__init__.py
new file mode 100644
index 0000000..78ea527
--- /dev/null
+++ b/src/brag/server/ceph_brag/tests/__init__.py
@@ -0,0 +1,22 @@
+import os
+from unittest import TestCase
+from pecan import set_config
+from pecan.testing import load_test_app
+
+__all__ = ['FunctionalTest']
+
+
+class FunctionalTest(TestCase):
+ """
+ Used for functional tests where you need to test your
+ literal application and its integration with the framework.
+ """
+
+ def setUp(self):
+ self.app = load_test_app(os.path.join(
+ os.path.dirname(__file__),
+ 'config.py'
+ ))
+
+ def tearDown(self):
+ set_config({}, overwrite=True)
diff --git a/src/brag/server/ceph_brag/tests/config.py b/src/brag/server/ceph_brag/tests/config.py
new file mode 100644
index 0000000..b6190da
--- /dev/null
+++ b/src/brag/server/ceph_brag/tests/config.py
@@ -0,0 +1,54 @@
+# Server Specific Configurations
+server = {
+ 'port': '8080',
+ 'host': '0.0.0.0'
+}
+
+# Pecan Application Configurations
+app = {
+ 'root': 'ceph_brag.controllers.root.RootController',
+ 'modules': ['ceph_brag'],
+ 'static_root': '%(confdir)s/public',
+ 'template_path': '%(confdir)s/ceph_brag/templates',
+ 'debug': True,
+ 'errors': {
+ 404: '/error/404',
+ '__force_dict__': True
+ }
+}
+
+logging = {
+ 'loggers': {
+ 'root': {'level': 'INFO', 'handlers': ['console']},
+ 'ceph_brag': {'level': 'DEBUG', 'handlers': ['console']},
+ 'py.warnings': {'handlers': ['console']},
+ '__force_dict__': True
+ },
+ 'handlers': {
+ 'console': {
+ 'level': 'DEBUG',
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'simple'
+ }
+ },
+ 'formatters': {
+ 'simple': {
+ 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
+ '[%(threadName)s] %(message)s')
+ }
+ }
+}
+
+sqlalchemy = {
+ 'url' : 'sqlite:////tmp/test.db',
+ 'echo' : False,
+ 'encoding' : 'utf-8'
+}
+
+
+# Custom Configurations must be in Python dictionary format::
+#
+# foo = {'bar':'baz'}
+#
+# All configurations are accessible at::
+# pecan.conf
diff --git a/src/brag/server/ceph_brag/tests/test_functional.py b/src/brag/server/ceph_brag/tests/test_functional.py
new file mode 100644
index 0000000..13285c4
--- /dev/null
+++ b/src/brag/server/ceph_brag/tests/test_functional.py
@@ -0,0 +1,68 @@
+from unittest import TestCase
+from webtest import TestApp
+from ceph_brag.tests import FunctionalTest
+import json, sys
+from pecan import request
+
+class TestRootController(FunctionalTest):
+ def test_1_get_invalid_url_format(self):
+ response = self.app.get('/1/2/3', expect_errors=True)
+ assert response.status_int == 400
+
+ def test_2_put(self):
+ with open ("sample.json", "r") as myfile:
+ data=myfile.read().replace('\n', '')
+ response = self.app.request('/', method='PUT', body=data)
+ assert response.status_int == 201
+
+ def test_3_put_invalid_json(self):
+ response = self.app.request('/', method='PUT', body='{asdfg', expect_errors=True)
+ assert response.status_int == 422
+
+ def test_4_put_invalid_entries_1(self):
+ response = self.app.request('/', method='PUT', body='{}', expect_errors=True)
+ assert response.status_int == 422
+
+ def test_5_put_incomplete_json(self):
+ response = self.app.request('/', method='PUT', body='{\"uuid\":\"adfs-12312ad\"}',
+ expect_errors=True)
+ assert response.status_int == 422
+
+ def test_6_get(self):
+ response = self.app.get('/')
+ js = json.loads(response.body)
+ for entry in js:
+ ci = entry
+ break
+
+ response = self.app.get('/'+ci['uuid']+'/'+str(ci['num_versions']))
+ assert response.status_int == 200
+
+ def test_7_get_invalid_uuid(self):
+ response = self.app.get('/xxxxxx', expect_errors=True)
+ assert response.status_int == 400
+
+ def test_8_get_invalid_version(self):
+ response = self.app.get('/')
+ js = json.loads(response.body)
+ for entry in js:
+ ci = entry
+ break
+
+ response = self.app.get('/'+ci['uuid']+'/'+str(0), expect_errors=True)
+ assert response.status_int == 400
+
+ def test_9_delete_invalid_parameters(self):
+ response = self.app.delete('/', expect_errors=True)
+ assert response.status_int == 400
+
+ def test_91_delete_wrong_uuid(self):
+ response = self.app.delete('/?uuid=xxxx', expect_errors=True)
+ assert response.status_int == 400
+
+ def test_92_delete(self):
+ response = self.app.get('/')
+ js = json.loads(response.body)
+ for entry in js:
+ response = self.app.delete('/?uuid='+entry['uuid'])
+ assert response.status_int == 200
diff --git a/src/brag/server/ceph_brag/tests/test_units.py b/src/brag/server/ceph_brag/tests/test_units.py
new file mode 100644
index 0000000..573fb68
--- /dev/null
+++ b/src/brag/server/ceph_brag/tests/test_units.py
@@ -0,0 +1,7 @@
+from unittest import TestCase
+
+
+class TestUnits(TestCase):
+
+ def test_units(self):
+ assert 5 * 5 == 25
diff --git a/src/brag/server/config.py b/src/brag/server/config.py
new file mode 100644
index 0000000..429a2ef
--- /dev/null
+++ b/src/brag/server/config.py
@@ -0,0 +1,52 @@
+# Server Specific Configurations
+server = {
+ 'port': '8080',
+ 'host': '0.0.0.0'
+}
+
+# Pecan Application Configurations
+app = {
+ 'root': 'ceph_brag.controllers.root.RootController',
+ 'modules': ['ceph_brag'],
+ 'debug': True,
+ 'errors': {
+ 404: '/error/404',
+ '__force_dict__': True
+ }
+}
+
+logging = {
+ 'loggers': {
+ 'root': {'level': 'INFO', 'handlers': ['console']},
+ 'ceph_brag': {'level': 'DEBUG', 'handlers': ['console']},
+ 'py.warnings': {'handlers': ['console']},
+ '__force_dict__': True
+ },
+ 'handlers': {
+ 'console': {
+ 'level': 'DEBUG',
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'simple'
+ }
+ },
+ 'formatters': {
+ 'simple': {
+ 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
+ '[%(threadName)s] %(message)s')
+ }
+ }
+}
+
+sqlalchemy = {
+ 'url' : 'sqlite:////tmp/test.db',
+ 'echo' : False,
+ 'encoding' : 'utf-8'
+}
+
+
+# Custom Configurations must be in Python dictionary format::
+#
+# foo = {'bar':'baz'}
+#
+# All configurations are accessible at::
+# pecan.conf
diff --git a/src/brag/server/sample.json b/src/brag/server/sample.json
new file mode 100644
index 0000000..194ec63
--- /dev/null
+++ b/src/brag/server/sample.json
@@ -0,0 +1,98 @@
+{
+ "cluster_creation_date": "2014-01-16 13:38:41.928551",
+ "uuid": "20679d0e-04b1-4004-8ee9-45ac271510e9",
+ "components_count": {
+ "num_pgs": 192,
+ "num_mdss": 1,
+ "num_osds": 1,
+ "num_bytes": 0,
+ "num_pools": 3,
+ "num_mons": 1,
+ "num_objects": 0
+ },
+ "crush_types": [
+ {
+ "count": 1,
+ "type": "osd"
+ },
+ {
+ "count": 1,
+ "type": "rack"
+ },
+ {
+ "count": 1,
+ "type": "host"
+ },
+ {
+ "count": 1,
+ "type": "root"
+ }
+ ],
+ "ownership": {
+ "organization": "eNovance",
+ "description": "Use case1",
+ "name": "Cluster1",
+ "email": "mail at enovance.com"
+ },
+ "pool_metadata": [
+ {
+ "type": 1,
+ "id": 0,
+ "size": 3
+ },
+ {
+ "type": 1,
+ "id": 1,
+ "size": 3
+ },
+ {
+ "type": 1,
+ "id": 2,
+ "size": 3
+ }
+ ],
+ "sysinfo": {
+ "kernel_types": [
+ {
+ "count": 1,
+ "type": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012"
+ }
+ ],
+ "cpu_archs": [
+ {
+ "count": 1,
+ "arch": "x86_64"
+ }
+ ],
+ "cpus": [
+ {
+ "count": 1,
+ "cpu": "Intel Xeon E312xx (Sandy Bridge)"
+ }
+ ],
+ "kernel_versions": [
+ {
+ "count": 1,
+ "version": "3.2.0-23-virtual"
+ }
+ ],
+ "ceph_versions": [
+ {
+ "count": 1,
+ "version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)"
+ }
+ ],
+ "os_info": [
+ {
+ "count": 1,
+ "os": "Linux"
+ }
+ ],
+ "distros": [
+ {
+ "count": 1,
+ "distro": "Ubuntu 12.04 precise (Ubuntu 12.04 LTS)"
+ }
+ ]
+ }
+}
diff --git a/src/brag/server/setup.cfg b/src/brag/server/setup.cfg
new file mode 100644
index 0000000..20d3a71
--- /dev/null
+++ b/src/brag/server/setup.cfg
@@ -0,0 +1,6 @@
+[nosetests]
+match=^test
+where=ceph_brag
+nocapture=1
+cover-package=ceph_brag
+cover-erase=1
diff --git a/src/brag/server/setup.py b/src/brag/server/setup.py
new file mode 100644
index 0000000..5144320
--- /dev/null
+++ b/src/brag/server/setup.py
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+try:
+ from setuptools import setup, find_packages
+except ImportError:
+ from ez_setup import use_setuptools
+ use_setuptools()
+ from setuptools import setup, find_packages
+
+setup(
+ name='ceph_brag',
+ version='0.1',
+ description='',
+ author='',
+ author_email='',
+ install_requires=[
+ "pecan",
+ ],
+ test_suite='ceph_brag',
+ zip_safe=False,
+ include_package_data=True,
+ packages=find_packages(exclude=['ez_setup'])
+)
diff --git a/src/ceph-disk b/src/ceph-disk
index 53e0afd..5a03639 100755
--- a/src/ceph-disk
+++ b/src/ceph-disk
@@ -303,6 +303,42 @@ def command_check_call(arguments):
return subprocess.check_call(arguments)
+def platform_distro():
+ """
+ Returns a normalized, lower case string without any leading nor trailing
+ whitespace that represents the distribution name of the current machine.
+ """
+ distro = platform_information()[0] or ''
+ return distro.strip().lower()
+
+
+def platform_information():
+ distro, release, codename = platform.linux_distribution()
+ if not codename and 'debian' in distro.lower(): # this could be an empty string in Debian
+ debian_codenames = {
+ '8': 'jessie',
+ '7': 'wheezy',
+ '6': 'squeeze',
+ }
+ major_version = release.split('.')[0]
+ codename = debian_codenames.get(major_version, '')
+
+ # In order to support newer jessie/sid or wheezy/sid strings we test this
+ # if sid is buried in the minor, we should use sid anyway.
+ if not codename and '/' in release:
+ major, minor = release.split('/')
+ if minor == 'sid':
+ codename = minor
+ else:
+ codename = major
+
+ return (
+ str(distro).strip(),
+ str(release).strip(),
+ str(codename).strip()
+ )
+
+
# a device "name" is something like
# sdb
# cciss!c0d1
@@ -954,13 +990,29 @@ def prepare_journal_dev(
# try to make sure the kernel refreshes the table. note
# that if this gets ebusy, we are probably racing with
# udev because it already updated it.. ignore failure here.
- LOG.debug('Calling partprobe on prepared device %s', journal)
- command(
- [
- 'partprobe',
- journal,
- ],
- )
+
+ # On RHEL and CentOS distros, calling partprobe forces a reboot of the
+ # server. Since we are not resizing partitons so we rely on calling
+ # partx
+ if platform_distro().startswith(('centos', 'red')):
+ LOG.info('calling partx on prepared device %s', journal)
+ LOG.info('re-reading known partitions will display errors')
+ command(
+ [
+ 'partx',
+ '-a',
+ journal,
+ ],
+ )
+
+ else:
+ LOG.debug('Calling partprobe on prepared device %s', journal)
+ command(
+ [
+ 'partprobe',
+ journal,
+ ],
+ )
# wait for udev event queue to clear
command(
@@ -1363,14 +1415,30 @@ def main_prepare(args):
# try to make sure the kernel refreshes the table. note
# that if this gets ebusy, we are probably racing with
# udev because it already updated it.. ignore failure here.
- LOG.debug('Calling partprobe on prepared device %s', args.data)
- command(
- [
- 'partprobe',
- args.data,
- ],
- )
+ # On RHEL and CentOS distros, calling partprobe forces a reboot of
+ # the server. Since we are not resizing partitons so we rely on
+ # calling partx
+ if platform_distro().startswith(('centos', 'red')):
+ LOG.info('calling partx on prepared device %s', args.data)
+ LOG.info('re-reading known partitions will display errors')
+
+ command(
+ [
+ 'partx',
+ '-a',
+ args.data,
+ ],
+ )
+
+ else:
+ LOG.debug('Calling partprobe on prepared device %s', args.data)
+ command(
+ [
+ 'partprobe',
+ args.data,
+ ],
+ )
except Error as e:
if journal_dm_keypath:
diff --git a/src/ceph.in b/src/ceph.in
index c1f5a84..0978882 100755
--- a/src/ceph.in
+++ b/src/ceph.in
@@ -705,8 +705,15 @@ def main():
# Repulsive hack to handle tell: lop off 'tell' and target
# and validate the rest of the command. 'target' is already
# determined in our callers, so it's ok to remove it here.
+ is_tell = False
if len(childargs) and childargs[0] == 'tell':
childargs = childargs[2:]
+ is_tell = True
+
+ if is_tell and not len(childargs):
+ print >> sys.stderr, \
+ 'Cannot use \'tell\' with interactive mode'
+ return errno.EINVAL
# fetch JSON sigs from command
# each line contains one command signature (a placeholder name
diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index f9752b4..0aa6b20 100644
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -412,7 +412,7 @@ int main(int argc, const char **argv)
if (prefork.is_parent()) {
return prefork.parent_wait();
}
- global_init_postfork(g_ceph_context, 0);
+ global_init_postfork_start(g_ceph_context);
}
common_init_finish(g_ceph_context);
global_init_chdir(g_ceph_context);
@@ -435,7 +435,7 @@ int main(int argc, const char **argv)
prefork.exit(1);
}
if (ret > 0) {
- cout << "converting monitor store, please do not interrupt..." << std::endl;
+ dout(0) << "converting monitor store, please do not interrupt..." << dendl;
int r = converter.convert();
if (r) {
derr << "failed to convert monitor store: " << cpp_strerror(r) << dendl;
@@ -452,18 +452,18 @@ int main(int argc, const char **argv)
bufferlist magicbl;
err = store->get(Monitor::MONITOR_NAME, "magic", magicbl);
if (!magicbl.length()) {
- cerr << "unable to read magic from mon data.. did you run mkcephfs?" << std::endl;
+ derr << "unable to read magic from mon data.. did you run mkcephfs?" << dendl;
prefork.exit(1);
}
string magic(magicbl.c_str(), magicbl.length()-1); // ignore trailing \n
if (strcmp(magic.c_str(), CEPH_MON_ONDISK_MAGIC)) {
- cerr << "mon fs magic '" << magic << "' != current '" << CEPH_MON_ONDISK_MAGIC << "'" << std::endl;
+ derr << "mon fs magic '" << magic << "' != current '" << CEPH_MON_ONDISK_MAGIC << "'" << dendl;
prefork.exit(1);
}
err = Monitor::check_features(store);
if (err < 0) {
- cerr << "error checking features: " << cpp_strerror(err) << std::endl;
+ derr << "error checking features: " << cpp_strerror(err) << dendl;
prefork.exit(1);
}
@@ -473,21 +473,23 @@ int main(int argc, const char **argv)
std::string error;
int r = bl.read_file(inject_monmap.c_str(), &error);
if (r) {
- cerr << "unable to read monmap from " << inject_monmap << ": "
- << error << std::endl;
+ derr << "unable to read monmap from " << inject_monmap << ": "
+ << error << dendl;
prefork.exit(1);
}
// get next version
version_t v = store->get("monmap", "last_committed");
- cout << "last committed monmap epoch is " << v << ", injected map will be " << (v+1) << std::endl;
+ dout(0) << "last committed monmap epoch is " << v << ", injected map will be " << (v+1)
+ << dendl;
v++;
// set the version
MonMap tmp;
tmp.decode(bl);
if (tmp.get_epoch() != v) {
- cout << "changing monmap epoch from " << tmp.get_epoch() << " to " << v << std::endl;
+ dout(0) << "changing monmap epoch from " << tmp.get_epoch()
+ << " to " << v << dendl;
tmp.set_epoch(v);
}
bufferlist mapbl;
@@ -503,7 +505,7 @@ int main(int argc, const char **argv)
t.put("monmap", "last_committed", v);
store->apply_transaction(t);
- cout << "done." << std::endl;
+ dout(0) << "done." << dendl;
prefork.exit(0);
}
@@ -549,9 +551,9 @@ int main(int argc, const char **argv)
if (g_conf->get_val_from_conf_file(my_sections, "mon addr",
mon_addr_str, true) == 0) {
if (conf_addr.parse(mon_addr_str.c_str()) && (ipaddr != conf_addr)) {
- cerr << "WARNING: 'mon addr' config option " << conf_addr
+ derr << "WARNING: 'mon addr' config option " << conf_addr
<< " does not match monmap file" << std::endl
- << " continuing with monmap configuration" << std::endl;
+ << " continuing with monmap configuration" << dendl;
}
}
} else {
@@ -568,7 +570,8 @@ int main(int argc, const char **argv)
MonMap tmpmap;
int err = tmpmap.build_initial(g_ceph_context, cerr);
if (err < 0) {
- cerr << argv[0] << ": error generating initial monmap: " << cpp_strerror(err) << std::endl;
+ derr << argv[0] << ": error generating initial monmap: "
+ << cpp_strerror(err) << dendl;
usage();
prefork.exit(1);
}
@@ -626,15 +629,17 @@ int main(int argc, const char **argv)
messenger->set_policy_throttlers(entity_name_t::TYPE_OSD, daemon_throttler, NULL);
messenger->set_policy_throttlers(entity_name_t::TYPE_MDS, daemon_throttler, NULL);
- cout << "starting " << g_conf->name << " rank " << rank
+ dout(0) << "starting " << g_conf->name << " rank " << rank
<< " at " << ipaddr
<< " mon_data " << g_conf->mon_data
<< " fsid " << monmap.get_fsid()
- << std::endl;
+ << dendl;
err = messenger->bind(ipaddr);
- if (err < 0)
+ if (err < 0) {
+ derr << "unable to bind monitor to " << ipaddr << dendl;
prefork.exit(1);
+ }
// start monitor
mon = new Monitor(g_ceph_context, g_conf->name.get_id(), store,
@@ -646,8 +651,10 @@ int main(int argc, const char **argv)
}
err = mon->preinit();
- if (err < 0)
+ if (err < 0) {
+ derr << "failed to initialize" << dendl;
prefork.exit(1);
+ }
if (compact || g_conf->mon_compact_on_start) {
derr << "compacting monitor store ..." << dendl;
@@ -655,8 +662,10 @@ int main(int argc, const char **argv)
derr << "done compacting" << dendl;
}
- if (g_conf->daemonize)
+ if (g_conf->daemonize) {
+ global_init_postfork_finish(g_ceph_context, 0);
prefork.daemonize();
+ }
messenger->start();
diff --git a/src/client/Client.cc b/src/client/Client.cc
index d7d9a50..c770286 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -1,4 +1,4 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
@@ -7,13 +7,12 @@
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
+ * License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
- *
+ *
*/
-
// unix-ey fs stuff
#include <unistd.h>
#include <sys/types.h>
@@ -111,6 +110,8 @@ bool Client::CommandHook::call(std::string command, cmdmap_t& cmdmap,
std::string format, bufferlist& out)
{
Formatter *f = new_formatter(format);
+ if (!f)
+ f = new_formatter("json-pretty");
f->open_object_section("result");
m_client->client_lock.Lock();
if (command == "mds_requests")
@@ -134,7 +135,7 @@ bool Client::CommandHook::call(std::string command, cmdmap_t& cmdmap,
dir_result_t::dir_result_t(Inode *in)
: inode(in), offset(0), this_offset(2), next_offset(2),
release_count(0), start_shared_gen(0),
- buffer(0) {
+ buffer(0) {
inode->get();
}
@@ -168,7 +169,7 @@ Client::Client(Messenger *m, MonClient *mc)
cwd = NULL;
- //
+ //
root = 0;
num_flushing_caps = 0;
@@ -185,7 +186,8 @@ Client::Client(Messenger *m, MonClient *mc)
// osd interfaces
osdmap = new OSDMap; // initially blank.. see mount()
mdsmap = new MDSMap;
- objecter = new Objecter(cct, messenger, monclient, osdmap, client_lock, timer);
+ objecter = new Objecter(cct, messenger, monclient, osdmap, client_lock, timer,
+ 0, 0);
objecter->set_client_incarnation(0); // client always 0, for now.
writeback_handler = new ObjecterWriteback(objecter);
objectcacher = new ObjectCacher(cct, "libcephfs", *writeback_handler, client_lock,
@@ -201,7 +203,7 @@ Client::Client(Messenger *m, MonClient *mc)
}
-Client::~Client()
+Client::~Client()
{
assert(!client_lock.is_locked());
@@ -260,6 +262,12 @@ inodeno_t Client::get_root_ino()
return root->ino;
}
+Inode *Client::get_root()
+{
+ root->ll_get();
+ return root;
+}
+
// debug crapola
@@ -590,7 +598,8 @@ void Client::update_inode_file_bits(Inode *in,
}
}
-Inode * Client::add_update_inode(InodeStat *st, utime_t from, MetaSession *session)
+Inode * Client::add_update_inode(InodeStat *st, utime_t from,
+ MetaSession *session)
{
Inode *in;
bool was_new = false;
@@ -665,11 +674,14 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, MetaSession *sessi
in->layout = st->layout;
in->ctime = st->ctime;
in->max_size = st->max_size; // right?
-
+
update_inode_file_bits(in, st->truncate_seq, st->truncate_size, st->size,
st->time_warp_seq, st->ctime, st->mtime, st->atime,
st->inline_version, st->inline_data,
issued);
+ } else if (st->inline_version > in->inline_version) {
+ in->inline_data = st->inline_data;
+ in->inline_version = st->inline_version;
}
// move me if/when version reflects fragtree changes.
@@ -696,7 +708,7 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, MetaSession *sessi
unlink(in->dir->dentry_map.begin()->second, true);
close_dir(in->dir);
}
- }
+ }
return in;
}
@@ -712,11 +724,11 @@ Dentry *Client::insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dl
Dentry *dn = NULL;
if (dir->dentries.count(dname))
dn = dir->dentries[dname];
-
+
ldout(cct, 12) << "insert_dentry_inode '" << dname << "' vino " << in->vino()
<< " in dir " << dir->parent_inode->vino() << " dn " << dn
<< dendl;
-
+
if (dn && dn->inode) {
if (dn->inode->vino() == in->vino()) {
touch_dn(dn);
@@ -2119,8 +2131,8 @@ void Client::handle_lease(MClientLease *m)
Dentry *dn = in->dir->dentries[m->dname];
ldout(cct, 10) << " revoked DN lease on " << dn << dendl;
dn->lease_mds = -1;
- }
-
+ }
+
revoke:
messenger->send_message(new MClientLease(CEPH_MDS_LEASE_RELEASE, seq,
m->get_mask(), m->get_ino(), m->get_first(), m->get_last(), m->dname),
@@ -2263,7 +2275,7 @@ void Client::put_cap_ref(Inode *in, int cap)
if (last) {
if (in->snapid == CEPH_NOSNAP) {
if ((cap & CEPH_CAP_FILE_WR) &&
- in->cap_snaps.size() &&
+ !in->cap_snaps.empty() &&
in->cap_snaps.rbegin()->second->writing) {
ldout(cct, 10) << "put_cap_ref finishing pending cap_snap on " << *in << dendl;
in->cap_snaps.rbegin()->second->writing = 0;
@@ -2300,7 +2312,7 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff)
in->wanted_max_size = endoff;
check_caps(in, false);
}
-
+
if (endoff >= 0 && endoff > (loff_t)in->max_size) {
ldout(cct, 10) << "waiting on max_size, endoff " << endoff << " max_size " << in->max_size << " on " << *in << dendl;
} else if (!in->cap_snaps.empty() && in->cap_snaps.rbegin()->second->writing) {
@@ -2433,7 +2445,7 @@ void Client::check_caps(Inode *in, bool is_delayed)
else
retain |= CEPH_CAP_ANY_SHARED;
}
-
+
ldout(cct, 10) << "check_caps on " << *in
<< " wanted " << ccap_string(wanted)
<< " used " << ccap_string(used)
@@ -2442,13 +2454,13 @@ void Client::check_caps(Inode *in, bool is_delayed)
if (in->snapid != CEPH_NOSNAP)
return; //snap caps last forever, can't write
-
+
if (in->caps.empty())
return; // guard if at end of func
- if (in->cap_snaps.size())
+ if (!in->cap_snaps.empty())
flush_snaps(in);
-
+
if (!is_delayed)
cap_delay_requeue(in);
else
@@ -2974,7 +2986,7 @@ void Client::remove_cap(Cap *cap)
void Client::remove_all_caps(Inode *in)
{
- while (in->caps.size())
+ while (!in->caps.empty())
remove_cap(in->caps.begin()->second);
}
@@ -3273,14 +3285,14 @@ inodeno_t Client::update_snap_trace(bufferlist& bl, bool flush)
SnapRealm *realm = q.front();
q.pop_front();
ldout(cct, 10) << " flushing caps on " << *realm << dendl;
-
+
xlist<Inode*>::iterator p = realm->inodes_with_caps.begin();
while (!p.end()) {
Inode *in = *p;
++p;
queue_cap_snap(in, realm->get_snap_context().seq);
}
-
+
for (set<SnapRealm*>::iterator p = realm->pchildren.begin();
p != realm->pchildren.end();
++p)
@@ -3616,7 +3628,7 @@ void Client::handle_cap_flushsnap_ack(MetaSession *session, Inode *in, MClientCa
<< " on " << *in << dendl;
// we may not have it if we send multiple FLUSHSNAP requests and (got multiple FLUSHEDSNAPs back)
}
-
+
m->put();
}
@@ -3668,14 +3680,13 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
const int old_caps = cap->issued;
const int new_caps = m->get_caps();
ldout(cct, 5) << "handle_cap_grant on in " << m->get_ino()
- << " mds." << mds << " seq " << m->get_seq()
- << " caps now " << ccap_string(new_caps)
- << " was " << ccap_string(old_caps) << dendl;
-
+ << " mds." << mds << " seq " << m->get_seq()
+ << " caps now " << ccap_string(new_caps)
+ << " was " << ccap_string(old_caps) << dendl;
cap->seq = m->get_seq();
in->layout = m->get_layout();
-
+
// update inode
int implemented = 0;
int issued = in->caps_issued(&implemented) | in->caps_dirty();
@@ -3771,6 +3782,18 @@ int Client::check_permissions(Inode *in, int flags, int uid, int gid)
return 0;
}
+vinodeno_t Client::_get_vino(Inode *in)
+{
+ /* The caller must hold the client lock */
+ return vinodeno_t(in->ino, in->snapid);
+}
+
+inodeno_t Client::_get_inodeno(Inode *in)
+{
+ /* The caller must hold the client lock */
+ return in->ino;
+}
+
// -------------------
// MOUNT
@@ -3801,7 +3824,7 @@ int Client::mount(const std::string &mount_root)
<< " and mdsmap " << mdsmap->get_epoch()
<< dendl;
-
+
// hack: get+pin root inode.
// fuse assumes it's always there.
MetaRequest *req = new MetaRequest(CEPH_MDS_OP_GETATTR);
@@ -3884,15 +3907,15 @@ void Client::unmount()
// flush/release all buffered data
ceph::unordered_map<vinodeno_t, Inode*>::iterator next;
for (ceph::unordered_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
- p != inode_map.end();
- p = next) {
+ p != inode_map.end();
+ p = next) {
next = p;
++next;
Inode *in = p->second;
if (!in) {
ldout(cct, 0) << "null inode_map entry ino " << p->first << dendl;
assert(in);
- }
+ }
if (!in->caps.empty()) {
in->get();
_release(in);
@@ -4082,7 +4105,7 @@ int Client::_lookup(Inode *dir, const string& dname, Inode **target)
r = -ENAMETOOLONG;
goto done;
}
-
+
if (dname == cct->_conf->client_snapdir &&
dir->snapid == CEPH_NOSNAP) {
*target = open_snapdir(dir);
@@ -4099,7 +4122,7 @@ int Client::_lookup(Inode *dir, const string& dname, Inode **target)
// is dn lease valid?
utime_t now = ceph_clock_now(cct);
- if (dn->lease_mds >= 0 &&
+ if (dn->lease_mds >= 0 &&
dn->lease_ttl > now &&
mds_sessions.count(dn->lease_mds)) {
MetaSession *s = mds_sessions[dn->lease_mds];
@@ -4449,13 +4472,18 @@ int Client::readlink(const char *relpath, char *buf, loff_t size)
int r = path_walk(path, &in, false);
if (r < 0)
return r;
-
+
+ return _readlink(in, buf, size);
+}
+
+int Client::_readlink(Inode *in, char *buf, size_t size)
+{
if (!in->is_symlink())
return -EINVAL;
// copy into buf (at most size bytes)
- r = in->symlink.length();
- if (r > size)
+ int r = in->symlink.length();
+ if (r > (int)size)
r = size;
memcpy(buf, in->symlink.c_str(), r);
return r;
@@ -4484,11 +4512,13 @@ int Client::_getattr(Inode *in, int mask, int uid, int gid, bool force)
return res;
}
-int Client::_setattr(Inode *in, struct stat *attr, int mask, int uid, int gid, Inode **inp)
+int Client::_setattr(Inode *in, struct stat *attr, int mask, int uid, int gid,
+ Inode **inp)
{
int issued = in->caps_issued();
- ldout(cct, 10) << "_setattr mask " << mask << " issued " << ccap_string(issued) << dendl;
+ ldout(cct, 10) << "_setattr mask " << mask << " issued " <<
+ ccap_string(issued) << dendl;
if (in->snapid != CEPH_NOSNAP) {
return -EROFS;
@@ -5157,6 +5187,8 @@ int Client::_readdir_cache_cb(dir_result_t *dirp, add_dirent_cb_t cb, void *p)
prev_name = dn->name;
dirp->offset = next_off;
+ if (r > 0)
+ return r;
}
ldout(cct, 10) << "_readdir_cache_cb " << dirp << " on " << dirp->inode->ino << " at end" << dendl;
@@ -5205,6 +5237,8 @@ int Client::readdir_r_cb(dir_result_t *d, add_dirent_cb_t cb, void *p)
dirp->offset = next_off;
off = next_off;
+ if (r > 0)
+ return r;
}
if (dirp->offset == 1) {
ldout(cct, 15) << " including .." << dendl;
@@ -5227,6 +5261,8 @@ int Client::readdir_r_cb(dir_result_t *d, add_dirent_cb_t cb, void *p)
dirp->offset = 2;
off = 2;
+ if (r > 0)
+ return r;
}
// can we read from our cache?
@@ -5241,7 +5277,7 @@ int Client::readdir_r_cb(dir_result_t *d, add_dirent_cb_t cb, void *p)
int err = _readdir_cache_cb(dirp, cb, p);
if (err != -EAGAIN)
return err;
- }
+ }
if (dirp->at_cache_name.length()) {
dirp->last_name = dirp->at_cache_name;
dirp->at_cache_name.clear();
@@ -5283,6 +5319,8 @@ int Client::readdir_r_cb(dir_result_t *d, add_dirent_cb_t cb, void *p)
off++;
dirp->offset++;
+ if (r > 0)
+ return r;
}
if (dirp->last_name.length()) {
@@ -5352,7 +5390,7 @@ static int _readdir_single_dirent_cb(void *p, struct dirent *de, struct stat *st
if (c->stmask)
*c->stmask = stmask;
c->full = true;
- return 0;
+ return 1;
}
struct dirent *Client::readdir(dir_result_t *d)
@@ -5634,7 +5672,7 @@ int Client::_release_fh(Fh *f)
in->snap_cap_refs--;
}
- put_inode( in );
+ put_inode(in);
delete f;
return 0;
@@ -5674,7 +5712,8 @@ int Client::_open(Inode *in, int flags, mode_t mode, Fh **fhp, int uid, int gid)
// success?
if (result >= 0) {
- *fhp = _create_fh(in, flags, cmode);
+ if (fhp)
+ *fhp = _create_fh(in, flags, cmode);
} else {
in->put_open_ref(cmode);
}
@@ -5724,7 +5763,7 @@ loff_t Client::_lseek(Fh *f, loff_t offset, int whence)
int r;
switch (whence) {
- case SEEK_SET:
+ case SEEK_SET:
f->pos = offset;
break;
@@ -5742,7 +5781,7 @@ loff_t Client::_lseek(Fh *f, loff_t offset, int whence)
default:
assert(0);
}
-
+
ldout(cct, 3) << "_lseek(" << f << ", " << offset << ", " << whence << ") = " << f->pos << dendl;
return f->pos;
}
@@ -5762,7 +5801,7 @@ void Client::lock_fh_pos(Fh *f)
assert(f->pos_waiters.front() == &cond);
f->pos_waiters.pop_front();
}
-
+
f->pos_locked = true;
}
@@ -5819,11 +5858,11 @@ int Client::uninline_data(Inode *in, Context *onfinish)
return 0;
}
-//
+//
// blocking osd interface
-int Client::read(int fd, char *buf, loff_t size, loff_t offset)
+int Client::read(int fd, char *buf, loff_t size, loff_t offset)
{
Mutex::Locker lock(client_lock);
tout(cct) << "read" << std::endl;
@@ -5851,6 +5890,13 @@ int Client::_read(Fh *f, int64_t offset, uint64_t size, bufferlist *bl)
//bool lazy = f->mode == CEPH_FILE_MODE_LAZY;
+ if (in->inline_version == 0) {
+ int r = _getattr(in, CEPH_STAT_CAP_INLINE_DATA, -1, -1, true);
+ if (r < 0)
+ return r;
+ assert(in->inline_version > 0);
+ }
+
int have;
int r = get_caps(in, CEPH_CAP_FILE_RD, CEPH_CAP_FILE_CACHE, &have, -1);
if (r < 0)
@@ -6025,7 +6071,7 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl)
}
}
}
-
+
// read (and possibly block)
int r, rvalue = 0;
Mutex flock("Client::_read_async flock");
@@ -6033,11 +6079,11 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl)
bool done = false;
Context *onfinish = new C_SafeCond(&flock, &cond, &done, &rvalue);
r = objectcacher->file_read(&in->oset, &in->layout, in->snapid,
- off, len, bl, 0, onfinish);
+ off, len, bl, 0, onfinish);
if (r == 0) {
client_lock.Unlock();
flock.Lock();
- while (!done)
+ while (!done)
cond.Wait(flock);
flock.Unlock();
client_lock.Lock();
@@ -6065,7 +6111,7 @@ int Client::_read_sync(Fh *f, uint64_t off, uint64_t len, bufferlist *bl)
bool done = false;
Context *onfinish = new C_SafeCond(&flock, &cond, &done, &r);
bufferlist tbl;
-
+
int wanted = left;
filer->read_trunc(in->ino, &in->layout, in->snapid,
pos, left, &tbl, 0,
@@ -6191,7 +6237,7 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
// use/adjust fd pos?
if (offset < 0) {
lock_fh_pos(f);
- /*
+ /*
* FIXME: this is racy in that we may block _after_ this point waiting for caps, and size may
* change out from under us.
*/
@@ -6209,6 +6255,13 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
// time it.
utime_t start = ceph_clock_now(cct);
+ if (in->inline_version == 0) {
+ int r = _getattr(in, CEPH_STAT_CAP_INLINE_DATA, -1, -1, true);
+ if (r < 0)
+ return r;
+ assert(in->inline_version > 0);
+ }
+
// copy into fresh buffer (since our write may be resub, async)
bufferptr bp;
if (size > 0) bp = buffer::copy(buf, size);
@@ -6305,6 +6358,7 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
client_lock.Unlock();
flock.Lock();
+
while (!done)
cond.Wait(flock);
flock.Unlock();
@@ -6521,7 +6575,7 @@ void Client::getcwd(string& dir)
if (!dn) {
// look it up
ldout(cct, 10) << "getcwd looking up parent for " << *in << dendl;
- MetaRequest *req = new MetaRequest(CEPH_MDS_OP_LOOKUPPARENT);
+ MetaRequest *req = new MetaRequest(CEPH_MDS_OP_LOOKUPNAME);
filepath path(in->ino);
req->set_filepath(path);
req->set_inode(in);
@@ -6557,7 +6611,7 @@ int Client::statfs(const char *path, struct statvfs *stbuf)
client_lock.Unlock();
lock.Lock();
- while (!done)
+ while (!done)
cond.Wait(lock);
lock.Unlock();
client_lock.Lock();
@@ -6587,9 +6641,11 @@ int Client::statfs(const char *path, struct statvfs *stbuf)
return rval;
}
-int Client::ll_statfs(vinodeno_t vino, struct statvfs *stbuf)
+int Client::ll_statfs(Inode *in, struct statvfs *stbuf)
{
- tout(cct) << "ll_statfs" << std::endl;
+ /* Since the only thing this does is wrap a call to statfs, and
+ statfs takes a lock, it doesn't seem we have a need to split it
+ out. */
return statfs(0, stbuf);
}
@@ -6767,34 +6823,19 @@ Inode *Client::open_snapdir(Inode *diri)
return in;
}
-int Client::ll_lookup(vinodeno_t parent, const char *name, struct stat *attr, int uid, int gid)
+int Client::ll_lookup(Inode *parent, const char *name, struct stat *attr,
+ Inode **out, int uid, int gid)
{
Mutex::Locker lock(client_lock);
ldout(cct, 3) << "ll_lookup " << parent << " " << name << dendl;
tout(cct) << "ll_lookup" << std::endl;
- tout(cct) << parent.ino.val << std::endl;
tout(cct) << name << std::endl;
- string dname = name;
- Inode *diri = 0;
- Inode *in = 0;
+ string dname(name);
+ Inode *in;
int r = 0;
- if (inode_map.count(parent) == 0) {
- ldout(cct, 1) << "ll_lookup " << parent << " " << name << " -> ENOENT (parent DNE... WTF)" << dendl;
- r = -ENOENT;
- attr->st_ino = 0;
- goto out;
- }
- diri = inode_map[parent];
- if (!diri->is_dir()) {
- ldout(cct, 1) << "ll_lookup " << parent << " " << name << " -> ENOTDIR (parent not a dir... WTF)" << dendl;
- r = -ENOTDIR;
- attr->st_ino = 0;
- goto out;
- }
-
- r = _lookup(diri, dname.c_str(), &in);
+ r = _lookup(parent, dname, &in);
if (r < 0) {
attr->st_ino = 0;
goto out;
@@ -6808,12 +6849,40 @@ int Client::ll_lookup(vinodeno_t parent, const char *name, struct stat *attr, in
ldout(cct, 3) << "ll_lookup " << parent << " " << name
<< " -> " << r << " (" << hex << attr->st_ino << dec << ")" << dendl;
tout(cct) << attr->st_ino << std::endl;
+ *out = in;
return r;
}
+int Client::ll_walk(const char* name, Inode **i, struct stat *attr)
+{
+ Mutex::Locker lock(client_lock);
+ filepath fp(name, 0);
+ Inode *destination = NULL;
+ int rc;
+
+ ldout(cct, 3) << "ll_walk" << name << dendl;
+ tout(cct) << "ll_walk" << std::endl;
+ tout(cct) << name << std::endl;
+
+ rc = path_walk(fp, &destination, false);
+ if (rc < 0)
+ {
+ attr->st_ino = 0;
+ *i = NULL;
+ return rc;
+ }
+ else
+ {
+ fill_stat(destination, attr);
+ *i = destination;
+ return 0;
+ }
+}
+
+
void Client::_ll_get(Inode *in)
{
- if (in->ll_ref == 0)
+ if (in->ll_ref == 0)
in->get();
in->ll_get();
ldout(cct, 20) << "_ll_get " << in << " " << in->ino << " -> " << in->ll_ref << dendl;
@@ -6846,45 +6915,61 @@ void Client::_ll_drop_pins()
}
}
-bool Client::ll_forget(vinodeno_t vino, int num)
+bool Client::ll_forget(Inode *in, int count)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_forget " << vino << " " << num << dendl;
+ inodeno_t ino = _get_inodeno(in);
+
+ ldout(cct, 3) << "ll_forget " << ino << " " << count << dendl;
tout(cct) << "ll_forget" << std::endl;
- tout(cct) << vino.ino.val << std::endl;
- tout(cct) << num << std::endl;
+ tout(cct) << ino.val << std::endl;
+ tout(cct) << count << std::endl;
- if (vino.ino == 1) return true; // ignore forget on root.
+ if (ino == 1) return true; // ignore forget on root.
bool last = false;
- if (inode_map.count(vino) == 0) {
- ldout(cct, 1) << "WARNING: ll_forget on " << vino << " " << num
- << ", which I don't have" << dendl;
- } else {
- Inode *in = inode_map[vino];
- assert(in);
- if (in->ll_ref < num) {
- ldout(cct, 1) << "WARNING: ll_forget on " << vino << " " << num << ", which only has ll_ref=" << in->ll_ref << dendl;
- _ll_put(in, in->ll_ref);
+ if (in->ll_ref < count) {
+ ldout(cct, 1) << "WARNING: ll_forget on " << ino << " " << count
+ << ", which only has ll_ref=" << in->ll_ref << dendl;
+ _ll_put(in, in->ll_ref);
last = true;
} else {
- if (_ll_put(in, num) == 0)
- last = true;
- }
+ if (_ll_put(in, count) == 0)
+ last = true;
}
+
return last;
}
-Inode *Client::_ll_get_inode(vinodeno_t vino)
+bool Client::ll_put(Inode *in)
{
- assert(inode_map.count(vino));
- return inode_map[vino];
+ /* ll_forget already takes the lock */
+ return ll_forget(in, 1);
}
+snapid_t Client::ll_get_snapid(Inode *in)
+{
+ Mutex::Locker lock(client_lock);
+ return in->snapid;
+}
-int Client::ll_getattr(vinodeno_t vino, struct stat *attr, int uid, int gid)
+Inode *Client::ll_get_inode(vinodeno_t vino)
{
Mutex::Locker lock(client_lock);
+ unordered_map<vinodeno_t,Inode*>::iterator p = inode_map.find(vino);
+ if (p == inode_map.end())
+ return NULL;
+ Inode *in = p->second;
+ _ll_get(in);
+ return in;
+}
+
+int Client::ll_getattr(Inode *in, struct stat *attr, int uid, int gid)
+{
+ Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_getattr " << vino << dendl;
tout(cct) << "ll_getattr" << std::endl;
tout(cct) << vino.ino.val << std::endl;
@@ -6896,7 +6981,6 @@ int Client::ll_getattr(vinodeno_t vino, struct stat *attr, int uid, int gid)
return 0;
}
- Inode *in = _ll_get_inode(vino);
int res;
if (vino.snapid < CEPH_NOSNAP)
res = 0;
@@ -6908,10 +6992,15 @@ int Client::ll_getattr(vinodeno_t vino, struct stat *attr, int uid, int gid)
return res;
}
-int Client::ll_setattr(vinodeno_t vino, struct stat *attr, int mask, int uid, int gid)
+int Client::ll_setattr(Inode *in, struct stat *attr, int mask, int uid,
+ int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_setattr " << vino << " mask " << hex << mask << dec << dendl;
+
+ vinodeno_t vino = _get_vino(in);
+
+ ldout(cct, 3) << "ll_setattr " << vino << " mask " << hex << mask << dec
+ << dendl;
tout(cct) << "ll_setattr" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << attr->st_mode << std::endl;
@@ -6922,7 +7011,6 @@ int Client::ll_setattr(vinodeno_t vino, struct stat *attr, int mask, int uid, in
tout(cct) << attr->st_atime << std::endl;
tout(cct) << mask << std::endl;
- Inode *in = _ll_get_inode(vino);
Inode *target = in;
int res = _setattr(in, attr, mask, uid, gid, &target);
if (res == 0) {
@@ -7086,15 +7174,18 @@ int Client::_getxattr(Inode *in, const char *name, void *value, size_t size,
return r;
}
-int Client::ll_getxattr(vinodeno_t vino, const char *name, void *value, size_t size, int uid, int gid)
+int Client::ll_getxattr(Inode *in, const char *name, void *value,
+ size_t size, int uid, int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_getxattr " << vino << " " << name << " size " << size << dendl;
tout(cct) << "ll_getxattr" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << name << std::endl;
- Inode *in = _ll_get_inode(vino);
return _getxattr(in, name, value, size, uid, gid);
}
@@ -7108,6 +7199,7 @@ int Client::_listxattr(Inode *in, char *name, size_t size, int uid, int gid)
p != in->xattrs.end();
++p)
r += p->first.length() + 1;
+
if (in->is_file())
r += sizeof(file_vxattrs);
else if (in->is_dir() && in->has_dir_layout())
@@ -7138,20 +7230,23 @@ int Client::_listxattr(Inode *in, char *name, size_t size, int uid, int gid)
return r;
}
-int Client::ll_listxattr(vinodeno_t vino, char *names, size_t size, int uid, int gid)
+int Client::ll_listxattr(Inode *in, char *names, size_t size, int uid,
+ int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_listxattr " << vino << " size " << size << dendl;
tout(cct) << "ll_listxattr" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << size << std::endl;
- Inode *in = _ll_get_inode(vino);
return _listxattr(in, names, size, uid, gid);
}
-int Client::_setxattr(Inode *in, const char *name, const void *value, size_t size, int flags,
- int uid, int gid)
+int Client::_setxattr(Inode *in, const char *name, const void *value,
+ size_t size, int flags, int uid, int gid)
{
if (in->snapid != CEPH_NOSNAP) {
return -EROFS;
@@ -7164,6 +7259,9 @@ int Client::_setxattr(Inode *in, const char *name, const void *value, size_t siz
strncmp(name, "ceph.", 5))
return -EOPNOTSUPP;
+ if (!value)
+ flags |= CEPH_XATTR_REMOVE;
+
MetaRequest *req = new MetaRequest(CEPH_MDS_OP_SETXATTR);
filepath path;
in->make_nosnap_relative_path(path);
@@ -7175,24 +7273,27 @@ int Client::_setxattr(Inode *in, const char *name, const void *value, size_t siz
bufferlist bl;
bl.append((const char*)value, size);
req->set_data(bl);
-
+
int res = make_request(req, uid, gid);
trim_cache();
- ldout(cct, 3) << "_setxattr(" << in->ino << ", \"" << name << "\") = " << res << dendl;
+ ldout(cct, 3) << "_setxattr(" << in->ino << ", \"" << name << "\") = " <<
+ res << dendl;
return res;
}
-int Client::ll_setxattr(vinodeno_t vino, const char *name, const void *value, size_t size, int flags,
- int uid, int gid)
+int Client::ll_setxattr(Inode *in, const char *name, const void *value,
+ size_t size, int flags, int uid, int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_setxattr " << vino << " " << name << " size " << size << dendl;
tout(cct) << "ll_setxattr" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << name << std::endl;
- Inode *in = _ll_get_inode(vino);
return _setxattr(in, name, value, size, flags, uid, gid);
}
@@ -7224,48 +7325,48 @@ int Client::_removexattr(Inode *in, const char *name, int uid, int gid)
}
-int Client::ll_removexattr(vinodeno_t vino, const char *name, int uid, int gid)
+int Client::ll_removexattr(Inode *in, const char *name, int uid, int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_removexattr " << vino << " " << name << dendl;
tout(cct) << "ll_removexattr" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << name << std::endl;
- Inode *in = _ll_get_inode(vino);
return _removexattr(in, name, uid, gid);
}
-int Client::ll_readlink(vinodeno_t vino, const char **value, int uid, int gid)
+int Client::ll_readlink(Inode *in, char *buf, size_t buflen, int uid, int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_readlink " << vino << dendl;
tout(cct) << "ll_readlink" << std::endl;
tout(cct) << vino.ino.val << std::endl;
- Inode *in = _ll_get_inode(vino);
set<Dentry*>::iterator dn = in->dn_set.begin();
while (dn != in->dn_set.end()) {
touch_dn(*dn);
++dn;
}
- int r = 0;
- if (in->is_symlink()) {
- *value = in->symlink.c_str();
- } else {
- *value = "";
- r = -EINVAL;
- }
- ldout(cct, 3) << "ll_readlink " << vino << " = " << r << " (" << *value << ")" << dendl;
+ int r = _readlink(in, buf, buflen);
+ ldout(cct, 3) << "ll_readlink " << vino << " = " << r << dendl;
return r;
}
-int Client::_mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev, int uid, int gid, Inode **inp)
-{
- ldout(cct, 3) << "_mknod(" << dir->ino << " " << name << ", 0" << oct << mode << dec << ", " << rdev
- << ", uid " << uid << ", gid " << gid << ")" << dendl;
+int Client::_mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev,
+ int uid, int gid, Inode **inp)
+{
+ ldout(cct, 3) << "_mknod(" << dir->ino << " " << name << ", 0" << oct
+ << mode << dec << ", " << rdev << ", uid " << uid << ", gid "
+ << gid << ")" << dendl;
if (strlen(name) > NAME_MAX)
return -ENAMETOOLONG;
@@ -7304,33 +7405,41 @@ int Client::_mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev, int ui
return res;
}
-int Client::ll_mknod(vinodeno_t parent, const char *name, mode_t mode, dev_t rdev, struct stat *attr, int uid, int gid)
+int Client::ll_mknod(Inode *parent, const char *name, mode_t mode,
+ dev_t rdev, struct stat *attr, Inode **out,
+ int uid, int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_mknod " << parent << " " << name << dendl;
+
+ vinodeno_t vparent = _get_vino(parent);
+
+ ldout(cct, 3) << "ll_mknod " << vparent << " " << name << dendl;
tout(cct) << "ll_mknod" << std::endl;
- tout(cct) << parent.ino.val << std::endl;
+ tout(cct) << vparent.ino.val << std::endl;
tout(cct) << name << std::endl;
tout(cct) << mode << std::endl;
tout(cct) << rdev << std::endl;
- Inode *diri = _ll_get_inode(parent);
- Inode *in = 0;
- int r = _mknod(diri, name, mode, rdev, uid, gid, &in);
+ Inode *in = NULL;
+ int r = _mknod(parent, name, mode, rdev, uid, gid, &in);
if (r == 0) {
fill_stat(in, attr);
_ll_get(in);
}
tout(cct) << attr->st_ino << std::endl;
- ldout(cct, 3) << "ll_mknod " << parent << " " << name
+ ldout(cct, 3) << "ll_mknod " << vparent << " " << name
<< " = " << r << " (" << hex << attr->st_ino << dec << ")" << dendl;
+ *out = in;
return r;
}
-int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp,
- int stripe_unit, int stripe_count, int object_size, const char *data_pool, bool *created, int uid, int gid)
+int Client::_create(Inode *dir, const char *name, int flags, mode_t mode,
+ Inode **inp, Fh **fhp, int stripe_unit, int stripe_count,
+ int object_size, const char *data_pool, bool *created,
+ int uid, int gid)
{
- ldout(cct, 3) << "_create(" << dir->ino << " " << name << ", 0" << oct << mode << dec << ")" << dendl;
+ ldout(cct, 3) << "_create(" << dir->ino << " " << name << ", 0" << oct <<
+ mode << dec << ")" << dendl;
if (strlen(name) > NAME_MAX)
return -ENAMETOOLONG;
@@ -7382,8 +7491,11 @@ int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode
goto reply_error;
}
- (*inp)->get_open_ref(cmode);
- *fhp = _create_fh(*inp, flags, cmode);
+ /* If the caller passed a value in fhp, do the open */
+ if(fhp) {
+ (*inp)->get_open_ref(cmode);
+ *fhp = _create_fh(*inp, flags, cmode);
+ }
reply_error:
trim_cache();
@@ -7404,8 +7516,9 @@ int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode
int Client::_mkdir(Inode *dir, const char *name, mode_t mode, int uid, int gid,
Inode **inp)
{
- ldout(cct, 3) << "_mkdir(" << dir->ino << " " << name << ", 0" << oct << mode << dec
- << ", uid " << uid << ", gid " << gid << ")" << dendl;
+ ldout(cct, 3) << "_mkdir(" << dir->ino << " " << name << ", 0" << oct
+ << mode << dec << ", uid " << uid << ", gid " << gid << ")"
+ << dendl;
if (strlen(name) > NAME_MAX)
return -ENAMETOOLONG;
@@ -7413,7 +7526,8 @@ int Client::_mkdir(Inode *dir, const char *name, mode_t mode, int uid, int gid,
if (dir->snapid != CEPH_NOSNAP && dir->snapid != CEPH_SNAPDIR) {
return -EROFS;
}
- MetaRequest *req = new MetaRequest(dir->snapid == CEPH_SNAPDIR ? CEPH_MDS_OP_MKSNAP:CEPH_MDS_OP_MKDIR);
+ MetaRequest *req = new MetaRequest(dir->snapid == CEPH_SNAPDIR ?
+ CEPH_MDS_OP_MKSNAP : CEPH_MDS_OP_MKDIR);
filepath path;
dir->make_nosnap_relative_path(path);
@@ -7422,7 +7536,7 @@ int Client::_mkdir(Inode *dir, const char *name, mode_t mode, int uid, int gid,
req->set_inode(dir);
req->head.args.mkdir.mode = mode;
req->dentry_drop = CEPH_CAP_FILE_SHARED;
- req->dentry_unless = CEPH_CAP_FILE_EXCL;
+ req->dentry_unless = CEPH_CAP_FILE_EXCL;
Dentry *de;
int res = get_or_create(dir, name, &de);
@@ -7444,31 +7558,34 @@ int Client::_mkdir(Inode *dir, const char *name, mode_t mode, int uid, int gid,
return res;
}
-int Client::ll_mkdir(vinodeno_t parent, const char *name, mode_t mode, struct stat *attr, int uid, int gid)
+int Client::ll_mkdir(Inode *parent, const char *name, mode_t mode,
+ struct stat *attr, Inode **out, int uid, int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_mkdir " << parent << " " << name << dendl;
+
+ vinodeno_t vparent = _get_vino(parent);
+
+ ldout(cct, 3) << "ll_mkdir " << vparent << " " << name << dendl;
tout(cct) << "ll_mkdir" << std::endl;
- tout(cct) << parent.ino.val << std::endl;
+ tout(cct) << vparent.ino.val << std::endl;
tout(cct) << name << std::endl;
tout(cct) << mode << std::endl;
- Inode *diri = _ll_get_inode(parent);
-
- Inode *in = 0;
- int r = _mkdir(diri, name, mode, uid, gid, &in);
+ Inode *in = NULL;
+ int r = _mkdir(parent, name, mode, uid, gid, &in);
if (r == 0) {
fill_stat(in, attr);
_ll_get(in);
}
tout(cct) << attr->st_ino << std::endl;
- ldout(cct, 3) << "ll_mkdir " << parent << " " << name
+ ldout(cct, 3) << "ll_mkdir " << vparent << " " << name
<< " = " << r << " (" << hex << attr->st_ino << dec << ")" << dendl;
+ *out = in;
return r;
}
-int Client::_symlink(Inode *dir, const char *name, const char *target, int uid, int gid,
- Inode **inp)
+int Client::_symlink(Inode *dir, const char *name, const char *target, int uid,
+ int gid, Inode **inp)
{
ldout(cct, 3) << "_symlink(" << dir->ino << " " << name << ", " << target
<< ", uid " << uid << ", gid " << gid << ")" << dendl;
@@ -7500,34 +7617,39 @@ int Client::_symlink(Inode *dir, const char *name, const char *target, int uid,
res = make_request(req, uid, gid, inp);
trim_cache();
- ldout(cct, 3) << "_symlink(\"" << path << "\", \"" << target << "\") = " << res << dendl;
+ ldout(cct, 3) << "_symlink(\"" << path << "\", \"" << target << "\") = " <<
+ res << dendl;
return res;
-
fail:
put_request(req);
return res;
}
-int Client::ll_symlink(vinodeno_t parent, const char *name, const char *value, struct stat *attr, int uid, int gid)
+int Client::ll_symlink(Inode *parent, const char *name, const char *value,
+ struct stat *attr, Inode **out, int uid, int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_symlink " << parent << " " << name << " -> " << value << dendl;
+
+ vinodeno_t vparent = _get_vino(parent);
+
+ ldout(cct, 3) << "ll_symlink " << vparent << " " << name << " -> " << value
+ << dendl;
tout(cct) << "ll_symlink" << std::endl;
- tout(cct) << parent.ino.val << std::endl;
+ tout(cct) << vparent.ino.val << std::endl;
tout(cct) << name << std::endl;
tout(cct) << value << std::endl;
- Inode *diri = _ll_get_inode(parent);
- Inode *in = 0;
- int r = _symlink(diri, name, value, uid, gid, &in);
+ Inode *in = NULL;
+ int r = _symlink(parent, name, value, uid, gid, &in);
if (r == 0) {
fill_stat(in, attr);
_ll_get(in);
}
tout(cct) << attr->st_ino << std::endl;
- ldout(cct, 3) << "ll_symlink " << parent << " " << name
+ ldout(cct, 3) << "ll_symlink " << vparent << " " << name
<< " = " << r << " (" << hex << attr->st_ino << dec << ")" << dendl;
+ *out = in;
return r;
}
@@ -7581,21 +7703,24 @@ int Client::_unlink(Inode *dir, const char *name, int uid, int gid)
return res;
}
-int Client::ll_unlink(vinodeno_t vino, const char *name, int uid, int gid)
+int Client::ll_unlink(Inode *in, const char *name, int uid, int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_unlink " << vino << " " << name << dendl;
tout(cct) << "ll_unlink" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << name << std::endl;
- Inode *diri = _ll_get_inode(vino);
- return _unlink(diri, name, uid, gid);
+ return _unlink(in, name, uid, gid);
}
int Client::_rmdir(Inode *dir, const char *name, int uid, int gid)
{
- ldout(cct, 3) << "_rmdir(" << dir->ino << " " << name << " uid " << uid << " gid " << gid << ")" << dendl;
+ ldout(cct, 3) << "_rmdir(" << dir->ino << " " << name << " uid " << uid <<
+ " gid " << gid << ")" << dendl;
if (dir->snapid != CEPH_NOSNAP && dir->snapid != CEPH_SNAPDIR) {
return -EROFS;
@@ -7627,7 +7752,7 @@ int Client::_rmdir(Inode *dir, const char *name, int uid, int gid)
if (dir->dir && dir->dir->dentries.count(name) ) {
Dentry *dn = dir->dir->dentries[name];
if (dn->inode->dir && dn->inode->dir->is_empty() &&
- (dn->inode->dn_set.size() == 1))
+ (dn->inode->dn_set.size() == 1))
close_dir(dn->inode->dir); // FIXME: maybe i shoudl proactively hose the whole subtree from cache?
unlink(dn, false);
}
@@ -7642,16 +7767,18 @@ int Client::_rmdir(Inode *dir, const char *name, int uid, int gid)
return res;
}
-int Client::ll_rmdir(vinodeno_t vino, const char *name, int uid, int gid)
+int Client::ll_rmdir(Inode *in, const char *name, int uid, int gid)
{
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_rmdir " << vino << " " << name << dendl;
tout(cct) << "ll_rmdir" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << name << std::endl;
- Inode *diri = _ll_get_inode(vino);
- return _rmdir(diri, name, uid, gid);
+ return _rmdir(in, name, uid, gid);
}
int Client::_rename(Inode *fromdir, const char *fromname, Inode *todir, const char *toname, int uid, int gid)
@@ -7725,20 +7852,23 @@ int Client::_rename(Inode *fromdir, const char *fromname, Inode *todir, const ch
return res;
}
-int Client::ll_rename(vinodeno_t parent, const char *name, vinodeno_t newparent, const char *newname, int uid, int gid)
+int Client::ll_rename(Inode *parent, const char *name, Inode *newparent,
+ const char *newname, int uid, int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_rename " << parent << " " << name << " to "
- << newparent << " " << newname << dendl;
+
+ vinodeno_t vparent = _get_vino(parent);
+ vinodeno_t vnewparent = _get_vino(newparent);
+
+ ldout(cct, 3) << "ll_rename " << vparent << " " << name << " to "
+ << vnewparent << " " << newname << dendl;
tout(cct) << "ll_rename" << std::endl;
- tout(cct) << parent.ino.val << std::endl;
+ tout(cct) << vparent.ino.val << std::endl;
tout(cct) << name << std::endl;
- tout(cct) << newparent.ino.val << std::endl;
+ tout(cct) << vnewparent.ino.val << std::endl;
tout(cct) << newname << std::endl;
- Inode *fromdiri = _ll_get_inode(parent);
- Inode *todiri = _ll_get_inode(newparent);
- return _rename(fromdiri, name, todiri, newname, uid, gid);
+ return _rename(parent, name, newparent, newname, uid, gid);
}
int Client::_link(Inode *in, Inode *dir, const char *newname, int uid, int gid, Inode **inp)
@@ -7782,83 +7912,162 @@ int Client::_link(Inode *in, Inode *dir, const char *newname, int uid, int gid,
return res;
}
-int Client::ll_link(vinodeno_t vino, vinodeno_t newparent, const char *newname, struct stat *attr, int uid, int gid)
+int Client::ll_link(Inode *parent, Inode *newparent, const char *newname,
+ struct stat *attr, int uid, int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_link " << vino << " to " << newparent << " " << newname << dendl;
+
+ vinodeno_t vparent = _get_vino(parent);
+ vinodeno_t vnewparent = _get_vino(newparent);
+
+ ldout(cct, 3) << "ll_link " << parent << " to " << vnewparent << " " <<
+ newname << dendl;
tout(cct) << "ll_link" << std::endl;
- tout(cct) << vino.ino.val << std::endl;
- tout(cct) << newparent << std::endl;
+ tout(cct) << vparent.ino.val << std::endl;
+ tout(cct) << vnewparent << std::endl;
tout(cct) << newname << std::endl;
- Inode *old = _ll_get_inode(vino);
- Inode *diri = _ll_get_inode(newparent);
-
- int r = _link(old, diri, newname, uid, gid, &old);
+ int r = _link(parent, newparent, newname, uid, gid, &parent);
if (r == 0) {
- Inode *in = _ll_get_inode(vino);
- fill_stat(in, attr);
- _ll_get(in);
+ fill_stat(parent, attr);
+ _ll_get(parent);
}
return r;
}
-int Client::ll_describe_layout(Fh *fh, ceph_file_layout* lp)
+int Client::ll_num_osds(void)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_describe_layout " << fh << " " << fh->inode->ino << dendl;
- tout(cct) << "ll_describe_layout" << std::endl;
+ return osdmap->get_num_osds();
+}
- Inode *in = fh->inode;
- *lp = in->layout;
+int Client::ll_osdaddr(int osd, uint32_t *addr)
+{
+ Mutex::Locker lock(client_lock);
+ entity_addr_t g = osdmap->get_addr(osd);
+ uint32_t nb_addr = (g.in4_addr()).sin_addr.s_addr;
+
+ if (!(osdmap->exists(osd))) {
+ return -1;
+ }
+
+ *addr = ntohl(nb_addr);
return 0;
}
-int Client::ll_opendir(vinodeno_t vino, void **dirpp, int uid, int gid)
+uint32_t Client::ll_stripe_unit(Inode *in)
{
Mutex::Locker lock(client_lock);
+ return in->layout.fl_stripe_unit;
+}
+
+uint64_t Client::ll_snap_seq(Inode *in)
+{
+ Mutex::Locker lock(client_lock);
+ return in->snaprealm->seq;
+}
+
+int Client::ll_file_layout(Inode *in, ceph_file_layout *layout)
+{
+ Mutex::Locker lock(client_lock);
+ *layout = in->layout;
+ return 0;
+}
+
+/* Currently we cannot take advantage of redundancy in reads, since we
+ would have to go through all possible placement groups (a
+ potentially quite large number determined by a hash), and use CRUSH
+ to calculate the appropriate set of OSDs for each placement group,
+ then index into that. An array with one entry per OSD is much more
+ tractable and works for demonstration purposes. */
+
+int Client::ll_get_stripe_osd(Inode *in, uint64_t blockno,
+ ceph_file_layout* layout)
+{
+ Mutex::Locker lock(client_lock);
+ inodeno_t ino = ll_get_inodeno(in);
+ uint32_t object_size = layout->fl_object_size;
+ uint32_t su = layout->fl_stripe_unit;
+ uint32_t stripe_count = layout->fl_stripe_count;
+ uint64_t stripes_per_object = object_size / su;
+
+ uint64_t stripeno = blockno / stripe_count; // which horizontal stripe (Y)
+ uint64_t stripepos = blockno % stripe_count; // which object in the object set (X)
+ uint64_t objectsetno = stripeno / stripes_per_object; // which object set
+ uint64_t objectno = objectsetno * stripe_count + stripepos; // object id
+
+ object_t oid = file_object_t(ino, objectno);
+ ceph_object_layout olayout
+ = objecter->osdmap->file_to_object_layout(oid, *layout, "");
+
+ pg_t pg = (pg_t)olayout.ol_pgid;
+ vector<int> osds;
+ int primary;
+ osdmap->pg_to_osds(pg, &osds, &primary);
+ return osds[0];
+}
+
+/* Return the offset of the block, internal to the object */
+
+uint64_t Client::ll_get_internal_offset(Inode *in, uint64_t blockno)
+{
+ Mutex::Locker lock(client_lock);
+ ceph_file_layout *layout=&(in->layout);
+ uint32_t object_size = layout->fl_object_size;
+ uint32_t su = layout->fl_stripe_unit;
+ uint64_t stripes_per_object = object_size / su;
+
+ return (blockno % stripes_per_object) * su;
+}
+
+int Client::ll_opendir(Inode *in, dir_result_t** dirpp, int uid, int gid)
+{
+ Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_opendir " << vino << dendl;
tout(cct) << "ll_opendir" << std::endl;
tout(cct) << vino.ino.val << std::endl;
-
- Inode *diri = inode_map[vino];
- assert(diri);
int r = 0;
if (vino.snapid == CEPH_SNAPDIR) {
- *dirpp = new dir_result_t(diri);
+ *dirpp = new dir_result_t(in);
} else {
- r = _opendir(diri, (dir_result_t**)dirpp);
+ r = _opendir(in, dirpp);
}
tout(cct) << (unsigned long)*dirpp << std::endl;
- ldout(cct, 3) << "ll_opendir " << vino << " = " << r << " (" << *dirpp << ")" << dendl;
+ ldout(cct, 3) << "ll_opendir " << vino << " = " << r << " (" << *dirpp << ")"
+ << dendl;
return r;
}
-void Client::ll_releasedir(void *dirp)
+int Client::ll_releasedir(dir_result_t *dirp)
{
Mutex::Locker lock(client_lock);
ldout(cct, 3) << "ll_releasedir " << dirp << dendl;
tout(cct) << "ll_releasedir" << std::endl;
tout(cct) << (unsigned long)dirp << std::endl;
- _closedir(static_cast<dir_result_t*>(dirp));
+ _closedir(dirp);
+ return 0;
}
-int Client::ll_open(vinodeno_t vino, int flags, Fh **fhp, int uid, int gid)
+int Client::ll_open(Inode *in, int flags, Fh **fhp, int uid, int gid)
{
assert(!(flags & O_CREAT));
Mutex::Locker lock(client_lock);
+
+ vinodeno_t vino = _get_vino(in);
+
ldout(cct, 3) << "ll_open " << vino << " " << flags << dendl;
tout(cct) << "ll_open" << std::endl;
tout(cct) << vino.ino.val << std::endl;
tout(cct) << flags << std::endl;
- Inode *in = _ll_get_inode(vino);
-
int r;
if (uid < 0) {
uid = geteuid();
@@ -7868,41 +8077,47 @@ int Client::ll_open(vinodeno_t vino, int flags, Fh **fhp, int uid, int gid)
if (r < 0)
goto out;
- r = _open(in, flags, 0, fhp, uid, gid);
+ r = _open(in, flags, 0, fhp /* may be NULL */, uid, gid);
out:
- tout(cct) << (unsigned long)*fhp << std::endl;
- ldout(cct, 3) << "ll_open " << vino << " " << flags << " = " << r << " (" << *fhp << ")" << dendl;
+ Fh *fhptr = fhp ? *fhp : NULL;
+ tout(cct) << (unsigned long)fhptr << std::endl;
+ ldout(cct, 3) << "ll_open " << vino << " " << flags << " = " << r << " (" <<
+ fhptr << ")" << dendl;
return r;
}
-int Client::ll_create(vinodeno_t parent, const char *name, mode_t mode, int flags,
- struct stat *attr, Fh **fhp, int uid, int gid)
+int Client::ll_create(Inode *parent, const char *name, mode_t mode,
+ int flags, struct stat *attr, Inode **outp, Fh **fhp,
+ int uid, int gid)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_create " << parent << " " << name << " 0" << oct << mode << dec << " " << flags << ", uid " << uid << ", gid " << gid << dendl;
+
+ vinodeno_t vparent = _get_vino(parent);
+
+ ldout(cct, 3) << "ll_create " << vparent << " " << name << " 0" << oct <<
+ mode << dec << " " << flags << ", uid " << uid << ", gid " << gid << dendl;
tout(cct) << "ll_create" << std::endl;
- tout(cct) << parent.ino.val << std::endl;
+ tout(cct) << vparent.ino.val << std::endl;
tout(cct) << name << std::endl;
tout(cct) << mode << std::endl;
tout(cct) << flags << std::endl;
- *fhp = NULL;
-
bool created = false;
Inode *in = NULL;
- Inode *dir = _ll_get_inode(parent);
- int r = _lookup(dir, name, &in);
+ int r = _lookup(parent, name, &in);
+
if (r == 0 && (flags & O_CREAT) && (flags & O_EXCL))
return -EEXIST;
- if (r == -ENOENT && (flags & O_CREAT)) {
- r = _create(dir, name, flags, mode, &in, fhp,
- 0, 0, 0,
- NULL, &created, uid, gid);
+
+ if (r == -ENOENT && (flags & O_CREAT)) {
+ r = _create(parent, name, flags, mode, &in, fhp /* may be NULL */,
+ 0, 0, 0, NULL, &created, uid, gid);
if (r < 0)
goto out;
- in = (*fhp)->inode;
+ if ((!in) && fhp)
+ in = (*fhp)->inode;
}
if (r < 0)
@@ -7910,18 +8125,17 @@ int Client::ll_create(vinodeno_t parent, const char *name, mode_t mode, int flag
assert(in);
fill_stat(in, attr);
- _ll_get(in);
ldout(cct, 20) << "ll_create created = " << created << dendl;
if (!created) {
r = check_permissions(in, flags, uid, gid);
if (r < 0) {
- if (*fhp) {
+ if (fhp && *fhp) {
_release_fh(*fhp);
}
goto out;
}
- if (*fhp == NULL) {
+ if (fhp && (*fhp == NULL)) {
r = _open(in, flags, mode, fhp);
if (r < 0)
goto out;
@@ -7932,13 +8146,33 @@ out:
if (r < 0)
attr->st_ino = 0;
- tout(cct) << (unsigned long)*fhp << std::endl;
+ Fh *fhptr = fhp ? *fhp : NULL;
+ tout(cct) << (unsigned long)fhptr << std::endl;
tout(cct) << attr->st_ino << std::endl;
- ldout(cct, 3) << "ll_create " << parent << " " << name << " 0" << oct << mode << dec << " " << flags
- << " = " << r << " (" << *fhp << " " << hex << attr->st_ino << dec << ")" << dendl;
+ ldout(cct, 3) << "ll_create " << parent << " " << name << " 0" << oct <<
+ mode << dec << " " << flags << " = " << r << " (" << fhptr << " " <<
+ hex << attr->st_ino << dec << ")" << dendl;
+
+ // passing an Inode in outp requires an additional ref
+ if (outp) {
+ if (in)
+ _ll_get(in);
+ *outp = in;
+ }
+
return r;
}
+loff_t Client::ll_lseek(Fh *fh, loff_t offset, int whence)
+{
+ Mutex::Locker lock(client_lock);
+ tout(cct) << "ll_lseek" << std::endl;
+ tout(cct) << offset << std::endl;
+ tout(cct) << whence << std::endl;
+
+ return _lseek(fh, offset, whence);
+}
+
int Client::ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl)
{
Mutex::Locker lock(client_lock);
@@ -7951,17 +8185,156 @@ int Client::ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl)
return _read(fh, off, len, bl);
}
+int Client::ll_read_block(Inode *in, uint64_t blockid,
+ char *buf,
+ uint64_t offset,
+ uint64_t length,
+ ceph_file_layout* layout)
+{
+ Mutex::Locker lock(client_lock);
+ Mutex flock("Client::ll_read_block flock");
+ Cond cond;
+ vinodeno_t vino = ll_get_vino(in);
+ object_t oid = file_object_t(vino.ino, blockid);
+ int r = 0;
+ bool done = false;
+ Context *onfinish = new C_SafeCond(&flock, &cond, &done, &r);
+ bufferlist bl;
+
+ objecter->read(oid,
+ object_locator_t(layout->fl_pg_pool),
+ offset,
+ length,
+ vino.snapid,
+ &bl,
+ CEPH_OSD_FLAG_READ,
+ onfinish);
+
+ while (!done)
+ cond.Wait(client_lock);
+
+ if (r >= 0) {
+ bl.copy(0, bl.length(), buf);
+ r = bl.length();
+ }
+
+ return r;
+}
+
+/* It appears that the OSD doesn't return success unless the entire
+ buffer was written, return the write length on success. */
+
+int Client::ll_write_block(Inode *in, uint64_t blockid,
+ char* buf, uint64_t offset,
+ uint64_t length, ceph_file_layout* layout,
+ uint64_t snapseq, uint32_t sync)
+{
+ Mutex flock("Client::ll_write_block flock");
+ vinodeno_t vino = ll_get_vino(in);
+ Cond cond;
+ bool done;
+ int r = 0;
+ Context *onack;
+ Context *onsafe;
+
+ if (length == 0) {
+ return -EINVAL;
+ }
+ if (true || sync) {
+ /* if write is stable, the epilogue is waiting on
+ * flock */
+ onack = new C_NoopContext;
+ onsafe = new C_SafeCond(&flock, &cond, &done, &r);
+ done = false;
+ } else {
+ /* if write is unstable, we just place a barrier for
+ * future commits to wait on */
+ onack = new C_NoopContext;
+ /*onsafe = new C_Block_Sync(this, vino.ino,
+ barrier_interval(offset, offset + length), &r);
+ */
+ done = true;
+ }
+ object_t oid = file_object_t(vino.ino, blockid);
+ SnapContext fakesnap;
+ bufferptr bp;
+ if (length > 0) bp = buffer::copy(buf, length);
+ bufferlist bl;
+ bl.push_back(bp);
+
+ ldout(cct, 1) << "ll_block_write for " << vino.ino << "." << blockid
+ << dendl;
+
+ fakesnap.seq = snapseq;
+
+ /* lock just in time */
+ client_lock.Lock();
+
+ objecter->write(oid,
+ object_locator_t(layout->fl_pg_pool),
+ offset,
+ length,
+ fakesnap,
+ bl,
+ ceph_clock_now(cct),
+ 0,
+ onack,
+ onsafe);
+
+ client_lock.Unlock();
+ if (!done /* also !sync */) {
+ flock.Lock();
+ while (! done)
+ cond.Wait(flock);
+ flock.Unlock();
+ }
+
+ if (r < 0) {
+ return r;
+ } else {
+ return length;
+ }
+}
+
+int Client::ll_commit_blocks(Inode *in,
+ uint64_t offset,
+ uint64_t length)
+{
+ Mutex::Locker lock(client_lock);
+ /*
+ BarrierContext *bctx;
+ vinodeno_t vino = ll_get_vino(in);
+ uint64_t ino = vino.ino;
+
+ ldout(cct, 1) << "ll_commit_blocks for " << vino.ino << " from "
+ << offset << " to " << length << dendl;
+
+ if (length == 0) {
+ return -EINVAL;
+ }
+
+ map<uint64_t, BarrierContext*>::iterator p = barriers.find(ino);
+ if (p != barriers.end()) {
+ barrier_interval civ(offset, offset + length);
+ p->second->commit_barrier(civ);
+ }
+ */
+ return 0;
+}
+
int Client::ll_write(Fh *fh, loff_t off, loff_t len, const char *data)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_write " << fh << " " << fh->inode->ino << " " << off << "~" << len << dendl;
+ ldout(cct, 3) << "ll_write " << fh << " " << fh->inode->ino << " " << off <<
+ "~" << len << dendl;
tout(cct) << "ll_write" << std::endl;
tout(cct) << (unsigned long)fh << std::endl;
tout(cct) << off << std::endl;
tout(cct) << len << std::endl;
int r = _write(fh, off, len, data);
- ldout(cct, 3) << "ll_write " << fh << " " << off << "~" << len << " = " << r << dendl;
+ ldout(cct, 3) << "ll_write " << fh << " " << off << "~" << len << " = " << r
+ << dendl;
return r;
}
@@ -7975,7 +8348,7 @@ int Client::ll_flush(Fh *fh)
return _flush(fh);
}
-int Client::ll_fsync(Fh *fh, bool syncdataonly)
+int Client::ll_fsync(Fh *fh, bool syncdataonly)
{
Mutex::Locker lock(client_lock);
ldout(cct, 3) << "ll_fsync " << fh << " " << fh->inode->ino << " " << dendl;
@@ -8146,8 +8519,9 @@ int Client::fallocate(int fd, int mode, loff_t offset, loff_t length)
int Client::ll_release(Fh *fh)
{
Mutex::Locker lock(client_lock);
- ldout(cct, 3) << "ll_release " << fh << " " << fh->inode->ino << " " << dendl;
- tout(cct) << "ll_release" << std::endl;
+ ldout(cct, 3) << "ll_release (fh)" << fh << " " << fh->inode->ino << " " <<
+ dendl;
+ tout(cct) << "ll_release (fh)" << std::endl;
tout(cct) << (unsigned long)fh << std::endl;
_release_fh(fh);
@@ -8155,10 +8529,6 @@ int Client::ll_release(Fh *fh)
}
-
-
-
-
// =========================================
// layout
@@ -8355,7 +8725,7 @@ void Client::ms_handle_connect(Connection *con)
objecter->ms_handle_connect(con);
}
-bool Client::ms_handle_reset(Connection *con)
+bool Client::ms_handle_reset(Connection *con)
{
ldout(cct, 0) << "ms_handle_reset on " << con->get_peer_addr() << dendl;
Mutex::Locker l(client_lock);
@@ -8363,7 +8733,7 @@ bool Client::ms_handle_reset(Connection *con)
return false;
}
-void Client::ms_handle_remote_reset(Connection *con)
+void Client::ms_handle_remote_reset(Connection *con)
{
ldout(cct, 0) << "ms_handle_remote_reset on " << con->get_peer_addr() << dendl;
Mutex::Locker l(client_lock);
diff --git a/src/client/Client.h b/src/client/Client.h
index af8876d..bd30c09 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -7,9 +7,9 @@
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
+ * License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
- *
+ *
*/
@@ -20,9 +20,11 @@
// stl
#include <string>
+#include <memory>
#include <set>
#include <map>
#include <fstream>
+#include <exception>
using std::set;
using std::map;
using std::fstream;
@@ -33,6 +35,8 @@ using std::fstream;
#include "include/interval_set.h"
#include "include/lru.h"
+//#include "barrier.h"
+
#include "mds/mdstypes.h"
#include "msg/Message.h"
@@ -205,7 +209,7 @@ class Client : public Dispatcher {
CommandHook m_command_hook;
// cluster descriptors
- MDSMap *mdsmap;
+ MDSMap *mdsmap;
OSDMap *osdmap;
SafeTimer timer;
@@ -310,6 +314,9 @@ protected:
int num_flushing_caps;
ceph::unordered_map<inodeno_t,SnapRealm*> snap_realms;
+ /* async block write barrier support */
+ //map<uint64_t, BarrierContext* > barriers;
+
SnapRealm *get_snap_realm(inodeno_t r);
SnapRealm *get_snap_realm_maybe(inodeno_t r);
void put_snap_realm(SnapRealm *realm);
@@ -365,6 +372,7 @@ protected:
friend class C_Client_PutInode; // calls put_inode()
friend class C_Client_CacheInvalidate; // calls ino_invalidate_cb
friend class C_Client_DentryInvalidate; // calls dentry_invalidate_cb
+ friend class C_Block_Sync; // Calls block map and protected helpers
//int get_cache_size() { return lru.lru_get_size(); }
//void set_cache_size(int m) { lru.lru_set_max(m); }
@@ -419,6 +427,7 @@ protected:
client_t get_nodeid() { return whoami; }
inodeno_t get_root_ino();
+ Inode *get_root();
int init() WARN_UNUSED_RESULT;
void shutdown();
@@ -562,6 +571,7 @@ private:
int _mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev, int uid=-1, int gid=-1, Inode **inp = 0);
int _setattr(Inode *in, struct stat *attr, int mask, int uid=-1, int gid=-1, Inode **inp = 0);
int _getattr(Inode *in, int mask, int uid=-1, int gid=-1, bool force=false);
+ int _readlink(Inode *in, char *buf, size_t size);
int _getxattr(Inode *in, const char *name, void *value, size_t len, int uid=-1, int gid=-1);
int _listxattr(Inode *in, char *names, size_t len, int uid=-1, int gid=-1);
int _setxattr(Inode *in, const char *name, const void *value, size_t len, int flags, int uid=-1, int gid=-1);
@@ -583,6 +593,9 @@ private:
int check_permissions(Inode *in, int flags, int uid, int gid);
+ vinodeno_t _get_vino(Inode *in);
+ inodeno_t _get_inodeno(Inode *in);
+
public:
int mount(const std::string &mount_root);
void unmount();
@@ -642,6 +655,7 @@ public:
// symlinks
int readlink(const char *path, char *buf, loff_t size);
+
int symlink(const char *existing, const char *newname);
// inode stuff
@@ -701,7 +715,7 @@ public:
int get_file_extent_osds(int fd, loff_t off, loff_t *len, vector<int>& osds);
int get_osd_addr(int osd, entity_addr_t& addr);
- // expose osdmap
+ // expose osdmap
int get_local_osd();
int get_pool_replication(int64_t pool);
int64_t get_pool_id(const char *pool_name);
@@ -718,37 +732,80 @@ public:
int get_caps_issued(int fd);
int get_caps_issued(const char *path);
- // low-level interface
- int ll_lookup(vinodeno_t parent, const char *name, struct stat *attr, int uid = -1, int gid = -1);
- bool ll_forget(vinodeno_t vino, int count);
- Inode *_ll_get_inode(vinodeno_t vino);
- int ll_getattr(vinodeno_t vino, struct stat *st, int uid = -1, int gid = -1);
- int ll_setattr(vinodeno_t vino, struct stat *st, int mask, int uid = -1, int gid = -1);
- int ll_getxattr(vinodeno_t vino, const char *name, void *value, size_t size, int uid=-1, int gid=-1);
- int ll_setxattr(vinodeno_t vino, const char *name, const void *value, size_t size, int flags, int uid=-1, int gid=-1);
- int ll_removexattr(vinodeno_t vino, const char *name, int uid=-1, int gid=-1);
- int ll_listxattr(vinodeno_t vino, char *list, size_t size, int uid=-1, int gid=-1);
- int ll_opendir(vinodeno_t vino, void **dirpp, int uid = -1, int gid = -1);
- void ll_releasedir(void *dirp);
- int ll_readlink(vinodeno_t vino, const char **value, int uid = -1, int gid = -1);
- int ll_mknod(vinodeno_t vino, const char *name, mode_t mode, dev_t rdev, struct stat *attr, int uid = -1, int gid = -1);
- int ll_mkdir(vinodeno_t vino, const char *name, mode_t mode, struct stat *attr, int uid = -1, int gid = -1);
- int ll_symlink(vinodeno_t vino, const char *name, const char *value, struct stat *attr, int uid = -1, int gid = -1);
- int ll_unlink(vinodeno_t vino, const char *name, int uid = -1, int gid = -1);
- int ll_rmdir(vinodeno_t vino, const char *name, int uid = -1, int gid = -1);
- int ll_rename(vinodeno_t parent, const char *name, vinodeno_t newparent, const char *newname, int uid = -1, int gid = -1);
- int ll_link(vinodeno_t vino, vinodeno_t newparent, const char *newname, struct stat *attr, int uid = -1, int gid = -1);
- int ll_describe_layout(Fh *fh, ceph_file_layout* layout);
- int ll_open(vinodeno_t vino, int flags, Fh **fh, int uid = -1, int gid = -1);
- int ll_create(vinodeno_t parent, const char *name, mode_t mode, int flags, struct stat *attr, Fh **fh, int uid = -1, int gid = -1);
+ // low-level interface v2
+ inodeno_t ll_get_inodeno(Inode *in) {
+ Mutex::Locker lock(client_lock);
+ return _get_inodeno(in);
+ }
+ snapid_t ll_get_snapid(Inode *in);
+ vinodeno_t ll_get_vino(Inode *in) {
+ Mutex::Locker lock(client_lock);
+ return _get_vino(in);
+ }
+ Inode *ll_get_inode(vinodeno_t vino);
+ int ll_lookup(Inode *parent, const char *name, struct stat *attr,
+ Inode **out, int uid = -1, int gid = -1);
+ bool ll_forget(Inode *in, int count);
+ bool ll_put(Inode *in);
+ int ll_getattr(Inode *in, struct stat *st, int uid = -1, int gid = -1);
+ int ll_setattr(Inode *in, struct stat *st, int mask, int uid = -1,
+ int gid = -1);
+ int ll_getxattr(Inode *in, const char *name, void *value, size_t size,
+ int uid=-1, int gid=-1);
+ int ll_setxattr(Inode *in, const char *name, const void *value, size_t size,
+ int flags, int uid=-1, int gid=-1);
+ int ll_removexattr(Inode *in, const char *name, int uid=-1, int gid=-1);
+ int ll_listxattr(Inode *in, char *list, size_t size, int uid=-1, int gid=-1);
+ int ll_opendir(Inode *in, dir_result_t **dirpp, int uid = -1, int gid = -1);
+ int ll_releasedir(dir_result_t* dirp);
+ int ll_readlink(Inode *in, char *buf, size_t bufsize, int uid = -1, int gid = -1);
+ int ll_mknod(Inode *in, const char *name, mode_t mode, dev_t rdev,
+ struct stat *attr, Inode **out, int uid = -1, int gid = -1);
+ int ll_mkdir(Inode *in, const char *name, mode_t mode, struct stat *attr,
+ Inode **out, int uid = -1, int gid = -1);
+ int ll_symlink(Inode *in, const char *name, const char *value,
+ struct stat *attr, Inode **out, int uid = -1, int gid = -1);
+ int ll_unlink(Inode *in, const char *name, int uid = -1, int gid = -1);
+ int ll_rmdir(Inode *in, const char *name, int uid = -1, int gid = -1);
+ int ll_rename(Inode *parent, const char *name, Inode *newparent,
+ const char *newname, int uid = -1, int gid = -1);
+ int ll_link(Inode *in, Inode *newparent, const char *newname,
+ struct stat *attr, int uid = -1, int gid = -1);
+ int ll_open(Inode *in, int flags, Fh **fh, int uid = -1, int gid = -1);
+ int ll_create(Inode *parent, const char *name, mode_t mode, int flags,
+ struct stat *attr, Inode **out, Fh **fhp, int uid = -1,
+ int gid = -1);
+ int ll_read_block(Inode *in, uint64_t blockid, char *buf, uint64_t offset,
+ uint64_t length, ceph_file_layout* layout);
+
+ int ll_write_block(Inode *in, uint64_t blockid,
+ char* buf, uint64_t offset,
+ uint64_t length, ceph_file_layout* layout,
+ uint64_t snapseq, uint32_t sync);
+ int ll_commit_blocks(Inode *in, uint64_t offset, uint64_t length);
+
+ int ll_statfs(Inode *in, struct statvfs *stbuf);
+ int ll_walk(const char* name, Inode **i, struct stat *attr); // XXX in?
+ int ll_listxattr_chunks(Inode *in, char *names, size_t size,
+ int *cookie, int *eol, int uid, int gid);
+ uint32_t ll_stripe_unit(Inode *in);
+ int ll_file_layout(Inode *in, ceph_file_layout *layout);
+ uint64_t ll_snap_seq(Inode *in);
+
int ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl);
int ll_write(Fh *fh, loff_t off, loff_t len, const char *data);
+ loff_t ll_lseek(Fh *fh, loff_t offset, int whence);
int ll_flush(Fh *fh);
int ll_fsync(Fh *fh, bool syncdataonly);
int ll_fallocate(Fh *fh, int mode, loff_t offset, loff_t length);
int ll_release(Fh *fh);
- int ll_statfs(vinodeno_t vino, struct statvfs *stbuf);
+ int ll_get_stripe_osd(struct Inode *in, uint64_t blockno,
+ ceph_file_layout* layout);
+ uint64_t ll_get_internal_offset(struct Inode *in, uint64_t blockno);
+ int ll_num_osds(void);
+ int ll_osdaddr(int osd, uint32_t *addr);
+ int ll_osdaddr(int osd, char* buf, size_t size);
void ll_register_ino_invalidate_cb(client_ino_callback_t cb, void *handle);
void ll_register_dentry_invalidate_cb(client_dentry_callback_t cb, void *handle);
diff --git a/src/client/Dentry.h b/src/client/Dentry.h
index 932f17d..c51a87c 100644
--- a/src/client/Dentry.h
+++ b/src/client/Dentry.h
@@ -19,18 +19,18 @@ class Dentry : public LRUObject {
uint64_t lease_gen;
ceph_seq_t lease_seq;
int cap_shared_gen;
-
+
/*
* ref==1 -> cached, unused
* ref >1 -> pinned in lru
*/
- void get() {
+ void get() {
assert(ref > 0);
if (++ref == 2)
- lru_pin();
+ lru_pin();
//cout << "dentry.get on " << this << " " << name << " now " << ref << std::endl;
}
- void put() {
+ void put() {
assert(ref > 0);
if (--ref == 1)
lru_unpin();
@@ -40,7 +40,7 @@ class Dentry : public LRUObject {
}
void dump(Formatter *f) const;
-
+
Dentry() : dir(0), inode(0), ref(1), offset(0), lease_mds(-1), lease_gen(0), lease_seq(0), cap_shared_gen(0) { }
private:
~Dentry() {
diff --git a/src/client/Fh.h b/src/client/Fh.h
index 3c573c2..083ccd1 100644
--- a/src/client/Fh.h
+++ b/src/client/Fh.h
@@ -10,7 +10,7 @@ class Cond;
struct Fh {
Inode *inode;
- loff_t pos;
+ loff_t pos;
int mds; // have to talk to mds we opened with (for now)
int mode; // the mode i opened the file with
diff --git a/src/client/Inode.cc b/src/client/Inode.cc
index 1a99624..4e53e14 100644
--- a/src/client/Inode.cc
+++ b/src/client/Inode.cc
@@ -21,7 +21,7 @@ ostream& operator<<(ostream &out, Inode &in)
out << "(";
for (map<int,Cap*>::iterator p = in.caps.begin(); p != in.caps.end(); ++p) {
if (p != in.caps.begin())
- out << ',';
+ out << ',';
out << p->first << '=' << ccap_string(p->second->issued);
}
out << ")";
@@ -126,7 +126,7 @@ bool Inode::put_cap_ref(int cap)
assert(cap_refs[c] > 0);
}
if (--cap_refs[c] == 0)
- last = true;
+ last = true;
//cout << "inode " << *this << " put " << cap_string(c) << " " << (cap_refs[c]+1) << " -> " << cap_refs[c] << std::endl;
}
cap >>= 1;
@@ -140,7 +140,7 @@ bool Inode::is_any_caps()
return caps.size() || exporting_mds >= 0;
}
-bool Inode::cap_is_valid(Cap* cap)
+bool Inode::cap_is_valid(Cap* cap)
{
/*cout << "cap_gen " << cap->session-> cap_gen << std::endl
<< "session gen " << cap->gen << std::endl
diff --git a/src/client/Inode.h b/src/client/Inode.h
index cd58b82..b6c7a38 100644
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -58,7 +58,7 @@ struct CapSnap {
xlist<CapSnap*>::item flushing_item;
CapSnap(Inode *i)
- : in(i), issued(0), dirty(0),
+ : in(i), issued(0), dirty(0),
size(0), time_warp_seq(0), mode(0), uid(0), gid(0), xattr_version(0),
writing(false), dirty_data(false), flush_tid(0),
flushing_item(this)
@@ -89,7 +89,7 @@ class Inode {
gid_t gid;
// nlink
- int32_t nlink;
+ int32_t nlink;
// file (data access)
ceph_dir_layout dir_layout;
@@ -106,7 +106,7 @@ class Inode {
// dirfrag, recursive accountin
frag_info_t dirstat;
nest_info_t rstat;
-
+
// special stuff
version_t version; // auth only
version_t xattr_version;
@@ -180,16 +180,16 @@ class Inode {
void make_long_path(filepath& p);
void make_nosnap_relative_path(filepath& p);
- void get() {
- _ref++;
+ void get() {
+ _ref++;
lsubdout(cct, mds, 15) << "inode.get on " << this << " " << ino << '.' << snapid
- << " now " << _ref << dendl;
+ << " now " << _ref << dendl;
}
/// private method to put a reference; see Client::put_inode()
int _put(int n=1) {
_ref -= n;
lsubdout(cct, mds, 15) << "inode.put on " << this << " " << ino << '.' << snapid
- << " now " << _ref << dendl;
+ << " now " << _ref << dendl;
assert(_ref >= 0);
return _ref;
}
diff --git a/src/client/SyntheticClient.cc b/src/client/SyntheticClient.cc
index aac509c..7129a5c 100644
--- a/src/client/SyntheticClient.cc
+++ b/src/client/SyntheticClient.cc
@@ -1003,7 +1003,6 @@ void SyntheticClient::up()
clear_dir();
}
-
int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
{
dout(4) << "play trace prefix '" << prefix << "'" << dendl;
@@ -1015,12 +1014,14 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
utime_t start = ceph_clock_now(client->cct);
ceph::unordered_map<int64_t, int64_t> open_files;
- ceph::unordered_map<int64_t, dir_result_t*> open_dirs;
+ ceph::unordered_map<int64_t, dir_result_t*> open_dirs;
ceph::unordered_map<int64_t, Fh*> ll_files;
- ceph::unordered_map<int64_t, void*> ll_dirs;
+ ceph::unordered_map<int64_t, dir_result_t*> ll_dirs;
ceph::unordered_map<uint64_t, int64_t> ll_inos;
+ Inode *i1, *i2;
+
ll_inos[1] = 1; // root inode is known.
// prefix?
@@ -1028,17 +1029,19 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
if (prefix.length()) {
client->mkdir(prefix.c_str(), 0755);
struct stat attr;
- if (client->ll_lookup(vinodeno_t(1, CEPH_NOSNAP), prefix.c_str(), &attr) == 0) {
+ i1 = client->ll_get_inode(vinodeno_t(1, CEPH_NOSNAP));
+ if (client->ll_lookup(i1, prefix.c_str(), &attr, &i2) == 0) {
ll_inos[1] = attr.st_ino;
dout(5) << "'root' ino is " << inodeno_t(attr.st_ino) << dendl;
+ client->ll_put(i1);
} else {
dout(0) << "warning: play_trace couldn't lookup up my per-client directory" << dendl;
}
- }
-
+ } else
+ (void) client->ll_get_inode(vinodeno_t(1, CEPH_NOSNAP));
utime_t last_status = start;
-
+
int n = 0;
// for object traces
@@ -1227,20 +1230,27 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
const char *name = t.get_string(buf, p);
int64_t r = t.get_int();
struct stat attr;
- if (ll_inos.count(i) &&
- client->ll_lookup(vinodeno_t(ll_inos[i],CEPH_NOSNAP), name, &attr) == 0)
- ll_inos[r] = attr.st_ino;
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_lookup(i1, name, &attr, &i2) == 0)
+ ll_inos[r] = attr.st_ino;
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_forget") == 0) {
int64_t i = t.get_int();
int64_t n = t.get_int();
if (ll_inos.count(i) &&
- client->ll_forget(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n))
+ client->ll_forget(
+ client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP)), n))
ll_inos.erase(i);
} else if (strcmp(op, "ll_getattr") == 0) {
int64_t i = t.get_int();
struct stat attr;
- if (ll_inos.count(i))
- client->ll_getattr(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &attr);
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ client->ll_getattr(i1, &attr);
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_setattr") == 0) {
int64_t i = t.get_int();
struct stat attr;
@@ -1252,13 +1262,19 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
attr.st_mtime = t.get_int();
attr.st_atime = t.get_int();
int mask = t.get_int();
- if (ll_inos.count(i))
- client->ll_setattr(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &attr, mask);
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ client->ll_setattr(i1, &attr, mask);
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_readlink") == 0) {
int64_t i = t.get_int();
- const char *value;
- if (ll_inos.count(i))
- client->ll_readlink(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &value);
+ char buf[PATH_MAX];
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ client->ll_readlink(i1, buf, sizeof(buf));
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_mknod") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
@@ -1266,62 +1282,88 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int r = t.get_int();
int64_t ri = t.get_int();
struct stat attr;
- if (ll_inos.count(i) &&
- client->ll_mknod(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, m, r, &attr) == 0)
- ll_inos[ri] = attr.st_ino;
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_mknod(i1, n, m, r, &attr, &i2) == 0)
+ ll_inos[ri] = attr.st_ino;
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_mkdir") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
int m = t.get_int();
int64_t ri = t.get_int();
struct stat attr;
- if (ll_inos.count(i) &&
- client->ll_mkdir(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, m, &attr) == 0)
- ll_inos[ri] = attr.st_ino;
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_mkdir(i1, n, m, &attr, &i2) == 0)
+ ll_inos[ri] = attr.st_ino;
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_symlink") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
const char *v = t.get_string(buf2, p);
int64_t ri = t.get_int();
struct stat attr;
- if (ll_inos.count(i) &&
- client->ll_symlink(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, v, &attr) == 0)
- ll_inos[ri] = attr.st_ino;
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_symlink(i1, n, v, &attr, &i2) == 0)
+ ll_inos[ri] = attr.st_ino;
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_unlink") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
- if (ll_inos.count(i))
- client->ll_unlink(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n);
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ client->ll_unlink(i1, n);
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_rmdir") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
- if (ll_inos.count(i))
- client->ll_rmdir(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n);
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ client->ll_rmdir(i1, n);
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_rename") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
int64_t ni = t.get_int();
const char *nn = t.get_string(buf2, p);
if (ll_inos.count(i) &&
- ll_inos.count(ni))
- client->ll_rename(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n,
- vinodeno_t(ll_inos[ni],CEPH_NOSNAP), nn);
+ ll_inos.count(ni)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ i2 = client->ll_get_inode(vinodeno_t(ll_inos[ni],CEPH_NOSNAP));
+ client->ll_rename(i1, n, i2, nn);
+ client->ll_put(i1);
+ client->ll_put(i2);
+ }
} else if (strcmp(op, "ll_link") == 0) {
int64_t i = t.get_int();
int64_t ni = t.get_int();
const char *nn = t.get_string(buf, p);
struct stat attr;
if (ll_inos.count(i) &&
- ll_inos.count(ni))
- client->ll_link(vinodeno_t(ll_inos[i],CEPH_NOSNAP),
- vinodeno_t(ll_inos[ni],CEPH_NOSNAP), nn, &attr);
+ ll_inos.count(ni)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ i2 = client->ll_get_inode(vinodeno_t(ll_inos[ni],CEPH_NOSNAP));
+ client->ll_link(i1, i2, nn, &attr);
+ client->ll_put(i1);
+ client->ll_put(i2);
+ }
} else if (strcmp(op, "ll_opendir") == 0) {
int64_t i = t.get_int();
int64_t r = t.get_int();
- void *dirp;
- if (ll_inos.count(i) &&
- client->ll_opendir(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &dirp) == 0)
- ll_dirs[r] = dirp;
+ dir_result_t *dirp;
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_opendir(i1, &dirp) == 0)
+ ll_dirs[r] = dirp;
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_releasedir") == 0) {
int64_t f = t.get_int();
if (ll_dirs.count(f)) {
@@ -1333,9 +1375,12 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t f = t.get_int();
int64_t r = t.get_int();
Fh *fhp;
- if (ll_inos.count(i) &&
- client->ll_open(vinodeno_t(ll_inos[i],CEPH_NOSNAP), f, &fhp) == 0)
- ll_files[r] = fhp;
+ if (ll_inos.count(i)) {
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_open(i1, f, &fhp) == 0)
+ ll_files[r] = fhp;
+ client->ll_put(i1);
+ }
} else if (strcmp(op, "ll_create") == 0) {
int64_t i = t.get_int();
const char *n = t.get_string(buf, p);
@@ -1343,12 +1388,15 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t f = t.get_int();
int64_t r = t.get_int();
int64_t ri = t.get_int();
- Fh *fhp;
struct stat attr;
- if (ll_inos.count(i) &&
- client->ll_create(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, m, f, &attr, &fhp) == 0) {
- ll_inos[ri] = attr.st_ino;
- ll_files[r] = fhp;
+ if (ll_inos.count(i)) {
+ Fh *fhp;
+ i1 = client->ll_get_inode(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
+ if (client->ll_create(i1, n, m, f, &attr, NULL, &fhp) == 0) {
+ ll_inos[ri] = attr.st_ino;
+ ll_files[r] = fhp;
+ }
+ client->ll_put(i1);
}
} else if (strcmp(op, "ll_read") == 0) {
int64_t f = t.get_int();
@@ -1394,7 +1442,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t i = t.get_int();
if (ll_inos.count(i))
{} //client->ll_statfs(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
- }
+ }
// object-level traces
@@ -1497,7 +1545,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
dout(1) << "leftover ll_release " << fi->second << dendl;
if (fi->second) client->ll_release(fi->second);
}
- for (ceph::unordered_map<int64_t,void*>::iterator fi = ll_dirs.begin();
+ for (ceph::unordered_map<int64_t,dir_result_t*>::iterator fi = ll_dirs.begin();
fi != ll_dirs.end();
++fi) {
dout(1) << "leftover ll_releasedir " << fi->second << dendl;
diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc
index 1c58b6c..d9f6a8d 100644
--- a/src/client/fuse_ll.cc
+++ b/src/client/fuse_ll.cc
@@ -29,6 +29,7 @@
#include "common/safe_io.h"
#include "include/types.h"
#include "Client.h"
+#include "Fh.h"
#include "ioctl.h"
#include "common/config.h"
#include "include/assert.h"
@@ -71,6 +72,8 @@ public:
uint64_t fino_snap(uint64_t fino);
vinodeno_t fino_vino(inodeno_t fino);
uint64_t make_fake_ino(inodeno_t ino, snapid_t snapid);
+ Inode * iget(inodeno_t fino);
+ void iput(Inode *in);
int fd_on_success;
Client *client;
@@ -81,6 +84,7 @@ public:
Mutex stag_lock;
int last_stag;
+
ceph::unordered_map<uint64_t,int> snap_stag_map;
ceph::unordered_map<int,uint64_t> stag_snap_map;
@@ -91,10 +95,11 @@ static void fuse_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name)
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
struct fuse_entry_param fe;
+ Inode *i2, *i1 = cfuse->iget(parent); // see below
int r;
memset(&fe, 0, sizeof(fe));
- r = cfuse->client->ll_lookup(cfuse->fino_vino(parent), name, &fe.attr, ctx->uid, ctx->gid);
+ r = cfuse->client->ll_lookup(i1, name, &fe.attr, &i2, ctx->uid, ctx->gid);
if (r >= 0) {
fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
fe.attr.st_rdev = new_encode_dev(fe.attr.st_rdev);
@@ -102,12 +107,17 @@ static void fuse_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name)
} else {
fuse_reply_err(req, -r);
}
+
+ // XXX NB, we dont iput(i2) because FUSE will do so in a matching
+ // fuse_ll_forget()
+ cfuse->iput(i1);
}
-static void fuse_ll_forget(fuse_req_t req, fuse_ino_t ino, long unsigned nlookup)
+static void fuse_ll_forget(fuse_req_t req, fuse_ino_t ino,
+ long unsigned nlookup)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
- cfuse->client->ll_forget(cfuse->fino_vino(ino), nlookup);
+ cfuse->client->ll_forget(cfuse->iget(ino), nlookup+1);
fuse_reply_none(req);
}
@@ -116,16 +126,20 @@ static void fuse_ll_getattr(fuse_req_t req, fuse_ino_t ino,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
struct stat stbuf;
- (void) fi;
+ (void) fi; // XXX
- if (cfuse->client->ll_getattr(cfuse->fino_vino(ino), &stbuf, ctx->uid, ctx->gid) == 0) {
+ if (cfuse->client->ll_getattr(in, &stbuf, ctx->uid, ctx->gid)
+ == 0) {
stbuf.st_ino = cfuse->make_fake_ino(stbuf.st_ino, stbuf.st_dev);
stbuf.st_rdev = new_encode_dev(stbuf.st_rdev);
fuse_reply_attr(req, &stbuf, 0);
} else
fuse_reply_err(req, ENOENT);
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr,
@@ -133,6 +147,7 @@ static void fuse_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
int mask = 0;
if (to_set & FUSE_SET_ATTR_MODE) mask |= CEPH_SETATTR_MODE;
@@ -142,11 +157,13 @@ static void fuse_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr,
if (to_set & FUSE_SET_ATTR_ATIME) mask |= CEPH_SETATTR_ATIME;
if (to_set & FUSE_SET_ATTR_SIZE) mask |= CEPH_SETATTR_SIZE;
- int r = cfuse->client->ll_setattr(cfuse->fino_vino(ino), attr, mask, ctx->uid, ctx->gid);
+ int r = cfuse->client->ll_setattr(in, attr, mask, ctx->uid, ctx->gid);
if (r == 0)
fuse_reply_attr(req, attr, 0);
else
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
// XATTRS
@@ -156,22 +173,31 @@ static void fuse_ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
- int r = cfuse->client->ll_setxattr(cfuse->fino_vino(ino), name, value, size, flags, ctx->uid, ctx->gid);
+ Inode *in = cfuse->iget(ino);
+
+ int r = cfuse->client->ll_setxattr(in, name, value, size, flags, ctx->uid,
+ ctx->gid);
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
char buf[size];
- int r = cfuse->client->ll_listxattr(cfuse->fino_vino(ino), buf, size, ctx->uid, ctx->gid);
+
+ int r = cfuse->client->ll_listxattr(in, buf, size, ctx->uid, ctx->gid);
if (size == 0 && r >= 0)
fuse_reply_xattr(req, r);
else if (r >= 0)
fuse_reply_buf(req, buf, r);
else
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
@@ -179,50 +205,70 @@ static void fuse_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
char buf[size];
- int r = cfuse->client->ll_getxattr(cfuse->fino_vino(ino), name, buf, size, ctx->uid, ctx->gid);
+
+ int r = cfuse->client->ll_getxattr(in, name, buf, size, ctx->uid, ctx->gid);
if (size == 0 && r >= 0)
fuse_reply_xattr(req, r);
else if (r >= 0)
fuse_reply_buf(req, buf, r);
else
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
-static void fuse_ll_removexattr(fuse_req_t req, fuse_ino_t ino, const char *name)
+static void fuse_ll_removexattr(fuse_req_t req, fuse_ino_t ino,
+ const char *name)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
- int r = cfuse->client->ll_removexattr(cfuse->fino_vino(ino), name, ctx->uid, ctx->gid);
- fuse_reply_err(req, -r);
-}
+ Inode *in = cfuse->iget(ino);
+ int r = cfuse->client->ll_removexattr(in, name, ctx->uid,
+ ctx->gid);
+ fuse_reply_err(req, -r);
+ cfuse->iput(in); // iput required
+}
-static void fuse_ll_opendir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
+static void fuse_ll_opendir(fuse_req_t req, fuse_ino_t ino,
+ struct fuse_file_info *fi)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
void *dirp;
- int r = cfuse->client->ll_opendir(cfuse->fino_vino(ino), &dirp, ctx->uid, ctx->gid);
+
+ int r = cfuse->client->ll_opendir(in, (dir_result_t **) &dirp, ctx->uid,
+ ctx->gid);
if (r >= 0) {
fi->fh = (long)dirp;
fuse_reply_open(req, fi);
} else {
fuse_reply_err(req, -r);
}
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_readlink(fuse_req_t req, fuse_ino_t ino)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
- const char *value;
- int r = cfuse->client->ll_readlink(cfuse->fino_vino(ino), &value, ctx->uid, ctx->gid);
- if (r == 0)
- fuse_reply_readlink(req, value);
- else
+ Inode *in = cfuse->iget(ino);
+ char buf[PATH_MAX + 1]; // leave room for a null terminator
+
+ int r = cfuse->client->ll_readlink(in, buf, sizeof(buf) - 1, ctx->uid, ctx->gid);
+ if (r >= 0) {
+ buf[r] = '\0';
+ fuse_reply_readlink(req, buf);
+ } else {
fuse_reply_err(req, -r);
+ }
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name,
@@ -230,10 +276,13 @@ static void fuse_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *i2, *i1 = cfuse->iget(parent);
struct fuse_entry_param fe;
+
memset(&fe, 0, sizeof(fe));
- int r = cfuse->client->ll_mknod(cfuse->fino_vino(parent), name, mode, new_decode_dev(rdev), &fe.attr, ctx->uid, ctx->gid);
+ int r = cfuse->client->ll_mknod(i1, name, mode, new_decode_dev(rdev),
+ &fe.attr, &i2, ctx->uid, ctx->gid);
if (r == 0) {
fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
fe.attr.st_rdev = new_encode_dev(fe.attr.st_rdev);
@@ -241,6 +290,10 @@ static void fuse_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name,
} else {
fuse_reply_err(req, -r);
}
+
+ // XXX NB, we dont iput(i2) because FUSE will do so in a matching
+ // fuse_ll_forget()
+ cfuse->iput(i1); // iput required
}
static void fuse_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name,
@@ -248,10 +301,13 @@ static void fuse_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *i2, *i1 = cfuse->iget(parent);
struct fuse_entry_param fe;
+
memset(&fe, 0, sizeof(fe));
- int r = cfuse->client->ll_mkdir(cfuse->fino_vino(parent), name, mode, &fe.attr, ctx->uid, ctx->gid);
+ int r = cfuse->client->ll_mkdir(i1, name, mode, &fe.attr, &i2, ctx->uid,
+ ctx->gid);
if (r == 0) {
fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
fe.attr.st_rdev = new_encode_dev(fe.attr.st_rdev);
@@ -259,32 +315,48 @@ static void fuse_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name,
} else {
fuse_reply_err(req, -r);
}
+
+ // XXX NB, we dont iput(i2) because FUSE will do so in a matching
+ // fuse_ll_forget()
+ cfuse->iput(i1); // iput required
}
static void fuse_ll_unlink(fuse_req_t req, fuse_ino_t parent, const char *name)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
- int r = cfuse->client->ll_unlink(cfuse->fino_vino(parent), name, ctx->uid, ctx->gid);
+ Inode *in = cfuse->iget(parent);
+
+ int r = cfuse->client->ll_unlink(in, name, ctx->uid, ctx->gid);
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_rmdir(fuse_req_t req, fuse_ino_t parent, const char *name)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
- int r = cfuse->client->ll_rmdir(cfuse->fino_vino(parent), name, ctx->uid, ctx->gid);
+ Inode *in = cfuse->iget(parent);
+
+ int r = cfuse->client->ll_rmdir(in, name, ctx->uid, ctx->gid);
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
-static void fuse_ll_symlink(fuse_req_t req, const char *existing, fuse_ino_t parent, const char *name)
+static void fuse_ll_symlink(fuse_req_t req, const char *existing,
+ fuse_ino_t parent, const char *name)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *i2, *i1 = cfuse->iget(parent);
struct fuse_entry_param fe;
+
memset(&fe, 0, sizeof(fe));
- int r = cfuse->client->ll_symlink(cfuse->fino_vino(parent), name, existing, &fe.attr, ctx->uid, ctx->gid);
+ int r = cfuse->client->ll_symlink(i1, name, existing, &fe.attr, &i2, ctx->uid,
+ ctx->gid);
if (r == 0) {
fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
fe.attr.st_rdev = new_encode_dev(fe.attr.st_rdev);
@@ -292,6 +364,10 @@ static void fuse_ll_symlink(fuse_req_t req, const char *existing, fuse_ino_t par
} else {
fuse_reply_err(req, -r);
}
+
+ // XXX NB, we dont iput(i2) because FUSE will do so in a matching
+ // fuse_ll_forget()
+ cfuse->iput(i1); // iput required
}
static void fuse_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name,
@@ -299,8 +375,14 @@ static void fuse_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
- int r = cfuse->client->ll_rename(cfuse->fino_vino(parent), name, cfuse->fino_vino(newparent), newname, ctx->uid, ctx->gid);
+ Inode *in = cfuse->iget(parent);
+ Inode *nin = cfuse->iget(newparent);
+
+ int r = cfuse->client->ll_rename(in, name, nin, newname, ctx->uid, ctx->gid);
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iputs required
+ cfuse->iput(nin);
}
static void fuse_ll_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t newparent,
@@ -308,10 +390,14 @@ static void fuse_ll_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t newparent,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
+ Inode *nin = cfuse->iget(newparent);
struct fuse_entry_param fe;
+
memset(&fe, 0, sizeof(fe));
- int r = cfuse->client->ll_link(cfuse->fino_vino(ino), cfuse->fino_vino(newparent), newname, &fe.attr, ctx->uid, ctx->gid);
+ int r = cfuse->client->ll_link(in, nin, newname, &fe.attr, ctx->uid,
+ ctx->gid);
if (r == 0) {
fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
fe.attr.st_rdev = new_encode_dev(fe.attr.st_rdev);
@@ -319,14 +405,20 @@ static void fuse_ll_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t newparent,
} else {
fuse_reply_err(req, -r);
}
+
+ cfuse->iput(in); // iputs required
+ cfuse->iput(nin);
}
-static void fuse_ll_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
+static void fuse_ll_open(fuse_req_t req, fuse_ino_t ino,
+ struct fuse_file_info *fi)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *in = cfuse->iget(ino);
Fh *fh = NULL;
- int r = cfuse->client->ll_open(cfuse->fino_vino(ino), fi->flags, &fh, ctx->uid, ctx->gid);
+
+ int r = cfuse->client->ll_open(in, fi->flags, &fh, ctx->uid, ctx->gid);
if (r == 0) {
fi->fh = (long)fh;
#if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8)
@@ -337,6 +429,8 @@ static void fuse_ll_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *
} else {
fuse_reply_err(req, -r);
}
+
+ cfuse->iput(in); // iput required
}
static void fuse_ll_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off,
@@ -364,7 +458,8 @@ static void fuse_ll_write(fuse_req_t req, fuse_ino_t ino, const char *buf,
fuse_reply_err(req, -r);
}
-static void fuse_ll_flush(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
+static void fuse_ll_flush(fuse_req_t req, fuse_ino_t ino,
+ struct fuse_file_info *fi)
{
// NOOP
fuse_reply_err(req, 0);
@@ -386,7 +481,7 @@ static void fuse_ll_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, st
struct ceph_file_layout layout;
struct ceph_ioctl_layout l;
Fh *fh = (Fh*)fi->fh;
- cfuse->client->ll_describe_layout(fh, &layout);
+ cfuse->client->ll_file_layout(fh->inode, &layout);
l.stripe_unit = layout.fl_stripe_unit;
l.stripe_count = layout.fl_stripe_count;
l.object_size = layout.fl_object_size;
@@ -414,7 +509,8 @@ static void fuse_ll_fallocate(fuse_req_t req, fuse_ino_t ino, int mode,
#endif
-static void fuse_ll_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
+static void fuse_ll_release(fuse_req_t req, fuse_ino_t ino,
+ struct fuse_file_info *fi)
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
Fh *fh = (Fh*)fi->fh;
@@ -442,7 +538,8 @@ struct readdir_context {
/*
* return 0 on success, -1 if out of space
*/
-static int fuse_ll_add_dirent(void *p, struct dirent *de, struct stat *st, int stmask, off_t next_off)
+static int fuse_ll_add_dirent(void *p, struct dirent *de, struct stat *st,
+ int stmask, off_t next_off)
{
struct readdir_context *c = (struct readdir_context *)p;
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(c->req);
@@ -499,28 +596,39 @@ static void fuse_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name,
{
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
const struct fuse_ctx *ctx = fuse_req_ctx(req);
+ Inode *i1 = cfuse->iget(parent), *i2;
struct fuse_entry_param fe;
- memset(&fe, 0, sizeof(fe));
Fh *fh = NULL;
- int r = cfuse->client->ll_create(cfuse->fino_vino(parent), name, mode, fi->flags, &fe.attr, &fh, ctx->uid, ctx->gid);
+
+ memset(&fe, 0, sizeof(fe));
+
+ // pass &i2 for the created inode so that ll_create takes an initial ll_ref
+ int r = cfuse->client->ll_create(i1, name, mode, fi->flags, &fe.attr, &i2,
+ &fh, ctx->uid, ctx->gid);
if (r == 0) {
fi->fh = (long)fh;
fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
fuse_reply_create(req, &fe, fi);
- } else {
+ } else
fuse_reply_err(req, -r);
- }
+ // XXX NB, we dont iput(i2) because FUSE will do so in a matching
+ // fuse_ll_forget()
+ cfuse->iput(i1); // iput required
}
static void fuse_ll_statfs(fuse_req_t req, fuse_ino_t ino)
{
struct statvfs stbuf;
CephFuse::Handle *cfuse = (CephFuse::Handle *)fuse_req_userdata(req);
- int r = cfuse->client->ll_statfs(cfuse->fino_vino(ino), &stbuf);
+ Inode *in = cfuse->iget(ino);
+
+ int r = cfuse->client->ll_statfs(in, &stbuf);
if (r == 0)
fuse_reply_statfs(req, &stbuf);
else
fuse_reply_err(req, -r);
+
+ cfuse->iput(in); // iput required
}
#if 0
@@ -551,7 +659,8 @@ static int getgroups_cb(void *handle, uid_t uid, gid_t **sgids)
}
#endif
-static void ino_invalidate_cb(void *handle, vinodeno_t vino, int64_t off, int64_t len)
+static void ino_invalidate_cb(void *handle, vinodeno_t vino, int64_t off,
+ int64_t len)
{
#if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8)
CephFuse::Handle *cfuse = (CephFuse::Handle *)handle;
@@ -794,6 +903,17 @@ vinodeno_t CephFuse::Handle::fino_vino(inodeno_t fino)
return vino;
}
+Inode * CephFuse::Handle::iget(inodeno_t fino)
+{
+ Inode *in =
+ client->ll_get_inode(fino_vino(fino));
+ return in;
+}
+
+void CephFuse::Handle::iput(Inode *in)
+{
+ client->ll_put(in);
+}
uint64_t CephFuse::Handle::make_fake_ino(inodeno_t ino, snapid_t snapid)
{
diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 2f5711e..cf301f7 100644
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -663,7 +663,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist
struct rgw_bucket_dir_entry remove_entry;
int ret = read_index_entry(hctx, remove_oid_name, &remove_entry);
if (ret < 0) {
- CLS_LOG(1, "rgw_bucket_complete_op(): removing entries, read_index_entry name=%s ret=%d\n", remove_oid_name.c_str(), rc);
+ CLS_LOG(1, "rgw_bucket_complete_op(): removing entries, read_index_entry name=%s ret=%d\n", remove_oid_name.c_str(), ret);
continue;
}
CLS_LOG(0, "rgw_bucket_complete_op(): entry.name=%s entry.meta.category=%d\n", remove_entry.name.c_str(), remove_entry.meta.category);
@@ -1427,7 +1427,7 @@ static int rgw_cls_gc_defer_entry(cls_method_context_t hctx, bufferlist *in, buf
return gc_defer_entry(hctx, op.tag, op.expiration_secs);
}
-static int gc_iterate_entries(cls_method_context_t hctx, const string& marker,
+static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, bool expired_only,
string& key_iter, uint32_t max_entries, bool *truncated,
int (*cb)(cls_method_context_t, const string&, cls_rgw_gc_obj_info&, void *),
void *param)
@@ -1449,12 +1449,14 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker,
start_key = key_iter;
}
- utime_t now = ceph_clock_now(g_ceph_context);
- string now_str;
- get_time_key(now, &now_str);
- prepend_index_prefix(now_str, GC_OBJ_TIME_INDEX, &end_key);
+ if (expired_only) {
+ utime_t now = ceph_clock_now(g_ceph_context);
+ string now_str;
+ get_time_key(now, &now_str);
+ prepend_index_prefix(now_str, GC_OBJ_TIME_INDEX, &end_key);
- CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key.c_str());
+ CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key.c_str());
+ }
string filter;
@@ -1475,7 +1477,7 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker,
CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str());
- if (key.compare(end_key) >= 0)
+ if (!end_key.empty() && key.compare(end_key) >= 0)
return 0;
if (!key_in_index(key, GC_OBJ_TIME_INDEX))
@@ -1512,10 +1514,11 @@ static int gc_list_cb(cls_method_context_t hctx, const string& key, cls_rgw_gc_o
}
static int gc_list_entries(cls_method_context_t hctx, const string& marker,
- uint32_t max, list<cls_rgw_gc_obj_info>& entries, bool *truncated)
+ uint32_t max, bool expired_only,
+ list<cls_rgw_gc_obj_info>& entries, bool *truncated)
{
string key_iter;
- int ret = gc_iterate_entries(hctx, marker,
+ int ret = gc_iterate_entries(hctx, marker, expired_only,
key_iter, max, truncated,
gc_list_cb, &entries);
return ret;
@@ -1534,7 +1537,7 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist
}
cls_rgw_gc_list_ret op_ret;
- int ret = gc_list_entries(hctx, op.marker, op.max, op_ret.entries, &op_ret.truncated);
+ int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, op_ret.entries, &op_ret.truncated);
if (ret < 0)
return ret;
diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index 2851f2b..a26d0af 100644
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -324,13 +324,14 @@ void cls_rgw_gc_defer_entry(ObjectWriteOperation& op, uint32_t expiration_secs,
op.exec("rgw", "gc_defer_entry", in);
}
-int cls_rgw_gc_list(IoCtx& io_ctx, string& oid, string& marker, uint32_t max,
+int cls_rgw_gc_list(IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only,
list<cls_rgw_gc_obj_info>& entries, bool *truncated)
{
bufferlist in, out;
cls_rgw_gc_list_op call;
call.marker = marker;
call.max = max;
+ call.expired_only = expired_only;
::encode(call, in);
int r = io_ctx.exec(oid, "rgw", "gc_list", in, out);
if (r < 0)
diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
index 39bb3c9..c6b5b75 100644
--- a/src/cls/rgw/cls_rgw_client.h
+++ b/src/cls/rgw/cls_rgw_client.h
@@ -61,7 +61,7 @@ void cls_rgw_usage_log_add(librados::ObjectWriteOperation& op, rgw_usage_log_inf
void cls_rgw_gc_set_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, cls_rgw_gc_obj_info& info);
void cls_rgw_gc_defer_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, const string& tag);
-int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max,
+int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only,
list<cls_rgw_gc_obj_info>& entries, bool *truncated);
void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const list<string>& tags);
diff --git a/src/cls/rgw/cls_rgw_ops.cc b/src/cls/rgw/cls_rgw_ops.cc
index 2ffc53c..01c0666 100644
--- a/src/cls/rgw/cls_rgw_ops.cc
+++ b/src/cls/rgw/cls_rgw_ops.cc
@@ -49,6 +49,7 @@ void cls_rgw_gc_list_op::dump(Formatter *f) const
{
f->dump_string("marker", marker);
f->dump_unsigned("max", max);
+ f->dump_bool("expired_only", expired_only);
}
void cls_rgw_gc_list_op::generate_test_instances(list<cls_rgw_gc_list_op*>& ls)
diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h
index 8f0ecfb..e5db060 100644
--- a/src/cls/rgw/cls_rgw_ops.h
+++ b/src/cls/rgw/cls_rgw_ops.h
@@ -345,20 +345,25 @@ WRITE_CLASS_ENCODER(cls_rgw_gc_defer_entry_op)
struct cls_rgw_gc_list_op {
string marker;
uint32_t max;
+ bool expired_only;
- cls_rgw_gc_list_op() : max(0) {}
+ cls_rgw_gc_list_op() : max(0), expired_only(true) {}
void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
+ ENCODE_START(2, 1, bl);
::encode(marker, bl);
::encode(max, bl);
+ ::encode(expired_only, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
+ DECODE_START(2, bl);
::decode(marker, bl);
::decode(max, bl);
+ if (struct_v >= 2) {
+ ::decode(expired_only, bl);
+ }
DECODE_FINISH(bl);
}
diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc
index a72ba33..003a834 100644
--- a/src/cls/user/cls_user.cc
+++ b/src/cls/user/cls_user.cc
@@ -114,6 +114,13 @@ static void dec_header_stats(cls_user_stats *stats, cls_user_bucket_entry& entry
stats->total_entries -= entry.count;
}
+static void apply_entry_stats(const cls_user_bucket_entry& src_entry, cls_user_bucket_entry *target_entry)
+{
+ target_entry->size = src_entry.size;
+ target_entry->size_rounded = src_entry.size_rounded;
+ target_entry->count = src_entry.count;
+}
+
static int cls_user_set_buckets_info(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
bufferlist::iterator in_iter = in->begin();
@@ -135,32 +142,35 @@ static int cls_user_set_buckets_info(cls_method_context_t hctx, bufferlist *in,
for (list<cls_user_bucket_entry>::iterator iter = op.entries.begin();
iter != op.entries.end(); ++iter) {
- cls_user_bucket_entry& entry = *iter;
+ cls_user_bucket_entry& update_entry = *iter;
string key;
- get_key_by_bucket_name(entry.bucket.name, &key);
+ get_key_by_bucket_name(update_entry.bucket.name, &key);
- cls_user_bucket_entry old_entry;
- ret = get_existing_bucket_entry(hctx, key, old_entry);
+ cls_user_bucket_entry entry;
+ ret = get_existing_bucket_entry(hctx, key, entry);
if (ret == -ENOENT) {
if (!op.add)
continue; /* racing bucket removal */
+ entry = update_entry;
+
ret = 0;
}
if (ret < 0) {
CLS_LOG(0, "ERROR: get_existing_bucket_entry() key=%s returned %d", key.c_str(), ret);
return ret;
- } else if (ret >= 0 && old_entry.user_stats_sync) {
- dec_header_stats(&header.stats, old_entry);
+ } else if (ret >= 0 && entry.user_stats_sync) {
+ dec_header_stats(&header.stats, entry);
}
CLS_LOG(20, "storing entry for key=%s size=%lld count=%lld",
- key.c_str(), (long long)entry.size, (long long)entry.count);
+ key.c_str(), (long long)update_entry.size, (long long)update_entry.count);
+ apply_entry_stats(update_entry, &entry);
entry.user_stats_sync = true;
ret = write_entry(hctx, key, entry);
diff --git a/src/common/Makefile.am b/src/common/Makefile.am
index ec1f31e..f39ab4e 100644
--- a/src/common/Makefile.am
+++ b/src/common/Makefile.am
@@ -40,6 +40,7 @@ libcommon_la_SOURCES = \
common/buffer.cc \
common/code_environment.cc \
common/dout.cc \
+ common/histogram.cc \
common/signal.cc \
common/simple_spin.cc \
common/Thread.cc \
@@ -82,6 +83,7 @@ libcommon_la_SOURCES += \
mon/MonMap.cc \
osd/OSDMap.cc \
osd/osd_types.cc \
+ osd/ECMsgTypes.cc \
osd/HitSet.cc \
mds/MDSMap.cc \
mds/inode_backtrace.cc \
@@ -141,6 +143,7 @@ noinst_HEADERS += \
common/fd.h \
common/version.h \
common/hex.h \
+ common/histogram.h \
common/entity_name.h \
common/errno.h \
common/environment.h \
diff --git a/src/common/PrioritizedQueue.h b/src/common/PrioritizedQueue.h
index e663f27..d936d5d 100644
--- a/src/common/PrioritizedQueue.h
+++ b/src/common/PrioritizedQueue.h
@@ -311,12 +311,16 @@ public:
void enqueue(K cl, unsigned priority, unsigned cost, T item) {
if (cost < min_cost)
cost = min_cost;
+ if (cost > max_tokens_per_subqueue)
+ cost = max_tokens_per_subqueue;
create_queue(priority)->enqueue(cl, cost, item);
}
void enqueue_front(K cl, unsigned priority, unsigned cost, T item) {
if (cost < min_cost)
cost = min_cost;
+ if (cost > max_tokens_per_subqueue)
+ cost = max_tokens_per_subqueue;
create_queue(priority)->enqueue_front(cl, cost, item);
}
diff --git a/src/common/Throttle.cc b/src/common/Throttle.cc
index 7c097e2..026d731 100644
--- a/src/common/Throttle.cc
+++ b/src/common/Throttle.cc
@@ -40,22 +40,24 @@ Throttle::Throttle(CephContext *cct, std::string n, int64_t m, bool _use_perf)
if (!use_perf)
return;
- PerfCountersBuilder b(cct, string("throttle-") + name, l_throttle_first, l_throttle_last);
- b.add_u64_counter(l_throttle_val, "val");
- b.add_u64_counter(l_throttle_max, "max");
- b.add_u64_counter(l_throttle_get, "get");
- b.add_u64_counter(l_throttle_get_sum, "get_sum");
- b.add_u64_counter(l_throttle_get_or_fail_fail, "get_or_fail_fail");
- b.add_u64_counter(l_throttle_get_or_fail_success, "get_or_fail_success");
- b.add_u64_counter(l_throttle_take, "take");
- b.add_u64_counter(l_throttle_take_sum, "take_sum");
- b.add_u64_counter(l_throttle_put, "put");
- b.add_u64_counter(l_throttle_put_sum, "put_sum");
- b.add_time_avg(l_throttle_wait, "wait");
-
- logger = b.create_perf_counters();
- cct->get_perfcounters_collection()->add(logger);
- logger->set(l_throttle_max, max.read());
+ if (cct->_conf->throttler_perf_counter) {
+ PerfCountersBuilder b(cct, string("throttle-") + name, l_throttle_first, l_throttle_last);
+ b.add_u64_counter(l_throttle_val, "val");
+ b.add_u64_counter(l_throttle_max, "max");
+ b.add_u64_counter(l_throttle_get, "get");
+ b.add_u64_counter(l_throttle_get_sum, "get_sum");
+ b.add_u64_counter(l_throttle_get_or_fail_fail, "get_or_fail_fail");
+ b.add_u64_counter(l_throttle_get_or_fail_success, "get_or_fail_success");
+ b.add_u64_counter(l_throttle_take, "take");
+ b.add_u64_counter(l_throttle_take_sum, "take_sum");
+ b.add_u64_counter(l_throttle_put, "put");
+ b.add_u64_counter(l_throttle_put_sum, "put_sum");
+ b.add_time_avg(l_throttle_wait, "wait");
+
+ logger = b.create_perf_counters();
+ cct->get_perfcounters_collection()->add(logger);
+ logger->set(l_throttle_max, max.read());
+ }
}
Throttle::~Throttle()
@@ -69,8 +71,10 @@ Throttle::~Throttle()
if (!use_perf)
return;
- cct->get_perfcounters_collection()->remove(logger);
- delete logger;
+ if (logger) {
+ cct->get_perfcounters_collection()->remove(logger);
+ delete logger;
+ }
}
void Throttle::_reset_max(int64_t m)
@@ -93,7 +97,8 @@ bool Throttle::_wait(int64_t c)
do {
if (!waited) {
ldout(cct, 2) << "_wait waiting..." << dendl;
- start = ceph_clock_now(cct);
+ if (logger)
+ start = ceph_clock_now(cct);
}
waited = true;
cv->Wait(lock);
@@ -101,9 +106,10 @@ bool Throttle::_wait(int64_t c)
if (waited) {
ldout(cct, 3) << "_wait finished waiting" << dendl;
- utime_t dur = ceph_clock_now(cct) - start;
- if (logger)
+ if (logger) {
+ utime_t dur = ceph_clock_now(cct) - start;
logger->tinc(l_throttle_wait, dur);
+ }
}
delete cv;
@@ -118,6 +124,10 @@ bool Throttle::_wait(int64_t c)
bool Throttle::wait(int64_t m)
{
+ if (0 == max.read()) {
+ return false;
+ }
+
Mutex::Locker l(lock);
if (m) {
assert(m > 0);
@@ -129,6 +139,9 @@ bool Throttle::wait(int64_t m)
int64_t Throttle::take(int64_t c)
{
+ if (0 == max.read()) {
+ return 0;
+ }
assert(c >= 0);
ldout(cct, 10) << "take " << c << dendl;
{
@@ -145,6 +158,10 @@ int64_t Throttle::take(int64_t c)
bool Throttle::get(int64_t c, int64_t m)
{
+ if (0 == max.read()) {
+ return false;
+ }
+
assert(c >= 0);
ldout(cct, 10) << "get " << c << " (" << count.read() << " -> " << (count.read() + c) << ")" << dendl;
bool waited = false;
@@ -170,6 +187,10 @@ bool Throttle::get(int64_t c, int64_t m)
*/
bool Throttle::get_or_fail(int64_t c)
{
+ if (0 == max.read()) {
+ return true;
+ }
+
assert (c >= 0);
Mutex::Locker l(lock);
if (_should_wait(c) || !cond.empty()) {
@@ -193,6 +214,10 @@ bool Throttle::get_or_fail(int64_t c)
int64_t Throttle::put(int64_t c)
{
+ if (0 == max.read()) {
+ return 0;
+ }
+
assert(c >= 0);
ldout(cct, 10) << "put " << c << " (" << count.read() << " -> " << (count.read()-c) << ")" << dendl;
Mutex::Locker l(lock);
diff --git a/src/common/TrackedOp.cc b/src/common/TrackedOp.cc
index d1dbc1e..ddb2f91 100644
--- a/src/common/TrackedOp.cc
+++ b/src/common/TrackedOp.cc
@@ -109,6 +109,8 @@ void OpTracker::dump_ops_in_flight(Formatter *f)
void OpTracker::register_inflight_op(xlist<TrackedOp*>::item *i)
{
+ if (!tracking_enabled)
+ return;
Mutex::Locker locker(ops_in_flight_lock);
ops_in_flight.push_back(i);
ops_in_flight.back()->seq = seq++;
@@ -116,18 +118,23 @@ void OpTracker::register_inflight_op(xlist<TrackedOp*>::item *i)
void OpTracker::unregister_inflight_op(TrackedOp *i)
{
+ // caller checks;
+ assert(tracking_enabled);
+
+ i->request->clear_data();
+ i->request->clear_payload();
+
Mutex::Locker locker(ops_in_flight_lock);
assert(i->xitem.get_list() == &ops_in_flight);
utime_t now = ceph_clock_now(cct);
i->xitem.remove_myself();
- i->request->clear_data();
history.insert(now, TrackedOpRef(i));
}
bool OpTracker::check_ops_in_flight(std::vector<string> &warning_vector)
{
Mutex::Locker locker(ops_in_flight_lock);
- if (!ops_in_flight.size())
+ if (!ops_in_flight.size()) // this covers tracking_enabled, too
return false;
utime_t now = ceph_clock_now(cct);
@@ -204,7 +211,7 @@ void OpTracker::get_age_ms_histogram(pow2_hist_t *h)
continue;
}
if (count)
- h->set(bin, count);
+ h->set_bin(bin, count);
while (lb > ms) {
bin--;
lb >>= 1;
@@ -212,11 +219,13 @@ void OpTracker::get_age_ms_histogram(pow2_hist_t *h)
count = 1;
}
if (count)
- h->set(bin, count);
+ h->set_bin(bin, count);
}
void OpTracker::mark_event(TrackedOp *op, const string &dest)
{
+ if (!tracking_enabled)
+ return;
utime_t now = ceph_clock_now(cct);
return _mark_event(op, dest, now);
}
@@ -232,6 +241,11 @@ void OpTracker::_mark_event(TrackedOp *op, const string &evt,
}
void OpTracker::RemoveOnDelete::operator()(TrackedOp *op) {
+ if (!tracker->tracking_enabled) {
+ op->request->clear_data();
+ delete op;
+ return;
+ }
op->mark_event("done");
tracker->unregister_inflight_op(op);
// Do not delete op, unregister_inflight_op took control
diff --git a/src/common/TrackedOp.h b/src/common/TrackedOp.h
index 9918482..4673c1b 100644
--- a/src/common/TrackedOp.h
+++ b/src/common/TrackedOp.h
@@ -17,7 +17,7 @@
#include <stdint.h>
#include <include/utime.h>
#include "common/Mutex.h"
-#include "include/histogram.h"
+#include "common/histogram.h"
#include "include/xlist.h"
#include "msg/Message.h"
#include "include/memory.h"
@@ -67,9 +67,11 @@ class OpTracker {
int log_threshold;
public:
+ bool tracking_enabled;
CephContext *cct;
- OpTracker(CephContext *cct_) : seq(0), ops_in_flight_lock("OpTracker mutex"),
- complaint_time(0), log_threshold(0), cct(cct_) {}
+ OpTracker(CephContext *cct_, bool tracking) : seq(0), ops_in_flight_lock("OpTracker mutex"),
+ complaint_time(0), log_threshold(0),
+ tracking_enabled(tracking), cct(cct_) {}
void set_complaint_and_threshold(float time, int threshold) {
complaint_time = time;
log_threshold = threshold;
diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc
index 46c6f3d..1190b6d 100644
--- a/src/common/admin_socket.cc
+++ b/src/common/admin_socket.cc
@@ -328,6 +328,9 @@ bool AdminSocket::do_accept()
return false;
}
cmd_getval(m_cct, cmdmap, "format", format);
+ if (format != "json" && format != "json-pretty" &&
+ format != "xml" && format != "xml-pretty")
+ format = "json-pretty";
cmd_getval(m_cct, cmdmap, "prefix", c);
string firstword;
@@ -451,6 +454,8 @@ public:
HelpHook(AdminSocket *as) : m_as(as) {}
bool call(string command, cmdmap_t &cmdmap, string format, bufferlist& out) {
Formatter *f = new_formatter(format);
+ if (!f)
+ f = new_formatter("json-pretty");
f->open_object_section("help");
for (map<string,string>::iterator p = m_as->m_help.begin();
p != m_as->m_help.end();
diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index eab9143..e6f989c 100644
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -94,7 +94,7 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
if (r < 0)
return r;
buf[r] = '\0';
- size = strict_strtol(buf, 10, &err);
+ size_t size = strict_strtol(buf, 10, &err);
if (!err.empty())
return -EIO;
buffer_max_pipe_size.set(size);
@@ -108,7 +108,7 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
if (size)
return size;
if (update_max_pipe_size() == 0)
- return buffer_max_pipe_size.read()
+ return buffer_max_pipe_size.read();
#endif
// this is the max size hardcoded in linux before 2.6.35
return 65536;
@@ -447,7 +447,7 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
if (r < (ssize_t)len) {
bdout << "raw_pipe: error reading from temp pipe:" << cpp_strerror(r)
<< bendl;
- delete data;
+ free(data);
data = NULL;
close_pipe(tmpfd);
throw error_code(r);
diff --git a/src/common/ceph_argparse.cc b/src/common/ceph_argparse.cc
index 6d1c613..eb000dc 100644
--- a/src/common/ceph_argparse.cc
+++ b/src/common/ceph_argparse.cc
@@ -56,6 +56,25 @@ void string_to_vec(std::vector<std::string>& args, std::string argstr)
}
}
+bool split_dashdash(const std::vector<const char*>& args,
+ std::vector<const char*>& options,
+ std::vector<const char*>& arguments) {
+ bool dashdash = false;
+ for (std::vector<const char*>::const_iterator i = args.begin();
+ i != args.end();
+ i++) {
+ if (dashdash) {
+ arguments.push_back(*i);
+ } else {
+ if (strcmp(*i, "--") == 0)
+ dashdash = true;
+ else
+ options.push_back(*i);
+ }
+ }
+ return dashdash;
+}
+
void env_to_vec(std::vector<const char*>& args, const char *name)
{
if (!name)
@@ -64,13 +83,32 @@ void env_to_vec(std::vector<const char*>& args, const char *name)
if (!p)
return;
+ bool dashdash = false;
+ std::vector<const char*> options;
+ std::vector<const char*> arguments;
+ if (split_dashdash(args, options, arguments))
+ dashdash = true;
+
+ std::vector<const char*> env_options;
+ std::vector<const char*> env_arguments;
static vector<string> str_vec;
+ std::vector<const char*> env;
str_vec.clear();
get_str_vec(p, " ", str_vec);
for (vector<string>::iterator i = str_vec.begin();
i != str_vec.end();
- i++)
- args.push_back(i->c_str());
+ ++i)
+ env.push_back(i->c_str());
+ if (split_dashdash(env, env_options, env_arguments))
+ dashdash = true;
+
+ args.clear();
+ args.insert(args.end(), options.begin(), options.end());
+ args.insert(args.end(), env_options.begin(), env_options.end());
+ if (dashdash)
+ args.push_back("--");
+ args.insert(args.end(), arguments.begin(), arguments.end());
+ args.insert(args.end(), env_arguments.begin(), env_arguments.end());
}
void argv_to_vec(int argc, const char **argv,
diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc
index f7c2df5..4ebf79e 100644
--- a/src/common/ceph_context.cc
+++ b/src/common/ceph_context.cc
@@ -172,6 +172,8 @@ void CephContext::do_command(std::string command, cmdmap_t& cmdmap,
std::string format, bufferlist *out)
{
Formatter *f = new_formatter(format);
+ if (!f)
+ f = new_formatter("json-pretty");
stringstream ss;
for (cmdmap_t::iterator it = cmdmap.begin(); it != cmdmap.end(); ++it) {
if (it->first != "prefix") {
diff --git a/src/common/ceph_strings.cc b/src/common/ceph_strings.cc
index 686f504..d82ecbb 100644
--- a/src/common/ceph_strings.cc
+++ b/src/common/ceph_strings.cc
@@ -101,7 +101,10 @@ const char *ceph_osd_op_name(int op)
case CEPH_OSD_OP_OMAPSETHEADER: return "omap-set-header";
case CEPH_OSD_OP_OMAPCLEAR: return "omap-clear";
case CEPH_OSD_OP_OMAPRMKEYS: return "omap-rm-keys";
+
+ case CEPH_OSD_OP_SETALLOCHINT: return "set-alloc-hint";
}
+
return "???";
}
@@ -170,6 +173,7 @@ const char *ceph_mds_op_name(int op)
case CEPH_MDS_OP_LOOKUPHASH: return "lookuphash";
case CEPH_MDS_OP_LOOKUPPARENT: return "lookupparent";
case CEPH_MDS_OP_LOOKUPINO: return "lookupino";
+ case CEPH_MDS_OP_LOOKUPNAME: return "lookupname";
case CEPH_MDS_OP_GETATTR: return "getattr";
case CEPH_MDS_OP_SETXATTR: return "setxattr";
case CEPH_MDS_OP_SETATTR: return "setattr";
diff --git a/src/common/config.cc b/src/common/config.cc
index a7f948f..4b85b6b 100644
--- a/src/common/config.cc
+++ b/src/common/config.cc
@@ -334,9 +334,11 @@ void md_config_t::_show_config(std::ostream *out, Formatter *f)
<< "/" << subsys.get_gather_level(o) << std::endl;
if (f) {
ostringstream ss;
+ std::string debug_name = "debug_";
+ debug_name += subsys.get_name(o);
ss << subsys.get_log_level(o)
<< "/" << subsys.get_gather_level(o);
- f->dump_string(subsys.get_name(o).c_str(), ss.str());
+ f->dump_string(debug_name.c_str(), ss.str());
}
}
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
@@ -995,7 +997,7 @@ bool md_config_t::expand_meta(std::string &origval,
*oss << "expansion stack: " << std::endl;
for (list<config_option *>::iterator j = stack.begin();
j != stack.end();
- j++) {
+ ++j) {
*oss << (*j)->name << "=" << *(string *)(*j)->conf_ptr(this) << std::endl;
}
return false;
diff --git a/src/common/config.h b/src/common/config.h
index d54642b..242b467 100644
--- a/src/common/config.h
+++ b/src/common/config.h
@@ -38,6 +38,8 @@ enum {
#define OSD_REP_SPLAY 1
#define OSD_REP_CHAIN 2
+#define OSD_POOL_ERASURE_CODE_STRIPE_WIDTH 4096
+
struct config_option;
class CephContext;
diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 41fc9e1..f8711e0 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -118,10 +118,11 @@ OPTION(ms_bind_port_min, OPT_INT, 6800)
OPTION(ms_bind_port_max, OPT_INT, 7300)
OPTION(ms_rwthread_stack_bytes, OPT_U64, 1024 << 10)
OPTION(ms_tcp_read_timeout, OPT_U64, 900)
-OPTION(ms_pq_max_tokens_per_priority, OPT_U64, 4194304)
+OPTION(ms_pq_max_tokens_per_priority, OPT_U64, 16777216)
OPTION(ms_pq_min_cost, OPT_U64, 65536)
OPTION(ms_inject_socket_failures, OPT_U64, 0)
OPTION(ms_inject_delay_type, OPT_STR, "") // "osd mds mon client" allowed
+OPTION(ms_inject_delay_msg_type, OPT_STR, "") // the type of message to delay, as returned by Message::get_type_name(). This is an additional restriction on the general type filter ms_inject_delay_type.
OPTION(ms_inject_delay_max, OPT_DOUBLE, 1) // seconds
OPTION(ms_inject_delay_probability, OPT_DOUBLE, 0) // range [0, 1]
OPTION(ms_inject_internal_delays, OPT_DOUBLE, 0) // seconds
@@ -150,6 +151,7 @@ OPTION(mon_osd_min_up_ratio, OPT_DOUBLE, .3) // min osds required to be up to
OPTION(mon_osd_min_in_ratio, OPT_DOUBLE, .3) // min osds required to be in to mark things out
OPTION(mon_osd_max_op_age, OPT_DOUBLE, 32) // max op age before we get concerned (make it a power of 2)
OPTION(mon_osd_max_split_count, OPT_INT, 32) // largest number of PGs per "involved" OSD to let split create
+OPTION(mon_osd_allow_primary_affinity, OPT_BOOL, false) // allow primary_affinity to be set in the osdmap
OPTION(mon_stat_smooth_intervals, OPT_INT, 2) // smooth stats over last N PGMap maps
OPTION(mon_lease, OPT_FLOAT, 5) // lease interval
OPTION(mon_lease_renew_interval, OPT_FLOAT, 3) // on leader, to renew the lease
@@ -164,6 +166,7 @@ OPTION(mon_pg_warn_min_per_osd, OPT_INT, 20) // min # pgs per (in) osd before w
OPTION(mon_pg_warn_max_object_skew, OPT_FLOAT, 10.0) // max skew few average in objects per pg
OPTION(mon_pg_warn_min_objects, OPT_INT, 10000) // do not warn below this object #
OPTION(mon_pg_warn_min_pool_objects, OPT_INT, 1000) // do not warn on pools below this object #
+OPTION(mon_cache_target_full_warn_ratio, OPT_FLOAT, .66) // position between pool cache_target_full and max where we start warning
OPTION(mon_osd_full_ratio, OPT_FLOAT, .95) // what % full makes an OSD "full"
OPTION(mon_osd_nearfull_ratio, OPT_FLOAT, .85) // what % full makes an OSD near full
OPTION(mon_globalid_prealloc, OPT_INT, 100) // how many globalids to prealloc
@@ -198,6 +201,7 @@ OPTION(mon_osd_min_down_reports, OPT_INT, 3) // number of times a down OSD m
OPTION(mon_osd_force_trim_to, OPT_INT, 0) // force mon to trim maps to this point, regardless of min_last_epoch_clean (dangerous, use with care)
OPTION(mon_mds_force_trim_to, OPT_INT, 0) // force mon to trim mdsmaps to this point (dangerous, use with care)
+OPTION(mon_advanced_debug_mode, OPT_BOOL, false) // true for developper oriented testing
// dump transactions
OPTION(mon_debug_dump_transactions, OPT_BOOL, false)
OPTION(mon_debug_dump_location, OPT_STR, "/var/log/ceph/$cluster-$name.tdump")
@@ -237,6 +241,8 @@ OPTION(auth_service_ticket_ttl, OPT_DOUBLE, 60*60)
OPTION(auth_debug, OPT_BOOL, false) // if true, assert when weird things happen
OPTION(mon_client_hunt_interval, OPT_DOUBLE, 3.0) // try new mon every N seconds until we connect
OPTION(mon_client_ping_interval, OPT_DOUBLE, 10.0) // ping every N seconds
+OPTION(mon_client_hunt_interval_backoff, OPT_DOUBLE, 2.0) // each time we reconnect to a monitor, double our timeout
+OPTION(mon_client_hunt_interval_max_multiple, OPT_DOUBLE, 10.0) // up to a max of 10*default (30 seconds)
OPTION(mon_client_max_log_entries_per_message, OPT_INT, 1000)
OPTION(mon_max_pool_pg_num, OPT_INT, 65536)
OPTION(mon_pool_quota_warn_threshold, OPT_INT, 0) // percent of quota at which to issue warnings
@@ -383,6 +389,18 @@ OPTION(osd_backfill_full_ratio, OPT_FLOAT, 0.85)
// Seconds to wait before retrying refused backfills
OPTION(osd_backfill_retry_interval, OPT_DOUBLE, 10.0)
+// max agent flush ops
+OPTION(osd_agent_max_ops, OPT_INT, 4)
+OPTION(osd_agent_min_evict_effort, OPT_FLOAT, .1)
+OPTION(osd_agent_quantize_effort, OPT_FLOAT, .1)
+
+// decay atime and hist histograms after how many objects go by
+OPTION(osd_agent_hist_halflife, OPT_INT, 1000)
+
+// must be this amount over the threshold to enable,
+// this amount below the threshold to disable.
+OPTION(osd_agent_slop, OPT_FLOAT, .02)
+
OPTION(osd_uuid, OPT_UUID, uuid_d())
OPTION(osd_data, OPT_STR, "/var/lib/ceph/osd/$cluster-$id")
OPTION(osd_journal, OPT_STR, "/var/lib/ceph/osd/$cluster-$id/journal")
@@ -396,7 +414,7 @@ OPTION(osd_pgp_bits, OPT_INT, 6) // bits per osd
OPTION(osd_crush_chooseleaf_type, OPT_INT, 1) // 1 = host
OPTION(osd_pool_default_crush_rule, OPT_INT, -1) // deprecated for osd_pool_default_crush_replicated_ruleset
OPTION(osd_pool_default_crush_replicated_ruleset, OPT_INT, CEPH_DEFAULT_CRUSH_REPLICATED_RULESET)
-OPTION(osd_pool_default_crush_erasure_ruleset, OPT_INT, CEPH_DEFAULT_CRUSH_ERASURE_RULESET)
+OPTION(osd_pool_erasure_code_stripe_width, OPT_U32, OSD_POOL_ERASURE_CODE_STRIPE_WIDTH) // in bytes
OPTION(osd_pool_default_size, OPT_INT, 3)
OPTION(osd_pool_default_min_size, OPT_INT, 0) // 0 means no specific default; ceph will use size-size/2
OPTION(osd_pool_default_pg_num, OPT_INT, 8) // number of PGs for new pools. Configure in global or mon section of ceph.conf
@@ -405,15 +423,25 @@ OPTION(osd_pool_default_erasure_code_directory, OPT_STR, CEPH_PKGLIBDIR"/erasure
OPTION(osd_pool_default_erasure_code_properties,
OPT_STR,
"erasure-code-plugin=jerasure "
- "erasure-code-technique=cauchy_good "
- "erasure-code-packetsize=3072 "
+ "erasure-code-technique=reed_sol_van "
"erasure-code-k=4 "
"erasure-code-m=2 "
) // default properties of osd pool create
OPTION(osd_pool_default_flags, OPT_INT, 0) // default flags for new pools
OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true) // use new pg hashing to prevent pool/pg overlap
+OPTION(osd_pool_default_hit_set_bloom_fpp, OPT_FLOAT, .05)
+OPTION(osd_pool_default_cache_target_dirty_ratio, OPT_FLOAT, .4)
+OPTION(osd_pool_default_cache_target_full_ratio, OPT_FLOAT, .8)
+OPTION(osd_pool_default_cache_min_flush_age, OPT_INT, 0) // seconds
+OPTION(osd_pool_default_cache_min_evict_age, OPT_INT, 0) // seconds
OPTION(osd_hit_set_min_size, OPT_INT, 1000) // min target size for a HitSet
OPTION(osd_hit_set_namespace, OPT_STR, ".ceph-internal") // rados namespace for hit_set tracking
+
+OPTION(osd_tier_default_cache_mode, OPT_STR, "writeback")
+OPTION(osd_tier_default_cache_hit_set_count, OPT_INT, 4)
+OPTION(osd_tier_default_cache_hit_set_period, OPT_INT, 1200)
+OPTION(osd_tier_default_cache_hit_set_type, OPT_STR, "bloom")
+
OPTION(osd_map_dedup, OPT_BOOL, true)
OPTION(osd_map_cache_size, OPT_INT, 500)
OPTION(osd_map_message_max, OPT_INT, 100) // max maps per MOSDMap message
@@ -503,6 +531,7 @@ OPTION(osd_debug_op_order, OPT_BOOL, false)
OPTION(osd_debug_verify_snaps_on_info, OPT_BOOL, false)
OPTION(osd_debug_verify_stray_on_activate, OPT_BOOL, false)
OPTION(osd_debug_skip_full_check_in_backfill_reservation, OPT_BOOL, false)
+OPTION(osd_enable_op_tracker, OPT_BOOL, true) // enable/disable OSD op tracking
OPTION(osd_op_history_size, OPT_U32, 20) // Max number of completed ops to track
OPTION(osd_op_history_duration, OPT_U32, 600) // Oldest completed op to track
OPTION(osd_target_transaction_size, OPT_INT, 30) // to adjust various transactions that batch smaller items
@@ -520,8 +549,8 @@ OPTION(osd_leveldb_log, OPT_STR, "") // enable OSD leveldb log file
// determines whether PGLog::check() compares written out log to stored log
OPTION(osd_debug_pg_log_writeout, OPT_BOOL, false)
-OPTION(leveldb_write_buffer_size, OPT_U64, 0) // leveldb write buffer size
-OPTION(leveldb_cache_size, OPT_U64, 0) // leveldb cache size
+OPTION(leveldb_write_buffer_size, OPT_U64, 8 *1024*1024) // leveldb write buffer size
+OPTION(leveldb_cache_size, OPT_U64, 128 *1024*1024) // leveldb cache size
OPTION(leveldb_block_size, OPT_U64, 0) // leveldb block size
OPTION(leveldb_bloom_size, OPT_INT, 0) // leveldb bloom bits per entry
OPTION(leveldb_max_open_files, OPT_INT, 0) // leveldb max open files
@@ -556,6 +585,11 @@ OPTION(osd_objectstore, OPT_STR, "filestore") // ObjectStore backend type
// Set to true for testing. Users should NOT set this.
OPTION(osd_debug_override_acting_compat, OPT_BOOL, false)
+OPTION(osd_bench_small_size_max_iops, OPT_U32, 100) // 100 IOPS
+OPTION(osd_bench_large_size_max_throughput, OPT_U64, 100 << 20) // 100 MB/s
+OPTION(osd_bench_max_block_size, OPT_U64, 64 << 20) // cap the block size at 64MB
+OPTION(osd_bench_duration, OPT_U32, 30) // duration of 'osd bench', capped at 30s to avoid triggering timeouts
+
OPTION(filestore_debug_disable_sharded_check, OPT_BOOL, false)
/// filestore wb throttle limits
@@ -599,6 +633,8 @@ OPTION(filestore_max_inline_xattrs_other, OPT_U32, 2)
OPTION(filestore_sloppy_crc, OPT_BOOL, false) // track sloppy crcs
OPTION(filestore_sloppy_crc_block_size, OPT_INT, 65536)
+OPTION(filestore_max_alloc_hint_size, OPT_U64, 1ULL << 20) // bytes
+
OPTION(filestore_max_sync_interval, OPT_DOUBLE, 5) // seconds
OPTION(filestore_min_sync_interval, OPT_DOUBLE, .01) // seconds
OPTION(filestore_btrfs_snap, OPT_BOOL, true)
@@ -633,6 +669,13 @@ OPTION(journal_dio, OPT_BOOL, true)
OPTION(journal_aio, OPT_BOOL, true)
OPTION(journal_force_aio, OPT_BOOL, false)
+OPTION(keyvaluestore_queue_max_ops, OPT_INT, 50)
+OPTION(keyvaluestore_queue_max_bytes, OPT_INT, 100 << 20)
+OPTION(keyvaluestore_debug_check_backend, OPT_BOOL, 0) // Expensive debugging check on sync
+OPTION(keyvaluestore_op_threads, OPT_INT, 2)
+OPTION(keyvaluestore_op_thread_timeout, OPT_INT, 60)
+OPTION(keyvaluestore_op_thread_suicide_timeout, OPT_INT, 180)
+
// max bytes to search ahead in journal searching for corruption
OPTION(journal_max_corrupt_search, OPT_U64, 10<<20)
OPTION(journal_block_align, OPT_BOOL, true)
@@ -646,6 +689,9 @@ OPTION(journal_replay_from, OPT_INT, 0)
OPTION(journal_zero_on_create, OPT_BOOL, false)
OPTION(journal_ignore_corruption, OPT_BOOL, false) // assume journal is not corrupt
+OPTION(rados_mon_op_timeout, OPT_DOUBLE, 0) // how many seconds to wait for a response from the monitor before returning an error from a rados operation. 0 means on limit.
+OPTION(rados_osd_op_timeout, OPT_DOUBLE, 0) // how many seconds to wait for a response from osds before returning an error from a rados operation. 0 means no limit.
+
OPTION(rbd_cache, OPT_BOOL, false) // whether to enable caching (writeback unless rbd_cache_max_dirty is 0)
OPTION(rbd_cache_writethrough_until_flush, OPT_BOOL, false) // whether to make writeback caching writethrough until flush is called, to be sure the user of librbd will send flushs so that writeback is safe
OPTION(rbd_cache_size, OPT_LONGLONG, 32<<20) // cache size in bytes
@@ -782,6 +828,7 @@ OPTION(rgw_user_quota_sync_wait_time, OPT_INT, 3600 * 24) // min time between tw
OPTION(rgw_multipart_min_part_size, OPT_INT, 5 * 1024 * 1024) // min size for each part (except for last one) in multipart upload
OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter
+OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter
// This will be set to true when it is safe to start threads.
// Once it is true, it will never change.
diff --git a/src/common/histogram.cc b/src/common/histogram.cc
new file mode 100644
index 0000000..e142292
--- /dev/null
+++ b/src/common/histogram.cc
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "common/histogram.h"
+#include "common/Formatter.h"
+
+// -- pow2_hist_t --
+void pow2_hist_t::dump(Formatter *f) const
+{
+ f->open_array_section("histogram");
+ for (std::vector<int32_t>::const_iterator p = h.begin(); p != h.end(); ++p)
+ f->dump_int("count", *p);
+ f->close_section();
+ f->dump_int("upper_bound", upper_bound());
+}
+
+void pow2_hist_t::encode(bufferlist& bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(h, bl);
+ ENCODE_FINISH(bl);
+}
+
+void pow2_hist_t::decode(bufferlist::iterator& p)
+{
+ DECODE_START(1, p);
+ ::decode(h, p);
+ DECODE_FINISH(p);
+}
+
+void pow2_hist_t::generate_test_instances(std::list<pow2_hist_t*>& ls)
+{
+ ls.push_back(new pow2_hist_t);
+ ls.push_back(new pow2_hist_t);
+ ls.back()->h.push_back(1);
+ ls.back()->h.push_back(3);
+ ls.back()->h.push_back(0);
+ ls.back()->h.push_back(2);
+}
+
+void pow2_hist_t::decay(int bits)
+{
+ for (std::vector<int32_t>::iterator p = h.begin(); p != h.end(); ++p) {
+ *p >>= bits;
+ }
+ _contract();
+}
diff --git a/src/include/histogram.h b/src/common/histogram.h
similarity index 52%
rename from src/include/histogram.h
rename to src/common/histogram.h
index c817b1e..bdbd75c 100644
--- a/src/include/histogram.h
+++ b/src/common/histogram.h
@@ -10,8 +10,17 @@
* Copyright 2013 Inktank
*/
-#ifndef HISTOGRAM_H_
-#define HISTOGRAM_H_
+#ifndef CEPH_HISTOGRAM_H
+#define CEPH_HISTOGRAM_H
+
+#include <vector>
+#include <list>
+
+#include "include/encoding.h"
+
+namespace ceph {
+ class Formatter;
+}
/**
* power of 2 histogram
@@ -23,7 +32,7 @@ struct pow2_hist_t { //
* bin size is 2^index
* value is count of elements that are <= the current bin but > the previous bin.
*/
- vector<int32_t> h;
+ std::vector<int32_t> h;
private:
/// expand to at least another's size
@@ -43,12 +52,55 @@ public:
void clear() {
h.clear();
}
- void set(int bin, int32_t v) {
+ void set_bin(int bin, int32_t count) {
+ _expand_to(bin + 1);
+ h[bin] = count;
+ _contract();
+ }
+
+ void add(int32_t v) {
+ int bin = calc_bits_of(v);
_expand_to(bin + 1);
- h[bin] = v;
+ h[bin]++;
_contract();
}
+ static int calc_bits_of(int t) {
+ int b = 0;
+ while (t > 0) {
+ t = t >> 1;
+ b++;
+ }
+ return b;
+ }
+
+ /// get a value's position in the histogram.
+ ///
+ /// positions are represented as values in the range [0..1000000]
+ /// (millionths on the unit interval).
+ ///
+ /// @param v [in] value (non-negative)
+ /// @param lower [out] pointer to lower-bound (0..1000000)
+ /// @param upper [out] pointer to the upper bound (0..1000000)
+ int get_position_micro(int32_t v, uint64_t *lower, uint64_t *upper) {
+ if (v < 0)
+ return -1;
+ unsigned bin = calc_bits_of(v);
+ uint64_t lower_sum = 0, upper_sum = 0, total = 0;
+ for (unsigned i=0; i<h.size(); ++i) {
+ if (i <= bin)
+ upper_sum += h[i];
+ if (i < bin)
+ lower_sum += h[i];
+ total += h[i];
+ }
+ if (total > 0) {
+ *lower = lower_sum * 1000000 / total;
+ *upper = upper_sum * 1000000 / total;
+ }
+ return 0;
+ }
+
void add(const pow2_hist_t& o) {
_expand_to(o.h.size());
for (unsigned p = 0; p < o.h.size(); ++p)
@@ -66,6 +118,9 @@ public:
return 1 << h.size();
}
+ /// decay histogram by N bits (default 1, for a halflife)
+ void decay(int bits = 1);
+
void dump(Formatter *f) const;
void encode(bufferlist &bl) const;
void decode(bufferlist::iterator &bl);
@@ -73,4 +128,4 @@ public:
};
WRITE_CLASS_ENCODER(pow2_hist_t)
-#endif /* HISTOGRAM_H_ */
+#endif /* CEPH_HISTOGRAM_H */
diff --git a/src/common/hobject.cc b/src/common/hobject.cc
index ffc73aa..ecc8cfd 100644
--- a/src/common/hobject.cc
+++ b/src/common/hobject.cc
@@ -272,7 +272,8 @@ void ghobject_t::generate_test_instances(list<ghobject_t*>& o)
ostream& operator<<(ostream& out, const ghobject_t& o)
{
out << o.hobj;
- if (o.generation != ghobject_t::NO_GEN) {
+ if (o.generation != ghobject_t::NO_GEN ||
+ o.shard_id != ghobject_t::NO_SHARD) {
assert(o.shard_id != ghobject_t::NO_SHARD);
out << "/" << o.generation << "/" << (unsigned)(o.shard_id);
}
diff --git a/src/common/hobject.h b/src/common/hobject.h
index 8d97411..2933483 100644
--- a/src/common/hobject.h
+++ b/src/common/hobject.h
@@ -95,11 +95,6 @@ public:
return ret;
}
- /// @return true if object is snapdir
- bool is_snapdir() const {
- return snap == CEPH_SNAPDIR;
- }
-
/// @return snapdir version of this hobject_t
hobject_t get_snapdir() const {
hobject_t ret(*this);
@@ -247,6 +242,7 @@ struct ghobject_t {
public:
static const shard_t NO_SHARD = UINT8_MAX;
+ static shard_t no_shard() { return NO_SHARD; }
static const gen_t NO_GEN = UINT64_MAX;
ghobject_t() : generation(NO_GEN), shard_id(NO_SHARD) {}
diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc
index d3c3b44..7067599 100644
--- a/src/common/obj_bencher.cc
+++ b/src/common/obj_bencher.cc
@@ -170,7 +170,6 @@ int ObjBencher::aio_bench(
int concurrentios, int op_size, bool cleanup) {
int object_size = op_size;
int num_objects = 0;
- char* contentsChars = new char[op_size];
int r = 0;
int prevPid = 0;
@@ -178,7 +177,6 @@ int ObjBencher::aio_bench(
if (operation != OP_WRITE) {
r = fetch_bench_metadata(BENCH_LASTRUN_METADATA, &object_size, &num_objects, &prevPid);
if (r < 0) {
- delete[] contentsChars;
if (r == -ENOENT)
cerr << "Must write data before running a read benchmark!" << std::endl;
return r;
@@ -187,6 +185,7 @@ int ObjBencher::aio_bench(
object_size = op_size;
}
+ char* contentsChars = new char[object_size];
lock.Lock();
data.done = false;
data.object_size = object_size;
diff --git a/src/common/shared_cache.hpp b/src/common/shared_cache.hpp
index a435ec4..df52178 100644
--- a/src/common/shared_cache.hpp
+++ b/src/common/shared_cache.hpp
@@ -29,6 +29,7 @@ class SharedLRU {
Mutex lock;
size_t max_size;
Cond cond;
+ unsigned size;
map<K, typename list<pair<K, VPtr> >::iterator > contents;
list<pair<K, VPtr> > lru;
@@ -36,23 +37,29 @@ class SharedLRU {
map<K, WeakVPtr> weak_refs;
void trim_cache(list<VPtr> *to_release) {
- while (lru.size() > max_size) {
+ while (size > max_size) {
to_release->push_back(lru.back().second);
lru_remove(lru.back().first);
}
}
void lru_remove(K key) {
- if (!contents.count(key))
+ typename map<K, typename list<pair<K, VPtr> >::iterator>::iterator i =
+ contents.find(key);
+ if (i == contents.end())
return;
- lru.erase(contents[key]);
- contents.erase(key);
+ lru.erase(i->second);
+ --size;
+ contents.erase(i);
}
void lru_add(K key, VPtr val, list<VPtr> *to_release) {
- if (contents.count(key)) {
- lru.splice(lru.begin(), lru, contents[key]);
+ typename map<K, typename list<pair<K, VPtr> >::iterator>::iterator i =
+ contents.find(key);
+ if (i != contents.end()) {
+ lru.splice(lru.begin(), lru, i->second);
} else {
+ ++size;
lru.push_front(make_pair(key, val));
contents[key] = lru.begin();
trim_cache(to_release);
@@ -77,7 +84,8 @@ class SharedLRU {
};
public:
- SharedLRU(size_t max_size = 20) : lock("SharedLRU::lock"), max_size(max_size) {}
+ SharedLRU(size_t max_size = 20)
+ : lock("SharedLRU::lock"), max_size(max_size), size(0) {}
~SharedLRU() {
contents.clear();
diff --git a/src/common/str_map.cc b/src/common/str_map.cc
index a17cf77..e635159 100644
--- a/src/common/str_map.cc
+++ b/src/common/str_map.cc
@@ -52,7 +52,7 @@ int get_str_map(const string &str,
list<string> pairs;
get_str_list(str, "\t\n ", pairs);
- for (list<string>::iterator i = pairs.begin(); i != pairs.end(); i++) {
+ for (list<string>::iterator i = pairs.begin(); i != pairs.end(); ++i) {
size_t equal = i->find('=');
if (equal == string::npos)
(*str_map)[*i] = string();
diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index a954fec..d75dddf 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -188,6 +188,8 @@ int CrushCompiler::decompile(ostream &out)
out << "tunable choose_total_tries " << crush.get_choose_total_tries() << "\n";
if (crush.get_chooseleaf_descend_once() != 0)
out << "tunable chooseleaf_descend_once " << crush.get_chooseleaf_descend_once() << "\n";
+ if (crush.get_chooseleaf_vary_r() != 0)
+ out << "tunable chooseleaf_vary_r " << crush.get_chooseleaf_vary_r() << "\n";
out << "\n# devices\n";
for (int i=0; i<crush.get_max_devices(); i++) {
@@ -269,6 +271,10 @@ int CrushCompiler::decompile(ostream &out)
out << "\tstep set_chooseleaf_tries " << crush.get_rule_arg1(i, j)
<< "\n";
break;
+ case CRUSH_RULE_SET_CHOOSELEAF_VARY_R:
+ out << "\tstep set_chooseleaf_vary_r " << crush.get_rule_arg1(i, j)
+ << "\n";
+ break;
case CRUSH_RULE_CHOOSE_FIRSTN:
out << "\tstep choose firstn "
<< crush.get_rule_arg1(i, j)
@@ -359,6 +365,8 @@ int CrushCompiler::parse_tunable(iter_t const& i)
crush.set_choose_total_tries(val);
else if (name == "chooseleaf_descend_once")
crush.set_chooseleaf_descend_once(val);
+ else if (name == "chooseleaf_vary_r")
+ crush.set_chooseleaf_vary_r(val);
else {
err << "tunable " << name << " not recognized" << std::endl;
return -1;
@@ -645,6 +653,13 @@ int CrushCompiler::parse_rule(iter_t const& i)
}
break;
+ case crush_grammar::_step_set_chooseleaf_vary_r:
+ {
+ int val = int_node(s->children[1]);
+ crush.set_rule_step_set_chooseleaf_vary_r(ruleno, step++, val);
+ }
+ break;
+
case crush_grammar::_step_choose:
case crush_grammar::_step_chooseleaf:
{
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index ae4ec1c..61b8a8a 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -24,6 +24,22 @@ bool CrushWrapper::has_v2_rules() const
return false;
}
+bool CrushWrapper::has_v3_rules() const
+{
+ // check rules for use of SET_CHOOSELEAF_VARY_R step
+ for (unsigned i=0; i<crush->max_rules; i++) {
+ crush_rule *r = crush->rules[i];
+ if (!r)
+ continue;
+ for (unsigned j=0; j<r->len; j++) {
+ if (r->steps[j].op == CRUSH_RULE_SET_CHOOSELEAF_VARY_R)
+ return true;
+ }
+ }
+ return false;
+}
+
+
void CrushWrapper::find_takes(set<int>& roots) const
{
for (unsigned i=0; i<crush->max_rules; i++) {
@@ -872,6 +888,7 @@ void CrushWrapper::encode(bufferlist& bl, bool lean) const
::encode(crush->choose_local_fallback_tries, bl);
::encode(crush->choose_total_tries, bl);
::encode(crush->chooseleaf_descend_once, bl);
+ ::encode(crush->chooseleaf_vary_r, bl);
}
static void decode_32_or_64_string_map(map<int32_t,string>& m, bufferlist::iterator& blp)
@@ -952,6 +969,9 @@ void CrushWrapper::decode(bufferlist::iterator& blp)
if (!blp.end()) {
::decode(crush->chooseleaf_descend_once, blp);
}
+ if (!blp.end()) {
+ ::decode(crush->chooseleaf_vary_r, blp);
+ }
finalize();
}
catch (...) {
@@ -1137,7 +1157,9 @@ void CrushWrapper::dump_tunables(Formatter *f) const
f->dump_int("chooseleaf_descend_once", get_chooseleaf_descend_once());
// be helpful about it
- if (has_bobtail_tunables())
+ if (has_firefly_tunables())
+ f->dump_string("profile", "firefly");
+ else if (has_bobtail_tunables())
f->dump_string("profile", "bobtail");
else if (has_argonaut_tunables())
f->dump_string("profile", "argonaut");
@@ -1155,66 +1177,77 @@ void CrushWrapper::dump_rules(Formatter *f) const
for (int i=0; i<get_max_rules(); i++) {
if (!rule_exists(i))
continue;
- f->open_object_section("rule");
- f->dump_int("rule_id", i);
- if (get_rule_name(i))
- f->dump_string("rule_name", get_rule_name(i));
- f->dump_int("ruleset", get_rule_mask_ruleset(i));
- f->dump_int("type", get_rule_mask_type(i));
- f->dump_int("min_size", get_rule_mask_min_size(i));
- f->dump_int("max_size", get_rule_mask_max_size(i));
- f->open_array_section("steps");
- for (int j=0; j<get_rule_len(i); j++) {
- f->open_object_section("step");
- switch (get_rule_op(i, j)) {
- case CRUSH_RULE_NOOP:
- f->dump_string("op", "noop");
- break;
- case CRUSH_RULE_TAKE:
- f->dump_string("op", "take");
- f->dump_int("item", get_rule_arg1(i, j));
- break;
- case CRUSH_RULE_EMIT:
- f->dump_string("op", "emit");
- break;
- case CRUSH_RULE_CHOOSE_FIRSTN:
- f->dump_string("op", "choose_firstn");
- f->dump_int("num", get_rule_arg1(i, j));
- f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
- break;
- case CRUSH_RULE_CHOOSE_INDEP:
- f->dump_string("op", "choose_indep");
- f->dump_int("num", get_rule_arg1(i, j));
- f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
- break;
- case CRUSH_RULE_CHOOSELEAF_FIRSTN:
- f->dump_string("op", "chooseleaf_firstn");
- f->dump_int("num", get_rule_arg1(i, j));
- f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
- break;
- case CRUSH_RULE_CHOOSELEAF_INDEP:
- f->dump_string("op", "chooseleaf_indep");
- f->dump_int("num", get_rule_arg1(i, j));
- f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
- break;
- case CRUSH_RULE_SET_CHOOSE_TRIES:
- f->dump_string("op", "set_choose_tries");
- f->dump_int("num", get_rule_arg1(i, j));
- break;
- case CRUSH_RULE_SET_CHOOSELEAF_TRIES:
- f->dump_string("op", "set_chooseleaf_tries");
- f->dump_int("num", get_rule_arg1(i, j));
- break;
- default:
- f->dump_int("opcode", get_rule_op(i, j));
- f->dump_int("arg1", get_rule_arg1(i, j));
- f->dump_int("arg2", get_rule_arg2(i, j));
+ dump_rule(i, f);
+ }
+}
+
+void CrushWrapper::dump_rule(int ruleset, Formatter *f) const
+{
+ f->open_object_section("rule");
+ f->dump_int("rule_id", ruleset);
+ if (get_rule_name(ruleset))
+ f->dump_string("rule_name", get_rule_name(ruleset));
+ f->dump_int("ruleset", get_rule_mask_ruleset(ruleset));
+ f->dump_int("type", get_rule_mask_type(ruleset));
+ f->dump_int("min_size", get_rule_mask_min_size(ruleset));
+ f->dump_int("max_size", get_rule_mask_max_size(ruleset));
+ f->open_array_section("steps");
+ for (int j=0; j<get_rule_len(ruleset); j++) {
+ f->open_object_section("step");
+ switch (get_rule_op(ruleset, j)) {
+ case CRUSH_RULE_NOOP:
+ f->dump_string("op", "noop");
+ break;
+ case CRUSH_RULE_TAKE:
+ f->dump_string("op", "take");
+ {
+ int item = get_rule_arg1(ruleset, j);
+ f->dump_int("item", item);
+
+ const char *name = get_item_name(item);
+ f->dump_string("item_name", name ? name : "");
}
- f->close_section();
+ break;
+ case CRUSH_RULE_EMIT:
+ f->dump_string("op", "emit");
+ break;
+ case CRUSH_RULE_CHOOSE_FIRSTN:
+ f->dump_string("op", "choose_firstn");
+ f->dump_int("num", get_rule_arg1(ruleset, j));
+ f->dump_string("type", get_type_name(get_rule_arg2(ruleset, j)));
+ break;
+ case CRUSH_RULE_CHOOSE_INDEP:
+ f->dump_string("op", "choose_indep");
+ f->dump_int("num", get_rule_arg1(ruleset, j));
+ f->dump_string("type", get_type_name(get_rule_arg2(ruleset, j)));
+ break;
+ case CRUSH_RULE_CHOOSELEAF_FIRSTN:
+ f->dump_string("op", "chooseleaf_firstn");
+ f->dump_int("num", get_rule_arg1(ruleset, j));
+ f->dump_string("type", get_type_name(get_rule_arg2(ruleset, j)));
+ break;
+ case CRUSH_RULE_CHOOSELEAF_INDEP:
+ f->dump_string("op", "chooseleaf_indep");
+ f->dump_int("num", get_rule_arg1(ruleset, j));
+ f->dump_string("type", get_type_name(get_rule_arg2(ruleset, j)));
+ break;
+ case CRUSH_RULE_SET_CHOOSE_TRIES:
+ f->dump_string("op", "set_choose_tries");
+ f->dump_int("num", get_rule_arg1(ruleset, j));
+ break;
+ case CRUSH_RULE_SET_CHOOSELEAF_TRIES:
+ f->dump_string("op", "set_chooseleaf_tries");
+ f->dump_int("num", get_rule_arg1(ruleset, j));
+ break;
+ default:
+ f->dump_int("opcode", get_rule_op(ruleset, j));
+ f->dump_int("arg1", get_rule_arg1(ruleset, j));
+ f->dump_int("arg2", get_rule_arg2(ruleset, j));
}
f->close_section();
- f->close_section();
}
+ f->close_section();
+ f->close_section();
}
void CrushWrapper::list_rules(Formatter *f) const
@@ -1375,9 +1408,9 @@ bool CrushWrapper::is_valid_crush_name(const string& s)
}
bool CrushWrapper::is_valid_crush_loc(CephContext *cct,
- const map<string,string> loc)
+ const map<string,string>& loc)
{
- for (map<string,string>::const_iterator l = loc.begin(); l != loc.end(); l++) {
+ for (map<string,string>::const_iterator l = loc.begin(); l != loc.end(); ++l) {
if (!is_valid_crush_name(l->first) ||
!is_valid_crush_name(l->second)) {
ldout(cct, 1) << "loc["
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index d0f34f0..3b2e6e6 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -105,19 +105,28 @@ public:
crush->choose_local_fallback_tries = 5;
crush->choose_total_tries = 19;
crush->chooseleaf_descend_once = 0;
+ crush->chooseleaf_vary_r = 0;
}
void set_tunables_bobtail() {
crush->choose_local_tries = 0;
crush->choose_local_fallback_tries = 0;
crush->choose_total_tries = 50;
crush->chooseleaf_descend_once = 1;
+ crush->chooseleaf_vary_r = 0;
+ }
+ void set_tunables_firefly() {
+ crush->choose_local_tries = 0;
+ crush->choose_local_fallback_tries = 0;
+ crush->choose_total_tries = 50;
+ crush->chooseleaf_descend_once = 1;
+ crush->chooseleaf_vary_r = 1;
}
void set_tunables_legacy() {
set_tunables_argonaut();
}
void set_tunables_optimal() {
- set_tunables_bobtail();
+ set_tunables_firefly();
}
void set_tunables_default() {
set_tunables_bobtail();
@@ -151,23 +160,40 @@ public:
crush->chooseleaf_descend_once = !!n;
}
+ int get_chooseleaf_vary_r() const {
+ return crush->chooseleaf_vary_r;
+ }
+ void set_chooseleaf_vary_r(int n) {
+ crush->chooseleaf_vary_r = n;
+ }
+
bool has_argonaut_tunables() const {
return
crush->choose_local_tries == 2 &&
crush->choose_local_fallback_tries == 5 &&
crush->choose_total_tries == 19 &&
- crush->chooseleaf_descend_once == 0;
+ crush->chooseleaf_descend_once == 0 &&
+ crush->chooseleaf_vary_r == 0;
}
bool has_bobtail_tunables() const {
return
crush->choose_local_tries == 0 &&
crush->choose_local_fallback_tries == 0 &&
crush->choose_total_tries == 50 &&
- crush->chooseleaf_descend_once == 1;
+ crush->chooseleaf_descend_once == 1 &&
+ crush->chooseleaf_vary_r == 0;
+ }
+ bool has_firefly_tunables() const {
+ return
+ crush->choose_local_tries == 0 &&
+ crush->choose_local_fallback_tries == 0 &&
+ crush->choose_total_tries == 50 &&
+ crush->chooseleaf_descend_once == 1 &&
+ crush->chooseleaf_vary_r == 1;
}
bool has_optimal_tunables() const {
- return has_bobtail_tunables();
+ return has_firefly_tunables();
}
bool has_legacy_tunables() const {
return has_argonaut_tunables();
@@ -183,7 +209,12 @@ public:
return
crush->chooseleaf_descend_once != 0;
}
+ bool has_nondefault_tunables3() const {
+ return
+ crush->chooseleaf_vary_r != 0;
+ }
bool has_v2_rules() const;
+ bool has_v3_rules() const;
// bucket types
@@ -630,6 +661,9 @@ public:
int set_rule_step_set_chooseleaf_tries(unsigned ruleno, unsigned step, int val) {
return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSELEAF_TRIES, val, 0);
}
+ int set_rule_step_set_chooseleaf_vary_r(unsigned ruleno, unsigned step, int val) {
+ return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSELEAF_VARY_R, val, 0);
+ }
int set_rule_step_choose_firstn(unsigned ruleno, unsigned step, int val, int type) {
return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_FIRSTN, val, type);
}
@@ -789,6 +823,8 @@ public:
/* modifiers */
int add_bucket(int bucketno, int alg, int hash, int type, int size,
int *items, int *weights, int *idout) {
+ if (type == 0)
+ return -EINVAL;
crush_bucket *b = crush_make_bucket(alg, hash, type, size, items, weights);
assert(b);
return crush_add_bucket(crush, bucketno, b, idout);
@@ -860,6 +896,7 @@ public:
void decode_crush_bucket(crush_bucket** bptr, bufferlist::iterator &blp);
void dump(Formatter *f) const;
void dump_rules(Formatter *f) const;
+ void dump_rule(int ruleset, Formatter *f) const;
void dump_tunables(Formatter *f) const;
void list_rules(Formatter *f) const;
void dump_tree(const vector<__u32>& w, ostream *out, Formatter *f) const;
@@ -869,7 +906,7 @@ public:
static bool is_valid_crush_name(const string& s);
static bool is_valid_crush_loc(CephContext *cct,
- const map<string,string> loc);
+ const map<string,string>& loc);
};
WRITE_CLASS_ENCODER(CrushWrapper)
diff --git a/src/crush/builder.c b/src/crush/builder.c
index c524cfc..eff0bf6 100644
--- a/src/crush/builder.c
+++ b/src/crush/builder.c
@@ -26,6 +26,7 @@ struct crush_map *crush_create()
m->choose_local_fallback_tries = 5;
m->choose_total_tries = 19;
m->chooseleaf_descend_once = 0;
+ m->chooseleaf_vary_r = 0;
return m;
}
diff --git a/src/crush/crush.h b/src/crush/crush.h
index 0da7180..8bac92a 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -59,6 +59,7 @@ enum {
CRUSH_RULE_SET_CHOOSELEAF_TRIES = 9, /* override chooseleaf_descend_once */
CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES = 10,
CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES = 11,
+ CRUSH_RULE_SET_CHOOSELEAF_VARY_R = 12
};
/*
@@ -182,6 +183,12 @@ struct crush_map {
* to. */
__u32 chooseleaf_descend_once;
+ /* if non-zero, feed r into chooseleaf, bit-shifted right by (r-1)
+ * bits. a value of 1 is best for new clusters. for legacy clusters
+ * that want to limit reshuffling, a value of 3 or 4 will make the
+ * mappings line up a bit better with previous mappings. */
+ __u8 chooseleaf_vary_r;
+
__u32 *choose_tries;
};
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index b1e1aef..42b0b8e 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -45,6 +45,7 @@ struct crush_grammar : public grammar<crush_grammar>
_bucket,
_step_take,
_step_set_chooseleaf_tries,
+ _step_set_chooseleaf_vary_r,
_step_set_choose_tries,
_step_set_choose_local_tries,
_step_set_choose_local_fallback_tries,
@@ -82,6 +83,7 @@ struct crush_grammar : public grammar<crush_grammar>
rule<ScannerT, parser_context<>, parser_tag<_step_set_choose_local_tries> > step_set_choose_local_tries;
rule<ScannerT, parser_context<>, parser_tag<_step_set_choose_local_fallback_tries> > step_set_choose_local_fallback_tries;
rule<ScannerT, parser_context<>, parser_tag<_step_set_chooseleaf_tries> > step_set_chooseleaf_tries;
+ rule<ScannerT, parser_context<>, parser_tag<_step_set_chooseleaf_vary_r> > step_set_chooseleaf_vary_r;
rule<ScannerT, parser_context<>, parser_tag<_step_choose> > step_choose;
rule<ScannerT, parser_context<>, parser_tag<_step_chooseleaf> > step_chooseleaf;
rule<ScannerT, parser_context<>, parser_tag<_step_emit> > step_emit;
@@ -128,6 +130,7 @@ struct crush_grammar : public grammar<crush_grammar>
step_set_choose_local_tries = str_p("set_choose_local_tries") >> posint;
step_set_choose_local_fallback_tries = str_p("set_choose_local_fallback_tries") >> posint;
step_set_chooseleaf_tries = str_p("set_chooseleaf_tries") >> posint;
+ step_set_chooseleaf_vary_r = str_p("set_chooseleaf_vary_r") >> posint;
step_choose = str_p("choose")
>> ( str_p("indep") | str_p("firstn") )
>> integer
@@ -142,6 +145,7 @@ struct crush_grammar : public grammar<crush_grammar>
step_set_choose_local_tries |
step_set_choose_local_fallback_tries |
step_set_chooseleaf_tries |
+ step_set_chooseleaf_vary_r |
step_choose |
step_chooseleaf |
step_emit );
diff --git a/src/crush/mapper.c b/src/crush/mapper.c
index 0b31844..22cde51 100644
--- a/src/crush/mapper.c
+++ b/src/crush/mapper.c
@@ -296,7 +296,9 @@ static int is_out(const struct crush_map *map,
* @local_retries: localized retries
* @local_fallback_retries: localized fallback retries
* @recurse_to_leaf: true if we want one device under each item of given type (chooseleaf instead of choose)
+ * @vary_r: pass r to recursive calls
* @out2: second output vector for leaf items (if @recurse_to_leaf)
+ * @parent_r: r value passed from the parent
*/
static int crush_choose_firstn(const struct crush_map *map,
struct crush_bucket *bucket,
@@ -308,7 +310,9 @@ static int crush_choose_firstn(const struct crush_map *map,
unsigned int local_retries,
unsigned int local_fallback_retries,
int recurse_to_leaf,
- int *out2)
+ unsigned int vary_r,
+ int *out2,
+ int parent_r)
{
int rep;
unsigned int ftotal, flocal;
@@ -320,8 +324,11 @@ static int crush_choose_firstn(const struct crush_map *map,
int itemtype;
int collide, reject;
- dprintk("CHOOSE%s bucket %d x %d outpos %d numrep %d\n", recurse_to_leaf ? "_LEAF" : "",
- bucket->id, x, outpos, numrep);
+ dprintk("CHOOSE%s bucket %d x %d outpos %d numrep %d tries %d recurse_tries %d local_retries %d local_fallback_retries %d parent_r %d\n",
+ recurse_to_leaf ? "_LEAF" : "",
+ bucket->id, x, outpos, numrep,
+ tries, recurse_tries, local_retries, local_fallback_retries,
+ parent_r);
for (rep = outpos; rep < numrep; rep++) {
/* keep trying until we get a non-out, non-colliding item */
@@ -336,7 +343,7 @@ static int crush_choose_firstn(const struct crush_map *map,
do {
collide = 0;
retry_bucket = 0;
- r = rep;
+ r = rep + parent_r;
/* r' = r + f_total */
r += ftotal;
@@ -388,6 +395,11 @@ static int crush_choose_firstn(const struct crush_map *map,
reject = 0;
if (!collide && recurse_to_leaf) {
if (item < 0) {
+ int sub_r;
+ if (vary_r)
+ sub_r = r >> (vary_r-1);
+ else
+ sub_r = 0;
if (crush_choose_firstn(map,
map->buckets[-1-item],
weight, weight_max,
@@ -397,7 +409,9 @@ static int crush_choose_firstn(const struct crush_map *map,
local_retries,
local_fallback_retries,
0,
- NULL) <= outpos)
+ vary_r,
+ NULL,
+ sub_r) <= outpos)
/* didn't get leaf */
reject = 1;
} else {
@@ -685,6 +699,8 @@ int crush_do_rule(const struct crush_map *map,
int choose_local_retries = map->choose_local_tries;
int choose_local_fallback_retries = map->choose_local_fallback_tries;
+ int vary_r = map->chooseleaf_vary_r;
+
if ((__u32)ruleno >= map->max_rules) {
dprintk(" bad ruleno %d\n", ruleno);
return 0;
@@ -725,6 +741,11 @@ int crush_do_rule(const struct crush_map *map,
choose_local_fallback_retries = curstep->arg1;
break;
+ case CRUSH_RULE_SET_CHOOSELEAF_VARY_R:
+ if (curstep->arg1 >= 0)
+ vary_r = curstep->arg1;
+ break;
+
case CRUSH_RULE_CHOOSELEAF_FIRSTN:
case CRUSH_RULE_CHOOSE_FIRSTN:
firstn = 1;
@@ -777,7 +798,9 @@ int crush_do_rule(const struct crush_map *map,
choose_local_retries,
choose_local_fallback_retries,
recurse_to_leaf,
- c+osize);
+ vary_r,
+ c+osize,
+ 0);
} else {
crush_choose_indep(
map,
diff --git a/src/osd/ErasureCodeInterface.h b/src/erasure-code/ErasureCodeInterface.h
similarity index 94%
rename from src/osd/ErasureCodeInterface.h
rename to src/erasure-code/ErasureCodeInterface.h
index 9a44be7..f8e22d1 100644
--- a/src/osd/ErasureCodeInterface.h
+++ b/src/erasure-code/ErasureCodeInterface.h
@@ -145,6 +145,8 @@
#include "include/memory.h"
#include "include/buffer.h"
+class CrushWrapper;
+
using namespace std;
namespace ceph {
@@ -154,6 +156,24 @@ namespace ceph {
virtual ~ErasureCodeInterface() {}
/**
+ * Create a new ruleset in **crush** under the name **name**,
+ * unless it already exists.
+ *
+ * Return the ruleset number that was created on success. If a
+ * ruleset **name** already exists, return -EEXISTS, otherwise
+ * return a negative value indicating an error with a semantic
+ * defined by the implementation.
+ *
+ * @param [in] name of the ruleset to create
+ * @param [in] crush crushmap in which the ruleset is created
+ * @param [out] ss contains informative messages when an error occurs
+ * @return **0** on success or a negative errno on error.
+ */
+ virtual int create_ruleset(const string &name,
+ CrushWrapper &crush,
+ ostream *ss) const = 0;
+
+ /**
* Return the number of chunks created by a call to the **encode**
* method.
*
diff --git a/src/osd/ErasureCodePlugin.cc b/src/erasure-code/ErasureCodePlugin.cc
similarity index 100%
rename from src/osd/ErasureCodePlugin.cc
rename to src/erasure-code/ErasureCodePlugin.cc
diff --git a/src/osd/ErasureCodePlugin.h b/src/erasure-code/ErasureCodePlugin.h
similarity index 100%
rename from src/osd/ErasureCodePlugin.h
rename to src/erasure-code/ErasureCodePlugin.h
diff --git a/src/erasure-code/Makefile.am b/src/erasure-code/Makefile.am
new file mode 100644
index 0000000..d9f0382
--- /dev/null
+++ b/src/erasure-code/Makefile.am
@@ -0,0 +1,17 @@
+## erasure code plugins
+erasure_codelibdir = $(pkglibdir)/erasure-code
+erasure_codelib_LTLIBRARIES =
+
+include erasure-code/jerasure/Makefile.am
+
+liberasure_code_la_SOURCES = \
+ erasure-code/ErasureCodePlugin.cc
+liberasure_code_la_LIBADD = $(LIBOSDC) $(LIBOS)
+if LINUX
+liberasure_code_la_LIBADD += -ldl
+endif # LINUX
+noinst_LTLIBRARIES += liberasure_code.la
+
+noinst_HEADERS += \
+ erasure-code/ErasureCodeInterface.h \
+ erasure-code/ErasureCodePlugin.h
diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc b/src/erasure-code/jerasure/ErasureCodeJerasure.cc
similarity index 91%
rename from src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
rename to src/erasure-code/jerasure/ErasureCodeJerasure.cc
index f1de0e4..26f978d 100644
--- a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
+++ b/src/erasure-code/jerasure/ErasureCodeJerasure.cc
@@ -18,6 +18,8 @@
#include <algorithm>
#include "common/debug.h"
#include "ErasureCodeJerasure.h"
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
#include "vectorop.h"
extern "C" {
#include "jerasure.h"
@@ -36,9 +38,24 @@ static ostream& _prefix(std::ostream* _dout)
return *_dout << "ErasureCodeJerasure: ";
}
-void ErasureCodeJerasure::init(const map<std::string,std::string> ¶meters)
+int ErasureCodeJerasure::create_ruleset(const string &name,
+ CrushWrapper &crush,
+ ostream *ss) const
+{
+ return crush.add_simple_ruleset(name, ruleset_root, ruleset_failure_domain,
+ "indep", pg_pool_t::TYPE_ERASURE, ss);
+}
+
+void ErasureCodeJerasure::init(const map<string,string> ¶meters)
{
dout(10) << "technique=" << technique << dendl;
+ map<string,string>::const_iterator parameter;
+ parameter = parameters.find("erasure-code-ruleset-root");
+ if (parameter != parameters.end())
+ ruleset_root = parameter->second;
+ parameter = parameters.find("erasure-code-ruleset-failure-domain");
+ if (parameter != parameters.end())
+ ruleset_failure_domain = parameter->second;
parse(parameters);
prepare();
}
@@ -96,11 +113,11 @@ int ErasureCodeJerasure::encode(const set<int> &want_to_encode,
bufferptr pad(padded_length - in.length());
pad.zero();
out.push_back(pad);
- out.rebuild_page_aligned();
}
unsigned coding_length = blocksize * m;
bufferptr coding(buffer::create_page_aligned(coding_length));
out.push_back(coding);
+ out.rebuild_page_aligned();
char *chunks[k + m];
for (int i = 0; i < k + m; i++) {
bufferlist &chunk = (*encoded)[i];
@@ -119,6 +136,22 @@ int ErasureCodeJerasure::decode(const set<int> &want_to_read,
const map<int, bufferlist> &chunks,
map<int, bufferlist> *decoded)
{
+ vector<int> have;
+ have.reserve(chunks.size());
+ for (map<int, bufferlist>::const_iterator i = chunks.begin();
+ i != chunks.end();
+ ++i) {
+ have.push_back(i->first);
+ }
+ if (includes(
+ have.begin(), have.end(), want_to_read.begin(), want_to_read.end())) {
+ for (set<int>::iterator i = want_to_read.begin();
+ i != want_to_read.end();
+ ++i) {
+ (*decoded)[*i] = chunks.find(*i)->second;
+ }
+ return 0;
+ }
unsigned blocksize = (*chunks.begin()).second.length();
int erasures[k + m + 1];
int erasures_count = 0;
@@ -128,10 +161,11 @@ int ErasureCodeJerasure::decode(const set<int> &want_to_read,
if (chunks.find(i) == chunks.end()) {
erasures[erasures_count] = i;
erasures_count++;
- bufferptr ptr(blocksize);
+ bufferptr ptr(buffer::create_page_aligned(blocksize));
(*decoded)[i].push_front(ptr);
} else {
(*decoded)[i] = chunks.find(i)->second;
+ (*decoded)[i].rebuild_page_aligned();
}
if (i < k)
data[i] = (*decoded)[i].c_str();
diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h b/src/erasure-code/jerasure/ErasureCodeJerasure.h
similarity index 95%
rename from src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h
rename to src/erasure-code/jerasure/ErasureCodeJerasure.h
index a0db6a0..811bc3d 100644
--- a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h
+++ b/src/erasure-code/jerasure/ErasureCodeJerasure.h
@@ -17,7 +17,7 @@
#ifndef CEPH_ERASURE_CODE_JERASURE_H
#define CEPH_ERASURE_CODE_JERASURE_H
-#include "osd/ErasureCodeInterface.h"
+#include "erasure-code/ErasureCodeInterface.h"
class ErasureCodeJerasure : public ErasureCodeInterface {
public:
@@ -25,13 +25,21 @@ public:
int m;
int w;
const char *technique;
+ string ruleset_root;
+ string ruleset_failure_domain;
ErasureCodeJerasure(const char *_technique) :
- technique(_technique)
+ technique(_technique),
+ ruleset_root("default"),
+ ruleset_failure_domain("host")
{}
virtual ~ErasureCodeJerasure() {}
+ virtual int create_ruleset(const string &name,
+ CrushWrapper &crush,
+ ostream *ss) const;
+
virtual unsigned int get_chunk_count() const {
return k + m;
}
diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc b/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc
similarity index 98%
rename from src/osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
rename to src/erasure-code/jerasure/ErasureCodePluginJerasure.cc
index d5cb1cd..a8e0510 100644
--- a/src/osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc
+++ b/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc
@@ -15,7 +15,7 @@
*/
#include "common/debug.h"
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
#include "ErasureCodeJerasure.h"
#define dout_subsys ceph_subsys_osd
diff --git a/src/erasure-code/jerasure/Makefile.am b/src/erasure-code/jerasure/Makefile.am
new file mode 100644
index 0000000..4e85404
--- /dev/null
+++ b/src/erasure-code/jerasure/Makefile.am
@@ -0,0 +1,28 @@
+# jerasure plugin
+libec_jerasure_la_SOURCES = \
+ erasure-code/jerasure/ErasureCodePluginJerasure.cc \
+ erasure-code/jerasure/ErasureCodeJerasure.cc \
+ erasure-code/jerasure/cauchy.c \
+ erasure-code/jerasure/galois.c \
+ erasure-code/jerasure/jerasure.c \
+ erasure-code/jerasure/liberation.c \
+ erasure-code/jerasure/reed_sol.c
+
+noinst_HEADERS += \
+ erasure-code/jerasure/ErasureCodeJerasure.h \
+ erasure-code/jerasure/cauchy.h \
+ erasure-code/jerasure/galois.h \
+ erasure-code/jerasure/jerasure.h \
+ erasure-code/jerasure/liberation.h \
+ erasure-code/jerasure/reed_sol.h \
+ erasure-code/jerasure/vectorop.h
+
+libec_jerasure_la_CFLAGS = ${AM_CFLAGS}
+libec_jerasure_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_jerasure_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0
+if LINUX
+libec_jerasure_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
+endif
+
+erasure_codelib_LTLIBRARIES += libec_jerasure.la
diff --git a/src/osd/ErasureCodePluginJerasure/cauchy.c b/src/erasure-code/jerasure/cauchy.c
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/cauchy.c
rename to src/erasure-code/jerasure/cauchy.c
diff --git a/src/osd/ErasureCodePluginJerasure/cauchy.h b/src/erasure-code/jerasure/cauchy.h
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/cauchy.h
rename to src/erasure-code/jerasure/cauchy.h
diff --git a/src/osd/ErasureCodePluginJerasure/galois.c b/src/erasure-code/jerasure/galois.c
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/galois.c
rename to src/erasure-code/jerasure/galois.c
diff --git a/src/osd/ErasureCodePluginJerasure/galois.h b/src/erasure-code/jerasure/galois.h
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/galois.h
rename to src/erasure-code/jerasure/galois.h
diff --git a/src/osd/ErasureCodePluginJerasure/jerasure.c b/src/erasure-code/jerasure/jerasure.c
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/jerasure.c
rename to src/erasure-code/jerasure/jerasure.c
diff --git a/src/osd/ErasureCodePluginJerasure/jerasure.h b/src/erasure-code/jerasure/jerasure.h
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/jerasure.h
rename to src/erasure-code/jerasure/jerasure.h
diff --git a/src/osd/ErasureCodePluginJerasure/liberation.c b/src/erasure-code/jerasure/liberation.c
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/liberation.c
rename to src/erasure-code/jerasure/liberation.c
diff --git a/src/osd/ErasureCodePluginJerasure/liberation.h b/src/erasure-code/jerasure/liberation.h
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/liberation.h
rename to src/erasure-code/jerasure/liberation.h
diff --git a/src/osd/ErasureCodePluginJerasure/reed_sol.c b/src/erasure-code/jerasure/reed_sol.c
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/reed_sol.c
rename to src/erasure-code/jerasure/reed_sol.c
diff --git a/src/osd/ErasureCodePluginJerasure/reed_sol.h b/src/erasure-code/jerasure/reed_sol.h
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/reed_sol.h
rename to src/erasure-code/jerasure/reed_sol.h
diff --git a/src/osd/ErasureCodePluginJerasure/vectorop.h b/src/erasure-code/jerasure/vectorop.h
similarity index 100%
rename from src/osd/ErasureCodePluginJerasure/vectorop.h
rename to src/erasure-code/jerasure/vectorop.h
diff --git a/src/global/global_init.cc b/src/global/global_init.cc
index e96c317..5e56e8f 100644
--- a/src/global/global_init.cc
+++ b/src/global/global_init.cc
@@ -180,10 +180,11 @@ void global_init_daemonize(CephContext *cct, int flags)
exit(1);
}
- global_init_postfork(cct, flags);
+ global_init_postfork_start(cct);
+ global_init_postfork_finish(cct, flags);
}
-void global_init_postfork(CephContext *cct, int flags)
+void global_init_postfork_start(CephContext *cct)
{
// restart log thread
g_ceph_context->_log->start();
@@ -215,6 +216,16 @@ void global_init_postfork(CephContext *cct, int flags)
<< err << dendl;
exit(1);
}
+
+ pidfile_write(g_conf);
+}
+
+void global_init_postfork_finish(CephContext *cct, int flags)
+{
+ /* We only close stderr once the caller decides the daemonization
+ * process is finished. This way we can allow error messages to be
+ * propagated in a manner that the user is able to see.
+ */
if (!(flags & CINIT_FLAG_NO_CLOSE_STDERR)) {
int ret = global_init_shutdown_stderr(cct);
if (ret) {
@@ -223,10 +234,10 @@ void global_init_postfork(CephContext *cct, int flags)
exit(1);
}
}
- pidfile_write(g_conf);
ldout(cct, 1) << "finished global_init_daemonize" << dendl;
}
+
void global_init_chdir(const CephContext *cct)
{
const md_config_t *conf = cct->_conf;
diff --git a/src/global/global_init.h b/src/global/global_init.h
index d2ba6ef..67586e4 100644
--- a/src/global/global_init.h
+++ b/src/global/global_init.h
@@ -41,10 +41,16 @@ void global_init(std::vector < const char * > *alt_def_args, std::vector < const
int global_init_prefork(CephContext *cct, int flags);
/*
- * perform all of the steps that global_init_daemonize performs just after
- * the fork.
+ * perform all the steps that global_init_daemonize performs just after
+ * the fork, except closing stderr, which we'll do later on.
*/
-void global_init_postfork(CephContext *cct, int flags);
+void global_init_postfork_start(CephContext *cct);
+
+/*
+ * close stderr, thus completing the postfork.
+ */
+void global_init_postfork_finish(CephContext *cct, int flags);
+
/*
* global_init_daemonize handles daemonizing a process.
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index 62cd62c..20cac9a 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -45,7 +45,6 @@ noinst_HEADERS += \
include/filepath.h \
include/frag.h \
include/hash.h \
- include/histogram.h \
include/intarith.h \
include/interval_set.h \
include/int_types.h \
diff --git a/src/include/buffer.h b/src/include/buffer.h
index 4c275ca..5491105 100644
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -448,6 +448,7 @@ public:
public:
hash() : crc(0) { }
+ hash(uint32_t init) : crc(init) { }
void update(buffer::list& bl) {
crc = bl.crc32c(crc);
diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index fc9b63d..f2b8a85 100644
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -48,6 +48,8 @@
this bit to determine if peers support NAK messages. */
#define CEPH_FEATURE_OSDMAP_ENC (1ULL<<39)
#define CEPH_FEATURE_MDS_INLINE_DATA (1ULL<<40)
+#define CEPH_FEATURE_CRUSH_TUNABLES3 (1ULL<<41)
+#define CEPH_FEATURE_OSD_PRIMARY_AFFINITY (1ULL<<41) /* overlap w/ tunables3 */
/*
* The introduction of CEPH_FEATURE_OSD_SNAPMAPPER caused the feature
@@ -116,6 +118,8 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) {
CEPH_FEATURE_OSD_ERASURE_CODES | \
CEPH_FEATURE_OSDMAP_ENC | \
CEPH_FEATURE_MDS_INLINE_DATA | \
+ CEPH_FEATURE_CRUSH_TUNABLES3 | \
+ CEPH_FEATURE_OSD_PRIMARY_AFFINITY | \
0ULL)
#define CEPH_FEATURES_SUPPORTED_DEFAULT CEPH_FEATURES_ALL
@@ -126,6 +130,7 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) {
#define CEPH_FEATURES_CRUSH \
(CEPH_FEATURE_CRUSH_TUNABLES | \
CEPH_FEATURE_CRUSH_TUNABLES2 | \
+ CEPH_FEATURE_CRUSH_TUNABLES3 | \
CEPH_FEATURE_CRUSH_V2)
#endif
diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 003f03c..10b52a5 100644
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -310,6 +310,7 @@ enum {
CEPH_MDS_OP_LOOKUPHASH = 0x00102,
CEPH_MDS_OP_LOOKUPPARENT = 0x00103,
CEPH_MDS_OP_LOOKUPINO = 0x00104,
+ CEPH_MDS_OP_LOOKUPNAME = 0x00105,
CEPH_MDS_OP_SETXATTR = 0x01105,
CEPH_MDS_OP_RMXATTR = 0x01106,
@@ -338,6 +339,7 @@ enum {
// internal op
CEPH_MDS_OP_FRAGMENTDIR= 0x01500,
+ CEPH_MDS_OP_EXPORTDIR = 0x01501,
};
extern const char *ceph_mds_op_name(int op);
@@ -354,8 +356,9 @@ extern const char *ceph_mds_op_name(int op);
/*
* Ceph setxattr request flags.
*/
-#define CEPH_XATTR_CREATE 1
-#define CEPH_XATTR_REPLACE 2
+#define CEPH_XATTR_CREATE (1 << 0)
+#define CEPH_XATTR_REPLACE (1 << 1)
+#define CEPH_XATTR_REMOVE (1 << 31)
union ceph_mds_request_args {
struct {
@@ -401,8 +404,8 @@ union ceph_mds_request_args {
struct {
__u8 rule; /* currently fcntl or flock */
__u8 type; /* shared, exclusive, remove*/
+ __le64 owner; /* who requests/holds the lock */
__le64 pid; /* process id requesting the lock */
- __le64 pid_namespace;
__le64 start; /* initial location to lock */
__le64 length; /* num bytes to lock from start */
__u8 wait; /* will caller wait for lock to become available? */
@@ -513,8 +516,8 @@ struct ceph_filelock {
__le64 start;/* file offset to start lock at */
__le64 length; /* num bytes to lock; 0 for all following start */
__le64 client; /* which client holds the lock */
+ __le64 owner; /* who requests/holds the lock */
__le64 pid; /* process id holding the lock on the client */
- __le64 pid_namespace;
__u8 type; /* shared lock, exclusive lock, or unlock */
} __attribute__ ((packed));
@@ -596,6 +599,8 @@ int ceph_flags_to_mode(int flags);
CEPH_CAP_LINK_SHARED | \
CEPH_CAP_FILE_SHARED | \
CEPH_CAP_XATTR_SHARED)
+#define CEPH_STAT_CAP_INLINE_DATA (CEPH_CAP_FILE_SHARED | \
+ CEPH_CAP_FILE_RD)
#define CEPH_CAP_ANY_SHARED (CEPH_CAP_AUTH_SHARED | \
CEPH_CAP_LINK_SHARED | \
diff --git a/src/include/cephfs/libcephfs.h b/src/include/cephfs/libcephfs.h
index 1370021..e600605 100644
--- a/src/include/cephfs/libcephfs.h
+++ b/src/include/cephfs/libcephfs.h
@@ -20,6 +20,8 @@
#include <sys/types.h>
#include <sys/statvfs.h>
#include <sys/socket.h>
+#include <stdint.h>
+#include <stdbool.h>
// FreeBSD compatibility
#ifdef __FreeBSD__
@@ -39,6 +41,56 @@ extern "C" {
# error libceph: must define __USE_FILE_OFFSET64 or readdir results will be corrupted
#endif
+/*
+ * XXXX redeclarations from ceph_fs.h, rados.h, etc. We need more of this
+ * in the interface, but shouldn't be re-typing it (and using different
+ * C data types).
+ */
+#ifndef __cplusplus
+
+#define CEPH_INO_ROOT 1
+#define CEPH_NOSNAP ((uint64_t)(-2))
+
+struct ceph_file_layout {
+ /* file -> object mapping */
+ uint32_t fl_stripe_unit; /* stripe unit, in bytes. must be multiple
+ of page size. */
+ uint32_t fl_stripe_count; /* over this many objects */
+ uint32_t fl_object_size; /* until objects are this big, then move to
+ new objects */
+ uint32_t fl_cas_hash; /* 0 = none; 1 = sha256 */
+
+ /* pg -> disk layout */
+ uint32_t fl_object_stripe_unit; /* for per-object parity, if any */
+
+ /* object -> pg layout */
+ uint32_t fl_pg_preferred; /* preferred primary for pg (-1 for none) */
+ uint32_t fl_pg_pool; /* namespace, crush ruleset, rep level */
+} __attribute__ ((packed));
+
+
+typedef struct _inodeno_t {
+ uint64_t val;
+} inodeno_t;
+
+typedef struct _snapid_t {
+ uint64_t val;
+} snapid_t;
+
+typedef struct vinodeno_t {
+ inodeno_t ino;
+ snapid_t snapid;
+} vinodeno_t;
+
+typedef struct Fh Fh;
+
+#endif /* ! __cplusplus */
+
+struct Inode;
+typedef struct Inode Inode;
+
+struct vinodeno_t;
+typedef struct vinodeno_t vinodeno;
struct ceph_mount_info;
struct ceph_dir_result;
struct CephContext;
@@ -445,7 +497,7 @@ int ceph_link(struct ceph_mount_info *cmount, const char *existing, const char *
* @param path the path to the symlink to read
* @param buf the buffer to hold the the path of the file that the symlink points to.
* @param size the length of the buffer
- * @returns 0 on success or negative error code on failure
+ * @returns number of bytes copied on success or negative error code on failure
*/
int ceph_readlink(struct ceph_mount_info *cmount, const char *path, char *buf, int64_t size);
@@ -1135,24 +1187,146 @@ int ceph_get_local_osd(struct ceph_mount_info *cmount);
/**
* Get the capabilities currently issued to the client.
- *
+ *
* @param cmount the ceph mount handle to use.
* @param fd the file descriptor to get issued
* @returns the current capabilities issued to this client
- * for the open file
+ * for the open file
*/
int ceph_debug_get_fd_caps(struct ceph_mount_info *cmount, int fd);
/**
* Get the capabilities currently issued to the client.
- *
+ *
* @param cmount the ceph mount handle to use.
* @param the path to the file
* @returns the current capabilities issued to this client
- * for the file
+ * for the file
*/
int ceph_debug_get_file_caps(struct ceph_mount_info *cmount, const char *path);
+/* Low Level */
+struct Inode *ceph_ll_get_inode(struct ceph_mount_info *cmount,
+ vinodeno_t vino);
+/**
+ * Get the root inode of FS. Increase counter of references for root Inode. You must call ceph_ll_forget for it!
+ *
+ * @param cmount the ceph mount handle to use.
+ * @param parent pointer to pointer to Inode struct. Pointer to root inode will be returned
+ * @returns 0 if all good
+ */
+int ceph_ll_lookup_root(struct ceph_mount_info *cmount,
+ Inode **parent);
+int ceph_ll_lookup(struct ceph_mount_info *cmount, struct Inode *parent,
+ const char *name, struct stat *attr,
+ Inode **out, int uid, int gid);
+int ceph_ll_put(struct ceph_mount_info *cmount, struct Inode *in);
+int ceph_ll_forget(struct ceph_mount_info *cmount, struct Inode *in,
+ int count);
+int ceph_ll_walk(struct ceph_mount_info *cmount, const char *name,
+ struct Inode **i,
+ struct stat *attr);
+int ceph_ll_getattr(struct ceph_mount_info *cmount, struct Inode *in,
+ struct stat *attr, int uid, int gid);
+int ceph_ll_setattr(struct ceph_mount_info *cmount, struct Inode *in,
+ struct stat *st, int mask, int uid, int gid);
+int ceph_ll_open(struct ceph_mount_info *cmount, struct Inode *in, int flags,
+ struct Fh **fh, int uid, int gid);
+loff_t ceph_ll_lseek(struct ceph_mount_info *cmount, struct Fh* filehandle,
+ loff_t offset, int whence);
+int ceph_ll_read(struct ceph_mount_info *cmount, struct Fh* filehandle,
+ int64_t off, uint64_t len, char* buf);
+int ceph_ll_fsync(struct ceph_mount_info *cmount, struct Fh *fh,
+ int syncdataonly);
+int ceph_ll_write(struct ceph_mount_info *cmount, struct Fh* filehandle,
+ int64_t off, uint64_t len, const char *data);
+int64_t ceph_ll_readv(struct ceph_mount_info *cmount, struct Fh *fh,
+ const struct iovec *iov, int iovcnt, int64_t off);
+int64_t ceph_ll_writev(struct ceph_mount_info *cmount, struct Fh *fh,
+ const struct iovec *iov, int iovcnt, int64_t off);
+int ceph_ll_close(struct ceph_mount_info *cmount, struct Fh* filehandle);
+int ceph_ll_iclose(struct ceph_mount_info *cmount, struct Inode *in, int mode);
+/**
+ * Get xattr value by xattr name.
+ *
+ * @param cmount the ceph mount handle to use.
+ * @param in file handle
+ * @param name name of attribute
+ * @param value pointer to begin buffer
+ * @param size buffer size
+ * @param uid user ID
+ * @param gid group ID
+ * @returns size of returned buffer. Negative number in error case
+ */
+int ceph_ll_getxattr(struct ceph_mount_info *cmount, struct Inode *in,
+ const char *name, void *value, size_t size, int uid,
+ int gid);
+int ceph_ll_setxattr(struct ceph_mount_info *cmount, struct Inode *in,
+ const char *name, const void *value, size_t size,
+ int flags, int uid, int gid);
+int ceph_ll_listxattr(struct ceph_mount_info *cmount, struct Inode *in,
+ char *list, size_t buf_size, size_t *list_size, int uid, int gid);
+int ceph_ll_removexattr(struct ceph_mount_info *cmount, struct Inode *in,
+ const char *name, int uid, int gid);
+int ceph_ll_create(struct ceph_mount_info *cmount, struct Inode *parent,
+ const char *name, mode_t mode, int flags,
+ struct stat *attr, struct Inode **out, Fh **fhp,
+ int uid, int gid);
+int ceph_ll_mkdir(struct ceph_mount_info *cmount, struct Inode *parent,
+ const char *name, mode_t mode, struct stat *attr,
+ Inode **out, int uid, int gid);
+int ceph_ll_link(struct ceph_mount_info *cmount, struct Inode *in,
+ struct Inode *newparrent, const char *name,
+ struct stat *attr, int uid, int gid);
+int ceph_ll_truncate(struct ceph_mount_info *cmount, struct Inode *in,
+ uint64_t length, int uid, int gid);
+int ceph_ll_opendir(struct ceph_mount_info *cmount, struct Inode *in,
+ struct ceph_dir_result **dirpp, int uid, int gid);
+int ceph_ll_releasedir(struct ceph_mount_info *cmount,
+ struct ceph_dir_result* dir);
+int ceph_ll_rename(struct ceph_mount_info *cmount, struct Inode *parent,
+ const char *name, struct Inode *newparent,
+ const char *newname, int uid, int gid);
+int ceph_ll_unlink(struct ceph_mount_info *cmount, struct Inode *in,
+ const char *name, int uid, int gid);
+int ceph_ll_statfs(struct ceph_mount_info *cmount, struct Inode *in,
+ struct statvfs *stbuf);
+int ceph_ll_readlink(struct ceph_mount_info *cmount, struct Inode *in,
+ char *buf, size_t bufsize, int uid, int gid);
+int ceph_ll_symlink(struct ceph_mount_info *cmount, struct Inode *parent,
+ const char *name, const char *value, struct stat *attr,
+ struct Inode **in, int uid, int gid);
+int ceph_ll_rmdir(struct ceph_mount_info *cmount, struct Inode *in,
+ const char *name, int uid, int gid);
+uint32_t ceph_ll_stripe_unit(struct ceph_mount_info *cmount,
+ struct Inode *in);
+uint32_t ceph_ll_file_layout(struct ceph_mount_info *cmount,
+ struct Inode *in,
+ struct ceph_file_layout *layout);
+uint64_t ceph_ll_snap_seq(struct ceph_mount_info *cmount,
+ struct Inode *in);
+int ceph_ll_get_stripe_osd(struct ceph_mount_info *cmount,
+ struct Inode *in,
+ uint64_t blockno,
+ struct ceph_file_layout* layout);
+int ceph_ll_num_osds(struct ceph_mount_info *cmount);
+int ceph_ll_osdaddr(struct ceph_mount_info *cmount,
+ int osd, uint32_t *addr);
+uint64_t ceph_ll_get_internal_offset(struct ceph_mount_info *cmount,
+ struct Inode *in, uint64_t blockno);
+int ceph_ll_read_block(struct ceph_mount_info *cmount,
+ struct Inode *in, uint64_t blockid,
+ char* bl, uint64_t offset, uint64_t length,
+ struct ceph_file_layout* layout);
+int ceph_ll_write_block(struct ceph_mount_info *cmount,
+ struct Inode *in, uint64_t blockid,
+ char* buf, uint64_t offset,
+ uint64_t length, struct ceph_file_layout* layout,
+ uint64_t snapseq, uint32_t sync);
+int ceph_ll_commit_blocks(struct ceph_mount_info *cmount,
+ struct Inode *in, uint64_t offset, uint64_t range);
+
+
#ifdef __cplusplus
}
#endif
diff --git a/src/include/cmp.h b/src/include/cmp.h
index d660c19..92ecef4 100644
--- a/src/include/cmp.h
+++ b/src/include/cmp.h
@@ -1,7 +1,5 @@
#ifndef __CEPH_CMP_H
#define __CEPH_CMP_H
-#include <boost/tuple/tuple.hpp>
-#include <boost/tuple/tuple_comparison.hpp>
/*
* macros to define comparison operators for classes with small numbers of members.
@@ -104,30 +102,48 @@
(l.d == r.d && l.e <= r.e))))))); \
}
-#define WRITE_EQ_OPERATORS_7(type, a, b, c, d, e, f, g) \
- inline bool operator==(const type &l, const type &r) { \
- return (boost::make_tuple(boost::cref(l.a), boost::cref(l.b), boost::cref(l.c), boost::cref(l.d), boost::cref(l.e), boost::cref(l.f), boost::cref(l.g)) == \
- boost::make_tuple(boost::cref(r.a), boost::cref(r.b), boost::cref(r.c), boost::cref(r.d), boost::cref(r.e), boost::cref(r.f), boost::cref(r.g))); \
- } \
- inline bool operator!=(const type &l, const type &r) { \
- return (boost::make_tuple(boost::cref(l.a), boost::cref(l.b), boost::cref(l.c), boost::cref(l.d), boost::cref(l.e), boost::cref(l.f), boost::cref(l.g)) != \
- boost::make_tuple(boost::cref(r.a), boost::cref(r.b), boost::cref(r.c), boost::cref(r.d), boost::cref(r.e), boost::cref(r.f), boost::cref(r.g))); \
- }
-#define WRITE_CMP_OPERATORS_7(type, a, b, c, d, e, f, g) \
- inline bool operator<=(const type &l, const type &r) { \
- return (boost::make_tuple(boost::cref(l.a), boost::cref(l.b), boost::cref(l.c), boost::cref(l.d), boost::cref(l.e), boost::cref(l.f), boost::cref(l.g)) <= \
- boost::make_tuple(boost::cref(r.a), boost::cref(r.b), boost::cref(r.c), boost::cref(r.d), boost::cref(r.e), boost::cref(r.f), boost::cref(r.g))); \
- } \
- inline bool operator>=(const type &l, const type &r) { \
- return (boost::make_tuple(boost::cref(l.a), boost::cref(l.b), boost::cref(l.c), boost::cref(l.d), boost::cref(l.e), boost::cref(l.f), boost::cref(l.g)) >= \
- boost::make_tuple(boost::cref(r.a), boost::cref(r.b), boost::cref(r.c), boost::cref(r.d), boost::cref(r.e), boost::cref(r.f), boost::cref(r.g))); \
- } \
- inline bool operator>(const type &l, const type &r) { \
- return (boost::make_tuple(boost::cref(l.a), boost::cref(l.b), boost::cref(l.c), boost::cref(l.d), boost::cref(l.e), boost::cref(l.f), boost::cref(l.g)) > \
- boost::make_tuple(boost::cref(r.a), boost::cref(r.b), boost::cref(r.c), boost::cref(r.d), boost::cref(r.e), boost::cref(r.f), boost::cref(r.g))); \
- } \
- inline bool operator<(const type &l, const type &r) { \
- return (boost::make_tuple(boost::cref(l.a), boost::cref(l.b), boost::cref(l.c), boost::cref(l.d), boost::cref(l.e), boost::cref(l.f), boost::cref(l.g)) < \
- boost::make_tuple(boost::cref(r.a), boost::cref(r.b), boost::cref(r.c), boost::cref(r.d), boost::cref(r.e), boost::cref(r.f), boost::cref(r.g))); \
- }
+#define WRITE_EQ_OPERATORS_7(type, a, b, c, d, e, f, g) \
+ inline bool operator==(const type &l, const type &r) { \
+ return l.a == r.a && l.b == r.b && l.c == r.c && l.d == r.d && l.e == r.e && l.f == r.f && l.g == r.g; \
+ } \
+ inline bool operator!=(const type &l, const type &r) { \
+ return l.a != r.a || l.b != r.b || l.c != r.c || l.d != r.d || l.e != r.e || l.f != r.f || l.g != r.g; \
+ }
+#define WRITE_CMP_OPERATORS_7(type, a, b, c, d, e, f, g) \
+ inline bool operator<=(const type &l, const type &r) { \
+ return l.a < r.a || \
+ (l.a == r.a && (l.b < r.b || \
+ (l.b == r.b && (l.c < r.c || \
+ (l.c == r.c && (l.d < r.d || \
+ (l.d == r.d && (l.e < r.e || \
+ (l.e == r.e && (l.f < r.f || \
+ (l.f == r.f && l.g <= r.g))))))))))); \
+ } \
+ inline bool operator>=(const type &l, const type &r) { \
+ return l.a > r.a || \
+ (l.a == r.a && (l.b > r.b || \
+ (l.b == r.b && (l.c > r.c || \
+ (l.c == r.c && (l.d > r.d || \
+ (l.d == r.d && (l.e > r.e || \
+ (l.e == r.e && (l.f > r.f || \
+ (l.f == r.f && l.g >= r.g))))))))))); \
+ } \
+ inline bool operator>(const type &l, const type &r) { \
+ return l.a > r.a || \
+ (l.a == r.a && (l.b > r.b || \
+ (l.b == r.b && (l.c > r.c || \
+ (l.c == r.c && (l.d > r.d || \
+ (l.d == r.d && (l.e > r.e || \
+ (l.e == r.e && (l.f > r.f || \
+ (l.f == r.f && l.g > r.g))))))))))); \
+ } \
+ inline bool operator<(const type &l, const type &r) { \
+ return l.a < r.a || \
+ (l.a == r.a && (l.b < r.b || \
+ (l.b == r.b && (l.c < r.c || \
+ (l.c == r.c && (l.d < r.d || \
+ (l.d == r.d && (l.e < r.e || \
+ (l.e == r.e && (l.f < r.f || \
+ (l.f == r.f && l.g < r.g))))))))))); \
+ }
#endif
diff --git a/src/include/encoding.h b/src/include/encoding.h
index ddb94ec..d097482 100644
--- a/src/include/encoding.h
+++ b/src/include/encoding.h
@@ -320,6 +320,12 @@ inline void decode(boost::optional<T> &p, bufferlist::iterator &bp)
// pair
template<class A, class B>
+inline void encode(const std::pair<A,B> &p, bufferlist &bl, uint64_t features)
+{
+ encode(p.first, bl, features);
+ encode(p.second, bl, features);
+}
+template<class A, class B>
inline void encode(const std::pair<A,B> &p, bufferlist &bl)
{
encode(p.first, bl);
diff --git a/src/include/rados.h b/src/include/rados.h
index 59cc77b..49391d9 100644
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -121,6 +121,9 @@ extern const char *ceph_osd_state_name(int s);
#define CEPH_OSD_IN 0x10000
#define CEPH_OSD_OUT 0
+#define CEPH_OSD_MAX_PRIMARY_AFFINITY 0x10000
+#define CEPH_OSD_DEFAULT_PRIMARY_AFFINITY 0x10000
+
/*
* osd map flag bits
@@ -138,6 +141,7 @@ extern const char *ceph_osd_state_name(int s);
#define CEPH_OSDMAP_NORECOVER (1<<10) /* block osd recovery and backfill */
#define CEPH_OSDMAP_NOSCRUB (1<<11) /* block periodic scrub */
#define CEPH_OSDMAP_NODEEP_SCRUB (1<<12) /* block periodic deep-scrub */
+#define CEPH_OSDMAP_NOTIERAGENT (1<<13) /* disable tiering agent */
/*
* The error code to return when an OSD can't handle a write
@@ -238,6 +242,9 @@ enum {
/* convert tmap to omap */
CEPH_OSD_OP_TMAP2OMAP = CEPH_OSD_OP_MODE_RMW | CEPH_OSD_OP_TYPE_DATA | 34,
+ /* hints */
+ CEPH_OSD_OP_SETALLOCHINT = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 35,
+
/** multi **/
CEPH_OSD_OP_CLONERANGE = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_MULTI | 1,
CEPH_OSD_OP_ASSERT_SRC_VERSION = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_MULTI | 2,
@@ -377,7 +384,6 @@ enum {
/* xattr comparison */
enum {
- CEPH_OSD_CMPXATTR_OP_NOP = 0,
CEPH_OSD_CMPXATTR_OP_EQ = 1,
CEPH_OSD_CMPXATTR_OP_NE = 2,
CEPH_OSD_CMPXATTR_OP_GT = 3,
@@ -460,6 +466,10 @@ struct ceph_osd_op {
struct {
__u8 flags;
} __attribute__ ((packed)) tmap2omap;
+ struct {
+ __le64 expected_object_size;
+ __le64 expected_write_size;
+ } __attribute__ ((packed)) alloc_hint;
};
__le32 payload_len;
} __attribute__ ((packed));
diff --git a/src/include/rados/buffer.h b/src/include/rados/buffer.h
index 4c275ca..5491105 100644
--- a/src/include/rados/buffer.h
+++ b/src/include/rados/buffer.h
@@ -448,6 +448,7 @@ public:
public:
hash() : crc(0) { }
+ hash(uint32_t init) : crc(init) { }
void update(buffer::list& bl) {
crc = bl.crc32c(crc);
diff --git a/src/include/rados/librados.h b/src/include/rados/librados.h
index 7dd1748..f0d45af 100644
--- a/src/include/rados/librados.h
+++ b/src/include/rados/librados.h
@@ -39,7 +39,7 @@ extern "C" {
#endif
#define LIBRADOS_VER_MAJOR 0
-#define LIBRADOS_VER_MINOR 68
+#define LIBRADOS_VER_MINOR 69
#define LIBRADOS_VER_EXTRA 0
#define LIBRADOS_VERSION(maj, min, extra) ((maj << 16) + (min << 8) + extra)
@@ -53,16 +53,31 @@ extern "C" {
*/
#define LIBRADOS_LOCK_FLAG_RENEW 0x1
+/*
+ * Constants for rados_write_op_create().
+ */
#define LIBRADOS_CREATE_EXCLUSIVE 1
#define LIBRADOS_CREATE_IDEMPOTENT 0
+/*
+ * Flags that can be set on a per-op basis via
+ * rados_read_op_set_flags() and rados_write_op_set_flags().
+ */
+// fail a create operation if the object already exists
+#define LIBRADOS_OP_FLAG_EXCL 1
+// allow the transaction to succeed even if the flagged op fails
+#define LIBRADOS_OP_FLAG_FAILOK 2
+
/**
* @defgroup librados_h_xattr_comp xattr comparison operations
+ * Operators for comparing xattrs on objects, and aborting the
+ * rados_read_op or rados_write_op transaction if the comparison
+ * fails.
+ *
* @{
*/
/** @cond TODO_enums_not_yet_in_asphyxiate */
enum {
- LIBRADOS_CMPXATTR_OP_NOP = 0,
LIBRADOS_CMPXATTR_OP_EQ = 1,
LIBRADOS_CMPXATTR_OP_NE = 2,
LIBRADOS_CMPXATTR_OP_GT = 3,
@@ -73,6 +88,26 @@ enum {
/** @endcond */
/** @} */
+/**
+ * @defgroup librados_h_operation_flags
+ * Flags for rados_read_op_opeprate(), rados_write_op_operate(),
+ * rados_aio_read_op_operate(), and rados_aio_write_op_operate().
+ * See librados.hpp for details.
+ * @{
+ */
+/** @cond TODO_enums_not_yet_in_asphyxiate */
+enum {
+ LIBRADOS_OPERATION_NOFLAG = 0,
+ LIBRADOS_OPERATION_BALANCE_READS = 1,
+ LIBRADOS_OPERATION_LOCALIZE_READS = 2,
+ LIBRADOS_OPERATION_ORDER_READS_WRITES = 4,
+ LIBRADOS_OPERATION_IGNORE_CACHE = 8,
+ LIBRADOS_OPERATION_SKIPRWLOCKS = 16,
+ LIBRADOS_OPERATION_IGNORE_OVERLAY = 32,
+};
+/** @endcond */
+/** @} */
+
/*
* snap id contants
*/
@@ -146,6 +181,15 @@ typedef uint64_t rados_snap_t;
typedef void *rados_xattrs_iter_t;
/**
+ * @typedef rados_omap_iter_t
+ * An iterator for listing omap key/value pairs on an object.
+ * Used with rados_read_op_omap_get_keys(), rados_read_op_omap_get_vals(),
+ * rados_read_op_omap_get_vals_by_keys(), rados_omap_get_next(), and
+ * rados_omap_get_end().
+ */
+typedef void *rados_omap_iter_t;
+
+/**
* @struct rados_pool_stat_t
* Usage information for a pool.
*/
@@ -190,15 +234,39 @@ struct rados_cluster_stat_t {
* - Extended attribute manipulation: rados_write_op_cmpxattr()
* rados_write_op_cmpxattr(), rados_write_op_setxattr(),
* rados_write_op_rmxattr()
+ * - Object map key/value pairs: rados_write_op_omap_set(),
+ * rados_write_op_omap_rm_keys(), rados_write_op_omap_clear(),
+ * rados_write_op_omap_cmp()
* - Creating objects: rados_write_op_create()
* - IO on objects: rados_write_op_append(), rados_write_op_write(), rados_write_op_zero
* rados_write_op_write_full(), rados_write_op_remove, rados_write_op_truncate(),
* rados_write_op_zero()
+ * - Hints: rados_write_op_set_alloc_hint()
* - Performing the operation: rados_write_op_operate(), rados_aio_write_op_operate()
*/
typedef void *rados_write_op_t;
/**
+ * @typedef rados_read_op_t
+ *
+ * An object read operation stores a number of operations which can be
+ * executed atomically. For usage, see:
+ * - Creation and deletion: rados_create_read_op() rados_release_read_op()
+ * - Extended attribute manipulation: rados_read_op_cmpxattr(),
+ * rados_read_op_getxattr(), rados_read_op_getxattrs()
+ * - Object map key/value pairs: rados_read_op_omap_get_vals(),
+ * rados_read_op_omap_get_keys(), rados_read_op_omap_get_vals_by_keys(),
+ * rados_read_op_omap_cmp()
+ * - Object properties: rados_read_op_stat(), rados_read_op_assert_exists()
+ * - IO on objects: rados_read_op_read()
+ * - Custom operations: rados_read_op_exec(), rados_read_op_exec_user_buf()
+ * - Request properties: rados_read_op_set_flags()
+ * - Performing the operation: rados_read_op_operate(),
+ * rados_aio_read_op_operate()
+ */
+typedef void *rados_read_op_t;
+
+/**
* Get the version of librados.
*
* The version number is major.minor.extra. Note that this is
@@ -1163,6 +1231,37 @@ void rados_getxattrs_end(rados_xattrs_iter_t iter);
/** @} Xattrs */
/**
+ * Get the next omap key/value pair on the object
+ *
+ * @pre iter is a valid iterator
+ *
+ * @post key and val are the next key/value pair. key is
+ * null-terminated, and val has length len. If the end of the list has
+ * been reached, key and val are NULL, and len is 0. key and val will
+ * not be accessible after rados_omap_get_end() is called on iter, so
+ * if they are needed after that they should be copied.
+ *
+ * @param iter iterator to advance
+ * @param key where to store the key of the next omap entry
+ * @param val where to store the value of the next omap entry
+ * @param len where to store the number of bytes in val
+ * @returns 0 on success, negative error code on failure
+ */
+int rados_omap_get_next(rados_omap_iter_t iter,
+ char **key,
+ char **val,
+ size_t *len);
+
+/**
+ * Close the omap iterator.
+ *
+ * iter should not be used after this is called.
+ *
+ * @param iter the iterator to close
+ */
+void rados_omap_get_end(rados_omap_iter_t iter);
+
+/**
* Get object stats (size/mtime)
*
* TODO: when are these set, and by whom? can they be out of date?
@@ -1683,7 +1782,47 @@ int rados_unwatch(rados_ioctx_t io, const char *o, uint64_t handle);
*/
int rados_notify(rados_ioctx_t io, const char *o, uint64_t ver, const char *buf, int buf_len);
-/** @} Atomic write operations */
+/** @} Watch/Notify */
+
+/**
+ * @defgroup librados_h_hints Hints
+ *
+ * @{
+ */
+
+/**
+ * Set allocation hint for an object
+ *
+ * This is an advisory operation, it will always succeed (as if it was
+ * submitted with a LIBRADOS_OP_FLAG_FAILOK flag set) and is not
+ * guaranteed to do anything on the backend.
+ *
+ * @param io the pool the object is in
+ * @param o the name of the object
+ * @param expected_object_size expected size of the object, in bytes
+ * @param expected_write_size expected size of writes to the object, in bytes
+ * @returns 0 on success, negative error code on failure
+ */
+int rados_set_alloc_hint(rados_ioctx_t io, const char *o,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size);
+
+/** @} Hints */
+
+/**
+ * @defgroup librados_h_obj_op Object Operations
+ *
+ * A single rados operation can do multiple operations on one object
+ * atomicly. The whole operation will suceed or fail, and no partial
+ * results will be visible.
+ *
+ * Operations may be either reads, which can return data, or writes,
+ * which cannot. The effects of writes are applied and visible all at
+ * once, so an operation that sets an xattr and then checks its value
+ * will not see the updated value.
+ *
+ * @{
+ */
/**
* Create a new rados_write_op_t write operation. This will store all actions
@@ -1701,6 +1840,13 @@ rados_write_op_t rados_create_write_op();
void rados_release_write_op(rados_write_op_t write_op);
/**
+ * Set flags for the last operation added to this write_op.
+ * At least one op must have been added to the write_op.
+ * @param flags see librados.h constants beginning with LIBRADOS_OP_FLAG
+ */
+void rados_write_op_set_flags(rados_write_op_t write_op, int flags);
+
+/**
* Ensure that the object exists before writing
* @param write_op operation to add this action to
*/
@@ -1722,6 +1868,26 @@ void rados_write_op_cmpxattr(rados_write_op_t write_op,
size_t value_len);
/**
+ * Ensure that the an omap value satisfies a comparison,
+ * with the supplied value on the right hand side (i.e.
+ * for OP_LT, the comparison is actual_value < value.
+ *
+ * @param write_op operation to add this action to
+ * @param key which omap value to compare
+ * @param comparison_operator one of LIBRADOS_CMPXATTR_OP_EQ,
+ LIBRADOS_CMPXATTR_OP_LT, or LIBRADOS_CMPXATTR_OP_GT
+ * @param val value to compare with
+ * @param val_len length of value in bytes
+ * @param prval where to store the return value from this action
+ */
+void rados_write_op_omap_cmp(rados_write_op_t write_op,
+ const char *key,
+ uint8_t comparison_operator,
+ const char *val,
+ size_t val_len,
+ int *prval);
+
+/**
* Set an xattr
* @param write_op operation to add this action to
* @param name name of the xattr
@@ -1802,19 +1968,84 @@ void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset);
* @len length to zero
*/
void rados_write_op_zero(rados_write_op_t write_op,
- uint64_t offset,
- uint64_t len);
+ uint64_t offset,
+ uint64_t len);
+
+/**
+ * Execute an OSD class method on an object
+ * See rados_exec() for general description.
+ *
+ * @param write_op operation to add this action to
+ * @param cls the name of the class
+ * @param method the name of the method
+ * @param in_buf where to find input
+ * @param in_len length of in_buf in bytes
+ * @param prval where to store the return value from the method
+ */
+void rados_write_op_exec(rados_write_op_t write_op,
+ const char *cls,
+ const char *method,
+ const char *in_buf,
+ size_t in_len,
+ int *prval);
+
+/**
+ * Set key/value pairs on an object
+ *
+ * @param write_op operation to add this action to
+ * @param keys array of null-terminated char arrays representing keys to set
+ * @param vals array of pointers to values to set
+ * @param lens array of lengths corresponding to each value
+ * @param num number of key/value pairs to set
+ */
+void rados_write_op_omap_set(rados_write_op_t write_op,
+ char const* const* keys,
+ char const* const* vals,
+ const size_t *lens,
+ size_t num);
+
+/**
+ * Remove key/value pairs from an object
+ *
+ * @param write_op operation to add this action to
+ * @param keys array of null-terminated char arrays representing keys to remove
+ * @param keys_len number of key/value pairs to remove
+ */
+void rados_write_op_omap_rm_keys(rados_write_op_t write_op,
+ char const* const* keys,
+ size_t keys_len);
+
+/**
+ * Remove all key/value pairs from an object
+ *
+ * @param write_op operation to add this action to
+ */
+void rados_write_op_omap_clear(rados_write_op_t write_op);
+
+/**
+ * Set allocation hint for an object
+ *
+ * @param write_op operation to add this action to
+ * @param expected_object_size expected size of the object, in bytes
+ * @param expected_write_size expected size of writes to the object, in bytes
+ */
+void rados_write_op_set_alloc_hint(rados_write_op_t write_op,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size);
+
/**
* Perform a write operation synchronously
* @param write_op operation to perform
* @io the ioctx that the object is in
* @oid the object id
* @mtime the time to set the mtime to, NULL for the current time
+ * @flags flags to apply to the entire operation (LIBRADOS_OPERATION_*)
*/
int rados_write_op_operate(rados_write_op_t write_op,
- rados_ioctx_t io,
- const char *oid,
- time_t *mtime);
+ rados_ioctx_t io,
+ const char *oid,
+ time_t *mtime,
+ int flags);
/**
* Perform a write operation asynchronously
* @param write_op operation to perform
@@ -1822,15 +2053,258 @@ int rados_write_op_operate(rados_write_op_t write_op,
* @param completion what to do when operation has been attempted
* @oid the object id
* @mtime the time to set the mtime to, NULL for the current time
+ * @flags flags to apply to the entire operation (LIBRADOS_OPERATION_*)
*/
int rados_aio_write_op_operate(rados_write_op_t write_op,
rados_ioctx_t io,
rados_completion_t completion,
const char *oid,
- time_t *mtime);
+ time_t *mtime,
+ int flags);
+
+/**
+ * Create a new rados_read_op_t write operation. This will store all
+ * actions to be performed atomically. You must call
+ * rados_release_read_op when you are finished with it (after it
+ * completes, or you decide not to send it in the first place).
+ *
+ * @returns non-NULL on success, NULL on memory allocation error.
+ */
+rados_read_op_t rados_create_read_op();
+/**
+ * Free a rados_read_op_t, must be called when you're done with it.
+ * @param read_op operation to deallocate, created with rados_create_read_op
+ */
+void rados_release_read_op(rados_read_op_t read_op);
-/** @} Watch/Notify */
+/**
+ * Set flags for the last operation added to this read_op.
+ * At least one op must have been added to the read_op.
+ * @param flags see librados.h constants beginning with LIBRADOS_OP_FLAG
+ */
+void rados_read_op_set_flags(rados_read_op_t read_op, int flags);
+
+/**
+ * Ensure that the object exists before reading
+ * @param read_op operation to add this action to
+ */
+void rados_read_op_assert_exists(rados_read_op_t read_op);
+
+/**
+ * Ensure that the an xattr satisfies a comparison
+ * @param read_op operation to add this action to
+ * @param name name of the xattr to look up
+ * @param comparison_operator currently undocumented, look for
+ * LIBRADOS_CMPXATTR_OP_EQ in librados.h
+ * @param value buffer to compare actual xattr value to
+ * @param value_len length of buffer to compare actual xattr value to
+ */
+void rados_read_op_cmpxattr(rados_read_op_t read_op,
+ const char *name,
+ uint8_t comparison_operator,
+ const char *value,
+ size_t value_len);
+
+/**
+ * Start iterating over xattrs on an object.
+ *
+ * @param read_op operation to add this action to
+ * @param iter where to store the iterator
+ * @param prval where to store the return value of this action
+ */
+void rados_read_op_getxattrs(rados_read_op_t read_op,
+ rados_xattrs_iter_t *iter,
+ int *prval);
+
+/**
+ * Ensure that the an omap value satisfies a comparison,
+ * with the supplied value on the right hand side (i.e.
+ * for OP_LT, the comparison is actual_value < value.
+ *
+ * @param read_op operation to add this action to
+ * @param key which omap value to compare
+ * @param comparison_operator one of LIBRADOS_CMPXATTR_OP_EQ,
+ LIBRADOS_CMPXATTR_OP_LT, or LIBRADOS_CMPXATTR_OP_GT
+ * @param val value to compare with
+ * @param val_len length of value in bytes
+ * @param prval where to store the return value from this action
+ */
+void rados_read_op_omap_cmp(rados_read_op_t read_op,
+ const char *key,
+ uint8_t comparison_operator,
+ const char *val,
+ size_t val_len,
+ int *prval);
+
+/**
+ * Get object size and mtime
+ * @param read_op operation to add this action to
+ * @param psize where to store object size
+ * @param pmtime where to store modification time
+ * @param prval where to store the return value of this action
+ */
+void rados_read_op_stat(rados_read_op_t read_op,
+ uint64_t *psize,
+ time_t *pmtime,
+ int *prval);
+
+/**
+ * Read bytes from offset into buffer.
+ *
+ * prlen will be filled with the number of bytes read if successful.
+ * A short read can only occur if the read reaches the end of the
+ * object.
+ *
+ * @param read_op operation to add this action to
+ * @param offset offset to read from
+ * @param buffer where to put the data
+ * @param len length of buffer
+ * @param prval where to store the return value of this action
+ * @param bytes_read where to store the number of bytes read by this action
+ */
+void rados_read_op_read(rados_read_op_t read_op,
+ uint64_t offset,
+ size_t len,
+ char *buf,
+ size_t *bytes_read,
+ int *prval);
+
+/**
+ * Execute an OSD class method on an object
+ * See rados_exec() for general description.
+ *
+ * The output buffer is allocated on the heap; the caller is
+ * expected to release that memory with rados_buffer_free(). The
+ * buffer and length pointers can all be NULL, in which case they are
+ * not filled in.
+ *
+ * @param read_op operation to add this action to
+ * @param cls the name of the class
+ * @param method the name of the method
+ * @param in_buf where to find input
+ * @param in_len length of in_buf in bytes
+ * @param out_buf where to put librados-allocated output buffer
+ * @param out_len length of out_buf in bytes
+ * @param prval where to store the return value from the method
+ */
+void rados_read_op_exec(rados_read_op_t read_op,
+ const char *cls,
+ const char *method,
+ const char *in_buf,
+ size_t in_len,
+ char **out_buf,
+ size_t *out_len,
+ int *prval);
+
+/**
+ * Execute an OSD class method on an object
+ * See rados_exec() for general description.
+ *
+ * If the output buffer is too small, prval will
+ * be set to -ERANGE and used_len will be 0.
+ *
+ * @param read_op operation to add this action to
+ * @param cls the name of the class
+ * @param method the name of the method
+ * @param in_buf where to find input
+ * @param in_len length of in_buf in bytes
+ * @param out_buf user-provided buffer to read into
+ * @param out_len length of out_buf in bytes
+ * @param used_len where to store the number of bytes read into out_buf
+ * @param prval where to store the return value from the method
+ */
+void rados_read_op_exec_user_buf(rados_read_op_t read_op,
+ const char *cls,
+ const char *method,
+ const char *in_buf,
+ size_t in_len,
+ char *out_buf,
+ size_t out_len,
+ size_t *used_len,
+ int *prval);
+
+/**
+ * Start iterating over key/value pairs on an object.
+ *
+ * They will be returned sorted by key.
+ *
+ * @param read_op operation to add this action to
+ * @param start_after list keys starting after start_after
+ * @param filter_prefix list only keys beginning with filter_prefix
+ * @parem max_return list no more than max_return key/value pairs
+ * @param iter where to store the iterator
+ * @param prval where to store the return value from this action
+ */
+void rados_read_op_omap_get_vals(rados_read_op_t read_op,
+ const char *start_after,
+ const char *filter_prefix,
+ uint64_t max_return,
+ rados_omap_iter_t *iter,
+ int *prval);
+
+/**
+ * Start iterating over keys on an object.
+ *
+ * They will be returned sorted by key, and the iterator
+ * will fill in NULL for all values if specified.
+ *
+ * @param read_op operation to add this action to
+ * @param start_after list keys starting after start_after
+ * @parem max_return list no more than max_return keys
+ * @param iter where to store the iterator
+ * @param prval where to store the return value from this action
+ */
+void rados_read_op_omap_get_keys(rados_read_op_t read_op,
+ const char *start_after,
+ uint64_t max_return,
+ rados_omap_iter_t *iter,
+ int *prval);
+
+/**
+ * Start iterating over specific key/value pairs
+ *
+ * They will be returned sorted by key.
+ *
+ * @param read_op operation to add this action to
+ * @param keys array of pointers to null-terminated keys to get
+ * @param keys_len the number of strings in keys
+ * @param iter where to store the iterator
+ * @param prval where to store the return value from this action
+ */
+void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op,
+ char const* const* keys,
+ size_t keys_len,
+ rados_omap_iter_t *iter,
+ int *prval);
+
+/**
+ * Perform a write operation synchronously
+ * @param read_op operation to perform
+ * @io the ioctx that the object is in
+ * @oid the object id
+ * @flags flags to apply to the entire operation (LIBRADOS_OPERATION_*)
+ */
+int rados_read_op_operate(rados_read_op_t read_op,
+ rados_ioctx_t io,
+ const char *oid,
+ int flags);
+
+/**
+ * Perform a write operation asynchronously
+ * @param read_op operation to perform
+ * @io the ioctx that the object is in
+ * @param completion what to do when operation has been attempted
+ * @oid the object id
+ * @flags flags to apply to the entire operation (LIBRADOS_OPERATION_*)
+ */
+int rados_aio_read_op_operate(rados_read_op_t read_op,
+ rados_ioctx_t io,
+ rados_completion_t completion,
+ const char *oid,
+ int flags);
+
+/** @} Object Operations */
/**
* Take an exclusive lock on an object.
diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index 6b059fe..dde6273 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -64,6 +64,9 @@ namespace librados
ObjectIterator() {}
ObjectIterator(ObjListCtx *ctx_);
~ObjectIterator();
+ ObjectIterator(const ObjectIterator &rhs);
+ ObjectIterator& operator=(const ObjectIterator& rhs);
+
bool operator==(const ObjectIterator& rhs) const;
bool operator!=(const ObjectIterator& rhs) const;
const std::pair<std::string, std::string>& operator*() const;
@@ -124,8 +127,8 @@ namespace librados
* ops added to an ObjectOperation.
*/
enum ObjectOperationFlags {
- OP_EXCL = 1,
- OP_FAILOK = 2,
+ OP_EXCL = LIBRADOS_OP_FLAG_EXCL,
+ OP_FAILOK = LIBRADOS_OP_FLAG_FAILOK,
};
class ObjectOperationCompletion {
@@ -310,6 +313,15 @@ namespace librados
*/
void undirty();
+ /**
+ * Set allocation hint for an object
+ *
+ * @param expected_object_size expected size of the object, in bytes
+ * @param expected_write_size expected size of writes to the object, in bytes
+ */
+ void set_alloc_hint(uint64_t expected_object_size,
+ uint64_t expected_write_size);
+
friend class IoCtx;
};
@@ -500,6 +512,9 @@ namespace librados
std::string get_pool_name();
+ bool pool_requires_alignment();
+ uint64_t pool_required_alignment();
+
// create an object
int create(const std::string& oid, bool exclusive);
int create(const std::string& oid, bool exclusive, const std::string& category);
@@ -797,6 +812,22 @@ namespace librados
int list_snaps(const std::string& o, snap_set_t *out_snaps);
void set_notify_timeout(uint32_t timeout);
+ /**
+ * Set allocation hint for an object
+ *
+ * This is an advisory operation, it will always succeed (as if it
+ * was submitted with a OP_FAILOK flag set) and is not guaranteed
+ * to do anything on the backend.
+ *
+ * @param o the name of the object
+ * @param expected_object_size expected size of the object, in bytes
+ * @param expected_write_size expected size of writes to the object, in bytes
+ * @returns 0 on success, negative error code on failure
+ */
+ int set_alloc_hint(const std::string& o,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size);
+
// assert version for next sync operations
void set_assert_version(uint64_t ver);
void set_assert_src_version(const std::string& o, uint64_t ver);
diff --git a/src/init-ceph.in b/src/init-ceph.in
index 925047f..dac00d9 100644
--- a/src/init-ceph.in
+++ b/src/init-ceph.in
@@ -183,6 +183,7 @@ fi
for name in $what; do
type=`echo $name | cut -c 1-3` # e.g. 'mon', if $item is 'mon1'
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+ cluster=`echo $conf | awk -F'/' '{print $(NF)}' | cut -d'.' -f 1`
num=$id
name="$type.$id"
@@ -269,7 +270,7 @@ for name in $what; do
[ -n "$wrap" ] && runmode="-f &" && runarg="-f"
[ -n "$max_open_files" ] && files="ulimit -n $max_open_files;"
- cmd="$files $wrap $cmd $runmode"
+ cmd="$files $wrap $cmd --cluster $cluster $runmode"
if [ $dofsmount -eq 1 ] && [ -n "$fs_devs" ]; then
get_conf pre_mount "true" "pre mount command"
@@ -322,17 +323,9 @@ for name in $what; do
get_conf osd_location_hook "$BINDIR/ceph-crush-location" "osd crush location hook"
osd_location=`$osd_location_hook --cluster ceph --id $id --type osd`
get_conf osd_weight "" "osd crush initial weight"
- defaultweight="$(do_cmd "df -P -k $osd_data/. | tail -1 | awk '{ d= \$2/1073741824 ; r = sprintf(\"%.2f\", d); print r }'")"
+ defaultweight="$(df -P -k $osd_data/. | tail -1 | awk '{ d=$2/1073741824 ; r = sprintf(\"%.2f\", d); print r }')"
get_conf osd_keyring "$osd_data/keyring" "keyring"
- do_cmd "timeout 10 $BINDIR/ceph \
- -c $conf \
- --name=osd.$id \
- --keyring=$osd_keyring \
- osd crush create-or-move \
- -- \
- $id \
- ${osd_weight:-${defaultweight:-1}} \
- $osd_location"
+ do_cmd "timeout 10 $BINDIR/ceph -c $conf --name=osd.$id --keyring=$osd_keyring osd crush create-or-move -- $id ${osd_weight:-${defaultweight:-1}} $osd_location"
fi
fi
diff --git a/src/libcephfs.cc b/src/libcephfs.cc
index cc24a31..74bd6c3 100644
--- a/src/libcephfs.cc
+++ b/src/libcephfs.cc
@@ -1145,9 +1145,330 @@ extern "C" int ceph_get_pool_id(struct ceph_mount_info *cmount, const char *pool
return (int)pool_id;
}
-extern "C" int ceph_get_pool_replication(struct ceph_mount_info *cmount, int pool_id)
+extern "C" int ceph_get_pool_replication(struct ceph_mount_info *cmount,
+ int pool_id)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->get_pool_replication(pool_id);
}
+/* Low-level exports */
+
+extern "C" int ceph_ll_lookup_root(struct ceph_mount_info *cmount,
+ Inode **parent)
+{
+ *parent = cmount->get_client()->get_root();
+ if (*parent)
+ return 0;
+ return EFAULT;
+}
+
+extern "C" struct Inode *ceph_ll_get_inode(class ceph_mount_info *cmount,
+ vinodeno_t vino)
+{
+ return (cmount->get_client())->ll_get_inode(vino);
+}
+
+extern "C" int ceph_ll_lookup(class ceph_mount_info *cmount,
+ struct Inode *parent, const char *name,
+ struct stat *attr, Inode **out,
+ int uid, int gid)
+{
+ return (cmount->get_client())->ll_lookup(parent, name, attr, out, uid, gid);
+}
+
+extern "C" int ceph_ll_put(class ceph_mount_info *cmount, Inode *in)
+{
+ return (cmount->get_client()->ll_put(in));
+}
+
+extern "C" int ceph_ll_forget(class ceph_mount_info *cmount, Inode *in,
+ int count)
+{
+ return (cmount->get_client()->ll_forget(in, count));
+}
+
+extern "C" int ceph_ll_walk(class ceph_mount_info *cmount, const char *name,
+ struct Inode **i,
+ struct stat *attr)
+{
+ return(cmount->get_client()->ll_walk(name, i, attr));
+}
+
+extern "C" int ceph_ll_getattr(class ceph_mount_info *cmount,
+ Inode *in, struct stat *attr,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_getattr(in, attr, uid, gid));
+}
+
+extern "C" int ceph_ll_setattr(class ceph_mount_info *cmount,
+ Inode *in, struct stat *st,
+ int mask, int uid, int gid)
+{
+ return (cmount->get_client()->ll_setattr(in, st, mask, uid, gid));
+}
+
+extern "C" int ceph_ll_open(class ceph_mount_info *cmount, Inode *in,
+ int flags, Fh **fh, int uid, int gid)
+{
+ return (cmount->get_client()->ll_open(in, flags, fh, uid, gid));
+}
+
+extern "C" int ceph_ll_read(class ceph_mount_info *cmount, Fh* filehandle,
+ int64_t off, uint64_t len, char* buf)
+{
+ bufferlist bl;
+ int r = 0;
+
+ r = cmount->get_client()->ll_read(filehandle, off, len, &bl);
+ if (r >= 0)
+ {
+ bl.copy(0, bl.length(), buf);
+ r = bl.length();
+ }
+ return r;
+}
+
+extern "C" int ceph_ll_read_block(class ceph_mount_info *cmount,
+ Inode *in, uint64_t blockid,
+ char* buf, uint64_t offset,
+ uint64_t length,
+ struct ceph_file_layout* layout)
+{
+
+ return (cmount->get_client()->ll_read_block(in, blockid, buf, offset,
+ length, layout));
+}
+
+extern "C" int ceph_ll_write_block(class ceph_mount_info *cmount,
+ Inode *in, uint64_t blockid,
+ char *buf, uint64_t offset,
+ uint64_t length,
+ struct ceph_file_layout *layout,
+ uint64_t snapseq, uint32_t sync)
+{
+ return (cmount->get_client()->ll_write_block(in, blockid, buf, offset,
+ length, layout, snapseq, sync));
+}
+
+extern "C" int ceph_ll_commit_blocks(class ceph_mount_info *cmount,
+ Inode *in, uint64_t offset,
+ uint64_t range)
+{
+ return (cmount->get_client()->ll_commit_blocks(in, offset, range));
+}
+
+extern "C" int ceph_ll_fsync(class ceph_mount_info *cmount,
+ Fh *fh, int syncdataonly)
+{
+ return (cmount->get_client()->ll_fsync(fh, syncdataonly));
+}
+
+extern "C" loff_t ceph_ll_lseek(class ceph_mount_info *cmount,
+ Fh *fh, loff_t offset, int whence)
+{
+ return (cmount->get_client()->ll_lseek(fh, offset, whence));
+}
+
+extern "C" int ceph_ll_write(class ceph_mount_info *cmount,
+ Fh *fh, int64_t off, uint64_t len,
+ const char *data)
+{
+ return (cmount->get_client()->ll_write(fh, off, len, data));
+}
+
+extern "C" int64_t ceph_ll_readv(class ceph_mount_info *cmount,
+ struct Fh *fh, const struct iovec *iov,
+ int iovcnt, int64_t off)
+{
+ return -1; // TODO: implement
+}
+
+extern "C" int64_t ceph_ll_writev(class ceph_mount_info *cmount,
+ struct Fh *fh, const struct iovec *iov,
+ int iovcnt, int64_t off)
+{
+ return -1; // TODO: implement
+}
+
+extern "C" int ceph_ll_close(class ceph_mount_info *cmount, Fh* fh)
+{
+ return (cmount->get_client()->ll_release(fh));
+}
+
+extern "C" int ceph_ll_create(class ceph_mount_info *cmount,
+ struct Inode *parent, const char *name,
+ mode_t mode, int flags, struct stat *attr,
+ struct Inode **out, Fh **fhp, int uid, int gid)
+{
+ return (cmount->get_client())->ll_create(parent, name, mode, flags,
+ attr, out, fhp, uid, gid);
+}
+
+extern "C" int ceph_ll_mkdir(class ceph_mount_info *cmount,
+ Inode *parent, const char *name,
+ mode_t mode, struct stat *attr, Inode **out,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_mkdir(parent, name, mode, attr, out, uid,
+ gid));
+}
+
+extern "C" int ceph_ll_link(class ceph_mount_info *cmount,
+ Inode *in, Inode *newparent,
+ const char *name, struct stat *attr, int uid,
+ int gid)
+{
+ return (cmount->get_client()->ll_link(in, newparent, name, attr, uid,
+ gid));
+}
+
+extern "C" int ceph_ll_truncate(class ceph_mount_info *cmount,
+ Inode *in, uint64_t length, int uid,
+ int gid)
+{
+ struct stat st;
+ st.st_size=length;
+
+ return(cmount->get_client()->ll_setattr(in, &st, CEPH_SETATTR_SIZE, uid,
+ gid));
+}
+
+extern "C" int ceph_ll_opendir(class ceph_mount_info *cmount,
+ Inode *in,
+ struct ceph_dir_result **dirpp,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_opendir(in, (dir_result_t**) dirpp, uid,
+ gid));
+}
+
+extern "C" int ceph_ll_releasedir(class ceph_mount_info *cmount,
+ ceph_dir_result *dir)
+{
+ (void) cmount->get_client()->ll_releasedir((dir_result_t*) dir);
+ return (0);
+}
+
+extern "C" int ceph_ll_rename(class ceph_mount_info *cmount,
+ Inode *parent, const char *name,
+ Inode *newparent, const char *newname,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_rename(parent, name, newparent, newname,
+ uid, gid));
+}
+
+extern "C" int ceph_ll_unlink(class ceph_mount_info *cmount,
+ Inode *in, const char *name,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_unlink(in, name, uid, gid));
+}
+
+extern "C" int ceph_ll_statfs(class ceph_mount_info *cmount,
+ Inode *in, struct statvfs *stbuf)
+{
+ return (cmount->get_client()->ll_statfs(in, stbuf));
+}
+
+extern "C" int ceph_ll_readlink(class ceph_mount_info *cmount,
+ Inode *in, char *buf, size_t bufsiz, int uid,
+ int gid)
+{
+ return (cmount->get_client()->ll_readlink(in, buf, bufsiz, uid, gid));
+}
+
+extern "C" int ceph_ll_symlink(class ceph_mount_info *cmount,
+ Inode *in, const char *name,
+ const char *value, struct stat *attr,
+ Inode **out, int uid, int gid)
+{
+ return (cmount->get_client()->ll_symlink(in, name, value, attr, out, uid,
+ gid));
+}
+
+extern "C" int ceph_ll_rmdir(class ceph_mount_info *cmount,
+ Inode *in, const char *name,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_rmdir(in, name, uid, gid));
+}
+
+extern "C" int ceph_ll_getxattr(class ceph_mount_info *cmount,
+ Inode *in, const char *name, void *value,
+ size_t size, int uid, int gid)
+{
+ return (cmount->get_client()->ll_getxattr(in, name, value, size, uid, gid));
+}
+extern "C" int ceph_ll_listxattr(struct ceph_mount_info *cmount,
+ Inode *in, char *list,
+ size_t buf_size, size_t *list_size, int uid, int gid)
+{
+ int res = cmount->get_client()->ll_listxattr(in, list, buf_size, uid, gid);
+ if (res >= 0) {
+ *list_size = (size_t)res;
+ return 0;
+ }
+ return res;
+}
+
+extern "C" int ceph_ll_setxattr(class ceph_mount_info *cmount,
+ Inode *in, const char *name,
+ const void *value, size_t size,
+ int flags, int uid, int gid)
+{
+ return (cmount->get_client()->ll_setxattr(in, name, value, size, flags, uid,
+ gid));
+}
+
+extern "C" int ceph_ll_removexattr(class ceph_mount_info *cmount,
+ Inode *in, const char *name,
+ int uid, int gid)
+{
+ return (cmount->get_client()->ll_removexattr(in, name, uid, gid));
+}
+
+extern "C" uint32_t ceph_ll_stripe_unit(class ceph_mount_info *cmount,
+ Inode *in)
+{
+ return (cmount->get_client()->ll_stripe_unit(in));
+}
+
+extern "C" uint32_t ceph_ll_file_layout(class ceph_mount_info *cmount,
+ Inode *in,
+ struct ceph_file_layout *layout)
+{
+ return (cmount->get_client()->ll_file_layout(in, layout));
+}
+
+uint64_t ceph_ll_snap_seq(class ceph_mount_info *cmount, Inode *in)
+{
+ return (cmount->get_client()->ll_snap_seq(in));
+}
+
+extern "C" int ceph_ll_get_stripe_osd(class ceph_mount_info *cmount,
+ Inode *in, uint64_t blockno,
+ struct ceph_file_layout* layout)
+{
+ return (cmount->get_client()->ll_get_stripe_osd(in, blockno, layout));
+}
+
+extern "C" int ceph_ll_num_osds(class ceph_mount_info *cmount)
+{
+ return (cmount->get_client()->ll_num_osds());
+}
+
+extern "C" int ceph_ll_osdaddr(class ceph_mount_info *cmount,
+ int osd, uint32_t *addr)
+{
+ return (cmount->get_client()->ll_osdaddr(osd, addr));
+}
+
+extern "C" uint64_t ceph_ll_get_internal_offset(class ceph_mount_info *cmount,
+ Inode *in,
+ uint64_t blockno)
+{
+ return (cmount->get_client()->ll_get_internal_offset(in, blockno));
+}
diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index 929e105..e3e6620 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -502,7 +502,7 @@ int librados::IoCtxImpl::clone_range(const object_t& dst_oid,
}
int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
- time_t *pmtime)
+ time_t *pmtime, int flags)
{
utime_t ut;
if (pmtime) {
@@ -529,7 +529,7 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
int op = o->ops[0].op.op;
ldout(client->cct, 10) << ceph_osd_op_name(op) << " oid=" << oid << " nspace=" << oloc.nspace << dendl;
Objecter::Op *objecter_op = objecter->prepare_mutate_op(oid, oloc,
- *o, snapc, ut, 0,
+ *o, snapc, ut, flags,
NULL, oncommit, &ver);
lock->Lock();
objecter->op_submit(objecter_op);
@@ -548,7 +548,9 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
}
int librados::IoCtxImpl::operate_read(const object_t& oid,
- ::ObjectOperation *o, bufferlist *pbl)
+ ::ObjectOperation *o,
+ bufferlist *pbl,
+ int flags)
{
if (!o->size())
return 0;
@@ -564,7 +566,7 @@ int librados::IoCtxImpl::operate_read(const object_t& oid,
int op = o->ops[0].op.op;
ldout(client->cct, 10) << ceph_osd_op_name(op) << " oid=" << oid << " nspace=" << oloc.nspace << dendl;
Objecter::Op *objecter_op = objecter->prepare_read_op(oid, oloc,
- *o, snap_seq, pbl, 0,
+ *o, snap_seq, pbl, flags,
onack, &ver);
lock->Lock();
objecter->op_submit(objecter_op);
@@ -924,7 +926,7 @@ int librados::IoCtxImpl::read(const object_t& oid,
::ObjectOperation rd;
prepare_assert_ops(&rd);
- rd.read(off, len, &bl, NULL);
+ rd.read(off, len, &bl, NULL, NULL);
int r = operate_read(oid, &rd, &bl);
if (r < 0)
return r;
@@ -1202,6 +1204,16 @@ int librados::IoCtxImpl::notify(const object_t& oid, uint64_t ver, bufferlist& b
return r;
}
+int librados::IoCtxImpl::set_alloc_hint(const object_t& oid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+{
+ ::ObjectOperation wr;
+ prepare_assert_ops(&wr);
+ wr.set_alloc_hint(expected_object_size, expected_write_size);
+ return operate(oid, &wr, NULL);
+}
+
version_t librados::IoCtxImpl::last_version()
{
return last_objver;
diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h
index 8693d6c..58e27ed 100644
--- a/src/librados/IoCtxImpl.h
+++ b/src/librados/IoCtxImpl.h
@@ -140,8 +140,8 @@ struct librados::IoCtxImpl {
int getxattrs(const object_t& oid, map<string, bufferlist>& attrset);
int rmxattr(const object_t& oid, const char *name);
- int operate(const object_t& oid, ::ObjectOperation *o, time_t *pmtime);
- int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl);
+ int operate(const object_t& oid, ::ObjectOperation *o, time_t *pmtime, int flags=0);
+ int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0);
int aio_operate(const object_t& oid, ::ObjectOperation *o,
AioCompletionImpl *c, const SnapContext& snap_context,
int flags);
@@ -202,6 +202,10 @@ struct librados::IoCtxImpl {
const object_t& oid, uint64_t notify_id, uint64_t ver,
uint64_t cookie);
+ int set_alloc_hint(const object_t& oid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size);
+
version_t last_version();
void set_assert_version(uint64_t ver);
void set_assert_src_version(const object_t& oid, uint64_t ver);
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index add1bb0..b656b5e 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -102,7 +102,9 @@ int64_t librados::RadosClient::lookup_pool(const char *name)
lock.Lock();
- wait_for_osdmap();
+ int r = wait_for_osdmap();
+ if (r < 0)
+ return r;
int64_t ret = osdmap.lookup_pg_pool_name(name);
pool_cache_rwl.get_write();
lock.Unlock();
@@ -120,6 +122,20 @@ int64_t librados::RadosClient::lookup_pool(const char *name)
return ret;
}
+bool librados::RadosClient::pool_requires_alignment(int64_t pool_id)
+{
+ Mutex::Locker l(lock);
+ return osdmap.have_pg_pool(pool_id) &&
+ osdmap.get_pg_pool(pool_id)->requires_aligned_append();
+}
+
+uint64_t librados::RadosClient::pool_required_alignment(int64_t pool_id)
+{
+ Mutex::Locker l(lock);
+ return osdmap.have_pg_pool(pool_id) ?
+ osdmap.get_pg_pool(pool_id)->required_alignment() : 0;
+}
+
const char *librados::RadosClient::get_pool_name(int64_t pool_id)
{
Mutex::Locker l(lock);
@@ -129,7 +145,9 @@ const char *librados::RadosClient::get_pool_name(int64_t pool_id)
int librados::RadosClient::pool_get_auid(uint64_t pool_id, unsigned long long *auid)
{
Mutex::Locker l(lock);
- wait_for_osdmap();
+ int r = wait_for_osdmap();
+ if (r < 0)
+ return r;
const pg_pool_t *pg = osdmap.get_pg_pool(pool_id);
if (!pg)
return -ENOENT;
@@ -140,7 +158,9 @@ int librados::RadosClient::pool_get_auid(uint64_t pool_id, unsigned long long *a
int librados::RadosClient::pool_get_name(uint64_t pool_id, std::string *s)
{
Mutex::Locker l(lock);
- wait_for_osdmap();
+ int r = wait_for_osdmap();
+ if (r < 0)
+ return r;
const char *str = osdmap.get_pool_name(pool_id);
if (!str)
return -ENOENT;
@@ -214,7 +234,9 @@ int librados::RadosClient::connect()
ldout(cct, 1) << "starting objecter" << dendl;
err = -ENOMEM;
- objecter = new Objecter(cct, messenger, &monclient, &osdmap, lock, timer);
+ objecter = new Objecter(cct, messenger, &monclient, &osdmap, lock, timer,
+ cct->_conf->rados_mon_op_timeout,
+ cct->_conf->rados_osd_op_timeout);
if (!objecter)
goto out;
objecter->set_balanced_budget();
@@ -406,15 +428,34 @@ bool librados::RadosClient::_dispatch(Message *m)
return true;
}
-void librados::RadosClient::wait_for_osdmap()
+int librados::RadosClient::wait_for_osdmap()
{
assert(lock.is_locked());
+
+ utime_t timeout;
+ if (cct->_conf->rados_mon_op_timeout > 0)
+ timeout.set_from_double(cct->_conf->rados_mon_op_timeout);
+
if (osdmap.get_epoch() == 0) {
ldout(cct, 10) << __func__ << " waiting" << dendl;
- while (osdmap.get_epoch() == 0)
- cond.Wait(lock);
+ utime_t start = ceph_clock_now(cct);
+
+ while (osdmap.get_epoch() == 0) {
+ cond.WaitInterval(cct, lock, timeout);
+
+ utime_t elapsed = ceph_clock_now(cct) - start;
+ if (!timeout.is_zero() && elapsed > timeout)
+ break;
+ }
+
ldout(cct, 10) << __func__ << " done waiting" << dendl;
+
+ if (osdmap.get_epoch() == 0) {
+ lderr(cct) << "timed out waiting for first osdmap from monitors" << dendl;
+ return -ETIMEDOUT;
+ }
}
+ return 0;
}
int librados::RadosClient::wait_for_latest_osdmap()
@@ -438,7 +479,9 @@ int librados::RadosClient::wait_for_latest_osdmap()
int librados::RadosClient::pool_list(std::list<std::string>& v)
{
Mutex::Locker l(lock);
- wait_for_osdmap();
+ int r = wait_for_osdmap();
+ if (r < 0)
+ return r;
for (map<int64_t,pg_pool_t>::const_iterator p = osdmap.get_pools().begin();
p != osdmap.get_pools().end();
++p)
@@ -452,9 +495,11 @@ int librados::RadosClient::get_pool_stats(std::list<string>& pools,
Mutex mylock("RadosClient::get_pool_stats::mylock");
Cond cond;
bool done;
+ int ret = 0;
lock.Lock();
- objecter->get_pool_stats(pools, &result, new C_SafeCond(&mylock, &cond, &done));
+ objecter->get_pool_stats(pools, &result, new C_SafeCond(&mylock, &cond, &done,
+ &ret));
lock.Unlock();
mylock.Lock();
@@ -462,7 +507,7 @@ int librados::RadosClient::get_pool_stats(std::list<string>& pools,
cond.Wait(mylock);
mylock.Unlock();
- return 0;
+ return ret;
}
int librados::RadosClient::get_fs_stats(ceph_statfs& stats)
@@ -470,15 +515,17 @@ int librados::RadosClient::get_fs_stats(ceph_statfs& stats)
Mutex mylock ("RadosClient::get_fs_stats::mylock");
Cond cond;
bool done;
+ int ret = 0;
+
lock.Lock();
- objecter->get_fs_stats(stats, new C_SafeCond(&mylock, &cond, &done));
+ objecter->get_fs_stats(stats, new C_SafeCond(&mylock, &cond, &done, &ret));
lock.Unlock();
mylock.Lock();
while (!done) cond.Wait(mylock);
mylock.Unlock();
- return 0;
+ return ret;
}
void librados::RadosClient::get() {
@@ -534,7 +581,9 @@ int librados::RadosClient::pool_create_async(string& name, PoolAsyncCompletionIm
int librados::RadosClient::pool_delete(const char *name)
{
lock.Lock();
- wait_for_osdmap();
+ int r = wait_for_osdmap();
+ if (r < 0)
+ return r;
int tmp_pool_id = osdmap.lookup_pg_pool_name(name);
if (tmp_pool_id < 0) {
lock.Unlock();
@@ -563,13 +612,15 @@ int librados::RadosClient::pool_delete(const char *name)
int librados::RadosClient::pool_delete_async(const char *name, PoolAsyncCompletionImpl *c)
{
Mutex::Locker l(lock);
- wait_for_osdmap();
+ int r = wait_for_osdmap();
+ if (r < 0)
+ return r;
int tmp_pool_id = osdmap.lookup_pg_pool_name(name);
if (tmp_pool_id < 0)
return -ENOENT;
Context *onfinish = new C_PoolAsync_Safe(c);
- int r = objecter->delete_pool(tmp_pool_id, onfinish);
+ r = objecter->delete_pool(tmp_pool_id, onfinish);
if (r < 0) {
delete onfinish;
}
diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index f9cc1f7..e608ced 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -79,7 +79,7 @@ private:
void *log_cb_arg;
string log_watch;
- void wait_for_osdmap();
+ int wait_for_osdmap();
public:
Finisher finisher;
@@ -99,6 +99,8 @@ public:
int get_fsid(std::string *s);
int64_t lookup_pool(const char *name);
const char *get_pool_name(int64_t pool_id);
+ bool pool_requires_alignment(int64_t pool_id);
+ uint64_t pool_required_alignment(int64_t pool_id);
int pool_get_auid(uint64_t pool_id, unsigned long long *auid);
int pool_get_name(uint64_t pool_id, std::string *auid);
diff --git a/src/librados/librados.cc b/src/librados/librados.cc
index 4925d5e..42476b5 100644
--- a/src/librados/librados.cc
+++ b/src/librados/librados.cc
@@ -75,16 +75,20 @@ size_t librados::ObjectOperation::size()
return o->size();
}
-void librados::ObjectOperation::set_op_flags(ObjectOperationFlags flags)
+static void set_op_flags(::ObjectOperation *o, int flags)
{
int rados_flags = 0;
- if (flags & OP_EXCL)
+ if (flags & LIBRADOS_OP_FLAG_EXCL)
rados_flags |= CEPH_OSD_OP_FLAG_EXCL;
- if (flags & OP_FAILOK)
+ if (flags & LIBRADOS_OP_FLAG_FAILOK)
rados_flags |= CEPH_OSD_OP_FLAG_FAILOK;
+ o->set_last_op_flags(rados_flags);
+}
+void librados::ObjectOperation::set_op_flags(ObjectOperationFlags flags)
+{
::ObjectOperation *o = (::ObjectOperation *)impl;
- o->set_last_op_flags(rados_flags);
+ ::set_op_flags(o, (int)flags);
}
void librados::ObjectOperation::cmpxattr(const char *name, uint8_t op, const bufferlist& v)
@@ -176,7 +180,7 @@ void librados::ObjectReadOperation::stat(uint64_t *psize, time_t *pmtime, int *p
void librados::ObjectReadOperation::read(size_t off, uint64_t len, bufferlist *pbl, int *prval)
{
::ObjectOperation *o = (::ObjectOperation *)impl;
- o->read(off, len, pbl, prval);
+ o->read(off, len, pbl, prval, NULL);
}
void librados::ObjectReadOperation::sparse_read(uint64_t off, uint64_t len,
@@ -449,6 +453,14 @@ void librados::ObjectWriteOperation::selfmanaged_snap_rollback(snap_t snapid)
o->rollback(snapid);
}
+void librados::ObjectWriteOperation::set_alloc_hint(
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+{
+ ::ObjectOperation *o = (::ObjectOperation *)impl;
+ o->set_alloc_hint(expected_object_size, expected_write_size);
+}
+
librados::WatchCtx::
~WatchCtx()
{
@@ -476,12 +488,35 @@ librados::ObjectIterator::~ObjectIterator()
ctx.reset();
}
+librados::ObjectIterator::ObjectIterator(const ObjectIterator &rhs)
+{
+ *this = rhs;
+}
+
+librados::ObjectIterator& librados::ObjectIterator::operator=(const librados::ObjectIterator &rhs)
+{
+ if (&rhs == this)
+ return *this;
+ if (rhs.ctx.get() == NULL) {
+ ctx.reset();
+ return *this;
+ }
+ Objecter::ListContext *list_ctx = new Objecter::ListContext(*rhs.ctx->lc);
+ ctx.reset(new ObjListCtx(rhs.ctx->ctx, list_ctx));
+ cur_obj = rhs.cur_obj;
+ return *this;
+}
+
bool librados::ObjectIterator::operator==(const librados::ObjectIterator& rhs) const {
- return (ctx.get() == rhs.ctx.get());
+ if (ctx.get() == NULL)
+ return rhs.ctx.get() == NULL || rhs.ctx->lc->at_end();
+ if (rhs.ctx.get() == NULL)
+ return ctx.get() == NULL || ctx->lc->at_end();
+ return ctx.get() == rhs.ctx.get();
}
bool librados::ObjectIterator::operator!=(const librados::ObjectIterator& rhs) const {
- return (ctx.get() != rhs.ctx.get());
+ return !(*this == rhs);
}
const pair<std::string, std::string>& librados::ObjectIterator::operator*() const {
@@ -515,10 +550,10 @@ uint32_t librados::ObjectIterator::seek(uint32_t pos)
void librados::ObjectIterator::get_next()
{
const char *entry, *key;
+ if (ctx->lc->at_end())
+ return;
int ret = rados_objects_list_next(ctx.get(), &entry, &key);
if (ret == -ENOENT) {
- ctx.reset();
- *this = __EndObjectIterator;
return;
}
else if (ret) {
@@ -724,6 +759,16 @@ int librados::IoCtx::get_auid(uint64_t *auid_)
return rados_ioctx_pool_get_auid(io_ctx_impl, auid_);
}
+bool librados::IoCtx::pool_requires_alignment()
+{
+ return io_ctx_impl->client->pool_requires_alignment(get_id());
+}
+
+uint64_t librados::IoCtx::pool_required_alignment()
+{
+ return io_ctx_impl->client->pool_required_alignment(get_id());
+}
+
std::string librados::IoCtx::get_pool_name()
{
std::string s;
@@ -1213,7 +1258,6 @@ librados::ObjectIterator librados::IoCtx::objects_begin(uint32_t pos)
rados_objects_list_open(io_ctx_impl, &listh);
ObjectIterator iter((ObjListCtx*)listh);
iter.seek(pos);
- iter.get_next();
return iter;
}
@@ -1380,6 +1424,15 @@ void librados::IoCtx::set_notify_timeout(uint32_t timeout)
io_ctx_impl->set_notify_timeout(timeout);
}
+int librados::IoCtx::set_alloc_hint(const std::string& o,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+{
+ object_t oid(o);
+ return io_ctx_impl->set_alloc_hint(oid, expected_object_size,
+ expected_write_size);
+}
+
void librados::IoCtx::set_assert_version(uint64_t ver)
{
io_ctx_impl->set_assert_version(ver);
@@ -1631,6 +1684,8 @@ int librados::Rados::get_pool_stats(std::list<string>& v, string& category,
{
map<string,::pool_stat_t> rawresult;
int r = client->get_pool_stats(v, rawresult);
+ if (r < 0)
+ return r;
for (map<string,::pool_stat_t>::iterator p = rawresult.begin();
p != rawresult.end();
++p) {
@@ -1945,7 +2000,9 @@ extern "C" int rados_pool_reverse_lookup(rados_t cluster, int64_t id,
{
librados::RadosClient *radosp = (librados::RadosClient *)cluster;
std::string name;
- radosp->pool_get_name(id, &name);
+ int r = radosp->pool_get_name(id, &name);
+ if (r < 0)
+ return r;
if (name.length() >= maxlen)
return -ERANGE;
strcpy(buf, name.c_str());
@@ -1968,7 +2025,9 @@ extern "C" int rados_pool_list(rados_t cluster, char *buf, size_t len)
{
librados::RadosClient *client = (librados::RadosClient *)cluster;
std::list<std::string> pools;
- client->pool_list(pools);
+ int r = client->pool_list(pools);
+ if (r < 0)
+ return r;
if (!buf)
return -EINVAL;
@@ -2912,6 +2971,15 @@ int rados_notify(rados_ioctx_t io, const char *o, uint64_t ver, const char *buf,
return ctx->notify(oid, ver, bl);
}
+extern "C" int rados_set_alloc_hint(rados_ioctx_t io, const char *o,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+{
+ librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
+ object_t oid(o);
+ return ctx->set_alloc_hint(oid, expected_object_size, expected_write_size);
+}
+
extern "C" int rados_lock_exclusive(rados_ioctx_t io, const char * o,
const char * name, const char * cookie,
const char * desc, struct timeval * duration,
@@ -3022,6 +3090,11 @@ extern "C" void rados_release_write_op(rados_write_op_t write_op)
delete (::ObjectOperation*)write_op;
}
+extern "C" void rados_write_op_set_flags(rados_write_op_t write_op, int flags)
+{
+ set_op_flags((::ObjectOperation *)write_op, flags);
+}
+
extern "C" void rados_write_op_assert_exists(rados_write_op_t write_op)
{
((::ObjectOperation *)write_op)->stat(NULL, (utime_t *)NULL, NULL);
@@ -3041,6 +3114,31 @@ extern "C" void rados_write_op_cmpxattr(rados_write_op_t write_op,
bl);
}
+static void rados_c_omap_cmp(ObjectOperation *op,
+ const char *key,
+ uint8_t comparison_operator,
+ const char *val,
+ size_t val_len,
+ int *prval)
+{
+ bufferlist bl;
+ bl.append(val, val_len);
+ std::map<std::string, pair<bufferlist, int> > assertions;
+ assertions[key] = std::make_pair(bl, comparison_operator);
+ op->omap_cmp(assertions, prval);
+}
+
+extern "C" void rados_write_op_omap_cmp(rados_write_op_t write_op,
+ const char *key,
+ uint8_t comparison_operator,
+ const char *val,
+ size_t val_len,
+ int *prval)
+{
+ rados_c_omap_cmp((::ObjectOperation *)write_op, key, comparison_operator,
+ val, val_len, prval);
+}
+
extern "C" void rados_write_op_setxattr(rados_write_op_t write_op,
const char *name,
const char *value,
@@ -3117,26 +3215,362 @@ extern "C" void rados_write_op_zero(rados_write_op_t write_op,
((::ObjectOperation *)write_op)->zero(offset, len);
}
+extern "C" void rados_write_op_exec(rados_write_op_t write_op,
+ const char *cls,
+ const char *method,
+ const char *in_buf,
+ size_t in_len,
+ int *prval)
+{
+ bufferlist inbl;
+ inbl.append(in_buf, in_len);
+ ((::ObjectOperation *)write_op)->call(cls, method, inbl, NULL, NULL, prval);
+}
+
+extern "C" void rados_write_op_omap_set(rados_write_op_t write_op,
+ char const* const* keys,
+ char const* const* vals,
+ const size_t *lens,
+ size_t num)
+{
+ std::map<std::string, bufferlist> entries;
+ for (size_t i = 0; i < num; ++i) {
+ bufferlist bl(lens[i]);
+ bl.append(vals[i], lens[i]);
+ entries[keys[i]] = bl;
+ }
+ ((::ObjectOperation *)write_op)->omap_set(entries);
+}
+
+extern "C" void rados_write_op_omap_rm_keys(rados_write_op_t write_op,
+ char const* const* keys,
+ size_t keys_len)
+{
+ std::set<std::string> to_remove(keys, keys + keys_len);
+ ((::ObjectOperation *)write_op)->omap_rm_keys(to_remove);
+}
+
+extern "C" void rados_write_op_omap_clear(rados_write_op_t write_op)
+{
+ ((::ObjectOperation *)write_op)->omap_clear();
+}
+
+extern "C" void rados_write_op_set_alloc_hint(rados_write_op_t write_op,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+{
+ ((::ObjectOperation *)write_op)->set_alloc_hint(expected_object_size,
+ expected_write_size);
+}
+
extern "C" int rados_write_op_operate(rados_write_op_t write_op,
rados_ioctx_t io,
const char *oid,
- time_t *mtime)
+ time_t *mtime,
+ int flags)
{
object_t obj(oid);
::ObjectOperation *oo = (::ObjectOperation *) write_op;
librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
- return ctx->operate(obj, oo, mtime);
+ return ctx->operate(obj, oo, mtime, flags);
}
extern "C" int rados_aio_write_op_operate(rados_write_op_t write_op,
- rados_ioctx_t io,
- rados_completion_t completion,
- const char *oid,
- time_t *mtime)
+ rados_ioctx_t io,
+ rados_completion_t completion,
+ const char *oid,
+ time_t *mtime,
+ int flags)
{
object_t obj(oid);
::ObjectOperation *oo = (::ObjectOperation *) write_op;
librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
librados::AioCompletionImpl *c = (librados::AioCompletionImpl*)completion;
- return ctx->aio_operate(obj, oo, c, ctx->snapc, 0);
+ return ctx->aio_operate(obj, oo, c, ctx->snapc, flags);
+}
+
+extern "C" rados_read_op_t rados_create_read_op()
+{
+ return new (std::nothrow)::ObjectOperation;
+}
+
+extern "C" void rados_release_read_op(rados_read_op_t read_op)
+{
+ delete (::ObjectOperation *)read_op;
+}
+
+extern "C" void rados_read_op_set_flags(rados_read_op_t read_op, int flags)
+{
+ set_op_flags((::ObjectOperation *)read_op, flags);
+}
+
+extern "C" void rados_read_op_assert_exists(rados_read_op_t read_op)
+{
+ ((::ObjectOperation *)read_op)->stat(NULL, (utime_t *)NULL, NULL);
+}
+
+extern "C" void rados_read_op_cmpxattr(rados_read_op_t read_op,
+ const char *name,
+ uint8_t comparison_operator,
+ const char *value,
+ size_t value_len)
+{
+ bufferlist bl;
+ bl.append(value, value_len);
+ ((::ObjectOperation *)read_op)->cmpxattr(name,
+ comparison_operator,
+ CEPH_OSD_CMPXATTR_MODE_STRING,
+ bl);
+}
+
+extern "C" void rados_read_op_omap_cmp(rados_read_op_t read_op,
+ const char *key,
+ uint8_t comparison_operator,
+ const char *val,
+ size_t val_len,
+ int *prval)
+{
+ rados_c_omap_cmp((::ObjectOperation *)read_op, key, comparison_operator,
+ val, val_len, prval);
+}
+
+extern "C" void rados_read_op_stat(rados_read_op_t read_op,
+ uint64_t *psize,
+ time_t *pmtime,
+ int *prval)
+{
+ ((::ObjectOperation *)read_op)->stat(psize, pmtime, prval);
+}
+
+class C_bl_to_buf : public Context {
+ char *out_buf;
+ size_t out_len;
+ size_t *bytes_read;
+ int *prval;
+public:
+ bufferlist out_bl;
+ C_bl_to_buf(char *out_buf,
+ size_t out_len,
+ size_t *bytes_read,
+ int *prval) : out_buf(out_buf), out_len(out_len),
+ bytes_read(bytes_read), prval(prval) {}
+ void finish(int r) {
+ if (out_bl.length() > out_len) {
+ if (prval)
+ *prval = -ERANGE;
+ if (bytes_read)
+ *bytes_read = 0;
+ return;
+ }
+ if (bytes_read)
+ *bytes_read = out_bl.length();
+ if (out_buf && out_bl.c_str() != out_buf)
+ out_bl.copy(0, out_bl.length(), out_buf);
+ }
+};
+
+extern "C" void rados_read_op_read(rados_read_op_t read_op,
+ uint64_t offset,
+ size_t len,
+ char *buf,
+ size_t *bytes_read,
+ int *prval)
+{
+ C_bl_to_buf *ctx = new C_bl_to_buf(buf, len, bytes_read, prval);
+ ctx->out_bl.push_back(buffer::create_static(len, buf));
+ ((::ObjectOperation *)read_op)->read(offset, len, &ctx->out_bl, prval, ctx);
+}
+
+class C_out_buffer : public Context {
+ char **out_buf;
+ size_t *out_len;
+public:
+ bufferlist out_bl;
+ C_out_buffer(char **out_buf, size_t *out_len) : out_buf(out_buf),
+ out_len(out_len) {}
+ void finish(int r) {
+ // ignore r since we don't know the meaning of return values
+ // from custom class methods
+ do_out_buffer(out_bl, out_buf, out_len);
+ }
+};
+
+extern "C" void rados_read_op_exec(rados_read_op_t read_op,
+ const char *cls,
+ const char *method,
+ const char *in_buf,
+ size_t in_len,
+ char **out_buf,
+ size_t *out_len,
+ int *prval)
+{
+ bufferlist inbl;
+ inbl.append(in_buf, in_len);
+ C_out_buffer *ctx = new C_out_buffer(out_buf, out_len);
+ ((::ObjectOperation *)read_op)->call(cls, method, inbl, &ctx->out_bl, ctx,
+ prval);
+}
+
+extern "C" void rados_read_op_exec_user_buf(rados_read_op_t read_op,
+ const char *cls,
+ const char *method,
+ const char *in_buf,
+ size_t in_len,
+ char *out_buf,
+ size_t out_len,
+ size_t *used_len,
+ int *prval)
+{
+ C_bl_to_buf *ctx = new C_bl_to_buf(out_buf, out_len, used_len, prval);
+ bufferlist inbl;
+ inbl.append(in_buf, in_len);
+ ((::ObjectOperation *)read_op)->call(cls, method, inbl, &ctx->out_bl, ctx,
+ prval);
+}
+
+struct RadosOmapIter {
+ std::map<std::string, bufferlist> values;
+ std::map<std::string, bufferlist>::iterator i;
+};
+
+class C_OmapIter : public Context {
+ RadosOmapIter *iter;
+public:
+ C_OmapIter(RadosOmapIter *iter) : iter(iter) {}
+ void finish(int r) {
+ iter->i = iter->values.begin();
+ }
+};
+
+class C_XattrsIter : public Context {
+ RadosXattrsIter *iter;
+public:
+ C_XattrsIter(RadosXattrsIter *iter) : iter(iter) {}
+ void finish(int r) {
+ iter->i = iter->attrset.begin();
+ }
+};
+
+extern "C" void rados_read_op_getxattrs(rados_read_op_t read_op,
+ rados_xattrs_iter_t *iter,
+ int *prval)
+{
+ RadosXattrsIter *xattrs_iter = new RadosXattrsIter;
+ ((::ObjectOperation *)read_op)->getxattrs(&xattrs_iter->attrset, prval);
+ ((::ObjectOperation *)read_op)->add_handler(new C_XattrsIter(xattrs_iter));
+ *iter = xattrs_iter;
+}
+
+extern "C" void rados_read_op_omap_get_vals(rados_read_op_t read_op,
+ const char *start_after,
+ const char *filter_prefix,
+ uint64_t max_return,
+ rados_omap_iter_t *iter,
+ int *prval)
+{
+ RadosOmapIter *omap_iter = new RadosOmapIter;
+ const char *start = start_after ? start_after : "";
+ const char *filter = filter_prefix ? filter_prefix : "";
+ ((::ObjectOperation *)read_op)->omap_get_vals(start,
+ filter,
+ max_return,
+ &omap_iter->values,
+ prval);
+ ((::ObjectOperation *)read_op)->add_handler(new C_OmapIter(omap_iter));
+ *iter = omap_iter;
+}
+
+struct C_OmapKeysIter : public Context {
+ RadosOmapIter *iter;
+ std::set<std::string> keys;
+ C_OmapKeysIter(RadosOmapIter *iter) : iter(iter) {}
+ void finish(int r) {
+ // map each key to an empty bl
+ for (std::set<std::string>::const_iterator i = keys.begin();
+ i != keys.end(); ++i) {
+ iter->values[*i];
+ }
+ iter->i = iter->values.begin();
+ }
+};
+
+extern "C" void rados_read_op_omap_get_keys(rados_read_op_t read_op,
+ const char *start_after,
+ uint64_t max_return,
+ rados_omap_iter_t *iter,
+ int *prval)
+{
+ RadosOmapIter *omap_iter = new RadosOmapIter;
+ C_OmapKeysIter *ctx = new C_OmapKeysIter(omap_iter);
+ ((::ObjectOperation *)read_op)->omap_get_keys(start_after ? start_after : "",
+ max_return, &ctx->keys, prval);
+ ((::ObjectOperation *)read_op)->add_handler(ctx);
+ *iter = omap_iter;
+}
+
+extern "C" void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op,
+ char const* const* keys,
+ size_t keys_len,
+ rados_omap_iter_t *iter,
+ int *prval)
+{
+ std::set<std::string> to_get(keys, keys + keys_len);
+
+ RadosOmapIter *omap_iter = new RadosOmapIter;
+ ((::ObjectOperation *)read_op)->omap_get_vals_by_keys(to_get,
+ &omap_iter->values,
+ prval);
+ ((::ObjectOperation *)read_op)->add_handler(new C_OmapIter(omap_iter));
+ *iter = omap_iter;
+}
+
+extern "C" int rados_omap_get_next(rados_omap_iter_t iter,
+ char **key,
+ char **val,
+ size_t *len)
+{
+ RadosOmapIter *it = (RadosOmapIter *)iter;
+ if (it->i == it->values.end()) {
+ *key = NULL;
+ *val = NULL;
+ *len = 0;
+ return 0;
+ }
+ if (key)
+ *key = (char*)it->i->first.c_str();
+ if (val)
+ *val = it->i->second.c_str();
+ if (len)
+ *len = it->i->second.length();
+ ++it->i;
+ return 0;
+}
+
+extern "C" void rados_omap_get_end(rados_omap_iter_t iter)
+{
+ RadosOmapIter *it = (RadosOmapIter *)iter;
+ delete it;
+}
+
+extern "C" int rados_read_op_operate(rados_read_op_t read_op,
+ rados_ioctx_t io,
+ const char *oid,
+ int flags)
+{
+ object_t obj(oid);
+ librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
+ return ctx->operate_read(obj, (::ObjectOperation *)read_op, NULL, flags);
+}
+
+extern "C" int rados_aio_read_op_operate(rados_read_op_t read_op,
+ rados_ioctx_t io,
+ rados_completion_t completion,
+ const char *oid,
+ int flags)
+{
+ object_t obj(oid);
+ librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
+ librados::AioCompletionImpl *c = (librados::AioCompletionImpl*)completion;
+ return ctx->aio_operate_read(obj, (::ObjectOperation *)read_op,
+ c, flags, NULL);
}
diff --git a/src/librbd/AioRequest.cc b/src/librbd/AioRequest.cc
index 6e4a05e..881f754 100644
--- a/src/librbd/AioRequest.cc
+++ b/src/librbd/AioRequest.cc
@@ -105,7 +105,7 @@ namespace librbd {
return r;
}
- /** read **/
+ /** write **/
AbstractWrite::AbstractWrite()
: m_state(LIBRBD_AIO_WRITE_FLAT),
@@ -236,6 +236,7 @@ namespace librbd {
}
void AbstractWrite::send_copyup() {
+ ldout(m_ictx->cct, 20) << "send_copyup " << this << " " << m_oid << " " << m_object_off << "~" << m_object_len << dendl;
if (!m_read_data.is_zero())
m_copyup.exec("rbd", "copyup", m_read_data);
add_copyup_ops();
@@ -246,4 +247,9 @@ namespace librbd {
m_snap_seq, m_snaps);
rados_completion->release();
}
+
+ void AioWrite::add_write_ops(librados::ObjectWriteOperation &wr) {
+ wr.set_alloc_hint(m_ictx->get_object_size(), m_ictx->get_object_size());
+ wr.write(m_object_off, m_write_data);
+ }
}
diff --git a/src/librbd/AioRequest.h b/src/librbd/AioRequest.h
index cf50ee2..d6103f9 100644
--- a/src/librbd/AioRequest.h
+++ b/src/librbd/AioRequest.h
@@ -160,16 +160,17 @@ namespace librbd {
completion, false),
m_write_data(data) {
guard_write();
- m_write.write(m_object_off, data);
+ add_write_ops(m_write);
}
virtual ~AioWrite() {}
protected:
virtual void add_copyup_ops() {
- m_copyup.write(m_object_off, m_write_data);
+ add_write_ops(m_copyup);
}
private:
+ void add_write_ops(librados::ObjectWriteOperation &wr);
ceph::bufferlist m_write_data;
};
diff --git a/src/logrotate.conf b/src/logrotate.conf
index e49285a..9181b56 100644
--- a/src/logrotate.conf
+++ b/src/logrotate.conf
@@ -11,13 +11,17 @@
fi
# Possibly reload twice, but depending on ceph.conf the reload above may be a no-op
if which initctl > /dev/null 2>&1 && [ -x `which initctl` ]; then
- # upstart reload isn't very helpful here:
- # https://bugs.launchpad.net/upstart/+bug/1012938
- initctl list \
- | sed -n 's/^\(ceph-\(mon\|osd\|mds\)\+\)[ \t]\+(\([^ \/]\+\)\/\([^ \/]\+\))[ \t]\+start\/.*$/\1 cluster=\3 id=\4/p' \
- | while read l; do
- initctl reload -- $l 2>/dev/null || :
- done
+ for daemon in osd mon mds ; do
+ find -L /var/lib/ceph/$daemon/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+ | while read f; do
+ if [ -e "/var/lib/ceph/$daemon/$f/done" ] && [ -e "/var/lib/ceph/$daemon/$f/upstart" ] && [ ! -e "/var/lib/ceph/$daemon/$f/sysvinit" ]; then
+ cluster="${f%%-*}"
+ id="${f#*-}"
+
+ initctl reload ceph-$daemon cluster="$cluster" id="$id" 2>/dev/null || :
+ fi
+ done
+ done
fi
endscript
missingok
diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 3697929..e244175 100644
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -154,7 +154,6 @@ ostream& CDir::print_db_line_prefix(ostream& out)
// CDir
CDir::CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth) :
- mseq(0),
dirty_rstat_inodes(member_offset(CInode, dirty_rstat_item)),
item_dirty(this), item_new(this),
pop_me(ceph_clock_now(g_ceph_context)),
@@ -875,7 +874,7 @@ void CDir::split(int bits, list<CDir*>& subs, list<Context*>& waiters, bool repl
int n = 0;
for (list<frag_t>::iterator p = frags.begin(); p != frags.end(); ++p) {
CDir *f = new CDir(inode, *p, cache, is_auth());
- f->state_set(state & MASK_STATE_FRAGMENT_KEPT);
+ f->state_set(state & (MASK_STATE_FRAGMENT_KEPT | STATE_COMPLETE));
f->replica_map = replica_map;
f->dir_auth = dir_auth;
f->init_fragment_pins();
@@ -897,6 +896,7 @@ void CDir::split(int bits, list<CDir*>& subs, list<Context*>& waiters, bool repl
subs.push_back(f);
inode->add_dirfrag(f);
+ f->set_dir_auth(get_dir_auth());
f->prepare_new_fragment(replay);
}
@@ -938,6 +938,7 @@ void CDir::merge(list<CDir*>& subs, list<Context*>& waiters, bool replay)
{
dout(10) << "merge " << subs << dendl;
+ set_dir_auth(subs.front()->get_dir_auth());
prepare_new_fragment(replay);
nest_info_t rstatdiff;
@@ -984,6 +985,9 @@ void CDir::merge(list<CDir*>& subs, list<Context*>& waiters, bool replay)
inode->close_dirfrag(dir->get_frag());
}
+ if (is_auth() && !replay)
+ mark_complete();
+
// FIXME: merge dirty old rstat
fnode.rstat.version = rstat_version;
fnode.accounted_rstat = fnode.rstat;
@@ -1520,6 +1524,8 @@ void CDir::_omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
}
}
+ list<CInode*> undef_inodes;
+
// purge stale snaps?
// only if we have past_parents open!
bool purged_any = false;
@@ -1634,10 +1640,7 @@ void CDir::_omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
if (in) {
dout(12) << "_fetched had dentry " << *dn << dendl;
if (in->state_test(CInode::STATE_REJOINUNDEF)) {
- assert(cache->mds->is_rejoin());
- assert(in->vino() == vinodeno_t(inode.ino, last));
- in->state_clear(CInode::STATE_REJOINUNDEF);
- cache->opened_undef_inode(in);
+ undef_inodes.push_back(in);
undef_inode = true;
}
} else
@@ -1747,6 +1750,15 @@ void CDir::_omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
// mark complete, !fetching
mark_complete();
state_clear(STATE_FETCHING);
+
+ // open & force frags
+ while (!undef_inodes.empty()) {
+ CInode *in = undef_inodes.front();
+ undef_inodes.pop_front();
+ in->state_clear(CInode::STATE_REJOINUNDEF);
+ cache->opened_undef_inode(in);
+ }
+
auth_unpin(this);
// kick waiters
@@ -2102,8 +2114,6 @@ void CDir::_committed(version_t v)
void CDir::encode_export(bufferlist& bl)
{
assert(!is_projected());
- ceph_seq_t seq = mseq + 1;
- ::encode(seq, bl);
::encode(first, bl);
::encode(fnode, bl);
::encode(dirty_old_rstat, bl);
@@ -2133,7 +2143,6 @@ void CDir::finish_export(utime_t now)
void CDir::decode_import(bufferlist::iterator& blp, utime_t now, LogSegment *ls)
{
- ::decode(mseq, blp);
::decode(first, blp);
::decode(fnode, blp);
::decode(dirty_old_rstat, blp);
@@ -2164,10 +2173,24 @@ void CDir::decode_import(bufferlist::iterator& blp, utime_t now, LogSegment *ls)
// did we import some dirty scatterlock data?
if (dirty_old_rstat.size() ||
- !(fnode.rstat == fnode.accounted_rstat))
+ !(fnode.rstat == fnode.accounted_rstat)) {
cache->mds->locker->mark_updated_scatterlock(&inode->nestlock);
- if (!(fnode.fragstat == fnode.accounted_fragstat))
+ ls->dirty_dirfrag_nest.push_back(&inode->item_dirty_dirfrag_nest);
+ }
+ if (!(fnode.fragstat == fnode.accounted_fragstat)) {
cache->mds->locker->mark_updated_scatterlock(&inode->filelock);
+ ls->dirty_dirfrag_dir.push_back(&inode->item_dirty_dirfrag_dir);
+ }
+ if (is_dirty_dft()) {
+ if (inode->dirfragtreelock.get_state() != LOCK_MIX &&
+ inode->dirfragtreelock.is_stable()) {
+ // clear stale dirtydft
+ state_clear(STATE_DIRTYDFT);
+ } else {
+ cache->mds->locker->mark_updated_scatterlock(&inode->dirfragtreelock);
+ ls->dirty_dirfrag_dirfragtree.push_back(&inode->item_dirty_dirfrag_dirfragtree);
+ }
+ }
}
diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index db11c06..955a04e 100644
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -107,6 +107,7 @@ public:
static const unsigned STATE_STICKY = (1<<15); // sticky pin due to inode stickydirs
static const unsigned STATE_DNPINNEDFRAG = (1<<16); // dir is refragmenting
static const unsigned STATE_ASSIMRSTAT = (1<<17); // assimilating inode->frag rstats
+ static const unsigned STATE_DIRTYDFT = (1<<18); // dirty dirfragtree
// common states
static const unsigned STATE_CLEAN = 0;
@@ -115,7 +116,7 @@ public:
// these state bits are preserved by an import/export
// ...except if the directory is hashed, in which case none of them are!
static const unsigned MASK_STATE_EXPORTED =
- (STATE_COMPLETE|STATE_DIRTY);
+ (STATE_COMPLETE|STATE_DIRTY|STATE_DIRTYDFT);
static const unsigned MASK_STATE_IMPORT_KEPT =
(
STATE_IMPORTING
@@ -129,10 +130,10 @@ public:
|STATE_FROZENDIR
|STATE_STICKY);
static const unsigned MASK_STATE_FRAGMENT_KEPT =
- (STATE_DIRTY |
- STATE_COMPLETE |
+ (STATE_DIRTY|
STATE_EXPORTBOUND |
- STATE_IMPORTBOUND);
+ STATE_IMPORTBOUND |
+ STATE_REJOINUNDEF);
// -- rep spec --
static const int REP_NONE = 0;
@@ -170,7 +171,6 @@ public:
fnode_t fnode;
snapid_t first;
- ceph_seq_t mseq; // migrate sequence
map<snapid_t,old_rstat_t> dirty_old_rstat; // [value.first,key]
// my inodes with dirty rstat data
@@ -473,6 +473,7 @@ private:
bool is_complete() { return state & STATE_COMPLETE; }
bool is_exporting() { return state & STATE_EXPORTING; }
bool is_importing() { return state & STATE_IMPORTING; }
+ bool is_dirty_dft() { return state & STATE_DIRTYDFT; }
int get_dir_rep() { return dir_rep; }
bool is_rep() {
@@ -532,7 +533,7 @@ protected:
map< inodeno_t, list<Context*> > waiting_on_ino;
public:
- bool is_waiting_for_dentry(const char *dname, snapid_t snap) {
+ bool is_waiting_for_dentry(const string& dname, snapid_t snap) {
return waiting_on_dentry.count(string_snap_t(dname, snap));
}
void add_dentry_waiter(const string& dentry, snapid_t snap, Context *c);
@@ -555,7 +556,6 @@ public:
void encode_export(bufferlist& bl);
void finish_export(utime_t now);
void abort_export() {
- mseq += 2;
put(PIN_TEMPEXPORTING);
}
void decode_import(bufferlist::iterator& blp, utime_t now, LogSegment *ls);
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 1933dea..02cabd3 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -460,13 +460,31 @@ bool CInode::get_dirfrags_under(frag_t fg, list<CDir*>& ls)
bool all = true;
list<frag_t> fglist;
dirfragtree.get_leaves_under(fg, fglist);
- for (list<frag_t>::iterator p = fglist.begin();
- p != fglist.end();
- ++p)
+ for (list<frag_t>::iterator p = fglist.begin(); p != fglist.end(); ++p)
if (dirfrags.count(*p))
ls.push_back(dirfrags[*p]);
else
all = false;
+
+ if (all)
+ return all;
+
+ fragtree_t tmpdft;
+ tmpdft.force_to_leaf(g_ceph_context, fg);
+ for (map<frag_t,CDir*>::iterator p = dirfrags.begin(); p != dirfrags.end(); ++p) {
+ tmpdft.force_to_leaf(g_ceph_context, p->first);
+ if (fg.contains(p->first) && !dirfragtree.is_leaf(p->first))
+ ls.push_back(p->second);
+ }
+
+ all = true;
+ tmpdft.get_leaves_under(fg, fglist);
+ for (list<frag_t>::iterator p = fglist.begin(); p != fglist.end(); ++p)
+ if (!dirfrags.count(*p)) {
+ all = false;
+ break;
+ }
+
return all;
}
@@ -809,6 +827,12 @@ version_t CInode::pre_dirty()
assert(is_base());
pv = get_projected_version() + 1;
}
+ // force update backtrace for old format inode (see inode_t::decode)
+ if (inode.backtrace_version == 0 && !projected_nodes.empty()) {
+ inode_t *pi = projected_nodes.back()->inode;
+ if (pi->backtrace_version == 0)
+ pi->update_backtrace(pv);
+ }
return pv;
}
@@ -1203,6 +1227,8 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
::encode(inode.layout, bl);
::encode(inode.size, bl);
::encode(inode.client_ranges, bl);
+ ::encode(inode.inline_data, bl);
+ ::encode(inode.inline_version, bl);
}
} else {
bool dirty = filelock.is_dirty();
@@ -1225,7 +1251,6 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
dout(20) << fg << " fragstat " << pf->fragstat << dendl;
dout(20) << fg << " accounted_fragstat " << pf->accounted_fragstat << dendl;
::encode(fg, tmp);
- ::encode(dir->mseq, tmp);
::encode(dir->first, tmp);
::encode(pf->fragstat, tmp);
::encode(pf->accounted_fragstat, tmp);
@@ -1261,7 +1286,6 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
dout(10) << fg << " " << pf->rstat << dendl;
dout(10) << fg << " " << dir->dirty_old_rstat << dendl;
::encode(fg, tmp);
- ::encode(dir->mseq, tmp);
::encode(dir->first, tmp);
::encode(pf->rstat, tmp);
::encode(pf->accounted_rstat, tmp);
@@ -1378,11 +1402,14 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
dirfragtree.swap(temp);
for (map<frag_t,CDir*>::iterator p = dirfrags.begin();
p != dirfrags.end();
- ++p)
+ ++p) {
if (!dirfragtree.is_leaf(p->first)) {
dout(10) << " forcing open dirfrag " << p->first << " to leaf (racing with split|merge)" << dendl;
dirfragtree.force_to_leaf(g_ceph_context, p->first);
}
+ if (p->second->is_auth())
+ p->second->state_clear(CDir::STATE_DIRTYDFT);
+ }
}
if (g_conf->mds_debug_frag)
verify_dirfrags();
@@ -1399,6 +1426,8 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
::decode(inode.layout, p);
::decode(inode.size, p);
::decode(inode.client_ranges, p);
+ ::decode(inode.inline_data, p);
+ ::decode(inode.inline_version, p);
}
} else {
bool replica_dirty;
@@ -1420,12 +1449,10 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
dout(10) << " ...got " << n << " fragstats on " << *this << dendl;
while (n--) {
frag_t fg;
- ceph_seq_t mseq;
snapid_t fgfirst;
frag_info_t fragstat;
frag_info_t accounted_fragstat;
::decode(fg, p);
- ::decode(mseq, p);
::decode(fgfirst, p);
::decode(fragstat, p);
::decode(accounted_fragstat, p);
@@ -1438,12 +1465,6 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
assert(dir); // i am auth; i had better have this dir open
dout(10) << fg << " first " << dir->first << " -> " << fgfirst
<< " on " << *dir << dendl;
- if (dir->fnode.fragstat.version == get_projected_inode()->dirstat.version &&
- ceph_seq_cmp(mseq, dir->mseq) < 0) {
- dout(10) << " mseq " << mseq << " < " << dir->mseq << ", ignoring" << dendl;
- continue;
- }
- dir->mseq = mseq;
dir->first = fgfirst;
dir->fnode.fragstat = fragstat;
dir->fnode.accounted_fragstat = accounted_fragstat;
@@ -1488,13 +1509,11 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
::decode(n, p);
while (n--) {
frag_t fg;
- ceph_seq_t mseq;
snapid_t fgfirst;
nest_info_t rstat;
nest_info_t accounted_rstat;
map<snapid_t,old_rstat_t> dirty_old_rstat;
::decode(fg, p);
- ::decode(mseq, p);
::decode(fgfirst, p);
::decode(rstat, p);
::decode(accounted_rstat, p);
@@ -1509,12 +1528,6 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
assert(dir); // i am auth; i had better have this dir open
dout(10) << fg << " first " << dir->first << " -> " << fgfirst
<< " on " << *dir << dendl;
- if (dir->fnode.rstat.version == get_projected_inode()->rstat.version &&
- ceph_seq_cmp(mseq, dir->mseq) < 0) {
- dout(10) << " mseq " << mseq << " < " << dir->mseq << ", ignoring" << dendl;
- continue;
- }
- dir->mseq = mseq;
dir->first = fgfirst;
dir->fnode.rstat = rstat;
dir->fnode.accounted_rstat = accounted_rstat;
@@ -1640,36 +1653,10 @@ void CInode::start_scatter(ScatterLock *lock)
case CEPH_LOCK_INEST:
finish_scatter_update(lock, dir, pi->rstat.version, pf->accounted_rstat.version);
break;
- }
- }
-}
-/*
- * set dirfrag_version to inode_version - 1. so that we can use dirfrag version
- * to check if we have gathered scatter state for a given dirfrag.
- */
-void CInode::start_scatter_gather(ScatterLock *lock, int auth)
-{
- assert(is_auth());
- inode_t *pi = get_projected_inode();
-
- for (map<frag_t,CDir*>::iterator p = dirfrags.begin();
- p != dirfrags.end();
- ++p) {
- CDir *dir = p->second;
-
- if (dir->is_auth())
- continue;
- if (auth >= 0 && dir->authority().first != auth)
- continue;
-
- switch (lock->get_type()) {
- case CEPH_LOCK_IFILE:
- dir->fnode.fragstat.version = pi->dirstat.version - 1;
- break;
- case CEPH_LOCK_INEST:
- dir->fnode.rstat.version = pi->rstat.version - 1;
- break;
+ case CEPH_LOCK_IDFT:
+ dir->state_clear(CDir::STATE_DIRTYDFT);
+ break;
}
}
}
@@ -2001,6 +1988,30 @@ bool CInode::is_freezing()
return false;
}
+void CInode::add_dir_waiter(frag_t fg, Context *c)
+{
+ if (waiting_on_dir.empty())
+ get(PIN_DIRWAITER);
+ waiting_on_dir[fg].push_back(c);
+ dout(10) << "add_dir_waiter frag " << fg << " " << c << " on " << *this << dendl;
+}
+
+void CInode::take_dir_waiting(frag_t fg, list<Context*>& ls)
+{
+ if (waiting_on_dir.empty())
+ return;
+
+ map<frag_t, list<Context*> >::iterator p = waiting_on_dir.find(fg);
+ if (p != waiting_on_dir.end()) {
+ dout(10) << "take_dir_waiting frag " << fg << " on " << *this << dendl;
+ ls.splice(ls.end(), p->second);
+ waiting_on_dir.erase(p);
+
+ if (waiting_on_dir.empty())
+ put(PIN_DIRWAITER);
+ }
+}
+
void CInode::add_waiter(uint64_t tag, Context *c)
{
dout(10) << "add_waiter tag " << std::hex << tag << std::dec << " " << c
@@ -2021,6 +2032,23 @@ void CInode::add_waiter(uint64_t tag, Context *c)
MDSCacheObject::add_waiter(tag, c);
}
+void CInode::take_waiting(uint64_t mask, list<Context*>& ls)
+{
+ if ((mask & WAIT_DIR) && !waiting_on_dir.empty()) {
+ // take all dentry waiters
+ while (!waiting_on_dir.empty()) {
+ map<frag_t, list<Context*> >::iterator p = waiting_on_dir.begin();
+ dout(10) << "take_waiting dirfrag " << p->first << " on " << *this << dendl;
+ ls.splice(ls.end(), p->second);
+ waiting_on_dir.erase(p);
+ }
+ put(PIN_DIRWAITER);
+ }
+
+ // waiting
+ MDSCacheObject::take_waiting(mask, ls);
+}
+
bool CInode::freeze_inode(int auth_pin_allowance)
{
assert(auth_pin_allowance > 0); // otherwise we need to adjust parent's nested_auth_pins
@@ -2749,8 +2777,10 @@ void CInode::replicate_relax_locks()
// =============================================
int CInode::encode_inodestat(bufferlist& bl, Session *session,
- SnapRealm *dir_realm,
- snapid_t snapid, unsigned max_bytes)
+ SnapRealm *dir_realm,
+ snapid_t snapid,
+ unsigned max_bytes,
+ int getattr_caps)
{
int client = session->info.inst.name.num();
assert(snapid);
@@ -2854,11 +2884,13 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
// inline data
version_t inline_version = 0;
bufferlist inline_data;
- if (!cap || (cap->client_inline_version < i->inline_version)) {
+ if (i->inline_version == CEPH_INLINE_NONE) {
+ inline_version = CEPH_INLINE_NONE;
+ } else if ((!cap && !no_caps) ||
+ (cap && cap->client_inline_version < i->inline_version) ||
+ (getattr_caps & CEPH_CAP_FILE_RD)) { // client requests inline data
inline_version = i->inline_version;
inline_data = i->inline_data;
- if (cap)
- cap->client_inline_version = i->inline_version;
}
// nest (do same as file... :/)
@@ -2969,9 +3001,20 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
<< " xattrv " << e.xattr_version << " len " << xbl.length()
<< dendl;
+ if (inline_data.length() && cap) {
+ if ((cap->pending() | getattr_caps) & CEPH_CAP_FILE_SHARED) {
+ dout(10) << "including inline version " << inline_version << dendl;
+ cap->client_inline_version = inline_version;
+ } else {
+ dout(10) << "dropping inline version " << inline_version << dendl;
+ inline_version = 0;
+ inline_data.clear();
+ }
+ }
+
// include those xattrs?
if (xbl.length() && cap) {
- if (cap->pending() & CEPH_CAP_XATTR_SHARED) {
+ if ((cap->pending() | getattr_caps) & CEPH_CAP_XATTR_SHARED) {
dout(10) << "including xattrs version " << i->xattr_version << dendl;
cap->client_xattr_version = i->xattr_version;
} else {
@@ -3132,6 +3175,18 @@ void CInode::_encode_locks_state_for_replica(bufferlist& bl)
flocklock.encode_state_for_replica(bl);
policylock.encode_state_for_replica(bl);
}
+void CInode::_encode_locks_state_for_rejoin(bufferlist& bl, int rep)
+{
+ authlock.encode_state_for_replica(bl);
+ linklock.encode_state_for_replica(bl);
+ dirfragtreelock.encode_state_for_rejoin(bl, rep);
+ filelock.encode_state_for_rejoin(bl, rep);
+ nestlock.encode_state_for_rejoin(bl, rep);
+ xattrlock.encode_state_for_replica(bl);
+ snaplock.encode_state_for_replica(bl);
+ flocklock.encode_state_for_replica(bl);
+ policylock.encode_state_for_replica(bl);
+}
void CInode::_decode_locks_state(bufferlist::iterator& p, bool is_new)
{
authlock.decode_state(p, is_new);
@@ -3144,7 +3199,8 @@ void CInode::_decode_locks_state(bufferlist::iterator& p, bool is_new)
flocklock.decode_state(p, is_new);
policylock.decode_state(p, is_new);
}
-void CInode::_decode_locks_rejoin(bufferlist::iterator& p, list<Context*>& waiters)
+void CInode::_decode_locks_rejoin(bufferlist::iterator& p, list<Context*>& waiters,
+ list<SimpleLock*>& eval_locks)
{
authlock.decode_state_rejoin(p, waiters);
linklock.decode_state_rejoin(p, waiters);
@@ -3155,6 +3211,13 @@ void CInode::_decode_locks_rejoin(bufferlist::iterator& p, list<Context*>& waite
snaplock.decode_state_rejoin(p, waiters);
flocklock.decode_state_rejoin(p, waiters);
policylock.decode_state_rejoin(p, waiters);
+
+ if (!dirfragtreelock.is_stable() && !dirfragtreelock.is_wrlocked())
+ eval_locks.push_back(&dirfragtreelock);
+ if (!filelock.is_stable() && !filelock.is_wrlocked())
+ eval_locks.push_back(&filelock);
+ if (!nestlock.is_stable() && !nestlock.is_wrlocked())
+ eval_locks.push_back(&nestlock);
}
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 3762e9d..3977859 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -110,6 +110,7 @@ public:
static const int PIN_DIRTYRSTAT = 21;
static const int PIN_EXPORTINGCAPS = 22;
static const int PIN_DIRTYPARENT = 23;
+ static const int PIN_DIRWAITER = 24;
const char *pin_name(int p) {
switch (p) {
@@ -135,6 +136,7 @@ public:
case PIN_NEEDSNAPFLUSH: return "needsnapflush";
case PIN_DIRTYRSTAT: return "dirtyrstat";
case PIN_DIRTYPARENT: return "dirtyparent";
+ case PIN_DIRWAITER: return "dirwaiter";
default: return generic_pin_name(p);
}
}
@@ -570,10 +572,17 @@ private:
_decode_locks_state(p, is_new);
}
-
// -- waiting --
+protected:
+ map<frag_t, list<Context*> > waiting_on_dir;
+public:
+ void add_dir_waiter(frag_t fg, Context *c);
+ void take_dir_waiting(frag_t fg, list<Context*>& ls);
+ bool is_waiting_for_dir(frag_t fg) {
+ return waiting_on_dir.count(fg);
+ }
void add_waiter(uint64_t tag, Context *c);
-
+ void take_waiting(uint64_t tag, list<Context*>& ls);
// -- encode/decode helpers --
void _encode_base(bufferlist& bl);
@@ -581,9 +590,10 @@ private:
void _encode_locks_full(bufferlist& bl);
void _decode_locks_full(bufferlist::iterator& p);
void _encode_locks_state_for_replica(bufferlist& bl);
+ void _encode_locks_state_for_rejoin(bufferlist& bl, int rep);
void _decode_locks_state(bufferlist::iterator& p, bool is_new);
- void _decode_locks_rejoin(bufferlist::iterator& p, list<Context*>& waiters);
-
+ void _decode_locks_rejoin(bufferlist::iterator& p, list<Context*>& waiters,
+ list<SimpleLock*>& eval_locks);
// -- import/export --
void encode_export(bufferlist& bl);
@@ -599,7 +609,8 @@ private:
// for giving to clients
int encode_inodestat(bufferlist& bl, Session *session, SnapRealm *realm,
- snapid_t snapid=CEPH_NOSNAP, unsigned max_bytes=0);
+ snapid_t snapid=CEPH_NOSNAP, unsigned max_bytes=0,
+ int getattr_wants=0);
void encode_cap_message(MClientCaps *m, Capability *cap);
@@ -653,7 +664,6 @@ public:
void clear_scatter_dirty(); // on rejoin ack
void start_scatter(ScatterLock *lock);
- void start_scatter_gather(ScatterLock *lock, int auth=-1);
void finish_scatter_update(ScatterLock *lock, CDir *dir,
version_t inode_version, version_t dir_accounted_version);
void finish_scatter_gather_update(int type);
diff --git a/src/mds/Dumper.cc b/src/mds/Dumper.cc
index fc9b481..cb570a5 100644
--- a/src/mds/Dumper.cc
+++ b/src/mds/Dumper.cc
@@ -51,7 +51,8 @@ void Dumper::init(int rank)
inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
unsigned pg_pool = MDS_METADATA_POOL;
osdmap = new OSDMap();
- objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer);
+ objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer,
+ 0, 0);
journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC,
objecter, 0, 0, &timer);
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 4f399e3..1906c49 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -174,7 +174,8 @@ bool Locker::acquire_locks(MDRequest *mdr,
set<SimpleLock*> &wrlocks,
set<SimpleLock*> &xlocks,
map<SimpleLock*,int> *remote_wrlocks,
- CInode *auth_pin_freeze)
+ CInode *auth_pin_freeze,
+ bool auth_pin_nonblock)
{
if (mdr->done_locking &&
!mdr->is_slave()) { // not on slaves! master requests locks piecemeal.
@@ -300,10 +301,15 @@ bool Locker::acquire_locks(MDRequest *mdr,
}
if (!object->can_auth_pin()) {
// wait
- dout(10) << " can't auth_pin (freezing?), waiting to authpin " << *object << dendl;
- object->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
mds->locker->drop_locks(mdr);
mdr->drop_local_auth_pins();
+ if (auth_pin_nonblock) {
+ dout(10) << " can't auth_pin (freezing?) " << *object << ", nonblocking" << dendl;
+ mdr->aborted = true;
+ return false;
+ }
+ dout(10) << " can't auth_pin (freezing?), waiting to authpin " << *object << dendl;
+ object->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
return false;
}
}
@@ -349,6 +355,8 @@ bool Locker::acquire_locks(MDRequest *mdr,
(*q)->set_object_info(req->get_authpin_freeze());
mdr->pin(*q);
}
+ if (auth_pin_nonblock)
+ req->mark_nonblock();
mds->send_message_mds(req, p->first);
// put in waiting list
@@ -368,6 +376,8 @@ bool Locker::acquire_locks(MDRequest *mdr,
for (set<SimpleLock*, SimpleLock::ptr_lt>::iterator p = sorted.begin();
p != sorted.end();
++p) {
+ bool need_wrlock = !!wrlocks.count(*p);
+ bool need_remote_wrlock = !!(remote_wrlocks && remote_wrlocks->count(*p));
// already locked?
if (existing != mdr->locks.end() && *existing == *p) {
@@ -378,21 +388,23 @@ bool Locker::acquire_locks(MDRequest *mdr,
dout(10) << " already xlocked " << *have << " " << *have->get_parent() << dendl;
continue;
}
- if (wrlocks.count(have) && mdr->wrlocks.count(have)) {
- dout(10) << " already wrlocked " << *have << " " << *have->get_parent() << dendl;
- continue;
+ if (mdr->remote_wrlocks.count(have)) {
+ if (!need_remote_wrlock ||
+ mdr->remote_wrlocks[have] != (*remote_wrlocks)[have]) {
+ dout(10) << " unlocking remote_wrlock on wrong mds." << mdr->remote_wrlocks[have]
+ << " " << *have << " " << *have->get_parent() << dendl;
+ remote_wrlock_finish(have, mdr->remote_wrlocks[have], mdr);
+ }
}
- if (remote_wrlocks && remote_wrlocks->count(have) &&
- mdr->remote_wrlocks.count(have)) {
- if (mdr->remote_wrlocks[have] == (*remote_wrlocks)[have]) {
- dout(10) << " already remote_wrlocked " << *have << " " << *have->get_parent() << dendl;
+ if (need_wrlock || need_remote_wrlock) {
+ if (need_wrlock == !!mdr->wrlocks.count(have) &&
+ need_remote_wrlock == !!mdr->remote_wrlocks.count(have)) {
+ if (need_wrlock)
+ dout(10) << " already wrlocked " << *have << " " << *have->get_parent() << dendl;
+ if (need_remote_wrlock)
+ dout(10) << " already remote_wrlocked " << *have << " " << *have->get_parent() << dendl;
continue;
}
- dout(10) << " unlocking remote_wrlock on wrong mds." << mdr->remote_wrlocks[have]
- << " (want mds." << (*remote_wrlocks)[have] << ") "
- << *have << " " << *have->get_parent() << dendl;
- remote_wrlock_finish(have, mdr->remote_wrlocks[have], mdr);
- // continue...
}
if (rdlocks.count(have) && mdr->rdlocks.count(have)) {
dout(10) << " already rdlocked " << *have << " " << *have->get_parent() << dendl;
@@ -401,19 +413,37 @@ bool Locker::acquire_locks(MDRequest *mdr,
}
// hose any stray locks
+ if (*existing == *p) {
+ assert(need_wrlock || need_remote_wrlock);
+ SimpleLock *lock = *existing;
+ if (mdr->wrlocks.count(lock)) {
+ if (!need_wrlock)
+ dout(10) << " unlocking extra " << *lock << " " << *lock->get_parent() << dendl;
+ else if (need_remote_wrlock) // acquire remote_wrlock first
+ dout(10) << " unlocking out-of-order " << *lock << " " << *lock->get_parent() << dendl;
+ bool need_issue = false;
+ wrlock_finish(lock, mdr, &need_issue);
+ if (need_issue)
+ issue_set.insert(static_cast<CInode*>(lock->get_parent()));
+ }
+ ++existing;
+ }
while (existing != mdr->locks.end()) {
SimpleLock *stray = *existing;
++existing;
dout(10) << " unlocking out-of-order " << *stray << " " << *stray->get_parent() << dendl;
bool need_issue = false;
- if (mdr->xlocks.count(stray))
+ if (mdr->xlocks.count(stray)) {
xlock_finish(stray, mdr, &need_issue);
- else if (mdr->wrlocks.count(stray))
- wrlock_finish(stray, mdr, &need_issue);
- else if (mdr->remote_wrlocks.count(stray))
- remote_wrlock_finish(stray, mdr->remote_wrlocks[stray], mdr);
- else
+ } else if (mdr->rdlocks.count(stray)) {
rdlock_finish(stray, mdr, &need_issue);
+ } else {
+ // may have acquired both wrlock and remore wrlock
+ if (mdr->wrlocks.count(stray))
+ wrlock_finish(stray, mdr, &need_issue);
+ if (mdr->remote_wrlocks.count(stray))
+ remote_wrlock_finish(stray, mdr->remote_wrlocks[stray], mdr);
+ }
if (need_issue)
issue_set.insert(static_cast<CInode*>(stray->get_parent()));
}
@@ -426,13 +456,23 @@ bool Locker::acquire_locks(MDRequest *mdr,
if (!xlock_start(*p, mdr))
goto out;
dout(10) << " got xlock on " << **p << " " << *(*p)->get_parent() << dendl;
- } else if (wrlocks.count(*p)) {
- if (!wrlock_start(*p, mdr))
+ } else if (need_wrlock || need_remote_wrlock) {
+ if (need_remote_wrlock && !mdr->remote_wrlocks.count(*p)) {
+ remote_wrlock_start(*p, (*remote_wrlocks)[*p], mdr);
goto out;
- dout(10) << " got wrlock on " << **p << " " << *(*p)->get_parent() << dendl;
- } else if (remote_wrlocks && remote_wrlocks->count(*p)) {
- remote_wrlock_start(*p, (*remote_wrlocks)[*p], mdr);
- goto out;
+ }
+ if (need_wrlock && !mdr->wrlocks.count(*p)) {
+ if (need_remote_wrlock && !(*p)->can_wrlock(mdr->get_client())) {
+ // can't take the wrlock because the scatter lock is gathering. need to
+ // release the remote wrlock, so that the gathering process can finish.
+ remote_wrlock_finish(*p, mdr->remote_wrlocks[*p], mdr);
+ remote_wrlock_start(*p, (*remote_wrlocks)[*p], mdr);
+ goto out;
+ }
+ if (!wrlock_start(*p, mdr))
+ goto out;
+ dout(10) << " got wrlock on " << **p << " " << *(*p)->get_parent() << dendl;
+ }
} else {
if (!rdlock_start(*p, mdr))
goto out;
@@ -446,14 +486,17 @@ bool Locker::acquire_locks(MDRequest *mdr,
++existing;
dout(10) << " unlocking extra " << *stray << " " << *stray->get_parent() << dendl;
bool need_issue = false;
- if (mdr->xlocks.count(stray))
+ if (mdr->xlocks.count(stray)) {
xlock_finish(stray, mdr, &need_issue);
- else if (mdr->wrlocks.count(stray))
- wrlock_finish(stray, mdr, &need_issue);
- else if (mdr->remote_wrlocks.count(stray))
- remote_wrlock_finish(stray, mdr->remote_wrlocks[stray], mdr);
- else
+ } else if (mdr->rdlocks.count(stray)) {
rdlock_finish(stray, mdr, &need_issue);
+ } else {
+ // may have acquired both wrlock and remore wrlock
+ if (mdr->wrlocks.count(stray))
+ wrlock_finish(stray, mdr, &need_issue);
+ if (mdr->remote_wrlocks.count(stray))
+ remote_wrlock_finish(stray, mdr->remote_wrlocks[stray], mdr);
+ }
if (need_issue)
issue_set.insert(static_cast<CInode*>(stray->get_parent()));
}
@@ -513,9 +556,11 @@ void Locker::_drop_non_rdlocks(Mutation *mut, set<CInode*> *pneed_issue)
}
while (!mut->remote_wrlocks.empty()) {
- slaves.insert(mut->remote_wrlocks.begin()->second);
- mut->locks.erase(mut->remote_wrlocks.begin()->first);
- mut->remote_wrlocks.erase(mut->remote_wrlocks.begin());
+ map<SimpleLock*,int>::iterator p = mut->remote_wrlocks.begin();
+ slaves.insert(p->second);
+ if (mut->wrlocks.count(p->first) == 0)
+ mut->locks.erase(p->first);
+ mut->remote_wrlocks.erase(p);
}
while (!mut->wrlocks.empty()) {
@@ -732,9 +777,8 @@ void Locker::eval_gather(SimpleLock *lock, bool first, bool *pneed_issue, list<C
lock->get_parent()->is_replicated()) {
dout(10) << " finished (local) gather for mix->lock, now gathering from replicas" << dendl;
send_lock_message(lock, LOCK_AC_LOCK);
- lock->set_state(LOCK_MIX_LOCK2);
lock->init_gather();
- in->start_scatter_gather(static_cast<ScatterLock *>(lock));
+ lock->set_state(LOCK_MIX_LOCK2);
return;
}
@@ -1217,25 +1261,16 @@ bool Locker::rdlock_try_set(set<SimpleLock*>& locks)
return true;
}
-void Locker::rdlock_take_set(set<SimpleLock*>& locks)
+void Locker::rdlock_take_set(set<SimpleLock*>& locks, Mutation *mut)
{
dout(10) << "rdlock_take_set " << locks << dendl;
- for (set<SimpleLock*>::iterator p = locks.begin(); p != locks.end(); ++p)
- (*p)->get_rdlock();
-}
-
-void Locker::rdlock_finish_set(set<SimpleLock*>& locks)
-{
- dout(10) << "rdlock_finish_set " << locks << dendl;
for (set<SimpleLock*>::iterator p = locks.begin(); p != locks.end(); ++p) {
- bool need_issue = false;
- rdlock_finish(*p, 0, &need_issue);
- if (need_issue)
- issue_caps((CInode*)(*p)->get_parent());
+ (*p)->get_rdlock();
+ mut->rdlocks.insert(*p);
+ mut->locks.insert(*p);
}
}
-
// ------------------
// wrlock
@@ -1260,15 +1295,16 @@ bool Locker::wrlock_start(SimpleLock *lock, MDRequest *mut, bool nowait)
dout(10) << "wrlock_start " << *lock << " on " << *lock->get_parent() << dendl;
- bool want_scatter = lock->get_parent()->is_auth() &&
- (static_cast<CInode*>(lock->get_parent()))->has_subtree_root_dirfrag();
-
CInode *in = static_cast<CInode *>(lock->get_parent());
client_t client = mut->get_client();
-
+ bool want_scatter = !nowait && lock->get_parent()->is_auth() &&
+ (in->has_subtree_root_dirfrag() ||
+ static_cast<ScatterLock*>(lock)->get_scatter_wanted());
+
while (1) {
// wrlock?
- if (lock->can_wrlock(client)) {
+ if (lock->can_wrlock(client) &&
+ (!want_scatter || lock->get_state() == LOCK_MIX)) {
lock->get_wrlock();
mut->wrlocks.insert(lock);
mut->locks.insert(lock);
@@ -1325,7 +1361,8 @@ void Locker::wrlock_finish(SimpleLock *lock, Mutation *mut, bool *pneed_issue)
lock->put_wrlock();
if (mut) {
mut->wrlocks.erase(lock);
- mut->locks.erase(lock);
+ if (mut->remote_wrlocks.count(lock) == 0)
+ mut->locks.erase(lock);
}
if (!lock->is_wrlocked()) {
@@ -1368,7 +1405,8 @@ void Locker::remote_wrlock_finish(SimpleLock *lock, int target, Mutation *mut)
{
// drop ref
mut->remote_wrlocks.erase(lock);
- mut->locks.erase(lock);
+ if (mut->wrlocks.count(lock) == 0)
+ mut->locks.erase(lock);
dout(7) << "remote_wrlock_finish releasing remote wrlock on mds." << target
<< " " << *lock->get_parent() << dendl;
@@ -2941,10 +2979,6 @@ bool Locker::_do_cap_update(CInode *in, Capability *cap,
wrlock_force(&in->xattrlock, mut);
}
-
- // update backtrace for old format inode. (see inode_t::decode)
- if (pi->backtrace_version == 0)
- pi->update_backtrace();
mut->auth_pin(in);
mdcache->predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY, 0, follows);
@@ -3486,7 +3520,7 @@ bool Locker::simple_sync(SimpleLock *lock, bool *need_issue)
assert(lock->is_stable());
CInode *in = 0;
- if (lock->get_type() != CEPH_LOCK_DN)
+ if (lock->get_cap_shift())
in = static_cast<CInode *>(lock->get_parent());
int old_state = lock->get_state();
@@ -3508,11 +3542,10 @@ bool Locker::simple_sync(SimpleLock *lock, bool *need_issue)
if (lock->get_parent()->is_replicated() && old_state == LOCK_MIX) {
send_lock_message(lock, LOCK_AC_SYNC);
lock->init_gather();
- in->start_scatter_gather(static_cast<ScatterLock *>(lock));
gather++;
}
- if (lock->get_cap_shift() && in->is_head()) {
+ if (in && in->is_head()) {
if (in->issued_caps_need_gather(lock)) {
if (need_issue)
*need_issue = true;
@@ -3623,7 +3656,7 @@ void Locker::simple_lock(SimpleLock *lock, bool *need_issue)
assert(lock->get_state() != LOCK_LOCK);
CInode *in = 0;
- if (lock->get_type() != CEPH_LOCK_DN)
+ if (lock->get_cap_shift())
in = static_cast<CInode *>(lock->get_parent());
int old_state = lock->get_state();
@@ -3650,7 +3683,7 @@ void Locker::simple_lock(SimpleLock *lock, bool *need_issue)
}
if (lock->is_rdlocked())
gather++;
- if (lock->get_cap_shift() && in->is_head()) {
+ if (in && in->is_head()) {
if (in->issued_caps_need_gather(lock)) {
if (need_issue)
*need_issue = true;
@@ -3683,8 +3716,6 @@ void Locker::simple_lock(SimpleLock *lock, bool *need_issue)
gather++;
send_lock_message(lock, LOCK_AC_LOCK);
lock->init_gather();
- if (lock->get_state() == LOCK_MIX_LOCK2)
- in->start_scatter_gather(static_cast<ScatterLock *>(lock));
}
}
@@ -4090,9 +4121,8 @@ void Locker::scatter_tempsync(ScatterLock *lock, bool *need_issue)
if (lock->get_state() == LOCK_MIX_TSYN &&
in->is_replicated()) {
- send_lock_message(lock, LOCK_AC_LOCK);
lock->init_gather();
- in->start_scatter_gather(static_cast<ScatterLock *>(lock));
+ send_lock_message(lock, LOCK_AC_LOCK);
gather++;
}
@@ -4416,8 +4446,6 @@ void Locker::file_excl(ScatterLock *lock, bool *need_issue)
lock->get_state() != LOCK_XSYN_EXCL) { // if we were lock, replicas are already lock.
send_lock_message(lock, LOCK_AC_LOCK);
lock->init_gather();
- if (lock->get_state() == LOCK_MIX_EXCL)
- in->start_scatter_gather(static_cast<ScatterLock *>(lock));
gather++;
}
if (lock->is_leased()) {
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 466f7c9..6056862 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -89,7 +89,8 @@ public:
set<SimpleLock*> &wrlocks,
set<SimpleLock*> &xlocks,
map<SimpleLock*,int> *remote_wrlocks=NULL,
- CInode *auth_pin_freeze=NULL);
+ CInode *auth_pin_freeze=NULL,
+ bool auth_pin_nonblock=false);
void cancel_locking(Mutation *mut, set<CInode*> *pneed_issue);
void drop_locks(Mutation *mut, set<CInode*> *pneed_issue=0);
@@ -132,8 +133,7 @@ public:
void rdlock_finish(SimpleLock *lock, Mutation *mut, bool *pneed_issue);
bool can_rdlock_set(set<SimpleLock*>& locks);
bool rdlock_try_set(set<SimpleLock*>& locks);
- void rdlock_take_set(set<SimpleLock*>& locks);
- void rdlock_finish_set(set<SimpleLock*>& locks);
+ void rdlock_take_set(set<SimpleLock*>& locks, Mutation *mut);
void wrlock_force(SimpleLock *lock, Mutation *mut);
bool wrlock_start(SimpleLock *lock, MDRequest *mut, bool nowait=false);
diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index 6a404c4..06989b5 100644
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -100,7 +100,8 @@ void MDBalancer::tick()
}
// hash?
- if (g_conf->mds_bal_frag && g_conf->mds_bal_fragment_interval > 0 &&
+ if ((g_conf->mds_bal_frag || g_conf->mds_thrash_fragments) &&
+ g_conf->mds_bal_fragment_interval > 0 &&
now.sec() - last_fragment.sec() > g_conf->mds_bal_fragment_interval) {
last_fragment = now;
do_fragmenting();
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index eb20c06..080fe1b 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -56,6 +56,7 @@
#include "events/EImportFinish.h"
#include "events/EFragment.h"
#include "events/ECommitted.h"
+#include "events/ESessions.h"
#include "messages/MGenericMessage.h"
@@ -1064,24 +1065,40 @@ void MDCache::get_force_dirfrag_bound_set(vector<dirfrag_t>& dfs, set<CDir*>& bo
if (!diri)
continue;
dout(10) << " checking fragset " << p->second.get() << " on " << *diri << dendl;
+
+ fragtree_t tmpdft;
+ for (set<frag_t>::iterator q = p->second.begin(); q != p->second.end(); ++q)
+ tmpdft.force_to_leaf(g_ceph_context, *q);
+
for (set<frag_t>::iterator q = p->second.begin(); q != p->second.end(); ++q) {
frag_t fg = *q;
- list<CDir*> u;
- diri->get_dirfrags_under(fg, u);
- dout(10) << " frag " << fg << " contains " << u << dendl;
- if (!u.empty())
- bounds.insert(u.begin(), u.end());
- frag_t t = fg;
- while (t != frag_t()) {
- t = t.parent();
- CDir *dir = diri->get_dirfrag(t);
- if (dir) {
- // ugh, we found a containing parent
- dout(10) << " ugh, splitting parent frag " << t << " " << *dir << dendl;
- force_dir_fragment(diri, fg);
- break;
+ list<frag_t> fgls;
+ diri->dirfragtree.get_leaves_under(fg, fgls);
+ if (fgls.empty()) {
+ bool all = true;
+ frag_t approx_fg = diri->dirfragtree[fg.value()];
+ list<frag_t> ls;
+ tmpdft.get_leaves_under(approx_fg, ls);
+ for (list<frag_t>::iterator r = ls.begin(); r != ls.end(); ++r) {
+ if (p->second.get().count(*r) == 0) {
+ // not bound, so the resolve message is from auth MDS of the dirfrag
+ force_dir_fragment(diri, *r);
+ all = false;
+ }
+ }
+ if (all) {
+ fgls.push_back(approx_fg);
+ } else {
+ diri->dirfragtree.get_leaves_under(fg, fgls);
+ assert(!fgls.empty());
}
}
+ dout(10) << " frag " << fg << " contains " << fgls << dendl;
+ for (list<frag_t>::iterator r = fgls.begin(); r != fgls.end(); ++r) {
+ CDir *dir = diri->get_dirfrag(*r);
+ if (dir)
+ bounds.insert(dir);
+ }
}
}
}
@@ -1282,7 +1299,9 @@ void MDCache::adjust_subtree_after_rename(CInode *diri, CDir *olddir,
// adjust subtree
list<CDir*> dfls;
- diri->get_dirfrags(dfls);
+ // make sure subtree dirfrags are at the front of the list
+ diri->get_subtree_dirfrags(dfls);
+ diri->get_nested_dirfrags(dfls);
for (list<CDir*>::iterator p = dfls.begin(); p != dfls.end(); ++p) {
CDir *dir = *p;
@@ -1819,7 +1838,7 @@ void MDCache::project_rstat_frag_to_inode(nest_info_t& rstat, nest_info_t& accou
pi->rstat.add(delta);
dout(20) << " result [" << first << "," << last << "] " << pi->rstat << dendl;
- if (pi->rstat.rbytes < 0)
+ if (pi->rstat.rbytes < 0 && pin->dirfragtree.is_leaf(frag_t()))
assert(!"negative rstat rbytes" == g_conf->mds_verify_scatter);
last = first-1;
@@ -1916,8 +1935,7 @@ void MDCache::predirty_journal_parents(Mutation *mut, EMetaBlob *blob,
pf->version = parent->pre_dirty();
if (do_parent_mtime || linkunlink) {
- assert(mut->wrlocks.count(&pin->filelock) ||
- mut->is_slave()); // we are slave. master will have wrlocked the dir.
+ assert(mut->wrlocks.count(&pin->filelock));
assert(cfollows == CEPH_NOSNAP);
// update stale fragstat?
@@ -2777,6 +2795,10 @@ void MDCache::handle_mds_recovery(int who)
{
dout(7) << "handle_mds_recovery mds." << who << dendl;
+ // exclude all discover waiters. kick_discovers() will do the job
+ static const uint64_t i_mask = CInode::WAIT_ANY_MASK & ~CInode::WAIT_DIR;
+ static const uint64_t d_mask = CDir::WAIT_ANY_MASK & ~CDir::WAIT_DENTRY;
+
list<Context*> waiters;
// wake up any waiters in their subtrees
@@ -2797,7 +2819,7 @@ void MDCache::handle_mds_recovery(int who)
while (!q.empty()) {
CDir *d = q.front();
q.pop_front();
- d->take_waiting(CDir::WAIT_ANY_MASK, waiters);
+ d->take_waiting(d_mask, waiters);
// inode waiters too
for (CDir::map_t::iterator p = d->items.begin();
@@ -2806,7 +2828,7 @@ void MDCache::handle_mds_recovery(int who)
CDentry *dn = p->second;
CDentry::linkage_t *dnl = dn->get_linkage();
if (dnl->is_primary()) {
- dnl->get_inode()->take_waiting(CInode::WAIT_ANY_MASK, waiters);
+ dnl->get_inode()->take_waiting(i_mask, waiters);
// recurse?
list<CDir*> ls;
@@ -3592,11 +3614,24 @@ void MDCache::rejoin_send_rejoins()
}
if (mds->is_rejoin()) {
+ map<client_t, set<int> > client_exports;
for (map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> >::iterator p = cap_exports.begin();
p != cap_exports.end();
++p) {
assert(cap_export_targets.count(p->first));
- rejoins[cap_export_targets[p->first]]->cap_exports[p->first] = p->second;
+ int target = cap_export_targets[p->first];
+ rejoins[target]->cap_exports[p->first] = p->second;
+ for (map<client_t,ceph_mds_cap_reconnect>::iterator q = p->second.begin();
+ q != p->second.end();
+ ++q)
+ client_exports[q->first].insert(target);
+ }
+ for (map<client_t, set<int> >::iterator p = client_exports.begin();
+ p != client_exports.end();
+ ++p) {
+ entity_inst_t inst = mds->sessionmap.get_inst(entity_name_t::CLIENT(p->first.v));
+ for (set<int>::iterator q = p->second.begin(); q != p->second.end(); ++q)
+ rejoins[*q]->client_map[p->first] = inst;
}
}
@@ -3809,7 +3844,6 @@ void MDCache::rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin)
dout(15) << " add_strong_dirfrag " << *dir << dendl;
rejoin->add_strong_dirfrag(dir->dirfrag(), dir->get_replica_nonce(), dir->get_dir_rep());
dir->state_set(CDir::STATE_REJOINING);
- dir->mseq = 0;
for (CDir::map_t::iterator p = dir->items.begin();
p != dir->items.end();
@@ -3950,6 +3984,8 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
assert(gather_locks.empty());
// check cap exports.
+ rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end());
+
for (map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> >::iterator p = weak->cap_exports.begin();
p != weak->cap_exports.end();
++p) {
@@ -3971,15 +4007,11 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
++p) {
CInode *in = get_inode(p->first);
assert(in);
- if (survivor) {
- in->start_scatter_gather(&in->filelock, from);
- in->start_scatter_gather(&in->nestlock, from);
- } else {
- rejoin_potential_updated_scatterlocks.insert(in);
- }
in->decode_lock_state(CEPH_LOCK_IFILE, p->second.file);
in->decode_lock_state(CEPH_LOCK_INEST, p->second.nest);
in->decode_lock_state(CEPH_LOCK_IDFT, p->second.dft);
+ if (!survivor)
+ rejoin_potential_updated_scatterlocks.insert(in);
}
// recovering peer may send incorrect dirfrags here. we need to
@@ -3995,20 +4027,32 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
if (!diri)
dout(0) << " missing dir ino " << p->ino << dendl;
assert(diri);
-
- frag_t fg = diri->dirfragtree[p->frag.value()];
- CDir *dir = diri->get_dirfrag(fg);
- if (!dir)
- dout(0) << " missing dir for " << p->frag << " (which maps to " << fg << ") on " << *diri << dendl;
- assert(dir);
- if (dirs_to_share.count(dir)) {
- dout(10) << " already have " << p->frag << " -> " << fg << " " << *dir << dendl;
+
+ list<frag_t> ls;
+ if (diri->dirfragtree.is_leaf(p->frag)) {
+ ls.push_back(p->frag);
} else {
- dirs_to_share.insert(dir);
- unsigned nonce = dir->add_replica(from);
- dout(10) << " have " << p->frag << " -> " << fg << " " << *dir << dendl;
- if (ack)
- ack->add_strong_dirfrag(dir->dirfrag(), nonce, dir->dir_rep);
+ diri->dirfragtree.get_leaves_under(p->frag, ls);
+ if (ls.empty())
+ ls.push_back(diri->dirfragtree[p->frag.value()]);
+ }
+ for (list<frag_t>::iterator q = ls.begin(); q != ls.end(); ++q) {
+ frag_t fg = *q;
+ CDir *dir = diri->get_dirfrag(fg);
+ if (!dir) {
+ dout(0) << " missing dir for " << p->frag << " (which maps to " << fg << ") on " << *diri << dendl;
+ continue;
+ }
+ assert(dir);
+ if (dirs_to_share.count(dir)) {
+ dout(10) << " already have " << p->frag << " -> " << fg << " " << *dir << dendl;
+ } else {
+ dirs_to_share.insert(dir);
+ unsigned nonce = dir->add_replica(from);
+ dout(10) << " have " << p->frag << " -> " << fg << " " << *dir << dendl;
+ if (ack)
+ ack->add_strong_dirfrag(dir->dirfrag(), nonce, dir->dir_rep);
+ }
}
}
@@ -4056,7 +4100,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
assert(in);
if (survivor && in->is_replica(from))
- inode_remove_replica(in, from, gather_locks);
+ inode_remove_replica(in, from, true, gather_locks);
unsigned inonce = in->add_replica(from);
dout(10) << " have " << *in << dendl;
@@ -4067,7 +4111,9 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
if (ack) {
acked_inodes.insert(in->vino());
ack->add_inode_base(in);
- ack->add_inode_locks(in, inonce);
+ bufferlist bl;
+ in->_encode_locks_state_for_rejoin(bl, from);
+ ack->add_inode_locks(in, inonce, bl);
}
}
}
@@ -4079,14 +4125,16 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
CInode *in = get_inode(*p);
assert(in); // hmm fixme wrt stray?
if (survivor && in->is_replica(from))
- inode_remove_replica(in, from, gather_locks);
+ inode_remove_replica(in, from, true, gather_locks);
unsigned inonce = in->add_replica(from);
dout(10) << " have base " << *in << dendl;
if (ack) {
acked_inodes.insert(in->vino());
ack->add_inode_base(in);
- ack->add_inode_locks(in, inonce);
+ bufferlist bl;
+ in->_encode_locks_state_for_rejoin(bl, from);
+ ack->add_inode_locks(in, inonce, bl);
}
}
@@ -4272,7 +4320,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
if (in->is_auth() &&
in->is_replica(from) &&
(ack == NULL || acked_inodes.count(p->second->vino()) == 0)) {
- inode_remove_replica(in, from, gather_locks);
+ inode_remove_replica(in, from, false, gather_locks);
dout(10) << " rem " << *in << dendl;
}
@@ -4622,6 +4670,7 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
// for sending cache expire message
set<CInode*> isolated_inodes;
+ set<CInode*> refragged_inodes;
// dirs
for (map<dirfrag_t, MMDSCacheRejoin::dirfrag_strong>::iterator p = ack->strong_dirfrags.begin();
@@ -4629,9 +4678,17 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
++p) {
// we may have had incorrect dir fragmentation; refragment based
// on what they auth tells us.
- CDir *dir = get_force_dirfrag(p->first);
+ CInode *diri = get_inode(p->first.ino);
+ CDir *dir = NULL;
+ if (diri) {
+ dir = diri->get_dirfrag(p->first.frag);
+ if (!dir) {
+ dir = force_dir_fragment(diri, p->first.frag, false);
+ if (dir)
+ refragged_inodes.insert(dir->get_inode());
+ }
+ }
if (!dir) {
- CInode *diri = get_inode(p->first.ino);
if (!diri) {
// barebones inode; the full inode loop below will clean up.
diri = new CInode(this, false);
@@ -4727,6 +4784,19 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
}
}
+ for (set<CInode*>::iterator p = refragged_inodes.begin();
+ p != refragged_inodes.end();
+ ++p) {
+ list<CDir*> ls;
+ (*p)->get_nested_dirfrags(ls);
+ for (list<CDir*>::iterator q = ls.begin(); q != ls.end(); ++q) {
+ if ((*q)->is_auth() || ack->strong_dirfrags.count((*q)->dirfrag()))
+ continue;
+ assert((*q)->get_num_any() == 0);
+ (*p)->close_dirfrag((*q)->get_frag());
+ }
+ }
+
// full dirfrags
for (map<dirfrag_t, bufferlist>::iterator p = ack->dirfrag_bases.begin();
p != ack->dirfrag_bases.end();
@@ -4773,7 +4843,7 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
assert(in);
in->set_replica_nonce(nonce);
bufferlist::iterator q = lockbl.begin();
- in->_decode_locks_rejoin(q, rejoin_waiters);
+ in->_decode_locks_rejoin(q, rejoin_waiters, rejoin_eval_locks);
in->state_clear(CInode::STATE_REJOINING);
dout(10) << " got inode locks " << *in << dendl;
}
@@ -4816,6 +4886,17 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
assert(rejoin_ack_gather.count(from));
rejoin_ack_gather.erase(from);
if (mds->is_rejoin()) {
+
+ if (rejoin_gather.empty()) {
+ // eval unstable scatter locks after all wrlocks are rejoined.
+ while (!rejoin_eval_locks.empty()) {
+ SimpleLock *lock = rejoin_eval_locks.front();
+ rejoin_eval_locks.pop_front();
+ if (!lock->is_stable())
+ mds->locker->eval_gather(lock);
+ }
+ }
+
if (rejoin_gather.empty() && // make sure we've gotten our FULL inodes, too.
rejoin_ack_gather.empty()) {
// finally, kickstart past snap parent opens
@@ -5034,13 +5115,36 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret)
cap_imports_num_opening--;
if (cap_imports_num_opening == 0) {
- if (rejoin_gather.count(mds->get_nodeid()))
- process_imported_caps();
- else
+ if (rejoin_gather.empty())
rejoin_gather_finish();
+ else if (rejoin_gather.count(mds->get_nodeid()))
+ process_imported_caps();
}
}
+class C_MDC_RejoinSessionsOpened : public Context {
+ MDCache *cache;
+public:
+ map<client_t,entity_inst_t> client_map;
+ map<client_t,uint64_t> sseqmap;
+
+ C_MDC_RejoinSessionsOpened(MDCache *c, map<client_t,entity_inst_t>& cm) :
+ cache(c), client_map(cm) {}
+ void finish(int r) {
+ assert(r == 0);
+ cache->rejoin_open_sessions_finish(client_map, sseqmap);
+ }
+};
+
+void MDCache::rejoin_open_sessions_finish(map<client_t,entity_inst_t> client_map,
+ map<client_t,uint64_t>& sseqmap)
+{
+ dout(10) << "rejoin_open_sessions_finish" << dendl;
+ mds->server->finish_force_open_sessions(client_map, sseqmap);
+ if (rejoin_gather.empty())
+ rejoin_gather_finish();
+}
+
bool MDCache::process_imported_caps()
{
dout(10) << "process_imported_caps" << dendl;
@@ -5067,6 +5171,22 @@ bool MDCache::process_imported_caps()
// called by rejoin_gather_finish() ?
if (rejoin_gather.count(mds->get_nodeid()) == 0) {
+ // if sessions for imported caps are all open ?
+ for (map<client_t,entity_inst_t>::iterator p = rejoin_client_map.begin();
+ p != rejoin_client_map.end();
+ ++p) {
+ if (!mds->sessionmap.have_session(entity_name_t::CLIENT(p->first.v))) {
+ C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this, rejoin_client_map);
+ version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map, finish->sseqmap);
+ ESessions *le = new ESessions(pv, rejoin_client_map);
+ mds->mdlog->start_submit_entry(le, finish);
+ mds->mdlog->flush();
+ rejoin_client_map.clear();
+ return true;
+ }
+ }
+ rejoin_client_map.clear();
+
// process caps that were exported by slave rename
for (map<inodeno_t,pair<int,map<client_t,Capability::Export> > >::iterator p = rejoin_slave_exports.begin();
p != rejoin_slave_exports.end();
@@ -5532,19 +5652,8 @@ bool MDCache::open_undef_inodes_dirfrags()
CInode *diri = dir->get_inode();
if (diri->state_test(CInode::STATE_REJOINUNDEF))
continue;
- if (dir->state_test(CDir::STATE_REJOINUNDEF) && dir->get_frag() == frag_t()) {
- rejoin_undef_dirfrags.erase(dir);
- dir->state_clear(CDir::STATE_REJOINUNDEF);
- diri->force_dirfrags();
- list<CDir*> ls;
- diri->get_dirfrags(ls);
- for (list<CDir*>::iterator q = ls.begin(); q != ls.end(); ++q) {
- rejoin_undef_dirfrags.insert(*q);
- (*q)->state_set(CDir::STATE_REJOINUNDEF);
- (*q)->fetch(gather.new_sub());
- }
- continue;
- }
+ if (dir->state_test(CDir::STATE_REJOINUNDEF))
+ assert(diri->dirfragtree.is_leaf(dir->get_frag()));
dir->fetch(gather.new_sub());
}
assert(gather.has_subs());
@@ -5552,6 +5661,23 @@ bool MDCache::open_undef_inodes_dirfrags()
return true;
}
+void MDCache::opened_undef_inode(CInode *in) {
+ dout(10) << "opened_undef_inode " << *in << dendl;
+ rejoin_undef_inodes.erase(in);
+ if (in->is_dir()) {
+ if (in->has_dirfrags() && !in->dirfragtree.is_leaf(frag_t())) {
+ CDir *dir = in->get_dirfrag(frag_t());
+ assert(dir);
+ rejoin_undef_dirfrags.erase(dir);
+ in->force_dirfrags();
+ list<CDir*> ls;
+ in->get_dirfrags(ls);
+ for (list<CDir*>::iterator p = ls.begin(); p != ls.end(); ++p)
+ rejoin_undef_dirfrags.insert(*p);
+ }
+ }
+}
+
void MDCache::finish_snaprealm_reconnect(client_t client, SnapRealm *realm, snapid_t seq)
{
if (seq < realm->get_newest_seq()) {
@@ -5675,7 +5801,9 @@ void MDCache::rejoin_send_acks()
r != in->replicas_end();
++r) {
ack[r->first]->add_inode_base(in);
- ack[r->first]->add_inode_locks(in, ++r->second);
+ bufferlist bl;
+ in->_encode_locks_state_for_rejoin(bl, r->first);
+ ack[r->first]->add_inode_locks(in, ++r->second, bl);
}
// subdirs in this subtree?
@@ -5690,14 +5818,18 @@ void MDCache::rejoin_send_acks()
r != root->replicas_end();
++r) {
ack[r->first]->add_inode_base(root);
- ack[r->first]->add_inode_locks(root, ++r->second);
+ bufferlist bl;
+ root->_encode_locks_state_for_rejoin(bl, r->first);
+ ack[r->first]->add_inode_locks(root, ++r->second, bl);
}
if (myin)
for (map<int,unsigned>::iterator r = myin->replicas_begin();
r != myin->replicas_end();
++r) {
ack[r->first]->add_inode_base(myin);
- ack[r->first]->add_inode_locks(myin, ++r->second);
+ bufferlist bl;
+ myin->_encode_locks_state_for_rejoin(bl, r->first);
+ ack[r->first]->add_inode_locks(myin, ++r->second, bl);
}
// include inode base for any inodes whose scatterlocks may have updated
@@ -6237,8 +6369,41 @@ bool MDCache::trim_dentry(CDentry *dn, map<int, MCacheExpire*>& expiremap)
assert(dn->is_auth());
}
+ // adjust the dir state
+ // NOTE: we can safely remove a clean, null dentry without effecting
+ // directory completeness.
+ // (check this _before_ we unlink the inode, below!)
+ bool null_dentry = false;
+ bool clear_complete = false;
+ if (!(dnl->is_null() && dn->is_clean()))
+ clear_complete = true;
+
+ // unlink the dentry
+ if (dnl->is_remote()) {
+ // just unlink.
+ dir->unlink_inode(dn);
+ } else if (dnl->is_primary()) {
+ // expire the inode, too.
+ CInode *in = dnl->get_inode();
+ assert(in);
+ if (trim_inode(dn, in, con, expiremap))
+ return true; // purging stray instead of trimming
+ } else {
+ assert(dnl->is_null());
+ null_dentry = true;
+ }
+
// notify dentry authority?
if (!dn->is_auth()) {
+ // If null replica dentry is not readable, it's likely we will
+ // receive a MDentryLink message soon. MDentryLink message only
+ // replicates an inode, so we should avoid trimming the inode's
+ // parent dentry. This is because that unconnected replicas are
+ // problematic for subtree migration.
+ if (null_dentry && !dn->lock.can_read(-1) &&
+ !dn->get_dir()->get_inode()->is_stray())
+ return true;
+
pair<int,int> auth = dn->authority();
for (int p=0; p<2; p++) {
@@ -6257,32 +6422,6 @@ bool MDCache::trim_dentry(CDentry *dn, map<int, MCacheExpire*>& expiremap)
}
}
- // adjust the dir state
- // NOTE: we can safely remove a clean, null dentry without effecting
- // directory completeness.
- // (check this _before_ we unlink the inode, below!)
- bool clear_complete = false;
- if (!(dnl->is_null() && dn->is_clean()))
- clear_complete = true;
-
- // unlink the dentry
- if (dnl->is_remote()) {
- // just unlink.
- dir->unlink_inode(dn);
- }
- else if (dnl->is_primary()) {
- // expire the inode, too.
- CInode *in = dnl->get_inode();
- assert(in);
- trim_inode(dn, in, con, expiremap);
- // purging stray instead of trimming ?
- if (dn->get_num_ref() > 0)
- return true;
- }
- else {
- assert(dnl->is_null());
- }
-
// remove dentry
if (dir->is_auth())
dir->add_to_bloom(dn);
@@ -6345,18 +6484,28 @@ void MDCache::trim_dirfrag(CDir *dir, CDir *con, map<int, MCacheExpire*>& expire
in->close_dirfrag(dir->dirfrag().frag);
}
-void MDCache::trim_inode(CDentry *dn, CInode *in, CDir *con, map<int, MCacheExpire*>& expiremap)
+bool MDCache::trim_inode(CDentry *dn, CInode *in, CDir *con, map<int, MCacheExpire*>& expiremap)
{
dout(15) << "trim_inode " << *in << dendl;
assert(in->get_num_ref() == 0);
-
- // DIR
- list<CDir*> dfls;
- in->get_dirfrags(dfls);
- for (list<CDir*>::iterator p = dfls.begin();
- p != dfls.end();
- ++p)
- trim_dirfrag(*p, con ? con:*p, expiremap); // if no container (e.g. root dirfrag), use *p
+
+ if (in->is_dir()) {
+ // If replica inode's dirfragtreelock is not readable, it's likely
+ // some dirfrags of the inode are being fragmented and we will receive
+ // MMDSFragmentNotify soon. MMDSFragmentNotify only replicates the new
+ // dirfrags, so we should avoid trimming these dirfrags' parent inode.
+ // This is because that unconnected replicas are problematic for
+ // subtree migration.
+ //
+ if (!in->is_auth() && !in->dirfragtreelock.can_read(-1))
+ return true;
+
+ // DIR
+ list<CDir*> dfls;
+ in->get_dirfrags(dfls);
+ for (list<CDir*>::iterator p = dfls.begin(); p != dfls.end(); ++p)
+ trim_dirfrag(*p, con ? con:*p, expiremap); // if no container (e.g. root dirfrag), use *p
+ }
// INODE
if (in->is_auth()) {
@@ -6364,7 +6513,7 @@ void MDCache::trim_inode(CDentry *dn, CInode *in, CDir *con, map<int, MCacheExpi
if (dn) {
maybe_eval_stray(in);
if (dn->get_num_ref() > 0)
- return;
+ return true;
}
} else {
pair<int,int> auth = in->authority();
@@ -6406,6 +6555,7 @@ void MDCache::trim_inode(CDentry *dn, CInode *in, CDir *con, map<int, MCacheExpi
if (dn)
dn->get_dir()->unlink_inode(dn);
remove_inode(in);
+ return false;
}
@@ -6742,7 +6892,7 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
// remove from our cached_by
dout(7) << " inode expire on " << *in << " from mds." << from
<< " cached_by was " << in->get_replicas() << dendl;
- inode_remove_replica(in, from, gather_locks);
+ inode_remove_replica(in, from, false, gather_locks);
}
else {
// this is an old nonce, ignore expire.
@@ -6761,10 +6911,19 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
unsigned nonce = it->second;
if (!dir) {
- dout(0) << " dir expire on " << it->first << " from " << from
+ CInode *diri = get_inode(it->first.ino);
+ if (diri) {
+ CDir *other = diri->get_approx_dirfrag(it->first.frag);
+ if (other) {
+ dout(7) << " dir expire on dirfrag " << it->first << " from mds." << from
+ << " have " << *other << ", mismatched frags, dropping" << dendl;
+ continue;
+ }
+ }
+ dout(0) << " dir expire on " << it->first << " from " << from
<< ", don't have it" << dendl;
assert(dir);
- }
+ }
assert(dir->is_auth());
// check nonce
@@ -6814,8 +6973,12 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
dn = dir->lookup(p->first.first, p->first.second);
}
- if (!dn)
- dout(0) << " missing dentry for " << p->first.first << " snap " << p->first.second << " in " << *dir << dendl;
+ if (!dn) {
+ if (dir)
+ dout(0) << " missing dentry for " << p->first.first << " snap " << p->first.second << " in " << *dir << dendl;
+ else
+ dout(0) << " missing dentry for " << p->first.first << " snap " << p->first.second << dendl;
+ }
assert(dn);
if (nonce == dn->get_replica_nonce(from)) {
@@ -6860,7 +7023,8 @@ void MDCache::discard_delayed_expire(CDir *dir)
delayed_expire.erase(dir);
}
-void MDCache::inode_remove_replica(CInode *in, int from, set<SimpleLock *>& gather_locks)
+void MDCache::inode_remove_replica(CInode *in, int from, bool rejoin,
+ set<SimpleLock *>& gather_locks)
{
in->remove_replica(from);
in->mds_caps_wanted.erase(from);
@@ -6869,14 +7033,17 @@ void MDCache::inode_remove_replica(CInode *in, int from, set<SimpleLock *>& gath
// fix lock
if (in->authlock.remove_replica(from)) gather_locks.insert(&in->authlock);
if (in->linklock.remove_replica(from)) gather_locks.insert(&in->linklock);
- if (in->dirfragtreelock.remove_replica(from)) gather_locks.insert(&in->dirfragtreelock);
- if (in->filelock.remove_replica(from)) gather_locks.insert(&in->filelock);
if (in->snaplock.remove_replica(from)) gather_locks.insert(&in->snaplock);
if (in->xattrlock.remove_replica(from)) gather_locks.insert(&in->xattrlock);
-
- if (in->nestlock.remove_replica(from)) gather_locks.insert(&in->nestlock);
if (in->flocklock.remove_replica(from)) gather_locks.insert(&in->flocklock);
if (in->policylock.remove_replica(from)) gather_locks.insert(&in->policylock);
+
+ // If 'rejoin' is true and the scatter lock is in LOCK_MIX_* state.
+ // Don't remove the recovering mds from lock's gathering list because
+ // it may hold rejoined wrlocks.
+ if (in->dirfragtreelock.remove_replica(from, rejoin)) gather_locks.insert(&in->dirfragtreelock);
+ if (in->filelock.remove_replica(from, rejoin)) gather_locks.insert(&in->filelock);
+ if (in->nestlock.remove_replica(from, rejoin)) gather_locks.insert(&in->nestlock);
}
void MDCache::dentry_remove_replica(CDentry *dn, int from, set<SimpleLock *>& gather_locks)
@@ -8185,25 +8352,9 @@ void MDCache::_open_ino_traverse_dir(inodeno_t ino, open_ino_info_t& info, int r
void MDCache::_open_ino_fetch_dir(inodeno_t ino, MMDSOpenIno *m, CDir *dir)
{
- if (dir->state_test(CDir::STATE_REJOINUNDEF) && dir->get_frag() == frag_t()) {
- rejoin_undef_dirfrags.erase(dir);
- dir->state_clear(CDir::STATE_REJOINUNDEF);
-
- CInode *diri = dir->get_inode();
- diri->force_dirfrags();
- list<CDir*> ls;
- diri->get_dirfrags(ls);
-
- C_GatherBuilder gather(g_ceph_context, _open_ino_get_waiter(ino, m));
- for (list<CDir*>::iterator p = ls.begin(); p != ls.end(); ++p) {
- rejoin_undef_dirfrags.insert(*p);
- (*p)->state_set(CDir::STATE_REJOINUNDEF);
- (*p)->fetch(gather.new_sub());
- }
- assert(gather.has_subs());
- gather.activate();
- } else
- dir->fetch(_open_ino_get_waiter(ino, m));
+ if (dir->state_test(CDir::STATE_REJOINUNDEF))
+ assert(dir->get_inode()->dirfragtree.is_leaf(dir->get_frag()));
+ dir->fetch(_open_ino_get_waiter(ino, m));
}
int MDCache::open_ino_traverse_dir(inodeno_t ino, MMDSOpenIno *m,
@@ -8257,10 +8408,9 @@ int MDCache::open_ino_traverse_dir(inodeno_t ino, MMDSOpenIno *m,
}
if (dir) {
inodeno_t next_ino = i > 0 ? ancestors[i - 1].dirino : ino;
+ CDentry *dn = dir->lookup(name);
+ CDentry::linkage_t *dnl = dn ? dn->get_linkage() : NULL;
if (dir->is_auth()) {
- CDentry *dn = dir->lookup(name);
- CDentry::linkage_t *dnl = dn ? dn->get_linkage() : NULL;
-
if (dnl && dnl->is_primary() &&
dnl->get_inode()->state_test(CInode::STATE_REJOINUNDEF)) {
dout(10) << " fetching undef " << *dnl->get_inode() << dendl;
@@ -8279,9 +8429,20 @@ int MDCache::open_ino_traverse_dir(inodeno_t ino, MMDSOpenIno *m,
if (i == 0)
err = -ENOENT;
} else if (discover) {
- discover_ino(dir, next_ino, _open_ino_get_waiter(ino, m),
- (i == 0 && want_xlocked));
- return 1;
+ if (!dnl) {
+ filepath path(name, 0);
+ discover_path(dir, CEPH_NOSNAP, path, _open_ino_get_waiter(ino, m),
+ (i == 0 && want_xlocked));
+ return 1;
+ }
+ if (dnl->is_null() && !dn->lock.can_read(-1)) {
+ dout(10) << " null " << *dn << " is not readable, waiting" << dendl;
+ dn->lock.add_waiter(SimpleLock::WAIT_RD, _open_ino_get_waiter(ino, m));
+ return 1;
+ }
+ dout(10) << " no ino " << next_ino << " in " << *dir << dendl;
+ if (i == 0)
+ err = -ENOENT;
}
}
if (hint && i == 0)
@@ -8485,8 +8646,10 @@ void MDCache::open_ino(inodeno_t ino, int64_t pool, Context* fin,
if (diri) {
frag_t fg = diri->pick_dirfrag(info.ancestors[0].dname);
CDir *dir = diri->get_dirfrag(fg);
- if (dir && !dir->is_auth())
- discover_ino(dir, ino, NULL, true);
+ if (dir && !dir->is_auth()) {
+ filepath path(info.ancestors[0].dname, 0);
+ discover_path(dir, CEPH_NOSNAP, path, NULL, true);
+ }
}
}
info.want_xlocked = true;
@@ -8756,6 +8919,9 @@ void MDCache::dispatch_request(MDRequest *mdr)
case CEPH_MDS_OP_FRAGMENTDIR:
dispatch_fragment_dir(mdr);
break;
+ case CEPH_MDS_OP_EXPORTDIR:
+ migrator->dispatch_export_dir(mdr);
+ break;
default:
assert(0);
}
@@ -9726,7 +9892,7 @@ void MDCache::discover_dir_frag(CInode *base,
dout(7) << "discover_dir_frag " << df
<< " from mds." << from << dendl;
- if (!base->is_waiter_for(CInode::WAIT_DIR) || !onfinish) { // FIXME: this is kind of weak!
+ if (!base->is_waiting_for_dir(approx_fg) || !onfinish) {
discover_info_t& d = _create_discover(from);
d.ino = base->ino();
d.frag = approx_fg;
@@ -9735,7 +9901,7 @@ void MDCache::discover_dir_frag(CInode *base,
}
if (onfinish)
- base->add_waiter(CInode::WAIT_DIR, onfinish);
+ base->add_dir_waiter(approx_fg, onfinish);
}
struct C_MDC_RetryDiscoverPath : public Context {
@@ -9778,10 +9944,12 @@ void MDCache::discover_path(CInode *base,
return;
}
+ frag_t fg = base->pick_dirfrag(want_path[0]);
if ((want_xlocked && want_path.depth() == 1) ||
- !base->is_waiter_for(CInode::WAIT_DIR) || !onfinish) { // FIXME: weak!
+ !base->is_waiting_for_dir(fg) || !onfinish) {
discover_info_t& d = _create_discover(from);
d.ino = base->ino();
+ d.frag = fg;
d.snap = snap;
d.want_path = want_path;
d.want_base_dir = true;
@@ -9791,7 +9959,7 @@ void MDCache::discover_path(CInode *base,
// register + wait
if (onfinish)
- base->add_waiter(CInode::WAIT_DIR, onfinish);
+ base->add_dir_waiter(fg, onfinish);
}
struct C_MDC_RetryDiscoverPath2 : public Context {
@@ -9904,8 +10072,11 @@ void MDCache::kick_discovers(int who)
{
for (map<tid_t,discover_info_t>::iterator p = discovers.begin();
p != discovers.end();
- ++p)
+ ++p) {
+ if (p->second.mds != who)
+ continue;
_send_discover(p->second);
+ }
}
@@ -9945,6 +10116,7 @@ void MDCache::handle_discover(MDiscover *dis)
<< dendl;
cur = get_inode(dis->get_base_ino());
+ assert(cur);
// add root
reply->starts_with = MDiscoverReply::INODE;
@@ -10004,8 +10176,10 @@ void MDCache::handle_discover(MDiscover *dis)
fg = cur->pick_dirfrag(dis->get_dentry(i));
} else {
// requester explicity specified the frag
- fg = dis->get_base_dir_frag();
assert(dis->wants_base_dir() || dis->get_want_ino() || MDS_INO_IS_BASE(dis->get_base_ino()));
+ fg = dis->get_base_dir_frag();
+ if (!cur->dirfragtree.is_leaf(fg))
+ fg = cur->dirfragtree[fg.value()];
}
CDir *curdir = cur->get_dirfrag(fg);
@@ -10051,15 +10225,19 @@ void MDCache::handle_discover(MDiscover *dis)
// is dir frozen?
if (curdir->is_frozen()) {
- if (reply->is_empty()) {
- dout(7) << *curdir << " is frozen, empty reply, waiting" << dendl;
- curdir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryMessage(mds, dis));
- reply->put();
- return;
- } else {
+ if (dis->wants_base_dir() && dis->get_base_dir_frag() != curdir->get_frag()) {
+ dout(7) << *curdir << " is frozen, dirfrag mismatch, stopping" << dendl;
+ reply->set_flag_error_dir();
+ break;
+ }
+ if (!reply->is_empty()) {
dout(7) << *curdir << " is frozen, non-empty reply, stopping" << dendl;
break;
}
+ dout(7) << *curdir << " is frozen, empty reply, waiting" << dendl;
+ curdir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryMessage(mds, dis));
+ reply->put();
+ return;
}
// add dir
@@ -10288,9 +10466,13 @@ void MDCache::handle_discover_reply(MDiscoverReply *m)
// dir
frag_t fg;
CDir *curdir = 0;
- if (next == MDiscoverReply::DIR)
+ if (next == MDiscoverReply::DIR) {
curdir = add_replica_dir(p, cur, m->get_source().num(), finished);
- else {
+ if (cur->ino() == m->get_base_ino() && curdir->get_frag() != m->get_base_dir_frag()) {
+ assert(m->get_wanted_base_dir());
+ cur->take_dir_waiting(m->get_base_dir_frag(), finished);
+ }
+ } else {
// note: this can only happen our first way around this loop.
if (p.end() && m->is_flag_error_dn()) {
fg = cur->pick_dirfrag(m->get_error_dentry());
@@ -10325,46 +10507,36 @@ void MDCache::handle_discover_reply(MDiscoverReply *m)
if (who >= 0)
dout(7) << " dir_auth_hint is " << m->get_dir_auth_hint() << dendl;
- // try again?
- if (m->get_error_dentry().length()) {
- // wanted a dentry
- frag_t fg = cur->pick_dirfrag(m->get_error_dentry());
- CDir *dir = cur->get_dirfrag(fg);
- filepath relpath(m->get_error_dentry(), 0);
-
- if (cur->is_waiter_for(CInode::WAIT_DIR)) {
- if (cur->is_auth() || dir)
- cur->take_waiting(CInode::WAIT_DIR, finished);
- else
- discover_path(cur, m->get_wanted_snapid(), relpath, 0, m->get_wanted_xlocked(), who);
- } else
- dout(7) << " doing nothing, nobody is waiting for dir" << dendl;
-
- if (dir) {
- // don't actaully need the hint, now
- if (dir->is_waiting_for_dentry(m->get_error_dentry().c_str(), m->get_wanted_snapid())) {
- if (dir->is_auth() || dir->lookup(m->get_error_dentry()))
- dir->take_dentry_waiting(m->get_error_dentry(), m->get_wanted_snapid(),
- m->get_wanted_snapid(), finished);
- else
- discover_path(dir, m->get_wanted_snapid(), relpath, 0, m->get_wanted_xlocked());
- } else
- dout(7) << " doing nothing, have dir but nobody is waiting on dentry "
- << m->get_error_dentry() << dendl;
- }
- } else {
- // wanted dir or ino
- frag_t fg = m->get_base_dir_frag();
- CDir *dir = cur->get_dirfrag(fg);
+ frag_t fg = m->get_base_dir_frag();
+ CDir *dir = cur->get_dirfrag(fg);
- if (cur->is_waiter_for(CInode::WAIT_DIR)) {
- if (cur->is_auth() || dir)
+ if (m->get_wanted_base_dir()) {
+ if (cur->is_waiting_for_dir(fg)) {
+ if (cur->is_auth())
cur->take_waiting(CInode::WAIT_DIR, finished);
+ else if (dir || !cur->dirfragtree.is_leaf(fg))
+ cur->take_dir_waiting(fg, finished);
else
discover_dir_frag(cur, fg, 0, who);
} else
dout(7) << " doing nothing, nobody is waiting for dir" << dendl;
+ }
+ // try again?
+ if (m->get_error_dentry().length()) {
+ // wanted a dentry
+ if (dir && dir->is_waiting_for_dentry(m->get_error_dentry(), m->get_wanted_snapid())) {
+ if (dir->is_auth() || dir->lookup(m->get_error_dentry())) {
+ dir->take_dentry_waiting(m->get_error_dentry(), m->get_wanted_snapid(),
+ m->get_wanted_snapid(), finished);
+ } else {
+ filepath relpath(m->get_error_dentry(), 0);
+ discover_path(dir, m->get_wanted_snapid(), relpath, 0, m->get_wanted_xlocked());
+ }
+ } else
+ dout(7) << " doing nothing, have dir but nobody is waiting on dentry "
+ << m->get_error_dentry() << dendl;
+ } else {
if (dir && m->get_wanted_ino() && dir->is_waiting_for_ino(m->get_wanted_ino())) {
if (dir->is_auth() || get_inode(m->get_wanted_ino()))
dir->take_ino_waiting(m->get_wanted_ino(), finished);
@@ -10424,7 +10596,7 @@ CDir *MDCache::add_replica_dir(bufferlist::iterator& p, CInode *diri, int from,
dout(7) << "add_replica_dir added " << *dir << " nonce " << dir->replica_nonce << dendl;
// get waiters
- diri->take_waiting(CInode::WAIT_DIR, finished);
+ diri->take_dir_waiting(df.frag, finished);
}
return dir;
@@ -10681,16 +10853,8 @@ void MDCache::handle_dentry_link(MDentryLink *m)
::decode(d_type, p);
dir->link_remote_inode(dn, ino, d_type);
}
- } else if (m->get_is_primary()) {
- CInode *in = add_replica_inode(p, NULL, finished);
- assert(in->get_num_ref() == 0);
- assert(in->get_parent_dn() == NULL);
- map<int, MCacheExpire*> expiremap;
- int from = m->get_source().num();
- expiremap[from] = new MCacheExpire(mds->get_nodeid());
- expiremap[from]->add_inode(m->get_subtree(), in->vino(), in->get_replica_nonce());
- send_expire_messages(expiremap);
- remove_inode(in);
+ } else {
+ assert(0);
}
if (!finished.empty())
@@ -10827,7 +10991,7 @@ void MDCache::adjust_dir_fragments(CInode *diri, frag_t basefrag, int bits,
adjust_dir_fragments(diri, srcfrags, basefrag, bits, resultfrags, waiters, replay);
}
-CDir *MDCache::force_dir_fragment(CInode *diri, frag_t fg)
+CDir *MDCache::force_dir_fragment(CInode *diri, frag_t fg, bool replay)
{
CDir *dir = diri->get_dirfrag(fg);
if (dir)
@@ -10846,7 +11010,7 @@ CDir *MDCache::force_dir_fragment(CInode *diri, frag_t fg)
int split = fg.bits() - parent.bits();
dout(10) << " splitting parent by " << split << " " << *pdir << dendl;
src.push_back(pdir);
- adjust_dir_fragments(diri, src, parent, split, result, waiters, true);
+ adjust_dir_fragments(diri, src, parent, split, result, waiters, replay);
dir = diri->get_dirfrag(fg);
if (dir)
dout(10) << "force_dir_fragment result " << *dir << dendl;
@@ -10866,7 +11030,7 @@ CDir *MDCache::force_dir_fragment(CInode *diri, frag_t fg)
return NULL;
}
dout(10) << " will combine frags under " << fg << ": " << src << dendl;
- adjust_dir_fragments(diri, src, fg, 0, result, waiters, true);
+ adjust_dir_fragments(diri, src, fg, 0, result, waiters, replay);
dir = result.front();
dout(10) << "force_dir_fragment result " << *dir << dendl;
return dir;
@@ -10991,13 +11155,12 @@ void MDCache::adjust_dir_fragments(CInode *diri,
class C_MDC_FragmentFrozen : public Context {
MDCache *mdcache;
- list<CDir*> dirs;
- frag_t basefrag;
- int by;
+ dirfrag_t basedirfrag;
public:
- C_MDC_FragmentFrozen(MDCache *m, list<CDir*> d, frag_t bf, int b) : mdcache(m), dirs(d), basefrag(bf), by(b) {}
+ C_MDC_FragmentFrozen(MDCache *m, dirfrag_t df) :
+ mdcache(m), basedirfrag(df) {}
virtual void finish(int r) {
- mdcache->fragment_frozen(dirs, basefrag, by);
+ mdcache->fragment_frozen(basedirfrag, r);
}
};
@@ -11049,8 +11212,13 @@ void MDCache::split_dir(CDir *dir, int bits)
if (!can_fragment(diri, dirs))
return;
- C_GatherBuilder gather(g_ceph_context,
- new C_MDC_FragmentFrozen(this, dirs, dir->get_frag(), bits));
+ assert(fragments.count(dir->dirfrag()) == 0);
+ fragment_info_t& info = fragments[dir->dirfrag()];
+ info.dirs.push_back(dir);
+ info.bits = bits;
+ info.last_cum_auth_pins_change = ceph_clock_now(g_ceph_context);
+
+ C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentFrozen(this, dir->dirfrag()));
fragment_freeze_dirs(dirs, gather);
gather.activate();
@@ -11080,8 +11248,15 @@ void MDCache::merge_dir(CInode *diri, frag_t frag)
int bits = first->get_frag().bits() - frag.bits();
dout(10) << " we are merginb by " << bits << " bits" << dendl;
+ dirfrag_t df(diri->ino(), frag);
+ assert(fragments.count(df) == 0);
+ fragment_info_t& info = fragments[df];
+ info.dirs = dirs;
+ info.bits = -bits;
+ info.last_cum_auth_pins_change = ceph_clock_now(g_ceph_context);
+
C_GatherBuilder gather(g_ceph_context,
- new C_MDC_FragmentFrozen(this, dirs, frag, -bits));
+ new C_MDC_FragmentFrozen(this, dirfrag_t(diri->ino(), frag)));
fragment_freeze_dirs(dirs, gather);
gather.activate();
@@ -11180,6 +11355,70 @@ void MDCache::fragment_unmark_unfreeze_dirs(list<CDir*>& dirs)
}
}
+void MDCache::fragment_freeze_inc_num_waiters(CDir *dir)
+{
+ map<dirfrag_t,fragment_info_t>::iterator p;
+ for (p = fragments.lower_bound(dirfrag_t(dir->ino(), 0));
+ p != fragments.end() && p->first.ino == dir->ino();
+ ++p) {
+ if (p->first.frag.contains(dir->get_frag())) {
+ p->second.num_remote_waiters++;
+ return;
+ }
+ }
+ assert(0);
+}
+
+void MDCache::find_stale_fragment_freeze()
+{
+ dout(10) << "find_stale_fragment_freeze" << dendl;
+ // see comment in Migrator::find_stale_export_freeze()
+ utime_t now = ceph_clock_now(g_ceph_context);
+ utime_t cutoff = now;
+ cutoff -= g_conf->mds_freeze_tree_timeout;
+
+ for (map<dirfrag_t,fragment_info_t>::iterator p = fragments.begin();
+ p != fragments.end(); ) {
+ dirfrag_t df = p->first;
+ fragment_info_t& info = p->second;
+ ++p;
+ if (info.dirs_frozen)
+ continue;
+ CDir *dir;
+ int total_auth_pins = 0;
+ for (list<CDir*>::iterator q = info.dirs.begin();
+ q != info.dirs.end();
+ ++q) {
+ dir = *q;
+ if (!dir->state_test(CDir::STATE_DNPINNEDFRAG)) {
+ total_auth_pins = -1;
+ break;
+ }
+ if (dir->is_frozen_dir())
+ continue;
+ total_auth_pins += dir->get_auth_pins() + dir->get_dir_auth_pins();
+ }
+ if (total_auth_pins < 0)
+ continue;
+ if (info.last_cum_auth_pins != total_auth_pins) {
+ info.last_cum_auth_pins = total_auth_pins;
+ info.last_cum_auth_pins_change = now;
+ continue;
+ }
+ if (info.last_cum_auth_pins_change >= cutoff)
+ continue;
+ dir = info.dirs.front();
+ if (info.num_remote_waiters > 0 ||
+ (!dir->inode->is_root() && dir->get_parent_dir()->is_freezing())) {
+ dout(10) << " cancel fragmenting " << df << " bit " << info.bits << dendl;
+ list<CDir*> dirs;
+ dirs.swap(info.dirs);
+ fragments.erase(df);
+ fragment_unmark_unfreeze_dirs(dirs);
+ }
+ }
+}
+
class C_MDC_FragmentPrep : public Context {
MDCache *mdcache;
MDRequest *mdr;
@@ -11205,8 +11444,8 @@ class C_MDC_FragmentCommit : public Context {
dirfrag_t basedirfrag;
list<CDir*> resultfrags;
public:
- C_MDC_FragmentCommit(MDCache *m, inodeno_t ino, frag_t f, list<CDir*>& l) :
- mdcache(m), basedirfrag(ino, f) {
+ C_MDC_FragmentCommit(MDCache *m, dirfrag_t df, list<CDir*>& l) :
+ mdcache(m), basedirfrag(df) {
resultfrags.swap(l);
}
virtual void finish(int r) {
@@ -11229,65 +11468,63 @@ public:
}
};
-void MDCache::fragment_frozen(list<CDir*>& dirs, frag_t basefrag, int bits)
+void MDCache::fragment_frozen(dirfrag_t basedirfrag, int r)
{
- dout(10) << "fragment_frozen " << dirs << " " << basefrag << " by " << bits
- << " on " << dirs.front()->get_inode() << dendl;
+ map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
+ if (r < 0) {
+ dout(7) << "fragment_frozen " << basedirfrag << " must have aborted" << dendl;
+ assert(it == fragments.end());
+ return;
+ }
+ assert(it != fragments.end());
+ fragment_info_t& info = it->second;
- if (bits > 0)
- assert(dirs.size() == 1);
- else if (bits < 0)
- assert(dirs.size() > 1);
- else
- assert(0);
+ dout(10) << "fragment_frozen " << basedirfrag.frag << " by " << info.bits
+ << " on " << info.dirs.front()->get_inode() << dendl;
- MDRequest *mdr = request_start_internal(CEPH_MDS_OP_FRAGMENTDIR);
- fragment_info_t &info = fragment_requests[mdr->reqid];
- info.basefrag = basefrag;
- info.bits = bits;
- info.dirs = dirs;
+ info.dirs_frozen = true;
+ MDRequest *mdr = request_start_internal(CEPH_MDS_OP_FRAGMENTDIR);
+ mdr->more()->fragment_base = basedirfrag;
dispatch_fragment_dir(mdr);
}
void MDCache::dispatch_fragment_dir(MDRequest *mdr)
{
- map<metareqid_t, fragment_info_t>::iterator it = fragment_requests.find(mdr->reqid);
- assert(it != fragment_requests.end());
- fragment_info_t &info = it->second;
+ dirfrag_t basedirfrag = mdr->more()->fragment_base;
+ map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
+ assert(it != fragments.end());
+ fragment_info_t& info = it->second;
CInode *diri = info.dirs.front()->get_inode();
- dout(10) << "dispatch_fragment_dir " << info.resultfrags << " "
- << info.basefrag << " bits " << info.bits << " on " << *diri << dendl;
-
- // avoid freeze dir deadlock
- if (!mdr->is_auth_pinned(diri)) {
- if (!diri->can_auth_pin()) {
- dout(10) << " can't auth_pin " << *diri << ", requeuing dir "
- << info.dirs.front()->dirfrag() << dendl;
- if (info.bits > 0)
- mds->balancer->queue_split(info.dirs.front());
- else
- mds->balancer->queue_merge(info.dirs.front());
- fragment_unmark_unfreeze_dirs(info.dirs);
- fragment_requests.erase(mdr->reqid);
- request_finish(mdr);
- return;
- }
- mdr->auth_pin(diri);
+ dout(10) << "dispatch_fragment_dir " << basedirfrag << " bits " << info.bits
+ << " on " << *diri << dendl;
+ if (!mdr->aborted) {
+ set<SimpleLock*> rdlocks, wrlocks, xlocks;
+ wrlocks.insert(&diri->dirfragtreelock);
+ // prevent a racing gather on any other scatterlocks too
+ wrlocks.insert(&diri->nestlock);
+ wrlocks.insert(&diri->filelock);
+ if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks, NULL, NULL, true))
+ if (!mdr->aborted)
+ return;
}
- set<SimpleLock*> rdlocks, wrlocks, xlocks;
- wrlocks.insert(&diri->dirfragtreelock);
- // prevent a racing gather on any other scatterlocks too
- wrlocks.insert(&diri->nestlock);
- wrlocks.insert(&diri->filelock);
- if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+ if (mdr->aborted) {
+ dout(10) << " can't auth_pin " << *diri << ", requeuing dir "
+ << info.dirs.front()->dirfrag() << dendl;
+ if (info.bits > 0)
+ mds->balancer->queue_split(info.dirs.front());
+ else
+ mds->balancer->queue_merge(info.dirs.front());
+ fragment_unmark_unfreeze_dirs(info.dirs);
+ fragments.erase(it);
+ request_finish(mdr);
return;
+ }
mdr->ls = mds->mdlog->get_current_segment();
- EFragment *le = new EFragment(mds->mdlog, EFragment::OP_PREPARE, diri->ino(),
- info.basefrag, info.bits);
+ EFragment *le = new EFragment(mds->mdlog, EFragment::OP_PREPARE, basedirfrag, info.bits);
mds->mdlog->start_entry(le);
for (list<CDir*>::iterator p = info.dirs.begin(); p != info.dirs.end(); ++p) {
@@ -11299,7 +11536,7 @@ void MDCache::dispatch_fragment_dir(MDRequest *mdr)
// refragment
list<Context*> waiters;
- adjust_dir_fragments(diri, info.dirs, info.basefrag, info.bits,
+ adjust_dir_fragments(diri, info.dirs, basedirfrag.frag, info.bits,
info.resultfrags, waiters, false);
if (g_conf->mds_debug_frag)
diri->verify_dirfrags();
@@ -11312,13 +11549,25 @@ void MDCache::dispatch_fragment_dir(MDRequest *mdr)
for (list<CDir*>::iterator p = info.resultfrags.begin();
p != info.resultfrags.end();
++p) {
- le->metablob.add_dir(*p, false);
+ if (diri->is_auth()) {
+ le->metablob.add_fragmented_dir(*p, false);
+ } else {
+ (*p)->state_set(CDir::STATE_DIRTYDFT);
+ le->metablob.add_fragmented_dir(*p, true);
+ }
}
// dft lock
- mds->locker->mark_updated_scatterlock(&diri->dirfragtreelock);
- mdr->ls->dirty_dirfrag_dirfragtree.push_back(&diri->item_dirty_dirfrag_dirfragtree);
- mdr->add_updated_lock(&diri->dirfragtreelock);
+ if (diri->is_auth()) {
+ // journal dirfragtree
+ inode_t *pi = diri->project_inode();
+ pi->version = diri->pre_dirty();
+ journal_dirty_inode(mdr, &le->metablob, diri);
+ } else {
+ mds->locker->mark_updated_scatterlock(&diri->dirfragtreelock);
+ mdr->ls->dirty_dirfrag_dirfragtree.push_back(&diri->item_dirty_dirfrag_dirfragtree);
+ mdr->add_updated_lock(&diri->dirfragtreelock);
+ }
/*
// filelock
@@ -11332,20 +11581,26 @@ void MDCache::dispatch_fragment_dir(MDRequest *mdr)
mut->add_updated_lock(&diri->nestlock);
*/
- add_uncommitted_fragment(dirfrag_t(diri->ino(), info.basefrag), info.bits, le->orig_frags, mdr->ls);
+ add_uncommitted_fragment(basedirfrag, info.bits, le->orig_frags, mdr->ls);
mds->mdlog->submit_entry(le, new C_MDC_FragmentPrep(this, mdr));
mds->mdlog->flush();
}
void MDCache::_fragment_logged(MDRequest *mdr)
{
- map<metareqid_t, fragment_info_t>::iterator it = fragment_requests.find(mdr->reqid);
- assert(it != fragment_requests.end());
+ dirfrag_t basedirfrag = mdr->more()->fragment_base;
+ map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
+ assert(it != fragments.end());
fragment_info_t &info = it->second;
CInode *diri = info.resultfrags.front()->get_inode();
- dout(10) << "fragment_logged " << info.resultfrags << " " << info.basefrag
- << " bits " << info.bits << " on " << *diri << dendl;
+ dout(10) << "fragment_logged " << basedirfrag << " bits " << info.bits
+ << " on " << *diri << dendl;
+
+ if (diri->is_auth())
+ diri->pop_and_dirty_projected_inode(mdr->ls);
+
+ mdr->apply(); // mark scatterlock
// store resulting frags
C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentStore(this, mdr));
@@ -11367,33 +11622,36 @@ void MDCache::_fragment_logged(MDRequest *mdr)
void MDCache::_fragment_stored(MDRequest *mdr)
{
- map<metareqid_t, fragment_info_t>::iterator it = fragment_requests.find(mdr->reqid);
- assert(it != fragment_requests.end());
+ dirfrag_t basedirfrag = mdr->more()->fragment_base;
+ map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
+ assert(it != fragments.end());
fragment_info_t &info = it->second;
CInode *diri = info.resultfrags.front()->get_inode();
- dout(10) << "fragment_stored " << info.resultfrags << " " << info.basefrag
- << " bits " << info.bits << " on " << *diri << dendl;
+ dout(10) << "fragment_stored " << basedirfrag << " bits " << info.bits
+ << " on " << *diri << dendl;
// tell peers
CDir *first = *info.resultfrags.begin();
for (map<int,unsigned>::iterator p = first->replicas_begin();
p != first->replica_map.end();
++p) {
- if (mds->mdsmap->get_state(p->first) <= MDSMap::STATE_REJOIN)
+ if (mds->mdsmap->get_state(p->first) < MDSMap::STATE_REJOIN ||
+ (mds->mdsmap->get_state(p->first) == MDSMap::STATE_REJOIN &&
+ rejoin_gather.count(p->first)))
continue;
- MMDSFragmentNotify *notify = new MMDSFragmentNotify(diri->ino(), info.basefrag, info.bits);
- /*
+ MMDSFragmentNotify *notify = new MMDSFragmentNotify(basedirfrag, info.bits);
+
// freshly replicate new dirs to peers
- for (list<CDir*>::iterator q = resultfrags.begin(); q != resultfrags.end(); q++)
+ for (list<CDir*>::iterator q = info.resultfrags.begin();
+ q != info.resultfrags.end();
+ ++q)
replicate_dir(*q, p->first, notify->basebl);
- */
mds->send_message_mds(notify, p->first);
- }
+ }
- mdr->apply(); // mark scatterlock
mds->locker->drop_locks(mdr);
// unfreeze resulting frags
@@ -11417,12 +11675,11 @@ void MDCache::_fragment_stored(MDRequest *mdr)
}
// journal commit
- EFragment *le = new EFragment(mds->mdlog, EFragment::OP_COMMIT,
- diri->ino(), info.basefrag, info.bits);
- mds->mdlog->start_submit_entry(le, new C_MDC_FragmentCommit(this, diri->ino(), info.basefrag,
+ EFragment *le = new EFragment(mds->mdlog, EFragment::OP_COMMIT, basedirfrag, info.bits);
+ mds->mdlog->start_submit_entry(le, new C_MDC_FragmentCommit(this, basedirfrag,
info.resultfrags));
- fragment_requests.erase(it);
+ fragments.erase(it);
request_finish(mdr);
}
@@ -11473,8 +11730,7 @@ void MDCache::_fragment_finish(dirfrag_t basedirfrag, list<CDir*>& resultfrags)
(*p)->auth_unpin(this);
}
- EFragment *le = new EFragment(mds->mdlog, EFragment::OP_FINISH,
- basedirfrag.ino, basedirfrag.frag, uf.bits);
+ EFragment *le = new EFragment(mds->mdlog, EFragment::OP_FINISH, basedirfrag, uf.bits);
mds->mdlog->start_submit_entry(le);
finish_uncommitted_fragment(basedirfrag, EFragment::OP_FINISH);
@@ -11495,6 +11751,7 @@ void MDCache::handle_fragment_notify(MMDSFragmentNotify *notify)
frag_t base = notify->get_basefrag();
int bits = notify->get_bits();
+/*
if ((bits < 0 && diri->dirfragtree.is_leaf(base)) ||
(bits > 0 && !diri->dirfragtree.is_leaf(base))) {
dout(10) << " dft " << diri->dirfragtree << " state doesn't match " << base << " by " << bits
@@ -11502,23 +11759,26 @@ void MDCache::handle_fragment_notify(MMDSFragmentNotify *notify)
notify->put();
return;
}
+*/
// refragment
list<Context*> waiters;
list<CDir*> resultfrags;
- adjust_dir_fragments(diri, base, bits,
- resultfrags, waiters, false);
+ adjust_dir_fragments(diri, base, bits, resultfrags, waiters, false);
if (g_conf->mds_debug_frag)
diri->verify_dirfrags();
- /*
+ for (list<CDir*>::iterator p = resultfrags.begin(); p != resultfrags.end(); ++p)
+ diri->take_dir_waiting((*p)->get_frag(), waiters);
+
// add new replica dirs values
bufferlist::iterator p = notify->basebl.begin();
- while (!p.end()) {
+ while (!p.end())
add_replica_dir(p, diri, notify->get_source().num(), waiters);
- */
mds->queue_waiters(waiters);
+ } else {
+ assert(0);
}
notify->put();
@@ -11534,8 +11794,15 @@ void MDCache::add_uncommitted_fragment(dirfrag_t basedirfrag, int bits, list<fra
uf.bits = bits;
uf.ls = ls;
ls->uncommitted_fragments.insert(basedirfrag);
- if (rollback)
+ if (rollback) {
uf.rollback.swap(*rollback);
+ // preserve COMPLETE flag for newly created dirfrag
+ if (bits > 0 && basedirfrag.frag == frag_t()) {
+ CDir *dir = get_dirfrag(basedirfrag);
+ if (dir && dir->is_complete())
+ uf.complete = true;
+ }
+ }
}
void MDCache::finish_uncommitted_fragment(dirfrag_t basedirfrag, int op)
@@ -11597,7 +11864,7 @@ void MDCache::rollback_uncommitted_fragments()
dout(10) << " rolling back " << p->first << " refragment by " << uf.bits << " bits" << dendl;
LogSegment *ls = mds->mdlog->get_current_segment();
- EFragment *le = new EFragment(mds->mdlog, EFragment::OP_ROLLBACK, diri->ino(), p->first.frag, uf.bits);
+ EFragment *le = new EFragment(mds->mdlog, EFragment::OP_ROLLBACK, p->first, uf.bits);
mds->mdlog->start_entry(le);
list<frag_t> old_frags;
@@ -11620,6 +11887,8 @@ void MDCache::rollback_uncommitted_fragments()
dir->set_version(rollback.fnode.version);
dir->fnode = rollback.fnode;
+ if (uf.complete)
+ dir->mark_complete();
dir->_mark_dirty(ls);
if (!(dir->fnode.rstat == dir->fnode.accounted_rstat)) {
@@ -11637,7 +11906,7 @@ void MDCache::rollback_uncommitted_fragments()
le->add_orig_frag(dir->get_frag());
le->metablob.add_dir_context(dir);
- le->metablob.add_dir(dir, true);
+ le->metablob.add_dir(dir, true, uf.complete);
}
}
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index ad0f5ff..b3c5ad5 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -413,6 +413,7 @@ protected:
set<int> rejoin_ack_gather; // nodes from whom i need a rejoin ack
map<int,map<inodeno_t,map<client_t,Capability::Import> > > rejoin_imported_caps;
map<inodeno_t,pair<int,map<client_t,Capability::Export> > > rejoin_slave_exports;
+ map<client_t,entity_inst_t> rejoin_client_map;
map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> > cap_exports; // ino -> client -> capex
map<inodeno_t,int> cap_export_targets; // ino -> auth mds
@@ -428,6 +429,7 @@ protected:
map<int, set<CInode*> > rejoin_unlinked_inodes;
vector<CInode*> rejoin_recover_q, rejoin_check_q;
+ list<SimpleLock*> rejoin_eval_locks;
list<Context*> rejoin_waiters;
void rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin);
@@ -489,7 +491,10 @@ public:
}
friend class C_MDC_RejoinOpenInoFinish;
+ friend class C_MDC_RejoinSessionsOpened;
void rejoin_open_ino_finish(inodeno_t ino, int ret);
+ void rejoin_open_sessions_finish(map<client_t,entity_inst_t> client_map,
+ map<client_t,uint64_t>& sseqmap);
bool process_imported_caps();
void choose_lock_states_and_reconnect_caps();
void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino,
@@ -514,12 +519,10 @@ public:
void open_snap_parents();
bool open_undef_inodes_dirfrags();
+ void opened_undef_inode(CInode *in);
void opened_undef_dirfrag(CDir *dir) {
rejoin_undef_dirfrags.erase(dir);
}
- void opened_undef_inode(CInode *in) {
- rejoin_undef_inodes.erase(in);
- }
void reissue_all_caps();
@@ -572,7 +575,7 @@ public:
bool trim_dentry(CDentry *dn, map<int, MCacheExpire*>& expiremap);
void trim_dirfrag(CDir *dir, CDir *con,
map<int, MCacheExpire*>& expiremap);
- void trim_inode(CDentry *dn, CInode *in, CDir *con,
+ bool trim_inode(CDentry *dn, CInode *in, CDir *con,
map<int,class MCacheExpire*>& expiremap);
void send_expire_messages(map<int, MCacheExpire*>& expiremap);
void trim_non_auth(); // trim out trimmable non-auth items
@@ -663,7 +666,8 @@ public:
}
protected:
- void inode_remove_replica(CInode *in, int rep, set<SimpleLock *>& gather_locks);
+ void inode_remove_replica(CInode *in, int rep, bool rejoin,
+ set<SimpleLock *>& gather_locks);
void dentry_remove_replica(CDentry *dn, int rep, set<SimpleLock *>& gather_locks);
void rename_file(CDentry *srcdn, CDentry *destdn);
@@ -942,21 +946,28 @@ private:
struct ufragment {
int bits;
bool committed;
+ bool complete;
LogSegment *ls;
list<Context*> waiters;
list<frag_t> old_frags;
bufferlist rollback;
- ufragment() : bits(0), committed(false), ls(NULL) {}
+ ufragment() : bits(0), committed(false), complete(false), ls(NULL) {}
};
map<dirfrag_t, ufragment> uncommitted_fragments;
struct fragment_info_t {
- frag_t basefrag;
int bits;
list<CDir*> dirs;
list<CDir*> resultfrags;
+ MDRequest *mdr;
+ // for deadlock detection
+ bool dirs_frozen;
+ utime_t last_cum_auth_pins_change;
+ int last_cum_auth_pins;
+ int num_remote_waiters; // number of remote authpin waiters
+ fragment_info_t() : last_cum_auth_pins(0), num_remote_waiters(0) {}
};
- map<metareqid_t, fragment_info_t> fragment_requests;
+ map<dirfrag_t,fragment_info_t> fragments;
void adjust_dir_fragments(CInode *diri, frag_t basefrag, int bits,
list<CDir*>& frags, list<Context*>& waiters, bool replay);
@@ -966,13 +977,13 @@ private:
list<CDir*>& resultfrags,
list<Context*>& waiters,
bool replay);
- CDir *force_dir_fragment(CInode *diri, frag_t fg);
+ CDir *force_dir_fragment(CInode *diri, frag_t fg, bool replay=true);
void get_force_dirfrag_bound_set(vector<dirfrag_t>& dfs, set<CDir*>& bounds);
bool can_fragment(CInode *diri, list<CDir*>& dirs);
void fragment_freeze_dirs(list<CDir*>& dirs, C_GatherBuilder &gather);
void fragment_mark_and_complete(list<CDir*>& dirs);
- void fragment_frozen(list<CDir*>& dirs, frag_t basefrag, int bits);
+ void fragment_frozen(dirfrag_t basedirfrag, int r);
void fragment_unmark_unfreeze_dirs(list<CDir*>& dirs);
void dispatch_fragment_dir(MDRequest *mdr);
void _fragment_logged(MDRequest *mdr);
@@ -1003,6 +1014,10 @@ public:
void merge_dir(CInode *diri, frag_t fg);
void rollback_uncommitted_fragments();
+ void find_stale_fragment_freeze();
+ void fragment_freeze_inc_num_waiters(CDir *dir);
+ int get_num_fragmenting_dirs() { return fragments.size(); }
+
// -- updates --
//int send_inode_updates(CInode *in);
//void handle_inode_update(MInodeUpdate *m);
diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index b8ae455..c62b75f 100644
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -117,7 +117,8 @@ MDS::MDS(const std::string &n, Messenger *m, MonClient *mc) :
mdsmap = new MDSMap;
osdmap = new OSDMap;
- objecter = new Objecter(m->cct, messenger, monc, osdmap, mds_lock, timer);
+ objecter = new Objecter(m->cct, messenger, monc, osdmap, mds_lock, timer,
+ 0, 0);
objecter->unset_honor_osdmap_full();
filer = new Filer(objecter);
@@ -606,6 +607,7 @@ void MDS::tick()
if (is_active()) {
balancer->tick();
+ mdcache->find_stale_fragment_freeze();
mdcache->migrator->find_stale_export_freeze();
if (snapserver)
snapserver->check_osd_map(false);
@@ -1037,28 +1039,31 @@ void MDS::handle_mds_map(MMDSMap *m)
if (g_conf->mds_dump_cache_after_rejoin &&
oldmap->is_rejoining() && !mdsmap->is_rejoining())
mdcache->dump_cache(); // for DEBUG only
-
- // ACTIVE|CLIENTREPLAY|REJOIN => we can discover from them.
- set<int> olddis, dis;
- oldmap->get_mds_set(olddis, MDSMap::STATE_ACTIVE);
- oldmap->get_mds_set(olddis, MDSMap::STATE_CLIENTREPLAY);
- oldmap->get_mds_set(olddis, MDSMap::STATE_REJOIN);
- mdsmap->get_mds_set(dis, MDSMap::STATE_ACTIVE);
- mdsmap->get_mds_set(dis, MDSMap::STATE_CLIENTREPLAY);
- mdsmap->get_mds_set(dis, MDSMap::STATE_REJOIN);
- for (set<int>::iterator p = dis.begin(); p != dis.end(); ++p)
- if (*p != whoami && // not me
- olddis.count(*p) == 0) { // newly so?
- mdcache->kick_discovers(*p);
- mdcache->kick_open_ino_peers(*p);
- }
+
+ if (oldstate >= MDSMap::STATE_REJOIN) {
+ // ACTIVE|CLIENTREPLAY|REJOIN => we can discover from them.
+ set<int> olddis, dis;
+ oldmap->get_mds_set(olddis, MDSMap::STATE_ACTIVE);
+ oldmap->get_mds_set(olddis, MDSMap::STATE_CLIENTREPLAY);
+ oldmap->get_mds_set(olddis, MDSMap::STATE_REJOIN);
+ mdsmap->get_mds_set(dis, MDSMap::STATE_ACTIVE);
+ mdsmap->get_mds_set(dis, MDSMap::STATE_CLIENTREPLAY);
+ mdsmap->get_mds_set(dis, MDSMap::STATE_REJOIN);
+ for (set<int>::iterator p = dis.begin(); p != dis.end(); ++p)
+ if (*p != whoami && // not me
+ olddis.count(*p) == 0) { // newly so?
+ mdcache->kick_discovers(*p);
+ mdcache->kick_open_ino_peers(*p);
+ }
+ }
}
if (oldmap->is_degraded() && !mdsmap->is_degraded() && state >= MDSMap::STATE_ACTIVE)
dout(1) << "cluster recovered." << dendl;
// did someone go active?
- if (is_clientreplay() || is_active() || is_stopping()) {
+ if (oldstate >= MDSMap::STATE_CLIENTREPLAY &&
+ (is_clientreplay() || is_active() || is_stopping())) {
set<int> oldactive, active;
oldmap->get_mds_set(oldactive, MDSMap::STATE_ACTIVE);
oldmap->get_mds_set(oldactive, MDSMap::STATE_CLIENTREPLAY);
@@ -2001,6 +2006,7 @@ bool MDS::_dispatch(Message *m)
// hack: thrash fragments
for (int i=0; i<g_conf->mds_thrash_fragments; i++) {
if (!is_active()) break;
+ if (mdcache->get_num_fragmenting_dirs() > 5) break;
dout(7) << "mds thrashing fragments pass " << (i+1) << "/" << g_conf->mds_thrash_fragments << dendl;
// pick a random dir inode
@@ -2012,9 +2018,10 @@ bool MDS::_dispatch(Message *m)
CDir *dir = ls.front();
if (!dir->get_parent_dir()) continue; // must be linked.
if (!dir->is_auth()) continue; // must be auth.
- if (dir->get_frag() == frag_t() || (rand() % 3 == 0)) {
+ frag_t fg = dir->get_frag();
+ if (fg == frag_t() || (rand() % (1 << fg.bits()) == 0))
mdcache->split_dir(dir, 1);
- } else
+ else
balancer->queue_merge(dir);
}
diff --git a/src/mds/MDS.h b/src/mds/MDS.h
index d6426de..ac68fea 100644
--- a/src/mds/MDS.h
+++ b/src/mds/MDS.h
@@ -35,7 +35,7 @@
#include "SessionMap.h"
-#define CEPH_MDS_PROTOCOL 20 /* cluster internal */
+#define CEPH_MDS_PROTOCOL 22 /* cluster internal */
enum {
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 117852c..233a511 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -24,6 +24,7 @@
#include "MDBalancer.h"
#include "MDLog.h"
#include "MDSMap.h"
+#include "Mutation.h"
#include "include/filepath.h"
@@ -209,37 +210,50 @@ void Migrator::find_stale_export_freeze()
* - client request tries authpinning items in subtree A
* (wait because subtree A is freezing)
*/
- for (set<pair<utime_t,CDir*> >::iterator p = export_freezing_dirs.begin();
- p != export_freezing_dirs.end(); ) {
- if (p->first >= cutoff)
- break;
- CDir *dir = p->second;
+ for (map<CDir*,export_state_t>::iterator p = export_state.begin();
+ p != export_state.end(); ) {
+ CDir* dir = p->first;
+ export_state_t& stat = p->second;
++p;
- if (export_freezing_state[dir].num_waiters > 0 ||
+ if (p->second.state != EXPORT_DISCOVERING &&
+ p->second.state != EXPORT_FREEZING)
+ continue;
+ if (stat.last_cum_auth_pins != dir->get_cum_auth_pins()) {
+ stat.last_cum_auth_pins = dir->get_cum_auth_pins();
+ stat.last_cum_auth_pins_change = now;
+ continue;
+ }
+ if (stat.last_cum_auth_pins_change >= cutoff)
+ continue;
+ if (stat.num_remote_waiters > 0 ||
(!dir->inode->is_root() && dir->get_parent_dir()->is_freezing())) {
- assert(get_export_state(dir) == EXPORT_DISCOVERING ||
- get_export_state(dir) == EXPORT_FREEZING);
export_try_cancel(dir);
}
}
}
-void Migrator::export_try_cancel(CDir *dir)
+void Migrator::export_try_cancel(CDir *dir, bool notify_peer)
{
+ dout(10) << "export_try_cancel " << *dir << dendl;
+
map<CDir*,export_state_t>::iterator it = export_state.find(dir);
assert(it != export_state.end());
int state = it->second.state;
switch (state) {
+ case EXPORT_LOCKING:
+ dout(10) << "export state=locking : dropping locks and removing auth_pin" << dendl;
+ it->second.state = EXPORT_CANCELLED;
+ dir->auth_unpin(this);
+ dir->state_clear(CDir::STATE_EXPORTING);
+ break;
case EXPORT_DISCOVERING:
dout(10) << "export state=discovering : canceling freeze and removing auth_pin" << dendl;
it->second.state = EXPORT_CANCELLED;
dir->unfreeze_tree(); // cancel the freeze
dir->auth_unpin(this);
- export_unlock(dir);
- export_freeze_finish(dir);
dir->state_clear(CDir::STATE_EXPORTING);
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+ if (notify_peer && mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
break;
@@ -247,9 +261,8 @@ void Migrator::export_try_cancel(CDir *dir)
dout(10) << "export state=freezing : canceling freeze" << dendl;
it->second.state = EXPORT_CANCELLED;
dir->unfreeze_tree(); // cancel the freeze
- export_freeze_finish(dir);
dir->state_clear(CDir::STATE_EXPORTING);
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+ if (notify_peer && mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
break;
@@ -284,9 +297,8 @@ void Migrator::export_try_cancel(CDir *dir)
dir->unfreeze_tree();
cache->adjust_subtree_auth(dir, mds->get_nodeid());
cache->try_subtree_merge(dir); // NOTE: this may journal subtree_map as side effect
- export_unlock(dir);
dir->state_clear(CDir::STATE_EXPORTING);
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+ if (notify_peer && mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
break;
@@ -311,9 +323,20 @@ void Migrator::export_try_cancel(CDir *dir)
if (it->second.state == EXPORT_CANCELLED) {
// wake up any waiters
mds->queue_waiters(it->second.waiting_for_finish);
+ // drop locks
+ if (state == EXPORT_LOCKING || state == EXPORT_DISCOVERING) {
+ MDRequest *mdr = dynamic_cast<MDRequest*>(it->second.mut);
+ assert(mdr);
+ if (mdr->more()->waiting_on_slave.empty())
+ mds->mdcache->request_finish(mdr);
+ } else if (it->second.mut) {
+ Mutation *mut = it->second.mut;
+ mds->locker->drop_locks(mut);
+ mut->cleanup();
+ delete mut;
+ }
export_state.erase(it);
-
// send pending import_maps? (these need to go out when all exports have finished.)
cache->maybe_send_pending_resolves();
@@ -360,6 +383,7 @@ void Migrator::handle_mds_failure_or_stop(int who)
// - that aren't frozen yet (to avoid auth_pin deadlock)
// - they havne't prepped yet (they may need to discover bounds to do that)
if (p->second.peer == who ||
+ p->second.state == EXPORT_LOCKING ||
p->second.state == EXPORT_DISCOVERING ||
p->second.state == EXPORT_FREEZING ||
p->second.state == EXPORT_PREPPING) {
@@ -487,15 +511,16 @@ void Migrator::handle_mds_failure_or_stop(int who)
break;
}
} else {
- if (q->second.state == IMPORT_ABORTING &&
- q->second.bystanders.count(who)) {
- assert(dir);
- dout(10) << "faking export_notify_ack from mds." << who
- << " on aborting import " << *dir << " from mds." << q->second.peer
- << dendl;
+ if (q->second.bystanders.count(who)) {
q->second.bystanders.erase(who);
- if (q->second.bystanders.empty()) {
- import_reverse_unfreeze(dir);
+ if (q->second.state == IMPORT_ABORTING) {
+ assert(dir);
+ dout(10) << "faking export_notify_ack from mds." << who
+ << " on aborting import " << *dir << " from mds." << q->second.peer
+ << dendl;
+ if (q->second.bystanders.empty()) {
+ import_reverse_unfreeze(dir);
+ }
}
}
}
@@ -583,7 +608,8 @@ void Migrator::audit()
p != export_state.end();
++p) {
CDir *dir = p->first;
- if (p->second.state == EXPORT_DISCOVERING ||
+ if (p->second.state == EXPORT_LOCKING ||
+ p->second.state == EXPORT_DISCOVERING ||
p->second.state == EXPORT_FREEZING) continue;
assert(dir->is_ambiguous_dir_auth());
assert(dir->authority().first == mds->get_nodeid() ||
@@ -613,6 +639,10 @@ void Migrator::export_dir_nicely(CDir *dir, int dest)
void Migrator::maybe_do_queued_export()
{
+ static bool running;
+ if (running)
+ return;
+ running = true;
while (!export_queue.empty() &&
export_state.size() <= 4) {
dirfrag_t df = export_queue.front().first;
@@ -627,6 +657,7 @@ void Migrator::maybe_do_queued_export()
export_dir(dir, dest);
}
+ running = false;
}
@@ -656,6 +687,9 @@ void Migrator::get_export_lock_set(CDir *dir, set<SimpleLock*>& locks)
++it)
locks.insert(&(*it)->lock);
+ // prevent scatter gather race
+ locks.insert(&dir->get_inode()->dirfragtreelock);
+
// bound dftlocks:
// NOTE: We need to take an rdlock on bounding dirfrags during
// migration for a rather irritating reason: when we export the
@@ -707,48 +741,88 @@ void Migrator::export_dir(CDir *dir, int dest)
dout(7) << "already exporting" << dendl;
return;
}
-
- // locks?
- set<SimpleLock*> locks;
- get_export_lock_set(dir, locks);
- if (!mds->locker->rdlock_try_set(locks)) {
- dout(7) << "export_dir can't rdlock needed locks, failing." << dendl;
- return;
- }
- // ok.
- mds->locker->rdlock_take_set(locks);
+ dir->auth_pin(this);
+ dir->state_set(CDir::STATE_EXPORTING);
+
+ MDRequest *mdr = mds->mdcache->request_start_internal(CEPH_MDS_OP_EXPORTDIR);
+ mdr->more()->export_dir = dir;
assert(export_state.count(dir) == 0);
export_state_t& stat = export_state[dir];
- stat.state = EXPORT_DISCOVERING;
+ stat.state = EXPORT_LOCKING;
stat.peer = dest;
- stat.tid = ++last_export_tid;
- stat.locks.swap(locks);
+ stat.tid = mdr->reqid.tid;
+ stat.mut = mdr;
+
+ dispatch_export_dir(mdr);
+}
+
+void Migrator::dispatch_export_dir(MDRequest *mdr)
+{
+ dout(7) << "dispatch_export_dir " << *mdr << dendl;
+ CDir *dir = mdr->more()->export_dir;
+
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ if (it == export_state.end() || it->second.tid != mdr->reqid.tid) {
+ // export must have aborted.
+ dout(7) << "export must have aborted " << *mdr << dendl;
+ mds->mdcache->request_finish(mdr);
+ return;
+ }
+ assert(it->second.state == EXPORT_LOCKING);
+
+ if (mdr->aborted || dir->is_frozen() || dir->is_freezing()) {
+ dout(7) << "wouldblock|freezing|frozen, canceling export" << dendl;
+ export_try_cancel(dir);
+ return;
+ }
+
+ // locks?
+ set<SimpleLock*> rdlocks;
+ set<SimpleLock*> xlocks;
+ set<SimpleLock*> wrlocks;
+ get_export_lock_set(dir, rdlocks);
+ // If auth MDS of the subtree root inode is neither the exporter MDS
+ // nor the importer MDS and it gathers subtree root's fragstat/neststat
+ // while the subtree is exporting. It's possible that the exporter MDS
+ // and the importer MDS both are auth MDS of the subtree root or both
+ // are not auth MDS of the subtree root at the time they receive the
+ // lock messages. So the auth MDS of the subtree root inode may get no
+ // or duplicated fragstat/neststat for the subtree root dirfrag.
+ wrlocks.insert(&dir->get_inode()->filelock);
+ wrlocks.insert(&dir->get_inode()->nestlock);
+ if (dir->get_inode()->is_auth()) {
+ dir->get_inode()->filelock.set_scatter_wanted();
+ dir->get_inode()->nestlock.set_scatter_wanted();
+ }
+
+ if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks, NULL, NULL, true)) {
+ if (mdr->aborted)
+ export_try_cancel(dir);
+ return;
+ }
- dir->state_set(CDir::STATE_EXPORTING);
assert(g_conf->mds_kill_export_at != 1);
+ it->second.state = EXPORT_DISCOVERING;
// send ExportDirDiscover (ask target)
filepath path;
dir->inode->make_path(path);
MExportDirDiscover *discover = new MExportDirDiscover(dir->dirfrag(), path,
- mds->get_nodeid(), stat.tid);
- mds->send_message_mds(discover, dest);
+ mds->get_nodeid(),
+ it->second.tid);
+ mds->send_message_mds(discover, it->second.peer);
assert(g_conf->mds_kill_export_at != 2);
- // start the freeze, but hold it up with an auth_pin.
- utime_t now = ceph_clock_now(g_ceph_context);
- export_freezing_dirs.insert(make_pair(now, dir));
- export_freezing_state[dir].start_time = now;
+ it->second.last_cum_auth_pins_change = ceph_clock_now(g_ceph_context);
- dir->auth_pin(this);
+ // start the freeze, but hold it up with an auth_pin.
dir->freeze_tree();
assert(dir->is_freezing_tree());
dir->add_waiter(CDir::WAIT_FROZEN, new C_MDC_ExportFreeze(this, dir));
}
-
/*
* called on receipt of MExportDirDiscoverAck
* the importer now has the directory's _inode_ in memory, and pinned.
@@ -771,7 +845,10 @@ void Migrator::handle_export_discover_ack(MExportDirDiscoverAck *m)
} else {
assert(it->second.state == EXPORT_DISCOVERING);
// release locks to avoid deadlock
- export_unlock(dir);
+ MDRequest *mdr = dynamic_cast<MDRequest*>(it->second.mut);
+ assert(mdr);
+ mds->mdcache->request_finish(mdr);
+ it->second.mut = NULL;
// freeze the subtree
it->second.state = EXPORT_FREEZING;
dir->auth_unpin(this);
@@ -821,16 +898,17 @@ void Migrator::export_frozen(CDir *dir)
assert(it != export_state.end());
assert(it->second.state == EXPORT_FREEZING);
- export_freeze_finish(dir);
-
- CInode *diri = dir->inode;
+ CInode *diri = dir->get_inode();
// ok, try to grab all my locks.
- set<SimpleLock*> locks;
- get_export_lock_set(dir, locks);
- if (!mds->locker->can_rdlock_set(locks)) {
- dout(7) << "export_dir couldn't rdlock all needed locks, failing. "
- << *diri << dendl;
+ set<SimpleLock*> rdlocks;
+ get_export_lock_set(dir, rdlocks);
+ if ((diri->is_auth() && diri->is_frozen()) ||
+ !mds->locker->can_rdlock_set(rdlocks) ||
+ !diri->filelock.can_wrlock(-1) ||
+ !diri->nestlock.can_wrlock(-1)) {
+ dout(7) << "export_dir couldn't acquire all needed locks, failing. "
+ << *dir << dendl;
// .. unwind ..
dir->unfreeze_tree();
@@ -843,9 +921,13 @@ void Migrator::export_frozen(CDir *dir)
return;
}
- mds->locker->rdlock_take_set(locks);
- it->second.locks.swap(locks);
-
+ it->second.mut = new Mutation;
+ if (diri->is_auth())
+ it->second.mut->auth_pin(diri);
+ mds->locker->rdlock_take_set(rdlocks, it->second.mut);
+ mds->locker->wrlock_force(&diri->filelock, it->second.mut);
+ mds->locker->wrlock_force(&diri->nestlock, it->second.mut);
+
cache->show_subtrees();
// note the bounds.
@@ -1019,8 +1101,15 @@ void Migrator::handle_export_prep_ack(MExportDirPrepAck *m)
m->put();
return;
}
-
assert(it->second.state == EXPORT_PREPPING);
+
+ if (!m->is_success()) {
+ dout(7) << "peer couldn't acquire all needed locks, canceling" << dendl;
+ export_try_cancel(dir, false);
+ m->put();
+ return;
+ }
+
assert (g_conf->mds_kill_export_at != 5);
// send warnings
set<CDir*> bounds;
@@ -1548,8 +1637,6 @@ void Migrator::export_reverse(CDir *dir)
// unfreeze
dir->unfreeze_tree();
- export_unlock(dir);
-
cache->show_cache();
}
@@ -1651,17 +1738,6 @@ void Migrator::handle_export_notify_ack(MExportDirNotifyAck *m)
m->put();
}
-void Migrator::export_unlock(CDir *dir)
-{
- dout(10) << "export_unlock " << *dir << dendl;
-
- mds->locker->rdlock_finish_set(export_state[dir].locks);
- export_state[dir].locks.clear();
-
- list<Context*> ls;
- mds->queue_waiters(ls);
-}
-
void Migrator::export_finish(CDir *dir)
{
dout(5) << "export_finish " << *dir << dendl;
@@ -1712,9 +1788,6 @@ void Migrator::export_finish(CDir *dir)
!dir->get_inode()->has_subtree_root_dirfrag(mds->get_nodeid()))
dir->get_inode()->clear_scatter_dirty();
- // unpin path
- export_unlock(dir);
-
// discard delayed expires
cache->discard_delayed_expire(dir);
@@ -1724,6 +1797,14 @@ void Migrator::export_finish(CDir *dir)
// queue finishers
mds->queue_waiters(it->second.waiting_for_finish);
+ // unpin path
+ Mutation *mut = it->second.mut;
+ if (mut) {
+ mds->locker->drop_locks(mut);
+ mut->cleanup();
+ delete mut;
+ }
+
export_state.erase(it);
cache->show_subtrees();
@@ -2030,27 +2111,37 @@ void Migrator::handle_export_prep(MExportDirPrep *m)
}
dout(7) << " all ready, noting auth and freezing import region" << dendl;
-
- // note that i am an ambiguous auth for this subtree.
- // specify bounds, since the exporter explicitly defines the region.
- cache->adjust_bounded_subtree_auth(dir, import_bounds,
- pair<int,int>(oldauth, mds->get_nodeid()));
- cache->verify_subtree_bounds(dir, import_bounds);
-
- // freeze.
- dir->_freeze_tree();
+
+ bool success = true;
+ if (dir->get_inode()->filelock.can_wrlock(-1) &&
+ dir->get_inode()->nestlock.can_wrlock(-1)) {
+ it->second.mut = new Mutation;
+ // force some locks. hacky.
+ mds->locker->wrlock_force(&dir->inode->filelock, it->second.mut);
+ mds->locker->wrlock_force(&dir->inode->nestlock, it->second.mut);
+
+ // note that i am an ambiguous auth for this subtree.
+ // specify bounds, since the exporter explicitly defines the region.
+ cache->adjust_bounded_subtree_auth(dir, import_bounds,
+ pair<int,int>(oldauth, mds->get_nodeid()));
+ cache->verify_subtree_bounds(dir, import_bounds);
+ // freeze.
+ dir->_freeze_tree();
+ // note new state
+ it->second.state = IMPORT_PREPPED;
+ } else {
+ dout(7) << " couldn't acquire all needed locks, failing. " << *dir << dendl;
+ success = false;
+ import_reverse_prepping(dir);
+ }
// ok!
dout(7) << " sending export_prep_ack on " << *dir << dendl;
- mds->send_message(new MExportDirPrepAck(dir->dirfrag(), m->get_tid()), m->get_connection());
-
- // note new state
- it->second.state = IMPORT_PREPPED;
+ mds->send_message(new MExportDirPrepAck(dir->dirfrag(), success, m->get_tid()), m->get_connection());
assert(g_conf->mds_kill_import_at != 4);
// done
m->put();
-
}
@@ -2090,6 +2181,9 @@ void Migrator::handle_export_dir(MExportDir *m)
dout(7) << "handle_export_dir importing " << *dir << " from " << oldauth << dendl;
assert(dir->is_auth() == false);
+ if (!dir->get_inode()->dirfragtree.is_leaf(dir->get_frag()))
+ dir->get_inode()->dirfragtree.force_to_leaf(g_ceph_context, dir->get_frag());
+
cache->show_subtrees();
C_MDS_ImportDirLoggedStart *onlogged = new C_MDS_ImportDirLoggedStart(this, dir, m->get_source().num());
@@ -2371,7 +2465,13 @@ void Migrator::import_reverse_final(CDir *dir)
dout(7) << "import_reverse_final " << *dir << dendl;
// clean up
- import_state.erase(dir->dirfrag());
+ map<dirfrag_t, import_state_t>::iterator it = import_state.find(dir->dirfrag());
+ if (it->second.mut) {
+ mds->locker->drop_locks(it->second.mut);
+ it->second.mut->cleanup();
+ delete it->second.mut;
+ }
+ import_state.erase(it);
// send pending import_maps?
mds->mdcache->maybe_send_pending_resolves();
@@ -2512,6 +2612,7 @@ void Migrator::import_finish(CDir *dir, bool notify, bool last)
it->second.peer_exports.swap(peer_exports);
// clear import state (we're done!)
+ Mutation *mut = it->second.mut;
import_state.erase(it);
mds->mdlog->start_submit_entry(new EImportFinish(dir, true));
@@ -2528,8 +2629,11 @@ void Migrator::import_finish(CDir *dir, bool notify, bool last)
cache->show_subtrees();
//audit(); // this fails, bc we munge up the subtree map during handle_import_map (resolve phase)
- list<Context*> ls;
- mds->queue_waiters(ls);
+ if (mut) {
+ mds->locker->drop_locks(mut);
+ mut->cleanup();
+ delete mut;
+ }
// re-eval imported caps
for (map<CInode*, map<client_t,Capability::Export> >::iterator p = peer_exports.begin();
@@ -2608,6 +2712,11 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::iterator& blp, int o
mds->locker->mark_updated_scatterlock(&in->filelock);
}
+ if (in->dirfragtreelock.is_dirty()) {
+ updated_scatterlocks.push_back(&in->dirfragtreelock);
+ mds->locker->mark_updated_scatterlock(&in->dirfragtreelock);
+ }
+
// adjust replica list
//assert(!in->is_replica(oldauth)); // not true on failed export
in->add_replica(oldauth, CInode::EXPORT_NONCE);
diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h
index 39542a3..0bf35f0 100644
--- a/src/mds/Migrator.h
+++ b/src/mds/Migrator.h
@@ -48,26 +48,28 @@ class MExportCapsAck;
class EImportStart;
+struct Mutation;
class Migrator {
private:
MDS *mds;
MDCache *cache;
- uint64_t last_export_tid;
// -- exports --
public:
// export stages. used to clean up intelligently if there's a failure.
- const static int EXPORT_CANCELLED = 0; // cancelled
- const static int EXPORT_DISCOVERING = 1; // dest is disovering export dir
- const static int EXPORT_FREEZING = 2; // we're freezing the dir tree
- const static int EXPORT_PREPPING = 3; // sending dest spanning tree to export bounds
- const static int EXPORT_WARNING = 4; // warning bystanders of dir_auth_pending
- const static int EXPORT_EXPORTING = 5; // sent actual export, waiting for ack
- const static int EXPORT_LOGGINGFINISH = 6; // logging EExportFinish
- const static int EXPORT_NOTIFYING = 7; // waiting for notifyacks
+ const static int EXPORT_CANCELLED = 0; // cancelled
+ const static int EXPORT_LOCKING = 1; // acquiring locks
+ const static int EXPORT_DISCOVERING = 2; // dest is disovering export dir
+ const static int EXPORT_FREEZING = 3; // we're freezing the dir tree
+ const static int EXPORT_PREPPING = 4; // sending dest spanning tree to export bounds
+ const static int EXPORT_WARNING = 5; // warning bystanders of dir_auth_pending
+ const static int EXPORT_EXPORTING = 6; // sent actual export, waiting for ack
+ const static int EXPORT_LOGGINGFINISH = 7; // logging EExportFinish
+ const static int EXPORT_NOTIFYING = 8; // waiting for notifyacks
static const char *get_export_statename(int s) {
switch (s) {
+ case EXPORT_LOCKING: return "locking";
case EXPORT_DISCOVERING: return "discovering";
case EXPORT_FREEZING: return "freezing";
case EXPORT_PREPPING: return "prepping";
@@ -85,25 +87,22 @@ protected:
int state;
int peer;
uint64_t tid;
- set<SimpleLock*> locks;
set<int> warning_ack_waiting;
set<int> notify_ack_waiting;
map<inodeno_t,map<client_t,Capability::Import> > peer_imported;
list<Context*> waiting_for_finish;
+ Mutation *mut;
+ // for freeze tree deadlock detection
+ utime_t last_cum_auth_pins_change;
+ int last_cum_auth_pins;
+ int num_remote_waiters; // number of remote authpin waiters
+ export_state_t() : mut(NULL), last_cum_auth_pins(0), num_remote_waiters(0) {}
};
map<CDir*, export_state_t> export_state;
list<pair<dirfrag_t,int> > export_queue;
- // for deadlock detection
- struct freezing_state_t {
- utime_t start_time;
- int num_waiters; // number of remote authpin waiters
- freezing_state_t() : num_waiters(0) {}
- };
- map<CDir*,freezing_state_t > export_freezing_state;
- set<pair<utime_t,CDir*> > export_freezing_dirs;
// -- imports --
public:
@@ -115,6 +114,7 @@ public:
const static int IMPORT_ACKING = 6; // logged EImportStart, sent ack, waiting for finish
const static int IMPORT_FINISHING = 7; // sent cap imports, waiting for finish
const static int IMPORT_ABORTING = 8; // notifying bystanders of an abort before unfreezing
+
static const char *get_import_statename(int s) {
switch (s) {
case IMPORT_DISCOVERING: return "discovering";
@@ -139,13 +139,15 @@ protected:
list<ScatterLock*> updated_scatterlocks;
map<client_t,entity_inst_t> client_map;
map<CInode*, map<client_t,Capability::Export> > peer_exports;
+ Mutation *mut;
+ import_state_t() : mut(NULL) {}
};
map<dirfrag_t, import_state_t> import_state;
public:
// -- cons --
- Migrator(MDS *m, MDCache *c) : mds(m), cache(c), last_export_tid(0) {}
+ Migrator(MDS *m, MDCache *c) : mds(m), cache(c) {}
void dispatch(Message*);
@@ -211,8 +213,9 @@ public:
}
void export_freeze_inc_num_waiters(CDir *dir) {
- assert(is_exporting(dir));
- export_freezing_state[dir].num_waiters++;
+ map<CDir*, export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ it->second.num_remote_waiters++;
}
void find_stale_export_freeze();
@@ -224,6 +227,7 @@ public:
// -- import/export --
// exporter
public:
+ void dispatch_export_dir(MDRequest *mdr);
void export_dir(CDir *dir, int dest);
void export_empty_import(CDir *dir);
@@ -272,21 +276,14 @@ public:
void export_sessions_flushed(CDir *dir, uint64_t tid);
void export_go(CDir *dir);
void export_go_synced(CDir *dir, uint64_t tid);
- void export_try_cancel(CDir *dir);
+ void export_try_cancel(CDir *dir, bool notify_peer=true);
void export_reverse(CDir *dir);
void export_notify_abort(CDir *dir, set<CDir*>& bounds);
void handle_export_ack(MExportDirAck *m);
void export_logged_finish(CDir *dir);
void handle_export_notify_ack(MExportDirNotifyAck *m);
- void export_unlock(CDir *dir);
void export_finish(CDir *dir);
- void export_freeze_finish(CDir *dir) {
- utime_t start = export_freezing_state[dir].start_time;
- export_freezing_dirs.erase(make_pair(start, dir));
- export_freezing_state.erase(dir);
- }
-
friend class C_MDC_ExportFreeze;
friend class C_MDS_ExportFinishLogged;
friend class C_M_ExportGo;
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 3d6963b..10ed53d 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -178,6 +178,7 @@ struct MDRequest : public Mutation {
int snap_caps;
bool did_early_reply;
bool o_trunc; ///< request is an O_TRUNC mutation
+ int getattr_caps; ///< caps requested by getattr
bufferlist reply_extra_bl;
@@ -236,6 +237,10 @@ struct MDRequest : public Mutation {
list<Context*> waiting_for_finish;
+ // export & fragment
+ CDir* export_dir;
+ dirfrag_t fragment_base;
+
More() :
srcdn_auth_mds(-1),
src_reanchor_atid(0), dst_reanchor_atid(0), inode_import_v(0),
@@ -252,6 +257,7 @@ struct MDRequest : public Mutation {
client_request(0), straydn(NULL), snapid(CEPH_NOSNAP), tracei(0), tracedn(0),
alloc_ino(0), used_prealloc_ino(0), snap_caps(0), did_early_reply(false),
o_trunc(false),
+ getattr_caps(0),
slave_request(0),
internal_op(-1),
retry(0),
@@ -266,6 +272,7 @@ struct MDRequest : public Mutation {
client_request(req), straydn(NULL), snapid(CEPH_NOSNAP), tracei(0), tracedn(0),
alloc_ino(0), used_prealloc_ino(0), snap_caps(0), did_early_reply(false),
o_trunc(false),
+ getattr_caps(0),
slave_request(0),
internal_op(-1),
retry(0),
@@ -280,6 +287,7 @@ struct MDRequest : public Mutation {
client_request(0), straydn(NULL), snapid(CEPH_NOSNAP), tracei(0), tracedn(0),
alloc_ino(0), used_prealloc_ino(0), snap_caps(0), did_early_reply(false),
o_trunc(false),
+ getattr_caps(0),
slave_request(0),
internal_op(-1),
retry(0),
diff --git a/src/mds/Resetter.cc b/src/mds/Resetter.cc
index cbf8f89..74da069 100644
--- a/src/mds/Resetter.cc
+++ b/src/mds/Resetter.cc
@@ -59,7 +59,8 @@ void Resetter::init(int rank)
inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
unsigned pg_pool = MDS_METADATA_POOL;
osdmap = new OSDMap();
- objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer);
+ objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer,
+ 0, 0);
journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC,
objecter, 0, 0, &timer);
diff --git a/src/mds/ScatterLock.h b/src/mds/ScatterLock.h
index 42745ce..a85caa8 100644
--- a/src/mds/ScatterLock.h
+++ b/src/mds/ScatterLock.h
@@ -189,6 +189,29 @@ public:
state = LOCK_LOCK;
}
+ void encode_state_for_rejoin(bufferlist& bl, int rep) const {
+ __s16 s = get_replica_state();
+ if (is_gathering(rep)) {
+ // the recovering mds may hold rejoined wrlocks
+ if (state == LOCK_MIX_SYNC)
+ s = LOCK_MIX_SYNC;
+ else
+ s = LOCK_MIX_LOCK;
+ }
+ ::encode(s, bl);
+ }
+
+ bool remove_replica(int from, bool rejoin) {
+ if (rejoin &&
+ (state == LOCK_MIX ||
+ state == LOCK_MIX_SYNC ||
+ state == LOCK_MIX_LOCK2 ||
+ state == LOCK_MIX_TSYN ||
+ state == LOCK_MIX_EXCL))
+ return false;
+ return SimpleLock::remove_replica(from);
+ }
+
virtual void print(ostream& out) const {
out << "(";
_print(out);
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 0e37727..8a4321e 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1,4 +1,4 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
@@ -7,9 +7,9 @@
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
+ * License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
- *
+ *
*/
#include <boost/lexical_cast.hpp>
@@ -1066,7 +1066,7 @@ void Server::set_trace_dist(Session *session, MClientReply *reply,
// inode
if (in) {
- in->encode_inodestat(bl, session, NULL, snapid);
+ in->encode_inodestat(bl, session, NULL, snapid, 0, mdr->getattr_caps);
dout(20) << "set_trace_dist added in " << *in << dendl;
reply->head.is_target = 1;
} else
@@ -1190,7 +1190,13 @@ void Server::dispatch_client_request(MDRequest *mdr)
switch (req->get_op()) {
case CEPH_MDS_OP_LOOKUPHASH:
case CEPH_MDS_OP_LOOKUPINO:
- handle_client_lookup_ino(mdr);
+ handle_client_lookup_ino(mdr, false, false);
+ break;
+ case CEPH_MDS_OP_LOOKUPPARENT:
+ handle_client_lookup_ino(mdr, true, false);
+ break;
+ case CEPH_MDS_OP_LOOKUPNAME:
+ handle_client_lookup_ino(mdr, false, true);
break;
// inodes ops.
@@ -1204,10 +1210,6 @@ void Server::dispatch_client_request(MDRequest *mdr)
handle_client_getattr(mdr, false);
break;
- case CEPH_MDS_OP_LOOKUPPARENT:
- handle_client_lookup_parent(mdr);
- break;
-
case CEPH_MDS_OP_SETATTR:
handle_client_setattr(mdr);
break;
@@ -1430,7 +1432,7 @@ void Server::handle_slave_request_reply(MMDSSlaveRequest *m)
assert(mdr->more()->waiting_on_slave.count(from));
mdr->more()->waiting_on_slave.erase(from);
assert(mdr->more()->waiting_on_slave.empty());
- dispatch_client_request(mdr);
+ mdcache->dispatch_request(mdr);
}
break;
@@ -1448,7 +1450,7 @@ void Server::handle_slave_request_reply(MMDSSlaveRequest *m)
assert(mdr->more()->waiting_on_slave.count(from));
mdr->more()->waiting_on_slave.erase(from);
assert(mdr->more()->waiting_on_slave.empty());
- dispatch_client_request(mdr);
+ mdcache->dispatch_request(mdr);
}
break;
@@ -1611,7 +1613,7 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
// build list of objects
list<MDSCacheObject*> objects;
CInode *auth_pin_freeze = NULL;
- bool fail = false;
+ bool fail = false, wouldblock = false;
for (vector<MDSCacheObjectInfo>::iterator p = mdr->slave_request->get_authpins().begin();
p != mdr->slave_request->get_authpins().end();
@@ -1639,6 +1641,12 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
break;
}
if (!mdr->can_auth_pin(*p)) {
+ if (mdr->slave_request->is_nonblock()) {
+ dout(10) << " can't auth_pin (freezing?) " << **p << " nonblocking" << dendl;
+ fail = true;
+ wouldblock = true;
+ break;
+ }
// wait
dout(10) << " waiting for authpinnable on " << **p << dendl;
(*p)->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
@@ -1653,12 +1661,15 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
} else {
assert(0);
}
- if (dir && dir->is_freezing_tree()) {
- while (!dir->is_freezing_tree_root())
- dir = dir->get_parent_dir();
- mdcache->migrator->export_freeze_inc_num_waiters(dir);
+ if (dir) {
+ if (dir->is_freezing_dir())
+ mdcache->fragment_freeze_inc_num_waiters(dir);
+ if (dir->is_freezing_tree()) {
+ while (!dir->is_freezing_tree_root())
+ dir = dir->get_parent_dir();
+ mdcache->migrator->export_freeze_inc_num_waiters(dir);
+ }
}
-
return;
}
}
@@ -1707,6 +1718,9 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
if (auth_pin_freeze)
auth_pin_freeze->set_object_info(reply->get_authpin_freeze());
+ if (wouldblock)
+ reply->mark_error_wouldblock();
+
mds->send_message_mds(reply, mdr->slave_to_mds);
// clean up this request
@@ -1749,6 +1763,9 @@ void Server::handle_slave_auth_pin_ack(MDRequest *mdr, MMDSSlaveRequest *ack)
++p;
}
}
+
+ if (ack->is_error_wouldblock())
+ mdr->aborted = true;
// note slave
mdr->more()->slaves.insert(from);
@@ -1759,7 +1776,7 @@ void Server::handle_slave_auth_pin_ack(MDRequest *mdr, MMDSSlaveRequest *ack)
// go again?
if (mdr->more()->waiting_on_slave.empty())
- dispatch_client_request(mdr);
+ mdcache->dispatch_request(mdr);
else
dout(10) << "still waiting on slaves " << mdr->more()->waiting_on_slave << dendl;
}
@@ -1795,7 +1812,7 @@ CDir *Server::validate_dentry_dir(MDRequest *mdr, CInode *diri, const string& dn
dir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
return NULL;
}
-
+
return dir;
}
@@ -1808,7 +1825,7 @@ CDentry* Server::prepare_null_dentry(MDRequest *mdr, CDir *dir, const string& dn
{
dout(10) << "prepare_null_dentry " << dname << " in " << *dir << dendl;
assert(dir->is_auth());
-
+
client_t client = mdr->get_client();
// does it already exist?
@@ -2332,6 +2349,10 @@ void Server::handle_client_getattr(MDRequest *mdr, bool is_lookup)
if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
return;
+ // note which caps are requested, so we return at least a snapshot
+ // value for them. (currently this matters for xattrs and inline data)
+ mdr->getattr_caps = mask;
+
mds->balancer->hit_inode(ceph_clock_now(g_ceph_context), ref, META_POP_IRD,
mdr->client_request->get_source().num());
@@ -2341,31 +2362,6 @@ void Server::handle_client_getattr(MDRequest *mdr, bool is_lookup)
is_lookup ? mdr->dn[0].back() : 0);
}
-/* This function will clean up the passed mdr*/
-void Server::handle_client_lookup_parent(MDRequest *mdr)
-{
- MClientRequest *req = mdr->client_request;
-
- CInode *in = mdcache->get_inode(req->get_filepath().get_ino());
- if (!in) {
- reply_request(mdr, -ESTALE);
- return;
- }
- if (in->is_base()) {
- reply_request(mdr, -EINVAL);
- return;
- }
-
- CDentry *dn = in->get_projected_parent_dn();
-
- set<SimpleLock*> rdlocks, wrlocks, xlocks;
- rdlocks.insert(&dn->lock);
- if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
- return;
-
- reply_request(mdr, 0, in, dn); // reply
-}
-
struct C_MDS_LookupIno2 : public Context {
Server *server;
MDRequest *mdr;
@@ -2379,7 +2375,7 @@ struct C_MDS_LookupIno2 : public Context {
/*
* filepath: ino
*/
-void Server::handle_client_lookup_ino(MDRequest *mdr)
+void Server::handle_client_lookup_ino(MDRequest *mdr, bool want_parent, bool want_dentry)
{
MClientRequest *req = mdr->client_request;
@@ -2394,9 +2390,36 @@ void Server::handle_client_lookup_ino(MDRequest *mdr)
return;
}
- dout(10) << "reply to lookup_ino " << *in << dendl;
- MClientReply *reply = new MClientReply(req, 0);
- reply_request(mdr, reply, in, NULL);
+ CDentry *dn = in->get_projected_parent_dn();
+ CInode *diri = dn ? dn->get_dir()->inode : NULL;
+ if (dn && (want_parent || want_dentry)) {
+ mdr->pin(dn);
+ set<SimpleLock*> rdlocks, wrlocks, xlocks;
+ rdlocks.insert(&dn->lock);
+ if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+ return;
+ }
+
+ if (want_parent) {
+ if (!diri || diri->is_stray()) {
+ reply_request(mdr, -ESTALE);
+ return;
+ }
+ dout(10) << "reply to lookup_parent " << *in << dendl;
+ reply_request(mdr, 0, diri);
+ } else {
+ if (want_dentry) {
+ inodeno_t dirino = req->get_filepath2().get_ino();
+ if (!diri || (dirino != inodeno_t() && diri->ino() != dirino)) {
+ reply_request(mdr, -ENOENT);
+ return;
+ }
+ dout(10) << "reply to lookup_name " << *in << dendl;
+ } else
+ dout(10) << "reply to lookup_ino " << *in << dendl;
+
+ reply_request(mdr, 0, in, want_dentry ? dn : NULL);
+ }
}
void Server::_lookup_ino_2(MDRequest *mdr, int r)
@@ -3049,8 +3072,8 @@ void Server::handle_client_file_setlock(MDRequest *mdr)
set_lock.start = req->head.args.filelock_change.start;
set_lock.length = req->head.args.filelock_change.length;
set_lock.client = req->get_orig_source().num();
+ set_lock.owner = req->head.args.filelock_change.owner;
set_lock.pid = req->head.args.filelock_change.pid;
- set_lock.pid_namespace = req->head.args.filelock_change.pid_namespace;
set_lock.type = req->head.args.filelock_change.type;
bool will_wait = req->head.args.filelock_change.wait;
@@ -3140,8 +3163,8 @@ void Server::handle_client_file_readlock(MDRequest *mdr)
checking_lock.start = req->head.args.filelock_change.start;
checking_lock.length = req->head.args.filelock_change.length;
checking_lock.client = req->get_orig_source().num();
+ checking_lock.owner = req->head.args.filelock_change.owner;
checking_lock.pid = req->head.args.filelock_change.pid;
- checking_lock.pid_namespace = req->head.args.filelock_change.pid_namespace;
checking_lock.type = req->head.args.filelock_change.type;
// get the appropriate lock state
@@ -3264,11 +3287,6 @@ void Server::handle_client_setattr(MDRequest *mdr)
// log + wait
le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
-
- // update backtrace for old format inode. (see inode_t::decode)
- if (pi->backtrace_version == 0)
- pi->update_backtrace();
-
mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
mdcache->journal_dirty_inode(mdr, &le->metablob, cur);
@@ -3328,8 +3346,14 @@ void Server::do_open_truncate(MDRequest *mdr, int cmode)
mdr->o_trunc = true;
- journal_and_reply(mdr, in, 0, le, new C_MDS_inode_update_finish(mds, mdr, in, old_size > 0,
- changed_ranges));
+ CDentry *dn = 0;
+ if (mdr->client_request->get_dentry_wanted()) {
+ assert(mdr->dn[0].size());
+ dn = mdr->dn[0].back();
+ }
+
+ journal_and_reply(mdr, in, dn, le, new C_MDS_inode_update_finish(mds, mdr, in, old_size > 0,
+ changed_ranges));
}
@@ -3783,12 +3807,13 @@ void Server::handle_client_setxattr(MDRequest *mdr)
if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
return;
- if ((flags & CEPH_XATTR_CREATE) && cur->xattrs.count(name)) {
+ map<string, bufferptr> *pxattrs = cur->get_projected_xattrs();
+ if ((flags & CEPH_XATTR_CREATE) && pxattrs->count(name)) {
dout(10) << "setxattr '" << name << "' XATTR_CREATE and EEXIST on " << *cur << dendl;
reply_request(mdr, -EEXIST);
return;
}
- if ((flags & CEPH_XATTR_REPLACE) && !cur->xattrs.count(name)) {
+ if ((flags & CEPH_XATTR_REPLACE) && !pxattrs->count(name)) {
dout(10) << "setxattr '" << name << "' XATTR_REPLACE and ENODATA on " << *cur << dendl;
reply_request(mdr, -ENODATA);
return;
@@ -3804,9 +3829,11 @@ void Server::handle_client_setxattr(MDRequest *mdr)
pi->ctime = ceph_clock_now(g_ceph_context);
pi->xattr_version++;
px->erase(name);
- (*px)[name] = buffer::create(len);
- if (len)
- req->get_data().copy(0, len, (*px)[name].c_str());
+ if (!(flags & CEPH_XATTR_REMOVE)) {
+ (*px)[name] = buffer::create(len);
+ if (len)
+ req->get_data().copy(0, len, (*px)[name].c_str());
+ }
// log + wait
mdr->ls = mdlog->get_current_segment();
@@ -6770,19 +6797,20 @@ void Server::_commit_slave_rename(MDRequest *mdr, int r,
mdlog->submit_entry(le, new C_MDS_CommittedSlave(this, mdr));
mdlog->flush();
} else {
- if (srcdn->is_auth() && destdnl->is_primary()) {
- dout(10) << " reversing inode export of " << *destdnl->get_inode() << dendl;
- destdnl->get_inode()->abort_export();
- }
// abort
// rollback_bl may be empty if we froze the inode but had to provide an expanded
// witness list from the master, and they failed before we tried prep again.
if (mdr->more()->rollback_bl.length()) {
+ if (mdr->more()->is_inode_exporter) {
+ dout(10) << " reversing inode export of " << *destdnl->get_inode() << dendl;
+ destdnl->get_inode()->abort_export();
+ }
if (mdcache->is_ambiguous_slave_update(mdr->reqid, mdr->slave_to_mds)) {
mdcache->remove_ambiguous_slave_update(mdr->reqid, mdr->slave_to_mds);
// rollback but preserve the slave request
do_rename_rollback(mdr->more()->rollback_bl, mdr->slave_to_mds, mdr, false);
+ mdr->more()->rollback_bl.clear();
} else
do_rename_rollback(mdr->more()->rollback_bl, mdr->slave_to_mds, mdr, true);
} else {
diff --git a/src/mds/Server.h b/src/mds/Server.h
index d7253f1..6ae9f59 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -142,8 +142,7 @@ public:
// requests on existing inodes.
void handle_client_getattr(MDRequest *mdr, bool is_lookup);
- void handle_client_lookup_parent(MDRequest *mdr);
- void handle_client_lookup_ino(MDRequest *mdr);
+ void handle_client_lookup_ino(MDRequest *mdr, bool want_parent, bool want_dentry);
void _lookup_ino_2(MDRequest *mdr, int r);
void handle_client_readdir(MDRequest *mdr);
void handle_client_file_setlock(MDRequest *mdr);
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 37c8235..4b9b7f5 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -330,7 +330,8 @@ public:
} else {
state = s;
}
- take_waiting(SimpleLock::WAIT_ALL, waiters);
+ if (is_stable())
+ take_waiting(SimpleLock::WAIT_ALL, waiters);
}
bool is_stable() const {
@@ -373,8 +374,10 @@ public:
++p)
more()->gather_set.insert(p->first);
}
- bool is_gathering() { return have_more() && !more()->gather_set.empty(); }
- bool is_gathering(int i) {
+ bool is_gathering() const {
+ return have_more() && !more()->gather_set.empty();
+ }
+ bool is_gathering(int i) const {
return have_more() && more()->gather_set.count(i);
}
void clear_gather() {
diff --git a/src/mds/events/EFragment.h b/src/mds/events/EFragment.h
index a9ddd54..8869137 100644
--- a/src/mds/events/EFragment.h
+++ b/src/mds/events/EFragment.h
@@ -37,9 +37,9 @@ public:
bufferlist rollback;
EFragment() : LogEvent(EVENT_FRAGMENT) { }
- EFragment(MDLog *mdlog, int o, inodeno_t i, frag_t bf, int b) :
+ EFragment(MDLog *mdlog, int o, dirfrag_t df, int b) :
LogEvent(EVENT_FRAGMENT), metablob(mdlog),
- op(o), ino(i), basefrag(bf), bits(b) { }
+ op(o), ino(df.ino), basefrag(df.frag), bits(b) { }
void print(ostream& out) const {
out << "EFragment " << op_name(op) << " " << ino << " " << basefrag << " by " << bits << " " << metablob;
diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h
index 685f5ff..14c640f 100644
--- a/src/mds/events/EMetaBlob.h
+++ b/src/mds/events/EMetaBlob.h
@@ -225,6 +225,7 @@ public:
static const int STATE_DIRTY = (1<<2); // dirty due to THIS journal item, that is!
static const int STATE_NEW = (1<<3); // new directory
static const int STATE_IMPORTING = (1<<4); // importing directory
+ static const int STATE_DIRTYDFT = (1<<5); // dirty dirfragtree
//version_t dirv;
fnode_t fnode;
@@ -249,6 +250,8 @@ public:
void mark_new() { state |= STATE_NEW; }
bool is_importing() { return state & STATE_IMPORTING; }
void mark_importing() { state |= STATE_IMPORTING; }
+ bool is_dirty_dft() { return state & STATE_DIRTYDFT; }
+ void mark_dirty_dft() { state |= STATE_DIRTYDFT; }
list<ceph::shared_ptr<fullbit> > &get_dfull() { return dfull; }
list<remotebit> &get_dremote() { return dremote; }
@@ -448,6 +451,9 @@ private:
//cout << "journaling " << in->inode.ino << " at " << my_offset << std::endl;
inode_t *pi = in->get_projected_inode();
+ if ((state & fullbit::STATE_DIRTY) && pi->is_backtrace_updated())
+ state |= fullbit::STATE_DIRTYPARENT;
+
bufferlist snapbl;
sr_t *sr = in->get_projected_srnode();
if (sr)
@@ -532,10 +538,15 @@ private:
dirlump& add_import_dir(CDir *dir) {
// dirty=false would be okay in some cases
return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
- dir->is_dirty(), dir->is_complete(), false, true);
+ dir->is_dirty(), dir->is_complete(), false, true, dir->is_dirty_dft());
+ }
+ dirlump& add_fragmented_dir(CDir *dir, bool dirtydft) {
+ return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
+ false, false, false, false, dirtydft);
}
dirlump& add_dir(dirfrag_t df, fnode_t *pf, version_t pv, bool dirty,
- bool complete=false, bool isnew=false, bool importing=false) {
+ bool complete=false, bool isnew=false,
+ bool importing=false, bool dirty_dft=false) {
if (lump_map.count(df) == 0)
lump_order.push_back(df);
@@ -546,6 +557,7 @@ private:
if (dirty) l.mark_dirty();
if (isnew) l.mark_new();
if (importing) l.mark_importing();
+ if (dirty_dft) l.mark_dirty_dft();
return l;
}
diff --git a/src/mds/flock.cc b/src/mds/flock.cc
index 5e329af..4e825c9 100644
--- a/src/mds/flock.cc
+++ b/src/mds/flock.cc
@@ -15,9 +15,7 @@ bool ceph_lock_state_t::is_waiting(ceph_filelock &fl)
if (p->second.start > fl.start)
return false;
if (p->second.length == fl.length &&
- p->second.client == fl.client &&
- p->second.pid == fl.pid &&
- p->second.pid_namespace == fl.pid_namespace)
+ ceph_filelock_owner_equal(p->second, fl))
return true;
++p;
}
@@ -31,9 +29,7 @@ void ceph_lock_state_t::remove_waiting(ceph_filelock& fl)
if (p->second.start > fl.start)
return;
if (p->second.length == fl.length &&
- p->second.client == fl.client &&
- p->second.pid == fl.pid &&
- p->second.pid_namespace == fl.pid_namespace) {
+ ceph_filelock_owner_equal(p->second, fl)) {
waiting_locks.erase(p);
--client_waiting_lock_counts[(client_t)fl.client];
if (!client_waiting_lock_counts[(client_t)fl.client]) {
@@ -466,17 +462,15 @@ void ceph_lock_state_t::split_by_owner(ceph_filelock& owner,
dout(15) << "owner lock: " << owner << dendl;
while (iter != locks.end()) {
dout(15) << "comparing to " << (*iter)->second << dendl;
- if ((*iter)->second.client == owner.client &&
- (*iter)->second.pid_namespace == owner.pid_namespace &&
- (*iter)->second.pid == owner.pid) {
+ if (ceph_filelock_owner_equal((*iter)->second, owner)) {
dout(15) << "success, pushing to owned_locks" << dendl;
owned_locks.push_back(*iter);
iter = locks.erase(iter);
} else {
dout(15) << "failure, something not equal in this group "
<< (*iter)->second.client << ":" << owner.client << ","
- << (*iter)->second.pid_namespace << ":" << owner.pid_namespace
- << "," << (*iter)->second.pid << ":" << owner.pid << dendl;
+ << (*iter)->second.owner << ":" << owner.owner << ","
+ << (*iter)->second.pid << ":" << owner.pid << dendl;
++iter;
}
}
diff --git a/src/mds/flock.h b/src/mds/flock.h
index b767fe5..a98446d 100644
--- a/src/mds/flock.h
+++ b/src/mds/flock.h
@@ -11,19 +11,29 @@
inline ostream& operator<<(ostream& out, ceph_filelock& l) {
out << "start: " << l.start << ", length: " << l.length
- << ", client: " << l.client << ", pid: " << l.pid
- << ", pid_ns: " << l.pid_namespace << ", type: " << (int)l.type
+ << ", client: " << l.client << ", owner: " << l.owner
+ << ", pid: " << l.pid << ", type: " << (int)l.type
<< std::endl;
return out;
}
+inline bool ceph_filelock_owner_equal(ceph_filelock& l, ceph_filelock& r)
+{
+ if (l.client != r.client || l.owner != r.owner)
+ return false;
+ // The file lock is from old client if the most significant bit of
+ // 'owner' is not set. Old clients use both 'owner' and 'pid' to
+ // identify the owner of lock.
+ if (l.owner & (1ULL << 63))
+ return true;
+ return l.pid == r.pid;
+}
+
inline bool operator==(ceph_filelock& l, ceph_filelock& r) {
return
l.length == r.length &&
- l.client == r.client &&
- l.pid == r.pid &&
- l.pid_namespace == r.pid_namespace &&
- l.type == r.type;
+ l.type == r.type &&
+ ceph_filelock_owner_equal(l, r);
}
class ceph_lock_state_t {
diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index e6d530f..f818d7c 100644
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1019,6 +1019,12 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
dout(10) << "EMetaBlob.replay clean fragstat on " << *dir << dendl;
}
}
+ if (lump.is_dirty_dft()) {
+ dout(10) << "EMetaBlob.replay dirty dirfragtree on " << *dir << dendl;
+ dir->state_set(CDir::STATE_DIRTYDFT);
+ mds->locker->mark_updated_scatterlock(&dir->inode->dirfragtreelock);
+ logseg->dirty_dirfrag_dirfragtree.push_back(&dir->inode->item_dirty_dirfrag_dirfragtree);
+ }
if (lump.is_new())
dir->mark_new(logseg);
if (lump.is_complete())
@@ -1072,15 +1078,14 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
if (p->is_dirty()) in->_mark_dirty(logseg);
dout(10) << "EMetaBlob.replay added " << *in << dendl;
} else {
+ if (in->get_parent_dn() && in->inode.anchored != p->inode.anchored)
+ in->get_parent_dn()->adjust_nested_anchors((int)p->inode.anchored - (int)in->inode.anchored);
+ p->update_inode(mds, in);
if (dn->get_linkage()->get_inode() != in && in->get_parent_dn()) {
dout(10) << "EMetaBlob.replay unlinking " << *in << dendl;
unlinked[in] = in->get_parent_dir();
in->get_parent_dir()->unlink_inode(in->get_parent_dn());
}
- if (in->get_parent_dn() && in->inode.anchored != p->inode.anchored)
- in->get_parent_dn()->adjust_nested_anchors( (int)p->inode.anchored - (int)in->inode.anchored );
- p->update_inode(mds, in);
- if (p->is_dirty()) in->_mark_dirty(logseg);
if (dn->get_linkage()->get_inode() != in) {
if (!dn->get_linkage()->is_null()) { // note: might be remote. as with stray reintegration.
if (dn->get_linkage()->is_primary()) {
@@ -1100,13 +1105,13 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
} else {
dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *in << dendl;
}
+ if (p->is_dirty()) in->_mark_dirty(logseg);
assert(in->first == p->dnfirst ||
(in->is_multiversion() && in->first > p->dnfirst));
}
-
- assert(g_conf->mds_kill_journal_replay_at != 2);
if (p->is_dirty_parent())
in->_mark_dirty_parent(logseg, p->is_dirty_pool());
+ assert(g_conf->mds_kill_journal_replay_at != 2);
}
// remote dentries
@@ -1154,9 +1159,15 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
dn->first = p->dnfirst;
if (!dn->get_linkage()->is_null()) {
dout(10) << "EMetaBlob.replay unlinking " << *dn << dendl;
- if (dn->get_linkage()->is_primary())
- unlinked[dn->get_linkage()->get_inode()] = dir;
- dir->unlink_inode(dn);
+ CInode *in = dn->get_linkage()->get_inode();
+ // For renamed inode, We may call CInode::force_dirfrag() later.
+ // CInode::force_dirfrag() doesn't work well when inode is detached
+ // from the hierarchy.
+ if (!renamed_diri || renamed_diri != in) {
+ if (dn->get_linkage()->is_primary())
+ unlinked[in] = dir;
+ dir->unlink_inode(dn);
+ }
}
dn->set_version(p->dnv);
if (p->dirty) dn->_mark_dirty(logseg);
@@ -2291,6 +2302,9 @@ void ESubtreeMap::replay(MDS *mds)
continue;
}
+ for (vector<dirfrag_t>::iterator q = p->second.begin(); q != p->second.end(); ++q)
+ mds->mdcache->get_force_dirfrag(*q);
+
set<CDir*> bounds;
mds->mdcache->get_subtree_bounds(dir, bounds);
for (vector<dirfrag_t>::iterator q = p->second.begin(); q != p->second.end(); ++q) {
diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index e39a723..aadf123 100644
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -204,7 +204,7 @@ ostream& operator<<(ostream& out, const client_writeable_range_t& r)
*/
void inode_t::encode(bufferlist &bl) const
{
- ENCODE_START(9, 6, bl);
+ ENCODE_START(10, 6, bl);
::encode(ino, bl);
::encode(rdev, bl);
@@ -247,7 +247,7 @@ void inode_t::encode(bufferlist &bl) const
void inode_t::decode(bufferlist::iterator &p)
{
- DECODE_START_LEGACY_COMPAT_LEN(9, 6, 6, p);
+ DECODE_START_LEGACY_COMPAT_LEN(10, 6, 6, p);
::decode(ino, p);
::decode(rdev, p);
@@ -297,8 +297,6 @@ void inode_t::decode(bufferlist::iterator &p)
::decode(backtrace_version, p);
if (struct_v >= 7)
::decode(old_pools, p);
- else
- backtrace_version = 0; // note inode which has no backtrace
if (struct_v >= 8)
::decode(max_size_ever, p);
if (struct_v >= 9) {
@@ -307,6 +305,8 @@ void inode_t::decode(bufferlist::iterator &p)
} else {
inline_version = CEPH_INLINE_NONE;
}
+ if (struct_v < 10)
+ backtrace_version = 0; // force update backtrace
DECODE_FINISH(p);
}
diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index 22eba07..8e7d9a9 100644
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -440,8 +440,8 @@ struct inode_t {
bool is_backtrace_updated() {
return backtrace_version == version;
}
- void update_backtrace() {
- backtrace_version = version;
+ void update_backtrace(version_t pv=0) {
+ backtrace_version = pv ? pv : version;
}
void add_old_pool(int64_t l) {
diff --git a/src/messages/MBackfillReserve.h b/src/messages/MBackfillReserve.h
index ce35ce7..d30e285 100644
--- a/src/messages/MBackfillReserve.h
+++ b/src/messages/MBackfillReserve.h
@@ -18,10 +18,10 @@
#include "msg/Message.h"
class MBackfillReserve : public Message {
- static const int HEAD_VERSION = 2;
+ static const int HEAD_VERSION = 3;
static const int COMPAT_VERSION = 1;
public:
- pg_t pgid;
+ spg_t pgid;
epoch_t query_epoch;
enum {
REQUEST = 0,
@@ -35,7 +35,7 @@ public:
: Message(MSG_OSD_BACKFILL_RESERVE, HEAD_VERSION, COMPAT_VERSION),
query_epoch(0), type(-1), priority(-1) {}
MBackfillReserve(int type,
- pg_t pgid,
+ spg_t pgid,
epoch_t query_epoch, unsigned prio = -1)
: Message(MSG_OSD_BACKFILL_RESERVE, HEAD_VERSION, COMPAT_VERSION),
pgid(pgid), query_epoch(query_epoch),
@@ -65,20 +65,26 @@ public:
void decode_payload() {
bufferlist::iterator p = payload.begin();
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(query_epoch, p);
::decode(type, p);
if (header.version > 1)
::decode(priority, p);
else
priority = 0;
+ if (header.version >= 3)
+ ::decode(pgid.shard, p);
+ else
+ pgid.shard = ghobject_t::no_shard();
+
}
void encode_payload(uint64_t features) {
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(query_epoch, payload);
::encode(type, payload);
::encode(priority, payload);
+ ::encode(pgid.shard, payload);
}
};
diff --git a/src/messages/MClientRequest.h b/src/messages/MClientRequest.h
index 7190ab2..0c17b16 100644
--- a/src/messages/MClientRequest.h
+++ b/src/messages/MClientRequest.h
@@ -180,8 +180,8 @@ public:
head.op == CEPH_MDS_OP_GETFILELOCK) {
out << "rule " << (int)head.args.filelock_change.rule
<< ", type " << (int)head.args.filelock_change.type
+ << ", owner " << head.args.filelock_change.owner
<< ", pid " << head.args.filelock_change.pid
- << ", pid_ns " << head.args.filelock_change.pid_namespace
<< ", start " << head.args.filelock_change.start
<< ", length " << head.args.filelock_change.length
<< ", wait " << (int)head.args.filelock_change.wait;
diff --git a/src/messages/MExportDirPrepAck.h b/src/messages/MExportDirPrepAck.h
index c3d7b49..319194d 100644
--- a/src/messages/MExportDirPrepAck.h
+++ b/src/messages/MExportDirPrepAck.h
@@ -20,30 +20,34 @@
class MExportDirPrepAck : public Message {
dirfrag_t dirfrag;
+ bool success;
public:
dirfrag_t get_dirfrag() { return dirfrag; }
MExportDirPrepAck() {}
- MExportDirPrepAck(dirfrag_t df, uint64_t tid) :
- Message(MSG_MDS_EXPORTDIRPREPACK), dirfrag(df) {
+ MExportDirPrepAck(dirfrag_t df, bool s, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIRPREPACK), dirfrag(df), success(s) {
set_tid(tid);
}
private:
~MExportDirPrepAck() {}
public:
+ bool is_success() { return success; }
const char *get_type_name() const { return "ExPAck"; }
void print(ostream& o) const {
- o << "export_prep_ack(" << dirfrag << ")";
+ o << "export_prep_ack(" << dirfrag << (success ? " success)" : " fail)");
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(dirfrag, p);
+ ::decode(success, p);
}
void encode_payload(uint64_t features) {
::encode(dirfrag, payload);
+ ::encode(success, payload);
}
};
diff --git a/src/messages/MMDSCacheRejoin.h b/src/messages/MMDSCacheRejoin.h
index d48cd67..bbafe64 100644
--- a/src/messages/MMDSCacheRejoin.h
+++ b/src/messages/MMDSCacheRejoin.h
@@ -168,6 +168,7 @@ class MMDSCacheRejoin : public Message {
// open
map<inodeno_t,map<client_t, ceph_mds_cap_reconnect> > cap_exports;
+ map<client_t, entity_inst_t> client_map;
bufferlist imported_caps;
// full
@@ -221,12 +222,10 @@ public:
void add_strong_inode(vinodeno_t i, int n, int cw, int dl, int nl, int dftl) {
strong_inodes[i] = inode_strong(n, cw, dl, nl, dftl);
}
- void add_inode_locks(CInode *in, __u32 nonce) {
+ void add_inode_locks(CInode *in, __u32 nonce, bufferlist& bl) {
::encode(in->inode.ino, inode_locks);
::encode(in->last, inode_locks);
::encode(nonce, inode_locks);
- bufferlist bl;
- in->_encode_locks_state_for_replica(bl);
::encode(bl, inode_locks);
}
void add_inode_base(CInode *in) {
@@ -300,6 +299,7 @@ public:
::encode(xlocked_inodes, payload);
::encode(wrlocked_inodes, payload);
::encode(cap_exports, payload);
+ ::encode(client_map, payload);
::encode(imported_caps, payload);
::encode(strong_dirfrags, payload);
::encode(dirfrag_bases, payload);
@@ -322,6 +322,7 @@ public:
::decode(xlocked_inodes, p);
::decode(wrlocked_inodes, p);
::decode(cap_exports, p);
+ ::decode(client_map, p);
::decode(imported_caps, p);
::decode(strong_dirfrags, p);
::decode(dirfrag_bases, p);
diff --git a/src/messages/MMDSFragmentNotify.h b/src/messages/MMDSFragmentNotify.h
index 2d86b57..46b8837 100644
--- a/src/messages/MMDSFragmentNotify.h
+++ b/src/messages/MMDSFragmentNotify.h
@@ -32,9 +32,9 @@ class MMDSFragmentNotify : public Message {
bufferlist basebl;
MMDSFragmentNotify() : Message(MSG_MDS_FRAGMENTNOTIFY) {}
- MMDSFragmentNotify(inodeno_t i, frag_t bf, int b) :
+ MMDSFragmentNotify(dirfrag_t df, int b) :
Message(MSG_MDS_FRAGMENTNOTIFY),
- ino(i), basefrag(bf), bits(b) { }
+ ino(df.ino), basefrag(df.frag), bits(b) { }
private:
~MMDSFragmentNotify() {}
diff --git a/src/messages/MMDSSlaveRequest.h b/src/messages/MMDSSlaveRequest.h
index f8f30b2..67f5c78 100644
--- a/src/messages/MMDSSlaveRequest.h
+++ b/src/messages/MMDSSlaveRequest.h
@@ -94,6 +94,10 @@ class MMDSSlaveRequest : public Message {
metareqid_t reqid;
__u32 attempt;
__s16 op;
+ __u16 flags;
+
+ static const unsigned FLAG_NONBLOCK = 1;
+ static const unsigned FLAG_WOULDBLOCK = 2;
// for locking
__u16 lock_type; // lock object type
@@ -125,6 +129,10 @@ public:
MDSCacheObjectInfo &get_authpin_freeze() { return object_info; }
vector<MDSCacheObjectInfo>& get_authpins() { return authpins; }
+ void mark_nonblock() { flags |= FLAG_NONBLOCK; }
+ bool is_nonblock() { return (flags & FLAG_NONBLOCK); }
+ void mark_error_wouldblock() { flags |= FLAG_WOULDBLOCK; }
+ bool is_error_wouldblock() { return (flags & FLAG_WOULDBLOCK); }
void set_lock_type(int t) { lock_type = t; }
@@ -133,7 +141,7 @@ public:
MMDSSlaveRequest() : Message(MSG_MDS_SLAVE_REQUEST) { }
MMDSSlaveRequest(metareqid_t ri, __u32 att, int o) :
Message(MSG_MDS_SLAVE_REQUEST),
- reqid(ri), attempt(att), op(o) { }
+ reqid(ri), attempt(att), op(o), flags(0) { }
private:
~MMDSSlaveRequest() {}
@@ -142,6 +150,7 @@ public:
::encode(reqid, payload);
::encode(attempt, payload);
::encode(op, payload);
+ ::encode(flags, payload);
::encode(lock_type, payload);
::encode(object_info, payload);
::encode(authpins, payload);
@@ -159,6 +168,7 @@ public:
::decode(reqid, p);
::decode(attempt, p);
::decode(op, p);
+ ::decode(flags, p);
::decode(lock_type, p);
::decode(object_info, p);
::decode(authpins, p);
diff --git a/src/messages/MOSDPGPull.h b/src/messages/MOSDECSubOpRead.h
similarity index 60%
copy from src/messages/MOSDPGPull.h
copy to src/messages/MOSDECSubOpRead.h
index 870db7f..99e62e6 100644
--- a/src/messages/MOSDPGPull.h
+++ b/src/messages/MOSDECSubOpRead.h
@@ -12,62 +12,49 @@
*
*/
-#ifndef MOSDPGPULL_H
-#define MOSDPGPULL_H
+#ifndef MOSDECSUBOPREAD_H
+#define MOSDECSUBOPREAD_H
#include "msg/Message.h"
#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
-class MOSDPGPull : public Message {
+class MOSDECSubOpRead : public Message {
static const int HEAD_VERSION = 1;
static const int COMPAT_VERSION = 1;
-
public:
- pg_t pgid;
+ spg_t pgid;
epoch_t map_epoch;
- vector<PullOp> pulls;
- uint64_t cost;
-
- MOSDPGPull() :
- Message(MSG_OSD_PG_PULL, HEAD_VERSION, COMPAT_VERSION),
- cost(0)
- {}
-
- void compute_cost(CephContext *cct) {
- cost = 0;
- for (vector<PullOp>::iterator i = pulls.begin();
- i != pulls.end();
- ++i) {
- cost += i->cost(cct);
- }
- }
+ ECSubRead op;
int get_cost() const {
- return cost;
+ return 0;
}
+ MOSDECSubOpRead() :
+ Message(MSG_OSD_EC_READ, HEAD_VERSION, COMPAT_VERSION)
+ {}
+
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(pgid, p);
::decode(map_epoch, p);
- ::decode(pulls, p);
- ::decode(cost, p);
+ ::decode(op, p);
}
virtual void encode_payload(uint64_t features) {
::encode(pgid, payload);
::encode(map_epoch, payload);
- ::encode(pulls, payload);
- ::encode(cost, payload);
+ ::encode(op, payload);
}
- const char *get_type_name() const { return "MOSDPGPull"; }
+ const char *get_type_name() const { return "MOSDECSubOpRead"; }
void print(ostream& out) const {
- out << "MOSDPGPull(" << pgid
+ out << "MOSDECSubOpRead(" << pgid
<< " " << map_epoch
- << " " << pulls;
+ << " " << op;
out << ")";
}
};
diff --git a/src/messages/MOSDPGPull.h b/src/messages/MOSDECSubOpReadReply.h
similarity index 60%
copy from src/messages/MOSDPGPull.h
copy to src/messages/MOSDECSubOpReadReply.h
index 870db7f..28e2cf7 100644
--- a/src/messages/MOSDPGPull.h
+++ b/src/messages/MOSDECSubOpReadReply.h
@@ -12,62 +12,49 @@
*
*/
-#ifndef MOSDPGPULL_H
-#define MOSDPGPULL_H
+#ifndef MOSDECSUBOPREADREPLY_H
+#define MOSDECSUBOPREADREPLY_H
#include "msg/Message.h"
#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
-class MOSDPGPull : public Message {
+class MOSDECSubOpReadReply : public Message {
static const int HEAD_VERSION = 1;
static const int COMPAT_VERSION = 1;
-
public:
- pg_t pgid;
+ spg_t pgid;
epoch_t map_epoch;
- vector<PullOp> pulls;
- uint64_t cost;
-
- MOSDPGPull() :
- Message(MSG_OSD_PG_PULL, HEAD_VERSION, COMPAT_VERSION),
- cost(0)
- {}
-
- void compute_cost(CephContext *cct) {
- cost = 0;
- for (vector<PullOp>::iterator i = pulls.begin();
- i != pulls.end();
- ++i) {
- cost += i->cost(cct);
- }
- }
+ ECSubReadReply op;
int get_cost() const {
- return cost;
+ return 0;
}
+ MOSDECSubOpReadReply() :
+ Message(MSG_OSD_EC_READ_REPLY, HEAD_VERSION, COMPAT_VERSION)
+ {}
+
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(pgid, p);
::decode(map_epoch, p);
- ::decode(pulls, p);
- ::decode(cost, p);
+ ::decode(op, p);
}
virtual void encode_payload(uint64_t features) {
::encode(pgid, payload);
::encode(map_epoch, payload);
- ::encode(pulls, payload);
- ::encode(cost, payload);
+ ::encode(op, payload);
}
- const char *get_type_name() const { return "MOSDPGPull"; }
+ const char *get_type_name() const { return "MOSDECSubOpReadReply"; }
void print(ostream& out) const {
- out << "MOSDPGPull(" << pgid
+ out << "MOSDECSubOpReadReply(" << pgid
<< " " << map_epoch
- << " " << pulls;
+ << " " << op;
out << ")";
}
};
diff --git a/src/messages/MOSDPGPull.h b/src/messages/MOSDECSubOpWrite.h
similarity index 60%
copy from src/messages/MOSDPGPull.h
copy to src/messages/MOSDECSubOpWrite.h
index 870db7f..a47bcef 100644
--- a/src/messages/MOSDPGPull.h
+++ b/src/messages/MOSDECSubOpWrite.h
@@ -12,64 +12,59 @@
*
*/
-#ifndef MOSDPGPULL_H
-#define MOSDPGPULL_H
+#ifndef MOSDECSUBOPWRITE_H
+#define MOSDECSUBOPWRITE_H
#include "msg/Message.h"
#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
-class MOSDPGPull : public Message {
+class MOSDECSubOpWrite : public Message {
static const int HEAD_VERSION = 1;
static const int COMPAT_VERSION = 1;
-
public:
- pg_t pgid;
+ spg_t pgid;
epoch_t map_epoch;
- vector<PullOp> pulls;
- uint64_t cost;
-
- MOSDPGPull() :
- Message(MSG_OSD_PG_PULL, HEAD_VERSION, COMPAT_VERSION),
- cost(0)
- {}
-
- void compute_cost(CephContext *cct) {
- cost = 0;
- for (vector<PullOp>::iterator i = pulls.begin();
- i != pulls.end();
- ++i) {
- cost += i->cost(cct);
- }
- }
+ ECSubWrite op;
int get_cost() const {
- return cost;
+ return 0;
}
+ MOSDECSubOpWrite()
+ : Message(MSG_OSD_EC_WRITE, HEAD_VERSION, COMPAT_VERSION)
+ {}
+ MOSDECSubOpWrite(ECSubWrite &op)
+ : Message(MSG_OSD_EC_WRITE, HEAD_VERSION, COMPAT_VERSION),
+ op(op) {}
+
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(pgid, p);
::decode(map_epoch, p);
- ::decode(pulls, p);
- ::decode(cost, p);
+ ::decode(op, p);
}
virtual void encode_payload(uint64_t features) {
::encode(pgid, payload);
::encode(map_epoch, payload);
- ::encode(pulls, payload);
- ::encode(cost, payload);
+ ::encode(op, payload);
}
- const char *get_type_name() const { return "MOSDPGPull"; }
+ const char *get_type_name() const { return "MOSDECSubOpWrite"; }
void print(ostream& out) const {
- out << "MOSDPGPull(" << pgid
+ out << "MOSDECSubOpWrite(" << pgid
<< " " << map_epoch
- << " " << pulls;
+ << " " << op;
out << ")";
}
+
+ void clear_buffers() {
+ op.t = ObjectStore::Transaction();
+ op.log_entries.clear();
+ }
};
#endif
diff --git a/src/messages/MOSDPGPull.h b/src/messages/MOSDECSubOpWriteReply.h
similarity index 60%
copy from src/messages/MOSDPGPull.h
copy to src/messages/MOSDECSubOpWriteReply.h
index 870db7f..c2edfb3 100644
--- a/src/messages/MOSDPGPull.h
+++ b/src/messages/MOSDECSubOpWriteReply.h
@@ -12,62 +12,49 @@
*
*/
-#ifndef MOSDPGPULL_H
-#define MOSDPGPULL_H
+#ifndef MOSDECSUBOPWRITEREPLY_H
+#define MOSDECSUBOPWRITEREPLY_H
#include "msg/Message.h"
#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
-class MOSDPGPull : public Message {
+class MOSDECSubOpWriteReply : public Message {
static const int HEAD_VERSION = 1;
static const int COMPAT_VERSION = 1;
-
public:
- pg_t pgid;
+ spg_t pgid;
epoch_t map_epoch;
- vector<PullOp> pulls;
- uint64_t cost;
-
- MOSDPGPull() :
- Message(MSG_OSD_PG_PULL, HEAD_VERSION, COMPAT_VERSION),
- cost(0)
- {}
-
- void compute_cost(CephContext *cct) {
- cost = 0;
- for (vector<PullOp>::iterator i = pulls.begin();
- i != pulls.end();
- ++i) {
- cost += i->cost(cct);
- }
- }
+ ECSubWriteReply op;
int get_cost() const {
- return cost;
+ return 0;
}
+ MOSDECSubOpWriteReply() :
+ Message(MSG_OSD_EC_WRITE_REPLY, HEAD_VERSION, COMPAT_VERSION)
+ {}
+
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(pgid, p);
::decode(map_epoch, p);
- ::decode(pulls, p);
- ::decode(cost, p);
+ ::decode(op, p);
}
virtual void encode_payload(uint64_t features) {
::encode(pgid, payload);
::encode(map_epoch, payload);
- ::encode(pulls, payload);
- ::encode(cost, payload);
+ ::encode(op, payload);
}
- const char *get_type_name() const { return "MOSDPGPull"; }
+ const char *get_type_name() const { return "MOSDECSubOpWriteReply"; }
void print(ostream& out) const {
- out << "MOSDPGPull(" << pgid
+ out << "MOSDECSubOpWriteReply(" << pgid
<< " " << map_epoch
- << " " << pulls;
+ << " " << op;
out << ")";
}
};
diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 32408be..7a27838 100644
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -337,6 +337,9 @@ struct ceph_osd_request_head {
OSDOp::split_osd_op_vector_in_data(ops, data);
}
+ void clear_buffers() {
+ ops.clear();
+ }
const char *get_type_name() const { return "osd_op"; }
void print(ostream& out) const {
@@ -363,6 +366,8 @@ struct ceph_osd_request_head {
out << " " << pgid;
if (is_retry_attempt())
out << " RETRY=" << get_retry_attempt();
+ if (reassert_version != eversion_t())
+ out << " reassert_version=" << reassert_version;
if (get_snap_seq())
out << " snapc " << get_snap_seq() << "=" << snaps;
out << " " << ceph_osd_flag_string(get_flags());
diff --git a/src/messages/MOSDOpReply.h b/src/messages/MOSDOpReply.h
index c0e989f..9739ae1 100644
--- a/src/messages/MOSDOpReply.h
+++ b/src/messages/MOSDOpReply.h
@@ -124,7 +124,7 @@ public:
public:
MOSDOpReply()
: Message(CEPH_MSG_OSD_OPREPLY, HEAD_VERSION, COMPAT_VERSION) { }
- MOSDOpReply(MOSDOp *req, int r, epoch_t e, int acktype)
+ MOSDOpReply(MOSDOp *req, int r, epoch_t e, int acktype, bool ignore_out_data)
: Message(CEPH_MSG_OSD_OPREPLY, HEAD_VERSION, COMPAT_VERSION) {
set_tid(req->get_tid());
ops = req->ops;
@@ -137,9 +137,12 @@ public:
user_version = 0;
retry_attempt = req->get_retry_attempt();
- // zero out ops payload_len
- for (unsigned i = 0; i < ops.size(); i++)
+ // zero out ops payload_len and possibly out data
+ for (unsigned i = 0; i < ops.size(); i++) {
ops[i].op.payload_len = 0;
+ if (ignore_out_data)
+ ops[i].outdata.clear();
+ }
}
private:
~MOSDOpReply() {}
diff --git a/src/messages/MOSDPGBackfill.h b/src/messages/MOSDPGBackfill.h
index 5700f96..e9ec661 100644
--- a/src/messages/MOSDPGBackfill.h
+++ b/src/messages/MOSDPGBackfill.h
@@ -19,7 +19,7 @@
#include "osd/osd_types.h"
class MOSDPGBackfill : public Message {
- static const int HEAD_VERSION = 2;
+ static const int HEAD_VERSION = 3;
static const int COMPAT_VERSION = 1;
public:
enum {
@@ -38,7 +38,7 @@ public:
__u32 op;
epoch_t map_epoch, query_epoch;
- pg_t pgid;
+ spg_t pgid;
hobject_t last_backfill;
bool compat_stat_sum;
pg_stat_t stats;
@@ -48,7 +48,7 @@ public:
::decode(op, p);
::decode(map_epoch, p);
::decode(query_epoch, p);
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(last_backfill, p);
// For compatibility with version 1
@@ -64,25 +64,31 @@ public:
if (!last_backfill.is_max() &&
last_backfill.pool == -1)
last_backfill.pool = pgid.pool();
+ if (header.version >= 3)
+ ::decode(pgid.shard, p);
+ else
+ pgid.shard = ghobject_t::no_shard();
}
virtual void encode_payload(uint64_t features) {
::encode(op, payload);
::encode(map_epoch, payload);
::encode(query_epoch, payload);
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(last_backfill, payload);
// For compatibility with version 1
::encode(stats.stats, payload);
::encode(stats, payload);
+
+ ::encode(pgid.shard, payload);
}
MOSDPGBackfill() :
Message(MSG_OSD_PG_BACKFILL, HEAD_VERSION, COMPAT_VERSION),
compat_stat_sum(false) {}
- MOSDPGBackfill(__u32 o, epoch_t e, epoch_t qe, pg_t p)
+ MOSDPGBackfill(__u32 o, epoch_t e, epoch_t qe, spg_t p)
: Message(MSG_OSD_PG_BACKFILL, HEAD_VERSION, COMPAT_VERSION),
op(o),
map_epoch(e), query_epoch(e),
diff --git a/src/messages/MOSDPGInfo.h b/src/messages/MOSDPGInfo.h
index 448b43b..83e74fb 100644
--- a/src/messages/MOSDPGInfo.h
+++ b/src/messages/MOSDPGInfo.h
@@ -20,7 +20,7 @@
#include "osd/osd_types.h"
class MOSDPGInfo : public Message {
- static const int HEAD_VERSION = 3;
+ static const int HEAD_VERSION = 4;
static const int COMPAT_VERSION = 1;
epoch_t epoch;
@@ -79,6 +79,14 @@ public:
p++)
::encode(pair<epoch_t, epoch_t>(
p->first.epoch_sent, p->first.query_epoch), payload);
+
+ // v4 needs from, to
+ for (vector<pair<pg_notify_t, pg_interval_map_t> >::iterator p = pg_list.begin();
+ p != pg_list.end();
+ ++p) {
+ ::encode(p->first.from, payload);
+ ::encode(p->first.to, payload);
+ }
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
@@ -113,6 +121,16 @@ public:
i->first.query_epoch = epoch;
}
}
+
+ // v4 needs from and to
+ if (header.version >= 4) {
+ for (vector<pair<pg_notify_t, pg_interval_map_t> >::iterator i = pg_list.begin();
+ i != pg_list.end();
+ i++) {
+ ::decode(i->first.from, p);
+ ::decode(i->first.to, p);
+ }
+ }
}
};
diff --git a/src/messages/MOSDPGLog.h b/src/messages/MOSDPGLog.h
index 906a859..44cd989 100644
--- a/src/messages/MOSDPGLog.h
+++ b/src/messages/MOSDPGLog.h
@@ -20,7 +20,7 @@
class MOSDPGLog : public Message {
- static const int HEAD_VERSION = 3;
+ static const int HEAD_VERSION = 4;
static const int COMPAT_VERSION = 2;
epoch_t epoch;
@@ -31,22 +31,29 @@ class MOSDPGLog : public Message {
epoch_t query_epoch;
public:
+ shard_id_t to;
+ shard_id_t from;
pg_info_t info;
pg_log_t log;
pg_missing_t missing;
pg_interval_map_t past_intervals;
epoch_t get_epoch() { return epoch; }
- pg_t get_pgid() { return info.pgid; }
+ spg_t get_pgid() { return spg_t(info.pgid.pgid, to); }
epoch_t get_query_epoch() { return query_epoch; }
MOSDPGLog() : Message(MSG_OSD_PG_LOG, HEAD_VERSION, COMPAT_VERSION) { }
- MOSDPGLog(version_t mv, pg_info_t& i)
+ MOSDPGLog(shard_id_t to, shard_id_t from, version_t mv, pg_info_t& i)
: Message(MSG_OSD_PG_LOG, HEAD_VERSION, COMPAT_VERSION),
- epoch(mv), query_epoch(mv), info(i) { }
- MOSDPGLog(version_t mv, pg_info_t& i, epoch_t query_epoch)
+ epoch(mv), query_epoch(mv),
+ to(to), from(from),
+ info(i) { }
+ MOSDPGLog(shard_id_t to, shard_id_t from,
+ version_t mv, pg_info_t& i, epoch_t query_epoch)
: Message(MSG_OSD_PG_LOG, HEAD_VERSION, COMPAT_VERSION),
- epoch(mv), query_epoch(query_epoch), info(i) { }
+ epoch(mv), query_epoch(query_epoch),
+ to(to), from(from),
+ info(i) { }
private:
~MOSDPGLog() {}
@@ -66,6 +73,8 @@ public:
::encode(missing, payload);
::encode(query_epoch, payload);
::encode(past_intervals, payload);
+ ::encode(to, payload);
+ ::encode(from, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
@@ -79,6 +88,13 @@ public:
if (header.version >= 3) {
::decode(past_intervals, p);
}
+ if (header.version >= 4) {
+ ::decode(to, p);
+ ::decode(from, p);
+ } else {
+ to = ghobject_t::NO_SHARD;
+ from = ghobject_t::NO_SHARD;
+ }
}
};
diff --git a/src/messages/MOSDPGNotify.h b/src/messages/MOSDPGNotify.h
index 3d2b269..6b9bdb3 100644
--- a/src/messages/MOSDPGNotify.h
+++ b/src/messages/MOSDPGNotify.h
@@ -25,7 +25,7 @@
class MOSDPGNotify : public Message {
- static const int HEAD_VERSION = 4;
+ static const int HEAD_VERSION = 5;
static const int COMPAT_VERSION = 2;
epoch_t epoch;
@@ -83,6 +83,14 @@ public:
::encode(pair<epoch_t, epoch_t>(
p->first.epoch_sent, p->first.query_epoch),
payload);
+
+ // v5 needs from, to
+ for (vector<pair<pg_notify_t, pg_interval_map_t> >::iterator p = pg_list.begin();
+ p != pg_list.end();
+ ++p) {
+ ::encode(p->first.from, payload);
+ ::encode(p->first.to, payload);
+ }
}
void decode_payload() {
epoch_t query_epoch;
@@ -120,6 +128,16 @@ public:
i->first.query_epoch = query_epoch;
}
}
+
+ // v5 needs from and to
+ if (header.version >= 5) {
+ for (vector<pair<pg_notify_t, pg_interval_map_t> >::iterator i = pg_list.begin();
+ i != pg_list.end();
+ i++) {
+ ::decode(i->first.from, p);
+ ::decode(i->first.to, p);
+ }
+ }
}
void print(ostream& out) const {
out << "pg_notify(";
diff --git a/src/messages/MOSDPGPull.h b/src/messages/MOSDPGPull.h
index 870db7f..9107232 100644
--- a/src/messages/MOSDPGPull.h
+++ b/src/messages/MOSDPGPull.h
@@ -19,12 +19,13 @@
#include "osd/osd_types.h"
class MOSDPGPull : public Message {
- static const int HEAD_VERSION = 1;
+ static const int HEAD_VERSION = 2;
static const int COMPAT_VERSION = 1;
public:
- pg_t pgid;
+ pg_shard_t from;
+ spg_t pgid;
epoch_t map_epoch;
vector<PullOp> pulls;
uint64_t cost;
@@ -49,17 +50,26 @@ public:
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(map_epoch, p);
::decode(pulls, p);
::decode(cost, p);
+ if (header.version >= 2) {
+ ::decode(pgid.shard, p);
+ ::decode(from, p);
+ } else {
+ pgid.shard = ghobject_t::NO_SHARD;
+ from = pg_shard_t(get_source().num(), ghobject_t::NO_SHARD);
+ }
}
virtual void encode_payload(uint64_t features) {
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(map_epoch, payload);
::encode(pulls, payload);
::encode(cost, payload);
+ ::encode(pgid.shard, payload);
+ ::encode(from, payload);
}
const char *get_type_name() const { return "MOSDPGPull"; }
diff --git a/src/messages/MOSDPGPush.h b/src/messages/MOSDPGPush.h
index acc0d2a..46a8f1b 100644
--- a/src/messages/MOSDPGPush.h
+++ b/src/messages/MOSDPGPush.h
@@ -19,12 +19,13 @@
#include "osd/osd_types.h"
class MOSDPGPush : public Message {
- static const int HEAD_VERSION = 1;
+ static const int HEAD_VERSION = 2;
static const int COMPAT_VERSION = 1;
public:
- pg_t pgid;
+ pg_shard_t from;
+ spg_t pgid;
epoch_t map_epoch;
vector<PushOp> pushes;
uint64_t cost;
@@ -49,17 +50,26 @@ public:
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(map_epoch, p);
::decode(pushes, p);
::decode(cost, p);
+ if (header.version >= 2) {
+ ::decode(pgid.shard, p);
+ ::decode(from, p);
+ } else {
+ pgid.shard = ghobject_t::NO_SHARD;
+ from = pg_shard_t(get_source().num(), ghobject_t::NO_SHARD);
+ }
}
virtual void encode_payload(uint64_t features) {
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(map_epoch, payload);
::encode(pushes, payload);
::encode(cost, payload);
+ ::encode(pgid.shard, payload);
+ ::encode(from, payload);
}
const char *get_type_name() const { return "MOSDPGPush"; }
diff --git a/src/messages/MOSDPGPushReply.h b/src/messages/MOSDPGPushReply.h
index 192dc2c..1875235 100644
--- a/src/messages/MOSDPGPushReply.h
+++ b/src/messages/MOSDPGPushReply.h
@@ -19,11 +19,12 @@
#include "osd/osd_types.h"
class MOSDPGPushReply : public Message {
- static const int HEAD_VERSION = 1;
+ static const int HEAD_VERSION = 2;
static const int COMPAT_VERSION = 1;
public:
- pg_t pgid;
+ pg_shard_t from;
+ spg_t pgid;
epoch_t map_epoch;
vector<PushReplyOp> replies;
uint64_t cost;
@@ -48,17 +49,27 @@ public:
virtual void decode_payload() {
bufferlist::iterator p = payload.begin();
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(map_epoch, p);
::decode(replies, p);
::decode(cost, p);
+
+ if (header.version >= 2) {
+ ::decode(pgid.shard, p);
+ ::decode(from, p);
+ } else {
+ pgid.shard = ghobject_t::NO_SHARD;
+ from = pg_shard_t(get_source().num(), ghobject_t::NO_SHARD);
+ }
}
virtual void encode_payload(uint64_t features) {
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(map_epoch, payload);
::encode(replies, payload);
::encode(cost, payload);
+ ::encode(pgid.shard, payload);
+ ::encode(from, payload);
}
void print(ostream& out) const {
diff --git a/src/messages/MOSDPGQuery.h b/src/messages/MOSDPGQuery.h
index b637412..c2c6f69 100644
--- a/src/messages/MOSDPGQuery.h
+++ b/src/messages/MOSDPGQuery.h
@@ -16,6 +16,7 @@
#ifndef CEPH_MOSDPGQUERY_H
#define CEPH_MOSDPGQUERY_H
+#include "common/hobject.h"
#include "msg/Message.h"
/*
@@ -23,18 +24,18 @@
*/
class MOSDPGQuery : public Message {
- static const int HEAD_VERSION = 2;
+ static const int HEAD_VERSION = 3;
static const int COMPAT_VERSION = 1;
version_t epoch;
public:
version_t get_epoch() { return epoch; }
- map<pg_t,pg_query_t> pg_list;
+ map<spg_t, pg_query_t> pg_list;
MOSDPGQuery() : Message(MSG_OSD_PG_QUERY,
HEAD_VERSION,
COMPAT_VERSION) {}
- MOSDPGQuery(epoch_t e, map<pg_t,pg_query_t>& ls) :
+ MOSDPGQuery(epoch_t e, map<spg_t,pg_query_t>& ls) :
Message(MSG_OSD_PG_QUERY,
HEAD_VERSION,
COMPAT_VERSION),
@@ -48,7 +49,8 @@ public:
const char *get_type_name() const { return "pg_query"; }
void print(ostream& out) const {
out << "pg_query(";
- for (map<pg_t,pg_query_t>::const_iterator p = pg_list.begin(); p != pg_list.end(); ++p) {
+ for (map<spg_t,pg_query_t>::const_iterator p = pg_list.begin();
+ p != pg_list.end(); ++p) {
if (p != pg_list.begin())
out << ",";
out << p->first;
@@ -58,15 +60,38 @@ public:
void encode_payload(uint64_t features) {
::encode(epoch, payload);
- ::encode(pg_list, payload, features);
+ vector<pair<pg_t, pg_query_t> > _pg_list;
+ _pg_list.reserve(pg_list.size());
+ vector<shard_id_t> _shard_list;
+ _shard_list.reserve(pg_list.size());
+ for (map<spg_t, pg_query_t>::iterator i = pg_list.begin();
+ i != pg_list.end();
+ ++i) {
+ _pg_list.push_back(make_pair(i->first.pgid, i->second));
+ _shard_list.push_back(i->first.shard);
+ }
+ ::encode(_pg_list, payload, features);
+ ::encode(_shard_list, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(epoch, p);
- ::decode(pg_list, p);
+ vector<pair<pg_t, pg_query_t> > _pg_list;
+ ::decode(_pg_list, p);
+ vector<shard_id_t> _shard_list(_pg_list.size(), ghobject_t::no_shard());
+ if (header.version >= 3) {
+ _shard_list.clear();
+ ::decode(_shard_list, p);
+ }
+ assert(_pg_list.size() == _shard_list.size());
+ for (unsigned i = 0; i < _pg_list.size(); ++i) {
+ pg_list.insert(
+ make_pair(
+ spg_t(_pg_list[i].first, _shard_list[i]), _pg_list[i].second));
+ }
if (header.version < 2) {
- for (map<pg_t, pg_query_t>::iterator i = pg_list.begin();
+ for (map<spg_t, pg_query_t>::iterator i = pg_list.begin();
i != pg_list.end();
++i) {
i->second.epoch_sent = epoch;
diff --git a/src/messages/MOSDPGRemove.h b/src/messages/MOSDPGRemove.h
index c6ec797..b55b5d2 100644
--- a/src/messages/MOSDPGRemove.h
+++ b/src/messages/MOSDPGRemove.h
@@ -16,20 +16,26 @@
#ifndef CEPH_MOSDPGREMOVE_H
#define CEPH_MOSDPGREMOVE_H
+#include "common/hobject.h"
#include "msg/Message.h"
class MOSDPGRemove : public Message {
+
+ static const int HEAD_VERSION = 2;
+ static const int COMPAT_VERSION = 1;
+
epoch_t epoch;
public:
- vector<pg_t> pg_list;
+ vector<spg_t> pg_list;
epoch_t get_epoch() { return epoch; }
- MOSDPGRemove() : Message(MSG_OSD_PG_REMOVE) {}
- MOSDPGRemove(epoch_t e, vector<pg_t>& l) :
- Message(MSG_OSD_PG_REMOVE) {
+ MOSDPGRemove() :
+ Message(MSG_OSD_PG_REMOVE, HEAD_VERSION, COMPAT_VERSION) {}
+ MOSDPGRemove(epoch_t e, vector<spg_t>& l) :
+ Message(MSG_OSD_PG_REMOVE, HEAD_VERSION, COMPAT_VERSION) {
this->epoch = e;
pg_list.swap(l);
}
@@ -41,16 +47,38 @@ public:
void encode_payload(uint64_t features) {
::encode(epoch, payload);
- ::encode(pg_list, payload);
+
+ vector<pg_t> _pg_list;
+ _pg_list.reserve(pg_list.size());
+ vector<shard_id_t> _shard_list;
+ _shard_list.reserve(pg_list.size());
+ for (vector<spg_t>::iterator i = pg_list.begin(); i != pg_list.end(); ++i) {
+ _pg_list.push_back(i->pgid);
+ _shard_list.push_back(i->shard);
+ }
+ ::encode(_pg_list, payload);
+ ::encode(_shard_list, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(epoch, p);
- ::decode(pg_list, p);
+ vector<pg_t> _pg_list;
+ ::decode(_pg_list, p);
+
+ vector<shard_id_t> _shard_list(_pg_list.size(), ghobject_t::no_shard());
+ if (header.version >= 2) {
+ _shard_list.clear();
+ ::decode(_shard_list, p);
+ }
+ assert(_shard_list.size() == _pg_list.size());
+ pg_list.reserve(_shard_list.size());
+ for (unsigned i = 0; i < _shard_list.size(); ++i) {
+ pg_list.push_back(spg_t(_pg_list[i], _shard_list[i]));
+ }
}
void print(ostream& out) const {
out << "osd pg remove(" << "epoch " << epoch << "; ";
- for (vector<pg_t>::const_iterator i = pg_list.begin();
+ for (vector<spg_t>::const_iterator i = pg_list.begin();
i != pg_list.end();
++i) {
out << "pg" << *i << "; ";
diff --git a/src/messages/MOSDPGScan.h b/src/messages/MOSDPGScan.h
index 4c86a3c..2c0c1ad 100644
--- a/src/messages/MOSDPGScan.h
+++ b/src/messages/MOSDPGScan.h
@@ -19,6 +19,10 @@
#include "osd/osd_types.h"
class MOSDPGScan : public Message {
+
+ static const int HEAD_VERSION = 2;
+ static const int COMPAT_VERSION = 1;
+
public:
enum {
OP_SCAN_GET_DIGEST = 1, // just objects and versions
@@ -34,7 +38,8 @@ public:
__u32 op;
epoch_t map_epoch, query_epoch;
- pg_t pgid;
+ pg_shard_t from;
+ spg_t pgid;
hobject_t begin, end;
virtual void decode_payload() {
@@ -42,7 +47,7 @@ public:
::decode(op, p);
::decode(map_epoch, p);
::decode(query_epoch, p);
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(begin, p);
::decode(end, p);
@@ -51,22 +56,36 @@ public:
begin.pool = pgid.pool();
if (!end.is_max() && end.pool == -1)
end.pool = pgid.pool();
+
+ if (header.version >= 2) {
+ ::decode(from, p);
+ ::decode(pgid.shard, p);
+ } else {
+ from = pg_shard_t(
+ get_source().num(),
+ ghobject_t::NO_SHARD);
+ pgid.shard = ghobject_t::NO_SHARD;
+ }
}
virtual void encode_payload(uint64_t features) {
::encode(op, payload);
::encode(map_epoch, payload);
::encode(query_epoch, payload);
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(begin, payload);
::encode(end, payload);
+ ::encode(from, payload);
+ ::encode(pgid.shard, payload);
}
- MOSDPGScan() : Message(MSG_OSD_PG_SCAN) {}
- MOSDPGScan(__u32 o, epoch_t e, epoch_t qe, pg_t p, hobject_t be, hobject_t en)
- : Message(MSG_OSD_PG_SCAN),
+ MOSDPGScan() : Message(MSG_OSD_PG_SCAN, HEAD_VERSION, COMPAT_VERSION) {}
+ MOSDPGScan(__u32 o, pg_shard_t from,
+ epoch_t e, epoch_t qe, spg_t p, hobject_t be, hobject_t en)
+ : Message(MSG_OSD_PG_SCAN, HEAD_VERSION, COMPAT_VERSION),
op(o),
map_epoch(e), query_epoch(e),
+ from(from),
pgid(p),
begin(be), end(en) {
}
diff --git a/src/messages/MOSDPGTrim.h b/src/messages/MOSDPGTrim.h
index ad52a7f..12a0e7c 100644
--- a/src/messages/MOSDPGTrim.h
+++ b/src/messages/MOSDPGTrim.h
@@ -18,15 +18,19 @@
#include "msg/Message.h"
class MOSDPGTrim : public Message {
+
+ static const int HEAD_VERSION = 2;
+ static const int COMPAT_VERSION = 1;
+
public:
epoch_t epoch;
- pg_t pgid;
+ spg_t pgid;
eversion_t trim_to;
epoch_t get_epoch() { return epoch; }
- MOSDPGTrim() : Message(MSG_OSD_PG_TRIM) {}
- MOSDPGTrim(version_t mv, pg_t p, eversion_t tt) :
+ MOSDPGTrim() : Message(MSG_OSD_PG_TRIM, HEAD_VERSION, COMPAT_VERSION) {}
+ MOSDPGTrim(version_t mv, spg_t p, eversion_t tt) :
Message(MSG_OSD_PG_TRIM),
epoch(mv), pgid(p), trim_to(tt) { }
private:
@@ -40,14 +44,19 @@ public:
void encode_payload(uint64_t features) {
::encode(epoch, payload);
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(trim_to, payload);
+ ::encode(pgid.shard, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(epoch, p);
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(trim_to, p);
+ if (header.version >= 2)
+ ::decode(pgid.shard, p);
+ else
+ pgid.shard = ghobject_t::no_shard();
}
};
diff --git a/src/messages/MOSDRepScrub.h b/src/messages/MOSDRepScrub.h
index 4fae008..3f67021 100644
--- a/src/messages/MOSDRepScrub.h
+++ b/src/messages/MOSDRepScrub.h
@@ -24,10 +24,10 @@
struct MOSDRepScrub : public Message {
- static const int HEAD_VERSION = 4;
+ static const int HEAD_VERSION = 5;
static const int COMPAT_VERSION = 2;
- pg_t pgid; // PG to scrub
+ spg_t pgid; // PG to scrub
eversion_t scrub_from; // only scrub log entries after scrub_from
eversion_t scrub_to; // last_update_applied when message sent
epoch_t map_epoch;
@@ -40,7 +40,7 @@ struct MOSDRepScrub : public Message {
chunky(false),
deep(false) { }
- MOSDRepScrub(pg_t pgid, eversion_t scrub_from, eversion_t scrub_to,
+ MOSDRepScrub(spg_t pgid, eversion_t scrub_from, eversion_t scrub_to,
epoch_t map_epoch)
: Message(MSG_OSD_REP_SCRUB, HEAD_VERSION, COMPAT_VERSION),
pgid(pgid),
@@ -50,7 +50,7 @@ struct MOSDRepScrub : public Message {
chunky(false),
deep(false) { }
- MOSDRepScrub(pg_t pgid, eversion_t scrub_to, epoch_t map_epoch,
+ MOSDRepScrub(spg_t pgid, eversion_t scrub_to, epoch_t map_epoch,
hobject_t start, hobject_t end, bool deep)
: Message(MSG_OSD_REP_SCRUB, HEAD_VERSION, COMPAT_VERSION),
pgid(pgid),
@@ -78,7 +78,7 @@ public:
}
void encode_payload(uint64_t features) {
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(scrub_from, payload);
::encode(scrub_to, payload);
::encode(map_epoch, payload);
@@ -86,10 +86,11 @@ public:
::encode(start, payload);
::encode(end, payload);
::encode(deep, payload);
+ ::encode(pgid.shard, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(scrub_from, p);
::decode(scrub_to, p);
::decode(map_epoch, p);
@@ -107,6 +108,12 @@ public:
chunky = false;
deep = false;
}
+
+ if (header.version >= 5) {
+ ::decode(pgid.shard, p);
+ } else {
+ pgid.shard = ghobject_t::no_shard();
+ }
}
};
diff --git a/src/messages/MOSDSubOp.h b/src/messages/MOSDSubOp.h
index 7e9f087..9c405e6 100644
--- a/src/messages/MOSDSubOp.h
+++ b/src/messages/MOSDSubOp.h
@@ -25,7 +25,7 @@
class MOSDSubOp : public Message {
- static const int HEAD_VERSION = 8;
+ static const int HEAD_VERSION = 9;
static const int COMPAT_VERSION = 1;
public:
@@ -35,7 +35,8 @@ public:
osd_reqid_t reqid;
// subop
- pg_t pgid;
+ pg_shard_t from;
+ spg_t pgid;
hobject_t poid;
object_locator_t oloc;
@@ -64,7 +65,7 @@ public:
eversion_t pg_trim_to; // primary->replica: trim to here
osd_peer_stat_t peer_stat;
- map<string,bufferptr> attrset;
+ map<string,bufferlist> attrset;
interval_set<uint64_t> data_subset;
map<hobject_t, interval_set<uint64_t> > clone_subsets;
@@ -100,7 +101,7 @@ public:
bufferlist::iterator p = payload.begin();
::decode(map_epoch, p);
::decode(reqid, p);
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(poid, p);
__u32 num_ops;
@@ -158,12 +159,22 @@ public:
::decode(new_temp_oid, p);
::decode(discard_temp_oid, p);
}
+
+ if (header.version >= 9) {
+ ::decode(from, p);
+ ::decode(pgid.shard, p);
+ } else {
+ from = pg_shard_t(
+ get_source().num(),
+ ghobject_t::NO_SHARD);
+ pgid.shard = ghobject_t::NO_SHARD;
+ }
}
virtual void encode_payload(uint64_t features) {
::encode(map_epoch, payload);
::encode(reqid, payload);
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(poid, payload);
__u32 num_ops = ops.size();
@@ -204,15 +215,19 @@ public:
::encode(omap_header, payload);
::encode(new_temp_oid, payload);
::encode(discard_temp_oid, payload);
+ ::encode(from, payload);
+ ::encode(pgid.shard, payload);
}
MOSDSubOp()
: Message(MSG_OSD_SUBOP, HEAD_VERSION, COMPAT_VERSION) { }
- MOSDSubOp(osd_reqid_t r, pg_t p, const hobject_t& po, bool noop_, int aw,
+ MOSDSubOp(osd_reqid_t r, pg_shard_t from,
+ spg_t p, const hobject_t& po, bool noop_, int aw,
epoch_t mape, tid_t rtid, eversion_t v)
: Message(MSG_OSD_SUBOP, HEAD_VERSION, COMPAT_VERSION),
map_epoch(mape),
reqid(r),
+ from(from),
pgid(p),
poid(po),
acks_wanted(aw),
diff --git a/src/messages/MOSDSubOpReply.h b/src/messages/MOSDSubOpReply.h
index 6b0738a..270629f 100644
--- a/src/messages/MOSDSubOpReply.h
+++ b/src/messages/MOSDSubOpReply.h
@@ -30,12 +30,15 @@
*/
class MOSDSubOpReply : public Message {
+ static const int HEAD_VERSION = 2;
+ static const int COMPAT_VERSION = 1;
public:
epoch_t map_epoch;
// subop metadata
osd_reqid_t reqid;
- pg_t pgid;
+ pg_shard_t from;
+ spg_t pgid;
hobject_t poid;
vector<OSDOp> ops;
@@ -54,7 +57,7 @@ public:
bufferlist::iterator p = payload.begin();
::decode(map_epoch, p);
::decode(reqid, p);
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(poid, p);
unsigned num_ops;
@@ -71,11 +74,21 @@ public:
if (!poid.is_max() && poid.pool == -1)
poid.pool = pgid.pool();
+
+ if (header.version >= 2) {
+ ::decode(from, p);
+ ::decode(pgid.shard, p);
+ } else {
+ from = pg_shard_t(
+ get_source().num(),
+ ghobject_t::NO_SHARD);
+ pgid.shard = ghobject_t::NO_SHARD;
+ }
}
virtual void encode_payload(uint64_t features) {
::encode(map_epoch, payload);
::encode(reqid, payload);
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(poid, payload);
__u32 num_ops = ops.size();
::encode(num_ops, payload);
@@ -87,11 +100,13 @@ public:
::encode(last_complete_ondisk, payload);
::encode(peer_stat, payload);
::encode(attrset, payload);
+ ::encode(from, payload);
+ ::encode(pgid.shard, payload);
}
epoch_t get_map_epoch() { return map_epoch; }
- pg_t get_pg() { return pgid; }
+ spg_t get_pg() { return pgid; }
hobject_t get_poid() { return poid; }
int get_ack_type() { return ack_type; }
@@ -110,11 +125,13 @@ public:
map<string,bufferptr>& get_attrset() { return attrset; }
public:
- MOSDSubOpReply(MOSDSubOp *req, int result_, epoch_t e, int at) :
- Message(MSG_OSD_SUBOPREPLY),
+ MOSDSubOpReply(
+ MOSDSubOp *req, pg_shard_t from, int result_, epoch_t e, int at) :
+ Message(MSG_OSD_SUBOPREPLY, HEAD_VERSION, COMPAT_VERSION),
map_epoch(e),
reqid(req->reqid),
- pgid(req->pgid),
+ from(from),
+ pgid(req->pgid.pgid, req->from.shard),
poid(req->poid),
ops(req->ops),
ack_type(at),
diff --git a/src/messages/MRecoveryReserve.h b/src/messages/MRecoveryReserve.h
index e87d8af..2a88bfd 100644
--- a/src/messages/MRecoveryReserve.h
+++ b/src/messages/MRecoveryReserve.h
@@ -18,10 +18,10 @@
#include "msg/Message.h"
class MRecoveryReserve : public Message {
- static const int HEAD_VERSION = 1;
+ static const int HEAD_VERSION = 2;
static const int COMPAT_VERSION = 1;
public:
- pg_t pgid;
+ spg_t pgid;
epoch_t query_epoch;
enum {
REQUEST = 0,
@@ -34,7 +34,7 @@ public:
: Message(MSG_OSD_RECOVERY_RESERVE, HEAD_VERSION, COMPAT_VERSION),
query_epoch(0), type(-1) {}
MRecoveryReserve(int type,
- pg_t pgid,
+ spg_t pgid,
epoch_t query_epoch)
: Message(MSG_OSD_RECOVERY_RESERVE, HEAD_VERSION, COMPAT_VERSION),
pgid(pgid), query_epoch(query_epoch),
@@ -63,15 +63,20 @@ public:
void decode_payload() {
bufferlist::iterator p = payload.begin();
- ::decode(pgid, p);
+ ::decode(pgid.pgid, p);
::decode(query_epoch, p);
::decode(type, p);
+ if (header.version >= 2)
+ ::decode(pgid.shard, p);
+ else
+ pgid.shard = ghobject_t::no_shard();
}
void encode_payload(uint64_t features) {
- ::encode(pgid, payload);
+ ::encode(pgid.pgid, payload);
::encode(query_epoch, payload);
::encode(type, payload);
+ ::encode(pgid.shard, payload);
}
};
diff --git a/src/messages/Makefile.am b/src/messages/Makefile.am
index c503d3f..cac4048 100644
--- a/src/messages/Makefile.am
+++ b/src/messages/Makefile.am
@@ -87,6 +87,10 @@ noinst_HEADERS += \
messages/MOSDPGQuery.h \
messages/MOSDPGRemove.h \
messages/MOSDPGScan.h \
+ messages/MOSDECSubOpWrite.h \
+ messages/MOSDECSubOpWriteReply.h \
+ messages/MOSDECSubOpRead.h \
+ messages/MOSDECSubOpReadReply.h \
messages/MBackfillReserve.h \
messages/MRecoveryReserve.h \
messages/MMonQuorumService.h \
diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc
index 511d714..3572c8b 100644
--- a/src/mon/Elector.cc
+++ b/src/mon/Elector.cc
@@ -148,7 +148,10 @@ void Elector::expire()
victory();
} else {
// whoever i deferred to didn't declare victory quickly enough.
- start();
+ if (mon->has_ever_joined)
+ start();
+ else
+ mon->bootstrap();
}
}
diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 0f7bf27..d25b0cb 100644
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -919,10 +919,19 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "mds set_state") {
int64_t gid;
- if (!cmd_getval(g_ceph_context, cmdmap, "gid", gid))
+ if (!cmd_getval(g_ceph_context, cmdmap, "gid", gid)) {
+ ss << "error parsing 'gid' value '"
+ << cmd_vartype_stringify(cmdmap["gid"]) << "'";
+ r = -EINVAL;
goto out;
+ }
int64_t state;
- cmd_getval(g_ceph_context, cmdmap, "state", state);
+ if (!cmd_getval(g_ceph_context, cmdmap, "state", state)) {
+ ss << "error parsing 'state' string value '"
+ << cmd_vartype_stringify(cmdmap["state"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
if (!pending_mdsmap.is_dne_gid(gid)) {
MDSMap::mds_info_t& info = pending_mdsmap.get_info_gid(gid);
info.state = state;
@@ -946,7 +955,12 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "mds rm") {
int64_t gid;
- cmd_getval(g_ceph_context, cmdmap, "gid", gid);
+ if (!cmd_getval(g_ceph_context, cmdmap, "gid", gid)) {
+ ss << "error parsing 'gid' value '"
+ << cmd_vartype_stringify(cmdmap["gid"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
int state = pending_mdsmap.get_state_gid(gid);
if (state == 0) {
ss << "mds gid " << gid << " dne";
@@ -967,7 +981,12 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
}
} else if (prefix == "mds rmfailed") {
int64_t w;
- cmd_getval(g_ceph_context, cmdmap, "who", w);
+ if (!cmd_getval(g_ceph_context, cmdmap, "who", w)) {
+ ss << "error parsing 'who' value '"
+ << cmd_vartype_stringify(cmdmap["who"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
pending_mdsmap.failed.erase(w);
stringstream ss;
ss << "removed failed mds." << w;
@@ -994,7 +1013,12 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
r = 0;
} else if (prefix == "mds compat rm_compat") {
int64_t f;
- cmd_getval(g_ceph_context, cmdmap, "feature", f);
+ if (!cmd_getval(g_ceph_context, cmdmap, "feature", f)) {
+ ss << "error parsing feature value '"
+ << cmd_vartype_stringify(cmdmap["feature"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
if (pending_mdsmap.compat.compat.contains(f)) {
ss << "removing compat feature " << f;
pending_mdsmap.compat.compat.remove(f);
@@ -1005,7 +1029,12 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
}
} else if (prefix == "mds compat rm_incompat") {
int64_t f;
- cmd_getval(g_ceph_context, cmdmap, "feature", f);
+ if (!cmd_getval(g_ceph_context, cmdmap, "feature", f)) {
+ ss << "error parsing feature value '"
+ << cmd_vartype_stringify(cmdmap["feature"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
if (pending_mdsmap.compat.incompat.contains(f)) {
ss << "removing incompat feature " << f;
pending_mdsmap.compat.incompat.remove(f);
@@ -1046,6 +1075,13 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
ss << "pool '" << poolname << "' does not exist";
}
}
+
+ if (pending_mdsmap.get_first_data_pool() == poolid) {
+ r = -EINVAL;
+ poolid = -1;
+ ss << "cannot remove default data pool";
+ }
+
if (poolid >= 0) {
cmd_getval(g_ceph_context, cmdmap, "poolid", poolid);
r = pending_mdsmap.remove_data_pool(poolid);
@@ -1057,8 +1093,18 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "mds newfs") {
MDSMap newmap;
int64_t metadata, data;
- cmd_getval(g_ceph_context, cmdmap, "metadata", metadata);
- cmd_getval(g_ceph_context, cmdmap, "data", data);
+ if (!cmd_getval(g_ceph_context, cmdmap, "metadata", metadata)) {
+ ss << "error parsing 'metadata' value '"
+ << cmd_vartype_stringify(cmdmap["metadata"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
+ if (!cmd_getval(g_ceph_context, cmdmap, "data", data)) {
+ ss << "error parsing 'data' value '"
+ << cmd_vartype_stringify(cmdmap["data"]) << "'";
+ r = -EINVAL;
+ goto out;
+ }
string sure;
cmd_getval(g_ceph_context, cmdmap, "sure", sure);
if (sure != "--yes-i-really-mean-it") {
diff --git a/src/mon/Makefile.am b/src/mon/Makefile.am
index 68c6503..2e91517 100644
--- a/src/mon/Makefile.am
+++ b/src/mon/Makefile.am
@@ -14,7 +14,7 @@ libmon_la_SOURCES = \
mon/HealthMonitor.cc \
mon/DataHealthService.cc \
mon/ConfigKeyService.cc
-libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS)
+libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS) $(LIBERASURE_CODE)
noinst_LTLIBRARIES += libmon.la
noinst_HEADERS += \
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index a77a5f8..25dd3cd 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -66,6 +66,9 @@ MonClient::MonClient(CephContext *cct_) :
want_monmap(true),
want_keys(0), global_id(0),
authenticate_err(0),
+ session_established_context(NULL),
+ had_a_connection(false),
+ reopen_interval_multiplier(1.0),
auth(NULL),
keyring(NULL),
rotating_secrets(NULL),
@@ -77,6 +80,7 @@ MonClient::MonClient(CephContext *cct_) :
MonClient::~MonClient()
{
delete auth_supported;
+ delete session_established_context;
delete auth;
delete keyring;
delete rotating_secrets;
@@ -462,6 +466,7 @@ int MonClient::authenticate(double timeout)
void MonClient::handle_auth(MAuthReply *m)
{
+ Context *cb = NULL;
bufferlist::iterator p = m->result_bl.begin();
if (state == MC_STATE_NEGOTIATING) {
if (!auth || (int)m->protocol != auth->get_protocol()) {
@@ -521,11 +526,20 @@ void MonClient::handle_auth(MAuthReply *m)
log_client->reset_session();
send_log();
}
+ if (session_established_context) {
+ cb = session_established_context;
+ session_established_context = NULL;
+ }
}
_check_auth_tickets();
}
auth_cond.SignalAll();
+ if (cb) {
+ monc_lock.Unlock();
+ cb->complete(0);
+ monc_lock.Lock();
+ }
}
@@ -601,8 +615,18 @@ void MonClient::_reopen_session(int rank, string name)
version_requests.erase(version_requests.begin());
}
+ // adjust timeouts if necessary
+ if (had_a_connection) {
+ reopen_interval_multiplier *= cct->_conf->mon_client_hunt_interval_backoff;
+ if (reopen_interval_multiplier >
+ cct->_conf->mon_client_hunt_interval_max_multiple)
+ reopen_interval_multiplier =
+ cct->_conf->mon_client_hunt_interval_max_multiple;
+ }
+
// restart authentication handshake
state = MC_STATE_NEGOTIATING;
+ hunting = true;
MAuth *m = new MAuth;
m->protocol = 0;
@@ -633,7 +657,6 @@ bool MonClient::ms_handle_reset(Connection *con)
return true;
ldout(cct, 0) << "hunting for new mon" << dendl;
- hunting = true;
_reopen_session();
}
}
@@ -646,6 +669,10 @@ void MonClient::_finish_hunting()
if (hunting) {
ldout(cct, 1) << "found mon." << cur_mon << dendl;
hunting = false;
+ had_a_connection = true;
+ reopen_interval_multiplier /= 2.0;
+ if (reopen_interval_multiplier < 1.0)
+ reopen_interval_multiplier = 1.0;
}
}
@@ -684,7 +711,8 @@ void MonClient::tick()
void MonClient::schedule_tick()
{
if (hunting)
- timer.add_event_after(cct->_conf->mon_client_hunt_interval, new C_Tick(this));
+ timer.add_event_after(cct->_conf->mon_client_hunt_interval
+ * reopen_interval_multiplier, new C_Tick(this));
else
timer.add_event_after(cct->_conf->mon_client_ping_interval, new C_Tick(this));
}
@@ -882,6 +910,23 @@ void MonClient::handle_mon_command_ack(MMonCommandAck *ack)
ack->put();
}
+int MonClient::_cancel_mon_command(uint64_t tid, int r)
+{
+ assert(monc_lock.is_locked());
+
+ map<tid_t, MonCommand*>::iterator it = mon_commands.find(tid);
+ if (it == mon_commands.end()) {
+ ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+ return -ENOENT;
+ }
+
+ ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+ MonCommand *cmd = it->second;
+ _finish_command(cmd, -ETIMEDOUT, "");
+ return 0;
+}
+
void MonClient::_finish_command(MonCommand *r, int ret, string rs)
{
ldout(cct, 10) << "_finish_command " << r->tid << " = " << ret << " " << rs << dendl;
@@ -907,13 +952,17 @@ int MonClient::start_mon_command(const vector<string>& cmd,
r->poutbl = outbl;
r->prs = outs;
r->onfinish = onfinish;
+ if (cct->_conf->rados_mon_op_timeout > 0) {
+ r->ontimeout = new C_CancelMonCommand(r->tid, this);
+ timer.add_event_after(cct->_conf->rados_mon_op_timeout, r->ontimeout);
+ }
mon_commands[r->tid] = r;
_send_command(r);
// can't fail
return 0;
}
-int MonClient::start_mon_command(string name,
+int MonClient::start_mon_command(const string &mon_name,
const vector<string>& cmd,
const bufferlist& inbl,
bufferlist *outbl, string *outs,
@@ -921,7 +970,7 @@ int MonClient::start_mon_command(string name,
{
Mutex::Locker l(monc_lock);
MonCommand *r = new MonCommand(++last_mon_command_tid);
- r->target_name = name;
+ r->target_name = mon_name;
r->cmd = cmd;
r->inbl = inbl;
r->poutbl = outbl;
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 1404f35..a2998bb 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -179,6 +179,9 @@ private:
int authenticate_err;
list<Message*> waiting_for_session;
+ Context *session_established_context;
+ bool had_a_connection;
+ double reopen_interval_multiplier;
string _pick_random_mon();
void _finish_hunting();
@@ -246,6 +249,22 @@ public:
Mutex::Locker l(monc_lock);
_sub_unwant(what);
}
+ /**
+ * Increase the requested subscription start point. If you do increase
+ * the value, apply the passed-in flags as well; otherwise do nothing.
+ */
+ bool sub_want_increment(string what, version_t start, unsigned flags) {
+ Mutex::Locker l(monc_lock);
+ map<string,ceph_mon_subscribe_item>::iterator i =
+ sub_have.find(what);
+ if (i == sub_have.end() || i->second.start < start) {
+ ceph_mon_subscribe_item& item = sub_have[what];
+ item.start = start;
+ item.flags = flags;
+ return true;
+ }
+ return false;
+ }
KeyRing *keyring;
RotatingKeyRing *rotating_secrets;
@@ -280,8 +299,19 @@ public:
Mutex::Locker l(monc_lock);
_send_mon_message(m);
}
- void reopen_session() {
+ /**
+ * If you specify a callback, you should not call
+ * reopen_session() again until it has been triggered. The MonClient
+ * will behave, but the first callback could be triggered after
+ * the session has been killed and the MonClient has started trying
+ * to reconnect to another monitor.
+ */
+ void reopen_session(Context *cb=NULL) {
Mutex::Locker l(monc_lock);
+ if (cb) {
+ delete session_established_context;
+ session_established_context = cb;
+ }
_reopen_session();
}
@@ -344,18 +374,30 @@ private:
bufferlist *poutbl;
string *prs;
int *prval;
- Context *onfinish;
+ Context *onfinish, *ontimeout;
MonCommand(uint64_t t)
: target_rank(-1),
tid(t),
- poutbl(NULL), prs(NULL), prval(NULL), onfinish(NULL)
+ poutbl(NULL), prs(NULL), prval(NULL), onfinish(NULL), ontimeout(NULL)
{}
};
map<uint64_t,MonCommand*> mon_commands;
+ class C_CancelMonCommand : public Context
+ {
+ uint64_t tid;
+ MonClient *monc;
+ public:
+ C_CancelMonCommand(uint64_t tid, MonClient *monc) : tid(tid), monc(monc) {}
+ void finish(int r) {
+ monc->_cancel_mon_command(tid, -ETIMEDOUT);
+ }
+ };
+
void _send_command(MonCommand *r);
void _resend_mon_commands();
+ int _cancel_mon_command(uint64_t tid, int r);
void _finish_command(MonCommand *r, int ret, string rs);
void handle_mon_command_ack(MMonCommandAck *ack);
@@ -367,7 +409,7 @@ public:
const vector<string>& cmd, const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish);
- int start_mon_command(const string mon_name, ///< mon name, with mon. prefix
+ int start_mon_command(const string &mon_name, ///< mon name, with mon. prefix
const vector<string>& cmd, const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish);
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index bff1f2e..a6bf5e5 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -363,8 +363,13 @@ COMMAND("osd lspools " \
COMMAND("osd blacklist ls", "show blacklisted clients", "osd", "r", "cli,rest")
COMMAND("osd crush rule list", "list crush rules", "osd", "r", "cli,rest")
COMMAND("osd crush rule ls", "list crush rules", "osd", "r", "cli,rest")
-COMMAND("osd crush rule dump", "dump crush rules", "osd", "r", "cli,rest")
-COMMAND("osd crush dump", "dump crush map", "osd", "r", "cli,rest")
+COMMAND("osd crush rule dump " \
+ "name=name,type=CephString,goodchars=[A-Za-z0-9-_.],req=false", \
+ "dump crush rule <name> (default all)", \
+ "osd", "r", "cli,rest")
+COMMAND("osd crush dump", \
+ "dump crush map", \
+ "osd", "r", "cli,rest")
COMMAND("osd setcrushmap", "set crush map from input file", \
"osd", "rw", "cli,rest")
COMMAND("osd crush set", "set crush map from input file", \
@@ -423,7 +428,7 @@ COMMAND("osd crush reweight " \
"change <name>'s weight to <weight> in crush map", \
"osd", "rw", "cli,rest")
COMMAND("osd crush tunables " \
- "name=profile,type=CephChoices,strings=legacy|argonaut|bobtail|optimal|default", \
+ "name=profile,type=CephChoices,strings=legacy|argonaut|bobtail|firefly|optimal|default", \
"set crush tunables values to <profile>", "osd", "rw", "cli,rest")
COMMAND("osd crush show-tunables", \
"show current crush tunables", "osd", "r", "cli,rest")
@@ -434,6 +439,11 @@ COMMAND("osd crush rule create-simple " \
"name=mode,type=CephChoices,strings=firstn|indep,req=false",
"create crush rule <name> to start from <root>, replicate across buckets of type <type>, using a choose mode of <firstn|indep> (default firstn; indep best for erasure pools)", \
"osd", "rw", "cli,rest")
+COMMAND("osd crush rule create-erasure " \
+ "name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+ "name=properties,type=CephString,n=N,req=false,goodchars=[A-Za-z0-9-_.=]", \
+ "create crush rule <name> suitable for erasure coded pool created with <properties>", \
+ "osd", "rw", "cli,rest")
COMMAND("osd crush rule rm " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] ", \
"remove crush rule <name>", "osd", "rw", "cli,rest")
@@ -443,10 +453,10 @@ COMMAND("osd setmaxosd " \
COMMAND("osd pause", "pause osd", "osd", "rw", "cli,rest")
COMMAND("osd unpause", "unpause osd", "osd", "rw", "cli,rest")
COMMAND("osd set " \
- "name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub", \
+ "name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub|notieragent", \
"set <key>", "osd", "rw", "cli,rest")
COMMAND("osd unset " \
- "name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub", \
+ "name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub|notieragent", \
"unset <key>", "osd", "rw", "cli,rest")
COMMAND("osd cluster_snap", "take cluster snapshot (disabled)", \
"osd", "r", "")
@@ -466,6 +476,11 @@ COMMAND("osd reweight " \
"name=id,type=CephInt,range=0 " \
"type=CephFloat,name=weight,range=0.0|1.0", \
"reweight osd to 0.0 < <weight> < 1.0", "osd", "rw", "cli,rest")
+COMMAND("osd primary-affinity " \
+ "name=id,type=CephOsdName " \
+ "type=CephFloat,name=weight,range=0.0|1.0", \
+ "adjust osd primary-affinity from 0.0 <= <weight> <= 1.0", \
+ "osd", "rw", "cli,rest")
COMMAND("osd lost " \
"name=id,type=CephInt,range=0 " \
"name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
@@ -511,7 +526,7 @@ COMMAND("osd pool get " \
"get pool parameter <var>", "osd", "r", "cli,rest")
COMMAND("osd pool set " \
"name=pool,type=CephPoolname " \
- "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|debug_fake_ec_pool " \
+ "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|debug_fake_ec_pool||target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age " \
"name=val,type=CephString", \
"set pool parameter <var> to <val>", "osd", "rw", "cli,rest")
// 'val' is a CephString because it can include a unit. Perhaps
@@ -537,7 +552,8 @@ COMMAND("osd thrash " \
// tiering
COMMAND("osd tier add " \
"name=pool,type=CephPoolname " \
- "name=tierpool,type=CephPoolname",
+ "name=tierpool,type=CephPoolname " \
+ "name=force_nonempty,type=CephChoices,strings=--force-nonempty,req=false",
"add the tier <tierpool> to base pool <pool>", "osd", "rw", "cli,rest")
COMMAND("osd tier remove " \
"name=pool,type=CephPoolname " \
@@ -555,6 +571,13 @@ COMMAND("osd tier remove-overlay " \
"name=pool,type=CephPoolname ", \
"remove the overlay pool for base pool <pool>", "osd", "rw", "cli,rest")
+COMMAND("osd tier add-cache " \
+ "name=pool,type=CephPoolname " \
+ "name=tierpool,type=CephPoolname " \
+ "name=size,type=CephInt,range=0", \
+ "add a cache <tierpool> of size <size> to existing pool <pool>", \
+ "osd", "rw", "cli,rest")
+
/*
* mon/ConfigKeyService.cc
*/
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 35d880c..dac9c3e 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -277,6 +277,8 @@ void Monitor::do_admin_command(string command, cmdmap_t& cmdmap, string format,
sync_force(f.get(), ss);
} else if (command.find("add_bootstrap_peer_hint") == 0) {
_add_bootstrap_peer_hint(command, cmdmap, ss);
+ } else if (command.find("osdmonitor_prepare_command") == 0) {
+ _osdmonitor_prepare_command(cmdmap, ss);
} else if (command == "quorum enter") {
elector.start_participating();
start_election();
@@ -518,6 +520,11 @@ int Monitor::preinit()
r = admin_socket->register_command("mon_status", "mon_status", admin_hook,
"show current monitor status");
assert(r == 0);
+ if (g_conf->mon_advanced_debug_mode) {
+ r = admin_socket->register_command("osdmonitor_prepare_command", "osdmonitor_prepare_command", admin_hook,
+ "call OSDMonitor::prepare_command");
+ assert(r == 0);
+ }
r = admin_socket->register_command("quorum_status", "quorum_status",
admin_hook, "show current quorum status");
assert(r == 0);
@@ -746,10 +753,34 @@ void Monitor::bootstrap()
}
}
+void Monitor::_osdmonitor_prepare_command(cmdmap_t& cmdmap, ostream& ss)
+{
+ if (!is_leader()) {
+ ss << "mon must be a leader";
+ return;
+ }
+
+ string cmd;
+ cmd_getval(g_ceph_context, cmdmap, "prepare", cmd);
+ cmdmap["prefix"] = cmdmap["prepare"];
+
+ OSDMonitor *monitor = osdmon();
+ MMonCommand *m = static_cast<MMonCommand *>((new MMonCommand())->get());
+ if (monitor->prepare_command_impl(m, cmdmap))
+ ss << "true";
+ else
+ ss << "false";
+ m->put();
+}
+
void Monitor::_add_bootstrap_peer_hint(string cmd, cmdmap_t& cmdmap, ostream& ss)
{
string addrstr;
- cmd_getval(g_ceph_context, cmdmap, "addr", addrstr);
+ if (!cmd_getval(g_ceph_context, cmdmap, "addr", addrstr)) {
+ ss << "unable to parse address string value '"
+ << cmd_vartype_stringify(cmdmap["addr"]) << "'";
+ return;
+ }
dout(10) << "_add_bootstrap_peer_hint '" << cmd << "' '"
<< addrstr << "'" << dendl;
@@ -1388,15 +1419,15 @@ void Monitor::handle_probe_reply(MMonProbe *m)
}
// new initial peer?
- if (monmap->contains(m->name)) {
- if (monmap->get_addr(m->name).is_blank_ip()) {
- dout(1) << " learned initial mon " << m->name << " addr " << m->get_source_addr() << dendl;
- monmap->set_addr(m->name, m->get_source_addr());
- m->put();
+ if (monmap->get_epoch() == 0 &&
+ monmap->contains(m->name) &&
+ monmap->get_addr(m->name).is_blank_ip()) {
+ dout(1) << " learned initial mon " << m->name << " addr " << m->get_source_addr() << dendl;
+ monmap->set_addr(m->name, m->get_source_addr());
+ m->put();
- bootstrap();
- return;
- }
+ bootstrap();
+ return;
}
// end discover phase
@@ -1560,12 +1591,22 @@ void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,
classic_mons = *classic_monitors;
paxos->leader_init();
- for (vector<PaxosService*>::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)
- (*p)->election_finished();
+ // NOTE: tell monmap monitor first. This is important for the
+ // bootstrap case to ensure that the very first paxos proposal
+ // codifies the monmap. Otherwise any manner of chaos can ensue
+ // when monitors are call elections or participating in a paxos
+ // round without agreeing on who the participants are.
+ monmon()->election_finished();
+ for (vector<PaxosService*>::iterator p = paxos_service.begin();
+ p != paxos_service.end(); ++p) {
+ if (*p != monmon())
+ (*p)->election_finished();
+ }
health_monitor->start(epoch);
finish_election();
- if (monmap->size() > 1)
+ if (monmap->size() > 1 &&
+ monmap->get_epoch() > 0)
timecheck_start();
}
@@ -1681,9 +1722,9 @@ void Monitor::_quorum_status(Formatter *f, ostream& ss)
f->dump_int("mon", *p);
f->close_section(); // quorum
- set<string> quorum_names = get_quorum_names();
+ list<string> quorum_names = get_quorum_names();
f->open_array_section("quorum_names");
- for (set<string>::iterator p = quorum_names.begin(); p != quorum_names.end(); ++p)
+ for (list<string>::iterator p = quorum_names.begin(); p != quorum_names.end(); ++p)
f->dump_string("mon", *p);
f->close_section(); // quorum_names
@@ -1939,7 +1980,7 @@ void Monitor::get_status(stringstream &ss, Formatter *f)
ss << " cluster " << monmap->get_fsid() << "\n";
ss << " health " << health << "\n";
ss << " monmap " << *monmap << ", election epoch " << get_epoch()
- << ", quorum " << get_quorum() << " " << get_quorum_names() << "\n";
+ << ", quorum " << get_quorum() << " " << get_quorum_names() << "\n";
if (mdsmon()->mdsmap.get_epoch() > 1)
ss << " mdsmap " << mdsmon()->mdsmap << "\n";
osdmon()->osdmap.print_summary(NULL, ss);
@@ -1985,7 +2026,7 @@ const MonCommand *Monitor::_get_moncommand(const string &cmd_prefix,
bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix,
const map<string,cmd_vartype>& cmdmap,
- const map<string,string> param_str_map,
+ const map<string,string>& param_str_map,
const MonCommand *this_cmd) {
bool cmd_r = (this_cmd->req_perms.find('r') != string::npos);
@@ -2382,6 +2423,9 @@ void Monitor::handle_command(MMonCommand *m)
start_election();
rs = "started responding to quorum, initiated new election";
r = 0;
+ } else {
+ rs = "needs a valid 'quorum' command";
+ r = -EINVAL;
}
}
@@ -2526,7 +2570,14 @@ void Monitor::try_send_message(Message *m, const entity_inst_t& to)
void Monitor::send_reply(PaxosServiceMessage *req, Message *reply)
{
- MonSession *session = static_cast<MonSession*>(req->get_connection()->get_priv());
+ ConnectionRef connection = req->get_connection();
+ if (!connection) {
+ dout(2) << "send_reply no connection, dropping reply " << *reply
+ << " to " << req << " " << *req << dendl;
+ reply->put();
+ return;
+ }
+ MonSession *session = static_cast<MonSession*>(connection->get_priv());
if (!session) {
dout(2) << "send_reply no session, dropping reply " << *reply
<< " to " << req << " " << *req << dendl;
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 47fa658..e8828a1 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -516,10 +516,10 @@ public:
epoch_t get_epoch();
int get_leader() { return leader; }
const set<int>& get_quorum() { return quorum; }
- set<string> get_quorum_names() {
- set<string> q;
+ list<string> get_quorum_names() {
+ list<string> q;
for (set<int>::iterator p = quorum.begin(); p != quorum.end(); ++p)
- q.insert(monmap->get_name(*p));
+ q.push_back(monmap->get_name(*p));
return q;
}
uint64_t get_quorum_features() const {
@@ -616,10 +616,11 @@ public:
MonCommand *cmds, int cmds_size);
bool _allowed_command(MonSession *s, string &module, string &prefix,
const map<string,cmd_vartype>& cmdmap,
- const map<string,string> param_str_map,
+ const map<string,string>& param_str_map,
const MonCommand *this_cmd);
void _mon_status(Formatter *f, ostream& ss);
void _quorum_status(Formatter *f, ostream& ss);
+ void _osdmonitor_prepare_command(cmdmap_t& cmdmap, ostream& ss);
void _add_bootstrap_peer_hint(string cmd, cmdmap_t& cmdmap, ostream& ss);
void handle_command(class MMonCommand *m);
void handle_route(MRoute *m);
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 8bf63cc..a3ec269 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -44,6 +44,8 @@
#include "common/config.h"
#include "common/errno.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
#include "include/compat.h"
#include "include/assert.h"
#include "include/stringify.h"
@@ -201,9 +203,17 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap)
if (t == NULL)
t = new MonitorDBStore::Transaction;
- // write out the full map for all past epochs
+ // Write out the full map for all past epochs. Encode the full
+ // map with the same features as the incremental. If we don't
+ // know, use the quorum features. If we don't know those either,
+ // encode with all features.
+ uint64_t f = inc.encode_features;
+ if (!f)
+ f = mon->quorum_features;
+ if (!f)
+ f = -1;
bufferlist full_bl;
- osdmap.encode(full_bl, inc.encode_features);
+ osdmap.encode(full_bl, f);
tx_size += full_bl.length();
put_version_full(t, osdmap.epoch, full_bl);
@@ -343,18 +353,30 @@ bool OSDMonitor::thrash()
while (n--)
++p;
for (int i=0; i<50; i++) {
+ unsigned size = osdmap.get_pg_size(p->first);
vector<int> v;
- for (int j=0; j<3; j++) {
+ bool have_real_osd = false;
+ for (int j=0; j < (int)size; j++) {
o = rand() % osdmap.get_num_osds();
- if (osdmap.exists(o) && std::find(v.begin(), v.end(), o) == v.end())
+ if (osdmap.exists(o) && std::find(v.begin(), v.end(), o) == v.end()) {
+ have_real_osd = true;
v.push_back(o);
+ }
}
- if (v.size() < 3) {
- for (vector<int>::iterator q = p->second.acting.begin(); q != p->second.acting.end(); ++q)
- if (std::find(v.begin(), v.end(), *q) == v.end())
- v.push_back(*q);
+ for (vector<int>::iterator q = p->second.acting.begin();
+ q != p->second.acting.end() && v.size() < size;
+ ++q) {
+ if (std::find(v.begin(), v.end(), *q) == v.end()) {
+ if (*q != CRUSH_ITEM_NONE)
+ have_real_osd = true;
+ v.push_back(*q);
+ }
}
- if (!v.empty())
+ if (osdmap.pg_is_ec(p->first)) {
+ while (v.size() < size)
+ v.push_back(CRUSH_ITEM_NONE);
+ }
+ if (!v.empty() && have_real_osd)
pending_inc.new_pg_temp[p->first] = v;
dout(5) << "thrash_map pg " << p->first << " pg_temp remapped to " << v << dendl;
@@ -725,7 +747,8 @@ bool OSDMonitor::should_propose(double& delay)
return true;
// adjust osd weights?
- if (osd_weight.size() == (unsigned)osdmap.get_max_osd()) {
+ if (!osd_weight.empty() &&
+ osd_weight.size() == (unsigned)osdmap.get_max_osd()) {
dout(0) << " adjusting osd weights based on " << osd_weight << dendl;
osdmap.adjust_osd_weights(osd_weight, pending_inc);
delay = 0.0;
@@ -1997,7 +2020,8 @@ void OSDMonitor::get_health(list<pair<health_status_t,string> >& summary,
CEPH_OSDMAP_NOBACKFILL |
CEPH_OSDMAP_NORECOVER |
CEPH_OSDMAP_NOSCRUB |
- CEPH_OSDMAP_NODEEP_SCRUB)) {
+ CEPH_OSDMAP_NODEEP_SCRUB |
+ CEPH_OSDMAP_NOTIERAGENT)) {
ostringstream ss;
ss << osdmap.get_flag_string() << " flag(s) set";
summary.push_back(make_pair(HEALTH_WARN, ss.str()));
@@ -2007,9 +2031,9 @@ void OSDMonitor::get_health(list<pair<health_status_t,string> >& summary,
// old crush tunables?
if (g_conf->mon_warn_on_legacy_crush_tunables) {
- if (!osdmap.crush->has_optimal_tunables()) {
+ if (osdmap.crush->has_legacy_tunables()) {
ostringstream ss;
- ss << "crush map has non-optimal tunables";
+ ss << "crush map has legacy tunables";
summary.push_back(make_pair(HEALTH_WARN, ss.str()));
if (detail) {
ss << "; see http://ceph.com/docs/master/rados/operations/crush-map/#tunables";
@@ -2185,7 +2209,12 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
}
} else if (prefix == "osd find") {
int64_t osd;
- cmd_getval(g_ceph_context, cmdmap, "id", osd);
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", osd)) {
+ ss << "unable to parse osd id value '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ r = -EINVAL;
+ goto reply;
+ }
if (!osdmap.exists(osd)) {
ss << "osd." << osd << " does not exist";
r = -ENOENT;
@@ -2207,7 +2236,12 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
f->flush(rdata);
} else if (prefix == "osd metadata") {
int64_t osd;
- cmd_getval(g_ceph_context, cmdmap, "id", osd);
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", osd)) {
+ ss << "unable to parse osd id value '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ r = -EINVAL;
+ goto reply;
+ }
if (!osdmap.exists(osd)) {
ss << "osd." << osd << " does not exist";
r = -ENOENT;
@@ -2526,15 +2560,27 @@ stats_out:
rs << "\n";
rdata.append(rs.str());
} else if (prefix == "osd crush rule dump") {
+ string name;
+ cmd_getval(g_ceph_context, cmdmap, "name", name);
string format;
cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
Formatter *fp = new_formatter(format);
if (!fp)
fp = new_formatter("json-pretty");
boost::scoped_ptr<Formatter> f(fp);
- f->open_array_section("rules");
- osdmap.crush->dump_rules(f.get());
- f->close_section();
+ if (name == "") {
+ f->open_array_section("rules");
+ osdmap.crush->dump_rules(f.get());
+ f->close_section();
+ } else {
+ int ruleset = osdmap.crush->get_rule_id(name);
+ if (ruleset < 0) {
+ ss << "unknown crush ruleset '" << name << "'";
+ r = ruleset;
+ goto reply;
+ }
+ osdmap.crush->dump_rule(ruleset, f.get());
+ }
ostringstream rs;
f->flush(rs);
rs << "\n";
@@ -2741,6 +2787,202 @@ int OSDMonitor::prepare_new_pool(MPoolOp *m)
properties, pg_pool_t::TYPE_REPLICATED, ss);
}
+int OSDMonitor::get_erasure_code(const map<string,string> &properties,
+ ErasureCodeInterfaceRef *erasure_code,
+ stringstream &ss)
+{
+ map<string,string>::const_iterator plugin =
+ properties.find("erasure-code-plugin");
+ if (plugin == properties.end()) {
+ ss << "cannot determine the erasure code plugin"
+ << " because erasure-code-plugin is not in the properties "
+ << properties;
+ return -EINVAL;
+ }
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ return instance.factory(plugin->second, properties, erasure_code);
+}
+
+int OSDMonitor::check_cluster_features(uint64_t features,
+ stringstream &ss)
+{
+ stringstream unsupported_ss;
+ int unsupported_count = 0;
+ if (!(mon->get_quorum_features() & features)) {
+ unsupported_ss << "the monitor cluster";
+ ++unsupported_count;
+ }
+
+ set<int32_t> up_osds;
+ osdmap.get_up_osds(up_osds);
+ for (set<int32_t>::iterator it = up_osds.begin();
+ it != up_osds.end(); ++it) {
+ const osd_xinfo_t &xi = osdmap.get_xinfo(*it);
+ if ((xi.features & features) != features) {
+ if (unsupported_count > 0)
+ unsupported_ss << ", ";
+ unsupported_ss << "osd." << *it;
+ unsupported_count ++;
+ }
+ }
+
+ if (unsupported_count > 0) {
+ ss << "features " << features << " unsupported by: "
+ << unsupported_ss.str();
+ return -ENOTSUP;
+ }
+
+ // check pending osd state, too!
+ for (map<int32_t,osd_xinfo_t>::const_iterator p =
+ pending_inc.new_xinfo.begin();
+ p != pending_inc.new_xinfo.end(); ++p) {
+ const osd_xinfo_t &xi = p->second;
+ if ((xi.features & features) != features) {
+ dout(10) << __func__ << " pending osd." << p->first
+ << " features are insufficient; retry" << dendl;
+ return -EAGAIN;
+ }
+ }
+
+ return 0;
+}
+
+int OSDMonitor::prepare_pool_properties(const unsigned pool_type,
+ const vector<string> &properties,
+ map<string,string> *properties_map,
+ stringstream &ss)
+{
+ if (pool_type == pg_pool_t::TYPE_ERASURE) {
+ int r = get_str_map(g_conf->osd_pool_default_erasure_code_properties,
+ ss,
+ properties_map);
+ if (r)
+ return r;
+ (*properties_map)["erasure-code-directory"] =
+ g_conf->osd_pool_default_erasure_code_directory;
+ }
+
+ for (vector<string>::const_iterator i = properties.begin();
+ i != properties.end();
+ ++i) {
+ size_t equal = i->find('=');
+ if (equal == string::npos)
+ (*properties_map)[*i] = string();
+ else {
+ const string key = i->substr(0, equal);
+ equal++;
+ const string value = i->substr(equal);
+ (*properties_map)[key] = value;
+ }
+ }
+
+ return 0;
+}
+
+int OSDMonitor::prepare_pool_size(const unsigned pool_type,
+ const map<string,string> &properties,
+ unsigned *size,
+ stringstream &ss)
+{
+ int err = 0;
+ switch (pool_type) {
+ case pg_pool_t::TYPE_REPLICATED:
+ *size = g_conf->osd_pool_default_size;
+ break;
+ case pg_pool_t::TYPE_ERASURE:
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ err = get_erasure_code(properties, &erasure_code, ss);
+ if (err == 0)
+ *size = erasure_code->get_chunk_count();
+ }
+ break;
+ default:
+ ss << "prepare_pool_size: " << pool_type << " is not a known pool type";
+ err = -EINVAL;
+ break;
+ }
+ return err;
+}
+
+int OSDMonitor::prepare_pool_stripe_width(const unsigned pool_type,
+ const map<string,string> &properties,
+ uint32_t *stripe_width,
+ stringstream &ss)
+{
+ int err = 0;
+ switch (pool_type) {
+ case pg_pool_t::TYPE_REPLICATED:
+ // ignored
+ break;
+ case pg_pool_t::TYPE_ERASURE:
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ err = get_erasure_code(properties, &erasure_code, ss);
+ uint32_t desired_stripe_width = g_conf->osd_pool_erasure_code_stripe_width;
+ if (err == 0)
+ *stripe_width = erasure_code->get_data_chunk_count() *
+ erasure_code->get_chunk_size(desired_stripe_width);
+ }
+ break;
+ default:
+ ss << "prepare_pool_stripe_width: "
+ << pool_type << " is not a known pool type";
+ err = -EINVAL;
+ break;
+ }
+ return err;
+}
+
+int OSDMonitor::prepare_pool_crush_ruleset(const unsigned pool_type,
+ const map<string,string> &properties,
+ int *crush_ruleset,
+ stringstream &ss)
+{
+ if (*crush_ruleset < 0) {
+ switch (pool_type) {
+ case pg_pool_t::TYPE_REPLICATED:
+ *crush_ruleset =
+ CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(g_ceph_context);
+ break;
+ case pg_pool_t::TYPE_ERASURE:
+ {
+ map<string,string>::const_iterator ruleset =
+ properties.find("crush_ruleset");
+ if (ruleset == properties.end()) {
+ ss << "prepare_pool_crush_ruleset: crush_ruleset is missing from "
+ << properties;
+ return -EINVAL;
+ }
+
+ *crush_ruleset = osdmap.crush->get_rule_id(ruleset->second);
+ if (*crush_ruleset < 0) {
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+
+ int rule = newcrush.get_rule_id(ruleset->second);
+ if (rule < 0) {
+ ss << "prepare_pool_crush_ruleset: ruleset " << ruleset->second
+ << " does not exist";
+ return -EINVAL;
+ } else {
+ dout(20) << "prepare_pool_crush_ruleset: ruleset "
+ << ruleset->second << " try again" << dendl;
+ return -EAGAIN;
+ }
+ }
+ }
+ break;
+ default:
+ ss << "prepare_pool_crush_ruleset: " << pool_type
+ << " is not a known pool type";
+ return -EINVAL;
+ break;
+ }
+ }
+
+ return 0;
+}
/**
* @param name The name of the new pool
* @param auid The auid of the pool owner. Can be -1
@@ -2759,16 +3001,21 @@ int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_ruleset,
const unsigned pool_type,
stringstream &ss)
{
- map<string,string> default_properties;
- if (pool_type == pg_pool_t::TYPE_ERASURE) {
- int r = get_str_map(g_conf->osd_pool_default_erasure_code_properties,
- ss,
- &default_properties);
- if (r)
- return r;
- default_properties["erasure-code-directory"] =
- g_conf->osd_pool_default_erasure_code_directory;
- }
+ map<string,string> properties_map;
+ int r = prepare_pool_properties(pool_type, properties, &properties_map, ss);
+ if (r)
+ return r;
+ r = prepare_pool_crush_ruleset(pool_type, properties_map, &crush_ruleset, ss);
+ if (r)
+ return r;
+ unsigned size;
+ r = prepare_pool_size(pool_type, properties_map, &size, ss);
+ if (r)
+ return r;
+ uint32_t stripe_width = 0;
+ r = prepare_pool_stripe_width(pool_type, properties_map, &stripe_width, ss);
+ if (r)
+ return r;
for (map<int64_t,string>::iterator p = pending_inc.new_pool_names.begin();
p != pending_inc.new_pool_names.end();
@@ -2787,40 +3034,22 @@ int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_ruleset,
if (g_conf->osd_pool_default_flag_hashpspool)
pi->flags |= pg_pool_t::FLAG_HASHPSPOOL;
- pi->size = g_conf->osd_pool_default_size;
+ pi->size = size;
pi->min_size = g_conf->get_osd_pool_default_min_size();
- if (crush_ruleset >= 0) {
- pi->crush_ruleset = crush_ruleset;
- } else {
- switch(pool_type) {
- case pg_pool_t::TYPE_REPLICATED:
- pi->crush_ruleset =
- CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(g_ceph_context);
- break;
- case pg_pool_t::TYPE_ERASURE:
- pi->crush_ruleset = g_conf->osd_pool_default_crush_erasure_ruleset;
- break;
- }
- }
+ pi->crush_ruleset = crush_ruleset;
pi->object_hash = CEPH_STR_HASH_RJENKINS;
pi->set_pg_num(pg_num ? pg_num : g_conf->osd_pool_default_pg_num);
pi->set_pgp_num(pgp_num ? pgp_num : g_conf->osd_pool_default_pgp_num);
pi->last_change = pending_inc.epoch;
pi->auid = auid;
- pi->properties = default_properties;
- for (vector<string>::const_iterator i = properties.begin();
- i != properties.end();
- ++i) {
- size_t equal = i->find('=');
- if (equal == string::npos)
- pi->properties[*i] = string();
- else {
- const string key = i->substr(0, equal);
- equal++;
- const string value = i->substr(equal);
- pi->properties[key] = value;
- }
- }
+ pi->properties = properties_map;
+ pi->stripe_width = stripe_width;
+ pi->cache_target_dirty_ratio_micro =
+ g_conf->osd_pool_default_cache_target_dirty_ratio * 1000000;
+ pi->cache_target_full_ratio_micro =
+ g_conf->osd_pool_default_cache_target_full_ratio * 1000000;
+ pi->cache_min_flush_age = g_conf->osd_pool_default_cache_min_flush_age;
+ pi->cache_min_evict_age = g_conf->osd_pool_default_cache_min_evict_age;
pending_inc.new_pool_names[pool] = name;
return 0;
}
@@ -2908,6 +3137,10 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
}
if (var == "size") {
+ if (p.type == pg_pool_t::TYPE_ERASURE) {
+ ss << "can not change the size of an erasure-coded pool";
+ return -ENOTSUP;
+ }
if (interr.length()) {
ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
@@ -2948,7 +3181,7 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
if (pgs_per_osd > g_conf->mon_osd_max_split_count) {
ss << "specified pg_num " << n << " is too large (creating "
<< new_pgs << " new PGs on ~" << expected_osds
- << " OSDs exceeds per-OSD max of" << g_conf->mon_osd_max_split_count
+ << " OSDs exceeds per-OSD max of " << g_conf->mon_osd_max_split_count
<< ')';
return -E2BIG;
}
@@ -2998,7 +3231,7 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
if (val == "true" || (interr.empty() && n == 1)) {
p.flags |= pg_pool_t::FLAG_HASHPSPOOL;
} else if (val == "false" || (interr.empty() && n == 0)) {
- p.flags ^= pg_pool_t::FLAG_HASHPSPOOL;
+ p.flags &= ~pg_pool_t::FLAG_HASHPSPOOL;
} else {
ss << "expecting value 'true', 'false', '0', or '1'";
return -EINVAL;
@@ -3008,7 +3241,7 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
p.hit_set_params = HitSet::Params();
else if (val == "bloom") {
BloomHitSet::Params *bsp = new BloomHitSet::Params;
- bsp->set_fpp(.01);
+ bsp->set_fpp(g_conf->osd_pool_default_hit_set_bloom_fpp);
p.hit_set_params = HitSet::Params(bsp);
} else if (val == "explicit_hash")
p.hit_set_params = HitSet::Params(new ExplicitHashHitSet::Params);
@@ -3045,7 +3278,50 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
if (val == "true" || (interr.empty() && n == 1)) {
p.flags |= pg_pool_t::FLAG_DEBUG_FAKE_EC_POOL;
}
- ss << " pool " << pool << " set debug_fake_ec_pool";
+ } else if (var == "target_max_objects") {
+ if (interr.length()) {
+ ss << "error parsing int '" << val << "': " << interr;
+ return -EINVAL;
+ }
+ p.target_max_objects = n;
+ } else if (var == "target_max_bytes") {
+ if (interr.length()) {
+ ss << "error parsing int '" << val << "': " << interr;
+ return -EINVAL;
+ }
+ p.target_max_bytes = n;
+ } else if (var == "cache_target_dirty_ratio") {
+ if (floaterr.length()) {
+ ss << "error parsing float '" << val << "': " << floaterr;
+ return -EINVAL;
+ }
+ if (f < 0 || f > 1.0) {
+ ss << "value must be in the range 0..1";
+ return -ERANGE;
+ }
+ p.cache_target_dirty_ratio_micro = f * 1000000;
+ } else if (var == "cache_target_full_ratio") {
+ if (floaterr.length()) {
+ ss << "error parsing float '" << val << "': " << floaterr;
+ return -EINVAL;
+ }
+ if (f < 0 || f > 1.0) {
+ ss << "value must be in the range 0..1";
+ return -ERANGE;
+ }
+ p.cache_target_full_ratio_micro = n;
+ } else if (var == "cache_min_flush_age") {
+ if (interr.length()) {
+ ss << "error parsing int '" << val << "': " << interr;
+ return -EINVAL;
+ }
+ p.cache_min_flush_age = n;
+ } else if (var == "cache_min_evict_age") {
+ if (interr.length()) {
+ ss << "error parsing int '" << val << "': " << interr;
+ return -EINVAL;
+ }
+ p.cache_min_evict_age = n;
} else {
ss << "unrecognized variable '" << var << "'";
return -EINVAL;
@@ -3058,12 +3334,7 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
bool OSDMonitor::prepare_command(MMonCommand *m)
{
- bool ret = false;
stringstream ss;
- string rs;
- bufferlist rdata;
- int err = 0;
-
map<string, cmd_vartype> cmdmap;
if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
string rs = ss.str();
@@ -3071,16 +3342,28 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
return true;
}
- string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
- boost::scoped_ptr<Formatter> f(new_formatter(format));
-
MonSession *session = m->get_session();
if (!session) {
mon->reply_command(m, -EACCES, "access denied", get_last_committed());
return true;
}
+ return prepare_command_impl(m, cmdmap);
+}
+
+bool OSDMonitor::prepare_command_impl(MMonCommand *m,
+ map<string,cmd_vartype> &cmdmap)
+{
+ bool ret = false;
+ stringstream ss;
+ string rs;
+ bufferlist rdata;
+ int err = 0;
+
+ string format;
+ cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+ boost::scoped_ptr<Formatter> f(new_formatter(format));
+
string prefix;
cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
@@ -3173,6 +3456,11 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
err = -EINVAL;
goto reply;
}
+ if (type == 0) {
+ ss << "type '" << typestr << "' is for devices, not buckets";
+ err = -EINVAL;
+ goto reply;
+ }
int bucketno;
err = newcrush.add_bucket(0, CRUSH_BUCKET_STRAW,
CRUSH_HASH_DEFAULT, type, 0, NULL,
@@ -3206,7 +3494,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
}
double weight;
- cmd_getval(g_ceph_context, cmdmap, "weight", weight);
+ if (!cmd_getval(g_ceph_context, cmdmap, "weight", weight)) {
+ ss << "unable to parse weight value '"
+ << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
string args;
vector<string> argvec;
@@ -3269,7 +3562,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
}
double weight;
- cmd_getval(g_ceph_context, cmdmap, "weight", weight);
+ if (!cmd_getval(g_ceph_context, cmdmap, "weight", weight)) {
+ ss << "unable to parse weight value '"
+ << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
string args;
vector<string> argvec;
@@ -3463,7 +3761,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
break;
}
double w;
- cmd_getval(g_ceph_context, cmdmap, "weight", w);
+ if (!cmd_getval(g_ceph_context, cmdmap, "weight", w)) {
+ ss << "unable to parse weight value '"
+ << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+ err = -EINVAL;
+ break;
+ }
err = newcrush.adjust_item_weightf(g_ceph_context, id, w);
if (err >= 0) {
@@ -3489,11 +3792,17 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
newcrush.set_tunables_legacy();
} else if (profile == "bobtail") {
newcrush.set_tunables_bobtail();
+ } else if (profile == "firefly") {
+ newcrush.set_tunables_firefly();
} else if (profile == "optimal") {
newcrush.set_tunables_optimal();
} else if (profile == "default") {
newcrush.set_tunables_default();
- }
+ } else {
+ ss << "unrecognized profile '" << profile << "'";
+ err = -EINVAL;
+ goto reply;
+ }
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush);
ss << "adjusted tunables profile to " << profile;
@@ -3539,6 +3848,59 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
get_last_committed() + 1));
return true;
+ } else if (prefix == "osd crush rule create-erasure") {
+ err = check_cluster_features(CEPH_FEATURE_CRUSH_V2, ss);
+ if (err == -EAGAIN)
+ goto wait;
+ if (err)
+ goto reply;
+ string name, poolstr;
+ cmd_getval(g_ceph_context, cmdmap, "name", name);
+ vector<string> properties;
+ cmd_getval(g_ceph_context, cmdmap, "properties", properties);
+
+ if (osdmap.crush->rule_exists(name)) {
+ ss << "rule " << name << " already exists";
+ err = 0;
+ goto reply;
+ }
+
+ map<string,string> properties_map;
+ err = prepare_pool_properties(pg_pool_t::TYPE_ERASURE,
+ properties, &properties_map, ss);
+ if (err)
+ goto reply;
+
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+
+ if (newcrush.rule_exists(name)) {
+ ss << "rule " << name << " already exists";
+ err = 0;
+ } else {
+ ErasureCodeInterfaceRef erasure_code;
+ err = get_erasure_code(properties_map, &erasure_code, ss);
+ if (err) {
+ ss << "failed to load plugin using properties " << properties_map;
+ goto reply;
+ }
+
+ int rule = erasure_code->create_ruleset(name, newcrush, &ss);
+ erasure_code.reset();
+ if (rule < 0) {
+ err = rule;
+ goto reply;
+ }
+ ss << "created rule " << name << " at " << rule;
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush);
+ }
+
+ getline(ss, rs);
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
+ return true;
+
} else if (prefix == "osd crush rule rm") {
string name;
cmd_getval(g_ceph_context, cmdmap, "name", name);
@@ -3584,7 +3946,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "osd setmaxosd") {
int64_t newmax;
- cmd_getval(g_ceph_context, cmdmap, "newmax", newmax);
+ if (!cmd_getval(g_ceph_context, cmdmap, "newmax", newmax)) {
+ ss << "unable to parse 'newmax' value '"
+ << cmd_vartype_stringify(cmdmap["newmax"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
if (newmax > g_conf->mon_max_osd) {
err = -ERANGE;
@@ -3627,6 +3994,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
return prepare_set_flag(m, CEPH_OSDMAP_NOSCRUB);
else if (key == "nodeep-scrub")
return prepare_set_flag(m, CEPH_OSDMAP_NODEEP_SCRUB);
+ else if (key == "notieragent")
+ return prepare_set_flag(m, CEPH_OSDMAP_NOTIERAGENT);
+ else {
+ ss << "unrecognized flag '" << key << "'";
+ err = -EINVAL;
+ }
} else if (prefix == "osd unset") {
string key;
@@ -3649,6 +4022,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
return prepare_unset_flag(m, CEPH_OSDMAP_NOSCRUB);
else if (key == "nodeep-scrub")
return prepare_unset_flag(m, CEPH_OSDMAP_NODEEP_SCRUB);
+ else if (key == "notieragent")
+ return prepare_unset_flag(m, CEPH_OSDMAP_NOTIERAGENT);
+ else {
+ ss << "unrecognized flag '" << key << "'";
+ err = -EINVAL;
+ }
} else if (prefix == "osd cluster_snap") {
// ** DISABLE THIS FOR NOW **
@@ -3723,14 +4102,62 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
get_last_committed() + 1));
return true;
}
+ } else if (prefix == "osd primary-affinity") {
+ int64_t id;
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+ ss << "invalid osd id value '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+ double w;
+ if (!cmd_getval(g_ceph_context, cmdmap, "weight", w)) {
+ ss << "unable to parse 'weight' value '"
+ << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+ long ww = (int)((double)CEPH_OSD_MAX_PRIMARY_AFFINITY*w);
+ if (ww < 0L) {
+ ss << "weight must be >= 0";
+ err = -EINVAL;
+ goto reply;
+ }
+ if (!g_conf->mon_osd_allow_primary_affinity) {
+ ss << "you must enable 'mon osd allow primary affinity = true' on the mons before you can adjust primary-affinity. note that older clients will no longer be able to communicate with the cluster.";
+ err = -EPERM;
+ goto reply;
+ }
+ err = check_cluster_features(CEPH_FEATURE_OSD_PRIMARY_AFFINITY, ss);
+ if (err == -EAGAIN)
+ goto wait;
+ if (err < 0)
+ goto reply;
+ if (osdmap.exists(id)) {
+ pending_inc.new_primary_affinity[id] = ww;
+ ss << "set osd." << id << " primary-affinity to " << w << " (" << ios::hex << ww << ios::dec << ")";
+ getline(ss, rs);
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ return true;
+ }
} else if (prefix == "osd reweight") {
int64_t id;
- cmd_getval(g_ceph_context, cmdmap, "id", id);
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+ ss << "unable to parse osd id value '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
double w;
- cmd_getval(g_ceph_context, cmdmap, "weight", w);
+ if (!cmd_getval(g_ceph_context, cmdmap, "weight", w)) {
+ ss << "unable to parse weight value '"
+ << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
long ww = (int)((double)CEPH_OSD_IN*w);
if (ww < 0L) {
- ss << "weight must be > 0";
+ ss << "weight must be >= 0";
err = -EINVAL;
goto reply;
}
@@ -3745,7 +4172,12 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "osd lost") {
int64_t id;
- cmd_getval(g_ceph_context, cmdmap, "id", id);
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+ ss << "unable to parse osd id value '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
string sure;
if (!cmd_getval(g_ceph_context, cmdmap, "sure", sure) || sure != "--yes-i-really-mean-it") {
ss << "are you SURE? this might mean real, permanent data loss. pass "
@@ -3999,35 +4431,13 @@ done:
if (pool_type_str == "replicated") {
pool_type = pg_pool_t::TYPE_REPLICATED;
} else if (pool_type_str == "erasure") {
-
- // make sure all the daemons support erasure coding
- stringstream ec_unsupported_ss;
- int ec_unsupported_count = 0;
- if (!(mon->get_quorum_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) {
- ec_unsupported_ss << "the monitor cluster";
- ++ec_unsupported_count;
- }
-
- set<int32_t> up_osds;
- osdmap.get_up_osds(up_osds);
- for (set<int32_t>::iterator it = up_osds.begin();
- it != up_osds.end(); it ++) {
- const osd_xinfo_t &xi = osdmap.get_xinfo(*it);
- if (!(xi.features & CEPH_FEATURE_OSD_ERASURE_CODES)) {
- if (ec_unsupported_count > 0)
- ec_unsupported_ss << ", ";
- ec_unsupported_ss << "osd." << *it;
- ec_unsupported_count ++;
- }
- }
-
- if (ec_unsupported_count > 0) {
- ss << "unable to create erasure pool; unsupported by: "
- << ec_unsupported_ss.str();
- err = -ENOTSUP;
- goto reply;
- }
-
+ err = check_cluster_features(CEPH_FEATURE_CRUSH_V2 |
+ CEPH_FEATURE_OSD_ERASURE_CODES,
+ ss);
+ if (err == -EAGAIN)
+ goto wait;
+ if (err)
+ goto reply;
pool_type = pg_pool_t::TYPE_ERASURE;
} else {
ss << "unknown pool type '" << pool_type_str << "'";
@@ -4040,11 +4450,18 @@ done:
pg_num, pgp_num,
properties, pool_type,
ss);
- if (err < 0 && err != -EEXIST) {
- goto reply;
- }
- if (err == -EEXIST) {
- ss << "pool '" << poolstr << "' already exists";
+ if (err < 0) {
+ switch(err) {
+ case -EEXIST:
+ ss << "pool '" << poolstr << "' already exists";
+ break;
+ case -EAGAIN:
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
+ default:
+ goto reply;
+ break;
+ }
} else {
ss << "pool '" << poolstr << "' created";
}
@@ -4073,13 +4490,14 @@ done:
err = -EPERM;
goto reply;
}
- int ret = _prepare_remove_pool(pool);
- if (ret == 0)
- ss << "pool '" << poolstr << "' deleted";
- getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, ret, rs,
- get_last_committed() + 1));
- return true;
+ err = _prepare_remove_pool(pool, &ss);
+ if (err == -EAGAIN) {
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
+ }
+ if (err < 0)
+ goto reply;
+ goto update;
} else if (prefix == "osd pool rename") {
string srcpoolstr, destpoolstr;
cmd_getval(g_ceph_context, cmdmap, "srcpool", srcpoolstr);
@@ -4164,9 +4582,26 @@ done:
err = -EINVAL;
goto reply;
}
+ // make sure new tier is empty
+ string force_nonempty;
+ cmd_getval(g_ceph_context, cmdmap, "force_nonempty", force_nonempty);
+ const pool_stat_t& tier_stats =
+ mon->pgmon()->pg_map.get_pg_pool_sum_stat(tierpool_id);
+ if (tier_stats.stats.sum.num_objects != 0 &&
+ force_nonempty != "--force-nonempty") {
+ ss << "tier pool '" << tierpoolstr << "' is not empty; --force-nonempty to force";
+ err = -ENOTEMPTY;
+ goto reply;
+ }
// go
- pending_inc.get_new_pool(pool_id, p)->tiers.insert(tierpool_id);
- pending_inc.get_new_pool(tierpool_id, tp)->tier_of = pool_id;
+ pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+ pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+ if (np->tiers.count(tierpool_id) || ntp->is_tier()) {
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
+ }
+ np->tiers.insert(tierpool_id);
+ ntp->tier_of = pool_id;
ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
get_last_committed() + 1));
@@ -4208,8 +4643,16 @@ done:
goto reply;
}
// go
- pending_inc.get_new_pool(pool_id, p)->tiers.erase(tierpool_id);
- pending_inc.get_new_pool(tierpool_id, tp)->clear_tier();
+ pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+ pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+ if (np->tiers.count(tierpool_id) == 0 ||
+ ntp->tier_of != pool_id ||
+ np->read_tier == tierpool_id) {
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
+ }
+ np->tiers.erase(tierpool_id);
+ ntp->clear_tier();
ss << "pool '" << tierpoolstr << "' is now (or already was) not a tier of '" << poolstr << "'";
wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
get_last_committed() + 1));
@@ -4311,6 +4754,95 @@ done:
wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
get_last_committed() + 1));
return true;
+ } else if (prefix == "osd tier add-cache") {
+ string poolstr;
+ cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+ int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+ if (pool_id < 0) {
+ ss << "unrecognized pool '" << poolstr << "'";
+ err = -ENOENT;
+ goto reply;
+ }
+ string tierpoolstr;
+ cmd_getval(g_ceph_context, cmdmap, "tierpool", tierpoolstr);
+ int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+ if (tierpool_id < 0) {
+ ss << "unrecognized pool '" << tierpoolstr << "'";
+ err = -ENOENT;
+ goto reply;
+ }
+ const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+ assert(p);
+ const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+ assert(tp);
+ if (p->tiers.count(tierpool_id)) {
+ assert(tp->tier_of == pool_id);
+ err = 0;
+ ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
+ goto reply;
+ }
+ if (tp->is_tier()) {
+ ss << "tier pool '" << tierpoolstr << "' is already a tier of '"
+ << osdmap.get_pool_name(tp->tier_of) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+ int64_t size = 0;
+ if (!cmd_getval(g_ceph_context, cmdmap, "size", size)) {
+ ss << "unable to parse 'size' value '"
+ << cmd_vartype_stringify(cmdmap["size"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+ // make sure new tier is empty
+ const pool_stat_t& tier_stats =
+ mon->pgmon()->pg_map.get_pg_pool_sum_stat(tierpool_id);
+ if (tier_stats.stats.sum.num_objects != 0) {
+ ss << "tier pool '" << tierpoolstr << "' is not empty";
+ err = -ENOTEMPTY;
+ goto reply;
+ }
+ string modestr = g_conf->osd_tier_default_cache_mode;
+ pg_pool_t::cache_mode_t mode = pg_pool_t::get_cache_mode_from_str(modestr);
+ if (mode < 0) {
+ ss << "osd tier cache default mode '" << modestr << "' is not a valid cache mode";
+ err = -EINVAL;
+ goto reply;
+ }
+ HitSet::Params hsp;
+ if (g_conf->osd_tier_default_cache_hit_set_type == "bloom") {
+ BloomHitSet::Params *bsp = new BloomHitSet::Params;
+ bsp->set_fpp(g_conf->osd_pool_default_hit_set_bloom_fpp);
+ hsp = HitSet::Params(bsp);
+ } else if (g_conf->osd_tier_default_cache_hit_set_type == "explicit_hash") {
+ hsp = HitSet::Params(new ExplicitHashHitSet::Params);
+ }
+ else if (g_conf->osd_tier_default_cache_hit_set_type == "explicit_object") {
+ hsp = HitSet::Params(new ExplicitObjectHitSet::Params);
+ } else {
+ ss << "osd tier cache default hit set type '" <<
+ g_conf->osd_tier_default_cache_hit_set_type << "' is not a known type";
+ err = -EINVAL;
+ goto reply;
+ }
+ // go
+ pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+ pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+ if (np->tiers.count(tierpool_id) || ntp->is_tier()) {
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
+ }
+ np->tiers.insert(tierpool_id);
+ ntp->tier_of = pool_id;
+ ntp->cache_mode = mode;
+ ntp->hit_set_count = g_conf->osd_tier_default_cache_hit_set_count;
+ ntp->hit_set_period = g_conf->osd_tier_default_cache_hit_set_period;
+ ntp->hit_set_params = hsp;
+ ntp->target_max_bytes = size;
+ ss << "pool '" << tierpoolstr << "' is now (or already was) a cache tier of '" << poolstr << "'";
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
+ return true;
} else if (prefix == "osd pool set-quota") {
string poolstr;
cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
@@ -4379,11 +4911,11 @@ done:
ss << "will thrash map for " << thrash_map << " epochs";
ret = thrash();
err = 0;
- } else {
- err = -EINVAL;
- }
+ } else {
+ err = -EINVAL;
+ }
-reply:
+ reply:
getline(ss, rs);
if (err < 0 && rs.length() == 0)
rs = cpp_strerror(err);
@@ -4395,6 +4927,10 @@ reply:
wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
get_last_committed() + 1));
return true;
+
+ wait:
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
}
bool OSDMonitor::preprocess_pool_op(MPoolOp *m)
@@ -4527,8 +5063,16 @@ bool OSDMonitor::prepare_pool_op(MPoolOp *m)
_pool_op_reply(m, ret, osdmap.get_epoch());
return false;
- case POOL_OP_CREATE_UNMANAGED_SNAP:
case POOL_OP_DELETE_UNMANAGED_SNAP:
+ // we won't allow removal of an unmanaged snapshot from a pool
+ // not in unmanaged snaps mode.
+ if (!pool->is_unmanaged_snaps_mode()) {
+ _pool_op_reply(m, -ENOTSUP, osdmap.get_epoch());
+ return false;
+ }
+ case POOL_OP_CREATE_UNMANAGED_SNAP:
+ // but we will allow creating an unmanaged snapshot on any pool
+ // as long as it is not in 'pool' snaps mode.
if (pool->is_pool_snaps_mode()) {
_pool_op_reply(m, -EINVAL, osdmap.get_epoch());
return false;
@@ -4626,20 +5170,64 @@ bool OSDMonitor::prepare_pool_op_create(MPoolOp *m)
return true;
}
-int OSDMonitor::_prepare_remove_pool(uint64_t pool)
+int OSDMonitor::_check_remove_pool(int64_t pool, const pg_pool_t *p,
+ ostream *ss)
+{
+ string poolstr = osdmap.get_pool_name(pool);
+
+ // If the Pool is in use by CephFS, refuse to delete it
+ MDSMap const &pending_mdsmap = mon->mdsmon()->pending_mdsmap;
+ if (pending_mdsmap.is_data_pool(pool) ||
+ pending_mdsmap.get_metadata_pool() == pool) {
+ *ss << "pool '" << poolstr << "' is in use by CephFS";
+ return -EBUSY;
+ }
+
+ if (p->tier_of >= 0) {
+ *ss << "pool '" << poolstr << "' is a tier of '"
+ << osdmap.get_pool_name(p->tier_of) << "'";
+ return -EBUSY;
+ }
+ if (!p->tiers.empty()) {
+ *ss << "pool '" << poolstr << "' includes tiers "
+ << p->tiers;
+ return -EBUSY;
+ }
+ *ss << "pool '" << poolstr << "' removed";
+ return 0;
+}
+
+int OSDMonitor::_prepare_remove_pool(int64_t pool, ostream *ss)
{
dout(10) << "_prepare_remove_pool " << pool << dendl;
+ const pg_pool_t *p = osdmap.get_pg_pool(pool);
+ int r = _check_remove_pool(pool, p, ss);
+ if (r < 0)
+ return r;
+
+ if (pending_inc.new_pools.count(pool)) {
+ // if there is a problem with the pending info, wait and retry
+ // this op.
+ pg_pool_t *p = &pending_inc.new_pools[pool];
+ int r = _check_remove_pool(pool, p, ss);
+ if (r < 0)
+ return -EAGAIN;
+ }
+
if (pending_inc.old_pools.count(pool)) {
- dout(10) << "_prepare_remove_pool " << pool << " pending removal" << dendl;
- return 0; // already removed
+ dout(10) << "_prepare_remove_pool " << pool << " already pending removal"
+ << dendl;
+ return 0;
}
+
+ // remove
pending_inc.old_pools.insert(pool);
// remove any pg_temp mappings for this pool too
for (map<pg_t,vector<int32_t> >::iterator p = osdmap.pg_temp->begin();
p != osdmap.pg_temp->end();
++p) {
- if (p->first.pool() == pool) {
+ if (p->first.pool() == (uint64_t)pool) {
dout(10) << "_prepare_remove_pool " << pool << " removing obsolete pg_temp "
<< p->first << dendl;
pending_inc.new_pg_temp[p->first].clear();
@@ -4648,7 +5236,7 @@ int OSDMonitor::_prepare_remove_pool(uint64_t pool)
for (map<pg_t,int>::iterator p = osdmap.primary_temp->begin();
p != osdmap.primary_temp->end();
++p) {
- if (p->first.pool() == pool) {
+ if (p->first.pool() == (uint64_t)pool) {
dout(10) << "_prepare_remove_pool " << pool
<< " removing obsolete primary_temp" << p->first << dendl;
pending_inc.new_primary_temp[p->first] = -1;
@@ -4678,8 +5266,16 @@ int OSDMonitor::_prepare_rename_pool(int64_t pool, string newname)
bool OSDMonitor::prepare_pool_op_delete(MPoolOp *m)
{
- int ret = _prepare_remove_pool(m->pool);
- wait_for_finished_proposal(new OSDMonitor::C_PoolOp(this, m, ret, pending_inc.epoch));
+ ostringstream ss;
+ int ret = _prepare_remove_pool(m->pool, &ss);
+ if (ret == -EAGAIN) {
+ wait_for_finished_proposal(new C_RetryMessage(this, m));
+ return true;
+ }
+ if (ret < 0)
+ dout(10) << __func__ << " got " << ret << " " << ss.str() << dendl;
+ wait_for_finished_proposal(new OSDMonitor::C_PoolOp(this, m, ret,
+ pending_inc.epoch));
return true;
}
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 81ea432..8a87374 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -37,6 +37,8 @@ class Monitor;
#include "messages/MOSDFailure.h"
#include "messages/MPoolOp.h"
+#include "erasure-code/ErasureCodeInterface.h"
+
#define OSD_METADATA_PREFIX "osd_metadata"
/// information about a particular peer's failure reports for one osd
@@ -180,6 +182,7 @@ private:
void encode_trim_extra(MonitorDBStore::Transaction *tx, version_t first);
void update_msgr_features();
+ int check_cluster_features(uint64_t features, stringstream &ss);
void share_map_with_random_osd();
@@ -228,7 +231,8 @@ private:
bool preprocess_pgtemp(class MOSDPGTemp *m);
bool prepare_pgtemp(class MOSDPGTemp *m);
- int _prepare_remove_pool(uint64_t pool);
+ int _check_remove_pool(int64_t pool, const pg_pool_t *pi, ostream *ss);
+ int _prepare_remove_pool(int64_t pool, ostream *ss);
int _prepare_rename_pool(int64_t pool, string newname);
bool preprocess_pool_op ( class MPoolOp *m);
@@ -236,6 +240,25 @@ private:
bool prepare_pool_op (MPoolOp *m);
bool prepare_pool_op_create (MPoolOp *m);
bool prepare_pool_op_delete(MPoolOp *m);
+ int get_erasure_code(const map<string,string> &properties,
+ ErasureCodeInterfaceRef *erasure_code,
+ stringstream &ss);
+ int prepare_pool_properties(const unsigned pool_type,
+ const vector<string> &properties,
+ map<string,string> *properties_map,
+ stringstream &ss);
+ int prepare_pool_crush_ruleset(const unsigned pool_type,
+ const map<string,string> &properties,
+ int *crush_ruleset,
+ stringstream &ss);
+ int prepare_pool_size(const unsigned pool_type,
+ const map<string,string> &properties,
+ unsigned *size,
+ stringstream &ss);
+ int prepare_pool_stripe_width(const unsigned pool_type,
+ const map<string,string> &properties,
+ unsigned *stripe_width,
+ stringstream &ss);
int prepare_new_pool(string& name, uint64_t auid, int crush_ruleset,
unsigned pg_num, unsigned pgp_num,
const vector<string> &properties,
@@ -326,6 +349,7 @@ private:
list<pair<health_status_t,string> > *detail) const;
bool preprocess_command(MMonCommand *m);
bool prepare_command(MMonCommand *m);
+ bool prepare_command_impl(MMonCommand *m, map<string,cmd_vartype> &cmdmap);
int prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
stringstream& ss);
diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 56376d8..fbed206 100644
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -229,8 +229,9 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
stat_osd_sub(t->second);
t->second = new_stats;
}
- assert(inc.get_osd_epochs().find(osd) != inc.get_osd_epochs().end());
- osd_epochs.insert(*(inc.get_osd_epochs().find(osd)));
+ map<int32_t,epoch_t>::const_iterator j = inc.get_osd_epochs().find(osd);
+ assert(j != inc.get_osd_epochs().end());
+ osd_epochs[j->first] = j->second;
stat_osd_add(new_stats);
@@ -390,8 +391,8 @@ void PGMap::stat_pg_add(const pg_t &pgid, const pg_stat_t &s)
pg_sum.add(s);
if (s.state & PG_STATE_CREATING) {
creating_pgs.insert(pgid);
- if (s.acting.size())
- creating_pgs_by_osd[s.acting[0]].insert(pgid);
+ if (s.acting_primary >= 0)
+ creating_pgs_by_osd[s.acting_primary].insert(pgid);
}
}
@@ -409,10 +410,10 @@ void PGMap::stat_pg_sub(const pg_t &pgid, const pg_stat_t &s)
pg_sum.sub(s);
if (s.state & PG_STATE_CREATING) {
creating_pgs.erase(pgid);
- if (s.acting.size()) {
- creating_pgs_by_osd[s.acting[0]].erase(pgid);
- if (creating_pgs_by_osd[s.acting[0]].size() == 0)
- creating_pgs_by_osd.erase(s.acting[0]);
+ if (s.acting_primary >= 0) {
+ creating_pgs_by_osd[s.acting_primary].erase(pgid);
+ if (creating_pgs_by_osd[s.acting_primary].size() == 0)
+ creating_pgs_by_osd.erase(s.acting_primary);
}
}
}
@@ -622,7 +623,7 @@ void PGMap::dump_osd_stats(Formatter *f) const
void PGMap::dump_pg_stats_plain(ostream& ss,
const ceph::unordered_map<pg_t, pg_stat_t>& pg_stats) const
{
- ss << "pg_stat\tobjects\tmip\tdegr\tunf\tbytes\tlog\tdisklog\tstate\tstate_stamp\tv\treported\tup\tacting\tlast_scrub\tscrub_stamp\tlast_deep_scrub\tdeep_scrub_stamp" << std::endl;
+ ss << "pg_stat\tobjects\tmip\tdegr\tunf\tbytes\tlog\tdisklog\tstate\tstate_stamp\tv\treported\tup\tup_primary\tacting\tacting_primary\tlast_scrub\tscrub_stamp\tlast_deep_scrub\tdeep_scrub_stamp" << std::endl;
for (ceph::unordered_map<pg_t, pg_stat_t>::const_iterator i = pg_stats.begin();
i != pg_stats.end(); ++i) {
const pg_stat_t &st(i->second);
@@ -640,7 +641,9 @@ void PGMap::dump_pg_stats_plain(ostream& ss,
<< "\t" << st.version
<< "\t" << st.reported_epoch << ":" << st.reported_seq
<< "\t" << st.up
+ << "\t" << st.up_primary
<< "\t" << st.acting
+ << "\t" << st.acting_primary
<< "\t" << st.last_scrub << "\t" << st.last_scrub_stamp
<< "\t" << st.last_deep_scrub << "\t" << st.last_deep_scrub_stamp
<< std::endl;
diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
index 9c34640..f4e8df5 100644
--- a/src/mon/PGMap.h
+++ b/src/mon/PGMap.h
@@ -205,6 +205,14 @@ public:
stamp = s;
}
+ pool_stat_t get_pg_pool_sum_stat(int64_t pool) const {
+ ceph::unordered_map<int,pool_stat_t>::const_iterator p =
+ pg_pool_sum.find(pool);
+ if (p != pg_pool_sum.end())
+ return p->second;
+ return pool_stat_t();
+ }
+
void update_pg(pg_t pgid, bufferlist& bl);
void remove_pg(pg_t pgid);
void update_osd(int osd, bufferlist& bl);
diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index cdfc83a..5adf535 100644
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -434,9 +434,10 @@ void PGMonitor::apply_pgmap_delta(bufferlist& bl)
while (!p.end()) {
pg_t pgid;
::decode(pgid, p);
- dout(20) << " refreshing pg " << pgid << dendl;
bufferlist bl;
int r = mon->store->get(pgmap_pg_prefix, stringify(pgid), bl);
+ dout(20) << " refreshing pg " << pgid << " got " << r << " len "
+ << bl.length() << dendl;
if (pg_pool_sum_old.count(pgid.pool()) == 0)
pg_pool_sum_old[pgid.pool()] = pg_map.pg_pool_sum[pgid.pool()];
@@ -1071,15 +1072,25 @@ void PGMonitor::map_pg_creates()
pg_stat_t& s = pg_map.pg_stat[pgid];
if (s.parent_split_bits)
on = s.parent;
- vector<int> acting;
- int nrep = osdmap->pg_to_acting_osds(on, acting);
- if (s.acting.size()) {
- pg_map.creating_pgs_by_osd[s.acting[0]].erase(pgid);
- if (pg_map.creating_pgs_by_osd[s.acting[0]].size() == 0)
- pg_map.creating_pgs_by_osd.erase(s.acting[0]);
+ vector<int> up, acting;
+ int up_primary, acting_primary;
+ osdmap->pg_to_up_acting_osds(
+ on,
+ &up,
+ &up_primary,
+ &acting,
+ &acting_primary);
+
+ if (s.acting_primary != -1) {
+ pg_map.creating_pgs_by_osd[s.acting_primary].erase(pgid);
+ if (pg_map.creating_pgs_by_osd[s.acting_primary].size() == 0)
+ pg_map.creating_pgs_by_osd.erase(s.acting_primary);
}
+ s.up = up;
+ s.up_primary = up_primary;
s.acting = acting;
+ s.acting_primary = acting_primary;
// don't send creates for localized pgs
if (pgid.preferred() >= 0)
@@ -1089,8 +1100,8 @@ void PGMonitor::map_pg_creates()
if (s.parent_split_bits)
continue;
- if (nrep) {
- pg_map.creating_pgs_by_osd[acting[0]].insert(pgid);
+ if (acting_primary != -1) {
+ pg_map.creating_pgs_by_osd[acting_primary].insert(pgid);
} else {
dout(20) << "map_pg_creates " << pgid << " -> no osds in epoch "
<< mon->osdmon()->osdmap.get_epoch() << ", skipping" << dendl;
@@ -1159,8 +1170,8 @@ bool PGMonitor::check_down_pgs()
p != pg_map.pg_stat.end();
++p) {
if ((p->second.state & PG_STATE_STALE) == 0 &&
- p->second.acting.size() &&
- osdmap->is_down(p->second.acting[0])) {
+ p->second.acting_primary != -1 &&
+ osdmap->is_down(p->second.acting_primary)) {
dout(10) << " marking pg " << p->first << " stale with acting " << p->second.acting << dendl;
map<pg_t,pg_stat_t>::iterator q = pending_inc.pg_stat_updates.find(p->first);
@@ -1199,6 +1210,7 @@ void PGMonitor::dump_object_stat_sum(TextTable &tbl, Formatter *f,
f->dump_int("bytes_used", sum.num_bytes);
f->dump_int("objects", sum.num_objects);
if (verbose) {
+ f->dump_int("dirty", sum.num_objects_dirty);
f->dump_int("rd", sum.num_rd);
f->dump_int("rd_kb", sum.num_rd_kb);
f->dump_int("wr", sum.num_wr);
@@ -1210,7 +1222,8 @@ void PGMonitor::dump_object_stat_sum(TextTable &tbl, Formatter *f,
tbl << percentify(((float)kb_used / pg_map.osd_sum.kb)*100);
tbl << sum.num_objects;
if (verbose) {
- tbl << stringify(si_t(sum.num_rd))
+ tbl << stringify(si_t(sum.num_objects_dirty))
+ << stringify(si_t(sum.num_rd))
<< stringify(si_t(sum.num_wr));
}
}
@@ -1231,6 +1244,7 @@ void PGMonitor::dump_pool_stats(stringstream &ss, Formatter *f, bool verbose)
tbl.define_column("\%USED", TextTable::LEFT, TextTable::LEFT);
tbl.define_column("OBJECTS", TextTable::LEFT, TextTable::LEFT);
if (verbose) {
+ tbl.define_column("DIRTY", TextTable::LEFT, TextTable::LEFT);
tbl.define_column("READ", TextTable::LEFT, TextTable::LEFT);
tbl.define_column("WRITE", TextTable::LEFT, TextTable::LEFT);
}
@@ -1526,12 +1540,12 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
r = -ENOENT;
goto reply;
}
- if (!pg_map.pg_stat[pgid].acting.size()) {
+ if (pg_map.pg_stat[pgid].acting_primary == -1) {
ss << "pg " << pgid << " has no primary osd";
r = -EAGAIN;
goto reply;
}
- int osd = pg_map.pg_stat[pgid].acting[0];
+ int osd = pg_map.pg_stat[pgid].acting_primary;
if (!mon->osdmon()->osdmap.is_up(osd)) {
ss << "pg " << pgid << " primary osd." << osd << " not up";
r = -EAGAIN;
@@ -1646,7 +1660,12 @@ bool PGMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "pg set_full_ratio" ||
prefix == "pg set_nearfull_ratio") {
double n;
- cmd_getval(g_ceph_context, cmdmap, "ratio", n);
+ if (!cmd_getval(g_ceph_context, cmdmap, "ratio", n)) {
+ ss << "unable to parse 'ratio' value '"
+ << cmd_vartype_stringify(cmdmap["who"]) << "'";
+ r = -EINVAL;
+ goto reply;
+ }
string op = prefix.substr(3, string::npos);
if (op == "set_full_ratio")
pending_inc.full_ratio = n;
@@ -1876,6 +1895,50 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
check_full_osd_health(summary, detail, pg_map.full_osds, "full", HEALTH_ERR);
check_full_osd_health(summary, detail, pg_map.nearfull_osds, "near full", HEALTH_WARN);
+ // near-target max pools
+ const map<int64_t,pg_pool_t>& pools = mon->osdmon()->osdmap.get_pools();
+ for (map<int64_t,pg_pool_t>::const_iterator p = pools.begin();
+ p != pools.end(); ++p) {
+ if ((!p->second.target_max_objects && !p->second.target_max_bytes) ||
+ !pg_map.pg_pool_sum.count(p->first))
+ continue;
+ bool nearfull = false;
+ const char *name = mon->osdmon()->osdmap.get_pool_name(p->first);
+ const pool_stat_t& st = pg_map.get_pg_pool_sum_stat(p->first);
+ uint64_t ratio = p->second.cache_target_full_ratio_micro +
+ ((1000000 - p->second.cache_target_full_ratio_micro) *
+ g_conf->mon_cache_target_full_warn_ratio);
+ if (p->second.target_max_objects && (uint64_t)st.stats.sum.num_objects >
+ p->second.target_max_objects * ratio / 1000000) {
+ nearfull = true;
+ if (detail) {
+ ostringstream ss;
+ ss << "cache pool '" << name << "' with "
+ << si_t(st.stats.sum.num_objects)
+ << " objects at/near target max "
+ << si_t(p->second.target_max_objects) << " objects";
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
+ if (p->second.target_max_bytes && (uint64_t)st.stats.sum.num_bytes >
+ p->second.target_max_bytes * ratio / 1000000) {
+ nearfull = true;
+ if (detail) {
+ ostringstream ss;
+ ss << "cache pool '" << mon->osdmon()->osdmap.get_pool_name(p->first)
+ << "' with " << si_t(st.stats.sum.num_bytes)
+ << "B at/near target max "
+ << si_t(p->second.target_max_bytes) << "B";
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
+ if (nearfull) {
+ ostringstream ss;
+ ss << "'" << name << "' at/near target max";
+ summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
+
// scrub
if (pg_map.pg_sum.stats.sum.num_scrub_errors) {
ostringstream ss;
@@ -1965,12 +2028,17 @@ int PGMonitor::dump_stuck_pg_stats(stringstream &ds,
{
PGMap::StuckPG stuck_type;
string type = args[0];
+
if (type == "inactive")
stuck_type = PGMap::STUCK_INACTIVE;
- if (type == "unclean")
+ else if (type == "unclean")
stuck_type = PGMap::STUCK_UNCLEAN;
- if (type == "stale")
+ else if (type == "stale")
stuck_type = PGMap::STUCK_STALE;
+ else {
+ ds << "Unknown type: " << type << std::endl;
+ return 0;
+ }
utime_t now(ceph_clock_now(g_ceph_context));
utime_t cutoff = now - utime_t(threshold, 0);
diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h
index d29f47c..09dd009 100644
--- a/src/mon/PGMonitor.h
+++ b/src/mon/PGMonitor.h
@@ -28,7 +28,7 @@ using namespace std;
#include "PaxosService.h"
#include "include/types.h"
#include "include/utime.h"
-#include "include/histogram.h"
+#include "common/histogram.h"
#include "msg/Messenger.h"
#include "common/config.h"
#include "mon/MonitorDBStore.h"
diff --git a/src/msg/Accepter.cc b/src/msg/Accepter.cc
index 5eebb2e..f8ae711 100644
--- a/src/msg/Accepter.cc
+++ b/src/msg/Accepter.cc
@@ -77,7 +77,7 @@ int Accepter::bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports)
int on = 1;
rc = ::setsockopt(listen_sd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
if (rc < 0) {
- ldout(msgr->cct,0) << "accepter.bind unable to setsockopt: "
+ lderr(msgr->cct) << "accepter.bind unable to setsockopt: "
<< cpp_strerror(errno) << dendl;
return -errno;
}
diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index b9dc548..6ecce71 100644
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -159,6 +159,11 @@ using namespace std;
#include "messages/MOSDPGPushReply.h"
#include "messages/MOSDPGPull.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+
#define DEBUGLVL 10 // debug level of output
#define dout_subsys ceph_subsys_ms
@@ -466,6 +471,18 @@ Message *decode_message(CephContext *cct, ceph_msg_header& header, ceph_msg_foot
case MSG_OSD_PG_PUSH_REPLY:
m = new MOSDPGPushReply;
break;
+ case MSG_OSD_EC_WRITE:
+ m = new MOSDECSubOpWrite;
+ break;
+ case MSG_OSD_EC_WRITE_REPLY:
+ m = new MOSDECSubOpWriteReply;
+ break;
+ case MSG_OSD_EC_READ:
+ m = new MOSDECSubOpRead;
+ break;
+ case MSG_OSD_EC_READ_REPLY:
+ m = new MOSDECSubOpReadReply;
+ break;
// auth
case CEPH_MSG_AUTH:
m = new MAuth;
diff --git a/src/msg/Message.h b/src/msg/Message.h
index f345e7a..740be7b 100644
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -107,6 +107,11 @@
#define MSG_OSD_PG_PULL 106
#define MSG_OSD_PG_PUSH_REPLY 107
+#define MSG_OSD_EC_WRITE 108
+#define MSG_OSD_EC_WRITE_REPLY 109
+#define MSG_OSD_EC_READ 110
+#define MSG_OSD_EC_READ_REPLY 111
+
// *** MDS ***
#define MSG_MDS_BEACON 100 // to monitor
@@ -398,10 +403,13 @@ public:
payload.clear();
middle.clear();
}
+
+ virtual void clear_buffers() {}
void clear_data() {
if (byte_throttler)
byte_throttler->put(data.length());
data.clear();
+ clear_buffers(); // let subclass drop buffers as well
}
bool empty_payload() { return payload.length() == 0; }
diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index 6389e20..9020888 100644
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -211,12 +211,14 @@ void *Pipe::DelayedDelivery::entry()
continue;
}
utime_t release = delay_queue.front().first;
- if (release > ceph_clock_now(pipe->msgr->cct)) {
+ Message *m = delay_queue.front().second;
+ string delay_msg_type = pipe->msgr->cct->_conf->ms_inject_delay_msg_type;
+ if (release > ceph_clock_now(pipe->msgr->cct) &&
+ (delay_msg_type.empty() || m->get_type_name() == delay_msg_type)) {
lgeneric_subdout(pipe->msgr->cct, ms, 10) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry sleeping on delay_cond until " << release << dendl;
delay_cond.WaitUntil(delay_lock, release);
continue;
}
- Message *m = delay_queue.front().second;
lgeneric_subdout(pipe->msgr->cct, ms, 10) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry dequeuing message " << m << " for delivery, past " << release << dendl;
delay_queue.pop_front();
pipe->in_q->enqueue(m, m->get_priority(), pipe->conn_id);
diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index ef97b01..78e0e69 100644
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -147,6 +147,7 @@ void SimpleMessenger::set_addr_unknowns(entity_addr_t &addr)
int port = my_inst.addr.get_port();
my_inst.addr.addr = addr.addr;
my_inst.addr.set_port(port);
+ init_local_connection();
}
}
@@ -294,8 +295,10 @@ int SimpleMessenger::start()
assert(!started);
started = true;
- if (!did_bind)
+ if (!did_bind) {
my_inst.addr.nonce = nonce;
+ init_local_connection();
+ }
lock.Unlock();
diff --git a/src/os/DBObjectMap.cc b/src/os/DBObjectMap.cc
index 886658f..5d2a2e6 100644
--- a/src/os/DBObjectMap.cc
+++ b/src/os/DBObjectMap.cc
@@ -156,9 +156,8 @@ string DBObjectMap::ghobject_key(const ghobject_t &oid)
t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.hobj.pool);
snprintf(t, end - t, ".%.*X", (int)(sizeof(oid.hobj.hash)*2), oid.hobj.hash);
- if (oid.generation != ghobject_t::NO_GEN) {
- assert(oid.shard_id != ghobject_t::NO_SHARD);
-
+ if (oid.generation != ghobject_t::NO_GEN ||
+ oid.shard_id != ghobject_t::NO_SHARD) {
t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.generation);
t += snprintf(t, end - t, ".%x", (int)oid.shard_id);
}
@@ -248,9 +247,9 @@ bool DBObjectMap::parse_ghobject_key_v0(const string &in, coll_t *c,
*c = coll_t(coll);
int64_t pool = -1;
- pg_t pg;
+ spg_t pg;
if (c->is_pg_prefix(pg))
- pool = (int64_t)pg.pool();
+ pool = (int64_t)pg.pgid.pool();
(*oid) = ghobject_t(hobject_t(name, key, snap, hash, pool, ""));
return true;
}
@@ -800,7 +799,7 @@ int DBObjectMap::clear_keys_header(const ghobject_t &oid,
// create new header
Header newheader = generate_new_header(oid, Header());
set_map_header(oid, *newheader, t);
- if (attrs.size())
+ if (!attrs.empty())
t->set(xattr_prefix(newheader), attrs);
return db->submit_transaction(t);
}
diff --git a/src/os/FileJournal.cc b/src/os/FileJournal.cc
index 3a344fa..54c0371 100644
--- a/src/os/FileJournal.cc
+++ b/src/os/FileJournal.cc
@@ -75,8 +75,8 @@ int FileJournal::_open(bool forwrite, bool create)
fd = TEMP_FAILURE_RETRY(::open(fn.c_str(), flags, 0644));
if (fd < 0) {
int err = errno;
- dout(2) << "FileJournal::_open: unable to open journal: open() failed: "
- << cpp_strerror(err) << dendl;
+ dout(2) << "FileJournal::_open unable to open journal "
+ << fn << ": " << cpp_strerror(err) << dendl;
return -err;
}
@@ -1576,9 +1576,12 @@ void FileJournal::put_throttle(uint64_t ops, uint64_t bytes)
}
}
-void FileJournal::make_writeable()
+int FileJournal::make_writeable()
{
- _open(true);
+ dout(10) << __func__ << dendl;
+ int r = _open(true);
+ if (r < 0)
+ return r;
if (read_pos > 0)
write_pos = read_pos;
@@ -1588,6 +1591,7 @@ void FileJournal::make_writeable()
must_write_header = true;
start_writer();
+ return 0;
}
void FileJournal::wrap_read_bl(
diff --git a/src/os/FileJournal.h b/src/os/FileJournal.h
index c69fc54..dbb1181 100644
--- a/src/os/FileJournal.h
+++ b/src/os/FileJournal.h
@@ -400,7 +400,7 @@ private:
bool is_writeable() {
return read_pos == 0;
}
- void make_writeable();
+ int make_writeable();
// writes
void commit_start(uint64_t seq);
diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index db55ccc..6a170ce 100644
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -48,6 +48,7 @@
#include "FileStore.h"
#include "GenericFileStoreBackend.h"
#include "BtrfsFileStoreBackend.h"
+#include "XfsFileStoreBackend.h"
#include "ZFSFileStoreBackend.h"
#include "common/BackTrace.h"
#include "include/types.h"
@@ -460,6 +461,7 @@ FileStore::FileStore(const std::string &base, const std::string &jdev, const cha
m_filestore_dump_fmt(true),
m_filestore_sloppy_crc(g_conf->filestore_sloppy_crc),
m_filestore_sloppy_crc_block_size(g_conf->filestore_sloppy_crc_block_size),
+ m_filestore_max_alloc_hint_size(g_conf->filestore_max_alloc_hint_size),
m_fs_type(FS_TYPE_NONE),
m_filestore_max_inline_xattr_size(0),
m_filestore_max_inline_xattrs(0)
@@ -679,6 +681,10 @@ int FileStore::mkfs()
#if defined(__linux__)
backend = new BtrfsFileStoreBackend(this);
#endif
+ } else if (basefs.f_type == XFS_SUPER_MAGIC) {
+#ifdef HAVE_LIBXFS
+ backend = new XfsFileStoreBackend(this);
+#endif
} else if (basefs.f_type == ZFS_SUPER_MAGIC) {
#ifdef HAVE_LIBZFS
backend = new ZFSFileStoreBackend(this);
@@ -866,30 +872,37 @@ int FileStore::_detect_fs()
blk_size = st.f_bsize;
m_fs_type = FS_TYPE_OTHER;
-#if defined(__linux__)
if (st.f_type == BTRFS_SUPER_MAGIC) {
+#if defined(__linux__)
dout(0) << "mount detected btrfs" << dendl;
backend = new BtrfsFileStoreBackend(this);
+ m_fs_type = FS_TYPE_BTRFS;
wbthrottle.set_fs(WBThrottle::BTRFS);
- m_fs_type = FS_TYPE_BTRFS;
+#endif
} else if (st.f_type == XFS_SUPER_MAGIC) {
- dout(1) << "mount detected xfs" << dendl;
+#ifdef HAVE_LIBXFS
+ dout(0) << "mount detected xfs (libxfs)" << dendl;
+ backend = new XfsFileStoreBackend(this);
+#else
+ dout(0) << "mount detected xfs" << dendl;
+#endif
m_fs_type = FS_TYPE_XFS;
+
+ // wbthrottle is constructed with fs(WBThrottle::XFS)
if (m_filestore_replica_fadvise) {
dout(1) << " disabling 'filestore replica fadvise' due to known issues with fadvise(DONTNEED) on xfs" << dendl;
g_conf->set_val("filestore_replica_fadvise", "false");
g_conf->apply_changes(NULL);
assert(m_filestore_replica_fadvise == false);
}
- }
-#endif
+ } else if (st.f_type == ZFS_SUPER_MAGIC) {
#ifdef HAVE_LIBZFS
- if (st.f_type == ZFS_SUPER_MAGIC) {
+ dout(0) << "mount detected zfs (libzfs)" << dendl;
backend = new ZFSFileStoreBackend(this);
m_fs_type = FS_TYPE_ZFS;
- }
#endif
+ }
set_xattr_limits_via_conf();
@@ -1705,7 +1718,9 @@ void FileStore::_finish_op(OpSequencer *osr)
if (o->onreadable_sync) {
o->onreadable_sync->complete(0);
}
- op_finisher.queue(o->onreadable);
+ if (o->onreadable) {
+ op_finisher.queue(o->onreadable);
+ }
delete o;
}
@@ -1734,6 +1749,9 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
tls, &onreadable, &ondisk, &onreadable_sync);
if (g_conf->filestore_blackhole) {
dout(0) << "queue_transactions filestore_blackhole = TRUE, dropping transaction" << dendl;
+ delete ondisk;
+ delete onreadable;
+ delete onreadable_sync;
return 0;
}
@@ -2427,6 +2445,18 @@ unsigned FileStore::_do_transaction(
}
break;
+ case Transaction::OP_SETALLOCHINT:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t expected_object_size = i.get_length();
+ uint64_t expected_write_size = i.get_length();
+ if (_check_replay_guard(cid, oid, spos) > 0)
+ r = _set_alloc_hint(cid, oid, expected_object_size,
+ expected_write_size);
+ }
+ break;
+
default:
derr << "bad op " << op << dendl;
assert(0);
@@ -2449,6 +2479,14 @@ unsigned FileStore::_do_transaction(
if (r == -ENODATA)
ok = true;
+ if (op == Transaction::OP_SETALLOCHINT)
+ // Either EOPNOTSUPP or EINVAL most probably. EINVAL in most
+ // cases means invalid hint size (e.g. too big, not a multiple
+ // of block size, etc) or, at least on xfs, an attempt to set
+ // or change it when the file is not empty. However,
+ // OP_SETALLOCHINT is advisory, so ignore all errors.
+ ok = true;
+
if (replaying && !backend->can_checkpoint()) {
if (r == -EEXIST && op == Transaction::OP_MKCOLL) {
dout(10) << "tolerating EEXIST during journal replay since checkpoint is not enabled" << dendl;
@@ -3630,7 +3668,7 @@ int FileStore::_setattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr
inline_to_set.insert(*p);
}
- if (spill_out != 1 && omap_set.size()) {
+ if (spill_out != 1 && !omap_set.empty()) {
chain_fsetxattr(**fd, XATTR_SPILL_OUT_NAME, XATTR_SPILL_OUT,
sizeof(XATTR_SPILL_OUT));
}
@@ -4429,10 +4467,12 @@ int FileStore::_collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
_inject_failure();
- // the name changed; link the omap content
- r = object_map->clone(oldoid, o, &spos);
- if (r == -ENOENT)
- r = 0;
+ if (r == 0) {
+ // the name changed; link the omap content
+ r = object_map->clone(oldoid, o, &spos);
+ if (r == -ENOENT)
+ r = 0;
+ }
_inject_failure();
@@ -4677,6 +4717,34 @@ int FileStore::_split_collection_create(coll_t cid,
return r;
}
+int FileStore::_set_alloc_hint(coll_t cid, const ghobject_t& oid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+{
+ dout(15) << "set_alloc_hint " << cid << "/" << oid << " object_size " << expected_object_size << " write_size " << expected_write_size << dendl;
+
+ FDRef fd;
+ int ret;
+
+ ret = lfn_open(cid, oid, false, &fd);
+ if (ret < 0)
+ goto out;
+
+ {
+ // TODO: a more elaborate hint calculation
+ uint64_t hint = MIN(expected_write_size, m_filestore_max_alloc_hint_size);
+
+ ret = backend->set_alloc_hint(**fd, hint);
+ dout(20) << "set_alloc_hint hint " << hint << " ret " << ret << dendl;
+ }
+
+ lfn_close(fd);
+out:
+ dout(10) << "set_alloc_hint " << cid << "/" << oid << " object_size " << expected_object_size << " write_size " << expected_write_size << " = " << ret << dendl;
+ assert(!m_filestore_fail_eio || ret != -EIO);
+ return ret;
+}
+
const char** FileStore::get_tracked_conf_keys() const
{
static const char* KEYS[] = {
@@ -4693,6 +4761,7 @@ const char** FileStore::get_tracked_conf_keys() const
"filestore_replica_fadvise",
"filestore_sloppy_crc",
"filestore_sloppy_crc_block_size",
+ "filestore_max_alloc_hint_size",
NULL
};
return KEYS;
@@ -4722,6 +4791,7 @@ void FileStore::handle_conf_change(const struct md_config_t *conf,
changed.count("filestore_fail_eio") ||
changed.count("filestore_sloppy_crc") ||
changed.count("filestore_sloppy_crc_block_size") ||
+ changed.count("filestore_max_alloc_hint_size") ||
changed.count("filestore_replica_fadvise")) {
Mutex::Locker l(lock);
m_filestore_min_sync_interval = conf->filestore_min_sync_interval;
@@ -4735,6 +4805,7 @@ void FileStore::handle_conf_change(const struct md_config_t *conf,
m_filestore_replica_fadvise = conf->filestore_replica_fadvise;
m_filestore_sloppy_crc = conf->filestore_sloppy_crc;
m_filestore_sloppy_crc_block_size = conf->filestore_sloppy_crc_block_size;
+ m_filestore_max_alloc_hint_size = conf->filestore_max_alloc_hint_size;
}
if (changed.count("filestore_commit_timeout")) {
Mutex::Locker l(sync_entry_timeo_lock);
diff --git a/src/os/FileStore.h b/src/os/FileStore.h
index d30eeba..4c9ffdb 100644
--- a/src/os/FileStore.h
+++ b/src/os/FileStore.h
@@ -63,33 +63,6 @@ static const __SWORD_TYPE XFS_SUPER_MAGIC(0x58465342);
static const __SWORD_TYPE ZFS_SUPER_MAGIC(0x2fc12fc1);
#endif
-enum {
- l_os_first = 84000,
- l_os_jq_max_ops,
- l_os_jq_ops,
- l_os_j_ops,
- l_os_jq_max_bytes,
- l_os_jq_bytes,
- l_os_j_bytes,
- l_os_j_lat,
- l_os_j_wr,
- l_os_j_wr_bytes,
- l_os_oq_max_ops,
- l_os_oq_ops,
- l_os_ops,
- l_os_oq_max_bytes,
- l_os_oq_bytes,
- l_os_bytes,
- l_os_apply_lat,
- l_os_committing,
- l_os_commit,
- l_os_commit_len,
- l_os_commit_lat,
- l_os_j_full,
- l_os_queue_lat,
- l_os_last,
-};
-
enum fs_types {
FS_TYPE_NONE = 0,
@@ -582,6 +555,11 @@ public:
int _collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
coll_t c, const ghobject_t& o,
const SequencerPosition& spos);
+
+ int _set_alloc_hint(coll_t cid, const ghobject_t& oid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size);
+
void dump_start(const std::string& file);
void dump_stop();
void dump_transactions(list<ObjectStore::Transaction*>& ls, uint64_t seq, OpSequencer *osr);
@@ -634,6 +612,7 @@ private:
atomic_t m_filestore_kill_at;
bool m_filestore_sloppy_crc;
int m_filestore_sloppy_crc_block_size;
+ uint64_t m_filestore_max_alloc_hint_size;
enum fs_types m_fs_type;
//Determined xattr handling based on fs type
@@ -711,6 +690,7 @@ public:
virtual bool has_fiemap() = 0;
virtual int do_fiemap(int fd, off_t start, size_t len, struct fiemap **pfiemap) = 0;
virtual int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) = 0;
+ virtual int set_alloc_hint(int fd, uint64_t hint) = 0;
// hooks for (sloppy) crc tracking
virtual int _crc_update_write(int fd, loff_t off, size_t len, const bufferlist& bl) = 0;
diff --git a/src/os/GenericFileStoreBackend.h b/src/os/GenericFileStoreBackend.h
index 5a09c24..374e193 100644
--- a/src/os/GenericFileStoreBackend.h
+++ b/src/os/GenericFileStoreBackend.h
@@ -42,6 +42,7 @@ public:
virtual int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) {
return _copy_range(from, to, srcoff, len, dstoff);
}
+ virtual int set_alloc_hint(int fd, uint64_t hint) { return -EOPNOTSUPP; }
private:
int _crc_load_or_init(int fd, SloppyCRCMap *cm);
diff --git a/src/os/GenericObjectMap.cc b/src/os/GenericObjectMap.cc
index 3be9b76..4d41c50 100644
--- a/src/os/GenericObjectMap.cc
+++ b/src/os/GenericObjectMap.cc
@@ -624,6 +624,7 @@ int GenericObjectMap::get_keys(const coll_t &cid, const ghobject_t &oid,
}
return 0;
}
+
int GenericObjectMap::get_values(const coll_t &cid, const ghobject_t &oid,
const string &prefix,
const set<string> &keys,
@@ -916,25 +917,44 @@ int GenericObjectMap::write_state(KeyValueDB::Transaction t)
return 0;
}
+// NOTE(haomai): It may occur dead lock if thread A hold header A try to header
+// B and thread hold header B try to get header A
GenericObjectMap::Header GenericObjectMap::_lookup_header(
const coll_t &cid, const ghobject_t &oid)
{
- // FIXME
- while (map_header_in_use.count(oid))
- header_cond.Wait(header_lock);
-
- map<string, bufferlist> out;
set<string> to_get;
to_get.insert(header_key(cid, oid));
- int r = db->get(GHOBJECT_TO_SEQ_PREFIX, to_get, &out);
- if (r < 0)
- return Header();
- if (out.empty())
- return Header();
+ _Header header;
- Header ret(new _Header(), RemoveMapHeaderOnDelete(this, cid, oid));
- bufferlist::iterator iter = out.begin()->second.begin();
- ret->decode(iter);
+ while (1) {
+ map<string, bufferlist> out;
+ bool try_again = false;
+
+ int r = db->get(GHOBJECT_TO_SEQ_PREFIX, to_get, &out);
+ if (r < 0)
+ return Header();
+ if (out.empty())
+ return Header();
+
+ bufferlist::iterator iter = out.begin()->second.begin();
+ header.decode(iter);
+
+ while (in_use.count(header.seq)) {
+ header_cond.Wait(header_lock);
+
+ // Another thread is hold this header, wait for it.
+ // Because the seq of this object may change, such as clone
+ // and rename operation, here need to look up "seq" again
+ try_again = true;
+ }
+
+ if (!try_again) {
+ break;
+ }
+ }
+
+ Header ret = Header(new _Header(header), RemoveOnDelete(this));
+ in_use.insert(ret->seq);
return ret;
}
@@ -968,6 +988,7 @@ GenericObjectMap::Header GenericObjectMap::lookup_parent(Header input)
dout(20) << "lookup_parent: parent " << input->parent
<< " for seq " << input->seq << dendl;
+
int r = db->get(parent_seq_prefix(input->parent), keys, &out);
if (r < 0) {
assert(0);
@@ -983,7 +1004,7 @@ GenericObjectMap::Header GenericObjectMap::lookup_parent(Header input)
bufferlist::iterator iter = out.begin()->second.begin();
header->decode(iter);
dout(20) << "lookup_parent: parent seq is " << header->seq << " with parent "
- << header->parent << dendl;
+ << header->parent << dendl;
in_use.insert(header->seq);
return header;
}
@@ -1018,6 +1039,7 @@ void GenericObjectMap::clear_header(Header header, KeyValueDB::Transaction t)
t->rmkeys(parent_seq_prefix(header->seq), keys);
}
+// only remove GHOBJECT_TO_SEQ
void GenericObjectMap::remove_header(const coll_t &cid,
const ghobject_t &oid, Header header,
KeyValueDB::Transaction t)
@@ -1043,6 +1065,7 @@ void GenericObjectMap::set_header(const coll_t &cid, const ghobject_t &oid,
int GenericObjectMap::list_objects(const coll_t &cid, ghobject_t start, int max,
vector<ghobject_t> *out, ghobject_t *next)
{
+ // FIXME
Mutex::Locker l(header_lock);
if (start.is_max())
diff --git a/src/os/GenericObjectMap.h b/src/os/GenericObjectMap.h
index fddbe53..c9c64bc 100644
--- a/src/os/GenericObjectMap.h
+++ b/src/os/GenericObjectMap.h
@@ -30,6 +30,8 @@
#include "osd/osd_types.h"
#include "common/Mutex.h"
#include "common/Cond.h"
+#include "common/simple_cache.hpp"
+
/**
* Genericobjectmap: Provide with key/value associated to ghobject_t APIs to caller
@@ -73,13 +75,11 @@ class GenericObjectMap {
*/
Mutex header_lock;
Cond header_cond;
- Cond map_header_cond;
/**
* Set of headers currently in use
*/
set<uint64_t> in_use;
- set<ghobject_t> map_header_in_use;
GenericObjectMap(KeyValueDB *db) : db(db), header_lock("GenericObjectMap") {}
@@ -365,11 +365,18 @@ private:
int adjust();
};
+protected:
typedef ceph::shared_ptr<GenericObjectMapIteratorImpl> GenericObjectMapIterator;
GenericObjectMapIterator _get_iterator(Header header, string prefix) {
return GenericObjectMapIterator(new GenericObjectMapIteratorImpl(this, header, prefix));
}
+ // Scan keys in header into out_keys and out_values (if nonnull)
+ int scan(Header header, const string &prefix, const set<string> &in_keys,
+ set<string> *out_keys, map<string, bufferlist> *out_values);
+
+ private:
+
/// Removes node corresponding to header
void clear_header(Header header, KeyValueDB::Transaction t);
@@ -399,10 +406,6 @@ private:
// Lookup header node for input
Header lookup_parent(Header input);
- // Scan keys in header into out_keys and out_values (if nonnull)
- int scan(Header header, const string &prefix, const set<string> &in_keys,
- set<string> *out_keys, map<string, bufferlist> *out_values);
-
// Remove header and all related prefixes
int _clear(Header header, KeyValueDB::Transaction t);
@@ -424,27 +427,8 @@ private:
KeyValueDB::Transaction t);
/**
- * Removes map header lock once Header is out of scope
- * @see lookup_header
- */
- class RemoveMapHeaderOnDelete {
- public:
- GenericObjectMap *db;
- coll_t cid;
- ghobject_t oid;
- RemoveMapHeaderOnDelete(GenericObjectMap *db, const coll_t &cid,
- const ghobject_t &oid) :
- db(db), cid(cid), oid(oid) {}
- void operator() (_Header *header) {
- Mutex::Locker l(db->header_lock);
- db->map_header_in_use.erase(oid);
- db->map_header_cond.Signal();
- delete header;
- }
- };
-
- /**
* Removes header seq lock once Header is out of scope
+ * @see _lookup_header
* @see lookup_parent
* @see generate_new_header
*/
diff --git a/src/os/Journal.h b/src/os/Journal.h
index 1d413bb..4f8658f 100644
--- a/src/os/Journal.h
+++ b/src/os/Journal.h
@@ -56,7 +56,7 @@ public:
// writes
virtual bool is_writeable() = 0;
- virtual void make_writeable() = 0;
+ virtual int make_writeable() = 0;
virtual void submit_entry(uint64_t seq, bufferlist& e, int alignment,
Context *oncommit,
TrackedOpRef osd_op = TrackedOpRef()) = 0;
diff --git a/src/os/JournalingObjectStore.cc b/src/os/JournalingObjectStore.cc
index e662580..402fa3c 100644
--- a/src/os/JournalingObjectStore.cc
+++ b/src/os/JournalingObjectStore.cc
@@ -98,7 +98,9 @@ int JournalingObjectStore::journal_replay(uint64_t fs_op_seq)
submit_manager.set_op_seq(op_seq);
// done reading, make writeable.
- journal->make_writeable();
+ err = journal->make_writeable();
+ if (err < 0)
+ return err;
return count;
}
diff --git a/src/os/KeyValueStore.cc b/src/os/KeyValueStore.cc
index e209dbc..8151789 100644
--- a/src/os/KeyValueStore.cc
+++ b/src/os/KeyValueStore.cc
@@ -21,6 +21,8 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/file.h>
+#include <sys/param.h>
+#include <sys/mount.h>
#include <errno.h>
#include <dirent.h>
@@ -82,28 +84,26 @@ void StripObjectMap::sync_wrap(StripObjectHeader &strip_header,
}
bool StripObjectMap::check_spos(const StripObjectHeader &header,
- const SequencerPosition *spos)
+ const SequencerPosition &spos)
{
- if (!spos || *spos > header.spos) {
+ if (spos > header.spos) {
stringstream out;
- if (spos)
- dout(10) << "cid: " << "oid: " << header.oid
- << " not skipping op, *spos " << *spos << dendl;
- else
- dout(10) << "cid: " << "oid: " << header.oid
- << " not skipping op, *spos " << "empty" << dendl;
+ dout(10) << "cid: " << "oid: " << header.oid
+ << " not skipping op, *spos " << spos << dendl;
dout(10) << " > header.spos " << header.spos << dendl;
return false;
} else {
- dout(10) << "cid: " << "oid: " << header.oid << " skipping op, *spos "
- << *spos << " <= header.spos " << header.spos << dendl;
+ dout(10) << "cid: " << "oid: " << header.oid << " skipping op, spos "
+ << spos << " <= header.spos " << header.spos << dendl;
return true;
}
}
int StripObjectMap::save_strip_header(StripObjectHeader &strip_header,
+ const SequencerPosition &spos,
KeyValueDB::Transaction t)
{
+ strip_header.spos = spos;
strip_header.header->data.clear();
::encode(strip_header, strip_header.header->data);
@@ -197,7 +197,6 @@ int StripObjectMap::file_to_extents(uint64_t offset, size_t len,
void StripObjectMap::clone_wrap(StripObjectHeader &old_header,
const coll_t &cid, const ghobject_t &oid,
KeyValueDB::Transaction t,
- const SequencerPosition &spos,
StripObjectHeader *origin_header,
StripObjectHeader *target_header)
{
@@ -211,19 +210,14 @@ void StripObjectMap::clone_wrap(StripObjectHeader &old_header,
old_header.header = new_origin_header;
- if (origin_header)
- origin_header->spos = spos;
-
if (target_header) {
target_header->oid = oid;
target_header->cid = cid;
- target_header->spos = spos;
}
}
void StripObjectMap::rename_wrap(const coll_t &cid, const ghobject_t &oid,
KeyValueDB::Transaction t,
- const SequencerPosition &spos,
StripObjectHeader *header)
{
assert(header);
@@ -232,11 +226,42 @@ void StripObjectMap::rename_wrap(const coll_t &cid, const ghobject_t &oid,
if (header) {
header->oid = oid;
header->cid = cid;
- header->spos = spos;
}
}
+int StripObjectMap::get_values_with_header(const StripObjectHeader &header,
+ const string &prefix,
+ const set<string> &keys,
+ map<string, bufferlist> *out)
+{
+ return scan(header.header, prefix, keys, 0, out);
+}
+int StripObjectMap::get_keys_with_header(const StripObjectHeader &header,
+ const string &prefix,
+ set<string> *keys)
+{
+ ObjectMap::ObjectMapIterator iter = _get_iterator(header.header, prefix);
+ for (; iter->valid(); iter->next()) {
+ if (iter->status())
+ return iter->status();
+ keys->insert(iter->key());
+ }
+ return 0;
+}
+
+int StripObjectMap::get_with_header(const StripObjectHeader &header,
+ const string &prefix, map<string, bufferlist> *out)
+{
+ ObjectMap::ObjectMapIterator iter = _get_iterator(header.header, prefix);
+ for (iter->seek_to_first(); iter->valid(); iter->next()) {
+ if (iter->status())
+ return iter->status();
+ out->insert(make_pair(iter->key(), iter->value()));
+ }
+
+ return 0;
+}
// =========== KeyValueStore::SubmitManager Implementation ==============
uint64_t KeyValueStore::SubmitManager::op_submit_start()
@@ -262,28 +287,11 @@ void KeyValueStore::SubmitManager::op_submit_finish(uint64_t op)
// ========= KeyValueStore::BufferTransaction Implementation ============
-int KeyValueStore::BufferTransaction::check_coll(const coll_t &cid)
-{
- int r = store->_check_coll(cid);
- if (r == 0)
- return r;
-
- StripHeaderMap::iterator it = strip_headers.find(
- make_pair(get_coll_for_coll(), make_ghobject_for_coll(cid)));
- if (it != strip_headers.end() && !it->second.deleted) {
- return 0;
- }
- return -ENOENT;
-}
-
int KeyValueStore::BufferTransaction::lookup_cached_header(
const coll_t &cid, const ghobject_t &oid,
StripObjectMap::StripObjectHeader **strip_header,
bool create_if_missing)
{
- if (check_coll(cid) < 0)
- return -ENOENT;
-
StripObjectMap::StripObjectHeader header;
int r = 0;
@@ -314,116 +322,109 @@ int KeyValueStore::BufferTransaction::lookup_cached_header(
return r;
}
-int KeyValueStore::BufferTransaction::get_buffer_key(
- StripObjectMap::StripObjectHeader *strip_header, const string &prefix,
- const string &key, bufferlist &bl)
+int KeyValueStore::BufferTransaction::get_buffer_keys(
+ StripObjectMap::StripObjectHeader &strip_header, const string &prefix,
+ const set<string> &keys, map<string, bufferlist> *out)
{
- if (strip_header->buffers.count(make_pair(prefix, key))) {
- bl.swap(strip_header->buffers[make_pair(prefix, key)]);
- return 0;
+ set<string> need_lookup;
+
+ for (set<string>::iterator it = keys.begin(); it != keys.end(); ++it) {
+ map<pair<string, string>, bufferlist>::iterator i =
+ strip_header.buffers.find(make_pair(prefix, *it));
+
+ if (i != strip_header.buffers.end()) {
+ (*out)[*it].swap(i->second);
+ } else {
+ need_lookup.insert(*it);
+ }
}
- set<string> keys;
- map<string, bufferlist> out;
- keys.insert(key);
- int r = store->backend->get_values(strip_header->cid, strip_header->oid,
- prefix, keys, &out);
- if (r < 0) {
- dout(10) << __func__ << " " << strip_header->cid << "/"
- << strip_header->oid << " " << " r = " << r << dendl;
- return r;
+ if (need_lookup.size()) {
+ int r = store->backend->get_values_with_header(strip_header, prefix,
+ need_lookup, out);
+ if (r < 0) {
+ dout(10) << __func__ << " " << strip_header.cid << "/"
+ << strip_header.oid << " " << " r = " << r << dendl;
+ return r;
+ }
}
- assert(out.size() == 1);
- bl.swap(out.begin()->second);
return 0;
}
void KeyValueStore::BufferTransaction::set_buffer_keys(
- const string &prefix, StripObjectMap::StripObjectHeader *strip_header,
- map<string, bufferlist> &values)
+ StripObjectMap::StripObjectHeader &strip_header,
+ const string &prefix, map<string, bufferlist> &values)
{
- if (store->backend->check_spos(*strip_header, &spos))
- return ;
-
- store->backend->set_keys(strip_header->header, prefix, values, t);
+ store->backend->set_keys(strip_header.header, prefix, values, t);
for (map<string, bufferlist>::iterator iter = values.begin();
iter != values.end(); ++iter) {
- strip_header->buffers[make_pair(prefix, iter->first)].swap(iter->second);
+ strip_header.buffers[make_pair(prefix, iter->first)].swap(iter->second);
}
}
int KeyValueStore::BufferTransaction::remove_buffer_keys(
- const string &prefix, StripObjectMap::StripObjectHeader *strip_header,
+ StripObjectMap::StripObjectHeader &strip_header, const string &prefix,
const set<string> &keys)
{
- if (store->backend->check_spos(*strip_header, &spos))
- return 0;
-
for (set<string>::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
- strip_header->buffers[make_pair(prefix, *iter)] = bufferlist();
+ strip_header.buffers[make_pair(prefix, *iter)] = bufferlist();
}
- return store->backend->rm_keys(strip_header->header, prefix, keys, t);
+ return store->backend->rm_keys(strip_header.header, prefix, keys, t);
}
void KeyValueStore::BufferTransaction::clear_buffer_keys(
- const string &prefix, StripObjectMap::StripObjectHeader *strip_header)
+ StripObjectMap::StripObjectHeader &strip_header, const string &prefix)
{
- for (map<pair<string, string>, bufferlist>::iterator iter = strip_header->buffers.begin();
- iter != strip_header->buffers.end(); ++iter) {
+ for (map<pair<string, string>, bufferlist>::iterator iter = strip_header.buffers.begin();
+ iter != strip_header.buffers.end(); ++iter) {
if (iter->first.first == prefix)
iter->second = bufferlist();
}
}
int KeyValueStore::BufferTransaction::clear_buffer(
- StripObjectMap::StripObjectHeader *strip_header)
+ StripObjectMap::StripObjectHeader &strip_header)
{
- if (store->backend->check_spos(*strip_header, &spos))
- return 0;
+ strip_header.deleted = true;
- strip_header->deleted = true;
-
- return store->backend->clear(strip_header->header, t);
+ return store->backend->clear(strip_header.header, t);
}
void KeyValueStore::BufferTransaction::clone_buffer(
- StripObjectMap::StripObjectHeader *old_header,
+ StripObjectMap::StripObjectHeader &old_header,
const coll_t &cid, const ghobject_t &oid)
{
- if (store->backend->check_spos(*old_header, &spos))
- return ;
-
// Remove target ahead to avoid dead lock
strip_headers.erase(make_pair(cid, oid));
StripObjectMap::StripObjectHeader new_origin_header, new_target_header;
- store->backend->clone_wrap(*old_header, cid, oid, t, spos,
+ store->backend->clone_wrap(old_header, cid, oid, t,
&new_origin_header, &new_target_header);
// FIXME: Lacking of lock for origin header(now become parent), it will
// cause other operation can get the origin header while submitting
// transactions
- strip_headers[make_pair(cid, old_header->oid)] = new_origin_header;
+ strip_headers[make_pair(cid, old_header.oid)] = new_origin_header;
strip_headers[make_pair(cid, oid)] = new_target_header;
}
void KeyValueStore::BufferTransaction::rename_buffer(
- StripObjectMap::StripObjectHeader *old_header,
+ StripObjectMap::StripObjectHeader &old_header,
const coll_t &cid, const ghobject_t &oid)
{
- if (store->backend->check_spos(*old_header, &spos))
+ if (store->backend->check_spos(old_header, spos))
return ;
// FIXME: Lacking of lock for origin header, it will cause other operation
// can get the origin header while submitting transactions
- store->backend->rename_wrap(cid, oid, t, spos, old_header);
+ store->backend->rename_wrap(cid, oid, t, &old_header);
- strip_headers.erase(make_pair(old_header->cid, old_header->oid));
- strip_headers[make_pair(cid, oid)] = *old_header;
+ strip_headers.erase(make_pair(old_header.cid, old_header.oid));
+ strip_headers[make_pair(cid, oid)] = old_header;
}
int KeyValueStore::BufferTransaction::submit_transaction()
@@ -434,13 +435,13 @@ int KeyValueStore::BufferTransaction::submit_transaction()
header_iter != strip_headers.end(); ++header_iter) {
StripObjectMap::StripObjectHeader header = header_iter->second;
- if (store->backend->check_spos(header, &spos))
+ if (store->backend->check_spos(header, spos))
continue;
if (header.deleted)
continue;
- r = store->backend->save_strip_header(header, t);
+ r = store->backend->save_strip_header(header, spos, t);
if (r < 0) {
dout(10) << __func__ << " save strip header failed " << dendl;
goto out;
@@ -498,14 +499,15 @@ KeyValueStore::KeyValueStore(const std::string &base,
lock("KeyValueStore::lock"),
default_osr("default"),
op_queue_len(0), op_queue_bytes(0),
+ op_throttle_lock("KeyValueStore::op_throttle_lock"),
op_finisher(g_ceph_context),
op_tp(g_ceph_context, "KeyValueStore::op_tp",
- g_conf->filestore_op_threads, "keyvaluestore_op_threads"),
- op_wq(this, g_conf->filestore_op_thread_timeout,
- g_conf->filestore_op_thread_suicide_timeout, &op_tp),
+ g_conf->keyvaluestore_op_threads, "keyvaluestore_op_threads"),
+ op_wq(this, g_conf->keyvaluestore_op_thread_timeout,
+ g_conf->keyvaluestore_op_thread_suicide_timeout, &op_tp),
logger(NULL),
- read_error_lock("KeyValueStore::read_error_lock"),
- m_fail_eio(g_conf->filestore_fail_eio),
+ m_keyvaluestore_queue_max_ops(g_conf->keyvaluestore_queue_max_ops),
+ m_keyvaluestore_queue_max_bytes(g_conf->keyvaluestore_queue_max_bytes),
do_update(do_update)
{
ostringstream oss;
@@ -517,7 +519,17 @@ KeyValueStore::KeyValueStore(const std::string &base,
current_op_seq_fn = sss.str();
// initialize logger
- PerfCountersBuilder plb(g_ceph_context, internal_name, 0, 1);
+ PerfCountersBuilder plb(g_ceph_context, internal_name, l_os_commit_lat, l_os_last);
+
+ plb.add_u64(l_os_oq_max_ops, "op_queue_max_ops");
+ plb.add_u64(l_os_oq_ops, "op_queue_ops");
+ plb.add_u64_counter(l_os_ops, "ops");
+ plb.add_u64(l_os_oq_max_bytes, "op_queue_max_bytes");
+ plb.add_u64(l_os_oq_bytes, "op_queue_bytes");
+ plb.add_u64_counter(l_os_bytes, "bytes");
+ plb.add_time_avg(l_os_apply_lat, "apply_latency");
+ plb.add_time_avg(l_os_queue_lat, "queue_transaction_latency_avg");
+
logger = plb.create_perf_counters();
g_ceph_context->get_perfcounters_collection()->add(logger);
@@ -536,7 +548,6 @@ int KeyValueStore::statfs(struct statfs *buf)
{
if (::statfs(basedir.c_str(), buf) < 0) {
int r = -errno;
- assert(!m_fail_eio || r != -EIO);
return r;
}
return 0;
@@ -853,7 +864,7 @@ int KeyValueStore::mount()
}
stringstream err2;
- if (g_conf->filestore_debug_omap_check && !dbomap->check(err2)) {
+ if (g_conf->keyvaluestore_debug_check_backend && !dbomap->check(err2)) {
derr << err2.str() << dendl;;
delete dbomap;
ret = -EINVAL;
@@ -877,7 +888,6 @@ close_fsid_fd:
TEMP_FAILURE_RETRY(::close(fsid_fd));
fsid_fd = -1;
done:
- assert(!m_fail_eio || ret != -EIO);
return ret;
}
@@ -949,6 +959,7 @@ int KeyValueStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
}
Op *o = build_op(tls, ondisk, onreadable, onreadable_sync, osd_op);
+ op_queue_reserve_throttle(o, handle);
uint64_t op = submit_manager.op_submit_start();
o->op = op;
dout(5) << "queue_transactions (trailing journal) " << op << " "
@@ -995,24 +1006,64 @@ void KeyValueStore::queue_op(OpSequencer *osr, Op *o)
osr->queue(o);
+ logger->inc(l_os_ops);
+ logger->inc(l_os_bytes, o->bytes);
+
dout(5) << "queue_op " << o << " seq " << o->op << " " << *osr << " "
<< o->bytes << " bytes" << " (queue has " << op_queue_len
<< " ops and " << op_queue_bytes << " bytes)" << dendl;
op_wq.queue(osr);
}
-void KeyValueStore::_do_op(OpSequencer *osr, ThreadPool::TPHandle &handle)
+void KeyValueStore::op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle)
{
- // inject a stall?
- if (g_conf->filestore_inject_stall) {
- int orig = g_conf->filestore_inject_stall;
- dout(5) << "_do_op filestore_inject_stall " << orig << ", sleeping" << dendl;
- for (int n = 0; n < g_conf->filestore_inject_stall; n++)
- sleep(1);
- g_conf->set_val("filestore_inject_stall", "0");
- dout(5) << "_do_op done stalling" << dendl;
+ uint64_t max_ops = m_keyvaluestore_queue_max_ops;
+ uint64_t max_bytes = m_keyvaluestore_queue_max_bytes;
+
+ logger->set(l_os_oq_max_ops, max_ops);
+ logger->set(l_os_oq_max_bytes, max_bytes);
+
+ utime_t start = ceph_clock_now(g_ceph_context);
+ {
+ Mutex::Locker l(op_throttle_lock);
+ while ((max_ops && (op_queue_len + 1) > max_ops) ||
+ (max_bytes && op_queue_bytes // let single large ops through!
+ && (op_queue_bytes + o->bytes) > max_bytes)) {
+ dout(2) << "waiting " << op_queue_len + 1 << " > " << max_ops
+ << " ops || " << op_queue_bytes + o->bytes << " > " << max_bytes
+ << dendl;
+ if (handle)
+ handle->suspend_tp_timeout();
+ op_throttle_cond.Wait(op_throttle_lock);
+ if (handle)
+ handle->reset_tp_timeout();
+ }
+
+ op_queue_len++;
+ op_queue_bytes += o->bytes;
+ }
+ utime_t end = ceph_clock_now(g_ceph_context);
+ logger->tinc(l_os_queue_lat, end - start);
+
+ logger->set(l_os_oq_ops, op_queue_len);
+ logger->set(l_os_oq_bytes, op_queue_bytes);
+}
+
+void KeyValueStore::op_queue_release_throttle(Op *o)
+{
+ {
+ Mutex::Locker l(op_throttle_lock);
+ op_queue_len--;
+ op_queue_bytes -= o->bytes;
+ op_throttle_cond.Signal();
}
+ logger->set(l_os_oq_ops, op_queue_len);
+ logger->set(l_os_oq_bytes, op_queue_bytes);
+}
+
+void KeyValueStore::_do_op(OpSequencer *osr, ThreadPool::TPHandle &handle)
+{
// FIXME: Suppose the collection of transaction only affect objects in the
// one PG, so this lock will ensure no other concurrent write operation
osr->apply_lock.Lock();
@@ -1038,9 +1089,11 @@ void KeyValueStore::_finish_op(OpSequencer *osr)
dout(10) << "_finish_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << dendl;
osr->apply_lock.Unlock(); // locked in _do_op
+ op_queue_release_throttle(o);
utime_t lat = ceph_clock_now(g_ceph_context);
lat -= o->start;
+ logger->tinc(l_os_apply_lat, lat);
if (o->onreadable_sync) {
o->onreadable_sync->complete(0);
@@ -1283,9 +1336,7 @@ unsigned KeyValueStore::_do_transaction(Transaction& transaction,
coll_t ocid = i.get_cid();
coll_t ncid = i.get_cid();
ghobject_t oid = i.get_oid();
- r = _collection_add(ocid, ncid, oid, t);
- if (r == 0)
- r = _remove(ocid, oid, t);
+ r = _collection_move_rename(ocid, oid, ncid, oid, t);
}
break;
@@ -1394,6 +1445,10 @@ unsigned KeyValueStore::_do_transaction(Transaction& transaction,
}
break;
+ case Transaction::OP_SETALLOCHINT:
+ // TODO: can kvstore make use of the hint?
+ break;
+
default:
derr << "bad op " << op << dendl;
assert(0);
@@ -1457,22 +1512,6 @@ unsigned KeyValueStore::_do_transaction(Transaction& transaction,
// =========== KeyValueStore Op Implementation ==============
// objects
-int KeyValueStore::_check_coll(const coll_t &cid)
-{
- if (is_coll_obj(cid))
- return 0;
-
- StripObjectMap::StripObjectHeader header;
- int r = backend->lookup_strip_header(get_coll_for_coll(),
- make_ghobject_for_coll(cid), header);
- if (r < 0) {
- dout(10) << __func__ << " could not find header r = " << r << dendl;
- return -ENOENT;
- }
-
- return 0;
-}
-
bool KeyValueStore::exists(coll_t cid, const ghobject_t& oid)
{
dout(10) << __func__ << "collection: " << cid << " object: " << oid
@@ -1480,11 +1519,6 @@ bool KeyValueStore::exists(coll_t cid, const ghobject_t& oid)
int r;
StripObjectMap::StripObjectHeader header;
- r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
r = backend->lookup_strip_header(cid, oid, header);
if (r < 0) {
return false;
@@ -1500,12 +1534,7 @@ int KeyValueStore::stat(coll_t cid, const ghobject_t& oid,
StripObjectMap::StripObjectHeader header;
- int r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
- r = backend->lookup_strip_header(cid, oid, header);
+ int r = backend->lookup_strip_header(cid, oid, header);
if (r < 0) {
dout(10) << "stat " << cid << "/" << oid << "=" << r << dendl;
return -ENOENT;
@@ -1521,43 +1550,14 @@ int KeyValueStore::stat(coll_t cid, const ghobject_t& oid,
return r;
}
-int KeyValueStore::_generic_read(coll_t cid, const ghobject_t& oid,
+int KeyValueStore::_generic_read(StripObjectMap::StripObjectHeader &header,
uint64_t offset, size_t len, bufferlist& bl,
bool allow_eio, BufferTransaction *bt)
{
- dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
- << len << dendl;
-
- int r;
- StripObjectMap::StripObjectHeader header;
-
- r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
- // use strip_header buffer
- if (bt) {
- StripObjectMap::StripObjectHeader *cache_header;
- r = bt->lookup_cached_header(cid, oid, &cache_header, false);
- if (r == 0) {
- header = *cache_header;
- }
- } else {
- r = backend->lookup_strip_header(cid, oid, header);
- }
-
- if (r < 0) {
- dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
- << len << " header isn't exist: r = " << r << dendl;
- return r;
- }
-
if (header.max_size < offset) {
- r = -EINVAL;
- dout(10) << __func__ << " " << cid << "/" << oid << ")"
+ dout(10) << __func__ << " " << header.cid << "/" << header.oid << ")"
<< " offset exceed the length of bl"<< dendl;
- return r;
+ return 0;
}
if (len == 0)
@@ -1578,6 +1578,7 @@ int KeyValueStore::_generic_read(coll_t cid, const ghobject_t& oid,
string key = strip_object_key(iter->no);
if (bt && header.buffers.count(make_pair(OBJECT_STRIP_PREFIX, key))) {
+ // use strip_header buffer
assert(header.bits[iter->no]);
out[key] = header.buffers[make_pair(OBJECT_STRIP_PREFIX, key)];
} else if (header.bits[iter->no]) {
@@ -1585,44 +1586,36 @@ int KeyValueStore::_generic_read(coll_t cid, const ghobject_t& oid,
}
}
- r = backend->get_values(cid, oid, OBJECT_STRIP_PREFIX, keys, &out);
+ int r = backend->get_values_with_header(header, OBJECT_STRIP_PREFIX, keys, &out);
if (r < 0) {
- dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
- << len << " = " << r << dendl;
- return r;
- }
- if (out.size() != keys.size()) {
- r = -EINVAL;
- dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
- << len << " get incorrect key/value pairs " << dendl;
+ dout(10) << __func__ << " " << header.cid << "/" << header.oid << " "
+ << offset << "~" << len << " = " << r << dendl;
return r;
+ } else if (out.size() != keys.size()) {
+ dout(0) << __func__ << " broken header or missing data in backend "
+ << header.cid << "/" << header.oid << " " << offset << "~"
+ << len << " = " << r << dendl;
+ return -EBADF;
}
- uint64_t readed = 0;
-
for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
iter != extents.end(); ++iter) {
string key = strip_object_key(iter->no);
- if (readed + header.strip_size > header.max_size) {
- if (header.bits[iter->no]) {
- out[key].copy(0, iter->len, bl);
- } else {
- bl.append_zero(iter->len);
- }
-
- break;
- }
if (header.bits[iter->no]) {
- bl.append(out[key]);
+ if (iter->len == header.strip_size) {
+ bl.claim_append(out[key]);
+ } else {
+ out[key].copy(iter->offset, iter->len, bl);
+ }
} else {
- bl.append_zero(header.strip_size);
+ bl.append_zero(iter->len);
}
- readed += header.strip_size;
}
- dout(10) << __func__ << " " << cid << "/" << oid << " " << offset
- << "~" << bl.length() << "/" << len << " r = " << r << dendl;
+ dout(10) << __func__ << " " << header.cid << "/" << header.oid << " "
+ << offset << "~" << bl.length() << "/" << len << " r = " << r
+ << dendl;
return bl.length();
}
@@ -1631,7 +1624,20 @@ int KeyValueStore::_generic_read(coll_t cid, const ghobject_t& oid,
int KeyValueStore::read(coll_t cid, const ghobject_t& oid, uint64_t offset,
size_t len, bufferlist& bl, bool allow_eio)
{
- return _generic_read(cid, oid, offset, len, bl, allow_eio);
+ dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << dendl;
+
+ StripObjectMap::StripObjectHeader header;
+
+ int r = backend->lookup_strip_header(cid, oid, header);
+
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << " header isn't exist: r = " << r << dendl;
+ return r;
+ }
+
+ return _generic_read(header, offset, len, bl, allow_eio);
}
int KeyValueStore::fiemap(coll_t cid, const ghobject_t& oid,
@@ -1642,11 +1648,6 @@ int KeyValueStore::fiemap(coll_t cid, const ghobject_t& oid,
int r;
StripObjectMap::StripObjectHeader header;
- r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
r = backend->lookup_strip_header(cid, oid, header);
if (r < 0) {
dout(10) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len
@@ -1682,7 +1683,7 @@ int KeyValueStore::_remove(coll_t cid, const ghobject_t& oid,
return r;
}
- r = t.clear_buffer(header);
+ r = t.clear_buffer(*header);
dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
return r;
@@ -1709,30 +1710,38 @@ int KeyValueStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size,
if (header->max_size > size) {
vector<StripObjectMap::StripExtent> extents;
- StripObjectMap::file_to_extents(size, header->max_size,
+ StripObjectMap::file_to_extents(size, header->max_size-size,
header->strip_size, extents);
assert(extents.size());
vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
- if (iter->offset != 0) {
+ if (header->bits[iter->no] && iter->offset != 0) {
bufferlist value;
- bufferlist old;
map<string, bufferlist> values;
- r = t.get_buffer_key(header, OBJECT_STRIP_PREFIX,
- strip_object_key(iter->no), old);
+ set<string> lookup_keys;
+ string key = strip_object_key(iter->no);
+
+ lookup_keys.insert(key);
+ r = t.get_buffer_keys(*header, OBJECT_STRIP_PREFIX,
+ lookup_keys, &values);
if (r < 0) {
dout(10) << __func__ << " " << cid << "/" << oid << " "
<< size << " = " << r << dendl;
return r;
+ } else if (values.size() != lookup_keys.size()) {
+ dout(0) << __func__ << " broken header or missing data in backend "
+ << header->cid << "/" << header->oid << " size " << size
+ << " r = " << r << dendl;
+ return -EBADF;
}
- old.copy(0, iter->offset, value);
+ values[key].copy(0, iter->offset, value);
value.append_zero(header->strip_size-iter->offset);
assert(value.length() == header->strip_size);
- ++iter;
+ value.swap(values[key]);
- values[strip_object_key(iter->no)] = value;
- t.set_buffer_keys(OBJECT_STRIP_PREFIX, header, values);
+ t.set_buffer_keys(*header, OBJECT_STRIP_PREFIX, values);
+ ++iter;
}
set<string> keys;
@@ -1742,7 +1751,7 @@ int KeyValueStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size,
header->bits[iter->no] = 0;
}
}
- r = t.remove_buffer_keys(OBJECT_STRIP_PREFIX, header, keys);
+ r = t.remove_buffer_keys(*header, OBJECT_STRIP_PREFIX, keys);
if (r < 0) {
dout(10) << __func__ << " " << cid << "/" << oid << " "
<< size << " = " << r << dendl;
@@ -1778,58 +1787,66 @@ int KeyValueStore::_touch(coll_t cid, const ghobject_t& oid,
return r;
}
-int KeyValueStore::_write(coll_t cid, const ghobject_t& oid,
- uint64_t offset, size_t len, const bufferlist& bl,
- BufferTransaction &t, bool replica)
+int KeyValueStore::_generic_write(StripObjectMap::StripObjectHeader &header,
+ uint64_t offset, size_t len,
+ const bufferlist& bl, BufferTransaction &t,
+ bool replica)
{
- dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
- << len << dendl;
- int r;
- StripObjectMap::StripObjectHeader *header;
-
- r = t.lookup_cached_header(cid, oid, &header, true);
- if (r < 0) {
- dout(10) << __func__ << " " << cid << "/" << oid << " " << offset
- << "~" << len << " failed to get header: r = " << r << dendl;
- return r;
- }
-
if (len > bl.length())
len = bl.length();
- if (len + offset > header->max_size) {
- header->max_size = len + offset;
- header->bits.resize(header->max_size/header->strip_size+1);
+ if (len + offset > header.max_size) {
+ header.max_size = len + offset;
+ header.bits.resize(header.max_size/header.strip_size+1);
}
vector<StripObjectMap::StripExtent> extents;
- StripObjectMap::file_to_extents(offset, len, header->strip_size,
+ StripObjectMap::file_to_extents(offset, len, header.strip_size,
extents);
+
+ map<string, bufferlist> out;
+ set<string> keys;
+ for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
+ iter != extents.end(); ++iter) {
+ if (header.bits[iter->no] && !(iter->offset == 0 &&
+ iter->len == header.strip_size))
+ keys.insert(strip_object_key(iter->no));
+ }
+
+ int r = t.get_buffer_keys(header, OBJECT_STRIP_PREFIX, keys, &out);
+ if (r < 0) {
+ dout(10) << __func__ << " failed to get value " << header.cid << "/"
+ << header.oid << " " << offset << "~" << len << " = " << r
+ << dendl;
+ return r;
+ } else if (keys.size() != out.size()) {
+ // Error on header.bits or the corresponding key/value pair is missing
+ dout(0) << __func__ << " broken header or missing data in backend "
+ << header.cid << "/" << header.oid << " " << offset << "~"
+ << len << " = " << r << dendl;
+ return -EBADF;
+ }
+
uint64_t bl_offset = 0;
map<string, bufferlist> values;
for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
iter != extents.end(); ++iter) {
bufferlist value;
string key = strip_object_key(iter->no);
- if (header->bits[iter->no]) {
- if (iter->offset == 0 && iter->len == header->strip_size) {
+ if (header.bits[iter->no]) {
+ if (iter->offset == 0 && iter->len == header.strip_size) {
bl.copy(bl_offset, iter->len, value);
bl_offset += iter->len;
} else {
- bufferlist old;
- r = t.get_buffer_key(header, OBJECT_STRIP_PREFIX, key, old);
- if (r < 0) {
- dout(10) << __func__ << " failed to get value " << cid << "/" << oid
- << " " << offset << "~" << len << " = " << r << dendl;
- return r;
- }
+ assert(out[key].length() == header.strip_size);
- old.copy(0, iter->offset, value);
+ out[key].copy(0, iter->offset, value);
bl.copy(bl_offset, iter->len, value);
bl_offset += iter->len;
- if (value.length() != header->strip_size)
- old.copy(value.length(), header->strip_size-value.length(), value);
+ if (value.length() != header.strip_size)
+ out[key].copy(value.length(), header.strip_size-value.length(),
+ value);
}
} else {
if (iter->offset)
@@ -1837,23 +1854,43 @@ int KeyValueStore::_write(coll_t cid, const ghobject_t& oid,
bl.copy(bl_offset, iter->len, value);
bl_offset += iter->len;
- if (value.length() < header->strip_size)
- value.append_zero(header->strip_size-value.length());
+ if (value.length() < header.strip_size)
+ value.append_zero(header.strip_size-value.length());
- header->bits[iter->no] = 1;
+ header.bits[iter->no] = 1;
}
- assert(value.length() == header->strip_size);
+ assert(value.length() == header.strip_size);
values[key].swap(value);
}
assert(bl_offset == len);
- t.set_buffer_keys(OBJECT_STRIP_PREFIX, header, values);
- dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~" << len
- << " = " << r << dendl;
+ t.set_buffer_keys(header, OBJECT_STRIP_PREFIX, values);
+ dout(10) << __func__ << " " << header.cid << "/" << header.oid << " "
+ << offset << "~" << len << " = " << r << dendl;
return r;
}
+int KeyValueStore::_write(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len, const bufferlist& bl,
+ BufferTransaction &t, bool replica)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << dendl;
+
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, true);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset
+ << "~" << len << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ return _generic_write(*header, offset, len, bl, t, replica);
+}
+
int KeyValueStore::_zero(coll_t cid, const ghobject_t& oid, uint64_t offset,
size_t len, BufferTransaction &t)
{
@@ -1865,7 +1902,7 @@ int KeyValueStore::_zero(coll_t cid, const ghobject_t& oid, uint64_t offset,
bl.push_back(bp);
int r = _write(cid, oid, offset, len, bl, t);
- dout(20) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
<< len << " = " << r << dendl;
return r;
}
@@ -1889,7 +1926,7 @@ int KeyValueStore::_clone(coll_t cid, const ghobject_t& oldoid,
return r;
}
- t.clone_buffer(old_header, cid, newoid);
+ t.clone_buffer(*old_header, cid, newoid);
dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
<< newoid << " = " << r << dendl;
@@ -1908,11 +1945,29 @@ int KeyValueStore::_clone_range(coll_t cid, const ghobject_t& oldoid,
int r;
bufferlist bl;
- r = _generic_read(cid, oldoid, srcoff, len, bl, &t);
+ StripObjectMap::StripObjectHeader *old_header, *new_header;
+
+ r = t.lookup_cached_header(cid, oldoid, &old_header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << " " << srcoff << "~" << len << " to " << dstoff
+ << " header isn't exist: r = " << r << dendl;
+ return r;
+ }
+
+ r = t.lookup_cached_header(cid, newoid, &new_header, true);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << " " << srcoff << "~" << len << " to " << dstoff
+ << " can't create header: r = " << r << dendl;
+ return r;
+ }
+
+ r = _generic_read(*old_header, srcoff, len, bl, &t);
if (r < 0)
goto out;
- r = _write(cid, newoid, dstoff, len, bl, t);
+ r = _generic_write(*new_header, dstoff, len, bl, t);
out:
dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
@@ -1933,11 +1988,6 @@ int KeyValueStore::getattr(coll_t cid, const ghobject_t& oid, const char *name,
map<string, bufferlist> got;
set<string> to_get;
- r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
to_get.insert(string(name));
r = backend->get_values(cid, oid, OBJECT_XATTR, to_get, &got);
if (r < 0 && r != -ENOENT) {
@@ -1964,11 +2014,6 @@ int KeyValueStore::getattrs(coll_t cid, const ghobject_t& oid,
int r;
map<string, bufferlist> attr_aset;
- r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
r = backend->get(cid, oid, OBJECT_XATTR, &attr_aset);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not get attrs r = " << r << dendl;
@@ -2020,7 +2065,7 @@ int KeyValueStore::_setattrs(coll_t cid, const ghobject_t& oid,
attrs[it->first].push_back(it->second);
}
- t.set_buffer_keys(OBJECT_XATTR, header, attrs);
+ t.set_buffer_keys(*header, OBJECT_XATTR, attrs);
out:
dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
@@ -2046,7 +2091,7 @@ int KeyValueStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name,
}
to_remove.insert(string(name));
- r = t.remove_buffer_keys(OBJECT_XATTR, header, to_remove);
+ r = t.remove_buffer_keys(*header, OBJECT_XATTR, to_remove);
dout(10) << __func__ << " " << cid << "/" << oid << " '" << name << "' = "
<< r << dendl;
@@ -2070,15 +2115,14 @@ int KeyValueStore::_rmattrs(coll_t cid, const ghobject_t& oid,
return r;
}
- r = backend->get_keys(cid, oid, OBJECT_XATTR, &attrs);
+ r = backend->get_keys_with_header(*header, OBJECT_XATTR, &attrs);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not get attrs r = " << r << dendl;
- assert(!m_fail_eio || r != -EIO);
return r;
}
- r = t.remove_buffer_keys(OBJECT_XATTR, header, attrs);
- t.clear_buffer_keys(OBJECT_XATTR, header);
+ r = t.remove_buffer_keys(*header, OBJECT_XATTR, attrs);
+ t.clear_buffer_keys(*header, OBJECT_XATTR);
dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
return r;
@@ -2119,17 +2163,12 @@ int KeyValueStore::collection_getattr(coll_t c, const char *name,
dout(15) << __func__ << " " << c.to_str() << " '" << name
<< "'" << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
set<string> keys;
map<string, bufferlist> out;
keys.insert(string(name));
- r = backend->get_values(get_coll_for_coll(), make_ghobject_for_coll(c),
- COLLECTION_ATTR, keys, &out);
+ int r = backend->get_values(get_coll_for_coll(), make_ghobject_for_coll(c),
+ COLLECTION_ATTR, keys, &out);
if (r < 0) {
dout(10) << __func__ << " could not get key" << string(name) << dendl;
r = -EINVAL;
@@ -2148,21 +2187,16 @@ int KeyValueStore::collection_getattrs(coll_t cid,
{
dout(10) << __func__ << " " << cid.to_str() << dendl;
- int r = _check_coll(cid);
- if (r < 0) {
- return r;
- }
-
map<string, bufferlist> out;
set<string> keys;
for (map<string, bufferptr>::iterator it = aset.begin();
- it != aset.end(); it++) {
+ it != aset.end(); ++it) {
keys.insert(it->first);
}
- r = backend->get_values(get_coll_for_coll(), make_ghobject_for_coll(cid),
- COLLECTION_ATTR, keys, &out);
+ int r = backend->get_values(get_coll_for_coll(), make_ghobject_for_coll(cid),
+ COLLECTION_ATTR, keys, &out);
if (r < 0) {
dout(10) << __func__ << " could not get keys" << dendl;
r = -EINVAL;
@@ -2203,7 +2237,7 @@ int KeyValueStore::_collection_setattr(coll_t c, const char *name,
bl.append(reinterpret_cast<const char*>(value), size);
out.insert(make_pair(string(name), bl));
- t.set_buffer_keys(COLLECTION_ATTR, header, out);
+ t.set_buffer_keys(*header, COLLECTION_ATTR, out);
dout(10) << __func__ << " " << c << " '"
<< name << "' len " << size << " = " << r << dendl;
@@ -2215,25 +2249,19 @@ int KeyValueStore::_collection_rmattr(coll_t c, const char *name,
{
dout(15) << __func__ << " " << c << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
bufferlist bl;
set<string> out;
StripObjectMap::StripObjectHeader *header;
- r = t.lookup_cached_header(get_coll_for_coll(),
- make_ghobject_for_coll(c),
- &header, false);
+ int r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(c), &header, false);
if (r < 0) {
dout(10) << __func__ << " could not find header r = " << r << dendl;
return r;
}
out.insert(string(name));
- r = t.remove_buffer_keys(COLLECTION_ATTR, header, out);
+ r = t.remove_buffer_keys(*header, COLLECTION_ATTR, out);
dout(10) << __func__ << " " << c << " = " << r << dendl;
return r;
@@ -2260,7 +2288,7 @@ int KeyValueStore::_collection_setattrs(coll_t cid,
attrs[it->first].push_back(it->second);
}
- t.set_buffer_keys(COLLECTION_ATTR, header, attrs);
+ t.set_buffer_keys(*header, COLLECTION_ATTR, attrs);
dout(10) << __func__ << " " << cid << " = " << r << dendl;
return r;
@@ -2310,7 +2338,7 @@ int KeyValueStore::_destroy_collection(coll_t c, BufferTransaction &t)
// All modified objects are marked deleted
for (BufferTransaction::StripHeaderMap::iterator iter = t.strip_headers.begin();
- iter != t.strip_headers.end(); iter++) {
+ iter != t.strip_headers.end(); ++iter) {
// sum the total modified object in this PG
if (iter->first.first != c)
continue;
@@ -2338,7 +2366,7 @@ int KeyValueStore::_destroy_collection(coll_t c, BufferTransaction &t)
}
}
- r = t.clear_buffer(header);
+ r = t.clear_buffer(*header);
out:
dout(10) << __func__ << " " << c << " = " << r << dendl;
@@ -2369,13 +2397,13 @@ int KeyValueStore::_collection_add(coll_t c, coll_t oldcid,
goto out;
}
- r = _generic_read(oldcid, o, 0, old_header->max_size, bl, &t);
+ r = _generic_read(*old_header, 0, old_header->max_size, bl, &t);
if (r < 0) {
r = -EINVAL;
goto out;
}
- r = _write(c, o, 0, bl.length(), bl, t);
+ r = _generic_write(*header, 0, bl.length(), bl, t);
if (r < 0) {
r = -EINVAL;
}
@@ -2410,7 +2438,7 @@ int KeyValueStore::_collection_move_rename(coll_t oldcid,
return r;
}
- t.rename_buffer(header, c, o);
+ t.rename_buffer(*header, c, o);
dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
<< oldoid << " = " << r << dendl;
@@ -2447,7 +2475,7 @@ int KeyValueStore::_collection_remove_recursive(const coll_t &cid,
}
}
- r = t.clear_buffer(header);
+ r = t.clear_buffer(*header);
dout(10) << __func__ << " " << cid << " r = " << r << dendl;
return 0;
@@ -2458,7 +2486,54 @@ int KeyValueStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
{
dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
<< dendl;
- return -EOPNOTSUPP;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(ncid),
+ &header, false);
+ if (r == 0) {
+ dout(2) << __func__ << ": " << ncid << " DNE" << dendl;
+ return -EEXIST;
+ }
+
+ r = t.lookup_cached_header(get_coll_for_coll(), make_ghobject_for_coll(cid),
+ &header, false);
+ if (r < 0) {
+ dout(2) << __func__ << ": " << cid << " DNE" << dendl;
+ return 0;
+ }
+
+ vector<ghobject_t> objects;
+ ghobject_t next, current;
+ int move_size = 0;
+ while (1) {
+ collection_list_partial(cid, current, get_ideal_list_min(),
+ get_ideal_list_max(), 0, &objects, &next);
+
+ dout(20) << __func__ << cid << "objects size: " << objects.size()
+ << dendl;
+
+ if (objects.empty())
+ break;
+
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end(); ++i) {
+ if (_collection_move_rename(cid, *i, ncid, *i, t) < 0) {
+ return -1;
+ }
+ move_size++;
+ }
+
+ objects.clear();
+ current = next;
+ }
+
+ t.rename_buffer(*header, get_coll_for_coll(), make_ghobject_for_coll(ncid));
+
+ dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
+ << dendl;
+ return 0;
}
int KeyValueStore::list_collections(vector<coll_t>& ls)
@@ -2483,8 +2558,8 @@ bool KeyValueStore::collection_exists(coll_t c)
dout(10) << __func__ << " " << dendl;
StripObjectMap::StripObjectHeader header;
-
- int r = _check_coll(c);
+ int r = backend->lookup_strip_header(get_coll_for_coll(),
+ make_ghobject_for_coll(c), header);
if (r < 0) {
return false;
}
@@ -2495,11 +2570,6 @@ bool KeyValueStore::collection_empty(coll_t c)
{
dout(10) << __func__ << " " << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return false;
- }
-
vector<ghobject_t> oids;
backend->list_objects(c, ghobject_t(), 1, &oids, 0);
@@ -2513,16 +2583,11 @@ int KeyValueStore::collection_list_range(coll_t c, ghobject_t start,
bool done = false;
ghobject_t next = start;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
while (!done) {
vector<ghobject_t> next_objects;
- r = collection_list_partial(c, next, get_ideal_list_min(),
- get_ideal_list_max(), seq,
- &next_objects, &next);
+ int r = collection_list_partial(c, next, get_ideal_list_min(),
+ get_ideal_list_max(), seq,
+ &next_objects, &next);
if (r < 0)
return r;
@@ -2580,22 +2645,41 @@ int KeyValueStore::collection_version_current(coll_t c, uint32_t *version)
// omap
int KeyValueStore::omap_get(coll_t c, const ghobject_t &hoid,
- bufferlist *header, map<string, bufferlist> *out)
+ bufferlist *bl, map<string, bufferlist> *out)
{
dout(15) << __func__ << " " << c << "/" << hoid << dendl;
- int r = _check_coll(c);
+ StripObjectMap::StripObjectHeader header;
+
+ int r = backend->lookup_strip_header(c, hoid, header);
if (r < 0) {
+ dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+ return r;
+ }
+
+
+ r = backend->get_with_header(header, OBJECT_OMAP, out);
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << __func__ << " err r =" << r << dendl;
return r;
}
- r = backend->get(c, hoid, OBJECT_OMAP, out);
+ set<string> keys;
+ map<string, bufferlist> got;
+
+ keys.insert(OBJECT_OMAP_HEADER_KEY);
+ r = backend->get_values_with_header(header, OBJECT_OMAP_HEADER, keys, &got);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " err r =" << r << dendl;
return r;
}
- return omap_get_header(c, hoid, header, false);
+ if (got.size()) {
+ assert(got.size() == 1);
+ bl->swap(got.begin()->second);
+ }
+
+ return 0;
}
int KeyValueStore::omap_get_header(coll_t c, const ghobject_t &hoid,
@@ -2603,23 +2687,17 @@ int KeyValueStore::omap_get_header(coll_t c, const ghobject_t &hoid,
{
dout(15) << __func__ << " " << c << "/" << hoid << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
set<string> keys;
map<string, bufferlist> got;
keys.insert(OBJECT_OMAP_HEADER_KEY);
- r = backend->get_values(c, hoid, OBJECT_OMAP_HEADER, keys, &got);
+ int r = backend->get_values(c, hoid, OBJECT_OMAP_HEADER, keys, &got);
if (r < 0 && r != -ENOENT) {
- assert(allow_eio || !m_fail_eio || r != -EIO);
dout(10) << __func__ << " err r =" << r << dendl;
return r;
}
- if (got.size()) {
+ if (!got.empty()) {
assert(got.size() == 1);
bl->swap(got.begin()->second);
}
@@ -2631,14 +2709,8 @@ int KeyValueStore::omap_get_keys(coll_t c, const ghobject_t &hoid, set<string> *
{
dout(15) << __func__ << " " << c << "/" << hoid << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
- r = backend->get_keys(c, hoid, OBJECT_OMAP, keys);
+ int r = backend->get_keys(c, hoid, OBJECT_OMAP, keys);
if (r < 0 && r != -ENOENT) {
- assert(!m_fail_eio || r != -EIO);
return r;
}
return 0;
@@ -2650,14 +2722,8 @@ int KeyValueStore::omap_get_values(coll_t c, const ghobject_t &hoid,
{
dout(15) << __func__ << " " << c << "/" << hoid << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
- r = backend->get_values(c, hoid, OBJECT_OMAP, keys, out);
+ int r = backend->get_values(c, hoid, OBJECT_OMAP, keys, out);
if (r < 0 && r != -ENOENT) {
- assert(!m_fail_eio || r != -EIO);
return r;
}
return 0;
@@ -2668,14 +2734,8 @@ int KeyValueStore::omap_check_keys(coll_t c, const ghobject_t &hoid,
{
dout(15) << __func__ << " " << c << "/" << hoid << dendl;
- int r = _check_coll(c);
- if (r < 0) {
- return r;
- }
-
- r = backend->check_keys(c, hoid, OBJECT_OMAP, keys, out);
+ int r = backend->check_keys(c, hoid, OBJECT_OMAP, keys, out);
if (r < 0 && r != -ENOENT) {
- assert(!m_fail_eio || r != -EIO);
return r;
}
return 0;
@@ -2703,14 +2763,13 @@ int KeyValueStore::_omap_clear(coll_t cid, const ghobject_t &hoid,
}
set<string> keys;
- r = backend->get_keys(cid, hoid, OBJECT_OMAP, &keys);
+ r = backend->get_keys_with_header(*header, OBJECT_OMAP, &keys);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not get omap_keys r = " << r << dendl;
- assert(!m_fail_eio || r != -EIO);
return r;
}
- r = t.remove_buffer_keys(OBJECT_OMAP, header, keys);
+ r = t.remove_buffer_keys(*header, OBJECT_OMAP, keys);
if (r < 0) {
dout(10) << __func__ << " could not remove keys r = " << r << dendl;
return r;
@@ -2718,13 +2777,13 @@ int KeyValueStore::_omap_clear(coll_t cid, const ghobject_t &hoid,
keys.clear();
keys.insert(OBJECT_OMAP_HEADER_KEY);
- r = t.remove_buffer_keys(OBJECT_OMAP_HEADER, header, keys);
+ r = t.remove_buffer_keys(*header, OBJECT_OMAP_HEADER, keys);
if (r < 0) {
dout(10) << __func__ << " could not remove keys r = " << r << dendl;
return r;
}
- t.clear_buffer_keys(OBJECT_OMAP_HEADER, header);
+ t.clear_buffer_keys(*header, OBJECT_OMAP_HEADER);
dout(10) << __func__ << " " << cid << "/" << hoid << " r = " << r << dendl;
return 0;
@@ -2745,7 +2804,7 @@ int KeyValueStore::_omap_setkeys(coll_t cid, const ghobject_t &hoid,
return r;
}
- t.set_buffer_keys(OBJECT_OMAP, header, aset);
+ t.set_buffer_keys(*header, OBJECT_OMAP, aset);
return 0;
}
@@ -2765,7 +2824,7 @@ int KeyValueStore::_omap_rmkeys(coll_t cid, const ghobject_t &hoid,
return r;
}
- r = t.remove_buffer_keys(OBJECT_OMAP, header, keys);
+ r = t.remove_buffer_keys(*header, OBJECT_OMAP, keys);
dout(10) << __func__ << " " << cid << "/" << hoid << " r = " << r << dendl;
return r;
@@ -2809,7 +2868,7 @@ int KeyValueStore::_omap_setheader(coll_t cid, const ghobject_t &hoid,
}
sets[OBJECT_OMAP_HEADER_KEY] = bl;
- t.set_buffer_keys(OBJECT_OMAP_HEADER, header, sets);
+ t.set_buffer_keys(*header, OBJECT_OMAP_HEADER, sets);
return 0;
}
@@ -2853,10 +2912,9 @@ int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
for (vector<ghobject_t>::iterator i = objects.begin();
i != objects.end(); ++i) {
if (i->match(bits, rem)) {
- if (_collection_add(dest, cid, *i, t) < 0) {
+ if (_collection_move_rename(cid, *i, dest, *i, t) < 0) {
return -1;
}
- _remove(cid, *i, t);
move_size++;
}
}
@@ -2909,19 +2967,8 @@ int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
const char** KeyValueStore::get_tracked_conf_keys() const
{
static const char* KEYS[] = {
- "filestore_min_sync_interval",
- "filestore_max_sync_interval",
- "filestore_queue_max_ops",
- "filestore_queue_max_bytes",
- "filestore_queue_committing_max_ops",
- "filestore_queue_committing_max_bytes",
- "filestore_commit_timeout",
- "filestore_dump_file",
- "filestore_kill_at",
- "filestore_fail_eio",
- "filestore_replica_fadvise",
- "filestore_sloppy_crc",
- "filestore_sloppy_crc_block_size",
+ "keyvaluestore_queue_max_ops",
+ "keyvaluestore_queue_max_bytes",
NULL
};
return KEYS;
@@ -2930,49 +2977,13 @@ const char** KeyValueStore::get_tracked_conf_keys() const
void KeyValueStore::handle_conf_change(const struct md_config_t *conf,
const std::set <std::string> &changed)
{
+ if (changed.count("keyvaluestore_queue_max_ops") ||
+ changed.count("keyvaluestore_queue_max_bytes")) {
+ m_keyvaluestore_queue_max_ops = conf->keyvaluestore_queue_max_ops;
+ m_keyvaluestore_queue_max_bytes = conf->keyvaluestore_queue_max_bytes;
+ }
}
void KeyValueStore::dump_transactions(list<ObjectStore::Transaction*>& ls, uint64_t seq, OpSequencer *osr)
{
}
-
-// ============== KeyValueStore Debug EIO Injection =================
-
-void KeyValueStore::inject_data_error(const ghobject_t &oid) {
- Mutex::Locker l(read_error_lock);
- dout(10) << __func__ << ": init error on " << oid << dendl;
- data_error_set.insert(oid);
-}
-
-void KeyValueStore::inject_mdata_error(const ghobject_t &oid) {
- Mutex::Locker l(read_error_lock);
- dout(10) << __func__ << ": init error on " << oid << dendl;
- mdata_error_set.insert(oid);
-}
-
-void KeyValueStore::debug_obj_on_delete(const ghobject_t &oid) {
- Mutex::Locker l(read_error_lock);
- dout(10) << __func__ << ": clear error on " << oid << dendl;
- data_error_set.erase(oid);
- mdata_error_set.erase(oid);
-}
-
-bool KeyValueStore::debug_data_eio(const ghobject_t &oid) {
- Mutex::Locker l(read_error_lock);
- if (data_error_set.count(oid)) {
- dout(10) << __func__ << ": inject error on " << oid << dendl;
- return true;
- } else {
- return false;
- }
-}
-
-bool KeyValueStore::debug_mdata_eio(const ghobject_t &oid) {
- Mutex::Locker l(read_error_lock);
- if (mdata_error_set.count(oid)) {
- dout(10) << __func__ << ": inject error on " << oid << dendl;
- return true;
- } else {
- return false;
- }
-}
diff --git a/src/os/KeyValueStore.h b/src/os/KeyValueStore.h
index 5a1d0f0..d7b9c0a 100644
--- a/src/os/KeyValueStore.h
+++ b/src/os/KeyValueStore.h
@@ -26,9 +26,6 @@
#include <fstream>
using namespace std;
-#include <ext/hash_map>
-using namespace __gnu_cxx;
-
#include "include/assert.h"
#include "ObjectStore.h"
@@ -100,7 +97,7 @@ class StripObjectMap: public GenericObjectMap {
};
bool check_spos(const StripObjectHeader &header,
- const SequencerPosition *spos);
+ const SequencerPosition &spos);
void sync_wrap(StripObjectHeader &strip_header, KeyValueDB::Transaction t,
const SequencerPosition &spos);
@@ -108,21 +105,38 @@ class StripObjectMap: public GenericObjectMap {
vector<StripExtent> &extents);
int lookup_strip_header(const coll_t & cid, const ghobject_t &oid,
StripObjectHeader &header);
- int save_strip_header(StripObjectHeader &header, KeyValueDB::Transaction t);
+ int save_strip_header(StripObjectHeader &header,
+ const SequencerPosition &spos,
+ KeyValueDB::Transaction t);
int create_strip_header(const coll_t &cid, const ghobject_t &oid,
StripObjectHeader &strip_header,
KeyValueDB::Transaction t);
void clone_wrap(StripObjectHeader &old_header,
const coll_t &cid, const ghobject_t &oid,
KeyValueDB::Transaction t,
- const SequencerPosition &spos,
StripObjectHeader *origin_header,
StripObjectHeader *target_header);
void rename_wrap(const coll_t &cid, const ghobject_t &oid,
KeyValueDB::Transaction t,
- const SequencerPosition &spos,
StripObjectHeader *header);
-
+ // Already hold header to avoid lock header seq again
+ int get_with_header(
+ const StripObjectHeader &header,
+ const string &prefix,
+ map<string, bufferlist> *out
+ );
+
+ int get_values_with_header(
+ const StripObjectHeader &header,
+ const string &prefix,
+ const set<string> &keys,
+ map<string, bufferlist> *out
+ );
+ int get_keys_with_header(
+ const StripObjectHeader &header,
+ const string &prefix,
+ set<string> *keys
+ );
StripObjectMap(KeyValueDB *db): GenericObjectMap(db) {}
@@ -206,25 +220,22 @@ class KeyValueStore : public ObjectStore,
SequencerPosition spos;
KeyValueDB::Transaction t;
- int check_coll(const coll_t &cid);
int lookup_cached_header(const coll_t &cid, const ghobject_t &oid,
StripObjectMap::StripObjectHeader **strip_header,
bool create_if_missing);
- int get_buffer_key(StripObjectMap::StripObjectHeader *strip_header,
- const string &prefix, const string &key,
- bufferlist &out);
- void set_buffer_keys(const string &prefix,
- StripObjectMap::StripObjectHeader *strip_header,
- map<string, bufferlist> &bl);
- int remove_buffer_keys(const string &prefix,
- StripObjectMap::StripObjectHeader *strip_header,
- const set<string> &keys);
- void clear_buffer_keys(const string &prefix,
- StripObjectMap::StripObjectHeader *strip_header);
- int clear_buffer(StripObjectMap::StripObjectHeader *strip_header);
- void clone_buffer(StripObjectMap::StripObjectHeader *old_header,
+ int get_buffer_keys(StripObjectMap::StripObjectHeader &strip_header,
+ const string &prefix, const set<string> &keys,
+ map<string, bufferlist> *out);
+ void set_buffer_keys(StripObjectMap::StripObjectHeader &strip_header,
+ const string &prefix, map<string, bufferlist> &bl);
+ int remove_buffer_keys(StripObjectMap::StripObjectHeader &strip_header,
+ const string &prefix, const set<string> &keys);
+ void clear_buffer_keys(StripObjectMap::StripObjectHeader &strip_header,
+ const string &prefix);
+ int clear_buffer(StripObjectMap::StripObjectHeader &strip_header);
+ void clone_buffer(StripObjectMap::StripObjectHeader &old_header,
const coll_t &cid, const ghobject_t &oid);
- void rename_buffer(StripObjectMap::StripObjectHeader *old_header,
+ void rename_buffer(StripObjectMap::StripObjectHeader &old_header,
const coll_t &cid, const ghobject_t &oid);
int submit_transaction();
@@ -305,6 +316,8 @@ class KeyValueStore : public ObjectStore,
Sequencer default_osr;
deque<OpSequencer*> op_queue;
uint64_t op_queue_len, op_queue_bytes;
+ Cond op_throttle_cond;
+ Mutex op_throttle_lock;
Finisher op_finisher;
ThreadPool op_tp;
@@ -344,11 +357,13 @@ class KeyValueStore : public ObjectStore,
}
} op_wq;
- void _do_op(OpSequencer *osr, ThreadPool::TPHandle &handle);
- void _finish_op(OpSequencer *osr);
Op *build_op(list<Transaction*>& tls, Context *ondisk, Context *onreadable,
Context *onreadable_sync, TrackedOpRef osd_op);
void queue_op(OpSequencer *osr, Op *o);
+ void op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle = NULL);
+ void _do_op(OpSequencer *osr, ThreadPool::TPHandle &handle);
+ void op_queue_release_throttle(Op *o);
+ void _finish_op(OpSequencer *osr);
PerfCounters *logger;
@@ -413,13 +428,13 @@ class KeyValueStore : public ObjectStore,
// ------------------
// objects
- // Read operation need call "check_coll", checking "coll_t" in write
- // operation is done by lookup_cached_header
- int _check_coll(const coll_t &cid);
+ int _generic_read(StripObjectMap::StripObjectHeader &header,
+ uint64_t offset, size_t len, bufferlist& bl,
+ bool allow_eio = false, BufferTransaction *bt = 0);
+ int _generic_write(StripObjectMap::StripObjectHeader &header,
+ uint64_t offset, size_t len, const bufferlist& bl,
+ BufferTransaction &t, bool replica = false);
- int _generic_read(coll_t cid, const ghobject_t& oid, uint64_t offset,
- size_t len, bufferlist& bl, bool allow_eio = false,
- BufferTransaction *bt = 0);
bool exists(coll_t cid, const ghobject_t& oid);
int stat(coll_t cid, const ghobject_t& oid, struct stat *st,
bool allow_eio = false);
@@ -451,16 +466,6 @@ class KeyValueStore : public ObjectStore,
void set_fsid(uuid_d u) { fsid = u; }
uuid_d get_fsid() { return fsid; }
- // DEBUG read error injection, an object is removed from both on delete()
- Mutex read_error_lock;
- set<ghobject_t> data_error_set; // read() will return -EIO
- set<ghobject_t> mdata_error_set; // getattr(),stat() will return -EIO
- void inject_data_error(const ghobject_t &oid);
- void inject_mdata_error(const ghobject_t &oid);
- void debug_obj_on_delete(const ghobject_t &oid);
- bool debug_data_eio(const ghobject_t &oid);
- bool debug_mdata_eio(const ghobject_t &oid);
-
// attrs
int getattr(coll_t cid, const ghobject_t& oid, const char *name,
bufferptr &bp);
@@ -553,8 +558,8 @@ class KeyValueStore : public ObjectStore,
const std::set <std::string> &changed);
std::string m_osd_rollback_to_cluster_snap;
- bool m_osd_use_stale_snap;
- bool m_fail_eio;
+ int m_keyvaluestore_queue_max_ops;
+ int m_keyvaluestore_queue_max_bytes;
int do_update;
diff --git a/src/os/LFNIndex.cc b/src/os/LFNIndex.cc
index a250016..0310cb5 100644
--- a/src/os/LFNIndex.cc
+++ b/src/os/LFNIndex.cc
@@ -661,8 +661,8 @@ string LFNIndex::lfn_generate_object_name(const ghobject_t &oid)
t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.pool);
full_name += string(buf);
- if (oid.generation != ghobject_t::NO_GEN) {
- assert(oid.shard_id != ghobject_t::NO_SHARD);
+ if (oid.generation != ghobject_t::NO_GEN ||
+ oid.shard_id != ghobject_t::NO_SHARD) {
full_name.append("_");
t = buf;
@@ -950,9 +950,9 @@ bool LFNIndex::lfn_parse_object_name_keyless(const string &long_name, ghobject_t
{
bool r = parse_object(long_name.c_str(), *out);
int64_t pool = -1;
- pg_t pg;
+ spg_t pg;
if (coll().is_pg_prefix(pg))
- pool = (int64_t)pg.pool();
+ pool = (int64_t)pg.pgid.pool();
out->hobj.pool = pool;
if (!r) return r;
string temp = lfn_generate_object_name(*out);
@@ -1043,9 +1043,9 @@ bool LFNIndex::lfn_parse_object_name_poolless(const string &long_name,
int64_t pool = -1;
- pg_t pg;
+ spg_t pg;
if (coll().is_pg_prefix(pg))
- pool = (int64_t)pg.pool();
+ pool = (int64_t)pg.pgid.pool();
(*out) = ghobject_t(hobject_t(name, key, snap, hash, pool, ""));
return true;
}
diff --git a/src/os/Makefile.am b/src/os/Makefile.am
index 2bba7aa..252c678 100644
--- a/src/os/Makefile.am
+++ b/src/os/Makefile.am
@@ -21,6 +21,10 @@ if LINUX
libos_la_SOURCES += os/BtrfsFileStoreBackend.cc
endif
+if WITH_LIBXFS
+libos_la_SOURCES += os/XfsFileStoreBackend.cc
+endif
+
if WITH_LIBZFS
libos_la_SOURCES += os/ZFSFileStoreBackend.cc
endif
@@ -52,6 +56,7 @@ noinst_HEADERS += \
os/ObjectStore.h \
os/SequencerPosition.h \
os/WBThrottle.h \
+ os/XfsFileStoreBackend.h \
os/ZFSFileStoreBackend.h
if WITH_LIBZFS
diff --git a/src/os/MemStore.cc b/src/os/MemStore.cc
index c736187..9e75b76 100644
--- a/src/os/MemStore.cc
+++ b/src/os/MemStore.cc
@@ -949,6 +949,10 @@ void MemStore::_do_transaction(Transaction& t)
}
break;
+ case Transaction::OP_SETALLOCHINT:
+ // nop
+ break;
+
default:
derr << "bad op " << op << dendl;
assert(0);
diff --git a/src/os/ObjectStore.cc b/src/os/ObjectStore.cc
index 7496c45..5b58c6f 100644
--- a/src/os/ObjectStore.cc
+++ b/src/os/ObjectStore.cc
@@ -323,8 +323,8 @@ void ObjectStore::Transaction::dump(ceph::Formatter *f)
case Transaction::OP_COLL_ADD:
{
- coll_t ocid = i.get_cid();
coll_t ncid = i.get_cid();
+ coll_t ocid = i.get_cid();
ghobject_t oid = i.get_oid();
f->dump_string("op_name", "collection_add");
f->dump_stream("src_collection") << ocid;
@@ -490,6 +490,34 @@ void ObjectStore::Transaction::dump(ceph::Formatter *f)
}
break;
+ case Transaction::OP_COLL_MOVE_RENAME:
+ {
+ coll_t old_cid(i.get_cid());
+ ghobject_t old_oid = i.get_oid();
+ coll_t new_cid(i.get_cid());
+ ghobject_t new_oid = i.get_oid();
+ f->dump_string("op_name", "op_coll_move_rename");
+ f->dump_stream("old_collection") << old_cid;
+ f->dump_stream("old_oid") << old_oid;
+ f->dump_stream("new_collection") << new_cid;
+ f->dump_stream("new_oid") << new_oid;
+ }
+ break;
+
+ case Transaction::OP_SETALLOCHINT:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t expected_object_size = i.get_length();
+ uint64_t expected_write_size = i.get_length();
+ f->dump_string("op_name", "op_setallochint");
+ f->dump_stream("collection") << cid;
+ f->dump_stream("oid") << oid;
+ f->dump_stream("expected_object_size") << expected_object_size;
+ f->dump_stream("expected_write_size") << expected_write_size;
+ }
+ break;
+
default:
f->dump_string("op_name", "unknown");
f->dump_unsigned("op_code", op);
diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index ef17db4..c19ebd0 100644
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -41,6 +41,34 @@ namespace ceph {
class Formatter;
}
+enum {
+ l_os_first = 84000,
+ l_os_jq_max_ops,
+ l_os_jq_ops,
+ l_os_j_ops,
+ l_os_jq_max_bytes,
+ l_os_jq_bytes,
+ l_os_j_bytes,
+ l_os_j_lat,
+ l_os_j_wr,
+ l_os_j_wr_bytes,
+ l_os_j_full,
+ l_os_committing,
+ l_os_commit,
+ l_os_commit_len,
+ l_os_commit_lat,
+ l_os_oq_max_ops,
+ l_os_oq_ops,
+ l_os_ops,
+ l_os_oq_max_bytes,
+ l_os_oq_bytes,
+ l_os_bytes,
+ l_os_apply_lat,
+ l_os_queue_lat,
+ l_os_last,
+};
+
+
/*
* low-level interface to the local OSD file system
*/
@@ -150,6 +178,8 @@ public:
doesn't create the destination */
OP_OMAP_RMKEYRANGE = 37, // cid, oid, firstkey, lastkey
OP_COLL_MOVE_RENAME = 38, // oldcid, oldoid, newcid, newoid
+
+ OP_SETALLOCHINT = 39, // cid, oid, object_size, write_size
};
private:
@@ -468,7 +498,7 @@ public:
::encode(attrset, tbl);
ops++;
}
- void setattrs(coll_t cid, const hobject_t& oid, map<string,bufferlist>& attrset) {
+ void setattrs(coll_t cid, const ghobject_t& oid, map<string,bufferlist>& attrset) {
__u32 op = OP_SETATTRS;
::encode(op, tbl);
::encode(cid, tbl);
@@ -687,6 +717,21 @@ public:
++ops;
}
+ void set_alloc_hint(
+ coll_t cid,
+ const ghobject_t &oid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size
+ ) {
+ __u32 op = OP_SETALLOCHINT;
+ ::encode(op, tbl);
+ ::encode(cid, tbl);
+ ::encode(oid, tbl);
+ ::encode(expected_object_size, tbl);
+ ::encode(expected_write_size, tbl);
+ ++ops;
+ }
+
// etc.
Transaction() :
ops(0), pad_unused_bytes(0), largest_data_len(0), largest_data_off(0), largest_data_off_in_tbl(0),
@@ -946,6 +991,14 @@ public:
value.push_back(bp);
return r;
}
+ int getattr(
+ coll_t cid, const ghobject_t& oid,
+ const string name, bufferlist& value) {
+ bufferptr bp;
+ int r = getattr(cid, oid, name.c_str(), bp);
+ value.push_back(bp);
+ return r;
+ }
virtual int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>& aset, bool user_only = false) = 0;
int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferlist>& aset, bool user_only = false) {
map<string,bufferptr> bmap;
diff --git a/src/os/XfsFileStoreBackend.cc b/src/os/XfsFileStoreBackend.cc
new file mode 100644
index 0000000..7b632d8
--- /dev/null
+++ b/src/os/XfsFileStoreBackend.cc
@@ -0,0 +1,124 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "XfsFileStoreBackend.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <xfs/xfs.h>
+
+#include "common/errno.h"
+#include "include/assert.h"
+#include "include/compat.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "xfsfilestorebackend(" << get_basedir_path() << ") "
+
+XfsFileStoreBackend::XfsFileStoreBackend(FileStore *fs):
+ GenericFileStoreBackend(fs), m_has_extsize(false) { }
+
+/*
+ * Set extsize attr on a file to val. Should be a free-standing
+ * function, but dout_prefix expanding to a call to get_basedir_path()
+ * protected member function won't let it.
+ */
+int XfsFileStoreBackend::set_extsize(int fd, unsigned int val)
+{
+ struct fsxattr fsx;
+ struct stat sb;
+ int ret;
+
+ if (fstat(fd, &sb) < 0) {
+ ret = -errno;
+ dout(0) << "set_extsize: fstat: " << cpp_strerror(ret) << dendl;
+ goto out;
+ }
+ if (!S_ISREG(sb.st_mode)) {
+ ret = -EINVAL;
+ dout(0) << "set_extsize: invalid target file type" << dendl;
+ goto out;
+ }
+
+ if (ioctl(fd, XFS_IOC_FSGETXATTR, &fsx) < 0) {
+ ret = -errno;
+ dout(0) << "set_extsize: FSGETXATTR: " << cpp_strerror(ret) << dendl;
+ goto out;
+ }
+
+ fsx.fsx_xflags |= XFS_XFLAG_EXTSIZE;
+ fsx.fsx_extsize = val;
+
+ if (ioctl(fd, XFS_IOC_FSSETXATTR, &fsx) < 0) {
+ ret = -errno;
+ dout(0) << "set_extsize: FSSETXATTR: " << cpp_strerror(ret) << dendl;
+ goto out;
+ }
+ ret = 0;
+
+out:
+ return ret;
+}
+
+int XfsFileStoreBackend::detect_features()
+{
+ int ret;
+
+ ret = GenericFileStoreBackend::detect_features();
+ if (ret < 0)
+ return ret;
+
+ // extsize?
+ int fd = ::openat(get_basedir_fd(), "extsize_test", O_CREAT|O_WRONLY, 0600);
+ if (fd < 0) {
+ ret = -errno;
+ dout(0) << "detect_feature: failed to create test file for extsize attr: "
+ << cpp_strerror(ret) << dendl;
+ goto out;
+ }
+ if (::unlinkat(get_basedir_fd(), "extsize_test", 0) < 0) {
+ ret = -errno;
+ dout(0) << "detect_feature: failed to unlink test file for extsize attr: "
+ << cpp_strerror(ret) << dendl;
+ goto out_close;
+ }
+
+ ret = set_extsize(fd, 1U << 15); // a few pages
+ if (ret) {
+ ret = 0;
+ dout(0) << "detect_feature: failed to set test file extsize, assuming extsize is NOT supported" << dendl;
+ goto out_close;
+ }
+
+ dout(0) << "detect_feature: extsize is supported" << dendl;
+ m_has_extsize = true;
+
+out_close:
+ TEMP_FAILURE_RETRY(::close(fd));
+out:
+ return ret;
+}
+
+int XfsFileStoreBackend::set_alloc_hint(int fd, uint64_t hint)
+{
+ if (!m_has_extsize)
+ return -EOPNOTSUPP;
+
+ assert(hint < UINT_MAX);
+ return set_extsize(fd, hint);
+}
diff --git a/src/os/XfsFileStoreBackend.h b/src/os/XfsFileStoreBackend.h
new file mode 100644
index 0000000..cb19bf7
--- /dev/null
+++ b/src/os/XfsFileStoreBackend.h
@@ -0,0 +1,33 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_XFSFILESTOREBACKEND_H
+#define CEPH_XFSFILESTOREBACKEND_H
+
+#include "GenericFileStoreBackend.h"
+
+#include "include/int_types.h"
+
+class XfsFileStoreBackend : public GenericFileStoreBackend {
+private:
+ bool m_has_extsize;
+ int set_extsize(int fd, unsigned int val);
+public:
+ XfsFileStoreBackend(FileStore *fs);
+ ~XfsFileStoreBackend() {};
+ int detect_features();
+ int set_alloc_hint(int fd, uint64_t hint);
+};
+
+#endif /* CEPH_XFSFILESTOREBACKEND_H */
diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
new file mode 100644
index 0000000..5738ab3
--- /dev/null
+++ b/src/osd/ECBackend.cc
@@ -0,0 +1,1765 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <boost/variant.hpp>
+#include <boost/optional.hpp>
+#include <iostream>
+#include <sstream>
+
+#include "ECUtil.h"
+#include "ECBackend.h"
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPushReply.h"
+
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
+ return *_dout << pgb->get_parent()->gen_dbg_prefix();
+}
+
+struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
+ list<ECBackend::RecoveryOp> ops;
+};
+
+static ostream &operator<<(ostream &lhs, const map<pg_shard_t, bufferlist> &rhs)
+{
+ lhs << "[";
+ for (map<pg_shard_t, bufferlist>::const_iterator i = rhs.begin();
+ i != rhs.end();
+ ++i) {
+ if (i != rhs.begin())
+ lhs << ", ";
+ lhs << make_pair(i->first, i->second.length());
+ }
+ return lhs << "]";
+}
+
+static ostream &operator<<(ostream &lhs, const map<int, bufferlist> &rhs)
+{
+ lhs << "[";
+ for (map<int, bufferlist>::const_iterator i = rhs.begin();
+ i != rhs.end();
+ ++i) {
+ if (i != rhs.begin())
+ lhs << ", ";
+ lhs << make_pair(i->first, i->second.length());
+ }
+ return lhs << "]";
+}
+
+static ostream &operator<<(
+ ostream &lhs,
+ const boost::tuple<uint64_t, uint64_t, map<pg_shard_t, bufferlist> > &rhs)
+{
+ return lhs << "(" << rhs.get<0>() << ", "
+ << rhs.get<1>() << ", " << rhs.get<2>() << ")";
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::read_request_t &rhs)
+{
+ return lhs << "read_request_t(to_read=[" << rhs.to_read << "]"
+ << ", need=" << rhs.need
+ << ", want_attrs=" << rhs.want_attrs
+ << ")";
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::read_result_t &rhs)
+{
+ lhs << "read_result_t(r=" << rhs.r
+ << ", errors=" << rhs.errors;
+ if (rhs.attrs) {
+ lhs << ", attrs=" << rhs.attrs;
+ } else {
+ lhs << ", noattrs";
+ }
+ return lhs << ", returned=" << rhs.returned;
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::ReadOp &rhs)
+{
+ lhs << "ReadOp(tid=" << rhs.tid;
+ if (rhs.op && rhs.op->get_req()) {
+ lhs << ", op=";
+ rhs.op->get_req()->print(lhs);
+ }
+ return lhs << ", to_read=" << rhs.to_read
+ << ", complete=" << rhs.complete
+ << ", priority=" << rhs.priority
+ << ", obj_to_source=" << rhs.obj_to_source
+ << ", source_to_obj=" << rhs.source_to_obj
+ << ", in_progress=" << rhs.in_progress << ")";
+}
+
+void ECBackend::ReadOp::dump(Formatter *f) const
+{
+ f->dump_stream("tid") << tid;
+ if (op && op->get_req()) {
+ f->dump_stream("op") << *(op->get_req());
+ }
+ f->dump_stream("to_read") << to_read;
+ f->dump_stream("complete") << complete;
+ f->dump_stream("priority") << priority;
+ f->dump_stream("obj_to_source") << obj_to_source;
+ f->dump_stream("source_to_obj") << source_to_obj;
+ f->dump_stream("in_progress") << in_progress;
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::Op &rhs)
+{
+ lhs << "Op(" << rhs.hoid
+ << " v=" << rhs.version
+ << " tt=" << rhs.trim_to
+ << " tid=" << rhs.tid
+ << " reqid=" << rhs.reqid;
+ if (rhs.client_op && rhs.client_op->get_req()) {
+ lhs << " client_op=";
+ rhs.client_op->get_req()->print(lhs);
+ }
+ lhs << " pending_commit=" << rhs.pending_commit
+ << " pending_apply=" << rhs.pending_apply
+ << ")";
+ return lhs;
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::RecoveryOp &rhs)
+{
+ return lhs << "RecoveryOp("
+ << "hoid=" << rhs.hoid
+ << " v=" << rhs.v
+ << " missing_on=" << rhs.missing_on
+ << " missing_on_shards=" << rhs.missing_on_shards
+ << " recovery_info=" << rhs.recovery_info
+ << " recovery_progress=" << rhs.recovery_progress
+ << " pending_read=" << rhs.pending_read
+ << " obc refcount=" << rhs.obc.use_count()
+ << " state=" << ECBackend::RecoveryOp::tostr(rhs.state)
+ << " waiting_on_pushes=" << rhs.waiting_on_pushes
+ << " extent_requested=" << rhs.extent_requested;
+}
+
+void ECBackend::RecoveryOp::dump(Formatter *f) const
+{
+ f->dump_stream("hoid") << hoid;
+ f->dump_stream("v") << v;
+ f->dump_stream("missing_on") << missing_on;
+ f->dump_stream("missing_on_shards") << missing_on_shards;
+ f->dump_stream("recovery_info") << recovery_info;
+ f->dump_stream("recovery_progress") << recovery_progress;
+ f->dump_stream("pending_read") << pending_read;
+ f->dump_stream("state") << tostr(state);
+ f->dump_stream("waiting_on_pushes") << waiting_on_pushes;
+ f->dump_stream("extent_requested") << extent_requested;
+}
+
+ECBackend::ECBackend(
+ PGBackend::Listener *pg,
+ coll_t coll,
+ coll_t temp_coll,
+ ObjectStore *store,
+ CephContext *cct,
+ ErasureCodeInterfaceRef ec_impl,
+ uint64_t stripe_width)
+ : PGBackend(pg, store, coll, temp_coll),
+ cct(cct),
+ ec_impl(ec_impl),
+ sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
+ assert((ec_impl->get_data_chunk_count() *
+ ec_impl->get_chunk_size(stripe_width)) == stripe_width);
+}
+
+PGBackend::RecoveryHandle *ECBackend::open_recovery_op()
+{
+ return new ECRecoveryHandle;
+}
+
+struct OnRecoveryReadComplete :
+ public GenContext<pair<RecoveryMessages*, ECBackend::read_result_t& > &> {
+ ECBackend *pg;
+ hobject_t hoid;
+ set<int> want;
+ OnRecoveryReadComplete(ECBackend *pg, const hobject_t &hoid)
+ : pg(pg), hoid(hoid) {}
+ void finish(pair<RecoveryMessages *, ECBackend::read_result_t &> &in) {
+ ECBackend::read_result_t &res = in.second;
+ assert(res.r == 0);
+ assert(res.errors.empty());
+ assert(res.returned.size() == 1);
+ pg->handle_recovery_read_complete(
+ hoid,
+ res.returned.back(),
+ res.attrs,
+ in.first);
+ }
+};
+
+struct RecoveryMessages {
+ map<hobject_t,
+ ECBackend::read_request_t> reads;
+ void read(
+ ECBackend *ec,
+ const hobject_t &hoid, uint64_t off, uint64_t len,
+ const set<pg_shard_t> &need,
+ bool attrs) {
+ list<pair<uint64_t, uint64_t> > to_read;
+ to_read.push_back(make_pair(off, len));
+ assert(!reads.count(hoid));
+ reads.insert(
+ make_pair(
+ hoid,
+ ECBackend::read_request_t(
+ hoid,
+ to_read,
+ need,
+ attrs,
+ new OnRecoveryReadComplete(
+ ec,
+ hoid))));
+ }
+
+ map<pg_shard_t, vector<PushOp> > pushes;
+ map<pg_shard_t, vector<PushReplyOp> > push_replies;
+ ObjectStore::Transaction *t;
+ RecoveryMessages() : t(new ObjectStore::Transaction) {}
+ ~RecoveryMessages() { assert(!t); }
+};
+
+void ECBackend::handle_recovery_push(
+ PushOp &op,
+ RecoveryMessages *m)
+{
+ bool oneshot = op.before_progress.first && op.after_progress.data_complete;
+ coll_t tcoll = oneshot ? coll : get_temp_coll(m->t);
+ if (op.before_progress.first) {
+ get_parent()->on_local_recover_start(
+ op.soid,
+ m->t);
+ m->t->remove(
+ get_temp_coll(m->t),
+ ghobject_t(
+ op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+ m->t->touch(
+ tcoll,
+ ghobject_t(
+ op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+ }
+
+ if (!op.data_included.empty()) {
+ uint64_t start = op.data_included.range_start();
+ uint64_t end = op.data_included.range_end();
+ assert(op.data.length() == (end - start));
+
+ m->t->write(
+ tcoll,
+ ghobject_t(
+ op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ start,
+ op.data.length(),
+ op.data);
+ } else {
+ assert(op.data.length() == 0);
+ }
+
+ if (op.before_progress.first) {
+ if (!oneshot)
+ add_temp_obj(op.soid);
+ assert(op.attrset.count(string("_")));
+ m->t->setattrs(
+ tcoll,
+ ghobject_t(
+ op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ op.attrset);
+ }
+
+ if (op.after_progress.data_complete && !oneshot) {
+ clear_temp_obj(op.soid);
+ m->t->collection_move(
+ coll,
+ tcoll,
+ ghobject_t(
+ op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+ }
+ if (op.after_progress.data_complete) {
+ if ((get_parent()->pgb_is_primary())) {
+ assert(recovery_ops.count(op.soid));
+ assert(recovery_ops[op.soid].obc);
+ object_stat_sum_t stats;
+ stats.num_objects_recovered = 1;
+ stats.num_bytes_recovered = recovery_ops[op.soid].obc->obs.oi.size;
+ get_parent()->on_local_recover(
+ op.soid,
+ stats,
+ op.recovery_info,
+ recovery_ops[op.soid].obc,
+ m->t);
+ } else {
+ get_parent()->on_local_recover(
+ op.soid,
+ object_stat_sum_t(),
+ op.recovery_info,
+ ObjectContextRef(),
+ m->t);
+ }
+ }
+ m->push_replies[get_parent()->primary_shard()].push_back(PushReplyOp());
+ m->push_replies[get_parent()->primary_shard()].back().soid = op.soid;
+}
+
+void ECBackend::handle_recovery_push_reply(
+ PushReplyOp &op,
+ pg_shard_t from,
+ RecoveryMessages *m)
+{
+ if (!recovery_ops.count(op.soid))
+ return;
+ RecoveryOp &rop = recovery_ops[op.soid];
+ assert(rop.waiting_on_pushes.count(from));
+ rop.waiting_on_pushes.erase(from);
+ continue_recovery_op(rop, m);
+}
+
+void ECBackend::handle_recovery_read_complete(
+ const hobject_t &hoid,
+ boost::tuple<uint64_t, uint64_t, map<pg_shard_t, bufferlist> > &to_read,
+ boost::optional<map<string, bufferlist> > attrs,
+ RecoveryMessages *m)
+{
+ dout(10) << __func__ << ": returned " << hoid << " "
+ << "(" << to_read.get<0>()
+ << ", " << to_read.get<1>()
+ << ", " << to_read.get<2>()
+ << ")"
+ << dendl;
+ assert(recovery_ops.count(hoid));
+ RecoveryOp &op = recovery_ops[hoid];
+ assert(op.returned_data.empty());
+ map<int, bufferlist*> target;
+ for (set<shard_id_t>::iterator i = op.missing_on_shards.begin();
+ i != op.missing_on_shards.end();
+ ++i) {
+ target[*i] = &(op.returned_data[*i]);
+ }
+ map<int, bufferlist> from;
+ for(map<pg_shard_t, bufferlist>::iterator i = to_read.get<2>().begin();
+ i != to_read.get<2>().end();
+ ++i) {
+ from[i->first.shard].claim(i->second);
+ }
+ dout(10) << __func__ << ": " << from << dendl;
+ ECUtil::decode(sinfo, ec_impl, from, target);
+ if (attrs) {
+ op.xattrs.swap(*attrs);
+
+ if (!op.obc) {
+ op.obc = get_parent()->get_obc(hoid, op.xattrs);
+ op.recovery_info.size = op.obc->obs.oi.size;
+ op.recovery_info.oi = op.obc->obs.oi;
+ }
+
+ ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
+ if (op.obc->obs.oi.size > 0) {
+ assert(op.xattrs.count(ECUtil::get_hinfo_key()));
+ bufferlist::iterator bp = op.xattrs[ECUtil::get_hinfo_key()].begin();
+ ::decode(hinfo, bp);
+ }
+ op.hinfo = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
+ }
+ assert(op.xattrs.size());
+ assert(op.obc);
+ continue_recovery_op(op, m);
+}
+
+struct SendPushReplies : public Context {
+ PGBackend::Listener *l;
+ epoch_t epoch;
+ map<int, MOSDPGPushReply*> replies;
+ SendPushReplies(
+ PGBackend::Listener *l,
+ epoch_t epoch,
+ map<int, MOSDPGPushReply*> &in) : l(l), epoch(epoch) {
+ replies.swap(in);
+ }
+ void finish(int) {
+ for (map<int, MOSDPGPushReply*>::iterator i = replies.begin();
+ i != replies.end();
+ ++i) {
+ l->send_message_osd_cluster(i->first, i->second, epoch);
+ }
+ replies.clear();
+ }
+ ~SendPushReplies() {
+ for (map<int, MOSDPGPushReply*>::iterator i = replies.begin();
+ i != replies.end();
+ ++i) {
+ i->second->put();
+ }
+ replies.clear();
+ }
+};
+
+void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
+{
+ for (map<pg_shard_t, vector<PushOp> >::iterator i = m.pushes.begin();
+ i != m.pushes.end();
+ m.pushes.erase(i++)) {
+ MOSDPGPush *msg = new MOSDPGPush();
+ msg->set_priority(priority);
+ msg->map_epoch = get_parent()->get_epoch();
+ msg->from = get_parent()->whoami_shard();
+ msg->pgid = spg_t(get_parent()->get_info().pgid.pgid, i->first.shard);
+ msg->pushes.swap(i->second);
+ msg->compute_cost(cct);
+ get_parent()->send_message(
+ i->first.osd,
+ msg);
+ }
+ map<int, MOSDPGPushReply*> replies;
+ for (map<pg_shard_t, vector<PushReplyOp> >::iterator i =
+ m.push_replies.begin();
+ i != m.push_replies.end();
+ m.push_replies.erase(i++)) {
+ MOSDPGPushReply *msg = new MOSDPGPushReply();
+ msg->set_priority(priority);
+ msg->map_epoch = get_parent()->get_epoch();
+ msg->from = get_parent()->whoami_shard();
+ msg->pgid = spg_t(get_parent()->get_info().pgid.pgid, i->first.shard);
+ msg->replies.swap(i->second);
+ msg->compute_cost(cct);
+ replies.insert(make_pair(i->first.osd, msg));
+ }
+ m.t->register_on_complete(
+ get_parent()->bless_context(
+ new SendPushReplies(
+ get_parent(),
+ get_parent()->get_epoch(),
+ replies)));
+ m.t->register_on_applied(
+ new ObjectStore::C_DeleteTransaction(m.t));
+ get_parent()->queue_transaction(m.t);
+ m.t = NULL;
+ if (m.reads.empty())
+ return;
+ start_read_op(
+ priority,
+ m.reads,
+ OpRequestRef());
+}
+
+void ECBackend::continue_recovery_op(
+ RecoveryOp &op,
+ RecoveryMessages *m)
+{
+ dout(10) << __func__ << ": continuing " << op << dendl;
+ while (1) {
+ switch (op.state) {
+ case RecoveryOp::IDLE: {
+ // start read
+ op.state = RecoveryOp::READING;
+ assert(!op.recovery_progress.data_complete);
+ set<int> want(op.missing_on_shards.begin(), op.missing_on_shards.end());
+ set<pg_shard_t> to_read;
+ int r = get_min_avail_to_read_shards(
+ op.hoid, want, true, &to_read);
+ assert(r == 0);
+ m->read(
+ this,
+ op.hoid,
+ op.recovery_progress.data_recovered_to,
+ get_recovery_chunk_size(),
+ to_read,
+ op.recovery_progress.first);
+ op.extent_requested = make_pair(op.recovery_progress.data_recovered_to,
+ get_recovery_chunk_size());
+ dout(10) << __func__ << ": IDLE return " << op << dendl;
+ return;
+ }
+ case RecoveryOp::READING: {
+ // read completed, start write
+ assert(op.xattrs.size());
+ assert(op.returned_data.size());
+ op.state = RecoveryOp::WRITING;
+ ObjectRecoveryProgress after_progress = op.recovery_progress;
+ after_progress.data_recovered_to += get_recovery_chunk_size();
+ after_progress.first = false;
+ if (after_progress.data_recovered_to >= op.obc->obs.oi.size) {
+ after_progress.data_recovered_to =
+ sinfo.logical_to_next_stripe_offset(
+ op.obc->obs.oi.size);
+ after_progress.data_complete = true;
+ }
+ for (set<pg_shard_t>::iterator mi = op.missing_on.begin();
+ mi != op.missing_on.end();
+ ++mi) {
+ assert(op.returned_data.count(mi->shard));
+ m->pushes[*mi].push_back(PushOp());
+ PushOp &pop = m->pushes[*mi].back();
+ pop.soid = op.hoid;
+ pop.version = op.v;
+ pop.data = op.returned_data[mi->shard];
+ dout(10) << __func__ << ": before_progress=" << op.recovery_progress
+ << ", after_progress=" << after_progress
+ << ", pop.data.length()=" << pop.data.length()
+ << ", size=" << op.obc->obs.oi.size << dendl;
+ assert(
+ pop.data.length() ==
+ sinfo.aligned_logical_offset_to_chunk_offset(
+ after_progress.data_recovered_to -
+ op.recovery_progress.data_recovered_to)
+ );
+ if (pop.data.length())
+ pop.data_included.insert(
+ sinfo.aligned_logical_offset_to_chunk_offset(
+ op.recovery_progress.data_recovered_to),
+ pop.data.length()
+ );
+ if (op.recovery_progress.first) {
+ pop.attrset = op.xattrs;
+ }
+ pop.recovery_info = op.recovery_info;
+ pop.before_progress = op.recovery_progress;
+ pop.after_progress = after_progress;
+ if (*mi != get_parent()->primary_shard())
+ get_parent()->begin_peer_recover(
+ *mi,
+ op.hoid);
+ }
+ op.returned_data.clear();
+ op.waiting_on_pushes = op.missing_on;
+ op.recovery_progress = after_progress;
+ dout(10) << __func__ << ": READING return " << op << dendl;
+ return;
+ }
+ case RecoveryOp::WRITING: {
+ if (op.waiting_on_pushes.empty()) {
+ if (op.recovery_progress.data_complete) {
+ op.state = RecoveryOp::COMPLETE;
+ for (set<pg_shard_t>::iterator i = op.missing_on.begin();
+ i != op.missing_on.end();
+ ++i) {
+ if (*i != get_parent()->primary_shard()) {
+ dout(10) << __func__ << ": on_peer_recover on " << *i
+ << ", obj " << op.hoid << dendl;
+ get_parent()->on_peer_recover(
+ *i,
+ op.hoid,
+ op.recovery_info,
+ object_stat_sum_t());
+ }
+ }
+ get_parent()->on_global_recover(op.hoid);
+ dout(10) << __func__ << ": WRITING return " << op << dendl;
+ recovery_ops.erase(op.hoid);
+ return;
+ } else {
+ op.state = RecoveryOp::IDLE;
+ dout(10) << __func__ << ": WRITING continue " << op << dendl;
+ continue;
+ }
+ }
+ return;
+ }
+ case RecoveryOp::COMPLETE: {
+ assert(0); // should never be called once complete
+ };
+ default:
+ assert(0);
+ }
+ }
+}
+
+void ECBackend::run_recovery_op(
+ RecoveryHandle *_h,
+ int priority)
+{
+ ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
+ RecoveryMessages m;
+ for (list<RecoveryOp>::iterator i = h->ops.begin();
+ i != h->ops.end();
+ ++i) {
+ dout(10) << __func__ << ": starting " << *i << dendl;
+ assert(!recovery_ops.count(i->hoid));
+ RecoveryOp &op = recovery_ops.insert(make_pair(i->hoid, *i)).first->second;
+ continue_recovery_op(op, &m);
+ }
+ dispatch_recovery_messages(m, priority);
+ delete _h;
+}
+
+void ECBackend::recover_object(
+ const hobject_t &hoid,
+ eversion_t v,
+ ObjectContextRef head,
+ ObjectContextRef obc,
+ RecoveryHandle *_h)
+{
+ ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
+ h->ops.push_back(RecoveryOp());
+ h->ops.back().v = v;
+ h->ops.back().hoid = hoid;
+ h->ops.back().obc = obc;
+ h->ops.back().recovery_info.soid = hoid;
+ h->ops.back().recovery_info.version = v;
+ if (obc) {
+ h->ops.back().recovery_info.size = obc->obs.oi.size;
+ h->ops.back().recovery_info.oi = obc->obs.oi;
+ }
+ h->ops.back().recovery_progress.omap_complete = true;
+ for (set<pg_shard_t>::const_iterator i =
+ get_parent()->get_actingbackfill_shards().begin();
+ i != get_parent()->get_actingbackfill_shards().end();
+ ++i) {
+ dout(10) << "checking " << *i << dendl;
+ if (get_parent()->get_shard_missing(*i).is_missing(hoid)) {
+ h->ops.back().missing_on.insert(*i);
+ h->ops.back().missing_on_shards.insert(i->shard);
+ }
+ }
+ dout(10) << __func__ << ": built op " << h->ops.back() << dendl;
+}
+
+bool ECBackend::can_handle_while_inactive(
+ OpRequestRef _op)
+{
+ return false;
+}
+
+bool ECBackend::handle_message(
+ OpRequestRef _op)
+{
+ dout(10) << __func__ << ": " << *_op->get_req() << dendl;
+ int priority = _op->get_req()->get_priority();
+ switch (_op->get_req()->get_type()) {
+ case MSG_OSD_EC_WRITE: {
+ MOSDECSubOpWrite *op = static_cast<MOSDECSubOpWrite*>(_op->get_req());
+ handle_sub_write(op->op.from, _op, op->op);
+ return true;
+ }
+ case MSG_OSD_EC_WRITE_REPLY: {
+ MOSDECSubOpWriteReply *op = static_cast<MOSDECSubOpWriteReply*>(
+ _op->get_req());
+ op->set_priority(priority);
+ handle_sub_write_reply(op->op.from, op->op);
+ return true;
+ }
+ case MSG_OSD_EC_READ: {
+ MOSDECSubOpRead *op = static_cast<MOSDECSubOpRead*>(_op->get_req());
+ MOSDECSubOpReadReply *reply = new MOSDECSubOpReadReply;
+ reply->pgid = get_parent()->primary_spg_t();
+ reply->map_epoch = get_parent()->get_epoch();
+ handle_sub_read(op->op.from, op->op, &(reply->op));
+ op->set_priority(priority);
+ get_parent()->send_message_osd_cluster(
+ op->op.from.osd, reply, get_parent()->get_epoch());
+ return true;
+ }
+ case MSG_OSD_EC_READ_REPLY: {
+ MOSDECSubOpReadReply *op = static_cast<MOSDECSubOpReadReply*>(
+ _op->get_req());
+ RecoveryMessages rm;
+ handle_sub_read_reply(op->op.from, op->op, &rm);
+ dispatch_recovery_messages(rm, priority);
+ return true;
+ }
+ case MSG_OSD_PG_PUSH: {
+ MOSDPGPush *op = static_cast<MOSDPGPush *>(_op->get_req());
+ RecoveryMessages rm;
+ for (vector<PushOp>::iterator i = op->pushes.begin();
+ i != op->pushes.end();
+ ++i) {
+ handle_recovery_push(*i, &rm);
+ }
+ dispatch_recovery_messages(rm, priority);
+ return true;
+ }
+ case MSG_OSD_PG_PUSH_REPLY: {
+ MOSDPGPushReply *op = static_cast<MOSDPGPushReply *>(_op->get_req());
+ RecoveryMessages rm;
+ for (vector<PushReplyOp>::iterator i = op->replies.begin();
+ i != op->replies.end();
+ ++i) {
+ handle_recovery_push_reply(*i, op->from, &rm);
+ }
+ dispatch_recovery_messages(rm, priority);
+ return true;
+ }
+ default:
+ return false;
+ }
+ return false;
+}
+
+struct SubWriteCommitted : public Context {
+ ECBackend *pg;
+ OpRequestRef msg;
+ tid_t tid;
+ eversion_t version;
+ eversion_t last_complete;
+ SubWriteCommitted(
+ ECBackend *pg,
+ OpRequestRef msg,
+ tid_t tid,
+ eversion_t version,
+ eversion_t last_complete)
+ : pg(pg), msg(msg), tid(tid),
+ version(version), last_complete(last_complete) {}
+ void finish(int) {
+ if (msg)
+ msg->mark_event("sub_op_committed");
+ pg->sub_write_committed(tid, version, last_complete);
+ }
+};
+void ECBackend::sub_write_committed(
+ tid_t tid, eversion_t version, eversion_t last_complete) {
+ if (get_parent()->pgb_is_primary()) {
+ ECSubWriteReply reply;
+ reply.tid = tid;
+ reply.last_complete = last_complete;
+ reply.committed = true;
+ reply.from = get_parent()->whoami_shard();
+ handle_sub_write_reply(
+ get_parent()->whoami_shard(),
+ reply);
+ } else {
+ get_parent()->update_last_complete_ondisk(last_complete);
+ MOSDECSubOpWriteReply *r = new MOSDECSubOpWriteReply;
+ r->pgid = get_parent()->primary_spg_t();
+ r->map_epoch = get_parent()->get_epoch();
+ r->op.tid = tid;
+ r->op.last_complete = last_complete;
+ r->op.committed = true;
+ r->op.from = get_parent()->whoami_shard();
+ get_parent()->send_message_osd_cluster(
+ get_parent()->primary_shard().osd, r, get_parent()->get_epoch());
+ }
+}
+
+struct SubWriteApplied : public Context {
+ ECBackend *pg;
+ OpRequestRef msg;
+ tid_t tid;
+ eversion_t version;
+ SubWriteApplied(
+ ECBackend *pg,
+ OpRequestRef msg,
+ tid_t tid,
+ eversion_t version)
+ : pg(pg), msg(msg), tid(tid), version(version) {}
+ void finish(int) {
+ if (msg)
+ msg->mark_event("sub_op_applied");
+ pg->sub_write_applied(tid, version);
+ }
+};
+void ECBackend::sub_write_applied(
+ tid_t tid, eversion_t version) {
+ parent->op_applied(version);
+ if (get_parent()->pgb_is_primary()) {
+ ECSubWriteReply reply;
+ reply.from = get_parent()->whoami_shard();
+ reply.tid = tid;
+ reply.applied = true;
+ handle_sub_write_reply(
+ get_parent()->whoami_shard(),
+ reply);
+ } else {
+ MOSDECSubOpWriteReply *r = new MOSDECSubOpWriteReply;
+ r->pgid = get_parent()->primary_spg_t();
+ r->map_epoch = get_parent()->get_epoch();
+ r->op.from = get_parent()->whoami_shard();
+ r->op.tid = tid;
+ r->op.applied = true;
+ get_parent()->send_message_osd_cluster(
+ get_parent()->primary_shard().osd, r, get_parent()->get_epoch());
+ }
+}
+
+void ECBackend::handle_sub_write(
+ pg_shard_t from,
+ OpRequestRef msg,
+ ECSubWrite &op,
+ Context *on_local_applied_sync)
+{
+ if (msg)
+ msg->mark_started();
+ assert(!get_parent()->get_log().get_missing().is_missing(op.soid));
+ if (!get_parent()->pgb_is_primary())
+ get_parent()->update_stats(op.stats);
+ ObjectStore::Transaction *localt = new ObjectStore::Transaction;
+ if (op.temp_added.size()) {
+ get_temp_coll(localt);
+ add_temp_objs(op.temp_added);
+ }
+ if (op.t.empty()) {
+ for (set<hobject_t>::iterator i = op.temp_removed.begin();
+ i != op.temp_removed.end();
+ ++i) {
+ dout(10) << __func__ << ": removing object " << *i
+ << " since we won't get the transaction" << dendl;
+ localt->remove(
+ temp_coll,
+ ghobject_t(
+ *i,
+ ghobject_t::NO_GEN,
+ get_parent()->whoami_shard().shard));
+ }
+ }
+ clear_temp_objs(op.temp_removed);
+ get_parent()->log_operation(
+ op.log_entries,
+ op.trim_to,
+ !(op.t.empty()),
+ localt);
+ localt->append(op.t);
+ if (on_local_applied_sync) {
+ dout(10) << "Queueing onreadable_sync: " << on_local_applied_sync << dendl;
+ localt->register_on_applied_sync(on_local_applied_sync);
+ }
+ localt->register_on_commit(
+ get_parent()->bless_context(
+ new SubWriteCommitted(
+ this, msg, op.tid,
+ op.at_version,
+ get_parent()->get_info().last_complete)));
+ localt->register_on_applied(
+ get_parent()->bless_context(
+ new SubWriteApplied(this, msg, op.tid, op.at_version)));
+ localt->register_on_applied(
+ new ObjectStore::C_DeleteTransaction(localt));
+ get_parent()->queue_transaction(localt, msg);
+}
+
+void ECBackend::handle_sub_read(
+ pg_shard_t from,
+ ECSubRead &op,
+ ECSubReadReply *reply)
+{
+ for(map<hobject_t, list<pair<uint64_t, uint64_t> > >::iterator i =
+ op.to_read.begin();
+ i != op.to_read.end();
+ ++i) {
+ for (list<pair<uint64_t, uint64_t> >::iterator j = i->second.begin();
+ j != i->second.end();
+ ++j) {
+ bufferlist bl;
+ int r = store->read(
+ i->first.is_temp() ? temp_coll : coll,
+ ghobject_t(
+ i->first, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ j->first,
+ j->second,
+ bl,
+ false);
+ if (r < 0) {
+ assert(0);
+ reply->buffers_read.erase(i->first);
+ reply->errors[i->first] = r;
+ break;
+ } else {
+ reply->buffers_read[i->first].push_back(
+ make_pair(
+ j->first,
+ bl)
+ );
+ }
+ }
+ }
+ for (set<hobject_t>::iterator i = op.attrs_to_read.begin();
+ i != op.attrs_to_read.end();
+ ++i) {
+ dout(10) << __func__ << ": fulfilling attr request on "
+ << *i << dendl;
+ if (reply->errors.count(*i))
+ continue;
+ int r = store->getattrs(
+ i->is_temp() ? temp_coll : coll,
+ ghobject_t(
+ *i, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ reply->attrs_read[*i]);
+ if (r < 0) {
+ assert(0);
+ reply->buffers_read.erase(*i);
+ reply->errors[*i] = r;
+ }
+ }
+ reply->from = get_parent()->whoami_shard();
+ reply->tid = op.tid;
+}
+
+void ECBackend::handle_sub_write_reply(
+ pg_shard_t from,
+ ECSubWriteReply &op)
+{
+ map<tid_t, Op>::iterator i = tid_to_op_map.find(op.tid);
+ assert(i != tid_to_op_map.end());
+ if (op.committed) {
+ assert(i->second.pending_commit.count(from));
+ i->second.pending_commit.erase(from);
+ if (from != get_parent()->whoami_shard()) {
+ get_parent()->update_peer_last_complete_ondisk(from, op.last_complete);
+ }
+ }
+ if (op.applied) {
+ assert(i->second.pending_apply.count(from));
+ i->second.pending_apply.erase(from);
+ }
+ check_op(&(i->second));
+}
+
+void ECBackend::handle_sub_read_reply(
+ pg_shard_t from,
+ ECSubReadReply &op,
+ RecoveryMessages *m)
+{
+ dout(10) << __func__ << ": reply " << op << dendl;
+ map<tid_t, ReadOp>::iterator iter = tid_to_read_map.find(op.tid);
+ if (iter == tid_to_read_map.end()) {
+ //canceled
+ return;
+ }
+ ReadOp &rop = iter->second;
+ for (map<hobject_t, list<pair<uint64_t, bufferlist> > >::iterator i =
+ op.buffers_read.begin();
+ i != op.buffers_read.end();
+ ++i) {
+ assert(!op.errors.count(i->first));
+ if (!rop.to_read.count(i->first)) {
+ // We canceled this read! @see filter_read_op
+ continue;
+ }
+ list<pair<uint64_t, uint64_t> >::const_iterator req_iter =
+ rop.to_read.find(i->first)->second.to_read.begin();
+ list<
+ boost::tuple<
+ uint64_t, uint64_t, map<pg_shard_t, bufferlist> > >::iterator riter =
+ rop.complete[i->first].returned.begin();
+ for (list<pair<uint64_t, bufferlist> >::iterator j = i->second.begin();
+ j != i->second.end();
+ ++j, ++req_iter, ++riter) {
+ assert(req_iter != rop.to_read.find(i->first)->second.to_read.end());
+ assert(riter != rop.complete[i->first].returned.end());
+ pair<uint64_t, uint64_t> adjusted =
+ sinfo.aligned_offset_len_to_chunk(
+ *req_iter);
+ assert(adjusted.first == j->first);
+ riter->get<2>()[from].claim(j->second);
+ }
+ }
+ for (map<hobject_t, map<string, bufferlist> >::iterator i = op.attrs_read.begin();
+ i != op.attrs_read.end();
+ ++i) {
+ assert(!op.errors.count(i->first));
+ if (!rop.to_read.count(i->first)) {
+ // We canceled this read! @see filter_read_op
+ continue;
+ }
+ rop.complete[i->first].attrs = map<string, bufferlist>();
+ (*(rop.complete[i->first].attrs)).swap(i->second);
+ }
+ for (map<hobject_t, int>::iterator i = op.errors.begin();
+ i != op.errors.end();
+ ++i) {
+ rop.complete[i->first].errors.insert(
+ make_pair(
+ from,
+ i->second));
+ if (rop.complete[i->first].r == 0)
+ rop.complete[i->first].r = i->second;
+ }
+
+ map<pg_shard_t, set<tid_t> >::iterator siter = shard_to_read_map.find(from);
+ assert(siter != shard_to_read_map.end());
+ assert(siter->second.count(op.tid));
+ siter->second.erase(op.tid);
+
+ assert(rop.in_progress.count(from));
+ rop.in_progress.erase(from);
+ if (!rop.in_progress.empty()) {
+ dout(10) << __func__ << " readop not complete: " << rop << dendl;
+ } else {
+ dout(10) << __func__ << " readop complete: " << rop << dendl;
+ complete_read_op(rop, m);
+ }
+}
+
+void ECBackend::complete_read_op(ReadOp &rop, RecoveryMessages *m)
+{
+ map<hobject_t, read_request_t>::iterator reqiter =
+ rop.to_read.begin();
+ map<hobject_t, read_result_t>::iterator resiter =
+ rop.complete.begin();
+ assert(rop.to_read.size() == rop.complete.size());
+ for (; reqiter != rop.to_read.end(); ++reqiter, ++resiter) {
+ if (reqiter->second.cb) {
+ pair<RecoveryMessages *, read_result_t &> arg(
+ m, resiter->second);
+ reqiter->second.cb->complete(arg);
+ reqiter->second.cb = NULL;
+ }
+ }
+ tid_to_read_map.erase(rop.tid);
+}
+
+struct FinishReadOp : public GenContext<ThreadPool::TPHandle&> {
+ ECBackend *ec;
+ tid_t tid;
+ FinishReadOp(ECBackend *ec, tid_t tid) : ec(ec), tid(tid) {}
+ void finish(ThreadPool::TPHandle &handle) {
+ assert(ec->tid_to_read_map.count(tid));
+ int priority = ec->tid_to_read_map[tid].priority;
+ RecoveryMessages rm;
+ ec->complete_read_op(ec->tid_to_read_map[tid], &rm);
+ ec->dispatch_recovery_messages(rm, priority);
+ }
+};
+
+void ECBackend::filter_read_op(
+ const OSDMapRef osdmap,
+ ReadOp &op)
+{
+ set<hobject_t> to_cancel;
+ for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
+ i != op.source_to_obj.end();
+ ++i) {
+ if (osdmap->is_down(i->first.osd)) {
+ to_cancel.insert(i->second.begin(), i->second.end());
+ op.in_progress.erase(i->first);
+ continue;
+ }
+ }
+
+ if (to_cancel.empty())
+ return;
+
+ for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
+ i != op.source_to_obj.end();
+ ) {
+ for (set<hobject_t>::iterator j = i->second.begin();
+ j != i->second.end();
+ ) {
+ if (to_cancel.count(*j))
+ i->second.erase(j++);
+ else
+ ++j;
+ }
+ if (i->second.empty()) {
+ op.source_to_obj.erase(i++);
+ } else {
+ assert(!osdmap->is_down(i->first.osd));
+ ++i;
+ }
+ }
+
+ for (set<hobject_t>::iterator i = to_cancel.begin();
+ i != to_cancel.end();
+ ++i) {
+ get_parent()->cancel_pull(*i);
+
+ assert(op.to_read.count(*i));
+ read_request_t &req = op.to_read.find(*i)->second;
+ dout(10) << __func__ << ": canceling " << req
+ << " for obj " << *i << dendl;
+ assert(req.cb);
+ delete req.cb;
+ req.cb = NULL;
+
+ op.to_read.erase(*i);
+ op.complete.erase(*i);
+ recovery_ops.erase(*i);
+ }
+
+ if (op.in_progress.empty()) {
+ get_parent()->schedule_work(
+ get_parent()->bless_gencontext(
+ new FinishReadOp(this, op.tid)));
+ }
+};
+
+void ECBackend::check_recovery_sources(const OSDMapRef osdmap)
+{
+ set<tid_t> tids_to_filter;
+ for (map<pg_shard_t, set<tid_t> >::iterator i = shard_to_read_map.begin();
+ i != shard_to_read_map.end();
+ ) {
+ if (osdmap->is_down(i->first.osd)) {
+ tids_to_filter.insert(i->second.begin(), i->second.end());
+ shard_to_read_map.erase(i++);
+ } else {
+ ++i;
+ }
+ }
+ for (set<tid_t>::iterator i = tids_to_filter.begin();
+ i != tids_to_filter.end();
+ ++i) {
+ map<tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
+ assert(j != tid_to_read_map.end());
+ filter_read_op(osdmap, j->second);
+ }
+}
+
+void ECBackend::_on_change(ObjectStore::Transaction *t)
+{
+ writing.clear();
+ tid_to_op_map.clear();
+ for (map<tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
+ i != tid_to_read_map.end();
+ ++i) {
+ dout(10) << __func__ << ": cancelling " << i->second << dendl;
+ for (map<hobject_t, read_request_t>::iterator j =
+ i->second.to_read.begin();
+ j != i->second.to_read.end();
+ ++j) {
+ delete j->second.cb;
+ j->second.cb = 0;
+ }
+ }
+ tid_to_read_map.clear();
+ for (list<ClientAsyncReadStatus>::iterator i = in_progress_client_reads.begin();
+ i != in_progress_client_reads.end();
+ ++i) {
+ delete i->on_complete;
+ i->on_complete = NULL;
+ }
+ in_progress_client_reads.clear();
+ shard_to_read_map.clear();
+ clear_state();
+}
+
+void ECBackend::clear_state()
+{
+ recovery_ops.clear();
+}
+
+void ECBackend::on_flushed()
+{
+}
+
+void ECBackend::dump_recovery_info(Formatter *f) const
+{
+ f->open_array_section("recovery_ops");
+ for (map<hobject_t, RecoveryOp>::const_iterator i = recovery_ops.begin();
+ i != recovery_ops.end();
+ ++i) {
+ f->open_object_section("op");
+ i->second.dump(f);
+ f->close_section();
+ }
+ f->close_section();
+ f->open_array_section("read_ops");
+ for (map<tid_t, ReadOp>::const_iterator i = tid_to_read_map.begin();
+ i != tid_to_read_map.end();
+ ++i) {
+ f->open_object_section("read_op");
+ i->second.dump(f);
+ f->close_section();
+ }
+ f->close_section();
+}
+
+PGBackend::PGTransaction *ECBackend::get_transaction()
+{
+ return new ECTransaction;
+}
+
+struct MustPrependHashInfo : public ObjectModDesc::Visitor {
+ enum { EMPTY, FOUND_APPEND, FOUND_CREATE_STASH } state;
+ MustPrependHashInfo() : state(EMPTY) {}
+ void append(uint64_t) {
+ if (state == EMPTY) {
+ state = FOUND_APPEND;
+ }
+ }
+ void rmobject(version_t) {
+ if (state == EMPTY) {
+ state = FOUND_CREATE_STASH;
+ }
+ }
+ void create() {
+ if (state == EMPTY) {
+ state = FOUND_CREATE_STASH;
+ }
+ }
+ bool must_prepend_hash_info() const { return state == FOUND_APPEND; }
+};
+
+void ECBackend::submit_transaction(
+ const hobject_t &hoid,
+ const eversion_t &at_version,
+ PGTransaction *_t,
+ const eversion_t &trim_to,
+ vector<pg_log_entry_t> &log_entries,
+ Context *on_local_applied_sync,
+ Context *on_all_applied,
+ Context *on_all_commit,
+ tid_t tid,
+ osd_reqid_t reqid,
+ OpRequestRef client_op
+ )
+{
+ assert(!tid_to_op_map.count(tid));
+ Op *op = &(tid_to_op_map[tid]);
+ op->hoid = hoid;
+ op->version = at_version;
+ op->trim_to = trim_to;
+ op->log_entries.swap(log_entries);
+ op->on_local_applied_sync = on_local_applied_sync;
+ op->on_all_applied = on_all_applied;
+ op->on_all_commit = on_all_commit;
+ op->tid = tid;
+ op->reqid = reqid;
+ op->client_op = client_op;
+
+ op->t = static_cast<ECTransaction*>(_t);
+
+ set<hobject_t> need_hinfos;
+ op->t->get_append_objects(&need_hinfos);
+ for (set<hobject_t>::iterator i = need_hinfos.begin();
+ i != need_hinfos.end();
+ ++i) {
+ op->unstable_hash_infos.insert(
+ make_pair(
+ *i,
+ get_hash_info(*i)));
+ }
+
+ for (vector<pg_log_entry_t>::iterator i = op->log_entries.begin();
+ i != op->log_entries.end();
+ ++i) {
+ MustPrependHashInfo vis;
+ i->mod_desc.visit(&vis);
+ if (vis.must_prepend_hash_info()) {
+ dout(10) << __func__ << ": stashing HashInfo for "
+ << i->soid << " for entry " << *i << dendl;
+ assert(op->unstable_hash_infos.count(i->soid));
+ ObjectModDesc desc;
+ map<string, boost::optional<bufferlist> > old_attrs;
+ bufferlist old_hinfo;
+ ::encode(*(op->unstable_hash_infos[i->soid]), old_hinfo);
+ old_attrs[ECUtil::get_hinfo_key()] = old_hinfo;
+ desc.setattrs(old_attrs);
+ i->mod_desc.swap(desc);
+ i->mod_desc.claim_append(desc);
+ assert(i->mod_desc.can_rollback());
+ }
+ }
+
+ dout(10) << __func__ << ": op " << *op << " starting" << dendl;
+ start_write(op);
+ writing.push_back(op);
+ dout(10) << "onreadable_sync: " << op->on_local_applied_sync << dendl;
+}
+
+int ECBackend::get_min_avail_to_read_shards(
+ const hobject_t &hoid,
+ const set<int> &want,
+ bool for_recovery,
+ set<pg_shard_t> *to_read)
+{
+ map<hobject_t, set<pg_shard_t> >::const_iterator miter =
+ get_parent()->get_missing_loc_shards().find(hoid);
+
+ set<int> have;
+ map<shard_id_t, pg_shard_t> shards;
+
+ for (set<pg_shard_t>::const_iterator i =
+ get_parent()->get_acting_shards().begin();
+ i != get_parent()->get_acting_shards().end();
+ ++i) {
+ dout(10) << __func__ << ": checking acting " << *i << dendl;
+ const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
+ if (!missing.is_missing(hoid)) {
+ assert(!have.count(i->shard));
+ have.insert(i->shard);
+ assert(!shards.count(i->shard));
+ shards.insert(make_pair(i->shard, *i));
+ }
+ }
+
+ if (for_recovery) {
+ for (set<pg_shard_t>::const_iterator i =
+ get_parent()->get_backfill_shards().begin();
+ i != get_parent()->get_backfill_shards().end();
+ ++i) {
+ if (have.count(i->shard)) {
+ assert(shards.count(i->shard));
+ continue;
+ }
+ dout(10) << __func__ << ": checking backfill " << *i << dendl;
+ assert(!shards.count(i->shard));
+ const pg_info_t &info = get_parent()->get_shard_info(*i);
+ const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
+ if (hoid < info.last_backfill && !missing.is_missing(hoid)) {
+ have.insert(i->shard);
+ shards.insert(make_pair(i->shard, *i));
+ }
+ }
+
+ if (miter != get_parent()->get_missing_loc_shards().end()) {
+ for (set<pg_shard_t>::iterator i = miter->second.begin();
+ i != miter->second.end();
+ ++i) {
+ dout(10) << __func__ << ": checking missing_loc " << *i << dendl;
+ boost::optional<const pg_missing_t &> m =
+ get_parent()->maybe_get_shard_missing(*i);
+ if (m) {
+ assert(!(*m).is_missing(hoid));
+ }
+ have.insert(i->shard);
+ shards.insert(make_pair(i->shard, *i));
+ }
+ }
+ }
+
+ set<int> need;
+ int r = ec_impl->minimum_to_decode(want, have, &need);
+ if (r < 0)
+ return r;
+
+ if (!to_read)
+ return 0;
+
+ for (set<int>::iterator i = need.begin();
+ i != need.end();
+ ++i) {
+ assert(shards.count(*i));
+ to_read->insert(shards[*i]);
+ }
+ return 0;
+}
+
+void ECBackend::start_read_op(
+ int priority,
+ map<hobject_t, read_request_t> &to_read,
+ OpRequestRef _op)
+{
+ tid_t tid = get_parent()->get_tid();
+ assert(!tid_to_read_map.count(tid));
+ ReadOp &op(tid_to_read_map[tid]);
+ op.priority = priority;
+ op.tid = tid;
+ op.to_read.swap(to_read);
+ op.op = _op;
+ dout(10) << __func__ << ": starting " << op << dendl;
+
+ map<pg_shard_t, ECSubRead> messages;
+ for (map<hobject_t, read_request_t>::iterator i = op.to_read.begin();
+ i != op.to_read.end();
+ ++i) {
+ list<boost::tuple<
+ uint64_t, uint64_t, map<pg_shard_t, bufferlist> > > &reslist =
+ op.complete[i->first].returned;
+ bool need_attrs = i->second.want_attrs;
+ for (set<pg_shard_t>::const_iterator j = i->second.need.begin();
+ j != i->second.need.end();
+ ++j) {
+ if (need_attrs) {
+ messages[*j].attrs_to_read.insert(i->first);
+ need_attrs = false;
+ }
+ op.obj_to_source[i->first].insert(*j);
+ op.source_to_obj[*j].insert(i->first);
+ }
+ for (list<pair<uint64_t, uint64_t> >::const_iterator j =
+ i->second.to_read.begin();
+ j != i->second.to_read.end();
+ ++j) {
+ reslist.push_back(
+ boost::make_tuple(
+ j->first,
+ j->second,
+ map<pg_shard_t, bufferlist>()));
+ pair<uint64_t, uint64_t> chunk_off_len =
+ sinfo.aligned_offset_len_to_chunk(
+ *j);
+ for (set<pg_shard_t>::const_iterator k = i->second.need.begin();
+ k != i->second.need.end();
+ ++k) {
+ messages[*k].to_read[i->first].push_back(chunk_off_len);
+ }
+ assert(!need_attrs);
+ }
+ }
+
+ for (map<pg_shard_t, ECSubRead>::iterator i = messages.begin();
+ i != messages.end();
+ ++i) {
+ op.in_progress.insert(i->first);
+ shard_to_read_map[i->first].insert(op.tid);
+ i->second.tid = tid;
+ MOSDECSubOpRead *msg = new MOSDECSubOpRead;
+ msg->set_priority(priority);
+ msg->pgid = spg_t(
+ get_parent()->whoami_spg_t().pgid,
+ i->first.shard);
+ msg->map_epoch = get_parent()->get_epoch();
+ msg->op = i->second;
+ msg->op.from = get_parent()->whoami_shard();
+ msg->op.tid = tid;
+ get_parent()->send_message_osd_cluster(
+ i->first.osd,
+ msg,
+ get_parent()->get_epoch());
+ }
+ dout(10) << __func__ << ": started " << op << dendl;
+}
+
+ECUtil::HashInfoRef ECBackend::get_hash_info(
+ const hobject_t &hoid)
+{
+ dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
+ ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid);
+ if (!ref) {
+ dout(10) << __func__ << ": not in cache " << hoid << dendl;
+ struct stat st;
+ int r = store->stat(
+ hoid.is_temp() ? temp_coll : coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ &st);
+ ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
+ if (r >= 0 && st.st_size > 0) {
+ dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
+ bufferlist bl;
+ r = store->getattr(
+ hoid.is_temp() ? temp_coll : coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ ECUtil::get_hinfo_key(),
+ bl);
+ if (r >= 0) {
+ bufferlist::iterator bp = bl.begin();
+ ::decode(hinfo, bp);
+ assert(hinfo.get_total_chunk_size() == (unsigned)st.st_size);
+ } else {
+ assert(0 == "missing hash attr");
+ }
+ }
+ ref = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
+ }
+ return ref;
+}
+
+void ECBackend::check_op(Op *op)
+{
+ if (op->pending_apply.empty() && op->on_all_applied) {
+ dout(10) << __func__ << " Calling on_all_applied on " << *op << dendl;
+ op->on_all_applied->complete(0);
+ op->on_all_applied = 0;
+ }
+ if (op->pending_commit.empty() && op->on_all_commit) {
+ dout(10) << __func__ << " Calling on_all_commit on " << *op << dendl;
+ op->on_all_commit->complete(0);
+ op->on_all_commit = 0;
+ }
+ if (op->pending_apply.empty() && op->pending_commit.empty()) {
+ // done!
+ assert(writing.front() == op);
+ dout(10) << __func__ << " Completing " << *op << dendl;
+ writing.pop_front();
+ tid_to_op_map.erase(op->tid);
+ }
+ for (map<tid_t, Op>::iterator i = tid_to_op_map.begin();
+ i != tid_to_op_map.end();
+ ++i) {
+ dout(20) << __func__ << " tid " << i->first <<": " << i->second << dendl;
+ }
+}
+
+void ECBackend::start_write(Op *op) {
+ map<shard_id_t, ObjectStore::Transaction> trans;
+ for (set<pg_shard_t>::const_iterator i =
+ get_parent()->get_actingbackfill_shards().begin();
+ i != get_parent()->get_actingbackfill_shards().end();
+ ++i) {
+ trans[i->shard];
+ }
+ op->t->generate_transactions(
+ op->unstable_hash_infos,
+ ec_impl,
+ get_parent()->get_info().pgid.pgid,
+ sinfo,
+ &trans,
+ &(op->temp_added),
+ &(op->temp_cleared));
+
+ dout(10) << "onreadable_sync: " << op->on_local_applied_sync << dendl;
+
+ for (set<pg_shard_t>::const_iterator i =
+ get_parent()->get_actingbackfill_shards().begin();
+ i != get_parent()->get_actingbackfill_shards().end();
+ ++i) {
+ op->pending_apply.insert(*i);
+ op->pending_commit.insert(*i);
+ map<shard_id_t, ObjectStore::Transaction>::iterator iter =
+ trans.find(i->shard);
+ assert(iter != trans.end());
+ bool should_send = get_parent()->should_send_op(*i, op->hoid);
+ pg_stat_t stats =
+ should_send ?
+ get_info().stats :
+ parent->get_shard_info().find(*i)->second.stats;
+
+ ECSubWrite sop(
+ get_parent()->whoami_shard(),
+ op->tid,
+ op->reqid,
+ op->hoid,
+ stats,
+ should_send ? iter->second : ObjectStore::Transaction(),
+ op->version,
+ op->trim_to,
+ op->log_entries,
+ op->temp_added,
+ op->temp_cleared);
+ if (*i == get_parent()->whoami_shard()) {
+ handle_sub_write(
+ get_parent()->whoami_shard(),
+ op->client_op,
+ sop,
+ op->on_local_applied_sync);
+ op->on_local_applied_sync = 0;
+ } else {
+ MOSDECSubOpWrite *r = new MOSDECSubOpWrite(sop);
+ r->set_priority(cct->_conf->osd_client_op_priority);
+ r->pgid = spg_t(get_parent()->primary_spg_t().pgid, i->shard);
+ r->map_epoch = get_parent()->get_epoch();
+ get_parent()->send_message_osd_cluster(
+ i->osd, r, get_parent()->get_epoch());
+ }
+ }
+}
+
+int ECBackend::objects_read_sync(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist *bl)
+{
+ return -EOPNOTSUPP;
+}
+
+struct CallClientContexts :
+ public GenContext<pair<RecoveryMessages*, ECBackend::read_result_t& > &> {
+ ECBackend *ec;
+ ECBackend::ClientAsyncReadStatus *status;
+ list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > to_read;
+ CallClientContexts(
+ ECBackend *ec,
+ ECBackend::ClientAsyncReadStatus *status,
+ const list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > &to_read)
+ : ec(ec), status(status), to_read(to_read) {}
+ void finish(pair<RecoveryMessages *, ECBackend::read_result_t &> &in) {
+ ECBackend::read_result_t &res = in.second;
+ assert(res.returned.size() == to_read.size());
+ assert(res.r == 0);
+ assert(res.errors.empty());
+ for (list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > >::iterator i = to_read.begin();
+ i != to_read.end();
+ to_read.erase(i++)) {
+ pair<uint64_t, uint64_t> adjusted =
+ ec->sinfo.offset_len_to_stripe_bounds(i->first);
+ assert(res.returned.front().get<0>() == adjusted.first &&
+ res.returned.front().get<1>() == adjusted.second);
+ map<int, bufferlist> to_decode;
+ bufferlist bl;
+ for (map<pg_shard_t, bufferlist>::iterator j =
+ res.returned.front().get<2>().begin();
+ j != res.returned.front().get<2>().end();
+ ++j) {
+ to_decode[j->first.shard].claim(j->second);
+ }
+ ECUtil::decode(
+ ec->sinfo,
+ ec->ec_impl,
+ to_decode,
+ &bl);
+ assert(i->second.second);
+ assert(i->second.first);
+ i->second.first->substr_of(
+ bl,
+ i->first.first - adjusted.first,
+ MIN(i->first.second, bl.length() - (i->first.first - adjusted.first)));
+ if (i->second.second) {
+ i->second.second->complete(i->second.first->length());
+ }
+ res.returned.pop_front();
+ }
+ status->complete = true;
+ list<ECBackend::ClientAsyncReadStatus> &ip =
+ ec->in_progress_client_reads;
+ while (ip.size() && ip.front().complete) {
+ if (ip.front().on_complete) {
+ ip.front().on_complete->complete(0);
+ ip.front().on_complete = NULL;
+ }
+ ip.pop_front();
+ }
+ }
+ ~CallClientContexts() {
+ for (list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > >::iterator i = to_read.begin();
+ i != to_read.end();
+ to_read.erase(i++)) {
+ delete i->second.second;
+ }
+ }
+};
+
+void ECBackend::objects_read_async(
+ const hobject_t &hoid,
+ const list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > &to_read,
+ Context *on_complete)
+{
+ in_progress_client_reads.push_back(ClientAsyncReadStatus(on_complete));
+ CallClientContexts *c = new CallClientContexts(
+ this, &(in_progress_client_reads.back()), to_read);
+ list<pair<uint64_t, uint64_t> > offsets;
+ for (list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > >::const_iterator i =
+ to_read.begin();
+ i != to_read.end();
+ ++i) {
+ offsets.push_back(
+ sinfo.offset_len_to_stripe_bounds(i->first));
+ }
+
+ set<int> want_to_read;
+ for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
+ want_to_read.insert(i);
+ }
+ set<pg_shard_t> shards;
+ int r = get_min_avail_to_read_shards(
+ hoid,
+ want_to_read,
+ false,
+ &shards);
+ assert(r == 0);
+
+ map<hobject_t, read_request_t> for_read_op;
+ for_read_op.insert(
+ make_pair(
+ hoid,
+ read_request_t(
+ hoid,
+ offsets,
+ shards,
+ false,
+ c)));
+
+ start_read_op(
+ cct->_conf->osd_client_op_priority,
+ for_read_op,
+ OpRequestRef());
+ return;
+}
+
+
+int ECBackend::objects_get_attrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> *out)
+{
+ int r = store->getattrs(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ *out);
+ if (r < 0)
+ return r;
+
+ for (map<string, bufferlist>::iterator i = out->begin();
+ i != out->end();
+ ) {
+ if (ECUtil::is_hinfo_key_string(i->first))
+ out->erase(i++);
+ else
+ ++i;
+ }
+ return r;
+}
+
+void ECBackend::rollback_append(
+ const hobject_t &hoid,
+ uint64_t old_size,
+ ObjectStore::Transaction *t)
+{
+ assert(old_size % sinfo.get_stripe_width() == 0);
+ t->truncate(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ sinfo.aligned_logical_offset_to_chunk_offset(
+ old_size));
+}
+
+void ECBackend::be_deep_scrub(
+ const hobject_t &poid,
+ ScrubMap::object &o,
+ ThreadPool::TPHandle &handle) {
+ bufferhash h(-1);
+ int r;
+ uint64_t stride = cct->_conf->osd_deep_scrub_stride;
+ if (stride % sinfo.get_chunk_size())
+ stride += sinfo.get_chunk_size() - (stride % sinfo.get_chunk_size());
+ uint64_t pos = 0;
+ while (true) {
+ bufferlist bl;
+ handle.reset_tp_timeout();
+ r = store->read(
+ coll,
+ ghobject_t(
+ poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ pos,
+ stride, bl,
+ true);
+ if (r < 0)
+ break;
+ if (bl.length() % sinfo.get_chunk_size()) {
+ r = -EIO;
+ break;
+ }
+ pos += r;
+ h << bl;
+ if ((unsigned)r < stride)
+ break;
+ }
+
+ ECUtil::HashInfoRef hinfo = get_hash_info(poid);
+ if (r == -EIO) {
+ dout(0) << "_scan_list " << poid << " got "
+ << r << " on read, read_error" << dendl;
+ o.read_error = true;
+ }
+
+ if (hinfo->get_chunk_hash(get_parent()->whoami_shard().shard) != h.digest()) {
+ dout(0) << "_scan_list " << poid << " got incorrect hash on read" << dendl;
+ o.read_error = true;
+ }
+
+ if (hinfo->get_total_chunk_size() != pos) {
+ dout(0) << "_scan_list " << poid << " got incorrect size on read" << dendl;
+ o.read_error = true;
+ }
+
+ /* We checked above that we match our own stored hash. We cannot
+ * send a hash of the actual object, so instead we simply send
+ * our locally stored hash of shard 0 on the assumption that if
+ * we match our chunk hash and our recollection of the hash for
+ * chunk 0 matches that of our peers, there is likely no corruption.
+ */
+ o.digest = hinfo->get_chunk_hash(0);
+ o.digest_present = true;
+
+ o.omap_digest = 0;
+ o.omap_digest_present = true;
+}
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
new file mode 100644
index 0000000..7844563
--- /dev/null
+++ b/src/osd/ECBackend.h
@@ -0,0 +1,476 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef ECBACKEND_H
+#define ECBACKEND_H
+
+#include "OSD.h"
+#include "PGBackend.h"
+#include "osd_types.h"
+#include <boost/optional.hpp>
+#include "erasure-code/ErasureCodeInterface.h"
+#include "ECTransaction.h"
+#include "ECMsgTypes.h"
+#include "ECUtil.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+
+struct RecoveryMessages;
+class ECBackend : public PGBackend {
+public:
+ RecoveryHandle *open_recovery_op();
+
+ void run_recovery_op(
+ RecoveryHandle *h,
+ int priority
+ );
+
+ void recover_object(
+ const hobject_t &hoid,
+ eversion_t v,
+ ObjectContextRef head,
+ ObjectContextRef obc,
+ RecoveryHandle *h
+ );
+
+ bool handle_message(
+ OpRequestRef op
+ );
+ bool can_handle_while_inactive(
+ OpRequestRef op
+ );
+ friend struct SubWriteApplied;
+ friend struct SubWriteCommitted;
+ void sub_write_applied(
+ tid_t tid, eversion_t version);
+ void sub_write_committed(
+ tid_t tid, eversion_t version, eversion_t last_complete);
+ void handle_sub_write(
+ pg_shard_t from,
+ OpRequestRef msg,
+ ECSubWrite &op,
+ Context *on_local_applied_sync = 0
+ );
+ void handle_sub_read(
+ pg_shard_t from,
+ ECSubRead &op,
+ ECSubReadReply *reply
+ );
+ void handle_sub_write_reply(
+ pg_shard_t from,
+ ECSubWriteReply &op
+ );
+ void handle_sub_read_reply(
+ pg_shard_t from,
+ ECSubReadReply &op,
+ RecoveryMessages *m
+ );
+
+ /// @see ReadOp below
+ void check_recovery_sources(const OSDMapRef osdmap);
+
+ void _on_change(ObjectStore::Transaction *t);
+ void clear_state();
+
+ void on_flushed();
+
+ void dump_recovery_info(Formatter *f) const;
+
+ /// @see osd/ECTransaction.cc/h
+ PGTransaction *get_transaction();
+
+ void submit_transaction(
+ const hobject_t &hoid,
+ const eversion_t &at_version,
+ PGTransaction *t,
+ const eversion_t &trim_to,
+ vector<pg_log_entry_t> &log_entries,
+ Context *on_local_applied_sync,
+ Context *on_all_applied,
+ Context *on_all_commit,
+ tid_t tid,
+ osd_reqid_t reqid,
+ OpRequestRef op
+ );
+
+ int objects_read_sync(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist *bl);
+
+ /**
+ * Async read mechanism
+ *
+ * Async reads use the same async read mechanism as does recovery.
+ * CallClientContexts is responsible for reconstructing the response
+ * buffer as well as for calling the callbacks.
+ *
+ * One tricky bit is that two reads may possibly not read from the same
+ * set of replicas. This could result in two reads completing in the
+ * wrong (from the interface user's point of view) order. Thus, we
+ * maintain a queue of in progress reads (@see in_progress_client_reads)
+ * to ensure that we always call the completion callback in order.
+ *
+ * Another subtely is that while we may read a degraded object, we will
+ * still only perform a client read from shards in the acting set. This
+ * ensures that we won't ever have to restart a client initiated read in
+ * check_recovery_sources.
+ */
+ friend struct CallClientContexts;
+ struct ClientAsyncReadStatus {
+ bool complete;
+ Context *on_complete;
+ ClientAsyncReadStatus(Context *on_complete)
+ : complete(false), on_complete(on_complete) {}
+ };
+ list<ClientAsyncReadStatus> in_progress_client_reads;
+ void objects_read_async(
+ const hobject_t &hoid,
+ const list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > &to_read,
+ Context *on_complete);
+
+private:
+ friend struct ECRecoveryHandle;
+ uint64_t get_recovery_chunk_size() const {
+ uint64_t max = cct->_conf->osd_recovery_max_chunk;
+ max -= max % sinfo.get_stripe_width();
+ max += sinfo.get_stripe_width();
+ return max;
+ }
+
+ /**
+ * Recovery
+ *
+ * Recovery uses the same underlying read mechanism as client reads
+ * with the slight difference that recovery reads may come from non
+ * acting shards. Thus, check_recovery_sources may wind up calling
+ * cancel_pull for a read originating with RecoveryOp.
+ *
+ * The recovery process is expressed as a state machine:
+ * - IDLE: Nothing is currently in progress, reads will be started and
+ * we will transition to READING
+ * - READING: We are awaiting a pending read op. Once complete, we will
+ * decode the buffers and proceed to WRITING
+ * - WRITING: We are awaiting a completed push. Once complete, we will
+ * either transition to COMPLETE or to IDLE to continue.
+ * - COMPLETE: complete
+ *
+ * We use the existing Push and PushReply messages and structures to
+ * handle actually shuffling the data over to the replicas. recovery_info
+ * and recovery_progress are expressed in terms of the logical offset
+ * space except for data_included which is in terms of the chunked object
+ * space (to match the passed buffer).
+ *
+ * xattrs are requested on the first read and used to initialize the
+ * object_context if missing on completion of the first read.
+ *
+ * In order to batch up reads and writes, we batch Push, PushReply,
+ * Transaction, and reads in a RecoveryMessages object which is passed
+ * among the recovery methods.
+ */
+ struct RecoveryOp {
+ hobject_t hoid;
+ eversion_t v;
+ set<pg_shard_t> missing_on;
+ set<shard_id_t> missing_on_shards;
+
+ ObjectRecoveryInfo recovery_info;
+ ObjectRecoveryProgress recovery_progress;
+
+ bool pending_read;
+ enum state_t { IDLE, READING, WRITING, COMPLETE } state;
+
+ static const char* tostr(state_t state) {
+ switch (state) {
+ case ECBackend::RecoveryOp::IDLE:
+ return "IDLE";
+ break;
+ case ECBackend::RecoveryOp::READING:
+ return "READING";
+ break;
+ case ECBackend::RecoveryOp::WRITING:
+ return "WRITING";
+ break;
+ case ECBackend::RecoveryOp::COMPLETE:
+ return "COMPLETE";
+ break;
+ default:
+ assert(0);
+ return "";
+ }
+ }
+
+ // must be filled if state == WRITING
+ map<shard_id_t, bufferlist> returned_data;
+ map<string, bufferlist> xattrs;
+ ECUtil::HashInfoRef hinfo;
+ ObjectContextRef obc;
+ set<pg_shard_t> waiting_on_pushes;
+
+ // valid in state READING
+ pair<uint64_t, uint64_t> extent_requested;
+
+ void dump(Formatter *f) const;
+
+ RecoveryOp() : pending_read(false), state(IDLE) {}
+ };
+ friend ostream &operator<<(ostream &lhs, const RecoveryOp &rhs);
+ map<hobject_t, RecoveryOp> recovery_ops;
+
+public:
+ /**
+ * Low level async read mechanism
+ *
+ * To avoid duplicating the logic for requesting and waiting for
+ * multiple object shards, there is a common async read mechanism
+ * taking a map of hobject_t->read_request_t which defines callbacks
+ * taking read_result_ts as arguments.
+ *
+ * tid_to_read_map gives open read ops. check_recovery_sources uses
+ * shard_to_read_map and ReadOp::source_to_obj to restart reads
+ * involving down osds.
+ *
+ * The user is responsible for specifying replicas on which to read
+ * and for reassembling the buffer on the other side since client
+ * reads require the original object buffer while recovery only needs
+ * the missing pieces.
+ *
+ * Rather than handling reads on the primary directly, we simply send
+ * ourselves a message. This avoids a dedicated primary path for that
+ * part.
+ */
+ struct read_result_t {
+ int r;
+ map<pg_shard_t, int> errors;
+ boost::optional<map<string, bufferlist> > attrs;
+ list<
+ boost::tuple<
+ uint64_t, uint64_t, map<pg_shard_t, bufferlist> > > returned;
+ read_result_t() : r(0) {}
+ };
+ struct read_request_t {
+ const list<pair<uint64_t, uint64_t> > to_read;
+ const set<pg_shard_t> need;
+ const bool want_attrs;
+ GenContext<pair<RecoveryMessages *, read_result_t& > &> *cb;
+ read_request_t(
+ const hobject_t &hoid,
+ const list<pair<uint64_t, uint64_t> > &to_read,
+ const set<pg_shard_t> &need,
+ bool want_attrs,
+ GenContext<pair<RecoveryMessages *, read_result_t& > &> *cb)
+ : to_read(to_read), need(need), want_attrs(want_attrs),
+ cb(cb) {}
+ };
+ friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
+
+ struct ReadOp {
+ int priority;
+ tid_t tid;
+ OpRequestRef op; // may be null if not on behalf of a client
+
+ map<hobject_t, read_request_t> to_read;
+ map<hobject_t, read_result_t> complete;
+
+ map<hobject_t, set<pg_shard_t> > obj_to_source;
+ map<pg_shard_t, set<hobject_t> > source_to_obj;
+
+ void dump(Formatter *f) const;
+
+ set<pg_shard_t> in_progress;
+ };
+ friend struct FinishReadOp;
+ void filter_read_op(
+ const OSDMapRef osdmap,
+ ReadOp &op);
+ void complete_read_op(ReadOp &rop, RecoveryMessages *m);
+ friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+ map<tid_t, ReadOp> tid_to_read_map;
+ map<pg_shard_t, set<tid_t> > shard_to_read_map;
+ void start_read_op(
+ int priority,
+ map<hobject_t, read_request_t> &to_read,
+ OpRequestRef op);
+
+
+ /**
+ * Client writes
+ *
+ * ECTransaction is responsible for generating a transaction for
+ * each shard to which we need to send the write. As required
+ * by the PGBackend interface, the ECBackend write mechanism
+ * passes trim information with the write and last_complete back
+ * with the reply.
+ *
+ * As with client reads, there is a possibility of out-of-order
+ * completions. Thus, callbacks and completion are called in order
+ * on the writing list.
+ */
+ struct Op {
+ hobject_t hoid;
+ eversion_t version;
+ eversion_t trim_to;
+ vector<pg_log_entry_t> log_entries;
+ Context *on_local_applied_sync;
+ Context *on_all_applied;
+ Context *on_all_commit;
+ tid_t tid;
+ osd_reqid_t reqid;
+ OpRequestRef client_op;
+
+ ECTransaction *t;
+
+ set<hobject_t> temp_added;
+ set<hobject_t> temp_cleared;
+
+ set<pg_shard_t> pending_commit;
+ set<pg_shard_t> pending_apply;
+
+ map<hobject_t, ECUtil::HashInfoRef> unstable_hash_infos;
+ ~Op() {
+ delete t;
+ delete on_local_applied_sync;
+ delete on_all_applied;
+ delete on_all_commit;
+ }
+ };
+ friend ostream &operator<<(ostream &lhs, const Op &rhs);
+
+ void continue_recovery_op(
+ RecoveryOp &op,
+ RecoveryMessages *m);
+
+ void dispatch_recovery_messages(RecoveryMessages &m, int priority);
+ friend struct OnRecoveryReadComplete;
+ void handle_recovery_read_complete(
+ const hobject_t &hoid,
+ boost::tuple<uint64_t, uint64_t, map<pg_shard_t, bufferlist> > &to_read,
+ boost::optional<map<string, bufferlist> > attrs,
+ RecoveryMessages *m);
+ void handle_recovery_push(
+ PushOp &op,
+ RecoveryMessages *m);
+ void handle_recovery_push_reply(
+ PushReplyOp &op,
+ pg_shard_t from,
+ RecoveryMessages *m);
+
+ map<tid_t, Op> tid_to_op_map; /// lists below point into here
+ list<Op*> writing;
+
+ CephContext *cct;
+ ErasureCodeInterfaceRef ec_impl;
+
+
+ /**
+ * ECRecPred
+ *
+ * Determines the whether _have is suffient to recover an object
+ */
+ class ECRecPred : public IsRecoverablePredicate {
+ set<int> want;
+ ErasureCodeInterfaceRef ec_impl;
+ public:
+ ECRecPred(ErasureCodeInterfaceRef ec_impl) : ec_impl(ec_impl) {
+ for (unsigned i = 0; i < ec_impl->get_data_chunk_count(); ++i) {
+ want.insert(i);
+ }
+ }
+ bool operator()(const set<pg_shard_t> &_have) const {
+ set<int> have;
+ for (set<pg_shard_t>::const_iterator i = _have.begin();
+ i != _have.end();
+ ++i) {
+ have.insert(i->shard);
+ }
+ set<int> min;
+ return ec_impl->minimum_to_decode(want, have, &min) == 0;
+ }
+ };
+ IsRecoverablePredicate *get_is_recoverable_predicate() {
+ return new ECRecPred(ec_impl);
+ }
+
+ /**
+ * ECReadPred
+ *
+ * Determines the whether _have is suffient to read an object
+ */
+ class ECReadPred : public IsReadablePredicate {
+ pg_shard_t whoami;
+ ECRecPred rec_pred;
+ public:
+ ECReadPred(
+ pg_shard_t whoami,
+ ErasureCodeInterfaceRef ec_impl) : whoami(whoami), rec_pred(ec_impl) {}
+ bool operator()(const set<pg_shard_t> &_have) const {
+ return _have.count(whoami) && rec_pred(_have);
+ }
+ };
+ IsReadablePredicate *get_is_readable_predicate() {
+ return new ECReadPred(get_parent()->whoami_shard(), ec_impl);
+ }
+
+
+ const ECUtil::stripe_info_t sinfo;
+ /// If modified, ensure that the ref is held until the update is applied
+ SharedPtrRegistry<hobject_t, ECUtil::HashInfo> unstable_hashinfo_registry;
+ ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid);
+
+ friend struct ReadCB;
+ void check_op(Op *op);
+ void start_write(Op *op);
+public:
+ ECBackend(
+ PGBackend::Listener *pg,
+ coll_t coll,
+ coll_t temp_coll,
+ ObjectStore *store,
+ CephContext *cct,
+ ErasureCodeInterfaceRef ec_impl,
+ uint64_t stripe_width);
+
+ /// Returns to_read replicas sufficient to reconstruct want
+ int get_min_avail_to_read_shards(
+ const hobject_t &hoid, ///< [in] object
+ const set<int> &want, ///< [in] desired shards
+ bool for_recovery, ///< [in] true if we may use non-acting replicas
+ set<pg_shard_t> *to_read ///< [out] shards to read
+ ); ///< @return error code, 0 on success
+
+ int objects_get_attrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> *out);
+
+ void rollback_append(
+ const hobject_t &hoid,
+ uint64_t old_size,
+ ObjectStore::Transaction *t);
+
+ bool scrub_supported() { return true; }
+
+ void be_deep_scrub(
+ const hobject_t &obj,
+ ScrubMap::object &o,
+ ThreadPool::TPHandle &handle);
+ uint64_t be_get_ondisk_size(uint64_t logical_size) {
+ return sinfo.logical_to_next_chunk_offset(logical_size);
+ }
+};
+
+#endif
diff --git a/src/osd/ECMsgTypes.cc b/src/osd/ECMsgTypes.cc
new file mode 100644
index 0000000..87e622b
--- /dev/null
+++ b/src/osd/ECMsgTypes.cc
@@ -0,0 +1,333 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "ECMsgTypes.h"
+
+void ECSubWrite::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(from, bl);
+ ::encode(tid, bl);
+ ::encode(reqid, bl);
+ ::encode(soid, bl);
+ ::encode(stats, bl);
+ ::encode(t, bl);
+ ::encode(at_version, bl);
+ ::encode(trim_to, bl);
+ ::encode(log_entries, bl);
+ ::encode(temp_added, bl);
+ ::encode(temp_removed, bl);
+ ENCODE_FINISH(bl);
+}
+
+void ECSubWrite::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(from, bl);
+ ::decode(tid, bl);
+ ::decode(reqid, bl);
+ ::decode(soid, bl);
+ ::decode(stats, bl);
+ ::decode(t, bl);
+ ::decode(at_version, bl);
+ ::decode(trim_to, bl);
+ ::decode(log_entries, bl);
+ ::decode(temp_added, bl);
+ ::decode(temp_removed, bl);
+ DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubWrite &rhs)
+{
+ return lhs
+ << "ECSubWrite(tid=" << rhs.tid
+ << ", reqid=" << rhs.reqid
+ << ", at_version=" << rhs.at_version
+ << ", trim_to=" << rhs.trim_to << ")";
+}
+
+void ECSubWrite::dump(Formatter *f) const
+{
+ f->dump_stream("tid") << tid;
+ f->dump_stream("reqid") << reqid;
+ f->dump_stream("at_version") << at_version;
+ f->dump_stream("trim_to") << trim_to;
+}
+
+void ECSubWrite::generate_test_instances(list<ECSubWrite*> &o)
+{
+ o.push_back(new ECSubWrite());
+ o.back()->tid = 1;
+ o.back()->at_version = eversion_t(2, 100);
+ o.back()->trim_to = eversion_t(1, 40);
+ o.push_back(new ECSubWrite());
+ o.back()->tid = 4;
+ o.back()->reqid = osd_reqid_t(entity_name_t::CLIENT(123), 1, 45678);
+ o.back()->at_version = eversion_t(10, 300);
+ o.back()->trim_to = eversion_t(5, 42);
+}
+
+void ECSubWriteReply::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(from, bl);
+ ::encode(tid, bl);
+ ::encode(last_complete, bl);
+ ::encode(committed, bl);
+ ::encode(applied, bl);
+ ENCODE_FINISH(bl);
+}
+
+void ECSubWriteReply::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(from, bl);
+ ::decode(tid, bl);
+ ::decode(last_complete, bl);
+ ::decode(committed, bl);
+ ::decode(applied, bl);
+ DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubWriteReply &rhs)
+{
+ return lhs
+ << "ECSubWriteReply(tid=" << rhs.tid
+ << ", last_complete=" << rhs.last_complete
+ << ", committed=" << rhs.committed
+ << ", applied=" << rhs.applied << ")";
+}
+
+void ECSubWriteReply::dump(Formatter *f) const
+{
+ f->dump_stream("tid") << tid;
+ f->dump_stream("last_complete") << last_complete;
+ f->dump_stream("committed") << committed;
+ f->dump_stream("applied") << applied;
+}
+
+void ECSubWriteReply::generate_test_instances(list<ECSubWriteReply*>& o)
+{
+ o.push_back(new ECSubWriteReply());
+ o.back()->tid = 20;
+ o.back()->last_complete = eversion_t(100, 2000);
+ o.back()->committed = true;
+ o.push_back(new ECSubWriteReply());
+ o.back()->tid = 80;
+ o.back()->last_complete = eversion_t(50, 200);
+ o.back()->applied = true;
+}
+
+void ECSubRead::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(from, bl);
+ ::encode(tid, bl);
+ ::encode(to_read, bl);
+ ::encode(attrs_to_read, bl);
+ ENCODE_FINISH(bl);
+}
+
+void ECSubRead::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(from, bl);
+ ::decode(tid, bl);
+ ::decode(to_read, bl);
+ ::decode(attrs_to_read, bl);
+ DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubRead &rhs)
+{
+ return lhs
+ << "ECSubRead(tid=" << rhs.tid
+ << ", to_read=" << rhs.to_read
+ << ", attrs_to_read=" << rhs.attrs_to_read << ")";
+}
+
+void ECSubRead::dump(Formatter *f) const
+{
+ f->dump_stream("from") << from;
+ f->dump_stream("tid") << tid;
+ f->open_array_section("objects");
+ for (map<hobject_t, list<pair<uint64_t, uint64_t> > >::const_iterator i =
+ to_read.begin();
+ i != to_read.end();
+ ++i) {
+ f->open_object_section("object");
+ f->dump_stream("oid") << i->first;
+ f->open_array_section("extents");
+ for (list<pair<uint64_t, uint64_t> >::const_iterator j =
+ i->second.begin();
+ j != i->second.end();
+ ++j) {
+ f->open_object_section("extent");
+ f->dump_unsigned("off", j->first);
+ f->dump_unsigned("len", j->second);
+ f->close_section();
+ }
+ f->close_section();
+ f->close_section();
+ }
+ f->close_section();
+
+ f->open_array_section("object_attrs_requested");
+ for (set<hobject_t>::const_iterator i = attrs_to_read.begin();
+ i != attrs_to_read.end();
+ ++i) {
+ f->open_object_section("object");
+ f->dump_stream("oid") << *i;
+ f->close_section();
+ }
+ f->close_section();
+}
+
+void ECSubRead::generate_test_instances(list<ECSubRead*>& o)
+{
+ hobject_t hoid1(sobject_t("asdf", 1));
+ hobject_t hoid2(sobject_t("asdf2", CEPH_NOSNAP));
+ o.push_back(new ECSubRead());
+ o.back()->from = pg_shard_t(2, 255);
+ o.back()->tid = 1;
+ o.back()->to_read[hoid1].push_back(make_pair(100, 200));
+ o.back()->to_read[hoid1].push_back(make_pair(400, 600));
+ o.back()->to_read[hoid2].push_back(make_pair(400, 600));
+ o.back()->attrs_to_read.insert(hoid1);
+ o.push_back(new ECSubRead());
+ o.back()->from = pg_shard_t(2, 255);
+ o.back()->tid = 300;
+ o.back()->to_read[hoid1].push_back(make_pair(300, 200));
+ o.back()->to_read[hoid2].push_back(make_pair(400, 600));
+ o.back()->to_read[hoid2].push_back(make_pair(2000, 600));
+ o.back()->attrs_to_read.insert(hoid2);
+}
+
+void ECSubReadReply::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(from, bl);
+ ::encode(tid, bl);
+ ::encode(buffers_read, bl);
+ ::encode(attrs_read, bl);
+ ::encode(errors, bl);
+ ENCODE_FINISH(bl);
+}
+
+void ECSubReadReply::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(from, bl);
+ ::decode(tid, bl);
+ ::decode(buffers_read, bl);
+ ::decode(attrs_read, bl);
+ ::decode(errors, bl);
+ DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubReadReply &rhs)
+{
+ return lhs
+ << "ECSubReadReply(tid=" << rhs.tid
+ << ", attrs_read=" << rhs.attrs_read.size()
+ << ")";
+}
+
+void ECSubReadReply::dump(Formatter *f) const
+{
+ f->dump_stream("from") << from;
+ f->dump_stream("tid") << tid;
+ f->open_array_section("buffers_read");
+ for (map<hobject_t, list<pair<uint64_t, bufferlist> > >::const_iterator i =
+ buffers_read.begin();
+ i != buffers_read.end();
+ ++i) {
+ f->open_object_section("object");
+ f->dump_stream("oid") << i->first;
+ f->open_array_section("data");
+ for (list<pair<uint64_t, bufferlist> >::const_iterator j =
+ i->second.begin();
+ j != i->second.end();
+ ++j) {
+ f->open_object_section("extent");
+ f->dump_unsigned("off", j->first);
+ f->dump_unsigned("buf_len", j->second.length());
+ f->close_section();
+ }
+ f->close_section();
+ f->close_section();
+ }
+ f->close_section();
+
+ f->open_array_section("attrs_returned");
+ for (map<hobject_t, map<string, bufferlist> >::const_iterator i =
+ attrs_read.begin();
+ i != attrs_read.end();
+ ++i) {
+ f->open_object_section("object_attrs");
+ f->dump_stream("oid") << i->first;
+ f->open_array_section("attrs");
+ for (map<string, bufferlist>::const_iterator j = i->second.begin();
+ j != i->second.end();
+ ++j) {
+ f->open_object_section("attr");
+ f->dump_string("attr", j->first);
+ f->dump_unsigned("val_len", j->second.length());
+ f->close_section();
+ }
+ f->close_section();
+ f->close_section();
+ }
+ f->close_section();
+
+ f->open_array_section("errors");
+ for (map<hobject_t, int>::const_iterator i = errors.begin();
+ i != errors.end();
+ ++i) {
+ f->open_object_section("error_pair");
+ f->dump_stream("oid") << i->first;
+ f->dump_int("error", i->second);
+ f->close_section();
+ }
+ f->close_section();
+}
+
+void ECSubReadReply::generate_test_instances(list<ECSubReadReply*>& o)
+{
+ hobject_t hoid1(sobject_t("asdf", 1));
+ hobject_t hoid2(sobject_t("asdf2", CEPH_NOSNAP));
+ bufferlist bl;
+ bl.append_zero(100);
+ bufferlist bl2;
+ bl2.append_zero(200);
+ o.push_back(new ECSubReadReply());
+ o.back()->from = pg_shard_t(2, 255);
+ o.back()->tid = 1;
+ o.back()->buffers_read[hoid1].push_back(make_pair(20, bl));
+ o.back()->buffers_read[hoid1].push_back(make_pair(2000, bl2));
+ o.back()->buffers_read[hoid2].push_back(make_pair(0, bl));
+ o.back()->attrs_read[hoid1]["foo"] = bl;
+ o.back()->attrs_read[hoid1]["_"] = bl2;
+ o.push_back(new ECSubReadReply());
+ o.back()->from = pg_shard_t(2, 255);
+ o.back()->tid = 300;
+ o.back()->buffers_read[hoid2].push_back(make_pair(0, bl2));
+ o.back()->attrs_read[hoid2]["foo"] = bl;
+ o.back()->attrs_read[hoid2]["_"] = bl2;
+ o.back()->errors[hoid1] = -2;
+}
diff --git a/src/osd/ECMsgTypes.h b/src/osd/ECMsgTypes.h
new file mode 100644
index 0000000..cadeb25
--- /dev/null
+++ b/src/osd/ECMsgTypes.h
@@ -0,0 +1,108 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef ECBMSGTYPES_H
+#define ECBMSGTYPES_H
+
+#include "osd_types.h"
+#include "include/buffer.h"
+#include "os/ObjectStore.h"
+
+struct ECSubWrite {
+ pg_shard_t from;
+ tid_t tid;
+ osd_reqid_t reqid;
+ hobject_t soid;
+ pg_stat_t stats;
+ ObjectStore::Transaction t;
+ eversion_t at_version;
+ eversion_t trim_to;
+ vector<pg_log_entry_t> log_entries;
+ set<hobject_t> temp_added;
+ set<hobject_t> temp_removed;
+ ECSubWrite() {}
+ ECSubWrite(
+ pg_shard_t from,
+ tid_t tid,
+ osd_reqid_t reqid,
+ hobject_t soid,
+ pg_stat_t stats,
+ ObjectStore::Transaction t,
+ eversion_t at_version,
+ eversion_t trim_to,
+ vector<pg_log_entry_t> log_entries,
+ const set<hobject_t> &temp_added,
+ const set<hobject_t> &temp_removed)
+ : from(from), tid(tid), reqid(reqid),
+ soid(soid), stats(stats), t(t),
+ at_version(at_version),
+ trim_to(trim_to), log_entries(log_entries),
+ temp_added(temp_added),
+ temp_removed(temp_removed) {}
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<ECSubWrite*>& o);
+};
+WRITE_CLASS_ENCODER(ECSubWrite)
+
+struct ECSubWriteReply {
+ pg_shard_t from;
+ tid_t tid;
+ eversion_t last_complete;
+ bool committed;
+ bool applied;
+ ECSubWriteReply() : committed(false), applied(false) {}
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<ECSubWriteReply*>& o);
+};
+WRITE_CLASS_ENCODER(ECSubWriteReply)
+
+struct ECSubRead {
+ pg_shard_t from;
+ tid_t tid;
+ map<hobject_t, list<pair<uint64_t, uint64_t> > > to_read;
+ set<hobject_t> attrs_to_read;
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<ECSubRead*>& o);
+};
+WRITE_CLASS_ENCODER(ECSubRead)
+
+struct ECSubReadReply {
+ pg_shard_t from;
+ tid_t tid;
+ map<hobject_t, list<pair<uint64_t, bufferlist> > > buffers_read;
+ map<hobject_t, map<string, bufferlist> > attrs_read;
+ map<hobject_t, int> errors;
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<ECSubReadReply*>& o);
+};
+WRITE_CLASS_ENCODER(ECSubReadReply)
+
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubWrite &rhs);
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubWriteReply &rhs);
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubRead &rhs);
+std::ostream &operator<<(
+ std::ostream &lhs, const ECSubReadReply &rhs);
+
+#endif
diff --git a/src/osd/ECTransaction.cc b/src/osd/ECTransaction.cc
new file mode 100644
index 0000000..c25023c
--- /dev/null
+++ b/src/osd/ECTransaction.cc
@@ -0,0 +1,283 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <boost/variant.hpp>
+#include <boost/optional.hpp>
+#include <iostream>
+#include <vector>
+#include <sstream>
+
+#include "ECBackend.h"
+#include "ECUtil.h"
+#include "os/ObjectStore.h"
+
+struct AppendObjectsGenerator: public boost::static_visitor<void> {
+ typedef void result_type;
+ set<hobject_t> *out;
+ AppendObjectsGenerator(set<hobject_t> *out) : out(out) {}
+ void operator()(const ECTransaction::AppendOp &op) {
+ out->insert(op.oid);
+ }
+ void operator()(const ECTransaction::TouchOp &op) {}
+ void operator()(const ECTransaction::CloneOp &op) {
+ out->insert(op.source);
+ out->insert(op.target);
+ }
+ void operator()(const ECTransaction::RenameOp &op) {
+ out->insert(op.source);
+ out->insert(op.destination);
+ }
+ void operator()(const ECTransaction::StashOp &op) {
+ out->insert(op.oid);
+ }
+ void operator()(const ECTransaction::RemoveOp &op) {
+ out->insert(op.oid);
+ }
+ void operator()(const ECTransaction::SetAttrsOp &op) {}
+ void operator()(const ECTransaction::RmAttrOp &op) {}
+ void operator()(const ECTransaction::AllocHintOp &op) {}
+ void operator()(const ECTransaction::NoOp &op) {}
+};
+void ECTransaction::get_append_objects(
+ set<hobject_t> *out) const
+{
+ AppendObjectsGenerator gen(out);
+ reverse_visit(gen);
+}
+
+struct TransGenerator : public boost::static_visitor<void> {
+ typedef void result_type;
+ map<hobject_t, ECUtil::HashInfoRef> &hash_infos;
+
+ ErasureCodeInterfaceRef &ecimpl;
+ const pg_t pgid;
+ const ECUtil::stripe_info_t sinfo;
+ map<shard_id_t, ObjectStore::Transaction> *trans;
+ set<int> want;
+ set<hobject_t> *temp_added;
+ set<hobject_t> *temp_removed;
+ stringstream *out;
+ TransGenerator(
+ map<hobject_t, ECUtil::HashInfoRef> &hash_infos,
+ ErasureCodeInterfaceRef &ecimpl,
+ pg_t pgid,
+ const ECUtil::stripe_info_t &sinfo,
+ map<shard_id_t, ObjectStore::Transaction> *trans,
+ set<hobject_t> *temp_added,
+ set<hobject_t> *temp_removed,
+ stringstream *out)
+ : hash_infos(hash_infos),
+ ecimpl(ecimpl), pgid(pgid),
+ sinfo(sinfo),
+ trans(trans),
+ temp_added(temp_added), temp_removed(temp_removed),
+ out(out) {
+ for (unsigned i = 0; i < ecimpl->get_chunk_count(); ++i) {
+ want.insert(i);
+ }
+ }
+
+ coll_t get_coll_ct(shard_id_t shard, const hobject_t &hoid) {
+ if (hoid.is_temp()) {
+ temp_removed->erase(hoid);
+ temp_added->insert(hoid);
+ }
+ return get_coll(shard, hoid);
+ }
+ coll_t get_coll_rm(shard_id_t shard, const hobject_t &hoid) {
+ if (hoid.is_temp()) {
+ temp_added->erase(hoid);
+ temp_removed->insert(hoid);
+ }
+ return get_coll(shard, hoid);
+ }
+ coll_t get_coll(shard_id_t shard, const hobject_t &hoid) {
+ if (hoid.is_temp())
+ return coll_t::make_temp_coll(spg_t(pgid, shard));
+ else
+ return coll_t(spg_t(pgid, shard));
+ }
+
+ void operator()(const ECTransaction::TouchOp &op) {
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.touch(
+ get_coll_ct(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first));
+ }
+ }
+ void operator()(const ECTransaction::AppendOp &op) {
+ uint64_t offset = op.off;
+ bufferlist bl(op.bl);
+ assert(bl.length());
+ assert(offset % sinfo.get_stripe_width() == 0);
+ map<int, bufferlist> buffers;
+
+ assert(hash_infos.count(op.oid));
+ ECUtil::HashInfoRef hinfo = hash_infos[op.oid];
+
+ // align
+ if (bl.length() % sinfo.get_stripe_width())
+ bl.append_zero(
+ sinfo.get_stripe_width() -
+ ((offset + bl.length()) % sinfo.get_stripe_width()));
+ assert(bl.length() - op.bl.length() < sinfo.get_stripe_width());
+ int r = ECUtil::encode(
+ sinfo, ecimpl, bl, want, &buffers);
+
+ hinfo->append(
+ sinfo.aligned_logical_offset_to_chunk_offset(op.off),
+ buffers);
+ bufferlist hbuf;
+ ::encode(
+ *hinfo,
+ hbuf);
+
+ assert(r == 0);
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ assert(buffers.count(i->first));
+ bufferlist &enc_bl = buffers[i->first];
+ i->second.write(
+ get_coll_ct(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+ sinfo.logical_to_prev_chunk_offset(
+ offset),
+ enc_bl.length(),
+ enc_bl);
+ i->second.setattr(
+ get_coll_ct(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+ ECUtil::get_hinfo_key(),
+ hbuf);
+ }
+ }
+ void operator()(const ECTransaction::CloneOp &op) {
+ assert(hash_infos.count(op.source));
+ assert(hash_infos.count(op.target));
+ *(hash_infos[op.target]) = *(hash_infos[op.source]);
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.clone(
+ get_coll_ct(i->first, op.source),
+ ghobject_t(op.source, ghobject_t::NO_GEN, i->first),
+ ghobject_t(op.target, ghobject_t::NO_GEN, i->first));
+ }
+ }
+ void operator()(const ECTransaction::RenameOp &op) {
+ assert(hash_infos.count(op.source));
+ assert(hash_infos.count(op.destination));
+ *(hash_infos[op.destination]) = *(hash_infos[op.source]);
+ hash_infos[op.source]->clear();
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.collection_move_rename(
+ get_coll_rm(i->first, op.source),
+ ghobject_t(op.source, ghobject_t::NO_GEN, i->first),
+ get_coll_ct(i->first, op.destination),
+ ghobject_t(op.destination, ghobject_t::NO_GEN, i->first));
+ }
+ }
+ void operator()(const ECTransaction::StashOp &op) {
+ assert(hash_infos.count(op.oid));
+ hash_infos[op.oid]->clear();
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ coll_t cid(get_coll_rm(i->first, op.oid));
+ i->second.collection_move_rename(
+ cid,
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+ cid,
+ ghobject_t(op.oid, op.version, i->first));
+ }
+ }
+ void operator()(const ECTransaction::RemoveOp &op) {
+ assert(hash_infos.count(op.oid));
+ hash_infos[op.oid]->clear();
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.remove(
+ get_coll_rm(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first));
+ }
+ }
+ void operator()(const ECTransaction::SetAttrsOp &op) {
+ map<string, bufferlist> attrs(op.attrs);
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.setattrs(
+ get_coll_ct(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+ attrs);
+ }
+ }
+ void operator()(const ECTransaction::RmAttrOp &op) {
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.rmattr(
+ get_coll_ct(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+ op.key);
+ }
+ }
+ void operator()(const ECTransaction::AllocHintOp &op) {
+ // logical_to_next_chunk_offset() scales down both aligned and
+ // unaligned offsets
+ uint64_t object_size = sinfo.logical_to_next_chunk_offset(
+ op.expected_object_size);
+ uint64_t write_size = sinfo.logical_to_next_chunk_offset(
+ op.expected_write_size);
+
+ for (map<shard_id_t, ObjectStore::Transaction>::iterator i = trans->begin();
+ i != trans->end();
+ ++i) {
+ i->second.set_alloc_hint(
+ get_coll_ct(i->first, op.oid),
+ ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+ object_size, write_size);
+ }
+ }
+ void operator()(const ECTransaction::NoOp &op) {}
+};
+
+
+void ECTransaction::generate_transactions(
+ map<hobject_t, ECUtil::HashInfoRef> &hash_infos,
+ ErasureCodeInterfaceRef &ecimpl,
+ pg_t pgid,
+ const ECUtil::stripe_info_t &sinfo,
+ map<shard_id_t, ObjectStore::Transaction> *transactions,
+ set<hobject_t> *temp_added,
+ set<hobject_t> *temp_removed,
+ stringstream *out) const
+{
+ TransGenerator gen(
+ hash_infos,
+ ecimpl,
+ pgid,
+ sinfo,
+ transactions,
+ temp_added,
+ temp_removed,
+ out);
+ visit(gen);
+}
diff --git a/src/osd/ECTransaction.h b/src/osd/ECTransaction.h
new file mode 100644
index 0000000..7b104c7
--- /dev/null
+++ b/src/osd/ECTransaction.h
@@ -0,0 +1,207 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef ECTRANSACTION_H
+#define ECTRANSACTION_H
+
+#include "OSD.h"
+#include "PGBackend.h"
+#include "osd_types.h"
+#include "ECUtil.h"
+#include <boost/optional.hpp>
+#include "erasure-code/ErasureCodeInterface.h"
+
+class ECTransaction : public PGBackend::PGTransaction {
+public:
+ struct AppendOp {
+ hobject_t oid;
+ uint64_t off;
+ bufferlist bl;
+ AppendOp(const hobject_t &oid, uint64_t off, bufferlist &bl)
+ : oid(oid), off(off), bl(bl) {}
+ };
+ struct CloneOp {
+ hobject_t source;
+ hobject_t target;
+ CloneOp(const hobject_t &source, const hobject_t &target)
+ : source(source), target(target) {}
+ };
+ struct RenameOp {
+ hobject_t source;
+ hobject_t destination;
+ RenameOp(const hobject_t &source, const hobject_t &destination)
+ : source(source), destination(destination) {}
+ };
+ struct StashOp {
+ hobject_t oid;
+ version_t version;
+ StashOp(const hobject_t &oid, version_t version)
+ : oid(oid), version(version) {}
+ };
+ struct TouchOp {
+ hobject_t oid;
+ TouchOp(const hobject_t &oid) : oid(oid) {}
+ };
+ struct RemoveOp {
+ hobject_t oid;
+ RemoveOp(const hobject_t &oid) : oid(oid) {}
+ };
+ struct SetAttrsOp {
+ hobject_t oid;
+ map<string, bufferlist> attrs;
+ SetAttrsOp(const hobject_t &oid, map<string, bufferlist> &_attrs)
+ : oid(oid) {
+ attrs.swap(_attrs);
+ }
+ SetAttrsOp(const hobject_t &oid, const string &key, bufferlist &val)
+ : oid(oid) {
+ attrs.insert(make_pair(key, val));
+ }
+ };
+ struct RmAttrOp {
+ hobject_t oid;
+ string key;
+ RmAttrOp(const hobject_t &oid, const string &key) : oid(oid), key(key) {}
+ };
+ struct AllocHintOp {
+ hobject_t oid;
+ uint64_t expected_object_size;
+ uint64_t expected_write_size;
+ AllocHintOp(const hobject_t &oid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size)
+ : oid(oid), expected_object_size(expected_object_size),
+ expected_write_size(expected_write_size) {}
+ };
+ struct NoOp {};
+ typedef boost::variant<
+ AppendOp,
+ CloneOp,
+ RenameOp,
+ StashOp,
+ TouchOp,
+ RemoveOp,
+ SetAttrsOp,
+ RmAttrOp,
+ AllocHintOp,
+ NoOp> Op;
+ list<Op> ops;
+ uint64_t written;
+
+ ECTransaction() : written(0) {}
+ /// Write
+ void touch(
+ const hobject_t &hoid) {
+ bufferlist bl;
+ ops.push_back(TouchOp(hoid));
+ }
+ void append(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist &bl) {
+ if (len == 0) {
+ touch(hoid);
+ return;
+ }
+ written += len;
+ assert(len == bl.length());
+ ops.push_back(AppendOp(hoid, off, bl));
+ }
+ void stash(
+ const hobject_t &hoid,
+ version_t former_version) {
+ ops.push_back(StashOp(hoid, former_version));
+ }
+ void remove(
+ const hobject_t &hoid) {
+ ops.push_back(RemoveOp(hoid));
+ }
+ void setattrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> &attrs) {
+ ops.push_back(SetAttrsOp(hoid, attrs));
+ }
+ void setattr(
+ const hobject_t &hoid,
+ const string &attrname,
+ bufferlist &bl) {
+ ops.push_back(SetAttrsOp(hoid, attrname, bl));
+ }
+ void rmattr(
+ const hobject_t &hoid,
+ const string &attrname) {
+ ops.push_back(RmAttrOp(hoid, attrname));
+ }
+ void clone(
+ const hobject_t &from,
+ const hobject_t &to) {
+ ops.push_back(CloneOp(from, to));
+ }
+ void rename(
+ const hobject_t &from,
+ const hobject_t &to) {
+ ops.push_back(RenameOp(from, to));
+ }
+ void set_alloc_hint(
+ const hobject_t &hoid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size) {
+ ops.push_back(AllocHintOp(hoid, expected_object_size, expected_write_size));
+ }
+
+ void append(PGTransaction *_to_append) {
+ ECTransaction *to_append = static_cast<ECTransaction*>(_to_append);
+ written += to_append->written;
+ to_append->written = 0;
+ ops.splice(ops.end(), to_append->ops,
+ to_append->ops.begin(), to_append->ops.end());
+ }
+ void nop() {
+ ops.push_back(NoOp());
+ }
+ bool empty() const {
+ return ops.empty();
+ }
+ uint64_t get_bytes_written() const {
+ return written;
+ }
+ template <typename T>
+ void visit(T &vis) const {
+ for (list<Op>::const_iterator i = ops.begin(); i != ops.end(); ++i) {
+ boost::apply_visitor(vis, *i);
+ }
+ }
+ template <typename T>
+ void reverse_visit(T &vis) const {
+ for (list<Op>::const_reverse_iterator i = ops.rbegin();
+ i != ops.rend();
+ ++i) {
+ boost::apply_visitor(vis, *i);
+ }
+ }
+ void get_append_objects(
+ set<hobject_t> *out) const;
+ void generate_transactions(
+ map<hobject_t, ECUtil::HashInfoRef> &hash_infos,
+ ErasureCodeInterfaceRef &ecimpl,
+ pg_t pgid,
+ const ECUtil::stripe_info_t &sinfo,
+ map<shard_id_t, ObjectStore::Transaction> *transactions,
+ set<hobject_t> *temp_added,
+ set<hobject_t> *temp_removed,
+ stringstream *out = 0) const;
+};
+
+#endif
diff --git a/src/osd/ECUtil.cc b/src/osd/ECUtil.cc
new file mode 100644
index 0000000..1f3b458
--- /dev/null
+++ b/src/osd/ECUtil.cc
@@ -0,0 +1,196 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include <errno.h>
+#include "include/encoding.h"
+#include "ECUtil.h"
+
+int ECUtil::decode(
+ const stripe_info_t &sinfo,
+ ErasureCodeInterfaceRef &ec_impl,
+ map<int, bufferlist> &to_decode,
+ bufferlist *out) {
+
+ uint64_t total_chunk_size = to_decode.begin()->second.length();
+
+ assert(to_decode.size());
+ assert(total_chunk_size % sinfo.get_chunk_size() == 0);
+ assert(out);
+ assert(out->length() == 0);
+
+ for (map<int, bufferlist>::iterator i = to_decode.begin();
+ i != to_decode.end();
+ ++i) {
+ assert(i->second.length() == total_chunk_size);
+ }
+
+ if (total_chunk_size == 0)
+ return 0;
+
+ for (uint64_t i = 0; i < total_chunk_size; i += sinfo.get_chunk_size()) {
+ map<int, bufferlist> chunks;
+ for (map<int, bufferlist>::iterator j = to_decode.begin();
+ j != to_decode.end();
+ ++j) {
+ chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size());
+ }
+ bufferlist bl;
+ int r = ec_impl->decode_concat(chunks, &bl);
+ assert(bl.length() == sinfo.get_stripe_width());
+ assert(r == 0);
+ out->claim_append(bl);
+ }
+ return 0;
+}
+
+int ECUtil::decode(
+ const stripe_info_t &sinfo,
+ ErasureCodeInterfaceRef &ec_impl,
+ map<int, bufferlist> &to_decode,
+ map<int, bufferlist*> &out) {
+
+ uint64_t total_chunk_size = to_decode.begin()->second.length();
+
+ assert(to_decode.size());
+ assert(total_chunk_size % sinfo.get_chunk_size() == 0);
+
+ for (map<int, bufferlist>::iterator i = to_decode.begin();
+ i != to_decode.end();
+ ++i) {
+ assert(i->second.length() == total_chunk_size);
+ }
+
+ if (total_chunk_size == 0)
+ return 0;
+
+ set<int> need;
+ for (map<int, bufferlist*>::iterator i = out.begin();
+ i != out.end();
+ ++i) {
+ assert(i->second);
+ assert(i->second->length() == 0);
+ need.insert(i->first);
+ }
+
+ for (uint64_t i = 0; i < total_chunk_size; i += sinfo.get_chunk_size()) {
+ map<int, bufferlist> chunks;
+ for (map<int, bufferlist>::iterator j = to_decode.begin();
+ j != to_decode.end();
+ ++j) {
+ chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size());
+ }
+ map<int, bufferlist> out_bls;
+ int r = ec_impl->decode(need, chunks, &out_bls);
+ assert(r == 0);
+ for (map<int, bufferlist*>::iterator j = out.begin();
+ j != out.end();
+ ++j) {
+ assert(out_bls.count(j->first));
+ assert(out_bls[j->first].length() == sinfo.get_chunk_size());
+ j->second->claim_append(out_bls[j->first]);
+ }
+ }
+ for (map<int, bufferlist*>::iterator i = out.begin();
+ i != out.end();
+ ++i) {
+ assert(i->second->length() == total_chunk_size);
+ }
+ return 0;
+}
+
+int ECUtil::encode(
+ const stripe_info_t &sinfo,
+ ErasureCodeInterfaceRef &ec_impl,
+ bufferlist &in,
+ const set<int> &want,
+ map<int, bufferlist> *out) {
+
+ uint64_t logical_size = in.length();
+
+ assert(logical_size % sinfo.get_stripe_width() == 0);
+ assert(out);
+ assert(out->empty());
+
+ if (logical_size == 0)
+ return 0;
+
+ for (uint64_t i = 0; i < logical_size; i += sinfo.get_stripe_width()) {
+ map<int, bufferlist> encoded;
+ bufferlist buf;
+ buf.substr_of(in, i, sinfo.get_stripe_width());
+ int r = ec_impl->encode(want, buf, &encoded);
+ assert(r == 0);
+ for (map<int, bufferlist>::iterator i = encoded.begin();
+ i != encoded.end();
+ ++i) {
+ assert(i->second.length() == sinfo.get_chunk_size());
+ (*out)[i->first].claim_append(i->second);
+ }
+ }
+
+ for (map<int, bufferlist>::iterator i = out->begin();
+ i != out->end();
+ ++i) {
+ assert(i->second.length() % sinfo.get_chunk_size() == 0);
+ assert(
+ sinfo.aligned_chunk_offset_to_logical_offset(i->second.length()) ==
+ logical_size);
+ }
+ return 0;
+}
+
+void ECUtil::HashInfo::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(total_chunk_size, bl);
+ ::encode(cumulative_shard_hashes, bl);
+ ENCODE_FINISH(bl);
+}
+
+void ECUtil::HashInfo::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(total_chunk_size, bl);
+ ::decode(cumulative_shard_hashes, bl);
+ DECODE_FINISH(bl);
+}
+
+void ECUtil::HashInfo::dump(Formatter *f) const
+{
+ f->dump_unsigned("total_chunk_size", total_chunk_size);
+ f->open_object_section("cumulative_shard_hashes");
+ for (unsigned i = 0; i != cumulative_shard_hashes.size(); ++i) {
+ f->open_object_section("hash");
+ f->dump_unsigned("shard", i);
+ f->dump_unsigned("hash", cumulative_shard_hashes[i]);
+ f->close_section();
+ }
+ f->close_section();
+}
+
+void ECUtil::HashInfo::generate_test_instances(list<HashInfo*>& o)
+{
+ o.push_back(new HashInfo(3));
+ {
+ bufferlist bl;
+ bl.append_zero(20);
+ map<int, bufferlist> buffers;
+ buffers[0] = bl;
+ buffers[1] = bl;
+ buffers[2] = bl;
+ o.back()->append(0, buffers);
+ o.back()->append(20, buffers);
+ }
+ o.push_back(new HashInfo(4));
+}
+
+const string HINFO_KEY = "hinfo_key";
+
+bool ECUtil::is_hinfo_key_string(const string &key)
+{
+ return key == HINFO_KEY;
+}
+
+const string &ECUtil::get_hinfo_key()
+{
+ return HINFO_KEY;
+}
diff --git a/src/osd/ECUtil.h b/src/osd/ECUtil.h
new file mode 100644
index 0000000..52d79aa
--- /dev/null
+++ b/src/osd/ECUtil.h
@@ -0,0 +1,154 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef ECUTIL_H
+#define ECUTIL_H
+
+#include <map>
+#include <set>
+
+#include "include/memory.h"
+#include "erasure-code/ErasureCodeInterface.h"
+#include "include/buffer.h"
+#include "include/assert.h"
+#include "include/encoding.h"
+#include "common/Formatter.h"
+
+namespace ECUtil {
+
+const uint64_t CHUNK_ALIGNMENT = 64;
+const uint64_t CHUNK_INFO = 8;
+const uint64_t CHUNK_PADDING = 8;
+const uint64_t CHUNK_OVERHEAD = 16; // INFO + PADDING
+
+class stripe_info_t {
+ const uint64_t stripe_size;
+ const uint64_t stripe_width;
+ const uint64_t chunk_size;
+public:
+ stripe_info_t(uint64_t stripe_size, uint64_t stripe_width)
+ : stripe_size(stripe_size), stripe_width(stripe_width),
+ chunk_size(stripe_width / stripe_size) {
+ assert(stripe_width % stripe_size == 0);
+ }
+ uint64_t get_stripe_width() const {
+ return stripe_width;
+ }
+ uint64_t get_chunk_size() const {
+ return chunk_size;
+ }
+ uint64_t logical_to_prev_chunk_offset(uint64_t offset) const {
+ return (offset / stripe_width) * chunk_size;
+ }
+ uint64_t logical_to_next_chunk_offset(uint64_t offset) const {
+ return ((offset + stripe_width - 1)/ stripe_width) * chunk_size;
+ }
+ uint64_t logical_to_prev_stripe_offset(uint64_t offset) const {
+ return offset - (offset % stripe_width);
+ }
+ uint64_t logical_to_next_stripe_offset(uint64_t offset) const {
+ return offset % stripe_width ?
+ offset - (offset % stripe_width) + stripe_width :
+ offset;
+ }
+ uint64_t aligned_logical_offset_to_chunk_offset(uint64_t offset) const {
+ assert(offset % stripe_width == 0);
+ return (offset / stripe_width) * chunk_size;
+ }
+ uint64_t aligned_chunk_offset_to_logical_offset(uint64_t offset) const {
+ assert(offset % chunk_size == 0);
+ return (offset / chunk_size) * stripe_width;
+ }
+ pair<uint64_t, uint64_t> aligned_offset_len_to_chunk(
+ pair<uint64_t, uint64_t> in) const {
+ return make_pair(
+ aligned_logical_offset_to_chunk_offset(in.first),
+ aligned_logical_offset_to_chunk_offset(in.second));
+ }
+ pair<uint64_t, uint64_t> offset_len_to_stripe_bounds(
+ pair<uint64_t, uint64_t> in) const {
+ uint64_t off = logical_to_prev_stripe_offset(in.first);
+ uint64_t len = logical_to_next_stripe_offset(
+ (in.first - off) + in.second);
+ return make_pair(off, len);
+ }
+};
+
+int decode(
+ const stripe_info_t &sinfo,
+ ErasureCodeInterfaceRef &ec_impl,
+ map<int, bufferlist> &to_decode,
+ bufferlist *out);
+
+int decode(
+ const stripe_info_t &sinfo,
+ ErasureCodeInterfaceRef &ec_impl,
+ map<int, bufferlist> &to_decode,
+ map<int, bufferlist*> &out);
+
+int encode(
+ const stripe_info_t &sinfo,
+ ErasureCodeInterfaceRef &ec_impl,
+ bufferlist &in,
+ const set<int> &want,
+ map<int, bufferlist> *out);
+
+class HashInfo {
+ uint64_t total_chunk_size;
+ vector<uint32_t> cumulative_shard_hashes;
+public:
+ HashInfo() : total_chunk_size(0) {}
+ HashInfo(unsigned num_chunks)
+ : total_chunk_size(0),
+ cumulative_shard_hashes(num_chunks, -1) {}
+ void append(uint64_t old_size, map<int, bufferlist> &to_append) {
+ assert(to_append.size() == cumulative_shard_hashes.size());
+ assert(old_size == total_chunk_size);
+ uint64_t size_to_append = to_append.begin()->second.length();
+ for (map<int, bufferlist>::iterator i = to_append.begin();
+ i != to_append.end();
+ ++i) {
+ assert(size_to_append == i->second.length());
+ assert((unsigned)i->first < cumulative_shard_hashes.size());
+ uint32_t new_hash = i->second.crc32c(cumulative_shard_hashes[i->first]);
+ cumulative_shard_hashes[i->first] = new_hash;
+ }
+ total_chunk_size += size_to_append;
+ }
+ void clear() {
+ total_chunk_size = 0;
+ cumulative_shard_hashes = vector<uint32_t>(
+ cumulative_shard_hashes.size(),
+ -1);
+ }
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<HashInfo*>& o);
+ uint32_t get_chunk_hash(int shard) const {
+ assert((unsigned)shard < cumulative_shard_hashes.size());
+ return cumulative_shard_hashes[shard];
+ }
+ uint64_t get_total_chunk_size() const {
+ return total_chunk_size;
+ }
+};
+typedef ceph::shared_ptr<HashInfo> HashInfoRef;
+
+bool is_hinfo_key_string(const string &key);
+const string &get_hinfo_key();
+
+};
+WRITE_CLASS_ENCODER(ECUtil::HashInfo)
+#endif
diff --git a/src/osd/ErasureCodePluginJerasure/Makefile.am b/src/osd/ErasureCodePluginJerasure/Makefile.am
deleted file mode 100644
index 51ef83b..0000000
--- a/src/osd/ErasureCodePluginJerasure/Makefile.am
+++ /dev/null
@@ -1,28 +0,0 @@
-# jerasure plugin
-libec_jerasure_la_SOURCES = \
- osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc \
- osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc \
- osd/ErasureCodePluginJerasure/cauchy.c \
- osd/ErasureCodePluginJerasure/galois.c \
- osd/ErasureCodePluginJerasure/jerasure.c \
- osd/ErasureCodePluginJerasure/liberation.c \
- osd/ErasureCodePluginJerasure/reed_sol.c
-
-noinst_HEADERS += \
- osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \
- osd/ErasureCodePluginJerasure/cauchy.h \
- osd/ErasureCodePluginJerasure/galois.h \
- osd/ErasureCodePluginJerasure/jerasure.h \
- osd/ErasureCodePluginJerasure/liberation.h \
- osd/ErasureCodePluginJerasure/reed_sol.h \
- osd/ErasureCodePluginJerasure/vectorop.h
-
-libec_jerasure_la_CFLAGS = ${AM_CFLAGS}
-libec_jerasure_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_jerasure_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0
-if LINUX
-libec_jerasure_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
-endif
-
-erasure_codelib_LTLIBRARIES += libec_jerasure.la
diff --git a/src/osd/HitSet.h b/src/osd/HitSet.h
index b6acce9..391dd63 100644
--- a/src/osd/HitSet.h
+++ b/src/osd/HitSet.h
@@ -170,6 +170,8 @@ private:
WRITE_CLASS_ENCODER(HitSet);
WRITE_CLASS_ENCODER(HitSet::Params);
+typedef boost::shared_ptr<HitSet> HitSetRef;
+
ostream& operator<<(ostream& out, const HitSet::Params& p);
/**
diff --git a/src/osd/Makefile.am b/src/osd/Makefile.am
index 9bbc7e4..3e91d7d 100644
--- a/src/osd/Makefile.am
+++ b/src/osd/Makefile.am
@@ -1,15 +1,11 @@
-## erasure code plugins
-erasure_codelibdir = $(pkglibdir)/erasure-code
-erasure_codelib_LTLIBRARIES =
-
-include osd/ErasureCodePluginJerasure/Makefile.am
-
libosd_la_SOURCES = \
- osd/ErasureCodePlugin.cc \
osd/PG.cc \
osd/PGLog.cc \
osd/ReplicatedPG.cc \
osd/ReplicatedBackend.cc \
+ osd/ECBackend.cc \
+ osd/ECMsgTypes.cc \
+ osd/ECTransaction.cc \
osd/PGBackend.cc \
osd/Ager.cc \
osd/HitSet.cc \
@@ -21,15 +17,14 @@ libosd_la_SOURCES = \
common/TrackedOp.cc \
osd/SnapMapper.cc \
osd/osd_types.cc \
+ osd/ECUtil.cc \
objclass/class_api.cc
-libosd_la_LIBADD = $(LIBOSDC) $(LIBOS)
+libosd_la_LIBADD = $(LIBOSDC) $(LIBOS) $(LIBERASURE_CODE)
noinst_LTLIBRARIES += libosd.la
noinst_HEADERS += \
osd/Ager.h \
osd/ClassHandler.h \
- osd/ErasureCodeInterface.h \
- osd/ErasureCodePlugin.h \
osd/HitSet.h \
osd/OSD.h \
osd/OSDCap.h \
@@ -42,6 +37,11 @@ noinst_HEADERS += \
osd/ReplicatedPG.h \
osd/PGBackend.h \
osd/ReplicatedBackend.h \
+ osd/TierAgentState.h \
+ osd/ECBackend.h \
+ osd/ECUtil.h \
+ osd/ECMsgTypes.h \
+ osd/ECTransaction.h \
osd/Watch.h \
osd/osd_types.h
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 38bb171..db14031 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -82,6 +82,10 @@
#include "messages/MOSDPGMissing.h"
#include "messages/MBackfillReserve.h"
#include "messages/MRecoveryReserve.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
#include "messages/MOSDAlive.h"
@@ -191,10 +195,16 @@ OSDService::OSDService(OSD *osd) :
pre_publish_lock("OSDService::pre_publish_lock"),
sched_scrub_lock("OSDService::sched_scrub_lock"), scrubs_pending(0),
scrubs_active(0),
+ agent_lock("OSD::agent_lock"),
+ agent_valid_iterator(false),
+ agent_ops(0),
+ agent_active(true),
+ agent_thread(this),
+ agent_stop_flag(false),
objecter_lock("OSD::objecter_lock"),
objecter_timer(osd->client_messenger->cct, objecter_lock),
objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, &objecter_osdmap,
- objecter_lock, objecter_timer)),
+ objecter_lock, objecter_timer, 0, 0)),
objecter_finisher(osd->client_messenger->cct),
objecter_dispatcher(this),
watch_lock("OSD::watch_lock"),
@@ -229,9 +239,9 @@ OSDService::~OSDService()
delete objecter;
}
-void OSDService::_start_split(pg_t parent, const set<pg_t> &children)
+void OSDService::_start_split(spg_t parent, const set<spg_t> &children)
{
- for (set<pg_t>::const_iterator i = children.begin();
+ for (set<spg_t>::const_iterator i = children.begin();
i != children.end();
++i) {
dout(10) << __func__ << ": Starting split on pg " << *i
@@ -245,12 +255,12 @@ void OSDService::_start_split(pg_t parent, const set<pg_t> &children)
}
}
-void OSDService::mark_split_in_progress(pg_t parent, const set<pg_t> &children)
+void OSDService::mark_split_in_progress(spg_t parent, const set<spg_t> &children)
{
Mutex::Locker l(in_progress_split_lock);
- map<pg_t, set<pg_t> >::iterator piter = rev_pending_splits.find(parent);
+ map<spg_t, set<spg_t> >::iterator piter = rev_pending_splits.find(parent);
assert(piter != rev_pending_splits.end());
- for (set<pg_t>::const_iterator i = children.begin();
+ for (set<spg_t>::const_iterator i = children.begin();
i != children.end();
++i) {
assert(piter->second.count(*i));
@@ -266,19 +276,19 @@ void OSDService::mark_split_in_progress(pg_t parent, const set<pg_t> &children)
rev_pending_splits.erase(piter);
}
-void OSDService::cancel_pending_splits_for_parent(pg_t parent)
+void OSDService::cancel_pending_splits_for_parent(spg_t parent)
{
Mutex::Locker l(in_progress_split_lock);
return _cancel_pending_splits_for_parent(parent);
}
-void OSDService::_cancel_pending_splits_for_parent(pg_t parent)
+void OSDService::_cancel_pending_splits_for_parent(spg_t parent)
{
- map<pg_t, set<pg_t> >::iterator piter = rev_pending_splits.find(parent);
+ map<spg_t, set<spg_t> >::iterator piter = rev_pending_splits.find(parent);
if (piter == rev_pending_splits.end())
return;
- for (set<pg_t>::iterator i = piter->second.begin();
+ for (set<spg_t>::iterator i = piter->second.begin();
i != piter->second.end();
++i) {
assert(pending_splits.count(*i));
@@ -293,11 +303,11 @@ void OSDService::_cancel_pending_splits_for_parent(pg_t parent)
void OSDService::_maybe_split_pgid(OSDMapRef old_map,
OSDMapRef new_map,
- pg_t pgid)
+ spg_t pgid)
{
assert(old_map->have_pg_pool(pgid.pool()));
if (pgid.ps() < static_cast<unsigned>(old_map->get_pg_num(pgid.pool()))) {
- set<pg_t> children;
+ set<spg_t> children;
pgid.is_split(old_map->get_pg_num(pgid.pool()),
new_map->get_pg_num(pgid.pool()), &children);
_start_split(pgid, children);
@@ -306,7 +316,7 @@ void OSDService::_maybe_split_pgid(OSDMapRef old_map,
}
}
-void OSDService::init_splits_between(pg_t pgid,
+void OSDService::init_splits_between(spg_t pgid,
OSDMapRef frommap,
OSDMapRef tomap)
{
@@ -317,7 +327,7 @@ void OSDService::init_splits_between(pg_t pgid,
tomap->get_pg_num(pgid.pool()),
NULL)) {
// Ok, a split happened, so we need to walk the osdmaps
- set<pg_t> new_pgs; // pgs to scan on each map
+ set<spg_t> new_pgs; // pgs to scan on each map
new_pgs.insert(pgid);
OSDMapRef curmap(get_map(frommap->get_epoch()));
for (epoch_t e = frommap->get_epoch() + 1;
@@ -326,9 +336,9 @@ void OSDService::init_splits_between(pg_t pgid,
OSDMapRef nextmap(try_get_map(e));
if (!nextmap)
continue;
- set<pg_t> even_newer_pgs; // pgs added in this loop
- for (set<pg_t>::iterator i = new_pgs.begin(); i != new_pgs.end(); ++i) {
- set<pg_t> split_pgs;
+ set<spg_t> even_newer_pgs; // pgs added in this loop
+ for (set<spg_t>::iterator i = new_pgs.begin(); i != new_pgs.end(); ++i) {
+ set<spg_t> split_pgs;
if (i->is_split(curmap->get_pg_num(i->pool()),
nextmap->get_pg_num(i->pool()),
&split_pgs)) {
@@ -347,7 +357,7 @@ void OSDService::expand_pg_num(OSDMapRef old_map,
OSDMapRef new_map)
{
Mutex::Locker l(in_progress_split_lock);
- for (set<pg_t>::iterator i = in_progress_splits.begin();
+ for (set<spg_t>::iterator i = in_progress_splits.begin();
i != in_progress_splits.end();
) {
if (!new_map->have_pg_pool(i->pool())) {
@@ -357,7 +367,7 @@ void OSDService::expand_pg_num(OSDMapRef old_map,
++i;
}
}
- for (map<pg_t, pg_t>::iterator i = pending_splits.begin();
+ for (map<spg_t, spg_t>::iterator i = pending_splits.begin();
i != pending_splits.end();
) {
if (!new_map->have_pg_pool(i->first.pool())) {
@@ -370,17 +380,17 @@ void OSDService::expand_pg_num(OSDMapRef old_map,
}
}
-bool OSDService::splitting(pg_t pgid)
+bool OSDService::splitting(spg_t pgid)
{
Mutex::Locker l(in_progress_split_lock);
return in_progress_splits.count(pgid) ||
pending_splits.count(pgid);
}
-void OSDService::complete_split(const set<pg_t> &pgs)
+void OSDService::complete_split(const set<spg_t> &pgs)
{
Mutex::Locker l(in_progress_split_lock);
- for (set<pg_t>::const_iterator i = pgs.begin();
+ for (set<spg_t>::const_iterator i = pgs.begin();
i != pgs.end();
++i) {
dout(10) << __func__ << ": Completing split on pg " << *i << dendl;
@@ -441,8 +451,84 @@ void OSDService::init()
objecter->init_locked();
}
watch_timer.init();
+
+ agent_thread.create();
+}
+
+void OSDService::activate_map()
+{
+ // wake/unwake the tiering agent
+ agent_lock.Lock();
+ agent_active =
+ !osdmap->test_flag(CEPH_OSDMAP_NOTIERAGENT) &&
+ osd->is_active();
+ agent_cond.Signal();
+ agent_lock.Unlock();
+}
+
+void OSDService::agent_entry()
+{
+ dout(10) << __func__ << " start" << dendl;
+ agent_lock.Lock();
+
+ while (!agent_stop_flag) {
+ if (agent_queue.empty()) {
+ dout(20) << __func__ << " empty queue" << dendl;
+ agent_cond.Wait(agent_lock);
+ continue;
+ }
+ uint64_t level = agent_queue.rbegin()->first;
+ set<PGRef>& top = agent_queue.rbegin()->second;
+ dout(10) << __func__
+ << " tiers " << agent_queue.size()
+ << ", top is " << level
+ << " with pgs " << top.size()
+ << ", ops " << agent_ops << "/"
+ << g_conf->osd_agent_max_ops
+ << (agent_active ? " active" : " NOT ACTIVE")
+ << dendl;
+ dout(20) << __func__ << " oids " << agent_oids << dendl;
+ if (agent_ops >= g_conf->osd_agent_max_ops || top.empty() ||
+ !agent_active) {
+ agent_cond.Wait(agent_lock);
+ continue;
+ }
+
+ if (!agent_valid_iterator || agent_queue_pos == top.end()) {
+ agent_queue_pos = top.begin();
+ agent_valid_iterator = true;
+ }
+ PGRef pg = *agent_queue_pos;
+ int max = g_conf->osd_agent_max_ops - agent_ops;
+ agent_lock.Unlock();
+ pg->agent_work(max);
+ agent_lock.Lock();
+ }
+ agent_lock.Unlock();
+ dout(10) << __func__ << " finish" << dendl;
+}
+
+void OSDService::agent_stop()
+{
+ {
+ Mutex::Locker l(agent_lock);
+
+ // By this time all ops should be cancelled
+ assert(agent_ops == 0);
+ // By this time all PGs are shutdown and dequeued
+ if (!agent_queue.empty()) {
+ set<PGRef>& top = agent_queue.rbegin()->second;
+ derr << "agent queue not empty, for example " << (*top.begin())->info.pgid << dendl;
+ assert(0 == "agent queue not empty");
+ }
+
+ agent_stop_flag = true;
+ agent_cond.Signal();
+ }
+ agent_thread.join();
}
+
#undef dout_prefix
#define dout_prefix *_dout
@@ -525,7 +611,7 @@ int OSD::do_convertfs(ObjectStore *store)
for (vector<coll_t>::iterator i = collections.begin();
i != collections.end();
++i) {
- pg_t pgid;
+ spg_t pgid;
if (i->is_temp(pgid))
recursive_remove_collection(store, *i);
else if (i->to_str() == "convertfs_temp" ||
@@ -818,7 +904,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
heartbeat_dispatcher(this),
stat_lock("OSD::stat_lock"),
finished_lock("OSD::finished_lock"),
- op_tracker(cct),
+ op_tracker(cct, cct->_conf->osd_enable_op_tracker),
test_ops_hook(NULL),
op_wq(this, cct->_conf->osd_op_thread_timeout, &op_tp),
peering_wq(this, cct->_conf->osd_op_thread_timeout, &op_tp),
@@ -828,6 +914,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
debug_drop_pg_create_duration(cct->_conf->osd_debug_drop_pg_create_duration),
debug_drop_pg_create_left(-1),
outstanding_pg_stats(false),
+ timeout_mon_on_pg_stats(true),
up_thru_wanted(0), up_thru_pending(0),
pg_stat_queue_lock("OSD::pg_stat_queue_lock"),
osd_stat_updated(false),
@@ -907,10 +994,24 @@ public:
bool OSD::asok_command(string command, cmdmap_t& cmdmap, string format,
ostream& ss)
{
- if (format == "")
- format = "json-pretty";
Formatter *f = new_formatter(format);
- if (command == "dump_ops_in_flight") {
+ if (!f)
+ f = new_formatter("json-pretty");
+ if (command == "status") {
+ f->open_object_section("status");
+ f->dump_stream("cluster_fsid") << superblock.cluster_fsid;
+ f->dump_stream("osd_fsid") << superblock.osd_fsid;
+ f->dump_unsigned("whoami", superblock.whoami);
+ f->dump_string("state", get_state_name(state));
+ f->dump_unsigned("oldest_map", superblock.oldest_map);
+ f->dump_unsigned("newest_map", superblock.newest_map);
+ osd_lock.Lock();
+ f->dump_unsigned("num_pgs", pg_map.size());
+ osd_lock.Unlock();
+ f->close_section();
+ } else if (command == "flush_journal") {
+ store->sync_and_flush();
+ } else if (command == "dump_ops_in_flight") {
op_tracker.dump_ops_in_flight(f);
} else if (command == "dump_historic_ops") {
op_tracker.dump_historic_ops(f);
@@ -938,7 +1039,7 @@ bool OSD::asok_command(string command, cmdmap_t& cmdmap, string format,
list<obj_watch_item_t> watchers;
osd_lock.Lock();
// scan pg's
- for (ceph::unordered_map<pg_t,PG*>::iterator it = pg_map.begin();
+ for (ceph::unordered_map<spg_t,PG*>::iterator it = pg_map.begin();
it != pg_map.end();
++it) {
@@ -1176,7 +1277,7 @@ int OSD::init()
consume_map();
peering_wq.drain();
- dout(10) << "done with init, starting boot process" << dendl;
+ dout(0) << "done with init, starting boot process" << dendl;
state = STATE_BOOTING;
start_boot();
@@ -1195,6 +1296,13 @@ void OSD::final_init()
int r;
AdminSocket *admin_socket = cct->get_admin_socket();
asok_hook = new OSDSocketHook(this);
+ r = admin_socket->register_command("status", "status", asok_hook,
+ "high-level status of OSD");
+ assert(r == 0);
+ r = admin_socket->register_command("flush_journal", "flush_journal",
+ asok_hook,
+ "flush the journal to permanent store");
+ assert(r == 0);
r = admin_socket->register_command("dump_ops_in_flight",
"dump_ops_in_flight", asok_hook,
"show the ops currently in flight");
@@ -1359,6 +1467,23 @@ void OSD::create_logger()
osd_plb.add_u64(l_osd_stat_bytes_used, "stat_bytes_used");
osd_plb.add_u64(l_osd_stat_bytes_avail, "stat_bytes_avail");
+ osd_plb.add_u64_counter(l_osd_copyfrom, "copyfrom");
+
+ osd_plb.add_u64_counter(l_osd_tier_promote, "tier_promote");
+ osd_plb.add_u64_counter(l_osd_tier_flush, "tier_flush");
+ osd_plb.add_u64_counter(l_osd_tier_flush_fail, "tier_flush_fail");
+ osd_plb.add_u64_counter(l_osd_tier_try_flush, "tier_try_flush");
+ osd_plb.add_u64_counter(l_osd_tier_try_flush_fail, "tier_try_flush_fail");
+ osd_plb.add_u64_counter(l_osd_tier_evict, "tier_evict");
+ osd_plb.add_u64_counter(l_osd_tier_whiteout, "tier_whiteout");
+ osd_plb.add_u64_counter(l_osd_tier_dirty, "tier_dirty");
+ osd_plb.add_u64_counter(l_osd_tier_clean, "tier_clean");
+
+ osd_plb.add_u64_counter(l_osd_agent_wake, "agent_wake");
+ osd_plb.add_u64_counter(l_osd_agent_skip, "agent_skip");
+ osd_plb.add_u64_counter(l_osd_agent_flush, "agent_flush");
+ osd_plb.add_u64_counter(l_osd_agent_evict, "agent_evict");
+
logger = osd_plb.create_perf_counters();
cct->get_perfcounters_collection()->add(logger);
}
@@ -1452,7 +1577,7 @@ int OSD::shutdown()
cct->_conf->apply_changes(NULL);
// Shutdown PGs
- for (ceph::unordered_map<pg_t, PG*>::iterator p = pg_map.begin();
+ for (ceph::unordered_map<spg_t, PG*>::iterator p = pg_map.begin();
p != pg_map.end();
++p) {
dout(20) << " kicking pg " << p->first << dendl;
@@ -1470,6 +1595,8 @@ int OSD::shutdown()
}
// unregister commands
+ cct->get_admin_socket()->unregister_command("status");
+ cct->get_admin_socket()->unregister_command("flush_journal");
cct->get_admin_socket()->unregister_command("dump_ops_in_flight");
cct->get_admin_socket()->unregister_command("dump_historic_ops");
cct->get_admin_socket()->unregister_command("dump_op_pq_state");
@@ -1510,7 +1637,10 @@ int OSD::shutdown()
disk_tp.drain();
disk_tp.stop();
- dout(10) << "disk tp paused (new), kicking all pgs" << dendl;
+ dout(10) << "disk tp paused (new)" << dendl;
+
+ dout(10) << "stopping agent" << dendl;
+ service.agent_stop();
osd_lock.Lock();
@@ -1548,7 +1678,7 @@ int OSD::shutdown()
#ifdef PG_DEBUG_REFS
service.dump_live_pgids();
#endif
- for (ceph::unordered_map<pg_t, PG*>::iterator p = pg_map.begin();
+ for (ceph::unordered_map<spg_t, PG*>::iterator p = pg_map.begin();
p != pg_map.end();
++p) {
dout(20) << " kicking pg " << p->first << dendl;
@@ -1622,13 +1752,17 @@ void OSD::recursive_remove_collection(ObjectStore *store, coll_t tmp)
store,
coll_t(),
make_snapmapper_oid());
- SnapMapper mapper(&driver, 0, 0, 0);
+
+ spg_t pg;
+ tmp.is_pg_prefix(pg);
+
+ ObjectStore::Transaction t;
+ SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
vector<ghobject_t> objects;
store->collection_list(tmp, objects);
// delete them.
- ObjectStore::Transaction t;
unsigned removed = 0;
for (vector<ghobject_t>::iterator p = objects.begin();
p != objects.end();
@@ -1677,7 +1811,7 @@ PGPool OSD::_get_pool(int id, OSDMapRef createmap)
PG *OSD::_open_lock_pg(
OSDMapRef createmap,
- pg_t pgid, bool no_lockdep_check, bool hold_map_lock)
+ spg_t pgid, bool no_lockdep_check, bool hold_map_lock)
{
assert(osd_lock.is_locked());
@@ -1692,7 +1826,7 @@ PG *OSD::_open_lock_pg(
PG* OSD::_make_pg(
OSDMapRef createmap,
- pg_t pgid)
+ spg_t pgid)
{
dout(10) << "_open_lock_pg " << pgid << dendl;
PGPool pool = _get_pool(pgid.pool(), createmap);
@@ -1701,7 +1835,8 @@ PG* OSD::_make_pg(
PG *pg;
hobject_t logoid = make_pg_log_oid(pgid);
hobject_t infooid = make_pg_biginfo_oid(pgid);
- if (createmap->get_pg_type(pgid) == pg_pool_t::TYPE_REPLICATED)
+ if (createmap->get_pg_type(pgid.pgid) == pg_pool_t::TYPE_REPLICATED ||
+ createmap->get_pg_type(pgid.pgid) == pg_pool_t::TYPE_ERASURE)
pg = new ReplicatedPG(&service, createmap, pool, pgid, logoid, infooid);
else
assert(0);
@@ -1717,14 +1852,14 @@ void OSD::add_newly_split_pg(PG *pg, PG::RecoveryCtx *rctx)
pg_map[pg->info.pgid] = pg;
dout(10) << "Adding newly split pg " << *pg << dendl;
vector<int> up, acting;
- pg->get_osdmap()->pg_to_up_acting_osds(pg->info.pgid, up, acting);
+ pg->get_osdmap()->pg_to_up_acting_osds(pg->info.pgid.pgid, up, acting);
int role = OSDMap::calc_pg_role(service.whoami, acting);
pg->set_role(role);
pg->reg_next_scrub();
pg->handle_loaded(rctx);
pg->write_if_dirty(*(rctx->transaction));
pg->queue_null(e, e);
- map<pg_t, list<PG::CephPeeringEvtRef> >::iterator to_wake =
+ map<spg_t, list<PG::CephPeeringEvtRef> >::iterator to_wake =
peering_wait_for_split.find(pg->info.pgid);
if (to_wake != peering_wait_for_split.end()) {
for (list<PG::CephPeeringEvtRef>::iterator i =
@@ -1741,13 +1876,13 @@ void OSD::add_newly_split_pg(PG *pg, PG::RecoveryCtx *rctx)
}
OSD::res_result OSD::_try_resurrect_pg(
- OSDMapRef curmap, pg_t pgid, pg_t *resurrected, PGRef *old_pg_state)
+ OSDMapRef curmap, spg_t pgid, spg_t *resurrected, PGRef *old_pg_state)
{
assert(resurrected);
assert(old_pg_state);
// find nearest ancestor
DeletingStateRef df;
- pg_t cur(pgid);
+ spg_t cur(pgid);
while (true) {
df = service.deleting_pgs.lookup(cur);
if (df)
@@ -1763,7 +1898,7 @@ OSD::res_result OSD::_try_resurrect_pg(
OSDMapRef create_map = df->old_pg_state->get_osdmap();
df->old_pg_state->unlock();
- set<pg_t> children;
+ set<spg_t> children;
if (cur == pgid) {
if (df->try_stop_deletion()) {
dout(10) << __func__ << ": halted deletion on pg " << pgid << dendl;
@@ -1800,11 +1935,14 @@ OSD::res_result OSD::_try_resurrect_pg(
PG *OSD::_create_lock_pg(
OSDMapRef createmap,
- pg_t pgid,
+ spg_t pgid,
bool newly_created,
bool hold_map_lock,
bool backfill,
- int role, vector<int>& up, vector<int>& acting, pg_history_t history,
+ int role,
+ vector<int>& up, int up_primary,
+ vector<int>& acting, int acting_primary,
+ pg_history_t history,
pg_interval_map_t& pi,
ObjectStore::Transaction& t)
{
@@ -1815,20 +1953,29 @@ PG *OSD::_create_lock_pg(
service.init_splits_between(pgid, pg->get_osdmap(), service.get_osdmap());
- pg->init(role, up, acting, history, pi, backfill, &t);
+ pg->init(
+ role,
+ up,
+ up_primary,
+ acting,
+ acting_primary,
+ history,
+ pi,
+ backfill,
+ &t);
dout(7) << "_create_lock_pg " << *pg << dendl;
return pg;
}
-bool OSD::_have_pg(pg_t pgid)
+bool OSD::_have_pg(spg_t pgid)
{
assert(osd_lock.is_locked());
return pg_map.count(pgid);
}
-PG *OSD::_lookup_lock_pg(pg_t pgid)
+PG *OSD::_lookup_lock_pg(spg_t pgid)
{
assert(osd_lock.is_locked());
if (!pg_map.count(pgid))
@@ -1839,7 +1986,7 @@ PG *OSD::_lookup_lock_pg(pg_t pgid)
}
-PG *OSD::_lookup_pg(pg_t pgid)
+PG *OSD::_lookup_pg(spg_t pgid)
{
assert(osd_lock.is_locked());
if (!pg_map.count(pgid))
@@ -1848,7 +1995,7 @@ PG *OSD::_lookup_pg(pg_t pgid)
return pg;
}
-PG *OSD::_lookup_lock_pg_with_map_lock_held(pg_t pgid)
+PG *OSD::_lookup_lock_pg_with_map_lock_held(spg_t pgid)
{
assert(osd_lock.is_locked());
assert(pg_map.count(pgid));
@@ -1860,7 +2007,7 @@ PG *OSD::_lookup_lock_pg_with_map_lock_held(pg_t pgid)
void OSD::load_pgs()
{
assert(osd_lock.is_locked());
- dout(10) << "load_pgs" << dendl;
+ dout(0) << "load_pgs" << dendl;
assert(pg_map.empty());
vector<coll_t> ls;
@@ -1869,12 +2016,12 @@ void OSD::load_pgs()
derr << "failed to list pgs: " << cpp_strerror(-r) << dendl;
}
- set<pg_t> head_pgs;
- map<pg_t, interval_set<snapid_t> > pgs;
+ set<spg_t> head_pgs;
+ map<spg_t, interval_set<snapid_t> > pgs;
for (vector<coll_t>::iterator it = ls.begin();
it != ls.end();
++it) {
- pg_t pgid;
+ spg_t pgid;
snapid_t snap;
uint64_t seq;
@@ -1901,10 +2048,10 @@ void OSD::load_pgs()
}
bool has_upgraded = false;
- for (map<pg_t, interval_set<snapid_t> >::iterator i = pgs.begin();
+ for (map<spg_t, interval_set<snapid_t> >::iterator i = pgs.begin();
i != pgs.end();
++i) {
- pg_t pgid(i->first);
+ spg_t pgid(i->first);
if (!head_pgs.count(pgid)) {
dout(10) << __func__ << ": " << pgid << " has orphan snap collections " << i->second
@@ -1971,7 +2118,15 @@ void OSD::load_pgs()
pg->reg_next_scrub();
// generate state for PG's current mapping
- pg->get_osdmap()->pg_to_up_acting_osds(pgid, pg->up, pg->acting);
+ int primary, up_primary;
+ vector<int> acting, up;
+ pg->get_osdmap()->pg_to_up_acting_osds(
+ pgid.pgid, &up, &up_primary, &acting, &primary);
+ pg->init_primary_up_acting(
+ up,
+ acting,
+ up_primary,
+ primary);
int role = OSDMap::calc_pg_role(whoami, pg->acting);
pg->set_role(role);
@@ -1981,7 +2136,7 @@ void OSD::load_pgs()
dout(10) << "load_pgs loaded " << *pg << " " << pg->pg_log.get_log() << dendl;
pg->unlock();
}
- dout(10) << "load_pgs done" << dendl;
+ dout(0) << "load_pgs opened " << pg_map.size() << " pgs" << dendl;
build_past_intervals_parallel();
}
@@ -2000,6 +2155,7 @@ struct pistate {
epoch_t start, end;
vector<int> old_acting, old_up;
epoch_t same_interval_since;
+ int primary;
};
void OSD::build_past_intervals_parallel()
@@ -2009,7 +2165,7 @@ void OSD::build_past_intervals_parallel()
// calculate untion of map range
epoch_t end_epoch = superblock.oldest_map;
epoch_t cur_epoch = superblock.newest_map;
- for (ceph::unordered_map<pg_t, PG*>::iterator i = pg_map.begin();
+ for (ceph::unordered_map<spg_t, PG*>::iterator i = pg_map.begin();
i != pg_map.end();
++i) {
PG *pg = i->second;
@@ -2051,7 +2207,9 @@ void OSD::build_past_intervals_parallel()
continue;
vector<int> acting, up;
- cur_map->pg_to_up_acting_osds(pg->info.pgid, up, acting);
+ int primary;
+ cur_map->pg_to_up_acting_osds(
+ pg->info.pgid.pgid, &up, 0, &acting, &primary);
if (p.same_interval_since == 0) {
dout(10) << __func__ << " epoch " << cur_epoch << " pg " << pg->info.pgid
@@ -2060,20 +2218,24 @@ void OSD::build_past_intervals_parallel()
p.same_interval_since = cur_epoch;
p.old_up = up;
p.old_acting = acting;
+ p.primary = primary;
continue;
}
assert(last_map);
std::stringstream debug;
- bool new_interval = pg_interval_t::check_new_interval(p.old_acting, acting,
- p.old_up, up,
- p.same_interval_since,
- pg->info.history.last_epoch_clean,
- cur_map, last_map,
- pg->info.pgid.pool(),
- pg->info.pgid,
- &pg->past_intervals,
- &debug);
+ bool new_interval = pg_interval_t::check_new_interval(
+ p.primary,
+ primary,
+ p.old_acting, acting,
+ p.old_up, up,
+ p.same_interval_since,
+ pg->info.history.last_epoch_clean,
+ cur_map, last_map,
+ pg->info.pgid.pool(),
+ pg->info.pgid.pgid,
+ &pg->past_intervals,
+ &debug);
if (new_interval) {
dout(10) << __func__ << " epoch " << cur_epoch << " pg " << pg->info.pgid
<< " " << debug.str() << dendl;
@@ -2115,37 +2277,40 @@ void OSD::build_past_intervals_parallel()
* hasn't changed since the given epoch and we are the primary.
*/
void OSD::handle_pg_peering_evt(
+ spg_t pgid,
const pg_info_t& info,
pg_interval_map_t& pi,
epoch_t epoch,
- int from,
+ pg_shard_t from,
bool primary,
PG::CephPeeringEvtRef evt)
{
- if (service.splitting(info.pgid)) {
- peering_wait_for_split[info.pgid].push_back(evt);
+ if (service.splitting(pgid)) {
+ peering_wait_for_split[pgid].push_back(evt);
return;
}
- if (!_have_pg(info.pgid)) {
+ if (!_have_pg(pgid)) {
// same primary?
- if (!osdmap->have_pg_pool(info.pgid.pool()))
+ if (!osdmap->have_pg_pool(pgid.pool()))
return;
+ int up_primary, acting_primary;
vector<int> up, acting;
- osdmap->pg_to_up_acting_osds(info.pgid, up, acting);
+ osdmap->pg_to_up_acting_osds(
+ pgid.pgid, &up, &up_primary, &acting, &acting_primary);
int role = osdmap->calc_pg_role(whoami, acting, acting.size());
pg_history_t history = info.history;
bool valid_history = project_pg_history(
- info.pgid, history, epoch, up, acting);
+ pgid, history, epoch, up, up_primary, acting, acting_primary);
if (!valid_history || epoch < history.same_interval_since) {
- dout(10) << "get_or_create_pg " << info.pgid << " acting changed in "
+ dout(10) << "get_or_create_pg " << pgid << " acting changed in "
<< history.same_interval_since << " (msg from " << epoch << ")" << dendl;
return;
}
- if (service.splitting(info.pgid)) {
+ if (service.splitting(pgid)) {
assert(0);
}
@@ -2154,29 +2319,29 @@ void OSD::handle_pg_peering_evt(
// DNE on source?
if (info.dne()) {
// is there a creation pending on this pg?
- if (creating_pgs.count(info.pgid)) {
- creating_pgs[info.pgid].prior.erase(from);
- if (!can_create_pg(info.pgid))
+ if (creating_pgs.count(pgid)) {
+ creating_pgs[pgid].prior.erase(from);
+ if (!can_create_pg(pgid))
return;
- history = creating_pgs[info.pgid].history;
+ history = creating_pgs[pgid].history;
create = true;
} else {
- dout(10) << "get_or_create_pg " << info.pgid
+ dout(10) << "get_or_create_pg " << pgid
<< " DNE on source, but creation probe, ignoring" << dendl;
return;
}
}
- creating_pgs.erase(info.pgid);
+ creating_pgs.erase(pgid);
} else {
assert(!info.dne()); // pg exists if we are hearing about it
}
// do we need to resurrect a deleting pg?
- pg_t resurrected;
+ spg_t resurrected;
PGRef old_pg_state;
res_result result = _try_resurrect_pg(
service.get_osdmap(),
- info.pgid,
+ pgid,
&resurrected,
&old_pg_state);
@@ -2184,11 +2349,14 @@ void OSD::handle_pg_peering_evt(
switch (result) {
case RES_NONE: {
// ok, create the pg locally using provided Info and History
- rctx.transaction->create_collection(coll_t(info.pgid));
+ rctx.transaction->create_collection(coll_t(pgid));
PG *pg = _create_lock_pg(
get_map(epoch),
- info.pgid, create, false, result == RES_SELF,
- role, up, acting, history, pi,
+ pgid, create, false, result == RES_SELF,
+ role,
+ up, up_primary,
+ acting, acting_primary,
+ history, pi,
*rctx.transaction);
pg->handle_create(&rctx);
pg->write_if_dirty(*rctx.transaction);
@@ -2213,7 +2381,9 @@ void OSD::handle_pg_peering_evt(
true,
old_pg_state->role,
old_pg_state->up,
+ old_pg_state->up_primary.osd,
old_pg_state->acting,
+ old_pg_state->primary.osd,
old_pg_state->info.history,
old_pg_state->past_intervals,
*rctx.transaction);
@@ -2242,7 +2412,9 @@ void OSD::handle_pg_peering_evt(
true,
old_pg_state->role,
old_pg_state->up,
+ old_pg_state->up_primary.osd,
old_pg_state->acting,
+ old_pg_state->primary.osd,
old_pg_state->info.history,
old_pg_state->past_intervals,
*rctx.transaction
@@ -2257,8 +2429,8 @@ void OSD::handle_pg_peering_evt(
// kick any waiters
wake_pg_waiters(parent->info.pgid);
- assert(service.splitting(info.pgid));
- peering_wait_for_split[info.pgid].push_back(evt);
+ assert(service.splitting(pgid));
+ peering_wait_for_split[pgid].push_back(evt);
//parent->queue_peering_event(evt);
parent->queue_null(osdmap->get_epoch(), osdmap->get_epoch());
@@ -2268,7 +2440,7 @@ void OSD::handle_pg_peering_evt(
}
} else {
// already had it. did the mapping change?
- PG *pg = _lookup_lock_pg(info.pgid);
+ PG *pg = _lookup_lock_pg(pgid);
if (epoch < pg->info.history.same_interval_since) {
dout(10) << *pg << " get_or_create_pg acting changed in "
<< pg->info.history.same_interval_since
@@ -2288,27 +2460,36 @@ void OSD::handle_pg_peering_evt(
* - from each epoch, include all osds up then AND now
* - if no osds from then are up now, include them all, even tho they're not reachable now
*/
-void OSD::calc_priors_during(pg_t pgid, epoch_t start, epoch_t end, set<int>& pset)
+void OSD::calc_priors_during(
+ spg_t pgid, epoch_t start, epoch_t end, set<pg_shard_t>& pset)
{
- dout(15) << "calc_priors_during " << pgid << " [" << start << "," << end << ")" << dendl;
+ dout(15) << "calc_priors_during " << pgid << " [" << start
+ << "," << end << ")" << dendl;
for (epoch_t e = start; e < end; e++) {
OSDMapRef oldmap = get_map(e);
vector<int> acting;
- oldmap->pg_to_acting_osds(pgid, acting);
+ oldmap->pg_to_acting_osds(pgid.pgid, acting);
dout(20) << " " << pgid << " in epoch " << e << " was " << acting << dendl;
int up = 0;
for (unsigned i=0; i<acting.size(); i++)
if (osdmap->is_up(acting[i])) {
- if (acting[i] != whoami)
- pset.insert(acting[i]);
+ if (acting[i] != whoami) {
+ pset.insert(
+ pg_shard_t(
+ acting[i],
+ osdmap->pg_is_ec(pgid.pgid) ? i : ghobject_t::NO_SHARD));
+ }
up++;
}
if (!up && !acting.empty()) {
// sucky. add down osds, even tho we can't reach them right now.
for (unsigned i=0; i<acting.size(); i++)
if (acting[i] != whoami)
- pset.insert(acting[i]);
+ pset.insert(
+ pg_shard_t(
+ acting[i],
+ osdmap->pg_is_ec(pgid.pgid) ? i : ghobject_t::NO_SHARD));
}
}
dout(10) << "calc_priors_during " << pgid
@@ -2321,9 +2502,11 @@ void OSD::calc_priors_during(pg_t pgid, epoch_t start, epoch_t end, set<int>& ps
* Fill in the passed history so you know same_interval_since, same_up_since,
* and same_primary_since.
*/
-bool OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
+bool OSD::project_pg_history(spg_t pgid, pg_history_t& h, epoch_t from,
const vector<int>& currentup,
- const vector<int>& currentacting)
+ int currentupprimary,
+ const vector<int>& currentacting,
+ int currentactingprimary)
{
dout(15) << "project_pg_history " << pgid
<< " from " << from << " to " << osdmap->get_epoch()
@@ -2342,14 +2525,25 @@ bool OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
}
assert(oldmap->have_pg_pool(pgid.pool()));
+ int upprimary, actingprimary;
vector<int> up, acting;
- oldmap->pg_to_up_acting_osds(pgid, up, acting);
+ oldmap->pg_to_up_acting_osds(
+ pgid.pgid,
+ &up,
+ &upprimary,
+ &acting,
+ &actingprimary);
// acting set change?
- if ((acting != currentacting || up != currentup) && e > h.same_interval_since) {
+ if ((actingprimary != currentactingprimary ||
+ upprimary != currentupprimary ||
+ acting != currentacting ||
+ up != currentup) && e > h.same_interval_since) {
dout(15) << "project_pg_history " << pgid << " acting|up changed in " << e
<< " from " << acting << "/" << up
+ << " " << actingprimary << "/" << upprimary
<< " -> " << currentacting << "/" << currentup
+ << " " << currentactingprimary << "/" << currentupprimary
<< dendl;
h.same_interval_since = e;
}
@@ -2360,14 +2554,20 @@ bool OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from,
h.same_interval_since = e;
}
// up set change?
- if (up != currentup && e > h.same_up_since) {
+ if ((up != currentup || upprimary != currentupprimary)
+ && e > h.same_up_since) {
dout(15) << "project_pg_history " << pgid << " up changed in " << e
- << " from " << up << " -> " << currentup << dendl;
+ << " from " << up << " " << upprimary
+ << " -> " << currentup << " " << currentupprimary << dendl;
h.same_up_since = e;
}
// primary change?
- if (!(!acting.empty() && !currentacting.empty() && acting[0] == currentacting[0]) &&
+ if (OSDMap::primary_changed(
+ actingprimary,
+ acting,
+ currentactingprimary,
+ currentacting) &&
e > h.same_primary_since) {
dout(15) << "project_pg_history " << pgid << " primary changed in " << e << dendl;
h.same_primary_since = e;
@@ -2587,7 +2787,7 @@ void OSD::maybe_update_heartbeat_peers()
// build heartbeat from set
if (is_active()) {
- for (ceph::unordered_map<pg_t, PG*>::iterator i = pg_map.begin();
+ for (ceph::unordered_map<spg_t, PG*>::iterator i = pg_map.begin();
i != pg_map.end();
++i) {
PG *pg = i->second;
@@ -2814,8 +3014,7 @@ void OSD::handle_osd_ping(MOSDPing *m)
case MOSDPing::YOU_DIED:
dout(10) << "handle_osd_ping " << m->get_source_inst() << " says i am down in " << m->map_epoch
<< dendl;
- if (monc->sub_want("osdmap", m->map_epoch, CEPH_SUBSCRIBE_ONETIME))
- monc->renew_subs();
+ osdmap_subscribe(m->map_epoch, false);
break;
}
@@ -2939,8 +3138,7 @@ void OSD::heartbeat()
if (now - last_mon_heartbeat > cct->_conf->osd_mon_heartbeat_interval && is_active()) {
last_mon_heartbeat = now;
dout(10) << "i have no heartbeat peers; checking mon for new map" << dendl;
- monc->sub_want("osdmap", osdmap->get_epoch() + 1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(osdmap->get_epoch() + 1, true);
}
}
@@ -3015,11 +3213,12 @@ void OSD::tick()
// mon report?
utime_t now = ceph_clock_now(cct);
- if (outstanding_pg_stats &&
+ if (outstanding_pg_stats && timeout_mon_on_pg_stats &&
(now - cct->_conf->osd_mon_ack_timeout) > last_pg_stats_ack) {
dout(1) << "mon hasn't acked PGStats in " << now - last_pg_stats_ack
<< " seconds, reconnecting elsewhere" << dendl;
- monc->reopen_session();
+ monc->reopen_session(new C_MonStatsAckTimer(this));
+ timeout_mon_on_pg_stats = false;
last_pg_stats_ack = ceph_clock_now(cct); // reset clock
last_pg_stats_sent = utime_t();
}
@@ -3099,7 +3298,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
command == "truncobj" || command == "injectmdataerr" ||
command == "injectdataerr"
) {
- pg_t rawpg, pgid;
+ pg_t rawpg;
int64_t pool;
OSDMapRef curmap = service->get_osdmap();
int r;
@@ -3130,7 +3329,11 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
ss << "Invalid namespace/objname";
return;
}
- pgid = curmap->raw_pg_to_pg(rawpg);
+ if (curmap->pg_is_ec(rawpg)) {
+ ss << "Must not call on ec pool";
+ return;
+ }
+ spg_t pgid = spg_t(curmap->raw_pg_to_pg(rawpg), ghobject_t::no_shard());
hobject_t obj(object_t(objname), string(""), CEPH_NOSNAP, rawpg.ps(), pool, nspace);
ObjectStore::Transaction t;
@@ -3424,10 +3627,9 @@ void OSD::_maybe_boot(epoch_t oldest, epoch_t newest)
// get all the latest maps
if (osdmap->get_epoch() > oldest)
- monc->sub_want("osdmap", osdmap->get_epoch(), CEPH_SUBSCRIBE_ONETIME);
+ osdmap_subscribe(osdmap->get_epoch() + 1, true);
else
- monc->sub_want("osdmap", oldest - 1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(oldest - 1, true);
}
void OSD::start_waiting_for_healthy()
@@ -3767,7 +3969,7 @@ void OSD::send_pg_stats(const utime_t &now)
}
pg->pg_stats_publish_lock.Lock();
if (pg->pg_stats_publish_valid) {
- m->pg_stat[pg->info.pgid] = pg->pg_stats_publish;
+ m->pg_stat[pg->info.pgid.pgid] = pg->pg_stats_publish;
dout(25) << " sending " << pg->info.pgid << " " << pg->pg_stats_publish.reported_epoch << ":"
<< pg->pg_stats_publish.reported_seq << dendl;
} else {
@@ -3811,8 +4013,8 @@ void OSD::handle_pg_stats_ack(MPGStatsAck *ack)
PGRef _pg(pg);
++p;
- if (ack->pg_stat.count(pg->info.pgid)) {
- pair<version_t,epoch_t> acked = ack->pg_stat[pg->info.pgid];
+ if (ack->pg_stat.count(pg->info.pgid.pgid)) {
+ pair<version_t,epoch_t> acked = ack->pg_stat[pg->info.pgid.pgid];
pg->pg_stats_publish_lock.Lock();
if (acked.first == pg->pg_stats_publish.reported_seq &&
acked.second == pg->pg_stats_publish.reported_epoch) {
@@ -3942,8 +4144,6 @@ COMMAND("list_missing " \
"osd", "r", "cli,rest")
// tell <osd.n> commands. Validation of osd.n must be special-cased in client
-
-// tell <osd.n> commands. Validation of osd.n must be special-cased in client
COMMAND("version", "report version of OSD", "osd", "r", "cli,rest")
COMMAND("injectargs " \
"name=injected_args,type=CephString,n=N",
@@ -3973,19 +4173,6 @@ COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics",
"osd", "r", "cli,rest")
COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics",
"osd", "rw", "cli,rest")
-
-// experiment: restate pg commands as "tell <pgid>". Validation of
-// pgid must be special-cased in client.
-COMMAND("query",
- "show details of a specific pg", "osd", "r", "cli,rest")
-COMMAND("mark_unfound_lost revert " \
- "name=mulcmd,type=CephChoices,strings=revert", \
- "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available",
- "osd", "rw", "cli,rest")
-COMMAND("list_missing " \
- "name=offset,type=CephString,req=false",
- "list missing objects on this pg, perhaps starting at an offset given in JSON",
- "osd", "rw", "cli,rest")
};
void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist& data)
@@ -4084,16 +4271,19 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
ss << "couldn't parse pgid '" << pgidstr << "'";
r = -EINVAL;
} else {
- PG *pg = _lookup_lock_pg(pgid);
- if (!pg) {
- ss << "i don't have pgid " << pgid;
- r = -ENOENT;
- } else {
+ spg_t pcand;
+ if (osdmap->get_primary_shard(pgid, &pcand) &&
+ _have_pg(pcand)) {
+ PG *pg = _lookup_lock_pg(pcand);
+ assert(pg);
// simulate pg <pgid> cmd= for pg->do-command
if (prefix != "pg")
cmd_putval(cct, cmdmap, "cmd", prefix);
r = pg->do_command(cmdmap, ss, data, odata);
pg->unlock();
+ } else {
+ ss << "i don't have pgid " << pgid;
+ r = -ENOENT;
}
}
}
@@ -4105,6 +4295,59 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
cmd_getval(cct, cmdmap, "count", count, (int64_t)1 << 30);
cmd_getval(cct, cmdmap, "size", bsize, (int64_t)4 << 20);
+ uint32_t duration = g_conf->osd_bench_duration;
+
+ if (bsize > (int64_t) g_conf->osd_bench_max_block_size) {
+ // let us limit the block size because the next checks rely on it
+ // having a sane value. If we allow any block size to be set things
+ // can still go sideways.
+ ss << "block 'size' values are capped at "
+ << prettybyte_t(g_conf->osd_bench_max_block_size) << ". If you wish to use"
+ << " a higher value, please adjust 'osd_bench_max_block_size'";
+ r = -EINVAL;
+ goto out;
+ } else if (bsize < (int64_t) (1 << 20)) {
+ // entering the realm of small block sizes.
+ // limit the count to a sane value, assuming a configurable amount of
+ // IOPS and duration, so that the OSD doesn't get hung up on this,
+ // preventing timeouts from going off
+ int64_t max_count =
+ bsize * duration * g_conf->osd_bench_small_size_max_iops;
+ if (count > max_count) {
+ ss << "'count' values greater than " << max_count
+ << " for a block size of " << prettybyte_t(bsize) << ", assuming "
+ << g_conf->osd_bench_small_size_max_iops << " IOPS,"
+ << " for " << duration << " seconds,"
+ << " can cause ill effects on osd. "
+ << " Please adjust 'osd_bench_small_size_max_iops' with a higher"
+ << " value if you wish to use a higher 'count'.";
+ r = -EINVAL;
+ goto out;
+ }
+ } else {
+ // 1MB block sizes are big enough so that we get more stuff done.
+ // However, to avoid the osd from getting hung on this and having
+ // timers being triggered, we are going to limit the count assuming
+ // a configurable throughput and duration.
+ int64_t total_throughput =
+ g_conf->osd_bench_large_size_max_throughput * duration;
+ int64_t max_count = (int64_t) (total_throughput / bsize);
+ if (count > max_count) {
+ ss << "'count' values greater than " << max_count
+ << " for a block size of " << prettybyte_t(bsize) << ", assuming "
+ << prettybyte_t(g_conf->osd_bench_large_size_max_throughput) << "/s,"
+ << " for " << duration << " seconds,"
+ << " can cause ill effects on osd. "
+ << " Please adjust 'osd_bench_large_size_max_throughput'"
+ << " with a higher value if you wish to use a higher 'count'.";
+ r = -EINVAL;
+ goto out;
+ }
+ }
+
+ dout(1) << " bench count " << count
+ << " bsize " << prettybyte_t(bsize) << dendl;
+
bufferlist bl;
bufferptr bp(bsize);
bp.zero();
@@ -4173,16 +4416,16 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
goto out;
}
- std::set <pg_t> keys;
- for (ceph::unordered_map<pg_t, PG*>::const_iterator pg_map_e = pg_map.begin();
+ std::set <spg_t> keys;
+ for (ceph::unordered_map<spg_t, PG*>::const_iterator pg_map_e = pg_map.begin();
pg_map_e != pg_map.end(); ++pg_map_e) {
keys.insert(pg_map_e->first);
}
fout << "*** osd " << whoami << ": dump_missing ***" << std::endl;
- for (std::set <pg_t>::iterator p = keys.begin();
+ for (std::set <spg_t>::iterator p = keys.begin();
p != keys.end(); ++p) {
- ceph::unordered_map<pg_t, PG*>::iterator q = pg_map.find(*p);
+ ceph::unordered_map<spg_t, PG*>::iterator q = pg_map.find(*p);
assert(q != pg_map.end());
PG *pg = q->second;
pg->lock();
@@ -4194,11 +4437,11 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
pg->pg_log.get_missing().missing.begin();
for (; mi != mend; ++mi) {
fout << mi->first << " -> " << mi->second << std::endl;
- map<hobject_t, set<int> >::const_iterator mli =
- pg->missing_loc.find(mi->first);
- if (mli == pg->missing_loc.end())
+ if (!pg->missing_loc.needs_recovery(mi->first))
continue;
- const set<int> &mls(mli->second);
+ if (pg->missing_loc.is_unfound(mi->first))
+ fout << " unfound ";
+ const set<pg_shard_t> &mls(pg->missing_loc.get_locations(mi->first));
if (mls.empty())
continue;
fout << "missing_loc: " << mls << std::endl;
@@ -4655,6 +4898,18 @@ void OSD::dispatch_op(OpRequestRef op)
case MSG_OSD_PG_PUSH_REPLY:
handle_replica_op<MOSDPGPushReply, MSG_OSD_PG_PUSH_REPLY>(op);
break;
+ case MSG_OSD_EC_WRITE:
+ handle_replica_op<MOSDECSubOpWrite, MSG_OSD_EC_WRITE>(op);
+ break;
+ case MSG_OSD_EC_WRITE_REPLY:
+ handle_replica_op<MOSDECSubOpWriteReply, MSG_OSD_EC_WRITE_REPLY>(op);
+ break;
+ case MSG_OSD_EC_READ:
+ handle_replica_op<MOSDECSubOpRead, MSG_OSD_EC_READ>(op);
+ break;
+ case MSG_OSD_EC_READ_REPLY:
+ handle_replica_op<MOSDECSubOpReadReply, MSG_OSD_EC_READ_REPLY>(op);
+ break;
}
}
@@ -4754,7 +5009,7 @@ void OSD::handle_scrub(MOSDScrub *m)
}
if (m->scrub_pgs.empty()) {
- for (ceph::unordered_map<pg_t, PG*>::iterator p = pg_map.begin();
+ for (ceph::unordered_map<spg_t, PG*>::iterator p = pg_map.begin();
p != pg_map.end();
++p) {
PG *pg = p->second;
@@ -4772,9 +5027,11 @@ void OSD::handle_scrub(MOSDScrub *m)
} else {
for (vector<pg_t>::iterator p = m->scrub_pgs.begin();
p != m->scrub_pgs.end();
- ++p)
- if (pg_map.count(*p)) {
- PG *pg = pg_map[*p];
+ ++p) {
+ spg_t pcand;
+ if (osdmap->get_primary_shard(*p, &pcand) &&
+ pg_map.count(pcand)) {
+ PG *pg = pg_map[pcand];
pg->lock();
if (pg->is_primary()) {
pg->unreg_next_scrub();
@@ -4786,6 +5043,7 @@ void OSD::handle_scrub(MOSDScrub *m)
}
pg->unlock();
}
+ }
}
m->put();
@@ -4834,11 +5092,11 @@ void OSD::sched_scrub()
//dout(20) << " " << last_scrub_pg << dendl;
- pair<utime_t, pg_t> pos;
+ pair<utime_t, spg_t> pos;
if (service.first_scrub_stamp(&pos)) {
do {
utime_t t = pos.first;
- pg_t pgid = pos.second;
+ spg_t pgid = pos.second;
dout(30) << "sched_scrub examine " << pgid << " at " << t << dendl;
utime_t diff = now - t;
@@ -4939,6 +5197,7 @@ bool OSDService::prepare_to_stop()
OSDMapRef osdmap = get_osdmap();
if (osdmap && osdmap->is_up(whoami)) {
+ dout(0) << __func__ << " telling mon we are shutting down" << dendl;
state = PREPARING_TO_STOP;
monc->send_mon_message(new MOSDMarkMeDown(monc->get_fsid(),
osdmap->get_inst(whoami),
@@ -4953,6 +5212,7 @@ bool OSDService::prepare_to_stop()
is_stopping_cond.WaitUntil(is_stopping_lock, timeout);
}
}
+ dout(0) << __func__ << " starting shutdown" << dendl;
state = STOPPING;
return true;
}
@@ -4960,7 +5220,7 @@ bool OSDService::prepare_to_stop()
void OSDService::got_stop_ack()
{
Mutex::Locker l(is_stopping_lock);
- dout(10) << "Got stop ack" << dendl;
+ dout(0) << __func__ << " starting shutdown" << dendl;
state = STOPPING;
is_stopping_cond.Signal();
}
@@ -4973,8 +5233,7 @@ void OSD::wait_for_new_map(OpRequestRef op)
{
// ask?
if (waiting_for_osdmap.empty()) {
- monc->sub_want("osdmap", osdmap->get_epoch() + 1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(osdmap->get_epoch() + 1, true);
}
logger->inc(l_osd_waiting_for_map);
@@ -5026,6 +5285,18 @@ struct C_OnMapApply : public Context {
}
};
+void OSD::osdmap_subscribe(version_t epoch, bool force_request)
+{
+ OSDMapRef osdmap = service.get_osdmap();
+ if (osdmap->get_epoch() >= epoch)
+ return;
+
+ if (monc->sub_want_increment("osdmap", epoch, CEPH_SUBSCRIBE_ONETIME) ||
+ force_request) {
+ monc->renew_subs();
+ }
+}
+
void OSD::handle_osd_map(MOSDMap *m)
{
assert(osd_lock.is_locked());
@@ -5077,14 +5348,16 @@ void OSD::handle_osd_map(MOSDMap *m)
return;
}
+ // even if this map isn't from a mon, we may have satisfied our subscription
+ monc->sub_got("osdmap", last);
+
// missing some?
bool skip_maps = false;
if (first > osdmap->get_epoch() + 1) {
dout(10) << "handle_osd_map message skips epochs " << osdmap->get_epoch() + 1
<< ".." << (first-1) << dendl;
if (m->oldest_map <= osdmap->get_epoch() + 1) {
- monc->sub_want("osdmap", osdmap->get_epoch()+1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(osdmap->get_epoch()+1, true);
m->put();
return;
}
@@ -5093,8 +5366,7 @@ void OSD::handle_osd_map(MOSDMap *m)
// 2- is at present the only way to ensure that we get a *full* map as
// the first map!
if (m->oldest_map < first) {
- monc->sub_want("osdmap", m->oldest_map - 1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(m->oldest_map - 1, true);
m->put();
return;
}
@@ -5229,6 +5501,10 @@ void OSD::handle_osd_map(MOSDMap *m)
if (is_booting()) {
dout(1) << "state: booting -> active" << dendl;
state = STATE_ACTIVE;
+
+ // set incarnation so that osd_reqid_t's we generate for our
+ // objecter requests are unique across restarts.
+ service.objecter->set_client_incarnation(osdmap->get_epoch());
}
}
@@ -5337,8 +5613,7 @@ void OSD::handle_osd_map(MOSDMap *m)
if (m->newest_map && m->newest_map > last) {
dout(10) << " msg say newest map is " << m->newest_map << ", requesting more" << dendl;
- monc->sub_want("osdmap", osdmap->get_epoch()+1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(osdmap->get_epoch()+1, true);
}
else if (is_booting()) {
start_boot(); // retry
@@ -5419,12 +5694,19 @@ void OSD::advance_pg(
continue;
vector<int> newup, newacting;
- nextmap->pg_to_up_acting_osds(pg->info.pgid, newup, newacting);
- pg->handle_advance_map(nextmap, lastmap, newup, newacting, rctx);
+ int up_primary, acting_primary;
+ nextmap->pg_to_up_acting_osds(
+ pg->info.pgid.pgid,
+ &newup, &up_primary,
+ &newacting, &acting_primary);
+ pg->handle_advance_map(
+ nextmap, lastmap, newup, up_primary,
+ newacting, acting_primary, rctx);
// Check for split!
- set<pg_t> children;
- if (pg->info.pgid.is_split(
+ set<spg_t> children;
+ spg_t parent(pg->info.pgid);
+ if (parent.is_split(
lastmap->get_pg_num(pg->pool.id),
nextmap->get_pg_num(pg->pool.id),
&children)) {
@@ -5464,15 +5746,15 @@ void OSD::advance_map(ObjectStore::Transaction& t, C_Contexts *tfin)
}
// scan pg creations
- ceph::unordered_map<pg_t, create_pg_info>::iterator n = creating_pgs.begin();
+ ceph::unordered_map<spg_t, create_pg_info>::iterator n = creating_pgs.begin();
while (n != creating_pgs.end()) {
- ceph::unordered_map<pg_t, create_pg_info>::iterator p = n++;
- pg_t pgid = p->first;
+ ceph::unordered_map<spg_t, create_pg_info>::iterator p = n++;
+ spg_t pgid = p->first;
// am i still primary?
vector<int> acting;
int primary;
- osdmap->pg_to_acting_osds(pgid, &acting, &primary);
+ osdmap->pg_to_acting_osds(pgid.pgid, &acting, &primary);
if (primary != whoami) {
dout(10) << " no longer primary for " << pgid << ", stopping creation" << dendl;
creating_pgs.erase(p);
@@ -5486,12 +5768,12 @@ void OSD::advance_map(ObjectStore::Transaction& t, C_Contexts *tfin)
}
// scan pgs with waiters
- map<pg_t, list<OpRequestRef> >::iterator p = waiting_for_pg.begin();
+ map<spg_t, list<OpRequestRef> >::iterator p = waiting_for_pg.begin();
while (p != waiting_for_pg.end()) {
- pg_t pgid = p->first;
+ spg_t pgid = p->first;
vector<int> acting;
- int nrep = osdmap->pg_to_acting_osds(pgid, acting);
+ int nrep = osdmap->pg_to_acting_osds(pgid.pgid, acting);
int role = osdmap->calc_pg_role(whoami, acting, nrep);
if (role >= 0) {
++p; // still me
@@ -5514,7 +5796,7 @@ void OSD::consume_map()
list<PGRef> to_remove;
// scan pg's
- for (ceph::unordered_map<pg_t,PG*>::iterator it = pg_map.begin();
+ for (ceph::unordered_map<spg_t,PG*>::iterator it = pg_map.begin();
it != pg_map.end();
++it) {
PG *pg = it->second;
@@ -5551,7 +5833,7 @@ void OSD::consume_map()
service.publish_map(osdmap);
// scan pg's
- for (ceph::unordered_map<pg_t,PG*>::iterator it = pg_map.begin();
+ for (ceph::unordered_map<spg_t,PG*>::iterator it = pg_map.begin();
it != pg_map.end();
++it) {
PG *pg = it->second;
@@ -5576,8 +5858,7 @@ void OSD::activate_map()
if (osdmap->test_flag(CEPH_OSDMAP_FULL)) {
dout(10) << " osdmap flagged full, doing onetime osdmap subscribe" << dendl;
- monc->sub_want("osdmap", osdmap->get_epoch() + 1, CEPH_SUBSCRIBE_ONETIME);
- monc->renew_subs();
+ osdmap_subscribe(osdmap->get_epoch() + 1, true);
}
// norecover?
@@ -5595,6 +5876,8 @@ void OSD::activate_map()
}
}
+ service.activate_map();
+
// process waiters
take_waiters(waiting_for_osdmap);
}
@@ -5834,7 +6117,7 @@ bool OSD::require_same_or_newer_map(OpRequestRef op, epoch_t epoch)
// pg creation
-bool OSD::can_create_pg(pg_t pgid)
+bool OSD::can_create_pg(spg_t pgid)
{
assert(creating_pgs.count(pgid));
@@ -5851,7 +6134,7 @@ bool OSD::can_create_pg(pg_t pgid)
void OSD::split_pgs(
PG *parent,
- const set<pg_t> &childpgids, set<boost::intrusive_ptr<PG> > *out_pgs,
+ const set<spg_t> &childpgids, set<boost::intrusive_ptr<PG> > *out_pgs,
OSDMapRef curmap,
OSDMapRef nextmap,
PG::RecoveryCtx *rctx)
@@ -5861,7 +6144,7 @@ void OSD::split_pgs(
parent->update_snap_mapper_bits(
parent->info.pgid.get_split_bits(pg_num)
);
- for (set<pg_t>::const_iterator i = childpgids.begin();
+ for (set<spg_t>::const_iterator i = childpgids.begin();
i != childpgids.end();
++i) {
dout(10) << "Splitting " << *parent << " into " << *i << dendl;
@@ -5878,10 +6161,10 @@ void OSD::split_pgs(
parent->split_colls(
*i,
split_bits,
- i->m_seed,
+ i->ps(),
rctx->transaction);
parent->split_into(
- *i,
+ i->pgid,
child,
split_bits);
@@ -5936,41 +6219,48 @@ void OSD::handle_pg_create(OpRequestRef op)
for (map<pg_t,pg_create_t>::iterator p = m->mkpg.begin();
p != m->mkpg.end();
++p) {
- pg_t pgid = p->first;
epoch_t created = p->second.created;
pg_t parent = p->second.parent;
if (p->second.split_bits) // Skip split pgs
continue;
- pg_t on = pgid;
+ pg_t on = p->first;
- if (pgid.preferred() >= 0) {
- dout(20) << "ignoring localized pg " << pgid << dendl;
+ if (on.preferred() >= 0) {
+ dout(20) << "ignoring localized pg " << on << dendl;
continue;
}
- if (!osdmap->have_pg_pool(pgid.pool())) {
- dout(20) << "ignoring pg on deleted pool " << pgid << dendl;
+
+ if (!osdmap->have_pg_pool(on.pool())) {
+ dout(20) << "ignoring pg on deleted pool " << on << dendl;
continue;
}
- dout(20) << "mkpg " << pgid << " e" << created << dendl;
+ dout(20) << "mkpg " << on << " e" << created << dendl;
// is it still ours?
vector<int> up, acting;
- int up_primary, acting_primary;
+ int up_primary = -1;
+ int acting_primary = -1;
osdmap->pg_to_up_acting_osds(on, &up, &up_primary, &acting, &acting_primary);
int role = osdmap->calc_pg_role(whoami, acting, acting.size());
if (up_primary != whoami) {
- dout(10) << "mkpg " << pgid << " not primary (role=" << role << "), skipping" << dendl;
+ dout(10) << "mkpg " << on << " not primary (role="
+ << role << "), skipping" << dendl;
continue;
}
if (up != acting) {
- dout(10) << "mkpg " << pgid << " up " << up << " != acting " << acting << ", ignoring" << dendl;
+ dout(10) << "mkpg " << on << " up " << up
+ << " != acting " << acting << ", ignoring" << dendl;
// we'll get a query soon anyway, since we know the pg
// must exist. we can ignore this.
continue;
}
+ spg_t pgid;
+ bool mapped = osdmap->get_primary_shard(on, &pgid);
+ assert(mapped);
+
// does it already exist?
if (_have_pg(pgid)) {
dout(10) << "mkpg " << pgid << " already exists, skipping" << dendl;
@@ -5986,8 +6276,8 @@ void OSD::handle_pg_create(OpRequestRef op)
utime_t now = ceph_clock_now(NULL);
history.last_scrub_stamp = now;
history.last_deep_scrub_stamp = now;
- bool valid_history =
- project_pg_history(pgid, history, created, up, acting);
+ bool valid_history = project_pg_history(
+ pgid, history, created, up, up_primary, acting, acting_primary);
/* the pg creation message must have come from a mon and therefore
* cannot be on the other side of a map gap
*/
@@ -6002,14 +6292,18 @@ void OSD::handle_pg_create(OpRequestRef op)
PG::RecoveryCtx rctx = create_context();
// poll priors
- set<int>& pset = creating_pgs[pgid].prior;
+ set<pg_shard_t>& pset = creating_pgs[pgid].prior;
dout(10) << "mkpg " << pgid << " e" << created
<< " h " << history
<< " : querying priors " << pset << dendl;
- for (set<int>::iterator p = pset.begin(); p != pset.end(); ++p)
- if (osdmap->is_up(*p))
- (*rctx.query_map)[*p][pgid] = pg_query_t(pg_query_t::INFO, history,
- osdmap->get_epoch());
+ for (set<pg_shard_t>::iterator p = pset.begin(); p != pset.end(); ++p)
+ if (osdmap->is_up(p->osd))
+ (*rctx.query_map)[p->osd][spg_t(pgid.pgid, p->shard)] =
+ pg_query_t(
+ pg_query_t::INFO,
+ p->shard, pgid.shard,
+ history,
+ osdmap->get_epoch());
PG *pg = NULL;
if (can_create_pg(pgid)) {
@@ -6017,7 +6311,8 @@ void OSD::handle_pg_create(OpRequestRef op)
rctx.transaction->create_collection(coll_t(pgid));
pg = _create_lock_pg(
osdmap, pgid, true, false, false,
- 0, creating_pgs[pgid].acting, creating_pgs[pgid].acting,
+ 0, creating_pgs[pgid].acting, whoami,
+ creating_pgs[pgid].acting, whoami,
history, pi,
*rctx.transaction);
pg->info.last_epoch_started = pg->info.history.last_epoch_started;
@@ -6044,10 +6339,10 @@ PG::RecoveryCtx OSD::create_context()
ObjectStore::Transaction *t = new ObjectStore::Transaction;
C_Contexts *on_applied = new C_Contexts(cct);
C_Contexts *on_safe = new C_Contexts(cct);
- map< int, map<pg_t,pg_query_t> > *query_map =
- new map<int, map<pg_t, pg_query_t> >;
+ map<int, map<spg_t,pg_query_t> > *query_map =
+ new map<int, map<spg_t, pg_query_t> >;
map<int,vector<pair<pg_notify_t, pg_interval_map_t> > > *notify_list =
- new map<int,vector<pair<pg_notify_t, pg_interval_map_t> > >;
+ new map<int, vector<pair<pg_notify_t, pg_interval_map_t> > >;
map<int,vector<pair<pg_notify_t, pg_interval_map_t> > > *info_map =
new map<int,vector<pair<pg_notify_t, pg_interval_map_t> > >;
PG::RecoveryCtx rctx(query_map, info_map, notify_list,
@@ -6074,16 +6369,25 @@ void OSD::dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg,
bool OSD::compat_must_dispatch_immediately(PG *pg)
{
assert(pg->is_locked());
- vector<int> *tmpacting = &pg->acting;
- if (pg->actingbackfill.size() > 0)
- tmpacting = &pg->actingbackfill;
- for (vector<int>::iterator i = tmpacting->begin();
- i != tmpacting->end();
+ set<pg_shard_t> tmpacting;
+ if (!pg->actingbackfill.empty()) {
+ tmpacting = pg->actingbackfill;
+ } else {
+ for (unsigned i = 0; i < pg->acting.size(); ++i) {
+ tmpacting.insert(
+ pg_shard_t(
+ pg->acting[i],
+ pg->pool.info.ec_pool() ? i : ghobject_t::NO_SHARD));
+ }
+ }
+
+ for (set<pg_shard_t>::iterator i = tmpacting.begin();
+ i != tmpacting.end();
++i) {
- if (*i == whoami)
+ if (i->osd == whoami)
continue;
ConnectionRef conn =
- service.get_con_osd_cluster(*i, pg->get_osdmap()->get_epoch());
+ service.get_con_osd_cluster(i->osd, pg->get_osdmap()->get_epoch());
if (conn && !conn->has_feature(CEPH_FEATURE_INDEP_PG_MAP)) {
return true;
}
@@ -6125,30 +6429,29 @@ void OSD::dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap,
*/
void OSD::do_notifies(
- map< int,vector<pair<pg_notify_t,pg_interval_map_t> > >& notify_list,
+ map<int,vector<pair<pg_notify_t,pg_interval_map_t> > >& notify_list,
OSDMapRef curmap)
{
- for (map< int, vector<pair<pg_notify_t,pg_interval_map_t> > >::iterator it = notify_list.begin();
+ for (map<int,
+ vector<pair<pg_notify_t,pg_interval_map_t> > >::iterator it =
+ notify_list.begin();
it != notify_list.end();
++it) {
- if (it->first == whoami) {
- dout(7) << "do_notify osd." << it->first << " is self, skipping" << dendl;
- continue;
- }
if (!curmap->is_up(it->first))
continue;
- ConnectionRef con = service.get_con_osd_cluster(it->first, curmap->get_epoch());
+ ConnectionRef con = service.get_con_osd_cluster(
+ it->first, curmap->get_epoch());
if (!con)
continue;
_share_map_outgoing(it->first, con.get(), curmap);
if (con->has_feature(CEPH_FEATURE_INDEP_PG_MAP)) {
- dout(7) << "do_notify osd." << it->first
+ dout(7) << "do_notify osd " << it->first
<< " on " << it->second.size() << " PGs" << dendl;
MOSDPGNotify *m = new MOSDPGNotify(curmap->get_epoch(),
it->second);
cluster_messenger->send_message(m, con.get());
} else {
- dout(7) << "do_notify osd." << it->first
+ dout(7) << "do_notify osd " << it->first
<< " sending seperate messages" << dendl;
for (vector<pair<pg_notify_t, pg_interval_map_t> >::iterator i =
it->second.begin();
@@ -6168,10 +6471,10 @@ void OSD::do_notifies(
/** do_queries
* send out pending queries for info | summaries
*/
-void OSD::do_queries(map< int, map<pg_t,pg_query_t> >& query_map,
+void OSD::do_queries(map<int, map<spg_t,pg_query_t> >& query_map,
OSDMapRef curmap)
{
- for (map< int, map<pg_t,pg_query_t> >::iterator pit = query_map.begin();
+ for (map<int, map<spg_t,pg_query_t> >::iterator pit = query_map.begin();
pit != query_map.end();
++pit) {
if (!curmap->is_up(pit->first))
@@ -6190,10 +6493,10 @@ void OSD::do_queries(map< int, map<pg_t,pg_query_t> >& query_map,
dout(7) << "do_queries querying osd." << who
<< " sending seperate messages "
<< " on " << pit->second.size() << " PGs" << dendl;
- for (map<pg_t, pg_query_t>::iterator i = pit->second.begin();
+ for (map<spg_t, pg_query_t>::iterator i = pit->second.begin();
i != pit->second.end();
++i) {
- map<pg_t, pg_query_t> to_send;
+ map<spg_t, pg_query_t> to_send;
to_send.insert(*i);
MOSDPGQuery *m = new MOSDPGQuery(i->second.epoch_sent, to_send);
cluster_messenger->send_message(m, con.get());
@@ -6203,10 +6506,13 @@ void OSD::do_queries(map< int, map<pg_t,pg_query_t> >& query_map,
}
-void OSD::do_infos(map<int,vector<pair<pg_notify_t, pg_interval_map_t> > >& info_map,
+void OSD::do_infos(map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > >& info_map,
OSDMapRef curmap)
{
- for (map<int,vector<pair<pg_notify_t, pg_interval_map_t> > >::iterator p = info_map.begin();
+ for (map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > >::iterator p =
+ info_map.begin();
p != info_map.end();
++p) {
if (!curmap->is_up(p->first))
@@ -6214,9 +6520,11 @@ void OSD::do_infos(map<int,vector<pair<pg_notify_t, pg_interval_map_t> > >& info
for (vector<pair<pg_notify_t,pg_interval_map_t> >::iterator i = p->second.begin();
i != p->second.end();
++i) {
- dout(20) << "Sending info " << i->first.info << " to osd." << p->first << dendl;
+ dout(20) << "Sending info " << i->first.info
+ << " to shard " << p->first << dendl;
}
- ConnectionRef con = service.get_con_osd_cluster(p->first, curmap->get_epoch());
+ ConnectionRef con = service.get_con_osd_cluster(
+ p->first, curmap->get_epoch());
if (!con)
continue;
_share_map_outgoing(p->first, con.get(), curmap);
@@ -6271,12 +6579,13 @@ void OSD::handle_pg_notify(OpRequestRef op)
}
handle_pg_peering_evt(
+ spg_t(it->first.info.pgid.pgid, it->first.to),
it->first.info, it->second,
- it->first.query_epoch, from, true,
+ it->first.query_epoch, pg_shard_t(from, it->first.from), true,
PG::CephPeeringEvtRef(
new PG::CephPeeringEvt(
it->first.epoch_sent, it->first.query_epoch,
- PG::MNotifyRec(from, it->first)))
+ PG::MNotifyRec(pg_shard_t(from, it->first.from), it->first)))
);
}
}
@@ -6300,12 +6609,13 @@ void OSD::handle_pg_log(OpRequestRef op)
op->mark_started();
handle_pg_peering_evt(
+ spg_t(m->info.pgid.pgid, m->to),
m->info, m->past_intervals, m->get_epoch(),
- from, false,
+ pg_shard_t(from, m->from), false,
PG::CephPeeringEvtRef(
new PG::CephPeeringEvt(
m->get_epoch(), m->get_query_epoch(),
- PG::MLogRec(from, m)))
+ PG::MLogRec(pg_shard_t(from, m->from), m)))
);
}
@@ -6332,12 +6642,15 @@ void OSD::handle_pg_info(OpRequestRef op)
}
handle_pg_peering_evt(
+ spg_t(p->first.info.pgid.pgid, p->first.to),
p->first.info, p->second, p->first.epoch_sent,
- from, false,
+ pg_shard_t(from, p->first.from), false,
PG::CephPeeringEvtRef(
new PG::CephPeeringEvt(
p->first.epoch_sent, p->first.query_epoch,
- PG::MInfoRec(from, p->first.info, p->first.epoch_sent)))
+ PG::MInfoRec(
+ pg_shard_t(
+ from, p->first.from), p->first.info, p->first.epoch_sent)))
);
}
}
@@ -6376,7 +6689,8 @@ void OSD::handle_pg_trim(OpRequestRef op)
if (pg->is_primary()) {
// peer is informing us of their last_complete_ondisk
dout(10) << *pg << " replica osd." << from << " lcod " << m->trim_to << dendl;
- pg->peer_last_complete_ondisk[from] = m->trim_to;
+ pg->peer_last_complete_ondisk[pg_shard_t(from, m->pgid.shard)] =
+ m->trim_to;
if (pg->calc_min_last_complete_ondisk()) {
dout(10) << *pg << " min lcod now " << pg->min_last_complete_ondisk << dendl;
pg->trim_peers();
@@ -6576,10 +6890,10 @@ void OSD::handle_pg_query(OpRequestRef op)
map< int, vector<pair<pg_notify_t, pg_interval_map_t> > > notify_list;
- for (map<pg_t,pg_query_t>::iterator it = m->pg_list.begin();
+ for (map<spg_t,pg_query_t>::iterator it = m->pg_list.begin();
it != m->pg_list.end();
++it) {
- pg_t pgid = it->first;
+ spg_t pgid = it->first;
if (pgid.preferred() >= 0) {
dout(10) << "ignoring localized pg " << pgid << dendl;
@@ -6591,15 +6905,17 @@ void OSD::handle_pg_query(OpRequestRef op)
PG::CephPeeringEvtRef(
new PG::CephPeeringEvt(
it->second.epoch_sent, it->second.epoch_sent,
- PG::MQuery(from, it->second, it->second.epoch_sent))));
+ PG::MQuery(pg_shard_t(from, it->second.from),
+ it->second, it->second.epoch_sent))));
continue;
}
if (pg_map.count(pgid)) {
PG *pg = 0;
pg = _lookup_lock_pg(pgid);
- pg->queue_query(it->second.epoch_sent, it->second.epoch_sent,
- from, it->second);
+ pg->queue_query(
+ it->second.epoch_sent, it->second.epoch_sent,
+ pg_shard_t(from, it->second.from), it->second);
pg->unlock();
continue;
}
@@ -6608,13 +6924,16 @@ void OSD::handle_pg_query(OpRequestRef op)
continue;
// get active crush mapping
+ int up_primary, acting_primary;
vector<int> up, acting;
- osdmap->pg_to_up_acting_osds(pgid, up, acting);
+ osdmap->pg_to_up_acting_osds(
+ pgid.pgid, &up, &up_primary, &acting, &acting_primary);
// same primary?
pg_history_t history = it->second.history;
- bool valid_history =
- project_pg_history(pgid, history, it->second.epoch_sent, up, acting);
+ bool valid_history = project_pg_history(
+ pgid, history, it->second.epoch_sent,
+ up, up_primary, acting, acting_primary);
if (!valid_history ||
it->second.epoch_sent < history.same_interval_since) {
@@ -6625,21 +6944,32 @@ void OSD::handle_pg_query(OpRequestRef op)
}
dout(10) << " pg " << pgid << " dne" << dendl;
- pg_info_t empty(pgid);
+ pg_info_t empty(spg_t(pgid.pgid, it->second.to));
+ /* This is racy, but that should be ok: if we complete the deletion
+ * before the pg is recreated, we'll just start it off backfilling
+ * instead of just empty */
+ if (service.deleting_pgs.lookup(pgid))
+ empty.last_backfill = hobject_t();
if (it->second.type == pg_query_t::LOG ||
it->second.type == pg_query_t::FULLLOG) {
ConnectionRef con = service.get_con_osd_cluster(from, osdmap->get_epoch());
if (con) {
- MOSDPGLog *mlog = new MOSDPGLog(osdmap->get_epoch(), empty,
- it->second.epoch_sent);
+ MOSDPGLog *mlog = new MOSDPGLog(
+ it->second.from, it->second.to,
+ osdmap->get_epoch(), empty,
+ it->second.epoch_sent);
_share_map_outgoing(from, con.get(), osdmap);
cluster_messenger->send_message(mlog, con.get());
}
} else {
- notify_list[from].push_back(make_pair(pg_notify_t(it->second.epoch_sent,
- osdmap->get_epoch(),
- empty),
- pg_interval_map_t()));
+ notify_list[from].push_back(
+ make_pair(
+ pg_notify_t(
+ it->second.from, it->second.to,
+ it->second.epoch_sent,
+ osdmap->get_epoch(),
+ empty),
+ pg_interval_map_t()));
}
}
do_notifies(notify_list, osdmap);
@@ -6662,10 +6992,10 @@ void OSD::handle_pg_remove(OpRequestRef op)
op->mark_started();
- for (vector<pg_t>::iterator it = m->pg_list.begin();
+ for (vector<spg_t>::iterator it = m->pg_list.begin();
it != m->pg_list.end();
++it) {
- pg_t pgid = *it;
+ spg_t pgid = *it;
if (pgid.preferred() >= 0) {
dout(10) << "ignoring localized pg " << pgid << dendl;
continue;
@@ -6678,14 +7008,16 @@ void OSD::handle_pg_remove(OpRequestRef op)
dout(5) << "queue_pg_for_deletion: " << pgid << dendl;
PG *pg = _lookup_lock_pg(pgid);
pg_history_t history = pg->info.history;
+ int up_primary, acting_primary;
vector<int> up, acting;
- osdmap->pg_to_up_acting_osds(pgid, up, acting);
- bool valid_history =
- project_pg_history(pg->info.pgid, history, pg->get_osdmap()->get_epoch(),
- up, acting);
+ osdmap->pg_to_up_acting_osds(
+ pgid.pgid, &up, &up_primary, &acting, &acting_primary);
+ bool valid_history = project_pg_history(
+ pg->info.pgid, history, pg->get_osdmap()->get_epoch(),
+ up, up_primary, acting, acting_primary);
if (valid_history &&
history.same_interval_since <= m->get_epoch()) {
- assert(pg->get_primary() == m->get_source().num());
+ assert(pg->get_primary().osd == m->get_source().num());
PGRef _pg(pg);
_remove_pg(pg);
pg->unlock();
@@ -6742,7 +7074,7 @@ void OSD::check_replay_queue()
assert(osd_lock.is_locked());
utime_t now = ceph_clock_now(cct);
- list< pair<pg_t,utime_t> > pgids;
+ list< pair<spg_t,utime_t> > pgids;
replay_queue_lock.Lock();
while (!replay_queue.empty() &&
replay_queue.front().second <= now) {
@@ -6751,8 +7083,8 @@ void OSD::check_replay_queue()
}
replay_queue_lock.Unlock();
- for (list< pair<pg_t,utime_t> >::iterator p = pgids.begin(); p != pgids.end(); ++p) {
- pg_t pgid = p->first;
+ for (list< pair<spg_t,utime_t> >::iterator p = pgids.begin(); p != pgids.end(); ++p) {
+ spg_t pgid = p->first;
if (pg_map.count(pgid)) {
PG *pg = _lookup_lock_pg_with_map_lock_held(pgid);
dout(10) << "check_replay_queue " << *pg << dendl;
@@ -6930,7 +7262,8 @@ void OSDService::reply_op_error(OpRequestRef op, int err, eversion_t v,
int flags;
flags = m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK);
- MOSDOpReply *reply = new MOSDOpReply(m, err, osdmap->get_epoch(), flags);
+ MOSDOpReply *reply = new MOSDOpReply(m, err, osdmap->get_epoch(), flags,
+ true);
reply->set_reply_versions(v, uv);
m->get_connection()->get_messenger()->send_message(reply, m->get_connection());
}
@@ -6945,6 +7278,41 @@ void OSDService::handle_misdirected_op(PG *pg, OpRequestRef op)
return;
}
+ if (pg->is_ec_pg()) {
+ /**
+ * OSD recomputes op target based on current OSDMap. With an EC pg, we
+ * can get this result:
+ * 1) client at map 512 sends an op to osd 3, pg_t 3.9 based on mapping
+ * [CRUSH_ITEM_NONE, 2, 3]/3
+ * 2) OSD 3 at map 513 remaps op to osd 3, spg_t 3.9s0 based on mapping
+ * [3, 2, 3]/3
+ * 3) PG 3.9s0 dequeues the op at epoch 512 and notices that it isn't primary
+ * -- misdirected op
+ * 4) client resends and this time PG 3.9s0 having caught up to 513 gets
+ * it and fulfils it
+ *
+ * We can't compute the op target based on the sending map epoch due to
+ * splitting. The simplest thing is to detect such cases here and drop
+ * them without an error (the client will resend anyway).
+ */
+ OSDMapRef opmap = try_get_map(m->get_map_epoch());
+ if (!opmap) {
+ dout(7) << __func__ << ": " << *pg << " no longer have map for "
+ << m->get_map_epoch() << ", dropping" << dendl;
+ return;
+ }
+ pg_t _pgid = m->get_pg();
+ spg_t pgid;
+ if ((m->get_flags() & CEPH_OSD_FLAG_PGOP) == 0)
+ _pgid = opmap->raw_pg_to_pg(_pgid);
+ if (opmap->get_primary_shard(_pgid, &pgid) &&
+ pgid.shard != pg->info.pgid.shard) {
+ dout(7) << __func__ << ": " << *pg << " primary changed since "
+ << m->get_map_epoch() << ", dropping" << dendl;
+ return;
+ }
+ }
+
dout(7) << *pg << " misdirected op in " << m->get_map_epoch() << dendl;
clog.warn() << m->get_source_inst() << " misdirected " << m->get_reqid()
<< " pg " << m->get_pg()
@@ -7033,18 +7401,24 @@ void OSD::handle_op(OpRequestRef op)
}
}
// calc actual pgid
- pg_t pgid = m->get_pg();
- int64_t pool = pgid.pool();
+ pg_t _pgid = m->get_pg();
+ int64_t pool = _pgid.pool();
if ((m->get_flags() & CEPH_OSD_FLAG_PGOP) == 0 &&
osdmap->have_pg_pool(pool))
- pgid = osdmap->raw_pg_to_pg(pgid);
+ _pgid = osdmap->raw_pg_to_pg(_pgid);
+
+ spg_t pgid;
+ if (!osdmap->get_primary_shard(_pgid, &pgid)) {
+ // missing pool or acting set empty -- drop
+ return;
+ }
// get and lock *pg.
PG *pg = _have_pg(pgid) ? _lookup_pg(pgid) : NULL;
if (!pg) {
dout(7) << "hit non-existent pg " << pgid << dendl;
- if (osdmap->get_pg_acting_role(pgid, whoami) >= 0) {
+ if (osdmap->get_pg_acting_role(pgid.pgid, whoami) >= 0) {
dout(7) << "we are valid target for op, waiting" << dendl;
waiting_for_pg[pgid].push_back(op);
op->mark_delayed("waiting for pg to exist locally");
@@ -7058,7 +7432,7 @@ void OSD::handle_op(OpRequestRef op)
}
OSDMapRef send_map = get_map(m->get_map_epoch());
- if (send_map->get_pg_acting_role(pgid, whoami) >= 0) {
+ if (send_map->get_pg_acting_role(pgid.pgid, whoami) >= 0) {
dout(7) << "dropping request; client will resend when they get new map" << dendl;
} else if (!send_map->have_pg_pool(pgid.pool())) {
dout(7) << "dropping request; pool did not exist" << dendl;
@@ -7071,9 +7445,6 @@ void OSD::handle_op(OpRequestRef op)
<< "\n";
} else {
dout(7) << "we are invalid target" << dendl;
- pgid = m->get_pg();
- if ((m->get_flags() & CEPH_OSD_FLAG_PGOP) == 0)
- pgid = send_map->raw_pg_to_pg(pgid);
clog.warn() << m->get_source_inst() << " misdirected " << m->get_reqid()
<< " pg " << m->get_pg()
<< " to osd." << whoami
@@ -7117,7 +7488,7 @@ void OSD::handle_replica_op(OpRequestRef op)
static_cast<Session*>(m->get_connection()->get_priv()));
// make sure we have the pg
- const pg_t pgid = m->pgid;
+ const spg_t pgid = m->pgid;
if (service.splitting(pgid)) {
waiting_for_pg[pgid].push_back(op);
return;
@@ -7171,13 +7542,11 @@ void OSD::OpWQ::_enqueue(pair<PGRef, OpRequestRef> item)
void OSD::OpWQ::_enqueue_front(pair<PGRef, OpRequestRef> item)
{
- {
- Mutex::Locker l(qlock);
- if (pg_for_processing.count(&*(item.first))) {
- pg_for_processing[&*(item.first)].push_front(item.second);
- item.second = pg_for_processing[&*(item.first)].back();
- pg_for_processing[&*(item.first)].pop_back();
- }
+ Mutex::Locker l(qlock);
+ if (pg_for_processing.count(&*(item.first))) {
+ pg_for_processing[&*(item.first)].push_front(item.second);
+ item.second = pg_for_processing[&*(item.first)].back();
+ pg_for_processing[&*(item.first)].pop_back();
}
unsigned priority = item.second->get_req()->get_priority();
unsigned cost = item.second->get_req()->get_cost();
@@ -7273,7 +7642,7 @@ struct C_CompleteSplits : public Context {
if (osd->is_stopping())
return;
PG::RecoveryCtx rctx = osd->create_context();
- set<pg_t> to_complete;
+ set<spg_t> to_complete;
for (set<boost::intrusive_ptr<PG> >::iterator i = pgs.begin();
i != pgs.end();
++i) {
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index cebceb7..7ee3442 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -122,6 +122,23 @@ enum {
l_osd_stat_bytes_used,
l_osd_stat_bytes_avail,
+ l_osd_copyfrom,
+
+ l_osd_tier_promote,
+ l_osd_tier_flush,
+ l_osd_tier_flush_fail,
+ l_osd_tier_try_flush,
+ l_osd_tier_try_flush_fail,
+ l_osd_tier_evict,
+ l_osd_tier_whiteout,
+ l_osd_tier_dirty,
+ l_osd_tier_clean,
+
+ l_osd_agent_wake,
+ l_osd_agent_skip,
+ l_osd_agent_flush,
+ l_osd_agent_evict,
+
l_osd_last,
};
@@ -197,9 +214,9 @@ class DeletingState {
} status;
bool stop_deleting;
public:
- const pg_t pgid;
+ const spg_t pgid;
const PGRef old_pg_state;
- DeletingState(const pair<pg_t, PGRef> &in) :
+ DeletingState(const pair<spg_t, PGRef> &in) :
lock("DeletingState::lock"), status(QUEUED), stop_deleting(false),
pgid(in.first), old_pg_state(in.second) {}
@@ -289,8 +306,8 @@ class OSDService {
public:
OSD *osd;
CephContext *cct;
- SharedPtrRegistry<pg_t, ObjectStore::Sequencer> osr_registry;
- SharedPtrRegistry<pg_t, DeletingState> deleting_pgs;
+ SharedPtrRegistry<spg_t, ObjectStore::Sequencer> osr_registry;
+ SharedPtrRegistry<spg_t, DeletingState> deleting_pgs;
const int whoami;
ObjectStore *&store;
LogClient &clog;
@@ -359,6 +376,9 @@ public:
Mutex::Locker l(pre_publish_lock);
next_osdmap = map;
}
+
+ void activate_map();
+
ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch);
pair<ConnectionRef,ConnectionRef> get_con_osd_hb(int peer, epoch_t from_epoch); // (back, front)
void send_message_osd_cluster(int peer, Message *m, epoch_t from_epoch);
@@ -382,33 +402,33 @@ public:
Mutex sched_scrub_lock;
int scrubs_pending;
int scrubs_active;
- set< pair<utime_t,pg_t> > last_scrub_pg;
+ set< pair<utime_t,spg_t> > last_scrub_pg;
- void reg_last_pg_scrub(pg_t pgid, utime_t t) {
+ void reg_last_pg_scrub(spg_t pgid, utime_t t) {
Mutex::Locker l(sched_scrub_lock);
- last_scrub_pg.insert(pair<utime_t,pg_t>(t, pgid));
+ last_scrub_pg.insert(pair<utime_t,spg_t>(t, pgid));
}
- void unreg_last_pg_scrub(pg_t pgid, utime_t t) {
+ void unreg_last_pg_scrub(spg_t pgid, utime_t t) {
Mutex::Locker l(sched_scrub_lock);
- pair<utime_t,pg_t> p(t, pgid);
- set<pair<utime_t,pg_t> >::iterator it = last_scrub_pg.find(p);
+ pair<utime_t,spg_t> p(t, pgid);
+ set<pair<utime_t,spg_t> >::iterator it = last_scrub_pg.find(p);
assert(it != last_scrub_pg.end());
last_scrub_pg.erase(it);
}
- bool first_scrub_stamp(pair<utime_t, pg_t> *out) {
+ bool first_scrub_stamp(pair<utime_t, spg_t> *out) {
Mutex::Locker l(sched_scrub_lock);
if (last_scrub_pg.empty())
return false;
- set< pair<utime_t, pg_t> >::iterator iter = last_scrub_pg.begin();
+ set< pair<utime_t, spg_t> >::iterator iter = last_scrub_pg.begin();
*out = *iter;
return true;
}
- bool next_scrub_stamp(pair<utime_t, pg_t> next,
- pair<utime_t, pg_t> *out) {
+ bool next_scrub_stamp(pair<utime_t, spg_t> next,
+ pair<utime_t, spg_t> *out) {
Mutex::Locker l(sched_scrub_lock);
if (last_scrub_pg.empty())
return false;
- set< pair<utime_t, pg_t> >::iterator iter = last_scrub_pg.lower_bound(next);
+ set< pair<utime_t, spg_t> >::iterator iter = last_scrub_pg.lower_bound(next);
if (iter == last_scrub_pg.end())
return false;
++iter;
@@ -427,6 +447,104 @@ public:
void reply_op_error(OpRequestRef op, int err, eversion_t v, version_t uv);
void handle_misdirected_op(PG *pg, OpRequestRef op);
+
+ // -- agent shared state --
+ Mutex agent_lock;
+ Cond agent_cond;
+ map<uint64_t, set<PGRef> > agent_queue;
+ set<PGRef>::iterator agent_queue_pos;
+ bool agent_valid_iterator;
+ int agent_ops;
+ set<hobject_t> agent_oids;
+ bool agent_active;
+ struct AgentThread : public Thread {
+ OSDService *osd;
+ AgentThread(OSDService *o) : osd(o) {}
+ void *entry() {
+ osd->agent_entry();
+ return NULL;
+ }
+ } agent_thread;
+ bool agent_stop_flag;
+
+ void agent_entry();
+ void agent_stop();
+
+ void _enqueue(PG *pg, uint64_t priority) {
+ if (!agent_queue.empty() &&
+ agent_queue.rbegin()->first < priority)
+ agent_valid_iterator = false; // inserting higher-priority queue
+ set<PGRef>& nq = agent_queue[priority];
+ if (nq.empty())
+ agent_cond.Signal();
+ nq.insert(pg);
+ }
+
+ void _dequeue(PG *pg, uint64_t old_priority) {
+ set<PGRef>& oq = agent_queue[old_priority];
+ set<PGRef>::iterator p = oq.find(pg);
+ assert(p != oq.end());
+ if (p == agent_queue_pos)
+ ++agent_queue_pos;
+ oq.erase(p);
+ if (oq.empty()) {
+ if (agent_queue.rbegin()->first == old_priority)
+ agent_valid_iterator = false;
+ agent_queue.erase(old_priority);
+ }
+ }
+
+ /// enable agent for a pg
+ void agent_enable_pg(PG *pg, uint64_t priority) {
+ Mutex::Locker l(agent_lock);
+ _enqueue(pg, priority);
+ }
+
+ /// adjust priority for an enagled pg
+ void agent_adjust_pg(PG *pg, uint64_t old_priority, uint64_t new_priority) {
+ Mutex::Locker l(agent_lock);
+ assert(new_priority != old_priority);
+ _enqueue(pg, new_priority);
+ _dequeue(pg, old_priority);
+ }
+
+ /// disable agent for a pg
+ void agent_disable_pg(PG *pg, uint64_t old_priority) {
+ Mutex::Locker l(agent_lock);
+ _dequeue(pg, old_priority);
+ }
+
+ /// note start of an async (flush) op
+ void agent_start_op(const hobject_t& oid) {
+ Mutex::Locker l(agent_lock);
+ ++agent_ops;
+ assert(agent_oids.count(oid) == 0);
+ agent_oids.insert(oid);
+ }
+
+ /// note finish or cancellation of an async (flush) op
+ void agent_finish_op(const hobject_t& oid) {
+ Mutex::Locker l(agent_lock);
+ assert(agent_ops > 0);
+ --agent_ops;
+ assert(agent_oids.count(oid) == 1);
+ agent_oids.erase(oid);
+ agent_cond.Signal();
+ }
+
+ /// check if we are operating on an object
+ bool agent_is_active_oid(const hobject_t& oid) {
+ Mutex::Locker l(agent_lock);
+ return agent_oids.count(oid);
+ }
+
+ /// get count of active agent ops
+ int agent_get_num_ops() {
+ Mutex::Locker l(agent_lock);
+ return agent_ops;
+ }
+
+
// -- Objecter, for teiring reads/writes from/to other OSDs --
Mutex objecter_lock;
SafeTimer objecter_timer;
@@ -476,11 +594,11 @@ public:
enum {
BACKFILL_LOW = 0, // backfill non-degraded PGs
BACKFILL_HIGH = 1, // backfill degraded PGs
- RECOVERY = AsyncReserver<pg_t>::MAX_PRIORITY // log based recovery
+ RECOVERY = AsyncReserver<spg_t>::MAX_PRIORITY // log based recovery
};
Finisher reserver_finisher;
- AsyncReserver<pg_t> local_reserver;
- AsyncReserver<pg_t> remote_reserver;
+ AsyncReserver<spg_t> local_reserver;
+ AsyncReserver<spg_t> remote_reserver;
// -- pg_temp --
Mutex pg_temp_lock;
@@ -551,26 +669,26 @@ public:
// split
Mutex in_progress_split_lock;
- map<pg_t, pg_t> pending_splits; // child -> parent
- map<pg_t, set<pg_t> > rev_pending_splits; // parent -> [children]
- set<pg_t> in_progress_splits; // child
+ map<spg_t, spg_t> pending_splits; // child -> parent
+ map<spg_t, set<spg_t> > rev_pending_splits; // parent -> [children]
+ set<spg_t> in_progress_splits; // child
- void _start_split(pg_t parent, const set<pg_t> &children);
- void start_split(pg_t parent, const set<pg_t> &children) {
+ void _start_split(spg_t parent, const set<spg_t> &children);
+ void start_split(spg_t parent, const set<spg_t> &children) {
Mutex::Locker l(in_progress_split_lock);
return _start_split(parent, children);
}
- void mark_split_in_progress(pg_t parent, const set<pg_t> &pgs);
- void complete_split(const set<pg_t> &pgs);
- void cancel_pending_splits_for_parent(pg_t parent);
- void _cancel_pending_splits_for_parent(pg_t parent);
- bool splitting(pg_t pgid);
+ void mark_split_in_progress(spg_t parent, const set<spg_t> &pgs);
+ void complete_split(const set<spg_t> &pgs);
+ void cancel_pending_splits_for_parent(spg_t parent);
+ void _cancel_pending_splits_for_parent(spg_t parent);
+ bool splitting(spg_t pgid);
void expand_pg_num(OSDMapRef old_map,
OSDMapRef new_map);
void _maybe_split_pgid(OSDMapRef old_map,
OSDMapRef new_map,
- pg_t pgid);
- void init_splits_between(pg_t pgid, OSDMapRef frommap, OSDMapRef tomap);
+ spg_t pgid);
+ void init_splits_between(spg_t pgid, OSDMapRef frommap, OSDMapRef tomap);
// -- OSD Full Status --
Mutex full_status_lock;
@@ -605,9 +723,9 @@ public:
#ifdef PG_DEBUG_REFS
Mutex pgid_lock;
- map<pg_t, int> pgid_tracker;
- map<pg_t, PG*> live_pgs;
- void add_pgid(pg_t pgid, PG *pg) {
+ map<spg_t, int> pgid_tracker;
+ map<spg_t, PG*> live_pgs;
+ void add_pgid(spg_t pgid, PG *pg) {
Mutex::Locker l(pgid_lock);
if (!pgid_tracker.count(pgid)) {
pgid_tracker[pgid] = 0;
@@ -615,7 +733,7 @@ public:
}
pgid_tracker[pgid]++;
}
- void remove_pgid(pg_t pgid, PG *pg) {
+ void remove_pgid(spg_t pgid, PG *pg) {
Mutex::Locker l(pgid_lock);
assert(pgid_tracker.count(pgid));
assert(pgid_tracker[pgid] > 0);
@@ -628,7 +746,7 @@ public:
void dump_live_pgids() {
Mutex::Locker l(pgid_lock);
derr << "live pgids:" << dendl;
- for (map<pg_t, int>::iterator i = pgid_tracker.begin();
+ for (map<spg_t, int>::iterator i = pgid_tracker.begin();
i != pgid_tracker.end();
++i) {
derr << "\t" << *i << dendl;
@@ -673,7 +791,7 @@ protected:
Messenger *cluster_messenger;
Messenger *client_messenger;
Messenger *objecter_messenger;
- MonClient *monc;
+ MonClient *monc; // check the "monc helpers" list before accessing directly
PerfCounters *logger;
PerfCounters *recoverystate_perf;
ObjectStore *store;
@@ -730,7 +848,7 @@ public:
0));
}
- static hobject_t make_pg_log_oid(pg_t pg) {
+ static hobject_t make_pg_log_oid(spg_t pg) {
stringstream ss;
ss << "pglog_" << pg;
string s;
@@ -738,7 +856,7 @@ public:
return hobject_t(sobject_t(object_t(s.c_str()), 0));
}
- static hobject_t make_pg_biginfo_oid(pg_t pg) {
+ static hobject_t make_pg_biginfo_oid(spg_t pg) {
stringstream ss;
ss << "pginfo_" << pg;
string s;
@@ -789,6 +907,17 @@ public:
static const int STATE_STOPPING = 4;
static const int STATE_WAITING_FOR_HEALTHY = 5;
+ static const char *get_state_name(int s) {
+ switch (s) {
+ case STATE_INITIALIZING: return "initializing";
+ case STATE_BOOTING: return "booting";
+ case STATE_ACTIVE: return "active";
+ case STATE_STOPPING: return "stopping";
+ case STATE_WAITING_FOR_HEALTHY: return "waiting_for_healthy";
+ default: return "???";
+ }
+ }
+
private:
int state;
epoch_t boot_epoch; // _first_ epoch we were marked up (after this process started)
@@ -825,6 +954,23 @@ public:
};
private:
+ /**
+ * @defgroup monc helpers
+ *
+ * Right now we only have the one
+ */
+
+ /**
+ * Ask the Monitors for a sequence of OSDMaps.
+ *
+ * @param epoch The epoch to start with when replying
+ * @param force_request True if this request forces a new subscription to
+ * the monitors; false if an outstanding request that encompasses it is
+ * sufficient.
+ */
+ void osdmap_subscribe(version_t epoch, bool force_request);
+ /** @} monc helpers */
+
// -- heartbeat --
/// information about a heartbeat peer
struct HeartbeatInfo {
@@ -1146,19 +1292,19 @@ private:
protected:
// -- placement groups --
- ceph::unordered_map<pg_t, PG*> pg_map;
- map<pg_t, list<OpRequestRef> > waiting_for_pg;
- map<pg_t, list<PG::CephPeeringEvtRef> > peering_wait_for_split;
+ ceph::unordered_map<spg_t, PG*> pg_map;
+ map<spg_t, list<OpRequestRef> > waiting_for_pg;
+ map<spg_t, list<PG::CephPeeringEvtRef> > peering_wait_for_split;
PGRecoveryStats pg_recovery_stats;
PGPool _get_pool(int id, OSDMapRef createmap);
- bool _have_pg(pg_t pgid);
- PG *_lookup_lock_pg_with_map_lock_held(pg_t pgid);
- PG *_lookup_lock_pg(pg_t pgid);
- PG *_lookup_pg(pg_t pgid);
+ bool _have_pg(spg_t pgid);
+ PG *_lookup_lock_pg_with_map_lock_held(spg_t pgid);
+ PG *_lookup_lock_pg(spg_t pgid);
+ PG *_lookup_pg(spg_t pgid);
PG *_open_lock_pg(OSDMapRef createmap,
- pg_t pg, bool no_lockdep_check=false,
+ spg_t pg, bool no_lockdep_check=false,
bool hold_map_lock=false);
enum res_result {
RES_PARENT, // resurrected a parent
@@ -1166,50 +1312,57 @@ protected:
RES_NONE // nothing relevant deleting
};
res_result _try_resurrect_pg(
- OSDMapRef curmap, pg_t pgid, pg_t *resurrected, PGRef *old_pg_state);
- PG *_create_lock_pg(OSDMapRef createmap,
- pg_t pgid,
- bool newly_created,
- bool hold_map_lock,
- bool backfill,
- int role,
- vector<int>& up,
- vector<int>& acting,
- pg_history_t history,
- pg_interval_map_t& pi,
- ObjectStore::Transaction& t);
- PG *_lookup_qlock_pg(pg_t pgid);
-
- PG* _make_pg(OSDMapRef createmap, pg_t pgid);
+ OSDMapRef curmap, spg_t pgid, spg_t *resurrected, PGRef *old_pg_state);
+ PG *_create_lock_pg(
+ OSDMapRef createmap,
+ spg_t pgid,
+ bool newly_created,
+ bool hold_map_lock,
+ bool backfill,
+ int role,
+ vector<int>& up, int up_primary,
+ vector<int>& acting, int acting_primary,
+ pg_history_t history,
+ pg_interval_map_t& pi,
+ ObjectStore::Transaction& t);
+ PG *_lookup_qlock_pg(spg_t pgid);
+
+ PG* _make_pg(OSDMapRef createmap, spg_t pgid);
void add_newly_split_pg(PG *pg,
PG::RecoveryCtx *rctx);
void handle_pg_peering_evt(
+ spg_t pgid,
const pg_info_t& info,
pg_interval_map_t& pi,
- epoch_t epoch, int from,
+ epoch_t epoch,
+ pg_shard_t from,
bool primary,
PG::CephPeeringEvtRef evt);
void load_pgs();
void build_past_intervals_parallel();
- void calc_priors_during(pg_t pgid, epoch_t start, epoch_t end, set<int>& pset);
+ void calc_priors_during(
+ spg_t pgid, epoch_t start, epoch_t end, set<pg_shard_t>& pset);
/// project pg history from from to now
bool project_pg_history(
- pg_t pgid, pg_history_t& h, epoch_t from,
- const vector<int>& lastup, const vector<int>& lastacting
+ spg_t pgid, pg_history_t& h, epoch_t from,
+ const vector<int>& lastup,
+ int lastupprimary,
+ const vector<int>& lastacting,
+ int lastactingprimary
); ///< @return false if there was a map gap between from and now
- void wake_pg_waiters(pg_t pgid) {
+ void wake_pg_waiters(spg_t pgid) {
if (waiting_for_pg.count(pgid)) {
take_waiters_front(waiting_for_pg[pgid]);
waiting_for_pg.erase(pgid);
}
}
void wake_all_pg_waiters() {
- for (map<pg_t, list<OpRequestRef> >::iterator p = waiting_for_pg.begin();
+ for (map<spg_t, list<OpRequestRef> >::iterator p = waiting_for_pg.begin();
p != waiting_for_pg.end();
++p)
take_waiters_front(p->second);
@@ -1221,20 +1374,20 @@ protected:
struct create_pg_info {
pg_history_t history;
vector<int> acting;
- set<int> prior;
+ set<pg_shard_t> prior;
pg_t parent;
};
- ceph::unordered_map<pg_t, create_pg_info> creating_pgs;
+ ceph::unordered_map<spg_t, create_pg_info> creating_pgs;
double debug_drop_pg_create_probability;
int debug_drop_pg_create_duration;
int debug_drop_pg_create_left; // 0 if we just dropped the last one, -1 if we can drop more
- bool can_create_pg(pg_t pgid);
+ bool can_create_pg(spg_t pgid);
void handle_pg_create(OpRequestRef op);
void split_pgs(
PG *parent,
- const set<pg_t> &childpgids, set<boost::intrusive_ptr<PG> > *out_pgs,
+ const set<spg_t> &childpgids, set<boost::intrusive_ptr<PG> > *out_pgs,
OSDMapRef curmap,
OSDMapRef nextmap,
PG::RecoveryCtx *rctx);
@@ -1251,6 +1404,22 @@ protected:
*/
utime_t last_pg_stats_ack;
bool outstanding_pg_stats; // some stat updates haven't been acked yet
+ bool timeout_mon_on_pg_stats;
+ void restart_stats_timer() {
+ Mutex::Locker l(osd_lock);
+ last_pg_stats_ack = ceph_clock_now(cct);
+ timeout_mon_on_pg_stats = true;
+ }
+
+ class C_MonStatsAckTimer : public Context {
+ OSD *osd;
+ public:
+ C_MonStatsAckTimer(OSD *o) : osd(o) {}
+ void finish(int r) {
+ osd->restart_stats_timer();
+ }
+ };
+ friend class C_MonStatsAckTimer;
void do_mon_report();
@@ -1327,13 +1496,16 @@ protected:
ThreadPool::TPHandle *handle = NULL);
void dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg,
ThreadPool::TPHandle *handle = NULL);
- void do_notifies(map< int,vector<pair<pg_notify_t, pg_interval_map_t> > >& notify_list,
+ void do_notifies(map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > >&
+ notify_list,
OSDMapRef map);
- void do_queries(map< int, map<pg_t,pg_query_t> >& query_map,
+ void do_queries(map<int, map<spg_t,pg_query_t> >& query_map,
OSDMapRef map);
- void do_infos(map<int, vector<pair<pg_notify_t, pg_interval_map_t> > >& info_map,
+ void do_infos(map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > >& info_map,
OSDMapRef map);
- void repeer(PG *pg, map< int, map<pg_t,pg_query_t> >& query_map);
+ void repeer(PG *pg, map< int, map<spg_t,pg_query_t> >& query_map);
bool require_mon_peer(Message *m);
bool require_osd_peer(OpRequestRef op);
@@ -1417,7 +1589,7 @@ protected:
utime_t defer_recovery_until;
int recovery_ops_active;
#ifdef DEBUG_RECOVERY_OIDS
- map<pg_t, set<hobject_t> > recovery_oids;
+ map<spg_t, set<hobject_t> > recovery_oids;
#endif
struct RecoveryWQ : public ThreadPool::WorkQueue<PG> {
@@ -1470,7 +1642,7 @@ protected:
// replay / delayed pg activation
Mutex replay_queue_lock;
- list< pair<pg_t, utime_t > > replay_queue;
+ list< pair<spg_t, utime_t > > replay_queue;
void check_replay_queue();
@@ -1697,7 +1869,7 @@ protected:
}
} remove_wq;
uint64_t next_removal_seq;
- coll_t get_next_removal_coll(pg_t pgid) {
+ coll_t get_next_removal_coll(spg_t pgid) {
return coll_t::make_removal_coll(next_removal_seq++, pgid);
}
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 902717a..57b7082 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -388,7 +388,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
ENCODE_START(7, 7, bl);
{
- ENCODE_START(1, 1, bl); // client-usable data
+ ENCODE_START(2, 1, bl); // client-usable data
::encode(fsid, bl);
::encode(epoch, bl);
::encode(modified, bl);
@@ -406,6 +406,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
::encode(new_weight, bl);
::encode(new_pg_temp, bl);
::encode(new_primary_temp, bl);
+ ::encode(new_primary_affinity, bl);
ENCODE_FINISH(bl); // client-usable data
}
@@ -541,7 +542,7 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
return;
}
{
- DECODE_START(1, bl); // client-usable data
+ DECODE_START(2, bl); // client-usable data
::decode(fsid, bl);
::decode(epoch, bl);
::decode(modified, bl);
@@ -559,6 +560,10 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
::decode(new_weight, bl);
::decode(new_pg_temp, bl);
::decode(new_primary_temp, bl);
+ if (struct_v >= 2)
+ ::decode(new_primary_affinity, bl);
+ else
+ new_primary_affinity.clear();
DECODE_FINISH(bl); // client-usable data
}
@@ -824,6 +829,8 @@ void OSDMap::set_max_osd(int m)
osd_addrs->hb_back_addr.resize(m);
osd_addrs->hb_front_addr.resize(m);
osd_uuid->resize(m);
+ if (osd_primary_affinity)
+ osd_primary_affinity->resize(m, CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
calc_num_osds();
}
@@ -932,6 +939,9 @@ uint64_t OSDMap::get_features(uint64_t *pmask) const
features |= CEPH_FEATURE_CRUSH_TUNABLES2;
if (crush->has_v2_rules())
features |= CEPH_FEATURE_CRUSH_V2;
+ if (crush->has_nondefault_tunables3() ||
+ crush->has_v3_rules())
+ features |= CEPH_FEATURE_CRUSH_TUNABLES3;
mask |= CEPH_FEATURES_CRUSH;
for (map<int64_t,pg_pool_t>::const_iterator p = pools.begin(); p != pools.end(); ++p) {
@@ -949,6 +959,16 @@ uint64_t OSDMap::get_features(uint64_t *pmask) const
mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL |
CEPH_FEATURE_OSD_ERASURE_CODES;
+ if (osd_primary_affinity) {
+ for (int i = 0; i < max_osd; ++i) {
+ if ((*osd_primary_affinity)[i] != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
+ features |= CEPH_FEATURE_OSD_PRIMARY_AFFINITY;
+ break;
+ }
+ }
+ }
+ mask |= CEPH_FEATURE_OSD_PRIMARY_AFFINITY;
+
if (pmask)
*pmask = mask;
return features;
@@ -1165,6 +1185,12 @@ int OSDMap::apply_incremental(const Incremental &inc)
osd_state[i->first] &= ~(CEPH_OSD_AUTOOUT | CEPH_OSD_NEW);
}
+ for (map<int32_t,uint32_t>::const_iterator i = inc.new_primary_affinity.begin();
+ i != inc.new_primary_affinity.end();
+ ++i) {
+ set_primary_affinity(i->first, i->second);
+ }
+
// up/down
for (map<int32_t,uint8_t>::const_iterator i = inc.new_state.begin();
i != inc.new_state.end();
@@ -1335,7 +1361,8 @@ void OSDMap::_remove_nonexistent_osds(const pg_pool_t& pool,
}
int OSDMap::_pg_to_osds(const pg_pool_t& pool, pg_t pg,
- vector<int> *osds, int *primary) const
+ vector<int> *osds, int *primary,
+ ps_t *ppps) const
{
// map to osds[]
ps_t pps = pool.raw_pg_to_pps(pg); // placement ps
@@ -1348,24 +1375,100 @@ int OSDMap::_pg_to_osds(const pg_pool_t& pool, pg_t pg,
_remove_nonexistent_osds(pool, *osds);
- *primary = (osds->empty() ? -1 : osds->front());
+ *primary = -1;
+ for (unsigned i = 0; i < osds->size(); ++i) {
+ if ((*osds)[i] != CRUSH_ITEM_NONE) {
+ *primary = (*osds)[i];
+ break;
+ }
+ }
+ if (ppps)
+ *ppps = pps;
return osds->size();
}
// pg -> (up osd list)
-void OSDMap::_raw_to_up_osds(pg_t pg, const vector<int>& raw,
+void OSDMap::_raw_to_up_osds(const pg_pool_t& pool, const vector<int>& raw,
vector<int> *up, int *primary) const
{
- up->clear();
- for (unsigned i=0; i<raw.size(); i++) {
- if (!exists(raw[i]) || is_down(raw[i]))
+ if (pool.can_shift_osds()) {
+ // shift left
+ up->clear();
+ for (unsigned i=0; i<raw.size(); i++) {
+ if (!exists(raw[i]) || is_down(raw[i]))
+ continue;
+ up->push_back(raw[i]);
+ }
+ *primary = (up->empty() ? -1 : up->front());
+ } else {
+ // set down/dne devices to NONE
+ *primary = -1;
+ up->resize(raw.size());
+ for (int i = raw.size() - 1; i >= 0; --i) {
+ if (!exists(raw[i]) || is_down(raw[i])) {
+ (*up)[i] = CRUSH_ITEM_NONE;
+ } else {
+ *primary = (*up)[i] = raw[i];
+ }
+ }
+ }
+}
+
+void OSDMap::_apply_primary_affinity(ps_t seed,
+ const pg_pool_t& pool,
+ vector<int> *osds,
+ int *primary) const
+{
+ // do we have any non-default primary_affinity values for these osds?
+ if (!osd_primary_affinity)
+ return;
+
+ bool any = false;
+ for (vector<int>::const_iterator p = osds->begin(); p != osds->end(); ++p) {
+ if (*p != CRUSH_ITEM_NONE &&
+ (*osd_primary_affinity)[*p] != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
+ any = true;
+ }
+ }
+ if (!any)
+ return;
+
+ // pick the primary. feed both the seed (for the pg) and the osd
+ // into the hash/rng so that a proportional fraction of an osd's pgs
+ // get rejected as primary.
+ int pos = -1;
+ for (unsigned i = 0; i < osds->size(); ++i) {
+ int o = (*osds)[i];
+ if (o == CRUSH_ITEM_NONE)
continue;
- up->push_back(raw[i]);
+ unsigned a = (*osd_primary_affinity)[o];
+ if (a < CEPH_OSD_MAX_PRIMARY_AFFINITY &&
+ (crush_hash32_2(CRUSH_HASH_RJENKINS1,
+ seed, o) >> 16) >= a) {
+ // we chose not to use this primary. note it anyway as a
+ // fallback in case we don't pick anyone else, but keep looking.
+ if (pos < 0)
+ pos = i;
+ } else {
+ pos = i;
+ break;
+ }
+ }
+ if (pos < 0)
+ return;
+
+ *primary = (*osds)[pos];
+
+ if (pool.can_shift_osds() && pos > 0) {
+ // move the new primary to the front.
+ for (int i = pos; i > 0; --i) {
+ (*osds)[i] = (*osds)[i-1];
+ }
+ (*osds)[0] = *primary;
}
- *primary = (up->empty() ? -1 : up->front());
}
-
+
void OSDMap::_get_temp_osds(const pg_pool_t& pool, pg_t pg,
vector<int> *temp_pg, int *temp_primary) const
{
@@ -1374,17 +1477,29 @@ void OSDMap::_get_temp_osds(const pg_pool_t& pool, pg_t pg,
temp_pg->clear();
if (p != pg_temp->end()) {
for (unsigned i=0; i<p->second.size(); i++) {
- if (!exists(p->second[i]) || is_down(p->second[i]))
- continue;
- temp_pg->push_back(p->second[i]);
+ if (!exists(p->second[i]) || is_down(p->second[i])) {
+ if (pool.can_shift_osds()) {
+ continue;
+ } else {
+ temp_pg->push_back(CRUSH_ITEM_NONE);
+ }
+ } else {
+ temp_pg->push_back(p->second[i]);
+ }
}
}
map<pg_t,int>::const_iterator pp = primary_temp->find(pg);
*temp_primary = -1;
- if (pp != primary_temp->end())
+ if (pp != primary_temp->end()) {
*temp_primary = pp->second;
- else if (!temp_pg->empty()) // apply pg_temp's primary
- *temp_primary = temp_pg->front();
+ } else if (!temp_pg->empty()) { // apply pg_temp's primary
+ for (unsigned i = 0; i < temp_pg->size(); ++i) {
+ if ((*temp_pg)[i] != CRUSH_ITEM_NONE) {
+ *temp_primary = (*temp_pg)[i];
+ break;
+ }
+ }
+ }
}
int OSDMap::pg_to_osds(pg_t pg, vector<int> *raw, int *primary) const
@@ -1394,7 +1509,7 @@ int OSDMap::pg_to_osds(pg_t pg, vector<int> *raw, int *primary) const
const pg_pool_t *pool = get_pg_pool(pg.pool());
if (!pool)
return 0;
- int r = _pg_to_osds(*pool, pg, raw, primary);
+ int r = _pg_to_osds(*pool, pg, raw, primary, NULL);
return r;
}
@@ -1409,8 +1524,10 @@ void OSDMap::pg_to_raw_up(pg_t pg, vector<int> *up, int *primary) const
return;
}
vector<int> raw;
- _pg_to_osds(*pool, pg, &raw, primary);
- _raw_to_up_osds(pg, raw, up, primary);
+ ps_t pps;
+ _pg_to_osds(*pool, pg, &raw, primary, &pps);
+ _raw_to_up_osds(*pool, raw, up, primary);
+ _apply_primary_affinity(pps, *pool, up, primary);
}
void OSDMap::_pg_to_up_acting_osds(pg_t pg, vector<int> *up, int *up_primary,
@@ -1433,13 +1550,17 @@ void OSDMap::_pg_to_up_acting_osds(pg_t pg, vector<int> *up, int *up_primary,
vector<int> _acting;
int _up_primary;
int _acting_primary;
- _pg_to_osds(*pool, pg, &raw, &_up_primary);
- _raw_to_up_osds(pg, raw, &_up, &_up_primary);
+ ps_t pps;
+ _pg_to_osds(*pool, pg, &raw, &_up_primary, &pps);
+ _raw_to_up_osds(*pool, raw, &_up, &_up_primary);
+ _apply_primary_affinity(pps, *pool, &_up, &_up_primary);
_get_temp_osds(*pool, pg, &_acting, &_acting_primary);
- if (_acting.empty())
+ if (_acting.empty()) {
_acting = _up;
- if (_acting_primary == -1)
- _acting_primary = _up_primary;
+ if (_acting_primary == -1) {
+ _acting_primary = _up_primary;
+ }
+ }
if (up)
up->swap(_up);
if (up_primary)
@@ -1450,7 +1571,7 @@ void OSDMap::_pg_to_up_acting_osds(pg_t pg, vector<int> *up, int *up_primary,
*acting_primary = _acting_primary;
}
-int OSDMap::calc_pg_rank(int osd, vector<int>& acting, int nrep)
+int OSDMap::calc_pg_rank(int osd, const vector<int>& acting, int nrep)
{
if (!nrep)
nrep = acting.size();
@@ -1460,13 +1581,31 @@ int OSDMap::calc_pg_rank(int osd, vector<int>& acting, int nrep)
return -1;
}
-int OSDMap::calc_pg_role(int osd, vector<int>& acting, int nrep)
+int OSDMap::calc_pg_role(int osd, const vector<int>& acting, int nrep)
{
if (!nrep)
nrep = acting.size();
return calc_pg_rank(osd, acting, nrep);
}
+bool OSDMap::primary_changed(
+ int oldprimary,
+ const vector<int> &oldacting,
+ int newprimary,
+ const vector<int> &newacting)
+{
+ if (oldacting.empty() && newacting.empty())
+ return false; // both still empty
+ if (oldacting.empty() ^ newacting.empty())
+ return true; // was empty, now not, or vice versa
+ if (oldprimary != newprimary)
+ return true; // primary changed
+ if (calc_pg_rank(oldprimary, oldacting) !=
+ calc_pg_rank(newprimary, newacting))
+ return true;
+ return false; // same primary (tho replicas may have changed)
+}
+
// serialize, unserialize
void OSDMap::encode_client_old(bufferlist& bl) const
@@ -1586,7 +1725,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
ENCODE_START(7, 7, bl);
{
- ENCODE_START(1, 1, bl); // client-usable data
+ ENCODE_START(2, 1, bl); // client-usable data
// base
::encode(fsid, bl);
::encode(epoch, bl);
@@ -1606,6 +1745,12 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
::encode(*pg_temp, bl);
::encode(*primary_temp, bl);
+ if (osd_primary_affinity) {
+ ::encode(*osd_primary_affinity, bl);
+ } else {
+ vector<__u32> v;
+ ::encode(v, bl);
+ }
// crush
bufferlist cbl;
@@ -1743,6 +1888,8 @@ void OSDMap::decode_classic(bufferlist::iterator& p)
else
osd_addrs->hb_front_addr.resize(osd_addrs->hb_back_addr.size());
+ osd_primary_affinity.reset();
+
post_decode();
}
@@ -1766,7 +1913,7 @@ void OSDMap::decode(bufferlist::iterator& bl)
* Since we made it past that hurdle, we can use our normal paths.
*/
{
- DECODE_START(1, bl); // client-usable data
+ DECODE_START(2, bl); // client-usable data
// base
::decode(fsid, bl);
::decode(epoch, bl);
@@ -1786,6 +1933,14 @@ void OSDMap::decode(bufferlist::iterator& bl)
::decode(*pg_temp, bl);
::decode(*primary_temp, bl);
+ if (struct_v >= 2) {
+ osd_primary_affinity.reset(new vector<__u32>);
+ ::decode(*osd_primary_affinity, bl);
+ if (osd_primary_affinity->empty())
+ osd_primary_affinity.reset();
+ } else {
+ osd_primary_affinity.reset();
+ }
// crush
bufferlist cbl;
@@ -1868,6 +2023,8 @@ void OSDMap::dump(Formatter *f) const
f->dump_stream("uuid") << get_uuid(i);
f->dump_int("up", is_up(i));
f->dump_int("in", is_in(i));
+ f->dump_float("weight", get_weightf(i));
+ f->dump_float("primary_affinity", get_primary_affinityf(i));
get_info(i).dump(f);
f->dump_stream("public_addr") << get_addr(i);
f->dump_stream("cluster_addr") << get_cluster_addr(i);
@@ -1971,6 +2128,8 @@ string OSDMap::get_flag_string(unsigned f)
s += ",noscrub";
if (f & CEPH_OSDMAP_NODEEP_SCRUB)
s += ",nodeep-scrub";
+ if (f & CEPH_OSDMAP_NOTIERAGENT)
+ s += ",notieragent";
if (s.length())
s = s.erase(0, 1);
return s;
@@ -2025,6 +2184,8 @@ void OSDMap::print(ostream& out) const
out << (is_up(i) ? " up ":" down");
out << (is_in(i) ? " in ":" out");
out << " weight " << get_weightf(i);
+ if (get_primary_affinity(i) != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY)
+ out << " primary_affinity " << get_primary_affinityf(i);
const osd_info_t& info(get_info(i));
out << " " << info;
out << " " << get_addr(i) << " " << get_cluster_addr(i) << " " << get_hb_back_addr(i)
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index a3acc02..44da1b0 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -135,6 +135,7 @@ public:
map<int32_t,uint32_t> new_weight;
map<pg_t,vector<int32_t> > new_pg_temp; // [] to remove
map<pg_t, int> new_primary_temp; // [-1] to remove
+ map<int32_t,uint32_t> new_primary_affinity;
map<int32_t,epoch_t> new_up_thru;
map<int32_t,pair<epoch_t,epoch_t> > new_last_clean_interval;
map<int32_t,epoch_t> new_lost;
@@ -208,6 +209,7 @@ private:
vector<osd_info_t> osd_info;
ceph::shared_ptr< map<pg_t,vector<int> > > pg_temp; // temp pg mapping (e.g. while we rebuild)
ceph::shared_ptr< map<pg_t,int > > primary_temp; // temp primary mapping (e.g. while we rebuild)
+ ceph::shared_ptr< vector<__u32> > osd_primary_affinity; ///< 16.16 fixed point, 0x10000 = baseline
map<int64_t,pg_pool_t> pools;
map<int64_t,string> pool_name;
@@ -341,6 +343,23 @@ public:
}
void adjust_osd_weights(const map<int,double>& weights, Incremental& inc) const;
+ void set_primary_affinity(int o, int w) {
+ assert(o < max_osd);
+ if (!osd_primary_affinity)
+ osd_primary_affinity.reset(new vector<__u32>(max_osd,
+ CEPH_OSD_DEFAULT_PRIMARY_AFFINITY));
+ (*osd_primary_affinity)[o] = w;
+ }
+ unsigned get_primary_affinity(int o) const {
+ assert(o < max_osd);
+ if (!osd_primary_affinity)
+ return CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;
+ return (*osd_primary_affinity)[o];
+ }
+ float get_primary_affinityf(int o) const {
+ return (float)get_primary_affinity(o) / (float)CEPH_OSD_MAX_PRIMARY_AFFINITY;
+ }
+
bool exists(int osd) const {
//assert(osd >= 0);
return osd >= 0 && osd < max_osd && (osd_state[osd] & CEPH_OSD_EXISTS);
@@ -536,11 +555,15 @@ public:
private:
/// pg -> (raw osd list)
int _pg_to_osds(const pg_pool_t& pool, pg_t pg,
- vector<int> *osds, int *primary) const;
+ vector<int> *osds, int *primary,
+ ps_t *ppps) const;
void _remove_nonexistent_osds(const pg_pool_t& pool, vector<int>& osds) const;
+ void _apply_primary_affinity(ps_t seed, const pg_pool_t& pool,
+ vector<int> *osds, int *primary) const;
+
/// pg -> (up osd list)
- void _raw_to_up_osds(pg_t pg, const vector<int>& raw,
+ void _raw_to_up_osds(const pg_pool_t& pool, const vector<int>& raw,
vector<int> *up, int *primary) const;
/**
@@ -575,7 +598,6 @@ public:
int pg_to_acting_osds(pg_t pg, vector<int>& acting) const {
int primary;
int r = pg_to_acting_osds(pg, &acting, &primary);
- assert(acting.empty() || primary == acting.front());
return r;
}
/**
@@ -597,8 +619,32 @@ public:
void pg_to_up_acting_osds(pg_t pg, vector<int>& up, vector<int>& acting) const {
int up_primary, acting_primary;
pg_to_up_acting_osds(pg, &up, &up_primary, &acting, &acting_primary);
- assert(up.empty() || up_primary == up.front());
- assert(acting.empty() || acting_primary == acting.front());
+ }
+ bool pg_is_ec(pg_t pg) const {
+ map<int64_t, pg_pool_t>::const_iterator i = pools.find(pg.pool());
+ assert(i != pools.end());
+ return i->second.ec_pool();
+ }
+ bool get_primary_shard(pg_t pgid, spg_t *out) const {
+ map<int64_t, pg_pool_t>::const_iterator i = get_pools().find(pgid.pool());
+ if (i == get_pools().end()) {
+ return false;
+ }
+ int primary;
+ vector<int> acting;
+ pg_to_acting_osds(pgid, &acting, &primary);
+ if (i->second.ec_pool()) {
+ for (shard_id_t i = 0; i < acting.size(); ++i) {
+ if (acting[i] == primary) {
+ *out = spg_t(pgid, i);
+ return true;
+ }
+ }
+ } else {
+ *out = spg_t(pgid);
+ return true;
+ }
+ return false;
}
int64_t lookup_pg_pool_name(const string& name) {
@@ -666,8 +712,13 @@ public:
/* what replica # is a given osd? 0 primary, -1 for none. */
- static int calc_pg_rank(int osd, vector<int>& acting, int nrep=0);
- static int calc_pg_role(int osd, vector<int>& acting, int nrep=0);
+ static int calc_pg_rank(int osd, const vector<int>& acting, int nrep=0);
+ static int calc_pg_role(int osd, const vector<int>& acting, int nrep=0);
+ static bool primary_changed(
+ int oldprimary,
+ const vector<int> &oldacting,
+ int newprimary,
+ const vector<int> &newacting);
/* rank is -1 (stray), 0 (primary), 1,2,3,... (replica) */
int get_pg_acting_rank(pg_t pg, int osd) const {
@@ -712,6 +763,11 @@ public:
bool crush_ruleset_in_use(int ruleset) const;
+ void clear_temp() {
+ pg_temp->clear();
+ primary_temp->clear();
+ }
+
private:
void print_osd_line(int cur, ostream *out, Formatter *f) const;
public:
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 4c1d909..82b1bdd 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -33,6 +33,10 @@
#include "messages/MOSDPGPush.h"
#include "messages/MOSDPGPushReply.h"
#include "messages/MOSDPGPull.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
#include "messages/MOSDSubOp.h"
#include "messages/MOSDSubOpReply.h"
@@ -43,9 +47,10 @@
#define dout_subsys ceph_subsys_osd
#undef dout_prefix
#define dout_prefix _prefix(_dout, this)
-static ostream& _prefix(std::ostream *_dout, const PG *pg)
+template <class T>
+static ostream& _prefix(std::ostream *_dout, T *t)
{
- return *_dout << pg->gen_prefix();
+ return *_dout << t->gen_prefix();
}
void PG::get(const string &tag)
@@ -139,16 +144,17 @@ void PGPool::update(OSDMapRef map)
}
PG::PG(OSDService *o, OSDMapRef curmap,
- const PGPool &_pool, pg_t p, const hobject_t& loid,
+ const PGPool &_pool, spg_t p, const hobject_t& loid,
const hobject_t& ioid) :
osd(o),
cct(o->cct),
osdriver(osd->store, coll_t(), OSD::make_snapmapper_oid()),
snap_mapper(
&osdriver,
- p.m_seed,
+ p.ps(),
p.get_split_bits(curmap->get_pg_num(_pool.id)),
- _pool.id),
+ _pool.id,
+ p.shard),
map_lock("PG::map_lock"),
osdmap_ref(curmap), last_persisted_osdmap_ref(curmap), pool(_pool),
_lock("PG::_lock"),
@@ -160,11 +166,13 @@ PG::PG(OSDService *o, OSDMapRef curmap,
info(p),
info_struct_v(0),
coll(p), pg_log(cct), log_oid(loid), biginfo_oid(ioid),
+ missing_loc(this),
recovery_item(this), scrub_item(this), scrub_finalize_item(this), snap_trim_item(this), stat_queue_item(this),
recovery_ops_active(0),
role(0),
state(0),
send_notify(false),
+ pg_whoami(osd->whoami, p.shard),
need_up_thru(false),
last_peering_reset(0),
heartbeat_peer_lock("PG::heartbeat_peer_lock"),
@@ -226,9 +234,12 @@ std::string PG::gen_prefix() const
/********* PG **********/
-void PG::proc_master_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, pg_missing_t& omissing, int from)
+void PG::proc_master_log(
+ ObjectStore::Transaction& t, pg_info_t &oinfo,
+ pg_log_t &olog, pg_missing_t& omissing, pg_shard_t from)
{
- dout(10) << "proc_master_log for osd." << from << ": " << olog << " " << omissing << dendl;
+ dout(10) << "proc_master_log for osd." << from << ": "
+ << olog << " " << omissing << dendl;
assert(!is_active() && is_primary());
// merge log into our own log to build master log. no need to
@@ -240,12 +251,13 @@ void PG::proc_master_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t
dout(10) << " peer osd." << from << " now " << oinfo << " " << omissing << dendl;
might_have_unfound.insert(from);
- search_for_missing(oinfo, &omissing, from);
peer_missing[from].swap(omissing);
}
-void PG::proc_replica_log(ObjectStore::Transaction& t,
- pg_info_t &oinfo, pg_log_t &olog, pg_missing_t& omissing, int from)
+void PG::proc_replica_log(
+ ObjectStore::Transaction& t,
+ pg_info_t &oinfo, pg_log_t &olog, pg_missing_t& omissing,
+ pg_shard_t from)
{
dout(10) << "proc_replica_log for osd." << from << ": "
<< oinfo << " " << olog << " " << omissing << dendl;
@@ -256,7 +268,6 @@ void PG::proc_replica_log(ObjectStore::Transaction& t,
dout(10) << " peer osd." << from << " now " << oinfo << " " << omissing << dendl;
might_have_unfound.insert(from);
- search_for_missing(oinfo, &omissing, from);
for (map<hobject_t, pg_missing_t::item>::iterator i = omissing.missing.begin();
i != omissing.missing.end();
++i) {
@@ -266,9 +277,9 @@ void PG::proc_replica_log(ObjectStore::Transaction& t,
peer_missing[from].swap(omissing);
}
-bool PG::proc_replica_info(int from, const pg_info_t &oinfo)
+bool PG::proc_replica_info(pg_shard_t from, const pg_info_t &oinfo)
{
- map<int,pg_info_t>::iterator p = peer_info.find(from);
+ map<pg_shard_t, pg_info_t>::iterator p = peer_info.find(from);
if (p != peer_info.end() && p->second.last_update == oinfo.last_update) {
dout(10) << " got dup osd." << from << " info " << oinfo << ", identical to ours" << dendl;
return false;
@@ -285,7 +296,7 @@ bool PG::proc_replica_info(int from, const pg_info_t &oinfo)
reg_next_scrub();
// stray?
- if (!is_acting(from)) {
+ if (!is_up(from) && !is_acting(from)) {
dout(10) << " osd." << from << " has stray content: " << oinfo << dendl;
stray_set.insert(from);
if (is_clean()) {
@@ -303,7 +314,9 @@ bool PG::proc_replica_info(int from, const pg_info_t &oinfo)
void PG::remove_snap_mapped_object(
ObjectStore::Transaction &t, const hobject_t &soid)
{
- t.remove(coll, soid);
+ t.remove(
+ coll,
+ ghobject_t(soid, ghobject_t::NO_GEN, pg_whoami.shard));
clear_object_snap_mapping(&t, soid);
}
@@ -341,7 +354,7 @@ void PG::update_object_snap_mapping(
}
void PG::merge_log(
- ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from)
+ ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, pg_shard_t from)
{
PGLogEntryHandler rollbacker;
pg_log.merge_log(
@@ -364,19 +377,58 @@ void PG::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead)
* TODO: if the missing set becomes very large, this could get expensive.
* Instead, we probably want to just iterate over our unfound set.
*/
-bool PG::search_for_missing(const pg_info_t &oinfo, const pg_missing_t *omissing,
- int fromosd)
-{
- bool stats_updated = false;
- bool found_missing = false;
+bool PG::search_for_missing(
+ const pg_info_t &oinfo, const pg_missing_t &omissing,
+ pg_shard_t from,
+ RecoveryCtx *ctx)
+{
+ unsigned num_unfound_before = missing_loc.num_unfound();
+ bool found_missing = missing_loc.add_source_info(
+ from, oinfo, omissing);
+ if (found_missing && num_unfound_before != missing_loc.num_unfound())
+ publish_stats_to_osd();
+ if (found_missing &&
+ (get_osdmap()->get_features(NULL) & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+ pg_info_t tinfo(oinfo);
+ tinfo.pgid.shard = pg_whoami.shard;
+ (*(ctx->info_map))[from.osd].push_back(
+ make_pair(
+ pg_notify_t(
+ from.shard, pg_whoami.shard,
+ get_osdmap()->get_epoch(),
+ get_osdmap()->get_epoch(),
+ tinfo),
+ past_intervals));
+ }
+ return found_missing;
+}
- // take note that we've probed this peer, for
- // all_unfound_are_queried_or_lost()'s benefit.
- peer_missing[fromosd];
+bool PG::MissingLoc::readable_with_acting(
+ const hobject_t &hoid,
+ const set<pg_shard_t> &acting) const {
+ if (!needs_recovery(hoid)) return true;
+ if (!missing_loc.count(hoid)) return false;
+ const set<pg_shard_t> &locs = missing_loc.find(hoid)->second;
+ dout(10) << __func__ << ": locs:" << locs << dendl;
+ set<pg_shard_t> have_acting;
+ for (set<pg_shard_t>::const_iterator i = locs.begin();
+ i != locs.end();
+ ++i) {
+ if (acting.count(*i))
+ have_acting.insert(*i);
+ }
+ return (*is_readable)(have_acting);
+}
+bool PG::MissingLoc::add_source_info(
+ pg_shard_t fromosd,
+ const pg_info_t &oinfo,
+ const pg_missing_t &omissing)
+{
+ bool found_missing = false;;
// found items?
- for (map<hobject_t,pg_missing_t::item>::const_iterator p = pg_log.get_missing().missing.begin();
- p != pg_log.get_missing().missing.end();
+ for (map<hobject_t,pg_missing_t::item>::const_iterator p = needs_recovery_map.begin();
+ p != needs_recovery_map.end();
++p) {
const hobject_t &soid(p->first);
eversion_t need = p->second.need;
@@ -397,70 +449,46 @@ bool PG::search_for_missing(const pg_info_t &oinfo, const pg_missing_t *omissing
continue;
}
if (oinfo.last_complete < need) {
- if (!omissing) {
- // We know that the peer lacks some objects at the revision we need.
- // Without the peer's missing set, we don't know whether it has this
- // particular object or not.
- dout(10) << __func__ << " " << soid << " " << need
- << " might also be missing on osd." << fromosd << dendl;
- continue;
- }
-
- if (omissing->is_missing(soid)) {
+ if (omissing.is_missing(soid)) {
dout(10) << "search_for_missing " << soid << " " << need
<< " also missing on osd." << fromosd << dendl;
continue;
}
}
+
dout(10) << "search_for_missing " << soid << " " << need
<< " is on osd." << fromosd << dendl;
- map<hobject_t, set<int> >::iterator ml = missing_loc.find(soid);
- if (ml == missing_loc.end()) {
- map<hobject_t, list<OpRequestRef> >::iterator wmo =
- waiting_for_missing_object.find(soid);
- if (wmo != waiting_for_missing_object.end()) {
- requeue_ops(wmo->second);
- }
- stats_updated = true;
- missing_loc[soid].insert(fromosd);
- missing_loc_sources.insert(fromosd);
- }
- else {
- ml->second.insert(fromosd);
- missing_loc_sources.insert(fromosd);
- }
+ missing_loc[soid].insert(fromosd);
+ missing_loc_sources.insert(fromosd);
found_missing = true;
}
- if (stats_updated) {
- publish_stats_to_osd();
- }
- dout(20) << "search_for_missing missing " << pg_log.get_missing().missing << dendl;
+ dout(20) << "needs_recovery_map missing " << needs_recovery_map << dendl;
return found_missing;
}
-void PG::discover_all_missing(map< int, map<pg_t,pg_query_t> > &query_map)
+void PG::discover_all_missing(map<int, map<spg_t,pg_query_t> > &query_map)
{
const pg_missing_t &missing = pg_log.get_missing();
- assert(missing.have_missing());
+ assert(have_unfound());
dout(10) << __func__ << " "
<< missing.num_missing() << " missing, "
<< get_num_unfound() << " unfound"
<< dendl;
- std::set<int>::const_iterator m = might_have_unfound.begin();
- std::set<int>::const_iterator mend = might_have_unfound.end();
+ std::set<pg_shard_t>::const_iterator m = might_have_unfound.begin();
+ std::set<pg_shard_t>::const_iterator mend = might_have_unfound.end();
for (; m != mend; ++m) {
- int peer(*m);
+ pg_shard_t peer(*m);
- if (!get_osdmap()->is_up(peer)) {
+ if (!get_osdmap()->is_up(peer.osd)) {
dout(20) << __func__ << " skipping down osd." << peer << dendl;
continue;
}
- map<int, pg_info_t>::const_iterator iter = peer_info.find(peer);
+ map<pg_shard_t, pg_info_t>::const_iterator iter = peer_info.find(peer);
if (iter != peer_info.end() &&
(iter->second.is_empty() || iter->second.dne())) {
// ignore empty peers
@@ -490,8 +518,11 @@ void PG::discover_all_missing(map< int, map<pg_t,pg_query_t> > &query_map)
dout(10) << __func__ << ": osd." << peer << ": requesting pg_missing_t"
<< dendl;
peer_missing_requested.insert(peer);
- query_map[peer][info.pgid] =
- pg_query_t(pg_query_t::MISSING, info.history, get_osdmap()->get_epoch());
+ query_map[peer.osd][spg_t(info.pgid.pgid, peer.shard)] =
+ pg_query_t(
+ pg_query_t::FULLLOG,
+ peer.shard, pg_whoami.shard,
+ info.history, get_osdmap()->get_epoch());
}
}
@@ -509,14 +540,14 @@ bool PG::needs_recovery() const
ret = true;
}
- assert(actingbackfill.size() > 0);
- vector<int>::const_iterator end = actingbackfill.end();
- vector<int>::const_iterator a = actingbackfill.begin();
+ assert(!actingbackfill.empty());
+ set<pg_shard_t>::const_iterator end = actingbackfill.end();
+ set<pg_shard_t>::const_iterator a = actingbackfill.begin();
assert(a != end);
- ++a;
for (; a != end; ++a) {
- int peer = *a;
- map<int, pg_missing_t>::const_iterator pm = peer_missing.find(peer);
+ if (*a == get_primary()) continue;
+ pg_shard_t peer = *a;
+ map<pg_shard_t, pg_missing_t>::const_iterator pm = peer_missing.find(peer);
if (pm == peer_missing.end()) {
dout(10) << __func__ << " osd." << peer << " don't have missing set" << dendl;
ret = true;
@@ -541,11 +572,11 @@ bool PG::needs_backfill() const
// We can assume that only possible osds that need backfill
// are on the backfill_targets vector nodes.
- vector<int>::const_iterator end = backfill_targets.end();
- vector<int>::const_iterator a = backfill_targets.begin();
+ set<pg_shard_t>::const_iterator end = backfill_targets.end();
+ set<pg_shard_t>::const_iterator a = backfill_targets.begin();
for (; a != end; ++a) {
- int peer = *a;
- map<int,pg_info_t>::const_iterator pi = peer_info.find(peer);
+ pg_shard_t peer = *a;
+ map<pg_shard_t, pg_info_t>::const_iterator pi = peer_info.find(peer);
if (!pi->second.last_backfill.is_max()) {
dout(10) << __func__ << " osd." << peer << " has last_backfill " << pi->second.last_backfill << dendl;
ret = true;
@@ -593,10 +624,13 @@ void PG::generate_past_intervals()
}
OSDMapRef last_map, cur_map;
+ int primary = -1;
+ int old_primary = -1;
vector<int> acting, up, old_acting, old_up;
cur_map = osd->get_map(cur_epoch);
- cur_map->pg_to_up_acting_osds(get_pgid(), up, acting);
+ cur_map->pg_to_up_acting_osds(
+ get_pgid().pgid, &up, 0, &acting, &primary);
epoch_t same_interval_since = cur_epoch;
dout(10) << __func__ << " over epochs " << cur_epoch << "-"
<< end_epoch << dendl;
@@ -605,12 +639,16 @@ void PG::generate_past_intervals()
last_map.swap(cur_map);
old_up.swap(up);
old_acting.swap(acting);
+ old_primary = primary;
cur_map = osd->get_map(cur_epoch);
- cur_map->pg_to_up_acting_osds(get_pgid(), up, acting);
+ cur_map->pg_to_up_acting_osds(
+ get_pgid().pgid, &up, 0, &acting, &primary);
std::stringstream debug;
bool new_interval = pg_interval_t::check_new_interval(
+ old_primary,
+ primary,
old_acting,
acting,
old_up,
@@ -620,7 +658,7 @@ void PG::generate_past_intervals()
cur_map,
last_map,
info.pgid.pool(),
- info.pgid,
+ info.pgid.pgid,
&past_intervals,
&debug);
if (new_interval) {
@@ -669,9 +707,9 @@ void PG::remove_down_peer_info(const OSDMapRef osdmap)
{
// Remove any downed osds from peer_info
bool removed = false;
- map<int,pg_info_t>::iterator p = peer_info.begin();
+ map<pg_shard_t, pg_info_t>::iterator p = peer_info.begin();
while (p != peer_info.end()) {
- if (!osdmap->is_up(p->first)) {
+ if (!osdmap->is_up(p->first.osd)) {
dout(10) << " dropping down osd." << p->first << " info " << p->second << dendl;
peer_missing.erase(p->first);
peer_log_requested.erase(p->first);
@@ -695,16 +733,16 @@ bool PG::all_unfound_are_queried_or_lost(const OSDMapRef osdmap) const
{
assert(is_primary());
- set<int>::const_iterator peer = might_have_unfound.begin();
- set<int>::const_iterator mend = might_have_unfound.end();
+ set<pg_shard_t>::const_iterator peer = might_have_unfound.begin();
+ set<pg_shard_t>::const_iterator mend = might_have_unfound.end();
for (; peer != mend; ++peer) {
if (peer_missing.count(*peer))
continue;
- map<int, pg_info_t>::const_iterator iter = peer_info.find(*peer);
+ map<pg_shard_t, pg_info_t>::const_iterator iter = peer_info.find(*peer);
if (iter != peer_info.end() &&
(iter->second.is_empty() || iter->second.dne()))
continue;
- const osd_info_t &osd_info(osdmap->get_info(*peer));
+ const osd_info_t &osd_info(osdmap->get_info(peer->osd));
if (osd_info.lost_at <= osd_info.up_from) {
// If there is even one OSD in might_have_unfound that isn't lost, we
// still might retrieve our unfound.
@@ -720,18 +758,22 @@ void PG::build_prior(std::auto_ptr<PriorSet> &prior_set)
{
if (1) {
// sanity check
- for (map<int,pg_info_t>::iterator it = peer_info.begin();
+ for (map<pg_shard_t,pg_info_t>::iterator it = peer_info.begin();
it != peer_info.end();
++it) {
assert(info.history.last_epoch_started >= it->second.history.last_epoch_started);
}
}
- prior_set.reset(new PriorSet(*get_osdmap(),
- past_intervals,
- up,
- acting,
- info,
- this));
+ prior_set.reset(
+ new PriorSet(
+ pool.info.ec_pool(),
+ get_pgbackend()->get_is_recoverable_predicate(),
+ *get_osdmap(),
+ past_intervals,
+ up,
+ acting,
+ info,
+ this));
PriorSet &prior(*prior_set.get());
if (prior.pg_down) {
@@ -752,7 +794,7 @@ void PG::build_prior(std::auto_ptr<PriorSet> &prior_set)
set_probe_targets(prior_set->probe);
}
-void PG::clear_primary_state()
+void PG::clear_primary_state(bool staying_primary)
{
dout(10) << "clear_primary_state" << dendl;
@@ -777,7 +819,6 @@ void PG::clear_primary_state()
finish_sync_event = 0; // so that _finish_recvoery doesn't go off in another thread
missing_loc.clear();
- missing_loc_sources.clear();
pg_log.reset_recovery_pointers();
@@ -786,6 +827,11 @@ void PG::clear_primary_state()
osd->recovery_wq.dequeue(this);
osd->snap_trim_wq.dequeue(this);
+
+ if (!staying_primary)
+ agent_clear();
+
+ osd->remove_want_pg_temp(info.pgid.pgid);
}
/**
@@ -796,11 +842,12 @@ void PG::clear_primary_state()
* 2) Prefer longer tail if it brings another info into contiguity
* 3) Prefer current primary
*/
-map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t> &infos) const
+map<pg_shard_t, pg_info_t>::const_iterator PG::find_best_info(
+ const map<pg_shard_t, pg_info_t> &infos) const
{
eversion_t min_last_update_acceptable = eversion_t::max();
epoch_t max_last_epoch_started_found = 0;
- for (map<int, pg_info_t>::const_iterator i = infos.begin();
+ for (map<pg_shard_t, pg_info_t>::const_iterator i = infos.begin();
i != infos.end();
++i) {
if (max_last_epoch_started_found < i->second.last_epoch_started) {
@@ -814,12 +861,12 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
}
assert(min_last_update_acceptable != eversion_t::max());
- map<int, pg_info_t>::const_iterator best = infos.end();
+ map<pg_shard_t, pg_info_t>::const_iterator best = infos.end();
// find osd with newest last_update (oldest for ec_pool).
// if there are multiples, prefer
// - a longer tail, if it brings another peer into log contiguity
// - the current primary
- for (map<int, pg_info_t>::const_iterator p = infos.begin();
+ for (map<pg_shard_t, pg_info_t>::const_iterator p = infos.begin();
p != infos.end();
++p) {
// Only consider peers with last_update >= min_last_update_acceptable
@@ -833,7 +880,7 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
continue;
}
// Prefer newer last_update
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (p->second.last_update > best->second.last_update)
continue;
if (p->second.last_update < best->second.last_update) {
@@ -848,26 +895,17 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
continue;
}
}
- // Prefer longer tail if it brings another peer into contiguity
- for (map<int, pg_info_t>::const_iterator q = infos.begin();
- q != infos.end();
- ++q) {
- if (q->second.is_incomplete())
- continue; // don't care about log contiguity
- if (q->second.last_update < best->second.log_tail &&
- q->second.last_update >= p->second.log_tail) {
- dout(10) << "calc_acting prefer osd." << p->first
- << " because it brings osd." << q->first << " into log contiguity" << dendl;
- best = p;
- continue;
- }
- if (q->second.last_update < p->second.log_tail &&
- q->second.last_update >= best->second.log_tail) {
- continue;
- }
+
+ // Prefer longer tail
+ if (p->second.log_tail > best->second.log_tail) {
+ continue;
+ } else if (p->second.log_tail < best->second.log_tail) {
+ best = p;
+ continue;
}
+
// prefer current primary (usually the caller), all things being equal
- if (p->first == acting[0]) {
+ if (p->first == pg_whoami) {
dout(10) << "calc_acting prefer osd." << p->first
<< " because it is current primary" << dendl;
best = p;
@@ -877,6 +915,85 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
return best;
}
+void PG::calc_ec_acting(
+ map<pg_shard_t, pg_info_t>::const_iterator auth_log_shard,
+ unsigned size,
+ const vector<int> &acting,
+ pg_shard_t acting_primary,
+ const vector<int> &up,
+ pg_shard_t up_primary,
+ const map<pg_shard_t, pg_info_t> &all_info,
+ bool compat_mode,
+ vector<int> *_want,
+ set<pg_shard_t> *backfill,
+ set<pg_shard_t> *acting_backfill,
+ pg_shard_t *want_primary,
+ ostream &ss) {
+ vector<int> want(size, CRUSH_ITEM_NONE);
+ map<shard_id_t, set<pg_shard_t> > all_info_by_shard;
+ unsigned usable = 0;
+ for(map<pg_shard_t, pg_info_t>::const_iterator i = all_info.begin();
+ i != all_info.end();
+ ++i) {
+ all_info_by_shard[i->first.shard].insert(i->first);
+ }
+ for (shard_id_t i = 0; i < want.size(); ++i) {
+ ss << "For position " << (unsigned)i << ": ";
+ if (up.size() > (unsigned)i && up[i] != CRUSH_ITEM_NONE &&
+ !all_info.find(pg_shard_t(up[i], i))->second.is_incomplete() &&
+ all_info.find(pg_shard_t(up[i], i))->second.last_update >=
+ auth_log_shard->second.log_tail) {
+ ss << " selecting up[i]: " << pg_shard_t(up[i], i) << std::endl;
+ want[i] = up[i];
+ ++usable;
+ continue;
+ }
+ if (up.size() > (unsigned)i && up[i] != CRUSH_ITEM_NONE) {
+ ss << " backfilling up[i]: " << pg_shard_t(up[i], i)
+ << " and ";
+ backfill->insert(pg_shard_t(up[i], i));
+ }
+
+ if (acting.size() > (unsigned)i && acting[i] != CRUSH_ITEM_NONE &&
+ !all_info.find(pg_shard_t(acting[i], i))->second.is_incomplete() &&
+ all_info.find(pg_shard_t(acting[i], i))->second.last_update >=
+ auth_log_shard->second.log_tail) {
+ ss << " selecting acting[i]: " << pg_shard_t(acting[i], i) << std::endl;
+ want[i] = acting[i];
+ ++usable;
+ } else {
+ for (set<pg_shard_t>::iterator j = all_info_by_shard[i].begin();
+ j != all_info_by_shard[i].end();
+ ++j) {
+ assert(j->shard == i);
+ if (!all_info.find(*j)->second.is_incomplete() &&
+ all_info.find(*j)->second.last_update >=
+ auth_log_shard->second.log_tail) {
+ ss << " selecting stray: " << *j << std::endl;
+ want[i] = j->osd;
+ ++usable;
+ break;
+ }
+ }
+ if (want[i] == CRUSH_ITEM_NONE)
+ ss << " failed to fill position " << i << std::endl;
+ }
+ }
+
+ bool found_primary = false;
+ for (shard_id_t i = 0; i < want.size(); ++i) {
+ if (want[i] != CRUSH_ITEM_NONE) {
+ acting_backfill->insert(pg_shard_t(want[i], i));
+ if (!found_primary) {
+ *want_primary = pg_shard_t(want[i], i);
+ found_primary = true;
+ }
+ }
+ }
+ acting_backfill->insert(backfill->begin(), backfill->end());
+ _want->swap(want);
+}
+
/**
* calculate the desired acting set.
*
@@ -884,69 +1001,45 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
* incomplete, or another osd has a longer tail that allows us to
* bring other up nodes up to date.
*/
-bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want, vector<int>& backfill) const
-{
- map<int, pg_info_t> all_info(peer_info.begin(), peer_info.end());
- all_info[osd->whoami] = info;
-
- for (map<int,pg_info_t>::iterator p = all_info.begin(); p != all_info.end(); ++p) {
- dout(10) << "calc_acting osd." << p->first << " " << p->second << dendl;
- }
-
- map<int, pg_info_t>::const_iterator newest_update_osd = find_best_info(all_info);
-
- if (newest_update_osd == all_info.end()) {
- if (up != acting) {
- dout(10) << "calc_acting no suitable info found (incomplete backfills?), reverting to up" << dendl;
- want = up;
- return true;
- } else {
- dout(10) << "calc_acting no suitable info found (incomplete backfills?)" << dendl;
- return false;
- }
- }
-
-
- dout(10) << "calc_acting newest update on osd." << newest_update_osd->first
- << " with " << newest_update_osd->second << dendl;
- newest_update_osd_id = newest_update_osd->first;
+void PG::calc_replicated_acting(
+ map<pg_shard_t, pg_info_t>::const_iterator auth_log_shard,
+ unsigned size,
+ const vector<int> &acting,
+ pg_shard_t acting_primary,
+ const vector<int> &up,
+ pg_shard_t up_primary,
+ const map<pg_shard_t, pg_info_t> &all_info,
+ bool compat_mode,
+ vector<int> *want,
+ set<pg_shard_t> *backfill,
+ set<pg_shard_t> *acting_backfill,
+ pg_shard_t *want_primary,
+ ostream &ss)
+{
+ ss << "calc_acting newest update on osd." << auth_log_shard->first
+ << " with " << auth_log_shard->second << std::endl;
+ pg_shard_t auth_log_shard_id = auth_log_shard->first;
// select primary
- map<int,pg_info_t>::const_iterator primary;
+ map<pg_shard_t,pg_info_t>::const_iterator primary;
if (up.size() &&
- !all_info[up[0]].is_incomplete() &&
- all_info[up[0]].last_update >= newest_update_osd->second.log_tail) {
- dout(10) << "up[0](osd." << up[0] << ") selected as primary" << dendl;
- primary = all_info.find(up[0]); // prefer up[0], all thing being equal
- } else if (!newest_update_osd->second.is_incomplete()) {
- dout(10) << "up[0] needs backfill, osd." << newest_update_osd_id
- << " selected as primary instead" << dendl;
- primary = newest_update_osd;
+ !all_info.find(up_primary)->second.is_incomplete() &&
+ all_info.find(up_primary)->second.last_update >=
+ auth_log_shard->second.log_tail) {
+ ss << "up_primary: " << up_primary << ") selected as primary" << std::endl;
+ primary = all_info.find(up_primary); // prefer up[0], all thing being equal
} else {
- map<int, pg_info_t> complete_infos;
- for (map<int, pg_info_t>::iterator i = all_info.begin();
- i != all_info.end();
- ++i) {
- if (!i->second.is_incomplete())
- complete_infos.insert(*i);
- }
- primary = find_best_info(complete_infos);
- if (primary == complete_infos.end() ||
- primary->second.last_update < newest_update_osd->second.log_tail) {
- dout(10) << "calc_acting no acceptable primary, reverting to up " << up << dendl;
- want = up;
- return true;
- } else {
- dout(10) << "up[0] and newest_update_osd need backfill, osd."
- << newest_update_osd_id
- << " selected as primary instead" << dendl;
- }
+ assert(!auth_log_shard->second.is_incomplete());
+ ss << "up[0] needs backfill, osd." << auth_log_shard_id
+ << " selected as primary instead" << std::endl;
+ primary = auth_log_shard;
}
-
- dout(10) << "calc_acting primary is osd." << primary->first
- << " with " << primary->second << dendl;
- want.push_back(primary->first);
+ ss << "calc_acting primary is osd." << primary->first
+ << " with " << primary->second << std::endl;
+ *want_primary = primary->first;
+ want->push_back(primary->first.osd);
+ acting_backfill->insert(primary->first);
unsigned usable = 1;
// select replicas that have log contiguity with primary.
@@ -954,24 +1047,35 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want, vector<int>&
for (vector<int>::const_iterator i = up.begin();
i != up.end();
++i) {
- if (*i == primary->first)
+ pg_shard_t up_cand = pg_shard_t(*i, ghobject_t::no_shard());
+ if (up_cand == primary->first)
continue;
- const pg_info_t &cur_info = all_info.find(*i)->second;
+ const pg_info_t &cur_info = all_info.find(up_cand)->second;
if (cur_info.is_incomplete() ||
cur_info.last_update < MIN(
primary->second.log_tail,
- newest_update_osd->second.log_tail)) {
- /* We include newest_update_osd->second.log_tail because in GetLog,
+ auth_log_shard->second.log_tail)) {
+ /* We include auth_log_shard->second.log_tail because in GetLog,
* we will request logs back to the min last_update over our
* acting_backfill set, which will result in our log being extended
* as far backwards as necessary to pick up any peers which can
- * be log recovered by newest_update_osd's log */
- dout(10) << " osd." << *i << " (up) backfill " << cur_info << dendl;
- backfill.push_back(*i);
+ * be log recovered by auth_log_shard's log */
+ ss << " shard " << up_cand << " (up) backfill " << cur_info << std::endl;
+ if (compat_mode) {
+ if (backfill->empty()) {
+ backfill->insert(up_cand);
+ want->push_back(*i);
+ acting_backfill->insert(up_cand);
+ }
+ } else {
+ backfill->insert(up_cand);
+ acting_backfill->insert(up_cand);
+ }
} else {
- want.push_back(*i);
+ want->push_back(*i);
+ acting_backfill->insert(up_cand);
usable++;
- dout(10) << " osd." << *i << " (up) accepted " << cur_info << dendl;
+ ss << " osd." << *i << " (up) accepted " << cur_info << std::endl;
}
}
@@ -979,52 +1083,60 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want, vector<int>&
for (vector<int>::const_iterator i = acting.begin();
i != acting.end();
++i) {
- if (usable >= get_osdmap()->get_pg_size(info.pgid))
+ pg_shard_t acting_cand(*i, ghobject_t::no_shard());
+ if (usable >= size)
break;
// skip up osds we already considered above
- if (*i == primary->first)
+ if (acting_cand == primary->first)
continue;
- vector<int>::const_iterator up_it = find(up.begin(), up.end(), *i);
+ vector<int>::const_iterator up_it = find(up.begin(), up.end(), acting_cand.osd);
if (up_it != up.end())
continue;
- const pg_info_t &cur_info = all_info.find(*i)->second;
- if (cur_info.is_incomplete() || cur_info.last_update < primary->second.log_tail) {
- dout(10) << " osd." << *i << " (stray) REJECTED " << cur_info << dendl;
+ const pg_info_t &cur_info = all_info.find(acting_cand)->second;
+ if (cur_info.is_incomplete() ||
+ cur_info.last_update < primary->second.log_tail) {
+ ss << " shard " << acting_cand << " (stray) REJECTED "
+ << cur_info << std::endl;
} else {
- want.push_back(*i);
- dout(10) << " osd." << *i << " (stray) accepted " << cur_info << dendl;
+ want->push_back(*i);
+ acting_backfill->insert(acting_cand);
+ ss << " shard " << acting_cand << " (stray) accepted "
+ << cur_info << std::endl;
usable++;
}
}
- for (map<int,pg_info_t>::const_iterator i = all_info.begin();
+ for (map<pg_shard_t,pg_info_t>::const_iterator i = all_info.begin();
i != all_info.end();
++i) {
- if (usable >= get_osdmap()->get_pg_size(info.pgid))
+ if (usable >= size)
break;
// skip up osds we already considered above
if (i->first == primary->first)
continue;
- vector<int>::const_iterator up_it = find(up.begin(), up.end(), i->first);
+ vector<int>::const_iterator up_it = find(up.begin(), up.end(), i->first.osd);
if (up_it != up.end())
continue;
- vector<int>::const_iterator acting_it = find(acting.begin(), acting.end(), i->first);
+ vector<int>::const_iterator acting_it = find(
+ acting.begin(), acting.end(), i->first.osd);
if (acting_it != acting.end())
continue;
- if (i->second.is_incomplete() || i->second.last_update < primary->second.log_tail) {
- dout(10) << " osd." << i->first << " (stray) REJECTED " << i->second << dendl;
+ if (i->second.is_incomplete() ||
+ i->second.last_update < primary->second.log_tail) {
+ ss << " shard " << i->first << " (stray) REJECTED "
+ << i->second << std::endl;
} else {
- want.push_back(i->first);
- dout(10) << " osd." << i->first << " (stray) accepted " << i->second << dendl;
+ want->push_back(i->first.osd);
+ acting_backfill->insert(i->first);
+ ss << " shard " << i->first << " (stray) accepted "
+ << i->second << std::endl;
usable++;
}
}
-
- return true;
}
/**
@@ -1033,31 +1145,66 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want, vector<int>&
* calculate the desired acting, and request a change with the monitor
* if it differs from the current acting.
*/
-bool PG::choose_acting(int& newest_update_osd)
+bool PG::choose_acting(pg_shard_t &auth_log_shard_id)
{
- vector<int> want, backfill;
+ map<pg_shard_t, pg_info_t> all_info(peer_info.begin(), peer_info.end());
+ all_info[pg_whoami] = info;
- if (!calc_acting(newest_update_osd, want, backfill)) {
- dout(10) << "choose_acting failed" << dendl;
- assert(want_acting.empty());
- return false;
+ for (map<pg_shard_t, pg_info_t>::iterator p = all_info.begin();
+ p != all_info.end();
+ ++p) {
+ dout(10) << "calc_acting osd." << p->first << " " << p->second << dendl;
+ }
+
+ map<pg_shard_t, pg_info_t>::const_iterator auth_log_shard =
+ find_best_info(all_info);
+
+ if (auth_log_shard == all_info.end()) {
+ if (up != acting) {
+ dout(10) << "choose_acting no suitable info found (incomplete backfills?),"
+ << " reverting to up" << dendl;
+ want_acting = up;
+ return true;
+ } else {
+ dout(10) << "choose_acting failed" << dendl;
+ assert(want_acting.empty());
+ return false;
+ }
}
+ if ((up.size() &&
+ !all_info.find(up_primary)->second.is_incomplete() &&
+ all_info.find(up_primary)->second.last_update >=
+ auth_log_shard->second.log_tail) &&
+ auth_log_shard->second.is_incomplete()) {
+ map<pg_shard_t, pg_info_t> complete_infos;
+ for (map<pg_shard_t, pg_info_t>::const_iterator i = all_info.begin();
+ i != all_info.end();
+ ++i) {
+ if (!i->second.is_incomplete())
+ complete_infos.insert(*i);
+ }
+ map<pg_shard_t, pg_info_t>::const_iterator i = find_best_info(
+ complete_infos);
+ if (i != complete_infos.end()) {
+ auth_log_shard = all_info.find(i->first);
+ }
+ }
+
+ auth_log_shard_id = auth_log_shard->first;
+
// Determine if compatibility needed
bool compat_mode = !cct->_conf->osd_debug_override_acting_compat;
-
if (compat_mode) {
bool all_support = true;
OSDMapRef osdmap = get_osdmap();
- vector<int> allpeers;
- allpeers = want;
- allpeers.insert(allpeers.end(), backfill.begin(), backfill.end());
- for (vector<int>::iterator it = allpeers.begin();
- it != allpeers.end(); ++it) {
- int peer = *it;
+ for (map<pg_shard_t, pg_info_t>::iterator it = all_info.begin();
+ it != all_info.end();
+ ++it) {
+ pg_shard_t peer = it->first;
- const osd_xinfo_t& xi = osdmap->get_xinfo(peer);
+ const osd_xinfo_t& xi = osdmap->get_xinfo(peer.osd);
if (!(xi.features & CEPH_FEATURE_OSD_ERASURE_CODES)) {
all_support = false;
break;
@@ -1067,21 +1214,65 @@ bool PG::choose_acting(int& newest_update_osd)
compat_mode = false;
}
- if (compat_mode && !backfill.empty()) {
- backfill.resize(1);
- }
+ set<pg_shard_t> want_backfill, want_acting_backfill;
+ vector<int> want;
+ pg_shard_t want_primary;
+ stringstream ss;
+ if (!pool.info.ec_pool())
+ calc_replicated_acting(
+ auth_log_shard,
+ get_osdmap()->get_pg_size(info.pgid.pgid),
+ acting,
+ primary,
+ up,
+ up_primary,
+ all_info,
+ compat_mode,
+ &want,
+ &want_backfill,
+ &want_acting_backfill,
+ &want_primary,
+ ss);
+ else
+ calc_ec_acting(
+ auth_log_shard,
+ get_osdmap()->get_pg_size(info.pgid.pgid),
+ acting,
+ primary,
+ up,
+ up_primary,
+ all_info,
+ compat_mode,
+ &want,
+ &want_backfill,
+ &want_acting_backfill,
+ &want_primary,
+ ss);
+ dout(10) << ss.str() << dendl;
// This might cause a problem if min_size is large
// and we need to backfill more than 1 osd. Older
// code would only include 1 backfill osd and now we
// have the resize above.
- if (want.size() + backfill.size() < pool.info.min_size) {
+ if (want_acting_backfill.size() < pool.info.min_size) {
want_acting.clear();
return false;
}
- if (compat_mode) {
- want.insert(want.end(), backfill.begin(), backfill.end());
+ /* Check whether we have enough acting shards to later perform recovery */
+ boost::scoped_ptr<PGBackend::IsRecoverablePredicate> recoverable_predicate(
+ get_pgbackend()->get_is_recoverable_predicate());
+ set<pg_shard_t> have;
+ for (int i = 0; i < (int)want.size(); ++i) {
+ if (want[i] != CRUSH_ITEM_NONE)
+ have.insert(
+ pg_shard_t(
+ want[i],
+ pool.info.ec_pool() ? i : ghobject_t::NO_SHARD));
+ }
+ if (!(*recoverable_predicate)(have)) {
+ want_acting.clear();
+ return false;
}
if (want != acting) {
@@ -1089,39 +1280,40 @@ bool PG::choose_acting(int& newest_update_osd)
<< ", requesting pg_temp change" << dendl;
want_acting = want;
- if (want == up) {
+ if (want_acting == up) {
// There can't be any pending backfill if
// want is the same as crush map up OSDs.
- assert(compat_mode || backfill.empty());
+ assert(compat_mode || want_backfill.empty());
vector<int> empty;
- osd->queue_want_pg_temp(info.pgid, empty);
+ osd->queue_want_pg_temp(info.pgid.pgid, empty);
} else
- osd->queue_want_pg_temp(info.pgid, want);
+ osd->queue_want_pg_temp(info.pgid.pgid, want);
return false;
}
want_acting.clear();
- // We can only get here when new interval has arrived and
- // we've accepted the acting set. Now we can create
- // actingbackfill and backfill_targets vectors.
- actingbackfill = acting;
- if (!compat_mode)
- actingbackfill.insert(actingbackfill.end(), backfill.begin(), backfill.end());
- assert(backfill_targets.empty() || backfill_targets == backfill);
+ actingbackfill = want_acting_backfill;
+ dout(10) << "actingbackfill is " << actingbackfill << dendl;
+ assert(backfill_targets.empty() || backfill_targets == want_backfill);
if (backfill_targets.empty()) {
- backfill_targets = backfill;
- for (unsigned i = 0; i < backfill.size() ; ++i) {
- stray_set.erase(backfill[i]);
+ // Caller is GetInfo
+ backfill_targets = want_backfill;
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ assert(!stray_set.count(*i));
}
} else {
// Will not change if already set because up would have had to change
- assert(backfill_targets == backfill);
+ assert(backfill_targets == want_backfill);
// Verify that nothing in backfill is in stray_set
- for (unsigned i = 0; i < backfill.size() ; ++i) {
- assert(stray_set.find(backfill[i]) == stray_set.end());
+ for (set<pg_shard_t>::iterator i = want_backfill.begin();
+ i != want_backfill.end();
+ ++i) {
+ assert(stray_set.find(*i) == stray_set.end());
}
}
dout(10) << "choose_acting want " << want << " (== acting) backfill_targets "
- << backfill << dendl;
+ << want_backfill << dendl;
return true;
}
@@ -1157,16 +1349,18 @@ void PG::build_might_have_unfound()
if (!interval.maybe_went_rw)
continue;
+ int i = 0;
std::vector<int>::const_iterator a = interval.acting.begin();
std::vector<int>::const_iterator a_end = interval.acting.end();
- for (; a != a_end; ++a) {
- if (*a != osd->whoami)
- might_have_unfound.insert(*a);
+ for (; a != a_end; ++a, ++i) {
+ pg_shard_t shard(*a, pool.info.ec_pool() ? i : ghobject_t::NO_SHARD);
+ if (*a != CRUSH_ITEM_NONE && shard != pg_whoami)
+ might_have_unfound.insert(shard);
}
}
// include any (stray) peers
- for (map<int,pg_info_t>::iterator p = peer_info.begin();
+ for (map<pg_shard_t, pg_info_t>::iterator p = peer_info.begin();
p != peer_info.end();
++p)
might_have_unfound.insert(p->first);
@@ -1187,8 +1381,12 @@ struct C_PG_ActivateCommitted : public Context {
void PG::activate(ObjectStore::Transaction& t,
epoch_t query_epoch,
list<Context*>& tfin,
- map< int, map<pg_t,pg_query_t> >& query_map,
- map<int, vector<pair<pg_notify_t, pg_interval_map_t> > > *activator_map)
+ map<int, map<spg_t,pg_query_t> >& query_map,
+ map<int,
+ vector<
+ pair<pg_notify_t,
+ pg_interval_map_t> > > *activator_map,
+ RecoveryCtx *ctx)
{
assert(!is_active());
assert(scrubber.callbacks.empty());
@@ -1206,31 +1404,22 @@ void PG::activate(ObjectStore::Transaction& t,
// TODOSAM: osd->osd-> is no good
osd->osd->replay_queue_lock.Lock();
- osd->osd->replay_queue.push_back(pair<pg_t,utime_t>(info.pgid, replay_until));
+ osd->osd->replay_queue.push_back(pair<spg_t,utime_t>(
+ info.pgid, replay_until));
osd->osd->replay_queue_lock.Unlock();
}
// twiddle pg state
- state_set(PG_STATE_ACTIVE);
state_clear(PG_STATE_DOWN);
send_notify = false;
- info.last_epoch_started = query_epoch;
+ if (is_acting(pg_whoami))
+ info.last_epoch_started = query_epoch;
const pg_missing_t &missing = pg_log.get_missing();
if (is_primary()) {
- // If necessary, create might_have_unfound to help us find our unfound objects.
- // NOTE: It's important that we build might_have_unfound before trimming the
- // past intervals.
- might_have_unfound.clear();
- if (missing.have_missing()) {
- build_might_have_unfound();
- }
- }
-
- if (is_primary()) {
last_update_ondisk = info.last_update;
min_last_complete_ondisk = eversion_t(0,0); // we don't know (yet)!
}
@@ -1243,12 +1432,8 @@ void PG::activate(ObjectStore::Transaction& t,
dirty_info = true;
dirty_big_info = true; // maybe
- // verify that there are no stray objects
- if (is_primary())
- check_local();
-
// find out when we commit
- tfin.push_back(new C_PG_ActivateCommitted(this, query_epoch));
+ t.register_on_complete(new C_PG_ActivateCommitted(this, query_epoch));
// initialize snap_trimq
if (is_primary()) {
@@ -1280,11 +1465,15 @@ void PG::activate(ObjectStore::Transaction& t,
// if primary..
if (is_primary()) {
+ assert(ctx);
// start up replicas
- assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++) {
- int peer = actingbackfill[i];
+ assert(!actingbackfill.empty());
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ pg_shard_t peer = *i;
assert(peer_info.count(peer));
pg_info_t& pi = peer_info[peer];
@@ -1299,18 +1488,31 @@ void PG::activate(ObjectStore::Transaction& t,
// empty log
if (!pi.is_empty() && activator_map) {
dout(10) << "activate peer osd." << peer << " is up to date, queueing in pending_activators" << dendl;
- (*activator_map)[peer].push_back(
+ (*activator_map)[peer.osd].push_back(
make_pair(
pg_notify_t(
+ peer.shard, pg_whoami.shard,
get_osdmap()->get_epoch(),
get_osdmap()->get_epoch(),
info),
past_intervals));
} else {
dout(10) << "activate peer osd." << peer << " is up to date, but sending pg_log anyway" << dendl;
- m = new MOSDPGLog(get_osdmap()->get_epoch(), info);
+ m = new MOSDPGLog(
+ i->shard, pg_whoami.shard,
+ get_osdmap()->get_epoch(), info);
}
- } else if (pg_log.get_tail() > pi.last_update || pi.last_backfill == hobject_t()) {
+ } else if (
+ pg_log.get_tail() > pi.last_update ||
+ pi.last_backfill == hobject_t() ||
+ (backfill_targets.count(*i) && pi.last_backfill.is_max())) {
+ /* This last case covers a situation where a replica is not contiguous
+ * with the auth_log, but is contiguous with this replica. Reshuffling
+ * the active set to handle this would be tricky, so instead we just go
+ * ahead and backfill it anyway. This is probably preferrable in any
+ * case since the replica in question would have to be significantly
+ * behind.
+ */
// backfill
osd->clog.info() << info.pgid << " restarting backfill on osd." << peer
<< " from (" << pi.log_tail << "," << pi.last_update << "] " << pi.last_backfill
@@ -1322,7 +1524,9 @@ void PG::activate(ObjectStore::Transaction& t,
pi.history = info.history;
pi.stats.stats.clear();
- m = new MOSDPGLog(get_osdmap()->get_epoch(), pi);
+ m = new MOSDPGLog(
+ i->shard, pg_whoami.shard,
+ get_osdmap()->get_epoch(), pi);
// send some recent log, so that op dup detection works well.
m->log.copy_up_to(pg_log.get_log(), cct->_conf->osd_min_pg_log_entries);
@@ -1333,7 +1537,9 @@ void PG::activate(ObjectStore::Transaction& t,
} else {
// catch up
assert(pg_log.get_tail() <= pi.last_update);
- m = new MOSDPGLog(get_osdmap()->get_epoch(), info);
+ m = new MOSDPGLog(
+ i->shard, pg_whoami.shard,
+ get_osdmap()->get_epoch(), info);
// send new stuff to append to replicas log
m->log.copy_after(pg_log.get_log(), pi.last_update);
}
@@ -1356,7 +1562,7 @@ void PG::activate(ObjectStore::Transaction& t,
if (m) {
dout(10) << "activate peer osd." << peer << " sending " << m->log << dendl;
//m->log.print(cout);
- osd->send_message_osd_cluster(peer, m, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(peer.osd, m, get_osdmap()->get_epoch());
}
// peer now has
@@ -1371,46 +1577,56 @@ void PG::activate(ObjectStore::Transaction& t,
}
}
- // degraded?
- if (get_osdmap()->get_pg_size(info.pgid) > acting.size())
- state_set(PG_STATE_DEGRADED);
-
- // all clean?
+ // Set up missing_loc
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) {
+ missing_loc.add_active_missing(pg_log.get_missing());
+ } else {
+ assert(peer_missing.count(*i));
+ missing_loc.add_active_missing(peer_missing[*i]);
+ }
+ }
+ // If necessary, create might_have_unfound to help us find our unfound objects.
+ // NOTE: It's important that we build might_have_unfound before trimming the
+ // past intervals.
+ might_have_unfound.clear();
if (needs_recovery()) {
- dout(10) << "activate not all replicas are up-to-date, queueing recovery" << dendl;
- queue_peering_event(
- CephPeeringEvtRef(
- new CephPeeringEvt(
- get_osdmap()->get_epoch(),
- get_osdmap()->get_epoch(),
- DoRecovery())));
- } else if (needs_backfill()) {
- dout(10) << "activate queueing backfill" << dendl;
- queue_peering_event(
- CephPeeringEvtRef(
- new CephPeeringEvt(
- get_osdmap()->get_epoch(),
- get_osdmap()->get_epoch(),
- RequestBackfill())));
- } else {
- dout(10) << "activate all replicas clean, no recovery" << dendl;
- queue_peering_event(
- CephPeeringEvtRef(
- new CephPeeringEvt(
- get_osdmap()->get_epoch(),
- get_osdmap()->get_epoch(),
- AllReplicasRecovered())));
+ missing_loc.add_source_info(pg_whoami, info, pg_log.get_missing());
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ dout(10) << __func__ << ": adding " << *i << " as a source" << dendl;
+ assert(peer_missing.count(*i));
+ assert(peer_info.count(*i));
+ missing_loc.add_source_info(
+ *i,
+ peer_info[*i],
+ peer_missing[*i]);
+ }
+ for (map<pg_shard_t, pg_missing_t>::iterator i = peer_missing.begin();
+ i != peer_missing.end();
+ ++i) {
+ if (is_actingbackfill(i->first))
+ continue;
+ assert(peer_info.count(i->first));
+ search_for_missing(
+ peer_info[i->first],
+ i->second,
+ i->first,
+ ctx);
+ }
+
+ build_might_have_unfound();
}
- publish_stats_to_osd();
- }
+ // degraded?
+ if (get_osdmap()->get_pg_size(info.pgid.pgid) > acting.size())
+ state_set(PG_STATE_DEGRADED);
- // waiters
- if (!is_replay() && flushes_in_progress == 0) {
- requeue_ops(waiting_for_active);
}
-
- on_activate();
}
bool PG::op_has_sufficient_caps(OpRequestRef op)
@@ -1484,7 +1700,7 @@ void PG::queue_op(OpRequestRef op)
void PG::replay_queued_ops()
{
- assert(is_replay() && is_active());
+ assert(is_replay());
eversion_t c = info.last_update;
list<OpRequestRef> replay;
dout(10) << "replay_queued_ops" << dendl;
@@ -1517,22 +1733,30 @@ void PG::_activate_committed(epoch_t e)
if (pg_has_reset_since(e)) {
dout(10) << "_activate_committed " << e << ", that was an old interval" << dendl;
} else if (is_primary()) {
- peer_activated.insert(osd->whoami);
+ peer_activated.insert(pg_whoami);
dout(10) << "_activate_committed " << e << " peer_activated now " << peer_activated
<< " last_epoch_started " << info.history.last_epoch_started
<< " same_interval_since " << info.history.same_interval_since << dendl;
- assert(actingbackfill.size() > 0);
+ assert(!actingbackfill.empty());
if (peer_activated.size() == actingbackfill.size())
all_activated_and_committed();
} else {
dout(10) << "_activate_committed " << e << " telling primary" << dendl;
MOSDPGInfo *m = new MOSDPGInfo(e);
- pg_notify_t i = pg_notify_t(get_osdmap()->get_epoch(),
- get_osdmap()->get_epoch(),
- info);
+ pg_notify_t i = pg_notify_t(
+ get_primary().shard, pg_whoami.shard,
+ get_osdmap()->get_epoch(),
+ get_osdmap()->get_epoch(),
+ info);
i.info.history.last_epoch_started = e;
m->pg_list.push_back(make_pair(i, pg_interval_map_t()));
- osd->send_message_osd_cluster(acting[0], m, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(get_primary().osd, m, get_osdmap()->get_epoch());
+
+ state_set(PG_STATE_ACTIVE);
+ // waiters
+ if (flushes_in_progress == 0) {
+ requeue_ops(waiting_for_active);
+ }
}
if (dirty_info) {
@@ -1555,7 +1779,7 @@ void PG::all_activated_and_committed()
dout(10) << "all_activated_and_committed" << dendl;
assert(is_primary());
assert(peer_activated.size() == actingbackfill.size());
- assert(actingbackfill.size() > 0);
+ assert(!actingbackfill.empty());
// info.last_epoch_started is set during activate()
info.history.last_epoch_started = info.last_epoch_started;
@@ -1611,7 +1835,7 @@ void PG::mark_clean()
{
// only mark CLEAN if we have the desired number of replicas AND we
// are not remapped.
- if (acting.size() == get_osdmap()->get_pg_size(info.pgid) &&
+ if (acting.size() == get_osdmap()->get_pg_size(info.pgid.pgid) &&
up == acting)
state_set(PG_STATE_CLEAN);
@@ -1738,9 +1962,10 @@ static void split_replay_queue(
}
void PG::split_ops(PG *child, unsigned split_bits) {
- unsigned match = child->info.pgid.m_seed;
+ unsigned match = child->info.pgid.ps();
assert(waiting_for_all_missing.empty());
- assert(waiting_for_missing_object.empty());
+ assert(waiting_for_cache_not_full.empty());
+ assert(waiting_for_unreadable_object.empty());
assert(waiting_for_degraded_object.empty());
assert(waiting_for_ack.empty());
assert(waiting_for_ondisk.empty());
@@ -1791,8 +2016,18 @@ void PG::split_into(pg_t child_pgid, PG *child, unsigned split_bits)
child->snap_trimq = snap_trimq;
// There can't be recovery/backfill going on now
- get_osdmap()->pg_to_up_acting_osds(child->info.pgid, child->up, child->acting);
+ int primary, up_primary;
+ vector<int> newup, newacting;
+ get_osdmap()->pg_to_up_acting_osds(
+ child->info.pgid.pgid, &newup, &up_primary, &newacting, &primary);
+ child->init_primary_up_acting(
+ newup,
+ newacting,
+ up_primary,
+ primary);
child->role = OSDMap::calc_pg_role(osd->whoami, child->acting);
+
+ // this comparison includes primary rank via pg_shard_t
if (get_primary() != child->get_primary())
child->info.history.same_primary_since = get_osdmap()->get_epoch();
@@ -1842,17 +2077,18 @@ void PG::purge_strays()
dout(10) << "purge_strays " << stray_set << dendl;
bool removed = false;
- for (set<int>::iterator p = stray_set.begin();
+ for (set<pg_shard_t>::iterator p = stray_set.begin();
p != stray_set.end();
++p) {
- if (get_osdmap()->is_up(*p)) {
+ assert(!is_actingbackfill(*p));
+ if (get_osdmap()->is_up(p->osd)) {
dout(10) << "sending PGRemove to osd." << *p << dendl;
- vector<pg_t> to_remove;
- to_remove.push_back(info.pgid);
+ vector<spg_t> to_remove;
+ to_remove.push_back(spg_t(info.pgid.pgid, p->shard));
MOSDPGRemove *m = new MOSDPGRemove(
get_osdmap()->get_epoch(),
to_remove);
- osd->send_message_osd_cluster(*p, m, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(p->osd, m, get_osdmap()->get_epoch());
stray_purged.insert(*p);
} else {
dout(10) << "not sending PGRemove to down osd." << *p << dendl;
@@ -1874,10 +2110,15 @@ void PG::purge_strays()
peer_missing_requested.clear();
}
-void PG::set_probe_targets(const set<int> &probe_set)
+void PG::set_probe_targets(const set<pg_shard_t> &probe_set)
{
Mutex::Locker l(heartbeat_peer_lock);
- probe_targets = probe_set;
+ probe_targets.clear();
+ for (set<pg_shard_t>::iterator i = probe_set.begin();
+ i != probe_set.end();
+ ++i) {
+ probe_targets.insert(i->osd);
+ }
}
void PG::clear_probe_targets()
@@ -1892,12 +2133,18 @@ void PG::update_heartbeat_peers()
set<int> new_peers;
if (is_primary()) {
- for (unsigned i=0; i<acting.size(); i++)
- new_peers.insert(acting[i]);
- for (unsigned i=0; i<up.size(); i++)
- new_peers.insert(up[i]);
- for (map<int,pg_info_t>::iterator p = peer_info.begin(); p != peer_info.end(); ++p)
- new_peers.insert(p->first);
+ for (unsigned i=0; i<acting.size(); i++) {
+ if (acting[i] != CRUSH_ITEM_NONE)
+ new_peers.insert(acting[i]);
+ }
+ for (unsigned i=0; i<up.size(); i++) {
+ if (up[i] != CRUSH_ITEM_NONE)
+ new_peers.insert(up[i]);
+ }
+ for (map<pg_shard_t,pg_info_t>::iterator p = peer_info.begin();
+ p != peer_info.end();
+ ++p)
+ new_peers.insert(p->first.osd);
}
bool need_update = false;
@@ -1933,7 +2180,8 @@ void PG::_update_calc_stats()
info.stats.ondisk_log_start = pg_log.get_tail();
// calc copies, degraded
- unsigned target = MAX(get_osdmap()->get_pg_size(info.pgid), actingbackfill.size());
+ unsigned target = MAX(
+ get_osdmap()->get_pg_size(info.pgid.pgid), actingbackfill.size());
info.stats.stats.calc_copies(target);
info.stats.stats.sum.num_objects_degraded = 0;
if ((is_degraded() || !is_clean()) && is_active()) {
@@ -1952,15 +2200,18 @@ void PG::_update_calc_stats()
pg_log.get_missing().num_missing();
degraded += pg_log.get_missing().num_missing();
- assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++) {
- assert(peer_missing.count(actingbackfill[i]));
+ assert(!actingbackfill.empty());
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ assert(peer_missing.count(*i));
// in missing set
- degraded += peer_missing[actingbackfill[i]].num_missing();
+ degraded += peer_missing[*i].num_missing();
// not yet backfilled
- degraded += num_objects - peer_info[actingbackfill[i]].stats.stats.sum.num_objects;
+ degraded += num_objects - peer_info[*i].stats.stats.sum.num_objects;
}
info.stats.stats.sum.num_objects_degraded = degraded;
info.stats.stats.sum.num_objects_unfound = get_num_unfound();
@@ -2037,11 +2288,14 @@ void PG::clear_publish_stats()
* @param backfill true if info should be marked as backfill
* @param t transaction to write out our new state in
*/
-void PG::init(int role, vector<int>& newup, vector<int>& newacting,
- pg_history_t& history,
- pg_interval_map_t& pi,
- bool backfill,
- ObjectStore::Transaction *t)
+void PG::init(
+ int role,
+ vector<int>& newup, int new_up_primary,
+ vector<int>& newacting, int new_acting_primary,
+ pg_history_t& history,
+ pg_interval_map_t& pi,
+ bool backfill,
+ ObjectStore::Transaction *t)
{
dout(10) << "init role " << role << " up " << newup << " acting " << newacting
<< " history " << history
@@ -2051,12 +2305,19 @@ void PG::init(int role, vector<int>& newup, vector<int>& newacting,
set_role(role);
acting = newacting;
up = newup;
+ init_primary_up_acting(
+ newup,
+ newacting,
+ new_up_primary,
+ new_acting_primary);
info.history = history;
past_intervals.swap(pi);
info.stats.up = up;
+ info.stats.up_primary = new_up_primary;
info.stats.acting = acting;
+ info.stats.acting_primary = new_acting_primary;
info.stats.mapping_epoch = info.history.same_interval_since;
if (backfill) {
@@ -2268,7 +2529,7 @@ void PG::write_info(ObjectStore::Transaction& t)
epoch_t PG::peek_map_epoch(ObjectStore *store, coll_t coll, hobject_t &infos_oid, bufferlist *bl)
{
assert(bl);
- pg_t pgid;
+ spg_t pgid;
snapid_t snap;
bool ok = coll.is_pg(pgid, snap);
assert(ok);
@@ -2311,12 +2572,19 @@ void PG::trim_peers()
calc_trim_to();
dout(10) << "trim_peers " << pg_trim_to << dendl;
if (pg_trim_to != eversion_t()) {
- assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++)
- osd->send_message_osd_cluster(actingbackfill[i],
- new MOSDPGTrim(get_osdmap()->get_epoch(), info.pgid,
- pg_trim_to),
- get_osdmap()->get_epoch());
+ assert(!actingbackfill.empty());
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ osd->send_message_osd_cluster(
+ i->osd,
+ new MOSDPGTrim(
+ get_osdmap()->get_epoch(),
+ spg_t(info.pgid.pgid, i->shard),
+ pg_trim_to),
+ get_osdmap()->get_epoch());
+ }
}
}
@@ -2335,6 +2603,12 @@ void PG::add_log_entry(pg_log_entry_t& e, bufferlist& log_bl)
if (e.user_version > info.last_user_version)
info.last_user_version = e.user_version;
+ /**
+ * Make sure we don't keep around more than we need to in the
+ * in-memory log
+ */
+ e.mod_desc.trim_bl();
+
// log mutation
pg_log.add(e);
dout(10) << "add_log_entry " << e << dendl;
@@ -2391,8 +2665,9 @@ std::string PG::get_corrupt_pg_log_name() const
dout(0) << "strftime failed" << dendl;
return "corrupt_log_unknown_time";
}
- info.pgid.print(buf + ret, MAX_BUF - ret);
- return buf;
+ string out(buf);
+ out += stringify(info.pgid);
+ return out;
}
int PG::read_info(
@@ -2675,7 +2950,7 @@ bool PG::sched_scrub()
if (osd->inc_scrubs_pending()) {
dout(20) << "sched_scrub: reserved locally, reserving replicas" << dendl;
scrubber.reserved = true;
- scrubber.reserved_peers.insert(osd->whoami);
+ scrubber.reserved_peers.insert(pg_whoami);
scrub_reserve_replicas();
} else {
dout(20) << "sched_scrub: failed to reserve locally" << dendl;
@@ -2733,28 +3008,28 @@ void PG::sub_op_scrub_map(OpRequestRef op)
op->mark_started();
- int from = m->get_source().num();
-
- dout(10) << " got osd." << from << " scrub map" << dendl;
+ dout(10) << " got " << m->from << " scrub map" << dendl;
bufferlist::iterator p = m->get_data().begin();
if (scrubber.is_chunky) { // chunky scrub
- scrubber.received_maps[from].decode(p, info.pgid.pool());
- dout(10) << "map version is " << scrubber.received_maps[from].valid_through << dendl;
+ scrubber.received_maps[m->from].decode(p, info.pgid.pool());
+ dout(10) << "map version is "
+ << scrubber.received_maps[m->from].valid_through
+ << dendl;
} else { // classic scrub
- if (scrubber.received_maps.count(from)) {
+ if (scrubber.received_maps.count(m->from)) {
ScrubMap incoming;
incoming.decode(p, info.pgid.pool());
- dout(10) << "from replica " << from << dendl;
+ dout(10) << "from replica " << m->from << dendl;
dout(10) << "map version is " << incoming.valid_through << dendl;
- scrubber.received_maps[from].merge_incr(incoming);
+ scrubber.received_maps[m->from].merge_incr(incoming);
} else {
- scrubber.received_maps[from].decode(p, info.pgid.pool());
+ scrubber.received_maps[m->from].decode(p, info.pgid.pool());
}
}
--scrubber.waiting_on;
- scrubber.waiting_on_whom.erase(from);
+ scrubber.waiting_on_whom.erase(m->from);
if (scrubber.waiting_on == 0) {
if (scrubber.is_chunky) { // chunky scrub
@@ -2768,7 +3043,7 @@ void PG::sub_op_scrub_map(OpRequestRef op)
scrubber.finalizing = true;
scrub_gather_replica_maps();
++scrubber.waiting_on;
- scrubber.waiting_on_whom.insert(osd->whoami);
+ scrubber.waiting_on_whom.insert(pg_whoami);
osd->scrub_wq.queue(this);
}
}
@@ -2777,27 +3052,33 @@ void PG::sub_op_scrub_map(OpRequestRef op)
}
// send scrub v2-compatible messages (classic scrub)
-void PG::_request_scrub_map_classic(int replica, eversion_t version)
+void PG::_request_scrub_map_classic(pg_shard_t replica, eversion_t version)
{
- assert(replica != osd->whoami);
+ assert(replica != pg_whoami);
dout(10) << "scrub requesting scrubmap from osd." << replica << dendl;
- MOSDRepScrub *repscrubop = new MOSDRepScrub(info.pgid, version,
- last_update_applied,
- get_osdmap()->get_epoch());
- osd->send_message_osd_cluster(replica, repscrubop, get_osdmap()->get_epoch());
+ MOSDRepScrub *repscrubop =
+ new MOSDRepScrub(
+ spg_t(info.pgid.pgid, replica.shard), version,
+ last_update_applied,
+ get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(
+ replica.osd, repscrubop, get_osdmap()->get_epoch());
}
// send scrub v3 messages (chunky scrub)
-void PG::_request_scrub_map(int replica, eversion_t version,
- hobject_t start, hobject_t end,
- bool deep)
+void PG::_request_scrub_map(
+ pg_shard_t replica, eversion_t version,
+ hobject_t start, hobject_t end,
+ bool deep)
{
- assert(replica != osd->whoami);
+ assert(replica != pg_whoami);
dout(10) << "scrub requesting scrubmap from osd." << replica << dendl;
- MOSDRepScrub *repscrubop = new MOSDRepScrub(info.pgid, version,
- get_osdmap()->get_epoch(),
- start, end, deep);
- osd->send_message_osd_cluster(replica, repscrubop, get_osdmap()->get_epoch());
+ MOSDRepScrub *repscrubop = new MOSDRepScrub(
+ spg_t(info.pgid.pgid, replica.shard), version,
+ get_osdmap()->get_epoch(),
+ start, end, deep);
+ osd->send_message_osd_cluster(
+ replica.osd, repscrubop, get_osdmap()->get_epoch());
}
void PG::sub_op_scrub_reserve(OpRequestRef op)
@@ -2815,7 +3096,8 @@ void PG::sub_op_scrub_reserve(OpRequestRef op)
scrubber.reserved = osd->inc_scrubs_pending();
- MOSDSubOpReply *reply = new MOSDSubOpReply(m, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+ MOSDSubOpReply *reply = new MOSDSubOpReply(
+ m, pg_whoami, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
::encode(scrubber.reserved, reply->get_data());
osd->send_message_osd_cluster(reply, m->get_connection());
}
@@ -2833,7 +3115,7 @@ void PG::sub_op_scrub_reserve_reply(OpRequestRef op)
op->mark_started();
- int from = reply->get_source().num();
+ pg_shard_t from = reply->from;
bufferlist::iterator p = reply->get_data().begin();
bool reserved;
::decode(reserved, p);
@@ -2874,17 +3156,18 @@ void PG::sub_op_scrub_stop(OpRequestRef op)
// see comment in sub_op_scrub_reserve
scrubber.reserved = false;
- MOSDSubOpReply *reply = new MOSDSubOpReply(m, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+ MOSDSubOpReply *reply = new MOSDSubOpReply(
+ m, pg_whoami, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
osd->send_message_osd_cluster(reply, m->get_connection());
}
void PG::reject_reservation()
{
osd->send_message_osd_cluster(
- acting[0],
+ primary.osd,
new MBackfillReserve(
MBackfillReserve::REJECT,
- info.pgid,
+ spg_t(info.pgid.pgid, primary.shard),
get_osdmap()->get_epoch()),
get_osdmap()->get_epoch());
}
@@ -2914,34 +3197,43 @@ void PG::clear_scrub_reserved()
void PG::scrub_reserve_replicas()
{
assert(backfill_targets.empty());
- for (unsigned i=1; i<acting.size(); i++) {
- dout(10) << "scrub requesting reserve from osd." << acting[i] << dendl;
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ dout(10) << "scrub requesting reserve from osd." << *i << dendl;
vector<OSDOp> scrub(1);
scrub[0].op.op = CEPH_OSD_OP_SCRUB_RESERVE;
hobject_t poid;
eversion_t v;
osd_reqid_t reqid;
- MOSDSubOp *subop = new MOSDSubOp(reqid, info.pgid, poid, false, 0,
- get_osdmap()->get_epoch(), osd->get_tid(), v);
+ MOSDSubOp *subop = new MOSDSubOp(
+ reqid, pg_whoami, spg_t(info.pgid.pgid, i->shard), poid, false, 0,
+ get_osdmap()->get_epoch(), osd->get_tid(), v);
subop->ops = scrub;
- osd->send_message_osd_cluster(acting[i], subop, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(
+ i->osd, subop, get_osdmap()->get_epoch());
}
}
void PG::scrub_unreserve_replicas()
{
assert(backfill_targets.empty());
- for (unsigned i=1; i<acting.size(); i++) {
- dout(10) << "scrub requesting unreserve from osd." << acting[i] << dendl;
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ dout(10) << "scrub requesting unreserve from osd." << *i << dendl;
vector<OSDOp> scrub(1);
scrub[0].op.op = CEPH_OSD_OP_SCRUB_UNRESERVE;
hobject_t poid;
eversion_t v;
osd_reqid_t reqid;
- MOSDSubOp *subop = new MOSDSubOp(reqid, info.pgid, poid, false, 0,
- get_osdmap()->get_epoch(), osd->get_tid(), v);
+ MOSDSubOp *subop = new MOSDSubOp(
+ reqid, pg_whoami, spg_t(info.pgid.pgid, i->shard), poid, false, 0,
+ get_osdmap()->get_epoch(), osd->get_tid(), v);
subop->ops = scrub;
- osd->send_message_osd_cluster(acting[i], subop, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(i->osd, subop, get_osdmap()->get_epoch());
}
}
@@ -3123,22 +3415,24 @@ void PG::build_inc_scrub_map(
osd->store->collection_getattrs(coll, map.attrs);
}
-void PG::repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer)
+void PG::repair_object(
+ const hobject_t& soid, ScrubMap::object *po,
+ pg_shard_t bad_peer, pg_shard_t ok_peer)
{
- dout(10) << "repair_object " << soid << " bad_peer osd." << bad_peer << " ok_peer osd." << ok_peer << dendl;
+ dout(10) << "repair_object " << soid << " bad_peer osd."
+ << bad_peer << " ok_peer osd." << ok_peer << dendl;
eversion_t v;
bufferlist bv;
bv.push_back(po->attrs[OI_ATTR]);
object_info_t oi(bv);
- if (bad_peer != acting[0]) {
+ if (bad_peer != primary) {
peer_missing[bad_peer].add(soid, oi.version, eversion_t());
} else {
// We should only be scrubbing if the PG is clean.
- assert(waiting_for_missing_object.empty());
+ assert(waiting_for_unreadable_object.empty());
pg_log.missing_add(soid, oi.version, eversion_t());
- missing_loc[soid].insert(ok_peer);
- missing_loc_sources.insert(ok_peer);
+ missing_loc.add_location(soid, ok_peer);
pg_log.set_last_requested(0);
}
@@ -3233,8 +3527,16 @@ void PG::replica_scrub(
hobject_t poid;
eversion_t v;
osd_reqid_t reqid;
- MOSDSubOp *subop = new MOSDSubOp(reqid, info.pgid, poid, false, 0,
- msg->map_epoch, osd->get_tid(), v);
+ MOSDSubOp *subop = new MOSDSubOp(
+ reqid,
+ pg_whoami,
+ spg_t(info.pgid.pgid, get_primary().shard),
+ poid,
+ false,
+ 0,
+ msg->map_epoch,
+ osd->get_tid(),
+ v);
::encode(map, subop->get_data());
subop->ops = scrub;
@@ -3270,7 +3572,11 @@ void PG::scrub(ThreadPool::TPHandle &handle)
OSDMapRef curmap = osd->get_osdmap();
scrubber.is_chunky = true;
assert(backfill_targets.empty());
- for (unsigned i=1; i<acting.size(); i++) {
+ for (unsigned i=0; i<acting.size(); i++) {
+ if (acting[i] == pg_whoami.osd)
+ continue;
+ if (acting[i] == CRUSH_ITEM_NONE)
+ continue;
ConnectionRef con = osd->get_con_osd_cluster(acting[i], get_osdmap()->get_epoch());
if (!con)
continue;
@@ -3363,11 +3669,16 @@ void PG::classic_scrub(ThreadPool::TPHandle &handle)
* last_update_applied == info.last_update)
*/
scrubber.waiting_on = acting.size();
- scrubber.waiting_on_whom.insert(acting.begin(), acting.end());
+ scrubber.waiting_on_whom.insert(
+ actingbackfill.begin(), actingbackfill.end());
+ scrubber.waiting_on_whom.erase(pg_whoami);
// request maps from replicas
- for (unsigned i=1; i<acting.size(); i++) {
- _request_scrub_map_classic(acting[i], eversion_t());
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ _request_scrub_map_classic(*i, eversion_t());
}
// Unlocks and relocks...
@@ -3382,7 +3693,7 @@ void PG::classic_scrub(ThreadPool::TPHandle &handle)
}
--scrubber.waiting_on;
- scrubber.waiting_on_whom.erase(osd->whoami);
+ scrubber.waiting_on_whom.erase(pg_whoami);
if (scrubber.waiting_on == 0) {
// the replicas have completed their scrub map, so lock out writes
@@ -3402,7 +3713,7 @@ void PG::classic_scrub(ThreadPool::TPHandle &handle)
// request incrementals from replicas
scrub_gather_replica_maps();
++scrubber.waiting_on;
- scrubber.waiting_on_whom.insert(osd->whoami);
+ scrubber.waiting_on_whom.insert(pg_whoami);
}
dout(10) << "clean up scrub" << dendl;
@@ -3424,7 +3735,7 @@ void PG::classic_scrub(ThreadPool::TPHandle &handle)
}
--scrubber.waiting_on;
- scrubber.waiting_on_whom.erase(osd->whoami);
+ scrubber.waiting_on_whom.erase(pg_whoami);
if (scrubber.waiting_on == 0) {
assert(last_update_applied == info.last_update);
osd->scrub_finalize_wq.queue(this);
@@ -3599,14 +3910,17 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle)
}
// ask replicas to wait until last_update_applied >= scrubber.subset_last_update and then scan
- scrubber.waiting_on_whom.insert(osd->whoami);
+ scrubber.waiting_on_whom.insert(pg_whoami);
++scrubber.waiting_on;
// request maps from replicas
- for (unsigned i=1; i<acting.size(); i++) {
- _request_scrub_map(acting[i], scrubber.subset_last_update,
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ _request_scrub_map(*i, scrubber.subset_last_update,
scrubber.start, scrubber.end, scrubber.deep);
- scrubber.waiting_on_whom.insert(acting[i]);
+ scrubber.waiting_on_whom.insert(*i);
++scrubber.waiting_on;
}
@@ -3649,7 +3963,7 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle)
}
--scrubber.waiting_on;
- scrubber.waiting_on_whom.erase(osd->whoami);
+ scrubber.waiting_on_whom.erase(pg_whoami);
scrubber.state = PG::Scrubber::WAIT_REPLICAS;
break;
@@ -3731,7 +4045,7 @@ bool PG::scrub_gather_replica_maps()
assert(scrubber.waiting_on == 0);
assert(_lock.is_locked());
- for (map<int,ScrubMap>::iterator p = scrubber.received_maps.begin();
+ for (map<pg_shard_t, ScrubMap>::iterator p = scrubber.received_maps.begin();
p != scrubber.received_maps.end();
++p) {
@@ -3750,8 +4064,6 @@ bool PG::scrub_gather_replica_maps()
}
}
-
-
void PG::scrub_compare_maps()
{
dout(10) << "scrub_compare_maps has maps, analyzing" << dendl;
@@ -3765,16 +4077,21 @@ void PG::scrub_compare_maps()
stringstream ss;
// Map from object with errors to good peer
- map<hobject_t, int> authoritative;
- map<int,ScrubMap *> maps;
+ map<hobject_t, pg_shard_t> authoritative;
+ map<pg_shard_t, ScrubMap *> maps;
dout(2) << "scrub osd." << acting[0] << " has "
<< scrubber.primary_scrubmap.objects.size() << " items" << dendl;
- maps[0] = &scrubber.primary_scrubmap;
- for (unsigned i=1; i<acting.size(); i++) {
- dout(2) << "scrub osd." << acting[i] << " has "
- << scrubber.received_maps[acting[i]].objects.size() << " items" << dendl;
- maps[i] = &scrubber.received_maps[acting[i]];
+ maps[pg_whoami] = &scrubber.primary_scrubmap;
+
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ dout(2) << "scrub replica " << *i << " has "
+ << scrubber.received_maps[*i].objects.size()
+ << " items" << dendl;
+ maps[*i] = &scrubber.received_maps[*i];
}
get_pgbackend()->be_compare_scrubmaps(
@@ -3793,7 +4110,7 @@ void PG::scrub_compare_maps()
osd->clog.error(ss);
}
- for (map<hobject_t, int>::iterator i = authoritative.begin();
+ for (map<hobject_t, pg_shard_t>::iterator i = authoritative.begin();
i != authoritative.end();
++i) {
scrubber.authoritative.insert(
@@ -3802,7 +4119,7 @@ void PG::scrub_compare_maps()
make_pair(maps[i->second]->objects[i->first], i->second)));
}
- for (map<hobject_t, int>::iterator i = authoritative.begin();
+ for (map<hobject_t, pg_shard_t>::iterator i = authoritative.begin();
i != authoritative.end();
++i) {
authmap.objects.erase(i->first);
@@ -3823,11 +4140,11 @@ void PG::scrub_process_inconsistent()
if (!scrubber.authoritative.empty() || !scrubber.inconsistent.empty()) {
stringstream ss;
- for (map<hobject_t, set<int> >::iterator obj =
+ for (map<hobject_t, set<pg_shard_t> >::iterator obj =
scrubber.inconsistent_snapcolls.begin();
obj != scrubber.inconsistent_snapcolls.end();
++obj) {
- for (set<int>::iterator j = obj->second.begin();
+ for (set<pg_shard_t>::iterator j = obj->second.begin();
j != obj->second.end();
++j) {
++scrubber.shallow_errors;
@@ -3836,26 +4153,28 @@ void PG::scrub_process_inconsistent()
}
}
- ss << info.pgid << " " << mode << " " << scrubber.missing.size() << " missing, "
+ ss << info.pgid << " " << mode << " "
+ << scrubber.missing.size() << " missing, "
<< scrubber.inconsistent.size() << " inconsistent objects\n";
dout(2) << ss.str() << dendl;
osd->clog.error(ss);
if (repair) {
state_clear(PG_STATE_CLEAN);
- for (map<hobject_t, pair<ScrubMap::object, int> >::iterator i =
+ for (map<hobject_t, pair<ScrubMap::object, pg_shard_t> >::iterator i =
scrubber.authoritative.begin();
i != scrubber.authoritative.end();
++i) {
- set<int>::iterator j;
+ set<pg_shard_t>::iterator j;
if (scrubber.missing.count(i->first)) {
for (j = scrubber.missing[i->first].begin();
j != scrubber.missing[i->first].end();
++j) {
- repair_object(i->first,
+ repair_object(
+ i->first,
&(i->second.first),
- acting[*j],
- acting[i->second.second]);
+ *j,
+ i->second.second);
++scrubber.fixed;
}
}
@@ -3865,8 +4184,8 @@ void PG::scrub_process_inconsistent()
++j) {
repair_object(i->first,
&(i->second.first),
- acting[*j],
- acting[i->second.second]);
+ *j,
+ i->second.second);
++scrubber.fixed;
}
}
@@ -3921,7 +4240,7 @@ void PG::scrub_finish()
{
stringstream oss;
- oss << info.pgid << " " << mode << " ";
+ oss << info.pgid.pgid << " " << mode << " ";
int total_errors = scrubber.shallow_errors + scrubber.deep_errors;
if (total_errors)
oss << total_errors << " errors";
@@ -4007,22 +4326,26 @@ void PG::share_pg_info()
dout(10) << "share_pg_info" << dendl;
// share new pg_info_t with replicas
- assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++) {
- int peer = actingbackfill[i];
- if (peer_info.count(i)) {
- peer_info[i].last_epoch_started = info.last_epoch_started;
- peer_info[i].history.merge(info.history);
+ assert(!actingbackfill.empty());
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == pg_whoami) continue;
+ pg_shard_t peer = *i;
+ if (peer_info.count(peer)) {
+ peer_info[peer].last_epoch_started = info.last_epoch_started;
+ peer_info[peer].history.merge(info.history);
}
MOSDPGInfo *m = new MOSDPGInfo(get_osdmap()->get_epoch());
m->pg_list.push_back(
make_pair(
pg_notify_t(
+ peer.shard, pg_whoami.shard,
get_osdmap()->get_epoch(),
get_osdmap()->get_epoch(),
info),
pg_interval_map_t()));
- osd->send_message_osd_cluster(peer, m, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(peer.osd, m, get_osdmap()->get_epoch());
}
}
@@ -4036,15 +4359,19 @@ void PG::share_pg_log()
dout(10) << __func__ << dendl;
assert(is_primary());
- vector<int>::const_iterator a = actingbackfill.begin();
+ set<pg_shard_t>::const_iterator a = actingbackfill.begin();
assert(a != actingbackfill.end());
- vector<int>::const_iterator end = actingbackfill.end();
- while (++a != end) {
- int peer(*a);
+ set<pg_shard_t>::const_iterator end = actingbackfill.end();
+ while (a != end) {
+ pg_shard_t peer(*a);
+ ++a;
+ if (peer == pg_whoami) continue;
pg_missing_t& pmissing(peer_missing[peer]);
pg_info_t& pinfo(peer_info[peer]);
- MOSDPGLog *m = new MOSDPGLog(info.last_update.epoch, info);
+ MOSDPGLog *m = new MOSDPGLog(
+ peer.shard, pg_whoami.shard,
+ info.last_update.epoch, info);
m->log.copy_after(pg_log.get_log(), pinfo.last_update);
for (list<pg_log_entry_t>::const_iterator i = m->log.log.begin();
@@ -4054,7 +4381,7 @@ void PG::share_pg_log()
}
pinfo.last_update = m->log.head;
- osd->send_message_osd_cluster(peer, m, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(peer.osd, m, get_osdmap()->get_epoch());
}
}
@@ -4065,11 +4392,11 @@ void PG::update_history_from_master(pg_history_t new_history)
reg_next_scrub();
}
-void PG::fulfill_info(int from, const pg_query_t &query,
- pair<int, pg_info_t> ¬ify_info)
+void PG::fulfill_info(
+ pg_shard_t from, const pg_query_t &query,
+ pair<pg_shard_t, pg_info_t> ¬ify_info)
{
- assert(!acting.empty());
- assert(from == acting[0]);
+ assert(from == primary);
assert(query.type == pg_query_t::INFO);
// info
@@ -4077,14 +4404,17 @@ void PG::fulfill_info(int from, const pg_query_t &query,
notify_info = make_pair(from, info);
}
-void PG::fulfill_log(int from, const pg_query_t &query, epoch_t query_epoch)
+void PG::fulfill_log(
+ pg_shard_t from, const pg_query_t &query, epoch_t query_epoch)
{
- assert(!acting.empty());
- assert(from == acting[0]);
+ dout(10) << "log request from " << from << dendl;
+ assert(from == primary);
assert(query.type != pg_query_t::INFO);
- MOSDPGLog *mlog = new MOSDPGLog(get_osdmap()->get_epoch(),
- info, query_epoch);
+ MOSDPGLog *mlog = new MOSDPGLog(
+ from.shard, pg_whoami.shard,
+ get_osdmap()->get_epoch(),
+ info, query_epoch);
mlog->missing = pg_log.get_missing();
// primary -> other, when building master log
@@ -4106,9 +4436,10 @@ void PG::fulfill_log(int from, const pg_query_t &query, epoch_t query_epoch)
dout(10) << " sending " << mlog->log << " " << mlog->missing << dendl;
- ConnectionRef con = osd->get_con_osd_cluster(from, get_osdmap()->get_epoch());
+ ConnectionRef con = osd->get_con_osd_cluster(
+ from.osd, get_osdmap()->get_epoch());
if (con) {
- osd->osd->_share_map_outgoing(from, con.get(), get_osdmap());
+ osd->osd->_share_map_outgoing(from.osd, con.get(), get_osdmap());
osd->send_message_osd_cluster(mlog, con.get());
} else {
mlog->put();
@@ -4145,6 +4476,8 @@ bool PG::may_need_replay(const OSDMapRef osdmap) const
// consider ACTING osds
for (unsigned i=0; i<interval.acting.size(); i++) {
int o = interval.acting[i];
+ if (o == CRUSH_ITEM_NONE)
+ continue;
const osd_info_t *pinfo = 0;
if (osdmap->exists(o))
@@ -4201,10 +4534,17 @@ bool PG::is_split(OSDMapRef lastmap, OSDMapRef nextmap)
0);
}
-bool PG::acting_up_affected(const vector<int>& newup, const vector<int>& newacting)
+bool PG::acting_up_affected(
+ int newupprimary,
+ int newactingprimary,
+ const vector<int>& newup, const vector<int>& newacting)
{
- if (acting != newacting || up != newup) {
- dout(20) << "acting_up_affected newup " << newup << " newacting " << newacting << dendl;
+ if (newupprimary != up_primary.osd ||
+ newactingprimary != primary.osd ||
+ acting != newacting ||
+ up != newup) {
+ dout(20) << "acting_up_affected newup " << newup
+ << " newacting " << newacting << dendl;
return true;
} else {
return false;
@@ -4256,10 +4596,11 @@ void PG::start_flush(ObjectStore::Transaction *t,
}
/* Called before initializing peering during advance_map */
-void PG::start_peering_interval(const OSDMapRef lastmap,
- const vector<int>& newup,
- const vector<int>& newacting,
- ObjectStore::Transaction *t)
+void PG::start_peering_interval(
+ const OSDMapRef lastmap,
+ const vector<int>& newup, int new_up_primary,
+ const vector<int>& newacting, int new_acting_primary,
+ ObjectStore::Transaction *t)
{
const OSDMapRef osdmap = get_osdmap();
@@ -4267,18 +4608,27 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
vector<int> oldacting, oldup;
int oldrole = get_role();
- int oldprimary = get_primary();
+
+ pg_shard_t old_acting_primary = get_primary();
+ pg_shard_t old_up_primary = up_primary;
bool was_old_primary = is_primary();
+
acting.swap(oldacting);
up.swap(oldup);
-
- up = newup;
- acting = newacting;
+ init_primary_up_acting(
+ newup,
+ newacting,
+ new_up_primary,
+ new_acting_primary);
if (info.stats.up != up ||
- info.stats.acting != acting) {
+ info.stats.acting != acting ||
+ info.stats.up_primary != new_up_primary ||
+ info.stats.acting_primary != new_acting_primary) {
info.stats.up = up;
+ info.stats.up_primary = new_up_primary;
info.stats.acting = acting;
+ info.stats.acting_primary = new_acting_primary;
info.stats.mapping_epoch = info.history.same_interval_since;
}
@@ -4290,7 +4640,10 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
state_clear(PG_STATE_REMAPPED);
int role = osdmap->calc_pg_role(osd->whoami, acting, acting.size());
- set_role(role);
+ if (role == pg_whoami.shard)
+ set_role(role);
+ else
+ set_role(-1);
// did acting, up, primary|acker change?
if (!lastmap) {
@@ -4300,6 +4653,8 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
} else {
std::stringstream debug;
bool new_interval = pg_interval_t::check_new_interval(
+ old_acting_primary.osd,
+ new_acting_primary,
oldacting, newacting,
oldup, newup,
info.history.same_interval_since,
@@ -4307,7 +4662,7 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
osdmap,
lastmap,
info.pgid.pool(),
- info.pgid,
+ info.pgid.pgid,
&past_intervals,
&debug);
dout(10) << __func__ << ": check_new_interval output: "
@@ -4319,18 +4674,26 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
}
}
- if (oldacting != acting || oldup != up || is_split(lastmap, osdmap)) {
+ if (old_up_primary != up_primary ||
+ old_acting_primary != primary ||
+ oldacting != acting ||
+ oldup != up ||
+ is_split(lastmap, osdmap)) {
info.history.same_interval_since = osdmap->get_epoch();
}
- if (oldup != up) {
+ if (old_up_primary != up_primary ||
+ oldup != up) {
info.history.same_up_since = osdmap->get_epoch();
}
- if (oldprimary != get_primary()) {
+ // this comparison includes primary rank via pg_shard_t
+ if (old_acting_primary != get_primary()) {
info.history.same_primary_since = osdmap->get_epoch();
}
dout(10) << " up " << oldup << " -> " << up
<< ", acting " << oldacting << " -> " << acting
+ << ", acting_primary " << old_acting_primary << " -> " << new_acting_primary
+ << ", up_primary " << old_up_primary << " -> " << new_up_primary
<< ", role " << oldrole << " -> " << role << dendl;
// deactivate.
@@ -4345,7 +4708,7 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
// reset primary state?
if (was_old_primary || is_primary())
- clear_primary_state();
+ clear_primary_state(was_old_primary && is_primary());
// pg->on_*
@@ -4353,6 +4716,9 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
assert(!deleting);
+ // should we tell the primary we are here?
+ send_notify = !is_primary();
+
if (role != oldrole ||
was_old_primary != is_primary()) {
// did primary change?
@@ -4375,19 +4741,13 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
// take active waiters
requeue_ops(waiting_for_active);
- // should we tell the primary we are here?
- send_notify = !is_primary();
-
} else {
// no role change.
// did primary change?
- if (get_primary() != oldprimary) {
- // we need to announce
- send_notify = true;
-
+ if (get_primary() != old_acting_primary) {
dout(10) << *this << " " << oldacting << " -> " << acting
<< ", acting primary "
- << oldprimary << " -> " << get_primary()
+ << old_acting_primary << " -> " << get_primary()
<< dendl;
} else {
// primary is the same.
@@ -4400,13 +4760,11 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
}
}
}
- // make sure we clear out any pg_temp change requests
- osd->remove_want_pg_temp(info.pgid);
cancel_recovery();
- if (acting.empty() && !up.empty() && up[0] == osd->whoami) {
+ if (acting.empty() && !up.empty() && up_primary == pg_whoami) {
dout(10) << " acting empty, but i am up[0], clearing pg_temp" << dendl;
- osd->queue_want_pg_temp(info.pgid, acting);
+ osd->queue_want_pg_temp(info.pgid.pgid, acting);
}
}
@@ -4623,9 +4981,18 @@ bool PG::can_discard_request(OpRequestRef op)
return can_discard_replica_op<MOSDPGPushReply, MSG_OSD_PG_PUSH_REPLY>(op);
case MSG_OSD_SUBOPREPLY:
return false;
+
+ case MSG_OSD_EC_WRITE:
+ return can_discard_replica_op<MOSDECSubOpWrite, MSG_OSD_EC_WRITE>(op);
+ case MSG_OSD_EC_WRITE_REPLY:
+ return can_discard_replica_op<MOSDECSubOpWriteReply, MSG_OSD_EC_WRITE_REPLY>(op);
+ case MSG_OSD_EC_READ:
+ return can_discard_replica_op<MOSDECSubOpRead, MSG_OSD_EC_READ>(op);
+ case MSG_OSD_EC_READ_REPLY:
+ return can_discard_replica_op<MOSDECSubOpReadReply, MSG_OSD_EC_READ_REPLY>(op);
+
case MSG_OSD_PG_SCAN:
return can_discard_scan(op);
-
case MSG_OSD_PG_BACKFILL:
return can_discard_backfill(op);
}
@@ -4684,6 +5051,26 @@ bool PG::op_must_wait_for_map(OSDMapRef curmap, OpRequestRef op)
return !have_same_or_newer_map(
curmap,
static_cast<MOSDPGPushReply*>(op->get_req())->map_epoch);
+
+ case MSG_OSD_EC_WRITE:
+ return !have_same_or_newer_map(
+ curmap,
+ static_cast<MOSDECSubOpWrite*>(op->get_req())->map_epoch);
+
+ case MSG_OSD_EC_WRITE_REPLY:
+ return !have_same_or_newer_map(
+ curmap,
+ static_cast<MOSDECSubOpWriteReply*>(op->get_req())->map_epoch);
+
+ case MSG_OSD_EC_READ:
+ return !have_same_or_newer_map(
+ curmap,
+ static_cast<MOSDECSubOpRead*>(op->get_req())->map_epoch);
+
+ case MSG_OSD_EC_READ_REPLY:
+ return !have_same_or_newer_map(
+ curmap,
+ static_cast<MOSDECSubOpReadReply*>(op->get_req())->map_epoch);
}
assert(0);
return false;
@@ -4723,9 +5110,9 @@ void PG::queue_peering_event(CephPeeringEvtRef evt)
void PG::queue_notify(epoch_t msg_epoch,
epoch_t query_epoch,
- int from, pg_notify_t& i)
+ pg_shard_t from, pg_notify_t& i)
{
- dout(10) << "notify " << i << " from osd." << from << dendl;
+ dout(10) << "notify " << i << " from replica " << from << dendl;
queue_peering_event(
CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
MNotifyRec(from, i))));
@@ -4733,9 +5120,9 @@ void PG::queue_notify(epoch_t msg_epoch,
void PG::queue_info(epoch_t msg_epoch,
epoch_t query_epoch,
- int from, pg_info_t& i)
+ pg_shard_t from, pg_info_t& i)
{
- dout(10) << "info " << i << " from osd." << from << dendl;
+ dout(10) << "info " << i << " from replica " << from << dendl;
queue_peering_event(
CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
MInfoRec(from, i, msg_epoch))));
@@ -4743,10 +5130,10 @@ void PG::queue_info(epoch_t msg_epoch,
void PG::queue_log(epoch_t msg_epoch,
epoch_t query_epoch,
- int from,
+ pg_shard_t from,
MOSDPGLog *msg)
{
- dout(10) << "log " << *msg << " from osd." << from << dendl;
+ dout(10) << "log " << *msg << " from replica " << from << dendl;
queue_peering_event(
CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
MLogRec(from, msg))));
@@ -4771,26 +5158,33 @@ void PG::queue_flushed(epoch_t e)
void PG::queue_query(epoch_t msg_epoch,
epoch_t query_epoch,
- int from, const pg_query_t& q)
+ pg_shard_t from, const pg_query_t& q)
{
- dout(10) << "handle_query " << q << " from osd." << from << dendl;
+ dout(10) << "handle_query " << q << " from replica " << from << dendl;
queue_peering_event(
CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
MQuery(from, q, query_epoch))));
}
-void PG::handle_advance_map(OSDMapRef osdmap, OSDMapRef lastmap,
- vector<int>& newup, vector<int>& newacting,
- RecoveryCtx *rctx)
+void PG::handle_advance_map(
+ OSDMapRef osdmap, OSDMapRef lastmap,
+ vector<int>& newup, int up_primary,
+ vector<int>& newacting, int acting_primary,
+ RecoveryCtx *rctx)
{
assert(lastmap->get_epoch() == osdmap_ref->get_epoch());
assert(lastmap == osdmap_ref);
- dout(10) << "handle_advance_map " << newup << "/" << newacting << dendl;
+ dout(10) << "handle_advance_map "
+ << newup << "/" << newacting
+ << " -- " << up_primary << "/" << acting_primary
+ << dendl;
update_osdmap_ref(osdmap);
pool.update(osdmap);
if (pool.info.last_change == osdmap_ref->get_epoch())
on_pool_change();
- AdvMap evt(osdmap, lastmap, newup, newacting);
+ AdvMap evt(
+ osdmap, lastmap, newup, up_primary,
+ newacting, acting_primary);
recovery_state.handle_event(evt, rctx);
}
@@ -4934,8 +5328,12 @@ boost::statechart::result PG::RecoveryState::Started::react(const AdvMap& advmap
{
dout(10) << "Started advmap" << dendl;
PG *pg = context< RecoveryMachine >().pg;
- if (pg->acting_up_affected(advmap.newup, advmap.newacting) ||
- pg->is_split(advmap.lastmap, advmap.osdmap)) {
+ if (pg->acting_up_affected(
+ advmap.up_primary,
+ advmap.acting_primary,
+ advmap.newup,
+ advmap.newacting) ||
+ pg->is_split(advmap.lastmap, advmap.osdmap)) {
dout(10) << "up or acting affected, transitioning to Reset" << dendl;
post_event(advmap);
return transit< Reset >();
@@ -4989,26 +5387,36 @@ boost::statechart::result PG::RecoveryState::Reset::react(const AdvMap& advmap)
// _before_ we are active.
pg->generate_past_intervals();
- pg->remove_down_peer_info(advmap.osdmap);
- if (pg->acting_up_affected(advmap.newup, advmap.newacting) ||
- pg->is_split(advmap.lastmap, advmap.osdmap)) {
+ if (pg->acting_up_affected(
+ advmap.up_primary,
+ advmap.acting_primary,
+ advmap.newup,
+ advmap.newacting) ||
+ pg->is_split(advmap.lastmap, advmap.osdmap)) {
dout(10) << "up or acting affected, calling start_peering_interval again"
<< dendl;
- pg->start_peering_interval(advmap.lastmap, advmap.newup, advmap.newacting,
- context< RecoveryMachine >().get_cur_transaction());
+ pg->start_peering_interval(
+ advmap.lastmap,
+ advmap.newup, advmap.up_primary,
+ advmap.newacting, advmap.acting_primary,
+ context< RecoveryMachine >().get_cur_transaction());
}
+ pg->remove_down_peer_info(advmap.osdmap);
return discard_event();
}
boost::statechart::result PG::RecoveryState::Reset::react(const ActMap&)
{
PG *pg = context< RecoveryMachine >().pg;
- if (pg->should_send_notify() && pg->get_primary() >= 0) {
- context< RecoveryMachine >().send_notify(pg->get_primary(),
- pg_notify_t(pg->get_osdmap()->get_epoch(),
- pg->get_osdmap()->get_epoch(),
- pg->info),
- pg->past_intervals);
+ if (pg->should_send_notify() && pg->get_primary().osd >= 0) {
+ context< RecoveryMachine >().send_notify(
+ pg->get_primary(),
+ pg_notify_t(
+ pg->get_primary().shard, pg->pg_whoami.shard,
+ pg->get_osdmap()->get_epoch(),
+ pg->get_osdmap()->get_epoch(),
+ pg->info),
+ pg->past_intervals);
}
pg->update_heartbeat_peers();
@@ -5069,13 +5477,6 @@ PG::RecoveryState::Primary::Primary(my_context ctx)
assert(pg->want_acting.empty());
}
-boost::statechart::result PG::RecoveryState::Primary::react(const AdvMap &advmap)
-{
- PG *pg = context< RecoveryMachine >().pg;
- pg->remove_down_peer_info(advmap.osdmap);
- return forward_event();
-}
-
boost::statechart::result PG::RecoveryState::Primary::react(const MNotifyRec& notevt)
{
dout(7) << "handle_pg_notify from osd." << notevt.from << dendl;
@@ -5156,15 +5557,19 @@ boost::statechart::result PG::RecoveryState::Peering::react(const QueryState& q)
q.f->close_section();
q.f->open_array_section("probing_osds");
- for (set<int>::iterator p = prior_set->probe.begin(); p != prior_set->probe.end(); ++p)
- q.f->dump_int("osd", *p);
+ for (set<pg_shard_t>::iterator p = prior_set->probe.begin();
+ p != prior_set->probe.end();
+ ++p)
+ q.f->dump_stream("osd") << *p;
q.f->close_section();
if (prior_set->pg_down)
q.f->dump_string("blocked", "peering is blocked due to down osds");
q.f->open_array_section("down_osds_we_would_probe");
- for (set<int>::iterator p = prior_set->down.begin(); p != prior_set->down.end(); ++p)
+ for (set<int>::iterator p = prior_set->down.begin();
+ p != prior_set->down.end();
+ ++p)
q.f->dump_int("osd", *p);
q.f->close_section();
@@ -5255,8 +5660,9 @@ PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteBackfillReserve
if (backfill_osd_it != context< Active >().sorted_backfill_set.end()) {
//The primary never backfills itself
- assert(*backfill_osd_it != pg->osd->whoami);
- ConnectionRef con = pg->osd->get_con_osd_cluster(*backfill_osd_it, pg->get_osdmap()->get_epoch());
+ assert(*backfill_osd_it != pg->pg_whoami);
+ ConnectionRef con = pg->osd->get_con_osd_cluster(
+ backfill_osd_it->osd, pg->get_osdmap()->get_epoch());
if (con) {
if (con->has_feature(CEPH_FEATURE_BACKFILL_RESERVATION)) {
unsigned priority = pg->is_degraded() ? OSDService::BACKFILL_HIGH
@@ -5264,7 +5670,7 @@ PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteBackfillReserve
pg->osd->send_message_osd_cluster(
new MBackfillReserve(
MBackfillReserve::REQUEST,
- pg->info.pgid,
+ spg_t(pg->info.pgid.pgid, backfill_osd_it->shard),
pg->get_osdmap()->get_epoch(), priority),
con.get());
} else {
@@ -5375,10 +5781,10 @@ PG::RecoveryState::RepWaitRecoveryReserved::react(const RemoteRecoveryReserved &
{
PG *pg = context< RecoveryMachine >().pg;
pg->osd->send_message_osd_cluster(
- pg->acting[0],
+ pg->primary.osd,
new MRecoveryReserve(
MRecoveryReserve::GRANT,
- pg->info.pgid,
+ spg_t(pg->info.pgid.pgid, pg->primary.shard),
pg->get_osdmap()->get_epoch()),
pg->get_osdmap()->get_epoch());
return transit<RepRecovering>();
@@ -5435,10 +5841,10 @@ PG::RecoveryState::RepWaitBackfillReserved::react(const RemoteBackfillReserved &
{
PG *pg = context< RecoveryMachine >().pg;
pg->osd->send_message_osd_cluster(
- pg->acting[0],
+ pg->primary.osd,
new MBackfillReserve(
MBackfillReserve::GRANT,
- pg->info.pgid,
+ spg_t(pg->info.pgid.pgid, pg->primary.shard),
pg->get_osdmap()->get_epoch()),
pg->get_osdmap()->get_epoch());
return transit<RepRecovering>();
@@ -5518,7 +5924,7 @@ void PG::RecoveryState::WaitLocalRecoveryReserved::exit()
PG::RecoveryState::WaitRemoteRecoveryReserved::WaitRemoteRecoveryReserved(my_context ctx)
: my_base(ctx),
NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteRecoveryReserved"),
- acting_osd_it(context< Active >().sorted_acting_set.begin())
+ acting_osd_it(context< Active >().sorted_actingbackfill_set.begin())
{
context< RecoveryMachine >().log_enter(state_name);
post_event(RemoteRecoveryReserved());
@@ -5528,20 +5934,22 @@ boost::statechart::result
PG::RecoveryState::WaitRemoteRecoveryReserved::react(const RemoteRecoveryReserved &evt) {
PG *pg = context< RecoveryMachine >().pg;
- if (acting_osd_it != context< Active >().sorted_acting_set.end()) {
+ if (acting_osd_it != context< Active >().sorted_actingbackfill_set.end()) {
// skip myself
- if (*acting_osd_it == pg->osd->whoami)
+ if (*acting_osd_it == pg->pg_whoami)
++acting_osd_it;
}
- if (acting_osd_it != context< Active >().sorted_acting_set.end()) {
- ConnectionRef con = pg->osd->get_con_osd_cluster(*acting_osd_it, pg->get_osdmap()->get_epoch());
+ if (acting_osd_it != context< Active >().sorted_actingbackfill_set.end()) {
+ ConnectionRef con = pg->osd->get_con_osd_cluster(
+ acting_osd_it->osd, pg->get_osdmap()->get_epoch());
if (con) {
if (con->has_feature(CEPH_FEATURE_RECOVERY_RESERVATION)) {
pg->osd->send_message_osd_cluster(
- new MRecoveryReserve(MRecoveryReserve::REQUEST,
- pg->info.pgid,
- pg->get_osdmap()->get_epoch()),
+ new MRecoveryReserve(
+ MRecoveryReserve::REQUEST,
+ spg_t(pg->info.pgid.pgid, acting_osd_it->shard),
+ pg->get_osdmap()->get_epoch()),
con.get());
} else {
post_event(RemoteRecoveryReserved());
@@ -5580,18 +5988,21 @@ void PG::RecoveryState::Recovering::release_reservations()
assert(!pg->pg_log.get_missing().have_missing());
// release remote reservations
- for (set<int>::const_iterator i = context< Active >().sorted_acting_set.begin();
- i != context< Active >().sorted_acting_set.end();
+ for (set<pg_shard_t>::const_iterator i =
+ context< Active >().sorted_actingbackfill_set.begin();
+ i != context< Active >().sorted_actingbackfill_set.end();
++i) {
- if (*i == pg->osd->whoami) // skip myself
+ if (*i == pg->pg_whoami) // skip myself
continue;
- ConnectionRef con = pg->osd->get_con_osd_cluster(*i, pg->get_osdmap()->get_epoch());
+ ConnectionRef con = pg->osd->get_con_osd_cluster(
+ i->osd, pg->get_osdmap()->get_epoch());
if (con) {
if (con->has_feature(CEPH_FEATURE_RECOVERY_RESERVATION)) {
pg->osd->send_message_osd_cluster(
- new MRecoveryReserve(MRecoveryReserve::RELEASE,
- pg->info.pgid,
- pg->get_osdmap()->get_epoch()),
+ new MRecoveryReserve(
+ MRecoveryReserve::RELEASE,
+ spg_t(pg->info.pgid.pgid, i->shard),
+ pg->get_osdmap()->get_epoch()),
con.get());
}
}
@@ -5628,7 +6039,7 @@ PG::RecoveryState::Recovered::Recovered(my_context ctx)
: my_base(ctx),
NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovered")
{
- int newest_update_osd;
+ pg_shard_t auth_log_shard;
context< RecoveryMachine >().log_enter(state_name);
@@ -5637,12 +6048,13 @@ PG::RecoveryState::Recovered::Recovered(my_context ctx)
// if we finished backfill, all acting are active; recheck if
// DEGRADED is appropriate.
- assert(pg->actingbackfill.size() > 0);
- if (pg->get_osdmap()->get_pg_size(pg->info.pgid) <= pg->actingbackfill.size())
+ assert(!pg->actingbackfill.empty());
+ if (pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid) <=
+ pg->actingbackfill.size())
pg->state_clear(PG_STATE_DEGRADED);
// adjust acting set? (e.g. because backfill completed...)
- if (pg->acting != pg->up && !pg->choose_acting(newest_update_osd))
+ if (pg->acting != pg->up && !pg->choose_acting(auth_log_shard))
assert(pg->want_acting.size());
assert(!pg->needs_recovery());
@@ -5691,10 +6103,12 @@ void PG::RecoveryState::Clean::exit()
PG::RecoveryState::Active::Active(my_context ctx)
: my_base(ctx),
NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active"),
- sorted_acting_set(context< RecoveryMachine >().pg->actingbackfill.begin(),
- context< RecoveryMachine >().pg->actingbackfill.end()),
- sorted_backfill_set(context< RecoveryMachine >().pg->backfill_targets.begin(),
- context< RecoveryMachine >().pg->backfill_targets.end()),
+ sorted_actingbackfill_set(
+ context< RecoveryMachine >().pg->actingbackfill.begin(),
+ context< RecoveryMachine >().pg->actingbackfill.end()),
+ sorted_backfill_set(
+ context< RecoveryMachine >().pg->backfill_targets.begin(),
+ context< RecoveryMachine >().pg->backfill_targets.end()),
all_replicas_activated(false)
{
context< RecoveryMachine >().log_enter(state_name);
@@ -5713,8 +6127,8 @@ PG::RecoveryState::Active::Active(my_context ctx)
pg->get_osdmap()->get_epoch(),
*context< RecoveryMachine >().get_on_safe_context_list(),
*context< RecoveryMachine >().get_query_map(),
- context< RecoveryMachine >().get_info_map());
- assert(pg->is_active());
+ context< RecoveryMachine >().get_info_map(),
+ context< RecoveryMachine >().get_recovery_ctx());
dout(10) << "Activate Finished" << dendl;
}
@@ -5741,9 +6155,9 @@ boost::statechart::result PG::RecoveryState::Active::react(const AdvMap& advmap)
/* Check for changes in pool size (if the acting set changed as a result,
* this does not matter) */
- if (advmap.lastmap->get_pg_size(pg->info.pgid) !=
- pg->get_osdmap()->get_pg_size(pg->info.pgid)) {
- if (pg->get_osdmap()->get_pg_size(pg->info.pgid) <= pg->acting.size())
+ if (advmap.lastmap->get_pg_size(pg->info.pgid.pgid) !=
+ pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid)) {
+ if (pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid) <= pg->acting.size())
pg->state_clear(PG_STATE_DEGRADED);
else
pg->state_set(PG_STATE_DEGRADED);
@@ -5764,7 +6178,6 @@ boost::statechart::result PG::RecoveryState::Active::react(const ActMap&)
{
PG *pg = context< RecoveryMachine >().pg;
dout(10) << "Active: handling ActMap" << dendl;
- assert(pg->is_active());
assert(pg->is_primary());
if (pg->have_unfound()) {
@@ -5775,7 +6188,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const ActMap&)
if (pg->cct->_conf->osd_check_for_log_corruption)
pg->check_log_for_corruption(pg->osd->store);
- int unfound = pg->pg_log.get_missing().num_missing() - pg->missing_loc.size();
+ int unfound = pg->missing_loc.num_unfound();
if (unfound > 0 &&
pg->all_unfound_are_queried_or_lost(pg->get_osdmap())) {
if (pg->cct->_conf->osd_auto_mark_unfound_lost) {
@@ -5802,7 +6215,6 @@ boost::statechart::result PG::RecoveryState::Active::react(const ActMap&)
boost::statechart::result PG::RecoveryState::Active::react(const MNotifyRec& notevt)
{
PG *pg = context< RecoveryMachine >().pg;
- assert(pg->is_active());
assert(pg->is_primary());
if (pg->peer_info.count(notevt.from)) {
dout(10) << "Active: got notify from " << notevt.from
@@ -5827,10 +6239,9 @@ boost::statechart::result PG::RecoveryState::Active::react(const MNotifyRec& not
boost::statechart::result PG::RecoveryState::Active::react(const MInfoRec& infoevt)
{
PG *pg = context< RecoveryMachine >().pg;
- assert(pg->is_active());
assert(pg->is_primary());
- assert(pg->actingbackfill.size() > 0);
+ assert(!pg->actingbackfill.empty());
// don't update history (yet) if we are active and primary; the replica
// may be telling us they have activated (and committed) but we can't
// share that until _everyone_ does the same.
@@ -5842,10 +6253,10 @@ boost::statechart::result PG::RecoveryState::Active::react(const MInfoRec& infoe
dout(10) << " peer osd." << infoevt.from << " activated and committed"
<< dendl;
pg->peer_activated.insert(infoevt.from);
- }
- if (pg->peer_activated.size() == pg->actingbackfill.size()) {
- pg->all_activated_and_committed();
+ if (pg->peer_activated.size() == pg->actingbackfill.size()) {
+ pg->all_activated_and_committed();
+ }
}
return discard_event();
}
@@ -5855,8 +6266,14 @@ boost::statechart::result PG::RecoveryState::Active::react(const MLogRec& logevt
dout(10) << "searching osd." << logevt.from
<< " log for unfound items" << dendl;
PG *pg = context< RecoveryMachine >().pg;
- bool got_missing = pg->search_for_missing(logevt.msg->info,
- &logevt.msg->missing, logevt.from);
+ pg->proc_replica_log(
+ *context<RecoveryMachine>().get_cur_transaction(),
+ logevt.msg->info, logevt.msg->log, logevt.msg->missing, logevt.from);
+ bool got_missing = pg->search_for_missing(
+ pg->peer_info[logevt.from],
+ pg->peer_missing[logevt.from],
+ logevt.from,
+ context< RecoveryMachine >().get_recovery_ctx());
if (got_missing)
pg->osd->queue_for_recovery(pg);
return discard_event();
@@ -5872,16 +6289,16 @@ boost::statechart::result PG::RecoveryState::Active::react(const QueryState& q)
{
q.f->open_array_section("might_have_unfound");
- for (set<int>::iterator p = pg->might_have_unfound.begin();
+ for (set<pg_shard_t>::iterator p = pg->might_have_unfound.begin();
p != pg->might_have_unfound.end();
++p) {
q.f->open_object_section("osd");
- q.f->dump_int("osd", *p);
+ q.f->dump_stream("osd") << *p;
if (pg->peer_missing.count(*p)) {
q.f->dump_string("status", "already probed");
} else if (pg->peer_missing_requested.count(*p)) {
q.f->dump_string("status", "querying");
- } else if (!pg->get_osdmap()->is_up(*p)) {
+ } else if (!pg->get_osdmap()->is_up(p->osd)) {
q.f->dump_string("status", "osd is down");
} else {
q.f->dump_string("status", "not queried");
@@ -5905,10 +6322,10 @@ boost::statechart::result PG::RecoveryState::Active::react(const QueryState& q)
q.f->dump_int("scrubber.waiting_on", pg->scrubber.waiting_on);
{
q.f->open_array_section("scrubber.waiting_on_whom");
- for (set<int>::iterator p = pg->scrubber.waiting_on_whom.begin();
+ for (set<pg_shard_t>::iterator p = pg->scrubber.waiting_on_whom.begin();
p != pg->scrubber.waiting_on_whom.end();
++p) {
- q.f->dump_int("osd", *p);
+ q.f->dump_stream("shard") << *p;
}
q.f->close_section();
}
@@ -5921,7 +6338,20 @@ boost::statechart::result PG::RecoveryState::Active::react(const QueryState& q)
boost::statechart::result PG::RecoveryState::Active::react(const AllReplicasActivated &evt)
{
+ PG *pg = context< RecoveryMachine >().pg;
all_replicas_activated = true;
+
+ pg->state_set(PG_STATE_ACTIVE);
+
+ pg->check_local();
+
+ // waiters
+ if (!pg->is_replay() && pg->flushes_in_progress == 0) {
+ pg->requeue_ops(pg->waiting_for_active);
+ }
+
+ pg->on_activate();
+
return discard_event();
}
@@ -5940,6 +6370,7 @@ void PG::RecoveryState::Active::exit()
pg->state_clear(PG_STATE_REPLAY);
utime_t dur = ceph_clock_now(pg->cct) - enter_time;
pg->osd->recoverystate_perf->tinc(rs_active_latency, dur);
+ pg->agent_stop();
}
/*------ReplicaActive-----*/
@@ -5961,11 +6392,11 @@ boost::statechart::result PG::RecoveryState::ReplicaActive::react(
const Activate& actevt) {
dout(10) << "In ReplicaActive, about to call activate" << dendl;
PG *pg = context< RecoveryMachine >().pg;
- map< int, map< pg_t, pg_query_t> > query_map;
+ map<int, map<spg_t, pg_query_t> > query_map;
pg->activate(*context< RecoveryMachine >().get_cur_transaction(),
actevt.query_epoch,
*context< RecoveryMachine >().get_on_safe_context_list(),
- query_map, NULL);
+ query_map, NULL, NULL);
dout(10) << "Activate Finished" << dendl;
return discard_event();
}
@@ -5992,12 +6423,15 @@ boost::statechart::result PG::RecoveryState::ReplicaActive::react(const MLogRec&
boost::statechart::result PG::RecoveryState::ReplicaActive::react(const ActMap&)
{
PG *pg = context< RecoveryMachine >().pg;
- if (pg->should_send_notify() && pg->get_primary() >= 0) {
- context< RecoveryMachine >().send_notify(pg->get_primary(),
- pg_notify_t(pg->get_osdmap()->get_epoch(),
- pg->get_osdmap()->get_epoch(),
- pg->info),
- pg->past_intervals);
+ if (pg->should_send_notify() && pg->get_primary().osd >= 0) {
+ context< RecoveryMachine >().send_notify(
+ pg->get_primary(),
+ pg_notify_t(
+ pg->get_primary().shard, pg->pg_whoami.shard,
+ pg->get_osdmap()->get_epoch(),
+ pg->get_osdmap()->get_epoch(),
+ pg->info),
+ pg->past_intervals);
}
pg->take_waiters();
return discard_event();
@@ -6100,14 +6534,17 @@ boost::statechart::result PG::RecoveryState::Stray::react(const MQuery& query)
{
PG *pg = context< RecoveryMachine >().pg;
if (query.query.type == pg_query_t::INFO) {
- pair<int, pg_info_t> notify_info;
+ pair<pg_shard_t, pg_info_t> notify_info;
pg->update_history_from_master(query.query.history);
pg->fulfill_info(query.from, query.query, notify_info);
- context< RecoveryMachine >().send_notify(notify_info.first,
- pg_notify_t(query.query_epoch,
- pg->get_osdmap()->get_epoch(),
- notify_info.second),
- pg->past_intervals);
+ context< RecoveryMachine >().send_notify(
+ notify_info.first,
+ pg_notify_t(
+ notify_info.first.shard, pg->pg_whoami.shard,
+ query.query_epoch,
+ pg->get_osdmap()->get_epoch(),
+ notify_info.second),
+ pg->past_intervals);
} else {
pg->fulfill_log(query.from, query.query, query.query_epoch);
}
@@ -6117,12 +6554,15 @@ boost::statechart::result PG::RecoveryState::Stray::react(const MQuery& query)
boost::statechart::result PG::RecoveryState::Stray::react(const ActMap&)
{
PG *pg = context< RecoveryMachine >().pg;
- if (pg->should_send_notify() && pg->get_primary() >= 0) {
- context< RecoveryMachine >().send_notify(pg->get_primary(),
- pg_notify_t(pg->get_osdmap()->get_epoch(),
- pg->get_osdmap()->get_epoch(),
- pg->info),
- pg->past_intervals);
+ if (pg->should_send_notify() && pg->get_primary().osd >= 0) {
+ context< RecoveryMachine >().send_notify(
+ pg->get_primary(),
+ pg_notify_t(
+ pg->get_primary().shard, pg->pg_whoami.shard,
+ pg->get_osdmap()->get_epoch(),
+ pg->get_osdmap()->get_epoch(),
+ pg->info),
+ pg->past_intervals);
}
pg->take_waiters();
return discard_event();
@@ -6163,11 +6603,11 @@ void PG::RecoveryState::GetInfo::get_infos()
PG *pg = context< RecoveryMachine >().pg;
auto_ptr<PriorSet> &prior_set = context< Peering >().prior_set;
- for (set<int>::const_iterator it = prior_set->probe.begin();
+ for (set<pg_shard_t>::const_iterator it = prior_set->probe.begin();
it != prior_set->probe.end();
++it) {
- int peer = *it;
- if (peer == pg->osd->whoami) {
+ pg_shard_t peer = *it;
+ if (peer == pg->pg_whoami) {
continue;
}
if (pg->peer_info.count(peer)) {
@@ -6176,12 +6616,13 @@ void PG::RecoveryState::GetInfo::get_infos()
}
if (peer_info_requested.count(peer)) {
dout(10) << " already requested info from osd." << peer << dendl;
- } else if (!pg->get_osdmap()->is_up(peer)) {
+ } else if (!pg->get_osdmap()->is_up(peer.osd)) {
dout(10) << " not querying info from down osd." << peer << dendl;
} else {
dout(10) << " querying info from osd." << peer << dendl;
context< RecoveryMachine >().send_query(
peer, pg_query_t(pg_query_t::INFO,
+ it->shard, pg->pg_whoami.shard,
pg->info.history,
pg->get_osdmap()->get_epoch()));
peer_info_requested.insert(peer);
@@ -6191,7 +6632,7 @@ void PG::RecoveryState::GetInfo::get_infos()
boost::statechart::result PG::RecoveryState::GetInfo::react(const MNotifyRec& infoevt)
{
- set<int>::iterator p = peer_info_requested.find(infoevt.from);
+ set<pg_shard_t>::iterator p = peer_info_requested.find(infoevt.from);
if (p != peer_info_requested.end())
peer_info_requested.erase(p);
@@ -6207,7 +6648,7 @@ boost::statechart::result PG::RecoveryState::GetInfo::react(const MNotifyRec& in
// filter out any osds that got dropped from the probe set from
// peer_info_requested. this is less expensive than restarting
// peering (which would re-probe everyone).
- set<int>::iterator p = peer_info_requested.begin();
+ set<pg_shard_t>::iterator p = peer_info_requested.begin();
while (p != peer_info_requested.end()) {
if (prior_set->probe.count(*p) == 0) {
dout(20) << " dropping osd." << *p << " from info_requested, no longer in probe set" << dendl;
@@ -6249,15 +6690,18 @@ boost::statechart::result PG::RecoveryState::GetInfo::react(const MNotifyRec& in
bool any_down_now = false;
for (unsigned i=0; i<interval.acting.size(); i++) {
int o = interval.acting[i];
+ if (o == CRUSH_ITEM_NONE)
+ continue;
+ pg_shard_t so(o, pg->pool.info.ec_pool() ? i : ghobject_t::NO_SHARD);
if (!osdmap->exists(o) || osdmap->get_info(o).lost_at > interval.first)
continue; // dne or lost
if (osdmap->is_up(o)) {
pg_info_t *pinfo;
- if (o == pg->osd->whoami) {
+ if (so == pg->pg_whoami) {
pinfo = &pg->info;
} else {
- assert(pg->peer_info.count(o));
- pinfo = &pg->peer_info[o];
+ assert(pg->peer_info.count(so));
+ pinfo = &pg->peer_info[so];
}
if (!pinfo->is_incomplete())
any_up_complete_now = true;
@@ -6287,9 +6731,11 @@ boost::statechart::result PG::RecoveryState::GetInfo::react(const QueryState& q)
q.f->dump_stream("enter_time") << enter_time;
q.f->open_array_section("requested_info_from");
- for (set<int>::iterator p = peer_info_requested.begin(); p != peer_info_requested.end(); ++p) {
+ for (set<pg_shard_t>::iterator p = peer_info_requested.begin();
+ p != peer_info_requested.end();
+ ++p) {
q.f->open_object_section("osd");
- q.f->dump_int("osd", *p);
+ q.f->dump_stream("osd") << *p;
if (pg->peer_info.count(*p)) {
q.f->open_object_section("got_info");
pg->peer_info[*p].dump(q.f);
@@ -6314,15 +6760,16 @@ void PG::RecoveryState::GetInfo::exit()
/*------GetLog------------*/
PG::RecoveryState::GetLog::GetLog(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetLog"),
- newest_update_osd(-1), msg(0)
+ NamedState(
+ context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetLog"),
+ msg(0)
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
// adjust acting?
- if (!pg->choose_acting(newest_update_osd)) {
+ if (!pg->choose_acting(auth_log_shard)) {
if (!pg->want_acting.empty()) {
post_event(NeedActingChange());
} else {
@@ -6332,36 +6779,41 @@ PG::RecoveryState::GetLog::GetLog(my_context ctx)
}
// am i the best?
- if (newest_update_osd == pg->osd->whoami) {
+ if (auth_log_shard == pg->pg_whoami) {
post_event(GotLog());
return;
}
- const pg_info_t& best = pg->peer_info[newest_update_osd];
+ const pg_info_t& best = pg->peer_info[auth_log_shard];
// am i broken?
if (pg->info.last_update < best.log_tail) {
- dout(10) << " not contiguous with osd." << newest_update_osd << ", down" << dendl;
+ dout(10) << " not contiguous with osd." << auth_log_shard << ", down" << dendl;
post_event(IsIncomplete());
return;
}
// how much log to request?
eversion_t request_log_from = pg->info.last_update;
- assert(pg->actingbackfill.size() > 0);
- for (vector<int>::iterator p = pg->actingbackfill.begin() + 1;
- p != pg->actingbackfill.end(); ++p) {
+ assert(!pg->actingbackfill.empty());
+ for (set<pg_shard_t>::iterator p = pg->actingbackfill.begin();
+ p != pg->actingbackfill.end();
+ ++p) {
+ if (*p == pg->pg_whoami) continue;
pg_info_t& ri = pg->peer_info[*p];
if (ri.last_update >= best.log_tail && ri.last_update < request_log_from)
request_log_from = ri.last_update;
}
// how much?
- dout(10) << " requesting log from osd." << newest_update_osd << dendl;
+ dout(10) << " requesting log from osd." << auth_log_shard << dendl;
context<RecoveryMachine>().send_query(
- newest_update_osd,
- pg_query_t(pg_query_t::LOG, request_log_from, pg->info.history,
- pg->get_osdmap()->get_epoch()));
+ auth_log_shard,
+ pg_query_t(
+ pg_query_t::LOG,
+ auth_log_shard.shard, pg->pg_whoami.shard,
+ request_log_from, pg->info.history,
+ pg->get_osdmap()->get_epoch()));
}
boost::statechart::result PG::RecoveryState::GetLog::react(const AdvMap& advmap)
@@ -6369,8 +6821,9 @@ boost::statechart::result PG::RecoveryState::GetLog::react(const AdvMap& advmap)
// make sure our log source didn't go down. we need to check
// explicitly because it may not be part of the prior set, which
// means the Peering state check won't catch it going down.
- if (!advmap.osdmap->is_up(newest_update_osd)) {
- dout(10) << "GetLog: newest_update_osd osd." << newest_update_osd << " went down" << dendl;
+ if (!advmap.osdmap->is_up(auth_log_shard.osd)) {
+ dout(10) << "GetLog: auth_log_shard osd."
+ << auth_log_shard.osd << " went down" << dendl;
post_event(advmap);
return transit< Reset >();
}
@@ -6382,9 +6835,9 @@ boost::statechart::result PG::RecoveryState::GetLog::react(const AdvMap& advmap)
boost::statechart::result PG::RecoveryState::GetLog::react(const MLogRec& logevt)
{
assert(!msg);
- if (logevt.from != newest_update_osd) {
+ if (logevt.from != auth_log_shard) {
dout(10) << "GetLog: discarding log from "
- << "non-newest_update_osd osd." << logevt.from << dendl;
+ << "non-auth_log_shard osd." << logevt.from << dendl;
return discard_event();
}
dout(10) << "GetLog: recieved master log from osd"
@@ -6402,7 +6855,7 @@ boost::statechart::result PG::RecoveryState::GetLog::react(const GotLog&)
dout(10) << "processing master log" << dendl;
pg->proc_master_log(*context<RecoveryMachine>().get_cur_transaction(),
msg->info, msg->log, msg->missing,
- newest_update_osd);
+ auth_log_shard);
}
pg->start_flush(
context< RecoveryMachine >().get_cur_transaction(),
@@ -6416,7 +6869,7 @@ boost::statechart::result PG::RecoveryState::GetLog::react(const QueryState& q)
q.f->open_object_section("state");
q.f->dump_string("name", state_name);
q.f->dump_stream("enter_time") << enter_time;
- q.f->dump_int("newest_update_osd", newest_update_osd);
+ q.f->dump_stream("auth_log_shard") << auth_log_shard;
q.f->close_section();
return forward_event();
}
@@ -6549,10 +7002,11 @@ PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
- assert(pg->actingbackfill.size() > 0);
- for (vector<int>::iterator i = pg->actingbackfill.begin() + 1;
+ assert(!pg->actingbackfill.empty());
+ for (set<pg_shard_t>::iterator i = pg->actingbackfill.begin();
i != pg->actingbackfill.end();
++i) {
+ if (*i == pg->get_primary()) continue;
const pg_info_t& pi = pg->peer_info[*i];
if (pi.is_empty())
@@ -6577,7 +7031,6 @@ PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
// can infer the rest!
dout(10) << " osd." << *i << " has no missing, identical log" << dendl;
pg->peer_missing[*i];
- pg->search_for_missing(pi, &pg->peer_missing[*i], *i);
continue;
}
@@ -6589,15 +7042,20 @@ PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
dout(10) << " requesting log+missing since " << since << " from osd." << *i << dendl;
context< RecoveryMachine >().send_query(
*i,
- pg_query_t(pg_query_t::LOG, since, pg->info.history,
- pg->get_osdmap()->get_epoch()));
+ pg_query_t(
+ pg_query_t::LOG,
+ i->shard, pg->pg_whoami.shard,
+ since, pg->info.history,
+ pg->get_osdmap()->get_epoch()));
} else {
dout(10) << " requesting fulllog+missing from osd." << *i
<< " (want since " << since << " < log.tail " << pi.log_tail << ")"
<< dendl;
context< RecoveryMachine >().send_query(
- *i, pg_query_t(pg_query_t::FULLLOG,
- pg->info.history, pg->get_osdmap()->get_epoch()));
+ *i, pg_query_t(
+ pg_query_t::FULLLOG,
+ i->shard, pg->pg_whoami.shard,
+ pg->info.history, pg->get_osdmap()->get_epoch()));
}
peer_missing_requested.insert(*i);
}
@@ -6610,7 +7068,7 @@ PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
}
// all good!
- post_event(CheckRepops());
+ post_event(Activate(pg->get_osdmap()->get_epoch()));
}
}
@@ -6629,7 +7087,7 @@ boost::statechart::result PG::RecoveryState::GetMissing::react(const MLogRec& lo
} else {
dout(10) << "Got last missing, don't need missing "
<< "posting CheckRepops" << dendl;
- post_event(CheckRepops());
+ post_event(Activate(pg->get_osdmap()->get_epoch()));
}
}
return discard_event();
@@ -6643,9 +7101,11 @@ boost::statechart::result PG::RecoveryState::GetMissing::react(const QueryState&
q.f->dump_stream("enter_time") << enter_time;
q.f->open_array_section("peer_missing_requested");
- for (set<int>::iterator p = peer_missing_requested.begin(); p != peer_missing_requested.end(); ++p) {
+ for (set<pg_shard_t>::iterator p = peer_missing_requested.begin();
+ p != peer_missing_requested.end();
+ ++p) {
q.f->open_object_section("osd");
- q.f->dump_int("osd", *p);
+ q.f->dump_stream("osd") << *p;
if (pg->peer_missing.count(*p)) {
q.f->open_object_section("got_missing");
pg->peer_missing[*p].dump(q.f);
@@ -6667,35 +7127,6 @@ void PG::RecoveryState::GetMissing::exit()
pg->osd->recoverystate_perf->tinc(rs_getmissing_latency, dur);
}
-/*---WaitFlushedPeering---*/
-PG::RecoveryState::WaitFlushedPeering::WaitFlushedPeering(my_context ctx)
- : my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitFlushedPeering")
-{
- PG *pg = context< RecoveryMachine >().pg;
- context< RecoveryMachine >().log_enter(state_name);
- if (context< RecoveryMachine >().pg->flushes_in_progress == 0)
- post_event(Activate(pg->get_osdmap()->get_epoch()));
-}
-
-boost::statechart::result
-PG::RecoveryState::WaitFlushedPeering::react(const FlushedEvt &evt)
-{
- PG *pg = context< RecoveryMachine >().pg;
- pg->on_flushed();
- return transit< WaitFlushedPeering >();
-}
-
-boost::statechart::result
-PG::RecoveryState::WaitFlushedPeering::react(const QueryState &q)
-{
- q.f->open_object_section("state");
- q.f->dump_string("name", state_name);
- q.f->dump_stream("enter_time") << enter_time;
- q.f->dump_string("comment", "waiting for flush");
- return forward_event();
-}
-
/*------WaitUpThru--------*/
PG::RecoveryState::WaitUpThru::WaitUpThru(my_context ctx)
: my_base(ctx),
@@ -6708,24 +7139,17 @@ boost::statechart::result PG::RecoveryState::WaitUpThru::react(const ActMap& am)
{
PG *pg = context< RecoveryMachine >().pg;
if (!pg->need_up_thru) {
- post_event(CheckRepops());
+ post_event(Activate(pg->get_osdmap()->get_epoch()));
}
return forward_event();
}
boost::statechart::result PG::RecoveryState::WaitUpThru::react(const MLogRec& logevt)
{
- dout(10) << "searching osd." << logevt.from
- << " log for unfound items" << dendl;
+ dout(10) << "Noting missing from osd." << logevt.from << dendl;
PG *pg = context< RecoveryMachine >().pg;
- bool got_missing = pg->search_for_missing(logevt.msg->info,
- &logevt.msg->missing, logevt.from);
-
- // hmm.. should we?
- (void)got_missing;
- //if (got_missing)
- //pg->osd->queue_for_recovery(pg);
-
+ pg->peer_missing[logevt.from].swap(logevt.msg->missing);
+ pg->peer_info[logevt.from] = logevt.msg->info;
return discard_event();
}
@@ -6772,13 +7196,15 @@ void PG::RecoveryState::RecoveryMachine::log_exit(const char *state_name, utime_
#undef dout_prefix
#define dout_prefix (*_dout << (debug_pg ? debug_pg->gen_prefix() : string()) << " PriorSet: ")
-PG::PriorSet::PriorSet(const OSDMap &osdmap,
+PG::PriorSet::PriorSet(bool ec_pool,
+ PGBackend::IsRecoverablePredicate *c,
+ const OSDMap &osdmap,
const map<epoch_t, pg_interval_t> &past_intervals,
const vector<int> &up,
const vector<int> &acting,
const pg_info_t &info,
const PG *debug_pg)
- : pg_down(false)
+ : ec_pool(ec_pool), pg_down(false), pcontdec(c)
{
/*
* We have to be careful to gracefully deal with situations like
@@ -6827,12 +7253,16 @@ PG::PriorSet::PriorSet(const OSDMap &osdmap,
// but because we want their pg_info to inform choose_acting(), and
// so that we know what they do/do not have explicitly before
// sending them any new info/logs/whatever.
- for (unsigned i=0; i<acting.size(); i++)
- probe.insert(acting[i]);
+ for (unsigned i=0; i<acting.size(); i++) {
+ if (acting[i] != CRUSH_ITEM_NONE)
+ probe.insert(pg_shard_t(acting[i], ec_pool ? i : ghobject_t::NO_SHARD));
+ }
// It may be possible to exlude the up nodes, but let's keep them in
// there for now.
- for (unsigned i=0; i<up.size(); i++)
- probe.insert(up[i]);
+ for (unsigned i=0; i<up.size(); i++) {
+ if (up[i] != CRUSH_ITEM_NONE)
+ probe.insert(pg_shard_t(up[i], ec_pool ? i : ghobject_t::NO_SHARD));
+ }
for (map<epoch_t,pg_interval_t>::const_reverse_iterator p = past_intervals.rbegin();
p != past_intervals.rend();
@@ -6852,12 +7282,15 @@ PG::PriorSet::PriorSet(const OSDMap &osdmap,
// look at candidate osds during this interval. each falls into
// one of three categories: up, down (but potentially
// interesting), or lost (down, but we won't wait for it).
- bool any_up_now = false; // any candidates up now
+ set<pg_shard_t> up_now;
bool any_down_now = false; // any candidates down now (that might have useful data)
// consider ACTING osds
for (unsigned i=0; i<interval.acting.size(); i++) {
int o = interval.acting[i];
+ if (o == CRUSH_ITEM_NONE)
+ continue;
+ pg_shard_t so(o, ec_pool ? i : ghobject_t::NO_SHARD);
const osd_info_t *pinfo = 0;
if (osdmap.exists(o))
@@ -6865,8 +7298,8 @@ PG::PriorSet::PriorSet(const OSDMap &osdmap,
if (osdmap.is_up(o)) {
// include past acting osds if they are up.
- probe.insert(o);
- any_up_now = true;
+ probe.insert(so);
+ up_now.insert(so);
} else if (!pinfo) {
dout(10) << "build_prior prior osd." << o << " no longer exists" << dendl;
down.insert(o);
@@ -6880,18 +7313,18 @@ PG::PriorSet::PriorSet(const OSDMap &osdmap,
}
}
- // if nobody survived this interval, and we may have gone rw,
+ // if not enough osds survived this interval, and we may have gone rw,
// then we need to wait for one of those osds to recover to
// ensure that we haven't lost any information.
- if (!any_up_now && any_down_now) {
+ if (!(*pcontdec)(up_now) && any_down_now) {
// fixme: how do we identify a "clean" shutdown anyway?
- dout(10) << "build_prior possibly went active+rw, none up; including down osds" << dendl;
+ dout(10) << "build_prior possibly went active+rw, insufficient up;"
+ << " including down osds" << dendl;
for (vector<int>::const_iterator i = interval.acting.begin();
i != interval.acting.end();
++i) {
if (osdmap.exists(*i) && // if it doesn't exist, we already consider it lost.
osdmap.is_down(*i)) {
- probe.insert(*i);
pg_down = true;
// make note of when any down osd in the cur set was lost, so that
@@ -6912,10 +7345,10 @@ PG::PriorSet::PriorSet(const OSDMap &osdmap,
// true if the given map affects the prior set
bool PG::PriorSet::affected_by_map(const OSDMapRef osdmap, const PG *debug_pg) const
{
- for (set<int>::iterator p = probe.begin();
+ for (set<pg_shard_t>::iterator p = probe.begin();
p != probe.end();
++p) {
- int o = *p;
+ int o = p->osd;
// did someone in the prior set go down?
if (osdmap->is_down(o) && down.count(o) == 0) {
@@ -6924,7 +7357,7 @@ bool PG::PriorSet::affected_by_map(const OSDMapRef osdmap, const PG *debug_pg) c
}
// did a down osd in cur get (re)marked as lost?
- map<int,epoch_t>::const_iterator r = blocked_by.find(o);
+ map<int, epoch_t>::const_iterator r = blocked_by.find(o);
if (r != blocked_by.end()) {
if (!osdmap->exists(o)) {
dout(10) << "affected_by_map osd." << o << " no longer exists" << dendl;
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 23c84fd..4c1b2cf 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -276,6 +276,9 @@ public:
bool dirty_info, dirty_big_info;
public:
+ bool is_ec_pg() const {
+ return pool.info.ec_pool();
+ }
// pg state
pg_info_t info;
__u8 info_struct_v;
@@ -289,19 +292,125 @@ public:
const coll_t coll;
PGLog pg_log;
- static string get_info_key(pg_t pgid) {
+ static string get_info_key(spg_t pgid) {
return stringify(pgid) + "_info";
}
- static string get_biginfo_key(pg_t pgid) {
+ static string get_biginfo_key(spg_t pgid) {
return stringify(pgid) + "_biginfo";
}
- static string get_epoch_key(pg_t pgid) {
+ static string get_epoch_key(spg_t pgid) {
return stringify(pgid) + "_epoch";
}
hobject_t log_oid;
hobject_t biginfo_oid;
- map<hobject_t, set<int> > missing_loc;
- set<int> missing_loc_sources; // superset of missing_loc locations
+
+ class MissingLoc {
+ map<hobject_t, pg_missing_t::item> needs_recovery_map;
+ map<hobject_t, set<pg_shard_t> > missing_loc;
+ set<pg_shard_t> missing_loc_sources;
+ PG *pg;
+ boost::scoped_ptr<PGBackend::IsReadablePredicate> is_readable;
+ boost::scoped_ptr<PGBackend::IsRecoverablePredicate> is_recoverable;
+ set<pg_shard_t> empty_set;
+ public:
+ MissingLoc(PG *pg)
+ : pg(pg) {}
+ void set_backend_predicates(
+ PGBackend::IsReadablePredicate *_is_readable,
+ PGBackend::IsRecoverablePredicate *_is_recoverable) {
+ is_readable.reset(_is_readable);
+ is_recoverable.reset(_is_recoverable);
+ }
+ string gen_prefix() const { return pg->gen_prefix(); }
+ bool needs_recovery(
+ const hobject_t &hoid,
+ eversion_t *v = 0) const {
+ map<hobject_t, pg_missing_t::item>::const_iterator i =
+ needs_recovery_map.find(hoid);
+ if (i == needs_recovery_map.end())
+ return false;
+ if (v)
+ *v = i->second.need;
+ return true;
+ }
+ bool is_unfound(const hobject_t &hoid) const {
+ return needs_recovery(hoid) && (
+ !missing_loc.count(hoid) ||
+ !(*is_recoverable)(missing_loc.find(hoid)->second));
+ }
+ bool readable_with_acting(
+ const hobject_t &hoid,
+ const set<pg_shard_t> &acting) const;
+ uint64_t num_unfound() const {
+ uint64_t ret = 0;
+ for (map<hobject_t, pg_missing_t::item>::const_iterator i =
+ needs_recovery_map.begin();
+ i != needs_recovery_map.end();
+ ++i) {
+ if (is_unfound(i->first))
+ ++ret;
+ }
+ return ret;
+ }
+
+ void clear() {
+ needs_recovery_map.clear();
+ missing_loc.clear();
+ missing_loc_sources.clear();
+ }
+
+ void add_location(const hobject_t &hoid, pg_shard_t location) {
+ missing_loc[hoid].insert(location);
+ }
+ void remove_location(const hobject_t &hoid, pg_shard_t location) {
+ missing_loc[hoid].erase(location);
+ }
+ void add_active_missing(const pg_missing_t &missing) {
+ for (map<hobject_t, pg_missing_t::item>::const_iterator i =
+ missing.missing.begin();
+ i != missing.missing.end();
+ ++i) {
+ map<hobject_t, pg_missing_t::item>::const_iterator j =
+ needs_recovery_map.find(i->first);
+ if (j == needs_recovery_map.end()) {
+ needs_recovery_map.insert(*i);
+ } else {
+ assert(i->second.need == j->second.need);
+ }
+ }
+ }
+ void revise_need(const hobject_t &hoid, eversion_t need) {
+ assert(needs_recovery(hoid));
+ needs_recovery_map[hoid].need = need;
+ }
+
+ /// Adds info about a possible recovery source
+ bool add_source_info(
+ pg_shard_t source, ///< [in] source
+ const pg_info_t &oinfo, ///< [in] info
+ const pg_missing_t &omissing ///< [in] (optional) missing
+ ); ///< @return whether a new object location was discovered
+
+ /// Uses osdmap to update structures for now down sources
+ void check_recovery_sources(const OSDMapRef osdmap);
+
+ /// Call when hoid is no longer missing in acting set
+ void recovered(const hobject_t &hoid) {
+ needs_recovery_map.erase(hoid);
+ missing_loc.erase(hoid);
+ }
+
+ const set<pg_shard_t> &get_locations(const hobject_t &hoid) const {
+ return missing_loc.count(hoid) ?
+ missing_loc.find(hoid)->second : empty_set;
+ }
+ const map<hobject_t, set<pg_shard_t> > &get_missing_locs() const {
+ return missing_loc;
+ }
+ const map<hobject_t, pg_missing_t::item> &get_needs_recovery() const {
+ return needs_recovery_map;
+ }
+ } missing_loc;
interval_set<snapid_t> snap_collections; // obsolete
map<epoch_t,pg_interval_t> past_intervals;
@@ -312,7 +421,7 @@ public:
* (if they have one) */
xlist<PG*>::item recovery_item, scrub_item, scrub_finalize_item, snap_trim_item, stat_queue_item;
int recovery_ops_active;
- set<int> waiting_on_backfill;
+ set<pg_shard_t> waiting_on_backfill;
#ifdef DEBUG_RECOVERY_OIDS
set<hobject_t> recovering_oids;
#endif
@@ -332,20 +441,28 @@ public:
// primary state
public:
- vector<int> up, acting, want_acting, actingbackfill;
- map<int,eversion_t> peer_last_complete_ondisk;
+ pg_shard_t primary;
+ pg_shard_t pg_whoami;
+ pg_shard_t up_primary;
+ vector<int> up, acting, want_acting;
+ set<pg_shard_t> actingbackfill, actingset;
+ map<pg_shard_t,eversion_t> peer_last_complete_ondisk;
eversion_t min_last_complete_ondisk; // up: min over last_complete_ondisk, peer_last_complete_ondisk
eversion_t pg_trim_to;
// [primary only] content recovery state
protected:
struct PriorSet {
- set<int> probe; /// current+prior OSDs we need to probe.
+ const bool ec_pool;
+ set<pg_shard_t> probe; /// current+prior OSDs we need to probe.
set<int> down; /// down osds that would normally be in @a probe and might be interesting.
- map<int,epoch_t> blocked_by; /// current lost_at values for any OSDs in cur set for which (re)marking them lost would affect cur set
+ map<int, epoch_t> blocked_by; /// current lost_at values for any OSDs in cur set for which (re)marking them lost would affect cur set
bool pg_down; /// some down osds are included in @a cur; the DOWN pg state bit should be set.
- PriorSet(const OSDMap &osdmap,
+ boost::scoped_ptr<PGBackend::IsRecoverablePredicate> pcontdec;
+ PriorSet(bool ec_pool,
+ PGBackend::IsRecoverablePredicate *c,
+ const OSDMap &osdmap,
const map<epoch_t, pg_interval_t> &past_intervals,
const vector<int> &up,
const vector<int> &acting,
@@ -364,15 +481,17 @@ public:
public:
struct RecoveryCtx {
utime_t start_time;
- map< int, map<pg_t, pg_query_t> > *query_map;
- map< int, vector<pair<pg_notify_t, pg_interval_map_t> > > *info_map;
- map< int, vector<pair<pg_notify_t, pg_interval_map_t> > > *notify_list;
+ map<int, map<spg_t, pg_query_t> > *query_map;
+ map<int, vector<pair<pg_notify_t, pg_interval_map_t> > > *info_map;
+ map<int, vector<pair<pg_notify_t, pg_interval_map_t> > > *notify_list;
C_Contexts *on_applied;
C_Contexts *on_safe;
ObjectStore::Transaction *transaction;
- RecoveryCtx(map< int, map<pg_t, pg_query_t> > *query_map,
- map< int, vector<pair<pg_notify_t, pg_interval_map_t> > > *info_map,
- map< int, vector<pair<pg_notify_t, pg_interval_map_t> > > *notify_list,
+ RecoveryCtx(map<int, map<spg_t, pg_query_t> > *query_map,
+ map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > > *info_map,
+ map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > > *notify_list,
C_Contexts *on_applied,
C_Contexts *on_safe,
ObjectStore::Transaction *transaction)
@@ -403,24 +522,26 @@ protected:
*/
bool need_up_thru;
- set<int> stray_set; // non-acting osds that have PG data.
+ set<pg_shard_t> stray_set; // non-acting osds that have PG data.
eversion_t oldest_update; // acting: lowest (valid) last_update in active set
- map<int,pg_info_t> peer_info; // info from peers (stray or prior)
- set<int> peer_purged; // peers purged
- map<int,pg_missing_t> peer_missing;
- set<int> peer_log_requested; // logs i've requested (and start stamps)
- set<int> peer_missing_requested;
- set<int> stray_purged; // i deleted these strays; ignore racing PGInfo from them
- set<int> peer_activated;
+ map<pg_shard_t, pg_info_t> peer_info; // info from peers (stray or prior)
+ set<pg_shard_t> peer_purged; // peers purged
+ map<pg_shard_t, pg_missing_t> peer_missing;
+ set<pg_shard_t> peer_log_requested; // logs i've requested (and start stamps)
+ set<pg_shard_t> peer_missing_requested;
+
+ // i deleted these strays; ignore racing PGInfo from them
+ set<pg_shard_t> stray_purged;
+ set<pg_shard_t> peer_activated;
// primary-only, recovery-only state
- set<int> might_have_unfound; // These osds might have objects on them
- // which are unfound on the primary
+ set<pg_shard_t> might_have_unfound; // These osds might have objects on them
+ // which are unfound on the primary
epoch_t last_peering_reset;
/* heartbeat peers */
- void set_probe_targets(const set<int> &probe_set);
+ void set_probe_targets(const set<pg_shard_t> &probe_set);
void clear_probe_targets();
public:
Mutex heartbeat_peer_lock;
@@ -505,21 +626,17 @@ protected:
};
BackfillInterval backfill_info;
- map<int, BackfillInterval> peer_backfill_info;
+ map<pg_shard_t, BackfillInterval> peer_backfill_info;
bool backfill_reserved;
bool backfill_reserving;
friend class OSD;
public:
- vector<int> backfill_targets;
+ set<pg_shard_t> backfill_targets;
- bool is_backfill_targets(int osd) {
- if (std::find(backfill_targets.begin(), backfill_targets.end(), osd)
- != backfill_targets.end())
- return true;
- else
- return false;
+ bool is_backfill_targets(pg_shard_t osd) {
+ return backfill_targets.count(osd);
}
protected:
@@ -530,8 +647,9 @@ protected:
// Ops waiting on backfill_pos to change
list<OpRequestRef> waiting_for_active;
+ list<OpRequestRef> waiting_for_cache_not_full;
list<OpRequestRef> waiting_for_all_missing;
- map<hobject_t, list<OpRequestRef> > waiting_for_missing_object,
+ map<hobject_t, list<OpRequestRef> > waiting_for_unreadable_object,
waiting_for_degraded_object,
waiting_for_blocked_object;
// Callbacks should assume pg (and nothing else) is locked
@@ -560,23 +678,25 @@ protected:
void clear_publish_stats();
public:
- void clear_primary_state();
+ void clear_primary_state(bool stay_primary);
public:
- bool is_acting(int osd) const {
- for (unsigned i=0; i<acting.size(); i++)
- if (acting[i] == osd) return true;
- return false;
+ bool is_actingbackfill(pg_shard_t osd) const {
+ return actingbackfill.count(osd);
}
- bool is_up(int osd) const {
- for (unsigned i=0; i<up.size(); i++)
- if (up[i] == osd) return true;
- return false;
+ bool is_acting(pg_shard_t osd) const {
+ if (pool.info.ec_pool()) {
+ return acting.size() > osd.shard && acting[osd.shard] == osd.osd;
+ } else {
+ return std::find(acting.begin(), acting.end(), osd.osd) != acting.end();
+ }
}
- bool is_actingbackfill(int osd) const {
- for (unsigned i=0; i<actingbackfill.size(); i++)
- if (actingbackfill[i] == osd) return true;
- return false;
+ bool is_up(pg_shard_t osd) const {
+ if (pool.info.ec_pool()) {
+ return up.size() > osd.shard && up[osd.shard] == osd.osd;
+ } else {
+ return std::find(up.begin(), up.end(), osd.osd) != up.end();
+ }
}
bool needs_recovery() const;
@@ -600,10 +720,13 @@ public:
bool calc_min_last_complete_ondisk() {
eversion_t min = last_complete_ondisk;
assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++) {
- if (peer_last_complete_ondisk.count(actingbackfill[i]) == 0)
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ if (peer_last_complete_ondisk.count(*i) == 0)
return false; // we don't have complete info
- eversion_t a = peer_last_complete_ondisk[actingbackfill[i]];
+ eversion_t a = peer_last_complete_ondisk[*i];
if (a < min)
min = a;
}
@@ -616,10 +739,10 @@ public:
virtual void calc_trim_to() = 0;
void proc_replica_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
- pg_missing_t& omissing, int from);
+ pg_missing_t& omissing, pg_shard_t from);
void proc_master_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
- pg_missing_t& omissing, int from);
- bool proc_replica_info(int from, const pg_info_t &info);
+ pg_missing_t& omissing, pg_shard_t from);
+ bool proc_replica_info(pg_shard_t from, const pg_info_t &info);
struct LogEntryTrimmer : public ObjectModDesc::Visitor {
@@ -653,8 +776,7 @@ public:
};
struct PGLogEntryHandler : public PGLog::LogEntryHandler {
- map<hobject_t, list<pg_log_entry_t> > to_rollback;
- set<hobject_t> cannot_rollback;
+ list<pg_log_entry_t> to_rollback;
set<hobject_t> to_remove;
list<pg_log_entry_t> to_trim;
@@ -663,30 +785,20 @@ public:
to_remove.insert(hoid);
}
void rollback(const pg_log_entry_t &entry) {
- assert(!cannot_rollback.count(entry.soid));
- to_rollback[entry.soid].push_back(entry);
- }
- void cant_rollback(const pg_log_entry_t &entry) {
- to_rollback.erase(entry.soid);
- cannot_rollback.insert(entry.soid);
+ to_rollback.push_back(entry);
}
void trim(const pg_log_entry_t &entry) {
to_trim.push_back(entry);
}
void apply(PG *pg, ObjectStore::Transaction *t) {
- for (map<hobject_t, list<pg_log_entry_t> >::iterator i =
- to_rollback.begin();
- i != to_rollback.end();
- ++i) {
- for (list<pg_log_entry_t>::reverse_iterator j = i->second.rbegin();
- j != i->second.rend();
- ++j) {
- assert(j->mod_desc.can_rollback());
- pg->get_pgbackend()->rollback(j->soid, j->mod_desc, t);
- SnapRollBacker rollbacker(j->soid, pg, t);
- j->mod_desc.visit(&rollbacker);
- }
+ for (list<pg_log_entry_t>::iterator j = to_rollback.begin();
+ j != to_rollback.end();
+ ++j) {
+ assert(j->mod_desc.can_rollback());
+ pg->get_pgbackend()->rollback(j->soid, j->mod_desc, t);
+ SnapRollBacker rollbacker(j->soid, pg, t);
+ j->mod_desc.visit(&rollbacker);
}
for (set<hobject_t>::iterator i = to_remove.begin();
i != to_remove.end();
@@ -713,38 +825,73 @@ public:
ObjectStore::Transaction *t, const hobject_t &soid);
void remove_snap_mapped_object(
ObjectStore::Transaction& t, const hobject_t& soid);
- void merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from);
+ void merge_log(
+ ObjectStore::Transaction& t, pg_info_t &oinfo,
+ pg_log_t &olog, pg_shard_t from);
void rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead);
- bool search_for_missing(const pg_info_t &oinfo, const pg_missing_t *omissing,
- int fromosd);
+ bool search_for_missing(
+ const pg_info_t &oinfo, const pg_missing_t &omissing,
+ pg_shard_t fromosd,
+ RecoveryCtx*);
void check_for_lost_objects();
void forget_lost_objects();
- void discover_all_missing(std::map< int, map<pg_t,pg_query_t> > &query_map);
+ void discover_all_missing(std::map<int, map<spg_t,pg_query_t> > &query_map);
void trim_write_ahead();
- map<int, pg_info_t>::const_iterator find_best_info(const map<int, pg_info_t> &infos) const;
- bool calc_acting(int& newest_update_osd, vector<int>& want, vector<int>& backfill) const;
- bool choose_acting(int& newest_update_osd);
+ map<pg_shard_t, pg_info_t>::const_iterator find_best_info(
+ const map<pg_shard_t, pg_info_t> &infos) const;
+ static void calc_ec_acting(
+ map<pg_shard_t, pg_info_t>::const_iterator auth_log_shard,
+ unsigned size,
+ const vector<int> &acting,
+ pg_shard_t acting_primary,
+ const vector<int> &up,
+ pg_shard_t up_primary,
+ const map<pg_shard_t, pg_info_t> &all_info,
+ bool compat_mode,
+ vector<int> *want,
+ set<pg_shard_t> *backfill,
+ set<pg_shard_t> *acting_backfill,
+ pg_shard_t *want_primary,
+ ostream &ss);
+ static void calc_replicated_acting(
+ map<pg_shard_t, pg_info_t>::const_iterator auth_log_shard,
+ unsigned size,
+ const vector<int> &acting,
+ pg_shard_t acting_primary,
+ const vector<int> &up,
+ pg_shard_t up_primary,
+ const map<pg_shard_t, pg_info_t> &all_info,
+ bool compat_mode,
+ vector<int> *want,
+ set<pg_shard_t> *backfill,
+ set<pg_shard_t> *acting_backfill,
+ pg_shard_t *want_primary,
+ ostream &ss);
+ bool choose_acting(pg_shard_t &auth_log_shard);
void build_might_have_unfound();
void replay_queued_ops();
- void activate(ObjectStore::Transaction& t,
- epoch_t query_epoch,
- list<Context*>& tfin,
- map< int, map<pg_t,pg_query_t> >& query_map,
- map<int, vector<pair<pg_notify_t, pg_interval_map_t> > > *activator_map=0);
+ void activate(
+ ObjectStore::Transaction& t,
+ epoch_t query_epoch,
+ list<Context*>& tfin,
+ map<int, map<spg_t,pg_query_t> >& query_map,
+ map<int,
+ vector<pair<pg_notify_t, pg_interval_map_t> > > *activator_map,
+ RecoveryCtx *ctx);
void _activate_committed(epoch_t e);
void all_activated_and_committed();
void proc_primary_info(ObjectStore::Transaction &t, const pg_info_t &info);
bool have_unfound() const {
- return pg_log.get_missing().num_missing() > missing_loc.size();
+ return missing_loc.num_unfound();
}
int get_num_unfound() const {
- return pg_log.get_missing().num_missing() - missing_loc.size();
+ return missing_loc.num_unfound();
}
virtual void check_local() = 0;
@@ -799,7 +946,7 @@ public:
}
// metadata
- set<int> reserved_peers;
+ set<pg_shard_t> reserved_peers;
bool reserved, reserve_failed;
epoch_t epoch_start;
@@ -808,12 +955,12 @@ public:
bool active;
bool queue_snap_trim;
int waiting_on;
- set<int> waiting_on_whom;
+ set<pg_shard_t> waiting_on_whom;
int shallow_errors;
int deep_errors;
int fixed;
ScrubMap primary_scrubmap;
- map<int,ScrubMap> received_maps;
+ map<pg_shard_t, ScrubMap> received_maps;
MOSDRepScrub *active_rep_scrub;
utime_t scrub_reg_stamp; // stamp we registered for
@@ -821,12 +968,12 @@ public:
bool must_scrub, must_deep_scrub, must_repair;
// Maps from objects with errors to missing/inconsistent peers
- map<hobject_t, set<int> > missing;
- map<hobject_t, set<int> > inconsistent;
- map<hobject_t, set<int> > inconsistent_snapcolls;
+ map<hobject_t, set<pg_shard_t> > missing;
+ map<hobject_t, set<pg_shard_t> > inconsistent;
+ map<hobject_t, set<pg_shard_t> > inconsistent_snapcolls;
// Map from object with errors to good peer
- map<hobject_t, pair<ScrubMap::object, int> > authoritative;
+ map<hobject_t, pair<ScrubMap::object, pg_shard_t> > authoritative;
// classic scrub
bool classic;
@@ -938,7 +1085,11 @@ public:
int active_pushes;
- void repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer);
+ void repair_object(
+ const hobject_t& soid, ScrubMap::object *po,
+ pg_shard_t bad_peer,
+ pg_shard_t ok_peer);
+
void scrub(ThreadPool::TPHandle &handle);
void classic_scrub(ThreadPool::TPHandle &handle);
void chunky_scrub(ThreadPool::TPHandle &handle);
@@ -949,8 +1100,8 @@ public:
void scrub_clear_state();
bool scrub_gather_replica_maps();
void _scan_snaps(ScrubMap &map);
- void _request_scrub_map_classic(int replica, eversion_t version);
- void _request_scrub_map(int replica, eversion_t version,
+ void _request_scrub_map_classic(pg_shard_t replica, eversion_t version);
+ void _request_scrub_map(pg_shard_t replica, eversion_t version,
hobject_t start, hobject_t end, bool deep);
int build_scrub_map_chunk(
ScrubMap &map,
@@ -964,14 +1115,14 @@ public:
virtual void _scrub_finish() { }
virtual void get_colls(list<coll_t> *out) = 0;
virtual void split_colls(
- pg_t child,
+ spg_t child,
int split_bits,
int seed,
ObjectStore::Transaction *t) = 0;
virtual bool _report_snap_collection_errors(
const hobject_t &hoid,
const map<string, bufferptr> &attrs,
- int osd,
+ pg_shard_t osd,
ostream &out) { return false; };
void clear_scrub_reserved();
void scrub_reserve_replicas();
@@ -1049,10 +1200,10 @@ public:
};
struct MInfoRec : boost::statechart::event< MInfoRec > {
- int from;
+ pg_shard_t from;
pg_info_t info;
epoch_t msg_epoch;
- MInfoRec(int from, pg_info_t &info, epoch_t msg_epoch) :
+ MInfoRec(pg_shard_t from, pg_info_t &info, epoch_t msg_epoch) :
from(from), info(info), msg_epoch(msg_epoch) {}
void print(std::ostream *out) const {
*out << "MInfoRec from " << from << " info: " << info;
@@ -1060,9 +1211,9 @@ public:
};
struct MLogRec : boost::statechart::event< MLogRec > {
- int from;
+ pg_shard_t from;
boost::intrusive_ptr<MOSDPGLog> msg;
- MLogRec(int from, MOSDPGLog *msg) :
+ MLogRec(pg_shard_t from, MOSDPGLog *msg) :
from(from), msg(msg) {}
void print(std::ostream *out) const {
*out << "MLogRec from " << from;
@@ -1070,9 +1221,9 @@ public:
};
struct MNotifyRec : boost::statechart::event< MNotifyRec > {
- int from;
+ pg_shard_t from;
pg_notify_t notify;
- MNotifyRec(int from, pg_notify_t ¬ify) :
+ MNotifyRec(pg_shard_t from, pg_notify_t ¬ify) :
from(from), notify(notify) {}
void print(std::ostream *out) const {
*out << "MNotifyRec from " << from << " notify: " << notify;
@@ -1080,10 +1231,10 @@ public:
};
struct MQuery : boost::statechart::event< MQuery > {
- int from;
+ pg_shard_t from;
pg_query_t query;
epoch_t query_epoch;
- MQuery(int from, const pg_query_t &query, epoch_t query_epoch):
+ MQuery(pg_shard_t from, const pg_query_t &query, epoch_t query_epoch):
from(from), query(query), query_epoch(query_epoch) {}
void print(std::ostream *out) const {
*out << "MQuery from " << from
@@ -1096,8 +1247,16 @@ public:
OSDMapRef osdmap;
OSDMapRef lastmap;
vector<int> newup, newacting;
- AdvMap(OSDMapRef osdmap, OSDMapRef lastmap, vector<int>& newup, vector<int>& newacting):
- osdmap(osdmap), lastmap(lastmap), newup(newup), newacting(newacting) {}
+ int up_primary, acting_primary;
+ AdvMap(
+ OSDMapRef osdmap, OSDMapRef lastmap,
+ vector<int>& newup, int up_primary,
+ vector<int>& newacting, int acting_primary):
+ osdmap(osdmap), lastmap(lastmap),
+ newup(newup),
+ newacting(newacting),
+ up_primary(up_primary),
+ acting_primary(acting_primary) {}
void print(std::ostream *out) const {
*out << "AdvMap";
}
@@ -1190,12 +1349,13 @@ public:
return state->rctx->transaction;
}
- void send_query(int to, const pg_query_t &query) {
+ void send_query(pg_shard_t to, const pg_query_t &query) {
assert(state->rctx->query_map);
- (*state->rctx->query_map)[to][pg->info.pgid] = query;
+ (*state->rctx->query_map)[to.osd][spg_t(pg->info.pgid.pgid, to.shard)] =
+ query;
}
- map<int, map<pg_t, pg_query_t> > *get_query_map() {
+ map<int, map<spg_t, pg_query_t> > *get_query_map() {
assert(state->rctx->query_map);
return state->rctx->query_map;
}
@@ -1215,9 +1375,12 @@ public:
return &(state->rctx->on_applied->contexts);
}
- void send_notify(int to, const pg_notify_t &info, const pg_interval_map_t &pi) {
+ RecoveryCtx *get_recovery_ctx() { return state->rctx; }
+
+ void send_notify(pg_shard_t to,
+ const pg_notify_t &info, const pg_interval_map_t &pi) {
assert(state->rctx->notify_list);
- (*state->rctx->notify_list)[to].push_back(make_pair(info, pi));
+ (*state->rctx->notify_list)[to.osd].push_back(make_pair(info, pi));
}
};
friend class RecoveryMachine;
@@ -1329,10 +1492,8 @@ public:
typedef boost::mpl::list <
boost::statechart::custom_reaction< ActMap >,
boost::statechart::custom_reaction< MNotifyRec >,
- boost::statechart::transition< NeedActingChange, WaitActingChange >,
- boost::statechart::custom_reaction< AdvMap>
+ boost::statechart::transition< NeedActingChange, WaitActingChange >
> reactions;
- boost::statechart::result react(const AdvMap&);
boost::statechart::result react(const ActMap&);
boost::statechart::result react(const MNotifyRec&);
};
@@ -1379,8 +1540,8 @@ public:
Active(my_context ctx);
void exit();
- const set<int> sorted_acting_set;
- const set<int> sorted_backfill_set;
+ const set<pg_shard_t> sorted_actingbackfill_set;
+ const set<pg_shard_t> sorted_backfill_set;
bool all_replicas_activated;
typedef boost::mpl::list <
@@ -1442,7 +1603,7 @@ public:
boost::statechart::custom_reaction< RemoteReservationRejected >,
boost::statechart::transition< AllBackfillsReserved, Backfilling >
> reactions;
- set<int>::const_iterator backfill_osd_it;
+ set<pg_shard_t>::const_iterator backfill_osd_it;
WaitRemoteBackfillReserved(my_context ctx);
void exit();
boost::statechart::result react(const RemoteBackfillReserved& evt);
@@ -1544,7 +1705,7 @@ public:
boost::statechart::custom_reaction< RemoteRecoveryReserved >,
boost::statechart::transition< AllRemotesReserved, Recovering >
> reactions;
- set<int>::const_iterator acting_osd_it;
+ set<pg_shard_t>::const_iterator acting_osd_it;
WaitRemoteRecoveryReserved(my_context ctx);
boost::statechart::result react(const RemoteRecoveryReserved &evt);
void exit();
@@ -1593,7 +1754,7 @@ public:
struct GetLog;
struct GetInfo : boost::statechart::state< GetInfo, Peering >, NamedState {
- set<int> peer_info_requested;
+ set<pg_shard_t> peer_info_requested;
GetInfo(my_context ctx);
void exit();
@@ -1614,7 +1775,7 @@ public:
};
struct GetLog : boost::statechart::state< GetLog, Peering >, NamedState {
- int newest_update_osd;
+ pg_shard_t auth_log_shard;
boost::intrusive_ptr<MOSDPGLog> msg;
GetLog(my_context ctx);
@@ -1634,10 +1795,9 @@ public:
};
struct WaitUpThru;
- struct WaitFlushedPeering;
struct GetMissing : boost::statechart::state< GetMissing, Peering >, NamedState {
- set<int> peer_missing_requested;
+ set<pg_shard_t> peer_missing_requested;
GetMissing(my_context ctx);
void exit();
@@ -1645,25 +1805,12 @@ public:
typedef boost::mpl::list <
boost::statechart::custom_reaction< QueryState >,
boost::statechart::custom_reaction< MLogRec >,
- boost::statechart::transition< NeedUpThru, WaitUpThru >,
- boost::statechart::transition< CheckRepops, WaitFlushedPeering>
+ boost::statechart::transition< NeedUpThru, WaitUpThru >
> reactions;
boost::statechart::result react(const QueryState& q);
boost::statechart::result react(const MLogRec& logevt);
};
- struct WaitFlushedPeering :
- boost::statechart::state< WaitFlushedPeering, Peering>, NamedState {
- WaitFlushedPeering(my_context ctx);
- void exit() {}
- typedef boost::mpl::list <
- boost::statechart::custom_reaction< QueryState >,
- boost::statechart::custom_reaction< FlushedEvt >
- > reactions;
- boost::statechart::result react(const FlushedEvt& evt);
- boost::statechart::result react(const QueryState& q);
- };
-
struct WaitUpThru : boost::statechart::state< WaitUpThru, Peering >, NamedState {
WaitUpThru(my_context ctx);
void exit();
@@ -1671,7 +1818,6 @@ public:
typedef boost::mpl::list <
boost::statechart::custom_reaction< QueryState >,
boost::statechart::custom_reaction< ActMap >,
- boost::statechart::transition< CheckRepops, WaitFlushedPeering>,
boost::statechart::custom_reaction< MLogRec >
> reactions;
boost::statechart::result react(const QueryState& q);
@@ -1719,7 +1865,7 @@ public:
public:
PG(OSDService *o, OSDMapRef curmap,
- const PGPool &pool, pg_t p, const hobject_t& loid, const hobject_t& ioid);
+ const PGPool &pool, spg_t p, const hobject_t& loid, const hobject_t& ioid);
virtual ~PG();
private:
@@ -1728,15 +1874,52 @@ public:
PG& operator=(const PG& rhs);
public:
- pg_t get_pgid() const { return info.pgid; }
+ spg_t get_pgid() const { return info.pgid; }
int get_nrep() const { return acting.size(); }
- int get_primary() { return acting.empty() ? -1:acting[0]; }
+ void init_primary_up_acting(
+ const vector<int> &newup,
+ const vector<int> &newacting,
+ int new_up_primary,
+ int new_acting_primary) {
+ actingset.clear();
+ acting = newacting;
+ for (shard_id_t i = 0; i < acting.size(); ++i) {
+ if (acting[i] != CRUSH_ITEM_NONE)
+ actingset.insert(
+ pg_shard_t(
+ acting[i],
+ pool.info.ec_pool() ? i : ghobject_t::NO_SHARD));
+ }
+ up = newup;
+ if (!pool.info.ec_pool()) {
+ up_primary = pg_shard_t(new_up_primary, ghobject_t::no_shard());
+ primary = pg_shard_t(new_acting_primary, ghobject_t::no_shard());
+ return;
+ }
+ up_primary = pg_shard_t();
+ primary = pg_shard_t();
+ for (shard_id_t i = 0; i < up.size(); ++i) {
+ if (up[i] == new_up_primary) {
+ up_primary = pg_shard_t(up[i], i);
+ break;
+ }
+ }
+ for (shard_id_t i = 0; i < acting.size(); ++i) {
+ if (acting[i] == new_acting_primary) {
+ primary = pg_shard_t(acting[i], i);
+ break;
+ }
+ }
+ assert(up_primary.osd == new_up_primary);
+ assert(primary.osd == new_acting_primary);
+ }
+ pg_shard_t get_primary() const { return primary; }
int get_role() const { return role; }
void set_role(int r) { role = r; }
- bool is_primary() const { return role == 0; }
+ bool is_primary() const { return pg_whoami == primary; }
bool is_replica() const { return role > 0; }
epoch_t get_last_peering_reset() const { return last_peering_reset; }
@@ -1764,7 +1947,9 @@ public:
void init(
int role,
vector<int>& up,
+ int up_primary,
vector<int>& acting,
+ int acting_primary,
pg_history_t& history,
pg_interval_map_t& pim,
bool backfill,
@@ -1825,10 +2010,11 @@ public:
/// share new pg log entries after a pg is active
void share_pg_log();
- void start_peering_interval(const OSDMapRef lastmap,
- const vector<int>& newup,
- const vector<int>& newacting,
- ObjectStore::Transaction *t);
+ void start_peering_interval(
+ const OSDMapRef lastmap,
+ const vector<int>& newup, int up_primary,
+ const vector<int>& newacting, int acting_primary,
+ ObjectStore::Transaction *t);
void start_flush(ObjectStore::Transaction *t,
list<Context *> *on_applied,
list<Context *> *on_safe);
@@ -1839,11 +2025,13 @@ public:
}
void update_history_from_master(pg_history_t new_history);
- void fulfill_info(int from, const pg_query_t &query,
- pair<int, pg_info_t> ¬ify_info);
- void fulfill_log(int from, const pg_query_t &query, epoch_t query_epoch);
+ void fulfill_info(pg_shard_t from, const pg_query_t &query,
+ pair<pg_shard_t, pg_info_t> ¬ify_info);
+ void fulfill_log(pg_shard_t from, const pg_query_t &query, epoch_t query_epoch);
bool is_split(OSDMapRef lastmap, OSDMapRef nextmap);
- bool acting_up_affected(const vector<int>& newup, const vector<int>& newacting);
+ bool acting_up_affected(
+ int newupprimary, int newactingprimary,
+ const vector<int>& newup, const vector<int>& newacting);
// OpRequest queueing
bool can_discard_op(OpRequestRef op);
@@ -1877,18 +2065,20 @@ public:
void queue_peering_event(CephPeeringEvtRef evt);
void handle_peering_event(CephPeeringEvtRef evt, RecoveryCtx *rctx);
void queue_notify(epoch_t msg_epoch, epoch_t query_epoch,
- int from, pg_notify_t& i);
+ pg_shard_t from, pg_notify_t& i);
void queue_info(epoch_t msg_epoch, epoch_t query_epoch,
- int from, pg_info_t& i);
- void queue_log(epoch_t msg_epoch, epoch_t query_epoch, int from,
+ pg_shard_t from, pg_info_t& i);
+ void queue_log(epoch_t msg_epoch, epoch_t query_epoch, pg_shard_t from,
MOSDPGLog *msg);
void queue_query(epoch_t msg_epoch, epoch_t query_epoch,
- int from, const pg_query_t& q);
+ pg_shard_t from, const pg_query_t& q);
void queue_null(epoch_t msg_epoch, epoch_t query_epoch);
void queue_flushed(epoch_t started_at);
- void handle_advance_map(OSDMapRef osdmap, OSDMapRef lastmap,
- vector<int>& newup, vector<int>& newacting,
- RecoveryCtx *rctx);
+ void handle_advance_map(
+ OSDMapRef osdmap, OSDMapRef lastmap,
+ vector<int>& newup, int up_primary,
+ vector<int>& newacting, int acting_primary,
+ RecoveryCtx *rctx);
void handle_activate_map(RecoveryCtx *rctx);
void handle_create(RecoveryCtx *rctx);
void handle_loaded(RecoveryCtx *rctx);
@@ -1928,6 +2118,10 @@ public:
virtual void on_shutdown() = 0;
virtual void check_blacklisted_watchers() = 0;
virtual void get_watchers(std::list<obj_watch_item_t>&) = 0;
+
+ virtual void agent_work(int max) = 0;
+ virtual void agent_stop() = 0;
+ virtual void agent_clear() = 0;
};
ostream& operator<<(ostream& out, const PG& pg);
diff --git a/src/osd/PGBackend.cc b/src/osd/PGBackend.cc
index 80e289e..19f3d99 100644
--- a/src/osd/PGBackend.cc
+++ b/src/osd/PGBackend.cc
@@ -13,7 +13,20 @@
*/
+#include "common/errno.h"
+#include "ReplicatedBackend.h"
+#include "ECBackend.h"
#include "PGBackend.h"
+#include "OSD.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, PGBackend *pgb) {
+ return *_dout << pgb->get_parent()->gen_dbg_prefix();
+}
// -- ObjectModDesc --
struct RollbackVisitor : public ObjectModDesc::Visitor {
@@ -64,3 +77,488 @@ void PGBackend::rollback(
}
+void PGBackend::on_change(ObjectStore::Transaction *t)
+{
+ dout(10) << __func__ << dendl;
+ // clear temp
+ for (set<hobject_t>::iterator i = temp_contents.begin();
+ i != temp_contents.end();
+ ++i) {
+ dout(10) << __func__ << ": Removing oid "
+ << *i << " from the temp collection" << dendl;
+ t->remove(
+ get_temp_coll(t),
+ ghobject_t(*i, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+ }
+ temp_contents.clear();
+ _on_change(t);
+}
+
+coll_t PGBackend::get_temp_coll(ObjectStore::Transaction *t)
+{
+ if (temp_created)
+ return temp_coll;
+ if (!store->collection_exists(temp_coll))
+ t->create_collection(temp_coll);
+ temp_created = true;
+ return temp_coll;
+}
+
+int PGBackend::objects_list_partial(
+ const hobject_t &begin,
+ int min,
+ int max,
+ snapid_t seq,
+ vector<hobject_t> *ls,
+ hobject_t *next)
+{
+ assert(ls);
+ ghobject_t _next(begin);
+ ls->reserve(max);
+ int r = 0;
+ while (!_next.is_max() && ls->size() < (unsigned)min) {
+ vector<ghobject_t> objects;
+ int r = store->collection_list_partial(
+ coll,
+ _next,
+ min - ls->size(),
+ max - ls->size(),
+ seq,
+ &objects,
+ &_next);
+ if (r != 0)
+ break;
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ if (i->is_no_gen()) {
+ ls->push_back(i->hobj);
+ }
+ }
+ }
+ if (r == 0)
+ *next = _next.hobj;
+ return r;
+}
+
+int PGBackend::objects_list_range(
+ const hobject_t &start,
+ const hobject_t &end,
+ snapid_t seq,
+ vector<hobject_t> *ls)
+{
+ assert(ls);
+ vector<ghobject_t> objects;
+ int r = store->collection_list_range(
+ coll,
+ start,
+ end,
+ seq,
+ &objects);
+ ls->reserve(objects.size());
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ if (i->is_no_gen()) {
+ ls->push_back(i->hobj);
+ }
+ }
+ return r;
+}
+
+int PGBackend::objects_get_attr(
+ const hobject_t &hoid,
+ const string &attr,
+ bufferlist *out)
+{
+ bufferptr bp;
+ int r = store->getattr(
+ hoid.is_temp() ? temp_coll : coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ attr.c_str(),
+ bp);
+ if (r >= 0 && out) {
+ out->clear();
+ out->push_back(bp);
+ }
+ return r;
+}
+
+int PGBackend::objects_get_attrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> *out)
+{
+ return store->getattrs(
+ hoid.is_temp() ? temp_coll : coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ *out);
+}
+
+void PGBackend::rollback_setattrs(
+ const hobject_t &hoid,
+ map<string, boost::optional<bufferlist> > &old_attrs,
+ ObjectStore::Transaction *t) {
+ map<string, bufferlist> to_set;
+ set<string> to_remove;
+ assert(!hoid.is_temp());
+ for (map<string, boost::optional<bufferlist> >::iterator i = old_attrs.begin();
+ i != old_attrs.end();
+ ++i) {
+ if (i->second) {
+ to_set[i->first] = i->second.get();
+ } else {
+ t->rmattr(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ i->first);
+ }
+ }
+ t->setattrs(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ to_set);
+}
+
+void PGBackend::rollback_append(
+ const hobject_t &hoid,
+ uint64_t old_size,
+ ObjectStore::Transaction *t) {
+ assert(!hoid.is_temp());
+ t->truncate(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ old_size);
+}
+
+void PGBackend::rollback_stash(
+ const hobject_t &hoid,
+ version_t old_version,
+ ObjectStore::Transaction *t) {
+ assert(!hoid.is_temp());
+ t->remove(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+ t->collection_move_rename(
+ coll,
+ ghobject_t(hoid, old_version, get_parent()->whoami_shard().shard),
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+}
+
+void PGBackend::rollback_create(
+ const hobject_t &hoid,
+ ObjectStore::Transaction *t) {
+ assert(!hoid.is_temp());
+ t->remove(
+ coll,
+ ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+}
+
+void PGBackend::trim_stashed_object(
+ const hobject_t &hoid,
+ version_t old_version,
+ ObjectStore::Transaction *t) {
+ assert(!hoid.is_temp());
+ t->remove(
+ coll, ghobject_t(hoid, old_version, get_parent()->whoami_shard().shard));
+}
+
+PGBackend *PGBackend::build_pg_backend(
+ const pg_pool_t &pool,
+ Listener *l,
+ coll_t coll,
+ coll_t temp_coll,
+ ObjectStore *store,
+ CephContext *cct)
+{
+ switch (pool.type) {
+ case pg_pool_t::TYPE_REPLICATED: {
+ return new ReplicatedBackend(l, coll, temp_coll, store, cct);
+ }
+ case pg_pool_t::TYPE_ERASURE: {
+ ErasureCodeInterfaceRef ec_impl;
+ assert(pool.properties.count("erasure-code-plugin"));
+ ceph::ErasureCodePluginRegistry::instance().factory(
+ pool.properties.find("erasure-code-plugin")->second,
+ pool.properties,
+ &ec_impl);
+ assert(ec_impl);
+ return new ECBackend(
+ l,
+ coll,
+ temp_coll,
+ store,
+ cct,
+ ec_impl,
+ pool.stripe_width);
+ }
+ default:
+ assert(0);
+ return NULL;
+ }
+}
+
+/*
+ * pg lock may or may not be held
+ */
+void PGBackend::be_scan_list(
+ ScrubMap &map, const vector<hobject_t> &ls, bool deep,
+ ThreadPool::TPHandle &handle)
+{
+ dout(10) << "_scan_list scanning " << ls.size() << " objects"
+ << (deep ? " deeply" : "") << dendl;
+ int i = 0;
+ for (vector<hobject_t>::const_iterator p = ls.begin();
+ p != ls.end();
+ ++p, i++) {
+ handle.reset_tp_timeout();
+ hobject_t poid = *p;
+
+ struct stat st;
+ int r = store->stat(
+ coll,
+ ghobject_t(
+ poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ &st,
+ true);
+ if (r == 0) {
+ ScrubMap::object &o = map.objects[poid];
+ o.size = st.st_size;
+ assert(!o.negative);
+ store->getattrs(
+ coll,
+ ghobject_t(
+ poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ o.attrs);
+
+ // calculate the CRC32 on deep scrubs
+ if (deep) {
+ be_deep_scrub(*p, o, handle);
+ }
+
+ dout(25) << "_scan_list " << poid << dendl;
+ } else if (r == -ENOENT) {
+ dout(25) << "_scan_list " << poid << " got " << r << ", skipping" << dendl;
+ } else if (r == -EIO) {
+ dout(25) << "_scan_list " << poid << " got " << r << ", read_error" << dendl;
+ ScrubMap::object &o = map.objects[poid];
+ o.read_error = true;
+ } else {
+ derr << "_scan_list got: " << cpp_strerror(r) << dendl;
+ assert(0);
+ }
+ }
+}
+
+enum scrub_error_type PGBackend::be_compare_scrub_objects(
+ const ScrubMap::object &auth,
+ const ScrubMap::object &candidate,
+ ostream &errorstream)
+{
+ enum scrub_error_type error = CLEAN;
+ if (candidate.read_error) {
+ // This can occur on stat() of a shallow scrub, but in that case size will
+ // be invalid, and this will be over-ridden below.
+ error = DEEP_ERROR;
+ errorstream << "candidate had a read error";
+ }
+ if (auth.digest_present && candidate.digest_present) {
+ if (auth.digest != candidate.digest) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = DEEP_ERROR;
+
+ errorstream << "digest " << candidate.digest
+ << " != known digest " << auth.digest;
+ }
+ }
+ if (auth.omap_digest_present && candidate.omap_digest_present) {
+ if (auth.omap_digest != candidate.omap_digest) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = DEEP_ERROR;
+
+ errorstream << "omap_digest " << candidate.omap_digest
+ << " != known omap_digest " << auth.omap_digest;
+ }
+ }
+ // Shallow error takes precendence because this will be seen by
+ // both types of scrubs.
+ if (auth.size != candidate.size) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "size " << candidate.size
+ << " != known size " << auth.size;
+ }
+ for (map<string,bufferptr>::const_iterator i = auth.attrs.begin();
+ i != auth.attrs.end();
+ ++i) {
+ if (!candidate.attrs.count(i->first)) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "missing attr " << i->first;
+ } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "attr value mismatch " << i->first;
+ }
+ }
+ for (map<string,bufferptr>::const_iterator i = candidate.attrs.begin();
+ i != candidate.attrs.end();
+ ++i) {
+ if (!auth.attrs.count(i->first)) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "extra attr " << i->first;
+ }
+ }
+ return error;
+}
+
+map<pg_shard_t, ScrubMap *>::const_iterator
+ PGBackend::be_select_auth_object(
+ const hobject_t &obj,
+ const map<pg_shard_t,ScrubMap*> &maps)
+{
+ map<pg_shard_t, ScrubMap *>::const_iterator auth = maps.end();
+ for (map<pg_shard_t, ScrubMap *>::const_iterator j = maps.begin();
+ j != maps.end();
+ ++j) {
+ map<hobject_t, ScrubMap::object>::iterator i =
+ j->second->objects.find(obj);
+ if (i == j->second->objects.end()) {
+ continue;
+ }
+ if (auth == maps.end()) {
+ // Something is better than nothing
+ // TODO: something is NOT better than nothing, do something like
+ // unfound_lost if no valid copies can be found, or just mark unfound
+ auth = j;
+ dout(10) << __func__ << ": selecting osd " << j->first
+ << " for obj " << obj
+ << ", auth == maps.end()"
+ << dendl;
+ continue;
+ }
+ if (i->second.read_error) {
+ // scrub encountered read error, probably corrupt
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", read_error"
+ << dendl;
+ continue;
+ }
+ map<string, bufferptr>::iterator k = i->second.attrs.find(OI_ATTR);
+ if (k == i->second.attrs.end()) {
+ // no object info on object, probably corrupt
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", no oi attr"
+ << dendl;
+ continue;
+ }
+
+ bufferlist bl;
+ bl.push_back(k->second);
+ object_info_t oi;
+ try {
+ bufferlist::iterator bliter = bl.begin();
+ ::decode(oi, bliter);
+ } catch (...) {
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", corrupt oi attr"
+ << dendl;
+ // invalid object info, probably corrupt
+ continue;
+ }
+ uint64_t correct_size = be_get_ondisk_size(oi.size);
+ if (correct_size != i->second.size) {
+ // invalid size, probably corrupt
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", size mismatch"
+ << dendl;
+ // invalid object info, probably corrupt
+ continue;
+ }
+ dout(10) << __func__ << ": selecting osd " << j->first
+ << " for obj " << obj
+ << dendl;
+ auth = j;
+ }
+ return auth;
+}
+
+void PGBackend::be_compare_scrubmaps(
+ const map<pg_shard_t,ScrubMap*> &maps,
+ map<hobject_t, set<pg_shard_t> > &missing,
+ map<hobject_t, set<pg_shard_t> > &inconsistent,
+ map<hobject_t, pg_shard_t> &authoritative,
+ map<hobject_t, set<pg_shard_t> > &invalid_snapcolls,
+ int &shallow_errors, int &deep_errors,
+ const spg_t pgid,
+ const vector<int> &acting,
+ ostream &errorstream)
+{
+ map<hobject_t,ScrubMap::object>::const_iterator i;
+ map<pg_shard_t, ScrubMap *>::const_iterator j;
+ set<hobject_t> master_set;
+
+ // Construct master set
+ for (j = maps.begin(); j != maps.end(); ++j) {
+ for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) {
+ master_set.insert(i->first);
+ }
+ }
+
+ // Check maps against master set and each other
+ for (set<hobject_t>::const_iterator k = master_set.begin();
+ k != master_set.end();
+ ++k) {
+ map<pg_shard_t, ScrubMap *>::const_iterator auth =
+ be_select_auth_object(*k, maps);
+ assert(auth != maps.end());
+ set<pg_shard_t> cur_missing;
+ set<pg_shard_t> cur_inconsistent;
+ for (j = maps.begin(); j != maps.end(); ++j) {
+ if (j == auth)
+ continue;
+ if (j->second->objects.count(*k)) {
+ // Compare
+ stringstream ss;
+ enum scrub_error_type error = be_compare_scrub_objects(auth->second->objects[*k],
+ j->second->objects[*k],
+ ss);
+ if (error != CLEAN) {
+ cur_inconsistent.insert(j->first);
+ if (error == SHALLOW_ERROR)
+ ++shallow_errors;
+ else
+ ++deep_errors;
+ errorstream << pgid << " shard " << j->first
+ << ": soid " << *k << " " << ss.str() << std::endl;
+ }
+ } else {
+ cur_missing.insert(j->first);
+ ++shallow_errors;
+ errorstream << pgid << " shard " << j->first
+ << " missing " << *k << std::endl;
+ }
+ }
+ assert(auth != maps.end());
+ if (!cur_missing.empty()) {
+ missing[*k] = cur_missing;
+ }
+ if (!cur_inconsistent.empty()) {
+ inconsistent[*k] = cur_inconsistent;
+ }
+ if (!cur_inconsistent.empty() || !cur_missing.empty()) {
+ authoritative[*k] = auth->first;
+ }
+ }
+}
diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h
index b887b14..d805782 100644
--- a/src/osd/PGBackend.h
+++ b/src/osd/PGBackend.h
@@ -22,6 +22,7 @@
#include "osd_types.h"
#include "include/Context.h"
#include "os/ObjectStore.h"
+#include "common/LogClient.h"
#include <string>
/**
@@ -37,6 +38,10 @@
* 4) Handling scrub, deep-scrub, repair
*/
class PGBackend {
+ protected:
+ ObjectStore *store;
+ const coll_t coll;
+ const coll_t temp_coll;
public:
/**
* Provides interfaces for PGBackend callbacks
@@ -73,18 +78,17 @@
* Called when peer is recovered
*/
virtual void on_peer_recover(
- int peer,
+ pg_shard_t peer,
const hobject_t &oid,
const ObjectRecoveryInfo &recovery_info,
const object_stat_sum_t &stat
) = 0;
virtual void begin_peer_recover(
- int peer,
+ pg_shard_t peer,
const hobject_t oid) = 0;
- virtual void failed_push(int from, const hobject_t &soid) = 0;
-
+ virtual void failed_push(pg_shard_t from, const hobject_t &soid) = 0;
virtual void cancel_pull(const hobject_t &soid) = 0;
@@ -103,28 +107,66 @@
ObjectStore::Transaction *t,
OpRequestRef op = OpRequestRef()
) = 0;
- virtual epoch_t get_epoch() = 0;
- virtual const vector<int> &get_actingbackfill() = 0;
+ virtual epoch_t get_epoch() const = 0;
+
+ virtual const set<pg_shard_t> &get_actingbackfill_shards() const = 0;
+ virtual const set<pg_shard_t> &get_acting_shards() const = 0;
+ virtual const set<pg_shard_t> &get_backfill_shards() const = 0;
+
virtual std::string gen_dbg_prefix() const = 0;
- virtual const map<hobject_t, set<int> > &get_missing_loc() = 0;
- virtual const map<int, pg_missing_t> &get_peer_missing() = 0;
- virtual const map<int, pg_info_t> &get_peer_info() = 0;
- virtual const pg_missing_t &get_local_missing() = 0;
- virtual const PGLog &get_log() = 0;
+ virtual const map<hobject_t, set<pg_shard_t> > &get_missing_loc_shards()
+ const = 0;
+
+ virtual const pg_missing_t &get_local_missing() const = 0;
+ virtual const map<pg_shard_t, pg_missing_t> &get_shard_missing()
+ const = 0;
+ virtual boost::optional<const pg_missing_t &> maybe_get_shard_missing(
+ pg_shard_t peer) const {
+ if (peer == primary_shard()) {
+ return get_local_missing();
+ } else {
+ map<pg_shard_t, pg_missing_t>::const_iterator i =
+ get_shard_missing().find(peer);
+ if (i == get_shard_missing().end()) {
+ return boost::optional<const pg_missing_t &>();
+ } else {
+ return i->second;
+ }
+ }
+ }
+ virtual const pg_missing_t &get_shard_missing(pg_shard_t peer) const {
+ boost::optional<const pg_missing_t &> m = maybe_get_shard_missing(peer);
+ assert(m);
+ return *m;
+ }
+
+ virtual const map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
+ virtual const pg_info_t &get_shard_info(pg_shard_t peer) const {
+ if (peer == primary_shard()) {
+ return get_info();
+ } else {
+ map<pg_shard_t, pg_info_t>::const_iterator i =
+ get_shard_info().find(peer);
+ assert(i != get_shard_info().end());
+ return i->second;
+ }
+ }
+
+ virtual const PGLog &get_log() const = 0;
virtual bool pgb_is_primary() const = 0;
virtual OSDMapRef pgb_get_osdmap() const = 0;
virtual const pg_info_t &get_info() const = 0;
virtual ObjectContextRef get_obc(
const hobject_t &hoid,
- map<string, bufferptr> &attrs) = 0;
+ map<string, bufferlist> &attrs) = 0;
virtual void op_applied(
const eversion_t &applied_version) = 0;
virtual bool should_send_op(
- int peer,
+ pg_shard_t peer,
const hobject_t &hoid) = 0;
virtual void log_operation(
@@ -134,7 +176,7 @@
ObjectStore::Transaction *t) = 0;
virtual void update_peer_last_complete_ondisk(
- int fromosd,
+ pg_shard_t fromosd,
eversion_t lcod) = 0;
virtual void update_last_complete_ondisk(
@@ -143,11 +185,44 @@
virtual void update_stats(
const pg_stat_t &stat) = 0;
+ virtual void schedule_work(
+ GenContext<ThreadPool::TPHandle&> *c) = 0;
+
+ virtual pg_shard_t whoami_shard() const = 0;
+ int whoami() const {
+ return whoami_shard().osd;
+ }
+ spg_t whoami_spg_t() const {
+ return get_info().pgid;
+ }
+
+ virtual spg_t primary_spg_t() const = 0;
+ virtual pg_shard_t primary_shard() const = 0;
+
+ virtual void send_message_osd_cluster(
+ int peer, Message *m, epoch_t from_epoch) = 0;
+ virtual void send_message_osd_cluster(
+ Message *m, Connection *con) = 0;
+ virtual void send_message_osd_cluster(
+ Message *m, const ConnectionRef& con) = 0;
+ virtual ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch) = 0;
+ virtual entity_name_t get_cluster_msgr_name() = 0;
+
+ virtual PerfCounters *get_logger() = 0;
+
+ virtual tid_t get_tid() = 0;
+
+ virtual LogClientTemp clog_error() = 0;
+
virtual ~Listener() {}
};
Listener *parent;
Listener *get_parent() const { return parent; }
- PGBackend(Listener *l) : parent(l) {}
+ PGBackend(Listener *l, ObjectStore *store, coll_t coll, coll_t temp_coll) :
+ store(store),
+ coll(coll),
+ temp_coll(temp_coll),
+ parent(l), temp_created(false) {}
bool is_primary() const { return get_parent()->pgb_is_primary(); }
OSDMapRef get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
const pg_info_t &get_info() { return get_parent()->get_info(); }
@@ -200,6 +275,7 @@
*/
virtual void recover_object(
const hobject_t &hoid, ///< [in] object to recover
+ eversion_t v, ///< [in] version to recover
ObjectContextRef head, ///< [in] context of the head/snapdir object
ObjectContextRef obc, ///< [in] context of the object
RecoveryHandle *h ///< [in,out] handle to attach recovery op to
@@ -223,25 +299,81 @@
* implementation should clear itself, contexts blessed prior to on_change
* won't be called after on_change()
*/
- virtual void on_change(ObjectStore::Transaction *t) = 0;
+ void on_change(ObjectStore::Transaction *t);
+ virtual void _on_change(ObjectStore::Transaction *t) = 0;
virtual void clear_state() = 0;
virtual void on_flushed() = 0;
+ class IsRecoverablePredicate {
+ public:
+ /**
+ * have encodes the shards available
+ */
+ virtual bool operator()(const set<pg_shard_t> &have) const = 0;
+ virtual ~IsRecoverablePredicate() {}
+ };
+ virtual IsRecoverablePredicate *get_is_recoverable_predicate() = 0;
+
+ class IsReadablePredicate {
+ public:
+ /**
+ * have encodes the shards available
+ */
+ virtual bool operator()(const set<pg_shard_t> &have) const = 0;
+ virtual ~IsReadablePredicate() {}
+ };
+ virtual IsReadablePredicate *get_is_readable_predicate() = 0;
- virtual void split_colls(
- pg_t child,
+ void temp_colls(list<coll_t> *out) {
+ if (temp_created)
+ out->push_back(temp_coll);
+ }
+ void split_colls(
+ spg_t child,
int split_bits,
int seed,
- ObjectStore::Transaction *t) = 0;
-
- virtual void temp_colls(list<coll_t> *out) = 0;
+ ObjectStore::Transaction *t) {
+ coll_t target = coll_t::make_temp_coll(child);
+ if (!temp_created)
+ return;
+ t->create_collection(target);
+ t->split_collection(
+ temp_coll,
+ split_bits,
+ seed,
+ target);
+ }
virtual void dump_recovery_info(Formatter *f) const = 0;
- virtual coll_t get_temp_coll(ObjectStore::Transaction *t) = 0;
- virtual void add_temp_obj(const hobject_t &oid) = 0;
- virtual void clear_temp_obj(const hobject_t &oid) = 0;
+ private:
+ bool temp_created;
+ set<hobject_t> temp_contents;
+ public:
+ coll_t get_temp_coll(ObjectStore::Transaction *t);
+ coll_t get_temp_coll() const {
+ return temp_coll;
+ }
+ bool have_temp_coll() const { return temp_created; }
+
+ // Track contents of temp collection, clear on reset
+ void add_temp_obj(const hobject_t &oid) {
+ temp_contents.insert(oid);
+ }
+ void add_temp_objs(const set<hobject_t> &oids) {
+ temp_contents.insert(oids.begin(), oids.end());
+ }
+ void clear_temp_obj(const hobject_t &oid) {
+ temp_contents.erase(oid);
+ }
+ void clear_temp_objs(const set<hobject_t> &oids) {
+ for (set<hobject_t>::const_iterator i = oids.begin();
+ i != oids.end();
+ ++i) {
+ temp_contents.erase(*i);
+ }
+ }
virtual ~PGBackend() {}
@@ -282,6 +414,11 @@
const hobject_t &from,
const hobject_t &to
) = 0;
+ virtual void set_alloc_hint(
+ const hobject_t &hoid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size
+ ) = 0;
/// Optional, not supported on ec-pool
virtual void write(
@@ -366,57 +503,57 @@
ObjectStore::Transaction *t);
/// Reapply old attributes
- virtual void rollback_setattrs(
+ void rollback_setattrs(
const hobject_t &hoid,
map<string, boost::optional<bufferlist> > &old_attrs,
- ObjectStore::Transaction *t) = 0;
+ ObjectStore::Transaction *t);
/// Truncate object to rollback append
virtual void rollback_append(
const hobject_t &hoid,
uint64_t old_size,
- ObjectStore::Transaction *t) = 0;
+ ObjectStore::Transaction *t);
/// Unstash object to rollback stash
- virtual void rollback_stash(
+ void rollback_stash(
const hobject_t &hoid,
version_t old_version,
- ObjectStore::Transaction *t) = 0;
+ ObjectStore::Transaction *t);
/// Delete object to rollback create
- virtual void rollback_create(
+ void rollback_create(
const hobject_t &hoid,
- ObjectStore::Transaction *t) = 0;
+ ObjectStore::Transaction *t);
/// Trim object stashed at stashed_version
- virtual void trim_stashed_object(
+ void trim_stashed_object(
const hobject_t &hoid,
version_t stashed_version,
- ObjectStore::Transaction *t) = 0;
+ ObjectStore::Transaction *t);
/// List objects in collection
- virtual int objects_list_partial(
+ int objects_list_partial(
const hobject_t &begin,
int min,
int max,
snapid_t seq,
vector<hobject_t> *ls,
- hobject_t *next) = 0;
+ hobject_t *next);
- virtual int objects_list_range(
+ int objects_list_range(
const hobject_t &start,
const hobject_t &end,
snapid_t seq,
- vector<hobject_t> *ls) = 0;
+ vector<hobject_t> *ls);
- virtual int objects_get_attr(
+ int objects_get_attr(
const hobject_t &hoid,
const string &attr,
- bufferlist *out) = 0;
+ bufferlist *out);
virtual int objects_get_attrs(
const hobject_t &hoid,
- map<string, bufferlist> *out) = 0;
+ map<string, bufferlist> *out);
virtual int objects_read_sync(
const hobject_t &hoid,
@@ -431,24 +568,64 @@
Context *on_complete) = 0;
virtual bool scrub_supported() { return false; }
- virtual void be_scan_list(ScrubMap &map, const vector<hobject_t> &ls, bool deep,
- ThreadPool::TPHandle &handle) { assert(0); }
- virtual enum scrub_error_type be_compare_scrub_objects(
- const ScrubMap::object &auth,
- const ScrubMap::object &candidate,
- ostream &errorstream) { assert(0); }
- virtual map<int, ScrubMap *>::const_iterator be_select_auth_object(
+ void be_scan_list(
+ ScrubMap &map, const vector<hobject_t> &ls, bool deep,
+ ThreadPool::TPHandle &handle);
+ enum scrub_error_type be_compare_scrub_objects(
+ const ScrubMap::object &auth,
+ const ScrubMap::object &candidate,
+ ostream &errorstream);
+ map<pg_shard_t, ScrubMap *>::const_iterator be_select_auth_object(
const hobject_t &obj,
- const map<int,ScrubMap*> &maps) { assert(0); }
- virtual void be_compare_scrubmaps(const map<int,ScrubMap*> &maps,
- map<hobject_t, set<int> > &missing,
- map<hobject_t, set<int> > &inconsistent,
- map<hobject_t, int> &authoritative,
- map<hobject_t, set<int> > &invalid_snapcolls,
- int &shallow_errors, int &deep_errors,
- const pg_t pgid,
- const vector<int> &acting,
- ostream &errorstream) { assert(0); }
+ const map<pg_shard_t,ScrubMap*> &maps);
+ void be_compare_scrubmaps(
+ const map<pg_shard_t,ScrubMap*> &maps,
+ map<hobject_t, set<pg_shard_t> > &missing,
+ map<hobject_t, set<pg_shard_t> > &inconsistent,
+ map<hobject_t, pg_shard_t> &authoritative,
+ map<hobject_t, set<pg_shard_t> > &invalid_snapcolls,
+ int &shallow_errors, int &deep_errors,
+ const spg_t pgid,
+ const vector<int> &acting,
+ ostream &errorstream);
+ virtual uint64_t be_get_ondisk_size(
+ uint64_t logical_size) { assert(0); return 0; }
+ virtual void be_deep_scrub(
+ const hobject_t &poid,
+ ScrubMap::object &o,
+ ThreadPool::TPHandle &handle) { assert(0); }
+
+ static PGBackend *build_pg_backend(
+ const pg_pool_t &pool,
+ Listener *l,
+ coll_t coll,
+ coll_t temp_coll,
+ ObjectStore *store,
+ CephContext *cct);
};
+struct PG_SendMessageOnConn: public Context {
+ PGBackend::Listener *pg;
+ Message *reply;
+ ConnectionRef conn;
+ PG_SendMessageOnConn(
+ PGBackend::Listener *pg,
+ Message *reply,
+ ConnectionRef conn) : pg(pg), reply(reply), conn(conn) {}
+ void finish(int) {
+ pg->send_message_osd_cluster(reply, conn.get());
+ }
+};
+
+struct PG_QueueAsync : public Context {
+ PGBackend::Listener *pg;
+ GenContext<ThreadPool::TPHandle&> *c;
+ PG_QueueAsync(
+ PGBackend::Listener *pg,
+ GenContext<ThreadPool::TPHandle&> *c) : pg(pg), c(c) {}
+ void finish(int) {
+ pg->schedule_work(c);
+ }
+};
+
#endif
diff --git a/src/osd/PGLog.cc b/src/osd/PGLog.cc
index e845bd6..c58a1c4 100644
--- a/src/osd/PGLog.cc
+++ b/src/osd/PGLog.cc
@@ -110,7 +110,7 @@ void PGLog::clear() {
}
void PGLog::clear_info_log(
- pg_t pgid,
+ spg_t pgid,
const hobject_t &infos_oid,
const hobject_t &log_oid,
ObjectStore::Transaction *t) {
@@ -144,8 +144,10 @@ void PGLog::trim(
}
}
-void PGLog::proc_replica_log(ObjectStore::Transaction& t,
- pg_info_t &oinfo, const pg_log_t &olog, pg_missing_t& omissing, int from) const
+void PGLog::proc_replica_log(
+ ObjectStore::Transaction& t,
+ pg_info_t &oinfo, const pg_log_t &olog, pg_missing_t& omissing,
+ pg_shard_t from) const
{
dout(10) << "proc_replica_log for osd." << from << ": "
<< oinfo << " " << olog << " " << omissing << dendl;
@@ -167,6 +169,21 @@ void PGLog::proc_replica_log(ObjectStore::Transaction& t,
<< " have " << i->second.have << dendl;
}
+ list<pg_log_entry_t>::const_iterator fromiter = log.log.end();
+ eversion_t lower_bound = log.tail;
+ while (1) {
+ if (fromiter == log.log.begin())
+ break;
+ --fromiter;
+ if (fromiter->version <= olog.head) {
+ dout(20) << "merge_log cut point (usually last shared) is "
+ << *fromiter << dendl;
+ lower_bound = fromiter->version;
+ ++fromiter;
+ break;
+ }
+ }
+
list<pg_log_entry_t> divergent;
list<pg_log_entry_t>::const_iterator pp = olog.log.end();
eversion_t lu(oinfo.last_update);
@@ -181,33 +198,32 @@ void PGLog::proc_replica_log(ObjectStore::Transaction& t,
// don't continue past the tail of our log.
if (oe.version <= log.tail) {
+ lu = oe.version;
+ ++pp;
break;
}
- ceph::unordered_map<hobject_t, pg_log_entry_t*>::const_iterator i =
- log.objects.find(oe.soid);
- if (i != log.objects.end() && i->second->version == oe.version) {
- dout(10) << " had " << oe << " new " << *(i->second)
- << " : match, stopping" << dendl;
- lu = pp->version;
+ if (oe.version <= lower_bound) {
+ lu = oe.version;
+ ++pp;
break;
}
divergent.push_front(oe);
}
- for (list<pg_log_entry_t>::iterator i = divergent.begin();
- i != divergent.end();
- ++i) {
- _merge_old_entry(
- t,
- *i,
- oinfo,
- omissing,
- olog.can_rollback_to,
- 0,
- 0);
- }
+
+ IndexedLog folog;
+ folog.log.insert(folog.log.begin(), olog.log.begin(), pp);
+ folog.index();
+ _merge_divergent_entries(
+ folog,
+ divergent,
+ oinfo,
+ olog.can_rollback_to,
+ omissing,
+ 0,
+ 0);
if (lu < oinfo.last_update) {
dout(10) << " peer osd." << from << " last_update now " << lu << dendl;
@@ -232,136 +248,186 @@ void PGLog::proc_replica_log(ObjectStore::Transaction& t,
}
}
-/*
- * merge an old (possibly divergent) log entry into the new log. this
- * happens _after_ new log items have been assimilated. thus, we assume
- * the index already references newer entries (if present), and missing
- * has been updated accordingly.
+/**
+ * _merge_object_divergent_entries
*
- * return true if entry is not divergent.
+ * There are 5 distinct cases:
+ * 1) There is a more recent update: in this case we assume we adjusted the
+ * store and missing during merge_log
+ * 2) The first entry in the divergent sequence is a create. This might
+ * either be because the object is a clone or because prior_version is
+ * eversion_t(). In this case the object does not exist and we must
+ * adjust missing and the store to match.
+ * 3) We are currently missing the object. In this case, we adjust the
+ * missing to our prior_version taking care to add a divergent_prior
+ * if necessary
+ * 4) We can rollback all of the entries. In this case, we do so using
+ * the rollbacker and return -- the object does not go into missing.
+ * 5) We cannot rollback at least 1 of the entries. In this case, we
+ * clear the object out of the store and add a missing entry at
+ * prior_version taking care to add a divergent_prior if
+ * necessary.
*/
-bool PGLog::_merge_old_entry(
- ObjectStore::Transaction& t,
- const pg_log_entry_t &oe,
- const pg_info_t& info,
- pg_missing_t &missing,
+void PGLog::_merge_object_divergent_entries(
+ const IndexedLog &log,
+ const hobject_t &hoid,
+ const list<pg_log_entry_t> &entries,
+ const pg_info_t &info,
eversion_t olog_can_rollback_to,
+ pg_missing_t &missing,
boost::optional<pair<eversion_t, hobject_t> > *new_divergent_prior,
- LogEntryHandler *rollbacker) const
+ LogEntryHandler *rollbacker
+ )
{
- if (oe.soid > info.last_backfill) {
- dout(20) << "merge_old_entry had " << oe
- << " : beyond last_backfill" << dendl;
- return false;
+ dout(10) << __func__ << ": merging hoid " << hoid
+ << " entries: " << entries << dendl;
+
+ if (hoid > info.last_backfill) {
+ dout(10) << __func__ << ": hoid " << hoid << " after last_backfill"
+ << dendl;
+ return;
}
- ceph::unordered_map<hobject_t, pg_log_entry_t*>::const_iterator objiter =
- log.objects.find(oe.soid);
- if (objiter != log.objects.end()) {
- pg_log_entry_t &ne = *(objiter->second); // new(er?) entry
-
- if (ne.version > oe.version) {
- dout(20) << "merge_old_entry had " << oe
- << " new " << ne << " : older, missing" << dendl;
- return false;
+
+ // entries is non-empty
+ assert(!entries.empty());
+ eversion_t last;
+ for (list<pg_log_entry_t>::const_iterator i = entries.begin();
+ i != entries.end();
+ ++i) {
+ // all entries are on hoid
+ assert(i->soid == hoid);
+ if (i != entries.begin() && i->prior_version != eversion_t()) {
+ // in increasing order of version
+ assert(i->version > last);
+ // prior_version correct
+ assert(i->prior_version == last);
}
- if (ne.version == oe.version) {
- dout(20) << "merge_old_entry had " << oe
- << " new " << ne << " : same" << dendl;
- return true;
+ last = i->version;
+
+ if (rollbacker)
+ rollbacker->trim(*i);
+ }
+
+ const eversion_t prior_version = entries.begin()->prior_version;
+ const eversion_t first_divergent_update = entries.begin()->version;
+ const eversion_t last_divergent_update = entries.rbegin()->version;
+ const bool object_not_in_store =
+ !missing.is_missing(hoid) &&
+ entries.rbegin()->is_delete();
+ dout(10) << __func__ << ": hoid " << hoid
+ << " prior_version: " << prior_version
+ << " first_divergent_update: " << first_divergent_update
+ << " last_divergent_update: " << last_divergent_update
+ << dendl;
+
+ ceph::unordered_map<hobject_t, pg_log_entry_t*>::const_iterator objiter =
+ log.objects.find(hoid);
+ if (objiter != log.objects.end() &&
+ objiter->second->version >= first_divergent_update) {
+ /// Case 1)
+ assert(objiter->second->version > last_divergent_update);
+
+ dout(10) << __func__ << ": more recent entry found: "
+ << *objiter->second << ", already merged" << dendl;
+
+ // ensure missing has been updated appropriately
+ if (objiter->second->is_update()) {
+ assert(missing.is_missing(hoid) &&
+ missing.missing[hoid].need == objiter->second->version);
+ } else {
+ assert(!missing.is_missing(hoid));
}
- if (oe.is_delete()) {
- if (ne.is_delete()) {
- // old and new are delete
- dout(20) << "merge_old_entry had " << oe
- << " new " << ne << " : both deletes" << dendl;
- } else {
- // old delete, new update.
- dout(20) << "merge_old_entry had " << oe
- << " new " << ne << " : missing" << dendl;
- missing.revise_need(ne.soid, ne.version);
- if (rollbacker)
- rollbacker->cant_rollback(oe);
- }
+ missing.revise_have(hoid, eversion_t());
+ if (rollbacker && !object_not_in_store)
+ rollbacker->remove(hoid);
+ return;
+ }
+
+ dout(10) << __func__ << ": hoid " << hoid
+ <<" has no more recent entries in log" << dendl;
+ if (prior_version == eversion_t() || entries.front().is_clone()) {
+ /// Case 2)
+ dout(10) << __func__ << ": hoid " << hoid
+ << " prior_version or op type indicates creation, deleting"
+ << dendl;
+ if (missing.is_missing(hoid))
+ missing.rm(missing.missing.find(hoid));
+ if (rollbacker && !object_not_in_store)
+ rollbacker->remove(hoid);
+ return;
+ }
+
+ if (missing.is_missing(hoid)) {
+ /// Case 3)
+ dout(10) << __func__ << ": hoid " << hoid
+ << " missing, " << missing.missing[hoid]
+ << " adjusting" << dendl;
+
+ if (missing.missing[hoid].have == prior_version) {
+ dout(10) << __func__ << ": hoid " << hoid
+ << " missing.have is prior_version " << prior_version
+ << " removing from missing" << dendl;
+ missing.rm(missing.missing.find(hoid));
} else {
- if (ne.is_delete()) {
- // old update, new delete
- dout(20) << "merge_old_entry had " << oe
- << " new " << ne << " : new delete supercedes" << dendl;
- if (rollbacker) {
- rollbacker->remove(oe.soid);
- rollbacker->cant_rollback(oe);
- }
- if (missing.is_missing(oe.soid))
- missing.rm(oe.soid, oe.version);
- } else {
- // old update, new update
- dout(20) << "merge_old_entry had " << oe
- << " new " << ne << " : new item supercedes" << dendl;
- if (oe.mod_desc.can_rollback() && oe.version > olog_can_rollback_to) {
- dout(20) << __func__ << ": ne.version < oe.version && can rollback, "
- << "rolling back " << oe << dendl;
- if (rollbacker)
- rollbacker->rollback(oe);
- } else {
- missing.revise_need(ne.soid, ne.version);
- if (rollbacker)
- rollbacker->cant_rollback(oe);
- }
+ dout(10) << __func__ << ": hoid " << hoid
+ << " missing.have is " << missing.missing[hoid].have
+ << ", adjusting" << dendl;
+ missing.revise_need(hoid, prior_version);
+ if (prior_version <= info.log_tail) {
+ dout(10) << __func__ << ": hoid " << hoid
+ << " prior_version " << prior_version << " <= info.log_tail "
+ << info.log_tail << dendl;
+ if (new_divergent_prior)
+ *new_divergent_prior = make_pair(prior_version, hoid);
}
}
- } else if (oe.op == pg_log_entry_t::CLONE) {
- assert(oe.soid.snap != CEPH_NOSNAP);
- dout(20) << "merge_old_entry had " << oe
- << ", clone with no non-divergent log entries, "
- << "deleting" << dendl;
- if (missing.is_missing(oe.soid)) {
- missing.rm(oe.soid, missing.missing[oe.soid].need);
+ return;
+ }
+
+ dout(10) << __func__ << ": hoid " << hoid
+ << " must be rolled back or recovered, attempting to rollback"
+ << dendl;
+ bool can_rollback = true;
+ /// Distinguish between 4) and 5)
+ for (list<pg_log_entry_t>::const_reverse_iterator i = entries.rbegin();
+ i != entries.rend();
+ ++i) {
+ if (!i->mod_desc.can_rollback() || i->version <= olog_can_rollback_to) {
+ dout(10) << __func__ << ": hoid " << hoid << " cannot rollback "
+ << *i << dendl;
+ can_rollback = false;
+ break;
}
+ }
- if (oe.mod_desc.can_rollback() && oe.version > olog_can_rollback_to) {
- dout(20) << __func__ << ": rolling back " << oe << dendl;
- if (rollbacker)
- rollbacker->rollback(oe);
- } else {
- dout(20) << __func__ << ": had " << oe << " deleting" << dendl;
+ if (can_rollback) {
+ /// Case 4)
+ for (list<pg_log_entry_t>::const_reverse_iterator i = entries.rbegin();
+ i != entries.rend();
+ ++i) {
+ assert(i->mod_desc.can_rollback() && i->version > olog_can_rollback_to);
+ dout(10) << __func__ << ": hoid " << hoid
+ << " rolling back " << *i << dendl;
if (rollbacker)
- rollbacker->remove(oe.soid);
+ rollbacker->rollback(*i);
}
- } else if (oe.prior_version > info.log_tail && missing.is_missing(oe.soid)) {
- /**
- * oe.prior_version is a previously divergent log entry
- * oe.soid must have already been handled and the missing
- * set updated appropriately
- */
- dout(20) << "merge_old_entry had oe " << oe
- << " with divergent prior_version " << oe.prior_version
- << " oe.soid " << oe.soid
- << " must already have been merged" << dendl;
+ dout(10) << __func__ << ": hoid " << hoid << " rolled back" << dendl;
+ return;
} else {
- if (oe.mod_desc.can_rollback() && oe.version > olog_can_rollback_to) {
- dout(20) << __func__ << ": rolling back " << oe << dendl;
- if (rollbacker)
- rollbacker->rollback(oe);
- } else {
- if (!oe.is_delete()) {
- if (rollbacker)
- rollbacker->remove(oe.soid);
- dout(20) << __func__ << ": had " << oe << " deleting" << dendl;
- }
- dout(20) << "merge_old_entry had " << oe << " updating missing to "
- << oe.prior_version << dendl;
- if (oe.prior_version > eversion_t()) {
- if (new_divergent_prior)
- *new_divergent_prior = make_pair(oe.prior_version, oe.soid);
- missing.revise_need(oe.soid, oe.prior_version);
- if (rollbacker)
- rollbacker->cant_rollback(oe);
- } else if (missing.is_missing(oe.soid)) {
- missing.rm(oe.soid, missing.missing[oe.soid].need);
- }
+ /// Case 5)
+ dout(10) << __func__ << ": hoid " << hoid << " cannot roll back, "
+ << "removing and adding to missing" << dendl;
+ if (rollbacker && !object_not_in_store)
+ rollbacker->remove(hoid);
+ missing.add(hoid, prior_version, eversion_t());
+ if (prior_version <= info.log_tail) {
+ dout(10) << __func__ << ": hoid " << hoid
+ << " prior_version " << prior_version << " <= info.log_tail "
+ << info.log_tail << dendl;
+ if (new_divergent_prior)
+ *new_divergent_prior = make_pair(prior_version, hoid);
}
}
- return false;
}
/**
@@ -405,11 +471,22 @@ void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead
info.last_complete = newhead;
log.index();
- for (list<pg_log_entry_t>::iterator d = divergent.begin();
- d != divergent.end();
- ++d) {
- merge_old_entry(t, *d, info, rollbacker);
- rollbacker->trim(*d);
+
+ map<eversion_t, hobject_t> new_priors;
+ _merge_divergent_entries(
+ log,
+ divergent,
+ info,
+ log.can_rollback_to,
+ missing,
+ &new_priors,
+ rollbacker);
+ for (map<eversion_t, hobject_t>::iterator i = new_priors.begin();
+ i != new_priors.end();
+ ++i) {
+ add_divergent_prior(
+ i->first,
+ i->second);
}
if (info.last_update < log.can_rollback_to)
@@ -420,7 +497,7 @@ void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead
}
void PGLog::merge_log(ObjectStore::Transaction& t,
- pg_info_t &oinfo, pg_log_t &olog, int fromosd,
+ pg_info_t &oinfo, pg_log_t &olog, pg_shard_t fromosd,
pg_info_t &info, LogEntryHandler *rollbacker,
bool &dirty_info, bool &dirty_big_info)
{
@@ -545,12 +622,21 @@ void PGLog::merge_log(ObjectStore::Transaction& t,
info.last_user_version = oinfo.last_user_version;
info.purged_snaps = oinfo.purged_snaps;
- // process divergent items
- for (list<pg_log_entry_t>::iterator d = divergent.begin();
- d != divergent.end();
- ++d) {
- merge_old_entry(t, *d, info, rollbacker);
- rollbacker->trim(*d);
+ map<eversion_t, hobject_t> new_priors;
+ _merge_divergent_entries(
+ log,
+ divergent,
+ info,
+ log.can_rollback_to,
+ missing,
+ &new_priors,
+ rollbacker);
+ for (map<eversion_t, hobject_t>::iterator i = new_priors.begin();
+ i != new_priors.end();
+ ++i) {
+ add_divergent_prior(
+ i->first,
+ i->second);
}
// We cannot rollback into the new log entries
@@ -749,7 +835,11 @@ bool PGLog::read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
if (i->is_delete()) continue;
bufferlist bv;
- int r = store->getattr(coll, i->soid, OI_ATTR, bv);
+ int r = store->getattr(
+ coll,
+ ghobject_t(i->soid, ghobject_t::NO_GEN, info.pgid.shard),
+ OI_ATTR,
+ bv);
if (r >= 0) {
object_info_t oi(bv);
if (oi.version < i->version) {
@@ -770,7 +860,11 @@ bool PGLog::read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
if (did.count(i->second)) continue;
did.insert(i->second);
bufferlist bv;
- int r = store->getattr(coll, i->second, OI_ATTR, bv);
+ int r = store->getattr(
+ coll,
+ ghobject_t(i->second, ghobject_t::NO_GEN, info.pgid.shard),
+ OI_ATTR,
+ bv);
if (r >= 0) {
object_info_t oi(bv);
/**
diff --git a/src/osd/PGLog.h b/src/osd/PGLog.h
index beafdbf..9309841 100644
--- a/src/osd/PGLog.h
+++ b/src/osd/PGLog.h
@@ -34,8 +34,6 @@ struct PGLog {
const hobject_t &hoid) = 0;
virtual void trim(
const pg_log_entry_t &entry) = 0;
- virtual void cant_rollback(
- const pg_log_entry_t &entry) = 0;
virtual ~LogEntryHandler() {}
};
@@ -313,7 +311,7 @@ public:
void reset_recovery_pointers() { log.reset_recovery_pointers(); }
static void clear_info_log(
- pg_t pgid,
+ spg_t pgid,
const hobject_t &infos_oid,
const hobject_t &log_oid,
ObjectStore::Transaction *t);
@@ -360,7 +358,7 @@ public:
break;
if (info.last_complete < log.complete_to->version)
info.last_complete = log.complete_to->version;
- log.complete_to++;
+ ++log.complete_to;
}
}
@@ -372,53 +370,111 @@ public:
log.complete_to = log.log.begin();
while (log.complete_to->version <
missing.missing[missing.rmissing.begin()->second].need)
- log.complete_to++;
+ ++log.complete_to;
assert(log.complete_to != log.log.end());
if (log.complete_to == log.log.begin()) {
info.last_complete = eversion_t();
} else {
- log.complete_to--;
+ --log.complete_to;
info.last_complete = log.complete_to->version;
- log.complete_to++;
+ ++log.complete_to;
}
log.last_requested = 0;
}
void proc_replica_log(ObjectStore::Transaction& t, pg_info_t &oinfo, const pg_log_t &olog,
- pg_missing_t& omissing, int from) const;
+ pg_missing_t& omissing, pg_shard_t from) const;
protected:
- bool _merge_old_entry(
- ObjectStore::Transaction& t,
- const pg_log_entry_t &oe,
- const pg_info_t& info,
- pg_missing_t &missing,
- eversion_t olog_can_rollback_to,
+ static void split_by_object(
+ list<pg_log_entry_t> &entries,
+ map<hobject_t, list<pg_log_entry_t> > *out_entries) {
+ while (!entries.empty()) {
+ list<pg_log_entry_t> &out_list = (*out_entries)[entries.front().soid];
+ out_list.splice(out_list.end(), entries, entries.begin());
+ }
+ }
+
+ /**
+ * Merge complete list of divergent entries for an object
+ *
+ * @param new_divergent_prior [out] filled out for a new divergent prior
+ */
+ static void _merge_object_divergent_entries(
+ const IndexedLog &log, ///< [in] log to merge against
+ const hobject_t &hoid, ///< [in] object we are merging
+ const list<pg_log_entry_t> &entries, ///< [in] entries for hoid to merge
+ const pg_info_t &oinfo, ///< [in] info for merging entries
+ eversion_t olog_can_rollback_to, ///< [in] rollback boundary
+ pg_missing_t &omissing, ///< [in,out] missing to adjust, use
boost::optional<pair<eversion_t, hobject_t> > *new_divergent_prior,
- LogEntryHandler *rollbacker) const;
- bool merge_old_entry(
+ LogEntryHandler *rollbacker ///< [in] optional rollbacker object
+ );
+
+ /// Merge all entries using above
+ static void _merge_divergent_entries(
+ const IndexedLog &log, ///< [in] log to merge against
+ list<pg_log_entry_t> &entries, ///< [in] entries to merge
+ const pg_info_t &oinfo, ///< [in] info for merging entries
+ eversion_t olog_can_rollback_to, ///< [in] rollback boundary
+ pg_missing_t &omissing, ///< [in,out] missing to adjust, use
+ map<eversion_t, hobject_t> *priors, ///< [out] target for new priors
+ LogEntryHandler *rollbacker ///< [in] optional rollbacker object
+ ) {
+ map<hobject_t, list<pg_log_entry_t> > split;
+ split_by_object(entries, &split);
+ for (map<hobject_t, list<pg_log_entry_t> >::iterator i = split.begin();
+ i != split.end();
+ ++i) {
+ boost::optional<pair<eversion_t, hobject_t> > new_divergent_prior;
+ _merge_object_divergent_entries(
+ log,
+ i->first,
+ i->second,
+ oinfo,
+ olog_can_rollback_to,
+ omissing,
+ &new_divergent_prior,
+ rollbacker);
+ if (priors && new_divergent_prior) {
+ (*priors)[new_divergent_prior->first] = new_divergent_prior->second;
+ }
+ }
+ }
+
+ /**
+ * Exists for use in TestPGLog for simply testing single divergent log
+ * cases
+ */
+ void merge_old_entry(
ObjectStore::Transaction& t,
const pg_log_entry_t& oe,
const pg_info_t& info,
LogEntryHandler *rollbacker) {
boost::optional<pair<eversion_t, hobject_t> > new_divergent_prior;
- bool merged = _merge_old_entry(
- t, oe, info, missing,
+ list<pg_log_entry_t> entries;
+ entries.push_back(oe);
+ _merge_object_divergent_entries(
+ log,
+ oe.soid,
+ entries,
+ info,
log.can_rollback_to,
+ missing,
&new_divergent_prior,
rollbacker);
if (new_divergent_prior)
add_divergent_prior(
(*new_divergent_prior).first,
(*new_divergent_prior).second);
- return merged;
}
public:
void rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead,
pg_info_t &info, LogEntryHandler *rollbacker,
bool &dirty_info, bool &dirty_big_info);
- void merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from,
+ void merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
+ pg_shard_t from,
pg_info_t &info, LogEntryHandler *rollbacker,
bool &dirty_info, bool &dirty_big_info);
diff --git a/src/osd/ReplicatedBackend.cc b/src/osd/ReplicatedBackend.cc
index 6d7d2e5..5a9e814 100644
--- a/src/osd/ReplicatedBackend.cc
+++ b/src/osd/ReplicatedBackend.cc
@@ -29,10 +29,14 @@ static ostream& _prefix(std::ostream *_dout, ReplicatedBackend *pgb) {
}
ReplicatedBackend::ReplicatedBackend(
- PGBackend::Listener *pg, coll_t coll, OSDService *osd) :
- PGBackend(pg), temp_created(false),
- temp_coll(coll_t::make_temp_coll(pg->get_info().pgid)),
- coll(coll), osd(osd), cct(osd->cct) {}
+ PGBackend::Listener *pg,
+ coll_t coll,
+ coll_t temp_coll,
+ ObjectStore *store,
+ CephContext *cct) :
+ PGBackend(pg, store,
+ coll, temp_coll),
+ cct(cct) {}
void ReplicatedBackend::run_recovery_op(
PGBackend::RecoveryHandle *_h,
@@ -46,6 +50,7 @@ void ReplicatedBackend::run_recovery_op(
void ReplicatedBackend::recover_object(
const hobject_t &hoid,
+ eversion_t v,
ObjectContextRef head,
ObjectContextRef obc,
RecoveryHandle *_h
@@ -57,6 +62,7 @@ void ReplicatedBackend::recover_object(
assert(!obc);
// pull
prepare_pull(
+ v,
hoid,
head,
h);
@@ -73,10 +79,10 @@ void ReplicatedBackend::recover_object(
void ReplicatedBackend::check_recovery_sources(const OSDMapRef osdmap)
{
- for(map<int, set<hobject_t> >::iterator i = pull_from_peer.begin();
+ for(map<pg_shard_t, set<hobject_t> >::iterator i = pull_from_peer.begin();
i != pull_from_peer.end();
) {
- if (osdmap->is_down(i->first)) {
+ if (osdmap->is_down(i->first.osd)) {
dout(10) << "check_recovery_sources resetting pulls from osd." << i->first
<< ", osdmap has it marked down" << dendl;
for (set<hobject_t>::iterator j = i->second.begin();
@@ -186,18 +192,8 @@ void ReplicatedBackend::clear_state()
pull_from_peer.clear();
}
-void ReplicatedBackend::on_change(ObjectStore::Transaction *t)
+void ReplicatedBackend::_on_change(ObjectStore::Transaction *t)
{
- dout(10) << __func__ << dendl;
- // clear temp
- for (set<hobject_t>::iterator i = temp_contents.begin();
- i != temp_contents.end();
- ++i) {
- dout(10) << __func__ << ": Removing oid "
- << *i << " from the temp collection" << dendl;
- t->remove(get_temp_coll(t), *i);
- }
- temp_contents.clear();
for (map<tid_t, InProgressOp>::iterator i = in_progress_ops.begin();
i != in_progress_ops.end();
in_progress_ops.erase(i++)) {
@@ -209,22 +205,12 @@ void ReplicatedBackend::on_change(ObjectStore::Transaction *t)
clear_state();
}
-coll_t ReplicatedBackend::get_temp_coll(ObjectStore::Transaction *t)
-{
- if (temp_created)
- return temp_coll;
- if (!osd->store->collection_exists(temp_coll))
- t->create_collection(temp_coll);
- temp_created = true;
- return temp_coll;
-}
-
void ReplicatedBackend::on_flushed()
{
if (have_temp_coll() &&
- !osd->store->collection_empty(get_temp_coll())) {
+ !store->collection_empty(get_temp_coll())) {
vector<hobject_t> objects;
- osd->store->collection_list(get_temp_coll(), objects);
+ store->collection_list(get_temp_coll(), objects);
derr << __func__ << ": found objects in the temp collection: "
<< objects << ", crashing now"
<< dendl;
@@ -232,106 +218,13 @@ void ReplicatedBackend::on_flushed()
}
}
-
-int ReplicatedBackend::objects_list_partial(
- const hobject_t &begin,
- int min,
- int max,
- snapid_t seq,
- vector<hobject_t> *ls,
- hobject_t *next)
-{
- assert(ls);
- ghobject_t _next(begin);
- ls->reserve(max);
- int r = 0;
- while (!_next.is_max() && ls->size() < (unsigned)min) {
- vector<ghobject_t> objects;
- int r = osd->store->collection_list_partial(
- coll,
- _next,
- min - ls->size(),
- max - ls->size(),
- seq,
- &objects,
- &_next);
- if (r != 0)
- break;
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- assert(i->is_no_shard());
- if (i->is_no_gen()) {
- ls->push_back(i->hobj);
- }
- }
- }
- if (r == 0)
- *next = _next.hobj;
- return r;
-}
-
-int ReplicatedBackend::objects_list_range(
- const hobject_t &start,
- const hobject_t &end,
- snapid_t seq,
- vector<hobject_t> *ls)
-{
- assert(ls);
- vector<ghobject_t> objects;
- int r = osd->store->collection_list_range(
- coll,
- start,
- end,
- seq,
- &objects);
- ls->reserve(objects.size());
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- assert(i->is_no_shard());
- if (i->is_no_gen()) {
- ls->push_back(i->hobj);
- }
- }
- return r;
-}
-
-int ReplicatedBackend::objects_get_attr(
- const hobject_t &hoid,
- const string &attr,
- bufferlist *out)
-{
- bufferptr bp;
- int r = osd->store->getattr(
- coll,
- hoid,
- attr.c_str(),
- bp);
- if (r >= 0 && out) {
- out->clear();
- out->push_back(bp);
- }
- return r;
-}
-
-int ReplicatedBackend::objects_get_attrs(
- const hobject_t &hoid,
- map<string, bufferlist> *out)
-{
- return osd->store->getattrs(
- coll,
- hoid,
- *out);
-}
-
int ReplicatedBackend::objects_read_sync(
const hobject_t &hoid,
uint64_t off,
uint64_t len,
bufferlist *bl)
{
- return osd->store->read(coll, hoid, off, len, *bl);
+ return store->read(coll, hoid, off, len, *bl);
}
struct AsyncReadCallback : public GenContext<ThreadPool::TPHandle&> {
@@ -358,17 +251,17 @@ void ReplicatedBackend::objects_read_async(
to_read.begin();
i != to_read.end() && r >= 0;
++i) {
- int _r = osd->store->read(coll, hoid, i->first.first,
- i->first.second, *(i->second.first));
+ int _r = store->read(coll, hoid, i->first.first,
+ i->first.second, *(i->second.first));
if (i->second.second) {
- osd->gen_wq.queue(
+ get_parent()->schedule_work(
get_parent()->bless_gencontext(
new AsyncReadCallback(_r, i->second.second)));
}
if (_r < 0)
r = _r;
}
- osd->gen_wq.queue(
+ get_parent()->schedule_work(
get_parent()->bless_gencontext(
new AsyncReadCallback(r, on_complete)));
}
@@ -436,7 +329,7 @@ public:
version_t former_version) {
t->collection_move_rename(
coll, hoid, coll,
- ghobject_t(hoid, former_version, 0));
+ ghobject_t(hoid, former_version, ghobject_t::NO_SHARD));
}
void setattrs(
const hobject_t &hoid,
@@ -528,10 +421,20 @@ public:
t->zero(get_coll(hoid), hoid, off, len);
}
+ void set_alloc_hint(
+ const hobject_t &hoid,
+ uint64_t expected_object_size,
+ uint64_t expected_write_size
+ ) {
+ t->set_alloc_hint(get_coll(hoid), hoid, expected_object_size,
+ expected_write_size);
+ }
+
void append(
PGTransaction *_to_append
) {
RPGTransaction *to_append = dynamic_cast<RPGTransaction*>(_to_append);
+ assert(to_append);
t->append(*(to_append->t));
for (set<hobject_t>::iterator i = to_append->temp_added.begin();
i != to_append->temp_added.end();
@@ -599,6 +502,7 @@ void ReplicatedBackend::submit_transaction(
OpRequestRef orig_op)
{
RPGTransaction *t = dynamic_cast<RPGTransaction*>(_t);
+ assert(t);
ObjectStore::Transaction *op_t = t->get_transaction();
assert(t->get_temp_added().size() <= 1);
@@ -614,6 +518,14 @@ void ReplicatedBackend::submit_transaction(
)
).first->second;
+ op.waiting_for_applied.insert(
+ parent->get_actingbackfill_shards().begin(),
+ parent->get_actingbackfill_shards().end());
+ op.waiting_for_commit.insert(
+ parent->get_actingbackfill_shards().begin(),
+ parent->get_actingbackfill_shards().end());
+
+
issue_op(
soid,
at_version,
@@ -627,20 +539,13 @@ void ReplicatedBackend::submit_transaction(
&op,
op_t);
- // add myself to gather set
- op.waiting_for_applied.insert(osd->whoami);
- op.waiting_for_commit.insert(osd->whoami);
-
ObjectStore::Transaction local_t;
if (t->get_temp_added().size()) {
get_temp_coll(&local_t);
- temp_contents.insert(t->get_temp_added().begin(), t->get_temp_added().end());
- }
- for (set<hobject_t>::const_iterator i = t->get_temp_cleared().begin();
- i != t->get_temp_cleared().end();
- ++i) {
- temp_contents.erase(*i);
+ add_temp_objs(t->get_temp_added());
}
+ clear_temp_objs(t->get_temp_cleared());
+
parent->log_operation(log_entries, trim_to, true, &local_t);
local_t.append(*op_t);
local_t.swap(*op_t);
@@ -666,7 +571,7 @@ void ReplicatedBackend::op_applied(
if (op->op)
op->op->mark_event("op_applied");
- op->waiting_for_applied.erase(osd->whoami);
+ op->waiting_for_applied.erase(get_parent()->whoami_shard());
parent->op_applied(op->v);
if (op->waiting_for_applied.empty()) {
@@ -686,7 +591,7 @@ void ReplicatedBackend::op_commit(
if (op->op)
op->op->mark_event("op_commit");
- op->waiting_for_commit.erase(osd->whoami);
+ op->waiting_for_commit.erase(get_parent()->whoami_shard());
if (op->waiting_for_commit.empty()) {
op->on_commit->complete(0);
@@ -707,7 +612,7 @@ void ReplicatedBackend::sub_op_modify_reply(OpRequestRef op)
// must be replication.
tid_t rep_tid = r->get_tid();
- int fromosd = r->get_source().num();
+ pg_shard_t from = r->from;
if (in_progress_ops.count(rep_tid)) {
map<tid_t, InProgressOp>::iterator iter =
@@ -719,31 +624,31 @@ void ReplicatedBackend::sub_op_modify_reply(OpRequestRef op)
if (m)
dout(7) << __func__ << ": tid " << ip_op.tid << " op " //<< *m
- << " ack_type " << r->ack_type
- << " from osd." << fromosd
+ << " ack_type " << (int)r->ack_type
+ << " from " << from
<< dendl;
else
dout(7) << __func__ << ": tid " << ip_op.tid << " (no op) "
- << " ack_type " << r->ack_type
- << " from osd." << fromosd
+ << " ack_type " << (int)r->ack_type
+ << " from " << from
<< dendl;
// oh, good.
if (r->ack_type & CEPH_OSD_FLAG_ONDISK) {
- assert(ip_op.waiting_for_commit.count(fromosd));
- ip_op.waiting_for_commit.erase(fromosd);
+ assert(ip_op.waiting_for_commit.count(from));
+ ip_op.waiting_for_commit.erase(from);
if (ip_op.op)
ip_op.op->mark_event("sub_op_commit_rec");
} else {
- assert(ip_op.waiting_for_applied.count(fromosd));
+ assert(ip_op.waiting_for_applied.count(from));
if (ip_op.op)
ip_op.op->mark_event("sub_op_applied_rec");
}
- ip_op.waiting_for_applied.erase(fromosd);
+ ip_op.waiting_for_applied.erase(from);
parent->update_peer_last_complete_ondisk(
- fromosd,
+ from,
r->get_last_complete_ondisk());
if (ip_op.waiting_for_applied.empty() &&
@@ -763,317 +668,80 @@ void ReplicatedBackend::sub_op_modify_reply(OpRequestRef op)
}
}
-/*
- * pg lock may or may not be held
- */
-void ReplicatedBackend::be_scan_list(
- ScrubMap &map, const vector<hobject_t> &ls, bool deep,
- ThreadPool::TPHandle &handle)
-{
- dout(10) << "_scan_list scanning " << ls.size() << " objects"
- << (deep ? " deeply" : "") << dendl;
- int i = 0;
- for (vector<hobject_t>::const_iterator p = ls.begin();
- p != ls.end();
- ++p, i++) {
+void ReplicatedBackend::be_deep_scrub(
+ const hobject_t &poid,
+ ScrubMap::object &o,
+ ThreadPool::TPHandle &handle) {
+ bufferhash h, oh;
+ bufferlist bl, hdrbl;
+ int r;
+ __u64 pos = 0;
+ while ( (r = store->read(
+ coll,
+ ghobject_t(
+ poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ pos,
+ cct->_conf->osd_deep_scrub_stride, bl,
+ true)) > 0) {
handle.reset_tp_timeout();
- hobject_t poid = *p;
-
- struct stat st;
- int r = osd->store->stat(coll, poid, &st, true);
- if (r == 0) {
- ScrubMap::object &o = map.objects[poid];
- o.size = st.st_size;
- assert(!o.negative);
- osd->store->getattrs(coll, poid, o.attrs);
-
- // calculate the CRC32 on deep scrubs
- if (deep) {
- bufferhash h, oh;
- bufferlist bl, hdrbl;
- int r;
- __u64 pos = 0;
- while ( (r = osd->store->read(coll, poid, pos,
- cct->_conf->osd_deep_scrub_stride, bl,
- true)) > 0) {
- handle.reset_tp_timeout();
- h << bl;
- pos += bl.length();
- bl.clear();
- }
- if (r == -EIO) {
- dout(25) << "_scan_list " << poid << " got "
- << r << " on read, read_error" << dendl;
- o.read_error = true;
- }
- o.digest = h.digest();
- o.digest_present = true;
-
- bl.clear();
- r = osd->store->omap_get_header(coll, poid, &hdrbl, true);
- if (r == 0) {
- dout(25) << "CRC header " << string(hdrbl.c_str(), hdrbl.length())
- << dendl;
- ::encode(hdrbl, bl);
- oh << bl;
- bl.clear();
- } else if (r == -EIO) {
- dout(25) << "_scan_list " << poid << " got "
- << r << " on omap header read, read_error" << dendl;
- o.read_error = true;
- }
-
- ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(
- coll, poid);
- assert(iter);
- uint64_t keys_scanned = 0;
- for (iter->seek_to_first(); iter->valid() ; iter->next()) {
- if (cct->_conf->osd_scan_list_ping_tp_interval &&
- (keys_scanned % cct->_conf->osd_scan_list_ping_tp_interval == 0)) {
- handle.reset_tp_timeout();
- }
- ++keys_scanned;
-
- dout(25) << "CRC key " << iter->key() << " value "
- << string(iter->value().c_str(), iter->value().length()) << dendl;
-
- ::encode(iter->key(), bl);
- ::encode(iter->value(), bl);
- oh << bl;
- bl.clear();
- }
- if (iter->status() == -EIO) {
- dout(25) << "_scan_list " << poid << " got "
- << r << " on omap scan, read_error" << dendl;
- o.read_error = true;
- break;
- }
-
- //Store final calculated CRC32 of omap header & key/values
- o.omap_digest = oh.digest();
- o.omap_digest_present = true;
- }
-
- dout(25) << "_scan_list " << poid << dendl;
- } else if (r == -ENOENT) {
- dout(25) << "_scan_list " << poid << " got " << r << ", skipping" << dendl;
- } else if (r == -EIO) {
- dout(25) << "_scan_list " << poid << " got " << r << ", read_error" << dendl;
- ScrubMap::object &o = map.objects[poid];
- o.read_error = true;
- } else {
- derr << "_scan_list got: " << cpp_strerror(r) << dendl;
- assert(0);
- }
+ h << bl;
+ pos += bl.length();
+ bl.clear();
}
-}
-
-enum scrub_error_type ReplicatedBackend::be_compare_scrub_objects(
- const ScrubMap::object &auth,
- const ScrubMap::object &candidate,
- ostream &errorstream)
-{
- enum scrub_error_type error = CLEAN;
- if (candidate.read_error) {
- // This can occur on stat() of a shallow scrub, but in that case size will
- // be invalid, and this will be over-ridden below.
- error = DEEP_ERROR;
- errorstream << "candidate had a read error";
- }
- if (auth.digest_present && candidate.digest_present) {
- if (auth.digest != candidate.digest) {
- if (error != CLEAN)
- errorstream << ", ";
- error = DEEP_ERROR;
-
- errorstream << "digest " << candidate.digest
- << " != known digest " << auth.digest;
- }
+ if (r == -EIO) {
+ dout(25) << "_scan_list " << poid << " got "
+ << r << " on read, read_error" << dendl;
+ o.read_error = true;
}
- if (auth.omap_digest_present && candidate.omap_digest_present) {
- if (auth.omap_digest != candidate.omap_digest) {
- if (error != CLEAN)
- errorstream << ", ";
- error = DEEP_ERROR;
+ o.digest = h.digest();
+ o.digest_present = true;
- errorstream << "omap_digest " << candidate.omap_digest
- << " != known omap_digest " << auth.omap_digest;
- }
- }
- // Shallow error takes precendence because this will be seen by
- // both types of scrubs.
- if (auth.size != candidate.size) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "size " << candidate.size
- << " != known size " << auth.size;
- }
- for (map<string,bufferptr>::const_iterator i = auth.attrs.begin();
- i != auth.attrs.end();
- ++i) {
- if (!candidate.attrs.count(i->first)) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "missing attr " << i->first;
- } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "attr value mismatch " << i->first;
- }
- }
- for (map<string,bufferptr>::const_iterator i = candidate.attrs.begin();
- i != candidate.attrs.end();
- ++i) {
- if (!auth.attrs.count(i->first)) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "extra attr " << i->first;
- }
+ bl.clear();
+ r = store->omap_get_header(
+ coll,
+ ghobject_t(
+ poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+ &hdrbl, true);
+ if (r == 0) {
+ dout(25) << "CRC header " << string(hdrbl.c_str(), hdrbl.length())
+ << dendl;
+ ::encode(hdrbl, bl);
+ oh << bl;
+ bl.clear();
+ } else if (r == -EIO) {
+ dout(25) << "_scan_list " << poid << " got "
+ << r << " on omap header read, read_error" << dendl;
+ o.read_error = true;
}
- return error;
-}
-map<int, ScrubMap *>::const_iterator ReplicatedBackend::be_select_auth_object(
- const hobject_t &obj,
- const map<int,ScrubMap*> &maps)
-{
- map<int, ScrubMap *>::const_iterator auth = maps.end();
- for (map<int, ScrubMap *>::const_iterator j = maps.begin();
- j != maps.end();
- ++j) {
- map<hobject_t, ScrubMap::object>::iterator i =
- j->second->objects.find(obj);
- if (i == j->second->objects.end()) {
- continue;
- }
- if (auth == maps.end()) {
- // Something is better than nothing
- // TODO: something is NOT better than nothing, do something like
- // unfound_lost if no valid copies can be found, or just mark unfound
- auth = j;
- dout(10) << __func__ << ": selecting osd " << j->first
- << " for obj " << obj
- << ", auth == maps.end()"
- << dendl;
- continue;
- }
- if (i->second.read_error) {
- // scrub encountered read error, probably corrupt
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", read_error"
- << dendl;
- continue;
- }
- map<string, bufferptr>::iterator k = i->second.attrs.find(OI_ATTR);
- if (k == i->second.attrs.end()) {
- // no object info on object, probably corrupt
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", no oi attr"
- << dendl;
- continue;
- }
- bufferlist bl;
- bl.push_back(k->second);
- object_info_t oi;
- try {
- bufferlist::iterator bliter = bl.begin();
- ::decode(oi, bliter);
- } catch (...) {
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", corrupt oi attr"
- << dendl;
- // invalid object info, probably corrupt
- continue;
- }
- if (oi.size != i->second.size) {
- // invalid size, probably corrupt
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", size mismatch"
- << dendl;
- // invalid object info, probably corrupt
- continue;
+ ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(
+ coll,
+ ghobject_t(
+ poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+ assert(iter);
+ uint64_t keys_scanned = 0;
+ for (iter->seek_to_first(); iter->valid() ; iter->next()) {
+ if (cct->_conf->osd_scan_list_ping_tp_interval &&
+ (keys_scanned % cct->_conf->osd_scan_list_ping_tp_interval == 0)) {
+ handle.reset_tp_timeout();
}
- dout(10) << __func__ << ": selecting osd " << j->first
- << " for obj " << obj
- << dendl;
- auth = j;
- }
- return auth;
-}
+ ++keys_scanned;
-void ReplicatedBackend::be_compare_scrubmaps(const map<int,ScrubMap*> &maps,
- map<hobject_t, set<int> > &missing,
- map<hobject_t, set<int> > &inconsistent,
- map<hobject_t, int> &authoritative,
- map<hobject_t, set<int> > &invalid_snapcolls,
- int &shallow_errors,
- int &deep_errors,
- const pg_t pgid,
- const vector<int> &acting,
- ostream &errorstream)
-{
- map<hobject_t,ScrubMap::object>::const_iterator i;
- map<int, ScrubMap *>::const_iterator j;
- set<hobject_t> master_set;
+ dout(25) << "CRC key " << iter->key() << " value "
+ << string(iter->value().c_str(), iter->value().length()) << dendl;
- // Construct master set
- for (j = maps.begin(); j != maps.end(); ++j) {
- for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) {
- master_set.insert(i->first);
- }
+ ::encode(iter->key(), bl);
+ ::encode(iter->value(), bl);
+ oh << bl;
+ bl.clear();
}
-
- // Check maps against master set and each other
- for (set<hobject_t>::const_iterator k = master_set.begin();
- k != master_set.end();
- ++k) {
- map<int, ScrubMap *>::const_iterator auth = be_select_auth_object(*k, maps);
- assert(auth != maps.end());
- set<int> cur_missing;
- set<int> cur_inconsistent;
- for (j = maps.begin(); j != maps.end(); ++j) {
- if (j == auth)
- continue;
- if (j->second->objects.count(*k)) {
- // Compare
- stringstream ss;
- enum scrub_error_type error = be_compare_scrub_objects(auth->second->objects[*k],
- j->second->objects[*k],
- ss);
- if (error != CLEAN) {
- cur_inconsistent.insert(j->first);
- if (error == SHALLOW_ERROR)
- ++shallow_errors;
- else
- ++deep_errors;
- errorstream << pgid << " osd." << acting[j->first]
- << ": soid " << *k << " " << ss.str() << std::endl;
- }
- } else {
- cur_missing.insert(j->first);
- ++shallow_errors;
- errorstream << pgid
- << " osd." << acting[j->first]
- << " missing " << *k << std::endl;
- }
- }
- assert(auth != maps.end());
- if (!cur_missing.empty()) {
- missing[*k] = cur_missing;
- }
- if (!cur_inconsistent.empty()) {
- inconsistent[*k] = cur_inconsistent;
- }
- if (!cur_inconsistent.empty() || !cur_missing.empty()) {
- authoritative[*k] = auth->first;
- }
+ if (iter->status() == -EIO) {
+ dout(25) << "_scan_list " << poid << " got "
+ << r << " on omap scan, read_error" << dendl;
+ o.read_error = true;
}
+
+ //Store final calculated CRC32 of omap header & key/values
+ o.omap_digest = oh.digest();
+ o.omap_digest_present = true;
}
diff --git a/src/osd/ReplicatedBackend.h b/src/osd/ReplicatedBackend.h
index 0f9cf56..64e91a6 100644
--- a/src/osd/ReplicatedBackend.h
+++ b/src/osd/ReplicatedBackend.h
@@ -23,26 +23,19 @@
struct C_ReplicatedBackend_OnPullComplete;
class ReplicatedBackend : public PGBackend {
struct RPGHandle : public PGBackend::RecoveryHandle {
- map<int, vector<PushOp> > pushes;
- map<int, vector<PullOp> > pulls;
+ map<pg_shard_t, vector<PushOp> > pushes;
+ map<pg_shard_t, vector<PullOp> > pulls;
};
friend struct C_ReplicatedBackend_OnPullComplete;
-private:
- bool temp_created;
- const coll_t temp_coll;
- coll_t get_temp_coll() const {
- return temp_coll;
- }
- bool have_temp_coll() const { return temp_created; }
-
- // Track contents of temp collection, clear on reset
- set<hobject_t> temp_contents;
public:
- coll_t coll;
- OSDService *osd;
CephContext *cct;
- ReplicatedBackend(PGBackend::Listener *pg, coll_t coll, OSDService *osd);
+ ReplicatedBackend(
+ PGBackend::Listener *pg,
+ coll_t coll,
+ coll_t temp_coll,
+ ObjectStore *store,
+ CephContext *cct);
/// @see PGBackend::open_recovery_op
RPGHandle *_open_recovery_op() {
@@ -60,6 +53,7 @@ public:
/// @see PGBackend::recover_object
void recover_object(
const hobject_t &hoid,
+ eversion_t v,
ObjectContextRef head,
ObjectContextRef obc,
RecoveryHandle *h
@@ -75,38 +69,40 @@ public:
OpRequestRef op
);
- void on_change(ObjectStore::Transaction *t);
+ void _on_change(ObjectStore::Transaction *t);
void clear_state();
void on_flushed();
- void temp_colls(list<coll_t> *out) {
- if (temp_created)
- out->push_back(temp_coll);
+ class RPCRecPred : public IsRecoverablePredicate {
+ public:
+ bool operator()(const set<pg_shard_t> &have) const {
+ return !have.empty();
+ }
+ };
+ IsRecoverablePredicate *get_is_recoverable_predicate() {
+ return new RPCRecPred;
}
- void split_colls(
- pg_t child,
- int split_bits,
- int seed,
- ObjectStore::Transaction *t) {
- coll_t target = coll_t::make_temp_coll(child);
- if (!temp_created)
- return;
- t->create_collection(target);
- t->split_collection(
- temp_coll,
- split_bits,
- seed,
- target);
+
+ class RPCReadPred : public IsReadablePredicate {
+ pg_shard_t whoami;
+ public:
+ RPCReadPred(pg_shard_t whoami) : whoami(whoami) {}
+ bool operator()(const set<pg_shard_t> &have) const {
+ return have.count(whoami);
+ }
+ };
+ IsReadablePredicate *get_is_readable_predicate() {
+ return new RPCReadPred(get_parent()->whoami_shard());
}
virtual void dump_recovery_info(Formatter *f) const {
{
f->open_array_section("pull_from_peer");
- for (map<int, set<hobject_t> >::const_iterator i = pull_from_peer.begin();
+ for (map<pg_shard_t, set<hobject_t> >::const_iterator i = pull_from_peer.begin();
i != pull_from_peer.end();
++i) {
f->open_object_section("pulling_from");
- f->dump_int("pull_from", i->first);
+ f->dump_stream("pull_from") << i->first;
{
f->open_array_section("pulls");
for (set<hobject_t>::const_iterator j = i->second.begin();
@@ -125,7 +121,7 @@ public:
}
{
f->open_array_section("pushing");
- for (map<hobject_t, map<int, PushInfo> >::const_iterator i =
+ for (map<hobject_t, map<pg_shard_t, PushInfo> >::const_iterator i =
pushing.begin();
i != pushing.end();
++i) {
@@ -133,11 +129,11 @@ public:
f->dump_stream("pushing") << i->first;
{
f->open_array_section("pushing_to");
- for (map<int, PushInfo>::const_iterator j = i->second.begin();
+ for (map<pg_shard_t, PushInfo>::const_iterator j = i->second.begin();
j != i->second.end();
++j) {
f->open_object_section("push_progress");
- f->dump_stream("object_pushing") << j->first;
+ f->dump_stream("pushing_to") << j->first;
{
f->open_object_section("push_info");
j->second.dump(f);
@@ -153,30 +149,6 @@ public:
}
}
- /// List objects in collection
- int objects_list_partial(
- const hobject_t &begin,
- int min,
- int max,
- snapid_t seq,
- vector<hobject_t> *ls,
- hobject_t *next);
-
- int objects_list_range(
- const hobject_t &start,
- const hobject_t &end,
- snapid_t seq,
- vector<hobject_t> *ls);
-
- int objects_get_attr(
- const hobject_t &hoid,
- const string &attr,
- bufferlist *out);
-
- int objects_get_attrs(
- const hobject_t &hoid,
- map<string, bufferlist> *out);
-
int objects_read_sync(
const hobject_t &hoid,
uint64_t off,
@@ -210,7 +182,7 @@ private:
}
}
};
- map<hobject_t, map<int, PushInfo> > pushing;
+ map<hobject_t, map<pg_shard_t, PushInfo> > pushing;
// pull
struct PullInfo {
@@ -238,18 +210,10 @@ private:
}
};
- coll_t get_temp_coll(ObjectStore::Transaction *t);
- void add_temp_obj(const hobject_t &oid) {
- temp_contents.insert(oid);
- }
- void clear_temp_obj(const hobject_t &oid) {
- temp_contents.erase(oid);
- }
-
map<hobject_t, PullInfo> pulling;
// Reverse mapping from osd peer to objects beging pulled from that peer
- map<int, set<hobject_t> > pull_from_peer;
+ map<pg_shard_t, set<hobject_t> > pull_from_peer;
void sub_op_push(OpRequestRef op);
void sub_op_push_reply(OpRequestRef op);
@@ -267,13 +231,13 @@ private:
void do_pull(OpRequestRef op);
void do_push_reply(OpRequestRef op);
- bool handle_push_reply(int peer, PushReplyOp &op, PushOp *reply);
- void handle_pull(int peer, PullOp &op, PushOp *reply);
+ bool handle_push_reply(pg_shard_t peer, PushReplyOp &op, PushOp *reply);
+ void handle_pull(pg_shard_t peer, PullOp &op, PushOp *reply);
bool handle_pull_response(
- int from, PushOp &op, PullOp *response,
+ pg_shard_t from, PushOp &op, PullOp *response,
list<hobject_t> *to_continue,
ObjectStore::Transaction *t);
- void handle_push(int from, PushOp &op, PushReplyOp *response,
+ void handle_push(pg_shard_t from, PushOp &op, PushReplyOp *response,
ObjectStore::Transaction *t);
static void trim_pushed_data(const interval_set<uint64_t> ©_subset,
@@ -281,18 +245,18 @@ private:
bufferlist data_received,
interval_set<uint64_t> *intervals_usable,
bufferlist *data_usable);
- void _failed_push(int from, const hobject_t &soid);
+ void _failed_push(pg_shard_t from, const hobject_t &soid);
- void send_pushes(int prio, map<int, vector<PushOp> > &pushes);
+ void send_pushes(int prio, map<pg_shard_t, vector<PushOp> > &pushes);
void prep_push_op_blank(const hobject_t& soid, PushOp *op);
- int send_push_op_legacy(int priority, int peer,
+ int send_push_op_legacy(int priority, pg_shard_t peer,
PushOp &pop);
- int send_pull_legacy(int priority, int peer,
+ int send_pull_legacy(int priority, pg_shard_t peer,
const ObjectRecoveryInfo& recovery_info,
ObjectRecoveryProgress progress);
void send_pulls(
int priority,
- map<int, vector<PullOp> > &pulls);
+ map<pg_shard_t, vector<PullOp> > &pulls);
int build_push_op(const ObjectRecoveryInfo &recovery_info,
const ObjectRecoveryProgress &progress,
@@ -305,7 +269,7 @@ private:
const interval_set<uint64_t> &intervals_included,
bufferlist data_included,
bufferlist omap_header,
- map<string, bufferptr> &attrs,
+ map<string, bufferlist> &attrs,
map<string, bufferlist> &omap_entries,
ObjectStore::Transaction *t);
void submit_push_complete(ObjectRecoveryInfo &recovery_info,
@@ -317,6 +281,7 @@ private:
interval_set<uint64_t>& data_subset,
map<hobject_t, interval_set<uint64_t> >& clone_subsets);
void prepare_pull(
+ eversion_t v,
const hobject_t& soid,
ObjectContextRef headctx,
RPGHandle *h);
@@ -325,13 +290,13 @@ private:
ObjectContextRef obj,
RPGHandle *h);
void prep_push_to_replica(
- ObjectContextRef obc, const hobject_t& soid, int peer,
+ ObjectContextRef obc, const hobject_t& soid, pg_shard_t peer,
PushOp *pop);
void prep_push(ObjectContextRef obc,
- const hobject_t& oid, int dest,
+ const hobject_t& oid, pg_shard_t dest,
PushOp *op);
void prep_push(ObjectContextRef obc,
- const hobject_t& soid, int peer,
+ const hobject_t& soid, pg_shard_t peer,
eversion_t version,
interval_set<uint64_t> &data_subset,
map<hobject_t, interval_set<uint64_t> >& clone_subsets,
@@ -351,8 +316,8 @@ private:
*/
struct InProgressOp {
tid_t tid;
- set<int> waiting_for_commit;
- set<int> waiting_for_applied;
+ set<pg_shard_t> waiting_for_commit;
+ set<pg_shard_t> waiting_for_applied;
Context *on_commit;
Context *on_applied;
OpRequestRef op;
@@ -386,56 +351,6 @@ public:
OpRequestRef op
);
- void rollback_setattrs(
- const hobject_t &hoid,
- map<string, boost::optional<bufferlist> > &old_attrs,
- ObjectStore::Transaction *t) {
- map<string, bufferlist> to_set;
- set<string> to_remove;
- for (map<string, boost::optional<bufferlist> >::iterator i = old_attrs.begin();
- i != old_attrs.end();
- ++i) {
- if (i->second) {
- to_set[i->first] = i->second.get();
- } else {
- t->rmattr(coll, hoid, i->first);
- }
- }
- t->setattrs(coll, hoid, to_set);
- }
-
- void rollback_append(
- const hobject_t &hoid,
- uint64_t old_size,
- ObjectStore::Transaction *t) {
- t->truncate(coll, hoid, old_size);
- }
-
- void rollback_stash(
- const hobject_t &hoid,
- version_t old_version,
- ObjectStore::Transaction *t) {
- t->remove(coll, hoid);
- t->collection_move_rename(
- coll,
- ghobject_t(hoid, old_version, 0),
- coll,
- hoid);
- }
-
- void rollback_create(
- const hobject_t &hoid,
- ObjectStore::Transaction *t) {
- t->remove(coll, hoid);
- }
-
- void trim_stashed_object(
- const hobject_t &hoid,
- version_t old_version,
- ObjectStore::Transaction *t) {
- t->remove(coll, ghobject_t(hoid, old_version, 0));
- }
-
private:
void issue_op(
const hobject_t &soid,
@@ -490,24 +405,12 @@ private:
void sub_op_modify_applied(RepModifyRef rm);
void sub_op_modify_commit(RepModifyRef rm);
bool scrub_supported() { return true; }
- void be_scan_list(ScrubMap &map, const vector<hobject_t> &ls, bool deep,
- ThreadPool::TPHandle &handle);
- enum scrub_error_type be_compare_scrub_objects(
- const ScrubMap::object &auth,
- const ScrubMap::object &candidate,
- ostream &errorstream);
- map<int, ScrubMap *>::const_iterator be_select_auth_object(
+
+ void be_deep_scrub(
const hobject_t &obj,
- const map<int,ScrubMap*> &maps);
- void be_compare_scrubmaps(const map<int,ScrubMap*> &maps,
- map<hobject_t, set<int> > &missing,
- map<hobject_t, set<int> > &inconsistent,
- map<hobject_t, int> &authoritative,
- map<hobject_t, set<int> > &invalid_snapcolls,
- int &shallow_errors, int &deep_errors,
- const pg_t pgid,
- const vector<int> &acting,
- ostream &errorstream);
+ ScrubMap::object &o,
+ ThreadPool::TPHandle &handle);
+ uint64_t be_get_ondisk_size(uint64_t logical_size) { return logical_size; }
};
#endif
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 2068af7..f0987bf 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -82,7 +82,7 @@ PGLSFilter::~PGLSFilter()
}
static void log_subop_stats(
- OSDService *osd,
+ PerfCounters *logger,
OpRequestRef op, int tag_inb, int tag_lat)
{
utime_t now = ceph_clock_now(g_ceph_context);
@@ -91,14 +91,14 @@ static void log_subop_stats(
uint64_t inb = op->get_req()->get_data().length();
- osd->logger->inc(l_osd_sop);
+ logger->inc(l_osd_sop);
- osd->logger->inc(l_osd_sop_inb, inb);
- osd->logger->tinc(l_osd_sop_lat, latency);
+ logger->inc(l_osd_sop_inb, inb);
+ logger->tinc(l_osd_sop_lat, latency);
if (tag_inb)
- osd->logger->inc(tag_inb, inb);
- osd->logger->tinc(tag_lat, latency);
+ logger->inc(tag_inb, inb);
+ logger->tinc(tag_lat, latency);
}
struct OnReadComplete : public Context {
@@ -162,11 +162,13 @@ public:
ctx->copy_cb = NULL;
if (r < 0) {
if (r != -ECANCELED) { // on cancel just toss it out; client resends
- ctx->pg->osd->reply_op_error(ctx->op, r);
+ if (ctx->op)
+ ctx->pg->osd->reply_op_error(ctx->op, r);
} else if (results->should_requeue) {
- ctx->pg->requeue_op(ctx->op);
+ if (ctx->op)
+ ctx->pg->requeue_op(ctx->op);
}
- ctx->pg->close_op_ctx(ctx);
+ ctx->pg->close_op_ctx(ctx, r);
}
}
@@ -191,7 +193,6 @@ void ReplicatedPG::on_local_recover_start(
{
pg_log.revise_have(oid, eversion_t());
remove_snap_mapped_object(*t, oid);
- t->remove(coll, oid);
}
void ReplicatedPG::on_local_recover(
@@ -202,6 +203,7 @@ void ReplicatedPG::on_local_recover(
ObjectStore::Transaction *t
)
{
+ dout(10) << __func__ << ": " << hoid << dendl;
ObjectRecoveryInfo recovery_info(_recovery_info);
if (recovery_info.soid.snap < CEPH_NOSNAP) {
assert(recovery_info.oi.snaps.size());
@@ -253,14 +255,17 @@ void ReplicatedPG::on_local_recover(
t->register_on_applied_sync(new C_OSD_OndiskWriteUnlock(obc));
publish_stats_to_osd();
- if (waiting_for_missing_object.count(hoid)) {
- dout(20) << " kicking waiters on " << hoid << dendl;
- requeue_ops(waiting_for_missing_object[hoid]);
- waiting_for_missing_object.erase(hoid);
- if (pg_log.get_missing().missing.size() == 0) {
- requeue_ops(waiting_for_all_missing);
- waiting_for_all_missing.clear();
- }
+ assert(missing_loc.needs_recovery(hoid));
+ missing_loc.add_location(hoid, pg_whoami);
+ if (!is_unreadable_object(hoid) &&
+ waiting_for_unreadable_object.count(hoid)) {
+ dout(20) << " kicking unreadable waiters on " << hoid << dendl;
+ requeue_ops(waiting_for_unreadable_object[hoid]);
+ waiting_for_unreadable_object.erase(hoid);
+ }
+ if (pg_log.get_missing().missing.size() == 0) {
+ requeue_ops(waiting_for_all_missing);
+ waiting_for_all_missing.clear();
}
} else {
t->register_on_applied(
@@ -283,21 +288,34 @@ void ReplicatedPG::on_local_recover(
void ReplicatedPG::on_global_recover(
const hobject_t &soid)
{
+ missing_loc.recovered(soid);
publish_stats_to_osd();
dout(10) << "pushed " << soid << " to all replicas" << dendl;
map<hobject_t, ObjectContextRef>::iterator i = recovering.find(soid);
assert(i != recovering.end());
+ if (backfills_in_flight.count(soid)) {
+ list<OpRequestRef> requeue_list;
+ i->second->drop_backfill_read(&requeue_list);
+ requeue_ops(requeue_list);
+ backfills_in_flight.erase(soid);
+ }
recovering.erase(i);
finish_recovery_op(soid);
if (waiting_for_degraded_object.count(soid)) {
+ dout(20) << " kicking degraded waiters on " << soid << dendl;
requeue_ops(waiting_for_degraded_object[soid]);
waiting_for_degraded_object.erase(soid);
}
+ if (waiting_for_unreadable_object.count(soid)) {
+ dout(20) << " kicking unreadable waiters on " << soid << dendl;
+ requeue_ops(waiting_for_unreadable_object[soid]);
+ waiting_for_unreadable_object.erase(soid);
+ }
finish_degraded_object(soid);
}
void ReplicatedPG::on_peer_recover(
- int peer,
+ pg_shard_t peer,
const hobject_t &soid,
const ObjectRecoveryInfo &recovery_info,
const object_stat_sum_t &stat)
@@ -306,23 +324,50 @@ void ReplicatedPG::on_peer_recover(
publish_stats_to_osd();
// done!
peer_missing[peer].got(soid, recovery_info.version);
- if (is_backfill_targets(peer) && backfills_in_flight.count(soid)) {
- map<hobject_t, ObjectContextRef>::iterator i = recovering.find(soid);
- assert(i != recovering.end());
- list<OpRequestRef> requeue_list;
- i->second->drop_backfill_read(&requeue_list);
- requeue_ops(requeue_list);
- backfills_in_flight.erase(soid);
- }
}
void ReplicatedPG::begin_peer_recover(
- int peer,
+ pg_shard_t peer,
const hobject_t soid)
{
peer_missing[peer].revise_have(soid, eversion_t());
}
+void ReplicatedPG::schedule_work(
+ GenContext<ThreadPool::TPHandle&> *c)
+{
+ osd->gen_wq.queue(c);
+}
+
+void ReplicatedPG::send_message_osd_cluster(
+ int peer, Message *m, epoch_t from_epoch)
+{
+ osd->send_message_osd_cluster(peer, m, from_epoch);
+}
+
+void ReplicatedPG::send_message_osd_cluster(
+ Message *m, Connection *con)
+{
+ osd->send_message_osd_cluster(m, con);
+}
+
+void ReplicatedPG::send_message_osd_cluster(
+ Message *m, const ConnectionRef& con)
+{
+ osd->send_message_osd_cluster(m, con);
+}
+
+ConnectionRef ReplicatedPG::get_con_osd_cluster(
+ int peer, epoch_t from_epoch)
+{
+ return osd->get_con_osd_cluster(peer, from_epoch);
+}
+
+PerfCounters *ReplicatedPG::get_logger()
+{
+ return osd->logger;
+}
+
// =======================
// pg changes
@@ -345,36 +390,32 @@ bool ReplicatedPG::same_for_rep_modify_since(epoch_t e)
// ====================
// missing objects
-bool ReplicatedPG::is_missing_object(const hobject_t& soid)
+bool ReplicatedPG::is_missing_object(const hobject_t& soid) const
{
return pg_log.get_missing().missing.count(soid);
}
-void ReplicatedPG::wait_for_missing_object(const hobject_t& soid, OpRequestRef op)
+void ReplicatedPG::wait_for_unreadable_object(
+ const hobject_t& soid, OpRequestRef op)
{
- assert(is_missing_object(soid));
+ assert(is_unreadable_object(soid));
- const pg_missing_t &missing = pg_log.get_missing();
-
- // we don't have it (yet).
- map<hobject_t, pg_missing_t::item>::const_iterator g = missing.missing.find(soid);
- assert(g != missing.missing.end());
- const eversion_t &v(g->second.need);
+ eversion_t v;
+ bool needs_recovery = missing_loc.needs_recovery(soid, &v);
+ assert(needs_recovery);
map<hobject_t, ObjectContextRef>::const_iterator p = recovering.find(soid);
if (p != recovering.end()) {
dout(7) << "missing " << soid << " v " << v << ", already recovering." << dendl;
- }
- else if (missing_loc.find(soid) == missing_loc.end()) {
+ } else if (missing_loc.is_unfound(soid)) {
dout(7) << "missing " << soid << " v " << v << ", is unfound." << dendl;
- }
- else {
+ } else {
dout(7) << "missing " << soid << " v " << v << ", recovering." << dendl;
PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
recover_missing(soid, v, cct->_conf->osd_client_op_priority, h);
pgbackend->run_recovery_op(h, cct->_conf->osd_client_op_priority);
}
- waiting_for_missing_object[soid].push_back(op);
+ waiting_for_unreadable_object[soid].push_back(op);
op->mark_delayed("waiting for missing object");
}
@@ -388,8 +429,11 @@ bool ReplicatedPG::is_degraded_object(const hobject_t& soid)
if (pg_log.get_missing().missing.count(soid))
return true;
assert(actingbackfill.size() > 0);
- for (unsigned i = 1; i < actingbackfill.size(); i++) {
- int peer = actingbackfill[i];
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_shard_t peer = *i;
if (peer_missing.count(peer) &&
peer_missing[peer].missing.count(soid))
return true;
@@ -415,6 +459,11 @@ void ReplicatedPG::wait_for_degraded_object(const hobject_t& soid, OpRequestRef
<< soid
<< ", already recovering"
<< dendl;
+ } else if (missing_loc.is_unfound(soid)) {
+ dout(7) << "degraded "
+ << soid
+ << ", still unfound, waiting"
+ << dendl;
} else {
dout(7) << "degraded "
<< soid
@@ -422,8 +471,11 @@ void ReplicatedPG::wait_for_degraded_object(const hobject_t& soid, OpRequestRef
<< dendl;
eversion_t v;
assert(actingbackfill.size() > 0);
- for (unsigned i = 1; i < actingbackfill.size(); i++) {
- int peer = actingbackfill[i];
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_shard_t peer = *i;
if (peer_missing.count(peer) &&
peer_missing[peer].missing.count(soid)) {
v = peer_missing[peer].missing[soid].need;
@@ -548,14 +600,18 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss,
f->close_section();
if (backfill_targets.size() > 0) {
f->open_array_section("backfill_targets");
- for (vector<int>::iterator p = backfill_targets.begin(); p != backfill_targets.end(); ++p)
- f->dump_unsigned("osd", *p);
+ for (set<pg_shard_t>::iterator p = backfill_targets.begin();
+ p != backfill_targets.end();
+ ++p)
+ f->dump_stream("shard") << *p;
f->close_section();
}
if (actingbackfill.size() > 0) {
f->open_array_section("actingbackfill");
- for (vector<int>::iterator p = actingbackfill.begin(); p != actingbackfill.end(); ++p)
- f->dump_unsigned("osd", *p);
+ for (set<pg_shard_t>::iterator p = actingbackfill.begin();
+ p != actingbackfill.end();
+ ++p)
+ f->dump_stream("shard") << *p;
f->close_section();
}
f->open_object_section("info");
@@ -564,11 +620,11 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss,
f->close_section();
f->open_array_section("peer_info");
- for (map<int,pg_info_t>::iterator p = peer_info.begin();
+ for (map<pg_shard_t, pg_info_t>::iterator p = peer_info.begin();
p != peer_info.end();
++p) {
f->open_object_section("info");
- f->dump_unsigned("peer", p->first);
+ f->dump_stream("peer") << p->first;
p->second.dump(f.get());
f->close_section();
}
@@ -596,7 +652,7 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss,
return -EROFS;
}
- int unfound = missing.num_missing() - missing_loc.size();
+ int unfound = missing_loc.num_unfound();
if (!unfound) {
ss << "pg has no unfound objects";
return 0; // make command idempotent
@@ -649,10 +705,13 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss,
p->second.dump(f.get()); // have, need keys
{
f->open_array_section("locations");
- map<hobject_t,set<int> >::iterator q = missing_loc.find(p->first);
- if (q != missing_loc.end())
- for (set<int>::iterator r = q->second.begin(); r != q->second.end(); ++r)
- f->dump_int("osd", *r);
+ if (missing_loc.needs_recovery(p->first)) {
+ for (set<pg_shard_t>::iterator r =
+ missing_loc.get_locations(p->first).begin();
+ r != missing_loc.get_locations(p->first).end();
+ ++r)
+ f->dump_stream("shard") << *r;
+ }
f->close_section();
}
f->close_section();
@@ -727,7 +786,7 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
// fall through
case CEPH_OSD_OP_PGLS:
- if (m->get_pg() != info.pgid) {
+ if (m->get_pg() != info.pgid.pgid) {
dout(10) << " pgls pg=" << m->get_pg() << " != " << info.pgid << dendl;
result = 0; // hmm?
} else {
@@ -780,8 +839,8 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
if (mcand == lcand) {
candidate = mcand;
if (!mcand.is_max()) {
- ls_iter++;
- missing_iter++;
+ ++ls_iter;
+ ++missing_iter;
}
} else if (mcand < lcand) {
candidate = mcand;
@@ -900,6 +959,11 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
result = -ENOENT;
break;
}
+ if (!pool.info.is_replicated()) {
+ // FIXME: EC not supported yet
+ result = -EOPNOTSUPP;
+ break;
+ }
result = osd->store->read(coll, oid, 0, 0, osd_op.outdata);
}
}
@@ -914,7 +978,8 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
// reply
MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(),
- CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
+ CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK,
+ false);
reply->claim_op_out_data(ops);
reply->set_result(result);
reply->set_reply_versions(info.last_update, info.last_user_version);
@@ -963,14 +1028,19 @@ void ReplicatedPG::calc_trim_to()
}
ReplicatedPG::ReplicatedPG(OSDService *o, OSDMapRef curmap,
- const PGPool &_pool, pg_t p, const hobject_t& oid,
+ const PGPool &_pool, spg_t p, const hobject_t& oid,
const hobject_t& ioid) :
PG(o, curmap, _pool, p, oid, ioid),
- pgbackend(new ReplicatedBackend(this, coll_t(p), o)),
+ pgbackend(
+ PGBackend::build_pg_backend(
+ _pool.info, this, coll_t(p), coll_t::make_temp_coll(p), o->store, cct)),
snapset_contexts_lock("ReplicatedPG::snapset_contexts"),
temp_seq(0),
snap_trimmer_machine(this)
{
+ missing_loc.set_backend_predicates(
+ pgbackend->get_is_readable_predicate(),
+ pgbackend->get_is_recoverable_predicate());
snap_trimmer_machine.initiate();
}
@@ -1051,9 +1121,11 @@ void ReplicatedPG::do_request(
hobject_t ReplicatedPG::earliest_backfill() const
{
hobject_t e = hobject_t::get_max();
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
- map<int, pg_info_t>::const_iterator iter = peer_info.find(bt);
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
+ map<pg_shard_t, pg_info_t>::const_iterator iter = peer_info.find(bt);
assert(iter != peer_info.end());
if (iter->second.last_backfill < e)
e = iter->second.last_backfill;
@@ -1071,9 +1143,12 @@ hobject_t ReplicatedPG::earliest_backfill() const
// take the larger of last_backfill_started and the replicas last_backfill.
bool ReplicatedPG::check_src_targ(const hobject_t& soid, const hobject_t& toid) const
{
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
- map<int, pg_info_t>::const_iterator iter = peer_info.find(bt);
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_shard_t bt = *i;
+ map<pg_shard_t, pg_info_t>::const_iterator iter = peer_info.find(bt);
assert(iter != peer_info.end());
if (toid <= MAX(last_backfill_started, iter->second.last_backfill) &&
@@ -1126,8 +1201,8 @@ void ReplicatedPG::do_op(OpRequestRef op)
}
// missing object?
- if (is_missing_object(head)) {
- wait_for_missing_object(head, op);
+ if (is_unreadable_object(head)) {
+ wait_for_unreadable_object(head, op);
return;
}
@@ -1141,8 +1216,8 @@ void ReplicatedPG::do_op(OpRequestRef op)
hobject_t snapdir(m->get_oid(), m->get_object_locator().key,
CEPH_SNAPDIR, m->get_pg().ps(), info.pgid.pool(),
m->get_object_locator().nspace);
- if (is_missing_object(snapdir)) {
- wait_for_missing_object(snapdir, op);
+ if (is_unreadable_object(snapdir)) {
+ wait_for_unreadable_object(snapdir, op);
return;
}
@@ -1178,9 +1253,14 @@ void ReplicatedPG::do_op(OpRequestRef op)
!(m->get_flags() & CEPH_OSD_FLAG_LOCALIZE_READS))) {
// missing the specific snap we need; requeue and wait.
assert(!can_create); // only happens on a read
- wait_for_missing_object(missing_oid, op);
+ wait_for_unreadable_object(missing_oid, op);
return;
}
+ } else if (r == 0 && is_unreadable_object(obc->obs.oi.soid)) {
+ dout(10) << __func__ << ": clone " << obc->obs.oi.soid
+ << " is unreadable, waiting" << dendl;
+ wait_for_unreadable_object(obc->obs.oi.soid, op);
+ return;
}
if (hit_set) {
@@ -1189,10 +1269,13 @@ void ReplicatedPG::do_op(OpRequestRef op)
hit_set_start_stamp + pool.info.hit_set_period <= m->get_recv_stamp()) {
hit_set_persist();
}
+
+ if (agent_state)
+ agent_choose_mode();
}
if ((m->get_flags() & CEPH_OSD_FLAG_IGNORE_CACHE) == 0 &&
- maybe_handle_cache(op, obc, r, missing_oid))
+ maybe_handle_cache(op, write_ordered, obc, r, missing_oid, false))
return;
if (r) {
@@ -1253,13 +1336,13 @@ void ReplicatedPG::do_op(OpRequestRef op)
int r;
if (src_oid.is_head() && is_missing_object(src_oid)) {
- wait_for_missing_object(src_oid, op);
+ wait_for_unreadable_object(src_oid, op);
} else if ((r = find_object_context(
src_oid, &sobc, false, &wait_oid)) == -EAGAIN) {
// missing the specific snap we need; requeue and wait.
- wait_for_missing_object(wait_oid, op);
+ wait_for_unreadable_object(wait_oid, op);
} else if (r) {
- if (!maybe_handle_cache(op, sobc, r, wait_oid, true))
+ if (!maybe_handle_cache(op, write_ordered, sobc, r, wait_oid, true))
osd->reply_op_error(op, r);
} else if (sobc->obs.oi.is_whiteout()) {
osd->reply_op_error(op, -ENOENT);
@@ -1317,9 +1400,9 @@ void ReplicatedPG::do_op(OpRequestRef op)
int r = find_object_context(clone_oid, &sobc, false, &wait_oid);
if (r == -EAGAIN) {
// missing the specific snap we need; requeue and wait.
- wait_for_missing_object(wait_oid, op);
+ wait_for_unreadable_object(wait_oid, op);
} else if (r) {
- if (!maybe_handle_cache(op, sobc, r, wait_oid, true))
+ if (!maybe_handle_cache(op, write_ordered, sobc, r, wait_oid, true))
osd->reply_op_error(op, r);
} else {
dout(10) << " clone_oid " << clone_oid << " obc " << sobc << dendl;
@@ -1338,6 +1421,10 @@ void ReplicatedPG::do_op(OpRequestRef op)
this);
ctx->op_t = pgbackend->get_transaction();
ctx->obc = obc;
+
+ if (!obc->obs.exists)
+ ctx->snapset_obc = get_object_context(obc->obs.oi.soid.get_snapdir(), false);
+
if (m->get_flags() & CEPH_OSD_FLAG_SKIPRWLOCKS) {
dout(20) << __func__ << ": skipping rw locks" << dendl;
} else if (m->get_flags() & CEPH_OSD_FLAG_FLUSH) {
@@ -1348,14 +1435,13 @@ void ReplicatedPG::do_op(OpRequestRef op)
map<hobject_t,FlushOpRef>::iterator p = flush_ops.find(obc->obs.oi.soid);
if (p == flush_ops.end()) {
dout(10) << __func__ << " no flush in progress, aborting" << dendl;
- close_op_ctx(ctx);
- osd->reply_op_error(op, -EINVAL);
+ reply_ctx(ctx, -EINVAL);
return;
}
} else if (!get_rw_locks(ctx)) {
dout(20) << __func__ << " waiting for rw locks " << dendl;
op->mark_delayed("waiting for rw locks");
- close_op_ctx(ctx);
+ close_op_ctx(ctx, -EBUSY);
return;
}
@@ -1363,14 +1449,12 @@ void ReplicatedPG::do_op(OpRequestRef op)
// This object is lost. Reading from it returns an error.
dout(20) << __func__ << ": object " << obc->obs.oi.soid
<< " is lost" << dendl;
- close_op_ctx(ctx);
- osd->reply_op_error(op, -ENFILE);
+ reply_ctx(ctx, -ENFILE);
return;
}
if (!op->may_write() && !op->may_cache() && (!obc->obs.exists ||
obc->obs.oi.is_whiteout())) {
- close_op_ctx(ctx);
- osd->reply_op_error(op, -ENOENT);
+ reply_ctx(ctx, -ENOENT);
return;
}
@@ -1380,7 +1464,9 @@ void ReplicatedPG::do_op(OpRequestRef op)
execute_ctx(ctx);
}
-bool ReplicatedPG::maybe_handle_cache(OpRequestRef op, ObjectContextRef obc,
+bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
+ bool write_ordered,
+ ObjectContextRef obc,
int r, const hobject_t& missing_oid,
bool must_promote)
{
@@ -1413,6 +1499,17 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op, ObjectContextRef obc,
if (obc.get() && obc->obs.exists) {
return false;
}
+ if (agent_state &&
+ agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
+ if (!op->may_write() && !op->may_cache() && !write_ordered) {
+ dout(20) << __func__ << " cache pool full, redirecting read" << dendl;
+ do_cache_redirect(op, obc);
+ return true;
+ }
+ dout(20) << __func__ << " cache pool full, waiting" << dendl;
+ waiting_for_cache_not_full.push_back(op);
+ return true;
+ }
if (!must_promote && can_skip_promote(op, obc)) {
return false;
}
@@ -1471,7 +1568,7 @@ void ReplicatedPG::do_cache_redirect(OpRequestRef op, ObjectContextRef obc)
MOSDOp *m = static_cast<MOSDOp*>(op->get_req());
int flags = m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK);
MOSDOpReply *reply = new MOSDOpReply(m, -ENOENT,
- get_osdmap()->get_epoch(), flags);
+ get_osdmap()->get_epoch(), flags, false);
request_redirect_t redir(m->get_object_locator(), pool.info.tier_of);
reply->set_redirect(redir);
dout(10) << "sending redirect to pool " << pool.info.tier_of << " for op "
@@ -1544,11 +1641,11 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
if (already_complete(oldv)) {
reply_ctx(ctx, 0, oldv, entry->user_version);
} else {
- close_op_ctx(ctx);
+ close_op_ctx(ctx, -EBUSY);
if (m->wants_ack()) {
if (already_ack(oldv)) {
- MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0);
+ MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, false);
reply->add_flags(CEPH_OSD_FLAG_ACK);
reply->set_reply_versions(oldv, entry->user_version);
osd->send_message_osd_client(reply, m->get_connection());
@@ -1633,7 +1730,7 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
if (result == -EAGAIN) {
// clean up after the ctx
- close_op_ctx(ctx);
+ close_op_ctx(ctx, result);
return;
}
@@ -1644,8 +1741,10 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
return;
}
+ bool successful_write = !ctx->op_t->empty() && op->may_write() && result >= 0;
// prepare the reply
- ctx->reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0);
+ ctx->reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0,
+ successful_write);
// Write operations aren't allowed to return a data payload because
// we can't do so reliably. If the client has to resend the request
@@ -1654,12 +1753,10 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
// possible to construct an operation that does a read, does a guard
// check (e.g., CMPXATTR), and then a write. Then we either succeed
// with the write, or return a CMPXATTR and the read value.
- if (!((ctx->op_t->empty() && !ctx->modify) || result < 0)) {
+ if (successful_write) {
// write. normalize the result code.
- if (result > 0) {
- dout(20) << " zeroing write result code " << result << dendl;
- result = 0;
- }
+ dout(20) << " zeroing write result code " << result << dendl;
+ result = 0;
}
ctx->reply->set_result(result);
@@ -1715,14 +1812,16 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
void ReplicatedPG::reply_ctx(OpContext *ctx, int r)
{
- osd->reply_op_error(ctx->op, r);
- close_op_ctx(ctx);
+ if (ctx->op)
+ osd->reply_op_error(ctx->op, r);
+ close_op_ctx(ctx, r);
}
void ReplicatedPG::reply_ctx(OpContext *ctx, int r, eversion_t v, version_t uv)
{
- osd->reply_op_error(ctx->op, r, v, uv);
- close_op_ctx(ctx);
+ if (ctx->op)
+ osd->reply_op_error(ctx->op, r, v, uv);
+ close_op_ctx(ctx, r);
}
void ReplicatedPG::log_op_stats(OpContext *ctx)
@@ -1868,9 +1967,11 @@ void ReplicatedPG::do_scan(
cct->_conf->osd_backfill_scan_max,
&bi,
handle);
- MOSDPGScan *reply = new MOSDPGScan(MOSDPGScan::OP_SCAN_DIGEST,
- get_osdmap()->get_epoch(), m->query_epoch,
- info.pgid, bi.begin, bi.end);
+ MOSDPGScan *reply = new MOSDPGScan(
+ MOSDPGScan::OP_SCAN_DIGEST,
+ pg_whoami,
+ get_osdmap()->get_epoch(), m->query_epoch,
+ spg_t(info.pgid.pgid, get_primary().shard), bi.begin, bi.end);
::encode(bi.objects, reply->get_data());
osd->send_message_osd_cluster(reply, m->get_connection());
}
@@ -1878,7 +1979,7 @@ void ReplicatedPG::do_scan(
case MOSDPGScan::OP_SCAN_DIGEST:
{
- int from = m->get_source().num();
+ pg_shard_t from = m->from;
// Check that from is in backfill_targets vector
assert(is_backfill_targets(from));
@@ -1920,7 +2021,7 @@ void ReplicatedBackend::_do_push(OpRequestRef op)
{
MOSDPGPush *m = static_cast<MOSDPGPush *>(op->get_req());
assert(m->get_header().type == MSG_OSD_PG_PUSH);
- int from = m->get_source().num();
+ pg_shard_t from = m->from;
vector<PushReplyOp> replies;
ObjectStore::Transaction *t = new ObjectStore::Transaction;
@@ -1932,6 +2033,7 @@ void ReplicatedBackend::_do_push(OpRequestRef op)
}
MOSDPGPushReply *reply = new MOSDPGPushReply;
+ reply->from = get_parent()->whoami_shard();
reply->set_priority(m->get_priority());
reply->pgid = get_info().pgid;
reply->map_epoch = m->map_epoch;
@@ -1939,8 +2041,8 @@ void ReplicatedBackend::_do_push(OpRequestRef op)
reply->compute_cost(cct);
t->register_on_complete(
- new C_OSD_SendMessageOnConn(
- osd, reply, m->get_connection()));
+ new PG_SendMessageOnConn(
+ get_parent(), reply, m->get_connection()));
t->register_on_applied(
new ObjectStore::C_DeleteTransaction(t));
@@ -1978,7 +2080,7 @@ void ReplicatedBackend::_do_pull_response(OpRequestRef op)
{
MOSDPGPush *m = static_cast<MOSDPGPush *>(op->get_req());
assert(m->get_header().type == MSG_OSD_PG_PUSH);
- int from = m->get_source().num();
+ pg_shard_t from = m->from;
vector<PullOp> replies(1);
ObjectStore::Transaction *t = new ObjectStore::Transaction;
@@ -1997,14 +2099,15 @@ void ReplicatedBackend::_do_pull_response(OpRequestRef op)
m->get_priority());
c->to_continue.swap(to_continue);
t->register_on_complete(
- new C_QueueInWQ(
- &osd->push_wq,
+ new PG_QueueAsync(
+ get_parent(),
get_parent()->bless_gencontext(c)));
}
replies.erase(replies.end() - 1);
if (replies.size()) {
MOSDPGPull *reply = new MOSDPGPull;
+ reply->from = parent->whoami_shard();
reply->set_priority(m->get_priority());
reply->pgid = get_info().pgid;
reply->map_epoch = m->map_epoch;
@@ -2012,8 +2115,8 @@ void ReplicatedBackend::_do_pull_response(OpRequestRef op)
reply->compute_cost(cct);
t->register_on_complete(
- new C_OSD_SendMessageOnConn(
- osd, reply, m->get_connection()));
+ new PG_SendMessageOnConn(
+ get_parent(), reply, m->get_connection()));
}
t->register_on_applied(
@@ -2025,9 +2128,9 @@ void ReplicatedBackend::do_pull(OpRequestRef op)
{
MOSDPGPull *m = static_cast<MOSDPGPull *>(op->get_req());
assert(m->get_header().type == MSG_OSD_PG_PULL);
- int from = m->get_source().num();
+ pg_shard_t from = m->from;
- map<int, vector<PushOp> > replies;
+ map<pg_shard_t, vector<PushOp> > replies;
for (vector<PullOp>::iterator i = m->pulls.begin();
i != m->pulls.end();
++i) {
@@ -2041,7 +2144,7 @@ void ReplicatedBackend::do_push_reply(OpRequestRef op)
{
MOSDPGPushReply *m = static_cast<MOSDPGPushReply *>(op->get_req());
assert(m->get_header().type == MSG_OSD_PG_PUSH_REPLY);
- int from = m->get_source().num();
+ pg_shard_t from = m->from;
vector<PushOp> replies(1);
for (vector<PushReplyOp>::iterator i = m->replies.begin();
@@ -2053,7 +2156,7 @@ void ReplicatedBackend::do_push_reply(OpRequestRef op)
}
replies.erase(replies.end() - 1);
- map<int, vector<PushOp> > _replies;
+ map<pg_shard_t, vector<PushOp> > _replies;
_replies[from].swap(replies);
send_pushes(m->get_priority(), _replies);
}
@@ -2071,9 +2174,11 @@ void ReplicatedPG::do_backfill(OpRequestRef op)
{
assert(cct->_conf->osd_kill_backfill_at != 1);
- MOSDPGBackfill *reply = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_FINISH_ACK,
- get_osdmap()->get_epoch(), m->query_epoch,
- info.pgid);
+ MOSDPGBackfill *reply = new MOSDPGBackfill(
+ MOSDPGBackfill::OP_BACKFILL_FINISH_ACK,
+ get_osdmap()->get_epoch(),
+ m->query_epoch,
+ spg_t(info.pgid.pgid, primary.shard));
reply->set_priority(cct->_conf->osd_recovery_op_priority);
osd->send_message_osd_cluster(reply, m->get_connection());
queue_peering_event(
@@ -2131,11 +2236,8 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
// get snap set context
if (!obc->ssc)
obc->ssc = get_snapset_context(
- coid.oid,
- coid.get_key(),
- coid.hash,
- false,
- coid.get_namespace());
+ coid,
+ false);
assert(obc->ssc);
SnapSet& snapset = obc->ssc->snapset;
@@ -2188,8 +2290,10 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
delta.num_objects--;
if (coi.is_dirty())
delta.num_objects_dirty--;
- if (coi.is_whiteout())
+ if (coi.is_whiteout()) {
+ dout(20) << __func__ << " trimming whiteout on " << coid << dendl;
delta.num_whiteouts--;
+ }
delta.num_object_clones--;
delta.num_bytes -= snapset.clone_size[last];
info.stats.stats.add(delta, obc->obs.oi.category);
@@ -2209,7 +2313,7 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
osd_reqid_t(),
ctx->mtime)
);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
set<snapid_t> snaps(
ctx->obc->obs.oi.snaps.begin(),
ctx->obc->obs.oi.snaps.end());
@@ -2246,14 +2350,14 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
osd_reqid_t(),
ctx->mtime)
);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
set<string> changing;
changing.insert(OI_ATTR);
ctx->obc->fill_in_setattrs(changing, &(ctx->log.back().mod_desc));
set<snapid_t> snaps(
ctx->obc->obs.oi.snaps.begin(),
ctx->obc->obs.oi.snaps.end());
- ctx->log.back().mod_desc.update_snaps(snaps);
+ ctx->log.back().mod_desc.update_snaps(old_snaps);
} else {
ctx->log.back().mod_desc.mark_unrollbackable();
}
@@ -2286,7 +2390,7 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
ctx->snapset_obc->obs.exists = false;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
t->stash(snapoid, ctx->at_version.version);
} else {
@@ -2321,7 +2425,7 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
::encode(ctx->snapset_obc->obs.oi, bl);
setattr_maybe_cache(ctx->snapset_obc, ctx, t, OI_ATTR, bl);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
set<string> changing;
changing.insert(OI_ATTR);
changing.insert(SS_ATTR);
@@ -2402,30 +2506,23 @@ int ReplicatedPG::do_xattr_cmp_u64(int op, __u64 v1, bufferlist& xattr)
int ReplicatedPG::do_xattr_cmp_str(int op, string& v1s, bufferlist& xattr)
{
- const char *v1, *v2;
- v1 = v1s.data();
- string v2s;
- if (xattr.length()) {
- v2s = string(xattr.c_str(), xattr.length());
- v2 = v2s.c_str();
- } else
- v2 = "";
+ string v2s(xattr.c_str(), xattr.length());
- dout(20) << "do_xattr_cmp_str '" << v1s << "' vs '" << v2 << "' op " << op << dendl;
+ dout(20) << "do_xattr_cmp_str '" << v1s << "' vs '" << v2s << "' op " << op << dendl;
switch (op) {
case CEPH_OSD_CMPXATTR_OP_EQ:
- return (strcmp(v1, v2) == 0);
+ return (v1s.compare(v2s) == 0);
case CEPH_OSD_CMPXATTR_OP_NE:
- return (strcmp(v1, v2) != 0);
+ return (v1s.compare(v2s) != 0);
case CEPH_OSD_CMPXATTR_OP_GT:
- return (strcmp(v1, v2) > 0);
+ return (v1s.compare(v2s) > 0);
case CEPH_OSD_CMPXATTR_OP_GTE:
- return (strcmp(v1, v2) >= 0);
+ return (v1s.compare(v2s) >= 0);
case CEPH_OSD_CMPXATTR_OP_LT:
- return (strcmp(v1, v2) < 0);
+ return (v1s.compare(v2s) < 0);
case CEPH_OSD_CMPXATTR_OP_LTE:
- return (strcmp(v1, v2) <= 0);
+ return (v1s.compare(v2s) <= 0);
default:
return -EINVAL;
}
@@ -2821,7 +2918,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// --- READS ---
case CEPH_OSD_OP_SYNC_READ:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -2830,21 +2927,28 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
++ctx->num_read;
{
__u32 seq = oi.truncate_seq;
+ uint64_t size = oi.size;
+ bool trimmed_read = false;
// are we beyond truncate_size?
if ( (seq < op.extent.truncate_seq) &&
- (op.extent.offset + op.extent.length > op.extent.truncate_size) ) {
-
- // truncated portion of the read
- unsigned from = MAX(op.extent.offset, op.extent.truncate_size); // also end of data
- unsigned to = op.extent.offset + op.extent.length;
- unsigned trim = to-from;
-
- op.extent.length = op.extent.length - trim;
+ (op.extent.offset + op.extent.length > op.extent.truncate_size) )
+ size = op.extent.truncate_size;
+
+ if (op.extent.offset >= size) {
+ op.extent.length = 0;
+ trimmed_read = true;
+ } else if (op.extent.offset + op.extent.length > size) {
+ op.extent.length = size - op.extent.offset;
+ trimmed_read = true;
}
// read into a buffer
bufferlist bl;
- if (pool.info.ec_pool()) {
+ if (trimmed_read && op.extent.length == 0) {
+ // read size was trimmed to zero and it is expected to do nothing
+ // a read operation of 0 bytes does *not* do nothing, this is why
+ // the trimmed_read boolean is needed
+ } else if (pool.info.require_rollback()) {
ctx->pending_async_reads.push_back(
make_pair(
make_pair(op.extent.offset, op.extent.length),
@@ -2874,7 +2978,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
/* map extents */
case CEPH_OSD_OP_MAPEXT:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -2894,7 +2998,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
/* map extents */
case CEPH_OSD_OP_SPARSE_READ:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3139,27 +3243,12 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
}
if (soid.snap == CEPH_NOSNAP) {
- // verify that all clones have been evicted
- dout(20) << __func__ << " verifying clones are absent "
- << ctx->new_snapset << dendl;
- result = 0;
- for (vector<snapid_t>::iterator p = ctx->new_snapset.clones.begin();
- p != ctx->new_snapset.clones.end();
- ++p) {
- hobject_t clone_oid = soid;
- clone_oid.snap = *p;
- ObjectContextRef clone_obc = get_object_context(clone_oid, false);
- if (clone_obc && clone_obc->obs.exists) {
- dout(10) << __func__ << " cannot evict head before clone "
- << clone_oid << dendl;
- result = -EBUSY;
- break;
- }
- }
+ result = _verify_no_head_clones(soid, ssc->snapset);
if (result < 0)
break;
}
result = _delete_head(ctx, true);
+ osd->logger->inc(l_osd_tier_evict);
}
break;
@@ -3186,21 +3275,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
case CEPH_OSD_OP_GETXATTRS:
++ctx->num_read;
{
- map<string,bufferlist> attrset;
+ map<string, bufferlist> out;
result = getattrs_maybe_cache(
ctx->obc,
- &attrset);
- map<string, bufferlist> out;
- for (map<string, bufferlist>::iterator i = attrset.begin();
- i != attrset.end();
- ++i) {
- if (i->first[0] != '_')
- continue;
- if (i->first == "_")
- continue;
- out[i->first.substr(1, i->first.size())].claim(
- i->second);
- }
+ &out,
+ true);
bufferlist bl;
::encode(out, bl);
@@ -3328,8 +3407,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
obj_list_snap_response_t resp;
if (!ssc) {
- ssc = ctx->obc->ssc = get_snapset_context(soid.oid,
- soid.get_key(), soid.hash, false, soid.get_namespace());
+ ssc = ctx->obc->ssc = get_snapset_context(soid, false);
}
assert(ssc);
@@ -3458,6 +3536,22 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
}
break;
+ case CEPH_OSD_OP_SETALLOCHINT:
+ ++ctx->num_write;
+ {
+ if (!obs.exists) {
+ ctx->mod_desc.create();
+ t->touch(soid);
+ ctx->delta_stats.num_objects++;
+ obs.exists = true;
+ }
+ t->set_alloc_hint(soid, op.alloc_hint.expected_object_size,
+ op.alloc_hint.expected_write_size);
+ ctx->delta_stats.num_wr++;
+ result = 0;
+ }
+ break;
+
// --- WRITES ---
@@ -3471,13 +3565,19 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
}
+ if (pool.info.requires_aligned_append() &&
+ (op.extent.offset % pool.info.required_alignment() != 0)) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+
if (!obs.exists) {
ctx->mod_desc.create();
} else if (op.extent.offset == oi.size) {
ctx->mod_desc.append(oi.size);
} else {
ctx->mod_desc.mark_unrollbackable();
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3517,7 +3617,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
if (result < 0)
break;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
t->append(soid, op.extent.offset, op.extent.length, osd_op.indata);
} else {
t->write(soid, op.extent.offset, op.extent.length, osd_op.indata);
@@ -3542,10 +3642,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
if (result < 0)
break;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (obs.exists) {
- if (ctx->mod_desc.rmobject(oi.version.version)) {
- t->stash(soid, oi.version.version);
+ if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+ t->stash(soid, ctx->at_version.version);
} else {
t->remove(soid);
}
@@ -3586,7 +3686,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_ZERO:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3650,7 +3750,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// falling through
case CEPH_OSD_OP_TRUNCATE:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3709,7 +3809,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
case CEPH_OSD_OP_CLONERANGE:
ctx->mod_desc.mark_unrollbackable();
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3741,6 +3841,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
case CEPH_OSD_OP_WATCH:
++ctx->num_write;
{
+ if (!obs.exists) {
+ result = -ENOENT;
+ break;
+ }
uint64_t cookie = op.watch.cookie;
bool do_watch = op.watch.flag & 1;
entity_name_t entity = ctx->reqid.name;
@@ -3799,7 +3903,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
string aname;
bp.copy(op.xattr.name_len, aname);
string name = "_" + aname;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
map<string, boost::optional<bufferlist> > to_set;
bufferlist old;
int r = getattr_maybe_cache(ctx->obc, name, &old);
@@ -3825,7 +3929,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
string aname;
bp.copy(op.xattr.name_len, aname);
string name = "_" + aname;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
map<string, boost::optional<bufferlist> > to_set;
bufferlist old;
int r = getattr_maybe_cache(ctx->obc, name, &old);
@@ -3869,7 +3973,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// -- trivial map --
case CEPH_OSD_OP_TMAPGET:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3886,7 +3990,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_TMAPPUT:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3944,7 +4048,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_TMAPUP:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3959,7 +4063,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// OMAP Read ops
case CEPH_OSD_OP_OMAPGETKEYS:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -3996,7 +4100,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAPGETVALS:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4041,7 +4145,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAPGETHEADER:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4054,7 +4158,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAPGETVALSBYKEYS:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4077,7 +4181,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAP_CMP:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4148,7 +4252,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// OMAP Write ops
case CEPH_OSD_OP_OMAPSETVALS:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4177,10 +4281,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
t->omap_setkeys(soid, to_set);
ctx->delta_stats.num_wr++;
}
+ obs.oi.set_flag(object_info_t::FLAG_OMAP);
break;
case CEPH_OSD_OP_OMAPSETHEADER:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4195,10 +4300,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
t->omap_setheader(soid, osd_op.indata);
ctx->delta_stats.num_wr++;
}
+ obs.oi.set_flag(object_info_t::FLAG_OMAP);
break;
case CEPH_OSD_OP_OMAPCLEAR:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4213,10 +4319,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
t->omap_clear(soid);
ctx->delta_stats.num_wr++;
}
+ obs.oi.set_flag(object_info_t::FLAG_OMAP);
break;
case CEPH_OSD_OP_OMAPRMKEYS:
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
result = -EOPNOTSUPP;
break;
}
@@ -4239,6 +4346,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
t->omap_rmkeys(soid, to_rm);
ctx->delta_stats.num_wr++;
}
+ obs.oi.set_flag(object_info_t::FLAG_OMAP);
break;
case CEPH_OSD_OP_COPY_GET_CLASSIC:
@@ -4307,6 +4415,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
ctx->bytes_read += osd_op.outdata.length();
fail:
+ osd_op.rval = result;
if (result < 0 && (op.flags & CEPH_OSD_OP_FLAG_FAILOK))
result = 0;
@@ -4340,6 +4449,27 @@ int ReplicatedPG::_get_tmap(OpContext *ctx, bufferlist *header, bufferlist *vals
return 0;
}
+int ReplicatedPG::_verify_no_head_clones(const hobject_t& soid,
+ const SnapSet& ss)
+{
+ // verify that all clones have been evicted
+ dout(20) << __func__ << " verifying clones are absent "
+ << ss << dendl;
+ for (vector<snapid_t>::const_iterator p = ss.clones.begin();
+ p != ss.clones.end();
+ ++p) {
+ hobject_t clone_oid = soid;
+ clone_oid.snap = *p;
+ ObjectContextRef clone_obc = get_object_context(clone_oid, false);
+ if (clone_obc && clone_obc->obs.exists) {
+ dout(10) << __func__ << " cannot evict head before clone "
+ << clone_oid << dendl;
+ return -EBUSY;
+ }
+ }
+ return 0;
+}
+
inline int ReplicatedPG::_delete_head(OpContext *ctx, bool no_whiteout)
{
SnapSet& snapset = ctx->new_snapset;
@@ -4351,12 +4481,14 @@ inline int ReplicatedPG::_delete_head(OpContext *ctx, bool no_whiteout)
if (!obs.exists || (obs.oi.is_whiteout() && !no_whiteout))
return -ENOENT;
- if (pool.info.ec_pool()) {
- if (ctx->mod_desc.rmobject(oi.version.version)) {
- t->stash(soid, oi.version.version);
+ if (pool.info.require_rollback()) {
+ if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+ t->stash(soid, ctx->at_version.version);
} else {
t->remove(soid);
}
+ map<string, bufferlist> new_attrs;
+ replace_cached_attrs(ctx, ctx->obc, new_attrs);
} else {
ctx->mod_desc.mark_unrollbackable();
t->remove(soid);
@@ -4378,14 +4510,15 @@ inline int ReplicatedPG::_delete_head(OpContext *ctx, bool no_whiteout)
oi.set_flag(object_info_t::FLAG_WHITEOUT);
ctx->delta_stats.num_whiteouts++;
t->touch(soid);
+ osd->logger->inc(l_osd_tier_whiteout);
return 0;
}
ctx->delta_stats.num_objects--;
- if (oi.is_dirty())
- ctx->delta_stats.num_objects_dirty--;
- if (oi.is_whiteout())
+ if (oi.is_whiteout()) {
+ dout(20) << __func__ << " deleting whiteout on " << soid << dendl;
ctx->delta_stats.num_whiteouts--;
+ }
snapset.head_exists = false;
obs.exists = false;
return 0;
@@ -4415,10 +4548,10 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
assert(is_missing_object(missing_oid));
dout(20) << "_rollback_to attempted to roll back to a missing object "
<< missing_oid << " (requested snapid: ) " << snapid << dendl;
- wait_for_missing_object(missing_oid, ctx->op);
+ wait_for_unreadable_object(missing_oid, ctx->op);
return ret;
}
- if (maybe_handle_cache(ctx->op, rollback_to, ret, missing_oid, true)) {
+ if (maybe_handle_cache(ctx->op, true, rollback_to, ret, missing_oid, true)) {
// promoting the rollback src, presumably
return -EAGAIN;
}
@@ -4455,14 +4588,15 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
dout(10) << "_rollback_to deleting " << soid.oid
<< " and rolling back to old snap" << dendl;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (obs.exists) {
- if (ctx->mod_desc.rmobject(oi.version.version)) {
- t->stash(soid, oi.version.version);
+ if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+ t->stash(soid, ctx->at_version.version);
} else {
t->remove(soid);
}
}
+ replace_cached_attrs(ctx, ctx->obc, rollback_to->attr_cache);
} else {
if (obs.exists) {
ctx->mod_desc.mark_unrollbackable();
@@ -4494,7 +4628,6 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
if (!obs.exists) {
obs.exists = true; //we're about to recreate it
ctx->delta_stats.num_objects++;
- ctx->delta_stats.num_objects_dirty++;
}
ctx->delta_stats.num_bytes -= obs.oi.size;
ctx->delta_stats.num_bytes += rollback_to->obs.oi.size;
@@ -4531,19 +4664,26 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
dout(20) << "make_writeable " << soid << " snapset=" << ctx->snapset
<< " snapc=" << snapc << dendl;;
- bool was_dirty = ctx->new_obs.oi.is_dirty();
-
+ bool was_dirty = ctx->obc->obs.oi.is_dirty();
if (ctx->new_obs.exists) {
// we will mark the object dirty
- if (ctx->undirty) {
+ if (ctx->undirty && was_dirty) {
dout(20) << " clearing DIRTY flag" << dendl;
assert(ctx->new_obs.oi.is_dirty());
ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
--ctx->delta_stats.num_objects_dirty;
- } else if (!ctx->new_obs.oi.test_flag(object_info_t::FLAG_DIRTY)) {
+ osd->logger->inc(l_osd_tier_clean);
+ } else if (!was_dirty && !ctx->undirty) {
dout(20) << " setting DIRTY flag" << dendl;
ctx->new_obs.oi.set_flag(object_info_t::FLAG_DIRTY);
++ctx->delta_stats.num_objects_dirty;
+ osd->logger->inc(l_osd_tier_dirty);
+ }
+ } else {
+ if (was_dirty) {
+ dout(20) << " deletion, decrementing num_dirty and clearing flag" << dendl;
+ ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
+ --ctx->delta_stats.num_objects_dirty;
}
}
@@ -4579,9 +4719,11 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
ctx->clone_obc->destructor_callback = new C_PG_ObjectContext(this, ctx->clone_obc.get());
ctx->clone_obc->obs.oi = static_snap_oi;
ctx->clone_obc->obs.exists = true;
- if (pool.info.ec_pool())
+ if (pool.info.require_rollback())
ctx->clone_obc->attr_cache = ctx->obc->attr_cache;
snap_oi = &ctx->clone_obc->obs.oi;
+ bool got = ctx->clone_obc->get_write(ctx->op);
+ assert(got);
} else {
snap_oi = &static_snap_oi;
}
@@ -4596,8 +4738,10 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
ctx->delta_stats.num_objects++;
if (snap_oi->is_dirty())
ctx->delta_stats.num_objects_dirty++;
- if (snap_oi->is_whiteout())
+ if (snap_oi->is_whiteout()) {
+ dout(20) << __func__ << " cloning whiteout on " << soid << " to " << coid << dendl;
ctx->delta_stats.num_whiteouts++;
+ }
ctx->delta_stats.num_object_clones++;
ctx->new_snapset.clones.push_back(coid.snap);
ctx->new_snapset.clone_size[coid.snap] = ctx->obs->oi.size;
@@ -4841,17 +4985,13 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
if (ctx->new_obs.exists) {
if (!ctx->obs->exists) {
- // if we logically recreated the head, remove old _snapdir object
- hobject_t snapoid(soid.oid, soid.get_key(), CEPH_SNAPDIR, soid.hash,
- info.pgid.pool(), soid.get_namespace());
-
- ctx->snapset_obc = get_object_context(snapoid, false);
if (ctx->snapset_obc && ctx->snapset_obc->obs.exists) {
+ hobject_t snapoid = soid.get_snapdir();
ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::DELETE, snapoid,
ctx->at_version,
- ctx->obs->oi.version,
+ ctx->snapset_obc->obs.oi.version,
0, osd_reqid_t(), ctx->mtime));
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
ctx->op_t->stash(snapoid, ctx->at_version.version);
} else {
@@ -4876,13 +5016,16 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
<< " in " << snapoid << dendl;
ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::MODIFY, snapoid,
ctx->at_version,
- ctx->obs->oi.version,
+ eversion_t(),
0, osd_reqid_t(), ctx->mtime));
ctx->snapset_obc = get_object_context(snapoid, true);
- if (pool.info.ec_pool() && !ctx->snapset_obc->obs.exists) {
+ bool got = ctx->snapset_obc->get_write(ctx->op);
+ assert(got);
+ ctx->release_snapset_obc = true;
+ if (pool.info.require_rollback() && !ctx->snapset_obc->obs.exists) {
ctx->log.back().mod_desc.create();
- } else if (!pool.info.ec_pool()) {
+ } else if (!pool.info.require_rollback()) {
ctx->log.back().mod_desc.mark_unrollbackable();
}
ctx->snapset_obc->obs.exists = true;
@@ -4895,7 +5038,7 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
ctx->op_t->touch(snapoid);
setattr_maybe_cache(ctx->snapset_obc, ctx, ctx->op_t, OI_ATTR, bv);
setattr_maybe_cache(ctx->snapset_obc, ctx, ctx->op_t, SS_ATTR, bss);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
map<string, boost::optional<bufferlist> > to_set;
to_set[SS_ATTR];
to_set[OI_ATTR];
@@ -4942,7 +5085,7 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
<< " in " << soid << dendl;
setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, SS_ATTR, bss);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
set<string> changing;
changing.insert(OI_ATTR);
changing.insert(SS_ATTR);
@@ -4951,6 +5094,8 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
} else {
dout(10) << " no snapset (this is a clone)" << dendl;
}
+ } else {
+ ctx->new_obs.oi = object_info_t(ctx->obc->obs.oi.soid);
}
// append to log
@@ -4985,8 +5130,10 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
ctx->obc->ssc->snapset = ctx->new_snapset;
info.stats.stats.add(ctx->delta_stats, ctx->obs->oi.category);
- for (unsigned i = 0; i < backfill_targets.size() ; ++i) {
- int bt = backfill_targets[i];
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
pg_info_t& pinfo = peer_info[bt];
if (soid <= pinfo.last_backfill)
pinfo.stats.stats.add(ctx->delta_stats, ctx->obs->oi.category);
@@ -5025,7 +5172,7 @@ void ReplicatedPG::complete_read_ctx(int result, OpContext *ctx)
reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
osd->send_message_osd_client(reply, m->get_connection());
- close_op_ctx(ctx);
+ close_op_ctx(ctx, 0);
}
// ========================================================================
@@ -5090,7 +5237,7 @@ int ReplicatedPG::fill_in_copy_get(
bool async_read_started = false;
object_copy_data_t _reply_obj;
C_CopyFrom_AsyncReadCb *cb = NULL;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
cb = new C_CopyFrom_AsyncReadCb(&osd_op, classic);
}
object_copy_data_t &reply_obj = cb ? cb->reply_obj : _reply_obj;
@@ -5108,9 +5255,16 @@ int ReplicatedPG::fill_in_copy_get(
// attrs
map<string,bufferlist>& out_attrs = reply_obj.attrs;
if (!cursor.attr_complete) {
- result = osd->store->getattrs(coll, soid, out_attrs, true);
- if (result < 0)
+ result = getattrs_maybe_cache(
+ ctx->obc,
+ &out_attrs,
+ true);
+ if (result < 0) {
+ if (cb) {
+ delete cb;
+ }
return result;
+ }
cursor.attr_complete = true;
dout(20) << " got attrs" << dendl;
}
@@ -5147,7 +5301,7 @@ int ReplicatedPG::fill_in_copy_get(
// omap
std::map<std::string,bufferlist>& out_omap = reply_obj.omap;
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
cursor.omap_complete = true;
} else {
if (left > 0 && !cursor.omap_complete) {
@@ -5159,11 +5313,11 @@ int ReplicatedPG::fill_in_copy_get(
osd->store->get_omap_iterator(coll, oi.soid);
assert(iter);
iter->upper_bound(cursor.omap_offset);
- if (iter->valid()) {
- for (; left > 0 && iter->valid(); iter->next()) {
- out_omap.insert(make_pair(iter->key(), iter->value()));
- left -= iter->key().length() + 4 + iter->value().length() + 4;
- }
+ for (; iter->valid(); iter->next()) {
+ out_omap.insert(make_pair(iter->key(), iter->value()));
+ left -= iter->key().length() + 4 + iter->value().length() + 4;
+ if (left <= 0)
+ break;
}
if (iter->valid()) {
cursor.omap_offset = iter->key();
@@ -5261,7 +5415,7 @@ void ReplicatedPG::_copy_some(ObjectContextRef obc, CopyOpRef cop)
// it already!
assert(cop->cursor.is_initial());
}
- op.copy_get(&cop->cursor, cct->_conf->osd_copyfrom_max_chunk,
+ op.copy_get(&cop->cursor, get_copy_chunk_size(),
&cop->results.object_size, &cop->results.mtime,
&cop->results.category,
&cop->attrs, &cop->data, &cop->omap_header, &cop->omap,
@@ -5302,7 +5456,7 @@ void ReplicatedPG::process_copy_chunk(hobject_t oid, tid_t tid, int r)
return;
}
- if (r >= 0 && pool.info.ec_pool() && cop->omap.size()) {
+ if (r >= 0 && pool.info.require_rollback() && cop->omap.size()) {
r = -EOPNOTSUPP;
}
cop->objecter_tid = 0;
@@ -5384,13 +5538,36 @@ void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t)
t->touch(cop->results.temp_oid);
for (map<string,bufferlist>::iterator p = cop->attrs.begin();
p != cop->attrs.end();
- ++p)
+ ++p) {
+ cop->results.attrs[string("_") + p->first] = p->second;
t->setattr(
cop->results.temp_oid,
string("_") + p->first, p->second);
+ }
cop->attrs.clear();
}
if (!cop->temp_cursor.data_complete) {
+ assert(cop->data.length() + cop->temp_cursor.data_offset ==
+ cop->cursor.data_offset);
+ if (pool.info.requires_aligned_append() &&
+ !cop->cursor.data_complete) {
+ /**
+ * Trim off the unaligned bit at the end, we'll adjust cursor.data_offset
+ * to pick it up on the next pass.
+ */
+ assert(cop->temp_cursor.data_offset %
+ pool.info.required_alignment() == 0);
+ if (cop->data.length() % pool.info.required_alignment() != 0) {
+ uint64_t to_trim =
+ cop->data.length() % pool.info.required_alignment();
+ bufferlist bl;
+ bl.substr_of(cop->data, 0, cop->data.length() - to_trim);
+ cop->data.swap(bl);
+ cop->cursor.data_offset -= to_trim;
+ assert(cop->data.length() + cop->temp_cursor.data_offset ==
+ cop->cursor.data_offset);
+ }
+ }
t->append(
cop->results.temp_oid,
cop->temp_cursor.data_offset,
@@ -5398,15 +5575,20 @@ void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t)
cop->data);
cop->data.clear();
}
- if (!cop->temp_cursor.omap_complete) {
- if (cop->omap_header.length()) {
- t->omap_setheader(
- cop->results.temp_oid,
- cop->omap_header);
- cop->omap_header.clear();
+ if (!pool.info.require_rollback()) {
+ if (!cop->temp_cursor.omap_complete) {
+ if (cop->omap_header.length()) {
+ t->omap_setheader(
+ cop->results.temp_oid,
+ cop->omap_header);
+ cop->omap_header.clear();
+ }
+ t->omap_setkeys(cop->results.temp_oid, cop->omap);
+ cop->omap.clear();
}
- t->omap_setkeys(cop->results.temp_oid, cop->omap);
- cop->omap.clear();
+ } else {
+ assert(cop->omap_header.length() == 0);
+ assert(cop->omap.empty());
}
cop->temp_cursor = cop->cursor;
}
@@ -5432,7 +5614,7 @@ void ReplicatedPG::finish_copyfrom(OpContext *ctx)
ObjectState& obs = ctx->new_obs;
CopyFromCallback *cb = static_cast<CopyFromCallback*>(ctx->copy_cb);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (obs.exists) {
if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
ctx->op_t->stash(obs.oi.soid, ctx->at_version.version);
@@ -5441,6 +5623,7 @@ void ReplicatedPG::finish_copyfrom(OpContext *ctx)
}
}
ctx->mod_desc.create();
+ replace_cached_attrs(ctx, ctx->obc, cb->results->attrs);
} else {
if (obs.exists) {
ctx->op_t->remove(obs.oi.soid);
@@ -5481,6 +5664,8 @@ void ReplicatedPG::finish_copyfrom(OpContext *ctx)
}
ctx->delta_stats.num_wr++;
ctx->delta_stats.num_wr_kb += SHIFT_ROUND_UP(obs.oi.size, 10);
+
+ osd->logger->inc(l_osd_copyfrom);
}
void ReplicatedPG::finish_promote(int r, OpRequestRef op,
@@ -5540,7 +5725,8 @@ void ReplicatedPG::finish_promote(int r, OpRequestRef op,
tctx->op_t->touch(soid);
tctx->new_obs.oi.set_flag(object_info_t::FLAG_WHITEOUT);
++tctx->delta_stats.num_whiteouts;
- dout(20) << __func__ << " creating whiteout" << dendl;
+ dout(20) << __func__ << " creating whiteout on " << soid << dendl;
+ osd->logger->inc(l_osd_tier_whiteout);
} else {
tctx->op_t->append(results->final_tx);
delete results->final_tx;
@@ -5584,8 +5770,8 @@ void ReplicatedPG::finish_promote(int r, OpRequestRef op,
tctx->new_snapset.head_exists = true;
dout(20) << __func__ << " new_snapset " << tctx->new_snapset << dendl;
- // take RWWRITE lock for duration of our local write
- if (!obc->rwstate.get_write_lock()) {
+ // take RWWRITE lock for duration of our local write. ignore starvation.
+ if (!obc->rwstate.take_write_lock()) {
assert(0 == "problem!");
}
tctx->lock_to_release = OpContext::W_LOCK;
@@ -5594,6 +5780,8 @@ void ReplicatedPG::finish_promote(int r, OpRequestRef op,
finish_ctx(tctx, pg_log_entry_t::PROMOTE);
simple_repop_submit(repop);
+
+ osd->logger->inc(l_osd_tier_promote);
}
void ReplicatedPG::cancel_copy(CopyOpRef cop, bool requeue)
@@ -5605,9 +5793,9 @@ void ReplicatedPG::cancel_copy(CopyOpRef cop, bool requeue)
// cancel objecter op, if we can
if (cop->objecter_tid) {
Mutex::Locker l(osd->objecter_lock);
- osd->objecter->op_cancel(cop->objecter_tid);
+ osd->objecter->op_cancel(cop->objecter_tid, -ECANCELED);
if (cop->objecter_tid2) {
- osd->objecter->op_cancel(cop->objecter_tid2);
+ osd->objecter->op_cancel(cop->objecter_tid2, -ECANCELED);
}
}
@@ -5728,7 +5916,8 @@ int ReplicatedPG::start_flush(OpContext *ctx, bool blocking)
if (fop->ctx->op == ctx->op) {
// we couldn't take the write lock on a cache-try-flush before;
// now we are trying again for the lock.
- close_op_ctx(fop->ctx); // clean up the previous ctx and use the new one.
+ // clean up the previous ctx and use the new one.
+ close_op_ctx(fop->ctx, -EAGAIN);
fop->ctx = ctx;
return try_flush_mark_clean(fop);
}
@@ -5784,7 +5973,7 @@ int ReplicatedPG::start_flush(OpContext *ctx, bool blocking)
// have been written before that.
vector<snapid_t>::iterator p = snapset.snaps.begin();
while (p != snapset.snaps.end() && *p >= oi.snaps.back())
- p++;
+ ++p;
snapc.snaps = vector<snapid_t>(p, snapset.snaps.end());
snapc.seq = oi.snaps.back() - 1;
}
@@ -5832,6 +6021,9 @@ void ReplicatedPG::finish_flush(hobject_t oid, tid_t tid, int r)
return;
}
+ delete fop->ctx->op_t;
+ fop->ctx->op_t = pgbackend->get_transaction();
+
r = try_flush_mark_clean(fop);
if (r == -EBUSY) {
reply_ctx(fop->ctx, -EBUSY, obc->obs.oi.version,
@@ -5862,6 +6054,10 @@ int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
kick_object_context_blocked(obc);
}
flush_ops.erase(oid);
+ if (fop->blocking)
+ osd->logger->inc(l_osd_tier_flush_fail);
+ else
+ osd->logger->inc(l_osd_tier_try_flush_fail);
return -EBUSY;
}
@@ -5869,10 +6065,16 @@ int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
if (!fop->blocking) {
// non-blocking: try to take the lock manually, since we don't
// have a ctx yet.
- dout(20) << __func__ << " taking write lock" << dendl;
- if (!obc->get_write(fop->ctx->op)) {
- dout(10) << __func__ << " waiting on lock" << dendl;
+ if (obc->get_write(fop->ctx->op)) {
+ dout(20) << __func__ << " took write lock" << dendl;
+ } else if (fop->ctx->op) {
+ dout(10) << __func__ << " waiting on write lock" << dendl;
return -EINPROGRESS; // will retry. this ctx is still alive!
+ } else {
+ dout(10) << __func__ << " failed write lock, no op; failing" << dendl;
+ osd->logger->inc(l_osd_tier_try_flush_fail);
+ cancel_flush(fop, false);
+ return -ECANCELED;
}
} else {
dout(20) << __func__ << " already holding write lock: "
@@ -5900,6 +6102,8 @@ int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
finish_ctx(ctx, pg_log_entry_t::CLEAN);
+ osd->logger->inc(l_osd_tier_clean);
+
if (!fop->dup_ops.empty()) {
dout(20) << __func__ << " queueing dups for " << ctx->at_version << dendl;
list<OpRequestRef>& ls = waiting_for_ondisk[ctx->at_version];
@@ -5909,6 +6113,12 @@ int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
simple_repop_submit(repop);
flush_ops.erase(oid);
+
+ if (fop->blocking)
+ osd->logger->inc(l_osd_tier_flush);
+ else
+ osd->logger->inc(l_osd_tier_try_flush);
+
return -EINPROGRESS;
}
@@ -5918,7 +6128,7 @@ void ReplicatedPG::cancel_flush(FlushOpRef fop, bool requeue)
<< fop->objecter_tid << dendl;
if (fop->objecter_tid) {
Mutex::Locker l(osd->objecter_lock);
- osd->objecter->op_cancel(fop->objecter_tid);
+ osd->objecter->op_cancel(fop->objecter_tid, -ECANCELED);
}
if (fop->ctx->op && requeue) {
requeue_op(fop->ctx->op);
@@ -5929,7 +6139,7 @@ void ReplicatedPG::cancel_flush(FlushOpRef fop, bool requeue)
kick_object_context_blocked(fop->ctx->obc);
}
flush_ops.erase(fop->ctx->obc->obs.oi.soid);
- close_op_ctx(fop->ctx);
+ close_op_ctx(fop->ctx, -ECANCELED);
}
void ReplicatedPG::cancel_flush_ops(bool requeue)
@@ -5963,6 +6173,10 @@ void ReplicatedPG::repop_all_applied(RepGather *repop)
repop->all_applied = true;
if (!repop->rep_aborted) {
eval_repop(repop);
+ if (repop->on_applied) {
+ repop->on_applied->complete(0);
+ repop->on_applied = NULL;
+ }
}
}
@@ -6010,7 +6224,7 @@ void ReplicatedPG::op_applied(const eversion_t &applied_version)
scrubber.finalizing = true;
scrub_gather_replica_maps();
++scrubber.waiting_on;
- scrubber.waiting_on_whom.insert(osd->whoami);
+ scrubber.waiting_on_whom.insert(pg_whoami);
osd->scrub_wq.queue(this);
}
} else {
@@ -6051,8 +6265,6 @@ void ReplicatedPG::eval_repop(RepGather *repop)
// ondisk?
if (repop->all_committed) {
- release_op_ctx_locks(repop->ctx);
-
log_op_stats(repop->ctx);
publish_stats_to_osd();
@@ -6080,7 +6292,7 @@ void ReplicatedPG::eval_repop(RepGather *repop)
if (reply)
repop->ctx->reply = NULL;
else {
- reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0);
+ reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, true);
reply->set_reply_versions(repop->ctx->at_version,
repop->ctx->user_at_version);
}
@@ -6102,7 +6314,7 @@ void ReplicatedPG::eval_repop(RepGather *repop)
i != waiting_for_ack[repop->v].end();
++i) {
MOSDOp *m = (MOSDOp*)(*i)->get_req();
- MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0);
+ MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, true);
reply->set_reply_versions(repop->ctx->at_version,
repop->ctx->user_at_version);
reply->add_flags(CEPH_OSD_FLAG_ACK);
@@ -6117,7 +6329,7 @@ void ReplicatedPG::eval_repop(RepGather *repop)
if (reply)
repop->ctx->reply = NULL;
else {
- reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0);
+ reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, true);
reply->set_reply_versions(repop->ctx->at_version,
repop->ctx->user_at_version);
}
@@ -6141,6 +6353,8 @@ void ReplicatedPG::eval_repop(RepGather *repop)
if (repop->all_applied && repop->all_committed) {
repop->rep_done = true;
+ release_op_ctx_locks(repop->ctx);
+
calc_min_last_complete_ondisk();
// kick snap_trimmer if necessary
@@ -6176,15 +6390,17 @@ void ReplicatedPG::issue_repop(RepGather *repop, utime_t now)
<< dendl;
repop->v = ctx->at_version;
-
- for (vector<int>::iterator i = actingbackfill.begin() + 1;
- i != actingbackfill.end();
- ++i) {
- pg_info_t &pinfo = peer_info[*i];
- // keep peer_info up to date
- if (pinfo.last_complete == pinfo.last_update)
- pinfo.last_complete = ctx->at_version;
- pinfo.last_update = ctx->at_version;
+ if (ctx->at_version > eversion_t()) {
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_info_t &pinfo = peer_info[*i];
+ // keep peer_info up to date
+ if (pinfo.last_complete == pinfo.last_update)
+ pinfo.last_complete = ctx->at_version;
+ pinfo.last_update = ctx->at_version;
+ }
}
repop->obc->ondisk_write_lock();
@@ -6200,7 +6416,7 @@ void ReplicatedPG::issue_repop(RepGather *repop, utime_t now)
repop->ctx->apply_pending_attrs();
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
for (vector<pg_log_entry_t>::iterator i = repop->ctx->log.begin();
i != repop->ctx->log.end();
++i) {
@@ -6244,25 +6460,27 @@ void ReplicatedBackend::issue_op(
{
int acks_wanted = CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK;
- if (parent->get_actingbackfill().size() > 1) {
+ if (parent->get_actingbackfill_shards().size() > 1) {
ostringstream ss;
- ss << "waiting for subops from " <<
- vector<int>(
- parent->get_actingbackfill().begin() + 1,
- parent->get_actingbackfill().end());
+ set<pg_shard_t> replicas = parent->get_actingbackfill_shards();
+ replicas.erase(parent->whoami_shard());
+ ss << "waiting for subops from " << replicas;
if (op->op)
op->op->mark_sub_op_sent(ss.str());
}
- for (unsigned i=1; i<parent->get_actingbackfill().size(); i++) {
- int peer = parent->get_actingbackfill()[i];
- const pg_info_t &pinfo = parent->get_peer_info().find(peer)->second;
-
- op->waiting_for_applied.insert(peer);
- op->waiting_for_commit.insert(peer);
+ for (set<pg_shard_t>::const_iterator i =
+ parent->get_actingbackfill_shards().begin();
+ i != parent->get_actingbackfill_shards().end();
+ ++i) {
+ if (*i == parent->whoami_shard()) continue;
+ pg_shard_t peer = *i;
+ const pg_info_t &pinfo = parent->get_shard_info().find(peer)->second;
// forward the write/update/whatever
MOSDSubOp *wr = new MOSDSubOp(
- reqid, get_info().pgid, soid,
+ reqid, parent->whoami_shard(),
+ spg_t(get_info().pgid.pgid, i->shard),
+ soid,
false, acks_wanted,
get_osdmap()->get_epoch(),
tid, at_version);
@@ -6292,8 +6510,8 @@ void ReplicatedBackend::issue_op(
wr->new_temp_oid = new_temp_oid;
wr->discard_temp_oid = discard_temp_oid;
- osd->send_message_osd_cluster(peer, wr, get_osdmap()->get_epoch());
-
+ get_parent()->send_message_osd_cluster(
+ peer.osd, wr, get_osdmap()->get_epoch());
}
}
@@ -6322,6 +6540,7 @@ void ReplicatedPG::remove_repop(RepGather *repop)
{
dout(20) << __func__ << " " << *repop << dendl;
release_op_ctx_locks(repop->ctx);
+ repop->ctx->finish(0); // FIXME: return value here is sloppy
repop_map.erase(repop->rep_tid);
repop->put();
@@ -6502,7 +6721,7 @@ void ReplicatedPG::handle_watch_timeout(WatchRef watch)
::encode(obc->obs.oi, bl);
setattr_maybe_cache(obc, repop->ctx, t, OI_ATTR, bl);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
map<string, boost::optional<bufferlist> > to_set;
to_set[OI_ATTR] = bl;
ctx->log.back().mod_desc.setattrs(to_set);
@@ -6534,7 +6753,7 @@ ObjectContextRef ReplicatedPG::create_object_context(const object_info_t& oi,
ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
bool can_create,
- map<string, bufferptr> *attrs)
+ map<string, bufferlist> *attrs)
{
assert(
attrs || !pg_log.get_missing().is_missing(soid) ||
@@ -6545,13 +6764,14 @@ ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
ObjectContextRef obc = object_contexts.lookup(soid);
if (obc) {
dout(10) << "get_object_context " << obc << " " << soid
- << " " << obc->rwstate << dendl;
+ << " " << obc->rwstate
+ << " oi:" << obc->obs.oi << dendl;
} else {
// check disk
bufferlist bv;
if (attrs) {
assert(attrs->count(OI_ATTR));
- bv.push_back(attrs->find(OI_ATTR)->second);
+ bv = attrs->find(OI_ATTR)->second;
} else {
int r = pgbackend->objects_get_attr(soid, OI_ATTR, &bv);
if (r < 0) {
@@ -6561,7 +6781,7 @@ ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
// new object.
object_info_t oi(soid);
SnapSetContext *ssc = get_snapset_context(
- soid.oid, soid.get_key(), soid.hash, true, soid.get_namespace(),
+ soid, true,
soid.has_snapset() ? attrs : 0);
return create_object_context(oi, ssc);
}
@@ -6577,22 +6797,15 @@ ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
obc->obs.exists = true;
obc->ssc = get_snapset_context(
- soid.oid, soid.get_key(), soid.hash,
- true, soid.get_namespace(),
+ soid, true,
soid.has_snapset() ? attrs : 0);
register_snapset_context(obc->ssc);
populate_obc_watchers(obc);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (attrs) {
- for (map<string, bufferptr>::iterator i = attrs->begin();
- i != attrs->end();
- ++i) {
- bufferlist bl;
- bl.append(i->second);
- obc->attr_cache.insert(make_pair(i->first, bl));
- }
+ obc->attr_cache = *attrs;
} else {
int r = pgbackend->objects_get_attrs(
soid,
@@ -6603,6 +6816,7 @@ ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
dout(10) << "get_object_context " << obc << " " << soid
<< " " << obc->rwstate
+ << " oi:" << obc->obs.oi
<< " 0 -> 1 read " << obc->obs.oi << dendl;
}
return obc;
@@ -6657,13 +6871,16 @@ int ReplicatedPG::find_object_context(const hobject_t& oid,
*pmissing = head;
return -ENOENT;
}
- dout(10) << "find_object_context " << oid << " @" << oid.snap << dendl;
+ dout(10) << "find_object_context " << oid
+ << " @" << oid.snap
+ << " oi=" << obc->obs.oi
+ << dendl;
*pobc = obc;
// always populate ssc for SNAPDIR...
if (!obc->ssc)
- obc->ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, true,
- oid.get_namespace());
+ obc->ssc = get_snapset_context(
+ oid, true);
return 0;
}
@@ -6675,12 +6892,14 @@ int ReplicatedPG::find_object_context(const hobject_t& oid,
*pmissing = head;
return -ENOENT;
}
- dout(10) << "find_object_context " << oid << " @" << oid.snap << dendl;
+ dout(10) << "find_object_context " << oid
+ << " @" << oid.snap
+ << " oi=" << obc->obs.oi
+ << dendl;
*pobc = obc;
if (can_create && !obc->ssc)
- obc->ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, true,
- oid.get_namespace());
+ obc->ssc = get_snapset_context(oid, true);
return 0;
}
@@ -6691,8 +6910,7 @@ int ReplicatedPG::find_object_context(const hobject_t& oid,
return -ENOENT;
}
- SnapSetContext *ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash,
- can_create, oid.get_namespace());
+ SnapSetContext *ssc = get_snapset_context(oid, can_create);
if (!ssc) {
dout(20) << __func__ << " " << oid << " no snapset" << dendl;
if (pmissing)
@@ -6811,11 +7029,7 @@ void ReplicatedPG::add_object_context_to_pg_stat(ObjectContextRef obc, pg_stat_t
stat.num_object_clones++;
if (!obc->ssc)
- obc->ssc = get_snapset_context(oi.soid.oid,
- oi.soid.get_key(),
- oi.soid.hash,
- false,
- oi.soid.get_namespace());
+ obc->ssc = get_snapset_context(oi.soid, false);
assert(obc->ssc);
// subtract off clone overlap
@@ -6853,47 +7067,41 @@ void ReplicatedPG::kick_object_context_blocked(ObjectContextRef obc)
waiting_for_blocked_object.erase(p);
}
-SnapSetContext *ReplicatedPG::create_snapset_context(const object_t& oid)
+SnapSetContext *ReplicatedPG::create_snapset_context(const hobject_t& oid)
{
Mutex::Locker l(snapset_contexts_lock);
- SnapSetContext *ssc = new SnapSetContext(oid);
+ SnapSetContext *ssc = new SnapSetContext(oid.get_snapdir());
_register_snapset_context(ssc);
ssc->ref++;
return ssc;
}
SnapSetContext *ReplicatedPG::get_snapset_context(
- const object_t& oid,
- const string& key,
- ps_t seed,
+ const hobject_t& oid,
bool can_create,
- const string& nspace,
- map<string, bufferptr> *attrs)
+ map<string, bufferlist> *attrs)
{
Mutex::Locker l(snapset_contexts_lock);
SnapSetContext *ssc;
- map<object_t, SnapSetContext*>::iterator p = snapset_contexts.find(oid);
+ map<hobject_t, SnapSetContext*>::iterator p = snapset_contexts.find(
+ oid.get_snapdir());
if (p != snapset_contexts.end()) {
ssc = p->second;
} else {
bufferlist bv;
if (!attrs) {
- hobject_t head(oid, key, CEPH_NOSNAP, seed,
- info.pgid.pool(), nspace);
- int r = pgbackend->objects_get_attr(head, SS_ATTR, &bv);
+ int r = pgbackend->objects_get_attr(oid.get_head(), SS_ATTR, &bv);
if (r < 0) {
// try _snapset
- hobject_t snapdir(oid, key, CEPH_SNAPDIR, seed,
- info.pgid.pool(), nspace);
- r = pgbackend->objects_get_attr(snapdir, SS_ATTR, &bv);
+ r = pgbackend->objects_get_attr(oid.get_snapdir(), SS_ATTR, &bv);
if (r < 0 && !can_create)
return NULL;
}
} else {
assert(attrs->count(SS_ATTR));
- bv.push_back(attrs->find(SS_ATTR)->second);
+ bv = attrs->find(SS_ATTR)->second;
}
- ssc = new SnapSetContext(oid);
+ ssc = new SnapSetContext(oid.get_snapdir());
_register_snapset_context(ssc);
if (bv.length()) {
bufferlist::iterator bvp = bv.begin();
@@ -6963,6 +7171,9 @@ void ReplicatedBackend::sub_op_modify(OpRequestRef op)
vector<pg_log_entry_t> log;
bufferlist::iterator p = m->get_data().begin();
+ ::decode(rm->opt, p);
+ if (!(m->get_connection()->get_features() & CEPH_FEATURE_OSD_SNAPMAPPER))
+ rm->opt.set_tolerate_collection_add_enoent();
if (m->new_temp_oid != hobject_t()) {
dout(20) << __func__ << " start tracking temp " << m->new_temp_oid << dendl;
@@ -6971,12 +7182,14 @@ void ReplicatedBackend::sub_op_modify(OpRequestRef op)
}
if (m->discard_temp_oid != hobject_t()) {
dout(20) << __func__ << " stop tracking temp " << m->discard_temp_oid << dendl;
+ if (rm->opt.empty()) {
+ dout(10) << __func__ << ": removing object " << m->discard_temp_oid
+ << " since we won't get the transaction" << dendl;
+ rm->localt.remove(temp_coll, m->discard_temp_oid);
+ }
clear_temp_obj(m->discard_temp_oid);
}
- ::decode(rm->opt, p);
- if (!(m->get_connection()->get_features() & CEPH_FEATURE_OSD_SNAPMAPPER))
- rm->opt.set_tolerate_collection_add_enoent();
p = m->logbl.begin();
::decode(log, p);
if (m->hobject_incorrect_pool) {
@@ -7044,9 +7257,12 @@ void ReplicatedBackend::sub_op_modify_applied(RepModifyRef rm)
if (!rm->committed) {
// send ack to acker only if we haven't sent a commit already
- MOSDSubOpReply *ack = new MOSDSubOpReply(m, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+ MOSDSubOpReply *ack = new MOSDSubOpReply(
+ m, parent->whoami_shard(),
+ 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
ack->set_priority(CEPH_MSG_PRIO_HIGH); // this better match commit priority!
- osd->send_message_osd_cluster(rm->ackerosd, ack, get_osdmap()->get_epoch());
+ get_parent()->send_message_osd_cluster(
+ rm->ackerosd, ack, get_osdmap()->get_epoch());
}
parent->op_applied(m->version);
@@ -7064,12 +7280,17 @@ void ReplicatedBackend::sub_op_modify_commit(RepModifyRef rm)
assert(get_osdmap()->is_up(rm->ackerosd));
get_parent()->update_last_complete_ondisk(rm->last_complete);
- MOSDSubOpReply *commit = new MOSDSubOpReply(static_cast<MOSDSubOp*>(rm->op->get_req()), 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ONDISK);
+ MOSDSubOpReply *commit = new MOSDSubOpReply(
+ static_cast<MOSDSubOp*>(rm->op->get_req()),
+ get_parent()->whoami_shard(),
+ 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ONDISK);
commit->set_last_complete_ondisk(rm->last_complete);
commit->set_priority(CEPH_MSG_PRIO_HIGH); // this better match ack priority!
- osd->send_message_osd_cluster(rm->ackerosd, commit, get_osdmap()->get_epoch());
+ get_parent()->send_message_osd_cluster(
+ rm->ackerosd, commit, get_osdmap()->get_epoch());
- log_subop_stats(osd, rm->op, l_osd_sop_w_inb, l_osd_sop_w_lat);
+ log_subop_stats(get_parent()->get_logger(), rm->op,
+ l_osd_sop_w_inb, l_osd_sop_w_lat);
}
@@ -7221,41 +7442,41 @@ void ReplicatedBackend::calc_clone_subsets(
enum { PULL_NONE, PULL_OTHER, PULL_YES };
void ReplicatedBackend::prepare_pull(
+ eversion_t v,
const hobject_t& soid,
ObjectContextRef headctx,
RPGHandle *h)
{
assert(get_parent()->get_local_missing().missing.count(soid));
- eversion_t v = get_parent()->get_local_missing().missing.find(
+ eversion_t _v = get_parent()->get_local_missing().missing.find(
soid)->second.need;
- const map<hobject_t, set<int> > &missing_loc(
- get_parent()->get_missing_loc());
- const map<int, pg_missing_t > &peer_missing(
- get_parent()->get_peer_missing());
- int fromosd = -1;
- map<hobject_t,set<int> >::const_iterator q = missing_loc.find(soid);
+ assert(_v == v);
+ const map<hobject_t, set<pg_shard_t> > &missing_loc(
+ get_parent()->get_missing_loc_shards());
+ const map<pg_shard_t, pg_missing_t > &peer_missing(
+ get_parent()->get_shard_missing());
+ map<hobject_t, set<pg_shard_t> >::const_iterator q = missing_loc.find(soid);
assert(q != missing_loc.end());
assert(!q->second.empty());
// pick a pullee
- vector<int> shuffle(q->second.begin(), q->second.end());
+ vector<pg_shard_t> shuffle(q->second.begin(), q->second.end());
random_shuffle(shuffle.begin(), shuffle.end());
- vector<int>::iterator p = shuffle.begin();
- assert(get_osdmap()->is_up(*p));
- fromosd = *p;
- assert(fromosd >= 0);
+ vector<pg_shard_t>::iterator p = shuffle.begin();
+ assert(get_osdmap()->is_up(p->osd));
+ pg_shard_t fromshard = *p;
dout(7) << "pull " << soid
<< "v " << v
<< " on osds " << *p
- << " from osd." << fromosd
+ << " from osd." << fromshard
<< dendl;
- assert(peer_missing.count(fromosd));
- const pg_missing_t &pmissing = peer_missing.find(fromosd)->second;
+ assert(peer_missing.count(fromshard));
+ const pg_missing_t &pmissing = peer_missing.find(fromshard)->second;
if (pmissing.is_missing(soid, v)) {
assert(pmissing.missing.find(soid)->second.have != v);
- dout(10) << "pulling soid " << soid << " from osd " << fromosd
+ dout(10) << "pulling soid " << soid << " from osd " << fromshard
<< " at version " << pmissing.missing.find(soid)->second.have
<< " rather than at version " << v << dendl;
v = pmissing.missing.find(soid)->second.have;
@@ -7292,8 +7513,8 @@ void ReplicatedBackend::prepare_pull(
recovery_info.size = ((uint64_t)-1);
}
- h->pulls[fromosd].push_back(PullOp());
- PullOp &op = h->pulls[fromosd].back();
+ h->pulls[fromshard].push_back(PullOp());
+ PullOp &op = h->pulls[fromshard].back();
op.soid = soid;
op.recovery_info = recovery_info;
@@ -7305,7 +7526,7 @@ void ReplicatedBackend::prepare_pull(
op.recovery_progress.first = true;
assert(!pulling.count(soid));
- pull_from_peer[fromosd].insert(soid);
+ pull_from_peer[fromshard].insert(soid);
PullInfo &pi = pulling[soid];
pi.head_ctx = headctx;
pi.recovery_info = op.recovery_info;
@@ -7317,8 +7538,7 @@ int ReplicatedPG::recover_missing(
int priority,
PGBackend::RecoveryHandle *h)
{
- map<hobject_t,set<int> >::iterator q = missing_loc.find(soid);
- if (q == missing_loc.end()) {
+ if (missing_loc.is_unfound(soid)) {
dout(7) << "pull " << soid
<< " v " << v
<< " but it is unfound" << dendl;
@@ -7376,13 +7596,15 @@ int ReplicatedPG::recover_missing(
recovering.insert(make_pair(soid, obc));
pgbackend->recover_object(
soid,
+ v,
head_obc,
obc,
h);
return PULL_YES;
}
-void ReplicatedPG::send_remove_op(const hobject_t& oid, eversion_t v, int peer)
+void ReplicatedPG::send_remove_op(
+ const hobject_t& oid, eversion_t v, pg_shard_t peer)
{
tid_t tid = osd->get_tid();
osd_reqid_t rid(osd->get_cluster_msgr_name(), 0, tid);
@@ -7390,12 +7612,14 @@ void ReplicatedPG::send_remove_op(const hobject_t& oid, eversion_t v, int peer)
dout(10) << "send_remove_op " << oid << " from osd." << peer
<< " tid " << tid << dendl;
- MOSDSubOp *subop = new MOSDSubOp(rid, info.pgid, oid, false, CEPH_OSD_FLAG_ACK,
- get_osdmap()->get_epoch(), tid, v);
+ MOSDSubOp *subop = new MOSDSubOp(
+ rid, pg_whoami, spg_t(info.pgid.pgid, peer.shard),
+ oid, false, CEPH_OSD_FLAG_ACK,
+ get_osdmap()->get_epoch(), tid, v);
subop->ops = vector<OSDOp>(1);
subop->ops[0].op.op = CEPH_OSD_OP_DELETE;
- osd->send_message_osd_cluster(peer, subop, get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(peer.osd, subop, get_osdmap()->get_epoch());
}
/*
@@ -7403,7 +7627,7 @@ void ReplicatedPG::send_remove_op(const hobject_t& oid, eversion_t v, int peer)
* clones/heads and dup data ranges where possible.
*/
void ReplicatedBackend::prep_push_to_replica(
- ObjectContextRef obc, const hobject_t& soid, int peer,
+ ObjectContextRef obc, const hobject_t& soid, pg_shard_t peer,
PushOp *pop)
{
const object_info_t& oi = obc->obs.oi;
@@ -7437,12 +7661,12 @@ void ReplicatedBackend::prep_push_to_replica(
SnapSetContext *ssc = obc->ssc;
assert(ssc);
dout(15) << "push_to_replica snapset is " << ssc->snapset << dendl;
- map<int, pg_missing_t>::const_iterator pm =
- get_parent()->get_peer_missing().find(peer);
- assert(pm != get_parent()->get_peer_missing().end());
- map<int, pg_info_t>::const_iterator pi =
- get_parent()->get_peer_info().find(peer);
- assert(pi != get_parent()->get_peer_info().end());
+ map<pg_shard_t, pg_missing_t>::const_iterator pm =
+ get_parent()->get_shard_missing().find(peer);
+ assert(pm != get_parent()->get_shard_missing().end());
+ map<pg_shard_t, pg_info_t>::const_iterator pi =
+ get_parent()->get_shard_info().find(peer);
+ assert(pi != get_parent()->get_shard_info().end());
calc_clone_subsets(ssc->snapset, soid,
pm->second,
pi->second.last_backfill,
@@ -7455,8 +7679,8 @@ void ReplicatedBackend::prep_push_to_replica(
dout(15) << "push_to_replica snapset is " << ssc->snapset << dendl;
calc_head_subsets(
obc,
- ssc->snapset, soid, get_parent()->get_peer_missing().find(peer)->second,
- get_parent()->get_peer_info().find(peer)->second.last_backfill,
+ ssc->snapset, soid, get_parent()->get_shard_missing().find(peer)->second,
+ get_parent()->get_shard_info().find(peer)->second.last_backfill,
data_subset, clone_subsets);
}
@@ -7464,7 +7688,7 @@ void ReplicatedBackend::prep_push_to_replica(
}
void ReplicatedBackend::prep_push(ObjectContextRef obc,
- const hobject_t& soid, int peer,
+ const hobject_t& soid, pg_shard_t peer,
PushOp *pop)
{
interval_set<uint64_t> data_subset;
@@ -7479,7 +7703,7 @@ void ReplicatedBackend::prep_push(ObjectContextRef obc,
void ReplicatedBackend::prep_push(
ObjectContextRef obc,
- const hobject_t& soid, int peer,
+ const hobject_t& soid, pg_shard_t peer,
eversion_t version,
interval_set<uint64_t> &data_subset,
map<hobject_t, interval_set<uint64_t> >& clone_subsets,
@@ -7510,13 +7734,13 @@ void ReplicatedBackend::prep_push(
pi.recovery_progress = new_progress;
}
-int ReplicatedBackend::send_pull_legacy(int prio, int peer,
+int ReplicatedBackend::send_pull_legacy(int prio, pg_shard_t peer,
const ObjectRecoveryInfo &recovery_info,
ObjectRecoveryProgress progress)
{
// send op
- tid_t tid = osd->get_tid();
- osd_reqid_t rid(osd->get_cluster_msgr_name(), 0, tid);
+ tid_t tid = get_parent()->get_tid();
+ osd_reqid_t rid(get_parent()->get_cluster_msgr_name(), 0, tid);
dout(10) << "send_pull_op " << recovery_info.soid << " "
<< recovery_info.version
@@ -7525,10 +7749,12 @@ int ReplicatedBackend::send_pull_legacy(int prio, int peer,
<< " from osd." << peer
<< " tid " << tid << dendl;
- MOSDSubOp *subop = new MOSDSubOp(rid, get_info().pgid, recovery_info.soid,
- false, CEPH_OSD_FLAG_ACK,
- get_osdmap()->get_epoch(), tid,
- recovery_info.version);
+ MOSDSubOp *subop = new MOSDSubOp(
+ rid, parent->whoami_shard(),
+ get_info().pgid, recovery_info.soid,
+ false, CEPH_OSD_FLAG_ACK,
+ get_osdmap()->get_epoch(), tid,
+ recovery_info.version);
subop->set_priority(prio);
subop->ops = vector<OSDOp>(1);
subop->ops[0].op.op = CEPH_OSD_OP_PULL;
@@ -7536,9 +7762,10 @@ int ReplicatedBackend::send_pull_legacy(int prio, int peer,
subop->recovery_info = recovery_info;
subop->recovery_progress = progress;
- osd->send_message_osd_cluster(peer, subop, get_osdmap()->get_epoch());
+ get_parent()->send_message_osd_cluster(
+ peer.osd, subop, get_osdmap()->get_epoch());
- osd->logger->inc(l_osd_pull);
+ get_parent()->get_logger()->inc(l_osd_pull);
return 0;
}
@@ -7549,7 +7776,7 @@ void ReplicatedBackend::submit_push_data(
const interval_set<uint64_t> &intervals_included,
bufferlist data_included,
bufferlist omap_header,
- map<string, bufferptr> &attrs,
+ map<string, bufferlist> &attrs,
map<string, bufferlist> &omap_entries,
ObjectStore::Transaction *t)
{
@@ -7559,7 +7786,7 @@ void ReplicatedBackend::submit_push_data(
} else {
dout(10) << __func__ << ": Creating oid "
<< recovery_info.soid << " in the temp collection" << dendl;
- temp_contents.insert(recovery_info.soid);
+ add_temp_obj(recovery_info.soid);
target_coll = get_temp_coll(t);
}
@@ -7587,10 +7814,9 @@ void ReplicatedBackend::submit_push_data(
if (complete) {
if (!first) {
- assert(temp_contents.count(recovery_info.soid));
dout(10) << __func__ << ": Removing oid "
<< recovery_info.soid << " from the temp collection" << dendl;
- temp_contents.erase(recovery_info.soid);
+ clear_temp_obj(recovery_info.soid);
t->collection_move(coll, target_coll, recovery_info.soid);
}
@@ -7633,7 +7859,7 @@ ObjectRecoveryInfo ReplicatedBackend::recalc_subsets(
}
bool ReplicatedBackend::handle_pull_response(
- int from, PushOp &pop, PullOp *response,
+ pg_shard_t from, PushOp &pop, PullOp *response,
list<hobject_t> *to_continue,
ObjectStore::Transaction *t
)
@@ -7730,12 +7956,12 @@ struct C_OnPushCommit : public Context {
C_OnPushCommit(ReplicatedPG *pg, OpRequestRef op) : pg(pg), op(op) {}
void finish(int) {
op->mark_event("committed");
- log_subop_stats(pg->osd, op, l_osd_push_inb, l_osd_sop_push_lat);
+ log_subop_stats(pg->osd->logger, op, l_osd_push_inb, l_osd_sop_push_lat);
}
};
void ReplicatedBackend::handle_push(
- int from, PushOp &pop, PushReplyOp *response,
+ pg_shard_t from, PushOp &pop, PushReplyOp *response,
ObjectStore::Transaction *t)
{
dout(10) << "handle_push "
@@ -7768,13 +7994,13 @@ void ReplicatedBackend::handle_push(
t);
}
-void ReplicatedBackend::send_pushes(int prio, map<int, vector<PushOp> > &pushes)
+void ReplicatedBackend::send_pushes(int prio, map<pg_shard_t, vector<PushOp> > &pushes)
{
- for (map<int, vector<PushOp> >::iterator i = pushes.begin();
+ for (map<pg_shard_t, vector<PushOp> >::iterator i = pushes.begin();
i != pushes.end();
++i) {
- ConnectionRef con = osd->get_con_osd_cluster(
- i->first,
+ ConnectionRef con = get_parent()->get_con_osd_cluster(
+ i->first.osd,
get_osdmap()->get_epoch());
if (!con)
continue;
@@ -7792,7 +8018,8 @@ void ReplicatedBackend::send_pushes(int prio, map<int, vector<PushOp> > &pushes)
uint64_t cost = 0;
uint64_t pushes = 0;
MOSDPGPush *msg = new MOSDPGPush();
- msg->pgid = get_info().pgid;
+ msg->from = get_parent()->whoami_shard();
+ msg->pgid = get_parent()->primary_spg_t();
msg->map_epoch = get_osdmap()->get_epoch();
msg->set_priority(prio);
for (;
@@ -7807,19 +8034,19 @@ void ReplicatedBackend::send_pushes(int prio, map<int, vector<PushOp> > &pushes)
msg->pushes.push_back(*j);
}
msg->compute_cost(cct);
- osd->send_message_osd_cluster(msg, con);
+ get_parent()->send_message_osd_cluster(msg, con);
}
}
}
}
-void ReplicatedBackend::send_pulls(int prio, map<int, vector<PullOp> > &pulls)
+void ReplicatedBackend::send_pulls(int prio, map<pg_shard_t, vector<PullOp> > &pulls)
{
- for (map<int, vector<PullOp> >::iterator i = pulls.begin();
+ for (map<pg_shard_t, vector<PullOp> >::iterator i = pulls.begin();
i != pulls.end();
++i) {
- ConnectionRef con = osd->get_con_osd_cluster(
- i->first,
+ ConnectionRef con = get_parent()->get_con_osd_cluster(
+ i->first.osd,
get_osdmap()->get_epoch());
if (!con)
continue;
@@ -7839,12 +8066,13 @@ void ReplicatedBackend::send_pulls(int prio, map<int, vector<PullOp> > &pulls)
dout(20) << __func__ << ": sending pulls " << i->second
<< " to osd." << i->first << dendl;
MOSDPGPull *msg = new MOSDPGPull();
+ msg->from = parent->whoami_shard();
msg->set_priority(prio);
- msg->pgid = get_info().pgid;
+ msg->pgid = get_parent()->primary_spg_t();
msg->map_epoch = get_osdmap()->get_epoch();
msg->pulls.swap(i->second);
msg->compute_cost(cct);
- osd->send_message_osd_cluster(msg, con);
+ get_parent()->send_message_osd_cluster(msg, con);
}
}
}
@@ -7868,20 +8096,19 @@ int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
<< dendl;
if (progress.first) {
- osd->store->omap_get_header(coll, recovery_info.soid, &out_op->omap_header);
- osd->store->getattrs(coll, recovery_info.soid, out_op->attrset);
+ store->omap_get_header(coll, recovery_info.soid, &out_op->omap_header);
+ store->getattrs(coll, recovery_info.soid, out_op->attrset);
// Debug
- bufferlist bv;
- bv.push_back(out_op->attrset[OI_ATTR]);
+ bufferlist bv = out_op->attrset[OI_ATTR];
object_info_t oi(bv);
if (oi.version != recovery_info.version) {
- osd->clog.error() << get_info().pgid << " push "
- << recovery_info.soid << " v "
- << recovery_info.version
- << " failed because local copy is "
- << oi.version << "\n";
+ get_parent()->clog_error() << get_info().pgid << " push "
+ << recovery_info.soid << " v "
+ << recovery_info.version
+ << " failed because local copy is "
+ << oi.version << "\n";
return -EINVAL;
}
@@ -7891,8 +8118,8 @@ int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
uint64_t available = cct->_conf->osd_recovery_max_chunk;
if (!progress.omap_complete) {
ObjectMap::ObjectMapIterator iter =
- osd->store->get_omap_iterator(coll,
- recovery_info.soid);
+ store->get_omap_iterator(coll,
+ recovery_info.soid);
for (iter->lower_bound(progress.omap_recovered_to);
iter->valid();
iter->next()) {
@@ -7924,7 +8151,7 @@ int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
p != out_op->data_included.end();
++p) {
bufferlist bit;
- osd->store->read(coll, recovery_info.soid,
+ store->read(coll, recovery_info.soid,
p.get_start(), p.get_len(), bit);
if (p.get_len() != bit.length()) {
dout(10) << " extent " << p.get_start() << "~" << p.get_len()
@@ -7950,8 +8177,8 @@ int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
stat->num_bytes_recovered += out_op->data.length();
}
- osd->logger->inc(l_osd_push);
- osd->logger->inc(l_osd_push_outb, out_op->data.length());
+ get_parent()->get_logger()->inc(l_osd_push);
+ get_parent()->get_logger()->inc(l_osd_push_outb, out_op->data.length());
// send
out_op->version = recovery_info.version;
@@ -7962,13 +8189,15 @@ int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
return 0;
}
-int ReplicatedBackend::send_push_op_legacy(int prio, int peer, PushOp &pop)
+int ReplicatedBackend::send_push_op_legacy(int prio, pg_shard_t peer, PushOp &pop)
{
- tid_t tid = osd->get_tid();
- osd_reqid_t rid(osd->get_cluster_msgr_name(), 0, tid);
- MOSDSubOp *subop = new MOSDSubOp(rid, get_info().pgid, pop.soid,
- false, 0, get_osdmap()->get_epoch(),
- tid, pop.recovery_info.version);
+ tid_t tid = get_parent()->get_tid();
+ osd_reqid_t rid(get_parent()->get_cluster_msgr_name(), 0, tid);
+ MOSDSubOp *subop = new MOSDSubOp(
+ rid, parent->whoami_shard(),
+ spg_t(get_info().pgid.pgid, peer.shard), pop.soid,
+ false, 0, get_osdmap()->get_epoch(),
+ tid, pop.recovery_info.version);
subop->ops = vector<OSDOp>(1);
subop->ops[0].op.op = CEPH_OSD_OP_PUSH;
@@ -7983,7 +8212,7 @@ int ReplicatedBackend::send_push_op_legacy(int prio, int peer, PushOp &pop)
subop->current_progress = pop.before_progress;
subop->recovery_progress = pop.after_progress;
- osd->send_message_osd_cluster(peer, subop, get_osdmap()->get_epoch());
+ get_parent()->send_message_osd_cluster(peer.osd, subop, get_osdmap()->get_epoch());
return 0;
}
@@ -8000,7 +8229,7 @@ void ReplicatedBackend::sub_op_push_reply(OpRequestRef op)
const hobject_t& soid = reply->get_poid();
assert(reply->get_header().type == MSG_OSD_SUBOPREPLY);
dout(10) << "sub_op_push_reply from " << reply->get_source() << " " << *reply << dendl;
- int peer = reply->get_source().num();
+ pg_shard_t peer = reply->from;
op->mark_started();
@@ -8012,7 +8241,7 @@ void ReplicatedBackend::sub_op_push_reply(OpRequestRef op)
send_push_op_legacy(op->get_req()->get_priority(), peer, pop);
}
-bool ReplicatedBackend::handle_push_reply(int peer, PushReplyOp &op, PushOp *reply)
+bool ReplicatedBackend::handle_push_reply(pg_shard_t peer, PushReplyOp &op, PushOp *reply)
{
const hobject_t &soid = op.soid;
if (pushing.count(soid) == 0) {
@@ -8110,24 +8339,24 @@ void ReplicatedBackend::sub_op_pull(OpRequestRef op)
pop.recovery_progress = m->recovery_progress;
PushOp reply;
- handle_pull(m->get_source().num(), pop, &reply);
+ handle_pull(m->from, pop, &reply);
send_push_op_legacy(
m->get_priority(),
- m->get_source().num(),
+ m->from,
reply);
- log_subop_stats(osd, op, 0, l_osd_sop_pull_lat);
+ log_subop_stats(get_parent()->get_logger(), op, 0, l_osd_sop_pull_lat);
}
-void ReplicatedBackend::handle_pull(int peer, PullOp &op, PushOp *reply)
+void ReplicatedBackend::handle_pull(pg_shard_t peer, PullOp &op, PushOp *reply)
{
const hobject_t &soid = op.soid;
struct stat st;
- int r = osd->store->stat(coll, soid, &st);
+ int r = store->stat(coll, soid, &st);
if (r != 0) {
- osd->clog.error() << get_info().pgid << " "
- << peer << " tried to pull " << soid
- << " but got " << cpp_strerror(-r) << "\n";
+ get_parent()->clog_error() << get_info().pgid << " "
+ << peer << " tried to pull " << soid
+ << " but got " << cpp_strerror(-r) << "\n";
prep_push_op_blank(soid, reply);
} else {
ObjectRecoveryInfo &recovery_info = op.recovery_info;
@@ -8160,10 +8389,13 @@ void ReplicatedPG::_committed_pushed_object(
if (!is_primary()) {
// Either we are a replica or backfill target.
// we are fully up to date. tell the primary!
- osd->send_message_osd_cluster(get_primary(),
- new MOSDPGTrim(get_osdmap()->get_epoch(), info.pgid,
- last_complete_ondisk),
- get_osdmap()->get_epoch());
+ osd->send_message_osd_cluster(
+ get_primary().osd,
+ new MOSDPGTrim(
+ get_osdmap()->get_epoch(),
+ spg_t(info.pgid.pgid, primary.shard),
+ last_complete_ondisk),
+ get_osdmap()->get_epoch());
} else {
// we are the primary. tell replicas to trim?
if (calc_min_last_complete_ondisk())
@@ -8216,10 +8448,6 @@ void ReplicatedPG::_applied_recovered_object_replica()
void ReplicatedPG::recover_got(hobject_t oid, eversion_t v)
{
dout(10) << "got missing " << oid << " v " << v << dendl;
- if (pg_log.get_missing().is_missing(oid, v)) {
- if (is_primary())
- missing_loc.erase(oid);
- }
pg_log.recover_got(oid, v, info);
if (pg_log.get_log().complete_to != pg_log.get_log().log.end()) {
dout(10) << "last_complete now " << info.last_complete
@@ -8305,12 +8533,12 @@ void ReplicatedBackend::sub_op_push(OpRequestRef op)
RPGHandle *h = _open_recovery_op();
list<hobject_t> to_continue;
bool more = handle_pull_response(
- m->get_source().num(), pop, &resp,
+ m->from, pop, &resp,
&to_continue, t);
if (more) {
send_pull_legacy(
m->get_priority(),
- m->get_source().num(),
+ m->from,
resp.recovery_info,
resp.recovery_progress);
} else {
@@ -8320,20 +8548,21 @@ void ReplicatedBackend::sub_op_push(OpRequestRef op)
op->get_req()->get_priority());
c->to_continue.swap(to_continue);
t->register_on_complete(
- new C_QueueInWQ(
- &osd->push_wq,
+ new PG_QueueAsync(
+ get_parent(),
get_parent()->bless_gencontext(c)));
}
run_recovery_op(h, op->get_req()->get_priority());
} else {
PushReplyOp resp;
MOSDSubOpReply *reply = new MOSDSubOpReply(
- m, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+ m, parent->whoami_shard(), 0,
+ get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
reply->set_priority(m->get_priority());
assert(entity_name_t::TYPE_OSD == m->get_connection()->peer_type);
- handle_push(m->get_source().num(), pop, &resp, t);
- t->register_on_complete(new C_OSD_SendMessageOnConn(
- osd, reply, m->get_connection()));
+ handle_push(m->from, pop, &resp, t);
+ t->register_on_complete(new PG_SendMessageOnConn(
+ get_parent(), reply, m->get_connection()));
}
t->register_on_applied(
new ObjectStore::C_DeleteTransaction(t));
@@ -8341,26 +8570,18 @@ void ReplicatedBackend::sub_op_push(OpRequestRef op)
return;
}
-void ReplicatedPG::failed_push(int from, const hobject_t &soid)
+void ReplicatedPG::failed_push(pg_shard_t from, const hobject_t &soid)
{
assert(recovering.count(soid));
recovering.erase(soid);
- map<hobject_t,set<int> >::iterator p = missing_loc.find(soid);
- if (p != missing_loc.end()) {
- dout(0) << "_failed_push " << soid << " from osd." << from
- << ", reps on " << p->second << dendl;
-
- p->second.erase(from); // forget about this (bad) peer replica
- if (p->second.empty())
- missing_loc.erase(p);
- } else {
- dout(0) << "_failed_push " << soid << " from osd." << from
- << " but not in missing_loc ???" << dendl;
- }
+ missing_loc.remove_location(soid, from);
+ dout(0) << "_failed_push " << soid << " from shard " << from
+ << ", reps on " << missing_loc.get_locations(soid)
+ << " unfound? " << missing_loc.is_unfound(soid) << dendl;
finish_recovery_op(soid); // close out this attempt,
}
-void ReplicatedBackend::_failed_push(int from, const hobject_t &soid)
+void ReplicatedBackend::_failed_push(pg_shard_t from, const hobject_t &soid)
{
get_parent()->failed_push(from, soid);
pull_from_peer[from].erase(soid);
@@ -8393,8 +8614,11 @@ eversion_t ReplicatedPG::pick_newest_available(const hobject_t& oid)
dout(10) << "pick_newest_available " << oid << " " << v << " on osd." << osd->whoami << " (local)" << dendl;
assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); ++i) {
- int peer = actingbackfill[i];
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_shard_t peer = *i;
if (!peer_missing[peer].is_missing(oid)) {
assert(is_backfill_targets(peer));
continue;
@@ -8419,8 +8643,8 @@ ObjectContextRef ReplicatedPG::mark_object_lost(ObjectStore::Transaction *t,
// Wake anyone waiting for this object. Now that it's been marked as lost,
// we will just return an error code.
map<hobject_t, list<OpRequestRef> >::iterator wmo =
- waiting_for_missing_object.find(oid);
- if (wmo != waiting_for_missing_object.end()) {
+ waiting_for_unreadable_object.find(oid);
+ if (wmo != waiting_for_unreadable_object.end()) {
requeue_ops(wmo->second);
}
@@ -8473,7 +8697,7 @@ void ReplicatedPG::mark_all_unfound_lost(int what)
map<hobject_t, pg_missing_t::item>::const_iterator mend = missing.missing.end();
while (m != mend) {
const hobject_t &oid(m->first);
- if (missing_loc.find(oid) != missing_loc.end()) {
+ if (!missing_loc.is_unfound(oid)) {
// We only care about unfound objects
++m;
continue;
@@ -8505,6 +8729,7 @@ void ReplicatedPG::mark_all_unfound_lost(int what)
// we are now missing the new version; recovery code will sort it out.
++m;
pg_log.revise_need(oid, info.last_update);
+ missing_loc.revise_need(oid, info.last_update);
break;
}
/** fall-thru **/
@@ -8523,6 +8748,7 @@ void ReplicatedPG::mark_all_unfound_lost(int what)
assert(0 == "not implemented.. tho i'm not sure how useful it really would be.");
}
pg_log.missing_rm(m++);
+ missing_loc.recovered(oid);
}
break;
@@ -8548,6 +8774,8 @@ void ReplicatedPG::mark_all_unfound_lost(int what)
dirty_info = true;
write_if_dirty(*t);
+ t->register_on_complete(new ObjectStore::C_DeleteTransaction(t));
+
osd->store->queue_transaction(osr.get(), t, c, NULL, new C_OSD_OndiskWriteUnlockList(&c->obcs));
// Send out the PG log to all replicas
@@ -8590,6 +8818,10 @@ void ReplicatedPG::apply_and_flush_repops(bool requeue)
repop_queue.pop_front();
dout(10) << " applying repop tid " << repop->rep_tid << dendl;
repop->rep_aborted = true;
+ if (repop->on_applied) {
+ delete repop->on_applied;
+ repop->on_applied = NULL;
+ }
if (requeue) {
if (repop->ctx->op) {
@@ -8612,7 +8844,20 @@ void ReplicatedPG::apply_and_flush_repops(bool requeue)
if (requeue) {
requeue_ops(rq);
- assert(waiting_for_ondisk.empty());
+ if (!waiting_for_ondisk.empty()) {
+ for (map<eversion_t, list<OpRequestRef> >::iterator i =
+ waiting_for_ondisk.begin();
+ i != waiting_for_ondisk.end();
+ ++i) {
+ for (list<OpRequestRef>::iterator j = i->second.begin();
+ j != i->second.end();
+ ++j) {
+ derr << __func__ << ": op " << *((*j)->get_req()) << " waiting on "
+ << i->first << dendl;
+ }
+ }
+ assert(waiting_for_ondisk.empty());
+ }
}
waiting_for_ondisk.clear();
@@ -8673,27 +8918,59 @@ void ReplicatedPG::on_shutdown()
osd->remote_reserver.cancel_reservation(info.pgid);
osd->local_reserver.cancel_reservation(info.pgid);
- clear_primary_state();
- osd->remove_want_pg_temp(info.pgid);
+ if (is_primary())
+ clear_primary_state(false); // Not staying primary
cancel_recovery();
}
void ReplicatedPG::on_activate()
{
+ // all clean?
+ if (needs_recovery()) {
+ dout(10) << "activate not all replicas are up-to-date, queueing recovery" << dendl;
+ queue_peering_event(
+ CephPeeringEvtRef(
+ new CephPeeringEvt(
+ get_osdmap()->get_epoch(),
+ get_osdmap()->get_epoch(),
+ DoRecovery())));
+ } else if (needs_backfill()) {
+ dout(10) << "activate queueing backfill" << dendl;
+ queue_peering_event(
+ CephPeeringEvtRef(
+ new CephPeeringEvt(
+ get_osdmap()->get_epoch(),
+ get_osdmap()->get_epoch(),
+ RequestBackfill())));
+ } else {
+ dout(10) << "activate all replicas clean, no recovery" << dendl;
+ queue_peering_event(
+ CephPeeringEvtRef(
+ new CephPeeringEvt(
+ get_osdmap()->get_epoch(),
+ get_osdmap()->get_epoch(),
+ AllReplicasRecovered())));
+ }
+
+ publish_stats_to_osd();
+
if (!backfill_targets.empty()) {
last_backfill_started = earliest_backfill();
new_backfill = true;
assert(!last_backfill_started.is_max());
dout(5) << "on activate: bft=" << backfill_targets
<< " from " << last_backfill_started << dendl;
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- dout(5) << "target osd." << backfill_targets[i]
- << " from " << peer_info[backfill_targets[i]].last_backfill
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ dout(5) << "target shard " << *i
+ << " from " << peer_info[*i].last_backfill
<< dendl;
}
}
hit_set_setup();
+ agent_setup();
}
void ReplicatedPG::on_change(ObjectStore::Transaction *t)
@@ -8717,7 +8994,7 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
in_progress_async_reads.begin();
i != in_progress_async_reads.end();
in_progress_async_reads.erase(i++)) {
- close_op_ctx(i->second);
+ close_op_ctx(i->second, -ECANCELED);
requeue_op(i->first);
}
@@ -8726,9 +9003,9 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
// requeue object waiters
if (is_primary()) {
- requeue_object_waiters(waiting_for_missing_object);
+ requeue_object_waiters(waiting_for_unreadable_object);
} else {
- waiting_for_missing_object.clear();
+ waiting_for_unreadable_object.clear();
}
for (map<hobject_t,list<OpRequestRef> >::iterator p = waiting_for_degraded_object.begin();
p != waiting_for_degraded_object.end();
@@ -8748,10 +9025,13 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
p->second.clear();
}
- if (is_primary())
+ if (is_primary()) {
+ requeue_ops(waiting_for_cache_not_full);
requeue_ops(waiting_for_all_missing);
- else
+ } else {
+ waiting_for_cache_not_full.clear();
waiting_for_all_missing.clear();
+ }
// this will requeue ops we were working on but didn't finish, and
// any dups
@@ -8779,13 +9059,13 @@ void ReplicatedPG::on_pool_change()
{
dout(10) << __func__ << dendl;
hit_set_setup();
+ agent_setup();
}
// clear state. called on recovery completion AND cancellation.
void ReplicatedPG::_clear_recovery_state()
{
missing_loc.clear();
- missing_loc_sources.clear();
#ifdef DEBUG_RECOVERY_OIDS
recovering_oids.clear();
#endif
@@ -8809,7 +9089,8 @@ void ReplicatedPG::cancel_pull(const hobject_t &soid)
assert(recovering.count(soid));
recovering.erase(soid);
finish_recovery_op(soid);
- pg_log.set_last_requested(0); // get recover_primary to start over
+ if (is_missing_object(soid))
+ pg_log.set_last_requested(0); // get recover_primary to start over
}
void ReplicatedPG::check_recovery_sources(const OSDMapRef osdmap)
@@ -8818,11 +9099,39 @@ void ReplicatedPG::check_recovery_sources(const OSDMapRef osdmap)
* check that any peers we are planning to (or currently) pulling
* objects from are dealt with.
*/
- set<int> now_down;
- for (set<int>::iterator p = missing_loc_sources.begin();
+ missing_loc.check_recovery_sources(osdmap);
+ pgbackend->check_recovery_sources(osdmap);
+
+ for (set<pg_shard_t>::iterator i = peer_log_requested.begin();
+ i != peer_log_requested.end();
+ ) {
+ if (!osdmap->is_up(i->osd)) {
+ dout(10) << "peer_log_requested removing " << *i << dendl;
+ peer_log_requested.erase(i++);
+ } else {
+ ++i;
+ }
+ }
+
+ for (set<pg_shard_t>::iterator i = peer_missing_requested.begin();
+ i != peer_missing_requested.end();
+ ) {
+ if (!osdmap->is_up(i->osd)) {
+ dout(10) << "peer_missing_requested removing " << *i << dendl;
+ peer_missing_requested.erase(i++);
+ } else {
+ ++i;
+ }
+ }
+}
+
+void PG::MissingLoc::check_recovery_sources(const OSDMapRef osdmap)
+{
+ set<pg_shard_t> now_down;
+ for (set<pg_shard_t>::iterator p = missing_loc_sources.begin();
p != missing_loc_sources.end();
) {
- if (osdmap->is_up(*p)) {
+ if (osdmap->is_up(p->osd)) {
++p;
continue;
}
@@ -8830,7 +9139,6 @@ void ReplicatedPG::check_recovery_sources(const OSDMapRef osdmap)
now_down.insert(*p);
missing_loc_sources.erase(p++);
}
- pgbackend->check_recovery_sources(osdmap);
if (now_down.empty()) {
dout(10) << "check_recovery_sources no source osds (" << missing_loc_sources << ") went down" << dendl;
@@ -8839,14 +9147,13 @@ void ReplicatedPG::check_recovery_sources(const OSDMapRef osdmap)
<< missing_loc_sources << dendl;
// filter missing_loc
- map<hobject_t, set<int> >::iterator p = missing_loc.begin();
+ map<hobject_t, set<pg_shard_t> >::iterator p = missing_loc.begin();
while (p != missing_loc.end()) {
- set<int>::iterator q = p->second.begin();
+ set<pg_shard_t>::iterator q = p->second.begin();
while (q != p->second.end())
if (now_down.count(*q)) {
p->second.erase(q++);
} else {
- assert(missing_loc_sources.count(*q));
++q;
}
if (p->second.empty())
@@ -8855,28 +9162,6 @@ void ReplicatedPG::check_recovery_sources(const OSDMapRef osdmap)
++p;
}
}
-
- for (set<int>::iterator i = peer_log_requested.begin();
- i != peer_log_requested.end();
- ) {
- if (!osdmap->is_up(*i)) {
- dout(10) << "peer_log_requested removing " << *i << dendl;
- peer_log_requested.erase(i++);
- } else {
- ++i;
- }
- }
-
- for (set<int>::iterator i = peer_missing_requested.begin();
- i != peer_missing_requested.end();
- ) {
- if (!osdmap->is_up(*i)) {
- dout(10) << "peer_missing_requested removing " << *i << dendl;
- peer_missing_requested.erase(i++);
- } else {
- ++i;
- }
- }
}
@@ -8961,6 +9246,12 @@ bool ReplicatedPG::start_recovery_ops(
assert(recovering.empty());
assert(recovery_ops_active == 0);
+ dout(10) << __func__ << " needs_recovery: "
+ << missing_loc.get_needs_recovery()
+ << dendl;
+ dout(10) << __func__ << " missing_loc: "
+ << missing_loc.get_missing_locs()
+ << dendl;
int unfound = get_num_unfound();
if (unfound) {
dout(10) << " still have " << unfound << " unfound" << dendl;
@@ -9058,7 +9349,7 @@ int ReplicatedPG::recover_primary(int max, ThreadPool::TPHandle &handle)
eversion_t need = item.need;
- bool unfound = (missing_loc.find(soid) == missing_loc.end());
+ bool unfound = missing_loc.is_unfound(soid);
dout(10) << "recover_primary "
<< soid << " " << item.need
@@ -9099,6 +9390,7 @@ int ReplicatedPG::recover_primary(int max, ThreadPool::TPHandle &handle)
t->setattr(coll, soid, OI_ATTR, b2);
recover_got(soid, latest->version);
+ missing_loc.add_location(soid, pg_whoami);
++active_pushes;
@@ -9126,14 +9418,16 @@ int ReplicatedPG::recover_primary(int max, ThreadPool::TPHandle &handle)
eversion_t alternate_need = latest->reverting_to;
dout(10) << " need to pull prior_version " << alternate_need << " for revert " << item << dendl;
- set<int>& loc = missing_loc[soid];
- for (map<int,pg_missing_t>::iterator p = peer_missing.begin(); p != peer_missing.end(); ++p)
+ for (map<pg_shard_t, pg_missing_t>::iterator p = peer_missing.begin();
+ p != peer_missing.end();
+ ++p)
if (p->second.is_missing(soid, need) &&
p->second.missing[soid].have == alternate_need) {
- missing_loc_sources.insert(p->first);
- loc.insert(p->first);
+ missing_loc.add_location(soid, p->first);
}
- dout(10) << " will pull " << alternate_need << " or " << need << " from one of " << loc << dendl;
+ dout(10) << " will pull " << alternate_need << " or " << need
+ << " from one of " << missing_loc.get_locations(soid)
+ << dendl;
unfound = false;
}
}
@@ -9186,15 +9480,18 @@ int ReplicatedPG::prep_object_replica_pushes(
ObjectContextRef obc = get_object_context(soid, false);
if (!obc) {
pg_log.missing_add(soid, v, eversion_t());
+ missing_loc.remove_location(soid, pg_whoami);
bool uhoh = true;
assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++) {
- int peer = actingbackfill[i];
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_shard_t peer = *i;
if (!peer_missing[peer].is_missing(soid, v)) {
- missing_loc[soid].insert(peer);
- missing_loc_sources.insert(peer);
+ missing_loc.add_location(soid, peer);
dout(10) << info.pgid << " unexpectedly missing " << soid << " v" << v
- << ", there should be a copy on osd." << peer << dendl;
+ << ", there should be a copy on shard " << peer << dendl;
uhoh = false;
}
}
@@ -9202,7 +9499,8 @@ int ReplicatedPG::prep_object_replica_pushes(
osd->clog.error() << info.pgid << " missing primary copy of " << soid << ", unfound\n";
else
osd->clog.error() << info.pgid << " missing primary copy of " << soid
- << ", will try copies on " << missing_loc[soid] << "\n";
+ << ", will try copies on " << missing_loc.get_locations(soid)
+ << "\n";
return 0;
}
@@ -9218,6 +9516,7 @@ int ReplicatedPG::prep_object_replica_pushes(
obc->ondisk_read_lock();
pgbackend->recover_object(
soid,
+ v,
ObjectContextRef(),
obc, // has snapset context
h);
@@ -9232,12 +9531,16 @@ int ReplicatedBackend::start_pushes(
{
int pushes = 0;
// who needs it?
- assert(get_parent()->get_actingbackfill().size() > 0);
- for (unsigned i=1; i<get_parent()->get_actingbackfill().size(); i++) {
- int peer = get_parent()->get_actingbackfill()[i];
- map<int, pg_missing_t>::const_iterator j =
- get_parent()->get_peer_missing().find(peer);
- assert(j != get_parent()->get_peer_missing().end());
+ assert(get_parent()->get_actingbackfill_shards().size() > 0);
+ for (set<pg_shard_t>::iterator i =
+ get_parent()->get_actingbackfill_shards().begin();
+ i != get_parent()->get_actingbackfill_shards().end();
+ ++i) {
+ if (*i == get_parent()->whoami_shard()) continue;
+ pg_shard_t peer = *i;
+ map<pg_shard_t, pg_missing_t>::const_iterator j =
+ get_parent()->get_shard_missing().find(peer);
+ assert(j != get_parent()->get_shard_missing().end());
if (j->second.is_missing(soid)) {
++pushes;
h->pushes[peer].push_back(PushOp());
@@ -9258,11 +9561,14 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
// this is FAR from an optimal recovery order. pretty lame, really.
assert(actingbackfill.size() > 0);
- for (unsigned i=1; i<actingbackfill.size(); i++) {
- int peer = actingbackfill[i];
- map<int, pg_missing_t>::const_iterator pm = peer_missing.find(peer);
+ for (set<pg_shard_t>::iterator i = actingbackfill.begin();
+ i != actingbackfill.end();
+ ++i) {
+ if (*i == get_primary()) continue;
+ pg_shard_t peer = *i;
+ map<pg_shard_t, pg_missing_t>::const_iterator pm = peer_missing.find(peer);
assert(pm != peer_missing.end());
- map<int, pg_info_t>::const_iterator pi = peer_info.find(peer);
+ map<pg_shard_t, pg_info_t>::const_iterator pi = peer_info.find(peer);
assert(pi != peer_info.end());
size_t m_sz = pm->second.num_missing();
@@ -9291,11 +9597,13 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
continue;
}
+ if (missing_loc.is_unfound(soid)) {
+ dout(10) << __func__ << ": " << soid << " still unfound" << dendl;
+ continue;
+ }
+
if (pg_log.get_missing().is_missing(soid)) {
- if (missing_loc.find(soid) == missing_loc.end())
- dout(10) << __func__ << ": " << soid << " still unfound" << dendl;
- else
- dout(10) << __func__ << ": " << soid << " still missing on primary" << dendl;
+ dout(10) << __func__ << ": " << soid << " still missing on primary" << dendl;
continue;
}
@@ -9313,9 +9621,11 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
hobject_t ReplicatedPG::earliest_peer_backfill() const
{
hobject_t e = hobject_t::get_max();
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int peer = backfill_targets[i];
- map<int, BackfillInterval>::const_iterator iter =
+ for (set<pg_shard_t>::const_iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t peer = *i;
+ map<pg_shard_t, BackfillInterval>::const_iterator iter =
peer_backfill_info.find(peer);
assert(iter != peer_backfill_info.end());
if (iter->second.begin < e)
@@ -9329,9 +9639,11 @@ bool ReplicatedPG::all_peer_done() const
// Primary hasn't got any more objects
assert(backfill_info.empty());
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
- map<int, BackfillInterval>::const_iterator piter =
+ for (set<pg_shard_t>::const_iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
+ map<pg_shard_t, BackfillInterval>::const_iterator piter =
peer_backfill_info.find(bt);
assert(piter != peer_backfill_info.end());
const BackfillInterval& pbi = piter->second;
@@ -9384,20 +9696,22 @@ int ReplicatedPG::recover_backfill(
// on_activate() was called prior to getting here
assert(last_backfill_started == earliest_backfill());
new_backfill = false;
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
- peer_backfill_info[bt].reset(peer_info[bt].last_backfill);
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ peer_backfill_info[*i].reset(peer_info[*i].last_backfill);
}
backfill_info.reset(last_backfill_started);
}
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
- dout(10) << "peer osd." << bt
- << " info " << peer_info[bt]
- << " interval " << peer_backfill_info[bt].begin
- << "-" << peer_backfill_info[bt].end
- << " " << peer_backfill_info[bt].objects.size() << " objects"
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ dout(10) << "peer osd." << *i
+ << " info " << peer_info[*i]
+ << " interval " << peer_backfill_info[*i].begin
+ << "-" << peer_backfill_info[*i].end
+ << " " << peer_backfill_info[*i].objects.size() << " objects"
<< dendl;
}
@@ -9407,13 +9721,14 @@ int ReplicatedPG::recover_backfill(
int ops = 0;
vector<boost::tuple<hobject_t, eversion_t,
- ObjectContextRef, vector<int> > > to_push;
- vector<boost::tuple<hobject_t, eversion_t, int> > to_remove;
+ ObjectContextRef, vector<pg_shard_t> > > to_push;
+ vector<boost::tuple<hobject_t, eversion_t, pg_shard_t> > to_remove;
set<hobject_t> add_to_stat;
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
- peer_backfill_info[bt].trim_to(last_backfill_started);
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ peer_backfill_info[*i].trim_to(last_backfill_started);
}
backfill_info.trim_to(last_backfill_started);
@@ -9436,19 +9751,23 @@ int ReplicatedPG::recover_backfill(
<< dendl;
bool sent_scan = false;
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
BackfillInterval& pbi = peer_backfill_info[bt];
- dout(20) << " peer osd." << bt << " backfill " << pbi.begin << "-"
+ dout(20) << " peer shard " << bt << " backfill " << pbi.begin << "-"
<< pbi.end << " " << pbi.objects << dendl;
if (pbi.begin <= backfill_info.begin &&
!pbi.extends_to_end() && pbi.empty()) {
dout(10) << " scanning peer osd." << bt << " from " << pbi.end << dendl;
epoch_t e = get_osdmap()->get_epoch();
- MOSDPGScan *m = new MOSDPGScan(MOSDPGScan::OP_SCAN_GET_DIGEST, e, e, info.pgid,
- pbi.end, hobject_t());
- osd->send_message_osd_cluster(bt, m, get_osdmap()->get_epoch());
+ MOSDPGScan *m = new MOSDPGScan(
+ MOSDPGScan::OP_SCAN_GET_DIGEST, pg_whoami, e, e,
+ spg_t(info.pgid.pgid, bt.shard),
+ pbi.end, hobject_t());
+ osd->send_message_osd_cluster(bt.osd, m, get_osdmap()->get_epoch());
assert(waiting_on_backfill.find(bt) == waiting_on_backfill.end());
waiting_on_backfill.insert(bt);
sent_scan = true;
@@ -9473,19 +9792,23 @@ int ReplicatedPG::recover_backfill(
if (check < backfill_info.begin) {
- vector<int> check_targets;
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
+ set<pg_shard_t> check_targets;
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
BackfillInterval& pbi = peer_backfill_info[bt];
if (pbi.begin == check)
- check_targets.push_back(bt);
+ check_targets.insert(bt);
}
assert(!check_targets.empty());
dout(20) << " BACKFILL removing " << check
<< " from peers " << check_targets << dendl;
- for (unsigned i = 0; i < check_targets.size(); ++i) {
- int bt = check_targets[i];
+ for (set<pg_shard_t>::iterator i = check_targets.begin();
+ i != check_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
BackfillInterval& pbi = peer_backfill_info[bt];
assert(pbi.begin == check);
@@ -9500,9 +9823,11 @@ int ReplicatedPG::recover_backfill(
} else {
eversion_t& obj_v = backfill_info.objects.begin()->second;
- vector<int> need_ver_targs, missing_targs, keep_ver_targs, skip_targs;
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
+ vector<pg_shard_t> need_ver_targs, missing_targs, keep_ver_targs, skip_targs;
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
BackfillInterval& pbi = peer_backfill_info[bt];
// Find all check peers that have the wrong version
if (check == backfill_info.begin && check == pbi.begin) {
@@ -9545,11 +9870,11 @@ int ReplicatedPG::recover_backfill(
<< " with ver " << obj_v
<< " to peers " << missing_targs << dendl;
}
- vector<int> all_push = need_ver_targs;
+ vector<pg_shard_t> all_push = need_ver_targs;
all_push.insert(all_push.end(), missing_targs.begin(), missing_targs.end());
to_push.push_back(
- boost::tuple<hobject_t, eversion_t, ObjectContextRef, vector<int> >
+ boost::tuple<hobject_t, eversion_t, ObjectContextRef, vector<pg_shard_t> >
(backfill_info.begin, obj_v, obc, all_push));
// Count all simultaneous pushes of the same object as a single op
ops++;
@@ -9569,10 +9894,12 @@ int ReplicatedPG::recover_backfill(
last_backfill_started = backfill_info.begin;
add_to_stat.insert(backfill_info.begin); // XXX: Only one for all pushes?
backfill_info.pop_front();
- vector<int> check_targets = need_ver_targs;
+ vector<pg_shard_t> check_targets = need_ver_targs;
check_targets.insert(check_targets.end(), keep_ver_targs.begin(), keep_ver_targs.end());
- for (unsigned i = 0; i < check_targets.size(); ++i) {
- int bt = check_targets[i];
+ for (vector<pg_shard_t>::iterator i = check_targets.begin();
+ i != check_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
BackfillInterval& pbi = peer_backfill_info[bt];
pbi.pop_front();
}
@@ -9621,8 +9948,10 @@ int ReplicatedPG::recover_backfill(
i->first < next_backfill_to_complete;
pending_backfill_updates.erase(i++)) {
assert(i->first > new_last_backfill);
- for (unsigned j = 0; j < backfill_targets.size(); ++j) {
- int bt = backfill_targets[j];
+ for (set<pg_shard_t>::iterator j = backfill_targets.begin();
+ j != backfill_targets.end();
+ ++j) {
+ pg_shard_t bt = *j;
pg_info_t& pinfo = peer_info[bt];
//Add stats to all peers that were missing object
if (i->first > pinfo.last_backfill)
@@ -9632,16 +9961,6 @@ int ReplicatedPG::recover_backfill(
}
dout(10) << "possible new_last_backfill at " << new_last_backfill << dendl;
- /* If last_backfill is snapdir, we know that head necessarily cannot exist,
- * therefore it's safe to bump the snap up to NOSNAP. This is necessary
- * since we need avoid having SNAPDIR backfilled and HEAD not backfilled
- * since a transaction on HEAD might change SNAPDIR
- */
- if (new_last_backfill.is_snapdir())
- new_last_backfill = new_last_backfill.get_head();
- if (last_backfill_started.is_snapdir())
- last_backfill_started = last_backfill_started.get_head();
-
assert(!pending_backfill_updates.empty() ||
new_last_backfill == last_backfill_started);
if (pending_backfill_updates.empty() &&
@@ -9655,8 +9974,10 @@ int ReplicatedPG::recover_backfill(
// If new_last_backfill == MAX, then we will send OP_BACKFILL_FINISH to
// all the backfill targets. Otherwise, we will move last_backfill up on
// those targets need it and send OP_BACKFILL_PROGRESS to them.
- for (unsigned i = 0; i < backfill_targets.size(); ++i) {
- int bt = backfill_targets[i];
+ for (set<pg_shard_t>::iterator i = backfill_targets.begin();
+ i != backfill_targets.end();
+ ++i) {
+ pg_shard_t bt = *i;
pg_info_t& pinfo = peer_info[bt];
if (new_last_backfill > pinfo.last_backfill) {
@@ -9664,7 +9985,11 @@ int ReplicatedPG::recover_backfill(
epoch_t e = get_osdmap()->get_epoch();
MOSDPGBackfill *m = NULL;
if (pinfo.last_backfill.is_max()) {
- m = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_FINISH, e, e, info.pgid);
+ m = new MOSDPGBackfill(
+ MOSDPGBackfill::OP_BACKFILL_FINISH,
+ e,
+ e,
+ spg_t(info.pgid.pgid, bt.shard));
// Use default priority here, must match sub_op priority
/* pinfo.stats might be wrong if we did log-based recovery on the
* backfilled portion in addition to continuing backfill.
@@ -9672,13 +9997,17 @@ int ReplicatedPG::recover_backfill(
pinfo.stats = info.stats;
start_recovery_op(hobject_t::get_max());
} else {
- m = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_PROGRESS, e, e, info.pgid);
+ m = new MOSDPGBackfill(
+ MOSDPGBackfill::OP_BACKFILL_PROGRESS,
+ e,
+ e,
+ spg_t(info.pgid.pgid, bt.shard));
// Use default priority here, must match sub_op priority
}
m->last_backfill = pinfo.last_backfill;
m->stats = pinfo.stats;
- osd->send_message_osd_cluster(bt, m, get_osdmap()->get_epoch());
- dout(10) << " peer osd." << bt
+ osd->send_message_osd_cluster(bt.osd, m, get_osdmap()->get_epoch());
+ dout(10) << " peer " << bt
<< " num_objects now " << pinfo.stats.stats.sum.num_objects
<< " / " << info.stats.stats.sum.num_objects << dendl;
}
@@ -9692,7 +10021,7 @@ int ReplicatedPG::recover_backfill(
void ReplicatedPG::prep_backfill_object_push(
hobject_t oid, eversion_t v,
ObjectContextRef obc,
- vector<int> peers,
+ vector<pg_shard_t> peers,
PGBackend::RecoveryHandle *h)
{
dout(10) << "push_backfill_object " << oid << " v " << v << " to peers " << peers << dendl;
@@ -9700,7 +10029,7 @@ void ReplicatedPG::prep_backfill_object_push(
backfills_in_flight.insert(oid);
for (unsigned int i = 0 ; i < peers.size(); ++i) {
- map<int, pg_missing_t>::iterator bpm = peer_missing.find(peers[i]);
+ map<pg_shard_t, pg_missing_t>::iterator bpm = peer_missing.find(peers[i]);
assert(bpm != peer_missing.end());
bpm->second.add(oid, eversion_t(), eversion_t());
}
@@ -9714,6 +10043,7 @@ void ReplicatedPG::prep_backfill_object_push(
obc->ondisk_read_lock();
pgbackend->recover_object(
oid,
+ v,
ObjectContextRef(),
obc,
h);
@@ -9853,7 +10183,10 @@ void ReplicatedPG::check_local()
dout(10) << " checking " << p->soid
<< " at " << p->version << dendl;
struct stat st;
- int r = osd->store->stat(coll, p->soid, &st);
+ int r = osd->store->stat(
+ coll,
+ ghobject_t(p->soid, ghobject_t::NO_GEN, pg_whoami.shard),
+ &st);
if (r != -ENOENT) {
derr << __func__ << " " << p->soid << " exists, but should have been "
<< "deleted" << dendl;
@@ -9895,8 +10228,9 @@ hobject_t ReplicatedPG::get_hit_set_archive_object(utime_t start, utime_t end)
void ReplicatedPG::hit_set_clear()
{
dout(20) << __func__ << dendl;
- hit_set.reset(NULL);
+ hit_set.reset();
hit_set_start_stamp = utime_t();
+ hit_set_flushing.clear();
}
void ReplicatedPG::hit_set_setup()
@@ -9986,37 +10320,66 @@ bool ReplicatedPG::hit_set_apply_log()
return true;
}
+struct C_HitSetFlushing : public Context {
+ ReplicatedPGRef pg;
+ time_t hit_set_name;
+ C_HitSetFlushing(ReplicatedPG *p, time_t n) : pg(p), hit_set_name(n) { }
+ void finish(int r) {
+ pg->hit_set_flushing.erase(hit_set_name);
+ }
+};
+
void ReplicatedPG::hit_set_persist()
{
dout(10) << __func__ << dendl;
bufferlist bl;
+ unsigned max = pool.info.hit_set_count;
utime_t now = ceph_clock_now(cct);
RepGather *repop;
hobject_t oid;
- bool reset = false;
+ time_t flush_time = 0;
+
+ // See what start is going to be used later
+ utime_t start = info.hit_set.current_info.begin;
+ if (!start)
+ start = hit_set_start_stamp;
+
+ // If any archives are degraded we skip this persist request
+ // account for the additional entry being added below
+ for (unsigned num = info.hit_set.history.size() + 1; num > max; --num) {
+ list<pg_hit_set_info_t>::iterator p = info.hit_set.history.begin();
+ assert(p != info.hit_set.history.end());
+ hobject_t aoid = get_hit_set_archive_object(p->begin, p->end);
+
+ // Once we hit a degraded object just skip further trim
+ if (is_degraded_object(aoid))
+ return;
+ }
+ oid = get_hit_set_archive_object(start, now);
+ // If the current object is degraded we skip this persist request
+ if (is_degraded_object(oid))
+ return;
if (!info.hit_set.current_info.begin)
info.hit_set.current_info.begin = hit_set_start_stamp;
- if (hit_set->is_full() ||
- hit_set_start_stamp + pool.info.hit_set_period <= now) {
- // archive
- hit_set->seal();
- ::encode(*hit_set, bl);
- info.hit_set.current_info.end = now;
- oid = get_hit_set_archive_object(info.hit_set.current_info.begin,
- info.hit_set.current_info.end);
- dout(20) << __func__ << " archive " << oid << dendl;
- reset = true;
- } else {
- // persist snapshot of current hitset
- ::encode(*hit_set, bl);
- oid = get_hit_set_current_object(now);
- dout(20) << __func__ << " checkpoint " << oid << dendl;
- }
+
+ hit_set->seal();
+ ::encode(*hit_set, bl);
+ info.hit_set.current_info.end = now;
+ dout(20) << __func__ << " archive " << oid << dendl;
+
+ if (agent_state)
+ agent_state->add_hit_set(info.hit_set.current_info.begin, hit_set);
+
+ // hold a ref until it is flushed to disk
+ hit_set_flushing[info.hit_set.current_info.begin] = hit_set;
+ flush_time = info.hit_set.current_info.begin;
ObjectContextRef obc = get_object_context(oid, true);
repop = simple_repop_create(obc);
+ if (flush_time != 0)
+ repop->on_applied = new C_HitSetFlushing(this, flush_time);
OpContext *ctx = repop->ctx;
ctx->at_version = get_next_version();
@@ -10034,7 +10397,7 @@ void ReplicatedPG::hit_set_persist()
0,
osd_reqid_t(),
ctx->mtime));
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
ctx->op_t->stash(old_obj, ctx->at_version.version);
} else {
@@ -10047,7 +10410,10 @@ void ReplicatedPG::hit_set_persist()
++ctx->at_version.version;
struct stat st;
- int r = osd->store->stat(coll, old_obj, &st);
+ int r = osd->store->stat(
+ coll,
+ ghobject_t(old_obj, ghobject_t::NO_GEN, pg_whoami.shard),
+ &st);
assert(r == 0);
--ctx->delta_stats.num_objects;
ctx->delta_stats.num_bytes -= st.st_size;
@@ -10055,14 +10421,11 @@ void ReplicatedPG::hit_set_persist()
info.hit_set.current_last_update = info.last_update; // *after* above remove!
info.hit_set.current_info.version = ctx->at_version;
- if (reset) {
- info.hit_set.history.push_back(info.hit_set.current_info);
- hit_set_create();
- info.hit_set.current_info = pg_hit_set_info_t();
- info.hit_set.current_last_stamp = utime_t();
- } else {
- info.hit_set.current_last_stamp = now;
- }
+
+ info.hit_set.history.push_back(info.hit_set.current_info);
+ hit_set_create();
+ info.hit_set.current_info = pg_hit_set_info_t();
+ info.hit_set.current_last_stamp = utime_t();
// fabricate an object_info_t and SnapSet
obc->obs.oi.version = ctx->at_version;
@@ -10094,13 +10457,13 @@ void ReplicatedPG::hit_set_persist()
osd_reqid_t(),
ctx->mtime)
);
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
ctx->log.back().mod_desc.create();
} else {
ctx->log.back().mod_desc.mark_unrollbackable();
}
- hit_set_trim(repop, pool.info.hit_set_count);
+ hit_set_trim(repop, max);
info.stats.stats.add(ctx->delta_stats, string());
@@ -10113,6 +10476,9 @@ void ReplicatedPG::hit_set_trim(RepGather *repop, unsigned max)
list<pg_hit_set_info_t>::iterator p = info.hit_set.history.begin();
assert(p != info.hit_set.history.end());
hobject_t oid = get_hit_set_archive_object(p->begin, p->end);
+
+ assert(!is_degraded_object(oid));
+
dout(20) << __func__ << " removing " << oid << dendl;
++repop->ctx->at_version.version;
repop->ctx->log.push_back(
@@ -10123,7 +10489,7 @@ void ReplicatedPG::hit_set_trim(RepGather *repop, unsigned max)
0,
osd_reqid_t(),
repop->ctx->mtime));
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
if (repop->ctx->log.back().mod_desc.rmobject(
repop->ctx->at_version.version)) {
repop->ctx->op_t->stash(oid, repop->ctx->at_version.version);
@@ -10134,10 +10500,15 @@ void ReplicatedPG::hit_set_trim(RepGather *repop, unsigned max)
repop->ctx->op_t->remove(oid);
repop->ctx->log.back().mod_desc.mark_unrollbackable();
}
+ if (agent_state)
+ agent_state->remove_oldest_hit_set();
info.hit_set.history.pop_front();
struct stat st;
- int r = osd->store->stat(coll, oid, &st);
+ int r = osd->store->stat(
+ coll,
+ ghobject_t(oid, ghobject_t::NO_GEN, pg_whoami.shard),
+ &st);
assert(r == 0);
--repop->ctx->delta_stats.num_objects;
repop->ctx->delta_stats.num_bytes -= st.st_size;
@@ -10145,6 +10516,504 @@ void ReplicatedPG::hit_set_trim(RepGather *repop, unsigned max)
}
+// =======================================
+// cache agent
+
+void ReplicatedPG::agent_setup()
+{
+ assert(is_locked());
+ if (!is_primary() ||
+ pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE ||
+ pool.info.tier_of < 0 ||
+ !get_osdmap()->have_pg_pool(pool.info.tier_of)) {
+ agent_clear();
+ return;
+ }
+ if (!agent_state) {
+ agent_state.reset(new TierAgentState);
+
+ // choose random starting position
+ agent_state->position = hobject_t();
+ agent_state->position.pool = info.pgid.pool();
+ agent_state->position.hash = pool.info.get_random_pg_position(
+ info.pgid.pgid,
+ rand());
+
+ dout(10) << __func__ << " allocated new state, position "
+ << agent_state->position << dendl;
+ } else {
+ dout(10) << __func__ << " keeping existing state" << dendl;
+ }
+
+ agent_choose_mode();
+}
+
+void ReplicatedPG::agent_clear()
+{
+ agent_stop();
+ agent_state.reset(NULL);
+}
+
+void ReplicatedPG::agent_work(int start_max)
+{
+ lock();
+ if (!agent_state) {
+ dout(10) << __func__ << " no agent state, stopping" << dendl;
+ unlock();
+ return;
+ }
+
+ assert(!deleting);
+
+ if (agent_state->is_idle()) {
+ dout(10) << __func__ << " idle, stopping" << dendl;
+ unlock();
+ return;
+ }
+
+ osd->logger->inc(l_osd_agent_wake);
+
+ dout(10) << __func__
+ << " max " << start_max
+ << ", flush " << agent_state->get_flush_mode_name()
+ << ", evict " << agent_state->get_evict_mode_name()
+ << ", pos " << agent_state->position
+ << dendl;
+ assert(is_primary());
+ assert(is_active());
+
+ agent_load_hit_sets();
+
+ const pg_pool_t *base_pool = get_osdmap()->get_pg_pool(pool.info.tier_of);
+ assert(base_pool);
+
+ int ls_min = 1;
+ int ls_max = 10; // FIXME?
+
+ // list some objects. this conveniently lists clones (oldest to
+ // newest) before heads... the same order we want to flush in.
+ //
+ // NOTE: do not flush the Sequencer. we will assume that the
+ // listing we get back is imprecise.
+ vector<hobject_t> ls;
+ hobject_t next;
+ int r = pgbackend->objects_list_partial(agent_state->position, ls_min, ls_max,
+ 0 /* no filtering by snapid */,
+ &ls, &next);
+ assert(r >= 0);
+ dout(20) << __func__ << " got " << ls.size() << " objects" << dendl;
+ int started = 0;
+ for (vector<hobject_t>::iterator p = ls.begin();
+ p != ls.end();
+ ++p) {
+ if (is_degraded_object(*p)) {
+ dout(20) << __func__ << " skip (degraded) " << *p << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+ ObjectContextRef obc = get_object_context(*p, false, NULL);
+ if (!obc) {
+ // we didn't flush; we may miss something here.
+ dout(20) << __func__ << " skip (no obc) " << *p << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+ if (!obc->obs.exists) {
+ dout(20) << __func__ << " skip (dne) " << obc->obs.oi.soid << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+ if (scrubber.write_blocked_by_scrub(obc->obs.oi.soid)) {
+ dout(20) << __func__ << " skip (scrubbing) " << obc->obs.oi << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+ if (obc->obs.oi.soid.nspace == cct->_conf->osd_hit_set_namespace) {
+ dout(20) << __func__ << " skip (hit set) " << obc->obs.oi << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+ if (obc->is_blocked()) {
+ dout(20) << __func__ << " skip (blocked) " << obc->obs.oi << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+
+ // be careful flushing omap to an EC pool.
+ if (base_pool->is_erasure() &&
+ obc->obs.oi.test_flag(object_info_t::FLAG_OMAP)) {
+ dout(20) << __func__ << " skip (omap to EC) " << obc->obs.oi << dendl;
+ osd->logger->inc(l_osd_agent_skip);
+ continue;
+ }
+
+ if (agent_state->flush_mode != TierAgentState::FLUSH_MODE_IDLE &&
+ agent_maybe_flush(obc))
+ ++started;
+ if (agent_state->evict_mode != TierAgentState::EVICT_MODE_IDLE &&
+ agent_maybe_evict(obc))
+ ++started;
+ if (started >= start_max)
+ break;
+ }
+
+ if (++agent_state->hist_age > g_conf->osd_agent_hist_halflife) {
+ dout(20) << __func__ << " resetting atime and temp histograms" << dendl;
+ agent_state->hist_age = 0;
+ agent_state->atime_hist.decay();
+ agent_state->temp_hist.decay();
+ }
+
+ if (next.is_max())
+ agent_state->position = hobject_t();
+ else
+ agent_state->position = next;
+ dout(20) << __func__ << " final position " << agent_state->position << dendl;
+ agent_choose_mode();
+ unlock();
+}
+
+void ReplicatedPG::agent_load_hit_sets()
+{
+ if (agent_state->evict_mode == TierAgentState::EVICT_MODE_IDLE) {
+ agent_state->discard_hit_sets();
+ return;
+ }
+
+ if (agent_state->hit_set_map.size() < info.hit_set.history.size()) {
+ dout(10) << __func__ << dendl;
+ for (list<pg_hit_set_info_t>::reverse_iterator p =
+ info.hit_set.history.rbegin();
+ p != info.hit_set.history.rend(); ++p) {
+ if (agent_state->hit_set_map.count(p->begin.sec()) == 0) {
+ dout(10) << __func__ << " loading " << p->begin << "-"
+ << p->end << dendl;
+ if (!pool.info.is_replicated()) {
+ // FIXME: EC not supported here yet
+ derr << __func__ << " on non-replicated pool" << dendl;
+ break;
+ }
+
+ // check if it's still in flight
+ if (hit_set_flushing.count(p->begin)) {
+ agent_state->add_hit_set(p->begin.sec(), hit_set_flushing[p->begin]);
+ } else {
+ bufferlist bl;
+ hobject_t oid = get_hit_set_archive_object(p->begin, p->end);
+ int r = osd->store->read(coll, oid, 0, 0, bl);
+ assert(r >= 0);
+ HitSetRef hs(new HitSet);
+ bufferlist::iterator pbl = bl.begin();
+ ::decode(*hs, pbl);
+ agent_state->add_hit_set(p->begin.sec(), hs);
+ }
+ }
+ }
+ }
+}
+
+struct C_AgentFlushStartStop : public Context {
+ ReplicatedPGRef pg;
+ hobject_t oid;
+ C_AgentFlushStartStop(ReplicatedPG *p, hobject_t o) : pg(p), oid(o) {
+ pg->osd->agent_start_op(oid);
+ }
+ void finish(int r) {
+ pg->osd->agent_finish_op(oid);
+ }
+};
+
+bool ReplicatedPG::agent_maybe_flush(ObjectContextRef& obc)
+{
+ if (!obc->obs.oi.is_dirty()) {
+ dout(20) << __func__ << " skip (clean) " << obc->obs.oi << dendl;
+ return false;
+ }
+
+ utime_t now = ceph_clock_now(NULL);
+ if (obc->obs.oi.mtime + utime_t(pool.info.cache_min_flush_age, 0) > now) {
+ dout(20) << __func__ << " skip (too young) " << obc->obs.oi << dendl;
+ return false;
+ }
+
+ if (osd->agent_is_active_oid(obc->obs.oi.soid)) {
+ dout(20) << __func__ << " skip (flushing) " << obc->obs.oi << dendl;
+ return false;
+ }
+
+ dout(10) << __func__ << " flushing " << obc->obs.oi << dendl;
+
+ // FIXME: flush anything dirty, regardless of what distribution of
+ // ages we expect.
+
+ vector<OSDOp> ops;
+ tid_t rep_tid = osd->get_tid();
+ osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
+ OpContext *ctx = new OpContext(OpRequestRef(), reqid, ops,
+ &obc->obs, obc->ssc, this);
+ ctx->op_t = pgbackend->get_transaction();
+ ctx->obc = obc;
+ ctx->mtime = ceph_clock_now(cct);
+ ctx->at_version = get_next_version();
+ ctx->on_finish = new C_AgentFlushStartStop(this, obc->obs.oi.soid);
+
+ start_flush(ctx, false);
+
+ osd->logger->inc(l_osd_agent_flush);
+ return true;
+}
+
+bool ReplicatedPG::agent_maybe_evict(ObjectContextRef& obc)
+{
+ const hobject_t& soid = obc->obs.oi.soid;
+ if (obc->obs.oi.is_dirty()) {
+ dout(20) << __func__ << " skip (dirty) " << obc->obs.oi << dendl;
+ return false;
+ }
+ if (!obc->obs.oi.watchers.empty()) {
+ dout(20) << __func__ << " skip (watchers) " << obc->obs.oi << dendl;
+ return false;
+ }
+
+ if (soid.snap == CEPH_NOSNAP) {
+ int result = _verify_no_head_clones(soid, obc->ssc->snapset);
+ if (result < 0) {
+ dout(20) << __func__ << " skip (clones) " << obc->obs.oi << dendl;
+ return false;
+ }
+ }
+
+ if (agent_state->evict_mode != TierAgentState::EVICT_MODE_FULL) {
+ // is this object old and/or cold enough?
+ int atime = -1, temp = 0;
+ agent_estimate_atime_temp(soid, &atime, NULL /*FIXME &temp*/);
+
+ uint64_t atime_upper = 0, atime_lower = 0;
+ if (atime < 0 && obc->obs.oi.mtime != utime_t())
+ atime = ceph_clock_now(NULL).sec() - obc->obs.oi.mtime;
+ if (atime < 0)
+ atime = pool.info.hit_set_period * pool.info.hit_set_count; // "infinite"
+ if (atime >= 0) {
+ agent_state->atime_hist.add(atime);
+ agent_state->atime_hist.get_position_micro(atime, &atime_lower,
+ &atime_upper);
+ }
+
+ unsigned temp_upper = 0, temp_lower = 0;
+ /*
+ // FIXME: bound atime based on creation time?
+ agent_state->temp_hist.add(atime);
+ agent_state->temp_hist.get_position_micro(temp, &temp_lower, &temp_upper);
+ */
+
+ dout(20) << __func__
+ << " atime " << atime
+ << " pos " << atime_lower << "-" << atime_upper
+ << ", temp " << temp
+ << " pos " << temp_lower << "-" << temp_upper
+ << ", evict_effort " << agent_state->evict_effort
+ << dendl;
+ dout(30) << "agent_state:\n";
+ Formatter *f = new_formatter("");
+ f->open_object_section("agent_state");
+ agent_state->dump(f);
+ f->close_section();
+ f->flush(*_dout);
+ delete f;
+ *_dout << dendl;
+
+ // FIXME: ignore temperature for now.
+
+ // KISS: if [lower,upper] spans our target effort, evict it.
+ if (atime_lower >= agent_state->evict_effort)
+ return false;
+ }
+
+ dout(10) << __func__ << " evicting " << obc->obs.oi << dendl;
+ RepGather *repop = simple_repop_create(obc);
+ OpContext *ctx = repop->ctx;
+ ctx->at_version = get_next_version();
+ assert(ctx->new_obs.exists);
+ int r = _delete_head(ctx, true);
+ assert(r == 0);
+ finish_ctx(ctx, pg_log_entry_t::DELETE);
+ simple_repop_submit(repop);
+ osd->logger->inc(l_osd_tier_evict);
+ osd->logger->inc(l_osd_agent_evict);
+ return true;
+}
+
+void ReplicatedPG::agent_stop()
+{
+ dout(20) << __func__ << dendl;
+ if (agent_state && !agent_state->is_idle()) {
+ agent_state->evict_mode = TierAgentState::EVICT_MODE_IDLE;
+ agent_state->flush_mode = TierAgentState::FLUSH_MODE_IDLE;
+ osd->agent_disable_pg(this, agent_state->evict_effort);
+ }
+}
+
+void ReplicatedPG::agent_choose_mode()
+{
+ uint64_t divisor = pool.info.get_pg_num_divisor(info.pgid.pgid);
+
+ // adjust (effective) user objects down based on the (max) number
+ // of HitSet objects, which should not count toward our total since
+ // they cannot be flushed.
+ uint64_t num_user_objects = info.stats.stats.sum.num_objects;
+ if (num_user_objects > pool.info.hit_set_count)
+ num_user_objects -= pool.info.hit_set_count;
+ else
+ num_user_objects = 0;
+
+ // get dirty, full ratios
+ uint64_t dirty_micro = 0;
+ uint64_t full_micro = 0;
+ if (pool.info.target_max_bytes && info.stats.stats.sum.num_objects) {
+ uint64_t avg_size = info.stats.stats.sum.num_bytes /
+ info.stats.stats.sum.num_objects;
+ dirty_micro =
+ info.stats.stats.sum.num_objects_dirty * avg_size * 1000000 /
+ (pool.info.target_max_bytes / divisor);
+ full_micro =
+ info.stats.stats.sum.num_bytes * 1000000 /
+ (pool.info.target_max_bytes / divisor);
+ }
+ if (pool.info.target_max_objects) {
+ uint64_t dirty_objects_micro =
+ info.stats.stats.sum.num_objects_dirty * 1000000 /
+ (pool.info.target_max_objects / divisor);
+ if (dirty_objects_micro > dirty_micro)
+ dirty_micro = dirty_objects_micro;
+ uint64_t full_objects_micro =
+ num_user_objects * 1000000 / (pool.info.target_max_objects / divisor);
+ if (full_objects_micro > full_micro)
+ full_micro = full_objects_micro;
+ }
+ dout(20) << __func__ << " dirty " << ((float)dirty_micro / 1000000.0)
+ << " full " << ((float)full_micro / 1000000.0)
+ << dendl;
+
+ // flush mode
+ TierAgentState::flush_mode_t flush_mode = TierAgentState::FLUSH_MODE_IDLE;
+ uint64_t flush_target = pool.info.cache_target_dirty_ratio_micro;
+ uint64_t flush_slop = (float)flush_target * g_conf->osd_agent_slop;
+ if (agent_state->flush_mode == TierAgentState::FLUSH_MODE_IDLE)
+ flush_target += flush_slop;
+ else
+ flush_target -= MIN(flush_target, flush_slop);
+ if (dirty_micro > flush_target)
+ flush_mode = TierAgentState::FLUSH_MODE_ACTIVE;
+
+ // evict mode
+ TierAgentState::evict_mode_t evict_mode = TierAgentState::EVICT_MODE_IDLE;
+ unsigned evict_effort = 0;
+ uint64_t evict_target = pool.info.cache_target_full_ratio_micro;
+ uint64_t evict_slop = (float)evict_target * g_conf->osd_agent_slop;
+ if (agent_state->evict_mode == TierAgentState::EVICT_MODE_IDLE)
+ evict_target += evict_slop;
+ else
+ evict_target -= MIN(evict_target, evict_slop);
+
+ if (full_micro > 1000000) {
+ // evict anything clean
+ evict_mode = TierAgentState::EVICT_MODE_FULL;
+ evict_effort = 1000000;
+ } else if (full_micro > evict_target) {
+ // set effort in [0..1] range based on where we are between
+ evict_mode = TierAgentState::EVICT_MODE_SOME;
+ uint64_t over = full_micro - evict_target;
+ uint64_t span = 1000000 - evict_target;
+ evict_effort = MAX(over * 1000000 / span,
+ (unsigned)(1000000.0 * g_conf->osd_agent_min_evict_effort));
+
+ // quantize effort to avoid too much reordering in the agent_queue.
+ uint64_t inc = g_conf->osd_agent_quantize_effort * 1000000;
+ assert(inc > 0);
+ uint64_t was = evict_effort;
+ evict_effort -= evict_effort % inc;
+ if (evict_effort < inc)
+ evict_effort = inc;
+ assert(evict_effort >= inc && evict_effort <= 1000000);
+ dout(30) << __func__ << " evict_effort " << was << " quantized by " << inc << " to " << evict_effort << dendl;
+ }
+
+ bool old_idle = agent_state->is_idle();
+ if (flush_mode != agent_state->flush_mode) {
+ dout(5) << __func__ << " flush_mode "
+ << TierAgentState::get_flush_mode_name(agent_state->flush_mode)
+ << " -> "
+ << TierAgentState::get_flush_mode_name(flush_mode)
+ << dendl;
+ agent_state->flush_mode = flush_mode;
+ }
+ if (evict_mode != agent_state->evict_mode) {
+ dout(5) << __func__ << " evict_mode "
+ << TierAgentState::get_evict_mode_name(agent_state->evict_mode)
+ << " -> "
+ << TierAgentState::get_evict_mode_name(evict_mode)
+ << dendl;
+ if (agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
+ requeue_ops(waiting_for_cache_not_full);
+ }
+ agent_state->evict_mode = evict_mode;
+ }
+ uint64_t old_effort = agent_state->evict_effort;
+ if (evict_effort != agent_state->evict_effort) {
+ dout(5) << __func__ << " evict_effort "
+ << ((float)agent_state->evict_effort / 1000000.0)
+ << " -> "
+ << ((float)evict_effort / 1000000.0)
+ << dendl;
+ agent_state->evict_effort = evict_effort;
+ }
+
+ // NOTE: we are using evict_effort as a proxy for *all* agent effort
+ // (including flush). This is probably fine (they should be
+ // correlated) but it is not precisely correct.
+ if (agent_state->is_idle()) {
+ if (!old_idle) {
+ osd->agent_disable_pg(this, old_effort);
+ }
+ } else {
+ if (old_idle) {
+ osd->agent_enable_pg(this, agent_state->evict_effort);
+ } else if (old_effort != agent_state->evict_effort) {
+ osd->agent_adjust_pg(this, old_effort, agent_state->evict_effort);
+ }
+ }
+}
+
+void ReplicatedPG::agent_estimate_atime_temp(const hobject_t& oid,
+ int *atime, int *temp)
+{
+ assert(hit_set);
+ *atime = -1;
+ if (temp)
+ *temp = 0;
+ if (hit_set->contains(oid)) {
+ *atime = 0;
+ if (temp)
+ ++(*temp);
+ else
+ return;
+ }
+ time_t now = ceph_clock_now(NULL).sec();
+ for (map<time_t,HitSetRef>::iterator p = agent_state->hit_set_map.begin();
+ p != agent_state->hit_set_map.end();
+ ++p) {
+ if (p->second->contains(oid)) {
+ if (*atime < 0)
+ *atime = now - p->first;
+ if (temp)
+ ++(*temp);
+ else
+ return;
+ }
+ }
+}
+
+
// ==========================================================================================
// SCRUB
@@ -10231,16 +11100,19 @@ void ReplicatedPG::_scrub(ScrubMap& scrubmap)
bv.push_back(p->second.attrs[OI_ATTR]);
object_info_t oi(bv);
- if (oi.size != p->second.size) {
+ if (pgbackend->be_get_ondisk_size(oi.size) != p->second.size) {
osd->clog.error() << mode << " " << info.pgid << " " << soid
<< " on disk size (" << p->second.size
- << ") does not match object info size (" << oi.size << ")";
+ << ") does not match object info size ("
+ << oi.size << ") ajusted for ondisk to ("
+ << pgbackend->be_get_ondisk_size(oi.size)
+ << ")";
++scrubber.shallow_errors;
}
dout(20) << mode << " " << soid << " " << oi << dendl;
- stat.num_bytes += p->second.size;
+ stat.num_bytes += oi.size;
if (oi.is_dirty())
++stat.num_objects_dirty;
@@ -10277,7 +11149,7 @@ void ReplicatedPG::_scrub(ScrubMap& scrubmap)
assert(soid.snap == *curclone);
}
- assert(p->second.size == snapset.clone_size[*curclone]);
+ assert(oi.size == snapset.clone_size[*curclone]);
// verify overlap?
// ...
@@ -10527,6 +11399,24 @@ boost::statechart::result ReplicatedPG::WaitingOnReplicas::react(const SnapTrim&
return transit< NotTrimming >();
}
+void ReplicatedPG::replace_cached_attrs(
+ OpContext *ctx,
+ ObjectContextRef obc,
+ const map<string, bufferlist> &new_attrs)
+{
+ ctx->pending_attrs[obc].clear();
+ for (map<string, bufferlist>::iterator i = obc->attr_cache.begin();
+ i != obc->attr_cache.end();
+ ++i) {
+ ctx->pending_attrs[obc][i->first] = boost::optional<bufferlist>();
+ }
+ for (map<string, bufferlist>::const_iterator i = new_attrs.begin();
+ i != new_attrs.end();
+ ++i) {
+ ctx->pending_attrs[obc][i->first] = i->second;
+ }
+}
+
void ReplicatedPG::setattr_maybe_cache(
ObjectContextRef obc,
OpContext *op,
@@ -10534,7 +11424,7 @@ void ReplicatedPG::setattr_maybe_cache(
const string &key,
bufferlist &val)
{
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
op->pending_attrs[obc][key] = val;
}
t->setattr(obc->obs.oi.soid, key, val);
@@ -10546,7 +11436,7 @@ void ReplicatedPG::rmattr_maybe_cache(
PGBackend::PGTransaction *t,
const string &key)
{
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
op->pending_attrs[obc][key] = boost::optional<bufferlist>();
}
t->rmattr(obc->obs.oi.soid, key);
@@ -10557,7 +11447,7 @@ int ReplicatedPG::getattr_maybe_cache(
const string &key,
bufferlist *val)
{
- if (pool.info.ec_pool()) {
+ if (pool.info.require_rollback()) {
map<string, bufferlist>::iterator i = obc->attr_cache.find(key);
if (i != obc->attr_cache.end()) {
if (val)
@@ -10572,14 +11462,27 @@ int ReplicatedPG::getattr_maybe_cache(
int ReplicatedPG::getattrs_maybe_cache(
ObjectContextRef obc,
- map<string, bufferlist> *out)
+ map<string, bufferlist> *out,
+ bool user_only)
{
- if (pool.info.ec_pool()) {
+ int r = 0;
+ if (pool.info.require_rollback()) {
if (out)
*out = obc->attr_cache;
- return 0;
+ } else {
+ r = pgbackend->objects_get_attrs(obc->obs.oi.soid, out);
+ }
+ if (out && user_only) {
+ map<string, bufferlist> tmp;
+ for (map<string, bufferlist>::iterator i = out->begin();
+ i != out->end();
+ ++i) {
+ if (i->first.size() > 1 && i->first[0] == '_')
+ tmp[i->first.substr(1, i->first.size())].claim(i->second);
+ }
+ tmp.swap(*out);
}
- return pgbackend->objects_get_attrs(obc->obs.oi.soid, out);
+ return r;
}
void intrusive_ptr_add_ref(ReplicatedPG *pg) { pg->get("intptr"); }
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 208ee40..4ddd8d0 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -28,6 +28,7 @@
#include "PG.h"
#include "Watch.h"
#include "OpRequest.h"
+#include "TierAgentState.h"
#include "messages/MOSDOp.h"
#include "messages/MOSDOpReply.h"
@@ -37,6 +38,7 @@
#include "PGBackend.h"
#include "ReplicatedBackend.h"
+#include "ECBackend.h"
class MOSDSubOpReply;
@@ -120,6 +122,7 @@ public:
snapid_t snap_seq; ///< src's snap_seq (if head)
librados::snap_set_t snapset; ///< src snapset (if head)
bool mirror_snapset;
+ map<string, bufferlist> attrs; ///< src user attrs
CopyResults() : object_size(0), started_temp_obj(false),
user_version(0), should_requeue(false),
mirror_snapset(false) {}
@@ -224,17 +227,17 @@ public:
ObjectStore::Transaction *t
);
void on_peer_recover(
- int peer,
+ pg_shard_t peer,
const hobject_t &oid,
const ObjectRecoveryInfo &recovery_info,
const object_stat_sum_t &stat
);
void begin_peer_recover(
- int peer,
+ pg_shard_t peer,
const hobject_t oid);
void on_global_recover(
const hobject_t &oid);
- void failed_push(int from, const hobject_t &soid);
+ void failed_push(pg_shard_t from, const hobject_t &soid);
void cancel_pull(const hobject_t &soid);
template <typename T>
@@ -287,27 +290,34 @@ public:
tls.push_back(t);
osd->store->queue_transaction(osr.get(), t, 0, 0, 0, op);
}
- epoch_t get_epoch() {
+ epoch_t get_epoch() const {
return get_osdmap()->get_epoch();
}
- const vector<int> &get_actingbackfill() {
+ const set<pg_shard_t> &get_actingbackfill_shards() const {
return actingbackfill;
}
+ const set<pg_shard_t> &get_acting_shards() const {
+ return actingset;
+ }
+ const set<pg_shard_t> &get_backfill_shards() const {
+ return backfill_targets;
+ }
+
std::string gen_dbg_prefix() const { return gen_prefix(); }
- const map<hobject_t, set<int> > &get_missing_loc() {
- return missing_loc;
+ const map<hobject_t, set<pg_shard_t> > &get_missing_loc_shards() const {
+ return missing_loc.get_missing_locs();
}
- const map<int, pg_missing_t> &get_peer_missing() {
+ const map<pg_shard_t, pg_missing_t> &get_shard_missing() const {
return peer_missing;
}
- const map<int, pg_info_t> &get_peer_info() {
+ const map<pg_shard_t, pg_info_t> &get_shard_info() const {
return peer_info;
}
- const pg_missing_t &get_local_missing() {
+ const pg_missing_t &get_local_missing() const {
return pg_log.get_missing();
}
- const PGLog &get_log() {
+ const PGLog &get_log() const {
return pg_log;
}
bool pgb_is_primary() const {
@@ -321,7 +331,7 @@ public:
}
ObjectContextRef get_obc(
const hobject_t &hoid,
- map<string, bufferptr> &attrs) {
+ map<string, bufferlist> &attrs) {
return get_object_context(hoid, true, &attrs);
}
void log_operation(
@@ -336,8 +346,10 @@ public:
const eversion_t &applied_version);
bool should_send_op(
- int peer,
+ pg_shard_t peer,
const hobject_t &hoid) {
+ if (peer == get_primary())
+ return true;
assert(peer_info.count(peer));
bool should_send = hoid.pool != (int64_t)info.pgid.pool() ||
hoid <= MAX(last_backfill_started, peer_info[peer].last_backfill);
@@ -347,7 +359,7 @@ public:
}
void update_peer_last_complete_ondisk(
- int fromosd,
+ pg_shard_t fromosd,
eversion_t lcod) {
peer_last_complete_ondisk[fromosd] = lcod;
}
@@ -362,13 +374,43 @@ public:
info.stats = stat;
}
+ void schedule_work(
+ GenContext<ThreadPool::TPHandle&> *c);
+
+ pg_shard_t whoami_shard() const {
+ return pg_whoami;
+ }
+ spg_t primary_spg_t() const {
+ return spg_t(info.pgid.pgid, primary.shard);
+ }
+ pg_shard_t primary_shard() const {
+ return primary;
+ }
+
+ void send_message_osd_cluster(
+ int peer, Message *m, epoch_t from_epoch);
+ void send_message_osd_cluster(
+ Message *m, Connection *con);
+ void send_message_osd_cluster(
+ Message *m, const ConnectionRef& con);
+ ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch);
+ entity_name_t get_cluster_msgr_name() {
+ return osd->get_cluster_msgr_name();
+ }
+
+ PerfCounters *get_logger();
+
+ tid_t get_tid() { return osd->get_tid(); }
+
+ LogClientTemp clog_error() { return osd->clog.error(); }
+
/*
* Capture all object state associated with an in-progress read or write.
*/
struct OpContext {
OpRequestRef op;
osd_reqid_t reqid;
- vector<OSDOp> ops;
+ vector<OSDOp> &ops;
const ObjectState *obs; // Old objectstate
const SnapSet *snapset; // Old snapset
@@ -436,14 +478,18 @@ public:
pending_attrs.begin();
i != pending_attrs.end();
++i) {
- for (map<string, boost::optional<bufferlist> >::iterator j =
- i->second.begin();
- j != i->second.end();
- ++j) {
- if (j->second)
- i->first->attr_cache[j->first] = j->second.get();
- else
- i->first->attr_cache.erase(j->first);
+ if (i->first->obs.exists) {
+ for (map<string, boost::optional<bufferlist> >::iterator j =
+ i->second.begin();
+ j != i->second.end();
+ ++j) {
+ if (j->second)
+ i->first->attr_cache[j->first] = j->second.get();
+ else
+ i->first->attr_cache.erase(j->first);
+ }
+ } else {
+ i->first->attr_cache.clear();
}
}
pending_attrs.clear();
@@ -465,9 +511,13 @@ public:
enum { W_LOCK, R_LOCK, NONE } lock_to_release;
+ Context *on_finish;
+
OpContext(const OpContext& other);
const OpContext& operator=(const OpContext& other);
+ bool release_snapset_obc;
+
OpContext(OpRequestRef _op, osd_reqid_t _reqid, vector<OSDOp>& _ops,
ObjectState *_obs, SnapSetContext *_ssc,
ReplicatedPG *_pg) :
@@ -483,7 +533,9 @@ public:
copy_cb(NULL),
async_read_result(0),
inflightreads(0),
- lock_to_release(NONE) {
+ lock_to_release(NONE),
+ on_finish(NULL),
+ release_snapset_obc(false) {
if (_ssc) {
new_snapset = _ssc->snapset;
snapset = &_ssc->snapset;
@@ -508,9 +560,16 @@ public:
pending_async_reads.erase(i++)) {
delete i->second.second;
}
+ assert(on_finish == NULL);
+ }
+ void finish(int r) {
+ if (on_finish) {
+ on_finish->complete(r);
+ on_finish = NULL;
+ }
}
};
- friend class OpContext;
+ friend struct OpContext;
/*
* State on the PG primary associated with the replicated mutation
@@ -541,6 +600,8 @@ public:
eversion_t pg_local_last_complete;
bool queue_snap_trimmer;
+
+ Context *on_applied;
RepGather(OpContext *c, ObjectContextRef pi, tid_t rt,
eversion_t lc) :
@@ -553,7 +614,8 @@ public:
//sent_nvram(false),
sent_disk(false),
pg_local_last_complete(lc),
- queue_snap_trimmer(false) { }
+ queue_snap_trimmer(false),
+ on_applied(NULL) { }
RepGather *get() {
nref++;
@@ -563,6 +625,7 @@ public:
assert(nref > 0);
if (--nref == 0) {
delete ctx; // must already be unlocked
+ assert(on_applied == NULL);
delete this;
//generic_dout(0) << "deleting " << this << dendl;
}
@@ -580,11 +643,30 @@ protected:
*/
bool get_rw_locks(OpContext *ctx) {
if (ctx->op->may_write() || ctx->op->may_cache()) {
- if (ctx->obc->get_write(ctx->op)) {
- ctx->lock_to_release = OpContext::W_LOCK;
+ /* If snapset_obc, !obc->obs->exists and we need to
+ * get a write lock on the snapdir as well as the
+ * head. Fortunately, we are guarranteed to get a
+ * write lock on the head if !obc->obs->exists
+ */
+ if (ctx->snapset_obc) {
+ assert(!ctx->obc->obs.exists);
+ if (ctx->snapset_obc->get_write(ctx->op)) {
+ ctx->release_snapset_obc = true;
+ ctx->lock_to_release = OpContext::W_LOCK;
+ } else {
+ return false;
+ }
+ // we are creating it and have the only ref
+ bool got = ctx->obc->get_write(ctx->op);
+ assert(got);
return true;
} else {
- return false;
+ if (ctx->obc->get_write(ctx->op)) {
+ ctx->lock_to_release = OpContext::W_LOCK;
+ return true;
+ } else {
+ return false;
+ }
}
} else {
assert(ctx->op->may_read());
@@ -602,10 +684,11 @@ protected:
*
* @param ctx [in] ctx to clean up
*/
- void close_op_ctx(OpContext *ctx) {
+ void close_op_ctx(OpContext *ctx, int r) {
release_op_ctx_locks(ctx);
delete ctx->op_t;
ctx->op_t = NULL;
+ ctx->finish(r);
delete ctx;
}
@@ -617,11 +700,17 @@ protected:
void release_op_ctx_locks(OpContext *ctx) {
list<OpRequestRef> to_req;
bool requeue_recovery = false;
+ bool requeue_recovery_clone = false;
+ bool requeue_recovery_snapset = false;
+ if (ctx->snapset_obc && ctx->release_snapset_obc) {
+ ctx->snapset_obc->put_write(&to_req, &requeue_recovery_snapset);
+ ctx->release_snapset_obc = false;
+ }
switch (ctx->lock_to_release) {
case OpContext::W_LOCK:
ctx->obc->put_write(&to_req, &requeue_recovery);
- if (requeue_recovery)
- osd->recovery_wq.queue(this);
+ if (ctx->clone_obc)
+ ctx->clone_obc->put_write(&to_req, &requeue_recovery_clone);
break;
case OpContext::R_LOCK:
ctx->obc->put_read(&to_req);
@@ -632,6 +721,8 @@ protected:
assert(0);
};
ctx->lock_to_release = OpContext::NONE;
+ if (requeue_recovery || requeue_recovery_clone || requeue_recovery_snapset)
+ osd->recovery_wq.queue(this);
requeue_ops(to_req);
}
@@ -653,9 +744,11 @@ protected:
void simple_repop_submit(RepGather *repop);
// hot/cold tracking
- boost::scoped_ptr<HitSet> hit_set; ///< currently accumulating HitSet
+ HitSetRef hit_set; ///< currently accumulating HitSet
utime_t hit_set_start_stamp; ///< time the current HitSet started recording
+ map<time_t,HitSetRef> hit_set_flushing; ///< currently being written, not yet readable
+
void hit_set_clear(); ///< discard any HitSet state
void hit_set_setup(); ///< initialize HitSet state
void hit_set_create(); ///< create a new HitSet
@@ -666,11 +759,43 @@ protected:
hobject_t get_hit_set_current_object(utime_t stamp);
hobject_t get_hit_set_archive_object(utime_t start, utime_t end);
+ // agent
+ boost::scoped_ptr<TierAgentState> agent_state;
+
+ friend class C_AgentFlushStartStop;
+ friend class C_HitSetFlushing;
+
+ void agent_setup(); ///< initialize agent state
+ void agent_work(int max); ///< entry point to do some agent work
+ bool agent_maybe_flush(ObjectContextRef& obc); ///< maybe flush
+ bool agent_maybe_evict(ObjectContextRef& obc); ///< maybe evict
+
+ void agent_load_hit_sets(); ///< load HitSets, if needed
+
+ /// estimate object atime and temperature
+ ///
+ /// @param oid [in] object name
+ /// @param atime [out] seconds since last access (lower bound)
+ /// @param temperature [out] relative temperature (# hitset bins we appear in)
+ void agent_estimate_atime_temp(const hobject_t& oid,
+ int *atime, int *temperature);
+
+ /// stop the agent
+ void agent_stop();
+
+ /// clear agent state
+ void agent_clear();
+
+ void agent_choose_mode(); ///< choose (new) agent mode(s)
+
/// true if we can send an ondisk/commit for v
bool already_complete(eversion_t v) {
for (xlist<RepGather*>::iterator i = repop_queue.begin();
!i.end();
++i) {
+ // skip copy from temp object ops
+ if ((*i)->v == eversion_t())
+ continue;
if ((*i)->v > v)
break;
if (!(*i)->all_committed)
@@ -683,6 +808,9 @@ protected:
for (xlist<RepGather*>::iterator i = repop_queue.begin();
!i.end();
++i) {
+ // skip copy from temp object ops
+ if ((*i)->v == eversion_t())
+ continue;
if ((*i)->v > v)
break;
if (!(*i)->all_applied)
@@ -695,7 +823,8 @@ protected:
// projected object info
SharedPtrRegistry<hobject_t, ObjectContext> object_contexts;
- map<object_t, SnapSetContext*> snapset_contexts;
+ // map from oid.snapdir() to SnapSetContext *
+ map<hobject_t, SnapSetContext*> snapset_contexts;
Mutex snapset_contexts_lock;
// debug order that client ops are applied
@@ -714,7 +843,7 @@ protected:
ObjectContextRef get_object_context(
const hobject_t& soid,
bool can_create,
- map<string, bufferptr> *attrs = 0
+ map<string, bufferlist> *attrs = 0
);
void context_registry_on_change();
@@ -738,11 +867,11 @@ protected:
void get_src_oloc(const object_t& oid, const object_locator_t& oloc, object_locator_t& src_oloc);
- SnapSetContext *create_snapset_context(const object_t& oid);
+ SnapSetContext *create_snapset_context(const hobject_t& oid);
SnapSetContext *get_snapset_context(
- const object_t& oid, const string &key,
- ps_t seed, bool can_create, const string &nspace,
- map<string, bufferptr> *attrs = 0
+ const hobject_t& oid,
+ bool can_create,
+ map<string, bufferlist> *attrs = 0
);
void register_snapset_context(SnapSetContext *ssc) {
Mutex::Locker l(snapset_contexts_lock);
@@ -779,14 +908,14 @@ protected:
void dump_recovery_info(Formatter *f) const {
f->open_array_section("backfill_targets");
- for (vector<int>::const_iterator p = backfill_targets.begin();
+ for (set<pg_shard_t>::const_iterator p = backfill_targets.begin();
p != backfill_targets.end(); ++p)
- f->dump_int("osd", *p);
+ f->dump_stream("replica") << *p;
f->close_section();
f->open_array_section("waiting_on_backfill");
- for (set<int>::const_iterator p = waiting_on_backfill.begin();
+ for (set<pg_shard_t>::const_iterator p = waiting_on_backfill.begin();
p != waiting_on_backfill.end(); ++p)
- f->dump_int("osd", *p);
+ f->dump_stream("osd") << *p;
f->close_section();
f->dump_stream("last_backfill_started") << last_backfill_started;
{
@@ -796,9 +925,10 @@ protected:
}
{
f->open_array_section("peer_backfill_info");
- for (map<int, BackfillInterval>::const_iterator pbi = peer_backfill_info.begin();
+ for (map<pg_shard_t, BackfillInterval>::const_iterator pbi =
+ peer_backfill_info.begin();
pbi != peer_backfill_info.end(); ++pbi) {
- f->dump_int("osd", pbi->first);
+ f->dump_stream("osd") << pbi->first;
f->open_object_section("BackfillInterval");
pbi->second.dump(f);
f->close_section();
@@ -872,9 +1002,11 @@ protected:
* This helper function is called from do_op if the ObjectContext lookup fails.
* @returns true if the caching code is handling the Op, false otherwise.
*/
- inline bool maybe_handle_cache(OpRequestRef op, ObjectContextRef obc, int r,
+ inline bool maybe_handle_cache(OpRequestRef op,
+ bool write_ordered,
+ ObjectContextRef obc, int r,
const hobject_t& missing_oid,
- bool must_promote = false);
+ bool must_promote);
/**
* This helper function tells the client to redirect their request elsewhere.
*/
@@ -936,9 +1068,9 @@ protected:
void prep_backfill_object_push(
hobject_t oid, eversion_t v, ObjectContextRef obc,
- vector<int> peer,
+ vector<pg_shard_t> peers,
PGBackend::RecoveryHandle *h);
- void send_remove_op(const hobject_t& oid, eversion_t v, int peer);
+ void send_remove_op(const hobject_t& oid, eversion_t v, pg_shard_t peer);
struct C_OSD_OndiskWriteUnlock : public Context {
@@ -1025,6 +1157,16 @@ protected:
bool mirror_snapset);
void process_copy_chunk(hobject_t oid, tid_t tid, int r);
void _write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t);
+ uint64_t get_copy_chunk_size() const {
+ uint64_t size = cct->_conf->osd_copyfrom_max_chunk;
+ if (pool.info.requires_aligned_append()) {
+ uint64_t alignment = pool.info.required_alignment();
+ if (size % alignment) {
+ size += alignment - (size % alignment);
+ }
+ }
+ return size;
+ }
void _copy_some(ObjectContextRef obc, CopyOpRef cop);
void _build_finish_copy_transaction(CopyOpRef cop,
PGBackend::PGTransaction *t);
@@ -1065,7 +1207,7 @@ protected:
public:
ReplicatedPG(OSDService *o, OSDMapRef curmap,
- const PGPool &_pool, pg_t p, const hobject_t& oid,
+ const PGPool &_pool, spg_t p, const hobject_t& oid,
const hobject_t& ioid);
~ReplicatedPG() {}
@@ -1107,7 +1249,7 @@ public:
return pgbackend->temp_colls(out);
}
void split_colls(
- pg_t child,
+ spg_t child,
int split_bits,
int seed,
ObjectStore::Transaction *t) {
@@ -1172,6 +1314,8 @@ private:
boost::statechart::result react(const SnapTrim&);
};
+ int _verify_no_head_clones(const hobject_t& soid,
+ const SnapSet& ss);
int _delete_head(OpContext *ctx, bool no_whiteout);
int _rollback_to(OpContext *ctx, ceph_osd_op& op);
public:
@@ -1179,8 +1323,12 @@ public:
bool same_for_modify_since(epoch_t e);
bool same_for_rep_modify_since(epoch_t e);
- bool is_missing_object(const hobject_t& oid);
- void wait_for_missing_object(const hobject_t& oid, OpRequestRef op);
+ bool is_missing_object(const hobject_t& oid) const;
+ bool is_unreadable_object(const hobject_t &oid) const {
+ return is_missing_object(oid) ||
+ !missing_loc.readable_with_acting(oid, actingset);
+ }
+ void wait_for_unreadable_object(const hobject_t& oid, OpRequestRef op);
void wait_for_all_missing(OpRequestRef op);
bool is_degraded_object(const hobject_t& oid);
@@ -1205,6 +1353,10 @@ public:
void on_shutdown();
// attr cache handling
+ void replace_cached_attrs(
+ OpContext *ctx,
+ ObjectContextRef obc,
+ const map<string, bufferlist> &new_attrs);
void setattr_maybe_cache(
ObjectContextRef obc,
OpContext *op,
@@ -1222,7 +1374,8 @@ public:
bufferlist *val);
int getattrs_maybe_cache(
ObjectContextRef obc,
- map<string, bufferlist> *out);
+ map<string, bufferlist> *out,
+ bool user_only = false);
};
inline ostream& operator<<(ostream& out, ReplicatedPG::RepGather& repop)
diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index 315e2e2..870fdd0 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -80,7 +80,7 @@ string SnapMapper::get_prefix(snapid_t snap)
string SnapMapper::to_raw_key(
const pair<snapid_t, hobject_t> &in)
{
- return get_prefix(in.first) + in.second.to_str();
+ return get_prefix(in.first) + shard_prefix + in.second.to_str();
}
pair<string, bufferlist> SnapMapper::to_raw(
@@ -110,7 +110,7 @@ bool SnapMapper::is_mapping(const string &to_test)
string SnapMapper::to_object_key(const hobject_t &hoid)
{
- return OBJECT_PREFIX + hoid.to_str();
+ return OBJECT_PREFIX + shard_prefix + hoid.to_str();
}
void SnapMapper::object_snaps::encode(bufferlist &bl) const
diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h
index f0d0baa..5565e17 100644
--- a/src/osd/SnapMapper.h
+++ b/src/osd/SnapMapper.h
@@ -115,10 +115,10 @@ private:
static std::string get_prefix(snapid_t snap);
- static std::string to_raw_key(
+ std::string to_raw_key(
const std::pair<snapid_t, hobject_t> &to_map);
- static std::pair<std::string, bufferlist> to_raw(
+ std::pair<std::string, bufferlist> to_raw(
const std::pair<snapid_t, hobject_t> &to_map);
static bool is_mapping(const std::string &to_test);
@@ -150,17 +150,29 @@ private:
);
public:
+ static string make_shard_prefix(shard_id_t shard) {
+ if (shard == ghobject_t::NO_SHARD)
+ return string();
+ char buf[20];
+ int r = snprintf(buf, sizeof(buf), ".%x", (int)shard);
+ assert(r < (int)sizeof(buf));
+ return string(buf, r) + '_';
+ }
uint32_t mask_bits;
const uint32_t match;
string last_key_checked;
const int64_t pool;
+ const shard_id_t shard;
+ const string shard_prefix;
SnapMapper(
MapCacher::StoreDriver<std::string, bufferlist> *driver,
uint32_t match, ///< [in] pgid
uint32_t bits, ///< [in] current split bits
- int64_t pool ///< [in] pool
+ int64_t pool, ///< [in] pool
+ shard_id_t shard ///< [in] shard
)
- : backend(driver), mask_bits(bits), match(match), pool(pool) {
+ : backend(driver), mask_bits(bits), match(match), pool(pool),
+ shard(shard), shard_prefix(make_shard_prefix(shard)) {
update_bits(mask_bits);
}
@@ -171,10 +183,16 @@ public:
) {
assert(new_bits >= mask_bits);
mask_bits = new_bits;
- prefixes = hobject_t::get_prefixes(
+ set<string> _prefixes = hobject_t::get_prefixes(
mask_bits,
match,
pool);
+ prefixes.clear();
+ for (set<string>::iterator i = _prefixes.begin();
+ i != _prefixes.end();
+ ++i) {
+ prefixes.insert(shard_prefix + *i);
+ }
}
/// Update snaps for oid, empty new_snaps removes the mapping
diff --git a/src/osd/TierAgentState.h b/src/osd/TierAgentState.h
new file mode 100644
index 0000000..b5f7910
--- /dev/null
+++ b/src/osd/TierAgentState.h
@@ -0,0 +1,112 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Sage Weil <sage at inktank.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_OSD_TIERAGENT_H
+#define CEPH_OSD_TIERAGENT_H
+
+struct TierAgentState {
+ /// current position iterating across pool
+ hobject_t position;
+
+ /// histogram of ages we've encountered
+ pow2_hist_t atime_hist;
+ pow2_hist_t temp_hist;
+ int hist_age;
+
+ /// past HitSet(s) (not current)
+ map<time_t,HitSetRef> hit_set_map;
+
+ /// a few recent things we've seen that are clean
+ list<hobject_t> recent_clean;
+
+ enum flush_mode_t {
+ FLUSH_MODE_IDLE, // nothing to flush
+ FLUSH_MODE_ACTIVE, // flush what we can to bring down dirty count
+ } flush_mode; ///< current flush behavior
+ static const char *get_flush_mode_name(flush_mode_t m) {
+ switch (m) {
+ case FLUSH_MODE_IDLE: return "idle";
+ case FLUSH_MODE_ACTIVE: return "active";
+ default: assert(0 == "bad flush mode");
+ }
+ }
+ const char *get_flush_mode_name() const {
+ return get_flush_mode_name(flush_mode);
+ }
+
+ enum evict_mode_t {
+ EVICT_MODE_IDLE, // no need to evict anything
+ EVICT_MODE_SOME, // evict some things as we are near the target
+ EVICT_MODE_FULL, // evict anything
+ } evict_mode; ///< current evict behavior
+ static const char *get_evict_mode_name(evict_mode_t m) {
+ switch (m) {
+ case EVICT_MODE_IDLE: return "idle";
+ case EVICT_MODE_SOME: return "some";
+ case EVICT_MODE_FULL: return "full";
+ default: assert(0 == "bad evict mode");
+ }
+ }
+ const char *get_evict_mode_name() const {
+ return get_evict_mode_name(evict_mode);
+ }
+
+ /// approximate ratio of objects (assuming they are uniformly
+ /// distributed) that i should aim to evict.
+ unsigned evict_effort;
+
+ TierAgentState()
+ : hist_age(0),
+ flush_mode(FLUSH_MODE_IDLE),
+ evict_mode(EVICT_MODE_IDLE),
+ evict_effort(0)
+ {}
+
+ /// false if we have any work to do
+ bool is_idle() const {
+ return
+ flush_mode == FLUSH_MODE_IDLE &&
+ evict_mode == EVICT_MODE_IDLE;
+ }
+
+ /// add archived HitSet
+ void add_hit_set(time_t start, HitSetRef hs) {
+ hit_set_map.insert(make_pair(start, hs));
+ }
+
+ /// remove old/trimmed HitSet
+ void remove_oldest_hit_set() {
+ if (!hit_set_map.empty())
+ hit_set_map.erase(hit_set_map.begin());
+ }
+
+ /// discard all open hit sets
+ void discard_hit_sets() {
+ hit_set_map.clear();
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_string("flush_mode", get_flush_mode_name());
+ f->dump_string("evict_mode", get_evict_mode_name());
+ f->dump_unsigned("evict_effort", evict_effort);
+ f->dump_stream("position") << position;
+ f->open_object_section("atime_hist");
+ atime_hist.dump(f);
+ f->close_section();
+ f->open_object_section("temp_hist");
+ temp_hist.dump(f);
+ f->close_section();
+ }
+};
+
+#endif
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index c1ab20b..9e6a8f6 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -62,6 +62,26 @@ string ceph_osd_flag_string(unsigned flags)
return string("-");
}
+void pg_shard_t::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(osd, bl);
+ ::encode(shard, bl);
+ ENCODE_FINISH(bl);
+}
+void pg_shard_t::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(osd, bl);
+ ::decode(shard, bl);
+ DECODE_FINISH(bl);
+}
+
+ostream &operator<<(ostream &lhs, const pg_shard_t &rhs)
+{
+ return lhs << '(' << rhs.osd << ',' << (unsigned)(rhs.shard) << ')';
+}
+
// -- osd_reqid_t --
void osd_reqid_t::encode(bufferlist &bl) const
{
@@ -193,40 +213,6 @@ void request_redirect_t::generate_test_instances(list<request_redirect_t*>& o)
o.push_back(new request_redirect_t(loc));
}
-// -- pow2_hist_t --
-void pow2_hist_t::dump(Formatter *f) const
-{
- f->open_array_section("histogram");
- for (vector<int>::const_iterator p = h.begin(); p != h.end(); ++p)
- f->dump_int("count", *p);
- f->close_section();
- f->dump_int("upper_bound", upper_bound());
-}
-
-void pow2_hist_t::encode(bufferlist& bl) const
-{
- ENCODE_START(1, 1, bl);
- ::encode(h, bl);
- ENCODE_FINISH(bl);
-}
-
-void pow2_hist_t::decode(bufferlist::iterator& p)
-{
- DECODE_START(1, p);
- ::decode(h, p);
- DECODE_FINISH(p);
-}
-
-void pow2_hist_t::generate_test_instances(list<pow2_hist_t*>& ls)
-{
- ls.push_back(new pow2_hist_t);
- ls.push_back(new pow2_hist_t);
- ls.back()->h.push_back(1);
- ls.back()->h.push_back(3);
- ls.back()->h.push_back(0);
- ls.back()->h.push_back(2);
-}
-
void objectstore_perf_stat_t::dump(Formatter *f) const
{
f->dump_unsigned("commit_latency_ms", filestore_commit_latency);
@@ -356,6 +342,50 @@ bool pg_t::parse(const char *s)
return true;
}
+bool spg_t::parse(const char *s)
+{
+ pgid.set_preferred(-1);
+ shard = ghobject_t::NO_SHARD;
+ uint64_t ppool;
+ uint32_t pseed;
+ int32_t pref;
+ uint32_t pshard;
+ int r = sscanf(s, "%llu.%x", (long long unsigned *)&ppool, &pseed);
+ if (r < 2)
+ return false;
+ pgid.set_pool(ppool);
+ pgid.set_ps(pseed);
+
+ const char *p = strchr(s, 'p');
+ if (p) {
+ r = sscanf(p, "p%d", &pref);
+ if (r == 1) {
+ pgid.set_preferred(pref);
+ } else {
+ return false;
+ }
+ }
+
+ p = strchr(s, 's');
+ if (p) {
+ r = sscanf(p, "s%d", &pshard);
+ if (r == 1) {
+ shard = pshard;
+ } else {
+ return false;
+ }
+ }
+ return true;
+}
+
+ostream& operator<<(ostream& out, const spg_t &pg)
+{
+ out << pg.pgid;
+ if (!pg.is_no_shard())
+ out << "s" << (unsigned)pg.shard;
+ return out;
+}
+
bool pg_t::is_split(unsigned old_pg_num, unsigned new_pg_num, set<pg_t> *children) const
{
assert(m_seed < old_pg_num);
@@ -451,7 +481,7 @@ ostream& operator<<(ostream& out, const pg_t &pg)
const coll_t coll_t::META_COLL("meta");
-bool coll_t::is_temp(pg_t& pgid) const
+bool coll_t::is_temp(spg_t& pgid) const
{
const char *cstr(str.c_str());
if (!pgid.parse(cstr))
@@ -464,7 +494,7 @@ bool coll_t::is_temp(pg_t& pgid) const
return false;
}
-bool coll_t::is_pg(pg_t& pgid, snapid_t& snap) const
+bool coll_t::is_pg(spg_t& pgid, snapid_t& snap) const
{
const char *cstr(str.c_str());
@@ -484,7 +514,7 @@ bool coll_t::is_pg(pg_t& pgid, snapid_t& snap) const
return true;
}
-bool coll_t::is_pg_prefix(pg_t& pgid) const
+bool coll_t::is_pg_prefix(spg_t& pgid) const
{
const char *cstr(str.c_str());
@@ -496,7 +526,7 @@ bool coll_t::is_pg_prefix(pg_t& pgid) const
return true;
}
-bool coll_t::is_removal(uint64_t *seq, pg_t *pgid) const
+bool coll_t::is_removal(uint64_t *seq, spg_t *pgid) const
{
if (str.substr(0, 11) != string("FORREMOVAL_"))
return false;
@@ -528,13 +558,13 @@ void coll_t::decode(bufferlist::iterator& bl)
::decode(struct_v, bl);
switch (struct_v) {
case 1: {
- pg_t pgid;
+ spg_t pgid;
snapid_t snap;
::decode(pgid, bl);
::decode(snap, bl);
// infer the type
- if (pgid == pg_t() && snap == 0)
+ if (pgid == spg_t() && snap == 0)
str = "meta";
else
str = pg_and_snap_to_str(pgid, snap);
@@ -543,7 +573,7 @@ void coll_t::decode(bufferlist::iterator& bl)
case 2: {
__u8 type;
- pg_t pgid;
+ spg_t pgid;
snapid_t snap;
::decode(type, bl);
@@ -743,6 +773,14 @@ void pg_pool_t::dump(Formatter *f) const
f->dump_int("read_tier", read_tier);
f->dump_int("write_tier", write_tier);
f->dump_string("cache_mode", get_cache_mode_name());
+ f->dump_unsigned("target_max_bytes", target_max_bytes);
+ f->dump_unsigned("target_max_objects", target_max_objects);
+ f->dump_unsigned("cache_target_dirty_ratio_micro",
+ cache_target_dirty_ratio_micro);
+ f->dump_unsigned("cache_target_full_ratio_micro",
+ cache_target_full_ratio_micro);
+ f->dump_unsigned("cache_min_flush_age", cache_min_flush_age);
+ f->dump_unsigned("cache_min_evict_age", cache_min_evict_age);
f->open_object_section("properties");
for (map<string,string>::const_iterator i = properties.begin();
i != properties.end();
@@ -756,6 +794,7 @@ void pg_pool_t::dump(Formatter *f) const
f->close_section(); // hit_set_params
f->dump_unsigned("hit_set_period", hit_set_period);
f->dump_unsigned("hit_set_count", hit_set_count);
+ f->dump_unsigned("stripe_width", get_stripe_width());
}
@@ -775,6 +814,17 @@ void pg_pool_t::calc_pg_masks()
pgp_num_mask = (1 << calc_bits_of(pgp_num-1)) - 1;
}
+unsigned pg_pool_t::get_pg_num_divisor(pg_t pgid) const
+{
+ if (pg_num == pg_num_mask + 1)
+ return pg_num; // power-of-2 split
+ unsigned mask = pg_num_mask >> 1;
+ if ((pgid.ps() & mask) < (pg_num & mask))
+ return pg_num_mask + 1; // smaller bin size (already split)
+ else
+ return (pg_num_mask + 1) >> 1; // bigger bin (not yet split)
+}
+
/*
* we have two snap modes:
* - pool global snaps
@@ -924,6 +974,23 @@ ps_t pg_pool_t::raw_pg_to_pps(pg_t pg) const
}
}
+uint32_t pg_pool_t::get_random_pg_position(pg_t pg, uint32_t seed) const
+{
+ uint32_t r = crush_hash32_2(CRUSH_HASH_RJENKINS1, seed, 123);
+ if (pg_num == pg_num_mask + 1) {
+ r &= ~pg_num_mask;
+ } else {
+ unsigned smaller_mask = pg_num_mask >> 1;
+ if ((pg.ps() & smaller_mask) < (pg_num & smaller_mask)) {
+ r &= ~pg_num_mask;
+ } else {
+ r &= ~smaller_mask;
+ }
+ }
+ r |= pg.ps();
+ return r;
+}
+
void pg_pool_t::encode(bufferlist& bl, uint64_t features) const
{
if ((features & CEPH_FEATURE_PGPOOL3) == 0) {
@@ -979,7 +1046,7 @@ void pg_pool_t::encode(bufferlist& bl, uint64_t features) const
}
__u8 encode_compat = 5;
- ENCODE_START(11, encode_compat, bl);
+ ENCODE_START(13, encode_compat, bl);
::encode(type, bl);
::encode(size, bl);
::encode(crush_ruleset, bl);
@@ -1012,12 +1079,19 @@ void pg_pool_t::encode(bufferlist& bl, uint64_t features) const
::encode(hit_set_params, bl);
::encode(hit_set_period, bl);
::encode(hit_set_count, bl);
+ ::encode(stripe_width, bl);
+ ::encode(target_max_bytes, bl);
+ ::encode(target_max_objects, bl);
+ ::encode(cache_target_dirty_ratio_micro, bl);
+ ::encode(cache_target_full_ratio_micro, bl);
+ ::encode(cache_min_flush_age, bl);
+ ::encode(cache_min_evict_age, bl);
ENCODE_FINISH_NEW_COMPAT(bl, encode_compat);
}
void pg_pool_t::decode(bufferlist::iterator& bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(11, 5, 5, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(13, 5, 5, bl);
::decode(type, bl);
::decode(size, bl);
::decode(crush_ruleset, bl);
@@ -1091,6 +1165,27 @@ void pg_pool_t::decode(bufferlist::iterator& bl)
hit_set_period = def.hit_set_period;
hit_set_count = def.hit_set_count;
}
+ if (struct_v >= 12) {
+ ::decode(stripe_width, bl);
+ } else {
+ set_stripe_width(0);
+ }
+ if (struct_v >= 13) {
+ ::decode(target_max_bytes, bl);
+ ::decode(target_max_objects, bl);
+ ::decode(cache_target_dirty_ratio_micro, bl);
+ ::decode(cache_target_full_ratio_micro, bl);
+ ::decode(cache_min_flush_age, bl);
+ ::decode(cache_min_evict_age, bl);
+ } else {
+ target_max_bytes = 0;
+ target_max_objects = 0;
+ cache_target_dirty_ratio_micro = 0;
+ cache_target_full_ratio_micro = 0;
+ cache_min_flush_age = 0;
+ cache_min_evict_age = 0;
+ }
+
DECODE_FINISH(bl);
calc_pg_masks();
}
@@ -1137,6 +1232,13 @@ void pg_pool_t::generate_test_instances(list<pg_pool_t*>& o)
a.hit_set_params = HitSet::Params(new BloomHitSet::Params);
a.hit_set_period = 3600;
a.hit_set_count = 8;
+ a.set_stripe_width(12345);
+ a.target_max_bytes = 1238132132;
+ a.target_max_objects = 1232132;
+ a.cache_target_dirty_ratio_micro = 187232;
+ a.cache_target_full_ratio_micro = 987222;
+ a.cache_min_flush_age = 231;
+ a.cache_min_evict_age = 2321;
o.push_back(new pg_pool_t(a));
}
@@ -1169,11 +1271,16 @@ ostream& operator<<(ostream& out, const pg_pool_t& p)
out << " write_tier " << p.write_tier;
if (p.cache_mode)
out << " cache_mode " << p.get_cache_mode_name();
+ if (p.target_max_bytes)
+ out << " target_bytes " << p.target_max_bytes;
+ if (p.target_max_objects)
+ out << " target_objects " << p.target_max_objects;
if (p.hit_set_params.get_type() != HitSet::TYPE_NONE) {
out << " hit_set " << p.hit_set_params
<< " " << p.hit_set_period << "s"
<< " x" << p.hit_set_count;
}
+ out << " stripe_width " << p.get_stripe_width();
return out;
}
@@ -1434,6 +1541,8 @@ void pg_stat_t::dump(Formatter *f) const
for (vector<int>::const_iterator p = acting.begin(); p != acting.end(); ++p)
f->dump_int("osd", *p);
f->close_section();
+ f->dump_int("up_primary", up_primary);
+ f->dump_int("acting_primary", acting_primary);
}
void pg_stat_t::dump_brief(Formatter *f) const
@@ -1447,11 +1556,13 @@ void pg_stat_t::dump_brief(Formatter *f) const
for (vector<int>::const_iterator p = acting.begin(); p != acting.end(); ++p)
f->dump_int("osd", *p);
f->close_section();
+ f->dump_int("up_primary", up_primary);
+ f->dump_int("acting_primary", acting_primary);
}
void pg_stat_t::encode(bufferlist &bl) const
{
- ENCODE_START(14, 8, bl);
+ ENCODE_START(15, 8, bl);
::encode(version, bl);
::encode(reported_seq, bl);
::encode(reported_epoch, bl);
@@ -1481,12 +1592,14 @@ void pg_stat_t::encode(bufferlist &bl) const
::encode(last_clean_scrub_stamp, bl);
::encode(last_became_active, bl);
::encode(dirty_stats_invalid, bl);
+ ::encode(up_primary, bl);
+ ::encode(acting_primary, bl);
ENCODE_FINISH(bl);
}
void pg_stat_t::decode(bufferlist::iterator &bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(14, 8, 8, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(15, 8, 8, bl);
::decode(version, bl);
::decode(reported_seq, bl);
::decode(reported_epoch, bl);
@@ -1573,6 +1686,13 @@ void pg_stat_t::decode(bufferlist::iterator &bl)
// encoder may not have supported num_objects_dirty accounting.
dirty_stats_invalid = true;
}
+ if (struct_v >= 15) {
+ ::decode(up_primary, bl);
+ ::decode(acting_primary, bl);
+ } else {
+ up_primary = up.size() ? up[0] : -1;
+ acting_primary = acting.size() ? acting[0] : -1;
+ }
DECODE_FINISH(bl);
}
@@ -1608,7 +1728,15 @@ void pg_stat_t::generate_test_instances(list<pg_stat_t*>& o)
a.log_size = 99;
a.ondisk_log_size = 88;
a.up.push_back(123);
+ a.up_primary = 123;
a.acting.push_back(456);
+ a.acting_primary = 456;
+ o.push_back(new pg_stat_t(a));
+
+ a.up.push_back(124);
+ a.up_primary = 124;
+ a.acting.push_back(124);
+ a.acting_primary = 124;
o.push_back(new pg_stat_t(a));
}
@@ -1771,8 +1899,8 @@ void pg_history_t::generate_test_instances(list<pg_history_t*>& o)
void pg_info_t::encode(bufferlist &bl) const
{
- ENCODE_START(29, 26, bl);
- ::encode(pgid, bl);
+ ENCODE_START(30, 26, bl);
+ ::encode(pgid.pgid, bl);
::encode(last_update, bl);
::encode(last_complete, bl);
::encode(log_tail, bl);
@@ -1783,6 +1911,7 @@ void pg_info_t::encode(bufferlist &bl) const
::encode(last_epoch_started, bl);
::encode(last_user_version, bl);
::encode(hit_set, bl);
+ ::encode(pgid.shard, bl);
ENCODE_FINISH(bl);
}
@@ -1792,9 +1921,9 @@ void pg_info_t::decode(bufferlist::iterator &bl)
if (struct_v < 23) {
old_pg_t opgid;
::decode(opgid, bl);
- pgid = opgid;
+ pgid.pgid = opgid;
} else {
- ::decode(pgid, bl);
+ ::decode(pgid.pgid, bl);
}
::decode(last_update, bl);
::decode(last_complete, bl);
@@ -1824,6 +1953,10 @@ void pg_info_t::decode(bufferlist::iterator &bl)
last_user_version = last_update.version;
if (struct_v >= 29)
::decode(hit_set, bl);
+ if (struct_v >= 30)
+ ::decode(pgid.shard, bl);
+ else
+ pgid.shard = ghobject_t::no_shard();
DECODE_FINISH(bl);
}
@@ -1862,7 +1995,7 @@ void pg_info_t::generate_test_instances(list<pg_info_t*>& o)
list<pg_history_t*> h;
pg_history_t::generate_test_instances(h);
o.back()->history = *h.back();
- o.back()->pgid = pg_t(1, 2, -1);
+ o.back()->pgid = spg_t(pg_t(1, 2, -1), ghobject_t::no_shard());
o.back()->last_update = eversion_t(3, 4);
o.back()->last_complete = eversion_t(5, 6);
o.back()->last_user_version = 2;
@@ -1883,24 +2016,35 @@ void pg_info_t::generate_test_instances(list<pg_info_t*>& o)
// -- pg_notify_t --
void pg_notify_t::encode(bufferlist &bl) const
{
- ENCODE_START(1, 1, bl);
+ ENCODE_START(2, 1, bl);
::encode(query_epoch, bl);
::encode(epoch_sent, bl);
::encode(info, bl);
+ ::encode(to, bl);
+ ::encode(from, bl);
ENCODE_FINISH(bl);
}
void pg_notify_t::decode(bufferlist::iterator &bl)
{
- DECODE_START(1, bl);
+ DECODE_START(2, bl);
::decode(query_epoch, bl);
::decode(epoch_sent, bl);
::decode(info, bl);
+ if (struct_v >= 2) {
+ ::decode(to, bl);
+ ::decode(from, bl);
+ } else {
+ to = ghobject_t::NO_SHARD;
+ from = ghobject_t::NO_SHARD;
+ }
DECODE_FINISH(bl);
}
void pg_notify_t::dump(Formatter *f) const
{
+ f->dump_int("from", from);
+ f->dump_int("to", to);
f->dump_stream("query_epoch") << query_epoch;
f->dump_stream("epoch_sent") << epoch_sent;
{
@@ -1912,38 +2056,50 @@ void pg_notify_t::dump(Formatter *f) const
void pg_notify_t::generate_test_instances(list<pg_notify_t*>& o)
{
- o.push_back(new pg_notify_t(1,1,pg_info_t()));
- o.push_back(new pg_notify_t(3,10,pg_info_t()));
+ o.push_back(new pg_notify_t(3, ghobject_t::NO_SHARD, 1 ,1 , pg_info_t()));
+ o.push_back(new pg_notify_t(0, 0, 3, 10, pg_info_t()));
}
ostream &operator<<(ostream &lhs, const pg_notify_t ¬ify)
{
- return lhs << "(query_epoch:" << notify.query_epoch
- << ", epoch_sent:" << notify.epoch_sent
- << ", info:" << notify.info << ")";
+ lhs << "(query_epoch:" << notify.query_epoch
+ << ", epoch_sent:" << notify.epoch_sent
+ << ", info:" << notify.info;
+ if (notify.from != ghobject_t::NO_SHARD ||
+ notify.to != ghobject_t::NO_SHARD)
+ lhs << " " << (unsigned)notify.from
+ << "->" << (unsigned)notify.to;
+ return lhs << ")";
}
// -- pg_interval_t --
void pg_interval_t::encode(bufferlist& bl) const
{
- ENCODE_START(2, 2, bl);
+ ENCODE_START(3, 2, bl);
::encode(first, bl);
::encode(last, bl);
::encode(up, bl);
::encode(acting, bl);
::encode(maybe_went_rw, bl);
+ ::encode(primary, bl);
ENCODE_FINISH(bl);
}
void pg_interval_t::decode(bufferlist::iterator& bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
::decode(first, bl);
::decode(last, bl);
::decode(up, bl);
::decode(acting, bl);
::decode(maybe_went_rw, bl);
+ if (struct_v >= 3) {
+ ::decode(primary, bl);
+ } else {
+ if (acting.size())
+ primary = acting[0];
+ }
DECODE_FINISH(bl);
}
@@ -1975,6 +2131,8 @@ void pg_interval_t::generate_test_instances(list<pg_interval_t*>& o)
}
bool pg_interval_t::check_new_interval(
+ int old_primary,
+ int new_primary,
const vector<int> &old_acting,
const vector<int> &new_acting,
const vector<int> &old_up,
@@ -1989,7 +2147,8 @@ bool pg_interval_t::check_new_interval(
std::ostream *out)
{
// remember past interval
- if (new_acting != old_acting || new_up != old_up ||
+ if (old_primary != new_primary ||
+ new_acting != old_acting || new_up != old_up ||
(!(lastmap->get_pools().count(pool_id))) ||
(lastmap->get_pools().find(pool_id)->second.min_size !=
osdmap->get_pools().find(pool_id)->second.min_size) ||
@@ -2000,24 +2159,25 @@ bool pg_interval_t::check_new_interval(
i.last = osdmap->get_epoch() - 1;
i.acting = old_acting;
i.up = old_up;
+ i.primary = old_primary;
- if (!i.acting.empty() &&
+ if (!i.acting.empty() && i.primary != -1 &&
i.acting.size() >=
lastmap->get_pools().find(pool_id)->second.min_size) {
if (out)
*out << "generate_past_intervals " << i
<< ": not rw,"
- << " up_thru " << lastmap->get_up_thru(i.acting[0])
- << " up_from " << lastmap->get_up_from(i.acting[0])
+ << " up_thru " << lastmap->get_up_thru(i.primary)
+ << " up_from " << lastmap->get_up_from(i.primary)
<< " last_epoch_clean " << last_epoch_clean
<< std::endl;
- if (lastmap->get_up_thru(i.acting[0]) >= i.first &&
- lastmap->get_up_from(i.acting[0]) <= i.first) {
+ if (lastmap->get_up_thru(i.primary) >= i.first &&
+ lastmap->get_up_from(i.primary) <= i.first) {
i.maybe_went_rw = true;
if (out)
*out << "generate_past_intervals " << i
- << " : primary up " << lastmap->get_up_from(i.acting[0])
- << "-" << lastmap->get_up_thru(i.acting[0])
+ << " : primary up " << lastmap->get_up_from(i.primary)
+ << "-" << lastmap->get_up_thru(i.primary)
<< " includes interval"
<< std::endl;
} else if (last_epoch_clean >= i.first &&
@@ -2039,8 +2199,8 @@ bool pg_interval_t::check_new_interval(
i.maybe_went_rw = false;
if (out)
*out << "generate_past_intervals " << i
- << " : primary up " << lastmap->get_up_from(i.acting[0])
- << "-" << lastmap->get_up_thru(i.acting[0])
+ << " : primary up " << lastmap->get_up_from(i.primary)
+ << "-" << lastmap->get_up_thru(i.primary)
<< " does not include interval"
<< std::endl;
}
@@ -2070,11 +2230,13 @@ ostream& operator<<(ostream& out, const pg_interval_t& i)
void pg_query_t::encode(bufferlist &bl, uint64_t features) const {
if (features & CEPH_FEATURE_QUERY_T) {
- ENCODE_START(2, 2, bl);
+ ENCODE_START(3, 2, bl);
::encode(type, bl);
::encode(since, bl);
history.encode(bl);
::encode(epoch_sent, bl);
+ ::encode(to, bl);
+ ::encode(from, bl);
ENCODE_FINISH(bl);
} else {
::encode(type, bl);
@@ -2086,11 +2248,18 @@ void pg_query_t::encode(bufferlist &bl, uint64_t features) const {
void pg_query_t::decode(bufferlist::iterator &bl) {
bufferlist::iterator bl2 = bl;
try {
- DECODE_START(2, bl);
+ DECODE_START(3, bl);
::decode(type, bl);
::decode(since, bl);
history.decode(bl);
::decode(epoch_sent, bl);
+ if (struct_v >= 3) {
+ ::decode(to, bl);
+ ::decode(from, bl);
+ } else {
+ to = ghobject_t::NO_SHARD;
+ from = ghobject_t::NO_SHARD;
+ }
DECODE_FINISH(bl);
} catch (...) {
bl = bl2;
@@ -2102,6 +2271,8 @@ void pg_query_t::decode(bufferlist::iterator &bl) {
void pg_query_t::dump(Formatter *f) const
{
+ f->dump_int("from", from);
+ f->dump_int("to", to);
f->dump_string("type", get_type_name());
f->dump_stream("since") << since;
f->dump_stream("epoch_sent") << epoch_sent;
@@ -2114,10 +2285,13 @@ void pg_query_t::generate_test_instances(list<pg_query_t*>& o)
o.push_back(new pg_query_t());
list<pg_history_t*> h;
pg_history_t::generate_test_instances(h);
- o.push_back(new pg_query_t(pg_query_t::INFO, *h.back(), 4));
- o.push_back(new pg_query_t(pg_query_t::MISSING, *h.back(), 4));
- o.push_back(new pg_query_t(pg_query_t::LOG, eversion_t(4, 5), *h.back(), 4));
- o.push_back(new pg_query_t(pg_query_t::FULLLOG, *h.back(), 5));
+ o.push_back(new pg_query_t(pg_query_t::INFO, 1, 2, *h.back(), 4));
+ o.push_back(new pg_query_t(pg_query_t::MISSING, 2, 3, *h.back(), 4));
+ o.push_back(new pg_query_t(pg_query_t::LOG, 0, 0,
+ eversion_t(4, 5), *h.back(), 4));
+ o.push_back(new pg_query_t(pg_query_t::FULLLOG,
+ ghobject_t::NO_SHARD, ghobject_t::NO_SHARD,
+ *h.back(), 5));
}
// -- ObjectModDesc --
@@ -4108,6 +4282,10 @@ ostream& operator<<(ostream& out, const OSDOp& op)
case CEPH_OSD_OP_COPY_FROM:
out << " ver " << op.op.copy_from.src_version;
break;
+ case CEPH_OSD_OP_SETALLOCHINT:
+ out << " object_size " << op.op.alloc_hint.expected_object_size
+ << " write_size " << op.op.alloc_hint.expected_write_size;
+ break;
default:
out << " " << op.op.extent.offset << "~" << op.op.extent.length;
if (op.op.extent.truncate_seq)
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index e2a4822..f529c30 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -27,7 +27,7 @@
#include "include/types.h"
#include "include/utime.h"
#include "include/CompatSet.h"
-#include "include/histogram.h"
+#include "common/histogram.h"
#include "include/interval_set.h"
#include "common/Formatter.h"
#include "common/bloom_filter.hpp"
@@ -36,7 +36,7 @@
#include "HitSet.h"
#include "Watch.h"
#include "OpRequest.h"
-#include "include/hash_namespace.h"
+#include "include/cmp.h"
#define CEPH_OSD_ONDISK_MAGIC "ceph osd volume v026"
@@ -63,6 +63,26 @@ const char *ceph_osd_flag_name(unsigned flag);
/// convert CEPH_OSD_FLAG_* op flags to a string
string ceph_osd_flag_string(unsigned flags);
+struct pg_shard_t {
+ int osd;
+ shard_id_t shard;
+ pg_shard_t() : osd(-1), shard(ghobject_t::NO_SHARD) {}
+ explicit pg_shard_t(int osd) : osd(osd), shard(ghobject_t::NO_SHARD) {}
+ pg_shard_t(int osd, shard_id_t shard) : osd(osd), shard(shard) {}
+ static pg_shard_t undefined_shard() {
+ return pg_shard_t(-1, ghobject_t::NO_SHARD);
+ }
+ bool is_undefined() const {
+ return osd == -1;
+ }
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+};
+WRITE_CLASS_ENCODER(pg_shard_t)
+WRITE_EQ_OPERATORS_2(pg_shard_t, osd, shard)
+WRITE_CMP_OPERATORS_2(pg_shard_t, osd, shard)
+ostream &operator<<(ostream &lhs, const pg_shard_t &rhs);
+
inline ostream& operator<<(ostream& out, const osd_reqid_t& r) {
return out << r.name << "." << r.inc << ":" << r.tid;
}
@@ -368,6 +388,74 @@ CEPH_HASH_NAMESPACE_START
};
CEPH_HASH_NAMESPACE_END
+struct spg_t {
+ pg_t pgid;
+ shard_id_t shard;
+ spg_t() : shard(ghobject_t::NO_SHARD) {}
+ spg_t(pg_t pgid, shard_id_t shard) : pgid(pgid), shard(shard) {}
+ explicit spg_t(pg_t pgid) : pgid(pgid), shard(ghobject_t::NO_SHARD) {}
+ unsigned get_split_bits(unsigned pg_num) const {
+ return pgid.get_split_bits(pg_num);
+ }
+ spg_t get_parent() const {
+ return spg_t(pgid.get_parent(), shard);
+ }
+ ps_t ps() const {
+ return pgid.ps();
+ }
+ uint64_t pool() const {
+ return pgid.pool();
+ }
+ int32_t preferred() const {
+ return pgid.preferred();
+ }
+ bool parse(const char *s);
+ bool is_split(unsigned old_pg_num, unsigned new_pg_num,
+ set<spg_t> *pchildren) const {
+ set<pg_t> _children;
+ set<pg_t> *children = pchildren ? &_children : NULL;
+ bool is_split = pgid.is_split(old_pg_num, new_pg_num, children);
+ if (pchildren && is_split) {
+ for (set<pg_t>::iterator i = _children.begin();
+ i != _children.end();
+ ++i) {
+ pchildren->insert(spg_t(*i, shard));
+ }
+ }
+ return is_split;
+ }
+ bool is_no_shard() const {
+ return shard == ghobject_t::NO_SHARD;
+ }
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(pgid, bl);
+ ::encode(shard, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(pgid, bl);
+ ::decode(shard, bl);
+ DECODE_FINISH(bl);
+ }
+};
+WRITE_CLASS_ENCODER(spg_t)
+WRITE_EQ_OPERATORS_2(spg_t, pgid, shard)
+WRITE_CMP_OPERATORS_2(spg_t, pgid, shard)
+
+CEPH_HASH_NAMESPACE_START
+ template<> struct hash< spg_t >
+ {
+ size_t operator()( const spg_t& x ) const
+ {
+ static hash<uint32_t> H;
+ return H(hash<pg_t>()(x.pgid) ^ x.shard);
+ }
+ };
+CEPH_HASH_NAMESPACE_END
+
+ostream& operator<<(ostream& out, const spg_t &pg);
// ----------------------
@@ -383,15 +471,15 @@ public:
: str(str_)
{ }
- explicit coll_t(pg_t pgid, snapid_t snap = CEPH_NOSNAP)
+ explicit coll_t(spg_t pgid, snapid_t snap = CEPH_NOSNAP)
: str(pg_and_snap_to_str(pgid, snap))
{ }
- static coll_t make_temp_coll(pg_t pgid) {
+ static coll_t make_temp_coll(spg_t pgid) {
return coll_t(pg_to_tmp_str(pgid));
}
- static coll_t make_removal_coll(uint64_t seq, pg_t pgid) {
+ static coll_t make_removal_coll(uint64_t seq, spg_t pgid) {
return coll_t(seq_to_removal_str(seq, pgid));
}
@@ -407,10 +495,10 @@ public:
return str < rhs.str;
}
- bool is_pg_prefix(pg_t& pgid) const;
- bool is_pg(pg_t& pgid, snapid_t& snap) const;
- bool is_temp(pg_t& pgid) const;
- bool is_removal(uint64_t *seq, pg_t *pgid) const;
+ bool is_pg_prefix(spg_t& pgid) const;
+ bool is_pg(spg_t& pgid, snapid_t& snap) const;
+ bool is_temp(spg_t& pgid) const;
+ bool is_removal(uint64_t *seq, spg_t *pgid) const;
void encode(bufferlist& bl) const;
void decode(bufferlist::iterator& bl);
inline bool operator==(const coll_t& rhs) const {
@@ -424,17 +512,17 @@ public:
static void generate_test_instances(list<coll_t*>& o);
private:
- static std::string pg_and_snap_to_str(pg_t p, snapid_t s) {
+ static std::string pg_and_snap_to_str(spg_t p, snapid_t s) {
std::ostringstream oss;
oss << p << "_" << s;
return oss.str();
}
- static std::string pg_to_tmp_str(pg_t p) {
+ static std::string pg_to_tmp_str(spg_t p) {
std::ostringstream oss;
oss << p << "_TEMP";
return oss.str();
}
- static std::string seq_to_removal_str(uint64_t seq, pg_t pgid) {
+ static std::string seq_to_removal_str(uint64_t seq, spg_t pgid) {
std::ostringstream oss;
oss << "FORREMOVAL_" << seq << "_" << pgid;
return oss.str();
@@ -811,7 +899,7 @@ public:
*/
interval_set<snapid_t> removed_snaps;
- int pg_num_mask, pgp_num_mask;
+ unsigned pg_num_mask, pgp_num_mask;
set<uint64_t> tiers; ///< pools that are tiers of us
int64_t tier_of; ///< pool for which we are a tier
@@ -827,10 +915,21 @@ public:
bool has_write_tier() const { return write_tier >= 0; }
void clear_write_tier() { write_tier = -1; }
+ uint64_t target_max_bytes; ///< tiering: target max pool size
+ uint64_t target_max_objects; ///< tiering: target max pool size
+
+ uint32_t cache_target_dirty_ratio_micro; ///< cache: fraction of target to leave dirty
+ uint32_t cache_target_full_ratio_micro; ///< cache: fraction of target to fill before we evict in earnest
+
+ uint32_t cache_min_flush_age; ///< minimum age (seconds) before we can flush
+ uint32_t cache_min_evict_age; ///< minimum age (seconds) before we can evict
+
HitSet::Params hit_set_params; ///< The HitSet params to use on this pool
uint32_t hit_set_period; ///< periodicity of HitSet segments (seconds)
uint32_t hit_set_count; ///< number of periods to retain
+ uint32_t stripe_width; ///< erasure coded stripe size in bytes
+
pg_pool_t()
: flags(0), type(0), size(0), min_size(0),
crush_ruleset(0), object_hash(0),
@@ -843,9 +942,15 @@ public:
pg_num_mask(0), pgp_num_mask(0),
tier_of(-1), read_tier(-1), write_tier(-1),
cache_mode(CACHEMODE_NONE),
+ target_max_bytes(0), target_max_objects(0),
+ cache_target_dirty_ratio_micro(0),
+ cache_target_full_ratio_micro(0),
+ cache_min_flush_age(0),
+ cache_min_evict_age(0),
hit_set_params(),
hit_set_period(0),
- hit_set_count(0)
+ hit_set_count(0),
+ stripe_width(0)
{ }
void dump(Formatter *f) const;
@@ -854,7 +959,10 @@ public:
/// This method will later return true for ec pools as well
bool ec_pool() const {
- return flags & FLAG_DEBUG_FAKE_EC_POOL;
+ return type == TYPE_ERASURE;
+ }
+ bool require_rollback() const {
+ return ec_pool() || flags & FLAG_DEBUG_FAKE_EC_POOL;
}
unsigned get_type() const { return type; }
@@ -874,9 +982,15 @@ public:
void set_snap_seq(snapid_t s) { snap_seq = s; }
void set_snap_epoch(epoch_t e) { snap_epoch = e; }
+ void set_stripe_width(uint32_t s) { stripe_width = s; }
+ uint32_t get_stripe_width() const { return stripe_width; }
+
bool is_replicated() const { return get_type() == TYPE_REPLICATED; }
bool is_erasure() const { return get_type() == TYPE_ERASURE; }
+ bool requires_aligned_append() const { return is_erasure(); }
+ uint64_t required_alignment() const { return stripe_width; }
+
bool can_shift_osds() const {
switch (get_type()) {
case TYPE_REPLICATED:
@@ -894,6 +1008,11 @@ public:
unsigned get_pg_num_mask() const { return pg_num_mask; }
unsigned get_pgp_num_mask() const { return pgp_num_mask; }
+ // if pg_num is not a multiple of two, pgs are not equally sized.
+ // return, for a given pg, the fraction (denominator) of the total
+ // pool size that it represents.
+ unsigned get_pg_num_divisor(pg_t pgid) const;
+
void set_pg_num(int p) {
pg_num = p;
calc_pg_masks();
@@ -965,6 +1084,9 @@ public:
*/
ps_t raw_pg_to_pps(pg_t pg) const;
+ /// choose a random hash position within a pg
+ uint32_t get_random_pg_position(pg_t pgid, uint32_t seed) const;
+
void encode(bufferlist& bl, uint64_t features) const;
void decode(bufferlist::iterator& bl);
@@ -1165,6 +1287,10 @@ struct pg_stat_t {
/// maintained starting from pool creation)
bool dirty_stats_invalid;
+ /// up, acting primaries
+ int up_primary;
+ int acting_primary;
+
pg_stat_t()
: reported_seq(0),
reported_epoch(0),
@@ -1174,7 +1300,9 @@ struct pg_stat_t {
stats_invalid(false),
log_size(0), ondisk_log_size(0),
mapping_epoch(0),
- dirty_stats_invalid(false)
+ dirty_stats_invalid(false),
+ up_primary(-1),
+ acting_primary(-1)
{ }
epoch_t get_effective_last_epoch_clean() const {
@@ -1407,7 +1535,7 @@ inline ostream& operator<<(ostream& out, const pg_history_t& h) {
* otherwise, we have no idea what the pg is supposed to contain.
*/
struct pg_info_t {
- pg_t pgid;
+ spg_t pgid;
eversion_t last_update; // last object version applied to store.
eversion_t last_complete; // last version pg was complete through.
epoch_t last_epoch_started;// last epoch at which this pg started on this osd
@@ -1429,7 +1557,7 @@ struct pg_info_t {
: last_epoch_started(0), last_user_version(0),
last_backfill(hobject_t::get_max())
{ }
- pg_info_t(pg_t p)
+ pg_info_t(spg_t p)
: pgid(p),
last_epoch_started(0), last_user_version(0),
last_backfill(hobject_t::get_max())
@@ -1443,6 +1571,11 @@ struct pg_info_t {
void encode(bufferlist& bl) const;
void decode(bufferlist::iterator& p);
void dump(Formatter *f) const;
+ bool overlaps_with(const pg_info_t &oinfo) const {
+ return last_update > oinfo.log_tail ?
+ oinfo.last_update >= log_tail :
+ last_update >= oinfo.log_tail;
+ }
static void generate_test_instances(list<pg_info_t*>& o);
};
WRITE_CLASS_ENCODER(pg_info_t)
@@ -1459,9 +1592,9 @@ inline ostream& operator<<(ostream& out, const pg_info_t& pgi)
if (pgi.last_complete != pgi.last_update)
out << " lc " << pgi.last_complete;
out << " (" << pgi.log_tail << "," << pgi.last_update << "]";
- if (pgi.is_incomplete())
- out << " lb " << pgi.last_backfill;
}
+ if (pgi.is_incomplete())
+ out << " lb " << pgi.last_backfill;
//out << " c " << pgi.epoch_created;
out << " local-les=" << pgi.last_epoch_started;
out << " n=" << pgi.stats.stats.sum.num_objects;
@@ -1474,13 +1607,22 @@ struct pg_notify_t {
epoch_t query_epoch;
epoch_t epoch_sent;
pg_info_t info;
- pg_notify_t() : query_epoch(0), epoch_sent(0) {}
- pg_notify_t(epoch_t query_epoch,
- epoch_t epoch_sent,
- const pg_info_t &info)
+ shard_id_t to;
+ shard_id_t from;
+ pg_notify_t() :
+ query_epoch(0), epoch_sent(0), to(ghobject_t::no_shard()),
+ from(ghobject_t::no_shard()) {}
+ pg_notify_t(
+ shard_id_t to,
+ shard_id_t from,
+ epoch_t query_epoch,
+ epoch_t epoch_sent,
+ const pg_info_t &info)
: query_epoch(query_epoch),
epoch_sent(epoch_sent),
- info(info) {}
+ info(info), to(to), from(from) {
+ assert(from == info.pgid.shard);
+ }
void encode(bufferlist &bl) const;
void decode(bufferlist::iterator &p);
void dump(Formatter *f) const;
@@ -1498,8 +1640,9 @@ struct pg_interval_t {
vector<int> up, acting;
epoch_t first, last;
bool maybe_went_rw;
+ int primary;
- pg_interval_t() : first(0), last(0), maybe_went_rw(false) {}
+ pg_interval_t() : first(0), last(0), maybe_went_rw(false), primary(-1) {}
void encode(bufferlist& bl) const;
void decode(bufferlist::iterator& bl);
@@ -1511,6 +1654,8 @@ struct pg_interval_t {
* if an interval was closed out.
*/
static bool check_new_interval(
+ int old_primary, ///< [in] primary as of lastmap
+ int new_primary, ///< [in] primary as of lastmap
const vector<int> &old_acting, ///< [in] acting as of lastmap
const vector<int> &new_acting, ///< [in] acting as of osdmap
const vector<int> &old_up, ///< [in] up as of lastmap
@@ -1558,18 +1703,32 @@ struct pg_query_t {
eversion_t since;
pg_history_t history;
epoch_t epoch_sent;
-
- pg_query_t() : type(-1), epoch_sent(0) {}
- pg_query_t(int t, const pg_history_t& h,
- epoch_t epoch_sent)
- : type(t), history(h),
- epoch_sent(epoch_sent) {
+ shard_id_t to;
+ shard_id_t from;
+
+ pg_query_t() : type(-1), epoch_sent(0), to(ghobject_t::NO_SHARD),
+ from(ghobject_t::NO_SHARD) {}
+ pg_query_t(
+ int t,
+ shard_id_t to,
+ shard_id_t from,
+ const pg_history_t& h,
+ epoch_t epoch_sent)
+ : type(t),
+ history(h),
+ epoch_sent(epoch_sent),
+ to(to), from(from) {
assert(t != LOG);
}
- pg_query_t(int t, eversion_t s, const pg_history_t& h,
- epoch_t epoch_sent)
+ pg_query_t(
+ int t,
+ shard_id_t to,
+ shard_id_t from,
+ eversion_t s,
+ const pg_history_t& h,
+ epoch_t epoch_sent)
: type(t), since(s), history(h),
- epoch_sent(epoch_sent) {
+ epoch_sent(epoch_sent), to(to), from(from) {
assert(t == LOG);
}
@@ -1619,6 +1778,27 @@ public:
can_local_rollback = other.can_local_rollback;
stashed = other.stashed;
}
+ void claim_append(ObjectModDesc &other) {
+ if (!can_local_rollback || stashed)
+ return;
+ if (!other.can_local_rollback) {
+ mark_unrollbackable();
+ return;
+ }
+ bl.claim_append(other.bl);
+ stashed = other.stashed;
+ }
+ void swap(ObjectModDesc &other) {
+ bl.swap(other.bl);
+
+ bool temp = other.can_local_rollback;
+ other.can_local_rollback = can_local_rollback;
+ can_local_rollback = temp;
+
+ temp = other.stashed;
+ other.stashed = stashed;
+ stashed = temp;
+ }
void append_id(ModID id) {
uint8_t _id(id);
::encode(_id, bl);
@@ -1676,6 +1856,16 @@ public:
bool empty() const {
return can_local_rollback && (bl.length() == 0);
}
+
+ /**
+ * Create fresh copy of bl bytes to avoid keeping large buffers around
+ * in the case that bl contains ptrs which point into a much larger
+ * message buffer
+ */
+ void trim_bl() {
+ if (bl.length() > 0)
+ bl.rebuild();
+ }
void encode(bufferlist &bl) const;
void decode(bufferlist::iterator &bl);
void dump(Formatter *f) const;
@@ -2302,6 +2492,7 @@ struct object_info_t {
FLAG_LOST = 1<<0,
FLAG_WHITEOUT = 1<<1, // object logically does not exist
FLAG_DIRTY = 1<<2, // object has been modified since last flushed or undirtied
+ FLAG_OMAP = 1 << 3, // has (or may have) some/any omap data
// ...
FLAG_USES_TMAP = 1<<8, // deprecated; no longer used.
} flag_t;
@@ -2318,6 +2509,8 @@ struct object_info_t {
s += "|dirty";
if (flags & FLAG_USES_TMAP)
s += "|uses_tmap";
+ if (flags & FLAG_OMAP)
+ s += "|omap";
if (s.length())
return s.substr(1);
return s;
@@ -2394,12 +2587,12 @@ struct ObjectState {
struct SnapSetContext {
- object_t oid;
+ hobject_t oid;
int ref;
bool registered;
SnapSet snapset;
- SnapSetContext(const object_t& o) : oid(o), ref(0), registered(false) { }
+ SnapSetContext(const hobject_t& o) : oid(o), ref(0), registered(false) { }
};
@@ -2496,7 +2689,8 @@ public:
if (get_write_lock()) {
return true;
} // else
- waiters.push_back(op);
+ if (op)
+ waiters.push_back(op);
return false;
}
bool get_write_lock() {
@@ -2520,14 +2714,21 @@ public:
return false;
}
}
+ /// same as get_write_lock, but ignore starvation
+ bool take_write_lock() {
+ if (state == RWWRITE) {
+ count++;
+ return true;
+ }
+ return get_write_lock();
+ }
void dec(list<OpRequestRef> *requeue) {
assert(count > 0);
assert(requeue);
- assert(requeue->empty());
count--;
if (count == 0) {
state = RWNONE;
- requeue->swap(waiters);
+ requeue->splice(requeue->end(), waiters);
}
}
void put_read(list<OpRequestRef> *requeue) {
@@ -2764,7 +2965,7 @@ struct PushOp {
interval_set<uint64_t> data_included;
bufferlist omap_header;
map<string, bufferlist> omap_entries;
- map<string, bufferptr> attrset;
+ map<string, bufferlist> attrset;
ObjectRecoveryInfo recovery_info;
ObjectRecoveryProgress before_progress;
diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index 6160d70..29dcaaa 100644
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -946,19 +946,14 @@ void ObjectCacher::flush(loff_t amount)
}
-void ObjectCacher::trim(loff_t max_bytes, loff_t max_ob)
+void ObjectCacher::trim()
{
assert(lock.is_locked());
- if (max_bytes < 0)
- max_bytes = max_size;
- if (max_ob < 0)
- max_ob = max_objects;
-
- ldout(cct, 10) << "trim start: bytes: max " << max_bytes << " clean " << get_stat_clean()
- << ", objects: max " << max_ob << " current " << ob_lru.lru_get_size()
+ ldout(cct, 10) << "trim start: bytes: max " << max_size << " clean " << get_stat_clean()
+ << ", objects: max " << max_objects << " current " << ob_lru.lru_get_size()
<< dendl;
- while (get_stat_clean() > max_bytes) {
+ while (get_stat_clean() > 0 && (uint64_t) get_stat_clean() > max_size) {
BufferHead *bh = static_cast<BufferHead*>(bh_lru_rest.lru_expire());
if (!bh)
break;
@@ -976,7 +971,7 @@ void ObjectCacher::trim(loff_t max_bytes, loff_t max_ob)
}
}
- while (ob_lru.lru_get_size() > max_ob) {
+ while (ob_lru.lru_get_size() > max_objects) {
Object *ob = static_cast<Object*>(ob_lru.lru_expire());
if (!ob)
break;
@@ -985,8 +980,8 @@ void ObjectCacher::trim(loff_t max_bytes, loff_t max_ob)
close_object(ob);
}
- ldout(cct, 10) << "trim finish: max " << max_bytes << " clean " << get_stat_clean()
- << ", objects: max " << max_ob << " current " << ob_lru.lru_get_size()
+ ldout(cct, 10) << "trim finish: max " << max_size << " clean " << get_stat_clean()
+ << ", objects: max " << max_objects << " current " << ob_lru.lru_get_size()
<< dendl;
}
@@ -1358,7 +1353,9 @@ void ObjectCacher::maybe_wait_for_writeback(uint64_t len)
// - do not wait for bytes other waiters are waiting on. this means that
// threads do not wait for each other. this effectively allows the cache
// size to balloon proportional to the data that is in flight.
- while (get_stat_dirty() + get_stat_tx() >= max_dirty + get_stat_dirty_waiting()) {
+ while (get_stat_dirty() + get_stat_tx() > 0 &&
+ (uint64_t) (get_stat_dirty() + get_stat_tx()) >=
+ max_dirty + get_stat_dirty_waiting()) {
ldout(cct, 10) << __func__ << " waiting for dirty|tx "
<< (get_stat_dirty() + get_stat_tx()) << " >= max "
<< max_dirty << " + dirty_waiting "
@@ -1413,7 +1410,7 @@ int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, M
}
// start writeback anyway?
- if (get_stat_dirty() > target_dirty) {
+ if (get_stat_dirty() > 0 && (uint64_t) get_stat_dirty() > target_dirty) {
ldout(cct, 10) << "wait_for_write " << get_stat_dirty() << " > target "
<< target_dirty << ", nudging flusher" << dendl;
flusher_cond.Signal();
@@ -1437,7 +1434,7 @@ void ObjectCacher::flusher_entry()
<< max_dirty << " max)"
<< dendl;
loff_t actual = get_stat_dirty() + get_stat_dirty_waiting();
- if (actual > target_dirty) {
+ if (actual > 0 && (uint64_t) actual > target_dirty) {
// flush some dirty pages
ldout(cct, 10) << "flusher "
<< get_stat_dirty() << " dirty + " << get_stat_dirty_waiting()
diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index d7ba9d8..54f028f 100644
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -332,7 +332,7 @@ class ObjectCacher {
string name;
Mutex& lock;
- int64_t max_dirty, target_dirty, max_size, max_objects;
+ uint64_t max_dirty, target_dirty, max_size, max_objects;
utime_t max_dirty_age;
bool block_writes_upfront;
@@ -434,7 +434,7 @@ class ObjectCacher {
void bh_read(BufferHead *bh);
void bh_write(BufferHead *bh);
- void trim(loff_t max_bytes=-1, loff_t max_objects=-1);
+ void trim();
void flush(loff_t amount=0);
/**
@@ -615,7 +615,7 @@ public:
// cache sizes
- void set_max_dirty(int64_t v) {
+ void set_max_dirty(uint64_t v) {
max_dirty = v;
}
void set_target_dirty(int64_t v) {
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 3da9be9..a295466 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -1175,7 +1175,7 @@ void Objecter::resend_mon_ops()
}
for (map<tid_t,PoolOp*>::iterator p = pool_ops.begin(); p!=pool_ops.end(); ++p) {
- pool_op_submit(p->second);
+ _pool_op_submit(p->second);
logger->inc(l_osdc_poolop_resend);
}
@@ -1205,6 +1205,19 @@ void Objecter::resend_mon_ops()
// read | write ---------------------------
+class C_CancelOp : public Context
+{
+ Objecter::Op *op;
+ Objecter *objecter;
+public:
+ C_CancelOp(Objecter::Op *op, Objecter *objecter) : op(op),
+ objecter(objecter) {}
+ void finish(int r) {
+ // note that objecter lock == timer lock, and is already held
+ objecter->op_cancel(op->tid, -ETIMEDOUT);
+ }
+};
+
tid_t Objecter::op_submit(Op *op)
{
assert(client_lock.is_locked());
@@ -1214,6 +1227,11 @@ tid_t Objecter::op_submit(Op *op)
assert(op->ops.size() == op->out_rval.size());
assert(op->ops.size() == op->out_handler.size());
+ if (osd_timeout > 0) {
+ op->ontimeout = new C_CancelOp(op, this);
+ timer.add_event_after(osd_timeout, op->ontimeout);
+ }
+
// throttle. before we look at any state, because
// take_op_budget() may drop our lock while it blocks.
take_op_budget(op);
@@ -1330,7 +1348,7 @@ tid_t Objecter::_op_submit(Op *op)
return op->tid;
}
-int Objecter::op_cancel(tid_t tid)
+int Objecter::op_cancel(tid_t tid, int r)
{
assert(client_lock.is_locked());
assert(initialized);
@@ -1344,11 +1362,11 @@ int Objecter::op_cancel(tid_t tid)
ldout(cct, 10) << __func__ << " tid " << tid << dendl;
Op *op = p->second;
if (op->onack) {
- op->onack->complete(-ECANCELED);
+ op->onack->complete(r);
op->onack = NULL;
}
if (op->oncommit) {
- op->oncommit->complete(-ECANCELED);
+ op->oncommit->complete(r);
op->oncommit = NULL;
}
op_cancel_map_check(op);
@@ -1356,15 +1374,20 @@ int Objecter::op_cancel(tid_t tid)
return 0;
}
-bool Objecter::is_pg_changed(vector<int>& o, vector<int>& n, bool any_change)
+bool Objecter::is_pg_changed(
+ int oldprimary,
+ const vector<int>& oldacting,
+ int newprimary,
+ const vector<int>& newacting,
+ bool any_change)
{
- if (o.empty() && n.empty())
- return false; // both still empty
- if (o.empty() ^ n.empty())
- return true; // was empty, now not, or vice versa
- if (o[0] != n[0])
- return true; // primary changed
- if (any_change && o != n)
+ if (OSDMap::primary_changed(
+ oldprimary,
+ oldacting,
+ newprimary,
+ newacting))
+ return true;
+ if (any_change && oldacting != newacting)
return true;
return false; // same primary (tho replicas may have changed)
}
@@ -1436,8 +1459,9 @@ int Objecter::recalc_op_target(Op *op)
if (ret == -ENOENT)
return RECALC_OP_TARGET_POOL_DNE;
}
+ int primary;
vector<int> acting;
- osdmap->pg_to_acting_osds(pgid, acting);
+ osdmap->pg_to_acting_osds(pgid, &acting, &primary);
bool need_resend = false;
@@ -1447,15 +1471,18 @@ int Objecter::recalc_op_target(Op *op)
need_resend = true;
}
- if (op->pgid != pgid || is_pg_changed(op->acting, acting, op->used_replica)) {
+ if (op->pgid != pgid ||
+ is_pg_changed(
+ op->primary, op->acting, primary, acting, op->used_replica)) {
op->pgid = pgid;
op->acting = acting;
+ op->primary = primary;
ldout(cct, 10) << "recalc_op_target tid " << op->tid
<< " pgid " << pgid << " acting " << acting << dendl;
OSDSession *s = NULL;
op->used_replica = false;
- if (!acting.empty()) {
+ if (primary != -1) {
int osd;
bool read = is_read && !is_write;
if (read && (op->flags & CEPH_OSD_FLAG_BALANCE_READS)) {
@@ -1468,7 +1495,7 @@ int Objecter::recalc_op_target(Op *op)
acting.size() > 1) {
// look for a local replica. prefer the primary if the
// distance is the same.
- int best;
+ int best = -1;
int best_locality;
for (unsigned i = 0; i < acting.size(); ++i) {
int locality = osdmap->crush->get_common_ancestor_distance(
@@ -1489,7 +1516,7 @@ int Objecter::recalc_op_target(Op *op)
assert(best >= 0);
osd = acting[best];
} else {
- osd = acting[0];
+ osd = primary;
}
s = get_session(osd);
}
@@ -1514,21 +1541,25 @@ int Objecter::recalc_op_target(Op *op)
bool Objecter::recalc_linger_op_target(LingerOp *linger_op)
{
+ int primary;
vector<int> acting;
pg_t pgid;
int ret = osdmap->object_locator_to_pg(linger_op->oid, linger_op->oloc, pgid);
if (ret == -ENOENT) {
return RECALC_OP_TARGET_POOL_DNE;
}
- osdmap->pg_to_acting_osds(pgid, acting);
+ osdmap->pg_to_acting_osds(pgid, &acting, &primary);
- if (pgid != linger_op->pgid || is_pg_changed(linger_op->acting, acting, true)) {
+ if (pgid != linger_op->pgid ||
+ is_pg_changed(
+ linger_op->primary, linger_op->acting, primary, acting, true)) {
linger_op->pgid = pgid;
linger_op->acting = acting;
+ linger_op->primary = primary;
ldout(cct, 10) << "recalc_linger_op_target tid " << linger_op->linger_id
<< " pgid " << pgid << " acting " << acting << dendl;
- OSDSession *s = acting.size() ? get_session(acting[0]) : NULL;
+ OSDSession *s = primary != -1 ? get_session(primary) : NULL;
if (linger_op->session != s) {
linger_op->session_item.remove_myself();
linger_op->session = s;
@@ -1563,6 +1594,9 @@ void Objecter::finish_op(Op *op)
logger->set(l_osdc_op_active, ops.size());
assert(check_latest_map_ops.find(op->tid) == check_latest_map_ops.end());
+ if (op->ontimeout)
+ timer.cancel_event(op->ontimeout);
+
delete op;
}
@@ -2108,8 +2142,30 @@ int Objecter::change_pool_auid(int64_t pool, Context *onfinish, uint64_t auid)
return 0;
}
+class C_CancelPoolOp : public Context
+{
+ tid_t tid;
+ Objecter *objecter;
+public:
+ C_CancelPoolOp(tid_t tid, Objecter *objecter) : tid(tid),
+ objecter(objecter) {}
+ void finish(int r) {
+ // note that objecter lock == timer lock, and is already held
+ objecter->pool_op_cancel(tid, -ETIMEDOUT);
+ }
+};
+
void Objecter::pool_op_submit(PoolOp *op)
{
+ if (mon_timeout > 0) {
+ op->ontimeout = new C_CancelPoolOp(op->tid, this);
+ timer.add_event_after(mon_timeout, op->ontimeout);
+ }
+ _pool_op_submit(op);
+}
+
+void Objecter::_pool_op_submit(PoolOp *op)
+{
ldout(cct, 10) << "pool_op_submit " << op->tid << dendl;
MPoolOp *m = new MPoolOp(monc->get_fsid(), op->tid, op->pool,
op->name, op->pool_op,
@@ -2150,11 +2206,7 @@ void Objecter::handle_pool_op_reply(MPoolOpReply *m)
op->onfinish->complete(m->replyCode);
}
op->onfinish = NULL;
- delete op;
- pool_ops.erase(tid);
-
- logger->set(l_osdc_poolop_active, pool_ops.size());
-
+ finish_pool_op(op);
} else {
ldout(cct, 10) << "unknown request " << tid << dendl;
}
@@ -2162,9 +2214,52 @@ void Objecter::handle_pool_op_reply(MPoolOpReply *m)
m->put();
}
+int Objecter::pool_op_cancel(tid_t tid, int r)
+{
+ assert(client_lock.is_locked());
+ assert(initialized);
+
+ map<tid_t, PoolOp*>::iterator it = pool_ops.find(tid);
+ if (it == pool_ops.end()) {
+ ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+ return -ENOENT;
+ }
+
+ ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+ PoolOp *op = it->second;
+ if (op->onfinish)
+ op->onfinish->complete(r);
+ finish_pool_op(op);
+ return 0;
+}
+
+void Objecter::finish_pool_op(PoolOp *op)
+{
+ pool_ops.erase(op->tid);
+ logger->set(l_osdc_poolop_active, pool_ops.size());
+
+ if (op->ontimeout)
+ timer.cancel_event(op->ontimeout);
+
+ delete op;
+}
// pool stats
+class C_CancelPoolStatOp : public Context
+{
+ tid_t tid;
+ Objecter *objecter;
+public:
+ C_CancelPoolStatOp(tid_t tid, Objecter *objecter) : tid(tid),
+ objecter(objecter) {}
+ void finish(int r) {
+ // note that objecter lock == timer lock, and is already held
+ objecter->pool_stat_op_cancel(tid, -ETIMEDOUT);
+ }
+};
+
void Objecter::get_pool_stats(list<string>& pools, map<string,pool_stat_t> *result,
Context *onfinish)
{
@@ -2175,6 +2270,11 @@ void Objecter::get_pool_stats(list<string>& pools, map<string,pool_stat_t> *resu
op->pools = pools;
op->pool_stats = result;
op->onfinish = onfinish;
+ op->ontimeout = NULL;
+ if (mon_timeout > 0) {
+ op->ontimeout = new C_CancelPoolStatOp(op->tid, this);
+ timer.add_event_after(mon_timeout, op->ontimeout);
+ }
poolstat_ops[op->tid] = op;
logger->set(l_osdc_poolstat_active, poolstat_ops.size());
@@ -2205,11 +2305,7 @@ void Objecter::handle_get_pool_stats_reply(MGetPoolStatsReply *m)
if (m->version > last_seen_pgmap_version)
last_seen_pgmap_version = m->version;
op->onfinish->complete(0);
- poolstat_ops.erase(tid);
- delete op;
-
- logger->set(l_osdc_poolstat_active, poolstat_ops.size());
-
+ finish_pool_stat_op(op);
} else {
ldout(cct, 10) << "unknown request " << tid << dendl;
}
@@ -2217,6 +2313,49 @@ void Objecter::handle_get_pool_stats_reply(MGetPoolStatsReply *m)
m->put();
}
+int Objecter::pool_stat_op_cancel(tid_t tid, int r)
+{
+ assert(client_lock.is_locked());
+ assert(initialized);
+
+ map<tid_t, PoolStatOp*>::iterator it = poolstat_ops.find(tid);
+ if (it == poolstat_ops.end()) {
+ ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+ return -ENOENT;
+ }
+
+ ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+ PoolStatOp *op = it->second;
+ if (op->onfinish)
+ op->onfinish->complete(r);
+ finish_pool_stat_op(op);
+ return 0;
+}
+
+void Objecter::finish_pool_stat_op(PoolStatOp *op)
+{
+ poolstat_ops.erase(op->tid);
+ logger->set(l_osdc_poolstat_active, poolstat_ops.size());
+
+ if (op->ontimeout)
+ timer.cancel_event(op->ontimeout);
+
+ delete op;
+}
+
+class C_CancelStatfsOp : public Context
+{
+ tid_t tid;
+ Objecter *objecter;
+public:
+ C_CancelStatfsOp(tid_t tid, Objecter *objecter) : tid(tid),
+ objecter(objecter) {}
+ void finish(int r) {
+ // note that objecter lock == timer lock, and is already held
+ objecter->statfs_op_cancel(tid, -ETIMEDOUT);
+ }
+};
void Objecter::get_fs_stats(ceph_statfs& result, Context *onfinish)
{
@@ -2226,6 +2365,11 @@ void Objecter::get_fs_stats(ceph_statfs& result, Context *onfinish)
op->tid = ++last_tid;
op->stats = &result;
op->onfinish = onfinish;
+ op->ontimeout = NULL;
+ if (mon_timeout > 0) {
+ op->ontimeout = new C_CancelStatfsOp(op->tid, this);
+ timer.add_event_after(mon_timeout, op->ontimeout);
+ }
statfs_ops[op->tid] = op;
logger->set(l_osdc_statfs_active, statfs_ops.size());
@@ -2256,11 +2400,7 @@ void Objecter::handle_fs_stats_reply(MStatfsReply *m)
if (m->h.version > last_seen_pgmap_version)
last_seen_pgmap_version = m->h.version;
op->onfinish->complete(0);
- statfs_ops.erase(tid);
- delete op;
-
- logger->set(l_osdc_statfs_active, statfs_ops.size());
-
+ finish_statfs_op(op);
} else {
ldout(cct, 10) << "unknown request " << tid << dendl;
}
@@ -2268,6 +2408,36 @@ void Objecter::handle_fs_stats_reply(MStatfsReply *m)
m->put();
}
+int Objecter::statfs_op_cancel(tid_t tid, int r)
+{
+ assert(client_lock.is_locked());
+ assert(initialized);
+
+ map<tid_t, StatfsOp*>::iterator it = statfs_ops.find(tid);
+ if (it == statfs_ops.end()) {
+ ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+ return -ENOENT;
+ }
+
+ ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+ StatfsOp *op = it->second;
+ if (op->onfinish)
+ op->onfinish->complete(r);
+ finish_statfs_op(op);
+ return 0;
+}
+
+void Objecter::finish_statfs_op(StatfsOp *op)
+{
+ statfs_ops.erase(op->tid);
+ logger->set(l_osdc_statfs_active, statfs_ops.size());
+
+ if (op->ontimeout)
+ timer.cancel_event(op->ontimeout);
+
+ delete op;
+}
// scatter/gather
@@ -2507,6 +2677,8 @@ bool Objecter::RequestStateHook::call(std::string command, cmdmap_t& cmdmap,
std::string format, bufferlist& out)
{
Formatter *f = new_formatter(format);
+ if (!f)
+ f = new_formatter("json-pretty");
m_objecter->client_lock.Lock();
m_objecter->dump_requests(f);
m_objecter->client_lock.Unlock();
@@ -2560,11 +2732,28 @@ void Objecter::handle_command_reply(MCommandReply *m)
m->put();
}
+class C_CancelCommandOp : public Context
+{
+ tid_t tid;
+ Objecter *objecter;
+public:
+ C_CancelCommandOp(tid_t tid, Objecter *objecter) : tid(tid),
+ objecter(objecter) {}
+ void finish(int r) {
+ // note that objecter lock == timer lock, and is already held
+ objecter->command_op_cancel(tid, -ETIMEDOUT);
+ }
+};
+
int Objecter::_submit_command(CommandOp *c, tid_t *ptid)
{
tid_t tid = ++last_tid;
ldout(cct, 10) << "_submit_command " << tid << " " << c->cmd << dendl;
c->tid = tid;
+ if (osd_timeout > 0) {
+ c->ontimeout = new C_CancelCommandOp(tid, this);
+ timer.add_event_after(osd_timeout, c->ontimeout);
+ }
command_ops[tid] = c;
num_homeless_ops++;
(void)recalc_command_target(c);
@@ -2603,10 +2792,11 @@ int Objecter::recalc_command_target(CommandOp *c)
c->map_check_error_str = "pool dne";
return RECALC_OP_TARGET_POOL_DNE;
}
+ int primary;
vector<int> acting;
- osdmap->pg_to_acting_osds(c->target_pg, acting);
- if (!acting.empty())
- s = get_session(acting[0]);
+ osdmap->pg_to_acting_osds(c->target_pg, &acting, &primary);
+ if (primary != -1)
+ s = get_session(primary);
}
if (c->session != s) {
ldout(cct, 10) << "recalc_command_target " << c->tid << " now " << c->session << dendl;
@@ -2638,6 +2828,25 @@ void Objecter::_send_command(CommandOp *c)
logger->inc(l_osdc_command_send);
}
+int Objecter::command_op_cancel(tid_t tid, int r)
+{
+ assert(client_lock.is_locked());
+ assert(initialized);
+
+ map<tid_t, CommandOp*>::iterator it = command_ops.find(tid);
+ if (it == command_ops.end()) {
+ ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+ return -ENOENT;
+ }
+
+ ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+ CommandOp *op = it->second;
+ command_cancel_map_check(op);
+ _finish_command(op, -ETIMEDOUT, "");
+ return 0;
+}
+
void Objecter::_finish_command(CommandOp *c, int r, string rs)
{
ldout(cct, 10) << "_finish_command " << c->tid << " = " << r << " " << rs << dendl;
@@ -2647,6 +2856,8 @@ void Objecter::_finish_command(CommandOp *c, int r, string rs)
if (c->onfinish)
c->onfinish->complete(r);
command_ops.erase(c->tid);
+ if (c->ontimeout)
+ timer.cancel_event(c->ontimeout);
c->put();
logger->set(l_osdc_command_active, command_ops.size());
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index dd0a471..2311f55 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -7,9 +7,9 @@
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
+ * License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
- *
+ *
*/
#ifndef CEPH_OBJECTER_H
@@ -75,6 +75,37 @@ struct ObjectOperation {
ops.rbegin()->op.flags = flags;
}
+ /**
+ * This is a more limited form of C_Contexts, but that requires
+ * a ceph_context which we don't have here.
+ */
+ class C_TwoContexts : public Context {
+ Context *first;
+ Context *second;
+ public:
+ C_TwoContexts(Context *first, Context *second) : first(first),
+ second(second) {}
+ void finish(int r) {
+ first->complete(r);
+ second->complete(r);
+ }
+ };
+
+ /**
+ * Add a callback to run when this operation completes,
+ * after any other callbacks for it.
+ */
+ void add_handler(Context *extra) {
+ size_t last = out_handler.size() - 1;
+ Context *orig = out_handler[last];
+ if (orig) {
+ Context *wrapper = new C_TwoContexts(orig, extra);
+ out_handler[last] = wrapper;
+ } else {
+ out_handler[last] = extra;
+ }
+ }
+
OSDOp& add_op(int op) {
int s = ops.size();
ops.resize(s+1);
@@ -166,6 +197,13 @@ struct ObjectOperation {
::encode(cookie, osd_op.indata);
osd_op.indata.append(filter);
}
+ void add_alloc_hint(int op, uint64_t expected_object_size,
+ uint64_t expected_write_size) {
+ OSDOp& osd_op = add_op(op);
+ osd_op.op.op = op;
+ osd_op.op.alloc_hint.expected_object_size = expected_object_size;
+ osd_op.op.alloc_hint.expected_write_size = expected_write_size;
+ }
// ------
@@ -237,12 +275,14 @@ struct ObjectOperation {
}
// object data
- void read(uint64_t off, uint64_t len, bufferlist *pbl, int *prval) {
+ void read(uint64_t off, uint64_t len, bufferlist *pbl, int *prval,
+ Context* ctx) {
bufferlist bl;
add_data(CEPH_OSD_OP_READ, off, len, bl);
unsigned p = ops.size() - 1;
out_bl[p] = pbl;
out_rval[p] = prval;
+ out_handler[p] = ctx;
}
struct C_ObjectOperation_sparse_read : public Context {
@@ -939,6 +979,17 @@ struct ObjectOperation {
void cache_evict() {
add_op(CEPH_OSD_OP_CACHE_EVICT);
}
+
+ void set_alloc_hint(uint64_t expected_object_size,
+ uint64_t expected_write_size ) {
+ add_alloc_hint(CEPH_OSD_OP_SETALLOCHINT, expected_object_size,
+ expected_write_size);
+
+ // CEPH_OSD_OP_SETALLOCHINT op is advisory and therefore deemed
+ // not worth a feature bit. Set FAILOK per-op flag to make
+ // sure older osds don't trip over an unsupported opcode.
+ set_last_op_flags(CEPH_OSD_OP_FLAG_FAILOK);
+ }
};
@@ -960,8 +1011,8 @@ public:
std::multimap<string,string> crush_location;
bool initialized;
-
- private:
+
+private:
tid_t last_tid;
int client_inc;
uint64_t max_linger_id;
@@ -1025,6 +1076,7 @@ public:
pg_t pgid; ///< last pg we mapped to
vector<int> acting; ///< acting for last pg we mapped to
+ int primary; ///< primary for last pg we mapped to
bool used_replica;
ConnectionRef con; // for rx buffer only
@@ -1041,7 +1093,7 @@ public:
vector<int*> out_rval;
int flags, priority;
- Context *onack, *oncommit;
+ Context *onack, *oncommit, *ontimeout;
tid_t tid;
eversion_t replay_version; // for op replay
@@ -1066,10 +1118,12 @@ public:
session(NULL), session_item(this), incarnation(0),
base_oid(o), base_oloc(ol),
precalc_pgid(false),
+ primary(-1),
used_replica(false), con(NULL),
snapid(CEPH_NOSNAP),
outbl(NULL),
flags(f), priority(0), onack(ac), oncommit(co),
+ ontimeout(NULL),
tid(0), attempts(0),
paused(false), objver(ov), reply_epoch(NULL),
map_dne_bound(0),
@@ -1213,7 +1267,7 @@ public:
list<string> pools;
map<string,pool_stat_t> *pool_stats;
- Context *onfinish;
+ Context *onfinish, *ontimeout;
utime_t last_submit;
};
@@ -1221,7 +1275,7 @@ public:
struct StatfsOp {
tid_t tid;
struct ceph_statfs *stats;
- Context *onfinish;
+ Context *onfinish, *ontimeout;
utime_t last_submit;
};
@@ -1230,7 +1284,7 @@ public:
tid_t tid;
int64_t pool;
string name;
- Context *onfinish;
+ Context *onfinish, *ontimeout;
int pool_op;
uint64_t auid;
__u8 crush_rule;
@@ -1238,7 +1292,7 @@ public:
bufferlist *blp;
utime_t last_submit;
- PoolOp() : tid(0), pool(0), onfinish(0), pool_op(0),
+ PoolOp() : tid(0), pool(0), onfinish(NULL), ontimeout(NULL), pool_op(0),
auid(0), crush_rule(0), snapid(0), blp(NULL) {}
};
@@ -1256,7 +1310,7 @@ public:
epoch_t map_dne_bound;
int map_check_error; // error to return if map check fails
const char *map_check_error_str;
- Context *onfinish;
+ Context *onfinish, *ontimeout;
utime_t last_submit;
CommandOp()
@@ -1265,12 +1319,13 @@ public:
map_dne_bound(0),
map_check_error(0),
map_check_error_str(NULL),
- onfinish(NULL) {}
+ onfinish(NULL), ontimeout(NULL) {}
};
int _submit_command(CommandOp *c, tid_t *ptid);
int recalc_command_target(CommandOp *c);
void _send_command(CommandOp *c);
+ int command_op_cancel(tid_t tid, int r);
void _finish_command(CommandOp *c, int r, string rs);
void handle_command_reply(MCommandReply *m);
@@ -1284,6 +1339,7 @@ public:
pg_t pgid;
vector<int> acting;
+ int primary;
snapid_t snap;
SnapContext snapc;
@@ -1304,7 +1360,8 @@ public:
tid_t register_tid;
epoch_t map_dne_bound;
- LingerOp() : linger_id(0), snap(CEPH_NOSNAP), flags(0),
+ LingerOp() : linger_id(0), primary(-1),
+ snap(CEPH_NOSNAP), flags(0),
poutbl(NULL), pobjver(NULL),
registered(false),
on_reg_ack(NULL), on_reg_commit(NULL),
@@ -1388,10 +1445,17 @@ public:
map<epoch_t,list< pair<Context*, int> > > waiting_for_map;
+ double mon_timeout, osd_timeout;
+
void send_op(Op *op);
void cancel_linger_op(Op *op);
void finish_op(Op *op);
- bool is_pg_changed(vector<int>& a, vector<int>& b, bool any_change=false);
+ static bool is_pg_changed(
+ int oldprimary,
+ const vector<int>& oldacting,
+ int newprimary,
+ const vector<int>& newacting,
+ bool any_change=false);
enum recalc_op_target_result {
RECALC_OP_TARGET_NO_ACTION = 0,
RECALC_OP_TARGET_NEED_RESEND,
@@ -1456,7 +1520,8 @@ public:
public:
Objecter(CephContext *cct_, Messenger *m, MonClient *mc,
- OSDMap *om, Mutex& l, SafeTimer& t) :
+ OSDMap *om, Mutex& l, SafeTimer& t, double mon_timeout,
+ double osd_timeout) :
messenger(m), monc(mc), osdmap(om), cct(cct_),
initialized(false),
last_tid(0), client_inc(-1), max_linger_id(0),
@@ -1469,6 +1534,8 @@ public:
logger(NULL), tick_event(NULL),
m_request_state_hook(NULL),
num_homeless_ops(0),
+ mon_timeout(mon_timeout),
+ osd_timeout(osd_timeout),
op_throttle_bytes(cct, "objecter_bytes", cct->_conf->objecter_inflight_op_bytes),
op_throttle_ops(cct, "objecter_ops", cct->_conf->objecter_inflight_ops)
{ }
@@ -1550,8 +1617,8 @@ public:
/** Clear the passed flags from the global op flag set */
void clear_global_op_flag(int flags) { global_op_flags &= ~flags; }
- /// cancel an in-progress request
- int op_cancel(tid_t tid);
+ /// cancel an in-progress request with the given return code
+ int op_cancel(tid_t tid, int r);
// commands
int osd_command(int osd, vector<string>& cmd,
@@ -1589,6 +1656,7 @@ public:
o->priority = op.priority;
o->mtime = mtime;
o->snapc = snapc;
+ o->out_rval.swap(op.out_rval);
return o;
}
tid_t mutate(const object_t& oid, const object_locator_t& oloc,
@@ -1760,9 +1828,9 @@ public:
}
tid_t getxattrs(const object_t& oid, const object_locator_t& oloc, snapid_t snap,
- map<string,bufferlist>& attrset,
- int flags, Context *onfinish,
- version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+ map<string,bufferlist>& attrset,
+ int flags, Context *onfinish,
+ version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
vector<OSDOp> ops;
int i = init_ops(ops, 1, extra_ops);
ops[i].op.op = CEPH_OSD_OP_GETXATTRS;
@@ -1779,9 +1847,10 @@ public:
version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
return read(oid, oloc, 0, 0, snap, pbl, flags | global_op_flags | CEPH_OSD_FLAG_READ, onfinish, objver);
}
+
// writes
- tid_t _modify(const object_t& oid, const object_locator_t& oloc,
+ tid_t _modify(const object_t& oid, const object_locator_t& oloc,
vector<OSDOp>& ops, utime_t mtime,
const SnapContext& snapc, int flags,
Context *onack, Context *oncommit,
@@ -1985,6 +2054,7 @@ public:
// pool ops
private:
void pool_op_submit(PoolOp *op);
+ void _pool_op_submit(PoolOp *op);
public:
int create_pool_snap(int64_t pool, string& snapName, Context *onfinish);
int allocate_selfmanaged_snap(int64_t pool, snapid_t *psnapid, Context *onfinish);
@@ -1997,6 +2067,8 @@ public:
int change_pool_auid(int64_t pool, Context *onfinish, uint64_t auid);
void handle_pool_op_reply(MPoolOpReply *m);
+ int pool_op_cancel(tid_t tid, int r);
+ void finish_pool_op(PoolOp *op);
// --------------------------
// pool stats
@@ -2006,6 +2078,8 @@ public:
void handle_get_pool_stats_reply(MGetPoolStatsReply *m);
void get_pool_stats(list<string>& pools, map<string,pool_stat_t> *result,
Context *onfinish);
+ int pool_stat_op_cancel(tid_t tid, int r);
+ void finish_pool_stat_op(PoolStatOp *op);
// ---------------------------
// df stats
@@ -2014,6 +2088,8 @@ private:
public:
void handle_fs_stats_reply(MStatfsReply *m);
void get_fs_stats(struct ceph_statfs& result, Context *onfinish);
+ int statfs_op_cancel(tid_t tid, int r);
+ void finish_statfs_op(StatfsOp *op);
// ---------------------------
// some scatter/gather hackery
diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py
index 97fa9ab..bf07576 100644
--- a/src/pybind/rbd.py
+++ b/src/pybind/rbd.py
@@ -73,6 +73,9 @@ class ArgumentOutOfRange(Error):
class ConnectionShutdown(Error):
pass
+class Timeout(Error):
+ pass
+
def make_ex(ret, msg):
"""
Translate a librbd return code into an exception.
@@ -95,7 +98,8 @@ def make_ex(ret, msg):
errno.ENOTEMPTY : ImageHasSnapshots,
errno.ENOSYS : FunctionNotSupported,
errno.EDOM : ArgumentOutOfRange,
- errno.ESHUTDOWN : ConnectionShutdown
+ errno.ESHUTDOWN : ConnectionShutdown,
+ errno.ETIMEDOUT : Timeout,
}
ret = abs(ret)
if ret in errors:
diff --git a/src/rbd_fuse/rbd-fuse.c b/src/rbd_fuse/rbd-fuse.c
index 2a6a8d2..a13ca44 100644
--- a/src/rbd_fuse/rbd-fuse.c
+++ b/src/rbd_fuse/rbd-fuse.c
@@ -17,6 +17,7 @@
#include <sys/types.h>
#include <unistd.h>
#include <getopt.h>
+#include <assert.h>
#include "include/rbd/librbd.h"
@@ -75,11 +76,11 @@ void simple_err(const char *msg, int err);
void
enumerate_images(struct rbd_image **head)
{
- char *ibuf;
- size_t ibuf_len;
+ char *ibuf = NULL;
+ size_t ibuf_len = 0;
struct rbd_image *im, *next;
char *ip;
- int actual_len;
+ int ret;
if (*head != NULL) {
for (im = *head; im != NULL;) {
@@ -90,17 +91,29 @@ enumerate_images(struct rbd_image **head)
*head = NULL;
}
- ibuf_len = 1024;
- ibuf = malloc(ibuf_len);
- actual_len = rbd_list(ioctx, ibuf, &ibuf_len);
- if (actual_len < 0) {
- simple_err("rbd_list: error %d\n", actual_len);
+ ret = rbd_list(ioctx, ibuf, &ibuf_len);
+ if (ret == -ERANGE) {
+ assert(ibuf_len > 0);
+ ibuf = malloc(ibuf_len);
+ if (!ibuf) {
+ simple_err("Failed to get ibuf", -ENOMEM);
+ return;
+ }
+ } else if (ret < 0) {
+ simple_err("Failed to get ibuf_len", ret);
+ return;
+ }
+
+ ret = rbd_list(ioctx, ibuf, &ibuf_len);
+ if (ret < 0) {
+ simple_err("Failed to populate ibuf", ret);
+ free(ibuf);
return;
}
+ assert(ret == (int)ibuf_len);
fprintf(stderr, "pool %s: ", pool_name);
- for (ip = ibuf; *ip != '\0' && ip < &ibuf[actual_len];
- ip += strlen(ip) + 1) {
+ for (ip = ibuf; ip < &ibuf[ibuf_len]; ip += strlen(ip) + 1) {
fprintf(stderr, "%s, ", ip);
im = malloc(sizeof(*im));
im->image_name = ip;
@@ -108,7 +121,6 @@ enumerate_images(struct rbd_image **head)
*head = im;
}
fprintf(stderr, "\n");
- return;
}
int
diff --git a/src/rgw/logrotate.conf b/src/rgw/logrotate.conf
index 7fb3391..7c6f74e 100644
--- a/src/rgw/logrotate.conf
+++ b/src/rgw/logrotate.conf
@@ -11,13 +11,14 @@
fi
# Possibly reload twice, but depending on ceph.conf the reload above may be a no-op
if which initctl > /dev/null 2>&1 && [ -x `which initctl` ]; then
- # upstart reload isn't very helpful here:
- # https://bugs.launchpad.net/upstart/+bug/1012938
- initctl list \
- | sed -n 's/^\(radosgw\+\)[ \t]\+(\([^ \/]\+\)\/\([^ \/]\+\))[ \t]\+start\/.*$/\1 cluster=\2 id=\3/p' \
- | while read l; do
- initctl reload -- $l 2>/dev/null || :
- done
+ find -L /var/lib/ceph/radosgw/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+ | while read f; do
+ if [ -e "/var/lib/ceph/radosgw/$f/done" ]; then
+ cluster="${f%%-*}"
+ id="${f#*-}"
+ initctl reload radosgw cluster="$cluster" id="$id" 2>/dev/null || :
+ fi
+ done
fi
endscript
missingok
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 26da02e..d1b7e96 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -87,7 +87,8 @@ void _usage()
cerr << " usage trim trim usage (by user, date range)\n";
cerr << " temp remove remove temporary objects that were created up to\n";
cerr << " specified date (and optional time)\n";
- cerr << " gc list dump expired garbage collection objects\n";
+ cerr << " gc list dump expired garbage collection objects (specify\n";
+ cerr << " --include-all to list all entries, including unexpired)\n";
cerr << " gc process manually process garbage\n";
cerr << " metadata get get metadata info\n";
cerr << " metadata put put metadata info\n";
@@ -812,6 +813,7 @@ int main(int argc, char **argv)
int64_t max_size = -1;
bool have_max_objects = false;
bool have_max_size = false;
+ int include_all = false;
int sync_stats = false;
@@ -952,6 +954,8 @@ int main(int argc, char **argv)
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) {
// do nothing
+ } else if (ceph_argparse_binary_flag(args, i, &include_all, NULL, "--include-all", (char*)NULL)) {
+ // do nothing
} else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
caps = val;
} else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) {
@@ -1352,7 +1356,9 @@ int main(int argc, char **argv)
case OPT_USER_INFO:
break;
case OPT_USER_CREATE:
- user_op.set_generate_key(); // generate a new key by default
+ if (!user_op.has_existing_user()) {
+ user_op.set_generate_key(); // generate a new key by default
+ }
ret = user.add(user_op, &err_msg);
if (ret < 0) {
cerr << "could not create user: " << err_msg << std::endl;
@@ -1891,7 +1897,7 @@ next:
do {
list<cls_rgw_gc_obj_info> result;
- int ret = store->list_gc_objs(&index, marker, 1000, result, &truncated);
+ int ret = store->list_gc_objs(&index, marker, 1000, !include_all, result, &truncated);
if (ret < 0) {
cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl;
return 1;
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 57b8ceb..e413a45 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -151,6 +151,7 @@ req_state::req_state(CephContext *_cct, class RGWEnv *e) : cct(_cct), cio(NULL),
perm_mask = 0;
content_length = 0;
object = NULL;
+ bucket_exists = false;
has_bad_meta = false;
length = NULL;
copy_source = NULL;
@@ -792,7 +793,7 @@ string rgw_trim_whitespace(const string& src)
}
int end = src.size() - 1;
- if (end <= start) {
+ if (end < start) {
return string();
}
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 30f8e95..1be0587 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -688,6 +688,11 @@ struct RGWObjVersionTracker {
write_version = obj_version();
}
+ void clear() {
+ read_version = obj_version();
+ write_version = obj_version();
+ }
+
void generate_new_write_ver(CephContext *cct);
};
diff --git a/src/rgw/rgw_dencoder.cc b/src/rgw/rgw_dencoder.cc
index b5c4942..2c8c758 100644
--- a/src/rgw/rgw_dencoder.cc
+++ b/src/rgw/rgw_dencoder.cc
@@ -95,16 +95,6 @@ void ACLGranteeType::generate_test_instances(list<ACLGranteeType*>& o)
static string rgw_uri_all_users = RGW_URI_ALL_USERS;
static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
-ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
-{
- if (uri.compare(rgw_uri_all_users) == 0)
- return ACL_GROUP_ALL_USERS;
- else if (uri.compare(rgw_uri_auth_users) == 0)
- return ACL_GROUP_AUTHENTICATED_USERS;
-
- return ACL_GROUP_NONE;
-}
-
void ACLGrant::generate_test_instances(list<ACLGrant*>& o)
{
string id, name, email;
diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 1b2b8ca..dab07f3 100644
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -90,13 +90,13 @@ int RGWGC::remove(int index, const std::list<string>& tags)
return store->gc_operate(obj_names[index], &op);
}
-int RGWGC::list(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
+int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
{
result.clear();
for (; *index < cct->_conf->rgw_gc_max_objs && result.size() < max; (*index)++, marker.clear()) {
std::list<cls_rgw_gc_obj_info> entries;
- int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), entries, truncated);
+ int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), expired_only, entries, truncated);
if (ret == -ENOENT)
continue;
if (ret < 0)
@@ -158,7 +158,7 @@ int RGWGC::process(int index, int max_secs)
do {
int max = 100;
std::list<cls_rgw_gc_obj_info> entries;
- ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, entries, &truncated);
+ ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, true, entries, &truncated);
if (ret == -ENOENT) {
ret = 0;
goto done;
diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h
index b19afc5..afaead4 100644
--- a/src/rgw/rgw_gc.h
+++ b/src/rgw/rgw_gc.h
@@ -49,7 +49,7 @@ public:
void initialize(CephContext *_cct, RGWRados *_store);
void finalize();
- int list(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
+ int list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
void list_init(int *index) { *index = 0; }
int process(int index, int process_max_secs);
int process();
diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 231475b..81ec599 100644
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -40,6 +40,14 @@ void RGWObjManifestPart::dump(Formatter *f) const
f->dump_unsigned("size", size);
}
+void RGWObjManifestRule::dump(Formatter *f) const
+{
+ encode_json("start_part_num", start_part_num, f);
+ encode_json("start_ofs", start_ofs, f);
+ encode_json("part_size", part_size, f);
+ encode_json("stripe_max_size", stripe_max_size, f);
+}
+
void RGWObjManifest::dump(Formatter *f) const
{
map<uint64_t, RGWObjManifestPart>::const_iterator iter = objs.begin();
@@ -52,6 +60,13 @@ void RGWObjManifest::dump(Formatter *f) const
}
f->close_section();
f->dump_unsigned("obj_size", obj_size);
+ ::encode_json("explicit_objs", explicit_objs, f);
+ ::encode_json("head_obj", head_obj, f);
+ ::encode_json("head_size", head_size, f);
+ ::encode_json("max_head_size", max_head_size, f);
+ ::encode_json("prefix", prefix, f);
+ ::encode_json("tail_bucket", tail_bucket, f);
+ ::encode_json("rules", rules, f);
}
void rgw_log_entry::dump(Formatter *f) const
diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index 9398dd0..2673761 100644
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -166,6 +166,9 @@ void rgw_log_usage_finalize()
static void log_usage(struct req_state *s, const string& op_name)
{
+ if (s->system_request) /* don't log system user operations */
+ return;
+
if (!usage_logger)
return;
diff --git a/src/rgw/rgw_multi.cc b/src/rgw/rgw_multi.cc
index 05863e3..74d18ba 100644
--- a/src/rgw/rgw_multi.cc
+++ b/src/rgw/rgw_multi.cc
@@ -48,7 +48,8 @@ bool RGWMultiCompleteUpload::xml_end(const char *el) {
XMLObj *RGWMultiXMLParser::alloc_obj(const char *el) {
XMLObj *obj = NULL;
- if (strcmp(el, "CompleteMultipartUpload") == 0) {
+ if (strcmp(el, "CompleteMultipartUpload") == 0 ||
+ strcmp(el, "MultipartUpload") == 0) {
obj = new RGWMultiCompleteUpload();
} else if (strcmp(el, "Part") == 0) {
obj = new RGWMultiPart();
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index a713285..074e7d8 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -342,7 +342,7 @@ static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bu
}
}
- if (s->bucket_name_str.size()) {
+ if (!s->bucket_name_str.empty()) {
s->bucket_exists = true;
if (s->bucket_instance_id.empty()) {
ret = store->get_bucket_info(s->obj_ctx, s->bucket_name_str, s->bucket_info, NULL, &s->bucket_attrs);
@@ -358,9 +358,13 @@ static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bu
}
s->bucket = s->bucket_info.bucket;
- string no_obj;
- RGWAccessControlPolicy bucket_acl(s->cct);
- ret = read_policy(store, s, s->bucket_info, s->bucket_attrs, s->bucket_acl, s->bucket, no_obj);
+ if (s->bucket_exists) {
+ string no_obj;
+ ret = read_policy(store, s, s->bucket_info, s->bucket_attrs, s->bucket_acl, s->bucket, no_obj);
+ } else {
+ s->bucket_acl->create_default(s->user.user_id, s->user.display_name);
+ ret = -ERR_NO_SUCH_BUCKET;
+ }
s->bucket_owner = s->bucket_acl->get_owner();
@@ -382,7 +386,10 @@ static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bu
/* we're passed only_bucket = true when we specifically need the bucket's
acls, that happens on write operations */
- if (!only_bucket) {
+ if (!only_bucket && !s->object_str.empty()) {
+ if (!s->bucket_exists) {
+ return -ERR_NO_SUCH_BUCKET;
+ }
s->object_acl = new RGWAccessControlPolicy(s->cct);
obj_str = s->object_str;
@@ -1150,6 +1157,7 @@ void RGWCreateBucket::execute()
RGWAccessControlPolicy old_policy(s->cct);
map<string, bufferlist> attrs;
bufferlist aclbl;
+ bufferlist corsbl;
bool existed;
int r;
rgw_obj obj(store->zone.domain_root, s->bucket_name_str);
@@ -1223,6 +1231,10 @@ void RGWCreateBucket::execute()
attrs[RGW_ATTR_ACL] = aclbl;
+ if (has_cors) {
+ cors_config.encode(corsbl);
+ attrs[RGW_ATTR_CORS] = corsbl;
+ }
s->bucket.name = s->bucket_name_str;
ret = store->create_bucket(s->user, s->bucket, region_name, placement_rule, attrs, info, pobjv,
&ep_objv, creation_time, pmaster_bucket, true);
@@ -1363,16 +1375,36 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, void *obj_ctx)
part_num = s->info.args.get("partNumber");
if (part_num.empty()) {
+ ldout(s->cct, 10) << "part number is empty" << dendl;
+ return -EINVAL;
+ }
+
+ string err;
+ uint64_t num = (uint64_t)strict_strtol(part_num.c_str(), 10, &err);
+
+ if (!err.empty()) {
+ ldout(s->cct, 10) << "bad part number: " << part_num << ": " << err << dendl;
return -EINVAL;
}
- oid = mp.get_part(part_num);
+ string upload_prefix = oid + "." + upload_id;
- head_obj.init_ns(bucket, oid, mp_ns);
- oid_prefix = oid;
- oid_prefix.append("_");
+ rgw_obj target_obj;
+ target_obj.init(bucket, oid);
+
+ manifest.set_prefix(upload_prefix);
+
+ manifest.set_multipart_part_rule(store->ctx()->_conf->rgw_obj_stripe_size, num);
+
+ int r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, target_obj);
+ if (r < 0) {
+ return r;
+ }
+
+ head_obj = manifest_gen.get_cur_obj();
cur_obj = head_obj;
- add_obj(head_obj);
+ add_obj(cur_obj);
+
return 0;
}
@@ -2582,7 +2614,7 @@ void RGWCompleteMultipart::execute()
return;
}
- char etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
+ char petag[CEPH_CRYPTO_MD5_DIGESTSIZE];
if (iter->first != (int)obj_iter->first) {
ldout(s->cct, 0) << "NOTICE: parts num mismatch: next requested: " << iter->first << " next uploaded: " << obj_iter->first << dendl;
ret = -ERR_INVALID_PART;
@@ -2595,8 +2627,8 @@ void RGWCompleteMultipart::execute()
return;
}
- hex_to_buf(obj_iter->second.etag.c_str(), etag, CEPH_CRYPTO_MD5_DIGESTSIZE);
- hash.Update((const byte *)etag, sizeof(etag));
+ hex_to_buf(obj_iter->second.etag.c_str(), petag, CEPH_CRYPTO_MD5_DIGESTSIZE);
+ hash.Update((const byte *)petag, sizeof(petag));
RGWUploadPartInfo& obj_part = obj_iter->second;
@@ -2606,11 +2638,9 @@ void RGWCompleteMultipart::execute()
src_obj.init_ns(s->bucket, oid, mp_ns);
if (obj_part.manifest.empty()) {
- RGWObjManifestPart& part = manifest.objs[ofs];
-
- part.loc = src_obj;
- part.loc_ofs = 0;
- part.size = part_size;
+ ldout(s->cct, 0) << "ERROR: empty manifest for object part: obj=" << src_obj << dendl;
+ ret = -ERR_INVALID_PART;
+ return;
} else {
manifest.append(obj_part.manifest);
}
@@ -2625,6 +2655,7 @@ void RGWCompleteMultipart::execute()
buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str);
snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2,
"-%lld", (long long)parts->parts.size());
+ etag = final_etag_str;
ldout(s->cct, 10) << "calculated etag: " << final_etag_str << dendl;
etag_bl.append(final_etag_str, strlen(final_etag_str) + 1);
@@ -2633,8 +2664,6 @@ void RGWCompleteMultipart::execute()
target_obj.init(s->bucket, s->object_str);
- manifest.obj_size = ofs;
-
store->set_atomic(s->obj_ctx, target_obj);
RGWRados::PutObjMetaExtraParams extra_params;
@@ -2713,10 +2742,10 @@ void RGWAbortMultipart::execute()
return;
} else {
RGWObjManifest& manifest = obj_part.manifest;
- map<uint64_t, RGWObjManifestPart>::iterator oiter;
- for (oiter = manifest.objs.begin(); oiter != manifest.objs.end(); ++oiter) {
- RGWObjManifestPart& part = oiter->second;
- ret = store->delete_obj(s->obj_ctx, owner, part.loc);
+ RGWObjManifest::obj_iterator oiter;
+ for (oiter = manifest.obj_begin(); oiter != manifest.obj_end(); ++oiter) {
+ rgw_obj loc = oiter.get_location();
+ ret = store->delete_obj(s->obj_ctx, owner, loc);
if (ret < 0 && ret != -ENOENT)
return;
}
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index b85f0e2..9c1fa53 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1,3 +1,5 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
/**
* All operations via the rados gateway are carried out by
* small classes known as RGWOps. This class contains a req_state
@@ -268,11 +270,13 @@ protected:
string placement_rule;
RGWBucketInfo info;
obj_version ep_objv;
+ bool has_cors;
+ RGWCORSConfiguration cors_config;
bufferlist in_data;
public:
- RGWCreateBucket() : ret(0) {}
+ RGWCreateBucket() : ret(0), has_cors(false) {}
int verify_permission();
void pre_exec();
@@ -826,8 +830,10 @@ protected:
public:
RGWListBucketMultiparts() {
+ max_uploads = 0;
ret = 0;
is_truncated = false;
+ default_max = 0;
}
virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index cce463b..cb82e05 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -527,8 +527,423 @@ void RGWObjVersionTracker::prepare_op_for_write(ObjectWriteOperation *op)
}
}
-void RGWObjManifest::append(RGWObjManifest& m)
+void RGWObjManifest::obj_iterator::update_explicit_pos()
{
+ ofs = explicit_iter->first;
+ stripe_ofs = ofs;
+
+ map<uint64_t, RGWObjManifestPart>::iterator next_iter = explicit_iter;
+ ++next_iter;
+ if (next_iter != manifest->objs.end()) {
+ stripe_size = next_iter->first - ofs;
+ } else {
+ stripe_size = manifest->obj_size - ofs;
+ }
+}
+
+void RGWObjManifest::obj_iterator::seek(uint64_t o)
+{
+ ofs = o;
+ if (manifest->explicit_objs) {
+ explicit_iter = manifest->objs.upper_bound(ofs);
+ if (explicit_iter != manifest->objs.begin()) {
+ --explicit_iter;
+ }
+ if (ofs >= manifest->obj_size) {
+ ofs = manifest->obj_size;
+ return;
+ }
+ update_explicit_pos();
+ update_location();
+ return;
+ }
+ if (o < manifest->get_head_size()) {
+ rule_iter = manifest->rules.begin();
+ stripe_ofs = 0;
+ stripe_size = manifest->get_head_size();
+ cur_part_id = rule_iter->second.start_part_num;
+ update_location();
+ return;
+ }
+
+ rule_iter = manifest->rules.upper_bound(ofs);
+ next_rule_iter = rule_iter;
+ if (rule_iter != manifest->rules.begin()) {
+ --rule_iter;
+ }
+
+ RGWObjManifestRule& rule = rule_iter->second;
+
+ if (rule.part_size > 0) {
+ cur_part_id = rule.start_part_num + (ofs - rule.start_ofs) / rule.part_size;
+ } else {
+ cur_part_id = rule.start_part_num;
+ }
+ part_ofs = rule.start_ofs + (cur_part_id - rule.start_part_num) * rule.part_size;
+
+ if (rule.stripe_max_size > 0) {
+ cur_stripe = (ofs - part_ofs) / rule.stripe_max_size;
+
+ stripe_ofs = part_ofs + cur_stripe * rule.stripe_max_size;
+ if (!cur_part_id && manifest->get_head_size() > 0) {
+ cur_stripe++;
+ }
+ } else {
+ cur_stripe = 0;
+ stripe_ofs = part_ofs;
+ }
+
+ if (!rule.part_size) {
+ stripe_size = rule.stripe_max_size;
+ stripe_size = MIN(manifest->get_obj_size() - stripe_ofs, stripe_size);
+ } else {
+ stripe_size = rule.part_size - (ofs - stripe_ofs);
+ stripe_size = MIN(stripe_size, rule.stripe_max_size);
+ }
+
+ update_location();
+}
+
+void RGWObjManifest::obj_iterator::update_location()
+{
+ if (manifest->explicit_objs) {
+ location = explicit_iter->second.loc;
+ return;
+ }
+
+ const rgw_obj& head = manifest->get_head();
+
+ if (ofs < manifest->get_head_size()) {
+ location = head;
+ return;
+ }
+
+ manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &location);
+}
+
+void RGWObjManifest::obj_iterator::operator++()
+{
+ if (manifest->explicit_objs) {
+ ++explicit_iter;
+
+ if (explicit_iter == manifest->objs.end()) {
+ ofs = manifest->obj_size;
+ return;
+ }
+
+ update_explicit_pos();
+
+ update_location();
+ return;
+ }
+
+ uint64_t obj_size = manifest->get_obj_size();
+ uint64_t head_size = manifest->get_head_size();
+
+ if (ofs == obj_size) {
+ return;
+ }
+
+ if (manifest->rules.empty()) {
+ return;
+ }
+
+ /* are we still pointing at the head? */
+ if (ofs < head_size) {
+ rule_iter = manifest->rules.begin();
+ RGWObjManifestRule *rule = &rule_iter->second;
+ ofs = MIN(head_size, obj_size);
+ stripe_ofs = ofs;
+ cur_stripe = 1;
+ stripe_size = MIN(obj_size - ofs, rule->stripe_max_size);
+ if (rule->part_size > 0) {
+ stripe_size = MIN(stripe_size, rule->part_size);
+ }
+ update_location();
+ return;
+ }
+
+ RGWObjManifestRule *rule = &rule_iter->second;
+
+ stripe_ofs += rule->stripe_max_size;
+ cur_stripe++;
+ dout(20) << "RGWObjManifest::operator++(): rule->part_size=" << rule->part_size << " rules.size()=" << manifest->rules.size() << dendl;
+
+ if (rule->part_size > 0) {
+ /* multi part, multi stripes object */
+
+ dout(20) << "RGWObjManifest::operator++(): stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
+
+ if (stripe_ofs >= part_ofs + rule->part_size) {
+ /* moved to the next part */
+ cur_stripe = 0;
+ part_ofs += rule->part_size;
+ stripe_ofs = part_ofs;
+
+ /* move to the next rule? */
+ if (next_rule_iter->second.start_ofs >= stripe_ofs) {
+ rule_iter = next_rule_iter;
+ bool last_rule = (next_rule_iter == manifest->rules.end());
+ if (!last_rule) {
+ ++next_rule_iter;
+ }
+ cur_part_id = rule_iter->second.start_part_num;
+ } else {
+ cur_part_id++;
+ }
+
+ rule = &rule_iter->second;
+ }
+
+ stripe_size = MIN(rule->part_size - (stripe_ofs - part_ofs), rule->stripe_max_size);
+ }
+
+ ofs = stripe_ofs;
+ if (ofs > obj_size) {
+ ofs = obj_size;
+ stripe_ofs = ofs;
+ stripe_size = 0;
+ }
+
+ dout(0) << "RGWObjManifest::operator++(): result: ofs=" << ofs << " stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
+ update_location();
+}
+
+int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m, rgw_bucket& _b, rgw_obj& _h)
+{
+ manifest = _m;
+
+ bucket = _b;
+ manifest->set_tail_bucket(_b);
+ manifest->set_head(_h);
+ last_ofs = 0;
+
+ char buf[33];
+ gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1);
+
+ if (manifest->get_prefix().empty()) {
+ string oid_prefix = ".";
+ oid_prefix.append(buf);
+ oid_prefix.append("_");
+
+ manifest->set_prefix(oid_prefix);
+ }
+
+ bool found = manifest->get_rule(0, &rule);
+ if (!found) {
+ derr << "ERROR: manifest->get_rule() could not find rule" << dendl;
+ return -EIO;
+ }
+
+ uint64_t head_size = manifest->get_head_size();
+
+ if (head_size > 0) {
+ cur_stripe_size = head_size;
+ } else {
+ cur_stripe_size = rule.stripe_max_size;
+ }
+
+ cur_part_id = rule.start_part_num;
+
+ manifest->get_implicit_location(cur_part_id, cur_stripe, 0, &cur_obj);
+
+ manifest->update_iterators();
+
+ return 0;
+}
+
+int RGWObjManifest::generator::create_next(uint64_t ofs)
+{
+ if (ofs < last_ofs) /* only going forward */
+ return -EINVAL;
+
+ string obj_name = manifest->prefix;
+
+ uint64_t max_head_size = manifest->get_max_head_size();
+
+ if (ofs <= max_head_size) {
+ manifest->set_head_size(ofs);
+ }
+
+ if (ofs >= max_head_size) {
+ manifest->set_head_size(max_head_size);
+ cur_stripe = (ofs - max_head_size) / rule.stripe_max_size;
+ cur_stripe_size = rule.stripe_max_size;
+
+ if (cur_part_id == 0 && max_head_size > 0) {
+ cur_stripe++;
+ }
+ }
+
+ last_ofs = ofs;
+ manifest->set_obj_size(ofs);
+
+
+ manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_obj);
+
+ manifest->update_iterators();
+
+ return 0;
+}
+
+const RGWObjManifest::obj_iterator& RGWObjManifest::obj_begin()
+{
+ return begin_iter;
+}
+
+const RGWObjManifest::obj_iterator& RGWObjManifest::obj_end()
+{
+ return end_iter;
+}
+
+void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, rgw_obj *location)
+{
+ string oid = prefix;
+ string ns;
+
+ if (!cur_part_id) {
+ if (ofs < max_head_size) {
+ *location = head_obj;
+ return;
+ } else {
+ char buf[16];
+ snprintf(buf, sizeof(buf), "%d", (int)cur_stripe);
+ oid += buf;
+ ns = shadow_ns;
+ }
+ } else {
+ char buf[32];
+ if (cur_stripe == 0) {
+ snprintf(buf, sizeof(buf), ".%d", (int)cur_part_id);
+ oid += buf;
+ ns= RGW_OBJ_NS_MULTIPART;
+ } else {
+ snprintf(buf, sizeof(buf), ".%d_%d", (int)cur_part_id, (int)cur_stripe);
+ oid += buf;
+ ns = shadow_ns;
+ }
+ }
+
+ rgw_bucket *bucket;
+
+ if (!tail_bucket.name.empty()) {
+ bucket = &tail_bucket;
+ } else {
+ bucket = &head_obj.bucket;
+ }
+
+ location->init_ns(*bucket, oid, ns);
+}
+
+RGWObjManifest::obj_iterator RGWObjManifest::obj_find(uint64_t ofs)
+{
+ if (ofs > obj_size) {
+ ofs = obj_size;
+ }
+ RGWObjManifest::obj_iterator iter(this);
+ iter.seek(ofs);
+ return iter;
+}
+
+int RGWObjManifest::append(RGWObjManifest& m)
+{
+ if (explicit_objs || m.explicit_objs) {
+ return append_explicit(m);
+ }
+
+ if (rules.empty()) {
+ *this = m;
+ return 0;
+ }
+
+ if (prefix.empty()) {
+ prefix = m.prefix;
+ } else if (prefix != m.prefix) {
+ return append_explicit(m);
+ }
+
+ map<uint64_t, RGWObjManifestRule>::iterator miter = m.rules.begin();
+ if (miter == m.rules.end()) {
+ return append_explicit(m);
+ }
+
+ for (; miter != m.rules.end(); ++miter) {
+ map<uint64_t, RGWObjManifestRule>::reverse_iterator last_rule = rules.rbegin();
+
+ RGWObjManifestRule& rule = last_rule->second;
+
+ if (rule.part_size == 0) {
+ rule.part_size = obj_size - rule.start_ofs;
+ }
+
+ RGWObjManifestRule& next_rule = miter->second;
+ if (!next_rule.part_size) {
+ next_rule.part_size = m.obj_size - next_rule.start_ofs;
+ }
+
+ if (rule.part_size != next_rule.part_size ||
+ rule.stripe_max_size != next_rule.stripe_max_size) {
+ append_rules(m, miter);
+ break;
+ }
+
+ uint64_t expected_part_num = rule.start_part_num + 1;
+ if (rule.part_size > 0) {
+ expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size;
+ }
+
+ if (expected_part_num != next_rule.start_part_num) {
+ append_rules(m, miter);
+ break;
+ }
+ }
+
+ set_obj_size(obj_size + m.obj_size);
+
+ return 0;
+}
+
+void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter)
+{
+ for (; miter != m.rules.end(); ++miter) {
+ RGWObjManifestRule rule = miter->second;
+ rule.start_ofs += obj_size;
+ rules[rule.start_ofs] = rule;
+ }
+}
+
+void RGWObjManifest::convert_to_explicit()
+{
+ if (explicit_objs) {
+ return;
+ }
+ obj_iterator iter = obj_begin();
+
+ while (iter != obj_end()) {
+ RGWObjManifestPart& part = objs[iter.get_stripe_ofs()];
+ part.loc = iter.get_location();
+ part.loc_ofs = 0;
+
+ uint64_t ofs = iter.get_stripe_ofs();
+ ++iter;
+ uint64_t next_ofs = iter.get_stripe_ofs();
+
+ part.size = next_ofs - ofs;
+ }
+
+ explicit_objs = true;
+ rules.clear();
+ prefix.clear();
+}
+
+int RGWObjManifest::append_explicit(RGWObjManifest& m)
+{
+ if (!explicit_objs) {
+ convert_to_explicit();
+ }
+ if (!m.explicit_objs) {
+ m.convert_to_explicit();
+ }
map<uint64_t, RGWObjManifestPart>::iterator iter;
uint64_t base = obj_size;
for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) {
@@ -536,6 +951,24 @@ void RGWObjManifest::append(RGWObjManifest& m)
objs[base + iter->first] = part;
}
obj_size += m.obj_size;
+
+ return 0;
+}
+
+bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule)
+{
+ if (rules.empty()) {
+ return false;
+ }
+
+ map<uint64_t, RGWObjManifestRule>::iterator iter = rules.upper_bound(ofs);
+ if (iter != rules.begin()) {
+ --iter;
+ }
+
+ *rule = iter->second;
+
+ return true;
}
void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct)
@@ -686,8 +1119,12 @@ int RGWPutObjProcessor_Aio::throttle_data(void *handle)
int RGWPutObjProcessor_Atomic::write_data(bufferlist& bl, off_t ofs, void **phandle)
{
- if (ofs >= next_part_ofs)
- prepare_next_part(ofs);
+ if (ofs >= next_part_ofs) {
+ int r = prepare_next_part(ofs);
+ if (r < 0) {
+ return r;
+ }
+ }
return RGWPutObjProcessor_Aio::handle_obj_data(cur_obj, bl, ofs - cur_part_ofs, ofs, phandle);
}
@@ -719,7 +1156,10 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, void **pha
if (!data_ofs && !immutable_head()) {
first_chunk.claim(bl);
obj_len = (uint64_t)first_chunk.length();
- prepare_next_part(first_chunk.length());
+ int r = prepare_next_part(first_chunk.length());
+ if (r < 0) {
+ return r;
+ }
data_ofs = obj_len;
return 0;
}
@@ -734,52 +1174,37 @@ int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, void *obj_ctx)
head_obj.init(bucket, obj_str);
- char buf[33];
- gen_rand_alphanumeric(store->ctx(), buf, sizeof(buf) - 1);
- oid_prefix.append("_");
- oid_prefix.append(buf);
- oid_prefix.append("_");
+ manifest.set_trivial_rule(RGW_MAX_CHUNK_SIZE, store->ctx()->_conf->rgw_obj_stripe_size);
+
+ int r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, head_obj);
+ if (r < 0) {
+ return r;
+ }
return 0;
}
-void RGWPutObjProcessor_Atomic::prepare_next_part(off_t ofs) {
- int num_parts = manifest.objs.size();
- RGWObjManifestPart *part;
+int RGWPutObjProcessor_Atomic::prepare_next_part(off_t ofs) {
- /* first update manifest for written data */
- if (!num_parts) {
- part = &manifest.objs[cur_part_ofs];
- part->loc = head_obj;
- } else {
- part = &manifest.objs[cur_part_ofs];
- part->loc = cur_obj;
+ int ret = manifest_gen.create_next(ofs);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: manifest_gen.create_next() returned ret=" << ret << dendl;
+ return ret;
}
- part->loc_ofs = 0;
- part->size = ofs - cur_part_ofs;
-
- if ((uint64_t)ofs > manifest.obj_size)
- manifest.obj_size = ofs;
-
- /* now update params for next part */
-
cur_part_ofs = ofs;
- next_part_ofs = cur_part_ofs + part_size;
- char buf[16];
-
- cur_part_id++;
- snprintf(buf, sizeof(buf), "%d", cur_part_id);
- string cur_oid = oid_prefix;
- cur_oid.append(buf);
- cur_obj.init_ns(bucket, cur_oid, shadow_ns);
-
+ next_part_ofs = ofs + manifest_gen.cur_stripe_max_size();
+ cur_obj = manifest_gen.get_cur_obj();
add_obj(cur_obj);
+
+ return 0;
};
-void RGWPutObjProcessor_Atomic::complete_parts()
+int RGWPutObjProcessor_Atomic::complete_parts()
{
- if (obj_len > (uint64_t)cur_part_ofs)
- prepare_next_part(obj_len);
+ if (obj_len > (uint64_t)cur_part_ofs) {
+ return prepare_next_part(obj_len);
+ }
+ return 0;
}
int RGWPutObjProcessor_Atomic::complete_writing_data()
@@ -801,9 +1226,12 @@ int RGWPutObjProcessor_Atomic::complete_writing_data()
return r;
}
}
- complete_parts();
+ int r = complete_parts();
+ if (r < 0) {
+ return r;
+ }
- int r = drain_pending();
+ r = drain_pending();
if (r < 0)
return r;
@@ -1936,6 +2364,7 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
ret = put_linked_bucket_info(info, exclusive, 0, pep_objv, &attrs, true);
if (ret == -EEXIST) {
/* we need to reread the info and return it, caller will have a use for it */
+ info.objv_tracker.clear();
r = get_bucket_info(NULL, bucket.name, info, NULL, NULL);
if (r < 0) {
if (r == -ENOENT) {
@@ -2752,13 +3181,12 @@ set_err_state:
bool copy_data = !astate->has_manifest;
bool copy_first = false;
if (astate->has_manifest) {
- if (astate->manifest.objs.size() < 2) {
+ if (!astate->manifest.has_tail()) {
copy_data = true;
} else {
- map<uint64_t, RGWObjManifestPart>::iterator iter = astate->manifest.objs.begin();
- RGWObjManifestPart part = iter->second;
- if (part.loc == src_obj) {
- if (part.size > RGW_MAX_CHUNK_SIZE) // should never happen
+ uint64_t head_size = astate->manifest.get_head_size();
+ if (head_size > 0) {
+ if (head_size > RGW_MAX_CHUNK_SIZE) // should never happen
copy_data = true;
else
copy_first = true;
@@ -2791,15 +3219,14 @@ set_err_state:
return copy_obj_data(ctx, dest_bucket_info.owner, &handle, end, dest_obj, src_obj, mtime, src_attrs, category, ptag, err);
}
- map<uint64_t, RGWObjManifestPart>::iterator miter = astate->manifest.objs.begin();
+ RGWObjManifest::obj_iterator miter = astate->manifest.obj_begin();
if (copy_first) // we need to copy first chunk, not increase refcount
++miter;
- RGWObjManifestPart *first_part = &miter->second;
string oid, key;
rgw_bucket bucket;
- get_obj_bucket_and_oid_key(first_part->loc, bucket, oid, key);
+ get_obj_bucket_and_oid_key(miter.get_location(), bucket, oid, key);
librados::IoCtx io_ctx;
PutObjMetaExtraParams ep;
@@ -2824,22 +3251,24 @@ set_err_state:
}
if (!copy_itself) {
- for (; miter != astate->manifest.objs.end(); ++miter) {
- RGWObjManifestPart& part = miter->second;
+ manifest = astate->manifest;
+ rgw_bucket& tail_bucket = manifest.get_tail_bucket();
+ if (tail_bucket.name.empty()) {
+ manifest.set_tail_bucket(src_obj.bucket);
+ }
+ for (; miter != astate->manifest.obj_end(); ++miter) {
ObjectWriteOperation op;
- manifest.objs[miter->first] = part;
cls_refcount_get(op, tag, true);
-
- get_obj_bucket_and_oid_key(part.loc, bucket, oid, key);
+ const rgw_obj& loc = miter.get_location();
+ get_obj_bucket_and_oid_key(loc, bucket, oid, key);
io_ctx.locator_set_key(key);
ret = io_ctx.operate(oid, &op);
if (ret < 0)
goto done_ret;
- ref_objs.push_back(part.loc);
+ ref_objs.push_back(loc);
}
- manifest.obj_size = total_len;
pmanifest = &manifest;
} else {
@@ -2853,10 +3282,8 @@ set_err_state:
if (ret < 0)
goto done_ret;
- first_part = &pmanifest->objs[0];
- first_part->loc = dest_obj;
- first_part->loc_ofs = 0;
- first_part->size = first_chunk.length();
+ pmanifest->set_head(dest_obj);
+ pmanifest->set_head_size(first_chunk.length());
}
ep.data = &first_chunk;
@@ -2906,6 +3333,7 @@ int RGWRados::copy_obj_data(void *ctx,
{
bufferlist first_chunk;
RGWObjManifest manifest;
+ map<uint64_t, RGWObjManifestPart> objs;
RGWObjManifestPart *first_part;
map<string, bufferlist>::iterator iter;
@@ -2947,18 +3375,19 @@ int RGWRados::copy_obj_data(void *ctx,
ofs += ret;
} while (ofs <= end);
- first_part = &manifest.objs[0];
+ first_part = &objs[0];
first_part->loc = dest_obj;
first_part->loc_ofs = 0;
first_part->size = first_chunk.length();
if (ofs > RGW_MAX_CHUNK_SIZE) {
- RGWObjManifestPart& tail = manifest.objs[RGW_MAX_CHUNK_SIZE];
+ RGWObjManifestPart& tail = objs[RGW_MAX_CHUNK_SIZE];
tail.loc = shadow_obj;
tail.loc_ofs = RGW_MAX_CHUNK_SIZE;
tail.size = ofs - RGW_MAX_CHUNK_SIZE;
}
- manifest.obj_size = ofs;
+
+ manifest.set_explicit(ofs, objs);
ep.data = &first_chunk;
ep.manifest = &manifest;
@@ -3095,9 +3524,9 @@ int RGWRados::complete_atomic_overwrite(RGWRadosCtx *rctx, RGWObjState *state, r
return 0;
cls_rgw_obj_chain chain;
- map<uint64_t, RGWObjManifestPart>::iterator iter;
- for (iter = state->manifest.objs.begin(); iter != state->manifest.objs.end(); ++iter) {
- rgw_obj& mobj = iter->second.loc;
+ RGWObjManifest::obj_iterator iter;
+ for (iter = state->manifest.obj_begin(); iter != state->manifest.obj_end(); ++iter) {
+ const rgw_obj& mobj = iter.get_location();
if (mobj == obj)
continue;
string oid, key;
@@ -3330,11 +3759,11 @@ static void generate_fake_tag(CephContext *cct, map<string, bufferlist>& attrset
{
string tag;
- map<uint64_t, RGWObjManifestPart>::iterator mi = manifest.objs.begin();
- if (mi != manifest.objs.end()) {
- if (manifest.objs.size() > 1) // first object usually points at the head, let's skip to a more unique part
+ RGWObjManifest::obj_iterator mi = manifest.obj_begin();
+ if (mi != manifest.obj_end()) {
+ if (manifest.has_tail()) // first object usually points at the head, let's skip to a more unique part
++mi;
- tag = mi->second.loc.object;
+ tag = mi.get_location().object;
tag.append("_");
}
@@ -3392,15 +3821,17 @@ int RGWRados::get_obj_state(RGWRadosCtx *rctx, rgw_obj& obj, RGWObjState **state
try {
::decode(s->manifest, miter);
s->has_manifest = true;
- s->size = s->manifest.obj_size;
+ s->size = s->manifest.get_obj_size();
} catch (buffer::error& err) {
ldout(cct, 20) << "ERROR: couldn't decode manifest" << dendl;
return -EIO;
}
- ldout(cct, 10) << "manifest: total_size = " << s->manifest.obj_size << dendl;
- map<uint64_t, RGWObjManifestPart>::iterator mi;
- for (mi = s->manifest.objs.begin(); mi != s->manifest.objs.end(); ++mi) {
- ldout(cct, 10) << "manifest: ofs=" << mi->first << " loc=" << mi->second.loc << dendl;
+ ldout(cct, 10) << "manifest: total_size = " << s->manifest.get_obj_size() << dendl;
+ if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20) && s->manifest.has_explicit_objs()) {
+ RGWObjManifest::obj_iterator mi;
+ for (mi = s->manifest.obj_begin(); mi != s->manifest.obj_end(); ++mi) {
+ ldout(cct, 20) << "manifest: ofs=" << mi.get_ofs() << " loc=" << mi.get_location() << dendl;
+ }
}
if (!s->obj_tag.length()) {
@@ -4072,20 +4503,14 @@ int RGWRados::get_obj(void *ctx, RGWObjVersionTracker *objv_tracker, void **hand
else
len = end - ofs + 1;
- if (astate->has_manifest && !astate->manifest.objs.empty()) {
+ if (astate->has_manifest && astate->manifest.has_tail()) {
/* now get the relevant object part */
- map<uint64_t, RGWObjManifestPart>::iterator iter = astate->manifest.objs.upper_bound(ofs);
- /* we're now pointing at the next part (unless the first part starts at a higher ofs),
- so retract to previous part */
- if (iter != astate->manifest.objs.begin()) {
- --iter;
- }
+ RGWObjManifest::obj_iterator iter = astate->manifest.obj_find(ofs);
- RGWObjManifestPart& part = iter->second;
- uint64_t part_ofs = iter->first;
- read_obj = part.loc;
- len = min(len, part.size - (ofs - part_ofs));
- read_ofs = part.loc_ofs + (ofs - part_ofs);
+ uint64_t stripe_ofs = iter.get_stripe_ofs();
+ read_obj = iter.get_location();
+ len = min(len, iter.get_stripe_size() - (ofs - stripe_ofs));
+ read_ofs = iter.location_ofs() + (ofs - stripe_ofs);
reading_from_head = (read_obj == obj);
if (!reading_from_head) {
@@ -4574,23 +4999,19 @@ int RGWRados::iterate_obj(void *ctx, rgw_obj& obj,
len = end - ofs + 1;
if (astate->has_manifest) {
- /* now get the relevant object part */
- map<uint64_t, RGWObjManifestPart>::iterator iter = astate->manifest.objs.upper_bound(ofs);
- /* we're now pointing at the next part (unless the first part starts at a higher ofs),
- so retract to previous part */
- if (iter != astate->manifest.objs.begin()) {
- --iter;
- }
+ /* now get the relevant object stripe */
+ RGWObjManifest::obj_iterator iter = astate->manifest.obj_find(ofs);
+
+ RGWObjManifest::obj_iterator obj_end = astate->manifest.obj_end();
- for (; iter != astate->manifest.objs.end() && ofs <= end; ++iter) {
- RGWObjManifestPart& part = iter->second;
- off_t part_ofs = iter->first;
- off_t next_part_ofs = part_ofs + part.size;
+ for (; iter != obj_end && ofs <= end; ++iter) {
+ off_t stripe_ofs = iter.get_stripe_ofs();
+ off_t next_stripe_ofs = stripe_ofs + iter.get_stripe_size();
- while (ofs < next_part_ofs && ofs <= end) {
- read_obj = part.loc;
- uint64_t read_len = min(len, part.size - (ofs - part_ofs));
- read_ofs = part.loc_ofs + (ofs - part_ofs);
+ while (ofs < next_stripe_ofs && ofs <= end) {
+ read_obj = iter.get_location();
+ uint64_t read_len = min(len, iter.get_stripe_size() - (ofs - stripe_ofs));
+ read_ofs = iter.location_ofs() + (ofs - stripe_ofs);
if (read_len > max_chunk_size) {
read_len = max_chunk_size;
@@ -5339,9 +5760,9 @@ int RGWRados::gc_operate(string& oid, librados::ObjectReadOperation *op, bufferl
return gc_pool_ctx.operate(oid, op, pbl);
}
-int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
+int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
{
- return gc->list(index, marker, max, result, truncated);
+ return gc->list(index, marker, max, expired_only, result, truncated);
}
int RGWRados::process_gc()
@@ -5673,12 +6094,10 @@ int RGWRados::check_disk_state(librados::IoCtx io_ctx,
}
if (astate->has_manifest) {
- map<uint64_t, RGWObjManifestPart>::iterator miter;
+ RGWObjManifest::obj_iterator miter;
RGWObjManifest& manifest = astate->manifest;
- for (miter = manifest.objs.begin(); miter != manifest.objs.end(); ++miter) {
- RGWObjManifestPart& part = miter->second;
-
- rgw_obj& loc = part.loc;
+ for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
+ rgw_obj loc = miter.get_location();
if (loc.ns == RGW_OBJ_NS_MULTIPART) {
dout(10) << "check_disk_state(): removing manifest part from index: " << loc << dendl;
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 64b9ef0..f6a810e 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -28,7 +28,7 @@ class RGWGC;
#define RGW_BUCKET_INSTANCE_MD_PREFIX ".bucket.meta."
-static inline void prepend_bucket_marker(rgw_bucket& bucket, string& orig_oid, string& oid)
+static inline void prepend_bucket_marker(rgw_bucket& bucket, const string& orig_oid, string& oid)
{
if (bucket.marker.empty() || orig_oid.empty()) {
oid = orig_oid;
@@ -39,7 +39,7 @@ static inline void prepend_bucket_marker(rgw_bucket& bucket, string& orig_oid, s
}
}
-static inline void get_obj_bucket_and_oid_key(rgw_obj& obj, rgw_bucket& bucket, string& oid, string& key)
+static inline void get_obj_bucket_and_oid_key(const rgw_obj& obj, rgw_bucket& bucket, string& oid, string& key)
{
bucket = obj.bucket;
prepend_bucket_marker(bucket, obj.object, oid);
@@ -118,32 +118,370 @@ struct RGWObjManifestPart {
};
WRITE_CLASS_ENCODER(RGWObjManifestPart);
-struct RGWObjManifest {
+/*
+ The manifest defines a set of rules for structuring the object parts.
+ There are a few terms to note:
+ - head: the head part of the object, which is the part that contains
+ the first chunk of data. An object might not have a head (as in the
+ case of multipart-part objects).
+ - stripe: data portion of a single rgw object that resides on a single
+ rados object.
+ - part: a collection of stripes that make a contiguous part of an
+ object. A regular object will only have one part (although might have
+ many stripes), a multipart object might have many parts. Each part
+ has a fixed stripe size, although the last stripe of a part might
+ be smaller than that. Consecutive parts may be merged if their stripe
+ value is the same.
+*/
+
+struct RGWObjManifestRule {
+ uint32_t start_part_num;
+ uint64_t start_ofs;
+ uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */
+ uint64_t stripe_max_size; /* underlying obj max size */
+
+ RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {}
+ RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) :
+ start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(start_part_num, bl);
+ ::encode(start_ofs, bl);
+ ::encode(part_size, bl);
+ ::encode(stripe_max_size, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(start_part_num, bl);
+ ::decode(start_ofs, bl);
+ ::decode(part_size, bl);
+ ::decode(stripe_max_size, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(RGWObjManifestRule);
+
+class RGWObjManifest {
+protected:
+ bool explicit_objs; /* old manifest? */
map<uint64_t, RGWObjManifestPart> objs;
+
uint64_t obj_size;
- RGWObjManifest() : obj_size(0) {}
+ rgw_obj head_obj;
+ uint64_t head_size;
+
+ uint64_t max_head_size;
+ string prefix;
+ rgw_bucket tail_bucket; /* might be different than the original bucket,
+ as object might have been copied across buckets */
+ map<uint64_t, RGWObjManifestRule> rules;
+
+ void convert_to_explicit();
+ int append_explicit(RGWObjManifest& m);
+ void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter);
+
+ void update_iterators() {
+ begin_iter.seek(0);
+ end_iter.seek(obj_size);
+ }
+public:
+
+ RGWObjManifest() : explicit_objs(false), obj_size(0), head_size(0), max_head_size(0),
+ begin_iter(this), end_iter(this) {}
+ RGWObjManifest(const RGWObjManifest& rhs) {
+ *this = rhs;
+ }
+ RGWObjManifest& operator=(const RGWObjManifest& rhs) {
+ explicit_objs = rhs.explicit_objs;
+ objs = rhs.objs;
+ obj_size = rhs.obj_size;
+ head_obj = rhs.head_obj;
+ head_size = rhs.head_size;
+ max_head_size = rhs.max_head_size;
+ prefix = rhs.prefix;
+ tail_bucket = rhs.tail_bucket;
+ rules = rhs.rules;
+
+ begin_iter.set_manifest(this);
+ end_iter.set_manifest(this);
+
+ begin_iter.seek(rhs.begin_iter.get_ofs());
+ end_iter.seek(rhs.end_iter.get_ofs());
+
+ return *this;
+ }
+
+
+ void set_explicit(uint64_t _size, map<uint64_t, RGWObjManifestPart>& _objs) {
+ explicit_objs = true;
+ obj_size = _size;
+ objs.swap(_objs);
+ }
+
+ void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, rgw_obj *location);
+
+ void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) {
+ RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size);
+ rules[0] = rule;
+ max_head_size = tail_ofs;
+ }
+
+ void set_multipart_part_rule(uint64_t stripe_max_size, uint64_t part_num) {
+ RGWObjManifestRule rule(0, 0, 0, stripe_max_size);
+ rule.start_part_num = part_num;
+ rules[0] = rule;
+ max_head_size = 0;
+ }
void encode(bufferlist& bl) const {
- ENCODE_START(2, 2, bl);
+ ENCODE_START(4, 3, bl);
::encode(obj_size, bl);
::encode(objs, bl);
+ ::encode(explicit_objs, bl);
+ ::encode(head_obj, bl);
+ ::encode(head_size, bl);
+ ::encode(max_head_size, bl);
+ ::encode(prefix, bl);
+ ::encode(rules, bl);
+ ::encode(tail_bucket, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
- DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
- ::decode(obj_size, bl);
- ::decode(objs, bl);
- DECODE_FINISH(bl);
+ DECODE_START_LEGACY_COMPAT_LEN_32(4, 2, 2, bl);
+ ::decode(obj_size, bl);
+ ::decode(objs, bl);
+ if (struct_v >= 3) {
+ ::decode(explicit_objs, bl);
+ ::decode(head_obj, bl);
+ ::decode(head_size, bl);
+ ::decode(max_head_size, bl);
+ ::decode(prefix, bl);
+ ::decode(rules, bl);
+ } else {
+ explicit_objs = true;
+ }
+
+ if (struct_v >= 4) {
+ ::decode(tail_bucket, bl);
+ }
+
+ update_iterators();
+ DECODE_FINISH(bl);
}
void dump(Formatter *f) const;
static void generate_test_instances(list<RGWObjManifest*>& o);
- void append(RGWObjManifest& m);
+ int append(RGWObjManifest& m);
+
+ bool get_rule(uint64_t ofs, RGWObjManifestRule *rule);
+
+ bool empty() {
+ if (explicit_objs)
+ return objs.empty();
+ return rules.empty();
+ }
+
+ bool has_explicit_objs() {
+ return explicit_objs;
+ }
+
+ bool has_tail() {
+ if (explicit_objs) {
+ return (objs.size() >= 2);
+ }
+ return (obj_size > head_size);
+ }
+
+ void set_head(const rgw_obj& _o) {
+ head_obj = _o;
+ }
+
+ const rgw_obj& get_head() {
+ return head_obj;
+ }
+
+ void set_tail_bucket(const rgw_bucket& _b) {
+ tail_bucket = _b;
+ }
+
+ rgw_bucket& get_tail_bucket() {
+ return tail_bucket;
+ }
+
+ void set_prefix(const string& _p) {
+ prefix = _p;
+ }
+
+ const string& get_prefix() {
+ return prefix;
+ }
+
+ void set_head_size(uint64_t _s) {
+ head_size = _s;
+ }
- bool empty() { return objs.empty(); }
+ void set_obj_size(uint64_t s) {
+ obj_size = s;
+
+ update_iterators();
+ }
+
+ uint64_t get_obj_size() {
+ return obj_size;
+ }
+
+ uint64_t get_head_size() {
+ return head_size;
+ }
+
+ void set_max_head_size(uint64_t s) {
+ max_head_size = s;
+ }
+
+ uint64_t get_max_head_size() {
+ return max_head_size;
+ }
+
+ class obj_iterator {
+ RGWObjManifest *manifest;
+ uint64_t part_ofs; /* where current part starts */
+ uint64_t stripe_ofs; /* where current stripe starts */
+ uint64_t ofs; /* current position within the object */
+ uint64_t stripe_size; /* current part size */
+
+ int cur_part_id;
+ int cur_stripe;
+
+ rgw_obj location;
+
+ map<uint64_t, RGWObjManifestRule>::iterator rule_iter;
+ map<uint64_t, RGWObjManifestRule>::iterator next_rule_iter;
+
+ map<uint64_t, RGWObjManifestPart>::iterator explicit_iter;
+
+ void init() {
+ part_ofs = 0;
+ stripe_ofs = 0;
+ stripe_size = 0;
+ cur_part_id = 0;
+ cur_stripe = 0;
+ }
+
+ void update_explicit_pos();
+
+
+ protected:
+
+ void set_manifest(RGWObjManifest *m) {
+ manifest = m;
+ }
+
+ public:
+ obj_iterator() : manifest(NULL) {
+ init();
+ }
+ obj_iterator(RGWObjManifest *_m) : manifest(_m) {
+ init();
+ seek(0);
+ }
+ obj_iterator(RGWObjManifest *_m, uint64_t _ofs) : manifest(_m) {
+ init();
+ seek(_ofs);
+ }
+ void seek(uint64_t ofs);
+
+ void operator++();
+ bool operator==(const obj_iterator& rhs) {
+ return (ofs == rhs.ofs);
+ }
+ bool operator!=(const obj_iterator& rhs) {
+ return (ofs != rhs.ofs);
+ }
+ const rgw_obj& get_location() {
+ return location;
+ }
+
+ /* start of current stripe */
+ uint64_t get_stripe_ofs() {
+ if (manifest->explicit_objs) {
+ return explicit_iter->first;
+ }
+ return stripe_ofs;
+ }
+
+ /* current ofs relative to start of rgw object */
+ uint64_t get_ofs() const {
+ return ofs;
+ }
+
+ /* current stripe size */
+ uint64_t get_stripe_size() {
+ if (manifest->explicit_objs) {
+ return explicit_iter->second.size;
+ }
+ return stripe_size;
+ }
+
+ /* offset where data starts within current stripe */
+ uint64_t location_ofs() {
+ if (manifest->explicit_objs) {
+ return explicit_iter->second.loc_ofs;
+ }
+ return 0; /* all stripes start at zero offset */
+ }
+
+ void update_location();
+
+ friend class RGWObjManifest;
+ };
+
+ const obj_iterator& obj_begin();
+ const obj_iterator& obj_end();
+ obj_iterator obj_find(uint64_t ofs);
+
+ obj_iterator begin_iter;
+ obj_iterator end_iter;
+
+ /*
+ * simple object generator. Using a simple single rule manifest.
+ */
+ class generator {
+ RGWObjManifest *manifest;
+ uint64_t last_ofs;
+ uint64_t cur_part_ofs;
+ int cur_part_id;
+ int cur_stripe;
+ uint64_t cur_stripe_size;
+ string cur_oid;
+
+ string oid_prefix;
+
+ rgw_obj cur_obj;
+ rgw_bucket bucket;
+
+
+ RGWObjManifestRule rule;
+
+ public:
+ generator() : last_ofs(0), cur_part_ofs(0), cur_part_id(0), cur_stripe(0), cur_stripe_size(0) {}
+ int create_begin(CephContext *cct, RGWObjManifest *manifest, rgw_bucket& bucket, rgw_obj& head);
+
+ int create_next(uint64_t ofs);
+
+ const rgw_obj& get_cur_obj() { return cur_obj; }
+
+ /* total max size of current stripe (including head obj) */
+ uint64_t cur_stripe_max_size() {
+ return cur_stripe_size;
+ }
+ };
};
WRITE_CLASS_ENCODER(RGWObjManifest);
@@ -192,7 +530,7 @@ protected:
list<rgw_obj> objs;
- void add_obj(rgw_obj& obj) {
+ void add_obj(const rgw_obj& obj) {
objs.push_back(obj);
}
public:
@@ -274,18 +612,18 @@ protected:
string unique_tag;
- string oid_prefix;
rgw_obj head_obj;
rgw_obj cur_obj;
RGWObjManifest manifest;
+ RGWObjManifest::generator manifest_gen;
virtual bool immutable_head() { return false; }
int write_data(bufferlist& bl, off_t ofs, void **phandle);
virtual int do_complete(string& etag, time_t *mtime, time_t set_mtime, map<string, bufferlist>& attrs);
- void prepare_next_part(off_t ofs);
- void complete_parts();
+ int prepare_next_part(off_t ofs);
+ int complete_parts();
int complete_writing_data();
public:
@@ -1431,7 +1769,7 @@ public:
int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op);
int gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl);
- int list_gc_objs(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
+ int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
int process_gc();
int defer_gc(void *ctx, rgw_obj& obj);
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 6524db8..507a7ff 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -284,9 +284,68 @@ void RGWStatBucket_ObjStore_SWIFT::send_response()
dump_start(s);
}
+static int get_swift_container_settings(req_state *s, RGWRados *store, RGWAccessControlPolicy *policy, bool *has_policy,
+ RGWCORSConfiguration *cors_config, bool *has_cors)
+{
+ string read_list, write_list;
+
+ const char *read_attr = s->info.env->get("HTTP_X_CONTAINER_READ");
+ if (read_attr) {
+ read_list = read_attr;
+ }
+ const char *write_attr = s->info.env->get("HTTP_X_CONTAINER_WRITE");
+ if (write_attr) {
+ write_list = write_attr;
+ }
+
+ *has_policy = false;
+
+ if (read_attr || write_attr) {
+ RGWAccessControlPolicy_SWIFT swift_policy(s->cct);
+ int r = swift_policy.create(store, s->user.user_id, s->user.display_name, read_list, write_list);
+ if (r < 0)
+ return r;
+
+ *policy = swift_policy;
+ *has_policy = true;
+ }
+
+ *has_cors = false;
+
+ /*Check and update CORS configuration*/
+ const char *allow_origins = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_ALLOW_ORIGIN");
+ const char *allow_headers = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_ALLOW_HEADERS");
+ const char *expose_headers = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_EXPOSE_HEADERS");
+ const char *max_age = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_MAX_AGE");
+ if (allow_origins) {
+ RGWCORSConfiguration_SWIFT *swift_cors = new RGWCORSConfiguration_SWIFT;
+ int r = swift_cors->create_update(allow_origins, allow_headers, expose_headers, max_age);
+ if (r < 0) {
+ dout(0) << "Error creating/updating the cors configuration" << dendl;
+ delete swift_cors;
+ return r;
+ }
+ *has_cors = true;
+ *cors_config = *swift_cors;
+ cors_config->dump();
+ delete swift_cors;
+ }
+
+ return 0;
+}
+
int RGWCreateBucket_ObjStore_SWIFT::get_params()
{
- policy.create_default(s->user.user_id, s->user.display_name);
+ bool has_policy;
+
+ int r = get_swift_container_settings(s, store, &policy, &has_policy, &cors_config, &has_cors);
+ if (r < 0) {
+ return r;
+ }
+
+ if (!has_policy) {
+ policy.create_default(s->user.user_id, s->user.display_name);
+ }
location_constraint = store->region.api_name;
@@ -371,44 +430,9 @@ int RGWPutMetadata_ObjStore_SWIFT::get_params()
return -EINVAL;
if (!s->object) {
- string read_list, write_list;
-
- const char *read_attr = s->info.env->get("HTTP_X_CONTAINER_READ");
- if (read_attr) {
- read_list = read_attr;
- }
- const char *write_attr = s->info.env->get("HTTP_X_CONTAINER_WRITE");
- if (write_attr) {
- write_list = write_attr;
- }
-
- if (read_attr || write_attr) {
- RGWAccessControlPolicy_SWIFT swift_policy(s->cct);
- int r = swift_policy.create(store, s->user.user_id, s->user.display_name, read_list, write_list);
- if (r < 0)
- return r;
-
- policy = swift_policy;
- has_policy = true;
- }
-
- /*Check and update CORS configuration*/
- const char *allow_origins = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_ALLOW_ORIGIN");
- const char *allow_headers = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_ALLOW_HEADERS");
- const char *expose_headers = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_EXPOSE_HEADERS");
- const char *max_age = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_MAX_AGE");
- if (allow_origins) {
- RGWCORSConfiguration_SWIFT *swift_cors = new RGWCORSConfiguration_SWIFT;
- int r = swift_cors->create_update(allow_origins, allow_headers, expose_headers, max_age);
- if (r < 0) {
- dout(0) << "Error creating/updating the cors configuration" << dendl;
- delete swift_cors;
- return r;
- }
- has_cors = true;
- cors_config = *swift_cors;
- cors_config.dump();
- delete swift_cors;
+ int r = get_swift_container_settings(s, store, &policy, &has_policy, &cors_config, &has_cors);
+ if (r < 0) {
+ return r;
}
}
@@ -536,7 +560,7 @@ int RGWGetObj_ObjStore_SWIFT::send_response_data(bufferlist& bl, off_t bl_ofs, o
goto send_data;
if (range_str)
- dump_range(s, ofs, start, s->obj_size);
+ dump_range(s, ofs, end, s->obj_size);
dump_content_length(s, total_len);
dump_last_modified(s, lastmod);
diff --git a/src/rgw/rgw_swift.cc b/src/rgw/rgw_swift.cc
index 1ae8f94..d9654a7 100644
--- a/src/rgw/rgw_swift.cc
+++ b/src/rgw/rgw_swift.cc
@@ -252,7 +252,9 @@ int RGWSwift::get_keystone_admin_token(std::string& token)
KeystoneToken t;
bufferlist token_bl;
RGWGetKeystoneAdminToken token_req(cct, &token_bl);
+ token_req.append_header("Content-Type", "application/json");
JSONFormatter jf;
+ jf.open_object_section("token_request");
jf.open_object_section("auth");
jf.open_object_section("passwordCredentials");
encode_json("username", cct->_conf->rgw_keystone_admin_user, &jf);
@@ -260,10 +262,11 @@ int RGWSwift::get_keystone_admin_token(std::string& token)
jf.close_section();
encode_json("tenantName", cct->_conf->rgw_keystone_admin_tenant, &jf);
jf.close_section();
+ jf.close_section();
std::stringstream ss;
jf.flush(ss);
token_req.set_post_data(ss.str());
- int ret = token_req.process(token_url.c_str());
+ int ret = token_req.process("POST", token_url.c_str());
if (ret < 0)
return ret;
if (t.parse(cct, token_bl) != 0)
diff --git a/src/test/Makefile.am b/src/test/Makefile.am
index 83b83f4..1f07a75 100644
--- a/src/test/Makefile.am
+++ b/src/test/Makefile.am
@@ -1,3 +1,5 @@
+include test/erasure-code/Makefile.am
+
## Unknown/other tests
ceph_test_timers_SOURCES = test/TestTimers.cc
@@ -53,6 +55,9 @@ ceph_dencoder_SOURCES = \
ceph_dencoder_LDADD = \
$(LIBOSD) $(LIBMDS) $(LIBMON) \
$(DENCODER_DEPS) $(CEPH_GLOBAL)
+if WITH_RADOSGW
+ceph_dencoder_LDADD += $(LIBRGW) $(LIBRGW_DEPS)
+endif
# These should always use explicit _CFLAGS/_CXXFLAGS so avoid basename conflicts
ceph_dencoder_CFLAGS = ${AM_CFLAGS}
@@ -175,21 +180,6 @@ ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_kvstorebench
endif
-ceph_multi_stress_watch_SOURCES = \
- test/multi_stress_watch.cc \
- test/librados/test.cc
-ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
-bin_DEBUGPROGRAMS += ceph_multi_stress_watch
-
-ceph_erasure_code_benchmark_SOURCES = \
- test/osd/ceph_erasure_code_benchmark.cc
-ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
-if LINUX
-ceph_erasure_code_benchmark_LDADD += -ldl
-endif
-bin_DEBUGPROGRAMS += ceph_erasure_code_benchmark
-
-
## System tests
if LINUX
@@ -244,11 +234,17 @@ check_SCRIPTS += \
unittest_bufferlist.sh \
test/encoding/check-generated.sh \
test/mon/osd-pool-create.sh \
+ test/mon/misc.sh \
+ test/mon/osd-crush.sh \
test/mon/mkfs.sh \
test/ceph-disk.sh \
test/mon/mon-handle-forward.sh \
test/vstart_wrapped_tests.sh
+EXTRA_DIST += \
+ $(srcdir)/test/mon/mon-test-helpers.sh \
+ $(srcdir)/test/osd/osd-test-helpers.sh
+
# target to build but not run the unit tests
unittests:: $(check_PROGRAMS)
@@ -276,16 +272,16 @@ unittest_bloom_filter_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_bloom_filter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
check_PROGRAMS += unittest_bloom_filter
+unittest_histogram_SOURCES = test/common/histogram.cc
+unittest_histogram_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_histogram_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_histogram
+
unittest_str_map_SOURCES = test/common/test_str_map.cc
unittest_str_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_str_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
check_PROGRAMS += unittest_str_map
-unittest_crushwrapper_SOURCES = test/test_crushwrapper.cc
-unittest_crushwrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_crushwrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
-check_PROGRAMS += unittest_crushwrapper
-
unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc
unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -361,75 +357,7 @@ unittest_ceph_compatset_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_ceph_compatset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_ceph_compatset
-libec_example_la_SOURCES = test/osd/ErasureCodePluginExample.cc
-libec_example_la_CFLAGS = ${AM_CFLAGS}
-libec_example_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_example_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_example.la
-
-libec_missing_entry_point_la_SOURCES = test/osd/ErasureCodePluginMissingEntryPoint.cc
-libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
-libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la
-
-libec_hangs_la_SOURCES = test/osd/ErasureCodePluginHangs.cc
-libec_hangs_la_CFLAGS = ${AM_CFLAGS}
-libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_hangs.la
-
-libec_fail_to_initialize_la_SOURCES = test/osd/ErasureCodePluginFailToInitialize.cc
-libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
-libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la
-
-libec_fail_to_register_la_SOURCES = test/osd/ErasureCodePluginFailToRegister.cc
-libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
-libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_fail_to_register.la
-
-unittest_erasure_code_plugin_SOURCES = test/osd/TestErasureCodePlugin.cc
-unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-if LINUX
-unittest_erasure_code_plugin_LDADD += -ldl
-endif
-check_PROGRAMS += unittest_erasure_code_plugin
-
-unittest_erasure_code_jerasure_SOURCES = \
- test/osd/TestErasureCodeJerasure.cc \
- $(libec_jerasure_la_SOURCES)
-unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-if LINUX
-unittest_erasure_code_jerasure_LDADD += -ldl
-endif
-check_PROGRAMS += unittest_erasure_code_jerasure
-
-unittest_erasure_code_plugin_jerasure_SOURCES = \
- test/osd/TestErasureCodePluginJerasure.cc
-unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
-unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-if LINUX
-unittest_erasure_code_plugin_jerasure_LDADD += -ldl
-endif
-check_PROGRAMS += unittest_erasure_code_plugin_jerasure
-
-unittest_erasure_code_example_SOURCES = test/osd/TestErasureCodeExample.cc
-noinst_HEADERS += test/osd/ErasureCodeExample.h
-unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-check_PROGRAMS += unittest_erasure_code_example
-
-unittest_osd_types_SOURCES = test/test_osd_types.cc
+unittest_osd_types_SOURCES = test/osd/types.cc
unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
check_PROGRAMS += unittest_osd_types
@@ -439,6 +367,11 @@ unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
check_PROGRAMS += unittest_pglog
+unittest_ecbackend_SOURCES = test/osd/TestECBackend.cc
+unittest_ecbackend_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_ecbackend_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_ecbackend
+
unittest_hitset_SOURCES = test/osd/hitset.cc
unittest_hitset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_hitset_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -523,7 +456,7 @@ unittest_escape_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_escape_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_escape
-unittest_chain_xattr_SOURCES = test/filestore/chain_xattr.cc
+unittest_chain_xattr_SOURCES = test/objectstore/chain_xattr.cc
unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_chain_xattr
@@ -548,6 +481,11 @@ unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_config
+unittest_context_SOURCES = test/common/test_context.cc
+unittest_context_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_context_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_context
+
unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc
unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -597,6 +535,11 @@ unittest_mon_moncap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_mon_moncap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_mon_moncap
+unittest_mon_pgmap_SOURCES = test/mon/PGMap.cc
+unittest_mon_pgmap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_mon_pgmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_mon_pgmap
+
#if WITH_RADOSGW
#unittest_librgw_SOURCES = test/librgw.cc
#unittest_librgw_LDFLAGS = -lrt $(PTHREAD_CFLAGS) -lcurl ${AM_LDFLAGS}
@@ -630,6 +573,15 @@ ceph_test_cors_LDADD = \
ceph_test_cors_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cors
+ceph_test_rgw_manifest_SOURCES = test/rgw/test_rgw_manifest.cc
+ceph_test_rgw_manifest_LDADD = \
+ $(LIBRADOS) $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) \
+ $(UNITTEST_LDADD) $(CRYPTO_LIBS) \
+ -lcurl -luuid -lexpat
+
+ceph_test_rgw_manifest_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rgw_manifest
+
ceph_test_cls_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc
ceph_test_cls_rgw_meta_LDADD = \
$(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
@@ -664,10 +616,19 @@ ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_rgw_opstate
endif # WITH_RADOSGW
-ceph_test_librbd_SOURCES = \
- test/librbd/test_librbd.cc \
- test/librados/test.cc
-ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+libradostest_la_SOURCES = \
+ test/librados/test.cc \
+ test/librados/TestCase.cc
+noinst_LTLIBRARIES += libradostest.la
+libradostest_la_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+RADOS_TEST_LDADD = libradostest.la
+
+ceph_multi_stress_watch_SOURCES = test/multi_stress_watch.cc
+ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+bin_DEBUGPROGRAMS += ceph_multi_stress_watch
+
+ceph_test_librbd_SOURCES = test/librbd/test_librbd.cc
+ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_librbd
@@ -678,71 +639,53 @@ ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format
bin_DEBUGPROGRAMS += ceph_test_librbd_fsx
endif
-ceph_test_cls_rbd_SOURCES = \
- test/cls_rbd/test_cls_rbd.cc \
- test/librados/test.cc
-ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD)
+ceph_test_cls_rbd_SOURCES = test/cls_rbd/test_cls_rbd.cc
+ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_rbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_rbd
-ceph_test_cls_refcount_SOURCES = \
- test/cls_refcount/test_cls_refcount.cc \
- test/librados/test.cc
-ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD)
+ceph_test_cls_refcount_SOURCES = test/cls_refcount/test_cls_refcount.cc
+ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_refcount_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_refcount
-ceph_test_cls_version_SOURCES = \
- test/cls_version/test_cls_version.cc \
- test/librados/test.cc
-ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD)
+ceph_test_cls_version_SOURCES = test/cls_version/test_cls_version.cc
+ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_version_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_version
-ceph_test_cls_log_SOURCES = \
- test/cls_log/test_cls_log.cc \
- test/librados/test.cc
-ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_cls_log_SOURCES = test/cls_log/test_cls_log.cc
+ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_log
-ceph_test_cls_statelog_SOURCES = \
- test/cls_statelog/test_cls_statelog.cc \
- test/librados/test.cc
-ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_cls_statelog_SOURCES = test/cls_statelog/test_cls_statelog.cc
+ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_statelog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_statelog
-ceph_test_cls_replica_log_SOURCES = \
- test/cls_replica_log/test_cls_replica_log.cc \
- test/librados/test.cc
+ceph_test_cls_replica_log_SOURCES = test/cls_replica_log/test_cls_replica_log.cc
ceph_test_cls_replica_log_LDADD = \
$(LIBRADOS) libcls_replica_log_client.a \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_replica_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_replica_log
-ceph_test_cls_lock_SOURCES = \
- test/cls_lock/test_cls_lock.cc \
- test/librados/test.cc
-ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD)
+ceph_test_cls_lock_SOURCES = test/cls_lock/test_cls_lock.cc
+ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_lock
-ceph_test_cls_hello_SOURCES = \
- test/cls_hello/test_cls_hello.cc \
- test/librados/test.cc
+ceph_test_cls_hello_SOURCES = test/cls_hello/test_cls_hello.cc
ceph_test_cls_hello_LDADD = \
$(LIBRADOS) $(CRYPTO_LIBS) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_cls_hello_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_hello
if WITH_RADOSGW
-ceph_test_cls_rgw_SOURCES = \
- test/cls_rgw/test_cls_rgw.cc \
- test/librados/test.cc
-ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD)
+ceph_test_cls_rgw_SOURCES = test/cls_rgw/test_cls_rgw.cc
+ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_cls_rgw_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_rgw
endif # WITH_RADOSGW
@@ -751,95 +694,77 @@ ceph_test_mon_workloadgen_SOURCES = test/mon/test_mon_workloadgen.cc
ceph_test_mon_workloadgen_LDADD = $(LIBOS) $(LIBOSDC) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_test_mon_workloadgen
-ceph_test_rados_api_cmd_SOURCES = \
- test/librados/cmd.cc \
- test/librados/test.cc
-ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_cmd_SOURCES = test/librados/cmd.cc
+ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_cmd
-ceph_test_rados_api_io_SOURCES = \
- test/librados/io.cc \
- test/librados/test.cc
-ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_io_SOURCES = test/librados/io.cc
+ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_io
ceph_test_rados_api_c_write_operations_SOURCES = \
- test/librados/c_write_operations.cc \
- test/librados/test.cc
-ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ test/librados/c_write_operations.cc
+ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_c_write_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_c_write_operations
-ceph_test_rados_api_aio_SOURCES = \
- test/librados/aio.cc \
- test/librados/test.cc
-ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_c_read_operations_SOURCES = \
+ test/librados/c_read_operations.cc
+ceph_test_rados_api_c_read_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_c_read_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_c_read_operations
+
+ceph_test_rados_api_aio_SOURCES = test/librados/aio.cc
+ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_aio_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_aio
-ceph_test_rados_api_list_SOURCES = \
- test/librados/list.cc \
- test/librados/test.cc
-ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_list_SOURCES = test/librados/list.cc
+ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_list_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_list
-ceph_test_rados_api_pool_SOURCES = \
- test/librados/pool.cc \
- test/librados/test.cc
-ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_pool_SOURCES = test/librados/pool.cc
+ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_pool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_pool
-ceph_test_rados_api_stat_SOURCES = \
- test/librados/stat.cc \
- test/librados/test.cc
-ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_stat_SOURCES = test/librados/stat.cc
+ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_stat_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_stat
-ceph_test_rados_api_watch_notify_SOURCES = \
- test/librados/watch_notify.cc \
- test/librados/test.cc
-ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_watch_notify_SOURCES = test/librados/watch_notify.cc
+ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_watch_notify_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_watch_notify
-ceph_test_rados_api_snapshots_SOURCES = \
- test/librados/snapshots.cc \
- test/librados/test.cc
-ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_snapshots_SOURCES = test/librados/snapshots.cc
+ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_snapshots_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_snapshots
-ceph_test_rados_api_cls_SOURCES = \
- test/librados/cls.cc \
- test/librados/test.cc
-ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_cls_SOURCES = test/librados/cls.cc
+ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_cls_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_cls
-ceph_test_rados_api_misc_SOURCES = \
- test/librados/misc.cc \
- test/librados/test.cc
-ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_rados_api_misc_SOURCES = test/librados/misc.cc
+ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_misc
ceph_test_rados_api_tier_SOURCES = \
test/librados/tier.cc \
- test/librados/test.cc \
osd/HitSet.cc
-ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
ceph_test_rados_api_tier_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_tier
-ceph_test_rados_api_lock_SOURCES = \
- test/librados/lock.cc \
- test/librados/test.cc
-ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_lock_SOURCES = test/librados/lock.cc
+ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_rados_api_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_lock
@@ -853,30 +778,30 @@ ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_libcephfs
if LINUX
-ceph_test_filestore_SOURCES = test/filestore/store_test.cc
-ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-bin_DEBUGPROGRAMS += ceph_test_filestore
+ceph_test_objectstore_SOURCES = test/objectstore/store_test.cc
+ceph_test_objectstore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_objectstore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_objectstore
endif
-ceph_test_filestore_workloadgen_SOURCES = \
- test/filestore/workload_generator.cc \
- test/filestore/TestFileStoreState.cc
-ceph_test_filestore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
-bin_DEBUGPROGRAMS += ceph_test_filestore_workloadgen
+ceph_test_objectstore_workloadgen_SOURCES = \
+ test/objectstore/workload_generator.cc \
+ test/objectstore/TestObjectStoreState.cc
+ceph_test_objectstore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_objectstore_workloadgen
ceph_test_filestore_idempotent_SOURCES = \
- test/filestore/test_idempotent.cc \
- test/filestore/FileStoreTracker.cc \
+ test/objectstore/test_idempotent.cc \
+ test/objectstore/FileStoreTracker.cc \
test/common/ObjectContents.cc
ceph_test_filestore_idempotent_LDADD = $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent
ceph_test_filestore_idempotent_sequence_SOURCES = \
- test/filestore/test_idempotent_sequence.cc \
- test/filestore/DeterministicOpSequence.cc \
- test/filestore/TestFileStoreState.cc \
- test/filestore/FileStoreDiff.cc
+ test/objectstore/test_idempotent_sequence.cc \
+ test/objectstore/DeterministicOpSequence.cc \
+ test/objectstore/TestObjectStoreState.cc \
+ test/objectstore/FileStoreDiff.cc
ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent_sequence
@@ -890,10 +815,8 @@ ceph_test_filejournal_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_filejournal_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_filejournal
-ceph_test_stress_watch_SOURCES = \
- test/test_stress_watch.cc \
- test/librados/test.cc
-ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_stress_watch_SOURCES = test/test_stress_watch.cc
+ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
ceph_test_stress_watch_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_stress_watch
@@ -950,16 +873,16 @@ noinst_HEADERS += \
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h \
test/encoding/types.h \
- test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreDiff.h \
- test/filestore/FileStoreTracker.h \
- test/filestore/TestFileStoreState.h \
- test/filestore/workload_generator.h \
+ test/objectstore/DeterministicOpSequence.h \
+ test/objectstore/FileStoreDiff.h \
+ test/objectstore/FileStoreTracker.h \
+ test/objectstore/TestObjectStoreState.h \
+ test/objectstore/workload_generator.h \
test/kv_store_bench.h \
test/librados/test.h \
+ test/librados/TestCase.h \
test/ObjectMap/KeyValueDBMemory.h \
test/omap_bench.h \
- test/osd/ceph_erasure_code_benchmark.h \
test/osdc/FakeWriteback.h \
test/osd/Object.h \
test/osd/RadosModel.h \
diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc
index 78384a8..8d58dfd 100644
--- a/src/test/admin_socket.cc
+++ b/src/test/admin_socket.cc
@@ -63,6 +63,38 @@ TEST(AdminSocket, TeardownSetup) {
ASSERT_EQ(true, asoct.shutdown());
}
+TEST(AdminSocket, SendHelp) {
+ std::auto_ptr<AdminSocket>
+ asokc(new AdminSocket(g_ceph_context));
+ AdminSocketTest asoct(asokc.get());
+ ASSERT_EQ(true, asoct.shutdown());
+ ASSERT_EQ(true, asoct.init(get_rand_socket_path()));
+ AdminSocketClient client(get_rand_socket_path());
+
+ {
+ string help;
+ ASSERT_EQ("", client.do_request("{\"prefix\":\"help\"}", &help));
+ ASSERT_NE(string::npos, help.find("\"list available commands\""));
+ }
+ {
+ string help;
+ ASSERT_EQ("", client.do_request("{"
+ " \"prefix\":\"help\","
+ " \"format\":\"xml\","
+ "}", &help));
+ ASSERT_NE(string::npos, help.find(">list available commands<"));
+ }
+ {
+ string help;
+ ASSERT_EQ("", client.do_request("{"
+ " \"prefix\":\"help\","
+ " \"format\":\"UNSUPPORTED\","
+ "}", &help));
+ ASSERT_NE(string::npos, help.find("\"list available commands\""));
+ }
+ ASSERT_EQ(true, asoct.shutdown());
+}
+
TEST(AdminSocket, SendNoOp) {
std::auto_ptr<AdminSocket>
asokc(new AdminSocket(g_ceph_context));
diff --git a/src/test/bufferlist.cc b/src/test/bufferlist.cc
index 64253c9..ff87238 100644
--- a/src/test/bufferlist.cc
+++ b/src/test/bufferlist.cc
@@ -306,7 +306,7 @@ TEST_F(TestRawPipe, buffer_list_write_fd_zero_copy) {
EXPECT_EQ(0, ::stat("testfile_out", &st));
EXPECT_EQ(len, st.st_size);
char buf[len + 1];
- EXPECT_EQ(len, safe_read(out_fd, buf, len + 1));
+ EXPECT_EQ((int)len, safe_read(out_fd, buf, len + 1));
EXPECT_EQ(0, memcmp(buf, "ABC\n", len));
::close(out_fd);
::unlink("testfile_out");
diff --git a/src/test/ceph_argparse.cc b/src/test/ceph_argparse.cc
index e4b5d6b..d1a790f 100644
--- a/src/test/ceph_argparse.cc
+++ b/src/test/ceph_argparse.cc
@@ -334,3 +334,112 @@ TEST(CephArgParse, WithInt) {
ASSERT_EQ(foo, 40);
ASSERT_EQ(bar, -1);
}
+
+TEST(CephArgParse, env_to_vec) {
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ unsetenv("WHATEVER");
+ env_to_vec(args);
+ EXPECT_EQ(0u, args.size());
+ env_to_vec(args, "WHATEVER");
+ EXPECT_EQ(0u, args.size());
+ args.push_back("a");
+ setenv("CEPH_ARGS", "b c", 0);
+ env_to_vec(args);
+ EXPECT_EQ(3u, args.size());
+ EXPECT_EQ(string("b"), args[1]);
+ EXPECT_EQ(string("c"), args[2]);
+ setenv("WHATEVER", "d e", 0);
+ env_to_vec(args, "WHATEVER");
+ EXPECT_EQ(5u, args.size());
+ EXPECT_EQ(string("d"), args[3]);
+ EXPECT_EQ(string("e"), args[4]);
+ }
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ args.push_back("a");
+ args.push_back("--");
+ args.push_back("c");
+ setenv("CEPH_ARGS", "b -- d", 0);
+ env_to_vec(args);
+ EXPECT_EQ(5u, args.size());
+ EXPECT_EQ(string("a"), args[0]);
+ EXPECT_EQ(string("b"), args[1]);
+ EXPECT_EQ(string("--"), args[2]);
+ EXPECT_EQ(string("c"), args[3]);
+ EXPECT_EQ(string("d"), args[4]);
+ }
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ args.push_back("a");
+ args.push_back("--");
+ setenv("CEPH_ARGS", "b -- c", 0);
+ env_to_vec(args);
+ EXPECT_EQ(4u, args.size());
+ EXPECT_EQ(string("a"), args[0]);
+ EXPECT_EQ(string("b"), args[1]);
+ EXPECT_EQ(string("--"), args[2]);
+ EXPECT_EQ(string("c"), args[3]);
+ }
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ args.push_back("--");
+ args.push_back("c");
+ setenv("CEPH_ARGS", "b -- d", 0);
+ env_to_vec(args);
+ EXPECT_EQ(4u, args.size());
+ EXPECT_EQ(string("b"), args[0]);
+ EXPECT_EQ(string("--"), args[1]);
+ EXPECT_EQ(string("c"), args[2]);
+ EXPECT_EQ(string("d"), args[3]);
+ }
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ args.push_back("b");
+ setenv("CEPH_ARGS", "c -- d", 0);
+ env_to_vec(args);
+ EXPECT_EQ(4u, args.size());
+ EXPECT_EQ(string("b"), args[0]);
+ EXPECT_EQ(string("c"), args[1]);
+ EXPECT_EQ(string("--"), args[2]);
+ EXPECT_EQ(string("d"), args[3]);
+ }
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ args.push_back("a");
+ args.push_back("--");
+ args.push_back("c");
+ setenv("CEPH_ARGS", "-- d", 0);
+ env_to_vec(args);
+ EXPECT_EQ(4u, args.size());
+ EXPECT_EQ(string("a"), args[0]);
+ EXPECT_EQ(string("--"), args[1]);
+ EXPECT_EQ(string("c"), args[2]);
+ EXPECT_EQ(string("d"), args[3]);
+ }
+ {
+ std::vector<const char*> args;
+ unsetenv("CEPH_ARGS");
+ args.push_back("a");
+ args.push_back("--");
+ args.push_back("c");
+ setenv("CEPH_ARGS", "d", 0);
+ env_to_vec(args);
+ EXPECT_EQ(4u, args.size());
+ EXPECT_EQ(string("a"), args[0]);
+ EXPECT_EQ(string("d"), args[1]);
+ EXPECT_EQ(string("--"), args[2]);
+ EXPECT_EQ(string("c"), args[3]);
+ }
+}
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make unittest_ceph_argparse && ./unittest_ceph_argparse"
+ * End:
+ */
diff --git a/src/test/cli/crushtool/help.t b/src/test/cli/crushtool/help.t
index 18617a3..3b48930 100644
--- a/src/test/cli/crushtool/help.t
+++ b/src/test/cli/crushtool/help.t
@@ -44,6 +44,8 @@
set choose total descent attempts
--set-chooseleaf-descend-once <0|1>
set chooseleaf to (not) retry the recursive descent
+ --set-chooseleaf-vary-r <0|1>
+ set chooseleaf to (not) vary r based on parent
--output-name name
prepend the data file(s) generated during the
testing routine with name
diff --git a/src/test/cli/crushtool/test-map-firefly-tunables.t b/src/test/cli/crushtool/test-map-firefly-tunables.t
new file mode 100644
index 0000000..481b6fd
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-firefly-tunables.t
@@ -0,0 +1,10259 @@
+ $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 0 --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 --set-chooseleaf-descend-once 1 --set-chooseleaf-vary-r 1 --weight 12 0 --weight 20 0 --weight 30 0
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 0 (data), x = 0..1023, numrep = 1..10
+ CRUSH rule 0 x 0 [101]
+ CRUSH rule 0 x 1 [80]
+ CRUSH rule 0 x 2 [91]
+ CRUSH rule 0 x 3 [51]
+ CRUSH rule 0 x 4 [50]
+ CRUSH rule 0 x 5 [89]
+ CRUSH rule 0 x 6 [91]
+ CRUSH rule 0 x 7 [104]
+ CRUSH rule 0 x 8 [78]
+ CRUSH rule 0 x 9 [101]
+ CRUSH rule 0 x 10 [61]
+ CRUSH rule 0 x 11 [13]
+ CRUSH rule 0 x 12 [83]
+ CRUSH rule 0 x 13 [108]
+ CRUSH rule 0 x 14 [105]
+ CRUSH rule 0 x 15 [18]
+ CRUSH rule 0 x 16 [103]
+ CRUSH rule 0 x 17 [85]
+ CRUSH rule 0 x 18 [11]
+ CRUSH rule 0 x 19 [75]
+ CRUSH rule 0 x 20 [79]
+ CRUSH rule 0 x 21 [84]
+ CRUSH rule 0 x 22 [23]
+ CRUSH rule 0 x 23 [118]
+ CRUSH rule 0 x 24 [83]
+ CRUSH rule 0 x 25 [81]
+ CRUSH rule 0 x 26 [38]
+ CRUSH rule 0 x 27 [76]
+ CRUSH rule 0 x 28 [76]
+ CRUSH rule 0 x 29 [24]
+ CRUSH rule 0 x 30 [94]
+ CRUSH rule 0 x 31 [76]
+ CRUSH rule 0 x 32 [72]
+ CRUSH rule 0 x 33 [77]
+ CRUSH rule 0 x 34 [7]
+ CRUSH rule 0 x 35 [22]
+ CRUSH rule 0 x 36 [104]
+ CRUSH rule 0 x 37 [61]
+ CRUSH rule 0 x 38 [72]
+ CRUSH rule 0 x 39 [68]
+ CRUSH rule 0 x 40 [103]
+ CRUSH rule 0 x 41 [85]
+ CRUSH rule 0 x 42 [106]
+ CRUSH rule 0 x 43 [10]
+ CRUSH rule 0 x 44 [101]
+ CRUSH rule 0 x 45 [83]
+ CRUSH rule 0 x 46 [65]
+ CRUSH rule 0 x 47 [106]
+ CRUSH rule 0 x 48 [34]
+ CRUSH rule 0 x 49 [0]
+ CRUSH rule 0 x 50 [42]
+ CRUSH rule 0 x 51 [104]
+ CRUSH rule 0 x 52 [83]
+ CRUSH rule 0 x 53 [32]
+ CRUSH rule 0 x 54 [9]
+ CRUSH rule 0 x 55 [14]
+ CRUSH rule 0 x 56 [21]
+ CRUSH rule 0 x 57 [93]
+ CRUSH rule 0 x 58 [45]
+ CRUSH rule 0 x 59 [80]
+ CRUSH rule 0 x 60 [90]
+ CRUSH rule 0 x 61 [88]
+ CRUSH rule 0 x 62 [81]
+ CRUSH rule 0 x 63 [79]
+ CRUSH rule 0 x 64 [1]
+ CRUSH rule 0 x 65 [32]
+ CRUSH rule 0 x 66 [48]
+ CRUSH rule 0 x 67 [94]
+ CRUSH rule 0 x 68 [102]
+ CRUSH rule 0 x 69 [62]
+ CRUSH rule 0 x 70 [84]
+ CRUSH rule 0 x 71 [9]
+ CRUSH rule 0 x 72 [97]
+ CRUSH rule 0 x 73 [64]
+ CRUSH rule 0 x 74 [29]
+ CRUSH rule 0 x 75 [29]
+ CRUSH rule 0 x 76 [55]
+ CRUSH rule 0 x 77 [107]
+ CRUSH rule 0 x 78 [11]
+ CRUSH rule 0 x 79 [64]
+ CRUSH rule 0 x 80 [0]
+ CRUSH rule 0 x 81 [71]
+ CRUSH rule 0 x 82 [37]
+ CRUSH rule 0 x 83 [92]
+ CRUSH rule 0 x 84 [49]
+ CRUSH rule 0 x 85 [54]
+ CRUSH rule 0 x 86 [37]
+ CRUSH rule 0 x 87 [116]
+ CRUSH rule 0 x 88 [38]
+ CRUSH rule 0 x 89 [76]
+ CRUSH rule 0 x 90 [14]
+ CRUSH rule 0 x 91 [68]
+ CRUSH rule 0 x 92 [86]
+ CRUSH rule 0 x 93 [44]
+ CRUSH rule 0 x 94 [61]
+ CRUSH rule 0 x 95 [93]
+ CRUSH rule 0 x 96 [66]
+ CRUSH rule 0 x 97 [111]
+ CRUSH rule 0 x 98 [93]
+ CRUSH rule 0 x 99 [78]
+ CRUSH rule 0 x 100 [6]
+ CRUSH rule 0 x 101 [84]
+ CRUSH rule 0 x 102 [82]
+ CRUSH rule 0 x 103 [66]
+ CRUSH rule 0 x 104 [14]
+ CRUSH rule 0 x 105 [87]
+ CRUSH rule 0 x 106 [69]
+ CRUSH rule 0 x 107 [1]
+ CRUSH rule 0 x 108 [94]
+ CRUSH rule 0 x 109 [112]
+ CRUSH rule 0 x 110 [54]
+ CRUSH rule 0 x 111 [10]
+ CRUSH rule 0 x 112 [89]
+ CRUSH rule 0 x 113 [69]
+ CRUSH rule 0 x 114 [79]
+ CRUSH rule 0 x 115 [50]
+ CRUSH rule 0 x 116 [96]
+ CRUSH rule 0 x 117 [87]
+ CRUSH rule 0 x 118 [23]
+ CRUSH rule 0 x 119 [104]
+ CRUSH rule 0 x 120 [57]
+ CRUSH rule 0 x 121 [105]
+ CRUSH rule 0 x 122 [45]
+ CRUSH rule 0 x 123 [112]
+ CRUSH rule 0 x 124 [110]
+ CRUSH rule 0 x 125 [66]
+ CRUSH rule 0 x 126 [51]
+ CRUSH rule 0 x 127 [70]
+ CRUSH rule 0 x 128 [90]
+ CRUSH rule 0 x 129 [103]
+ CRUSH rule 0 x 130 [50]
+ CRUSH rule 0 x 131 [23]
+ CRUSH rule 0 x 132 [69]
+ CRUSH rule 0 x 133 [52]
+ CRUSH rule 0 x 134 [78]
+ CRUSH rule 0 x 135 [78]
+ CRUSH rule 0 x 136 [32]
+ CRUSH rule 0 x 137 [11]
+ CRUSH rule 0 x 138 [17]
+ CRUSH rule 0 x 139 [89]
+ CRUSH rule 0 x 140 [39]
+ CRUSH rule 0 x 141 [89]
+ CRUSH rule 0 x 142 [70]
+ CRUSH rule 0 x 143 [51]
+ CRUSH rule 0 x 144 [13]
+ CRUSH rule 0 x 145 [77]
+ CRUSH rule 0 x 146 [8]
+ CRUSH rule 0 x 147 [22]
+ CRUSH rule 0 x 148 [74]
+ CRUSH rule 0 x 149 [76]
+ CRUSH rule 0 x 150 [14]
+ CRUSH rule 0 x 151 [90]
+ CRUSH rule 0 x 152 [49]
+ CRUSH rule 0 x 153 [71]
+ CRUSH rule 0 x 154 [94]
+ CRUSH rule 0 x 155 [75]
+ CRUSH rule 0 x 156 [94]
+ CRUSH rule 0 x 157 [112]
+ CRUSH rule 0 x 158 [26]
+ CRUSH rule 0 x 159 [52]
+ CRUSH rule 0 x 160 [41]
+ CRUSH rule 0 x 161 [19]
+ CRUSH rule 0 x 162 [55]
+ CRUSH rule 0 x 163 [54]
+ CRUSH rule 0 x 164 [45]
+ CRUSH rule 0 x 165 [25]
+ CRUSH rule 0 x 166 [73]
+ CRUSH rule 0 x 167 [89]
+ CRUSH rule 0 x 168 [47]
+ CRUSH rule 0 x 169 [51]
+ CRUSH rule 0 x 170 [68]
+ CRUSH rule 0 x 171 [73]
+ CRUSH rule 0 x 172 [33]
+ CRUSH rule 0 x 173 [102]
+ CRUSH rule 0 x 174 [116]
+ CRUSH rule 0 x 175 [3]
+ CRUSH rule 0 x 176 [94]
+ CRUSH rule 0 x 177 [52]
+ CRUSH rule 0 x 178 [39]
+ CRUSH rule 0 x 179 [72]
+ CRUSH rule 0 x 180 [60]
+ CRUSH rule 0 x 181 [18]
+ CRUSH rule 0 x 182 [22]
+ CRUSH rule 0 x 183 [11]
+ CRUSH rule 0 x 184 [92]
+ CRUSH rule 0 x 185 [97]
+ CRUSH rule 0 x 186 [67]
+ CRUSH rule 0 x 187 [116]
+ CRUSH rule 0 x 188 [69]
+ CRUSH rule 0 x 189 [47]
+ CRUSH rule 0 x 190 [90]
+ CRUSH rule 0 x 191 [49]
+ CRUSH rule 0 x 192 [68]
+ CRUSH rule 0 x 193 [0]
+ CRUSH rule 0 x 194 [17]
+ CRUSH rule 0 x 195 [119]
+ CRUSH rule 0 x 196 [72]
+ CRUSH rule 0 x 197 [106]
+ CRUSH rule 0 x 198 [114]
+ CRUSH rule 0 x 199 [0]
+ CRUSH rule 0 x 200 [35]
+ CRUSH rule 0 x 201 [14]
+ CRUSH rule 0 x 202 [98]
+ CRUSH rule 0 x 203 [36]
+ CRUSH rule 0 x 204 [10]
+ CRUSH rule 0 x 205 [22]
+ CRUSH rule 0 x 206 [49]
+ CRUSH rule 0 x 207 [80]
+ CRUSH rule 0 x 208 [63]
+ CRUSH rule 0 x 209 [85]
+ CRUSH rule 0 x 210 [79]
+ CRUSH rule 0 x 211 [26]
+ CRUSH rule 0 x 212 [28]
+ CRUSH rule 0 x 213 [91]
+ CRUSH rule 0 x 214 [78]
+ CRUSH rule 0 x 215 [61]
+ CRUSH rule 0 x 216 [99]
+ CRUSH rule 0 x 217 [86]
+ CRUSH rule 0 x 218 [93]
+ CRUSH rule 0 x 219 [28]
+ CRUSH rule 0 x 220 [56]
+ CRUSH rule 0 x 221 [0]
+ CRUSH rule 0 x 222 [50]
+ CRUSH rule 0 x 223 [29]
+ CRUSH rule 0 x 224 [52]
+ CRUSH rule 0 x 225 [61]
+ CRUSH rule 0 x 226 [44]
+ CRUSH rule 0 x 227 [42]
+ CRUSH rule 0 x 228 [117]
+ CRUSH rule 0 x 229 [100]
+ CRUSH rule 0 x 230 [41]
+ CRUSH rule 0 x 231 [56]
+ CRUSH rule 0 x 232 [23]
+ CRUSH rule 0 x 233 [88]
+ CRUSH rule 0 x 234 [4]
+ CRUSH rule 0 x 235 [26]
+ CRUSH rule 0 x 236 [32]
+ CRUSH rule 0 x 237 [92]
+ CRUSH rule 0 x 238 [10]
+ CRUSH rule 0 x 239 [15]
+ CRUSH rule 0 x 240 [109]
+ CRUSH rule 0 x 241 [47]
+ CRUSH rule 0 x 242 [24]
+ CRUSH rule 0 x 243 [76]
+ CRUSH rule 0 x 244 [96]
+ CRUSH rule 0 x 245 [27]
+ CRUSH rule 0 x 246 [35]
+ CRUSH rule 0 x 247 [99]
+ CRUSH rule 0 x 248 [8]
+ CRUSH rule 0 x 249 [85]
+ CRUSH rule 0 x 250 [79]
+ CRUSH rule 0 x 251 [28]
+ CRUSH rule 0 x 252 [95]
+ CRUSH rule 0 x 253 [109]
+ CRUSH rule 0 x 254 [80]
+ CRUSH rule 0 x 255 [112]
+ CRUSH rule 0 x 256 [37]
+ CRUSH rule 0 x 257 [69]
+ CRUSH rule 0 x 258 [34]
+ CRUSH rule 0 x 259 [70]
+ CRUSH rule 0 x 260 [98]
+ CRUSH rule 0 x 261 [94]
+ CRUSH rule 0 x 262 [42]
+ CRUSH rule 0 x 263 [65]
+ CRUSH rule 0 x 264 [36]
+ CRUSH rule 0 x 265 [66]
+ CRUSH rule 0 x 266 [75]
+ CRUSH rule 0 x 267 [58]
+ CRUSH rule 0 x 268 [38]
+ CRUSH rule 0 x 269 [43]
+ CRUSH rule 0 x 270 [58]
+ CRUSH rule 0 x 271 [19]
+ CRUSH rule 0 x 272 [73]
+ CRUSH rule 0 x 273 [108]
+ CRUSH rule 0 x 274 [47]
+ CRUSH rule 0 x 275 [92]
+ CRUSH rule 0 x 276 [7]
+ CRUSH rule 0 x 277 [19]
+ CRUSH rule 0 x 278 [116]
+ CRUSH rule 0 x 279 [101]
+ CRUSH rule 0 x 280 [113]
+ CRUSH rule 0 x 281 [14]
+ CRUSH rule 0 x 282 [106]
+ CRUSH rule 0 x 283 [8]
+ CRUSH rule 0 x 284 [10]
+ CRUSH rule 0 x 285 [88]
+ CRUSH rule 0 x 286 [27]
+ CRUSH rule 0 x 287 [84]
+ CRUSH rule 0 x 288 [103]
+ CRUSH rule 0 x 289 [9]
+ CRUSH rule 0 x 290 [115]
+ CRUSH rule 0 x 291 [48]
+ CRUSH rule 0 x 292 [52]
+ CRUSH rule 0 x 293 [27]
+ CRUSH rule 0 x 294 [79]
+ CRUSH rule 0 x 295 [37]
+ CRUSH rule 0 x 296 [56]
+ CRUSH rule 0 x 297 [35]
+ CRUSH rule 0 x 298 [71]
+ CRUSH rule 0 x 299 [79]
+ CRUSH rule 0 x 300 [67]
+ CRUSH rule 0 x 301 [51]
+ CRUSH rule 0 x 302 [78]
+ CRUSH rule 0 x 303 [19]
+ CRUSH rule 0 x 304 [101]
+ CRUSH rule 0 x 305 [81]
+ CRUSH rule 0 x 306 [0]
+ CRUSH rule 0 x 307 [44]
+ CRUSH rule 0 x 308 [91]
+ CRUSH rule 0 x 309 [15]
+ CRUSH rule 0 x 310 [26]
+ CRUSH rule 0 x 311 [36]
+ CRUSH rule 0 x 312 [33]
+ CRUSH rule 0 x 313 [104]
+ CRUSH rule 0 x 314 [28]
+ CRUSH rule 0 x 315 [16]
+ CRUSH rule 0 x 316 [4]
+ CRUSH rule 0 x 317 [118]
+ CRUSH rule 0 x 318 [32]
+ CRUSH rule 0 x 319 [24]
+ CRUSH rule 0 x 320 [36]
+ CRUSH rule 0 x 321 [26]
+ CRUSH rule 0 x 322 [87]
+ CRUSH rule 0 x 323 [73]
+ CRUSH rule 0 x 324 [64]
+ CRUSH rule 0 x 325 [52]
+ CRUSH rule 0 x 326 [111]
+ CRUSH rule 0 x 327 [62]
+ CRUSH rule 0 x 328 [7]
+ CRUSH rule 0 x 329 [93]
+ CRUSH rule 0 x 330 [24]
+ CRUSH rule 0 x 331 [41]
+ CRUSH rule 0 x 332 [61]
+ CRUSH rule 0 x 333 [16]
+ CRUSH rule 0 x 334 [94]
+ CRUSH rule 0 x 335 [71]
+ CRUSH rule 0 x 336 [16]
+ CRUSH rule 0 x 337 [37]
+ CRUSH rule 0 x 338 [109]
+ CRUSH rule 0 x 339 [13]
+ CRUSH rule 0 x 340 [119]
+ CRUSH rule 0 x 341 [63]
+ CRUSH rule 0 x 342 [92]
+ CRUSH rule 0 x 343 [49]
+ CRUSH rule 0 x 344 [103]
+ CRUSH rule 0 x 345 [56]
+ CRUSH rule 0 x 346 [3]
+ CRUSH rule 0 x 347 [106]
+ CRUSH rule 0 x 348 [10]
+ CRUSH rule 0 x 349 [96]
+ CRUSH rule 0 x 350 [63]
+ CRUSH rule 0 x 351 [60]
+ CRUSH rule 0 x 352 [103]
+ CRUSH rule 0 x 353 [10]
+ CRUSH rule 0 x 354 [55]
+ CRUSH rule 0 x 355 [73]
+ CRUSH rule 0 x 356 [114]
+ CRUSH rule 0 x 357 [70]
+ CRUSH rule 0 x 358 [97]
+ CRUSH rule 0 x 359 [4]
+ CRUSH rule 0 x 360 [106]
+ CRUSH rule 0 x 361 [27]
+ CRUSH rule 0 x 362 [28]
+ CRUSH rule 0 x 363 [45]
+ CRUSH rule 0 x 364 [23]
+ CRUSH rule 0 x 365 [57]
+ CRUSH rule 0 x 366 [14]
+ CRUSH rule 0 x 367 [108]
+ CRUSH rule 0 x 368 [103]
+ CRUSH rule 0 x 369 [11]
+ CRUSH rule 0 x 370 [11]
+ CRUSH rule 0 x 371 [34]
+ CRUSH rule 0 x 372 [58]
+ CRUSH rule 0 x 373 [6]
+ CRUSH rule 0 x 374 [110]
+ CRUSH rule 0 x 375 [19]
+ CRUSH rule 0 x 376 [22]
+ CRUSH rule 0 x 377 [93]
+ CRUSH rule 0 x 378 [67]
+ CRUSH rule 0 x 379 [77]
+ CRUSH rule 0 x 380 [3]
+ CRUSH rule 0 x 381 [55]
+ CRUSH rule 0 x 382 [26]
+ CRUSH rule 0 x 383 [48]
+ CRUSH rule 0 x 384 [15]
+ CRUSH rule 0 x 385 [82]
+ CRUSH rule 0 x 386 [108]
+ CRUSH rule 0 x 387 [70]
+ CRUSH rule 0 x 388 [5]
+ CRUSH rule 0 x 389 [14]
+ CRUSH rule 0 x 390 [68]
+ CRUSH rule 0 x 391 [113]
+ CRUSH rule 0 x 392 [72]
+ CRUSH rule 0 x 393 [115]
+ CRUSH rule 0 x 394 [38]
+ CRUSH rule 0 x 395 [0]
+ CRUSH rule 0 x 396 [59]
+ CRUSH rule 0 x 397 [87]
+ CRUSH rule 0 x 398 [44]
+ CRUSH rule 0 x 399 [9]
+ CRUSH rule 0 x 400 [19]
+ CRUSH rule 0 x 401 [79]
+ CRUSH rule 0 x 402 [107]
+ CRUSH rule 0 x 403 [23]
+ CRUSH rule 0 x 404 [76]
+ CRUSH rule 0 x 405 [10]
+ CRUSH rule 0 x 406 [38]
+ CRUSH rule 0 x 407 [70]
+ CRUSH rule 0 x 408 [55]
+ CRUSH rule 0 x 409 [102]
+ CRUSH rule 0 x 410 [59]
+ CRUSH rule 0 x 411 [34]
+ CRUSH rule 0 x 412 [108]
+ CRUSH rule 0 x 413 [54]
+ CRUSH rule 0 x 414 [70]
+ CRUSH rule 0 x 415 [107]
+ CRUSH rule 0 x 416 [21]
+ CRUSH rule 0 x 417 [8]
+ CRUSH rule 0 x 418 [51]
+ CRUSH rule 0 x 419 [8]
+ CRUSH rule 0 x 420 [109]
+ CRUSH rule 0 x 421 [114]
+ CRUSH rule 0 x 422 [109]
+ CRUSH rule 0 x 423 [59]
+ CRUSH rule 0 x 424 [71]
+ CRUSH rule 0 x 425 [101]
+ CRUSH rule 0 x 426 [47]
+ CRUSH rule 0 x 427 [8]
+ CRUSH rule 0 x 428 [68]
+ CRUSH rule 0 x 429 [76]
+ CRUSH rule 0 x 430 [69]
+ CRUSH rule 0 x 431 [70]
+ CRUSH rule 0 x 432 [46]
+ CRUSH rule 0 x 433 [6]
+ CRUSH rule 0 x 434 [64]
+ CRUSH rule 0 x 435 [16]
+ CRUSH rule 0 x 436 [89]
+ CRUSH rule 0 x 437 [29]
+ CRUSH rule 0 x 438 [105]
+ CRUSH rule 0 x 439 [29]
+ CRUSH rule 0 x 440 [38]
+ CRUSH rule 0 x 441 [112]
+ CRUSH rule 0 x 442 [55]
+ CRUSH rule 0 x 443 [44]
+ CRUSH rule 0 x 444 [72]
+ CRUSH rule 0 x 445 [19]
+ CRUSH rule 0 x 446 [40]
+ CRUSH rule 0 x 447 [13]
+ CRUSH rule 0 x 448 [7]
+ CRUSH rule 0 x 449 [67]
+ CRUSH rule 0 x 450 [117]
+ CRUSH rule 0 x 451 [93]
+ CRUSH rule 0 x 452 [70]
+ CRUSH rule 0 x 453 [82]
+ CRUSH rule 0 x 454 [53]
+ CRUSH rule 0 x 455 [91]
+ CRUSH rule 0 x 456 [101]
+ CRUSH rule 0 x 457 [113]
+ CRUSH rule 0 x 458 [53]
+ CRUSH rule 0 x 459 [25]
+ CRUSH rule 0 x 460 [105]
+ CRUSH rule 0 x 461 [102]
+ CRUSH rule 0 x 462 [98]
+ CRUSH rule 0 x 463 [108]
+ CRUSH rule 0 x 464 [19]
+ CRUSH rule 0 x 465 [29]
+ CRUSH rule 0 x 466 [66]
+ CRUSH rule 0 x 467 [6]
+ CRUSH rule 0 x 468 [97]
+ CRUSH rule 0 x 469 [98]
+ CRUSH rule 0 x 470 [50]
+ CRUSH rule 0 x 471 [40]
+ CRUSH rule 0 x 472 [74]
+ CRUSH rule 0 x 473 [95]
+ CRUSH rule 0 x 474 [51]
+ CRUSH rule 0 x 475 [49]
+ CRUSH rule 0 x 476 [110]
+ CRUSH rule 0 x 477 [25]
+ CRUSH rule 0 x 478 [47]
+ CRUSH rule 0 x 479 [70]
+ CRUSH rule 0 x 480 [62]
+ CRUSH rule 0 x 481 [26]
+ CRUSH rule 0 x 482 [84]
+ CRUSH rule 0 x 483 [15]
+ CRUSH rule 0 x 484 [37]
+ CRUSH rule 0 x 485 [47]
+ CRUSH rule 0 x 486 [92]
+ CRUSH rule 0 x 487 [106]
+ CRUSH rule 0 x 488 [42]
+ CRUSH rule 0 x 489 [76]
+ CRUSH rule 0 x 490 [68]
+ CRUSH rule 0 x 491 [80]
+ CRUSH rule 0 x 492 [21]
+ CRUSH rule 0 x 493 [99]
+ CRUSH rule 0 x 494 [4]
+ CRUSH rule 0 x 495 [40]
+ CRUSH rule 0 x 496 [93]
+ CRUSH rule 0 x 497 [102]
+ CRUSH rule 0 x 498 [68]
+ CRUSH rule 0 x 499 [10]
+ CRUSH rule 0 x 500 [50]
+ CRUSH rule 0 x 501 [60]
+ CRUSH rule 0 x 502 [11]
+ CRUSH rule 0 x 503 [117]
+ CRUSH rule 0 x 504 [90]
+ CRUSH rule 0 x 505 [91]
+ CRUSH rule 0 x 506 [82]
+ CRUSH rule 0 x 507 [81]
+ CRUSH rule 0 x 508 [34]
+ CRUSH rule 0 x 509 [88]
+ CRUSH rule 0 x 510 [11]
+ CRUSH rule 0 x 511 [72]
+ CRUSH rule 0 x 512 [118]
+ CRUSH rule 0 x 513 [22]
+ CRUSH rule 0 x 514 [82]
+ CRUSH rule 0 x 515 [27]
+ CRUSH rule 0 x 516 [66]
+ CRUSH rule 0 x 517 [83]
+ CRUSH rule 0 x 518 [18]
+ CRUSH rule 0 x 519 [67]
+ CRUSH rule 0 x 520 [15]
+ CRUSH rule 0 x 521 [63]
+ CRUSH rule 0 x 522 [56]
+ CRUSH rule 0 x 523 [36]
+ CRUSH rule 0 x 524 [33]
+ CRUSH rule 0 x 525 [3]
+ CRUSH rule 0 x 526 [83]
+ CRUSH rule 0 x 527 [37]
+ CRUSH rule 0 x 528 [108]
+ CRUSH rule 0 x 529 [107]
+ CRUSH rule 0 x 530 [49]
+ CRUSH rule 0 x 531 [27]
+ CRUSH rule 0 x 532 [68]
+ CRUSH rule 0 x 533 [5]
+ CRUSH rule 0 x 534 [97]
+ CRUSH rule 0 x 535 [8]
+ CRUSH rule 0 x 536 [3]
+ CRUSH rule 0 x 537 [116]
+ CRUSH rule 0 x 538 [85]
+ CRUSH rule 0 x 539 [10]
+ CRUSH rule 0 x 540 [100]
+ CRUSH rule 0 x 541 [111]
+ CRUSH rule 0 x 542 [50]
+ CRUSH rule 0 x 543 [45]
+ CRUSH rule 0 x 544 [106]
+ CRUSH rule 0 x 545 [43]
+ CRUSH rule 0 x 546 [108]
+ CRUSH rule 0 x 547 [67]
+ CRUSH rule 0 x 548 [58]
+ CRUSH rule 0 x 549 [60]
+ CRUSH rule 0 x 550 [47]
+ CRUSH rule 0 x 551 [14]
+ CRUSH rule 0 x 552 [70]
+ CRUSH rule 0 x 553 [96]
+ CRUSH rule 0 x 554 [61]
+ CRUSH rule 0 x 555 [76]
+ CRUSH rule 0 x 556 [106]
+ CRUSH rule 0 x 557 [39]
+ CRUSH rule 0 x 558 [70]
+ CRUSH rule 0 x 559 [106]
+ CRUSH rule 0 x 560 [94]
+ CRUSH rule 0 x 561 [27]
+ CRUSH rule 0 x 562 [97]
+ CRUSH rule 0 x 563 [64]
+ CRUSH rule 0 x 564 [96]
+ CRUSH rule 0 x 565 [66]
+ CRUSH rule 0 x 566 [27]
+ CRUSH rule 0 x 567 [88]
+ CRUSH rule 0 x 568 [106]
+ CRUSH rule 0 x 569 [102]
+ CRUSH rule 0 x 570 [98]
+ CRUSH rule 0 x 571 [95]
+ CRUSH rule 0 x 572 [62]
+ CRUSH rule 0 x 573 [51]
+ CRUSH rule 0 x 574 [89]
+ CRUSH rule 0 x 575 [87]
+ CRUSH rule 0 x 576 [112]
+ CRUSH rule 0 x 577 [8]
+ CRUSH rule 0 x 578 [64]
+ CRUSH rule 0 x 579 [78]
+ CRUSH rule 0 x 580 [68]
+ CRUSH rule 0 x 581 [55]
+ CRUSH rule 0 x 582 [15]
+ CRUSH rule 0 x 583 [74]
+ CRUSH rule 0 x 584 [22]
+ CRUSH rule 0 x 585 [35]
+ CRUSH rule 0 x 586 [33]
+ CRUSH rule 0 x 587 [106]
+ CRUSH rule 0 x 588 [0]
+ CRUSH rule 0 x 589 [7]
+ CRUSH rule 0 x 590 [40]
+ CRUSH rule 0 x 591 [42]
+ CRUSH rule 0 x 592 [45]
+ CRUSH rule 0 x 593 [89]
+ CRUSH rule 0 x 594 [27]
+ CRUSH rule 0 x 595 [7]
+ CRUSH rule 0 x 596 [82]
+ CRUSH rule 0 x 597 [72]
+ CRUSH rule 0 x 598 [34]
+ CRUSH rule 0 x 599 [119]
+ CRUSH rule 0 x 600 [24]
+ CRUSH rule 0 x 601 [104]
+ CRUSH rule 0 x 602 [48]
+ CRUSH rule 0 x 603 [24]
+ CRUSH rule 0 x 604 [89]
+ CRUSH rule 0 x 605 [104]
+ CRUSH rule 0 x 606 [49]
+ CRUSH rule 0 x 607 [95]
+ CRUSH rule 0 x 608 [112]
+ CRUSH rule 0 x 609 [61]
+ CRUSH rule 0 x 610 [106]
+ CRUSH rule 0 x 611 [66]
+ CRUSH rule 0 x 612 [103]
+ CRUSH rule 0 x 613 [13]
+ CRUSH rule 0 x 614 [81]
+ CRUSH rule 0 x 615 [61]
+ CRUSH rule 0 x 616 [41]
+ CRUSH rule 0 x 617 [111]
+ CRUSH rule 0 x 618 [26]
+ CRUSH rule 0 x 619 [92]
+ CRUSH rule 0 x 620 [108]
+ CRUSH rule 0 x 621 [106]
+ CRUSH rule 0 x 622 [67]
+ CRUSH rule 0 x 623 [94]
+ CRUSH rule 0 x 624 [115]
+ CRUSH rule 0 x 625 [111]
+ CRUSH rule 0 x 626 [3]
+ CRUSH rule 0 x 627 [19]
+ CRUSH rule 0 x 628 [65]
+ CRUSH rule 0 x 629 [6]
+ CRUSH rule 0 x 630 [22]
+ CRUSH rule 0 x 631 [35]
+ CRUSH rule 0 x 632 [81]
+ CRUSH rule 0 x 633 [65]
+ CRUSH rule 0 x 634 [87]
+ CRUSH rule 0 x 635 [40]
+ CRUSH rule 0 x 636 [23]
+ CRUSH rule 0 x 637 [102]
+ CRUSH rule 0 x 638 [43]
+ CRUSH rule 0 x 639 [31]
+ CRUSH rule 0 x 640 [113]
+ CRUSH rule 0 x 641 [45]
+ CRUSH rule 0 x 642 [47]
+ CRUSH rule 0 x 643 [64]
+ CRUSH rule 0 x 644 [31]
+ CRUSH rule 0 x 645 [76]
+ CRUSH rule 0 x 646 [37]
+ CRUSH rule 0 x 647 [58]
+ CRUSH rule 0 x 648 [31]
+ CRUSH rule 0 x 649 [88]
+ CRUSH rule 0 x 650 [116]
+ CRUSH rule 0 x 651 [97]
+ CRUSH rule 0 x 652 [57]
+ CRUSH rule 0 x 653 [8]
+ CRUSH rule 0 x 654 [49]
+ CRUSH rule 0 x 655 [89]
+ CRUSH rule 0 x 656 [0]
+ CRUSH rule 0 x 657 [47]
+ CRUSH rule 0 x 658 [75]
+ CRUSH rule 0 x 659 [26]
+ CRUSH rule 0 x 660 [65]
+ CRUSH rule 0 x 661 [91]
+ CRUSH rule 0 x 662 [111]
+ CRUSH rule 0 x 663 [88]
+ CRUSH rule 0 x 664 [59]
+ CRUSH rule 0 x 665 [78]
+ CRUSH rule 0 x 666 [112]
+ CRUSH rule 0 x 667 [97]
+ CRUSH rule 0 x 668 [97]
+ CRUSH rule 0 x 669 [85]
+ CRUSH rule 0 x 670 [41]
+ CRUSH rule 0 x 671 [116]
+ CRUSH rule 0 x 672 [44]
+ CRUSH rule 0 x 673 [83]
+ CRUSH rule 0 x 674 [36]
+ CRUSH rule 0 x 675 [88]
+ CRUSH rule 0 x 676 [62]
+ CRUSH rule 0 x 677 [88]
+ CRUSH rule 0 x 678 [98]
+ CRUSH rule 0 x 679 [33]
+ CRUSH rule 0 x 680 [55]
+ CRUSH rule 0 x 681 [115]
+ CRUSH rule 0 x 682 [27]
+ CRUSH rule 0 x 683 [57]
+ CRUSH rule 0 x 684 [22]
+ CRUSH rule 0 x 685 [106]
+ CRUSH rule 0 x 686 [86]
+ CRUSH rule 0 x 687 [32]
+ CRUSH rule 0 x 688 [80]
+ CRUSH rule 0 x 689 [6]
+ CRUSH rule 0 x 690 [43]
+ CRUSH rule 0 x 691 [34]
+ CRUSH rule 0 x 692 [40]
+ CRUSH rule 0 x 693 [29]
+ CRUSH rule 0 x 694 [6]
+ CRUSH rule 0 x 695 [19]
+ CRUSH rule 0 x 696 [36]
+ CRUSH rule 0 x 697 [96]
+ CRUSH rule 0 x 698 [61]
+ CRUSH rule 0 x 699 [47]
+ CRUSH rule 0 x 700 [99]
+ CRUSH rule 0 x 701 [42]
+ CRUSH rule 0 x 702 [0]
+ CRUSH rule 0 x 703 [92]
+ CRUSH rule 0 x 704 [10]
+ CRUSH rule 0 x 705 [105]
+ CRUSH rule 0 x 706 [74]
+ CRUSH rule 0 x 707 [0]
+ CRUSH rule 0 x 708 [84]
+ CRUSH rule 0 x 709 [114]
+ CRUSH rule 0 x 710 [94]
+ CRUSH rule 0 x 711 [68]
+ CRUSH rule 0 x 712 [34]
+ CRUSH rule 0 x 713 [29]
+ CRUSH rule 0 x 714 [81]
+ CRUSH rule 0 x 715 [71]
+ CRUSH rule 0 x 716 [40]
+ CRUSH rule 0 x 717 [61]
+ CRUSH rule 0 x 718 [40]
+ CRUSH rule 0 x 719 [59]
+ CRUSH rule 0 x 720 [69]
+ CRUSH rule 0 x 721 [62]
+ CRUSH rule 0 x 722 [115]
+ CRUSH rule 0 x 723 [117]
+ CRUSH rule 0 x 724 [45]
+ CRUSH rule 0 x 725 [53]
+ CRUSH rule 0 x 726 [84]
+ CRUSH rule 0 x 727 [109]
+ CRUSH rule 0 x 728 [76]
+ CRUSH rule 0 x 729 [108]
+ CRUSH rule 0 x 730 [28]
+ CRUSH rule 0 x 731 [78]
+ CRUSH rule 0 x 732 [55]
+ CRUSH rule 0 x 733 [84]
+ CRUSH rule 0 x 734 [27]
+ CRUSH rule 0 x 735 [83]
+ CRUSH rule 0 x 736 [70]
+ CRUSH rule 0 x 737 [117]
+ CRUSH rule 0 x 738 [118]
+ CRUSH rule 0 x 739 [87]
+ CRUSH rule 0 x 740 [29]
+ CRUSH rule 0 x 741 [96]
+ CRUSH rule 0 x 742 [106]
+ CRUSH rule 0 x 743 [105]
+ CRUSH rule 0 x 744 [23]
+ CRUSH rule 0 x 745 [28]
+ CRUSH rule 0 x 746 [56]
+ CRUSH rule 0 x 747 [65]
+ CRUSH rule 0 x 748 [48]
+ CRUSH rule 0 x 749 [102]
+ CRUSH rule 0 x 750 [50]
+ CRUSH rule 0 x 751 [36]
+ CRUSH rule 0 x 752 [69]
+ CRUSH rule 0 x 753 [116]
+ CRUSH rule 0 x 754 [9]
+ CRUSH rule 0 x 755 [98]
+ CRUSH rule 0 x 756 [113]
+ CRUSH rule 0 x 757 [47]
+ CRUSH rule 0 x 758 [57]
+ CRUSH rule 0 x 759 [74]
+ CRUSH rule 0 x 760 [53]
+ CRUSH rule 0 x 761 [78]
+ CRUSH rule 0 x 762 [87]
+ CRUSH rule 0 x 763 [13]
+ CRUSH rule 0 x 764 [106]
+ CRUSH rule 0 x 765 [109]
+ CRUSH rule 0 x 766 [76]
+ CRUSH rule 0 x 767 [41]
+ CRUSH rule 0 x 768 [13]
+ CRUSH rule 0 x 769 [91]
+ CRUSH rule 0 x 770 [105]
+ CRUSH rule 0 x 771 [10]
+ CRUSH rule 0 x 772 [118]
+ CRUSH rule 0 x 773 [116]
+ CRUSH rule 0 x 774 [100]
+ CRUSH rule 0 x 775 [102]
+ CRUSH rule 0 x 776 [69]
+ CRUSH rule 0 x 777 [76]
+ CRUSH rule 0 x 778 [38]
+ CRUSH rule 0 x 779 [46]
+ CRUSH rule 0 x 780 [63]
+ CRUSH rule 0 x 781 [105]
+ CRUSH rule 0 x 782 [117]
+ CRUSH rule 0 x 783 [60]
+ CRUSH rule 0 x 784 [82]
+ CRUSH rule 0 x 785 [27]
+ CRUSH rule 0 x 786 [41]
+ CRUSH rule 0 x 787 [13]
+ CRUSH rule 0 x 788 [4]
+ CRUSH rule 0 x 789 [50]
+ CRUSH rule 0 x 790 [58]
+ CRUSH rule 0 x 791 [96]
+ CRUSH rule 0 x 792 [80]
+ CRUSH rule 0 x 793 [6]
+ CRUSH rule 0 x 794 [14]
+ CRUSH rule 0 x 795 [51]
+ CRUSH rule 0 x 796 [114]
+ CRUSH rule 0 x 797 [79]
+ CRUSH rule 0 x 798 [42]
+ CRUSH rule 0 x 799 [48]
+ CRUSH rule 0 x 800 [91]
+ CRUSH rule 0 x 801 [2]
+ CRUSH rule 0 x 802 [116]
+ CRUSH rule 0 x 803 [37]
+ CRUSH rule 0 x 804 [33]
+ CRUSH rule 0 x 805 [96]
+ CRUSH rule 0 x 806 [67]
+ CRUSH rule 0 x 807 [47]
+ CRUSH rule 0 x 808 [76]
+ CRUSH rule 0 x 809 [27]
+ CRUSH rule 0 x 810 [119]
+ CRUSH rule 0 x 811 [75]
+ CRUSH rule 0 x 812 [25]
+ CRUSH rule 0 x 813 [64]
+ CRUSH rule 0 x 814 [110]
+ CRUSH rule 0 x 815 [84]
+ CRUSH rule 0 x 816 [25]
+ CRUSH rule 0 x 817 [40]
+ CRUSH rule 0 x 818 [34]
+ CRUSH rule 0 x 819 [88]
+ CRUSH rule 0 x 820 [104]
+ CRUSH rule 0 x 821 [58]
+ CRUSH rule 0 x 822 [29]
+ CRUSH rule 0 x 823 [100]
+ CRUSH rule 0 x 824 [102]
+ CRUSH rule 0 x 825 [47]
+ CRUSH rule 0 x 826 [45]
+ CRUSH rule 0 x 827 [101]
+ CRUSH rule 0 x 828 [60]
+ CRUSH rule 0 x 829 [45]
+ CRUSH rule 0 x 830 [51]
+ CRUSH rule 0 x 831 [6]
+ CRUSH rule 0 x 832 [57]
+ CRUSH rule 0 x 833 [34]
+ CRUSH rule 0 x 834 [90]
+ CRUSH rule 0 x 835 [14]
+ CRUSH rule 0 x 836 [38]
+ CRUSH rule 0 x 837 [51]
+ CRUSH rule 0 x 838 [6]
+ CRUSH rule 0 x 839 [106]
+ CRUSH rule 0 x 840 [33]
+ CRUSH rule 0 x 841 [110]
+ CRUSH rule 0 x 842 [66]
+ CRUSH rule 0 x 843 [11]
+ CRUSH rule 0 x 844 [74]
+ CRUSH rule 0 x 845 [74]
+ CRUSH rule 0 x 846 [98]
+ CRUSH rule 0 x 847 [10]
+ CRUSH rule 0 x 848 [89]
+ CRUSH rule 0 x 849 [42]
+ CRUSH rule 0 x 850 [40]
+ CRUSH rule 0 x 851 [65]
+ CRUSH rule 0 x 852 [31]
+ CRUSH rule 0 x 853 [49]
+ CRUSH rule 0 x 854 [90]
+ CRUSH rule 0 x 855 [2]
+ CRUSH rule 0 x 856 [40]
+ CRUSH rule 0 x 857 [15]
+ CRUSH rule 0 x 858 [10]
+ CRUSH rule 0 x 859 [29]
+ CRUSH rule 0 x 860 [114]
+ CRUSH rule 0 x 861 [22]
+ CRUSH rule 0 x 862 [22]
+ CRUSH rule 0 x 863 [79]
+ CRUSH rule 0 x 864 [68]
+ CRUSH rule 0 x 865 [25]
+ CRUSH rule 0 x 866 [18]
+ CRUSH rule 0 x 867 [3]
+ CRUSH rule 0 x 868 [81]
+ CRUSH rule 0 x 869 [22]
+ CRUSH rule 0 x 870 [73]
+ CRUSH rule 0 x 871 [25]
+ CRUSH rule 0 x 872 [39]
+ CRUSH rule 0 x 873 [92]
+ CRUSH rule 0 x 874 [21]
+ CRUSH rule 0 x 875 [27]
+ CRUSH rule 0 x 876 [98]
+ CRUSH rule 0 x 877 [73]
+ CRUSH rule 0 x 878 [64]
+ CRUSH rule 0 x 879 [29]
+ CRUSH rule 0 x 880 [56]
+ CRUSH rule 0 x 881 [109]
+ CRUSH rule 0 x 882 [60]
+ CRUSH rule 0 x 883 [93]
+ CRUSH rule 0 x 884 [67]
+ CRUSH rule 0 x 885 [31]
+ CRUSH rule 0 x 886 [2]
+ CRUSH rule 0 x 887 [5]
+ CRUSH rule 0 x 888 [16]
+ CRUSH rule 0 x 889 [3]
+ CRUSH rule 0 x 890 [48]
+ CRUSH rule 0 x 891 [86]
+ CRUSH rule 0 x 892 [64]
+ CRUSH rule 0 x 893 [118]
+ CRUSH rule 0 x 894 [16]
+ CRUSH rule 0 x 895 [40]
+ CRUSH rule 0 x 896 [97]
+ CRUSH rule 0 x 897 [60]
+ CRUSH rule 0 x 898 [10]
+ CRUSH rule 0 x 899 [75]
+ CRUSH rule 0 x 900 [102]
+ CRUSH rule 0 x 901 [66]
+ CRUSH rule 0 x 902 [102]
+ CRUSH rule 0 x 903 [5]
+ CRUSH rule 0 x 904 [50]
+ CRUSH rule 0 x 905 [19]
+ CRUSH rule 0 x 906 [75]
+ CRUSH rule 0 x 907 [47]
+ CRUSH rule 0 x 908 [96]
+ CRUSH rule 0 x 909 [94]
+ CRUSH rule 0 x 910 [88]
+ CRUSH rule 0 x 911 [102]
+ CRUSH rule 0 x 912 [91]
+ CRUSH rule 0 x 913 [29]
+ CRUSH rule 0 x 914 [84]
+ CRUSH rule 0 x 915 [70]
+ CRUSH rule 0 x 916 [32]
+ CRUSH rule 0 x 917 [43]
+ CRUSH rule 0 x 918 [91]
+ CRUSH rule 0 x 919 [13]
+ CRUSH rule 0 x 920 [18]
+ CRUSH rule 0 x 921 [104]
+ CRUSH rule 0 x 922 [33]
+ CRUSH rule 0 x 923 [28]
+ CRUSH rule 0 x 924 [69]
+ CRUSH rule 0 x 925 [71]
+ CRUSH rule 0 x 926 [64]
+ CRUSH rule 0 x 927 [99]
+ CRUSH rule 0 x 928 [13]
+ CRUSH rule 0 x 929 [117]
+ CRUSH rule 0 x 930 [31]
+ CRUSH rule 0 x 931 [46]
+ CRUSH rule 0 x 932 [60]
+ CRUSH rule 0 x 933 [88]
+ CRUSH rule 0 x 934 [68]
+ CRUSH rule 0 x 935 [31]
+ CRUSH rule 0 x 936 [104]
+ CRUSH rule 0 x 937 [110]
+ CRUSH rule 0 x 938 [29]
+ CRUSH rule 0 x 939 [77]
+ CRUSH rule 0 x 940 [76]
+ CRUSH rule 0 x 941 [66]
+ CRUSH rule 0 x 942 [83]
+ CRUSH rule 0 x 943 [4]
+ CRUSH rule 0 x 944 [113]
+ CRUSH rule 0 x 945 [17]
+ CRUSH rule 0 x 946 [37]
+ CRUSH rule 0 x 947 [107]
+ CRUSH rule 0 x 948 [55]
+ CRUSH rule 0 x 949 [45]
+ CRUSH rule 0 x 950 [96]
+ CRUSH rule 0 x 951 [40]
+ CRUSH rule 0 x 952 [93]
+ CRUSH rule 0 x 953 [55]
+ CRUSH rule 0 x 954 [84]
+ CRUSH rule 0 x 955 [31]
+ CRUSH rule 0 x 956 [72]
+ CRUSH rule 0 x 957 [3]
+ CRUSH rule 0 x 958 [8]
+ CRUSH rule 0 x 959 [42]
+ CRUSH rule 0 x 960 [113]
+ CRUSH rule 0 x 961 [116]
+ CRUSH rule 0 x 962 [13]
+ CRUSH rule 0 x 963 [0]
+ CRUSH rule 0 x 964 [59]
+ CRUSH rule 0 x 965 [47]
+ CRUSH rule 0 x 966 [88]
+ CRUSH rule 0 x 967 [71]
+ CRUSH rule 0 x 968 [73]
+ CRUSH rule 0 x 969 [53]
+ CRUSH rule 0 x 970 [3]
+ CRUSH rule 0 x 971 [87]
+ CRUSH rule 0 x 972 [3]
+ CRUSH rule 0 x 973 [113]
+ CRUSH rule 0 x 974 [114]
+ CRUSH rule 0 x 975 [40]
+ CRUSH rule 0 x 976 [81]
+ CRUSH rule 0 x 977 [95]
+ CRUSH rule 0 x 978 [35]
+ CRUSH rule 0 x 979 [98]
+ CRUSH rule 0 x 980 [52]
+ CRUSH rule 0 x 981 [89]
+ CRUSH rule 0 x 982 [1]
+ CRUSH rule 0 x 983 [34]
+ CRUSH rule 0 x 984 [78]
+ CRUSH rule 0 x 985 [99]
+ CRUSH rule 0 x 986 [4]
+ CRUSH rule 0 x 987 [78]
+ CRUSH rule 0 x 988 [79]
+ CRUSH rule 0 x 989 [87]
+ CRUSH rule 0 x 990 [47]
+ CRUSH rule 0 x 991 [61]
+ CRUSH rule 0 x 992 [83]
+ CRUSH rule 0 x 993 [75]
+ CRUSH rule 0 x 994 [74]
+ CRUSH rule 0 x 995 [100]
+ CRUSH rule 0 x 996 [41]
+ CRUSH rule 0 x 997 [89]
+ CRUSH rule 0 x 998 [92]
+ CRUSH rule 0 x 999 [101]
+ CRUSH rule 0 x 1000 [9]
+ CRUSH rule 0 x 1001 [49]
+ CRUSH rule 0 x 1002 [99]
+ CRUSH rule 0 x 1003 [43]
+ CRUSH rule 0 x 1004 [89]
+ CRUSH rule 0 x 1005 [105]
+ CRUSH rule 0 x 1006 [45]
+ CRUSH rule 0 x 1007 [19]
+ CRUSH rule 0 x 1008 [31]
+ CRUSH rule 0 x 1009 [19]
+ CRUSH rule 0 x 1010 [42]
+ CRUSH rule 0 x 1011 [25]
+ CRUSH rule 0 x 1012 [68]
+ CRUSH rule 0 x 1013 [5]
+ CRUSH rule 0 x 1014 [33]
+ CRUSH rule 0 x 1015 [106]
+ CRUSH rule 0 x 1016 [88]
+ CRUSH rule 0 x 1017 [0]
+ CRUSH rule 0 x 1018 [63]
+ CRUSH rule 0 x 1019 [104]
+ CRUSH rule 0 x 1020 [96]
+ CRUSH rule 0 x 1021 [117]
+ CRUSH rule 0 x 1022 [73]
+ CRUSH rule 0 x 1023 [0]
+ rule 0 (data) num_rep 1 result size == 1:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [101,114]
+ CRUSH rule 0 x 1 [80,79]
+ CRUSH rule 0 x 2 [91,96]
+ CRUSH rule 0 x 3 [51,4]
+ CRUSH rule 0 x 4 [50,89]
+ CRUSH rule 0 x 5 [89,94]
+ CRUSH rule 0 x 6 [91,76]
+ CRUSH rule 0 x 7 [104,25]
+ CRUSH rule 0 x 8 [78,57]
+ CRUSH rule 0 x 9 [101,102]
+ CRUSH rule 0 x 10 [61,58]
+ CRUSH rule 0 x 11 [13,31]
+ CRUSH rule 0 x 12 [83,46]
+ CRUSH rule 0 x 13 [108,85]
+ CRUSH rule 0 x 14 [105,72]
+ CRUSH rule 0 x 15 [18,7]
+ CRUSH rule 0 x 16 [103,3]
+ CRUSH rule 0 x 17 [85,110]
+ CRUSH rule 0 x 18 [11,65]
+ CRUSH rule 0 x 19 [75,50]
+ CRUSH rule 0 x 20 [79,70]
+ CRUSH rule 0 x 21 [84,49]
+ CRUSH rule 0 x 22 [23,104]
+ CRUSH rule 0 x 23 [118,63]
+ CRUSH rule 0 x 24 [83,38]
+ CRUSH rule 0 x 25 [81,64]
+ CRUSH rule 0 x 26 [38,99]
+ CRUSH rule 0 x 27 [76,107]
+ CRUSH rule 0 x 28 [76,71]
+ CRUSH rule 0 x 29 [24,71]
+ CRUSH rule 0 x 30 [94,87]
+ CRUSH rule 0 x 31 [76,95]
+ CRUSH rule 0 x 32 [72,95]
+ CRUSH rule 0 x 33 [77,86]
+ CRUSH rule 0 x 34 [7,108]
+ CRUSH rule 0 x 35 [22,88]
+ CRUSH rule 0 x 36 [104,65]
+ CRUSH rule 0 x 37 [61,109]
+ CRUSH rule 0 x 38 [72,85]
+ CRUSH rule 0 x 39 [68,103]
+ CRUSH rule 0 x 40 [103,78]
+ CRUSH rule 0 x 41 [85,11]
+ CRUSH rule 0 x 42 [106,33]
+ CRUSH rule 0 x 43 [10,68]
+ CRUSH rule 0 x 44 [101,4]
+ CRUSH rule 0 x 45 [83,15]
+ CRUSH rule 0 x 46 [65,1]
+ CRUSH rule 0 x 47 [106,53]
+ CRUSH rule 0 x 48 [34,33]
+ CRUSH rule 0 x 49 [0,81]
+ CRUSH rule 0 x 50 [42,6]
+ CRUSH rule 0 x 51 [104,75]
+ CRUSH rule 0 x 52 [83,19]
+ CRUSH rule 0 x 53 [32,69]
+ CRUSH rule 0 x 54 [9,79]
+ CRUSH rule 0 x 55 [14,5]
+ CRUSH rule 0 x 56 [21,72]
+ CRUSH rule 0 x 57 [93,84]
+ CRUSH rule 0 x 58 [45,106]
+ CRUSH rule 0 x 59 [80,41]
+ CRUSH rule 0 x 60 [90,57]
+ CRUSH rule 0 x 61 [88,37]
+ CRUSH rule 0 x 62 [81,1]
+ CRUSH rule 0 x 63 [79,113]
+ CRUSH rule 0 x 64 [1,35]
+ CRUSH rule 0 x 65 [32,103]
+ CRUSH rule 0 x 66 [48,99]
+ CRUSH rule 0 x 67 [94,103]
+ CRUSH rule 0 x 68 [102,91]
+ CRUSH rule 0 x 69 [62,77]
+ CRUSH rule 0 x 70 [84,105]
+ CRUSH rule 0 x 71 [9,33]
+ CRUSH rule 0 x 72 [97,42]
+ CRUSH rule 0 x 73 [64,83]
+ CRUSH rule 0 x 74 [29,50]
+ CRUSH rule 0 x 75 [29,28]
+ CRUSH rule 0 x 76 [55,0]
+ CRUSH rule 0 x 77 [107,21]
+ CRUSH rule 0 x 78 [11,89]
+ CRUSH rule 0 x 79 [64,51]
+ CRUSH rule 0 x 80 [0,31]
+ CRUSH rule 0 x 81 [71,109]
+ CRUSH rule 0 x 82 [37,21]
+ CRUSH rule 0 x 83 [92,103]
+ CRUSH rule 0 x 84 [49,115]
+ CRUSH rule 0 x 85 [54,101]
+ CRUSH rule 0 x 86 [37,7]
+ CRUSH rule 0 x 87 [116,4]
+ CRUSH rule 0 x 88 [38,27]
+ CRUSH rule 0 x 89 [76,77]
+ CRUSH rule 0 x 90 [14,50]
+ CRUSH rule 0 x 91 [68,19]
+ CRUSH rule 0 x 92 [86,9]
+ CRUSH rule 0 x 93 [44,65]
+ CRUSH rule 0 x 94 [61,102]
+ CRUSH rule 0 x 95 [93,86]
+ CRUSH rule 0 x 96 [66,87]
+ CRUSH rule 0 x 97 [111,9]
+ CRUSH rule 0 x 98 [93,102]
+ CRUSH rule 0 x 99 [78,3]
+ CRUSH rule 0 x 100 [6,63]
+ CRUSH rule 0 x 101 [84,16]
+ CRUSH rule 0 x 102 [82,105]
+ CRUSH rule 0 x 103 [66,6]
+ CRUSH rule 0 x 104 [14,95]
+ CRUSH rule 0 x 105 [87,1]
+ CRUSH rule 0 x 106 [69,116]
+ CRUSH rule 0 x 107 [1,55]
+ CRUSH rule 0 x 108 [94,53]
+ CRUSH rule 0 x 109 [112,13]
+ CRUSH rule 0 x 110 [54,61]
+ CRUSH rule 0 x 111 [10,78]
+ CRUSH rule 0 x 112 [89,9]
+ CRUSH rule 0 x 113 [69,2]
+ CRUSH rule 0 x 114 [79,110]
+ CRUSH rule 0 x 115 [50,85]
+ CRUSH rule 0 x 116 [96,16]
+ CRUSH rule 0 x 117 [87,42]
+ CRUSH rule 0 x 118 [23,56]
+ CRUSH rule 0 x 119 [104,11]
+ CRUSH rule 0 x 120 [57,5]
+ CRUSH rule 0 x 121 [105,9]
+ CRUSH rule 0 x 122 [45,110]
+ CRUSH rule 0 x 123 [112,35]
+ CRUSH rule 0 x 124 [110,49]
+ CRUSH rule 0 x 125 [66,105]
+ CRUSH rule 0 x 126 [51,28]
+ CRUSH rule 0 x 127 [70,6]
+ CRUSH rule 0 x 128 [90,16]
+ CRUSH rule 0 x 129 [103,110]
+ CRUSH rule 0 x 130 [50,11]
+ CRUSH rule 0 x 131 [23,60]
+ CRUSH rule 0 x 132 [69,70]
+ CRUSH rule 0 x 133 [52,25]
+ CRUSH rule 0 x 134 [78,29]
+ CRUSH rule 0 x 135 [78,3]
+ CRUSH rule 0 x 136 [32,29]
+ CRUSH rule 0 x 137 [11,78]
+ CRUSH rule 0 x 138 [17,94]
+ CRUSH rule 0 x 139 [89,60]
+ CRUSH rule 0 x 140 [39,62]
+ CRUSH rule 0 x 141 [89,98]
+ CRUSH rule 0 x 142 [70,61]
+ CRUSH rule 0 x 143 [51,28]
+ CRUSH rule 0 x 144 [13,81]
+ CRUSH rule 0 x 145 [77,119]
+ CRUSH rule 0 x 146 [8,64]
+ CRUSH rule 0 x 147 [22,37]
+ CRUSH rule 0 x 148 [74,69]
+ CRUSH rule 0 x 149 [76,13]
+ CRUSH rule 0 x 150 [14,47]
+ CRUSH rule 0 x 151 [90,4]
+ CRUSH rule 0 x 152 [49,18]
+ CRUSH rule 0 x 153 [71,44]
+ CRUSH rule 0 x 154 [94,81]
+ CRUSH rule 0 x 155 [75,6]
+ CRUSH rule 0 x 156 [94,85]
+ CRUSH rule 0 x 157 [112,43]
+ CRUSH rule 0 x 158 [26,17]
+ CRUSH rule 0 x 159 [52,29]
+ CRUSH rule 0 x 160 [41,0]
+ CRUSH rule 0 x 161 [19,78]
+ CRUSH rule 0 x 162 [55,2]
+ CRUSH rule 0 x 163 [54,31]
+ CRUSH rule 0 x 164 [45,5]
+ CRUSH rule 0 x 165 [25,72]
+ CRUSH rule 0 x 166 [73,36]
+ CRUSH rule 0 x 167 [89,58]
+ CRUSH rule 0 x 168 [47,40]
+ CRUSH rule 0 x 169 [51,21]
+ CRUSH rule 0 x 170 [68,91]
+ CRUSH rule 0 x 171 [73,90]
+ CRUSH rule 0 x 172 [33,15]
+ CRUSH rule 0 x 173 [102,59]
+ CRUSH rule 0 x 174 [116,25]
+ CRUSH rule 0 x 175 [3,41]
+ CRUSH rule 0 x 176 [94,91]
+ CRUSH rule 0 x 177 [52,85]
+ CRUSH rule 0 x 178 [39,2]
+ CRUSH rule 0 x 179 [72,97]
+ CRUSH rule 0 x 180 [60,7]
+ CRUSH rule 0 x 181 [18,59]
+ CRUSH rule 0 x 182 [22,90]
+ CRUSH rule 0 x 183 [11,74]
+ CRUSH rule 0 x 184 [92,101]
+ CRUSH rule 0 x 185 [97,8]
+ CRUSH rule 0 x 186 [67,116]
+ CRUSH rule 0 x 187 [116,11]
+ CRUSH rule 0 x 188 [69,92]
+ CRUSH rule 0 x 189 [47,84]
+ CRUSH rule 0 x 190 [90,13]
+ CRUSH rule 0 x 191 [49,17]
+ CRUSH rule 0 x 192 [68,93]
+ CRUSH rule 0 x 193 [0,33]
+ CRUSH rule 0 x 194 [17,58]
+ CRUSH rule 0 x 195 [119,41]
+ CRUSH rule 0 x 196 [72,27]
+ CRUSH rule 0 x 197 [106,83]
+ CRUSH rule 0 x 198 [114,95]
+ CRUSH rule 0 x 199 [0,83]
+ CRUSH rule 0 x 200 [35,86]
+ CRUSH rule 0 x 201 [14,29]
+ CRUSH rule 0 x 202 [98,33]
+ CRUSH rule 0 x 203 [36,22]
+ CRUSH rule 0 x 204 [10,98]
+ CRUSH rule 0 x 205 [22,61]
+ CRUSH rule 0 x 206 [49,112]
+ CRUSH rule 0 x 207 [80,39]
+ CRUSH rule 0 x 208 [63,26]
+ CRUSH rule 0 x 209 [85,111]
+ CRUSH rule 0 x 210 [79,18]
+ CRUSH rule 0 x 211 [26,10]
+ CRUSH rule 0 x 212 [28,103]
+ CRUSH rule 0 x 213 [91,0]
+ CRUSH rule 0 x 214 [78,47]
+ CRUSH rule 0 x 215 [61,22]
+ CRUSH rule 0 x 216 [99,3]
+ CRUSH rule 0 x 217 [86,89]
+ CRUSH rule 0 x 218 [93,96]
+ CRUSH rule 0 x 219 [28,59]
+ CRUSH rule 0 x 220 [56,8]
+ CRUSH rule 0 x 221 [0,9]
+ CRUSH rule 0 x 222 [50,63]
+ CRUSH rule 0 x 223 [29,1]
+ CRUSH rule 0 x 224 [52,10]
+ CRUSH rule 0 x 225 [61,11]
+ CRUSH rule 0 x 226 [44,22]
+ CRUSH rule 0 x 227 [42,3]
+ CRUSH rule 0 x 228 [117,49]
+ CRUSH rule 0 x 229 [100,79]
+ CRUSH rule 0 x 230 [41,114]
+ CRUSH rule 0 x 231 [56,95]
+ CRUSH rule 0 x 232 [23,44]
+ CRUSH rule 0 x 233 [88,103]
+ CRUSH rule 0 x 234 [4,101]
+ CRUSH rule 0 x 235 [26,10]
+ CRUSH rule 0 x 236 [32,37]
+ CRUSH rule 0 x 237 [92,3]
+ CRUSH rule 0 x 238 [10,26]
+ CRUSH rule 0 x 239 [15,105]
+ CRUSH rule 0 x 240 [109,85]
+ CRUSH rule 0 x 241 [47,108]
+ CRUSH rule 0 x 242 [24,99]
+ CRUSH rule 0 x 243 [76,8]
+ CRUSH rule 0 x 244 [96,19]
+ CRUSH rule 0 x 245 [27,28]
+ CRUSH rule 0 x 246 [35,82]
+ CRUSH rule 0 x 247 [99,102]
+ CRUSH rule 0 x 248 [8,29]
+ CRUSH rule 0 x 249 [85,1]
+ CRUSH rule 0 x 250 [79,102]
+ CRUSH rule 0 x 251 [28,103]
+ CRUSH rule 0 x 252 [95,22]
+ CRUSH rule 0 x 253 [109,27]
+ CRUSH rule 0 x 254 [80,103]
+ CRUSH rule 0 x 255 [112,22]
+ CRUSH rule 0 x 256 [37,38]
+ CRUSH rule 0 x 257 [69,117]
+ CRUSH rule 0 x 258 [34,55]
+ CRUSH rule 0 x 259 [70,17]
+ CRUSH rule 0 x 260 [98,29]
+ CRUSH rule 0 x 261 [94,83]
+ CRUSH rule 0 x 262 [42,49]
+ CRUSH rule 0 x 263 [65,42]
+ CRUSH rule 0 x 264 [36,17]
+ CRUSH rule 0 x 265 [66,63]
+ CRUSH rule 0 x 266 [75,92]
+ CRUSH rule 0 x 267 [58,35]
+ CRUSH rule 0 x 268 [38,9]
+ CRUSH rule 0 x 269 [43,104]
+ CRUSH rule 0 x 270 [58,37]
+ CRUSH rule 0 x 271 [19,33]
+ CRUSH rule 0 x 272 [73,9]
+ CRUSH rule 0 x 273 [108,29]
+ CRUSH rule 0 x 274 [47,64]
+ CRUSH rule 0 x 275 [92,19]
+ CRUSH rule 0 x 276 [7,79]
+ CRUSH rule 0 x 277 [19,68]
+ CRUSH rule 0 x 278 [116,95]
+ CRUSH rule 0 x 279 [101,3]
+ CRUSH rule 0 x 280 [113,69]
+ CRUSH rule 0 x 281 [14,93]
+ CRUSH rule 0 x 282 [106,7]
+ CRUSH rule 0 x 283 [8,118]
+ CRUSH rule 0 x 284 [10,110]
+ CRUSH rule 0 x 285 [88,63]
+ CRUSH rule 0 x 286 [27,4]
+ CRUSH rule 0 x 287 [84,65]
+ CRUSH rule 0 x 288 [103,8]
+ CRUSH rule 0 x 289 [9,104]
+ CRUSH rule 0 x 290 [115,7]
+ CRUSH rule 0 x 291 [48,45]
+ CRUSH rule 0 x 292 [52,16]
+ CRUSH rule 0 x 293 [27,24]
+ CRUSH rule 0 x 294 [79,36]
+ CRUSH rule 0 x 295 [37,116]
+ CRUSH rule 0 x 296 [56,61]
+ CRUSH rule 0 x 297 [35,40]
+ CRUSH rule 0 x 298 [71,118]
+ CRUSH rule 0 x 299 [79,1]
+ CRUSH rule 0 x 300 [67,5]
+ CRUSH rule 0 x 301 [51,110]
+ CRUSH rule 0 x 302 [78,67]
+ CRUSH rule 0 x 303 [19,94]
+ CRUSH rule 0 x 304 [101,66]
+ CRUSH rule 0 x 305 [81,62]
+ CRUSH rule 0 x 306 [0,23]
+ CRUSH rule 0 x 307 [44,15]
+ CRUSH rule 0 x 308 [91,98]
+ CRUSH rule 0 x 309 [15,18]
+ CRUSH rule 0 x 310 [26,89]
+ CRUSH rule 0 x 311 [36,41]
+ CRUSH rule 0 x 312 [33,22]
+ CRUSH rule 0 x 313 [104,16]
+ CRUSH rule 0 x 314 [28,4]
+ CRUSH rule 0 x 315 [16,8]
+ CRUSH rule 0 x 316 [4,1]
+ CRUSH rule 0 x 317 [118,8]
+ CRUSH rule 0 x 318 [32,47]
+ CRUSH rule 0 x 319 [24,83]
+ CRUSH rule 0 x 320 [36,97]
+ CRUSH rule 0 x 321 [26,85]
+ CRUSH rule 0 x 322 [87,42]
+ CRUSH rule 0 x 323 [73,0]
+ CRUSH rule 0 x 324 [64,37]
+ CRUSH rule 0 x 325 [52,16]
+ CRUSH rule 0 x 326 [111,93]
+ CRUSH rule 0 x 327 [62,16]
+ CRUSH rule 0 x 328 [7,42]
+ CRUSH rule 0 x 329 [93,34]
+ CRUSH rule 0 x 330 [24,4]
+ CRUSH rule 0 x 331 [41,21]
+ CRUSH rule 0 x 332 [61,110]
+ CRUSH rule 0 x 333 [16,8]
+ CRUSH rule 0 x 334 [94,35]
+ CRUSH rule 0 x 335 [71,74]
+ CRUSH rule 0 x 336 [16,19]
+ CRUSH rule 0 x 337 [37,11]
+ CRUSH rule 0 x 338 [109,69]
+ CRUSH rule 0 x 339 [13,64]
+ CRUSH rule 0 x 340 [119,15]
+ CRUSH rule 0 x 341 [63,114]
+ CRUSH rule 0 x 342 [92,25]
+ CRUSH rule 0 x 343 [49,26]
+ CRUSH rule 0 x 344 [103,26]
+ CRUSH rule 0 x 345 [56,25]
+ CRUSH rule 0 x 346 [3,79]
+ CRUSH rule 0 x 347 [106,27]
+ CRUSH rule 0 x 348 [10,117]
+ CRUSH rule 0 x 349 [96,37]
+ CRUSH rule 0 x 350 [63,32]
+ CRUSH rule 0 x 351 [60,85]
+ CRUSH rule 0 x 352 [103,84]
+ CRUSH rule 0 x 353 [10,113]
+ CRUSH rule 0 x 354 [55,52]
+ CRUSH rule 0 x 355 [73,68]
+ CRUSH rule 0 x 356 [114,41]
+ CRUSH rule 0 x 357 [70,13]
+ CRUSH rule 0 x 358 [97,13]
+ CRUSH rule 0 x 359 [4,117]
+ CRUSH rule 0 x 360 [106,69]
+ CRUSH rule 0 x 361 [27,46]
+ CRUSH rule 0 x 362 [28,33]
+ CRUSH rule 0 x 363 [45,26]
+ CRUSH rule 0 x 364 [23,50]
+ CRUSH rule 0 x 365 [57,114]
+ CRUSH rule 0 x 366 [14,58]
+ CRUSH rule 0 x 367 [108,65]
+ CRUSH rule 0 x 368 [103,32]
+ CRUSH rule 0 x 369 [11,57]
+ CRUSH rule 0 x 370 [11,89]
+ CRUSH rule 0 x 371 [34,55]
+ CRUSH rule 0 x 372 [58,10]
+ CRUSH rule 0 x 373 [6,42]
+ CRUSH rule 0 x 374 [110,95]
+ CRUSH rule 0 x 375 [19,92]
+ CRUSH rule 0 x 376 [22,86]
+ CRUSH rule 0 x 377 [93,113]
+ CRUSH rule 0 x 378 [67,36]
+ CRUSH rule 0 x 379 [77,115]
+ CRUSH rule 0 x 380 [3,108]
+ CRUSH rule 0 x 381 [55,1]
+ CRUSH rule 0 x 382 [26,51]
+ CRUSH rule 0 x 383 [48,25]
+ CRUSH rule 0 x 384 [15,100]
+ CRUSH rule 0 x 385 [82,4]
+ CRUSH rule 0 x 386 [108,63]
+ CRUSH rule 0 x 387 [70,41]
+ CRUSH rule 0 x 388 [5,67]
+ CRUSH rule 0 x 389 [14,1]
+ CRUSH rule 0 x 390 [68,10]
+ CRUSH rule 0 x 391 [113,14]
+ CRUSH rule 0 x 392 [72,14]
+ CRUSH rule 0 x 393 [115,6]
+ CRUSH rule 0 x 394 [38,21]
+ CRUSH rule 0 x 395 [0,27]
+ CRUSH rule 0 x 396 [59,92]
+ CRUSH rule 0 x 397 [87,1]
+ CRUSH rule 0 x 398 [44,75]
+ CRUSH rule 0 x 399 [9,2]
+ CRUSH rule 0 x 400 [19,63]
+ CRUSH rule 0 x 401 [79,34]
+ CRUSH rule 0 x 402 [107,98]
+ CRUSH rule 0 x 403 [23,82]
+ CRUSH rule 0 x 404 [76,75]
+ CRUSH rule 0 x 405 [10,32]
+ CRUSH rule 0 x 406 [38,16]
+ CRUSH rule 0 x 407 [70,85]
+ CRUSH rule 0 x 408 [55,72]
+ CRUSH rule 0 x 409 [102,15]
+ CRUSH rule 0 x 410 [59,13]
+ CRUSH rule 0 x 411 [34,29]
+ CRUSH rule 0 x 412 [108,99]
+ CRUSH rule 0 x 413 [54,107]
+ CRUSH rule 0 x 414 [70,4]
+ CRUSH rule 0 x 415 [107,36]
+ CRUSH rule 0 x 416 [21,68]
+ CRUSH rule 0 x 417 [8,70]
+ CRUSH rule 0 x 418 [51,46]
+ CRUSH rule 0 x 419 [8,66]
+ CRUSH rule 0 x 420 [109,105]
+ CRUSH rule 0 x 421 [114,17]
+ CRUSH rule 0 x 422 [109,87]
+ CRUSH rule 0 x 423 [59,98]
+ CRUSH rule 0 x 424 [71,5]
+ CRUSH rule 0 x 425 [101,111]
+ CRUSH rule 0 x 426 [47,46]
+ CRUSH rule 0 x 427 [8,115]
+ CRUSH rule 0 x 428 [68,103]
+ CRUSH rule 0 x 429 [76,6]
+ CRUSH rule 0 x 430 [69,86]
+ CRUSH rule 0 x 431 [70,83]
+ CRUSH rule 0 x 432 [46,37]
+ CRUSH rule 0 x 433 [6,101]
+ CRUSH rule 0 x 434 [64,69]
+ CRUSH rule 0 x 435 [16,50]
+ CRUSH rule 0 x 436 [89,102]
+ CRUSH rule 0 x 437 [29,114]
+ CRUSH rule 0 x 438 [105,98]
+ CRUSH rule 0 x 439 [29,119]
+ CRUSH rule 0 x 440 [38,7]
+ CRUSH rule 0 x 441 [112,105]
+ CRUSH rule 0 x 442 [55,108]
+ CRUSH rule 0 x 443 [44,57]
+ CRUSH rule 0 x 444 [72,27]
+ CRUSH rule 0 x 445 [19,5]
+ CRUSH rule 0 x 446 [40,47]
+ CRUSH rule 0 x 447 [13,61]
+ CRUSH rule 0 x 448 [7,68]
+ CRUSH rule 0 x 449 [67,19]
+ CRUSH rule 0 x 450 [117,79]
+ CRUSH rule 0 x 451 [93,108]
+ CRUSH rule 0 x 452 [70,49]
+ CRUSH rule 0 x 453 [82,22]
+ CRUSH rule 0 x 454 [53,18]
+ CRUSH rule 0 x 455 [91,92]
+ CRUSH rule 0 x 456 [101,104]
+ CRUSH rule 0 x 457 [113,51]
+ CRUSH rule 0 x 458 [53,34]
+ CRUSH rule 0 x 459 [25,115]
+ CRUSH rule 0 x 460 [105,9]
+ CRUSH rule 0 x 461 [102,35]
+ CRUSH rule 0 x 462 [98,107]
+ CRUSH rule 0 x 463 [108,105]
+ CRUSH rule 0 x 464 [19,109]
+ CRUSH rule 0 x 465 [29,86]
+ CRUSH rule 0 x 466 [66,7]
+ CRUSH rule 0 x 467 [6,57]
+ CRUSH rule 0 x 468 [97,26]
+ CRUSH rule 0 x 469 [98,75]
+ CRUSH rule 0 x 470 [50,3]
+ CRUSH rule 0 x 471 [40,79]
+ CRUSH rule 0 x 472 [74,79]
+ CRUSH rule 0 x 473 [95,21]
+ CRUSH rule 0 x 474 [51,32]
+ CRUSH rule 0 x 475 [49,110]
+ CRUSH rule 0 x 476 [110,31]
+ CRUSH rule 0 x 477 [25,106]
+ CRUSH rule 0 x 478 [47,46]
+ CRUSH rule 0 x 479 [70,37]
+ CRUSH rule 0 x 480 [62,57]
+ CRUSH rule 0 x 481 [26,19]
+ CRUSH rule 0 x 482 [84,85]
+ CRUSH rule 0 x 483 [15,116]
+ CRUSH rule 0 x 484 [37,36]
+ CRUSH rule 0 x 485 [47,117]
+ CRUSH rule 0 x 486 [92,10]
+ CRUSH rule 0 x 487 [106,51]
+ CRUSH rule 0 x 488 [42,9]
+ CRUSH rule 0 x 489 [76,16]
+ CRUSH rule 0 x 490 [68,17]
+ CRUSH rule 0 x 491 [80,71]
+ CRUSH rule 0 x 492 [21,57]
+ CRUSH rule 0 x 493 [99,78]
+ CRUSH rule 0 x 494 [4,87]
+ CRUSH rule 0 x 495 [40,43]
+ CRUSH rule 0 x 496 [93,38]
+ CRUSH rule 0 x 497 [102,71]
+ CRUSH rule 0 x 498 [68,83]
+ CRUSH rule 0 x 499 [10,26]
+ CRUSH rule 0 x 500 [50,6]
+ CRUSH rule 0 x 501 [60,9]
+ CRUSH rule 0 x 502 [11,64]
+ CRUSH rule 0 x 503 [117,25]
+ CRUSH rule 0 x 504 [90,41]
+ CRUSH rule 0 x 505 [91,100]
+ CRUSH rule 0 x 506 [82,103]
+ CRUSH rule 0 x 507 [81,54]
+ CRUSH rule 0 x 508 [34,87]
+ CRUSH rule 0 x 509 [88,63]
+ CRUSH rule 0 x 510 [11,73]
+ CRUSH rule 0 x 511 [72,27]
+ CRUSH rule 0 x 512 [118,73]
+ CRUSH rule 0 x 513 [22,76]
+ CRUSH rule 0 x 514 [82,11]
+ CRUSH rule 0 x 515 [27,0]
+ CRUSH rule 0 x 516 [66,13]
+ CRUSH rule 0 x 517 [83,60]
+ CRUSH rule 0 x 518 [18,3]
+ CRUSH rule 0 x 519 [67,119]
+ CRUSH rule 0 x 520 [15,88]
+ CRUSH rule 0 x 521 [63,113]
+ CRUSH rule 0 x 522 [56,73]
+ CRUSH rule 0 x 523 [36,35]
+ CRUSH rule 0 x 524 [33,38]
+ CRUSH rule 0 x 525 [3,119]
+ CRUSH rule 0 x 526 [83,50]
+ CRUSH rule 0 x 527 [37,0]
+ CRUSH rule 0 x 528 [108,87]
+ CRUSH rule 0 x 529 [107,60]
+ CRUSH rule 0 x 530 [49,3]
+ CRUSH rule 0 x 531 [27,104]
+ CRUSH rule 0 x 532 [68,14]
+ CRUSH rule 0 x 533 [5,85]
+ CRUSH rule 0 x 534 [97,24]
+ CRUSH rule 0 x 535 [8,75]
+ CRUSH rule 0 x 536 [3,37]
+ CRUSH rule 0 x 537 [116,7]
+ CRUSH rule 0 x 538 [85,56]
+ CRUSH rule 0 x 539 [10,9]
+ CRUSH rule 0 x 540 [100,101]
+ CRUSH rule 0 x 541 [111,77]
+ CRUSH rule 0 x 542 [50,27]
+ CRUSH rule 0 x 543 [45,21]
+ CRUSH rule 0 x 544 [106,65]
+ CRUSH rule 0 x 545 [43,114]
+ CRUSH rule 0 x 546 [108,79]
+ CRUSH rule 0 x 547 [67,50]
+ CRUSH rule 0 x 548 [58,61]
+ CRUSH rule 0 x 549 [60,22]
+ CRUSH rule 0 x 550 [47,68]
+ CRUSH rule 0 x 551 [14,88]
+ CRUSH rule 0 x 552 [70,65]
+ CRUSH rule 0 x 553 [96,105]
+ CRUSH rule 0 x 554 [61,94]
+ CRUSH rule 0 x 555 [76,37]
+ CRUSH rule 0 x 556 [106,89]
+ CRUSH rule 0 x 557 [39,113]
+ CRUSH rule 0 x 558 [70,79]
+ CRUSH rule 0 x 559 [106,69]
+ CRUSH rule 0 x 560 [94,97]
+ CRUSH rule 0 x 561 [27,76]
+ CRUSH rule 0 x 562 [97,62]
+ CRUSH rule 0 x 563 [64,103]
+ CRUSH rule 0 x 564 [96,41]
+ CRUSH rule 0 x 565 [66,71]
+ CRUSH rule 0 x 566 [27,38]
+ CRUSH rule 0 x 567 [88,8]
+ CRUSH rule 0 x 568 [106,17]
+ CRUSH rule 0 x 569 [102,63]
+ CRUSH rule 0 x 570 [98,27]
+ CRUSH rule 0 x 571 [95,98]
+ CRUSH rule 0 x 572 [62,83]
+ CRUSH rule 0 x 573 [51,118]
+ CRUSH rule 0 x 574 [89,78]
+ CRUSH rule 0 x 575 [87,19]
+ CRUSH rule 0 x 576 [112,73]
+ CRUSH rule 0 x 577 [8,84]
+ CRUSH rule 0 x 578 [64,99]
+ CRUSH rule 0 x 579 [78,77]
+ CRUSH rule 0 x 580 [68,95]
+ CRUSH rule 0 x 581 [55,52]
+ CRUSH rule 0 x 582 [15,113]
+ CRUSH rule 0 x 583 [74,105]
+ CRUSH rule 0 x 584 [22,92]
+ CRUSH rule 0 x 585 [35,1]
+ CRUSH rule 0 x 586 [33,1]
+ CRUSH rule 0 x 587 [106,99]
+ CRUSH rule 0 x 588 [0,83]
+ CRUSH rule 0 x 589 [7,95]
+ CRUSH rule 0 x 590 [40,69]
+ CRUSH rule 0 x 591 [42,23]
+ CRUSH rule 0 x 592 [45,22]
+ CRUSH rule 0 x 593 [89,14]
+ CRUSH rule 0 x 594 [27,76]
+ CRUSH rule 0 x 595 [7,10]
+ CRUSH rule 0 x 596 [82,59]
+ CRUSH rule 0 x 597 [72,83]
+ CRUSH rule 0 x 598 [34,19]
+ CRUSH rule 0 x 599 [119,61]
+ CRUSH rule 0 x 600 [24,27]
+ CRUSH rule 0 x 601 [104,15]
+ CRUSH rule 0 x 602 [48,45]
+ CRUSH rule 0 x 603 [24,13]
+ CRUSH rule 0 x 604 [89,0]
+ CRUSH rule 0 x 605 [104,87]
+ CRUSH rule 0 x 606 [49,34]
+ CRUSH rule 0 x 607 [95,40]
+ CRUSH rule 0 x 608 [112,91]
+ CRUSH rule 0 x 609 [61,66]
+ CRUSH rule 0 x 610 [106,16]
+ CRUSH rule 0 x 611 [66,87]
+ CRUSH rule 0 x 612 [103,8]
+ CRUSH rule 0 x 613 [13,91]
+ CRUSH rule 0 x 614 [81,88]
+ CRUSH rule 0 x 615 [61,19]
+ CRUSH rule 0 x 616 [41,15]
+ CRUSH rule 0 x 617 [111,69]
+ CRUSH rule 0 x 618 [26,99]
+ CRUSH rule 0 x 619 [92,27]
+ CRUSH rule 0 x 620 [108,103]
+ CRUSH rule 0 x 621 [106,99]
+ CRUSH rule 0 x 622 [67,48]
+ CRUSH rule 0 x 623 [94,61]
+ CRUSH rule 0 x 624 [115,59]
+ CRUSH rule 0 x 625 [111,27]
+ CRUSH rule 0 x 626 [3,55]
+ CRUSH rule 0 x 627 [19,29]
+ CRUSH rule 0 x 628 [65,88]
+ CRUSH rule 0 x 629 [6,46]
+ CRUSH rule 0 x 630 [22,72]
+ CRUSH rule 0 x 631 [35,22]
+ CRUSH rule 0 x 632 [81,0]
+ CRUSH rule 0 x 633 [65,68]
+ CRUSH rule 0 x 634 [87,50]
+ CRUSH rule 0 x 635 [40,73]
+ CRUSH rule 0 x 636 [23,70]
+ CRUSH rule 0 x 637 [102,45]
+ CRUSH rule 0 x 638 [43,114]
+ CRUSH rule 0 x 639 [31,78]
+ CRUSH rule 0 x 640 [113,73]
+ CRUSH rule 0 x 641 [45,96]
+ CRUSH rule 0 x 642 [47,66]
+ CRUSH rule 0 x 643 [64,47]
+ CRUSH rule 0 x 644 [31,21]
+ CRUSH rule 0 x 645 [76,43]
+ CRUSH rule 0 x 646 [37,54]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21]
+ CRUSH rule 0 x 649 [88,45]
+ CRUSH rule 0 x 650 [116,7]
+ CRUSH rule 0 x 651 [97,106]
+ CRUSH rule 0 x 652 [57,112]
+ CRUSH rule 0 x 653 [8,116]
+ CRUSH rule 0 x 654 [49,32]
+ CRUSH rule 0 x 655 [89,62]
+ CRUSH rule 0 x 656 [0,49]
+ CRUSH rule 0 x 657 [47,17]
+ CRUSH rule 0 x 658 [75,82]
+ CRUSH rule 0 x 659 [26,83]
+ CRUSH rule 0 x 660 [65,112]
+ CRUSH rule 0 x 661 [91,48]
+ CRUSH rule 0 x 662 [111,99]
+ CRUSH rule 0 x 663 [88,35]
+ CRUSH rule 0 x 664 [59,78]
+ CRUSH rule 0 x 665 [78,15]
+ CRUSH rule 0 x 666 [112,4]
+ CRUSH rule 0 x 667 [97,46]
+ CRUSH rule 0 x 668 [97,8]
+ CRUSH rule 0 x 669 [85,66]
+ CRUSH rule 0 x 670 [41,48]
+ CRUSH rule 0 x 671 [116,97]
+ CRUSH rule 0 x 672 [44,55]
+ CRUSH rule 0 x 673 [83,50]
+ CRUSH rule 0 x 674 [36,8]
+ CRUSH rule 0 x 675 [88,14]
+ CRUSH rule 0 x 676 [62,8]
+ CRUSH rule 0 x 677 [88,67]
+ CRUSH rule 0 x 678 [98,83]
+ CRUSH rule 0 x 679 [33,78]
+ CRUSH rule 0 x 680 [55,94]
+ CRUSH rule 0 x 681 [115,95]
+ CRUSH rule 0 x 682 [27,94]
+ CRUSH rule 0 x 683 [57,80]
+ CRUSH rule 0 x 684 [22,65]
+ CRUSH rule 0 x 685 [106,55]
+ CRUSH rule 0 x 686 [86,95]
+ CRUSH rule 0 x 687 [32,57]
+ CRUSH rule 0 x 688 [80,22]
+ CRUSH rule 0 x 689 [6,48]
+ CRUSH rule 0 x 690 [43,70]
+ CRUSH rule 0 x 691 [34,105]
+ CRUSH rule 0 x 692 [40,97]
+ CRUSH rule 0 x 693 [29,84]
+ CRUSH rule 0 x 694 [6,84]
+ CRUSH rule 0 x 695 [19,69]
+ CRUSH rule 0 x 696 [36,75]
+ CRUSH rule 0 x 697 [96,99]
+ CRUSH rule 0 x 698 [61,11]
+ CRUSH rule 0 x 699 [47,62]
+ CRUSH rule 0 x 700 [99,82]
+ CRUSH rule 0 x 701 [42,11]
+ CRUSH rule 0 x 702 [0,71]
+ CRUSH rule 0 x 703 [92,3]
+ CRUSH rule 0 x 704 [10,19]
+ CRUSH rule 0 x 705 [105,21]
+ CRUSH rule 0 x 706 [74,105]
+ CRUSH rule 0 x 707 [0,77]
+ CRUSH rule 0 x 708 [84,8]
+ CRUSH rule 0 x 709 [114,97]
+ CRUSH rule 0 x 710 [94,7]
+ CRUSH rule 0 x 711 [68,49]
+ CRUSH rule 0 x 712 [34,75]
+ CRUSH rule 0 x 713 [29,0]
+ CRUSH rule 0 x 714 [81,115]
+ CRUSH rule 0 x 715 [71,84]
+ CRUSH rule 0 x 716 [40,17]
+ CRUSH rule 0 x 717 [61,62]
+ CRUSH rule 0 x 718 [40,85]
+ CRUSH rule 0 x 719 [59,42]
+ CRUSH rule 0 x 720 [69,72]
+ CRUSH rule 0 x 721 [62,21]
+ CRUSH rule 0 x 722 [115,8]
+ CRUSH rule 0 x 723 [117,41]
+ CRUSH rule 0 x 724 [45,102]
+ CRUSH rule 0 x 725 [53,113]
+ CRUSH rule 0 x 726 [84,19]
+ CRUSH rule 0 x 727 [109,14]
+ CRUSH rule 0 x 728 [76,16]
+ CRUSH rule 0 x 729 [108,47]
+ CRUSH rule 0 x 730 [28,47]
+ CRUSH rule 0 x 731 [78,37]
+ CRUSH rule 0 x 732 [55,90]
+ CRUSH rule 0 x 733 [84,3]
+ CRUSH rule 0 x 734 [27,117]
+ CRUSH rule 0 x 735 [83,4]
+ CRUSH rule 0 x 736 [70,67]
+ CRUSH rule 0 x 737 [117,15]
+ CRUSH rule 0 x 738 [118,22]
+ CRUSH rule 0 x 739 [87,38]
+ CRUSH rule 0 x 740 [29,38]
+ CRUSH rule 0 x 741 [96,73]
+ CRUSH rule 0 x 742 [106,83]
+ CRUSH rule 0 x 743 [105,94]
+ CRUSH rule 0 x 744 [23,14]
+ CRUSH rule 0 x 745 [28,6]
+ CRUSH rule 0 x 746 [56,47]
+ CRUSH rule 0 x 747 [65,70]
+ CRUSH rule 0 x 748 [48,89]
+ CRUSH rule 0 x 749 [102,51]
+ CRUSH rule 0 x 750 [50,3]
+ CRUSH rule 0 x 751 [36,25]
+ CRUSH rule 0 x 752 [69,52]
+ CRUSH rule 0 x 753 [116,65]
+ CRUSH rule 0 x 754 [9,57]
+ CRUSH rule 0 x 755 [98,81]
+ CRUSH rule 0 x 756 [113,8]
+ CRUSH rule 0 x 757 [47,66]
+ CRUSH rule 0 x 758 [57,88]
+ CRUSH rule 0 x 759 [74,97]
+ CRUSH rule 0 x 760 [53,90]
+ CRUSH rule 0 x 761 [78,97]
+ CRUSH rule 0 x 762 [87,104]
+ CRUSH rule 0 x 763 [13,45]
+ CRUSH rule 0 x 764 [106,81]
+ CRUSH rule 0 x 765 [109,91]
+ CRUSH rule 0 x 766 [76,97]
+ CRUSH rule 0 x 767 [41,116]
+ CRUSH rule 0 x 768 [13,114]
+ CRUSH rule 0 x 769 [91,96]
+ CRUSH rule 0 x 770 [105,19]
+ CRUSH rule 0 x 771 [10,76]
+ CRUSH rule 0 x 772 [118,17]
+ CRUSH rule 0 x 773 [116,75]
+ CRUSH rule 0 x 774 [100,43]
+ CRUSH rule 0 x 775 [102,43]
+ CRUSH rule 0 x 776 [69,38]
+ CRUSH rule 0 x 777 [76,49]
+ CRUSH rule 0 x 778 [38,13]
+ CRUSH rule 0 x 779 [46,21]
+ CRUSH rule 0 x 780 [63,102]
+ CRUSH rule 0 x 781 [105,92]
+ CRUSH rule 0 x 782 [117,31]
+ CRUSH rule 0 x 783 [60,93]
+ CRUSH rule 0 x 784 [82,81]
+ CRUSH rule 0 x 785 [27,84]
+ CRUSH rule 0 x 786 [41,80]
+ CRUSH rule 0 x 787 [13,54]
+ CRUSH rule 0 x 788 [4,100]
+ CRUSH rule 0 x 789 [50,37]
+ CRUSH rule 0 x 790 [58,16]
+ CRUSH rule 0 x 791 [96,14]
+ CRUSH rule 0 x 792 [80,4]
+ CRUSH rule 0 x 793 [6,71]
+ CRUSH rule 0 x 794 [14,89]
+ CRUSH rule 0 x 795 [51,3]
+ CRUSH rule 0 x 796 [114,77]
+ CRUSH rule 0 x 797 [79,100]
+ CRUSH rule 0 x 798 [42,10]
+ CRUSH rule 0 x 799 [48,11]
+ CRUSH rule 0 x 800 [91,7]
+ CRUSH rule 0 x 801 [2,6]
+ CRUSH rule 0 x 802 [116,89]
+ CRUSH rule 0 x 803 [37,32]
+ CRUSH rule 0 x 804 [33,4]
+ CRUSH rule 0 x 805 [96,22]
+ CRUSH rule 0 x 806 [67,90]
+ CRUSH rule 0 x 807 [47,42]
+ CRUSH rule 0 x 808 [76,79]
+ CRUSH rule 0 x 809 [27,26]
+ CRUSH rule 0 x 810 [119,61]
+ CRUSH rule 0 x 811 [75,72]
+ CRUSH rule 0 x 812 [25,52]
+ CRUSH rule 0 x 813 [64,13]
+ CRUSH rule 0 x 814 [110,53]
+ CRUSH rule 0 x 815 [84,61]
+ CRUSH rule 0 x 816 [25,22]
+ CRUSH rule 0 x 817 [40,73]
+ CRUSH rule 0 x 818 [34,13]
+ CRUSH rule 0 x 819 [88,19]
+ CRUSH rule 0 x 820 [104,49]
+ CRUSH rule 0 x 821 [58,69]
+ CRUSH rule 0 x 822 [29,72]
+ CRUSH rule 0 x 823 [100,103]
+ CRUSH rule 0 x 824 [102,81]
+ CRUSH rule 0 x 825 [47,17]
+ CRUSH rule 0 x 826 [45,34]
+ CRUSH rule 0 x 827 [101,11]
+ CRUSH rule 0 x 828 [60,27]
+ CRUSH rule 0 x 829 [45,90]
+ CRUSH rule 0 x 830 [51,96]
+ CRUSH rule 0 x 831 [6,64]
+ CRUSH rule 0 x 832 [57,78]
+ CRUSH rule 0 x 833 [34,97]
+ CRUSH rule 0 x 834 [90,33]
+ CRUSH rule 0 x 835 [14,46]
+ CRUSH rule 0 x 836 [38,43]
+ CRUSH rule 0 x 837 [51,74]
+ CRUSH rule 0 x 838 [6,32]
+ CRUSH rule 0 x 839 [106,8]
+ CRUSH rule 0 x 840 [33,109]
+ CRUSH rule 0 x 841 [110,15]
+ CRUSH rule 0 x 842 [66,67]
+ CRUSH rule 0 x 843 [11,63]
+ CRUSH rule 0 x 844 [74,13]
+ CRUSH rule 0 x 845 [74,43]
+ CRUSH rule 0 x 846 [98,107]
+ CRUSH rule 0 x 847 [10,3]
+ CRUSH rule 0 x 848 [89,17]
+ CRUSH rule 0 x 849 [42,59]
+ CRUSH rule 0 x 850 [40,73]
+ CRUSH rule 0 x 851 [65,94]
+ CRUSH rule 0 x 852 [31,94]
+ CRUSH rule 0 x 853 [49,11]
+ CRUSH rule 0 x 854 [90,31]
+ CRUSH rule 0 x 855 [2,19]
+ CRUSH rule 0 x 856 [40,22]
+ CRUSH rule 0 x 857 [15,82]
+ CRUSH rule 0 x 858 [10,80]
+ CRUSH rule 0 x 859 [29,48]
+ CRUSH rule 0 x 860 [114,75]
+ CRUSH rule 0 x 861 [22,33]
+ CRUSH rule 0 x 862 [22,25]
+ CRUSH rule 0 x 863 [79,50]
+ CRUSH rule 0 x 864 [68,6]
+ CRUSH rule 0 x 865 [25,92]
+ CRUSH rule 0 x 866 [18,89]
+ CRUSH rule 0 x 867 [3,78]
+ CRUSH rule 0 x 868 [81,98]
+ CRUSH rule 0 x 869 [22,104]
+ CRUSH rule 0 x 870 [73,98]
+ CRUSH rule 0 x 871 [25,54]
+ CRUSH rule 0 x 872 [39,48]
+ CRUSH rule 0 x 873 [92,9]
+ CRUSH rule 0 x 874 [21,43]
+ CRUSH rule 0 x 875 [27,108]
+ CRUSH rule 0 x 876 [98,75]
+ CRUSH rule 0 x 877 [73,5]
+ CRUSH rule 0 x 878 [64,45]
+ CRUSH rule 0 x 879 [29,18]
+ CRUSH rule 0 x 880 [56,91]
+ CRUSH rule 0 x 881 [109,69]
+ CRUSH rule 0 x 882 [60,33]
+ CRUSH rule 0 x 883 [93,96]
+ CRUSH rule 0 x 884 [67,58]
+ CRUSH rule 0 x 885 [31,8]
+ CRUSH rule 0 x 886 [2,107]
+ CRUSH rule 0 x 887 [5,93]
+ CRUSH rule 0 x 888 [16,13]
+ CRUSH rule 0 x 889 [3,76]
+ CRUSH rule 0 x 890 [48,63]
+ CRUSH rule 0 x 891 [86,79]
+ CRUSH rule 0 x 892 [64,9]
+ CRUSH rule 0 x 893 [118,33]
+ CRUSH rule 0 x 894 [16,111]
+ CRUSH rule 0 x 895 [40,107]
+ CRUSH rule 0 x 896 [97,96]
+ CRUSH rule 0 x 897 [60,67]
+ CRUSH rule 0 x 898 [10,2]
+ CRUSH rule 0 x 899 [75,80]
+ CRUSH rule 0 x 900 [102,81]
+ CRUSH rule 0 x 901 [66,87]
+ CRUSH rule 0 x 902 [102,49]
+ CRUSH rule 0 x 903 [5,14]
+ CRUSH rule 0 x 904 [50,16]
+ CRUSH rule 0 x 905 [19,51]
+ CRUSH rule 0 x 906 [75,119]
+ CRUSH rule 0 x 907 [47,5]
+ CRUSH rule 0 x 908 [96,9]
+ CRUSH rule 0 x 909 [94,75]
+ CRUSH rule 0 x 910 [88,63]
+ CRUSH rule 0 x 911 [102,23]
+ CRUSH rule 0 x 912 [91,60]
+ CRUSH rule 0 x 913 [29,17]
+ CRUSH rule 0 x 914 [84,29]
+ CRUSH rule 0 x 915 [70,22]
+ CRUSH rule 0 x 916 [32,9]
+ CRUSH rule 0 x 917 [43,26]
+ CRUSH rule 0 x 918 [91,98]
+ CRUSH rule 0 x 919 [13,69]
+ CRUSH rule 0 x 920 [18,87]
+ CRUSH rule 0 x 921 [104,33]
+ CRUSH rule 0 x 922 [33,19]
+ CRUSH rule 0 x 923 [28,8]
+ CRUSH rule 0 x 924 [69,88]
+ CRUSH rule 0 x 925 [71,32]
+ CRUSH rule 0 x 926 [64,69]
+ CRUSH rule 0 x 927 [99,106]
+ CRUSH rule 0 x 928 [13,113]
+ CRUSH rule 0 x 929 [117,61]
+ CRUSH rule 0 x 930 [31,82]
+ CRUSH rule 0 x 931 [46,79]
+ CRUSH rule 0 x 932 [60,13]
+ CRUSH rule 0 x 933 [88,31]
+ CRUSH rule 0 x 934 [68,4]
+ CRUSH rule 0 x 935 [31,18]
+ CRUSH rule 0 x 936 [104,57]
+ CRUSH rule 0 x 937 [110,22]
+ CRUSH rule 0 x 938 [29,106]
+ CRUSH rule 0 x 939 [77,13]
+ CRUSH rule 0 x 940 [76,33]
+ CRUSH rule 0 x 941 [66,37]
+ CRUSH rule 0 x 942 [83,94]
+ CRUSH rule 0 x 943 [4,74]
+ CRUSH rule 0 x 944 [113,53]
+ CRUSH rule 0 x 945 [17,52]
+ CRUSH rule 0 x 946 [37,111]
+ CRUSH rule 0 x 947 [107,74]
+ CRUSH rule 0 x 948 [55,98]
+ CRUSH rule 0 x 949 [45,72]
+ CRUSH rule 0 x 950 [96,23]
+ CRUSH rule 0 x 951 [40,93]
+ CRUSH rule 0 x 952 [93,46]
+ CRUSH rule 0 x 953 [55,92]
+ CRUSH rule 0 x 954 [84,57]
+ CRUSH rule 0 x 955 [31,117]
+ CRUSH rule 0 x 956 [72,11]
+ CRUSH rule 0 x 957 [3,74]
+ CRUSH rule 0 x 958 [8,106]
+ CRUSH rule 0 x 959 [42,59]
+ CRUSH rule 0 x 960 [113,107]
+ CRUSH rule 0 x 961 [116,8]
+ CRUSH rule 0 x 962 [13,62]
+ CRUSH rule 0 x 963 [0,99]
+ CRUSH rule 0 x 964 [59,21]
+ CRUSH rule 0 x 965 [47,115]
+ CRUSH rule 0 x 966 [88,63]
+ CRUSH rule 0 x 967 [71,108]
+ CRUSH rule 0 x 968 [73,7]
+ CRUSH rule 0 x 969 [53,6]
+ CRUSH rule 0 x 970 [3,40]
+ CRUSH rule 0 x 971 [87,38]
+ CRUSH rule 0 x 972 [3,37]
+ CRUSH rule 0 x 973 [113,27]
+ CRUSH rule 0 x 974 [114,23]
+ CRUSH rule 0 x 975 [40,59]
+ CRUSH rule 0 x 976 [81,38]
+ CRUSH rule 0 x 977 [95,102]
+ CRUSH rule 0 x 978 [35,56]
+ CRUSH rule 0 x 979 [98,6]
+ CRUSH rule 0 x 980 [52,69]
+ CRUSH rule 0 x 981 [89,117]
+ CRUSH rule 0 x 982 [1,47]
+ CRUSH rule 0 x 983 [34,61]
+ CRUSH rule 0 x 984 [78,25]
+ CRUSH rule 0 x 985 [99,52]
+ CRUSH rule 0 x 986 [4,59]
+ CRUSH rule 0 x 987 [78,21]
+ CRUSH rule 0 x 988 [79,2]
+ CRUSH rule 0 x 989 [87,17]
+ CRUSH rule 0 x 990 [47,118]
+ CRUSH rule 0 x 991 [61,18]
+ CRUSH rule 0 x 992 [83,66]
+ CRUSH rule 0 x 993 [75,62]
+ CRUSH rule 0 x 994 [74,57]
+ CRUSH rule 0 x 995 [100,97]
+ CRUSH rule 0 x 996 [41,6]
+ CRUSH rule 0 x 997 [89,76]
+ CRUSH rule 0 x 998 [92,47]
+ CRUSH rule 0 x 999 [101,11]
+ CRUSH rule 0 x 1000 [9,119]
+ CRUSH rule 0 x 1001 [49,32]
+ CRUSH rule 0 x 1002 [99,113]
+ CRUSH rule 0 x 1003 [43,18]
+ CRUSH rule 0 x 1004 [89,54]
+ CRUSH rule 0 x 1005 [105,84]
+ CRUSH rule 0 x 1006 [45,111]
+ CRUSH rule 0 x 1007 [19,57]
+ CRUSH rule 0 x 1008 [31,24]
+ CRUSH rule 0 x 1009 [19,111]
+ CRUSH rule 0 x 1010 [42,89]
+ CRUSH rule 0 x 1011 [25,114]
+ CRUSH rule 0 x 1012 [68,71]
+ CRUSH rule 0 x 1013 [5,65]
+ CRUSH rule 0 x 1014 [33,4]
+ CRUSH rule 0 x 1015 [106,45]
+ CRUSH rule 0 x 1016 [88,39]
+ CRUSH rule 0 x 1017 [0,89]
+ CRUSH rule 0 x 1018 [63,5]
+ CRUSH rule 0 x 1019 [104,97]
+ CRUSH rule 0 x 1020 [96,9]
+ CRUSH rule 0 x 1021 [117,6]
+ CRUSH rule 0 x 1022 [73,21]
+ CRUSH rule 0 x 1023 [0,16]
+ rule 0 (data) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 3 result size == 2:\t2/1024 (esc)
+ rule 0 (data) num_rep 3 result size == 3:\t1022/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 4 result size == 2:\t2/1024 (esc)
+ rule 0 (data) num_rep 4 result size == 3:\t1022/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 5 result size == 2:\t2/1024 (esc)
+ rule 0 (data) num_rep 5 result size == 3:\t1022/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76,9]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 6 result size == 2:\t1/1024 (esc)
+ rule 0 (data) num_rep 6 result size == 3:\t1023/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76,9]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 7 result size == 2:\t1/1024 (esc)
+ rule 0 (data) num_rep 7 result size == 3:\t1023/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76,9]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 8 result size == 2:\t1/1024 (esc)
+ rule 0 (data) num_rep 8 result size == 3:\t1023/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76,9]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87,19]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 9 result size == 3:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [101,114,14]
+ CRUSH rule 0 x 1 [80,79,17]
+ CRUSH rule 0 x 2 [91,96,4]
+ CRUSH rule 0 x 3 [51,4,109]
+ CRUSH rule 0 x 4 [50,89,8]
+ CRUSH rule 0 x 5 [89,94,11]
+ CRUSH rule 0 x 6 [91,76,7]
+ CRUSH rule 0 x 7 [104,25,17]
+ CRUSH rule 0 x 8 [78,57,8]
+ CRUSH rule 0 x 9 [101,102,4]
+ CRUSH rule 0 x 10 [61,58,22]
+ CRUSH rule 0 x 11 [13,31,26]
+ CRUSH rule 0 x 12 [83,46,13]
+ CRUSH rule 0 x 13 [108,85,17]
+ CRUSH rule 0 x 14 [105,72,13]
+ CRUSH rule 0 x 15 [18,7,29]
+ CRUSH rule 0 x 16 [103,3,50]
+ CRUSH rule 0 x 17 [85,110,9]
+ CRUSH rule 0 x 18 [11,65,52]
+ CRUSH rule 0 x 19 [75,50,22]
+ CRUSH rule 0 x 20 [79,70,15]
+ CRUSH rule 0 x 21 [84,49,9]
+ CRUSH rule 0 x 22 [23,104,21]
+ CRUSH rule 0 x 23 [118,63,6]
+ CRUSH rule 0 x 24 [83,38,8]
+ CRUSH rule 0 x 25 [81,64,3]
+ CRUSH rule 0 x 26 [38,99,3]
+ CRUSH rule 0 x 27 [76,107,17]
+ CRUSH rule 0 x 28 [76,71,15]
+ CRUSH rule 0 x 29 [24,71,19]
+ CRUSH rule 0 x 30 [94,87,19]
+ CRUSH rule 0 x 31 [76,95,22]
+ CRUSH rule 0 x 32 [72,95,19]
+ CRUSH rule 0 x 33 [77,86,3]
+ CRUSH rule 0 x 34 [7,108,83]
+ CRUSH rule 0 x 35 [22,88,83]
+ CRUSH rule 0 x 36 [104,65,15]
+ CRUSH rule 0 x 37 [61,109,11]
+ CRUSH rule 0 x 38 [72,85,3]
+ CRUSH rule 0 x 39 [68,103,8]
+ CRUSH rule 0 x 40 [103,78,3]
+ CRUSH rule 0 x 41 [85,11,110]
+ CRUSH rule 0 x 42 [106,33,9]
+ CRUSH rule 0 x 43 [10,68,11]
+ CRUSH rule 0 x 44 [101,4,109]
+ CRUSH rule 0 x 45 [83,15,24]
+ CRUSH rule 0 x 46 [65,1,7]
+ CRUSH rule 0 x 47 [106,53,7]
+ CRUSH rule 0 x 48 [34,33,14]
+ CRUSH rule 0 x 49 [0,81,4]
+ CRUSH rule 0 x 50 [42,6,101]
+ CRUSH rule 0 x 51 [104,75,9]
+ CRUSH rule 0 x 52 [83,19,58]
+ CRUSH rule 0 x 53 [32,69,7]
+ CRUSH rule 0 x 54 [9,79,104]
+ CRUSH rule 0 x 55 [14,5,37]
+ CRUSH rule 0 x 56 [21,72,63]
+ CRUSH rule 0 x 57 [93,84,3]
+ CRUSH rule 0 x 58 [45,106,13]
+ CRUSH rule 0 x 59 [80,41,15]
+ CRUSH rule 0 x 60 [90,57,15]
+ CRUSH rule 0 x 61 [88,37,3]
+ CRUSH rule 0 x 62 [81,1,9]
+ CRUSH rule 0 x 63 [79,113,9]
+ CRUSH rule 0 x 64 [1,35,9]
+ CRUSH rule 0 x 65 [32,103,15]
+ CRUSH rule 0 x 66 [48,99,9]
+ CRUSH rule 0 x 67 [94,103,15]
+ CRUSH rule 0 x 68 [102,91,6]
+ CRUSH rule 0 x 69 [62,77,11]
+ CRUSH rule 0 x 70 [84,105,4]
+ CRUSH rule 0 x 71 [9,33,38]
+ CRUSH rule 0 x 72 [97,42,22]
+ CRUSH rule 0 x 73 [64,83,6]
+ CRUSH rule 0 x 74 [29,50,11]
+ CRUSH rule 0 x 75 [29,28,4]
+ CRUSH rule 0 x 76 [55,0,7]
+ CRUSH rule 0 x 77 [107,21,0]
+ CRUSH rule 0 x 78 [11,89,102]
+ CRUSH rule 0 x 79 [64,51,7]
+ CRUSH rule 0 x 80 [0,31,14]
+ CRUSH rule 0 x 81 [71,109,19]
+ CRUSH rule 0 x 82 [37,21,74]
+ CRUSH rule 0 x 83 [92,103,3]
+ CRUSH rule 0 x 84 [49,115,7]
+ CRUSH rule 0 x 85 [54,101,19]
+ CRUSH rule 0 x 86 [37,7,109]
+ CRUSH rule 0 x 87 [116,4,33]
+ CRUSH rule 0 x 88 [38,27,17]
+ CRUSH rule 0 x 89 [76,77,19]
+ CRUSH rule 0 x 90 [14,50,39]
+ CRUSH rule 0 x 91 [68,19,105]
+ CRUSH rule 0 x 92 [86,9,73]
+ CRUSH rule 0 x 93 [44,65,19]
+ CRUSH rule 0 x 94 [61,102,22]
+ CRUSH rule 0 x 95 [93,86,21]
+ CRUSH rule 0 x 96 [66,87,17]
+ CRUSH rule 0 x 97 [111,9,89]
+ CRUSH rule 0 x 98 [93,102,6]
+ CRUSH rule 0 x 99 [78,3,81]
+ CRUSH rule 0 x 100 [6,63,104]
+ CRUSH rule 0 x 101 [84,16,17]
+ CRUSH rule 0 x 102 [82,105,7]
+ CRUSH rule 0 x 103 [66,6,49]
+ CRUSH rule 0 x 104 [14,95,50]
+ CRUSH rule 0 x 105 [87,1,7]
+ CRUSH rule 0 x 106 [69,116,4]
+ CRUSH rule 0 x 107 [1,55,4]
+ CRUSH rule 0 x 108 [94,53,4]
+ CRUSH rule 0 x 109 [112,13,25]
+ CRUSH rule 0 x 110 [54,61,13]
+ CRUSH rule 0 x 111 [10,78,3]
+ CRUSH rule 0 x 112 [89,9,109]
+ CRUSH rule 0 x 113 [69,2,9]
+ CRUSH rule 0 x 114 [79,110,9]
+ CRUSH rule 0 x 115 [50,85,6]
+ CRUSH rule 0 x 116 [96,16,4]
+ CRUSH rule 0 x 117 [87,42,13]
+ CRUSH rule 0 x 118 [23,56,13]
+ CRUSH rule 0 x 119 [104,11,71]
+ CRUSH rule 0 x 120 [57,5,22]
+ CRUSH rule 0 x 121 [105,9,114]
+ CRUSH rule 0 x 122 [45,110,4]
+ CRUSH rule 0 x 123 [112,35,14]
+ CRUSH rule 0 x 124 [110,49,17]
+ CRUSH rule 0 x 125 [66,105,13]
+ CRUSH rule 0 x 126 [51,28,4]
+ CRUSH rule 0 x 127 [70,6,65]
+ CRUSH rule 0 x 128 [90,16,8]
+ CRUSH rule 0 x 129 [103,110,8]
+ CRUSH rule 0 x 130 [50,11,63]
+ CRUSH rule 0 x 131 [23,60,9]
+ CRUSH rule 0 x 132 [69,70,19]
+ CRUSH rule 0 x 133 [52,25,6]
+ CRUSH rule 0 x 134 [78,29,8]
+ CRUSH rule 0 x 135 [78,3,29]
+ CRUSH rule 0 x 136 [32,29,17]
+ CRUSH rule 0 x 137 [11,78,75]
+ CRUSH rule 0 x 138 [17,94,85]
+ CRUSH rule 0 x 139 [89,60,8]
+ CRUSH rule 0 x 140 [39,62,13]
+ CRUSH rule 0 x 141 [89,98,3]
+ CRUSH rule 0 x 142 [70,61,4]
+ CRUSH rule 0 x 143 [51,28,7]
+ CRUSH rule 0 x 144 [13,81,60]
+ CRUSH rule 0 x 145 [77,119,17]
+ CRUSH rule 0 x 146 [8,64,53]
+ CRUSH rule 0 x 147 [22,37,94]
+ CRUSH rule 0 x 148 [74,69,11]
+ CRUSH rule 0 x 149 [76,13,81]
+ CRUSH rule 0 x 150 [14,47,110]
+ CRUSH rule 0 x 151 [90,4,65]
+ CRUSH rule 0 x 152 [49,18,15]
+ CRUSH rule 0 x 153 [71,44,9]
+ CRUSH rule 0 x 154 [94,81,13]
+ CRUSH rule 0 x 155 [75,6,70]
+ CRUSH rule 0 x 156 [94,85,7]
+ CRUSH rule 0 x 157 [112,43,3]
+ CRUSH rule 0 x 158 [26,17,99]
+ CRUSH rule 0 x 159 [52,29,3]
+ CRUSH rule 0 x 160 [41,0,7]
+ CRUSH rule 0 x 161 [19,78,95]
+ CRUSH rule 0 x 162 [55,2,9]
+ CRUSH rule 0 x 163 [54,31,9]
+ CRUSH rule 0 x 164 [45,5,14]
+ CRUSH rule 0 x 165 [25,72,7]
+ CRUSH rule 0 x 166 [73,36,7]
+ CRUSH rule 0 x 167 [89,58,14]
+ CRUSH rule 0 x 168 [47,40,15]
+ CRUSH rule 0 x 169 [51,21,0]
+ CRUSH rule 0 x 170 [68,91,17]
+ CRUSH rule 0 x 171 [73,90,13]
+ CRUSH rule 0 x 172 [33,15,102]
+ CRUSH rule 0 x 173 [102,59,19]
+ CRUSH rule 0 x 174 [116,25,15]
+ CRUSH rule 0 x 175 [3,41,102]
+ CRUSH rule 0 x 176 [94,91,3]
+ CRUSH rule 0 x 177 [52,85,8]
+ CRUSH rule 0 x 178 [39,2,15]
+ CRUSH rule 0 x 179 [72,97,15]
+ CRUSH rule 0 x 180 [60,7,99]
+ CRUSH rule 0 x 181 [18,59,15]
+ CRUSH rule 0 x 182 [22,90,25]
+ CRUSH rule 0 x 183 [11,74,103]
+ CRUSH rule 0 x 184 [92,101,6]
+ CRUSH rule 0 x 185 [97,8,24]
+ CRUSH rule 0 x 186 [67,116,4]
+ CRUSH rule 0 x 187 [116,11,31]
+ CRUSH rule 0 x 188 [69,92,9]
+ CRUSH rule 0 x 189 [47,84,3]
+ CRUSH rule 0 x 190 [90,13,23]
+ CRUSH rule 0 x 191 [49,17,60]
+ CRUSH rule 0 x 192 [68,93,7]
+ CRUSH rule 0 x 193 [0,33,6]
+ CRUSH rule 0 x 194 [17,58,61]
+ CRUSH rule 0 x 195 [119,41,9]
+ CRUSH rule 0 x 196 [72,27,22]
+ CRUSH rule 0 x 197 [106,83,13]
+ CRUSH rule 0 x 198 [114,95,14]
+ CRUSH rule 0 x 199 [0,83,11]
+ CRUSH rule 0 x 200 [35,86,14]
+ CRUSH rule 0 x 201 [14,29,109]
+ CRUSH rule 0 x 202 [98,33,17]
+ CRUSH rule 0 x 203 [36,22,101]
+ CRUSH rule 0 x 204 [10,98,17]
+ CRUSH rule 0 x 205 [22,61,72]
+ CRUSH rule 0 x 206 [49,112,15]
+ CRUSH rule 0 x 207 [80,39,14]
+ CRUSH rule 0 x 208 [63,26,7]
+ CRUSH rule 0 x 209 [85,111,8]
+ CRUSH rule 0 x 210 [79,18,11]
+ CRUSH rule 0 x 211 [26,10,19]
+ CRUSH rule 0 x 212 [28,103,15]
+ CRUSH rule 0 x 213 [91,0,8]
+ CRUSH rule 0 x 214 [78,47,13]
+ CRUSH rule 0 x 215 [61,22,102]
+ CRUSH rule 0 x 216 [99,3,104]
+ CRUSH rule 0 x 217 [86,89,15]
+ CRUSH rule 0 x 218 [93,96,4]
+ CRUSH rule 0 x 219 [28,59,6]
+ CRUSH rule 0 x 220 [56,8,83]
+ CRUSH rule 0 x 221 [0,9,71]
+ CRUSH rule 0 x 222 [50,63,21]
+ CRUSH rule 0 x 223 [29,1,15]
+ CRUSH rule 0 x 224 [52,10,19]
+ CRUSH rule 0 x 225 [61,11,64]
+ CRUSH rule 0 x 226 [44,22,93]
+ CRUSH rule 0 x 227 [42,3,81]
+ CRUSH rule 0 x 228 [117,49,22]
+ CRUSH rule 0 x 229 [100,79,9]
+ CRUSH rule 0 x 230 [41,114,11]
+ CRUSH rule 0 x 231 [56,95,8]
+ CRUSH rule 0 x 232 [23,44,11]
+ CRUSH rule 0 x 233 [88,103,21]
+ CRUSH rule 0 x 234 [4,101,18]
+ CRUSH rule 0 x 235 [26,10,11]
+ CRUSH rule 0 x 236 [32,37,3]
+ CRUSH rule 0 x 237 [92,3,61]
+ CRUSH rule 0 x 238 [10,26,22]
+ CRUSH rule 0 x 239 [15,105,2]
+ CRUSH rule 0 x 240 [109,85,14]
+ CRUSH rule 0 x 241 [47,108,15]
+ CRUSH rule 0 x 242 [24,99,9]
+ CRUSH rule 0 x 243 [76,8,99]
+ CRUSH rule 0 x 244 [96,19,105]
+ CRUSH rule 0 x 245 [27,28,19]
+ CRUSH rule 0 x 246 [35,82,19]
+ CRUSH rule 0 x 247 [99,102,4]
+ CRUSH rule 0 x 248 [8,29,42]
+ CRUSH rule 0 x 249 [85,1,13]
+ CRUSH rule 0 x 250 [79,102,13]
+ CRUSH rule 0 x 251 [28,103,19]
+ CRUSH rule 0 x 252 [95,22,92]
+ CRUSH rule 0 x 253 [109,27,17]
+ CRUSH rule 0 x 254 [80,103,3]
+ CRUSH rule 0 x 255 [112,22,85]
+ CRUSH rule 0 x 256 [37,38,11]
+ CRUSH rule 0 x 257 [69,117,9]
+ CRUSH rule 0 x 258 [34,55,19]
+ CRUSH rule 0 x 259 [70,17,91]
+ CRUSH rule 0 x 260 [98,29,4]
+ CRUSH rule 0 x 261 [94,83,22]
+ CRUSH rule 0 x 262 [42,49,14]
+ CRUSH rule 0 x 263 [65,42,14]
+ CRUSH rule 0 x 264 [36,17,107]
+ CRUSH rule 0 x 265 [66,63,4]
+ CRUSH rule 0 x 266 [75,92,7]
+ CRUSH rule 0 x 267 [58,35,6]
+ CRUSH rule 0 x 268 [38,9,63]
+ CRUSH rule 0 x 269 [43,104,7]
+ CRUSH rule 0 x 270 [58,37,4]
+ CRUSH rule 0 x 271 [19,33,114]
+ CRUSH rule 0 x 272 [73,9,100]
+ CRUSH rule 0 x 273 [108,29,22]
+ CRUSH rule 0 x 274 [47,64,22]
+ CRUSH rule 0 x 275 [92,19,43]
+ CRUSH rule 0 x 276 [7,79,118]
+ CRUSH rule 0 x 277 [19,68,10]
+ CRUSH rule 0 x 278 [116,95,19]
+ CRUSH rule 0 x 279 [101,3,76]
+ CRUSH rule 0 x 280 [113,69,4]
+ CRUSH rule 0 x 281 [14,93,96]
+ CRUSH rule 0 x 282 [106,7,47]
+ CRUSH rule 0 x 283 [8,118,101]
+ CRUSH rule 0 x 284 [10,110,22]
+ CRUSH rule 0 x 285 [88,63,15]
+ CRUSH rule 0 x 286 [27,4,18]
+ CRUSH rule 0 x 287 [84,65,4]
+ CRUSH rule 0 x 288 [103,8,70]
+ CRUSH rule 0 x 289 [9,104,45]
+ CRUSH rule 0 x 290 [115,7,101]
+ CRUSH rule 0 x 291 [48,45,13]
+ CRUSH rule 0 x 292 [52,16,14]
+ CRUSH rule 0 x 293 [27,24,17]
+ CRUSH rule 0 x 294 [79,36,13]
+ CRUSH rule 0 x 295 [37,116,7]
+ CRUSH rule 0 x 296 [56,61,7]
+ CRUSH rule 0 x 297 [35,40,9]
+ CRUSH rule 0 x 298 [71,118,8]
+ CRUSH rule 0 x 299 [79,1,19]
+ CRUSH rule 0 x 300 [67,5,9]
+ CRUSH rule 0 x 301 [51,110,8]
+ CRUSH rule 0 x 302 [78,67,19]
+ CRUSH rule 0 x 303 [19,94,31]
+ CRUSH rule 0 x 304 [101,66,13]
+ CRUSH rule 0 x 305 [81,62,6]
+ CRUSH rule 0 x 306 [0,23,9]
+ CRUSH rule 0 x 307 [44,15,95]
+ CRUSH rule 0 x 308 [91,98,21]
+ CRUSH rule 0 x 309 [15,18,99]
+ CRUSH rule 0 x 310 [26,89,11]
+ CRUSH rule 0 x 311 [36,41,9]
+ CRUSH rule 0 x 312 [33,22,113]
+ CRUSH rule 0 x 313 [104,16,3]
+ CRUSH rule 0 x 314 [28,4,23]
+ CRUSH rule 0 x 315 [16,8,96]
+ CRUSH rule 0 x 316 [4,1,79]
+ CRUSH rule 0 x 317 [118,8,31]
+ CRUSH rule 0 x 318 [32,47,7]
+ CRUSH rule 0 x 319 [24,83,4]
+ CRUSH rule 0 x 320 [36,97,17]
+ CRUSH rule 0 x 321 [26,85,11]
+ CRUSH rule 0 x 322 [87,42,21]
+ CRUSH rule 0 x 323 [73,0,13]
+ CRUSH rule 0 x 324 [64,37,21]
+ CRUSH rule 0 x 325 [52,16,3]
+ CRUSH rule 0 x 326 [111,93,13]
+ CRUSH rule 0 x 327 [62,16,19]
+ CRUSH rule 0 x 328 [7,42,67]
+ CRUSH rule 0 x 329 [93,34,11]
+ CRUSH rule 0 x 330 [24,4,63]
+ CRUSH rule 0 x 331 [41,21,111]
+ CRUSH rule 0 x 332 [61,110,3]
+ CRUSH rule 0 x 333 [16,8,116]
+ CRUSH rule 0 x 334 [94,35,15]
+ CRUSH rule 0 x 335 [71,74,7]
+ CRUSH rule 0 x 336 [16,19,66]
+ CRUSH rule 0 x 337 [37,11,52]
+ CRUSH rule 0 x 338 [109,69,13]
+ CRUSH rule 0 x 339 [13,64,93]
+ CRUSH rule 0 x 340 [119,15,107]
+ CRUSH rule 0 x 341 [63,114,14]
+ CRUSH rule 0 x 342 [92,25,17]
+ CRUSH rule 0 x 343 [49,26,17]
+ CRUSH rule 0 x 344 [103,26,7]
+ CRUSH rule 0 x 345 [56,25,8]
+ CRUSH rule 0 x 346 [3,79,24]
+ CRUSH rule 0 x 347 [106,27,21]
+ CRUSH rule 0 x 348 [10,117,19]
+ CRUSH rule 0 x 349 [96,37,8]
+ CRUSH rule 0 x 350 [63,32,9]
+ CRUSH rule 0 x 351 [60,85,22]
+ CRUSH rule 0 x 352 [103,84,17]
+ CRUSH rule 0 x 353 [10,113,13]
+ CRUSH rule 0 x 354 [55,52,11]
+ CRUSH rule 0 x 355 [73,68,14]
+ CRUSH rule 0 x 356 [114,41,14]
+ CRUSH rule 0 x 357 [70,13,75]
+ CRUSH rule 0 x 358 [97,13,42]
+ CRUSH rule 0 x 359 [4,117,87]
+ CRUSH rule 0 x 360 [106,69,15]
+ CRUSH rule 0 x 361 [27,46,6]
+ CRUSH rule 0 x 362 [28,33,17]
+ CRUSH rule 0 x 363 [45,26,6]
+ CRUSH rule 0 x 364 [23,50,4]
+ CRUSH rule 0 x 365 [57,114,19]
+ CRUSH rule 0 x 366 [14,58,16]
+ CRUSH rule 0 x 367 [108,65,8]
+ CRUSH rule 0 x 368 [103,32,3]
+ CRUSH rule 0 x 369 [11,57,110]
+ CRUSH rule 0 x 370 [11,89,66]
+ CRUSH rule 0 x 371 [34,55,19]
+ CRUSH rule 0 x 372 [58,10,9]
+ CRUSH rule 0 x 373 [6,42,27]
+ CRUSH rule 0 x 374 [110,95,4]
+ CRUSH rule 0 x 375 [19,92,103]
+ CRUSH rule 0 x 376 [22,86,91]
+ CRUSH rule 0 x 377 [93,113,11]
+ CRUSH rule 0 x 378 [67,36,15]
+ CRUSH rule 0 x 379 [77,115,7]
+ CRUSH rule 0 x 380 [3,108,83]
+ CRUSH rule 0 x 381 [55,1,14]
+ CRUSH rule 0 x 382 [26,51,17]
+ CRUSH rule 0 x 383 [48,25,13]
+ CRUSH rule 0 x 384 [15,100,81]
+ CRUSH rule 0 x 385 [82,4,67]
+ CRUSH rule 0 x 386 [108,63,11]
+ CRUSH rule 0 x 387 [70,41,21]
+ CRUSH rule 0 x 388 [5,67,19]
+ CRUSH rule 0 x 389 [14,1,45]
+ CRUSH rule 0 x 390 [68,10,13]
+ CRUSH rule 0 x 391 [113,14,27]
+ CRUSH rule 0 x 392 [72,14,77]
+ CRUSH rule 0 x 393 [115,6,81]
+ CRUSH rule 0 x 394 [38,21,16]
+ CRUSH rule 0 x 395 [0,27,13]
+ CRUSH rule 0 x 396 [59,92,11]
+ CRUSH rule 0 x 397 [87,1,7]
+ CRUSH rule 0 x 398 [44,75,14]
+ CRUSH rule 0 x 399 [9,2,95]
+ CRUSH rule 0 x 400 [19,63,98]
+ CRUSH rule 0 x 401 [79,34,11]
+ CRUSH rule 0 x 402 [107,98,8]
+ CRUSH rule 0 x 403 [23,82,13]
+ CRUSH rule 0 x 404 [76,75,7]
+ CRUSH rule 0 x 405 [10,32,15]
+ CRUSH rule 0 x 406 [38,16,7]
+ CRUSH rule 0 x 407 [70,85,9]
+ CRUSH rule 0 x 408 [55,72,14]
+ CRUSH rule 0 x 409 [102,15,73]
+ CRUSH rule 0 x 410 [59,13,118]
+ CRUSH rule 0 x 411 [34,29,21]
+ CRUSH rule 0 x 412 [108,99,9]
+ CRUSH rule 0 x 413 [54,107,8]
+ CRUSH rule 0 x 414 [70,4,73]
+ CRUSH rule 0 x 415 [107,36,13]
+ CRUSH rule 0 x 416 [21,68,57]
+ CRUSH rule 0 x 417 [8,70,61]
+ CRUSH rule 0 x 418 [51,46,3]
+ CRUSH rule 0 x 419 [8,66,79]
+ CRUSH rule 0 x 420 [109,105,7]
+ CRUSH rule 0 x 421 [114,17,67]
+ CRUSH rule 0 x 422 [109,87,17]
+ CRUSH rule 0 x 423 [59,98,9]
+ CRUSH rule 0 x 424 [71,5,17]
+ CRUSH rule 0 x 425 [101,111,15]
+ CRUSH rule 0 x 426 [47,46,19]
+ CRUSH rule 0 x 427 [8,115,65]
+ CRUSH rule 0 x 428 [68,103,21]
+ CRUSH rule 0 x 429 [76,6,75]
+ CRUSH rule 0 x 430 [69,86,13]
+ CRUSH rule 0 x 431 [70,83,17]
+ CRUSH rule 0 x 432 [46,37,19]
+ CRUSH rule 0 x 433 [6,101,68]
+ CRUSH rule 0 x 434 [64,69,4]
+ CRUSH rule 0 x 435 [16,50,6]
+ CRUSH rule 0 x 436 [89,102,21]
+ CRUSH rule 0 x 437 [29,114,9]
+ CRUSH rule 0 x 438 [105,98,6]
+ CRUSH rule 0 x 439 [29,119,7]
+ CRUSH rule 0 x 440 [38,7,87]
+ CRUSH rule 0 x 441 [112,105,13]
+ CRUSH rule 0 x 442 [55,108,21]
+ CRUSH rule 0 x 443 [44,57,9]
+ CRUSH rule 0 x 444 [72,27,9]
+ CRUSH rule 0 x 445 [19,5,39]
+ CRUSH rule 0 x 446 [40,47,7]
+ CRUSH rule 0 x 447 [13,61,90]
+ CRUSH rule 0 x 448 [7,68,55]
+ CRUSH rule 0 x 449 [67,19,66]
+ CRUSH rule 0 x 450 [117,79,17]
+ CRUSH rule 0 x 451 [93,108,8]
+ CRUSH rule 0 x 452 [70,49,11]
+ CRUSH rule 0 x 453 [82,22,59]
+ CRUSH rule 0 x 454 [53,18,21]
+ CRUSH rule 0 x 455 [91,92,3]
+ CRUSH rule 0 x 456 [101,104,9]
+ CRUSH rule 0 x 457 [113,51,4]
+ CRUSH rule 0 x 458 [53,34,21]
+ CRUSH rule 0 x 459 [25,115,11]
+ CRUSH rule 0 x 460 [105,9,74]
+ CRUSH rule 0 x 461 [102,35,13]
+ CRUSH rule 0 x 462 [98,107,8]
+ CRUSH rule 0 x 463 [108,105,11]
+ CRUSH rule 0 x 464 [19,109,105]
+ CRUSH rule 0 x 465 [29,86,21]
+ CRUSH rule 0 x 466 [66,7,16]
+ CRUSH rule 0 x 467 [6,57,44]
+ CRUSH rule 0 x 468 [97,26,7]
+ CRUSH rule 0 x 469 [98,75,9]
+ CRUSH rule 0 x 470 [50,3,45]
+ CRUSH rule 0 x 471 [40,79,17]
+ CRUSH rule 0 x 472 [74,79,6]
+ CRUSH rule 0 x 473 [95,21,36]
+ CRUSH rule 0 x 474 [51,32,15]
+ CRUSH rule 0 x 475 [49,110,22]
+ CRUSH rule 0 x 476 [110,31,11]
+ CRUSH rule 0 x 477 [25,106,7]
+ CRUSH rule 0 x 478 [47,46,6]
+ CRUSH rule 0 x 479 [70,37,6]
+ CRUSH rule 0 x 480 [62,57,6]
+ CRUSH rule 0 x 481 [26,19,49]
+ CRUSH rule 0 x 482 [84,85,11]
+ CRUSH rule 0 x 483 [15,116,63]
+ CRUSH rule 0 x 484 [37,36,8]
+ CRUSH rule 0 x 485 [47,117,17]
+ CRUSH rule 0 x 486 [92,10,6]
+ CRUSH rule 0 x 487 [106,51,11]
+ CRUSH rule 0 x 488 [42,9,87]
+ CRUSH rule 0 x 489 [76,16,21]
+ CRUSH rule 0 x 490 [68,17,101]
+ CRUSH rule 0 x 491 [80,71,8]
+ CRUSH rule 0 x 492 [21,57,86]
+ CRUSH rule 0 x 493 [99,78,14]
+ CRUSH rule 0 x 494 [4,87,114]
+ CRUSH rule 0 x 495 [40,43,17]
+ CRUSH rule 0 x 496 [93,38,3]
+ CRUSH rule 0 x 497 [102,71,6]
+ CRUSH rule 0 x 498 [68,83,3]
+ CRUSH rule 0 x 499 [10,26,7]
+ CRUSH rule 0 x 500 [50,6,95]
+ CRUSH rule 0 x 501 [60,9,103]
+ CRUSH rule 0 x 502 [11,64,53]
+ CRUSH rule 0 x 503 [117,25,14]
+ CRUSH rule 0 x 504 [90,41,9]
+ CRUSH rule 0 x 505 [91,100,21]
+ CRUSH rule 0 x 506 [82,103,14]
+ CRUSH rule 0 x 507 [81,54,6]
+ CRUSH rule 0 x 508 [34,87,19]
+ CRUSH rule 0 x 509 [88,63,8]
+ CRUSH rule 0 x 510 [11,73,106]
+ CRUSH rule 0 x 511 [72,27,21]
+ CRUSH rule 0 x 512 [118,73,13]
+ CRUSH rule 0 x 513 [22,76,77]
+ CRUSH rule 0 x 514 [82,11,29]
+ CRUSH rule 0 x 515 [27,0,22]
+ CRUSH rule 0 x 516 [66,13,43]
+ CRUSH rule 0 x 517 [83,60,8]
+ CRUSH rule 0 x 518 [18,3,83]
+ CRUSH rule 0 x 519 [67,119,14]
+ CRUSH rule 0 x 520 [15,88,53]
+ CRUSH rule 0 x 521 [63,113,7]
+ CRUSH rule 0 x 522 [56,73,19]
+ CRUSH rule 0 x 523 [36,35,3]
+ CRUSH rule 0 x 524 [33,38,13]
+ CRUSH rule 0 x 525 [3,119,45]
+ CRUSH rule 0 x 526 [83,50,3]
+ CRUSH rule 0 x 527 [37,0,11]
+ CRUSH rule 0 x 528 [108,87,15]
+ CRUSH rule 0 x 529 [107,60,4]
+ CRUSH rule 0 x 530 [49,3,56]
+ CRUSH rule 0 x 531 [27,104,21]
+ CRUSH rule 0 x 532 [68,14,107]
+ CRUSH rule 0 x 533 [5,85,3]
+ CRUSH rule 0 x 534 [97,24,19]
+ CRUSH rule 0 x 535 [8,75,88]
+ CRUSH rule 0 x 536 [3,37,86]
+ CRUSH rule 0 x 537 [116,7,59]
+ CRUSH rule 0 x 538 [85,56,17]
+ CRUSH rule 0 x 539 [10,9,117]
+ CRUSH rule 0 x 540 [100,101,14]
+ CRUSH rule 0 x 541 [111,77,11]
+ CRUSH rule 0 x 542 [50,27,13]
+ CRUSH rule 0 x 543 [45,21,109]
+ CRUSH rule 0 x 544 [106,65,21]
+ CRUSH rule 0 x 545 [43,114,17]
+ CRUSH rule 0 x 546 [108,79,17]
+ CRUSH rule 0 x 547 [67,50,4]
+ CRUSH rule 0 x 548 [58,61,6]
+ CRUSH rule 0 x 549 [60,22,89]
+ CRUSH rule 0 x 550 [47,68,21]
+ CRUSH rule 0 x 551 [14,88,59]
+ CRUSH rule 0 x 552 [70,65,22]
+ CRUSH rule 0 x 553 [96,105,9]
+ CRUSH rule 0 x 554 [61,94,22]
+ CRUSH rule 0 x 555 [76,37,9]
+ CRUSH rule 0 x 556 [106,89,9]
+ CRUSH rule 0 x 557 [39,113,17]
+ CRUSH rule 0 x 558 [70,79,8]
+ CRUSH rule 0 x 559 [106,69,14]
+ CRUSH rule 0 x 560 [94,97,8]
+ CRUSH rule 0 x 561 [27,76,9]
+ CRUSH rule 0 x 562 [97,62,7]
+ CRUSH rule 0 x 563 [64,103,15]
+ CRUSH rule 0 x 564 [96,41,14]
+ CRUSH rule 0 x 565 [66,71,19]
+ CRUSH rule 0 x 566 [27,38,11]
+ CRUSH rule 0 x 567 [88,8,25]
+ CRUSH rule 0 x 568 [106,17,33]
+ CRUSH rule 0 x 569 [102,63,17]
+ CRUSH rule 0 x 570 [98,27,19]
+ CRUSH rule 0 x 571 [95,98,4]
+ CRUSH rule 0 x 572 [62,83,7]
+ CRUSH rule 0 x 573 [51,118,4]
+ CRUSH rule 0 x 574 [89,78,13]
+ CRUSH rule 0 x 575 [87,19,38]
+ CRUSH rule 0 x 576 [112,73,19]
+ CRUSH rule 0 x 577 [8,84,41]
+ CRUSH rule 0 x 578 [64,99,7]
+ CRUSH rule 0 x 579 [78,77,17]
+ CRUSH rule 0 x 580 [68,95,7]
+ CRUSH rule 0 x 581 [55,52,7]
+ CRUSH rule 0 x 582 [15,113,77]
+ CRUSH rule 0 x 583 [74,105,15]
+ CRUSH rule 0 x 584 [22,92,87]
+ CRUSH rule 0 x 585 [35,1,15]
+ CRUSH rule 0 x 586 [33,1,13]
+ CRUSH rule 0 x 587 [106,99,22]
+ CRUSH rule 0 x 588 [0,83,7]
+ CRUSH rule 0 x 589 [7,95,90]
+ CRUSH rule 0 x 590 [40,69,4]
+ CRUSH rule 0 x 591 [42,23,11]
+ CRUSH rule 0 x 592 [45,22,108]
+ CRUSH rule 0 x 593 [89,14,42]
+ CRUSH rule 0 x 594 [27,76,9]
+ CRUSH rule 0 x 595 [7,10,34]
+ CRUSH rule 0 x 596 [82,59,19]
+ CRUSH rule 0 x 597 [72,83,9]
+ CRUSH rule 0 x 598 [34,19,69]
+ CRUSH rule 0 x 599 [119,61,7]
+ CRUSH rule 0 x 600 [24,27,21]
+ CRUSH rule 0 x 601 [104,15,49]
+ CRUSH rule 0 x 602 [48,45,3]
+ CRUSH rule 0 x 603 [24,13,41]
+ CRUSH rule 0 x 604 [89,0,14]
+ CRUSH rule 0 x 605 [104,87,13]
+ CRUSH rule 0 x 606 [49,34,13]
+ CRUSH rule 0 x 607 [95,40,15]
+ CRUSH rule 0 x 608 [112,91,6]
+ CRUSH rule 0 x 609 [61,66,11]
+ CRUSH rule 0 x 610 [106,16,14]
+ CRUSH rule 0 x 611 [66,87,3]
+ CRUSH rule 0 x 612 [103,8,44]
+ CRUSH rule 0 x 613 [13,91,96]
+ CRUSH rule 0 x 614 [81,88,11]
+ CRUSH rule 0 x 615 [61,19,64]
+ CRUSH rule 0 x 616 [41,15,106]
+ CRUSH rule 0 x 617 [111,69,15]
+ CRUSH rule 0 x 618 [26,99,9]
+ CRUSH rule 0 x 619 [92,27,19]
+ CRUSH rule 0 x 620 [108,103,15]
+ CRUSH rule 0 x 621 [106,99,3]
+ CRUSH rule 0 x 622 [67,48,14]
+ CRUSH rule 0 x 623 [94,61,15]
+ CRUSH rule 0 x 624 [115,59,15]
+ CRUSH rule 0 x 625 [111,27,19]
+ CRUSH rule 0 x 626 [3,55,80]
+ CRUSH rule 0 x 627 [19,29,90]
+ CRUSH rule 0 x 628 [65,88,7]
+ CRUSH rule 0 x 629 [6,46,87]
+ CRUSH rule 0 x 630 [22,72,55]
+ CRUSH rule 0 x 631 [35,22,94]
+ CRUSH rule 0 x 632 [81,0,14]
+ CRUSH rule 0 x 633 [65,68,13]
+ CRUSH rule 0 x 634 [87,50,7]
+ CRUSH rule 0 x 635 [40,73,13]
+ CRUSH rule 0 x 636 [23,70,3]
+ CRUSH rule 0 x 637 [102,45,3]
+ CRUSH rule 0 x 638 [43,114,19]
+ CRUSH rule 0 x 639 [31,78,11]
+ CRUSH rule 0 x 640 [113,73,22]
+ CRUSH rule 0 x 641 [45,96,3]
+ CRUSH rule 0 x 642 [47,66,3]
+ CRUSH rule 0 x 643 [64,47,21]
+ CRUSH rule 0 x 644 [31,21,119]
+ CRUSH rule 0 x 645 [76,43,6]
+ CRUSH rule 0 x 646 [37,54,8]
+ CRUSH rule 0 x 647 [58,87,19]
+ CRUSH rule 0 x 648 [31,21,102]
+ CRUSH rule 0 x 649 [88,45,14]
+ CRUSH rule 0 x 650 [116,7,107]
+ CRUSH rule 0 x 651 [97,106,3]
+ CRUSH rule 0 x 652 [57,112,9]
+ CRUSH rule 0 x 653 [8,116,97]
+ CRUSH rule 0 x 654 [49,32,7]
+ CRUSH rule 0 x 655 [89,62,17]
+ CRUSH rule 0 x 656 [0,49,22]
+ CRUSH rule 0 x 657 [47,17,58]
+ CRUSH rule 0 x 658 [75,82,17]
+ CRUSH rule 0 x 659 [26,83,8]
+ CRUSH rule 0 x 660 [65,112,13]
+ CRUSH rule 0 x 661 [91,48,3]
+ CRUSH rule 0 x 662 [111,99,17]
+ CRUSH rule 0 x 663 [88,35,3]
+ CRUSH rule 0 x 664 [59,78,8]
+ CRUSH rule 0 x 665 [78,15,67]
+ CRUSH rule 0 x 666 [112,4,61]
+ CRUSH rule 0 x 667 [97,46,8]
+ CRUSH rule 0 x 668 [97,8,56]
+ CRUSH rule 0 x 669 [85,66,3]
+ CRUSH rule 0 x 670 [41,48,14]
+ CRUSH rule 0 x 671 [116,97,13]
+ CRUSH rule 0 x 672 [44,55,17]
+ CRUSH rule 0 x 673 [83,50,14]
+ CRUSH rule 0 x 674 [36,8,65]
+ CRUSH rule 0 x 675 [88,14,43]
+ CRUSH rule 0 x 676 [62,8,99]
+ CRUSH rule 0 x 677 [88,67,8]
+ CRUSH rule 0 x 678 [98,83,3]
+ CRUSH rule 0 x 679 [33,78,3]
+ CRUSH rule 0 x 680 [55,94,17]
+ CRUSH rule 0 x 681 [115,95,3]
+ CRUSH rule 0 x 682 [27,94,15]
+ CRUSH rule 0 x 683 [57,80,9]
+ CRUSH rule 0 x 684 [22,65,44]
+ CRUSH rule 0 x 685 [106,55,8]
+ CRUSH rule 0 x 686 [86,95,4]
+ CRUSH rule 0 x 687 [32,57,13]
+ CRUSH rule 0 x 688 [80,22,49]
+ CRUSH rule 0 x 689 [6,48,71]
+ CRUSH rule 0 x 690 [43,70,14]
+ CRUSH rule 0 x 691 [34,105,4]
+ CRUSH rule 0 x 692 [40,97,13]
+ CRUSH rule 0 x 693 [29,84,21]
+ CRUSH rule 0 x 694 [6,84,57]
+ CRUSH rule 0 x 695 [19,69,112]
+ CRUSH rule 0 x 696 [36,75,11]
+ CRUSH rule 0 x 697 [96,99,14]
+ CRUSH rule 0 x 698 [61,11,84]
+ CRUSH rule 0 x 699 [47,62,15]
+ CRUSH rule 0 x 700 [99,82,22]
+ CRUSH rule 0 x 701 [42,11,91]
+ CRUSH rule 0 x 702 [0,71,22]
+ CRUSH rule 0 x 703 [92,3,89]
+ CRUSH rule 0 x 704 [10,19,88]
+ CRUSH rule 0 x 705 [105,21,2]
+ CRUSH rule 0 x 706 [74,105,13]
+ CRUSH rule 0 x 707 [0,77,15]
+ CRUSH rule 0 x 708 [84,8,39]
+ CRUSH rule 0 x 709 [114,97,19]
+ CRUSH rule 0 x 710 [94,7,33]
+ CRUSH rule 0 x 711 [68,49,8]
+ CRUSH rule 0 x 712 [34,75,11]
+ CRUSH rule 0 x 713 [29,0,21]
+ CRUSH rule 0 x 714 [81,115,3]
+ CRUSH rule 0 x 715 [71,84,6]
+ CRUSH rule 0 x 716 [40,17,69]
+ CRUSH rule 0 x 717 [61,62,14]
+ CRUSH rule 0 x 718 [40,85,13]
+ CRUSH rule 0 x 719 [59,42,3]
+ CRUSH rule 0 x 720 [69,72,14]
+ CRUSH rule 0 x 721 [62,21,35]
+ CRUSH rule 0 x 722 [115,8,43]
+ CRUSH rule 0 x 723 [117,41,13]
+ CRUSH rule 0 x 724 [45,102,4]
+ CRUSH rule 0 x 725 [53,113,13]
+ CRUSH rule 0 x 726 [84,19,103]
+ CRUSH rule 0 x 727 [109,14,31]
+ CRUSH rule 0 x 728 [76,16,11]
+ CRUSH rule 0 x 729 [108,47,11]
+ CRUSH rule 0 x 730 [28,47,21]
+ CRUSH rule 0 x 731 [78,37,14]
+ CRUSH rule 0 x 732 [55,90,4]
+ CRUSH rule 0 x 733 [84,3,99]
+ CRUSH rule 0 x 734 [27,117,4]
+ CRUSH rule 0 x 735 [83,4,54]
+ CRUSH rule 0 x 736 [70,67,21]
+ CRUSH rule 0 x 737 [117,15,101]
+ CRUSH rule 0 x 738 [118,22,65]
+ CRUSH rule 0 x 739 [87,38,11]
+ CRUSH rule 0 x 740 [29,38,19]
+ CRUSH rule 0 x 741 [96,73,4]
+ CRUSH rule 0 x 742 [106,83,8]
+ CRUSH rule 0 x 743 [105,94,9]
+ CRUSH rule 0 x 744 [23,14,78]
+ CRUSH rule 0 x 745 [28,6,87]
+ CRUSH rule 0 x 746 [56,47,13]
+ CRUSH rule 0 x 747 [65,70,19]
+ CRUSH rule 0 x 748 [48,89,17]
+ CRUSH rule 0 x 749 [102,51,6]
+ CRUSH rule 0 x 750 [50,3,59]
+ CRUSH rule 0 x 751 [36,25,9]
+ CRUSH rule 0 x 752 [69,52,15]
+ CRUSH rule 0 x 753 [116,65,21]
+ CRUSH rule 0 x 754 [9,57,40]
+ CRUSH rule 0 x 755 [98,81,4]
+ CRUSH rule 0 x 756 [113,8,43]
+ CRUSH rule 0 x 757 [47,66,14]
+ CRUSH rule 0 x 758 [57,88,4]
+ CRUSH rule 0 x 759 [74,97,6]
+ CRUSH rule 0 x 760 [53,90,8]
+ CRUSH rule 0 x 761 [78,97,7]
+ CRUSH rule 0 x 762 [87,104,8]
+ CRUSH rule 0 x 763 [13,45,92]
+ CRUSH rule 0 x 764 [106,81,22]
+ CRUSH rule 0 x 765 [109,91,6]
+ CRUSH rule 0 x 766 [76,97,7]
+ CRUSH rule 0 x 767 [41,116,6]
+ CRUSH rule 0 x 768 [13,114,57]
+ CRUSH rule 0 x 769 [91,96,13]
+ CRUSH rule 0 x 770 [105,19,104]
+ CRUSH rule 0 x 771 [10,76,17]
+ CRUSH rule 0 x 772 [118,17,69]
+ CRUSH rule 0 x 773 [116,75,6]
+ CRUSH rule 0 x 774 [100,43,19]
+ CRUSH rule 0 x 775 [102,43,13]
+ CRUSH rule 0 x 776 [69,38,14]
+ CRUSH rule 0 x 777 [76,49,17]
+ CRUSH rule 0 x 778 [38,13,89]
+ CRUSH rule 0 x 779 [46,21,29]
+ CRUSH rule 0 x 780 [63,102,6]
+ CRUSH rule 0 x 781 [105,92,22]
+ CRUSH rule 0 x 782 [117,31,13]
+ CRUSH rule 0 x 783 [60,93,13]
+ CRUSH rule 0 x 784 [82,81,15]
+ CRUSH rule 0 x 785 [27,84,8]
+ CRUSH rule 0 x 786 [41,80,19]
+ CRUSH rule 0 x 787 [13,54,43]
+ CRUSH rule 0 x 788 [4,100,41]
+ CRUSH rule 0 x 789 [50,37,14]
+ CRUSH rule 0 x 790 [58,16,15]
+ CRUSH rule 0 x 791 [96,14,105]
+ CRUSH rule 0 x 792 [80,4,35]
+ CRUSH rule 0 x 793 [6,71,82]
+ CRUSH rule 0 x 794 [14,89,52]
+ CRUSH rule 0 x 795 [51,3,78]
+ CRUSH rule 0 x 796 [114,77,19]
+ CRUSH rule 0 x 797 [79,100,15]
+ CRUSH rule 0 x 798 [42,10,7]
+ CRUSH rule 0 x 799 [48,11,101]
+ CRUSH rule 0 x 800 [91,7,18]
+ CRUSH rule 0 x 801 [2,6,73]
+ CRUSH rule 0 x 802 [116,89,7]
+ CRUSH rule 0 x 803 [37,32,7]
+ CRUSH rule 0 x 804 [33,4,106]
+ CRUSH rule 0 x 805 [96,22,41]
+ CRUSH rule 0 x 806 [67,90,9]
+ CRUSH rule 0 x 807 [47,42,17]
+ CRUSH rule 0 x 808 [76,79,14]
+ CRUSH rule 0 x 809 [27,26,3]
+ CRUSH rule 0 x 810 [119,61,8]
+ CRUSH rule 0 x 811 [75,72,15]
+ CRUSH rule 0 x 812 [25,52,13]
+ CRUSH rule 0 x 813 [64,13,77]
+ CRUSH rule 0 x 814 [110,53,3]
+ CRUSH rule 0 x 815 [84,61,4]
+ CRUSH rule 0 x 816 [25,22,84]
+ CRUSH rule 0 x 817 [40,73,13]
+ CRUSH rule 0 x 818 [34,13,45]
+ CRUSH rule 0 x 819 [88,19,85]
+ CRUSH rule 0 x 820 [104,49,11]
+ CRUSH rule 0 x 821 [58,69,14]
+ CRUSH rule 0 x 822 [29,72,6]
+ CRUSH rule 0 x 823 [100,103,17]
+ CRUSH rule 0 x 824 [102,81,4]
+ CRUSH rule 0 x 825 [47,17,94]
+ CRUSH rule 0 x 826 [45,34,22]
+ CRUSH rule 0 x 827 [101,11,66]
+ CRUSH rule 0 x 828 [60,27,19]
+ CRUSH rule 0 x 829 [45,90,9]
+ CRUSH rule 0 x 830 [51,96,17]
+ CRUSH rule 0 x 831 [6,64,73]
+ CRUSH rule 0 x 832 [57,78,13]
+ CRUSH rule 0 x 833 [34,97,3]
+ CRUSH rule 0 x 834 [90,33,6]
+ CRUSH rule 0 x 835 [14,46,25]
+ CRUSH rule 0 x 836 [38,43,7]
+ CRUSH rule 0 x 837 [51,74,15]
+ CRUSH rule 0 x 838 [6,32,107]
+ CRUSH rule 0 x 839 [106,8,39]
+ CRUSH rule 0 x 840 [33,109,3]
+ CRUSH rule 0 x 841 [110,15,71]
+ CRUSH rule 0 x 842 [66,67,13]
+ CRUSH rule 0 x 843 [11,63,48]
+ CRUSH rule 0 x 844 [74,13,59]
+ CRUSH rule 0 x 845 [74,43,22]
+ CRUSH rule 0 x 846 [98,107,19]
+ CRUSH rule 0 x 847 [10,3,88]
+ CRUSH rule 0 x 848 [89,17,111]
+ CRUSH rule 0 x 849 [42,59,14]
+ CRUSH rule 0 x 850 [40,73,13]
+ CRUSH rule 0 x 851 [65,94,11]
+ CRUSH rule 0 x 852 [31,94,7]
+ CRUSH rule 0 x 853 [49,11,114]
+ CRUSH rule 0 x 854 [90,31,21]
+ CRUSH rule 0 x 855 [2,19,81]
+ CRUSH rule 0 x 856 [40,22,61]
+ CRUSH rule 0 x 857 [15,82,91]
+ CRUSH rule 0 x 858 [10,80,19]
+ CRUSH rule 0 x 859 [29,48,4]
+ CRUSH rule 0 x 860 [114,75,21]
+ CRUSH rule 0 x 861 [22,33,98]
+ CRUSH rule 0 x 862 [22,25,76]
+ CRUSH rule 0 x 863 [79,50,11]
+ CRUSH rule 0 x 864 [68,6,41]
+ CRUSH rule 0 x 865 [25,92,14]
+ CRUSH rule 0 x 866 [18,89,22]
+ CRUSH rule 0 x 867 [3,78,41]
+ CRUSH rule 0 x 868 [81,98,11]
+ CRUSH rule 0 x 869 [22,104,89]
+ CRUSH rule 0 x 870 [73,98,3]
+ CRUSH rule 0 x 871 [25,54,19]
+ CRUSH rule 0 x 872 [39,48,11]
+ CRUSH rule 0 x 873 [92,9,75]
+ CRUSH rule 0 x 874 [21,43,66]
+ CRUSH rule 0 x 875 [27,108,7]
+ CRUSH rule 0 x 876 [98,75,13]
+ CRUSH rule 0 x 877 [73,5,4]
+ CRUSH rule 0 x 878 [64,45,22]
+ CRUSH rule 0 x 879 [29,18,9]
+ CRUSH rule 0 x 880 [56,91,13]
+ CRUSH rule 0 x 881 [109,69,4]
+ CRUSH rule 0 x 882 [60,33,11]
+ CRUSH rule 0 x 883 [93,96,11]
+ CRUSH rule 0 x 884 [67,58,4]
+ CRUSH rule 0 x 885 [31,8,104]
+ CRUSH rule 0 x 886 [2,107,9]
+ CRUSH rule 0 x 887 [5,93,19]
+ CRUSH rule 0 x 888 [16,13,26]
+ CRUSH rule 0 x 889 [3,76,93]
+ CRUSH rule 0 x 890 [48,63,4]
+ CRUSH rule 0 x 891 [86,79,22]
+ CRUSH rule 0 x 892 [64,9,10]
+ CRUSH rule 0 x 893 [118,33,22]
+ CRUSH rule 0 x 894 [16,111,11]
+ CRUSH rule 0 x 895 [40,107,4]
+ CRUSH rule 0 x 896 [97,96,14]
+ CRUSH rule 0 x 897 [60,67,22]
+ CRUSH rule 0 x 898 [10,2,21]
+ CRUSH rule 0 x 899 [75,80,4]
+ CRUSH rule 0 x 900 [102,81,8]
+ CRUSH rule 0 x 901 [66,87,14]
+ CRUSH rule 0 x 902 [102,49,8]
+ CRUSH rule 0 x 903 [5,14,33]
+ CRUSH rule 0 x 904 [50,16,4]
+ CRUSH rule 0 x 905 [19,51,110]
+ CRUSH rule 0 x 906 [75,119,13]
+ CRUSH rule 0 x 907 [47,5,7]
+ CRUSH rule 0 x 908 [96,9,29]
+ CRUSH rule 0 x 909 [94,75,19]
+ CRUSH rule 0 x 910 [88,63,15]
+ CRUSH rule 0 x 911 [102,23,3]
+ CRUSH rule 0 x 912 [91,60,13]
+ CRUSH rule 0 x 913 [29,17,96]
+ CRUSH rule 0 x 914 [84,29,17]
+ CRUSH rule 0 x 915 [70,22,107]
+ CRUSH rule 0 x 916 [32,9,57]
+ CRUSH rule 0 x 917 [43,26,3]
+ CRUSH rule 0 x 918 [91,98,6]
+ CRUSH rule 0 x 919 [13,69,56]
+ CRUSH rule 0 x 920 [18,87,11]
+ CRUSH rule 0 x 921 [104,33,14]
+ CRUSH rule 0 x 922 [33,19,117]
+ CRUSH rule 0 x 923 [28,8,101]
+ CRUSH rule 0 x 924 [69,88,9]
+ CRUSH rule 0 x 925 [71,32,17]
+ CRUSH rule 0 x 926 [64,69,15]
+ CRUSH rule 0 x 927 [99,106,13]
+ CRUSH rule 0 x 928 [13,113,95]
+ CRUSH rule 0 x 929 [117,61,21]
+ CRUSH rule 0 x 930 [31,82,3]
+ CRUSH rule 0 x 931 [46,79,22]
+ CRUSH rule 0 x 932 [60,13,103]
+ CRUSH rule 0 x 933 [88,31,6]
+ CRUSH rule 0 x 934 [68,4,99]
+ CRUSH rule 0 x 935 [31,18,4]
+ CRUSH rule 0 x 936 [104,57,6]
+ CRUSH rule 0 x 937 [110,22,95]
+ CRUSH rule 0 x 938 [29,106,13]
+ CRUSH rule 0 x 939 [77,13,52]
+ CRUSH rule 0 x 940 [76,33,7]
+ CRUSH rule 0 x 941 [66,37,8]
+ CRUSH rule 0 x 942 [83,94,9]
+ CRUSH rule 0 x 943 [4,74,89]
+ CRUSH rule 0 x 944 [113,53,21]
+ CRUSH rule 0 x 945 [17,52,16]
+ CRUSH rule 0 x 946 [37,111,11]
+ CRUSH rule 0 x 947 [107,74,7]
+ CRUSH rule 0 x 948 [55,98,9]
+ CRUSH rule 0 x 949 [45,72,21]
+ CRUSH rule 0 x 950 [96,23,3]
+ CRUSH rule 0 x 951 [40,93,7]
+ CRUSH rule 0 x 952 [93,46,6]
+ CRUSH rule 0 x 953 [55,92,6]
+ CRUSH rule 0 x 954 [84,57,7]
+ CRUSH rule 0 x 955 [31,117,13]
+ CRUSH rule 0 x 956 [72,11,55]
+ CRUSH rule 0 x 957 [3,74,87]
+ CRUSH rule 0 x 958 [8,106,43]
+ CRUSH rule 0 x 959 [42,59,22]
+ CRUSH rule 0 x 960 [113,107,11]
+ CRUSH rule 0 x 961 [116,8,53]
+ CRUSH rule 0 x 962 [13,62,79]
+ CRUSH rule 0 x 963 [0,99,14]
+ CRUSH rule 0 x 964 [59,21,32]
+ CRUSH rule 0 x 965 [47,115,9]
+ CRUSH rule 0 x 966 [88,63,13]
+ CRUSH rule 0 x 967 [71,108,14]
+ CRUSH rule 0 x 968 [73,7,54]
+ CRUSH rule 0 x 969 [53,6,2]
+ CRUSH rule 0 x 970 [3,40,65]
+ CRUSH rule 0 x 971 [87,38,9]
+ CRUSH rule 0 x 972 [3,37,109]
+ CRUSH rule 0 x 973 [113,27,4]
+ CRUSH rule 0 x 974 [114,23,13]
+ CRUSH rule 0 x 975 [40,59,8]
+ CRUSH rule 0 x 976 [81,38,19]
+ CRUSH rule 0 x 977 [95,102,11]
+ CRUSH rule 0 x 978 [35,56,15]
+ CRUSH rule 0 x 979 [98,6,45]
+ CRUSH rule 0 x 980 [52,69,3]
+ CRUSH rule 0 x 981 [89,117,15]
+ CRUSH rule 0 x 982 [1,47,22]
+ CRUSH rule 0 x 983 [34,61,13]
+ CRUSH rule 0 x 984 [78,25,8]
+ CRUSH rule 0 x 985 [99,52,6]
+ CRUSH rule 0 x 986 [4,59,84]
+ CRUSH rule 0 x 987 [78,21,27]
+ CRUSH rule 0 x 988 [79,2,11]
+ CRUSH rule 0 x 989 [87,17,32]
+ CRUSH rule 0 x 990 [47,118,9]
+ CRUSH rule 0 x 991 [61,18,6]
+ CRUSH rule 0 x 992 [83,66,17]
+ CRUSH rule 0 x 993 [75,62,8]
+ CRUSH rule 0 x 994 [74,57,9]
+ CRUSH rule 0 x 995 [100,97,7]
+ CRUSH rule 0 x 996 [41,6,58]
+ CRUSH rule 0 x 997 [89,76,7]
+ CRUSH rule 0 x 998 [92,47,13]
+ CRUSH rule 0 x 999 [101,11,66]
+ CRUSH rule 0 x 1000 [9,119,37]
+ CRUSH rule 0 x 1001 [49,32,7]
+ CRUSH rule 0 x 1002 [99,113,7]
+ CRUSH rule 0 x 1003 [43,18,6]
+ CRUSH rule 0 x 1004 [89,54,15]
+ CRUSH rule 0 x 1005 [105,84,8]
+ CRUSH rule 0 x 1006 [45,111,6]
+ CRUSH rule 0 x 1007 [19,57,5]
+ CRUSH rule 0 x 1008 [31,24,13]
+ CRUSH rule 0 x 1009 [19,111,61]
+ CRUSH rule 0 x 1010 [42,89,13]
+ CRUSH rule 0 x 1011 [25,114,6]
+ CRUSH rule 0 x 1012 [68,71,21]
+ CRUSH rule 0 x 1013 [5,65,3]
+ CRUSH rule 0 x 1014 [33,4,109]
+ CRUSH rule 0 x 1015 [106,45,9]
+ CRUSH rule 0 x 1016 [88,39,4]
+ CRUSH rule 0 x 1017 [0,89,7]
+ CRUSH rule 0 x 1018 [63,5,7]
+ CRUSH rule 0 x 1019 [104,97,4]
+ CRUSH rule 0 x 1020 [96,9,91]
+ CRUSH rule 0 x 1021 [117,6,43]
+ CRUSH rule 0 x 1022 [73,21,36]
+ CRUSH rule 0 x 1023 [0,16,3]
+ rule 0 (data) num_rep 10 result size == 3:\t1024/1024 (esc)
diff --git a/src/test/cli/crushtool/test-map-vary-r-0.t b/src/test/cli/crushtool/test-map-vary-r-0.t
new file mode 100644
index 0000000..663ef65
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-vary-r-0.t
@@ -0,0 +1,3081 @@
+ $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 0 --weight 0 0 --weight 4 0 --weight 9 0
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,12]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,52]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,82]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,40]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,107]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38]
+ CRUSH rule 3 x 38 [72,57]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,2]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,29]
+ CRUSH rule 3 x 54 [28,77]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,105]
+ CRUSH rule 3 x 69 [62]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,88]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,43]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,39]
+ CRUSH rule 3 x 79 [64,65]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,83]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,68]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,26]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,17]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,83]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,100]
+ CRUSH rule 3 x 146 [12,15]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,81]
+ CRUSH rule 3 x 154 [19,56]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,22]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,103]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,100]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,85]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90]
+ CRUSH rule 3 x 191 [49,113]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,66]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,86]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,118]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,32]
+ CRUSH rule 3 x 234 [32,55]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,10]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,87]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,31]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,54]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,75]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,18]
+ CRUSH rule 3 x 291 [35,117]
+ CRUSH rule 3 x 292 [20,74]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,36]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,105]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,119]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,15]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,17]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,116]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,118]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,112]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,32]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,81]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,12]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,113]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,70]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,17]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,44]
+ CRUSH rule 3 x 410 [70,8]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,15]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,41]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,21]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,68]
+ CRUSH rule 3 x 456 [101,60]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,106]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,84]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,61]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99]
+ CRUSH rule 3 x 492 [21,58]
+ CRUSH rule 3 x 493 [94,89]
+ CRUSH rule 3 x 494 [56,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,82]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,6]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,77]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,30]
+ CRUSH rule 3 x 518 [18,37]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,104]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,92]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,19]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,116]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,35]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,96]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,24]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,44]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,50]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,114]
+ CRUSH rule 3 x 638 [43,78]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,99]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,39]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,107]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,102]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,100]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,116]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,26]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36]
+ CRUSH rule 3 x 697 [19,38]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,60]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,113]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,38]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,26]
+ CRUSH rule 3 x 723 [117,75]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,38]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,94]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,56]
+ CRUSH rule 3 x 752 [82,16]
+ CRUSH rule 3 x 753 [116,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,65]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,90]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,53]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,93]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,35]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,94]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,19]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,102]
+ CRUSH rule 3 x 824 [102,95]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,33]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,75]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,39]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,31]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,29]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,66]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,62]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,80]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,25]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,60]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,107]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,10]
+ CRUSH rule 3 x 942 [80,37]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,111]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,16]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,46]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,103]
+ CRUSH rule 3 x 969 [53]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,19]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,100]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,113]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,33]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,22]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,35]
+ CRUSH rule 3 x 1014 [33,48]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,111]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [59,88]
+ rule 3 (delltestrule) num_rep 2 result size == 1:\t27/1024 (esc)
+ rule 3 (delltestrule) num_rep 2 result size == 2:\t997/1024 (esc)
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,12]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,52]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,82]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,40]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,107]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38]
+ CRUSH rule 3 x 38 [72,57]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,2]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,29]
+ CRUSH rule 3 x 54 [28,77]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,105]
+ CRUSH rule 3 x 69 [62]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,88]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,43]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,39]
+ CRUSH rule 3 x 79 [64,65]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,83]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,68]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,26]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,17]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,83]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,100]
+ CRUSH rule 3 x 146 [12,15]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,81]
+ CRUSH rule 3 x 154 [19,56]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,22]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,103]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,100]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,85]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90]
+ CRUSH rule 3 x 191 [49,113]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,66]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,86]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,118]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,32]
+ CRUSH rule 3 x 234 [32,55]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,10]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,87]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,31]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,54]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,75]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,18]
+ CRUSH rule 3 x 291 [35,117]
+ CRUSH rule 3 x 292 [20,74]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,36]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,105]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,119]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,15]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,17]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,116]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,118]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,112]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,32]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,81]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,12]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,113]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,70]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,17]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,44]
+ CRUSH rule 3 x 410 [70,8]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,15]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,41]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,21]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,68]
+ CRUSH rule 3 x 456 [101,60]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,106]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,84]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,61]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99]
+ CRUSH rule 3 x 492 [21,58]
+ CRUSH rule 3 x 493 [94,89]
+ CRUSH rule 3 x 494 [56,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,82]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,6]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,77]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,30]
+ CRUSH rule 3 x 518 [18,37]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,104]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,92]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,19]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,116]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,35]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,96]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,24]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,44]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,50]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,114]
+ CRUSH rule 3 x 638 [43,78]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,99]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,39]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,107]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,102]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,100]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,116]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,26]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36]
+ CRUSH rule 3 x 697 [19,38]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,60]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,113]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,38]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,26]
+ CRUSH rule 3 x 723 [117,75]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,38]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,94]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,56]
+ CRUSH rule 3 x 752 [82,16]
+ CRUSH rule 3 x 753 [116,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,65]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,90]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,53]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,93]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,35]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,94]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,19]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,102]
+ CRUSH rule 3 x 824 [102,95]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,33]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,75]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,39]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,31]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,29]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,66]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,62]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,80]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,25]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,60]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,107]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,10]
+ CRUSH rule 3 x 942 [80,37]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,111]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,16]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,46]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,103]
+ CRUSH rule 3 x 969 [53]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,19]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,100]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,113]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,33]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,22]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,35]
+ CRUSH rule 3 x 1014 [33,48]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,111]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [59,88]
+ rule 3 (delltestrule) num_rep 3 result size == 1:\t27/1024 (esc)
+ rule 3 (delltestrule) num_rep 3 result size == 2:\t997/1024 (esc)
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,12]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,52]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,82]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,40]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,107]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38]
+ CRUSH rule 3 x 38 [72,57]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,2]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,29]
+ CRUSH rule 3 x 54 [28,77]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,105]
+ CRUSH rule 3 x 69 [62]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,88]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,43]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,39]
+ CRUSH rule 3 x 79 [64,65]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,83]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,68]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,26]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,17]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,83]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,100]
+ CRUSH rule 3 x 146 [12,15]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,81]
+ CRUSH rule 3 x 154 [19,56]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,22]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,103]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,100]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,85]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90]
+ CRUSH rule 3 x 191 [49,113]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,66]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,86]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,118]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,32]
+ CRUSH rule 3 x 234 [32,55]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,10]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,87]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,31]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,54]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,75]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,18]
+ CRUSH rule 3 x 291 [35,117]
+ CRUSH rule 3 x 292 [20,74]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,36]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,105]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,119]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,15]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,17]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,116]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,118]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,112]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,32]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,81]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,12]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,113]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,70]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,17]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,44]
+ CRUSH rule 3 x 410 [70,8]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,15]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,41]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,21]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,68]
+ CRUSH rule 3 x 456 [101,60]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,106]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,84]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,61]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99]
+ CRUSH rule 3 x 492 [21,58]
+ CRUSH rule 3 x 493 [94,89]
+ CRUSH rule 3 x 494 [56,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,82]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,6]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,77]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,30]
+ CRUSH rule 3 x 518 [18,37]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,104]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,92]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,19]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,116]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,35]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,96]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,24]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,44]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,50]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,114]
+ CRUSH rule 3 x 638 [43,78]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,99]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,39]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,107]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,102]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,100]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,116]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,26]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36]
+ CRUSH rule 3 x 697 [19,38]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,60]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,113]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,38]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,26]
+ CRUSH rule 3 x 723 [117,75]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,38]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,94]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,56]
+ CRUSH rule 3 x 752 [82,16]
+ CRUSH rule 3 x 753 [116,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,65]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,90]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,53]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,93]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,35]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,94]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,19]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,102]
+ CRUSH rule 3 x 824 [102,95]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,33]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,75]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,39]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,31]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,29]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,66]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,62]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,80]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,25]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,60]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,107]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,10]
+ CRUSH rule 3 x 942 [80,37]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,111]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,16]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,46]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,103]
+ CRUSH rule 3 x 969 [53]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,19]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,100]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,113]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,33]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,22]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,35]
+ CRUSH rule 3 x 1014 [33,48]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,111]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [59,88]
+ rule 3 (delltestrule) num_rep 4 result size == 1:\t27/1024 (esc)
+ rule 3 (delltestrule) num_rep 4 result size == 2:\t997/1024 (esc)
diff --git a/src/test/cli/crushtool/test-map-vary-r-1.t b/src/test/cli/crushtool/test-map-vary-r-1.t
new file mode 100644
index 0000000..4ac4c22
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-vary-r-1.t
@@ -0,0 +1,3078 @@
+ $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 1 --weight 0 0 --weight 4 0 --weight 9 0
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+ CRUSH rule 3 x 0 [94,6]
+ CRUSH rule 3 x 1 [73,52]
+ CRUSH rule 3 x 2 [91,48]
+ CRUSH rule 3 x 3 [51,48]
+ CRUSH rule 3 x 4 [45,114]
+ CRUSH rule 3 x 5 [89,94]
+ CRUSH rule 3 x 6 [91,76]
+ CRUSH rule 3 x 7 [104,73]
+ CRUSH rule 3 x 8 [41,98]
+ CRUSH rule 3 x 9 [46,47]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,40]
+ CRUSH rule 3 x 12 [83,26]
+ CRUSH rule 3 x 13 [27,28]
+ CRUSH rule 3 x 14 [105,64]
+ CRUSH rule 3 x 15 [18,7]
+ CRUSH rule 3 x 16 [103,30]
+ CRUSH rule 3 x 17 [85,118]
+ CRUSH rule 3 x 18 [11,106]
+ CRUSH rule 3 x 19 [75,50]
+ CRUSH rule 3 x 20 [111,67]
+ CRUSH rule 3 x 21 [84,61]
+ CRUSH rule 3 x 22 [23,104]
+ CRUSH rule 3 x 23 [19,86]
+ CRUSH rule 3 x 24 [83,60]
+ CRUSH rule 3 x 25 [81,64]
+ CRUSH rule 3 x 26 [17,38]
+ CRUSH rule 3 x 27 [33,84]
+ CRUSH rule 3 x 28 [45,90]
+ CRUSH rule 3 x 29 [8,109]
+ CRUSH rule 3 x 30 [55,42]
+ CRUSH rule 3 x 31 [76,95]
+ CRUSH rule 3 x 32 [72,11]
+ CRUSH rule 3 x 33 [86,53]
+ CRUSH rule 3 x 34 [7,108]
+ CRUSH rule 3 x 35 [108,13]
+ CRUSH rule 3 x 36 [67,66]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,105]
+ CRUSH rule 3 x 39 [68,103]
+ CRUSH rule 3 x 40 [30,85]
+ CRUSH rule 3 x 41 [52,11]
+ CRUSH rule 3 x 42 [106,75]
+ CRUSH rule 3 x 43 [10,104]
+ CRUSH rule 3 x 44 [101,28]
+ CRUSH rule 3 x 45 [83,64]
+ CRUSH rule 3 x 46 [54,31]
+ CRUSH rule 3 x 47 [106,61]
+ CRUSH rule 3 x 48 [34,41]
+ CRUSH rule 3 x 49 [79,110]
+ CRUSH rule 3 x 50 [42,13]
+ CRUSH rule 3 x 51 [6,94]
+ CRUSH rule 3 x 52 [82,19]
+ CRUSH rule 3 x 53 [32,91]
+ CRUSH rule 3 x 54 [108,8]
+ CRUSH rule 3 x 55 [14,94]
+ CRUSH rule 3 x 56 [21,72]
+ CRUSH rule 3 x 57 [69,88]
+ CRUSH rule 3 x 58 [48,87]
+ CRUSH rule 3 x 59 [21,113]
+ CRUSH rule 3 x 60 [90,73]
+ CRUSH rule 3 x 61 [88,63]
+ CRUSH rule 3 x 62 [100,13]
+ CRUSH rule 3 x 63 [79,5]
+ CRUSH rule 3 x 64 [1,89]
+ CRUSH rule 3 x 65 [32,103]
+ CRUSH rule 3 x 66 [48,79]
+ CRUSH rule 3 x 67 [94,11]
+ CRUSH rule 3 x 68 [102,15]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,11]
+ CRUSH rule 3 x 71 [12,33]
+ CRUSH rule 3 x 72 [26,99]
+ CRUSH rule 3 x 73 [29,114]
+ CRUSH rule 3 x 74 [29,1]
+ CRUSH rule 3 x 75 [60,65]
+ CRUSH rule 3 x 76 [55,62]
+ CRUSH rule 3 x 77 [107,100]
+ CRUSH rule 3 x 78 [86,107]
+ CRUSH rule 3 x 79 [64,16]
+ CRUSH rule 3 x 80 [19,100]
+ CRUSH rule 3 x 81 [64,16]
+ CRUSH rule 3 x 82 [37,40]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,115]
+ CRUSH rule 3 x 85 [87,88]
+ CRUSH rule 3 x 86 [37,68]
+ CRUSH rule 3 x 87 [116,77]
+ CRUSH rule 3 x 88 [38,55]
+ CRUSH rule 3 x 89 [76,25]
+ CRUSH rule 3 x 90 [14,50]
+ CRUSH rule 3 x 91 [68,61]
+ CRUSH rule 3 x 92 [86,95]
+ CRUSH rule 3 x 93 [44,35]
+ CRUSH rule 3 x 94 [46,71]
+ CRUSH rule 3 x 95 [108,53]
+ CRUSH rule 3 x 96 [66,87]
+ CRUSH rule 3 x 97 [111,45]
+ CRUSH rule 3 x 98 [93,110]
+ CRUSH rule 3 x 99 [78,43]
+ CRUSH rule 3 x 100 [28,61]
+ CRUSH rule 3 x 101 [91,110]
+ CRUSH rule 3 x 102 [82,7]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,79]
+ CRUSH rule 3 x 105 [34,87]
+ CRUSH rule 3 x 106 [69,12]
+ CRUSH rule 3 x 107 [1,59]
+ CRUSH rule 3 x 108 [7,109]
+ CRUSH rule 3 x 109 [112,67]
+ CRUSH rule 3 x 110 [54,61]
+ CRUSH rule 3 x 111 [10,92]
+ CRUSH rule 3 x 112 [80,11]
+ CRUSH rule 3 x 113 [69,38]
+ CRUSH rule 3 x 114 [79,38]
+ CRUSH rule 3 x 115 [10,48]
+ CRUSH rule 3 x 116 [37,108]
+ CRUSH rule 3 x 117 [87,56]
+ CRUSH rule 3 x 118 [23,56]
+ CRUSH rule 3 x 119 [104,31]
+ CRUSH rule 3 x 120 [44,93]
+ CRUSH rule 3 x 121 [80,16]
+ CRUSH rule 3 x 122 [45,54]
+ CRUSH rule 3 x 123 [22,112]
+ CRUSH rule 3 x 124 [97,50]
+ CRUSH rule 3 x 125 [66,6]
+ CRUSH rule 3 x 126 [70,39]
+ CRUSH rule 3 x 127 [70,75]
+ CRUSH rule 3 x 128 [11,111]
+ CRUSH rule 3 x 129 [103,46]
+ CRUSH rule 3 x 130 [50,73]
+ CRUSH rule 3 x 131 [44,15]
+ CRUSH rule 3 x 132 [69,58]
+ CRUSH rule 3 x 133 [67,115]
+ CRUSH rule 3 x 134 [37,60]
+ CRUSH rule 3 x 135 [78,61]
+ CRUSH rule 3 x 136 [32,29]
+ CRUSH rule 3 x 137 [92,87]
+ CRUSH rule 3 x 138 [54,8]
+ CRUSH rule 3 x 139 [89,60]
+ CRUSH rule 3 x 140 [39,50]
+ CRUSH rule 3 x 141 [89,62]
+ CRUSH rule 3 x 142 [22,86]
+ CRUSH rule 3 x 143 [96,16]
+ CRUSH rule 3 x 144 [13,1]
+ CRUSH rule 3 x 145 [77,54]
+ CRUSH rule 3 x 146 [12,43]
+ CRUSH rule 3 x 147 [2,59]
+ CRUSH rule 3 x 148 [85,50]
+ CRUSH rule 3 x 149 [103,68]
+ CRUSH rule 3 x 150 [14,50]
+ CRUSH rule 3 x 151 [75,56]
+ CRUSH rule 3 x 152 [49,18]
+ CRUSH rule 3 x 153 [92,79]
+ CRUSH rule 3 x 154 [19,26]
+ CRUSH rule 3 x 155 [12,13]
+ CRUSH rule 3 x 156 [107,18]
+ CRUSH rule 3 x 157 [15,78]
+ CRUSH rule 3 x 158 [11,28]
+ CRUSH rule 3 x 159 [33,88]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,78]
+ CRUSH rule 3 x 162 [55,96]
+ CRUSH rule 3 x 163 [54,55]
+ CRUSH rule 3 x 164 [72,99]
+ CRUSH rule 3 x 165 [25,116]
+ CRUSH rule 3 x 166 [2,23]
+ CRUSH rule 3 x 167 [89,40]
+ CRUSH rule 3 x 168 [68,49]
+ CRUSH rule 3 x 169 [51,50]
+ CRUSH rule 3 x 170 [68,91]
+ CRUSH rule 3 x 171 [88,51]
+ CRUSH rule 3 x 172 [117,23]
+ CRUSH rule 3 x 173 [29,1]
+ CRUSH rule 3 x 174 [67,40]
+ CRUSH rule 3 x 175 [48,41]
+ CRUSH rule 3 x 176 [94,91]
+ CRUSH rule 3 x 177 [53,70]
+ CRUSH rule 3 x 178 [39,110]
+ CRUSH rule 3 x 179 [72,89]
+ CRUSH rule 3 x 180 [3,38]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,46]
+ CRUSH rule 3 x 183 [11,78]
+ CRUSH rule 3 x 184 [79,92]
+ CRUSH rule 3 x 185 [97,92]
+ CRUSH rule 3 x 186 [67,116]
+ CRUSH rule 3 x 187 [6,96]
+ CRUSH rule 3 x 188 [76,16]
+ CRUSH rule 3 x 189 [96,71]
+ CRUSH rule 3 x 190 [90,6]
+ CRUSH rule 3 x 191 [49,84]
+ CRUSH rule 3 x 192 [93,114]
+ CRUSH rule 3 x 193 [89,60]
+ CRUSH rule 3 x 194 [62,61]
+ CRUSH rule 3 x 195 [119,95]
+ CRUSH rule 3 x 196 [20,28]
+ CRUSH rule 3 x 197 [6,64]
+ CRUSH rule 3 x 198 [55,112]
+ CRUSH rule 3 x 199 [66,17]
+ CRUSH rule 3 x 200 [12,63]
+ CRUSH rule 3 x 201 [52,23]
+ CRUSH rule 3 x 202 [98,33]
+ CRUSH rule 3 x 203 [36,22]
+ CRUSH rule 3 x 204 [10,100]
+ CRUSH rule 3 x 205 [38,29]
+ CRUSH rule 3 x 206 [38,27]
+ CRUSH rule 3 x 207 [19,108]
+ CRUSH rule 3 x 208 [63,26]
+ CRUSH rule 3 x 209 [70,11]
+ CRUSH rule 3 x 210 [79,58]
+ CRUSH rule 3 x 211 [26,59]
+ CRUSH rule 3 x 212 [107,114]
+ CRUSH rule 3 x 213 [100,3]
+ CRUSH rule 3 x 214 [91,118]
+ CRUSH rule 3 x 215 [92,16]
+ CRUSH rule 3 x 216 [99,94]
+ CRUSH rule 3 x 217 [86,99]
+ CRUSH rule 3 x 218 [70,15]
+ CRUSH rule 3 x 219 [61,58]
+ CRUSH rule 3 x 220 [23,56]
+ CRUSH rule 3 x 221 [21,92]
+ CRUSH rule 3 x 222 [102,29]
+ CRUSH rule 3 x 223 [34,73]
+ CRUSH rule 3 x 224 [107,68]
+ CRUSH rule 3 x 225 [61,98]
+ CRUSH rule 3 x 226 [44,13]
+ CRUSH rule 3 x 227 [55,60]
+ CRUSH rule 3 x 228 [117,55]
+ CRUSH rule 3 x 229 [100,67]
+ CRUSH rule 3 x 230 [41,109]
+ CRUSH rule 3 x 231 [30,71]
+ CRUSH rule 3 x 232 [23,1]
+ CRUSH rule 3 x 233 [47,90]
+ CRUSH rule 3 x 234 [55,62]
+ CRUSH rule 3 x 235 [20,60]
+ CRUSH rule 3 x 236 [95,24]
+ CRUSH rule 3 x 237 [21,106]
+ CRUSH rule 3 x 238 [109,15]
+ CRUSH rule 3 x 239 [40,101]
+ CRUSH rule 3 x 240 [63,60]
+ CRUSH rule 3 x 241 [47,12]
+ CRUSH rule 3 x 242 [73,74]
+ CRUSH rule 3 x 243 [76,8]
+ CRUSH rule 3 x 244 [103,50]
+ CRUSH rule 3 x 245 [106,95]
+ CRUSH rule 3 x 246 [35,82]
+ CRUSH rule 3 x 247 [116,101]
+ CRUSH rule 3 x 248 [8,119]
+ CRUSH rule 3 x 249 [2,17]
+ CRUSH rule 3 x 250 [34,89]
+ CRUSH rule 3 x 251 [28,69]
+ CRUSH rule 3 x 252 [95,80]
+ CRUSH rule 3 x 253 [109,3]
+ CRUSH rule 3 x 254 [99,80]
+ CRUSH rule 3 x 255 [112,85]
+ CRUSH rule 3 x 256 [94,63]
+ CRUSH rule 3 x 257 [100,87]
+ CRUSH rule 3 x 258 [34,63]
+ CRUSH rule 3 x 259 [70,107]
+ CRUSH rule 3 x 260 [89,115]
+ CRUSH rule 3 x 261 [94,83]
+ CRUSH rule 3 x 262 [42,45]
+ CRUSH rule 3 x 263 [113,101]
+ CRUSH rule 3 x 264 [36,81]
+ CRUSH rule 3 x 265 [14,88]
+ CRUSH rule 3 x 266 [75,96]
+ CRUSH rule 3 x 267 [6,5]
+ CRUSH rule 3 x 268 [38,47]
+ CRUSH rule 3 x 269 [86,59]
+ CRUSH rule 3 x 270 [87,70]
+ CRUSH rule 3 x 271 [19,108]
+ CRUSH rule 3 x 272 [73,5]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,64]
+ CRUSH rule 3 x 275 [29,34]
+ CRUSH rule 3 x 276 [7,100]
+ CRUSH rule 3 x 277 [74,6]
+ CRUSH rule 3 x 278 [107,115]
+ CRUSH rule 3 x 279 [112,20]
+ CRUSH rule 3 x 280 [113,15]
+ CRUSH rule 3 x 281 [89,56]
+ CRUSH rule 3 x 282 [20,38]
+ CRUSH rule 3 x 283 [8,114]
+ CRUSH rule 3 x 284 [66,75]
+ CRUSH rule 3 x 285 [99,94]
+ CRUSH rule 3 x 286 [78,6]
+ CRUSH rule 3 x 287 [12,27]
+ CRUSH rule 3 x 288 [24,22]
+ CRUSH rule 3 x 289 [105,64]
+ CRUSH rule 3 x 290 [25,46]
+ CRUSH rule 3 x 291 [35,116]
+ CRUSH rule 3 x 292 [20,109]
+ CRUSH rule 3 x 293 [27,92]
+ CRUSH rule 3 x 294 [60,93]
+ CRUSH rule 3 x 295 [37,2]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,55]
+ CRUSH rule 3 x 298 [70,53]
+ CRUSH rule 3 x 299 [116,10]
+ CRUSH rule 3 x 300 [67,26]
+ CRUSH rule 3 x 301 [117,23]
+ CRUSH rule 3 x 302 [78,67]
+ CRUSH rule 3 x 303 [19,5]
+ CRUSH rule 3 x 304 [101,50]
+ CRUSH rule 3 x 305 [5,59]
+ CRUSH rule 3 x 306 [41,18]
+ CRUSH rule 3 x 307 [65,5]
+ CRUSH rule 3 x 308 [91,2]
+ CRUSH rule 3 x 309 [38,53]
+ CRUSH rule 3 x 310 [26,15]
+ CRUSH rule 3 x 311 [36,95]
+ CRUSH rule 3 x 312 [114,61]
+ CRUSH rule 3 x 313 [104,65]
+ CRUSH rule 3 x 314 [28,6]
+ CRUSH rule 3 x 315 [118,31]
+ CRUSH rule 3 x 316 [98,95]
+ CRUSH rule 3 x 317 [118,13]
+ CRUSH rule 3 x 318 [17,30]
+ CRUSH rule 3 x 319 [53,1]
+ CRUSH rule 3 x 320 [36,41]
+ CRUSH rule 3 x 321 [33,5]
+ CRUSH rule 3 x 322 [68,10]
+ CRUSH rule 3 x 323 [66,29]
+ CRUSH rule 3 x 324 [21,72]
+ CRUSH rule 3 x 325 [52,16]
+ CRUSH rule 3 x 326 [7,109]
+ CRUSH rule 3 x 327 [62,17]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,100]
+ CRUSH rule 3 x 330 [24,11]
+ CRUSH rule 3 x 331 [84,95]
+ CRUSH rule 3 x 332 [61,111]
+ CRUSH rule 3 x 333 [116,25]
+ CRUSH rule 3 x 334 [94,103]
+ CRUSH rule 3 x 335 [71,118]
+ CRUSH rule 3 x 336 [24,99]
+ CRUSH rule 3 x 337 [18,83]
+ CRUSH rule 3 x 338 [43,28]
+ CRUSH rule 3 x 339 [13,64]
+ CRUSH rule 3 x 340 [81,111]
+ CRUSH rule 3 x 341 [46,105]
+ CRUSH rule 3 x 342 [92,23]
+ CRUSH rule 3 x 343 [49,112]
+ CRUSH rule 3 x 344 [1,87]
+ CRUSH rule 3 x 345 [56,35]
+ CRUSH rule 3 x 346 [3,54]
+ CRUSH rule 3 x 347 [106,27]
+ CRUSH rule 3 x 348 [10,117]
+ CRUSH rule 3 x 349 [96,87]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,41]
+ CRUSH rule 3 x 352 [36,13]
+ CRUSH rule 3 x 353 [10,82]
+ CRUSH rule 3 x 354 [55,52]
+ CRUSH rule 3 x 355 [73,94]
+ CRUSH rule 3 x 356 [75,66]
+ CRUSH rule 3 x 357 [70,93]
+ CRUSH rule 3 x 358 [97,56]
+ CRUSH rule 3 x 359 [110,105]
+ CRUSH rule 3 x 360 [106,57]
+ CRUSH rule 3 x 361 [27,42]
+ CRUSH rule 3 x 362 [28,55]
+ CRUSH rule 3 x 363 [68,20]
+ CRUSH rule 3 x 364 [23,50]
+ CRUSH rule 3 x 365 [57,76]
+ CRUSH rule 3 x 366 [42,75]
+ CRUSH rule 3 x 367 [103,82]
+ CRUSH rule 3 x 368 [103,104]
+ CRUSH rule 3 x 369 [12,57]
+ CRUSH rule 3 x 370 [11,26]
+ CRUSH rule 3 x 371 [34,55]
+ CRUSH rule 3 x 372 [58,14]
+ CRUSH rule 3 x 373 [6,42]
+ CRUSH rule 3 x 374 [110,95]
+ CRUSH rule 3 x 375 [5,43]
+ CRUSH rule 3 x 376 [91,86]
+ CRUSH rule 3 x 377 [93,116]
+ CRUSH rule 3 x 378 [68,6]
+ CRUSH rule 3 x 379 [77,44]
+ CRUSH rule 3 x 380 [76,83]
+ CRUSH rule 3 x 381 [36,27]
+ CRUSH rule 3 x 382 [26,77]
+ CRUSH rule 3 x 383 [76,99]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,93]
+ CRUSH rule 3 x 386 [83,92]
+ CRUSH rule 3 x 387 [16,26]
+ CRUSH rule 3 x 388 [29,113]
+ CRUSH rule 3 x 389 [92,29]
+ CRUSH rule 3 x 390 [68,77]
+ CRUSH rule 3 x 391 [15,88]
+ CRUSH rule 3 x 392 [21,32]
+ CRUSH rule 3 x 393 [91,18]
+ CRUSH rule 3 x 394 [38,73]
+ CRUSH rule 3 x 395 [21,119]
+ CRUSH rule 3 x 396 [12,13]
+ CRUSH rule 3 x 397 [40,63]
+ CRUSH rule 3 x 398 [44,3]
+ CRUSH rule 3 x 399 [5,95]
+ CRUSH rule 3 x 400 [19,102]
+ CRUSH rule 3 x 401 [79,52]
+ CRUSH rule 3 x 402 [107,98]
+ CRUSH rule 3 x 403 [23,82]
+ CRUSH rule 3 x 404 [87,68]
+ CRUSH rule 3 x 405 [90,97]
+ CRUSH rule 3 x 406 [15,117]
+ CRUSH rule 3 x 407 [70,35]
+ CRUSH rule 3 x 408 [55,72]
+ CRUSH rule 3 x 409 [73,62]
+ CRUSH rule 3 x 410 [70,73]
+ CRUSH rule 3 x 411 [34,25]
+ CRUSH rule 3 x 412 [105,117]
+ CRUSH rule 3 x 413 [41,110]
+ CRUSH rule 3 x 414 [70,65]
+ CRUSH rule 3 x 415 [107,5]
+ CRUSH rule 3 x 416 [2,22]
+ CRUSH rule 3 x 417 [26,14]
+ CRUSH rule 3 x 418 [51,46]
+ CRUSH rule 3 x 419 [8,82]
+ CRUSH rule 3 x 420 [109,105]
+ CRUSH rule 3 x 421 [114,75]
+ CRUSH rule 3 x 422 [109,87]
+ CRUSH rule 3 x 423 [59,24]
+ CRUSH rule 3 x 424 [92,51]
+ CRUSH rule 3 x 425 [101,111]
+ CRUSH rule 3 x 426 [36,6]
+ CRUSH rule 3 x 427 [8,24]
+ CRUSH rule 3 x 428 [68,35]
+ CRUSH rule 3 x 429 [76,75]
+ CRUSH rule 3 x 430 [67,117]
+ CRUSH rule 3 x 431 [70,25]
+ CRUSH rule 3 x 432 [7,34]
+ CRUSH rule 3 x 433 [49,84]
+ CRUSH rule 3 x 434 [64,31]
+ CRUSH rule 3 x 435 [110,13]
+ CRUSH rule 3 x 436 [106,89]
+ CRUSH rule 3 x 437 [26,65]
+ CRUSH rule 3 x 438 [118,63]
+ CRUSH rule 3 x 439 [40,21]
+ CRUSH rule 3 x 440 [45,119]
+ CRUSH rule 3 x 441 [112,105]
+ CRUSH rule 3 x 442 [55,113]
+ CRUSH rule 3 x 443 [44,33]
+ CRUSH rule 3 x 444 [71,38]
+ CRUSH rule 3 x 445 [58,81]
+ CRUSH rule 3 x 446 [40,10]
+ CRUSH rule 3 x 447 [100,61]
+ CRUSH rule 3 x 448 [111,73]
+ CRUSH rule 3 x 449 [67,66]
+ CRUSH rule 3 x 450 [117,61]
+ CRUSH rule 3 x 451 [66,81]
+ CRUSH rule 3 x 452 [70,8]
+ CRUSH rule 3 x 453 [82,85]
+ CRUSH rule 3 x 454 [53,18]
+ CRUSH rule 3 x 455 [91,42]
+ CRUSH rule 3 x 456 [101,46]
+ CRUSH rule 3 x 457 [113,51]
+ CRUSH rule 3 x 458 [119,25]
+ CRUSH rule 3 x 459 [50,67]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,51]
+ CRUSH rule 3 x 462 [98,107]
+ CRUSH rule 3 x 463 [108,105]
+ CRUSH rule 3 x 464 [19,109]
+ CRUSH rule 3 x 465 [62,23]
+ CRUSH rule 3 x 466 [53,12]
+ CRUSH rule 3 x 467 [40,57]
+ CRUSH rule 3 x 468 [97,44]
+ CRUSH rule 3 x 469 [98,75]
+ CRUSH rule 3 x 470 [50,29]
+ CRUSH rule 3 x 471 [40,13]
+ CRUSH rule 3 x 472 [27,18]
+ CRUSH rule 3 x 473 [48,35]
+ CRUSH rule 3 x 474 [51,32]
+ CRUSH rule 3 x 475 [49,117]
+ CRUSH rule 3 x 476 [110,31]
+ CRUSH rule 3 x 477 [80,97]
+ CRUSH rule 3 x 478 [78,99]
+ CRUSH rule 3 x 479 [31,66]
+ CRUSH rule 3 x 480 [75,88]
+ CRUSH rule 3 x 481 [26,20]
+ CRUSH rule 3 x 482 [84,53]
+ CRUSH rule 3 x 483 [15,116]
+ CRUSH rule 3 x 484 [37,114]
+ CRUSH rule 3 x 485 [84,8]
+ CRUSH rule 3 x 486 [92,10]
+ CRUSH rule 3 x 487 [106,17]
+ CRUSH rule 3 x 488 [42,20]
+ CRUSH rule 3 x 489 [89,2]
+ CRUSH rule 3 x 490 [22,114]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,66]
+ CRUSH rule 3 x 493 [94,14]
+ CRUSH rule 3 x 494 [59,86]
+ CRUSH rule 3 x 495 [95,58]
+ CRUSH rule 3 x 496 [46,41]
+ CRUSH rule 3 x 497 [102,27]
+ CRUSH rule 3 x 498 [21,116]
+ CRUSH rule 3 x 499 [5,49]
+ CRUSH rule 3 x 500 [50,49]
+ CRUSH rule 3 x 501 [60,3]
+ CRUSH rule 3 x 502 [65,110]
+ CRUSH rule 3 x 503 [21,112]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,93]
+ CRUSH rule 3 x 506 [79,64]
+ CRUSH rule 3 x 507 [34,107]
+ CRUSH rule 3 x 508 [45,114]
+ CRUSH rule 3 x 509 [19,88]
+ CRUSH rule 3 x 510 [117,45]
+ CRUSH rule 3 x 511 [14,104]
+ CRUSH rule 3 x 512 [59,26]
+ CRUSH rule 3 x 513 [102,93]
+ CRUSH rule 3 x 514 [75,72]
+ CRUSH rule 3 x 515 [84,41]
+ CRUSH rule 3 x 516 [37,30]
+ CRUSH rule 3 x 517 [83,115]
+ CRUSH rule 3 x 518 [18,83]
+ CRUSH rule 3 x 519 [67,88]
+ CRUSH rule 3 x 520 [15,114]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,51]
+ CRUSH rule 3 x 523 [68,101]
+ CRUSH rule 3 x 524 [33,38]
+ CRUSH rule 3 x 525 [63,115]
+ CRUSH rule 3 x 526 [83,50]
+ CRUSH rule 3 x 527 [37,56]
+ CRUSH rule 3 x 528 [108,81]
+ CRUSH rule 3 x 529 [74,33]
+ CRUSH rule 3 x 530 [49,92]
+ CRUSH rule 3 x 531 [117,105]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,85]
+ CRUSH rule 3 x 534 [97,24]
+ CRUSH rule 3 x 535 [48,75]
+ CRUSH rule 3 x 536 [113,101]
+ CRUSH rule 3 x 537 [116,47]
+ CRUSH rule 3 x 538 [85,74]
+ CRUSH rule 3 x 539 [72,43]
+ CRUSH rule 3 x 540 [39,34]
+ CRUSH rule 3 x 541 [53,84]
+ CRUSH rule 3 x 542 [27,32]
+ CRUSH rule 3 x 543 [45,113]
+ CRUSH rule 3 x 544 [59,42]
+ CRUSH rule 3 x 545 [118,95]
+ CRUSH rule 3 x 546 [18,79]
+ CRUSH rule 3 x 547 [67,30]
+ CRUSH rule 3 x 548 [53,100]
+ CRUSH rule 3 x 549 [60,45]
+ CRUSH rule 3 x 550 [92,101]
+ CRUSH rule 3 x 551 [77,88]
+ CRUSH rule 3 x 552 [61,94]
+ CRUSH rule 3 x 553 [71,78]
+ CRUSH rule 3 x 554 [61,115]
+ CRUSH rule 3 x 555 [76,77]
+ CRUSH rule 3 x 556 [106,55]
+ CRUSH rule 3 x 557 [26,22]
+ CRUSH rule 3 x 558 [41,84]
+ CRUSH rule 3 x 559 [65,24]
+ CRUSH rule 3 x 560 [94,16]
+ CRUSH rule 3 x 561 [27,5]
+ CRUSH rule 3 x 562 [78,59]
+ CRUSH rule 3 x 563 [59,70]
+ CRUSH rule 3 x 564 [96,8]
+ CRUSH rule 3 x 565 [8,48]
+ CRUSH rule 3 x 566 [119,17]
+ CRUSH rule 3 x 567 [7,38]
+ CRUSH rule 3 x 568 [57,94]
+ CRUSH rule 3 x 569 [65,26]
+ CRUSH rule 3 x 570 [98,27]
+ CRUSH rule 3 x 571 [95,30]
+ CRUSH rule 3 x 572 [62,83]
+ CRUSH rule 3 x 573 [1,79]
+ CRUSH rule 3 x 574 [89,42]
+ CRUSH rule 3 x 575 [87,113]
+ CRUSH rule 3 x 576 [21,68]
+ CRUSH rule 3 x 577 [8,84]
+ CRUSH rule 3 x 578 [75,115]
+ CRUSH rule 3 x 579 [105,68]
+ CRUSH rule 3 x 580 [51,28]
+ CRUSH rule 3 x 581 [55,113]
+ CRUSH rule 3 x 582 [27,113]
+ CRUSH rule 3 x 583 [6,78]
+ CRUSH rule 3 x 584 [10,30]
+ CRUSH rule 3 x 585 [20,111]
+ CRUSH rule 3 x 586 [48,27]
+ CRUSH rule 3 x 587 [29,94]
+ CRUSH rule 3 x 588 [103,90]
+ CRUSH rule 3 x 589 [88,95]
+ CRUSH rule 3 x 590 [76,101]
+ CRUSH rule 3 x 591 [42,43]
+ CRUSH rule 3 x 592 [78,51]
+ CRUSH rule 3 x 593 [82,71]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,59]
+ CRUSH rule 3 x 597 [16,36]
+ CRUSH rule 3 x 598 [37,56]
+ CRUSH rule 3 x 599 [10,84]
+ CRUSH rule 3 x 600 [24,69]
+ CRUSH rule 3 x 601 [104,14]
+ CRUSH rule 3 x 602 [48,45]
+ CRUSH rule 3 x 603 [93,32]
+ CRUSH rule 3 x 604 [118,79]
+ CRUSH rule 3 x 605 [104,53]
+ CRUSH rule 3 x 606 [90,83]
+ CRUSH rule 3 x 607 [95,110]
+ CRUSH rule 3 x 608 [112,101]
+ CRUSH rule 3 x 609 [34,99]
+ CRUSH rule 3 x 610 [106,16]
+ CRUSH rule 3 x 611 [66,87]
+ CRUSH rule 3 x 612 [2,81]
+ CRUSH rule 3 x 613 [13,86]
+ CRUSH rule 3 x 614 [50,3]
+ CRUSH rule 3 x 615 [24,73]
+ CRUSH rule 3 x 616 [41,119]
+ CRUSH rule 3 x 617 [81,106]
+ CRUSH rule 3 x 618 [3,104]
+ CRUSH rule 3 x 619 [92,7]
+ CRUSH rule 3 x 620 [108,11]
+ CRUSH rule 3 x 621 [105,115]
+ CRUSH rule 3 x 622 [67,48]
+ CRUSH rule 3 x 623 [69,74]
+ CRUSH rule 3 x 624 [115,49]
+ CRUSH rule 3 x 625 [73,109]
+ CRUSH rule 3 x 626 [52,3]
+ CRUSH rule 3 x 627 [116,3]
+ CRUSH rule 3 x 628 [98,91]
+ CRUSH rule 3 x 629 [6,112]
+ CRUSH rule 3 x 630 [22,72]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,71]
+ CRUSH rule 3 x 633 [65,12]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,46]
+ CRUSH rule 3 x 636 [23,70]
+ CRUSH rule 3 x 637 [99,24]
+ CRUSH rule 3 x 638 [43,114]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,73]
+ CRUSH rule 3 x 641 [45,84]
+ CRUSH rule 3 x 642 [47,66]
+ CRUSH rule 3 x 643 [64,8]
+ CRUSH rule 3 x 644 [31,82]
+ CRUSH rule 3 x 645 [77,64]
+ CRUSH rule 3 x 646 [37,86]
+ CRUSH rule 3 x 647 [65,56]
+ CRUSH rule 3 x 648 [84,13]
+ CRUSH rule 3 x 649 [88,55]
+ CRUSH rule 3 x 650 [21,76]
+ CRUSH rule 3 x 651 [63,116]
+ CRUSH rule 3 x 652 [57,112]
+ CRUSH rule 3 x 653 [38,61]
+ CRUSH rule 3 x 654 [104,67]
+ CRUSH rule 3 x 655 [89,54]
+ CRUSH rule 3 x 656 [84,49]
+ CRUSH rule 3 x 657 [47,32]
+ CRUSH rule 3 x 658 [80,29]
+ CRUSH rule 3 x 659 [11,112]
+ CRUSH rule 3 x 660 [65,111]
+ CRUSH rule 3 x 661 [96,73]
+ CRUSH rule 3 x 662 [111,73]
+ CRUSH rule 3 x 663 [83,60]
+ CRUSH rule 3 x 664 [59,80]
+ CRUSH rule 3 x 665 [31,117]
+ CRUSH rule 3 x 666 [112,101]
+ CRUSH rule 3 x 667 [70,47]
+ CRUSH rule 3 x 668 [96,57]
+ CRUSH rule 3 x 669 [56,39]
+ CRUSH rule 3 x 670 [98,105]
+ CRUSH rule 3 x 671 [57,48]
+ CRUSH rule 3 x 672 [37,36]
+ CRUSH rule 3 x 673 [83,2]
+ CRUSH rule 3 x 674 [36,25]
+ CRUSH rule 3 x 675 [88,14]
+ CRUSH rule 3 x 676 [3,110]
+ CRUSH rule 3 x 677 [88,67]
+ CRUSH rule 3 x 678 [27,44]
+ CRUSH rule 3 x 679 [33,116]
+ CRUSH rule 3 x 680 [111,39]
+ CRUSH rule 3 x 681 [53,12]
+ CRUSH rule 3 x 682 [12,87]
+ CRUSH rule 3 x 683 [24,85]
+ CRUSH rule 3 x 684 [98,65]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,72]
+ CRUSH rule 3 x 688 [16,114]
+ CRUSH rule 3 x 689 [32,31]
+ CRUSH rule 3 x 690 [96,33]
+ CRUSH rule 3 x 691 [34,6]
+ CRUSH rule 3 x 692 [97,84]
+ CRUSH rule 3 x 693 [29,118]
+ CRUSH rule 3 x 694 [6,30]
+ CRUSH rule 3 x 695 [31,72]
+ CRUSH rule 3 x 696 [104,97]
+ CRUSH rule 3 x 697 [19,96]
+ CRUSH rule 3 x 698 [30,69]
+ CRUSH rule 3 x 699 [47,76]
+ CRUSH rule 3 x 700 [82,55]
+ CRUSH rule 3 x 701 [53,80]
+ CRUSH rule 3 x 702 [95,98]
+ CRUSH rule 3 x 703 [92,65]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,1]
+ CRUSH rule 3 x 706 [74,35]
+ CRUSH rule 3 x 707 [91,115]
+ CRUSH rule 3 x 708 [95,112]
+ CRUSH rule 3 x 709 [73,72]
+ CRUSH rule 3 x 710 [94,47]
+ CRUSH rule 3 x 711 [68,41]
+ CRUSH rule 3 x 712 [107,18]
+ CRUSH rule 3 x 713 [29,109]
+ CRUSH rule 3 x 714 [86,61]
+ CRUSH rule 3 x 715 [74,13]
+ CRUSH rule 3 x 716 [101,56]
+ CRUSH rule 3 x 717 [12,29]
+ CRUSH rule 3 x 718 [83,24]
+ CRUSH rule 3 x 719 [26,10]
+ CRUSH rule 3 x 720 [69,2]
+ CRUSH rule 3 x 721 [51,42]
+ CRUSH rule 3 x 722 [15,74]
+ CRUSH rule 3 x 723 [117,14]
+ CRUSH rule 3 x 724 [45,38]
+ CRUSH rule 3 x 725 [53,110]
+ CRUSH rule 3 x 726 [103,68]
+ CRUSH rule 3 x 727 [89,100]
+ CRUSH rule 3 x 728 [76,16]
+ CRUSH rule 3 x 729 [35,90]
+ CRUSH rule 3 x 730 [28,103]
+ CRUSH rule 3 x 731 [78,41]
+ CRUSH rule 3 x 732 [1,27]
+ CRUSH rule 3 x 733 [35,100]
+ CRUSH rule 3 x 734 [119,85]
+ CRUSH rule 3 x 735 [102,43]
+ CRUSH rule 3 x 736 [37,92]
+ CRUSH rule 3 x 737 [117,11]
+ CRUSH rule 3 x 738 [57,32]
+ CRUSH rule 3 x 739 [87,1]
+ CRUSH rule 3 x 740 [29,80]
+ CRUSH rule 3 x 741 [47,111]
+ CRUSH rule 3 x 742 [106,83]
+ CRUSH rule 3 x 743 [105,94]
+ CRUSH rule 3 x 744 [23,64]
+ CRUSH rule 3 x 745 [37,112]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,95]
+ CRUSH rule 3 x 748 [48,14]
+ CRUSH rule 3 x 749 [102,101]
+ CRUSH rule 3 x 750 [83,78]
+ CRUSH rule 3 x 751 [25,104]
+ CRUSH rule 3 x 752 [82,95]
+ CRUSH rule 3 x 753 [14,113]
+ CRUSH rule 3 x 754 [114,51]
+ CRUSH rule 3 x 755 [87,26]
+ CRUSH rule 3 x 756 [113,87]
+ CRUSH rule 3 x 757 [47,66]
+ CRUSH rule 3 x 758 [54,63]
+ CRUSH rule 3 x 759 [74,20]
+ CRUSH rule 3 x 760 [88,22]
+ CRUSH rule 3 x 761 [73,86]
+ CRUSH rule 3 x 762 [34,17]
+ CRUSH rule 3 x 763 [13,78]
+ CRUSH rule 3 x 764 [89,42]
+ CRUSH rule 3 x 765 [109,91]
+ CRUSH rule 3 x 766 [19,66]
+ CRUSH rule 3 x 767 [41,26]
+ CRUSH rule 3 x 768 [106,57]
+ CRUSH rule 3 x 769 [91,104]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,35]
+ CRUSH rule 3 x 772 [97,108]
+ CRUSH rule 3 x 773 [116,47]
+ CRUSH rule 3 x 774 [100,31]
+ CRUSH rule 3 x 775 [102,43]
+ CRUSH rule 3 x 776 [69,38]
+ CRUSH rule 3 x 777 [91,52]
+ CRUSH rule 3 x 778 [83,119]
+ CRUSH rule 3 x 779 [47,60]
+ CRUSH rule 3 x 780 [63,70]
+ CRUSH rule 3 x 781 [105,2]
+ CRUSH rule 3 x 782 [117,59]
+ CRUSH rule 3 x 783 [19,109]
+ CRUSH rule 3 x 784 [63,114]
+ CRUSH rule 3 x 785 [27,84]
+ CRUSH rule 3 x 786 [41,110]
+ CRUSH rule 3 x 787 [108,73]
+ CRUSH rule 3 x 788 [74,103]
+ CRUSH rule 3 x 789 [50,17]
+ CRUSH rule 3 x 790 [20,106]
+ CRUSH rule 3 x 791 [96,87]
+ CRUSH rule 3 x 792 [80,97]
+ CRUSH rule 3 x 793 [6,26]
+ CRUSH rule 3 x 794 [14,42]
+ CRUSH rule 3 x 795 [30,8]
+ CRUSH rule 3 x 796 [87,36]
+ CRUSH rule 3 x 797 [64,61]
+ CRUSH rule 3 x 798 [42,69]
+ CRUSH rule 3 x 799 [19,117]
+ CRUSH rule 3 x 800 [106,8]
+ CRUSH rule 3 x 801 [2,57]
+ CRUSH rule 3 x 802 [63,68]
+ CRUSH rule 3 x 803 [46,35]
+ CRUSH rule 3 x 804 [33,26]
+ CRUSH rule 3 x 805 [96,49]
+ CRUSH rule 3 x 806 [48,25]
+ CRUSH rule 3 x 807 [48,83]
+ CRUSH rule 3 x 808 [76,31]
+ CRUSH rule 3 x 809 [27,48]
+ CRUSH rule 3 x 810 [119,71]
+ CRUSH rule 3 x 811 [111,91]
+ CRUSH rule 3 x 812 [25,111]
+ CRUSH rule 3 x 813 [81,28]
+ CRUSH rule 3 x 814 [95,42]
+ CRUSH rule 3 x 815 [84,61]
+ CRUSH rule 3 x 816 [64,35]
+ CRUSH rule 3 x 817 [63,60]
+ CRUSH rule 3 x 818 [69,46]
+ CRUSH rule 3 x 819 [88,75]
+ CRUSH rule 3 x 820 [104,57]
+ CRUSH rule 3 x 821 [58,21]
+ CRUSH rule 3 x 822 [20,80]
+ CRUSH rule 3 x 823 [63,118]
+ CRUSH rule 3 x 824 [102,13]
+ CRUSH rule 3 x 825 [47,118]
+ CRUSH rule 3 x 826 [44,7]
+ CRUSH rule 3 x 827 [101,88]
+ CRUSH rule 3 x 828 [60,41]
+ CRUSH rule 3 x 829 [45,102]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,75]
+ CRUSH rule 3 x 833 [57,32]
+ CRUSH rule 3 x 834 [90,33]
+ CRUSH rule 3 x 835 [6,1]
+ CRUSH rule 3 x 836 [63,68]
+ CRUSH rule 3 x 837 [76,71]
+ CRUSH rule 3 x 838 [106,20]
+ CRUSH rule 3 x 839 [87,96]
+ CRUSH rule 3 x 840 [33,32]
+ CRUSH rule 3 x 841 [110,55]
+ CRUSH rule 3 x 842 [66,87]
+ CRUSH rule 3 x 843 [11,80]
+ CRUSH rule 3 x 844 [74,103]
+ CRUSH rule 3 x 845 [74,43]
+ CRUSH rule 3 x 846 [43,76]
+ CRUSH rule 3 x 847 [62,20]
+ CRUSH rule 3 x 848 [92,17]
+ CRUSH rule 3 x 849 [93,36]
+ CRUSH rule 3 x 850 [83,82]
+ CRUSH rule 3 x 851 [65,94]
+ CRUSH rule 3 x 852 [60,22]
+ CRUSH rule 3 x 853 [88,29]
+ CRUSH rule 3 x 854 [83,54]
+ CRUSH rule 3 x 855 [2,101]
+ CRUSH rule 3 x 856 [40,41]
+ CRUSH rule 3 x 857 [69,82]
+ CRUSH rule 3 x 858 [98,81]
+ CRUSH rule 3 x 859 [56,43]
+ CRUSH rule 3 x 860 [11,26]
+ CRUSH rule 3 x 861 [22,110]
+ CRUSH rule 3 x 862 [22,70]
+ CRUSH rule 3 x 863 [79,84]
+ CRUSH rule 3 x 864 [77,24]
+ CRUSH rule 3 x 865 [119,17]
+ CRUSH rule 3 x 866 [18,49]
+ CRUSH rule 3 x 867 [3,84]
+ CRUSH rule 3 x 868 [100,107]
+ CRUSH rule 3 x 869 [22,104]
+ CRUSH rule 3 x 870 [73,30]
+ CRUSH rule 3 x 871 [84,105]
+ CRUSH rule 3 x 872 [72,75]
+ CRUSH rule 3 x 873 [81,96]
+ CRUSH rule 3 x 874 [21,72]
+ CRUSH rule 3 x 875 [115,59]
+ CRUSH rule 3 x 876 [98,49]
+ CRUSH rule 3 x 877 [80,79]
+ CRUSH rule 3 x 878 [87,94]
+ CRUSH rule 3 x 879 [29,18]
+ CRUSH rule 3 x 880 [23,40]
+ CRUSH rule 3 x 881 [109,69]
+ CRUSH rule 3 x 882 [31,118]
+ CRUSH rule 3 x 883 [102,8]
+ CRUSH rule 3 x 884 [80,19]
+ CRUSH rule 3 x 885 [46,101]
+ CRUSH rule 3 x 886 [2,65]
+ CRUSH rule 3 x 887 [5,99]
+ CRUSH rule 3 x 888 [16,70]
+ CRUSH rule 3 x 889 [84,93]
+ CRUSH rule 3 x 890 [65,118]
+ CRUSH rule 3 x 891 [86,105]
+ CRUSH rule 3 x 892 [64,10]
+ CRUSH rule 3 x 893 [20,110]
+ CRUSH rule 3 x 894 [32,47]
+ CRUSH rule 3 x 895 [40,21]
+ CRUSH rule 3 x 896 [113,14]
+ CRUSH rule 3 x 897 [107,80]
+ CRUSH rule 3 x 898 [76,71]
+ CRUSH rule 3 x 899 [75,82]
+ CRUSH rule 3 x 900 [83,82]
+ CRUSH rule 3 x 901 [66,61]
+ CRUSH rule 3 x 902 [25,56]
+ CRUSH rule 3 x 903 [53,46]
+ CRUSH rule 3 x 904 [50,101]
+ CRUSH rule 3 x 905 [99,110]
+ CRUSH rule 3 x 906 [68,27]
+ CRUSH rule 3 x 907 [109,47]
+ CRUSH rule 3 x 908 [47,1]
+ CRUSH rule 3 x 909 [73,2]
+ CRUSH rule 3 x 910 [71,74]
+ CRUSH rule 3 x 911 [39,42]
+ CRUSH rule 3 x 912 [90,7]
+ CRUSH rule 3 x 913 [29,96]
+ CRUSH rule 3 x 914 [84,45]
+ CRUSH rule 3 x 915 [49,115]
+ CRUSH rule 3 x 916 [32,77]
+ CRUSH rule 3 x 917 [46,23]
+ CRUSH rule 3 x 918 [82,73]
+ CRUSH rule 3 x 919 [13,28]
+ CRUSH rule 3 x 920 [25,26]
+ CRUSH rule 3 x 921 [55,119]
+ CRUSH rule 3 x 922 [33,32]
+ CRUSH rule 3 x 923 [28,15]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,22]
+ CRUSH rule 3 x 926 [64,69]
+ CRUSH rule 3 x 927 [32,16]
+ CRUSH rule 3 x 928 [13,113]
+ CRUSH rule 3 x 929 [85,115]
+ CRUSH rule 3 x 930 [104,20]
+ CRUSH rule 3 x 931 [46,79]
+ CRUSH rule 3 x 932 [43,52]
+ CRUSH rule 3 x 933 [18,55]
+ CRUSH rule 3 x 934 [68,81]
+ CRUSH rule 3 x 935 [28,57]
+ CRUSH rule 3 x 936 [104,57]
+ CRUSH rule 3 x 937 [110,10]
+ CRUSH rule 3 x 938 [48,23]
+ CRUSH rule 3 x 939 [77,42]
+ CRUSH rule 3 x 940 [76,49]
+ CRUSH rule 3 x 941 [66,101]
+ CRUSH rule 3 x 942 [80,87]
+ CRUSH rule 3 x 943 [75,74]
+ CRUSH rule 3 x 944 [82,53]
+ CRUSH rule 3 x 945 [71,74]
+ CRUSH rule 3 x 946 [37,42]
+ CRUSH rule 3 x 947 [107,30]
+ CRUSH rule 3 x 948 [108,95]
+ CRUSH rule 3 x 949 [46,103]
+ CRUSH rule 3 x 950 [96,101]
+ CRUSH rule 3 x 951 [40,14]
+ CRUSH rule 3 x 952 [114,21]
+ CRUSH rule 3 x 953 [62,23]
+ CRUSH rule 3 x 954 [103,5]
+ CRUSH rule 3 x 955 [42,73]
+ CRUSH rule 3 x 956 [72,103]
+ CRUSH rule 3 x 957 [117,22]
+ CRUSH rule 3 x 958 [23,106]
+ CRUSH rule 3 x 959 [42,93]
+ CRUSH rule 3 x 960 [113,8]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,51]
+ CRUSH rule 3 x 963 [101,106]
+ CRUSH rule 3 x 964 [66,89]
+ CRUSH rule 3 x 965 [47,102]
+ CRUSH rule 3 x 966 [88,63]
+ CRUSH rule 3 x 967 [71,46]
+ CRUSH rule 3 x 968 [74,51]
+ CRUSH rule 3 x 969 [53,78]
+ CRUSH rule 3 x 970 [3,30]
+ CRUSH rule 3 x 971 [66,107]
+ CRUSH rule 3 x 972 [3,66]
+ CRUSH rule 3 x 973 [113,20]
+ CRUSH rule 3 x 974 [114,35]
+ CRUSH rule 3 x 975 [83,58]
+ CRUSH rule 3 x 976 [81,48]
+ CRUSH rule 3 x 977 [95,102]
+ CRUSH rule 3 x 978 [119,41]
+ CRUSH rule 3 x 979 [98,6]
+ CRUSH rule 3 x 980 [39,108]
+ CRUSH rule 3 x 981 [89,84]
+ CRUSH rule 3 x 982 [19,94]
+ CRUSH rule 3 x 983 [34,45]
+ CRUSH rule 3 x 984 [78,63]
+ CRUSH rule 3 x 985 [99,52]
+ CRUSH rule 3 x 986 [44,99]
+ CRUSH rule 3 x 987 [25,32]
+ CRUSH rule 3 x 988 [79,2]
+ CRUSH rule 3 x 989 [87,26]
+ CRUSH rule 3 x 990 [72,69]
+ CRUSH rule 3 x 991 [90,8]
+ CRUSH rule 3 x 992 [30,67]
+ CRUSH rule 3 x 993 [74,49]
+ CRUSH rule 3 x 994 [74,105]
+ CRUSH rule 3 x 995 [100,97]
+ CRUSH rule 3 x 996 [41,58]
+ CRUSH rule 3 x 997 [89,76]
+ CRUSH rule 3 x 998 [92,47]
+ CRUSH rule 3 x 999 [117,16]
+ CRUSH rule 3 x 1000 [50,47]
+ CRUSH rule 3 x 1001 [83,102]
+ CRUSH rule 3 x 1002 [94,37]
+ CRUSH rule 3 x 1003 [43,88]
+ CRUSH rule 3 x 1004 [89,54]
+ CRUSH rule 3 x 1005 [105,84]
+ CRUSH rule 3 x 1006 [45,111]
+ CRUSH rule 3 x 1007 [19,66]
+ CRUSH rule 3 x 1008 [31,76]
+ CRUSH rule 3 x 1009 [1,95]
+ CRUSH rule 3 x 1010 [31,113]
+ CRUSH rule 3 x 1011 [64,81]
+ CRUSH rule 3 x 1012 [68,49]
+ CRUSH rule 3 x 1013 [5,93]
+ CRUSH rule 3 x 1014 [33,66]
+ CRUSH rule 3 x 1015 [106,45]
+ CRUSH rule 3 x 1016 [107,86]
+ CRUSH rule 3 x 1017 [12,61]
+ CRUSH rule 3 x 1018 [61,26]
+ CRUSH rule 3 x 1019 [27,104]
+ CRUSH rule 3 x 1020 [31,86]
+ CRUSH rule 3 x 1021 [22,26]
+ CRUSH rule 3 x 1022 [73,34]
+ CRUSH rule 3 x 1023 [88,79]
+ rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,6]
+ CRUSH rule 3 x 1 [73,52]
+ CRUSH rule 3 x 2 [91,48]
+ CRUSH rule 3 x 3 [51,48]
+ CRUSH rule 3 x 4 [45,114]
+ CRUSH rule 3 x 5 [89,94]
+ CRUSH rule 3 x 6 [91,76]
+ CRUSH rule 3 x 7 [104,73]
+ CRUSH rule 3 x 8 [41,98]
+ CRUSH rule 3 x 9 [46,47]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,40]
+ CRUSH rule 3 x 12 [83,26]
+ CRUSH rule 3 x 13 [27,28]
+ CRUSH rule 3 x 14 [105,64]
+ CRUSH rule 3 x 15 [18,7]
+ CRUSH rule 3 x 16 [103,30]
+ CRUSH rule 3 x 17 [85,118]
+ CRUSH rule 3 x 18 [11,106]
+ CRUSH rule 3 x 19 [75,50]
+ CRUSH rule 3 x 20 [111,67]
+ CRUSH rule 3 x 21 [84,61]
+ CRUSH rule 3 x 22 [23,104]
+ CRUSH rule 3 x 23 [19,86]
+ CRUSH rule 3 x 24 [83,60]
+ CRUSH rule 3 x 25 [81,64]
+ CRUSH rule 3 x 26 [17,38]
+ CRUSH rule 3 x 27 [33,84]
+ CRUSH rule 3 x 28 [45,90]
+ CRUSH rule 3 x 29 [8,109]
+ CRUSH rule 3 x 30 [55,42]
+ CRUSH rule 3 x 31 [76,95]
+ CRUSH rule 3 x 32 [72,11]
+ CRUSH rule 3 x 33 [86,53]
+ CRUSH rule 3 x 34 [7,108]
+ CRUSH rule 3 x 35 [108,13]
+ CRUSH rule 3 x 36 [67,66]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,105]
+ CRUSH rule 3 x 39 [68,103]
+ CRUSH rule 3 x 40 [30,85]
+ CRUSH rule 3 x 41 [52,11]
+ CRUSH rule 3 x 42 [106,75]
+ CRUSH rule 3 x 43 [10,104]
+ CRUSH rule 3 x 44 [101,28]
+ CRUSH rule 3 x 45 [83,64]
+ CRUSH rule 3 x 46 [54,31]
+ CRUSH rule 3 x 47 [106,61]
+ CRUSH rule 3 x 48 [34,41]
+ CRUSH rule 3 x 49 [79,110]
+ CRUSH rule 3 x 50 [42,13]
+ CRUSH rule 3 x 51 [6,94]
+ CRUSH rule 3 x 52 [82,19]
+ CRUSH rule 3 x 53 [32,91]
+ CRUSH rule 3 x 54 [108,8]
+ CRUSH rule 3 x 55 [14,94]
+ CRUSH rule 3 x 56 [21,72]
+ CRUSH rule 3 x 57 [69,88]
+ CRUSH rule 3 x 58 [48,87]
+ CRUSH rule 3 x 59 [21,113]
+ CRUSH rule 3 x 60 [90,73]
+ CRUSH rule 3 x 61 [88,63]
+ CRUSH rule 3 x 62 [100,13]
+ CRUSH rule 3 x 63 [79,5]
+ CRUSH rule 3 x 64 [1,89]
+ CRUSH rule 3 x 65 [32,103]
+ CRUSH rule 3 x 66 [48,79]
+ CRUSH rule 3 x 67 [94,11]
+ CRUSH rule 3 x 68 [102,15]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,11]
+ CRUSH rule 3 x 71 [12,33]
+ CRUSH rule 3 x 72 [26,99]
+ CRUSH rule 3 x 73 [29,114]
+ CRUSH rule 3 x 74 [29,1]
+ CRUSH rule 3 x 75 [60,65]
+ CRUSH rule 3 x 76 [55,62]
+ CRUSH rule 3 x 77 [107,100]
+ CRUSH rule 3 x 78 [86,107]
+ CRUSH rule 3 x 79 [64,16]
+ CRUSH rule 3 x 80 [19,100]
+ CRUSH rule 3 x 81 [64,16]
+ CRUSH rule 3 x 82 [37,40]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,115]
+ CRUSH rule 3 x 85 [87,88]
+ CRUSH rule 3 x 86 [37,68]
+ CRUSH rule 3 x 87 [116,77]
+ CRUSH rule 3 x 88 [38,55]
+ CRUSH rule 3 x 89 [76,25]
+ CRUSH rule 3 x 90 [14,50]
+ CRUSH rule 3 x 91 [68,61]
+ CRUSH rule 3 x 92 [86,95]
+ CRUSH rule 3 x 93 [44,35]
+ CRUSH rule 3 x 94 [46,71]
+ CRUSH rule 3 x 95 [108,53]
+ CRUSH rule 3 x 96 [66,87]
+ CRUSH rule 3 x 97 [111,45]
+ CRUSH rule 3 x 98 [93,110]
+ CRUSH rule 3 x 99 [78,43]
+ CRUSH rule 3 x 100 [28,61]
+ CRUSH rule 3 x 101 [91,110]
+ CRUSH rule 3 x 102 [82,7]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,79]
+ CRUSH rule 3 x 105 [34,87]
+ CRUSH rule 3 x 106 [69,12]
+ CRUSH rule 3 x 107 [1,59]
+ CRUSH rule 3 x 108 [7,109]
+ CRUSH rule 3 x 109 [112,67]
+ CRUSH rule 3 x 110 [54,61]
+ CRUSH rule 3 x 111 [10,92]
+ CRUSH rule 3 x 112 [80,11]
+ CRUSH rule 3 x 113 [69,38]
+ CRUSH rule 3 x 114 [79,38]
+ CRUSH rule 3 x 115 [10,48]
+ CRUSH rule 3 x 116 [37,108]
+ CRUSH rule 3 x 117 [87,56]
+ CRUSH rule 3 x 118 [23,56]
+ CRUSH rule 3 x 119 [104,31]
+ CRUSH rule 3 x 120 [44,93]
+ CRUSH rule 3 x 121 [80,16]
+ CRUSH rule 3 x 122 [45,54]
+ CRUSH rule 3 x 123 [22,112]
+ CRUSH rule 3 x 124 [97,50]
+ CRUSH rule 3 x 125 [66,6]
+ CRUSH rule 3 x 126 [70,39]
+ CRUSH rule 3 x 127 [70,75]
+ CRUSH rule 3 x 128 [11,111]
+ CRUSH rule 3 x 129 [103,46]
+ CRUSH rule 3 x 130 [50,73]
+ CRUSH rule 3 x 131 [44,15]
+ CRUSH rule 3 x 132 [69,58]
+ CRUSH rule 3 x 133 [67,115]
+ CRUSH rule 3 x 134 [37,60]
+ CRUSH rule 3 x 135 [78,61]
+ CRUSH rule 3 x 136 [32,29]
+ CRUSH rule 3 x 137 [92,87]
+ CRUSH rule 3 x 138 [54,8]
+ CRUSH rule 3 x 139 [89,60]
+ CRUSH rule 3 x 140 [39,50]
+ CRUSH rule 3 x 141 [89,62]
+ CRUSH rule 3 x 142 [22,86]
+ CRUSH rule 3 x 143 [96,16]
+ CRUSH rule 3 x 144 [13,1]
+ CRUSH rule 3 x 145 [77,54]
+ CRUSH rule 3 x 146 [12,43]
+ CRUSH rule 3 x 147 [2,59]
+ CRUSH rule 3 x 148 [85,50]
+ CRUSH rule 3 x 149 [103,68]
+ CRUSH rule 3 x 150 [14,50]
+ CRUSH rule 3 x 151 [75,56]
+ CRUSH rule 3 x 152 [49,18]
+ CRUSH rule 3 x 153 [92,79]
+ CRUSH rule 3 x 154 [19,26]
+ CRUSH rule 3 x 155 [12,13]
+ CRUSH rule 3 x 156 [107,18]
+ CRUSH rule 3 x 157 [15,78]
+ CRUSH rule 3 x 158 [11,28]
+ CRUSH rule 3 x 159 [33,88]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,78]
+ CRUSH rule 3 x 162 [55,96]
+ CRUSH rule 3 x 163 [54,55]
+ CRUSH rule 3 x 164 [72,99]
+ CRUSH rule 3 x 165 [25,116]
+ CRUSH rule 3 x 166 [2,23]
+ CRUSH rule 3 x 167 [89,40]
+ CRUSH rule 3 x 168 [68,49]
+ CRUSH rule 3 x 169 [51,50]
+ CRUSH rule 3 x 170 [68,91]
+ CRUSH rule 3 x 171 [88,51]
+ CRUSH rule 3 x 172 [117,23]
+ CRUSH rule 3 x 173 [29,1]
+ CRUSH rule 3 x 174 [67,40]
+ CRUSH rule 3 x 175 [48,41]
+ CRUSH rule 3 x 176 [94,91]
+ CRUSH rule 3 x 177 [53,70]
+ CRUSH rule 3 x 178 [39,110]
+ CRUSH rule 3 x 179 [72,89]
+ CRUSH rule 3 x 180 [3,38]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,46]
+ CRUSH rule 3 x 183 [11,78]
+ CRUSH rule 3 x 184 [79,92]
+ CRUSH rule 3 x 185 [97,92]
+ CRUSH rule 3 x 186 [67,116]
+ CRUSH rule 3 x 187 [6,96]
+ CRUSH rule 3 x 188 [76,16]
+ CRUSH rule 3 x 189 [96,71]
+ CRUSH rule 3 x 190 [90,6]
+ CRUSH rule 3 x 191 [49,84]
+ CRUSH rule 3 x 192 [93,114]
+ CRUSH rule 3 x 193 [89,60]
+ CRUSH rule 3 x 194 [62,61]
+ CRUSH rule 3 x 195 [119,95]
+ CRUSH rule 3 x 196 [20,28]
+ CRUSH rule 3 x 197 [6,64]
+ CRUSH rule 3 x 198 [55,112]
+ CRUSH rule 3 x 199 [66,17]
+ CRUSH rule 3 x 200 [12,63]
+ CRUSH rule 3 x 201 [52,23]
+ CRUSH rule 3 x 202 [98,33]
+ CRUSH rule 3 x 203 [36,22]
+ CRUSH rule 3 x 204 [10,100]
+ CRUSH rule 3 x 205 [38,29]
+ CRUSH rule 3 x 206 [38,27]
+ CRUSH rule 3 x 207 [19,108]
+ CRUSH rule 3 x 208 [63,26]
+ CRUSH rule 3 x 209 [70,11]
+ CRUSH rule 3 x 210 [79,58]
+ CRUSH rule 3 x 211 [26,59]
+ CRUSH rule 3 x 212 [107,114]
+ CRUSH rule 3 x 213 [100,3]
+ CRUSH rule 3 x 214 [91,118]
+ CRUSH rule 3 x 215 [92,16]
+ CRUSH rule 3 x 216 [99,94]
+ CRUSH rule 3 x 217 [86,99]
+ CRUSH rule 3 x 218 [70,15]
+ CRUSH rule 3 x 219 [61,58]
+ CRUSH rule 3 x 220 [23,56]
+ CRUSH rule 3 x 221 [21,92]
+ CRUSH rule 3 x 222 [102,29]
+ CRUSH rule 3 x 223 [34,73]
+ CRUSH rule 3 x 224 [107,68]
+ CRUSH rule 3 x 225 [61,98]
+ CRUSH rule 3 x 226 [44,13]
+ CRUSH rule 3 x 227 [55,60]
+ CRUSH rule 3 x 228 [117,55]
+ CRUSH rule 3 x 229 [100,67]
+ CRUSH rule 3 x 230 [41,109]
+ CRUSH rule 3 x 231 [30,71]
+ CRUSH rule 3 x 232 [23,1]
+ CRUSH rule 3 x 233 [47,90]
+ CRUSH rule 3 x 234 [55,62]
+ CRUSH rule 3 x 235 [20,60]
+ CRUSH rule 3 x 236 [95,24]
+ CRUSH rule 3 x 237 [21,106]
+ CRUSH rule 3 x 238 [109,15]
+ CRUSH rule 3 x 239 [40,101]
+ CRUSH rule 3 x 240 [63,60]
+ CRUSH rule 3 x 241 [47,12]
+ CRUSH rule 3 x 242 [73,74]
+ CRUSH rule 3 x 243 [76,8]
+ CRUSH rule 3 x 244 [103,50]
+ CRUSH rule 3 x 245 [106,95]
+ CRUSH rule 3 x 246 [35,82]
+ CRUSH rule 3 x 247 [116,101]
+ CRUSH rule 3 x 248 [8,119]
+ CRUSH rule 3 x 249 [2,17]
+ CRUSH rule 3 x 250 [34,89]
+ CRUSH rule 3 x 251 [28,69]
+ CRUSH rule 3 x 252 [95,80]
+ CRUSH rule 3 x 253 [109,3]
+ CRUSH rule 3 x 254 [99,80]
+ CRUSH rule 3 x 255 [112,85]
+ CRUSH rule 3 x 256 [94,63]
+ CRUSH rule 3 x 257 [100,87]
+ CRUSH rule 3 x 258 [34,63]
+ CRUSH rule 3 x 259 [70,107]
+ CRUSH rule 3 x 260 [89,115]
+ CRUSH rule 3 x 261 [94,83]
+ CRUSH rule 3 x 262 [42,45]
+ CRUSH rule 3 x 263 [113,101]
+ CRUSH rule 3 x 264 [36,81]
+ CRUSH rule 3 x 265 [14,88]
+ CRUSH rule 3 x 266 [75,96]
+ CRUSH rule 3 x 267 [6,5]
+ CRUSH rule 3 x 268 [38,47]
+ CRUSH rule 3 x 269 [86,59]
+ CRUSH rule 3 x 270 [87,70]
+ CRUSH rule 3 x 271 [19,108]
+ CRUSH rule 3 x 272 [73,5]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,64]
+ CRUSH rule 3 x 275 [29,34]
+ CRUSH rule 3 x 276 [7,100]
+ CRUSH rule 3 x 277 [74,6]
+ CRUSH rule 3 x 278 [107,115]
+ CRUSH rule 3 x 279 [112,20]
+ CRUSH rule 3 x 280 [113,15]
+ CRUSH rule 3 x 281 [89,56]
+ CRUSH rule 3 x 282 [20,38]
+ CRUSH rule 3 x 283 [8,114]
+ CRUSH rule 3 x 284 [66,75]
+ CRUSH rule 3 x 285 [99,94]
+ CRUSH rule 3 x 286 [78,6]
+ CRUSH rule 3 x 287 [12,27]
+ CRUSH rule 3 x 288 [24,22]
+ CRUSH rule 3 x 289 [105,64]
+ CRUSH rule 3 x 290 [25,46]
+ CRUSH rule 3 x 291 [35,116]
+ CRUSH rule 3 x 292 [20,109]
+ CRUSH rule 3 x 293 [27,92]
+ CRUSH rule 3 x 294 [60,93]
+ CRUSH rule 3 x 295 [37,2]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,55]
+ CRUSH rule 3 x 298 [70,53]
+ CRUSH rule 3 x 299 [116,10]
+ CRUSH rule 3 x 300 [67,26]
+ CRUSH rule 3 x 301 [117,23]
+ CRUSH rule 3 x 302 [78,67]
+ CRUSH rule 3 x 303 [19,5]
+ CRUSH rule 3 x 304 [101,50]
+ CRUSH rule 3 x 305 [5,59]
+ CRUSH rule 3 x 306 [41,18]
+ CRUSH rule 3 x 307 [65,5]
+ CRUSH rule 3 x 308 [91,2]
+ CRUSH rule 3 x 309 [38,53]
+ CRUSH rule 3 x 310 [26,15]
+ CRUSH rule 3 x 311 [36,95]
+ CRUSH rule 3 x 312 [114,61]
+ CRUSH rule 3 x 313 [104,65]
+ CRUSH rule 3 x 314 [28,6]
+ CRUSH rule 3 x 315 [118,31]
+ CRUSH rule 3 x 316 [98,95]
+ CRUSH rule 3 x 317 [118,13]
+ CRUSH rule 3 x 318 [17,30]
+ CRUSH rule 3 x 319 [53,1]
+ CRUSH rule 3 x 320 [36,41]
+ CRUSH rule 3 x 321 [33,5]
+ CRUSH rule 3 x 322 [68,10]
+ CRUSH rule 3 x 323 [66,29]
+ CRUSH rule 3 x 324 [21,72]
+ CRUSH rule 3 x 325 [52,16]
+ CRUSH rule 3 x 326 [7,109]
+ CRUSH rule 3 x 327 [62,17]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,100]
+ CRUSH rule 3 x 330 [24,11]
+ CRUSH rule 3 x 331 [84,95]
+ CRUSH rule 3 x 332 [61,111]
+ CRUSH rule 3 x 333 [116,25]
+ CRUSH rule 3 x 334 [94,103]
+ CRUSH rule 3 x 335 [71,118]
+ CRUSH rule 3 x 336 [24,99]
+ CRUSH rule 3 x 337 [18,83]
+ CRUSH rule 3 x 338 [43,28]
+ CRUSH rule 3 x 339 [13,64]
+ CRUSH rule 3 x 340 [81,111]
+ CRUSH rule 3 x 341 [46,105]
+ CRUSH rule 3 x 342 [92,23]
+ CRUSH rule 3 x 343 [49,112]
+ CRUSH rule 3 x 344 [1,87]
+ CRUSH rule 3 x 345 [56,35]
+ CRUSH rule 3 x 346 [3,54]
+ CRUSH rule 3 x 347 [106,27]
+ CRUSH rule 3 x 348 [10,117]
+ CRUSH rule 3 x 349 [96,87]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,41]
+ CRUSH rule 3 x 352 [36,13]
+ CRUSH rule 3 x 353 [10,82]
+ CRUSH rule 3 x 354 [55,52]
+ CRUSH rule 3 x 355 [73,94]
+ CRUSH rule 3 x 356 [75,66]
+ CRUSH rule 3 x 357 [70,93]
+ CRUSH rule 3 x 358 [97,56]
+ CRUSH rule 3 x 359 [110,105]
+ CRUSH rule 3 x 360 [106,57]
+ CRUSH rule 3 x 361 [27,42]
+ CRUSH rule 3 x 362 [28,55]
+ CRUSH rule 3 x 363 [68,20]
+ CRUSH rule 3 x 364 [23,50]
+ CRUSH rule 3 x 365 [57,76]
+ CRUSH rule 3 x 366 [42,75]
+ CRUSH rule 3 x 367 [103,82]
+ CRUSH rule 3 x 368 [103,104]
+ CRUSH rule 3 x 369 [12,57]
+ CRUSH rule 3 x 370 [11,26]
+ CRUSH rule 3 x 371 [34,55]
+ CRUSH rule 3 x 372 [58,14]
+ CRUSH rule 3 x 373 [6,42]
+ CRUSH rule 3 x 374 [110,95]
+ CRUSH rule 3 x 375 [5,43]
+ CRUSH rule 3 x 376 [91,86]
+ CRUSH rule 3 x 377 [93,116]
+ CRUSH rule 3 x 378 [68,6]
+ CRUSH rule 3 x 379 [77,44]
+ CRUSH rule 3 x 380 [76,83]
+ CRUSH rule 3 x 381 [36,27]
+ CRUSH rule 3 x 382 [26,77]
+ CRUSH rule 3 x 383 [76,99]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,93]
+ CRUSH rule 3 x 386 [83,92]
+ CRUSH rule 3 x 387 [16,26]
+ CRUSH rule 3 x 388 [29,113]
+ CRUSH rule 3 x 389 [92,29]
+ CRUSH rule 3 x 390 [68,77]
+ CRUSH rule 3 x 391 [15,88]
+ CRUSH rule 3 x 392 [21,32]
+ CRUSH rule 3 x 393 [91,18]
+ CRUSH rule 3 x 394 [38,73]
+ CRUSH rule 3 x 395 [21,119]
+ CRUSH rule 3 x 396 [12,13]
+ CRUSH rule 3 x 397 [40,63]
+ CRUSH rule 3 x 398 [44,3]
+ CRUSH rule 3 x 399 [5,95]
+ CRUSH rule 3 x 400 [19,102]
+ CRUSH rule 3 x 401 [79,52]
+ CRUSH rule 3 x 402 [107,98]
+ CRUSH rule 3 x 403 [23,82]
+ CRUSH rule 3 x 404 [87,68]
+ CRUSH rule 3 x 405 [90,97]
+ CRUSH rule 3 x 406 [15,117]
+ CRUSH rule 3 x 407 [70,35]
+ CRUSH rule 3 x 408 [55,72]
+ CRUSH rule 3 x 409 [73,62]
+ CRUSH rule 3 x 410 [70,73]
+ CRUSH rule 3 x 411 [34,25]
+ CRUSH rule 3 x 412 [105,117]
+ CRUSH rule 3 x 413 [41,110]
+ CRUSH rule 3 x 414 [70,65]
+ CRUSH rule 3 x 415 [107,5]
+ CRUSH rule 3 x 416 [2,22]
+ CRUSH rule 3 x 417 [26,14]
+ CRUSH rule 3 x 418 [51,46]
+ CRUSH rule 3 x 419 [8,82]
+ CRUSH rule 3 x 420 [109,105]
+ CRUSH rule 3 x 421 [114,75]
+ CRUSH rule 3 x 422 [109,87]
+ CRUSH rule 3 x 423 [59,24]
+ CRUSH rule 3 x 424 [92,51]
+ CRUSH rule 3 x 425 [101,111]
+ CRUSH rule 3 x 426 [36,6]
+ CRUSH rule 3 x 427 [8,24]
+ CRUSH rule 3 x 428 [68,35]
+ CRUSH rule 3 x 429 [76,75]
+ CRUSH rule 3 x 430 [67,117]
+ CRUSH rule 3 x 431 [70,25]
+ CRUSH rule 3 x 432 [7,34]
+ CRUSH rule 3 x 433 [49,84]
+ CRUSH rule 3 x 434 [64,31]
+ CRUSH rule 3 x 435 [110,13]
+ CRUSH rule 3 x 436 [106,89]
+ CRUSH rule 3 x 437 [26,65]
+ CRUSH rule 3 x 438 [118,63]
+ CRUSH rule 3 x 439 [40,21]
+ CRUSH rule 3 x 440 [45,119]
+ CRUSH rule 3 x 441 [112,105]
+ CRUSH rule 3 x 442 [55,113]
+ CRUSH rule 3 x 443 [44,33]
+ CRUSH rule 3 x 444 [71,38]
+ CRUSH rule 3 x 445 [58,81]
+ CRUSH rule 3 x 446 [40,10]
+ CRUSH rule 3 x 447 [100,61]
+ CRUSH rule 3 x 448 [111,73]
+ CRUSH rule 3 x 449 [67,66]
+ CRUSH rule 3 x 450 [117,61]
+ CRUSH rule 3 x 451 [66,81]
+ CRUSH rule 3 x 452 [70,8]
+ CRUSH rule 3 x 453 [82,85]
+ CRUSH rule 3 x 454 [53,18]
+ CRUSH rule 3 x 455 [91,42]
+ CRUSH rule 3 x 456 [101,46]
+ CRUSH rule 3 x 457 [113,51]
+ CRUSH rule 3 x 458 [119,25]
+ CRUSH rule 3 x 459 [50,67]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,51]
+ CRUSH rule 3 x 462 [98,107]
+ CRUSH rule 3 x 463 [108,105]
+ CRUSH rule 3 x 464 [19,109]
+ CRUSH rule 3 x 465 [62,23]
+ CRUSH rule 3 x 466 [53,12]
+ CRUSH rule 3 x 467 [40,57]
+ CRUSH rule 3 x 468 [97,44]
+ CRUSH rule 3 x 469 [98,75]
+ CRUSH rule 3 x 470 [50,29]
+ CRUSH rule 3 x 471 [40,13]
+ CRUSH rule 3 x 472 [27,18]
+ CRUSH rule 3 x 473 [48,35]
+ CRUSH rule 3 x 474 [51,32]
+ CRUSH rule 3 x 475 [49,117]
+ CRUSH rule 3 x 476 [110,31]
+ CRUSH rule 3 x 477 [80,97]
+ CRUSH rule 3 x 478 [78,99]
+ CRUSH rule 3 x 479 [31,66]
+ CRUSH rule 3 x 480 [75,88]
+ CRUSH rule 3 x 481 [26,20]
+ CRUSH rule 3 x 482 [84,53]
+ CRUSH rule 3 x 483 [15,116]
+ CRUSH rule 3 x 484 [37,114]
+ CRUSH rule 3 x 485 [84,8]
+ CRUSH rule 3 x 486 [92,10]
+ CRUSH rule 3 x 487 [106,17]
+ CRUSH rule 3 x 488 [42,20]
+ CRUSH rule 3 x 489 [89,2]
+ CRUSH rule 3 x 490 [22,114]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,66]
+ CRUSH rule 3 x 493 [94,14]
+ CRUSH rule 3 x 494 [59,86]
+ CRUSH rule 3 x 495 [95,58]
+ CRUSH rule 3 x 496 [46,41]
+ CRUSH rule 3 x 497 [102,27]
+ CRUSH rule 3 x 498 [21,116]
+ CRUSH rule 3 x 499 [5,49]
+ CRUSH rule 3 x 500 [50,49]
+ CRUSH rule 3 x 501 [60,3]
+ CRUSH rule 3 x 502 [65,110]
+ CRUSH rule 3 x 503 [21,112]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,93]
+ CRUSH rule 3 x 506 [79,64]
+ CRUSH rule 3 x 507 [34,107]
+ CRUSH rule 3 x 508 [45,114]
+ CRUSH rule 3 x 509 [19,88]
+ CRUSH rule 3 x 510 [117,45]
+ CRUSH rule 3 x 511 [14,104]
+ CRUSH rule 3 x 512 [59,26]
+ CRUSH rule 3 x 513 [102,93]
+ CRUSH rule 3 x 514 [75,72]
+ CRUSH rule 3 x 515 [84,41]
+ CRUSH rule 3 x 516 [37,30]
+ CRUSH rule 3 x 517 [83,115]
+ CRUSH rule 3 x 518 [18,83]
+ CRUSH rule 3 x 519 [67,88]
+ CRUSH rule 3 x 520 [15,114]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,51]
+ CRUSH rule 3 x 523 [68,101]
+ CRUSH rule 3 x 524 [33,38]
+ CRUSH rule 3 x 525 [63,115]
+ CRUSH rule 3 x 526 [83,50]
+ CRUSH rule 3 x 527 [37,56]
+ CRUSH rule 3 x 528 [108,81]
+ CRUSH rule 3 x 529 [74,33]
+ CRUSH rule 3 x 530 [49,92]
+ CRUSH rule 3 x 531 [117,105]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,85]
+ CRUSH rule 3 x 534 [97,24]
+ CRUSH rule 3 x 535 [48,75]
+ CRUSH rule 3 x 536 [113,101]
+ CRUSH rule 3 x 537 [116,47]
+ CRUSH rule 3 x 538 [85,74]
+ CRUSH rule 3 x 539 [72,43]
+ CRUSH rule 3 x 540 [39,34]
+ CRUSH rule 3 x 541 [53,84]
+ CRUSH rule 3 x 542 [27,32]
+ CRUSH rule 3 x 543 [45,113]
+ CRUSH rule 3 x 544 [59,42]
+ CRUSH rule 3 x 545 [118,95]
+ CRUSH rule 3 x 546 [18,79]
+ CRUSH rule 3 x 547 [67,30]
+ CRUSH rule 3 x 548 [53,100]
+ CRUSH rule 3 x 549 [60,45]
+ CRUSH rule 3 x 550 [92,101]
+ CRUSH rule 3 x 551 [77,88]
+ CRUSH rule 3 x 552 [61,94]
+ CRUSH rule 3 x 553 [71,78]
+ CRUSH rule 3 x 554 [61,115]
+ CRUSH rule 3 x 555 [76,77]
+ CRUSH rule 3 x 556 [106,55]
+ CRUSH rule 3 x 557 [26,22]
+ CRUSH rule 3 x 558 [41,84]
+ CRUSH rule 3 x 559 [65,24]
+ CRUSH rule 3 x 560 [94,16]
+ CRUSH rule 3 x 561 [27,5]
+ CRUSH rule 3 x 562 [78,59]
+ CRUSH rule 3 x 563 [59,70]
+ CRUSH rule 3 x 564 [96,8]
+ CRUSH rule 3 x 565 [8,48]
+ CRUSH rule 3 x 566 [119,17]
+ CRUSH rule 3 x 567 [7,38]
+ CRUSH rule 3 x 568 [57,94]
+ CRUSH rule 3 x 569 [65,26]
+ CRUSH rule 3 x 570 [98,27]
+ CRUSH rule 3 x 571 [95,30]
+ CRUSH rule 3 x 572 [62,83]
+ CRUSH rule 3 x 573 [1,79]
+ CRUSH rule 3 x 574 [89,42]
+ CRUSH rule 3 x 575 [87,113]
+ CRUSH rule 3 x 576 [21,68]
+ CRUSH rule 3 x 577 [8,84]
+ CRUSH rule 3 x 578 [75,115]
+ CRUSH rule 3 x 579 [105,68]
+ CRUSH rule 3 x 580 [51,28]
+ CRUSH rule 3 x 581 [55,113]
+ CRUSH rule 3 x 582 [27,113]
+ CRUSH rule 3 x 583 [6,78]
+ CRUSH rule 3 x 584 [10,30]
+ CRUSH rule 3 x 585 [20,111]
+ CRUSH rule 3 x 586 [48,27]
+ CRUSH rule 3 x 587 [29,94]
+ CRUSH rule 3 x 588 [103,90]
+ CRUSH rule 3 x 589 [88,95]
+ CRUSH rule 3 x 590 [76,101]
+ CRUSH rule 3 x 591 [42,43]
+ CRUSH rule 3 x 592 [78,51]
+ CRUSH rule 3 x 593 [82,71]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,59]
+ CRUSH rule 3 x 597 [16,36]
+ CRUSH rule 3 x 598 [37,56]
+ CRUSH rule 3 x 599 [10,84]
+ CRUSH rule 3 x 600 [24,69]
+ CRUSH rule 3 x 601 [104,14]
+ CRUSH rule 3 x 602 [48,45]
+ CRUSH rule 3 x 603 [93,32]
+ CRUSH rule 3 x 604 [118,79]
+ CRUSH rule 3 x 605 [104,53]
+ CRUSH rule 3 x 606 [90,83]
+ CRUSH rule 3 x 607 [95,110]
+ CRUSH rule 3 x 608 [112,101]
+ CRUSH rule 3 x 609 [34,99]
+ CRUSH rule 3 x 610 [106,16]
+ CRUSH rule 3 x 611 [66,87]
+ CRUSH rule 3 x 612 [2,81]
+ CRUSH rule 3 x 613 [13,86]
+ CRUSH rule 3 x 614 [50,3]
+ CRUSH rule 3 x 615 [24,73]
+ CRUSH rule 3 x 616 [41,119]
+ CRUSH rule 3 x 617 [81,106]
+ CRUSH rule 3 x 618 [3,104]
+ CRUSH rule 3 x 619 [92,7]
+ CRUSH rule 3 x 620 [108,11]
+ CRUSH rule 3 x 621 [105,115]
+ CRUSH rule 3 x 622 [67,48]
+ CRUSH rule 3 x 623 [69,74]
+ CRUSH rule 3 x 624 [115,49]
+ CRUSH rule 3 x 625 [73,109]
+ CRUSH rule 3 x 626 [52,3]
+ CRUSH rule 3 x 627 [116,3]
+ CRUSH rule 3 x 628 [98,91]
+ CRUSH rule 3 x 629 [6,112]
+ CRUSH rule 3 x 630 [22,72]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,71]
+ CRUSH rule 3 x 633 [65,12]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,46]
+ CRUSH rule 3 x 636 [23,70]
+ CRUSH rule 3 x 637 [99,24]
+ CRUSH rule 3 x 638 [43,114]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,73]
+ CRUSH rule 3 x 641 [45,84]
+ CRUSH rule 3 x 642 [47,66]
+ CRUSH rule 3 x 643 [64,8]
+ CRUSH rule 3 x 644 [31,82]
+ CRUSH rule 3 x 645 [77,64]
+ CRUSH rule 3 x 646 [37,86]
+ CRUSH rule 3 x 647 [65,56]
+ CRUSH rule 3 x 648 [84,13]
+ CRUSH rule 3 x 649 [88,55]
+ CRUSH rule 3 x 650 [21,76]
+ CRUSH rule 3 x 651 [63,116]
+ CRUSH rule 3 x 652 [57,112]
+ CRUSH rule 3 x 653 [38,61]
+ CRUSH rule 3 x 654 [104,67]
+ CRUSH rule 3 x 655 [89,54]
+ CRUSH rule 3 x 656 [84,49]
+ CRUSH rule 3 x 657 [47,32]
+ CRUSH rule 3 x 658 [80,29]
+ CRUSH rule 3 x 659 [11,112]
+ CRUSH rule 3 x 660 [65,111]
+ CRUSH rule 3 x 661 [96,73]
+ CRUSH rule 3 x 662 [111,73]
+ CRUSH rule 3 x 663 [83,60]
+ CRUSH rule 3 x 664 [59,80]
+ CRUSH rule 3 x 665 [31,117]
+ CRUSH rule 3 x 666 [112,101]
+ CRUSH rule 3 x 667 [70,47]
+ CRUSH rule 3 x 668 [96,57]
+ CRUSH rule 3 x 669 [56,39]
+ CRUSH rule 3 x 670 [98,105]
+ CRUSH rule 3 x 671 [57,48]
+ CRUSH rule 3 x 672 [37,36]
+ CRUSH rule 3 x 673 [83,2]
+ CRUSH rule 3 x 674 [36,25]
+ CRUSH rule 3 x 675 [88,14]
+ CRUSH rule 3 x 676 [3,110]
+ CRUSH rule 3 x 677 [88,67]
+ CRUSH rule 3 x 678 [27,44]
+ CRUSH rule 3 x 679 [33,116]
+ CRUSH rule 3 x 680 [111,39]
+ CRUSH rule 3 x 681 [53,12]
+ CRUSH rule 3 x 682 [12,87]
+ CRUSH rule 3 x 683 [24,85]
+ CRUSH rule 3 x 684 [98,65]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,72]
+ CRUSH rule 3 x 688 [16,114]
+ CRUSH rule 3 x 689 [32,31]
+ CRUSH rule 3 x 690 [96,33]
+ CRUSH rule 3 x 691 [34,6]
+ CRUSH rule 3 x 692 [97,84]
+ CRUSH rule 3 x 693 [29,118]
+ CRUSH rule 3 x 694 [6,30]
+ CRUSH rule 3 x 695 [31,72]
+ CRUSH rule 3 x 696 [104,97]
+ CRUSH rule 3 x 697 [19,96]
+ CRUSH rule 3 x 698 [30,69]
+ CRUSH rule 3 x 699 [47,76]
+ CRUSH rule 3 x 700 [82,55]
+ CRUSH rule 3 x 701 [53,80]
+ CRUSH rule 3 x 702 [95,98]
+ CRUSH rule 3 x 703 [92,65]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,1]
+ CRUSH rule 3 x 706 [74,35]
+ CRUSH rule 3 x 707 [91,115]
+ CRUSH rule 3 x 708 [95,112]
+ CRUSH rule 3 x 709 [73,72]
+ CRUSH rule 3 x 710 [94,47]
+ CRUSH rule 3 x 711 [68,41]
+ CRUSH rule 3 x 712 [107,18]
+ CRUSH rule 3 x 713 [29,109]
+ CRUSH rule 3 x 714 [86,61]
+ CRUSH rule 3 x 715 [74,13]
+ CRUSH rule 3 x 716 [101,56]
+ CRUSH rule 3 x 717 [12,29]
+ CRUSH rule 3 x 718 [83,24]
+ CRUSH rule 3 x 719 [26,10]
+ CRUSH rule 3 x 720 [69,2]
+ CRUSH rule 3 x 721 [51,42]
+ CRUSH rule 3 x 722 [15,74]
+ CRUSH rule 3 x 723 [117,14]
+ CRUSH rule 3 x 724 [45,38]
+ CRUSH rule 3 x 725 [53,110]
+ CRUSH rule 3 x 726 [103,68]
+ CRUSH rule 3 x 727 [89,100]
+ CRUSH rule 3 x 728 [76,16]
+ CRUSH rule 3 x 729 [35,90]
+ CRUSH rule 3 x 730 [28,103]
+ CRUSH rule 3 x 731 [78,41]
+ CRUSH rule 3 x 732 [1,27]
+ CRUSH rule 3 x 733 [35,100]
+ CRUSH rule 3 x 734 [119,85]
+ CRUSH rule 3 x 735 [102,43]
+ CRUSH rule 3 x 736 [37,92]
+ CRUSH rule 3 x 737 [117,11]
+ CRUSH rule 3 x 738 [57,32]
+ CRUSH rule 3 x 739 [87,1]
+ CRUSH rule 3 x 740 [29,80]
+ CRUSH rule 3 x 741 [47,111]
+ CRUSH rule 3 x 742 [106,83]
+ CRUSH rule 3 x 743 [105,94]
+ CRUSH rule 3 x 744 [23,64]
+ CRUSH rule 3 x 745 [37,112]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,95]
+ CRUSH rule 3 x 748 [48,14]
+ CRUSH rule 3 x 749 [102,101]
+ CRUSH rule 3 x 750 [83,78]
+ CRUSH rule 3 x 751 [25,104]
+ CRUSH rule 3 x 752 [82,95]
+ CRUSH rule 3 x 753 [14,113]
+ CRUSH rule 3 x 754 [114,51]
+ CRUSH rule 3 x 755 [87,26]
+ CRUSH rule 3 x 756 [113,87]
+ CRUSH rule 3 x 757 [47,66]
+ CRUSH rule 3 x 758 [54,63]
+ CRUSH rule 3 x 759 [74,20]
+ CRUSH rule 3 x 760 [88,22]
+ CRUSH rule 3 x 761 [73,86]
+ CRUSH rule 3 x 762 [34,17]
+ CRUSH rule 3 x 763 [13,78]
+ CRUSH rule 3 x 764 [89,42]
+ CRUSH rule 3 x 765 [109,91]
+ CRUSH rule 3 x 766 [19,66]
+ CRUSH rule 3 x 767 [41,26]
+ CRUSH rule 3 x 768 [106,57]
+ CRUSH rule 3 x 769 [91,104]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,35]
+ CRUSH rule 3 x 772 [97,108]
+ CRUSH rule 3 x 773 [116,47]
+ CRUSH rule 3 x 774 [100,31]
+ CRUSH rule 3 x 775 [102,43]
+ CRUSH rule 3 x 776 [69,38]
+ CRUSH rule 3 x 777 [91,52]
+ CRUSH rule 3 x 778 [83,119]
+ CRUSH rule 3 x 779 [47,60]
+ CRUSH rule 3 x 780 [63,70]
+ CRUSH rule 3 x 781 [105,2]
+ CRUSH rule 3 x 782 [117,59]
+ CRUSH rule 3 x 783 [19,109]
+ CRUSH rule 3 x 784 [63,114]
+ CRUSH rule 3 x 785 [27,84]
+ CRUSH rule 3 x 786 [41,110]
+ CRUSH rule 3 x 787 [108,73]
+ CRUSH rule 3 x 788 [74,103]
+ CRUSH rule 3 x 789 [50,17]
+ CRUSH rule 3 x 790 [20,106]
+ CRUSH rule 3 x 791 [96,87]
+ CRUSH rule 3 x 792 [80,97]
+ CRUSH rule 3 x 793 [6,26]
+ CRUSH rule 3 x 794 [14,42]
+ CRUSH rule 3 x 795 [30,8]
+ CRUSH rule 3 x 796 [87,36]
+ CRUSH rule 3 x 797 [64,61]
+ CRUSH rule 3 x 798 [42,69]
+ CRUSH rule 3 x 799 [19,117]
+ CRUSH rule 3 x 800 [106,8]
+ CRUSH rule 3 x 801 [2,57]
+ CRUSH rule 3 x 802 [63,68]
+ CRUSH rule 3 x 803 [46,35]
+ CRUSH rule 3 x 804 [33,26]
+ CRUSH rule 3 x 805 [96,49]
+ CRUSH rule 3 x 806 [48,25]
+ CRUSH rule 3 x 807 [48,83]
+ CRUSH rule 3 x 808 [76,31]
+ CRUSH rule 3 x 809 [27,48]
+ CRUSH rule 3 x 810 [119,71]
+ CRUSH rule 3 x 811 [111,91]
+ CRUSH rule 3 x 812 [25,111]
+ CRUSH rule 3 x 813 [81,28]
+ CRUSH rule 3 x 814 [95,42]
+ CRUSH rule 3 x 815 [84,61]
+ CRUSH rule 3 x 816 [64,35]
+ CRUSH rule 3 x 817 [63,60]
+ CRUSH rule 3 x 818 [69,46]
+ CRUSH rule 3 x 819 [88,75]
+ CRUSH rule 3 x 820 [104,57]
+ CRUSH rule 3 x 821 [58,21]
+ CRUSH rule 3 x 822 [20,80]
+ CRUSH rule 3 x 823 [63,118]
+ CRUSH rule 3 x 824 [102,13]
+ CRUSH rule 3 x 825 [47,118]
+ CRUSH rule 3 x 826 [44,7]
+ CRUSH rule 3 x 827 [101,88]
+ CRUSH rule 3 x 828 [60,41]
+ CRUSH rule 3 x 829 [45,102]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,75]
+ CRUSH rule 3 x 833 [57,32]
+ CRUSH rule 3 x 834 [90,33]
+ CRUSH rule 3 x 835 [6,1]
+ CRUSH rule 3 x 836 [63,68]
+ CRUSH rule 3 x 837 [76,71]
+ CRUSH rule 3 x 838 [106,20]
+ CRUSH rule 3 x 839 [87,96]
+ CRUSH rule 3 x 840 [33,32]
+ CRUSH rule 3 x 841 [110,55]
+ CRUSH rule 3 x 842 [66,87]
+ CRUSH rule 3 x 843 [11,80]
+ CRUSH rule 3 x 844 [74,103]
+ CRUSH rule 3 x 845 [74,43]
+ CRUSH rule 3 x 846 [43,76]
+ CRUSH rule 3 x 847 [62,20]
+ CRUSH rule 3 x 848 [92,17]
+ CRUSH rule 3 x 849 [93,36]
+ CRUSH rule 3 x 850 [83,82]
+ CRUSH rule 3 x 851 [65,94]
+ CRUSH rule 3 x 852 [60,22]
+ CRUSH rule 3 x 853 [88,29]
+ CRUSH rule 3 x 854 [83,54]
+ CRUSH rule 3 x 855 [2,101]
+ CRUSH rule 3 x 856 [40,41]
+ CRUSH rule 3 x 857 [69,82]
+ CRUSH rule 3 x 858 [98,81]
+ CRUSH rule 3 x 859 [56,43]
+ CRUSH rule 3 x 860 [11,26]
+ CRUSH rule 3 x 861 [22,110]
+ CRUSH rule 3 x 862 [22,70]
+ CRUSH rule 3 x 863 [79,84]
+ CRUSH rule 3 x 864 [77,24]
+ CRUSH rule 3 x 865 [119,17]
+ CRUSH rule 3 x 866 [18,49]
+ CRUSH rule 3 x 867 [3,84]
+ CRUSH rule 3 x 868 [100,107]
+ CRUSH rule 3 x 869 [22,104]
+ CRUSH rule 3 x 870 [73,30]
+ CRUSH rule 3 x 871 [84,105]
+ CRUSH rule 3 x 872 [72,75]
+ CRUSH rule 3 x 873 [81,96]
+ CRUSH rule 3 x 874 [21,72]
+ CRUSH rule 3 x 875 [115,59]
+ CRUSH rule 3 x 876 [98,49]
+ CRUSH rule 3 x 877 [80,79]
+ CRUSH rule 3 x 878 [87,94]
+ CRUSH rule 3 x 879 [29,18]
+ CRUSH rule 3 x 880 [23,40]
+ CRUSH rule 3 x 881 [109,69]
+ CRUSH rule 3 x 882 [31,118]
+ CRUSH rule 3 x 883 [102,8]
+ CRUSH rule 3 x 884 [80,19]
+ CRUSH rule 3 x 885 [46,101]
+ CRUSH rule 3 x 886 [2,65]
+ CRUSH rule 3 x 887 [5,99]
+ CRUSH rule 3 x 888 [16,70]
+ CRUSH rule 3 x 889 [84,93]
+ CRUSH rule 3 x 890 [65,118]
+ CRUSH rule 3 x 891 [86,105]
+ CRUSH rule 3 x 892 [64,10]
+ CRUSH rule 3 x 893 [20,110]
+ CRUSH rule 3 x 894 [32,47]
+ CRUSH rule 3 x 895 [40,21]
+ CRUSH rule 3 x 896 [113,14]
+ CRUSH rule 3 x 897 [107,80]
+ CRUSH rule 3 x 898 [76,71]
+ CRUSH rule 3 x 899 [75,82]
+ CRUSH rule 3 x 900 [83,82]
+ CRUSH rule 3 x 901 [66,61]
+ CRUSH rule 3 x 902 [25,56]
+ CRUSH rule 3 x 903 [53,46]
+ CRUSH rule 3 x 904 [50,101]
+ CRUSH rule 3 x 905 [99,110]
+ CRUSH rule 3 x 906 [68,27]
+ CRUSH rule 3 x 907 [109,47]
+ CRUSH rule 3 x 908 [47,1]
+ CRUSH rule 3 x 909 [73,2]
+ CRUSH rule 3 x 910 [71,74]
+ CRUSH rule 3 x 911 [39,42]
+ CRUSH rule 3 x 912 [90,7]
+ CRUSH rule 3 x 913 [29,96]
+ CRUSH rule 3 x 914 [84,45]
+ CRUSH rule 3 x 915 [49,115]
+ CRUSH rule 3 x 916 [32,77]
+ CRUSH rule 3 x 917 [46,23]
+ CRUSH rule 3 x 918 [82,73]
+ CRUSH rule 3 x 919 [13,28]
+ CRUSH rule 3 x 920 [25,26]
+ CRUSH rule 3 x 921 [55,119]
+ CRUSH rule 3 x 922 [33,32]
+ CRUSH rule 3 x 923 [28,15]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,22]
+ CRUSH rule 3 x 926 [64,69]
+ CRUSH rule 3 x 927 [32,16]
+ CRUSH rule 3 x 928 [13,113]
+ CRUSH rule 3 x 929 [85,115]
+ CRUSH rule 3 x 930 [104,20]
+ CRUSH rule 3 x 931 [46,79]
+ CRUSH rule 3 x 932 [43,52]
+ CRUSH rule 3 x 933 [18,55]
+ CRUSH rule 3 x 934 [68,81]
+ CRUSH rule 3 x 935 [28,57]
+ CRUSH rule 3 x 936 [104,57]
+ CRUSH rule 3 x 937 [110,10]
+ CRUSH rule 3 x 938 [48,23]
+ CRUSH rule 3 x 939 [77,42]
+ CRUSH rule 3 x 940 [76,49]
+ CRUSH rule 3 x 941 [66,101]
+ CRUSH rule 3 x 942 [80,87]
+ CRUSH rule 3 x 943 [75,74]
+ CRUSH rule 3 x 944 [82,53]
+ CRUSH rule 3 x 945 [71,74]
+ CRUSH rule 3 x 946 [37,42]
+ CRUSH rule 3 x 947 [107,30]
+ CRUSH rule 3 x 948 [108,95]
+ CRUSH rule 3 x 949 [46,103]
+ CRUSH rule 3 x 950 [96,101]
+ CRUSH rule 3 x 951 [40,14]
+ CRUSH rule 3 x 952 [114,21]
+ CRUSH rule 3 x 953 [62,23]
+ CRUSH rule 3 x 954 [103,5]
+ CRUSH rule 3 x 955 [42,73]
+ CRUSH rule 3 x 956 [72,103]
+ CRUSH rule 3 x 957 [117,22]
+ CRUSH rule 3 x 958 [23,106]
+ CRUSH rule 3 x 959 [42,93]
+ CRUSH rule 3 x 960 [113,8]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,51]
+ CRUSH rule 3 x 963 [101,106]
+ CRUSH rule 3 x 964 [66,89]
+ CRUSH rule 3 x 965 [47,102]
+ CRUSH rule 3 x 966 [88,63]
+ CRUSH rule 3 x 967 [71,46]
+ CRUSH rule 3 x 968 [74,51]
+ CRUSH rule 3 x 969 [53,78]
+ CRUSH rule 3 x 970 [3,30]
+ CRUSH rule 3 x 971 [66,107]
+ CRUSH rule 3 x 972 [3,66]
+ CRUSH rule 3 x 973 [113,20]
+ CRUSH rule 3 x 974 [114,35]
+ CRUSH rule 3 x 975 [83,58]
+ CRUSH rule 3 x 976 [81,48]
+ CRUSH rule 3 x 977 [95,102]
+ CRUSH rule 3 x 978 [119,41]
+ CRUSH rule 3 x 979 [98,6]
+ CRUSH rule 3 x 980 [39,108]
+ CRUSH rule 3 x 981 [89,84]
+ CRUSH rule 3 x 982 [19,94]
+ CRUSH rule 3 x 983 [34,45]
+ CRUSH rule 3 x 984 [78,63]
+ CRUSH rule 3 x 985 [99,52]
+ CRUSH rule 3 x 986 [44,99]
+ CRUSH rule 3 x 987 [25,32]
+ CRUSH rule 3 x 988 [79,2]
+ CRUSH rule 3 x 989 [87,26]
+ CRUSH rule 3 x 990 [72,69]
+ CRUSH rule 3 x 991 [90,8]
+ CRUSH rule 3 x 992 [30,67]
+ CRUSH rule 3 x 993 [74,49]
+ CRUSH rule 3 x 994 [74,105]
+ CRUSH rule 3 x 995 [100,97]
+ CRUSH rule 3 x 996 [41,58]
+ CRUSH rule 3 x 997 [89,76]
+ CRUSH rule 3 x 998 [92,47]
+ CRUSH rule 3 x 999 [117,16]
+ CRUSH rule 3 x 1000 [50,47]
+ CRUSH rule 3 x 1001 [83,102]
+ CRUSH rule 3 x 1002 [94,37]
+ CRUSH rule 3 x 1003 [43,88]
+ CRUSH rule 3 x 1004 [89,54]
+ CRUSH rule 3 x 1005 [105,84]
+ CRUSH rule 3 x 1006 [45,111]
+ CRUSH rule 3 x 1007 [19,66]
+ CRUSH rule 3 x 1008 [31,76]
+ CRUSH rule 3 x 1009 [1,95]
+ CRUSH rule 3 x 1010 [31,113]
+ CRUSH rule 3 x 1011 [64,81]
+ CRUSH rule 3 x 1012 [68,49]
+ CRUSH rule 3 x 1013 [5,93]
+ CRUSH rule 3 x 1014 [33,66]
+ CRUSH rule 3 x 1015 [106,45]
+ CRUSH rule 3 x 1016 [107,86]
+ CRUSH rule 3 x 1017 [12,61]
+ CRUSH rule 3 x 1018 [61,26]
+ CRUSH rule 3 x 1019 [27,104]
+ CRUSH rule 3 x 1020 [31,86]
+ CRUSH rule 3 x 1021 [22,26]
+ CRUSH rule 3 x 1022 [73,34]
+ CRUSH rule 3 x 1023 [88,79]
+ rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,6]
+ CRUSH rule 3 x 1 [73,52]
+ CRUSH rule 3 x 2 [91,48]
+ CRUSH rule 3 x 3 [51,48]
+ CRUSH rule 3 x 4 [45,114]
+ CRUSH rule 3 x 5 [89,94]
+ CRUSH rule 3 x 6 [91,76]
+ CRUSH rule 3 x 7 [104,73]
+ CRUSH rule 3 x 8 [41,98]
+ CRUSH rule 3 x 9 [46,47]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,40]
+ CRUSH rule 3 x 12 [83,26]
+ CRUSH rule 3 x 13 [27,28]
+ CRUSH rule 3 x 14 [105,64]
+ CRUSH rule 3 x 15 [18,7]
+ CRUSH rule 3 x 16 [103,30]
+ CRUSH rule 3 x 17 [85,118]
+ CRUSH rule 3 x 18 [11,106]
+ CRUSH rule 3 x 19 [75,50]
+ CRUSH rule 3 x 20 [111,67]
+ CRUSH rule 3 x 21 [84,61]
+ CRUSH rule 3 x 22 [23,104]
+ CRUSH rule 3 x 23 [19,86]
+ CRUSH rule 3 x 24 [83,60]
+ CRUSH rule 3 x 25 [81,64]
+ CRUSH rule 3 x 26 [17,38]
+ CRUSH rule 3 x 27 [33,84]
+ CRUSH rule 3 x 28 [45,90]
+ CRUSH rule 3 x 29 [8,109]
+ CRUSH rule 3 x 30 [55,42]
+ CRUSH rule 3 x 31 [76,95]
+ CRUSH rule 3 x 32 [72,11]
+ CRUSH rule 3 x 33 [86,53]
+ CRUSH rule 3 x 34 [7,108]
+ CRUSH rule 3 x 35 [108,13]
+ CRUSH rule 3 x 36 [67,66]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,105]
+ CRUSH rule 3 x 39 [68,103]
+ CRUSH rule 3 x 40 [30,85]
+ CRUSH rule 3 x 41 [52,11]
+ CRUSH rule 3 x 42 [106,75]
+ CRUSH rule 3 x 43 [10,104]
+ CRUSH rule 3 x 44 [101,28]
+ CRUSH rule 3 x 45 [83,64]
+ CRUSH rule 3 x 46 [54,31]
+ CRUSH rule 3 x 47 [106,61]
+ CRUSH rule 3 x 48 [34,41]
+ CRUSH rule 3 x 49 [79,110]
+ CRUSH rule 3 x 50 [42,13]
+ CRUSH rule 3 x 51 [6,94]
+ CRUSH rule 3 x 52 [82,19]
+ CRUSH rule 3 x 53 [32,91]
+ CRUSH rule 3 x 54 [108,8]
+ CRUSH rule 3 x 55 [14,94]
+ CRUSH rule 3 x 56 [21,72]
+ CRUSH rule 3 x 57 [69,88]
+ CRUSH rule 3 x 58 [48,87]
+ CRUSH rule 3 x 59 [21,113]
+ CRUSH rule 3 x 60 [90,73]
+ CRUSH rule 3 x 61 [88,63]
+ CRUSH rule 3 x 62 [100,13]
+ CRUSH rule 3 x 63 [79,5]
+ CRUSH rule 3 x 64 [1,89]
+ CRUSH rule 3 x 65 [32,103]
+ CRUSH rule 3 x 66 [48,79]
+ CRUSH rule 3 x 67 [94,11]
+ CRUSH rule 3 x 68 [102,15]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,11]
+ CRUSH rule 3 x 71 [12,33]
+ CRUSH rule 3 x 72 [26,99]
+ CRUSH rule 3 x 73 [29,114]
+ CRUSH rule 3 x 74 [29,1]
+ CRUSH rule 3 x 75 [60,65]
+ CRUSH rule 3 x 76 [55,62]
+ CRUSH rule 3 x 77 [107,100]
+ CRUSH rule 3 x 78 [86,107]
+ CRUSH rule 3 x 79 [64,16]
+ CRUSH rule 3 x 80 [19,100]
+ CRUSH rule 3 x 81 [64,16]
+ CRUSH rule 3 x 82 [37,40]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,115]
+ CRUSH rule 3 x 85 [87,88]
+ CRUSH rule 3 x 86 [37,68]
+ CRUSH rule 3 x 87 [116,77]
+ CRUSH rule 3 x 88 [38,55]
+ CRUSH rule 3 x 89 [76,25]
+ CRUSH rule 3 x 90 [14,50]
+ CRUSH rule 3 x 91 [68,61]
+ CRUSH rule 3 x 92 [86,95]
+ CRUSH rule 3 x 93 [44,35]
+ CRUSH rule 3 x 94 [46,71]
+ CRUSH rule 3 x 95 [108,53]
+ CRUSH rule 3 x 96 [66,87]
+ CRUSH rule 3 x 97 [111,45]
+ CRUSH rule 3 x 98 [93,110]
+ CRUSH rule 3 x 99 [78,43]
+ CRUSH rule 3 x 100 [28,61]
+ CRUSH rule 3 x 101 [91,110]
+ CRUSH rule 3 x 102 [82,7]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,79]
+ CRUSH rule 3 x 105 [34,87]
+ CRUSH rule 3 x 106 [69,12]
+ CRUSH rule 3 x 107 [1,59]
+ CRUSH rule 3 x 108 [7,109]
+ CRUSH rule 3 x 109 [112,67]
+ CRUSH rule 3 x 110 [54,61]
+ CRUSH rule 3 x 111 [10,92]
+ CRUSH rule 3 x 112 [80,11]
+ CRUSH rule 3 x 113 [69,38]
+ CRUSH rule 3 x 114 [79,38]
+ CRUSH rule 3 x 115 [10,48]
+ CRUSH rule 3 x 116 [37,108]
+ CRUSH rule 3 x 117 [87,56]
+ CRUSH rule 3 x 118 [23,56]
+ CRUSH rule 3 x 119 [104,31]
+ CRUSH rule 3 x 120 [44,93]
+ CRUSH rule 3 x 121 [80,16]
+ CRUSH rule 3 x 122 [45,54]
+ CRUSH rule 3 x 123 [22,112]
+ CRUSH rule 3 x 124 [97,50]
+ CRUSH rule 3 x 125 [66,6]
+ CRUSH rule 3 x 126 [70,39]
+ CRUSH rule 3 x 127 [70,75]
+ CRUSH rule 3 x 128 [11,111]
+ CRUSH rule 3 x 129 [103,46]
+ CRUSH rule 3 x 130 [50,73]
+ CRUSH rule 3 x 131 [44,15]
+ CRUSH rule 3 x 132 [69,58]
+ CRUSH rule 3 x 133 [67,115]
+ CRUSH rule 3 x 134 [37,60]
+ CRUSH rule 3 x 135 [78,61]
+ CRUSH rule 3 x 136 [32,29]
+ CRUSH rule 3 x 137 [92,87]
+ CRUSH rule 3 x 138 [54,8]
+ CRUSH rule 3 x 139 [89,60]
+ CRUSH rule 3 x 140 [39,50]
+ CRUSH rule 3 x 141 [89,62]
+ CRUSH rule 3 x 142 [22,86]
+ CRUSH rule 3 x 143 [96,16]
+ CRUSH rule 3 x 144 [13,1]
+ CRUSH rule 3 x 145 [77,54]
+ CRUSH rule 3 x 146 [12,43]
+ CRUSH rule 3 x 147 [2,59]
+ CRUSH rule 3 x 148 [85,50]
+ CRUSH rule 3 x 149 [103,68]
+ CRUSH rule 3 x 150 [14,50]
+ CRUSH rule 3 x 151 [75,56]
+ CRUSH rule 3 x 152 [49,18]
+ CRUSH rule 3 x 153 [92,79]
+ CRUSH rule 3 x 154 [19,26]
+ CRUSH rule 3 x 155 [12,13]
+ CRUSH rule 3 x 156 [107,18]
+ CRUSH rule 3 x 157 [15,78]
+ CRUSH rule 3 x 158 [11,28]
+ CRUSH rule 3 x 159 [33,88]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,78]
+ CRUSH rule 3 x 162 [55,96]
+ CRUSH rule 3 x 163 [54,55]
+ CRUSH rule 3 x 164 [72,99]
+ CRUSH rule 3 x 165 [25,116]
+ CRUSH rule 3 x 166 [2,23]
+ CRUSH rule 3 x 167 [89,40]
+ CRUSH rule 3 x 168 [68,49]
+ CRUSH rule 3 x 169 [51,50]
+ CRUSH rule 3 x 170 [68,91]
+ CRUSH rule 3 x 171 [88,51]
+ CRUSH rule 3 x 172 [117,23]
+ CRUSH rule 3 x 173 [29,1]
+ CRUSH rule 3 x 174 [67,40]
+ CRUSH rule 3 x 175 [48,41]
+ CRUSH rule 3 x 176 [94,91]
+ CRUSH rule 3 x 177 [53,70]
+ CRUSH rule 3 x 178 [39,110]
+ CRUSH rule 3 x 179 [72,89]
+ CRUSH rule 3 x 180 [3,38]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,46]
+ CRUSH rule 3 x 183 [11,78]
+ CRUSH rule 3 x 184 [79,92]
+ CRUSH rule 3 x 185 [97,92]
+ CRUSH rule 3 x 186 [67,116]
+ CRUSH rule 3 x 187 [6,96]
+ CRUSH rule 3 x 188 [76,16]
+ CRUSH rule 3 x 189 [96,71]
+ CRUSH rule 3 x 190 [90,6]
+ CRUSH rule 3 x 191 [49,84]
+ CRUSH rule 3 x 192 [93,114]
+ CRUSH rule 3 x 193 [89,60]
+ CRUSH rule 3 x 194 [62,61]
+ CRUSH rule 3 x 195 [119,95]
+ CRUSH rule 3 x 196 [20,28]
+ CRUSH rule 3 x 197 [6,64]
+ CRUSH rule 3 x 198 [55,112]
+ CRUSH rule 3 x 199 [66,17]
+ CRUSH rule 3 x 200 [12,63]
+ CRUSH rule 3 x 201 [52,23]
+ CRUSH rule 3 x 202 [98,33]
+ CRUSH rule 3 x 203 [36,22]
+ CRUSH rule 3 x 204 [10,100]
+ CRUSH rule 3 x 205 [38,29]
+ CRUSH rule 3 x 206 [38,27]
+ CRUSH rule 3 x 207 [19,108]
+ CRUSH rule 3 x 208 [63,26]
+ CRUSH rule 3 x 209 [70,11]
+ CRUSH rule 3 x 210 [79,58]
+ CRUSH rule 3 x 211 [26,59]
+ CRUSH rule 3 x 212 [107,114]
+ CRUSH rule 3 x 213 [100,3]
+ CRUSH rule 3 x 214 [91,118]
+ CRUSH rule 3 x 215 [92,16]
+ CRUSH rule 3 x 216 [99,94]
+ CRUSH rule 3 x 217 [86,99]
+ CRUSH rule 3 x 218 [70,15]
+ CRUSH rule 3 x 219 [61,58]
+ CRUSH rule 3 x 220 [23,56]
+ CRUSH rule 3 x 221 [21,92]
+ CRUSH rule 3 x 222 [102,29]
+ CRUSH rule 3 x 223 [34,73]
+ CRUSH rule 3 x 224 [107,68]
+ CRUSH rule 3 x 225 [61,98]
+ CRUSH rule 3 x 226 [44,13]
+ CRUSH rule 3 x 227 [55,60]
+ CRUSH rule 3 x 228 [117,55]
+ CRUSH rule 3 x 229 [100,67]
+ CRUSH rule 3 x 230 [41,109]
+ CRUSH rule 3 x 231 [30,71]
+ CRUSH rule 3 x 232 [23,1]
+ CRUSH rule 3 x 233 [47,90]
+ CRUSH rule 3 x 234 [55,62]
+ CRUSH rule 3 x 235 [20,60]
+ CRUSH rule 3 x 236 [95,24]
+ CRUSH rule 3 x 237 [21,106]
+ CRUSH rule 3 x 238 [109,15]
+ CRUSH rule 3 x 239 [40,101]
+ CRUSH rule 3 x 240 [63,60]
+ CRUSH rule 3 x 241 [47,12]
+ CRUSH rule 3 x 242 [73,74]
+ CRUSH rule 3 x 243 [76,8]
+ CRUSH rule 3 x 244 [103,50]
+ CRUSH rule 3 x 245 [106,95]
+ CRUSH rule 3 x 246 [35,82]
+ CRUSH rule 3 x 247 [116,101]
+ CRUSH rule 3 x 248 [8,119]
+ CRUSH rule 3 x 249 [2,17]
+ CRUSH rule 3 x 250 [34,89]
+ CRUSH rule 3 x 251 [28,69]
+ CRUSH rule 3 x 252 [95,80]
+ CRUSH rule 3 x 253 [109,3]
+ CRUSH rule 3 x 254 [99,80]
+ CRUSH rule 3 x 255 [112,85]
+ CRUSH rule 3 x 256 [94,63]
+ CRUSH rule 3 x 257 [100,87]
+ CRUSH rule 3 x 258 [34,63]
+ CRUSH rule 3 x 259 [70,107]
+ CRUSH rule 3 x 260 [89,115]
+ CRUSH rule 3 x 261 [94,83]
+ CRUSH rule 3 x 262 [42,45]
+ CRUSH rule 3 x 263 [113,101]
+ CRUSH rule 3 x 264 [36,81]
+ CRUSH rule 3 x 265 [14,88]
+ CRUSH rule 3 x 266 [75,96]
+ CRUSH rule 3 x 267 [6,5]
+ CRUSH rule 3 x 268 [38,47]
+ CRUSH rule 3 x 269 [86,59]
+ CRUSH rule 3 x 270 [87,70]
+ CRUSH rule 3 x 271 [19,108]
+ CRUSH rule 3 x 272 [73,5]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,64]
+ CRUSH rule 3 x 275 [29,34]
+ CRUSH rule 3 x 276 [7,100]
+ CRUSH rule 3 x 277 [74,6]
+ CRUSH rule 3 x 278 [107,115]
+ CRUSH rule 3 x 279 [112,20]
+ CRUSH rule 3 x 280 [113,15]
+ CRUSH rule 3 x 281 [89,56]
+ CRUSH rule 3 x 282 [20,38]
+ CRUSH rule 3 x 283 [8,114]
+ CRUSH rule 3 x 284 [66,75]
+ CRUSH rule 3 x 285 [99,94]
+ CRUSH rule 3 x 286 [78,6]
+ CRUSH rule 3 x 287 [12,27]
+ CRUSH rule 3 x 288 [24,22]
+ CRUSH rule 3 x 289 [105,64]
+ CRUSH rule 3 x 290 [25,46]
+ CRUSH rule 3 x 291 [35,116]
+ CRUSH rule 3 x 292 [20,109]
+ CRUSH rule 3 x 293 [27,92]
+ CRUSH rule 3 x 294 [60,93]
+ CRUSH rule 3 x 295 [37,2]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,55]
+ CRUSH rule 3 x 298 [70,53]
+ CRUSH rule 3 x 299 [116,10]
+ CRUSH rule 3 x 300 [67,26]
+ CRUSH rule 3 x 301 [117,23]
+ CRUSH rule 3 x 302 [78,67]
+ CRUSH rule 3 x 303 [19,5]
+ CRUSH rule 3 x 304 [101,50]
+ CRUSH rule 3 x 305 [5,59]
+ CRUSH rule 3 x 306 [41,18]
+ CRUSH rule 3 x 307 [65,5]
+ CRUSH rule 3 x 308 [91,2]
+ CRUSH rule 3 x 309 [38,53]
+ CRUSH rule 3 x 310 [26,15]
+ CRUSH rule 3 x 311 [36,95]
+ CRUSH rule 3 x 312 [114,61]
+ CRUSH rule 3 x 313 [104,65]
+ CRUSH rule 3 x 314 [28,6]
+ CRUSH rule 3 x 315 [118,31]
+ CRUSH rule 3 x 316 [98,95]
+ CRUSH rule 3 x 317 [118,13]
+ CRUSH rule 3 x 318 [17,30]
+ CRUSH rule 3 x 319 [53,1]
+ CRUSH rule 3 x 320 [36,41]
+ CRUSH rule 3 x 321 [33,5]
+ CRUSH rule 3 x 322 [68,10]
+ CRUSH rule 3 x 323 [66,29]
+ CRUSH rule 3 x 324 [21,72]
+ CRUSH rule 3 x 325 [52,16]
+ CRUSH rule 3 x 326 [7,109]
+ CRUSH rule 3 x 327 [62,17]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,100]
+ CRUSH rule 3 x 330 [24,11]
+ CRUSH rule 3 x 331 [84,95]
+ CRUSH rule 3 x 332 [61,111]
+ CRUSH rule 3 x 333 [116,25]
+ CRUSH rule 3 x 334 [94,103]
+ CRUSH rule 3 x 335 [71,118]
+ CRUSH rule 3 x 336 [24,99]
+ CRUSH rule 3 x 337 [18,83]
+ CRUSH rule 3 x 338 [43,28]
+ CRUSH rule 3 x 339 [13,64]
+ CRUSH rule 3 x 340 [81,111]
+ CRUSH rule 3 x 341 [46,105]
+ CRUSH rule 3 x 342 [92,23]
+ CRUSH rule 3 x 343 [49,112]
+ CRUSH rule 3 x 344 [1,87]
+ CRUSH rule 3 x 345 [56,35]
+ CRUSH rule 3 x 346 [3,54]
+ CRUSH rule 3 x 347 [106,27]
+ CRUSH rule 3 x 348 [10,117]
+ CRUSH rule 3 x 349 [96,87]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,41]
+ CRUSH rule 3 x 352 [36,13]
+ CRUSH rule 3 x 353 [10,82]
+ CRUSH rule 3 x 354 [55,52]
+ CRUSH rule 3 x 355 [73,94]
+ CRUSH rule 3 x 356 [75,66]
+ CRUSH rule 3 x 357 [70,93]
+ CRUSH rule 3 x 358 [97,56]
+ CRUSH rule 3 x 359 [110,105]
+ CRUSH rule 3 x 360 [106,57]
+ CRUSH rule 3 x 361 [27,42]
+ CRUSH rule 3 x 362 [28,55]
+ CRUSH rule 3 x 363 [68,20]
+ CRUSH rule 3 x 364 [23,50]
+ CRUSH rule 3 x 365 [57,76]
+ CRUSH rule 3 x 366 [42,75]
+ CRUSH rule 3 x 367 [103,82]
+ CRUSH rule 3 x 368 [103,104]
+ CRUSH rule 3 x 369 [12,57]
+ CRUSH rule 3 x 370 [11,26]
+ CRUSH rule 3 x 371 [34,55]
+ CRUSH rule 3 x 372 [58,14]
+ CRUSH rule 3 x 373 [6,42]
+ CRUSH rule 3 x 374 [110,95]
+ CRUSH rule 3 x 375 [5,43]
+ CRUSH rule 3 x 376 [91,86]
+ CRUSH rule 3 x 377 [93,116]
+ CRUSH rule 3 x 378 [68,6]
+ CRUSH rule 3 x 379 [77,44]
+ CRUSH rule 3 x 380 [76,83]
+ CRUSH rule 3 x 381 [36,27]
+ CRUSH rule 3 x 382 [26,77]
+ CRUSH rule 3 x 383 [76,99]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,93]
+ CRUSH rule 3 x 386 [83,92]
+ CRUSH rule 3 x 387 [16,26]
+ CRUSH rule 3 x 388 [29,113]
+ CRUSH rule 3 x 389 [92,29]
+ CRUSH rule 3 x 390 [68,77]
+ CRUSH rule 3 x 391 [15,88]
+ CRUSH rule 3 x 392 [21,32]
+ CRUSH rule 3 x 393 [91,18]
+ CRUSH rule 3 x 394 [38,73]
+ CRUSH rule 3 x 395 [21,119]
+ CRUSH rule 3 x 396 [12,13]
+ CRUSH rule 3 x 397 [40,63]
+ CRUSH rule 3 x 398 [44,3]
+ CRUSH rule 3 x 399 [5,95]
+ CRUSH rule 3 x 400 [19,102]
+ CRUSH rule 3 x 401 [79,52]
+ CRUSH rule 3 x 402 [107,98]
+ CRUSH rule 3 x 403 [23,82]
+ CRUSH rule 3 x 404 [87,68]
+ CRUSH rule 3 x 405 [90,97]
+ CRUSH rule 3 x 406 [15,117]
+ CRUSH rule 3 x 407 [70,35]
+ CRUSH rule 3 x 408 [55,72]
+ CRUSH rule 3 x 409 [73,62]
+ CRUSH rule 3 x 410 [70,73]
+ CRUSH rule 3 x 411 [34,25]
+ CRUSH rule 3 x 412 [105,117]
+ CRUSH rule 3 x 413 [41,110]
+ CRUSH rule 3 x 414 [70,65]
+ CRUSH rule 3 x 415 [107,5]
+ CRUSH rule 3 x 416 [2,22]
+ CRUSH rule 3 x 417 [26,14]
+ CRUSH rule 3 x 418 [51,46]
+ CRUSH rule 3 x 419 [8,82]
+ CRUSH rule 3 x 420 [109,105]
+ CRUSH rule 3 x 421 [114,75]
+ CRUSH rule 3 x 422 [109,87]
+ CRUSH rule 3 x 423 [59,24]
+ CRUSH rule 3 x 424 [92,51]
+ CRUSH rule 3 x 425 [101,111]
+ CRUSH rule 3 x 426 [36,6]
+ CRUSH rule 3 x 427 [8,24]
+ CRUSH rule 3 x 428 [68,35]
+ CRUSH rule 3 x 429 [76,75]
+ CRUSH rule 3 x 430 [67,117]
+ CRUSH rule 3 x 431 [70,25]
+ CRUSH rule 3 x 432 [7,34]
+ CRUSH rule 3 x 433 [49,84]
+ CRUSH rule 3 x 434 [64,31]
+ CRUSH rule 3 x 435 [110,13]
+ CRUSH rule 3 x 436 [106,89]
+ CRUSH rule 3 x 437 [26,65]
+ CRUSH rule 3 x 438 [118,63]
+ CRUSH rule 3 x 439 [40,21]
+ CRUSH rule 3 x 440 [45,119]
+ CRUSH rule 3 x 441 [112,105]
+ CRUSH rule 3 x 442 [55,113]
+ CRUSH rule 3 x 443 [44,33]
+ CRUSH rule 3 x 444 [71,38]
+ CRUSH rule 3 x 445 [58,81]
+ CRUSH rule 3 x 446 [40,10]
+ CRUSH rule 3 x 447 [100,61]
+ CRUSH rule 3 x 448 [111,73]
+ CRUSH rule 3 x 449 [67,66]
+ CRUSH rule 3 x 450 [117,61]
+ CRUSH rule 3 x 451 [66,81]
+ CRUSH rule 3 x 452 [70,8]
+ CRUSH rule 3 x 453 [82,85]
+ CRUSH rule 3 x 454 [53,18]
+ CRUSH rule 3 x 455 [91,42]
+ CRUSH rule 3 x 456 [101,46]
+ CRUSH rule 3 x 457 [113,51]
+ CRUSH rule 3 x 458 [119,25]
+ CRUSH rule 3 x 459 [50,67]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,51]
+ CRUSH rule 3 x 462 [98,107]
+ CRUSH rule 3 x 463 [108,105]
+ CRUSH rule 3 x 464 [19,109]
+ CRUSH rule 3 x 465 [62,23]
+ CRUSH rule 3 x 466 [53,12]
+ CRUSH rule 3 x 467 [40,57]
+ CRUSH rule 3 x 468 [97,44]
+ CRUSH rule 3 x 469 [98,75]
+ CRUSH rule 3 x 470 [50,29]
+ CRUSH rule 3 x 471 [40,13]
+ CRUSH rule 3 x 472 [27,18]
+ CRUSH rule 3 x 473 [48,35]
+ CRUSH rule 3 x 474 [51,32]
+ CRUSH rule 3 x 475 [49,117]
+ CRUSH rule 3 x 476 [110,31]
+ CRUSH rule 3 x 477 [80,97]
+ CRUSH rule 3 x 478 [78,99]
+ CRUSH rule 3 x 479 [31,66]
+ CRUSH rule 3 x 480 [75,88]
+ CRUSH rule 3 x 481 [26,20]
+ CRUSH rule 3 x 482 [84,53]
+ CRUSH rule 3 x 483 [15,116]
+ CRUSH rule 3 x 484 [37,114]
+ CRUSH rule 3 x 485 [84,8]
+ CRUSH rule 3 x 486 [92,10]
+ CRUSH rule 3 x 487 [106,17]
+ CRUSH rule 3 x 488 [42,20]
+ CRUSH rule 3 x 489 [89,2]
+ CRUSH rule 3 x 490 [22,114]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,66]
+ CRUSH rule 3 x 493 [94,14]
+ CRUSH rule 3 x 494 [59,86]
+ CRUSH rule 3 x 495 [95,58]
+ CRUSH rule 3 x 496 [46,41]
+ CRUSH rule 3 x 497 [102,27]
+ CRUSH rule 3 x 498 [21,116]
+ CRUSH rule 3 x 499 [5,49]
+ CRUSH rule 3 x 500 [50,49]
+ CRUSH rule 3 x 501 [60,3]
+ CRUSH rule 3 x 502 [65,110]
+ CRUSH rule 3 x 503 [21,112]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,93]
+ CRUSH rule 3 x 506 [79,64]
+ CRUSH rule 3 x 507 [34,107]
+ CRUSH rule 3 x 508 [45,114]
+ CRUSH rule 3 x 509 [19,88]
+ CRUSH rule 3 x 510 [117,45]
+ CRUSH rule 3 x 511 [14,104]
+ CRUSH rule 3 x 512 [59,26]
+ CRUSH rule 3 x 513 [102,93]
+ CRUSH rule 3 x 514 [75,72]
+ CRUSH rule 3 x 515 [84,41]
+ CRUSH rule 3 x 516 [37,30]
+ CRUSH rule 3 x 517 [83,115]
+ CRUSH rule 3 x 518 [18,83]
+ CRUSH rule 3 x 519 [67,88]
+ CRUSH rule 3 x 520 [15,114]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,51]
+ CRUSH rule 3 x 523 [68,101]
+ CRUSH rule 3 x 524 [33,38]
+ CRUSH rule 3 x 525 [63,115]
+ CRUSH rule 3 x 526 [83,50]
+ CRUSH rule 3 x 527 [37,56]
+ CRUSH rule 3 x 528 [108,81]
+ CRUSH rule 3 x 529 [74,33]
+ CRUSH rule 3 x 530 [49,92]
+ CRUSH rule 3 x 531 [117,105]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,85]
+ CRUSH rule 3 x 534 [97,24]
+ CRUSH rule 3 x 535 [48,75]
+ CRUSH rule 3 x 536 [113,101]
+ CRUSH rule 3 x 537 [116,47]
+ CRUSH rule 3 x 538 [85,74]
+ CRUSH rule 3 x 539 [72,43]
+ CRUSH rule 3 x 540 [39,34]
+ CRUSH rule 3 x 541 [53,84]
+ CRUSH rule 3 x 542 [27,32]
+ CRUSH rule 3 x 543 [45,113]
+ CRUSH rule 3 x 544 [59,42]
+ CRUSH rule 3 x 545 [118,95]
+ CRUSH rule 3 x 546 [18,79]
+ CRUSH rule 3 x 547 [67,30]
+ CRUSH rule 3 x 548 [53,100]
+ CRUSH rule 3 x 549 [60,45]
+ CRUSH rule 3 x 550 [92,101]
+ CRUSH rule 3 x 551 [77,88]
+ CRUSH rule 3 x 552 [61,94]
+ CRUSH rule 3 x 553 [71,78]
+ CRUSH rule 3 x 554 [61,115]
+ CRUSH rule 3 x 555 [76,77]
+ CRUSH rule 3 x 556 [106,55]
+ CRUSH rule 3 x 557 [26,22]
+ CRUSH rule 3 x 558 [41,84]
+ CRUSH rule 3 x 559 [65,24]
+ CRUSH rule 3 x 560 [94,16]
+ CRUSH rule 3 x 561 [27,5]
+ CRUSH rule 3 x 562 [78,59]
+ CRUSH rule 3 x 563 [59,70]
+ CRUSH rule 3 x 564 [96,8]
+ CRUSH rule 3 x 565 [8,48]
+ CRUSH rule 3 x 566 [119,17]
+ CRUSH rule 3 x 567 [7,38]
+ CRUSH rule 3 x 568 [57,94]
+ CRUSH rule 3 x 569 [65,26]
+ CRUSH rule 3 x 570 [98,27]
+ CRUSH rule 3 x 571 [95,30]
+ CRUSH rule 3 x 572 [62,83]
+ CRUSH rule 3 x 573 [1,79]
+ CRUSH rule 3 x 574 [89,42]
+ CRUSH rule 3 x 575 [87,113]
+ CRUSH rule 3 x 576 [21,68]
+ CRUSH rule 3 x 577 [8,84]
+ CRUSH rule 3 x 578 [75,115]
+ CRUSH rule 3 x 579 [105,68]
+ CRUSH rule 3 x 580 [51,28]
+ CRUSH rule 3 x 581 [55,113]
+ CRUSH rule 3 x 582 [27,113]
+ CRUSH rule 3 x 583 [6,78]
+ CRUSH rule 3 x 584 [10,30]
+ CRUSH rule 3 x 585 [20,111]
+ CRUSH rule 3 x 586 [48,27]
+ CRUSH rule 3 x 587 [29,94]
+ CRUSH rule 3 x 588 [103,90]
+ CRUSH rule 3 x 589 [88,95]
+ CRUSH rule 3 x 590 [76,101]
+ CRUSH rule 3 x 591 [42,43]
+ CRUSH rule 3 x 592 [78,51]
+ CRUSH rule 3 x 593 [82,71]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,59]
+ CRUSH rule 3 x 597 [16,36]
+ CRUSH rule 3 x 598 [37,56]
+ CRUSH rule 3 x 599 [10,84]
+ CRUSH rule 3 x 600 [24,69]
+ CRUSH rule 3 x 601 [104,14]
+ CRUSH rule 3 x 602 [48,45]
+ CRUSH rule 3 x 603 [93,32]
+ CRUSH rule 3 x 604 [118,79]
+ CRUSH rule 3 x 605 [104,53]
+ CRUSH rule 3 x 606 [90,83]
+ CRUSH rule 3 x 607 [95,110]
+ CRUSH rule 3 x 608 [112,101]
+ CRUSH rule 3 x 609 [34,99]
+ CRUSH rule 3 x 610 [106,16]
+ CRUSH rule 3 x 611 [66,87]
+ CRUSH rule 3 x 612 [2,81]
+ CRUSH rule 3 x 613 [13,86]
+ CRUSH rule 3 x 614 [50,3]
+ CRUSH rule 3 x 615 [24,73]
+ CRUSH rule 3 x 616 [41,119]
+ CRUSH rule 3 x 617 [81,106]
+ CRUSH rule 3 x 618 [3,104]
+ CRUSH rule 3 x 619 [92,7]
+ CRUSH rule 3 x 620 [108,11]
+ CRUSH rule 3 x 621 [105,115]
+ CRUSH rule 3 x 622 [67,48]
+ CRUSH rule 3 x 623 [69,74]
+ CRUSH rule 3 x 624 [115,49]
+ CRUSH rule 3 x 625 [73,109]
+ CRUSH rule 3 x 626 [52,3]
+ CRUSH rule 3 x 627 [116,3]
+ CRUSH rule 3 x 628 [98,91]
+ CRUSH rule 3 x 629 [6,112]
+ CRUSH rule 3 x 630 [22,72]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,71]
+ CRUSH rule 3 x 633 [65,12]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,46]
+ CRUSH rule 3 x 636 [23,70]
+ CRUSH rule 3 x 637 [99,24]
+ CRUSH rule 3 x 638 [43,114]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,73]
+ CRUSH rule 3 x 641 [45,84]
+ CRUSH rule 3 x 642 [47,66]
+ CRUSH rule 3 x 643 [64,8]
+ CRUSH rule 3 x 644 [31,82]
+ CRUSH rule 3 x 645 [77,64]
+ CRUSH rule 3 x 646 [37,86]
+ CRUSH rule 3 x 647 [65,56]
+ CRUSH rule 3 x 648 [84,13]
+ CRUSH rule 3 x 649 [88,55]
+ CRUSH rule 3 x 650 [21,76]
+ CRUSH rule 3 x 651 [63,116]
+ CRUSH rule 3 x 652 [57,112]
+ CRUSH rule 3 x 653 [38,61]
+ CRUSH rule 3 x 654 [104,67]
+ CRUSH rule 3 x 655 [89,54]
+ CRUSH rule 3 x 656 [84,49]
+ CRUSH rule 3 x 657 [47,32]
+ CRUSH rule 3 x 658 [80,29]
+ CRUSH rule 3 x 659 [11,112]
+ CRUSH rule 3 x 660 [65,111]
+ CRUSH rule 3 x 661 [96,73]
+ CRUSH rule 3 x 662 [111,73]
+ CRUSH rule 3 x 663 [83,60]
+ CRUSH rule 3 x 664 [59,80]
+ CRUSH rule 3 x 665 [31,117]
+ CRUSH rule 3 x 666 [112,101]
+ CRUSH rule 3 x 667 [70,47]
+ CRUSH rule 3 x 668 [96,57]
+ CRUSH rule 3 x 669 [56,39]
+ CRUSH rule 3 x 670 [98,105]
+ CRUSH rule 3 x 671 [57,48]
+ CRUSH rule 3 x 672 [37,36]
+ CRUSH rule 3 x 673 [83,2]
+ CRUSH rule 3 x 674 [36,25]
+ CRUSH rule 3 x 675 [88,14]
+ CRUSH rule 3 x 676 [3,110]
+ CRUSH rule 3 x 677 [88,67]
+ CRUSH rule 3 x 678 [27,44]
+ CRUSH rule 3 x 679 [33,116]
+ CRUSH rule 3 x 680 [111,39]
+ CRUSH rule 3 x 681 [53,12]
+ CRUSH rule 3 x 682 [12,87]
+ CRUSH rule 3 x 683 [24,85]
+ CRUSH rule 3 x 684 [98,65]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,72]
+ CRUSH rule 3 x 688 [16,114]
+ CRUSH rule 3 x 689 [32,31]
+ CRUSH rule 3 x 690 [96,33]
+ CRUSH rule 3 x 691 [34,6]
+ CRUSH rule 3 x 692 [97,84]
+ CRUSH rule 3 x 693 [29,118]
+ CRUSH rule 3 x 694 [6,30]
+ CRUSH rule 3 x 695 [31,72]
+ CRUSH rule 3 x 696 [104,97]
+ CRUSH rule 3 x 697 [19,96]
+ CRUSH rule 3 x 698 [30,69]
+ CRUSH rule 3 x 699 [47,76]
+ CRUSH rule 3 x 700 [82,55]
+ CRUSH rule 3 x 701 [53,80]
+ CRUSH rule 3 x 702 [95,98]
+ CRUSH rule 3 x 703 [92,65]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,1]
+ CRUSH rule 3 x 706 [74,35]
+ CRUSH rule 3 x 707 [91,115]
+ CRUSH rule 3 x 708 [95,112]
+ CRUSH rule 3 x 709 [73,72]
+ CRUSH rule 3 x 710 [94,47]
+ CRUSH rule 3 x 711 [68,41]
+ CRUSH rule 3 x 712 [107,18]
+ CRUSH rule 3 x 713 [29,109]
+ CRUSH rule 3 x 714 [86,61]
+ CRUSH rule 3 x 715 [74,13]
+ CRUSH rule 3 x 716 [101,56]
+ CRUSH rule 3 x 717 [12,29]
+ CRUSH rule 3 x 718 [83,24]
+ CRUSH rule 3 x 719 [26,10]
+ CRUSH rule 3 x 720 [69,2]
+ CRUSH rule 3 x 721 [51,42]
+ CRUSH rule 3 x 722 [15,74]
+ CRUSH rule 3 x 723 [117,14]
+ CRUSH rule 3 x 724 [45,38]
+ CRUSH rule 3 x 725 [53,110]
+ CRUSH rule 3 x 726 [103,68]
+ CRUSH rule 3 x 727 [89,100]
+ CRUSH rule 3 x 728 [76,16]
+ CRUSH rule 3 x 729 [35,90]
+ CRUSH rule 3 x 730 [28,103]
+ CRUSH rule 3 x 731 [78,41]
+ CRUSH rule 3 x 732 [1,27]
+ CRUSH rule 3 x 733 [35,100]
+ CRUSH rule 3 x 734 [119,85]
+ CRUSH rule 3 x 735 [102,43]
+ CRUSH rule 3 x 736 [37,92]
+ CRUSH rule 3 x 737 [117,11]
+ CRUSH rule 3 x 738 [57,32]
+ CRUSH rule 3 x 739 [87,1]
+ CRUSH rule 3 x 740 [29,80]
+ CRUSH rule 3 x 741 [47,111]
+ CRUSH rule 3 x 742 [106,83]
+ CRUSH rule 3 x 743 [105,94]
+ CRUSH rule 3 x 744 [23,64]
+ CRUSH rule 3 x 745 [37,112]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,95]
+ CRUSH rule 3 x 748 [48,14]
+ CRUSH rule 3 x 749 [102,101]
+ CRUSH rule 3 x 750 [83,78]
+ CRUSH rule 3 x 751 [25,104]
+ CRUSH rule 3 x 752 [82,95]
+ CRUSH rule 3 x 753 [14,113]
+ CRUSH rule 3 x 754 [114,51]
+ CRUSH rule 3 x 755 [87,26]
+ CRUSH rule 3 x 756 [113,87]
+ CRUSH rule 3 x 757 [47,66]
+ CRUSH rule 3 x 758 [54,63]
+ CRUSH rule 3 x 759 [74,20]
+ CRUSH rule 3 x 760 [88,22]
+ CRUSH rule 3 x 761 [73,86]
+ CRUSH rule 3 x 762 [34,17]
+ CRUSH rule 3 x 763 [13,78]
+ CRUSH rule 3 x 764 [89,42]
+ CRUSH rule 3 x 765 [109,91]
+ CRUSH rule 3 x 766 [19,66]
+ CRUSH rule 3 x 767 [41,26]
+ CRUSH rule 3 x 768 [106,57]
+ CRUSH rule 3 x 769 [91,104]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,35]
+ CRUSH rule 3 x 772 [97,108]
+ CRUSH rule 3 x 773 [116,47]
+ CRUSH rule 3 x 774 [100,31]
+ CRUSH rule 3 x 775 [102,43]
+ CRUSH rule 3 x 776 [69,38]
+ CRUSH rule 3 x 777 [91,52]
+ CRUSH rule 3 x 778 [83,119]
+ CRUSH rule 3 x 779 [47,60]
+ CRUSH rule 3 x 780 [63,70]
+ CRUSH rule 3 x 781 [105,2]
+ CRUSH rule 3 x 782 [117,59]
+ CRUSH rule 3 x 783 [19,109]
+ CRUSH rule 3 x 784 [63,114]
+ CRUSH rule 3 x 785 [27,84]
+ CRUSH rule 3 x 786 [41,110]
+ CRUSH rule 3 x 787 [108,73]
+ CRUSH rule 3 x 788 [74,103]
+ CRUSH rule 3 x 789 [50,17]
+ CRUSH rule 3 x 790 [20,106]
+ CRUSH rule 3 x 791 [96,87]
+ CRUSH rule 3 x 792 [80,97]
+ CRUSH rule 3 x 793 [6,26]
+ CRUSH rule 3 x 794 [14,42]
+ CRUSH rule 3 x 795 [30,8]
+ CRUSH rule 3 x 796 [87,36]
+ CRUSH rule 3 x 797 [64,61]
+ CRUSH rule 3 x 798 [42,69]
+ CRUSH rule 3 x 799 [19,117]
+ CRUSH rule 3 x 800 [106,8]
+ CRUSH rule 3 x 801 [2,57]
+ CRUSH rule 3 x 802 [63,68]
+ CRUSH rule 3 x 803 [46,35]
+ CRUSH rule 3 x 804 [33,26]
+ CRUSH rule 3 x 805 [96,49]
+ CRUSH rule 3 x 806 [48,25]
+ CRUSH rule 3 x 807 [48,83]
+ CRUSH rule 3 x 808 [76,31]
+ CRUSH rule 3 x 809 [27,48]
+ CRUSH rule 3 x 810 [119,71]
+ CRUSH rule 3 x 811 [111,91]
+ CRUSH rule 3 x 812 [25,111]
+ CRUSH rule 3 x 813 [81,28]
+ CRUSH rule 3 x 814 [95,42]
+ CRUSH rule 3 x 815 [84,61]
+ CRUSH rule 3 x 816 [64,35]
+ CRUSH rule 3 x 817 [63,60]
+ CRUSH rule 3 x 818 [69,46]
+ CRUSH rule 3 x 819 [88,75]
+ CRUSH rule 3 x 820 [104,57]
+ CRUSH rule 3 x 821 [58,21]
+ CRUSH rule 3 x 822 [20,80]
+ CRUSH rule 3 x 823 [63,118]
+ CRUSH rule 3 x 824 [102,13]
+ CRUSH rule 3 x 825 [47,118]
+ CRUSH rule 3 x 826 [44,7]
+ CRUSH rule 3 x 827 [101,88]
+ CRUSH rule 3 x 828 [60,41]
+ CRUSH rule 3 x 829 [45,102]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,75]
+ CRUSH rule 3 x 833 [57,32]
+ CRUSH rule 3 x 834 [90,33]
+ CRUSH rule 3 x 835 [6,1]
+ CRUSH rule 3 x 836 [63,68]
+ CRUSH rule 3 x 837 [76,71]
+ CRUSH rule 3 x 838 [106,20]
+ CRUSH rule 3 x 839 [87,96]
+ CRUSH rule 3 x 840 [33,32]
+ CRUSH rule 3 x 841 [110,55]
+ CRUSH rule 3 x 842 [66,87]
+ CRUSH rule 3 x 843 [11,80]
+ CRUSH rule 3 x 844 [74,103]
+ CRUSH rule 3 x 845 [74,43]
+ CRUSH rule 3 x 846 [43,76]
+ CRUSH rule 3 x 847 [62,20]
+ CRUSH rule 3 x 848 [92,17]
+ CRUSH rule 3 x 849 [93,36]
+ CRUSH rule 3 x 850 [83,82]
+ CRUSH rule 3 x 851 [65,94]
+ CRUSH rule 3 x 852 [60,22]
+ CRUSH rule 3 x 853 [88,29]
+ CRUSH rule 3 x 854 [83,54]
+ CRUSH rule 3 x 855 [2,101]
+ CRUSH rule 3 x 856 [40,41]
+ CRUSH rule 3 x 857 [69,82]
+ CRUSH rule 3 x 858 [98,81]
+ CRUSH rule 3 x 859 [56,43]
+ CRUSH rule 3 x 860 [11,26]
+ CRUSH rule 3 x 861 [22,110]
+ CRUSH rule 3 x 862 [22,70]
+ CRUSH rule 3 x 863 [79,84]
+ CRUSH rule 3 x 864 [77,24]
+ CRUSH rule 3 x 865 [119,17]
+ CRUSH rule 3 x 866 [18,49]
+ CRUSH rule 3 x 867 [3,84]
+ CRUSH rule 3 x 868 [100,107]
+ CRUSH rule 3 x 869 [22,104]
+ CRUSH rule 3 x 870 [73,30]
+ CRUSH rule 3 x 871 [84,105]
+ CRUSH rule 3 x 872 [72,75]
+ CRUSH rule 3 x 873 [81,96]
+ CRUSH rule 3 x 874 [21,72]
+ CRUSH rule 3 x 875 [115,59]
+ CRUSH rule 3 x 876 [98,49]
+ CRUSH rule 3 x 877 [80,79]
+ CRUSH rule 3 x 878 [87,94]
+ CRUSH rule 3 x 879 [29,18]
+ CRUSH rule 3 x 880 [23,40]
+ CRUSH rule 3 x 881 [109,69]
+ CRUSH rule 3 x 882 [31,118]
+ CRUSH rule 3 x 883 [102,8]
+ CRUSH rule 3 x 884 [80,19]
+ CRUSH rule 3 x 885 [46,101]
+ CRUSH rule 3 x 886 [2,65]
+ CRUSH rule 3 x 887 [5,99]
+ CRUSH rule 3 x 888 [16,70]
+ CRUSH rule 3 x 889 [84,93]
+ CRUSH rule 3 x 890 [65,118]
+ CRUSH rule 3 x 891 [86,105]
+ CRUSH rule 3 x 892 [64,10]
+ CRUSH rule 3 x 893 [20,110]
+ CRUSH rule 3 x 894 [32,47]
+ CRUSH rule 3 x 895 [40,21]
+ CRUSH rule 3 x 896 [113,14]
+ CRUSH rule 3 x 897 [107,80]
+ CRUSH rule 3 x 898 [76,71]
+ CRUSH rule 3 x 899 [75,82]
+ CRUSH rule 3 x 900 [83,82]
+ CRUSH rule 3 x 901 [66,61]
+ CRUSH rule 3 x 902 [25,56]
+ CRUSH rule 3 x 903 [53,46]
+ CRUSH rule 3 x 904 [50,101]
+ CRUSH rule 3 x 905 [99,110]
+ CRUSH rule 3 x 906 [68,27]
+ CRUSH rule 3 x 907 [109,47]
+ CRUSH rule 3 x 908 [47,1]
+ CRUSH rule 3 x 909 [73,2]
+ CRUSH rule 3 x 910 [71,74]
+ CRUSH rule 3 x 911 [39,42]
+ CRUSH rule 3 x 912 [90,7]
+ CRUSH rule 3 x 913 [29,96]
+ CRUSH rule 3 x 914 [84,45]
+ CRUSH rule 3 x 915 [49,115]
+ CRUSH rule 3 x 916 [32,77]
+ CRUSH rule 3 x 917 [46,23]
+ CRUSH rule 3 x 918 [82,73]
+ CRUSH rule 3 x 919 [13,28]
+ CRUSH rule 3 x 920 [25,26]
+ CRUSH rule 3 x 921 [55,119]
+ CRUSH rule 3 x 922 [33,32]
+ CRUSH rule 3 x 923 [28,15]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,22]
+ CRUSH rule 3 x 926 [64,69]
+ CRUSH rule 3 x 927 [32,16]
+ CRUSH rule 3 x 928 [13,113]
+ CRUSH rule 3 x 929 [85,115]
+ CRUSH rule 3 x 930 [104,20]
+ CRUSH rule 3 x 931 [46,79]
+ CRUSH rule 3 x 932 [43,52]
+ CRUSH rule 3 x 933 [18,55]
+ CRUSH rule 3 x 934 [68,81]
+ CRUSH rule 3 x 935 [28,57]
+ CRUSH rule 3 x 936 [104,57]
+ CRUSH rule 3 x 937 [110,10]
+ CRUSH rule 3 x 938 [48,23]
+ CRUSH rule 3 x 939 [77,42]
+ CRUSH rule 3 x 940 [76,49]
+ CRUSH rule 3 x 941 [66,101]
+ CRUSH rule 3 x 942 [80,87]
+ CRUSH rule 3 x 943 [75,74]
+ CRUSH rule 3 x 944 [82,53]
+ CRUSH rule 3 x 945 [71,74]
+ CRUSH rule 3 x 946 [37,42]
+ CRUSH rule 3 x 947 [107,30]
+ CRUSH rule 3 x 948 [108,95]
+ CRUSH rule 3 x 949 [46,103]
+ CRUSH rule 3 x 950 [96,101]
+ CRUSH rule 3 x 951 [40,14]
+ CRUSH rule 3 x 952 [114,21]
+ CRUSH rule 3 x 953 [62,23]
+ CRUSH rule 3 x 954 [103,5]
+ CRUSH rule 3 x 955 [42,73]
+ CRUSH rule 3 x 956 [72,103]
+ CRUSH rule 3 x 957 [117,22]
+ CRUSH rule 3 x 958 [23,106]
+ CRUSH rule 3 x 959 [42,93]
+ CRUSH rule 3 x 960 [113,8]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,51]
+ CRUSH rule 3 x 963 [101,106]
+ CRUSH rule 3 x 964 [66,89]
+ CRUSH rule 3 x 965 [47,102]
+ CRUSH rule 3 x 966 [88,63]
+ CRUSH rule 3 x 967 [71,46]
+ CRUSH rule 3 x 968 [74,51]
+ CRUSH rule 3 x 969 [53,78]
+ CRUSH rule 3 x 970 [3,30]
+ CRUSH rule 3 x 971 [66,107]
+ CRUSH rule 3 x 972 [3,66]
+ CRUSH rule 3 x 973 [113,20]
+ CRUSH rule 3 x 974 [114,35]
+ CRUSH rule 3 x 975 [83,58]
+ CRUSH rule 3 x 976 [81,48]
+ CRUSH rule 3 x 977 [95,102]
+ CRUSH rule 3 x 978 [119,41]
+ CRUSH rule 3 x 979 [98,6]
+ CRUSH rule 3 x 980 [39,108]
+ CRUSH rule 3 x 981 [89,84]
+ CRUSH rule 3 x 982 [19,94]
+ CRUSH rule 3 x 983 [34,45]
+ CRUSH rule 3 x 984 [78,63]
+ CRUSH rule 3 x 985 [99,52]
+ CRUSH rule 3 x 986 [44,99]
+ CRUSH rule 3 x 987 [25,32]
+ CRUSH rule 3 x 988 [79,2]
+ CRUSH rule 3 x 989 [87,26]
+ CRUSH rule 3 x 990 [72,69]
+ CRUSH rule 3 x 991 [90,8]
+ CRUSH rule 3 x 992 [30,67]
+ CRUSH rule 3 x 993 [74,49]
+ CRUSH rule 3 x 994 [74,105]
+ CRUSH rule 3 x 995 [100,97]
+ CRUSH rule 3 x 996 [41,58]
+ CRUSH rule 3 x 997 [89,76]
+ CRUSH rule 3 x 998 [92,47]
+ CRUSH rule 3 x 999 [117,16]
+ CRUSH rule 3 x 1000 [50,47]
+ CRUSH rule 3 x 1001 [83,102]
+ CRUSH rule 3 x 1002 [94,37]
+ CRUSH rule 3 x 1003 [43,88]
+ CRUSH rule 3 x 1004 [89,54]
+ CRUSH rule 3 x 1005 [105,84]
+ CRUSH rule 3 x 1006 [45,111]
+ CRUSH rule 3 x 1007 [19,66]
+ CRUSH rule 3 x 1008 [31,76]
+ CRUSH rule 3 x 1009 [1,95]
+ CRUSH rule 3 x 1010 [31,113]
+ CRUSH rule 3 x 1011 [64,81]
+ CRUSH rule 3 x 1012 [68,49]
+ CRUSH rule 3 x 1013 [5,93]
+ CRUSH rule 3 x 1014 [33,66]
+ CRUSH rule 3 x 1015 [106,45]
+ CRUSH rule 3 x 1016 [107,86]
+ CRUSH rule 3 x 1017 [12,61]
+ CRUSH rule 3 x 1018 [61,26]
+ CRUSH rule 3 x 1019 [27,104]
+ CRUSH rule 3 x 1020 [31,86]
+ CRUSH rule 3 x 1021 [22,26]
+ CRUSH rule 3 x 1022 [73,34]
+ CRUSH rule 3 x 1023 [88,79]
+ rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/src/test/cli/crushtool/test-map-vary-r-2.t b/src/test/cli/crushtool/test-map-vary-r-2.t
new file mode 100644
index 0000000..c9e78c6
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-vary-r-2.t
@@ -0,0 +1,3078 @@
+ $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 2 --weight 0 0 --weight 4 0 --weight 9 0
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+ CRUSH rule 3 x 0 [94,45]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,118]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,96]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,70]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,76]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,100]
+ CRUSH rule 3 x 17 [85,110]
+ CRUSH rule 3 x 18 [11,119]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,47]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,90]
+ CRUSH rule 3 x 24 [83,38]
+ CRUSH rule 3 x 25 [81,5]
+ CRUSH rule 3 x 26 [17,100]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,119]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,69]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,52]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,29]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,11]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,68]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,21]
+ CRUSH rule 3 x 47 [106,53]
+ CRUSH rule 3 x 48 [34,33]
+ CRUSH rule 3 x 49 [99,104]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,52]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,75]
+ CRUSH rule 3 x 54 [28,93]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,40]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,33]
+ CRUSH rule 3 x 61 [88,3]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,113]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,61]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,7]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,44]
+ CRUSH rule 3 x 74 [29,74]
+ CRUSH rule 3 x 75 [60,89]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,7]
+ CRUSH rule 3 x 79 [64,41]
+ CRUSH rule 3 x 80 [73,104]
+ CRUSH rule 3 x 81 [64,19]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,68]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,95]
+ CRUSH rule 3 x 89 [76,3]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,65]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,23]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,100]
+ CRUSH rule 3 x 99 [78,22]
+ CRUSH rule 3 x 100 [28,63]
+ CRUSH rule 3 x 101 [91,36]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,95]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,116]
+ CRUSH rule 3 x 107 [1,55]
+ CRUSH rule 3 x 108 [7,72]
+ CRUSH rule 3 x 109 [112,63]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,78]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,54]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,42]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,16]
+ CRUSH rule 3 x 122 [45,110]
+ CRUSH rule 3 x 123 [22,86]
+ CRUSH rule 3 x 124 [97,72]
+ CRUSH rule 3 x 125 [66,71]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,97]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,110]
+ CRUSH rule 3 x 130 [50,63]
+ CRUSH rule 3 x 131 [44,25]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,2]
+ CRUSH rule 3 x 134 [37,94]
+ CRUSH rule 3 x 135 [78,17]
+ CRUSH rule 3 x 136 [32,91]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,62]
+ CRUSH rule 3 x 141 [89,96]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,87]
+ CRUSH rule 3 x 144 [13,86]
+ CRUSH rule 3 x 145 [77,60]
+ CRUSH rule 3 x 146 [12,53]
+ CRUSH rule 3 x 147 [2,59]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,58]
+ CRUSH rule 3 x 150 [14,54]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,33]
+ CRUSH rule 3 x 154 [19,111]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,5]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,58]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,31]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,47]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,107]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,34]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,2]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,1]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,90]
+ CRUSH rule 3 x 183 [11,74]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,113]
+ CRUSH rule 3 x 186 [67,96]
+ CRUSH rule 3 x 187 [6,36]
+ CRUSH rule 3 x 188 [76,20]
+ CRUSH rule 3 x 189 [96,89]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,60]
+ CRUSH rule 3 x 192 [93,64]
+ CRUSH rule 3 x 193 [89,112]
+ CRUSH rule 3 x 194 [62,63]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [97,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,29]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,98]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,35]
+ CRUSH rule 3 x 207 [19,36]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,76]
+ CRUSH rule 3 x 211 [26,89]
+ CRUSH rule 3 x 212 [115,107]
+ CRUSH rule 3 x 213 [100,8]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,89]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,2]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,12]
+ CRUSH rule 3 x 222 [50,63]
+ CRUSH rule 3 x 223 [34,59]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,102]
+ CRUSH rule 3 x 226 [44,105]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,78]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,44]
+ CRUSH rule 3 x 233 [47,113]
+ CRUSH rule 3 x 234 [55,18]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,53]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,38]
+ CRUSH rule 3 x 242 [73,52]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,71]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,81]
+ CRUSH rule 3 x 250 [34,41]
+ CRUSH rule 3 x 251 [28,15]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,77]
+ CRUSH rule 3 x 256 [94,45]
+ CRUSH rule 3 x 257 [100,81]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,80]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,53]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,18]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,86]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,70]
+ CRUSH rule 3 x 271 [19,88]
+ CRUSH rule 3 x 272 [73,52]
+ CRUSH rule 3 x 273 [69,92]
+ CRUSH rule 3 x 274 [47,88]
+ CRUSH rule 3 x 275 [112,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,87]
+ CRUSH rule 3 x 278 [107,44]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,10]
+ CRUSH rule 3 x 281 [89,109]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,118]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,101]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,26]
+ CRUSH rule 3 x 290 [25,12]
+ CRUSH rule 3 x 291 [35,46]
+ CRUSH rule 3 x 292 [20,28]
+ CRUSH rule 3 x 293 [27,24]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,116]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,63]
+ CRUSH rule 3 x 298 [70,87]
+ CRUSH rule 3 x 299 [116,61]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,94]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,53]
+ CRUSH rule 3 x 306 [41,12]
+ CRUSH rule 3 x 307 [65,64]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,69]
+ CRUSH rule 3 x 311 [36,83]
+ CRUSH rule 3 x 312 [114,97]
+ CRUSH rule 3 x 313 [104,31]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,22]
+ CRUSH rule 3 x 316 [98,8]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,48]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,30]
+ CRUSH rule 3 x 330 [24,55]
+ CRUSH rule 3 x 331 [84,91]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,35]
+ CRUSH rule 3 x 335 [71,66]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,74]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,52]
+ CRUSH rule 3 x 341 [46,81]
+ CRUSH rule 3 x 342 [92,6]
+ CRUSH rule 3 x 343 [49,26]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,87]
+ CRUSH rule 3 x 346 [3,70]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,103]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,91]
+ CRUSH rule 3 x 353 [10,118]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,5]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,89]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,46]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,75]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,115]
+ CRUSH rule 3 x 366 [42,33]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,23]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,76]
+ CRUSH rule 3 x 378 [68,10]
+ CRUSH rule 3 x 379 [77,30]
+ CRUSH rule 3 x 380 [76,25]
+ CRUSH rule 3 x 381 [36,55]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,25]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,15]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,118]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,2]
+ CRUSH rule 3 x 393 [91,112]
+ CRUSH rule 3 x 394 [38,13]
+ CRUSH rule 3 x 395 [21,117]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,103]
+ CRUSH rule 3 x 400 [19,100]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,110]
+ CRUSH rule 3 x 403 [23,92]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,52]
+ CRUSH rule 3 x 407 [70,85]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,1]
+ CRUSH rule 3 x 410 [70,107]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,68]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,66]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,17]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,21]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,73]
+ CRUSH rule 3 x 427 [8,115]
+ CRUSH rule 3 x 428 [68,31]
+ CRUSH rule 3 x 429 [76,6]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,1]
+ CRUSH rule 3 x 434 [64,31]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,25]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,85]
+ CRUSH rule 3 x 439 [40,21]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,57]
+ CRUSH rule 3 x 442 [55,108]
+ CRUSH rule 3 x 443 [44,14]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,19]
+ CRUSH rule 3 x 449 [67,66]
+ CRUSH rule 3 x 450 [117,97]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,41]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,92]
+ CRUSH rule 3 x 456 [101,104]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,43]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,64]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,60]
+ CRUSH rule 3 x 469 [98,71]
+ CRUSH rule 3 x 470 [50,27]
+ CRUSH rule 3 x 471 [40,31]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,75]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,90]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,36]
+ CRUSH rule 3 x 485 [84,43]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,51]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,86]
+ CRUSH rule 3 x 493 [94,21]
+ CRUSH rule 3 x 494 [59,98]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,12]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,37]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,28]
+ CRUSH rule 3 x 503 [21,96]
+ CRUSH rule 3 x 504 [67,1]
+ CRUSH rule 3 x 505 [12,10]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,67]
+ CRUSH rule 3 x 508 [45,56]
+ CRUSH rule 3 x 509 [19,70]
+ CRUSH rule 3 x 510 [117,73]
+ CRUSH rule 3 x 511 [14,117]
+ CRUSH rule 3 x 512 [59,26]
+ CRUSH rule 3 x 513 [102,93]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,39]
+ CRUSH rule 3 x 516 [37,12]
+ CRUSH rule 3 x 517 [83,68]
+ CRUSH rule 3 x 518 [18,107]
+ CRUSH rule 3 x 519 [67,119]
+ CRUSH rule 3 x 520 [15,88]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,3]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,113]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,56]
+ CRUSH rule 3 x 528 [108,35]
+ CRUSH rule 3 x 529 [74,15]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,104]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,37]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,56]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,94]
+ CRUSH rule 3 x 541 [53,86]
+ CRUSH rule 3 x 542 [27,114]
+ CRUSH rule 3 x 543 [45,78]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,49]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,82]
+ CRUSH rule 3 x 549 [60,71]
+ CRUSH rule 3 x 550 [92,71]
+ CRUSH rule 3 x 551 [77,88]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,20]
+ CRUSH rule 3 x 556 [106,89]
+ CRUSH rule 3 x 557 [26,45]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,105]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,25]
+ CRUSH rule 3 x 563 [59,72]
+ CRUSH rule 3 x 564 [96,59]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,35]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,38]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,11]
+ CRUSH rule 3 x 574 [89,78]
+ CRUSH rule 3 x 575 [87,50]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,113]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,116]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,66]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,116]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,63]
+ CRUSH rule 3 x 591 [42,77]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,31]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,94]
+ CRUSH rule 3 x 598 [37,60]
+ CRUSH rule 3 x 599 [10,76]
+ CRUSH rule 3 x 600 [24,7]
+ CRUSH rule 3 x 601 [104,87]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,60]
+ CRUSH rule 3 x 604 [118,71]
+ CRUSH rule 3 x 605 [104,87]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,40]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,7]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [81,54]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,27]
+ CRUSH rule 3 x 620 [108,103]
+ CRUSH rule 3 x 621 [105,110]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,64]
+ CRUSH rule 3 x 624 [115,29]
+ CRUSH rule 3 x 625 [73,98]
+ CRUSH rule 3 x 626 [52,55]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,35]
+ CRUSH rule 3 x 629 [6,46]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,80]
+ CRUSH rule 3 x 632 [80,95]
+ CRUSH rule 3 x 633 [65,68]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,72]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,68]
+ CRUSH rule 3 x 638 [43,109]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,96]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,7]
+ CRUSH rule 3 x 644 [31,5]
+ CRUSH rule 3 x 645 [77,1]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,56]
+ CRUSH rule 3 x 648 [56,79]
+ CRUSH rule 3 x 649 [88,103]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,14]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [7,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,2]
+ CRUSH rule 3 x 660 [65,110]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,78]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,101]
+ CRUSH rule 3 x 667 [70,11]
+ CRUSH rule 3 x 668 [96,63]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,99]
+ CRUSH rule 3 x 671 [57,2]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,62]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,17]
+ CRUSH rule 3 x 676 [3,100]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,26]
+ CRUSH rule 3 x 680 [111,43]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,95]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,41]
+ CRUSH rule 3 x 690 [96,103]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,72]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,70]
+ CRUSH rule 3 x 695 [31,62]
+ CRUSH rule 3 x 696 [42,97]
+ CRUSH rule 3 x 697 [19,86]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,106]
+ CRUSH rule 3 x 700 [35,82]
+ CRUSH rule 3 x 701 [53,30]
+ CRUSH rule 3 x 702 [101,32]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,8]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,42]
+ CRUSH rule 3 x 708 [95,84]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,23]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,26]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,21]
+ CRUSH rule 3 x 716 [101,72]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,96]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,58]
+ CRUSH rule 3 x 722 [15,80]
+ CRUSH rule 3 x 723 [117,41]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,116]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,42]
+ CRUSH rule 3 x 730 [28,47]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,89]
+ CRUSH rule 3 x 733 [35,62]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,73]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,90]
+ CRUSH rule 3 x 742 [106,31]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,89]
+ CRUSH rule 3 x 749 [102,71]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,74]
+ CRUSH rule 3 x 752 [82,83]
+ CRUSH rule 3 x 753 [14,32]
+ CRUSH rule 3 x 754 [114,57]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,83]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,29]
+ CRUSH rule 3 x 760 [88,105]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,41]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,116]
+ CRUSH rule 3 x 768 [106,71]
+ CRUSH rule 3 x 769 [91,48]
+ CRUSH rule 3 x 770 [72,43]
+ CRUSH rule 3 x 771 [115,97]
+ CRUSH rule 3 x 772 [97,111]
+ CRUSH rule 3 x 773 [116,75]
+ CRUSH rule 3 x 774 [100,43]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,118]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,40]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,58]
+ CRUSH rule 3 x 787 [108,16]
+ CRUSH rule 3 x 788 [74,6]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,97]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,90]
+ CRUSH rule 3 x 797 [64,63]
+ CRUSH rule 3 x 798 [42,91]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,49]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,5]
+ CRUSH rule 3 x 803 [57,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,63]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,59]
+ CRUSH rule 3 x 809 [27,26]
+ CRUSH rule 3 x 810 [119,107]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,52]
+ CRUSH rule 3 x 813 [81,72]
+ CRUSH rule 3 x 814 [95,32]
+ CRUSH rule 3 x 815 [84,15]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,15]
+ CRUSH rule 3 x 820 [104,49]
+ CRUSH rule 3 x 821 [58,85]
+ CRUSH rule 3 x 822 [20,98]
+ CRUSH rule 3 x 823 [63,90]
+ CRUSH rule 3 x 824 [102,81]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,3]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,3]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,46]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,17]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,109]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,57]
+ CRUSH rule 3 x 845 [74,63]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,117]
+ CRUSH rule 3 x 852 [60,27]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,84]
+ CRUSH rule 3 x 855 [2,105]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,79]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,54]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,68]
+ CRUSH rule 3 x 865 [119,14]
+ CRUSH rule 3 x 866 [18,89]
+ CRUSH rule 3 x 867 [3,78]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,21]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,112]
+ CRUSH rule 3 x 874 [21,44]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,75]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,96]
+ CRUSH rule 3 x 881 [109,69]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,51]
+ CRUSH rule 3 x 884 [80,103]
+ CRUSH rule 3 x 885 [46,7]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,40]
+ CRUSH rule 3 x 889 [84,93]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,105]
+ CRUSH rule 3 x 892 [64,87]
+ CRUSH rule 3 x 893 [20,115]
+ CRUSH rule 3 x 894 [32,3]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,93]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,80]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,104]
+ CRUSH rule 3 x 903 [53,64]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,5]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,86]
+ CRUSH rule 3 x 911 [39,84]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,12]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,54]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,86]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,23]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,33]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,80]
+ CRUSH rule 3 x 930 [104,61]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,42]
+ CRUSH rule 3 x 933 [18,63]
+ CRUSH rule 3 x 934 [68,51]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,11]
+ CRUSH rule 3 x 938 [48,73]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,22]
+ CRUSH rule 3 x 942 [80,8]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,7]
+ CRUSH rule 3 x 945 [71,56]
+ CRUSH rule 3 x 946 [37,114]
+ CRUSH rule 3 x 947 [107,74]
+ CRUSH rule 3 x 948 [108,79]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,87]
+ CRUSH rule 3 x 953 [62,105]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,91]
+ CRUSH rule 3 x 957 [117,16]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,107]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,92]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,108]
+ CRUSH rule 3 x 968 [74,6]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,10]
+ CRUSH rule 3 x 972 [3,58]
+ CRUSH rule 3 x 973 [113,81]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,117]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [119,93]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,26]
+ CRUSH rule 3 x 981 [89,117]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,21]
+ CRUSH rule 3 x 984 [78,15]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,103]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,86]
+ CRUSH rule 3 x 990 [72,35]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,53]
+ CRUSH rule 3 x 994 [74,20]
+ CRUSH rule 3 x 995 [100,17]
+ CRUSH rule 3 x 996 [41,30]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,65]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,36]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,118]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,113]
+ CRUSH rule 3 x 1008 [31,36]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,34]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,71]
+ CRUSH rule 3 x 1013 [5,39]
+ CRUSH rule 3 x 1014 [33,80]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,109]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,109]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,108]
+ CRUSH rule 3 x 1022 [73,106]
+ CRUSH rule 3 x 1023 [88,89]
+ rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,45]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,118]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,96]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,70]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,76]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,100]
+ CRUSH rule 3 x 17 [85,110]
+ CRUSH rule 3 x 18 [11,119]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,47]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,90]
+ CRUSH rule 3 x 24 [83,38]
+ CRUSH rule 3 x 25 [81,5]
+ CRUSH rule 3 x 26 [17,100]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,119]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,69]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,52]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,29]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,11]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,68]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,21]
+ CRUSH rule 3 x 47 [106,53]
+ CRUSH rule 3 x 48 [34,33]
+ CRUSH rule 3 x 49 [99,104]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,52]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,75]
+ CRUSH rule 3 x 54 [28,93]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,40]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,33]
+ CRUSH rule 3 x 61 [88,3]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,113]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,61]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,7]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,44]
+ CRUSH rule 3 x 74 [29,74]
+ CRUSH rule 3 x 75 [60,89]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,7]
+ CRUSH rule 3 x 79 [64,41]
+ CRUSH rule 3 x 80 [73,104]
+ CRUSH rule 3 x 81 [64,19]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,68]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,95]
+ CRUSH rule 3 x 89 [76,3]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,65]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,23]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,100]
+ CRUSH rule 3 x 99 [78,22]
+ CRUSH rule 3 x 100 [28,63]
+ CRUSH rule 3 x 101 [91,36]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,95]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,116]
+ CRUSH rule 3 x 107 [1,55]
+ CRUSH rule 3 x 108 [7,72]
+ CRUSH rule 3 x 109 [112,63]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,78]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,54]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,42]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,16]
+ CRUSH rule 3 x 122 [45,110]
+ CRUSH rule 3 x 123 [22,86]
+ CRUSH rule 3 x 124 [97,72]
+ CRUSH rule 3 x 125 [66,71]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,97]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,110]
+ CRUSH rule 3 x 130 [50,63]
+ CRUSH rule 3 x 131 [44,25]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,2]
+ CRUSH rule 3 x 134 [37,94]
+ CRUSH rule 3 x 135 [78,17]
+ CRUSH rule 3 x 136 [32,91]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,62]
+ CRUSH rule 3 x 141 [89,96]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,87]
+ CRUSH rule 3 x 144 [13,86]
+ CRUSH rule 3 x 145 [77,60]
+ CRUSH rule 3 x 146 [12,53]
+ CRUSH rule 3 x 147 [2,59]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,58]
+ CRUSH rule 3 x 150 [14,54]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,33]
+ CRUSH rule 3 x 154 [19,111]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,5]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,58]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,31]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,47]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,107]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,34]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,2]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,1]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,90]
+ CRUSH rule 3 x 183 [11,74]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,113]
+ CRUSH rule 3 x 186 [67,96]
+ CRUSH rule 3 x 187 [6,36]
+ CRUSH rule 3 x 188 [76,20]
+ CRUSH rule 3 x 189 [96,89]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,60]
+ CRUSH rule 3 x 192 [93,64]
+ CRUSH rule 3 x 193 [89,112]
+ CRUSH rule 3 x 194 [62,63]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [97,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,29]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,98]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,35]
+ CRUSH rule 3 x 207 [19,36]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,76]
+ CRUSH rule 3 x 211 [26,89]
+ CRUSH rule 3 x 212 [115,107]
+ CRUSH rule 3 x 213 [100,8]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,89]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,2]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,12]
+ CRUSH rule 3 x 222 [50,63]
+ CRUSH rule 3 x 223 [34,59]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,102]
+ CRUSH rule 3 x 226 [44,105]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,78]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,44]
+ CRUSH rule 3 x 233 [47,113]
+ CRUSH rule 3 x 234 [55,18]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,53]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,38]
+ CRUSH rule 3 x 242 [73,52]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,71]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,81]
+ CRUSH rule 3 x 250 [34,41]
+ CRUSH rule 3 x 251 [28,15]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,77]
+ CRUSH rule 3 x 256 [94,45]
+ CRUSH rule 3 x 257 [100,81]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,80]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,53]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,18]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,86]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,70]
+ CRUSH rule 3 x 271 [19,88]
+ CRUSH rule 3 x 272 [73,52]
+ CRUSH rule 3 x 273 [69,92]
+ CRUSH rule 3 x 274 [47,88]
+ CRUSH rule 3 x 275 [112,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,87]
+ CRUSH rule 3 x 278 [107,44]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,10]
+ CRUSH rule 3 x 281 [89,109]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,118]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,101]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,26]
+ CRUSH rule 3 x 290 [25,12]
+ CRUSH rule 3 x 291 [35,46]
+ CRUSH rule 3 x 292 [20,28]
+ CRUSH rule 3 x 293 [27,24]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,116]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,63]
+ CRUSH rule 3 x 298 [70,87]
+ CRUSH rule 3 x 299 [116,61]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,94]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,53]
+ CRUSH rule 3 x 306 [41,12]
+ CRUSH rule 3 x 307 [65,64]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,69]
+ CRUSH rule 3 x 311 [36,83]
+ CRUSH rule 3 x 312 [114,97]
+ CRUSH rule 3 x 313 [104,31]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,22]
+ CRUSH rule 3 x 316 [98,8]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,48]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,30]
+ CRUSH rule 3 x 330 [24,55]
+ CRUSH rule 3 x 331 [84,91]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,35]
+ CRUSH rule 3 x 335 [71,66]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,74]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,52]
+ CRUSH rule 3 x 341 [46,81]
+ CRUSH rule 3 x 342 [92,6]
+ CRUSH rule 3 x 343 [49,26]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,87]
+ CRUSH rule 3 x 346 [3,70]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,103]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,91]
+ CRUSH rule 3 x 353 [10,118]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,5]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,89]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,46]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,75]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,115]
+ CRUSH rule 3 x 366 [42,33]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,23]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,76]
+ CRUSH rule 3 x 378 [68,10]
+ CRUSH rule 3 x 379 [77,30]
+ CRUSH rule 3 x 380 [76,25]
+ CRUSH rule 3 x 381 [36,55]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,25]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,15]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,118]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,2]
+ CRUSH rule 3 x 393 [91,112]
+ CRUSH rule 3 x 394 [38,13]
+ CRUSH rule 3 x 395 [21,117]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,103]
+ CRUSH rule 3 x 400 [19,100]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,110]
+ CRUSH rule 3 x 403 [23,92]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,52]
+ CRUSH rule 3 x 407 [70,85]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,1]
+ CRUSH rule 3 x 410 [70,107]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,68]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,66]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,17]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,21]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,73]
+ CRUSH rule 3 x 427 [8,115]
+ CRUSH rule 3 x 428 [68,31]
+ CRUSH rule 3 x 429 [76,6]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,1]
+ CRUSH rule 3 x 434 [64,31]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,25]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,85]
+ CRUSH rule 3 x 439 [40,21]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,57]
+ CRUSH rule 3 x 442 [55,108]
+ CRUSH rule 3 x 443 [44,14]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,19]
+ CRUSH rule 3 x 449 [67,66]
+ CRUSH rule 3 x 450 [117,97]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,41]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,92]
+ CRUSH rule 3 x 456 [101,104]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,43]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,64]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,60]
+ CRUSH rule 3 x 469 [98,71]
+ CRUSH rule 3 x 470 [50,27]
+ CRUSH rule 3 x 471 [40,31]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,75]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,90]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,36]
+ CRUSH rule 3 x 485 [84,43]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,51]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,86]
+ CRUSH rule 3 x 493 [94,21]
+ CRUSH rule 3 x 494 [59,98]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,12]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,37]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,28]
+ CRUSH rule 3 x 503 [21,96]
+ CRUSH rule 3 x 504 [67,1]
+ CRUSH rule 3 x 505 [12,10]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,67]
+ CRUSH rule 3 x 508 [45,56]
+ CRUSH rule 3 x 509 [19,70]
+ CRUSH rule 3 x 510 [117,73]
+ CRUSH rule 3 x 511 [14,117]
+ CRUSH rule 3 x 512 [59,26]
+ CRUSH rule 3 x 513 [102,93]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,39]
+ CRUSH rule 3 x 516 [37,12]
+ CRUSH rule 3 x 517 [83,68]
+ CRUSH rule 3 x 518 [18,107]
+ CRUSH rule 3 x 519 [67,119]
+ CRUSH rule 3 x 520 [15,88]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,3]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,113]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,56]
+ CRUSH rule 3 x 528 [108,35]
+ CRUSH rule 3 x 529 [74,15]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,104]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,37]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,56]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,94]
+ CRUSH rule 3 x 541 [53,86]
+ CRUSH rule 3 x 542 [27,114]
+ CRUSH rule 3 x 543 [45,78]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,49]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,82]
+ CRUSH rule 3 x 549 [60,71]
+ CRUSH rule 3 x 550 [92,71]
+ CRUSH rule 3 x 551 [77,88]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,20]
+ CRUSH rule 3 x 556 [106,89]
+ CRUSH rule 3 x 557 [26,45]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,105]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,25]
+ CRUSH rule 3 x 563 [59,72]
+ CRUSH rule 3 x 564 [96,59]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,35]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,38]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,11]
+ CRUSH rule 3 x 574 [89,78]
+ CRUSH rule 3 x 575 [87,50]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,113]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,116]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,66]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,116]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,63]
+ CRUSH rule 3 x 591 [42,77]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,31]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,94]
+ CRUSH rule 3 x 598 [37,60]
+ CRUSH rule 3 x 599 [10,76]
+ CRUSH rule 3 x 600 [24,7]
+ CRUSH rule 3 x 601 [104,87]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,60]
+ CRUSH rule 3 x 604 [118,71]
+ CRUSH rule 3 x 605 [104,87]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,40]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,7]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [81,54]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,27]
+ CRUSH rule 3 x 620 [108,103]
+ CRUSH rule 3 x 621 [105,110]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,64]
+ CRUSH rule 3 x 624 [115,29]
+ CRUSH rule 3 x 625 [73,98]
+ CRUSH rule 3 x 626 [52,55]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,35]
+ CRUSH rule 3 x 629 [6,46]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,80]
+ CRUSH rule 3 x 632 [80,95]
+ CRUSH rule 3 x 633 [65,68]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,72]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,68]
+ CRUSH rule 3 x 638 [43,109]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,96]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,7]
+ CRUSH rule 3 x 644 [31,5]
+ CRUSH rule 3 x 645 [77,1]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,56]
+ CRUSH rule 3 x 648 [56,79]
+ CRUSH rule 3 x 649 [88,103]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,14]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [7,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,2]
+ CRUSH rule 3 x 660 [65,110]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,78]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,101]
+ CRUSH rule 3 x 667 [70,11]
+ CRUSH rule 3 x 668 [96,63]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,99]
+ CRUSH rule 3 x 671 [57,2]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,62]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,17]
+ CRUSH rule 3 x 676 [3,100]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,26]
+ CRUSH rule 3 x 680 [111,43]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,95]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,41]
+ CRUSH rule 3 x 690 [96,103]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,72]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,70]
+ CRUSH rule 3 x 695 [31,62]
+ CRUSH rule 3 x 696 [42,97]
+ CRUSH rule 3 x 697 [19,86]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,106]
+ CRUSH rule 3 x 700 [35,82]
+ CRUSH rule 3 x 701 [53,30]
+ CRUSH rule 3 x 702 [101,32]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,8]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,42]
+ CRUSH rule 3 x 708 [95,84]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,23]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,26]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,21]
+ CRUSH rule 3 x 716 [101,72]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,96]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,58]
+ CRUSH rule 3 x 722 [15,80]
+ CRUSH rule 3 x 723 [117,41]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,116]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,42]
+ CRUSH rule 3 x 730 [28,47]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,89]
+ CRUSH rule 3 x 733 [35,62]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,73]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,90]
+ CRUSH rule 3 x 742 [106,31]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,89]
+ CRUSH rule 3 x 749 [102,71]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,74]
+ CRUSH rule 3 x 752 [82,83]
+ CRUSH rule 3 x 753 [14,32]
+ CRUSH rule 3 x 754 [114,57]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,83]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,29]
+ CRUSH rule 3 x 760 [88,105]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,41]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,116]
+ CRUSH rule 3 x 768 [106,71]
+ CRUSH rule 3 x 769 [91,48]
+ CRUSH rule 3 x 770 [72,43]
+ CRUSH rule 3 x 771 [115,97]
+ CRUSH rule 3 x 772 [97,111]
+ CRUSH rule 3 x 773 [116,75]
+ CRUSH rule 3 x 774 [100,43]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,118]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,40]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,58]
+ CRUSH rule 3 x 787 [108,16]
+ CRUSH rule 3 x 788 [74,6]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,97]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,90]
+ CRUSH rule 3 x 797 [64,63]
+ CRUSH rule 3 x 798 [42,91]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,49]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,5]
+ CRUSH rule 3 x 803 [57,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,63]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,59]
+ CRUSH rule 3 x 809 [27,26]
+ CRUSH rule 3 x 810 [119,107]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,52]
+ CRUSH rule 3 x 813 [81,72]
+ CRUSH rule 3 x 814 [95,32]
+ CRUSH rule 3 x 815 [84,15]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,15]
+ CRUSH rule 3 x 820 [104,49]
+ CRUSH rule 3 x 821 [58,85]
+ CRUSH rule 3 x 822 [20,98]
+ CRUSH rule 3 x 823 [63,90]
+ CRUSH rule 3 x 824 [102,81]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,3]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,3]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,46]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,17]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,109]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,57]
+ CRUSH rule 3 x 845 [74,63]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,117]
+ CRUSH rule 3 x 852 [60,27]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,84]
+ CRUSH rule 3 x 855 [2,105]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,79]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,54]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,68]
+ CRUSH rule 3 x 865 [119,14]
+ CRUSH rule 3 x 866 [18,89]
+ CRUSH rule 3 x 867 [3,78]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,21]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,112]
+ CRUSH rule 3 x 874 [21,44]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,75]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,96]
+ CRUSH rule 3 x 881 [109,69]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,51]
+ CRUSH rule 3 x 884 [80,103]
+ CRUSH rule 3 x 885 [46,7]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,40]
+ CRUSH rule 3 x 889 [84,93]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,105]
+ CRUSH rule 3 x 892 [64,87]
+ CRUSH rule 3 x 893 [20,115]
+ CRUSH rule 3 x 894 [32,3]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,93]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,80]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,104]
+ CRUSH rule 3 x 903 [53,64]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,5]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,86]
+ CRUSH rule 3 x 911 [39,84]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,12]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,54]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,86]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,23]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,33]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,80]
+ CRUSH rule 3 x 930 [104,61]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,42]
+ CRUSH rule 3 x 933 [18,63]
+ CRUSH rule 3 x 934 [68,51]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,11]
+ CRUSH rule 3 x 938 [48,73]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,22]
+ CRUSH rule 3 x 942 [80,8]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,7]
+ CRUSH rule 3 x 945 [71,56]
+ CRUSH rule 3 x 946 [37,114]
+ CRUSH rule 3 x 947 [107,74]
+ CRUSH rule 3 x 948 [108,79]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,87]
+ CRUSH rule 3 x 953 [62,105]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,91]
+ CRUSH rule 3 x 957 [117,16]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,107]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,92]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,108]
+ CRUSH rule 3 x 968 [74,6]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,10]
+ CRUSH rule 3 x 972 [3,58]
+ CRUSH rule 3 x 973 [113,81]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,117]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [119,93]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,26]
+ CRUSH rule 3 x 981 [89,117]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,21]
+ CRUSH rule 3 x 984 [78,15]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,103]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,86]
+ CRUSH rule 3 x 990 [72,35]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,53]
+ CRUSH rule 3 x 994 [74,20]
+ CRUSH rule 3 x 995 [100,17]
+ CRUSH rule 3 x 996 [41,30]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,65]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,36]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,118]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,113]
+ CRUSH rule 3 x 1008 [31,36]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,34]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,71]
+ CRUSH rule 3 x 1013 [5,39]
+ CRUSH rule 3 x 1014 [33,80]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,109]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,109]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,108]
+ CRUSH rule 3 x 1022 [73,106]
+ CRUSH rule 3 x 1023 [88,89]
+ rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,45]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,118]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,96]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,70]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,76]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,100]
+ CRUSH rule 3 x 17 [85,110]
+ CRUSH rule 3 x 18 [11,119]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,47]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,90]
+ CRUSH rule 3 x 24 [83,38]
+ CRUSH rule 3 x 25 [81,5]
+ CRUSH rule 3 x 26 [17,100]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,119]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,69]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,52]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,29]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,11]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,68]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,21]
+ CRUSH rule 3 x 47 [106,53]
+ CRUSH rule 3 x 48 [34,33]
+ CRUSH rule 3 x 49 [99,104]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,52]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,75]
+ CRUSH rule 3 x 54 [28,93]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,40]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,33]
+ CRUSH rule 3 x 61 [88,3]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,113]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,61]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,7]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,44]
+ CRUSH rule 3 x 74 [29,74]
+ CRUSH rule 3 x 75 [60,89]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,7]
+ CRUSH rule 3 x 79 [64,41]
+ CRUSH rule 3 x 80 [73,104]
+ CRUSH rule 3 x 81 [64,19]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,68]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,95]
+ CRUSH rule 3 x 89 [76,3]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,65]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,23]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,100]
+ CRUSH rule 3 x 99 [78,22]
+ CRUSH rule 3 x 100 [28,63]
+ CRUSH rule 3 x 101 [91,36]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,95]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,116]
+ CRUSH rule 3 x 107 [1,55]
+ CRUSH rule 3 x 108 [7,72]
+ CRUSH rule 3 x 109 [112,63]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,78]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,54]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,42]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,16]
+ CRUSH rule 3 x 122 [45,110]
+ CRUSH rule 3 x 123 [22,86]
+ CRUSH rule 3 x 124 [97,72]
+ CRUSH rule 3 x 125 [66,71]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,97]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,110]
+ CRUSH rule 3 x 130 [50,63]
+ CRUSH rule 3 x 131 [44,25]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,2]
+ CRUSH rule 3 x 134 [37,94]
+ CRUSH rule 3 x 135 [78,17]
+ CRUSH rule 3 x 136 [32,91]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,62]
+ CRUSH rule 3 x 141 [89,96]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,87]
+ CRUSH rule 3 x 144 [13,86]
+ CRUSH rule 3 x 145 [77,60]
+ CRUSH rule 3 x 146 [12,53]
+ CRUSH rule 3 x 147 [2,59]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,58]
+ CRUSH rule 3 x 150 [14,54]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,33]
+ CRUSH rule 3 x 154 [19,111]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,5]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,58]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,31]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,47]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,107]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,34]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,2]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,1]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,90]
+ CRUSH rule 3 x 183 [11,74]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,113]
+ CRUSH rule 3 x 186 [67,96]
+ CRUSH rule 3 x 187 [6,36]
+ CRUSH rule 3 x 188 [76,20]
+ CRUSH rule 3 x 189 [96,89]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,60]
+ CRUSH rule 3 x 192 [93,64]
+ CRUSH rule 3 x 193 [89,112]
+ CRUSH rule 3 x 194 [62,63]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [97,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,29]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,98]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,35]
+ CRUSH rule 3 x 207 [19,36]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,76]
+ CRUSH rule 3 x 211 [26,89]
+ CRUSH rule 3 x 212 [115,107]
+ CRUSH rule 3 x 213 [100,8]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,89]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,2]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,12]
+ CRUSH rule 3 x 222 [50,63]
+ CRUSH rule 3 x 223 [34,59]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,102]
+ CRUSH rule 3 x 226 [44,105]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,78]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,44]
+ CRUSH rule 3 x 233 [47,113]
+ CRUSH rule 3 x 234 [55,18]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,53]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,38]
+ CRUSH rule 3 x 242 [73,52]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,71]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,81]
+ CRUSH rule 3 x 250 [34,41]
+ CRUSH rule 3 x 251 [28,15]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,77]
+ CRUSH rule 3 x 256 [94,45]
+ CRUSH rule 3 x 257 [100,81]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,80]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,53]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,18]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,86]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,70]
+ CRUSH rule 3 x 271 [19,88]
+ CRUSH rule 3 x 272 [73,52]
+ CRUSH rule 3 x 273 [69,92]
+ CRUSH rule 3 x 274 [47,88]
+ CRUSH rule 3 x 275 [112,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,87]
+ CRUSH rule 3 x 278 [107,44]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,10]
+ CRUSH rule 3 x 281 [89,109]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,118]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,101]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,26]
+ CRUSH rule 3 x 290 [25,12]
+ CRUSH rule 3 x 291 [35,46]
+ CRUSH rule 3 x 292 [20,28]
+ CRUSH rule 3 x 293 [27,24]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,116]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,63]
+ CRUSH rule 3 x 298 [70,87]
+ CRUSH rule 3 x 299 [116,61]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,94]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,53]
+ CRUSH rule 3 x 306 [41,12]
+ CRUSH rule 3 x 307 [65,64]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,69]
+ CRUSH rule 3 x 311 [36,83]
+ CRUSH rule 3 x 312 [114,97]
+ CRUSH rule 3 x 313 [104,31]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,22]
+ CRUSH rule 3 x 316 [98,8]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,48]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,30]
+ CRUSH rule 3 x 330 [24,55]
+ CRUSH rule 3 x 331 [84,91]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,35]
+ CRUSH rule 3 x 335 [71,66]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,74]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,52]
+ CRUSH rule 3 x 341 [46,81]
+ CRUSH rule 3 x 342 [92,6]
+ CRUSH rule 3 x 343 [49,26]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,87]
+ CRUSH rule 3 x 346 [3,70]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,103]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,91]
+ CRUSH rule 3 x 353 [10,118]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,5]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,89]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,46]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,75]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,115]
+ CRUSH rule 3 x 366 [42,33]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,23]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,76]
+ CRUSH rule 3 x 378 [68,10]
+ CRUSH rule 3 x 379 [77,30]
+ CRUSH rule 3 x 380 [76,25]
+ CRUSH rule 3 x 381 [36,55]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,25]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,15]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,118]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,2]
+ CRUSH rule 3 x 393 [91,112]
+ CRUSH rule 3 x 394 [38,13]
+ CRUSH rule 3 x 395 [21,117]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,103]
+ CRUSH rule 3 x 400 [19,100]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,110]
+ CRUSH rule 3 x 403 [23,92]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,52]
+ CRUSH rule 3 x 407 [70,85]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,1]
+ CRUSH rule 3 x 410 [70,107]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,68]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,66]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,17]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,21]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,73]
+ CRUSH rule 3 x 427 [8,115]
+ CRUSH rule 3 x 428 [68,31]
+ CRUSH rule 3 x 429 [76,6]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,1]
+ CRUSH rule 3 x 434 [64,31]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,25]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,85]
+ CRUSH rule 3 x 439 [40,21]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,57]
+ CRUSH rule 3 x 442 [55,108]
+ CRUSH rule 3 x 443 [44,14]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,19]
+ CRUSH rule 3 x 449 [67,66]
+ CRUSH rule 3 x 450 [117,97]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,41]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,92]
+ CRUSH rule 3 x 456 [101,104]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,43]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,64]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,60]
+ CRUSH rule 3 x 469 [98,71]
+ CRUSH rule 3 x 470 [50,27]
+ CRUSH rule 3 x 471 [40,31]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,75]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,90]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,36]
+ CRUSH rule 3 x 485 [84,43]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,51]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,86]
+ CRUSH rule 3 x 493 [94,21]
+ CRUSH rule 3 x 494 [59,98]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,12]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,37]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,28]
+ CRUSH rule 3 x 503 [21,96]
+ CRUSH rule 3 x 504 [67,1]
+ CRUSH rule 3 x 505 [12,10]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,67]
+ CRUSH rule 3 x 508 [45,56]
+ CRUSH rule 3 x 509 [19,70]
+ CRUSH rule 3 x 510 [117,73]
+ CRUSH rule 3 x 511 [14,117]
+ CRUSH rule 3 x 512 [59,26]
+ CRUSH rule 3 x 513 [102,93]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,39]
+ CRUSH rule 3 x 516 [37,12]
+ CRUSH rule 3 x 517 [83,68]
+ CRUSH rule 3 x 518 [18,107]
+ CRUSH rule 3 x 519 [67,119]
+ CRUSH rule 3 x 520 [15,88]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,3]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,113]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,56]
+ CRUSH rule 3 x 528 [108,35]
+ CRUSH rule 3 x 529 [74,15]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,104]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,37]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,56]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,94]
+ CRUSH rule 3 x 541 [53,86]
+ CRUSH rule 3 x 542 [27,114]
+ CRUSH rule 3 x 543 [45,78]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,49]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,82]
+ CRUSH rule 3 x 549 [60,71]
+ CRUSH rule 3 x 550 [92,71]
+ CRUSH rule 3 x 551 [77,88]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,20]
+ CRUSH rule 3 x 556 [106,89]
+ CRUSH rule 3 x 557 [26,45]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,105]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,25]
+ CRUSH rule 3 x 563 [59,72]
+ CRUSH rule 3 x 564 [96,59]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,35]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,38]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,11]
+ CRUSH rule 3 x 574 [89,78]
+ CRUSH rule 3 x 575 [87,50]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,113]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,116]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,66]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,116]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,63]
+ CRUSH rule 3 x 591 [42,77]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,31]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,94]
+ CRUSH rule 3 x 598 [37,60]
+ CRUSH rule 3 x 599 [10,76]
+ CRUSH rule 3 x 600 [24,7]
+ CRUSH rule 3 x 601 [104,87]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,60]
+ CRUSH rule 3 x 604 [118,71]
+ CRUSH rule 3 x 605 [104,87]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,40]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,7]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [81,54]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,27]
+ CRUSH rule 3 x 620 [108,103]
+ CRUSH rule 3 x 621 [105,110]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,64]
+ CRUSH rule 3 x 624 [115,29]
+ CRUSH rule 3 x 625 [73,98]
+ CRUSH rule 3 x 626 [52,55]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,35]
+ CRUSH rule 3 x 629 [6,46]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,80]
+ CRUSH rule 3 x 632 [80,95]
+ CRUSH rule 3 x 633 [65,68]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,72]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,68]
+ CRUSH rule 3 x 638 [43,109]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,96]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,7]
+ CRUSH rule 3 x 644 [31,5]
+ CRUSH rule 3 x 645 [77,1]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,56]
+ CRUSH rule 3 x 648 [56,79]
+ CRUSH rule 3 x 649 [88,103]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,14]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [7,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,2]
+ CRUSH rule 3 x 660 [65,110]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,78]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,101]
+ CRUSH rule 3 x 667 [70,11]
+ CRUSH rule 3 x 668 [96,63]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,99]
+ CRUSH rule 3 x 671 [57,2]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,62]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,17]
+ CRUSH rule 3 x 676 [3,100]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,26]
+ CRUSH rule 3 x 680 [111,43]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,95]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,41]
+ CRUSH rule 3 x 690 [96,103]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,72]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,70]
+ CRUSH rule 3 x 695 [31,62]
+ CRUSH rule 3 x 696 [42,97]
+ CRUSH rule 3 x 697 [19,86]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,106]
+ CRUSH rule 3 x 700 [35,82]
+ CRUSH rule 3 x 701 [53,30]
+ CRUSH rule 3 x 702 [101,32]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,8]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,42]
+ CRUSH rule 3 x 708 [95,84]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,23]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,26]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,21]
+ CRUSH rule 3 x 716 [101,72]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,96]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,58]
+ CRUSH rule 3 x 722 [15,80]
+ CRUSH rule 3 x 723 [117,41]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,116]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,42]
+ CRUSH rule 3 x 730 [28,47]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,89]
+ CRUSH rule 3 x 733 [35,62]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,73]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,90]
+ CRUSH rule 3 x 742 [106,31]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,89]
+ CRUSH rule 3 x 749 [102,71]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,74]
+ CRUSH rule 3 x 752 [82,83]
+ CRUSH rule 3 x 753 [14,32]
+ CRUSH rule 3 x 754 [114,57]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,83]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,29]
+ CRUSH rule 3 x 760 [88,105]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,41]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,116]
+ CRUSH rule 3 x 768 [106,71]
+ CRUSH rule 3 x 769 [91,48]
+ CRUSH rule 3 x 770 [72,43]
+ CRUSH rule 3 x 771 [115,97]
+ CRUSH rule 3 x 772 [97,111]
+ CRUSH rule 3 x 773 [116,75]
+ CRUSH rule 3 x 774 [100,43]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,118]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,40]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,58]
+ CRUSH rule 3 x 787 [108,16]
+ CRUSH rule 3 x 788 [74,6]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,97]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,90]
+ CRUSH rule 3 x 797 [64,63]
+ CRUSH rule 3 x 798 [42,91]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,49]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,5]
+ CRUSH rule 3 x 803 [57,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,63]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,59]
+ CRUSH rule 3 x 809 [27,26]
+ CRUSH rule 3 x 810 [119,107]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,52]
+ CRUSH rule 3 x 813 [81,72]
+ CRUSH rule 3 x 814 [95,32]
+ CRUSH rule 3 x 815 [84,15]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,15]
+ CRUSH rule 3 x 820 [104,49]
+ CRUSH rule 3 x 821 [58,85]
+ CRUSH rule 3 x 822 [20,98]
+ CRUSH rule 3 x 823 [63,90]
+ CRUSH rule 3 x 824 [102,81]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,3]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,3]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,46]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,17]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,109]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,57]
+ CRUSH rule 3 x 845 [74,63]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,117]
+ CRUSH rule 3 x 852 [60,27]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,84]
+ CRUSH rule 3 x 855 [2,105]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,79]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,54]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,68]
+ CRUSH rule 3 x 865 [119,14]
+ CRUSH rule 3 x 866 [18,89]
+ CRUSH rule 3 x 867 [3,78]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,21]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,112]
+ CRUSH rule 3 x 874 [21,44]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,75]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,96]
+ CRUSH rule 3 x 881 [109,69]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,51]
+ CRUSH rule 3 x 884 [80,103]
+ CRUSH rule 3 x 885 [46,7]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,40]
+ CRUSH rule 3 x 889 [84,93]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,105]
+ CRUSH rule 3 x 892 [64,87]
+ CRUSH rule 3 x 893 [20,115]
+ CRUSH rule 3 x 894 [32,3]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,93]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,80]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,104]
+ CRUSH rule 3 x 903 [53,64]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,5]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,86]
+ CRUSH rule 3 x 911 [39,84]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,12]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,54]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,86]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,23]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,33]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,80]
+ CRUSH rule 3 x 930 [104,61]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,42]
+ CRUSH rule 3 x 933 [18,63]
+ CRUSH rule 3 x 934 [68,51]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,11]
+ CRUSH rule 3 x 938 [48,73]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,22]
+ CRUSH rule 3 x 942 [80,8]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,7]
+ CRUSH rule 3 x 945 [71,56]
+ CRUSH rule 3 x 946 [37,114]
+ CRUSH rule 3 x 947 [107,74]
+ CRUSH rule 3 x 948 [108,79]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,87]
+ CRUSH rule 3 x 953 [62,105]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,91]
+ CRUSH rule 3 x 957 [117,16]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,107]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,92]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,108]
+ CRUSH rule 3 x 968 [74,6]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,10]
+ CRUSH rule 3 x 972 [3,58]
+ CRUSH rule 3 x 973 [113,81]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,117]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [119,93]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,26]
+ CRUSH rule 3 x 981 [89,117]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,21]
+ CRUSH rule 3 x 984 [78,15]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,103]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,86]
+ CRUSH rule 3 x 990 [72,35]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,53]
+ CRUSH rule 3 x 994 [74,20]
+ CRUSH rule 3 x 995 [100,17]
+ CRUSH rule 3 x 996 [41,30]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,65]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,36]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,118]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,113]
+ CRUSH rule 3 x 1008 [31,36]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,34]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,71]
+ CRUSH rule 3 x 1013 [5,39]
+ CRUSH rule 3 x 1014 [33,80]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,109]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,109]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,108]
+ CRUSH rule 3 x 1022 [73,106]
+ CRUSH rule 3 x 1023 [88,89]
+ rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/src/test/cli/crushtool/test-map-vary-r-3.t b/src/test/cli/crushtool/test-map-vary-r-3.t
new file mode 100644
index 0000000..ad02e73
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-vary-r-3.t
@@ -0,0 +1,3078 @@
+ $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 3 --weight 0 0 --weight 4 0 --weight 9 0
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,62]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,60]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,48]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,111]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,20]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,85]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,12]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,93]
+ CRUSH rule 3 x 54 [28,3]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,91]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,92]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,16]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,89]
+ CRUSH rule 3 x 79 [64,75]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,57]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,117]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,2]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,55]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,29]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,119]
+ CRUSH rule 3 x 146 [12,73]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,8]
+ CRUSH rule 3 x 154 [19,5]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,10]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,47]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,86]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,17]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,38]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,118]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,32]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,54]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,104]
+ CRUSH rule 3 x 234 [55,94]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,105]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,41]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,22]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,66]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,74]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,69]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,112]
+ CRUSH rule 3 x 291 [35,52]
+ CRUSH rule 3 x 292 [20,60]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,66]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,55]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,80]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,16]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,16]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,92]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,88]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,40]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,113]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,101]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,58]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,94]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,114]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,20]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,92]
+ CRUSH rule 3 x 410 [70,11]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,21]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,15]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,22]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,34]
+ CRUSH rule 3 x 456 [101,119]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,2]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,109]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,13]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,113]
+ CRUSH rule 3 x 493 [94,105]
+ CRUSH rule 3 x 494 [66,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,78]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,81]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,54]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,17]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59,111]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,60]
+ CRUSH rule 3 x 518 [18,13]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,119]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,66]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,21]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,18]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,29]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,108]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,62]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,40]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,64]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,30]
+ CRUSH rule 3 x 638 [43,113]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,97]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,45]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,87]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,82]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,86]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,50]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3,40]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,84]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36,97]
+ CRUSH rule 3 x 697 [19,60]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,62]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,94]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,30]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,56]
+ CRUSH rule 3 x 723 [117,25]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,48]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,44]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,119]
+ CRUSH rule 3 x 752 [82,47]
+ CRUSH rule 3 x 753 [24,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,27]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,80]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,37]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,75]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,29]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,30]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,91]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,88]
+ CRUSH rule 3 x 824 [102,81]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,11]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,35]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,85]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,77]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,16]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,98]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,58]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,12]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,41]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,24]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,21]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,37]
+ CRUSH rule 3 x 942 [80,8]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,52]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,55]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,78]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,37]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,22]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,38]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,98]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,59]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,31]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,10]
+ CRUSH rule 3 x 1014 [33,98]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,44]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [83,88]
+ rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,62]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,60]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,48]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,111]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,20]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,85]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,12]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,93]
+ CRUSH rule 3 x 54 [28,3]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,91]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,92]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,16]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,89]
+ CRUSH rule 3 x 79 [64,75]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,57]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,117]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,2]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,55]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,29]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,119]
+ CRUSH rule 3 x 146 [12,73]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,8]
+ CRUSH rule 3 x 154 [19,5]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,10]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,47]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,86]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,17]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,38]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,118]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,32]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,54]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,104]
+ CRUSH rule 3 x 234 [55,94]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,105]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,41]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,22]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,66]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,74]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,69]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,112]
+ CRUSH rule 3 x 291 [35,52]
+ CRUSH rule 3 x 292 [20,60]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,66]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,55]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,80]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,16]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,16]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,92]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,88]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,40]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,113]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,101]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,58]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,94]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,114]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,20]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,92]
+ CRUSH rule 3 x 410 [70,11]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,21]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,15]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,22]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,34]
+ CRUSH rule 3 x 456 [101,119]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,2]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,109]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,13]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,113]
+ CRUSH rule 3 x 493 [94,105]
+ CRUSH rule 3 x 494 [66,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,78]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,81]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,54]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,17]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59,111]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,60]
+ CRUSH rule 3 x 518 [18,13]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,119]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,66]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,21]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,18]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,29]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,108]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,62]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,40]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,64]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,30]
+ CRUSH rule 3 x 638 [43,113]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,97]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,45]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,87]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,82]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,86]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,50]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3,40]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,84]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36,97]
+ CRUSH rule 3 x 697 [19,60]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,62]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,94]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,30]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,56]
+ CRUSH rule 3 x 723 [117,25]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,48]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,44]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,119]
+ CRUSH rule 3 x 752 [82,47]
+ CRUSH rule 3 x 753 [24,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,27]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,80]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,37]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,75]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,29]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,30]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,91]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,88]
+ CRUSH rule 3 x 824 [102,81]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,11]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,35]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,85]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,77]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,16]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,98]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,58]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,12]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,41]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,24]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,21]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,37]
+ CRUSH rule 3 x 942 [80,8]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,52]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,55]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,78]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,37]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,22]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,38]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,98]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,59]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,31]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,10]
+ CRUSH rule 3 x 1014 [33,98]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,44]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [83,88]
+ rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,62]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,60]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,48]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,111]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,20]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,85]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,12]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,93]
+ CRUSH rule 3 x 54 [28,3]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,91]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,92]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,16]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,89]
+ CRUSH rule 3 x 79 [64,75]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,57]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,117]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,2]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,55]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,29]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,119]
+ CRUSH rule 3 x 146 [12,73]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,8]
+ CRUSH rule 3 x 154 [19,5]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,10]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,47]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,86]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,17]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,38]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,118]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,32]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,54]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,104]
+ CRUSH rule 3 x 234 [55,94]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,105]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,41]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,22]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,66]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,74]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,69]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,112]
+ CRUSH rule 3 x 291 [35,52]
+ CRUSH rule 3 x 292 [20,60]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,66]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,55]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,80]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,16]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,16]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,92]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,88]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,40]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,113]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,101]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,58]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,94]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,114]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,20]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,92]
+ CRUSH rule 3 x 410 [70,11]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,21]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,15]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,22]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,34]
+ CRUSH rule 3 x 456 [101,119]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,2]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,109]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,13]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,113]
+ CRUSH rule 3 x 493 [94,105]
+ CRUSH rule 3 x 494 [66,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,78]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,81]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,54]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,17]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59,111]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,60]
+ CRUSH rule 3 x 518 [18,13]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,119]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,66]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,21]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,18]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,29]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,108]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,62]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,40]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,64]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,30]
+ CRUSH rule 3 x 638 [43,113]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,97]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,45]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,87]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,82]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,86]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,50]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3,40]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,84]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36,97]
+ CRUSH rule 3 x 697 [19,60]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,62]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,94]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,30]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,56]
+ CRUSH rule 3 x 723 [117,25]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,48]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,44]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,119]
+ CRUSH rule 3 x 752 [82,47]
+ CRUSH rule 3 x 753 [24,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,27]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,80]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,37]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,75]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,29]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,30]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,91]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,88]
+ CRUSH rule 3 x 824 [102,81]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,11]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,35]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,85]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,77]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,16]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,98]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,58]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,12]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,41]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,24]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,21]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,37]
+ CRUSH rule 3 x 942 [80,8]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,52]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,55]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,78]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,37]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,22]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,38]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,98]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,59]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,31]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,10]
+ CRUSH rule 3 x 1014 [33,98]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,44]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [83,88]
+ rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/src/test/cli/crushtool/test-map-vary-r-4.t b/src/test/cli/crushtool/test-map-vary-r-4.t
new file mode 100644
index 0000000..059da77
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-vary-r-4.t
@@ -0,0 +1,3078 @@
+ $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 4 --weight 0 0 --weight 4 0 --weight 9 0
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,12]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,52]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,46]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,40]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,107]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,57]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,114]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,29]
+ CRUSH rule 3 x 54 [28,77]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,105]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,88]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,43]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,39]
+ CRUSH rule 3 x 79 [64,65]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,83]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,68]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,26]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,17]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,83]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,100]
+ CRUSH rule 3 x 146 [12,15]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,81]
+ CRUSH rule 3 x 154 [19,56]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,22]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,103]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,100]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,85]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,113]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,66]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,86]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,102]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,32]
+ CRUSH rule 3 x 234 [55,78]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,10]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,87]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,31]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,54]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,75]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,18]
+ CRUSH rule 3 x 291 [35,42]
+ CRUSH rule 3 x 292 [20,74]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,36]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,105]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,119]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,15]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,17]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,116]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,118]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,112]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,32]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,81]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,12]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,113]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,70]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,21]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,44]
+ CRUSH rule 3 x 410 [70,47]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,15]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,79]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,21]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,68]
+ CRUSH rule 3 x 456 [101,60]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,106]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,106]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,61]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,58]
+ CRUSH rule 3 x 493 [94,89]
+ CRUSH rule 3 x 494 [56,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,82]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,6]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,77]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59,111]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,30]
+ CRUSH rule 3 x 518 [18,37]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,104]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,92]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,19]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,116]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,35]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,32]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,24]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,44]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,50]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,114]
+ CRUSH rule 3 x 638 [43,78]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,99]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,39]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,107]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,102]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,100]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,116]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3,40]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,26]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36,97]
+ CRUSH rule 3 x 697 [19,38]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,60]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,113]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,38]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,26]
+ CRUSH rule 3 x 723 [117,75]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,38]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,94]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,56]
+ CRUSH rule 3 x 752 [82,16]
+ CRUSH rule 3 x 753 [116,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,65]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,90]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,53]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,93]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,61]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,94]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,19]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,102]
+ CRUSH rule 3 x 824 [102,95]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,33]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,75]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,39]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,31]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,29]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,66]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,62]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,80]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,25]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,60]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,107]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,10]
+ CRUSH rule 3 x 942 [80,37]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,111]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,16]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,46]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,75]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,19]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,100]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,113]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,33]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,22]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,35]
+ CRUSH rule 3 x 1014 [33,48]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,111]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [59,88]
+ rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,12]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,52]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,46]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,40]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,107]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,57]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,114]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,29]
+ CRUSH rule 3 x 54 [28,77]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,105]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,88]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,43]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,39]
+ CRUSH rule 3 x 79 [64,65]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,83]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,68]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,26]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,17]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,83]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,100]
+ CRUSH rule 3 x 146 [12,15]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,81]
+ CRUSH rule 3 x 154 [19,56]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,22]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,103]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,100]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,85]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,113]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,66]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,86]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,102]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,32]
+ CRUSH rule 3 x 234 [55,78]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,10]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,87]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,31]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,54]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,75]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,18]
+ CRUSH rule 3 x 291 [35,42]
+ CRUSH rule 3 x 292 [20,74]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,36]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,105]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,119]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,15]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,17]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,116]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,118]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,112]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,32]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,81]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,12]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,113]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,70]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,21]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,44]
+ CRUSH rule 3 x 410 [70,47]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,15]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,79]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,21]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,68]
+ CRUSH rule 3 x 456 [101,60]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,106]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,106]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,61]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,58]
+ CRUSH rule 3 x 493 [94,89]
+ CRUSH rule 3 x 494 [56,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,82]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,6]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,77]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59,111]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,30]
+ CRUSH rule 3 x 518 [18,37]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,104]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,92]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,19]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,116]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,35]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,32]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,24]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,44]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,50]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,114]
+ CRUSH rule 3 x 638 [43,78]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,99]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,39]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,107]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,102]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,100]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,116]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3,40]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,26]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36,97]
+ CRUSH rule 3 x 697 [19,38]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,60]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,113]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,38]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,26]
+ CRUSH rule 3 x 723 [117,75]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,38]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,94]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,56]
+ CRUSH rule 3 x 752 [82,16]
+ CRUSH rule 3 x 753 [116,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,65]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,90]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,53]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,93]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,61]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,94]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,19]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,102]
+ CRUSH rule 3 x 824 [102,95]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,33]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,75]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,39]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,31]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,29]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,66]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,62]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,80]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,25]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,60]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,107]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,10]
+ CRUSH rule 3 x 942 [80,37]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,111]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,16]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,46]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,75]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,19]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,100]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,113]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,33]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,22]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,35]
+ CRUSH rule 3 x 1014 [33,48]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,111]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [59,88]
+ rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [94,85]
+ CRUSH rule 3 x 1 [73,78]
+ CRUSH rule 3 x 2 [91,104]
+ CRUSH rule 3 x 3 [51,94]
+ CRUSH rule 3 x 4 [45,28]
+ CRUSH rule 3 x 5 [89,113]
+ CRUSH rule 3 x 6 [91,12]
+ CRUSH rule 3 x 7 [104,71]
+ CRUSH rule 3 x 8 [41,12]
+ CRUSH rule 3 x 9 [46,35]
+ CRUSH rule 3 x 10 [61,60]
+ CRUSH rule 3 x 11 [13,74]
+ CRUSH rule 3 x 12 [83,62]
+ CRUSH rule 3 x 13 [27,117]
+ CRUSH rule 3 x 14 [105,115]
+ CRUSH rule 3 x 15 [18,87]
+ CRUSH rule 3 x 16 [103,52]
+ CRUSH rule 3 x 17 [85,80]
+ CRUSH rule 3 x 18 [11,46]
+ CRUSH rule 3 x 19 [75,114]
+ CRUSH rule 3 x 20 [111,27]
+ CRUSH rule 3 x 21 [84,7]
+ CRUSH rule 3 x 22 [23,66]
+ CRUSH rule 3 x 23 [19,84]
+ CRUSH rule 3 x 24 [83,40]
+ CRUSH rule 3 x 25 [81,108]
+ CRUSH rule 3 x 26 [17,117]
+ CRUSH rule 3 x 27 [33,58]
+ CRUSH rule 3 x 28 [45,98]
+ CRUSH rule 3 x 29 [8,46]
+ CRUSH rule 3 x 30 [55,119]
+ CRUSH rule 3 x 31 [76,35]
+ CRUSH rule 3 x 32 [72,13]
+ CRUSH rule 3 x 33 [86,107]
+ CRUSH rule 3 x 34 [7,38]
+ CRUSH rule 3 x 35 [108,31]
+ CRUSH rule 3 x 36 [67,24]
+ CRUSH rule 3 x 37 [38,17]
+ CRUSH rule 3 x 38 [72,57]
+ CRUSH rule 3 x 39 [68,73]
+ CRUSH rule 3 x 40 [30,25]
+ CRUSH rule 3 x 41 [52,91]
+ CRUSH rule 3 x 42 [106,39]
+ CRUSH rule 3 x 43 [10,115]
+ CRUSH rule 3 x 44 [101,115]
+ CRUSH rule 3 x 45 [83,80]
+ CRUSH rule 3 x 46 [54,33]
+ CRUSH rule 3 x 47 [106,41]
+ CRUSH rule 3 x 48 [34,65]
+ CRUSH rule 3 x 49 [99,46]
+ CRUSH rule 3 x 50 [42,85]
+ CRUSH rule 3 x 51 [6,114]
+ CRUSH rule 3 x 52 [82,14]
+ CRUSH rule 3 x 53 [32,29]
+ CRUSH rule 3 x 54 [28,77]
+ CRUSH rule 3 x 55 [14,44]
+ CRUSH rule 3 x 56 [21,112]
+ CRUSH rule 3 x 57 [93,26]
+ CRUSH rule 3 x 58 [48,95]
+ CRUSH rule 3 x 59 [21,104]
+ CRUSH rule 3 x 60 [90,75]
+ CRUSH rule 3 x 61 [88,39]
+ CRUSH rule 3 x 62 [100,8]
+ CRUSH rule 3 x 63 [79,96]
+ CRUSH rule 3 x 64 [1,77]
+ CRUSH rule 3 x 65 [32,25]
+ CRUSH rule 3 x 66 [48,93]
+ CRUSH rule 3 x 67 [94,91]
+ CRUSH rule 3 x 68 [102,105]
+ CRUSH rule 3 x 69 [62,20]
+ CRUSH rule 3 x 70 [84,27]
+ CRUSH rule 3 x 71 [12,99]
+ CRUSH rule 3 x 72 [26,69]
+ CRUSH rule 3 x 73 [29,88]
+ CRUSH rule 3 x 74 [29,60]
+ CRUSH rule 3 x 75 [60,43]
+ CRUSH rule 3 x 76 [55,60]
+ CRUSH rule 3 x 77 [107,78]
+ CRUSH rule 3 x 78 [86,39]
+ CRUSH rule 3 x 79 [64,65]
+ CRUSH rule 3 x 80 [73,26]
+ CRUSH rule 3 x 81 [64,57]
+ CRUSH rule 3 x 82 [37,1]
+ CRUSH rule 3 x 83 [92,22]
+ CRUSH rule 3 x 84 [49,40]
+ CRUSH rule 3 x 85 [87,30]
+ CRUSH rule 3 x 86 [37,119]
+ CRUSH rule 3 x 87 [116,3]
+ CRUSH rule 3 x 88 [38,22]
+ CRUSH rule 3 x 89 [76,41]
+ CRUSH rule 3 x 90 [14,98]
+ CRUSH rule 3 x 91 [68,27]
+ CRUSH rule 3 x 92 [86,13]
+ CRUSH rule 3 x 93 [44,83]
+ CRUSH rule 3 x 94 [46,15]
+ CRUSH rule 3 x 95 [108,6]
+ CRUSH rule 3 x 96 [66,25]
+ CRUSH rule 3 x 97 [111,33]
+ CRUSH rule 3 x 98 [93,36]
+ CRUSH rule 3 x 99 [78,17]
+ CRUSH rule 3 x 100 [28,55]
+ CRUSH rule 3 x 101 [91,34]
+ CRUSH rule 3 x 102 [82,93]
+ CRUSH rule 3 x 103 [66,105]
+ CRUSH rule 3 x 104 [116,10]
+ CRUSH rule 3 x 105 [34,69]
+ CRUSH rule 3 x 106 [69,66]
+ CRUSH rule 3 x 107 [1,41]
+ CRUSH rule 3 x 108 [7,68]
+ CRUSH rule 3 x 109 [112,87]
+ CRUSH rule 3 x 110 [54,10]
+ CRUSH rule 3 x 111 [10,86]
+ CRUSH rule 3 x 112 [80,29]
+ CRUSH rule 3 x 113 [69,26]
+ CRUSH rule 3 x 114 [79,46]
+ CRUSH rule 3 x 115 [10,111]
+ CRUSH rule 3 x 116 [37,86]
+ CRUSH rule 3 x 117 [87,50]
+ CRUSH rule 3 x 118 [23,106]
+ CRUSH rule 3 x 119 [104,14]
+ CRUSH rule 3 x 120 [44,3]
+ CRUSH rule 3 x 121 [80,14]
+ CRUSH rule 3 x 122 [45,68]
+ CRUSH rule 3 x 123 [112,22]
+ CRUSH rule 3 x 124 [97,118]
+ CRUSH rule 3 x 125 [66,7]
+ CRUSH rule 3 x 126 [70,23]
+ CRUSH rule 3 x 127 [70,13]
+ CRUSH rule 3 x 128 [11,119]
+ CRUSH rule 3 x 129 [103,108]
+ CRUSH rule 3 x 130 [50,17]
+ CRUSH rule 3 x 131 [44,55]
+ CRUSH rule 3 x 132 [69,1]
+ CRUSH rule 3 x 133 [67,104]
+ CRUSH rule 3 x 134 [37,66]
+ CRUSH rule 3 x 135 [78,101]
+ CRUSH rule 3 x 136 [32,83]
+ CRUSH rule 3 x 137 [92,81]
+ CRUSH rule 3 x 138 [54,17]
+ CRUSH rule 3 x 139 [89,92]
+ CRUSH rule 3 x 140 [39,1]
+ CRUSH rule 3 x 141 [89,28]
+ CRUSH rule 3 x 142 [22,26]
+ CRUSH rule 3 x 143 [96,77]
+ CRUSH rule 3 x 144 [13,111]
+ CRUSH rule 3 x 145 [77,100]
+ CRUSH rule 3 x 146 [12,15]
+ CRUSH rule 3 x 147 [2,11]
+ CRUSH rule 3 x 148 [85,108]
+ CRUSH rule 3 x 149 [103,62]
+ CRUSH rule 3 x 150 [14,78]
+ CRUSH rule 3 x 151 [75,119]
+ CRUSH rule 3 x 152 [49,84]
+ CRUSH rule 3 x 153 [92,81]
+ CRUSH rule 3 x 154 [19,56]
+ CRUSH rule 3 x 155 [12,75]
+ CRUSH rule 3 x 156 [107,112]
+ CRUSH rule 3 x 157 [15,28]
+ CRUSH rule 3 x 158 [11,113]
+ CRUSH rule 3 x 159 [33,52]
+ CRUSH rule 3 x 160 [86,35]
+ CRUSH rule 3 x 161 [19,117]
+ CRUSH rule 3 x 162 [55,113]
+ CRUSH rule 3 x 163 [54,87]
+ CRUSH rule 3 x 164 [72,8]
+ CRUSH rule 3 x 165 [25,74]
+ CRUSH rule 3 x 166 [2,22]
+ CRUSH rule 3 x 167 [89,56]
+ CRUSH rule 3 x 168 [68,103]
+ CRUSH rule 3 x 169 [51,12]
+ CRUSH rule 3 x 170 [68,53]
+ CRUSH rule 3 x 171 [88,79]
+ CRUSH rule 3 x 172 [117,89]
+ CRUSH rule 3 x 173 [29,40]
+ CRUSH rule 3 x 174 [67,86]
+ CRUSH rule 3 x 175 [48,85]
+ CRUSH rule 3 x 176 [94,83]
+ CRUSH rule 3 x 177 [53,18]
+ CRUSH rule 3 x 178 [39,30]
+ CRUSH rule 3 x 179 [72,17]
+ CRUSH rule 3 x 180 [3,114]
+ CRUSH rule 3 x 181 [18,16]
+ CRUSH rule 3 x 182 [75,5]
+ CRUSH rule 3 x 183 [11,110]
+ CRUSH rule 3 x 184 [79,48]
+ CRUSH rule 3 x 185 [97,100]
+ CRUSH rule 3 x 186 [67,44]
+ CRUSH rule 3 x 187 [6,50]
+ CRUSH rule 3 x 188 [76,85]
+ CRUSH rule 3 x 189 [96,7]
+ CRUSH rule 3 x 190 [90,95]
+ CRUSH rule 3 x 191 [49,113]
+ CRUSH rule 3 x 192 [93,58]
+ CRUSH rule 3 x 193 [89,66]
+ CRUSH rule 3 x 194 [62,3]
+ CRUSH rule 3 x 195 [119,85]
+ CRUSH rule 3 x 196 [20,72]
+ CRUSH rule 3 x 197 [6,116]
+ CRUSH rule 3 x 198 [55,92]
+ CRUSH rule 3 x 199 [77,66]
+ CRUSH rule 3 x 200 [12,81]
+ CRUSH rule 3 x 201 [52,71]
+ CRUSH rule 3 x 202 [98,59]
+ CRUSH rule 3 x 203 [36,19]
+ CRUSH rule 3 x 204 [10,113]
+ CRUSH rule 3 x 205 [38,79]
+ CRUSH rule 3 x 206 [38,105]
+ CRUSH rule 3 x 207 [19,86]
+ CRUSH rule 3 x 208 [63,92]
+ CRUSH rule 3 x 209 [70,99]
+ CRUSH rule 3 x 210 [79,102]
+ CRUSH rule 3 x 211 [26,27]
+ CRUSH rule 3 x 212 [28,107]
+ CRUSH rule 3 x 213 [100,49]
+ CRUSH rule 3 x 214 [91,88]
+ CRUSH rule 3 x 215 [92,7]
+ CRUSH rule 3 x 216 [99,108]
+ CRUSH rule 3 x 217 [86,97]
+ CRUSH rule 3 x 218 [70,10]
+ CRUSH rule 3 x 219 [61,112]
+ CRUSH rule 3 x 220 [23,66]
+ CRUSH rule 3 x 221 [51,66]
+ CRUSH rule 3 x 222 [50,65]
+ CRUSH rule 3 x 223 [34,45]
+ CRUSH rule 3 x 224 [107,44]
+ CRUSH rule 3 x 225 [61,102]
+ CRUSH rule 3 x 226 [44,87]
+ CRUSH rule 3 x 227 [55,66]
+ CRUSH rule 3 x 228 [117,103]
+ CRUSH rule 3 x 229 [100,27]
+ CRUSH rule 3 x 230 [41,32]
+ CRUSH rule 3 x 231 [30,16]
+ CRUSH rule 3 x 232 [23,102]
+ CRUSH rule 3 x 233 [47,32]
+ CRUSH rule 3 x 234 [55,78]
+ CRUSH rule 3 x 235 [20,32]
+ CRUSH rule 3 x 236 [95,118]
+ CRUSH rule 3 x 237 [21,72]
+ CRUSH rule 3 x 238 [109,53]
+ CRUSH rule 3 x 239 [40,10]
+ CRUSH rule 3 x 240 [63,96]
+ CRUSH rule 3 x 241 [47,1]
+ CRUSH rule 3 x 242 [73,24]
+ CRUSH rule 3 x 243 [76,79]
+ CRUSH rule 3 x 244 [103,115]
+ CRUSH rule 3 x 245 [106,29]
+ CRUSH rule 3 x 246 [35,5]
+ CRUSH rule 3 x 247 [116,37]
+ CRUSH rule 3 x 248 [8,34]
+ CRUSH rule 3 x 249 [2,105]
+ CRUSH rule 3 x 250 [34,79]
+ CRUSH rule 3 x 251 [28,87]
+ CRUSH rule 3 x 252 [95,24]
+ CRUSH rule 3 x 253 [109,97]
+ CRUSH rule 3 x 254 [99,56]
+ CRUSH rule 3 x 255 [112,31]
+ CRUSH rule 3 x 256 [94,31]
+ CRUSH rule 3 x 257 [100,39]
+ CRUSH rule 3 x 258 [34,83]
+ CRUSH rule 3 x 259 [70,87]
+ CRUSH rule 3 x 260 [89,24]
+ CRUSH rule 3 x 261 [94,77]
+ CRUSH rule 3 x 262 [42,97]
+ CRUSH rule 3 x 263 [113,37]
+ CRUSH rule 3 x 264 [36,89]
+ CRUSH rule 3 x 265 [14,46]
+ CRUSH rule 3 x 266 [75,48]
+ CRUSH rule 3 x 267 [6,46]
+ CRUSH rule 3 x 268 [38,3]
+ CRUSH rule 3 x 269 [86,91]
+ CRUSH rule 3 x 270 [87,54]
+ CRUSH rule 3 x 271 [19,78]
+ CRUSH rule 3 x 272 [73,110]
+ CRUSH rule 3 x 273 [69,113]
+ CRUSH rule 3 x 274 [47,26]
+ CRUSH rule 3 x 275 [92,29]
+ CRUSH rule 3 x 276 [7,38]
+ CRUSH rule 3 x 277 [74,95]
+ CRUSH rule 3 x 278 [107,62]
+ CRUSH rule 3 x 279 [112,53]
+ CRUSH rule 3 x 280 [113,75]
+ CRUSH rule 3 x 281 [89,40]
+ CRUSH rule 3 x 282 [20,46]
+ CRUSH rule 3 x 283 [8,36]
+ CRUSH rule 3 x 284 [66,85]
+ CRUSH rule 3 x 285 [99,109]
+ CRUSH rule 3 x 286 [78,89]
+ CRUSH rule 3 x 287 [12,79]
+ CRUSH rule 3 x 288 [24,37]
+ CRUSH rule 3 x 289 [105,74]
+ CRUSH rule 3 x 290 [25,18]
+ CRUSH rule 3 x 291 [35,42]
+ CRUSH rule 3 x 292 [20,74]
+ CRUSH rule 3 x 293 [27,118]
+ CRUSH rule 3 x 294 [60,75]
+ CRUSH rule 3 x 295 [37,36]
+ CRUSH rule 3 x 296 [16,28]
+ CRUSH rule 3 x 297 [36,29]
+ CRUSH rule 3 x 298 [70,105]
+ CRUSH rule 3 x 299 [116,85]
+ CRUSH rule 3 x 300 [67,36]
+ CRUSH rule 3 x 301 [117,71]
+ CRUSH rule 3 x 302 [78,105]
+ CRUSH rule 3 x 303 [19,82]
+ CRUSH rule 3 x 304 [101,38]
+ CRUSH rule 3 x 305 [5,49]
+ CRUSH rule 3 x 306 [41,64]
+ CRUSH rule 3 x 307 [65,119]
+ CRUSH rule 3 x 308 [91,115]
+ CRUSH rule 3 x 309 [38,41]
+ CRUSH rule 3 x 310 [26,43]
+ CRUSH rule 3 x 311 [36,75]
+ CRUSH rule 3 x 312 [114,15]
+ CRUSH rule 3 x 313 [104,79]
+ CRUSH rule 3 x 314 [28,43]
+ CRUSH rule 3 x 315 [118,17]
+ CRUSH rule 3 x 316 [98,39]
+ CRUSH rule 3 x 317 [118,21]
+ CRUSH rule 3 x 318 [17,94]
+ CRUSH rule 3 x 319 [53,62]
+ CRUSH rule 3 x 320 [36,3]
+ CRUSH rule 3 x 321 [33,60]
+ CRUSH rule 3 x 322 [68,3]
+ CRUSH rule 3 x 323 [66,95]
+ CRUSH rule 3 x 324 [21,42]
+ CRUSH rule 3 x 325 [52,43]
+ CRUSH rule 3 x 326 [7,90]
+ CRUSH rule 3 x 327 [62,3]
+ CRUSH rule 3 x 328 [61,42]
+ CRUSH rule 3 x 329 [19,115]
+ CRUSH rule 3 x 330 [24,15]
+ CRUSH rule 3 x 331 [84,14]
+ CRUSH rule 3 x 332 [61,72]
+ CRUSH rule 3 x 333 [116,6]
+ CRUSH rule 3 x 334 [94,29]
+ CRUSH rule 3 x 335 [71,116]
+ CRUSH rule 3 x 336 [24,11]
+ CRUSH rule 3 x 337 [18,23]
+ CRUSH rule 3 x 338 [43,118]
+ CRUSH rule 3 x 339 [13,50]
+ CRUSH rule 3 x 340 [81,115]
+ CRUSH rule 3 x 341 [46,65]
+ CRUSH rule 3 x 342 [92,71]
+ CRUSH rule 3 x 343 [49,56]
+ CRUSH rule 3 x 344 [1,25]
+ CRUSH rule 3 x 345 [56,11]
+ CRUSH rule 3 x 346 [3,112]
+ CRUSH rule 3 x 347 [106,85]
+ CRUSH rule 3 x 348 [10,114]
+ CRUSH rule 3 x 349 [96,51]
+ CRUSH rule 3 x 350 [63,32]
+ CRUSH rule 3 x 351 [60,20]
+ CRUSH rule 3 x 352 [36,21]
+ CRUSH rule 3 x 353 [10,32]
+ CRUSH rule 3 x 354 [55,74]
+ CRUSH rule 3 x 355 [73,80]
+ CRUSH rule 3 x 356 [75,96]
+ CRUSH rule 3 x 357 [70,89]
+ CRUSH rule 3 x 358 [97,92]
+ CRUSH rule 3 x 359 [119,20]
+ CRUSH rule 3 x 360 [106,15]
+ CRUSH rule 3 x 361 [27,56]
+ CRUSH rule 3 x 362 [28,22]
+ CRUSH rule 3 x 363 [68,81]
+ CRUSH rule 3 x 364 [23,2]
+ CRUSH rule 3 x 365 [57,12]
+ CRUSH rule 3 x 366 [42,61]
+ CRUSH rule 3 x 367 [103,108]
+ CRUSH rule 3 x 368 [103,119]
+ CRUSH rule 3 x 369 [12,11]
+ CRUSH rule 3 x 370 [11,109]
+ CRUSH rule 3 x 371 [34,65]
+ CRUSH rule 3 x 372 [58,29]
+ CRUSH rule 3 x 373 [6,64]
+ CRUSH rule 3 x 374 [110,89]
+ CRUSH rule 3 x 375 [5,89]
+ CRUSH rule 3 x 376 [91,98]
+ CRUSH rule 3 x 377 [93,113]
+ CRUSH rule 3 x 378 [68,41]
+ CRUSH rule 3 x 379 [77,94]
+ CRUSH rule 3 x 380 [76,107]
+ CRUSH rule 3 x 381 [36,20]
+ CRUSH rule 3 x 382 [26,107]
+ CRUSH rule 3 x 383 [48,93]
+ CRUSH rule 3 x 384 [15,100]
+ CRUSH rule 3 x 385 [82,27]
+ CRUSH rule 3 x 386 [83,24]
+ CRUSH rule 3 x 387 [16,70]
+ CRUSH rule 3 x 388 [29,66]
+ CRUSH rule 3 x 389 [92,67]
+ CRUSH rule 3 x 390 [68,13]
+ CRUSH rule 3 x 391 [15,2]
+ CRUSH rule 3 x 392 [21,110]
+ CRUSH rule 3 x 393 [91,113]
+ CRUSH rule 3 x 394 [38,21]
+ CRUSH rule 3 x 395 [21,92]
+ CRUSH rule 3 x 396 [12,59]
+ CRUSH rule 3 x 397 [40,51]
+ CRUSH rule 3 x 398 [44,21]
+ CRUSH rule 3 x 399 [5,33]
+ CRUSH rule 3 x 400 [19,64]
+ CRUSH rule 3 x 401 [79,109]
+ CRUSH rule 3 x 402 [107,72]
+ CRUSH rule 3 x 403 [23,74]
+ CRUSH rule 3 x 404 [87,78]
+ CRUSH rule 3 x 405 [90,93]
+ CRUSH rule 3 x 406 [15,98]
+ CRUSH rule 3 x 407 [70,25]
+ CRUSH rule 3 x 408 [55,104]
+ CRUSH rule 3 x 409 [73,44]
+ CRUSH rule 3 x 410 [70,47]
+ CRUSH rule 3 x 411 [34,15]
+ CRUSH rule 3 x 412 [105,44]
+ CRUSH rule 3 x 413 [41,86]
+ CRUSH rule 3 x 414 [70,71]
+ CRUSH rule 3 x 415 [107,80]
+ CRUSH rule 3 x 416 [2,23]
+ CRUSH rule 3 x 417 [26,23]
+ CRUSH rule 3 x 418 [51,114]
+ CRUSH rule 3 x 419 [8,94]
+ CRUSH rule 3 x 420 [109,15]
+ CRUSH rule 3 x 421 [114,77]
+ CRUSH rule 3 x 422 [109,39]
+ CRUSH rule 3 x 423 [59,98]
+ CRUSH rule 3 x 424 [92,65]
+ CRUSH rule 3 x 425 [101,50]
+ CRUSH rule 3 x 426 [36,57]
+ CRUSH rule 3 x 427 [8,38]
+ CRUSH rule 3 x 428 [68,63]
+ CRUSH rule 3 x 429 [76,13]
+ CRUSH rule 3 x 430 [67,100]
+ CRUSH rule 3 x 431 [70,53]
+ CRUSH rule 3 x 432 [7,50]
+ CRUSH rule 3 x 433 [49,24]
+ CRUSH rule 3 x 434 [64,59]
+ CRUSH rule 3 x 435 [110,71]
+ CRUSH rule 3 x 436 [106,47]
+ CRUSH rule 3 x 437 [26,29]
+ CRUSH rule 3 x 438 [118,95]
+ CRUSH rule 3 x 439 [40,83]
+ CRUSH rule 3 x 440 [45,68]
+ CRUSH rule 3 x 441 [112,15]
+ CRUSH rule 3 x 442 [55,18]
+ CRUSH rule 3 x 443 [44,37]
+ CRUSH rule 3 x 444 [71,119]
+ CRUSH rule 3 x 445 [58,63]
+ CRUSH rule 3 x 446 [40,20]
+ CRUSH rule 3 x 447 [100,43]
+ CRUSH rule 3 x 448 [111,15]
+ CRUSH rule 3 x 449 [67,102]
+ CRUSH rule 3 x 450 [117,79]
+ CRUSH rule 3 x 451 [66,75]
+ CRUSH rule 3 x 452 [70,33]
+ CRUSH rule 3 x 453 [82,21]
+ CRUSH rule 3 x 454 [53,28]
+ CRUSH rule 3 x 455 [91,68]
+ CRUSH rule 3 x 456 [101,60]
+ CRUSH rule 3 x 457 [113,97]
+ CRUSH rule 3 x 458 [119,41]
+ CRUSH rule 3 x 459 [50,55]
+ CRUSH rule 3 x 460 [105,30]
+ CRUSH rule 3 x 461 [102,45]
+ CRUSH rule 3 x 462 [98,25]
+ CRUSH rule 3 x 463 [108,57]
+ CRUSH rule 3 x 464 [19,50]
+ CRUSH rule 3 x 465 [62,95]
+ CRUSH rule 3 x 466 [53,106]
+ CRUSH rule 3 x 467 [40,95]
+ CRUSH rule 3 x 468 [97,108]
+ CRUSH rule 3 x 469 [98,16]
+ CRUSH rule 3 x 470 [50,3]
+ CRUSH rule 3 x 471 [40,14]
+ CRUSH rule 3 x 472 [27,28]
+ CRUSH rule 3 x 473 [48,17]
+ CRUSH rule 3 x 474 [51,113]
+ CRUSH rule 3 x 475 [49,66]
+ CRUSH rule 3 x 476 [110,55]
+ CRUSH rule 3 x 477 [80,8]
+ CRUSH rule 3 x 478 [78,25]
+ CRUSH rule 3 x 479 [31,106]
+ CRUSH rule 3 x 480 [75,5]
+ CRUSH rule 3 x 481 [26,37]
+ CRUSH rule 3 x 482 [84,87]
+ CRUSH rule 3 x 483 [15,113]
+ CRUSH rule 3 x 484 [37,28]
+ CRUSH rule 3 x 485 [84,61]
+ CRUSH rule 3 x 486 [92,61]
+ CRUSH rule 3 x 487 [106,53]
+ CRUSH rule 3 x 488 [42,7]
+ CRUSH rule 3 x 489 [89,98]
+ CRUSH rule 3 x 490 [22,119]
+ CRUSH rule 3 x 491 [99,5]
+ CRUSH rule 3 x 492 [21,58]
+ CRUSH rule 3 x 493 [94,89]
+ CRUSH rule 3 x 494 [56,59]
+ CRUSH rule 3 x 495 [95,119]
+ CRUSH rule 3 x 496 [46,43]
+ CRUSH rule 3 x 497 [102,89]
+ CRUSH rule 3 x 498 [21,82]
+ CRUSH rule 3 x 499 [5,95]
+ CRUSH rule 3 x 500 [50,6]
+ CRUSH rule 3 x 501 [60,75]
+ CRUSH rule 3 x 502 [65,1]
+ CRUSH rule 3 x 503 [21,115]
+ CRUSH rule 3 x 504 [67,5]
+ CRUSH rule 3 x 505 [12,91]
+ CRUSH rule 3 x 506 [79,110]
+ CRUSH rule 3 x 507 [34,77]
+ CRUSH rule 3 x 508 [34,45]
+ CRUSH rule 3 x 509 [19,74]
+ CRUSH rule 3 x 510 [117,69]
+ CRUSH rule 3 x 511 [14,34]
+ CRUSH rule 3 x 512 [59,111]
+ CRUSH rule 3 x 513 [102,13]
+ CRUSH rule 3 x 514 [75,111]
+ CRUSH rule 3 x 515 [84,83]
+ CRUSH rule 3 x 516 [37,80]
+ CRUSH rule 3 x 517 [83,30]
+ CRUSH rule 3 x 518 [18,37]
+ CRUSH rule 3 x 519 [67,52]
+ CRUSH rule 3 x 520 [15,70]
+ CRUSH rule 3 x 521 [70,22]
+ CRUSH rule 3 x 522 [56,3]
+ CRUSH rule 3 x 523 [36,23]
+ CRUSH rule 3 x 524 [33,94]
+ CRUSH rule 3 x 525 [63,104]
+ CRUSH rule 3 x 526 [83,118]
+ CRUSH rule 3 x 527 [37,5]
+ CRUSH rule 3 x 528 [108,43]
+ CRUSH rule 3 x 529 [74,7]
+ CRUSH rule 3 x 530 [49,12]
+ CRUSH rule 3 x 531 [117,107]
+ CRUSH rule 3 x 532 [31,68]
+ CRUSH rule 3 x 533 [5,73]
+ CRUSH rule 3 x 534 [97,104]
+ CRUSH rule 3 x 535 [48,41]
+ CRUSH rule 3 x 536 [113,71]
+ CRUSH rule 3 x 537 [116,7]
+ CRUSH rule 3 x 538 [85,40]
+ CRUSH rule 3 x 539 [72,85]
+ CRUSH rule 3 x 540 [39,12]
+ CRUSH rule 3 x 541 [53,64]
+ CRUSH rule 3 x 542 [27,54]
+ CRUSH rule 3 x 543 [45,106]
+ CRUSH rule 3 x 544 [59,26]
+ CRUSH rule 3 x 545 [118,15]
+ CRUSH rule 3 x 546 [18,71]
+ CRUSH rule 3 x 547 [67,80]
+ CRUSH rule 3 x 548 [53,92]
+ CRUSH rule 3 x 549 [60,51]
+ CRUSH rule 3 x 550 [92,37]
+ CRUSH rule 3 x 551 [77,52]
+ CRUSH rule 3 x 552 [61,80]
+ CRUSH rule 3 x 553 [71,84]
+ CRUSH rule 3 x 554 [61,52]
+ CRUSH rule 3 x 555 [76,69]
+ CRUSH rule 3 x 556 [106,10]
+ CRUSH rule 3 x 557 [26,35]
+ CRUSH rule 3 x 558 [41,46]
+ CRUSH rule 3 x 559 [65,86]
+ CRUSH rule 3 x 560 [94,91]
+ CRUSH rule 3 x 561 [27,98]
+ CRUSH rule 3 x 562 [78,19]
+ CRUSH rule 3 x 563 [59,82]
+ CRUSH rule 3 x 564 [96,15]
+ CRUSH rule 3 x 565 [8,92]
+ CRUSH rule 3 x 566 [119,81]
+ CRUSH rule 3 x 567 [7,46]
+ CRUSH rule 3 x 568 [57,96]
+ CRUSH rule 3 x 569 [65,100]
+ CRUSH rule 3 x 570 [98,103]
+ CRUSH rule 3 x 571 [95,110]
+ CRUSH rule 3 x 572 [62,75]
+ CRUSH rule 3 x 573 [1,20]
+ CRUSH rule 3 x 574 [89,64]
+ CRUSH rule 3 x 575 [87,54]
+ CRUSH rule 3 x 576 [21,113]
+ CRUSH rule 3 x 577 [8,113]
+ CRUSH rule 3 x 578 [75,116]
+ CRUSH rule 3 x 579 [105,96]
+ CRUSH rule 3 x 580 [51,12]
+ CRUSH rule 3 x 581 [55,40]
+ CRUSH rule 3 x 582 [27,106]
+ CRUSH rule 3 x 583 [6,102]
+ CRUSH rule 3 x 584 [10,90]
+ CRUSH rule 3 x 585 [20,88]
+ CRUSH rule 3 x 586 [48,67]
+ CRUSH rule 3 x 587 [29,5]
+ CRUSH rule 3 x 588 [103,40]
+ CRUSH rule 3 x 589 [88,85]
+ CRUSH rule 3 x 590 [76,11]
+ CRUSH rule 3 x 591 [42,17]
+ CRUSH rule 3 x 592 [78,6]
+ CRUSH rule 3 x 593 [82,35]
+ CRUSH rule 3 x 594 [27,76]
+ CRUSH rule 3 x 595 [52,10]
+ CRUSH rule 3 x 596 [82,99]
+ CRUSH rule 3 x 597 [16,32]
+ CRUSH rule 3 x 598 [37,36]
+ CRUSH rule 3 x 599 [10,24]
+ CRUSH rule 3 x 600 [24,37]
+ CRUSH rule 3 x 601 [104,21]
+ CRUSH rule 3 x 602 [48,39]
+ CRUSH rule 3 x 603 [93,44]
+ CRUSH rule 3 x 604 [118,87]
+ CRUSH rule 3 x 605 [104,63]
+ CRUSH rule 3 x 606 [90,103]
+ CRUSH rule 3 x 607 [95,72]
+ CRUSH rule 3 x 608 [112,71]
+ CRUSH rule 3 x 609 [34,16]
+ CRUSH rule 3 x 610 [106,73]
+ CRUSH rule 3 x 611 [66,37]
+ CRUSH rule 3 x 612 [2,20]
+ CRUSH rule 3 x 613 [13,92]
+ CRUSH rule 3 x 614 [50,65]
+ CRUSH rule 3 x 615 [24,39]
+ CRUSH rule 3 x 616 [41,46]
+ CRUSH rule 3 x 617 [111,81]
+ CRUSH rule 3 x 618 [3,72]
+ CRUSH rule 3 x 619 [92,31]
+ CRUSH rule 3 x 620 [108,31]
+ CRUSH rule 3 x 621 [105,50]
+ CRUSH rule 3 x 622 [67,102]
+ CRUSH rule 3 x 623 [69,117]
+ CRUSH rule 3 x 624 [115,79]
+ CRUSH rule 3 x 625 [73,94]
+ CRUSH rule 3 x 626 [52,25]
+ CRUSH rule 3 x 627 [116,105]
+ CRUSH rule 3 x 628 [98,87]
+ CRUSH rule 3 x 629 [6,116]
+ CRUSH rule 3 x 630 [22,50]
+ CRUSH rule 3 x 631 [35,96]
+ CRUSH rule 3 x 632 [80,53]
+ CRUSH rule 3 x 633 [65,110]
+ CRUSH rule 3 x 634 [87,50]
+ CRUSH rule 3 x 635 [107,111]
+ CRUSH rule 3 x 636 [23,30]
+ CRUSH rule 3 x 637 [99,114]
+ CRUSH rule 3 x 638 [43,78]
+ CRUSH rule 3 x 639 [30,31]
+ CRUSH rule 3 x 640 [113,87]
+ CRUSH rule 3 x 641 [45,58]
+ CRUSH rule 3 x 642 [47,30]
+ CRUSH rule 3 x 643 [64,99]
+ CRUSH rule 3 x 644 [31,119]
+ CRUSH rule 3 x 645 [77,90]
+ CRUSH rule 3 x 646 [37,26]
+ CRUSH rule 3 x 647 [65,112]
+ CRUSH rule 3 x 648 [31,84]
+ CRUSH rule 3 x 649 [88,39]
+ CRUSH rule 3 x 650 [21,44]
+ CRUSH rule 3 x 651 [63,12]
+ CRUSH rule 3 x 652 [57,28]
+ CRUSH rule 3 x 653 [38,63]
+ CRUSH rule 3 x 654 [104,107]
+ CRUSH rule 3 x 655 [89,109]
+ CRUSH rule 3 x 656 [79,84]
+ CRUSH rule 3 x 657 [47,18]
+ CRUSH rule 3 x 658 [80,49]
+ CRUSH rule 3 x 659 [11,104]
+ CRUSH rule 3 x 660 [65,102]
+ CRUSH rule 3 x 661 [96,67]
+ CRUSH rule 3 x 662 [111,43]
+ CRUSH rule 3 x 663 [83,115]
+ CRUSH rule 3 x 664 [59,52]
+ CRUSH rule 3 x 665 [31,86]
+ CRUSH rule 3 x 666 [112,8]
+ CRUSH rule 3 x 667 [70,107]
+ CRUSH rule 3 x 668 [96,43]
+ CRUSH rule 3 x 669 [56,25]
+ CRUSH rule 3 x 670 [98,83]
+ CRUSH rule 3 x 671 [57,100]
+ CRUSH rule 3 x 672 [37,98]
+ CRUSH rule 3 x 673 [83,116]
+ CRUSH rule 3 x 674 [36,95]
+ CRUSH rule 3 x 675 [88,91]
+ CRUSH rule 3 x 676 [3,40]
+ CRUSH rule 3 x 677 [88,105]
+ CRUSH rule 3 x 678 [27,100]
+ CRUSH rule 3 x 679 [33,118]
+ CRUSH rule 3 x 680 [111,81]
+ CRUSH rule 3 x 681 [53,68]
+ CRUSH rule 3 x 682 [12,83]
+ CRUSH rule 3 x 683 [24,67]
+ CRUSH rule 3 x 684 [98,45]
+ CRUSH rule 3 x 685 [106,25]
+ CRUSH rule 3 x 686 [86,45]
+ CRUSH rule 3 x 687 [49,102]
+ CRUSH rule 3 x 688 [16,52]
+ CRUSH rule 3 x 689 [32,101]
+ CRUSH rule 3 x 690 [96,79]
+ CRUSH rule 3 x 691 [34,99]
+ CRUSH rule 3 x 692 [97,68]
+ CRUSH rule 3 x 693 [29,38]
+ CRUSH rule 3 x 694 [6,26]
+ CRUSH rule 3 x 695 [31,112]
+ CRUSH rule 3 x 696 [36,97]
+ CRUSH rule 3 x 697 [19,38]
+ CRUSH rule 3 x 698 [30,103]
+ CRUSH rule 3 x 699 [47,60]
+ CRUSH rule 3 x 700 [99,82]
+ CRUSH rule 3 x 701 [53,72]
+ CRUSH rule 3 x 702 [101,113]
+ CRUSH rule 3 x 703 [92,20]
+ CRUSH rule 3 x 704 [34,47]
+ CRUSH rule 3 x 705 [105,88]
+ CRUSH rule 3 x 706 [74,20]
+ CRUSH rule 3 x 707 [95,40]
+ CRUSH rule 3 x 708 [95,38]
+ CRUSH rule 3 x 709 [73,94]
+ CRUSH rule 3 x 710 [94,7]
+ CRUSH rule 3 x 711 [68,16]
+ CRUSH rule 3 x 712 [107,64]
+ CRUSH rule 3 x 713 [29,2]
+ CRUSH rule 3 x 714 [86,97]
+ CRUSH rule 3 x 715 [74,95]
+ CRUSH rule 3 x 716 [101,74]
+ CRUSH rule 3 x 717 [12,57]
+ CRUSH rule 3 x 718 [83,106]
+ CRUSH rule 3 x 719 [26,39]
+ CRUSH rule 3 x 720 [69,64]
+ CRUSH rule 3 x 721 [51,119]
+ CRUSH rule 3 x 722 [15,26]
+ CRUSH rule 3 x 723 [117,75]
+ CRUSH rule 3 x 724 [45,106]
+ CRUSH rule 3 x 725 [53,66]
+ CRUSH rule 3 x 726 [103,38]
+ CRUSH rule 3 x 727 [89,115]
+ CRUSH rule 3 x 728 [76,65]
+ CRUSH rule 3 x 729 [35,48]
+ CRUSH rule 3 x 730 [28,37]
+ CRUSH rule 3 x 731 [78,6]
+ CRUSH rule 3 x 732 [1,93]
+ CRUSH rule 3 x 733 [35,44]
+ CRUSH rule 3 x 734 [119,93]
+ CRUSH rule 3 x 735 [102,17]
+ CRUSH rule 3 x 736 [37,78]
+ CRUSH rule 3 x 737 [117,35]
+ CRUSH rule 3 x 738 [57,56]
+ CRUSH rule 3 x 739 [87,24]
+ CRUSH rule 3 x 740 [29,34]
+ CRUSH rule 3 x 741 [47,94]
+ CRUSH rule 3 x 742 [106,107]
+ CRUSH rule 3 x 743 [105,5]
+ CRUSH rule 3 x 744 [23,30]
+ CRUSH rule 3 x 745 [37,106]
+ CRUSH rule 3 x 746 [56,47]
+ CRUSH rule 3 x 747 [56,107]
+ CRUSH rule 3 x 748 [48,25]
+ CRUSH rule 3 x 749 [102,93]
+ CRUSH rule 3 x 750 [83,102]
+ CRUSH rule 3 x 751 [25,56]
+ CRUSH rule 3 x 752 [82,16]
+ CRUSH rule 3 x 753 [116,14]
+ CRUSH rule 3 x 754 [114,39]
+ CRUSH rule 3 x 755 [87,60]
+ CRUSH rule 3 x 756 [113,77]
+ CRUSH rule 3 x 757 [47,112]
+ CRUSH rule 3 x 758 [54,107]
+ CRUSH rule 3 x 759 [74,65]
+ CRUSH rule 3 x 760 [88,47]
+ CRUSH rule 3 x 761 [73,98]
+ CRUSH rule 3 x 762 [34,33]
+ CRUSH rule 3 x 763 [13,116]
+ CRUSH rule 3 x 764 [89,2]
+ CRUSH rule 3 x 765 [109,77]
+ CRUSH rule 3 x 766 [19,92]
+ CRUSH rule 3 x 767 [41,80]
+ CRUSH rule 3 x 768 [106,16]
+ CRUSH rule 3 x 769 [91,2]
+ CRUSH rule 3 x 770 [72,19]
+ CRUSH rule 3 x 771 [115,63]
+ CRUSH rule 3 x 772 [97,102]
+ CRUSH rule 3 x 773 [116,91]
+ CRUSH rule 3 x 774 [100,105]
+ CRUSH rule 3 x 775 [102,95]
+ CRUSH rule 3 x 776 [69,44]
+ CRUSH rule 3 x 777 [91,102]
+ CRUSH rule 3 x 778 [83,110]
+ CRUSH rule 3 x 779 [47,80]
+ CRUSH rule 3 x 780 [63,117]
+ CRUSH rule 3 x 781 [105,106]
+ CRUSH rule 3 x 782 [117,107]
+ CRUSH rule 3 x 783 [19,30]
+ CRUSH rule 3 x 784 [63,82]
+ CRUSH rule 3 x 785 [27,50]
+ CRUSH rule 3 x 786 [41,90]
+ CRUSH rule 3 x 787 [108,27]
+ CRUSH rule 3 x 788 [74,75]
+ CRUSH rule 3 x 789 [50,67]
+ CRUSH rule 3 x 790 [20,108]
+ CRUSH rule 3 x 791 [96,53]
+ CRUSH rule 3 x 792 [80,13]
+ CRUSH rule 3 x 793 [6,82]
+ CRUSH rule 3 x 794 [14,90]
+ CRUSH rule 3 x 795 [30,67]
+ CRUSH rule 3 x 796 [87,60]
+ CRUSH rule 3 x 797 [64,93]
+ CRUSH rule 3 x 798 [42,19]
+ CRUSH rule 3 x 799 [19,113]
+ CRUSH rule 3 x 800 [106,22]
+ CRUSH rule 3 x 801 [2,11]
+ CRUSH rule 3 x 802 [63,1]
+ CRUSH rule 3 x 803 [37,46]
+ CRUSH rule 3 x 804 [33,66]
+ CRUSH rule 3 x 805 [96,3]
+ CRUSH rule 3 x 806 [48,57]
+ CRUSH rule 3 x 807 [48,85]
+ CRUSH rule 3 x 808 [76,15]
+ CRUSH rule 3 x 809 [27,90]
+ CRUSH rule 3 x 810 [119,61]
+ CRUSH rule 3 x 811 [111,93]
+ CRUSH rule 3 x 812 [25,94]
+ CRUSH rule 3 x 813 [81,50]
+ CRUSH rule 3 x 814 [95,48]
+ CRUSH rule 3 x 815 [84,6]
+ CRUSH rule 3 x 816 [64,3]
+ CRUSH rule 3 x 817 [63,117]
+ CRUSH rule 3 x 818 [69,52]
+ CRUSH rule 3 x 819 [88,19]
+ CRUSH rule 3 x 820 [104,29]
+ CRUSH rule 3 x 821 [58,107]
+ CRUSH rule 3 x 822 [20,18]
+ CRUSH rule 3 x 823 [63,102]
+ CRUSH rule 3 x 824 [102,95]
+ CRUSH rule 3 x 825 [47,46]
+ CRUSH rule 3 x 826 [44,33]
+ CRUSH rule 3 x 827 [101,115]
+ CRUSH rule 3 x 828 [60,39]
+ CRUSH rule 3 x 829 [45,24]
+ CRUSH rule 3 x 830 [51,96]
+ CRUSH rule 3 x 831 [78,53]
+ CRUSH rule 3 x 832 [28,15]
+ CRUSH rule 3 x 833 [57,72]
+ CRUSH rule 3 x 834 [90,77]
+ CRUSH rule 3 x 835 [14,50]
+ CRUSH rule 3 x 836 [63,100]
+ CRUSH rule 3 x 837 [76,85]
+ CRUSH rule 3 x 838 [106,75]
+ CRUSH rule 3 x 839 [87,12]
+ CRUSH rule 3 x 840 [33,117]
+ CRUSH rule 3 x 841 [110,13]
+ CRUSH rule 3 x 842 [66,97]
+ CRUSH rule 3 x 843 [11,50]
+ CRUSH rule 3 x 844 [74,22]
+ CRUSH rule 3 x 845 [74,20]
+ CRUSH rule 3 x 846 [43,113]
+ CRUSH rule 3 x 847 [62,105]
+ CRUSH rule 3 x 848 [92,19]
+ CRUSH rule 3 x 849 [93,118]
+ CRUSH rule 3 x 850 [83,119]
+ CRUSH rule 3 x 851 [65,56]
+ CRUSH rule 3 x 852 [60,11]
+ CRUSH rule 3 x 853 [88,11]
+ CRUSH rule 3 x 854 [83,52]
+ CRUSH rule 3 x 855 [2,22]
+ CRUSH rule 3 x 856 [40,13]
+ CRUSH rule 3 x 857 [69,110]
+ CRUSH rule 3 x 858 [98,27]
+ CRUSH rule 3 x 859 [56,41]
+ CRUSH rule 3 x 860 [11,30]
+ CRUSH rule 3 x 861 [22,68]
+ CRUSH rule 3 x 862 [22,52]
+ CRUSH rule 3 x 863 [79,32]
+ CRUSH rule 3 x 864 [77,32]
+ CRUSH rule 3 x 865 [119,99]
+ CRUSH rule 3 x 866 [18,39]
+ CRUSH rule 3 x 867 [3,58]
+ CRUSH rule 3 x 868 [100,22]
+ CRUSH rule 3 x 869 [22,86]
+ CRUSH rule 3 x 870 [73,94]
+ CRUSH rule 3 x 871 [84,51]
+ CRUSH rule 3 x 872 [72,91]
+ CRUSH rule 3 x 873 [81,72]
+ CRUSH rule 3 x 874 [21,38]
+ CRUSH rule 3 x 875 [115,27]
+ CRUSH rule 3 x 876 [98,16]
+ CRUSH rule 3 x 877 [80,25]
+ CRUSH rule 3 x 878 [87,114]
+ CRUSH rule 3 x 879 [29,1]
+ CRUSH rule 3 x 880 [23,2]
+ CRUSH rule 3 x 881 [109,97]
+ CRUSH rule 3 x 882 [31,36]
+ CRUSH rule 3 x 883 [102,17]
+ CRUSH rule 3 x 884 [80,23]
+ CRUSH rule 3 x 885 [46,31]
+ CRUSH rule 3 x 886 [2,11]
+ CRUSH rule 3 x 887 [5,85]
+ CRUSH rule 3 x 888 [16,64]
+ CRUSH rule 3 x 889 [84,45]
+ CRUSH rule 3 x 890 [65,50]
+ CRUSH rule 3 x 891 [86,59]
+ CRUSH rule 3 x 892 [64,11]
+ CRUSH rule 3 x 893 [20,118]
+ CRUSH rule 3 x 894 [32,14]
+ CRUSH rule 3 x 895 [40,91]
+ CRUSH rule 3 x 896 [113,29]
+ CRUSH rule 3 x 897 [107,112]
+ CRUSH rule 3 x 898 [76,51]
+ CRUSH rule 3 x 899 [75,66]
+ CRUSH rule 3 x 900 [83,111]
+ CRUSH rule 3 x 901 [66,17]
+ CRUSH rule 3 x 902 [25,5]
+ CRUSH rule 3 x 903 [53,54]
+ CRUSH rule 3 x 904 [50,10]
+ CRUSH rule 3 x 905 [99,106]
+ CRUSH rule 3 x 906 [68,73]
+ CRUSH rule 3 x 907 [109,45]
+ CRUSH rule 3 x 908 [47,24]
+ CRUSH rule 3 x 909 [73,94]
+ CRUSH rule 3 x 910 [71,26]
+ CRUSH rule 3 x 911 [39,62]
+ CRUSH rule 3 x 912 [90,39]
+ CRUSH rule 3 x 913 [29,80]
+ CRUSH rule 3 x 914 [84,99]
+ CRUSH rule 3 x 915 [49,62]
+ CRUSH rule 3 x 916 [32,7]
+ CRUSH rule 3 x 917 [46,91]
+ CRUSH rule 3 x 918 [82,71]
+ CRUSH rule 3 x 919 [13,109]
+ CRUSH rule 3 x 920 [25,100]
+ CRUSH rule 3 x 921 [55,32]
+ CRUSH rule 3 x 922 [33,96]
+ CRUSH rule 3 x 923 [28,79]
+ CRUSH rule 3 x 924 [1,41]
+ CRUSH rule 3 x 925 [113,25]
+ CRUSH rule 3 x 926 [64,65]
+ CRUSH rule 3 x 927 [32,23]
+ CRUSH rule 3 x 928 [13,94]
+ CRUSH rule 3 x 929 [85,60]
+ CRUSH rule 3 x 930 [104,55]
+ CRUSH rule 3 x 931 [46,91]
+ CRUSH rule 3 x 932 [43,54]
+ CRUSH rule 3 x 933 [18,93]
+ CRUSH rule 3 x 934 [68,107]
+ CRUSH rule 3 x 935 [28,23]
+ CRUSH rule 3 x 936 [104,51]
+ CRUSH rule 3 x 937 [110,37]
+ CRUSH rule 3 x 938 [48,69]
+ CRUSH rule 3 x 939 [77,32]
+ CRUSH rule 3 x 940 [76,19]
+ CRUSH rule 3 x 941 [66,10]
+ CRUSH rule 3 x 942 [80,37]
+ CRUSH rule 3 x 943 [75,82]
+ CRUSH rule 3 x 944 [113,15]
+ CRUSH rule 3 x 945 [71,111]
+ CRUSH rule 3 x 946 [37,115]
+ CRUSH rule 3 x 947 [107,48]
+ CRUSH rule 3 x 948 [108,8]
+ CRUSH rule 3 x 949 [46,14]
+ CRUSH rule 3 x 950 [96,13]
+ CRUSH rule 3 x 951 [40,63]
+ CRUSH rule 3 x 952 [114,16]
+ CRUSH rule 3 x 953 [62,53]
+ CRUSH rule 3 x 954 [103,68]
+ CRUSH rule 3 x 955 [42,63]
+ CRUSH rule 3 x 956 [72,6]
+ CRUSH rule 3 x 957 [117,6]
+ CRUSH rule 3 x 958 [23,74]
+ CRUSH rule 3 x 959 [42,87]
+ CRUSH rule 3 x 960 [113,91]
+ CRUSH rule 3 x 961 [116,61]
+ CRUSH rule 3 x 962 [60,41]
+ CRUSH rule 3 x 963 [103,46]
+ CRUSH rule 3 x 964 [66,15]
+ CRUSH rule 3 x 965 [47,108]
+ CRUSH rule 3 x 966 [88,69]
+ CRUSH rule 3 x 967 [71,74]
+ CRUSH rule 3 x 968 [74,75]
+ CRUSH rule 3 x 969 [53,30]
+ CRUSH rule 3 x 970 [3,2]
+ CRUSH rule 3 x 971 [66,19]
+ CRUSH rule 3 x 972 [3,115]
+ CRUSH rule 3 x 973 [113,89]
+ CRUSH rule 3 x 974 [114,73]
+ CRUSH rule 3 x 975 [83,96]
+ CRUSH rule 3 x 976 [81,100]
+ CRUSH rule 3 x 977 [95,76]
+ CRUSH rule 3 x 978 [35,119]
+ CRUSH rule 3 x 979 [98,13]
+ CRUSH rule 3 x 980 [39,113]
+ CRUSH rule 3 x 981 [89,46]
+ CRUSH rule 3 x 982 [19,66]
+ CRUSH rule 3 x 983 [34,107]
+ CRUSH rule 3 x 984 [78,23]
+ CRUSH rule 3 x 985 [99,24]
+ CRUSH rule 3 x 986 [44,33]
+ CRUSH rule 3 x 987 [25,98]
+ CRUSH rule 3 x 988 [79,84]
+ CRUSH rule 3 x 989 [87,60]
+ CRUSH rule 3 x 990 [72,22]
+ CRUSH rule 3 x 991 [90,71]
+ CRUSH rule 3 x 992 [30,75]
+ CRUSH rule 3 x 993 [74,27]
+ CRUSH rule 3 x 994 [74,75]
+ CRUSH rule 3 x 995 [100,45]
+ CRUSH rule 3 x 996 [41,34]
+ CRUSH rule 3 x 997 [89,32]
+ CRUSH rule 3 x 998 [92,41]
+ CRUSH rule 3 x 999 [117,13]
+ CRUSH rule 3 x 1000 [50,31]
+ CRUSH rule 3 x 1001 [83,116]
+ CRUSH rule 3 x 1002 [94,13]
+ CRUSH rule 3 x 1003 [43,54]
+ CRUSH rule 3 x 1004 [89,106]
+ CRUSH rule 3 x 1005 [105,76]
+ CRUSH rule 3 x 1006 [45,5]
+ CRUSH rule 3 x 1007 [19,111]
+ CRUSH rule 3 x 1008 [31,74]
+ CRUSH rule 3 x 1009 [1,51]
+ CRUSH rule 3 x 1010 [31,108]
+ CRUSH rule 3 x 1011 [64,3]
+ CRUSH rule 3 x 1012 [68,81]
+ CRUSH rule 3 x 1013 [5,35]
+ CRUSH rule 3 x 1014 [33,48]
+ CRUSH rule 3 x 1015 [106,99]
+ CRUSH rule 3 x 1016 [107,111]
+ CRUSH rule 3 x 1017 [12,69]
+ CRUSH rule 3 x 1018 [61,60]
+ CRUSH rule 3 x 1019 [27,88]
+ CRUSH rule 3 x 1020 [31,111]
+ CRUSH rule 3 x 1021 [22,36]
+ CRUSH rule 3 x 1022 [73,28]
+ CRUSH rule 3 x 1023 [59,88]
+ rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/src/test/cli/crushtool/test-map-vary-r.crushmap b/src/test/cli/crushtool/test-map-vary-r.crushmap
new file mode 100644
index 0000000..41886ae
Binary files /dev/null and b/src/test/cli/crushtool/test-map-vary-r.crushmap differ
diff --git a/src/test/cli/osdmaptool/clobber.t b/src/test/cli/osdmaptool/clobber.t
index b9986d3..37399fd 100644
--- a/src/test/cli/osdmaptool/clobber.t
+++ b/src/test/cli/osdmaptool/clobber.t
@@ -20,9 +20,9 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
- pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
- pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool stripe_width 0
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool stripe_width 0
max_osd 3
@@ -43,9 +43,9 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
- pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool
- pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool stripe_width 0
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool stripe_width 0
max_osd 1
diff --git a/src/test/cli/osdmaptool/create-print.t b/src/test/cli/osdmaptool/create-print.t
index c8e405e..84c37ff 100644
--- a/src/test/cli/osdmaptool/create-print.t
+++ b/src/test/cli/osdmaptool/create-print.t
@@ -75,9 +75,9 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
- pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
- pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool stripe_width 0
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool stripe_width 0
max_osd 3
@@ -86,7 +86,7 @@
osdmaptool: writing epoch 1 to myosdmap
$ osdmaptool --print myosdmap | grep 'pool 0'
osdmaptool: osdmap file 'myosdmap'
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 66 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 66 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
$ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_rule 55 myosdmap 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
osdmaptool: osdmap file 'myosdmap'
default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
@@ -97,7 +97,7 @@
default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
$ osdmaptool --print myosdmap | grep 'pool 0'
osdmaptool: osdmap file 'myosdmap'
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
$ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_replicated_ruleset 66 --osd_pool_default_crush_rule 55 myosdmap 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
osdmaptool: osdmap file 'myosdmap'
default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
@@ -108,5 +108,5 @@
default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
$ osdmaptool --print myosdmap | grep 'pool 0'
osdmaptool: osdmap file 'myosdmap'
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
$ rm -f myosdmap
diff --git a/src/test/cli/osdmaptool/create-racks.t b/src/test/cli/osdmaptool/create-racks.t
index 4d9b65a..f502806 100644
--- a/src/test/cli/osdmaptool/create-racks.t
+++ b/src/test/cli/osdmaptool/create-racks.t
@@ -788,9 +788,9 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
- pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool
- pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool stripe_width 0
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool stripe_width 0
max_osd 239
@@ -800,7 +800,7 @@
osdmaptool: writing epoch 1 to om
$ osdmaptool --print om | grep 'pool 0'
osdmaptool: osdmap file 'om'
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
$ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_rule 55 om -c $TESTDIR/ceph.conf.withracks 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
osdmaptool: osdmap file 'om'
default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
@@ -811,7 +811,7 @@
default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
$ osdmaptool --print om | grep 'pool 0'
osdmaptool: osdmap file 'om'
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
$ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_replicated_ruleset 66 --osd_pool_default_crush_rule 55 om -c $TESTDIR/ceph.conf.withracks 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
osdmaptool: osdmap file 'om'
default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
@@ -822,5 +822,5 @@
default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
$ osdmaptool --print om | grep 'pool 0'
osdmaptool: osdmap file 'om'
- pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
$ rm -f om
diff --git a/src/test/cli/osdmaptool/crush.t b/src/test/cli/osdmaptool/crush.t
new file mode 100644
index 0000000..5833da8
--- /dev/null
+++ b/src/test/cli/osdmaptool/crush.t
@@ -0,0 +1,10 @@
+ $ osdmaptool --createsimple 3 myosdmap
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: writing epoch 1 to myosdmap
+ $ osdmaptool --export-crush oc myosdmap
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: exported crush map to oc
+ $ osdmaptool --import-crush oc myosdmap
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: imported 486 byte crush map from oc
+ osdmaptool: writing epoch 3 to myosdmap
diff --git a/src/test/cli/osdmaptool/help.t b/src/test/cli/osdmaptool/help.t
index c09768b..2c5a41d 100644
--- a/src/test/cli/osdmaptool/help.t
+++ b/src/test/cli/osdmaptool/help.t
@@ -3,6 +3,10 @@
usage: [--print] [--createsimple <numosd> [--clobber] [--pg_bits <bitsperosd>]] <mapfilename>
--export-crush <file> write osdmap's crush map to <file>
--import-crush <file> replace osdmap's crush map with <file>
+ --test-map-pgs [--pool <poolid>] map all pgs
+ --mark-up-in mark osds up and in (but do not persist)
+ --clear-temp clear pg_temp and primary_temp
+ --test-random do random placements
--test-map-pg <pgid> map a pgid to osds
--test-map-object <objectname> [--pool <poolid>] map an object to osds
[1]
diff --git a/src/test/cli/osdmaptool/simple.t b/src/test/cli/osdmaptool/missing-argument.t
similarity index 64%
rename from src/test/cli/osdmaptool/simple.t
rename to src/test/cli/osdmaptool/missing-argument.t
index 1783a9e..d0740ab 100644
--- a/src/test/cli/osdmaptool/simple.t
+++ b/src/test/cli/osdmaptool/missing-argument.t
@@ -3,6 +3,10 @@
usage: [--print] [--createsimple <numosd> [--clobber] [--pg_bits <bitsperosd>]] <mapfilename>
--export-crush <file> write osdmap's crush map to <file>
--import-crush <file> replace osdmap's crush map with <file>
+ --test-map-pgs [--pool <poolid>] map all pgs
+ --mark-up-in mark osds up and in (but do not persist)
+ --clear-temp clear pg_temp and primary_temp
+ --test-random do random placements
--test-map-pg <pgid> map a pgid to osds
--test-map-object <objectname> [--pool <poolid>] map an object to osds
[1]
diff --git a/src/test/cli/osdmaptool/pool.t b/src/test/cli/osdmaptool/pool.t
new file mode 100644
index 0000000..0adb240
--- /dev/null
+++ b/src/test/cli/osdmaptool/pool.t
@@ -0,0 +1,54 @@
+ $ osdmaptool --createsimple 3 myosdmap
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: writing epoch 1 to myosdmap
+
+#
+# --test-map-object / --pool
+#
+ $ osdmaptool myosdmap --test-map-object foo --pool
+ Option --pool requires an argument.
+ [1]
+
+ $ osdmaptool myosdmap --test-map-object foo --pool bar
+ strict_strtoll: expected integer, got: 'bar'
+ [1]
+
+ $ osdmaptool myosdmap --test-map-object foo --pool 123
+ osdmaptool: osdmap file 'myosdmap'
+ There is no pool 123
+ [1]
+
+ $ osdmaptool myosdmap --test-map-object foo --pool 2
+ osdmaptool: osdmap file 'myosdmap'
+ object 'foo' \-\> 2\..* (re)
+
+ $ osdmaptool myosdmap --test-map-object foo
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: assuming pool 0 (use --pool to override)
+ object 'foo' \-\> 0\..* (re)
+
+#
+# --test-map-pgs / --pool
+#
+ $ osdmaptool myosdmap --test-map-pgs --pool
+ Option --pool requires an argument.
+ [1]
+
+ $ osdmaptool myosdmap --test-map-pgs --pool baz
+ strict_strtoll: expected integer, got: 'baz'
+ [1]
+
+ $ osdmaptool myosdmap --test-map-pgs --pool 123
+ osdmaptool: osdmap file 'myosdmap'
+ There is no pool 123
+ [1]
+
+ $ osdmaptool myosdmap --mark-up-in --test-map-pgs --pool 2 | grep pool
+ osdmaptool: osdmap file 'myosdmap'
+ pool 2 pg_num .* (re)
+
+ $ osdmaptool myosdmap --mark-up-in --test-map-pgs | grep pool
+ osdmaptool: osdmap file 'myosdmap'
+ pool 0 pg_num .* (re)
+ pool 1 pg_num .* (re)
+ pool 2 pg_num .* (re)
diff --git a/src/test/cli/osdmaptool/test-map-pgs.t b/src/test/cli/osdmaptool/test-map-pgs.t
new file mode 100644
index 0000000..b64f2d9
--- /dev/null
+++ b/src/test/cli/osdmaptool/test-map-pgs.t
@@ -0,0 +1,52 @@
+ $ NUM_OSDS=500
+ $ POOL_COUNT=3 # data + metadata + rbd
+ $ SIZE=3
+ $ PG_BITS=4
+#
+# create an osdmap with a few hundred devices and a realistic crushmap
+#
+ $ OSD_MAP="osdmap"
+ $ osdmaptool --osd_pool_default_size $SIZE --pg_bits $PG_BITS --createsimple $NUM_OSDS "$OSD_MAP" > /dev/null
+ osdmaptool: osdmap file 'osdmap'
+ $ CRUSH_MAP="crushmap"
+ $ CEPH_ARGS="--debug-crush 0" crushtool --outfn "$CRUSH_MAP" --build --num_osds $NUM_OSDS node straw 10 rack straw 10 root straw 0
+ $ osdmaptool --import-crush "$CRUSH_MAP" "$OSD_MAP" > /dev/null
+ osdmaptool: osdmap file 'osdmap'
+ $ OUT="$TESTDIR/out"
+#
+# --test-map-pgs
+#
+ $ osdmaptool --mark-up-in --test-map-pgs "$OSD_MAP" > "$OUT"
+ osdmaptool: osdmap file 'osdmap'
+ $ PG_NUM=$(($NUM_OSDS << $PG_BITS))
+ $ grep "pg_num $PG_NUM" "$OUT" || cat $OUT
+ pool 0 pg_num 8000
+ pool 1 pg_num 8000
+ pool 2 pg_num 8000
+ $ TOTAL=$((POOL_COUNT * $PG_NUM))
+ $ PATTERN=$(echo "size $SIZE\t$TOTAL")
+ $ grep "$PATTERN" $OUT || cat "$OUT"
+ size 3\t24000 (esc)
+ $ STATS_CRUSH=$(grep '^ avg ' "$OUT")
+#
+# --test-map-pgs --test-random is expected to change nothing regarding the totals
+#
+ $ osdmaptool --mark-up-in --test-random --test-map-pgs "$OSD_MAP" > "$OUT"
+ osdmaptool: osdmap file 'osdmap'
+ $ PG_NUM=$(($NUM_OSDS << $PG_BITS))
+ $ grep "pg_num $PG_NUM" "$OUT" || cat $OUT
+ pool 0 pg_num 8000
+ pool 1 pg_num 8000
+ pool 2 pg_num 8000
+ $ TOTAL=$((POOL_COUNT * $PG_NUM))
+ $ PATTERN=$(echo "size $SIZE\t$TOTAL")
+ $ grep "$PATTERN" $OUT || cat "$OUT"
+ size 3\t24000 (esc)
+ $ STATS_RANDOM=$(grep '^ avg ' "$OUT")
+# it is almost impossible to get the same stats with random and crush
+# if they are, it most probably means something went wrong somewhere
+ $ test "$STATS_CRUSH" != "$STATS_RANDOM"
+#
+# cleanup
+#
+ $ rm -f "$CRUSH_MAP" "$OSD_MAP" "$OUT"
diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index a7921e2..ca3e747 100644
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -48,7 +48,8 @@
usage trim trim usage (by user, date range)
temp remove remove temporary objects that were created up to
specified date (and optional time)
- gc list dump expired garbage collection objects
+ gc list dump expired garbage collection objects (specify
+ --include-all to list all entries, including unexpired)
gc process manually process garbage
metadata get get metadata info
metadata put put metadata info
diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
index 7a37dee..44cb303 100644
--- a/src/test/cls_rgw/test_cls_rgw.cc
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -409,14 +409,14 @@ TEST(cls_rgw, gc_set)
string marker;
/* list chains, verify truncated */
- ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, entries, &truncated));
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated));
ASSERT_EQ(8, (int)entries.size());
ASSERT_EQ(1, truncated);
entries.clear();
/* list all chains, verify not truncated */
- ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, entries, &truncated));
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, true, entries, &truncated));
ASSERT_EQ(10, (int)entries.size());
ASSERT_EQ(0, truncated);
@@ -483,7 +483,7 @@ TEST(cls_rgw, gc_defer)
string marker;
/* list chains, verify num entries as expected */
- ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
ASSERT_EQ(1, (int)entries.size());
ASSERT_EQ(0, truncated);
@@ -496,7 +496,7 @@ TEST(cls_rgw, gc_defer)
entries.clear();
/* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */
- ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
ASSERT_EQ(0, (int)entries.size());
ASSERT_EQ(0, truncated);
@@ -504,7 +504,7 @@ TEST(cls_rgw, gc_defer)
sleep(5);
/* verify list shows deferred entry */
- ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
ASSERT_EQ(1, (int)entries.size());
ASSERT_EQ(0, truncated);
@@ -519,7 +519,7 @@ TEST(cls_rgw, gc_defer)
entries.clear();
/* verify entry was removed */
- ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
ASSERT_EQ(0, (int)entries.size());
ASSERT_EQ(0, truncated);
diff --git a/src/test/common/histogram.cc b/src/test/common/histogram.cc
new file mode 100644
index 0000000..2fd3cfe
--- /dev/null
+++ b/src/test/common/histogram.cc
@@ -0,0 +1,126 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank <info at inktank.com>
+ *
+ * LGPL2.1 (see COPYING-LGPL2.1) or later
+ */
+
+#include <iostream>
+#include <gtest/gtest.h>
+
+#include "common/histogram.h"
+#include "include/stringify.h"
+
+TEST(Histogram, Basic) {
+ pow2_hist_t h;
+
+ h.add(0);
+ h.add(0);
+ h.add(0);
+ ASSERT_EQ(3, h.h[0]);
+ ASSERT_EQ(1u, h.h.size());
+
+ h.add(1);
+ ASSERT_EQ(3, h.h[0]);
+ ASSERT_EQ(1, h.h[1]);
+ ASSERT_EQ(2u, h.h.size());
+
+ h.add(2);
+ h.add(2);
+ ASSERT_EQ(3, h.h[0]);
+ ASSERT_EQ(1, h.h[1]);
+ ASSERT_EQ(2, h.h[2]);
+ ASSERT_EQ(3u, h.h.size());
+}
+
+TEST(Histogram, Set) {
+ pow2_hist_t h;
+ h.set_bin(0, 12);
+ h.set_bin(2, 12);
+ ASSERT_EQ(12, h.h[0]);
+ ASSERT_EQ(0, h.h[1]);
+ ASSERT_EQ(12, h.h[2]);
+ ASSERT_EQ(3u, h.h.size());
+}
+
+TEST(Histogram, Position) {
+ {
+ pow2_hist_t h;
+ uint64_t lb, ub;
+ h.add(0);
+ ASSERT_EQ(-1, h.get_position_micro(-20, &lb, &ub));
+ }
+ {
+ pow2_hist_t h;
+ h.add(0);
+ uint64_t lb, ub;
+ h.get_position_micro(0, &lb, &ub);
+ ASSERT_EQ(0u, lb);
+ ASSERT_EQ(1000000u, ub);
+ h.add(0);
+ h.add(0);
+ h.add(0);
+ h.get_position_micro(0, &lb, &ub);
+ ASSERT_EQ(0u, lb);
+ ASSERT_EQ(1000000u, ub);
+ }
+ {
+ pow2_hist_t h;
+ h.add(1);
+ h.add(1);
+ uint64_t lb, ub;
+ h.get_position_micro(0, &lb, &ub);
+ ASSERT_EQ(0u, lb);
+ ASSERT_EQ(0u, ub);
+ h.add(0);
+ h.get_position_micro(0, &lb, &ub);
+ ASSERT_EQ(0u, lb);
+ ASSERT_EQ(333333u, ub);
+ h.get_position_micro(1, &lb, &ub);
+ ASSERT_EQ(333333u, lb);
+ ASSERT_EQ(1000000u, ub);
+ }
+ {
+ pow2_hist_t h;
+ h.h.resize(10, 0);
+ h.h[0] = 1;
+ h.h[5] = 1;
+ uint64_t lb, ub;
+ h.get_position_micro(4, &lb, &ub);
+ ASSERT_EQ(500000u, lb);
+ ASSERT_EQ(500000u, ub);
+ }
+ {
+ pow2_hist_t h;
+ h.h.resize(10, 0);
+ h.h[0] = UINT_MAX;
+ h.h[5] = UINT_MAX;
+ uint64_t lb, ub;
+ ASSERT_EQ(500000u, lb);
+ ASSERT_EQ(500000u, ub);
+ }
+}
+
+TEST(Histogram, Decay) {
+ pow2_hist_t h;
+ h.set_bin(0, 123);
+ h.set_bin(3, 12);
+ h.set_bin(5, 1);
+ h.decay(1);
+ ASSERT_EQ(61, h.h[0]);
+ ASSERT_EQ(6, h.h[3]);
+ ASSERT_EQ(4u, h.h.size());
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_histogram &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./unittest_histogram
+ * "
+ * End:
+ */
diff --git a/src/test/common/test_context.cc b/src/test/common/test_context.cc
new file mode 100644
index 0000000..ca745c9
--- /dev/null
+++ b/src/test/common/test_context.cc
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Library Public License for more details.
+ *
+ *
+ */
+#include "gtest/gtest.h"
+#include "include/types.h"
+#include "include/msgr.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+
+TEST(CephContext, do_command)
+{
+ CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
+
+ string key("key");
+ string value("value");
+ cct->_conf->set_val(key.c_str(), value.c_str(), false);
+ cmdmap_t cmdmap;
+ cmdmap["var"] = key;
+
+ {
+ bufferlist out;
+ cct->do_command("config get", cmdmap, "xml", &out);
+ string s(out.c_str(), out.length());
+ EXPECT_EQ("<config get><key>" + value + "</key></config get>", s);
+ }
+
+ {
+ bufferlist out;
+ cct->do_command("config get", cmdmap, "UNSUPPORTED", &out);
+ string s(out.c_str(), out.length());
+ EXPECT_EQ("{ \"key\": \"value\"}", s);
+ }
+
+ cct->put();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make unittest_context &&
+ * valgrind \
+ * --max-stackframe=20000000 --tool=memcheck \
+ * ./unittest_context # --gtest_filter=CephContext.*
+ * "
+ * End:
+ */
diff --git a/src/test/crush/TestCrushWrapper.cc b/src/test/crush/TestCrushWrapper.cc
index d936a6c..d70a525 100644
--- a/src/test/crush/TestCrushWrapper.cc
+++ b/src/test/crush/TestCrushWrapper.cc
@@ -27,6 +27,7 @@
#include "global/global_init.h"
#include "global/global_context.h"
#include "include/Context.h"
+#include "osd/osd_types.h"
#include "crush/CrushWrapper.h"
@@ -62,11 +63,13 @@ TEST(CrushWrapper, get_immediate_parent) {
EXPECT_EQ(0, ret);
EXPECT_EQ("root", loc.first);
EXPECT_EQ("default", loc.second);
+
+ delete c;
}
TEST(CrushWrapper, move_bucket) {
CrushWrapper *c = new CrushWrapper;
-
+
const int ROOT_TYPE = 2;
c->set_type_name(ROOT_TYPE, "root");
const int HOST_TYPE = 1;
@@ -120,6 +123,8 @@ TEST(CrushWrapper, move_bucket) {
EXPECT_EQ("root", loc.first);
EXPECT_EQ("root1", loc.second);
}
+
+ delete c;
}
TEST(CrushWrapper, check_item_loc) {
@@ -185,6 +190,8 @@ TEST(CrushWrapper, check_item_loc) {
EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
EXPECT_EQ(expected_weight, weight);
}
+
+ delete c;
}
TEST(CrushWrapper, update_item) {
@@ -279,6 +286,8 @@ TEST(CrushWrapper, update_item) {
EXPECT_EQ(modified_weight, c->get_item_weightf(item));
EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, other_loc, &weight));
+
+ delete c;
}
TEST(CrushWrapper, insert_item) {
@@ -393,8 +402,9 @@ TEST(CrushWrapper, insert_item) {
{
// create an OSD bucket
int osdno;
- c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
- OSD_TYPE, 0, NULL, NULL, &osdno);
+ int r = c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ 10, 0, NULL, NULL, &osdno);
+ ASSERT_EQ(0, r);
c->set_item_name(osdno, "myosd");
map<string,string> loc;
loc["root"] = "default";
@@ -463,6 +473,143 @@ TEST(CrushWrapper, is_valid_crush_loc) {
}
}
+TEST(CrushWrapper, dump_rules) {
+ CrushWrapper *c = new CrushWrapper;
+
+ const int ROOT_TYPE = 1;
+ c->set_type_name(ROOT_TYPE, "root");
+ const int OSD_TYPE = 0;
+ c->set_type_name(OSD_TYPE, "osd");
+
+ string failure_domain_type("osd");
+ string root_name("default");
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, root_name);
+
+ int item = 0;
+
+ pair <string,string> loc;
+ int ret;
+ loc = c->get_immediate_parent(item, &ret);
+ EXPECT_EQ(-ENOENT, ret);
+
+ {
+ map<string,string> loc;
+ loc["root"] = root_name;
+
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+ "osd.0", loc));
+ }
+
+ // no ruleset by default
+ {
+ Formatter *f = new_formatter("json-pretty");
+ c->dump_rules(f);
+ stringstream ss;
+ f->flush(ss);
+ delete f;
+ EXPECT_EQ("", ss.str());
+ }
+
+ string name("NAME");
+ int ruleset = c->add_simple_ruleset(name, root_name, failure_domain_type,
+ "firstn", pg_pool_t::TYPE_ERASURE);
+ EXPECT_EQ(0, ruleset);
+
+ {
+ Formatter *f = new_formatter("xml");
+ c->dump_rules(f);
+ stringstream ss;
+ f->flush(ss);
+ delete f;
+ EXPECT_EQ((unsigned)0, ss.str().find("<rule><rule_id>0</rule_id><rule_name>NAME</rule_name>"));
+ }
+
+ {
+ Formatter *f = new_formatter("xml");
+ c->dump_rule(ruleset, f);
+ stringstream ss;
+ f->flush(ss);
+ delete f;
+ EXPECT_EQ((unsigned)0, ss.str().find("<rule><rule_id>0</rule_id><rule_name>NAME</rule_name>"));
+ EXPECT_NE(string::npos,
+ ss.str().find("<item_name>default</item_name></step>"));
+ }
+
+ delete c;
+}
+
+TEST(CrushWrapper, distance) {
+ CrushWrapper c;
+ c.create();
+ c.set_type_name(1, "host");
+ c.set_type_name(2, "rack");
+ c.set_type_name(3, "root");
+ int bno;
+ int r = c.add_bucket(0, CRUSH_BUCKET_STRAW,
+ CRUSH_HASH_DEFAULT, 3, 0, NULL,
+ NULL, &bno);
+ ASSERT_EQ(0, r);
+ ASSERT_EQ(-1, bno);
+ c.set_item_name(bno, "default");
+
+ c.set_max_devices(10);
+
+ //JSONFormatter jf(true);
+
+ map<string,string> loc;
+ loc["host"] = "a1";
+ loc["rack"] = "a";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 0, 1, "osd.0", loc);
+
+ loc.clear();
+ loc["host"] = "a2";
+ loc["rack"] = "a";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 1, 1, "osd.1", loc);
+
+ loc.clear();
+ loc["host"] = "b1";
+ loc["rack"] = "b";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 2, 1, "osd.2", loc);
+
+ loc.clear();
+ loc["host"] = "b2";
+ loc["rack"] = "b";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 3, 1, "osd.3", loc);
+
+ vector<pair<string,string> > ol;
+ c.get_full_location_ordered(3, ol);
+ ASSERT_EQ(3u, ol.size());
+ ASSERT_EQ(make_pair(string("host"),string("b2")), ol[0]);
+ ASSERT_EQ(make_pair(string("rack"),string("b")), ol[1]);
+ ASSERT_EQ(make_pair(string("root"),string("default")), ol[2]);
+
+ //c.dump(&jf);
+ //jf.flush(cout);
+
+ multimap<string,string> p;
+ p.insert(make_pair("host","b2"));
+ p.insert(make_pair("rack","b"));
+ p.insert(make_pair("root","default"));
+ ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 0, p));
+ ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 1, p));
+ ASSERT_EQ(2, c.get_common_ancestor_distance(g_ceph_context, 2, p));
+ ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
+ ASSERT_EQ(-ENOENT, c.get_common_ancestor_distance(g_ceph_context, 123, p));
+
+ // make sure a "multipath" location will reflect a minimal
+ // distance for both paths
+ p.insert(make_pair("host","b1"));
+ ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 2, p));
+ ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
+}
+
int main(int argc, char **argv) {
vector<const char*> args;
argv_to_vec(argc, (const char **)argv, args);
@@ -477,9 +624,10 @@ int main(int argc, char **argv) {
/*
* Local Variables:
- * compile-command: "cd ../.. ; make unittest_crush_wrapper &&
+ * compile-command: "cd ../.. ; make -j4 unittest_crush_wrapper &&
* valgrind \
* --max-stackframe=20000000 --tool=memcheck \
- * ./unittest_crush_wrapper --log-to-stderr=true --debug-crush=20 # --gtest_filter=CrushWrapper.insert_item"
+ * ./unittest_crush_wrapper --log-to-stderr=true --debug-crush=20 \
+ * # --gtest_filter=CrushWrapper.insert_item"
* End:
*/
diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h
index ce0af02..678bb62 100644
--- a/src/test/encoding/types.h
+++ b/src/test/encoding/types.h
@@ -36,7 +36,7 @@ TYPEWITHSTRAYDATA(OSDMap::Incremental)
#include "crush/CrushWrapper.h"
TYPE_NOCOPY(CrushWrapper)
-#include "include/histogram.h"
+#include "common/histogram.h"
TYPE(pow2_hist_t)
#include "osd/osd_types.h"
@@ -82,6 +82,15 @@ TYPE(PullOp)
TYPE(PushOp)
TYPE(PushReplyOp)
+#include "osd/ECUtil.h"
+TYPE(ECUtil::HashInfo)
+
+#include "osd/ECMsgTypes.h"
+TYPE(ECSubWrite)
+TYPE(ECSubWriteReply)
+TYPE(ECSubRead)
+TYPE(ECSubReadReply)
+
#include "osd/HitSet.h"
TYPE(ExplicitHashHitSet)
TYPE(ExplicitObjectHitSet)
diff --git a/src/test/osd/ErasureCodeExample.h b/src/test/erasure-code/ErasureCodeExample.h
similarity index 94%
rename from src/test/osd/ErasureCodeExample.h
rename to src/test/erasure-code/ErasureCodeExample.h
index ab2bcf0..2b77d51 100644
--- a/src/test/osd/ErasureCodeExample.h
+++ b/src/test/erasure-code/ErasureCodeExample.h
@@ -21,7 +21,10 @@
#include <errno.h>
#include <algorithm>
#include <sstream>
-#include "osd/ErasureCodeInterface.h"
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "erasure-code/ErasureCodeInterface.h"
#define FIRST_DATA_CHUNK 0
#define SECOND_DATA_CHUNK 1
@@ -35,6 +38,13 @@
class ErasureCodeExample : public ErasureCodeInterface {
public:
virtual ~ErasureCodeExample() {}
+
+ virtual int create_ruleset(const string &name,
+ CrushWrapper &crush,
+ ostream *ss) const {
+ return crush.add_simple_ruleset(name, "default", "host",
+ "indep", pg_pool_t::TYPE_ERASURE, ss);
+ }
virtual int minimum_to_decode(const set<int> &want_to_read,
const set<int> &available_chunks,
diff --git a/src/test/osd/ErasureCodePluginExample.cc b/src/test/erasure-code/ErasureCodePluginExample.cc
similarity index 96%
rename from src/test/osd/ErasureCodePluginExample.cc
rename to src/test/erasure-code/ErasureCodePluginExample.cc
index 6ae61c0..c1b5b3c 100644
--- a/src/test/osd/ErasureCodePluginExample.cc
+++ b/src/test/erasure-code/ErasureCodePluginExample.cc
@@ -16,7 +16,7 @@
#include <unistd.h>
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
#include "ErasureCodeExample.h"
class ErasureCodePluginExample : public ErasureCodePlugin {
diff --git a/src/test/osd/ErasureCodePluginFailToInitialize.cc b/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
similarity index 93%
rename from src/test/osd/ErasureCodePluginFailToInitialize.cc
rename to src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
index cded6ee..d3043a4 100644
--- a/src/test/osd/ErasureCodePluginFailToInitialize.cc
+++ b/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
@@ -15,7 +15,7 @@
*/
#include <errno.h>
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
int __erasure_code_init(char *plugin_name)
{
diff --git a/src/test/osd/ErasureCodePluginFailToRegister.cc b/src/test/erasure-code/ErasureCodePluginFailToRegister.cc
similarity index 93%
rename from src/test/osd/ErasureCodePluginFailToRegister.cc
rename to src/test/erasure-code/ErasureCodePluginFailToRegister.cc
index ea980b7..c26ac5b 100644
--- a/src/test/osd/ErasureCodePluginFailToRegister.cc
+++ b/src/test/erasure-code/ErasureCodePluginFailToRegister.cc
@@ -14,7 +14,7 @@
*
*/
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
int __erasure_code_init(char *plugin_name)
{
diff --git a/src/test/osd/ErasureCodePluginHangs.cc b/src/test/erasure-code/ErasureCodePluginHangs.cc
similarity index 93%
rename from src/test/osd/ErasureCodePluginHangs.cc
rename to src/test/erasure-code/ErasureCodePluginHangs.cc
index ea73786..01c2fa8 100644
--- a/src/test/osd/ErasureCodePluginHangs.cc
+++ b/src/test/erasure-code/ErasureCodePluginHangs.cc
@@ -15,7 +15,7 @@
*/
#include <unistd.h>
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
int __erasure_code_init(char *plugin_name)
{
diff --git a/src/test/osd/ErasureCodePluginMissingEntryPoint.cc b/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
similarity index 100%
rename from src/test/osd/ErasureCodePluginMissingEntryPoint.cc
rename to src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
diff --git a/src/test/erasure-code/Makefile.am b/src/test/erasure-code/Makefile.am
new file mode 100644
index 0000000..30f461c
--- /dev/null
+++ b/src/test/erasure-code/Makefile.am
@@ -0,0 +1,89 @@
+check_SCRIPTS += \
+ test/erasure-code/test-erasure-code.sh
+
+ceph_erasure_code_benchmark_SOURCES = \
+ test/erasure-code/ceph_erasure_code_benchmark.cc
+ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+if LINUX
+ceph_erasure_code_benchmark_LDADD += -ldl
+endif
+bin_DEBUGPROGRAMS += ceph_erasure_code_benchmark
+
+ceph_erasure_code_SOURCES = \
+ test/erasure-code/ceph_erasure_code.cc
+ceph_erasure_code_LDADD = $(LIBOSD) $(LIBCOMMON) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+if LINUX
+ceph_erasure_code_LDADD += -ldl
+endif
+bin_DEBUGPROGRAMS += ceph_erasure_code
+
+libec_example_la_SOURCES = test/erasure-code/ErasureCodePluginExample.cc
+libec_example_la_CFLAGS = ${AM_CFLAGS}
+libec_example_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_example.la
+
+libec_missing_entry_point_la_SOURCES = test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
+libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
+libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la
+
+libec_hangs_la_SOURCES = test/erasure-code/ErasureCodePluginHangs.cc
+libec_hangs_la_CFLAGS = ${AM_CFLAGS}
+libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_hangs.la
+
+libec_fail_to_initialize_la_SOURCES = test/erasure-code/ErasureCodePluginFailToInitialize.cc
+libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la
+
+libec_fail_to_register_la_SOURCES = test/erasure-code/ErasureCodePluginFailToRegister.cc
+libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_fail_to_register.la
+
+unittest_erasure_code_plugin_SOURCES = test/erasure-code/TestErasureCodePlugin.cc
+unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_plugin_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_plugin
+
+unittest_erasure_code_jerasure_SOURCES = \
+ test/erasure-code/TestErasureCodeJerasure.cc \
+ $(libec_jerasure_la_SOURCES)
+unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_jerasure_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_jerasure
+
+unittest_erasure_code_plugin_jerasure_SOURCES = \
+ test/erasure-code/TestErasureCodePluginJerasure.cc
+unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_plugin_jerasure_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_plugin_jerasure
+
+unittest_erasure_code_example_SOURCES = test/erasure-code/TestErasureCodeExample.cc
+noinst_HEADERS += test/erasure-code/ErasureCodeExample.h
+unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_erasure_code_example
+
+noinst_HEADERS += \
+ test/erasure-code/ceph_erasure_code_benchmark.h
diff --git a/src/test/osd/TestErasureCodeExample.cc b/src/test/erasure-code/TestErasureCodeExample.cc
similarity index 89%
rename from src/test/osd/TestErasureCodeExample.cc
rename to src/test/erasure-code/TestErasureCodeExample.cc
index faa863e..4df0762 100644
--- a/src/test/osd/TestErasureCodeExample.cc
+++ b/src/test/erasure-code/TestErasureCodeExample.cc
@@ -14,6 +14,7 @@
*
*/
+#include "include/stringify.h"
#include "global/global_init.h"
#include "ErasureCodeExample.h"
#include "common/ceph_argparse.h"
@@ -201,6 +202,37 @@ TEST(ErasureCodeExample, decode)
EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
}
+TEST(ErasureCodeExample, create_ruleset)
+{
+ CrushWrapper *c = new CrushWrapper;
+ c->create();
+ c->set_type_name(2, "root");
+ c->set_type_name(1, "host");
+ c->set_type_name(0, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ 5, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ stringstream ss;
+ ErasureCodeExample example;
+ EXPECT_EQ(0, example.create_ruleset("myrule", *c, &ss));
+}
+
int main(int argc, char **argv) {
vector<const char*> args;
argv_to_vec(argc, (const char **)argv, args);
diff --git a/src/test/osd/TestErasureCodeJerasure.cc b/src/test/erasure-code/TestErasureCodeJerasure.cc
similarity index 75%
rename from src/test/osd/TestErasureCodeJerasure.cc
rename to src/test/erasure-code/TestErasureCodeJerasure.cc
index 0e2aaa1..87f5966 100644
--- a/src/test/osd/TestErasureCodeJerasure.cc
+++ b/src/test/erasure-code/TestErasureCodeJerasure.cc
@@ -15,8 +15,11 @@
*/
#include <errno.h>
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
#include "global/global_init.h"
-#include "osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h"
+#include "erasure-code/jerasure/ErasureCodeJerasure.h"
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "gtest/gtest.h"
@@ -79,8 +82,7 @@ TYPED_TEST(ErasureCodeTest, encode_decode)
EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2),
encoded,
&decoded));
- // always decode all, regardless of want_to_decode
- EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(2u, decoded.size());
EXPECT_EQ(length, decoded[0].length());
EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
@@ -257,6 +259,85 @@ TEST(ErasureCodeTest, encode)
}
}
+TEST(ErasureCodeTest, create_ruleset)
+{
+ CrushWrapper *c = new CrushWrapper;
+ c->create();
+ int root_type = 2;
+ c->set_type_name(root_type, "root");
+ int host_type = 1;
+ c->set_type_name(host_type, "host");
+ int osd_type = 0;
+ c->set_type_name(osd_type, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ root_type, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 4;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ jerasure.init(parameters);
+ int ruleset = jerasure.create_ruleset("myrule", *c, &ss);
+ EXPECT_EQ(0, ruleset);
+ EXPECT_EQ(-EEXIST, jerasure.create_ruleset("myrule", *c, &ss));
+ //
+ // the minimum that is expected from the created ruleset is to
+ // successfully map get_chunk_count() devices from the crushmap,
+ // at least once.
+ //
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ vector<int> out;
+ int x = 0;
+ c->do_rule(ruleset, x, out, jerasure.get_chunk_count(), weight);
+ ASSERT_EQ(out.size(), jerasure.get_chunk_count());
+ for (unsigned i=0; i<out.size(); ++i)
+ ASSERT_NE(CRUSH_ITEM_NONE, out[i]);
+ }
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ parameters["erasure-code-ruleset-root"] = "BAD";
+ jerasure.init(parameters);
+ EXPECT_EQ(-ENOENT, jerasure.create_ruleset("otherrule", *c, &ss));
+ EXPECT_EQ("root item BAD does not exist", ss.str());
+ }
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ parameters["erasure-code-ruleset-failure-domain"] = "WORSE";
+ jerasure.init(parameters);
+ EXPECT_EQ(-EINVAL, jerasure.create_ruleset("otherrule", *c, &ss));
+ EXPECT_EQ("unknown type WORSE", ss.str());
+ }
+}
+
int main(int argc, char **argv)
{
vector<const char*> args;
diff --git a/src/test/osd/TestErasureCodePlugin.cc b/src/test/erasure-code/TestErasureCodePlugin.cc
similarity index 98%
rename from src/test/osd/TestErasureCodePlugin.cc
rename to src/test/erasure-code/TestErasureCodePlugin.cc
index 46ed4b1..f97b140 100644
--- a/src/test/osd/TestErasureCodePlugin.cc
+++ b/src/test/erasure-code/TestErasureCodePlugin.cc
@@ -18,7 +18,7 @@
#include <signal.h>
#include "common/Thread.h"
#include "global/global_init.h"
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "gtest/gtest.h"
diff --git a/src/test/osd/TestErasureCodePluginJerasure.cc b/src/test/erasure-code/TestErasureCodePluginJerasure.cc
similarity index 97%
rename from src/test/osd/TestErasureCodePluginJerasure.cc
rename to src/test/erasure-code/TestErasureCodePluginJerasure.cc
index 2f55893..0013ce8 100644
--- a/src/test/osd/TestErasureCodePluginJerasure.cc
+++ b/src/test/erasure-code/TestErasureCodePluginJerasure.cc
@@ -16,7 +16,7 @@
#include <errno.h>
#include "global/global_init.h"
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "gtest/gtest.h"
diff --git a/src/test/erasure-code/ceph_erasure_code.cc b/src/test/erasure-code/ceph_erasure_code.cc
new file mode 100644
index 0000000..a364de5
--- /dev/null
+++ b/src/test/erasure-code/ceph_erasure_code.cc
@@ -0,0 +1,166 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "common/Clock.h"
+#include "include/utime.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
+namespace po = boost::program_options;
+
+class ErasureCodeCommand {
+ po::variables_map vm;
+ map<string,string> parameters;
+public:
+ int setup(int argc, char** argv);
+ int run();
+};
+
+int ErasureCodeCommand::setup(int argc, char** argv) {
+
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help,h", "produce help message")
+ ("all", "implies "
+ "--get_chunk_size 1024 "
+ "--get_data_chunk_count "
+ "--get_chunk_count ")
+ ("get_chunk_size", po::value<unsigned int>(),
+ "display get_chunk_size(<object size>)")
+ ("get_data_chunk_count", "display get_data_chunk_count()")
+ ("get_chunk_count", "display get_chunk_count()")
+ ("parameter,P", po::value<vector<string> >(),
+ "parameters")
+ ;
+
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+ po::store(
+ parsed,
+ vm);
+ po::notify(vm);
+
+ vector<const char *> ceph_options, def_args;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ global_init(
+ &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ if (vm.count("help")) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ if (vm.count("parameter")) {
+ const vector<string> &p = vm["parameter"].as< vector<string> >();
+ for (vector<string>::const_iterator i = p.begin();
+ i != p.end();
+ ++i) {
+ std::vector<std::string> strs;
+ boost::split(strs, *i, boost::is_any_of("="));
+ if (strs.size() != 2) {
+ cerr << "--parameter " << *i
+ << " ignored because it does not contain exactly one =" << endl;
+ } else {
+ parameters[strs[0]] = strs[1];
+ }
+ }
+ }
+
+ if (parameters.count("erasure-code-directory") == 0)
+ parameters["erasure-code-directory"] = ".libs";
+ if (parameters.count("erasure-code-plugin") == 0) {
+ cerr << "--parameter erasure-code-plugin=<plugin> is mandatory" << endl;
+ return 1;
+ }
+
+ return 0;
+}
+
+int ErasureCodeCommand::run() {
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ instance.disable_dlclose = true;
+ ErasureCodeInterfaceRef erasure_code;
+ int code = instance.factory(parameters["erasure-code-plugin"],
+ parameters,
+ &erasure_code);
+ if (code)
+ return code;
+
+ if (vm.count("all") || vm.count("get_chunk_size")) {
+ unsigned int object_size = 1024;
+ if (vm.count("get_chunk_size"))
+ object_size = vm["get_chunk_size"].as<unsigned int>();
+ cout << "get_chunk_size(" << object_size << ")\t"
+ << erasure_code->get_chunk_size(object_size) << endl;
+ }
+ if (vm.count("all") || vm.count("get_data_chunk_count"))
+ cout << "get_data_chunk_count\t"
+ << erasure_code->get_data_chunk_count() << endl;
+ if (vm.count("all") || vm.count("get_chunk_count"))
+ cout << "get_chunk_count\t"
+ << erasure_code->get_chunk_count() << endl;
+ return 0;
+}
+
+int main(int argc, char** argv) {
+ ErasureCodeCommand eccommand;
+ int err = eccommand.setup(argc, argv);
+ if (err)
+ return err;
+ return eccommand.run();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make -j4 ceph_erasure_code &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./ceph_erasure_code \
+ * --parameter erasure-code-plugin=jerasure \
+ * --parameter erasure-code-directory=.libs \
+ * --parameter erasure-code-technique=reed_sol_van \
+ * --parameter erasure-code-k=2 \
+ * --parameter erasure-code-m=2 \
+ * --get_chunk_size 1024 \
+ * --get_data_chunk_count \
+ * --get_chunk_count \
+ * "
+ * End:
+ */
diff --git a/src/test/osd/ceph_erasure_code_benchmark.cc b/src/test/erasure-code/ceph_erasure_code_benchmark.cc
similarity index 97%
rename from src/test/osd/ceph_erasure_code_benchmark.cc
rename to src/test/erasure-code/ceph_erasure_code_benchmark.cc
index f4fdaf3..ce768ae 100644
--- a/src/test/osd/ceph_erasure_code_benchmark.cc
+++ b/src/test/erasure-code/ceph_erasure_code_benchmark.cc
@@ -3,7 +3,7 @@
/*
* Ceph - scalable distributed file system
*
- * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing at cloudwatt.com>
*
* Author: Loic Dachary <loic at dachary.org>
*
@@ -23,14 +23,14 @@
#include <boost/program_options/parsers.hpp>
#include <boost/algorithm/string.hpp>
-#include "test/osd/ceph_erasure_code_benchmark.h"
+#include "ceph_erasure_code_benchmark.h"
#include "global/global_context.h"
#include "global/global_init.h"
#include "common/ceph_argparse.h"
#include "common/config.h"
#include "common/Clock.h"
#include "include/utime.h"
-#include "osd/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCodePlugin.h"
namespace po = boost::program_options;
@@ -87,7 +87,7 @@ int ErasureCodeBench::setup(int argc, char** argv) {
const vector<string> &p = vm["parameter"].as< vector<string> >();
for (vector<string>::const_iterator i = p.begin();
i != p.end();
- i++) {
+ ++i) {
std::vector<std::string> strs;
boost::split(strs, *i, boost::is_any_of("="));
if (strs.size() != 2) {
diff --git a/src/test/osd/ceph_erasure_code_benchmark.h b/src/test/erasure-code/ceph_erasure_code_benchmark.h
similarity index 92%
rename from src/test/osd/ceph_erasure_code_benchmark.h
rename to src/test/erasure-code/ceph_erasure_code_benchmark.h
index 8ea60f9..df73aa7 100644
--- a/src/test/osd/ceph_erasure_code_benchmark.h
+++ b/src/test/erasure-code/ceph_erasure_code_benchmark.h
@@ -3,7 +3,7 @@
/*
* Ceph - scalable distributed file system
*
- * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing at cloudwatt.com>
*
* Author: Loic Dachary <loic at dachary.org>
*
diff --git a/src/test/libcephfs/test.cc b/src/test/libcephfs/test.cc
index 825c10c..b42a8e8 100644
--- a/src/test/libcephfs/test.cc
+++ b/src/test/libcephfs/test.cc
@@ -471,6 +471,45 @@ TEST(LibCephFS, Xattrs) {
ceph_close(cmount, fd);
ceph_shutdown(cmount);
+
+}
+
+TEST(LibCephFS, Xattrs_ll) {
+ struct ceph_mount_info *cmount;
+ ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+ ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+ ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+ ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+ char test_xattr_file[256];
+ sprintf(test_xattr_file, "test_xattr_%d", getpid());
+ int fd = ceph_open(cmount, test_xattr_file, O_CREAT, 0666);
+ ASSERT_GT(fd, 0);
+ ceph_close(cmount, fd);
+
+ Inode *root = NULL;
+ Inode *existent_file_handle = NULL;
+ struct stat attr;
+
+ int res = ceph_ll_lookup_root(cmount, &root);
+ ASSERT_EQ(res, 0);
+ res = ceph_ll_lookup(cmount, root, test_xattr_file, &attr, &existent_file_handle, 0, 0);
+ ASSERT_EQ(res, 0);
+
+ const char *valid_name = "user.attrname";
+ const char *value = "attrvalue";
+ char value_buf[256] = { 0 };
+
+ res = ceph_ll_setxattr(cmount, existent_file_handle, valid_name, value, strlen(value), 0, 0, 0);
+ ASSERT_EQ(res, 0);
+
+ res = ceph_ll_getxattr(cmount, existent_file_handle, valid_name, value_buf, 256, 0, 0);
+ ASSERT_EQ(res, (int)strlen(value));
+
+ value_buf[res] = '\0';
+ ASSERT_STREQ(value_buf, value);
+
+ ceph_shutdown(cmount);
}
TEST(LibCephFS, LstatSlashdot) {
diff --git a/src/test/librados/TestCase.cc b/src/test/librados/TestCase.cc
new file mode 100644
index 0000000..70c5ac8
--- /dev/null
+++ b/src/test/librados/TestCase.cc
@@ -0,0 +1,93 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <errno.h>
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+using namespace librados;
+
+std::string RadosTest::pool_name;
+rados_t RadosTest::s_cluster = NULL;
+
+void RadosTest::SetUpTestCase()
+{
+ pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTest::TearDownTestCase()
+{
+ ASSERT_EQ(0, destroy_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTest::SetUp()
+{
+ cluster = RadosTest::s_cluster;
+ ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+ std::string nspace = get_temp_pool_name();
+ rados_ioctx_set_namespace(ioctx, nspace.c_str());
+}
+
+void RadosTest::TearDown()
+{
+ cleanup_default_namespace(ioctx);
+ rados_ioctx_destroy(ioctx);
+}
+
+void RadosTest::cleanup_default_namespace(rados_ioctx_t ioctx)
+{
+ // remove all objects from the default namespace to avoid polluting
+ // other tests
+ rados_ioctx_set_namespace(ioctx, "");
+ rados_list_ctx_t list_ctx;
+ ASSERT_EQ(0, rados_objects_list_open(ioctx, &list_ctx));
+ int r;
+ const char *entry = NULL;
+ const char *key = NULL;
+ while ((r = rados_objects_list_next(list_ctx, &entry, &key)) != -ENOENT) {
+ ASSERT_EQ(0, r);
+ rados_ioctx_locator_set_key(ioctx, key);
+ ASSERT_EQ(0, rados_remove(ioctx, entry));
+ }
+ rados_objects_list_close(list_ctx);
+}
+
+std::string RadosTestPP::pool_name;
+Rados RadosTestPP::s_cluster;
+
+void RadosTestPP::SetUpTestCase()
+{
+ pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPP::TearDownTestCase()
+{
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPP::SetUp()
+{
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+ ns = get_temp_pool_name();
+ ioctx.set_namespace(ns);
+}
+
+void RadosTestPP::TearDown()
+{
+ cleanup_default_namespace(ioctx);
+ ioctx.close();
+}
+
+void RadosTestPP::cleanup_default_namespace(librados::IoCtx ioctx)
+{
+ // remove all objects from the default namespace to avoid polluting
+ // other tests
+ ioctx.set_namespace("");
+ for (ObjectIterator it = ioctx.objects_begin();
+ it != ioctx.objects_end(); ++it) {
+ ioctx.locator_set_key(it->second);
+ ASSERT_EQ(0, ioctx.remove(it->first));
+ }
+}
diff --git a/src/test/librados/TestCase.h b/src/test/librados/TestCase.h
new file mode 100644
index 0000000..31fa040
--- /dev/null
+++ b/src/test/librados/TestCase.h
@@ -0,0 +1,56 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_RADOS_TESTCASE_H
+#define CEPH_TEST_RADOS_TESTCASE_H
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "gtest/gtest.h"
+
+#include <string>
+
+/**
+ * These test cases create a temporary pool that lives as long as the
+ * test case. Each test within a test case gets a new ioctx set to a
+ * unique namespace within the pool.
+ *
+ * Since pool creation and deletion is slow, this allows many tests to
+ * run faster.
+ */
+class RadosTest : public ::testing::Test {
+public:
+ RadosTest() {}
+ virtual ~RadosTest() {}
+protected:
+ static void SetUpTestCase();
+ static void TearDownTestCase();
+ static void cleanup_default_namespace(rados_ioctx_t ioctx);
+ static rados_t s_cluster;
+ static std::string pool_name;
+
+ virtual void SetUp();
+ virtual void TearDown();
+ rados_t cluster;
+ rados_ioctx_t ioctx;
+};
+
+class RadosTestPP : public ::testing::Test {
+public:
+ RadosTestPP() : cluster(s_cluster) {}
+ virtual ~RadosTestPP() {}
+protected:
+ static void SetUpTestCase();
+ static void TearDownTestCase();
+ static void cleanup_default_namespace(librados::IoCtx ioctx);
+ static librados::Rados s_cluster;
+ static std::string pool_name;
+
+ virtual void SetUp();
+ virtual void TearDown();
+ librados::Rados &cluster;
+ librados::IoCtx ioctx;
+ std::string ns;
+};
+
+#endif
diff --git a/src/test/librados/aio.cc b/src/test/librados/aio.cc
index 41318b8..30c794a 100644
--- a/src/test/librados/aio.cc
+++ b/src/test/librados/aio.cc
@@ -14,7 +14,6 @@
using std::ostringstream;
using namespace librados;
using std::pair;
-using std::make_pair;
class AioTestData
{
@@ -1157,7 +1156,6 @@ TEST(LibRadosAio, OmapPP) {
{
boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0, 0));
ObjectWriteOperation op;
- string val = "bar";
to_set["foo"] = header_to_set;
to_set["foo2"] = header_to_set;
to_set["qfoo3"] = header_to_set;
@@ -1188,7 +1186,7 @@ TEST(LibRadosAio, OmapPP) {
TestAlarm alarm;
ASSERT_EQ(0, my_completion->wait_for_complete());
}
- ASSERT_EQ(0, r);
+ ASSERT_EQ(-ECANCELED, r);
}
{
diff --git a/src/test/librados/c_read_operations.cc b/src/test/librados/c_read_operations.cc
new file mode 100644
index 0000000..106f6b4
--- /dev/null
+++ b/src/test/librados/c_read_operations.cc
@@ -0,0 +1,539 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// Tests for the C API coverage of atomic read operations
+
+#include <errno.h>
+#include <string>
+
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+const char *data = "testdata";
+const char *obj = "testobj";
+const int len = strlen(data);
+
+class CReadOpsTest : public RadosTest {
+protected:
+ void write_object() {
+ // Create an object and write to it
+ ASSERT_EQ(len, rados_write(ioctx, obj, data, len, 0));
+ }
+ void remove_object() {
+ ASSERT_EQ(0, rados_remove(ioctx, obj));
+ }
+ int cmp_xattr(const char *xattr, const char *value, size_t value_len,
+ uint8_t cmp_op)
+ {
+ rados_read_op_t op = rados_create_read_op();
+ rados_read_op_cmpxattr(op, xattr, cmp_op, value, value_len);
+ int r = rados_read_op_operate(op, ioctx, obj, 0);
+ rados_release_read_op(op);
+ return r;
+ }
+
+ void fetch_and_verify_omap_vals(char const* const* keys,
+ char const* const* vals,
+ const size_t *lens,
+ size_t len)
+ {
+ rados_omap_iter_t iter_vals, iter_keys, iter_vals_by_key;
+ int r_vals, r_keys, r_vals_by_key;
+ rados_read_op_t op = rados_create_read_op();
+ rados_read_op_omap_get_vals(op, NULL, NULL, 100, &iter_vals, &r_vals);
+ rados_read_op_omap_get_keys(op, NULL, 100, &iter_keys, &r_keys);
+ rados_read_op_omap_get_vals_by_keys(op, keys, len,
+ &iter_vals_by_key, &r_vals_by_key);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+ ASSERT_EQ(0, r_vals);
+ ASSERT_EQ(0, r_keys);
+ ASSERT_EQ(0, r_vals_by_key);
+
+ const char *zeros[len];
+ size_t zero_lens[len];
+ memset(zeros, 0, len);
+ memset(zero_lens, 0, len * sizeof(size_t));
+ compare_omap_vals(keys, vals, lens, len, iter_vals);
+ compare_omap_vals(keys, zeros, zero_lens, len, iter_keys);
+ compare_omap_vals(keys, vals, lens, len, iter_vals_by_key);
+ }
+
+ void compare_omap_vals(char const* const* keys,
+ char const* const* vals,
+ const size_t *lens,
+ size_t len,
+ rados_omap_iter_t iter)
+ {
+ size_t i = 0;
+ char *key = NULL;
+ char *val = NULL;
+ size_t val_len = 0;
+ while (i < len) {
+ ASSERT_EQ(0, rados_omap_get_next(iter, &key, &val, &val_len));
+ if (len == 0 && key == NULL && val == NULL)
+ break;
+ if (key)
+ EXPECT_EQ(std::string(keys[i]), std::string(key));
+ else
+ EXPECT_EQ(keys[i], key);
+ ASSERT_EQ(0, memcmp(vals[i], val, val_len));
+ ASSERT_EQ(lens[i], val_len);
+ ++i;
+ }
+ ASSERT_EQ(i, len);
+ ASSERT_EQ(0, rados_omap_get_next(iter, &key, &val, &val_len));
+ ASSERT_EQ((char*)NULL, key);
+ ASSERT_EQ((char*)NULL, val);
+ ASSERT_EQ(0u, val_len);
+ rados_omap_get_end(iter);
+ }
+
+ void compare_xattrs(char const* const* keys,
+ char const* const* vals,
+ const size_t *lens,
+ size_t len,
+ rados_xattrs_iter_t iter)
+ {
+ size_t i = 0;
+ char *key = NULL;
+ char *val = NULL;
+ size_t val_len = 0;
+ while (i < len) {
+ ASSERT_EQ(0, rados_getxattrs_next(iter, (const char**) &key,
+ (const char**) &val, &val_len));
+ if (len == 0 && key == NULL && val == NULL)
+ break;
+ EXPECT_EQ(std::string(keys[i]), std::string(key));
+ EXPECT_EQ(0, memcmp(vals[i], val, val_len));
+ EXPECT_EQ(lens[i], val_len);
+ ++i;
+ }
+ ASSERT_EQ(i, len);
+ ASSERT_EQ(0, rados_getxattrs_next(iter, (const char**)&key,
+ (const char**)&val, &val_len));
+ ASSERT_EQ((char*)NULL, key);
+ ASSERT_EQ((char*)NULL, val);
+ ASSERT_EQ(0u, val_len);
+ rados_getxattrs_end(iter);
+ }
+};
+
+TEST_F(CReadOpsTest, NewDelete) {
+ rados_read_op_t op = rados_create_read_op();
+ ASSERT_TRUE(op);
+ rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, SetOpFlags) {
+ write_object();
+
+ rados_read_op_t op = rados_create_read_op();
+ size_t bytes_read = 0;
+ char *out = NULL;
+ int rval = 0;
+ rados_read_op_exec(op, "rbd", "get_id", NULL, 0, &out,
+ &bytes_read, &rval);
+ rados_read_op_set_flags(op, LIBRADOS_OP_FLAG_FAILOK);
+ EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ EXPECT_EQ(-EIO, rval);
+ EXPECT_EQ(0u, bytes_read);
+ EXPECT_EQ((char*)NULL, out);
+ rados_release_read_op(op);
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, AssertExists) {
+ rados_read_op_t op = rados_create_read_op();
+ rados_read_op_assert_exists(op);
+
+ ASSERT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+
+ op = rados_create_read_op();
+ rados_read_op_assert_exists(op);
+
+ rados_completion_t completion;
+ ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
+ ASSERT_EQ(0, rados_aio_read_op_operate(op, ioctx, completion, obj, 0));
+ rados_aio_wait_for_complete(completion);
+ ASSERT_EQ(-ENOENT, rados_aio_get_return_value(completion));
+ rados_release_read_op(op);
+
+ write_object();
+
+ op = rados_create_read_op();
+ rados_read_op_assert_exists(op);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, CmpXattr) {
+ write_object();
+
+ char buf[len];
+ memset(buf, 0xcc, sizeof(buf));
+
+ const char *xattr = "test";
+ rados_setxattr(ioctx, obj, xattr, buf, sizeof(buf));
+
+ // equal value
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_EQ));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_NE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GTE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LTE));
+
+ // < value
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_EQ));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_NE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_GT));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_GTE));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_LT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_LTE));
+
+ // > value
+ memset(buf, 0xcd, sizeof(buf));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_EQ));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_NE));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GTE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LT));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LTE));
+
+ // check that null bytes are compared correctly
+ rados_setxattr(ioctx, obj, xattr, "\0\0", 2);
+ buf[0] = '\0';
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_EQ));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_NE));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GTE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LT));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LTE));
+
+ buf[1] = '\0';
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_EQ));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_NE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GTE));
+ EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LT));
+ EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LTE));
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, Read) {
+ write_object();
+
+ char buf[len];
+ // check that using read_ops returns the same data with
+ // or without bytes_read and rval out params
+ {
+ rados_read_op_t op = rados_create_read_op();
+ rados_read_op_read(op, 0, len, buf, NULL, NULL);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ {
+ rados_read_op_t op = rados_create_read_op();
+ int rval;
+ rados_read_op_read(op, 0, len, buf, NULL, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ {
+ rados_read_op_t op = rados_create_read_op();
+ size_t bytes_read = 0;
+ rados_read_op_read(op, 0, len, buf, &bytes_read, NULL);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(len, (int)bytes_read);
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ {
+ rados_read_op_t op = rados_create_read_op();
+ size_t bytes_read = 0;
+ int rval;
+ rados_read_op_read(op, 0, len, buf, &bytes_read, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(len, (int)bytes_read);
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, ShortRead) {
+ write_object();
+
+ char buf[len * 2];
+ // check that using read_ops returns the same data with
+ // or without bytes_read and rval out params
+ {
+ rados_read_op_t op = rados_create_read_op();
+ rados_read_op_read(op, 0, len * 2, buf, NULL, NULL);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ {
+ rados_read_op_t op = rados_create_read_op();
+ int rval;
+ rados_read_op_read(op, 0, len * 2, buf, NULL, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ {
+ rados_read_op_t op = rados_create_read_op();
+ size_t bytes_read = 0;
+ rados_read_op_read(op, 0, len * 2, buf, &bytes_read, NULL);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(len, (int)bytes_read);
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ {
+ rados_read_op_t op = rados_create_read_op();
+ size_t bytes_read = 0;
+ int rval;
+ rados_read_op_read(op, 0, len * 2, buf, &bytes_read, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ ASSERT_EQ(len, (int)bytes_read);
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(data, buf, len));
+ rados_release_read_op(op);
+ }
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, Exec) {
+ // create object so we don't get -ENOENT
+ write_object();
+
+ rados_read_op_t op = rados_create_read_op();
+ ASSERT_TRUE(op);
+ size_t bytes_read = 0;
+ char *out = NULL;
+ int rval = 0;
+ rados_read_op_exec(op, "rbd", "get_all_features", NULL, 0, &out,
+ &bytes_read, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+ EXPECT_EQ(0, rval);
+ EXPECT_TRUE(out);
+ uint64_t features;
+ EXPECT_EQ(sizeof(features), bytes_read);
+ // make sure buffer is at least as long as it claims
+ ASSERT_TRUE(out[bytes_read-1] == out[bytes_read-1]);
+ rados_buffer_free(out);
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, ExecUserBuf) {
+ // create object so we don't get -ENOENT
+ write_object();
+
+ rados_read_op_t op = rados_create_read_op();
+ size_t bytes_read = 0;
+ uint64_t features;
+ char out[sizeof(features)];
+ int rval = 0;
+ rados_read_op_exec_user_buf(op, "rbd", "get_all_features", NULL, 0, out,
+ sizeof(out), &bytes_read, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+ EXPECT_EQ(0, rval);
+ EXPECT_EQ(sizeof(features), bytes_read);
+
+ // buffer too short
+ bytes_read = 1024;
+ op = rados_create_read_op();
+ rados_read_op_exec_user_buf(op, "rbd", "get_all_features", NULL, 0, out,
+ sizeof(features) - 1, &bytes_read, &rval);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ EXPECT_EQ(0u, bytes_read);
+ EXPECT_EQ(-ERANGE, rval);
+
+ // input buffer and no rval or bytes_read
+ op = rados_create_read_op();
+ rados_read_op_exec_user_buf(op, "rbd", "get_all_features", out, sizeof(out),
+ out, sizeof(out), NULL, NULL);
+ ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, Stat) {
+ rados_read_op_t op = rados_create_read_op();
+ uint64_t size = 1;
+ int rval;
+ rados_read_op_stat(op, &size, NULL, &rval);
+ EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+ EXPECT_EQ(-EIO, rval);
+ EXPECT_EQ(1u, size);
+ rados_release_read_op(op);
+
+ write_object();
+
+ op = rados_create_read_op();
+ rados_read_op_stat(op, &size, NULL, &rval);
+ EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ EXPECT_EQ(0, rval);
+ EXPECT_EQ(len, (int)size);
+ rados_release_read_op(op);
+
+ op = rados_create_read_op();
+ rados_read_op_stat(op, NULL, NULL, NULL);
+ EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+
+ remove_object();
+
+ op = rados_create_read_op();
+ rados_read_op_stat(op, NULL, NULL, NULL);
+ EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+ rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, Omap) {
+ char *keys[] = {(char*)"bar",
+ (char*)"foo",
+ (char*)"test1",
+ (char*)"test2"};
+ char *vals[] = {(char*)"",
+ (char*)"\0",
+ (char*)"abc",
+ (char*)"va\0lue"};
+ size_t lens[] = {0, 1, 3, 6};
+
+ // check for -ENOENT before the object exists and when it exists
+ // with no omap entries
+ rados_omap_iter_t iter_vals;
+ rados_read_op_t rop = rados_create_read_op();
+ rados_read_op_omap_get_vals(rop, "", "", 10, &iter_vals, NULL);
+ ASSERT_EQ(-ENOENT, rados_read_op_operate(rop, ioctx, obj, 0));
+ rados_release_read_op(rop);
+ compare_omap_vals(NULL, NULL, NULL, 0, iter_vals);
+
+ write_object();
+
+ fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+ // write and check for the k/v pairs
+ rados_write_op_t op = rados_create_write_op();
+ rados_write_op_omap_set(op, keys, vals, lens, 4);
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+ rados_release_write_op(op);
+
+ fetch_and_verify_omap_vals(keys, vals, lens, 4);
+
+ rados_omap_iter_t iter_keys;
+ int r_vals = -1, r_keys = -1;
+ rop = rados_create_read_op();
+ rados_read_op_omap_get_vals(rop, "", "test", 1, &iter_vals, &r_vals);
+ rados_read_op_omap_get_keys(rop, "test", 1, &iter_keys, &r_keys);
+ ASSERT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+ rados_release_read_op(rop);
+ EXPECT_EQ(0, r_vals);
+ EXPECT_EQ(0, r_keys);
+
+ compare_omap_vals(&keys[2], &vals[2], &lens[2], 1, iter_vals);
+ compare_omap_vals(&keys[2], &vals[0], &lens[0], 1, iter_keys);
+
+ // check omap_cmp finds all expected values
+ rop = rados_create_read_op();
+ int rvals[4];
+ for (int i = 0; i < 4; ++i)
+ rados_read_op_omap_cmp(rop, keys[i], LIBRADOS_CMPXATTR_OP_EQ,
+ vals[i], lens[i], &rvals[i]);
+ EXPECT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+ rados_release_read_op(rop);
+ for (int i = 0; i < 4; ++i)
+ EXPECT_EQ(0, rvals[i]);
+
+ // try to remove keys with a guard that should fail
+ op = rados_create_write_op();
+ rados_write_op_omap_cmp(op, keys[2], LIBRADOS_CMPXATTR_OP_LT,
+ vals[2], lens[2], &r_vals);
+ rados_write_op_omap_rm_keys(op, keys, 2);
+ EXPECT_EQ(-ECANCELED, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+ rados_release_write_op(op);
+ ASSERT_EQ(-ECANCELED, r_vals);
+
+ // verifying the keys are still there, and then remove them
+ op = rados_create_write_op();
+ rados_write_op_omap_cmp(op, keys[0], LIBRADOS_CMPXATTR_OP_EQ,
+ vals[0], lens[0], NULL);
+ rados_write_op_omap_cmp(op, keys[1], LIBRADOS_CMPXATTR_OP_EQ,
+ vals[1], lens[1], NULL);
+ rados_write_op_omap_rm_keys(op, keys, 2);
+ EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+ rados_release_write_op(op);
+
+ fetch_and_verify_omap_vals(&keys[2], &vals[2], &lens[2], 2);
+
+ // clear the rest and check there are none left
+ op = rados_create_write_op();
+ rados_write_op_omap_clear(op);
+ EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+ rados_release_write_op(op);
+
+ fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+ remove_object();
+}
+
+TEST_F(CReadOpsTest, GetXattrs) {
+ write_object();
+
+ char *keys[] = {(char*)"bar",
+ (char*)"foo",
+ (char*)"test1",
+ (char*)"test2"};
+ char *vals[] = {(char*)"",
+ (char*)"\0",
+ (char*)"abc",
+ (char*)"va\0lue"};
+ size_t lens[] = {0, 1, 3, 6};
+
+ int rval = 1;
+ rados_read_op_t op = rados_create_read_op();
+ rados_xattrs_iter_t it;
+ rados_read_op_getxattrs(op, &it, &rval);
+ EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ EXPECT_EQ(0, rval);
+ rados_release_read_op(op);
+ compare_xattrs(keys, vals, lens, 0, it);
+
+ for (int i = 0; i < 4; ++i)
+ rados_setxattr(ioctx, obj, keys[i], vals[i], lens[i]);
+
+ rval = 1;
+ op = rados_create_read_op();
+ rados_read_op_getxattrs(op, &it, &rval);
+ EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+ EXPECT_EQ(0, rval);
+ rados_release_read_op(op);
+ compare_xattrs(keys, vals, lens, 4, it);
+
+ remove_object();
+}
diff --git a/src/test/librados/c_write_operations.cc b/src/test/librados/c_write_operations.cc
index 906a386..4dede36 100644
--- a/src/test/librados/c_write_operations.cc
+++ b/src/test/librados/c_write_operations.cc
@@ -22,7 +22,7 @@ TEST(LibRadosCWriteOps, assertExists) {
rados_write_op_assert_exists(op);
// -2, ENOENT
- ASSERT_EQ(-2, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(-2, rados_write_op_operate(op, ioctx, "test", NULL, 0));
rados_release_write_op(op);
rados_write_op_t op2 = rados_create_write_op();
@@ -31,7 +31,7 @@ TEST(LibRadosCWriteOps, assertExists) {
rados_completion_t completion;
ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
- ASSERT_EQ(0, rados_aio_write_op_operate(op2, ioctx, completion, "test", NULL));
+ ASSERT_EQ(0, rados_aio_write_op_operate(op2, ioctx, completion, "test", NULL, 0));
rados_aio_wait_for_complete(completion);
ASSERT_EQ(-2, rados_aio_get_return_value(completion));
@@ -52,7 +52,7 @@ TEST(LibRadosCWriteOps, Xattrs) {
ASSERT_TRUE(op);
rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
rados_write_op_setxattr(op, "key", "value", 5);
- ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
rados_release_write_op(op);
// Check that xattr exists, if it does, delete it.
@@ -61,7 +61,7 @@ TEST(LibRadosCWriteOps, Xattrs) {
rados_write_op_create(op, LIBRADOS_CREATE_IDEMPOTENT, NULL);
rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
rados_write_op_rmxattr(op, "key");
- ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
rados_release_write_op(op);
// Check the xattr exits, if it does, add it again (will fail) with -125
@@ -70,9 +70,10 @@ TEST(LibRadosCWriteOps, Xattrs) {
ASSERT_TRUE(op);
rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
rados_write_op_setxattr(op, "key", "value", 5);
- ASSERT_EQ(-125, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(-125, rados_write_op_operate(op, ioctx, "test", NULL, 0));
rados_release_write_op(op);
+ rados_ioctx_destroy(ioctx);
ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
@@ -89,7 +90,7 @@ TEST(LibRadosCWriteOps, Write) {
rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
rados_write_op_write(op, "four", 4, 0);
rados_write_op_write_full(op, "hi", 2);
- ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
char hi[4];
ASSERT_EQ(2, rados_read(ioctx, "test", hi, 4, 0));
rados_release_write_op(op);
@@ -99,7 +100,7 @@ TEST(LibRadosCWriteOps, Write) {
ASSERT_TRUE(op);
rados_write_op_truncate(op, 1);
rados_write_op_append(op, "hi", 2);
- ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
ASSERT_EQ(3, rados_read(ioctx, "test", hi, 4, 0));
rados_release_write_op(op);
@@ -108,10 +109,34 @@ TEST(LibRadosCWriteOps, Write) {
ASSERT_TRUE(op);
rados_write_op_zero(op, 0, 3);
rados_write_op_remove(op);
- ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
// ENOENT
ASSERT_EQ(-2, rados_read(ioctx, "test", hi, 4, 0));
rados_release_write_op(op);
+ rados_ioctx_destroy(ioctx);
+ ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Exec) {
+ rados_t cluster;
+ rados_ioctx_t ioctx;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+ rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+ int rval = 1;
+ rados_write_op_t op = rados_create_write_op();
+ rados_write_op_exec(op, "hello", "record_hello", "test", 4, &rval);
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+ rados_release_write_op(op);
+ ASSERT_EQ(0, rval);
+
+ char hi[100];
+ ASSERT_EQ(12, rados_read(ioctx, "test", hi, 100, 0));
+ hi[12] = '\0';
+ ASSERT_EQ(0, strcmp("Hello, test!", hi));
+
+ rados_ioctx_destroy(ioctx);
ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
diff --git a/src/test/librados/cmd.cc b/src/test/librados/cmd.cc
index 79dc658..4f327a0 100644
--- a/src/test/librados/cmd.cc
+++ b/src/test/librados/cmd.cc
@@ -1,3 +1,6 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
#include "mds/mdstypes.h"
#include "include/buffer.h"
#include "include/rbd_types.h"
@@ -22,8 +25,7 @@ using std::string;
TEST(LibRadosCmd, MonDescribe) {
rados_t cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+ ASSERT_EQ("", connect_cluster(&cluster));
char *buf, *st;
size_t buflen, stlen;
@@ -53,30 +55,24 @@ TEST(LibRadosCmd, MonDescribe) {
//ASSERT_LT(0u, stlen);
rados_buffer_free(buf);
rados_buffer_free(st);
-
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ rados_shutdown(cluster);
}
TEST(LibRadosCmd, MonDescribePP) {
Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
-
+ ASSERT_EQ("", connect_cluster_pp(cluster));
bufferlist inbl, outbl;
string outs;
ASSERT_EQ(0, cluster.mon_command("{\"prefix\": \"get_command_descriptions\"}",
inbl, &outbl, &outs));
ASSERT_LT(0u, outbl.length());
ASSERT_LE(0u, outs.length());
-
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ cluster.shutdown();
}
TEST(LibRadosCmd, OSDCmd) {
rados_t cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
-
+ ASSERT_EQ("", connect_cluster(&cluster));
int r;
char *buf, *st;
size_t buflen, stlen;
@@ -95,8 +91,7 @@ TEST(LibRadosCmd, OSDCmd) {
ASSERT_TRUE((r == 0 && buflen > 0) || (r == -ENXIO && buflen == 0));
rados_buffer_free(buf);
rados_buffer_free(st);
-
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ rados_shutdown(cluster);
}
TEST(LibRadosCmd, PGCmd) {
@@ -172,9 +167,7 @@ void log_cb(void *arg,
TEST(LibRadosCmd, WatchLog) {
rados_t cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
-
+ ASSERT_EQ("", connect_cluster(&cluster));
char *buf, *st;
char *cmd[2];
cmd[1] = NULL;
@@ -213,7 +206,5 @@ TEST(LibRadosCmd, WatchLog) {
ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
sleep(2);
ASSERT_FALSE(l.contains("fourxx"));
-
-
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ rados_shutdown(cluster);
}
diff --git a/src/test/librados/io.cc b/src/test/librados/io.cc
index f1bcc06..75742bf 100644
--- a/src/test/librados/io.cc
+++ b/src/test/librados/io.cc
@@ -1,6 +1,10 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include <errno.h>
#include "gtest/gtest.h"
@@ -8,63 +12,163 @@
using namespace librados;
using std::string;
-TEST(LibRadosIo, SimpleWrite) {
+typedef RadosTest LibRadosIo;
+typedef RadosTestPP LibRadosIoPP;
+
+TEST_F(LibRadosIo, SimpleWrite) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
rados_ioctx_set_namespace(ioctx, "nspace");
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, SimpleWritePP) {
+TEST_F(LibRadosIoPP, SimpleWritePP) {
char buf[128];
- std::string pool_name = get_temp_pool_name();
- Rados cluster;
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl, sizeof(buf), 0));
ioctx.set_namespace("nspace");
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl, sizeof(buf), 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, RoundTrip) {
+TEST_F(LibRadosIoPP, ReadOpPP) {
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl;
+ bl.append(buf, sizeof(buf));
+ ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl, sizeof(buf), 0));
+
+ {
+ bufferlist op_bl;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), NULL, NULL);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+ ASSERT_EQ(sizeof(buf), op_bl.length());
+ ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist read_bl, op_bl;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), &read_bl, NULL);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+ ASSERT_EQ(sizeof(buf), read_bl.length());
+ ASSERT_EQ(sizeof(buf), op_bl.length());
+ ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+ ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist op_bl;
+ int rval = 1000;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), NULL, &rval);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+ ASSERT_EQ(sizeof(buf), op_bl.length());
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist read_bl, op_bl;
+ int rval = 1000;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), &read_bl, &rval);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+ ASSERT_EQ(sizeof(buf), read_bl.length());
+ ASSERT_EQ(sizeof(buf), op_bl.length());
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+ ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist read_bl1, read_bl2, op_bl;
+ int rval1 = 1000, rval2 = 1002;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), &read_bl1, &rval1);
+ op.read(0, sizeof(buf), &read_bl2, &rval2);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+ ASSERT_EQ(sizeof(buf), read_bl1.length());
+ ASSERT_EQ(sizeof(buf), read_bl2.length());
+ ASSERT_EQ(sizeof(buf) * 2, op_bl.length());
+ ASSERT_EQ(0, rval1);
+ ASSERT_EQ(0, rval2);
+ ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+ ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+ ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+ ASSERT_EQ(0, memcmp(op_bl.c_str() + sizeof(buf), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist op_bl;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), NULL, NULL);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+ ASSERT_EQ(sizeof(buf), op_bl.length());
+ ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist read_bl;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), &read_bl, NULL);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_EQ(sizeof(buf), read_bl.length());
+ ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ int rval = 1000;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), NULL, &rval);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_EQ(0, rval);
+ }
+
+ {
+ bufferlist read_bl;
+ int rval = 1000;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), &read_bl, &rval);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_EQ(sizeof(buf), read_bl.length());
+ ASSERT_EQ(0, rval);
+ ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+ }
+
+ {
+ bufferlist read_bl1, read_bl2;
+ int rval1 = 1000, rval2 = 1002;
+ ObjectReadOperation op;
+ op.read(0, sizeof(buf), &read_bl1, &rval1);
+ op.read(0, sizeof(buf), &read_bl2, &rval2);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_EQ(sizeof(buf), read_bl1.length());
+ ASSERT_EQ(sizeof(buf), read_bl2.length());
+ ASSERT_EQ(0, rval1);
+ ASSERT_EQ(0, rval2);
+ ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+ ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+ }
+}
+
+TEST_F(LibRadosIo, RoundTrip) {
char buf[128];
char buf2[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
memset(buf2, 0, sizeof(buf2));
ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, RoundTripPP) {
+TEST_F(LibRadosIoPP, RoundTripPP) {
char buf[128];
char buf2[128];
Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
@@ -72,19 +176,12 @@ TEST(LibRadosIo, RoundTripPP) {
bufferlist cl;
ASSERT_EQ((int)sizeof(buf2), ioctx.read("foo", cl, sizeof(buf), 0));
ASSERT_EQ(0, memcmp(buf, cl.c_str(), sizeof(buf)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, OverlappingWriteRoundTrip) {
+TEST_F(LibRadosIo, OverlappingWriteRoundTrip) {
char buf[128];
char buf2[64];
char buf3[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
memset(buf2, 0xdd, sizeof(buf2));
@@ -93,18 +190,11 @@ TEST(LibRadosIo, OverlappingWriteRoundTrip) {
ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
ASSERT_EQ(0, memcmp(buf3 + sizeof(buf2), buf, sizeof(buf) - sizeof(buf2)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, OverlappingWriteRoundTripPP) {
+TEST_F(LibRadosIoPP, OverlappingWriteRoundTripPP) {
char buf[128];
char buf2[64];
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -117,19 +207,12 @@ TEST(LibRadosIo, OverlappingWriteRoundTripPP) {
ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
ASSERT_EQ(0, memcmp(bl3.c_str() + sizeof(buf2), buf, sizeof(buf) - sizeof(buf2)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, WriteFullRoundTrip) {
+TEST_F(LibRadosIo, WriteFullRoundTrip) {
char buf[128];
char buf2[64];
char buf3[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
memset(buf2, 0xdd, sizeof(buf2));
@@ -137,18 +220,11 @@ TEST(LibRadosIo, WriteFullRoundTrip) {
memset(buf3, 0xdd, sizeof(buf3));
ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
ASSERT_EQ(0, memcmp(buf2, buf2, sizeof(buf2)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, WriteFullRoundTripPP) {
+TEST_F(LibRadosIoPP, WriteFullRoundTripPP) {
char buf[128];
char buf2[64];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -160,19 +236,12 @@ TEST(LibRadosIo, WriteFullRoundTripPP) {
bufferlist bl3;
ASSERT_EQ((int)sizeof(buf2), ioctx.read("foo", bl3, sizeof(buf), 0));
ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, AppendRoundTrip) {
+TEST_F(LibRadosIo, AppendRoundTrip) {
char buf[64];
char buf2[64];
char buf3[sizeof(buf) + sizeof(buf2)];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xde, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_append(ioctx, "foo", buf, sizeof(buf)));
memset(buf2, 0xad, sizeof(buf2));
@@ -181,18 +250,11 @@ TEST(LibRadosIo, AppendRoundTrip) {
ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
ASSERT_EQ(0, memcmp(buf3 + sizeof(buf), buf2, sizeof(buf2)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, AppendRoundTripPP) {
+TEST_F(LibRadosIoPP, AppendRoundTripPP) {
char buf[64];
char buf2[64];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xde, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -207,35 +269,21 @@ TEST(LibRadosIo, AppendRoundTripPP) {
const char *bl3_str = bl3.c_str();
ASSERT_EQ(0, memcmp(bl3_str, buf, sizeof(buf)));
ASSERT_EQ(0, memcmp(bl3_str + sizeof(buf), buf2, sizeof(buf2)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, TruncTest) {
+TEST_F(LibRadosIo, TruncTest) {
char buf[128];
char buf2[sizeof(buf)];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xaa, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_append(ioctx, "foo", buf, sizeof(buf)));
ASSERT_EQ(0, rados_trunc(ioctx, "foo", sizeof(buf) / 2));
memset(buf2, 0, sizeof(buf2));
ASSERT_EQ((int)(sizeof(buf)/2), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)/2));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, TruncTestPP) {
+TEST_F(LibRadosIoPP, TruncTestPP) {
char buf[128];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xaa, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
@@ -244,34 +292,20 @@ TEST(LibRadosIo, TruncTestPP) {
bufferlist bl2;
ASSERT_EQ((int)(sizeof(buf)/2), ioctx.read("foo", bl2, sizeof(buf), 0));
ASSERT_EQ(0, memcmp(bl2.c_str(), buf, sizeof(buf)/2));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, RemoveTest) {
+TEST_F(LibRadosIo, RemoveTest) {
char buf[128];
char buf2[sizeof(buf)];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xaa, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_append(ioctx, "foo", buf, sizeof(buf)));
ASSERT_EQ(0, rados_remove(ioctx, "foo"));
memset(buf2, 0, sizeof(buf2));
ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, RemoveTestPP) {
+TEST_F(LibRadosIoPP, RemoveTestPP) {
char buf[128];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xaa, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -279,19 +313,12 @@ TEST(LibRadosIo, RemoveTestPP) {
ASSERT_EQ(0, ioctx.remove("foo"));
bufferlist bl2;
ASSERT_EQ(-ENOENT, ioctx.read("foo", bl2, sizeof(buf), 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, XattrsRoundTrip) {
+TEST_F(LibRadosIo, XattrsRoundTrip) {
char buf[128];
char attr1[] = "attr1";
char attr1_buf[] = "foo bar baz";
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xaa, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_append(ioctx, "foo", buf, sizeof(buf)));
ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
@@ -299,19 +326,12 @@ TEST(LibRadosIo, XattrsRoundTrip) {
ASSERT_EQ((int)sizeof(attr1_buf),
rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
ASSERT_EQ(0, memcmp(attr1_buf, buf, sizeof(attr1_buf)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, XattrsRoundTripPP) {
+TEST_F(LibRadosIoPP, XattrsRoundTripPP) {
char buf[128];
char attr1[] = "attr1";
char attr1_buf[] = "foo bar baz";
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xaa, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -325,38 +345,24 @@ TEST(LibRadosIo, XattrsRoundTripPP) {
ASSERT_EQ((int)sizeof(attr1_buf),
ioctx.getxattr("foo", attr1, bl4));
ASSERT_EQ(0, memcmp(bl4.c_str(), attr1_buf, sizeof(attr1_buf)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, RmXattr) {
+TEST_F(LibRadosIo, RmXattr) {
char buf[128];
char attr1[] = "attr1";
char attr1_buf[] = "foo bar baz";
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xaa, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_append(ioctx, "foo", buf, sizeof(buf)));
ASSERT_EQ(0,
rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
ASSERT_EQ(0, rados_rmxattr(ioctx, "foo", attr1));
ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, RmXattrPP) {
+TEST_F(LibRadosIoPP, RmXattrPP) {
char buf[128];
char attr1[] = "attr1";
char attr1_buf[] = "foo bar baz";
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xaa, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -367,11 +373,9 @@ TEST(LibRadosIo, RmXattrPP) {
ASSERT_EQ(0, ioctx.rmxattr("foo", attr1));
bufferlist bl3;
ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl3));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosIo, XattrIter) {
+TEST_F(LibRadosIo, XattrIter) {
char buf[128];
char attr1[] = "attr1";
char attr1_buf[] = "foo bar baz";
@@ -380,11 +384,6 @@ TEST(LibRadosIo, XattrIter) {
for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
attr2_buf[j] = j % 0xff;
}
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xaa, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_append(ioctx, "foo", buf, sizeof(buf)));
ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
@@ -414,11 +413,9 @@ TEST(LibRadosIo, XattrIter) {
}
}
rados_getxattrs_end(iter);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosIo, XattrListPP) {
+TEST_F(LibRadosIoPP, XattrListPP) {
char buf[128];
char attr1[] = "attr1";
char attr1_buf[] = "foo bar baz";
@@ -427,11 +424,6 @@ TEST(LibRadosIo, XattrListPP) {
for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
attr2_buf[j] = j % 0xff;
}
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xaa, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -456,6 +448,4 @@ TEST(LibRadosIo, XattrListPP) {
ASSERT_EQ(0, 1);
}
}
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
diff --git a/src/test/librados/list.cc b/src/test/librados/list.cc
index 30e18b7..c530b60 100644
--- a/src/test/librados/list.cc
+++ b/src/test/librados/list.cc
@@ -3,6 +3,7 @@
#include "include/rados/librados.hpp"
#include "include/stringify.h"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include "include/types.h"
#include "gtest/gtest.h"
@@ -11,13 +12,11 @@
using namespace librados;
-TEST(LibRadosList, ListObjects) {
+typedef RadosTest LibRadosList;
+typedef RadosTestPP LibRadosListPP;
+
+TEST_F(LibRadosList, ListObjects) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
rados_list_ctx_t ctx;
@@ -30,16 +29,25 @@ TEST(LibRadosList, ListObjects) {
}
ASSERT_TRUE(foundit);
rados_objects_list_close(ctx);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosList, ListObjectsPP) {
- std::string pool_name = get_temp_pool_name();
- Rados cluster;
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosListPP, ListObjectsPP) {
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl1;
+ bl1.append(buf, sizeof(buf));
+ ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl1, sizeof(buf), 0));
+ ObjectIterator iter(ioctx.objects_begin());
+ bool foundit = false;
+ while (iter != ioctx.objects_end()) {
+ foundit = true;
+ ASSERT_EQ((*iter).first, "foo");
+ ++iter;
+ }
+ ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, ListObjectsTwicePP) {
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
@@ -53,8 +61,73 @@ TEST(LibRadosList, ListObjectsPP) {
++iter;
}
ASSERT_TRUE(foundit);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ ++iter;
+ ASSERT_TRUE(iter == ioctx.objects_end());
+ foundit = false;
+ iter.seek(0);
+ while (iter != ioctx.objects_end()) {
+ foundit = true;
+ ASSERT_EQ((*iter).first, "foo");
+ ++iter;
+ }
+ ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, ListObjectsCopyIterPP) {
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl1;
+ bl1.append(buf, sizeof(buf));
+ ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl1, sizeof(buf), 0));
+
+ // make sure this is still valid after the original iterators are gone
+ ObjectIterator iter3;
+ {
+ ObjectIterator iter(ioctx.objects_begin());
+ ObjectIterator iter2(iter);
+ iter3 = iter2;
+ ASSERT_EQ((*iter).first, "foo");
+ ++iter;
+ ASSERT_TRUE(iter == ioctx.objects_end());
+ ++iter;
+ ASSERT_TRUE(iter == ioctx.objects_end());
+
+ ASSERT_EQ(iter2->first, "foo");
+ ASSERT_EQ(iter3->first, "foo");
+ ++iter2;
+ ASSERT_TRUE(iter2 == ioctx.objects_end());
+ }
+
+ ASSERT_EQ(iter3->first, "foo");
+ iter3 = iter3;
+ ASSERT_EQ(iter3->first, "foo");
+ ++iter3;
+ ASSERT_TRUE(iter3 == ioctx.objects_end());
+}
+
+TEST_F(LibRadosListPP, ListObjectsEndIter) {
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl1;
+ bl1.append(buf, sizeof(buf));
+ ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl1, sizeof(buf), 0));
+
+ ObjectIterator iter(ioctx.objects_begin());
+ ObjectIterator iter_end(ioctx.objects_end());
+ ObjectIterator iter_end2 = ioctx.objects_end();
+ ASSERT_TRUE(iter_end == iter_end2);
+ ASSERT_TRUE(iter_end == ioctx.objects_end());
+ ASSERT_TRUE(iter_end2 == ioctx.objects_end());
+
+ ASSERT_EQ(iter->first, "foo");
+ ++iter;
+ ASSERT_TRUE(iter == ioctx.objects_end());
+ ASSERT_TRUE(iter == iter_end);
+ ASSERT_TRUE(iter == iter_end2);
+ ObjectIterator iter2 = iter;
+ ASSERT_TRUE(iter2 == ioctx.objects_end());
+ ASSERT_TRUE(iter2 == iter_end);
+ ASSERT_TRUE(iter2 == iter_end2);
}
static void check_list(std::set<std::string>& myset, rados_list_ctx_t& ctx)
@@ -74,15 +147,11 @@ static void check_list(std::set<std::string>& myset, rados_list_ctx_t& ctx)
ASSERT_TRUE(myset.empty());
}
-TEST(LibRadosList, ListObjectsNS) {
+TEST_F(LibRadosList, ListObjectsNS) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
// Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+ rados_ioctx_set_namespace(ioctx, "");
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
rados_ioctx_set_namespace(ioctx, "ns1");
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
@@ -124,9 +193,6 @@ TEST(LibRadosList, ListObjectsNS) {
ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
check_list(ns2, ctx);
rados_objects_list_close(ctx);
-
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
static void check_listpp(std::set<std::string>& myset, IoCtx& ioctx)
@@ -147,17 +213,13 @@ static void check_listpp(std::set<std::string>& myset, IoCtx& ioctx)
ASSERT_TRUE(myset.empty());
}
-TEST(LibRadosList, ListObjectsPPNS) {
- std::string pool_name = get_temp_pool_name();
- Rados cluster;
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosListPP, ListObjectsPPNS) {
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
// Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+ ioctx.set_namespace("");
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo1", bl1, sizeof(buf), 0));
ioctx.set_namespace("ns1");
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo1", bl1, sizeof(buf), 0));
@@ -189,18 +251,9 @@ TEST(LibRadosList, ListObjectsPPNS) {
ioctx.set_namespace("ns2");
check_listpp(ns2, ioctx);
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosList, ListObjectsManyPP) {
- std::string pool_name = get_temp_pool_name();
- Rados cluster;
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosListPP, ListObjectsManyPP) {
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
@@ -224,20 +277,9 @@ TEST(LibRadosList, ListObjectsManyPP) {
// make sure they are 0..n
for (unsigned i = 0; i < saw_pg.size(); ++i)
ASSERT_TRUE(saw_pg.count(i));
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-
-
-TEST(LibRadosList, ListObjectsStart) {
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
-
+TEST_F(LibRadosList, ListObjectsStart) {
char buf[128];
memset(buf, 0xcc, sizeof(buf));
@@ -268,17 +310,9 @@ TEST(LibRadosList, ListObjectsStart) {
++p;
}
rados_objects_list_close(ctx);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosList, ListObjectsStartPP) {
- std::string pool_name = get_temp_pool_name();
- Rados cluster;
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosListPP, ListObjectsStartPP) {
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
@@ -304,7 +338,4 @@ TEST(LibRadosList, ListObjectsStartPP) {
ASSERT_TRUE(p->second.count(it->first));
++p;
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
diff --git a/src/test/librados/lock.cc b/src/test/librados/lock.cc
index 1d33d46..6894ed4 100644
--- a/src/test/librados/lock.cc
+++ b/src/test/librados/lock.cc
@@ -1,6 +1,7 @@
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include "cls/lock/cls_lock_client.h"
#include <algorithm>
@@ -10,171 +11,90 @@
using namespace librados;
-TEST(LibRadosLock, LockExclusive) {
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+typedef RadosTest LibRadosLock;
+typedef RadosTestPP LibRadosLockPP;
+
+TEST_F(LibRadosLock, LockExclusive) {
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, LockExclusivePP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosLockPP, LockExclusivePP) {
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, LockShared) {
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+TEST_F(LibRadosLock, LockShared) {
ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
ASSERT_EQ(-EEXIST, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, LockSharedPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosLockPP, LockSharedPP) {
ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
ASSERT_EQ(-EEXIST, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, LockExclusiveDur) {
+TEST_F(LibRadosLock, LockExclusiveDur) {
struct timeval tv;
tv.tv_sec = 1;
tv.tv_usec = 0;
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", &tv, 0));
sleep(1);
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, LockExclusiveDurPP) {
+TEST_F(LibRadosLockPP, LockExclusiveDurPP) {
struct timeval tv;
tv.tv_sec = 1;
tv.tv_usec = 0;
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", &tv, 0));
sleep(1);
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, LockSharedDur) {
+TEST_F(LibRadosLock, LockSharedDur) {
struct timeval tv;
tv.tv_sec = 1;
tv.tv_usec = 0;
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", &tv, 0));
sleep(1);
ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, LockSharedDurPP) {
+TEST_F(LibRadosLockPP, LockSharedDurPP) {
struct timeval tv;
tv.tv_sec = 1;
tv.tv_usec = 0;
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", &tv, 0));
sleep(1);
ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, LockRenew) {
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+TEST_F(LibRadosLock, LockRenew) {
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, LOCK_FLAG_RENEW));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, LockRenewPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosLockPP, LockRenewPP) {
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, LOCK_FLAG_RENEW));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, Unlock) {
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+TEST_F(LibRadosLock, Unlock) {
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock", "Cookie"));
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, UnlockPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosLockPP, UnlockPP) {
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(0, ioctx.unlock("foo", "TestLock", "Cookie"));
ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, ListLockers) {
+TEST_F(LibRadosLock, ListLockers) {
int exclusive;
char tag[1024];
char clients[1024];
@@ -184,14 +104,9 @@ TEST(LibRadosLock, ListLockers) {
size_t clients_len = 1024;
size_t cookies_len = 1024;
size_t addresses_len = 1024;
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
std::stringstream sstm;
sstm << "client." << rados_get_instance_id(cluster);
std::string me = sstm.str();
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock", "Cookie"));
ASSERT_EQ(0, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
@@ -209,16 +124,9 @@ TEST(LibRadosLock, ListLockers) {
ASSERT_EQ(me.size() + 1, clients_len);
ASSERT_EQ(0, strcmp(cookies, "Cookie"));
ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, ListLockersPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosLockPP, ListLockersPP) {
std::stringstream sstm;
sstm << "client." << cluster.get_instance_id();
std::string me = sstm.str();
@@ -241,11 +149,9 @@ TEST(LibRadosLock, ListLockersPP) {
ASSERT_EQ(me, it->client);
ASSERT_EQ("Cookie", it->cookie);
}
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosLock, BreakLock) {
+TEST_F(LibRadosLock, BreakLock) {
int exclusive;
char tag[1024];
char clients[1024];
@@ -255,14 +161,9 @@ TEST(LibRadosLock, BreakLock) {
size_t clients_len = 1024;
size_t cookies_len = 1024;
size_t addresses_len = 1024;
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
std::stringstream sstm;
sstm << "client." << rados_get_instance_id(cluster);
std::string me = sstm.str();
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
ASSERT_EQ(1, exclusive);
@@ -273,19 +174,12 @@ TEST(LibRadosLock, BreakLock) {
ASSERT_EQ(0, strcmp(cookies, "Cookie"));
ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
ASSERT_EQ(0, rados_break_lock(ioctx, "foo", "TestLock", clients, "Cookie"));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosLock, BreakLockPP) {
+TEST_F(LibRadosLockPP, BreakLockPP) {
int exclusive;
std::string tag;
std::list<librados::locker_t> lockers;
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
std::stringstream sstm;
sstm << "client." << cluster.get_instance_id();
std::string me = sstm.str();
@@ -296,6 +190,4 @@ TEST(LibRadosLock, BreakLockPP) {
ASSERT_EQ(me, it->client);
ASSERT_EQ("Cookie", it->cookie);
ASSERT_EQ(0, ioctx.break_lock("foo", "TestLock", it->client, "Cookie"));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
diff --git a/src/test/librados/misc.cc b/src/test/librados/misc.cc
index 0b2ba6c..233cc6b 100644
--- a/src/test/librados/misc.cc
+++ b/src/test/librados/misc.cc
@@ -13,6 +13,7 @@
#include "common/ceph_argparse.h"
#include "common/common_init.h"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include <errno.h>
#include <map>
@@ -25,37 +26,28 @@ using std::map;
using std::ostringstream;
using std::string;
-TEST(LibRadosMisc, Version) {
+typedef RadosTest LibRadosMisc;
+typedef RadosTestPP LibRadosMiscPP;
+
+TEST(LibRadosMiscVersion, Version) {
int major, minor, extra;
rados_version(&major, &minor, &extra);
}
-TEST(LibRadosMisc, VersionPP) {
+TEST(LibRadosMiscVersion, VersionPP) {
int major, minor, extra;
Rados::version(&major, &minor, &extra);
}
-TEST(LibRadosMisc, ClusterFSID) {
- rados_t cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
-
+TEST_F(LibRadosMisc, ClusterFSID) {
char fsid[37];
ASSERT_EQ(-ERANGE, rados_cluster_fsid(cluster, fsid, sizeof(fsid) - 1));
ASSERT_EQ(sizeof(fsid) - 1,
(size_t)rados_cluster_fsid(cluster, fsid, sizeof(fsid)));
-
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosMisc, WaitOSDMapPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
-
+TEST_F(LibRadosMiscPP, WaitOSDMapPP) {
ASSERT_EQ(0, cluster.wait_for_latest_osdmap());
-
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
static std::string read_key_from_tmap(IoCtx& ioctx, const std::string &obj,
@@ -119,13 +111,7 @@ static int remove_key_from_tmap(IoCtx &ioctx, const std::string &obj,
return ret;
}
-TEST(LibRadosMisc, TmapUpdatePP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, TmapUpdatePP) {
// create tmap
{
__u8 c = CEPH_OSD_TMAP_CREATE;
@@ -152,18 +138,9 @@ TEST(LibRadosMisc, TmapUpdatePP) {
// key should be removed
ASSERT_EQ(string(""), read_key_from_tmap(ioctx, "foo", "key1"));
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, TmapUpdateMisorderedPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, TmapUpdateMisorderedPP) {
// create tmap
{
__u8 c = CEPH_OSD_TMAP_CREATE;
@@ -229,18 +206,9 @@ TEST(LibRadosMisc, TmapUpdateMisorderedPP) {
ASSERT_EQ(0, remove_key_from_tmap(ioctx, "foo", "b"));
ASSERT_EQ(string(""), read_key_from_tmap(ioctx, "foo", "a"));
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, TmapUpdateMisorderedPutPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, TmapUpdateMisorderedPutPP) {
// create unsorted tmap
string h("header");
bufferlist bl;
@@ -260,18 +228,9 @@ TEST(LibRadosMisc, TmapUpdateMisorderedPutPP) {
bufferlist newbl;
ioctx.read("foo", newbl, orig.length(), 0);
ASSERT_EQ(orig.contents_equal(newbl), false);
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, Tmap2OmapPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, Tmap2OmapPP) {
// create tmap
bufferlist hdr;
hdr.append("header");
@@ -325,18 +284,10 @@ TEST(LibRadosMisc, Tmap2OmapPP) {
}
ASSERT_TRUE(same);
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, Exec) {
+TEST_F(LibRadosMisc, Exec) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
char buf2[512];
@@ -349,16 +300,9 @@ TEST(LibRadosMisc, Exec) {
uint64_t all_features;
::decode(all_features, iter);
ASSERT_EQ(all_features, (uint64_t)RBD_FEATURES_ALL);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosMisc, ExecPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosMiscPP, ExecPP) {
bufferlist bl;
ASSERT_EQ(0, ioctx.write("foo", bl, 0, 0));
bufferlist bl2, out;
@@ -368,17 +312,9 @@ TEST(LibRadosMisc, ExecPP) {
uint64_t all_features;
::decode(all_features, iter);
ASSERT_EQ(all_features, (uint64_t)RBD_FEATURES_ALL);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, Operate1PP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, Operate1PP) {
ObjectWriteOperation o;
{
bufferlist bl;
@@ -408,25 +344,17 @@ TEST(LibRadosMisc, Operate1PP) {
o2.cmpxattr("key1", CEPH_OSD_CMPXATTR_OP_EQ, bl);
o2.rmxattr("key1");
}
- ASSERT_EQ(0, ioctx.operate("foo", &o2));
+ ASSERT_EQ(-ECANCELED, ioctx.operate("foo", &o2));
ObjectWriteOperation o3;
{
bufferlist bl;
bl.append(val1);
o3.cmpxattr("key1", CEPH_OSD_CMPXATTR_OP_EQ, bl);
}
- ASSERT_LT(ioctx.operate("foo", &o3), 0);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ ASSERT_EQ(-ECANCELED, ioctx.operate("foo", &o3));
}
-TEST(LibRadosMisc, Operate2PP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, Operate2PP) {
ObjectWriteOperation o;
{
bufferlist bl;
@@ -445,17 +373,9 @@ TEST(LibRadosMisc, Operate2PP) {
time_t mtime;
ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
ASSERT_EQ(0U, size);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, BigObjectPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, BigObjectPP) {
bufferlist bl;
bl.append("abcdefg");
ASSERT_EQ((int)bl.length(), ioctx.write("foo", bl, bl.length(), 0));
@@ -485,9 +405,6 @@ TEST(LibRadosMisc, BigObjectPP) {
// this test only works on 64-bit platforms
ASSERT_EQ(-EFBIG, ioctx.write("foo", bl, bl.length(), 500000000000ull));
#endif
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
void set_completion_complete(rados_completion_t cb, void *arg)
@@ -496,13 +413,7 @@ void set_completion_complete(rados_completion_t cb, void *arg)
*my_aio_complete = true;
}
-TEST(LibRadosMisc, AioOperatePP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
-
+TEST_F(LibRadosMiscPP, AioOperatePP) {
bool my_aio_complete = false;
AioCompletion *my_completion = cluster.aio_create_completion(
(void*)&my_aio_complete, set_completion_complete, NULL);
@@ -533,16 +444,9 @@ TEST(LibRadosMisc, AioOperatePP) {
time_t mtime;
ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
ASSERT_EQ(1024U, size);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, CloneRangePP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+TEST_F(LibRadosMiscPP, CloneRangePP) {
char buf[64];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
@@ -553,17 +457,10 @@ TEST(LibRadosMisc, CloneRangePP) {
bufferlist bl2;
ASSERT_EQ(sizeof(buf), (size_t)ioctx.read("bar", bl2, sizeof(buf), 0));
ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, CloneRange) {
+TEST_F(LibRadosMisc, CloneRange) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "src", buf, sizeof(buf), 0));
rados_ioctx_locator_set_key(ioctx, "src");
@@ -572,17 +469,9 @@ TEST(LibRadosMisc, CloneRange) {
memset(buf2, 0, sizeof(buf2));
ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "dst", buf2, sizeof(buf2), 0));
ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosMisc, AssertExistsPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosMiscPP, AssertExistsPP) {
char buf[64];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
@@ -595,18 +484,9 @@ TEST(LibRadosMisc, AssertExistsPP) {
ASSERT_EQ(0, ioctx.create("asdffoo", true));
ASSERT_EQ(0, ioctx.operate("asdffoo", &op));
ASSERT_EQ(-EEXIST, ioctx.create("asdffoo", true));
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, BigAttrPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosMiscPP, BigAttrPP) {
char buf[64];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
@@ -641,18 +521,9 @@ TEST(LibRadosMisc, BigAttrPP) {
ASSERT_EQ((int)bl.length(), ioctx.getxattr("foo", n, got));
ASSERT_TRUE(bl.contents_equal(got));
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, CopyPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosMiscPP, CopyPP) {
bufferlist bl, x;
bl.append("hi there");
x.append("bar");
@@ -733,9 +604,6 @@ TEST(LibRadosMisc, CopyPP) {
ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2));
ASSERT_TRUE(x.contents_equal(x2));
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
int main(int argc, char **argv)
diff --git a/src/test/librados/snapshots.cc b/src/test/librados/snapshots.cc
index d2bbe74..947d9c6 100644
--- a/src/test/librados/snapshots.cc
+++ b/src/test/librados/snapshots.cc
@@ -1,5 +1,6 @@
#include "include/rados/librados.hpp"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include <algorithm>
#include <errno.h>
@@ -9,34 +10,28 @@
using namespace librados;
using std::string;
+typedef RadosTest LibRadosSnapshots;
+typedef RadosTest LibRadosSnapshotsSelfManaged;
+typedef RadosTestPP LibRadosSnapshotsPP;
+typedef RadosTestPP LibRadosSnapshotsSelfManagedPP;
+
const int bufsize = 128;
-TEST(LibRadosSnapshots, SnapList) {
+TEST_F(LibRadosSnapshots, SnapList) {
char buf[bufsize];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
rados_snap_t snaps[10];
- ASSERT_EQ(1, rados_ioctx_snap_list(ioctx, snaps,
- sizeof(snaps) / sizeof(snaps[0])));
+ EXPECT_EQ(1, rados_ioctx_snap_list(ioctx, snaps,
+ sizeof(snaps) / sizeof(snaps[0])));
rados_snap_t rid;
- ASSERT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
- ASSERT_EQ(rid, snaps[0]);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+ EXPECT_EQ(rid, snaps[0]);
+ EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
}
-TEST(LibRadosSnapshots, SnapListPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosSnapshotsPP, SnapListPP) {
char buf[bufsize];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
@@ -44,22 +39,16 @@ TEST(LibRadosSnapshots, SnapListPP) {
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl1, sizeof(buf), 0));
ASSERT_EQ(0, ioctx.snap_create("snap1"));
std::vector<snap_t> snaps;
- ASSERT_EQ(0, ioctx.snap_list(&snaps));
- ASSERT_EQ(1U, snaps.size());
+ EXPECT_EQ(0, ioctx.snap_list(&snaps));
+ EXPECT_EQ(1U, snaps.size());
snap_t rid;
- ASSERT_EQ(0, ioctx.snap_lookup("snap1", &rid));
- ASSERT_EQ(rid, snaps[0]);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ EXPECT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+ EXPECT_EQ(rid, snaps[0]);
+ EXPECT_EQ(0, ioctx.snap_remove("snap1"));
}
-TEST(LibRadosSnapshots, SnapRemove) {
+TEST_F(LibRadosSnapshots, SnapRemove) {
char buf[bufsize];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
@@ -68,17 +57,10 @@ TEST(LibRadosSnapshots, SnapRemove) {
ASSERT_EQ(-EEXIST, rados_ioctx_snap_create(ioctx, "snap1"));
ASSERT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
ASSERT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosSnapshots, SnapRemovePP) {
+TEST_F(LibRadosSnapshotsPP, SnapRemovePP) {
char buf[bufsize];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -88,38 +70,25 @@ TEST(LibRadosSnapshots, SnapRemovePP) {
ASSERT_EQ(0, ioctx.snap_lookup("snap1", &rid));
ASSERT_EQ(0, ioctx.snap_remove("snap1"));
ASSERT_EQ(-ENOENT, ioctx.snap_lookup("snap1", &rid));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosSnapshots, Rollback) {
+TEST_F(LibRadosSnapshots, Rollback) {
char buf[bufsize];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
char buf2[sizeof(buf)];
memset(buf2, 0xdd, sizeof(buf2));
- ASSERT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
- ASSERT_EQ(0, rados_rollback(ioctx, "foo", "snap1"));
+ EXPECT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+ EXPECT_EQ(0, rados_rollback(ioctx, "foo", "snap1"));
char buf3[sizeof(buf)];
- ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
- ASSERT_EQ(0, memcmp(buf, buf3, sizeof(buf)));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ EXPECT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+ EXPECT_EQ(0, memcmp(buf, buf3, sizeof(buf)));
+ EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
}
-TEST(LibRadosSnapshots, RollbackPP) {
+TEST_F(LibRadosSnapshotsPP, RollbackPP) {
char buf[bufsize];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
@@ -129,70 +98,51 @@ TEST(LibRadosSnapshots, RollbackPP) {
memset(buf2, 0xdd, sizeof(buf2));
bufferlist bl2;
bl2.append(buf2, sizeof(buf2));
- ASSERT_EQ(0, ioctx.write_full("foo", bl2));
- ASSERT_EQ(0, ioctx.rollback("foo", "snap1"));
+ EXPECT_EQ(0, ioctx.write_full("foo", bl2));
+ EXPECT_EQ(0, ioctx.rollback("foo", "snap1"));
bufferlist bl3;
- ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
- ASSERT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ EXPECT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+ EXPECT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
+ EXPECT_EQ(0, ioctx.snap_remove("snap1"));
}
-TEST(LibRadosSnapshots, SnapGetName) {
+TEST_F(LibRadosSnapshots, SnapGetName) {
char buf[bufsize];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snapfoo"));
rados_snap_t rid;
- ASSERT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snapfoo", &rid));
- ASSERT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snapbar", &rid));
+ EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snapfoo", &rid));
+ EXPECT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snapbar", &rid));
char name[128];
memset(name, 0, sizeof(name));
- ASSERT_EQ(0, rados_ioctx_snap_get_name(ioctx, rid, name, sizeof(name)));
+ EXPECT_EQ(0, rados_ioctx_snap_get_name(ioctx, rid, name, sizeof(name)));
time_t snaptime;
- ASSERT_EQ(0, rados_ioctx_snap_get_stamp(ioctx, rid, &snaptime));
- ASSERT_EQ(0, strcmp(name, "snapfoo"));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ EXPECT_EQ(0, rados_ioctx_snap_get_stamp(ioctx, rid, &snaptime));
+ EXPECT_EQ(0, strcmp(name, "snapfoo"));
+ EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snapfoo"));
}
-TEST(LibRadosSnapshots, SnapGetNamePP) {
+TEST_F(LibRadosSnapshotsPP, SnapGetNamePP) {
char buf[bufsize];
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl, sizeof(buf), 0));
ASSERT_EQ(0, ioctx.snap_create("snapfoo"));
rados_snap_t rid;
- ASSERT_EQ(0, ioctx.snap_lookup("snapfoo", &rid));
- ASSERT_EQ(-ENOENT, ioctx.snap_lookup("snapbar", &rid));
+ EXPECT_EQ(0, ioctx.snap_lookup("snapfoo", &rid));
+ EXPECT_EQ(-ENOENT, ioctx.snap_lookup("snapbar", &rid));
std::string name;
- ASSERT_EQ(0, ioctx.snap_get_name(rid, &name));
+ EXPECT_EQ(0, ioctx.snap_get_name(rid, &name));
time_t snaptime;
- ASSERT_EQ(0, ioctx.snap_get_stamp(rid, &snaptime));
- ASSERT_EQ(0, strcmp(name.c_str(), "snapfoo"));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ EXPECT_EQ(0, ioctx.snap_get_stamp(rid, &snaptime));
+ EXPECT_EQ(0, strcmp(name.c_str(), "snapfoo"));
+ EXPECT_EQ(0, ioctx.snap_remove("snapfoo"));
}
-TEST(LibRadosSnapshots, SelfManagedSnapTest) {
+TEST_F(LibRadosSnapshotsSelfManaged, Snap) {
std::vector<uint64_t> my_snaps;
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
-
my_snaps.push_back(-2);
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
::std::reverse(my_snaps.begin(), my_snaps.end());
@@ -224,18 +174,12 @@ TEST(LibRadosSnapshots, SelfManagedSnapTest) {
my_snaps.pop_back();
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
my_snaps.pop_back();
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+ ASSERT_EQ(0, rados_remove(ioctx, "foo"));
}
-TEST(LibRadosSnapshots, SelfManagedRollbackTest) {
+TEST_F(LibRadosSnapshotsSelfManaged, Rollback) {
std::vector<uint64_t> my_snaps;
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
-
my_snaps.push_back(-2);
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
::std::reverse(my_snaps.begin(), my_snaps.end());
@@ -264,18 +208,11 @@ TEST(LibRadosSnapshots, SelfManagedRollbackTest) {
my_snaps.pop_back();
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
my_snaps.pop_back();
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+ ASSERT_EQ(0, rados_remove(ioctx, "foo"));
}
-TEST(LibRadosSnapshots, SelfManagedSnapTestPP) {
+TEST_F(LibRadosSnapshotsSelfManagedPP, SnapPP) {
std::vector<uint64_t> my_snaps;
- Rados cluster;
- IoCtx ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
my_snaps.push_back(-2);
ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
::std::reverse(my_snaps.begin(), my_snaps.end());
@@ -307,19 +244,15 @@ TEST(LibRadosSnapshots, SelfManagedSnapTestPP) {
my_snaps.pop_back();
ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
my_snaps.pop_back();
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+ ASSERT_EQ(0, ioctx.remove("foo"));
}
-TEST(LibRadosSnapshots, SelfManagedSnapRollbackPP) {
+TEST_F(LibRadosSnapshotsSelfManagedPP, RollbackPP) {
std::vector<uint64_t> my_snaps;
- Rados cluster;
- IoCtx ioctx;
IoCtx readioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+ readioctx.set_namespace(ns);
readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
my_snaps.push_back(-2);
@@ -396,19 +329,14 @@ TEST(LibRadosSnapshots, SelfManagedSnapRollbackPP) {
my_snaps.pop_back();
ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
my_snaps.pop_back();
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ readioctx.close();
}
-TEST(LibRadosSnapshots, SelfManagedSnapOverlapPP) {
+TEST_F(LibRadosSnapshotsSelfManagedPP, SnapOverlapPP) {
std::vector<uint64_t> my_snaps;
- Rados cluster;
- IoCtx ioctx;
IoCtx readioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+ readioctx.set_namespace(ns);
readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
my_snaps.push_back(-2);
@@ -528,6 +456,7 @@ TEST(LibRadosSnapshots, SelfManagedSnapOverlapPP) {
my_snaps.pop_back();
ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
my_snaps.pop_back();
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+ my_snaps.pop_back();
+ readioctx.close();
}
diff --git a/src/test/librados/stat.cc b/src/test/librados/stat.cc
index 481e4a5..47706dd 100644
--- a/src/test/librados/stat.cc
+++ b/src/test/librados/stat.cc
@@ -1,6 +1,7 @@
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include <algorithm>
#include <errno.h>
@@ -8,13 +9,11 @@
using namespace librados;
-TEST(LibRadosStat, Stat) {
+typedef RadosTest LibRadosStat;
+typedef RadosTestPP LibRadosStatPP;
+
+TEST_F(LibRadosStat, Stat) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
uint64_t size;
@@ -22,17 +21,10 @@ TEST(LibRadosStat, Stat) {
ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
ASSERT_EQ(sizeof(buf), size);
ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosStat, StatPP) {
+TEST_F(LibRadosStatPP, StatPP) {
char buf[128];
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
@@ -42,18 +34,12 @@ TEST(LibRadosStat, StatPP) {
ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
ASSERT_EQ(sizeof(buf), size);
ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosStat, StatNS) {
+TEST_F(LibRadosStat, StatNS) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
+ rados_ioctx_set_namespace(ioctx, "");
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
@@ -74,21 +60,14 @@ TEST(LibRadosStat, StatNS) {
ASSERT_EQ(sizeof(buf2), size);
ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
ASSERT_EQ(-ENOENT, rados_stat(ioctx, "foo2", &size, &mtime));
-
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosStat, StatPPNS) {
+TEST_F(LibRadosStatPP, StatPPNS) {
char buf[128];
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
+ ioctx.set_namespace("");
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl, sizeof(buf), 0));
ASSERT_EQ((int)sizeof(buf), ioctx.write("foo2", bl, sizeof(buf), 0));
@@ -111,36 +90,20 @@ TEST(LibRadosStat, StatPPNS) {
ASSERT_EQ(sizeof(buf2), size);
ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
ASSERT_EQ(-ENOENT, ioctx.stat("foo2", &size, &mtime));
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosStat, ClusterStat) {
- rados_t cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+TEST_F(LibRadosStat, ClusterStat) {
struct rados_cluster_stat_t result;
ASSERT_EQ(0, rados_cluster_stat(cluster, &result));
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosStat, ClusterStatPP) {
- Rados cluster;
+TEST_F(LibRadosStatPP, ClusterStatPP) {
cluster_stat_t cstat;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
ASSERT_EQ(0, cluster.cluster_stat(cstat));
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosStat, PoolStat) {
+TEST_F(LibRadosStat, PoolStat) {
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
char actual_pool_name[80];
unsigned l = rados_ioctx_get_pool_name(ioctx, actual_pool_name, sizeof(actual_pool_name));
ASSERT_EQ(strlen(actual_pool_name), l);
@@ -150,16 +113,9 @@ TEST(LibRadosStat, PoolStat) {
struct rados_pool_stat_t stats;
memset(&stats, 0, sizeof(stats));
ASSERT_EQ(0, rados_ioctx_pool_stat(ioctx, &stats));
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
-TEST(LibRadosStat, PoolStatPP) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
+TEST_F(LibRadosStatPP, PoolStatPP) {
std::string n = ioctx.get_pool_name();
ASSERT_EQ(n, pool_name);
char buf[128];
@@ -170,6 +126,4 @@ TEST(LibRadosStat, PoolStatPP) {
std::list<std::string> v;
std::map<std::string,stats_map> stats;
ASSERT_EQ(0, cluster.get_pool_stats(v, stats));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
diff --git a/src/test/librados/test.cc b/src/test/librados/test.cc
index a1aa24c..83f11b0 100644
--- a/src/test/librados/test.cc
+++ b/src/test/librados/test.cc
@@ -1,3 +1,6 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
#include "test/librados/test.h"
@@ -27,70 +30,59 @@ std::string get_temp_pool_name()
std::string create_one_pool(const std::string &pool_name, rados_t *cluster)
{
- int ret;
- ret = rados_create(cluster, NULL);
- if (ret) {
- std::ostringstream oss;
- oss << "rados_create failed with error " << ret;
- return oss.str();
- }
- ret = rados_conf_read_file(*cluster, NULL);
+ std::string err = connect_cluster(cluster);
+ if (err.length())
+ return err;
+ int ret = rados_pool_create(*cluster, pool_name.c_str());
if (ret) {
rados_shutdown(*cluster);
std::ostringstream oss;
- oss << "rados_conf_read_file failed with error " << ret;
- return oss.str();
- }
- rados_conf_parse_env(*cluster, NULL);
- ret = rados_connect(*cluster);
- if (ret) {
- rados_shutdown(*cluster);
- std::ostringstream oss;
- oss << "rados_connect failed with error " << ret;
+ oss << "rados_pool_create(" << pool_name << ") failed with error " << ret;
return oss.str();
}
- ret = rados_pool_create(*cluster, pool_name.c_str());
+ return "";
+}
+
+std::string create_one_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+ std::string err = connect_cluster_pp(cluster);
+ if (err.length())
+ return err;
+ int ret = cluster.pool_create(pool_name.c_str());
if (ret) {
- rados_shutdown(*cluster);
+ cluster.shutdown();
std::ostringstream oss;
- oss << "rados_pool_create(" << pool_name << ") failed with error " << ret;
+ oss << "cluster.pool_create(" << pool_name << ") failed with error " << ret;
return oss.str();
}
return "";
}
-std::string create_one_pool_pp(const std::string &pool_name, Rados &cluster)
+std::string connect_cluster(rados_t *cluster)
{
char *id = getenv("CEPH_CLIENT_ID");
if (id) std::cerr << "Client id is: " << id << std::endl;
int ret;
- ret = cluster.init(id);
- if (ret) {
- std::ostringstream oss;
- oss << "cluster.init failed with error " << ret;
- return oss.str();
- }
- ret = cluster.conf_read_file(NULL);
+ ret = rados_create(cluster, NULL);
if (ret) {
- cluster.shutdown();
std::ostringstream oss;
- oss << "cluster.conf_read_file failed with error " << ret;
+ oss << "rados_create failed with error " << ret;
return oss.str();
}
- cluster.conf_parse_env(NULL);
- ret = cluster.connect();
+ ret = rados_conf_read_file(*cluster, NULL);
if (ret) {
- cluster.shutdown();
+ rados_shutdown(*cluster);
std::ostringstream oss;
- oss << "cluster.connect failed with error " << ret;
+ oss << "rados_conf_read_file failed with error " << ret;
return oss.str();
}
- ret = cluster.pool_create(pool_name.c_str());
+ rados_conf_parse_env(*cluster, NULL);
+ ret = rados_connect(*cluster);
if (ret) {
- cluster.shutdown();
+ rados_shutdown(*cluster);
std::ostringstream oss;
- oss << "cluster.pool_create(" << pool_name << ") failed with error " << ret;
+ oss << "rados_connect failed with error " << ret;
return oss.str();
}
return "";
diff --git a/src/test/librados/test.h b/src/test/librados/test.h
index df27ba0..652c235 100644
--- a/src/test/librados/test.h
+++ b/src/test/librados/test.h
@@ -26,6 +26,7 @@ std::string get_temp_pool_name();
std::string create_one_pool(const std::string &pool_name, rados_t *cluster);
std::string create_one_pool_pp(const std::string &pool_name,
librados::Rados &cluster);
+std::string connect_cluster(rados_t *cluster);
std::string connect_cluster_pp(librados::Rados &cluster);
int destroy_one_pool(const std::string &pool_name, rados_t *cluster);
int destroy_one_pool_pp(const std::string &pool_name, librados::Rados &cluster);
diff --git a/src/test/librados/tier.cc b/src/test/librados/tier.cc
index 065ad34..1ff3d2e 100644
--- a/src/test/librados/tier.cc
+++ b/src/test/librados/tier.cc
@@ -15,6 +15,7 @@
#include "common/common_init.h"
#include "common/Cond.h"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include "json_spirit/json_spirit.h"
#include "osd/HitSet.h"
@@ -30,13 +31,42 @@ using std::map;
using std::ostringstream;
using std::string;
-TEST(LibRadosTier, Dirty) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+typedef RadosTestPP LibRadosTierPP;
+
+class LibRadosTwoPoolsPP : public RadosTestPP
+{
+public:
+ LibRadosTwoPoolsPP() {};
+ virtual ~LibRadosTwoPoolsPP() {};
+protected:
+ static void SetUpTestCase() {
+ pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+ cache_pool_name = get_temp_pool_name();
+ ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
+ }
+ static void TearDownTestCase() {
+ ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+ }
+ static std::string cache_pool_name;
+
+ virtual void SetUp() {
+ RadosTestPP::SetUp();
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ cache_ioctx.set_namespace(ns);
+ }
+ virtual void TearDown() {
+ RadosTestPP::TearDown();
+ cleanup_default_namespace(cache_ioctx);
+ cache_ioctx.close();
+ }
+ librados::IoCtx cache_ioctx;
+};
+
+std::string LibRadosTwoPoolsPP::cache_pool_name;
+TEST_F(LibRadosTierPP, Dirty) {
{
ObjectWriteOperation op;
op.undirty();
@@ -89,29 +119,16 @@ TEST(LibRadosTier, Dirty) {
ASSERT_TRUE(dirty);
ASSERT_EQ(0, r);
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosTier, Overlay) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, Overlay) {
// create objects
{
bufferlist bl;
bl.append("base");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
{
bufferlist bl;
@@ -124,11 +141,12 @@ TEST(LibRadosTier, Overlay) {
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
@@ -138,7 +156,7 @@ TEST(LibRadosTier, Overlay) {
// by default, the overlay sends us to cache pool
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
{
@@ -153,7 +171,7 @@ TEST(LibRadosTier, Overlay) {
ObjectReadOperation op;
op.read(0, 1, &bl, NULL);
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_OVERLAY, NULL));
completion->wait_for_safe();
@@ -164,49 +182,34 @@ TEST(LibRadosTier, Overlay) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, Promote) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, Promote) {
// create object
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -220,14 +223,14 @@ TEST(LibRadosTier, Promote) {
// read, trigger a promote
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
}
// read, trigger a whiteout
{
bufferlist bl;
- ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
- ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
}
// verify the object is present in the cache tier
@@ -243,102 +246,87 @@ TEST(LibRadosTier, Promote) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, PromoteSnap) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
// create object
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ ASSERT_EQ(0, ioctx.operate("bar", &op));
}
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ ASSERT_EQ(0, ioctx.operate("baz", &op));
}
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ ASSERT_EQ(0, ioctx.operate("bam", &op));
}
// create a snapshot, clone
vector<uint64_t> my_snaps(1);
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
my_snaps));
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ ASSERT_EQ(0, ioctx.operate("bar", &op));
}
{
ObjectWriteOperation op;
op.remove();
- ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ ASSERT_EQ(0, ioctx.operate("baz", &op));
}
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ ASSERT_EQ(0, ioctx.operate("bam", &op));
}
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -352,118 +340,103 @@ TEST(LibRadosTier, PromoteSnap) {
// read, trigger a promote on the head
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("bam", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
// read foo snap
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('h', bl[0]);
}
// read bar snap
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
ASSERT_EQ('h', bl[0]);
}
// read baz snap
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("baz", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
ASSERT_EQ('h', bl[0]);
}
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
// read foo
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
// read bar
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
// read baz
{
bufferlist bl;
- ASSERT_EQ(-ENOENT, base_ioctx.read("baz", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
}
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, PromoteSnapTrimRace) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
// create object
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// create a snapshot, clone
vector<uint64_t> my_snaps(1);
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
my_snaps));
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -475,61 +448,46 @@ TEST(LibRadosTier, PromoteSnapTrimRace) {
cluster.wait_for_latest_osdmap();
// delete the snap
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_remove(my_snaps[0]));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
// read foo snap
{
bufferlist bl;
- ASSERT_EQ(-ENOENT, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
}
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- // cluster.pool_delete(cache_pool_name.c_str());
- //ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, Whiteout) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, Whiteout) {
// create object
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -541,10 +499,10 @@ TEST(LibRadosTier, Whiteout) {
cluster.wait_for_latest_osdmap();
// create some whiteouts, verify they behave
- ASSERT_EQ(0, base_ioctx.remove("foo"));
+ ASSERT_EQ(0, ioctx.remove("foo"));
- ASSERT_EQ(-ENOENT, base_ioctx.remove("bar"));
- ASSERT_EQ(-ENOENT, base_ioctx.remove("bar"));
+ ASSERT_EQ(-ENOENT, ioctx.remove("bar"));
+ ASSERT_EQ(-ENOENT, ioctx.remove("bar"));
// verify the whiteouts are there in the cache tier
{
@@ -557,7 +515,7 @@ TEST(LibRadosTier, Whiteout) {
ASSERT_TRUE(it == cache_ioctx.objects_end());
}
- ASSERT_EQ(-ENOENT, base_ioctx.remove("foo"));
+ ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
// recreate an object and verify we can read it
{
@@ -565,59 +523,44 @@ TEST(LibRadosTier, Whiteout) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('h', bl[0]);
}
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, Evict) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, Evict) {
// create object
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -631,15 +574,15 @@ TEST(LibRadosTier, Evict) {
// read, trigger a promote
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
}
// read, trigger a whiteout, and a dirty object
{
bufferlist bl;
- ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
- ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
- ASSERT_EQ(0, base_ioctx.write("bar", bl, bl.length(), 0));
+ ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
}
// verify the object is present in the cache tier
@@ -690,102 +633,87 @@ TEST(LibRadosTier, Evict) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, EvictSnap) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
// create object
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ ASSERT_EQ(0, ioctx.operate("bar", &op));
}
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ ASSERT_EQ(0, ioctx.operate("baz", &op));
}
{
bufferlist bl;
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ ASSERT_EQ(0, ioctx.operate("bam", &op));
}
// create a snapshot, clone
vector<uint64_t> my_snaps(1);
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
my_snaps));
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ ASSERT_EQ(0, ioctx.operate("bar", &op));
}
{
ObjectWriteOperation op;
op.remove();
- ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ ASSERT_EQ(0, ioctx.operate("baz", &op));
}
{
bufferlist bl;
bl.append("ciao!");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ ASSERT_EQ(0, ioctx.operate("bam", &op));
}
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -799,12 +727,12 @@ TEST(LibRadosTier, EvictSnap) {
// read, trigger a promote on the head
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("bam", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
@@ -834,10 +762,10 @@ TEST(LibRadosTier, EvictSnap) {
}
// read foo snap
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('h', bl[0]);
}
@@ -846,7 +774,7 @@ TEST(LibRadosTier, EvictSnap) {
ObjectReadOperation op;
op.cache_evict();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -859,7 +787,7 @@ TEST(LibRadosTier, EvictSnap) {
ObjectReadOperation op;
op.read(1, 0, &bl, NULL);
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -867,13 +795,13 @@ TEST(LibRadosTier, EvictSnap) {
completion->release();
}
// head is still there...
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
bufferlist bl;
ObjectReadOperation op;
op.read(1, 0, &bl, NULL);
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -882,26 +810,26 @@ TEST(LibRadosTier, EvictSnap) {
}
// promote head + snap of bar
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
ASSERT_EQ('h', bl[0]);
}
// evict bar head (fail)
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
ObjectReadOperation op;
op.cache_evict();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"bar", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -910,12 +838,12 @@ TEST(LibRadosTier, EvictSnap) {
}
// evict bar snap
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
ObjectReadOperation op;
op.cache_evict();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"bar", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -923,13 +851,13 @@ TEST(LibRadosTier, EvictSnap) {
completion->release();
}
// ...and then head
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
bufferlist bl;
ObjectReadOperation op;
op.read(1, 0, &bl, NULL);
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"bar", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -940,7 +868,7 @@ TEST(LibRadosTier, EvictSnap) {
ObjectReadOperation op;
op.cache_evict();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"bar", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -950,40 +878,25 @@ TEST(LibRadosTier, EvictSnap) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, TryFlush) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, TryFlush) {
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -1000,7 +913,7 @@ TEST(LibRadosTier, TryFlush) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// verify the object is present in the cache tier
@@ -1014,8 +927,8 @@ TEST(LibRadosTier, TryFlush) {
// verify the object is NOT present in the base tier
{
- ObjectIterator it = base_ioctx.objects_begin();
- ASSERT_TRUE(it == base_ioctx.objects_end());
+ ObjectIterator it = ioctx.objects_begin();
+ ASSERT_TRUE(it == ioctx.objects_end());
}
// verify dirty
@@ -1056,11 +969,11 @@ TEST(LibRadosTier, TryFlush) {
// verify in base tier
{
- ObjectIterator it = base_ioctx.objects_begin();
- ASSERT_TRUE(it != base_ioctx.objects_end());
+ ObjectIterator it = ioctx.objects_begin();
+ ASSERT_TRUE(it != ioctx.objects_end());
ASSERT_TRUE(it->first == string("foo"));
++it;
- ASSERT_TRUE(it == base_ioctx.objects_end());
+ ASSERT_TRUE(it == ioctx.objects_end());
}
// evict it
@@ -1083,40 +996,25 @@ TEST(LibRadosTier, TryFlush) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, Flush) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, Flush) {
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -1135,7 +1033,7 @@ TEST(LibRadosTier, Flush) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// verify the object is present in the cache tier
@@ -1149,8 +1047,8 @@ TEST(LibRadosTier, Flush) {
// verify the object is NOT present in the base tier
{
- ObjectIterator it = base_ioctx.objects_begin();
- ASSERT_TRUE(it == base_ioctx.objects_end());
+ ObjectIterator it = ioctx.objects_begin();
+ ASSERT_TRUE(it == ioctx.objects_end());
}
// verify dirty
@@ -1191,11 +1089,11 @@ TEST(LibRadosTier, Flush) {
// verify in base tier
{
- ObjectIterator it = base_ioctx.objects_begin();
- ASSERT_TRUE(it != base_ioctx.objects_end());
+ ObjectIterator it = ioctx.objects_begin();
+ ASSERT_TRUE(it != ioctx.objects_end());
ASSERT_TRUE(it->first == string("foo"));
++it;
- ASSERT_TRUE(it == base_ioctx.objects_end());
+ ASSERT_TRUE(it == ioctx.objects_end());
}
// evict it
@@ -1227,7 +1125,7 @@ TEST(LibRadosTier, Flush) {
{
ObjectWriteOperation op;
op.remove();
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// flush whiteout
@@ -1262,46 +1160,31 @@ TEST(LibRadosTier, Flush) {
}
// or base tier
{
- ObjectIterator it = base_ioctx.objects_begin();
- ASSERT_TRUE(it == base_ioctx.objects_end());
+ ObjectIterator it = ioctx.objects_begin();
+ ASSERT_TRUE(it == ioctx.objects_end());
}
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, FlushSnap) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -1318,34 +1201,34 @@ TEST(LibRadosTier, FlushSnap) {
bl.append("a");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// create a snapshot, clone
vector<uint64_t> my_snaps(1);
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
my_snaps));
{
bufferlist bl;
bl.append("b");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// and another
my_snaps.resize(2);
my_snaps[1] = my_snaps[0];
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
- ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
my_snaps));
{
bufferlist bl;
bl.append("c");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// verify the object is present in the cache tier
@@ -1359,17 +1242,17 @@ TEST(LibRadosTier, FlushSnap) {
// verify the object is NOT present in the base tier
{
- ObjectIterator it = base_ioctx.objects_begin();
- ASSERT_TRUE(it == base_ioctx.objects_end());
+ ObjectIterator it = ioctx.objects_begin();
+ ASSERT_TRUE(it == ioctx.objects_end());
}
// flush on head (should fail)
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
ObjectReadOperation op;
op.cache_flush();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -1377,12 +1260,12 @@ TEST(LibRadosTier, FlushSnap) {
completion->release();
}
// flush on recent snap (should fail)
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
ObjectReadOperation op;
op.cache_flush();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -1390,12 +1273,12 @@ TEST(LibRadosTier, FlushSnap) {
completion->release();
}
// flush on oldest snap
- base_ioctx.snap_set_read(my_snaps[1]);
+ ioctx.snap_set_read(my_snaps[1]);
{
ObjectReadOperation op;
op.cache_flush();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -1403,12 +1286,12 @@ TEST(LibRadosTier, FlushSnap) {
completion->release();
}
// flush on next oldest snap
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
ObjectReadOperation op;
op.cache_flush();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -1416,12 +1299,12 @@ TEST(LibRadosTier, FlushSnap) {
completion->release();
}
// flush on head
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
ObjectReadOperation op;
op.cache_flush();
librados::AioCompletion *completion = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion, &op,
librados::OPERATION_IGNORE_CACHE, NULL));
completion->wait_for_safe();
@@ -1430,32 +1313,32 @@ TEST(LibRadosTier, FlushSnap) {
}
// verify i can read the snaps from the cache pool
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('b', bl[0]);
}
- base_ioctx.snap_set_read(my_snaps[1]);
+ ioctx.snap_set_read(my_snaps[1]);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('a', bl[0]);
}
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
@@ -1463,51 +1346,45 @@ TEST(LibRadosTier, FlushSnap) {
cluster.wait_for_latest_osdmap();
// verify i can read the snaps from the base pool
- base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ ioctx.snap_set_read(librados::SNAP_HEAD);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('c', bl[0]);
}
- base_ioctx.snap_set_read(my_snaps[0]);
+ ioctx.snap_set_read(my_snaps[0]);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('b', bl[0]);
}
- base_ioctx.snap_set_read(my_snaps[1]);
+ ioctx.snap_set_read(my_snaps[1]);
{
bufferlist bl;
- ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
ASSERT_EQ('a', bl[0]);
}
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, FlushWriteRaces) {
+TEST_F(LibRadosTierPP, FlushWriteRaces) {
Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ std::string pool_name = get_temp_pool_name();
+ std::string cache_pool_name = pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
IoCtx cache_ioctx;
ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+ IoCtx ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -1524,7 +1401,7 @@ TEST(LibRadosTier, FlushWriteRaces) {
{
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// flush + write
@@ -1539,7 +1416,7 @@ TEST(LibRadosTier, FlushWriteRaces) {
ObjectWriteOperation op2;
op2.write_full(bl);
librados::AioCompletion *completion2 = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate(
+ ASSERT_EQ(0, ioctx.aio_operate(
"foo", completion2, &op2, 0));
completion->wait_for_safe();
@@ -1558,7 +1435,7 @@ TEST(LibRadosTier, FlushWriteRaces) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// try-flush + write
@@ -1574,7 +1451,7 @@ TEST(LibRadosTier, FlushWriteRaces) {
ObjectWriteOperation op2;
op2.write_full(bl);
librados::AioCompletion *completion2 = cluster.aio_create_completion();
- ASSERT_EQ(0, base_ioctx.aio_operate("foo", completion2, &op2, 0));
+ ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
completion->wait_for_safe();
completion2->wait_for_safe();
@@ -1592,40 +1469,25 @@ TEST(LibRadosTier, FlushWriteRaces) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, FlushTryFlushRaces) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -1642,7 +1504,7 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// flush + flush
@@ -1675,7 +1537,7 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// flush + try-flush
@@ -1711,7 +1573,7 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// try-flush + flush
@@ -1752,7 +1614,7 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
bl.append("hi there");
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// try-flush + try-flush
@@ -1783,19 +1645,13 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
@@ -1832,25 +1688,16 @@ void flush_read_race_cb(completion_t cb, void *arg)
test_lock.Unlock();
}
-TEST(LibRadosTier, TryFlushReadRace) {
- Rados cluster;
- std::string base_pool_name = get_temp_pool_name();
- std::string cache_pool_name = base_pool_name + "-cache";
- ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
- ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
- IoCtx cache_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
- IoCtx base_ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
-
+TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
// configure cache
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
- "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name +
+ "\", \"force_nonempty\": \"--force-nonempty\" }",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
"\", \"overlaypool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
@@ -1870,11 +1717,11 @@ TEST(LibRadosTier, TryFlushReadRace) {
bl.append(bp);
ObjectWriteOperation op;
op.write_full(bl);
- ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
}
// start a continuous stream of reads
- read_ioctx = &base_ioctx;
+ read_ioctx = &ioctx;
test_lock.Lock();
for (int i = 0; i < max_reads; ++i) {
start_flush_read();
@@ -1904,28 +1751,16 @@ TEST(LibRadosTier, TryFlushReadRace) {
// tear down tiers
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
"\"}",
inbl, NULL, NULL));
ASSERT_EQ(0, cluster.mon_command(
- "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
"\", \"tierpool\": \"" + cache_pool_name + "\"}",
inbl, NULL, NULL));
-
- base_ioctx.close();
- cache_ioctx.close();
-
- cluster.pool_delete(cache_pool_name.c_str());
- ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
}
-TEST(LibRadosTier, HitSetNone) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosTierPP, HitSetNone) {
{
list< pair<time_t,time_t> > ls;
AioCompletion *c = librados::Rados::aio_create_completion();
@@ -1943,9 +1778,6 @@ TEST(LibRadosTier, HitSetNone) {
ASSERT_EQ(-ENOENT, c->get_return_value());
c->release();
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
string set_pool_str(string pool, string var, string val)
@@ -1958,17 +1790,11 @@ string set_pool_str(string pool, string var, string val)
string set_pool_str(string pool, string var, int val)
{
return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
- + string("\",\"var\": \"") + var + string("\",\"val\": ")
- + stringify(val) + string("}");
+ + string("\",\"var\": \"") + var + string("\",\"val\": \"")
+ + stringify(val) + string("\"}");
}
-TEST(LibRadosTier, HitSetRead) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosTierPP, HitSetRead) {
// enable hitset tracking for this pool
bufferlist inbl;
ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 2),
@@ -1982,6 +1808,8 @@ TEST(LibRadosTier, HitSetRead) {
// wait for maps to settle
cluster.wait_for_latest_osdmap();
+ ioctx.set_namespace("");
+
// keep reading until we see our object appear in the HitSet
utime_t start = ceph_clock_now(NULL);
utime_t hard_stop = start + utime_t(600, 0);
@@ -2019,9 +1847,6 @@ TEST(LibRadosTier, HitSetRead) {
sleep(1);
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
static int _get_pg_num(Rados& cluster, string pool_name)
@@ -2053,13 +1878,7 @@ static int _get_pg_num(Rados& cluster, string pool_name)
}
-TEST(LibRadosTier, HitSetWrite) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosTierPP, HitSetWrite) {
int num_pg = _get_pg_num(cluster, pool_name);
assert(num_pg > 0);
@@ -2076,6 +1895,8 @@ TEST(LibRadosTier, HitSetWrite) {
// wait for maps to settle
cluster.wait_for_latest_osdmap();
+ ioctx.set_namespace("");
+
// do a bunch of writes
for (int i=0; i<1000; ++i) {
bufferlist bl;
@@ -2128,18 +1949,9 @@ TEST(LibRadosTier, HitSetWrite) {
}
ASSERT_TRUE(found);
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosTier, HitSetTrim) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
+TEST_F(LibRadosTierPP, HitSetTrim) {
unsigned count = 3;
unsigned period = 3;
@@ -2157,6 +1969,8 @@ TEST(LibRadosTier, HitSetTrim) {
// wait for maps to settle
cluster.wait_for_latest_osdmap();
+ ioctx.set_namespace("");
+
// do a bunch of writes and make sure the hitsets rotate
utime_t start = ceph_clock_now(NULL);
utime_t hard_stop = start + utime_t(count * period * 12, 0);
@@ -2196,9 +2010,6 @@ TEST(LibRadosTier, HitSetTrim) {
sleep(1);
}
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
diff --git a/src/test/librados/watch_notify.cc b/src/test/librados/watch_notify.cc
index 2fdb44d..8616b5c 100644
--- a/src/test/librados/watch_notify.cc
+++ b/src/test/librados/watch_notify.cc
@@ -2,6 +2,7 @@
#include "include/rados/librados.hpp"
#include "include/rados/rados_types.h"
#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
#include <errno.h>
#include <semaphore.h>
@@ -9,6 +10,9 @@
using namespace librados;
+typedef RadosTest LibRadosWatchNotify;
+typedef RadosTestPP LibRadosWatchNotifyPP;
+
static sem_t sem;
static void watch_notify_test_cb(uint8_t opcode, uint64_t ver, void *arg)
@@ -25,14 +29,9 @@ public:
}
};
-TEST(LibRadosWatchNotify, WatchNotifyTest) {
+TEST_F(LibRadosWatchNotify, WatchNotifyTest) {
ASSERT_EQ(0, sem_init(&sem, 0, 0));
char buf[128];
- rados_t cluster;
- rados_ioctx_t ioctx;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool(pool_name, &cluster));
- rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
memset(buf, 0xcc, sizeof(buf));
ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, "foo", buf, sizeof(buf), 0));
uint64_t handle;
@@ -42,18 +41,11 @@ TEST(LibRadosWatchNotify, WatchNotifyTest) {
TestAlarm alarm;
sem_wait(&sem);
rados_unwatch(ioctx, "foo", handle);
- rados_ioctx_destroy(ioctx);
- ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
sem_destroy(&sem);
}
-TEST(LibRadosWatchNotify, WatchNotifyTestPP) {
+TEST_F(LibRadosWatchNotifyPP, WatchNotifyTestPP) {
ASSERT_EQ(0, sem_init(&sem, 0, 0));
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
@@ -70,23 +62,21 @@ TEST(LibRadosWatchNotify, WatchNotifyTestPP) {
TestAlarm alarm;
sem_wait(&sem);
ioctx.unwatch("foo", handle);
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
sem_destroy(&sem);
}
-TEST(LibRadosWatchNotify, WatchNotifyTimeoutTestPP) {
+TEST_F(LibRadosWatchNotifyPP, WatchNotifyTimeoutTestPP) {
ASSERT_EQ(0, sem_init(&sem, 0, 0));
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- cluster.ioctx_create(pool_name.c_str(), ioctx);
ioctx.set_notify_timeout(1);
uint64_t handle;
WatchNotifyTestCtx ctx;
+
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl1;
+ bl1.append(buf, sizeof(buf));
+ ASSERT_EQ((int)sizeof(buf), ioctx.write("foo", bl1, sizeof(buf), 0));
+
ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
sem_destroy(&sem);
}
diff --git a/src/test/mon/PGMap.cc b/src/test/mon/PGMap.cc
new file mode 100644
index 0000000..f7fbe7e
--- /dev/null
+++ b/src/test/mon/PGMap.cc
@@ -0,0 +1,96 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank <info at inktank.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2, as published by the Free Software
+ * Foundation. See file COPYING.
+ */
+
+#include "mon/PGMap.h"
+#include "gtest/gtest.h"
+
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+TEST(pgmap, min_last_epoch_clean)
+{
+ PGMap pg_map;
+ PGMap::Incremental inc;
+ osd_stat_t os;
+ pg_stat_t ps;
+
+ ps.last_epoch_clean = 999;
+ inc.pg_stat_updates[pg_t(9,9)] = ps;
+ inc.version = 1;
+ inc.update_stat(0, 123, os);
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(123u, pg_map.calc_min_last_epoch_clean());
+
+ inc = PGMap::Incremental();
+ inc.version = 2;
+ inc.update_stat(1, 222, os);
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(123u, pg_map.calc_min_last_epoch_clean());
+
+ inc = PGMap::Incremental();
+ inc.version = 3;
+ inc.update_stat(0, 222, os);
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(222u, pg_map.calc_min_last_epoch_clean());
+
+ inc = PGMap::Incremental();
+ inc.version = 4;
+ inc.update_stat(0, 333, os);
+ inc.update_stat(1, 333, os);
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(333u, pg_map.calc_min_last_epoch_clean());
+
+ ps.last_epoch_clean = 222;
+ inc = PGMap::Incremental();
+ inc.version = 5;
+ inc.pg_stat_updates[pg_t(1,1)] = ps;
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(222u, pg_map.calc_min_last_epoch_clean());
+
+ ps.last_epoch_clean = 223;
+ inc = PGMap::Incremental();
+ inc.version = 6;
+ inc.pg_stat_updates[pg_t(1,1)] = ps;
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(223u, pg_map.calc_min_last_epoch_clean());
+
+ ps.last_epoch_clean = 224;
+ inc = PGMap::Incremental();
+ inc.version = 7;
+ inc.pg_stat_updates[pg_t(2,2)] = ps;
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(223u, pg_map.calc_min_last_epoch_clean());
+
+ ps.last_epoch_clean = 225;
+ inc = PGMap::Incremental();
+ inc.version = 8;
+ inc.pg_stat_updates[pg_t(1,1)] = ps;
+ pg_map.apply_incremental(g_ceph_context, inc);
+ ASSERT_EQ(224u, pg_map.calc_min_last_epoch_clean());
+
+}
+
+
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+ env_to_vec(args);
+
+ vector<const char*> def_args;
+ global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/src/test/mon/mon-test-helpers.sh b/src/test/mon/mon-test-helpers.sh
new file mode 100644
index 0000000..32db2bb
--- /dev/null
+++ b/src/test/mon/mon-test-helpers.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+#
+# Author: Loic Dachary <loic at dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library Public License for more details.
+#
+function setup() {
+ local dir=$1
+ teardown $dir
+ mkdir $dir
+}
+
+function teardown() {
+ local dir=$1
+ kill_daemons $dir
+ rm -fr $dir
+}
+
+function run_mon() {
+ local dir=$1
+ shift
+ local id=$1
+ shift
+ dir+=/$id
+
+ ./ceph-mon \
+ --id $id \
+ --mkfs \
+ --mon-data=$dir --run-dir=$dir \
+ "$@"
+
+ ./ceph-mon \
+ --id $id \
+ --paxos-propose-interval=0.1 \
+ --osd-pool-default-erasure-code-directory=.libs \
+ --debug-mon 20 \
+ --debug-ms 20 \
+ --debug-paxos 20 \
+ --mon-advanced-debug-mode \
+ --chdir= \
+ --mon-data=$dir \
+ --log-file=$dir/log \
+ --mon-cluster-log-file=$dir/log \
+ --run-dir=$dir \
+ --pid-file=$dir/pidfile \
+ "$@"
+}
+
+function kill_daemons() {
+ local dir=$1
+ for pidfile in $(find $dir | grep pidfile) ; do
+ for try in 0 1 1 1 2 3 ; do
+ kill -9 $(cat $pidfile 2> /dev/null) 2> /dev/null || break
+ sleep $try
+ done
+ done
+}
+
+function main() {
+ local dir=$1
+
+ export PATH=:$PATH # make sure program from sources are prefered
+
+ PS4='${FUNCNAME[0]}: $LINENO: '
+ export CEPH_CONF=/dev/null
+ unset CEPH_ARGS
+
+ set -x
+ setup $dir || return 1
+ run $dir || return 1
+ teardown $dir || return 1
+}
diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc
index 138bdc0..3c6ff56 100644
--- a/src/test/mon/test_mon_workloadgen.cc
+++ b/src/test/mon/test_mon_workloadgen.cc
@@ -257,7 +257,7 @@ class ClientStub : public TestStub
<< messenger->get_myaddr() << dendl;
objecter.reset(new Objecter(cct, messenger.get(), &monc, &osdmap,
- lock, timer));
+ lock, timer, 0, 0));
assert(objecter.get() != NULL);
objecter->set_balanced_budget();
diff --git a/src/test/filestore/DeterministicOpSequence.cc b/src/test/objectstore/DeterministicOpSequence.cc
similarity index 99%
rename from src/test/filestore/DeterministicOpSequence.cc
rename to src/test/objectstore/DeterministicOpSequence.cc
index d9a0be4..09c9f5e 100644
--- a/src/test/filestore/DeterministicOpSequence.cc
+++ b/src/test/objectstore/DeterministicOpSequence.cc
@@ -18,7 +18,7 @@
#include <stdlib.h>
#include <signal.h>
#include <sstream>
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include "common/ceph_argparse.h"
#include "global/global_init.h"
#include "common/debug.h"
@@ -34,9 +34,9 @@
#undef dout_prefix
#define dout_prefix *_dout << "deterministic_seq "
-DeterministicOpSequence::DeterministicOpSequence(FileStore *store,
+DeterministicOpSequence::DeterministicOpSequence(ObjectStore *store,
std::string status)
- : TestFileStoreState(store),
+ : TestObjectStoreState(store),
txn(0),
m_osr("OSR")
{
diff --git a/src/test/filestore/DeterministicOpSequence.h b/src/test/objectstore/DeterministicOpSequence.h
similarity index 93%
rename from src/test/filestore/DeterministicOpSequence.h
rename to src/test/objectstore/DeterministicOpSequence.h
index 818d0ed..1980c98 100644
--- a/src/test/filestore/DeterministicOpSequence.h
+++ b/src/test/objectstore/DeterministicOpSequence.h
@@ -16,18 +16,18 @@
#include <iostream>
#include <fstream>
#include <set>
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include <boost/scoped_ptr.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
-#include "TestFileStoreState.h"
+#include "TestObjectStoreState.h"
typedef boost::mt11213b rngen_t;
-class DeterministicOpSequence : public TestFileStoreState {
+class DeterministicOpSequence : public TestObjectStoreState {
public:
- DeterministicOpSequence(FileStore *store, std::string status = std::string());
+ DeterministicOpSequence(ObjectStore *store, std::string status = std::string());
virtual ~DeterministicOpSequence();
virtual void generate(int seed, int num_txs);
diff --git a/src/test/filestore/FileStoreDiff.cc b/src/test/objectstore/FileStoreDiff.cc
similarity index 100%
rename from src/test/filestore/FileStoreDiff.cc
rename to src/test/objectstore/FileStoreDiff.cc
diff --git a/src/test/filestore/FileStoreDiff.h b/src/test/objectstore/FileStoreDiff.h
similarity index 100%
rename from src/test/filestore/FileStoreDiff.h
rename to src/test/objectstore/FileStoreDiff.h
diff --git a/src/test/filestore/FileStoreTracker.cc b/src/test/objectstore/FileStoreTracker.cc
similarity index 100%
rename from src/test/filestore/FileStoreTracker.cc
rename to src/test/objectstore/FileStoreTracker.cc
diff --git a/src/test/filestore/FileStoreTracker.h b/src/test/objectstore/FileStoreTracker.h
similarity index 100%
rename from src/test/filestore/FileStoreTracker.h
rename to src/test/objectstore/FileStoreTracker.h
diff --git a/src/test/filestore/TestFileStoreState.cc b/src/test/objectstore/TestObjectStoreState.cc
similarity index 83%
rename from src/test/filestore/TestFileStoreState.cc
rename to src/test/objectstore/TestObjectStoreState.cc
index a34e526..37f99c7 100644
--- a/src/test/filestore/TestFileStoreState.cc
+++ b/src/test/objectstore/TestObjectStoreState.cc
@@ -16,23 +16,23 @@
#include <time.h>
#include <stdlib.h>
#include <signal.h>
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include "common/ceph_argparse.h"
#include "global/global_init.h"
#include "common/debug.h"
#include <boost/scoped_ptr.hpp>
#include <boost/lexical_cast.hpp>
-#include "TestFileStoreState.h"
+#include "TestObjectStoreState.h"
#include "include/assert.h"
#define dout_subsys ceph_subsys_filestore
#undef dout_prefix
-#define dout_prefix *_dout << "ceph_test_filestore_state "
+#define dout_prefix *_dout << "ceph_test_objectstore_state "
-const coll_t TestFileStoreState::META_COLL("meta");
-const coll_t TestFileStoreState::TEMP_COLL("temp");
+const coll_t TestObjectStoreState::META_COLL("meta");
+const coll_t TestObjectStoreState::TEMP_COLL("temp");
-void TestFileStoreState::init(int colls, int objs)
+void TestObjectStoreState::init(int colls, int objs)
{
dout(5) << "init " << colls << " colls " << objs << " objs" << dendl;
@@ -77,7 +77,7 @@ void TestFileStoreState::init(int colls, int objs)
dout(5) << "init finished" << dendl;
}
-TestFileStoreState::coll_entry_t *TestFileStoreState::coll_create(int id)
+TestObjectStoreState::coll_entry_t *TestObjectStoreState::coll_create(int id)
{
char buf[100];
char meta_buf[100];
@@ -88,8 +88,8 @@ TestFileStoreState::coll_entry_t *TestFileStoreState::coll_create(int id)
return (new coll_entry_t(id, buf, meta_buf));
}
-TestFileStoreState::coll_entry_t*
-TestFileStoreState::get_coll(int key, bool erase)
+TestObjectStoreState::coll_entry_t*
+TestObjectStoreState::get_coll(int key, bool erase)
{
dout(5) << "get_coll id " << key << dendl;
@@ -115,8 +115,8 @@ TestFileStoreState::get_coll(int key, bool erase)
return entry;
}
-TestFileStoreState::coll_entry_t*
-TestFileStoreState::get_coll_at(int pos, bool erase)
+TestObjectStoreState::coll_entry_t*
+TestObjectStoreState::get_coll_at(int pos, bool erase)
{
dout(5) << "get_coll_at pos " << pos << dendl;
@@ -147,7 +147,7 @@ TestFileStoreState::get_coll_at(int pos, bool erase)
return entry;
}
-TestFileStoreState::coll_entry_t::~coll_entry_t()
+TestObjectStoreState::coll_entry_t::~coll_entry_t()
{
if (m_objects.size() > 0) {
map<int, hobject_t*>::iterator it = m_objects.begin();
@@ -161,14 +161,14 @@ TestFileStoreState::coll_entry_t::~coll_entry_t()
}
}
-bool TestFileStoreState::coll_entry_t::check_for_obj(int id)
+bool TestObjectStoreState::coll_entry_t::check_for_obj(int id)
{
if (m_objects.count(id))
return true;
return false;
}
-hobject_t *TestFileStoreState::coll_entry_t::touch_obj(int id)
+hobject_t *TestObjectStoreState::coll_entry_t::touch_obj(int id)
{
map<int, hobject_t*>::iterator it = m_objects.find(id);
if (it != m_objects.end()) {
@@ -188,7 +188,7 @@ hobject_t *TestFileStoreState::coll_entry_t::touch_obj(int id)
return obj;
}
-hobject_t *TestFileStoreState::coll_entry_t::get_obj(int id)
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj(int id)
{
return get_obj(id, false);
}
@@ -198,12 +198,12 @@ hobject_t *TestFileStoreState::coll_entry_t::get_obj(int id)
* @param id Object's id in the map.
* @return The object or NULL in case of error.
*/
-hobject_t *TestFileStoreState::coll_entry_t::remove_obj(int id)
+hobject_t *TestObjectStoreState::coll_entry_t::remove_obj(int id)
{
return get_obj(id, true);
}
-hobject_t *TestFileStoreState::coll_entry_t::get_obj(int id, bool remove)
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj(int id, bool remove)
{
map<int, hobject_t*>::iterator it = m_objects.find(id);
if (it == m_objects.end()) {
@@ -222,7 +222,7 @@ hobject_t *TestFileStoreState::coll_entry_t::get_obj(int id, bool remove)
return obj;
}
-hobject_t *TestFileStoreState::coll_entry_t::get_obj_at(int pos, int *key)
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj_at(int pos, int *key)
{
return get_obj_at(pos, false, key);
}
@@ -232,12 +232,12 @@ hobject_t *TestFileStoreState::coll_entry_t::get_obj_at(int pos, int *key)
* @param pos The map's position in which the object lies.
* @return The object or NULL in case of error.
*/
-hobject_t *TestFileStoreState::coll_entry_t::remove_obj_at(int pos, int *key)
+hobject_t *TestObjectStoreState::coll_entry_t::remove_obj_at(int pos, int *key)
{
return get_obj_at(pos, true, key);
}
-hobject_t *TestFileStoreState::coll_entry_t::get_obj_at(int pos,
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj_at(int pos,
bool remove, int *key)
{
if (m_objects.empty()) {
@@ -274,13 +274,13 @@ hobject_t *TestFileStoreState::coll_entry_t::get_obj_at(int pos,
}
hobject_t*
-TestFileStoreState::coll_entry_t::replace_obj(int id, hobject_t *obj) {
+TestObjectStoreState::coll_entry_t::replace_obj(int id, hobject_t *obj) {
hobject_t *old_obj = remove_obj(id);
m_objects.insert(make_pair(id, obj));
return old_obj;
}
-int TestFileStoreState::coll_entry_t::get_random_obj_id(rngen_t& gen)
+int TestObjectStoreState::coll_entry_t::get_random_obj_id(rngen_t& gen)
{
ceph_assert(!m_objects.empty());
diff --git a/src/test/filestore/TestFileStoreState.h b/src/test/objectstore/TestObjectStoreState.h
similarity index 91%
rename from src/test/filestore/TestFileStoreState.h
rename to src/test/objectstore/TestObjectStoreState.h
index 7cd4527..dad2aab 100644
--- a/src/test/filestore/TestFileStoreState.h
+++ b/src/test/objectstore/TestObjectStoreState.h
@@ -10,10 +10,10 @@
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*/
-#ifndef TEST_FILESTORE_STATE_H_
-#define TEST_FILESTORE_STATE_H_
+#ifndef TEST_OBJECTSTORE_STATE_H_
+#define TEST_OBJECTSTORE_STATE_H_
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include <boost/scoped_ptr.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
@@ -22,7 +22,7 @@
typedef boost::mt11213b rngen_t;
-class TestFileStoreState {
+class TestObjectStoreState {
public:
struct coll_entry_t {
int m_id;
@@ -96,13 +96,13 @@ public:
static const int m_default_num_colls = 30;
public:
- TestFileStoreState(FileStore *store) :
+ TestObjectStoreState(ObjectStore *store) :
m_next_coll_nr(0), m_num_objs_per_coll(10), m_num_objects(0),
m_max_in_flight(0), m_finished_lock("Finished Lock") {
m_in_flight.set(0);
m_store.reset(store);
}
- ~TestFileStoreState() {
+ ~TestObjectStoreState() {
map<int, coll_entry_t*>::iterator it = m_collections.begin();
while (it != m_collections.end()) {
if (it->second)
@@ -128,11 +128,11 @@ public:
class C_OnFinished: public Context {
protected:
- TestFileStoreState *m_state;
+ TestObjectStoreState *m_state;
ObjectStore::Transaction *m_tx;
public:
- C_OnFinished(TestFileStoreState *state,
+ C_OnFinished(TestObjectStoreState *state,
ObjectStore::Transaction *t) : m_state(state), m_tx(t) { }
void finish(int r) {
@@ -145,4 +145,4 @@ public:
};
};
-#endif /* TEST_FILESTORE_STATE_H_ */
+#endif /* TEST_OBJECTSTORE_STATE_H_ */
diff --git a/src/test/filestore/chain_xattr.cc b/src/test/objectstore/chain_xattr.cc
similarity index 100%
rename from src/test/filestore/chain_xattr.cc
rename to src/test/objectstore/chain_xattr.cc
diff --git a/src/test/filestore/store_test.cc b/src/test/objectstore/store_test.cc
similarity index 84%
rename from src/test/filestore/store_test.cc
rename to src/test/objectstore/store_test.cc
index 8e88ab3..8133d4d 100644
--- a/src/test/filestore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -1,4 +1,4 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
@@ -7,9 +7,9 @@
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
+ * License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
- *
+ *
*/
#include <stdio.h>
@@ -17,6 +17,7 @@
#include <iostream>
#include <time.h>
#include <sys/mount.h>
+#include "os/ObjectStore.h"
#include "os/FileStore.h"
#include "os/KeyValueStore.h"
#include "include/Context.h"
@@ -269,6 +270,7 @@ TEST_P(StoreTest, ManyObjectTest) {
}
}
+
class ObjectGenerator {
public:
virtual ghobject_t create_object(gen_type *gen) = 0;
@@ -303,10 +305,12 @@ class SyntheticWorkloadState {
public:
static const unsigned max_in_flight = 16;
static const unsigned max_objects = 3000;
+ static const unsigned max_object_len = 1024 * 20;
coll_t cid;
unsigned in_flight;
+ map<ghobject_t, bufferlist> contents;
set<ghobject_t> available_objects;
- set<ghobject_t> in_use_objects;
+ set<ghobject_t> in_flight_objects;
ObjectGenerator *object_gen;
gen_type *rng;
ObjectStore *store;
@@ -321,25 +325,39 @@ public:
ObjectStore::Transaction *t;
ghobject_t hoid;
C_SyntheticOnReadable(SyntheticWorkloadState *state,
- ObjectStore::Transaction *t, ghobject_t hoid)
+ ObjectStore::Transaction *t, ghobject_t hoid)
: state(state), t(t), hoid(hoid) {}
void finish(int r) {
- ASSERT_TRUE(r >= 0);
Mutex::Locker locker(state->lock);
- if (state->in_use_objects.count(hoid)) {
- state->available_objects.insert(hoid);
- state->in_use_objects.erase(hoid);
- }
+ ASSERT_TRUE(state->in_flight_objects.count(hoid));
+ ASSERT_EQ(r, 0);
+ state->in_flight_objects.erase(hoid);
+ if (state->contents.count(hoid))
+ state->available_objects.insert(hoid);
--(state->in_flight);
state->cond.Signal();
}
};
-
-
- SyntheticWorkloadState(ObjectStore *store,
- ObjectGenerator *gen,
- gen_type *rng,
+
+ static void filled_byte_array(bufferlist& bl, size_t size)
+ {
+ static const char alphanum[] = "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+
+ bufferptr bp(size);
+ for (unsigned int i = 0; i < size - 1; i++) {
+ bp[i] = alphanum[rand() % sizeof(alphanum)];
+ }
+ bp[size - 1] = '\0';
+
+ bl.append(bp);
+ }
+
+ SyntheticWorkloadState(ObjectStore *store,
+ ObjectGenerator *gen,
+ gen_type *rng,
ObjectStore::Sequencer *osr,
coll_t cid)
: cid(cid), in_flight(0), object_gen(gen), rng(rng), store(store), osr(osr),
@@ -359,7 +377,6 @@ public:
set<ghobject_t>::iterator i = available_objects.begin();
for ( ; index > 0; --index, ++i) ;
ghobject_t ret = *i;
- available_objects.erase(i);
return ret;
}
@@ -373,13 +390,13 @@ public:
while (in_flight)
cond.Wait(lock);
}
-
+
bool can_create() {
- return (available_objects.size() + in_use_objects.size()) < max_objects;
+ return (available_objects.size() + in_flight_objects.size()) < max_objects;
}
bool can_unlink() {
- return (available_objects.size() + in_use_objects.size()) > 0;
+ return (available_objects.size() + in_flight_objects.size()) > 0;
}
int touch() {
@@ -388,14 +405,114 @@ public:
return -ENOSPC;
wait_for_ready();
ghobject_t new_obj = object_gen->create_object(rng);
- in_use_objects.insert(new_obj);
available_objects.erase(new_obj);
ObjectStore::Transaction *t = new ObjectStore::Transaction;
t->touch(cid, new_obj);
++in_flight;
+ in_flight_objects.insert(new_obj);
+ if (!contents.count(new_obj))
+ contents[new_obj] = bufferlist();
+ return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, new_obj));
+ }
+
+ int write() {
+ Mutex::Locker locker(lock);
+ if (!can_unlink())
+ return -ENOENT;
+ wait_for_ready();
+
+ ghobject_t new_obj = get_uniform_random_object();
+ available_objects.erase(new_obj);
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+
+ boost::uniform_int<> u1(0, max_object_len/2);
+ boost::uniform_int<> u2(0, max_object_len);
+ uint64_t offset = u1(*rng);
+ uint64_t len = u2(*rng);
+ bufferlist bl;
+ if (offset > len)
+ swap(offset, len);
+
+ filled_byte_array(bl, len);
+
+ if (contents[new_obj].length() <= offset) {
+ contents[new_obj].append_zero(offset-contents[new_obj].length());
+ contents[new_obj].append(bl);
+ } else {
+ bufferlist value;
+ contents[new_obj].copy(0, offset, value);
+ value.append(bl);
+ if (value.length() < contents[new_obj].length())
+ contents[new_obj].copy(value.length(), contents[new_obj].length()-value.length(), value);
+ value.swap(contents[new_obj]);
+ }
+
+ t->write(cid, new_obj, offset, len, bl);
+ ++in_flight;
+ in_flight_objects.insert(new_obj);
return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, new_obj));
}
+ void read() {
+ boost::uniform_int<> u1(0, max_object_len/2);
+ boost::uniform_int<> u2(0, max_object_len);
+ uint64_t offset = u1(*rng);
+ uint64_t len = u2(*rng);
+ if (offset > len)
+ swap(offset, len);
+
+ ghobject_t obj;
+ int r;
+ {
+ Mutex::Locker locker(lock);
+ if (!can_unlink())
+ return ;
+ wait_for_ready();
+
+ obj = get_uniform_random_object();
+ }
+ bufferlist bl, result;
+ r = store->read(cid, obj, offset, len, result);
+ if (offset >= contents[obj].length()) {
+ ASSERT_EQ(r, 0);
+ } else {
+ size_t max_len = contents[obj].length() - offset;
+ if (len > max_len)
+ len = max_len;
+ ASSERT_EQ(len, result.length());
+ contents[obj].copy(offset, len, bl);
+ ASSERT_EQ(r, (int)len);
+ ASSERT_TRUE(result.contents_equal(bl));
+ }
+ }
+
+ int truncate() {
+ Mutex::Locker locker(lock);
+ if (!can_unlink())
+ return -ENOENT;
+ wait_for_ready();
+
+ ghobject_t obj = get_uniform_random_object();
+ available_objects.erase(obj);
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+
+ boost::uniform_int<> choose(0, max_object_len);
+ size_t len = choose(*rng);
+ bufferlist bl;
+
+ t->truncate(cid, obj, len);
+ ++in_flight;
+ in_flight_objects.insert(obj);
+ if (contents[obj].length() <= len)
+ contents[obj].append_zero(len - contents[obj].length());
+ else {
+ contents[obj].copy(0, len, bl);
+ bl.swap(contents[obj]);
+ }
+
+ return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, obj));
+ }
+
void scan() {
Mutex::Locker locker(lock);
while (in_flight)
@@ -405,7 +522,7 @@ public:
ghobject_t next, current;
while (1) {
cerr << "scanning..." << std::endl;
- int r = store->collection_list_partial(cid, current, 50, 100,
+ int r = store->collection_list_partial(cid, current, 50, 100,
0, &objects, &next);
ASSERT_EQ(r, 0);
ASSERT_TRUE(sorted(objects));
@@ -432,26 +549,28 @@ public:
}
}
- int stat() {
+ void stat() {
ghobject_t hoid;
{
Mutex::Locker locker(lock);
if (!can_unlink())
- return -ENOENT;
+ return ;
hoid = get_uniform_random_object();
- in_use_objects.insert(hoid);
+ in_flight_objects.insert(hoid);
+ available_objects.erase(hoid);
++in_flight;
}
struct stat buf;
int r = store->stat(cid, hoid, &buf);
+ ASSERT_EQ(0, r);
+ ASSERT_TRUE(buf.st_size == contents[hoid].length());
{
Mutex::Locker locker(lock);
--in_flight;
cond.Signal();
- in_use_objects.erase(hoid);
+ in_flight_objects.erase(hoid);
available_objects.insert(hoid);
}
- return r;
}
int unlink() {
@@ -462,14 +581,17 @@ public:
ObjectStore::Transaction *t = new ObjectStore::Transaction;
t->remove(cid, to_remove);
++in_flight;
+ available_objects.erase(to_remove);
+ in_flight_objects.insert(to_remove);
+ contents.erase(to_remove);
return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, to_remove));
}
void print_internal_state() {
Mutex::Locker locker(lock);
cerr << "available_objects: " << available_objects.size()
- << " in_use_objects: " << in_use_objects.size()
- << " total objects: " << in_use_objects.size() + available_objects.size()
+ << " in_flight_objects: " << in_flight_objects.size()
+ << " total objects: " << in_flight_objects.size() + available_objects.size()
<< " in_flight " << in_flight << std::endl;
}
};
@@ -479,7 +601,7 @@ TEST_P(StoreTest, Synthetic) {
MixedGenerator gen;
gen_type rng(time(NULL));
coll_t cid("synthetic_1");
-
+
SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid);
test_obj.init();
for (int i = 0; i < 1000; ++i) {
@@ -495,12 +617,16 @@ TEST_P(StoreTest, Synthetic) {
int val = true_false(rng);
if (val > 97) {
test_obj.scan();
- } else if (val > 50) {
+ } else if (val > 90) {
test_obj.stat();
- } else if (val > 30) {
+ } else if (val > 85) {
test_obj.unlink();
+ } else if (val > 50) {
+ test_obj.write();
+ } else if (val > 10) {
+ test_obj.read();
} else {
- test_obj.touch();
+ test_obj.truncate();
}
}
test_obj.wait_for_done();
@@ -676,6 +802,28 @@ TEST_P(StoreTest, OMapTest) {
++i;
}
+ {
+ bufferlist bl1;
+ bl1.append("omap_header");
+ ObjectStore::Transaction t;
+ t.omap_setheader(cid, hoid, bl1);
+ store->apply_transaction(t);
+
+ bufferlist bl2;
+ bl2.append("value");
+ map<string, bufferlist> to_add;
+ to_add.insert(pair<string, bufferlist>("key", bl2));
+ t.omap_setkeys(cid, hoid, to_add);
+ store->apply_transaction(t);
+
+ bufferlist bl3;
+ map<string, bufferlist> cur_attrs;
+ r = store->omap_get(cid, hoid, &bl3, &cur_attrs);
+ ASSERT_EQ(r, 0);
+ ASSERT_EQ(cur_attrs.size(), size_t(1));
+ ASSERT_TRUE(bl3.contents_equal(bl1));
+ }
+
ObjectStore::Transaction t;
t.remove(cid, hoid);
t.remove_collection(cid);
@@ -1061,5 +1209,5 @@ int main(int argc, char **argv) {
}
// Local Variables:
-// compile-command: "cd ../.. ; make ceph_test_filestore ; ./ceph_test_filestore --gtest_filter=StoreTest.* --log-to-stderr=true --debug-filestore=20"
+// compile-command: "cd ../.. ; make ceph_test_objectstore ; ./ceph_test_objectstore --gtest_filter=StoreTest.* --log-to-stderr=true --debug-filestore=20"
// End:
diff --git a/src/test/filestore/test_idempotent.cc b/src/test/objectstore/test_idempotent.cc
similarity index 100%
rename from src/test/filestore/test_idempotent.cc
rename to src/test/objectstore/test_idempotent.cc
diff --git a/src/test/filestore/test_idempotent_sequence.cc b/src/test/objectstore/test_idempotent_sequence.cc
similarity index 100%
rename from src/test/filestore/test_idempotent_sequence.cc
rename to src/test/objectstore/test_idempotent_sequence.cc
diff --git a/src/test/filestore/workload_generator.cc b/src/test/objectstore/workload_generator.cc
similarity index 96%
rename from src/test/filestore/workload_generator.cc
rename to src/test/objectstore/workload_generator.cc
index aadb475..acf0fc1 100644
--- a/src/test/filestore/workload_generator.cc
+++ b/src/test/objectstore/workload_generator.cc
@@ -20,7 +20,7 @@
#include <cctype>
#include <errno.h>
#include <sys/time.h>
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include "common/ceph_argparse.h"
#include "global/global_init.h"
#include "common/debug.h"
@@ -29,7 +29,7 @@
#include "workload_generator.h"
#include "include/assert.h"
-#include "TestFileStoreState.h"
+#include "TestObjectStoreState.h"
static const char *our_name = NULL;
void usage();
@@ -40,7 +40,7 @@ boost::scoped_ptr<WorkloadGenerator> wrkldgen;
WorkloadGenerator::WorkloadGenerator(vector<const char*> args)
- : TestFileStoreState(NULL),
+ : TestObjectStoreState(NULL),
m_max_in_flight(def_max_in_flight),
m_num_ops(-1),
m_destroy_coll_every_nr_runs(def_destroy_coll_every_nr_runs),
@@ -67,7 +67,10 @@ WorkloadGenerator::WorkloadGenerator(vector<const char*> args)
err = ::mkdir(g_conf->osd_data.c_str(), 0755);
ceph_assert(err == 0 || (err < 0 && errno == EEXIST));
- ObjectStore *store_ptr = new FileStore(g_conf->osd_data, g_conf->osd_journal);
+ ObjectStore *store_ptr = ObjectStore::create(g_ceph_context,
+ g_conf->osd_objectstore,
+ g_conf->osd_data,
+ g_conf->osd_journal);
m_store.reset(store_ptr);
err = m_store->mkfs();
ceph_assert(err == 0);
@@ -193,7 +196,7 @@ int WorkloadGenerator::get_uniform_random_value(int min, int max)
return value(m_rng);
}
-TestFileStoreState::coll_entry_t *WorkloadGenerator::get_rnd_coll_entry(bool erase = false)
+TestObjectStoreState::coll_entry_t *WorkloadGenerator::get_rnd_coll_entry(bool erase = false)
{
int index = get_uniform_random_value(0, m_collections_ids.size()-1);
coll_entry_t *entry = get_coll_at(index, erase);
@@ -358,7 +361,7 @@ void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t,
t->remove(META_COLL, entry->m_meta_obj);
}
-TestFileStoreState::coll_entry_t
+TestObjectStoreState::coll_entry_t
*WorkloadGenerator::do_create_collection(ObjectStore::Transaction *t,
C_StatState *stat)
{
@@ -429,7 +432,7 @@ void WorkloadGenerator::run()
ObjectStore::Transaction *t = new ObjectStore::Transaction;
Context *c;
bool destroy_collection = false;
- TestFileStoreState::coll_entry_t *entry = NULL;
+ TestObjectStoreState::coll_entry_t *entry = NULL;
if (m_do_stats) {
@@ -508,6 +511,7 @@ void usage()
\n\
Global Options:\n\
-c FILE Read configuration from FILE\n\
+ --osd-objectstore TYPE Set OSD ObjectStore type\n\
--osd-data PATH Set OSD Data path\n\
--osd-journal PATH Set OSD Journal path\n\
--osd-journal-size VAL Set Journal size\n\
diff --git a/src/test/filestore/workload_generator.h b/src/test/objectstore/workload_generator.h
similarity index 93%
rename from src/test/filestore/workload_generator.h
rename to src/test/objectstore/workload_generator.h
index 80e95da..3235fe8 100644
--- a/src/test/filestore/workload_generator.h
+++ b/src/test/objectstore/workload_generator.h
@@ -13,18 +13,18 @@
#ifndef WORKLOAD_GENERATOR_H_
#define WORKLOAD_GENERATOR_H_
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include <boost/scoped_ptr.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <map>
#include <sys/time.h>
-#include "TestFileStoreState.h"
+#include "TestObjectStoreState.h"
typedef boost::mt11213b rngen_t;
-class WorkloadGenerator : public TestFileStoreState {
+class WorkloadGenerator : public TestObjectStoreState {
public:
static const int def_max_in_flight = 50;
@@ -125,17 +125,17 @@ public:
m_store->umount();
}
- class C_OnReadable: public TestFileStoreState::C_OnFinished {
+ class C_OnReadable: public TestObjectStoreState::C_OnFinished {
WorkloadGenerator *wrkldgen_state;
public:
C_OnReadable(WorkloadGenerator *state,
ObjectStore::Transaction *t)
- :TestFileStoreState::C_OnFinished(state, t), wrkldgen_state(state) { }
+ :TestObjectStoreState::C_OnFinished(state, t), wrkldgen_state(state) { }
void finish(int r)
{
- TestFileStoreState::C_OnFinished::finish(r);
+ TestObjectStoreState::C_OnFinished::finish(r);
wrkldgen_state->m_nr_runs.inc();
}
};
diff --git a/src/test/osd/Object.cc b/src/test/osd/Object.cc
index 279ae23..c98d62d 100644
--- a/src/test/osd/Object.cc
+++ b/src/test/osd/Object.cc
@@ -44,13 +44,15 @@ void AppendGenerator::get_ranges_map(
uint64_t pos = off;
uint64_t limit = off + get_append_size(cont);
while (pos < limit) {
- uint64_t segment_length = (
- rand() % (max_append_size - min_append_size)) + min_append_size;
- assert(segment_length < max_append_size);
+ uint64_t segment_length = round_up(
+ rand() % (max_append_size - min_append_size),
+ alignment) + min_append_size;
assert(segment_length >= min_append_size);
if (segment_length + pos > limit) {
segment_length = limit - pos;
}
+ if (alignment)
+ assert(segment_length % alignment == 0);
out.insert(make_pair(pos, segment_length));
pos += segment_length;
}
diff --git a/src/test/osd/Object.h b/src/test/osd/Object.h
index 1f4defd..d39d36c 100644
--- a/src/test/osd/Object.h
+++ b/src/test/osd/Object.h
@@ -238,21 +238,34 @@ public:
class AppendGenerator : public RandGenerator {
uint64_t off;
+ uint64_t alignment;
uint64_t min_append_size;
uint64_t max_append_size;
uint64_t max_append_total;
+
+ uint64_t round_up(uint64_t in, uint64_t by) {
+ if (by)
+ in += (by - (in % by));
+ return in;
+ }
+
public:
AppendGenerator(
uint64_t off,
+ uint64_t alignment,
uint64_t min_append_size,
- uint64_t max_append_size,
- uint64_t max_append_total) :
- off(off), min_append_size(min_append_size),
- max_append_size(max_append_size),
- max_append_total(max_append_total) {}
+ uint64_t _max_append_size,
+ uint64_t max_append_multiple) :
+ off(off), alignment(alignment),
+ min_append_size(round_up(min_append_size, alignment)),
+ max_append_size(round_up(_max_append_size, alignment)) {
+ if (_max_append_size == min_append_size)
+ max_append_size += alignment;
+ max_append_total = max_append_multiple * max_append_size;
+ }
uint64_t get_append_size(const ContDesc &in) {
RandWrap rand(in.seqnum);
- return rand() % max_append_total;
+ return round_up(rand() % max_append_total, alignment);
}
uint64_t get_length(const ContDesc &in) {
return off + get_append_size(in);
diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h
index 4a5b861..f199d03 100644
--- a/src/test/osd/RadosModel.h
+++ b/src/test/osd/RadosModel.h
@@ -378,6 +378,7 @@ public:
{
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, contents));
+ pool_obj_cont[current_snap][oid].dirty = true;
}
void update_object_undirty(const string &oid)
@@ -543,8 +544,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- comp = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
+ comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
}
@@ -634,8 +635,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- comp = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
+ comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
}
@@ -708,9 +709,11 @@ public:
0;
cont_gen = new AppendGenerator(
prev_length,
+ (context->io_ctx.pool_requires_alignment() ?
+ context->io_ctx.pool_required_alignment() : 0),
context->min_stride_size,
context->max_stride_size,
- 3 * context->max_stride_size);
+ 3);
} else {
cont_gen = new VarLenGenerator(
context->max_size, context->min_stride_size, context->max_stride_size);
@@ -748,7 +751,8 @@ public:
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(tid));
librados::AioCompletion *completion =
- context->rados.aio_create_completion((void*) cb_arg, &write_callback, NULL);
+ context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
waiting.insert(completion);
librados::ObjectWriteOperation op;
if (do_append) {
@@ -768,7 +772,7 @@ public:
this,
new TestOp::CallbackInfo(++tid));
librados::AioCompletion *completion = context->rados.aio_create_completion(
- (void*) cb_arg, &write_callback, NULL);
+ (void*) cb_arg, NULL, &write_callback);
waiting.insert(completion);
waiting_on++;
write_op.setxattr("_header", contbl);
@@ -783,7 +787,7 @@ public:
this,
new TestOp::CallbackInfo(++tid));
rcompletion = context->rados.aio_create_completion(
- (void*) cb_arg, &write_callback, NULL);
+ (void*) cb_arg, NULL, &write_callback);
waiting_on++;
read_op.read(0, 1, &rbuffer, 0);
context->io_ctx.aio_operate(
@@ -1027,10 +1031,12 @@ public:
map<string, bufferlist>::iterator iter = xattrs.find("_header");
bufferlist headerbl;
if (iter == xattrs.end()) {
- cerr << num << ": Error: did not find header attr, has_contents: "
- << old_value.has_contents()
- << std::endl;
- assert(!old_value.has_contents());
+ if (old_value.has_contents()) {
+ cerr << num << ": Error: did not find header attr, has_contents: "
+ << old_value.has_contents()
+ << std::endl;
+ assert(!old_value.has_contents());
+ }
} else {
headerbl = iter->second;
xattrs.erase(iter);
@@ -1340,8 +1346,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- comp = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
+ comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
}
@@ -1425,16 +1431,15 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- comp = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
+ comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
// queue up a racing read, too.
pair<TestOp*, TestOp::CallbackInfo*> *read_cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(1));
- comp_racing_read = context->rados.aio_create_completion((void*) read_cb_arg, &write_callback,
- NULL);
+ comp_racing_read = context->rados.aio_create_completion((void*) read_cb_arg, NULL, &write_callback);
rd_op.stat(NULL, NULL, NULL);
context->io_ctx.aio_operate(context->prefix+oid, comp_racing_read, &rd_op,
librados::OPERATION_ORDER_READS_WRITES, // order wrt previous write/update
@@ -1524,8 +1529,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- comp1 = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
+ comp1 = context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
int r = context->io_ctx.hit_set_list(hash, comp1, &ls);
assert(r == 0);
}
@@ -1545,8 +1550,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- comp2 = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
+ comp2 = context->rados.aio_create_completion((void*) cb_arg, NULL,
+ &write_callback);
r = context->io_ctx.hit_set_get(hash, comp2, p->second, &bl);
assert(r == 0);
}
@@ -1599,8 +1604,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- completion = context->rados.aio_create_completion((void *) cb_arg,
- &write_callback, 0);
+ completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+ &write_callback);
context->oid_in_use.insert(oid);
context->oid_not_in_use.erase(oid);
@@ -1673,8 +1678,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- completion = context->rados.aio_create_completion((void *) cb_arg,
- &write_callback, 0);
+ completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+ &write_callback);
context->oid_in_use.insert(oid);
context->oid_not_in_use.erase(oid);
@@ -1781,8 +1786,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- completion = context->rados.aio_create_completion((void *) cb_arg,
- &write_callback, 0);
+ completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+ &write_callback);
// leave object in unused list so that we race with other operations
//context->oid_in_use.insert(oid);
//context->oid_not_in_use.erase(oid);
@@ -1880,8 +1885,8 @@ public:
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(0));
- completion = context->rados.aio_create_completion((void *) cb_arg,
- &write_callback, 0);
+ completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+ &write_callback);
// leave object in unused list so that we race with other operations
//context->oid_in_use.insert(oid);
//context->oid_not_in_use.erase(oid);
diff --git a/src/test/osd/TestECBackend.cc b/src/test/osd/TestECBackend.cc
new file mode 100644
index 0000000..affff36
--- /dev/null
+++ b/src/test/osd/TestECBackend.cc
@@ -0,0 +1,60 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <iostream>
+#include <sstream>
+#include <errno.h>
+#include <signal.h>
+#include "osd/ECBackend.h"
+#include "gtest/gtest.h"
+
+TEST(ECUtil, stripe_info_t)
+{
+ const uint64_t swidth = 4096;
+ const uint64_t ssize = 4;
+
+ ECUtil::stripe_info_t s(ssize, swidth);
+ ASSERT_EQ(s.get_stripe_width(), swidth);
+
+ ASSERT_EQ(s.logical_to_next_chunk_offset(0), 0u);
+ ASSERT_EQ(s.logical_to_next_chunk_offset(1), s.get_chunk_size());
+ ASSERT_EQ(s.logical_to_next_chunk_offset(swidth - 1),
+ s.get_chunk_size());
+
+ ASSERT_EQ(s.logical_to_prev_chunk_offset(0), 0u);
+ ASSERT_EQ(s.logical_to_prev_chunk_offset(swidth), s.get_chunk_size());
+ ASSERT_EQ(s.logical_to_prev_chunk_offset((swidth * 2) - 1),
+ s.get_chunk_size());
+
+ ASSERT_EQ(s.logical_to_next_stripe_offset(0), 0u);
+ ASSERT_EQ(s.logical_to_next_stripe_offset(swidth - 1),
+ s.get_stripe_width());
+
+ ASSERT_EQ(s.logical_to_prev_stripe_offset(swidth), s.get_stripe_width());
+ ASSERT_EQ(s.logical_to_prev_stripe_offset(swidth), s.get_stripe_width());
+ ASSERT_EQ(s.logical_to_prev_stripe_offset((swidth * 2) - 1),
+ s.get_stripe_width());
+
+ ASSERT_EQ(s.aligned_logical_offset_to_chunk_offset(2*swidth),
+ 2*s.get_chunk_size());
+ ASSERT_EQ(s.aligned_chunk_offset_to_logical_offset(2*s.get_chunk_size()),
+ 2*s.get_stripe_width());
+
+ ASSERT_EQ(s.aligned_offset_len_to_chunk(make_pair(swidth, 10*swidth)),
+ make_pair(s.get_chunk_size(), 10*s.get_chunk_size()));
+
+ ASSERT_EQ(s.offset_len_to_stripe_bounds(make_pair(swidth-10, (uint64_t)20)),
+ make_pair((uint64_t)0, 2*swidth));
+}
+
diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc
index ee5e9e3..0ff12c8 100644
--- a/src/test/osd/TestOSDMap.cc
+++ b/src/test/osd/TestOSDMap.cc
@@ -49,8 +49,35 @@ public:
pending_inc.new_uuid[i] = sample_uuid;
}
osdmap.apply_incremental(pending_inc);
+
+ // kludge to get an erasure coding rule and pool
+ int r = osdmap.crush->add_simple_ruleset("erasure", "default", "osd",
+ "indep", pg_pool_t::TYPE_ERASURE,
+ &cerr);
+ pg_pool_t *p = (pg_pool_t *)osdmap.get_pg_pool(2);
+ p->type = pg_pool_t::TYPE_ERASURE;
+ p->crush_ruleset = r;
}
unsigned int get_num_osds() { return num_osds; }
+
+ void test_mappings(int pool,
+ int num,
+ vector<int> *any,
+ vector<int> *first,
+ vector<int> *primary) {
+ for (int i=0; i<num; ++i) {
+ vector<int> o;
+ int p;
+ pg_t pgid(i, pool);
+ osdmap.pg_to_acting_osds(pgid, &o, &p);
+ for (unsigned j=0; j<o.size(); ++j)
+ (*any)[o[j]]++;
+ if (!o.empty())
+ (*first)[o[0]]++;
+ if (p >= 0)
+ (*primary)[p]++;
+ }
+ }
};
TEST_F(OSDMapTest, Create) {
@@ -237,3 +264,85 @@ TEST_F(OSDMapTest, KeepsNecessaryTemps) {
EXPECT_FALSE(pending_inc.new_pg_temp.count(pgid));
EXPECT_FALSE(pending_inc.new_primary_temp.count(pgid));
}
+
+TEST_F(OSDMapTest, PrimaryAffinity) {
+ set_up_map();
+
+ /*
+ osdmap.print(cout);
+ Formatter *f = new_formatter("json-pretty");
+ f->open_object_section("CRUSH");
+ osdmap.crush->dump(f);
+ f->close_section();
+ f->flush(cout);
+ delete f;
+ */
+
+ int n = get_num_osds();
+ for (map<int64_t,pg_pool_t>::const_iterator p = osdmap.get_pools().begin();
+ p != osdmap.get_pools().end();
+ ++p) {
+ int pool = p->first;
+ cout << "pool " << pool << std::endl;
+ {
+ vector<int> any(n, 0);
+ vector<int> first(n, 0);
+ vector<int> primary(n, 0);
+ test_mappings(0, 10000, &any, &first, &primary);
+ for (int i=0; i<n; ++i) {
+ //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl;
+ ASSERT_LT(0, any[i]);
+ ASSERT_LT(0, first[i]);
+ ASSERT_LT(0, primary[i]);
+ }
+ }
+
+ osdmap.set_primary_affinity(0, 0);
+ osdmap.set_primary_affinity(1, 0);
+ {
+ vector<int> any(n, 0);
+ vector<int> first(n, 0);
+ vector<int> primary(n, 0);
+ test_mappings(pool, 10000, &any, &first, &primary);
+ for (int i=0; i<n; ++i) {
+ //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl;
+ ASSERT_LT(0, any[i]);
+ if (i >= 2) {
+ ASSERT_LT(0, first[i]);
+ ASSERT_LT(0, primary[i]);
+ } else {
+ if (p->second.is_replicated())
+ ASSERT_EQ(0, first[i]);
+ ASSERT_EQ(0, primary[i]);
+ }
+ }
+ }
+
+ osdmap.set_primary_affinity(0, 0x8000);
+ osdmap.set_primary_affinity(1, 0);
+ {
+ vector<int> any(n, 0);
+ vector<int> first(n, 0);
+ vector<int> primary(n, 0);
+ test_mappings(pool, 10000, &any, &first, &primary);
+ for (int i=0; i<n; ++i) {
+ //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl;
+ ASSERT_LT(0, any[i]);
+ if (i >= 2) {
+ ASSERT_LT(0, first[i]);
+ ASSERT_LT(0, primary[i]);
+ } else if (i == 1) {
+ if (p->second.is_replicated())
+ ASSERT_EQ(0, first[i]);
+ ASSERT_EQ(0, primary[i]);
+ } else {
+ ASSERT_LT(10000/6/4, primary[0]);
+ ASSERT_GT(10000/6/4*3, primary[0]);
+ }
+ }
+ }
+
+ osdmap.set_primary_affinity(0, 0x10000);
+ osdmap.set_primary_affinity(1, 0x10000);
+ }
+}
diff --git a/src/test/osd/TestPGLog.cc b/src/test/osd/TestPGLog.cc
index 39af882..c2063b8 100644
--- a/src/test/osd/TestPGLog.cc
+++ b/src/test/osd/TestPGLog.cc
@@ -33,6 +33,223 @@ public:
virtual void TearDown() {
clear();
}
+
+ static hobject_t mk_obj(unsigned id) {
+ hobject_t hoid;
+ stringstream ss;
+ ss << "obj_" << id;
+ hoid.oid = ss.str();
+ hoid.hash = id;
+ return hoid;
+ }
+ static eversion_t mk_evt(unsigned ep, unsigned v) {
+ return eversion_t(ep, v);
+ }
+ static pg_log_entry_t mk_ple_mod(
+ const hobject_t &hoid, eversion_t v, eversion_t pv) {
+ pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
+ e.op = pg_log_entry_t::MODIFY;
+ e.soid = hoid;
+ e.version = v;
+ e.prior_version = pv;
+ return e;
+ }
+ static pg_log_entry_t mk_ple_dt(
+ const hobject_t &hoid, eversion_t v, eversion_t pv) {
+ pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
+ e.op = pg_log_entry_t::DELETE;
+ e.soid = hoid;
+ e.version = v;
+ e.prior_version = pv;
+ return e;
+ }
+ static pg_log_entry_t mk_ple_mod_rb(
+ const hobject_t &hoid, eversion_t v, eversion_t pv) {
+ pg_log_entry_t e;
+ e.op = pg_log_entry_t::MODIFY;
+ e.soid = hoid;
+ e.version = v;
+ e.prior_version = pv;
+ return e;
+ }
+ static pg_log_entry_t mk_ple_dt_rb(
+ const hobject_t &hoid, eversion_t v, eversion_t pv) {
+ pg_log_entry_t e;
+ e.op = pg_log_entry_t::DELETE;
+ e.soid = hoid;
+ e.version = v;
+ e.prior_version = pv;
+ return e;
+ }
+
+ struct TestCase {
+ list<pg_log_entry_t> base;
+ list<pg_log_entry_t> auth;
+ list<pg_log_entry_t> div;
+
+ pg_missing_t init;
+ pg_missing_t final;
+
+ set<hobject_t> toremove;
+ list<pg_log_entry_t> torollback;
+
+ private:
+ IndexedLog fullauth;
+ IndexedLog fulldiv;
+ pg_info_t authinfo;
+ pg_info_t divinfo;
+ public:
+ void setup() {
+ fullauth.log.insert(fullauth.log.end(), base.begin(), base.end());
+ fullauth.log.insert(fullauth.log.end(), auth.begin(), auth.end());
+ fulldiv.log.insert(fulldiv.log.end(), base.begin(), base.end());
+ fulldiv.log.insert(fulldiv.log.end(), div.begin(), div.end());
+
+ fullauth.head = authinfo.last_update = fullauth.log.rbegin()->version;
+ authinfo.last_complete = fullauth.log.rbegin()->version;
+ authinfo.log_tail = fullauth.log.begin()->version;
+ authinfo.log_tail.version--;
+ fullauth.tail = authinfo.log_tail;
+ authinfo.last_backfill = hobject_t::get_max();
+
+ fulldiv.head = divinfo.last_update = fulldiv.log.rbegin()->version;
+ divinfo.last_complete = eversion_t();
+ divinfo.log_tail = fulldiv.log.begin()->version;
+ divinfo.log_tail.version--;
+ fulldiv.tail = divinfo.log_tail;
+ divinfo.last_backfill = hobject_t::get_max();
+
+ if (init.missing.empty()) {
+ divinfo.last_complete = divinfo.last_update;
+ } else {
+ eversion_t fmissing = init.missing[init.rmissing.begin()->second].need;
+ for (list<pg_log_entry_t>::const_iterator i = fulldiv.log.begin();
+ i != fulldiv.log.end();
+ ++i) {
+ if (i->version < fmissing)
+ divinfo.last_complete = i->version;
+ else
+ break;
+ }
+ }
+
+ fullauth.index();
+ fulldiv.index();
+ }
+ const IndexedLog &get_fullauth() const { return fullauth; }
+ const IndexedLog &get_fulldiv() const { return fulldiv; }
+ const pg_info_t &get_authinfo() const { return authinfo; }
+ const pg_info_t &get_divinfo() const { return divinfo; }
+ };
+
+ struct LogHandler : public PGLog::LogEntryHandler {
+ set<hobject_t> removed;
+ list<pg_log_entry_t> rolledback;
+
+ void rollback(
+ const pg_log_entry_t &entry) {
+ rolledback.push_back(entry);
+ }
+ void remove(
+ const hobject_t &hoid) {
+ removed.insert(hoid);
+ }
+ void trim(
+ const pg_log_entry_t &entry) {}
+ };
+
+ void verify_missing(
+ const TestCase &tcase,
+ const pg_missing_t &missing) {
+ ASSERT_EQ(tcase.final.missing.size(), missing.missing.size());
+ for (map<hobject_t, pg_missing_t::item>::const_iterator i =
+ missing.missing.begin();
+ i != missing.missing.end();
+ ++i) {
+ EXPECT_TRUE(tcase.final.missing.count(i->first));
+ EXPECT_EQ(tcase.final.missing.find(i->first)->second.need, i->second.need);
+ EXPECT_EQ(tcase.final.missing.find(i->first)->second.have, i->second.have);
+ }
+ }
+
+ void verify_sideeffects(
+ const TestCase &tcase,
+ const LogHandler &handler) {
+ ASSERT_EQ(tcase.toremove.size(), handler.removed.size());
+ ASSERT_EQ(tcase.torollback.size(), handler.rolledback.size());
+
+ {
+ list<pg_log_entry_t>::const_iterator titer = tcase.torollback.begin();
+ list<pg_log_entry_t>::const_iterator hiter = handler.rolledback.begin();
+ for (; titer != tcase.torollback.end(); ++titer, ++hiter) {
+ EXPECT_EQ(titer->version, hiter->version);
+ }
+ }
+
+ {
+ set<hobject_t>::const_iterator titer = tcase.toremove.begin();
+ set<hobject_t>::const_iterator hiter = handler.removed.begin();
+ for (; titer != tcase.toremove.end(); ++titer, ++hiter) {
+ EXPECT_EQ(*titer, *hiter);
+ }
+ }
+ }
+
+ void test_merge_log(const TestCase &tcase) {
+ clear();
+ ObjectStore::Transaction t;
+ log = tcase.get_fulldiv();
+ pg_info_t info = tcase.get_divinfo();
+
+ missing = tcase.init;
+
+ IndexedLog olog;
+ olog = tcase.get_fullauth();
+ pg_info_t oinfo = tcase.get_authinfo();
+
+ LogHandler h;
+ bool dirty_info = false;
+ bool dirty_big_info = false;
+ merge_log(
+ t, oinfo, olog, pg_shard_t(1, 0), info,
+ &h, dirty_info, dirty_big_info);
+
+ ASSERT_EQ(info.last_update, oinfo.last_update);
+ verify_missing(tcase, missing);
+ verify_sideeffects(tcase, h);
+ };
+ void test_proc_replica_log(const TestCase &tcase) {
+ clear();
+ ObjectStore::Transaction t;
+ log = tcase.get_fullauth();
+ pg_info_t info = tcase.get_authinfo();
+
+ pg_missing_t omissing = tcase.init;
+
+ IndexedLog olog;
+ olog = tcase.get_fulldiv();
+ pg_info_t oinfo = tcase.get_divinfo();
+
+ proc_replica_log(
+ t, oinfo, olog, omissing, pg_shard_t(1, 0));
+
+ if (!tcase.base.empty()) {
+ ASSERT_EQ(tcase.base.rbegin()->version, oinfo.last_update);
+ }
+
+ for (list<pg_log_entry_t>::const_iterator i = tcase.auth.begin();
+ i != tcase.auth.end();
+ ++i) {
+ omissing.add_next_event(*i);
+ }
+ verify_missing(tcase, omissing);
+ }
+ void run_test_case(const TestCase &tcase) {
+ test_merge_log(tcase);
+ test_proc_replica_log(tcase);
+ }
};
struct TestHandler : public PGLog::LogEntryHandler {
@@ -115,6 +332,7 @@ TEST_F(PGLogTest, rewind_divergent_log) {
e.op = pg_log_entry_t::MODIFY;
log.log.push_back(e);
e.version = divergent_version = eversion_t(1, 5);
+ e.prior_version = eversion_t(1, 4);
e.soid = divergent;
divergent_object = e.soid;
e.op = pg_log_entry_t::DELETE;
@@ -241,79 +459,15 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(log.empty());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
- EXPECT_TRUE(log.empty());
- }
-
- // a clone with no non-divergent log entry is deleted
- {
- clear();
-
- ObjectStore::Transaction t;
- pg_log_entry_t oe;
- oe.mod_desc.mark_unrollbackable();
- pg_info_t info;
- list<hobject_t> remove_snap;
-
- oe.op = pg_log_entry_t::CLONE;
-
- oe.soid.snap = CEPH_NOSNAP;
- TestHandler h(remove_snap);
- EXPECT_THROW(merge_old_entry(t, oe, info, &h), FailedAssertion);
- oe.soid.snap = 1U;
- missing.add(oe.soid, eversion_t(), eversion_t());
+ merge_old_entry(t, oe, info, &h);
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
EXPECT_TRUE(t.empty());
- EXPECT_TRUE(missing.have_missing());
- EXPECT_TRUE(missing.is_missing(oe.soid));
- EXPECT_TRUE(log.empty());
-
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
-
- EXPECT_FALSE(is_dirty());
- EXPECT_EQ(oe.soid, remove_snap.front());
- EXPECT_TRUE(t.empty());
EXPECT_FALSE(missing.have_missing());
EXPECT_TRUE(log.empty());
}
- // the new entry (from the logs) old entry (from the log entry
- // given in argument) have the same version : do nothing and return true.
- {
- clear();
-
- ObjectStore::Transaction t;
- pg_log_entry_t oe;
- oe.mod_desc.mark_unrollbackable();
- pg_info_t info;
- list<hobject_t> remove_snap;
-
- oe.version = eversion_t(1,1);
- log.add(oe);
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
- EXPECT_EQ(1U, log.log.size());
-
- TestHandler h(remove_snap);
- EXPECT_TRUE(merge_old_entry(t, oe, info, &h));
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
- EXPECT_EQ(1U, log.log.size());
- }
-
// the new entry (from the logs) has a version that is higher than
// the old entry (from the log entry given in argument) : do
// nothing and return false
@@ -344,7 +498,7 @@ TEST_F(PGLogTest, merge_old_entry) {
oe.version = eversion_t(1,1);
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
}
// if the newer entry is not DELETE, the object must be in missing
@@ -357,13 +511,13 @@ TEST_F(PGLogTest, merge_old_entry) {
oe.version = eversion_t(1,1);
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
missing.rm(ne.soid, ne.version);
}
EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
+ EXPECT_FALSE(remove_snap.empty());
EXPECT_TRUE(t.empty());
EXPECT_FALSE(missing.have_missing());
EXPECT_EQ(1U, log.log.size());
@@ -399,7 +553,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_EQ(1U, log.log.size());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -410,69 +564,6 @@ TEST_F(PGLogTest, merge_old_entry) {
// the new entry (from the logs) has a version that is lower than
// the old entry (from the log entry given in argument) and
- // new is update :
- // if the object is not already in missing, add it
- // if the object is already in missing, revise the version it needs
- // return false
- {
- __s32 ops[2] = { pg_log_entry_t::MODIFY, pg_log_entry_t::DELETE };
- for (int i = 0; i < 2; i++) {
- __s32 oe_op = ops[i];
-
- clear();
-
- ObjectStore::Transaction t;
- pg_log_entry_t oe;
- oe.mod_desc.mark_unrollbackable();
- pg_info_t info;
- list<hobject_t> remove_snap;
-
- pg_log_entry_t ne;
- ne.mod_desc.mark_unrollbackable();
- ne.version = eversion_t(1,1);
- ne.op = pg_log_entry_t::MODIFY;
- log.add(ne);
-
- oe.version = eversion_t(2,1);
- oe.op = oe_op;
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
- EXPECT_EQ(1U, log.log.size());
-
- eversion_t old_version(0, 0);
- // if the object is not already in missing, add it
- {
- TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
-
- EXPECT_TRUE(missing.is_missing(ne.soid, ne.version));
- EXPECT_FALSE(missing.is_missing(ne.soid, old_version));
- }
- // if the object is already in missing, revise the version it needs
- {
- missing.revise_need(ne.soid, old_version);
- EXPECT_TRUE(missing.is_missing(ne.soid, old_version));
-
- TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
-
- EXPECT_TRUE(missing.is_missing(ne.soid, ne.version));
- EXPECT_FALSE(missing.is_missing(ne.soid, old_version));
- }
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_TRUE(missing.is_missing(ne.soid));
- EXPECT_EQ(1U, log.log.size());
- }
- }
-
- // the new entry (from the logs) has a version that is lower than
- // the old entry (from the log entry given in argument) and
// old is update and new is DELETE :
// if the object is in missing, it is removed
{
@@ -501,7 +592,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_EQ(1U, log.log.size());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.size() > 0);
@@ -534,7 +625,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(log.empty());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -571,7 +662,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(log.empty());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
EXPECT_TRUE(is_dirty());
EXPECT_EQ(oe.soid, remove_snap.front());
@@ -609,7 +700,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(log.empty());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
EXPECT_TRUE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -619,43 +710,6 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_EQ(oe.soid, divergent_priors[oe.prior_version]);
}
- // there is no new entry (from the logs) and
- // the old entry (from the log entry given in argument) is not a CLONE and
- // the old entry (from the log entry given in argument) is a DELETE and
- // the old entry prior_version is eversion_t() :
- // remove the prior_version of the object from missing, if any and
- // return false
- {
- clear();
-
- ObjectStore::Transaction t;
- pg_log_entry_t oe;
- oe.mod_desc.mark_unrollbackable();
- pg_info_t info;
- list<hobject_t> remove_snap;
-
- info.log_tail = eversion_t(10,1);
- oe.soid.hash = 1;
- oe.op = pg_log_entry_t::DELETE;
- oe.prior_version = eversion_t();
-
- missing.add(oe.soid, eversion_t(1,1), eversion_t());
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_TRUE(missing.is_missing(oe.soid));
- EXPECT_TRUE(log.empty());
-
- TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
-
- EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
- EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
- EXPECT_TRUE(log.empty());
- }
// there is no new entry (from the logs) and
// the old entry (from the log entry given in argument) is not a CLONE and
@@ -687,7 +741,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(log.empty());
TestHandler h(remove_snap);
- EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
+ merge_old_entry(t, oe, info, &h);
EXPECT_FALSE(is_dirty());
EXPECT_EQ(oe.soid, remove_snap.front());
@@ -707,7 +761,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -755,7 +809,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -842,7 +896,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -937,7 +991,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -1052,7 +1106,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -1125,7 +1179,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -1166,7 +1220,7 @@ TEST_F(PGLogTest, merge_log) {
ObjectStore::Transaction t;
pg_log_t olog;
pg_info_t oinfo;
- int fromosd = -1;
+ pg_shard_t fromosd;
pg_info_t info;
list<hobject_t> remove_snap;
bool dirty_info = false;
@@ -1213,7 +1267,7 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_log_t olog;
pg_info_t oinfo;
pg_missing_t omissing;
- int from = -1;
+ pg_shard_t from;
eversion_t last_update(1, 1);
oinfo.last_update = last_update;
@@ -1265,7 +1319,7 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_log_t olog;
pg_info_t oinfo;
pg_missing_t omissing;
- int from = -1;
+ pg_shard_t from;
{
pg_log_entry_t e;
@@ -1305,8 +1359,6 @@ TEST_F(PGLogTest, proc_replica_log) {
EXPECT_TRUE(t.empty());
EXPECT_FALSE(omissing.have_missing());
- EXPECT_EQ(olog.head, oinfo.last_update);
- EXPECT_EQ(olog.head, oinfo.last_complete);
}
{
@@ -1316,7 +1368,7 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_log_t olog;
pg_info_t oinfo;
pg_missing_t omissing;
- int from = -1;
+ pg_shard_t from;
hobject_t divergent_object;
@@ -1332,6 +1384,7 @@ TEST_F(PGLogTest, proc_replica_log) {
log.log.push_back(e);
e.soid = divergent_object;
+ e.prior_version = eversion_t(1, 1);
e.version = eversion_t(1, 2);
log.tail = e.version;
log.log.push_back(e);
@@ -1368,9 +1421,11 @@ TEST_F(PGLogTest, proc_replica_log) {
olog.log.push_back(e);
e.soid = divergent_object;
+ e.prior_version = eversion_t(1, 1);
e.version = eversion_t(1, 2);
olog.log.push_back(e);
+ e.prior_version = eversion_t(0, 0);
e.soid.hash = 0x3;
e.version = eversion_t(1, 4);
olog.log.push_back(e);
@@ -1383,12 +1438,12 @@ TEST_F(PGLogTest, proc_replica_log) {
e.version = eversion_t(1, 6);
olog.log.push_back(e);
- e.soid.hash = 0x9; // should be ignored, matches above
- e.op = pg_log_entry_t::DELETE;
+ e.soid.hash = 0x9; // should not be added to missing, create
+ e.op = pg_log_entry_t::MODIFY;
e.version = eversion_t(1, 7);
olog.log.push_back(e);
- e.soid = divergent_object; // should be marked missing at version 1'2
+ e.soid = divergent_object; // should be added to missing at 1,2
e.op = pg_log_entry_t::MODIFY;
e.version = eversion_t(1, 8);
e.prior_version = eversion_t(1, 2);
@@ -1445,7 +1500,7 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_log_t olog;
pg_info_t oinfo;
pg_missing_t omissing;
- int from = -1;
+ pg_shard_t from;
eversion_t last_update(1, 2);
@@ -1528,7 +1583,7 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_log_t olog;
pg_info_t oinfo;
pg_missing_t omissing;
- int from = -1;
+ pg_shard_t from;
eversion_t last_update(1, 2);
hobject_t divergent_object;
@@ -1591,23 +1646,24 @@ TEST_F(PGLogTest, proc_replica_log) {
| |object | |
|version | hash | version |
| | | |
- tail > (1,1) | x5 | (1,1) < tail
+ tail > (1,1) | x9 | (1,1) < tail
| | | |
| | | |
| (1,2) | x3 | (1,2) |
| | | |
| | | |
- head > (1,3) | x9 | |
- | MODIFY | | |
- | | | |
- | | x9 | (2,3) < head
+ | | x9 | (1,3) < head
| | | MODIFY |
| | | |
+ head > (2,3) | x9 | |
+ | DELETE | | |
+ | | | |
+--------+-------+---------+
- The log entry (1,3) deletes the object x9 but the olog entry
- (2,3) modifies it : remove it from omissing.
-
+ The log entry (2,3) deletes the object x9 but the olog entry
+ (1,3) modifies it : proc_replica_log should adjust missing to
+ 1,1 for that object until add_next_event in PG::activate processes
+ the delete.
*/
{
clear();
@@ -1616,39 +1672,42 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_log_t olog;
pg_info_t oinfo;
pg_missing_t omissing;
- int from = -1;
+ pg_shard_t from;
eversion_t last_update(1, 2);
hobject_t divergent_object;
- eversion_t new_version(1, 3);
- eversion_t divergent_version(2, 3);
+ eversion_t new_version(2, 3);
+ eversion_t divergent_version(1, 3);
{
pg_log_entry_t e;
e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
- e.soid.hash = 0x5;
+ e.soid.hash = 0x9;
log.tail = e.version;
log.log.push_back(e);
e.version = last_update;
e.soid.hash = 0x3;
log.log.push_back(e);
e.version = new_version;
+ e.prior_version = eversion_t(1, 1);
e.soid.hash = 0x9;
- e.op = pg_log_entry_t::MODIFY;
+ e.op = pg_log_entry_t::DELETE;
log.log.push_back(e);
log.head = e.version;
log.index();
+ e.op = pg_log_entry_t::MODIFY;
e.version = eversion_t(1, 1);
- e.soid.hash = 0x5;
+ e.soid.hash = 0x9;
olog.tail = e.version;
olog.log.push_back(e);
e.version = last_update;
e.soid.hash = 0x3;
olog.log.push_back(e);
e.version = divergent_version;
+ e.prior_version = eversion_t(1, 1);
e.soid.hash = 0x9;
divergent_object = e.soid;
omissing.add(divergent_object, e.version, eversion_t());
@@ -1671,14 +1730,138 @@ TEST_F(PGLogTest, proc_replica_log) {
EXPECT_TRUE(t.empty());
EXPECT_TRUE(omissing.have_missing());
- EXPECT_TRUE(omissing.is_missing(divergent_object));
- EXPECT_EQ(new_version, omissing.missing[divergent_object].need);
+ EXPECT_TRUE(omissing.missing.begin()->second.need == eversion_t(1, 1));
EXPECT_EQ(last_update, oinfo.last_update);
- EXPECT_EQ(last_update, oinfo.last_complete);
+ EXPECT_EQ(eversion_t(0, 0), oinfo.last_complete);
}
}
+TEST_F(PGLogTest, merge_log_1) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+
+ t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0));
+
+ t.toremove.insert(mk_obj(1));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_2) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+ t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+ t.torollback.insert(
+ t.torollback.begin(), t.div.rbegin(), t.div.rend());
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_3) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+ t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+ t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0));
+
+ t.toremove.insert(mk_obj(1));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_4) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+ t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+ t.init.add(mk_obj(1), mk_evt(10, 102), mk_evt(0, 0));
+ t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_5) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+ t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+ t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+ t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(0, 0));
+
+ t.toremove.insert(mk_obj(1));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_6) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+ t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_7) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+ t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80));
+ t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_8) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.auth.push_back(mk_ple_dt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+ t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80));
+
+ t.toremove.insert(mk_obj(1));
+
+ t.setup();
+ run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_prior_version_have) {
+ TestCase t;
+ t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+ t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+
+ t.init.add(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100));
+
+ t.setup();
+ run_test_case(t);
+}
+
int main(int argc, char **argv) {
vector<const char*> args;
argv_to_vec(argc, (const char **)argv, args);
diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc
index 79c367c..02e7735 100644
--- a/src/test/osd/TestRados.cc
+++ b/src/test/osd/TestRados.cc
@@ -281,7 +281,7 @@ int main(int argc, char **argv)
else if (strcmp(argv[i], "--max-stride-size") == 0)
max_stride_size = atoi(argv[++i]);
else if (strcmp(argv[i], "--ec-pool") == 0) {
- if (op_weights.size()) {
+ if (!op_weights.empty()) {
cerr << "--ec-pool must be specified prior to any ops" << std::endl;
exit(1);
}
diff --git a/src/test/osd/osd-test-helpers.sh b/src/test/osd/osd-test-helpers.sh
new file mode 100644
index 0000000..341c171
--- /dev/null
+++ b/src/test/osd/osd-test-helpers.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Copyright (C) 2014 Cloudwatt <libre.licensing at cloudwatt.com>
+#
+# Author: Loic Dachary <loic at dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library Public License for more details.
+#
+
+function run_osd() {
+ local dir=$1
+ shift
+ local id=$1
+ shift
+ local osd_data=$dir/$id
+
+ local ceph_disk_args
+ ceph_disk_args+=" --statedir=$dir"
+ ceph_disk_args+=" --sysconfdir=$dir"
+ ceph_disk_args+=" --prepend-to-path="
+ ceph_disk_args+=" --verbose"
+
+ touch $dir/ceph.conf
+
+ ./ceph-disk $ceph_disk_args \
+ prepare $osd_data || return 1
+
+ local ceph_args="$CEPH_ARGS"
+ ceph_args+=" --osd-journal-size=100"
+ ceph_args+=" --osd-data=$osd_data"
+ ceph_args+=" --chdir="
+ ceph_args+=" --run-dir=$dir"
+ ceph_args+=" --log-file=$dir/osd-\$id.log"
+ ceph_args+=" --pid-file=$dir/osd-\$id.pidfile"
+ ceph_args+=" "
+ ceph_args+="$@"
+ CEPH_ARGS="$ceph_args" ./ceph-disk $ceph_disk_args \
+ activate \
+ --mark-init=none \
+ $osd_data || return 1
+
+ [ "$id" = "$(cat $osd_data/whoami)" ] || return 1
+
+ ./ceph osd crush create-or-move "$id" 1 root=default host=localhost
+}
diff --git a/src/test/test_osd_types.cc b/src/test/osd/types.cc
similarity index 89%
rename from src/test/test_osd_types.cc
rename to src/test/osd/types.cc
index bcc71c3..6380211 100644
--- a/src/test/test_osd_types.cc
+++ b/src/test/osd/types.cc
@@ -1,4 +1,4 @@
-// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
@@ -151,6 +151,8 @@ TEST(pg_interval_t, check_new_interval)
new_acting.push_back(osd_id);
new_acting.push_back(osd_id + 1);
vector<int> old_acting = new_acting;
+ int old_primary = osd_id;
+ int new_primary = osd_id;
vector<int> new_up;
new_up.push_back(osd_id);
vector<int> old_up = new_up;
@@ -166,7 +168,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_FALSE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_FALSE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -192,7 +196,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -215,13 +221,15 @@ TEST(pg_interval_t, check_new_interval)
//
{
vector<int> new_acting;
- int new_primary = osd_id + 1;
- new_acting.push_back(new_primary);
+ int _new_primary = osd_id + 1;
+ new_acting.push_back(_new_primary);
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -232,6 +240,7 @@ TEST(pg_interval_t, check_new_interval)
pool_id,
pgid,
&past_intervals));
+ old_primary = new_primary;
ASSERT_EQ((unsigned int)1, past_intervals.size());
ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
@@ -244,13 +253,15 @@ TEST(pg_interval_t, check_new_interval)
//
{
vector<int> new_up;
- int new_primary = osd_id + 1;
- new_up.push_back(new_primary);
+ int _new_primary = osd_id + 1;
+ new_up.push_back(_new_primary);
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -285,7 +296,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -320,7 +333,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -337,7 +352,7 @@ TEST(pg_interval_t, check_new_interval)
ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
}
-
+
//
// The old acting set was empty : the previous interval could not
// have been rw
@@ -350,7 +365,9 @@ TEST(pg_interval_t, check_new_interval)
ostringstream out;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -368,16 +385,16 @@ TEST(pg_interval_t, check_new_interval)
}
//
- // The old acting set did not have enough osd : it could
+ // The old acting set did not have enough osd : it could
// not have been rw
//
{
vector<int> old_acting;
- old_acting.push_back(osd_id);
+ old_acting.push_back(osd_id);
//
// see http://tracker.ceph.com/issues/5780
- // the size of the old acting set should be compared
+ // the size of the old acting set should be compared
// with the min_size of the old osdmap
//
// The new osdmap is created so that it triggers the
@@ -398,7 +415,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -421,15 +440,17 @@ TEST(pg_interval_t, check_new_interval)
//
{
vector<int> new_acting;
- new_acting.push_back(osd_id + 4);
- new_acting.push_back(osd_id + 5);
-
+ new_acting.push_back(osd_id + 4);
+ new_acting.push_back(osd_id + 5);
+
ostringstream out;
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -446,14 +467,14 @@ TEST(pg_interval_t, check_new_interval)
ASSERT_NE(string::npos, out.str().find("includes interval"));
}
//
- // The acting set changes. The old acting set primary was not up
+ // The acting set changes. The old acting set primary was not up
// during the old interval but last_epoch_clean is in the
// old interval and it may have been rw.
//
{
vector<int> new_acting;
- new_acting.push_back(osd_id + 4);
- new_acting.push_back(osd_id + 5);
+ new_acting.push_back(osd_id + 4);
+ new_acting.push_back(osd_id + 5);
ceph::shared_ptr<OSDMap> lastmap(new OSDMap());
lastmap->set_max_osd(10);
@@ -470,7 +491,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -488,15 +511,15 @@ TEST(pg_interval_t, check_new_interval)
}
//
- // The acting set changes. The old acting set primary was not up
+ // The acting set changes. The old acting set primary was not up
// during the old interval and last_epoch_clean is before the
// old interval : the previous interval could not possibly have
// been rw.
//
{
vector<int> new_acting;
- new_acting.push_back(osd_id + 4);
- new_acting.push_back(osd_id + 5);
+ new_acting.push_back(osd_id + 4);
+ new_acting.push_back(osd_id + 5);
epoch_t last_epoch_clean = epoch - 10;
@@ -515,7 +538,9 @@ TEST(pg_interval_t, check_new_interval)
map<epoch_t, pg_interval_t> past_intervals;
ASSERT_TRUE(past_intervals.empty());
- ASSERT_TRUE(pg_interval_t::check_new_interval(old_acting,
+ ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+ new_primary,
+ old_acting,
new_acting,
old_up,
new_up,
@@ -573,7 +598,7 @@ TEST(pg_t, split)
ASSERT_EQ(0u, s.size());
pgid = pg_t(1, 0, -1);
-
+
s.clear();
b = pgid.is_split(2, 4, &s);
ASSERT_TRUE(b);
@@ -665,8 +690,8 @@ TEST(pg_missing_t, swap)
EXPECT_FALSE(other.have_missing());
other.swap(missing);
- EXPECT_FALSE(missing.have_missing());
- EXPECT_TRUE(other.have_missing());
+ EXPECT_FALSE(missing.have_missing());
+ EXPECT_TRUE(other.have_missing());
}
TEST(pg_missing_t, is_missing)
@@ -737,7 +762,7 @@ TEST(pg_missing_t, add_next_event)
EXPECT_EQ(1U, missing.num_missing());
EXPECT_EQ(1U, missing.rmissing.size());
}
-
+
// new object (CLONE)
{
pg_missing_t missing;
@@ -785,7 +810,7 @@ TEST(pg_missing_t, add_next_event)
EXPECT_EQ(1U, missing.num_missing());
EXPECT_EQ(1U, missing.rmissing.size());
}
-
+
// object with prior version (MODIFY)
{
pg_missing_t missing;
@@ -813,7 +838,7 @@ TEST(pg_missing_t, add_next_event)
EXPECT_FALSE(missing.is_missing(oid));
EXPECT_THROW(missing.add_next_event(e), FailedAssertion);
}
-
+
// adding a DELETE matching an existing event
{
pg_missing_t missing;
@@ -957,10 +982,10 @@ TEST(pg_missing_t, split_into)
hobject_t oid2(object_t("objname"), "key2", 123, hash2, 0, "");
pg_missing_t missing;
missing.add(oid1, eversion_t(), eversion_t());
- missing.add(oid2, eversion_t(), eversion_t());
+ missing.add(oid2, eversion_t(), eversion_t());
pg_t child_pgid;
child_pgid.m_seed = 1;
- pg_missing_t child;
+ pg_missing_t child;
unsigned split_bits = 1;
missing.split_into(child_pgid, split_bits, &child);
EXPECT_TRUE(child.is_missing(oid1));
@@ -978,11 +1003,11 @@ protected:
public:
ObjectContext &obc;
- Thread_read_lock(ObjectContext& _obc) :
+ Thread_read_lock(ObjectContext& _obc) :
obc(_obc)
{
}
-
+
virtual void *entry() {
obc.ondisk_read_lock();
return NULL;
@@ -993,11 +1018,11 @@ protected:
public:
ObjectContext &obc;
- Thread_write_lock(ObjectContext& _obc) :
+ Thread_write_lock(ObjectContext& _obc) :
obc(_obc)
{
}
-
+
virtual void *entry() {
obc.ondisk_write_lock();
return NULL;
@@ -1083,7 +1108,7 @@ TEST_F(ObjectContextTest, read_write_lock)
do {
cout << "Trying (2) with delay " << delay << "us\n";
usleep(delay);
- } while ((obc.readers == 0 || obc.readers_waiting == 1) &&
+ } while ((obc.readers == 0 || obc.readers_waiting == 1) &&
( delay = delay * 2 + 1) < DELAY_MAX);
EXPECT_EQ(0, obc.readers_waiting);
EXPECT_EQ(1, obc.readers);
@@ -1157,15 +1182,55 @@ TEST_F(ObjectContextTest, read_write_lock)
t.join();
}
-
+
+}
+
+TEST(pg_pool_t_test, get_pg_num_divisor) {
+ pg_pool_t p;
+ p.set_pg_num(16);
+ p.set_pgp_num(16);
+
+ for (int i = 0; i < 16; ++i)
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(i, 1)));
+
+ p.set_pg_num(12);
+ p.set_pgp_num(12);
+ //cout << "num " << p.get_pg_num()
+ // << " mask " << p.get_pg_num_mask() << std::endl;
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(0, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(1, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(2, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(3, 1)));
+ ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(4, 1)));
+ ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(5, 1)));
+ ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(6, 1)));
+ ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(7, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(8, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(9, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(10, 1)));
+ ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(11, 1)));
+}
+
+TEST(pg_pool_t_test, get_random_pg_position) {
+ srand(getpid());
+ for (int i = 0; i < 100; ++i) {
+ pg_pool_t p;
+ p.set_pg_num(1 + (rand() % 1000));
+ p.set_pgp_num(p.get_pg_num());
+ pg_t pgid(rand() % p.get_pg_num(), 1);
+ uint32_t h = p.get_random_pg_position(pgid, rand());
+ uint32_t ps = p.raw_hash_to_pg(h);
+ cout << p.get_pg_num() << " " << pgid << ": "
+ << h << " -> " << pg_t(ps, 1) << std::endl;
+ ASSERT_EQ(pgid.ps(), ps);
+ }
}
/*
* Local Variables:
- * compile-command: "cd .. ;
- * make unittest_osd_types ;
- * ./unittest_osd_types # --gtest_filter=pg_missing_t.constructor
+ * compile-command: "cd .. ;
+ * make unittest_osd_types ;
+ * ./unittest_osd_types # --gtest_filter=pg_missing_t.constructor
* "
* End:
*/
-
diff --git a/src/test/rgw/test_rgw_manifest.cc b/src/test/rgw/test_rgw_manifest.cc
new file mode 100644
index 0000000..502eaf9
--- /dev/null
+++ b/src/test/rgw/test_rgw_manifest.cc
@@ -0,0 +1,227 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing at enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include "global/global_init.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#define GTEST
+#ifdef GTEST
+#include <gtest/gtest.h>
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+ else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+ else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c)
+#endif
+using namespace std;
+
+static void init_bucket(rgw_bucket *bucket, const char *name)
+{
+ *bucket = rgw_bucket(name, ".data-pool", ".index-pool", "marker.", "bucket-id", NULL);
+}
+
+void append_head(list<rgw_obj> *objs, rgw_obj& head)
+{
+ objs->push_back(head);
+}
+
+void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size)
+{
+ string prefix = manifest.get_prefix();
+ rgw_bucket bucket = manifest.get_head().bucket;
+
+ int i = 0;
+ for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) {
+ char buf[16];
+ snprintf(buf, sizeof(buf), "%d", ++i);
+ string oid = prefix + buf;
+ cout << "oid=" << oid << std::endl;
+ rgw_obj obj;
+ obj.init_ns(bucket, oid, "shadow");
+ objs->push_back(obj);
+ }
+}
+
+static void gen_obj(uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
+ RGWObjManifest *manifest, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen,
+ list<rgw_obj> *test_objs)
+{
+ manifest->set_trivial_rule(head_max_size, stripe_size);
+
+ init_bucket(bucket, "buck");
+
+ *head = rgw_obj(*bucket, "oid");
+ gen->create_begin(g_ceph_context, manifest, *bucket, *head);
+
+ append_head(test_objs, *head);
+ cout << "test_objs.size()=" << test_objs->size() << std::endl;
+ append_stripes(test_objs, *manifest, obj_size, stripe_size);
+
+ cout << "test_objs.size()=" << test_objs->size() << std::endl;
+
+ ASSERT_EQ((int)manifest->get_obj_size(), 0);
+ ASSERT_EQ((int)manifest->get_head_size(), 0);
+ ASSERT_EQ(manifest->has_tail(), false);
+
+ uint64_t ofs = 0;
+ list<rgw_obj>::iterator iter = test_objs->begin();
+
+ while (ofs < obj_size) {
+ rgw_obj obj = gen->get_cur_obj();
+cout << "obj=" << obj << std::endl;
+ ASSERT_TRUE(obj == *iter);
+
+ ofs = MIN(ofs + gen->cur_stripe_max_size(), obj_size);
+ gen->create_next(ofs);
+
+ cout << "obj=" << obj << " *iter=" << *iter << std::endl;
+ cout << "test_objs.size()=" << test_objs->size() << std::endl;
+ ++iter;
+
+ }
+
+ if (manifest->has_tail()) {
+ rgw_obj obj = gen->get_cur_obj();
+ ASSERT_TRUE(obj == *iter);
+ ++iter;
+ }
+ ASSERT_TRUE(iter == test_objs->end());
+ ASSERT_EQ(manifest->get_obj_size(), obj_size);
+ ASSERT_EQ(manifest->get_head_size(), MIN(obj_size, head_max_size));
+ ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size));
+}
+
+TEST(TestRGWManifest, head_only_obj) {
+ RGWObjManifest manifest;
+ rgw_bucket bucket;
+ rgw_obj head;
+ RGWObjManifest::generator gen;
+
+ int obj_size = 256 * 1024;
+
+ list<rgw_obj> objs;
+
+ gen_obj(obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, &bucket, &head, &gen, &objs);
+
+ cout << " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl;
+ cout << " manifest.get_head_size()=" << manifest.get_head_size() << std::endl;
+ list<rgw_obj>::iterator liter;
+
+ RGWObjManifest::obj_iterator iter;
+ for (iter = manifest.obj_begin(), liter = objs.begin();
+ iter != manifest.obj_end() && liter != objs.end();
+ ++iter, ++liter) {
+ ASSERT_TRUE(*liter == iter.get_location());
+ }
+
+ ASSERT_TRUE(iter == manifest.obj_end());
+ ASSERT_TRUE(liter == objs.end());
+
+ iter = manifest.obj_find(100 * 1024);
+ ASSERT_TRUE(iter.get_location() == head);
+ ASSERT_EQ((int)iter.get_stripe_size(), obj_size);
+}
+
+TEST(TestRGWManifest, obj_with_head_and_tail) {
+ RGWObjManifest manifest;
+ rgw_bucket bucket;
+ rgw_obj head;
+ RGWObjManifest::generator gen;
+
+ list<rgw_obj> objs;
+
+ int obj_size = 21 * 1024 * 1024 + 1000;
+ int stripe_size = 4 * 1024 * 1024;
+ int head_size = 512 * 1024;
+
+ gen_obj(obj_size, head_size, stripe_size, &manifest, &bucket, &head, &gen, &objs);
+
+ list<rgw_obj>::iterator liter;
+
+ rgw_obj last_obj;
+
+ RGWObjManifest::obj_iterator iter;
+ for (iter = manifest.obj_begin(), liter = objs.begin();
+ iter != manifest.obj_end() && liter != objs.end();
+ ++iter, ++liter) {
+ cout << "*liter=" << *liter << " iter.get_location()=" << iter.get_location() << std::endl;
+ ASSERT_TRUE(*liter == iter.get_location());
+
+ last_obj = iter.get_location();
+ }
+
+ ASSERT_TRUE(iter == manifest.obj_end());
+ ASSERT_TRUE(liter == objs.end());
+
+ iter = manifest.obj_find(100 * 1024);
+ ASSERT_TRUE(iter.get_location() == head);
+ ASSERT_EQ((int)iter.get_stripe_size(), head_size);
+
+ uint64_t ofs = 20 * 1024 * 1024 + head_size;
+ iter = manifest.obj_find(ofs + 100);
+
+ ASSERT_TRUE(iter.get_location() == last_obj);
+ ASSERT_EQ(iter.get_stripe_ofs(), ofs);
+ ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs);
+}
+
+TEST(TestRGWManifest, multipart) {
+ int num_parts = 16;
+ RGWObjManifest pm[num_parts];
+ rgw_bucket bucket;
+ uint64_t part_size = 10 * 1024 * 1024;
+ uint64_t stripe_size = 4 * 1024 * 1024;
+
+ string upload_id = "abc123";
+
+ for (int i = 0; i < num_parts; ++i) {
+ RGWObjManifest& manifest = pm[i];
+ RGWObjManifest::generator gen;
+ manifest.set_prefix(upload_id);
+
+ manifest.set_multipart_part_rule(stripe_size, i + 1);
+
+ uint64_t ofs;
+ rgw_obj head;
+ for (ofs = 0; ofs < part_size; ofs += stripe_size) {
+ if (ofs == 0) {
+ int r = gen.create_begin(g_ceph_context, &manifest, bucket, head);
+ ASSERT_EQ(r, 0);
+ continue;
+ }
+ gen.create_next(ofs);
+ }
+
+ if (ofs > part_size) {
+ gen.create_next(part_size);
+ }
+ }
+
+ RGWObjManifest m;
+
+ for (int i = 0; i < num_parts; i++) {
+ m.append(pm[i]);
+ }
+ RGWObjManifest::obj_iterator iter;
+ for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) {
+ RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs());
+ ASSERT_TRUE(fiter.get_location() == iter.get_location());
+ }
+
+ ASSERT_EQ(m.get_obj_size(), num_parts * part_size);
+}
+
diff --git a/src/test/system/rados_watch_notify.cc b/src/test/system/rados_watch_notify.cc
index e549d38..6517f33 100644
--- a/src/test/system/rados_watch_notify.cc
+++ b/src/test/system/rados_watch_notify.cc
@@ -70,7 +70,7 @@ int main(int argc, const char **argv)
{
StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
StRadosWatch r2(argc, argv, setup_sem, watch_sem, notify_sem,
- 1, pool, "0.obj");
+ 1, 0, pool, "0.obj");
StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
0, pool, "0.obj");
vector<SysTestRunnable*> vec;
@@ -94,7 +94,7 @@ int main(int argc, const char **argv)
{
StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 0, ".obj");
StRadosWatch r2(argc, argv, setup_sem, watch_sem, notify_sem,
- 0, pool, "0.obj");
+ 0, -ENOENT, pool, "0.obj");
StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
-ENOENT, pool, "0.obj");
vector<SysTestRunnable*> vec;
@@ -127,7 +127,7 @@ int main(int argc, const char **argv)
{
StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
StRadosWatch r2(argc, argv, setup_sem, watch_sem, finished_notifies_sem,
- 1, pool, "0.obj");
+ 1, 0, pool, "0.obj");
StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
0, pool, "0.obj");
StRadosDeletePool r4(argc, argv, notify_sem, deleted_sem, pool);
@@ -165,7 +165,7 @@ int main(int argc, const char **argv)
{
StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
StRadosWatch r2(argc, argv, setup_sem, watch_sem, finished_notifies_sem,
- 1, pool, "0.obj");
+ 1, 0, pool, "0.obj");
StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
0, pool, "0.obj");
StRadosDeleteObjs r4(argc, argv, notify_sem, deleted_sem, 1, pool, ".obj");
diff --git a/src/test/system/st_rados_watch.cc b/src/test/system/st_rados_watch.cc
index 696b086..38b7b5e 100644
--- a/src/test/system/st_rados_watch.cc
+++ b/src/test/system/st_rados_watch.cc
@@ -28,6 +28,7 @@ StRadosWatch::StRadosWatch(int argc, const char **argv,
CrossProcessSem *watch_sem,
CrossProcessSem *notify_sem,
int num_notifies,
+ int watch_retcode,
const std::string &pool_name,
const std::string &obj_name)
: SysTestRunnable(argc, argv),
@@ -35,6 +36,7 @@ StRadosWatch::StRadosWatch(int argc, const char **argv,
m_watch_sem(watch_sem),
m_notify_sem(notify_sem),
m_num_notifies(num_notifies),
+ m_watch_retcode(watch_retcode),
m_pool_name(pool_name),
m_obj_name(obj_name)
{
@@ -65,9 +67,13 @@ run()
RETURN1_IF_NONZERO(rados_connect(cl));
RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
printf("%s: watching object %s\n", get_id_str(), m_obj_name.c_str());
- RETURN1_IF_NONZERO(rados_watch(io_ctx, m_obj_name.c_str(), 0, &handle,
- reinterpret_cast<rados_watchcb_t>(notify_cb),
- reinterpret_cast<void*>(&num_notifies)));
+
+ RETURN1_IF_NOT_VAL(
+ rados_watch(io_ctx, m_obj_name.c_str(), 0, &handle,
+ reinterpret_cast<rados_watchcb_t>(notify_cb),
+ reinterpret_cast<void*>(&num_notifies)),
+ m_watch_retcode
+ );
if (m_watch_sem) {
m_watch_sem->post();
}
diff --git a/src/test/system/st_rados_watch.h b/src/test/system/st_rados_watch.h
index e5ae130..e3e78c0 100644
--- a/src/test/system/st_rados_watch.h
+++ b/src/test/system/st_rados_watch.h
@@ -38,6 +38,7 @@ public:
CrossProcessSem *watch_sem,
CrossProcessSem *notify_sem,
int num_notifies,
+ int watch_retcode,
const std::string &pool_name,
const std::string &obj_name);
~StRadosWatch();
@@ -47,6 +48,7 @@ private:
CrossProcessSem *m_watch_sem;
CrossProcessSem *m_notify_sem;
int m_num_notifies;
+ int m_watch_retcode;
std::string m_pool_name;
std::string m_obj_name;
};
diff --git a/src/test/test_crushwrapper.cc b/src/test/test_crushwrapper.cc
deleted file mode 100644
index eeec8f7..0000000
--- a/src/test/test_crushwrapper.cc
+++ /dev/null
@@ -1,107 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Inktank <info at inktank.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Library Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Library Public License for more details.
- *
- */
-
-#include "crush/CrushWrapper.h"
-#include "gtest/gtest.h"
-#include "global/global_init.h"
-#include "global/global_context.h"
-#include "common/ceph_argparse.h"
-#include "common/Formatter.h"
-
-TEST(CrushWrapper, distance) {
- CrushWrapper c;
- c.create();
- c.set_type_name(1, "host");
- c.set_type_name(2, "rack");
- c.set_type_name(3, "root");
- int bno;
- int r = c.add_bucket(0, CRUSH_BUCKET_STRAW,
- CRUSH_HASH_DEFAULT, 3, 0, NULL,
- NULL, &bno);
- ASSERT_EQ(0, r);
- ASSERT_EQ(-1, bno);
- c.set_item_name(bno, "default");
-
- c.set_max_devices(10);
-
- //JSONFormatter jf(true);
-
- map<string,string> loc;
- loc["host"] = "a1";
- loc["rack"] = "a";
- loc["root"] = "default";
- c.insert_item(g_ceph_context, 0, 1, "osd.0", loc);
-
- loc.clear();
- loc["host"] = "a2";
- loc["rack"] = "a";
- loc["root"] = "default";
- c.insert_item(g_ceph_context, 1, 1, "osd.1", loc);
-
- loc.clear();
- loc["host"] = "b1";
- loc["rack"] = "b";
- loc["root"] = "default";
- c.insert_item(g_ceph_context, 2, 1, "osd.2", loc);
-
- loc.clear();
- loc["host"] = "b2";
- loc["rack"] = "b";
- loc["root"] = "default";
- c.insert_item(g_ceph_context, 3, 1, "osd.3", loc);
-
- vector<pair<string,string> > ol;
- c.get_full_location_ordered(3, ol);
- ASSERT_EQ(3u, ol.size());
- ASSERT_EQ(make_pair(string("host"),string("b2")), ol[0]);
- ASSERT_EQ(make_pair(string("rack"),string("b")), ol[1]);
- ASSERT_EQ(make_pair(string("root"),string("default")), ol[2]);
-
- //c.dump(&jf);
- //jf.flush(cout);
-
- multimap<string,string> p;
- p.insert(make_pair("host","b2"));
- p.insert(make_pair("rack","b"));
- p.insert(make_pair("root","default"));
- ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 0, p));
- ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 1, p));
- ASSERT_EQ(2, c.get_common_ancestor_distance(g_ceph_context, 2, p));
- ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
- ASSERT_EQ(-ENOENT, c.get_common_ancestor_distance(g_ceph_context, 123, p));
-
- // make sure a "multipath" location will reflect a minimal
- // distance for both paths
- p.insert(make_pair("host","b1"));
- ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 2, p));
- ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
-}
-
-
-int main(int argc, char **argv) {
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
- env_to_vec(args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
diff --git a/src/test/test_snap_mapper.cc b/src/test/test_snap_mapper.cc
index eaf308c..04db762 100644
--- a/src/test/test_snap_mapper.cc
+++ b/src/test/test_snap_mapper.cc
@@ -453,7 +453,8 @@ public:
uint32_t mask,
uint32_t bits)
: driver(driver),
- mapper(new SnapMapper(driver, mask, bits, 0)), mask(mask), bits(bits),
+ mapper(new SnapMapper(driver, mask, bits, 0, 1)),
+ mask(mask), bits(bits),
lock("lock") {}
hobject_t random_hobject() {
diff --git a/src/tools/ceph-filestore-dump.cc b/src/tools/ceph-filestore-dump.cc
index b4220ba..6c8b309 100644
--- a/src/tools/ceph-filestore-dump.cc
+++ b/src/tools/ceph-filestore-dump.cc
@@ -374,11 +374,13 @@ int get_log(ObjectStore *fs, coll_t coll, pg_t pgid, const pg_info_t &info,
//Based on RemoveWQ::_process()
void remove_coll(ObjectStore *store, const coll_t &coll)
{
+ spg_t pg;
+ coll.is_pg_prefix(pg);
OSDriver driver(
store,
coll_t(),
OSD::make_snapmapper_oid());
- SnapMapper mapper(&driver, 0, 0, 0);
+ SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
vector<ghobject_t> objects;
ghobject_t next;
@@ -432,7 +434,7 @@ int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
for (vector<coll_t>::iterator it = ls.begin();
it != ls.end();
++it) {
- pg_t pgid;
+ spg_t pgid;
snapid_t snap;
if (it->is_temp(pgid)) {
@@ -465,12 +467,13 @@ int initiate_new_remove_pg(ObjectStore *store, pg_t r_pgid,
{
ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
- if (store->collection_exists(coll_t(r_pgid))) {
+ if (store->collection_exists(coll_t(spg_t(r_pgid, ghobject_t::no_shard())))) {
coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++,
- r_pgid);
- cout << "collection rename " << coll_t(r_pgid) << " to " << to_remove
+ spg_t(r_pgid, ghobject_t::no_shard()));
+ cout << "collection rename " << coll_t(spg_t(r_pgid, ghobject_t::no_shard()))
+ << " to " << to_remove
<< std::endl;
- rmt->collection_rename(coll_t(r_pgid), to_remove);
+ rmt->collection_rename(coll_t(spg_t(r_pgid, ghobject_t::no_shard())), to_remove);
} else {
delete rmt;
return ENOENT;
@@ -856,7 +859,9 @@ int get_object(ObjectStore *store, coll_t coll, bufferlist &bl)
store,
coll_t(),
OSD::make_snapmapper_oid());
- SnapMapper mapper(&driver, 0, 0, 0);
+ spg_t pg;
+ coll.is_pg_prefix(pg);
+ SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
t->touch(coll, ob.hoid);
@@ -988,11 +993,11 @@ int do_import(ObjectStore *store, OSDSuperblock sb)
return 1;
}
- log_oid = OSD::make_pg_log_oid(pgid);
- biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
+ log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
+ biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
//Check for PG already present.
- coll_t coll(pgid);
+ coll_t coll(spg_t(pgid, ghobject_t::no_shard()));
if (store->collection_exists(coll)) {
cout << "pgid " << pgid << " already exists" << std::endl;
return 1;
@@ -1000,7 +1005,8 @@ int do_import(ObjectStore *store, OSDSuperblock sb)
//Switch to collection which will be removed automatically if
//this program is interupted.
- coll_t rmcoll = coll_t::make_removal_coll(next_removal_seq, pgid);
+ coll_t rmcoll = coll_t::make_removal_coll(
+ next_removal_seq, spg_t(pgid, ghobject_t::no_shard()));
ObjectStore::Transaction *t = new ObjectStore::Transaction;
t->create_collection(rmcoll);
store->apply_transaction(*t);
@@ -1286,8 +1292,8 @@ int main(int argc, char **argv)
goto out;
}
- log_oid = OSD::make_pg_log_oid(pgid);
- biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
+ log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
+ biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
if (type == "remove") {
uint64_t next_removal_seq = 0; //My local seq
@@ -1311,13 +1317,13 @@ int main(int argc, char **argv)
for (it = ls.begin(); it != ls.end(); ++it) {
snapid_t snap;
- pg_t tmppgid;
+ spg_t tmppgid;
if (!it->is_pg(tmppgid, snap)) {
continue;
}
- if (tmppgid != pgid) {
+ if (tmppgid.pgid != pgid) {
continue;
}
if (snap != CEPH_NOSNAP && debug) {
@@ -1340,9 +1346,10 @@ int main(int argc, char **argv)
if (debug)
cerr << "map_epoch " << map_epoch << std::endl;
- pg_info_t info(pgid);
+ pg_info_t info(spg_t(pgid, ghobject_t::no_shard()));
map<epoch_t,pg_interval_t> past_intervals;
- hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
+ hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(
+ spg_t(pgid, ghobject_t::no_shard()));
interval_set<snapid_t> snap_collections;
__u8 struct_ver;
diff --git a/src/tools/ceph-filestore-tool.cc b/src/tools/ceph-filestore-tool.cc
index e9b1351..eb9f8da 100644
--- a/src/tools/ceph-filestore-tool.cc
+++ b/src/tools/ceph-filestore-tool.cc
@@ -162,7 +162,7 @@ int main(int argc, char **argv)
vector<coll_t> colls_to_check;
if (pgidstr.length()) {
- pg_t pgid;
+ spg_t pgid;
if (!pgid.parse(pgidstr.c_str())) {
cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
exit(1);
@@ -178,7 +178,7 @@ int main(int argc, char **argv)
for (vector<coll_t>::iterator i = candidates.begin();
i != candidates.end();
++i) {
- pg_t pgid;
+ spg_t pgid;
snapid_t snap;
if (i->is_pg(pgid, snap)) {
colls_to_check.push_back(*i);
diff --git a/src/tools/crushtool.cc b/src/tools/crushtool.cc
index 7709a2a..0c6d7ba 100644
--- a/src/tools/crushtool.cc
+++ b/src/tools/crushtool.cc
@@ -129,6 +129,8 @@ void usage()
cout << " set choose total descent attempts\n";
cout << " --set-chooseleaf-descend-once <0|1>\n";
cout << " set chooseleaf to (not) retry the recursive descent\n";
+ cout << " --set-chooseleaf-vary-r <0|1>\n";
+ cout << " set chooseleaf to (not) vary r based on parent\n";
cout << " --output-name name\n";
cout << " prepend the data file(s) generated during the\n";
cout << " testing routine with name\n";
@@ -187,6 +189,7 @@ int main(int argc, const char **argv)
int choose_local_fallback_tries = -1;
int choose_total_tries = -1;
int chooseleaf_descend_once = -1;
+ int chooseleaf_vary_r = -1;
CrushWrapper crush;
@@ -257,6 +260,9 @@ int main(int argc, const char **argv)
} else if (ceph_argparse_withint(args, i, &chooseleaf_descend_once, &err,
"--set_chooseleaf_descend_once", (char*)NULL)) {
adjust = true;
+ } else if (ceph_argparse_withint(args, i, &chooseleaf_vary_r, &err,
+ "--set_chooseleaf_vary_r", (char*)NULL)) {
+ adjust = true;
} else if (ceph_argparse_flag(args, i, "--reweight", (char*)NULL)) {
reweight = true;
} else if (ceph_argparse_withint(args, i, &add_item, &err, "--add_item", (char*)NULL)) {
@@ -702,6 +708,10 @@ int main(int argc, const char **argv)
crush.set_chooseleaf_descend_once(chooseleaf_descend_once);
modified = true;
}
+ if (chooseleaf_vary_r >= 0) {
+ crush.set_chooseleaf_vary_r(chooseleaf_vary_r);
+ modified = true;
+ }
if (modified) {
crush.finalize();
diff --git a/src/tools/osdmaptool.cc b/src/tools/osdmaptool.cc
index 7148572..0db39da 100644
--- a/src/tools/osdmaptool.cc
+++ b/src/tools/osdmaptool.cc
@@ -34,6 +34,10 @@ void usage()
cout << " usage: [--print] [--createsimple <numosd> [--clobber] [--pg_bits <bitsperosd>]] <mapfilename>" << std::endl;
cout << " --export-crush <file> write osdmap's crush map to <file>" << std::endl;
cout << " --import-crush <file> replace osdmap's crush map with <file>" << std::endl;
+ cout << " --test-map-pgs [--pool <poolid>] map all pgs" << std::endl;
+ cout << " --mark-up-in mark osds up and in (but do not persist)" << std::endl;
+ cout << " --clear-temp clear pg_temp and primary_temp" << std::endl;
+ cout << " --test-random do random placements" << std::endl;
cout << " --test-map-pg <pgid> map a pgid to osds" << std::endl;
cout << " --test-map-object <objectname> [--pool <poolid>] map an object to osds"
<< std::endl;
@@ -67,7 +71,11 @@ int main(int argc, const char **argv)
bool test_crush = false;
int range_first = -1;
int range_last = -1;
- int pool = 0;
+ int pool = -1;
+ bool mark_up_in = false;
+ bool clear_temp = false;
+ bool test_map_pgs = false;
+ bool test_random = false;
std::string val;
std::ostringstream err;
@@ -90,6 +98,14 @@ int main(int argc, const char **argv)
createsimple = true;
} else if (ceph_argparse_flag(args, i, "--create-from-conf", (char*)NULL)) {
create_from_conf = true;
+ } else if (ceph_argparse_flag(args, i, "--mark-up-in", (char*)NULL)) {
+ mark_up_in = true;
+ } else if (ceph_argparse_flag(args, i, "--clear-temp", (char*)NULL)) {
+ clear_temp = true;
+ } else if (ceph_argparse_flag(args, i, "--test-map-pgs", (char*)NULL)) {
+ test_map_pgs = true;
+ } else if (ceph_argparse_flag(args, i, "--test-random", (char*)NULL)) {
+ test_random = true;
} else if (ceph_argparse_flag(args, i, "--clobber", (char*)NULL)) {
clobber = true;
} else if (ceph_argparse_withint(args, i, &pg_bits, &err, "--pg_bits", (char*)NULL)) {
@@ -115,6 +131,10 @@ int main(int argc, const char **argv)
} else if (ceph_argparse_withint(args, i, &range_first, &err, "--range_first", (char*)NULL)) {
} else if (ceph_argparse_withint(args, i, &range_last, &err, "--range_last", (char*)NULL)) {
} else if (ceph_argparse_withint(args, i, &pool, &err, "--pool", (char*)NULL)) {
+ if (!err.str().empty()) {
+ cerr << err.str() << std::endl;
+ exit(EXIT_FAILURE);
+ }
} else {
++i;
}
@@ -197,6 +217,20 @@ int main(int argc, const char **argv)
modified = true;
}
+ if (mark_up_in) {
+ cout << "marking all OSDs up and in" << std::endl;
+ int n = osdmap.get_max_osd();
+ for (int i=0; i<n; i++) {
+ osdmap.set_state(i, osdmap.get_state(i) | CEPH_OSD_UP);
+ osdmap.set_weight(i, CEPH_OSD_IN);
+ osdmap.crush->adjust_item_weightf(g_ceph_context, i, 1.0);
+ }
+ }
+ if (clear_temp) {
+ cout << "clearing pg/primary temp" << std::endl;
+ osdmap.clear_temp();
+ }
+
if (!import_crush.empty()) {
bufferlist cbl;
std::string error;
@@ -241,6 +275,10 @@ int main(int argc, const char **argv)
if (!test_map_object.empty()) {
object_t oid(test_map_object);
+ if (pool == -1) {
+ cout << me << ": assuming pool 0 (use --pool to override)" << std::endl;
+ pool = 0;
+ }
if (!osdmap.have_pg_pool(pool)) {
cerr << "There is no pool " << pool << std::endl;
exit(1);
@@ -275,6 +313,111 @@ int main(int argc, const char **argv)
<< ") acting (" << acting << ", p" << acting_primary << ")"
<< std::endl;
}
+ if (test_map_pgs) {
+ if (pool != -1 && !osdmap.have_pg_pool(pool)) {
+ cerr << "There is no pool " << pool << std::endl;
+ exit(1);
+ }
+ int n = osdmap.get_max_osd();
+ vector<int> count(n, 0);
+ vector<int> first_count(n, 0);
+ vector<int> primary_count(n, 0);
+ vector<int> size(30, 0);
+ if (test_random)
+ srand(getpid());
+ const map<int64_t,pg_pool_t>& pools = osdmap.get_pools();
+ for (map<int64_t,pg_pool_t>::const_iterator p = pools.begin();
+ p != pools.end(); ++p) {
+ if (pool != -1 && p->first != pool)
+ continue;
+ cout << "pool " << p->first
+ << " pg_num " << p->second.get_pg_num() << std::endl;
+ for (unsigned i = 0; i < p->second.get_pg_num(); ++i) {
+ pg_t pgid = pg_t(i, p->first);
+
+ vector<int> osds;
+ int primary;
+ if (test_random) {
+ osds.resize(p->second.size);
+ for (unsigned i=0; i<osds.size(); ++i) {
+ osds[i] = rand() % osdmap.get_max_osd();
+ }
+ primary = osds[0];
+ } else {
+ osdmap.pg_to_acting_osds(pgid, &osds, &primary);
+ }
+ size[osds.size()]++;
+
+ for (unsigned i=0; i<osds.size(); i++) {
+ //cout << " rep " << i << " on " << osds[i] << std::endl;
+ count[osds[i]]++;
+ }
+ if (osds.size())
+ first_count[osds[0]]++;
+ if (primary >= 0)
+ primary_count[primary]++;
+ }
+ }
+
+ uint64_t total = 0;
+ int in = 0;
+ int min_osd = -1;
+ int max_osd = -1;
+ cout << "#osd\tcount\tfirst\tprimary\tc wt\twt\n";
+ for (int i=0; i<n; i++) {
+ if (!osdmap.is_in(i))
+ continue;
+ if (osdmap.crush->get_item_weight(i) <= 0)
+ continue;
+ in++;
+ cout << "osd." << i
+ << "\t" << count[i]
+ << "\t" << first_count[i]
+ << "\t" << primary_count[i]
+ << "\t" << osdmap.crush->get_item_weightf(i)
+ << "\t" << osdmap.get_weightf(i)
+ << std::endl;
+ total += count[i];
+ if (count[i] &&
+ (min_osd < 0 ||
+ count[i] < count[min_osd]))
+ min_osd = i;
+ if (count[i] &&
+ (max_osd < 0 ||
+ count[i] > count[max_osd]))
+ max_osd = i;
+
+ }
+ uint64_t avg = total / in;
+ double dev = 0;
+ for (int i=0; i<n; i++) {
+ if (!osdmap.is_in(i))
+ continue;
+ if (osdmap.crush->get_item_weight(i) <= 0)
+ continue;
+ dev += (avg - count[i]) * (avg - count[i]);
+ }
+ dev /= in;
+ dev = sqrt(dev);
+
+ //double edev = sqrt(pgavg) * (double)avg / pgavg;
+ double edev = sqrt((double)total / (double)in * (1.0 - (1.0 / (double)in)));
+ cout << " in " << in << std::endl;
+ cout << " avg " << avg
+ << " stddev " << dev
+ << " (" << (dev/avg) << "x)"
+ << " (expected " << edev << " " << (edev/avg) << "x))"
+ << std::endl;
+
+ if (min_osd >= 0)
+ cout << " min osd." << min_osd << " " << count[min_osd] << std::endl;
+ if (max_osd >= 0)
+ cout << " max osd." << max_osd << " " << count[max_osd] << std::endl;
+
+ for (int i=0; i<4; i++) {
+ cout << "size " << i << "\t" << size[i] << std::endl;
+ }
+ }
if (test_crush) {
int pass = 0;
while (1) {
@@ -308,7 +451,8 @@ int main(int argc, const char **argv)
if (!print && !print_json && !tree && !modified &&
export_crush.empty() && import_crush.empty() &&
- test_map_pg.empty() && test_map_object.empty()) {
+ test_map_pg.empty() && test_map_object.empty() &&
+ !test_map_pgs) {
cerr << me << ": no action specified?" << std::endl;
usage();
}
diff --git a/src/tools/psim.cc b/src/tools/psim.cc
index ecc176d..7f094b9 100644
--- a/src/tools/psim.cc
+++ b/src/tools/psim.cc
@@ -21,15 +21,25 @@ int main(int argc, char **argv)
OSDMap osdmap;
osdmap.decode(bl);
+ //osdmap.set_primary_affinity(0, 0x8000);
+ //osdmap.set_primary_affinity(3, 0);
+
int n = osdmap.get_max_osd();
int count[n];
+ int first_count[n];
+ int primary_count[n];
for (int i=0; i<n; i++) {
osdmap.set_state(i, osdmap.get_state(i) | CEPH_OSD_UP);
- //if (i<8)
+ //if (i<12)
osdmap.set_weight(i, CEPH_OSD_IN);
count[i] = 0;
+ first_count[i] = 0;
+ primary_count[i] = 0;
}
+ //pg_pool_t *p = (pg_pool_t *)osdmap.get_pg_pool(0);
+ //p->type = pg_pool_t::TYPE_ERASURE;
+
int size[4];
for (int i=0; i<4; i++)
size[i] = 0;
@@ -48,7 +58,7 @@ int main(int argc, char **argv)
pg_t pgid = pg_t(l.ol_pgid);
//pgid.u.ps = f * 4 + b;
int primary;
- osdmap.pg_to_osds(pgid, &osds, &primary);
+ osdmap.pg_to_acting_osds(pgid, &osds, &primary);
size[osds.size()]++;
#if 0
if (0) {
@@ -66,13 +76,20 @@ int main(int argc, char **argv)
//cout << " rep " << i << " on " << osds[i] << std::endl;
count[osds[i]]++;
}
+ if (osds.size())
+ first_count[osds[0]]++;
+ if (primary >= 0)
+ primary_count[primary]++;
}
}
}
uint64_t avg = 0;
for (int i=0; i<n; i++) {
- cout << "osd." << i << "\t" << count[i] << std::endl;
+ cout << "osd." << i << "\t" << count[i]
+ << "\t" << first_count[i]
+ << "\t" << primary_count[i]
+ << std::endl;
avg += count[i];
}
avg /= n;
diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index e5ee834..9fb60d1 100644
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -101,6 +101,8 @@ void usage(ostream& out)
" setomapheader <obj-name> <val>\n"
" tmap-to-omap <obj-name> convert tmap keys/values to omap\n"
" listwatchers <obj-name> list the watchers of this object\n"
+" set-alloc-hint <obj-name> <expected-object-size> <expected-write-size>\n"
+" set allocation hint for an object\n"
"\n"
"IMPORT AND EXPORT\n"
" import [options] <local-directory> <rados-pool>\n"
@@ -845,7 +847,7 @@ protected:
return io_ctx.read(oid, bl, len, 0);
}
int sync_write(const std::string& oid, bufferlist& bl, size_t len) {
- return io_ctx.write(oid, bl, len, 0);
+ return io_ctx.write_full(oid, bl);
}
int sync_remove(const std::string& oid) {
@@ -1358,7 +1360,11 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
// list pools?
if (strcmp(nargs[0], "lspools") == 0) {
list<string> vec;
- rados.pool_list(vec);
+ ret = rados.pool_list(vec);
+ if (ret < 0) {
+ cerr << "error listing pools: " << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
for (list<string>::iterator i = vec.begin(); i != vec.end(); ++i)
cout << *i << std::endl;
}
@@ -1366,13 +1372,22 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
// pools
list<string> vec;
- if (!pool_name)
- rados.pool_list(vec);
- else
+ if (!pool_name) {
+ ret = rados.pool_list(vec);
+ if (ret < 0) {
+ cerr << "error listing pools: " << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
+ } else {
vec.push_back(pool_name);
+ }
map<string, map<string, pool_stat_t> > stats;
- rados.get_pool_stats(vec, category, stats);
+ ret = rados.get_pool_stats(vec, category, stats);
+ if (ret < 0) {
+ cerr << "error fetching pool stats: " << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
if (!formatter) {
printf("%-15s %-15s"
@@ -1449,7 +1464,11 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
// total
cluster_stat_t tstats;
- rados.cluster_stat(tstats);
+ ret = rados.cluster_stat(tstats);
+ if (ret < 0) {
+ cerr << "error getting total cluster usage: " << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
if (!formatter) {
printf(" total used %12lld %12lld\n", (long long unsigned)tstats.kb_used,
(long long unsigned)tstats.num_objects);
@@ -2180,6 +2199,27 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
ret = io_ctx.notify(oid, 0, bl);
if (ret != 0)
cerr << "error calling notify: " << ret << std::endl;
+ } else if (strcmp(nargs[0], "set-alloc-hint") == 0) {
+ if (!pool_name || nargs.size() < 4)
+ usage_exit();
+ string err;
+ string oid(nargs[1]);
+ uint64_t expected_object_size = strict_strtoll(nargs[2], 10, &err);
+ if (!err.empty()) {
+ cerr << "couldn't parse expected_object_size: " << err << std::endl;
+ usage_exit();
+ }
+ uint64_t expected_write_size = strict_strtoll(nargs[3], 10, &err);
+ if (!err.empty()) {
+ cerr << "couldn't parse expected_write_size: " << err << std::endl;
+ usage_exit();
+ }
+ ret = io_ctx.set_alloc_hint(oid, expected_object_size, expected_write_size);
+ if (ret < 0) {
+ cerr << "error setting alloc-hint " << pool_name << "/" << oid << ": "
+ << strerror_r(-ret, buf, sizeof(buf)) << std::endl;
+ goto out;
+ }
} else if (strcmp(nargs[0], "load-gen") == 0) {
if (!pool_name) {
cerr << "error: must specify pool" << std::endl;
diff --git a/src/tools/rados/rados_sync.cc b/src/tools/rados/rados_sync.cc
index d5ca0e7..595df81 100644
--- a/src/tools/rados/rados_sync.cc
+++ b/src/tools/rados/rados_sync.cc
@@ -537,7 +537,7 @@ void BackedUpObject::get_xattrs(std::list < std::string > &xattrs_) const
}
}
-const Xattr* BackedUpObject::get_xattr(const std::string name) const
+const Xattr* BackedUpObject::get_xattr(const std::string &name) const
{
std::map < std::string, Xattr* >::const_iterator x = xattrs.find(name);
if (x == xattrs.end())
diff --git a/src/tools/rados/rados_sync.h b/src/tools/rados/rados_sync.h
index 3d4cf19..d762450 100644
--- a/src/tools/rados/rados_sync.h
+++ b/src/tools/rados/rados_sync.h
@@ -177,7 +177,7 @@ public:
void get_xattrs(std::list < std::string > &xattrs_) const;
- const Xattr* get_xattr(const std::string name) const;
+ const Xattr* get_xattr(const std::string &name) const;
const char *get_rados_name() const;
diff --git a/src/upstart/rbdmap.conf b/src/upstart/rbdmap.conf
index a581c2a..eeefec3 100644
--- a/src/upstart/rbdmap.conf
+++ b/src/upstart/rbdmap.conf
@@ -27,7 +27,7 @@ pre-start script
DEV=rbd/$DEV
;;
esac
- for PARAM in $(echo $PARAM | tr ',' '\n'); do
+ for PARAM in $(echo $PARAMS | tr ',' '\n'); do
CMDPARAMS="$CMDPARAMS --$(echo $PARAM | tr '=' ' ')"
done
if [ ! -b /dev/rbd/$DEV ]; then
diff --git a/src/vstart.sh b/src/vstart.sh
index 9c76c53..181f812 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -327,6 +327,7 @@ if [ "$start_mon" -eq 1 ]; then
osd pgp bits = 5 ; (invalid, but ceph should cope!)
osd crush chooseleaf type = 0
osd pool default min size = 1
+ osd pool default erasure code directory = .libs
run dir = $CEPH_OUT_DIR
EOF
if [ "$cephx" -eq 1 ] ; then
@@ -368,6 +369,7 @@ $COSDMEMSTORE
$extra_conf
[mon]
mon pg warn min per osd = 10
+ mon osd allow primary affinity = true
$DAEMONOPTS
$CMONDEBUG
$extra_conf
--
Alioth's hooks/post-receive on /srv/git.debian.org/git/pkg-ceph/ceph.git
More information about the Pkg-ceph-commits
mailing list