[Pkg-ceph-commits] [ceph] 01/01: Imported Upstream version 0.77
James Downing Page
jamespage at moszumanska.debian.org
Sat Mar 8 16:49:49 UTC 2014
This is an automated email from the git hooks/post-receive script.
jamespage pushed a commit to annotated tag upstream/0.77
in repository ceph.
commit ac0edd8d661a7343580e832767cf7c35b7d91198
Author: James Page <james.page at ubuntu.com>
Date: Tue Mar 4 10:21:09 2014 +0000
Imported Upstream version 0.77
---
Makefile.in | 10 +-
README | 16 +-
aclocal.m4 | 64 +
ar-lib | 265 +
autogen.sh | 15 +-
ceph.spec | 32 +-
ceph.spec.in | 30 +-
configure | 5770 +--
configure.ac | 164 +-
m4/ax_c_pretty_func.m4 | 18 +
m4/ax_c_var_func.m4 | 66 +
m4/ax_cxx_static_cast.m4 | 43 +
man/Makefile.in | 6 +
man/ceph-authtool.8 | 102 +-
man/ceph-clsinfo.8 | 35 +-
man/ceph-conf.8 | 52 +-
man/ceph-debugpack.8 | 36 +-
man/ceph-dencoder.8 | 47 +-
man/ceph-fuse.8 | 39 +-
man/ceph-mds.8 | 35 +-
man/ceph-mon.8 | 45 +-
man/ceph-osd.8 | 42 +-
man/ceph-post-file.8 | 6 +-
man/ceph-rbdnamer.8 | 39 +-
man/ceph-rest-api.8 | 45 +-
man/ceph-run.8 | 35 +-
man/ceph-syn.8 | 43 +-
man/ceph.8 | 50 +-
man/cephfs.8 | 37 +-
man/crushtool.8 | 364 +-
man/librados-config.8 | 37 +-
man/monmaptool.8 | 54 +-
man/mount.ceph.8 | 59 +-
man/osdmaptool.8 | 46 +-
man/rados.8 | 68 +-
man/radosgw-admin.8 | 69 +-
man/radosgw.8 | 67 +-
man/rbd-fuse.8 | 43 +-
man/rbd.8 | 160 +-
src/.git_version | 4 +-
src/Makefile-env.am | 24 +-
src/Makefile.am | 21 +-
src/Makefile.in | 2049 +-
src/acconfig.h.in | 119 +
src/arch/intel.c | 6 +-
src/arch/intel.h | 2 +-
src/auth/AuthAuthorizeHandler.h | 3 +-
src/auth/AuthMethodList.cc | 2 +
src/auth/KeyRing.cc | 1 +
src/auth/KeyRing.h | 2 +-
src/ceph-crush-location | 87 +
src/ceph-crush-location.in | 87 +
src/ceph-disk | 611 +-
src/ceph.in | 2 +-
src/ceph_mds.cc | 4 +-
src/ceph_mon.cc | 136 +-
src/ceph_osd.cc | 40 +-
src/civetweb/civetweb.h | 496 +
src/civetweb/include/civetweb.h | 496 +
src/civetweb/src/civetweb.c | 6453 ++++
src/civetweb/src/md5.h | 461 +
src/client/Client.cc | 378 +-
src/client/Client.h | 15 +-
src/client/Dir.h | 2 +-
src/client/Inode.h | 5 +
src/client/MetaSession.h | 4 +-
src/client/SyntheticClient.cc | 40 +-
src/client/SyntheticClient.h | 2 +
src/client/Trace.cc | 2 -
src/client/fuse_ll.cc | 10 +-
src/cls/Makefile.am | 34 +-
src/cls/user/cls_user.cc | 372 +
src/cls/user/cls_user_client.cc | 145 +
src/cls/user/cls_user_client.h | 33 +
src/cls/user/cls_user_ops.cc | 115 +
src/cls/user/cls_user_ops.h | 179 +
src/cls/user/cls_user_types.cc | 107 +
src/cls/user/cls_user_types.h | 182 +
src/common/Finisher.cc | 1 +
src/common/Makefile.am | 13 +-
src/common/TrackedOp.h | 9 +-
src/common/WorkQueue.cc | 1 +
src/common/admin_socket.cc | 25 +-
src/common/admin_socket_client.cc | 8 +
src/common/admin_socket_client.h | 1 +
src/common/blkdev.cc | 66 +-
src/common/bloom_filter.hpp | 8 +
src/common/buffer.cc | 365 +-
src/common/ceph_argparse.cc | 75 +-
src/common/ceph_argparse.h | 5 -
src/common/ceph_context.cc | 68 +-
src/common/ceph_context.h | 3 +-
src/common/ceph_strings.cc | 8 +
src/common/common_init.cc | 6 +-
src/common/common_init.h | 2 +-
src/common/config.cc | 166 +-
src/common/config.h | 17 +-
src/common/config_opts.h | 76 +-
src/common/hobject.cc | 2 +-
src/common/hobject.h | 31 +-
src/common/ipaddr.cc | 2 +-
src/common/linux_version.c | 25 +
src/common/linux_version.h | 22 +
src/common/lockdep.cc | 15 +-
src/common/map_cacher.hpp | 2 +-
src/common/obj_bencher.cc | 191 +-
src/common/obj_bencher.h | 1 +
src/common/pipe.c | 42 +-
src/common/safe_io.c | 35 +
src/common/safe_io.h | 13 +
src/common/sctp_crc32.c | 12 +-
src/common/shared_cache.hpp | 4 +-
src/common/sharedptr_registry.hpp | 4 +-
src/common/str_map.cc | 68 +
src/crush/CrushCompiler.cc | 70 +-
src/crush/CrushTester.cc | 18 +-
src/crush/CrushTester.h | 76 +-
src/crush/CrushWrapper.cc | 350 +-
src/crush/CrushWrapper.h | 101 +-
src/crush/builder.c | 2 +
src/crush/crush.c | 7 +-
src/crush/crush.h | 19 +-
src/crush/grammar.h | 18 +-
src/crush/mapper.c | 408 +-
src/crush/mapper.h | 3 +-
src/crush/sample.txt | 2 +-
src/gtest/Makefile.am | 2 +-
src/gtest/Makefile.in | 2 +-
src/include/Context.h | 4 +-
src/include/Makefile.am | 13 +-
src/include/Spinlock.h | 66 +-
src/include/assert.h | 45 +-
src/include/atomic.h | 33 +-
src/include/buffer.h | 35 +-
src/include/byteorder.h | 47 +-
src/include/ceph_features.h | 18 +-
src/include/ceph_fs.h | 36 +-
src/include/cephfs/libcephfs.h | 203 +-
src/include/compat.h | 15 +-
src/include/encoding.h | 78 +-
src/include/hash_namespace.h | 24 +
src/include/int_types.h | 37 +-
src/include/linux_fiemap.h | 2 +
src/include/memory.h | 26 +
src/include/object.h | 12 +-
src/include/on_exit.h | 49 +
src/include/rados.h | 46 +-
src/include/rados/buffer.h | 35 +-
src/include/rados/librados.h | 204 +-
src/include/rados/librados.hpp | 104 +-
src/include/rados/memory.h | 26 +
src/include/rados/rados_types.hpp | 2 +
src/include/stat.h | 145 +
src/include/str_map.h | 59 +
src/include/types.h | 45 +-
src/include/unordered_map.h | 24 +
src/include/unordered_set.h | 24 +
src/include/utime.h | 5 +-
src/init-ceph.in | 13 +-
src/init-radosgw | 43 +-
src/init-radosgw.sysv | 45 +-
src/java/Makefile.in | 6 +
src/json_spirit/Makefile.am | 3 +-
src/json_spirit/json_spirit_value.cpp | 8 -
src/key_value_store/Makefile.am | 4 +-
src/key_value_store/kv_flat_btree_async.cc | 8 +-
src/libcephfs.cc | 210 +-
src/librados/AioCompletionImpl.h | 5 +-
src/librados/IoCtxImpl.cc | 98 +-
src/librados/IoCtxImpl.h | 13 +-
src/librados/Makefile.am | 7 +-
src/librados/RadosClient.cc | 81 +-
src/librados/RadosClient.h | 21 +-
src/librados/librados.cc | 286 +-
src/librbd/AioCompletion.h | 10 +-
src/librbd/AioRequest.cc | 2 +-
src/librbd/ImageCtx.cc | 7 +-
src/librbd/ImageCtx.h | 3 +
src/librbd/LibrbdWriteback.cc | 2 +-
src/librbd/LibrbdWriteback.h | 2 +-
src/librbd/Makefile.am | 5 +-
src/librbd/internal.cc | 43 +-
src/libs3/debian/changelog | 5 -
src/libs3/debian/changelog.Debian | 6 -
src/libs3/debian/control | 12 -
src/libs3/debian/control.dev | 26 -
src/libs3/debian/postinst | 3 -
src/log/Log.cc | 8 +-
src/mds/CDentry.h | 2 +-
src/mds/CDir.cc | 453 +-
src/mds/CDir.h | 26 +-
src/mds/CInode.cc | 63 +-
src/mds/CInode.h | 2 +-
src/mds/Capability.cc | 30 +
src/mds/Capability.h | 83 +-
src/mds/Dumper.cc | 4 +-
src/mds/Locker.cc | 104 +-
src/mds/Locker.h | 1 +
src/mds/LogSegment.h | 6 +-
src/mds/MDCache.cc | 458 +-
src/mds/MDCache.h | 29 +-
src/mds/MDS.cc | 37 +-
src/mds/MDS.h | 5 +-
src/mds/MDSMap.cc | 27 +-
src/mds/MDSMap.h | 13 +-
src/mds/Migrator.cc | 1127 +-
src/mds/Migrator.h | 148 +-
src/mds/Mutation.h | 2 +
src/mds/Resetter.cc | 4 +-
src/mds/Server.cc | 256 +-
src/mds/Server.h | 6 +-
src/mds/SessionMap.cc | 11 +-
src/mds/SessionMap.h | 32 +-
src/mds/SimpleLock.h | 2 +-
src/mds/events/EMetaBlob.h | 16 +-
src/mds/journal.cc | 16 +-
src/mds/mdstypes.cc | 14 +-
src/mds/mdstypes.h | 80 +-
src/messages/MCacheExpire.h | 30 +-
src/messages/MClientCaps.h | 59 +-
src/messages/MClientReply.h | 9 +
src/messages/MExportDir.h | 6 +-
src/messages/MExportDirAck.h | 11 +-
src/messages/MExportDirCancel.h | 7 +-
src/messages/MExportDirDiscover.h | 10 +-
src/messages/MExportDirDiscoverAck.h | 7 +-
src/messages/MExportDirFinish.h | 12 +-
src/messages/MExportDirNotify.h | 6 +-
src/messages/MExportDirNotifyAck.h | 6 +-
src/messages/MExportDirPrep.h | 7 +-
src/messages/MExportDirPrepAck.h | 7 +-
src/messages/MForward.h | 33 +-
src/messages/MMDSCacheRejoin.h | 9 +-
src/messages/MMDSResolve.h | 8 +-
src/messages/MMDSResolveAck.h | 4 +-
src/messages/MMonElection.h | 8 +-
src/messages/MOSDBoot.h | 6 +-
src/messages/MOSDMap.h | 5 +-
src/messages/MOSDOp.h | 11 +-
src/messages/MOSDPGLog.h | 1 +
src/messages/MOSDPGScan.h | 4 +-
src/messages/MOSDSubOp.h | 2 +-
src/messages/MOSDSubOpReply.h | 2 +-
src/mon/AuthMonitor.cc | 38 +-
src/mon/DataHealthService.cc | 19 +-
src/mon/{MonCommands.h => DumplingMonCommands.h} | 80 +-
src/mon/Elector.cc | 99 +-
src/mon/Elector.h | 41 +-
src/mon/LogMonitor.cc | 6 +-
src/mon/MDSMonitor.cc | 162 +-
src/mon/MDSMonitor.h | 3 +-
src/mon/Makefile.am | 1 +
src/mon/MonCap.cc | 2 +-
src/mon/MonClient.cc | 9 +-
src/mon/MonClient.h | 6 +-
src/mon/MonCommands.h | 38 +-
src/mon/MonMap.cc | 1 +
src/mon/Monitor.cc | 385 +-
src/mon/Monitor.h | 106 +-
src/mon/MonitorDBStore.h | 34 +-
src/mon/MonmapMonitor.cc | 91 +-
src/mon/OSDMonitor.cc | 690 +-
src/mon/OSDMonitor.h | 14 +-
src/mon/PGMap.cc | 90 +-
src/mon/PGMap.h | 22 +-
src/mon/PGMonitor.cc | 42 +-
src/mon/Session.h | 3 +-
src/mount/mount.ceph.c | 10 +-
src/msg/Accepter.cc | 2 -
src/msg/Pipe.cc | 30 +-
src/msg/SimpleMessenger.cc | 9 +-
src/msg/SimpleMessenger.h | 16 +-
src/msg/msg_types.h | 13 +-
src/objclass/class_api.cc | 4 +-
src/ocf/Makefile.in | 6 +
src/os/CollectionIndex.h | 14 +-
src/os/DBObjectMap.cc | 38 +-
src/os/DBObjectMap.h | 15 +-
src/os/FDCache.h | 2 +-
src/os/FileJournal.cc | 88 +-
src/os/FileStore.cc | 243 +-
src/os/FileStore.h | 49 +-
src/os/FlatIndex.cc | 2 +-
src/os/FlatIndex.h | 6 +-
src/os/GenericObjectMap.cc | 1112 +
src/os/GenericObjectMap.h | 468 +
src/os/HashIndex.cc | 2 +-
src/os/HashIndex.h | 2 +-
src/os/IndexManager.cc | 2 +-
src/os/IndexManager.h | 6 +-
src/os/JournalingObjectStore.h | 9 +-
src/os/KeyValueDB.h | 16 +-
src/os/KeyValueStore.cc | 2978 ++
src/os/KeyValueStore.h | 594 +
src/os/LFNIndex.cc | 2 +-
src/os/LFNIndex.h | 10 +-
src/os/LevelDBStore.cc | 28 +-
src/os/LevelDBStore.h | 18 +-
src/os/Makefile.am | 49 +-
src/os/MemStore.cc | 1478 +
src/os/MemStore.h | 357 +
src/os/ObjectMap.h | 14 +-
src/os/ObjectStore.cc | 60 +-
src/os/ObjectStore.h | 120 +-
src/os/WBThrottle.cc | 37 +-
src/os/WBThrottle.h | 4 +-
src/osd/ErasureCodeInterface.h | 126 +-
src/osd/ErasureCodePlugin.cc | 9 +-
src/osd/ErasureCodePlugin.h | 1 +
.../ErasureCodeJerasure.cc | 62 +-
.../ErasureCodeJerasure.h | 24 +-
src/osd/ErasureCodePluginJerasure/Makefile.am | 10 +-
src/osd/ErasureCodePluginJerasure/cauchy.c | 0
src/osd/ErasureCodePluginJerasure/cauchy.h | 0
src/osd/ErasureCodePluginJerasure/galois.c | 48 +-
src/osd/ErasureCodePluginJerasure/galois.h | 0
src/osd/ErasureCodePluginJerasure/jerasure.c | 6 +-
src/osd/ErasureCodePluginJerasure/jerasure.h | 0
src/osd/ErasureCodePluginJerasure/liberation.c | 0
src/osd/ErasureCodePluginJerasure/liberation.h | 0
src/osd/ErasureCodePluginJerasure/reed_sol.c | 0
src/osd/ErasureCodePluginJerasure/reed_sol.h | 0
src/osd/ErasureCodePluginJerasure/vectorop.h | 51 +
src/osd/HitSet.cc | 218 +
src/osd/HitSet.h | 475 +
src/osd/Makefile.am | 5 +-
src/osd/OSD.cc | 449 +-
src/osd/OSD.h | 50 +-
src/osd/OSDCap.cc | 3 +-
src/osd/OSDMap.cc | 879 +-
src/osd/OSDMap.h | 195 +-
src/osd/OpRequest.cc | 2 +
src/osd/OpRequest.h | 14 +-
src/osd/PG.cc | 864 +-
src/osd/PG.h | 189 +-
src/osd/PGBackend.cc | 66 +
src/osd/PGBackend.h | 228 +-
src/osd/PGLog.cc | 244 +-
src/osd/PGLog.h | 67 +-
src/osd/ReplicatedBackend.cc | 855 +-
src/osd/ReplicatedBackend.h | 184 +
src/osd/ReplicatedPG.cc | 3894 +-
src/osd/ReplicatedPG.h | 492 +-
src/osd/Watch.h | 18 +-
src/osd/osd_types.cc | 540 +-
src/osd/osd_types.h | 438 +-
src/osdc/ObjectCacher.cc | 18 +-
src/osdc/ObjectCacher.h | 2 +-
src/osdc/Objecter.cc | 353 +-
src/osdc/Objecter.h | 281 +-
src/pybind/ceph_rest_api.py | 4 +-
src/pybind/cephfs.py | 18 +-
src/pybind/rados.py | 13 +-
src/pybind/rbd.py | 20 +-
src/rbd.cc | 326 +-
src/rgw/Makefile.am | 17 +-
src/rgw/rgw_admin.cc | 125 +-
src/rgw/rgw_bucket.cc | 169 +-
src/rgw/rgw_bucket.h | 20 +-
src/rgw/rgw_cache.cc | 26 +-
src/rgw/rgw_cache.h | 33 +-
src/rgw/rgw_civetweb.cc | 172 +
src/rgw/rgw_civetweb.h | 41 +
src/rgw/rgw_client_io.cc | 16 +
src/rgw/rgw_client_io.h | 15 +-
src/rgw/rgw_common.cc | 18 +-
src/rgw/rgw_common.h | 75 +-
src/rgw/rgw_cors.cc | 35 +-
src/rgw/rgw_cors.h | 3 +-
src/rgw/rgw_env.cc | 42 +-
src/rgw/rgw_fcgi.cc | 31 +-
src/rgw/rgw_fcgi.h | 7 +-
src/rgw/rgw_gc.cc | 8 +-
src/rgw/rgw_http_client.cc | 4 +-
src/rgw/rgw_http_errors.h | 46 +
src/rgw/rgw_json_enc.cc | 7 +-
src/rgw/rgw_loadgen.cc | 111 +
src/rgw/rgw_loadgen.h | 45 +
src/rgw/rgw_main.cc | 670 +-
src/rgw/rgw_metadata.cc | 2 +-
src/rgw/rgw_op.cc | 575 +-
src/rgw/rgw_op.h | 43 +-
src/rgw/rgw_quota.cc | 701 +-
src/rgw/rgw_quota.h | 7 +-
src/rgw/rgw_rados.cc | 394 +-
src/rgw/rgw_rados.h | 89 +-
src/rgw/rgw_resolve.cc | 6 +
src/rgw/rgw_rest.cc | 67 +-
src/rgw/rgw_rest.h | 43 +
src/rgw/rgw_rest_client.cc | 10 +-
src/rgw/rgw_rest_conn.cc | 2 +-
src/rgw/rgw_rest_log.cc | 2 +-
src/rgw/rgw_rest_metadata.cc | 1 -
src/rgw/rgw_rest_replica_log.cc | 29 +-
src/rgw/rgw_rest_s3.cc | 51 +-
src/rgw/rgw_rest_swift.cc | 61 +-
src/rgw/rgw_rest_swift.h | 10 +
src/rgw/rgw_rest_user.cc | 265 +
src/rgw/rgw_swift.cc | 189 +-
src/rgw/rgw_swift.h | 2 +
src/rgw/rgw_swift_auth.cc | 10 +-
src/rgw/rgw_user.cc | 78 +-
src/rgw/rgw_user.h | 23 +-
src/sample.ceph.conf | 485 +-
src/stop.sh | 27 +-
src/test/Makefile.am | 122 +-
src/test/ObjectMap/KeyValueDBMemory.cc | 6 +-
src/test/ObjectMap/KeyValueDBMemory.h | 12 +-
src/test/ObjectMap/test_keyvaluedb_atomicity.cc | 2 +-
src/test/ObjectMap/test_keyvaluedb_iterators.cc | 2 +-
src/test/ObjectMap/test_object_map.cc | 41 +-
src/test/admin_socket.cc | 135 +-
src/test/bench/bencher.cc | 18 +-
src/test/bench/bencher.h | 20 +-
src/test/bench/dumb_backend.cc | 10 +
src/test/bench/rbd_backend.cc | 4 +-
src/test/bench/rbd_backend.h | 4 +-
src/test/bench/small_io_bench_dumb.cc | 12 +
src/test/bench/small_io_bench_fs.cc | 10 +-
src/test/bench/small_io_bench_rbd.cc | 4 +-
src/test/bufferlist.cc | 214 +-
src/test/ceph_crypto.cc | 2 +
src/test/cli/ceph-conf/show-config-value.t | 15 +
src/test/cli/ceph-conf/show-config.t | 6 +
src/test/cli/crushtool/bad-mappings.crushmap.txt | 39 +
src/test/cli/crushtool/bad-mappings.t | 6 +
src/test/cli/crushtool/build.t | 104 +
src/test/cli/crushtool/set-choose.crushmap.txt | 144 +
src/test/cli/crushtool/set-choose.t | 36941 +++++++++++++++++++
...ndered-devices.crushmap => test-map-a.crushmap} | Bin 31979 -> 31995 bytes
...unable-values.t => test-map-bobtail-tunables.t} | 3 +-
...w-tunables.crushmap => test-map-indep.crushmap} | Bin 31979 -> 31995 bytes
src/test/cli/crushtool/test-map-indep.t | 10253 +++++
...tunable-values.t => test-map-legacy-tunables.t} | 2 +-
.../crushtool/test-map-tries-vs-retries.crushmap | Bin 0 -> 808 bytes
src/test/cli/crushtool/test-map-tries-vs-retries.t | 10259 +++++
src/test/cli/osdmaptool/clobber.t | 12 +-
src/test/cli/osdmaptool/create-print.t | 100 +-
src/test/cli/osdmaptool/create-racks.t | 824 +-
src/test/cli/radosgw-admin/help.t | 4 +
src/test/cli/rbd/help.t | 1 +
src/test/cls_rbd/test_cls_rbd.cc | 14 +-
src/test/cls_statelog/test_cls_statelog.cc | 2 +-
src/test/common/Throttle.cc | 17 +-
src/test/common/get_command_descriptions.cc | 7 +-
src/test/common/test_config.cc | 187 +
src/test/common/test_str_map.cc | 70 +
src/test/confutils.cc | 22 +-
src/test/crush/TestCrushWrapper.cc | 485 +
src/test/crush/indep.cc | 262 +
src/test/daemon_config.cc | 24 +-
src/test/encoding/ceph_dencoder.cc | 73 +-
src/test/encoding/check-generated.sh | 27 +-
src/test/encoding/types.h | 32 +-
src/test/filestore/store_test.cc | 73 +-
src/test/filestore/workload_generator.cc | 3 +-
src/test/libcephfs/caps.cc | 1 -
src/test/libcephfs/readdir_r_cb.cc | 6 +
src/test/libcephfs/test.cc | 12 +-
src/test/librados/c_write_operations.cc | 117 +
src/test/librados/cmd.cc | 15 +
src/test/librados/list.cc | 117 +
src/test/librados/misc.cc | 147 +-
src/test/librados/snapshots.cc | 5 +-
src/test/librados/tier.cc | 2216 ++
src/test/mon/moncap.cc | 3 +-
src/test/mon/test_mon_workloadgen.cc | 18 +-
src/test/on_exit.cc | 113 +
src/test/os/TestFlatIndex.cc | 6 +-
src/test/os/TestLFNIndex.cc | 2 +-
src/test/osd/ErasureCodeExample.h | 33 +-
src/test/osd/ErasureCodePluginMissingEntryPoint.cc | 3 +
src/test/osd/Object.cc | 153 +-
src/test/osd/Object.h | 190 +-
src/test/osd/RadosModel.h | 967 +-
src/test/osd/TestErasureCodeExample.cc | 71 +-
src/test/osd/TestErasureCodeJerasure.cc | 30 +-
src/test/osd/TestOSDMap.cc | 239 +
src/test/osd/TestOpStat.h | 2 +-
src/test/osd/TestPGLog.cc | 302 +-
src/test/osd/TestRados.cc | 156 +-
src/test/osd/ceph_erasure_code_benchmark.cc | 218 +
src/test/osd/ceph_erasure_code_benchmark.h | 38 +
src/test/osd/hitset.cc | 197 +
src/test/osd/osdcap.cc | 1 +
src/test/osdc/object_cacher_stress.cc | 4 +-
src/test/run-cli-tests | 2 +-
src/test/streamtest.cc | 12 +-
src/test/{test_arch.c => test_arch.cc} | 1 +
src/test/test_c_headers.c | 22 +
src/test/test_crushwrapper.cc | 107 +
src/test/test_filejournal.cc | 32 +-
src/test/test_get_blkdev_size.cc | 35 +
src/test/test_ipaddr.cc | 22 +
src/test/test_osd_types.cc | 28 +-
src/test/test_rgw_admin_opstate.cc | 2 +-
src/test/test_snap_mapper.cc | 6 +-
src/test/test_striper.cc | 31 +
src/test/testmsgr.cc | 12 +-
src/test/xattr_bench.cc | 2 +-
src/tools/Makefile.am | 12 +-
src/tools/ceph-kvstore-tool.cc | 54 +-
src/tools/ceph_conf.cc | 6 +
src/tools/crushtool.cc | 90 +-
src/tools/dupstore.cc | 5 +-
src/tools/osdmaptool.cc | 33 +-
src/tools/psim.cc | 3 +-
src/tools/rados/rados.cc | 152 +-
src/tools/rados/rados_sync.cc | 9 +-
src/tools/rados/rados_sync.h | 3 +-
src/unittest_bufferlist.sh | 2 +-
src/upstart/ceph-osd.conf | 15 +-
src/upstart/rbdmap.conf | 48 +
src/vstart.sh | 79 +-
514 files changed, 111112 insertions(+), 11397 deletions(-)
diff --git a/Makefile.in b/Makefile.in
index 94be359..0da4246 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -38,8 +38,8 @@ subdir = .
DIST_COMMON = README $(am__configure_deps) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in $(srcdir)/ceph.spec.in \
$(top_srcdir)/configure AUTHORS COPYING ChangeLog INSTALL NEWS \
- compile config.guess config.sub depcomp install-sh ltmain.sh \
- missing py-compile
+ ar-lib compile config.guess config.sub depcomp install-sh \
+ ltmain.sh missing py-compile
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
$(top_srcdir)/m4/ac_prog_jar.m4 \
@@ -47,7 +47,10 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
$(top_srcdir)/m4/ac_prog_javac_works.m4 \
$(top_srcdir)/m4/ac_prog_javah.m4 \
$(top_srcdir)/m4/acx_pthread.m4 \
+ $(top_srcdir)/m4/ax_c_pretty_func.m4 \
+ $(top_srcdir)/m4/ax_c_var_func.m4 \
$(top_srcdir)/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/m4/ax_cxx_static_cast.m4 \
$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/pkg.m4 \
@@ -132,6 +135,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
+BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@
CC = @CC@
CCAS = @CCAS@
CCASDEPMODE = @CCASDEPMODE@
@@ -173,6 +177,7 @@ JAR = @JAR@
JAVAC = @JAVAC@
JAVAH = @JAVAH@
JDK_CPPFLAGS = @JDK_CPPFLAGS@
+KEYUTILS_LIB = @KEYUTILS_LIB@
LD = @LD@
LDFLAGS = @LDFLAGS@
LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@
@@ -218,6 +223,7 @@ PYTHON_PLATFORM = @PYTHON_PLATFORM@
PYTHON_PREFIX = @PYTHON_PREFIX@
PYTHON_VERSION = @PYTHON_VERSION@
RANLIB = @RANLIB@
+RESOLV_LIBS = @RESOLV_LIBS@
RPM_RELEASE = @RPM_RELEASE@
SED = @SED@
SET_MAKE = @SET_MAKE@
diff --git a/README b/README
index 3662d0e..197a7bf 100644
--- a/README
+++ b/README
@@ -99,7 +99,6 @@ To build the documentation, ensure that you are in the top-level `/ceph director
Build Prerequisites
===================
-
debian-based
------------
To build the source code, you must install the following:
@@ -123,6 +122,7 @@ To build the source code, you must install the following:
- libgoogle-perftools-dev
- libkeyutils-dev
- uuid-dev
+- libblkid-dev
- libatomic-ops-dev
- libboost-program-options-dev
- libboost-thread-dev
@@ -135,7 +135,7 @@ To build the source code, you must install the following:
For example:
- $ apt-get install automake autoconf pkg-config gcc g++ make libboost-dev libedit-dev libssl-dev libtool libfcgi libfcgi-dev libfuse-dev linux-kernel-headers libcrypto++-dev libaio-dev libgoogle-perftools-dev libkeyutils-dev uuid-dev libatomic-ops-dev libboost-program-options-dev libboost-thread-dev libexpat1-dev libleveldb-dev libsnappy-dev libcurl4-gnutls-dev python-argparse python-flask
+ $ apt-get install automake autoconf pkg-config gcc g++ make libboost-dev libedit-dev libssl-dev libtool libfcgi libfcgi-dev libfuse-dev linux-kernel-headers libcrypto++-dev libaio-dev libgoogle-perftools-dev libkeyutils-dev uuid-dev libblkid-dev libatomic-ops-dev libboost-program-options-dev libboost-thread-dev libexpat1-dev libleveldb-dev libsnappy-dev libcurl4-gnutls-dev python-argparse python-flask
rpm-based
---------
@@ -144,12 +144,19 @@ These are the rpm packages needed to install in an rpm-based OS:
autoconf
automake
gcc
+ gcc-c++
make
libtool
python-argparse
python-flask
libuuid-devel
+ libblkid-devel
+ keyutils-libs-devel
+ cryptopp-devel
nss-devel
+ fcgi-devel
+ expat-devel
+ libcurl-devel
fuse-devel
gperftools-devel
libedit-devel
@@ -158,3 +165,8 @@ These are the rpm packages needed to install in an rpm-based OS:
leveldb-devel
libaio-devel
boost-devel
+
+For example:
+
+ $ yum install autoconf automake gcc gcc-c++ make libtool python-argparse python-flask libuuid-devel libblkid-devel keyutils-libs-devel cryptopp-devel nss-devel fcgi-devel expat-devel libcurl-devel fuse-devel gperftools-devel libedit-devel libatomic_ops-devel snappy-devel leveldb-devel libaio-devel boost-devel
+
diff --git a/aclocal.m4 b/aclocal.m4
index 4256ed5..2da43c3 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -59,6 +59,67 @@ m4_ifndef([AC_AUTOCONF_VERSION],
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 1
+
+# AM_PROG_AR([ACT-IF-FAIL])
+# -------------------------
+# Try to determine the archiver interface, and trigger the ar-lib wrapper
+# if it is needed. If the detection of archiver interface fails, run
+# ACT-IF-FAIL (default is to abort configure with a proper error message).
+AC_DEFUN([AM_PROG_AR],
+[AC_BEFORE([$0], [LT_INIT])dnl
+AC_BEFORE([$0], [AC_PROG_LIBTOOL])dnl
+AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl
+AC_REQUIRE_AUX_FILE([ar-lib])dnl
+AC_CHECK_TOOLS([AR], [ar lib "link -lib"], [false])
+: ${AR=ar}
+
+AC_CACHE_CHECK([the archiver ($AR) interface], [am_cv_ar_interface],
+ [am_cv_ar_interface=ar
+ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[int some_variable = 0;]])],
+ [am_ar_try='$AR cru libconftest.a conftest.$ac_objext >&AS_MESSAGE_LOG_FD'
+ AC_TRY_EVAL([am_ar_try])
+ if test "$ac_status" -eq 0; then
+ am_cv_ar_interface=ar
+ else
+ am_ar_try='$AR -NOLOGO -OUT:conftest.lib conftest.$ac_objext >&AS_MESSAGE_LOG_FD'
+ AC_TRY_EVAL([am_ar_try])
+ if test "$ac_status" -eq 0; then
+ am_cv_ar_interface=lib
+ else
+ am_cv_ar_interface=unknown
+ fi
+ fi
+ rm -f conftest.lib libconftest.a
+ ])
+ ])
+
+case $am_cv_ar_interface in
+ar)
+ ;;
+lib)
+ # Microsoft lib, so override with the ar-lib wrapper script.
+ # FIXME: It is wrong to rewrite AR.
+ # But if we don't then we get into trouble of one sort or another.
+ # A longer-term fix would be to have automake use am__AR in this case,
+ # and then we could set am__AR="$am_aux_dir/ar-lib \$(AR)" or something
+ # similar.
+ AR="$am_aux_dir/ar-lib $AR"
+ ;;
+unknown)
+ m4_default([$1],
+ [AC_MSG_ERROR([could not determine $AR interface])])
+ ;;
+esac
+AC_SUBST([AR])dnl
+])
+
# Figure out how to run the assembler. -*- Autoconf -*-
# Copyright (C) 2001, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
@@ -1311,7 +1372,10 @@ m4_include([m4/ac_prog_javac.m4])
m4_include([m4/ac_prog_javac_works.m4])
m4_include([m4/ac_prog_javah.m4])
m4_include([m4/acx_pthread.m4])
+m4_include([m4/ax_c_pretty_func.m4])
+m4_include([m4/ax_c_var_func.m4])
m4_include([m4/ax_check_compile_flag.m4])
+m4_include([m4/ax_cxx_static_cast.m4])
m4_include([m4/libtool.m4])
m4_include([m4/ltoptions.m4])
m4_include([m4/ltsugar.m4])
diff --git a/ar-lib b/ar-lib
new file mode 100755
index 0000000..c0286a4
--- /dev/null
+++ b/ar-lib
@@ -0,0 +1,265 @@
+#! /bin/sh
+# Wrapper for Microsoft lib.exe
+
+me=ar-lib
+scriptversion=2012-01-30.22; # UTC
+
+# Copyright (C) 2010, 2012 Free Software Foundation, Inc.
+# Written by Peter Rosin <peda at lysator.liu.se>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <bug-automake at gnu.org> or send patches to
+# <automake-patches at gnu.org>.
+
+
+# func_error message
+func_error ()
+{
+ echo "$me: $1" 1>&2
+ exit 1
+}
+
+file_conv=
+
+# func_file_conv build_file
+# Convert a $build file to $host form and store it in $file
+# Currently only supports Windows hosts.
+func_file_conv ()
+{
+ file=$1
+ case $file in
+ / | /[!/]*) # absolute file, and not a UNC file
+ if test -z "$file_conv"; then
+ # lazily determine how to convert abs files
+ case `uname -s` in
+ MINGW*)
+ file_conv=mingw
+ ;;
+ CYGWIN*)
+ file_conv=cygwin
+ ;;
+ *)
+ file_conv=wine
+ ;;
+ esac
+ fi
+ case $file_conv in
+ mingw)
+ file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'`
+ ;;
+ cygwin)
+ file=`cygpath -m "$file" || echo "$file"`
+ ;;
+ wine)
+ file=`winepath -w "$file" || echo "$file"`
+ ;;
+ esac
+ ;;
+ esac
+}
+
+# func_at_file at_file operation archive
+# Iterate over all members in AT_FILE performing OPERATION on ARCHIVE
+# for each of them.
+# When interpreting the content of the @FILE, do NOT use func_file_conv,
+# since the user would need to supply preconverted file names to
+# binutils ar, at least for MinGW.
+func_at_file ()
+{
+ operation=$2
+ archive=$3
+ at_file_contents=`cat "$1"`
+ eval set x "$at_file_contents"
+ shift
+
+ for member
+ do
+ $AR -NOLOGO $operation:"$member" "$archive" || exit $?
+ done
+}
+
+case $1 in
+ '')
+ func_error "no command. Try '$0 --help' for more information."
+ ;;
+ -h | --h*)
+ cat <<EOF
+Usage: $me [--help] [--version] PROGRAM ACTION ARCHIVE [MEMBER...]
+
+Members may be specified in a file named with @FILE.
+EOF
+ exit $?
+ ;;
+ -v | --v*)
+ echo "$me, version $scriptversion"
+ exit $?
+ ;;
+esac
+
+if test $# -lt 3; then
+ func_error "you must specify a program, an action and an archive"
+fi
+
+AR=$1
+shift
+while :
+do
+ if test $# -lt 2; then
+ func_error "you must specify a program, an action and an archive"
+ fi
+ case $1 in
+ -lib | -LIB \
+ | -ltcg | -LTCG \
+ | -machine* | -MACHINE* \
+ | -subsystem* | -SUBSYSTEM* \
+ | -verbose | -VERBOSE \
+ | -wx* | -WX* )
+ AR="$AR $1"
+ shift
+ ;;
+ *)
+ action=$1
+ shift
+ break
+ ;;
+ esac
+done
+orig_archive=$1
+shift
+func_file_conv "$orig_archive"
+archive=$file
+
+# strip leading dash in $action
+action=${action#-}
+
+delete=
+extract=
+list=
+replace=
+create=
+
+while test -n "$action"
+do
+ case $action in
+ d*) delete=yes ;;
+ x*) extract=yes ;;
+ t*) list=yes ;;
+ r*) replace=yes ;;
+ c*) create=yes ;;
+ u*) ;; # TODO: don't ignore the update modifier
+ v*) ;; # TODO: don't ignore the verbose modifier
+ *)
+ func_error "unknown action specified"
+ ;;
+ esac
+ action=${action#?}
+done
+
+case $delete$extract$list$replace in
+ yes)
+ ;;
+ yesyes*)
+ func_error "more than one action specified"
+ ;;
+ *)
+ func_error "no action specified"
+ ;;
+esac
+
+if test -n "$delete"; then
+ if test ! -f "$orig_archive"; then
+ func_error "archive not found"
+ fi
+ for member
+ do
+ case $1 in
+ @*)
+ func_at_file "${1#@}" -REMOVE "$archive"
+ ;;
+ *)
+ func_file_conv "$1"
+ $AR -NOLOGO -REMOVE:"$file" "$archive" || exit $?
+ ;;
+ esac
+ done
+
+elif test -n "$extract"; then
+ if test ! -f "$orig_archive"; then
+ func_error "archive not found"
+ fi
+ if test $# -gt 0; then
+ for member
+ do
+ case $1 in
+ @*)
+ func_at_file "${1#@}" -EXTRACT "$archive"
+ ;;
+ *)
+ func_file_conv "$1"
+ $AR -NOLOGO -EXTRACT:"$file" "$archive" || exit $?
+ ;;
+ esac
+ done
+ else
+ $AR -NOLOGO -LIST "$archive" | sed -e 's/\\/\\\\/g' | while read member
+ do
+ $AR -NOLOGO -EXTRACT:"$member" "$archive" || exit $?
+ done
+ fi
+
+elif test -n "$replace"; then
+ if test ! -f "$orig_archive"; then
+ if test -z "$create"; then
+ echo "$me: creating $orig_archive"
+ fi
+ orig_archive=
+ else
+ orig_archive=$archive
+ fi
+
+ for member
+ do
+ case $1 in
+ @*)
+ func_file_conv "${1#@}"
+ set x "$@" "@$file"
+ ;;
+ *)
+ func_file_conv "$1"
+ set x "$@" "$file"
+ ;;
+ esac
+ shift
+ shift
+ done
+
+ if test -n "$orig_archive"; then
+ $AR -NOLOGO -OUT:"$archive" "$orig_archive" "$@" || exit $?
+ else
+ $AR -NOLOGO -OUT:"$archive" "$@" || exit $?
+ fi
+
+elif test -n "$list"; then
+ if test ! -f "$orig_archive"; then
+ func_error "archive not found"
+ fi
+ $AR -NOLOGO -LIST "$archive" || exit $?
+fi
diff --git a/autogen.sh b/autogen.sh
index f90c49d..103caa4 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+set -e
+
check_for_pkg_config() {
which pkg-config >/dev/null && return
@@ -11,10 +13,21 @@ check_for_pkg_config() {
exit 1
}
+if [ `which libtoolize` ]; then
+ LIBTOOLIZE=libtoolize
+elif [ `which glibtoolize` ]; then
+ LIBTOOLIZE=glibtoolize
+else
+ echo "Error: could not find libtoolize"
+ echo " Please install libtoolize or glibtoolize."
+ exit 1
+fi
+
rm -f config.cache
aclocal -I m4 --install
check_for_pkg_config
-libtoolize --force --copy
+$LIBTOOLIZE --force --copy
+aclocal -I m4 --install
autoconf
autoheader
automake -a --add-missing -Wall
diff --git a/ceph.spec b/ceph.spec
index 1835de7..31dc0b8 100644
--- a/ceph.spec
+++ b/ceph.spec
@@ -9,7 +9,7 @@
# common
#################################################################################
Name: ceph
-Version: 0.72.2
+Version: 0.77
Release: 0%{?dist}
Summary: User space components of the Ceph file system
License: GPL-2.0
@@ -29,6 +29,7 @@ Requires: util-linux
Requires: hdparm
Requires(post): binutils
BuildRoot: %{_tmppath}/%{name}-%{version}-build
+BuildRequires: make
BuildRequires: gcc-c++
BuildRequires: libtool
BuildRequires: boost-devel
@@ -38,10 +39,12 @@ BuildRequires: gdbm
BuildRequires: pkgconfig
BuildRequires: python
BuildRequires: python-nose
+BuildRequires: python-argparse
BuildRequires: libaio-devel
BuildRequires: libcurl-devel
BuildRequires: libxml2-devel
BuildRequires: libuuid-devel
+BuildRequires: libblkid-devel >= 2.17
BuildRequires: leveldb-devel > 1.2
BuildRequires: yasm
%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora}
@@ -312,7 +315,7 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config
install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules
install -m 0644 -D udev/60-ceph-partuuid-workaround.rules $RPM_BUILD_ROOT/lib/udev/rules.d/60-ceph-partuuid-workaround.rules
-%if 0%{?centos}
+%if (0%{?rhel} || 0%{?rhel} < 7)
install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
%else
install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
@@ -328,7 +331,7 @@ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/osd
mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/mds
mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-osd
mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-mds
-
+mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/log/radosgw
%if %{defined suse_version}
# Fedora seems to have some problems with this macro, use it only on SUSE
@@ -364,7 +367,7 @@ fi
%endif
# Package removal cleanup
if [ "$1" -eq "0" ] ; then
- rm -rf /var/log/ceph
+ rm -rf /var/log/ceph
rm -rf /etc/ceph
fi
@@ -391,6 +394,7 @@ fi
%{_bindir}/ceph-authtool
%{_bindir}/ceph-syn
%{_bindir}/ceph-post-file
+%{_bindir}/ceph-crush-location
%{_bindir}/ceph-run
%{_bindir}/ceph-mon
%{_bindir}/ceph-mds
@@ -412,6 +416,7 @@ fi
%{_sbindir}/rcceph
/sbin/mkcephfs
/sbin/mount.ceph
+%{_libdir}/ceph
%dir %{_libdir}/rados-classes
%{_libdir}/rados-classes/libcls_rbd.so*
%{_libdir}/rados-classes/libcls_hello.so*
@@ -423,7 +428,13 @@ fi
%{_libdir}/rados-classes/libcls_replica_log.so*
%{_libdir}/rados-classes/libcls_statelog.so*
%{_libdir}/rados-classes/libcls_version.so*
-%{_libdir}/ceph
+%dir %{_libdir}/ceph/erasure-code
+%{_libdir}/ceph/erasure-code/libec_example.so*
+%{_libdir}/ceph/erasure-code/libec_fail_to_initialize.so*
+%{_libdir}/ceph/erasure-code/libec_fail_to_register.so*
+%{_libdir}/ceph/erasure-code/libec_hangs.so*
+%{_libdir}/ceph/erasure-code/libec_jerasure.so*
+%{_libdir}/ceph/erasure-code/libec_missing_entry_point.so*
/lib/udev/rules.d/50-rbd.rules
/lib/udev/rules.d/60-ceph-partuuid-workaround.rules
/lib/udev/rules.d/95-ceph-osd.rules
@@ -497,6 +508,7 @@ fi
%{_includedir}/rados/crc32c.h
%{_includedir}/rados/rados_types.h
%{_includedir}/rados/rados_types.hpp
+%{_includedir}/rados/memory.h
%dir %{_includedir}/rbd
%{_includedir}/rbd/librbd.h
%{_includedir}/rbd/librbd.hpp
@@ -515,6 +527,7 @@ fi
%{_mandir}/man8/radosgw.8*
%{_mandir}/man8/radosgw-admin.8*
%{_sbindir}/rcceph-radosgw
+%dir %{_localstatedir}/log/radosgw/
%post radosgw
/sbin/ldconfig
@@ -533,6 +546,11 @@ fi
%restart_on_update ceph-radosgw
%insserv_cleanup
%endif
+# Package removal cleanup
+if [ "$1" -eq "0" ] ; then
+ rm -rf /var/log/radosgw
+fi
+
#################################################################################
%if %{with ocf}
@@ -598,6 +616,7 @@ fi
%{_bindir}/ceph_dupstore
%{_bindir}/ceph_kvstorebench
%{_bindir}/ceph_multi_stress_watch
+%{_bindir}/ceph_erasure_code_benchmark
%{_bindir}/ceph_omapbench
%{_bindir}/ceph_psim
%{_bindir}/ceph_radosacl
@@ -612,6 +631,7 @@ fi
%{_bindir}/ceph_filestore_dump
%{_bindir}/ceph_filestore_tool
%{_bindir}/ceph_streamtest
+%{_bindir}/ceph_test_c_headers
%{_bindir}/ceph_test_cfuse_cache_invalidate
%{_bindir}/ceph_test_cls_hello
%{_bindir}/ceph_test_cls_lock
@@ -631,6 +651,7 @@ fi
%{_bindir}/ceph_test_filestore_idempotent
%{_bindir}/ceph_test_filestore_idempotent_sequence
%{_bindir}/ceph_test_filestore_workloadgen
+%{_bindir}/ceph_test_get_blkdev_size
%{_bindir}/ceph_test_ioctls
%{_bindir}/ceph_test_keyvaluedb_atomicity
%{_bindir}/ceph_test_keyvaluedb_iterators
@@ -648,6 +669,7 @@ fi
%{_bindir}/ceph_test_rados_api_list
%{_bindir}/ceph_test_rados_api_lock
%{_bindir}/ceph_test_rados_api_misc
+%{_bindir}/ceph_test_rados_api_tier
%{_bindir}/ceph_test_rados_api_pool
%{_bindir}/ceph_test_rados_api_snapshots
%{_bindir}/ceph_test_rados_api_stat
diff --git a/ceph.spec.in b/ceph.spec.in
index 27400b1..d18aa4f 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -29,6 +29,7 @@ Requires: util-linux
Requires: hdparm
Requires(post): binutils
BuildRoot: %{_tmppath}/%{name}-%{version}-build
+BuildRequires: make
BuildRequires: gcc-c++
BuildRequires: libtool
BuildRequires: boost-devel
@@ -38,10 +39,12 @@ BuildRequires: gdbm
BuildRequires: pkgconfig
BuildRequires: python
BuildRequires: python-nose
+BuildRequires: python-argparse
BuildRequires: libaio-devel
BuildRequires: libcurl-devel
BuildRequires: libxml2-devel
BuildRequires: libuuid-devel
+BuildRequires: libblkid-devel >= 2.17
BuildRequires: leveldb-devel > 1.2
BuildRequires: yasm
%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora}
@@ -312,7 +315,7 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config
install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules
install -m 0644 -D udev/60-ceph-partuuid-workaround.rules $RPM_BUILD_ROOT/lib/udev/rules.d/60-ceph-partuuid-workaround.rules
-%if 0%{?centos}
+%if (0%{?rhel} || 0%{?rhel} < 7)
install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
%else
install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
@@ -328,7 +331,7 @@ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/osd
mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/mds
mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-osd
mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-mds
-
+mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/log/radosgw
%if %{defined suse_version}
# Fedora seems to have some problems with this macro, use it only on SUSE
@@ -364,7 +367,7 @@ fi
%endif
# Package removal cleanup
if [ "$1" -eq "0" ] ; then
- rm -rf /var/log/ceph
+ rm -rf /var/log/ceph
rm -rf /etc/ceph
fi
@@ -391,6 +394,7 @@ fi
%{_bindir}/ceph-authtool
%{_bindir}/ceph-syn
%{_bindir}/ceph-post-file
+%{_bindir}/ceph-crush-location
%{_bindir}/ceph-run
%{_bindir}/ceph-mon
%{_bindir}/ceph-mds
@@ -412,6 +416,7 @@ fi
%{_sbindir}/rcceph
/sbin/mkcephfs
/sbin/mount.ceph
+%{_libdir}/ceph
%dir %{_libdir}/rados-classes
%{_libdir}/rados-classes/libcls_rbd.so*
%{_libdir}/rados-classes/libcls_hello.so*
@@ -423,7 +428,13 @@ fi
%{_libdir}/rados-classes/libcls_replica_log.so*
%{_libdir}/rados-classes/libcls_statelog.so*
%{_libdir}/rados-classes/libcls_version.so*
-%{_libdir}/ceph
+%dir %{_libdir}/ceph/erasure-code
+%{_libdir}/ceph/erasure-code/libec_example.so*
+%{_libdir}/ceph/erasure-code/libec_fail_to_initialize.so*
+%{_libdir}/ceph/erasure-code/libec_fail_to_register.so*
+%{_libdir}/ceph/erasure-code/libec_hangs.so*
+%{_libdir}/ceph/erasure-code/libec_jerasure.so*
+%{_libdir}/ceph/erasure-code/libec_missing_entry_point.so*
/lib/udev/rules.d/50-rbd.rules
/lib/udev/rules.d/60-ceph-partuuid-workaround.rules
/lib/udev/rules.d/95-ceph-osd.rules
@@ -497,6 +508,7 @@ fi
%{_includedir}/rados/crc32c.h
%{_includedir}/rados/rados_types.h
%{_includedir}/rados/rados_types.hpp
+%{_includedir}/rados/memory.h
%dir %{_includedir}/rbd
%{_includedir}/rbd/librbd.h
%{_includedir}/rbd/librbd.hpp
@@ -515,6 +527,7 @@ fi
%{_mandir}/man8/radosgw.8*
%{_mandir}/man8/radosgw-admin.8*
%{_sbindir}/rcceph-radosgw
+%dir %{_localstatedir}/log/radosgw/
%post radosgw
/sbin/ldconfig
@@ -533,6 +546,11 @@ fi
%restart_on_update ceph-radosgw
%insserv_cleanup
%endif
+# Package removal cleanup
+if [ "$1" -eq "0" ] ; then
+ rm -rf /var/log/radosgw
+fi
+
#################################################################################
%if %{with ocf}
@@ -598,6 +616,7 @@ fi
%{_bindir}/ceph_dupstore
%{_bindir}/ceph_kvstorebench
%{_bindir}/ceph_multi_stress_watch
+%{_bindir}/ceph_erasure_code_benchmark
%{_bindir}/ceph_omapbench
%{_bindir}/ceph_psim
%{_bindir}/ceph_radosacl
@@ -612,6 +631,7 @@ fi
%{_bindir}/ceph_filestore_dump
%{_bindir}/ceph_filestore_tool
%{_bindir}/ceph_streamtest
+%{_bindir}/ceph_test_c_headers
%{_bindir}/ceph_test_cfuse_cache_invalidate
%{_bindir}/ceph_test_cls_hello
%{_bindir}/ceph_test_cls_lock
@@ -631,6 +651,7 @@ fi
%{_bindir}/ceph_test_filestore_idempotent
%{_bindir}/ceph_test_filestore_idempotent_sequence
%{_bindir}/ceph_test_filestore_workloadgen
+%{_bindir}/ceph_test_get_blkdev_size
%{_bindir}/ceph_test_ioctls
%{_bindir}/ceph_test_keyvaluedb_atomicity
%{_bindir}/ceph_test_keyvaluedb_iterators
@@ -648,6 +669,7 @@ fi
%{_bindir}/ceph_test_rados_api_list
%{_bindir}/ceph_test_rados_api_lock
%{_bindir}/ceph_test_rados_api_misc
+%{_bindir}/ceph_test_rados_api_tier
%{_bindir}/ceph_test_rados_api_pool
%{_bindir}/ceph_test_rados_api_snapshots
%{_bindir}/ceph_test_rados_api_stat
diff --git a/configure b/configure
index f6b1ab2..2f27917 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.68 for ceph 0.72.2.
+# Generated by GNU Autoconf 2.68 for ceph 0.77.
#
# Report bugs to <ceph-devel at vger.kernel.org>.
#
@@ -570,8 +570,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='ceph'
PACKAGE_TARNAME='ceph'
-PACKAGE_VERSION='0.72.2'
-PACKAGE_STRING='ceph 0.72.2'
+PACKAGE_VERSION='0.77'
+PACKAGE_STRING='ceph 0.77'
PACKAGE_BUGREPORT='ceph-devel at vger.kernel.org'
PACKAGE_URL=''
@@ -627,6 +627,7 @@ PYTHON_VERSION
PYTHON
WITH_BUILD_TESTS_FALSE
WITH_BUILD_TESTS_TRUE
+BOOST_PROGRAM_OPTIONS_LIBS
USE_BOOST_SPIRIT_OLD_HDR_FALSE
USE_BOOST_SPIRIT_OLD_HDR_TRUE
WITH_LIBZFS_FALSE
@@ -678,6 +679,8 @@ CRYPTOPP_CFLAGS
PKG_CONFIG_LIBDIR
PKG_CONFIG_PATH
PKG_CONFIG
+KEYUTILS_LIB
+RESOLV_LIBS
PTHREAD_CFLAGS
PTHREAD_LIBS
PTHREAD_CC
@@ -721,8 +724,6 @@ NMEDIT
DSYMUTIL
MANIFEST_TOOL
RANLIB
-ac_ct_AR
-AR
DLLTOOL
OBJDUMP
LN_S
@@ -745,13 +746,6 @@ AMDEP_TRUE
am__quote
am__include
DEPDIR
-OBJEXT
-EXEEXT
-ac_ct_CC
-CPPFLAGS
-LDFLAGS
-CFLAGS
-CC
am__untar
am__tar
AMTAR
@@ -775,6 +769,15 @@ am__isrc
INSTALL_DATA
INSTALL_SCRIPT
INSTALL_PROGRAM
+OBJEXT
+EXEEXT
+ac_ct_CC
+CPPFLAGS
+LDFLAGS
+CFLAGS
+CC
+ac_ct_AR
+AR
target_os
target_vendor
target_cpu
@@ -1427,7 +1430,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures ceph 0.72.2 to adapt to many kinds of systems.
+\`configure' configures ceph 0.77 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1498,7 +1501,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of ceph 0.72.2:";;
+ short | recursive ) echo "Configuration of ceph 0.77:";;
esac
cat <<\_ACEOF
@@ -1642,7 +1645,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-ceph configure 0.72.2
+ceph configure 0.77
generated by GNU Autoconf 2.68
Copyright (C) 2010 Free Software Foundation, Inc.
@@ -2276,11 +2279,65 @@ $as_echo "$ac_res" >&6; }
eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
} # ac_fn_c_check_member
+
+# ac_fn_c_check_type LINENO TYPE VAR INCLUDES
+# -------------------------------------------
+# Tests whether TYPE exists after having included INCLUDES, setting cache
+# variable VAR accordingly.
+ac_fn_c_check_type ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+$as_echo_n "checking for $2... " >&6; }
+if eval \${$3+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ eval "$3=no"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+int
+main ()
+{
+if (sizeof ($2))
+ return 0;
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+int
+main ()
+{
+if (sizeof (($2)))
+ return 0;
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+else
+ eval "$3=yes"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+ eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_type
cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by ceph $as_me 0.72.2, which was
+It was created by ceph $as_me 0.77, which was
generated by GNU Autoconf 2.68. Invocation command line was
$ $0 $@
@@ -2825,208 +2882,118 @@ test -n "$target_alias" &&
NONENONEs,x,x, &&
program_prefix=${target_alias}-
-# Automake
-am__api_version='1.11'
+# Fix automake problems in 1.12
+# expand $ac_aux_dir to an absolute path
+am_aux_dir=`cd $ac_aux_dir && pwd`
-# Find a good install program. We prefer a C program (faster),
-# so one script is as good as another. But avoid the broken or
-# incompatible versions:
-# SysV /etc/install, /usr/sbin/install
-# SunOS /usr/etc/install
-# IRIX /sbin/install
-# AIX /bin/install
-# AmigaOS /C/install, which installs bootblocks on floppy discs
-# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
-# AFS /usr/afsws/bin/install, which mishandles nonexistent args
-# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
-# OS/2's system install, which has a completely different semantic
-# ./install, which can be erroneously created by make from ./install.sh.
-# Reject install programs that cannot install multiple files.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5
-$as_echo_n "checking for a BSD-compatible install... " >&6; }
-if test -z "$INSTALL"; then
-if ${ac_cv_path_install+:} false; then :
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_CC+:} false; then :
$as_echo_n "(cached) " >&6
else
- as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
- # Account for people who put trailing slashes in PATH elements.
-case $as_dir/ in #((
- ./ | .// | /[cC]/* | \
- /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
- ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \
- /usr/ucb/* ) ;;
- *)
- # OSF1 and SCO ODT 3.0 have their own names for install.
- # Don't use installbsd from OSF since it installs stuff as root
- # by default.
- for ac_prog in ginstall scoinst install; do
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then
- if test $ac_prog = install &&
- grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
- # AIX install. It has an incompatible calling convention.
- :
- elif test $ac_prog = install &&
- grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
- # program-specific install script used by HP pwplus--don't use.
- :
- else
- rm -rf conftest.one conftest.two conftest.dir
- echo one > conftest.one
- echo two > conftest.two
- mkdir conftest.dir
- if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" &&
- test -s conftest.one && test -s conftest.two &&
- test -s conftest.dir/conftest.one &&
- test -s conftest.dir/conftest.two
- then
- ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
- break 3
- fi
- fi
- fi
- done
- done
- ;;
-esac
-
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_CC="${ac_tool_prefix}gcc"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
done
IFS=$as_save_IFS
-rm -rf conftest.one conftest.two conftest.dir
-
fi
- if test "${ac_cv_path_install+set}" = set; then
- INSTALL=$ac_cv_path_install
- else
- # As a last resort, use the slow shell script. Don't cache a
- # value for INSTALL within a source directory, because that will
- # break other packages using the cache if that directory is
- # removed, or if the value is a relative name.
- INSTALL=$ac_install_sh
- fi
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5
-$as_echo "$INSTALL" >&6; }
-
-# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
-# It thinks the first close brace ends the variable substitution.
-test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
-
-test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
-
-test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5
-$as_echo_n "checking whether build environment is sane... " >&6; }
-# Just in case
-sleep 1
-echo timestamp > conftest.file
-# Reject unsafe characters in $srcdir or the absolute working directory
-# name. Accept space and tab only in the latter.
-am_lf='
-'
-case `pwd` in
- *[\\\"\#\$\&\'\`$am_lf]*)
- as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;;
-esac
-case $srcdir in
- *[\\\"\#\$\&\'\`$am_lf\ \ ]*)
- as_fn_error $? "unsafe srcdir value: \`$srcdir'" "$LINENO" 5;;
-esac
-
-# Do `set' in a subshell so we don't clobber the current shell's
-# arguments. Must try -L first in case configure is actually a
-# symlink; some systems play weird games with the mod time of symlinks
-# (eg FreeBSD returns the mod time of the symlink's containing
-# directory).
-if (
- set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null`
- if test "$*" = "X"; then
- # -L didn't work.
- set X `ls -t "$srcdir/configure" conftest.file`
- fi
- rm -f conftest.file
- if test "$*" != "X $srcdir/configure conftest.file" \
- && test "$*" != "X conftest.file $srcdir/configure"; then
-
- # If neither matched, then we have a broken ls. This can happen
- # if, for instance, CONFIG_SHELL is bash and it inherits a
- # broken ls alias from the environment. This has actually
- # happened. Such a system could not be considered "sane".
- as_fn_error $? "ls -t appears to fail. Make sure there is not a broken
-alias in your environment" "$LINENO" 5
- fi
-
- test "$2" = conftest.file
- )
-then
- # Ok.
- :
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
else
- as_fn_error $? "newly created file is older than distributed files!
-Check your system clock" "$LINENO" 5
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-test "$program_prefix" != NONE &&
- program_transform_name="s&^&$program_prefix&;$program_transform_name"
-# Use a double $ so make ignores it.
-test "$program_suffix" != NONE &&
- program_transform_name="s&\$&$program_suffix&;$program_transform_name"
-# Double any \ or $.
-# By default was `s,x,x', remove it if useless.
-ac_script='s/[\\$]/&&/g;s/;s,x,x,$//'
-program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"`
-# expand $ac_aux_dir to an absolute path
-am_aux_dir=`cd $ac_aux_dir && pwd`
-if test x"${MISSING+set}" != xset; then
- case $am_aux_dir in
- *\ * | *\ *)
- MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;;
- *)
- MISSING="\${SHELL} $am_aux_dir/missing" ;;
- esac
fi
-# Use eval to expand $SHELL
-if eval "$MISSING --run true"; then
- am_missing_run="$MISSING --run "
+if test -z "$ac_cv_prog_CC"; then
+ ac_ct_CC=$CC
+ # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_ac_ct_CC+:} false; then :
+ $as_echo_n "(cached) " >&6
else
- am_missing_run=
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`missing' script is too old or missing" >&5
-$as_echo "$as_me: WARNING: \`missing' script is too old or missing" >&2;}
+ if test -n "$ac_ct_CC"; then
+ ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_CC="gcc"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+$as_echo "$ac_ct_CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-if test x"${install_sh}" != xset; then
- case $am_aux_dir in
- *\ * | *\ *)
- install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;;
- *)
- install_sh="\${SHELL} $am_aux_dir/install-sh"
- esac
+ if test "x$ac_ct_CC" = x; then
+ CC=""
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ CC=$ac_ct_CC
+ fi
+else
+ CC="$ac_cv_prog_CC"
fi
-# Installed binaries are usually stripped using `strip' when the user
-# run `make install-strip'. However `strip' might not be the right
-# tool to use in cross-compilation environments, therefore Automake
-# will honor the `STRIP' environment variable to overrule this program.
-if test "$cross_compiling" != no; then
- if test -n "$ac_tool_prefix"; then
- # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args.
-set dummy ${ac_tool_prefix}strip; ac_word=$2
+if test -z "$CC"; then
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_STRIP+:} false; then :
+if ${ac_cv_prog_CC+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$STRIP"; then
- ac_cv_prog_STRIP="$STRIP" # Let the user override the test.
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
else
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
@@ -3035,7 +3002,7 @@ do
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_STRIP="${ac_tool_prefix}strip"
+ ac_cv_prog_CC="${ac_tool_prefix}cc"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
@@ -3045,29 +3012,30 @@ IFS=$as_save_IFS
fi
fi
-STRIP=$ac_cv_prog_STRIP
-if test -n "$STRIP"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5
-$as_echo "$STRIP" >&6; }
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
+ fi
fi
-if test -z "$ac_cv_prog_STRIP"; then
- ac_ct_STRIP=$STRIP
- # Extract the first word of "strip", so it can be a program name with args.
-set dummy strip; ac_word=$2
+if test -z "$CC"; then
+ # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_ac_ct_STRIP+:} false; then :
+if ${ac_cv_prog_CC+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$ac_ct_STRIP"; then
- ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test.
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
else
+ ac_prog_rejected=no
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
@@ -3075,7 +3043,11 @@ do
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_ac_ct_STRIP="strip"
+ if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+ ac_prog_rejected=yes
+ continue
+ fi
+ ac_cv_prog_CC="cc"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
@@ -3083,94 +3055,88 @@ done
done
IFS=$as_save_IFS
+if test $ac_prog_rejected = yes; then
+ # We found a bogon in the path, so make sure we never use it.
+ set dummy $ac_cv_prog_CC
+ shift
+ if test $# != 0; then
+ # We chose a different compiler from the bogus one.
+ # However, it has the same basename, so the bogon will be chosen
+ # first if we set CC to just the basename; use the full file name.
+ shift
+ ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+ fi
fi
fi
-ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP
-if test -n "$ac_ct_STRIP"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5
-$as_echo "$ac_ct_STRIP" >&6; }
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
- if test "x$ac_ct_STRIP" = x; then
- STRIP=":"
- else
- case $cross_compiling:$ac_tool_warned in
-yes:)
-{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
-$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
-ac_tool_warned=yes ;;
-esac
- STRIP=$ac_ct_STRIP
- fi
-else
- STRIP="$ac_cv_prog_STRIP"
-fi
fi
-INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s"
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a thread-safe mkdir -p" >&5
-$as_echo_n "checking for a thread-safe mkdir -p... " >&6; }
-if test -z "$MKDIR_P"; then
- if ${ac_cv_path_mkdir+:} false; then :
+if test -z "$CC"; then
+ if test -n "$ac_tool_prefix"; then
+ for ac_prog in cl.exe
+ do
+ # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_CC+:} false; then :
$as_echo_n "(cached) " >&6
else
- as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
- for ac_prog in mkdir gmkdir; do
- for ac_exec_ext in '' $ac_executable_extensions; do
- { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; } || continue
- case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
- 'mkdir (GNU coreutils) '* | \
- 'mkdir (coreutils) '* | \
- 'mkdir (fileutils) '4.1*)
- ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext
- break 3;;
- esac
- done
- done
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
done
IFS=$as_save_IFS
fi
-
- test -d ./--version && rmdir ./--version
- if test "${ac_cv_path_mkdir+set}" = set; then
- MKDIR_P="$ac_cv_path_mkdir -p"
- else
- # As a last resort, use the slow shell script. Don't cache a
- # value for MKDIR_P within a source directory, because that will
- # break other packages using the cache if that directory is
- # removed, or if the value is a relative name.
- MKDIR_P="$ac_install_sh -d"
- fi
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5
-$as_echo "$MKDIR_P" >&6; }
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
-mkdir_p="$MKDIR_P"
-case $mkdir_p in
- [\\/$]* | ?:[\\/]*) ;;
- */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
-esac
-for ac_prog in gawk mawk nawk awk
+ test -n "$CC" && break
+ done
+fi
+if test -z "$CC"; then
+ ac_ct_CC=$CC
+ for ac_prog in cl.exe
do
# Extract the first word of "$ac_prog", so it can be a program name with args.
set dummy $ac_prog; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_AWK+:} false; then :
+if ${ac_cv_prog_ac_ct_CC+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$AWK"; then
- ac_cv_prog_AWK="$AWK" # Let the user override the test.
+ if test -n "$ac_ct_CC"; then
+ ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
else
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
@@ -3179,7 +3145,7 @@ do
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_AWK="$ac_prog"
+ ac_cv_prog_ac_ct_CC="$ac_prog"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
@@ -3189,199 +3155,540 @@ IFS=$as_save_IFS
fi
fi
-AWK=$ac_cv_prog_AWK
-if test -n "$AWK"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5
-$as_echo "$AWK" >&6; }
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+$as_echo "$ac_ct_CC" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
- test -n "$AWK" && break
+ test -n "$ac_ct_CC" && break
done
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} sets \$(MAKE)" >&5
-$as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; }
-set x ${MAKE-make}
-ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'`
-if eval \${ac_cv_prog_make_${ac_make}_set+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat >conftest.make <<\_ACEOF
-SHELL = /bin/sh
-all:
- @echo '@@@%%%=$(MAKE)=@@@%%%'
-_ACEOF
-# GNU make sometimes prints "make[1]: Entering ...", which would confuse us.
-case `${MAKE-make} -f conftest.make 2>/dev/null` in
- *@@@%%%=?*=@@@%%%*)
- eval ac_cv_prog_make_${ac_make}_set=yes;;
- *)
- eval ac_cv_prog_make_${ac_make}_set=no;;
+ if test "x$ac_ct_CC" = x; then
+ CC=""
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
esac
-rm -f conftest.make
-fi
-if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
- SET_MAKE=
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
- SET_MAKE="MAKE=${MAKE-make}"
+ CC=$ac_ct_CC
+ fi
fi
-rm -rf .tst 2>/dev/null
-mkdir .tst 2>/dev/null
-if test -d .tst; then
- am__leading_dot=.
-else
- am__leading_dot=_
fi
-rmdir .tst 2>/dev/null
-if test "`cd $srcdir && pwd`" != "`pwd`"; then
- # Use -I$(srcdir) only when $(srcdir) != ., so that make's output
- # is not polluted with repeated "-I."
- am__isrc=' -I$(srcdir)'
- # test to see if srcdir already configured
- if test -f $srcdir/config.status; then
- as_fn_error $? "source directory already configured; run \"make distclean\" there first" "$LINENO" 5
- fi
-fi
-# test whether we have cygpath
-if test -z "$CYGPATH_W"; then
- if (cygpath --version) >/dev/null 2>/dev/null; then
- CYGPATH_W='cygpath -w'
- else
- CYGPATH_W=echo
- fi
-fi
+test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "no acceptable C compiler found in \$PATH
+See \`config.log' for more details" "$LINENO" 5; }
+# Provide some information about the compiler.
+$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5
+set X $ac_compile
+ac_compiler=$2
+for ac_option in --version -v -V -qversion; do
+ { { ac_try="$ac_compiler $ac_option >&5"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_compiler $ac_option >&5") 2>conftest.err
+ ac_status=$?
+ if test -s conftest.err; then
+ sed '10a\
+... rest of stderr output deleted ...
+ 10q' conftest.err >conftest.er1
+ cat conftest.er1 >&5
+ fi
+ rm -f conftest.er1 conftest.err
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+done
-# Define the identity of the package.
- PACKAGE='ceph'
- VERSION='0.72.2'
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+int
+main ()
+{
-cat >>confdefs.h <<_ACEOF
-#define PACKAGE "$PACKAGE"
+ ;
+ return 0;
+}
_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5
+$as_echo_n "checking whether the C compiler works... " >&6; }
+ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+# The possible output files:
+ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*"
-cat >>confdefs.h <<_ACEOF
-#define VERSION "$VERSION"
-_ACEOF
+ac_rmfiles=
+for ac_file in $ac_files
+do
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
+ * ) ac_rmfiles="$ac_rmfiles $ac_file";;
+ esac
+done
+rm -f $ac_rmfiles
-# Some tools Automake needs.
+if { { ac_try="$ac_link_default"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link_default") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then :
+ # Autoconf-2.13 could set the ac_cv_exeext variable to `no'.
+# So ignore a value of `no', otherwise this would lead to `EXEEXT = no'
+# in a Makefile. We should not override ac_cv_exeext if it was cached,
+# so that the user can short-circuit this test for compilers unknown to
+# Autoconf.
+for ac_file in $ac_files ''
+do
+ test -f "$ac_file" || continue
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj )
+ ;;
+ [ab].out )
+ # We found the default executable, but exeext='' is most
+ # certainly right.
+ break;;
+ *.* )
+ if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no;
+ then :; else
+ ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+ fi
+ # We set ac_cv_exeext here because the later test for it is not
+ # safe: cross compilers may not add the suffix if given an `-o'
+ # argument, so we may need to know it at that point already.
+ # Even if this section looks crufty: it has the advantage of
+ # actually working.
+ break;;
+ * )
+ break;;
+ esac
+done
+test "$ac_cv_exeext" = no && ac_cv_exeext=
-ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"}
+else
+ ac_file=''
+fi
+if test -z "$ac_file"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+$as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error 77 "C compiler cannot create executables
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5
+$as_echo_n "checking for C compiler default output file name... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5
+$as_echo "$ac_file" >&6; }
+ac_exeext=$ac_cv_exeext
-AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"}
+rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5
+$as_echo_n "checking for suffix of executables... " >&6; }
+if { { ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then :
+ # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+ test -f "$ac_file" || continue
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
+ *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+ break;;
+ * ) break;;
+ esac
+done
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+rm -f conftest conftest$ac_cv_exeext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5
+$as_echo "$ac_cv_exeext" >&6; }
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdio.h>
+int
+main ()
+{
+FILE *f = fopen ("conftest.out", "w");
+ return ferror (f) || fclose (f) != 0;
-AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"}
+ ;
+ return 0;
+}
+_ACEOF
+ac_clean_files="$ac_clean_files conftest.out"
+# Check that the compiler produces executables we can run. If not, either
+# the compiler is broken, or we cross compile.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5
+$as_echo_n "checking whether we are cross compiling... " >&6; }
+if test "$cross_compiling" != yes; then
+ { { ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+ if { ac_try='./conftest$ac_cv_exeext'
+ { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }; then
+ cross_compiling=no
+ else
+ if test "$cross_compiling" = maybe; then
+ cross_compiling=yes
+ else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5
+$as_echo "$cross_compiling" >&6; }
+rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out
+ac_clean_files=$ac_clean_files_save
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5
+$as_echo_n "checking for suffix of object files... " >&6; }
+if ${ac_cv_objext+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
-AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"}
+int
+main ()
+{
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { { ac_try="$ac_compile"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_compile") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then :
+ for ac_file in conftest.o conftest.obj conftest.*; do
+ test -f "$ac_file" || continue;
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;;
+ *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+ break;;
+ esac
+done
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
-MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"}
+{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot compute suffix of object files: cannot compile
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5
+$as_echo "$ac_cv_objext" >&6; }
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5
+$as_echo_n "checking whether we are using the GNU C compiler... " >&6; }
+if ${ac_cv_c_compiler_gnu+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
-# We need awk for the "check" target. The system "awk" is bad on
-# some platforms.
-# Always define AMTAR for backward compatibility. Yes, it's still used
-# in the wild :-( We should find a proper way to deprecate it ...
-AMTAR='$${TAR-tar}'
+int
+main ()
+{
+#ifndef __GNUC__
+ choke me
+#endif
-am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -'
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_compiler_gnu=yes
+else
+ ac_compiler_gnu=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5
+$as_echo "$ac_cv_c_compiler_gnu" >&6; }
+if test $ac_compiler_gnu = yes; then
+ GCC=yes
+else
+ GCC=
+fi
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5
+$as_echo_n "checking whether $CC accepts -g... " >&6; }
+if ${ac_cv_prog_cc_g+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_save_c_werror_flag=$ac_c_werror_flag
+ ac_c_werror_flag=yes
+ ac_cv_prog_cc_g=no
+ CFLAGS="-g"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+int
+main ()
+{
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_g=yes
+else
+ CFLAGS=""
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+int
+main ()
+{
-DEPDIR="${am__leading_dot}deps"
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
-ac_config_commands="$ac_config_commands depfiles"
+else
+ ac_c_werror_flag=$ac_save_c_werror_flag
+ CFLAGS="-g"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+int
+main ()
+{
-am_make=${MAKE-make}
-cat > confinc << 'END'
-am__doit:
- @echo this is the am__doit target
-.PHONY: am__doit
-END
-# If we don't find an include directive, just comment out the code.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for style of include used by $am_make" >&5
-$as_echo_n "checking for style of include used by $am_make... " >&6; }
-am__include="#"
-am__quote=
-_am_result=none
-# First try GNU make style include.
-echo "include confinc" > confmf
-# Ignore all kinds of additional output from `make'.
-case `$am_make -s -f confmf 2> /dev/null` in #(
-*the\ am__doit\ target*)
- am__include=include
- am__quote=
- _am_result=GNU
- ;;
-esac
-# Now try BSD make style include.
-if test "$am__include" = "#"; then
- echo '.include "confinc"' > confmf
- case `$am_make -s -f confmf 2> /dev/null` in #(
- *the\ am__doit\ target*)
- am__include=.include
- am__quote="\""
- _am_result=BSD
- ;;
- esac
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_g=yes
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ ac_c_werror_flag=$ac_save_c_werror_flag
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5
+$as_echo "$ac_cv_prog_cc_g" >&6; }
+if test "$ac_test_CFLAGS" = set; then
+ CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+ if test "$GCC" = yes; then
+ CFLAGS="-g -O2"
+ else
+ CFLAGS="-g"
+ fi
+else
+ if test "$GCC" = yes; then
+ CFLAGS="-O2"
+ else
+ CFLAGS=
+ fi
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5
+$as_echo_n "checking for $CC option to accept ISO C89... " >&6; }
+if ${ac_cv_prog_cc_c89+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_cv_prog_cc_c89=no
+ac_save_CC=$CC
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+ char **p;
+ int i;
+{
+ return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+ char *s;
+ va_list v;
+ va_start (v,p);
+ s = g (p, va_arg (v,int));
+ va_end (v);
+ return s;
+}
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has
+ function prototypes and stuff, but not '\xHH' hex character constants.
+ These don't provoke an error unfortunately, instead are silently treated
+ as 'x'. The following induces an error, until -std is added to get
+ proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an
+ array size at least. It's necessary to write '\x00'==0 to get something
+ that's true only with -std. */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $_am_result" >&5
-$as_echo "$_am_result" >&6; }
-rm -f confinc confmf
+/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters
+ inside strings and character constants. */
+#define FOO(x) 'x'
+int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1];
-# Check whether --enable-dependency-tracking was given.
-if test "${enable_dependency_tracking+set}" = set; then :
- enableval=$enable_dependency_tracking;
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1];
+ ;
+ return 0;
+}
+_ACEOF
+for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \
+ -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+ CC="$ac_save_CC $ac_arg"
+ if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_c89=$ac_arg
fi
+rm -f core conftest.err conftest.$ac_objext
+ test "x$ac_cv_prog_cc_c89" != "xno" && break
+done
+rm -f conftest.$ac_ext
+CC=$ac_save_CC
-if test "x$enable_dependency_tracking" != xno; then
- am_depcomp="$ac_aux_dir/depcomp"
- AMDEPBACKSLASH='\'
- am__nodep='_no'
-fi
- if test "x$enable_dependency_tracking" != xno; then
- AMDEP_TRUE=
- AMDEP_FALSE='#'
-else
- AMDEP_TRUE='#'
- AMDEP_FALSE=
fi
+# AC_CACHE_VAL
+case "x$ac_cv_prog_cc_c89" in
+ x)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
+$as_echo "none needed" >&6; } ;;
+ xno)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
+$as_echo "unsupported" >&6; } ;;
+ *)
+ CC="$CC $ac_cv_prog_cc_c89"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5
+$as_echo "$ac_cv_prog_cc_c89" >&6; } ;;
+esac
+if test "x$ac_cv_prog_cc_c89" != xno; then :
+fi
ac_ext=c
ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
if test -n "$ac_tool_prefix"; then
- # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
-set dummy ${ac_tool_prefix}gcc; ac_word=$2
+ for ac_prog in ar lib "link -lib"
+ do
+ # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_CC+:} false; then :
+if ${ac_cv_prog_AR+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$CC"; then
- ac_cv_prog_CC="$CC" # Let the user override the test.
+ if test -n "$AR"; then
+ ac_cv_prog_AR="$AR" # Let the user override the test.
else
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
@@ -3390,7 +3697,7 @@ do
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_CC="${ac_tool_prefix}gcc"
+ ac_cv_prog_AR="$ac_tool_prefix$ac_prog"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
@@ -3400,28 +3707,32 @@ IFS=$as_save_IFS
fi
fi
-CC=$ac_cv_prog_CC
-if test -n "$CC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
-$as_echo "$CC" >&6; }
+AR=$ac_cv_prog_AR
+if test -n "$AR"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5
+$as_echo "$AR" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
+ test -n "$AR" && break
+ done
fi
-if test -z "$ac_cv_prog_CC"; then
- ac_ct_CC=$CC
- # Extract the first word of "gcc", so it can be a program name with args.
-set dummy gcc; ac_word=$2
+if test -z "$AR"; then
+ ac_ct_AR=$AR
+ for ac_prog in ar lib "link -lib"
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_ac_ct_CC+:} false; then :
+if ${ac_cv_prog_ac_ct_AR+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$ac_ct_CC"; then
- ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+ if test -n "$ac_ct_AR"; then
+ ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test.
else
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
@@ -3430,7 +3741,7 @@ do
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_ac_ct_CC="gcc"
+ ac_cv_prog_ac_ct_AR="$ac_prog"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
@@ -3440,17 +3751,21 @@ IFS=$as_save_IFS
fi
fi
-ac_ct_CC=$ac_cv_prog_ac_ct_CC
-if test -n "$ac_ct_CC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
-$as_echo "$ac_ct_CC" >&6; }
+ac_ct_AR=$ac_cv_prog_ac_ct_AR
+if test -n "$ac_ct_AR"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_AR" >&5
+$as_echo "$ac_ct_AR" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
- if test "x$ac_ct_CC" = x; then
- CC=""
+
+ test -n "$ac_ct_AR" && break
+done
+
+ if test "x$ac_ct_AR" = x; then
+ AR="false"
else
case $cross_compiling:$ac_tool_warned in
yes:)
@@ -3458,166 +3773,270 @@ yes:)
$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
ac_tool_warned=yes ;;
esac
- CC=$ac_ct_CC
+ AR=$ac_ct_AR
fi
-else
- CC="$ac_cv_prog_CC"
fi
-if test -z "$CC"; then
- if test -n "$ac_tool_prefix"; then
- # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
-set dummy ${ac_tool_prefix}cc; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_CC+:} false; then :
+: ${AR=ar}
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the archiver ($AR) interface" >&5
+$as_echo_n "checking the archiver ($AR) interface... " >&6; }
+if ${am_cv_ar_interface+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$CC"; then
- ac_cv_prog_CC="$CC" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_CC="${ac_tool_prefix}cc"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
- done
-IFS=$as_save_IFS
+ am_cv_ar_interface=ar
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+int some_variable = 0;
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ am_ar_try='$AR cru libconftest.a conftest.$ac_objext >&5'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$am_ar_try\""; } >&5
+ (eval $am_ar_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+ if test "$ac_status" -eq 0; then
+ am_cv_ar_interface=ar
+ else
+ am_ar_try='$AR -NOLOGO -OUT:conftest.lib conftest.$ac_objext >&5'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$am_ar_try\""; } >&5
+ (eval $am_ar_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+ if test "$ac_status" -eq 0; then
+ am_cv_ar_interface=lib
+ else
+ am_cv_ar_interface=unknown
+ fi
+ fi
+ rm -f conftest.lib libconftest.a
fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
fi
-CC=$ac_cv_prog_CC
-if test -n "$CC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
-$as_echo "$CC" >&6; }
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_ar_interface" >&5
+$as_echo "$am_cv_ar_interface" >&6; }
+case $am_cv_ar_interface in
+ar)
+ ;;
+lib)
+ # Microsoft lib, so override with the ar-lib wrapper script.
+ # FIXME: It is wrong to rewrite AR.
+ # But if we don't then we get into trouble of one sort or another.
+ # A longer-term fix would be to have automake use am__AR in this case,
+ # and then we could set am__AR="$am_aux_dir/ar-lib \$(AR)" or something
+ # similar.
+ AR="$am_aux_dir/ar-lib $AR"
+ ;;
+unknown)
+ as_fn_error $? "could not determine $AR interface" "$LINENO" 5
+ ;;
+esac
- fi
-fi
-if test -z "$CC"; then
- # Extract the first word of "cc", so it can be a program name with args.
-set dummy cc; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_CC+:} false; then :
+
+# Automake
+am__api_version='1.11'
+
+# Find a good install program. We prefer a C program (faster),
+# so one script is as good as another. But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+# Reject install programs that cannot install multiple files.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5
+$as_echo_n "checking for a BSD-compatible install... " >&6; }
+if test -z "$INSTALL"; then
+if ${ac_cv_path_install+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$CC"; then
- ac_cv_prog_CC="$CC" # Let the user override the test.
-else
- ac_prog_rejected=no
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
- ac_prog_rejected=yes
- continue
- fi
- ac_cv_prog_CC="cc"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
+ # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in #((
+ ./ | .// | /[cC]/* | \
+ /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+ ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \
+ /usr/ucb/* ) ;;
+ *)
+ # OSF1 and SCO ODT 3.0 have their own names for install.
+ # Don't use installbsd from OSF since it installs stuff as root
+ # by default.
+ for ac_prog in ginstall scoinst install; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then
+ if test $ac_prog = install &&
+ grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+ # AIX install. It has an incompatible calling convention.
+ :
+ elif test $ac_prog = install &&
+ grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+ # program-specific install script used by HP pwplus--don't use.
+ :
+ else
+ rm -rf conftest.one conftest.two conftest.dir
+ echo one > conftest.one
+ echo two > conftest.two
+ mkdir conftest.dir
+ if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" &&
+ test -s conftest.one && test -s conftest.two &&
+ test -s conftest.dir/conftest.one &&
+ test -s conftest.dir/conftest.two
+ then
+ ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+ break 3
+ fi
+ fi
+ fi
+ done
+ done
+ ;;
+esac
+
done
IFS=$as_save_IFS
-if test $ac_prog_rejected = yes; then
- # We found a bogon in the path, so make sure we never use it.
- set dummy $ac_cv_prog_CC
- shift
- if test $# != 0; then
- # We chose a different compiler from the bogus one.
- # However, it has the same basename, so the bogon will be chosen
- # first if we set CC to just the basename; use the full file name.
- shift
- ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
- fi
-fi
-fi
+rm -rf conftest.one conftest.two conftest.dir
+
fi
-CC=$ac_cv_prog_CC
-if test -n "$CC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
-$as_echo "$CC" >&6; }
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ if test "${ac_cv_path_install+set}" = set; then
+ INSTALL=$ac_cv_path_install
+ else
+ # As a last resort, use the slow shell script. Don't cache a
+ # value for INSTALL within a source directory, because that will
+ # break other packages using the cache if that directory is
+ # removed, or if the value is a relative name.
+ INSTALL=$ac_install_sh
+ fi
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5
+$as_echo "$INSTALL" >&6; }
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5
+$as_echo_n "checking whether build environment is sane... " >&6; }
+# Just in case
+sleep 1
+echo timestamp > conftest.file
+# Reject unsafe characters in $srcdir or the absolute working directory
+# name. Accept space and tab only in the latter.
+am_lf='
+'
+case `pwd` in
+ *[\\\"\#\$\&\'\`$am_lf]*)
+ as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;;
+esac
+case $srcdir in
+ *[\\\"\#\$\&\'\`$am_lf\ \ ]*)
+ as_fn_error $? "unsafe srcdir value: \`$srcdir'" "$LINENO" 5;;
+esac
+
+# Do `set' in a subshell so we don't clobber the current shell's
+# arguments. Must try -L first in case configure is actually a
+# symlink; some systems play weird games with the mod time of symlinks
+# (eg FreeBSD returns the mod time of the symlink's containing
+# directory).
+if (
+ set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null`
+ if test "$*" = "X"; then
+ # -L didn't work.
+ set X `ls -t "$srcdir/configure" conftest.file`
+ fi
+ rm -f conftest.file
+ if test "$*" != "X $srcdir/configure conftest.file" \
+ && test "$*" != "X conftest.file $srcdir/configure"; then
+ # If neither matched, then we have a broken ls. This can happen
+ # if, for instance, CONFIG_SHELL is bash and it inherits a
+ # broken ls alias from the environment. This has actually
+ # happened. Such a system could not be considered "sane".
+ as_fn_error $? "ls -t appears to fail. Make sure there is not a broken
+alias in your environment" "$LINENO" 5
+ fi
-fi
-if test -z "$CC"; then
- if test -n "$ac_tool_prefix"; then
- for ac_prog in cl.exe
- do
- # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
-set dummy $ac_tool_prefix$ac_prog; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_CC+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- if test -n "$CC"; then
- ac_cv_prog_CC="$CC" # Let the user override the test.
+ test "$2" = conftest.file
+ )
+then
+ # Ok.
+ :
else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
- done
-IFS=$as_save_IFS
-
+ as_fn_error $? "newly created file is older than distributed files!
+Check your system clock" "$LINENO" 5
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+test "$program_prefix" != NONE &&
+ program_transform_name="s&^&$program_prefix&;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+ program_transform_name="s&\$&$program_suffix&;$program_transform_name"
+# Double any \ or $.
+# By default was `s,x,x', remove it if useless.
+ac_script='s/[\\$]/&&/g;s/;s,x,x,$//'
+program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"`
+
+if test x"${MISSING+set}" != xset; then
+ case $am_aux_dir in
+ *\ * | *\ *)
+ MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;;
+ *)
+ MISSING="\${SHELL} $am_aux_dir/missing" ;;
+ esac
fi
-CC=$ac_cv_prog_CC
-if test -n "$CC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
-$as_echo "$CC" >&6; }
+# Use eval to expand $SHELL
+if eval "$MISSING --run true"; then
+ am_missing_run="$MISSING --run "
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ am_missing_run=
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`missing' script is too old or missing" >&5
+$as_echo "$as_me: WARNING: \`missing' script is too old or missing" >&2;}
fi
-
- test -n "$CC" && break
- done
+if test x"${install_sh}" != xset; then
+ case $am_aux_dir in
+ *\ * | *\ *)
+ install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;;
+ *)
+ install_sh="\${SHELL} $am_aux_dir/install-sh"
+ esac
fi
-if test -z "$CC"; then
- ac_ct_CC=$CC
- for ac_prog in cl.exe
-do
- # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
+
+# Installed binaries are usually stripped using `strip' when the user
+# run `make install-strip'. However `strip' might not be the right
+# tool to use in cross-compilation environments, therefore Automake
+# will honor the `STRIP' environment variable to overrule this program.
+if test "$cross_compiling" != no; then
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args.
+set dummy ${ac_tool_prefix}strip; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_ac_ct_CC+:} false; then :
+if ${ac_cv_prog_STRIP+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$ac_ct_CC"; then
- ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+ if test -n "$STRIP"; then
+ ac_cv_prog_STRIP="$STRIP" # Let the user override the test.
else
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
@@ -3626,536 +4045,336 @@ do
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_ac_ct_CC="$ac_prog"
+ ac_cv_prog_STRIP="${ac_tool_prefix}strip"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
- done
-IFS=$as_save_IFS
-
-fi
-fi
-ac_ct_CC=$ac_cv_prog_ac_ct_CC
-if test -n "$ac_ct_CC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
-$as_echo "$ac_ct_CC" >&6; }
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
-
-
- test -n "$ac_ct_CC" && break
-done
-
- if test "x$ac_ct_CC" = x; then
- CC=""
- else
- case $cross_compiling:$ac_tool_warned in
-yes:)
-{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
-$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
-ac_tool_warned=yes ;;
-esac
- CC=$ac_ct_CC
- fi
-fi
-
-fi
-
-
-test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "no acceptable C compiler found in \$PATH
-See \`config.log' for more details" "$LINENO" 5; }
-
-# Provide some information about the compiler.
-$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5
-set X $ac_compile
-ac_compiler=$2
-for ac_option in --version -v -V -qversion; do
- { { ac_try="$ac_compiler $ac_option >&5"
-case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
- (eval "$ac_compiler $ac_option >&5") 2>conftest.err
- ac_status=$?
- if test -s conftest.err; then
- sed '10a\
-... rest of stderr output deleted ...
- 10q' conftest.err >conftest.er1
- cat conftest.er1 >&5
- fi
- rm -f conftest.er1 conftest.err
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }
-done
-
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-ac_clean_files_save=$ac_clean_files
-ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out"
-# Try to create an executable without -o first, disregard a.out.
-# It will help us diagnose broken compilers, and finding out an intuition
-# of exeext.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5
-$as_echo_n "checking whether the C compiler works... " >&6; }
-ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
-
-# The possible output files:
-ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*"
-
-ac_rmfiles=
-for ac_file in $ac_files
-do
- case $ac_file in
- *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
- * ) ac_rmfiles="$ac_rmfiles $ac_file";;
- esac
-done
-rm -f $ac_rmfiles
-
-if { { ac_try="$ac_link_default"
-case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
- (eval "$ac_link_default") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; then :
- # Autoconf-2.13 could set the ac_cv_exeext variable to `no'.
-# So ignore a value of `no', otherwise this would lead to `EXEEXT = no'
-# in a Makefile. We should not override ac_cv_exeext if it was cached,
-# so that the user can short-circuit this test for compilers unknown to
-# Autoconf.
-for ac_file in $ac_files ''
-do
- test -f "$ac_file" || continue
- case $ac_file in
- *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj )
- ;;
- [ab].out )
- # We found the default executable, but exeext='' is most
- # certainly right.
- break;;
- *.* )
- if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no;
- then :; else
- ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
- fi
- # We set ac_cv_exeext here because the later test for it is not
- # safe: cross compilers may not add the suffix if given an `-o'
- # argument, so we may need to know it at that point already.
- # Even if this section looks crufty: it has the advantage of
- # actually working.
- break;;
- * )
- break;;
- esac
-done
-test "$ac_cv_exeext" = no && ac_cv_exeext=
+ done
+IFS=$as_save_IFS
-else
- ac_file=''
fi
-if test -z "$ac_file"; then :
+fi
+STRIP=$ac_cv_prog_STRIP
+if test -n "$STRIP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5
+$as_echo "$STRIP" >&6; }
+else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
-$as_echo "$as_me: failed program was:" >&5
-sed 's/^/| /' conftest.$ac_ext >&5
-
-{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error 77 "C compiler cannot create executables
-See \`config.log' for more details" "$LINENO" 5; }
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5
-$as_echo_n "checking for C compiler default output file name... " >&6; }
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5
-$as_echo "$ac_file" >&6; }
-ac_exeext=$ac_cv_exeext
-rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out
-ac_clean_files=$ac_clean_files_save
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5
-$as_echo_n "checking for suffix of executables... " >&6; }
-if { { ac_try="$ac_link"
-case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
- (eval "$ac_link") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; then :
- # If both `conftest.exe' and `conftest' are `present' (well, observable)
-# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will
-# work properly (i.e., refer to `conftest.exe'), while it won't with
-# `rm'.
-for ac_file in conftest.exe conftest conftest.*; do
- test -f "$ac_file" || continue
- case $ac_file in
- *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
- *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
- break;;
- * ) break;;
- esac
+
+fi
+if test -z "$ac_cv_prog_STRIP"; then
+ ac_ct_STRIP=$STRIP
+ # Extract the first word of "strip", so it can be a program name with args.
+set dummy strip; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_ac_ct_STRIP+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_STRIP"; then
+ ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_STRIP="strip"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP
+if test -n "$ac_ct_STRIP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5
+$as_echo "$ac_ct_STRIP" >&6; }
else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "cannot compute suffix of executables: cannot compile and link
-See \`config.log' for more details" "$LINENO" 5; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-rm -f conftest conftest$ac_cv_exeext
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5
-$as_echo "$ac_cv_exeext" >&6; }
-rm -f conftest.$ac_ext
-EXEEXT=$ac_cv_exeext
-ac_exeext=$EXEEXT
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <stdio.h>
-int
-main ()
-{
-FILE *f = fopen ("conftest.out", "w");
- return ferror (f) || fclose (f) != 0;
-
- ;
- return 0;
-}
-_ACEOF
-ac_clean_files="$ac_clean_files conftest.out"
-# Check that the compiler produces executables we can run. If not, either
-# the compiler is broken, or we cross compile.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5
-$as_echo_n "checking whether we are cross compiling... " >&6; }
-if test "$cross_compiling" != yes; then
- { { ac_try="$ac_link"
-case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
- (eval "$ac_link") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }
- if { ac_try='./conftest$ac_cv_exeext'
- { { case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
- (eval "$ac_try") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; }; then
- cross_compiling=no
+ if test "x$ac_ct_STRIP" = x; then
+ STRIP=":"
else
- if test "$cross_compiling" = maybe; then
- cross_compiling=yes
- else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "cannot run C compiled programs.
-If you meant to cross compile, use \`--host'.
-See \`config.log' for more details" "$LINENO" 5; }
- fi
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ STRIP=$ac_ct_STRIP
fi
+else
+ STRIP="$ac_cv_prog_STRIP"
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5
-$as_echo "$cross_compiling" >&6; }
-rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out
-ac_clean_files=$ac_clean_files_save
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5
-$as_echo_n "checking for suffix of object files... " >&6; }
-if ${ac_cv_objext+:} false; then :
+fi
+INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s"
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a thread-safe mkdir -p" >&5
+$as_echo_n "checking for a thread-safe mkdir -p... " >&6; }
+if test -z "$MKDIR_P"; then
+ if ${ac_cv_path_mkdir+:} false; then :
$as_echo_n "(cached) " >&6
else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_prog in mkdir gmkdir; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; } || continue
+ case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
+ 'mkdir (GNU coreutils) '* | \
+ 'mkdir (coreutils) '* | \
+ 'mkdir (fileutils) '4.1*)
+ ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext
+ break 3;;
+ esac
+ done
+ done
+ done
+IFS=$as_save_IFS
-int
-main ()
-{
+fi
- ;
- return 0;
-}
-_ACEOF
-rm -f conftest.o conftest.obj
-if { { ac_try="$ac_compile"
-case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
+ test -d ./--version && rmdir ./--version
+ if test "${ac_cv_path_mkdir+set}" = set; then
+ MKDIR_P="$ac_cv_path_mkdir -p"
+ else
+ # As a last resort, use the slow shell script. Don't cache a
+ # value for MKDIR_P within a source directory, because that will
+ # break other packages using the cache if that directory is
+ # removed, or if the value is a relative name.
+ MKDIR_P="$ac_install_sh -d"
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5
+$as_echo "$MKDIR_P" >&6; }
+
+mkdir_p="$MKDIR_P"
+case $mkdir_p in
+ [\\/$]* | ?:[\\/]*) ;;
+ */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
- (eval "$ac_compile") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; then :
- for ac_file in conftest.o conftest.obj conftest.*; do
- test -f "$ac_file" || continue;
- case $ac_file in
- *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;;
- *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
- break;;
- esac
+
+for ac_prog in gawk mawk nawk awk
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_AWK+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$AWK"; then
+ ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_AWK="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
done
-else
- $as_echo "$as_me: failed program was:" >&5
-sed 's/^/| /' conftest.$ac_ext >&5
+ done
+IFS=$as_save_IFS
-{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "cannot compute suffix of object files: cannot compile
-See \`config.log' for more details" "$LINENO" 5; }
fi
-rm -f conftest.$ac_cv_objext conftest.$ac_ext
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5
-$as_echo "$ac_cv_objext" >&6; }
-OBJEXT=$ac_cv_objext
-ac_objext=$OBJEXT
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5
-$as_echo_n "checking whether we are using the GNU C compiler... " >&6; }
-if ${ac_cv_c_compiler_gnu+:} false; then :
- $as_echo_n "(cached) " >&6
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5
+$as_echo "$AWK" >&6; }
else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
-int
-main ()
-{
-#ifndef __GNUC__
- choke me
-#endif
- ;
- return 0;
-}
+ test -n "$AWK" && break
+done
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+$as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; }
+set x ${MAKE-make}
+ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'`
+if eval \${ac_cv_prog_make_${ac_make}_set+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat >conftest.make <<\_ACEOF
+SHELL = /bin/sh
+all:
+ @echo '@@@%%%=$(MAKE)=@@@%%%'
_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- ac_compiler_gnu=yes
+# GNU make sometimes prints "make[1]: Entering ...", which would confuse us.
+case `${MAKE-make} -f conftest.make 2>/dev/null` in
+ *@@@%%%=?*=@@@%%%*)
+ eval ac_cv_prog_make_${ac_make}_set=yes;;
+ *)
+ eval ac_cv_prog_make_${ac_make}_set=no;;
+esac
+rm -f conftest.make
+fi
+if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ SET_MAKE=
else
- ac_compiler_gnu=no
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ SET_MAKE="MAKE=${MAKE-make}"
fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-ac_cv_c_compiler_gnu=$ac_compiler_gnu
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5
-$as_echo "$ac_cv_c_compiler_gnu" >&6; }
-if test $ac_compiler_gnu = yes; then
- GCC=yes
+rm -rf .tst 2>/dev/null
+mkdir .tst 2>/dev/null
+if test -d .tst; then
+ am__leading_dot=.
else
- GCC=
+ am__leading_dot=_
fi
-ac_test_CFLAGS=${CFLAGS+set}
-ac_save_CFLAGS=$CFLAGS
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5
-$as_echo_n "checking whether $CC accepts -g... " >&6; }
-if ${ac_cv_prog_cc_g+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_save_c_werror_flag=$ac_c_werror_flag
- ac_c_werror_flag=yes
- ac_cv_prog_cc_g=no
- CFLAGS="-g"
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+rmdir .tst 2>/dev/null
-int
-main ()
-{
+DEPDIR="${am__leading_dot}deps"
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- ac_cv_prog_cc_g=yes
-else
- CFLAGS=""
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+ac_config_commands="$ac_config_commands depfiles"
-int
-main ()
-{
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+am_make=${MAKE-make}
+cat > confinc << 'END'
+am__doit:
+ @echo this is the am__doit target
+.PHONY: am__doit
+END
+# If we don't find an include directive, just comment out the code.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for style of include used by $am_make" >&5
+$as_echo_n "checking for style of include used by $am_make... " >&6; }
+am__include="#"
+am__quote=
+_am_result=none
+# First try GNU make style include.
+echo "include confinc" > confmf
+# Ignore all kinds of additional output from `make'.
+case `$am_make -s -f confmf 2> /dev/null` in #(
+*the\ am__doit\ target*)
+ am__include=include
+ am__quote=
+ _am_result=GNU
+ ;;
+esac
+# Now try BSD make style include.
+if test "$am__include" = "#"; then
+ echo '.include "confinc"' > confmf
+ case `$am_make -s -f confmf 2> /dev/null` in #(
+ *the\ am__doit\ target*)
+ am__include=.include
+ am__quote="\""
+ _am_result=BSD
+ ;;
+ esac
+fi
-else
- ac_c_werror_flag=$ac_save_c_werror_flag
- CFLAGS="-g"
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-int
-main ()
-{
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $_am_result" >&5
+$as_echo "$_am_result" >&6; }
+rm -f confinc confmf
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- ac_cv_prog_cc_g=yes
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+# Check whether --enable-dependency-tracking was given.
+if test "${enable_dependency_tracking+set}" = set; then :
+ enableval=$enable_dependency_tracking;
fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test "x$enable_dependency_tracking" != xno; then
+ am_depcomp="$ac_aux_dir/depcomp"
+ AMDEPBACKSLASH='\'
+ am__nodep='_no'
fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
- ac_c_werror_flag=$ac_save_c_werror_flag
+ if test "x$enable_dependency_tracking" != xno; then
+ AMDEP_TRUE=
+ AMDEP_FALSE='#'
+else
+ AMDEP_TRUE='#'
+ AMDEP_FALSE=
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5
-$as_echo "$ac_cv_prog_cc_g" >&6; }
-if test "$ac_test_CFLAGS" = set; then
- CFLAGS=$ac_save_CFLAGS
-elif test $ac_cv_prog_cc_g = yes; then
- if test "$GCC" = yes; then
- CFLAGS="-g -O2"
- else
- CFLAGS="-g"
+
+
+if test "`cd $srcdir && pwd`" != "`pwd`"; then
+ # Use -I$(srcdir) only when $(srcdir) != ., so that make's output
+ # is not polluted with repeated "-I."
+ am__isrc=' -I$(srcdir)'
+ # test to see if srcdir already configured
+ if test -f $srcdir/config.status; then
+ as_fn_error $? "source directory already configured; run \"make distclean\" there first" "$LINENO" 5
fi
-else
- if test "$GCC" = yes; then
- CFLAGS="-O2"
+fi
+
+# test whether we have cygpath
+if test -z "$CYGPATH_W"; then
+ if (cygpath --version) >/dev/null 2>/dev/null; then
+ CYGPATH_W='cygpath -w'
else
- CFLAGS=
+ CYGPATH_W=echo
fi
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5
-$as_echo_n "checking for $CC option to accept ISO C89... " >&6; }
-if ${ac_cv_prog_cc_c89+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_cv_prog_cc_c89=no
-ac_save_CC=$CC
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <stdarg.h>
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */
-struct buf { int x; };
-FILE * (*rcsopen) (struct buf *, struct stat *, int);
-static char *e (p, i)
- char **p;
- int i;
-{
- return p[i];
-}
-static char *f (char * (*g) (char **, int), char **p, ...)
-{
- char *s;
- va_list v;
- va_start (v,p);
- s = g (p, va_arg (v,int));
- va_end (v);
- return s;
-}
-/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has
- function prototypes and stuff, but not '\xHH' hex character constants.
- These don't provoke an error unfortunately, instead are silently treated
- as 'x'. The following induces an error, until -std is added to get
- proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an
- array size at least. It's necessary to write '\x00'==0 to get something
- that's true only with -std. */
-int osf4_cc_array ['\x00' == 0 ? 1 : -1];
-/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters
- inside strings and character constants. */
-#define FOO(x) 'x'
-int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1];
+# Define the identity of the package.
+ PACKAGE='ceph'
+ VERSION='0.77'
-int test (int i, double x);
-struct s1 {int (*f) (int a);};
-struct s2 {int (*f) (double a);};
-int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
-int argc;
-char **argv;
-int
-main ()
-{
-return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1];
- ;
- return 0;
-}
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
_ACEOF
-for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \
- -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
-do
- CC="$ac_save_CC $ac_arg"
- if ac_fn_c_try_compile "$LINENO"; then :
- ac_cv_prog_cc_c89=$ac_arg
-fi
-rm -f core conftest.err conftest.$ac_objext
- test "x$ac_cv_prog_cc_c89" != "xno" && break
-done
-rm -f conftest.$ac_ext
-CC=$ac_save_CC
-fi
-# AC_CACHE_VAL
-case "x$ac_cv_prog_cc_c89" in
- x)
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
-$as_echo "none needed" >&6; } ;;
- xno)
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
-$as_echo "unsupported" >&6; } ;;
- *)
- CC="$CC $ac_cv_prog_cc_c89"
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5
-$as_echo "$ac_cv_prog_cc_c89" >&6; } ;;
-esac
-if test "x$ac_cv_prog_cc_c89" != xno; then :
-fi
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+# Some tools Automake needs.
+
+ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"}
+
+
+AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"}
+
+
+AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"}
+
+
+AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"}
+
+
+MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"}
+
+# We need awk for the "check" target. The system "awk" is bad on
+# some platforms.
+# Always define AMTAR for backward compatibility. Yes, it's still used
+# in the wild :-( We should find a proper way to deprecate it ...
+AMTAR='$${TAR-tar}'
+
+am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -'
+
+
-ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
depcc="$CC" am_compiler_list=
@@ -4285,6 +4504,7 @@ else
fi
+
if test "x$CC" != xcc; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC and cc understand -c and -o together" >&5
$as_echo_n "checking whether $CC and cc understand -c and -o together... " >&6; }
@@ -5930,7 +6150,6 @@ test -z "$sharedlib_from_linklib_cmd" && sharedlib_from_linklib_cmd=$ECHO
-
if test -n "$ac_tool_prefix"; then
for ac_prog in ar
do
@@ -12096,7 +12315,7 @@ fi
# Define the identity of the package.
PACKAGE='ceph'
- VERSION='0.72.2'
+ VERSION='0.77'
cat >>confdefs.h <<_ACEOF
@@ -16067,7 +16286,33 @@ if test "$CXX" = no || test "$CXX:$GXX" = "g++:"; then
as_fn_error $? "no C++ compiler found" "$LINENO" 5
fi
- if test x"$CXX" = x"clang++"; then
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if compiler is clang" >&5
+$as_echo_n "checking if compiler is clang... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+#ifndef __clang__
+#error "Not Clang"
+#endif
+return 0;
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ CLANG=yes
+else
+ CLANG=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CLANG" >&5
+$as_echo "$CLANG" >&6; }
+ if test "$CLANG" = "yes"; then
CLANG_TRUE=
CLANG_FALSE='#'
else
@@ -16279,6 +16524,118 @@ else
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports static_cast<>" >&5
+$as_echo_n "checking whether the compiler supports static_cast<>... " >&6; }
+if ${ax_cv_cxx_static_cast+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+
+ ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <typeinfo>
+class Base { public : Base () {} virtual void f () = 0; };
+class Derived : public Base { public : Derived () {} virtual void f () {} };
+int g (Derived&) { return 0; }
+int
+main ()
+{
+
+Derived d; Base& b = d; Derived& s = static_cast<Derived&> (b); return g (s);
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+ ax_cv_cxx_static_cast=yes
+else
+ ax_cv_cxx_static_cast=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_cxx_static_cast" >&5
+$as_echo "$ax_cv_cxx_static_cast" >&6; }
+if test "$ax_cv_cxx_static_cast" = yes; then
+
+$as_echo "#define HAVE_STATIC_CAST /**/" >>confdefs.h
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC recognizes __func__" >&5
+$as_echo_n "checking whether $CC recognizes __func__... " >&6; }
+if ${ac_cv_c_var_func+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+char *s = __func__;
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+$as_echo "#define HAVE_FUNC /**/" >>confdefs.h
+ ac_cv_c_var_func=yes
+else
+ ac_cv_c_var_func=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_var_func" >&5
+$as_echo "$ac_cv_c_var_func" >&6; }
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC recognizes __PRETTY_FUNCTION__" >&5
+$as_echo_n "checking whether $CC recognizes __PRETTY_FUNCTION__... " >&6; }
+if ${ac_cv_c_pretty_func+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+char *s = __PRETTY_FUNCTION__;
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+$as_echo "#define HAVE_PRETTY_FUNC /**/" >>confdefs.h
+ ac_cv_c_pretty_func=yes
+else
+ ac_cv_c_pretty_func=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_pretty_func" >&5
+$as_echo "$ac_cv_c_pretty_func" >&6; }
+
+
# Checks for libraries.
@@ -16873,6 +17230,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for uuid_parse in -luuid" >&5
$as_echo_n "checking for uuid_parse in -luuid... " >&6; }
if ${ac_cv_lib_uuid_uuid_parse+:} false; then :
@@ -16918,58 +17276,155 @@ as_fn_error $? "libuuid not found
See \`config.log' for more details" "$LINENO" 5; }
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pow in -lm" >&5
-$as_echo_n "checking for pow in -lm... " >&6; }
-if ${ac_cv_lib_m_pow+:} false; then :
+
+if test x"$linux" = x"yes"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for blkid_devno_to_wholedisk in -lblkid" >&5
+$as_echo_n "checking for blkid_devno_to_wholedisk in -lblkid... " >&6; }
+if ${ac_cv_lib_blkid_blkid_devno_to_wholedisk+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
-LIBS="-lm $LIBS"
+LIBS="-lblkid $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char pow ();
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char blkid_devno_to_wholedisk ();
+int
+main ()
+{
+return blkid_devno_to_wholedisk ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_blkid_blkid_devno_to_wholedisk=yes
+else
+ ac_cv_lib_blkid_blkid_devno_to_wholedisk=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_blkid_blkid_devno_to_wholedisk" >&5
+$as_echo "$ac_cv_lib_blkid_blkid_devno_to_wholedisk" >&6; }
+if test "x$ac_cv_lib_blkid_blkid_devno_to_wholedisk" = xyes; then :
+ true
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "libblkid not found
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+
+fi
+
+#
+# Check for res_nquery in libresolv. There are several variations. On OSX
+# res_nquery is a macro defined in resolv.h, so the typical AC_CHECK_LIB
+# doesn't work. On FreeBSD res_nquery can be found in libc. The required
+# library for linking (if any) is defined RESOLV_LIBS.
+#
+ac_fn_c_check_header_compile "$LINENO" "resolv.h" "ac_cv_header_resolv_h" "#include <netinet/in.h>
+"
+if test "x$ac_cv_header_resolv_h" = xyes; then :
+
+fi
+
+
+
+
+
+RESOLV_LIBS=""
+{
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if res_nquery will link (LIBS=$RESOLV_LIBS)" >&5
+$as_echo_n "checking if res_nquery will link (LIBS=$RESOLV_LIBS)... " >&6; }
+ saved_LIBS="${LIBS}"
+ LIBS="$RESOLV_LIBS"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+ #include <netinet/in.h>
+ #include <resolv.h>
+
+int
+main ()
+{
+res_nquery(0, 0, 0, 0, 0, 0);
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ resolv_libs="ok"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ LIBS="${saved_LIBS}"
+}
+if test x"$resolv_libs" != "xok"; then
+ RESOLV_LIBS="-lresolv"
+ {
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if res_nquery will link (LIBS=$RESOLV_LIBS)" >&5
+$as_echo_n "checking if res_nquery will link (LIBS=$RESOLV_LIBS)... " >&6; }
+ saved_LIBS="${LIBS}"
+ LIBS="$RESOLV_LIBS"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+ #include <netinet/in.h>
+ #include <resolv.h>
+
int
main ()
{
-return pow ();
+res_nquery(0, 0, 0, 0, 0, 0);
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_m_pow=yes
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ resolv_libs="ok"
else
- ac_cv_lib_m_pow=no
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_m_pow" >&5
-$as_echo "$ac_cv_lib_m_pow" >&6; }
-if test "x$ac_cv_lib_m_pow" = xyes; then :
- true
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+ LIBS="${saved_LIBS}"
+}
+ if test x"$resolv_libs" != "xok"; then
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libm not found
+as_fn_error $? "no resolv library found
See \`config.log' for more details" "$LINENO" 5; }
+ fi
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __res_nquery in -lresolv" >&5
-$as_echo_n "checking for __res_nquery in -lresolv... " >&6; }
-if ${ac_cv_lib_resolv___res_nquery+:} false; then :
+
+KEYUTILS_LIB=""
+if test x"$linux" = x"yes"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for add_key in -lkeyutils" >&5
+$as_echo_n "checking for add_key in -lkeyutils... " >&6; }
+if ${ac_cv_lib_keyutils_add_key+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
-LIBS="-lresolv $LIBS"
+LIBS="-lkeyutils $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
@@ -16979,43 +17434,46 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char __res_nquery ();
+char add_key ();
int
main ()
{
-return __res_nquery ();
+return add_key ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_resolv___res_nquery=yes
+ ac_cv_lib_keyutils_add_key=yes
else
- ac_cv_lib_resolv___res_nquery=no
+ ac_cv_lib_keyutils_add_key=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv___res_nquery" >&5
-$as_echo "$ac_cv_lib_resolv___res_nquery" >&6; }
-if test "x$ac_cv_lib_resolv___res_nquery" = xyes; then :
- true
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_keyutils_add_key" >&5
+$as_echo "$ac_cv_lib_keyutils_add_key" >&6; }
+if test "x$ac_cv_lib_keyutils_add_key" = xyes; then :
+ KEYUTILS_LIB="-lkeyutils"
else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libresolv not found
+as_fn_error $? "libkeyutils not found
See \`config.log' for more details" "$LINENO" 5; }
fi
-if test x"$linux" = x"yes"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for add_key in -lkeyutils" >&5
-$as_echo_n "checking for add_key in -lkeyutils... " >&6; }
-if ${ac_cv_lib_keyutils_add_key+:} false; then :
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pow in -lm" >&5
+$as_echo_n "checking for pow in -lm... " >&6; }
+if ${ac_cv_lib_m_pow+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
-LIBS="-lkeyutils $LIBS"
+LIBS="-lm $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
@@ -17025,36 +17483,35 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char add_key ();
+char pow ();
int
main ()
{
-return add_key ();
+return pow ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_keyutils_add_key=yes
+ ac_cv_lib_m_pow=yes
else
- ac_cv_lib_keyutils_add_key=no
+ ac_cv_lib_m_pow=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_keyutils_add_key" >&5
-$as_echo "$ac_cv_lib_keyutils_add_key" >&6; }
-if test "x$ac_cv_lib_keyutils_add_key" = xyes; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_m_pow" >&5
+$as_echo "$ac_cv_lib_m_pow" >&6; }
+if test "x$ac_cv_lib_m_pow" = xyes; then :
true
else
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libkeyutils not found
+as_fn_error $? "libm not found
See \`config.log' for more details" "$LINENO" 5; }
fi
-fi
for ac_func in syncfs
do :
ac_fn_c_check_func "$LINENO" "syncfs" "ac_cv_func_syncfs"
@@ -17743,7 +18200,193 @@ if ${ac_cv_lib_expat_XML_Parse+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
-LIBS="-lexpat $LIBS"
+LIBS="-lexpat $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char XML_Parse ();
+int
+main ()
+{
+return XML_Parse ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_expat_XML_Parse=yes
+else
+ ac_cv_lib_expat_XML_Parse=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_expat_XML_Parse" >&5
+$as_echo "$ac_cv_lib_expat_XML_Parse" >&6; }
+if test "x$ac_cv_lib_expat_XML_Parse" = xyes; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_easy_init in -lcurl" >&5
+$as_echo_n "checking for curl_easy_init in -lcurl... " >&6; }
+if ${ac_cv_lib_curl_curl_easy_init+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char curl_easy_init ();
+int
+main ()
+{
+return curl_easy_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_curl_curl_easy_init=yes
+else
+ ac_cv_lib_curl_curl_easy_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_init" >&5
+$as_echo "$ac_cv_lib_curl_curl_easy_init" >&6; }
+if test "x$ac_cv_lib_curl_curl_easy_init" = xyes; then :
+ RADOSGW=1
+ ac_fn_c_check_header_mongrel "$LINENO" "fastcgi/fcgiapp.h" "ac_cv_header_fastcgi_fcgiapp_h" "$ac_includes_default"
+if test "x$ac_cv_header_fastcgi_fcgiapp_h" = xyes; then :
+
+$as_echo "#define FASTCGI_INCLUDE_DIR 1" >>confdefs.h
+
+fi
+
+
+
+else
+ if test "x$with_radosgw" != "xcheck"; then
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "--with-radosgw was given but libcurl (libcurl-dev on debian) not found
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+fi
+
+
+else
+ if test "x$with_radosgw" != "xcheck"; then
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "--with-radosgw was given but libexpat (libexpat1-dev on debian) not found
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+fi
+
+
+else
+ if test "x$with_radosgw" != "xcheck"; then
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "--with-radosgw was given but libfcgi (libfcgi-dev on debian) not found
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+fi
+
+fi
+ if test "$RADOSGW" = "1"; then
+ WITH_RADOSGW_TRUE=
+ WITH_RADOSGW_FALSE='#'
+else
+ WITH_RADOSGW_TRUE='#'
+ WITH_RADOSGW_FALSE=
+fi
+
+
+if test "$RADOSGW" = "1"; then :
+
+$as_echo "#define WITH_RADOSGW 1" >>confdefs.h
+
+fi
+
+if test "$RADOSGW" = "1"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_multi_wait in -lcurl" >&5
+$as_echo_n "checking for curl_multi_wait in -lcurl... " >&6; }
+if ${ac_cv_lib_curl_curl_multi_wait+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char curl_multi_wait ();
+int
+main ()
+{
+return curl_multi_wait ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_curl_curl_multi_wait=yes
+else
+ ac_cv_lib_curl_curl_multi_wait=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_multi_wait" >&5
+$as_echo "$ac_cv_lib_curl_curl_multi_wait" >&6; }
+if test "x$ac_cv_lib_curl_curl_multi_wait" = xyes; then :
+
+$as_echo "#define HAVE_CURL_MULTI_WAIT 1" >>confdefs.h
+
+fi
+
+
+fi
+
+# fuse?
+
+# Check whether --with-fuse was given.
+if test "${with_fuse+set}" = set; then :
+ withval=$with_fuse;
+else
+ with_fuse=yes
+fi
+
+LIBFUSE=
+if test "x$with_fuse" != xno; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for fuse_main in -lfuse" >&5
+$as_echo_n "checking for fuse_main in -lfuse... " >&6; }
+if ${ac_cv_lib_fuse_fuse_main+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lfuse $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
@@ -17753,34 +18396,84 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char XML_Parse ();
+char fuse_main ();
int
main ()
{
-return XML_Parse ();
+return fuse_main ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_expat_XML_Parse=yes
+ ac_cv_lib_fuse_fuse_main=yes
else
- ac_cv_lib_expat_XML_Parse=no
+ ac_cv_lib_fuse_fuse_main=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_expat_XML_Parse" >&5
-$as_echo "$ac_cv_lib_expat_XML_Parse" >&6; }
-if test "x$ac_cv_lib_expat_XML_Parse" = xyes; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_easy_init in -lcurl" >&5
-$as_echo_n "checking for curl_easy_init in -lcurl... " >&6; }
-if ${ac_cv_lib_curl_curl_easy_init+:} false; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_fuse_fuse_main" >&5
+$as_echo "$ac_cv_lib_fuse_fuse_main" >&6; }
+if test "x$ac_cv_lib_fuse_fuse_main" = xyes; then :
+ LIBFUSE="-lfuse"
+
+
+$as_echo "#define HAVE_LIBFUSE 1" >>confdefs.h
+
+ HAVE_LIBFUSE=1
+ # look for fuse_getgroups and define FUSE_GETGROUPS if found
+ LIBS_saved="$LIBS"
+ LIBS="$LIBS -lfuse"
+ for ac_func in fuse_getgroups
+do :
+ ac_fn_c_check_func "$LINENO" "fuse_getgroups" "ac_cv_func_fuse_getgroups"
+if test "x$ac_cv_func_fuse_getgroups" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_FUSE_GETGROUPS 1
+_ACEOF
+
+fi
+done
+
+ LIBS="$LIBS_saved"
+
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "no FUSE found (use --without-fuse to disable)
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+
+fi
+ if test "$HAVE_LIBFUSE" = "1"; then
+ WITH_FUSE_TRUE=
+ WITH_FUSE_FALSE='#'
+else
+ WITH_FUSE_TRUE='#'
+ WITH_FUSE_FALSE=
+fi
+
+
+# tcmalloc?
+
+# Check whether --with-tcmalloc was given.
+if test "${with_tcmalloc+set}" = set; then :
+ withval=$with_tcmalloc;
+else
+ with_tcmalloc=yes
+fi
+
+TCMALLOC=
+if test "x$with_tcmalloc" != xno; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for malloc in -ltcmalloc" >&5
+$as_echo_n "checking for malloc in -ltcmalloc... " >&6; }
+if ${ac_cv_lib_tcmalloc_malloc+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
-LIBS="-lcurl $LIBS"
+LIBS="-ltcmalloc $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
@@ -17790,740 +18483,1004 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char curl_easy_init ();
+char malloc ();
int
main ()
{
-return curl_easy_init ();
+return malloc ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_curl_curl_easy_init=yes
+ ac_cv_lib_tcmalloc_malloc=yes
else
- ac_cv_lib_curl_curl_easy_init=no
+ ac_cv_lib_tcmalloc_malloc=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_init" >&5
-$as_echo "$ac_cv_lib_curl_curl_easy_init" >&6; }
-if test "x$ac_cv_lib_curl_curl_easy_init" = xyes; then :
- RADOSGW=1
- ac_fn_c_check_header_mongrel "$LINENO" "fastcgi/fcgiapp.h" "ac_cv_header_fastcgi_fcgiapp_h" "$ac_includes_default"
-if test "x$ac_cv_header_fastcgi_fcgiapp_h" = xyes; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_tcmalloc_malloc" >&5
+$as_echo "$ac_cv_lib_tcmalloc_malloc" >&6; }
+if test "x$ac_cv_lib_tcmalloc_malloc" = xyes; then :
+ LIBTCMALLOC="-ltcmalloc"
-$as_echo "#define FASTCGI_INCLUDE_DIR 1" >>confdefs.h
+$as_echo "#define HAVE_LIBTCMALLOC 1" >>confdefs.h
+
+ HAVE_LIBTCMALLOC=1
+
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "no tcmalloc found (use --without-tcmalloc to disable)
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+
+fi
+ if test "$HAVE_LIBTCMALLOC" = "1"; then
+ WITH_TCMALLOC_TRUE=
+ WITH_TCMALLOC_FALSE='#'
+else
+ WITH_TCMALLOC_TRUE='#'
+ WITH_TCMALLOC_FALSE=
+fi
+
+
+#set pg ref debugging?
+# Check whether --enable-pgrefdebugging was given.
+if test "${enable_pgrefdebugging+set}" = set; then :
+ enableval=$enable_pgrefdebugging;
+$as_echo "#define PG_DEBUG_REFS 1" >>confdefs.h
+
+fi
+
+
+#
+# Java is painful
+# - adapted from OMPI wrappers package
+# - this might become bigger. maybe should be own m4 file
+#
+# Check whether --enable-cephfs-java was given.
+if test "${enable_cephfs_java+set}" = set; then :
+ enableval=$enable_cephfs_java;
+else
+ enable_cephfs_java=no
+fi
+
+
+ if test "x$enable_cephfs_java" = "xyes"; then
+ ENABLE_CEPHFS_JAVA_TRUE=
+ ENABLE_CEPHFS_JAVA_FALSE='#'
+else
+ ENABLE_CEPHFS_JAVA_TRUE='#'
+ ENABLE_CEPHFS_JAVA_FALSE=
+fi
+
+
+
+# Check whether --with-jdk-dir was given.
+if test "${with_jdk_dir+set}" = set; then :
+ withval=$with_jdk_dir;
+fi
+
+
+if test "x$enable_cephfs_java" = "xyes"; then
+
+ # setup bin/include dirs from --with-jdk-dir (search for jni.h, javac)
+ if test -n "$with_jdk_dir"; then :
+
+ javac_prog=`find $with_jdk_dir/ -name javac | head -n 1`
+ if test -x "$javac_prog"; then :
+
+ EXTRA_JDK_BIN_DIR=`dirname $javac_prog`
+fi
+ jnih=`find $with_jdk_dir/ -name jni.h | head -n 1`
+ if test -r "$jnih"; then :
+
+ EXTRA_JDK_INC_DIR=`dirname $jnih`
+fi
+fi
+
+ # setup defaults for Debian default-jdk package (without --with-jdk-dir)
+ if test -z "$with_jdk_dir"; then :
+
+ # This works with Debian's and CentOS' default-jdk package
+ for dir in '/usr/lib/jvm/default-java/' '/usr/lib/jvm/java/' ; do
+ # only test if a suitable path has not yet been found
+ if test "$EXTRA_JDK_BIN_DIR" == ""; then :
+
+ if test -x "$javac_prog"; then :
+
+ EXTRA_JDK_BIN_DIR=`dirname $javac_prog`
+fi
+ jnih=`find $dir -name jni.h | head -n 1`
+ if test -r "$jnih"; then :
+
+ EXTRA_JDK_INC_DIR=`dirname $jnih`
+fi
+
+fi
+ done
+
+fi
+
+ # cephfs_java_test only makes sense if java is already turned on
+ # setup CLASSPATH for Debian default junit4.jar package
+ #
+ # Configuring --with-debug and --enable-cephfs-java will throw an error if
+ # JUnit4 cannot be found. While currently this works for users who have
+ # installed via the package manager on Ubuntu, we need to expand this
+ # check to 1 support other distrubtions and 2 allow users to influence
+ # the search path.
+ if test "x$with_debug" = "xyes"; then :
+
+ dir='/usr/share/java'
+ junit4_jar=`find $dir -name junit4.jar | head -n 1`
+ if test -r "$junit4_jar"; then :
+
+ EXTRA_CLASSPATH_JAR=`dirname $junit4_jar`/junit4.jar
+
+ have_junit4=1
+else
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: Cannot find junit4.jar (apt-get install junit4)" >&5
+$as_echo "$as_me: Cannot find junit4.jar (apt-get install junit4)" >&6;}
+ have_junit4=0
+fi
+fi
+
+
+if test "x$CLASSPATH" = x; then
+ echo "You have no CLASSPATH, I hope it is good"
+else
+ echo "You have CLASSPATH $CLASSPATH, hope it is correct"
fi
-else
- if test "x$with_radosgw" != "xcheck"; then
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "--with-radosgw was given but libcurl (libcurl-dev on debian) not found
-See \`config.log' for more details" "$LINENO" 5; }
- fi
+if test "x$JAVAPREFIX" = x; then
+ test "x$JAVAC" = x && for ac_prog in "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT javac$EXEEXT
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_JAVAC+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$JAVAC"; then
+ ac_cv_prog_JAVAC="$JAVAC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_JAVAC="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
fi
-
-
+fi
+JAVAC=$ac_cv_prog_JAVAC
+if test -n "$JAVAC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAC" >&5
+$as_echo "$JAVAC" >&6; }
else
- if test "x$with_radosgw" != "xcheck"; then
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "--with-radosgw was given but libexpat (libexpat1-dev on debian) not found
-See \`config.log' for more details" "$LINENO" 5; }
- fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
+ test -n "$JAVAC" && break
+done
+
else
- if test "x$with_radosgw" != "xcheck"; then
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "--with-radosgw was given but libfcgi (libfcgi-dev on debian) not found
-See \`config.log' for more details" "$LINENO" 5; }
- fi
-fi
+ test "x$JAVAC" = x && for ac_prog in "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT javac$EXEEXT
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_JAVAC+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$JAVAC"; then
+ ac_cv_prog_JAVAC="$JAVAC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_JAVAC="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
fi
- if test "$RADOSGW" = "1"; then
- WITH_RADOSGW_TRUE=
- WITH_RADOSGW_FALSE='#'
+fi
+JAVAC=$ac_cv_prog_JAVAC
+if test -n "$JAVAC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAC" >&5
+$as_echo "$JAVAC" >&6; }
else
- WITH_RADOSGW_TRUE='#'
- WITH_RADOSGW_FALSE=
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-if test "$RADOSGW" = "1"; then :
-
-$as_echo "#define WITH_RADOSGW 1" >>confdefs.h
+ test -n "$JAVAC" && break
+done
+test -n "$JAVAC" || JAVAC="$JAVAPREFIX"
fi
+test "x$JAVAC" = x && as_fn_error $? "no acceptable Java compiler found in \$PATH" "$LINENO" 5
-if test "$RADOSGW" = "1"; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_multi_wait in -lcurl" >&5
-$as_echo_n "checking for curl_multi_wait in -lcurl... " >&6; }
-if ${ac_cv_lib_curl_curl_multi_wait+:} false; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $JAVAC works" >&5
+$as_echo_n "checking if $JAVAC works... " >&6; }
+if ${ac_cv_prog_javac_works+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lcurl $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char curl_multi_wait ();
-int
-main ()
-{
-return curl_multi_wait ();
- ;
- return 0;
+JAVA_TEST=Test.java
+CLASS_TEST=Test.class
+cat << \EOF > $JAVA_TEST
+/* #line 18742 "configure" */
+public class Test {
}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_curl_curl_multi_wait=yes
+EOF
+if { ac_try='$JAVAC $JAVACFLAGS $JAVA_TEST'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; } >/dev/null 2>&1; then
+ ac_cv_prog_javac_works=yes
else
- ac_cv_lib_curl_curl_multi_wait=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+ as_fn_error $? "The Java compiler $JAVAC failed (see config.log, check the CLASSPATH?)" "$LINENO" 5
+ echo "configure: failed program was:" >&5
+ cat $JAVA_TEST >&5
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_multi_wait" >&5
-$as_echo "$ac_cv_lib_curl_curl_multi_wait" >&6; }
-if test "x$ac_cv_lib_curl_curl_multi_wait" = xyes; then :
-
-$as_echo "#define HAVE_CURL_MULTI_WAIT 1" >>confdefs.h
+rm -f $JAVA_TEST $CLASS_TEST
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_javac_works" >&5
+$as_echo "$ac_cv_prog_javac_works" >&6; }
-fi
-# fuse?
+# Extract the first word of "javah", so it can be a program name with args.
+set dummy javah; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_JAVAH+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $JAVAH in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_JAVAH="$JAVAH" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_path_JAVAH="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
-# Check whether --with-fuse was given.
-if test "${with_fuse+set}" = set; then :
- withval=$with_fuse;
+ ;;
+esac
+fi
+JAVAH=$ac_cv_path_JAVAH
+if test -n "$JAVAH"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAH" >&5
+$as_echo "$JAVAH" >&6; }
else
- with_fuse=yes
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-LIBFUSE=
-if test "x$with_fuse" != xno; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for fuse_main in -lfuse" >&5
-$as_echo_n "checking for fuse_main in -lfuse... " >&6; }
-if ${ac_cv_lib_fuse_fuse_main+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lfuse $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+
+if test x"`eval 'echo $ac_cv_path_JAVAH'`" != x ; then
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
+#include <jni.h>
+_ACEOF
+if ac_fn_c_try_cpp "$LINENO"; then :
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char fuse_main ();
-int
-main ()
-{
-return fuse_main ();
- ;
- return 0;
-}
+else
+
+ ac_save_CPPFLAGS="$CPPFLAGS"
+ ac_dir=`echo $ac_cv_path_JAVAH | sed 's,\(.*\)/[^/]*/[^/]*$,\1/include,'`
+ ac_machdep=`echo $build_os | sed 's,[-0-9].*,,' | sed 's,cygwin,win32,'`
+ CPPFLAGS="$ac_save_CPPFLAGS -I$ac_dir -I$ac_dir/$ac_machdep"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <jni.h>
_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_fuse_fuse_main=yes
+if ac_fn_c_try_cpp "$LINENO"; then :
+ ac_save_CPPFLAGS="$CPPFLAGS"
else
- ac_cv_lib_fuse_fuse_main=no
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unable to include <jni.h>" >&5
+$as_echo "$as_me: WARNING: unable to include <jni.h>" >&2;}
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+rm -f conftest.err conftest.i conftest.$ac_ext
+ CPPFLAGS="$ac_save_CPPFLAGS"
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_fuse_fuse_main" >&5
-$as_echo "$ac_cv_lib_fuse_fuse_main" >&6; }
-if test "x$ac_cv_lib_fuse_fuse_main" = xyes; then :
- LIBFUSE="-lfuse"
-
-
-$as_echo "#define HAVE_LIBFUSE 1" >>confdefs.h
-
- HAVE_LIBFUSE=1
- # look for fuse_getgroups and define FUSE_GETGROUPS if found
- LIBS_saved="$LIBS"
- LIBS="$LIBS -lfuse"
- for ac_func in fuse_getgroups
-do :
- ac_fn_c_check_func "$LINENO" "fuse_getgroups" "ac_cv_func_fuse_getgroups"
-if test "x$ac_cv_func_fuse_getgroups" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_FUSE_GETGROUPS 1
-_ACEOF
-
+rm -f conftest.err conftest.i conftest.$ac_ext
fi
+
+if test "x$JAVAPREFIX" = x; then
+ test "x$JAR" = x && for ac_prog in jar$EXEEXT
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_JAR+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$JAR"; then
+ ac_cv_prog_JAR="$JAR" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_JAR="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
done
+ done
+IFS=$as_save_IFS
- LIBS="$LIBS_saved"
-
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "no FUSE found (use --without-fuse to disable)
-See \`config.log' for more details" "$LINENO" 5; }
fi
-
fi
- if test "$HAVE_LIBFUSE" = "1"; then
- WITH_FUSE_TRUE=
- WITH_FUSE_FALSE='#'
+JAR=$ac_cv_prog_JAR
+if test -n "$JAR"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAR" >&5
+$as_echo "$JAR" >&6; }
else
- WITH_FUSE_TRUE='#'
- WITH_FUSE_FALSE=
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-# tcmalloc?
+ test -n "$JAR" && break
+done
-# Check whether --with-tcmalloc was given.
-if test "${with_tcmalloc+set}" = set; then :
- withval=$with_tcmalloc;
else
- with_tcmalloc=yes
-fi
-
-TCMALLOC=
-if test "x$with_tcmalloc" != xno; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for malloc in -ltcmalloc" >&5
-$as_echo_n "checking for malloc in -ltcmalloc... " >&6; }
-if ${ac_cv_lib_tcmalloc_malloc+:} false; then :
+ test "x$JAR" = x && for ac_prog in jar
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_JAR+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-ltcmalloc $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+ if test -n "$JAR"; then
+ ac_cv_prog_JAR="$JAR" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_JAR="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char malloc ();
-int
-main ()
-{
-return malloc ();
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_tcmalloc_malloc=yes
+fi
+fi
+JAR=$ac_cv_prog_JAR
+if test -n "$JAR"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAR" >&5
+$as_echo "$JAR" >&6; }
else
- ac_cv_lib_tcmalloc_malloc=no
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+
+
+ test -n "$JAR" && break
+done
+test -n "$JAR" || JAR="$JAVAPREFIX"
+
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_tcmalloc_malloc" >&5
-$as_echo "$ac_cv_lib_tcmalloc_malloc" >&6; }
-if test "x$ac_cv_lib_tcmalloc_malloc" = xyes; then :
- LIBTCMALLOC="-ltcmalloc"
+test "x$JAR" = x && as_fn_error $? "no acceptable jar program found in \$PATH" "$LINENO" 5
-$as_echo "#define HAVE_LIBTCMALLOC 1" >>confdefs.h
+ CLASSPATH=$CLASSPATH:$EXTRA_CLASSPATH_JAR
+ export CLASSPATH
+ { $as_echo "$as_me:${as_lineno-$LINENO}: classpath - $CLASSPATH" >&5
+$as_echo "$as_me: classpath - $CLASSPATH" >&6;}
- HAVE_LIBTCMALLOC=1
+ # Check for jni.h
+ CPPFLAGS_save=$CPPFLAGS
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "no tcmalloc found (use --without-tcmalloc to disable)
-See \`config.log' for more details" "$LINENO" 5; }
+ if test -n "$EXTRA_JDK_INC_DIR"; then :
+ JDK_CPPFLAGS="-I$EXTRA_JDK_INC_DIR"
+ if test -d "$EXTRA_JDK_INC_DIR/linux"; then :
+ JDK_CPPFLAGS="$JDK_CPPFLAGS -I$EXTRA_JDK_INC_DIR/linux"
fi
-
+ CPPFLAGS="$CPPFLAGS $JDK_CPPFLAGS"
fi
- if test "$HAVE_LIBTCMALLOC" = "1"; then
- WITH_TCMALLOC_TRUE=
- WITH_TCMALLOC_FALSE='#'
+
+ ac_fn_c_check_header_mongrel "$LINENO" "jni.h" "ac_cv_header_jni_h" "$ac_includes_default"
+if test "x$ac_cv_header_jni_h" = xyes; then :
+
else
- WITH_TCMALLOC_TRUE='#'
- WITH_TCMALLOC_FALSE=
+ as_fn_error $? "Cannot find header 'jni.h'. Try setting --with-jdk-dir" "$LINENO" 5
fi
-#set pg ref debugging?
-# Check whether --enable-pgrefdebugging was given.
-if test "${enable_pgrefdebugging+set}" = set; then :
- enableval=$enable_pgrefdebugging;
-$as_echo "#define PG_DEBUG_REFS 1" >>confdefs.h
+ CPPFLAGS=$CPPFLAGS_save
+
+ # Setup output var
+
+fi
+ if test "$have_junit4" = "1"; then
+ HAVE_JUNIT4_TRUE=
+ HAVE_JUNIT4_FALSE='#'
+else
+ HAVE_JUNIT4_TRUE='#'
+ HAVE_JUNIT4_FALSE=
fi
#
-# Java is painful
-# - adapted from OMPI wrappers package
-# - this might become bigger. maybe should be own m4 file
+# FreeBSD has it in base.
#
-# Check whether --enable-cephfs-java was given.
-if test "${enable_cephfs_java+set}" = set; then :
- enableval=$enable_cephfs_java;
-else
- enable_cephfs_java=no
-fi
+if test x"$freebsd" != x"yes"; then
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBEDIT" >&5
+$as_echo_n "checking for LIBEDIT... " >&6; }
- if test "x$enable_cephfs_java" = "xyes"; then
- ENABLE_CEPHFS_JAVA_TRUE=
- ENABLE_CEPHFS_JAVA_FALSE='#'
+if test -n "$LIBEDIT_CFLAGS"; then
+ pkg_cv_LIBEDIT_CFLAGS="$LIBEDIT_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libedit >= 2.11\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "libedit >= 2.11") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LIBEDIT_CFLAGS=`$PKG_CONFIG --cflags "libedit >= 2.11" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
else
- ENABLE_CEPHFS_JAVA_TRUE='#'
- ENABLE_CEPHFS_JAVA_FALSE=
+ pkg_failed=yes
fi
-
-
-
-# Check whether --with-jdk-dir was given.
-if test "${with_jdk_dir+set}" = set; then :
- withval=$with_jdk_dir;
+ else
+ pkg_failed=untried
+fi
+if test -n "$LIBEDIT_LIBS"; then
+ pkg_cv_LIBEDIT_LIBS="$LIBEDIT_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libedit >= 2.11\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "libedit >= 2.11") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LIBEDIT_LIBS=`$PKG_CONFIG --libs "libedit >= 2.11" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
fi
-if test "x$enable_cephfs_java" = "xyes"; then
-
- # setup bin/include dirs from --with-jdk-dir (search for jni.h, javac)
- if test -n "$with_jdk_dir"; then :
- javac_prog=`find $with_jdk_dir/ -name javac | head -n 1`
- if test -x "$javac_prog"; then :
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
- EXTRA_JDK_BIN_DIR=`dirname $javac_prog`
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
fi
- jnih=`find $with_jdk_dir/ -name jni.h | head -n 1`
- if test -r "$jnih"; then :
+ if test $_pkg_short_errors_supported = yes; then
+ LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit >= 2.11" 2>&1`
+ else
+ LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit >= 2.11" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LIBEDIT_PKG_ERRORS" >&5
+
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "No usable version of libedit found.
+See \`config.log' for more details" "$LINENO" 5; }
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "No usable version of libedit found.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS
+ LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
- EXTRA_JDK_INC_DIR=`dirname $jnih`
fi
+else
+ LIBEDIT_LIBS="-ledit"
fi
- # setup defaults for Debian default-jdk package (without --with-jdk-dir)
- if test -z "$with_jdk_dir"; then :
-
- # This works with Debian's and CentOS' default-jdk package
- for dir in '/usr/lib/jvm/default-java/' '/usr/lib/jvm/java/' ; do
- # only test if a suitable path has not yet been found
- if test "$EXTRA_JDK_BIN_DIR" == ""; then :
-
- if test -x "$javac_prog"; then :
-
- EXTRA_JDK_BIN_DIR=`dirname $javac_prog`
-fi
- jnih=`find $dir -name jni.h | head -n 1`
- if test -r "$jnih"; then :
+#libatomic-ops? You want it!
- EXTRA_JDK_INC_DIR=`dirname $jnih`
+# Check whether --with-libatomic-ops was given.
+if test "${with_libatomic_ops+set}" = set; then :
+ withval=$with_libatomic_ops;
+else
+ with_libatomic_ops=yes
fi
-fi
- done
+if test "x$with_libatomic_ops" != xno; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "atomic_ops.h" "ac_cv_header_atomic_ops_h" "$ac_includes_default"
+if test "x$ac_cv_header_atomic_ops_h" = xyes; then :
+ HAVE_ATOMIC_OPS=1
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "no libatomic-ops found (use --without-libatomic-ops to disable)
+See \`config.log' for more details" "$LINENO" 5; }
fi
- # cephfs_java_test only makes sense if java is already turned on
- # setup CLASSPATH for Debian default junit4.jar package
- #
- # Configuring --with-debug and --enable-cephfs-java will throw an error if
- # JUnit4 cannot be found. While currently this works for users who have
- # installed via the package manager on Ubuntu, we need to expand this
- # check to 1 support other distrubtions and 2 allow users to influence
- # the search path.
- if test "x$with_debug" = "xyes"; then :
-
- dir='/usr/share/java'
- junit4_jar=`find $dir -name junit4.jar | head -n 1`
- if test -r "$junit4_jar"; then :
- EXTRA_CLASSPATH_JAR=`dirname $junit4_jar`/junit4.jar
+fi
+if test "$HAVE_ATOMIC_OPS" = "1"; then :
- have_junit4=1
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: Cannot find junit4.jar (apt-get install junit4)" >&5
-$as_echo "$as_me: Cannot find junit4.jar (apt-get install junit4)" >&6;}
- have_junit4=0
-fi
-fi
+$as_echo "#define NO_ATOMIC_OPS 1" >>confdefs.h
+fi
-if test "x$CLASSPATH" = x; then
- echo "You have no CLASSPATH, I hope it is good"
+ if test "$HAVE_ATOMIC_OPS" = "1"; then
+ WITH_LIBATOMIC_TRUE=
+ WITH_LIBATOMIC_FALSE='#'
else
- echo "You have CLASSPATH $CLASSPATH, hope it is correct"
+ WITH_LIBATOMIC_TRUE='#'
+ WITH_LIBATOMIC_FALSE=
fi
+# newsyn? requires mpi.
+#AC_ARG_WITH([newsyn],
+# [AS_HELP_STRING([--with-newsyn], [build newsyn target requires mpi])],
+# [],
+# [with_newsyn=no])
-if test "x$JAVAPREFIX" = x; then
- test "x$JAVAC" = x && for ac_prog in "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT javac$EXEEXT
-do
- # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_JAVAC+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- if test -n "$JAVAC"; then
- ac_cv_prog_JAVAC="$JAVAC" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_JAVAC="$ac_prog"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
- done
-IFS=$as_save_IFS
-fi
-fi
-JAVAC=$ac_cv_prog_JAVAC
-if test -n "$JAVAC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAC" >&5
-$as_echo "$JAVAC" >&6; }
+# Check whether --with-ocf was given.
+if test "${with_ocf+set}" = set; then :
+ withval=$with_ocf;
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ with_ocf=no
fi
+ if test "$with_ocf" = "yes" ; then
+ WITH_OCF_TRUE=
+ WITH_OCF_FALSE='#'
+else
+ WITH_OCF_TRUE='#'
+ WITH_OCF_FALSE=
+fi
- test -n "$JAVAC" && break
-done
-else
- test "x$JAVAC" = x && for ac_prog in "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT javac$EXEEXT
-do
- # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_JAVAC+:} false; then :
+# check is snappy-devel is installed, needed by leveldb
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for snappy_compress in -lsnappy" >&5
+$as_echo_n "checking for snappy_compress in -lsnappy... " >&6; }
+if ${ac_cv_lib_snappy_snappy_compress+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$JAVAC"; then
- ac_cv_prog_JAVAC="$JAVAC" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_JAVAC="$ac_prog"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
- done
-IFS=$as_save_IFS
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lsnappy $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
-fi
-fi
-JAVAC=$ac_cv_prog_JAVAC
-if test -n "$JAVAC"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAC" >&5
-$as_echo "$JAVAC" >&6; }
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char snappy_compress ();
+int
+main ()
+{
+return snappy_compress ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_snappy_snappy_compress=yes
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ ac_cv_lib_snappy_snappy_compress=no
fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_snappy_snappy_compress" >&5
+$as_echo "$ac_cv_lib_snappy_snappy_compress" >&6; }
+if test "x$ac_cv_lib_snappy_snappy_compress" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBSNAPPY 1
+_ACEOF
+ LIBS="-lsnappy $LIBS"
- test -n "$JAVAC" && break
-done
-test -n "$JAVAC" || JAVAC="$JAVAPREFIX"
-
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "libsnappy not found
+See \`config.log' for more details" "$LINENO" 5; }
fi
-test "x$JAVAC" = x && as_fn_error $? "no acceptable Java compiler found in \$PATH" "$LINENO" 5
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $JAVAC works" >&5
-$as_echo_n "checking if $JAVAC works... " >&6; }
-if ${ac_cv_prog_javac_works+:} false; then :
+# use system leveldb
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for leveldb_open in -lleveldb" >&5
+$as_echo_n "checking for leveldb_open in -lleveldb... " >&6; }
+if ${ac_cv_lib_leveldb_leveldb_open+:} false; then :
$as_echo_n "(cached) " >&6
else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lleveldb -lsnappy -lpthread $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
-JAVA_TEST=Test.java
-CLASS_TEST=Test.class
-cat << \EOF > $JAVA_TEST
-/* #line 18285 "configure" */
-public class Test {
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char leveldb_open ();
+int
+main ()
+{
+return leveldb_open ();
+ ;
+ return 0;
}
-EOF
-if { ac_try='$JAVAC $JAVACFLAGS $JAVA_TEST'
- { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
- (eval $ac_try) 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; } >/dev/null 2>&1; then
- ac_cv_prog_javac_works=yes
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_leveldb_leveldb_open=yes
+else
+ ac_cv_lib_leveldb_leveldb_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_leveldb_leveldb_open" >&5
+$as_echo "$ac_cv_lib_leveldb_leveldb_open" >&6; }
+if test "x$ac_cv_lib_leveldb_leveldb_open" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBLEVELDB 1
+_ACEOF
+
+ LIBS="-lleveldb $LIBS"
+
else
- as_fn_error $? "The Java compiler $JAVAC failed (see config.log, check the CLASSPATH?)" "$LINENO" 5
- echo "configure: failed program was:" >&5
- cat $JAVA_TEST >&5
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "libleveldb not found
+See \`config.log' for more details" "$LINENO" 5; }
fi
-rm -f $JAVA_TEST $CLASS_TEST
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_javac_works" >&5
-$as_echo "$ac_cv_prog_javac_works" >&6; }
+# see if we can use bloom filters with leveldb
+ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+ac_fn_cxx_check_header_mongrel "$LINENO" "leveldb/filter_policy.h" "ac_cv_header_leveldb_filter_policy_h" "$ac_includes_default"
+if test "x$ac_cv_header_leveldb_filter_policy_h" = xyes; then :
-# Extract the first word of "javah", so it can be a program name with args.
-set dummy javah; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_path_JAVAH+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- case $JAVAH in
- [\\/]* | ?:[\\/]*)
- ac_cv_path_JAVAH="$JAVAH" # Let the user override the test with a path.
- ;;
- *)
- as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_path_JAVAH="$as_dir/$ac_word$ac_exec_ext"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
- done
-IFS=$as_save_IFS
+$as_echo "#define HAVE_LEVELDB_FILTER_POLICY 1" >>confdefs.h
- ;;
-esac
-fi
-JAVAH=$ac_cv_path_JAVAH
-if test -n "$JAVAH"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAH" >&5
-$as_echo "$JAVAH" >&6; }
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
fi
-if test x"`eval 'echo $ac_cv_path_JAVAH'`" != x ; then
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <jni.h>
-_ACEOF
-if ac_fn_c_try_cpp "$LINENO"; then :
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+# use system libs3?
+# Check whether --with-system-libs3 was given.
+if test "${with_system_libs3+set}" = set; then :
+ withval=$with_system_libs3;
else
+ with_system_libs3=no
+fi
- ac_save_CPPFLAGS="$CPPFLAGS"
- ac_dir=`echo $ac_cv_path_JAVAH | sed 's,\(.*\)/[^/]*/[^/]*$,\1/include,'`
- ac_machdep=`echo $build_os | sed 's,[-0-9].*,,' | sed 's,cygwin,win32,'`
- CPPFLAGS="$ac_save_CPPFLAGS -I$ac_dir -I$ac_dir/$ac_machdep"
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+if test "x$with_system_libs3" = xyes; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for S3_initialize in -ls3" >&5
+$as_echo_n "checking for S3_initialize in -ls3... " >&6; }
+if ${ac_cv_lib_s3_S3_initialize+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-ls3 -lpthread $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <jni.h>
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char S3_initialize ();
+int
+main ()
+{
+return S3_initialize ();
+ ;
+ return 0;
+}
_ACEOF
-if ac_fn_c_try_cpp "$LINENO"; then :
- ac_save_CPPFLAGS="$CPPFLAGS"
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_s3_S3_initialize=yes
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unable to include <jni.h>" >&5
-$as_echo "$as_me: WARNING: unable to include <jni.h>" >&2;}
+ ac_cv_lib_s3_S3_initialize=no
fi
-rm -f conftest.err conftest.i conftest.$ac_ext
- CPPFLAGS="$ac_save_CPPFLAGS"
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-rm -f conftest.err conftest.i conftest.$ac_ext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_s3_S3_initialize" >&5
+$as_echo "$ac_cv_lib_s3_S3_initialize" >&6; }
+if test "x$ac_cv_lib_s3_S3_initialize" = xyes; then :
+ true
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "libs3 not found
+See \`config.log' for more details" "$LINENO" 5; }
fi
-if test "x$JAVAPREFIX" = x; then
- test "x$JAR" = x && for ac_prog in jar$EXEEXT
-do
- # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_JAR+:} false; then :
+fi
+if test "x$with_system_libs3" = xcheck; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing S3_initialize" >&5
+$as_echo_n "checking for library containing S3_initialize... " >&6; }
+if ${ac_cv_search_S3_initialize+:} false; then :
$as_echo_n "(cached) " >&6
else
- if test -n "$JAR"; then
- ac_cv_prog_JAR="$JAR" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_JAR="$ac_prog"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
+ ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char S3_initialize ();
+int
+main ()
+{
+return S3_initialize ();
+ ;
+ return 0;
+}
+_ACEOF
+for ac_lib in '' s3; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib -lpthread $ac_func_search_save_LIBS"
fi
+ if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_search_S3_initialize=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext
+ if ${ac_cv_search_S3_initialize+:} false; then :
+ break
+fi
done
- done
-IFS=$as_save_IFS
+if ${ac_cv_search_S3_initialize+:} false; then :
+else
+ ac_cv_search_S3_initialize=no
fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
fi
-JAR=$ac_cv_prog_JAR
-if test -n "$JAR"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAR" >&5
-$as_echo "$JAR" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_S3_initialize" >&5
+$as_echo "$ac_cv_search_S3_initialize" >&6; }
+ac_res=$ac_cv_search_S3_initialize
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+ with_system_libs3=yes
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ true
fi
+fi
+ if test "$with_system_libs3" = "yes" ; then
+ WITH_SYSTEM_LIBS3_TRUE=
+ WITH_SYSTEM_LIBS3_FALSE='#'
+else
+ WITH_SYSTEM_LIBS3_TRUE='#'
+ WITH_SYSTEM_LIBS3_FALSE=
+fi
- test -n "$JAR" && break
-done
-else
- test "x$JAR" = x && for ac_prog in jar
-do
- # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
-$as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_JAR+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- if test -n "$JAR"; then
- ac_cv_prog_JAR="$JAR" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
-do
- IFS=$as_save_IFS
- test -z "$as_dir" && as_dir=.
- for ac_exec_ext in '' $ac_executable_extensions; do
- if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
- ac_cv_prog_JAR="$ac_prog"
- $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
- break 2
- fi
-done
- done
-IFS=$as_save_IFS
+# rest-bench?
+# Check whether --with-rest-bench was given.
+if test "${with_rest_bench+set}" = set; then :
+ withval=$with_rest_bench;
+else
+ with_rest_bench=no
fi
-fi
-JAR=$ac_cv_prog_JAR
-if test -n "$JAR"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAR" >&5
-$as_echo "$JAR" >&6; }
+
+ if test "$with_rest_bench" = "yes" ; then
+ WITH_REST_BENCH_TRUE=
+ WITH_REST_BENCH_FALSE='#'
else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ WITH_REST_BENCH_TRUE='#'
+ WITH_REST_BENCH_FALSE=
fi
- test -n "$JAR" && break
-done
-test -n "$JAR" || JAR="$JAVAPREFIX"
+# use libaio?
+# Check whether --with-libaio was given.
+if test "${with_libaio+set}" = set; then :
+ withval=$with_libaio;
+else
+ with_libaio=yes
fi
-test "x$JAR" = x && as_fn_error $? "no acceptable jar program found in \$PATH" "$LINENO" 5
-
-
- CLASSPATH=$CLASSPATH:$EXTRA_CLASSPATH_JAR
- export CLASSPATH
- { $as_echo "$as_me:${as_lineno-$LINENO}: classpath - $CLASSPATH" >&5
-$as_echo "$as_me: classpath - $CLASSPATH" >&6;}
- # Check for jni.h
- CPPFLAGS_save=$CPPFLAGS
+if test "x$with_libaio" != xno; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for io_submit in -laio" >&5
+$as_echo_n "checking for io_submit in -laio... " >&6; }
+if ${ac_cv_lib_aio_io_submit+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-laio $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
- if test -n "$EXTRA_JDK_INC_DIR"; then :
- JDK_CPPFLAGS="-I$EXTRA_JDK_INC_DIR"
- if test -d "$EXTRA_JDK_INC_DIR/linux"; then :
- JDK_CPPFLAGS="$JDK_CPPFLAGS -I$EXTRA_JDK_INC_DIR/linux"
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char io_submit ();
+int
+main ()
+{
+return io_submit ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_aio_io_submit=yes
+else
+ ac_cv_lib_aio_io_submit=no
fi
- CPPFLAGS="$CPPFLAGS $JDK_CPPFLAGS"
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-
- ac_fn_c_check_header_mongrel "$LINENO" "jni.h" "ac_cv_header_jni_h" "$ac_includes_default"
-if test "x$ac_cv_header_jni_h" = xyes; then :
-
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_aio_io_submit" >&5
+$as_echo "$ac_cv_lib_aio_io_submit" >&6; }
+if test "x$ac_cv_lib_aio_io_submit" = xyes; then :
+ true
else
- as_fn_error $? "Cannot find header 'jni.h'. Try setting --with-jdk-dir" "$LINENO" 5
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "libaio not found
+See \`config.log' for more details" "$LINENO" 5; }
fi
+fi
+if test "x$with_libaio" != xno; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "libaio.h" "ac_cv_header_libaio_h" "$ac_includes_default"
+if test "x$ac_cv_header_libaio_h" = xyes; then :
+
+fi
- CPPFLAGS=$CPPFLAGS_save
+fi
+if test "$with_libaio" = "yes"; then :
- # Setup output var
+$as_echo "#define HAVE_LIBAIO 1" >>confdefs.h
fi
- if test "$have_junit4" = "1"; then
- HAVE_JUNIT4_TRUE=
- HAVE_JUNIT4_FALSE='#'
+ if test "$with_libaio" = "yes" ; then
+ WITH_LIBAIO_TRUE=
+ WITH_LIBAIO_FALSE='#'
else
- HAVE_JUNIT4_TRUE='#'
- HAVE_JUNIT4_FALSE=
+ WITH_LIBAIO_TRUE='#'
+ WITH_LIBAIO_FALSE=
fi
-#
-# FreeBSD has it in base.
-#
-if test x"$freebsd" != x"yes"; then
+# use libzfs
+
+# Check whether --with-libzfs was given.
+if test "${with_libzfs+set}" = set; then :
+ withval=$with_libzfs;
+else
+ with_libzfs=no
+fi
+
+if test "x$with_libzfs" = xyes; then :
pkg_failed=no
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBEDIT" >&5
-$as_echo_n "checking for LIBEDIT... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBZFS" >&5
+$as_echo_n "checking for LIBZFS... " >&6; }
-if test -n "$LIBEDIT_CFLAGS"; then
- pkg_cv_LIBEDIT_CFLAGS="$LIBEDIT_CFLAGS"
+if test -n "$LIBZFS_CFLAGS"; then
+ pkg_cv_LIBZFS_CFLAGS="$LIBZFS_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
- { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libedit >= 2.11\""; } >&5
- ($PKG_CONFIG --exists --print-errors "libedit >= 2.11") 2>&5
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zfs\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "zfs") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
- pkg_cv_LIBEDIT_CFLAGS=`$PKG_CONFIG --cflags "libedit >= 2.11" 2>/dev/null`
+ pkg_cv_LIBZFS_CFLAGS=`$PKG_CONFIG --cflags "zfs" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
@@ -18531,16 +19488,16 @@ fi
else
pkg_failed=untried
fi
-if test -n "$LIBEDIT_LIBS"; then
- pkg_cv_LIBEDIT_LIBS="$LIBEDIT_LIBS"
+if test -n "$LIBZFS_LIBS"; then
+ pkg_cv_LIBZFS_LIBS="$LIBZFS_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
- { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libedit >= 2.11\""; } >&5
- ($PKG_CONFIG --exists --print-errors "libedit >= 2.11") 2>&5
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zfs\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "zfs") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
- pkg_cv_LIBEDIT_LIBS=`$PKG_CONFIG --libs "libedit >= 2.11" 2>/dev/null`
+ pkg_cv_LIBZFS_LIBS=`$PKG_CONFIG --libs "zfs" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
@@ -18561,240 +19518,90 @@ else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
- LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit >= 2.11" 2>&1`
+ LIBZFS_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zfs" 2>&1`
else
- LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit >= 2.11" 2>&1`
+ LIBZFS_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zfs" 2>&1`
fi
# Put the nasty error message in config.log where it belongs
- echo "$LIBEDIT_PKG_ERRORS" >&5
+ echo "$LIBZFS_PKG_ERRORS" >&5
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "No usable version of libedit found.
-See \`config.log' for more details" "$LINENO" 5; }
+ true
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "No usable version of libedit found.
-See \`config.log' for more details" "$LINENO" 5; }
-else
- LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS
- LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-
-fi
-else
- LIBEDIT_LIBS="-ledit"
-fi
-
-#libatomic-ops? You want it!
-
-# Check whether --with-libatomic-ops was given.
-if test "${with_libatomic_ops+set}" = set; then :
- withval=$with_libatomic_ops;
-else
- with_libatomic_ops=yes
-fi
-
-if test "x$with_libatomic_ops" != xno; then :
- ac_fn_c_check_header_mongrel "$LINENO" "atomic_ops.h" "ac_cv_header_atomic_ops_h" "$ac_includes_default"
-if test "x$ac_cv_header_atomic_ops_h" = xyes; then :
- HAVE_ATOMIC_OPS=1
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "no libatomic-ops found (use --without-libatomic-ops to disable)
-See \`config.log' for more details" "$LINENO" 5; }
-
-fi
-
-
-fi
-if test "$HAVE_ATOMIC_OPS" = "1"; then :
-
-else
-
-$as_echo "#define NO_ATOMIC_OPS 1" >>confdefs.h
-
-fi
-
- if test "$HAVE_ATOMIC_OPS" = "1"; then
- WITH_LIBATOMIC_TRUE=
- WITH_LIBATOMIC_FALSE='#'
-else
- WITH_LIBATOMIC_TRUE='#'
- WITH_LIBATOMIC_FALSE=
-fi
-
-
-# newsyn? requires mpi.
-#AC_ARG_WITH([newsyn],
-# [AS_HELP_STRING([--with-newsyn], [build newsyn target requires mpi])],
-# [],
-# [with_newsyn=no])
-
-
-# Check whether --with-ocf was given.
-if test "${with_ocf+set}" = set; then :
- withval=$with_ocf;
-else
- with_ocf=no
-fi
-
- if test "$with_ocf" = "yes" ; then
- WITH_OCF_TRUE=
- WITH_OCF_FALSE='#'
-else
- WITH_OCF_TRUE='#'
- WITH_OCF_FALSE=
-fi
-
-
-# check is snappy-devel is installed, needed by leveldb
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for snappy_compress in -lsnappy" >&5
-$as_echo_n "checking for snappy_compress in -lsnappy... " >&6; }
-if ${ac_cv_lib_snappy_snappy_compress+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lsnappy $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char snappy_compress ();
-int
-main ()
-{
-return snappy_compress ();
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_snappy_snappy_compress=yes
+ true
else
- ac_cv_lib_snappy_snappy_compress=no
+ LIBZFS_CFLAGS=$pkg_cv_LIBZFS_CFLAGS
+ LIBZFS_LIBS=$pkg_cv_LIBZFS_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_snappy_snappy_compress" >&5
-$as_echo "$ac_cv_lib_snappy_snappy_compress" >&6; }
-if test "x$ac_cv_lib_snappy_snappy_compress" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBSNAPPY 1
-_ACEOF
+if test "x$with_libzfs" = xyes; then :
- LIBS="-lsnappy $LIBS"
+$as_echo "#define HAVE_LIBZFS 1" >>confdefs.h
+fi
+ if test "$with_libzfs" = "yes" ; then
+ WITH_LIBZFS_TRUE=
+ WITH_LIBZFS_FALSE='#'
else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libsnappy not found
-See \`config.log' for more details" "$LINENO" 5; }
+ WITH_LIBZFS_TRUE='#'
+ WITH_LIBZFS_FALSE=
fi
-# use system leveldb
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for leveldb_open in -lleveldb" >&5
-$as_echo_n "checking for leveldb_open in -lleveldb... " >&6; }
-if ${ac_cv_lib_leveldb_leveldb_open+:} false; then :
+
+# Checks for header files.
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+ as_ac_Header=`$as_echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_hdr that defines DIR" >&5
+$as_echo_n "checking for $ac_hdr that defines DIR... " >&6; }
+if eval \${$as_ac_Header+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lleveldb -lsnappy -lpthread $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
+#include <sys/types.h>
+#include <$ac_hdr>
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char leveldb_open ();
int
main ()
{
-return leveldb_open ();
+if ((DIR *) 0)
+return 0;
;
return 0;
}
_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_leveldb_leveldb_open=yes
+if ac_fn_c_try_compile "$LINENO"; then :
+ eval "$as_ac_Header=yes"
else
- ac_cv_lib_leveldb_leveldb_open=no
+ eval "$as_ac_Header=no"
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_leveldb_leveldb_open" >&5
-$as_echo "$ac_cv_lib_leveldb_leveldb_open" >&6; }
-if test "x$ac_cv_lib_leveldb_leveldb_open" = xyes; then :
+eval ac_res=\$$as_ac_Header
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBLEVELDB 1
+#define `$as_echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
_ACEOF
- LIBS="-lleveldb $LIBS"
-
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libleveldb not found
-See \`config.log' for more details" "$LINENO" 5; }
-fi
-
-# see if we can use bloom filters with leveldb
-ac_ext=cpp
-ac_cpp='$CXXCPP $CPPFLAGS'
-ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
-
-
-ac_fn_cxx_check_header_mongrel "$LINENO" "leveldb/filter_policy.h" "ac_cv_header_leveldb_filter_policy_h" "$ac_includes_default"
-if test "x$ac_cv_header_leveldb_filter_policy_h" = xyes; then :
-
-$as_echo "#define HAVE_LEVELDB_FILTER_POLICY 1" >>confdefs.h
-
-fi
-
-
-ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-
-# use system libs3?
-
-# Check whether --with-system-libs3 was given.
-if test "${with_system_libs3+set}" = set; then :
- withval=$with_system_libs3;
-else
- with_system_libs3=no
+ac_header_dirent=$ac_hdr; break
fi
-if test "x$with_system_libs3" = xyes; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for S3_initialize in -ls3" >&5
-$as_echo_n "checking for S3_initialize in -ls3... " >&6; }
-if ${ac_cv_lib_s3_S3_initialize+:} false; then :
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing opendir" >&5
+$as_echo_n "checking for library containing opendir... " >&6; }
+if ${ac_cv_search_opendir+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-ls3 -lpthread $LIBS"
+ ac_func_search_save_LIBS=$LIBS
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
@@ -18804,40 +19611,51 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char S3_initialize ();
+char opendir ();
int
main ()
{
-return S3_initialize ();
+return opendir ();
;
return 0;
}
_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_s3_S3_initialize=yes
-else
- ac_cv_lib_s3_S3_initialize=no
+for ac_lib in '' dir; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ fi
+ if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_search_opendir=$ac_res
fi
rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+ conftest$ac_exeext
+ if ${ac_cv_search_opendir+:} false; then :
+ break
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_s3_S3_initialize" >&5
-$as_echo "$ac_cv_lib_s3_S3_initialize" >&6; }
-if test "x$ac_cv_lib_s3_S3_initialize" = xyes; then :
- true
+done
+if ${ac_cv_search_opendir+:} false; then :
+
else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libs3 not found
-See \`config.log' for more details" "$LINENO" 5; }
+ ac_cv_search_opendir=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5
+$as_echo "$ac_cv_search_opendir" >&6; }
+ac_res=$ac_cv_search_opendir
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
fi
-if test "x$with_system_libs3" = xcheck; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing S3_initialize" >&5
-$as_echo_n "checking for library containing S3_initialize... " >&6; }
-if ${ac_cv_search_S3_initialize+:} false; then :
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing opendir" >&5
+$as_echo_n "checking for library containing opendir... " >&6; }
+if ${ac_cv_search_opendir+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_func_search_save_LIBS=$LIBS
@@ -18850,877 +19668,1011 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char S3_initialize ();
+char opendir ();
int
main ()
{
-return S3_initialize ();
+return opendir ();
;
return 0;
}
_ACEOF
-for ac_lib in '' s3; do
+for ac_lib in '' x; do
if test -z "$ac_lib"; then
ac_res="none required"
else
ac_res=-l$ac_lib
- LIBS="-l$ac_lib -lpthread $ac_func_search_save_LIBS"
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
fi
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_search_S3_initialize=$ac_res
+ ac_cv_search_opendir=$ac_res
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext
- if ${ac_cv_search_S3_initialize+:} false; then :
+ if ${ac_cv_search_opendir+:} false; then :
break
fi
done
-if ${ac_cv_search_S3_initialize+:} false; then :
+if ${ac_cv_search_opendir+:} false; then :
else
- ac_cv_search_S3_initialize=no
+ ac_cv_search_opendir=no
fi
rm conftest.$ac_ext
LIBS=$ac_func_search_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_S3_initialize" >&5
-$as_echo "$ac_cv_search_S3_initialize" >&6; }
-ac_res=$ac_cv_search_S3_initialize
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5
+$as_echo "$ac_cv_search_opendir" >&6; }
+ac_res=$ac_cv_search_opendir
if test "$ac_res" != no; then :
test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
- with_system_libs3=yes
+
+fi
+
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5
+$as_echo_n "checking for ANSI C header files... " >&6; }
+if ${ac_cv_header_stdc+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_header_stdc=yes
+else
+ ac_cv_header_stdc=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+ # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+ $EGREP "memchr" >/dev/null 2>&1; then :
+
+else
+ ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+ # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+ $EGREP "free" >/dev/null 2>&1; then :
+
else
- true
+ ac_cv_header_stdc=no
fi
+rm -f conftest*
fi
- if test "$with_system_libs3" = "yes" ; then
- WITH_SYSTEM_LIBS3_TRUE=
- WITH_SYSTEM_LIBS3_FALSE='#'
-else
- WITH_SYSTEM_LIBS3_TRUE='#'
- WITH_SYSTEM_LIBS3_FALSE=
-fi
+if test $ac_cv_header_stdc = yes; then
+ # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+ if test "$cross_compiling" = yes; then :
+ :
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <ctype.h>
+#include <stdlib.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+ (('a' <= (c) && (c) <= 'i') \
+ || ('j' <= (c) && (c) <= 'r') \
+ || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
-# rest-bench?
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+ int i;
+ for (i = 0; i < 256; i++)
+ if (XOR (islower (i), ISLOWER (i))
+ || toupper (i) != TOUPPER (i))
+ return 2;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_run "$LINENO"; then :
-# Check whether --with-rest-bench was given.
-if test "${with_rest_bench+set}" = set; then :
- withval=$with_rest_bench;
else
- with_rest_bench=no
+ ac_cv_header_stdc=no
fi
-
- if test "$with_rest_bench" = "yes" ; then
- WITH_REST_BENCH_TRUE=
- WITH_REST_BENCH_FALSE='#'
-else
- WITH_REST_BENCH_TRUE='#'
- WITH_REST_BENCH_FALSE=
+rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
+ conftest.$ac_objext conftest.beam conftest.$ac_ext
fi
+fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5
+$as_echo "$ac_cv_header_stdc" >&6; }
+if test $ac_cv_header_stdc = yes; then
-# use libaio?
+$as_echo "#define STDC_HEADERS 1" >>confdefs.h
-# Check whether --with-libaio was given.
-if test "${with_libaio+set}" = set; then :
- withval=$with_libaio;
-else
- with_libaio=yes
fi
-if test "x$with_libaio" != xno; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for io_submit in -laio" >&5
-$as_echo_n "checking for io_submit in -laio... " >&6; }
-if ${ac_cv_lib_aio_io_submit+:} false; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for sys/wait.h that is POSIX.1 compatible" >&5
+$as_echo_n "checking for sys/wait.h that is POSIX.1 compatible... " >&6; }
+if ${ac_cv_header_sys_wait_h+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-laio $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
+#include <sys/types.h>
+#include <sys/wait.h>
+#ifndef WEXITSTATUS
+# define WEXITSTATUS(stat_val) ((unsigned int) (stat_val) >> 8)
#endif
-char io_submit ();
+#ifndef WIFEXITED
+# define WIFEXITED(stat_val) (((stat_val) & 255) == 0)
+#endif
+
int
main ()
{
-return io_submit ();
+ int s;
+ wait (&s);
+ s = WIFEXITED (s) ? WEXITSTATUS (s) : 1;
;
return 0;
}
_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_lib_aio_io_submit=yes
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_header_sys_wait_h=yes
else
- ac_cv_lib_aio_io_submit=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+ ac_cv_header_sys_wait_h=no
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_aio_io_submit" >&5
-$as_echo "$ac_cv_lib_aio_io_submit" >&6; }
-if test "x$ac_cv_lib_aio_io_submit" = xyes; then :
- true
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "libaio not found
-See \`config.log' for more details" "$LINENO" 5; }
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_sys_wait_h" >&5
+$as_echo "$ac_cv_header_sys_wait_h" >&6; }
+if test $ac_cv_header_sys_wait_h = yes; then
-fi
-if test "x$with_libaio" != xno; then :
- ac_fn_c_check_header_mongrel "$LINENO" "libaio.h" "ac_cv_header_libaio_h" "$ac_includes_default"
-if test "x$ac_cv_header_libaio_h" = xyes; then :
+$as_echo "#define HAVE_SYS_WAIT_H 1" >>confdefs.h
fi
-fi
-if test "$with_libaio" = "yes"; then :
-$as_echo "#define HAVE_LIBAIO 1" >>confdefs.h
+# spirit?
+ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+
+ac_fn_cxx_check_header_mongrel "$LINENO" "boost/spirit/include/classic_core.hpp" "ac_cv_header_boost_spirit_include_classic_core_hpp" "$ac_includes_default"
+if test "x$ac_cv_header_boost_spirit_include_classic_core_hpp" = xyes; then :
-fi
- if test "$with_libaio" = "yes" ; then
- WITH_LIBAIO_TRUE=
- WITH_LIBAIO_FALSE='#'
else
- WITH_LIBAIO_TRUE='#'
- WITH_LIBAIO_FALSE=
+ ac_fn_cxx_check_header_mongrel "$LINENO" "boost/spirit.hpp" "ac_cv_header_boost_spirit_hpp" "$ac_includes_default"
+if test "x$ac_cv_header_boost_spirit_hpp" = xyes; then :
+ use_bspirit_old_hdr=yes
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "\"Can't find boost spirit headers\"
+See \`config.log' for more details" "$LINENO" 5; }
fi
-# use libzfs
+fi
-# Check whether --with-libzfs was given.
-if test "${with_libzfs+set}" = set; then :
- withval=$with_libzfs;
+
+ if test "$use_bspirit_old_hdr" = "yes"; then
+ USE_BOOST_SPIRIT_OLD_HDR_TRUE=
+ USE_BOOST_SPIRIT_OLD_HDR_FALSE='#'
else
- with_libzfs=no
+ USE_BOOST_SPIRIT_OLD_HDR_TRUE='#'
+ USE_BOOST_SPIRIT_OLD_HDR_FALSE=
fi
-if test "x$with_libzfs" = xyes; then :
-pkg_failed=no
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBZFS" >&5
-$as_echo_n "checking for LIBZFS... " >&6; }
+ac_fn_cxx_check_header_mongrel "$LINENO" "boost/random/discrete_distribution.hpp" "ac_cv_header_boost_random_discrete_distribution_hpp" "$ac_includes_default"
+if test "x$ac_cv_header_boost_random_discrete_distribution_hpp" = xyes; then :
+
+$as_echo "#define HAVE_BOOST_RANDOM_DISCRETE_DISTRIBUTION /**/" >>confdefs.h
-if test -n "$LIBZFS_CFLAGS"; then
- pkg_cv_LIBZFS_CFLAGS="$LIBZFS_CFLAGS"
- elif test -n "$PKG_CONFIG"; then
- if test -n "$PKG_CONFIG" && \
- { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zfs\""; } >&5
- ($PKG_CONFIG --exists --print-errors "zfs") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; then
- pkg_cv_LIBZFS_CFLAGS=`$PKG_CONFIG --cflags "zfs" 2>/dev/null`
- test "x$?" != "x0" && pkg_failed=yes
-else
- pkg_failed=yes
-fi
- else
- pkg_failed=untried
-fi
-if test -n "$LIBZFS_LIBS"; then
- pkg_cv_LIBZFS_LIBS="$LIBZFS_LIBS"
- elif test -n "$PKG_CONFIG"; then
- if test -n "$PKG_CONFIG" && \
- { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zfs\""; } >&5
- ($PKG_CONFIG --exists --print-errors "zfs") 2>&5
- ac_status=$?
- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
- test $ac_status = 0; }; then
- pkg_cv_LIBZFS_LIBS=`$PKG_CONFIG --libs "zfs" 2>/dev/null`
- test "x$?" != "x0" && pkg_failed=yes
-else
- pkg_failed=yes
-fi
- else
- pkg_failed=untried
fi
-if test $pkg_failed = yes; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+ac_fn_cxx_check_header_mongrel "$LINENO" "boost/statechart/state.hpp" "ac_cv_header_boost_statechart_state_hpp" "$ac_includes_default"
+if test "x$ac_cv_header_boost_statechart_state_hpp" = xyes; then :
-if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
- _pkg_short_errors_supported=yes
else
- _pkg_short_errors_supported=no
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "\"Can't find boost statechart headers; need 1.34 or later\"
+See \`config.log' for more details" "$LINENO" 5; }
fi
- if test $_pkg_short_errors_supported = yes; then
- LIBZFS_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zfs" 2>&1`
- else
- LIBZFS_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zfs" 2>&1`
- fi
- # Put the nasty error message in config.log where it belongs
- echo "$LIBZFS_PKG_ERRORS" >&5
-
- true
-elif test $pkg_failed = untried; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
- true
-else
- LIBZFS_CFLAGS=$pkg_cv_LIBZFS_CFLAGS
- LIBZFS_LIBS=$pkg_cv_LIBZFS_LIBS
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-fi
-fi
-if test "x$with_libzfs" = xyes; then :
-$as_echo "#define HAVE_LIBZFS 1" >>confdefs.h
+ac_fn_cxx_check_header_mongrel "$LINENO" "boost/program_options/option.hpp" "ac_cv_header_boost_program_options_option_hpp" "$ac_includes_default"
+if test "x$ac_cv_header_boost_program_options_option_hpp" = xyes; then :
-fi
- if test "$with_libzfs" = "yes" ; then
- WITH_LIBZFS_TRUE=
- WITH_LIBZFS_FALSE='#'
else
- WITH_LIBZFS_TRUE='#'
- WITH_LIBZFS_FALSE=
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "\"Can't find boost program_options headers\"
+See \`config.log' for more details" "$LINENO" 5; }
fi
-# Checks for header files.
-ac_header_dirent=no
-for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
- as_ac_Header=`$as_echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_hdr that defines DIR" >&5
-$as_echo_n "checking for $ac_hdr that defines DIR... " >&6; }
-if eval \${$as_ac_Header+:} false; then :
+
+# If we have the boost system library installed, then we may want to link
+# with it.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_system-mt" >&5
+$as_echo_n "checking for main in -lboost_system-mt... " >&6; }
+if ${ac_cv_lib_boost_system_mt_main+:} false; then :
$as_echo_n "(cached) " >&6
else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lboost_system-mt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <sys/types.h>
-#include <$ac_hdr>
+
int
main ()
{
-if ((DIR *) 0)
-return 0;
+return main ();
;
return 0;
}
_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- eval "$as_ac_Header=yes"
+if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_lib_boost_system_mt_main=yes
else
- eval "$as_ac_Header=no"
+ ac_cv_lib_boost_system_mt_main=no
fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-eval ac_res=\$$as_ac_Header
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
-$as_echo "$ac_res" >&6; }
-if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_system_mt_main" >&5
+$as_echo "$ac_cv_lib_boost_system_mt_main" >&6; }
+if test "x$ac_cv_lib_boost_system_mt_main" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define `$as_echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+#define HAVE_LIBBOOST_SYSTEM_MT 1
_ACEOF
-ac_header_dirent=$ac_hdr; break
-fi
+ LIBS="-lboost_system-mt $LIBS"
-done
-# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
-if test $ac_header_dirent = dirent.h; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing opendir" >&5
-$as_echo_n "checking for library containing opendir... " >&6; }
-if ${ac_cv_search_opendir+:} false; then :
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_system" >&5
+$as_echo_n "checking for main in -lboost_system... " >&6; }
+if ${ac_cv_lib_boost_system_main+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_func_search_save_LIBS=$LIBS
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lboost_system $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char opendir ();
+
int
main ()
{
-return opendir ();
+return main ();
;
return 0;
}
_ACEOF
-for ac_lib in '' dir; do
- if test -z "$ac_lib"; then
- ac_res="none required"
- else
- ac_res=-l$ac_lib
- LIBS="-l$ac_lib $ac_func_search_save_LIBS"
- fi
- if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_search_opendir=$ac_res
+if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_lib_boost_system_main=yes
+else
+ ac_cv_lib_boost_system_main=no
fi
rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext
- if ${ac_cv_search_opendir+:} false; then :
- break
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-done
-if ${ac_cv_search_opendir+:} false; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_system_main" >&5
+$as_echo "$ac_cv_lib_boost_system_main" >&6; }
+if test "x$ac_cv_lib_boost_system_main" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBBOOST_SYSTEM 1
+_ACEOF
+
+ LIBS="-lboost_system $LIBS"
else
- ac_cv_search_opendir=no
-fi
-rm conftest.$ac_ext
-LIBS=$ac_func_search_save_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: \"Boost system library not found.\"" >&5
+$as_echo "$as_me: \"Boost system library not found.\"" >&6;}
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5
-$as_echo "$ac_cv_search_opendir" >&6; }
-ac_res=$ac_cv_search_opendir
-if test "$ac_res" != no; then :
- test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
fi
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing opendir" >&5
-$as_echo_n "checking for library containing opendir... " >&6; }
-if ${ac_cv_search_opendir+:} false; then :
+
+# Find the right boost_thread library.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_thread-mt" >&5
+$as_echo_n "checking for main in -lboost_thread-mt... " >&6; }
+if ${ac_cv_lib_boost_thread_mt_main+:} false; then :
$as_echo_n "(cached) " >&6
else
- ac_func_search_save_LIBS=$LIBS
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lboost_thread-mt $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char opendir ();
+
int
main ()
{
-return opendir ();
+return main ();
;
return 0;
}
_ACEOF
-for ac_lib in '' x; do
- if test -z "$ac_lib"; then
- ac_res="none required"
- else
- ac_res=-l$ac_lib
- LIBS="-l$ac_lib $ac_func_search_save_LIBS"
- fi
- if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_search_opendir=$ac_res
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext
- if ${ac_cv_search_opendir+:} false; then :
- break
-fi
-done
-if ${ac_cv_search_opendir+:} false; then :
-
+if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_lib_boost_thread_mt_main=yes
else
- ac_cv_search_opendir=no
-fi
-rm conftest.$ac_ext
-LIBS=$ac_func_search_save_LIBS
+ ac_cv_lib_boost_thread_mt_main=no
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5
-$as_echo "$ac_cv_search_opendir" >&6; }
-ac_res=$ac_cv_search_opendir
-if test "$ac_res" != no; then :
- test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
-
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_thread_mt_main" >&5
+$as_echo "$ac_cv_lib_boost_thread_mt_main" >&6; }
+if test "x$ac_cv_lib_boost_thread_mt_main" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBBOOST_THREAD_MT 1
+_ACEOF
-fi
+ LIBS="-lboost_thread-mt $LIBS"
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5
-$as_echo_n "checking for ANSI C header files... " >&6; }
-if ${ac_cv_header_stdc+:} false; then :
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_thread" >&5
+$as_echo_n "checking for main in -lboost_thread... " >&6; }
+if ${ac_cv_lib_boost_thread_main+:} false; then :
$as_echo_n "(cached) " >&6
else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lboost_thread $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <float.h>
+
int
main ()
{
-
+return main ();
;
return 0;
}
_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- ac_cv_header_stdc=yes
-else
- ac_cv_header_stdc=no
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-
-if test $ac_cv_header_stdc = yes; then
- # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <string.h>
-
-_ACEOF
-if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
- $EGREP "memchr" >/dev/null 2>&1; then :
-
+if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_lib_boost_thread_main=yes
else
- ac_cv_header_stdc=no
+ ac_cv_lib_boost_thread_main=no
fi
-rm -f conftest*
-
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-
-if test $ac_cv_header_stdc = yes; then
- # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <stdlib.h>
-
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_thread_main" >&5
+$as_echo "$ac_cv_lib_boost_thread_main" >&6; }
+if test "x$ac_cv_lib_boost_thread_main" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBBOOST_THREAD 1
_ACEOF
-if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
- $EGREP "free" >/dev/null 2>&1; then :
+
+ LIBS="-lboost_thread $LIBS"
else
- ac_cv_header_stdc=no
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "\"Boost thread library not found.\"
+See \`config.log' for more details" "$LINENO" 5; }
fi
-rm -f conftest*
fi
-if test $ac_cv_header_stdc = yes; then
- # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
- if test "$cross_compiling" = yes; then :
- :
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <ctype.h>
-#include <stdlib.h>
-#if ((' ' & 0x0FF) == 0x020)
-# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
-# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
-#else
-# define ISLOWER(c) \
- (('a' <= (c) && (c) <= 'i') \
- || ('j' <= (c) && (c) <= 'r') \
- || ('s' <= (c) && (c) <= 'z'))
-# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
-#endif
-
-#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+
+#
+# Check for boost_program_options library (defines BOOST_PROGRAM_OPTIONS_LIBS).
+#
+BOOST_PROGRAM_OPTIONS_LIBS=""
+saved_LIBS="${LIBS}"
+LIBS=""
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_program_options-mt" >&5
+$as_echo_n "checking for main in -lboost_program_options-mt... " >&6; }
+if ${ac_cv_lib_boost_program_options_mt_main+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lboost_program_options-mt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+
int
main ()
{
- int i;
- for (i = 0; i < 256; i++)
- if (XOR (islower (i), ISLOWER (i))
- || toupper (i) != TOUPPER (i))
- return 2;
+return main ();
+ ;
return 0;
}
_ACEOF
-if ac_fn_c_try_run "$LINENO"; then :
-
+if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_lib_boost_program_options_mt_main=yes
else
- ac_cv_header_stdc=no
-fi
-rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
- conftest.$ac_objext conftest.beam conftest.$ac_ext
-fi
-
+ ac_cv_lib_boost_program_options_mt_main=no
fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5
-$as_echo "$ac_cv_header_stdc" >&6; }
-if test $ac_cv_header_stdc = yes; then
-
-$as_echo "#define STDC_HEADERS 1" >>confdefs.h
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_program_options_mt_main" >&5
+$as_echo "$ac_cv_lib_boost_program_options_mt_main" >&6; }
+if test "x$ac_cv_lib_boost_program_options_mt_main" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBBOOST_PROGRAM_OPTIONS_MT 1
+_ACEOF
-fi
+ LIBS="-lboost_program_options-mt $LIBS"
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for sys/wait.h that is POSIX.1 compatible" >&5
-$as_echo_n "checking for sys/wait.h that is POSIX.1 compatible... " >&6; }
-if ${ac_cv_header_sys_wait_h+:} false; then :
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_program_options" >&5
+$as_echo_n "checking for main in -lboost_program_options... " >&6; }
+if ${ac_cv_lib_boost_program_options_main+:} false; then :
$as_echo_n "(cached) " >&6
else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lboost_program_options $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <sys/types.h>
-#include <sys/wait.h>
-#ifndef WEXITSTATUS
-# define WEXITSTATUS(stat_val) ((unsigned int) (stat_val) >> 8)
-#endif
-#ifndef WIFEXITED
-# define WIFEXITED(stat_val) (((stat_val) & 255) == 0)
-#endif
+
int
main ()
{
- int s;
- wait (&s);
- s = WIFEXITED (s) ? WEXITSTATUS (s) : 1;
+return main ();
;
return 0;
}
_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- ac_cv_header_sys_wait_h=yes
+if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_lib_boost_program_options_main=yes
else
- ac_cv_header_sys_wait_h=no
+ ac_cv_lib_boost_program_options_main=no
fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_sys_wait_h" >&5
-$as_echo "$ac_cv_header_sys_wait_h" >&6; }
-if test $ac_cv_header_sys_wait_h = yes; then
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_program_options_main" >&5
+$as_echo "$ac_cv_lib_boost_program_options_main" >&6; }
+if test "x$ac_cv_lib_boost_program_options_main" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBBOOST_PROGRAM_OPTIONS 1
+_ACEOF
-$as_echo "#define HAVE_SYS_WAIT_H 1" >>confdefs.h
+ LIBS="-lboost_program_options $LIBS"
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "\"Boost program options library not found.\"
+See \`config.log' for more details" "$LINENO" 5; }
fi
+fi
+BOOST_PROGRAM_OPTIONS_LIBS="${LIBS}"
+LIBS="${saved_LIBS}"
-# spirit?
-ac_ext=cpp
-ac_cpp='$CXXCPP $CPPFLAGS'
-ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
-ac_fn_cxx_check_header_mongrel "$LINENO" "boost/spirit/include/classic_core.hpp" "ac_cv_header_boost_spirit_include_classic_core_hpp" "$ac_includes_default"
-if test "x$ac_cv_header_boost_spirit_include_classic_core_hpp" = xyes; then :
+
+ac_fn_c_check_member "$LINENO" "struct fiemap_extent" "fe_logical" "ac_cv_member_struct_fiemap_extent_fe_logical" "#include <linux/fiemap.h>
+"
+if test "x$ac_cv_member_struct_fiemap_extent_fe_logical" = xyes; then :
+
+$as_echo "#define HAVE_FIEMAP_H /**/" >>confdefs.h
else
- ac_fn_cxx_check_header_mongrel "$LINENO" "boost/spirit.hpp" "ac_cv_header_boost_spirit_hpp" "$ac_includes_default"
-if test "x$ac_cv_header_boost_spirit_hpp" = xyes; then :
- use_bspirit_old_hdr=yes
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "\"Can't find boost spirit headers\"
-See \`config.log' for more details" "$LINENO" 5; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: linux/fiemap.h was not found or not usable; using local Ceph copy" >&5
+$as_echo "$as_me: linux/fiemap.h was not found or not usable; using local Ceph copy" >&6;}
fi
+for ac_header in \
+ arpa/inet.h \
+ linux/version.h \
+ netdb.h \
+ netinet/in.h \
+ sys/file.h \
+ sys/ioctl.h \
+ sys/mount.h \
+ sys/param.h \
+ sys/socket.h \
+ sys/statvfs.h \
+ sys/time.h \
+ sys/vfs.h \
+ sys/xattr.h \
+ syslog.h \
+ utime.h \
+
+do :
+ as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
+ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
+if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
+ cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
fi
+done
+
+
+# sync_file_range
+ac_fn_c_check_func "$LINENO" "sync_file_range" "ac_cv_func_sync_file_range"
+if test "x$ac_cv_func_sync_file_range" = xyes; then :
+
+$as_echo "#define HAVE_SYNC_FILE_RANGE /**/" >>confdefs.h
- if test "$use_bspirit_old_hdr" = "yes"; then
- USE_BOOST_SPIRIT_OLD_HDR_TRUE=
- USE_BOOST_SPIRIT_OLD_HDR_FALSE='#'
-else
- USE_BOOST_SPIRIT_OLD_HDR_TRUE='#'
- USE_BOOST_SPIRIT_OLD_HDR_FALSE=
fi
-ac_fn_cxx_check_header_mongrel "$LINENO" "boost/random/discrete_distribution.hpp" "ac_cv_header_boost_random_discrete_distribution_hpp" "$ac_includes_default"
-if test "x$ac_cv_header_boost_random_discrete_distribution_hpp" = xyes; then :
+# fallocate
+ac_fn_c_check_func "$LINENO" "fallocate" "ac_cv_func_fallocate"
+if test "x$ac_cv_func_fallocate" = xyes; then :
-$as_echo "#define HAVE_BOOST_RANDOM_DISCRETE_DISTRIBUTION /**/" >>confdefs.h
+$as_echo "#define CEPH_HAVE_FALLOCATE /**/" >>confdefs.h
fi
+#
+# Test for time-related `struct stat` members.
+#
-ac_fn_cxx_check_header_mongrel "$LINENO" "boost/statechart/state.hpp" "ac_cv_header_boost_statechart_state_hpp" "$ac_includes_default"
-if test "x$ac_cv_header_boost_statechart_state_hpp" = xyes; then :
+ac_fn_c_check_member "$LINENO" "struct stat" "st_mtim.tv_nsec" "ac_cv_member_struct_stat_st_mtim_tv_nsec" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_mtim_tv_nsec" = xyes; then :
+
+$as_echo "#define HAVE_STAT_ST_MTIM_TV_NSEC 1" >>confdefs.h
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "\"Can't find boost statechart headers; need 1.34 or later\"
-See \`config.log' for more details" "$LINENO" 5; }
fi
-ac_fn_cxx_check_header_mongrel "$LINENO" "boost/program_options/option.hpp" "ac_cv_header_boost_program_options_option_hpp" "$ac_includes_default"
-if test "x$ac_cv_header_boost_program_options_option_hpp" = xyes; then :
+ac_fn_c_check_member "$LINENO" "struct stat" "st_mtimespec.tv_nsec" "ac_cv_member_struct_stat_st_mtimespec_tv_nsec" "$ac_includes_default"
+if test "x$ac_cv_member_struct_stat_st_mtimespec_tv_nsec" = xyes; then :
+
+$as_echo "#define HAVE_STAT_ST_MTIMESPEC_TV_NSEC 1" >>confdefs.h
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "\"Can't find boost program_options headers\"
-See \`config.log' for more details" "$LINENO" 5; }
fi
+# splice/tee
+ac_fn_c_check_func "$LINENO" "splice" "ac_cv_func_splice"
+if test "x$ac_cv_func_splice" = xyes; then :
+
+$as_echo "#define CEPH_HAVE_SPLICE /**/" >>confdefs.h
+
+fi
+
+
+
+for ac_header in arpa/nameser_compat.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "arpa/nameser_compat.h" "ac_cv_header_arpa_nameser_compat_h" "$ac_includes_default"
+if test "x$ac_cv_header_arpa_nameser_compat_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_ARPA_NAMESER_COMPAT_H 1
+_ACEOF
+
+fi
+
+done
+
-# If we have the boost system library installed, then we may want to link
-# with it.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_system-mt" >&5
-$as_echo_n "checking for main in -lboost_system-mt... " >&6; }
-if ${ac_cv_lib_boost_system_mt_main+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lboost_system-mt $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
+#include <fcntl.h>
+F_SETPIPE_SZ
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+$as_echo "#define CEPH_HAVE_SETPIPE_SZ /**/" >>confdefs.h
-int
-main ()
-{
-return main ();
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_link "$LINENO"; then :
- ac_cv_lib_boost_system_mt_main=yes
else
- ac_cv_lib_boost_system_mt_main=no
+ { $as_echo "$as_me:${as_lineno-$LINENO}: \"F_SETPIPE_SZ not found, zero-copy may be less efficent\"" >&5
+$as_echo "$as_me: \"F_SETPIPE_SZ not found, zero-copy may be less efficent\"" >&6;}
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+
+for ac_header in arpa/nameser_compat.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "arpa/nameser_compat.h" "ac_cv_header_arpa_nameser_compat_h" "$ac_includes_default"
+if test "x$ac_cv_header_arpa_nameser_compat_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_ARPA_NAMESER_COMPAT_H 1
+_ACEOF
+
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_system_mt_main" >&5
-$as_echo "$ac_cv_lib_boost_system_mt_main" >&6; }
-if test "x$ac_cv_lib_boost_system_mt_main" = xyes; then :
+
+done
+
+for ac_func in posix_fallocate
+do :
+ ac_fn_c_check_func "$LINENO" "posix_fallocate" "ac_cv_func_posix_fallocate"
+if test "x$ac_cv_func_posix_fallocate" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBBOOST_SYSTEM_MT 1
+#define HAVE_POSIX_FALLOCATE 1
_ACEOF
- LIBS="-lboost_system-mt $LIBS"
+fi
+done
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_system" >&5
-$as_echo_n "checking for main in -lboost_system... " >&6; }
-if ${ac_cv_lib_boost_system_main+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lboost_system $LIBS"
+for ac_header in sys/prctl.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/prctl.h" "ac_cv_header_sys_prctl_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_prctl_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_SYS_PRCTL_H 1
+_ACEOF
+
+fi
+
+done
+
+for ac_func in prctl
+do :
+ ac_fn_c_check_func "$LINENO" "prctl" "ac_cv_func_prctl"
+if test "x$ac_cv_func_prctl" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_PRCTL 1
+_ACEOF
+
+fi
+done
+
+for ac_func in pipe2
+do :
+ ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2"
+if test "x$ac_cv_func_pipe2" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_PIPE2 1
+_ACEOF
+
+fi
+done
+
+for ac_func in posix_fadvise
+do :
+ ac_fn_c_check_func "$LINENO" "posix_fadvise" "ac_cv_func_posix_fadvise"
+if test "x$ac_cv_func_posix_fadvise" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_POSIX_FADVISE 1
+_ACEOF
+
+fi
+done
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for fdatasync" >&5
+$as_echo_n "checking for fdatasync... " >&6; }
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
+#include <unistd.h>
int
main ()
{
-return main ();
+
+#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
+return fdatasync(0);
+#else
+#error Not supported
+#endif
+
;
return 0;
}
_ACEOF
-if ac_fn_cxx_try_link "$LINENO"; then :
- ac_cv_lib_boost_system_main=yes
+if ac_fn_c_try_compile "$LINENO"; then :
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+$as_echo "#define HAVE_FDATASYNC 1" >>confdefs.h
+
+
else
- ac_cv_lib_boost_system_main=no
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+#
+# Check for pthread spinlock (depends on ACX_PTHREAD)
+#
+saved_LIBS="$LIBS"
+saved_CFLAGS="$CFLAGS"
+LIBS="$PTHREAD_LIBS $LIBS"
+CFLAGS="$PTHREAD_CFLAGS $CFLAGS"
+ac_fn_c_check_func "$LINENO" "pthread_spin_init" "ac_cv_func_pthread_spin_init"
+if test "x$ac_cv_func_pthread_spin_init" = xyes; then :
+
+$as_echo "#define HAVE_PTHREAD_SPINLOCK 1" >>confdefs.h
+
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_system_main" >&5
-$as_echo "$ac_cv_lib_boost_system_main" >&6; }
-if test "x$ac_cv_lib_boost_system_main" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBBOOST_SYSTEM 1
+
+LIBS="$saved_LIBS"
+CFLAGS="$saved_CFLAGS"
+
+ac_fn_c_check_type "$LINENO" "int8_t" "ac_cv_type_int8_t" "$ac_includes_default"
+if test "x$ac_cv_type_int8_t" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_INT8_T 1
_ACEOF
- LIBS="-lboost_system $LIBS"
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: \"Boost system library not found.\"" >&5
-$as_echo "$as_me: \"Boost system library not found.\"" >&6;}
fi
+ac_fn_c_check_type "$LINENO" "uint8_t" "ac_cv_type_uint8_t" "$ac_includes_default"
+if test "x$ac_cv_type_uint8_t" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_UINT8_T 1
+_ACEOF
+
fi
+ac_fn_c_check_type "$LINENO" "int16_t" "ac_cv_type_int16_t" "$ac_includes_default"
+if test "x$ac_cv_type_int16_t" = xyes; then :
+cat >>confdefs.h <<_ACEOF
+#define HAVE_INT16_T 1
+_ACEOF
-# Find the right boost_thread library.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_thread-mt" >&5
-$as_echo_n "checking for main in -lboost_thread-mt... " >&6; }
-if ${ac_cv_lib_boost_thread_mt_main+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lboost_thread-mt $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+fi
+ac_fn_c_check_type "$LINENO" "uint16_t" "ac_cv_type_uint16_t" "$ac_includes_default"
+if test "x$ac_cv_type_uint16_t" = xyes; then :
-int
-main ()
-{
-return main ();
- ;
- return 0;
-}
+cat >>confdefs.h <<_ACEOF
+#define HAVE_UINT16_T 1
_ACEOF
-if ac_fn_cxx_try_link "$LINENO"; then :
- ac_cv_lib_boost_thread_mt_main=yes
-else
- ac_cv_lib_boost_thread_mt_main=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+
+
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_thread_mt_main" >&5
-$as_echo "$ac_cv_lib_boost_thread_mt_main" >&6; }
-if test "x$ac_cv_lib_boost_thread_mt_main" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBBOOST_THREAD_MT 1
+ac_fn_c_check_type "$LINENO" "int32_t" "ac_cv_type_int32_t" "$ac_includes_default"
+if test "x$ac_cv_type_int32_t" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_INT32_T 1
_ACEOF
- LIBS="-lboost_thread-mt $LIBS"
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_thread" >&5
-$as_echo_n "checking for main in -lboost_thread... " >&6; }
-if ${ac_cv_lib_boost_thread_main+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lboost_thread $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
+fi
+ac_fn_c_check_type "$LINENO" "uint32_t" "ac_cv_type_uint32_t" "$ac_includes_default"
+if test "x$ac_cv_type_uint32_t" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_UINT32_T 1
+_ACEOF
-int
-main ()
-{
-return main ();
- ;
- return 0;
-}
+fi
+ac_fn_c_check_type "$LINENO" "int64_t" "ac_cv_type_int64_t" "$ac_includes_default"
+if test "x$ac_cv_type_int64_t" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_INT64_T 1
_ACEOF
-if ac_fn_cxx_try_link "$LINENO"; then :
- ac_cv_lib_boost_thread_main=yes
-else
- ac_cv_lib_boost_thread_main=no
+
+
fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
+ac_fn_c_check_type "$LINENO" "uint64_t" "ac_cv_type_uint64_t" "$ac_includes_default"
+if test "x$ac_cv_type_uint64_t" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_UINT64_T 1
+_ACEOF
+
+
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_thread_main" >&5
-$as_echo "$ac_cv_lib_boost_thread_main" >&6; }
-if test "x$ac_cv_lib_boost_thread_main" = xyes; then :
+
+
+for ac_header in linux/types.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "linux/types.h" "ac_cv_header_linux_types_h" "$ac_includes_default"
+if test "x$ac_cv_header_linux_types_h" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBBOOST_THREAD 1
+#define HAVE_LINUX_TYPES_H 1
_ACEOF
- LIBS="-lboost_thread $LIBS"
-
-else
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "\"Boost thread library not found.\"
-See \`config.log' for more details" "$LINENO" 5; }
fi
+done
+
+ac_fn_c_check_type "$LINENO" "__u8" "ac_cv_type___u8" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___u8" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___U8 1
+_ACEOF
+
+
fi
+ac_fn_c_check_type "$LINENO" "__s8" "ac_cv_type___s8" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___s8" = xyes; then :
+cat >>confdefs.h <<_ACEOF
+#define HAVE___S8 1
+_ACEOF
-ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
+fi
+ac_fn_c_check_type "$LINENO" "__u16" "ac_cv_type___u16" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___u16" = xyes; then :
-ac_fn_c_check_member "$LINENO" "struct fiemap_extent" "fe_logical" "ac_cv_member_struct_fiemap_extent_fe_logical" "#include <linux/fiemap.h>
+cat >>confdefs.h <<_ACEOF
+#define HAVE___U16 1
+_ACEOF
+
+
+fi
+ac_fn_c_check_type "$LINENO" "__s16" "ac_cv_type___s16" "#include <linux/types.h>
"
-if test "x$ac_cv_member_struct_fiemap_extent_fe_logical" = xyes; then :
+if test "x$ac_cv_type___s16" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___S16 1
+_ACEOF
-$as_echo "#define HAVE_FIEMAP_H /**/" >>confdefs.h
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: linux/fiemap.h was not found or not usable; using local Ceph copy" >&5
-$as_echo "$as_me: linux/fiemap.h was not found or not usable; using local Ceph copy" >&6;}
fi
+ac_fn_c_check_type "$LINENO" "__u32" "ac_cv_type___u32" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___u32" = xyes; then :
+cat >>confdefs.h <<_ACEOF
+#define HAVE___U32 1
+_ACEOF
-for ac_header in sys/xattr.h arpa/inet.h netdb.h netinet/in.h sys/file.h sys/ioctl.h sys/mount.h sys/param.h sys/socket.h sys/statvfs.h sys/time.h sys/vfs.h syslog.h utime.h
-do :
- as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
-ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
-if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
- cat >>confdefs.h <<_ACEOF
-#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+
+fi
+ac_fn_c_check_type "$LINENO" "__s32" "ac_cv_type___s32" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___s32" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___S32 1
_ACEOF
+
fi
+ac_fn_c_check_type "$LINENO" "__u64" "ac_cv_type___u64" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___u64" = xyes; then :
-done
+cat >>confdefs.h <<_ACEOF
+#define HAVE___U64 1
+_ACEOF
-# sync_file_range
-ac_fn_c_check_func "$LINENO" "sync_file_range" "ac_cv_func_sync_file_range"
-if test "x$ac_cv_func_sync_file_range" = xyes; then :
+fi
+ac_fn_c_check_type "$LINENO" "__s64" "ac_cv_type___s64" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___s64" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___S64 1
+_ACEOF
-$as_echo "#define HAVE_SYNC_FILE_RANGE /**/" >>confdefs.h
fi
+ac_fn_c_check_type "$LINENO" "__le16" "ac_cv_type___le16" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___le16" = xyes; then :
+cat >>confdefs.h <<_ACEOF
+#define HAVE___LE16 1
+_ACEOF
-# fallocate
-ac_fn_c_check_func "$LINENO" "fallocate" "ac_cv_func_fallocate"
-if test "x$ac_cv_func_fallocate" = xyes; then :
-$as_echo "#define CEPH_HAVE_FALLOCATE /**/" >>confdefs.h
+fi
+ac_fn_c_check_type "$LINENO" "__be16" "ac_cv_type___be16" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___be16" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___BE16 1
+_ACEOF
+
fi
+ac_fn_c_check_type "$LINENO" "__le32" "ac_cv_type___le32" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___le32" = xyes; then :
+cat >>confdefs.h <<_ACEOF
+#define HAVE___LE32 1
+_ACEOF
-for ac_header in sys/prctl.h
-do :
- ac_fn_c_check_header_mongrel "$LINENO" "sys/prctl.h" "ac_cv_header_sys_prctl_h" "$ac_includes_default"
-if test "x$ac_cv_header_sys_prctl_h" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_SYS_PRCTL_H 1
+fi
+ac_fn_c_check_type "$LINENO" "__be32" "ac_cv_type___be32" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___be32" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___BE32 1
_ACEOF
+
fi
+ac_fn_c_check_type "$LINENO" "__le64" "ac_cv_type___le64" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___le64" = xyes; then :
-done
+cat >>confdefs.h <<_ACEOF
+#define HAVE___LE64 1
+_ACEOF
-for ac_func in prctl
-do :
- ac_fn_c_check_func "$LINENO" "prctl" "ac_cv_func_prctl"
-if test "x$ac_cv_func_prctl" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_PRCTL 1
+
+fi
+ac_fn_c_check_type "$LINENO" "__be64" "ac_cv_type___be64" "#include <linux/types.h>
+"
+if test "x$ac_cv_type___be64" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE___BE64 1
_ACEOF
+
fi
-done
# Checks for typedefs, structures, and compiler characteristics.
@@ -20117,14 +21069,6 @@ LIBOBJS=$ac_libobjs
LTLIBOBJS=$ac_ltlibobjs
- if test -n "$EXEEXT"; then
- am__EXEEXT_TRUE=
- am__EXEEXT_FALSE='#'
-else
- am__EXEEXT_TRUE='#'
- am__EXEEXT_FALSE=
-fi
-
if test -z "${AMDEP_TRUE}" && test -z "${AMDEP_FALSE}"; then
as_fn_error $? "conditional \"AMDEP\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
@@ -20133,6 +21077,14 @@ if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then
as_fn_error $? "conditional \"am__fastdepCC\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
+ if test -n "$EXEEXT"; then
+ am__EXEEXT_TRUE=
+ am__EXEEXT_FALSE='#'
+else
+ am__EXEEXT_TRUE='#'
+ am__EXEEXT_FALSE=
+fi
+
if test -z "${am__fastdepCCAS_TRUE}" && test -z "${am__fastdepCCAS_FALSE}"; then
as_fn_error $? "conditional \"am__fastdepCCAS\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
@@ -20654,7 +21606,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by ceph $as_me 0.72.2, which was
+This file was extended by ceph $as_me 0.77, which was
generated by GNU Autoconf 2.68. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -20720,7 +21672,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-ceph config.status 0.72.2
+ceph config.status 0.77
configured by $0, generated by GNU Autoconf 2.68,
with options \\"\$ac_cs_config\\"
diff --git a/configure.ac b/configure.ac
index 30ea52d..5da65fc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -8,7 +8,7 @@ AC_PREREQ(2.59)
# VERSION define is not used by the code. It gets a version string
# from 'git describe'; see src/ceph_ver.[ch]
-AC_INIT([ceph], [0.72.2], [ceph-devel at vger.kernel.org])
+AC_INIT([ceph], [0.77], [ceph-devel at vger.kernel.org])
# Create release string. Used with VERSION for RPMs.
RPM_RELEASE=0
@@ -29,6 +29,9 @@ AC_CONFIG_SUBDIRS([src/gtest])
AC_CANONICAL_HOST
AC_CANONICAL_TARGET
+# Fix automake problems in 1.12
+m4_ifdef([AM_PROG_AR], [AM_PROG_AR])
+
# Automake
AM_INIT_AUTOMAKE
AM_PROG_CC_C_O
@@ -61,7 +64,16 @@ if test "$CXX" = no || test "$CXX:$GXX" = "g++:"; then
AC_MSG_ERROR([no C++ compiler found])
fi
-AM_CONDITIONAL(CLANG, test x"$CXX" = x"clang++")
+AC_MSG_CHECKING([if compiler is clang])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [[
+#ifndef __clang__
+#error "Not Clang"
+#endif
+return 0;
+]])],
+[CLANG=yes], [CLANG=no])
+AC_MSG_RESULT([$CLANG])
+AM_CONDITIONAL(CLANG, test "$CLANG" = "yes")
#AC_PROG_CC
AC_PROG_MAKE_SET
@@ -113,14 +125,60 @@ AM_CONDITIONAL([ENABLE_FPU_NEON], [case $target_cpu in arm*) true;; *) false;; e
AX_CHECK_COMPILE_FLAG([-fvar-tracking-assignments], [HAS_VTA_SUPPORT=1], [HAS_VTA_SUPPORT=0])
AM_CONDITIONAL(COMPILER_HAS_VTA, [test "$HAS_VTA_SUPPORT" = 1])
+AX_CXX_STATIC_CAST
+AX_C_VAR_FUNC
+AX_C_PRETTY_FUNC
+
# Checks for libraries.
ACX_PTHREAD
+
AC_CHECK_LIB([uuid], [uuid_parse], [true], AC_MSG_FAILURE([libuuid not found]))
-AC_CHECK_LIB([m], [pow], [true], AC_MSG_FAILURE([libm not found]))
-AC_CHECK_LIB([resolv], [__res_nquery], [true], AC_MSG_FAILURE([libresolv not found]))
+
if test x"$linux" = x"yes"; then
- AC_CHECK_LIB([keyutils], [add_key], [true], AC_MSG_FAILURE([libkeyutils not found]))
+ AC_CHECK_LIB([blkid], [blkid_devno_to_wholedisk], [true], AC_MSG_FAILURE([libblkid not found]))
+fi
+
+#
+# Check for res_nquery in libresolv. There are several variations. On OSX
+# res_nquery is a macro defined in resolv.h, so the typical AC_CHECK_LIB
+# doesn't work. On FreeBSD res_nquery can be found in libc. The required
+# library for linking (if any) is defined RESOLV_LIBS.
+#
+AC_CHECK_HEADER([resolv.h], [], [], [#include <netinet/in.h>])
+
+AC_DEFUN([CHECK_RESOLV_LIBS], [{
+ AC_MSG_CHECKING([if res_nquery will link (LIBS=$1)])
+ saved_LIBS="${LIBS}"
+ LIBS="$1"
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+ #include <netinet/in.h>
+ #include <resolv.h>
+ ]], [[res_nquery(0, 0, 0, 0, 0, 0);]])],
+ [AC_MSG_RESULT([yes])
+ [$2]],
+ AC_MSG_RESULT([no]))
+ LIBS="${saved_LIBS}"
+}])
+
+RESOLV_LIBS=""
+CHECK_RESOLV_LIBS([$RESOLV_LIBS], [resolv_libs="ok"])
+if test x"$resolv_libs" != "xok"; then
+ RESOLV_LIBS="-lresolv"
+ CHECK_RESOLV_LIBS([$RESOLV_LIBS], [resolv_libs="ok"])
+ if test x"$resolv_libs" != "xok"; then
+ AC_MSG_FAILURE([no resolv library found])
+ fi
fi
+AC_SUBST([RESOLV_LIBS])
+
+dnl check for libkeyutils on linux
+KEYUTILS_LIB=""
+AS_IF([test x"$linux" = x"yes"], [
+ AC_CHECK_LIB([keyutils], [add_key], [KEYUTILS_LIB="-lkeyutils"], [
+ AC_MSG_FAILURE([libkeyutils not found])])])
+AC_SUBST(KEYUTILS_LIB)
+
+AC_CHECK_LIB([m], [pow], [true], AC_MSG_FAILURE([libm not found]))
AC_CHECK_FUNCS([syncfs], AC_DEFINE([HAVE_SYS_SYNCFS], [1], [we have syncfs]), [])
# Find some crypto library for us to use, while letting user to decide which one to use.
@@ -516,6 +574,19 @@ AC_CHECK_LIB(boost_thread-mt, main, [],
[AC_CHECK_LIB(boost_thread, main, [],
AC_MSG_FAILURE(["Boost thread library not found."]))])
+#
+# Check for boost_program_options library (defines BOOST_PROGRAM_OPTIONS_LIBS).
+#
+BOOST_PROGRAM_OPTIONS_LIBS=""
+saved_LIBS="${LIBS}"
+LIBS=""
+AC_CHECK_LIB(boost_program_options-mt, main, [],
+ [AC_CHECK_LIB(boost_program_options, main, [],
+ AC_MSG_FAILURE(["Boost program options library not found."]))])
+BOOST_PROGRAM_OPTIONS_LIBS="${LIBS}"
+LIBS="${saved_LIBS}"
+AC_SUBST(BOOST_PROGRAM_OPTIONS_LIBS)
+
AC_LANG([C])
AC_CHECK_MEMBER([struct fiemap_extent.fe_logical],
@@ -523,7 +594,23 @@ AC_CHECK_MEMBER([struct fiemap_extent.fe_logical],
[AC_MSG_NOTICE([linux/fiemap.h was not found or not usable; using local Ceph copy])],
[[#include <linux/fiemap.h>]])
-AC_CHECK_HEADERS([sys/xattr.h arpa/inet.h netdb.h netinet/in.h sys/file.h sys/ioctl.h sys/mount.h sys/param.h sys/socket.h sys/statvfs.h sys/time.h sys/vfs.h syslog.h utime.h])
+AC_CHECK_HEADERS([ \
+ arpa/inet.h \
+ linux/version.h \
+ netdb.h \
+ netinet/in.h \
+ sys/file.h \
+ sys/ioctl.h \
+ sys/mount.h \
+ sys/param.h \
+ sys/socket.h \
+ sys/statvfs.h \
+ sys/time.h \
+ sys/vfs.h \
+ sys/xattr.h \
+ syslog.h \
+ utime.h \
+])
# sync_file_range
AC_CHECK_FUNC([sync_file_range],
@@ -535,9 +622,74 @@ AC_CHECK_FUNC([fallocate],
[AC_DEFINE([CEPH_HAVE_FALLOCATE], [], [fallocate(2) is supported])],
[])
+#
+# Test for time-related `struct stat` members.
+#
+
+AC_CHECK_MEMBER([struct stat.st_mtim.tv_nsec],
+ [AC_DEFINE(HAVE_STAT_ST_MTIM_TV_NSEC, 1,
+ [Define if you have struct stat.st_mtim.tv_nsec])])
+
+AC_CHECK_MEMBER([struct stat.st_mtimespec.tv_nsec],
+ [AC_DEFINE(HAVE_STAT_ST_MTIMESPEC_TV_NSEC, 1,
+ [Define if you have struct stat.st_mtimespec.tv_nsec])])
+
+# splice/tee
+AC_CHECK_FUNC([splice],
+ [AC_DEFINE([CEPH_HAVE_SPLICE], [], [splice(2) is supported])],
+ [])
+
+AC_CHECK_HEADERS([arpa/nameser_compat.h])
+
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[#include <fcntl.h>
+F_SETPIPE_SZ]])],
+ [AC_DEFINE([CEPH_HAVE_SETPIPE_SZ], [], [F_SETPIPE_SZ is supported])],
+ [AC_MSG_NOTICE(["F_SETPIPE_SZ not found, zero-copy may be less efficent"])])
+
+
+AC_CHECK_HEADERS([arpa/nameser_compat.h])
+AC_CHECK_FUNCS([posix_fallocate])
AC_CHECK_HEADERS([sys/prctl.h])
AC_CHECK_FUNCS([prctl])
+AC_CHECK_FUNCS([pipe2])
+AC_CHECK_FUNCS([posix_fadvise])
+
+AC_MSG_CHECKING([for fdatasync])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#include <unistd.h>
+]], [[
+#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
+return fdatasync(0);
+#else
+#error Not supported
+#endif
+]])], [
+AC_MSG_RESULT([yes])
+AC_DEFINE([HAVE_FDATASYNC], 1, [Define to 1 if you have fdatasync.])
+], [
+AC_MSG_RESULT([no])
+])
+
+#
+# Check for pthread spinlock (depends on ACX_PTHREAD)
+#
+saved_LIBS="$LIBS"
+saved_CFLAGS="$CFLAGS"
+LIBS="$PTHREAD_LIBS $LIBS"
+CFLAGS="$PTHREAD_CFLAGS $CFLAGS"
+AC_CHECK_FUNC([pthread_spin_init],
+ [AC_DEFINE(HAVE_PTHREAD_SPINLOCK, 1, [Define if you have pthread_spin_init])])
+LIBS="$saved_LIBS"
+CFLAGS="$saved_CFLAGS"
+
+AC_CHECK_TYPES([int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t,
+ int64_t, uint64_t])
+
+dnl check for Linux types
+AC_CHECK_HEADERS([linux/types.h])
+AC_CHECK_TYPES([__u8, __s8, __u16, __s16, __u32, __s32, __u64, __s64, __le16,
+ __be16, __le32, __be32, __le64, __be64], [], [], [[#include <linux/types.h>]])
# Checks for typedefs, structures, and compiler characteristics.
#AC_HEADER_STDBOOL
diff --git a/m4/ax_c_pretty_func.m4 b/m4/ax_c_pretty_func.m4
new file mode 100644
index 0000000..ad76709
--- /dev/null
+++ b/m4/ax_c_pretty_func.m4
@@ -0,0 +1,18 @@
+#
+# Test for C compiler support of __PRETTY_FUNCTION__
+#
+# - Adapted from ax_c_var_func (Noah Watkins)
+#
+
+AU_ALIAS([AC_C_PRETTY_FUNC], [AX_C_PRETTY_FUNC])
+AC_DEFUN([AX_C_PRETTY_FUNC],
+[AC_REQUIRE([AC_PROG_CC])
+AC_CACHE_CHECK(whether $CC recognizes __PRETTY_FUNCTION__, ac_cv_c_pretty_func,
+AC_TRY_COMPILE(,
+[
+char *s = __PRETTY_FUNCTION__;
+],
+AC_DEFINE(HAVE_PRETTY_FUNC,,
+[Define if the C complier supports __PRETTY_FUNCTION__]) ac_cv_c_pretty_func=yes,
+ac_cv_c_pretty_func=no) )
+])dnl
diff --git a/m4/ax_c_var_func.m4 b/m4/ax_c_var_func.m4
new file mode 100644
index 0000000..8b57563
--- /dev/null
+++ b/m4/ax_c_var_func.m4
@@ -0,0 +1,66 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_c_var_func.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_C_VAR_FUNC
+#
+# DESCRIPTION
+#
+# This macro tests if the C complier supports the C9X standard __func__
+# indentifier.
+#
+# The new C9X standard for the C language stipulates that the identifier
+# __func__ shall be implictly declared by the compiler as if, immediately
+# following the opening brace of each function definition, the declaration
+#
+# static const char __func__[] = "function-name";
+#
+# appeared, where function-name is the name of the function where the
+# __func__ identifier is used.
+#
+# LICENSE
+#
+# Copyright (c) 2008 Christopher Currie <christopher at currie.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# As a special exception, the respective Autoconf Macro's copyright owner
+# gives unlimited permission to copy, distribute and modify the configure
+# scripts that are the output of Autoconf when processing the Macro. You
+# need not follow the terms of the GNU General Public License when using
+# or distributing such scripts, even though portions of the text of the
+# Macro appear in them. The GNU General Public License (GPL) does govern
+# all other use of the material that constitutes the Autoconf Macro.
+#
+# This special exception to the GPL applies to versions of the Autoconf
+# Macro released by the Autoconf Archive. When you make and distribute a
+# modified version of the Autoconf Macro, you may extend this special
+# exception to the GPL to apply to your modified version as well.
+
+#serial 5
+
+AU_ALIAS([AC_C_VAR_FUNC], [AX_C_VAR_FUNC])
+AC_DEFUN([AX_C_VAR_FUNC],
+[AC_REQUIRE([AC_PROG_CC])
+AC_CACHE_CHECK(whether $CC recognizes __func__, ac_cv_c_var_func,
+AC_TRY_COMPILE(,
+[
+char *s = __func__;
+],
+AC_DEFINE(HAVE_FUNC,,
+[Define if the C complier supports __func__]) ac_cv_c_var_func=yes,
+ac_cv_c_var_func=no) )
+])dnl
diff --git a/m4/ax_cxx_static_cast.m4 b/m4/ax_cxx_static_cast.m4
new file mode 100644
index 0000000..e09e6c4
--- /dev/null
+++ b/m4/ax_cxx_static_cast.m4
@@ -0,0 +1,43 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_cxx_static_cast.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_CXX_STATIC_CAST
+#
+# DESCRIPTION
+#
+# If the compiler supports static_cast<>, define HAVE_STATIC_CAST.
+#
+# LICENSE
+#
+# Copyright (c) 2008 Todd Veldhuizen
+# Copyright (c) 2008 Luc Maisonobe <luc at spaceroots.org>
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved. This file is offered as-is, without any
+# warranty.
+
+#serial 6
+
+AU_ALIAS([AC_CXX_STATIC_CAST], [AX_CXX_STATIC_CAST])
+AC_DEFUN([AX_CXX_STATIC_CAST],
+[AC_CACHE_CHECK(whether the compiler supports static_cast<>,
+ax_cv_cxx_static_cast,
+[AC_LANG_SAVE
+ AC_LANG_CPLUSPLUS
+ AC_TRY_COMPILE([#include <typeinfo>
+class Base { public : Base () {} virtual void f () = 0; };
+class Derived : public Base { public : Derived () {} virtual void f () {} };
+int g (Derived&) { return 0; }],[
+Derived d; Base& b = d; Derived& s = static_cast<Derived&> (b); return g (s);],
+ ax_cv_cxx_static_cast=yes, ax_cv_cxx_static_cast=no)
+ AC_LANG_RESTORE
+])
+if test "$ax_cv_cxx_static_cast" = yes; then
+ AC_DEFINE(HAVE_STATIC_CAST,,
+ [define if the compiler supports static_cast<>])
+fi
+])
diff --git a/man/Makefile.in b/man/Makefile.in
index 5443f74..dab2731 100644
--- a/man/Makefile.in
+++ b/man/Makefile.in
@@ -44,7 +44,10 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
$(top_srcdir)/m4/ac_prog_javac_works.m4 \
$(top_srcdir)/m4/ac_prog_javah.m4 \
$(top_srcdir)/m4/acx_pthread.m4 \
+ $(top_srcdir)/m4/ax_c_pretty_func.m4 \
+ $(top_srcdir)/m4/ax_c_var_func.m4 \
$(top_srcdir)/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/m4/ax_cxx_static_cast.m4 \
$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/pkg.m4 \
@@ -104,6 +107,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
+BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@
CC = @CC@
CCAS = @CCAS@
CCASDEPMODE = @CCASDEPMODE@
@@ -145,6 +149,7 @@ JAR = @JAR@
JAVAC = @JAVAC@
JAVAH = @JAVAH@
JDK_CPPFLAGS = @JDK_CPPFLAGS@
+KEYUTILS_LIB = @KEYUTILS_LIB@
LD = @LD@
LDFLAGS = @LDFLAGS@
LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@
@@ -190,6 +195,7 @@ PYTHON_PLATFORM = @PYTHON_PLATFORM@
PYTHON_PREFIX = @PYTHON_PREFIX@
PYTHON_VERSION = @PYTHON_VERSION@
RANLIB = @RANLIB@
+RESOLV_LIBS = @RESOLV_LIBS@
RPM_RELEASE = @RPM_RELEASE@
SED = @SED@
SET_MAKE = @SET_MAKE@
diff --git a/man/ceph-authtool.8 b/man/ceph-authtool.8
index e64cac9..aae41d3 100644
--- a/man/ceph-authtool.8
+++ b/man/ceph-authtool.8
@@ -1,4 +1,6 @@
-.TH "CEPH-AUTHTOOL" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-AUTHTOOL" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-authtool \- ceph keyring manipulation tool
.
@@ -28,13 +30,38 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-authtool\fP \fIkeyringfile\fP [ \-l | \-\-list ] [ \-C | \-\-create\-keyring
] [ \-p | \-\-print ] [ \-n | \-\-name \fIentityname\fP ] [ \-\-gen\-key ] [ \-a |
-\-\-add\-key \fIbase64_key\fP ] [ \-\-caps \fIcapfils\fP ]
+\-\-add\-key \fIbase64_key\fP ] [ \-\-caps \fIcapfile\fP ]
.fi
.sp
.SH DESCRIPTION
@@ -43,7 +70,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
file. A keyring file stores one or more Ceph authentication keys and
possibly an associated capability specification. Each key is
associated with an entity name, of the form
-\fB{client,mon,mds,osd}.name\fP.
+\fB{client,mon,mds,osd}.name\fP\&.
.sp
\fBWARNING\fP Ceph provides authentication and protection against
man\-in\-the\-middle attacks once secret keys are in place. However,
@@ -69,33 +96,16 @@ will create a new keyring, overwriting any existing keyringfile
.UNINDENT
.INDENT 0.0
.TP
-.B \-g, \-\-gen\-key
+.B \-\-gen\-key
will generate a new secret key for the specified entityname
.UNINDENT
.INDENT 0.0
.TP
-.B \-a, \-\-add\-key
+.B \-\-add\-key
will add an encoded key to the keyring
.UNINDENT
.INDENT 0.0
.TP
-.B \-u, \-\-set\-uid
-sets the auid (authenticated user id) for the specified entityname
-.UNINDENT
-.INDENT 0.0
-.TP
-.B \-\-gen\-print\-key
-will generate and print a new secret key without adding it to the keyringfile
-
-NOTE: will work without a given keyringfile
-.UNINDENT
-.INDENT 0.0
-.TP
-.B \-\-import\-keyring
-will import the content of a given keyring into the keyringfile
-.UNINDENT
-.INDENT 0.0
-.TP
.B \-\-cap subsystem capability
will set the capability for given subsystem
.UNINDENT
@@ -107,7 +117,7 @@ will set all of capabilities associated with a given key, for all subsystems
.SH CAPABILITIES
.sp
The subsystem is the name of a Ceph subsystem: \fBmon\fP, \fBmds\fP, or
-\fBosd\fP.
+\fBosd\fP\&.
.sp
The capability is a string describing what the given user is allowed
to do. This takes the form of a comma separated list of allow
@@ -116,6 +126,8 @@ read, write, and execute permission. The \fBallow *\fP grants full
superuser permissions for the given subsystem.
.sp
For example:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -129,8 +141,12 @@ mds = "allow"
mon = "allow rwx"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
A librados user restricted to a single pool might look like:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -139,8 +155,12 @@ mon = "allow r"
osd = "allow rw pool foo"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
A client using rbd with read access to one pool and read/write access to another:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -149,8 +169,12 @@ mon = "allow r"
osd = "allow class\-read object_prefix rbd_children, allow pool templates r class\-read, allow pool vms rwx"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
A client mounting the file system with minimal permissions would need caps like:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -161,9 +185,13 @@ osd = "allow rw pool data"
mon = "allow r"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH OSD CAPABILITIES
.sp
In general, an osd capability follows the grammar:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -173,8 +201,12 @@ match := [pool[=]<poolname> | object_prefix <prefix>]
capspec := * | [r][w][x] [class\-read] [class\-write]
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
The capspec determines what kind of operations the entity can perform:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -187,6 +219,8 @@ class\-write = can call class methods that are writes
i.e. ceph osd tell ...
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
The match criteria restrict a grant based on the pool being accessed.
Grants are additive if the client fulfills the match condition. For
@@ -204,46 +238,62 @@ value is the capability string (see above).
.SH EXAMPLE
.sp
To create a new keyring containing a key for client.foo:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-authtool \-C \-n client.foo \-\-gen\-key keyring
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To associate some capabilities with the key (namely, the ability to
mount a Ceph filesystem):
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-authtool \-n client.foo \-\-cap mds \(aqallow\(aq \-\-cap osd \(aqallow rw pool=data\(aq \-\-cap mon \(aqallow r\(aq keyring
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To display the contents of the keyring:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-authtool \-l keyring
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
When mount a Ceph file system, you can grab the appropriately encoded secret key with:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount \-t ceph serverhost:/ mountpoint \-o name=foo,secret=\(gaceph\-authtool \-p \-n client.foo keyring\(ga
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-authtool\fP is part of the Ceph distributed file system. Please
+\fBceph\-authtool\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-clsinfo.8 b/man/ceph-clsinfo.8
index 08ba0cb..4fb8274 100644
--- a/man/ceph-clsinfo.8
+++ b/man/ceph-clsinfo.8
@@ -1,4 +1,6 @@
-.TH "CEPH-CLSINFO" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-CLSINFO" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-clsinfo \- show class object information
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-clsinfo\fP [ \fIoptions\fP ] ... \fIfilename\fP
@@ -57,13 +84,13 @@ Shows the class architecture
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-clsinfo\fP is part of the Ceph distributed file system. Please
+\fBceph\-clsinfo\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-conf.8 b/man/ceph-conf.8
index bfec815..754f7fc 100644
--- a/man/ceph-conf.8
+++ b/man/ceph-conf.8
@@ -1,4 +1,6 @@
-.TH "CEPH-CONF" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-CONF" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-conf \- ceph conf file tool
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-conf\fP \-c \fIconffile\fP \-\-list\-all\-sections
@@ -72,46 +99,61 @@ we will default to doing a lookup.
.SH EXAMPLES
.sp
To find out what value osd 0 will use for the "osd data" option:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-conf \-c foo.conf \-\-name osd.0 \-\-lookup "osd data"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To find out what value will mds a use for the "log file" option:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-conf \-c foo.conf \-\-name mds.a "log file"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To list all sections that begin with osd:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-conf \-c foo.conf \-l osd
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To list all sections:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-conf \-c foo.conf \-L
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-conf\fP is part of the Ceph distributed file system. Please refer
+\fBceph\-conf\fP is part of the Ceph distributed storage system. Please refer
to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
-\fBmkcephfs\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-debugpack.8 b/man/ceph-debugpack.8
index 69db7c3..9fc016f 100644
--- a/man/ceph-debugpack.8
+++ b/man/ceph-debugpack.8
@@ -1,4 +1,6 @@
-.TH "CEPH-DEBUGPACK" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-DEBUGPACK" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-debugpack \- ceph debug packer utility
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-debugpack\fP [ \fIoptions\fP ] \fIfilename.tar.gz\fP
@@ -55,13 +82,14 @@ startup.
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-debugpack\fP is part of the Ceph distributed file system. Please
+\fBceph\-debugpack\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
+\fBceph\-post\-file\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-dencoder.8 b/man/ceph-dencoder.8
index 3f6ec89..51b2704 100644
--- a/man/ceph-dencoder.8
+++ b/man/ceph-dencoder.8
@@ -1,4 +1,6 @@
-.TH "CEPH-DENCODER" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-DENCODER" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-dencoder \- ceph encoder/decoder utility
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-dencoder\fP [commands...]
@@ -64,7 +91,7 @@ file.
.INDENT 0.0
.TP
.B list_types
-List the data types known to this build of \fBceph\-dencoder\fP.
+List the data types known to this build of \fBceph\-dencoder\fP\&.
.UNINDENT
.INDENT 0.0
.TP
@@ -104,19 +131,21 @@ of the type.
.TP
.B get_features
Print the decimal value of the feature set supported by this version
-of \fBceph\-dencoder\fP. Each bit represents a feature. These correspond to
+of \fBceph\-dencoder\fP\&. Each bit represents a feature. These correspond to
CEPH_FEATURE_* defines in src/include/ceph_features.h.
.UNINDENT
.INDENT 0.0
.TP
.B set_features <f>
-Set the feature bits provided to \fBencode\fP to \fIf\fP. This allows
+Set the feature bits provided to \fBencode\fP to \fIf\fP\&. This allows
you to encode objects such that they can be understood by old
versions of the software (for those types that support it).
.UNINDENT
.SH EXAMPLE
.sp
-Say you want to examine an attribute on an object stored by \fBceph\-osd\fP. You can do:
+Say you want to examine an attribute on an object stored by \fBceph\-osd\fP\&. You can do:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -148,15 +177,17 @@ $ ceph\-dencoder type object_info_t import /tmp/a decode dump_json
"watchers": {}}
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-dencoder\fP is part of the Ceph distributed file system. Please
+\fBceph\-dencoder\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-fuse.8 b/man/ceph-fuse.8
index e463214..f41b7b9 100644
--- a/man/ceph-fuse.8
+++ b/man/ceph-fuse.8
@@ -1,4 +1,6 @@
-.TH "CEPH-FUSE" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-FUSE" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-fuse \- FUSE-based client for ceph
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-fuse\fP [ \-m \fImonaddr\fP:\fIport\fP ] \fImountpoint\fP [ \fIfuse options\fP ]
@@ -43,12 +70,16 @@ via the \-m option for described by ceph.conf (see below) at the
specific mount point.
.sp
The file system can be unmounted with:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
fusermount \-u mountpoint
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
or by sending \fBSIGINT\fP to the \fBceph\-fuse\fP process.
.SH OPTIONS
@@ -77,13 +108,13 @@ Use root_directory as the mounted root, rather than the full Ceph tree.
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-fuse\fP is part of the Ceph distributed file system. Please refer to
+\fBceph\-fuse\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
fusermount(8),
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-mds.8 b/man/ceph-mds.8
index 5399308..398b010 100644
--- a/man/ceph-mds.8
+++ b/man/ceph-mds.8
@@ -1,4 +1,6 @@
-.TH "CEPH-MDS" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-MDS" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-mds \- ceph metadata server daemon
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-mds\fP \-i \fIname\fP [[ \-\-hot\-standby [\fIrank\fP] ]|[\-\-journal_check \fIrank\fP]]
@@ -81,7 +108,7 @@ Connect to specified monitor (instead of looking through
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-mds\fP is part of the Ceph distributed file system. Please refer to the Ceph documentation at
+\fBceph\-mon\fP is part of the Ceph distributed storage system. Please refer to the Ceph documentation at
\fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
@@ -89,6 +116,6 @@ Connect to specified monitor (instead of looking through
\fBceph\-mon\fP(8),
\fBceph\-osd\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-mon.8 b/man/ceph-mon.8
index 9dc494f..64e3bf9 100644
--- a/man/ceph-mon.8
+++ b/man/ceph-mon.8
@@ -1,4 +1,6 @@
-.TH "CEPH-MON" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-MON" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-mon \- ceph monitor daemon
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-mon\fP \-i \fImonid\fP [ \-\-mon\-data \fImondatapath\fP ]
@@ -73,30 +100,30 @@ cluster. Three pieces of information must be provided:
.INDENT 7.0
.IP \(bu 2
The cluster fsid. This can come from a monmap (\fB\-\-monmap <path>\fP) or
-explicitly via \fB\-\-fsid <uuid>\fP.
+explicitly via \fB\-\-fsid <uuid>\fP\&.
.IP \(bu 2
A list of monitors and their addresses. This list of monitors
can come from a monmap (\fB\-\-monmap <path>\fP), the \fBmon host\fP
configuration value (in \fIceph.conf\fP or via \fB\-m
-host1,host2,...\fP), or \fBmon addr\fP lines in \fIceph.conf\fP. If this
+host1,host2,...\fP), or \fBmon addr\fP lines in \fIceph.conf\fP\&. If this
monitor is to be part of the initial monitor quorum for a new
Ceph cluster, then it must be included in the initial list,
matching either the name or address of a monitor in the list.
When matching by address, either the \fBpublic addr\fP or \fBpublic
subnet\fP options may be used.
.IP \(bu 2
-The monitor secret key \fBmon.\fP. This must be included in the
-keyring provided via \fB\-\-keyring <path>\fP.
+The monitor secret key \fBmon.\fP\&. This must be included in the
+keyring provided via \fB\-\-keyring <path>\fP\&.
.UNINDENT
.UNINDENT
.INDENT 0.0
.TP
.B \-\-keyring
-Specify a keyring for use with \fB\-\-mkfs\fP.
+Specify a keyring for use with \fB\-\-mkfs\fP\&.
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-mon\fP is part of the Ceph distributed file system. Please refer
+\fBceph\-mon\fP is part of the Ceph distributed storage system. Please refer
to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
@@ -105,6 +132,6 @@ information.
\fBceph\-mds\fP(8),
\fBceph\-osd\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-osd.8 b/man/ceph-osd.8
index f60ce18..24ec595 100644
--- a/man/ceph-osd.8
+++ b/man/ceph-osd.8
@@ -1,4 +1,6 @@
-.TH "CEPH-OSD" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-OSD" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-osd \- ceph object storage daemon
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-osd\fP \-i \fIosdnum\fP [ \-\-osd\-data \fIdatapath\fP ] [ \-\-osd\-journal
@@ -61,18 +88,17 @@ Debug mode: like \fB\-f\fP, but also send all log output to stderr.
.INDENT 0.0
.TP
.B \-\-osd\-data osddata
-Use object store at \fIosddata\fP.
+Use object store at \fIosddata\fP\&.
.UNINDENT
.INDENT 0.0
.TP
.B \-\-osd\-journal journal
-Journal updates to \fIjournal\fP.
+Journal updates to \fIjournal\fP\&.
.UNINDENT
.INDENT 0.0
.TP
.B \-\-mkfs
-Create an empty object repository. Normally invoked by
-\fBmkcephfs\fP(8). This also initializes the journal
+Create an empty object repository. This also initializes the journal
(if one is defined).
.UNINDENT
.INDENT 0.0
@@ -129,7 +155,7 @@ Connect to specified monitor (instead of looking through
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-osd\fP is part of the Ceph distributed file system. Please refer to
+\fBceph\-osd\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
@@ -138,6 +164,6 @@ the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
\fBceph\-mon\fP(8),
\fBceph\-authtool\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-post-file.8 b/man/ceph-post-file.8
index 110f3c2..a037821 100644
--- a/man/ceph-post-file.8
+++ b/man/ceph-post-file.8
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "CEPH-POST-FILE" "8" "August 14, 2013" "dev" "Ceph"
+.TH "CEPH-POST-FILE" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-post-file \- post files for ceph developers
.
@@ -118,13 +118,13 @@ ceph\-post\-file \-d \(aqmon data directories\(aq /var/log/ceph/mon/*
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-post\-file\fP is part of the Ceph distributed file system. Please refer to
+\fBceph\-post\-file\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
\fBceph\-debugpack\fP(8),
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-rbdnamer.8 b/man/ceph-rbdnamer.8
index 6aa2cfa..dab2c10 100644
--- a/man/ceph-rbdnamer.8
+++ b/man/ceph-rbdnamer.8
@@ -1,4 +1,6 @@
-.TH "CEPH-RBDNAMER" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-RBDNAMER" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-rbdnamer \- udev helper to name RBD devices
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-rbdnamer\fP \fInum\fP
@@ -40,15 +67,19 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
\fBceph\-rbdnamer\fP prints the pool and image name for the given RBD devices
to stdout. It is used by \fIudev\fP (using a rule like the one below) to
set up a device symlink.
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
KERNEL=="rbd[0\-9]*", PROGRAM="/usr/bin/ceph\-rbdnamer %n", SYMLINK+="rbd/%c{1}/%c{2}"
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-rbdnamer\fP is part of the Ceph distributed file system. Please
+\fBceph\-rbdnamer\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
@@ -56,6 +87,6 @@ information.
\fBrbd\fP(8),
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-rest-api.8 b/man/ceph-rest-api.8
index 5170b7f..0d443f6 100644
--- a/man/ceph-rest-api.8
+++ b/man/ceph-rest-api.8
@@ -1,4 +1,6 @@
-.TH "CEPH-REST-API" "8" "July 26, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-REST-API" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-rest-api \- ceph RESTlike administration server
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-rest\-api\fP [ \-c \fIconffile\fP ] [\-\-cluster \fIclustername\fP ] [ \-n \fIname\fP ] [\-i \fIid\fP ]
@@ -125,17 +152,21 @@ desired structured output, or you may use a .json or .xml addition
to the requested PATH. Parameters are supplied as query parameters
in the request; for parameters that take more than one value, repeat
the key=val construct. For instance, to remove OSDs 2 and 3,
-send a PUT request to \fBosd/rm?ids=2&ids=3\fP.
+send a PUT request to \fBosd/rm?ids=2&ids=3\fP\&.
.SH DISCOVERY
.sp
Human\-readable discovery of supported commands and parameters, along
with a small description of each command, is provided when the requested
path is incomplete/partially matching. Requesting / will redirect to
the value of \fBrestapi base url\fP, and that path will give a full list
-of all known commands. The command set is very similar to the commands
+of all known commands.
+For example, requesting \fBapi/vX.X/mon\fP will return the list of API calls for
+monitors \- \fBapi/vX.X/osd\fP will return the list of API calls for OSD and so on.
+.sp
+The command set is very similar to the commands
supported by the \fBceph\fP tool. One notable exception is that the
\fBceph pg <pgid> <command>\fP style of commands is supported here
-as \fBtell/<pgid>/command?args\fP.
+as \fBtell/<pgid>/command?args\fP\&.
.SH DEPLOYMENT AS WSGI APPLICATION
.sp
When deploying as WSGI application (say, with Apache/mod_wsgi,
@@ -166,12 +197,12 @@ exception to be raised; see your WSGI server documentation for how to
see those messages in case of problem.
.SH AVAILABILITY
.sp
-\fBceph\-rest\-api\fP is part of the Ceph distributed file system. Please refer to the Ceph documentation at
+\fBceph\-rest\-api\fP is part of the Ceph distributed storage system. Please refer to the Ceph documentation at
\fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-run.8 b/man/ceph-run.8
index 77e8f32..0baa632 100644
--- a/man/ceph-run.8
+++ b/man/ceph-run.8
@@ -1,4 +1,6 @@
-.TH "CEPH-RUN" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-RUN" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-run \- restart daemon on core dump
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\-run\fP \fIcommand\fP ...
@@ -48,7 +75,7 @@ that means the \fB\-f\fP option.
None
.SH AVAILABILITY
.sp
-\fBceph\-run\fP is part of the Ceph distributed file system. Please refer to
+\fBceph\-run\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
@@ -57,6 +84,6 @@ the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
\fBceph\-mds\fP(8),
\fBceph\-osd\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph-syn.8 b/man/ceph-syn.8
index 03a068a..6b12a09 100644
--- a/man/ceph-syn.8
+++ b/man/ceph-syn.8
@@ -1,4 +1,6 @@
-.TH "CEPH-SYN" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH-SYN" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph-syn \- ceph synthetic workload generator
.
@@ -28,11 +30,36 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
-\fBceph\-syn\fP [ \-m \fImonaddr\fP:\fIport\fP ] \-\-syn \fIcommand\fP \fI...\fP
+\fBceph\-syn\fP [ \-m \fImonaddr\fP:\fIport\fP ] \-\-syn \fIcommand\fP \fI\&...\fP
.fi
.sp
.SH DESCRIPTION
@@ -81,13 +108,13 @@ line. This is not a complete list.
.INDENT 0.0
.TP
.B \fBmknap\fP \fIpath\fP \fIsnapname\fP
-Create a snapshot called \fIsnapname\fP on \fIpath\fP.
+Create a snapshot called \fIsnapname\fP on \fIpath\fP\&.
.TP
.B \fBrmsnap\fP \fIpath\fP \fIsnapname\fP
-Delete snapshot called \fIsnapname\fP on \fIpath\fP.
+Delete snapshot called \fIsnapname\fP on \fIpath\fP\&.
.TP
.B \fBrmfile\fP \fIpath\fP
-Delete/unlink \fIpath\fP.
+Delete/unlink \fIpath\fP\&.
.TP
.B \fBwritefile\fP \fIsizeinmb\fP \fIblocksize\fP
Create a file, named after our client id, that is \fIsizeinmb\fP MB by
@@ -109,13 +136,13 @@ Recursively walk the file system (like find).
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\-syn\fP is part of the Ceph distributed file system. Please refer to
+\fBceph\-syn\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
\fBceph\-fuse\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/ceph.8 b/man/ceph.8
index 4f2caf3..9bb903c 100644
--- a/man/ceph.8
+++ b/man/ceph.8
@@ -1,4 +1,6 @@
-.TH "CEPH" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPH" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
ceph \- ceph file system control utility
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBceph\fP [ \-m \fImonaddr\fP ] [ \-w | \fIcommand\fP ... ]
@@ -38,7 +65,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.SH DESCRIPTION
.sp
\fBceph\fP is a control utility for communicating with the monitor
-cluster of a running Ceph distributed file system.
+cluster of a running Ceph distributed storage system.
.sp
There are three basic modes of operation.
.SS Interactive mode
@@ -48,12 +75,16 @@ To start in interactive mode, no arguments are necessary. Control\-d or
.SS Watch mode
.sp
Watch mode shows cluster state changes as they occur. For example:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph \-w
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SS Command line mode
.sp
Finally, to send a single instruction to the monitor cluster (and wait
@@ -87,20 +118,28 @@ Connect to specified monitor (instead of looking through ceph.conf).
.SH EXAMPLES
.sp
To grab a copy of the current OSD map:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph \-m 1.2.3.4:6789 osd getmap \-o osdmap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To get a dump of placement group (PG) state:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph pg dump \-o pg.txt
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH MONITOR COMMANDS
.sp
A more complete summary of commands understood by the monitor cluster can be found in the
@@ -112,13 +151,12 @@ online documentation, at
.UNINDENT
.SH AVAILABILITY
.sp
-\fBceph\fP is part of the Ceph distributed file system. Please refer to the Ceph documentation at
+\fBceph\fP is part of the Ceph distributed storage system. Please refer to the Ceph documentation at
\fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
-\fBmkcephfs\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/cephfs.8 b/man/cephfs.8
index f53b208..2fa5583 100644
--- a/man/cephfs.8
+++ b/man/cephfs.8
@@ -1,4 +1,6 @@
-.TH "CEPHFS" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CEPHFS" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
cephfs \- ceph file system options utility
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBcephfs\fP [ \fIpath\fP \fIcommand\fP \fIoptions\fP ]
@@ -38,7 +65,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.SH DESCRIPTION
.sp
\fBcephfs\fP is a control utility for accessing and manipulating file
-layout and location data in the Ceph distributed file system.
+layout and location data in the Ceph distributed storage system.
.sp
Choose one of the following three commands:
.INDENT 0.0
@@ -105,13 +132,13 @@ preferred OSD for placement. This feature is unsupported and ignored
in modern versions of the Ceph servers; do not use it.
.SH AVAILABILITY
.sp
-\fBcephfs\fP is part of the Ceph distributed file system. Please refer
+\fBcephfs\fP is part of the Ceph distributed storage system. Please refer
to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/crushtool.8 b/man/crushtool.8
index 26b21a3..38a379b 100644
--- a/man/crushtool.8
+++ b/man/crushtool.8
@@ -1,4 +1,6 @@
-.TH "CRUSHTOOL" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "CRUSHTOOL" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
crushtool \- CRUSH map manipulation tool
.
@@ -28,18 +30,45 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
-\fBcrushtool\fP ( \-d \fImap\fP | \-c \fImap.txt\fP | \-\-build \fInumosds\fP
-\fIlayer1\fP \fI...\fP ) [ \-o \fIoutfile\fP [ \-\-clobber ]]
+\fBcrushtool\fP ( \-d \fImap\fP | \-c \fImap.txt\fP | \-\-build \-\-num_osds \fInumosds\fP
+\fIlayer1\fP \fI\&...\fP | \-\-test ) [ \-o \fIoutfile\fP ]
.fi
.sp
.SH DESCRIPTION
-.sp
-\fBcrushtool\fP is a utility that lets you create, compile, and
-decompile CRUSH map files.
+.INDENT 0.0
+.TP
+.B \fBcrushtool\fP is a utility that lets you create, compile, decompile
+and test CRUSH map files.
+.UNINDENT
.sp
CRUSH is a pseudo\-random data distribution algorithm that efficiently
maps input values (typically data objects) across a heterogeneous,
@@ -55,101 +84,350 @@ some since then):
The tool has four modes of operation.
.INDENT 0.0
.TP
-.B \-c map.txt
+.B \-\-compile|\-c map.txt
will compile a plaintext map.txt into a binary map file.
.UNINDENT
.INDENT 0.0
.TP
-.B \-d map
+.B \-\-decompile|\-d map
will take the compiled map and decompile it into a plaintext source
file, suitable for editing.
.UNINDENT
.INDENT 0.0
.TP
.B \-\-build \-\-num_osds {num\-osds} layer1 ...
-will create a relatively generic map with the given layer
-structure. See below for examples.
+will create map with the given layer structure. See below for a
+detailed explanation.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-test
+will perform a dry run of a CRUSH mapping for a range of input
+object names. See below for a detailed explanation.
+.UNINDENT
+.sp
+Unlike other Ceph tools, \fBcrushtool\fP does not accept generic options
+such as \fB\-\-debug\-crush\fP from the command line. They can however be
+provided via the CEPH_ARGS environment variable. For instance, to
+silence all output from the CRUSH subsystem:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+CEPH_ARGS="\-\-debug\-crush 0" crushtool ...
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.SH RUNNING TESTS WITH --TEST
+.sp
+The test mode will use the input crush map ( as specified with \fB\-i
+map\fP ) and perform a dry run of CRUSH mapping or random placement (
+if \fB\-\-simulate\fP is set ). On completion, two kinds of reports can be
+created. The \fB\-\-show\-...\fP options output human readable informations
+on stderr. The \fB\-\-output\-csv\fP option creates CSV files that are
+documented by the \fB\-\-help\-output\fP option.
+.INDENT 0.0
+.TP
+.B \-\-show\-statistics
+for each rule display the mapping of each object. For instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+CRUSH rule 1 x 24 [11,6]
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+shows that object \fB24\fP is mapped to devices \fB[11,6]\fP by rule
+\fB1\fP\&. At the end of the mapping details, a summary of the
+distribution is displayed. For instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+rule 1 (metadata) num_rep 5 result size == 5: 1024/1024
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+shows that rule \fB1\fP which is named \fBmetadata\fP successfully
+mapped \fB1024\fP objects to \fBresult size == 5\fP devices when trying
+to map them to \fBnum_rep 5\fP replicas. When it fails to provide the
+required mapping, presumably because the number of \fBtries\fP must
+be increased, a breakdown of the failures is displays. For instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+rule 1 (metadata) num_rep 10 result size == 8: 4/1024
+rule 1 (metadata) num_rep 10 result size == 9: 93/1024
+rule 1 (metadata) num_rep 10 result size == 10: 927/1024
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+shows that although \fBnum_rep 10\fP replicas were required, \fB4\fP
+out of \fB1024\fP objects ( \fB4/1024\fP ) were mapped to \fBresult size
+== 8\fP devices only.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-show\-bad\-mappings
+display which object failed to be mapped to the required number of
+devices. For instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9]
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+shows that when rule \fB1\fP was required to map \fB7\fP devices, it
+could only map six : \fB[8,10,2,11,6,9]\fP\&.
.UNINDENT
.INDENT 0.0
.TP
-.B \-\-test ...
+.B \-\-show\-utilization
+display the expected and actual utilisation for each device, for
+each number of replicas. For instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+device 0: stored : 951 expected : 853.333
+device 1: stored : 963 expected : 853.333
+\&...
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+shows that device \fB0\fP stored \fB951\fP objects and was expected to store \fB853\fP\&.
+Implies \fB\-\-show\-statistics\fP\&.
+.UNINDENT
+.INDENT 0.0
.TP
-.B will perform a dry run of a CRUSH mapping for a range of input object
+.B \-\-show\-utilization\-all
+displays the same as \fB\-\-show\-utilization\fP but does not suppress
+output when the weight of a device is zero.
+Implies \fB\-\-show\-statistics\fP\&.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-show\-choose\-tries
+display how many attempts were needed to find a device mapping.
+For instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+0: 95224
+1: 3745
+2: 2225
+\&..
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+shows that \fB95224\fP mappings succeeded without retries, \fB3745\fP
+mappings succeeded with one attempts, etc. There are as many rows
+as the value of the \fB\-\-set\-choose\-total\-tries\fP option.
+.UNINDENT
+.INDENT 0.0
.TP
-.B \-\-help for more information.
+.B \-\-output\-csv
+create CSV files (in the current directory) containing information
+documented by \fB\-\-help\-output\fP\&. The files are named after the rule
+used when collecting the statistics. For instance, if the rule
+metadata is used, the CSV files will be:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+metadata\-absolute_weights.csv
+metadata\-device_utilization.csv
+\&...
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+The first line of the file shortly explains the column layout. For
+instance:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+metadata\-absolute_weights.csv
+Device ID, Absolute Weight
+0,1
+\&...
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
.UNINDENT
-.SH OPTIONS
.INDENT 0.0
.TP
-.B \-o outfile
-will specify the output file.
+.B \-\-output\-name NAME
+prepend \fBNAME\fP to the file names generated when \fB\-\-output\-csv\fP
+is specified. For instance \fB\-\-output\-name FOO\fP will create
+files:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+FOO\-metadata\-absolute_weights.csv
+FOO\-metadata\-device_utilization.csv
+\&...
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.sp
+The \fB\-\-set\-...\fP options can be used to modify the tunables of the
+input crush map. The input crush map is modified in
+memory. For example:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ crushtool \-i mymap \-\-test \-\-show\-bad\-mappings
+bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9]
+.ft P
+.fi
+.UNINDENT
.UNINDENT
+.sp
+could be fixed by increasing the \fBchoose\-total\-tries\fP as follows:
+.INDENT 0.0
+.INDENT 3.5
.INDENT 0.0
.TP
-.B \-\-clobber
-will allow the tool to overwrite an existing outfile (it will normally refuse).
+.B $ crushtool \-i mymap \-\-test
+\-\-show\-bad\-mappings \-\-set\-choose\-total\-tries 500
+.UNINDENT
.UNINDENT
-.SH BUILDING A MAP
+.UNINDENT
+.SH BUILDING A MAP WITH --BUILD
.sp
-The build mode will generate relatively generic hierarchical maps. The
-first argument simply specifies the number of devices (leaves) in the
-CRUSH hierarchy. Each layer describes how the layer (or raw devices)
-preceding it should be grouped.
+The build mode will generate hierarchical maps. The first argument
+specifies the number of devices (leaves) in the CRUSH hierarchy. Each
+layer describes how the layer (or devices) preceding it should be
+grouped.
.sp
Each layer consists of:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
-name ( uniform | list | tree | straw ) size
+bucket ( uniform | list | tree | straw ) size
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
-The first element is the name for the elements in the layer
-(e.g. "rack"). Each element\(aqs name will be append a number to the
-provided name.
+The \fBbucket\fP is the type of the buckets in the layer
+(e.g. "rack"). Each bucket name will be built by appending a unique
+number to the \fBbucket\fP string (e.g. "rack0", "rack1"...).
.sp
-The second component is the type of CRUSH bucket.
+The second component is the type of bucket: \fBstraw\fP should be used
+most of the time.
.sp
-The third component is the maximum size of the bucket. If the size is
-0, a single bucket will be generated that includes everything in the
-preceding layer.
+The third component is the maximum size of the bucket. A size of zero
+means a bucket of infinite capacity.
.SH EXAMPLE
.sp
-Suppose we have 128 devices, each grouped into shelves with 4 devices
-each, and 8 shelves per rack. We could create a three level hierarchy
-with:
+Suppose we have two rows with two racks each and 20 nodes per rack. Suppose
+each node contains 4 storage devices for Ceph OSD Daemons. This configuration
+allows us to deploy 320 Ceph OSD Daemons. Lets assume a 42U rack with 2U nodes,
+leaving an extra 2U for a rack switch.
+.sp
+To reflect our hierarchy of devices, nodes, racks and rows, we would execute
+the following:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
-crushtool \-\-build 128 shelf uniform 4 rack straw 8 root straw 0 \-o map
+$ crushtool \-o crushmap \-\-build \-\-num_osds 320 \e
+ node straw 4 \e
+ rack straw 20 \e
+ row straw 2 \e
+ root straw 0
+# id weight type name reweight
+\-87 320 root root
+\-85 160 row row0
+\-81 80 rack rack0
+\-1 4 node node0
+0 1 osd.0 1
+1 1 osd.1 1
+2 1 osd.2 1
+3 1 osd.3 1
+\-2 4 node node1
+4 1 osd.4 1
+5 1 osd.5 1
+\&...
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
-To adjust the default (generic) mapping rules, we can run:
+CRUSH rulesets are created so the generated crushmap can be
+tested. They are the same rulesets as the one created by default when
+creating a new Ceph cluster. They can be further edited with:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
# decompile
-crushtool \-d map \-o map.txt
+crushtool \-d crushmap \-o map.txt
# edit
-vi map.txt
+emacs map.txt
# recompile
-crushtool \-c map.txt \-o map
+crushtool \-c map.txt \-o crushmap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBcrushtool\fP is part of the Ceph distributed file system. Please
+\fBcrushtool\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
\fBosdmaptool\fP(8),
-\fBmkcephfs\fP(8)
+.SH AUTHORS
+.sp
+John Wilkins, Sage Weil, Loic Dachary
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/librados-config.8 b/man/librados-config.8
index fc5dbd1..48238fe 100644
--- a/man/librados-config.8
+++ b/man/librados-config.8
@@ -1,4 +1,6 @@
-.TH "LIBRADOS-CONFIG" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "LIBRADOS-CONFIG" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
librados-config \- display information about librados
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBlibrados\-config\fP [ \-\-version ] [ \-\-vernum ]
@@ -39,7 +66,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.INDENT 0.0
.TP
.B \fBlibrados\-config\fP is a utility that displays information about the
-installed \fBlibrados\fP.
+installed \fBlibrados\fP\&.
.UNINDENT
.SH OPTIONS
.INDENT 0.0
@@ -54,7 +81,7 @@ Display the \fBlibrados\fP version code
.UNINDENT
.SH AVAILABILITY
.sp
-\fBlibrados\-config\fP is part of the Ceph distributed file system.
+\fBlibrados\-config\fP is part of the Ceph distributed storage system.
Please refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for
more information.
.SH SEE ALSO
@@ -62,6 +89,6 @@ more information.
\fBceph\fP(8),
\fBrados\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/monmaptool.8 b/man/monmaptool.8
index 7630854..1bd9f30 100644
--- a/man/monmaptool.8
+++ b/man/monmaptool.8
@@ -1,4 +1,6 @@
-.TH "MONMAPTOOL" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "MONMAPTOOL" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
monmaptool \- ceph monitor cluster map manipulation tool
.
@@ -28,18 +30,43 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBmonmaptool\fP \fImapfilename\fP [ \-\-clobber ] [ \-\-print ] [ \-\-create ]
-[ \-\-add \fIip\fP:\fIport\fP \fI...\fP ] [ \-\-rm \fIip\fP:\fIport\fP \fI...\fP ]
+[ \-\-add \fIip\fP:\fIport\fP \fI\&...\fP ] [ \-\-rm \fIip\fP:\fIport\fP \fI\&...\fP ]
.fi
.sp
.SH DESCRIPTION
.sp
\fBmonmaptool\fP is a utility to create, view, and modify a monitor
-cluster map for the Ceph distributed file system. The monitor map
+cluster map for the Ceph distributed storage system. The monitor map
specifies the only fixed addresses in the Ceph distributed system.
All other daemons bind to arbitrary addresses and register themselves
with the monitors.
@@ -111,6 +138,8 @@ will set the fsid to the given uuid. If not specified with \-\-create, a random
.SH EXAMPLE
.sp
To create a new map with three monitors (for a fresh Ceph file system):
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -118,33 +147,42 @@ monmaptool \-\-create \-\-add mon.a 192.168.0.10:6789 \-\-add mon.b 192.168.0
\-\-add mon.c 192.168.0.12:6789 \-\-clobber monmap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To display the contents of the map:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
-monmaptool \-\-print onmap
+monmaptool \-\-print monmap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To replace one monitor:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
monmaptool \-\-rm mon.a \-\-add mon.a 192.168.0.9:6789 \-\-clobber monmap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBmonmaptool\fP is part of the Ceph distributed file system. Please
+\fBmonmaptool\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
\fBcrushtool\fP(8),
-\fBmkcephfs\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/mount.ceph.8 b/man/mount.ceph.8
index e11b671..c3549c0 100644
--- a/man/mount.ceph.8
+++ b/man/mount.ceph.8
@@ -1,4 +1,6 @@
-.TH "MOUNT.CEPH" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "MOUNT.CEPH" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
mount.ceph \- mount a ceph file system
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBmount.ceph\fP \fImonaddr1\fP[,\fImonaddr2\fP,...]:/[\fIsubdir\fP] \fIdir\fP [
@@ -44,12 +71,16 @@ addresses and read authentication keys from disk; the Linux kernel
client component does most of the real work. In fact, it is possible
to mount a non\-authenticated Ceph file system without mount.ceph by
specifying monitor address(es) by IP:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount \-t ceph 1.2.3.4:/ mountpoint
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Each monitor address monaddr takes the form host[:port]. If the port
is not specified, the Ceph default of 6789 is assumed.
@@ -151,49 +182,69 @@ no dcache readdir
.SH EXAMPLES
.sp
Mount the full file system:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount.ceph monhost:/ /mnt/foo
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
If there are multiple monitors:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount.ceph monhost1,monhost2,monhost3:/ /mnt/foo
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
If \fBceph\-mon\fP(8) is running on a non\-standard
port:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount.ceph monhost1:7000,monhost2:7000,monhost3:7000:/ /mnt/foo
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To mount only part of the namespace:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount.ceph monhost1:/some/small/thing /mnt/thing
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Assuming mount.ceph(8) is installed properly, it should be
automatically invoked by mount(8) like so:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
mount \-t ceph monhost:/ /mnt/foo
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBmount.ceph\fP is part of the Ceph distributed file system. Please
+\fBmount.ceph\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
@@ -201,6 +252,6 @@ information.
\fBceph\-fuse\fP(8),
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/osdmaptool.8 b/man/osdmaptool.8
index edc6dc2..cf3660b 100644
--- a/man/osdmaptool.8
+++ b/man/osdmaptool.8
@@ -1,4 +1,6 @@
-.TH "OSDMAPTOOL" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "OSDMAPTOOL" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
osdmaptool \- ceph osd cluster map manipulation tool
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBosdmaptool\fP \fImapfilename\fP [\-\-print] [\-\-createsimple \fInumosd\fP
@@ -39,7 +66,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.SH DESCRIPTION
.sp
\fBosdmaptool\fP is a utility that lets you create, view, and manipulate
-OSD cluster maps from the Ceph distributed file system. Notably, it
+OSD cluster maps from the Ceph distributed storage system. Notably, it
lets you extract the embedded CRUSH map or import a new CRUSH map.
.SH OPTIONS
.INDENT 0.0
@@ -75,31 +102,38 @@ attribute will be set to numosd shifted by bitsperosd.
.SH EXAMPLE
.sp
To create a simple map with 16 devices:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
osdmaptool \-\-createsimple 16 osdmap \-\-clobber
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To view the result:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
osdmaptool \-\-print osdmap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBosdmaptool\fP is part of the Ceph distributed file system. Please
+\fBosdmaptool\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
\fBcrushtool\fP(8),
-\fBmkcephfs\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/rados.8 b/man/rados.8
index ff067a3..0b9881f 100644
--- a/man/rados.8
+++ b/man/rados.8
@@ -1,4 +1,6 @@
-.TH "RADOS" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "RADOS" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
rados \- rados object storage utility
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBrados\fP [ \-m \fImonaddr\fP ] [ mkpool | rmpool \fIfoo\fP ] [ \-p | \-\-pool
@@ -40,7 +67,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.SH DESCRIPTION
.sp
\fBrados\fP is a utility for interacting with a Ceph object storage
-cluster (RADOS), part of the Ceph distributed file system.
+cluster (RADOS), part of the Ceph distributed storage system.
.SH OPTIONS
.INDENT 0.0
.TP
@@ -111,11 +138,14 @@ List objects in given pool and write to outfile.
.B \fBlssnap\fP
List snapshots for given pool.
.TP
+.B \fBclonedata\fP \fIsrcname\fP \fIdstname\fP \-\-object\-locator \fIkey\fP
+Clone object byte data from \fIsrcname\fP to \fIdstname\fP\&. Both objects must be stored with the locator key \fIkey\fP (usually either \fIsrcname\fP or \fIdstname\fP). Object attributes and omap keys are not copied or cloned.
+.TP
.B \fBmksnap\fP \fIfoo\fP
-Create pool snapshot named \fIfoo\fP.
+Create pool snapshot named \fIfoo\fP\&.
.TP
.B \fBrmsnap\fP \fIfoo\fP
-Remove pool snapshot names \fIfoo\fP.
+Remove pool snapshot named \fIfoo\fP\&.
.TP
.B \fBbench\fP \fIseconds\fP \fImode\fP [ \-b \fIobjsize\fP ] [ \-t \fIthreads\fP ]
Benchmark for seconds. The mode can be write or read. The default
@@ -147,60 +177,84 @@ Set the value of the object map header of object name.
.SH EXAMPLES
.sp
To view cluster utilization:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rados df
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To get a list object in pool foo sent to stdout:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rados \-p foo ls \-
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To write an object:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rados \-p foo put myobject blah.txt
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To create a snapshot:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rados \-p foo mksnap mysnap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To delete the object:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rados \-p foo rm myobject
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To read a previously snapshotted version of an object:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rados \-p foo \-s mysnap get myobject blah.txt.old
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBrados\fP is part of the Ceph distributed file system. Please refer to
+\fBrados\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/radosgw-admin.8 b/man/radosgw-admin.8
index 2091df5..64c54ae 100644
--- a/man/radosgw-admin.8
+++ b/man/radosgw-admin.8
@@ -1,4 +1,6 @@
-.TH "RADOSGW-ADMIN" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "RADOSGW-ADMIN" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
radosgw-admin \- rados REST gateway user administration utility
.
@@ -28,11 +30,36 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
-\fBradosgw\-admin\fP \fIcommand\fP [ \fIoptions\fP \fI...\fP ]
+\fBradosgw\-admin\fP \fIcommand\fP [ \fIoptions\fP \fI\&...\fP ]
.fi
.sp
.SH DESCRIPTION
@@ -186,6 +213,8 @@ Defer removal of object tail
.SH EXAMPLES
.sp
Generate a new user:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -203,40 +232,60 @@ $ radosgw\-admin user create \-\-display\-name="johnny rotten" \-\-uid=johnny
"swift_keys": []}
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Remove a user:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
$ radosgw\-admin user rm \-\-uid=johnny
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Remove a user and all associated buckets with their contents:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
$ radosgw\-admin user rm \-\-uid=johnny \-\-purge\-data
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Remove a bucket:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
$ radosgw\-admin bucket unlink \-\-bucket=foo
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Show the logs of a bucket from April 1st, 2012:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
$ radosgw\-admin log show \-\-bucket=foo \-\-date=2012=04\-01
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Show usage information for user from March 1st to (but not including) April 1st, 2012:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -244,31 +293,41 @@ $ radosgw\-admin usage show \-\-uid=johnny \e
\-\-start\-date=2012\-03\-01 \-\-end\-date=2012\-04\-01
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Show only summary of usage information for all users:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
$ radosgw\-admin usage show \-\-show\-log\-entries=false
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Trim usage information for user until March 1st, 2012:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
$ radosgw\-admin usage trim \-\-uid=johnny \-\-end\-date=2012\-04\-01
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBradosgw\-admin\fP is part of the Ceph distributed file system. Please
+\fBradosgw\-admin\fP is part of the Ceph distributed storage system. Please
refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
.sp
\fBceph\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/radosgw.8 b/man/radosgw.8
index 44f8607..054fb79 100644
--- a/man/radosgw.8
+++ b/man/radosgw.8
@@ -1,4 +1,6 @@
-.TH "RADOSGW" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "RADOSGW" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
radosgw \- rados REST gateway
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBradosgw\fP
@@ -62,6 +89,8 @@ Specify a unix domain socket path.
.SH CONFIGURATION
.sp
Currently it\(aqs the easiest to use the RADOS Gateway with Apache and mod_fastcgi:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -92,8 +121,12 @@ FastCgiExternalServer /var/www/s3gw.fcgi \-socket /tmp/radosgw.sock
</VirtualHost>
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
And the corresponding radosgw script (/var/www/s3gw.fcgi):
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -101,10 +134,14 @@ And the corresponding radosgw script (/var/www/s3gw.fcgi):
exec /usr/bin/radosgw \-c /etc/ceph/ceph.conf \-n client.radosgw.gateway
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
The radosgw daemon is a standalone process which needs a configuration
section in the ceph.conf The section name should start with
\(aqclient.radosgw.\(aq as specified in /etc/init.d/radosgw:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -114,26 +151,38 @@ section in the ceph.conf The section name should start with
rgw socket path = /tmp/radosgw.sock
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
You will also have to generate a key for the radosgw to use for
authentication with the cluster:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph\-authtool \-C \-n client.radosgw.gateway \-\-gen\-key /etc/ceph/keyring.radosgw.gateway
-ceph\-authtool \-n client.radosgw.gateway \-\-cap mon \(aqallow r\(aq \-\-cap osd \(aqallow rwx\(aq /etc/ceph/keyring.radosgw.gateway
+ceph\-authtool \-n client.radosgw.gateway \-\-cap mon \(aqallow rw\(aq \-\-cap osd \(aqallow rwx\(aq /etc/ceph/keyring.radosgw.gateway
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
And add the key to the auth entries:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
ceph auth add client.radosgw.gateway \-\-in\-file=keyring.radosgw.gateway
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Now you can start Apache and the radosgw daemon:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -141,11 +190,13 @@ Now you can start Apache and the radosgw daemon:
/etc/init.d/radosgw start
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH USAGE LOGGING
.sp
The \fBradosgw\fP maintains an asynchronous usage log. It accumulates
statistics about user operations and flushes it periodically. The
-logs can be accessed and managed through \fBradosgw\-admin\fP.
+logs can be accessed and managed through \fBradosgw\-admin\fP\&.
.sp
The information that is being logged contains total data transfer,
total operations, and total successful operations. The data is being
@@ -154,6 +205,8 @@ operation was done on the service (e.g., when listing a bucket) in
which case it is accounted under the operating user.
.sp
Following is an example configuration:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
@@ -165,6 +218,8 @@ Following is an example configuration:
rgw usage max user shards = 1
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
The total number of shards determines how many total objects hold the
usage log information. The per\-user number of shards specify how many
@@ -174,7 +229,7 @@ threshold specify how many entries can be kept before resorting to
synchronous flush.
.SH AVAILABILITY
.sp
-\fBradosgw\fP is part of the Ceph distributed file system. Please refer
+\fBradosgw\fP is part of the Ceph distributed storage system. Please refer
to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more
information.
.SH SEE ALSO
@@ -182,6 +237,6 @@ information.
\fBceph\fP(8)
\fBradosgw\-admin\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/rbd-fuse.8 b/man/rbd-fuse.8
index a2989e6..3ba0636 100644
--- a/man/rbd-fuse.8
+++ b/man/rbd-fuse.8
@@ -1,4 +1,6 @@
-.TH "RBD-FUSE" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "RBD-FUSE" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
rbd-fuse \- expose rbd images as files
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBrbd\-fuse\fP [ \-p pool ] [\-c conffile] \fImountpoint\fP [ \fIfuse options\fP ]
@@ -40,15 +67,19 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
\fBrbd\-fuse\fP is a FUSE (File system in USErspace) client for RADOS
block device (rbd) images. Given a pool containing rbd images,
it will mount a userspace filesystem allowing access to those images
-as regular files at \fBmountpoint\fP.
+as regular files at \fBmountpoint\fP\&.
.sp
The file system can be unmounted with:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
fusermount \-u mountpoint
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
or by sending \fBSIGINT\fP to the \fBrbd\-fuse\fP process.
.SH OPTIONS
@@ -63,17 +94,17 @@ Use \fIceph.conf\fP configuration file instead of the default
.INDENT 0.0
.TP
.B \-p pool
-Use \fIpool\fP as the pool to search for rbd images. Default is \fBrbd\fP.
+Use \fIpool\fP as the pool to search for rbd images. Default is \fBrbd\fP\&.
.UNINDENT
.SH AVAILABILITY
.sp
-\fBrbd\-fuse\fP is part of the Ceph distributed file system. Please refer to
+\fBrbd\-fuse\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
fusermount(8),
\fBrbd\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/man/rbd.8 b/man/rbd.8
index 8804867..1bec65d 100644
--- a/man/rbd.8
+++ b/man/rbd.8
@@ -1,4 +1,6 @@
-.TH "RBD" "8" "April 29, 2013" "dev" "Ceph"
+.\" Man page generated from reStructuredText.
+.
+.TH "RBD" "8" "January 12, 2014" "dev" "Ceph"
.SH NAME
rbd \- manage rados block device (RBD) images
.
@@ -28,8 +30,33 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
-.\" Man page generated from reStructuredText.
.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
.SH SYNOPSIS
.nf
\fBrbd\fP [ \-c \fIceph.conf\fP ] [ \-m \fImonaddr\fP ] [ \-p | \-\-pool \fIpool\fP ] [
@@ -92,7 +119,7 @@ Specifies the size (in megabytes) of the new rbd image.
.TP
.B \-\-order bits
Specifies the object size expressed as a number of bits, such that
-the object size is \fB1 << order\fP. The default is 22 (4 MB).
+the object size is \fB1 << order\fP\&. The default is 22 (4 MB).
.UNINDENT
.INDENT 0.0
.TP
@@ -150,8 +177,15 @@ Make json or xml formatted output more human\-readable.
.UNINDENT
.INDENT 0.0
.TP
+.B \-o map\-options, \-\-options map\-options
+Specifies which options to use when mapping an image. map\-options is
+a comma\-separated string of options (similar to mount(8) mount options).
+See map options section below for more details.
+.UNINDENT
+.INDENT 0.0
+.TP
.B \-\-read\-only
-Set device readonly when mapping image.
+Map the image read\-only. Equivalent to \-o ro.
.UNINDENT
.SH COMMANDS
.INDENT 0.0
@@ -176,7 +210,7 @@ Object order will be identical to that of the parent image unless
specified. Size will be the same as the parent snapshot.
.sp
The parent snapshot must be protected (see \fIrbd snap protect\fP).
-This requires format 2.
+This requires image format 2.
.TP
.B \fBflatten\fP [\fIimage\-name\fP]
If image is a clone, copy all shared blocks from the parent snapshot and
@@ -184,13 +218,13 @@ make the child independent of the parent, severing the link between
parent snap and child. The parent snapshot can be unprotected and
deleted if it has no further dependent clones.
.sp
-This requires format 2.
+This requires image format 2.
.TP
.B \fBchildren\fP [\fIimage\-name\fP]
List the clones of the image at the given snapshot. This checks
every pool, and outputs the resulting poolname/imagename.
.sp
-This requires format 2.
+This requires image format 2.
.TP
.B \fBresize\fP [\fIimage\-name\fP] [\-\-allow\-shrink]
Resizes rbd image. The size parameter also needs to be specified.
@@ -231,7 +265,7 @@ whether the region is known to be zeros or may contain other data.
.TP
.B \fBcp\fP [\fIsrc\-image\fP] [\fIdest\-image\fP]
Copies the content of a src\-image into the newly created dest\-image.
-dest\-image will have the same size, order, and format as src\-image.
+dest\-image will have the same size, order, and image format as src\-image.
.TP
.B \fBmv\fP [\fIsrc\-image\fP] [\fIdest\-image\fP]
Renames an image. Note: rename across pools is not supported.
@@ -259,16 +293,16 @@ protection implies that there exist dependent cloned children that
refer to this snapshot. \fIrbd clone\fP will fail on a nonprotected
snapshot.
.sp
-This requires format 2.
+This requires image format 2.
.TP
.B \fBsnap\fP unprotect [\fIimage\-name\fP]
Unprotect a snapshot from deletion (undo \fIsnap protect\fP). If cloned
children remain, \fIsnap unprotect\fP fails. (Note that clones may exist
in different pools than the parent snapshot.)
.sp
-This requires format 2.
+This requires image format 2.
.TP
-.B \fBmap\fP [\fIimage\-name\fP]
+.B \fBmap\fP [\fIimage\-name\fP] [\-o | \-\-options \fImap\-options\fP ] [\-\-read\-only]
Maps the specified image to a block device via the rbd kernel module.
.TP
.B \fBunmap\fP [\fIdevice\-path\fP]
@@ -302,12 +336,16 @@ write throughput and latency. Defaults are: \-\-io\-size 4096, \-\-io\-threads
.sp
In addition to using the \-\-pool and the \-\-snap options, the image name can include both
the pool name and the snapshot name. The image name format is as follows:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
[pool/]image\-name[@snap]
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
Thus an image name that contains a slash character (\(aq/\(aq) requires specifying the pool
name explicitly.
@@ -350,141 +388,229 @@ The striping is controlled by three parameters:
By default, [\fIstripe_unit\fP] is the same as the object size and [\fIstripe_count\fP] is 1. Specifying a different
[\fIstripe_unit\fP] requires that the STRIPINGV2 feature be supported (added in Ceph v0.53) and format 2 images be
used.
+.SH MAP OPTIONS
+.sp
+Most of these options are useful mainly for debugging and benchmarking. The
+default values are set in the kernel and may therefore depend on the version of
+the running kernel.
+.INDENT 0.0
+.IP \(bu 2
+fsid=aaaaaaaa\-bbbb\-cccc\-dddd\-eeeeeeeeeeee \- FSID that should be assumed by
+the client.
+.IP \(bu 2
+ip=a.b.c.d[:p] \- IP and, optionally, port the client should use.
+.IP \(bu 2
+share \- Enable sharing of client instances with other mappings (default).
+.IP \(bu 2
+noshare \- Disable sharing of client instances with other mappings.
+.IP \(bu 2
+crc \- Enable CRC32C checksumming for data writes (default).
+.IP \(bu 2
+nocrc \- Disable CRC32C checksumming for data writes.
+.IP \(bu 2
+osdkeepalive=x \- OSD keepalive timeout (default is 5 seconds).
+.IP \(bu 2
+osd_idle_ttl=x \- OSD idle TTL (default is 60 seconds).
+.IP \(bu 2
+rw \- Map the image read\-write (default).
+.IP \(bu 2
+ro \- Map the image read\-only. Equivalent to \-\-read\-only.
+.UNINDENT
.SH EXAMPLES
.sp
To create a new rbd image that is 100 GB:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd \-p mypool create myimage \-\-size 102400
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
or alternatively:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd create mypool/myimage \-\-size 102400
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To use a non\-default object size (8 MB):
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd create mypool/myimage \-\-size 102400 \-\-order 23
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To delete an rbd image (be careful!):
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd rm mypool/myimage
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To create a new snapshot:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd snap create mypool/myimage at mysnap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To create a copy\-on\-write clone of a protected snapshot:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd clone mypool/myimage at mysnap otherpool/cloneimage
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To see which clones of a snapshot exist:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd children mypool/myimage at mysnap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To delete a snapshot:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd snap rm mypool/myimage at mysnap
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To map an image via the kernel with cephx enabled:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd map mypool/myimage \-\-id admin \-\-keyfile secretfile
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To unmap an image:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd unmap /dev/rbd0
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To create an image and a clone from it:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
-rbd import \-\-format 2 image mypool/parent
+rbd import \-\-image\-format 2 image mypool/parent
rbd snap create \-\-snap snapname mypool/parent
rbd snap protect mypool/parent at snap
rbd clone mypool/parent at snap otherpool/child
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To create an image with a smaller stripe_unit (to better distribute small writes in some workloads):
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd \-p mypool create myimage \-\-size 102400 \-\-stripe\-unit 65536 \-\-stripe\-count 16
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
-To change an image from one format to another, export it and then
-import it as the desired format:
+To change an image from one image format to another, export it and then
+import it as the desired image format:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd export mypool/myimage at snap /tmp/img
-rbd import \-\-format 2 /tmp/img mypool/myimage2
+rbd import \-\-image\-format 2 /tmp/img mypool/myimage2
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To lock an image for exclusive use:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd lock add mypool/myimage mylockid
.ft P
.fi
+.UNINDENT
+.UNINDENT
.sp
To release a lock:
+.INDENT 0.0
+.INDENT 3.5
.sp
.nf
.ft C
rbd lock remove mypool/myimage mylockid client.2485
.ft P
.fi
+.UNINDENT
+.UNINDENT
.SH AVAILABILITY
.sp
-\fBrbd\fP is part of the Ceph distributed file system. Please refer to
+\fBrbd\fP is part of the Ceph distributed storage system. Please refer to
the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information.
.SH SEE ALSO
.sp
\fBceph\fP(8),
\fBrados\fP(8)
.SH COPYRIGHT
-2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
+2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA
.\" Generated by docutils manpage writer.
.
diff --git a/src/.git_version b/src/.git_version
index c8c9163..95f3115 100644
--- a/src/.git_version
+++ b/src/.git_version
@@ -1,2 +1,2 @@
-a913ded2ff138aefb8cb84d347d72164099cfd60
-v0.72.2
+1bca9c5c412b3af722d5250f07fd562a23cf35ff
+v0.77
diff --git a/src/Makefile-env.am b/src/Makefile-env.am
index 9bc6ee7..f637eff 100644
--- a/src/Makefile-env.am
+++ b/src/Makefile-env.am
@@ -54,10 +54,11 @@ AM_COMMON_CPPFLAGS = \
-D_THREAD_SAFE \
-D__STDC_FORMAT_MACROS \
-D_GNU_SOURCE \
- -DCEPH_LIBDIR=\"${libdir}\"
+ -DCEPH_LIBDIR=\"${libdir}\" \
+ -DCEPH_PKGLIBDIR=\"${pkglibdir}\" \
+ -DGTEST_HAS_TR1_TUPLE=0
AM_COMMON_CFLAGS = \
- -rdynamic \
-Wall \
${WARN_TYPE_LIMITS} \
${WARN_IGNORED_QUALIFIERS} \
@@ -66,15 +67,21 @@ AM_COMMON_CFLAGS = \
-Werror=format-security \
-fno-strict-aliasing \
-fsigned-char
+if !CLANG
+ AM_COMMON_CFLAGS += -rdynamic
+endif
AM_CFLAGS = $(AM_COMMON_CFLAGS)
AM_CPPFLAGS = $(AM_COMMON_CPPFLAGS)
AM_CXXFLAGS = \
@AM_CXXFLAGS@ \
$(AM_COMMON_CFLAGS) \
+ -ftemplate-depth-1024 \
-Wnon-virtual-dtor \
- -Wno-invalid-offsetof \
- -Wstrict-null-sentinel
+ -Wno-invalid-offsetof
+if !CLANG
+ AM_CXXFLAGS += -Wstrict-null-sentinel
+endif
# note: this is position dependant, it affects the -l options that
# come after it on the command line. when you use ${AM_LDFLAGS} in
@@ -84,7 +91,10 @@ AM_CXXFLAGS = \
# http://www.gentoo.org/proj/en/qa/asneeded.xml
# http://gcc.gnu.org/ml/gcc-help/2010-12/msg00338.html
# http://sigquit.wordpress.com/2011/02/16/why-asneeded-doesnt-work-as-expected-for-your-libraries-on-your-autotools-project/
-AM_LDFLAGS = -Wl,--as-needed
+AM_LDFLAGS =
+if LINUX
+AM_LDFLAGS += -Wl,--as-needed
+endif
if USE_BOOST_SPIRIT_OLD_HDR
AM_CXXFLAGS += -DUSE_BOOST_SPIRIT_OLD_HDR
@@ -106,7 +116,7 @@ AM_CCASFLAGS = -f elf64
#####################
## library definitions and dependencies
-EXTRALIBS = -luuid -lm -lkeyutils
+EXTRALIBS = -luuid -lm $(KEYUTILS_LIB)
if FREEBSD
EXTRALIBS += -lexecinfo
endif # FREEBSD
@@ -168,7 +178,7 @@ LIBMDS += $(LIBPERFGLUE)
LIBOS += -lleveldb -lsnappy
# Use this for binaries requiring libglobal
-CEPH_GLOBAL = $(LIBGLOBAL) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS)
+CEPH_GLOBAL = $(LIBGLOBAL) $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS)
# This is set by [lib]/Makefile.am and used for build tests
LIBCOMMON_DEPS =
diff --git a/src/Makefile.am b/src/Makefile.am
index d9189bd..24f8fa9 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -35,18 +35,18 @@ include tools/Makefile.am
# core daemons
ceph_mon_SOURCES = ceph_mon.cc
-ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL)
+ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) $(LIBCOMMON)
bin_PROGRAMS += ceph-mon
ceph_osd_SOURCES = ceph_osd.cc
-ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL)
+ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON)
if LINUX
ceph_osd_LDADD += -ldl
endif # LINUX
bin_PROGRAMS += ceph-osd
ceph_mds_SOURCES = ceph_mds.cc
-ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL)
+ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL) $(LIBCOMMON)
bin_PROGRAMS += ceph-mds
@@ -75,8 +75,8 @@ ceph_syn_SOURCES += client/SyntheticClient.cc # uses g_conf.. needs cleanup
ceph_syn_LDADD = $(LIBCLIENT) $(CEPH_GLOBAL)
bin_PROGRAMS += ceph-syn
-rbd_SOURCES = rbd.cc
-rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL)
+rbd_SOURCES = rbd.cc
+rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) -lblkid
if LINUX
bin_PROGRAMS += rbd
endif #LINUX
@@ -128,7 +128,7 @@ editpaths = sed \
-e 's|@datadir[@]|$(pkgdatadir)|g' \
-e 's|@prefix[@]|$(prefix)|g' \
-e 's|@@GCOV_PREFIX_STRIP[@][@]|$(GCOV_PREFIX_STRIP)|g'
-shell_scripts = ceph-debugpack ceph-post-file
+shell_scripts = ceph-debugpack ceph-post-file ceph-crush-location
$(shell_scripts): Makefile
$(shell_scripts): %: %.in
rm -f $@ $@.tmp
@@ -173,6 +173,7 @@ EXTRA_DIST += \
$(srcdir)/upstart/radosgw.conf \
$(srcdir)/upstart/radosgw-all.conf \
$(srcdir)/upstart/radosgw-all-starter.conf \
+ $(srcdir)/upstart/rbdmap.conf \
ceph.in \
ceph-disk \
ceph-disk-prepare \
@@ -180,6 +181,7 @@ EXTRA_DIST += \
ceph-disk-udev \
ceph-create-keys \
ceph-rest-api \
+ ceph-crush-location \
mount.fuse.ceph \
rbdmap \
unittest_bufferlist.sh \
@@ -236,7 +238,8 @@ bin_SCRIPTS += \
ceph-clsinfo \
ceph-debugpack \
ceph-rbdnamer \
- ceph-post-file
+ ceph-post-file \
+ ceph-crush-location
BUILT_SOURCES += init-ceph
su_sbin_SCRIPTS += mkcephfs
@@ -308,7 +311,9 @@ ceph: ceph.in ./ceph_ver.h Makefile
# cleaning
clean-local:
- -rm *.so *.gcno *.gcda
+ rm -f *.so
+ find . -name '*.gcno' -o -name '*.gcda' -o -name '*.lcov' | xargs rm -f
+ rm -f ceph java/java/com/ceph/crush/Bucket.class
# pybind
diff --git a/src/Makefile.in b/src/Makefile.in
index 48e84aa..32d31ac 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -58,80 +58,93 @@ DIST_COMMON = README $(am__noinst_HEADERS_DIST) $(dist_bin_SCRIPTS) \
$(srcdir)/osd/Makefile.am $(srcdir)/osdc/Makefile.am \
$(srcdir)/perfglue/Makefile.am $(srcdir)/rgw/Makefile.am \
$(srcdir)/test/Makefile.am $(srcdir)/tools/Makefile.am TODO
-bin_PROGRAMS = $(am__EXEEXT_6) $(am__EXEEXT_7) ceph-dencoder$(EXEEXT) \
- ceph_filestore_tool$(EXEEXT) ceph_filestore_dump$(EXEEXT) \
- monmaptool$(EXEEXT) crushtool$(EXEEXT) osdmaptool$(EXEEXT) \
- rados$(EXEEXT) $(am__EXEEXT_8) ceph-conf$(EXEEXT) \
- ceph-authtool$(EXEEXT) ceph_mon_store_converter$(EXEEXT) \
- ceph-mon$(EXEEXT) ceph-osd$(EXEEXT) ceph-mds$(EXEEXT) \
- cephfs$(EXEEXT) librados-config$(EXEEXT) ceph-syn$(EXEEXT) \
- $(am__EXEEXT_9) $(am__EXEEXT_10)
+bin_PROGRAMS = $(am__EXEEXT_10) $(am__EXEEXT_11) \
+ ceph-dencoder$(EXEEXT) ceph_filestore_tool$(EXEEXT) \
+ ceph_filestore_dump$(EXEEXT) monmaptool$(EXEEXT) \
+ crushtool$(EXEEXT) osdmaptool$(EXEEXT) rados$(EXEEXT) \
+ $(am__EXEEXT_12) ceph-conf$(EXEEXT) ceph-authtool$(EXEEXT) \
+ ceph_mon_store_converter$(EXEEXT) ceph-mon$(EXEEXT) \
+ ceph-osd$(EXEEXT) ceph-mds$(EXEEXT) cephfs$(EXEEXT) \
+ librados-config$(EXEEXT) ceph-syn$(EXEEXT) $(am__EXEEXT_13) \
+ $(am__EXEEXT_14)
noinst_PROGRAMS = get_command_descriptions$(EXEEXT)
sbin_PROGRAMS =
-su_sbin_PROGRAMS = $(am__EXEEXT_11)
+su_sbin_PROGRAMS = $(am__EXEEXT_15)
check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
- unittest_bloom_filter$(EXEEXT) \
+ unittest_bloom_filter$(EXEEXT) unittest_str_map$(EXEEXT) \
+ unittest_crushwrapper$(EXEEXT) \
unittest_sharedptr_registry$(EXEEXT) \
unittest_sloppy_crc_map$(EXEEXT) unittest_util$(EXEEXT) \
+ unittest_crush_indep$(EXEEXT) unittest_osdmap$(EXEEXT) \
unittest_workqueue$(EXEEXT) unittest_striper$(EXEEXT) \
unittest_prebufferedstreambuf$(EXEEXT) \
unittest_str_list$(EXEEXT) unittest_log$(EXEEXT) \
- unittest_throttle$(EXEEXT) unittest_base64$(EXEEXT) \
- unittest_ceph_argparse$(EXEEXT) \
+ unittest_throttle$(EXEEXT) unittest_crush_wrapper$(EXEEXT) \
+ unittest_base64$(EXEEXT) unittest_ceph_argparse$(EXEEXT) \
unittest_ceph_compatset$(EXEEXT) \
unittest_erasure_code_plugin$(EXEEXT) \
unittest_erasure_code_jerasure$(EXEEXT) \
unittest_erasure_code_plugin_jerasure$(EXEEXT) \
unittest_erasure_code_example$(EXEEXT) \
unittest_osd_types$(EXEEXT) unittest_pglog$(EXEEXT) \
- unittest_gather$(EXEEXT) unittest_run_cmd$(EXEEXT) \
- unittest_signals$(EXEEXT) unittest_simple_spin$(EXEEXT) \
- unittest_librados$(EXEEXT) unittest_bufferlist$(EXEEXT) \
- unittest_crc32c$(EXEEXT) unittest_arch$(EXEEXT) \
- unittest_crypto$(EXEEXT) unittest_perf_counters$(EXEEXT) \
- unittest_admin_socket$(EXEEXT) unittest_ceph_crypto$(EXEEXT) \
- unittest_utf8$(EXEEXT) unittest_mime$(EXEEXT) \
- unittest_escape$(EXEEXT) unittest_chain_xattr$(EXEEXT) \
- unittest_flatindex$(EXEEXT) unittest_strtol$(EXEEXT) \
- unittest_confutils$(EXEEXT) unittest_heartbeatmap$(EXEEXT) \
+ unittest_hitset$(EXEEXT) unittest_gather$(EXEEXT) \
+ unittest_run_cmd$(EXEEXT) unittest_signals$(EXEEXT) \
+ unittest_simple_spin$(EXEEXT) unittest_librados$(EXEEXT) \
+ unittest_bufferlist$(EXEEXT) unittest_crc32c$(EXEEXT) \
+ unittest_arch$(EXEEXT) unittest_crypto$(EXEEXT) \
+ unittest_perf_counters$(EXEEXT) unittest_admin_socket$(EXEEXT) \
+ unittest_ceph_crypto$(EXEEXT) unittest_utf8$(EXEEXT) \
+ unittest_mime$(EXEEXT) unittest_escape$(EXEEXT) \
+ unittest_chain_xattr$(EXEEXT) unittest_flatindex$(EXEEXT) \
+ unittest_strtol$(EXEEXT) unittest_confutils$(EXEEXT) \
+ unittest_config$(EXEEXT) unittest_heartbeatmap$(EXEEXT) \
unittest_formatter$(EXEEXT) unittest_libcephfs_config$(EXEEXT) \
unittest_lfnindex$(EXEEXT) unittest_librados_config$(EXEEXT) \
unittest_daemon_config$(EXEEXT) unittest_osd_osdcap$(EXEEXT) \
unittest_mon_moncap$(EXEEXT) unittest_ipaddr$(EXEEXT) \
- unittest_texttable$(EXEEXT)
+ unittest_texttable$(EXEEXT) unittest_on_exit$(EXEEXT)
# when doing a debug build, make sure to make the targets
@WITH_DEBUG_TRUE at am__append_1 = $(bin_DEBUGPROGRAMS)
- at USE_BOOST_SPIRIT_OLD_HDR_TRUE@am__append_2 = -DUSE_BOOST_SPIRIT_OLD_HDR
- at WITH_LIBATOMIC_TRUE@am__append_3 = -latomic_ops
- at ENABLE_COVERAGE_TRUE@am__append_4 = -fprofile-arcs -ftest-coverage
- at ENABLE_COVERAGE_TRUE@am__append_5 = -fprofile-arcs -ftest-coverage -O0
- at FREEBSD_TRUE@am__append_6 = -lexecinfo
- at LINUX_TRUE@am__append_7 = -lrt
- at WITH_PROFILER_TRUE@am__append_8 = -lprofiler
- at WITH_LIBAIO_TRUE@am__append_9 = -laio
- at WITH_LIBZFS_TRUE@am__append_10 = libos_zfs.a -lzfs
- at WITH_TCMALLOC_TRUE@am__append_11 = -ltcmalloc
- at ENABLE_COVERAGE_TRUE@am__append_12 = -lgcov
- at WITH_LIBZFS_TRUE@am__append_13 = libos_zfs.a
- at WITH_LIBZFS_TRUE@am__append_14 = os/ZFS.h
- at WITH_FUSE_TRUE@am__append_15 = libclient_fuse.la
- at WITH_FUSE_TRUE@am__append_16 = client/fuse_ll.h
- at WITH_TCMALLOC_TRUE@am__append_17 = perfglue/heap_profiler.cc
- at WITH_TCMALLOC_TRUE@am__append_18 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
- at WITH_TCMALLOC_TRUE@am__append_19 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
- at WITH_TCMALLOC_FALSE@am__append_20 = perfglue/disabled_heap_profiler.cc
- at WITH_PROFILER_TRUE@am__append_21 = perfglue/cpu_profiler.cc
- at WITH_PROFILER_FALSE@am__append_22 = perfglue/disabled_stubs.cc
- at LINUX_TRUE@am__append_23 = common/secret.c
- at WITH_GOOD_YASM_ELF64_TRUE@am__append_24 = common/crc32c_intel_fast_asm.S common/crc32c_intel_fast_zero_asm.S
- at LINUX_TRUE@am__append_25 = -lrt
- at WITH_RADOSGW_TRUE@am__append_26 = librgw.la
- at WITH_RADOSGW_TRUE@am__append_27 = \
+ at LINUX_TRUE@am__append_2 = -Wl,--as-needed
+ at USE_BOOST_SPIRIT_OLD_HDR_TRUE@am__append_3 = -DUSE_BOOST_SPIRIT_OLD_HDR
+ at WITH_LIBATOMIC_TRUE@am__append_4 = -latomic_ops
+ at ENABLE_COVERAGE_TRUE@am__append_5 = -fprofile-arcs -ftest-coverage
+ at ENABLE_COVERAGE_TRUE@am__append_6 = -fprofile-arcs -ftest-coverage -O0
+ at FREEBSD_TRUE@am__append_7 = -lexecinfo
+ at LINUX_TRUE@am__append_8 = -lrt
+ at WITH_PROFILER_TRUE@am__append_9 = -lprofiler
+ at WITH_LIBAIO_TRUE@am__append_10 = -laio
+ at WITH_LIBZFS_TRUE@am__append_11 = libos_zfs.a -lzfs
+ at WITH_TCMALLOC_TRUE@am__append_12 = -ltcmalloc
+ at ENABLE_COVERAGE_TRUE@am__append_13 = -lgcov
+ at LINUX_TRUE@am__append_14 = os/BtrfsFileStoreBackend.cc
+ at WITH_LIBZFS_TRUE@am__append_15 = os/ZFSFileStoreBackend.cc
+ at WITH_LIBZFS_TRUE@am__append_16 = libos_zfs.a
+ at WITH_LIBZFS_TRUE@am__append_17 = os/ZFS.h
+ at LINUX_TRUE@am__append_18 = -export-symbols-regex '.*__erasure_code_.*'
+ at WITH_FUSE_TRUE@am__append_19 = libclient_fuse.la
+ at WITH_FUSE_TRUE@am__append_20 = client/fuse_ll.h
+ at WITH_TCMALLOC_TRUE@am__append_21 = perfglue/heap_profiler.cc
+ at WITH_TCMALLOC_TRUE@am__append_22 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
+ at WITH_TCMALLOC_TRUE@am__append_23 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
+ at WITH_TCMALLOC_FALSE@am__append_24 = perfglue/disabled_heap_profiler.cc
+ at WITH_PROFILER_TRUE@am__append_25 = perfglue/cpu_profiler.cc
+ at WITH_PROFILER_FALSE@am__append_26 = perfglue/disabled_stubs.cc
+ at LINUX_TRUE@am__append_27 = \
+ at LINUX_TRUE@ common/secret.c
+
+ at WITH_GOOD_YASM_ELF64_TRUE@am__append_28 = common/crc32c_intel_fast_asm.S common/crc32c_intel_fast_zero_asm.S
+ at LINUX_TRUE@am__append_29 = -lrt
+ at LINUX_TRUE@am__append_30 = -export-symbols-regex '^rados_.*'
+ at LINUX_TRUE@am__append_31 = -export-symbols-regex '^rbd_.*'
+ at WITH_RADOSGW_TRUE@am__append_32 = librgw.la
+ at WITH_RADOSGW_TRUE@am__append_33 = \
@WITH_RADOSGW_TRUE@ $(LIBRADOS) \
@WITH_RADOSGW_TRUE@ libcls_rgw_client.la \
@WITH_RADOSGW_TRUE@ libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_user_client.a \
@WITH_RADOSGW_TRUE@ libcls_replica_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_lock_client.la \
@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
@@ -139,45 +152,56 @@ check_PROGRAMS = unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \
@WITH_RADOSGW_TRUE@ -lcurl \
@WITH_RADOSGW_TRUE@ -lexpat \
@WITH_RADOSGW_TRUE@ -lm \
- at WITH_RADOSGW_TRUE@ -lfcgi
+ at WITH_RADOSGW_TRUE@ -lfcgi \
+ at WITH_RADOSGW_TRUE@ -ldl
- at WITH_RADOSGW_TRUE@am__append_28 = radosgw radosgw-admin
- at WITH_RADOSGW_TRUE@am__append_29 = ceph_rgw_multiparser \
+ at WITH_RADOSGW_TRUE@am__append_34 = radosgw radosgw-admin
+ at WITH_RADOSGW_TRUE@am__append_35 = ceph_rgw_multiparser \
@WITH_RADOSGW_TRUE@ ceph_rgw_jsonparser
# inject rgw stuff in the decoder testcase
- at WITH_RADOSGW_TRUE@am__append_30 = \
+ at WITH_RADOSGW_TRUE@am__append_36 = \
@WITH_RADOSGW_TRUE@ rgw/rgw_dencoder.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_acl.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_common.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_env.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.cc
- at COMPILER_HAS_VTA_TRUE@am__append_31 = -fno-var-tracking-assignments
- at COMPILER_HAS_VTA_TRUE@am__append_32 = -fno-var-tracking-assignments
- at WITH_BUILD_TESTS_TRUE@am__append_33 = test_build_libcommon \
+ at LINUX_TRUE@am__append_37 = libcls_kvs.la
+ at COMPILER_HAS_VTA_TRUE@am__append_38 = -fno-var-tracking-assignments
+ at COMPILER_HAS_VTA_TRUE@am__append_39 = -fno-var-tracking-assignments
+ at WITH_BUILD_TESTS_TRUE@am__append_40 = test_build_libcommon \
@WITH_BUILD_TESTS_TRUE@ test_build_librados test_build_librgw \
@WITH_BUILD_TESTS_TRUE@ test_build_libcephfs
- at LINUX_TRUE@am__append_34 = -ldl
- at LINUX_TRUE@am__append_35 = -ldl
- at LINUX_TRUE@am__append_36 = -ldl
- at LINUX_TRUE@am__append_37 = -ldl
- at WITH_RADOSGW_TRUE@am__append_38 = ceph_test_cors \
+ at LINUX_TRUE@am__append_41 = ceph_kvstorebench
+ at LINUX_TRUE@am__append_42 = -ldl
+ at LINUX_TRUE@am__append_43 = libsystest.la
+ at LINUX_TRUE@am__append_44 = ceph_test_rados_list_parallel \
+ at LINUX_TRUE@ ceph_test_rados_open_pools_parallel \
+ at LINUX_TRUE@ ceph_test_rados_delete_pools_parallel \
+ at LINUX_TRUE@ ceph_test_rados_watch_notify
+ at LINUX_TRUE@am__append_45 = -ldl
+ at LINUX_TRUE@am__append_46 = -ldl
+ at LINUX_TRUE@am__append_47 = -ldl
+ at LINUX_TRUE@am__append_48 = -ldl
+ at WITH_RADOSGW_TRUE@am__append_49 = ceph_test_cors \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_meta \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_log \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_opstate
- at WITH_RADOSGW_TRUE@am__append_39 = ceph_test_cls_rgw
- at LINUX_TRUE@am__append_40 = -ldl
- at LINUX_TRUE@am__append_41 = -ldl
- at WITH_REST_BENCH_TRUE@am__append_42 = rest-bench
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_TRUE at am__append_43 = -ls3
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_44 = libs3/build/lib/libs3.a -lcurl -lxml2
- at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_45 = libs3
- at LINUX_TRUE@am__append_46 = -ldl
- at LINUX_TRUE@am__append_47 = mount.ceph
- at LINUX_TRUE@am__append_48 = rbd
- at WITH_FUSE_TRUE@am__append_49 = ceph-fuse rbd-fuse
- at ENABLE_CEPHFS_JAVA_TRUE@am__append_50 = libcephfs_jni.la
+ at LINUX_TRUE@am__append_50 = ceph_test_librbd_fsx
+ at WITH_RADOSGW_TRUE@am__append_51 = ceph_test_cls_rgw
+ at LINUX_TRUE@am__append_52 = ceph_test_filestore
+ at LINUX_TRUE@am__append_53 = -ldl
+ at LINUX_TRUE@am__append_54 = -ldl
+ at WITH_REST_BENCH_TRUE@am__append_55 = rest-bench
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_TRUE at am__append_56 = -ls3
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_57 = libs3/build/lib/libs3.a -lcurl -lxml2
+ at WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at am__append_58 = libs3
+ at LINUX_TRUE@am__append_59 = -ldl
+ at LINUX_TRUE@am__append_60 = mount.ceph
+ at LINUX_TRUE@am__append_61 = rbd
+ at WITH_FUSE_TRUE@am__append_62 = ceph-fuse rbd-fuse
+ at ENABLE_CEPHFS_JAVA_TRUE@am__append_63 = libcephfs_jni.la
subdir = src
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
@@ -186,7 +210,10 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
$(top_srcdir)/m4/ac_prog_javac_works.m4 \
$(top_srcdir)/m4/ac_prog_javah.m4 \
$(top_srcdir)/m4/acx_pthread.m4 \
+ $(top_srcdir)/m4/ax_c_pretty_func.m4 \
+ $(top_srcdir)/m4/ax_c_var_func.m4 \
$(top_srcdir)/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/m4/ax_cxx_static_cast.m4 \
$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/pkg.m4 \
@@ -224,6 +251,12 @@ am_libcls_statelog_client_a_OBJECTS = \
cls/statelog/cls_statelog_client.$(OBJEXT)
libcls_statelog_client_a_OBJECTS = \
$(am_libcls_statelog_client_a_OBJECTS)
+libcls_user_client_a_AR = $(AR) $(ARFLAGS)
+libcls_user_client_a_LIBADD =
+am_libcls_user_client_a_OBJECTS = cls/user/cls_user_client.$(OBJEXT) \
+ cls/user/cls_user_types.$(OBJEXT) \
+ cls/user/cls_user_ops.$(OBJEXT)
+libcls_user_client_a_OBJECTS = $(am_libcls_user_client_a_OBJECTS)
libcls_version_client_a_AR = $(AR) $(ARFLAGS)
libcls_version_client_a_LIBADD =
am_libcls_version_client_a_OBJECTS = \
@@ -297,7 +330,8 @@ am_libauth_la_OBJECTS = auth/AuthAuthorizeHandler.lo \
libauth_la_OBJECTS = $(am_libauth_la_OBJECTS)
am__DEPENDENCIES_1 =
am__DEPENDENCIES_2 = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
- $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
+ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_1)
libcephfs_la_DEPENDENCIES = $(LIBCLIENT) $(LIBCOMMON) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
@@ -338,14 +372,16 @@ libcls_hello_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_hello_la_LDFLAGS) \
$(LDFLAGS) -o $@
-libcls_kvs_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
- $(am__DEPENDENCIES_2)
-am_libcls_kvs_la_OBJECTS = key_value_store/cls_kvs.lo
+ at LINUX_TRUE@libcls_kvs_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
+ at LINUX_TRUE@ $(am__DEPENDENCIES_2)
+am__libcls_kvs_la_SOURCES_DIST = key_value_store/cls_kvs.cc
+ at LINUX_TRUE@am_libcls_kvs_la_OBJECTS = key_value_store/cls_kvs.lo
libcls_kvs_la_OBJECTS = $(am_libcls_kvs_la_OBJECTS)
libcls_kvs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_kvs_la_LDFLAGS) $(LDFLAGS) \
-o $@
+ at LINUX_TRUE@am_libcls_kvs_la_rpath = -rpath $(radoslibdir)
libcls_lock_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
am_libcls_lock_la_OBJECTS = cls/lock/cls_lock.lo
@@ -421,6 +457,14 @@ libcls_statelog_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_statelog_la_LDFLAGS) \
$(LDFLAGS) -o $@
+libcls_user_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_2)
+am_libcls_user_la_OBJECTS = cls/user/cls_user.lo
+libcls_user_la_OBJECTS = $(am_libcls_user_la_OBJECTS)
+libcls_user_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_user_la_LDFLAGS) \
+ $(LDFLAGS) -o $@
libcls_version_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2)
am_libcls_version_la_OBJECTS = cls/version/cls_version.lo
@@ -430,7 +474,8 @@ libcls_version_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_version_la_LDFLAGS) \
$(LDFLAGS) -o $@
am__DEPENDENCIES_3 = libcommon_crc.la $(LIBMSG) $(LIBAUTH) $(LIBCRUSH) \
- $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) $(am__DEPENDENCIES_1)
+ $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_1)
libcommon_la_DEPENDENCIES = $(am__DEPENDENCIES_3)
am__libcommon_la_SOURCES_DIST = ceph_ver.c common/DecayCounter.cc \
common/LogClient.cc common/LogEntry.cc \
@@ -443,22 +488,24 @@ am__libcommon_la_SOURCES_DIST = ceph_ver.c common/DecayCounter.cc \
common/assert.cc common/run_cmd.cc common/WorkQueue.cc \
common/ConfUtils.cc common/MemoryModel.cc common/armor.c \
common/fd.cc common/xattr.c common/safe_io.c \
- common/snap_types.cc common/str_list.cc common/errno.cc \
- common/RefCountedObj.cc common/blkdev.cc common/common_init.cc \
- common/pipe.c common/ceph_argparse.cc common/ceph_context.cc \
- common/buffer.cc common/code_environment.cc common/dout.cc \
- common/signal.cc common/simple_spin.cc common/Thread.cc \
- common/Formatter.cc common/HeartbeatMap.cc common/config.cc \
- common/utf8.c common/mime.c common/strtol.cc common/page.cc \
+ common/snap_types.cc common/str_list.cc common/str_map.cc \
+ common/errno.cc common/RefCountedObj.cc common/blkdev.cc \
+ common/common_init.cc common/pipe.c common/ceph_argparse.cc \
+ common/ceph_context.cc common/buffer.cc \
+ common/code_environment.cc common/dout.cc common/signal.cc \
+ common/simple_spin.cc common/Thread.cc common/Formatter.cc \
+ common/HeartbeatMap.cc common/config.cc common/utf8.c \
+ common/mime.c common/strtol.cc common/page.cc \
common/lockdep.cc common/version.cc common/hex.cc \
common/entity_name.cc common/ceph_crypto.cc \
common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \
common/pick_address.cc common/util.cc common/TextTable.cc \
common/ceph_fs.cc common/ceph_hash.cc common/ceph_strings.cc \
common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \
- common/bloom_filter.cc common/secret.c mon/MonCap.cc \
- mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc osd/osd_types.cc \
- mds/MDSMap.cc mds/inode_backtrace.cc mds/mdstypes.cc
+ common/bloom_filter.cc common/linux_version.c common/secret.c \
+ mon/MonCap.cc mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc \
+ osd/osd_types.cc osd/HitSet.cc mds/MDSMap.cc \
+ mds/inode_backtrace.cc mds/mdstypes.cc
@LINUX_TRUE at am__objects_1 = common/secret.lo
am_libcommon_la_OBJECTS = ceph_ver.lo common/DecayCounter.lo \
common/LogClient.lo common/LogEntry.lo \
@@ -471,22 +518,24 @@ am_libcommon_la_OBJECTS = ceph_ver.lo common/DecayCounter.lo \
common/assert.lo common/run_cmd.lo common/WorkQueue.lo \
common/ConfUtils.lo common/MemoryModel.lo common/armor.lo \
common/fd.lo common/xattr.lo common/safe_io.lo \
- common/snap_types.lo common/str_list.lo common/errno.lo \
- common/RefCountedObj.lo common/blkdev.lo common/common_init.lo \
- common/pipe.lo common/ceph_argparse.lo common/ceph_context.lo \
- common/buffer.lo common/code_environment.lo common/dout.lo \
- common/signal.lo common/simple_spin.lo common/Thread.lo \
- common/Formatter.lo common/HeartbeatMap.lo common/config.lo \
- common/utf8.lo common/mime.lo common/strtol.lo common/page.lo \
+ common/snap_types.lo common/str_list.lo common/str_map.lo \
+ common/errno.lo common/RefCountedObj.lo common/blkdev.lo \
+ common/common_init.lo common/pipe.lo common/ceph_argparse.lo \
+ common/ceph_context.lo common/buffer.lo \
+ common/code_environment.lo common/dout.lo common/signal.lo \
+ common/simple_spin.lo common/Thread.lo common/Formatter.lo \
+ common/HeartbeatMap.lo common/config.lo common/utf8.lo \
+ common/mime.lo common/strtol.lo common/page.lo \
common/lockdep.lo common/version.lo common/hex.lo \
common/entity_name.lo common/ceph_crypto.lo \
common/ceph_crypto_cms.lo common/ceph_json.lo common/ipaddr.lo \
common/pick_address.lo common/util.lo common/TextTable.lo \
common/ceph_fs.lo common/ceph_hash.lo common/ceph_strings.lo \
common/ceph_frag.lo common/addr_parsing.lo common/hobject.lo \
- common/bloom_filter.lo $(am__objects_1) mon/MonCap.lo \
- mon/MonClient.lo mon/MonMap.lo osd/OSDMap.lo osd/osd_types.lo \
- mds/MDSMap.lo mds/inode_backtrace.lo mds/mdstypes.lo
+ common/bloom_filter.lo common/linux_version.lo \
+ $(am__objects_1) mon/MonCap.lo mon/MonClient.lo mon/MonMap.lo \
+ osd/OSDMap.lo osd/osd_types.lo osd/HitSet.lo mds/MDSMap.lo \
+ mds/inode_backtrace.lo mds/mdstypes.lo
libcommon_la_OBJECTS = $(am_libcommon_la_OBJECTS)
libcommon_crc_la_LIBADD =
am__libcommon_crc_la_SOURCES_DIST = common/sctp_crc32.c \
@@ -575,8 +624,7 @@ am_libglobal_la_OBJECTS = global/global_context.lo \
libglobal_la_OBJECTS = $(am_libglobal_la_OBJECTS)
libjson_spirit_la_LIBADD =
am_libjson_spirit_la_OBJECTS = json_spirit/json_spirit_reader.lo \
- json_spirit/json_spirit_writer.lo \
- json_spirit/json_spirit_value.lo
+ json_spirit/json_spirit_writer.lo
libjson_spirit_la_OBJECTS = $(am_libjson_spirit_la_OBJECTS)
liblog_la_LIBADD =
am_liblog_la_OBJECTS = log/Log.lo log/SubsystemMap.lo
@@ -608,20 +656,31 @@ am_libmsg_la_OBJECTS = msg/Accepter.lo msg/DispatchQueue.lo \
msg/SimpleMessenger.lo msg/msg_types.lo
libmsg_la_OBJECTS = $(am_libmsg_la_OBJECTS)
libos_la_LIBADD =
-am_libos_la_OBJECTS = os/FileJournal.lo os/FileStore.lo \
- os/chain_xattr.lo os/ObjectStore.lo \
- os/JournalingObjectStore.lo os/LFNIndex.lo os/HashIndex.lo \
- os/IndexManager.lo os/FlatIndex.lo os/DBObjectMap.lo \
- os/LevelDBStore.lo os/WBThrottle.lo \
- os/BtrfsFileStoreBackend.lo os/GenericFileStoreBackend.lo \
- os/ZFSFileStoreBackend.lo common/TrackedOp.lo
+am__libos_la_SOURCES_DIST = os/chain_xattr.cc os/DBObjectMap.cc \
+ os/GenericObjectMap.cc os/FileJournal.cc os/FileStore.cc \
+ os/FlatIndex.cc os/GenericFileStoreBackend.cc os/HashIndex.cc \
+ os/IndexManager.cc os/JournalingObjectStore.cc \
+ os/LevelDBStore.cc os/LFNIndex.cc os/MemStore.cc \
+ os/KeyValueStore.cc os/ObjectStore.cc os/WBThrottle.cc \
+ common/TrackedOp.cc os/BtrfsFileStoreBackend.cc \
+ os/ZFSFileStoreBackend.cc
+ at LINUX_TRUE@am__objects_3 = os/BtrfsFileStoreBackend.lo
+ at WITH_LIBZFS_TRUE@am__objects_4 = os/ZFSFileStoreBackend.lo
+am_libos_la_OBJECTS = os/chain_xattr.lo os/DBObjectMap.lo \
+ os/GenericObjectMap.lo os/FileJournal.lo os/FileStore.lo \
+ os/FlatIndex.lo os/GenericFileStoreBackend.lo os/HashIndex.lo \
+ os/IndexManager.lo os/JournalingObjectStore.lo \
+ os/LevelDBStore.lo os/LFNIndex.lo os/MemStore.lo \
+ os/KeyValueStore.lo os/ObjectStore.lo os/WBThrottle.lo \
+ common/TrackedOp.lo $(am__objects_3) $(am__objects_4)
libos_la_OBJECTS = $(am_libos_la_OBJECTS)
libosd_la_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_5)
am_libosd_la_OBJECTS = osd/ErasureCodePlugin.lo osd/PG.lo osd/PGLog.lo \
- osd/ReplicatedPG.lo osd/ReplicatedBackend.lo osd/Ager.lo \
- osd/OSD.lo osd/OSDCap.lo osd/Watch.lo osd/ClassHandler.lo \
- osd/OpRequest.lo common/TrackedOp.lo osd/SnapMapper.lo \
- osd/osd_types.lo objclass/class_api.lo
+ osd/ReplicatedPG.lo osd/ReplicatedBackend.lo osd/PGBackend.lo \
+ osd/Ager.lo osd/HitSet.lo osd/OSD.lo osd/OSDCap.lo \
+ osd/Watch.lo osd/ClassHandler.lo osd/OpRequest.lo \
+ common/TrackedOp.lo osd/SnapMapper.lo osd/osd_types.lo \
+ objclass/class_api.lo
libosd_la_OBJECTS = $(am_libosd_la_OBJECTS)
libosdc_la_LIBADD =
am_libosdc_la_OBJECTS = osdc/Objecter.lo osdc/ObjectCacher.lo \
@@ -631,21 +690,23 @@ libperfglue_la_DEPENDENCIES =
am__libperfglue_la_SOURCES_DIST = perfglue/heap_profiler.cc \
perfglue/disabled_heap_profiler.cc perfglue/cpu_profiler.cc \
perfglue/disabled_stubs.cc
- at WITH_TCMALLOC_TRUE@am__objects_3 = perfglue/heap_profiler.lo
- at WITH_TCMALLOC_FALSE@am__objects_4 = \
+ at WITH_TCMALLOC_TRUE@am__objects_5 = perfglue/heap_profiler.lo
+ at WITH_TCMALLOC_FALSE@am__objects_6 = \
@WITH_TCMALLOC_FALSE@ perfglue/disabled_heap_profiler.lo
- at WITH_PROFILER_TRUE@am__objects_5 = perfglue/cpu_profiler.lo
- at WITH_PROFILER_FALSE@am__objects_6 = perfglue/disabled_stubs.lo
-am_libperfglue_la_OBJECTS = $(am__objects_3) $(am__objects_4) \
- $(am__objects_5) $(am__objects_6)
+ at WITH_PROFILER_TRUE@am__objects_7 = perfglue/cpu_profiler.lo
+ at WITH_PROFILER_FALSE@am__objects_8 = perfglue/disabled_stubs.lo
+am_libperfglue_la_OBJECTS = $(am__objects_5) $(am__objects_6) \
+ $(am__objects_7) $(am__objects_8)
libperfglue_la_OBJECTS = $(am_libperfglue_la_OBJECTS)
librados_la_DEPENDENCIES = $(LIBRADOS_DEPS) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2)
-am_librados_la_OBJECTS = librados/librados.lo librados/RadosClient.lo \
- librados/IoCtxImpl.lo librados/snap_set_diff.lo
+am_librados_la_OBJECTS = librados/librados_la-librados.lo \
+ librados/librados_la-RadosClient.lo \
+ librados/librados_la-IoCtxImpl.lo \
+ librados/librados_la-snap_set_diff.lo
librados_la_OBJECTS = $(am_librados_la_OBJECTS)
librados_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
- $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(librados_la_CXXFLAGS) \
$(CXXFLAGS) $(librados_la_LDFLAGS) $(LDFLAGS) -o $@
librbd_la_DEPENDENCIES = $(LIBRADOS) $(LIBOSDC) libcls_rbd_client.la \
libcls_lock_client.la $(am__DEPENDENCIES_1) \
@@ -708,25 +769,37 @@ librgw_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(librgw_la_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
@WITH_RADOSGW_TRUE at am_librgw_la_rpath =
-am__DEPENDENCIES_6 = $(LIBGLOBAL) $(am__DEPENDENCIES_1) \
+am__DEPENDENCIES_6 = $(LIBGLOBAL) $(LIBCOMMON) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2)
-libsystest_la_DEPENDENCIES = $(am__DEPENDENCIES_6)
-am_libsystest_la_OBJECTS = test/system/cross_process_sem.lo \
- test/system/systest_runnable.lo \
- test/system/systest_settings.lo
+ at LINUX_TRUE@libsystest_la_DEPENDENCIES = $(am__DEPENDENCIES_6)
+am__libsystest_la_SOURCES_DIST = test/system/cross_process_sem.cc \
+ test/system/systest_runnable.cc \
+ test/system/systest_settings.cc
+ at LINUX_TRUE@am_libsystest_la_OBJECTS = \
+ at LINUX_TRUE@ test/system/cross_process_sem.lo \
+ at LINUX_TRUE@ test/system/systest_runnable.lo \
+ at LINUX_TRUE@ test/system/systest_settings.lo
libsystest_la_OBJECTS = $(am_libsystest_la_OBJECTS)
+ at LINUX_TRUE@am_libsystest_la_rpath =
@WITH_RADOSGW_TRUE at am__EXEEXT_1 = ceph_rgw_multiparser$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_rgw_jsonparser$(EXEEXT)
@WITH_BUILD_TESTS_TRUE at am__EXEEXT_2 = test_build_libcommon$(EXEEXT) \
@WITH_BUILD_TESTS_TRUE@ test_build_librados$(EXEEXT) \
@WITH_BUILD_TESTS_TRUE@ test_build_librgw$(EXEEXT) \
@WITH_BUILD_TESTS_TRUE@ test_build_libcephfs$(EXEEXT)
- at WITH_RADOSGW_TRUE@am__EXEEXT_3 = ceph_test_cors$(EXEEXT) \
+ at LINUX_TRUE@am__EXEEXT_3 = ceph_kvstorebench$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_4 = ceph_test_rados_list_parallel$(EXEEXT) \
+ at LINUX_TRUE@ ceph_test_rados_open_pools_parallel$(EXEEXT) \
+ at LINUX_TRUE@ ceph_test_rados_delete_pools_parallel$(EXEEXT) \
+ at LINUX_TRUE@ ceph_test_rados_watch_notify$(EXEEXT)
+ at WITH_RADOSGW_TRUE@am__EXEEXT_5 = ceph_test_cors$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_meta$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_log$(EXEEXT) \
@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_opstate$(EXEEXT)
- at WITH_RADOSGW_TRUE@am__EXEEXT_4 = ceph_test_cls_rgw$(EXEEXT)
-am__EXEEXT_5 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
+ at LINUX_TRUE@am__EXEEXT_6 = ceph_test_librbd_fsx$(EXEEXT)
+ at WITH_RADOSGW_TRUE@am__EXEEXT_7 = ceph_test_cls_rgw$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_8 = ceph_test_filestore$(EXEEXT)
+am__EXEEXT_9 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
ceph_test_timers$(EXEEXT) ceph_test_signal_handlers$(EXEEXT) \
ceph_test_rados$(EXEEXT) ceph_test_mutate$(EXEEXT) \
ceph_test_rewrite_latency$(EXEEXT) ceph_test_msgr$(EXEEXT) \
@@ -735,21 +808,20 @@ am__EXEEXT_5 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
$(am__EXEEXT_2) ceph_smalliobench$(EXEEXT) \
ceph_smalliobenchfs$(EXEEXT) ceph_smalliobenchdumb$(EXEEXT) \
ceph_smalliobenchrbd$(EXEEXT) ceph_tpbench$(EXEEXT) \
- ceph_omapbench$(EXEEXT) ceph_kvstorebench$(EXEEXT) \
+ ceph_omapbench$(EXEEXT) $(am__EXEEXT_3) \
ceph_multi_stress_watch$(EXEEXT) \
- ceph_test_rados_list_parallel$(EXEEXT) \
- ceph_test_rados_open_pools_parallel$(EXEEXT) \
- ceph_test_rados_delete_pools_parallel$(EXEEXT) \
- ceph_test_rados_watch_notify$(EXEEXT) ceph_bench_log$(EXEEXT) \
- $(am__EXEEXT_3) ceph_test_librbd$(EXEEXT) \
- ceph_test_librbd_fsx$(EXEEXT) ceph_test_cls_rbd$(EXEEXT) \
- ceph_test_cls_refcount$(EXEEXT) ceph_test_cls_version$(EXEEXT) \
- ceph_test_cls_log$(EXEEXT) ceph_test_cls_statelog$(EXEEXT) \
+ ceph_erasure_code_benchmark$(EXEEXT) $(am__EXEEXT_4) \
+ ceph_bench_log$(EXEEXT) $(am__EXEEXT_5) \
+ ceph_test_librbd$(EXEEXT) $(am__EXEEXT_6) \
+ ceph_test_cls_rbd$(EXEEXT) ceph_test_cls_refcount$(EXEEXT) \
+ ceph_test_cls_version$(EXEEXT) ceph_test_cls_log$(EXEEXT) \
+ ceph_test_cls_statelog$(EXEEXT) \
ceph_test_cls_replica_log$(EXEEXT) ceph_test_cls_lock$(EXEEXT) \
- ceph_test_cls_hello$(EXEEXT) $(am__EXEEXT_4) \
+ ceph_test_cls_hello$(EXEEXT) $(am__EXEEXT_7) \
ceph_test_mon_workloadgen$(EXEEXT) \
ceph_test_rados_api_cmd$(EXEEXT) \
ceph_test_rados_api_io$(EXEEXT) \
+ ceph_test_rados_api_c_write_operations$(EXEEXT) \
ceph_test_rados_api_aio$(EXEEXT) \
ceph_test_rados_api_list$(EXEEXT) \
ceph_test_rados_api_pool$(EXEEXT) \
@@ -758,9 +830,9 @@ am__EXEEXT_5 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
ceph_test_rados_api_snapshots$(EXEEXT) \
ceph_test_rados_api_cls$(EXEEXT) \
ceph_test_rados_api_misc$(EXEEXT) \
+ ceph_test_rados_api_tier$(EXEEXT) \
ceph_test_rados_api_lock$(EXEEXT) ceph_test_libcephfs$(EXEEXT) \
- ceph_test_filestore$(EXEEXT) \
- ceph_test_filestore_workloadgen$(EXEEXT) \
+ $(am__EXEEXT_8) ceph_test_filestore_workloadgen$(EXEEXT) \
ceph_test_filestore_idempotent$(EXEEXT) \
ceph_test_filestore_idempotent_sequence$(EXEEXT) \
ceph_xattr_bench$(EXEEXT) ceph_test_filejournal$(EXEEXT) \
@@ -770,38 +842,40 @@ am__EXEEXT_5 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \
ceph_test_keyvaluedb_atomicity$(EXEEXT) \
ceph_test_keyvaluedb_iterators$(EXEEXT) \
ceph_test_cfuse_cache_invalidate$(EXEEXT) \
- ceph-osdomap-tool$(EXEEXT) ceph-monstore-tool$(EXEEXT) \
- ceph-kvstore-tool$(EXEEXT) ceph_scratchtool$(EXEEXT) \
- ceph_scratchtoolpp$(EXEEXT) ceph_psim$(EXEEXT) \
- ceph_dupstore$(EXEEXT) ceph_radosacl$(EXEEXT)
- at WITH_DEBUG_TRUE@am__EXEEXT_6 = $(am__EXEEXT_5)
- at WITH_RADOSGW_TRUE@am__EXEEXT_7 = radosgw$(EXEEXT) \
+ ceph_test_c_headers$(EXEEXT) \
+ ceph_test_get_blkdev_size$(EXEEXT) ceph-osdomap-tool$(EXEEXT) \
+ ceph-monstore-tool$(EXEEXT) ceph-kvstore-tool$(EXEEXT) \
+ ceph_scratchtool$(EXEEXT) ceph_scratchtoolpp$(EXEEXT) \
+ ceph_psim$(EXEEXT) ceph_dupstore$(EXEEXT) \
+ ceph_radosacl$(EXEEXT)
+ at WITH_DEBUG_TRUE@am__EXEEXT_10 = $(am__EXEEXT_9)
+ at WITH_RADOSGW_TRUE@am__EXEEXT_11 = radosgw$(EXEEXT) \
@WITH_RADOSGW_TRUE@ radosgw-admin$(EXEEXT)
- at WITH_REST_BENCH_TRUE@am__EXEEXT_8 = rest-bench$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_9 = rbd$(EXEEXT)
- at WITH_FUSE_TRUE@am__EXEEXT_10 = ceph-fuse$(EXEEXT) rbd-fuse$(EXEEXT)
- at LINUX_TRUE@am__EXEEXT_11 = mount.ceph$(EXEEXT)
+ at WITH_REST_BENCH_TRUE@am__EXEEXT_12 = rest-bench$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_13 = rbd$(EXEEXT)
+ at WITH_FUSE_TRUE@am__EXEEXT_14 = ceph-fuse$(EXEEXT) rbd-fuse$(EXEEXT)
+ at LINUX_TRUE@am__EXEEXT_15 = mount.ceph$(EXEEXT)
PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) $(sbin_PROGRAMS) \
$(su_sbin_PROGRAMS)
am_ceph_authtool_OBJECTS = tools/ceph_authtool.$(OBJEXT)
ceph_authtool_OBJECTS = $(am_ceph_authtool_OBJECTS)
-ceph_authtool_DEPENDENCIES = $(am__DEPENDENCIES_6)
+ceph_authtool_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON)
am_ceph_conf_OBJECTS = tools/ceph_conf.$(OBJEXT)
ceph_conf_OBJECTS = $(am_ceph_conf_OBJECTS)
-ceph_conf_DEPENDENCIES = $(am__DEPENDENCIES_6)
+ceph_conf_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON)
am__ceph_dencoder_SOURCES_DIST = test/encoding/ceph_dencoder.cc \
rgw/rgw_dencoder.cc rgw/rgw_acl.cc rgw/rgw_common.cc \
rgw/rgw_env.cc rgw/rgw_json_enc.cc
- at WITH_RADOSGW_TRUE@am__objects_7 = \
+ at WITH_RADOSGW_TRUE@am__objects_9 = \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_dencoder.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_acl.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_common.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_env.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_json_enc.$(OBJEXT)
-am__objects_8 = $(am__objects_7)
+am__objects_10 = $(am__objects_9)
am_ceph_dencoder_OBJECTS = \
test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT) \
- $(am__objects_8)
+ $(am__objects_10)
ceph_dencoder_OBJECTS = $(am_ceph_dencoder_OBJECTS)
am__DEPENDENCIES_7 = libperfglue.la $(am__DEPENDENCIES_1)
am__DEPENDENCIES_8 = libosd.la $(LIBOSDC) $(am__DEPENDENCIES_5) \
@@ -832,23 +906,23 @@ ceph_kvstore_tool_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am_ceph_mds_OBJECTS = ceph_mds.$(OBJEXT)
ceph_mds_OBJECTS = $(am_ceph_mds_OBJECTS)
ceph_mds_DEPENDENCIES = $(am__DEPENDENCIES_9) $(LIBOSDC) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_6) $(LIBCOMMON)
am_ceph_mon_OBJECTS = ceph_mon.$(OBJEXT)
ceph_mon_OBJECTS = $(am_ceph_mon_OBJECTS)
ceph_mon_DEPENDENCIES = $(am__DEPENDENCIES_10) $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_6) $(LIBCOMMON)
am_ceph_monstore_tool_OBJECTS = tools/ceph-monstore-tool.$(OBJEXT)
ceph_monstore_tool_OBJECTS = $(am_ceph_monstore_tool_OBJECTS)
ceph_monstore_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_6) $(am__DEPENDENCIES_1)
am_ceph_osd_OBJECTS = ceph_osd.$(OBJEXT)
ceph_osd_OBJECTS = $(am_ceph_osd_OBJECTS)
ceph_osd_DEPENDENCIES = $(am__DEPENDENCIES_8) $(am__DEPENDENCIES_6) \
- $(am__DEPENDENCIES_1)
+ $(LIBCOMMON) $(am__DEPENDENCIES_1)
am_ceph_osdomap_tool_OBJECTS = tools/ceph-osdomap-tool.$(OBJEXT)
ceph_osdomap_tool_OBJECTS = $(am_ceph_osdomap_tool_OBJECTS)
ceph_osdomap_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_6) $(am__DEPENDENCIES_1)
am_ceph_syn_OBJECTS = ceph_syn.$(OBJEXT) \
client/SyntheticClient.$(OBJEXT)
ceph_syn_OBJECTS = $(am_ceph_syn_OBJECTS)
@@ -860,20 +934,31 @@ am_ceph_dupstore_OBJECTS = tools/dupstore.$(OBJEXT)
ceph_dupstore_OBJECTS = $(am_ceph_dupstore_OBJECTS)
ceph_dupstore_DEPENDENCIES = $(am__DEPENDENCIES_5) \
$(am__DEPENDENCIES_6)
+am_ceph_erasure_code_benchmark_OBJECTS = \
+ test/osd/ceph_erasure_code_benchmark.$(OBJEXT)
+ceph_erasure_code_benchmark_OBJECTS = \
+ $(am_ceph_erasure_code_benchmark_OBJECTS)
+ceph_erasure_code_benchmark_DEPENDENCIES = $(am__DEPENDENCIES_8) \
+ $(LIBCOMMON) $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) \
+ $(am__DEPENDENCIES_1)
am_ceph_filestore_dump_OBJECTS = tools/ceph-filestore-dump.$(OBJEXT)
ceph_filestore_dump_OBJECTS = $(am_ceph_filestore_dump_OBJECTS)
ceph_filestore_dump_DEPENDENCIES = $(am__DEPENDENCIES_8) \
$(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) \
- $(am__DEPENDENCIES_1)
+ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
am_ceph_filestore_tool_OBJECTS = tools/ceph-filestore-tool.$(OBJEXT)
ceph_filestore_tool_OBJECTS = $(am_ceph_filestore_tool_OBJECTS)
ceph_filestore_tool_DEPENDENCIES = $(am__DEPENDENCIES_8) \
$(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) \
$(am__DEPENDENCIES_1)
-am_ceph_kvstorebench_OBJECTS = test/kv_store_bench.$(OBJEXT) \
- key_value_store/kv_flat_btree_async.$(OBJEXT)
+am__ceph_kvstorebench_SOURCES_DIST = test/kv_store_bench.cc \
+ key_value_store/kv_flat_btree_async.cc
+ at LINUX_TRUE@am_ceph_kvstorebench_OBJECTS = \
+ at LINUX_TRUE@ test/kv_store_bench.$(OBJEXT) \
+ at LINUX_TRUE@ key_value_store/kv_flat_btree_async.$(OBJEXT)
ceph_kvstorebench_OBJECTS = $(am_ceph_kvstorebench_OBJECTS)
-ceph_kvstorebench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6)
+ at LINUX_TRUE@ceph_kvstorebench_DEPENDENCIES = $(LIBRADOS) \
+ at LINUX_TRUE@ $(am__DEPENDENCIES_6)
am_ceph_mon_store_converter_OBJECTS = \
tools/mon_store_converter.$(OBJEXT)
ceph_mon_store_converter_OBJECTS = \
@@ -906,6 +991,7 @@ ceph_rgw_jsonparser_OBJECTS = $(am_ceph_rgw_jsonparser_OBJECTS)
@WITH_RADOSGW_TRUE at am__DEPENDENCIES_11 = $(LIBRADOS) \
@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_user_client.a \
@WITH_RADOSGW_TRUE@ libcls_replica_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_lock_client.la \
@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
@@ -932,23 +1018,24 @@ am_ceph_smalliobench_OBJECTS = test/bench/small_io_bench.$(OBJEXT) \
test/bench/detailed_stat_collector.$(OBJEXT) \
test/bench/bencher.$(OBJEXT)
ceph_smalliobench_OBJECTS = $(am_ceph_smalliobench_OBJECTS)
-ceph_smalliobench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6)
+ceph_smalliobench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_6)
am_ceph_smalliobenchdumb_OBJECTS = \
test/bench/small_io_bench_dumb.$(OBJEXT) \
test/bench/dumb_backend.$(OBJEXT) \
test/bench/detailed_stat_collector.$(OBJEXT) \
test/bench/bencher.$(OBJEXT)
ceph_smalliobenchdumb_OBJECTS = $(am_ceph_smalliobenchdumb_OBJECTS)
-ceph_smalliobenchdumb_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
+ceph_smalliobenchdumb_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6)
am_ceph_smalliobenchfs_OBJECTS = \
test/bench/small_io_bench_fs.$(OBJEXT) \
test/bench/testfilestore_backend.$(OBJEXT) \
test/bench/detailed_stat_collector.$(OBJEXT) \
test/bench/bencher.$(OBJEXT)
ceph_smalliobenchfs_OBJECTS = $(am_ceph_smalliobenchfs_OBJECTS)
-ceph_smalliobenchfs_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
+ceph_smalliobenchfs_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6)
am_ceph_smalliobenchrbd_OBJECTS = \
test/bench/small_io_bench_rbd.$(OBJEXT) \
test/bench/rbd_backend.$(OBJEXT) \
@@ -956,11 +1043,14 @@ am_ceph_smalliobenchrbd_OBJECTS = \
test/bench/bencher.$(OBJEXT)
ceph_smalliobenchrbd_OBJECTS = $(am_ceph_smalliobenchrbd_OBJECTS)
ceph_smalliobenchrbd_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) \
- $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6)
am_ceph_streamtest_OBJECTS = test/streamtest.$(OBJEXT)
ceph_streamtest_OBJECTS = $(am_ceph_streamtest_OBJECTS)
ceph_streamtest_DEPENDENCIES = $(am__DEPENDENCIES_5) \
$(am__DEPENDENCIES_6)
+am_ceph_test_c_headers_OBJECTS = test/test_c_headers.$(OBJEXT)
+ceph_test_c_headers_OBJECTS = $(am_ceph_test_c_headers_OBJECTS)
+ceph_test_c_headers_DEPENDENCIES = $(LIBRADOS) $(LIBCEPHFS)
am_ceph_test_cfuse_cache_invalidate_OBJECTS = \
test/test_cfuse_cache_invalidate.$(OBJEXT)
ceph_test_cfuse_cache_invalidate_OBJECTS = \
@@ -1050,7 +1140,8 @@ ceph_test_cls_rgw_log_OBJECTS = $(am_ceph_test_cls_rgw_log_OBJECTS)
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_lock_client.la
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_lock_client.la
ceph_test_cls_rgw_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1065,7 +1156,8 @@ ceph_test_cls_rgw_meta_OBJECTS = $(am_ceph_test_cls_rgw_meta_OBJECTS)
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_lock_client.la
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_lock_client.la
ceph_test_cls_rgw_meta_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_rgw_meta_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1082,7 +1174,8 @@ ceph_test_cls_rgw_opstate_OBJECTS = \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \
@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_lock_client.la
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a \
+ at WITH_RADOSGW_TRUE@ libcls_lock_client.la
ceph_test_cls_rgw_opstate_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) \
@@ -1129,11 +1222,11 @@ ceph_test_filejournal_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_filestore_OBJECTS = \
- test/filestore/ceph_test_filestore-store_test.$(OBJEXT)
+am__ceph_test_filestore_SOURCES_DIST = test/filestore/store_test.cc
+ at LINUX_TRUE@am_ceph_test_filestore_OBJECTS = test/filestore/ceph_test_filestore-store_test.$(OBJEXT)
ceph_test_filestore_OBJECTS = $(am_ceph_test_filestore_OBJECTS)
-ceph_test_filestore_DEPENDENCIES = $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ at LINUX_TRUE@ceph_test_filestore_DEPENDENCIES = $(am__DEPENDENCIES_5) \
+ at LINUX_TRUE@ $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
ceph_test_filestore_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_filestore_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
@@ -1162,6 +1255,11 @@ ceph_test_filestore_workloadgen_OBJECTS = \
$(am_ceph_test_filestore_workloadgen_OBJECTS)
ceph_test_filestore_workloadgen_DEPENDENCIES = $(am__DEPENDENCIES_5) \
$(am__DEPENDENCIES_6)
+am_ceph_test_get_blkdev_size_OBJECTS = \
+ test/test_get_blkdev_size.$(OBJEXT)
+ceph_test_get_blkdev_size_OBJECTS = \
+ $(am_ceph_test_get_blkdev_size_OBJECTS)
+ceph_test_get_blkdev_size_DEPENDENCIES = $(LIBCOMMON)
am_ceph_test_ioctls_OBJECTS = client/test_ioctls.$(OBJEXT)
ceph_test_ioctls_OBJECTS = $(am_ceph_test_ioctls_OBJECTS)
ceph_test_ioctls_LDADD = $(LDADD)
@@ -1209,10 +1307,11 @@ ceph_test_librbd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_ceph_test_librbd_fsx_OBJECTS = \
- test/librbd/ceph_test_librbd_fsx-fsx.$(OBJEXT)
+am__ceph_test_librbd_fsx_SOURCES_DIST = test/librbd/fsx.c
+ at LINUX_TRUE@am_ceph_test_librbd_fsx_OBJECTS = \
+ at LINUX_TRUE@ test/librbd/ceph_test_librbd_fsx-fsx.$(OBJEXT)
ceph_test_librbd_fsx_OBJECTS = $(am_ceph_test_librbd_fsx_OBJECTS)
-ceph_test_librbd_fsx_DEPENDENCIES = $(LIBRBD) $(LIBRADOS)
+ at LINUX_TRUE@ceph_test_librbd_fsx_DEPENDENCIES = $(LIBRBD) $(LIBRADOS)
ceph_test_librbd_fsx_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
$(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
@@ -1262,6 +1361,16 @@ ceph_test_rados_api_aio_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_ceph_test_rados_api_c_write_operations_OBJECTS = test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT) \
+ test/librados/ceph_test_rados_api_c_write_operations-test.$(OBJEXT)
+ceph_test_rados_api_c_write_operations_OBJECTS = \
+ $(am_ceph_test_rados_api_c_write_operations_OBJECTS)
+ceph_test_rados_api_c_write_operations_DEPENDENCIES = $(LIBRADOS) \
+ $(am__DEPENDENCIES_13)
+ceph_test_rados_api_c_write_operations_LINK = $(LIBTOOL) $(AM_V_lt) \
+ --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
+ $(CXXLD) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) \
+ $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
am_ceph_test_rados_api_cls_OBJECTS = \
test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT) \
test/librados/ceph_test_rados_api_cls-test.$(OBJEXT)
@@ -1359,6 +1468,18 @@ ceph_test_rados_api_stat_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_ceph_test_rados_api_tier_OBJECTS = \
+ test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT) \
+ test/librados/ceph_test_rados_api_tier-test.$(OBJEXT) \
+ osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT)
+ceph_test_rados_api_tier_OBJECTS = \
+ $(am_ceph_test_rados_api_tier_OBJECTS)
+ceph_test_rados_api_tier_DEPENDENCIES = $(LIBRADOS) \
+ $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+ceph_test_rados_api_tier_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_ceph_test_rados_api_watch_notify_OBJECTS = test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT) \
test/librados/ceph_test_rados_api_watch_notify-test.$(OBJEXT)
ceph_test_rados_api_watch_notify_OBJECTS = \
@@ -1369,41 +1490,59 @@ ceph_test_rados_api_watch_notify_LINK = $(LIBTOOL) $(AM_V_lt) \
--tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
$(CXXLD) $(ceph_test_rados_api_watch_notify_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
-am_ceph_test_rados_delete_pools_parallel_OBJECTS = \
- test/system/rados_delete_pools_parallel.$(OBJEXT) \
- test/system/st_rados_create_pool.$(OBJEXT) \
- test/system/st_rados_delete_pool.$(OBJEXT) \
- test/system/st_rados_list_objects.$(OBJEXT)
+am__ceph_test_rados_delete_pools_parallel_SOURCES_DIST = \
+ test/system/rados_delete_pools_parallel.cc \
+ test/system/st_rados_create_pool.cc \
+ test/system/st_rados_delete_pool.cc \
+ test/system/st_rados_list_objects.cc
+ at LINUX_TRUE@am_ceph_test_rados_delete_pools_parallel_OBJECTS = \
+ at LINUX_TRUE@ test/system/rados_delete_pools_parallel.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_delete_pool.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_list_objects.$(OBJEXT)
ceph_test_rados_delete_pools_parallel_OBJECTS = \
$(am_ceph_test_rados_delete_pools_parallel_OBJECTS)
-ceph_test_rados_delete_pools_parallel_DEPENDENCIES = $(LIBRADOS) \
- libsystest.la $(am__DEPENDENCIES_1)
-am_ceph_test_rados_list_parallel_OBJECTS = \
- test/system/rados_list_parallel.$(OBJEXT) \
- test/system/st_rados_create_pool.$(OBJEXT) \
- test/system/st_rados_list_objects.$(OBJEXT)
+ at LINUX_TRUE@ceph_test_rados_delete_pools_parallel_DEPENDENCIES = \
+ at LINUX_TRUE@ $(LIBRADOS) libsystest.la $(am__DEPENDENCIES_1)
+am__ceph_test_rados_list_parallel_SOURCES_DIST = \
+ test/system/rados_list_parallel.cc \
+ test/system/st_rados_create_pool.cc \
+ test/system/st_rados_list_objects.cc
+ at LINUX_TRUE@am_ceph_test_rados_list_parallel_OBJECTS = \
+ at LINUX_TRUE@ test/system/rados_list_parallel.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_list_objects.$(OBJEXT)
ceph_test_rados_list_parallel_OBJECTS = \
$(am_ceph_test_rados_list_parallel_OBJECTS)
-ceph_test_rados_list_parallel_DEPENDENCIES = $(LIBRADOS) libsystest.la \
- $(am__DEPENDENCIES_1)
-am_ceph_test_rados_open_pools_parallel_OBJECTS = \
- test/system/rados_open_pools_parallel.$(OBJEXT) \
- test/system/st_rados_create_pool.$(OBJEXT)
+ at LINUX_TRUE@ceph_test_rados_list_parallel_DEPENDENCIES = $(LIBRADOS) \
+ at LINUX_TRUE@ libsystest.la $(am__DEPENDENCIES_1)
+am__ceph_test_rados_open_pools_parallel_SOURCES_DIST = \
+ test/system/rados_open_pools_parallel.cc \
+ test/system/st_rados_create_pool.cc
+ at LINUX_TRUE@am_ceph_test_rados_open_pools_parallel_OBJECTS = \
+ at LINUX_TRUE@ test/system/rados_open_pools_parallel.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT)
ceph_test_rados_open_pools_parallel_OBJECTS = \
$(am_ceph_test_rados_open_pools_parallel_OBJECTS)
-ceph_test_rados_open_pools_parallel_DEPENDENCIES = $(LIBRADOS) \
- libsystest.la $(am__DEPENDENCIES_1)
-am_ceph_test_rados_watch_notify_OBJECTS = \
- test/system/rados_watch_notify.$(OBJEXT) \
- test/system/st_rados_create_pool.$(OBJEXT) \
- test/system/st_rados_delete_pool.$(OBJEXT) \
- test/system/st_rados_delete_objs.$(OBJEXT) \
- test/system/st_rados_watch.$(OBJEXT) \
- test/system/st_rados_notify.$(OBJEXT)
+ at LINUX_TRUE@ceph_test_rados_open_pools_parallel_DEPENDENCIES = \
+ at LINUX_TRUE@ $(LIBRADOS) libsystest.la $(am__DEPENDENCIES_1)
+am__ceph_test_rados_watch_notify_SOURCES_DIST = \
+ test/system/rados_watch_notify.cc \
+ test/system/st_rados_create_pool.cc \
+ test/system/st_rados_delete_pool.cc \
+ test/system/st_rados_delete_objs.cc \
+ test/system/st_rados_watch.cc test/system/st_rados_notify.cc
+ at LINUX_TRUE@am_ceph_test_rados_watch_notify_OBJECTS = \
+ at LINUX_TRUE@ test/system/rados_watch_notify.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_delete_pool.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_delete_objs.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_watch.$(OBJEXT) \
+ at LINUX_TRUE@ test/system/st_rados_notify.$(OBJEXT)
ceph_test_rados_watch_notify_OBJECTS = \
$(am_ceph_test_rados_watch_notify_OBJECTS)
-ceph_test_rados_watch_notify_DEPENDENCIES = $(LIBRADOS) libsystest.la \
- $(am__DEPENDENCIES_1)
+ at LINUX_TRUE@ceph_test_rados_watch_notify_DEPENDENCIES = $(LIBRADOS) \
+ at LINUX_TRUE@ libsystest.la $(am__DEPENDENCIES_1)
am_ceph_test_rewrite_latency_OBJECTS = \
test/test_rewrite_latency.$(OBJEXT)
ceph_test_rewrite_latency_OBJECTS = \
@@ -1445,8 +1584,8 @@ ceph_test_trans_DEPENDENCIES = $(am__DEPENDENCIES_5) \
am_ceph_tpbench_OBJECTS = test/bench/tp_bench.$(OBJEXT) \
test/bench/detailed_stat_collector.$(OBJEXT)
ceph_tpbench_OBJECTS = $(am_ceph_tpbench_OBJECTS)
-ceph_tpbench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_5) \
- $(am__DEPENDENCIES_6)
+ceph_tpbench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6)
am_ceph_xattr_bench_OBJECTS = \
test/ceph_xattr_bench-xattr_bench.$(OBJEXT)
ceph_xattr_bench_OBJECTS = $(am_ceph_xattr_bench_OBJECTS)
@@ -1473,7 +1612,7 @@ librados_config_OBJECTS = $(am_librados_config_OBJECTS)
librados_config_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6)
am_monmaptool_OBJECTS = tools/monmaptool.$(OBJEXT)
monmaptool_OBJECTS = $(am_monmaptool_OBJECTS)
-monmaptool_DEPENDENCIES = $(am__DEPENDENCIES_6)
+monmaptool_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON)
am_mount_ceph_OBJECTS = mount/mount.ceph.$(OBJEXT)
mount_ceph_OBJECTS = $(am_mount_ceph_OBJECTS)
mount_ceph_DEPENDENCIES = $(LIBCOMMON)
@@ -1494,6 +1633,7 @@ am__radosgw_SOURCES_DIST = rgw/rgw_resolve.cc rgw/rgw_rest.cc \
rgw/rgw_rest_log.cc rgw/rgw_rest_opstate.cc \
rgw/rgw_rest_replica_log.cc rgw/rgw_rest_config.cc \
rgw/rgw_http_client.cc rgw/rgw_swift.cc rgw/rgw_swift_auth.cc \
+ rgw/rgw_loadgen.cc rgw/rgw_civetweb.cc civetweb/src/civetweb.c \
rgw/rgw_main.cc
@WITH_RADOSGW_TRUE at am_radosgw_OBJECTS = rgw/rgw_resolve.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/rgw_rest.$(OBJEXT) \
@@ -1511,11 +1651,14 @@ am__radosgw_SOURCES_DIST = rgw/rgw_resolve.cc rgw/rgw_rest.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_http_client.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/rgw_swift.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/rgw_swift_auth.$(OBJEXT) \
+ at WITH_RADOSGW_TRUE@ rgw/rgw_loadgen.$(OBJEXT) \
+ at WITH_RADOSGW_TRUE@ rgw/rgw_civetweb.$(OBJEXT) \
+ at WITH_RADOSGW_TRUE@ civetweb/src/radosgw-civetweb.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/rgw_main.$(OBJEXT)
radosgw_OBJECTS = $(am_radosgw_OBJECTS)
@WITH_RADOSGW_TRUE at radosgw_DEPENDENCIES = $(LIBRGW) \
@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12) \
- at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6)
+ at WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6)
am__radosgw_admin_SOURCES_DIST = rgw/rgw_admin.cc
@WITH_RADOSGW_TRUE at am_radosgw_admin_OBJECTS = rgw/rgw_admin.$(OBJEXT)
radosgw_admin_OBJECTS = $(am_radosgw_admin_OBJECTS)
@@ -1546,13 +1689,13 @@ rest_bench_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
am__test_build_libcephfs_SOURCES_DIST = test/buildtest_skeleton.cc \
osdc/Objecter.cc osdc/ObjectCacher.cc osdc/Filer.cc \
osdc/Striper.cc osdc/Journaler.cc
-am__objects_9 = osdc/test_build_libcephfs-Objecter.$(OBJEXT) \
+am__objects_11 = osdc/test_build_libcephfs-Objecter.$(OBJEXT) \
osdc/test_build_libcephfs-ObjectCacher.$(OBJEXT) \
osdc/test_build_libcephfs-Filer.$(OBJEXT) \
osdc/test_build_libcephfs-Striper.$(OBJEXT) \
osdc/test_build_libcephfs-Journaler.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_libcephfs_OBJECTS = test/test_build_libcephfs-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_9)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_11)
test_build_libcephfs_OBJECTS = $(am_test_build_libcephfs_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_libcephfs_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(LIBCEPHFS) $(am__DEPENDENCIES_1) \
@@ -1574,25 +1717,27 @@ am__test_build_libcommon_SOURCES_DIST = test/buildtest_skeleton.cc \
common/assert.cc common/run_cmd.cc common/WorkQueue.cc \
common/ConfUtils.cc common/MemoryModel.cc common/armor.c \
common/fd.cc common/xattr.c common/safe_io.c \
- common/snap_types.cc common/str_list.cc common/errno.cc \
- common/RefCountedObj.cc common/blkdev.cc common/common_init.cc \
- common/pipe.c common/ceph_argparse.cc common/ceph_context.cc \
- common/buffer.cc common/code_environment.cc common/dout.cc \
- common/signal.cc common/simple_spin.cc common/Thread.cc \
- common/Formatter.cc common/HeartbeatMap.cc common/config.cc \
- common/utf8.c common/mime.c common/strtol.cc common/page.cc \
+ common/snap_types.cc common/str_list.cc common/str_map.cc \
+ common/errno.cc common/RefCountedObj.cc common/blkdev.cc \
+ common/common_init.cc common/pipe.c common/ceph_argparse.cc \
+ common/ceph_context.cc common/buffer.cc \
+ common/code_environment.cc common/dout.cc common/signal.cc \
+ common/simple_spin.cc common/Thread.cc common/Formatter.cc \
+ common/HeartbeatMap.cc common/config.cc common/utf8.c \
+ common/mime.c common/strtol.cc common/page.cc \
common/lockdep.cc common/version.cc common/hex.cc \
common/entity_name.cc common/ceph_crypto.cc \
common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \
common/pick_address.cc common/util.cc common/TextTable.cc \
common/ceph_fs.cc common/ceph_hash.cc common/ceph_strings.cc \
common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \
- common/bloom_filter.cc common/secret.c mon/MonCap.cc \
- mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc osd/osd_types.cc \
- mds/MDSMap.cc mds/inode_backtrace.cc mds/mdstypes.cc
- at LINUX_TRUE@am__objects_10 = \
+ common/bloom_filter.cc common/linux_version.c common/secret.c \
+ mon/MonCap.cc mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc \
+ osd/osd_types.cc osd/HitSet.cc mds/MDSMap.cc \
+ mds/inode_backtrace.cc mds/mdstypes.cc
+ at LINUX_TRUE@am__objects_12 = \
@LINUX_TRUE@ common/test_build_libcommon-secret.$(OBJEXT)
-am__objects_11 = test_build_libcommon-ceph_ver.$(OBJEXT) \
+am__objects_13 = test_build_libcommon-ceph_ver.$(OBJEXT) \
common/test_build_libcommon-DecayCounter.$(OBJEXT) \
common/test_build_libcommon-LogClient.$(OBJEXT) \
common/test_build_libcommon-LogEntry.$(OBJEXT) \
@@ -1622,6 +1767,7 @@ am__objects_11 = test_build_libcommon-ceph_ver.$(OBJEXT) \
common/test_build_libcommon-safe_io.$(OBJEXT) \
common/test_build_libcommon-snap_types.$(OBJEXT) \
common/test_build_libcommon-str_list.$(OBJEXT) \
+ common/test_build_libcommon-str_map.$(OBJEXT) \
common/test_build_libcommon-errno.$(OBJEXT) \
common/test_build_libcommon-RefCountedObj.$(OBJEXT) \
common/test_build_libcommon-blkdev.$(OBJEXT) \
@@ -1660,16 +1806,18 @@ am__objects_11 = test_build_libcommon-ceph_ver.$(OBJEXT) \
common/test_build_libcommon-addr_parsing.$(OBJEXT) \
common/test_build_libcommon-hobject.$(OBJEXT) \
common/test_build_libcommon-bloom_filter.$(OBJEXT) \
- $(am__objects_10) mon/test_build_libcommon-MonCap.$(OBJEXT) \
+ common/test_build_libcommon-linux_version.$(OBJEXT) \
+ $(am__objects_12) mon/test_build_libcommon-MonCap.$(OBJEXT) \
mon/test_build_libcommon-MonClient.$(OBJEXT) \
mon/test_build_libcommon-MonMap.$(OBJEXT) \
osd/test_build_libcommon-OSDMap.$(OBJEXT) \
osd/test_build_libcommon-osd_types.$(OBJEXT) \
+ osd/test_build_libcommon-HitSet.$(OBJEXT) \
mds/test_build_libcommon-MDSMap.$(OBJEXT) \
mds/test_build_libcommon-inode_backtrace.$(OBJEXT) \
mds/test_build_libcommon-mdstypes.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_libcommon_OBJECTS = test/test_build_libcommon-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_11)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_13)
test_build_libcommon_OBJECTS = $(am_test_build_libcommon_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_libcommon_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_3) \
@@ -1683,12 +1831,12 @@ test_build_libcommon_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
am__test_build_librados_SOURCES_DIST = test/buildtest_skeleton.cc \
librados/librados.cc librados/RadosClient.cc \
librados/IoCtxImpl.cc librados/snap_set_diff.cc
-am__objects_12 = librados/test_build_librados-librados.$(OBJEXT) \
+am__objects_14 = librados/test_build_librados-librados.$(OBJEXT) \
librados/test_build_librados-RadosClient.$(OBJEXT) \
librados/test_build_librados-IoCtxImpl.$(OBJEXT) \
librados/test_build_librados-snap_set_diff.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_librados_OBJECTS = test/test_build_librados-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_12)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_14)
test_build_librados_OBJECTS = $(am_test_build_librados_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_librados_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(LIBRADOS_DEPS) $(am__DEPENDENCIES_1) \
@@ -1710,7 +1858,7 @@ am__test_build_librgw_SOURCES_DIST = test/buildtest_skeleton.cc \
rgw/rgw_multi_del.cc rgw/rgw_env.cc rgw/rgw_cors.cc \
rgw/rgw_cors_s3.cc rgw/rgw_auth_s3.cc rgw/rgw_metadata.cc \
rgw/rgw_replica_log.cc rgw/rgw_keystone.cc rgw/rgw_quota.cc
- at WITH_RADOSGW_TRUE@am__objects_13 = \
+ at WITH_RADOSGW_TRUE@am__objects_15 = \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-librgw.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl_s3.$(OBJEXT) \
@@ -1745,7 +1893,7 @@ am__test_build_librgw_SOURCES_DIST = test/buildtest_skeleton.cc \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_keystone.$(OBJEXT) \
@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_quota.$(OBJEXT)
@WITH_BUILD_TESTS_TRUE at am_test_build_librgw_OBJECTS = test/test_build_librgw-buildtest_skeleton.$(OBJEXT) \
- at WITH_BUILD_TESTS_TRUE@ $(am__objects_13)
+ at WITH_BUILD_TESTS_TRUE@ $(am__objects_15)
test_build_librgw_OBJECTS = $(am_test_build_librgw_OBJECTS)
@WITH_BUILD_TESTS_TRUE at test_build_librgw_DEPENDENCIES = \
@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_12) \
@@ -1774,10 +1922,14 @@ unittest_admin_socket_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-am_unittest_arch_OBJECTS = test/test_arch.$(OBJEXT)
+am_unittest_arch_OBJECTS = test/unittest_arch-test_arch.$(OBJEXT)
unittest_arch_OBJECTS = $(am_unittest_arch_OBJECTS)
unittest_arch_DEPENDENCIES = $(am__DEPENDENCIES_13) \
$(am__DEPENDENCIES_6)
+unittest_arch_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_arch_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \
+ -o $@
am_unittest_base64_OBJECTS = test/unittest_base64-base64.$(OBJEXT)
unittest_base64_OBJECTS = $(am_unittest_base64_OBJECTS)
unittest_base64_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13)
@@ -1840,6 +1992,15 @@ unittest_chain_xattr_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_config_OBJECTS = \
+ test/common/unittest_config-test_config.$(OBJEXT)
+unittest_config_OBJECTS = $(am_unittest_config_OBJECTS)
+unittest_config_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+ $(am__DEPENDENCIES_6)
+unittest_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_config_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_confutils_OBJECTS = \
test/unittest_confutils-confutils.$(OBJEXT)
unittest_confutils_OBJECTS = $(am_unittest_confutils_OBJECTS)
@@ -1858,6 +2019,34 @@ unittest_crc32c_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_crush_indep_OBJECTS = \
+ test/crush/unittest_crush_indep-indep.$(OBJEXT)
+unittest_crush_indep_OBJECTS = $(am_unittest_crush_indep_OBJECTS)
+unittest_crush_indep_DEPENDENCIES = $(LIBCOMMON) \
+ $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_2) \
+ $(am__DEPENDENCIES_6)
+unittest_crush_indep_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
+am_unittest_crush_wrapper_OBJECTS = \
+ test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT)
+unittest_crush_wrapper_OBJECTS = $(am_unittest_crush_wrapper_OBJECTS)
+unittest_crush_wrapper_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+ $(am__DEPENDENCIES_6) $(LIBCRUSH)
+unittest_crush_wrapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
+am_unittest_crushwrapper_OBJECTS = \
+ test/unittest_crushwrapper-test_crushwrapper.$(OBJEXT)
+unittest_crushwrapper_OBJECTS = $(am_unittest_crushwrapper_OBJECTS)
+unittest_crushwrapper_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+ $(am__DEPENDENCIES_6) $(LIBCRUSH)
+unittest_crushwrapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_crypto_OBJECTS = test/unittest_crypto-crypto.$(OBJEXT)
unittest_crypto_OBJECTS = $(am_unittest_crypto_OBJECTS)
unittest_crypto_DEPENDENCIES = $(am__DEPENDENCIES_13) \
@@ -1893,7 +2082,7 @@ unittest_erasure_code_example_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) \
$(AM_LDFLAGS) $(LDFLAGS) -o $@
-am__objects_14 = osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT) \
+am__objects_16 = osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT) \
osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT) \
osd/ErasureCodePluginJerasure/cauchy.$(OBJEXT) \
osd/ErasureCodePluginJerasure/galois.$(OBJEXT) \
@@ -1901,7 +2090,7 @@ am__objects_14 = osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-Er
osd/ErasureCodePluginJerasure/liberation.$(OBJEXT) \
osd/ErasureCodePluginJerasure/reed_sol.$(OBJEXT)
am_unittest_erasure_code_jerasure_OBJECTS = test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT) \
- $(am__objects_14)
+ $(am__objects_16)
unittest_erasure_code_jerasure_OBJECTS = \
$(am_unittest_erasure_code_jerasure_OBJECTS)
unittest_erasure_code_jerasure_DEPENDENCIES = $(am__DEPENDENCIES_8) \
@@ -1975,6 +2164,15 @@ unittest_heartbeatmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_hitset_OBJECTS = \
+ test/osd/unittest_hitset-hitset.$(OBJEXT)
+unittest_hitset_OBJECTS = $(am_unittest_hitset_OBJECTS)
+unittest_hitset_DEPENDENCIES = $(am__DEPENDENCIES_8) \
+ $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6)
+unittest_hitset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_ipaddr_OBJECTS = \
test/unittest_ipaddr-test_ipaddr.$(OBJEXT)
unittest_ipaddr_OBJECTS = $(am_unittest_ipaddr_OBJECTS)
@@ -2043,6 +2241,9 @@ unittest_mon_moncap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_on_exit_OBJECTS = test/on_exit.$(OBJEXT)
+unittest_on_exit_OBJECTS = $(am_unittest_on_exit_OBJECTS)
+unittest_on_exit_DEPENDENCIES = $(am__DEPENDENCIES_1)
am_unittest_osd_osdcap_OBJECTS = \
test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT)
unittest_osd_osdcap_OBJECTS = $(am_unittest_osd_osdcap_OBJECTS)
@@ -2061,6 +2262,15 @@ unittest_osd_types_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_osdmap_OBJECTS = \
+ test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT)
+unittest_osdmap_OBJECTS = $(am_unittest_osdmap_OBJECTS)
+unittest_osdmap_DEPENDENCIES = $(am__DEPENDENCIES_13) $(LIBCOMMON) \
+ $(am__DEPENDENCIES_6)
+unittest_osdmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_perf_counters_OBJECTS = \
test/unittest_perf_counters-perf_counters.$(OBJEXT)
unittest_perf_counters_OBJECTS = $(am_unittest_perf_counters_OBJECTS)
@@ -2140,6 +2350,15 @@ unittest_str_list_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
$(unittest_str_list_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
+am_unittest_str_map_OBJECTS = \
+ test/common/unittest_str_map-test_str_map.$(OBJEXT)
+unittest_str_map_OBJECTS = $(am_unittest_str_map_OBJECTS)
+unittest_str_map_DEPENDENCIES = $(am__DEPENDENCIES_13) \
+ $(am__DEPENDENCIES_6)
+unittest_str_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+ $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
am_unittest_striper_OBJECTS = \
test/unittest_striper-test_striper.$(OBJEXT)
unittest_striper_OBJECTS = $(am_unittest_striper_OBJECTS)
@@ -2253,6 +2472,7 @@ am__v_GEN_0 = @echo " GEN " $@;
SOURCES = $(libcls_log_client_a_SOURCES) \
$(libcls_replica_log_client_a_SOURCES) \
$(libcls_statelog_client_a_SOURCES) \
+ $(libcls_user_client_a_SOURCES) \
$(libcls_version_client_a_SOURCES) $(libos_zfs_a_SOURCES) \
$(libarch_la_SOURCES) $(libauth_la_SOURCES) \
$(libcephfs_la_SOURCES) $(libcephfs_jni_la_SOURCES) \
@@ -2264,9 +2484,9 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(libcls_refcount_client_la_SOURCES) \
$(libcls_replica_log_la_SOURCES) $(libcls_rgw_la_SOURCES) \
$(libcls_rgw_client_la_SOURCES) $(libcls_statelog_la_SOURCES) \
- $(libcls_version_la_SOURCES) $(libcommon_la_SOURCES) \
- $(libcommon_crc_la_SOURCES) $(libcrush_la_SOURCES) \
- $(libec_example_la_SOURCES) \
+ $(libcls_user_la_SOURCES) $(libcls_version_la_SOURCES) \
+ $(libcommon_la_SOURCES) $(libcommon_crc_la_SOURCES) \
+ $(libcrush_la_SOURCES) $(libec_example_la_SOURCES) \
$(libec_fail_to_initialize_la_SOURCES) \
$(libec_fail_to_register_la_SOURCES) $(libec_hangs_la_SOURCES) \
$(libec_jerasure_la_SOURCES) \
@@ -2283,8 +2503,10 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_mon_SOURCES) $(ceph_monstore_tool_SOURCES) \
$(ceph_osd_SOURCES) $(ceph_osdomap_tool_SOURCES) \
$(ceph_syn_SOURCES) $(ceph_bench_log_SOURCES) \
- $(ceph_dupstore_SOURCES) $(ceph_filestore_dump_SOURCES) \
- $(ceph_filestore_tool_SOURCES) $(ceph_kvstorebench_SOURCES) \
+ $(ceph_dupstore_SOURCES) \
+ $(ceph_erasure_code_benchmark_SOURCES) \
+ $(ceph_filestore_dump_SOURCES) $(ceph_filestore_tool_SOURCES) \
+ $(ceph_kvstorebench_SOURCES) \
$(ceph_mon_store_converter_SOURCES) \
$(ceph_multi_stress_watch_SOURCES) $(ceph_omapbench_SOURCES) \
$(ceph_psim_SOURCES) $(ceph_radosacl_SOURCES) \
@@ -2292,7 +2514,7 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_scratchtool_SOURCES) $(ceph_scratchtoolpp_SOURCES) \
$(ceph_smalliobench_SOURCES) $(ceph_smalliobenchdumb_SOURCES) \
$(ceph_smalliobenchfs_SOURCES) $(ceph_smalliobenchrbd_SOURCES) \
- $(ceph_streamtest_SOURCES) \
+ $(ceph_streamtest_SOURCES) $(ceph_test_c_headers_SOURCES) \
$(ceph_test_cfuse_cache_invalidate_SOURCES) \
$(ceph_test_cls_hello_SOURCES) $(ceph_test_cls_lock_SOURCES) \
$(ceph_test_cls_log_SOURCES) $(ceph_test_cls_rbd_SOURCES) \
@@ -2308,6 +2530,7 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_filestore_idempotent_SOURCES) \
$(ceph_test_filestore_idempotent_sequence_SOURCES) \
$(ceph_test_filestore_workloadgen_SOURCES) \
+ $(ceph_test_get_blkdev_size_SOURCES) \
$(ceph_test_ioctls_SOURCES) $(ceph_test_keys_SOURCES) \
$(ceph_test_keyvaluedb_atomicity_SOURCES) \
$(ceph_test_keyvaluedb_iterators_SOURCES) \
@@ -2317,6 +2540,7 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_mutate_SOURCES) $(ceph_test_object_map_SOURCES) \
$(ceph_test_objectcacher_stress_SOURCES) \
$(ceph_test_rados_SOURCES) $(ceph_test_rados_api_aio_SOURCES) \
+ $(ceph_test_rados_api_c_write_operations_SOURCES) \
$(ceph_test_rados_api_cls_SOURCES) \
$(ceph_test_rados_api_cmd_SOURCES) \
$(ceph_test_rados_api_io_SOURCES) \
@@ -2326,6 +2550,7 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_rados_api_pool_SOURCES) \
$(ceph_test_rados_api_snapshots_SOURCES) \
$(ceph_test_rados_api_stat_SOURCES) \
+ $(ceph_test_rados_api_tier_SOURCES) \
$(ceph_test_rados_api_watch_notify_SOURCES) \
$(ceph_test_rados_delete_pools_parallel_SOURCES) \
$(ceph_test_rados_list_parallel_SOURCES) \
@@ -2351,8 +2576,11 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(unittest_ceph_argparse_SOURCES) \
$(unittest_ceph_compatset_SOURCES) \
$(unittest_ceph_crypto_SOURCES) \
- $(unittest_chain_xattr_SOURCES) $(unittest_confutils_SOURCES) \
- $(unittest_crc32c_SOURCES) $(unittest_crypto_SOURCES) \
+ $(unittest_chain_xattr_SOURCES) $(unittest_config_SOURCES) \
+ $(unittest_confutils_SOURCES) $(unittest_crc32c_SOURCES) \
+ $(unittest_crush_indep_SOURCES) \
+ $(unittest_crush_wrapper_SOURCES) \
+ $(unittest_crushwrapper_SOURCES) $(unittest_crypto_SOURCES) \
$(unittest_daemon_config_SOURCES) $(unittest_encoding_SOURCES) \
$(unittest_erasure_code_example_SOURCES) \
$(unittest_erasure_code_jerasure_SOURCES) \
@@ -2360,39 +2588,43 @@ SOURCES = $(libcls_log_client_a_SOURCES) \
$(unittest_erasure_code_plugin_jerasure_SOURCES) \
$(unittest_escape_SOURCES) $(unittest_flatindex_SOURCES) \
$(unittest_formatter_SOURCES) $(unittest_gather_SOURCES) \
- $(unittest_heartbeatmap_SOURCES) $(unittest_ipaddr_SOURCES) \
- $(unittest_lfnindex_SOURCES) \
+ $(unittest_heartbeatmap_SOURCES) $(unittest_hitset_SOURCES) \
+ $(unittest_ipaddr_SOURCES) $(unittest_lfnindex_SOURCES) \
$(unittest_libcephfs_config_SOURCES) \
$(unittest_librados_SOURCES) \
$(unittest_librados_config_SOURCES) $(unittest_log_SOURCES) \
$(unittest_mime_SOURCES) $(unittest_mon_moncap_SOURCES) \
- $(unittest_osd_osdcap_SOURCES) $(unittest_osd_types_SOURCES) \
+ $(unittest_on_exit_SOURCES) $(unittest_osd_osdcap_SOURCES) \
+ $(unittest_osd_types_SOURCES) $(unittest_osdmap_SOURCES) \
$(unittest_perf_counters_SOURCES) $(unittest_pglog_SOURCES) \
$(unittest_prebufferedstreambuf_SOURCES) \
$(unittest_run_cmd_SOURCES) \
$(unittest_sharedptr_registry_SOURCES) \
$(unittest_signals_SOURCES) $(unittest_simple_spin_SOURCES) \
$(unittest_sloppy_crc_map_SOURCES) \
- $(unittest_str_list_SOURCES) $(unittest_striper_SOURCES) \
- $(unittest_strtol_SOURCES) $(unittest_texttable_SOURCES) \
- $(unittest_throttle_SOURCES) $(unittest_utf8_SOURCES) \
- $(unittest_util_SOURCES) $(unittest_workqueue_SOURCES)
+ $(unittest_str_list_SOURCES) $(unittest_str_map_SOURCES) \
+ $(unittest_striper_SOURCES) $(unittest_strtol_SOURCES) \
+ $(unittest_texttable_SOURCES) $(unittest_throttle_SOURCES) \
+ $(unittest_utf8_SOURCES) $(unittest_util_SOURCES) \
+ $(unittest_workqueue_SOURCES)
DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(libcls_replica_log_client_a_SOURCES) \
$(libcls_statelog_client_a_SOURCES) \
+ $(libcls_user_client_a_SOURCES) \
$(libcls_version_client_a_SOURCES) \
$(am__libos_zfs_a_SOURCES_DIST) $(libarch_la_SOURCES) \
$(libauth_la_SOURCES) $(libcephfs_la_SOURCES) \
$(am__libcephfs_jni_la_SOURCES_DIST) $(libclient_la_SOURCES) \
$(am__libclient_fuse_la_SOURCES_DIST) \
- $(libcls_hello_la_SOURCES) $(libcls_kvs_la_SOURCES) \
+ $(libcls_hello_la_SOURCES) $(am__libcls_kvs_la_SOURCES_DIST) \
$(libcls_lock_la_SOURCES) $(libcls_lock_client_la_SOURCES) \
$(libcls_log_la_SOURCES) $(libcls_rbd_la_SOURCES) \
$(libcls_rbd_client_la_SOURCES) $(libcls_refcount_la_SOURCES) \
$(libcls_refcount_client_la_SOURCES) \
$(libcls_replica_log_la_SOURCES) $(libcls_rgw_la_SOURCES) \
$(libcls_rgw_client_la_SOURCES) $(libcls_statelog_la_SOURCES) \
- $(libcls_version_la_SOURCES) $(am__libcommon_la_SOURCES_DIST) \
+ $(libcls_user_la_SOURCES) $(libcls_version_la_SOURCES) \
+ $(am__libcommon_la_SOURCES_DIST) \
$(am__libcommon_crc_la_SOURCES_DIST) $(libcrush_la_SOURCES) \
$(libec_example_la_SOURCES) \
$(libec_fail_to_initialize_la_SOURCES) \
@@ -2401,19 +2633,20 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(libec_missing_entry_point_la_SOURCES) \
$(libglobal_la_SOURCES) $(libjson_spirit_la_SOURCES) \
$(liblog_la_SOURCES) $(libmds_la_SOURCES) $(libmon_la_SOURCES) \
- $(libmsg_la_SOURCES) $(libos_la_SOURCES) $(libosd_la_SOURCES) \
- $(libosdc_la_SOURCES) $(am__libperfglue_la_SOURCES_DIST) \
- $(librados_la_SOURCES) $(librbd_la_SOURCES) \
- $(am__librgw_la_SOURCES_DIST) $(libsystest_la_SOURCES) \
- $(ceph_authtool_SOURCES) $(ceph_conf_SOURCES) \
- $(am__ceph_dencoder_SOURCES_DIST) \
+ $(libmsg_la_SOURCES) $(am__libos_la_SOURCES_DIST) \
+ $(libosd_la_SOURCES) $(libosdc_la_SOURCES) \
+ $(am__libperfglue_la_SOURCES_DIST) $(librados_la_SOURCES) \
+ $(librbd_la_SOURCES) $(am__librgw_la_SOURCES_DIST) \
+ $(am__libsystest_la_SOURCES_DIST) $(ceph_authtool_SOURCES) \
+ $(ceph_conf_SOURCES) $(am__ceph_dencoder_SOURCES_DIST) \
$(am__ceph_fuse_SOURCES_DIST) $(ceph_kvstore_tool_SOURCES) \
$(ceph_mds_SOURCES) $(ceph_mon_SOURCES) \
$(ceph_monstore_tool_SOURCES) $(ceph_osd_SOURCES) \
$(ceph_osdomap_tool_SOURCES) $(ceph_syn_SOURCES) \
$(ceph_bench_log_SOURCES) $(ceph_dupstore_SOURCES) \
+ $(ceph_erasure_code_benchmark_SOURCES) \
$(ceph_filestore_dump_SOURCES) $(ceph_filestore_tool_SOURCES) \
- $(ceph_kvstorebench_SOURCES) \
+ $(am__ceph_kvstorebench_SOURCES_DIST) \
$(ceph_mon_store_converter_SOURCES) \
$(ceph_multi_stress_watch_SOURCES) $(ceph_omapbench_SOURCES) \
$(ceph_psim_SOURCES) $(ceph_radosacl_SOURCES) \
@@ -2422,7 +2655,7 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_scratchtool_SOURCES) $(ceph_scratchtoolpp_SOURCES) \
$(ceph_smalliobench_SOURCES) $(ceph_smalliobenchdumb_SOURCES) \
$(ceph_smalliobenchfs_SOURCES) $(ceph_smalliobenchrbd_SOURCES) \
- $(ceph_streamtest_SOURCES) \
+ $(ceph_streamtest_SOURCES) $(ceph_test_c_headers_SOURCES) \
$(ceph_test_cfuse_cache_invalidate_SOURCES) \
$(ceph_test_cls_hello_SOURCES) $(ceph_test_cls_lock_SOURCES) \
$(ceph_test_cls_log_SOURCES) $(ceph_test_cls_rbd_SOURCES) \
@@ -2436,19 +2669,21 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_cls_version_SOURCES) \
$(am__ceph_test_cors_SOURCES_DIST) $(ceph_test_crypto_SOURCES) \
$(ceph_test_filejournal_SOURCES) \
- $(ceph_test_filestore_SOURCES) \
+ $(am__ceph_test_filestore_SOURCES_DIST) \
$(ceph_test_filestore_idempotent_SOURCES) \
$(ceph_test_filestore_idempotent_sequence_SOURCES) \
$(ceph_test_filestore_workloadgen_SOURCES) \
+ $(ceph_test_get_blkdev_size_SOURCES) \
$(ceph_test_ioctls_SOURCES) $(ceph_test_keys_SOURCES) \
$(ceph_test_keyvaluedb_atomicity_SOURCES) \
$(ceph_test_keyvaluedb_iterators_SOURCES) \
$(ceph_test_libcephfs_SOURCES) $(ceph_test_librbd_SOURCES) \
- $(ceph_test_librbd_fsx_SOURCES) \
+ $(am__ceph_test_librbd_fsx_SOURCES_DIST) \
$(ceph_test_mon_workloadgen_SOURCES) $(ceph_test_msgr_SOURCES) \
$(ceph_test_mutate_SOURCES) $(ceph_test_object_map_SOURCES) \
$(ceph_test_objectcacher_stress_SOURCES) \
$(ceph_test_rados_SOURCES) $(ceph_test_rados_api_aio_SOURCES) \
+ $(ceph_test_rados_api_c_write_operations_SOURCES) \
$(ceph_test_rados_api_cls_SOURCES) \
$(ceph_test_rados_api_cmd_SOURCES) \
$(ceph_test_rados_api_io_SOURCES) \
@@ -2458,11 +2693,12 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(ceph_test_rados_api_pool_SOURCES) \
$(ceph_test_rados_api_snapshots_SOURCES) \
$(ceph_test_rados_api_stat_SOURCES) \
+ $(ceph_test_rados_api_tier_SOURCES) \
$(ceph_test_rados_api_watch_notify_SOURCES) \
- $(ceph_test_rados_delete_pools_parallel_SOURCES) \
- $(ceph_test_rados_list_parallel_SOURCES) \
- $(ceph_test_rados_open_pools_parallel_SOURCES) \
- $(ceph_test_rados_watch_notify_SOURCES) \
+ $(am__ceph_test_rados_delete_pools_parallel_SOURCES_DIST) \
+ $(am__ceph_test_rados_list_parallel_SOURCES_DIST) \
+ $(am__ceph_test_rados_open_pools_parallel_SOURCES_DIST) \
+ $(am__ceph_test_rados_watch_notify_SOURCES_DIST) \
$(ceph_test_rewrite_latency_SOURCES) \
$(ceph_test_signal_handlers_SOURCES) \
$(ceph_test_snap_mapper_SOURCES) \
@@ -2486,8 +2722,11 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(unittest_ceph_argparse_SOURCES) \
$(unittest_ceph_compatset_SOURCES) \
$(unittest_ceph_crypto_SOURCES) \
- $(unittest_chain_xattr_SOURCES) $(unittest_confutils_SOURCES) \
- $(unittest_crc32c_SOURCES) $(unittest_crypto_SOURCES) \
+ $(unittest_chain_xattr_SOURCES) $(unittest_config_SOURCES) \
+ $(unittest_confutils_SOURCES) $(unittest_crc32c_SOURCES) \
+ $(unittest_crush_indep_SOURCES) \
+ $(unittest_crush_wrapper_SOURCES) \
+ $(unittest_crushwrapper_SOURCES) $(unittest_crypto_SOURCES) \
$(unittest_daemon_config_SOURCES) $(unittest_encoding_SOURCES) \
$(unittest_erasure_code_example_SOURCES) \
$(unittest_erasure_code_jerasure_SOURCES) \
@@ -2495,23 +2734,25 @@ DIST_SOURCES = $(libcls_log_client_a_SOURCES) \
$(unittest_erasure_code_plugin_jerasure_SOURCES) \
$(unittest_escape_SOURCES) $(unittest_flatindex_SOURCES) \
$(unittest_formatter_SOURCES) $(unittest_gather_SOURCES) \
- $(unittest_heartbeatmap_SOURCES) $(unittest_ipaddr_SOURCES) \
- $(unittest_lfnindex_SOURCES) \
+ $(unittest_heartbeatmap_SOURCES) $(unittest_hitset_SOURCES) \
+ $(unittest_ipaddr_SOURCES) $(unittest_lfnindex_SOURCES) \
$(unittest_libcephfs_config_SOURCES) \
$(unittest_librados_SOURCES) \
$(unittest_librados_config_SOURCES) $(unittest_log_SOURCES) \
$(unittest_mime_SOURCES) $(unittest_mon_moncap_SOURCES) \
- $(unittest_osd_osdcap_SOURCES) $(unittest_osd_types_SOURCES) \
+ $(unittest_on_exit_SOURCES) $(unittest_osd_osdcap_SOURCES) \
+ $(unittest_osd_types_SOURCES) $(unittest_osdmap_SOURCES) \
$(unittest_perf_counters_SOURCES) $(unittest_pglog_SOURCES) \
$(unittest_prebufferedstreambuf_SOURCES) \
$(unittest_run_cmd_SOURCES) \
$(unittest_sharedptr_registry_SOURCES) \
$(unittest_signals_SOURCES) $(unittest_simple_spin_SOURCES) \
$(unittest_sloppy_crc_map_SOURCES) \
- $(unittest_str_list_SOURCES) $(unittest_striper_SOURCES) \
- $(unittest_strtol_SOURCES) $(unittest_texttable_SOURCES) \
- $(unittest_throttle_SOURCES) $(unittest_utf8_SOURCES) \
- $(unittest_util_SOURCES) $(unittest_workqueue_SOURCES)
+ $(unittest_str_list_SOURCES) $(unittest_str_map_SOURCES) \
+ $(unittest_striper_SOURCES) $(unittest_strtol_SOURCES) \
+ $(unittest_texttable_SOURCES) $(unittest_throttle_SOURCES) \
+ $(unittest_utf8_SOURCES) $(unittest_util_SOURCES) \
+ $(unittest_workqueue_SOURCES)
RECURSIVE_TARGETS = all-recursive check-recursive dvi-recursive \
html-recursive info-recursive install-data-recursive \
install-dvi-recursive install-exec-recursive \
@@ -2548,12 +2789,13 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
mon/AuthMonitor.h mon/DataHealthService.h mon/Elector.h \
mon/LogMonitor.h mon/ConfigKeyService.h mon/HealthMonitor.h \
mon/HealthService.h mon/MDSMonitor.h mon/MonmapMonitor.h \
- mon/MonCap.h mon/MonClient.h mon/MonCommands.h mon/MonMap.h \
- mon/Monitor.h mon/MonitorStore.h mon/MonitorDBStore.h \
- mon/OSDMonitor.h mon/PGMap.h mon/PGMonitor.h mon/Paxos.h \
- mon/PaxosService.h mon/QuorumService.h mon/Session.h \
- mon/mon_types.h mds/inode_backtrace.h mds/flock.h mds/locks.c \
- mds/locks.h mds/Anchor.h mds/AnchorClient.h mds/AnchorServer.h \
+ mon/MonCap.h mon/MonClient.h mon/MonCommands.h \
+ mon/DumplingMonCommands.h mon/MonMap.h mon/Monitor.h \
+ mon/MonitorStore.h mon/MonitorDBStore.h mon/OSDMonitor.h \
+ mon/PGMap.h mon/PGMonitor.h mon/Paxos.h mon/PaxosService.h \
+ mon/QuorumService.h mon/Session.h mon/mon_types.h \
+ mds/inode_backtrace.h mds/flock.h mds/locks.c mds/locks.h \
+ mds/Anchor.h mds/AnchorClient.h mds/AnchorServer.h \
mds/CDentry.h mds/CDir.h mds/CInode.h mds/Capability.h \
mds/Dumper.h mds/InoTable.h mds/LocalLock.h mds/Locker.h \
mds/LogEvent.h mds/LogSegment.h mds/MDBalancer.h mds/MDCache.h \
@@ -2570,26 +2812,28 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
mds/events/ESessions.h mds/events/ESlaveUpdate.h \
mds/events/ESubtreeMap.h mds/events/ETableClient.h \
mds/events/ETableServer.h mds/events/EUpdate.h \
- os/btrfs_ioctl.h os/chain_xattr.h os/CollectionIndex.h \
- os/FileJournal.h os/FileStore.h os/BtrfsFileStoreBackend.h \
- os/GenericFileStoreBackend.h os/ZFSFileStoreBackend.h \
- os/FlatIndex.h os/HashIndex.h os/FDCache.h os/WBThrottle.h \
- os/IndexManager.h os/Journal.h os/JournalingObjectStore.h \
- os/LFNIndex.h os/ObjectStore.h os/SequencerPosition.h \
- os/ObjectMap.h os/DBObjectMap.h os/KeyValueDB.h \
- os/LevelDBStore.h os/ZFS.h \
+ os/btrfs_ioctl.h os/chain_xattr.h os/BtrfsFileStoreBackend.h \
+ os/CollectionIndex.h os/DBObjectMap.h os/GenericObjectMap.h \
+ os/FileJournal.h os/FileStore.h os/FlatIndex.h os/FDCache.h \
+ os/GenericFileStoreBackend.h os/HashIndex.h os/IndexManager.h \
+ os/Journal.h os/JournalingObjectStore.h os/KeyValueDB.h \
+ os/LevelDBStore.h os/LFNIndex.h os/MemStore.h \
+ os/KeyValueStore.h os/ObjectMap.h os/ObjectStore.h \
+ os/SequencerPosition.h os/WBThrottle.h \
+ os/ZFSFileStoreBackend.h os/ZFS.h \
osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \
osd/ErasureCodePluginJerasure/cauchy.h \
osd/ErasureCodePluginJerasure/galois.h \
osd/ErasureCodePluginJerasure/jerasure.h \
osd/ErasureCodePluginJerasure/liberation.h \
- osd/ErasureCodePluginJerasure/reed_sol.h osd/Ager.h \
+ osd/ErasureCodePluginJerasure/reed_sol.h \
+ osd/ErasureCodePluginJerasure/vectorop.h osd/Ager.h \
osd/ClassHandler.h osd/ErasureCodeInterface.h \
- osd/ErasureCodePlugin.h osd/OSD.h osd/OSDCap.h osd/OSDMap.h \
- osd/ObjectVersioner.h osd/OpRequest.h osd/SnapMapper.h \
- osd/PG.h osd/PGLog.h osd/ReplicatedPG.h osd/PGBackend.h \
- osd/ReplicatedBackend.h osd/Watch.h osd/osd_types.h \
- osdc/Blinker.h osdc/Filer.h osdc/Journaler.h \
+ osd/ErasureCodePlugin.h osd/HitSet.h osd/OSD.h osd/OSDCap.h \
+ osd/OSDMap.h osd/ObjectVersioner.h osd/OpRequest.h \
+ osd/SnapMapper.h osd/PG.h osd/PGLog.h osd/ReplicatedPG.h \
+ osd/PGBackend.h osd/ReplicatedBackend.h osd/Watch.h \
+ osd/osd_types.h osdc/Blinker.h osdc/Filer.h osdc/Journaler.h \
osdc/ObjectCacher.h osdc/Objecter.h osdc/Striper.h \
osdc/WritebackHandler.h client/Client.h client/Dentry.h \
client/Dir.h client/Fh.h client/Inode.h client/MetaRequest.h \
@@ -2640,9 +2884,9 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
common/utf8.h common/mime.h common/pick_address.h \
common/secret.h common/strtol.h common/static_assert.h \
common/AsyncReserver.h common/sync_filesystem.h \
- common/cmdparse.h common/hobject.h msg/Accepter.h \
- msg/DispatchQueue.h msg/Dispatcher.h msg/Message.h \
- msg/Messenger.h msg/Pipe.h msg/SimpleMessenger.h \
+ common/cmdparse.h common/hobject.h common/linux_version.h \
+ msg/Accepter.h msg/DispatchQueue.h msg/Dispatcher.h \
+ msg/Message.h msg/Messenger.h msg/Pipe.h msg/SimpleMessenger.h \
msg/msg_types.h messages/MAuth.h messages/MAuthReply.h \
messages/MCacheExpire.h messages/MClientCaps.h \
messages/MClientCapRelease.h messages/MClientLease.h \
@@ -2708,40 +2952,46 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
include/ipaddr.h include/linux_fiemap.h include/lru.h \
include/msgr.h include/object.h include/page.h \
include/rangeset.h include/rados.h include/rbd_types.h \
- include/statlite.h include/str_list.h include/stringify.h \
- include/triple.h include/types.h include/utime.h \
- include/dlist.h include/elist.h include/uuid.h include/xlist.h \
- include/rados/librados.h include/rados/rados_types.h \
- include/rados/rados_types.hpp include/rados/librados.hpp \
- include/rados/librgw.h include/rados/page.h \
- include/rados/crc32c.h include/rados/buffer.h \
- include/rbd/features.h include/rbd/librbd.h \
- include/rbd/librbd.hpp include/util.h librados/snap_set_diff.h \
- librados/AioCompletionImpl.h librados/IoCtxImpl.h \
- librados/PoolAsyncCompletionImpl.h librados/RadosClient.h \
- librbd/AioCompletion.h librbd/AioRequest.h librbd/ImageCtx.h \
- librbd/internal.h librbd/LibrbdWriteback.h \
- librbd/parent_types.h librbd/SnapInfo.h librbd/WatchCtx.h \
- rgw/logrotate.conf rgw/rgw_acl.h rgw/rgw_acl_s3.h \
- rgw/rgw_acl_swift.h rgw/rgw_client_io.h rgw/rgw_fcgi.h \
- rgw/rgw_xml.h rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
+ include/statlite.h include/str_list.h include/str_map.h \
+ include/stringify.h include/triple.h include/types.h \
+ include/utime.h include/dlist.h include/elist.h include/uuid.h \
+ include/xlist.h include/rados/librados.h \
+ include/rados/rados_types.h include/rados/rados_types.hpp \
+ include/rados/librados.hpp include/rados/librgw.h \
+ include/rados/page.h include/rados/crc32c.h \
+ include/rados/buffer.h include/rbd/features.h \
+ include/rbd/librbd.h include/rbd/librbd.hpp include/util.h \
+ include/stat.h include/on_exit.h include/memory.h \
+ include/rados/memory.h include/hash_namespace.h \
+ include/unordered_set.h include/unordered_map.h \
+ librados/snap_set_diff.h librados/AioCompletionImpl.h \
+ librados/IoCtxImpl.h librados/PoolAsyncCompletionImpl.h \
+ librados/RadosClient.h librbd/AioCompletion.h \
+ librbd/AioRequest.h librbd/ImageCtx.h librbd/internal.h \
+ librbd/LibrbdWriteback.h librbd/parent_types.h \
+ librbd/SnapInfo.h librbd/WatchCtx.h rgw/logrotate.conf \
+ rgw/rgw_acl.h rgw/rgw_acl_s3.h rgw/rgw_acl_swift.h \
+ rgw/rgw_client_io.h rgw/rgw_fcgi.h rgw/rgw_xml.h \
+ rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
rgw/rgw_cors_s3.h rgw/rgw_cors_swift.h rgw/rgw_string.h \
rgw/rgw_formats.h rgw/rgw_http_errors.h rgw/rgw_log.h \
- rgw/rgw_multi.h rgw/rgw_policy_s3.h rgw/rgw_gc.h \
- rgw/rgw_metadata.h rgw/rgw_multi_del.h rgw/rgw_op.h \
- rgw/rgw_http_client.h rgw/rgw_swift.h rgw/rgw_swift_auth.h \
- rgw/rgw_quota.h rgw/rgw_rados.h rgw/rgw_replica_log.h \
- rgw/rgw_resolve.h rgw/rgw_rest.h rgw/rgw_rest_swift.h \
- rgw/rgw_rest_s3.h rgw/rgw_auth_s3.h rgw/rgw_rest_admin.h \
- rgw/rgw_rest_usage.h rgw/rgw_rest_user.h rgw/rgw_rest_bucket.h \
- rgw/rgw_rest_client.h rgw/rgw_rest_conn.h rgw/rgw_tools.h \
- rgw/rgw_rest_metadata.h rgw/rgw_rest_log.h \
- rgw/rgw_rest_opstate.h rgw/rgw_rest_replica_log.h \
- rgw/rgw_rest_config.h rgw/rgw_usage.h rgw/rgw_user.h \
- rgw/rgw_bucket.h rgw/rgw_keystone.h cls/lock/cls_lock_types.h \
- cls/lock/cls_lock_ops.h cls/lock/cls_lock_client.h \
- cls/rbd/cls_rbd.h cls/rbd/cls_rbd_client.h \
- cls/refcount/cls_refcount_ops.h \
+ rgw/rgw_loadgen.h rgw/rgw_multi.h rgw/rgw_policy_s3.h \
+ rgw/rgw_gc.h rgw/rgw_metadata.h rgw/rgw_multi_del.h \
+ rgw/rgw_op.h rgw/rgw_http_client.h rgw/rgw_swift.h \
+ rgw/rgw_swift_auth.h rgw/rgw_quota.h rgw/rgw_rados.h \
+ rgw/rgw_replica_log.h rgw/rgw_resolve.h rgw/rgw_rest.h \
+ rgw/rgw_rest_swift.h rgw/rgw_rest_s3.h rgw/rgw_auth_s3.h \
+ rgw/rgw_rest_admin.h rgw/rgw_rest_usage.h rgw/rgw_rest_user.h \
+ rgw/rgw_rest_bucket.h rgw/rgw_rest_client.h \
+ rgw/rgw_rest_conn.h rgw/rgw_tools.h rgw/rgw_rest_metadata.h \
+ rgw/rgw_rest_log.h rgw/rgw_rest_opstate.h \
+ rgw/rgw_rest_replica_log.h rgw/rgw_rest_config.h \
+ rgw/rgw_usage.h rgw/rgw_user.h rgw/rgw_bucket.h \
+ rgw/rgw_keystone.h rgw/rgw_civetweb.h civetweb/civetweb.h \
+ civetweb/include/civetweb.h civetweb/src/md5.h \
+ cls/lock/cls_lock_types.h cls/lock/cls_lock_ops.h \
+ cls/lock/cls_lock_client.h cls/rbd/cls_rbd.h \
+ cls/rbd/cls_rbd_client.h cls/refcount/cls_refcount_ops.h \
cls/refcount/cls_refcount_client.h \
cls/version/cls_version_types.h cls/version/cls_version_ops.h \
cls/version/cls_version_client.h cls/log/cls_log_types.h \
@@ -2753,29 +3003,31 @@ am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \
cls/replica_log/cls_replica_log_ops.h \
cls/replica_log/cls_replica_log_client.h \
cls/rgw/cls_rgw_client.h cls/rgw/cls_rgw_ops.h \
- cls/rgw/cls_rgw_types.h key_value_store/key_value_structure.h \
+ cls/rgw/cls_rgw_types.h cls/user/cls_user_client.h \
+ cls/user/cls_user_ops.h cls/user/cls_user_types.h \
+ key_value_store/key_value_structure.h \
key_value_store/kv_flat_btree_async.h \
key_value_store/kvs_arg_types.h test/osd/ErasureCodeExample.h \
- test/osd/RadosModel.h test/osd/Object.h test/osd/TestOpStat.h \
- test/bench/distribution.h test/bench/rados_backend.h \
- test/bench/rbd_backend.h test/bench/bencher.h \
- test/bench/backend.h test/bench/dumb_backend.h \
- test/bench/stat_collector.h \
- test/bench/detailed_stat_collector.h \
+ test/bench/backend.h test/bench/bencher.h \
+ test/bench/detailed_stat_collector.h test/bench/distribution.h \
+ test/bench/dumb_backend.h test/bench/rados_backend.h \
+ test/bench/rbd_backend.h test/bench/stat_collector.h \
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h test/encoding/types.h \
test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreTracker.h \
test/filestore/FileStoreDiff.h \
+ test/filestore/FileStoreTracker.h \
test/filestore/TestFileStoreState.h \
test/filestore/workload_generator.h test/kv_store_bench.h \
test/librados/test.h test/ObjectMap/KeyValueDBMemory.h \
- test/omap_bench.h test/osdc/FakeWriteback.h \
+ test/omap_bench.h test/osd/ceph_erasure_code_benchmark.h \
+ test/osdc/FakeWriteback.h test/osd/Object.h \
+ test/osd/RadosModel.h test/osd/TestOpStat.h \
test/system/cross_process_sem.h \
test/system/st_rados_create_pool.h \
- test/system/st_rados_list_objects.h \
test/system/st_rados_delete_objs.h \
test/system/st_rados_delete_pool.h \
+ test/system/st_rados_list_objects.h \
test/system/st_rados_notify.h test/system/st_rados_watch.h \
test/system/systest_runnable.h test/system/systest_settings.h \
test/unit.h tools/rados/rados_sync.h tools/common.h cls_acl.cc \
@@ -2821,15 +3073,16 @@ am__relativize = \
reldir="$$dir2"
ACLOCAL = @ACLOCAL@
AMTAR = @AMTAR@
-AM_CXXFLAGS = @AM_CXXFLAGS@ $(AM_COMMON_CFLAGS) -Wnon-virtual-dtor \
- -Wno-invalid-offsetof -Wstrict-null-sentinel $(am__append_2) \
- $(am__append_5) $(am__append_19)
+AM_CXXFLAGS = @AM_CXXFLAGS@ $(AM_COMMON_CFLAGS) -ftemplate-depth-1024 \
+ -Wnon-virtual-dtor -Wno-invalid-offsetof $(am__append_3) \
+ $(am__append_6) $(am__append_23)
AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
AR = @AR@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
+BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@
CC = @CC@
CCAS = ${srcdir}/yasm-wrapper
CCASDEPMODE = @CCASDEPMODE@
@@ -2871,6 +3124,7 @@ JAR = @JAR@
JAVAC = @JAVAC@
JAVAH = @JAVAH@
JDK_CPPFLAGS = @JDK_CPPFLAGS@
+KEYUTILS_LIB = @KEYUTILS_LIB@
LD = @LD@
LDFLAGS = @LDFLAGS@
LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@
@@ -2916,6 +3170,7 @@ PYTHON_PLATFORM = @PYTHON_PLATFORM@
PYTHON_PREFIX = @PYTHON_PREFIX@
PYTHON_VERSION = @PYTHON_VERSION@
RANLIB = @RANLIB@
+RESOLV_LIBS = @RESOLV_LIBS@
RPM_RELEASE = @RPM_RELEASE@
SED = @SED@
SET_MAKE = @SET_MAKE@
@@ -2988,7 +3243,7 @@ top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
AUTOMAKE_OPTIONS = gnu subdir-objects
-SUBDIRS = ocf java $(am__append_45)
+SUBDIRS = ocf java $(am__append_58)
DIST_SUBDIRS = gtest ocf libs3 java
BUILT_SOURCES = init-ceph
@@ -3015,15 +3270,16 @@ EXTRA_DIST = $(srcdir)/$(shell_scripts:%=%.in) \
$(srcdir)/upstart/ceph-mds-all-starter.conf \
$(srcdir)/upstart/radosgw.conf \
$(srcdir)/upstart/radosgw-all.conf \
- $(srcdir)/upstart/radosgw-all-starter.conf ceph.in ceph-disk \
+ $(srcdir)/upstart/radosgw-all-starter.conf \
+ $(srcdir)/upstart/rbdmap.conf ceph.in ceph-disk \
ceph-disk-prepare ceph-disk-activate ceph-disk-udev \
- ceph-create-keys ceph-rest-api mount.fuse.ceph rbdmap \
- unittest_bufferlist.sh yasm-wrapper libs3/COPYING \
- libs3/ChangeLog libs3/GNUmakefile libs3/GNUmakefile.mingw \
- libs3/GNUmakefile.osx libs3/INSTALL libs3/LICENSE libs3/README \
- libs3/TODO libs3/archlinux libs3/debian libs3/doxyfile \
- libs3/inc libs3/libs3.spec libs3/mswin libs3/src libs3/test \
- unittest_bufferlist.sh
+ ceph-create-keys ceph-rest-api ceph-crush-location \
+ mount.fuse.ceph rbdmap unittest_bufferlist.sh yasm-wrapper \
+ libs3/COPYING libs3/ChangeLog libs3/GNUmakefile \
+ libs3/GNUmakefile.mingw libs3/GNUmakefile.osx libs3/INSTALL \
+ libs3/LICENSE libs3/README libs3/TODO libs3/archlinux \
+ libs3/debian libs3/doxyfile libs3/inc libs3/libs3.spec \
+ libs3/mswin libs3/src libs3/test unittest_bufferlist.sh
CLEANFILES = $(shell_scripts) ceph_ver.h sample.fetch_config
# everything else we want to include in a 'make dist'
@@ -3052,12 +3308,13 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
mon/AuthMonitor.h mon/DataHealthService.h mon/Elector.h \
mon/LogMonitor.h mon/ConfigKeyService.h mon/HealthMonitor.h \
mon/HealthService.h mon/MDSMonitor.h mon/MonmapMonitor.h \
- mon/MonCap.h mon/MonClient.h mon/MonCommands.h mon/MonMap.h \
- mon/Monitor.h mon/MonitorStore.h mon/MonitorDBStore.h \
- mon/OSDMonitor.h mon/PGMap.h mon/PGMonitor.h mon/Paxos.h \
- mon/PaxosService.h mon/QuorumService.h mon/Session.h \
- mon/mon_types.h mds/inode_backtrace.h mds/flock.h mds/locks.c \
- mds/locks.h mds/Anchor.h mds/AnchorClient.h mds/AnchorServer.h \
+ mon/MonCap.h mon/MonClient.h mon/MonCommands.h \
+ mon/DumplingMonCommands.h mon/MonMap.h mon/Monitor.h \
+ mon/MonitorStore.h mon/MonitorDBStore.h mon/OSDMonitor.h \
+ mon/PGMap.h mon/PGMonitor.h mon/Paxos.h mon/PaxosService.h \
+ mon/QuorumService.h mon/Session.h mon/mon_types.h \
+ mds/inode_backtrace.h mds/flock.h mds/locks.c mds/locks.h \
+ mds/Anchor.h mds/AnchorClient.h mds/AnchorServer.h \
mds/CDentry.h mds/CDir.h mds/CInode.h mds/Capability.h \
mds/Dumper.h mds/InoTable.h mds/LocalLock.h mds/Locker.h \
mds/LogEvent.h mds/LogSegment.h mds/MDBalancer.h mds/MDCache.h \
@@ -3075,31 +3332,34 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
mds/events/ESlaveUpdate.h mds/events/ESubtreeMap.h \
mds/events/ETableClient.h mds/events/ETableServer.h \
mds/events/EUpdate.h os/btrfs_ioctl.h os/chain_xattr.h \
- os/CollectionIndex.h os/FileJournal.h os/FileStore.h \
- os/BtrfsFileStoreBackend.h os/GenericFileStoreBackend.h \
- os/ZFSFileStoreBackend.h os/FlatIndex.h os/HashIndex.h \
- os/FDCache.h os/WBThrottle.h os/IndexManager.h os/Journal.h \
- os/JournalingObjectStore.h os/LFNIndex.h os/ObjectStore.h \
- os/SequencerPosition.h os/ObjectMap.h os/DBObjectMap.h \
- os/KeyValueDB.h os/LevelDBStore.h $(am__append_14) \
+ os/BtrfsFileStoreBackend.h os/CollectionIndex.h \
+ os/DBObjectMap.h os/GenericObjectMap.h os/FileJournal.h \
+ os/FileStore.h os/FlatIndex.h os/FDCache.h \
+ os/GenericFileStoreBackend.h os/HashIndex.h os/IndexManager.h \
+ os/Journal.h os/JournalingObjectStore.h os/KeyValueDB.h \
+ os/LevelDBStore.h os/LFNIndex.h os/MemStore.h \
+ os/KeyValueStore.h os/ObjectMap.h os/ObjectStore.h \
+ os/SequencerPosition.h os/WBThrottle.h \
+ os/ZFSFileStoreBackend.h $(am__append_17) \
osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \
osd/ErasureCodePluginJerasure/cauchy.h \
osd/ErasureCodePluginJerasure/galois.h \
osd/ErasureCodePluginJerasure/jerasure.h \
osd/ErasureCodePluginJerasure/liberation.h \
- osd/ErasureCodePluginJerasure/reed_sol.h osd/Ager.h \
+ osd/ErasureCodePluginJerasure/reed_sol.h \
+ osd/ErasureCodePluginJerasure/vectorop.h osd/Ager.h \
osd/ClassHandler.h osd/ErasureCodeInterface.h \
- osd/ErasureCodePlugin.h osd/OSD.h osd/OSDCap.h osd/OSDMap.h \
- osd/ObjectVersioner.h osd/OpRequest.h osd/SnapMapper.h \
- osd/PG.h osd/PGLog.h osd/ReplicatedPG.h osd/PGBackend.h \
- osd/ReplicatedBackend.h osd/Watch.h osd/osd_types.h \
- osdc/Blinker.h osdc/Filer.h osdc/Journaler.h \
+ osd/ErasureCodePlugin.h osd/HitSet.h osd/OSD.h osd/OSDCap.h \
+ osd/OSDMap.h osd/ObjectVersioner.h osd/OpRequest.h \
+ osd/SnapMapper.h osd/PG.h osd/PGLog.h osd/ReplicatedPG.h \
+ osd/PGBackend.h osd/ReplicatedBackend.h osd/Watch.h \
+ osd/osd_types.h osdc/Blinker.h osdc/Filer.h osdc/Journaler.h \
osdc/ObjectCacher.h osdc/Objecter.h osdc/Striper.h \
osdc/WritebackHandler.h client/Client.h client/Dentry.h \
client/Dir.h client/Fh.h client/Inode.h client/MetaRequest.h \
client/MetaSession.h client/ClientSnapRealm.h \
client/SyntheticClient.h client/Trace.h client/ioctl.h \
- client/ObjecterWriteback.h $(am__append_16) global/pidfile.h \
+ client/ObjecterWriteback.h $(am__append_20) global/pidfile.h \
global/global_init.h global/global_context.h \
global/signal_handler.h json_spirit/json_spirit.h \
json_spirit/json_spirit_error_position.h \
@@ -3144,9 +3404,9 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
common/utf8.h common/mime.h common/pick_address.h \
common/secret.h common/strtol.h common/static_assert.h \
common/AsyncReserver.h common/sync_filesystem.h \
- common/cmdparse.h common/hobject.h msg/Accepter.h \
- msg/DispatchQueue.h msg/Dispatcher.h msg/Message.h \
- msg/Messenger.h msg/Pipe.h msg/SimpleMessenger.h \
+ common/cmdparse.h common/hobject.h common/linux_version.h \
+ msg/Accepter.h msg/DispatchQueue.h msg/Dispatcher.h \
+ msg/Message.h msg/Messenger.h msg/Pipe.h msg/SimpleMessenger.h \
msg/msg_types.h messages/MAuth.h messages/MAuthReply.h \
messages/MCacheExpire.h messages/MClientCaps.h \
messages/MClientCapRelease.h messages/MClientLease.h \
@@ -3212,40 +3472,46 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
include/ipaddr.h include/linux_fiemap.h include/lru.h \
include/msgr.h include/object.h include/page.h \
include/rangeset.h include/rados.h include/rbd_types.h \
- include/statlite.h include/str_list.h include/stringify.h \
- include/triple.h include/types.h include/utime.h \
- include/dlist.h include/elist.h include/uuid.h include/xlist.h \
- include/rados/librados.h include/rados/rados_types.h \
- include/rados/rados_types.hpp include/rados/librados.hpp \
- include/rados/librgw.h include/rados/page.h \
- include/rados/crc32c.h include/rados/buffer.h \
- include/rbd/features.h include/rbd/librbd.h \
- include/rbd/librbd.hpp include/util.h librados/snap_set_diff.h \
- librados/AioCompletionImpl.h librados/IoCtxImpl.h \
- librados/PoolAsyncCompletionImpl.h librados/RadosClient.h \
- librbd/AioCompletion.h librbd/AioRequest.h librbd/ImageCtx.h \
- librbd/internal.h librbd/LibrbdWriteback.h \
- librbd/parent_types.h librbd/SnapInfo.h librbd/WatchCtx.h \
- rgw/logrotate.conf rgw/rgw_acl.h rgw/rgw_acl_s3.h \
- rgw/rgw_acl_swift.h rgw/rgw_client_io.h rgw/rgw_fcgi.h \
- rgw/rgw_xml.h rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
+ include/statlite.h include/str_list.h include/str_map.h \
+ include/stringify.h include/triple.h include/types.h \
+ include/utime.h include/dlist.h include/elist.h include/uuid.h \
+ include/xlist.h include/rados/librados.h \
+ include/rados/rados_types.h include/rados/rados_types.hpp \
+ include/rados/librados.hpp include/rados/librgw.h \
+ include/rados/page.h include/rados/crc32c.h \
+ include/rados/buffer.h include/rbd/features.h \
+ include/rbd/librbd.h include/rbd/librbd.hpp include/util.h \
+ include/stat.h include/on_exit.h include/memory.h \
+ include/rados/memory.h include/hash_namespace.h \
+ include/unordered_set.h include/unordered_map.h \
+ librados/snap_set_diff.h librados/AioCompletionImpl.h \
+ librados/IoCtxImpl.h librados/PoolAsyncCompletionImpl.h \
+ librados/RadosClient.h librbd/AioCompletion.h \
+ librbd/AioRequest.h librbd/ImageCtx.h librbd/internal.h \
+ librbd/LibrbdWriteback.h librbd/parent_types.h \
+ librbd/SnapInfo.h librbd/WatchCtx.h rgw/logrotate.conf \
+ rgw/rgw_acl.h rgw/rgw_acl_s3.h rgw/rgw_acl_swift.h \
+ rgw/rgw_client_io.h rgw/rgw_fcgi.h rgw/rgw_xml.h \
+ rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \
rgw/rgw_cors_s3.h rgw/rgw_cors_swift.h rgw/rgw_string.h \
rgw/rgw_formats.h rgw/rgw_http_errors.h rgw/rgw_log.h \
- rgw/rgw_multi.h rgw/rgw_policy_s3.h rgw/rgw_gc.h \
- rgw/rgw_metadata.h rgw/rgw_multi_del.h rgw/rgw_op.h \
- rgw/rgw_http_client.h rgw/rgw_swift.h rgw/rgw_swift_auth.h \
- rgw/rgw_quota.h rgw/rgw_rados.h rgw/rgw_replica_log.h \
- rgw/rgw_resolve.h rgw/rgw_rest.h rgw/rgw_rest_swift.h \
- rgw/rgw_rest_s3.h rgw/rgw_auth_s3.h rgw/rgw_rest_admin.h \
- rgw/rgw_rest_usage.h rgw/rgw_rest_user.h rgw/rgw_rest_bucket.h \
- rgw/rgw_rest_client.h rgw/rgw_rest_conn.h rgw/rgw_tools.h \
- rgw/rgw_rest_metadata.h rgw/rgw_rest_log.h \
- rgw/rgw_rest_opstate.h rgw/rgw_rest_replica_log.h \
- rgw/rgw_rest_config.h rgw/rgw_usage.h rgw/rgw_user.h \
- rgw/rgw_bucket.h rgw/rgw_keystone.h cls/lock/cls_lock_types.h \
- cls/lock/cls_lock_ops.h cls/lock/cls_lock_client.h \
- cls/rbd/cls_rbd.h cls/rbd/cls_rbd_client.h \
- cls/refcount/cls_refcount_ops.h \
+ rgw/rgw_loadgen.h rgw/rgw_multi.h rgw/rgw_policy_s3.h \
+ rgw/rgw_gc.h rgw/rgw_metadata.h rgw/rgw_multi_del.h \
+ rgw/rgw_op.h rgw/rgw_http_client.h rgw/rgw_swift.h \
+ rgw/rgw_swift_auth.h rgw/rgw_quota.h rgw/rgw_rados.h \
+ rgw/rgw_replica_log.h rgw/rgw_resolve.h rgw/rgw_rest.h \
+ rgw/rgw_rest_swift.h rgw/rgw_rest_s3.h rgw/rgw_auth_s3.h \
+ rgw/rgw_rest_admin.h rgw/rgw_rest_usage.h rgw/rgw_rest_user.h \
+ rgw/rgw_rest_bucket.h rgw/rgw_rest_client.h \
+ rgw/rgw_rest_conn.h rgw/rgw_tools.h rgw/rgw_rest_metadata.h \
+ rgw/rgw_rest_log.h rgw/rgw_rest_opstate.h \
+ rgw/rgw_rest_replica_log.h rgw/rgw_rest_config.h \
+ rgw/rgw_usage.h rgw/rgw_user.h rgw/rgw_bucket.h \
+ rgw/rgw_keystone.h rgw/rgw_civetweb.h civetweb/civetweb.h \
+ civetweb/include/civetweb.h civetweb/src/md5.h \
+ cls/lock/cls_lock_types.h cls/lock/cls_lock_ops.h \
+ cls/lock/cls_lock_client.h cls/rbd/cls_rbd.h \
+ cls/rbd/cls_rbd_client.h cls/refcount/cls_refcount_ops.h \
cls/refcount/cls_refcount_client.h \
cls/version/cls_version_types.h cls/version/cls_version_ops.h \
cls/version/cls_version_client.h cls/log/cls_log_types.h \
@@ -3257,30 +3523,31 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
cls/replica_log/cls_replica_log_ops.h \
cls/replica_log/cls_replica_log_client.h \
cls/rgw/cls_rgw_client.h cls/rgw/cls_rgw_ops.h \
- cls/rgw/cls_rgw_types.h key_value_store/key_value_structure.h \
+ cls/rgw/cls_rgw_types.h cls/user/cls_user_client.h \
+ cls/user/cls_user_ops.h cls/user/cls_user_types.h \
+ key_value_store/key_value_structure.h \
key_value_store/kv_flat_btree_async.h \
key_value_store/kvs_arg_types.h test/osd/ErasureCodeExample.h \
- test/osd/RadosModel.h test/osd/Object.h test/osd/TestOpStat.h \
- test/bench/distribution.h test/bench/rados_backend.h \
- test/bench/rbd_backend.h test/bench/bencher.h \
- test/bench/backend.h test/bench/dumb_backend.h \
- test/bench/stat_collector.h \
- test/bench/detailed_stat_collector.h \
+ test/bench/backend.h test/bench/bencher.h \
+ test/bench/detailed_stat_collector.h test/bench/distribution.h \
+ test/bench/dumb_backend.h test/bench/rados_backend.h \
+ test/bench/rbd_backend.h test/bench/stat_collector.h \
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h test/encoding/types.h \
test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreTracker.h \
test/filestore/FileStoreDiff.h \
+ test/filestore/FileStoreTracker.h \
test/filestore/TestFileStoreState.h \
test/filestore/workload_generator.h test/kv_store_bench.h \
test/librados/test.h test/ObjectMap/KeyValueDBMemory.h \
- test/omap_bench.h test/osd/Object.h test/osd/RadosModel.h \
- test/osd/TestOpStat.h test/osdc/FakeWriteback.h \
+ test/omap_bench.h test/osd/ceph_erasure_code_benchmark.h \
+ test/osdc/FakeWriteback.h test/osd/Object.h \
+ test/osd/RadosModel.h test/osd/TestOpStat.h \
test/system/cross_process_sem.h \
test/system/st_rados_create_pool.h \
- test/system/st_rados_list_objects.h \
test/system/st_rados_delete_objs.h \
test/system/st_rados_delete_pool.h \
+ test/system/st_rados_list_objects.h \
test/system/st_rados_notify.h test/system/st_rados_watch.h \
test/system/systest_runnable.h test/system/systest_settings.h \
test/unit.h tools/rados/rados_sync.h tools/common.h cls_acl.cc \
@@ -3289,58 +3556,57 @@ noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \
bash_completion/radosgw-admin mount/canonicalize.c \
mount/mtab.c objclass/objclass.h
bin_SCRIPTS = ceph ceph-run ceph-rest-api ceph-clsinfo ceph-debugpack \
- ceph-rbdnamer ceph-post-file ceph-coverage
+ ceph-rbdnamer ceph-post-file ceph-crush-location ceph-coverage
sbin_SCRIPTS =
su_sbin_SCRIPTS = mount.fuse.ceph mkcephfs
dist_bin_SCRIPTS =
-lib_LTLIBRARIES = librados.la librbd.la libcephfs.la $(am__append_50)
+lib_LTLIBRARIES = librados.la librbd.la libcephfs.la $(am__append_63)
noinst_LTLIBRARIES = libarch.la libauth.la libcrush.la libmon.la \
libmds.la libos.la libosd.la libosdc.la libclient.la \
- $(am__append_15) libglobal.la libjson_spirit.la liblog.la \
+ $(am__append_19) libglobal.la libjson_spirit.la liblog.la \
libperfglue.la libcommon_crc.la libcommon.la libmsg.la \
- $(am__append_26) libcls_lock_client.la \
+ $(am__append_32) libcls_lock_client.la \
libcls_refcount_client.la libcls_rgw_client.la \
- libcls_rbd_client.la libsystest.la
-noinst_LIBRARIES = $(am__append_13) libcls_version_client.a \
+ libcls_rbd_client.la $(am__append_43)
+noinst_LIBRARIES = $(am__append_16) libcls_version_client.a \
libcls_log_client.a libcls_statelog_client.a \
- libcls_replica_log_client.a
+ libcls_replica_log_client.a libcls_user_client.a
radoslib_LTLIBRARIES = libcls_hello.la libcls_rbd.la libcls_lock.la \
libcls_refcount.la libcls_version.la libcls_log.la \
- libcls_statelog.la libcls_replica_log.la libcls_rgw.la \
- libcls_kvs.la
+ libcls_statelog.la libcls_replica_log.la libcls_user.la \
+ libcls_rgw.la $(am__append_37)
# like bin_PROGRAMS, but these targets are only built for debug builds
-bin_DEBUGPROGRAMS = ceph_test_ioctls $(am__append_29) ceph_test_timers \
+bin_DEBUGPROGRAMS = ceph_test_ioctls $(am__append_35) ceph_test_timers \
ceph_test_signal_handlers ceph_test_rados ceph_test_mutate \
ceph_test_rewrite_latency ceph_test_msgr ceph_streamtest \
ceph_test_trans ceph_test_crypto ceph_test_keys \
- $(am__append_33) ceph_smalliobench ceph_smalliobenchfs \
+ $(am__append_40) ceph_smalliobench ceph_smalliobenchfs \
ceph_smalliobenchdumb ceph_smalliobenchrbd ceph_tpbench \
- ceph_omapbench ceph_kvstorebench ceph_multi_stress_watch \
- ceph_test_rados_list_parallel \
- ceph_test_rados_open_pools_parallel \
- ceph_test_rados_delete_pools_parallel \
- ceph_test_rados_watch_notify ceph_bench_log $(am__append_38) \
- ceph_test_librbd ceph_test_librbd_fsx ceph_test_cls_rbd \
- ceph_test_cls_refcount ceph_test_cls_version ceph_test_cls_log \
- ceph_test_cls_statelog ceph_test_cls_replica_log \
- ceph_test_cls_lock ceph_test_cls_hello $(am__append_39) \
- ceph_test_mon_workloadgen ceph_test_rados_api_cmd \
- ceph_test_rados_api_io ceph_test_rados_api_aio \
+ ceph_omapbench $(am__append_41) ceph_multi_stress_watch \
+ ceph_erasure_code_benchmark $(am__append_44) ceph_bench_log \
+ $(am__append_49) ceph_test_librbd $(am__append_50) \
+ ceph_test_cls_rbd ceph_test_cls_refcount ceph_test_cls_version \
+ ceph_test_cls_log ceph_test_cls_statelog \
+ ceph_test_cls_replica_log ceph_test_cls_lock \
+ ceph_test_cls_hello $(am__append_51) ceph_test_mon_workloadgen \
+ ceph_test_rados_api_cmd ceph_test_rados_api_io \
+ ceph_test_rados_api_c_write_operations ceph_test_rados_api_aio \
ceph_test_rados_api_list ceph_test_rados_api_pool \
ceph_test_rados_api_stat ceph_test_rados_api_watch_notify \
ceph_test_rados_api_snapshots ceph_test_rados_api_cls \
- ceph_test_rados_api_misc ceph_test_rados_api_lock \
- ceph_test_libcephfs ceph_test_filestore \
+ ceph_test_rados_api_misc ceph_test_rados_api_tier \
+ ceph_test_rados_api_lock ceph_test_libcephfs $(am__append_52) \
ceph_test_filestore_workloadgen ceph_test_filestore_idempotent \
ceph_test_filestore_idempotent_sequence ceph_xattr_bench \
ceph_test_filejournal ceph_test_stress_watch \
ceph_test_objectcacher_stress ceph_test_snap_mapper \
ceph_test_object_map ceph_test_keyvaluedb_atomicity \
ceph_test_keyvaluedb_iterators \
- ceph_test_cfuse_cache_invalidate ceph-osdomap-tool \
- ceph-monstore-tool ceph-kvstore-tool ceph_scratchtool \
- ceph_scratchtoolpp ceph_psim ceph_dupstore ceph_radosacl
+ ceph_test_cfuse_cache_invalidate ceph_test_c_headers \
+ ceph_test_get_blkdev_size ceph-osdomap-tool ceph-monstore-tool \
+ ceph-kvstore-tool ceph_scratchtool ceph_scratchtoolpp \
+ ceph_psim ceph_dupstore ceph_radosacl
# like sbin_SCRIPTS but can be used to install to e.g. /usr/sbin
ceph_sbindir = $(sbindir)
@@ -3350,7 +3616,9 @@ su_sbindir = /sbin
# tests scripts will be appended to this
check_SCRIPTS = unittest_bufferlist.sh \
- test/encoding/check-generated.sh \
+ test/encoding/check-generated.sh test/mon/osd-pool-create.sh \
+ test/mon/mkfs.sh test/ceph-disk.sh \
+ test/mon/mon-handle-forward.sh test/vstart_wrapped_tests.sh \
test/pybind/test_ceph_argparse.py
##################################
@@ -3361,10 +3629,11 @@ AM_COMMON_CPPFLAGS = \
-D_THREAD_SAFE \
-D__STDC_FORMAT_MACROS \
-D_GNU_SOURCE \
- -DCEPH_LIBDIR=\"${libdir}\"
+ -DCEPH_LIBDIR=\"${libdir}\" \
+ -DCEPH_PKGLIBDIR=\"${pkglibdir}\" \
+ -DGTEST_HAS_TR1_TUPLE=0
AM_COMMON_CFLAGS = \
- -rdynamic \
-Wall \
${WARN_TYPE_LIMITS} \
${WARN_IGNORED_QUALIFIERS} \
@@ -3374,7 +3643,7 @@ AM_COMMON_CFLAGS = \
-fno-strict-aliasing \
-fsigned-char
-AM_CFLAGS = $(AM_COMMON_CFLAGS) $(am__append_4) $(am__append_18)
+AM_CFLAGS = $(AM_COMMON_CFLAGS) $(am__append_5) $(am__append_22)
AM_CPPFLAGS = $(AM_COMMON_CPPFLAGS)
# note: this is position dependant, it affects the -l options that
@@ -3385,16 +3654,16 @@ AM_CPPFLAGS = $(AM_COMMON_CPPFLAGS)
# http://www.gentoo.org/proj/en/qa/asneeded.xml
# http://gcc.gnu.org/ml/gcc-help/2010-12/msg00338.html
# http://sigquit.wordpress.com/2011/02/16/why-asneeded-doesnt-work-as-expected-for-your-libraries-on-your-autotools-project/
-AM_LDFLAGS = -Wl,--as-needed $(am__append_3)
+AM_LDFLAGS = $(am__append_2) $(am__append_4)
AM_CCASFLAGS = -f elf64
#####################
-EXTRALIBS = -luuid -lm -lkeyutils $(am__append_6) $(am__append_7) \
- $(am__append_8) $(am__append_12)
+EXTRALIBS = -luuid -lm $(KEYUTILS_LIB) $(am__append_7) $(am__append_8) \
+ $(am__append_9) $(am__append_13)
LIBGLOBAL = libglobal.la
LIBCOMMON = libcommon.la
LIBARCH = libarch.la
-LIBPERFGLUE = libperfglue.la $(am__append_11)
+LIBPERFGLUE = libperfglue.la $(am__append_12)
LIBAUTH = libauth.la
LIBMSG = libmsg.la
LIBCRUSH = libcrush.la
@@ -3402,7 +3671,7 @@ LIBJSON_SPIRIT = libjson_spirit.la
LIBLOG = liblog.la
# Always use system leveldb
-LIBOS = libos.la $(am__append_9) $(am__append_10) -lleveldb -lsnappy
+LIBOS = libos.la $(am__append_10) $(am__append_11) -lleveldb -lsnappy
# Libosd always needs osdc and os
LIBOSD = libosd.la $(LIBOSDC) $(LIBOS) $(LIBPERFGLUE)
@@ -3419,21 +3688,22 @@ LIBRBD = librbd.la
LIBCEPHFS = libcephfs.la
# Use this for binaries requiring libglobal
-CEPH_GLOBAL = $(LIBGLOBAL) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS)
+CEPH_GLOBAL = $(LIBGLOBAL) $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS)
# This is set by [lib]/Makefile.am and used for build tests
# important; libmsg before libauth!
LIBCOMMON_DEPS = libcommon_crc.la $(LIBMSG) $(LIBAUTH) $(LIBCRUSH) \
- $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) -lkeyutils \
- $(am__append_25)
+ $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) $(KEYUTILS_LIB) \
+ $(am__append_29)
LIBRADOS_DEPS = libcls_lock_client.la $(LIBOSDC) $(LIBCOMMON)
-LIBRGW_DEPS = $(am__append_27)
+LIBRGW_DEPS = $(am__append_33)
# This is used by the dencoder test
-DENCODER_SOURCES = $(am__append_30)
+DENCODER_SOURCES = $(am__append_36)
DENCODER_DEPS = libcls_lock_client.la libcls_refcount_client.la \
- libcls_replica_log_client.a libcls_rgw_client.la
+ libcls_replica_log_client.a libcls_rgw_client.la \
+ libcls_user_client.a
radoslibdir = $(libdir)/rados-classes
libarch_la_SOURCES = \
arch/intel.c \
@@ -3517,27 +3787,16 @@ libmds_la_SOURCES = \
mds/MDLog.cc
libmds_la_LIBADD = $(LIBOSDC)
-libos_la_SOURCES = \
- os/FileJournal.cc \
- os/FileStore.cc \
- os/chain_xattr.cc \
- os/ObjectStore.cc \
- os/JournalingObjectStore.cc \
- os/LFNIndex.cc \
- os/HashIndex.cc \
- os/IndexManager.cc \
- os/FlatIndex.cc \
- os/DBObjectMap.cc \
- os/LevelDBStore.cc \
- os/WBThrottle.cc \
- os/BtrfsFileStoreBackend.cc \
- os/GenericFileStoreBackend.cc \
- os/ZFSFileStoreBackend.cc \
- common/TrackedOp.cc
-
+libos_la_SOURCES = os/chain_xattr.cc os/DBObjectMap.cc \
+ os/GenericObjectMap.cc os/FileJournal.cc os/FileStore.cc \
+ os/FlatIndex.cc os/GenericFileStoreBackend.cc os/HashIndex.cc \
+ os/IndexManager.cc os/JournalingObjectStore.cc \
+ os/LevelDBStore.cc os/LFNIndex.cc os/MemStore.cc \
+ os/KeyValueStore.cc os/ObjectStore.cc os/WBThrottle.cc \
+ common/TrackedOp.cc $(am__append_14) $(am__append_15)
@WITH_LIBZFS_TRUE at libos_zfs_a_SOURCES = os/ZFS.cc
@WITH_LIBZFS_TRUE at libos_zfs_a_CXXFLAGS = ${AM_CXXFLAGS} ${LIBZFS_CFLAGS}
-erasure_codelibdir = $(libdir)/erasure-code
+erasure_codelibdir = $(pkglibdir)/erasure-code
erasure_codelib_LTLIBRARIES = libec_jerasure.la libec_example.la \
libec_missing_entry_point.la libec_hangs.la \
libec_fail_to_initialize.la libec_fail_to_register.la
@@ -3555,14 +3814,17 @@ libec_jerasure_la_SOURCES = \
libec_jerasure_la_CFLAGS = ${AM_CFLAGS}
libec_jerasure_la_CXXFLAGS = ${AM_CXXFLAGS}
libec_jerasure_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__erasure_code_.*'
+libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 \
+ $(am__append_18)
libosd_la_SOURCES = \
osd/ErasureCodePlugin.cc \
osd/PG.cc \
osd/PGLog.cc \
osd/ReplicatedPG.cc \
osd/ReplicatedBackend.cc \
+ osd/PGBackend.cc \
osd/Ager.cc \
+ osd/HitSet.cc \
osd/OSD.cc \
osd/OSDCap.cc \
osd/Watch.cc \
@@ -3603,15 +3865,14 @@ libglobal_la_SOURCES = \
libglobal_la_LIBADD = $(LIBCOMMON)
libjson_spirit_la_SOURCES = \
json_spirit/json_spirit_reader.cpp \
- json_spirit/json_spirit_writer.cpp \
- json_spirit/json_spirit_value.cpp
+ json_spirit/json_spirit_writer.cpp
liblog_la_SOURCES = \
log/Log.cc \
log/SubsystemMap.cc
-libperfglue_la_SOURCES = $(am__append_17) $(am__append_20) \
- $(am__append_21) $(am__append_22)
+libperfglue_la_SOURCES = $(am__append_21) $(am__append_24) \
+ $(am__append_25) $(am__append_26)
@WITH_TCMALLOC_TRUE at libperfglue_la_LIBADD = -ltcmalloc
# these should go out of libcommon
@@ -3626,27 +3887,29 @@ libcommon_la_SOURCES = ceph_ver.c common/DecayCounter.cc \
common/assert.cc common/run_cmd.cc common/WorkQueue.cc \
common/ConfUtils.cc common/MemoryModel.cc common/armor.c \
common/fd.cc common/xattr.c common/safe_io.c \
- common/snap_types.cc common/str_list.cc common/errno.cc \
- common/RefCountedObj.cc common/blkdev.cc common/common_init.cc \
- common/pipe.c common/ceph_argparse.cc common/ceph_context.cc \
- common/buffer.cc common/code_environment.cc common/dout.cc \
- common/signal.cc common/simple_spin.cc common/Thread.cc \
- common/Formatter.cc common/HeartbeatMap.cc common/config.cc \
- common/utf8.c common/mime.c common/strtol.cc common/page.cc \
+ common/snap_types.cc common/str_list.cc common/str_map.cc \
+ common/errno.cc common/RefCountedObj.cc common/blkdev.cc \
+ common/common_init.cc common/pipe.c common/ceph_argparse.cc \
+ common/ceph_context.cc common/buffer.cc \
+ common/code_environment.cc common/dout.cc common/signal.cc \
+ common/simple_spin.cc common/Thread.cc common/Formatter.cc \
+ common/HeartbeatMap.cc common/config.cc common/utf8.c \
+ common/mime.c common/strtol.cc common/page.cc \
common/lockdep.cc common/version.cc common/hex.cc \
common/entity_name.cc common/ceph_crypto.cc \
common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \
common/pick_address.cc common/util.cc common/TextTable.cc \
common/ceph_fs.cc common/ceph_hash.cc common/ceph_strings.cc \
common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \
- common/bloom_filter.cc $(am__append_23) mon/MonCap.cc \
- mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc osd/osd_types.cc \
- mds/MDSMap.cc mds/inode_backtrace.cc mds/mdstypes.cc
+ common/bloom_filter.cc common/linux_version.c $(am__append_27) \
+ mon/MonCap.cc mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc \
+ osd/osd_types.cc osd/HitSet.cc mds/MDSMap.cc \
+ mds/inode_backtrace.cc mds/mdstypes.cc
# inject crc in common
libcommon_crc_la_SOURCES = common/sctp_crc32.c common/crc32c.cc \
common/crc32c_intel_baseline.c common/crc32c_intel_fast.c \
- $(am__append_24)
+ $(am__append_28)
@WITH_GOOD_YASM_ELF64_TRUE at libcommon_crc_la_LIBTOOLFLAGS = --tag=CC
libcommon_la_LIBADD = $(LIBCOMMON_DEPS)
libmsg_la_SOURCES = \
@@ -3674,7 +3937,8 @@ rados_include_DATA = \
$(srcdir)/include/rados/librados.hpp \
$(srcdir)/include/buffer.h \
$(srcdir)/include/page.h \
- $(srcdir)/include/crc32c.h
+ $(srcdir)/include/crc32c.h \
+ $(srcdir)/include/memory.h
librados_la_SOURCES = \
librados/librados.cc \
@@ -3684,9 +3948,10 @@ librados_la_SOURCES = \
# We need this to avoid basename conflicts with the librados build tests in test/Makefile.am
-librados_la_CFLAGS = ${AM_CFLAGS}
+librados_la_CXXFLAGS = ${AM_CXXFLAGS}
librados_la_LIBADD = $(LIBRADOS_DEPS) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS)
-librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 -export-symbols-regex '^rados_.*'
+librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \
+ $(am__append_30)
librbd_la_SOURCES = \
librbd/librbd.cc \
librbd/AioCompletion.cc \
@@ -3701,7 +3966,7 @@ librbd_la_LIBADD = \
libcls_rbd_client.la libcls_lock_client.la \
$(PTHREAD_LIBS) $(EXTRALIBS)
-librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^rbd_.*'
+librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 $(am__append_31)
@WITH_RADOSGW_TRUE at librgw_la_SOURCES = \
@WITH_RADOSGW_TRUE@ rgw/librgw.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_acl.cc \
@@ -3755,9 +4020,13 @@ librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^rb
@WITH_RADOSGW_TRUE@ rgw/rgw_http_client.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_swift.cc \
@WITH_RADOSGW_TRUE@ rgw/rgw_swift_auth.cc \
+ at WITH_RADOSGW_TRUE@ rgw/rgw_loadgen.cc \
+ at WITH_RADOSGW_TRUE@ rgw/rgw_civetweb.cc \
+ at WITH_RADOSGW_TRUE@ civetweb/src/civetweb.c \
@WITH_RADOSGW_TRUE@ rgw/rgw_main.cc
- at WITH_RADOSGW_TRUE@radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) -lresolv $(CEPH_GLOBAL)
+ at WITH_RADOSGW_TRUE@radosgw_CFLAGS = -Icivetweb/include
+ at WITH_RADOSGW_TRUE@radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(RESOLV_LIBS) $(CEPH_GLOBAL)
@WITH_RADOSGW_TRUE at radosgw_admin_SOURCES = rgw/rgw_admin.cc
@WITH_RADOSGW_TRUE at radosgw_admin_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL)
@WITH_RADOSGW_TRUE at ceph_rgw_multiparser_SOURCES = rgw/rgw_multiparser.cc
@@ -3771,32 +4040,35 @@ librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^rb
@WITH_RADOSGW_TRUE at ceph_rgw_jsonparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL)
libcls_hello_la_SOURCES = cls/hello/cls_hello.cc
libcls_hello_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_rbd_la_SOURCES = cls/rbd/cls_rbd.cc
libcls_rbd_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_lock_la_SOURCES = cls/lock/cls_lock.cc
libcls_lock_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_refcount_la_SOURCES = \
cls/refcount/cls_refcount.cc \
cls/refcount/cls_refcount_ops.cc \
common/ceph_json.cc
libcls_refcount_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_version_la_SOURCES = cls/version/cls_version.cc
libcls_version_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_log_la_SOURCES = cls/log/cls_log.cc
libcls_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_statelog_la_SOURCES = cls/statelog/cls_statelog.cc
libcls_statelog_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_replica_log_la_SOURCES = cls/replica_log/cls_replica_log.cc
libcls_replica_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
+libcls_user_la_SOURCES = cls/user/cls_user.cc
+libcls_user_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libcls_user_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
libcls_rgw_la_SOURCES = \
cls/rgw/cls_rgw.cc \
cls/rgw/cls_rgw_ops.cc \
@@ -3804,7 +4076,7 @@ libcls_rgw_la_SOURCES = \
common/ceph_json.cc
libcls_rgw_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
libcls_lock_client_la_SOURCES = \
cls/lock/cls_lock_client.cc \
cls/lock/cls_lock_types.cc \
@@ -3831,9 +4103,13 @@ libcls_rgw_client_la_SOURCES = \
cls/rgw/cls_rgw_ops.cc
libcls_rbd_client_la_SOURCES = cls/rbd/cls_rbd_client.cc
-libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc
-libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_user_client_a_SOURCES = cls/user/cls_user_client.cc \
+ cls/user/cls_user_types.cc \
+ cls/user/cls_user_ops.cc
+
+ at LINUX_TRUE@libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc
+ at LINUX_TRUE@libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+ at LINUX_TRUE@libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
ceph_test_timers_SOURCES = test/TestTimers.cc
ceph_test_timers_LDADD = $(CEPH_GLOBAL)
ceph_test_signal_handlers_SOURCES = test/TestSignalHandlers.cc
@@ -3869,8 +4145,8 @@ ceph_dencoder_LDADD = \
# These should always use explicit _CFLAGS/_CXXFLAGS so avoid basename conflicts
-ceph_dencoder_CFLAGS = ${AM_CFLAGS} $(am__append_31)
-ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} $(am__append_32)
+ceph_dencoder_CFLAGS = ${AM_CFLAGS} $(am__append_38)
+ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} $(am__append_39)
get_command_descriptions_SOURCES = test/common/get_command_descriptions.cc
get_command_descriptions_LDADD = $(LIBMON) $(LIBCOMMON) $(CEPH_GLOBAL)
@@ -3928,78 +4204,83 @@ ceph_smalliobench_SOURCES = \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobench_LDADD = $(LIBRADOS) -lboost_program_options $(CEPH_GLOBAL)
+ceph_smalliobench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
ceph_smalliobenchfs_SOURCES = \
test/bench/small_io_bench_fs.cc \
test/bench/testfilestore_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobenchfs_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL)
+ceph_smalliobenchfs_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
ceph_smalliobenchdumb_SOURCES = \
test/bench/small_io_bench_dumb.cc \
test/bench/dumb_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobenchdumb_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL)
+ceph_smalliobenchdumb_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
ceph_smalliobenchrbd_SOURCES = \
test/bench/small_io_bench_rbd.cc \
test/bench/rbd_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) -lboost_program_options $(CEPH_GLOBAL)
+ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
ceph_tpbench_SOURCES = \
test/bench/tp_bench.cc \
test/bench/detailed_stat_collector.cc
-ceph_tpbench_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL)
+ceph_tpbench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
ceph_omapbench_SOURCES = test/omap_bench.cc
ceph_omapbench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
-ceph_kvstorebench_SOURCES = \
- test/kv_store_bench.cc \
- key_value_store/kv_flat_btree_async.cc
+ at LINUX_TRUE@ceph_kvstorebench_SOURCES = \
+ at LINUX_TRUE@ test/kv_store_bench.cc \
+ at LINUX_TRUE@ key_value_store/kv_flat_btree_async.cc
-ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+ at LINUX_TRUE@ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
ceph_multi_stress_watch_SOURCES = \
test/multi_stress_watch.cc \
test/librados/test.cc
ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
-libsystest_la_SOURCES = \
- test/system/cross_process_sem.cc \
- test/system/systest_runnable.cc \
- test/system/systest_settings.cc
-
-libsystest_la_LIBADD = $(CEPH_GLOBAL)
-ceph_test_rados_list_parallel_SOURCES = \
- test/system/rados_list_parallel.cc \
- test/system/st_rados_create_pool.cc \
- test/system/st_rados_list_objects.cc
-
-ceph_test_rados_list_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
-ceph_test_rados_open_pools_parallel_SOURCES = \
- test/system/rados_open_pools_parallel.cc \
- test/system/st_rados_create_pool.cc
-
-ceph_test_rados_open_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
-ceph_test_rados_delete_pools_parallel_SOURCES = \
- test/system/rados_delete_pools_parallel.cc \
- test/system/st_rados_create_pool.cc \
- test/system/st_rados_delete_pool.cc \
- test/system/st_rados_list_objects.cc
-
-ceph_test_rados_delete_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
-ceph_test_rados_watch_notify_SOURCES = \
- test/system/rados_watch_notify.cc \
- test/system/st_rados_create_pool.cc \
- test/system/st_rados_delete_pool.cc \
- test/system/st_rados_delete_objs.cc \
- test/system/st_rados_watch.cc \
- test/system/st_rados_notify.cc
-
-ceph_test_rados_watch_notify_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+ceph_erasure_code_benchmark_SOURCES = \
+ test/osd/ceph_erasure_code_benchmark.cc
+
+ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) \
+ $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) $(am__append_42)
+ at LINUX_TRUE@libsystest_la_SOURCES = \
+ at LINUX_TRUE@ test/system/cross_process_sem.cc \
+ at LINUX_TRUE@ test/system/systest_runnable.cc \
+ at LINUX_TRUE@ test/system/systest_settings.cc
+
+ at LINUX_TRUE@libsystest_la_LIBADD = $(CEPH_GLOBAL)
+ at LINUX_TRUE@ceph_test_rados_list_parallel_SOURCES = \
+ at LINUX_TRUE@ test/system/rados_list_parallel.cc \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.cc \
+ at LINUX_TRUE@ test/system/st_rados_list_objects.cc
+
+ at LINUX_TRUE@ceph_test_rados_list_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+ at LINUX_TRUE@ceph_test_rados_open_pools_parallel_SOURCES = \
+ at LINUX_TRUE@ test/system/rados_open_pools_parallel.cc \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.cc
+
+ at LINUX_TRUE@ceph_test_rados_open_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+ at LINUX_TRUE@ceph_test_rados_delete_pools_parallel_SOURCES = \
+ at LINUX_TRUE@ test/system/rados_delete_pools_parallel.cc \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.cc \
+ at LINUX_TRUE@ test/system/st_rados_delete_pool.cc \
+ at LINUX_TRUE@ test/system/st_rados_list_objects.cc
+
+ at LINUX_TRUE@ceph_test_rados_delete_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+ at LINUX_TRUE@ceph_test_rados_watch_notify_SOURCES = \
+ at LINUX_TRUE@ test/system/rados_watch_notify.cc \
+ at LINUX_TRUE@ test/system/st_rados_create_pool.cc \
+ at LINUX_TRUE@ test/system/st_rados_delete_pool.cc \
+ at LINUX_TRUE@ test/system/st_rados_delete_objs.cc \
+ at LINUX_TRUE@ test/system/st_rados_watch.cc \
+ at LINUX_TRUE@ test/system/st_rados_notify.cc
+
+ at LINUX_TRUE@ceph_test_rados_watch_notify_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
ceph_bench_log_SOURCES = test/bench_log.cc
ceph_bench_log_LDADD = $(CEPH_GLOBAL)
UNITTEST_CXXFLAGS = \
@@ -4021,6 +4302,12 @@ unittest_addrs_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_bloom_filter_SOURCES = test/common/test_bloom_filter.cc
unittest_bloom_filter_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_bloom_filter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_str_map_SOURCES = test/common/test_str_map.cc
+unittest_str_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_str_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_crushwrapper_SOURCES = test/test_crushwrapper.cc
+unittest_crushwrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_crushwrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc
unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -4030,6 +4317,12 @@ unittest_sloppy_crc_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_util_SOURCES = test/common/test_util.cc
unittest_util_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_util_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CRYPTO_LIBS) $(EXTRALIBS)
+unittest_crush_indep_SOURCES = test/crush/indep.cc
+unittest_crush_indep_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_crush_indep_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CEPH_CRUSH) $(EXTRALIBS) $(CEPH_GLOBAL)
+unittest_osdmap_SOURCES = test/osd/TestOSDMap.cc
+unittest_osdmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_osdmap_LDADD = $(UNITTEST_LDADD) $(LIBCOMMON) $(CEPH_GLOBAL)
unittest_workqueue_SOURCES = test/test_workqueue.cc
unittest_workqueue_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_workqueue_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -4048,6 +4341,9 @@ unittest_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
unittest_throttle_SOURCES = test/common/Throttle.cc
unittest_throttle_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_throttle_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
+unittest_crush_wrapper_SOURCES = test/crush/TestCrushWrapper.cc
+unittest_crush_wrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
+unittest_crush_wrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
unittest_base64_SOURCES = test/base64.cc
unittest_base64_LDADD = $(LIBCEPHFS) -lm $(UNITTEST_LDADD)
unittest_base64_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -4085,20 +4381,20 @@ libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__era
unittest_erasure_code_plugin_SOURCES = test/osd/TestErasureCodePlugin.cc
unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_34)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_45)
unittest_erasure_code_jerasure_SOURCES = \
test/osd/TestErasureCodeJerasure.cc \
$(libec_jerasure_la_SOURCES)
unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_35)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_46)
unittest_erasure_code_plugin_jerasure_SOURCES = \
test/osd/TestErasureCodePluginJerasure.cc
unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \
- $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_36)
+ $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_47)
unittest_erasure_code_example_SOURCES = test/osd/TestErasureCodeExample.cc
unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -4108,7 +4404,10 @@ unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_pglog_SOURCES = test/osd/TestPGLog.cc
unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) \
- $(am__append_37)
+ $(am__append_48)
+unittest_hitset_SOURCES = test/osd/hitset.cc
+unittest_hitset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_hitset_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_gather_SOURCES = test/gather.cc
unittest_gather_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_gather_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -4130,7 +4429,7 @@ unittest_bufferlist_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_crc32c_SOURCES = test/common/test_crc32c.cc
unittest_crc32c_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_crc32c_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_arch_SOURCES = test/test_arch.c
+unittest_arch_SOURCES = test/test_arch.cc
unittest_arch_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_arch_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_crypto_SOURCES = test/crypto.cc
@@ -4166,6 +4465,9 @@ unittest_strtol_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_confutils_SOURCES = test/confutils.cc
unittest_confutils_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_config_SOURCES = test/common/test_config.cc
+unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc
unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -4215,6 +4517,8 @@ unittest_ipaddr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_texttable_SOURCES = test/test_texttable.cc
unittest_texttable_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD)
unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_on_exit_SOURCES = test/on_exit.cc
+unittest_on_exit_LDADD = $(PTHREAD_LIBS)
@WITH_RADOSGW_TRUE at ceph_test_cors_SOURCES = test/test_cors.cc
@WITH_RADOSGW_TRUE at ceph_test_cors_LDADD = \
@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
@@ -4229,7 +4533,7 @@ unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_lock_client.la
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_meta_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_log_SOURCES = test/test_rgw_admin_log.cc
@@ -4239,7 +4543,7 @@ unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_lock_client.la
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_opstate_SOURCES = test/test_rgw_admin_opstate.cc
@@ -4249,7 +4553,7 @@ unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat \
@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \
@WITH_RADOSGW_TRUE@ libcls_statelog_client.a libcls_refcount_client.la \
- at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_lock_client.la
+ at WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
@WITH_RADOSGW_TRUE at ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_librbd_SOURCES = \
@@ -4258,9 +4562,9 @@ ceph_test_librbd_SOURCES = \
ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c
-ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm
-ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format
+ at LINUX_TRUE@ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c
+ at LINUX_TRUE@ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm
+ at LINUX_TRUE@ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format
ceph_test_cls_rbd_SOURCES = \
test/cls_rbd/test_cls_rbd.cc \
test/librados/test.cc
@@ -4335,6 +4639,12 @@ ceph_test_rados_api_io_SOURCES = \
ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+ceph_test_rados_api_c_write_operations_SOURCES = \
+ test/librados/c_write_operations.cc \
+ test/librados/test.cc
+
+ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_c_write_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_rados_api_aio_SOURCES = \
test/librados/aio.cc \
test/librados/test.cc
@@ -4383,6 +4693,13 @@ ceph_test_rados_api_misc_SOURCES = \
ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+ceph_test_rados_api_tier_SOURCES = \
+ test/librados/tier.cc \
+ test/librados/test.cc \
+ osd/HitSet.cc
+
+ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_rados_api_tier_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_rados_api_lock_SOURCES = \
test/librados/lock.cc \
test/librados/test.cc
@@ -4397,9 +4714,9 @@ ceph_test_libcephfs_SOURCES = \
ceph_test_libcephfs_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-ceph_test_filestore_SOURCES = test/filestore/store_test.cc
-ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+ at LINUX_TRUE@ceph_test_filestore_SOURCES = test/filestore/store_test.cc
+ at LINUX_TRUE@ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ at LINUX_TRUE@ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_filestore_workloadgen_SOURCES = \
test/filestore/workload_generator.cc \
test/filestore/TestFileStoreState.cc
@@ -4454,21 +4771,25 @@ ceph_test_keyvaluedb_iterators_SOURCES = \
ceph_test_keyvaluedb_iterators_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_keyvaluedb_iterators_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc
+ceph_test_c_headers_SOURCES = test/test_c_headers.c
+ceph_test_c_headers_LDADD = $(LIBRADOS) $(LIBCEPHFS)
+ceph_test_get_blkdev_size_SOURCES = test/test_get_blkdev_size.cc
+ceph_test_get_blkdev_size_LDADD = $(LIBCOMMON)
ceph_osdomap_tool_SOURCES = tools/ceph-osdomap-tool.cc
-ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc
-ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
ceph_kvstore_tool_SOURCES = tools/ceph-kvstore-tool.cc
ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL)
ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
ceph_filestore_tool_SOURCES = tools/ceph-filestore-tool.cc
ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) \
- -lboost_program_options $(am__append_40)
+ -lboost_program_options $(am__append_53)
ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) \
- -lboost_program_options $(am__append_41)
+ $(BOOST_PROGRAM_OPTIONS_LIBS) $(am__append_54)
monmaptool_SOURCES = tools/monmaptool.cc
-monmaptool_LDADD = $(CEPH_GLOBAL)
+monmaptool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
crushtool_SOURCES = tools/crushtool.cc
crushtool_LDADD = $(CEPH_GLOBAL)
osdmaptool_SOURCES = tools/osdmaptool.cc
@@ -4492,12 +4813,12 @@ rados_LDADD = libcls_lock_client.la $(LIBRADOS) $(CEPH_GLOBAL)
@WITH_REST_BENCH_TRUE@ common/obj_bencher.cc # needs cleanup so \
@WITH_REST_BENCH_TRUE@ it can go in libcommon.la
@WITH_REST_BENCH_TRUE at rest_bench_LDADD = $(CEPH_GLOBAL) \
- at WITH_REST_BENCH_TRUE@ $(am__append_43) $(am__append_44)
+ at WITH_REST_BENCH_TRUE@ $(am__append_56) $(am__append_57)
@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE at rest_bench_CXXFLAGS = ${AM_CXXFLAGS} -I$(top_srcdir)/src/libs3/inc
ceph_conf_SOURCES = tools/ceph_conf.cc
-ceph_conf_LDADD = $(CEPH_GLOBAL)
+ceph_conf_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
ceph_authtool_SOURCES = tools/ceph_authtool.cc
-ceph_authtool_LDADD = $(CEPH_GLOBAL)
+ceph_authtool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
ceph_mon_store_converter_SOURCES = tools/mon_store_converter.cc
ceph_mon_store_converter_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL)
@@ -4505,11 +4826,12 @@ ceph_mon_store_converter_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL)
# core daemons
ceph_mon_SOURCES = ceph_mon.cc
-ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL)
+ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) $(LIBCOMMON)
ceph_osd_SOURCES = ceph_osd.cc
-ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(am__append_46)
+ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON) \
+ $(am__append_59)
ceph_mds_SOURCES = ceph_mds.cc
-ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL)
+ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL) $(LIBCOMMON)
# admin tools
@@ -4523,8 +4845,8 @@ librados_config_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
ceph_syn_SOURCES = ceph_syn.cc client/SyntheticClient.cc # uses \
g_conf.. needs cleanup
ceph_syn_LDADD = $(LIBCLIENT) $(CEPH_GLOBAL)
-rbd_SOURCES = rbd.cc
-rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL)
+rbd_SOURCES = rbd.cc
+rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) -lblkid
# Fuse targets
@WITH_FUSE_TRUE at ceph_fuse_SOURCES = ceph_fuse.cc
@@ -4560,8 +4882,8 @@ editpaths = sed \
# coverage
-shell_scripts = ceph-debugpack ceph-post-file init-ceph mkcephfs \
- ceph-coverage
+shell_scripts = ceph-debugpack ceph-post-file ceph-crush-location \
+ init-ceph mkcephfs ceph-coverage
doc_DATA = $(srcdir)/sample.ceph.conf sample.fetch_config
# various scripts
@@ -4697,6 +5019,22 @@ libcls_statelog_client.a: $(libcls_statelog_client_a_OBJECTS) $(libcls_statelog_
$(AM_V_at)-rm -f libcls_statelog_client.a
$(AM_V_AR)$(libcls_statelog_client_a_AR) libcls_statelog_client.a $(libcls_statelog_client_a_OBJECTS) $(libcls_statelog_client_a_LIBADD)
$(AM_V_at)$(RANLIB) libcls_statelog_client.a
+cls/user/$(am__dirstamp):
+ @$(MKDIR_P) cls/user
+ @: > cls/user/$(am__dirstamp)
+cls/user/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) cls/user/$(DEPDIR)
+ @: > cls/user/$(DEPDIR)/$(am__dirstamp)
+cls/user/cls_user_client.$(OBJEXT): cls/user/$(am__dirstamp) \
+ cls/user/$(DEPDIR)/$(am__dirstamp)
+cls/user/cls_user_types.$(OBJEXT): cls/user/$(am__dirstamp) \
+ cls/user/$(DEPDIR)/$(am__dirstamp)
+cls/user/cls_user_ops.$(OBJEXT): cls/user/$(am__dirstamp) \
+ cls/user/$(DEPDIR)/$(am__dirstamp)
+libcls_user_client.a: $(libcls_user_client_a_OBJECTS) $(libcls_user_client_a_DEPENDENCIES) $(EXTRA_libcls_user_client_a_DEPENDENCIES)
+ $(AM_V_at)-rm -f libcls_user_client.a
+ $(AM_V_AR)$(libcls_user_client_a_AR) libcls_user_client.a $(libcls_user_client_a_OBJECTS) $(libcls_user_client_a_LIBADD)
+ $(AM_V_at)$(RANLIB) libcls_user_client.a
cls/version/$(am__dirstamp):
@$(MKDIR_P) cls/version
@: > cls/version/$(am__dirstamp)
@@ -4954,7 +5292,7 @@ key_value_store/$(DEPDIR)/$(am__dirstamp):
key_value_store/cls_kvs.lo: key_value_store/$(am__dirstamp) \
key_value_store/$(DEPDIR)/$(am__dirstamp)
libcls_kvs.la: $(libcls_kvs_la_OBJECTS) $(libcls_kvs_la_DEPENDENCIES) $(EXTRA_libcls_kvs_la_DEPENDENCIES)
- $(AM_V_CXXLD)$(libcls_kvs_la_LINK) -rpath $(radoslibdir) $(libcls_kvs_la_OBJECTS) $(libcls_kvs_la_LIBADD) $(LIBS)
+ $(AM_V_CXXLD)$(libcls_kvs_la_LINK) $(am_libcls_kvs_la_rpath) $(libcls_kvs_la_OBJECTS) $(libcls_kvs_la_LIBADD) $(LIBS)
cls/lock/$(am__dirstamp):
@$(MKDIR_P) cls/lock
@: > cls/lock/$(am__dirstamp)
@@ -5041,6 +5379,10 @@ cls/statelog/cls_statelog.lo: cls/statelog/$(am__dirstamp) \
cls/statelog/$(DEPDIR)/$(am__dirstamp)
libcls_statelog.la: $(libcls_statelog_la_OBJECTS) $(libcls_statelog_la_DEPENDENCIES) $(EXTRA_libcls_statelog_la_DEPENDENCIES)
$(AM_V_CXXLD)$(libcls_statelog_la_LINK) -rpath $(radoslibdir) $(libcls_statelog_la_OBJECTS) $(libcls_statelog_la_LIBADD) $(LIBS)
+cls/user/cls_user.lo: cls/user/$(am__dirstamp) \
+ cls/user/$(DEPDIR)/$(am__dirstamp)
+libcls_user.la: $(libcls_user_la_OBJECTS) $(libcls_user_la_DEPENDENCIES) $(EXTRA_libcls_user_la_DEPENDENCIES)
+ $(AM_V_CXXLD)$(libcls_user_la_LINK) -rpath $(radoslibdir) $(libcls_user_la_OBJECTS) $(libcls_user_la_LIBADD) $(LIBS)
cls/version/cls_version.lo: cls/version/$(am__dirstamp) \
cls/version/$(DEPDIR)/$(am__dirstamp)
libcls_version.la: $(libcls_version_la_OBJECTS) $(libcls_version_la_DEPENDENCIES) $(EXTRA_libcls_version_la_DEPENDENCIES)
@@ -5102,6 +5444,8 @@ common/snap_types.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/str_list.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
+common/str_map.lo: common/$(am__dirstamp) \
+ common/$(DEPDIR)/$(am__dirstamp)
common/errno.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/RefCountedObj.lo: common/$(am__dirstamp) \
@@ -5175,6 +5519,8 @@ common/hobject.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/bloom_filter.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
+common/linux_version.lo: common/$(am__dirstamp) \
+ common/$(DEPDIR)/$(am__dirstamp)
common/secret.lo: common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
mon/$(am__dirstamp):
@@ -5194,6 +5540,7 @@ osd/$(DEPDIR)/$(am__dirstamp):
@: > osd/$(DEPDIR)/$(am__dirstamp)
osd/OSDMap.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/osd_types.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
+osd/HitSet.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
mds/$(am__dirstamp):
@$(MKDIR_P) mds
@: > mds/$(am__dirstamp)
@@ -5320,8 +5667,6 @@ json_spirit/json_spirit_reader.lo: json_spirit/$(am__dirstamp) \
json_spirit/$(DEPDIR)/$(am__dirstamp)
json_spirit/json_spirit_writer.lo: json_spirit/$(am__dirstamp) \
json_spirit/$(DEPDIR)/$(am__dirstamp)
-json_spirit/json_spirit_value.lo: json_spirit/$(am__dirstamp) \
- json_spirit/$(DEPDIR)/$(am__dirstamp)
libjson_spirit.la: $(libjson_spirit_la_OBJECTS) $(libjson_spirit_la_DEPENDENCIES) $(EXTRA_libjson_spirit_la_DEPENDENCIES)
$(AM_V_CXXLD)$(CXXLINK) $(libjson_spirit_la_OBJECTS) $(libjson_spirit_la_LIBADD) $(LIBS)
log/$(am__dirstamp):
@@ -5405,27 +5750,31 @@ msg/SimpleMessenger.lo: msg/$(am__dirstamp) \
msg/msg_types.lo: msg/$(am__dirstamp) msg/$(DEPDIR)/$(am__dirstamp)
libmsg.la: $(libmsg_la_OBJECTS) $(libmsg_la_DEPENDENCIES) $(EXTRA_libmsg_la_DEPENDENCIES)
$(AM_V_CXXLD)$(CXXLINK) $(libmsg_la_OBJECTS) $(libmsg_la_LIBADD) $(LIBS)
+os/chain_xattr.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/DBObjectMap.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/GenericObjectMap.lo: os/$(am__dirstamp) \
+ os/$(DEPDIR)/$(am__dirstamp)
os/FileJournal.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
os/FileStore.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
-os/chain_xattr.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
-os/ObjectStore.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
-os/JournalingObjectStore.lo: os/$(am__dirstamp) \
+os/FlatIndex.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/GenericFileStoreBackend.lo: os/$(am__dirstamp) \
os/$(DEPDIR)/$(am__dirstamp)
-os/LFNIndex.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
os/HashIndex.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
os/IndexManager.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
-os/FlatIndex.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
-os/DBObjectMap.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/JournalingObjectStore.lo: os/$(am__dirstamp) \
+ os/$(DEPDIR)/$(am__dirstamp)
os/LevelDBStore.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/LFNIndex.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/MemStore.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/KeyValueStore.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+os/ObjectStore.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
os/WBThrottle.lo: os/$(am__dirstamp) os/$(DEPDIR)/$(am__dirstamp)
+common/TrackedOp.lo: common/$(am__dirstamp) \
+ common/$(DEPDIR)/$(am__dirstamp)
os/BtrfsFileStoreBackend.lo: os/$(am__dirstamp) \
os/$(DEPDIR)/$(am__dirstamp)
-os/GenericFileStoreBackend.lo: os/$(am__dirstamp) \
- os/$(DEPDIR)/$(am__dirstamp)
os/ZFSFileStoreBackend.lo: os/$(am__dirstamp) \
os/$(DEPDIR)/$(am__dirstamp)
-common/TrackedOp.lo: common/$(am__dirstamp) \
- common/$(DEPDIR)/$(am__dirstamp)
libos.la: $(libos_la_OBJECTS) $(libos_la_DEPENDENCIES) $(EXTRA_libos_la_DEPENDENCIES)
$(AM_V_CXXLD)$(CXXLINK) $(libos_la_OBJECTS) $(libos_la_LIBADD) $(LIBS)
osd/ErasureCodePlugin.lo: osd/$(am__dirstamp) \
@@ -5435,6 +5784,7 @@ osd/PGLog.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/ReplicatedPG.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/ReplicatedBackend.lo: osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
+osd/PGBackend.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/Ager.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/OSD.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
osd/OSDCap.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp)
@@ -5488,13 +5838,13 @@ librados/$(am__dirstamp):
librados/$(DEPDIR)/$(am__dirstamp):
@$(MKDIR_P) librados/$(DEPDIR)
@: > librados/$(DEPDIR)/$(am__dirstamp)
-librados/librados.lo: librados/$(am__dirstamp) \
+librados/librados_la-librados.lo: librados/$(am__dirstamp) \
librados/$(DEPDIR)/$(am__dirstamp)
-librados/RadosClient.lo: librados/$(am__dirstamp) \
+librados/librados_la-RadosClient.lo: librados/$(am__dirstamp) \
librados/$(DEPDIR)/$(am__dirstamp)
-librados/IoCtxImpl.lo: librados/$(am__dirstamp) \
+librados/librados_la-IoCtxImpl.lo: librados/$(am__dirstamp) \
librados/$(DEPDIR)/$(am__dirstamp)
-librados/snap_set_diff.lo: librados/$(am__dirstamp) \
+librados/librados_la-snap_set_diff.lo: librados/$(am__dirstamp) \
librados/$(DEPDIR)/$(am__dirstamp)
librados.la: $(librados_la_OBJECTS) $(librados_la_DEPENDENCIES) $(EXTRA_librados_la_DEPENDENCIES)
$(AM_V_CXXLD)$(librados_la_LINK) -rpath $(libdir) $(librados_la_OBJECTS) $(librados_la_LIBADD) $(LIBS)
@@ -5607,7 +5957,7 @@ test/system/systest_runnable.lo: test/system/$(am__dirstamp) \
test/system/systest_settings.lo: test/system/$(am__dirstamp) \
test/system/$(DEPDIR)/$(am__dirstamp)
libsystest.la: $(libsystest_la_OBJECTS) $(libsystest_la_DEPENDENCIES) $(EXTRA_libsystest_la_DEPENDENCIES)
- $(AM_V_CXXLD)$(CXXLINK) $(libsystest_la_OBJECTS) $(libsystest_la_LIBADD) $(LIBS)
+ $(AM_V_CXXLD)$(CXXLINK) $(am_libsystest_la_rpath) $(libsystest_la_OBJECTS) $(libsystest_la_LIBADD) $(LIBS)
install-binPROGRAMS: $(bin_PROGRAMS)
@$(NORMAL_INSTALL)
test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
@@ -5841,6 +6191,11 @@ tools/dupstore.$(OBJEXT): tools/$(am__dirstamp) \
ceph_dupstore$(EXEEXT): $(ceph_dupstore_OBJECTS) $(ceph_dupstore_DEPENDENCIES) $(EXTRA_ceph_dupstore_DEPENDENCIES)
@rm -f ceph_dupstore$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_dupstore_OBJECTS) $(ceph_dupstore_LDADD) $(LIBS)
+test/osd/ceph_erasure_code_benchmark.$(OBJEXT): \
+ test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+ceph_erasure_code_benchmark$(EXEEXT): $(ceph_erasure_code_benchmark_OBJECTS) $(ceph_erasure_code_benchmark_DEPENDENCIES) $(EXTRA_ceph_erasure_code_benchmark_DEPENDENCIES)
+ @rm -f ceph_erasure_code_benchmark$(EXEEXT)
+ $(AM_V_CXXLD)$(CXXLINK) $(ceph_erasure_code_benchmark_OBJECTS) $(ceph_erasure_code_benchmark_LDADD) $(LIBS)
tools/ceph-filestore-dump.$(OBJEXT): tools/$(am__dirstamp) \
tools/$(DEPDIR)/$(am__dirstamp)
ceph_filestore_dump$(EXEEXT): $(ceph_filestore_dump_OBJECTS) $(ceph_filestore_dump_DEPENDENCIES) $(EXTRA_ceph_filestore_dump_DEPENDENCIES)
@@ -5963,6 +6318,11 @@ test/streamtest.$(OBJEXT): test/$(am__dirstamp) \
ceph_streamtest$(EXEEXT): $(ceph_streamtest_OBJECTS) $(ceph_streamtest_DEPENDENCIES) $(EXTRA_ceph_streamtest_DEPENDENCIES)
@rm -f ceph_streamtest$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_streamtest_OBJECTS) $(ceph_streamtest_LDADD) $(LIBS)
+test/test_c_headers.$(OBJEXT): test/$(am__dirstamp) \
+ test/$(DEPDIR)/$(am__dirstamp)
+ceph_test_c_headers$(EXEEXT): $(ceph_test_c_headers_OBJECTS) $(ceph_test_c_headers_DEPENDENCIES) $(EXTRA_ceph_test_c_headers_DEPENDENCIES)
+ @rm -f ceph_test_c_headers$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(ceph_test_c_headers_OBJECTS) $(ceph_test_c_headers_LDADD) $(LIBS)
test/test_cfuse_cache_invalidate.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
ceph_test_cfuse_cache_invalidate$(EXEEXT): $(ceph_test_cfuse_cache_invalidate_OBJECTS) $(ceph_test_cfuse_cache_invalidate_DEPENDENCIES) $(EXTRA_ceph_test_cfuse_cache_invalidate_DEPENDENCIES)
@@ -6183,6 +6543,11 @@ test/filestore/workload_generator.$(OBJEXT): \
ceph_test_filestore_workloadgen$(EXEEXT): $(ceph_test_filestore_workloadgen_OBJECTS) $(ceph_test_filestore_workloadgen_DEPENDENCIES) $(EXTRA_ceph_test_filestore_workloadgen_DEPENDENCIES)
@rm -f ceph_test_filestore_workloadgen$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(ceph_test_filestore_workloadgen_OBJECTS) $(ceph_test_filestore_workloadgen_LDADD) $(LIBS)
+test/test_get_blkdev_size.$(OBJEXT): test/$(am__dirstamp) \
+ test/$(DEPDIR)/$(am__dirstamp)
+ceph_test_get_blkdev_size$(EXEEXT): $(ceph_test_get_blkdev_size_OBJECTS) $(ceph_test_get_blkdev_size_DEPENDENCIES) $(EXTRA_ceph_test_get_blkdev_size_DEPENDENCIES)
+ @rm -f ceph_test_get_blkdev_size$(EXEEXT)
+ $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_get_blkdev_size_OBJECTS) $(ceph_test_get_blkdev_size_LDADD) $(LIBS)
client/test_ioctls.$(OBJEXT): client/$(am__dirstamp) \
client/$(DEPDIR)/$(am__dirstamp)
ceph_test_ioctls$(EXEEXT): $(ceph_test_ioctls_OBJECTS) $(ceph_test_ioctls_DEPENDENCIES) $(EXTRA_ceph_test_ioctls_DEPENDENCIES)
@@ -6319,6 +6684,15 @@ test/librados/ceph_test_rados_api_aio-test.$(OBJEXT): \
ceph_test_rados_api_aio$(EXEEXT): $(ceph_test_rados_api_aio_OBJECTS) $(ceph_test_rados_api_aio_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_aio_DEPENDENCIES)
@rm -f ceph_test_rados_api_aio$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_aio_LINK) $(ceph_test_rados_api_aio_OBJECTS) $(ceph_test_rados_api_aio_LDADD) $(LIBS)
+test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT): \
+ test/librados/$(am__dirstamp) \
+ test/librados/$(DEPDIR)/$(am__dirstamp)
+test/librados/ceph_test_rados_api_c_write_operations-test.$(OBJEXT): \
+ test/librados/$(am__dirstamp) \
+ test/librados/$(DEPDIR)/$(am__dirstamp)
+ceph_test_rados_api_c_write_operations$(EXEEXT): $(ceph_test_rados_api_c_write_operations_OBJECTS) $(ceph_test_rados_api_c_write_operations_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_c_write_operations_DEPENDENCIES)
+ @rm -f ceph_test_rados_api_c_write_operations$(EXEEXT)
+ $(AM_V_CXXLD)$(ceph_test_rados_api_c_write_operations_LINK) $(ceph_test_rados_api_c_write_operations_OBJECTS) $(ceph_test_rados_api_c_write_operations_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
@@ -6400,6 +6774,17 @@ test/librados/ceph_test_rados_api_stat-test.$(OBJEXT): \
ceph_test_rados_api_stat$(EXEEXT): $(ceph_test_rados_api_stat_OBJECTS) $(ceph_test_rados_api_stat_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_stat_DEPENDENCIES)
@rm -f ceph_test_rados_api_stat$(EXEEXT)
$(AM_V_CXXLD)$(ceph_test_rados_api_stat_LINK) $(ceph_test_rados_api_stat_OBJECTS) $(ceph_test_rados_api_stat_LDADD) $(LIBS)
+test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT): \
+ test/librados/$(am__dirstamp) \
+ test/librados/$(DEPDIR)/$(am__dirstamp)
+test/librados/ceph_test_rados_api_tier-test.$(OBJEXT): \
+ test/librados/$(am__dirstamp) \
+ test/librados/$(DEPDIR)/$(am__dirstamp)
+osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT): osd/$(am__dirstamp) \
+ osd/$(DEPDIR)/$(am__dirstamp)
+ceph_test_rados_api_tier$(EXEEXT): $(ceph_test_rados_api_tier_OBJECTS) $(ceph_test_rados_api_tier_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_tier_DEPENDENCIES)
+ @rm -f ceph_test_rados_api_tier$(EXEEXT)
+ $(AM_V_CXXLD)$(ceph_test_rados_api_tier_LINK) $(ceph_test_rados_api_tier_OBJECTS) $(ceph_test_rados_api_tier_LDADD) $(LIBS)
test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT): \
test/librados/$(am__dirstamp) \
test/librados/$(DEPDIR)/$(am__dirstamp)
@@ -6580,6 +6965,18 @@ rgw/rgw_swift.$(OBJEXT): rgw/$(am__dirstamp) \
rgw/$(DEPDIR)/$(am__dirstamp)
rgw/rgw_swift_auth.$(OBJEXT): rgw/$(am__dirstamp) \
rgw/$(DEPDIR)/$(am__dirstamp)
+rgw/rgw_loadgen.$(OBJEXT): rgw/$(am__dirstamp) \
+ rgw/$(DEPDIR)/$(am__dirstamp)
+rgw/rgw_civetweb.$(OBJEXT): rgw/$(am__dirstamp) \
+ rgw/$(DEPDIR)/$(am__dirstamp)
+civetweb/src/$(am__dirstamp):
+ @$(MKDIR_P) civetweb/src
+ @: > civetweb/src/$(am__dirstamp)
+civetweb/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) civetweb/src/$(DEPDIR)
+ @: > civetweb/src/$(DEPDIR)/$(am__dirstamp)
+civetweb/src/radosgw-civetweb.$(OBJEXT): civetweb/src/$(am__dirstamp) \
+ civetweb/src/$(DEPDIR)/$(am__dirstamp)
rgw/rgw_main.$(OBJEXT): rgw/$(am__dirstamp) \
rgw/$(DEPDIR)/$(am__dirstamp)
radosgw$(EXEEXT): $(radosgw_OBJECTS) $(radosgw_DEPENDENCIES) $(EXTRA_radosgw_DEPENDENCIES)
@@ -6686,6 +7083,8 @@ common/test_build_libcommon-snap_types.$(OBJEXT): \
common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-str_list.$(OBJEXT): \
common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp)
+common/test_build_libcommon-str_map.$(OBJEXT): common/$(am__dirstamp) \
+ common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-errno.$(OBJEXT): common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-RefCountedObj.$(OBJEXT): \
@@ -6762,6 +7161,8 @@ common/test_build_libcommon-hobject.$(OBJEXT): common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-bloom_filter.$(OBJEXT): \
common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp)
+common/test_build_libcommon-linux_version.$(OBJEXT): \
+ common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp)
common/test_build_libcommon-secret.$(OBJEXT): common/$(am__dirstamp) \
common/$(DEPDIR)/$(am__dirstamp)
mon/test_build_libcommon-MonCap.$(OBJEXT): mon/$(am__dirstamp) \
@@ -6774,6 +7175,8 @@ osd/test_build_libcommon-OSDMap.$(OBJEXT): osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
osd/test_build_libcommon-osd_types.$(OBJEXT): osd/$(am__dirstamp) \
osd/$(DEPDIR)/$(am__dirstamp)
+osd/test_build_libcommon-HitSet.$(OBJEXT): osd/$(am__dirstamp) \
+ osd/$(DEPDIR)/$(am__dirstamp)
mds/test_build_libcommon-MDSMap.$(OBJEXT): mds/$(am__dirstamp) \
mds/$(DEPDIR)/$(am__dirstamp)
mds/test_build_libcommon-inode_backtrace.$(OBJEXT): \
@@ -6877,11 +7280,11 @@ test/unittest_admin_socket-admin_socket.$(OBJEXT): \
unittest_admin_socket$(EXEEXT): $(unittest_admin_socket_OBJECTS) $(unittest_admin_socket_DEPENDENCIES) $(EXTRA_unittest_admin_socket_DEPENDENCIES)
@rm -f unittest_admin_socket$(EXEEXT)
$(AM_V_CXXLD)$(unittest_admin_socket_LINK) $(unittest_admin_socket_OBJECTS) $(unittest_admin_socket_LDADD) $(LIBS)
-test/test_arch.$(OBJEXT): test/$(am__dirstamp) \
+test/unittest_arch-test_arch.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_arch$(EXEEXT): $(unittest_arch_OBJECTS) $(unittest_arch_DEPENDENCIES) $(EXTRA_unittest_arch_DEPENDENCIES)
@rm -f unittest_arch$(EXEEXT)
- $(AM_V_CCLD)$(LINK) $(unittest_arch_OBJECTS) $(unittest_arch_LDADD) $(LIBS)
+ $(AM_V_CXXLD)$(unittest_arch_LINK) $(unittest_arch_OBJECTS) $(unittest_arch_LDADD) $(LIBS)
test/unittest_base64-base64.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_base64$(EXEEXT): $(unittest_base64_OBJECTS) $(unittest_base64_DEPENDENCIES) $(EXTRA_unittest_base64_DEPENDENCIES)
@@ -6919,6 +7322,12 @@ test/filestore/unittest_chain_xattr-chain_xattr.$(OBJEXT): \
unittest_chain_xattr$(EXEEXT): $(unittest_chain_xattr_OBJECTS) $(unittest_chain_xattr_DEPENDENCIES) $(EXTRA_unittest_chain_xattr_DEPENDENCIES)
@rm -f unittest_chain_xattr$(EXEEXT)
$(AM_V_CXXLD)$(unittest_chain_xattr_LINK) $(unittest_chain_xattr_OBJECTS) $(unittest_chain_xattr_LDADD) $(LIBS)
+test/common/unittest_config-test_config.$(OBJEXT): \
+ test/common/$(am__dirstamp) \
+ test/common/$(DEPDIR)/$(am__dirstamp)
+unittest_config$(EXEEXT): $(unittest_config_OBJECTS) $(unittest_config_DEPENDENCIES) $(EXTRA_unittest_config_DEPENDENCIES)
+ @rm -f unittest_config$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_config_LINK) $(unittest_config_OBJECTS) $(unittest_config_LDADD) $(LIBS)
test/unittest_confutils-confutils.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_confutils$(EXEEXT): $(unittest_confutils_OBJECTS) $(unittest_confutils_DEPENDENCIES) $(EXTRA_unittest_confutils_DEPENDENCIES)
@@ -6930,6 +7339,29 @@ test/common/unittest_crc32c-test_crc32c.$(OBJEXT): \
unittest_crc32c$(EXEEXT): $(unittest_crc32c_OBJECTS) $(unittest_crc32c_DEPENDENCIES) $(EXTRA_unittest_crc32c_DEPENDENCIES)
@rm -f unittest_crc32c$(EXEEXT)
$(AM_V_CXXLD)$(unittest_crc32c_LINK) $(unittest_crc32c_OBJECTS) $(unittest_crc32c_LDADD) $(LIBS)
+test/crush/$(am__dirstamp):
+ @$(MKDIR_P) test/crush
+ @: > test/crush/$(am__dirstamp)
+test/crush/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) test/crush/$(DEPDIR)
+ @: > test/crush/$(DEPDIR)/$(am__dirstamp)
+test/crush/unittest_crush_indep-indep.$(OBJEXT): \
+ test/crush/$(am__dirstamp) \
+ test/crush/$(DEPDIR)/$(am__dirstamp)
+unittest_crush_indep$(EXEEXT): $(unittest_crush_indep_OBJECTS) $(unittest_crush_indep_DEPENDENCIES) $(EXTRA_unittest_crush_indep_DEPENDENCIES)
+ @rm -f unittest_crush_indep$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_crush_indep_LINK) $(unittest_crush_indep_OBJECTS) $(unittest_crush_indep_LDADD) $(LIBS)
+test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT): \
+ test/crush/$(am__dirstamp) \
+ test/crush/$(DEPDIR)/$(am__dirstamp)
+unittest_crush_wrapper$(EXEEXT): $(unittest_crush_wrapper_OBJECTS) $(unittest_crush_wrapper_DEPENDENCIES) $(EXTRA_unittest_crush_wrapper_DEPENDENCIES)
+ @rm -f unittest_crush_wrapper$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_crush_wrapper_LINK) $(unittest_crush_wrapper_OBJECTS) $(unittest_crush_wrapper_LDADD) $(LIBS)
+test/unittest_crushwrapper-test_crushwrapper.$(OBJEXT): \
+ test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp)
+unittest_crushwrapper$(EXEEXT): $(unittest_crushwrapper_OBJECTS) $(unittest_crushwrapper_DEPENDENCIES) $(EXTRA_unittest_crushwrapper_DEPENDENCIES)
+ @rm -f unittest_crushwrapper$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_crushwrapper_LINK) $(unittest_crushwrapper_OBJECTS) $(unittest_crushwrapper_LDADD) $(LIBS)
test/unittest_crypto-crypto.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_crypto$(EXEEXT): $(unittest_crypto_OBJECTS) $(unittest_crypto_DEPENDENCIES) $(EXTRA_unittest_crypto_DEPENDENCIES)
@@ -7019,6 +7451,11 @@ test/unittest_heartbeatmap-heartbeat_map.$(OBJEXT): \
unittest_heartbeatmap$(EXEEXT): $(unittest_heartbeatmap_OBJECTS) $(unittest_heartbeatmap_DEPENDENCIES) $(EXTRA_unittest_heartbeatmap_DEPENDENCIES)
@rm -f unittest_heartbeatmap$(EXEEXT)
$(AM_V_CXXLD)$(unittest_heartbeatmap_LINK) $(unittest_heartbeatmap_OBJECTS) $(unittest_heartbeatmap_LDADD) $(LIBS)
+test/osd/unittest_hitset-hitset.$(OBJEXT): test/osd/$(am__dirstamp) \
+ test/osd/$(DEPDIR)/$(am__dirstamp)
+unittest_hitset$(EXEEXT): $(unittest_hitset_OBJECTS) $(unittest_hitset_DEPENDENCIES) $(EXTRA_unittest_hitset_DEPENDENCIES)
+ @rm -f unittest_hitset$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_hitset_LINK) $(unittest_hitset_OBJECTS) $(unittest_hitset_LDADD) $(LIBS)
test/unittest_ipaddr-test_ipaddr.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_ipaddr$(EXEEXT): $(unittest_ipaddr_OBJECTS) $(unittest_ipaddr_DEPENDENCIES) $(EXTRA_unittest_ipaddr_DEPENDENCIES)
@@ -7061,6 +7498,11 @@ test/mon/unittest_mon_moncap-moncap.$(OBJEXT): \
unittest_mon_moncap$(EXEEXT): $(unittest_mon_moncap_OBJECTS) $(unittest_mon_moncap_DEPENDENCIES) $(EXTRA_unittest_mon_moncap_DEPENDENCIES)
@rm -f unittest_mon_moncap$(EXEEXT)
$(AM_V_CXXLD)$(unittest_mon_moncap_LINK) $(unittest_mon_moncap_OBJECTS) $(unittest_mon_moncap_LDADD) $(LIBS)
+test/on_exit.$(OBJEXT): test/$(am__dirstamp) \
+ test/$(DEPDIR)/$(am__dirstamp)
+unittest_on_exit$(EXEEXT): $(unittest_on_exit_OBJECTS) $(unittest_on_exit_DEPENDENCIES) $(EXTRA_unittest_on_exit_DEPENDENCIES)
+ @rm -f unittest_on_exit$(EXEEXT)
+ $(AM_V_CXXLD)$(CXXLINK) $(unittest_on_exit_OBJECTS) $(unittest_on_exit_LDADD) $(LIBS)
test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT): \
test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
unittest_osd_osdcap$(EXEEXT): $(unittest_osd_osdcap_OBJECTS) $(unittest_osd_osdcap_DEPENDENCIES) $(EXTRA_unittest_osd_osdcap_DEPENDENCIES)
@@ -7071,6 +7513,11 @@ test/unittest_osd_types-test_osd_types.$(OBJEXT): \
unittest_osd_types$(EXEEXT): $(unittest_osd_types_OBJECTS) $(unittest_osd_types_DEPENDENCIES) $(EXTRA_unittest_osd_types_DEPENDENCIES)
@rm -f unittest_osd_types$(EXEEXT)
$(AM_V_CXXLD)$(unittest_osd_types_LINK) $(unittest_osd_types_OBJECTS) $(unittest_osd_types_LDADD) $(LIBS)
+test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT): \
+ test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp)
+unittest_osdmap$(EXEEXT): $(unittest_osdmap_OBJECTS) $(unittest_osdmap_DEPENDENCIES) $(EXTRA_unittest_osdmap_DEPENDENCIES)
+ @rm -f unittest_osdmap$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_osdmap_LINK) $(unittest_osdmap_OBJECTS) $(unittest_osdmap_LDADD) $(LIBS)
test/unittest_perf_counters-perf_counters.$(OBJEXT): \
test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp)
unittest_perf_counters$(EXEEXT): $(unittest_perf_counters_OBJECTS) $(unittest_perf_counters_DEPENDENCIES) $(EXTRA_unittest_perf_counters_DEPENDENCIES)
@@ -7118,6 +7565,12 @@ test/unittest_str_list-test_str_list.$(OBJEXT): test/$(am__dirstamp) \
unittest_str_list$(EXEEXT): $(unittest_str_list_OBJECTS) $(unittest_str_list_DEPENDENCIES) $(EXTRA_unittest_str_list_DEPENDENCIES)
@rm -f unittest_str_list$(EXEEXT)
$(AM_V_CXXLD)$(unittest_str_list_LINK) $(unittest_str_list_OBJECTS) $(unittest_str_list_LDADD) $(LIBS)
+test/common/unittest_str_map-test_str_map.$(OBJEXT): \
+ test/common/$(am__dirstamp) \
+ test/common/$(DEPDIR)/$(am__dirstamp)
+unittest_str_map$(EXEEXT): $(unittest_str_map_OBJECTS) $(unittest_str_map_DEPENDENCIES) $(EXTRA_unittest_str_map_DEPENDENCIES)
+ @rm -f unittest_str_map$(EXEEXT)
+ $(AM_V_CXXLD)$(unittest_str_map_LINK) $(unittest_str_map_OBJECTS) $(unittest_str_map_LDADD) $(LIBS)
test/unittest_striper-test_striper.$(OBJEXT): test/$(am__dirstamp) \
test/$(DEPDIR)/$(am__dirstamp)
unittest_striper$(EXEEXT): $(unittest_striper_OBJECTS) $(unittest_striper_DEPENDENCIES) $(EXTRA_unittest_striper_DEPENDENCIES)
@@ -7388,6 +7841,7 @@ mostlyclean-compile:
-rm -f auth/none/AuthNoneAuthorizeHandler.lo
-rm -f auth/unknown/AuthUnknownAuthorizeHandler.$(OBJEXT)
-rm -f auth/unknown/AuthUnknownAuthorizeHandler.lo
+ -rm -f civetweb/src/radosgw-civetweb.$(OBJEXT)
-rm -f client/Client.$(OBJEXT)
-rm -f client/Client.lo
-rm -f client/ClientSnapRealm.$(OBJEXT)
@@ -7445,6 +7899,11 @@ mostlyclean-compile:
-rm -f cls/statelog/cls_statelog.$(OBJEXT)
-rm -f cls/statelog/cls_statelog.lo
-rm -f cls/statelog/cls_statelog_client.$(OBJEXT)
+ -rm -f cls/user/cls_user.$(OBJEXT)
+ -rm -f cls/user/cls_user.lo
+ -rm -f cls/user/cls_user_client.$(OBJEXT)
+ -rm -f cls/user/cls_user_ops.$(OBJEXT)
+ -rm -f cls/user/cls_user_types.$(OBJEXT)
-rm -f cls/version/cls_version.$(OBJEXT)
-rm -f cls/version/cls_version.lo
-rm -f cls/version/cls_version_client.$(OBJEXT)
@@ -7563,6 +8022,8 @@ mostlyclean-compile:
-rm -f common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo
-rm -f common/libcommon_crc_la-sctp_crc32.$(OBJEXT)
-rm -f common/libcommon_crc_la-sctp_crc32.lo
+ -rm -f common/linux_version.$(OBJEXT)
+ -rm -f common/linux_version.lo
-rm -f common/lockdep.$(OBJEXT)
-rm -f common/lockdep.lo
-rm -f common/mime.$(OBJEXT)
@@ -7591,6 +8052,8 @@ mostlyclean-compile:
-rm -f common/snap_types.lo
-rm -f common/str_list.$(OBJEXT)
-rm -f common/str_list.lo
+ -rm -f common/str_map.$(OBJEXT)
+ -rm -f common/str_map.lo
-rm -f common/strtol.$(OBJEXT)
-rm -f common/strtol.lo
-rm -f common/test_build_libcommon-BackTrace.$(OBJEXT)
@@ -7643,6 +8106,7 @@ mostlyclean-compile:
-rm -f common/test_build_libcommon-hex.$(OBJEXT)
-rm -f common/test_build_libcommon-hobject.$(OBJEXT)
-rm -f common/test_build_libcommon-ipaddr.$(OBJEXT)
+ -rm -f common/test_build_libcommon-linux_version.$(OBJEXT)
-rm -f common/test_build_libcommon-lockdep.$(OBJEXT)
-rm -f common/test_build_libcommon-mime.$(OBJEXT)
-rm -f common/test_build_libcommon-page.$(OBJEXT)
@@ -7656,6 +8120,7 @@ mostlyclean-compile:
-rm -f common/test_build_libcommon-simple_spin.$(OBJEXT)
-rm -f common/test_build_libcommon-snap_types.$(OBJEXT)
-rm -f common/test_build_libcommon-str_list.$(OBJEXT)
+ -rm -f common/test_build_libcommon-str_map.$(OBJEXT)
-rm -f common/test_build_libcommon-strtol.$(OBJEXT)
-rm -f common/test_build_libcommon-utf8.$(OBJEXT)
-rm -f common/test_build_libcommon-util.$(OBJEXT)
@@ -7697,21 +8162,19 @@ mostlyclean-compile:
-rm -f java/native/libcephfs_jni_la-libcephfs_jni.lo
-rm -f json_spirit/json_spirit_reader.$(OBJEXT)
-rm -f json_spirit/json_spirit_reader.lo
- -rm -f json_spirit/json_spirit_value.$(OBJEXT)
- -rm -f json_spirit/json_spirit_value.lo
-rm -f json_spirit/json_spirit_writer.$(OBJEXT)
-rm -f json_spirit/json_spirit_writer.lo
-rm -f key_value_store/cls_kvs.$(OBJEXT)
-rm -f key_value_store/cls_kvs.lo
-rm -f key_value_store/kv_flat_btree_async.$(OBJEXT)
- -rm -f librados/IoCtxImpl.$(OBJEXT)
- -rm -f librados/IoCtxImpl.lo
- -rm -f librados/RadosClient.$(OBJEXT)
- -rm -f librados/RadosClient.lo
- -rm -f librados/librados.$(OBJEXT)
- -rm -f librados/librados.lo
- -rm -f librados/snap_set_diff.$(OBJEXT)
- -rm -f librados/snap_set_diff.lo
+ -rm -f librados/librados_la-IoCtxImpl.$(OBJEXT)
+ -rm -f librados/librados_la-IoCtxImpl.lo
+ -rm -f librados/librados_la-RadosClient.$(OBJEXT)
+ -rm -f librados/librados_la-RadosClient.lo
+ -rm -f librados/librados_la-librados.$(OBJEXT)
+ -rm -f librados/librados_la-librados.lo
+ -rm -f librados/librados_la-snap_set_diff.$(OBJEXT)
+ -rm -f librados/librados_la-snap_set_diff.lo
-rm -f librados/test_build_librados-IoCtxImpl.$(OBJEXT)
-rm -f librados/test_build_librados-RadosClient.$(OBJEXT)
-rm -f librados/test_build_librados-librados.$(OBJEXT)
@@ -7870,16 +8333,22 @@ mostlyclean-compile:
-rm -f os/FlatIndex.lo
-rm -f os/GenericFileStoreBackend.$(OBJEXT)
-rm -f os/GenericFileStoreBackend.lo
+ -rm -f os/GenericObjectMap.$(OBJEXT)
+ -rm -f os/GenericObjectMap.lo
-rm -f os/HashIndex.$(OBJEXT)
-rm -f os/HashIndex.lo
-rm -f os/IndexManager.$(OBJEXT)
-rm -f os/IndexManager.lo
-rm -f os/JournalingObjectStore.$(OBJEXT)
-rm -f os/JournalingObjectStore.lo
+ -rm -f os/KeyValueStore.$(OBJEXT)
+ -rm -f os/KeyValueStore.lo
-rm -f os/LFNIndex.$(OBJEXT)
-rm -f os/LFNIndex.lo
-rm -f os/LevelDBStore.$(OBJEXT)
-rm -f os/LevelDBStore.lo
+ -rm -f os/MemStore.$(OBJEXT)
+ -rm -f os/MemStore.lo
-rm -f os/ObjectStore.$(OBJEXT)
-rm -f os/ObjectStore.lo
-rm -f os/WBThrottle.$(OBJEXT)
@@ -7916,6 +8385,8 @@ mostlyclean-compile:
-rm -f osd/ErasureCodePluginJerasure/reed_sol.$(OBJEXT)
-rm -f osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT)
-rm -f osd/ErasureCodePluginJerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT)
+ -rm -f osd/HitSet.$(OBJEXT)
+ -rm -f osd/HitSet.lo
-rm -f osd/OSD.$(OBJEXT)
-rm -f osd/OSD.lo
-rm -f osd/OSDCap.$(OBJEXT)
@@ -7926,6 +8397,8 @@ mostlyclean-compile:
-rm -f osd/OpRequest.lo
-rm -f osd/PG.$(OBJEXT)
-rm -f osd/PG.lo
+ -rm -f osd/PGBackend.$(OBJEXT)
+ -rm -f osd/PGBackend.lo
-rm -f osd/PGLog.$(OBJEXT)
-rm -f osd/PGLog.lo
-rm -f osd/ReplicatedBackend.$(OBJEXT)
@@ -7936,8 +8409,10 @@ mostlyclean-compile:
-rm -f osd/SnapMapper.lo
-rm -f osd/Watch.$(OBJEXT)
-rm -f osd/Watch.lo
+ -rm -f osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT)
-rm -f osd/osd_types.$(OBJEXT)
-rm -f osd/osd_types.lo
+ -rm -f osd/test_build_libcommon-HitSet.$(OBJEXT)
-rm -f osd/test_build_libcommon-OSDMap.$(OBJEXT)
-rm -f osd/test_build_libcommon-osd_types.$(OBJEXT)
-rm -f osdc/Filer.$(OBJEXT)
@@ -8036,11 +8511,13 @@ mostlyclean-compile:
-rm -f rgw/librgw_la-rgw_xml.$(OBJEXT)
-rm -f rgw/librgw_la-rgw_xml.lo
-rm -f rgw/rgw_admin.$(OBJEXT)
+ -rm -f rgw/rgw_civetweb.$(OBJEXT)
-rm -f rgw/rgw_common.$(OBJEXT)
-rm -f rgw/rgw_env.$(OBJEXT)
-rm -f rgw/rgw_http_client.$(OBJEXT)
-rm -f rgw/rgw_json_enc.$(OBJEXT)
-rm -f rgw/rgw_jsonparser.$(OBJEXT)
+ -rm -f rgw/rgw_loadgen.$(OBJEXT)
-rm -f rgw/rgw_main.$(OBJEXT)
-rm -f rgw/rgw_multiparser.$(OBJEXT)
-rm -f rgw/rgw_replica_log.$(OBJEXT)
@@ -8131,11 +8608,15 @@ mostlyclean-compile:
-rm -f test/common/ObjectContents.$(OBJEXT)
-rm -f test/common/get_command_descriptions.$(OBJEXT)
-rm -f test/common/unittest_bloom_filter-test_bloom_filter.$(OBJEXT)
+ -rm -f test/common/unittest_config-test_config.$(OBJEXT)
-rm -f test/common/unittest_crc32c-test_crc32c.$(OBJEXT)
-rm -f test/common/unittest_sharedptr_registry-test_sharedptr_registry.$(OBJEXT)
-rm -f test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.$(OBJEXT)
+ -rm -f test/common/unittest_str_map-test_str_map.$(OBJEXT)
-rm -f test/common/unittest_throttle-Throttle.$(OBJEXT)
-rm -f test/common/unittest_util-test_util.$(OBJEXT)
+ -rm -f test/crush/unittest_crush_indep-indep.$(OBJEXT)
+ -rm -f test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT)
-rm -f test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT)
-rm -f test/filestore/DeterministicOpSequence.$(OBJEXT)
-rm -f test/filestore/FileStoreDiff.$(OBJEXT)
@@ -8163,6 +8644,8 @@ mostlyclean-compile:
-rm -f test/librados/ceph_test_librbd-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_aio-test.$(OBJEXT)
+ -rm -f test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT)
+ -rm -f test/librados/ceph_test_rados_api_c_write_operations-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_cls-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT)
@@ -8181,6 +8664,8 @@ mostlyclean-compile:
-rm -f test/librados/ceph_test_rados_api_snapshots-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_stat-test.$(OBJEXT)
+ -rm -f test/librados/ceph_test_rados_api_tier-test.$(OBJEXT)
+ -rm -f test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_watch_notify-test.$(OBJEXT)
-rm -f test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT)
-rm -f test/librados/ceph_test_stress_watch-test.$(OBJEXT)
@@ -8193,12 +8678,14 @@ mostlyclean-compile:
-rm -f test/mon/unittest_mon_moncap-moncap.$(OBJEXT)
-rm -f test/multi_stress_watch.$(OBJEXT)
-rm -f test/omap_bench.$(OBJEXT)
+ -rm -f test/on_exit.$(OBJEXT)
-rm -f test/os/unittest_flatindex-TestFlatIndex.$(OBJEXT)
-rm -f test/os/unittest_lfnindex-TestLFNIndex.$(OBJEXT)
-rm -f test/osd/Object.$(OBJEXT)
-rm -f test/osd/RadosModel.$(OBJEXT)
-rm -f test/osd/TestOpStat.$(OBJEXT)
-rm -f test/osd/TestRados.$(OBJEXT)
+ -rm -f test/osd/ceph_erasure_code_benchmark.$(OBJEXT)
-rm -f test/osd/libec_example_la-ErasureCodePluginExample.$(OBJEXT)
-rm -f test/osd/libec_example_la-ErasureCodePluginExample.lo
-rm -f test/osd/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.$(OBJEXT)
@@ -8213,7 +8700,9 @@ mostlyclean-compile:
-rm -f test/osd/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT)
-rm -f test/osd/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT)
-rm -f test/osd/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT)
+ -rm -f test/osd/unittest_hitset-hitset.$(OBJEXT)
-rm -f test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT)
+ -rm -f test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT)
-rm -f test/osd/unittest_pglog-TestPGLog.$(OBJEXT)
-rm -f test/osdc/FakeWriteback.$(OBJEXT)
-rm -f test/osdc/object_cacher_stress.$(OBJEXT)
@@ -8234,12 +8723,13 @@ mostlyclean-compile:
-rm -f test/system/systest_runnable.lo
-rm -f test/system/systest_settings.$(OBJEXT)
-rm -f test/system/systest_settings.lo
- -rm -f test/test_arch.$(OBJEXT)
-rm -f test/test_build_libcephfs-buildtest_skeleton.$(OBJEXT)
-rm -f test/test_build_libcommon-buildtest_skeleton.$(OBJEXT)
-rm -f test/test_build_librados-buildtest_skeleton.$(OBJEXT)
-rm -f test/test_build_librgw-buildtest_skeleton.$(OBJEXT)
+ -rm -f test/test_c_headers.$(OBJEXT)
-rm -f test/test_cfuse_cache_invalidate.$(OBJEXT)
+ -rm -f test/test_get_blkdev_size.$(OBJEXT)
-rm -f test/test_mutate.$(OBJEXT)
-rm -f test/test_rewrite_latency.$(OBJEXT)
-rm -f test/test_trans.$(OBJEXT)
@@ -8248,12 +8738,14 @@ mostlyclean-compile:
-rm -f test/testmsgr.$(OBJEXT)
-rm -f test/unittest_addrs-test_addrs.$(OBJEXT)
-rm -f test/unittest_admin_socket-admin_socket.$(OBJEXT)
+ -rm -f test/unittest_arch-test_arch.$(OBJEXT)
-rm -f test/unittest_base64-base64.$(OBJEXT)
-rm -f test/unittest_bufferlist-bufferlist.$(OBJEXT)
-rm -f test/unittest_ceph_argparse-ceph_argparse.$(OBJEXT)
-rm -f test/unittest_ceph_compatset-ceph_compatset.$(OBJEXT)
-rm -f test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT)
-rm -f test/unittest_confutils-confutils.$(OBJEXT)
+ -rm -f test/unittest_crushwrapper-test_crushwrapper.$(OBJEXT)
-rm -f test/unittest_crypto-crypto.$(OBJEXT)
-rm -f test/unittest_daemon_config-daemon_config.$(OBJEXT)
-rm -f test/unittest_encoding-encoding.$(OBJEXT)
@@ -8331,6 +8823,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at auth/cephx/$(DEPDIR)/CephxSessionHandler.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at auth/none/$(DEPDIR)/AuthNoneAuthorizeHandler.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at auth/unknown/$(DEPDIR)/AuthUnknownAuthorizeHandler.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at civetweb/src/$(DEPDIR)/radosgw-civetweb.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at client/$(DEPDIR)/Client.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at client/$(DEPDIR)/ClientSnapRealm.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at client/$(DEPDIR)/Dentry.Plo at am__quote@
@@ -8363,6 +8856,10 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at cls/rgw/$(DEPDIR)/cls_rgw_types.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at cls/statelog/$(DEPDIR)/cls_statelog.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at cls/statelog/$(DEPDIR)/cls_statelog_client.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at cls/user/$(DEPDIR)/cls_user.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at cls/user/$(DEPDIR)/cls_user_client.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at cls/user/$(DEPDIR)/cls_user_ops.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at cls/user/$(DEPDIR)/cls_user_types.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at cls/version/$(DEPDIR)/cls_version.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at cls/version/$(DEPDIR)/cls_version_client.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at cls/version/$(DEPDIR)/cls_version_types.Po at am__quote@
@@ -8423,6 +8920,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_asm.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_zero_asm.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/libcommon_crc_la-sctp_crc32.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/linux_version.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/lockdep.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/mime.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/obj_bencher.Po at am__quote@
@@ -8438,6 +8936,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/simple_spin.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/snap_types.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/str_list.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/str_map.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/strtol.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-BackTrace.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-Clock.Po at am__quote@
@@ -8489,6 +8988,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-hex.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-hobject.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-ipaddr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-linux_version.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-lockdep.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-mime.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-page.Po at am__quote@
@@ -8502,6 +9002,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-simple_spin.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-snap_types.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-str_list.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-str_map.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-strtol.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-utf8.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at common/$(DEPDIR)/test_build_libcommon-util.Po at am__quote@
@@ -8525,14 +9026,13 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at java/native/$(DEPDIR)/libcephfs_jni_la-JniConstants.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at java/native/$(DEPDIR)/libcephfs_jni_la-libcephfs_jni.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at json_spirit/$(DEPDIR)/json_spirit_reader.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at json_spirit/$(DEPDIR)/json_spirit_value.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at json_spirit/$(DEPDIR)/json_spirit_writer.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at key_value_store/$(DEPDIR)/cls_kvs.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at key_value_store/$(DEPDIR)/kv_flat_btree_async.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/IoCtxImpl.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/RadosClient.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/librados.Plo at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/snap_set_diff.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/librados_la-IoCtxImpl.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/librados_la-RadosClient.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/librados_la-librados.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/librados_la-snap_set_diff.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/test_build_librados-RadosClient.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at librados/$(DEPDIR)/test_build_librados-librados.Po at am__quote@
@@ -8618,11 +9118,14 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/FileStore.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/FlatIndex.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/GenericFileStoreBackend.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/GenericObjectMap.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/HashIndex.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/IndexManager.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/JournalingObjectStore.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/KeyValueStore.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/LFNIndex.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/LevelDBStore.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/MemStore.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/ObjectStore.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/WBThrottle.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at os/$(DEPDIR)/ZFSFileStoreBackend.Plo at am__quote@
@@ -8631,17 +9134,21 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/Ager.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ClassHandler.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ErasureCodePlugin.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/HitSet.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/OSD.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/OSDCap.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/OSDMap.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/OpRequest.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/PG.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/PGBackend.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/PGLog.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ReplicatedBackend.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ReplicatedPG.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/SnapMapper.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/Watch.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/osd_types.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-HitSet.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-OSDMap.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/$(DEPDIR)/test_build_libcommon-osd_types.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at osd/ErasureCodePluginJerasure/$(DEPDIR)/cauchy.Po at am__quote@
@@ -8712,11 +9219,13 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/librgw_la-rgw_user.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/librgw_la-rgw_xml.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_admin.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_civetweb.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_common.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_env.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_http_client.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_json_enc.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_jsonparser.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_loadgen.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_main.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_multiparser.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at rgw/$(DEPDIR)/rgw_replica_log.Po at am__quote@
@@ -8782,13 +9291,15 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/kv_store_bench.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/multi_stress_watch.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/omap_bench.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/on_exit.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/streamtest.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_arch.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_c_headers.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_cfuse_cache_invalidate.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_get_blkdev_size.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_mutate.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_rewrite_latency.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/test_trans.Po at am__quote@
@@ -8797,12 +9308,14 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/testmsgr.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_addrs-test_addrs.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_admin_socket-admin_socket.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_arch-test_arch.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_base64-base64.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_bufferlist-bufferlist.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_confutils-confutils.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_crypto-crypto.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_daemon_config-daemon_config.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/$(DEPDIR)/unittest_encoding-encoding.Po at am__quote@
@@ -8853,11 +9366,15 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/ObjectContents.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/get_command_descriptions.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_config-test_config.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_throttle-Throttle.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/common/$(DEPDIR)/unittest_util-test_util.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/DeterministicOpSequence.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/filestore/$(DEPDIR)/FileStoreDiff.Po at am__quote@
@@ -8884,6 +9401,8 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_librbd-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_aio-test.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cls-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Po at am__quote@
@@ -8902,6 +9421,8 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_stat-test.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-test.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/librados/$(DEPDIR)/ceph_test_stress_watch-test.Po at am__quote@
@@ -8918,6 +9439,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/RadosModel.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/TestOpStat.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/TestRados.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/ceph_erasure_code_benchmark.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo at am__quote@
@@ -8927,7 +9449,9 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_hitset-hitset.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osdc/$(DEPDIR)/FakeWriteback.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at test/osdc/$(DEPDIR)/object_cacher_stress.Po at am__quote@
@@ -9098,6 +9622,20 @@ test/librbd/ceph_test_librbd_fsx-fsx.obj: test/librbd/fsx.c
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) -c -o test/librbd/ceph_test_librbd_fsx-fsx.obj `if test -f 'test/librbd/fsx.c'; then $(CYGPATH_W) 'test/librbd/fsx.c'; else $(CYGPATH_W) '$(srcdir)/test/librbd/fsx.c'; fi`
+civetweb/src/radosgw-civetweb.o: civetweb/src/civetweb.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -MT civetweb/src/radosgw-civetweb.o -MD -MP -MF civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo -c -o civetweb/src/radosgw-civetweb.o `test -f 'civetweb/src/civetweb.c' || echo '$(srcdir)/'`civetweb/src/civetweb.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo civetweb/src/$(DEPDIR)/radosgw-civetweb.Po
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='civetweb/src/civetweb.c' object='civetweb/src/radosgw-civetweb.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -c -o civetweb/src/radosgw-civetweb.o `test -f 'civetweb/src/civetweb.c' || echo '$(srcdir)/'`civetweb/src/civetweb.c
+
+civetweb/src/radosgw-civetweb.obj: civetweb/src/civetweb.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -MT civetweb/src/radosgw-civetweb.obj -MD -MP -MF civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo -c -o civetweb/src/radosgw-civetweb.obj `if test -f 'civetweb/src/civetweb.c'; then $(CYGPATH_W) 'civetweb/src/civetweb.c'; else $(CYGPATH_W) '$(srcdir)/civetweb/src/civetweb.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo civetweb/src/$(DEPDIR)/radosgw-civetweb.Po
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='civetweb/src/civetweb.c' object='civetweb/src/radosgw-civetweb.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -c -o civetweb/src/radosgw-civetweb.obj `if test -f 'civetweb/src/civetweb.c'; then $(CYGPATH_W) 'civetweb/src/civetweb.c'; else $(CYGPATH_W) '$(srcdir)/civetweb/src/civetweb.c'; fi`
+
test_build_libcommon-ceph_ver.o: ceph_ver.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT test_build_libcommon-ceph_ver.o -MD -MP -MF $(DEPDIR)/test_build_libcommon-ceph_ver.Tpo -c -o test_build_libcommon-ceph_ver.o `test -f 'ceph_ver.c' || echo '$(srcdir)/'`ceph_ver.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/test_build_libcommon-ceph_ver.Tpo $(DEPDIR)/test_build_libcommon-ceph_ver.Po
@@ -9224,6 +9762,20 @@ common/test_build_libcommon-addr_parsing.obj: common/addr_parsing.c
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-addr_parsing.obj `if test -f 'common/addr_parsing.c'; then $(CYGPATH_W) 'common/addr_parsing.c'; else $(CYGPATH_W) '$(srcdir)/common/addr_parsing.c'; fi`
+common/test_build_libcommon-linux_version.o: common/linux_version.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-linux_version.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo -c -o common/test_build_libcommon-linux_version.o `test -f 'common/linux_version.c' || echo '$(srcdir)/'`common/linux_version.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo common/$(DEPDIR)/test_build_libcommon-linux_version.Po
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/linux_version.c' object='common/test_build_libcommon-linux_version.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-linux_version.o `test -f 'common/linux_version.c' || echo '$(srcdir)/'`common/linux_version.c
+
+common/test_build_libcommon-linux_version.obj: common/linux_version.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-linux_version.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo -c -o common/test_build_libcommon-linux_version.obj `if test -f 'common/linux_version.c'; then $(CYGPATH_W) 'common/linux_version.c'; else $(CYGPATH_W) '$(srcdir)/common/linux_version.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo common/$(DEPDIR)/test_build_libcommon-linux_version.Po
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/linux_version.c' object='common/test_build_libcommon-linux_version.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-linux_version.obj `if test -f 'common/linux_version.c'; then $(CYGPATH_W) 'common/linux_version.c'; else $(CYGPATH_W) '$(srcdir)/common/linux_version.c'; fi`
+
common/test_build_libcommon-secret.o: common/secret.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-secret.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-secret.Tpo -c -o common/test_build_libcommon-secret.o `test -f 'common/secret.c' || echo '$(srcdir)/'`common/secret.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-secret.Tpo common/$(DEPDIR)/test_build_libcommon-secret.Po
@@ -9346,6 +9898,34 @@ test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: tes
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo `test -f 'test/osd/ErasureCodePluginMissingEntryPoint.cc' || echo '$(srcdir)/'`test/osd/ErasureCodePluginMissingEntryPoint.cc
+librados/librados_la-librados.lo: librados/librados.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-librados.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-librados.Tpo -c -o librados/librados_la-librados.lo `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-librados.Tpo librados/$(DEPDIR)/librados_la-librados.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/librados.cc' object='librados/librados_la-librados.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-librados.lo `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc
+
+librados/librados_la-RadosClient.lo: librados/RadosClient.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-RadosClient.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-RadosClient.Tpo -c -o librados/librados_la-RadosClient.lo `test -f 'librados/RadosClient.cc' || echo '$(srcdir)/'`librados/RadosClient.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-RadosClient.Tpo librados/$(DEPDIR)/librados_la-RadosClient.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/RadosClient.cc' object='librados/librados_la-RadosClient.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-RadosClient.lo `test -f 'librados/RadosClient.cc' || echo '$(srcdir)/'`librados/RadosClient.cc
+
+librados/librados_la-IoCtxImpl.lo: librados/IoCtxImpl.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-IoCtxImpl.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-IoCtxImpl.Tpo -c -o librados/librados_la-IoCtxImpl.lo `test -f 'librados/IoCtxImpl.cc' || echo '$(srcdir)/'`librados/IoCtxImpl.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-IoCtxImpl.Tpo librados/$(DEPDIR)/librados_la-IoCtxImpl.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/IoCtxImpl.cc' object='librados/librados_la-IoCtxImpl.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-IoCtxImpl.lo `test -f 'librados/IoCtxImpl.cc' || echo '$(srcdir)/'`librados/IoCtxImpl.cc
+
+librados/librados_la-snap_set_diff.lo: librados/snap_set_diff.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-snap_set_diff.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-snap_set_diff.Tpo -c -o librados/librados_la-snap_set_diff.lo `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-snap_set_diff.Tpo librados/$(DEPDIR)/librados_la-snap_set_diff.Plo
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/snap_set_diff.cc' object='librados/librados_la-snap_set_diff.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-snap_set_diff.lo `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc
+
rgw/librgw_la-librgw.lo: rgw/librgw.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-librgw.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-librgw.Tpo -c -o rgw/librgw_la-librgw.lo `test -f 'rgw/librgw.cc' || echo '$(srcdir)/'`rgw/librgw.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-librgw.Tpo rgw/$(DEPDIR)/librgw_la-librgw.Plo
@@ -10193,6 +10773,34 @@ test/librados/ceph_test_rados_api_aio-test.obj: test/librados/test.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_aio-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o: test/librados/c_write_operations.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o `test -f 'test/librados/c_write_operations.cc' || echo '$(srcdir)/ [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_write_operations.cc' object='test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o `test -f 'test/librados/c_write_operations.cc' || echo '$(srcdir)/'`test/librados/c_write_operations.cc
+
+test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj: test/librados/c_write_operations.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj `if test -f 'test/librados/c_write_operations.cc'; then $(CYGP [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_write_operations.cc' object='test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj `if test -f 'test/librados/c_write_operations.cc'; then $(CYGPATH_W) 'test/librados/c_write_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_write_operations.cc'; fi`
+
+test/librados/ceph_test_rados_api_c_write_operations-test.o: test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_c_write_operations-test.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+
+test/librados/ceph_test_rados_api_c_write_operations-test.obj: test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(src [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-test.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_c_write_operations-test.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+
test/librados/ceph_test_rados_api_cls-cls.o: test/librados/cls.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cls-cls.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo -c -o test/librados/ceph_test_rados_api_cls-cls.o `test -f 'test/librados/cls.cc' || echo '$(srcdir)/'`test/librados/cls.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po
@@ -10445,6 +11053,48 @@ test/librados/ceph_test_rados_api_stat-test.obj: test/librados/test.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_stat-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+test/librados/ceph_test_rados_api_tier-tier.o: test/librados/tier.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-tier.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo -c -o test/librados/ceph_test_rados_api_tier-tier.o `test -f 'test/librados/tier.cc' || echo '$(srcdir)/'`test/librados/tier.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/tier.cc' object='test/librados/ceph_test_rados_api_tier-tier.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-tier.o `test -f 'test/librados/tier.cc' || echo '$(srcdir)/'`test/librados/tier.cc
+
+test/librados/ceph_test_rados_api_tier-tier.obj: test/librados/tier.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-tier.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo -c -o test/librados/ceph_test_rados_api_tier-tier.obj `if test -f 'test/librados/tier.cc'; then $(CYGPATH_W) 'test/librados/tier.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/tier.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/tier.cc' object='test/librados/ceph_test_rados_api_tier-tier.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-tier.obj `if test -f 'test/librados/tier.cc'; then $(CYGPATH_W) 'test/librados/tier.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/tier.cc'; fi`
+
+test/librados/ceph_test_rados_api_tier-test.o: test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-test.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo -c -o test/librados/ceph_test_rados_api_tier-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_tier-test.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-test.o `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc
+
+test/librados/ceph_test_rados_api_tier-test.obj: test/librados/test.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-test.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo -c -o test/librados/ceph_test_rados_api_tier-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-test.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/ceph_test_rados_api_tier-test.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-test.obj `if test -f 'test/librados/test.cc'; then $(CYGPATH_W) 'test/librados/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/test.cc'; fi`
+
+osd/ceph_test_rados_api_tier-HitSet.o: osd/HitSet.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT osd/ceph_test_rados_api_tier-HitSet.o -MD -MP -MF osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo -c -o osd/ceph_test_rados_api_tier-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/ceph_test_rados_api_tier-HitSet.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o osd/ceph_test_rados_api_tier-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc
+
+osd/ceph_test_rados_api_tier-HitSet.obj: osd/HitSet.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT osd/ceph_test_rados_api_tier-HitSet.obj -MD -MP -MF osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo -c -o osd/ceph_test_rados_api_tier-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/ceph_test_rados_api_tier-HitSet.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o osd/ceph_test_rados_api_tier-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi`
+
test/librados/ceph_test_rados_api_watch_notify-watch_notify.o: test/librados/watch_notify.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_watch_notify-watch_notify.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Tpo -c -o test/librados/ceph_test_rados_api_watch_notify-watch_notify.o `test -f 'test/librados/watch_notify.cc' || echo '$(srcdir)/'`test/librados/watch_notify.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Po
@@ -11005,6 +11655,20 @@ common/test_build_libcommon-str_list.obj: common/str_list.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_list.obj `if test -f 'common/str_list.cc'; then $(CYGPATH_W) 'common/str_list.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_list.cc'; fi`
+common/test_build_libcommon-str_map.o: common/str_map.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-str_map.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-str_map.Tpo -c -o common/test_build_libcommon-str_map.o `test -f 'common/str_map.cc' || echo '$(srcdir)/'`common/str_map.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-str_map.Tpo common/$(DEPDIR)/test_build_libcommon-str_map.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/str_map.cc' object='common/test_build_libcommon-str_map.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_map.o `test -f 'common/str_map.cc' || echo '$(srcdir)/'`common/str_map.cc
+
+common/test_build_libcommon-str_map.obj: common/str_map.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-str_map.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-str_map.Tpo -c -o common/test_build_libcommon-str_map.obj `if test -f 'common/str_map.cc'; then $(CYGPATH_W) 'common/str_map.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_map.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-str_map.Tpo common/$(DEPDIR)/test_build_libcommon-str_map.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/str_map.cc' object='common/test_build_libcommon-str_map.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_map.obj `if test -f 'common/str_map.cc'; then $(CYGPATH_W) 'common/str_map.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_map.cc'; fi`
+
common/test_build_libcommon-errno.o: common/errno.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-errno.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-errno.Tpo -c -o common/test_build_libcommon-errno.o `test -f 'common/errno.cc' || echo '$(srcdir)/'`common/errno.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-errno.Tpo common/$(DEPDIR)/test_build_libcommon-errno.Po
@@ -11551,6 +12215,20 @@ osd/test_build_libcommon-osd_types.obj: osd/osd_types.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-osd_types.obj `if test -f 'osd/osd_types.cc'; then $(CYGPATH_W) 'osd/osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/osd/osd_types.cc'; fi`
+osd/test_build_libcommon-HitSet.o: osd/HitSet.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-HitSet.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo -c -o osd/test_build_libcommon-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo osd/$(DEPDIR)/test_build_libcommon-HitSet.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/test_build_libcommon-HitSet.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc
+
+osd/test_build_libcommon-HitSet.obj: osd/HitSet.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-HitSet.obj -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo -c -o osd/test_build_libcommon-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo osd/$(DEPDIR)/test_build_libcommon-HitSet.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/test_build_libcommon-HitSet.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi`
+
mds/test_build_libcommon-MDSMap.o: mds/MDSMap.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-MDSMap.o -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-MDSMap.Tpo -c -o mds/test_build_libcommon-MDSMap.o `test -f 'mds/MDSMap.cc' || echo '$(srcdir)/'`mds/MDSMap.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-MDSMap.Tpo mds/$(DEPDIR)/test_build_libcommon-MDSMap.Po
@@ -12167,6 +12845,20 @@ test/unittest_admin_socket-admin_socket.obj: test/admin_socket.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_admin_socket-admin_socket.obj `if test -f 'test/admin_socket.cc'; then $(CYGPATH_W) 'test/admin_socket.cc'; else $(CYGPATH_W) '$(srcdir)/test/admin_socket.cc'; fi`
+test/unittest_arch-test_arch.o: test/test_arch.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_arch-test_arch.o -MD -MP -MF test/$(DEPDIR)/unittest_arch-test_arch.Tpo -c -o test/unittest_arch-test_arch.o `test -f 'test/test_arch.cc' || echo '$(srcdir)/'`test/test_arch.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_arch-test_arch.Tpo test/$(DEPDIR)/unittest_arch-test_arch.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_arch.cc' object='test/unittest_arch-test_arch.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_arch-test_arch.o `test -f 'test/test_arch.cc' || echo '$(srcdir)/'`test/test_arch.cc
+
+test/unittest_arch-test_arch.obj: test/test_arch.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_arch-test_arch.obj -MD -MP -MF test/$(DEPDIR)/unittest_arch-test_arch.Tpo -c -o test/unittest_arch-test_arch.obj `if test -f 'test/test_arch.cc'; then $(CYGPATH_W) 'test/test_arch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_arch.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_arch-test_arch.Tpo test/$(DEPDIR)/unittest_arch-test_arch.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_arch.cc' object='test/unittest_arch-test_arch.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_arch-test_arch.obj `if test -f 'test/test_arch.cc'; then $(CYGPATH_W) 'test/test_arch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_arch.cc'; fi`
+
test/unittest_base64-base64.o: test/base64.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_base64_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_base64-base64.o -MD -MP -MF test/$(DEPDIR)/unittest_base64-base64.Tpo -c -o test/unittest_base64-base64.o `test -f 'test/base64.cc' || echo '$(srcdir)/'`test/base64.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_base64-base64.Tpo test/$(DEPDIR)/unittest_base64-base64.Po
@@ -12265,6 +12957,20 @@ test/filestore/unittest_chain_xattr-chain_xattr.obj: test/filestore/chain_xattr.
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/filestore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/filestore/chain_xattr.cc'; then $(CYGPATH_W) 'test/filestore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/filestore/chain_xattr.cc'; fi`
+test/common/unittest_config-test_config.o: test/common/test_config.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_config-test_config.o -MD -MP -MF test/common/$(DEPDIR)/unittest_config-test_config.Tpo -c -o test/common/unittest_config-test_config.o `test -f 'test/common/test_config.cc' || echo '$(srcdir)/'`test/common/test_config.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_config-test_config.Tpo test/common/$(DEPDIR)/unittest_config-test_config.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_config.cc' object='test/common/unittest_config-test_config.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_config-test_config.o `test -f 'test/common/test_config.cc' || echo '$(srcdir)/'`test/common/test_config.cc
+
+test/common/unittest_config-test_config.obj: test/common/test_config.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_config-test_config.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_config-test_config.Tpo -c -o test/common/unittest_config-test_config.obj `if test -f 'test/common/test_config.cc'; then $(CYGPATH_W) 'test/common/test_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_config.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_config-test_config.Tpo test/common/$(DEPDIR)/unittest_config-test_config.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_config.cc' object='test/common/unittest_config-test_config.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_config-test_config.obj `if test -f 'test/common/test_config.cc'; then $(CYGPATH_W) 'test/common/test_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_config.cc'; fi`
+
test/unittest_confutils-confutils.o: test/confutils.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_confutils-confutils.o -MD -MP -MF test/$(DEPDIR)/unittest_confutils-confutils.Tpo -c -o test/unittest_confutils-confutils.o `test -f 'test/confutils.cc' || echo '$(srcdir)/'`test/confutils.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_confutils-confutils.Tpo test/$(DEPDIR)/unittest_confutils-confutils.Po
@@ -12293,6 +12999,48 @@ test/common/unittest_crc32c-test_crc32c.obj: test/common/test_crc32c.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_crc32c-test_crc32c.obj `if test -f 'test/common/test_crc32c.cc'; then $(CYGPATH_W) 'test/common/test_crc32c.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_crc32c.cc'; fi`
+test/crush/unittest_crush_indep-indep.o: test/crush/indep.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_indep-indep.o -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo -c -o test/crush/unittest_crush_indep-indep.o `test -f 'test/crush/indep.cc' || echo '$(srcdir)/'`test/crush/indep.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/indep.cc' object='test/crush/unittest_crush_indep-indep.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_indep-indep.o `test -f 'test/crush/indep.cc' || echo '$(srcdir)/'`test/crush/indep.cc
+
+test/crush/unittest_crush_indep-indep.obj: test/crush/indep.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_indep-indep.obj -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo -c -o test/crush/unittest_crush_indep-indep.obj `if test -f 'test/crush/indep.cc'; then $(CYGPATH_W) 'test/crush/indep.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/indep.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/indep.cc' object='test/crush/unittest_crush_indep-indep.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_indep-indep.obj `if test -f 'test/crush/indep.cc'; then $(CYGPATH_W) 'test/crush/indep.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/indep.cc'; fi`
+
+test/crush/unittest_crush_wrapper-TestCrushWrapper.o: test/crush/TestCrushWrapper.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_wrapper-TestCrushWrapper.o -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.o `test -f 'test/crush/TestCrushWrapper.cc' || echo '$(srcdir)/'`test/crush/TestCrushWrapper.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/TestCrushWrapper.cc' object='test/crush/unittest_crush_wrapper-TestCrushWrapper.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.o `test -f 'test/crush/TestCrushWrapper.cc' || echo '$(srcdir)/'`test/crush/TestCrushWrapper.cc
+
+test/crush/unittest_crush_wrapper-TestCrushWrapper.obj: test/crush/TestCrushWrapper.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_wrapper-TestCrushWrapper.obj -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.obj `if test -f 'test/crush/TestCrushWrapper.cc'; then $(CYGPATH_W) 'test/crush/TestCrushWrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/Tes [...]
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/TestCrushWrapper.cc' object='test/crush/unittest_crush_wrapper-TestCrushWrapper.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.obj `if test -f 'test/crush/TestCrushWrapper.cc'; then $(CYGPATH_W) 'test/crush/TestCrushWrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/TestCrushWrapper.cc'; fi`
+
+test/unittest_crushwrapper-test_crushwrapper.o: test/test_crushwrapper.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crushwrapper-test_crushwrapper.o -MD -MP -MF test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo -c -o test/unittest_crushwrapper-test_crushwrapper.o `test -f 'test/test_crushwrapper.cc' || echo '$(srcdir)/'`test/test_crushwrapper.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_crushwrapper.cc' object='test/unittest_crushwrapper-test_crushwrapper.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crushwrapper-test_crushwrapper.o `test -f 'test/test_crushwrapper.cc' || echo '$(srcdir)/'`test/test_crushwrapper.cc
+
+test/unittest_crushwrapper-test_crushwrapper.obj: test/test_crushwrapper.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crushwrapper-test_crushwrapper.obj -MD -MP -MF test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo -c -o test/unittest_crushwrapper-test_crushwrapper.obj `if test -f 'test/test_crushwrapper.cc'; then $(CYGPATH_W) 'test/test_crushwrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_crushwrapper.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Tpo test/$(DEPDIR)/unittest_crushwrapper-test_crushwrapper.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_crushwrapper.cc' object='test/unittest_crushwrapper-test_crushwrapper.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crushwrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crushwrapper-test_crushwrapper.obj `if test -f 'test/test_crushwrapper.cc'; then $(CYGPATH_W) 'test/test_crushwrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_crushwrapper.cc'; fi`
+
test/unittest_crypto-crypto.o: test/crypto.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crypto-crypto.o -MD -MP -MF test/$(DEPDIR)/unittest_crypto-crypto.Tpo -c -o test/unittest_crypto-crypto.o `test -f 'test/crypto.cc' || echo '$(srcdir)/'`test/crypto.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crypto-crypto.Tpo test/$(DEPDIR)/unittest_crypto-crypto.Po
@@ -12503,6 +13251,20 @@ test/unittest_heartbeatmap-heartbeat_map.obj: test/heartbeat_map.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_heartbeatmap-heartbeat_map.obj `if test -f 'test/heartbeat_map.cc'; then $(CYGPATH_W) 'test/heartbeat_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/heartbeat_map.cc'; fi`
+test/osd/unittest_hitset-hitset.o: test/osd/hitset.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_hitset-hitset.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo -c -o test/osd/unittest_hitset-hitset.o `test -f 'test/osd/hitset.cc' || echo '$(srcdir)/'`test/osd/hitset.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo test/osd/$(DEPDIR)/unittest_hitset-hitset.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/hitset.cc' object='test/osd/unittest_hitset-hitset.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_hitset-hitset.o `test -f 'test/osd/hitset.cc' || echo '$(srcdir)/'`test/osd/hitset.cc
+
+test/osd/unittest_hitset-hitset.obj: test/osd/hitset.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_hitset-hitset.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo -c -o test/osd/unittest_hitset-hitset.obj `if test -f 'test/osd/hitset.cc'; then $(CYGPATH_W) 'test/osd/hitset.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/hitset.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo test/osd/$(DEPDIR)/unittest_hitset-hitset.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/hitset.cc' object='test/osd/unittest_hitset-hitset.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_hitset-hitset.obj `if test -f 'test/osd/hitset.cc'; then $(CYGPATH_W) 'test/osd/hitset.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/hitset.cc'; fi`
+
test/unittest_ipaddr-test_ipaddr.o: test/test_ipaddr.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ipaddr_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ipaddr-test_ipaddr.o -MD -MP -MF test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Tpo -c -o test/unittest_ipaddr-test_ipaddr.o `test -f 'test/test_ipaddr.cc' || echo '$(srcdir)/'`test/test_ipaddr.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Tpo test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Po
@@ -12643,6 +13405,20 @@ test/unittest_osd_types-test_osd_types.obj: test/test_osd_types.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_osd_types-test_osd_types.obj `if test -f 'test/test_osd_types.cc'; then $(CYGPATH_W) 'test/test_osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_osd_types.cc'; fi`
+test/osd/unittest_osdmap-TestOSDMap.o: test/osd/TestOSDMap.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osdmap-TestOSDMap.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo -c -o test/osd/unittest_osdmap-TestOSDMap.o `test -f 'test/osd/TestOSDMap.cc' || echo '$(srcdir)/'`test/osd/TestOSDMap.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestOSDMap.cc' object='test/osd/unittest_osdmap-TestOSDMap.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osdmap-TestOSDMap.o `test -f 'test/osd/TestOSDMap.cc' || echo '$(srcdir)/'`test/osd/TestOSDMap.cc
+
+test/osd/unittest_osdmap-TestOSDMap.obj: test/osd/TestOSDMap.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osdmap-TestOSDMap.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo -c -o test/osd/unittest_osdmap-TestOSDMap.obj `if test -f 'test/osd/TestOSDMap.cc'; then $(CYGPATH_W) 'test/osd/TestOSDMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestOSDMap.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestOSDMap.cc' object='test/osd/unittest_osdmap-TestOSDMap.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osdmap-TestOSDMap.obj `if test -f 'test/osd/TestOSDMap.cc'; then $(CYGPATH_W) 'test/osd/TestOSDMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestOSDMap.cc'; fi`
+
test/unittest_perf_counters-perf_counters.o: test/perf_counters.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_perf_counters_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_perf_counters-perf_counters.o -MD -MP -MF test/$(DEPDIR)/unittest_perf_counters-perf_counters.Tpo -c -o test/unittest_perf_counters-perf_counters.o `test -f 'test/perf_counters.cc' || echo '$(srcdir)/'`test/perf_counters.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_perf_counters-perf_counters.Tpo test/$(DEPDIR)/unittest_perf_counters-perf_counters.Po
@@ -12769,6 +13545,20 @@ test/unittest_str_list-test_str_list.obj: test/test_str_list.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_list_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_str_list-test_str_list.obj `if test -f 'test/test_str_list.cc'; then $(CYGPATH_W) 'test/test_str_list.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_str_list.cc'; fi`
+test/common/unittest_str_map-test_str_map.o: test/common/test_str_map.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_str_map-test_str_map.o -MD -MP -MF test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo -c -o test/common/unittest_str_map-test_str_map.o `test -f 'test/common/test_str_map.cc' || echo '$(srcdir)/'`test/common/test_str_map.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_str_map.cc' object='test/common/unittest_str_map-test_str_map.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_str_map-test_str_map.o `test -f 'test/common/test_str_map.cc' || echo '$(srcdir)/'`test/common/test_str_map.cc
+
+test/common/unittest_str_map-test_str_map.obj: test/common/test_str_map.cc
+ at am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_str_map-test_str_map.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo -c -o test/common/unittest_str_map-test_str_map.obj `if test -f 'test/common/test_str_map.cc'; then $(CYGPATH_W) 'test/common/test_str_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_str_map.cc'; fi`
+ at am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_str_map.cc' object='test/common/unittest_str_map-test_str_map.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCXX_FALSE@ $(AM_V_CXX at am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_str_map-test_str_map.obj `if test -f 'test/common/test_str_map.cc'; then $(CYGPATH_W) 'test/common/test_str_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_str_map.cc'; fi`
+
test/unittest_striper-test_striper.o: test/test_striper.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_striper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_striper-test_striper.o -MD -MP -MF test/$(DEPDIR)/unittest_striper-test_striper.Tpo -c -o test/unittest_striper-test_striper.o `test -f 'test/test_striper.cc' || echo '$(srcdir)/'`test/test_striper.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_striper-test_striper.Tpo test/$(DEPDIR)/unittest_striper-test_striper.Po
@@ -12910,6 +13700,7 @@ clean-libtool:
-rm -rf cls/replica_log/.libs cls/replica_log/_libs
-rm -rf cls/rgw/.libs cls/rgw/_libs
-rm -rf cls/statelog/.libs cls/statelog/_libs
+ -rm -rf cls/user/.libs cls/user/_libs
-rm -rf cls/version/.libs cls/version/_libs
-rm -rf common/.libs common/_libs
-rm -rf crush/.libs crush/_libs
@@ -13399,6 +14190,8 @@ distclean-generic:
-rm -f auth/none/$(am__dirstamp)
-rm -f auth/unknown/$(DEPDIR)/$(am__dirstamp)
-rm -f auth/unknown/$(am__dirstamp)
+ -rm -f civetweb/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f civetweb/src/$(am__dirstamp)
-rm -f client/$(DEPDIR)/$(am__dirstamp)
-rm -f client/$(am__dirstamp)
-rm -f cls/hello/$(DEPDIR)/$(am__dirstamp)
@@ -13417,6 +14210,8 @@ distclean-generic:
-rm -f cls/rgw/$(am__dirstamp)
-rm -f cls/statelog/$(DEPDIR)/$(am__dirstamp)
-rm -f cls/statelog/$(am__dirstamp)
+ -rm -f cls/user/$(DEPDIR)/$(am__dirstamp)
+ -rm -f cls/user/$(am__dirstamp)
-rm -f cls/version/$(DEPDIR)/$(am__dirstamp)
-rm -f cls/version/$(am__dirstamp)
-rm -f common/$(DEPDIR)/$(am__dirstamp)
@@ -13487,6 +14282,8 @@ distclean-generic:
-rm -f test/cls_version/$(am__dirstamp)
-rm -f test/common/$(DEPDIR)/$(am__dirstamp)
-rm -f test/common/$(am__dirstamp)
+ -rm -f test/crush/$(DEPDIR)/$(am__dirstamp)
+ -rm -f test/crush/$(am__dirstamp)
-rm -f test/encoding/$(DEPDIR)/$(am__dirstamp)
-rm -f test/encoding/$(am__dirstamp)
-rm -f test/filestore/$(DEPDIR)/$(am__dirstamp)
@@ -13526,7 +14323,7 @@ clean-am: clean-binPROGRAMS clean-checkPROGRAMS \
clean-sbinPROGRAMS clean-su_sbinPROGRAMS mostlyclean-am
distclean: distclean-recursive
- -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$(DEPDIR) librbd/$(DEPDIR) log/$(DEPDIR) mds [...]
+ -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$( [...]
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-hdr distclean-tags
@@ -13580,7 +14377,7 @@ install-ps-am:
installcheck-am:
maintainer-clean: maintainer-clean-recursive
- -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$(DEPDIR) librbd/$(DEPDIR) log/$(DEPDIR) mds [...]
+ -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$( [...]
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
@@ -13654,6 +14451,8 @@ uninstall-am: uninstall-bash_completionDATA uninstall-binPROGRAMS \
# python unit tests need to know where the scripts are located
export PYTHONPATH=$(top_srcdir)/src/pybind
+ at CLANG_FALSE@ AM_COMMON_CFLAGS += -rdynamic
+ at CLANG_FALSE@ AM_CXXFLAGS += -Wstrict-null-sentinel
#crush_includedir = $(includedir)/crush
#crush_include_DATA = \
@@ -13724,7 +14523,9 @@ ceph: ceph.in ./ceph_ver.h Makefile
# cleaning
clean-local:
- -rm *.so *.gcno *.gcda
+ rm -f *.so
+ find . -name '*.gcno' -o -name '*.gcda' -o -name '*.lcov' | xargs rm -f
+ rm -f ceph java/java/com/ceph/crush/Bucket.class
install-coverage:
@ENABLE_COVERAGE_TRUE@ -mkdir -p $(COV_DIR)/.libs
diff --git a/src/acconfig.h.in b/src/acconfig.h.in
index c2aad6b..43e8c30 100644
--- a/src/acconfig.h.in
+++ b/src/acconfig.h.in
@@ -3,6 +3,12 @@
/* fallocate(2) is supported */
#undef CEPH_HAVE_FALLOCATE
+/* F_SETPIPE_SZ is supported */
+#undef CEPH_HAVE_SETPIPE_SZ
+
+/* splice(2) is supported */
+#undef CEPH_HAVE_SPLICE
+
/* Define if darwin/osx */
#undef DARWIN
@@ -18,6 +24,9 @@
/* Define to 1 if you have the <arpa/inet.h> header file. */
#undef HAVE_ARPA_INET_H
+/* Define to 1 if you have the <arpa/nameser_compat.h> header file. */
+#undef HAVE_ARPA_NAMESER_COMPAT_H
+
/* have boost::random::discrete_distribution */
#undef HAVE_BOOST_RANDOM_DISCRETE_DISTRIBUTION
@@ -31,15 +40,33 @@
/* Define to 1 if you have the <dlfcn.h> header file. */
#undef HAVE_DLFCN_H
+/* Define to 1 if you have fdatasync. */
+#undef HAVE_FDATASYNC
+
/* linux/fiemap.h was found, fiemap ioctl will be used */
#undef HAVE_FIEMAP_H
+/* Define if the C complier supports __func__ */
+#undef HAVE_FUNC
+
/* Define to 1 if you have the `fuse_getgroups' function. */
#undef HAVE_FUSE_GETGROUPS
/* we have a recent yasm and are x86_64 */
#undef HAVE_GOOD_YASM_ELF64
+/* Define to 1 if the system has the type `int16_t'. */
+#undef HAVE_INT16_T
+
+/* Define to 1 if the system has the type `int32_t'. */
+#undef HAVE_INT32_T
+
+/* Define to 1 if the system has the type `int64_t'. */
+#undef HAVE_INT64_T
+
+/* Define to 1 if the system has the type `int8_t'. */
+#undef HAVE_INT8_T
+
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
@@ -49,6 +76,14 @@
/* Defined if you don't have atomic_ops */
#undef HAVE_LIBAIO
+/* Define to 1 if you have the `boost_program_options' library
+ (-lboost_program_options). */
+#undef HAVE_LIBBOOST_PROGRAM_OPTIONS
+
+/* Define to 1 if you have the `boost_program_options-mt' library
+ (-lboost_program_options-mt). */
+#undef HAVE_LIBBOOST_PROGRAM_OPTIONS_MT
+
/* Define to 1 if you have the `boost_system' library (-lboost_system). */
#undef HAVE_LIBBOOST_SYSTEM
@@ -81,6 +116,12 @@
/* Defined if you have libzfs enabled */
#undef HAVE_LIBZFS
+/* Define to 1 if you have the <linux/types.h> header file. */
+#undef HAVE_LINUX_TYPES_H
+
+/* Define to 1 if you have the <linux/version.h> header file. */
+#undef HAVE_LINUX_VERSION_H
+
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
@@ -93,15 +134,39 @@
/* Define to 1 if you have the <netinet/in.h> header file. */
#undef HAVE_NETINET_IN_H
+/* Define to 1 if you have the `pipe2' function. */
+#undef HAVE_PIPE2
+
+/* Define to 1 if you have the `posix_fadvise' function. */
+#undef HAVE_POSIX_FADVISE
+
+/* Define to 1 if you have the `posix_fallocate' function. */
+#undef HAVE_POSIX_FALLOCATE
+
/* Define to 1 if you have the `prctl' function. */
#undef HAVE_PRCTL
+/* Define if the C complier supports __PRETTY_FUNCTION__ */
+#undef HAVE_PRETTY_FUNC
+
/* Define if you have perftools profiler enabled */
#undef HAVE_PROFILER
/* Define if you have POSIX threads libraries and header files. */
#undef HAVE_PTHREAD
+/* Define if you have pthread_spin_init */
+#undef HAVE_PTHREAD_SPINLOCK
+
+/* define if the compiler supports static_cast<> */
+#undef HAVE_STATIC_CAST
+
+/* Define if you have struct stat.st_mtimespec.tv_nsec */
+#undef HAVE_STAT_ST_MTIMESPEC_TV_NSEC
+
+/* Define if you have struct stat.st_mtim.tv_nsec */
+#undef HAVE_STAT_ST_MTIM_TV_NSEC
+
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
@@ -173,12 +238,66 @@
/* Define to 1 if you have the <sys/xattr.h> header file. */
#undef HAVE_SYS_XATTR_H
+/* Define to 1 if the system has the type `uint16_t'. */
+#undef HAVE_UINT16_T
+
+/* Define to 1 if the system has the type `uint32_t'. */
+#undef HAVE_UINT32_T
+
+/* Define to 1 if the system has the type `uint64_t'. */
+#undef HAVE_UINT64_T
+
+/* Define to 1 if the system has the type `uint8_t'. */
+#undef HAVE_UINT8_T
+
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to 1 if you have the <utime.h> header file. */
#undef HAVE_UTIME_H
+/* Define to 1 if the system has the type `__be16'. */
+#undef HAVE___BE16
+
+/* Define to 1 if the system has the type `__be32'. */
+#undef HAVE___BE32
+
+/* Define to 1 if the system has the type `__be64'. */
+#undef HAVE___BE64
+
+/* Define to 1 if the system has the type `__le16'. */
+#undef HAVE___LE16
+
+/* Define to 1 if the system has the type `__le32'. */
+#undef HAVE___LE32
+
+/* Define to 1 if the system has the type `__le64'. */
+#undef HAVE___LE64
+
+/* Define to 1 if the system has the type `__s16'. */
+#undef HAVE___S16
+
+/* Define to 1 if the system has the type `__s32'. */
+#undef HAVE___S32
+
+/* Define to 1 if the system has the type `__s64'. */
+#undef HAVE___S64
+
+/* Define to 1 if the system has the type `__s8'. */
+#undef HAVE___S8
+
+/* Define to 1 if the system has the type `__u16'. */
+#undef HAVE___U16
+
+/* Define to 1 if the system has the type `__u32'. */
+#undef HAVE___U32
+
+/* Define to 1 if the system has the type `__u64'. */
+#undef HAVE___U64
+
+/* Define to 1 if the system has the type `__u8'. */
+#undef HAVE___U8
+
/* Define to the sub-directory in which libtool stores uninstalled libraries.
*/
#undef LT_OBJDIR
diff --git a/src/arch/intel.c b/src/arch/intel.c
index 8b2d2cc..9f2d3e4 100644
--- a/src/arch/intel.c
+++ b/src/arch/intel.c
@@ -2,7 +2,7 @@
/* flags we export */
int ceph_arch_intel_sse42 = 0;
-
+int ceph_arch_intel_sse2 = 0;
#ifdef __x86_64__
@@ -31,6 +31,10 @@ int ceph_arch_intel_probe(void)
if ((ecx & (1 << 20)) != 0) {
ceph_arch_intel_sse42 = 1;
}
+ if ((edx & (1 << 26)) != 0) {
+ ceph_arch_intel_sse2 = 1;
+ }
+
return 0;
}
diff --git a/src/arch/intel.h b/src/arch/intel.h
index aefb64e..7703bd7 100644
--- a/src/arch/intel.h
+++ b/src/arch/intel.h
@@ -6,7 +6,7 @@ extern "C" {
#endif
extern int ceph_arch_intel_sse42; /* true if we have sse 4.2 features */
-
+extern int ceph_arch_intel_sse2; /* true if we have sse 2 features */
extern int ceph_arch_intel_probe(void);
#ifdef __cplusplus
diff --git a/src/auth/AuthAuthorizeHandler.h b/src/auth/AuthAuthorizeHandler.h
index 10803d0..7b67876 100644
--- a/src/auth/AuthAuthorizeHandler.h
+++ b/src/auth/AuthAuthorizeHandler.h
@@ -41,12 +41,11 @@ struct AuthAuthorizeHandler {
class AuthAuthorizeHandlerRegistry {
Mutex m_lock;
map<int,AuthAuthorizeHandler*> m_authorizers;
- CephContext *cct;
AuthMethodList supported;
public:
AuthAuthorizeHandlerRegistry(CephContext *cct_, std::string methods)
- : m_lock("AuthAuthorizeHandlerRegistry::m_lock"), cct(cct_), supported(cct_, methods)
+ : m_lock("AuthAuthorizeHandlerRegistry::m_lock"), supported(cct_, methods)
{}
~AuthAuthorizeHandlerRegistry();
diff --git a/src/auth/AuthMethodList.cc b/src/auth/AuthMethodList.cc
index 50d58bf..0e507fb 100644
--- a/src/auth/AuthMethodList.cc
+++ b/src/auth/AuthMethodList.cc
@@ -12,6 +12,8 @@
*
*/
+#include <algorithm>
+
#include "common/Mutex.h"
#include "common/config.h"
#include "common/debug.h"
diff --git a/src/auth/KeyRing.cc b/src/auth/KeyRing.cc
index 5a8b228..96e56cd 100644
--- a/src/auth/KeyRing.cc
+++ b/src/auth/KeyRing.cc
@@ -16,6 +16,7 @@
#include <map>
#include <memory>
#include <sstream>
+#include <algorithm>
#include "auth/AuthMethodList.h"
#include "auth/Crypto.h"
diff --git a/src/auth/KeyRing.h b/src/auth/KeyRing.h
index 6b041ab..b69921c 100644
--- a/src/auth/KeyRing.h
+++ b/src/auth/KeyRing.h
@@ -25,8 +25,8 @@ class KeyRing : public KeyStore {
map<EntityName, EntityAuth> keys;
int set_modifier(const char *type, const char *val, EntityName& name, map<string, bufferlist>& caps);
- void decode_plaintext(bufferlist::iterator& bl);
public:
+ void decode_plaintext(bufferlist::iterator& bl);
/* Create a KeyRing from a Ceph context.
* We will use the configuration stored inside the context. */
int from_ceph_context(CephContext *cct);
diff --git a/src/ceph-crush-location b/src/ceph-crush-location
new file mode 100755
index 0000000..1f33f38
--- /dev/null
+++ b/src/ceph-crush-location
@@ -0,0 +1,87 @@
+#!/bin/sh
+#
+# Generate a CRUSH location for the given entity
+#
+# The CRUSH location consists of a list of key=value pairs, separated
+# by spaces, all on a single line. This describes where in CRUSH
+# hierarhcy this entity should be placed.
+#
+# Arguments:
+# --cluster <clustername> name of the cluster (see /etc/ceph/$cluster.conf)
+# --type <osd|mds|client> daemon/entity type
+# --id <id> id (osd number, mds name, client name)
+#
+
+# if we start up as ./ceph-crush-location, assume everything else is
+# in the current directory too.
+if [ `dirname $0` = "." ] && [ $PWD != "/usr/bin" ]; then
+ BINDIR=.
+ SBINDIR=.
+ LIBDIR=.
+ ETCDIR=.
+else
+ BINDIR=/usr/bin
+ SBINDIR=/usr/sbin
+ LIBDIR=/usr/lib/ceph
+ ETCDIR=/etc/ceph
+fi
+
+usage_exit() {
+ echo "usage: $0 [--cluster <cluster>] --id <id> --type <osd|mds|client>"
+ exit
+}
+
+cluster="ceph"
+type=""
+id=""
+while [ $# -ge 1 ]; do
+ case $1 in
+ --cluster | -C)
+ shift
+ cluster="$1"
+ shift
+ ;;
+ --id | -i)
+ shift
+ id="$1"
+ shift
+ ;;
+ --type | -t)
+ shift
+ type="$1"
+ shift
+ ;;
+ *)
+ echo "unrecognized option '$1'"
+ usage_exit
+ ;;
+ esac
+done
+
+if [ -z "$type" ]; then
+ echo "must specify entity type"
+ usage_exit
+fi
+
+if [ -z "$id" ]; then
+ echo "must specify id"
+ usage_exit
+fi
+
+# try a type-specific config, e.g. 'osd crush location'
+location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup ${type}_crush_location || :)"
+if [ -n "$location" ]; then
+ echo $location
+ exit 0
+fi
+
+# try a generic location
+location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup crush_location || :)"
+if [ -n "$location" ]; then
+ echo $location
+ exit 0
+fi
+
+# spit out something generic
+echo "host=$(hostname -s) root=default"
+
diff --git a/src/ceph-crush-location.in b/src/ceph-crush-location.in
new file mode 100755
index 0000000..b5043f2
--- /dev/null
+++ b/src/ceph-crush-location.in
@@ -0,0 +1,87 @@
+#!/bin/sh
+#
+# Generate a CRUSH location for the given entity
+#
+# The CRUSH location consists of a list of key=value pairs, separated
+# by spaces, all on a single line. This describes where in CRUSH
+# hierarhcy this entity should be placed.
+#
+# Arguments:
+# --cluster <clustername> name of the cluster (see /etc/ceph/$cluster.conf)
+# --type <osd|mds|client> daemon/entity type
+# --id <id> id (osd number, mds name, client name)
+#
+
+# if we start up as ./ceph-crush-location, assume everything else is
+# in the current directory too.
+if [ `dirname $0` = "." ] && [ $PWD != "/usr/bin" ]; then
+ BINDIR=.
+ SBINDIR=.
+ LIBDIR=.
+ ETCDIR=.
+else
+ BINDIR=@bindir@
+ SBINDIR=@prefix@/sbin
+ LIBDIR=@libdir@/ceph
+ ETCDIR=@sysconfdir@/ceph
+fi
+
+usage_exit() {
+ echo "usage: $0 [--cluster <cluster>] --id <id> --type <osd|mds|client>"
+ exit
+}
+
+cluster="ceph"
+type=""
+id=""
+while [ $# -ge 1 ]; do
+ case $1 in
+ --cluster | -C)
+ shift
+ cluster="$1"
+ shift
+ ;;
+ --id | -i)
+ shift
+ id="$1"
+ shift
+ ;;
+ --type | -t)
+ shift
+ type="$1"
+ shift
+ ;;
+ *)
+ echo "unrecognized option '$1'"
+ usage_exit
+ ;;
+ esac
+done
+
+if [ -z "$type" ]; then
+ echo "must specify entity type"
+ usage_exit
+fi
+
+if [ -z "$id" ]; then
+ echo "must specify id"
+ usage_exit
+fi
+
+# try a type-specific config, e.g. 'osd crush location'
+location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup ${type}_crush_location || :)"
+if [ -n "$location" ]; then
+ echo $location
+ exit 0
+fi
+
+# try a generic location
+location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup crush_location || :)"
+if [ -n "$location" ]; then
+ echo $location
+ exit 0
+fi
+
+# spit out something generic
+echo "host=$(hostname -s) root=default"
+
diff --git a/src/ceph-disk b/src/ceph-disk
index 64d944d..53e0afd 100755
--- a/src/ceph-disk
+++ b/src/ceph-disk
@@ -1,4 +1,20 @@
#!/usr/bin/env python
+#
+# Copyright (C) 2014 Inktank <info at inktank.com>
+# Copyright (C) 2014 Cloudwatt <libre.licensing at cloudwatt.com>
+#
+# Author: Loic Dachary <loic at dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library Public License for more details.
+#
import argparse
import errno
@@ -55,7 +71,6 @@ knew the GPT partition type.
"""
-
CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106'
@@ -97,8 +112,13 @@ INIT_SYSTEMS = [
'sysvinit',
'systemd',
'auto',
+ 'none',
]
+STATEDIR = '/var/lib/ceph'
+
+SYSCONFDIR = '/etc/ceph'
+
# Nuke the TERM variable to avoid confusing any subprocesses we call.
# For example, libreadline will print weird control sequences for some
# TERM values.
@@ -130,11 +150,6 @@ class filelock(object):
fcntl.lockf(self.fd, fcntl.LOCK_UN)
self.fd = None
-
-prepare_lock = filelock('/var/lib/ceph/tmp/ceph-disk.prepare.lock')
-activate_lock = filelock('/var/lib/ceph/tmp/ceph-disk.activate.lock')
-
-
###### exceptions ########
class Error(Exception):
@@ -174,8 +189,20 @@ class TooManyLinesError(Error):
class FilesystemTypeError(Error):
"""
Cannot discover filesystem type
+ """
+
+class CephDiskException(Exception):
+ """
+ A base exception for ceph-disk to provide custom (ad-hoc) messages that
+ will be caught and dealt with when main() is executed
"""
+ pass
+class ExecutableNotFound(CephDiskException):
+ """
+ Exception to report on executables not available in PATH
+ """
+ pass
####### utils
@@ -198,6 +225,84 @@ def maybe_mkdir(*a, **kw):
raise
+def which(executable):
+ """find the location of an executable"""
+ if 'PATH' in os.environ:
+ envpath = os.environ['PATH']
+ else:
+ envpath = os.defpath
+ PATH = envpath.split(os.pathsep)
+
+ locations = PATH + [
+ '/usr/local/bin',
+ '/bin',
+ '/usr/bin',
+ '/usr/local/sbin',
+ '/usr/sbin',
+ '/sbin',
+ ]
+
+ for location in locations:
+ executable_path = os.path.join(location, executable)
+ if os.path.exists(executable_path):
+ return executable_path
+
+
+def _get_command_executable(arguments):
+ """
+ Return the full path for an executable, raise if the executable is not
+ found. If the executable has already a full path do not perform any checks.
+ """
+ if arguments[0].startswith('/'): # an absolute path
+ return arguments
+ executable = which(arguments[0])
+ if not executable:
+ command_msg = 'Could not run command: %s' % ' '.join(arguments)
+ executable_msg = '%s not in path.' % arguments[0]
+ raise ExecutableNotFound('%s %s' % (executable_msg, command_msg))
+
+ # swap the old executable for the new one
+ arguments[0] = executable
+ return arguments
+
+
+def command(arguments, **kwargs):
+ """
+ Safely execute a ``subprocess.Popen`` call making sure that the
+ executable exists and raising a helpful error message
+ if it does not.
+
+ .. note:: This should be the prefered way of calling ``subprocess.Popen``
+ since it provides the caller with the safety net of making sure that
+ executables *will* be found and will error nicely otherwise.
+
+ This returns the output of the command and the return code of the
+ process in a tuple: (output, returncode).
+ """
+ arguments = _get_command_executable(arguments)
+
+ process = subprocess.Popen(
+ arguments,
+ stdout=subprocess.PIPE,
+ **kwargs)
+ out, _ = process.communicate()
+ return out, process.returncode
+
+
+def command_check_call(arguments):
+ """
+ Safely execute a ``subprocess.check_call`` call making sure that the
+ executable exists and raising a helpful error message if it does not.
+
+ .. note:: This should be the prefered way of calling
+ ``subprocess.check_call`` since it provides the caller with the safety net
+ of making sure that executables *will* be found and will error nicely
+ otherwise.
+ """
+ arguments = _get_command_executable(arguments)
+ return subprocess.check_call(arguments)
+
+
# a device "name" is something like
# sdb
# cciss!c0d1
@@ -253,6 +358,9 @@ def list_all_partitions():
"""
dev_part_list = {}
for name in os.listdir('/sys/block'):
+ # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
+ if re.match(r'^fd\d$', name):
+ continue
if not os.path.exists(os.path.join('/sys/block', name, 'device')):
continue
dev_part_list[name] = list_partitions(name)
@@ -432,7 +540,7 @@ def check_osd_id(osd_id):
Ensures osd id is numeric.
"""
if not re.match(r'^[0-9]+$', osd_id):
- raise Error('osd id is not numeric')
+ raise Error('osd id is not numeric', osd_id)
def allocate_osd_id(
@@ -451,7 +559,7 @@ def allocate_osd_id(
try:
osd_id = _check_output(
args=[
- '/usr/bin/ceph',
+ 'ceph',
'--cluster', cluster,
'--name', 'client.bootstrap-osd',
'--keyring', keyring,
@@ -460,7 +568,7 @@ def allocate_osd_id(
],
)
except subprocess.CalledProcessError as e:
- raise Error('ceph osd create failed', e)
+ raise Error('ceph osd create failed', e, e.output)
osd_id = must_be_one_line(osd_id)
check_osd_id(osd_id)
return osd_id
@@ -476,17 +584,10 @@ def get_osd_id(path):
return osd_id
-def _check_output(*args, **kwargs):
- process = subprocess.Popen(
- stdout=subprocess.PIPE,
- *args, **kwargs)
- out, _ = process.communicate()
- ret = process.wait()
+def _check_output(args=None, **kwargs):
+ out, ret = command(args, **kwargs)
if ret:
- cmd = kwargs.get("args")
- if cmd is None:
- cmd = args[0]
- #raise subprocess.CalledProcessError(ret, cmd, output=out)
+ cmd = args[0]
error = subprocess.CalledProcessError(ret, cmd)
error.output = out
raise error
@@ -502,9 +603,9 @@ def get_conf(cluster, variable):
:return: The variable value or None.
"""
try:
- process = subprocess.Popen(
- args=[
- '/usr/bin/ceph-conf',
+ out, ret = command(
+ [
+ 'ceph-conf',
'--cluster={cluster}'.format(
cluster=cluster,
),
@@ -512,19 +613,16 @@ def get_conf(cluster, variable):
'--lookup',
variable,
],
- stdout=subprocess.PIPE,
close_fds=True,
)
except OSError as e:
raise Error('error executing ceph-conf', e)
- (out, _err) = process.communicate()
- ret = process.wait()
if ret == 1:
# config entry not found
return None
elif ret != 0:
raise Error('getting variable from configuration failed')
- value = str(out).split('\n', 1)[0]
+ value = out.split('\n', 1)[0]
# don't differentiate between "var=" and no var set
if not value:
return None
@@ -567,7 +665,7 @@ def get_fsid(cluster):
:return: The fsid or raises Error.
"""
- fsid = get_conf(cluster=cluster, variable='fsid')
+ fsid = get_conf_with_default(cluster=cluster, variable='fsid')
if fsid is None:
raise Error('getting cluster uuid from configuration failed')
return fsid.lower()
@@ -622,7 +720,7 @@ def dmcrypt_map(
rawdev,
]
try:
- subprocess.check_call(args)
+ command_check_call(args)
return dev
except subprocess.CalledProcessError as e:
@@ -642,7 +740,7 @@ def dmcrypt_unmap(
]
try:
- subprocess.check_call(args)
+ command_check_call(args)
except subprocess.CalledProcessError as e:
raise Error('unable to unmap device', _uuid, e)
@@ -664,12 +762,12 @@ def mount(
# mount
path = tempfile.mkdtemp(
prefix='mnt.',
- dir='/var/lib/ceph/tmp',
+ dir=STATEDIR + '/tmp',
)
try:
LOG.debug('Mounting %s on %s with options %s', dev, path, options)
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'mount',
'-t', fstype,
'-o', options,
@@ -696,8 +794,8 @@ def unmount(
"""
try:
LOG.debug('Unmounting %s', path)
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'/bin/umount',
'--',
path,
@@ -738,7 +836,7 @@ def get_free_partition_index(dev):
lines = str(lines).splitlines(True)
# work around buggy libreadline(?) library in rhel/centos.
- idiot_prefix = '\x1b\x5b\x3f\x31\x30\x33\x34\x68';
+ idiot_prefix = '\x1b\x5b\x3f\x31\x30\x33\x34\x68'
if lines[0].startswith(idiot_prefix):
lines[0] = lines[0][8:]
@@ -777,16 +875,16 @@ def zap(dev):
dev_file.seek(-size, os.SEEK_END)
dev_file.write(size*'\0')
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'sgdisk',
'--zap-all',
'--clear',
'--mbrtogpt',
'--',
dev,
- ],
- )
+ ],
+ )
except subprocess.CalledProcessError as e:
raise Error(e)
@@ -834,8 +932,8 @@ def prepare_journal_dev(
try:
LOG.debug('Creating journal partition num %d size %d on %s', num, journal_size, journal)
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'sgdisk',
'--new={part}'.format(part=journal_part),
'--change-name={num}:ceph journal'.format(num=num),
@@ -847,25 +945,26 @@ def prepare_journal_dev(
num=num,
uuid=ptype,
),
+ '--mbrtogpt',
'--',
journal,
- ],
- )
+ ],
+ )
# try to make sure the kernel refreshes the table. note
# that if this gets ebusy, we are probably racing with
# udev because it already updated it.. ignore failure here.
LOG.debug('Calling partprobe on prepared device %s', journal)
- subprocess.call(
- args=[
+ command(
+ [
'partprobe',
journal,
],
)
# wait for udev event queue to clear
- subprocess.call(
- args=[
+ command(
+ [
'udevadm',
'settle',
],
@@ -931,7 +1030,7 @@ def prepare_journal(
raise Error('Journal is not a regular file', journal)
return prepare_journal_dev(data, journal, journal_size, journal_uuid, journal_dm_keypath)
- raise Error('Journal %s is neither a block device nor regular file', journal)
+ raise Error('Journal %s is neither a block device nor regular file' % journal)
def adjust_symlink(target, path):
@@ -966,7 +1065,12 @@ def prepare_dir(
journal_uuid,
journal_dmcrypt = None,
):
- LOG.debug('Preparing osd data dir %s', path)
+
+ if os.path.exists(os.path.join(path, 'magic')):
+ LOG.debug('Data dir %s already exists', path)
+ return
+ else:
+ LOG.debug('Preparing osd data dir %s', path)
if osd_uuid is None:
osd_uuid = str(uuid.uuid4())
@@ -985,12 +1089,14 @@ def prepare_dir(
write_one_line(path, 'ceph_fsid', cluster_uuid)
write_one_line(path, 'fsid', osd_uuid)
- write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC)
if journal_uuid is not None:
# i.e., journal is a tagged partition
write_one_line(path, 'journal_uuid', journal_uuid)
+ write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC)
+
+
def prepare_dev(
data,
journal,
@@ -1026,8 +1132,8 @@ def prepare_dev(
else:
LOG.debug('Creating osd partition on %s', data)
try:
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'sgdisk',
'--largest-new=1',
'--change-name=1:ceph data',
@@ -1037,10 +1143,10 @@ def prepare_dev(
'--typecode=1:%s' % ptype_tobe,
'--',
data,
- ],
- )
- subprocess.call(
- args=[
+ ],
+ )
+ command(
+ [
# wait for udev event queue to clear
'udevadm',
'settle',
@@ -1075,7 +1181,7 @@ def prepare_dev(
])
try:
LOG.debug('Creating %s fs on %s', fstype, dev)
- subprocess.check_call(args=args)
+ command_check_call(args)
except subprocess.CalledProcessError as e:
raise Error(e)
@@ -1102,14 +1208,14 @@ def prepare_dev(
if not is_partition(data):
try:
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'sgdisk',
'--typecode=1:%s' % ptype_osd,
'--',
data,
- ],
- )
+ ],
+ )
except subprocess.CalledProcessError as e:
raise Error(e)
@@ -1121,7 +1227,10 @@ def main_prepare(args):
try:
prepare_lock.acquire()
if not os.path.exists(args.data):
- raise Error('data path does not exist', args.data)
+ if args.data_dev:
+ raise Error('data path does not exist', args.data)
+ else:
+ os.mkdir(args.data)
# in use?
dmode = os.stat(args.data).st_mode
@@ -1255,8 +1364,8 @@ def main_prepare(args):
# that if this gets ebusy, we are probably racing with
# udev because it already updated it.. ignore failure here.
LOG.debug('Calling partprobe on prepared device %s', args.data)
- subprocess.call(
- args=[
+ command(
+ [
'partprobe',
args.data,
],
@@ -1283,9 +1392,9 @@ def mkfs(
keyring,
):
monmap = os.path.join(path, 'activate.monmap')
- subprocess.check_call(
- args=[
- '/usr/bin/ceph',
+ command_check_call(
+ [
+ 'ceph',
'--cluster', cluster,
'--name', 'client.bootstrap-osd',
'--keyring', keyring,
@@ -1293,9 +1402,9 @@ def mkfs(
],
)
- subprocess.check_call(
- args=[
- '/usr/bin/ceph-osd',
+ command_check_call(
+ [
+ 'ceph-osd',
'--cluster', cluster,
'--mkfs',
'--mkkey',
@@ -1319,9 +1428,9 @@ def auth_key(
):
try:
# try dumpling+ cap scheme
- subprocess.check_call(
- args=[
- '/usr/bin/ceph',
+ command_check_call(
+ [
+ 'ceph',
'--cluster', cluster,
'--name', 'client.bootstrap-osd',
'--keyring', keyring,
@@ -1332,11 +1441,11 @@ def auth_key(
],
)
except subprocess.CalledProcessError as err:
- if err.errno == errno.EACCES:
+ if err.returncode == errno.EACCES:
# try old cap scheme
- subprocess.check_call(
- args=[
- '/usr/bin/ceph',
+ command_check_call(
+ [
+ 'ceph',
'--cluster', cluster,
'--name', 'client.bootstrap-osd',
'--keyring', keyring,
@@ -1359,7 +1468,7 @@ def move_mount(
mount_options,
):
LOG.debug('Moving mount to final location...')
- parent = '/var/lib/ceph/osd'
+ parent = STATEDIR + '/osd'
osd_data = os.path.join(
parent,
'{cluster}-{osd_id}'.format(cluster=cluster, osd_id=osd_id),
@@ -1376,8 +1485,8 @@ def move_mount(
# /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
# this being 2013. Instead, mount the original device at the final
# location.
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'/bin/mount',
'-o',
mount_options,
@@ -1386,8 +1495,8 @@ def move_mount(
osd_data,
],
)
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'/bin/umount',
'-l', # lazy, in case someone else is peeking at the
# wrong moment
@@ -1403,14 +1512,14 @@ def start_daemon(
):
LOG.debug('Starting %s osd.%s...', cluster, osd_id)
- path = '/var/lib/ceph/osd/{cluster}-{osd_id}'.format(
+ path = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
cluster=cluster, osd_id=osd_id)
# upstart?
try:
if os.path.exists(os.path.join(path,'upstart')):
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
'/sbin/initctl',
# use emit, not start, because start would fail if the
# instance was already running
@@ -1430,8 +1539,8 @@ def start_daemon(
svc = '/usr/sbin/service'
else:
svc = '/sbin/service'
- subprocess.check_call(
- args=[
+ command_check_call(
+ [
svc,
'ceph',
'start',
@@ -1514,16 +1623,16 @@ def mount_activate(
other = False
src_dev = os.stat(path).st_dev
try:
- dst_dev = os.stat('/var/lib/ceph/osd/{cluster}-{osd_id}'.format(
+ dst_dev = os.stat((STATEDIR + '/osd/{cluster}-{osd_id}').format(
cluster=cluster,
osd_id=osd_id)).st_dev
if src_dev == dst_dev:
active = True
else:
- parent_dev = os.stat('/var/lib/ceph/osd').st_dev
+ parent_dev = os.stat(STATEDIR + '/osd').st_dev
if dst_dev != parent_dev:
other = True
- elif os.listdir('/var/lib/ceph/osd/{cluster}-{osd_id}'.format(
+ elif os.listdir((STATEDIR + '/osd/{cluster}-{osd_id}').format(
cluster=cluster,
osd_id=osd_id,
)):
@@ -1570,28 +1679,30 @@ def activate_dir(
)
(osd_id, cluster) = activate(path, activate_key_template, init)
- canonical = '/var/lib/ceph/osd/{cluster}-{osd_id}'.format(
- cluster=cluster,
- osd_id=osd_id)
- if path != canonical:
- # symlink it from the proper location
- create = True
- if os.path.lexists(canonical):
- old = os.readlink(canonical)
- if old != path:
- LOG.debug('Removing old symlink %s -> %s', canonical, old)
+
+ if init not in ( None, 'none' ):
+ canonical = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
+ cluster=cluster,
+ osd_id=osd_id)
+ if path != canonical:
+ # symlink it from the proper location
+ create = True
+ if os.path.lexists(canonical):
+ old = os.readlink(canonical)
+ if old != path:
+ LOG.debug('Removing old symlink %s -> %s', canonical, old)
+ try:
+ os.unlink(canonical)
+ except:
+ raise Error('unable to remove old symlink', canonical)
+ else:
+ create = False
+ if create:
+ LOG.debug('Creating symlink %s -> %s', canonical, path)
try:
- os.unlink(canonical)
+ os.symlink(path, canonical)
except:
- raise Error('unable to remove old symlink %s', canonical)
- else:
- create = False
- if create:
- LOG.debug('Creating symlink %s -> %s', canonical, path)
- try:
- os.symlink(path, canonical)
- except:
- raise Error('unable to create symlink %s -> %s', canonical, path)
+ raise Error('unable to create symlink %s -> %s' % (canonical, path))
return (cluster, osd_id)
@@ -1603,15 +1714,15 @@ def find_cluster_by_uuid(_uuid):
"""
_uuid = _uuid.lower()
no_fsid = []
- if not os.path.exists('/etc/ceph'):
+ if not os.path.exists(SYSCONFDIR):
return None
- for conf_file in os.listdir('/etc/ceph'):
+ for conf_file in os.listdir(SYSCONFDIR):
if not conf_file.endswith('.conf'):
continue
cluster = conf_file[:-5]
try:
fsid = get_fsid(cluster)
- except Error as e:
+ except Error as e:
if e.message != 'getting cluster uuid from configuration failed':
raise e
no_fsid.append(cluster)
@@ -1620,7 +1731,7 @@ def find_cluster_by_uuid(_uuid):
return cluster
# be tolerant of /etc/ceph/ceph.conf without an fsid defined.
if len(no_fsid) == 1 and no_fsid[0] == 'ceph':
- LOG.warning('No fsid defined in /etc/ceph/ceph.conf; using anyway')
+ LOG.warning('No fsid defined in ' + SYSCONFDIR + '/ceph.conf; using anyway')
return 'ceph'
return None
@@ -1630,88 +1741,84 @@ def activate(
init,
):
- try:
- check_osd_magic(path)
+ check_osd_magic(path)
- ceph_fsid = read_one_line(path, 'ceph_fsid')
- if ceph_fsid is None:
- raise Error('No cluster uuid assigned.')
- LOG.debug('Cluster uuid is %s', ceph_fsid)
+ ceph_fsid = read_one_line(path, 'ceph_fsid')
+ if ceph_fsid is None:
+ raise Error('No cluster uuid assigned.')
+ LOG.debug('Cluster uuid is %s', ceph_fsid)
- cluster = find_cluster_by_uuid(ceph_fsid)
- if cluster is None:
- raise Error('No cluster conf found in /etc/ceph with fsid %s' % ceph_fsid)
- LOG.debug('Cluster name is %s', cluster)
+ cluster = find_cluster_by_uuid(ceph_fsid)
+ if cluster is None:
+ raise Error('No cluster conf found in ' + SYSCONFDIR + ' with fsid %s' % ceph_fsid)
+ LOG.debug('Cluster name is %s', cluster)
- fsid = read_one_line(path, 'fsid')
- if fsid is None:
- raise Error('No OSD uuid assigned.')
- LOG.debug('OSD uuid is %s', fsid)
+ fsid = read_one_line(path, 'fsid')
+ if fsid is None:
+ raise Error('No OSD uuid assigned.')
+ LOG.debug('OSD uuid is %s', fsid)
- keyring = activate_key_template.format(cluster=cluster)
+ keyring = activate_key_template.format(cluster=cluster)
- osd_id = get_osd_id(path)
- if osd_id is None:
- osd_id = allocate_osd_id(
- cluster=cluster,
- fsid=fsid,
- keyring=keyring,
- )
- write_one_line(path, 'whoami', osd_id)
- LOG.debug('OSD id is %s', osd_id)
+ osd_id = get_osd_id(path)
+ if osd_id is None:
+ osd_id = allocate_osd_id(
+ cluster=cluster,
+ fsid=fsid,
+ keyring=keyring,
+ )
+ write_one_line(path, 'whoami', osd_id)
+ LOG.debug('OSD id is %s', osd_id)
+
+ if not os.path.exists(os.path.join(path, 'ready')):
+ LOG.debug('Initializing OSD...')
+ # re-running mkfs is safe, so just run until it completes
+ mkfs(
+ path=path,
+ cluster=cluster,
+ osd_id=osd_id,
+ fsid=fsid,
+ keyring=keyring,
+ )
- if not os.path.exists(os.path.join(path, 'ready')):
- LOG.debug('Initializing OSD...')
- # re-running mkfs is safe, so just run until it completes
- mkfs(
- path=path,
+ if init not in ( None, 'none' ):
+ if init == 'auto':
+ conf_val = get_conf(
cluster=cluster,
- osd_id=osd_id,
- fsid=fsid,
- keyring=keyring,
+ variable='init'
)
-
- if init is not None:
- if init == 'auto':
- conf_val = get_conf(
- cluster=cluster,
- variable='init'
- )
- if conf_val is not None:
- init = conf_val
+ if conf_val is not None:
+ init = conf_val
+ else:
+ (distro, release, codename) = platform.dist()
+ if distro == 'Ubuntu':
+ init = 'upstart'
else:
- (distro, release, codename) = platform.dist()
- if distro == 'Ubuntu':
- init = 'upstart'
- else:
- init = 'sysvinit'
-
- LOG.debug('Marking with init system %s', init)
- with file(os.path.join(path, init), 'w'):
- pass
+ init = 'sysvinit'
- # remove markers for others, just in case.
- for other in INIT_SYSTEMS:
- if other != init:
- try:
- os.unlink(os.path.join(path, other))
- except OSError:
- pass
+ LOG.debug('Marking with init system %s', init)
+ with file(os.path.join(path, init), 'w'):
+ pass
- if not os.path.exists(os.path.join(path, 'active')):
- LOG.debug('Authorizing OSD key...')
- auth_key(
- path=path,
- cluster=cluster,
- osd_id=osd_id,
- keyring=keyring,
- )
- write_one_line(path, 'active', 'ok')
- LOG.debug('%s osd.%s data dir is ready at %s', cluster, osd_id, path)
- return (osd_id, cluster)
- except:
- raise
+ # remove markers for others, just in case.
+ for other in INIT_SYSTEMS:
+ if other != init:
+ try:
+ os.unlink(os.path.join(path, other))
+ except OSError:
+ pass
+ if not os.path.exists(os.path.join(path, 'active')):
+ LOG.debug('Authorizing OSD key...')
+ auth_key(
+ path=path,
+ cluster=cluster,
+ osd_id=osd_id,
+ keyring=keyring,
+ )
+ write_one_line(path, 'active', 'ok')
+ LOG.debug('%s osd.%s data dir is ready at %s', cluster, osd_id, path)
+ return (osd_id, cluster)
def main_activate(args):
@@ -1719,7 +1826,7 @@ def main_activate(args):
osd_id = None
if not os.path.exists(args.path):
- raise Error('%s does not exist', args.path)
+ raise Error('%s does not exist' % args.path)
if is_suppressed(args.path):
LOG.info('suppressed activate request on %s', args.path)
@@ -1734,22 +1841,36 @@ def main_activate(args):
activate_key_template=args.activate_key_template,
init=args.mark_init,
)
+
elif stat.S_ISDIR(mode):
(cluster, osd_id) = activate_dir(
path=args.path,
activate_key_template=args.activate_key_template,
init=args.mark_init,
)
+
+ if args.mark_init == 'none':
+ command_check_call(
+ [
+ 'ceph-osd',
+ '--cluster={cluster}'.format(cluster=cluster),
+ '--id={osd_id}'.format(osd_id=osd_id),
+ '--osd-data={path}'.format(path=args.path),
+ '--osd-journal={path}/journal'.format(path=args.path),
+ ],
+ )
+
else:
- raise Error('%s is not a directory or block device', args.path)
+ raise Error('%s is not a directory or block device' % args.path)
- start_daemon(
- cluster=cluster,
- osd_id=osd_id,
+ if args.mark_init not in ( None, 'none' ):
+
+ start_daemon(
+ cluster=cluster,
+ osd_id=osd_id,
)
- activate_lock.release()
- except:
+ finally:
activate_lock.release()
@@ -1757,11 +1878,11 @@ def main_activate(args):
def get_journal_osd_uuid(path):
if not os.path.exists(path):
- raise Error('%s does not exist', path)
+ raise Error('%s does not exist' % path)
mode = os.stat(path).st_mode
if not stat.S_ISBLK(mode):
- raise Error('%s is not a block device', path)
+ raise Error('%s is not a block device' % path)
try:
out = _check_output(
@@ -1785,7 +1906,7 @@ def get_journal_osd_uuid(path):
def main_activate_journal(args):
if not os.path.exists(args.dev):
- raise Error('%s does not exist', args.dev)
+ raise Error('%s does not exist' % args.dev)
cluster = None
osd_id = None
@@ -1806,12 +1927,9 @@ def main_activate_journal(args):
osd_id=osd_id,
)
+ finally:
activate_lock.release()
- except:
- activate_lock.release()
- raise
-
###########################
def main_activate_all(args):
@@ -1876,15 +1994,14 @@ def get_oneliner(base, name):
return None
def get_dev_fs(dev):
- fscheck = subprocess.Popen(
+ fscheck, _ = command(
[
'blkid',
'-s',
'TYPE',
- dev
- ],
- stdout = subprocess.PIPE,
- stderr=subprocess.PIPE).stdout.read()
+ dev,
+ ],
+ )
if 'TYPE' in fscheck:
fstype = fscheck.split()[1].split('"')[1]
return fstype
@@ -1893,14 +2010,14 @@ def get_dev_fs(dev):
def get_partition_type(part):
(base, partnum) = re.match('(\D+)(\d+)', part).group(1, 2)
- sgdisk = subprocess.Popen(
+ sgdisk, _ = command(
[
'sgdisk',
'-p',
base,
- ],
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE).stdout.read()
+ ]
+ )
+
for line in sgdisk.splitlines():
m = re.search('\s+(\d+)\s+\d+\s+\d+\s+\S+ \S+B\s+\S+\s+(.*)', line)
if m is not None:
@@ -1912,10 +2029,7 @@ def get_partition_type(part):
def get_partition_uuid(dev):
(base, partnum) = re.match('(\D+)(\d+)', dev).group(1, 2)
- out = subprocess.Popen(
- [ 'sgdisk', '-i', partnum, base ],
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE).stdout.read()
+ out, _ = command(['sgdisk', '-i', partnum, base])
for line in out.splitlines():
m = re.match('Partition unique GUID: (\S+)', line)
if m:
@@ -2043,8 +2157,6 @@ def main_list(args):
# means suppressing sdb will stop activate on sdb1, sdb2, etc.
#
-SUPPRESS_PREFIX = '/var/lib/ceph/tmp/suppress-activate.'
-
def is_suppressed(path):
disk = os.path.realpath(path)
try:
@@ -2102,6 +2214,27 @@ def main_zap(args):
###########################
+def setup_statedir(dir):
+ global STATEDIR
+ STATEDIR = dir
+
+ if not os.path.exists(STATEDIR):
+ os.mkdir(STATEDIR)
+ if not os.path.exists(STATEDIR + "/tmp"):
+ os.mkdir(STATEDIR + "/tmp")
+
+ global prepare_lock
+ prepare_lock = filelock(STATEDIR + '/tmp/ceph-disk.prepare.lock')
+
+ global activate_lock
+ activate_lock = filelock(STATEDIR + '/tmp/ceph-disk.activate.lock')
+
+ global SUPPRESS_PREFIX
+ SUPPRESS_PREFIX = STATEDIR + '/tmp/suppress-activate.'
+
+def setup_sysconfdir(dir):
+ global SYSCONFDIR
+ SYSCONFDIR = dir
def parse_args():
parser = argparse.ArgumentParser(
@@ -2112,6 +2245,24 @@ def parse_args():
action='store_true', default=None,
help='be more verbose',
)
+ parser.add_argument(
+ '--prepend-to-path',
+ metavar='PATH',
+ default='/usr/bin',
+ help='prepend PATH to $PATH for backward compatibility (default /usr/bin)',
+ )
+ parser.add_argument(
+ '--statedir',
+ metavar='PATH',
+ default='/var/lib/ceph',
+ help='directory in which ceph state is preserved (default /var/lib/ceph)',
+ )
+ parser.add_argument(
+ '--sysconfdir',
+ metavar='PATH',
+ default='/etc/ceph',
+ help='directory in which ceph configuration files are found (default /etc/ceph)',
+ )
parser.set_defaults(
# we want to hold on to this, for later
prog=parser.prog,
@@ -2227,7 +2378,7 @@ def parse_args():
help='path to block device or directory',
)
activate_parser.set_defaults(
- activate_key_template='/var/lib/ceph/bootstrap-osd/{cluster}.keyring',
+ activate_key_template=STATEDIR + '/bootstrap-osd/{cluster}.keyring',
func=main_activate,
)
@@ -2251,7 +2402,7 @@ def parse_args():
choices=INIT_SYSTEMS,
)
activate_journal_parser.set_defaults(
- activate_key_template='/var/lib/ceph/bootstrap-osd/{cluster}.keyring',
+ activate_key_template=STATEDIR + '/bootstrap-osd/{cluster}.keyring',
func=main_activate_journal,
)
@@ -2270,7 +2421,7 @@ def parse_args():
choices=INIT_SYSTEMS,
)
activate_all_parser.set_defaults(
- activate_key_template='/var/lib/ceph/bootstrap-osd/{cluster}.keyring',
+ activate_key_template=STATEDIR + '/bootstrap-osd/{cluster}.keyring',
func=main_activate_all,
)
@@ -2305,7 +2456,7 @@ def parse_args():
zap_parser.add_argument(
'dev',
metavar='DEV',
- nargs='*',
+ nargs='+',
help='path to block device',
)
zap_parser.set_defaults(
@@ -2327,15 +2478,33 @@ def main():
level=loglevel,
)
+ if args.prepend_to_path != '':
+ path = os.environ.get('PATH', os.defpath)
+ os.environ['PATH'] = args.prepend_to_path + ":" + path
+
+ setup_statedir(args.statedir)
+ setup_sysconfdir(args.sysconfdir)
+
try:
args.func(args)
except Error as e:
- print >> sys.stderr, '{prog}: {msg}'.format(
- prog=args.prog,
- msg=e,
+ raise SystemExit(
+ '{prog}: {msg}'.format(
+ prog=args.prog,
+ msg=e,
)
- sys.exit(1)
+ )
+
+ except CephDiskException as error:
+ exc_name = error.__class__.__name__
+ raise SystemExit(
+ '{prog} {exc_name}: {msg}'.format(
+ prog=args.prog,
+ exc_name=exc_name,
+ msg=error,
+ )
+ )
if __name__ == '__main__':
diff --git a/src/ceph.in b/src/ceph.in
index 57816d3..c1f5a84 100755
--- a/src/ceph.in
+++ b/src/ceph.in
@@ -44,7 +44,7 @@ if MYDIR.endswith('src') and \
print >> sys.stderr, DEVMODEMSG
os.execvp('python', ['python'] + sys.argv)
sys.path.insert(0, os.path.join(MYDIR, 'pybind'))
- if MYDIR not in os.environ['PATH']:
+ if os.environ.has_key('PATH') and MYDIR not in os.environ['PATH']:
os.environ['PATH'] += ':' + MYDIR
import argparse
diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc
index 88b807b..2cfe285 100644
--- a/src/ceph_mds.cc
+++ b/src/ceph_mds.cc
@@ -243,8 +243,10 @@ int main(int argc, const char **argv)
CEPH_FEATURE_UID |
CEPH_FEATURE_NOSRCADDR |
CEPH_FEATURE_DIRLAYOUTHASH |
+ CEPH_FEATURE_MDS_INLINE_DATA |
CEPH_FEATURE_PGID64 |
- CEPH_FEATURE_MSG_AUTH;
+ CEPH_FEATURE_MSG_AUTH |
+ CEPH_FEATURE_EXPORT_PEER;
uint64_t required =
CEPH_FEATURE_OSDREPLYMUX;
messenger->set_default_policy(Messenger::Policy::lossy_client(supported, required));
diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index 2fc9e91..f9752b4 100644
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -100,16 +100,87 @@ int obtain_monmap(MonitorDBStore &store, bufferlist &bl)
return -ENOENT;
}
+int mon_data_exists(bool *r)
+{
+ string mon_data = g_conf->mon_data;
+ struct stat buf;
+ if (::stat(mon_data.c_str(), &buf)) {
+ if (errno == ENOENT) {
+ *r = false;
+ } else {
+ cerr << "stat(" << mon_data << ") " << strerror(errno) << std::endl;
+ return -errno;
+ }
+ } else {
+ *r = true;
+ }
+ return 0;
+}
+
+int mon_data_empty(bool *r)
+{
+ string mon_data = g_conf->mon_data;
+
+ DIR *dir = ::opendir(mon_data.c_str());
+ if (!dir) {
+ cerr << "opendir(" << mon_data << ") " << strerror(errno) << std::endl;
+ return -errno;
+ }
+ char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+
+ *r = false;
+ int code = 0;
+ struct dirent *de;
+ errno = 0;
+ while (!::readdir_r(dir, reinterpret_cast<struct dirent*>(buf), &de)) {
+ if (!de) {
+ if (errno) {
+ cerr << "readdir(" << mon_data << ") " << strerror(errno) << std::endl;
+ code = -errno;
+ }
+ break;
+ }
+ if (string(".") != de->d_name &&
+ string("..") != de->d_name) {
+ *r = true;
+ break;
+ }
+ }
+
+ ::closedir(dir);
+
+ return code;
+}
+
+int mon_exists(bool *r)
+{
+ int code = mon_data_exists(r);
+ if (code || *r == false)
+ return code;
+ return mon_data_empty(r);
+}
void usage()
{
- cerr << "usage: ceph-mon -i monid [--mon-data=pathtodata] [flags]" << std::endl;
+ cerr << "usage: ceph-mon -i monid [flags]" << std::endl;
cerr << " --debug_mon n\n";
cerr << " debug monitor level (e.g. 10)\n";
cerr << " --mkfs\n";
cerr << " build fresh monitor fs\n";
cerr << " --force-sync\n";
cerr << " force a sync from another mon by wiping local data (BE CAREFUL)\n";
+ cerr << " --yes-i-really-mean-it\n";
+ cerr << " mandatory safeguard for --force-sync\n";
+ cerr << " --compact\n";
+ cerr << " compact the monitor store\n";
+ cerr << " --osdmap <filename>\n";
+ cerr << " only used when --mkfs is provided: load the osdmap from <filename>\n";
+ cerr << " --inject-monmap <filename>\n";
+ cerr << " write the <filename> monmap to the local monitor store and exit\n";
+ cerr << " --extract-monmap <filename>\n";
+ cerr << " extract the monmap from the local monitor store and exit\n";
+ cerr << " --mon-data <directory>\n";
+ cerr << " where the mon store and keyring are located\n";
generic_server_usage();
}
@@ -127,7 +198,27 @@ int main(int argc, const char **argv)
argv_to_vec(argc, argv, args);
env_to_vec(args);
- global_init(NULL, args, CEPH_ENTITY_TYPE_MON, CODE_ENVIRONMENT_DAEMON, 0);
+ int flags = 0;
+ {
+ vector<const char*> args_copy = args;
+ std::string val;
+ for (std::vector<const char*>::iterator i = args_copy.begin();
+ i != args_copy.end(); ) {
+ if (ceph_argparse_double_dash(args_copy, i)) {
+ break;
+ } else if (ceph_argparse_flag(args_copy, i, "--mkfs", (char*)NULL)) {
+ flags |= CINIT_FLAG_NO_DAEMON_ACTIONS;
+ } else if (ceph_argparse_witharg(args_copy, i, &val, "--inject_monmap", (char*)NULL)) {
+ flags |= CINIT_FLAG_NO_DAEMON_ACTIONS;
+ } else if (ceph_argparse_witharg(args_copy, i, &val, "--extract-monmap", (char*)NULL)) {
+ flags |= CINIT_FLAG_NO_DAEMON_ACTIONS;
+ } else {
+ ++i;
+ }
+ }
+ }
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_MON, CODE_ENVIRONMENT_DAEMON, flags);
uuid_d fsid;
std::string val;
@@ -176,12 +267,33 @@ int main(int argc, const char **argv)
usage();
}
+ bool exists;
+ if (mon_exists(&exists))
+ exit(1);
+
+ if (mkfs && exists) {
+ cerr << g_conf->mon_data << " already exists" << std::endl;
+ exit(0);
+ }
+
// -- mkfs --
if (mkfs) {
+
+ if (mon_data_exists(&exists))
+ exit(1);
+
+ if (!exists) {
+ if (::mkdir(g_conf->mon_data.c_str(), 0755)) {
+ cerr << "mkdir(" << g_conf->mon_data << ") : "
+ << strerror(errno) << std::endl;
+ exit(1);
+ }
+ }
+
// resolve public_network -> public_addr
pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC);
- common_init_finish(g_ceph_context);
+ common_init_finish(g_ceph_context, flags);
bufferlist monmapbl, osdmapbl;
std::string error;
@@ -293,16 +405,18 @@ int main(int argc, const char **argv)
// we fork early to prevent leveldb's environment static state from
// screwing us over
Preforker prefork;
- if (g_conf->daemonize) {
- global_init_prefork(g_ceph_context, 0);
- prefork.prefork();
- if (prefork.is_parent()) {
- return prefork.parent_wait();
+ if (!(flags & CINIT_FLAG_NO_DAEMON_ACTIONS)) {
+ if (g_conf->daemonize) {
+ global_init_prefork(g_ceph_context, 0);
+ prefork.prefork();
+ if (prefork.is_parent()) {
+ return prefork.parent_wait();
+ }
+ global_init_postfork(g_ceph_context, 0);
}
- global_init_postfork(g_ceph_context, 0);
+ common_init_finish(g_ceph_context);
+ global_init_chdir(g_ceph_context);
}
- common_init_finish(g_ceph_context);
- global_init_chdir(g_ceph_context);
MonitorDBStore *store = new MonitorDBStore(g_conf->mon_data);
diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 2388762..029ef28 100644
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -23,7 +23,7 @@
using namespace std;
#include "osd/OSD.h"
-#include "os/FileStore.h"
+#include "os/ObjectStore.h"
#include "mon/MonClient.h"
#include "include/ceph_features.h"
@@ -162,6 +162,16 @@ int main(int argc, const char **argv)
usage();
}
+ // the store
+ ObjectStore *store = ObjectStore::create(g_ceph_context,
+ g_conf->osd_objectstore,
+ g_conf->osd_data,
+ g_conf->osd_journal);
+ if (!store) {
+ derr << "unable to create object store" << dendl;
+ return -ENODEV;
+ }
+
if (mkfs) {
common_init_finish(g_ceph_context);
MonClient mc(g_ceph_context);
@@ -170,7 +180,8 @@ int main(int argc, const char **argv)
if (mc.get_monmap_privately() < 0)
return -1;
- int err = OSD::mkfs(g_ceph_context, g_conf->osd_data, g_conf->osd_journal, mc.monmap.fsid, whoami);
+ int err = OSD::mkfs(g_ceph_context, store, g_conf->osd_data,
+ mc.monmap.fsid, whoami);
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error creating empty object store in "
<< g_conf->osd_data << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
@@ -213,7 +224,7 @@ int main(int argc, const char **argv)
exit(0);
if (mkjournal) {
common_init_finish(g_ceph_context);
- int err = OSD::mkjournal(g_ceph_context, g_conf->osd_data, g_conf->osd_journal);
+ int err = store->mkjournal();
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error creating fresh journal " << g_conf->osd_journal
<< " for object store " << g_conf->osd_data
@@ -226,13 +237,15 @@ int main(int argc, const char **argv)
}
if (flushjournal) {
common_init_finish(g_ceph_context);
- int err = OSD::flushjournal(g_ceph_context, g_conf->osd_data, g_conf->osd_journal);
+ int err = store->mount();
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error flushing journal " << g_conf->osd_journal
<< " for object store " << g_conf->osd_data
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
exit(1);
}
+ store->sync_and_flush();
+ store->umount();
derr << "flushed journal " << g_conf->osd_journal
<< " for object store " << g_conf->osd_data
<< dendl;
@@ -240,7 +253,7 @@ int main(int argc, const char **argv)
}
if (dump_journal) {
common_init_finish(g_ceph_context);
- int err = OSD::dump_journal(g_ceph_context, g_conf->osd_data, g_conf->osd_journal, cout);
+ int err = store->dump_journal(cout);
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error dumping journal " << g_conf->osd_journal
<< " for object store " << g_conf->osd_data
@@ -256,7 +269,7 @@ int main(int argc, const char **argv)
if (convertfilestore) {
- int err = OSD::convertfs(g_conf->osd_data, g_conf->osd_journal);
+ int err = OSD::do_convertfs(store);
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
@@ -267,7 +280,7 @@ int main(int argc, const char **argv)
if (get_journal_fsid) {
uuid_d fsid;
- int r = OSD::peek_journal_fsid(g_conf->osd_journal, fsid);
+ int r = store->peek_journal_fsid(&fsid);
if (r == 0)
cout << fsid << std::endl;
exit(r);
@@ -276,7 +289,7 @@ int main(int argc, const char **argv)
string magic;
uuid_d cluster_fsid, osd_fsid;
int w;
- int r = OSD::peek_meta(g_conf->osd_data, magic, cluster_fsid, osd_fsid, w);
+ int r = OSD::peek_meta(store, magic, cluster_fsid, osd_fsid, w);
if (r < 0) {
derr << TEXT_RED << " ** ERROR: unable to open OSD superblock on "
<< g_conf->osd_data << ": " << cpp_strerror(-r)
@@ -357,7 +370,8 @@ int main(int argc, const char **argv)
CEPH_FEATURE_UID |
CEPH_FEATURE_NOSRCADDR |
CEPH_FEATURE_PGID64 |
- CEPH_FEATURE_MSG_AUTH;
+ CEPH_FEATURE_MSG_AUTH |
+ CEPH_FEATURE_OSD_ERASURE_CODES;
ms_public->set_default_policy(Messenger::Policy::stateless_server(supported, 0));
ms_public->set_policy_throttlers(entity_name_t::TYPE_CLIENT,
@@ -423,8 +437,8 @@ int main(int argc, const char **argv)
global_init_daemonize(g_ceph_context, 0);
common_init_finish(g_ceph_context);
- if (g_conf->filestore_update_to >= (int)FileStore::target_version) {
- int err = OSD::convertfs(g_conf->osd_data, g_conf->osd_journal);
+ if (g_conf->filestore_update_to >= (int)store->get_target_version()) {
+ int err = OSD::do_convertfs(store);
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
@@ -437,7 +451,9 @@ int main(int argc, const char **argv)
return -1;
global_init_chdir(g_ceph_context);
- osd = new OSD(g_ceph_context, whoami,
+ osd = new OSD(g_ceph_context,
+ store,
+ whoami,
ms_cluster,
ms_public,
ms_hbclient,
diff --git a/src/civetweb/civetweb.h b/src/civetweb/civetweb.h
new file mode 100644
index 0000000..b8be9c4
--- /dev/null
+++ b/src/civetweb/civetweb.h
@@ -0,0 +1,496 @@
+/* Copyright (c) 2004-2013 Sergey Lyubka
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef CIVETWEB_HEADER_INCLUDED
+#define CIVETWEB_HEADER_INCLUDED
+
+#ifndef CIVETWEB_VERSION
+#define CIVETWEB_VERSION "1.6"
+#endif
+
+#include <stdio.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+struct mg_context; /* Handle for the HTTP service itself */
+struct mg_connection; /* Handle for the individual connection */
+
+
+/* This structure contains information about the HTTP request. */
+struct mg_request_info {
+ const char *request_method; /* "GET", "POST", etc */
+ const char *uri; /* URL-decoded URI */
+ const char *http_version; /* E.g. "1.0", "1.1" */
+ const char *query_string; /* URL part after '?', not including '?', or
+ NULL */
+ const char *remote_user; /* Authenticated user, or NULL if no auth
+ used */
+ long remote_ip; /* Client's IP address */
+ int remote_port; /* Client's port */
+ int is_ssl; /* 1 if SSL-ed, 0 if not */
+ void *user_data; /* User data pointer passed to mg_start() */
+ void *conn_data; /* Connection-specific user data */
+
+ int num_headers; /* Number of HTTP headers */
+ struct mg_header {
+ const char *name; /* HTTP header name */
+ const char *value; /* HTTP header value */
+ } http_headers[64]; /* Maximum 64 headers */
+};
+
+
+/* This structure needs to be passed to mg_start(), to let civetweb know
+ which callbacks to invoke. For detailed description, see
+ https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md */
+struct mg_callbacks {
+ /* Called when civetweb has received new HTTP request.
+ If callback returns non-zero,
+ callback must process the request by sending valid HTTP headers and
+ body, and civetweb will not do any further processing.
+ If callback returns 0, civetweb processes the request itself. In this
+ case, callback must not send any data to the client. */
+ int (*begin_request)(struct mg_connection *);
+
+ /* Called when civetweb has finished processing request. */
+ void (*end_request)(const struct mg_connection *, int reply_status_code);
+
+ /* Called when civetweb is about to log a message. If callback returns
+ non-zero, civetweb does not log anything. */
+ int (*log_message)(const struct mg_connection *, const char *message);
+
+ /* Called when civetweb initializes SSL library. */
+ int (*init_ssl)(void *ssl_context, void *user_data);
+
+ /* Called when websocket request is received, before websocket handshake.
+ If callback returns 0, civetweb proceeds with handshake, otherwise
+ cinnection is closed immediately. */
+ int (*websocket_connect)(const struct mg_connection *);
+
+ /* Called when websocket handshake is successfully completed, and
+ connection is ready for data exchange. */
+ void (*websocket_ready)(struct mg_connection *);
+
+ /* Called when data frame has been received from the client.
+ Parameters:
+ bits: first byte of the websocket frame, see websocket RFC at
+ http://tools.ietf.org/html/rfc6455, section 5.2
+ data, data_len: payload, with mask (if any) already applied.
+ Return value:
+ non-0: keep this websocket connection opened.
+ 0: close this websocket connection. */
+ int (*websocket_data)(struct mg_connection *, int bits,
+ char *data, size_t data_len);
+
+ /* Called when civetweb is closing a connection. The per-context mutex is
+ locked when this is invoked. This is primarily useful for noting when
+ a websocket is closing and removing it from any application-maintained
+ list of clients. */
+ void (*connection_close)(struct mg_connection *);
+
+ /* Called when civetweb tries to open a file. Used to intercept file open
+ calls, and serve file data from memory instead.
+ Parameters:
+ path: Full path to the file to open.
+ data_len: Placeholder for the file size, if file is served from
+ memory.
+ Return value:
+ NULL: do not serve file from memory, proceed with normal file open.
+ non-NULL: pointer to the file contents in memory. data_len must be
+ initilized with the size of the memory block. */
+ const char * (*open_file)(const struct mg_connection *,
+ const char *path, size_t *data_len);
+
+ /* Called when civetweb is about to serve Lua server page (.lp file), if
+ Lua support is enabled.
+ Parameters:
+ lua_context: "lua_State *" pointer. */
+ void (*init_lua)(struct mg_connection *, void *lua_context);
+
+ /* Called when civetweb has uploaded a file to a temporary directory as a
+ result of mg_upload() call.
+ Parameters:
+ file_file: full path name to the uploaded file. */
+ void (*upload)(struct mg_connection *, const char *file_name);
+
+ /* Called when civetweb is about to send HTTP error to the client.
+ Implementing this callback allows to create custom error pages.
+ Parameters:
+ status: HTTP error status code. */
+ int (*http_error)(struct mg_connection *, int status);
+};
+
+/* Start web server.
+
+ Parameters:
+ callbacks: mg_callbacks structure with user-defined callbacks.
+ options: NULL terminated list of option_name, option_value pairs that
+ specify Civetweb configuration parameters.
+
+ Side-effects: on UNIX, ignores SIGCHLD and SIGPIPE signals. If custom
+ processing is required for these, signal handlers must be set up
+ after calling mg_start().
+
+
+ Example:
+ const char *options[] = {
+ "document_root", "/var/www",
+ "listening_ports", "80,443s",
+ NULL
+ };
+ struct mg_context *ctx = mg_start(&my_func, NULL, options);
+
+ Refer to https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md
+ for the list of valid option and their possible values.
+
+ Return:
+ web server context, or NULL on error. */
+struct mg_context *mg_start(const struct mg_callbacks *callbacks,
+ void *user_data,
+ const char **configuration_options);
+
+
+/* Stop the web server.
+
+ Must be called last, when an application wants to stop the web server and
+ release all associated resources. This function blocks until all Civetweb
+ threads are stopped. Context pointer becomes invalid. */
+void mg_stop(struct mg_context *);
+
+/* mg_request_handler
+
+ Called when a new request comes in. This callback is URI based
+ and configured with mg_set_request_handler().
+
+ Parameters:
+ conn: current connection information.
+ cbdata: the callback data configured with mg_set_request_handler().
+ Returns:
+ 0: the handler could not handle the request, so fall through.
+ 1: the handler processed the request. */
+typedef int (* mg_request_handler)(struct mg_connection *conn, void *cbdata);
+
+/* mg_set_request_handler
+
+ Sets or removes a URI mapping for a request handler.
+
+ URI's are ordered and prefixed URI's are supported. For example,
+ consider two URIs: /a/b and /a
+ /a matches /a
+ /a/b matches /a/b
+ /a/c matches /a
+
+ Parameters:
+ ctx: server context
+ uri: the URI to configure
+ handler: the callback handler to use when the URI is requested.
+ If NULL, the URI will be removed.
+ cbdata: the callback data to give to the handler when it s requested. */
+void mg_set_request_handler(struct mg_context *ctx, const char *uri, mg_request_handler handler, void *cbdata);
+
+
+/* Get the value of particular configuration parameter.
+ The value returned is read-only. Civetweb does not allow changing
+ configuration at run time.
+ If given parameter name is not valid, NULL is returned. For valid
+ names, return value is guaranteed to be non-NULL. If parameter is not
+ set, zero-length string is returned. */
+const char *mg_get_option(const struct mg_context *ctx, const char *name);
+
+
+/* Return array of strings that represent valid configuration options.
+ For each option, option name and default value is returned, i.e. the
+ number of entries in the array equals to number_of_options x 2.
+ Array is NULL terminated. */
+const char **mg_get_valid_option_names(void);
+
+/* Get the list of ports that civetweb is listening on.
+ size is the size of the ports int array and ssl int array to fill.
+ It is the caller's responsibility to make sure ports and ssl each
+ contain at least size int elements worth of memory to write into.
+ Return value is the number of ports and ssl information filled in.
+ The value returned is read-only. Civetweb does not allow changing
+ configuration at run time. */
+size_t mg_get_ports(const struct mg_context *ctx, size_t size, int* ports, int* ssl);
+
+/* Add, edit or delete the entry in the passwords file.
+
+ This function allows an application to manipulate .htpasswd files on the
+ fly by adding, deleting and changing user records. This is one of the
+ several ways of implementing authentication on the server side. For another,
+ cookie-based way please refer to the examples/chat in the source tree.
+
+ If password is not NULL, entry is added (or modified if already exists).
+ If password is NULL, entry is deleted.
+
+ Return:
+ 1 on success, 0 on error. */
+int mg_modify_passwords_file(const char *passwords_file_name,
+ const char *domain,
+ const char *user,
+ const char *password);
+
+
+/* Return information associated with the request. */
+struct mg_request_info *mg_get_request_info(struct mg_connection *);
+
+
+/* Send data to the client.
+ Return:
+ 0 when the connection has been closed
+ -1 on error
+ >0 number of bytes written on success */
+int mg_write(struct mg_connection *, const void *buf, size_t len);
+
+
+/* Send data to a websocket client wrapped in a websocket frame. Uses mg_lock
+ to ensure that the transmission is not interrupted, i.e., when the
+ application is proactively communicating and responding to a request
+ simultaneously.
+
+ Send data to a websocket client wrapped in a websocket frame.
+ This function is available when civetweb is compiled with -DUSE_WEBSOCKET
+
+ Return:
+ 0 when the connection has been closed
+ -1 on error
+ >0 number of bytes written on success */
+int mg_websocket_write(struct mg_connection* conn, int opcode,
+ const char *data, size_t data_len);
+
+/* Blocks until unique access is obtained to this connection. Intended for use
+ with websockets only.
+ Invoke this before mg_write or mg_printf when communicating with a
+ websocket if your code has server-initiated communication as well as
+ communication in direct response to a message. */
+void mg_lock(struct mg_connection* conn);
+void mg_unlock(struct mg_connection* conn);
+
+/* Opcodes, from http://tools.ietf.org/html/rfc6455 */
+enum {
+ WEBSOCKET_OPCODE_CONTINUATION = 0x0,
+ WEBSOCKET_OPCODE_TEXT = 0x1,
+ WEBSOCKET_OPCODE_BINARY = 0x2,
+ WEBSOCKET_OPCODE_CONNECTION_CLOSE = 0x8,
+ WEBSOCKET_OPCODE_PING = 0x9,
+ WEBSOCKET_OPCODE_PONG = 0xa
+};
+
+
+/* Macros for enabling compiler-specific checks for printf-like arguments. */
+#undef PRINTF_FORMAT_STRING
+#if defined(_MSC_VER) && _MSC_VER >= 1400
+#include <sal.h>
+#if defined(_MSC_VER) && _MSC_VER > 1400
+#define PRINTF_FORMAT_STRING(s) _Printf_format_string_ s
+#else
+#define PRINTF_FORMAT_STRING(s) __format_string s
+#endif
+#else
+#define PRINTF_FORMAT_STRING(s) s
+#endif
+
+#ifdef __GNUC__
+#define PRINTF_ARGS(x, y) __attribute__((format(printf, x, y)))
+#else
+#define PRINTF_ARGS(x, y)
+#endif
+
+/* Send data to the client using printf() semantics.
+
+ Works exactly like mg_write(), but allows to do message formatting. */
+int mg_printf(struct mg_connection *,
+ PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3);
+
+
+/* Send contents of the entire file together with HTTP headers. */
+void mg_send_file(struct mg_connection *conn, const char *path);
+
+
+/* Read data from the remote end, return number of bytes read.
+ Return:
+ 0 connection has been closed by peer. No more data could be read.
+ < 0 read error. No more data could be read from the connection.
+ > 0 number of bytes read into the buffer. */
+int mg_read(struct mg_connection *, void *buf, size_t len);
+
+
+/* Get the value of particular HTTP header.
+
+ This is a helper function. It traverses request_info->http_headers array,
+ and if the header is present in the array, returns its value. If it is
+ not present, NULL is returned. */
+const char *mg_get_header(const struct mg_connection *, const char *name);
+
+
+/* Get a value of particular form variable.
+
+ Parameters:
+ data: pointer to form-uri-encoded buffer. This could be either POST data,
+ or request_info.query_string.
+ data_len: length of the encoded data.
+ var_name: variable name to decode from the buffer
+ dst: destination buffer for the decoded variable
+ dst_len: length of the destination buffer
+
+ Return:
+ On success, length of the decoded variable.
+ On error:
+ -1 (variable not found).
+ -2 (destination buffer is NULL, zero length or too small to hold the
+ decoded variable).
+
+ Destination buffer is guaranteed to be '\0' - terminated if it is not
+ NULL or zero length. */
+int mg_get_var(const char *data, size_t data_len,
+ const char *var_name, char *dst, size_t dst_len);
+
+/* Get a value of particular form variable.
+
+ Parameters:
+ data: pointer to form-uri-encoded buffer. This could be either POST data,
+ or request_info.query_string.
+ data_len: length of the encoded data.
+ var_name: variable name to decode from the buffer
+ dst: destination buffer for the decoded variable
+ dst_len: length of the destination buffer
+ occurrence: which occurrence of the variable, 0 is the first, 1 the
+ second...
+ this makes it possible to parse a query like
+ b=x&a=y&a=z which will have occurrence values b:0, a:0 and a:1
+
+ Return:
+ On success, length of the decoded variable.
+ On error:
+ -1 (variable not found).
+ -2 (destination buffer is NULL, zero length or too small to hold the
+ decoded variable).
+
+ Destination buffer is guaranteed to be '\0' - terminated if it is not
+ NULL or zero length. */
+int mg_get_var2(const char *data, size_t data_len,
+ const char *var_name, char *dst, size_t dst_len, size_t occurrence);
+
+/* Fetch value of certain cookie variable into the destination buffer.
+
+ Destination buffer is guaranteed to be '\0' - terminated. In case of
+ failure, dst[0] == '\0'. Note that RFC allows many occurrences of the same
+ parameter. This function returns only first occurrence.
+
+ Return:
+ On success, value length.
+ On error:
+ -1 (either "Cookie:" header is not present at all or the requested
+ parameter is not found).
+ -2 (destination buffer is NULL, zero length or too small to hold the
+ value). */
+int mg_get_cookie(const char *cookie, const char *var_name,
+ char *buf, size_t buf_len);
+
+
+/* Download data from the remote web server.
+ host: host name to connect to, e.g. "foo.com", or "10.12.40.1".
+ port: port number, e.g. 80.
+ use_ssl: wether to use SSL connection.
+ error_buffer, error_buffer_size: error message placeholder.
+ request_fmt,...: HTTP request.
+ Return:
+ On success, valid pointer to the new connection, suitable for mg_read().
+ On error, NULL. error_buffer contains error message.
+ Example:
+ char ebuf[100];
+ struct mg_connection *conn;
+ conn = mg_download("google.com", 80, 0, ebuf, sizeof(ebuf),
+ "%s", "GET / HTTP/1.0\r\nHost: google.com\r\n\r\n");
+ */
+struct mg_connection *mg_download(const char *host, int port, int use_ssl,
+ char *error_buffer, size_t error_buffer_size,
+ PRINTF_FORMAT_STRING(const char *request_fmt),
+ ...) PRINTF_ARGS(6, 7);
+
+
+/* Close the connection opened by mg_download(). */
+void mg_close_connection(struct mg_connection *conn);
+
+
+/* File upload functionality. Each uploaded file gets saved into a temporary
+ file and MG_UPLOAD event is sent.
+ Return number of uploaded files. */
+int mg_upload(struct mg_connection *conn, const char *destination_dir);
+
+
+/* Convenience function -- create detached thread.
+ Return: 0 on success, non-0 on error. */
+typedef void * (*mg_thread_func_t)(void *);
+int mg_start_thread(mg_thread_func_t f, void *p);
+
+
+/* Return builtin mime type for the given file name.
+ For unrecognized extensions, "text/plain" is returned. */
+const char *mg_get_builtin_mime_type(const char *file_name);
+
+
+/* Return Civetweb version. */
+const char *mg_version(void);
+
+/* URL-decode input buffer into destination buffer.
+ 0-terminate the destination buffer.
+ form-url-encoded data differs from URI encoding in a way that it
+ uses '+' as character for space, see RFC 1866 section 8.2.1
+ http://ftp.ics.uci.edu/pub/ietf/html/rfc1866.txt
+ Return: length of the decoded data, or -1 if dst buffer is too small. */
+int mg_url_decode(const char *src, int src_len, char *dst,
+ int dst_len, int is_form_url_encoded);
+
+/* URL-encode input buffer into destination buffer.
+ returns the length of the resulting buffer or -1
+ is the buffer is too small. */
+int mg_url_encode(const char *src, char *dst, size_t dst_len);
+
+/* MD5 hash given strings.
+ Buffer 'buf' must be 33 bytes long. Varargs is a NULL terminated list of
+ ASCIIz strings. When function returns, buf will contain human-readable
+ MD5 hash. Example:
+ char buf[33];
+ mg_md5(buf, "aa", "bb", NULL); */
+char *mg_md5(char buf[33], ...);
+
+
+/* Print error message to the opened error log stream.
+ This utilizes the provided logging configuration.
+ conn: connection
+ fmt: format string without the line return
+ ...: variable argument list
+ Example:
+ mg_cry(conn,"i like %s", "logging"); */
+void mg_cry(struct mg_connection *conn,
+ PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3);
+
+/* utility method to compare two buffers, case incensitive. */
+int mg_strncasecmp(const char *s1, const char *s2, size_t len);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* CIVETWEB_HEADER_INCLUDED */
diff --git a/src/civetweb/include/civetweb.h b/src/civetweb/include/civetweb.h
new file mode 100644
index 0000000..b8be9c4
--- /dev/null
+++ b/src/civetweb/include/civetweb.h
@@ -0,0 +1,496 @@
+/* Copyright (c) 2004-2013 Sergey Lyubka
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef CIVETWEB_HEADER_INCLUDED
+#define CIVETWEB_HEADER_INCLUDED
+
+#ifndef CIVETWEB_VERSION
+#define CIVETWEB_VERSION "1.6"
+#endif
+
+#include <stdio.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+struct mg_context; /* Handle for the HTTP service itself */
+struct mg_connection; /* Handle for the individual connection */
+
+
+/* This structure contains information about the HTTP request. */
+struct mg_request_info {
+ const char *request_method; /* "GET", "POST", etc */
+ const char *uri; /* URL-decoded URI */
+ const char *http_version; /* E.g. "1.0", "1.1" */
+ const char *query_string; /* URL part after '?', not including '?', or
+ NULL */
+ const char *remote_user; /* Authenticated user, or NULL if no auth
+ used */
+ long remote_ip; /* Client's IP address */
+ int remote_port; /* Client's port */
+ int is_ssl; /* 1 if SSL-ed, 0 if not */
+ void *user_data; /* User data pointer passed to mg_start() */
+ void *conn_data; /* Connection-specific user data */
+
+ int num_headers; /* Number of HTTP headers */
+ struct mg_header {
+ const char *name; /* HTTP header name */
+ const char *value; /* HTTP header value */
+ } http_headers[64]; /* Maximum 64 headers */
+};
+
+
+/* This structure needs to be passed to mg_start(), to let civetweb know
+ which callbacks to invoke. For detailed description, see
+ https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md */
+struct mg_callbacks {
+ /* Called when civetweb has received new HTTP request.
+ If callback returns non-zero,
+ callback must process the request by sending valid HTTP headers and
+ body, and civetweb will not do any further processing.
+ If callback returns 0, civetweb processes the request itself. In this
+ case, callback must not send any data to the client. */
+ int (*begin_request)(struct mg_connection *);
+
+ /* Called when civetweb has finished processing request. */
+ void (*end_request)(const struct mg_connection *, int reply_status_code);
+
+ /* Called when civetweb is about to log a message. If callback returns
+ non-zero, civetweb does not log anything. */
+ int (*log_message)(const struct mg_connection *, const char *message);
+
+ /* Called when civetweb initializes SSL library. */
+ int (*init_ssl)(void *ssl_context, void *user_data);
+
+ /* Called when websocket request is received, before websocket handshake.
+ If callback returns 0, civetweb proceeds with handshake, otherwise
+ cinnection is closed immediately. */
+ int (*websocket_connect)(const struct mg_connection *);
+
+ /* Called when websocket handshake is successfully completed, and
+ connection is ready for data exchange. */
+ void (*websocket_ready)(struct mg_connection *);
+
+ /* Called when data frame has been received from the client.
+ Parameters:
+ bits: first byte of the websocket frame, see websocket RFC at
+ http://tools.ietf.org/html/rfc6455, section 5.2
+ data, data_len: payload, with mask (if any) already applied.
+ Return value:
+ non-0: keep this websocket connection opened.
+ 0: close this websocket connection. */
+ int (*websocket_data)(struct mg_connection *, int bits,
+ char *data, size_t data_len);
+
+ /* Called when civetweb is closing a connection. The per-context mutex is
+ locked when this is invoked. This is primarily useful for noting when
+ a websocket is closing and removing it from any application-maintained
+ list of clients. */
+ void (*connection_close)(struct mg_connection *);
+
+ /* Called when civetweb tries to open a file. Used to intercept file open
+ calls, and serve file data from memory instead.
+ Parameters:
+ path: Full path to the file to open.
+ data_len: Placeholder for the file size, if file is served from
+ memory.
+ Return value:
+ NULL: do not serve file from memory, proceed with normal file open.
+ non-NULL: pointer to the file contents in memory. data_len must be
+ initilized with the size of the memory block. */
+ const char * (*open_file)(const struct mg_connection *,
+ const char *path, size_t *data_len);
+
+ /* Called when civetweb is about to serve Lua server page (.lp file), if
+ Lua support is enabled.
+ Parameters:
+ lua_context: "lua_State *" pointer. */
+ void (*init_lua)(struct mg_connection *, void *lua_context);
+
+ /* Called when civetweb has uploaded a file to a temporary directory as a
+ result of mg_upload() call.
+ Parameters:
+ file_file: full path name to the uploaded file. */
+ void (*upload)(struct mg_connection *, const char *file_name);
+
+ /* Called when civetweb is about to send HTTP error to the client.
+ Implementing this callback allows to create custom error pages.
+ Parameters:
+ status: HTTP error status code. */
+ int (*http_error)(struct mg_connection *, int status);
+};
+
+/* Start web server.
+
+ Parameters:
+ callbacks: mg_callbacks structure with user-defined callbacks.
+ options: NULL terminated list of option_name, option_value pairs that
+ specify Civetweb configuration parameters.
+
+ Side-effects: on UNIX, ignores SIGCHLD and SIGPIPE signals. If custom
+ processing is required for these, signal handlers must be set up
+ after calling mg_start().
+
+
+ Example:
+ const char *options[] = {
+ "document_root", "/var/www",
+ "listening_ports", "80,443s",
+ NULL
+ };
+ struct mg_context *ctx = mg_start(&my_func, NULL, options);
+
+ Refer to https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md
+ for the list of valid option and their possible values.
+
+ Return:
+ web server context, or NULL on error. */
+struct mg_context *mg_start(const struct mg_callbacks *callbacks,
+ void *user_data,
+ const char **configuration_options);
+
+
+/* Stop the web server.
+
+ Must be called last, when an application wants to stop the web server and
+ release all associated resources. This function blocks until all Civetweb
+ threads are stopped. Context pointer becomes invalid. */
+void mg_stop(struct mg_context *);
+
+/* mg_request_handler
+
+ Called when a new request comes in. This callback is URI based
+ and configured with mg_set_request_handler().
+
+ Parameters:
+ conn: current connection information.
+ cbdata: the callback data configured with mg_set_request_handler().
+ Returns:
+ 0: the handler could not handle the request, so fall through.
+ 1: the handler processed the request. */
+typedef int (* mg_request_handler)(struct mg_connection *conn, void *cbdata);
+
+/* mg_set_request_handler
+
+ Sets or removes a URI mapping for a request handler.
+
+ URI's are ordered and prefixed URI's are supported. For example,
+ consider two URIs: /a/b and /a
+ /a matches /a
+ /a/b matches /a/b
+ /a/c matches /a
+
+ Parameters:
+ ctx: server context
+ uri: the URI to configure
+ handler: the callback handler to use when the URI is requested.
+ If NULL, the URI will be removed.
+ cbdata: the callback data to give to the handler when it s requested. */
+void mg_set_request_handler(struct mg_context *ctx, const char *uri, mg_request_handler handler, void *cbdata);
+
+
+/* Get the value of particular configuration parameter.
+ The value returned is read-only. Civetweb does not allow changing
+ configuration at run time.
+ If given parameter name is not valid, NULL is returned. For valid
+ names, return value is guaranteed to be non-NULL. If parameter is not
+ set, zero-length string is returned. */
+const char *mg_get_option(const struct mg_context *ctx, const char *name);
+
+
+/* Return array of strings that represent valid configuration options.
+ For each option, option name and default value is returned, i.e. the
+ number of entries in the array equals to number_of_options x 2.
+ Array is NULL terminated. */
+const char **mg_get_valid_option_names(void);
+
+/* Get the list of ports that civetweb is listening on.
+ size is the size of the ports int array and ssl int array to fill.
+ It is the caller's responsibility to make sure ports and ssl each
+ contain at least size int elements worth of memory to write into.
+ Return value is the number of ports and ssl information filled in.
+ The value returned is read-only. Civetweb does not allow changing
+ configuration at run time. */
+size_t mg_get_ports(const struct mg_context *ctx, size_t size, int* ports, int* ssl);
+
+/* Add, edit or delete the entry in the passwords file.
+
+ This function allows an application to manipulate .htpasswd files on the
+ fly by adding, deleting and changing user records. This is one of the
+ several ways of implementing authentication on the server side. For another,
+ cookie-based way please refer to the examples/chat in the source tree.
+
+ If password is not NULL, entry is added (or modified if already exists).
+ If password is NULL, entry is deleted.
+
+ Return:
+ 1 on success, 0 on error. */
+int mg_modify_passwords_file(const char *passwords_file_name,
+ const char *domain,
+ const char *user,
+ const char *password);
+
+
+/* Return information associated with the request. */
+struct mg_request_info *mg_get_request_info(struct mg_connection *);
+
+
+/* Send data to the client.
+ Return:
+ 0 when the connection has been closed
+ -1 on error
+ >0 number of bytes written on success */
+int mg_write(struct mg_connection *, const void *buf, size_t len);
+
+
+/* Send data to a websocket client wrapped in a websocket frame. Uses mg_lock
+ to ensure that the transmission is not interrupted, i.e., when the
+ application is proactively communicating and responding to a request
+ simultaneously.
+
+ Send data to a websocket client wrapped in a websocket frame.
+ This function is available when civetweb is compiled with -DUSE_WEBSOCKET
+
+ Return:
+ 0 when the connection has been closed
+ -1 on error
+ >0 number of bytes written on success */
+int mg_websocket_write(struct mg_connection* conn, int opcode,
+ const char *data, size_t data_len);
+
+/* Blocks until unique access is obtained to this connection. Intended for use
+ with websockets only.
+ Invoke this before mg_write or mg_printf when communicating with a
+ websocket if your code has server-initiated communication as well as
+ communication in direct response to a message. */
+void mg_lock(struct mg_connection* conn);
+void mg_unlock(struct mg_connection* conn);
+
+/* Opcodes, from http://tools.ietf.org/html/rfc6455 */
+enum {
+ WEBSOCKET_OPCODE_CONTINUATION = 0x0,
+ WEBSOCKET_OPCODE_TEXT = 0x1,
+ WEBSOCKET_OPCODE_BINARY = 0x2,
+ WEBSOCKET_OPCODE_CONNECTION_CLOSE = 0x8,
+ WEBSOCKET_OPCODE_PING = 0x9,
+ WEBSOCKET_OPCODE_PONG = 0xa
+};
+
+
+/* Macros for enabling compiler-specific checks for printf-like arguments. */
+#undef PRINTF_FORMAT_STRING
+#if defined(_MSC_VER) && _MSC_VER >= 1400
+#include <sal.h>
+#if defined(_MSC_VER) && _MSC_VER > 1400
+#define PRINTF_FORMAT_STRING(s) _Printf_format_string_ s
+#else
+#define PRINTF_FORMAT_STRING(s) __format_string s
+#endif
+#else
+#define PRINTF_FORMAT_STRING(s) s
+#endif
+
+#ifdef __GNUC__
+#define PRINTF_ARGS(x, y) __attribute__((format(printf, x, y)))
+#else
+#define PRINTF_ARGS(x, y)
+#endif
+
+/* Send data to the client using printf() semantics.
+
+ Works exactly like mg_write(), but allows to do message formatting. */
+int mg_printf(struct mg_connection *,
+ PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3);
+
+
+/* Send contents of the entire file together with HTTP headers. */
+void mg_send_file(struct mg_connection *conn, const char *path);
+
+
+/* Read data from the remote end, return number of bytes read.
+ Return:
+ 0 connection has been closed by peer. No more data could be read.
+ < 0 read error. No more data could be read from the connection.
+ > 0 number of bytes read into the buffer. */
+int mg_read(struct mg_connection *, void *buf, size_t len);
+
+
+/* Get the value of particular HTTP header.
+
+ This is a helper function. It traverses request_info->http_headers array,
+ and if the header is present in the array, returns its value. If it is
+ not present, NULL is returned. */
+const char *mg_get_header(const struct mg_connection *, const char *name);
+
+
+/* Get a value of particular form variable.
+
+ Parameters:
+ data: pointer to form-uri-encoded buffer. This could be either POST data,
+ or request_info.query_string.
+ data_len: length of the encoded data.
+ var_name: variable name to decode from the buffer
+ dst: destination buffer for the decoded variable
+ dst_len: length of the destination buffer
+
+ Return:
+ On success, length of the decoded variable.
+ On error:
+ -1 (variable not found).
+ -2 (destination buffer is NULL, zero length or too small to hold the
+ decoded variable).
+
+ Destination buffer is guaranteed to be '\0' - terminated if it is not
+ NULL or zero length. */
+int mg_get_var(const char *data, size_t data_len,
+ const char *var_name, char *dst, size_t dst_len);
+
+/* Get a value of particular form variable.
+
+ Parameters:
+ data: pointer to form-uri-encoded buffer. This could be either POST data,
+ or request_info.query_string.
+ data_len: length of the encoded data.
+ var_name: variable name to decode from the buffer
+ dst: destination buffer for the decoded variable
+ dst_len: length of the destination buffer
+ occurrence: which occurrence of the variable, 0 is the first, 1 the
+ second...
+ this makes it possible to parse a query like
+ b=x&a=y&a=z which will have occurrence values b:0, a:0 and a:1
+
+ Return:
+ On success, length of the decoded variable.
+ On error:
+ -1 (variable not found).
+ -2 (destination buffer is NULL, zero length or too small to hold the
+ decoded variable).
+
+ Destination buffer is guaranteed to be '\0' - terminated if it is not
+ NULL or zero length. */
+int mg_get_var2(const char *data, size_t data_len,
+ const char *var_name, char *dst, size_t dst_len, size_t occurrence);
+
+/* Fetch value of certain cookie variable into the destination buffer.
+
+ Destination buffer is guaranteed to be '\0' - terminated. In case of
+ failure, dst[0] == '\0'. Note that RFC allows many occurrences of the same
+ parameter. This function returns only first occurrence.
+
+ Return:
+ On success, value length.
+ On error:
+ -1 (either "Cookie:" header is not present at all or the requested
+ parameter is not found).
+ -2 (destination buffer is NULL, zero length or too small to hold the
+ value). */
+int mg_get_cookie(const char *cookie, const char *var_name,
+ char *buf, size_t buf_len);
+
+
+/* Download data from the remote web server.
+ host: host name to connect to, e.g. "foo.com", or "10.12.40.1".
+ port: port number, e.g. 80.
+ use_ssl: wether to use SSL connection.
+ error_buffer, error_buffer_size: error message placeholder.
+ request_fmt,...: HTTP request.
+ Return:
+ On success, valid pointer to the new connection, suitable for mg_read().
+ On error, NULL. error_buffer contains error message.
+ Example:
+ char ebuf[100];
+ struct mg_connection *conn;
+ conn = mg_download("google.com", 80, 0, ebuf, sizeof(ebuf),
+ "%s", "GET / HTTP/1.0\r\nHost: google.com\r\n\r\n");
+ */
+struct mg_connection *mg_download(const char *host, int port, int use_ssl,
+ char *error_buffer, size_t error_buffer_size,
+ PRINTF_FORMAT_STRING(const char *request_fmt),
+ ...) PRINTF_ARGS(6, 7);
+
+
+/* Close the connection opened by mg_download(). */
+void mg_close_connection(struct mg_connection *conn);
+
+
+/* File upload functionality. Each uploaded file gets saved into a temporary
+ file and MG_UPLOAD event is sent.
+ Return number of uploaded files. */
+int mg_upload(struct mg_connection *conn, const char *destination_dir);
+
+
+/* Convenience function -- create detached thread.
+ Return: 0 on success, non-0 on error. */
+typedef void * (*mg_thread_func_t)(void *);
+int mg_start_thread(mg_thread_func_t f, void *p);
+
+
+/* Return builtin mime type for the given file name.
+ For unrecognized extensions, "text/plain" is returned. */
+const char *mg_get_builtin_mime_type(const char *file_name);
+
+
+/* Return Civetweb version. */
+const char *mg_version(void);
+
+/* URL-decode input buffer into destination buffer.
+ 0-terminate the destination buffer.
+ form-url-encoded data differs from URI encoding in a way that it
+ uses '+' as character for space, see RFC 1866 section 8.2.1
+ http://ftp.ics.uci.edu/pub/ietf/html/rfc1866.txt
+ Return: length of the decoded data, or -1 if dst buffer is too small. */
+int mg_url_decode(const char *src, int src_len, char *dst,
+ int dst_len, int is_form_url_encoded);
+
+/* URL-encode input buffer into destination buffer.
+ returns the length of the resulting buffer or -1
+ is the buffer is too small. */
+int mg_url_encode(const char *src, char *dst, size_t dst_len);
+
+/* MD5 hash given strings.
+ Buffer 'buf' must be 33 bytes long. Varargs is a NULL terminated list of
+ ASCIIz strings. When function returns, buf will contain human-readable
+ MD5 hash. Example:
+ char buf[33];
+ mg_md5(buf, "aa", "bb", NULL); */
+char *mg_md5(char buf[33], ...);
+
+
+/* Print error message to the opened error log stream.
+ This utilizes the provided logging configuration.
+ conn: connection
+ fmt: format string without the line return
+ ...: variable argument list
+ Example:
+ mg_cry(conn,"i like %s", "logging"); */
+void mg_cry(struct mg_connection *conn,
+ PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3);
+
+/* utility method to compare two buffers, case incensitive. */
+int mg_strncasecmp(const char *s1, const char *s2, size_t len);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* CIVETWEB_HEADER_INCLUDED */
diff --git a/src/civetweb/src/civetweb.c b/src/civetweb/src/civetweb.c
new file mode 100644
index 0000000..b639f1f
--- /dev/null
+++ b/src/civetweb/src/civetweb.c
@@ -0,0 +1,6453 @@
+/* Copyright (c) 2004-2013 Sergey Lyubka
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#if defined(_WIN32)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS /* Disable deprecation warning in VS2005 */
+#endif
+#else
+#ifdef __linux__
+#define _XOPEN_SOURCE 600 /* For flockfile() on Linux */
+#endif
+#define _LARGEFILE_SOURCE /* Enable 64-bit file offsets */
+#ifndef __STDC_FORMAT_MACROS
+# define __STDC_FORMAT_MACROS /* <inttypes.h> wants this for C++ */
+#endif
+#define __STDC_LIMIT_MACROS /* C++ wants that for INT64_MAX */
+#endif
+
+#if defined (_MSC_VER)
+/* 'type cast' : conversion from 'int' to 'HANDLE' of greater size */
+#pragma warning (disable : 4306 )
+/* conditional expression is constant: introduced by FD_SET(..) */
+#pragma warning (disable : 4127)
+/* non-constant aggregate initializer: issued due to missing C99 support */
+#pragma warning (disable : 4204)
+#endif
+
+/* Disable WIN32_LEAN_AND_MEAN.
+ This makes windows.h always include winsock2.h */
+#if defined(WIN32_LEAN_AND_MEAN) && (_MSC_VER <= 1400)
+#undef WIN32_LEAN_AND_MEAN
+#endif
+
+#if defined USE_IPV6 && defined(_WIN32)
+#include <ws2tcpip.h>
+#endif
+
+#if defined(__SYMBIAN32__)
+#define NO_SSL /* SSL is not supported */
+#define NO_CGI /* CGI is not supported */
+#define PATH_MAX FILENAME_MAX
+#endif /* __SYMBIAN32__ */
+
+#ifndef IGNORE_UNUSED_RESULT
+#define IGNORE_UNUSED_RESULT(a) (void)((a) && 1)
+#endif
+
+#ifndef _WIN32_WCE /* Some ANSI #includes are not available on Windows CE */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <signal.h>
+#include <fcntl.h>
+#endif /* !_WIN32_WCE */
+
+#include <time.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+#include <limits.h>
+#include <stddef.h>
+#include <stdio.h>
+
+#define MAX_WORKER_THREADS 1024
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__) /* Windows specific */
+#if defined(_MSC_VER) && _MSC_VER <= 1400
+#undef _WIN32_WINNT
+#define _WIN32_WINNT 0x0400 /* To make it link in VS2005 */
+#endif
+#include <windows.h>
+
+#ifndef PATH_MAX
+#define PATH_MAX MAX_PATH
+#endif
+
+#ifndef _IN_PORT_T
+#ifndef in_port_t
+#define in_port_t u_short
+#endif
+#endif
+
+#ifndef _WIN32_WCE
+#include <process.h>
+#include <direct.h>
+#include <io.h>
+#else /* _WIN32_WCE */
+#define NO_CGI /* WinCE has no pipes */
+
+typedef long off_t;
+
+#define errno GetLastError()
+#define strerror(x) _ultoa(x, (char *) _alloca(sizeof(x) *3 ), 10)
+#endif /* _WIN32_WCE */
+
+#define MAKEUQUAD(lo, hi) ((uint64_t)(((uint32_t)(lo)) | \
+ ((uint64_t)((uint32_t)(hi))) << 32))
+#define RATE_DIFF 10000000 /* 100 nsecs */
+#define EPOCH_DIFF MAKEUQUAD(0xd53e8000, 0x019db1de)
+#define SYS2UNIX_TIME(lo, hi) \
+ (time_t) ((MAKEUQUAD((lo), (hi)) - EPOCH_DIFF) / RATE_DIFF)
+
+/* Visual Studio 6 does not know __func__ or __FUNCTION__
+ The rest of MS compilers use __FUNCTION__, not C99 __func__
+ Also use _strtoui64 on modern M$ compilers */
+#if defined(_MSC_VER) && _MSC_VER < 1300
+#define STRX(x) #x
+#define STR(x) STRX(x)
+#define __func__ __FILE__ ":" STR(__LINE__)
+#define strtoull(x, y, z) (unsigned __int64) _atoi64(x)
+#define strtoll(x, y, z) _atoi64(x)
+#else
+#define __func__ __FUNCTION__
+#define strtoull(x, y, z) _strtoui64(x, y, z)
+#define strtoll(x, y, z) _strtoi64(x, y, z)
+#endif /* _MSC_VER */
+
+#define ERRNO GetLastError()
+#define NO_SOCKLEN_T
+#define SSL_LIB "ssleay32.dll"
+#define CRYPTO_LIB "libeay32.dll"
+#define O_NONBLOCK 0
+#if !defined(EWOULDBLOCK)
+#define EWOULDBLOCK WSAEWOULDBLOCK
+#endif /* !EWOULDBLOCK */
+#define _POSIX_
+#define INT64_FMT "I64d"
+
+#define WINCDECL __cdecl
+#define SHUT_WR 1
+#define snprintf _snprintf
+#define vsnprintf _vsnprintf
+#define mg_sleep(x) Sleep(x)
+
+#define pipe(x) _pipe(x, MG_BUF_LEN, _O_BINARY)
+#ifndef popen
+#define popen(x, y) _popen(x, y)
+#endif
+#ifndef pclose
+#define pclose(x) _pclose(x)
+#endif
+#define close(x) _close(x)
+#define dlsym(x,y) GetProcAddress((HINSTANCE) (x), (y))
+#define RTLD_LAZY 0
+#define fseeko(x, y, z) _lseeki64(_fileno(x), (y), (z))
+#define fdopen(x, y) _fdopen((x), (y))
+#define write(x, y, z) _write((x), (y), (unsigned) z)
+#define read(x, y, z) _read((x), (y), (unsigned) z)
+#define flockfile(x) EnterCriticalSection(&global_log_file_lock)
+#define funlockfile(x) LeaveCriticalSection(&global_log_file_lock)
+#define sleep(x) Sleep((x) * 1000)
+#define rmdir(x) _rmdir(x)
+
+#if !defined(va_copy)
+#define va_copy(x, y) x = y
+#endif /* !va_copy MINGW #defines va_copy */
+
+#if !defined(fileno)
+#define fileno(x) _fileno(x)
+#endif /* !fileno MINGW #defines fileno */
+
+typedef HANDLE pthread_mutex_t;
+typedef DWORD pthread_key_t;
+typedef HANDLE pthread_t;
+typedef struct {
+ CRITICAL_SECTION threadIdSec;
+ int waitingthreadcount; /* The number of threads queued. */
+ pthread_t *waitingthreadhdls; /* The thread handles. */
+} pthread_cond_t;
+
+typedef DWORD clockid_t;
+#define CLOCK_MONOTONIC (1)
+#define CLOCK_REALTIME (2)
+
+struct timespec {
+ time_t tv_sec; /* seconds */
+ long tv_nsec; /* nanoseconds */
+};
+
+#define pid_t HANDLE /* MINGW typedefs pid_t to int. Using #define here. */
+
+static int pthread_mutex_lock(pthread_mutex_t *);
+static int pthread_mutex_unlock(pthread_mutex_t *);
+static void to_unicode(const char *path, wchar_t *wbuf, size_t wbuf_len);
+struct file;
+static char *mg_fgets(char *buf, size_t size, struct file *filep, char **p);
+
+#if defined(HAVE_STDINT)
+#include <stdint.h>
+#else
+typedef unsigned int uint32_t;
+typedef unsigned short uint16_t;
+typedef unsigned __int64 uint64_t;
+typedef __int64 int64_t;
+#define INT64_MAX 9223372036854775807
+#endif /* HAVE_STDINT */
+
+/* POSIX dirent interface */
+struct dirent {
+ char d_name[PATH_MAX];
+};
+
+typedef struct DIR {
+ HANDLE handle;
+ WIN32_FIND_DATAW info;
+ struct dirent result;
+} DIR;
+
+#if !defined(USE_IPV6) && defined(_WIN32)
+#ifndef HAVE_POLL
+struct pollfd {
+ SOCKET fd;
+ short events;
+ short revents;
+};
+#define POLLIN 1
+#endif
+#endif
+
+/* Mark required libraries */
+#ifdef _MSC_VER
+#pragma comment(lib, "Ws2_32.lib")
+#endif
+
+#else /* UNIX specific */
+#include <sys/wait.h>
+#include <sys/socket.h>
+#include <sys/poll.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/time.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <netdb.h>
+
+#include <pwd.h>
+#include <unistd.h>
+#include <dirent.h>
+#if !defined(NO_SSL_DL) && !defined(NO_SSL)
+#include <dlfcn.h>
+#endif
+#include <pthread.h>
+#if defined(__MACH__)
+#define SSL_LIB "libssl.dylib"
+#define CRYPTO_LIB "libcrypto.dylib"
+#else
+#if !defined(SSL_LIB)
+#define SSL_LIB "libssl.so"
+#endif
+#if !defined(CRYPTO_LIB)
+#define CRYPTO_LIB "libcrypto.so"
+#endif
+#endif
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif /* O_BINARY */
+#define closesocket(a) close(a)
+#define mg_mkdir(x, y) mkdir(x, y)
+#define mg_remove(x) remove(x)
+#define mg_sleep(x) usleep((x) * 1000)
+#define ERRNO errno
+#define INVALID_SOCKET (-1)
+#define INT64_FMT PRId64
+typedef int SOCKET;
+#define WINCDECL
+
+#endif /* End of Windows and UNIX specific includes */
+
+#include "civetweb.h"
+
+#define PASSWORDS_FILE_NAME ".htpasswd"
+#define CGI_ENVIRONMENT_SIZE 4096
+#define MAX_CGI_ENVIR_VARS 64
+#define MG_BUF_LEN 8192
+#ifndef MAX_REQUEST_SIZE
+#define MAX_REQUEST_SIZE 16384
+#endif
+#define ARRAY_SIZE(array) (sizeof(array) / sizeof(array[0]))
+
+#ifdef _WIN32
+static CRITICAL_SECTION global_log_file_lock;
+static DWORD pthread_self(void)
+{
+ return GetCurrentThreadId();
+}
+
+int pthread_key_create(pthread_key_t *key, void (*_must_be_zero)(void*) /* destructor function not supported for windows */)
+{
+ assert(_must_be_zero == NULL);
+ if ((key!=0) && (_must_be_zero == NULL)) {
+ *key = TlsAlloc();
+ return (*key != TLS_OUT_OF_INDEXES) ? 0 : -1;
+ }
+ return -2;
+}
+
+int pthread_key_delete(pthread_key_t key)
+{
+ return TlsFree(key) ? 0 : 1;
+}
+
+int pthread_setspecific(pthread_key_t key, void * value)
+{
+ return TlsSetValue(key, value) ? 0 : 1;
+}
+
+void *pthread_getspecific(pthread_key_t key)
+{
+ return TlsGetValue(key);
+}
+#endif /* _WIN32 */
+
+#define MD5_STATIC static
+#include "md5.h"
+
+#ifdef DEBUG_TRACE
+#undef DEBUG_TRACE
+#define DEBUG_TRACE(x)
+#else
+#if defined(DEBUG)
+#define DEBUG_TRACE(x) do { \
+ flockfile(stdout); \
+ printf("*** %lu.%p.%s.%d: ", \
+ (unsigned long) time(NULL), (void *) pthread_self(), \
+ __func__, __LINE__); \
+ printf x; \
+ putchar('\n'); \
+ fflush(stdout); \
+ funlockfile(stdout); \
+} while (0)
+#else
+#define DEBUG_TRACE(x)
+#endif /* DEBUG */
+#endif /* DEBUG_TRACE */
+
+/* Darwin prior to 7.0 and Win32 do not have socklen_t */
+#ifdef NO_SOCKLEN_T
+typedef int socklen_t;
+#endif /* NO_SOCKLEN_T */
+#define _DARWIN_UNLIMITED_SELECT
+
+#define IP_ADDR_STR_LEN 50 /* IPv6 hex string is 46 chars */
+
+#if !defined(MSG_NOSIGNAL)
+#define MSG_NOSIGNAL 0
+#endif
+
+#if !defined(SOMAXCONN)
+#define SOMAXCONN 100
+#endif
+
+#if !defined(PATH_MAX)
+#define PATH_MAX 4096
+#endif
+
+/* Size of the accepted socket queue */
+#if !defined(MGSQLEN)
+#define MGSQLEN 20
+#endif
+
+static const char *http_500_error = "Internal Server Error";
+
+#if defined(NO_SSL_DL)
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#else
+/* SSL loaded dynamically from DLL.
+ I put the prototypes here to be independent from OpenSSL source
+ installation. */
+
+typedef struct ssl_st SSL;
+typedef struct ssl_method_st SSL_METHOD;
+typedef struct ssl_ctx_st SSL_CTX;
+
+struct ssl_func {
+ const char *name; /* SSL function name */
+ void (*ptr)(void); /* Function pointer */
+};
+
+#define SSL_free (* (void (*)(SSL *)) ssl_sw[0].ptr)
+#define SSL_accept (* (int (*)(SSL *)) ssl_sw[1].ptr)
+#define SSL_connect (* (int (*)(SSL *)) ssl_sw[2].ptr)
+#define SSL_read (* (int (*)(SSL *, void *, int)) ssl_sw[3].ptr)
+#define SSL_write (* (int (*)(SSL *, const void *,int)) ssl_sw[4].ptr)
+#define SSL_get_error (* (int (*)(SSL *, int)) ssl_sw[5].ptr)
+#define SSL_set_fd (* (int (*)(SSL *, SOCKET)) ssl_sw[6].ptr)
+#define SSL_new (* (SSL * (*)(SSL_CTX *)) ssl_sw[7].ptr)
+#define SSL_CTX_new (* (SSL_CTX * (*)(SSL_METHOD *)) ssl_sw[8].ptr)
+#define SSLv23_server_method (* (SSL_METHOD * (*)(void)) ssl_sw[9].ptr)
+#define SSL_library_init (* (int (*)(void)) ssl_sw[10].ptr)
+#define SSL_CTX_use_PrivateKey_file (* (int (*)(SSL_CTX *, \
+ const char *, int)) ssl_sw[11].ptr)
+#define SSL_CTX_use_certificate_file (* (int (*)(SSL_CTX *, \
+ const char *, int)) ssl_sw[12].ptr)
+#define SSL_CTX_set_default_passwd_cb \
+ (* (void (*)(SSL_CTX *, mg_callback_t)) ssl_sw[13].ptr)
+#define SSL_CTX_free (* (void (*)(SSL_CTX *)) ssl_sw[14].ptr)
+#define SSL_load_error_strings (* (void (*)(void)) ssl_sw[15].ptr)
+#define SSL_CTX_use_certificate_chain_file \
+ (* (int (*)(SSL_CTX *, const char *)) ssl_sw[16].ptr)
+#define SSLv23_client_method (* (SSL_METHOD * (*)(void)) ssl_sw[17].ptr)
+#define SSL_pending (* (int (*)(SSL *)) ssl_sw[18].ptr)
+#define SSL_CTX_set_verify (* (void (*)(SSL_CTX *, int, int)) ssl_sw[19].ptr)
+#define SSL_shutdown (* (int (*)(SSL *)) ssl_sw[20].ptr)
+
+#define CRYPTO_num_locks (* (int (*)(void)) crypto_sw[0].ptr)
+#define CRYPTO_set_locking_callback \
+ (* (void (*)(void (*)(int, int, const char *, int))) crypto_sw[1].ptr)
+#define CRYPTO_set_id_callback \
+ (* (void (*)(unsigned long (*)(void))) crypto_sw[2].ptr)
+#define ERR_get_error (* (unsigned long (*)(void)) crypto_sw[3].ptr)
+#define ERR_error_string (* (char * (*)(unsigned long,char *)) crypto_sw[4].ptr)
+
+/* set_ssl_option() function updates this array.
+ It loads SSL library dynamically and changes NULLs to the actual addresses
+ of respective functions. The macros above (like SSL_connect()) are really
+ just calling these functions indirectly via the pointer. */
+static struct ssl_func ssl_sw[] = {
+ {"SSL_free", NULL},
+ {"SSL_accept", NULL},
+ {"SSL_connect", NULL},
+ {"SSL_read", NULL},
+ {"SSL_write", NULL},
+ {"SSL_get_error", NULL},
+ {"SSL_set_fd", NULL},
+ {"SSL_new", NULL},
+ {"SSL_CTX_new", NULL},
+ {"SSLv23_server_method", NULL},
+ {"SSL_library_init", NULL},
+ {"SSL_CTX_use_PrivateKey_file", NULL},
+ {"SSL_CTX_use_certificate_file",NULL},
+ {"SSL_CTX_set_default_passwd_cb",NULL},
+ {"SSL_CTX_free", NULL},
+ {"SSL_load_error_strings", NULL},
+ {"SSL_CTX_use_certificate_chain_file", NULL},
+ {"SSLv23_client_method", NULL},
+ {"SSL_pending", NULL},
+ {"SSL_CTX_set_verify", NULL},
+ {"SSL_shutdown", NULL},
+ {NULL, NULL}
+};
+
+/* Similar array as ssl_sw. These functions could be located in different
+ lib. */
+#if !defined(NO_SSL)
+static struct ssl_func crypto_sw[] = {
+ {"CRYPTO_num_locks", NULL},
+ {"CRYPTO_set_locking_callback", NULL},
+ {"CRYPTO_set_id_callback", NULL},
+ {"ERR_get_error", NULL},
+ {"ERR_error_string", NULL},
+ {NULL, NULL}
+};
+#endif /* NO_SSL */
+#endif /* NO_SSL_DL */
+
+static const char *month_names[] = {
+ "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
+};
+
+/* Unified socket address. For IPv6 support, add IPv6 address structure
+ in the union u. */
+union usa {
+ struct sockaddr sa;
+ struct sockaddr_in sin;
+#if defined(USE_IPV6)
+ struct sockaddr_in6 sin6;
+#endif
+};
+
+/* Describes a string (chunk of memory). */
+struct vec {
+ const char *ptr;
+ size_t len;
+};
+
+struct file {
+ int is_directory;
+ time_t modification_time;
+ int64_t size;
+ FILE *fp;
+ const char *membuf; /* Non-NULL if file data is in memory */
+ /* set to 1 if the content is gzipped
+ in which case we need a content-encoding: gzip header */
+ int gzipped;
+};
+#define STRUCT_FILE_INITIALIZER {0, 0, 0, NULL, NULL, 0}
+
+/* Describes listening socket, or socket which was accept()-ed by the master
+ thread and queued for future handling by the worker thread. */
+struct socket {
+ SOCKET sock; /* Listening socket */
+ union usa lsa; /* Local socket address */
+ union usa rsa; /* Remote socket address */
+ unsigned is_ssl:1; /* Is port SSL-ed */
+ unsigned ssl_redir:1; /* Is port supposed to redirect everything to SSL
+ port */
+};
+
+/* NOTE(lsm): this enum shoulds be in sync with the config_options below. */
+enum {
+ CGI_EXTENSIONS, CGI_ENVIRONMENT, PUT_DELETE_PASSWORDS_FILE, CGI_INTERPRETER,
+ PROTECT_URI, AUTHENTICATION_DOMAIN, SSI_EXTENSIONS, THROTTLE,
+ ACCESS_LOG_FILE, ENABLE_DIRECTORY_LISTING, ERROR_LOG_FILE,
+ GLOBAL_PASSWORDS_FILE, INDEX_FILES, ENABLE_KEEP_ALIVE, ACCESS_CONTROL_LIST,
+ EXTRA_MIME_TYPES, LISTENING_PORTS, DOCUMENT_ROOT, SSL_CERTIFICATE,
+ NUM_THREADS, RUN_AS_USER, REWRITE, HIDE_FILES, REQUEST_TIMEOUT,
+
+#if defined(USE_LUA)
+ LUA_SCRIPT_EXTENSIONS, LUA_SERVER_PAGE_EXTENSIONS,
+#endif
+#if defined(USE_WEBSOCKET)
+ WEBSOCKET_ROOT,
+#endif
+
+ NUM_OPTIONS
+};
+
+static const char *config_options[] = {
+ "cgi_pattern", "**.cgi$|**.pl$|**.php$",
+ "cgi_environment", NULL,
+ "put_delete_auth_file", NULL,
+ "cgi_interpreter", NULL,
+ "protect_uri", NULL,
+ "authentication_domain", "mydomain.com",
+ "ssi_pattern", "**.shtml$|**.shtm$",
+ "throttle", NULL,
+ "access_log_file", NULL,
+ "enable_directory_listing", "yes",
+ "error_log_file", NULL,
+ "global_auth_file", NULL,
+ "index_files",
+#ifdef USE_LUA
+ "index.html,index.htm,index.lp,index.lsp,index.lua,index.cgi,index.shtml,index.php",
+#else
+ "index.html,index.htm,index.cgi,index.shtml,index.php",
+#endif
+ "enable_keep_alive", "no",
+ "access_control_list", NULL,
+ "extra_mime_types", NULL,
+ "listening_ports", "8080",
+ "document_root", NULL,
+ "ssl_certificate", NULL,
+ "num_threads", "50",
+ "run_as_user", NULL,
+ "url_rewrite_patterns", NULL,
+ "hide_files_patterns", NULL,
+ "request_timeout_ms", "30000",
+
+#if defined(USE_LUA)
+ "lua_script_pattern", "**.lua$",
+ "lua_server_page_pattern", "**.lp$|**.lsp$",
+#endif
+#if defined(USE_WEBSOCKET)
+ "websocket_root", NULL,
+#endif
+
+ NULL
+};
+
+struct mg_request_handler_info {
+ char *uri;
+ size_t uri_len;
+ mg_request_handler handler;
+ void *cbdata;
+ struct mg_request_handler_info *next;
+};
+
+struct mg_context {
+ volatile int stop_flag; /* Should we stop event loop */
+ void *ssllib_dll_handle; /* Store the ssl library handle. */
+ void *cryptolib_dll_handle; /* Store the crypto library handle. */
+ SSL_CTX *ssl_ctx; /* SSL context */
+ char *config[NUM_OPTIONS]; /* Civetweb configuration parameters */
+ struct mg_callbacks callbacks; /* User-defined callback function */
+ void *user_data; /* User-defined data */
+
+ struct socket *listening_sockets;
+ in_port_t *listening_ports;
+ int num_listening_sockets;
+
+ volatile int num_threads; /* Number of threads */
+ pthread_mutex_t mutex; /* Protects (max|num)_threads */
+ pthread_cond_t cond; /* Condvar for tracking workers terminations */
+
+ struct socket queue[MGSQLEN]; /* Accepted sockets */
+ volatile int sq_head; /* Head of the socket queue */
+ volatile int sq_tail; /* Tail of the socket queue */
+ pthread_cond_t sq_full; /* Signaled when socket is produced */
+ pthread_cond_t sq_empty; /* Signaled when socket is consumed */
+ pthread_t masterthreadid; /* The master thread ID. */
+ int workerthreadcount; /* The amount of worker threads. */
+ pthread_t *workerthreadids;/* The worker thread IDs. */
+
+ /* linked list of uri handlers */
+ struct mg_request_handler_info *request_handlers;
+};
+
+struct mg_connection {
+ struct mg_request_info request_info;
+ struct mg_context *ctx;
+ SSL *ssl; /* SSL descriptor */
+ SSL_CTX *client_ssl_ctx; /* SSL context for client connections */
+ struct socket client; /* Connected client */
+ time_t birth_time; /* Time when request was received */
+ int64_t num_bytes_sent; /* Total bytes sent to client */
+ int64_t content_len; /* Content-Length header value */
+ int64_t consumed_content; /* How many bytes of content have been read */
+ char *buf; /* Buffer for received data */
+ char *path_info; /* PATH_INFO part of the URL */
+ int must_close; /* 1 if connection must be closed */
+ int buf_size; /* Buffer size */
+ int request_len; /* Size of the request + headers in a buffer */
+ int data_len; /* Total size of data in a buffer */
+ int status_code; /* HTTP reply status code, e.g. 200 */
+ int throttle; /* Throttling, bytes/sec. <= 0 means no
+ throttle */
+ time_t last_throttle_time; /* Last time throttled data was sent */
+ int64_t last_throttle_bytes;/* Bytes sent this second */
+ pthread_mutex_t mutex; /* Used by mg_lock/mg_unlock to ensure atomic
+ transmissions for websockets */
+#if defined(USE_LUA) && defined(USE_WEBSOCKET)
+ void * lua_websocket_state; /* Lua_State for a websocket connection */
+#endif
+};
+
+static pthread_key_t sTlsKey; /* Thread local storage index */
+static int sTlsInit = 0;
+
+struct mg_workerTLS {
+ int is_master;
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ HANDLE pthread_cond_helper_mutex;
+#endif
+};
+
+/* Directory entry */
+struct de {
+ struct mg_connection *conn;
+ char *file_name;
+ struct file file;
+};
+
+#if defined(USE_WEBSOCKET)
+static int is_websocket_request(const struct mg_connection *conn);
+#endif
+
+const char **mg_get_valid_option_names(void)
+{
+ return config_options;
+}
+
+static int is_file_in_memory(struct mg_connection *conn, const char *path,
+ struct file *filep)
+{
+ size_t size = 0;
+ if ((filep->membuf = conn->ctx->callbacks.open_file == NULL ? NULL :
+ conn->ctx->callbacks.open_file(conn, path, &size)) != NULL) {
+ /* NOTE: override filep->size only on success. Otherwise, it might
+ break constructs like if (!mg_stat() || !mg_fopen()) ... */
+ filep->size = size;
+ }
+ return filep->membuf != NULL;
+}
+
+static int is_file_opened(const struct file *filep)
+{
+ return filep->membuf != NULL || filep->fp != NULL;
+}
+
+static int mg_fopen(struct mg_connection *conn, const char *path,
+ const char *mode, struct file *filep)
+{
+ if (!is_file_in_memory(conn, path, filep)) {
+#ifdef _WIN32
+ wchar_t wbuf[PATH_MAX], wmode[20];
+ to_unicode(path, wbuf, ARRAY_SIZE(wbuf));
+ MultiByteToWideChar(CP_UTF8, 0, mode, -1, wmode, ARRAY_SIZE(wmode));
+ filep->fp = _wfopen(wbuf, wmode);
+#else
+ filep->fp = fopen(path, mode);
+#endif
+ }
+
+ return is_file_opened(filep);
+}
+
+static void mg_fclose(struct file *filep)
+{
+ if (filep != NULL && filep->fp != NULL) {
+ fclose(filep->fp);
+ }
+}
+
+static int get_option_index(const char *name)
+{
+ int i;
+
+ for (i = 0; config_options[i * 2] != NULL; i++) {
+ if (strcmp(config_options[i * 2], name) == 0) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+const char *mg_get_option(const struct mg_context *ctx, const char *name)
+{
+ int i;
+ if ((i = get_option_index(name)) == -1) {
+ return NULL;
+ } else if (ctx->config[i] == NULL) {
+ return "";
+ } else {
+ return ctx->config[i];
+ }
+}
+
+size_t mg_get_ports(const struct mg_context *ctx, size_t size, int* ports, int* ssl)
+{
+ size_t i;
+ for (i = 0; i < size && i < (size_t)ctx->num_listening_sockets; i++)
+ {
+ ssl[i] = ctx->listening_sockets[i].is_ssl;
+ ports[i] = ctx->listening_ports[i];
+ }
+ return i;
+}
+
+static void sockaddr_to_string(char *buf, size_t len,
+ const union usa *usa)
+{
+ buf[0] = '\0';
+#if defined(USE_IPV6)
+ inet_ntop(usa->sa.sa_family, usa->sa.sa_family == AF_INET ?
+ (void *) &usa->sin.sin_addr :
+ (void *) &usa->sin6.sin6_addr, buf, len);
+#elif defined(_WIN32)
+ /* Only Windows Vista (and newer) have inet_ntop() */
+ strncpy(buf, inet_ntoa(usa->sin.sin_addr), len);
+#else
+ inet_ntop(usa->sa.sa_family, (void *) &usa->sin.sin_addr, buf, len);
+#endif
+}
+
+/* Convert time_t to a string. According to RFC2616, Sec 14.18, this must be included in all responses other than 100, 101, 5xx. */
+static void gmt_time_string(char *buf, size_t buf_len, time_t *t)
+{
+ struct tm *tm;
+
+ tm = gmtime(t);
+ if (tm != NULL) {
+ strftime(buf, buf_len, "%a, %d %b %Y %H:%M:%S GMT", tm);
+ } else {
+ strncpy(buf, "Thu, 01 Jan 1970 00:00:00 GMT", buf_len);
+ buf[buf_len - 1] = '\0';
+ }
+}
+
+/* Print error message to the opened error log stream. */
+void mg_cry(struct mg_connection *conn, const char *fmt, ...)
+{
+ char buf[MG_BUF_LEN], src_addr[IP_ADDR_STR_LEN];
+ va_list ap;
+ FILE *fp;
+ time_t timestamp;
+
+ va_start(ap, fmt);
+ IGNORE_UNUSED_RESULT(vsnprintf(buf, sizeof(buf), fmt, ap));
+ va_end(ap);
+
+ /* Do not lock when getting the callback value, here and below.
+ I suppose this is fine, since function cannot disappear in the
+ same way string option can. */
+ if (conn->ctx->callbacks.log_message == NULL ||
+ conn->ctx->callbacks.log_message(conn, buf) == 0) {
+ fp = conn->ctx->config[ERROR_LOG_FILE] == NULL ? NULL :
+ fopen(conn->ctx->config[ERROR_LOG_FILE], "a+");
+
+ if (fp != NULL) {
+ flockfile(fp);
+ timestamp = time(NULL);
+
+ sockaddr_to_string(src_addr, sizeof(src_addr), &conn->client.rsa);
+ fprintf(fp, "[%010lu] [error] [client %s] ", (unsigned long) timestamp,
+ src_addr);
+
+ if (conn->request_info.request_method != NULL) {
+ fprintf(fp, "%s %s: ", conn->request_info.request_method,
+ conn->request_info.uri);
+ }
+
+ fprintf(fp, "%s", buf);
+ fputc('\n', fp);
+ funlockfile(fp);
+ fclose(fp);
+ }
+ }
+}
+
+/* Return fake connection structure. Used for logging, if connection
+ is not applicable at the moment of logging. */
+static struct mg_connection *fc(struct mg_context *ctx)
+{
+ static struct mg_connection fake_connection;
+ fake_connection.ctx = ctx;
+ return &fake_connection;
+}
+
+const char *mg_version(void)
+{
+ return CIVETWEB_VERSION;
+}
+
+struct mg_request_info *mg_get_request_info(struct mg_connection *conn)
+{
+ return &conn->request_info;
+}
+
+static void mg_strlcpy(register char *dst, register const char *src, size_t n)
+{
+ for (; *src != '\0' && n > 1; n--) {
+ *dst++ = *src++;
+ }
+ *dst = '\0';
+}
+
+static int lowercase(const char *s)
+{
+ return tolower(* (const unsigned char *) s);
+}
+
+int mg_strncasecmp(const char *s1, const char *s2, size_t len)
+{
+ int diff = 0;
+
+ if (len > 0)
+ do {
+ diff = lowercase(s1++) - lowercase(s2++);
+ } while (diff == 0 && s1[-1] != '\0' && --len > 0);
+
+ return diff;
+}
+
+static int mg_strcasecmp(const char *s1, const char *s2)
+{
+ int diff;
+
+ do {
+ diff = lowercase(s1++) - lowercase(s2++);
+ } while (diff == 0 && s1[-1] != '\0');
+
+ return diff;
+}
+
+static char * mg_strndup(const char *ptr, size_t len)
+{
+ char *p;
+
+ if ((p = (char *) malloc(len + 1)) != NULL) {
+ mg_strlcpy(p, ptr, len + 1);
+ }
+
+ return p;
+}
+
+static char * mg_strdup(const char *str)
+{
+ return mg_strndup(str, strlen(str));
+}
+
+static const char *mg_strcasestr(const char *big_str, const char *small_str)
+{
+ int i, big_len = (int)strlen(big_str), small_len = (int)strlen(small_str);
+
+ for (i = 0; i <= big_len - small_len; i++) {
+ if (mg_strncasecmp(big_str + i, small_str, small_len) == 0) {
+ return big_str + i;
+ }
+ }
+
+ return NULL;
+}
+
+/* Like snprintf(), but never returns negative value, or a value
+ that is larger than a supplied buffer.
+ Thanks to Adam Zeldis to pointing snprintf()-caused vulnerability
+ in his audit report. */
+static int mg_vsnprintf(struct mg_connection *conn, char *buf, size_t buflen,
+ const char *fmt, va_list ap)
+{
+ int n;
+
+ if (buflen == 0)
+ return 0;
+
+ n = vsnprintf(buf, buflen, fmt, ap);
+
+ if (n < 0) {
+ mg_cry(conn, "vsnprintf error");
+ n = 0;
+ } else if (n >= (int) buflen) {
+ mg_cry(conn, "truncating vsnprintf buffer: [%.*s]",
+ n > 200 ? 200 : n, buf);
+ n = (int) buflen - 1;
+ }
+ buf[n] = '\0';
+
+ return n;
+}
+
+static int mg_snprintf(struct mg_connection *conn, char *buf, size_t buflen,
+ PRINTF_FORMAT_STRING(const char *fmt), ...)
+PRINTF_ARGS(4, 5);
+
+static int mg_snprintf(struct mg_connection *conn, char *buf, size_t buflen,
+ const char *fmt, ...)
+{
+ va_list ap;
+ int n;
+
+ va_start(ap, fmt);
+ n = mg_vsnprintf(conn, buf, buflen, fmt, ap);
+ va_end(ap);
+
+ return n;
+}
+
+/* Skip the characters until one of the delimiters characters found.
+ 0-terminate resulting word. Skip the delimiter and following whitespaces.
+ Advance pointer to buffer to the next word. Return found 0-terminated word.
+ Delimiters can be quoted with quotechar. */
+static char *skip_quoted(char **buf, const char *delimiters,
+ const char *whitespace, char quotechar)
+{
+ char *p, *begin_word, *end_word, *end_whitespace;
+
+ begin_word = *buf;
+ end_word = begin_word + strcspn(begin_word, delimiters);
+
+ /* Check for quotechar */
+ if (end_word > begin_word) {
+ p = end_word - 1;
+ while (*p == quotechar) {
+ /* If there is anything beyond end_word, copy it */
+ if (*end_word == '\0') {
+ *p = '\0';
+ break;
+ } else {
+ size_t end_off = strcspn(end_word + 1, delimiters);
+ memmove (p, end_word, end_off + 1);
+ p += end_off; /* p must correspond to end_word - 1 */
+ end_word += end_off + 1;
+ }
+ }
+ for (p++; p < end_word; p++) {
+ *p = '\0';
+ }
+ }
+
+ if (*end_word == '\0') {
+ *buf = end_word;
+ } else {
+ end_whitespace = end_word + 1 + strspn(end_word + 1, whitespace);
+
+ for (p = end_word; p < end_whitespace; p++) {
+ *p = '\0';
+ }
+
+ *buf = end_whitespace;
+ }
+
+ return begin_word;
+}
+
+/* Simplified version of skip_quoted without quote char
+ and whitespace == delimiters */
+static char *skip(char **buf, const char *delimiters)
+{
+ return skip_quoted(buf, delimiters, delimiters, 0);
+}
+
+
+/* Return HTTP header value, or NULL if not found. */
+static const char *get_header(const struct mg_request_info *ri,
+ const char *name)
+{
+ int i;
+
+ for (i = 0; i < ri->num_headers; i++)
+ if (!mg_strcasecmp(name, ri->http_headers[i].name))
+ return ri->http_headers[i].value;
+
+ return NULL;
+}
+
+const char *mg_get_header(const struct mg_connection *conn, const char *name)
+{
+ return get_header(&conn->request_info, name);
+}
+
+/* A helper function for traversing a comma separated list of values.
+ It returns a list pointer shifted to the next value, or NULL if the end
+ of the list found.
+ Value is stored in val vector. If value has form "x=y", then eq_val
+ vector is initialized to point to the "y" part, and val vector length
+ is adjusted to point only to "x". */
+static const char *next_option(const char *list, struct vec *val,
+ struct vec *eq_val)
+{
+ if (list == NULL || *list == '\0') {
+ /* End of the list */
+ list = NULL;
+ } else {
+ val->ptr = list;
+ if ((list = strchr(val->ptr, ',')) != NULL) {
+ /* Comma found. Store length and shift the list ptr */
+ val->len = list - val->ptr;
+ list++;
+ } else {
+ /* This value is the last one */
+ list = val->ptr + strlen(val->ptr);
+ val->len = list - val->ptr;
+ }
+
+ if (eq_val != NULL) {
+ /* Value has form "x=y", adjust pointers and lengths
+ so that val points to "x", and eq_val points to "y". */
+ eq_val->len = 0;
+ eq_val->ptr = (const char *) memchr(val->ptr, '=', val->len);
+ if (eq_val->ptr != NULL) {
+ eq_val->ptr++; /* Skip over '=' character */
+ eq_val->len = val->ptr + val->len - eq_val->ptr;
+ val->len = (eq_val->ptr - val->ptr) - 1;
+ }
+ }
+ }
+
+ return list;
+}
+
+/* Perform case-insensitive match of string against pattern */
+static int match_prefix(const char *pattern, int pattern_len, const char *str)
+{
+ const char *or_str;
+ int i, j, len, res;
+
+ if ((or_str = (const char *) memchr(pattern, '|', pattern_len)) != NULL) {
+ res = match_prefix(pattern, (int)(or_str - pattern), str);
+ return res > 0 ? res :
+ match_prefix(or_str + 1, (int)((pattern + pattern_len) - (or_str + 1)), str);
+ }
+
+ i = j = 0;
+ res = -1;
+ for (; i < pattern_len; i++, j++) {
+ if (pattern[i] == '?' && str[j] != '\0') {
+ continue;
+ } else if (pattern[i] == '$') {
+ return str[j] == '\0' ? j : -1;
+ } else if (pattern[i] == '*') {
+ i++;
+ if (pattern[i] == '*') {
+ i++;
+ len = (int) strlen(str + j);
+ } else {
+ len = (int) strcspn(str + j, "/");
+ }
+ if (i == pattern_len) {
+ return j + len;
+ }
+ do {
+ res = match_prefix(pattern + i, pattern_len - i, str + j + len);
+ } while (res == -1 && len-- > 0);
+ return res == -1 ? -1 : j + res + len;
+ } else if (lowercase(&pattern[i]) != lowercase(&str[j])) {
+ return -1;
+ }
+ }
+ return j;
+}
+
+/* HTTP 1.1 assumes keep alive if "Connection:" header is not set
+ This function must tolerate situations when connection info is not
+ set up, for example if request parsing failed. */
+static int should_keep_alive(const struct mg_connection *conn)
+{
+ const char *http_version = conn->request_info.http_version;
+ const char *header = mg_get_header(conn, "Connection");
+ if (conn->must_close ||
+ conn->status_code == 401 ||
+ mg_strcasecmp(conn->ctx->config[ENABLE_KEEP_ALIVE], "yes") != 0 ||
+ (header != NULL && mg_strcasecmp(header, "keep-alive") != 0) ||
+ (header == NULL && http_version && strcmp(http_version, "1.1"))) {
+ return 0;
+ }
+ return 1;
+}
+
+static const char *suggest_connection_header(const struct mg_connection *conn)
+{
+ return should_keep_alive(conn) ? "keep-alive" : "close";
+}
+
+static void send_http_error(struct mg_connection *, int, const char *,
+ PRINTF_FORMAT_STRING(const char *fmt), ...)
+PRINTF_ARGS(4, 5);
+
+
+static void send_http_error(struct mg_connection *conn, int status,
+ const char *reason, const char *fmt, ...)
+{
+ char buf[MG_BUF_LEN];
+ va_list ap;
+ int len = 0;
+ char date[64];
+ time_t curtime = time(NULL);
+
+ conn->status_code = status;
+ if (conn->ctx->callbacks.http_error == NULL ||
+ conn->ctx->callbacks.http_error(conn, status)) {
+ buf[0] = '\0';
+
+ gmt_time_string(date, sizeof(date), &curtime);
+
+ /* Errors 1xx, 204 and 304 MUST NOT send a body */
+ if (status > 199 && status != 204 && status != 304) {
+ len = mg_snprintf(conn, buf, sizeof(buf), "Error %d: %s", status, reason);
+ buf[len++] = '\n';
+
+ va_start(ap, fmt);
+ len += mg_vsnprintf(conn, buf + len, sizeof(buf) - len, fmt, ap);
+ va_end(ap);
+ }
+ DEBUG_TRACE(("[%s]", buf));
+
+ mg_printf(conn, "HTTP/1.1 %d %s\r\n"
+ "Content-Length: %d\r\n"
+ "Date: %s\r\n"
+ "Connection: %s\r\n\r\n",
+ status, reason, len, date,
+ suggest_connection_header(conn));
+ conn->num_bytes_sent += mg_printf(conn, "%s", buf);
+ }
+}
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+static int pthread_mutex_init(pthread_mutex_t *mutex, void *unused)
+{
+ (void) unused;
+ *mutex = CreateMutex(NULL, FALSE, NULL);
+ return *mutex == NULL ? -1 : 0;
+}
+
+static int pthread_mutex_destroy(pthread_mutex_t *mutex)
+{
+ return CloseHandle(*mutex) == 0 ? -1 : 0;
+}
+
+static int pthread_mutex_lock(pthread_mutex_t *mutex)
+{
+ return WaitForSingleObject(*mutex, INFINITE) == WAIT_OBJECT_0? 0 : -1;
+}
+
+static int pthread_mutex_unlock(pthread_mutex_t *mutex)
+{
+ return ReleaseMutex(*mutex) == 0 ? -1 : 0;
+}
+
+static int clock_gettime(clockid_t clk_id, struct timespec *tp)
+{
+ FILETIME ft;
+ ULARGE_INTEGER li;
+ BOOL ok = FALSE;
+ double d;
+ static double perfcnt_per_sec = 0.0;
+
+ if (tp) {
+ if (clk_id == CLOCK_REALTIME) {
+ GetSystemTimeAsFileTime(&ft);
+ li.LowPart = ft.dwLowDateTime;
+ li.HighPart = ft.dwHighDateTime;
+ li.QuadPart -= 116444736000000000; /* 1.1.1970 in filedate */
+ tp->tv_sec = (time_t)(li.QuadPart / 10000000);
+ tp->tv_nsec = (long)(li.QuadPart % 10000000) * 100;
+ ok = TRUE;
+ } else if (clk_id == CLOCK_MONOTONIC) {
+ if (perfcnt_per_sec==0) {
+ QueryPerformanceFrequency((LARGE_INTEGER *) &li);
+ perfcnt_per_sec = 1.0 / li.QuadPart;
+ }
+ if (perfcnt_per_sec!=0) {
+ QueryPerformanceCounter((LARGE_INTEGER *) &li);
+ d = li.QuadPart * perfcnt_per_sec;
+ tp->tv_sec = (time_t)d;
+ d -= tp->tv_sec;
+ tp->tv_nsec = (long)(d*1.0E9);
+ ok = TRUE;
+ }
+ }
+ }
+
+ return ok ? 0 : -1;
+}
+
+static int pthread_cond_init(pthread_cond_t *cv, const void *unused)
+{
+ (void) unused;
+ InitializeCriticalSection(&cv->threadIdSec);
+ cv->waitingthreadcount = 0;
+ cv->waitingthreadhdls = calloc(MAX_WORKER_THREADS, sizeof(pthread_t));
+ return (cv->waitingthreadhdls!=NULL) ? 0 : -1;
+}
+
+static int pthread_cond_timedwait(pthread_cond_t *cv, pthread_mutex_t *mutex, const struct timespec * abstime)
+{
+ struct mg_workerTLS * tls = (struct mg_workerTLS *)TlsGetValue(sTlsKey);
+ int ok;
+ struct timespec tsnow;
+ int64_t nsnow, nswaitabs, nswaitrel;
+ DWORD mswaitrel;
+
+ EnterCriticalSection(&cv->threadIdSec);
+ assert(cv->waitingthreadcount < MAX_WORKER_THREADS);
+ cv->waitingthreadhdls[cv->waitingthreadcount] = tls->pthread_cond_helper_mutex;
+ cv->waitingthreadcount++;
+ LeaveCriticalSection(&cv->threadIdSec);
+
+ if (abstime) {
+ clock_gettime(CLOCK_REALTIME, &tsnow);
+ nsnow = (((uint64_t)tsnow.tv_sec)<<32) + tsnow.tv_nsec;
+ nswaitabs = (((uint64_t)abstime->tv_sec)<<32) + abstime->tv_nsec;
+ nswaitrel = nswaitabs - nsnow;
+ if (nswaitrel<0) nswaitrel=0;
+ mswaitrel = (DWORD)(nswaitrel / 1000000);
+ } else {
+ mswaitrel = INFINITE;
+ }
+
+ pthread_mutex_unlock(mutex);
+ ok = (WAIT_OBJECT_0 == WaitForSingleObject(tls->pthread_cond_helper_mutex, mswaitrel));
+ pthread_mutex_lock(mutex);
+
+ return ok ? 0 : -1;
+}
+
+static int pthread_cond_wait(pthread_cond_t *cv, pthread_mutex_t *mutex)
+{
+ return pthread_cond_timedwait(cv, mutex, NULL);
+}
+
+static int pthread_cond_signal(pthread_cond_t *cv)
+{
+ int i;
+ HANDLE wkup = NULL;
+ BOOL ok = FALSE;
+
+ EnterCriticalSection(&cv->threadIdSec);
+ if (cv->waitingthreadcount) {
+ wkup = cv->waitingthreadhdls[0];
+ ok = SetEvent(wkup);
+
+ for (i=1; i<cv->waitingthreadcount; i++) {
+ cv->waitingthreadhdls[i-1] = cv->waitingthreadhdls[i];
+ }
+ cv->waitingthreadcount--;
+
+ assert(ok);
+ }
+ LeaveCriticalSection(&cv->threadIdSec);
+
+ return ok ? 0 : 1;
+}
+
+static int pthread_cond_broadcast(pthread_cond_t *cv)
+{
+ EnterCriticalSection(&cv->threadIdSec);
+ while (cv->waitingthreadcount) {
+ pthread_cond_signal(cv);
+ }
+ LeaveCriticalSection(&cv->threadIdSec);
+
+ return 0;
+}
+
+static int pthread_cond_destroy(pthread_cond_t *cv)
+{
+ EnterCriticalSection(&cv->threadIdSec);
+ assert(cv->waitingthreadcount==0);
+ cv->waitingthreadhdls = 0;
+ free(cv->waitingthreadhdls);
+ LeaveCriticalSection(&cv->threadIdSec);
+ DeleteCriticalSection(&cv->threadIdSec);
+
+ return 0;
+}
+
+/* For Windows, change all slashes to backslashes in path names. */
+static void change_slashes_to_backslashes(char *path)
+{
+ int i;
+
+ for (i = 0; path[i] != '\0'; i++) {
+ if (path[i] == '/')
+ path[i] = '\\';
+ /* i > 0 check is to preserve UNC paths, like \\server\file.txt */
+ if (path[i] == '\\' && i > 0)
+ while (path[i + 1] == '\\' || path[i + 1] == '/')
+ (void) memmove(path + i + 1,
+ path + i + 2, strlen(path + i + 1));
+ }
+}
+
+/* Encode 'path' which is assumed UTF-8 string, into UNICODE string.
+ wbuf and wbuf_len is a target buffer and its length. */
+static void to_unicode(const char *path, wchar_t *wbuf, size_t wbuf_len)
+{
+ char buf[PATH_MAX], buf2[PATH_MAX];
+
+ mg_strlcpy(buf, path, sizeof(buf));
+ change_slashes_to_backslashes(buf);
+
+ /* Convert to Unicode and back. If doubly-converted string does not
+ match the original, something is fishy, reject. */
+ memset(wbuf, 0, wbuf_len * sizeof(wchar_t));
+ MultiByteToWideChar(CP_UTF8, 0, buf, -1, wbuf, (int) wbuf_len);
+ WideCharToMultiByte(CP_UTF8, 0, wbuf, (int) wbuf_len, buf2, sizeof(buf2),
+ NULL, NULL);
+ if (strcmp(buf, buf2) != 0) {
+ wbuf[0] = L'\0';
+ }
+}
+
+#if defined(_WIN32_WCE)
+static time_t time(time_t *ptime)
+{
+ time_t t;
+ SYSTEMTIME st;
+ FILETIME ft;
+
+ GetSystemTime(&st);
+ SystemTimeToFileTime(&st, &ft);
+ t = SYS2UNIX_TIME(ft.dwLowDateTime, ft.dwHighDateTime);
+
+ if (ptime != NULL) {
+ *ptime = t;
+ }
+
+ return t;
+}
+
+static struct tm *localtime(const time_t *ptime, struct tm *ptm)
+{
+ int64_t t = ((int64_t) *ptime) * RATE_DIFF + EPOCH_DIFF;
+ FILETIME ft, lft;
+ SYSTEMTIME st;
+ TIME_ZONE_INFORMATION tzinfo;
+
+ if (ptm == NULL) {
+ return NULL;
+ }
+
+ * (int64_t *) &ft = t;
+ FileTimeToLocalFileTime(&ft, &lft);
+ FileTimeToSystemTime(&lft, &st);
+ ptm->tm_year = st.wYear - 1900;
+ ptm->tm_mon = st.wMonth - 1;
+ ptm->tm_wday = st.wDayOfWeek;
+ ptm->tm_mday = st.wDay;
+ ptm->tm_hour = st.wHour;
+ ptm->tm_min = st.wMinute;
+ ptm->tm_sec = st.wSecond;
+ ptm->tm_yday = 0; /* hope nobody uses this */
+ ptm->tm_isdst =
+ GetTimeZoneInformation(&tzinfo) == TIME_ZONE_ID_DAYLIGHT ? 1 : 0;
+
+ return ptm;
+}
+
+static struct tm *gmtime(const time_t *ptime, struct tm *ptm)
+{
+ /* FIXME(lsm): fix this. */
+ return localtime(ptime, ptm);
+}
+
+static size_t strftime(char *dst, size_t dst_size, const char *fmt,
+ const struct tm *tm)
+{
+ (void) snprintf(dst, dst_size, "implement strftime() for WinCE");
+ return 0;
+}
+#endif
+
+/* Windows happily opens files with some garbage at the end of file name.
+ For example, fopen("a.cgi ", "r") on Windows successfully opens
+ "a.cgi", despite one would expect an error back.
+ This function returns non-0 if path ends with some garbage. */
+static int path_cannot_disclose_cgi(const char *path)
+{
+ static const char *allowed_last_characters = "_-";
+ int last = path[strlen(path) - 1];
+ return isalnum(last) || strchr(allowed_last_characters, last) != NULL;
+}
+
+static int mg_stat(struct mg_connection *conn, const char *path,
+ struct file *filep)
+{
+ wchar_t wbuf[PATH_MAX];
+ WIN32_FILE_ATTRIBUTE_DATA info;
+
+ if (!is_file_in_memory(conn, path, filep)) {
+ to_unicode(path, wbuf, ARRAY_SIZE(wbuf));
+ if (GetFileAttributesExW(wbuf, GetFileExInfoStandard, &info) != 0) {
+ filep->size = MAKEUQUAD(info.nFileSizeLow, info.nFileSizeHigh);
+ filep->modification_time = SYS2UNIX_TIME(
+ info.ftLastWriteTime.dwLowDateTime,
+ info.ftLastWriteTime.dwHighDateTime);
+ filep->is_directory = info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY;
+ /* If file name is fishy, reset the file structure and return
+ error.
+ Note it is important to reset, not just return the error, cause
+ functions like is_file_opened() check the struct. */
+ if (!filep->is_directory && !path_cannot_disclose_cgi(path)) {
+ memset(filep, 0, sizeof(*filep));
+ }
+ }
+ }
+
+ return filep->membuf != NULL || filep->modification_time != 0;
+}
+
+static int mg_remove(const char *path)
+{
+ wchar_t wbuf[PATH_MAX];
+ to_unicode(path, wbuf, ARRAY_SIZE(wbuf));
+ return DeleteFileW(wbuf) ? 0 : -1;
+}
+
+static int mg_mkdir(const char *path, int mode)
+{
+ char buf[PATH_MAX];
+ wchar_t wbuf[PATH_MAX];
+
+ (void) mode;
+ mg_strlcpy(buf, path, sizeof(buf));
+ change_slashes_to_backslashes(buf);
+
+ (void) MultiByteToWideChar(CP_UTF8, 0, buf, -1, wbuf, ARRAY_SIZE(wbuf));
+
+ return CreateDirectoryW(wbuf, NULL) ? 0 : -1;
+}
+
+/* Implementation of POSIX opendir/closedir/readdir for Windows. */
+static DIR * opendir(const char *name)
+{
+ DIR *dir = NULL;
+ wchar_t wpath[PATH_MAX];
+ DWORD attrs;
+
+ if (name == NULL) {
+ SetLastError(ERROR_BAD_ARGUMENTS);
+ } else if ((dir = (DIR *) malloc(sizeof(*dir))) == NULL) {
+ SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+ } else {
+ to_unicode(name, wpath, ARRAY_SIZE(wpath));
+ attrs = GetFileAttributesW(wpath);
+ if (attrs != 0xFFFFFFFF &&
+ ((attrs & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY)) {
+ (void) wcscat(wpath, L"\\*");
+ dir->handle = FindFirstFileW(wpath, &dir->info);
+ dir->result.d_name[0] = '\0';
+ } else {
+ free(dir);
+ dir = NULL;
+ }
+ }
+
+ return dir;
+}
+
+static int closedir(DIR *dir)
+{
+ int result = 0;
+
+ if (dir != NULL) {
+ if (dir->handle != INVALID_HANDLE_VALUE)
+ result = FindClose(dir->handle) ? 0 : -1;
+
+ free(dir);
+ } else {
+ result = -1;
+ SetLastError(ERROR_BAD_ARGUMENTS);
+ }
+
+ return result;
+}
+
+static struct dirent *readdir(DIR *dir)
+{
+ struct dirent *result = 0;
+
+ if (dir) {
+ if (dir->handle != INVALID_HANDLE_VALUE) {
+ result = &dir->result;
+ (void) WideCharToMultiByte(CP_UTF8, 0,
+ dir->info.cFileName, -1, result->d_name,
+ sizeof(result->d_name), NULL, NULL);
+
+ if (!FindNextFileW(dir->handle, &dir->info)) {
+ (void) FindClose(dir->handle);
+ dir->handle = INVALID_HANDLE_VALUE;
+ }
+
+ } else {
+ SetLastError(ERROR_FILE_NOT_FOUND);
+ }
+ } else {
+ SetLastError(ERROR_BAD_ARGUMENTS);
+ }
+
+ return result;
+}
+
+#ifndef HAVE_POLL
+static int poll(struct pollfd *pfd, int n, int milliseconds)
+{
+ struct timeval tv;
+ fd_set set;
+ int i, result;
+ SOCKET maxfd = 0;
+
+ tv.tv_sec = milliseconds / 1000;
+ tv.tv_usec = (milliseconds % 1000) * 1000;
+ FD_ZERO(&set);
+
+ for (i = 0; i < n; i++) {
+ FD_SET((SOCKET) pfd[i].fd, &set);
+ pfd[i].revents = 0;
+
+ if (pfd[i].fd > maxfd) {
+ maxfd = pfd[i].fd;
+ }
+ }
+
+ if ((result = select((int)maxfd + 1, &set, NULL, NULL, &tv)) > 0) {
+ for (i = 0; i < n; i++) {
+ if (FD_ISSET(pfd[i].fd, &set)) {
+ pfd[i].revents = POLLIN;
+ }
+ }
+ }
+
+ return result;
+}
+#endif /* HAVE_POLL */
+
+static void set_close_on_exec(SOCKET sock, struct mg_connection *conn /* may be null */)
+{
+ (void) conn; /* Unused. */
+ (void) SetHandleInformation((HANDLE) sock, HANDLE_FLAG_INHERIT, 0);
+}
+
+int mg_start_thread(mg_thread_func_t f, void *p)
+{
+#if defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1)
+ /* Compile-time option to control stack size, e.g. -DUSE_STACK_SIZE=16384 */
+ return (long)_beginthread((void (__cdecl *)(void *)) f, USE_STACK_SIZE, p) == -1L ? -1 : 0;
+#else
+ return (long)_beginthread((void (__cdecl *)(void *)) f, 0, p) == -1L ? -1 : 0;
+#endif /* defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) */
+}
+
+/* Start a thread storing the thread context. */
+
+static int mg_start_thread_with_id(unsigned (__stdcall *f)(void *), void *p,
+ pthread_t *threadidptr)
+{
+ uintptr_t uip;
+ HANDLE threadhandle;
+ int result;
+
+ uip = _beginthreadex(NULL, 0, (unsigned (__stdcall *)(void *)) f, p, 0,
+ NULL);
+ threadhandle = (HANDLE) uip;
+ if (threadidptr != NULL) {
+ *threadidptr = threadhandle;
+ }
+ result = (threadhandle == NULL) ? -1 : 0;
+
+ return result;
+}
+
+/* Wait for a thread to finish. */
+
+static int mg_join_thread(pthread_t threadid)
+{
+ int result;
+ DWORD dwevent;
+
+ result = -1;
+ dwevent = WaitForSingleObject(threadid, INFINITE);
+ if (dwevent == WAIT_FAILED) {
+ int err;
+
+ err = GetLastError();
+ DEBUG_TRACE(("WaitForSingleObject() failed, error %d", err));
+ } else {
+ if (dwevent == WAIT_OBJECT_0) {
+ CloseHandle(threadid);
+ result = 0;
+ }
+ }
+
+ return result;
+}
+
+static HANDLE dlopen(const char *dll_name, int flags)
+{
+ wchar_t wbuf[PATH_MAX];
+ (void) flags;
+ to_unicode(dll_name, wbuf, ARRAY_SIZE(wbuf));
+ return LoadLibraryW(wbuf);
+}
+
+static int dlclose(void *handle)
+{
+ int result;
+
+ if (FreeLibrary(handle) != 0) {
+ result = 0;
+ } else {
+ result = -1;
+ }
+
+ return result;
+}
+
+#if !defined(NO_CGI)
+#define SIGKILL 0
+static int kill(pid_t pid, int sig_num)
+{
+ (void) TerminateProcess(pid, sig_num);
+ (void) CloseHandle(pid);
+ return 0;
+}
+
+static void trim_trailing_whitespaces(char *s)
+{
+ char *e = s + strlen(s) - 1;
+ while (e > s && isspace(* (unsigned char *) e)) {
+ *e-- = '\0';
+ }
+}
+
+static pid_t spawn_process(struct mg_connection *conn, const char *prog,
+ char *envblk, char *envp[], int fdin,
+ int fdout, const char *dir)
+{
+ HANDLE me;
+ char *p, *interp, full_interp[PATH_MAX], full_dir[PATH_MAX],
+ cmdline[PATH_MAX], buf[PATH_MAX];
+ struct file file = STRUCT_FILE_INITIALIZER;
+ STARTUPINFOA si;
+ PROCESS_INFORMATION pi = { 0 };
+
+ (void) envp;
+
+ memset(&si, 0, sizeof(si));
+ si.cb = sizeof(si);
+
+ /* TODO(lsm): redirect CGI errors to the error log file */
+ si.dwFlags = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW;
+ si.wShowWindow = SW_HIDE;
+
+ me = GetCurrentProcess();
+ DuplicateHandle(me, (HANDLE) _get_osfhandle(fdin), me,
+ &si.hStdInput, 0, TRUE, DUPLICATE_SAME_ACCESS);
+ DuplicateHandle(me, (HANDLE) _get_osfhandle(fdout), me,
+ &si.hStdOutput, 0, TRUE, DUPLICATE_SAME_ACCESS);
+
+ /* If CGI file is a script, try to read the interpreter line */
+ interp = conn->ctx->config[CGI_INTERPRETER];
+ if (interp == NULL) {
+ buf[0] = buf[1] = '\0';
+
+ /* Read the first line of the script into the buffer */
+ snprintf(cmdline, sizeof(cmdline), "%s%c%s", dir, '/', prog);
+ if (mg_fopen(conn, cmdline, "r", &file)) {
+ p = (char *) file.membuf;
+ mg_fgets(buf, sizeof(buf), &file, &p);
+ mg_fclose(&file);
+ buf[sizeof(buf) - 1] = '\0';
+ }
+
+ if (buf[0] == '#' && buf[1] == '!') {
+ trim_trailing_whitespaces(buf + 2);
+ } else {
+ buf[2] = '\0';
+ }
+ interp = buf + 2;
+ }
+
+ if (interp[0] != '\0') {
+ GetFullPathNameA(interp, sizeof(full_interp), full_interp, NULL);
+ interp = full_interp;
+ }
+ GetFullPathNameA(dir, sizeof(full_dir), full_dir, NULL);
+
+ mg_snprintf(conn, cmdline, sizeof(cmdline), "%s%s\"%s\\%s\"",
+ interp, interp[0] == '\0' ? "" : " ", full_dir, prog);
+
+ DEBUG_TRACE(("Running [%s]", cmdline));
+ if (CreateProcessA(NULL, cmdline, NULL, NULL, TRUE,
+ CREATE_NEW_PROCESS_GROUP, envblk, NULL, &si, &pi) == 0) {
+ mg_cry(conn, "%s: CreateProcess(%s): %ld",
+ __func__, cmdline, ERRNO);
+ pi.hProcess = (pid_t) -1;
+ }
+
+ (void) CloseHandle(si.hStdOutput);
+ (void) CloseHandle(si.hStdInput);
+ if (pi.hThread != NULL)
+ (void) CloseHandle(pi.hThread);
+
+ return (pid_t) pi.hProcess;
+}
+#endif /* !NO_CGI */
+
+static int set_non_blocking_mode(SOCKET sock)
+{
+ unsigned long on = 1;
+ return ioctlsocket(sock, FIONBIO, &on);
+}
+
+#else
+static int mg_stat(struct mg_connection *conn, const char *path,
+ struct file *filep)
+{
+ struct stat st;
+
+ if (!is_file_in_memory(conn, path, filep) && !stat(path, &st)) {
+ filep->size = st.st_size;
+ filep->modification_time = st.st_mtime;
+ filep->is_directory = S_ISDIR(st.st_mode);
+ } else {
+ filep->modification_time = (time_t) 0;
+ }
+
+ return filep->membuf != NULL || filep->modification_time != (time_t) 0;
+}
+
+static void set_close_on_exec(int fd, struct mg_connection *conn /* may be null */)
+{
+ if (fcntl(fd, F_SETFD, FD_CLOEXEC) != 0) {
+ if (conn)
+ mg_cry(conn, "%s: fcntl(F_SETFD FD_CLOEXEC) failed: %s",
+ __func__, strerror(ERRNO));
+ }
+}
+
+int mg_start_thread(mg_thread_func_t func, void *param)
+{
+ pthread_t thread_id;
+ pthread_attr_t attr;
+ int result;
+
+ (void) pthread_attr_init(&attr);
+ (void) pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+
+#if defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1)
+ /* Compile-time option to control stack size,
+ e.g. -DUSE_STACK_SIZE=16384 */
+ (void) pthread_attr_setstacksize(&attr, USE_STACK_SIZE);
+#endif /* defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) */
+
+ result = pthread_create(&thread_id, &attr, func, param);
+ pthread_attr_destroy(&attr);
+
+ return result;
+}
+
+/* Start a thread storing the thread context. */
+
+static int mg_start_thread_with_id(mg_thread_func_t func, void *param,
+ pthread_t *threadidptr)
+{
+ pthread_t thread_id;
+ pthread_attr_t attr;
+ int result;
+
+ (void) pthread_attr_init(&attr);
+
+#if defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1)
+ /* Compile-time option to control stack size,
+ e.g. -DUSE_STACK_SIZE=16384 */
+ (void) pthread_attr_setstacksize(&attr, USE_STACK_SIZE);
+#endif /* defined(USE_STACK_SIZE) && USE_STACK_SIZE > 1 */
+
+ result = pthread_create(&thread_id, &attr, func, param);
+ pthread_attr_destroy(&attr);
+ if (threadidptr != NULL) {
+ *threadidptr = thread_id;
+ }
+ return result;
+}
+
+/* Wait for a thread to finish. */
+
+static int mg_join_thread(pthread_t threadid)
+{
+ int result;
+
+ result = pthread_join(threadid, NULL);
+ return result;
+}
+
+#ifndef NO_CGI
+static pid_t spawn_process(struct mg_connection *conn, const char *prog,
+ char *envblk, char *envp[], int fdin,
+ int fdout, const char *dir)
+{
+ pid_t pid;
+ const char *interp;
+
+ (void) envblk;
+
+ if ((pid = fork()) == -1) {
+ /* Parent */
+ send_http_error(conn, 500, http_500_error, "fork(): %s", strerror(ERRNO));
+ } else if (pid == 0) {
+ /* Child */
+ if (chdir(dir) != 0) {
+ mg_cry(conn, "%s: chdir(%s): %s", __func__, dir, strerror(ERRNO));
+ } else if (dup2(fdin, 0) == -1) {
+ mg_cry(conn, "%s: dup2(%d, 0): %s", __func__, fdin, strerror(ERRNO));
+ } else if (dup2(fdout, 1) == -1) {
+ mg_cry(conn, "%s: dup2(%d, 1): %s", __func__, fdout, strerror(ERRNO));
+ } else {
+ /* Not redirecting stderr to stdout, to avoid output being littered
+ with the error messages. */
+ (void) close(fdin);
+ (void) close(fdout);
+
+ /* After exec, all signal handlers are restored to their default
+ values, with one exception of SIGCHLD. According to
+ POSIX.1-2001 and Linux's implementation, SIGCHLD's handler will
+ leave unchanged after exec if it was set to be ignored. Restore
+ it to default action. */
+ signal(SIGCHLD, SIG_DFL);
+
+ interp = conn->ctx->config[CGI_INTERPRETER];
+ if (interp == NULL) {
+ (void) execle(prog, prog, NULL, envp);
+ mg_cry(conn, "%s: execle(%s): %s", __func__, prog, strerror(ERRNO));
+ } else {
+ (void) execle(interp, interp, prog, NULL, envp);
+ mg_cry(conn, "%s: execle(%s %s): %s", __func__, interp, prog,
+ strerror(ERRNO));
+ }
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ return pid;
+}
+#endif /* !NO_CGI */
+
+static int set_non_blocking_mode(SOCKET sock)
+{
+ int flags;
+
+ flags = fcntl(sock, F_GETFL, 0);
+ (void) fcntl(sock, F_SETFL, flags | O_NONBLOCK);
+
+ return 0;
+}
+#endif /* _WIN32 */
+
+/* Write data to the IO channel - opened file descriptor, socket or SSL
+ descriptor. Return number of bytes written. */
+static int64_t push(FILE *fp, SOCKET sock, SSL *ssl, const char *buf,
+ int64_t len)
+{
+ int64_t sent;
+ int n, k;
+
+ (void) ssl; /* Get rid of warning */
+ sent = 0;
+ while (sent < len) {
+
+ /* How many bytes we send in this iteration */
+ k = len - sent > INT_MAX ? INT_MAX : (int) (len - sent);
+
+#ifndef NO_SSL
+ if (ssl != NULL) {
+ n = SSL_write(ssl, buf + sent, k);
+ } else
+#endif
+ if (fp != NULL) {
+ n = (int) fwrite(buf + sent, 1, (size_t) k, fp);
+ if (ferror(fp))
+ n = -1;
+ } else {
+ n = send(sock, buf + sent, (size_t) k, MSG_NOSIGNAL);
+ }
+
+ if (n <= 0)
+ break;
+
+ sent += n;
+ }
+
+ return sent;
+}
+
+/* Read from IO channel - opened file descriptor, socket, or SSL descriptor.
+ Return negative value on error, or number of bytes read on success. */
+static int pull(FILE *fp, struct mg_connection *conn, char *buf, int len)
+{
+ int nread;
+
+ if (fp != NULL) {
+ /* Use read() instead of fread(), because if we're reading from the
+ CGI pipe, fread() may block until IO buffer is filled up. We cannot
+ afford to block and must pass all read bytes immediately to the
+ client. */
+ nread = read(fileno(fp), buf, (size_t) len);
+#ifndef NO_SSL
+ } else if (conn->ssl != NULL) {
+ nread = SSL_read(conn->ssl, buf, len);
+#endif
+ } else {
+ nread = recv(conn->client.sock, buf, (size_t) len, 0);
+ }
+
+ return conn->ctx->stop_flag ? -1 : nread;
+}
+
+static int pull_all(FILE *fp, struct mg_connection *conn, char *buf, int len)
+{
+ int n, nread = 0;
+
+ while (len > 0 && conn->ctx->stop_flag == 0) {
+ n = pull(fp, conn, buf + nread, len);
+ if (n < 0) {
+ nread = n; /* Propagate the error */
+ break;
+ } else if (n == 0) {
+ break; /* No more data to read */
+ } else {
+ conn->consumed_content += n;
+ nread += n;
+ len -= n;
+ }
+ }
+
+ return nread;
+}
+
+int mg_read(struct mg_connection *conn, void *buf, size_t len)
+{
+ int n, buffered_len, nread;
+ const char *body;
+
+ /* If Content-Length is not set, read until socket is closed */
+ if (conn->consumed_content == 0 && conn->content_len == 0) {
+ conn->content_len = INT64_MAX;
+ conn->must_close = 1;
+ }
+
+ nread = 0;
+ if (conn->consumed_content < conn->content_len) {
+ /* Adjust number of bytes to read. */
+ int64_t to_read = conn->content_len - conn->consumed_content;
+ if (to_read < (int64_t) len) {
+ len = (size_t) to_read;
+ }
+
+ /* Return buffered data */
+ body = conn->buf + conn->request_len + conn->consumed_content;
+ buffered_len = (int)(&conn->buf[conn->data_len] - body);
+ if (buffered_len > 0) {
+ if (len < (size_t) buffered_len) {
+ buffered_len = (int) len;
+ }
+ memcpy(buf, body, (size_t) buffered_len);
+ len -= buffered_len;
+ conn->consumed_content += buffered_len;
+ nread += buffered_len;
+ buf = (char *) buf + buffered_len;
+ }
+
+ /* We have returned all buffered data. Read new data from the remote
+ socket. */
+ n = pull_all(NULL, conn, (char *) buf, (int) len);
+ nread = n >= 0 ? nread + n : n;
+ }
+ return nread;
+}
+
+int mg_write(struct mg_connection *conn, const void *buf, size_t len)
+{
+ time_t now;
+ int64_t n, total, allowed;
+
+ if (conn->throttle > 0) {
+ if ((now = time(NULL)) != conn->last_throttle_time) {
+ conn->last_throttle_time = now;
+ conn->last_throttle_bytes = 0;
+ }
+ allowed = conn->throttle - conn->last_throttle_bytes;
+ if (allowed > (int64_t) len) {
+ allowed = len;
+ }
+ if ((total = push(NULL, conn->client.sock, conn->ssl, (const char *) buf,
+ (int64_t) allowed)) == allowed) {
+ buf = (char *) buf + total;
+ conn->last_throttle_bytes += total;
+ while (total < (int64_t) len && conn->ctx->stop_flag == 0) {
+ allowed = conn->throttle > (int64_t) len - total ?
+ (int64_t) len - total : conn->throttle;
+ if ((n = push(NULL, conn->client.sock, conn->ssl, (const char *) buf,
+ (int64_t) allowed)) != allowed) {
+ break;
+ }
+ sleep(1);
+ conn->last_throttle_bytes = allowed;
+ conn->last_throttle_time = time(NULL);
+ buf = (char *) buf + n;
+ total += n;
+ }
+ }
+ } else {
+ total = push(NULL, conn->client.sock, conn->ssl, (const char *) buf,
+ (int64_t) len);
+ }
+ return (int) total;
+}
+
+/* Alternative alloc_vprintf() for non-compliant C runtimes */
+static int alloc_vprintf2(char **buf, const char *fmt, va_list ap)
+{
+ va_list ap_copy;
+ int size = MG_BUF_LEN;
+ int len = -1;
+
+ *buf = NULL;
+ while (len == -1) {
+ if (*buf) free(*buf);
+ *buf = (char *)malloc(size *= 4);
+ if (!*buf) break;
+ va_copy(ap_copy, ap);
+ len = vsnprintf(*buf, size, fmt, ap_copy);
+ va_end(ap_copy);
+ }
+
+ return len;
+}
+
+/* Print message to buffer. If buffer is large enough to hold the message,
+ return buffer. If buffer is to small, allocate large enough buffer on heap,
+ and return allocated buffer. */
+static int alloc_vprintf(char **buf, size_t size, const char *fmt, va_list ap)
+{
+ va_list ap_copy;
+ int len;
+
+ /* Windows is not standard-compliant, and vsnprintf() returns -1 if
+ buffer is too small. Also, older versions of msvcrt.dll do not have
+ _vscprintf(). However, if size is 0, vsnprintf() behaves correctly.
+ Therefore, we make two passes: on first pass, get required message
+ length.
+ On second pass, actually print the message. */
+ va_copy(ap_copy, ap);
+ len = vsnprintf(NULL, 0, fmt, ap_copy);
+ va_end(ap_copy);
+
+ if (len < 0) {
+ /* C runtime is not standard compliant, vsnprintf() returned -1.
+ Switch to alternative code path that uses incremental allocations.
+ */
+ va_copy(ap_copy, ap);
+ len = alloc_vprintf2(buf, fmt, ap);
+ va_end(ap_copy);
+ } else if (len > (int) size &&
+ (size = len + 1) > 0 &&
+ (*buf = (char *) malloc(size)) == NULL) {
+ len = -1; /* Allocation failed, mark failure */
+ } else {
+ va_copy(ap_copy, ap);
+ IGNORE_UNUSED_RESULT(vsnprintf(*buf, size, fmt, ap_copy));
+ va_end(ap_copy);
+ }
+
+ return len;
+}
+
+int mg_vprintf(struct mg_connection *conn, const char *fmt, va_list ap);
+
+int mg_vprintf(struct mg_connection *conn, const char *fmt, va_list ap)
+{
+ char mem[MG_BUF_LEN], *buf = mem;
+ int len;
+
+ if ((len = alloc_vprintf(&buf, sizeof(mem), fmt, ap)) > 0) {
+ len = mg_write(conn, buf, (size_t) len);
+ }
+ if (buf != mem && buf != NULL) {
+ free(buf);
+ }
+
+ return len;
+}
+
+int mg_printf(struct mg_connection *conn, const char *fmt, ...)
+{
+ va_list ap;
+ int result;
+
+ va_start(ap, fmt);
+ result = mg_vprintf(conn, fmt, ap);
+ va_end(ap);
+
+ return result;
+}
+
+int mg_url_decode(const char *src, int src_len, char *dst,
+ int dst_len, int is_form_url_encoded)
+{
+ int i, j, a, b;
+#define HEXTOI(x) (isdigit(x) ? x - '0' : x - 'W')
+
+ for (i = j = 0; i < src_len && j < dst_len - 1; i++, j++) {
+ if (src[i] == '%' && i < src_len - 2 &&
+ isxdigit(* (const unsigned char *) (src + i + 1)) &&
+ isxdigit(* (const unsigned char *) (src + i + 2))) {
+ a = tolower(* (const unsigned char *) (src + i + 1));
+ b = tolower(* (const unsigned char *) (src + i + 2));
+ dst[j] = (char) ((HEXTOI(a) << 4) | HEXTOI(b));
+ i += 2;
+ } else if (is_form_url_encoded && src[i] == '+') {
+ dst[j] = ' ';
+ } else {
+ dst[j] = src[i];
+ }
+ }
+
+ dst[j] = '\0'; /* Null-terminate the destination */
+
+ return i >= src_len ? j : -1;
+}
+
+int mg_get_var(const char *data, size_t data_len, const char *name,
+ char *dst, size_t dst_len)
+{
+ return mg_get_var2(data,data_len,name,dst,dst_len,0);
+}
+
+int mg_get_var2(const char *data, size_t data_len, const char *name,
+ char *dst, size_t dst_len, size_t occurrence)
+{
+ const char *p, *e, *s;
+ size_t name_len;
+ int len;
+
+ if (dst == NULL || dst_len == 0) {
+ len = -2;
+ } else if (data == NULL || name == NULL || data_len == 0) {
+ len = -1;
+ dst[0] = '\0';
+ } else {
+ name_len = strlen(name);
+ e = data + data_len;
+ len = -1;
+ dst[0] = '\0';
+
+ /* data is "var1=val1&var2=val2...". Find variable first */
+ for (p = data; p + name_len < e; p++) {
+ if ((p == data || p[-1] == '&') && p[name_len] == '=' &&
+ !mg_strncasecmp(name, p, name_len) && 0 == occurrence--) {
+
+ /* Point p to variable value */
+ p += name_len + 1;
+
+ /* Point s to the end of the value */
+ s = (const char *) memchr(p, '&', (size_t)(e - p));
+ if (s == NULL) {
+ s = e;
+ }
+ assert(s >= p);
+
+ /* Decode variable into destination buffer */
+ len = mg_url_decode(p, (int)(s - p), dst, (int)dst_len, 1);
+
+ /* Redirect error code from -1 to -2 (destination buffer too
+ small). */
+ if (len == -1) {
+ len = -2;
+ }
+ break;
+ }
+ }
+ }
+
+ return len;
+}
+
+int mg_get_cookie(const char *cookie_header, const char *var_name,
+ char *dst, size_t dst_size)
+{
+ const char *s, *p, *end;
+ int name_len, len = -1;
+
+ if (dst == NULL || dst_size == 0) {
+ len = -2;
+ } else if (var_name == NULL || (s = cookie_header) == NULL) {
+ len = -1;
+ dst[0] = '\0';
+ } else {
+ name_len = (int) strlen(var_name);
+ end = s + strlen(s);
+ dst[0] = '\0';
+
+ for (; (s = mg_strcasestr(s, var_name)) != NULL; s += name_len) {
+ if (s[name_len] == '=') {
+ s += name_len + 1;
+ if ((p = strchr(s, ' ')) == NULL)
+ p = end;
+ if (p[-1] == ';')
+ p--;
+ if (*s == '"' && p[-1] == '"' && p > s + 1) {
+ s++;
+ p--;
+ }
+ if ((size_t) (p - s) < dst_size) {
+ len = (int)(p - s);
+ mg_strlcpy(dst, s, (size_t) len + 1);
+ } else {
+ len = -3;
+ }
+ break;
+ }
+ }
+ }
+ return len;
+}
+
+static void convert_uri_to_file_name(struct mg_connection *conn, char *buf,
+ size_t buf_len, struct file *filep,
+ int * is_script_ressource)
+{
+ struct vec a, b;
+ const char *rewrite, *uri = conn->request_info.uri,
+ *root = conn->ctx->config[DOCUMENT_ROOT];
+ char *p;
+ int match_len;
+ char gz_path[PATH_MAX];
+ char const* accept_encoding;
+
+ *is_script_ressource = 0;
+
+#if defined(USE_WEBSOCKET)
+ if (is_websocket_request(conn) && conn->ctx->config[WEBSOCKET_ROOT]) {
+ root = conn->ctx->config[WEBSOCKET_ROOT];
+ }
+#endif
+
+ /* Using buf_len - 1 because memmove() for PATH_INFO may shift part
+ of the path one byte on the right.
+ If document_root is NULL, leave the file empty. */
+ mg_snprintf(conn, buf, buf_len - 1, "%s%s",
+ root == NULL ? "" : root,
+ root == NULL ? "" : uri);
+
+ rewrite = conn->ctx->config[REWRITE];
+ while ((rewrite = next_option(rewrite, &a, &b)) != NULL) {
+ if ((match_len = match_prefix(a.ptr, (int) a.len, uri)) > 0) {
+ mg_snprintf(conn, buf, buf_len - 1, "%.*s%s", (int) b.len, b.ptr,
+ uri + match_len);
+ break;
+ }
+ }
+
+ if (mg_stat(conn, buf, filep)) return;
+
+ /* if we can't find the actual file, look for the file
+ with the same name but a .gz extension. If we find it,
+ use that and set the gzipped flag in the file struct
+ to indicate that the response need to have the content-
+ encoding: gzip header
+ we can only do this if the browser declares support */
+ if ((accept_encoding = mg_get_header(conn, "Accept-Encoding")) != NULL) {
+ if (strstr(accept_encoding,"gzip") != NULL) {
+ snprintf(gz_path, sizeof(gz_path), "%s.gz", buf);
+ if (mg_stat(conn, gz_path, filep)) {
+ filep->gzipped = 1;
+ return;
+ }
+ }
+ }
+
+ /* Support PATH_INFO for CGI scripts. */
+ for (p = buf + strlen(buf); p > buf + 1; p--) {
+ if (*p == '/') {
+ *p = '\0';
+ if ((match_prefix(conn->ctx->config[CGI_EXTENSIONS],
+ (int)strlen(conn->ctx->config[CGI_EXTENSIONS]), buf) > 0
+#ifdef USE_LUA
+ ||
+ match_prefix(conn->ctx->config[LUA_SCRIPT_EXTENSIONS],
+ (int)strlen(conn->ctx->config[LUA_SCRIPT_EXTENSIONS]), buf) > 0
+#endif
+ ) && mg_stat(conn, buf, filep)) {
+ /* Shift PATH_INFO block one character right, e.g.
+ "/x.cgi/foo/bar\x00" => "/x.cgi\x00/foo/bar\x00"
+ conn->path_info is pointing to the local variable "path"
+ declared in handle_request(), so PATH_INFO is not valid
+ after handle_request returns. */
+ conn->path_info = p + 1;
+ memmove(p + 2, p + 1, strlen(p + 1) + 1); /* +1 is for
+ trailing \0 */
+ p[1] = '/';
+ *is_script_ressource = 1;
+ break;
+ } else {
+ *p = '/';
+ }
+ }
+ }
+}
+
+/* Check whether full request is buffered. Return:
+ -1 if request is malformed
+ 0 if request is not yet fully buffered
+ >0 actual request length, including last \r\n\r\n */
+static int get_request_len(const char *buf, int buflen)
+{
+ const char *s, *e;
+ int len = 0;
+
+ for (s = buf, e = s + buflen - 1; len <= 0 && s < e; s++)
+ /* Control characters are not allowed but >=128 is. */
+ if (!isprint(* (const unsigned char *) s) && *s != '\r' &&
+ *s != '\n' && * (const unsigned char *) s < 128) {
+ len = -1;
+ break; /* [i_a] abort scan as soon as one malformed character is
+ found; */
+ /* don't let subsequent \r\n\r\n win us over anyhow */
+ } else if (s[0] == '\n' && s[1] == '\n') {
+ len = (int) (s - buf) + 2;
+ } else if (s[0] == '\n' && &s[1] < e &&
+ s[1] == '\r' && s[2] == '\n') {
+ len = (int) (s - buf) + 3;
+ }
+
+ return len;
+}
+
+/* Convert month to the month number. Return -1 on error, or month number */
+static int get_month_index(const char *s)
+{
+ size_t i;
+
+ for (i = 0; i < ARRAY_SIZE(month_names); i++)
+ if (!strcmp(s, month_names[i]))
+ return (int) i;
+
+ return -1;
+}
+
+static int num_leap_years(int year)
+{
+ return year / 4 - year / 100 + year / 400;
+}
+
+/* Parse UTC date-time string, and return the corresponding time_t value. */
+static time_t parse_date_string(const char *datetime)
+{
+ static const unsigned short days_before_month[] = {
+ 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334
+ };
+ char month_str[32];
+ int second, minute, hour, day, month, year, leap_days, days;
+ time_t result = (time_t) 0;
+
+ if (((sscanf(datetime, "%d/%3s/%d %d:%d:%d",
+ &day, month_str, &year, &hour, &minute, &second) == 6) ||
+ (sscanf(datetime, "%d %3s %d %d:%d:%d",
+ &day, month_str, &year, &hour, &minute, &second) == 6) ||
+ (sscanf(datetime, "%*3s, %d %3s %d %d:%d:%d",
+ &day, month_str, &year, &hour, &minute, &second) == 6) ||
+ (sscanf(datetime, "%d-%3s-%d %d:%d:%d",
+ &day, month_str, &year, &hour, &minute, &second) == 6)) &&
+ year > 1970 &&
+ (month = get_month_index(month_str)) != -1) {
+ leap_days = num_leap_years(year) - num_leap_years(1970);
+ year -= 1970;
+ days = year * 365 + days_before_month[month] + (day - 1) + leap_days;
+ result = (time_t) days * 24 * 3600 + (time_t) hour * 3600 +
+ minute * 60 + second;
+ }
+
+ return result;
+}
+
+/* Protect against directory disclosure attack by removing '..',
+ excessive '/' and '\' characters */
+static void remove_double_dots_and_double_slashes(char *s)
+{
+ char *p = s;
+
+ while (*s != '\0') {
+ *p++ = *s++;
+ if (s[-1] == '/' || s[-1] == '\\') {
+ /* Skip all following slashes, backslashes and double-dots */
+ while (s[0] != '\0') {
+ if (s[0] == '/' || s[0] == '\\') {
+ s++;
+ } else if (s[0] == '.' && s[1] == '.') {
+ s += 2;
+ } else {
+ break;
+ }
+ }
+ }
+ }
+ *p = '\0';
+}
+
+static const struct {
+ const char *extension;
+ size_t ext_len;
+ const char *mime_type;
+} builtin_mime_types[] = {
+ /* IANA registered MIME types (http://www.iana.org/assignments/media-types)
+ application types */
+ {".doc", 4, "application/msword"},
+ {".eps", 4, "application/postscript"},
+ {".exe", 4, "application/octet-stream"},
+ {".js", 3, "application/javascript"},
+ {".json", 5, "application/json"},
+ {".pdf", 4, "application/pdf"},
+ {".ps", 3, "application/postscript"},
+ {".rtf", 4, "application/rtf"},
+ {".xhtml", 6, "application/xhtml+xml"},
+ {".xsl", 4, "application/xml"},
+ {".xslt", 5, "application/xml"},
+
+ /* audio */
+ {".mp3", 4, "audio/mpeg"},
+ {".oga", 4, "audio/ogg"},
+ {".ogg", 4, "audio/ogg"},
+
+ /* image */
+ {".gif", 4, "image/gif"},
+ {".ief", 4, "image/ief"},
+ {".jpeg", 5, "image/jpeg"},
+ {".jpg", 4, "image/jpeg"},
+ {".jpm", 4, "image/jpm"},
+ {".jpx", 4, "image/jpx"},
+ {".png", 4, "image/png"},
+ {".svg", 4, "image/svg+xml"},
+ {".tif", 4, "image/tiff"},
+ {".tiff", 5, "image/tiff"},
+
+ /* model */
+ {".wrl", 4, "model/vrml"},
+
+ /* text */
+ {".css", 4, "text/css"},
+ {".csv", 4, "text/csv"},
+ {".htm", 4, "text/html"},
+ {".html", 5, "text/html"},
+ {".sgm", 4, "text/sgml"},
+ {".shtm", 5, "text/html"},
+ {".shtml", 6, "text/html"},
+ {".txt", 4, "text/plain"},
+ {".xml", 4, "text/xml"},
+
+ /* video */
+ {".mov", 4, "video/quicktime"},
+ {".mp4", 4, "video/mp4"},
+ {".mpeg", 5, "video/mpeg"},
+ {".mpg", 4, "video/mpeg"},
+ {".ogv", 4, "video/ogg"},
+ {".qt", 3, "video/quicktime"},
+
+ /* not registered types
+ (http://reference.sitepoint.com/html/mime-types-full,
+ http://www.hansenb.pdx.edu/DMKB/dict/tutorials/mime_typ.php, ..) */
+ {".arj", 4, "application/x-arj-compressed"},
+ {".gz", 3, "application/x-gunzip"},
+ {".rar", 4, "application/x-arj-compressed"},
+ {".swf", 4, "application/x-shockwave-flash"},
+ {".tar", 4, "application/x-tar"},
+ {".tgz", 4, "application/x-tar-gz"},
+ {".torrent", 8, "application/x-bittorrent"},
+ {".ppt", 4, "application/x-mspowerpoint"},
+ {".xls", 4, "application/x-msexcel"},
+ {".zip", 4, "application/x-zip-compressed"},
+ {".aac", 4, "audio/aac"}, /* http://en.wikipedia.org/wiki/Advanced_Audio_Coding */
+ {".aif", 4, "audio/x-aif"},
+ {".m3u", 4, "audio/x-mpegurl"},
+ {".mid", 4, "audio/x-midi"},
+ {".ra", 3, "audio/x-pn-realaudio"},
+ {".ram", 4, "audio/x-pn-realaudio"},
+ {".wav", 4, "audio/x-wav"},
+ {".bmp", 4, "image/bmp"},
+ {".ico", 4, "image/x-icon"},
+ {".pct", 4, "image/x-pct"},
+ {".pict", 5, "image/pict"},
+ {".rgb", 4, "image/x-rgb"},
+ {".webm", 5, "video/webm"}, /* http://en.wikipedia.org/wiki/WebM */
+ {".asf", 4, "video/x-ms-asf"},
+ {".avi", 4, "video/x-msvideo"},
+ {".m4v", 4, "video/x-m4v"},
+ {NULL, 0, NULL}
+};
+
+const char *mg_get_builtin_mime_type(const char *path)
+{
+ const char *ext;
+ size_t i, path_len;
+
+ path_len = strlen(path);
+
+ for (i = 0; builtin_mime_types[i].extension != NULL; i++) {
+ ext = path + (path_len - builtin_mime_types[i].ext_len);
+ if (path_len > builtin_mime_types[i].ext_len &&
+ mg_strcasecmp(ext, builtin_mime_types[i].extension) == 0) {
+ return builtin_mime_types[i].mime_type;
+ }
+ }
+
+ return "text/plain";
+}
+
+/* Look at the "path" extension and figure what mime type it has.
+ Store mime type in the vector. */
+static void get_mime_type(struct mg_context *ctx, const char *path,
+ struct vec *vec)
+{
+ struct vec ext_vec, mime_vec;
+ const char *list, *ext;
+ size_t path_len;
+
+ path_len = strlen(path);
+
+ /* Scan user-defined mime types first, in case user wants to
+ override default mime types. */
+ list = ctx->config[EXTRA_MIME_TYPES];
+ while ((list = next_option(list, &ext_vec, &mime_vec)) != NULL) {
+ /* ext now points to the path suffix */
+ ext = path + path_len - ext_vec.len;
+ if (mg_strncasecmp(ext, ext_vec.ptr, ext_vec.len) == 0) {
+ *vec = mime_vec;
+ return;
+ }
+ }
+
+ vec->ptr = mg_get_builtin_mime_type(path);
+ vec->len = strlen(vec->ptr);
+}
+
+/* Stringify binary data. Output buffer must be twice as big as input,
+ because each byte takes 2 bytes in string representation */
+static void bin2str(char *to, const unsigned char *p, size_t len)
+{
+ static const char *hex = "0123456789abcdef";
+
+ for (; len--; p++) {
+ *to++ = hex[p[0] >> 4];
+ *to++ = hex[p[0] & 0x0f];
+ }
+ *to = '\0';
+}
+
+/* Return stringified MD5 hash for list of strings. Buffer must be 33 bytes. */
+char *mg_md5(char buf[33], ...)
+{
+ md5_byte_t hash[16];
+ const char *p;
+ va_list ap;
+ md5_state_t ctx;
+
+ md5_init(&ctx);
+
+ va_start(ap, buf);
+ while ((p = va_arg(ap, const char *)) != NULL) {
+ md5_append(&ctx, (const md5_byte_t *) p, (int) strlen(p));
+ }
+ va_end(ap);
+
+ md5_finish(&ctx, hash);
+ bin2str(buf, hash, sizeof(hash));
+ return buf;
+}
+
+/* Check the user's password, return 1 if OK */
+static int check_password(const char *method, const char *ha1, const char *uri,
+ const char *nonce, const char *nc, const char *cnonce,
+ const char *qop, const char *response)
+{
+ char ha2[32 + 1], expected_response[32 + 1];
+
+ /* Some of the parameters may be NULL */
+ if (method == NULL || nonce == NULL || nc == NULL || cnonce == NULL ||
+ qop == NULL || response == NULL) {
+ return 0;
+ }
+
+ /* NOTE(lsm): due to a bug in MSIE, we do not compare the URI */
+ /* TODO(lsm): check for authentication timeout */
+ if (/* strcmp(dig->uri, c->ouri) != 0 || */
+ strlen(response) != 32
+ /* || now - strtoul(dig->nonce, NULL, 10) > 3600 */
+ ) {
+ return 0;
+ }
+
+ mg_md5(ha2, method, ":", uri, NULL);
+ mg_md5(expected_response, ha1, ":", nonce, ":", nc,
+ ":", cnonce, ":", qop, ":", ha2, NULL);
+
+ return mg_strcasecmp(response, expected_response) == 0;
+}
+
+/* Use the global passwords file, if specified by auth_gpass option,
+ or search for .htpasswd in the requested directory. */
+static void open_auth_file(struct mg_connection *conn, const char *path,
+ struct file *filep)
+{
+ char name[PATH_MAX];
+ const char *p, *e, *gpass = conn->ctx->config[GLOBAL_PASSWORDS_FILE];
+ struct file file = STRUCT_FILE_INITIALIZER;
+
+ if (gpass != NULL) {
+ /* Use global passwords file */
+ if (!mg_fopen(conn, gpass, "r", filep)) {
+#ifdef DEBUG
+ mg_cry(conn, "fopen(%s): %s", gpass, strerror(ERRNO));
+#endif
+ }
+ /* Important: using local struct file to test path for is_directory
+ flag.
+ If filep is used, mg_stat() makes it appear as if auth file was
+ opened. */
+ } else if (mg_stat(conn, path, &file) && file.is_directory) {
+ mg_snprintf(conn, name, sizeof(name), "%s%c%s",
+ path, '/', PASSWORDS_FILE_NAME);
+ if (!mg_fopen(conn, name, "r", filep)) {
+#ifdef DEBUG
+ mg_cry(conn, "fopen(%s): %s", name, strerror(ERRNO));
+#endif
+ }
+ } else {
+ /* Try to find .htpasswd in requested directory. */
+ for (p = path, e = p + strlen(p) - 1; e > p; e--)
+ if (e[0] == '/')
+ break;
+ mg_snprintf(conn, name, sizeof(name), "%.*s%c%s",
+ (int) (e - p), p, '/', PASSWORDS_FILE_NAME);
+ if (!mg_fopen(conn, name, "r", filep)) {
+#ifdef DEBUG
+ mg_cry(conn, "fopen(%s): %s", name, strerror(ERRNO));
+#endif
+ }
+ }
+}
+
+/* Parsed Authorization header */
+struct ah {
+ char *user, *uri, *cnonce, *response, *qop, *nc, *nonce;
+};
+
+/* Return 1 on success. Always initializes the ah structure. */
+static int parse_auth_header(struct mg_connection *conn, char *buf,
+ size_t buf_size, struct ah *ah)
+{
+ char *name, *value, *s;
+ const char *auth_header;
+
+ (void) memset(ah, 0, sizeof(*ah));
+ if ((auth_header = mg_get_header(conn, "Authorization")) == NULL ||
+ mg_strncasecmp(auth_header, "Digest ", 7) != 0) {
+ return 0;
+ }
+
+ /* Make modifiable copy of the auth header */
+ (void) mg_strlcpy(buf, auth_header + 7, buf_size);
+ s = buf;
+
+ /* Parse authorization header */
+ for (;;) {
+ /* Gobble initial spaces */
+ while (isspace(* (unsigned char *) s)) {
+ s++;
+ }
+ name = skip_quoted(&s, "=", " ", 0);
+ /* Value is either quote-delimited, or ends at first comma or space. */
+ if (s[0] == '\"') {
+ s++;
+ value = skip_quoted(&s, "\"", " ", '\\');
+ if (s[0] == ',') {
+ s++;
+ }
+ } else {
+ value = skip_quoted(&s, ", ", " ", 0); /* IE uses commas, FF uses
+ spaces */
+ }
+ if (*name == '\0') {
+ break;
+ }
+
+ if (!strcmp(name, "username")) {
+ ah->user = value;
+ } else if (!strcmp(name, "cnonce")) {
+ ah->cnonce = value;
+ } else if (!strcmp(name, "response")) {
+ ah->response = value;
+ } else if (!strcmp(name, "uri")) {
+ ah->uri = value;
+ } else if (!strcmp(name, "qop")) {
+ ah->qop = value;
+ } else if (!strcmp(name, "nc")) {
+ ah->nc = value;
+ } else if (!strcmp(name, "nonce")) {
+ ah->nonce = value;
+ }
+ }
+
+ /* CGI needs it as REMOTE_USER */
+ if (ah->user != NULL) {
+ conn->request_info.remote_user = mg_strdup(ah->user);
+ } else {
+ return 0;
+ }
+
+ return 1;
+}
+
+static char *mg_fgets(char *buf, size_t size, struct file *filep, char **p)
+{
+ char *eof;
+ size_t len;
+ char *memend;
+
+ if (filep->membuf != NULL && *p != NULL) {
+ memend = (char *) &filep->membuf[filep->size];
+ eof = (char *) memchr(*p, '\n', memend - *p); /* Search for \n from p
+ till the end of
+ stream */
+ if (eof != NULL) {
+ eof += 1; /* Include \n */
+ } else {
+ eof = memend; /* Copy remaining data */
+ }
+ len = (size_t) (eof - *p) > size - 1 ? size - 1 : (size_t) (eof - *p);
+ memcpy(buf, *p, len);
+ buf[len] = '\0';
+ *p += len;
+ return len ? eof : NULL;
+ } else if (filep->fp != NULL) {
+ return fgets(buf, (int)size, filep->fp);
+ } else {
+ return NULL;
+ }
+}
+
+/* Authorize against the opened passwords file. Return 1 if authorized. */
+static int authorize(struct mg_connection *conn, struct file *filep)
+{
+ struct ah ah;
+ char line[256], f_user[256] = "", ha1[256] = "", f_domain[256] = "", buf[MG_BUF_LEN], *p;
+
+ if (!parse_auth_header(conn, buf, sizeof(buf), &ah)) {
+ return 0;
+ }
+
+ /* Loop over passwords file */
+ p = (char *) filep->membuf;
+ while (mg_fgets(line, sizeof(line), filep, &p) != NULL) {
+ if (sscanf(line, "%[^:]:%[^:]:%s", f_user, f_domain, ha1) != 3) {
+ continue;
+ }
+
+ if (!strcmp(ah.user, f_user) &&
+ !strcmp(conn->ctx->config[AUTHENTICATION_DOMAIN], f_domain))
+ return check_password(conn->request_info.request_method, ha1, ah.uri,
+ ah.nonce, ah.nc, ah.cnonce, ah.qop, ah.response);
+ }
+
+ return 0;
+}
+
+/* Return 1 if request is authorised, 0 otherwise. */
+static int check_authorization(struct mg_connection *conn, const char *path)
+{
+ char fname[PATH_MAX];
+ struct vec uri_vec, filename_vec;
+ const char *list;
+ struct file file = STRUCT_FILE_INITIALIZER;
+ int authorized = 1;
+
+ list = conn->ctx->config[PROTECT_URI];
+ while ((list = next_option(list, &uri_vec, &filename_vec)) != NULL) {
+ if (!memcmp(conn->request_info.uri, uri_vec.ptr, uri_vec.len)) {
+ mg_snprintf(conn, fname, sizeof(fname), "%.*s",
+ (int) filename_vec.len, filename_vec.ptr);
+ if (!mg_fopen(conn, fname, "r", &file)) {
+ mg_cry(conn, "%s: cannot open %s: %s", __func__, fname, strerror(errno));
+ }
+ break;
+ }
+ }
+
+ if (!is_file_opened(&file)) {
+ open_auth_file(conn, path, &file);
+ }
+
+ if (is_file_opened(&file)) {
+ authorized = authorize(conn, &file);
+ mg_fclose(&file);
+ }
+
+ return authorized;
+}
+
+static void send_authorization_request(struct mg_connection *conn)
+{
+ char date[64];
+ time_t curtime = time(NULL);
+
+ conn->status_code = 401;
+ conn->must_close = 1;
+
+ gmt_time_string(date, sizeof(date), &curtime);
+
+ mg_printf(conn,
+ "HTTP/1.1 401 Unauthorized\r\n"
+ "Date: %s\r\n"
+ "Connection: %s\r\n"
+ "Content-Length: 0\r\n"
+ "WWW-Authenticate: Digest qop=\"auth\", realm=\"%s\", nonce=\"%lu\"\r\n\r\n",
+ date, suggest_connection_header(conn),
+ conn->ctx->config[AUTHENTICATION_DOMAIN],
+ (unsigned long) time(NULL));
+}
+
+static int is_authorized_for_put(struct mg_connection *conn)
+{
+ struct file file = STRUCT_FILE_INITIALIZER;
+ const char *passfile = conn->ctx->config[PUT_DELETE_PASSWORDS_FILE];
+ int ret = 0;
+
+ if (passfile != NULL && mg_fopen(conn, passfile, "r", &file)) {
+ ret = authorize(conn, &file);
+ mg_fclose(&file);
+ }
+
+ return ret;
+}
+
+int mg_modify_passwords_file(const char *fname, const char *domain,
+ const char *user, const char *pass)
+{
+ int found;
+ char line[512], u[512] = "", d[512] ="", ha1[33], tmp[PATH_MAX+1];
+ FILE *fp, *fp2;
+
+ found = 0;
+ fp = fp2 = NULL;
+
+ /* Regard empty password as no password - remove user record. */
+ if (pass != NULL && pass[0] == '\0') {
+ pass = NULL;
+ }
+
+ (void) snprintf(tmp, sizeof(tmp) - 1, "%s.tmp", fname);
+ tmp[sizeof(tmp) - 1] = 0;
+
+ /* Create the file if does not exist */
+ if ((fp = fopen(fname, "a+")) != NULL) {
+ (void) fclose(fp);
+ }
+
+ /* Open the given file and temporary file */
+ if ((fp = fopen(fname, "r")) == NULL) {
+ return 0;
+ } else if ((fp2 = fopen(tmp, "w+")) == NULL) {
+ fclose(fp);
+ return 0;
+ }
+
+ /* Copy the stuff to temporary file */
+ while (fgets(line, sizeof(line), fp) != NULL) {
+ if (sscanf(line, "%[^:]:%[^:]:%*s", u, d) != 2) {
+ continue;
+ }
+
+ if (!strcmp(u, user) && !strcmp(d, domain)) {
+ found++;
+ if (pass != NULL) {
+ mg_md5(ha1, user, ":", domain, ":", pass, NULL);
+ fprintf(fp2, "%s:%s:%s\n", user, domain, ha1);
+ }
+ } else {
+ fprintf(fp2, "%s", line);
+ }
+ }
+
+ /* If new user, just add it */
+ if (!found && pass != NULL) {
+ mg_md5(ha1, user, ":", domain, ":", pass, NULL);
+ fprintf(fp2, "%s:%s:%s\n", user, domain, ha1);
+ }
+
+ /* Close files */
+ fclose(fp);
+ fclose(fp2);
+
+ /* Put the temp file in place of real file */
+ IGNORE_UNUSED_RESULT(remove(fname));
+ IGNORE_UNUSED_RESULT(rename(tmp, fname));
+
+ return 1;
+}
+
+static SOCKET conn2(struct mg_context *ctx /* may be null */, const char *host, int port,
+ int use_ssl, char *ebuf, size_t ebuf_len)
+{
+ struct sockaddr_in sain;
+ struct hostent *he;
+ SOCKET sock = INVALID_SOCKET;
+
+ if (host == NULL) {
+ snprintf(ebuf, ebuf_len, "%s", "NULL host");
+ } else if (use_ssl && SSLv23_client_method == NULL) {
+ snprintf(ebuf, ebuf_len, "%s", "SSL is not initialized");
+ /* TODO(lsm): use something threadsafe instead of gethostbyname() */
+ } else if ((he = gethostbyname(host)) == NULL) {
+ snprintf(ebuf, ebuf_len, "gethostbyname(%s): %s", host, strerror(ERRNO));
+ } else if ((sock = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) {
+ snprintf(ebuf, ebuf_len, "socket(): %s", strerror(ERRNO));
+ } else {
+ set_close_on_exec(sock, fc(ctx));
+ memset(&sain, '\0', sizeof(sain));
+ sain.sin_family = AF_INET;
+ sain.sin_port = htons((uint16_t) port);
+ sain.sin_addr = * (struct in_addr *) he->h_addr_list[0];
+ if (connect(sock, (struct sockaddr *) &sain, sizeof(sain)) != 0) {
+ snprintf(ebuf, ebuf_len, "connect(%s:%d): %s",
+ host, port, strerror(ERRNO));
+ closesocket(sock);
+ sock = INVALID_SOCKET;
+ }
+ }
+ return sock;
+}
+
+int mg_url_encode(const char *src, char *dst, size_t dst_len)
+{
+ static const char *dont_escape = "._-$,;~()";
+ static const char *hex = "0123456789abcdef";
+ char *pos = dst;
+ const char *end = dst + dst_len - 1;
+
+ for (; *src != '\0' && pos < end; src++, pos++) {
+ if (isalnum(*(const unsigned char *) src) ||
+ strchr(dont_escape, * (const unsigned char *) src) != NULL) {
+ *pos = *src;
+ } else if (pos + 2 < end) {
+ pos[0] = '%';
+ pos[1] = hex[(* (const unsigned char *) src) >> 4];
+ pos[2] = hex[(* (const unsigned char *) src) & 0xf];
+ pos += 2;
+ } else {
+ return -1;
+ }
+ }
+
+ *pos = '\0';
+ return (*src == '\0') ? (int)(pos - dst) : -1;
+}
+
+static void print_dir_entry(struct de *de)
+{
+ char size[64], mod[64], href[PATH_MAX];
+ struct tm *tm;
+
+ if (de->file.is_directory) {
+ mg_snprintf(de->conn, size, sizeof(size), "%s", "[DIRECTORY]");
+ } else {
+ /* We use (signed) cast below because MSVC 6 compiler cannot
+ convert unsigned __int64 to double. Sigh. */
+ if (de->file.size < 1024) {
+ mg_snprintf(de->conn, size, sizeof(size), "%d", (int) de->file.size);
+ } else if (de->file.size < 0x100000) {
+ mg_snprintf(de->conn, size, sizeof(size),
+ "%.1fk", (double) de->file.size / 1024.0);
+ } else if (de->file.size < 0x40000000) {
+ mg_snprintf(de->conn, size, sizeof(size),
+ "%.1fM", (double) de->file.size / 1048576);
+ } else {
+ mg_snprintf(de->conn, size, sizeof(size),
+ "%.1fG", (double) de->file.size / 1073741824);
+ }
+ }
+ tm = localtime(&de->file.modification_time);
+ if (tm != NULL) {
+ strftime(mod, sizeof(mod), "%d-%b-%Y %H:%M", tm);
+ } else {
+ strncpy(mod, "01-Jan-1970 00:00", sizeof(mod));
+ mod[sizeof(mod) - 1] = '\0';
+ }
+ mg_url_encode(de->file_name, href, sizeof(href));
+ de->conn->num_bytes_sent += mg_printf(de->conn,
+ "<tr><td><a href=\"%s%s%s\">%s%s</a></td>"
+ "<td> %s</td><td> %s</td></tr>\n",
+ de->conn->request_info.uri, href, de->file.is_directory ? "/" : "",
+ de->file_name, de->file.is_directory ? "/" : "", mod, size);
+}
+
+/* This function is called from send_directory() and used for
+ sorting directory entries by size, or name, or modification time.
+ On windows, __cdecl specification is needed in case if project is built
+ with __stdcall convention. qsort always requires __cdels callback. */
+static int WINCDECL compare_dir_entries(const void *p1, const void *p2)
+{
+ const struct de *a = (const struct de *) p1, *b = (const struct de *) p2;
+ const char *query_string = a->conn->request_info.query_string;
+ int cmp_result = 0;
+
+ if (query_string == NULL) {
+ query_string = "na";
+ }
+
+ if (a->file.is_directory && !b->file.is_directory) {
+ return -1; /* Always put directories on top */
+ } else if (!a->file.is_directory && b->file.is_directory) {
+ return 1; /* Always put directories on top */
+ } else if (*query_string == 'n') {
+ cmp_result = strcmp(a->file_name, b->file_name);
+ } else if (*query_string == 's') {
+ cmp_result = a->file.size == b->file.size ? 0 :
+ a->file.size > b->file.size ? 1 : -1;
+ } else if (*query_string == 'd') {
+ cmp_result = a->file.modification_time == b->file.modification_time ? 0 :
+ a->file.modification_time > b->file.modification_time ? 1 : -1;
+ }
+
+ return query_string[1] == 'd' ? -cmp_result : cmp_result;
+}
+
+static int must_hide_file(struct mg_connection *conn, const char *path)
+{
+ const char *pw_pattern = "**" PASSWORDS_FILE_NAME "$";
+ const char *pattern = conn->ctx->config[HIDE_FILES];
+ return match_prefix(pw_pattern, (int)strlen(pw_pattern), path) > 0 ||
+ (pattern != NULL && match_prefix(pattern, (int)strlen(pattern), path) > 0);
+}
+
+static int scan_directory(struct mg_connection *conn, const char *dir,
+ void *data, void (*cb)(struct de *, void *))
+{
+ char path[PATH_MAX];
+ struct dirent *dp;
+ DIR *dirp;
+ struct de de;
+
+ if ((dirp = opendir(dir)) == NULL) {
+ return 0;
+ } else {
+ de.conn = conn;
+
+ while ((dp = readdir(dirp)) != NULL) {
+ /* Do not show current dir and hidden files */
+ if (!strcmp(dp->d_name, ".") ||
+ !strcmp(dp->d_name, "..") ||
+ must_hide_file(conn, dp->d_name)) {
+ continue;
+ }
+
+ mg_snprintf(conn, path, sizeof(path), "%s%c%s", dir, '/', dp->d_name);
+
+ /* If we don't memset stat structure to zero, mtime will have
+ garbage and strftime() will segfault later on in
+ print_dir_entry(). memset is required only if mg_stat()
+ fails. For more details, see
+ http://code.google.com/p/mongoose/issues/detail?id=79 */
+ memset(&de.file, 0, sizeof(de.file));
+ if (!mg_stat(conn, path, &de.file)) {
+ mg_cry(conn, "%s: mg_stat(%s) failed: %s",
+ __func__, path, strerror(ERRNO));
+ }
+
+ de.file_name = dp->d_name;
+ cb(&de, data);
+ }
+ (void) closedir(dirp);
+ }
+ return 1;
+}
+
+static int remove_directory(struct mg_connection *conn, const char *dir)
+{
+ char path[PATH_MAX];
+ struct dirent *dp;
+ DIR *dirp;
+ struct de de;
+
+ if ((dirp = opendir(dir)) == NULL) {
+ return 0;
+ } else {
+ de.conn = conn;
+
+ while ((dp = readdir(dirp)) != NULL) {
+ /* Do not show current dir (but show hidden files as they will
+ also be removed) */
+ if (!strcmp(dp->d_name, ".") ||
+ !strcmp(dp->d_name, "..")) {
+ continue;
+ }
+
+ mg_snprintf(conn, path, sizeof(path), "%s%c%s", dir, '/', dp->d_name);
+
+ /* If we don't memset stat structure to zero, mtime will have
+ garbage and strftime() will segfault later on in
+ print_dir_entry(). memset is required only if mg_stat()
+ fails. For more details, see
+ http://code.google.com/p/mongoose/issues/detail?id=79 */
+ memset(&de.file, 0, sizeof(de.file));
+ if (!mg_stat(conn, path, &de.file)) {
+ mg_cry(conn, "%s: mg_stat(%s) failed: %s",
+ __func__, path, strerror(ERRNO));
+ }
+ if(de.file.modification_time) {
+ if(de.file.is_directory) {
+ remove_directory(conn, path);
+ } else {
+ mg_remove(path);
+ }
+ }
+
+ }
+ (void) closedir(dirp);
+
+ IGNORE_UNUSED_RESULT(rmdir(dir));
+ }
+
+ return 1;
+}
+
+struct dir_scan_data {
+ struct de *entries;
+ int num_entries;
+ int arr_size;
+};
+
+/* Behaves like realloc(), but frees original pointer on failure */
+static void *realloc2(void *ptr, size_t size)
+{
+ void *new_ptr = realloc(ptr, size);
+ if (new_ptr == NULL) {
+ free(ptr);
+ }
+ return new_ptr;
+}
+
+static void dir_scan_callback(struct de *de, void *data)
+{
+ struct dir_scan_data *dsd = (struct dir_scan_data *) data;
+
+ if (dsd->entries == NULL || dsd->num_entries >= dsd->arr_size) {
+ dsd->arr_size *= 2;
+ dsd->entries = (struct de *) realloc2(dsd->entries, dsd->arr_size *
+ sizeof(dsd->entries[0]));
+ }
+ if (dsd->entries == NULL) {
+ /* TODO(lsm): propagate an error to the caller */
+ dsd->num_entries = 0;
+ } else {
+ dsd->entries[dsd->num_entries].file_name = mg_strdup(de->file_name);
+ dsd->entries[dsd->num_entries].file = de->file;
+ dsd->entries[dsd->num_entries].conn = de->conn;
+ dsd->num_entries++;
+ }
+}
+
+static void handle_directory_request(struct mg_connection *conn,
+ const char *dir)
+{
+ int i, sort_direction;
+ struct dir_scan_data data = { NULL, 0, 128 };
+ char date[64];
+ time_t curtime = time(NULL);
+
+ if (!scan_directory(conn, dir, &data, dir_scan_callback)) {
+ send_http_error(conn, 500, "Cannot open directory",
+ "Error: opendir(%s): %s", dir, strerror(ERRNO));
+ return;
+ }
+
+ gmt_time_string(date, sizeof(date), &curtime);
+
+ sort_direction = conn->request_info.query_string != NULL &&
+ conn->request_info.query_string[1] == 'd' ? 'a' : 'd';
+
+ conn->must_close = 1;
+ mg_printf(conn, "HTTP/1.1 200 OK\r\n"
+ "Date: %s\r\n"
+ "Connection: close\r\n"
+ "Content-Type: text/html; charset=utf-8\r\n\r\n",
+ date);
+
+ conn->num_bytes_sent += mg_printf(conn,
+ "<html><head><title>Index of %s</title>"
+ "<style>th {text-align: left;}</style></head>"
+ "<body><h1>Index of %s</h1><pre><table cellpadding=\"0\">"
+ "<tr><th><a href=\"?n%c\">Name</a></th>"
+ "<th><a href=\"?d%c\">Modified</a></th>"
+ "<th><a href=\"?s%c\">Size</a></th></tr>"
+ "<tr><td colspan=\"3\"><hr></td></tr>",
+ conn->request_info.uri, conn->request_info.uri,
+ sort_direction, sort_direction, sort_direction);
+
+ /* Print first entry - link to a parent directory */
+ conn->num_bytes_sent += mg_printf(conn,
+ "<tr><td><a href=\"%s%s\">%s</a></td>"
+ "<td> %s</td><td> %s</td></tr>\n",
+ conn->request_info.uri, "..", "Parent directory", "-", "-");
+
+ /* Sort and print directory entries */
+ if (data.entries != NULL) {
+ qsort(data.entries, (size_t) data.num_entries,
+ sizeof(data.entries[0]), compare_dir_entries);
+ for (i = 0; i < data.num_entries; i++) {
+ print_dir_entry(&data.entries[i]);
+ free(data.entries[i].file_name);
+ }
+ free(data.entries);
+ }
+
+ conn->num_bytes_sent += mg_printf(conn, "%s", "</table></body></html>");
+ conn->status_code = 200;
+}
+
+/* Send len bytes from the opened file to the client. */
+static void send_file_data(struct mg_connection *conn, struct file *filep,
+ int64_t offset, int64_t len)
+{
+ char buf[MG_BUF_LEN];
+ int to_read, num_read, num_written;
+
+ /* Sanity check the offset */
+ offset = offset < 0 ? 0 : offset > filep->size ? filep->size : offset;
+
+ if (len > 0 && filep->membuf != NULL && filep->size > 0) {
+ if (len > filep->size - offset) {
+ len = filep->size - offset;
+ }
+ mg_write(conn, filep->membuf + offset, (size_t) len);
+ } else if (len > 0 && filep->fp != NULL) {
+ if (offset > 0 && fseeko(filep->fp, offset, SEEK_SET) != 0) {
+ mg_cry(conn, "%s: fseeko() failed: %s",
+ __func__, strerror(ERRNO));
+ }
+ while (len > 0) {
+ /* Calculate how much to read from the file in the buffer */
+ to_read = sizeof(buf);
+ if ((int64_t) to_read > len) {
+ to_read = (int) len;
+ }
+
+ /* Read from file, exit the loop on error */
+ if ((num_read = (int) fread(buf, 1, (size_t) to_read, filep->fp)) <= 0) {
+ break;
+ }
+
+ /* Send read bytes to the client, exit the loop on error */
+ if ((num_written = mg_write(conn, buf, (size_t) num_read)) != num_read) {
+ break;
+ }
+
+ /* Both read and were successful, adjust counters */
+ conn->num_bytes_sent += num_written;
+ len -= num_written;
+ }
+ }
+}
+
+static int parse_range_header(const char *header, int64_t *a, int64_t *b)
+{
+ return sscanf(header, "bytes=%" INT64_FMT "-%" INT64_FMT, a, b);
+}
+
+static void construct_etag(char *buf, size_t buf_len,
+ const struct file *filep)
+{
+ snprintf(buf, buf_len, "\"%lx.%" INT64_FMT "\"",
+ (unsigned long) filep->modification_time, filep->size);
+}
+
+static void fclose_on_exec(struct file *filep, struct mg_connection *conn)
+{
+ if (filep != NULL && filep->fp != NULL) {
+#ifdef _WIN32
+ (void) conn; /* Unused. */
+#else
+ if (fcntl(fileno(filep->fp), F_SETFD, FD_CLOEXEC) != 0) {
+ mg_cry(conn, "%s: fcntl(F_SETFD FD_CLOEXEC) failed: %s",
+ __func__, strerror(ERRNO));
+ }
+#endif
+ }
+}
+
+static void handle_file_request(struct mg_connection *conn, const char *path,
+ struct file *filep)
+{
+ char date[64], lm[64], etag[64], range[64];
+ const char *msg = "OK", *hdr;
+ time_t curtime = time(NULL);
+ int64_t cl, r1, r2;
+ struct vec mime_vec;
+ int n;
+ char gz_path[PATH_MAX];
+ char const* encoding = "";
+
+ get_mime_type(conn->ctx, path, &mime_vec);
+ cl = filep->size;
+ conn->status_code = 200;
+ range[0] = '\0';
+
+ /* if this file is in fact a pre-gzipped file, rewrite its filename
+ it's important to rewrite the filename after resolving
+ the mime type from it, to preserve the actual file's type */
+ if (filep->gzipped) {
+ snprintf(gz_path, sizeof(gz_path), "%s.gz", path);
+ path = gz_path;
+ encoding = "Content-Encoding: gzip\r\n";
+ }
+
+ if (!mg_fopen(conn, path, "rb", filep)) {
+ send_http_error(conn, 500, http_500_error,
+ "fopen(%s): %s", path, strerror(ERRNO));
+ return;
+ }
+
+ fclose_on_exec(filep, conn);
+
+ /* If Range: header specified, act accordingly */
+ r1 = r2 = 0;
+ hdr = mg_get_header(conn, "Range");
+ if (hdr != NULL && (n = parse_range_header(hdr, &r1, &r2)) > 0 &&
+ r1 >= 0 && r2 >= 0) {
+ /* actually, range requests don't play well with a pre-gzipped
+ file (since the range is specified in the uncompressed space) */
+ if (filep->gzipped) {
+ send_http_error(conn, 501, "Not Implemented", "range requests in gzipped files are not supported");
+ mg_fclose(filep);
+ return;
+ }
+ conn->status_code = 206;
+cl = n == 2 ? (r2 > cl ? cl : r2) - r1 + 1: cl - r1;
+ mg_snprintf(conn, range, sizeof(range),
+ "Content-Range: bytes "
+ "%" INT64_FMT "-%"
+ INT64_FMT "/%" INT64_FMT "\r\n",
+ r1, r1 + cl - 1, filep->size);
+ msg = "Partial Content";
+ }
+
+ /* Prepare Etag, Date, Last-Modified headers. Must be in UTC, according to
+ http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3 */
+ gmt_time_string(date, sizeof(date), &curtime);
+ gmt_time_string(lm, sizeof(lm), &filep->modification_time);
+ construct_etag(etag, sizeof(etag), filep);
+
+ (void) mg_printf(conn,
+ "HTTP/1.1 %d %s\r\n"
+ "Date: %s\r\n"
+ "Last-Modified: %s\r\n"
+ "Etag: %s\r\n"
+ "Content-Type: %.*s\r\n"
+ "Content-Length: %" INT64_FMT "\r\n"
+ "Connection: %s\r\n"
+ "Accept-Ranges: bytes\r\n"
+ "%s%s\r\n",
+ conn->status_code, msg, date, lm, etag, (int) mime_vec.len,
+ mime_vec.ptr, cl, suggest_connection_header(conn), range, encoding);
+
+ if (strcmp(conn->request_info.request_method, "HEAD") != 0) {
+ send_file_data(conn, filep, r1, cl);
+ }
+ mg_fclose(filep);
+}
+
+void mg_send_file(struct mg_connection *conn, const char *path)
+{
+ struct file file = STRUCT_FILE_INITIALIZER;
+ if (mg_stat(conn, path, &file)) {
+ handle_file_request(conn, path, &file);
+ } else {
+ send_http_error(conn, 404, "Not Found", "%s", "File not found");
+ }
+}
+
+
+/* Parse HTTP headers from the given buffer, advance buffer to the point
+ where parsing stopped. */
+static void parse_http_headers(char **buf, struct mg_request_info *ri)
+{
+ int i;
+
+ for (i = 0; i < (int) ARRAY_SIZE(ri->http_headers); i++) {
+ ri->http_headers[i].name = skip_quoted(buf, ":", " ", 0);
+ ri->http_headers[i].value = skip(buf, "\r\n");
+ if (ri->http_headers[i].name[0] == '\0')
+ break;
+ ri->num_headers = i + 1;
+ }
+}
+
+static int is_valid_http_method(const char *method)
+{
+ return !strcmp(method, "GET") || !strcmp(method, "POST") ||
+ !strcmp(method, "HEAD") || !strcmp(method, "CONNECT") ||
+ !strcmp(method, "PUT") || !strcmp(method, "DELETE") ||
+ !strcmp(method, "OPTIONS") || !strcmp(method, "PROPFIND")
+ || !strcmp(method, "MKCOL")
+ ;
+}
+
+/* Parse HTTP request, fill in mg_request_info structure.
+ This function modifies the buffer by NUL-terminating
+ HTTP request components, header names and header values. */
+static int parse_http_message(char *buf, int len, struct mg_request_info *ri)
+{
+ int is_request, request_length = get_request_len(buf, len);
+ if (request_length > 0) {
+ /* Reset attributes. DO NOT TOUCH is_ssl, remote_ip, remote_port */
+ ri->remote_user = ri->request_method = ri->uri = ri->http_version = NULL;
+ ri->num_headers = 0;
+
+ buf[request_length - 1] = '\0';
+
+ /* RFC says that all initial whitespaces should be ingored */
+ while (*buf != '\0' && isspace(* (unsigned char *) buf)) {
+ buf++;
+ }
+ ri->request_method = skip(&buf, " ");
+ ri->uri = skip(&buf, " ");
+ ri->http_version = skip(&buf, "\r\n");
+
+ /* HTTP message could be either HTTP request or HTTP response, e.g.
+ "GET / HTTP/1.0 ...." or "HTTP/1.0 200 OK ..." */
+ is_request = is_valid_http_method(ri->request_method);
+ if ((is_request && memcmp(ri->http_version, "HTTP/", 5) != 0) ||
+ (!is_request && memcmp(ri->request_method, "HTTP/", 5) != 0)) {
+ request_length = -1;
+ } else {
+ if (is_request) {
+ ri->http_version += 5;
+ }
+ parse_http_headers(&buf, ri);
+ }
+ }
+ return request_length;
+}
+
+/* Keep reading the input (either opened file descriptor fd, or socket sock,
+ or SSL descriptor ssl) into buffer buf, until \r\n\r\n appears in the
+ buffer (which marks the end of HTTP request). Buffer buf may already
+ have some data. The length of the data is stored in nread.
+ Upon every read operation, increase nread by the number of bytes read. */
+static int read_request(FILE *fp, struct mg_connection *conn,
+ char *buf, int bufsiz, int *nread)
+{
+ int request_len, n = 0;
+
+ request_len = get_request_len(buf, *nread);
+ while (conn->ctx->stop_flag == 0 &&
+ *nread < bufsiz && request_len == 0 &&
+ (n = pull(fp, conn, buf + *nread, bufsiz - *nread)) > 0) {
+ *nread += n;
+ assert(*nread <= bufsiz);
+ request_len = get_request_len(buf, *nread);
+ }
+
+ return request_len <= 0 && n <= 0 ? -1 : request_len;
+}
+
+/* For given directory path, substitute it to valid index file.
+ Return 1 if index file has been found, 0 if not found.
+ If the file is found, it's stats is returned in stp. */
+static int substitute_index_file(struct mg_connection *conn, char *path,
+ size_t path_len, struct file *filep)
+{
+ const char *list = conn->ctx->config[INDEX_FILES];
+ struct file file = STRUCT_FILE_INITIALIZER;
+ struct vec filename_vec;
+ size_t n = strlen(path);
+ int found = 0;
+
+ /* The 'path' given to us points to the directory. Remove all trailing
+ directory separator characters from the end of the path, and
+ then append single directory separator character. */
+ while (n > 0 && path[n - 1] == '/') {
+ n--;
+ }
+ path[n] = '/';
+
+ /* Traverse index files list. For each entry, append it to the given
+ path and see if the file exists. If it exists, break the loop */
+ while ((list = next_option(list, &filename_vec, NULL)) != NULL) {
+
+ /* Ignore too long entries that may overflow path buffer */
+ if (filename_vec.len > path_len - (n + 2))
+ continue;
+
+ /* Prepare full path to the index file */
+ mg_strlcpy(path + n + 1, filename_vec.ptr, filename_vec.len + 1);
+
+ /* Does it exist? */
+ if (mg_stat(conn, path, &file)) {
+ /* Yes it does, break the loop */
+ *filep = file;
+ found = 1;
+ break;
+ }
+ }
+
+ /* If no index file exists, restore directory path */
+ if (!found) {
+ path[n] = '\0';
+ }
+
+ return found;
+}
+
+/* Return True if we should reply 304 Not Modified. */
+static int is_not_modified(const struct mg_connection *conn,
+ const struct file *filep)
+{
+ char etag[64];
+ const char *ims = mg_get_header(conn, "If-Modified-Since");
+ const char *inm = mg_get_header(conn, "If-None-Match");
+ construct_etag(etag, sizeof(etag), filep);
+ return (inm != NULL && !mg_strcasecmp(etag, inm)) ||
+ (ims != NULL && filep->modification_time <= parse_date_string(ims));
+}
+
+static int forward_body_data(struct mg_connection *conn, FILE *fp,
+ SOCKET sock, SSL *ssl)
+{
+ const char *expect, *body;
+ char buf[MG_BUF_LEN];
+ int to_read, nread, buffered_len, success = 0;
+
+ expect = mg_get_header(conn, "Expect");
+ assert(fp != NULL);
+
+ if (conn->content_len == -1) {
+ send_http_error(conn, 411, "Length Required", "%s", "");
+ } else if (expect != NULL && mg_strcasecmp(expect, "100-continue")) {
+ send_http_error(conn, 417, "Expectation Failed", "%s", "");
+ } else {
+ if (expect != NULL) {
+ (void) mg_printf(conn, "%s", "HTTP/1.1 100 Continue\r\n\r\n");
+ }
+
+ body = conn->buf + conn->request_len + conn->consumed_content;
+ buffered_len = (int)(&conn->buf[conn->data_len] - body);
+ assert(buffered_len >= 0);
+ assert(conn->consumed_content == 0);
+
+ if (buffered_len > 0) {
+ if ((int64_t) buffered_len > conn->content_len) {
+ buffered_len = (int) conn->content_len;
+ }
+ push(fp, sock, ssl, body, (int64_t) buffered_len);
+ conn->consumed_content += buffered_len;
+ }
+
+ nread = 0;
+ while (conn->consumed_content < conn->content_len) {
+ to_read = sizeof(buf);
+ if ((int64_t) to_read > conn->content_len - conn->consumed_content) {
+ to_read = (int) (conn->content_len - conn->consumed_content);
+ }
+ nread = pull(NULL, conn, buf, to_read);
+ if (nread <= 0 || push(fp, sock, ssl, buf, nread) != nread) {
+ break;
+ }
+ conn->consumed_content += nread;
+ }
+
+ if (conn->consumed_content == conn->content_len) {
+ success = nread >= 0;
+ }
+
+ /* Each error code path in this function must send an error */
+ if (!success) {
+ send_http_error(conn, 577, http_500_error, "%s", "");
+ }
+ }
+
+ return success;
+}
+
+#if !defined(NO_CGI)
+/* This structure helps to create an environment for the spawned CGI program.
+ Environment is an array of "VARIABLE=VALUE\0" ASCIIZ strings,
+ last element must be NULL.
+ However, on Windows there is a requirement that all these VARIABLE=VALUE\0
+ strings must reside in a contiguous buffer. The end of the buffer is
+ marked by two '\0' characters.
+ We satisfy both worlds: we create an envp array (which is vars), all
+ entries are actually pointers inside buf. */
+struct cgi_env_block {
+ struct mg_connection *conn;
+ char buf[CGI_ENVIRONMENT_SIZE]; /* Environment buffer */
+ int len; /* Space taken */
+ char *vars[MAX_CGI_ENVIR_VARS]; /* char **envp */
+ int nvars; /* Number of variables */
+};
+
+static char *addenv(struct cgi_env_block *block,
+ PRINTF_FORMAT_STRING(const char *fmt), ...)
+PRINTF_ARGS(2, 3);
+
+/* Append VARIABLE=VALUE\0 string to the buffer, and add a respective
+ pointer into the vars array. */
+static char *addenv(struct cgi_env_block *block, const char *fmt, ...)
+{
+ int n, space;
+ char *added;
+ va_list ap;
+
+ /* Calculate how much space is left in the buffer */
+ space = sizeof(block->buf) - block->len - 2;
+ assert(space >= 0);
+
+ /* Make a pointer to the free space int the buffer */
+ added = block->buf + block->len;
+
+ /* Copy VARIABLE=VALUE\0 string into the free space */
+ va_start(ap, fmt);
+ n = mg_vsnprintf(block->conn, added, (size_t) space, fmt, ap);
+ va_end(ap);
+
+ /* Make sure we do not overflow buffer and the envp array */
+ if (n > 0 && n + 1 < space &&
+ block->nvars < (int) ARRAY_SIZE(block->vars) - 2) {
+ /* Append a pointer to the added string into the envp array */
+ block->vars[block->nvars++] = added;
+ /* Bump up used length counter. Include \0 terminator */
+ block->len += n + 1;
+ } else {
+ mg_cry(block->conn, "%s: CGI env buffer truncated for [%s]", __func__, fmt);
+ }
+
+ return added;
+}
+
+static void prepare_cgi_environment(struct mg_connection *conn,
+ const char *prog,
+ struct cgi_env_block *blk)
+{
+ const char *s, *slash;
+ struct vec var_vec;
+ char *p, src_addr[IP_ADDR_STR_LEN];
+ int i;
+
+ blk->len = blk->nvars = 0;
+ blk->conn = conn;
+ sockaddr_to_string(src_addr, sizeof(src_addr), &conn->client.rsa);
+
+ addenv(blk, "SERVER_NAME=%s", conn->ctx->config[AUTHENTICATION_DOMAIN]);
+ addenv(blk, "SERVER_ROOT=%s", conn->ctx->config[DOCUMENT_ROOT]);
+ addenv(blk, "DOCUMENT_ROOT=%s", conn->ctx->config[DOCUMENT_ROOT]);
+ addenv(blk, "SERVER_SOFTWARE=%s/%s", "Civetweb", mg_version());
+
+ /* Prepare the environment block */
+ addenv(blk, "%s", "GATEWAY_INTERFACE=CGI/1.1");
+ addenv(blk, "%s", "SERVER_PROTOCOL=HTTP/1.1");
+ addenv(blk, "%s", "REDIRECT_STATUS=200"); /* For PHP */
+
+ /* TODO(lsm): fix this for IPv6 case */
+ addenv(blk, "SERVER_PORT=%d", ntohs(conn->client.lsa.sin.sin_port));
+
+ addenv(blk, "REQUEST_METHOD=%s", conn->request_info.request_method);
+ addenv(blk, "REMOTE_ADDR=%s", src_addr);
+ addenv(blk, "REMOTE_PORT=%d", conn->request_info.remote_port);
+ addenv(blk, "REQUEST_URI=%s", conn->request_info.uri);
+
+ /* SCRIPT_NAME */
+ assert(conn->request_info.uri[0] == '/');
+ slash = strrchr(conn->request_info.uri, '/');
+ if ((s = strrchr(prog, '/')) == NULL)
+ s = prog;
+ addenv(blk, "SCRIPT_NAME=%.*s%s", (int) (slash - conn->request_info.uri),
+ conn->request_info.uri, s);
+
+ addenv(blk, "SCRIPT_FILENAME=%s", prog);
+ addenv(blk, "PATH_TRANSLATED=%s", prog);
+ addenv(blk, "HTTPS=%s", conn->ssl == NULL ? "off" : "on");
+
+ if ((s = mg_get_header(conn, "Content-Type")) != NULL)
+ addenv(blk, "CONTENT_TYPE=%s", s);
+
+ if (conn->request_info.query_string != NULL)
+ addenv(blk, "QUERY_STRING=%s", conn->request_info.query_string);
+
+ if ((s = mg_get_header(conn, "Content-Length")) != NULL)
+ addenv(blk, "CONTENT_LENGTH=%s", s);
+
+ if ((s = getenv("PATH")) != NULL)
+ addenv(blk, "PATH=%s", s);
+
+ if (conn->path_info != NULL) {
+ addenv(blk, "PATH_INFO=%s", conn->path_info);
+ }
+
+#if defined(_WIN32)
+ if ((s = getenv("COMSPEC")) != NULL) {
+ addenv(blk, "COMSPEC=%s", s);
+ }
+ if ((s = getenv("SYSTEMROOT")) != NULL) {
+ addenv(blk, "SYSTEMROOT=%s", s);
+ }
+ if ((s = getenv("SystemDrive")) != NULL) {
+ addenv(blk, "SystemDrive=%s", s);
+ }
+ if ((s = getenv("ProgramFiles")) != NULL) {
+ addenv(blk, "ProgramFiles=%s", s);
+ }
+ if ((s = getenv("ProgramFiles(x86)")) != NULL) {
+ addenv(blk, "ProgramFiles(x86)=%s", s);
+ }
+#else
+ if ((s = getenv("LD_LIBRARY_PATH")) != NULL)
+ addenv(blk, "LD_LIBRARY_PATH=%s", s);
+#endif /* _WIN32 */
+
+ if ((s = getenv("PERLLIB")) != NULL)
+ addenv(blk, "PERLLIB=%s", s);
+
+ if (conn->request_info.remote_user != NULL) {
+ addenv(blk, "REMOTE_USER=%s", conn->request_info.remote_user);
+ addenv(blk, "%s", "AUTH_TYPE=Digest");
+ }
+
+ /* Add all headers as HTTP_* variables */
+ for (i = 0; i < conn->request_info.num_headers; i++) {
+ p = addenv(blk, "HTTP_%s=%s",
+ conn->request_info.http_headers[i].name,
+ conn->request_info.http_headers[i].value);
+
+ /* Convert variable name into uppercase, and change - to _ */
+ for (; *p != '=' && *p != '\0'; p++) {
+ if (*p == '-')
+ *p = '_';
+ *p = (char) toupper(* (unsigned char *) p);
+ }
+ }
+
+ /* Add user-specified variables */
+ s = conn->ctx->config[CGI_ENVIRONMENT];
+ while ((s = next_option(s, &var_vec, NULL)) != NULL) {
+ addenv(blk, "%.*s", (int) var_vec.len, var_vec.ptr);
+ }
+
+ blk->vars[blk->nvars++] = NULL;
+ blk->buf[blk->len++] = '\0';
+
+ assert(blk->nvars < (int) ARRAY_SIZE(blk->vars));
+ assert(blk->len > 0);
+ assert(blk->len < (int) sizeof(blk->buf));
+}
+
+static void handle_cgi_request(struct mg_connection *conn, const char *prog)
+{
+ char *buf;
+ size_t buflen;
+ int headers_len, data_len, i, fdin[2] = { 0, 0 }, fdout[2] = { 0, 0 };
+ const char *status, *status_text, *connection_state;
+ char *pbuf, dir[PATH_MAX], *p;
+ struct mg_request_info ri;
+ struct cgi_env_block blk;
+ FILE *in = NULL, *out = NULL;
+ struct file fout = STRUCT_FILE_INITIALIZER;
+ pid_t pid = (pid_t) -1;
+
+ buf = NULL;
+ buflen = 16384;
+ prepare_cgi_environment(conn, prog, &blk);
+
+ /* CGI must be executed in its own directory. 'dir' must point to the
+ directory containing executable program, 'p' must point to the
+ executable program name relative to 'dir'. */
+ (void) mg_snprintf(conn, dir, sizeof(dir), "%s", prog);
+ if ((p = strrchr(dir, '/')) != NULL) {
+ *p++ = '\0';
+ } else {
+ dir[0] = '.', dir[1] = '\0';
+ p = (char *) prog;
+ }
+
+ if (pipe(fdin) != 0 || pipe(fdout) != 0) {
+ send_http_error(conn, 500, http_500_error,
+ "Cannot create CGI pipe: %s", strerror(ERRNO));
+ goto done;
+ }
+
+ pid = spawn_process(conn, p, blk.buf, blk.vars, fdin[0], fdout[1], dir);
+ if (pid == (pid_t) -1) {
+ send_http_error(conn, 500, http_500_error,
+ "Cannot spawn CGI process [%s]: %s", prog, strerror(ERRNO));
+ goto done;
+ }
+
+ /* Make sure child closes all pipe descriptors. It must dup them to 0,1 */
+ set_close_on_exec(fdin[0], conn);
+ set_close_on_exec(fdin[1], conn);
+ set_close_on_exec(fdout[0], conn);
+ set_close_on_exec(fdout[1], conn);
+
+ /* Parent closes only one side of the pipes.
+ If we don't mark them as closed, close() attempt before
+ return from this function throws an exception on Windows.
+ Windows does not like when closed descriptor is closed again. */
+ (void) close(fdin[0]);
+ (void) close(fdout[1]);
+ fdin[0] = fdout[1] = -1;
+
+
+ if ((in = fdopen(fdin[1], "wb")) == NULL ||
+ (out = fdopen(fdout[0], "rb")) == NULL) {
+ send_http_error(conn, 500, http_500_error,
+ "fopen: %s", strerror(ERRNO));
+ goto done;
+ }
+
+ setbuf(in, NULL);
+ setbuf(out, NULL);
+ fout.fp = out;
+
+ /* Send POST data to the CGI process if needed */
+ if (!strcmp(conn->request_info.request_method, "POST") &&
+ !forward_body_data(conn, in, INVALID_SOCKET, NULL)) {
+ goto done;
+ }
+
+ /* Close so child gets an EOF. */
+ fclose(in);
+ in = NULL;
+ fdin[1] = -1;
+
+ /* Now read CGI reply into a buffer. We need to set correct
+ status code, thus we need to see all HTTP headers first.
+ Do not send anything back to client, until we buffer in all
+ HTTP headers. */
+ data_len = 0;
+ buf = malloc(buflen);
+ if (buf == NULL) {
+ send_http_error(conn, 500, http_500_error,
+ "Not enough memory for buffer (%u bytes)",
+ (unsigned int) buflen);
+ goto done;
+ }
+ headers_len = read_request(out, conn, buf, (int) buflen, &data_len);
+ if (headers_len <= 0) {
+ send_http_error(conn, 500, http_500_error,
+ "CGI program sent malformed or too big (>%u bytes) "
+ "HTTP headers: [%.*s]",
+ (unsigned) buflen, data_len, buf);
+ goto done;
+ }
+ pbuf = buf;
+ buf[headers_len - 1] = '\0';
+ parse_http_headers(&pbuf, &ri);
+
+ /* Make up and send the status line */
+ status_text = "OK";
+ if ((status = get_header(&ri, "Status")) != NULL) {
+ conn->status_code = atoi(status);
+ status_text = status;
+ while (isdigit(* (unsigned char *) status_text) || *status_text == ' ') {
+ status_text++;
+ }
+ } else if (get_header(&ri, "Location") != NULL) {
+ conn->status_code = 302;
+ } else {
+ conn->status_code = 200;
+ }
+ connection_state = get_header(&ri, "Connection");
+ if (connection_state == NULL ||
+ mg_strcasecmp(connection_state, "keep-alive")) {
+ conn->must_close = 1;
+ }
+ (void) mg_printf(conn, "HTTP/1.1 %d %s\r\n", conn->status_code,
+ status_text);
+
+ /* Send headers */
+ for (i = 0; i < ri.num_headers; i++) {
+ mg_printf(conn, "%s: %s\r\n",
+ ri.http_headers[i].name, ri.http_headers[i].value);
+ }
+ mg_write(conn, "\r\n", 2);
+
+ /* Send chunk of data that may have been read after the headers */
+ conn->num_bytes_sent += mg_write(conn, buf + headers_len,
+ (size_t)(data_len - headers_len));
+
+ /* Read the rest of CGI output and send to the client */
+ send_file_data(conn, &fout, 0, INT64_MAX);
+
+done:
+ if (pid != (pid_t) -1) {
+ kill(pid, SIGKILL);
+#if !defined(_WIN32)
+ {
+ int st;
+ while (waitpid(pid, &st, 0) != -1); /* clean zombies */
+ }
+#endif
+ }
+ if (fdin[0] != -1) {
+ close(fdin[0]);
+ }
+ if (fdout[1] != -1) {
+ close(fdout[1]);
+ }
+
+ if (in != NULL) {
+ fclose(in);
+ } else if (fdin[1] != -1) {
+ close(fdin[1]);
+ }
+
+ if (out != NULL) {
+ fclose(out);
+ } else if (fdout[0] != -1) {
+ close(fdout[0]);
+ }
+ if (buf != NULL) {
+ free(buf);
+ }
+}
+#endif /* !NO_CGI */
+
+/* For a given PUT path, create all intermediate subdirectories
+ for given path. Return 0 if the path itself is a directory,
+ or -1 on error, 1 if OK. */
+static int put_dir(struct mg_connection *conn, const char *path)
+{
+ char buf[PATH_MAX];
+ const char *s, *p;
+ struct file file = STRUCT_FILE_INITIALIZER;
+ int len, res = 1;
+
+ for (s = p = path + 2; (p = strchr(s, '/')) != NULL; s = ++p) {
+ len = (int)(p - path);
+ if (len >= (int) sizeof(buf)) {
+ res = -1;
+ break;
+ }
+ memcpy(buf, path, len);
+ buf[len] = '\0';
+
+ /* Try to create intermediate directory */
+ DEBUG_TRACE(("mkdir(%s)", buf));
+ if (!mg_stat(conn, buf, &file) && mg_mkdir(buf, 0755) != 0) {
+ res = -1;
+ break;
+ }
+
+ /* Is path itself a directory? */
+ if (p[1] == '\0') {
+ res = 0;
+ }
+ }
+
+ return res;
+}
+
+static void mkcol(struct mg_connection *conn, const char *path)
+{
+ int rc, body_len;
+ struct de de;
+ char date[64];
+ time_t curtime = time(NULL);
+
+ memset(&de.file, 0, sizeof(de.file));
+ if (!mg_stat(conn, path, &de.file)) {
+ mg_cry(conn, "%s: mg_stat(%s) failed: %s",
+ __func__, path, strerror(ERRNO));
+ }
+
+ if(de.file.modification_time) {
+ send_http_error(conn, 405, "Method Not Allowed",
+ "mkcol(%s): %s", path, strerror(ERRNO));
+ return;
+ }
+
+ body_len = conn->data_len - conn->request_len;
+ if(body_len > 0) {
+ send_http_error(conn, 415, "Unsupported media type",
+ "mkcol(%s): %s", path, strerror(ERRNO));
+ return;
+ }
+
+ rc = mg_mkdir(path, 0755);
+
+ if (rc == 0) {
+ conn->status_code = 201;
+ gmt_time_string(date, sizeof(date), &curtime);
+ mg_printf(conn, "HTTP/1.1 %d Created\r\nDate: %s\r\nContent-Length: 0\r\nConnection: %s\r\n\r\n",
+ conn->status_code, date, suggest_connection_header(conn));
+ } else if (rc == -1) {
+ if(errno == EEXIST)
+ send_http_error(conn, 405, "Method Not Allowed",
+ "mkcol(%s): %s", path, strerror(ERRNO));
+ else if(errno == EACCES)
+ send_http_error(conn, 403, "Forbidden",
+ "mkcol(%s): %s", path, strerror(ERRNO));
+ else if(errno == ENOENT)
+ send_http_error(conn, 409, "Conflict",
+ "mkcol(%s): %s", path, strerror(ERRNO));
+ else
+ send_http_error(conn, 500, http_500_error,
+ "fopen(%s): %s", path, strerror(ERRNO));
+ }
+}
+
+static void put_file(struct mg_connection *conn, const char *path)
+{
+ struct file file = STRUCT_FILE_INITIALIZER;
+ const char *range;
+ int64_t r1, r2;
+ int rc;
+ char date[64];
+ time_t curtime = time(NULL);
+
+ conn->status_code = mg_stat(conn, path, &file) ? 200 : 201;
+
+ if ((rc = put_dir(conn, path)) == 0) {
+ gmt_time_string(date, sizeof(date), &curtime);
+ mg_printf(conn, "HTTP/1.1 %d OK\r\nDate: %s\r\nContent-Length: 0\r\nConnection: %s\r\n\r\n",
+ conn->status_code, date, suggest_connection_header(conn));
+ } else if (rc == -1) {
+ send_http_error(conn, 500, http_500_error,
+ "put_dir(%s): %s", path, strerror(ERRNO));
+ } else if (!mg_fopen(conn, path, "wb+", &file) || file.fp == NULL) {
+ mg_fclose(&file);
+ send_http_error(conn, 500, http_500_error,
+ "fopen(%s): %s", path, strerror(ERRNO));
+ } else {
+ fclose_on_exec(&file, conn);
+ range = mg_get_header(conn, "Content-Range");
+ r1 = r2 = 0;
+ if (range != NULL && parse_range_header(range, &r1, &r2) > 0) {
+ conn->status_code = 206;
+ fseeko(file.fp, r1, SEEK_SET);
+ }
+ if (!forward_body_data(conn, file.fp, INVALID_SOCKET, NULL)) {
+ conn->status_code = 500;
+ }
+ gmt_time_string(date, sizeof(date), &curtime);
+ mg_printf(conn, "HTTP/1.1 %d OK\r\nDate: %s\r\nContent-Length: 0\r\nConnection: %s\r\n\r\n",
+ conn->status_code, date, suggest_connection_header(conn));
+ mg_fclose(&file);
+ }
+}
+
+static void send_ssi_file(struct mg_connection *, const char *,
+ struct file *, int);
+
+static void do_ssi_include(struct mg_connection *conn, const char *ssi,
+ char *tag, int include_level)
+{
+ char file_name[MG_BUF_LEN], path[PATH_MAX], *p;
+ struct file file = STRUCT_FILE_INITIALIZER;
+
+ /* sscanf() is safe here, since send_ssi_file() also uses buffer
+ of size MG_BUF_LEN to get the tag. So strlen(tag) is
+ always < MG_BUF_LEN. */
+ if (sscanf(tag, " virtual=\"%[^\"]\"", file_name) == 1) {
+ /* File name is relative to the webserver root */
+ (void) mg_snprintf(conn, path, sizeof(path), "%s%c%s",
+ conn->ctx->config[DOCUMENT_ROOT], '/', file_name);
+ } else if (sscanf(tag, " abspath=\"%[^\"]\"", file_name) == 1) {
+ /* File name is relative to the webserver working directory
+ or it is absolute system path */
+ (void) mg_snprintf(conn, path, sizeof(path), "%s", file_name);
+ } else if (sscanf(tag, " file=\"%[^\"]\"", file_name) == 1 ||
+ sscanf(tag, " \"%[^\"]\"", file_name) == 1) {
+ /* File name is relative to the currect document */
+ (void) mg_snprintf(conn, path, sizeof(path), "%s", ssi);
+ if ((p = strrchr(path, '/')) != NULL) {
+ p[1] = '\0';
+ }
+ (void) mg_snprintf(conn, path + strlen(path),
+ sizeof(path) - strlen(path), "%s", file_name);
+ } else {
+ mg_cry(conn, "Bad SSI #include: [%s]", tag);
+ return;
+ }
+
+ if (!mg_fopen(conn, path, "rb", &file)) {
+ mg_cry(conn, "Cannot open SSI #include: [%s]: fopen(%s): %s",
+ tag, path, strerror(ERRNO));
+ } else {
+ fclose_on_exec(&file, conn);
+ if (match_prefix(conn->ctx->config[SSI_EXTENSIONS],
+ (int)strlen(conn->ctx->config[SSI_EXTENSIONS]), path) > 0) {
+ send_ssi_file(conn, path, &file, include_level + 1);
+ } else {
+ send_file_data(conn, &file, 0, INT64_MAX);
+ }
+ mg_fclose(&file);
+ }
+}
+
+#if !defined(NO_POPEN)
+static void do_ssi_exec(struct mg_connection *conn, char *tag)
+{
+ char cmd[MG_BUF_LEN] = "";
+ struct file file = STRUCT_FILE_INITIALIZER;
+
+ if (sscanf(tag, " \"%[^\"]\"", cmd) != 1) {
+ mg_cry(conn, "Bad SSI #exec: [%s]", tag);
+ } else if ((file.fp = popen(cmd, "r")) == NULL) {
+ mg_cry(conn, "Cannot SSI #exec: [%s]: %s", cmd, strerror(ERRNO));
+ } else {
+ send_file_data(conn, &file, 0, INT64_MAX);
+ pclose(file.fp);
+ }
+}
+#endif /* !NO_POPEN */
+
+static int mg_fgetc(struct file *filep, int offset)
+{
+ if (filep->membuf != NULL && offset >=0 && offset < filep->size) {
+ return ((unsigned char *) filep->membuf)[offset];
+ } else if (filep->fp != NULL) {
+ return fgetc(filep->fp);
+ } else {
+ return EOF;
+ }
+}
+
+static void send_ssi_file(struct mg_connection *conn, const char *path,
+ struct file *filep, int include_level)
+{
+ char buf[MG_BUF_LEN];
+ int ch, offset, len, in_ssi_tag;
+
+ if (include_level > 10) {
+ mg_cry(conn, "SSI #include level is too deep (%s)", path);
+ return;
+ }
+
+ in_ssi_tag = len = offset = 0;
+ while ((ch = mg_fgetc(filep, offset)) != EOF) {
+ if (in_ssi_tag && ch == '>') {
+ in_ssi_tag = 0;
+ buf[len++] = (char) ch;
+ buf[len] = '\0';
+ assert(len <= (int) sizeof(buf));
+ if (len < 6 || memcmp(buf, "<!--#", 5) != 0) {
+ /* Not an SSI tag, pass it */
+ (void) mg_write(conn, buf, (size_t) len);
+ } else {
+ if (!memcmp(buf + 5, "include", 7)) {
+ do_ssi_include(conn, path, buf + 12, include_level);
+#if !defined(NO_POPEN)
+ } else if (!memcmp(buf + 5, "exec", 4)) {
+ do_ssi_exec(conn, buf + 9);
+#endif /* !NO_POPEN */
+ } else {
+ mg_cry(conn, "%s: unknown SSI " "command: \"%s\"", path, buf);
+ }
+ }
+ len = 0;
+ } else if (in_ssi_tag) {
+ if (len == 5 && memcmp(buf, "<!--#", 5) != 0) {
+ /* Not an SSI tag */
+ in_ssi_tag = 0;
+ } else if (len == (int) sizeof(buf) - 2) {
+ mg_cry(conn, "%s: SSI tag is too large", path);
+ len = 0;
+ }
+ buf[len++] = ch & 0xff;
+ } else if (ch == '<') {
+ in_ssi_tag = 1;
+ if (len > 0) {
+ mg_write(conn, buf, (size_t) len);
+ }
+ len = 0;
+ buf[len++] = ch & 0xff;
+ } else {
+ buf[len++] = ch & 0xff;
+ if (len == (int) sizeof(buf)) {
+ mg_write(conn, buf, (size_t) len);
+ len = 0;
+ }
+ }
+ }
+
+ /* Send the rest of buffered data */
+ if (len > 0) {
+ mg_write(conn, buf, (size_t) len);
+ }
+}
+
+static void handle_ssi_file_request(struct mg_connection *conn,
+ const char *path)
+{
+ struct file file = STRUCT_FILE_INITIALIZER;
+ char date[64];
+ time_t curtime = time(NULL);
+
+ if (!mg_fopen(conn, path, "rb", &file)) {
+ send_http_error(conn, 500, http_500_error, "fopen(%s): %s", path,
+ strerror(ERRNO));
+ } else {
+ conn->must_close = 1;
+ gmt_time_string(date, sizeof(date), &curtime);
+ fclose_on_exec(&file, conn);
+ mg_printf(conn, "HTTP/1.1 200 OK\r\n"
+ "Date: %s\r\n"
+ "Content-Type: text/html\r\n"
+ "Connection: %s\r\n\r\n",
+ date, suggest_connection_header(conn));
+ send_ssi_file(conn, path, &file, 0);
+ mg_fclose(&file);
+ }
+}
+
+static void send_options(struct mg_connection *conn)
+{
+ char date[64];
+ time_t curtime = time(NULL);
+
+ conn->status_code = 200;
+ conn->must_close = 1;
+ gmt_time_string(date, sizeof(date), &curtime);
+
+ mg_printf(conn, "HTTP/1.1 200 OK\r\n"
+ "Date: %s\r\n"
+ "Connection: %s\r\n"
+ "Allow: GET, POST, HEAD, CONNECT, PUT, DELETE, OPTIONS, PROPFIND, MKCOL\r\n"
+ "DAV: 1\r\n\r\n",
+ date, suggest_connection_header(conn));
+}
+
+/* Writes PROPFIND properties for a collection element */
+static void print_props(struct mg_connection *conn, const char* uri,
+ struct file *filep)
+{
+ char mtime[64];
+ gmt_time_string(mtime, sizeof(mtime), &filep->modification_time);
+ conn->num_bytes_sent += mg_printf(conn,
+ "<d:response>"
+ "<d:href>%s</d:href>"
+ "<d:propstat>"
+ "<d:prop>"
+ "<d:resourcetype>%s</d:resourcetype>"
+ "<d:getcontentlength>%" INT64_FMT "</d:getcontentlength>"
+ "<d:getlastmodified>%s</d:getlastmodified>"
+ "</d:prop>"
+ "<d:status>HTTP/1.1 200 OK</d:status>"
+ "</d:propstat>"
+ "</d:response>\n",
+ uri,
+ filep->is_directory ? "<d:collection/>" : "",
+ filep->size,
+ mtime);
+}
+
+static void print_dav_dir_entry(struct de *de, void *data)
+{
+ char href[PATH_MAX];
+ char href_encoded[PATH_MAX];
+ struct mg_connection *conn = (struct mg_connection *) data;
+ mg_snprintf(conn, href, sizeof(href), "%s%s",
+ conn->request_info.uri, de->file_name);
+ mg_url_encode(href, href_encoded, PATH_MAX-1);
+ print_props(conn, href_encoded, &de->file);
+}
+
+static void handle_propfind(struct mg_connection *conn, const char *path,
+ struct file *filep)
+{
+ const char *depth = mg_get_header(conn, "Depth");
+ char date[64];
+ time_t curtime = time(NULL);
+
+ gmt_time_string(date, sizeof(date), &curtime);
+
+ conn->must_close = 1;
+ conn->status_code = 207;
+ mg_printf(conn, "HTTP/1.1 207 Multi-Status\r\n"
+ "Date: %s\r\n"
+ "Connection: %s\r\n"
+ "Content-Type: text/xml; charset=utf-8\r\n\r\n",
+ date, suggest_connection_header(conn));
+
+ conn->num_bytes_sent += mg_printf(conn,
+ "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+ "<d:multistatus xmlns:d='DAV:'>\n");
+
+ /* Print properties for the requested resource itself */
+ print_props(conn, conn->request_info.uri, filep);
+
+ /* If it is a directory, print directory entries too if Depth is not 0 */
+ if (filep->is_directory &&
+ !mg_strcasecmp(conn->ctx->config[ENABLE_DIRECTORY_LISTING], "yes") &&
+ (depth == NULL || strcmp(depth, "0") != 0)) {
+ scan_directory(conn, path, conn, &print_dav_dir_entry);
+ }
+
+ conn->num_bytes_sent += mg_printf(conn, "%s\n", "</d:multistatus>");
+}
+
+void mg_lock(struct mg_connection* conn)
+{
+ (void) pthread_mutex_lock(&conn->mutex);
+}
+
+void mg_unlock(struct mg_connection* conn)
+{
+ (void) pthread_mutex_unlock(&conn->mutex);
+}
+
+#ifdef USE_LUA
+#include "mod_lua.inl"
+#endif /* USE_LUA */
+
+#if defined(USE_WEBSOCKET)
+
+/* START OF SHA-1 code
+ Copyright(c) By Steve Reid <steve at edmweb.com> */
+#define SHA1HANDSOFF
+#if defined(__sun)
+#include "solarisfixes.h"
+#endif
+
+static int is_big_endian(void)
+{
+ static const int n = 1;
+ return ((char *) &n)[0] == 0;
+}
+
+union char64long16 {
+ unsigned char c[64];
+ uint32_t l[16];
+};
+
+#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))
+
+static uint32_t blk0(union char64long16 *block, int i)
+{
+ /* Forrest: SHA expect BIG_ENDIAN, swap if LITTLE_ENDIAN */
+ if (!is_big_endian()) {
+ block->l[i] = (rol(block->l[i], 24) & 0xFF00FF00) |
+ (rol(block->l[i], 8) & 0x00FF00FF);
+ }
+ return block->l[i];
+}
+
+#define blk(i) (block->l[i&15] = rol(block->l[(i+13)&15]^block->l[(i+8)&15] \
+ ^block->l[(i+2)&15]^block->l[i&15],1))
+#define R0(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk0(block, i)+0x5A827999+rol(v,5);w=rol(w,30);
+#define R1(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=rol(w,30);
+#define R2(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=rol(w,30);
+#define R3(v,w,x,y,z,i) z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=rol(w,30);
+#define R4(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=rol(w,30);
+
+typedef struct {
+ uint32_t state[5];
+ uint32_t count[2];
+ unsigned char buffer[64];
+} SHA1_CTX;
+
+static void SHA1Transform(uint32_t state[5], const unsigned char buffer[64])
+{
+ uint32_t a, b, c, d, e;
+ union char64long16 block[1];
+
+ memcpy(block, buffer, 64);
+ a = state[0];
+ b = state[1];
+ c = state[2];
+ d = state[3];
+ e = state[4];
+ R0(a,b,c,d,e, 0);
+ R0(e,a,b,c,d, 1);
+ R0(d,e,a,b,c, 2);
+ R0(c,d,e,a,b, 3);
+ R0(b,c,d,e,a, 4);
+ R0(a,b,c,d,e, 5);
+ R0(e,a,b,c,d, 6);
+ R0(d,e,a,b,c, 7);
+ R0(c,d,e,a,b, 8);
+ R0(b,c,d,e,a, 9);
+ R0(a,b,c,d,e,10);
+ R0(e,a,b,c,d,11);
+ R0(d,e,a,b,c,12);
+ R0(c,d,e,a,b,13);
+ R0(b,c,d,e,a,14);
+ R0(a,b,c,d,e,15);
+ R1(e,a,b,c,d,16);
+ R1(d,e,a,b,c,17);
+ R1(c,d,e,a,b,18);
+ R1(b,c,d,e,a,19);
+ R2(a,b,c,d,e,20);
+ R2(e,a,b,c,d,21);
+ R2(d,e,a,b,c,22);
+ R2(c,d,e,a,b,23);
+ R2(b,c,d,e,a,24);
+ R2(a,b,c,d,e,25);
+ R2(e,a,b,c,d,26);
+ R2(d,e,a,b,c,27);
+ R2(c,d,e,a,b,28);
+ R2(b,c,d,e,a,29);
+ R2(a,b,c,d,e,30);
+ R2(e,a,b,c,d,31);
+ R2(d,e,a,b,c,32);
+ R2(c,d,e,a,b,33);
+ R2(b,c,d,e,a,34);
+ R2(a,b,c,d,e,35);
+ R2(e,a,b,c,d,36);
+ R2(d,e,a,b,c,37);
+ R2(c,d,e,a,b,38);
+ R2(b,c,d,e,a,39);
+ R3(a,b,c,d,e,40);
+ R3(e,a,b,c,d,41);
+ R3(d,e,a,b,c,42);
+ R3(c,d,e,a,b,43);
+ R3(b,c,d,e,a,44);
+ R3(a,b,c,d,e,45);
+ R3(e,a,b,c,d,46);
+ R3(d,e,a,b,c,47);
+ R3(c,d,e,a,b,48);
+ R3(b,c,d,e,a,49);
+ R3(a,b,c,d,e,50);
+ R3(e,a,b,c,d,51);
+ R3(d,e,a,b,c,52);
+ R3(c,d,e,a,b,53);
+ R3(b,c,d,e,a,54);
+ R3(a,b,c,d,e,55);
+ R3(e,a,b,c,d,56);
+ R3(d,e,a,b,c,57);
+ R3(c,d,e,a,b,58);
+ R3(b,c,d,e,a,59);
+ R4(a,b,c,d,e,60);
+ R4(e,a,b,c,d,61);
+ R4(d,e,a,b,c,62);
+ R4(c,d,e,a,b,63);
+ R4(b,c,d,e,a,64);
+ R4(a,b,c,d,e,65);
+ R4(e,a,b,c,d,66);
+ R4(d,e,a,b,c,67);
+ R4(c,d,e,a,b,68);
+ R4(b,c,d,e,a,69);
+ R4(a,b,c,d,e,70);
+ R4(e,a,b,c,d,71);
+ R4(d,e,a,b,c,72);
+ R4(c,d,e,a,b,73);
+ R4(b,c,d,e,a,74);
+ R4(a,b,c,d,e,75);
+ R4(e,a,b,c,d,76);
+ R4(d,e,a,b,c,77);
+ R4(c,d,e,a,b,78);
+ R4(b,c,d,e,a,79);
+ state[0] += a;
+ state[1] += b;
+ state[2] += c;
+ state[3] += d;
+ state[4] += e;
+ a = b = c = d = e = 0;
+ memset(block, '\0', sizeof(block));
+}
+
+static void SHA1Init(SHA1_CTX* context)
+{
+ context->state[0] = 0x67452301;
+ context->state[1] = 0xEFCDAB89;
+ context->state[2] = 0x98BADCFE;
+ context->state[3] = 0x10325476;
+ context->state[4] = 0xC3D2E1F0;
+ context->count[0] = context->count[1] = 0;
+}
+
+static void SHA1Update(SHA1_CTX* context, const unsigned char* data,
+ uint32_t len)
+{
+ uint32_t i, j;
+
+ j = context->count[0];
+ if ((context->count[0] += len << 3) < j)
+ context->count[1]++;
+ context->count[1] += (len>>29);
+ j = (j >> 3) & 63;
+ if ((j + len) > 63) {
+ memcpy(&context->buffer[j], data, (i = 64-j));
+ SHA1Transform(context->state, context->buffer);
+ for ( ; i + 63 < len; i += 64) {
+ SHA1Transform(context->state, &data[i]);
+ }
+ j = 0;
+ } else i = 0;
+ memcpy(&context->buffer[j], &data[i], len - i);
+}
+
+static void SHA1Final(unsigned char digest[20], SHA1_CTX* context)
+{
+ unsigned i;
+ unsigned char finalcount[8], c;
+
+ for (i = 0; i < 8; i++) {
+ finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)]
+ >> ((3-(i & 3)) * 8) ) & 255);
+ }
+ c = 0200;
+ SHA1Update(context, &c, 1);
+ while ((context->count[0] & 504) != 448) {
+ c = 0000;
+ SHA1Update(context, &c, 1);
+ }
+ SHA1Update(context, finalcount, 8);
+ for (i = 0; i < 20; i++) {
+ digest[i] = (unsigned char)
+ ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+ }
+ memset(context, '\0', sizeof(*context));
+ memset(&finalcount, '\0', sizeof(finalcount));
+}
+/* END OF SHA1 CODE */
+
+static void base64_encode(const unsigned char *src, int src_len, char *dst)
+{
+ static const char *b64 =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ int i, j, a, b, c;
+
+ for (i = j = 0; i < src_len; i += 3) {
+ a = src[i];
+ b = i + 1 >= src_len ? 0 : src[i + 1];
+ c = i + 2 >= src_len ? 0 : src[i + 2];
+
+ dst[j++] = b64[a >> 2];
+ dst[j++] = b64[((a & 3) << 4) | (b >> 4)];
+ if (i + 1 < src_len) {
+ dst[j++] = b64[(b & 15) << 2 | (c >> 6)];
+ }
+ if (i + 2 < src_len) {
+ dst[j++] = b64[c & 63];
+ }
+ }
+ while (j % 4 != 0) {
+ dst[j++] = '=';
+ }
+ dst[j++] = '\0';
+}
+
+static void send_websocket_handshake(struct mg_connection *conn)
+{
+ static const char *magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
+ char buf[100], sha[20], b64_sha[sizeof(sha) * 2];
+ SHA1_CTX sha_ctx;
+
+ mg_snprintf(conn, buf, sizeof(buf), "%s%s",
+ mg_get_header(conn, "Sec-WebSocket-Key"), magic);
+ SHA1Init(&sha_ctx);
+ SHA1Update(&sha_ctx, (unsigned char *) buf, (uint32_t)strlen(buf));
+ SHA1Final((unsigned char *) sha, &sha_ctx);
+ base64_encode((unsigned char *) sha, sizeof(sha), b64_sha);
+ mg_printf(conn, "%s%s%s",
+ "HTTP/1.1 101 Switching Protocols\r\n"
+ "Upgrade: websocket\r\n"
+ "Connection: Upgrade\r\n"
+ "Sec-WebSocket-Accept: ", b64_sha, "\r\n\r\n");
+}
+
+static void read_websocket(struct mg_connection *conn)
+{
+ /* Pointer to the beginning of the portion of the incoming websocket
+ message queue.
+ The original websocket upgrade request is never removed, so the queue
+ begins after it. */
+ unsigned char *buf = (unsigned char *) conn->buf + conn->request_len;
+ int n, error;
+
+ /* body_len is the length of the entire queue in bytes
+ len is the length of the current message
+ data_len is the length of the current message's data payload
+ header_len is the length of the current message's header */
+ size_t i, len, mask_len, data_len, header_len, body_len;
+
+ /* "The masking key is a 32-bit value chosen at random by the client."
+ http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-17#section-5 */
+ unsigned char mask[4];
+
+ /* data points to the place where the message is stored when passed to the
+ websocket_data callback. This is either mem on the stack, or a
+ dynamically allocated buffer if it is too large. */
+ char mem[4096];
+ char *data = mem;
+ unsigned char mop; /* mask flag and opcode */
+
+ /* Loop continuously, reading messages from the socket, invoking the
+ callback, and waiting repeatedly until an error occurs. */
+ assert(conn->content_len == 0);
+ for (;;) {
+ header_len = 0;
+ assert(conn->data_len >= conn->request_len);
+ if ((body_len = conn->data_len - conn->request_len) >= 2) {
+ len = buf[1] & 127;
+ mask_len = buf[1] & 128 ? 4 : 0;
+ if (len < 126 && body_len >= mask_len) {
+ data_len = len;
+ header_len = 2 + mask_len;
+ } else if (len == 126 && body_len >= 4 + mask_len) {
+ header_len = 4 + mask_len;
+ data_len = ((((int) buf[2]) << 8) + buf[3]);
+ } else if (body_len >= 10 + mask_len) {
+ header_len = 10 + mask_len;
+ data_len = (((uint64_t) ntohl(* (uint32_t *) &buf[2])) << 32) +
+ ntohl(* (uint32_t *) &buf[6]);
+ }
+ }
+
+ if (header_len > 0) {
+ /* Allocate space to hold websocket payload */
+ data = mem;
+ if (data_len > sizeof(mem)) {
+ data = (char *)malloc(data_len);
+ if (data == NULL) {
+ /* Allocation failed, exit the loop and then close the
+ connection */
+ mg_cry(conn, "websocket malloc() failed; closing connection");
+ break;
+ }
+ }
+
+ /* Copy the mask before we shift the queue and destroy it */
+ if (mask_len > 0) {
+ *(uint32_t*)mask = *(uint32_t*)(buf + header_len - mask_len);
+ } else {
+ *(uint32_t*)mask = 0;
+ }
+
+ /* Read frame payload from the first message in the queue into
+ data and advance the queue by moving the memory in place. */
+ assert(body_len >= header_len);
+ if (data_len + header_len > body_len) {
+ mop = buf[0]; /* current mask and opcode */
+ /* Overflow case */
+ len = body_len - header_len;
+ memcpy(data, buf + header_len, len);
+ error = 0;
+ while (len < data_len) {
+ int n = pull(NULL, conn, data + len, (int)(data_len - len));
+ if (n < 0) {
+ error = 1;
+ break;
+ }
+ len += n;
+ }
+ if (error) {
+ mg_cry(conn, "Websocket pull failed; closing connection");
+ break;
+ }
+ conn->data_len = conn->request_len;
+ } else {
+ mop = buf[0]; /* current mask and opcode, overwritten by memmove() */
+ /* Length of the message being read at the front of the
+ queue */
+ len = data_len + header_len;
+
+ /* Copy the data payload into the data pointer for the
+ callback */
+ memcpy(data, buf + header_len, data_len);
+
+ /* Move the queue forward len bytes */
+ memmove(buf, buf + len, body_len - len);
+
+ /* Mark the queue as advanced */
+ conn->data_len -= (int)len;
+ }
+
+ /* Apply mask if necessary */
+ if (mask_len > 0) {
+ for (i = 0; i < data_len; ++i) {
+ data[i] ^= mask[i & 3];
+ }
+ }
+
+ /* Exit the loop if callback signalled to exit,
+ or "connection close" opcode received. */
+ if ((conn->ctx->callbacks.websocket_data != NULL &&
+#ifdef USE_LUA
+ (conn->lua_websocket_state == NULL) &&
+#endif
+ !conn->ctx->callbacks.websocket_data(conn, mop, data, data_len)) ||
+#ifdef USE_LUA
+ (conn->lua_websocket_state &&
+ !lua_websocket_data(conn, mop, data, data_len)) ||
+#endif
+ (buf[0] & 0xf) == WEBSOCKET_OPCODE_CONNECTION_CLOSE) { /* Opcode == 8, connection close */
+ break;
+ }
+
+ if (data != mem) {
+ free(data);
+ }
+ /* Not breaking the loop, process next websocket frame. */
+ } else {
+ /* Read from the socket into the next available location in the
+ message queue. */
+ if ((n = pull(NULL, conn, conn->buf + conn->data_len,
+ conn->buf_size - conn->data_len)) <= 0) {
+ /* Error, no bytes read */
+ break;
+ }
+ conn->data_len += n;
+ }
+ }
+}
+
+int mg_websocket_write(struct mg_connection* conn, int opcode, const char* data, size_t dataLen)
+{
+ unsigned char header[10];
+ size_t headerLen = 1;
+
+ int retval = -1;
+
+ header[0] = 0x80 + (opcode & 0xF);
+
+ /* Frame format: http://tools.ietf.org/html/rfc6455#section-5.2 */
+ if (dataLen < 126) {
+ /* inline 7-bit length field */
+ header[1] = (unsigned char)dataLen;
+ headerLen = 2;
+ } else if (dataLen <= 0xFFFF) {
+ /* 16-bit length field */
+ header[1] = 126;
+ *(uint16_t*)(header + 2) = htons((uint16_t)dataLen);
+ headerLen = 4;
+ } else {
+ /* 64-bit length field */
+ header[1] = 127;
+ *(uint32_t*)(header + 2) = htonl((uint64_t)dataLen >> 32);
+ *(uint32_t*)(header + 6) = htonl(dataLen & 0xFFFFFFFF);
+ headerLen = 10;
+ }
+
+ /* Note that POSIX/Winsock's send() is threadsafe
+ http://stackoverflow.com/questions/1981372/are-parallel-calls-to-send-recv-on-the-same-socket-valid
+ but mongoose's mg_printf/mg_write is not (because of the loop in
+ push(), although that is only a problem if the packet is large or
+ outgoing buffer is full). */
+ (void) mg_lock(conn);
+ retval = mg_write(conn, header, headerLen);
+ retval = mg_write(conn, data, dataLen);
+ mg_unlock(conn);
+
+ return retval;
+}
+
+static void handle_websocket_request(struct mg_connection *conn, const char *path, int is_script_resource)
+{
+ const char *version = mg_get_header(conn, "Sec-WebSocket-Version");
+ if (version == NULL || strcmp(version, "13") != 0) {
+ send_http_error(conn, 426, "Upgrade Required", "%s", "Upgrade Required");
+ } else if (conn->ctx->callbacks.websocket_connect != NULL &&
+ conn->ctx->callbacks.websocket_connect(conn) != 0) {
+ /* C callback has returned non-zero, do not proceed with handshake. */
+ /* The C callback is called before Lua and may prevent Lua from handling the websocket. */
+ } else {
+#ifdef USE_LUA
+ if (match_prefix(conn->ctx->config[LUA_SCRIPT_EXTENSIONS],
+ (int)strlen(conn->ctx->config[LUA_SCRIPT_EXTENSIONS]),
+ path)) {
+ conn->lua_websocket_state = new_lua_websocket(path, conn);
+ if (conn->lua_websocket_state) {
+ send_websocket_handshake(conn);
+ if (lua_websocket_ready(conn)) {
+ read_websocket(conn);
+ }
+ }
+ } else
+#endif
+ {
+ /* No Lua websock script specified. */
+ send_websocket_handshake(conn);
+ if (conn->ctx->callbacks.websocket_ready != NULL) {
+ conn->ctx->callbacks.websocket_ready(conn);
+ }
+ read_websocket(conn);
+ }
+ }
+}
+
+static int is_websocket_request(const struct mg_connection *conn)
+{
+ const char *host, *upgrade, *connection, *version, *key;
+
+ host = mg_get_header(conn, "Host");
+ upgrade = mg_get_header(conn, "Upgrade");
+ connection = mg_get_header(conn, "Connection");
+ key = mg_get_header(conn, "Sec-WebSocket-Key");
+ version = mg_get_header(conn, "Sec-WebSocket-Version");
+
+ return host != NULL && upgrade != NULL && connection != NULL &&
+ key != NULL && version != NULL &&
+ mg_strcasestr(upgrade, "websocket") != NULL &&
+ mg_strcasestr(connection, "Upgrade") != NULL;
+}
+#endif /* !USE_WEBSOCKET */
+
+static int isbyte(int n)
+{
+ return n >= 0 && n <= 255;
+}
+
+static int parse_net(const char *spec, uint32_t *net, uint32_t *mask)
+{
+ int n, a, b, c, d, slash = 32, len = 0;
+
+ if ((sscanf(spec, "%d.%d.%d.%d/%d%n", &a, &b, &c, &d, &slash, &n) == 5 ||
+ sscanf(spec, "%d.%d.%d.%d%n", &a, &b, &c, &d, &n) == 4) &&
+ isbyte(a) && isbyte(b) && isbyte(c) && isbyte(d) &&
+ slash >= 0 && slash < 33) {
+ len = n;
+ *net = ((uint32_t)a << 24) | ((uint32_t)b << 16) | ((uint32_t)c << 8) | d;
+ *mask = slash ? 0xffffffffU << (32 - slash) : 0;
+ }
+
+ return len;
+}
+
+static int set_throttle(const char *spec, uint32_t remote_ip, const char *uri)
+{
+ int throttle = 0;
+ struct vec vec, val;
+ uint32_t net, mask;
+ char mult;
+ double v;
+
+ while ((spec = next_option(spec, &vec, &val)) != NULL) {
+ mult = ',';
+ if (sscanf(val.ptr, "%lf%c", &v, &mult) < 1 || v < 0 ||
+ (lowercase(&mult) != 'k' && lowercase(&mult) != 'm' && mult != ',')) {
+ continue;
+ }
+ v *= lowercase(&mult) == 'k' ? 1024 : lowercase(&mult) == 'm' ? 1048576 : 1;
+ if (vec.len == 1 && vec.ptr[0] == '*') {
+ throttle = (int) v;
+ } else if (parse_net(vec.ptr, &net, &mask) > 0) {
+ if ((remote_ip & mask) == net) {
+ throttle = (int) v;
+ }
+ } else if (match_prefix(vec.ptr, (int)vec.len, uri) > 0) {
+ throttle = (int) v;
+ }
+ }
+
+ return throttle;
+}
+
+static uint32_t get_remote_ip(const struct mg_connection *conn)
+{
+ return ntohl(* (uint32_t *) &conn->client.rsa.sin.sin_addr);
+}
+
+int mg_upload(struct mg_connection *conn, const char *destination_dir)
+{
+ const char *content_type_header, *boundary_start;
+ char buf[MG_BUF_LEN], path[PATH_MAX], fname[1024], boundary[100], *s;
+ FILE *fp;
+ int bl, n, i, j, headers_len, boundary_len, eof,
+ len = 0, num_uploaded_files = 0;
+
+ /* Request looks like this:
+
+ POST /upload HTTP/1.1
+ Host: 127.0.0.1:8080
+ Content-Length: 244894
+ Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryRVr
+
+ ------WebKitFormBoundaryRVr
+ Content-Disposition: form-data; name="file"; filename="accum.png"
+ Content-Type: image/png
+
+ <89>PNG
+ <PNG DATA>
+ ------WebKitFormBoundaryRVr */
+
+ /* Extract boundary string from the Content-Type header */
+ if ((content_type_header = mg_get_header(conn, "Content-Type")) == NULL ||
+ (boundary_start = mg_strcasestr(content_type_header,
+ "boundary=")) == NULL ||
+ (sscanf(boundary_start, "boundary=\"%99[^\"]\"", boundary) == 0 &&
+ sscanf(boundary_start, "boundary=%99s", boundary) == 0) ||
+ boundary[0] == '\0') {
+ return num_uploaded_files;
+ }
+
+ boundary_len = (int)strlen(boundary);
+ bl = boundary_len + 4; /* \r\n--<boundary> */
+ for (;;) {
+ /* Pull in headers */
+ assert(len >= 0 && len <= (int) sizeof(buf));
+ while ((n = mg_read(conn, buf + len, sizeof(buf) - len)) > 0) {
+ len += n;
+ }
+ if ((headers_len = get_request_len(buf, len)) <= 0) {
+ break;
+ }
+
+ /* Fetch file name. */
+ fname[0] = '\0';
+ for (i = j = 0; i < headers_len; i++) {
+ if (buf[i] == '\r' && buf[i + 1] == '\n') {
+ buf[i] = buf[i + 1] = '\0';
+ /* TODO(lsm): don't expect filename to be the 3rd field,
+ parse the header properly instead. */
+ IGNORE_UNUSED_RESULT(sscanf(&buf[j], "Content-Disposition: %*s %*s filename=\"%1023[^\"]",
+ fname));
+ j = i + 2;
+ }
+ }
+
+ /* Give up if the headers are not what we expect */
+ if (fname[0] == '\0') {
+ break;
+ }
+
+ /* Move data to the beginning of the buffer */
+ assert(len >= headers_len);
+ memmove(buf, &buf[headers_len], len - headers_len);
+ len -= headers_len;
+
+ /* We open the file with exclusive lock held. This guarantee us
+ there is no other thread can save into the same file
+ simultaneously. */
+ fp = NULL;
+ /* Construct destination file name. Do not allow paths to have
+ slashes. */
+ if ((s = strrchr(fname, '/')) == NULL &&
+ (s = strrchr(fname, '\\')) == NULL) {
+ s = fname;
+ }
+
+ /* Open file in binary mode. TODO: set an exclusive lock. */
+ snprintf(path, sizeof(path), "%s/%s", destination_dir, s);
+ if ((fp = fopen(path, "wb")) == NULL) {
+ break;
+ }
+
+ /* Read POST data, write into file until boundary is found. */
+ eof = n = 0;
+ do {
+ len += n;
+ for (i = 0; i < len - bl; i++) {
+ if (!memcmp(&buf[i], "\r\n--", 4) &&
+ !memcmp(&buf[i + 4], boundary, boundary_len)) {
+ /* Found boundary, that's the end of file data. */
+ fwrite(buf, 1, i, fp);
+ eof = 1;
+ memmove(buf, &buf[i + bl], len - (i + bl));
+ len -= i + bl;
+ break;
+ }
+ }
+ if (!eof && len > bl) {
+ fwrite(buf, 1, len - bl, fp);
+ memmove(buf, &buf[len - bl], bl);
+ len = bl;
+ }
+ } while (!eof && (n = mg_read(conn, buf + len, sizeof(buf) - len)) > 0);
+ fclose(fp);
+ if (eof) {
+ num_uploaded_files++;
+ if (conn->ctx->callbacks.upload != NULL) {
+ conn->ctx->callbacks.upload(conn, path);
+ }
+ }
+ }
+
+ return num_uploaded_files;
+}
+
+static int is_put_or_delete_request(const struct mg_connection *conn)
+{
+ const char *s = conn->request_info.request_method;
+ return s != NULL && (!strcmp(s, "PUT") ||
+ !strcmp(s, "DELETE") ||
+ !strcmp(s, "MKCOL"));
+}
+
+static int get_first_ssl_listener_index(const struct mg_context *ctx)
+{
+ int i, idx = -1;
+ for (i = 0; idx == -1 && i < ctx->num_listening_sockets; i++) {
+ idx = ctx->listening_sockets[i].is_ssl ? i : -1;
+ }
+ return idx;
+}
+
+static void redirect_to_https_port(struct mg_connection *conn, int ssl_index)
+{
+ char host[1025];
+ const char *host_header;
+ size_t hostlen;
+
+ host_header = mg_get_header(conn, "Host");
+ hostlen = sizeof(host);
+ if (host_header != NULL) {
+ char *pos;
+
+ strncpy(host, host_header, hostlen);
+ host[hostlen - 1] = '\0';
+ pos = strchr(host, ':');
+ if (pos != NULL) {
+ *pos = '\0';
+ }
+ } else {
+ /* Cannot get host from the Host: header.
+ Fallback to our IP address. */
+ sockaddr_to_string(host, hostlen, &conn->client.lsa);
+ }
+
+ mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: https://%s:%d%s\r\n\r\n",
+ host, (int) ntohs(conn->ctx->listening_sockets[ssl_index].
+ lsa.sin.sin_port), conn->request_info.uri);
+}
+
+
+void mg_set_request_handler(struct mg_context *ctx, const char *uri, mg_request_handler handler, void *cbdata)
+{
+ struct mg_request_handler_info *tmp_rh, *lastref = 0;
+ size_t urilen = strlen(uri);
+
+ /* first see it the uri exists */
+ for (tmp_rh = ctx->request_handlers;
+ tmp_rh != NULL && strcmp(uri, tmp_rh->uri);
+ lastref = tmp_rh, tmp_rh = tmp_rh->next) {
+ /* first try for an exact match */
+ if (urilen == tmp_rh->uri_len && !strcmp(tmp_rh->uri,uri)) {
+ /* already there... */
+
+ if (handler != NULL) {
+ /* change this entry */
+ tmp_rh->handler = handler;
+ tmp_rh->cbdata = cbdata;
+ } else {
+ /* remove this entry */
+ if (lastref != NULL)
+ lastref->next = tmp_rh->next;
+ else
+ ctx->request_handlers = tmp_rh->next;
+ free(tmp_rh->uri);
+ free(tmp_rh);
+ }
+ return;
+ }
+
+ /* next try for a partial match, we will accept uri/something */
+ if (tmp_rh->uri_len < urilen
+ && uri[tmp_rh->uri_len] == '/'
+ && memcmp(tmp_rh->uri, uri, tmp_rh->uri_len) == 0) {
+ /* if there is a partical match this new entry MUST go BEFORE
+ the current position otherwise it will never be matched. */
+ break;
+ }
+
+ }
+
+ if (handler == NULL) {
+ /* no handler to set, this was a remove request */
+ return;
+ }
+
+ tmp_rh = (struct mg_request_handler_info *)malloc(sizeof(struct mg_request_handler_info));
+ if (tmp_rh == NULL) {
+ mg_cry(fc(ctx), "%s", "Cannot create new request handler struct, OOM");
+ return;
+ }
+ tmp_rh->uri = mg_strdup(uri);
+ tmp_rh->uri_len = strlen(uri);
+ tmp_rh->handler = handler;
+ tmp_rh->cbdata = cbdata;
+
+ if (lastref == NULL) {
+ tmp_rh->next = ctx->request_handlers;
+ ctx->request_handlers = tmp_rh;
+ } else {
+ tmp_rh->next = lastref->next;
+ lastref->next = tmp_rh;
+ }
+
+}
+
+static int use_request_handler(struct mg_connection *conn)
+{
+ struct mg_request_info *request_info = mg_get_request_info(conn);
+ const char *uri = request_info->uri;
+ size_t urilen = strlen(uri);
+ struct mg_request_handler_info *tmp_rh = conn->ctx->request_handlers;
+
+ for (; tmp_rh != NULL; tmp_rh = tmp_rh->next) {
+
+ /* first try for an exact match */
+ if (urilen == tmp_rh->uri_len && !strcmp(tmp_rh->uri,uri)) {
+ return tmp_rh->handler(conn, tmp_rh->cbdata);
+ }
+
+ /* next try for a partial match */
+ /* we will accept uri/something */
+ if (tmp_rh->uri_len < urilen
+ && uri[tmp_rh->uri_len] == '/'
+ && memcmp(tmp_rh->uri, uri, tmp_rh->uri_len) == 0) {
+
+ return tmp_rh->handler(conn, tmp_rh->cbdata);
+ }
+
+ }
+
+ return 0; /* none found */
+}
+
+/* This is the heart of the Civetweb's logic.
+ This function is called when the request is read, parsed and validated,
+ and Civetweb must decide what action to take: serve a file, or
+ a directory, or call embedded function, etcetera. */
+static void handle_request(struct mg_connection *conn)
+{
+ struct mg_request_info *ri = &conn->request_info;
+ char path[PATH_MAX];
+ int uri_len, ssl_index, is_script_resource;
+ struct file file = STRUCT_FILE_INITIALIZER;
+ char date[64];
+ time_t curtime = time(NULL);
+
+ if ((conn->request_info.query_string = strchr(ri->uri, '?')) != NULL) {
+ * ((char *) conn->request_info.query_string++) = '\0';
+ }
+ uri_len = (int) strlen(ri->uri);
+ mg_url_decode(ri->uri, uri_len, (char *) ri->uri, uri_len + 1, 0);
+ remove_double_dots_and_double_slashes((char *) ri->uri);
+ path[0] = '\0';
+ convert_uri_to_file_name(conn, path, sizeof(path), &file, &is_script_resource);
+ conn->throttle = set_throttle(conn->ctx->config[THROTTLE],
+ get_remote_ip(conn), ri->uri);
+
+ DEBUG_TRACE(("%s", ri->uri));
+ /* Perform redirect and auth checks before calling begin_request() handler.
+ Otherwise, begin_request() would need to perform auth checks and
+ redirects. */
+ if (!conn->client.is_ssl && conn->client.ssl_redir &&
+ (ssl_index = get_first_ssl_listener_index(conn->ctx)) > -1) {
+ redirect_to_https_port(conn, ssl_index);
+ } else if (!is_script_resource && !is_put_or_delete_request(conn) &&
+ !check_authorization(conn, path)) {
+ send_authorization_request(conn);
+ } else if (conn->ctx->callbacks.begin_request != NULL &&
+ conn->ctx->callbacks.begin_request(conn)) {
+ /* Do nothing, callback has served the request */
+#if defined(USE_WEBSOCKET)
+ } else if (is_websocket_request(conn)) {
+ handle_websocket_request(conn, path, is_script_resource);
+#endif
+ } else if (conn->ctx->request_handlers != NULL &&
+ use_request_handler(conn)) {
+ /* Do nothing, callback has served the request */
+ } else if (!is_script_resource && !strcmp(ri->request_method, "OPTIONS")) {
+ send_options(conn);
+ } else if (conn->ctx->config[DOCUMENT_ROOT] == NULL) {
+ send_http_error(conn, 404, "Not Found", "Not Found");
+ } else if (!is_script_resource && is_put_or_delete_request(conn) &&
+ (is_authorized_for_put(conn) != 1)) {
+ send_authorization_request(conn);
+ } else if (!is_script_resource && !strcmp(ri->request_method, "PUT")) {
+ put_file(conn, path);
+ } else if (!is_script_resource && !strcmp(ri->request_method, "MKCOL")) {
+ mkcol(conn, path);
+ } else if (!is_script_resource && !strcmp(ri->request_method, "DELETE")) {
+ struct de de;
+ memset(&de.file, 0, sizeof(de.file));
+ if(!mg_stat(conn, path, &de.file)) {
+ send_http_error(conn, 404, "Not Found", "%s", "File not found");
+ } else {
+ if(de.file.modification_time) {
+ if(de.file.is_directory) {
+ remove_directory(conn, path);
+ send_http_error(conn, 204, "No Content", "%s", "");
+ } else if (mg_remove(path) == 0) {
+ send_http_error(conn, 204, "No Content", "%s", "");
+ } else {
+ send_http_error(conn, 423, "Locked", "remove(%s): %s", path,
+ strerror(ERRNO));
+ }
+ } else {
+ send_http_error(conn, 500, http_500_error, "remove(%s): %s", path,
+ strerror(ERRNO));
+ }
+ }
+ } else if ((file.membuf == NULL && file.modification_time == (time_t) 0) ||
+ must_hide_file(conn, path)) {
+ send_http_error(conn, 404, "Not Found", "%s", "File not found");
+ } else if (file.is_directory && ri->uri[uri_len - 1] != '/') {
+ gmt_time_string(date, sizeof(date), &curtime);
+ mg_printf(conn, "HTTP/1.1 301 Moved Permanently\r\n"
+ "Location: %s/\r\n"
+ "Date: %s\r\n"
+ "Content-Length: 0\r\n"
+ "Connection: %s\r\n\r\n",
+ ri->uri, date, suggest_connection_header(conn));
+ } else if (!is_script_resource && !strcmp(ri->request_method, "PROPFIND")) {
+ handle_propfind(conn, path, &file);
+ } else if (file.is_directory &&
+ !substitute_index_file(conn, path, sizeof(path), &file)) {
+ if (!mg_strcasecmp(conn->ctx->config[ENABLE_DIRECTORY_LISTING], "yes")) {
+ handle_directory_request(conn, path);
+ } else {
+ send_http_error(conn, 403, "Directory Listing Denied",
+ "Directory listing denied");
+ }
+#ifdef USE_LUA
+ } else if (match_prefix(conn->ctx->config[LUA_SERVER_PAGE_EXTENSIONS],
+ (int)strlen(conn->ctx->config[LUA_SERVER_PAGE_EXTENSIONS]),
+ path) > 0) {
+ /* Lua server page: an SSI like page containing mostly plain html code plus some tags with server generated contents. */
+ handle_lsp_request(conn, path, &file, NULL);
+ } else if (match_prefix(conn->ctx->config[LUA_SCRIPT_EXTENSIONS],
+ (int)strlen(conn->ctx->config[LUA_SCRIPT_EXTENSIONS]),
+ path) > 0) {
+ /* Lua in-server module script: a CGI like script used to generate the entire reply. */
+ mg_exec_lua_script(conn, path, NULL);
+#endif
+#if !defined(NO_CGI)
+ } else if (match_prefix(conn->ctx->config[CGI_EXTENSIONS],
+ (int)strlen(conn->ctx->config[CGI_EXTENSIONS]),
+ path) > 0) {
+ /* TODO: check unsupported methods -> 501
+ if (strcmp(ri->request_method, "POST") &&
+ strcmp(ri->request_method, "HEAD") &&
+ strcmp(ri->request_method, "GET")) {
+ send_http_error(conn, 501, "Not Implemented",
+ "Method %s is not implemented", ri->request_method);
+ } else {
+ handle_cgi_request(conn, path);
+ } */
+ handle_cgi_request(conn, path);
+#endif /* !NO_CGI */
+ } else if (match_prefix(conn->ctx->config[SSI_EXTENSIONS],
+ (int)strlen(conn->ctx->config[SSI_EXTENSIONS]),
+ path) > 0) {
+ handle_ssi_file_request(conn, path);
+ } else if (is_not_modified(conn, &file)) {
+ send_http_error(conn, 304, "Not Modified", "%s", "");
+ } else {
+ handle_file_request(conn, path, &file);
+ }
+}
+
+static void close_all_listening_sockets(struct mg_context *ctx)
+{
+ int i;
+ for (i = 0; i < ctx->num_listening_sockets; i++) {
+ closesocket(ctx->listening_sockets[i].sock);
+ }
+ free(ctx->listening_sockets);
+}
+
+static int is_valid_port(unsigned int port)
+{
+ return port < 0xffff;
+}
+
+/* Valid listening port specification is: [ip_address:]port[s]
+ Examples: 80, 443s, 127.0.0.1:3128, 1.2.3.4:8080s
+ TODO(lsm): add parsing of the IPv6 address */
+static int parse_port_string(const struct vec *vec, struct socket *so)
+{
+ unsigned int a, b, c, d, ch, len, port;
+#if defined(USE_IPV6)
+ char buf[100];
+#endif
+
+ /* MacOS needs that. If we do not zero it, subsequent bind() will fail.
+ Also, all-zeroes in the socket address means binding to all addresses
+ for both IPv4 and IPv6 (INADDR_ANY and IN6ADDR_ANY_INIT). */
+ memset(so, 0, sizeof(*so));
+ so->lsa.sin.sin_family = AF_INET;
+
+ if (sscanf(vec->ptr, "%u.%u.%u.%u:%u%n", &a, &b, &c, &d, &port, &len) == 5) {
+ /* Bind to a specific IPv4 address, e.g. 192.168.1.5:8080 */
+ so->lsa.sin.sin_addr.s_addr = htonl((a << 24) | (b << 16) | (c << 8) | d);
+ so->lsa.sin.sin_port = htons((uint16_t) port);
+#if defined(USE_IPV6)
+
+ } else if (sscanf(vec->ptr, "[%49[^]]]:%d%n", buf, &port, &len) == 2 &&
+ inet_pton(AF_INET6, buf, &so->lsa.sin6.sin6_addr)) {
+ /* IPv6 address, e.g. [3ffe:2a00:100:7031::1]:8080 */
+ so->lsa.sin6.sin6_family = AF_INET6;
+ so->lsa.sin6.sin6_port = htons((uint16_t) port);
+#endif
+ } else if (sscanf(vec->ptr, "%u%n", &port, &len) == 1) {
+ /* If only port is specified, bind to IPv4, INADDR_ANY */
+ so->lsa.sin.sin_port = htons((uint16_t) port);
+ } else {
+ port = len = 0; /* Parsing failure. Make port invalid. */
+ }
+
+ ch = vec->ptr[len]; /* Next character after the port number */
+ so->is_ssl = ch == 's';
+ so->ssl_redir = ch == 'r';
+
+ /* Make sure the port is valid and vector ends with 's', 'r' or ',' */
+ return is_valid_port(port) &&
+ (ch == '\0' || ch == 's' || ch == 'r' || ch == ',');
+}
+
+static int set_ports_option(struct mg_context *ctx)
+{
+ const char *list = ctx->config[LISTENING_PORTS];
+ int on = 1, success = 1;
+#if defined(USE_IPV6)
+ int off = 0;
+#endif
+ struct vec vec;
+ struct socket so, *ptr;
+
+ in_port_t *portPtr;
+ struct sockaddr_in sin;
+ socklen_t len;
+
+ memset(&sin, 0, sizeof(sin));
+ len = sizeof(sin);
+
+ while (success && (list = next_option(list, &vec, NULL)) != NULL) {
+ if (!parse_port_string(&vec, &so)) {
+ mg_cry(fc(ctx), "%s: %.*s: invalid port spec. Expecting list of: %s",
+ __func__, (int) vec.len, vec.ptr, "[IP_ADDRESS:]PORT[s|r]");
+ success = 0;
+ } else if (so.is_ssl && ctx->ssl_ctx == NULL) {
+ mg_cry(fc(ctx), "Cannot add SSL socket, is -ssl_certificate option set?");
+ success = 0;
+ } else if ((so.sock = socket(so.lsa.sa.sa_family, SOCK_STREAM, 6)) ==
+ INVALID_SOCKET ||
+ /* On Windows, SO_REUSEADDR is recommended only for
+ broadcast UDP sockets */
+ setsockopt(so.sock, SOL_SOCKET, SO_REUSEADDR,
+ (void *) &on, sizeof(on)) != 0 ||
+#if defined(USE_IPV6)
+ (so.lsa.sa.sa_family == AF_INET6 &&
+ setsockopt(so.sock, IPPROTO_IPV6, IPV6_V6ONLY, (void *) &off,
+ sizeof(off)) != 0) ||
+#endif
+ bind(so.sock, &so.lsa.sa, so.lsa.sa.sa_family == AF_INET ?
+ sizeof(so.lsa.sin) : sizeof(so.lsa)) != 0 ||
+ listen(so.sock, SOMAXCONN) != 0 ||
+ getsockname(so.sock, (struct sockaddr *)&sin, &len) != 0) {
+ mg_cry(fc(ctx), "%s: cannot bind to %.*s: %d (%s)", __func__,
+ (int) vec.len, vec.ptr, ERRNO, strerror(errno));
+ if (so.sock != INVALID_SOCKET) {
+ closesocket(so.sock);
+ }
+ success = 0;
+ } else if ((ptr = (struct socket *) realloc(ctx->listening_sockets,
+ (ctx->num_listening_sockets + 1) *
+ sizeof(ctx->listening_sockets[0]))) == NULL) {
+ closesocket(so.sock);
+ success = 0;
+ } else if ((portPtr = (in_port_t*) realloc(ctx->listening_ports,
+ (ctx->num_listening_sockets + 1) *
+ sizeof(ctx->listening_ports[0]))) == NULL) {
+ closesocket(so.sock);
+ success = 0;
+ }
+ else {
+ set_close_on_exec(so.sock, fc(ctx));
+ ctx->listening_sockets = ptr;
+ ctx->listening_sockets[ctx->num_listening_sockets] = so;
+ ctx->listening_ports = portPtr;
+ ctx->listening_ports[ctx->num_listening_sockets] = ntohs(sin.sin_port);
+ ctx->num_listening_sockets++;
+ }
+ }
+
+ if (!success) {
+ close_all_listening_sockets(ctx);
+ }
+
+ return success;
+}
+
+static void log_header(const struct mg_connection *conn, const char *header,
+ FILE *fp)
+{
+ const char *header_value;
+
+ if ((header_value = mg_get_header(conn, header)) == NULL) {
+ (void) fprintf(fp, "%s", " -");
+ } else {
+ (void) fprintf(fp, " \"%s\"", header_value);
+ }
+}
+
+static void log_access(const struct mg_connection *conn)
+{
+ const struct mg_request_info *ri;
+ FILE *fp;
+ char date[64], src_addr[IP_ADDR_STR_LEN];
+ struct tm *tm;
+
+ fp = conn->ctx->config[ACCESS_LOG_FILE] == NULL ? NULL :
+ fopen(conn->ctx->config[ACCESS_LOG_FILE], "a+");
+
+ if (fp == NULL)
+ return;
+
+ tm = localtime(&conn->birth_time);
+ if (tm != NULL) {
+ strftime(date, sizeof(date), "%d/%b/%Y:%H:%M:%S %z", tm);
+ } else {
+ strncpy(date, "01/Jan/1970:00:00:00 +0000", sizeof(date));
+ date[sizeof(date) - 1] = '\0';
+ }
+
+ ri = &conn->request_info;
+ flockfile(fp);
+
+ sockaddr_to_string(src_addr, sizeof(src_addr), &conn->client.rsa);
+ fprintf(fp, "%s - %s [%s] \"%s %s HTTP/%s\" %d %" INT64_FMT,
+ src_addr, ri->remote_user == NULL ? "-" : ri->remote_user, date,
+ ri->request_method ? ri->request_method : "-",
+ ri->uri ? ri->uri : "-", ri->http_version,
+ conn->status_code, conn->num_bytes_sent);
+ log_header(conn, "Referer", fp);
+ log_header(conn, "User-Agent", fp);
+ fputc('\n', fp);
+ fflush(fp);
+
+ funlockfile(fp);
+ fclose(fp);
+}
+
+/* Verify given socket address against the ACL.
+ Return -1 if ACL is malformed, 0 if address is disallowed, 1 if allowed. */
+static int check_acl(struct mg_context *ctx, uint32_t remote_ip)
+{
+ int allowed, flag;
+ uint32_t net, mask;
+ struct vec vec;
+ const char *list = ctx->config[ACCESS_CONTROL_LIST];
+
+ /* If any ACL is set, deny by default */
+ allowed = list == NULL ? '+' : '-';
+
+ while ((list = next_option(list, &vec, NULL)) != NULL) {
+ flag = vec.ptr[0];
+ if ((flag != '+' && flag != '-') ||
+ parse_net(&vec.ptr[1], &net, &mask) == 0) {
+ mg_cry(fc(ctx), "%s: subnet must be [+|-]x.x.x.x[/x]", __func__);
+ return -1;
+ }
+
+ if (net == (remote_ip & mask)) {
+ allowed = flag;
+ }
+ }
+
+ return allowed == '+';
+}
+
+#if !defined(_WIN32)
+static int set_uid_option(struct mg_context *ctx)
+{
+ struct passwd *pw;
+ const char *uid = ctx->config[RUN_AS_USER];
+ int success = 0;
+
+ if (uid == NULL) {
+ success = 1;
+ } else {
+ if ((pw = getpwnam(uid)) == NULL) {
+ mg_cry(fc(ctx), "%s: unknown user [%s]", __func__, uid);
+ } else if (setgid(pw->pw_gid) == -1) {
+ mg_cry(fc(ctx), "%s: setgid(%s): %s", __func__, uid, strerror(errno));
+ } else if (setuid(pw->pw_uid) == -1) {
+ mg_cry(fc(ctx), "%s: setuid(%s): %s", __func__, uid, strerror(errno));
+ } else {
+ success = 1;
+ }
+ }
+
+ return success;
+}
+#endif /* !_WIN32 */
+
+#if !defined(NO_SSL)
+static pthread_mutex_t *ssl_mutexes;
+
+static int sslize(struct mg_connection *conn, SSL_CTX *s, int (*func)(SSL *))
+{
+ return (conn->ssl = SSL_new(s)) != NULL &&
+ SSL_set_fd(conn->ssl, conn->client.sock) == 1 &&
+ func(conn->ssl) == 1;
+}
+
+/* Return OpenSSL error message */
+static const char *ssl_error(void)
+{
+ unsigned long err;
+ err = ERR_get_error();
+ return err == 0 ? "" : ERR_error_string(err, NULL);
+}
+
+static void ssl_locking_callback(int mode, int mutex_num, const char *file,
+ int line)
+{
+ (void) line;
+ (void) file;
+
+ if (mode & 1) { /* 1 is CRYPTO_LOCK */
+ (void) pthread_mutex_lock(&ssl_mutexes[mutex_num]);
+ } else {
+ (void) pthread_mutex_unlock(&ssl_mutexes[mutex_num]);
+ }
+}
+
+static unsigned long ssl_id_callback(void)
+{
+ return (unsigned long) pthread_self();
+}
+
+#if !defined(NO_SSL_DL)
+static void *load_dll(struct mg_context *ctx, const char *dll_name,
+ struct ssl_func *sw)
+{
+ union {
+ void *p;
+ void (*fp)(void);
+ } u;
+ void *dll_handle;
+ struct ssl_func *fp;
+
+ if ((dll_handle = dlopen(dll_name, RTLD_LAZY)) == NULL) {
+ mg_cry(fc(ctx), "%s: cannot load %s", __func__, dll_name);
+ return NULL;
+ }
+
+ for (fp = sw; fp->name != NULL; fp++) {
+#ifdef _WIN32
+ /* GetProcAddress() returns pointer to function */
+ u.fp = (void (*)(void)) dlsym(dll_handle, fp->name);
+#else
+ /* dlsym() on UNIX returns void *. ISO C forbids casts of data
+ pointers to function pointers. We need to use a union to make a
+ cast. */
+ u.p = dlsym(dll_handle, fp->name);
+#endif /* _WIN32 */
+ if (u.fp == NULL) {
+ mg_cry(fc(ctx), "%s: %s: cannot find %s", __func__, dll_name, fp->name);
+ dlclose(dll_handle);
+ return NULL;
+ } else {
+ fp->ptr = u.fp;
+ }
+ }
+
+ return dll_handle;
+}
+#endif /* NO_SSL_DL */
+
+/* Dynamically load SSL library. Set up ctx->ssl_ctx pointer. */
+static int set_ssl_option(struct mg_context *ctx)
+{
+ int i, size;
+ const char *pem;
+
+ /* If PEM file is not specified and the init_ssl callback
+ is not specified, skip SSL initialization. */
+ if ((pem = ctx->config[SSL_CERTIFICATE]) == NULL &&
+ ctx->callbacks.init_ssl == NULL) {
+ return 1;
+ }
+
+#if !defined(NO_SSL_DL)
+ ctx->ssllib_dll_handle = load_dll(ctx, SSL_LIB, ssl_sw);
+ ctx->cryptolib_dll_handle = load_dll(ctx, CRYPTO_LIB, crypto_sw);
+ if (!ctx->ssllib_dll_handle || !ctx->cryptolib_dll_handle) {
+ return 0;
+ }
+#endif /* NO_SSL_DL */
+
+ /* Initialize SSL library */
+ SSL_library_init();
+ SSL_load_error_strings();
+
+ if ((ctx->ssl_ctx = SSL_CTX_new(SSLv23_server_method())) == NULL) {
+ mg_cry(fc(ctx), "SSL_CTX_new (server) error: %s", ssl_error());
+ return 0;
+ }
+
+ /* If user callback returned non-NULL, that means that user callback has
+ set up certificate itself. In this case, skip sertificate setting. */
+ if ((ctx->callbacks.init_ssl == NULL ||
+ !ctx->callbacks.init_ssl(ctx->ssl_ctx, ctx->user_data)) &&
+ (SSL_CTX_use_certificate_file(ctx->ssl_ctx, pem, 1) == 0 ||
+ SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, pem, 1) == 0)) {
+ mg_cry(fc(ctx), "%s: cannot open %s: %s", __func__, pem, ssl_error());
+ return 0;
+ }
+
+ if (pem != NULL) {
+ (void) SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, pem);
+ }
+
+ /* Initialize locking callbacks, needed for thread safety.
+ http://www.openssl.org/support/faq.html#PROG1 */
+ size = sizeof(pthread_mutex_t) * CRYPTO_num_locks();
+ if ((ssl_mutexes = (pthread_mutex_t *) malloc((size_t)size)) == NULL) {
+ mg_cry(fc(ctx), "%s: cannot allocate mutexes: %s", __func__, ssl_error());
+ return 0;
+ }
+
+ for (i = 0; i < CRYPTO_num_locks(); i++) {
+ pthread_mutex_init(&ssl_mutexes[i], NULL);
+ }
+
+ CRYPTO_set_locking_callback(&ssl_locking_callback);
+ CRYPTO_set_id_callback(&ssl_id_callback);
+
+ return 1;
+}
+
+static void uninitialize_ssl(struct mg_context *ctx)
+{
+ int i;
+ if (ctx->ssl_ctx != NULL) {
+ CRYPTO_set_locking_callback(NULL);
+ for (i = 0; i < CRYPTO_num_locks(); i++) {
+ pthread_mutex_destroy(&ssl_mutexes[i]);
+ }
+ CRYPTO_set_locking_callback(NULL);
+ CRYPTO_set_id_callback(NULL);
+ }
+}
+#endif /* !NO_SSL */
+
+static int set_gpass_option(struct mg_context *ctx)
+{
+ struct file file = STRUCT_FILE_INITIALIZER;
+ const char *path = ctx->config[GLOBAL_PASSWORDS_FILE];
+ if (path != NULL && !mg_stat(fc(ctx), path, &file)) {
+ mg_cry(fc(ctx), "Cannot open %s: %s", path, strerror(ERRNO));
+ return 0;
+ }
+ return 1;
+}
+
+static int set_acl_option(struct mg_context *ctx)
+{
+ return check_acl(ctx, (uint32_t) 0x7f000001UL) != -1;
+}
+
+static void reset_per_request_attributes(struct mg_connection *conn)
+{
+ conn->path_info = NULL;
+ conn->num_bytes_sent = conn->consumed_content = 0;
+ conn->status_code = -1;
+ conn->must_close = conn->request_len = conn->throttle = 0;
+}
+
+static void close_socket_gracefully(struct mg_connection *conn)
+{
+#if defined(_WIN32)
+ char buf[MG_BUF_LEN];
+ int n;
+#endif
+ struct linger linger;
+
+ /* Set linger option to avoid socket hanging out after close. This prevent
+ ephemeral port exhaust problem under high QPS. */
+ linger.l_onoff = 1;
+ linger.l_linger = 1;
+ if (setsockopt(conn->client.sock, SOL_SOCKET, SO_LINGER,
+ (char *) &linger, sizeof(linger)) != 0) {
+ mg_cry(conn, "%s: setsockopt(SOL_SOCKET SO_LINGER) failed: %s",
+ __func__, strerror(ERRNO));
+ }
+
+ /* Send FIN to the client */
+ shutdown(conn->client.sock, SHUT_WR);
+ set_non_blocking_mode(conn->client.sock);
+
+#if defined(_WIN32)
+ /* Read and discard pending incoming data. If we do not do that and close
+ the socket, the data in the send buffer may be discarded. This
+ behaviour is seen on Windows, when client keeps sending data
+ when server decides to close the connection; then when client
+ does recv() it gets no data back. */
+ do {
+ n = pull(NULL, conn, buf, sizeof(buf));
+ } while (n > 0);
+#endif
+
+ /* Now we know that our FIN is ACK-ed, safe to close */
+ closesocket(conn->client.sock);
+}
+
+static void close_connection(struct mg_connection *conn)
+{
+#if defined(USE_LUA) && defined(USE_WEBSOCKET)
+ if (conn->lua_websocket_state) {
+ lua_websocket_close(conn);
+ }
+#endif
+
+ /* call the connection_close callback if assigned */
+ if (conn->ctx->callbacks.connection_close != NULL)
+ conn->ctx->callbacks.connection_close(conn);
+
+ mg_lock(conn);
+
+ conn->must_close = 1;
+
+#ifndef NO_SSL
+ if (conn->ssl != NULL) {
+ /* Run SSL_shutdown twice to ensure completly close SSL connection */
+ SSL_shutdown(conn->ssl);
+ SSL_free(conn->ssl);
+ conn->ssl = NULL;
+ }
+#endif
+ if (conn->client.sock != INVALID_SOCKET) {
+ close_socket_gracefully(conn);
+ conn->client.sock = INVALID_SOCKET;
+ }
+
+ mg_unlock(conn);
+}
+
+void mg_close_connection(struct mg_connection *conn)
+{
+#ifndef NO_SSL
+ if (conn->client_ssl_ctx != NULL) {
+ SSL_CTX_free((SSL_CTX *) conn->client_ssl_ctx);
+ }
+#endif
+ close_connection(conn);
+ (void) pthread_mutex_destroy(&conn->mutex);
+ free(conn);
+}
+
+struct mg_connection *mg_connect(const char *host, int port, int use_ssl,
+ char *ebuf, size_t ebuf_len);
+
+struct mg_connection *mg_connect(const char *host, int port, int use_ssl,
+ char *ebuf, size_t ebuf_len)
+{
+ static struct mg_context fake_ctx;
+ struct mg_connection *conn = NULL;
+ SOCKET sock;
+
+ if ((sock = conn2(&fake_ctx, host, port, use_ssl, ebuf,
+ ebuf_len)) == INVALID_SOCKET) {
+ } else if ((conn = (struct mg_connection *)
+ calloc(1, sizeof(*conn) + MAX_REQUEST_SIZE)) == NULL) {
+ snprintf(ebuf, ebuf_len, "calloc(): %s", strerror(ERRNO));
+ closesocket(sock);
+#ifndef NO_SSL
+ } else if (use_ssl && (conn->client_ssl_ctx =
+ SSL_CTX_new(SSLv23_client_method())) == NULL) {
+ snprintf(ebuf, ebuf_len, "SSL_CTX_new error");
+ closesocket(sock);
+ free(conn);
+ conn = NULL;
+#endif /* NO_SSL */
+ } else {
+ socklen_t len = sizeof(struct sockaddr);
+ conn->buf_size = MAX_REQUEST_SIZE;
+ conn->buf = (char *) (conn + 1);
+ conn->ctx = &fake_ctx;
+ conn->client.sock = sock;
+ if (getsockname(sock, &conn->client.rsa.sa, &len) != 0) {
+ mg_cry(conn, "%s: getsockname() failed: %s",
+ __func__, strerror(ERRNO));
+ }
+ conn->client.is_ssl = use_ssl;
+ (void) pthread_mutex_init(&conn->mutex, NULL);
+#ifndef NO_SSL
+ if (use_ssl) {
+ /* SSL_CTX_set_verify call is needed to switch off server
+ certificate checking, which is off by default in OpenSSL and on
+ in yaSSL. */
+ SSL_CTX_set_verify(conn->client_ssl_ctx, 0, 0);
+ sslize(conn, conn->client_ssl_ctx, SSL_connect);
+ }
+#endif
+ }
+
+ return conn;
+}
+
+static int is_valid_uri(const char *uri)
+{
+ /* Conform to
+ http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
+ URI can be an asterisk (*) or should start with slash. */
+ return uri[0] == '/' || (uri[0] == '*' && uri[1] == '\0');
+}
+
+static int getreq(struct mg_connection *conn, char *ebuf, size_t ebuf_len)
+{
+ const char *cl;
+
+ ebuf[0] = '\0';
+ reset_per_request_attributes(conn);
+ conn->request_len = read_request(NULL, conn, conn->buf, conn->buf_size,
+ &conn->data_len);
+ assert(conn->request_len < 0 || conn->data_len >= conn->request_len);
+
+ if (conn->request_len == 0 && conn->data_len == conn->buf_size) {
+ snprintf(ebuf, ebuf_len, "%s", "Request Too Large");
+ } else if (conn->request_len <= 0) {
+ snprintf(ebuf, ebuf_len, "%s", "Client closed connection");
+ } else if (parse_http_message(conn->buf, conn->buf_size,
+ &conn->request_info) <= 0) {
+ snprintf(ebuf, ebuf_len, "Bad request: [%.*s]", conn->data_len, conn->buf);
+ } else {
+ /* Request is valid */
+ if ((cl = get_header(&conn->request_info, "Content-Length")) != NULL) {
+ conn->content_len = strtoll(cl, NULL, 10);
+ } else if (!mg_strcasecmp(conn->request_info.request_method, "POST") ||
+ !mg_strcasecmp(conn->request_info.request_method, "PUT")) {
+ conn->content_len = -1;
+ } else {
+ conn->content_len = 0;
+ }
+ conn->birth_time = time(NULL);
+ }
+ return ebuf[0] == '\0';
+}
+
+struct mg_connection *mg_download(const char *host, int port, int use_ssl,
+ char *ebuf, size_t ebuf_len,
+ const char *fmt, ...)
+{
+ struct mg_connection *conn;
+ va_list ap;
+
+ va_start(ap, fmt);
+ ebuf[0] = '\0';
+ if ((conn = mg_connect(host, port, use_ssl, ebuf, ebuf_len)) == NULL) {
+ } else if (mg_vprintf(conn, fmt, ap) <= 0) {
+ snprintf(ebuf, ebuf_len, "%s", "Error sending request");
+ } else {
+ getreq(conn, ebuf, ebuf_len);
+ }
+ if (ebuf[0] != '\0' && conn != NULL) {
+ mg_close_connection(conn);
+ conn = NULL;
+ }
+ va_end(ap);
+
+ return conn;
+}
+
+static void process_new_connection(struct mg_connection *conn)
+{
+ struct mg_request_info *ri = &conn->request_info;
+ int keep_alive_enabled, keep_alive, discard_len;
+ char ebuf[100];
+
+ keep_alive_enabled = !strcmp(conn->ctx->config[ENABLE_KEEP_ALIVE], "yes");
+ keep_alive = 0;
+
+ /* Important: on new connection, reset the receiving buffer. Credit goes
+ to crule42. */
+ conn->data_len = 0;
+ do {
+ if (!getreq(conn, ebuf, sizeof(ebuf))) {
+ send_http_error(conn, 500, "Server Error", "%s", ebuf);
+ conn->must_close = 1;
+ } else if (!is_valid_uri(conn->request_info.uri)) {
+ snprintf(ebuf, sizeof(ebuf), "Invalid URI: [%s]", ri->uri);
+ send_http_error(conn, 400, "Bad Request", "%s", ebuf);
+ } else if (strcmp(ri->http_version, "1.0") &&
+ strcmp(ri->http_version, "1.1")) {
+ snprintf(ebuf, sizeof(ebuf), "Bad HTTP version: [%s]", ri->http_version);
+ send_http_error(conn, 505, "Bad HTTP version", "%s", ebuf);
+ }
+
+ if (ebuf[0] == '\0') {
+ handle_request(conn);
+ if (conn->ctx->callbacks.end_request != NULL) {
+ conn->ctx->callbacks.end_request(conn, conn->status_code);
+ }
+ log_access(conn);
+ }
+ if (ri->remote_user != NULL) {
+ free((void *) ri->remote_user);
+ /* Important! When having connections with and without auth
+ would cause double free and then crash */
+ ri->remote_user = NULL;
+ }
+
+ /* NOTE(lsm): order is important here. should_keep_alive() call is
+ using parsed request, which will be invalid after memmove's below.
+ Therefore, memorize should_keep_alive() result now for later use
+ in loop exit condition. */
+ keep_alive = conn->ctx->stop_flag == 0 && keep_alive_enabled &&
+ conn->content_len >= 0 && should_keep_alive(conn);
+
+ /* Discard all buffered data for this request */
+ discard_len = conn->content_len >= 0 && conn->request_len > 0 &&
+ conn->request_len + conn->content_len < (int64_t) conn->data_len ?
+ (int) (conn->request_len + conn->content_len) : conn->data_len;
+ assert(discard_len >= 0);
+ memmove(conn->buf, conn->buf + discard_len, conn->data_len - discard_len);
+ conn->data_len -= discard_len;
+ assert(conn->data_len >= 0);
+ assert(conn->data_len <= conn->buf_size);
+ } while (keep_alive);
+}
+
+/* Worker threads take accepted socket from the queue */
+static int consume_socket(struct mg_context *ctx, struct socket *sp)
+{
+ (void) pthread_mutex_lock(&ctx->mutex);
+ DEBUG_TRACE(("going idle"));
+
+ /* If the queue is empty, wait. We're idle at this point. */
+ while (ctx->sq_head == ctx->sq_tail && ctx->stop_flag == 0) {
+ pthread_cond_wait(&ctx->sq_full, &ctx->mutex);
+ }
+
+ /* If we're stopping, sq_head may be equal to sq_tail. */
+ if (ctx->sq_head > ctx->sq_tail) {
+ /* Copy socket from the queue and increment tail */
+ *sp = ctx->queue[ctx->sq_tail % ARRAY_SIZE(ctx->queue)];
+ ctx->sq_tail++;
+ DEBUG_TRACE(("grabbed socket %d, going busy", sp->sock));
+
+ /* Wrap pointers if needed */
+ while (ctx->sq_tail > (int) ARRAY_SIZE(ctx->queue)) {
+ ctx->sq_tail -= ARRAY_SIZE(ctx->queue);
+ ctx->sq_head -= ARRAY_SIZE(ctx->queue);
+ }
+ }
+
+ (void) pthread_cond_signal(&ctx->sq_empty);
+ (void) pthread_mutex_unlock(&ctx->mutex);
+
+ return !ctx->stop_flag;
+}
+
+static void *worker_thread_run(void *thread_func_param)
+{
+ struct mg_context *ctx = (struct mg_context *) thread_func_param;
+ struct mg_connection *conn;
+ struct mg_workerTLS tls;
+
+ tls.is_master = 0;
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ tls.pthread_cond_helper_mutex = CreateEvent(NULL, FALSE, FALSE, NULL);
+#endif
+
+ conn = (struct mg_connection *) calloc(1, sizeof(*conn) + MAX_REQUEST_SIZE);
+ if (conn == NULL) {
+ mg_cry(fc(ctx), "%s", "Cannot create new connection struct, OOM");
+ } else {
+ pthread_setspecific(sTlsKey, &tls);
+ conn->buf_size = MAX_REQUEST_SIZE;
+ conn->buf = (char *) (conn + 1);
+ conn->ctx = ctx;
+ conn->request_info.user_data = ctx->user_data;
+ /* Allocate a mutex for this connection to allow communication both
+ within the request handler and from elsewhere in the application */
+ (void) pthread_mutex_init(&conn->mutex, NULL);
+
+ /* Call consume_socket() even when ctx->stop_flag > 0, to let it
+ signal sq_empty condvar to wake up the master waiting in
+ produce_socket() */
+ while (consume_socket(ctx, &conn->client)) {
+ conn->birth_time = time(NULL);
+
+ /* Fill in IP, port info early so even if SSL setup below fails,
+ error handler would have the corresponding info.
+ Thanks to Johannes Winkelmann for the patch.
+ TODO(lsm): Fix IPv6 case */
+ conn->request_info.remote_port = ntohs(conn->client.rsa.sin.sin_port);
+ memcpy(&conn->request_info.remote_ip,
+ &conn->client.rsa.sin.sin_addr.s_addr, 4);
+ conn->request_info.remote_ip = ntohl(conn->request_info.remote_ip);
+ conn->request_info.is_ssl = conn->client.is_ssl;
+
+ if (!conn->client.is_ssl
+#ifndef NO_SSL
+ || sslize(conn, conn->ctx->ssl_ctx, SSL_accept)
+#endif
+ ) {
+ process_new_connection(conn);
+ }
+
+ close_connection(conn);
+ }
+ }
+
+ /* Signal master that we're done with connection and exiting */
+ (void) pthread_mutex_lock(&ctx->mutex);
+ ctx->num_threads--;
+ (void) pthread_cond_signal(&ctx->cond);
+ assert(ctx->num_threads >= 0);
+ (void) pthread_mutex_unlock(&ctx->mutex);
+
+ pthread_setspecific(sTlsKey, 0);
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ CloseHandle(tls.pthread_cond_helper_mutex);
+#endif
+ free(conn);
+
+ DEBUG_TRACE(("exiting"));
+ return NULL;
+}
+
+/* Threads have different return types on Windows and Unix. */
+
+#ifdef _WIN32
+static unsigned __stdcall worker_thread(void *thread_func_param)
+{
+ worker_thread_run(thread_func_param);
+ return 0;
+}
+#else
+static void *worker_thread(void *thread_func_param)
+{
+ worker_thread_run(thread_func_param);
+ return NULL;
+}
+#endif /* _WIN32 */
+
+/* Master thread adds accepted socket to a queue */
+static void produce_socket(struct mg_context *ctx, const struct socket *sp)
+{
+ (void) pthread_mutex_lock(&ctx->mutex);
+
+ /* If the queue is full, wait */
+ while (ctx->stop_flag == 0 &&
+ ctx->sq_head - ctx->sq_tail >= (int) ARRAY_SIZE(ctx->queue)) {
+ (void) pthread_cond_wait(&ctx->sq_empty, &ctx->mutex);
+ }
+
+ if (ctx->sq_head - ctx->sq_tail < (int) ARRAY_SIZE(ctx->queue)) {
+ /* Copy socket to the queue and increment head */
+ ctx->queue[ctx->sq_head % ARRAY_SIZE(ctx->queue)] = *sp;
+ ctx->sq_head++;
+ DEBUG_TRACE(("queued socket %d", sp->sock));
+ }
+
+ (void) pthread_cond_signal(&ctx->sq_full);
+ (void) pthread_mutex_unlock(&ctx->mutex);
+}
+
+static int set_sock_timeout(SOCKET sock, int milliseconds)
+{
+#ifdef _WIN32
+ DWORD t = milliseconds;
+#else
+ struct timeval t;
+ t.tv_sec = milliseconds / 1000;
+ t.tv_usec = (milliseconds * 1000) % 1000000;
+#endif
+ return setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (void *) &t, sizeof(t)) ||
+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (void *) &t, sizeof(t));
+}
+
+static void accept_new_connection(const struct socket *listener,
+ struct mg_context *ctx)
+{
+ struct socket so;
+ char src_addr[IP_ADDR_STR_LEN];
+ socklen_t len = sizeof(so.rsa);
+ int on = 1;
+
+ if ((so.sock = accept(listener->sock, &so.rsa.sa, &len)) == INVALID_SOCKET) {
+ } else if (!check_acl(ctx, ntohl(* (uint32_t *) &so.rsa.sin.sin_addr))) {
+ sockaddr_to_string(src_addr, sizeof(src_addr), &so.rsa);
+ mg_cry(fc(ctx), "%s: %s is not allowed to connect", __func__, src_addr);
+ closesocket(so.sock);
+ } else {
+ /* Put so socket structure into the queue */
+ DEBUG_TRACE(("Accepted socket %d", (int) so.sock));
+ set_close_on_exec(so.sock, fc(ctx));
+ so.is_ssl = listener->is_ssl;
+ so.ssl_redir = listener->ssl_redir;
+ if (getsockname(so.sock, &so.lsa.sa, &len) != 0) {
+ mg_cry(fc(ctx), "%s: getsockname() failed: %s",
+ __func__, strerror(ERRNO));
+ }
+ /* Set TCP keep-alive. This is needed because if HTTP-level keep-alive
+ is enabled, and client resets the connection, server won't get
+ TCP FIN or RST and will keep the connection open forever. With TCP
+ keep-alive, next keep-alive handshake will figure out that the
+ client is down and will close the server end.
+ Thanks to Igor Klopov who suggested the patch. */
+ if (setsockopt(so.sock, SOL_SOCKET, SO_KEEPALIVE, (void *) &on,
+ sizeof(on)) != 0) {
+ mg_cry(fc(ctx),
+ "%s: setsockopt(SOL_SOCKET SO_KEEPALIVE) failed: %s",
+ __func__, strerror(ERRNO));
+ }
+ set_sock_timeout(so.sock, atoi(ctx->config[REQUEST_TIMEOUT]));
+ produce_socket(ctx, &so);
+ }
+}
+
+static void master_thread_run(void *thread_func_param)
+{
+ struct mg_context *ctx = (struct mg_context *) thread_func_param;
+ struct mg_workerTLS tls;
+ struct pollfd *pfd;
+ int i;
+ int workerthreadcount;
+
+ /* Increase priority of the master thread */
+#if defined(_WIN32)
+ SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL);
+#elif defined(USE_MASTER_THREAD_PRIORITY)
+ int min_prio = sched_get_priority_min(SCHED_RR);
+ int max_prio = sched_get_priority_max(SCHED_RR);
+ if ((min_prio >=0) && (max_prio >= 0) &&
+ ((USE_MASTER_THREAD_PRIORITY) <= max_prio) &&
+ ((USE_MASTER_THREAD_PRIORITY) >= min_prio)
+ ) {
+ struct sched_param sched_param = {0};
+ sched_param.sched_priority = (USE_MASTER_THREAD_PRIORITY);
+ pthread_setschedparam(pthread_self(), SCHED_RR, &sched_param);
+ }
+#endif
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ tls.pthread_cond_helper_mutex = CreateEvent(NULL, FALSE, FALSE, NULL);
+#endif
+ tls.is_master = 1;
+ pthread_setspecific(sTlsKey, &tls);
+
+ pfd = (struct pollfd *) calloc(ctx->num_listening_sockets, sizeof(pfd[0]));
+ while (pfd != NULL && ctx->stop_flag == 0) {
+ for (i = 0; i < ctx->num_listening_sockets; i++) {
+ pfd[i].fd = ctx->listening_sockets[i].sock;
+ pfd[i].events = POLLIN;
+ }
+
+ if (poll(pfd, ctx->num_listening_sockets, 200) > 0) {
+ for (i = 0; i < ctx->num_listening_sockets; i++) {
+ /* NOTE(lsm): on QNX, poll() returns POLLRDNORM after the
+ successful poll, and POLLIN is defined as
+ (POLLRDNORM | POLLRDBAND)
+ Therefore, we're checking pfd[i].revents & POLLIN, not
+ pfd[i].revents == POLLIN. */
+ if (ctx->stop_flag == 0 && (pfd[i].revents & POLLIN)) {
+ accept_new_connection(&ctx->listening_sockets[i], ctx);
+ }
+ }
+ }
+ }
+ free(pfd);
+ DEBUG_TRACE(("stopping workers"));
+
+ /* Stop signal received: somebody called mg_stop. Quit. */
+ close_all_listening_sockets(ctx);
+
+ /* Wakeup workers that are waiting for connections to handle. */
+ pthread_cond_broadcast(&ctx->sq_full);
+
+ /* Wait until all threads finish */
+ (void) pthread_mutex_lock(&ctx->mutex);
+ while (ctx->num_threads > 0) {
+ (void) pthread_cond_wait(&ctx->cond, &ctx->mutex);
+ }
+ (void) pthread_mutex_unlock(&ctx->mutex);
+
+ /* Join all worker threads to avoid leaking threads. */
+ workerthreadcount = ctx->workerthreadcount;
+ for (i = 0; i < workerthreadcount; i++) {
+ mg_join_thread(ctx->workerthreadids[i]);
+ }
+
+ /* All threads exited, no sync is needed. Destroy mutex and condvars */
+ (void) pthread_mutex_destroy(&ctx->mutex);
+ (void) pthread_cond_destroy(&ctx->cond);
+ (void) pthread_cond_destroy(&ctx->sq_empty);
+ (void) pthread_cond_destroy(&ctx->sq_full);
+
+#if !defined(NO_SSL)
+ uninitialize_ssl(ctx);
+#endif
+ DEBUG_TRACE(("exiting"));
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ CloseHandle(tls.pthread_cond_helper_mutex);
+#endif
+ pthread_setspecific(sTlsKey, 0);
+
+ /* Signal mg_stop() that we're done.
+ WARNING: This must be the very last thing this
+ thread does, as ctx becomes invalid after this line. */
+ ctx->stop_flag = 2;
+}
+
+/* Threads have different return types on Windows and Unix. */
+
+#ifdef _WIN32
+static unsigned __stdcall master_thread(void *thread_func_param)
+{
+ master_thread_run(thread_func_param);
+ return 0;
+}
+#else
+static void *master_thread(void *thread_func_param)
+{
+ master_thread_run(thread_func_param);
+ return NULL;
+}
+#endif /* _WIN32 */
+
+static void free_context(struct mg_context *ctx)
+{
+ int i;
+ struct mg_request_handler_info *tmp_rh;
+
+ if (ctx == NULL)
+ return;
+
+ /* Deallocate config parameters */
+ for (i = 0; i < NUM_OPTIONS; i++) {
+ if (ctx->config[i] != NULL)
+#ifdef WIN32
+#pragma warning(suppress: 6001)
+#endif
+ free(ctx->config[i]);
+ }
+
+ /* Deallocate request handlers */
+ while (ctx->request_handlers) {
+ tmp_rh = ctx->request_handlers;
+ ctx->request_handlers = tmp_rh->next;
+ free(tmp_rh->uri);
+ free(tmp_rh);
+ }
+
+#ifndef NO_SSL
+ /* Deallocate SSL context */
+ if (ctx->ssl_ctx != NULL) {
+ SSL_CTX_free(ctx->ssl_ctx);
+ }
+ if (ssl_mutexes != NULL) {
+ free(ssl_mutexes);
+ ssl_mutexes = NULL;
+ }
+#endif /* !NO_SSL */
+
+ /* Deallocate worker thread ID array */
+ if (ctx->workerthreadids != NULL) {
+ free(ctx->workerthreadids);
+ }
+
+ /* Deallocate the tls variable */
+ sTlsInit--;
+ if (sTlsInit==0) {
+ pthread_key_delete(sTlsKey);
+ }
+
+ /* Deallocate context itself */
+ free(ctx);
+}
+
+void mg_stop(struct mg_context *ctx)
+{
+ ctx->stop_flag = 1;
+
+ /* Wait until mg_fini() stops */
+ while (ctx->stop_flag != 2) {
+ (void) mg_sleep(10);
+ }
+ mg_join_thread(ctx->masterthreadid);
+ free_context(ctx);
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ (void) WSACleanup();
+#endif /* _WIN32 && !__SYMBIAN32__ */
+}
+
+struct mg_context *mg_start(const struct mg_callbacks *callbacks,
+ void *user_data,
+ const char **options)
+{
+ struct mg_context *ctx;
+ const char *name, *value, *default_value;
+ int i;
+ int workerthreadcount;
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+ WSADATA data;
+ WSAStartup(MAKEWORD(2,2), &data);
+#pragma warning(suppress: 28125)
+ InitializeCriticalSection(&global_log_file_lock);
+#endif /* _WIN32 && !__SYMBIAN32__ */
+
+ /* Check if the config_options and the corresponding enum have compatible sizes. */
+ /* Could use static_assert, once it is verified that all compilers support this. */
+ assert(sizeof(config_options)/sizeof(config_options[0]) == NUM_OPTIONS*2+1);
+
+ /* Allocate context and initialize reasonable general case defaults.
+ TODO(lsm): do proper error handling here. */
+ if ((ctx = (struct mg_context *) calloc(1, sizeof(*ctx))) == NULL) {
+ return NULL;
+ }
+
+ if (sTlsInit==0) {
+ if (0 != pthread_key_create(&sTlsKey, NULL)) {
+ mg_cry(fc(ctx), "Cannot initialize thread local storage");
+ return NULL;
+ }
+ sTlsInit++;
+ }
+
+ ctx->callbacks = *callbacks;
+ ctx->user_data = user_data;
+ ctx->request_handlers = 0;
+
+ while (options && (name = *options++) != NULL) {
+ if ((i = get_option_index(name)) == -1) {
+ mg_cry(fc(ctx), "Invalid option: %s", name);
+ free_context(ctx);
+ return NULL;
+ } else if ((value = *options++) == NULL) {
+ mg_cry(fc(ctx), "%s: option value cannot be NULL", name);
+ free_context(ctx);
+ return NULL;
+ }
+ if (ctx->config[i] != NULL) {
+ mg_cry(fc(ctx), "warning: %s: duplicate option", name);
+ free(ctx->config[i]);
+ }
+ ctx->config[i] = mg_strdup(value);
+ DEBUG_TRACE(("[%s] -> [%s]", name, value));
+ }
+
+ /* Set default value if needed */
+ for (i = 0; config_options[i * 2] != NULL; i++) {
+ default_value = config_options[i * 2 + 1];
+ if (ctx->config[i] == NULL && default_value != NULL) {
+ ctx->config[i] = mg_strdup(default_value);
+ }
+ }
+
+ /* NOTE(lsm): order is important here. SSL certificates must
+ be initialized before listening ports. UID must be set last. */
+ if (!set_gpass_option(ctx) ||
+#if !defined(NO_SSL)
+ !set_ssl_option(ctx) ||
+#endif
+ !set_ports_option(ctx) ||
+#if !defined(_WIN32)
+ !set_uid_option(ctx) ||
+#endif
+ !set_acl_option(ctx)) {
+ free_context(ctx);
+ return NULL;
+ }
+
+#if !defined(_WIN32) && !defined(__SYMBIAN32__)
+ /* Ignore SIGPIPE signal, so if browser cancels the request, it
+ won't kill the whole process. */
+ (void) signal(SIGPIPE, SIG_IGN);
+#endif /* !_WIN32 && !__SYMBIAN32__ */
+
+ (void) pthread_mutex_init(&ctx->mutex, NULL);
+ (void) pthread_cond_init(&ctx->cond, NULL);
+ (void) pthread_cond_init(&ctx->sq_empty, NULL);
+ (void) pthread_cond_init(&ctx->sq_full, NULL);
+
+ workerthreadcount = atoi(ctx->config[NUM_THREADS]);
+
+ if (workerthreadcount > MAX_WORKER_THREADS) {
+ mg_cry(fc(ctx), "Too many worker threads");
+ free_context(ctx);
+ return NULL;
+ }
+
+ if (workerthreadcount > 0) {
+ ctx->workerthreadcount = workerthreadcount;
+ ctx->workerthreadids = calloc(workerthreadcount, sizeof(pthread_t));
+ if (ctx->workerthreadids == NULL) {
+ mg_cry(fc(ctx), "Not enough memory for worker thread ID array");
+ free_context(ctx);
+ return NULL;
+ }
+ }
+
+ /* Start master (listening) thread */
+ mg_start_thread_with_id(master_thread, ctx, &ctx->masterthreadid);
+
+ /* Start worker threads */
+ for (i = 0; i < workerthreadcount; i++) {
+ (void) pthread_mutex_lock(&ctx->mutex);
+ ctx->num_threads++;
+ (void) pthread_mutex_unlock(&ctx->mutex);
+ if (mg_start_thread_with_id(worker_thread, ctx,
+ &ctx->workerthreadids[i]) != 0) {
+ (void) pthread_mutex_lock(&ctx->mutex);
+ ctx->num_threads--;
+ (void) pthread_mutex_unlock(&ctx->mutex);
+ mg_cry(fc(ctx), "Cannot start worker thread: %ld", (long) ERRNO);
+ }
+ }
+
+ return ctx;
+}
diff --git a/src/civetweb/src/md5.h b/src/civetweb/src/md5.h
new file mode 100644
index 0000000..4da933d
--- /dev/null
+++ b/src/civetweb/src/md5.h
@@ -0,0 +1,461 @@
+/*
+ * This an amalgamation of md5.c and md5.h into a single file
+ * with all static declaration to reduce linker conflicts
+ * in Civetweb.
+ *
+ * The MD5_STATIC declaration was added to facilitate static
+ * inclusion.
+ * No Face Press, LLC
+ */
+
+/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */
+/*
+ Independent implementation of MD5 (RFC 1321).
+
+ This code implements the MD5 Algorithm defined in RFC 1321, whose
+ text is available at
+ http://www.ietf.org/rfc/rfc1321.txt
+ The code is derived from the text of the RFC, including the test suite
+ (section A.5) but excluding the rest of Appendix A. It does not include
+ any code or documentation that is identified in the RFC as being
+ copyrighted.
+
+ The original and principal author of md5.h is L. Peter Deutsch
+ <ghost at aladdin.com>. Other authors are noted in the change history
+ that follows (in reverse chronological order):
+
+ 2002-04-13 lpd Removed support for non-ANSI compilers; removed
+ references to Ghostscript; clarified derivation from RFC 1321;
+ now handles byte order either statically or dynamically.
+ 1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
+ 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5);
+ added conditionalization for C++ compilation from Martin
+ Purschke <purschke at bnl.gov>.
+ 1999-05-03 lpd Original version.
+ */
+
+#ifndef md5_INCLUDED
+# define md5_INCLUDED
+
+/*
+ * This package supports both compile-time and run-time determination of CPU
+ * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be
+ * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is
+ * defined as non-zero, the code will be compiled to run only on big-endian
+ * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to
+ * run on either big- or little-endian CPUs, but will run slightly less
+ * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined.
+ */
+
+typedef unsigned char md5_byte_t; /* 8-bit byte */
+typedef unsigned int md5_word_t; /* 32-bit word */
+
+/* Define the state of the MD5 Algorithm. */
+typedef struct md5_state_s {
+ md5_word_t count[2]; /* message length in bits, lsw first */
+ md5_word_t abcd[4]; /* digest buffer */
+ md5_byte_t buf[64]; /* accumulate block */
+} md5_state_t;
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* Initialize the algorithm. */
+MD5_STATIC void md5_init(md5_state_t *pms);
+
+/* Append a string to the message. */
+MD5_STATIC void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes);
+
+/* Finish the message and return the digest. */
+MD5_STATIC void md5_finish(md5_state_t *pms, md5_byte_t digest[16]);
+
+#ifdef __cplusplus
+} /* end extern "C" */
+#endif
+
+#endif /* md5_INCLUDED */
+
+/*
+ Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved.
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+ claim that you wrote the original software. If you use this software
+ in a product, an acknowledgment in the product documentation would be
+ appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+
+ L. Peter Deutsch
+ ghost at aladdin.com
+
+ */
+/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */
+/*
+ Independent implementation of MD5 (RFC 1321).
+
+ This code implements the MD5 Algorithm defined in RFC 1321, whose
+ text is available at
+ http://www.ietf.org/rfc/rfc1321.txt
+ The code is derived from the text of the RFC, including the test suite
+ (section A.5) but excluding the rest of Appendix A. It does not include
+ any code or documentation that is identified in the RFC as being
+ copyrighted.
+
+ The original and principal author of md5.c is L. Peter Deutsch
+ <ghost at aladdin.com>. Other authors are noted in the change history
+ that follows (in reverse chronological order):
+
+ 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order
+ either statically or dynamically; added missing #include <string.h>
+ in library.
+ 2002-03-11 lpd Corrected argument list for main(), and added int return
+ type, in test program and T value program.
+ 2002-02-21 lpd Added missing #include <stdio.h> in test program.
+ 2000-07-03 lpd Patched to eliminate warnings about "constant is
+ unsigned in ANSI C, signed in traditional"; made test program
+ self-checking.
+ 1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
+ 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5).
+ 1999-05-03 lpd Original version.
+ */
+
+#ifndef MD5_STATIC
+#include <string.h>
+#endif
+
+#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */
+#ifdef ARCH_IS_BIG_ENDIAN
+# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1)
+#else
+# define BYTE_ORDER 0
+#endif
+
+#define T_MASK ((md5_word_t)~0)
+#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87)
+#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9)
+#define T3 0x242070db
+#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111)
+#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050)
+#define T6 0x4787c62a
+#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec)
+#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe)
+#define T9 0x698098d8
+#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850)
+#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e)
+#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841)
+#define T13 0x6b901122
+#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c)
+#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71)
+#define T16 0x49b40821
+#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d)
+#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf)
+#define T19 0x265e5a51
+#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855)
+#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2)
+#define T22 0x02441453
+#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e)
+#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437)
+#define T25 0x21e1cde6
+#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829)
+#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278)
+#define T28 0x455a14ed
+#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa)
+#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07)
+#define T31 0x676f02d9
+#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375)
+#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd)
+#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e)
+#define T35 0x6d9d6122
+#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3)
+#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb)
+#define T38 0x4bdecfa9
+#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f)
+#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f)
+#define T41 0x289b7ec6
+#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805)
+#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a)
+#define T44 0x04881d05
+#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6)
+#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a)
+#define T47 0x1fa27cf8
+#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a)
+#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb)
+#define T50 0x432aff97
+#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58)
+#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6)
+#define T53 0x655b59c3
+#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d)
+#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82)
+#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e)
+#define T57 0x6fa87e4f
+#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f)
+#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb)
+#define T60 0x4e0811a1
+#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d)
+#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca)
+#define T63 0x2ad7d2bb
+#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e)
+
+
+static void
+md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/)
+{
+ md5_word_t
+ a = pms->abcd[0], b = pms->abcd[1],
+ c = pms->abcd[2], d = pms->abcd[3];
+ md5_word_t t;
+#if BYTE_ORDER > 0
+ /* Define storage only for big-endian CPUs. */
+ md5_word_t X[16];
+#else
+ /* Define storage for little-endian or both types of CPUs. */
+ md5_word_t xbuf[16];
+ const md5_word_t *X;
+#endif
+
+ {
+#if BYTE_ORDER == 0
+ /*
+ * Determine dynamically whether this is a big-endian or
+ * little-endian machine, since we can use a more efficient
+ * algorithm on the latter.
+ */
+ static const int w = 1;
+
+ if (*((const md5_byte_t *)&w)) /* dynamic little-endian */
+#endif
+#if BYTE_ORDER <= 0 /* little-endian */
+ {
+ /*
+ * On little-endian machines, we can process properly aligned
+ * data without copying it.
+ */
+ if (!((data - (const md5_byte_t *)0) & 3)) {
+ /* data are properly aligned */
+ X = (const md5_word_t *)data;
+ } else {
+ /* not aligned */
+ memcpy(xbuf, data, 64);
+ X = xbuf;
+ }
+ }
+#endif
+#if BYTE_ORDER == 0
+ else /* dynamic big-endian */
+#endif
+#if BYTE_ORDER >= 0 /* big-endian */
+ {
+ /*
+ * On big-endian machines, we must arrange the bytes in the
+ * right order.
+ */
+ const md5_byte_t *xp = data;
+ int i;
+
+# if BYTE_ORDER == 0
+ X = xbuf; /* (dynamic only) */
+# else
+# define xbuf X /* (static only) */
+# endif
+ for (i = 0; i < 16; ++i, xp += 4)
+ xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24);
+ }
+#endif
+ }
+
+#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
+
+ /* Round 1. */
+ /* Let [abcd k s i] denote the operation
+ a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */
+#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
+#define SET(a, b, c, d, k, s, Ti)\
+ t = a + F(b,c,d) + X[k] + Ti;\
+ a = ROTATE_LEFT(t, s) + b
+ /* Do the following 16 operations. */
+ SET(a, b, c, d, 0, 7, T1);
+ SET(d, a, b, c, 1, 12, T2);
+ SET(c, d, a, b, 2, 17, T3);
+ SET(b, c, d, a, 3, 22, T4);
+ SET(a, b, c, d, 4, 7, T5);
+ SET(d, a, b, c, 5, 12, T6);
+ SET(c, d, a, b, 6, 17, T7);
+ SET(b, c, d, a, 7, 22, T8);
+ SET(a, b, c, d, 8, 7, T9);
+ SET(d, a, b, c, 9, 12, T10);
+ SET(c, d, a, b, 10, 17, T11);
+ SET(b, c, d, a, 11, 22, T12);
+ SET(a, b, c, d, 12, 7, T13);
+ SET(d, a, b, c, 13, 12, T14);
+ SET(c, d, a, b, 14, 17, T15);
+ SET(b, c, d, a, 15, 22, T16);
+#undef SET
+
+ /* Round 2. */
+ /* Let [abcd k s i] denote the operation
+ a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */
+#define G(x, y, z) (((x) & (z)) | ((y) & ~(z)))
+#define SET(a, b, c, d, k, s, Ti)\
+ t = a + G(b,c,d) + X[k] + Ti;\
+ a = ROTATE_LEFT(t, s) + b
+ /* Do the following 16 operations. */
+ SET(a, b, c, d, 1, 5, T17);
+ SET(d, a, b, c, 6, 9, T18);
+ SET(c, d, a, b, 11, 14, T19);
+ SET(b, c, d, a, 0, 20, T20);
+ SET(a, b, c, d, 5, 5, T21);
+ SET(d, a, b, c, 10, 9, T22);
+ SET(c, d, a, b, 15, 14, T23);
+ SET(b, c, d, a, 4, 20, T24);
+ SET(a, b, c, d, 9, 5, T25);
+ SET(d, a, b, c, 14, 9, T26);
+ SET(c, d, a, b, 3, 14, T27);
+ SET(b, c, d, a, 8, 20, T28);
+ SET(a, b, c, d, 13, 5, T29);
+ SET(d, a, b, c, 2, 9, T30);
+ SET(c, d, a, b, 7, 14, T31);
+ SET(b, c, d, a, 12, 20, T32);
+#undef SET
+
+ /* Round 3. */
+ /* Let [abcd k s t] denote the operation
+ a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */
+#define H(x, y, z) ((x) ^ (y) ^ (z))
+#define SET(a, b, c, d, k, s, Ti)\
+ t = a + H(b,c,d) + X[k] + Ti;\
+ a = ROTATE_LEFT(t, s) + b
+ /* Do the following 16 operations. */
+ SET(a, b, c, d, 5, 4, T33);
+ SET(d, a, b, c, 8, 11, T34);
+ SET(c, d, a, b, 11, 16, T35);
+ SET(b, c, d, a, 14, 23, T36);
+ SET(a, b, c, d, 1, 4, T37);
+ SET(d, a, b, c, 4, 11, T38);
+ SET(c, d, a, b, 7, 16, T39);
+ SET(b, c, d, a, 10, 23, T40);
+ SET(a, b, c, d, 13, 4, T41);
+ SET(d, a, b, c, 0, 11, T42);
+ SET(c, d, a, b, 3, 16, T43);
+ SET(b, c, d, a, 6, 23, T44);
+ SET(a, b, c, d, 9, 4, T45);
+ SET(d, a, b, c, 12, 11, T46);
+ SET(c, d, a, b, 15, 16, T47);
+ SET(b, c, d, a, 2, 23, T48);
+#undef SET
+
+ /* Round 4. */
+ /* Let [abcd k s t] denote the operation
+ a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */
+#define I(x, y, z) ((y) ^ ((x) | ~(z)))
+#define SET(a, b, c, d, k, s, Ti)\
+ t = a + I(b,c,d) + X[k] + Ti;\
+ a = ROTATE_LEFT(t, s) + b
+ /* Do the following 16 operations. */
+ SET(a, b, c, d, 0, 6, T49);
+ SET(d, a, b, c, 7, 10, T50);
+ SET(c, d, a, b, 14, 15, T51);
+ SET(b, c, d, a, 5, 21, T52);
+ SET(a, b, c, d, 12, 6, T53);
+ SET(d, a, b, c, 3, 10, T54);
+ SET(c, d, a, b, 10, 15, T55);
+ SET(b, c, d, a, 1, 21, T56);
+ SET(a, b, c, d, 8, 6, T57);
+ SET(d, a, b, c, 15, 10, T58);
+ SET(c, d, a, b, 6, 15, T59);
+ SET(b, c, d, a, 13, 21, T60);
+ SET(a, b, c, d, 4, 6, T61);
+ SET(d, a, b, c, 11, 10, T62);
+ SET(c, d, a, b, 2, 15, T63);
+ SET(b, c, d, a, 9, 21, T64);
+#undef SET
+
+ /* Then perform the following additions. (That is increment each
+ of the four registers by the value it had before this block
+ was started.) */
+ pms->abcd[0] += a;
+ pms->abcd[1] += b;
+ pms->abcd[2] += c;
+ pms->abcd[3] += d;
+}
+
+MD5_STATIC void
+md5_init(md5_state_t *pms)
+{
+ pms->count[0] = pms->count[1] = 0;
+ pms->abcd[0] = 0x67452301;
+ pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476;
+ pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301;
+ pms->abcd[3] = 0x10325476;
+}
+
+MD5_STATIC void
+md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes)
+{
+ const md5_byte_t *p = data;
+ int left = nbytes;
+ int offset = (pms->count[0] >> 3) & 63;
+ md5_word_t nbits = (md5_word_t)(nbytes << 3);
+
+ if (nbytes <= 0)
+ return;
+
+ /* Update the message length. */
+ pms->count[1] += nbytes >> 29;
+ pms->count[0] += nbits;
+ if (pms->count[0] < nbits)
+ pms->count[1]++;
+
+ /* Process an initial partial block. */
+ if (offset) {
+ int copy = (offset + nbytes > 64 ? 64 - offset : nbytes);
+
+ memcpy(pms->buf + offset, p, copy);
+ if (offset + copy < 64)
+ return;
+ p += copy;
+ left -= copy;
+ md5_process(pms, pms->buf);
+ }
+
+ /* Process full blocks. */
+ for (; left >= 64; p += 64, left -= 64)
+ md5_process(pms, p);
+
+ /* Process a final partial block. */
+ if (left)
+ memcpy(pms->buf, p, left);
+}
+
+MD5_STATIC void
+md5_finish(md5_state_t *pms, md5_byte_t digest[16])
+{
+ static const md5_byte_t pad[64] = {
+ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ };
+ md5_byte_t data[8];
+ int i;
+
+ /* Save the length before padding. */
+ for (i = 0; i < 8; ++i)
+ data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3));
+ /* Pad to 56 bytes mod 64. */
+ md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1);
+ /* Append the length. */
+ md5_append(pms, data, 8);
+ for (i = 0; i < 16; ++i)
+ digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3));
+}
diff --git a/src/client/Client.cc b/src/client/Client.cc
index 2065189..d7d9a50 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -84,6 +84,7 @@ using namespace std;
#include "ObjecterWriteback.h"
#include "include/assert.h"
+#include "include/stat.h"
#undef dout_prefix
#define dout_prefix *_dout << "client." << whoami << " "
@@ -109,7 +110,6 @@ Client::CommandHook::CommandHook(Client *client) :
bool Client::CommandHook::call(std::string command, cmdmap_t& cmdmap,
std::string format, bufferlist& out)
{
- stringstream ss;
Formatter *f = new_formatter(format);
f->open_object_section("result");
m_client->client_lock.Lock();
@@ -226,7 +226,7 @@ Client::~Client()
void Client::tear_down_cache()
{
// fd's
- for (hash_map<int, Fh*>::iterator it = fd_map.begin();
+ for (ceph::unordered_map<int, Fh*>::iterator it = fd_map.begin();
it != fd_map.end();
++it) {
Fh *fh = it->second;
@@ -286,7 +286,7 @@ void Client::dump_inode(Formatter *f, Inode *in, set<Inode*>& did, bool disconne
did.insert(in);
if (in->dir) {
ldout(cct, 1) << " dir " << in->dir << " size " << in->dir->dentries.size() << dendl;
- for (hash_map<string, Dentry*>::iterator it = in->dir->dentries.begin();
+ for (ceph::unordered_map<string, Dentry*>::iterator it = in->dir->dentries.begin();
it != in->dir->dentries.end();
++it) {
ldout(cct, 1) << " " << in->ino << " dn " << it->first << " " << it->second << " ref " << it->second->ref << dendl;
@@ -314,7 +314,7 @@ void Client::dump_cache(Formatter *f)
dump_inode(f, root, did, true);
// make a second pass to catch anything disconnected
- for (hash_map<vinodeno_t, Inode*>::iterator it = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t, Inode*>::iterator it = inode_map.begin();
it != inode_map.end();
++it) {
if (did.count(it->second))
@@ -494,6 +494,8 @@ void Client::update_inode_file_bits(Inode *in,
uint64_t time_warp_seq, utime_t ctime,
utime_t mtime,
utime_t atime,
+ version_t inline_version,
+ bufferlist& inline_data,
int issued)
{
bool warn = false;
@@ -504,6 +506,11 @@ void Client::update_inode_file_bits(Inode *in,
<< " local " << in->time_warp_seq << dendl;
uint64_t prior_size = in->size;
+ if (inline_version > in->inline_version) {
+ in->inline_data = inline_data;
+ in->inline_version = inline_version;
+ }
+
if (truncate_seq > in->truncate_seq ||
(truncate_seq == in->truncate_seq && size > in->size)) {
ldout(cct, 10) << "size " << in->size << " -> " << size << dendl;
@@ -520,6 +527,13 @@ void Client::update_inode_file_bits(Inode *in,
_invalidate_inode_cache(in, truncate_size, prior_size - truncate_size, true);
}
}
+
+ // truncate inline data
+ if (in->inline_version < CEPH_INLINE_NONE) {
+ uint32_t len = in->inline_data.length();
+ if (size < len)
+ in->inline_data.splice(size, len - size);
+ }
}
if (truncate_seq >= in->truncate_seq &&
in->truncate_size != truncate_size) {
@@ -654,6 +668,7 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, MetaSession *sessi
update_inode_file_bits(in, st->truncate_seq, st->truncate_size, st->size,
st->time_warp_seq, st->ctime, st->mtime, st->atime,
+ st->inline_version, st->inline_data,
issued);
}
@@ -1184,7 +1199,7 @@ int Client::verify_reply_trace(int r,
bufferlist extra_bl;
inodeno_t created_ino;
bool got_created_ino = false;
- hash_map<vinodeno_t, Inode*>::iterator p;
+ ceph::unordered_map<vinodeno_t, Inode*>::iterator p;
extra_bl.claim(reply->get_extra_bl());
if (extra_bl.length() >= 8) {
@@ -1600,6 +1615,11 @@ void Client::handle_client_session(MClientSession *m)
trim_caps(session, m->get_max_caps());
break;
+ case CEPH_SESSION_FLUSHMSG:
+ messenger->send_message(new MClientSession(CEPH_SESSION_FLUSHMSG_ACK, m->get_seq()),
+ session->con);
+ break;
+
default:
assert(0);
}
@@ -1953,8 +1973,8 @@ void Client::send_reconnect(MetaSession *session)
MClientReconnect *m = new MClientReconnect;
// i have an open session.
- hash_set<inodeno_t> did_snaprealm;
- for (hash_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
+ ceph::unordered_set<inodeno_t> did_snaprealm;
+ for (ceph::unordered_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
p != inode_map.end();
++p) {
Inode *in = p->second;
@@ -2384,6 +2404,11 @@ void Client::send_cap(Inode *in, MetaSession *session, Cap *cap,
in->ctime.encode_timeval(&m->head.ctime);
m->head.time_warp_seq = in->time_warp_seq;
+ if (flush & CEPH_CAP_FILE_WR) {
+ m->inline_version = in->inline_version;
+ m->inline_data = in->inline_data;
+ }
+
in->reported_size = in->size;
m->set_snap_follows(follows);
cap->wanted = want;
@@ -2921,8 +2946,6 @@ void Client::remove_cap(Cap *cap)
i.seq = cap->issue_seq;
i.migrate_seq = cap->mseq;
session->release->caps.push_back(i);
-
- cap->cap_item.remove_myself();
if (in->auth_cap == cap) {
if (in->flushing_cap_item.is_on_list()) {
@@ -2933,7 +2956,13 @@ void Client::remove_cap(Cap *cap)
}
assert(in->caps.count(mds));
in->caps.erase(mds);
- delete cap;
+
+ if (cap == session->s_cap_iterator) {
+ cap->inode = NULL;
+ } else {
+ cap->cap_item.remove_myself();
+ delete cap;
+ }
if (!in->is_any_caps()) {
ldout(cct, 15) << "remove_cap last one, closing snaprealm " << in->snaprealm << dendl;
@@ -2966,7 +2995,7 @@ void Client::trim_caps(MetaSession *s, int max)
xlist<Cap*>::iterator p = s->caps.begin();
while (s->caps.size() > max && !p.end()) {
Cap *cap = *p;
- ++p;
+ s->s_cap_iterator = cap;
Inode *in = cap->inode;
if (in->caps.size() > 1 && cap != in->auth_cap) {
// disposable non-auth cap
@@ -2992,7 +3021,14 @@ void Client::trim_caps(MetaSession *s, int max)
if (all)
trimmed++;
}
+
+ ++p;
+ if (!cap->inode) {
+ cap->cap_item.remove_myself();
+ delete cap;
+ }
}
+ s->s_cap_iterator = NULL;
}
void Client::mark_caps_dirty(Inode *in, int caps)
@@ -3513,7 +3549,9 @@ void Client::handle_cap_trunc(MetaSession *session, Inode *in, MClientCaps *m)
issued |= implemented;
update_inode_file_bits(in, m->get_truncate_seq(), m->get_truncate_size(),
m->get_size(), m->get_time_warp_seq(), m->get_ctime(),
- m->get_mtime(), m->get_atime(), issued);
+ m->get_mtime(), m->get_atime(),
+ m->inline_version, m->inline_data,
+ issued);
m->put();
}
@@ -3663,7 +3701,8 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
in->xattr_version = m->head.xattr_version;
}
update_inode_file_bits(in, m->get_truncate_seq(), m->get_truncate_size(), m->get_size(),
- m->get_time_warp_seq(), m->get_ctime(), m->get_mtime(), m->get_atime(), issued);
+ m->get_time_warp_seq(), m->get_ctime(), m->get_mtime(), m->get_atime(),
+ m->inline_version, m->inline_data, issued);
// max_size
if (cap == in->auth_cap &&
@@ -3843,8 +3882,8 @@ void Client::unmount()
if (cct->_conf->client_oc) {
// flush/release all buffered data
- hash_map<vinodeno_t, Inode*>::iterator next;
- for (hash_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
+ ceph::unordered_map<vinodeno_t, Inode*>::iterator next;
+ for (ceph::unordered_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
p != inode_map.end();
p = next) {
next = p;
@@ -4492,9 +4531,9 @@ int Client::_setattr(Inode *in, struct stat *attr, int mask, int uid, int gid, I
if (in->caps_issued_mask(CEPH_CAP_FILE_EXCL)) {
if (mask & (CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME)) {
if (mask & CEPH_SETATTR_MTIME)
- in->mtime = utime_t(attr->st_mtim.tv_sec, attr->st_mtim.tv_nsec);
+ in->mtime = utime_t(stat_get_mtime_sec(attr), stat_get_mtime_nsec(attr));
if (mask & CEPH_SETATTR_ATIME)
- in->atime = utime_t(attr->st_atim.tv_sec, attr->st_atim.tv_nsec);
+ in->atime = utime_t(stat_get_atime_sec(attr), stat_get_atime_nsec(attr));
in->ctime = ceph_clock_now(cct);
in->time_warp_seq++;
mark_caps_dirty(in, CEPH_CAP_FILE_EXCL);
@@ -4524,14 +4563,14 @@ int Client::_setattr(Inode *in, struct stat *attr, int mask, int uid, int gid, I
req->inode_drop |= CEPH_CAP_AUTH_SHARED;
}
if (mask & CEPH_SETATTR_MTIME) {
- req->head.args.setattr.mtime =
- utime_t(attr->st_mtim.tv_sec, attr->st_mtim.tv_nsec);
+ utime_t mtime = utime_t(stat_get_mtime_sec(attr), stat_get_mtime_nsec(attr));
+ req->head.args.setattr.mtime = mtime;
req->inode_drop |= CEPH_CAP_AUTH_SHARED | CEPH_CAP_FILE_RD |
CEPH_CAP_FILE_WR;
}
if (mask & CEPH_SETATTR_ATIME) {
- req->head.args.setattr.atime =
- utime_t(attr->st_atim.tv_sec, attr->st_atim.tv_nsec);
+ utime_t atime = utime_t(stat_get_atime_sec(attr), stat_get_atime_nsec(attr));
+ req->head.args.setattr.atime = atime;
req->inode_drop |= CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_RD |
CEPH_CAP_FILE_WR;
}
@@ -4641,16 +4680,16 @@ int Client::fill_stat(Inode *in, struct stat *st, frag_info_t *dirstat, nest_inf
st->st_uid = in->uid;
st->st_gid = in->gid;
if (in->ctime.sec() > in->mtime.sec()) {
- st->st_ctim.tv_sec = in->ctime.sec();
- st->st_ctim.tv_nsec = in->ctime.nsec();
+ stat_set_ctime_sec(st, in->ctime.sec());
+ stat_set_ctime_nsec(st, in->ctime.nsec());
} else {
- st->st_ctim.tv_sec = in->mtime.sec();
- st->st_ctim.tv_nsec = in->mtime.nsec();
+ stat_set_ctime_sec(st, in->mtime.sec());
+ stat_set_ctime_nsec(st, in->mtime.nsec());
}
- st->st_atim.tv_sec = in->atime.sec();
- st->st_atim.tv_nsec = in->atime.nsec();
- st->st_mtim.tv_sec = in->mtime.sec();
- st->st_mtim.tv_nsec = in->mtime.nsec();
+ stat_set_atime_sec(st, in->atime.sec());
+ stat_set_atime_nsec(st, in->atime.nsec());
+ stat_set_mtime_sec(st, in->mtime.sec());
+ stat_set_mtime_nsec(st, in->mtime.nsec());
if (in->is_dir()) {
//st->st_size = in->dirstat.size();
st->st_size = in->rstat.rbytes;
@@ -4796,10 +4835,10 @@ int Client::utime(const char *relpath, struct utimbuf *buf)
if (r < 0)
return r;
struct stat attr;
- attr.st_mtim.tv_sec = buf->modtime;
- attr.st_mtim.tv_nsec = 0;
- attr.st_atim.tv_sec = buf->actime;
- attr.st_atim.tv_nsec = 0;
+ stat_set_mtime_sec(&attr, buf->modtime);
+ stat_set_mtime_nsec(&attr, 0);
+ stat_set_atime_sec(&attr, buf->actime);
+ stat_set_atime_nsec(&attr, 0);
return _setattr(in, &attr, CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME);
}
@@ -4817,10 +4856,10 @@ int Client::lutime(const char *relpath, struct utimbuf *buf)
if (r < 0)
return r;
struct stat attr;
- attr.st_mtim.tv_sec = buf->modtime;
- attr.st_mtim.tv_nsec = 0;
- attr.st_atim.tv_sec = buf->actime;
- attr.st_atim.tv_nsec = 0;
+ stat_set_mtime_sec(&attr, buf->modtime);
+ stat_set_mtime_nsec(&attr, 0);
+ stat_set_atime_sec(&attr, buf->actime);
+ stat_set_atime_nsec(&attr, 0);
return _setattr(in, &attr, CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME);
}
@@ -5733,6 +5772,52 @@ void Client::unlock_fh_pos(Fh *f)
f->pos_locked = false;
}
+int Client::uninline_data(Inode *in, Context *onfinish)
+{
+ if (!in->inline_data.length()) {
+ onfinish->complete(0);
+ return 0;
+ }
+
+ char oid_buf[32];
+ snprintf(oid_buf, sizeof(oid_buf), "%llx.00000000", (long long unsigned)in->ino);
+ object_t oid = oid_buf;
+
+ ObjectOperation create_ops;
+ create_ops.create(false);
+
+ objecter->mutate(oid,
+ OSDMap::file_to_object_locator(in->layout),
+ create_ops,
+ in->snaprealm->get_snap_context(),
+ ceph_clock_now(cct),
+ 0,
+ NULL,
+ NULL);
+
+ bufferlist inline_version_bl;
+ ::encode(in->inline_version, inline_version_bl);
+
+ ObjectOperation uninline_ops;
+ uninline_ops.cmpxattr("inline_version",
+ CEPH_OSD_CMPXATTR_OP_GT,
+ CEPH_OSD_CMPXATTR_MODE_U64,
+ inline_version_bl);
+ bufferlist inline_data = in->inline_data;
+ uninline_ops.write(0, inline_data, in->truncate_size, in->truncate_seq);
+ uninline_ops.setxattr("inline_version", inline_version_bl);
+
+ objecter->mutate(oid,
+ OSDMap::file_to_object_locator(in->layout),
+ uninline_ops,
+ in->snaprealm->get_snap_context(),
+ ceph_clock_now(cct),
+ 0,
+ NULL,
+ onfinish);
+
+ return 0;
+}
//
@@ -5778,6 +5863,41 @@ int Client::_read(Fh *f, int64_t offset, uint64_t size, bufferlist *bl)
movepos = true;
}
+ Mutex uninline_flock("Clinet::_read_uninline_data flock");
+ Cond uninline_cond;
+ bool uninline_done = false;
+ int uninline_ret = 0;
+ Context *onuninline = NULL;
+
+ if (in->inline_version < CEPH_INLINE_NONE) {
+ if (!(have & CEPH_CAP_FILE_CACHE)) {
+ onuninline = new C_SafeCond(&uninline_flock,
+ &uninline_cond,
+ &uninline_done,
+ &uninline_ret);
+ uninline_data(in, onuninline);
+ } else {
+ uint32_t len = in->inline_data.length();
+
+ uint64_t endoff = offset + size;
+ if (endoff > in->size)
+ endoff = in->size;
+
+ if (offset < len) {
+ if (endoff <= len) {
+ bl->substr_of(in->inline_data, offset, endoff - offset);
+ } else {
+ bl->substr_of(in->inline_data, offset, len - offset);
+ bl->append_zero(endoff - len);
+ }
+ } else if ((uint64_t)offset < endoff) {
+ bl->append_zero(endoff - offset);
+ }
+
+ goto success;
+ }
+ }
+
if (!conf->client_debug_force_sync_read &&
(cct->_conf->client_oc && (have & CEPH_CAP_FILE_CACHE))) {
@@ -5794,6 +5914,8 @@ int Client::_read(Fh *f, int64_t offset, uint64_t size, bufferlist *bl)
goto done;
}
+success:
+
if (movepos) {
// adjust fd pos
f->pos = offset+bl->length();
@@ -5815,6 +5937,24 @@ int Client::_read(Fh *f, int64_t offset, uint64_t size, bufferlist *bl)
done:
// done!
+
+ if (onuninline) {
+ client_lock.Unlock();
+ uninline_flock.Lock();
+ while (!uninline_done)
+ uninline_cond.Wait(uninline_flock);
+ uninline_flock.Unlock();
+ client_lock.Lock();
+
+ if (uninline_ret >= 0 || uninline_ret == -ECANCELED) {
+ in->inline_data.clear();
+ in->inline_version = CEPH_INLINE_NONE;
+ mark_caps_dirty(in, CEPH_CAP_FILE_WR);
+ check_caps(in, false);
+ } else
+ r = uninline_ret;
+ }
+
put_cap_ref(in, CEPH_CAP_FILE_RD);
return r;
}
@@ -5955,7 +6095,7 @@ int Client::_read_sync(Fh *f, uint64_t off, uint64_t len, bufferlist *bl)
if (r >= 0 && r < wanted) {
if (pos < in->size) {
// zero up to known EOF
- int some = in->size - pos;
+ int64_t some = in->size - pos;
if (some > left)
some = left;
bufferptr z(some);
@@ -6085,6 +6225,43 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
ldout(cct, 10) << " snaprealm " << *in->snaprealm << dendl;
+ Mutex uninline_flock("Clinet::_write_uninline_data flock");
+ Cond uninline_cond;
+ bool uninline_done = false;
+ int uninline_ret = 0;
+ Context *onuninline = NULL;
+
+ if (in->inline_version < CEPH_INLINE_NONE) {
+ if (endoff > cct->_conf->client_max_inline_size ||
+ endoff > CEPH_INLINE_MAX_SIZE ||
+ !(have & CEPH_CAP_FILE_BUFFER)) {
+ onuninline = new C_SafeCond(&uninline_flock,
+ &uninline_cond,
+ &uninline_done,
+ &uninline_ret);
+ uninline_data(in, onuninline);
+ } else {
+ get_cap_ref(in, CEPH_CAP_FILE_BUFFER);
+
+ uint32_t len = in->inline_data.length();
+
+ if (endoff < len)
+ in->inline_data.copy(endoff, len - endoff, bl);
+
+ if (offset < len)
+ in->inline_data.splice(offset, len - offset);
+ else if (offset > len)
+ in->inline_data.append_zero(offset - len);
+
+ in->inline_data.append(bl);
+ in->inline_version++;
+
+ put_cap_ref(in, CEPH_CAP_FILE_BUFFER);
+
+ goto success;
+ }
+ }
+
if (cct->_conf->client_oc && (have & CEPH_CAP_FILE_BUFFER)) {
// do buffered write
if (!in->oset.dirty_or_tx)
@@ -6135,7 +6312,7 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
}
// if we get here, write was successful, update client metadata
-
+success:
// time
lat = ceph_clock_now(cct);
lat -= start;
@@ -6163,6 +6340,24 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
mark_caps_dirty(in, CEPH_CAP_FILE_WR);
done:
+
+ if (onuninline) {
+ client_lock.Unlock();
+ uninline_flock.Lock();
+ while (!uninline_done)
+ uninline_cond.Wait(uninline_flock);
+ uninline_flock.Unlock();
+ client_lock.Lock();
+
+ if (uninline_ret >= 0 || uninline_ret == -ECANCELED) {
+ in->inline_data.clear();
+ in->inline_version = CEPH_INLINE_NONE;
+ mark_caps_dirty(in, CEPH_CAP_FILE_WR);
+ check_caps(in, false);
+ } else
+ r = uninline_ret;
+ }
+
put_cap_ref(in, CEPH_CAP_FILE_WR);
return r;
}
@@ -6639,8 +6834,8 @@ int Client::_ll_put(Inode *in, int num)
void Client::_ll_drop_pins()
{
ldout(cct, 10) << "_ll_drop_pins" << dendl;
- hash_map<vinodeno_t, Inode*>::iterator next;
- for (hash_map<vinodeno_t, Inode*>::iterator it = inode_map.begin();
+ ceph::unordered_map<vinodeno_t, Inode*>::iterator next;
+ for (ceph::unordered_map<vinodeno_t, Inode*>::iterator it = inode_map.begin();
it != inode_map.end();
it = next) {
Inode *in = it->second;
@@ -7819,34 +8014,69 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length)
if (r < 0)
return r;
+ Mutex uninline_flock("Clinet::_fallocate_uninline_data flock");
+ Cond uninline_cond;
+ bool uninline_done = false;
+ int uninline_ret = 0;
+ Context *onuninline = NULL;
+
if (mode & FALLOC_FL_PUNCH_HOLE) {
- Mutex flock("Client::_punch_hole flock");
- Cond cond;
- bool done = false;
- Context *onfinish = new C_SafeCond(&flock, &cond, &done);
- Context *onsafe = new C_Client_SyncCommit(this, in);
+ if (in->inline_version < CEPH_INLINE_NONE &&
+ (have & CEPH_CAP_FILE_BUFFER)) {
+ bufferlist bl;
+ int len = in->inline_data.length();
+ if (offset < len) {
+ if (offset > 0)
+ in->inline_data.copy(0, offset, bl);
+ int size = length;
+ if (offset + size > len)
+ size = len - offset;
+ if (size > 0)
+ bl.append_zero(size);
+ if (offset + size < len)
+ in->inline_data.copy(offset + size, len - offset - size, bl);
+ in->inline_data = bl;
+ in->inline_version++;
+ }
+ in->mtime = ceph_clock_now(cct);
+ mark_caps_dirty(in, CEPH_CAP_FILE_WR);
+ } else {
+ if (in->inline_version < CEPH_INLINE_NONE) {
+ onuninline = new C_SafeCond(&uninline_flock,
+ &uninline_cond,
+ &uninline_done,
+ &uninline_ret);
+ uninline_data(in, onuninline);
+ }
- unsafe_sync_write++;
- get_cap_ref(in, CEPH_CAP_FILE_BUFFER);
+ Mutex flock("Client::_punch_hole flock");
+ Cond cond;
+ bool done = false;
+ Context *onfinish = new C_SafeCond(&flock, &cond, &done);
+ Context *onsafe = new C_Client_SyncCommit(this, in);
- _invalidate_inode_cache(in, offset, length, true);
- r = filer->zero(in->ino, &in->layout,
- in->snaprealm->get_snap_context(),
- offset, length,
- ceph_clock_now(cct),
- 0, true, onfinish, onsafe);
- if (r < 0)
- goto done;
+ unsafe_sync_write++;
+ get_cap_ref(in, CEPH_CAP_FILE_BUFFER);
- in->mtime = ceph_clock_now(cct);
- mark_caps_dirty(in, CEPH_CAP_FILE_WR);
+ _invalidate_inode_cache(in, offset, length, true);
+ r = filer->zero(in->ino, &in->layout,
+ in->snaprealm->get_snap_context(),
+ offset, length,
+ ceph_clock_now(cct),
+ 0, true, onfinish, onsafe);
+ if (r < 0)
+ goto done;
- client_lock.Unlock();
- flock.Lock();
- while (!done)
- cond.Wait(flock);
- flock.Unlock();
- client_lock.Lock();
+ in->mtime = ceph_clock_now(cct);
+ mark_caps_dirty(in, CEPH_CAP_FILE_WR);
+
+ client_lock.Unlock();
+ flock.Lock();
+ while (!done)
+ cond.Wait(flock);
+ flock.Unlock();
+ client_lock.Lock();
+ }
} else if (!(mode & FALLOC_FL_KEEP_SIZE)) {
uint64_t size = offset + length;
if (size > in->size) {
@@ -7861,6 +8091,24 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length)
}
done:
+
+ if (onuninline) {
+ client_lock.Unlock();
+ uninline_flock.Lock();
+ while (!uninline_done)
+ uninline_cond.Wait(uninline_flock);
+ uninline_flock.Unlock();
+ client_lock.Lock();
+
+ if (uninline_ret >= 0 || uninline_ret == -ECANCELED) {
+ in->inline_data.clear();
+ in->inline_version = CEPH_INLINE_NONE;
+ mark_caps_dirty(in, CEPH_CAP_FILE_WR);
+ check_caps(in, false);
+ } else
+ r = uninline_ret;
+ }
+
put_cap_ref(in, CEPH_CAP_FILE_WR);
return r;
}
@@ -8015,6 +8263,8 @@ int Client::get_file_extent_osds(int fd, loff_t off, loff_t *len, vector<int>& o
int Client::get_osd_crush_location(int id, vector<pair<string, string> >& path)
{
Mutex::Locker lock(client_lock);
+ if (id < 0)
+ return -EINVAL;
return osdmap->crush->get_full_location_ordered(id, path);
}
diff --git a/src/client/Client.h b/src/client/Client.h
index 649bacc..af8876d 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -27,8 +27,7 @@ using std::set;
using std::map;
using std::fstream;
-#include <ext/hash_map>
-using namespace __gnu_cxx;
+#include "include/unordered_map.h"
#include "include/filepath.h"
#include "include/interval_set.h"
@@ -302,14 +301,14 @@ protected:
WritebackHandler *writeback_handler;
// cache
- hash_map<vinodeno_t, Inode*> inode_map;
+ ceph::unordered_map<vinodeno_t, Inode*> inode_map;
Inode* root;
LRU lru; // lru list of Dentry's in our local metadata cache.
// all inodes with caps sit on either cap_list or delayed_caps.
xlist<Inode*> delayed_caps, cap_list;
int num_flushing_caps;
- hash_map<inodeno_t,SnapRealm*> snap_realms;
+ ceph::unordered_map<inodeno_t,SnapRealm*> snap_realms;
SnapRealm *get_snap_realm(inodeno_t r);
SnapRealm *get_snap_realm_maybe(inodeno_t r);
@@ -324,7 +323,7 @@ protected:
// file handles, etc.
interval_set<int> free_fd_set; // unused fds
- hash_map<int, Fh*> fd_map;
+ ceph::unordered_map<int, Fh*> fd_map;
int get_fd() {
int fd = free_fd_set.range_start();
@@ -339,7 +338,7 @@ protected:
* Resolve file descriptor, or return NULL.
*/
Fh *get_filehandle(int fd) {
- hash_map<int, Fh*>::iterator p = fd_map.find(fd);
+ ceph::unordered_map<int, Fh*>::iterator p = fd_map.find(fd);
if (p == fd_map.end())
return NULL;
return p->second;
@@ -429,6 +428,9 @@ protected:
void handle_lease(MClientLease *m);
+ // inline data
+ int uninline_data(Inode *in, Context *onfinish);
+
// file caps
void check_cap_issue(Inode *in, Cap *cap, unsigned issued);
void add_update_cap(Inode *in, MetaSession *session, uint64_t cap_id,
@@ -508,6 +510,7 @@ protected:
void update_inode_file_bits(Inode *in,
uint64_t truncate_seq, uint64_t truncate_size, uint64_t size,
uint64_t time_warp_seq, utime_t ctime, utime_t mtime, utime_t atime,
+ version_t inline_version, bufferlist& inline_data,
int issued);
Inode *add_update_inode(InodeStat *st, utime_t ttl, MetaSession *session);
Dentry *insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dlease,
diff --git a/src/client/Dir.h b/src/client/Dir.h
index 7e05d37..f0029b1 100644
--- a/src/client/Dir.h
+++ b/src/client/Dir.h
@@ -6,7 +6,7 @@ class Inode;
class Dir {
public:
Inode *parent_inode; // my inode
- hash_map<string, Dentry*> dentries;
+ ceph::unordered_map<string, Dentry*> dentries;
map<string, Dentry*> dentry_map;
uint64_t release_count;
uint64_t max_offset;
diff --git a/src/client/Inode.h b/src/client/Inode.h
index cc054a6..cd58b82 100644
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -111,6 +111,10 @@ class Inode {
version_t version; // auth only
version_t xattr_version;
+ // inline data
+ version_t inline_version;
+ bufferlist inline_data;
+
bool is_symlink() const { return (mode & S_IFMT) == S_IFLNK; }
bool is_dir() const { return (mode & S_IFMT) == S_IFDIR; }
bool is_file() const { return (mode & S_IFMT) == S_IFREG; }
@@ -207,6 +211,7 @@ class Inode {
rdev(0), mode(0), uid(0), gid(0), nlink(0),
size(0), truncate_seq(1), truncate_size(-1),
time_warp_seq(0), max_size(0), version(0), xattr_version(0),
+ inline_version(0),
flags(0),
dir_hashed(false), dir_replicated(false), auth_cap(NULL),
dirty_caps(0), flushing_caps(0), flushing_cap_seq(0), shared_gen(0), cache_gen(0),
diff --git a/src/client/MetaSession.h b/src/client/MetaSession.h
index a6cf634..d9fcbb8 100644
--- a/src/client/MetaSession.h
+++ b/src/client/MetaSession.h
@@ -43,12 +43,14 @@ struct MetaSession {
xlist<MetaRequest*> requests;
xlist<MetaRequest*> unsafe_requests;
+ Cap *s_cap_iterator;
+
MClientCapRelease *release;
MetaSession()
: mds_num(-1), con(NULL),
seq(0), cap_gen(0), cap_renew_seq(0), num_caps(0),
- state(STATE_NEW),
+ state(STATE_NEW), s_cap_iterator(NULL),
release(NULL)
{}
~MetaSession();
diff --git a/src/client/SyntheticClient.cc b/src/client/SyntheticClient.cc
index 6b2c7b1..aac509c 100644
--- a/src/client/SyntheticClient.cc
+++ b/src/client/SyntheticClient.cc
@@ -1014,12 +1014,12 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
utime_t start = ceph_clock_now(client->cct);
- hash_map<int64_t, int64_t> open_files;
- hash_map<int64_t, dir_result_t*> open_dirs;
+ ceph::unordered_map<int64_t, int64_t> open_files;
+ ceph::unordered_map<int64_t, dir_result_t*> open_dirs;
- hash_map<int64_t, Fh*> ll_files;
- hash_map<int64_t, void*> ll_dirs;
- hash_map<uint64_t, int64_t> ll_inos;
+ ceph::unordered_map<int64_t, Fh*> ll_files;
+ ceph::unordered_map<int64_t, void*> ll_dirs;
+ ceph::unordered_map<uint64_t, int64_t> ll_inos;
ll_inos[1] = 1; // root inode is known.
@@ -1404,7 +1404,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t ol = t.get_int();
object_t oid = file_object_t(oh, ol);
lock.Lock();
- object_locator_t oloc(CEPH_DATA_RULE);
+ object_locator_t oloc(SYNCLIENT_FIRST_POOL);
uint64_t size;
utime_t mtime;
client->objecter->stat(oid, oloc, CEPH_NOSNAP, &size, &mtime, 0, new C_SafeCond(&lock, &cond, &ack));
@@ -1417,7 +1417,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t off = t.get_int();
int64_t len = t.get_int();
object_t oid = file_object_t(oh, ol);
- object_locator_t oloc(CEPH_DATA_RULE);
+ object_locator_t oloc(SYNCLIENT_FIRST_POOL);
lock.Lock();
bufferlist bl;
client->objecter->read(oid, oloc, off, len, CEPH_NOSNAP, &bl, 0, new C_SafeCond(&lock, &cond, &ack));
@@ -1430,7 +1430,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t off = t.get_int();
int64_t len = t.get_int();
object_t oid = file_object_t(oh, ol);
- object_locator_t oloc(CEPH_DATA_RULE);
+ object_locator_t oloc(SYNCLIENT_FIRST_POOL);
lock.Lock();
bufferptr bp(len);
bufferlist bl;
@@ -1449,7 +1449,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
int64_t off = t.get_int();
int64_t len = t.get_int();
object_t oid = file_object_t(oh, ol);
- object_locator_t oloc(CEPH_DATA_RULE);
+ object_locator_t oloc(SYNCLIENT_FIRST_POOL);
lock.Lock();
SnapContext snapc;
client->objecter->zero(oid, oloc, off, len, snapc, ceph_clock_now(client->cct), 0,
@@ -1479,25 +1479,25 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
lock.Unlock();
// close open files
- for (hash_map<int64_t, int64_t>::iterator fi = open_files.begin();
+ for (ceph::unordered_map<int64_t, int64_t>::iterator fi = open_files.begin();
fi != open_files.end();
++fi) {
dout(1) << "leftover close " << fi->second << dendl;
if (fi->second > 0) client->close(fi->second);
}
- for (hash_map<int64_t, dir_result_t*>::iterator fi = open_dirs.begin();
+ for (ceph::unordered_map<int64_t, dir_result_t*>::iterator fi = open_dirs.begin();
fi != open_dirs.end();
++fi) {
dout(1) << "leftover closedir " << fi->second << dendl;
if (fi->second != 0) client->closedir(fi->second);
}
- for (hash_map<int64_t,Fh*>::iterator fi = ll_files.begin();
+ for (ceph::unordered_map<int64_t,Fh*>::iterator fi = ll_files.begin();
fi != ll_files.end();
++fi) {
dout(1) << "leftover ll_release " << fi->second << dendl;
if (fi->second) client->ll_release(fi->second);
}
- for (hash_map<int64_t,void*>::iterator fi = ll_dirs.begin();
+ for (ceph::unordered_map<int64_t,void*>::iterator fi = ll_dirs.begin();
fi != ll_dirs.end();
++fi) {
dout(1) << "leftover ll_releasedir " << fi->second << dendl;
@@ -1559,8 +1559,8 @@ int SyntheticClient::full_walk(string& basedir)
memset(&empty, 0, sizeof(empty));
statq.push_back(empty);
- hash_map<inodeno_t, int> nlink;
- hash_map<inodeno_t, int> nlink_seen;
+ ceph::unordered_map<inodeno_t, int> nlink;
+ ceph::unordered_map<inodeno_t, int> nlink_seen;
while (!dirq.empty()) {
string dir = dirq.front();
@@ -1638,7 +1638,7 @@ int SyntheticClient::full_walk(string& basedir)
}
}
- for (hash_map<inodeno_t,int>::iterator p = nlink.begin(); p != nlink.end(); ++p) {
+ for (ceph::unordered_map<inodeno_t,int>::iterator p = nlink.begin(); p != nlink.end(); ++p) {
if (nlink_seen[p->first] != p->second)
dout(0) << p->first << " nlink " << p->second << " != " << nlink_seen[p->first] << "seen" << dendl;
}
@@ -1679,7 +1679,7 @@ int SyntheticClient::dump_placement(string& fn) {
for (vector<ObjectExtent>::iterator i = extents.begin();
i != extents.end(); ++i) {
- int osd = client->osdmap->get_pg_primary(client->osdmap->object_locator_to_pg(i->oid, i->oloc));
+ int osd = client->osdmap->get_pg_acting_primary(client->osdmap->object_locator_to_pg(i->oid, i->oloc));
// run through all the buffer extents
for (vector<pair<uint64_t, uint64_t> >::iterator j = i->buffer_extents.begin();
@@ -1959,7 +1959,7 @@ int SyntheticClient::overload_osd_0(int n, int size, int wrsize) {
int SyntheticClient::check_first_primary(int fh) {
vector<ObjectExtent> extents;
client->enumerate_layout(fh, extents, 1, 0);
- return client->osdmap->get_pg_primary(client->osdmap->object_locator_to_pg(extents.begin()->oid,
+ return client->osdmap->get_pg_acting_primary(client->osdmap->object_locator_to_pg(extents.begin()->oid,
extents.begin()->oloc));
}
@@ -2218,7 +2218,7 @@ int SyntheticClient::create_objects(int nobj, int osize, int inflight)
if (time_to_stop()) break;
object_t oid = file_object_t(999, i);
- object_locator_t oloc(CEPH_DATA_RULE);
+ object_locator_t oloc(SYNCLIENT_FIRST_POOL);
SnapContext snapc;
if (i % inflight == 0) {
@@ -2319,7 +2319,7 @@ int SyntheticClient::object_rw(int nobj, int osize, int wrpc,
o = (long)trunc(pow(r, rskew) * (double)nobj); // exponentially skew towards 0
}
object_t oid = file_object_t(999, o);
- object_locator_t oloc(CEPH_DATA_RULE);
+ object_locator_t oloc(SYNCLIENT_FIRST_POOL);
SnapContext snapc;
client->client_lock.Lock();
diff --git a/src/client/SyntheticClient.h b/src/client/SyntheticClient.h
index f9f4e3d..e0687b4 100644
--- a/src/client/SyntheticClient.h
+++ b/src/client/SyntheticClient.h
@@ -23,6 +23,8 @@
#include "Trace.h"
+#define SYNCLIENT_FIRST_POOL 0
+
#define SYNCLIENT_MODE_RANDOMWALK 1
#define SYNCLIENT_MODE_FULLWALK 2
#define SYNCLIENT_MODE_REPEATWALK 3
diff --git a/src/client/Trace.cc b/src/client/Trace.cc
index f718595..dbd9fce 100644
--- a/src/client/Trace.cc
+++ b/src/client/Trace.cc
@@ -19,8 +19,6 @@
#include <iostream>
#include <map>
-#include <ext/rope>
-using namespace __gnu_cxx;
#include "common/Mutex.h"
diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc
index 88f727e..1c58b6c 100644
--- a/src/client/fuse_ll.cc
+++ b/src/client/fuse_ll.cc
@@ -81,8 +81,8 @@ public:
Mutex stag_lock;
int last_stag;
- hash_map<uint64_t,int> snap_stag_map;
- hash_map<int,uint64_t> stag_snap_map;
+ ceph::unordered_map<uint64_t,int> snap_stag_map;
+ ceph::unordered_map<int,uint64_t> stag_snap_map;
};
@@ -769,7 +769,11 @@ done:
int CephFuse::Handle::loop()
{
- return fuse_session_loop(se);
+ if (client->cct->_conf->fuse_multithreaded) {
+ return fuse_session_loop_mt(se);
+ } else {
+ return fuse_session_loop(se);
+ }
}
uint64_t CephFuse::Handle::fino_snap(uint64_t fino)
diff --git a/src/cls/Makefile.am b/src/cls/Makefile.am
index 2d3d43c..ea44fe7 100644
--- a/src/cls/Makefile.am
+++ b/src/cls/Makefile.am
@@ -2,17 +2,17 @@
libcls_hello_la_SOURCES = cls/hello/cls_hello.cc
libcls_hello_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_hello.la
libcls_rbd_la_SOURCES = cls/rbd/cls_rbd.cc
libcls_rbd_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_rbd.la
libcls_lock_la_SOURCES = cls/lock/cls_lock.cc
libcls_lock_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_lock.la
libcls_refcount_la_SOURCES = \
@@ -20,36 +20,41 @@ libcls_refcount_la_SOURCES = \
cls/refcount/cls_refcount_ops.cc \
common/ceph_json.cc
libcls_refcount_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_refcount.la
libcls_version_la_SOURCES = cls/version/cls_version.cc
libcls_version_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_version.la
libcls_log_la_SOURCES = cls/log/cls_log.cc
libcls_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_log.la
libcls_statelog_la_SOURCES = cls/statelog/cls_statelog.cc
libcls_statelog_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_statelog.la
libcls_replica_log_la_SOURCES = cls/replica_log/cls_replica_log.cc
libcls_replica_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_replica_log.la
+libcls_user_la_SOURCES = cls/user/cls_user.cc
+libcls_user_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libcls_user_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+radoslib_LTLIBRARIES += libcls_user.la
+
libcls_rgw_la_SOURCES = \
cls/rgw/cls_rgw.cc \
cls/rgw/cls_rgw_ops.cc \
cls/rgw/cls_rgw_types.cc \
common/ceph_json.cc
libcls_rgw_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_rgw.la
## Rados object client classes
@@ -95,6 +100,12 @@ DENCODER_DEPS += libcls_rgw_client.la
libcls_rbd_client_la_SOURCES = cls/rbd/cls_rbd_client.cc
noinst_LTLIBRARIES += libcls_rbd_client.la
+libcls_user_client_a_SOURCES = cls/user/cls_user_client.cc \
+ cls/user/cls_user_types.cc \
+ cls/user/cls_user_ops.cc
+DENCODER_DEPS += libcls_user_client.a
+
+noinst_LIBRARIES += libcls_user_client.a
noinst_HEADERS += \
cls/lock/cls_lock_types.h \
@@ -118,5 +129,8 @@ noinst_HEADERS += \
cls/replica_log/cls_replica_log_client.h \
cls/rgw/cls_rgw_client.h \
cls/rgw/cls_rgw_ops.h \
- cls/rgw/cls_rgw_types.h
+ cls/rgw/cls_rgw_types.h \
+ cls/user/cls_user_client.h \
+ cls/user/cls_user_ops.h \
+ cls/user/cls_user_types.h
diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc
new file mode 100644
index 0000000..a72ba33
--- /dev/null
+++ b/src/cls/user/cls_user.cc
@@ -0,0 +1,372 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <iostream>
+
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include "include/types.h"
+#include "include/utime.h"
+#include "objclass/objclass.h"
+
+#include "cls_user_types.h"
+#include "cls_user_ops.h"
+
+CLS_VER(1,0)
+CLS_NAME(user)
+
+cls_handle_t h_class;
+cls_method_handle_t h_user_set_buckets_info;
+cls_method_handle_t h_user_complete_stats_sync;
+cls_method_handle_t h_user_remove_bucket;
+cls_method_handle_t h_user_list_buckets;
+cls_method_handle_t h_user_get_header;
+
+static int write_entry(cls_method_context_t hctx, const string& key, const cls_user_bucket_entry& entry)
+{
+ bufferlist bl;
+ ::encode(entry, bl);
+
+ int ret = cls_cxx_map_set_val(hctx, key, &bl);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static int remove_entry(cls_method_context_t hctx, const string& key)
+{
+ int ret = cls_cxx_map_remove_key(hctx, key);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static void get_key_by_bucket_name(const string& bucket_name, string *key)
+{
+ *key = bucket_name;
+}
+
+static int get_existing_bucket_entry(cls_method_context_t hctx, const string& bucket_name,
+ cls_user_bucket_entry& entry)
+{
+ if (bucket_name.empty()) {
+ return -EINVAL;
+ }
+
+ string key;
+ get_key_by_bucket_name(bucket_name, &key);
+
+ bufferlist bl;
+ int rc = cls_cxx_map_get_val(hctx, key, &bl);
+ if (rc < 0) {
+ CLS_LOG(10, "could not read entry %s", key.c_str());
+ return rc;
+ }
+ try {
+ bufferlist::iterator iter = bl.begin();
+ ::decode(entry, iter);
+ } catch (buffer::error& err) {
+ CLS_LOG(0, "ERROR: failed to decode entry %s", key.c_str());
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static int read_header(cls_method_context_t hctx, cls_user_header *header)
+{
+ bufferlist bl;
+
+ int ret = cls_cxx_map_read_header(hctx, &bl);
+ if (ret < 0)
+ return ret;
+
+ if (bl.length() == 0) {
+ *header = cls_user_header();
+ return 0;
+ }
+
+ try {
+ ::decode(*header, bl);
+ } catch (buffer::error& err) {
+ CLS_LOG(0, "ERROR: failed to decode user header");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static void add_header_stats(cls_user_stats *stats, cls_user_bucket_entry& entry)
+{
+ stats->total_entries += entry.count;
+ stats->total_bytes += entry.size;
+ stats->total_bytes_rounded += entry.size_rounded;
+}
+
+static void dec_header_stats(cls_user_stats *stats, cls_user_bucket_entry& entry)
+{
+ stats->total_bytes -= entry.size;
+ stats->total_bytes_rounded -= entry.size_rounded;
+ stats->total_entries -= entry.count;
+}
+
+static int cls_user_set_buckets_info(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+ bufferlist::iterator in_iter = in->begin();
+
+ cls_user_set_buckets_op op;
+ try {
+ ::decode(op, in_iter);
+ } catch (buffer::error& err) {
+ CLS_LOG(1, "ERROR: cls_user_add_op(): failed to decode op");
+ return -EINVAL;
+ }
+
+ cls_user_header header;
+ int ret = read_header(hctx, &header);
+ if (ret < 0) {
+ CLS_LOG(0, "ERROR: failed to read user info header ret=%d", ret);
+ return ret;
+ }
+
+ for (list<cls_user_bucket_entry>::iterator iter = op.entries.begin();
+ iter != op.entries.end(); ++iter) {
+ cls_user_bucket_entry& entry = *iter;
+
+ string key;
+
+ get_key_by_bucket_name(entry.bucket.name, &key);
+
+ cls_user_bucket_entry old_entry;
+ ret = get_existing_bucket_entry(hctx, key, old_entry);
+
+ if (ret == -ENOENT) {
+ if (!op.add)
+ continue; /* racing bucket removal */
+
+ ret = 0;
+ }
+
+ if (ret < 0) {
+ CLS_LOG(0, "ERROR: get_existing_bucket_entry() key=%s returned %d", key.c_str(), ret);
+ return ret;
+ } else if (ret >= 0 && old_entry.user_stats_sync) {
+ dec_header_stats(&header.stats, old_entry);
+ }
+
+ CLS_LOG(20, "storing entry for key=%s size=%lld count=%lld",
+ key.c_str(), (long long)entry.size, (long long)entry.count);
+
+ entry.user_stats_sync = true;
+
+ ret = write_entry(hctx, key, entry);
+ if (ret < 0)
+ return ret;
+
+ add_header_stats(&header.stats, entry);
+ }
+
+ bufferlist bl;
+
+ CLS_LOG(20, "header: total bytes=%lld entries=%lld", (long long)header.stats.total_bytes, (long long)header.stats.total_entries);
+
+ if (header.last_stats_update < op.time)
+ header.last_stats_update = op.time;
+
+ ::encode(header, bl);
+
+ ret = cls_cxx_map_write_header(hctx, &bl);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static int cls_user_complete_stats_sync(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+ bufferlist::iterator in_iter = in->begin();
+
+ cls_user_complete_stats_sync_op op;
+ try {
+ ::decode(op, in_iter);
+ } catch (buffer::error& err) {
+ CLS_LOG(1, "ERROR: cls_user_add_op(): failed to decode op");
+ return -EINVAL;
+ }
+
+ cls_user_header header;
+ int ret = read_header(hctx, &header);
+ if (ret < 0) {
+ CLS_LOG(0, "ERROR: failed to read user info header ret=%d", ret);
+ return ret;
+ }
+
+ if (header.last_stats_sync < op.time)
+ header.last_stats_sync = op.time;
+
+ bufferlist bl;
+
+ ::encode(header, bl);
+
+ ret = cls_cxx_map_write_header(hctx, &bl);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static int cls_user_remove_bucket(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+ bufferlist::iterator in_iter = in->begin();
+
+ cls_user_remove_bucket_op op;
+ try {
+ ::decode(op, in_iter);
+ } catch (buffer::error& err) {
+ CLS_LOG(1, "ERROR: cls_user_add_op(): failed to decode op");
+ return -EINVAL;
+ }
+
+ cls_user_header header;
+ int ret = read_header(hctx, &header);
+ if (ret < 0) {
+ CLS_LOG(0, "ERROR: failed to read user info header ret=%d", ret);
+ return ret;
+ }
+
+ string key;
+
+ get_key_by_bucket_name(op.bucket.name, &key);
+
+ cls_user_bucket_entry entry;
+ ret = get_existing_bucket_entry(hctx, key, entry);
+ if (ret == -ENOENT) {
+ return 0; /* idempotent removal */
+ }
+ if (ret < 0) {
+ CLS_LOG(0, "ERROR: get existing bucket entry, key=%s ret=%d", key.c_str(), ret);
+ return ret;
+ }
+
+ if (entry.user_stats_sync) {
+ dec_header_stats(&header.stats, entry);
+ }
+
+ CLS_LOG(20, "removing entry at %s", key.c_str());
+
+ ret = remove_entry(hctx, key);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+ bufferlist::iterator in_iter = in->begin();
+
+ cls_user_list_buckets_op op;
+ try {
+ ::decode(op, in_iter);
+ } catch (buffer::error& err) {
+ CLS_LOG(1, "ERROR: cls_user_list_op(): failed to decode op");
+ return -EINVAL;
+ }
+
+ map<string, bufferlist> keys;
+
+ string from_index = op.marker;
+
+#define MAX_ENTRIES 1000
+ size_t max_entries = op.max_entries;
+ if (!max_entries || max_entries > MAX_ENTRIES)
+ max_entries = MAX_ENTRIES;
+
+ string match_prefix;
+
+ int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys);
+ if (rc < 0)
+ return rc;
+
+ CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str());
+ cls_user_list_buckets_ret ret;
+
+ list<cls_user_bucket_entry>& entries = ret.entries;
+ map<string, bufferlist>::iterator iter = keys.begin();
+
+ bool done = false;
+ string marker;
+
+ size_t i;
+ for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) {
+ const string& index = iter->first;
+ marker = index;
+
+ bufferlist& bl = iter->second;
+ bufferlist::iterator biter = bl.begin();
+ try {
+ cls_user_bucket_entry e;
+ ::decode(e, biter);
+ entries.push_back(e);
+ } catch (buffer::error& err) {
+ CLS_LOG(0, "ERROR: cls_user_list: could not decode entry, index=%s", index.c_str());
+ }
+ }
+
+ if (iter == keys.end())
+ done = true;
+ else
+ ret.marker = marker;
+
+ ret.truncated = !done;
+
+ ::encode(ret, *out);
+
+ return 0;
+}
+
+static int cls_user_get_header(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+ bufferlist::iterator in_iter = in->begin();
+
+ cls_user_get_header_op op;
+ try {
+ ::decode(op, in_iter);
+ } catch (buffer::error& err) {
+ CLS_LOG(1, "ERROR: cls_user_get_header_op(): failed to decode op");
+ return -EINVAL;
+ }
+
+ cls_user_get_header_ret op_ret;
+
+ int ret = read_header(hctx, &op_ret.header);
+ if (ret < 0)
+ return ret;
+
+ ::encode(op_ret, *out);
+
+ return 0;
+}
+
+void __cls_init()
+{
+ CLS_LOG(1, "Loaded user class!");
+
+ cls_register("user", &h_class);
+
+ /* log */
+ cls_register_cxx_method(h_class, "set_buckets_info", CLS_METHOD_RD | CLS_METHOD_WR,
+ cls_user_set_buckets_info, &h_user_set_buckets_info);
+ cls_register_cxx_method(h_class, "complete_stats_sync", CLS_METHOD_RD | CLS_METHOD_WR,
+ cls_user_complete_stats_sync, &h_user_set_buckets_info);
+ cls_register_cxx_method(h_class, "remove_bucket", CLS_METHOD_RD | CLS_METHOD_WR, cls_user_remove_bucket, &h_user_remove_bucket);
+ cls_register_cxx_method(h_class, "list_buckets", CLS_METHOD_RD, cls_user_list_buckets, &h_user_list_buckets);
+ cls_register_cxx_method(h_class, "get_header", CLS_METHOD_RD, cls_user_get_header, &h_user_get_header);
+
+ return;
+}
+
diff --git a/src/cls/user/cls_user_client.cc b/src/cls/user/cls_user_client.cc
new file mode 100644
index 0000000..d86c5bf
--- /dev/null
+++ b/src/cls/user/cls_user_client.cc
@@ -0,0 +1,145 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <errno.h>
+
+#include "include/types.h"
+#include "cls/user/cls_user_ops.h"
+#include "cls/user/cls_user_client.h"
+#include "include/rados/librados.hpp"
+
+
+using namespace librados;
+
+
+void cls_user_set_buckets(librados::ObjectWriteOperation& op, list<cls_user_bucket_entry>& entries, bool add)
+{
+ bufferlist in;
+ cls_user_set_buckets_op call;
+ call.entries = entries;
+ call.add = add;
+ call.time = ceph_clock_now(NULL);
+ ::encode(call, in);
+ op.exec("user", "set_buckets_info", in);
+}
+
+void cls_user_complete_stats_sync(librados::ObjectWriteOperation& op)
+{
+ bufferlist in;
+ cls_user_complete_stats_sync_op call;
+ call.time = ceph_clock_now(NULL);
+ ::encode(call, in);
+ op.exec("user", "complete_stats_sync", in);
+}
+
+void cls_user_remove_bucket(librados::ObjectWriteOperation& op, const cls_user_bucket& bucket)
+{
+ bufferlist in;
+ cls_user_remove_bucket_op call;
+ call.bucket = bucket;
+ ::encode(call, in);
+ op.exec("user", "remove_bucket", in);
+}
+
+class ClsUserListCtx : public ObjectOperationCompletion {
+ list<cls_user_bucket_entry> *entries;
+ string *marker;
+ bool *truncated;
+ int *pret;
+public:
+ ClsUserListCtx(list<cls_user_bucket_entry> *_entries, string *_marker, bool *_truncated, int *_pret) :
+ entries(_entries), marker(_marker), truncated(_truncated), pret(_pret) {}
+ void handle_completion(int r, bufferlist& outbl) {
+ if (r >= 0) {
+ cls_user_list_buckets_ret ret;
+ try {
+ bufferlist::iterator iter = outbl.begin();
+ ::decode(ret, iter);
+ if (entries)
+ *entries = ret.entries;
+ if (truncated)
+ *truncated = ret.truncated;
+ if (marker)
+ *marker = ret.marker;
+ } catch (buffer::error& err) {
+ r = -EIO;
+ }
+ }
+ if (pret) {
+ *pret = r;
+ }
+ }
+};
+
+void cls_user_bucket_list(librados::ObjectReadOperation& op,
+ const string& in_marker, int max_entries, list<cls_user_bucket_entry>& entries,
+ string *out_marker, bool *truncated, int *pret)
+{
+ bufferlist inbl;
+ cls_user_list_buckets_op call;
+ call.marker = in_marker;
+ call.max_entries = max_entries;
+
+ ::encode(call, inbl);
+
+ op.exec("user", "list_buckets", inbl, new ClsUserListCtx(&entries, out_marker, truncated, pret));
+}
+
+class ClsUserGetHeaderCtx : public ObjectOperationCompletion {
+ cls_user_header *header;
+ RGWGetUserHeader_CB *ret_ctx;
+ int *pret;
+public:
+ ClsUserGetHeaderCtx(cls_user_header *_h, RGWGetUserHeader_CB *_ctx, int *_pret) : header(_h), ret_ctx(_ctx), pret(_pret) {}
+ ~ClsUserGetHeaderCtx() {
+ if (ret_ctx) {
+ ret_ctx->put();
+ }
+ }
+ void handle_completion(int r, bufferlist& outbl) {
+ if (r >= 0) {
+ cls_user_get_header_ret ret;
+ try {
+ bufferlist::iterator iter = outbl.begin();
+ ::decode(ret, iter);
+ if (header)
+ *header = ret.header;
+ } catch (buffer::error& err) {
+ r = -EIO;
+ }
+ if (ret_ctx) {
+ ret_ctx->handle_response(r, ret.header);
+ }
+ }
+ if (pret) {
+ *pret = r;
+ }
+ }
+};
+
+void cls_user_get_header(librados::ObjectReadOperation& op,
+ cls_user_header *header, int *pret)
+{
+ bufferlist inbl;
+ cls_user_get_header_op call;
+
+ ::encode(call, inbl);
+
+ op.exec("user", "get_header", inbl, new ClsUserGetHeaderCtx(header, NULL, pret));
+}
+
+int cls_user_get_header_async(IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx)
+{
+ bufferlist in, out;
+ cls_user_get_header_op call;
+ ::encode(call, in);
+ ObjectReadOperation op;
+ op.exec("user", "get_header", in, new ClsUserGetHeaderCtx(NULL, ctx, NULL)); /* no need to pass pret, as we'll call ctx->handle_response() with correct error */
+ AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
+ int r = io_ctx.aio_operate(oid, c, &op, NULL);
+ c->release();
+ if (r < 0)
+ return r;
+
+ return 0;
+}
diff --git a/src/cls/user/cls_user_client.h b/src/cls/user/cls_user_client.h
new file mode 100644
index 0000000..dcfdab6
--- /dev/null
+++ b/src/cls/user/cls_user_client.h
@@ -0,0 +1,33 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_CLS_USER_CLIENT_H
+#define CEPH_CLS_USER_CLIENT_H
+
+#include "include/types.h"
+#include "include/rados/librados.hpp"
+#include "cls_user_types.h"
+#include "common/RefCountedObj.h"
+
+class RGWGetUserHeader_CB : public RefCountedObject {
+public:
+ virtual ~RGWGetUserHeader_CB() {}
+ virtual void handle_response(int r, cls_user_header& header) = 0;
+};
+
+/*
+ * user objclass
+ */
+
+void cls_user_set_buckets(librados::ObjectWriteOperation& op, list<cls_user_bucket_entry>& entries, bool add);
+void cls_user_complete_stats_sync(librados::ObjectWriteOperation& op);
+void cls_user_remove_bucket(librados::ObjectWriteOperation& op, const cls_user_bucket& bucket);
+void cls_user_bucket_list(librados::ObjectReadOperation& op,
+ const string& in_marker, int max_entries,
+ list<cls_user_bucket_entry>& entries,
+ string *out_marker, bool *truncated,
+ int *pret);
+void cls_user_get_header(librados::ObjectReadOperation& op, cls_user_header *header, int *pret);
+int cls_user_get_header_async(librados::IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx);
+
+#endif
diff --git a/src/cls/user/cls_user_ops.cc b/src/cls/user/cls_user_ops.cc
new file mode 100644
index 0000000..c50ab6a
--- /dev/null
+++ b/src/cls/user/cls_user_ops.cc
@@ -0,0 +1,115 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "cls/user/cls_user_types.h"
+#include "cls/user/cls_user_ops.h"
+#include "common/Formatter.h"
+#include "common/ceph_json.h"
+
+void cls_user_set_buckets_op::dump(Formatter *f) const
+{
+ encode_json("entries", entries, f);
+ encode_json("add", add, f);
+ encode_json("time", time, f);
+}
+
+void cls_user_set_buckets_op::generate_test_instances(list<cls_user_set_buckets_op*>& ls)
+{
+ ls.push_back(new cls_user_set_buckets_op);
+ cls_user_set_buckets_op *op = new cls_user_set_buckets_op;
+ for (int i = 0; i < 3; i++) {
+ cls_user_bucket_entry e;
+ cls_user_gen_test_bucket_entry(&e, i);
+ op->entries.push_back(e);
+ }
+ op->add = true;
+ op->time = utime_t(1, 0);
+ ls.push_back(op);
+}
+
+void cls_user_remove_bucket_op::dump(Formatter *f) const
+{
+ encode_json("bucket", bucket, f);
+}
+
+void cls_user_remove_bucket_op::generate_test_instances(list<cls_user_remove_bucket_op*>& ls)
+{
+ ls.push_back(new cls_user_remove_bucket_op);
+ cls_user_remove_bucket_op *op = new cls_user_remove_bucket_op;
+ cls_user_gen_test_bucket(&op->bucket, 0);
+ ls.push_back(op);
+}
+
+void cls_user_list_buckets_op::dump(Formatter *f) const
+{
+ encode_json("marker", marker, f);
+ encode_json("max_entries", max_entries, f);
+}
+
+void cls_user_list_buckets_op::generate_test_instances(list<cls_user_list_buckets_op*>& ls)
+{
+ ls.push_back(new cls_user_list_buckets_op);
+ cls_user_list_buckets_op *op = new cls_user_list_buckets_op;
+ op->marker = "marker";
+ op->max_entries = 1000;
+ ls.push_back(op);
+}
+
+void cls_user_list_buckets_ret::dump(Formatter *f) const
+{
+ encode_json("entries", entries, f);
+ encode_json("marker", marker, f);
+ encode_json("truncated", truncated, f);
+}
+
+void cls_user_list_buckets_ret::generate_test_instances(list<cls_user_list_buckets_ret*>& ls)
+{
+ ls.push_back(new cls_user_list_buckets_ret);
+ cls_user_list_buckets_ret *ret = new cls_user_list_buckets_ret;
+ for (int i = 0; i < 3; i++) {
+ cls_user_bucket_entry e;
+ cls_user_gen_test_bucket_entry(&e, i);
+ ret->entries.push_back(e);
+ }
+ ret->marker = "123";
+ ret->truncated = true;
+ ls.push_back(ret);
+}
+
+void cls_user_get_header_op::dump(Formatter *f) const
+{
+ // empty!
+}
+
+void cls_user_get_header_op::generate_test_instances(list<cls_user_get_header_op*>& ls)
+{
+ ls.push_back(new cls_user_get_header_op);
+}
+
+void cls_user_get_header_ret::dump(Formatter *f) const
+{
+ encode_json("header", header, f);
+}
+
+void cls_user_get_header_ret::generate_test_instances(list<cls_user_get_header_ret*>& ls)
+{
+ ls.push_back(new cls_user_get_header_ret);
+ cls_user_get_header_ret *ret = new cls_user_get_header_ret;
+ cls_user_gen_test_header(&ret->header);
+ ls.push_back(ret);
+}
+
+void cls_user_complete_stats_sync_op::dump(Formatter *f) const
+{
+ encode_json("time", time, f);
+}
+
+void cls_user_complete_stats_sync_op::generate_test_instances(list<cls_user_complete_stats_sync_op*>& ls)
+{
+ ls.push_back(new cls_user_complete_stats_sync_op);
+ cls_user_complete_stats_sync_op *op = new cls_user_complete_stats_sync_op;
+ op->time = utime_t(12345, 0);
+ ls.push_back(op);
+}
+
+
diff --git a/src/cls/user/cls_user_ops.h b/src/cls/user/cls_user_ops.h
new file mode 100644
index 0000000..e1a169b
--- /dev/null
+++ b/src/cls/user/cls_user_ops.h
@@ -0,0 +1,179 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_CLS_USER_OPS_H
+#define CEPH_CLS_USER_OPS_H
+
+#include "include/types.h"
+#include "cls_user_types.h"
+
+struct cls_user_set_buckets_op {
+ list<cls_user_bucket_entry> entries;
+ bool add;
+ utime_t time; /* op time */
+
+ cls_user_set_buckets_op() : add(false) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(entries, bl);
+ ::encode(add, bl);
+ ::encode(time, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(entries, bl);
+ ::decode(add, bl);
+ ::decode(time, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_set_buckets_op*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_set_buckets_op)
+
+struct cls_user_remove_bucket_op {
+ cls_user_bucket bucket;
+
+ cls_user_remove_bucket_op() {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(bucket, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(bucket, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_remove_bucket_op*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_remove_bucket_op)
+
+struct cls_user_list_buckets_op {
+ string marker;
+ int max_entries; /* upperbound to returned num of entries
+ might return less than that and still be truncated */
+
+ cls_user_list_buckets_op()
+ : max_entries(0) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(marker, bl);
+ ::encode(max_entries, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(marker, bl);
+ ::decode(max_entries, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_list_buckets_op*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_list_buckets_op)
+
+struct cls_user_list_buckets_ret {
+ list<cls_user_bucket_entry> entries;
+ string marker;
+ bool truncated;
+
+ cls_user_list_buckets_ret() : truncated(false) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(entries, bl);
+ ::encode(marker, bl);
+ ::encode(truncated, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(entries, bl);
+ ::decode(marker, bl);
+ ::decode(truncated, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_list_buckets_ret*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_list_buckets_ret)
+
+
+struct cls_user_get_header_op {
+ cls_user_get_header_op() {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_get_header_op*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_get_header_op)
+
+struct cls_user_get_header_ret {
+ cls_user_header header;
+
+ cls_user_get_header_ret() {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(header, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(header, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_get_header_ret*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_get_header_ret)
+
+struct cls_user_complete_stats_sync_op {
+ utime_t time;
+
+ cls_user_complete_stats_sync_op() {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(time, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(time, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_complete_stats_sync_op*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_complete_stats_sync_op)
+
+
+#endif
diff --git a/src/cls/user/cls_user_types.cc b/src/cls/user/cls_user_types.cc
new file mode 100644
index 0000000..1ccd19b
--- /dev/null
+++ b/src/cls/user/cls_user_types.cc
@@ -0,0 +1,107 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "cls/user/cls_user_types.h"
+#include "common/Formatter.h"
+#include "common/ceph_json.h"
+
+void cls_user_gen_test_bucket(cls_user_bucket *bucket, int i)
+{
+ char buf[16];
+ snprintf(buf, sizeof(buf), ".%d", i);
+
+ bucket->name = string("buck") + buf;
+ bucket->data_pool = string(".data.pool") + buf;
+ bucket->index_pool = string(".index.pool") + buf;
+ bucket->marker = string("mark") + buf;
+ bucket->bucket_id = string("bucket.id") + buf;
+}
+
+void cls_user_bucket::dump(Formatter *f) const
+{
+ encode_json("name", name, f);
+ encode_json("data_pool", data_pool,f);
+ encode_json("index_pool", index_pool,f);
+ encode_json("marker", marker,f);
+ encode_json("bucket_id", bucket_id,f);
+}
+
+void cls_user_bucket::generate_test_instances(list<cls_user_bucket*>& ls)
+{
+ ls.push_back(new cls_user_bucket);
+ cls_user_bucket *b = new cls_user_bucket;
+ cls_user_gen_test_bucket(b, 0);
+ ls.push_back(b);
+}
+
+void cls_user_bucket_entry::dump(Formatter *f) const
+{
+ encode_json("bucket", bucket, f);
+ encode_json("size", size, f);
+ encode_json("size_rounded", size_rounded, f);
+ encode_json("creation_time", creation_time, f);
+ encode_json("count", count, f);
+ encode_json("user_stats_sync", user_stats_sync, f);
+}
+
+void cls_user_gen_test_bucket_entry(cls_user_bucket_entry *entry, int i)
+{
+ cls_user_gen_test_bucket(&entry->bucket, i);
+ entry->size = i + 1;
+ entry->size_rounded = i + 2;
+ entry->creation_time = i + 3;
+ entry->count = i + 4;
+ entry->user_stats_sync = true;
+}
+
+void cls_user_bucket_entry::generate_test_instances(list<cls_user_bucket_entry*>& ls)
+{
+ ls.push_back(new cls_user_bucket_entry);
+ cls_user_bucket_entry *entry = new cls_user_bucket_entry;
+ cls_user_gen_test_bucket_entry(entry, 0);
+ ls.push_back(entry);
+}
+
+void cls_user_gen_test_stats(cls_user_stats *s)
+{
+ s->total_entries = 1;
+ s->total_bytes = 2;
+ s->total_bytes_rounded = 3;
+}
+
+void cls_user_stats::dump(Formatter *f) const
+{
+ f->dump_int("total_entries", total_entries);
+ f->dump_int("total_bytes", total_bytes);
+ f->dump_int("total_bytes_rounded", total_bytes_rounded);
+}
+
+void cls_user_stats::generate_test_instances(list<cls_user_stats*>& ls)
+{
+ ls.push_back(new cls_user_stats);
+ cls_user_stats *s = new cls_user_stats;
+ cls_user_gen_test_stats(s);
+ ls.push_back(s);
+}
+
+void cls_user_gen_test_header(cls_user_header *h)
+{
+ cls_user_gen_test_stats(&h->stats);
+ h->last_stats_sync = utime_t(1, 0);
+ h->last_stats_update = utime_t(2, 0);
+}
+
+void cls_user_header::dump(Formatter *f) const
+{
+ encode_json("stats", stats, f);
+ encode_json("last_stats_sync", last_stats_sync, f);
+ encode_json("last_stats_update", last_stats_update, f);
+}
+
+void cls_user_header::generate_test_instances(list<cls_user_header*>& ls)
+{
+ ls.push_back(new cls_user_header);
+ cls_user_header *h = new cls_user_header;
+ cls_user_gen_test_header(h);
+ ls.push_back(h);
+}
diff --git a/src/cls/user/cls_user_types.h b/src/cls/user/cls_user_types.h
new file mode 100644
index 0000000..fce25c3
--- /dev/null
+++ b/src/cls/user/cls_user_types.h
@@ -0,0 +1,182 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_CLS_USER_TYPES_H
+#define CEPH_CLS_USER_TYPES_H
+
+#include "include/encoding.h"
+#include "include/types.h"
+#include "include/utime.h"
+
+/*
+ * this needs to be compatible with with rgw_bucket, as it replaces it
+ */
+struct cls_user_bucket {
+ std::string name;
+ std::string data_pool;
+ std::string index_pool;
+ std::string marker;
+ std::string bucket_id;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(6, 3, bl);
+ ::encode(name, bl);
+ ::encode(data_pool, bl);
+ ::encode(marker, bl);
+ ::encode(bucket_id, bl);
+ ::encode(index_pool, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
+ ::decode(name, bl);
+ ::decode(data_pool, bl);
+ if (struct_v >= 2) {
+ ::decode(marker, bl);
+ if (struct_v <= 3) {
+ uint64_t id;
+ ::decode(id, bl);
+ char buf[16];
+ snprintf(buf, sizeof(buf), "%llu", (long long)id);
+ bucket_id = buf;
+ } else {
+ ::decode(bucket_id, bl);
+ }
+ }
+ if (struct_v >= 5) {
+ ::decode(index_pool, bl);
+ } else {
+ index_pool = data_pool;
+ }
+ DECODE_FINISH(bl);
+ }
+
+ bool operator<(const cls_user_bucket& b) const {
+ return name.compare(b.name) < 0;
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_bucket*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_bucket)
+
+/*
+ * this structure overrides RGWBucketEnt
+ */
+struct cls_user_bucket_entry {
+ cls_user_bucket bucket;
+ size_t size;
+ size_t size_rounded;
+ time_t creation_time;
+ uint64_t count;
+ bool user_stats_sync;
+
+ cls_user_bucket_entry() : size(0), size_rounded(0), creation_time(0), count(0), user_stats_sync(false) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(6, 5, bl);
+ uint64_t s = size;
+ __u32 mt = creation_time;
+ string empty_str; // originally had the bucket name here, but we encode bucket later
+ ::encode(empty_str, bl);
+ ::encode(s, bl);
+ ::encode(mt, bl);
+ ::encode(count, bl);
+ ::encode(bucket, bl);
+ s = size_rounded;
+ ::encode(s, bl);
+ ::encode(user_stats_sync, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl);
+ __u32 mt;
+ uint64_t s;
+ string empty_str; // backward compatibility
+ ::decode(empty_str, bl);
+ ::decode(s, bl);
+ ::decode(mt, bl);
+ size = s;
+ creation_time = mt;
+ if (struct_v >= 2)
+ ::decode(count, bl);
+ if (struct_v >= 3)
+ ::decode(bucket, bl);
+ if (struct_v >= 4)
+ ::decode(s, bl);
+ size_rounded = s;
+ if (struct_v >= 6)
+ ::decode(user_stats_sync, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_bucket_entry*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_bucket_entry)
+
+struct cls_user_stats {
+ uint64_t total_entries;
+ uint64_t total_bytes;
+ uint64_t total_bytes_rounded;
+
+ cls_user_stats()
+ : total_entries(0),
+ total_bytes(0),
+ total_bytes_rounded(0) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(total_entries, bl);
+ ::encode(total_bytes, bl);
+ ::encode(total_bytes_rounded, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(total_entries, bl);
+ ::decode(total_bytes, bl);
+ ::decode(total_bytes_rounded, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_stats*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_stats)
+
+/*
+ * this needs to be compatible with with rgw_bucket, as it replaces it
+ */
+struct cls_user_header {
+ cls_user_stats stats;
+ utime_t last_stats_sync; /* last time a full stats sync completed */
+ utime_t last_stats_update; /* last time a stats update was done */
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(stats, bl);
+ ::encode(last_stats_sync, bl);
+ ::encode(last_stats_update, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(stats, bl);
+ ::decode(last_stats_sync, bl);
+ ::decode(last_stats_update, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<cls_user_header*>& ls);
+};
+WRITE_CLASS_ENCODER(cls_user_header)
+
+void cls_user_gen_test_bucket(cls_user_bucket *bucket, int i);
+void cls_user_gen_test_bucket_entry(cls_user_bucket_entry *entry, int i);
+void cls_user_gen_test_stats(cls_user_stats *stats);
+void cls_user_gen_test_header(cls_user_header *h);
+
+#endif
+
+
diff --git a/src/common/Finisher.cc b/src/common/Finisher.cc
index a1462a1..0cbfc44 100644
--- a/src/common/Finisher.cc
+++ b/src/common/Finisher.cc
@@ -80,6 +80,7 @@ void *Finisher::finisher_thread_entry()
finisher_empty_cond.Signal();
ldout(cct, 10) << "finisher_thread stop" << dendl;
+ finisher_stop = false;
finisher_lock.Unlock();
return 0;
}
diff --git a/src/common/Makefile.am b/src/common/Makefile.am
index 080e276..ec1f31e 100644
--- a/src/common/Makefile.am
+++ b/src/common/Makefile.am
@@ -29,6 +29,7 @@ libcommon_la_SOURCES = \
common/safe_io.c \
common/snap_types.cc \
common/str_list.cc \
+ common/str_map.cc \
common/errno.cc \
common/RefCountedObj.cc \
common/blkdev.cc \
@@ -66,10 +67,12 @@ libcommon_la_SOURCES = \
common/ceph_frag.cc \
common/addr_parsing.c \
common/hobject.cc \
- common/bloom_filter.cc
+ common/bloom_filter.cc \
+ common/linux_version.c
if LINUX
-libcommon_la_SOURCES += common/secret.c
+libcommon_la_SOURCES += \
+ common/secret.c
endif
# these should go out of libcommon
@@ -79,6 +82,7 @@ libcommon_la_SOURCES += \
mon/MonMap.cc \
osd/OSDMap.cc \
osd/osd_types.cc \
+ osd/HitSet.cc \
mds/MDSMap.cc \
mds/inode_backtrace.cc \
mds/mdstypes.cc
@@ -108,7 +112,7 @@ noinst_HEADERS += \
LIBCOMMON_DEPS += \
$(LIBMSG) $(LIBAUTH) \
$(LIBCRUSH) $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) \
- -lkeyutils
+ $(KEYUTILS_LIB)
if LINUX
LIBCOMMON_DEPS += -lrt
@@ -195,7 +199,8 @@ noinst_HEADERS += \
common/AsyncReserver.h \
common/sync_filesystem.h \
common/cmdparse.h \
- common/hobject.h
+ common/hobject.h \
+ common/linux_version.h
noinst_LTLIBRARIES += libcommon.la
diff --git a/src/common/TrackedOp.h b/src/common/TrackedOp.h
index 44e0390..9918482 100644
--- a/src/common/TrackedOp.h
+++ b/src/common/TrackedOp.h
@@ -20,10 +20,10 @@
#include "include/histogram.h"
#include "include/xlist.h"
#include "msg/Message.h"
-#include <tr1/memory>
+#include "include/memory.h"
class TrackedOp;
-typedef std::tr1::shared_ptr<TrackedOp> TrackedOpRef;
+typedef ceph::shared_ptr<TrackedOp> TrackedOpRef;
class OpTracker;
class OpHistory {
@@ -31,12 +31,11 @@ class OpHistory {
set<pair<double, TrackedOpRef> > duration;
void cleanup(utime_t now);
bool shutdown;
- OpTracker *tracker;
uint32_t history_size;
uint32_t history_duration;
public:
- OpHistory(OpTracker *tracker_) : shutdown(false), tracker(tracker_),
+ OpHistory() : shutdown(false),
history_size(0), history_duration(0) {}
~OpHistory() {
assert(arrived.empty());
@@ -70,7 +69,7 @@ class OpTracker {
public:
CephContext *cct;
OpTracker(CephContext *cct_) : seq(0), ops_in_flight_lock("OpTracker mutex"),
- history(this), complaint_time(0), log_threshold(0), cct(cct_) {}
+ complaint_time(0), log_threshold(0), cct(cct_) {}
void set_complaint_and_threshold(float time, int threshold) {
complaint_time = time;
log_threshold = threshold;
diff --git a/src/common/WorkQueue.cc b/src/common/WorkQueue.cc
index 6b648a7..f47435b 100644
--- a/src/common/WorkQueue.cc
+++ b/src/common/WorkQueue.cc
@@ -210,6 +210,7 @@ void ThreadPool::stop(bool clear_after)
_lock.Lock();
for (unsigned i=0; i<work_queues.size(); i++)
work_queues[i]->_clear();
+ _stop = false;
_lock.Unlock();
ldout(cct,15) << "stopped" << dendl;
}
diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc
index 12e5868..46c6f3d 100644
--- a/src/common/admin_socket.cc
+++ b/src/common/admin_socket.cc
@@ -16,6 +16,7 @@
#include "common/Thread.h"
#include "common/admin_socket.h"
+#include "common/admin_socket_client.h"
#include "common/config.h"
#include "common/cmdparse.h"
#include "common/dout.h"
@@ -184,15 +185,21 @@ std::string AdminSocket::bind_and_listen(const std::string &sock_path, int *fd)
sizeof(struct sockaddr_un)) != 0) {
int err = errno;
if (err == EADDRINUSE) {
- // The old UNIX domain socket must still be there.
- // Let's unlink it and try again.
- TEMP_FAILURE_RETRY(unlink(sock_path.c_str()));
- if (bind(sock_fd, (struct sockaddr*)&address,
- sizeof(struct sockaddr_un)) == 0) {
- err = 0;
- }
- else {
- err = errno;
+ AdminSocketClient client(sock_path);
+ bool ok;
+ client.ping(&ok);
+ if (ok) {
+ ldout(m_cct, 20) << "socket " << sock_path << " is in use" << dendl;
+ err = EEXIST;
+ } else {
+ ldout(m_cct, 20) << "unlink stale file " << sock_path << dendl;
+ TEMP_FAILURE_RETRY(unlink(sock_path.c_str()));
+ if (bind(sock_fd, (struct sockaddr*)&address,
+ sizeof(struct sockaddr_un)) == 0) {
+ err = 0;
+ } else {
+ err = errno;
+ }
}
}
if (err != 0) {
diff --git a/src/common/admin_socket_client.cc b/src/common/admin_socket_client.cc
index 335695f..8750560 100644
--- a/src/common/admin_socket_client.cc
+++ b/src/common/admin_socket_client.cc
@@ -127,6 +127,14 @@ AdminSocketClient(const std::string &path)
{
}
+std::string AdminSocketClient::ping(bool *ok)
+{
+ std::string version;
+ std::string result = do_request("{\"prefix\":\"0\"}", &version);
+ *ok = result == "" && version.length() == 1;
+ return result;
+}
+
std::string AdminSocketClient::do_request(std::string request, std::string *result)
{
int socket_fd = 0, res;
diff --git a/src/common/admin_socket_client.h b/src/common/admin_socket_client.h
index 45654bb..b46ed24 100644
--- a/src/common/admin_socket_client.h
+++ b/src/common/admin_socket_client.h
@@ -26,6 +26,7 @@ class AdminSocketClient
public:
AdminSocketClient(const std::string &path);
std::string do_request(std::string request, std::string *result);
+ std::string ping(bool *ok);
private:
std::string m_path;
};
diff --git a/src/common/blkdev.cc b/src/common/blkdev.cc
index b0dc0a5..9c7240c 100644
--- a/src/common/blkdev.cc
+++ b/src/common/blkdev.cc
@@ -1,40 +1,56 @@
-#include "include/int_types.h"
-
-#include <fcntl.h>
-#include <sys/ioctl.h>
#include <errno.h>
#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mount.h>
-#include <iostream>
-
-#include "acconfig.h"
-#include "include/compat.h"
+#include <sys/ioctl.h>
+#include "include/int_types.h"
-#if defined(__FreeBSD__)
-#include <sys/disk.h>
-#endif
+#ifdef __linux__
+#include <linux/fs.h>
int get_block_device_size(int fd, int64_t *psize)
{
- int ret = 0;
-
-#if defined(__FreeBSD__)
- ret = ::ioctl(fd, DIOCGMEDIASIZE, psize);
-#elif defined(__linux__)
#ifdef BLKGETSIZE64
- // ioctl block device
- ret = ::ioctl(fd, BLKGETSIZE64, psize);
+ int ret = ::ioctl(fd, BLKGETSIZE64, psize);
#elif BLKGETSIZE
- // hrm, try the 32 bit ioctl?
unsigned long sectors = 0;
- ret = ::ioctl(fd, BLKGETSIZE, §ors);
+ int ret = ::ioctl(fd, BLKGETSIZE, §ors);
*psize = sectors * 512ULL;
-#endif
#else
-#error "Compile error: we don't know how to get the size of a raw block device."
-#endif /* !__FreeBSD__ */
+# error "Linux configuration error (get_block_device_size)"
+#endif
if (ret < 0)
ret = -errno;
return ret;
}
+
+#elif defined(__APPLE__)
+#include <sys/disk.h>
+
+int get_block_device_size(int fd, int64_t *psize)
+{
+ unsigned long blocksize = 0;
+ int ret = ::ioctl(fd, DKIOCGETBLOCKSIZE, &blocksize);
+ if (!ret) {
+ unsigned long nblocks;
+ ret = ::ioctl(fd, DKIOCGETBLOCKCOUNT, &nblocks);
+ if (!ret)
+ *psize = (int64_t)nblocks * blocksize;
+ }
+ if (ret < 0)
+ ret = -errno;
+ return ret;
+}
+
+#elif defined(__FreeBSD__)
+#include <sys/disk.h>
+
+int get_block_device_size(int fd, int64_t *psize)
+{
+ int ret = ::ioctl(fd, DIOCGMEDIASIZE, psize);
+ if (ret < 0)
+ ret = -errno;
+ return ret;
+}
+
+#else
+# error "Unable to query block device size: unsupported platform, please report."
+#endif
diff --git a/src/common/bloom_filter.hpp b/src/common/bloom_filter.hpp
index 93787a8..735d6c4 100644
--- a/src/common/bloom_filter.hpp
+++ b/src/common/bloom_filter.hpp
@@ -80,6 +80,7 @@ public:
target_element_count_(predicted_inserted_element_count),
random_seed_((random_seed) ? random_seed : 0xA5A5A5A5)
{
+ assert(false_positive_probability > 0.0);
find_optimal_parameters(predicted_inserted_element_count, false_positive_probability,
&salt_count_, &table_size_);
init();
@@ -110,6 +111,7 @@ public:
}
bloom_filter(const bloom_filter& filter)
+ : bit_table_(0)
{
this->operator=(filter);
}
@@ -120,6 +122,7 @@ public:
salt_count_ = filter.salt_count_;
table_size_ = filter.table_size_;
insert_count_ = filter.insert_count_;
+ target_element_count_ = filter.target_element_count_;
random_seed_ = filter.random_seed_;
delete[] bit_table_;
bit_table_ = new cell_type[table_size_];
@@ -307,6 +310,11 @@ public:
return insert_count_;
}
+ inline bool is_full() const
+ {
+ return insert_count_ >= target_element_count_;
+ }
+
/*
* density of bits set. inconvenient units, but:
* .3 = ~50% target insertions
diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index 4930705..eab9143 100644
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -18,6 +18,7 @@
#include "common/errno.h"
#include "common/safe_io.h"
#include "common/simple_spin.h"
+#include "common/strtol.h"
#include "include/atomic.h"
#include "include/types.h"
#include "include/compat.h"
@@ -69,6 +70,52 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
return buffer_cached_crc_adjusted.read();
}
+ atomic_t buffer_c_str_accesses;
+ bool buffer_track_c_str = get_env_bool("CEPH_BUFFER_TRACK");
+
+ void buffer::track_c_str(bool b) {
+ buffer_track_c_str = b;
+ }
+ int buffer::get_c_str_accesses() {
+ return buffer_c_str_accesses.read();
+ }
+
+ atomic_t buffer_max_pipe_size;
+ int update_max_pipe_size() {
+#ifdef CEPH_HAVE_SETPIPE_SZ
+ char buf[32];
+ int r;
+ std::string err;
+ struct stat stat_result;
+ if (::stat("/proc/sys/fs/pipe-max-size", &stat_result) == -1)
+ return -errno;
+ r = safe_read_file("/proc/sys/fs/", "pipe-max-size",
+ buf, sizeof(buf) - 1);
+ if (r < 0)
+ return r;
+ buf[r] = '\0';
+ size = strict_strtol(buf, 10, &err);
+ if (!err.empty())
+ return -EIO;
+ buffer_max_pipe_size.set(size);
+#endif
+ return 0;
+ }
+
+ size_t get_max_pipe_size() {
+#ifdef CEPH_HAVE_SETPIPE_SZ
+ size_t size = buffer_max_pipe_size.read();
+ if (size)
+ return size;
+ if (update_max_pipe_size() == 0)
+ return buffer_max_pipe_size.read()
+#endif
+ // this is the max size hardcoded in linux before 2.6.35
+ return 65536;
+ }
+
+ buffer::error_code::error_code(int error) :
+ buffer::malformed_input(cpp_strerror(error).c_str()), code(error) {}
class buffer::raw {
public:
@@ -89,18 +136,22 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
raw(const raw &other);
const raw& operator=(const raw &other);
+ virtual char *get_data() {
+ return data;
+ }
virtual raw* clone_empty() = 0;
raw *clone() {
raw *c = clone_empty();
memcpy(c->data, data, len);
return c;
}
-
- unsigned length() const {
- return len;
+ virtual bool can_zero_copy() const {
+ return false;
}
-
- bool is_page_aligned() {
+ virtual int zero_copy_to_fd(int fd, loff_t *offset) {
+ return -ENOTSUP;
+ }
+ virtual bool is_page_aligned() {
return ((long)data & ~CEPH_PAGE_MASK) == 0;
}
bool is_n_page_sized() {
@@ -228,6 +279,187 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
};
#endif
+#ifdef CEPH_HAVE_SPLICE
+ class buffer::raw_pipe : public buffer::raw {
+ public:
+ raw_pipe(unsigned len) : raw(len), source_consumed(false) {
+ size_t max = get_max_pipe_size();
+ if (len > max) {
+ bdout << "raw_pipe: requested length " << len
+ << " > max length " << max << bendl;
+ throw malformed_input("length larger than max pipe size");
+ }
+ pipefds[0] = -1;
+ pipefds[1] = -1;
+
+ int r;
+ if (::pipe(pipefds) == -1) {
+ r = -errno;
+ bdout << "raw_pipe: error creating pipe: " << cpp_strerror(r) << bendl;
+ throw error_code(r);
+ }
+
+ r = set_nonblocking(pipefds);
+ if (r < 0) {
+ bdout << "raw_pipe: error setting nonblocking flag on temp pipe: "
+ << cpp_strerror(r) << bendl;
+ throw error_code(r);
+ }
+
+ r = set_pipe_size(pipefds, len);
+ if (r < 0) {
+ bdout << "raw_pipe: could not set pipe size" << bendl;
+ // continue, since the pipe should become large enough as needed
+ }
+
+ inc_total_alloc(len);
+ bdout << "raw_pipe " << this << " alloc " << len << " "
+ << buffer::get_total_alloc() << bendl;
+ }
+
+ ~raw_pipe() {
+ if (data)
+ delete data;
+ close_pipe(pipefds);
+ dec_total_alloc(len);
+ bdout << "raw_pipe " << this << " free " << (void *)data << " "
+ << buffer::get_total_alloc() << bendl;
+ }
+
+ bool can_zero_copy() const {
+ return true;
+ }
+
+ bool is_page_aligned() {
+ return false;
+ }
+
+ int set_source(int fd, loff_t *off) {
+ int flags = SPLICE_F_NONBLOCK;
+ ssize_t r = safe_splice(fd, off, pipefds[1], NULL, len, flags);
+ if (r < 0) {
+ bdout << "raw_pipe: error splicing into pipe: " << cpp_strerror(r)
+ << bendl;
+ return r;
+ }
+ // update length with actual amount read
+ len = r;
+ return 0;
+ }
+
+ int zero_copy_to_fd(int fd, loff_t *offset) {
+ assert(!source_consumed);
+ int flags = SPLICE_F_NONBLOCK;
+ ssize_t r = safe_splice_exact(pipefds[0], NULL, fd, offset, len, flags);
+ if (r < 0) {
+ bdout << "raw_pipe: error splicing from pipe to fd: "
+ << cpp_strerror(r) << bendl;
+ return r;
+ }
+ source_consumed = true;
+ return 0;
+ }
+
+ buffer::raw* clone_empty() {
+ // cloning doesn't make sense for pipe-based buffers,
+ // and is only used by unit tests for other types of buffers
+ return NULL;
+ }
+
+ char *get_data() {
+ if (data)
+ return data;
+ return copy_pipe(pipefds);
+ }
+
+ private:
+ int set_pipe_size(int *fds, long length) {
+#ifdef CEPH_HAVE_SETPIPE_SZ
+ if (::fcntl(fds[1], F_SETPIPE_SZ, length) == -1) {
+ int r = -errno;
+ if (r == -EPERM) {
+ // pipe limit must have changed - EPERM means we requested
+ // more than the maximum size as an unprivileged user
+ update_max_pipe_size();
+ throw malformed_input("length larger than new max pipe size");
+ }
+ return r;
+ }
+#endif
+ return 0;
+ }
+
+ int set_nonblocking(int *fds) {
+ if (::fcntl(fds[0], F_SETFL, O_NONBLOCK) == -1)
+ return -errno;
+ if (::fcntl(fds[1], F_SETFL, O_NONBLOCK) == -1)
+ return -errno;
+ return 0;
+ }
+
+ void close_pipe(int *fds) {
+ if (fds[0] >= 0)
+ TEMP_FAILURE_RETRY(::close(fds[0]));
+ if (fds[1] >= 0)
+ TEMP_FAILURE_RETRY(::close(fds[1]));
+ }
+ char *copy_pipe(int *fds) {
+ /* preserve original pipe contents by copying into a temporary
+ * pipe before reading.
+ */
+ int tmpfd[2];
+ int r;
+
+ assert(!source_consumed);
+ assert(fds[0] >= 0);
+
+ if (::pipe(tmpfd) == -1) {
+ r = -errno;
+ bdout << "raw_pipe: error creating temp pipe: " << cpp_strerror(r)
+ << bendl;
+ throw error_code(r);
+ }
+ r = set_nonblocking(tmpfd);
+ if (r < 0) {
+ bdout << "raw_pipe: error setting nonblocking flag on temp pipe: "
+ << cpp_strerror(r) << bendl;
+ throw error_code(r);
+ }
+ r = set_pipe_size(tmpfd, len);
+ if (r < 0) {
+ bdout << "raw_pipe: error setting pipe size on temp pipe: "
+ << cpp_strerror(r) << bendl;
+ }
+ int flags = SPLICE_F_NONBLOCK;
+ if (::tee(fds[0], tmpfd[1], len, flags) == -1) {
+ r = errno;
+ bdout << "raw_pipe: error tee'ing into temp pipe: " << cpp_strerror(r)
+ << bendl;
+ close_pipe(tmpfd);
+ throw error_code(r);
+ }
+ data = (char *)malloc(len);
+ if (!data) {
+ close_pipe(tmpfd);
+ throw bad_alloc();
+ }
+ r = safe_read(tmpfd[0], data, len);
+ if (r < (ssize_t)len) {
+ bdout << "raw_pipe: error reading from temp pipe:" << cpp_strerror(r)
+ << bendl;
+ delete data;
+ data = NULL;
+ close_pipe(tmpfd);
+ throw error_code(r);
+ }
+ close_pipe(tmpfd);
+ return data;
+ }
+ bool source_consumed;
+ int pipefds[2];
+ };
+#endif // CEPH_HAVE_SPLICE
+
/*
* primitive buffer types
*/
@@ -293,6 +525,20 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
#endif
}
+ buffer::raw* buffer::create_zero_copy(unsigned len, int fd, int64_t *offset) {
+#ifdef CEPH_HAVE_SPLICE
+ buffer::raw_pipe* buf = new raw_pipe(len);
+ int r = buf->set_source(fd, (loff_t*)offset);
+ if (r < 0) {
+ delete buf;
+ throw error_code(r);
+ }
+ return buf;
+#else
+ throw error_code(-ENOTSUP);
+#endif
+ }
+
buffer::ptr::ptr(raw *r) : _raw(r), _off(0), _len(r->len) // no lock needed; this is an unref raw.
{
r->nref.inc();
@@ -375,8 +621,18 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
bool buffer::ptr::at_buffer_tail() const { return _off + _len == _raw->len; }
- const char *buffer::ptr::c_str() const { assert(_raw); return _raw->data + _off; }
- char *buffer::ptr::c_str() { assert(_raw); return _raw->data + _off; }
+ const char *buffer::ptr::c_str() const {
+ assert(_raw);
+ if (buffer_track_c_str)
+ buffer_c_str_accesses.inc();
+ return _raw->get_data() + _off;
+ }
+ char *buffer::ptr::c_str() {
+ assert(_raw);
+ if (buffer_track_c_str)
+ buffer_c_str_accesses.inc();
+ return _raw->get_data() + _off;
+ }
unsigned buffer::ptr::unused_tail_length() const
{
@@ -389,13 +645,13 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
{
assert(_raw);
assert(n < _len);
- return _raw->data[_off + n];
+ return _raw->get_data()[_off + n];
}
char& buffer::ptr::operator[](unsigned n)
{
assert(_raw);
assert(n < _len);
- return _raw->data[_off + n];
+ return _raw->get_data()[_off + n];
}
const char *buffer::ptr::raw_c_str() const { assert(_raw); return _raw->data; }
@@ -408,7 +664,7 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
return _raw->len - _len;
}
- int buffer::ptr::cmp(const ptr& o)
+ int buffer::ptr::cmp(const ptr& o) const
{
int l = _len < o._len ? _len : o._len;
if (l) {
@@ -472,6 +728,15 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
memset(c_str()+o, 0, l);
}
+ bool buffer::ptr::can_zero_copy() const
+ {
+ return _raw->can_zero_copy();
+ }
+
+ int buffer::ptr::zero_copy_to_fd(int fd, int64_t *offset) const
+ {
+ return _raw->zero_copy_to_fd(fd, (loff_t*)offset);
+ }
// -- buffer::list::iterator --
/*
@@ -734,6 +999,16 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
}
}
+ bool buffer::list::can_zero_copy() const
+ {
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it)
+ if (!it->can_zero_copy())
+ return false;
+ return true;
+ }
+
bool buffer::list::is_page_aligned() const
{
for (std::list<ptr>::const_iterator it = _buffers.begin();
@@ -806,6 +1081,11 @@ static uint32_t simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZE
nb = buffer::create_page_aligned(_len);
else
nb = buffer::create(_len);
+ rebuild(nb);
+ }
+
+ void buffer::list::rebuild(ptr& nb)
+ {
unsigned pos = 0;
for (std::list<ptr>::iterator it = _buffers.begin();
it != _buffers.end();
@@ -849,7 +1129,8 @@ void buffer::list::rebuild_page_aligned()
(!p->is_page_aligned() ||
!p->is_n_page_sized() ||
(offset & ~CEPH_PAGE_MASK)));
- unaligned.rebuild();
+ ptr nb(buffer::create_page_aligned(unaligned._len));
+ unaligned.rebuild(nb);
_buffers.insert(p, unaligned._buffers.front());
}
}
@@ -998,7 +1279,8 @@ void buffer::list::rebuild_page_aligned()
std::string s;
getline(in, s);
append(s.c_str(), s.length());
- append("\n", 1);
+ if (s.length())
+ append("\n", 1);
}
}
@@ -1039,7 +1321,7 @@ void buffer::list::rebuild_page_aligned()
return 0; // no buffers
std::list<ptr>::const_iterator iter = _buffers.begin();
- iter++;
+ ++iter;
if (iter != _buffers.end())
rebuild();
@@ -1087,6 +1369,9 @@ void buffer::list::rebuild_page_aligned()
// funky modifer
void buffer::list::splice(unsigned off, unsigned len, list *claim_by /*, bufferlist& replace_with */)
{ // fixme?
+ if (len == 0)
+ return;
+
if (off >= length())
throw end_of_buffer();
@@ -1226,8 +1511,14 @@ int buffer::list::read_file(const char *fn, std::string *error)
return 0;
}
-ssize_t buffer::list::read_fd(int fd, size_t len)
+ssize_t buffer::list::read_fd(int fd, size_t len)
{
+ // try zero copy first
+ if (false && read_fd_zero_copy(fd, len) == 0) {
+ // TODO fix callers to not require correct read size, which is not
+ // available for raw_pipe until we actually inspect the data
+ return 0;
+ }
int s = ROUND_UP_TO(len, CEPH_PAGE_SIZE);
bufferptr bp = buffer::create_page_aligned(s);
ssize_t ret = safe_read(fd, (void*)bp.c_str(), len);
@@ -1238,6 +1529,23 @@ ssize_t buffer::list::read_fd(int fd, size_t len)
return ret;
}
+int buffer::list::read_fd_zero_copy(int fd, size_t len)
+{
+#ifdef CEPH_HAVE_SPLICE
+ try {
+ bufferptr bp = buffer::create_zero_copy(len, fd, NULL);
+ append(bp);
+ } catch (buffer::error_code e) {
+ return e.code;
+ } catch (buffer::malformed_input) {
+ return -EIO;
+ }
+ return 0;
+#else
+ return -ENOTSUP;
+#endif
+}
+
int buffer::list::write_file(const char *fn, int mode)
{
int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC, mode));
@@ -1265,12 +1573,15 @@ int buffer::list::write_file(const char *fn, int mode)
int buffer::list::write_fd(int fd) const
{
+ if (can_zero_copy())
+ return write_fd_zero_copy(fd);
+
// use writev!
iovec iov[IOV_MAX];
int iovlen = 0;
ssize_t bytes = 0;
- std::list<ptr>::const_iterator p = _buffers.begin();
+ std::list<ptr>::const_iterator p = _buffers.begin();
while (p != _buffers.end()) {
if (p->length() > 0) {
iov[iovlen].iov_base = (void *)p->c_str();
@@ -1315,6 +1626,30 @@ int buffer::list::write_fd(int fd) const
return 0;
}
+int buffer::list::write_fd_zero_copy(int fd) const
+{
+ if (!can_zero_copy())
+ return -ENOTSUP;
+ /* pass offset to each call to avoid races updating the fd seek
+ * position, since the I/O may be non-blocking
+ */
+ int64_t offset = ::lseek(fd, 0, SEEK_CUR);
+ int64_t *off_p = &offset;
+ if (offset < 0 && offset != ESPIPE)
+ return (int) offset;
+ if (offset == ESPIPE)
+ off_p = NULL;
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end(); ++it) {
+ int r = it->zero_copy_to_fd(fd, off_p);
+ if (r < 0)
+ return r;
+ if (off_p)
+ offset += it->length();
+ }
+ return 0;
+}
+
__u32 buffer::list::crc32c(__u32 crc) const
{
for (std::list<ptr>::const_iterator it = _buffers.begin();
diff --git a/src/common/ceph_argparse.cc b/src/common/ceph_argparse.cc
index 6c80538..6d1c613 100644
--- a/src/common/ceph_argparse.cc
+++ b/src/common/ceph_argparse.cc
@@ -64,22 +64,13 @@ void env_to_vec(std::vector<const char*>& args, const char *name)
if (!p)
return;
- static char buf[1000];
- int len = MIN(strlen(p), sizeof(buf)-1); // bleh.
- memcpy(buf, p, len);
- buf[len] = 0;
- //cout << "CEPH_ARGS='" << p << ";" << endl;
-
- p = buf;
- while (*p && p < buf + len) {
- char *e = p;
- while (*e && *e != ' ')
- e++;
- *e = 0;
- args.push_back(p);
- //cout << "arg " << p << std::endl;
- p = e+1;
- }
+ static vector<string> str_vec;
+ str_vec.clear();
+ get_str_vec(p, " ", str_vec);
+ for (vector<string>::iterator i = str_vec.begin();
+ i != str_vec.end();
+ i++)
+ args.push_back(i->c_str());
}
void argv_to_vec(int argc, const char **argv,
@@ -249,8 +240,7 @@ bool ceph_argparse_binary_flag(std::vector<const char*> &args,
}
static bool va_ceph_argparse_witharg(std::vector<const char*> &args,
- std::vector<const char*>::iterator &i, std::string *ret, bool cli,
- va_list ap)
+ std::vector<const char*>::iterator &i, std::string *ret, va_list ap)
{
const char *first = *i;
char tmp[strlen(first)+1];
@@ -274,12 +264,8 @@ static bool va_ceph_argparse_witharg(std::vector<const char*> &args,
else if (first[strlen_a] == '\0') {
// find second part (or not)
if (i+1 == args.end()) {
- if (cli) {
- cerr << "Option " << *i << " requires an argument." << std::endl;
- _exit(1);
- } else {
- return false;
- }
+ cerr << "Option " << *i << " requires an argument." << std::endl;
+ _exit(1);
}
i = args.erase(i);
*ret = *i;
@@ -296,21 +282,11 @@ bool ceph_argparse_witharg(std::vector<const char*> &args,
bool r;
va_list ap;
va_start(ap, ret);
- r = va_ceph_argparse_witharg(args, i, ret, false, ap);
+ r = va_ceph_argparse_witharg(args, i, ret, ap);
va_end(ap);
return r;
}
-bool ceph_argparse_witharg_daemon(std::vector<const char*> &args,
- std::vector<const char*>::iterator &i, std::string *ret, ...)
-{
- bool r;
- va_list ap;
- va_start(ap, ret);
- r = va_ceph_argparse_witharg(args, i, ret, false, ap);
- va_end(ap);
- return r;
-}
bool ceph_argparse_withint(std::vector<const char*> &args,
std::vector<const char*>::iterator &i, int *ret,
std::ostream *oss, ...)
@@ -319,30 +295,7 @@ bool ceph_argparse_withint(std::vector<const char*> &args,
va_list ap;
std::string str;
va_start(ap, oss);
- r = va_ceph_argparse_witharg(args, i, &str, true, ap);
- va_end(ap);
- if (!r) {
- return false;
- }
-
- std::string err;
- int myret = strict_strtol(str.c_str(), 10, &err);
- *ret = myret;
- if (!err.empty()) {
- *oss << err;
- }
- return true;
-}
-
-bool ceph_argparse_withint_daemon(std::vector<const char*> &args,
- std::vector<const char*>::iterator &i, int *ret,
- std::ostream *oss, ...)
-{
- bool r;
- va_list ap;
- std::string str;
- va_start(ap, oss);
- r = va_ceph_argparse_witharg(args, i, &str, false, ap);
+ r = va_ceph_argparse_witharg(args, i, &str, ap);
va_end(ap);
if (!r) {
return false;
@@ -365,7 +318,7 @@ bool ceph_argparse_withlonglong(std::vector<const char*> &args,
va_list ap;
std::string str;
va_start(ap, oss);
- r = va_ceph_argparse_witharg(args, i, &str, false, ap);
+ r = va_ceph_argparse_witharg(args, i, &str, ap);
va_end(ap);
if (!r) {
return false;
@@ -388,7 +341,7 @@ bool ceph_argparse_withfloat(std::vector<const char*> &args,
va_list ap;
std::string str;
va_start(ap, oss);
- r = va_ceph_argparse_witharg(args, i, &str, false, ap);
+ r = va_ceph_argparse_witharg(args, i, &str, ap);
va_end(ap);
if (!r) {
return false;
diff --git a/src/common/ceph_argparse.h b/src/common/ceph_argparse.h
index 5249704..3ef0251 100644
--- a/src/common/ceph_argparse.h
+++ b/src/common/ceph_argparse.h
@@ -57,8 +57,6 @@ bool ceph_argparse_flag(std::vector<const char*> &args,
std::vector<const char*>::iterator &i, ...);
bool ceph_argparse_witharg(std::vector<const char*> &args,
std::vector<const char*>::iterator &i, std::string *ret, ...);
-bool ceph_argparse_witharg_daemon(std::vector<const char*> &args,
- std::vector<const char*>::iterator &i, std::string *ret, ...);
bool ceph_argparse_binary_flag(std::vector<const char*> &args,
std::vector<const char*>::iterator &i, int *ret,
std::ostream *oss, ...);
@@ -68,9 +66,6 @@ extern CephInitParameters ceph_argparse_early_args
extern bool ceph_argparse_withint(std::vector<const char*> &args,
std::vector<const char*>::iterator &i, int *ret,
std::ostream *oss, ...);
-extern bool ceph_argparse_withint_daemon(std::vector<const char*> &args,
- std::vector<const char*>::iterator &i, int *ret,
- std::ostream *oss, ...);
extern bool ceph_argparse_withfloat(std::vector<const char*> &args,
std::vector<const char*>::iterator &i, float *ret,
std::ostream *oss, ...);
diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc
index e694a2f..f7c2df5 100644
--- a/src/common/ceph_context.cc
+++ b/src/common/ceph_context.cc
@@ -27,10 +27,13 @@
#include "log/Log.h"
#include "auth/Crypto.h"
#include "include/str_list.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
#include <iostream>
#include <pthread.h>
-#include <semaphore.h>
+
+#include "include/Spinlock.h"
using ceph::HeartbeatMap;
@@ -38,30 +41,28 @@ class CephContextServiceThread : public Thread
{
public:
CephContextServiceThread(CephContext *cct)
- : _reopen_logs(false), _exit_thread(false), _cct(cct)
+ : _lock("CephContextServiceThread::_lock"),
+ _reopen_logs(false), _exit_thread(false), _cct(cct)
{
- sem_init(&_sem, 0, 0);
- };
+ }
- ~CephContextServiceThread()
- {
- sem_destroy(&_sem);
- };
+ ~CephContextServiceThread() {}
void *entry()
{
while (1) {
+ Mutex::Locker l(_lock);
+
if (_cct->_conf->heartbeat_interval) {
- struct timespec timeout;
- clock_gettime(CLOCK_REALTIME, &timeout);
- timeout.tv_sec += _cct->_conf->heartbeat_interval;
- sem_timedwait(&_sem, &timeout);
- } else {
- sem_wait(&_sem);
- }
+ utime_t interval(_cct->_conf->heartbeat_interval, 0);
+ _cond.WaitInterval(_cct, _lock, interval);
+ } else
+ _cond.Wait(_lock);
+
if (_exit_thread) {
break;
}
+
if (_reopen_logs) {
_cct->_log->reopen_log_file();
_reopen_logs = false;
@@ -73,20 +74,23 @@ public:
void reopen_logs()
{
+ Mutex::Locker l(_lock);
_reopen_logs = true;
- sem_post(&_sem);
+ _cond.Signal();
}
void exit_thread()
{
+ Mutex::Locker l(_lock);
_exit_thread = true;
- sem_post(&_sem);
+ _cond.Signal();
}
private:
- volatile bool _reopen_logs;
- volatile bool _exit_thread;
- sem_t _sem;
+ Mutex _lock;
+ Cond _cond;
+ bool _reopen_logs;
+ bool _exit_thread;
CephContext *_cct;
};
@@ -258,7 +262,7 @@ CephContext::CephContext(uint32_t module_type_)
_crypto_none(NULL),
_crypto_aes(NULL)
{
- pthread_spin_init(&_service_thread_lock, PTHREAD_PROCESS_SHARED);
+ ceph_spin_init(&_service_thread_lock);
_log = new ceph::log::Log(&_conf->subsys);
_log->start();
@@ -290,12 +294,12 @@ CephContext::CephContext(uint32_t module_type_)
CephContext::~CephContext()
{
+ join_service_thread();
+
if (_conf->lockdep) {
lockdep_unregister_ceph_context(this);
}
- join_service_thread();
-
_admin_socket->unregister_command("perfcounters_dump");
_admin_socket->unregister_command("perf dump");
_admin_socket->unregister_command("1");
@@ -328,7 +332,7 @@ CephContext::~CephContext()
_log = NULL;
delete _conf;
- pthread_spin_destroy(&_service_thread_lock);
+ ceph_spin_destroy(&_service_thread_lock);
delete _crypto_none;
delete _crypto_aes;
@@ -336,14 +340,14 @@ CephContext::~CephContext()
void CephContext::start_service_thread()
{
- pthread_spin_lock(&_service_thread_lock);
+ ceph_spin_lock(&_service_thread_lock);
if (_service_thread) {
- pthread_spin_unlock(&_service_thread_lock);
+ ceph_spin_unlock(&_service_thread_lock);
return;
}
_service_thread = new CephContextServiceThread(this);
_service_thread->create();
- pthread_spin_unlock(&_service_thread_lock);
+ ceph_spin_unlock(&_service_thread_lock);
// make logs flush on_exit()
if (_conf->log_flush_on_exit)
@@ -361,22 +365,22 @@ void CephContext::start_service_thread()
void CephContext::reopen_logs()
{
- pthread_spin_lock(&_service_thread_lock);
+ ceph_spin_lock(&_service_thread_lock);
if (_service_thread)
_service_thread->reopen_logs();
- pthread_spin_unlock(&_service_thread_lock);
+ ceph_spin_unlock(&_service_thread_lock);
}
void CephContext::join_service_thread()
{
- pthread_spin_lock(&_service_thread_lock);
+ ceph_spin_lock(&_service_thread_lock);
CephContextServiceThread *thread = _service_thread;
if (!thread) {
- pthread_spin_unlock(&_service_thread_lock);
+ ceph_spin_unlock(&_service_thread_lock);
return;
}
_service_thread = NULL;
- pthread_spin_unlock(&_service_thread_lock);
+ ceph_spin_unlock(&_service_thread_lock);
thread->exit_thread();
thread->join();
diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h
index 08efece..ba60620 100644
--- a/src/common/ceph_context.h
+++ b/src/common/ceph_context.h
@@ -21,6 +21,7 @@
#include "include/buffer.h"
#include "include/atomic.h"
#include "common/cmdparse.h"
+#include "include/Spinlock.h"
class AdminSocket;
class CephContextServiceThread;
@@ -126,7 +127,7 @@ private:
AdminSocket *_admin_socket;
/* lock which protects service thread creation, destruction, etc. */
- pthread_spinlock_t _service_thread_lock;
+ ceph_spinlock_t _service_thread_lock;
/* The collection of profiling loggers associated with this context */
PerfCountersCollection *_perf_counters_collection;
diff --git a/src/common/ceph_strings.cc b/src/common/ceph_strings.cc
index 3eb5e67..686f504 100644
--- a/src/common/ceph_strings.cc
+++ b/src/common/ceph_strings.cc
@@ -19,6 +19,7 @@ const char *ceph_osd_op_name(int op)
{
switch (op) {
case CEPH_OSD_OP_READ: return "read";
+ case CEPH_OSD_OP_SYNC_READ: return "sync_read";
case CEPH_OSD_OP_STAT: return "stat";
case CEPH_OSD_OP_MAPEXT: return "mapext";
case CEPH_OSD_OP_SPARSE_READ: return "sparse-read";
@@ -53,6 +54,9 @@ const char *ceph_osd_op_name(int op)
case CEPH_OSD_OP_COPY_FROM: return "copy-from";
case CEPH_OSD_OP_UNDIRTY: return "undirty";
case CEPH_OSD_OP_ISDIRTY: return "isdirty";
+ case CEPH_OSD_OP_CACHE_FLUSH: return "cache-flush";
+ case CEPH_OSD_OP_CACHE_EVICT: return "cache-evict";
+ case CEPH_OSD_OP_CACHE_TRY_FLUSH: return "cache-try-flush";
case CEPH_OSD_OP_CLONERANGE: return "clonerange";
case CEPH_OSD_OP_ASSERT_SRC_VERSION: return "assert-src-version";
@@ -87,6 +91,8 @@ const char *ceph_osd_op_name(int op)
case CEPH_OSD_OP_PGLS: return "pgls";
case CEPH_OSD_OP_PGLS_FILTER: return "pgls-filter";
+ case CEPH_OSD_OP_PG_HITSET_LS: return "pg-hitset-ls";
+ case CEPH_OSD_OP_PG_HITSET_GET: return "pg-hitset-get";
case CEPH_OSD_OP_OMAPGETKEYS: return "omap-get-keys";
case CEPH_OSD_OP_OMAPGETVALS: return "omap-get-vals";
case CEPH_OSD_OP_OMAPGETHEADER: return "omap-get-header";
@@ -151,6 +157,8 @@ const char *ceph_session_op_name(int op)
case CEPH_SESSION_RENEWCAPS: return "renewcaps";
case CEPH_SESSION_STALE: return "stale";
case CEPH_SESSION_RECALL_STATE: return "recall_state";
+ case CEPH_SESSION_FLUSHMSG: return "flushmsg";
+ case CEPH_SESSION_FLUSHMSG_ACK: return "flushmsg_ack";
}
return "???";
}
diff --git a/src/common/common_init.cc b/src/common/common_init.cc
index 8fb688c..6538b18 100644
--- a/src/common/common_init.cc
+++ b/src/common/common_init.cc
@@ -112,10 +112,12 @@ void complain_about_parse_errors(CephContext *cct,
/* Please be sure that this can safely be called multiple times by the
* same application. */
-void common_init_finish(CephContext *cct)
+void common_init_finish(CephContext *cct, int flags)
{
ceph::crypto::init(cct);
- cct->start_service_thread();
+
+ if (!(flags & CINIT_FLAG_NO_DAEMON_ACTIONS))
+ cct->start_service_thread();
if (cct->_conf->lockdep) {
g_lockdep = true;
diff --git a/src/common/common_init.h b/src/common/common_init.h
index f48b349..d6aa9fa 100644
--- a/src/common/common_init.h
+++ b/src/common/common_init.h
@@ -75,6 +75,6 @@ void complain_about_parse_errors(CephContext *cct,
* libraries. The most obvious reason for this is that the threads started by
* the Ceph libraries would be destroyed by a fork().
*/
-void common_init_finish(CephContext *cct);
+void common_init_finish(CephContext *cct, int flags = 0);
#endif
diff --git a/src/common/config.cc b/src/common/config.cc
index 5c64f4e..a7f948f 100644
--- a/src/common/config.cc
+++ b/src/common/config.cc
@@ -48,6 +48,7 @@
#undef dendl
using std::map;
+using std::list;
using std::multimap;
using std::ostringstream;
using std::pair;
@@ -229,7 +230,7 @@ int md_config_t::parse_config_files_impl(const std::list<std::string> &conf_file
for (c = conf_files.begin(); c != conf_files.end(); ++c) {
cf.clear();
string fn = *c;
- expand_meta(fn);
+ expand_meta(fn, warnings);
int ret = cf.parse_file(fn.c_str(), parse_errors, warnings);
if (ret == 0)
break;
@@ -357,6 +358,10 @@ int md_config_t::parse_argv(std::vector<const char*>& args)
return -ENOSYS;
}
+ bool show_config = false;
+ bool show_config_value = false;
+ string show_config_value_arg;
+
// In this function, don't change any parts of the configuration directly.
// Instead, use set_val to set them. This will allow us to send the proper
// observer notifications later.
@@ -373,24 +378,11 @@ int md_config_t::parse_argv(std::vector<const char*>& args)
_exit(0);
}
else if (ceph_argparse_flag(args, i, "--show_config", (char*)NULL)) {
- expand_all_meta();
- _show_config(&cout, NULL);
- _exit(0);
+ show_config = true;
}
else if (ceph_argparse_witharg(args, i, &val, "--show_config_value", (char*)NULL)) {
- char *buf = 0;
- int r = _get_val(val.c_str(), &buf, -1);
- if (r < 0) {
- if (r == -ENOENT)
- std::cerr << "failed to get config option '" << val << "': option not found" << std::endl;
- else
- std::cerr << "failed to get config option '" << val << "': " << strerror(-r) << std::endl;
- _exit(1);
- }
- string s = buf;
- expand_meta(s);
- std::cout << s << std::endl;
- _exit(0);
+ show_config_value = true;
+ show_config_value_arg = val;
}
else if (ceph_argparse_flag(args, i, "--foreground", "-f", (char*)NULL)) {
set_val_or_die("daemonize", "false");
@@ -429,6 +421,31 @@ int md_config_t::parse_argv(std::vector<const char*>& args)
parse_option(args, i, NULL);
}
}
+
+ if (show_config) {
+ expand_all_meta();
+ _show_config(&cout, NULL);
+ _exit(0);
+ }
+
+ if (show_config_value) {
+ char *buf = 0;
+ int r = _get_val(show_config_value_arg.c_str(), &buf, -1);
+ if (r < 0) {
+ if (r == -ENOENT)
+ std::cerr << "failed to get config option '" <<
+ show_config_value_arg << "': option not found" << std::endl;
+ else
+ std::cerr << "failed to get config option '" <<
+ show_config_value_arg << "': " << strerror(-r) << std::endl;
+ _exit(1);
+ }
+ string s = buf;
+ expand_meta(s, &std::cerr);
+ std::cout << s << std::endl;
+ _exit(0);
+ }
+
return 0;
}
@@ -651,7 +668,7 @@ int md_config_t::set_val(const char *key, const char *val, bool meta)
std::string v(val);
if (meta)
- expand_meta(v);
+ expand_meta(v, &std::cerr);
string k(ConfFile::normalize_key_name(key));
@@ -834,7 +851,7 @@ int md_config_t::_get_val_from_conf_file(const std::vector <std::string> §io
int ret = cf.read(s->c_str(), key, out);
if (ret == 0) {
if (emeta)
- expand_meta(out);
+ expand_meta(out, &std::cerr);
return 0;
}
else if (ret != -ENOENT)
@@ -936,28 +953,62 @@ static const int NUM_CONF_METAVARIABLES =
void md_config_t::expand_all_meta()
{
// Expand all metavariables
+ ostringstream oss;
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
config_option *opt = config_optionsp + i;
if (opt->type == OPT_STR) {
std::string *str = (std::string *)opt->conf_ptr(this);
- expand_meta(*str);
+ list<config_option *> stack;
+ expand_meta(*str, opt, stack, &oss);
}
}
+ cerr << oss.str();
+}
+
+bool md_config_t::expand_meta(std::string &origval,
+ std::ostream *oss) const
+{
+ list<config_option *> stack;
+ return expand_meta(origval, NULL, stack, oss);
}
-bool md_config_t::expand_meta(std::string &origval) const
+bool md_config_t::expand_meta(std::string &origval,
+ config_option *opt,
+ std::list<config_option *> stack,
+ std::ostream *oss) const
{
assert(lock.is_locked());
- bool found_meta = false;
- set<string> resolved;
+ // no $ means no variable expansion is necessary
+ if (origval.find("$") == string::npos)
+ return false;
+
+ // ignore an expansion loop and create a human readable
+ // message about it
+ if (opt) {
+ for (list<config_option *>::iterator i = stack.begin();
+ i != stack.end();
+ ++i) {
+ if (strcmp(opt->name, (*i)->name) == 0) {
+ *oss << "variable expansion loop at "
+ << opt->name << "=" << origval << std::endl;
+ *oss << "expansion stack: " << std::endl;
+ for (list<config_option *>::iterator j = stack.begin();
+ j != stack.end();
+ j++) {
+ *oss << (*j)->name << "=" << *(string *)(*j)->conf_ptr(this) << std::endl;
+ }
+ return false;
+ }
+ }
+ }
- string val = origval;
+ if (opt)
+ stack.push_front(opt);
- restart:
+ bool found_meta = false;
string out;
- out.reserve(val.size());
-
+ string val = origval;
for (string::size_type s = 0; s < val.size(); ) {
if (val[s] != '$') {
out += val[s++];
@@ -965,7 +1016,7 @@ bool md_config_t::expand_meta(std::string &origval) const
}
// try to parse the variable name into var, either \$\{(.+)\} or
- // \$([a-z\_]+)
+ // \$[a-z\_]+
const char *valid_chars = "abcdefghijklmnopqrstuvwxyz_";
string var;
size_t endpos = 0;
@@ -985,8 +1036,8 @@ bool md_config_t::expand_meta(std::string &origval) const
else
var = val.substr(s+1);
}
- //cout << "var='" << var << "'" << std::endl;
+ bool expanded = false;
if (var.length()) {
// special metavariable?
for (int i = 0; i < NUM_CONF_METAVARIABLES; ++i) {
@@ -1009,42 +1060,39 @@ bool md_config_t::expand_meta(std::string &origval) const
out += stringify(getpid());
else
assert(0); // unreachable
- found_meta = true;
- if (endpos != std::string::npos)
- out += val.substr(endpos);
- //cout << "val '" << val << "' s " << s << " out '" << out << "'" << std::endl;
- val = out;
- goto restart;
+ expanded = true;
}
- // config option?
- for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
- config_option *opt = &config_optionsp[i];
- if (var != opt->name)
- continue;
-
- // avoid loops
- if (resolved.count(opt->name))
- continue; // loop; skip
- resolved.insert(opt->name);
-
- found_meta = true;
- char *vv = NULL;
- _get_val(opt->name, &vv, -1);
- out += vv;
- if (endpos != std::string::npos)
- out += val.substr(endpos);
- //cout << "val '" << val << "' s " << s << " out '" << out << "' after sub " << opt->name << " -> " << vv << std::endl;
- val = out;
- free(vv);
- goto restart;
+ if (!expanded) {
+ // config option?
+ for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
+ config_option *opt = &config_optionsp[i];
+ if (var == opt->name) {
+ if (opt->type == OPT_STR) {
+ string *origval = (string *)opt->conf_ptr(this);
+ expand_meta(*origval, opt, stack, oss);
+ out += *origval;
+ } else {
+ char *vv = NULL;
+ _get_val(opt->name, &vv, -1);
+ out += vv;
+ free(vv);
+ }
+ expanded = true;
+ break;
+ }
+ }
}
}
- // pass it thru
- out += val[s++];
+ if (expanded) {
+ found_meta = true;
+ s = endpos;
+ } else {
+ out += val[s++];
+ }
}
- //cout << "done '" << origval << "' -> '" << out << "'" << std::endl;
+ // override the original value with the expanded value
origval = out;
return found_meta;
}
diff --git a/src/common/config.h b/src/common/config.h
index 08ae660..d54642b 100644
--- a/src/common/config.h
+++ b/src/common/config.h
@@ -29,6 +29,11 @@ extern struct ceph_file_layout g_default_file_layout;
#include "common/config_obs.h"
#include "msg/msg_types.h"
+enum {
+ CEPH_DEFAULT_CRUSH_REPLICATED_RULESET,
+ CEPH_DEFAULT_CRUSH_ERASURE_RULESET,
+};
+
#define OSD_REP_PRIMARY 0
#define OSD_REP_SPLAY 1
#define OSD_REP_CHAIN 2
@@ -168,9 +173,13 @@ private:
void init_subsys();
- // Expand metavariables in the provided string.
- // Returns true if any metavariables were found and expanded.
- bool expand_meta(std::string &val) const;
+ bool expand_meta(std::string &val,
+ std::ostream *oss) const;
+
+ bool expand_meta(std::string &val,
+ config_option *opt,
+ std::list<config_option *> stack,
+ std::ostream *oss) const;
/// expand all metavariables in config structure.
void expand_all_meta();
@@ -228,6 +237,8 @@ public:
* It is best if this lock comes first in the lock hierarchy. We will
* hold this lock when calling configuration observers. */
mutable Mutex lock;
+
+ friend class test_md_config_t;
};
typedef enum {
diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 0b3938e..41fc9e1 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -79,6 +79,7 @@ SUBSYS(osd, 0, 5)
SUBSYS(optracker, 0, 5)
SUBSYS(objclass, 0, 5)
SUBSYS(filestore, 1, 3)
+SUBSYS(keyvaluestore, 1, 3)
SUBSYS(journal, 1, 3)
SUBSYS(ms, 0, 5)
SUBSYS(mon, 1, 5)
@@ -148,6 +149,7 @@ OPTION(mon_osd_down_out_subtree_limit, OPT_STR, "rack") // smallest crush unit
OPTION(mon_osd_min_up_ratio, OPT_DOUBLE, .3) // min osds required to be up to mark things down
OPTION(mon_osd_min_in_ratio, OPT_DOUBLE, .3) // min osds required to be in to mark things out
OPTION(mon_osd_max_op_age, OPT_DOUBLE, 32) // max op age before we get concerned (make it a power of 2)
+OPTION(mon_osd_max_split_count, OPT_INT, 32) // largest number of PGs per "involved" OSD to let split create
OPTION(mon_stat_smooth_intervals, OPT_INT, 2) // smooth stats over last N PGMap maps
OPTION(mon_lease, OPT_FLOAT, 5) // lease interval
OPTION(mon_lease_renew_interval, OPT_FLOAT, 3) // on leader, to renew the lease
@@ -167,9 +169,12 @@ OPTION(mon_osd_nearfull_ratio, OPT_FLOAT, .85) // what % full makes an OSD near
OPTION(mon_globalid_prealloc, OPT_INT, 100) // how many globalids to prealloc
OPTION(mon_osd_report_timeout, OPT_INT, 900) // grace period before declaring unresponsive OSDs dead
OPTION(mon_force_standby_active, OPT_BOOL, true) // should mons force standby-replay mds to be active
+OPTION(mon_warn_on_old_mons, OPT_BOOL, true) // should mons set health to WARN if part of quorum is old?
+OPTION(mon_warn_on_legacy_crush_tunables, OPT_BOOL, true) // warn if crush tunables are not optimal
OPTION(mon_min_osdmap_epochs, OPT_INT, 500)
OPTION(mon_max_pgmap_epochs, OPT_INT, 500)
OPTION(mon_max_log_epochs, OPT_INT, 500)
+OPTION(mon_max_mdsmap_epochs, OPT_INT, 500)
OPTION(mon_max_osd, OPT_INT, 10000)
OPTION(mon_probe_timeout, OPT_DOUBLE, 2.0)
OPTION(mon_slurp_timeout, OPT_DOUBLE, 10.0)
@@ -191,6 +196,7 @@ OPTION(mon_inject_sync_get_chunk_delay, OPT_DOUBLE, 0) // inject N second delay
OPTION(mon_osd_min_down_reporters, OPT_INT, 1) // number of OSDs who need to report a down OSD for it to count
OPTION(mon_osd_min_down_reports, OPT_INT, 3) // number of times a down OSD must be reported for it to count
OPTION(mon_osd_force_trim_to, OPT_INT, 0) // force mon to trim maps to this point, regardless of min_last_epoch_clean (dangerous, use with care)
+OPTION(mon_mds_force_trim_to, OPT_INT, 0) // force mon to trim mdsmaps to this point (dangerous, use with care)
// dump transactions
OPTION(mon_debug_dump_transactions, OPT_BOOL, false)
@@ -205,7 +211,7 @@ OPTION(mon_leveldb_bloom_size, OPT_INT, 0) // monitor's leveldb bloom bits per e
OPTION(mon_leveldb_max_open_files, OPT_INT, 0) // monitor's leveldb max open files
OPTION(mon_leveldb_compression, OPT_BOOL, false) // monitor's leveldb uses compression
OPTION(mon_leveldb_paranoid, OPT_BOOL, false) // monitor's leveldb paranoid flag
-OPTION(mon_leveldb_log, OPT_STR, "/dev/null")
+OPTION(mon_leveldb_log, OPT_STR, "")
OPTION(mon_leveldb_size_warn, OPT_U64, 40*1024*1024*1024) // issue a warning when the monitor's leveldb goes over 40GB (in bytes)
OPTION(paxos_stash_full_interval, OPT_INT, 25) // how often (in commits) to stash a full copy of the PaxosService state
OPTION(paxos_max_join_drift, OPT_INT, 10) // max paxos iterations before we must first sync the monitor stores
@@ -257,6 +263,7 @@ OPTION(client_oc_max_dirty_age, OPT_DOUBLE, 5.0) // max age in cache before
OPTION(client_oc_max_objects, OPT_INT, 1000) // max objects in cache
OPTION(client_debug_force_sync_read, OPT_BOOL, false) // always read synchronously (go to osds)
OPTION(client_debug_inject_tick_delay, OPT_INT, 0) // delay the client tick for a number of seconds
+OPTION(client_max_inline_size, OPT_U64, 4096)
// note: the max amount of "in flight" dirty data is roughly (max - target)
OPTION(fuse_use_invalidate_cb, OPT_BOOL, false) // use fuse 2.8+ invalidate callback to keep page cache consistent
OPTION(fuse_allow_other, OPT_BOOL, true)
@@ -264,6 +271,10 @@ OPTION(fuse_default_permissions, OPT_BOOL, true)
OPTION(fuse_big_writes, OPT_BOOL, true)
OPTION(fuse_atomic_o_trunc, OPT_BOOL, true)
OPTION(fuse_debug, OPT_BOOL, false)
+OPTION(fuse_multithreaded, OPT_BOOL, false)
+
+OPTION(crush_location, OPT_STR, "") // whitespace-separated list of key=value pairs describing crush location
+
OPTION(objecter_tick_interval, OPT_DOUBLE, 5.0)
OPTION(objecter_timeout, OPT_DOUBLE, 10.0) // before we ask for a map
OPTION(objecter_inflight_op_bytes, OPT_U64, 1024*1024*100) // max in-flight data (both directions)
@@ -279,14 +290,14 @@ OPTION(mds_max_file_size, OPT_U64, 1ULL << 40)
OPTION(mds_cache_size, OPT_INT, 100000)
OPTION(mds_cache_mid, OPT_FLOAT, .7)
OPTION(mds_mem_max, OPT_INT, 1048576) // KB
-OPTION(mds_dir_commit_ratio, OPT_FLOAT, .5)
-OPTION(mds_dir_max_commit_size, OPT_INT, 90) // MB
+OPTION(mds_dir_max_commit_size, OPT_INT, 10) // MB
OPTION(mds_decay_halflife, OPT_FLOAT, 5)
OPTION(mds_beacon_interval, OPT_FLOAT, 4)
OPTION(mds_beacon_grace, OPT_FLOAT, 15)
OPTION(mds_enforce_unique_name, OPT_BOOL, true)
OPTION(mds_blacklist_interval, OPT_FLOAT, 24.0*60.0) // how long to blacklist failed nodes
OPTION(mds_session_timeout, OPT_FLOAT, 60) // cap bits and leases time out if client idle
+OPTION(mds_freeze_tree_timeout, OPT_FLOAT, 30) // cap bits and leases time out if client idle
OPTION(mds_session_autoclose, OPT_FLOAT, 300) // autoclose idle session
OPTION(mds_reconnect_timeout, OPT_FLOAT, 45) // seconds to wait for clients during mds restart
// make it (mds_session_timeout - mds_beacon_grace)
@@ -295,7 +306,6 @@ OPTION(mds_dirstat_min_interval, OPT_FLOAT, 1) // try to avoid propagating mo
OPTION(mds_scatter_nudge_interval, OPT_FLOAT, 5) // how quickly dirstat changes propagate up the hierarchy
OPTION(mds_client_prealloc_inos, OPT_INT, 1000)
OPTION(mds_early_reply, OPT_BOOL, true)
-OPTION(mds_use_tmap, OPT_BOOL, true) // use trivialmap for dir updates
OPTION(mds_default_dir_hash, OPT_INT, CEPH_STR_HASH_RJENKINS)
OPTION(mds_log, OPT_BOOL, true)
OPTION(mds_log_skip_corrupt_events, OPT_BOOL, false)
@@ -384,15 +394,26 @@ OPTION(osd_client_message_cap, OPT_U64, 100) // num client messages
OPTION(osd_pg_bits, OPT_INT, 6) // bits per osd
OPTION(osd_pgp_bits, OPT_INT, 6) // bits per osd
OPTION(osd_crush_chooseleaf_type, OPT_INT, 1) // 1 = host
-OPTION(osd_min_rep, OPT_INT, 1)
-OPTION(osd_max_rep, OPT_INT, 10)
-OPTION(osd_pool_default_crush_rule, OPT_INT, 0)
-OPTION(osd_pool_default_size, OPT_INT, 2)
+OPTION(osd_pool_default_crush_rule, OPT_INT, -1) // deprecated for osd_pool_default_crush_replicated_ruleset
+OPTION(osd_pool_default_crush_replicated_ruleset, OPT_INT, CEPH_DEFAULT_CRUSH_REPLICATED_RULESET)
+OPTION(osd_pool_default_crush_erasure_ruleset, OPT_INT, CEPH_DEFAULT_CRUSH_ERASURE_RULESET)
+OPTION(osd_pool_default_size, OPT_INT, 3)
OPTION(osd_pool_default_min_size, OPT_INT, 0) // 0 means no specific default; ceph will use size-size/2
OPTION(osd_pool_default_pg_num, OPT_INT, 8) // number of PGs for new pools. Configure in global or mon section of ceph.conf
OPTION(osd_pool_default_pgp_num, OPT_INT, 8) // number of PGs for placement purposes. Should be equal to pg_num
+OPTION(osd_pool_default_erasure_code_directory, OPT_STR, CEPH_PKGLIBDIR"/erasure-code") // default for the erasure-code-directory=XXX property of osd pool create
+OPTION(osd_pool_default_erasure_code_properties,
+ OPT_STR,
+ "erasure-code-plugin=jerasure "
+ "erasure-code-technique=cauchy_good "
+ "erasure-code-packetsize=3072 "
+ "erasure-code-k=4 "
+ "erasure-code-m=2 "
+ ) // default properties of osd pool create
OPTION(osd_pool_default_flags, OPT_INT, 0) // default flags for new pools
-OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, false) // use new pg hashing to prevent pool/pg overlap
+OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true) // use new pg hashing to prevent pool/pg overlap
+OPTION(osd_hit_set_min_size, OPT_INT, 1000) // min target size for a HitSet
+OPTION(osd_hit_set_namespace, OPT_STR, ".ceph-internal") // rados namespace for hit_set tracking
OPTION(osd_map_dedup, OPT_BOOL, true)
OPTION(osd_map_cache_size, OPT_INT, 500)
OPTION(osd_map_message_max, OPT_INT, 100) // max maps per MOSDMap message
@@ -494,11 +515,21 @@ OPTION(osd_leveldb_bloom_size, OPT_INT, 0) // OSD's leveldb bloom bits per entry
OPTION(osd_leveldb_max_open_files, OPT_INT, 0) // OSD's leveldb max open files
OPTION(osd_leveldb_compression, OPT_BOOL, true) // OSD's leveldb uses compression
OPTION(osd_leveldb_paranoid, OPT_BOOL, false) // OSD's leveldb paranoid flag
-OPTION(osd_leveldb_log, OPT_STR, "/dev/null") // enable OSD leveldb log file
+OPTION(osd_leveldb_log, OPT_STR, "") // enable OSD leveldb log file
// determines whether PGLog::check() compares written out log to stored log
OPTION(osd_debug_pg_log_writeout, OPT_BOOL, false)
+OPTION(leveldb_write_buffer_size, OPT_U64, 0) // leveldb write buffer size
+OPTION(leveldb_cache_size, OPT_U64, 0) // leveldb cache size
+OPTION(leveldb_block_size, OPT_U64, 0) // leveldb block size
+OPTION(leveldb_bloom_size, OPT_INT, 0) // leveldb bloom bits per entry
+OPTION(leveldb_max_open_files, OPT_INT, 0) // leveldb max open files
+OPTION(leveldb_compression, OPT_BOOL, true) // leveldb uses compression
+OPTION(leveldb_paranoid, OPT_BOOL, false) // leveldb paranoid flag
+OPTION(leveldb_log, OPT_STR, "/dev/null") // enable leveldb log file
+OPTION(leveldb_compact_on_mount, OPT_BOOL, false)
+
/**
* osd_client_op_priority and osd_recovery_op_priority adjust the relative
* priority of client io vs recovery io.
@@ -520,7 +551,12 @@ OPTION(osd_mon_shutdown_timeout, OPT_DOUBLE, 5)
OPTION(osd_max_object_size, OPT_U64, 100*1024L*1024L*1024L) // OSD's maximum object size
OPTION(osd_max_attr_size, OPT_U64, 0)
-OPTION(filestore, OPT_BOOL, false)
+OPTION(osd_objectstore, OPT_STR, "filestore") // ObjectStore backend type
+// Override maintaining compatibility with older OSDs
+// Set to true for testing. Users should NOT set this.
+OPTION(osd_debug_override_acting_compat, OPT_BOOL, false)
+
+OPTION(filestore_debug_disable_sharded_check, OPT_BOOL, false)
/// filestore wb throttle limits
OPTION(filestore_wbthrottle_enable, OPT_BOOL, true)
@@ -620,6 +656,8 @@ OPTION(rbd_cache_block_writes_upfront, OPT_BOOL, false) // whether to block writ
OPTION(rbd_concurrent_management_ops, OPT_INT, 10) // how many operations can be in flight for a management operation like deleting or resizing an image
OPTION(rbd_balance_snap_reads, OPT_BOOL, false)
OPTION(rbd_localize_snap_reads, OPT_BOOL, false)
+OPTION(rbd_balance_parent_reads, OPT_BOOL, false)
+OPTION(rbd_localize_parent_reads, OPT_BOOL, true)
/*
* The following options change the behavior for librbd's image creation methods that
@@ -659,8 +697,12 @@ OPTION(rgw_swift_url, OPT_STR, "") // the swift url, being published
OPTION(rgw_swift_url_prefix, OPT_STR, "swift") // entry point for which a url is considered a swift url
OPTION(rgw_swift_auth_url, OPT_STR, "") // default URL to go and verify tokens for v1 auth (if not using internal swift auth)
OPTION(rgw_swift_auth_entry, OPT_STR, "auth") // entry point for which a url is considered a swift auth url
+OPTION(rgw_swift_tenant_name, OPT_STR, "") // tenant name to use for swift access
OPTION(rgw_keystone_url, OPT_STR, "") // url for keystone server
OPTION(rgw_keystone_admin_token, OPT_STR, "") // keystone admin token (shared secret)
+OPTION(rgw_keystone_admin_user, OPT_STR, "") // keystone admin user name
+OPTION(rgw_keystone_admin_password, OPT_STR, "") // keystone admin user password
+OPTION(rgw_keystone_admin_tenant, OPT_STR, "") // keystone admin user tenant
OPTION(rgw_keystone_accepted_roles, OPT_STR, "Member, admin") // roles required to serve requests
OPTION(rgw_keystone_token_cache_size, OPT_INT, 10000) // max number of entries in keystone token cache
OPTION(rgw_keystone_revocation_interval, OPT_INT, 15 * 60) // seconds between tokens revocation check
@@ -709,6 +751,7 @@ OPTION(rgw_exit_timeout_secs, OPT_INT, 120) // how many seconds to wait for proc
OPTION(rgw_get_obj_window_size, OPT_INT, 16 << 20) // window size in bytes for single get obj request
OPTION(rgw_get_obj_max_req_size, OPT_INT, 4 << 20) // max length of a single get obj rados op
OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket name rules for US region buckets
+OPTION(rgw_defer_to_bucket_acls, OPT_STR, "") // if the user has bucket perms, use those before key perms (recurse and full_control)
OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets
OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log
OPTION(rgw_num_zone_opstate_shards, OPT_INT, 128) // max shards for keeping inter-region copy progress info
@@ -727,6 +770,17 @@ OPTION(rgw_bucket_quota_ttl, OPT_INT, 600) // time for cached bucket stats to be
OPTION(rgw_bucket_quota_soft_threshold, OPT_DOUBLE, 0.95) // threshold from which we don't rely on cached info for quota decisions
OPTION(rgw_bucket_quota_cache_size, OPT_INT, 10000) // number of entries in bucket quota cache
+OPTION(rgw_expose_bucket, OPT_BOOL, false) // Return the bucket name in the 'Bucket' response header
+
+OPTION(rgw_frontends, OPT_STR, "") // alternative front ends
+
+OPTION(rgw_user_quota_bucket_sync_interval, OPT_INT, 180) // time period for accumulating modified buckets before syncing stats
+OPTION(rgw_user_quota_sync_interval, OPT_INT, 3600 * 24) // time period for accumulating modified buckets before syncing entire user stats
+OPTION(rgw_user_quota_sync_idle_users, OPT_BOOL, false) // whether stats for idle users be fully synced
+OPTION(rgw_user_quota_sync_wait_time, OPT_INT, 3600 * 24) // min time between two full stats syc for non-idle users
+
+OPTION(rgw_multipart_min_part_size, OPT_INT, 5 * 1024 * 1024) // min size for each part (except for last one) in multipart upload
+
OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter
// This will be set to true when it is safe to start threads.
diff --git a/src/common/hobject.cc b/src/common/hobject.cc
index b68baed..ffc73aa 100644
--- a/src/common/hobject.cc
+++ b/src/common/hobject.cc
@@ -274,7 +274,7 @@ ostream& operator<<(ostream& out, const ghobject_t& o)
out << o.hobj;
if (o.generation != ghobject_t::NO_GEN) {
assert(o.shard_id != ghobject_t::NO_SHARD);
- out << "/" << o.generation << "/" << o.shard_id;
+ out << "/" << o.generation << "/" << (unsigned)(o.shard_id);
}
return out;
}
diff --git a/src/common/hobject.h b/src/common/hobject.h
index 2f88ca4..8d97411 100644
--- a/src/common/hobject.h
+++ b/src/common/hobject.h
@@ -35,6 +35,7 @@ struct hobject_t {
uint32_t hash;
private:
bool max;
+ static const int64_t POOL_IS_TEMP = -1;
public:
int64_t pool;
string nspace;
@@ -55,6 +56,14 @@ public:
bool match(uint32_t bits, uint32_t match) const {
return match_hash(hash, bits, match);
}
+
+ static hobject_t make_temp(const string &name) {
+ hobject_t ret(object_t(name), "", CEPH_NOSNAP, 0, POOL_IS_TEMP, "");
+ return ret;
+ }
+ bool is_temp() const {
+ return pool == POOL_IS_TEMP;
+ }
hobject_t() : snap(0), hash(0), max(false), pool(-1) {}
@@ -116,7 +125,7 @@ public:
/* Do not use when a particular hash function is needed */
explicit hobject_t(const sobject_t &o) :
oid(o.oid), snap(o.snap), max(false), pool(-1) {
- hash = __gnu_cxx::hash<sobject_t>()(o);
+ hash = CEPH_HASH_NAMESPACE::hash<sobject_t>()(o);
}
// maximum sorted value.
@@ -194,11 +203,11 @@ public:
friend bool operator>=(const hobject_t&, const hobject_t&);
friend bool operator==(const hobject_t&, const hobject_t&);
friend bool operator!=(const hobject_t&, const hobject_t&);
- friend class ghobject_t;
+ friend struct ghobject_t;
};
WRITE_CLASS_ENCODER(hobject_t)
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<hobject_t> {
size_t operator()(const hobject_t &r) const {
static hash<object_t> H;
@@ -206,7 +215,7 @@ namespace __gnu_cxx {
return H(r.oid) ^ I(r.snap);
}
};
-}
+CEPH_HASH_NAMESPACE_END
ostream& operator<<(ostream& out, const hobject_t& o);
@@ -221,7 +230,7 @@ WRITE_CMP_OPERATORS_7(hobject_t,
oid,
snap)
-typedef uint64_t gen_t;
+typedef version_t gen_t;
typedef uint8_t shard_t;
#ifndef UINT8_MAX
@@ -268,6 +277,14 @@ public:
return generation == NO_GEN && shard_id == NO_SHARD;
}
+ bool is_no_gen() const {
+ return generation == NO_GEN;
+ }
+
+ bool is_no_shard() const {
+ return shard_id == NO_SHARD;
+ }
+
// maximum sorted value.
static ghobject_t get_max() {
ghobject_t h(hobject_t::get_max());
@@ -297,7 +314,7 @@ public:
};
WRITE_CLASS_ENCODER(ghobject_t)
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<ghobject_t> {
size_t operator()(const ghobject_t &r) const {
static hash<object_t> H;
@@ -305,7 +322,7 @@ namespace __gnu_cxx {
return H(r.hobj.oid) ^ I(r.hobj.snap);
}
};
-}
+CEPH_HASH_NAMESPACE_END
ostream& operator<<(ostream& out, const ghobject_t& o);
diff --git a/src/common/ipaddr.cc b/src/common/ipaddr.cc
index 253a7c6..3147d37 100644
--- a/src/common/ipaddr.cc
+++ b/src/common/ipaddr.cc
@@ -80,7 +80,7 @@ const struct sockaddr *find_ipv6_in_subnet(const struct ifaddrs *addrs,
struct in6_addr *cur = &((struct sockaddr_in6*)addrs->ifa_addr)->sin6_addr;
netmask_ipv6(cur, prefix_len, &temp);
- if (memcmp(temp.s6_addr32, want.s6_addr32, sizeof(temp)) == 0)
+ if (IN6_ARE_ADDR_EQUAL(&temp, &want))
return addrs->ifa_addr;
}
diff --git a/src/common/linux_version.c b/src/common/linux_version.c
new file mode 100644
index 0000000..b83dc71
--- /dev/null
+++ b/src/common/linux_version.c
@@ -0,0 +1,25 @@
+#include "common/linux_version.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/utsname.h>
+
+int get_linux_version(void)
+{
+ struct utsname ubuf;
+ int a, b, c;
+ int n;
+
+ if (uname(&ubuf) || strcmp(ubuf.sysname, "Linux"))
+ return 0;
+
+ n = sscanf(ubuf.release, "%d.%d.%d", &a, &b, &c);
+ switch (n) {
+ case 3:
+ return KERNEL_VERSION(a, b, c);
+ case 2:
+ return KERNEL_VERSION(a, b, 0);
+ default:
+ return 0;
+ }
+}
diff --git a/src/common/linux_version.h b/src/common/linux_version.h
new file mode 100644
index 0000000..5588c55
--- /dev/null
+++ b/src/common/linux_version.h
@@ -0,0 +1,22 @@
+#ifndef CEPH_LINUX_VERSION_H
+#define CEPH_LINUX_VERSION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef HAVE_LINUX_VERSION_H
+# include <linux/version.h>
+#endif
+
+#ifndef KERNEL_VERSION
+# define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))
+#endif
+
+int get_linux_version(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CEPH_LINUX_VERSION_H */
diff --git a/src/common/lockdep.cc b/src/common/lockdep.cc
index 7f34f80..5c2ee55 100644
--- a/src/common/lockdep.cc
+++ b/src/common/lockdep.cc
@@ -18,10 +18,11 @@
#include "include/types.h"
#include "lockdep.h"
-#include <ext/hash_map>
+#include "include/unordered_map.h"
+#include "include/hash_namespace.h"
#if defined(__FreeBSD__) && defined(__LP64__) // On FreeBSD pthread_t is a pointer.
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<>
struct hash<pthread_t>
{
@@ -29,7 +30,7 @@ namespace __gnu_cxx {
operator()(pthread_t __x) const
{ return (uintptr_t)__x; }
};
-}
+CEPH_HASH_NAMESPACE_END
#endif
/******* Constants **********/
@@ -50,10 +51,10 @@ struct lockdep_stopper_t {
static pthread_mutex_t lockdep_mutex = PTHREAD_MUTEX_INITIALIZER;
static CephContext *g_lockdep_ceph_ctx = NULL;
static lockdep_stopper_t lockdep_stopper;
-static hash_map<const char *, int> lock_ids;
+static ceph::unordered_map<const char *, int> lock_ids;
static map<int, const char *> lock_names;
static int last_id = 0;
-static hash_map<pthread_t, map<int,BackTrace*> > held;
+static ceph::unordered_map<pthread_t, map<int,BackTrace*> > held;
static BackTrace *follows[MAX_LOCKS][MAX_LOCKS]; // follows[a][b] means b taken after a
/******* Functions **********/
@@ -79,7 +80,7 @@ int lockdep_dump_locks()
{
pthread_mutex_lock(&lockdep_mutex);
- for (hash_map<pthread_t, map<int,BackTrace*> >::iterator p = held.begin();
+ for (ceph::unordered_map<pthread_t, map<int,BackTrace*> >::iterator p = held.begin();
p != held.end();
++p) {
lockdep_dout(0) << "--- thread " << p->first << " ---" << dendl;
@@ -109,7 +110,7 @@ int lockdep_register(const char *name)
for (int j=0; j<MAX_LOCKS; j++)
follows[i][j] = NULL;
- hash_map<const char *, int>::iterator p = lock_ids.find(name);
+ ceph::unordered_map<const char *, int>::iterator p = lock_ids.find(name);
if (p == lock_ids.end()) {
assert(last_id < MAX_LOCKS);
id = last_id++;
diff --git a/src/common/map_cacher.hpp b/src/common/map_cacher.hpp
index 04c84cd..da79e82 100644
--- a/src/common/map_cacher.hpp
+++ b/src/common/map_cacher.hpp
@@ -17,7 +17,7 @@
#include <boost/scoped_ptr.hpp>
#include <boost/optional.hpp>
-#include <tr1/memory>
+#include "include/memory.h"
#include <set>
#include <map>
#include <utility>
diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc
index 599cbbc..d3c3b44 100644
--- a/src/common/obj_bencher.cc
+++ b/src/common/obj_bencher.cc
@@ -214,8 +214,8 @@ int ObjBencher::aio_bench(
if (r != 0) goto out;
}
else if (OP_RAND_READ == operation) {
- cerr << "Random test not implemented yet!" << std::endl;
- r = -1;
+ r = rand_read_bench(secondsToRun, num_objects, concurrentios, prevPid);
+ if (r != 0) goto out;
}
if (OP_WRITE == operation && cleanup) {
@@ -683,6 +683,193 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre
return -5;
}
+int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurrentios, int pid)
+{
+ lock_cond lc(&lock);
+ std::vector<string> name(concurrentios);
+ std::string newName;
+ bufferlist* contents[concurrentios];
+ int index[concurrentios];
+ int errors = 0;
+ utime_t start_time;
+ std::vector<utime_t> start_times(concurrentios);
+ utime_t time_to_run;
+ time_to_run.set_from_double(seconds_to_run);
+ double total_latency = 0;
+ int r = 0;
+ utime_t runtime;
+ sanitize_object_contents(&data, data.object_size); //clean it up once; subsequent
+ //changes will be safe because string length should remain the same
+
+ srand (time(NULL));
+
+ r = completions_init(concurrentios);
+ if (r < 0)
+ return r;
+
+ //set up initial reads
+ for (int i = 0; i < concurrentios; ++i) {
+ name[i] = generate_object_name(i, pid);
+ contents[i] = new bufferlist();
+ }
+
+ lock.Lock();
+ data.finished = 0;
+ data.start_time = ceph_clock_now(g_ceph_context);
+ lock.Unlock();
+
+ pthread_t print_thread;
+ pthread_create(&print_thread, NULL, status_printer, (void *)this);
+
+ utime_t finish_time = data.start_time + time_to_run;
+ //start initial reads
+ for (int i = 0; i < concurrentios; ++i) {
+ index[i] = i;
+ start_times[i] = ceph_clock_now(g_ceph_context);
+ create_completion(i, _aio_cb, (void *)&lc);
+ r = aio_read(name[i], i, contents[i], data.object_size);
+ if (r < 0) { //naughty, doesn't clean up heap -- oh, or handle the print thread!
+ cerr << "r = " << r << std::endl;
+ goto ERR;
+ }
+ lock.Lock();
+ ++data.started;
+ ++data.in_flight;
+ lock.Unlock();
+ }
+
+ //keep on adding new reads as old ones complete
+ int slot;
+ bufferlist *cur_contents;
+ int rand_id;
+
+ slot = 0;
+ while (seconds_to_run && (ceph_clock_now(g_ceph_context) < finish_time)) {
+ lock.Lock();
+ int old_slot = slot;
+ bool found = false;
+ while (1) {
+ do {
+ if (completion_is_done(slot)) {
+ found = true;
+ break;
+ }
+ slot++;
+ if (slot == concurrentios) {
+ slot = 0;
+ }
+ } while (slot != old_slot);
+ if (found) {
+ break;
+ }
+ lc.cond.Wait(lock);
+ }
+ lock.Unlock();
+ rand_id = rand() % num_objects;
+ newName = generate_object_name(rand_id, pid);
+ int current_index = index[slot];
+ index[slot] = rand_id;
+ completion_wait(slot);
+ lock.Lock();
+ r = completion_ret(slot);
+ if (r != 0) {
+ cerr << "read got " << r << std::endl;
+ lock.Unlock();
+ goto ERR;
+ }
+ data.cur_latency = ceph_clock_now(g_ceph_context) - start_times[slot];
+ total_latency += data.cur_latency;
+ if( data.cur_latency > data.max_latency) data.max_latency = data.cur_latency;
+ if (data.cur_latency < data.min_latency) data.min_latency = data.cur_latency;
+ ++data.finished;
+ data.avg_latency = total_latency / data.finished;
+ --data.in_flight;
+ lock.Unlock();
+ release_completion(slot);
+ cur_contents = contents[slot];
+
+ //start new read and check data if requested
+ start_times[slot] = ceph_clock_now(g_ceph_context);
+ contents[slot] = new bufferlist();
+ create_completion(slot, _aio_cb, (void *)&lc);
+ r = aio_read(newName, slot, contents[slot], data.object_size);
+ if (r < 0) {
+ goto ERR;
+ }
+ lock.Lock();
+ ++data.started;
+ ++data.in_flight;
+ snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index);
+ lock.Unlock();
+ if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) {
+ cerr << name[slot] << " is not correct!" << std::endl;
+ ++errors;
+ }
+ name[slot] = newName;
+ delete cur_contents;
+ }
+
+ //wait for final reads to complete
+ while (data.finished < data.started) {
+ slot = data.finished % concurrentios;
+ completion_wait(slot);
+ lock.Lock();
+ r = completion_ret(slot);
+ if (r != 0) {
+ cerr << "read got " << r << std::endl;
+ lock.Unlock();
+ goto ERR;
+ }
+ data.cur_latency = ceph_clock_now(g_ceph_context) - start_times[slot];
+ total_latency += data.cur_latency;
+ if (data.cur_latency > data.max_latency) data.max_latency = data.cur_latency;
+ if (data.cur_latency < data.min_latency) data.min_latency = data.cur_latency;
+ ++data.finished;
+ data.avg_latency = total_latency / data.finished;
+ --data.in_flight;
+ release_completion(slot);
+ snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", index[slot]);
+ lock.Unlock();
+ if (memcmp(data.object_contents, contents[slot]->c_str(), data.object_size) != 0) {
+ cerr << name[slot] << " is not correct!" << std::endl;
+ ++errors;
+ }
+ delete contents[slot];
+ }
+
+ runtime = ceph_clock_now(g_ceph_context) - data.start_time;
+ lock.Lock();
+ data.done = true;
+ lock.Unlock();
+
+ pthread_join(print_thread, NULL);
+
+ double bandwidth;
+ bandwidth = ((double)data.finished)*((double)data.object_size)/(double)runtime;
+ bandwidth = bandwidth/(1024*1024); // we want it in MB/sec
+ char bw[20];
+ snprintf(bw, sizeof(bw), "%.3lf \n", bandwidth);
+
+ out(cout) << "Total time run: " << runtime << std::endl
+ << "Total reads made: " << data.finished << std::endl
+ << "Read size: " << data.object_size << std::endl
+ << "Bandwidth (MB/sec): " << bw << std::endl
+ << "Average Latency: " << data.avg_latency << std::endl
+ << "Max latency: " << data.max_latency << std::endl
+ << "Min latency: " << data.min_latency << std::endl;
+
+ completions_done();
+
+ return 0;
+
+ ERR:
+ lock.Lock();
+ data.done = 1;
+ lock.Unlock();
+ pthread_join(print_thread, NULL);
+ return -5;
+}
+
int ObjBencher::clean_up(const std::string& prefix, int concurrentios) {
int r = 0;
int object_size;
diff --git a/src/common/obj_bencher.h b/src/common/obj_bencher.h
index c8f671f..b87821a 100644
--- a/src/common/obj_bencher.h
+++ b/src/common/obj_bencher.h
@@ -65,6 +65,7 @@ protected:
int write_bench(int secondsToRun, int maxObjects, int concurrentios);
int seq_read_bench(int secondsToRun, int concurrentios, int num_objects, int writePid);
+ int rand_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid);
int clean_up(int num_objects, int prevPid, int concurrentios);
int clean_up_slow(const std::string& prefix, int concurrentios);
diff --git a/src/common/pipe.c b/src/common/pipe.c
index 9c01b32..8144558 100644
--- a/src/common/pipe.c
+++ b/src/common/pipe.c
@@ -11,8 +11,10 @@
* Foundation. See file COPYING.
*
*/
+#include "acconfig.h"
#include "common/pipe.h"
+#include "include/compat.h"
#include <errno.h>
#include <fcntl.h>
@@ -20,24 +22,40 @@
int pipe_cloexec(int pipefd[2])
{
-#if defined(O_CLOEXEC) && !defined(__FreeBSD__)
int ret;
+
+#if defined(HAVE_PIPE2) && defined(O_CLOEXEC)
ret = pipe2(pipefd, O_CLOEXEC);
- if (ret) {
- ret = -errno;
- return ret;
- }
+ if (ret == -1)
+ return -errno;
return 0;
#else
- /* The old-fashioned, race-condition prone way that we have to fall back on if
- * O_CLOEXEC does not exist. */
- int ret = pipe(pipefd);
- if (ret) {
+ ret = pipe(pipefd);
+ if (ret == -1)
+ return -errno;
+
+ /*
+ * The old-fashioned, race-condition prone way that we have to fall
+ * back on if O_CLOEXEC does not exist.
+ */
+ ret = fcntl(pipefd[0], F_SETFD, FD_CLOEXEC);
+ if (ret == -1) {
+ ret = -errno;
+ goto out;
+ }
+
+ ret = fcntl(pipefd[1], F_SETFD, FD_CLOEXEC);
+ if (ret == -1) {
ret = -errno;
- return ret;
+ goto out;
}
- fcntl(pipefd[0], F_SETFD, FD_CLOEXEC);
- fcntl(pipefd[1], F_SETFD, FD_CLOEXEC);
+
return 0;
+
+out:
+ TEMP_FAILURE_RETRY(close(pipefd[0]));
+ TEMP_FAILURE_RETRY(close(pipefd[1]));
+
+ return ret;
#endif
}
diff --git a/src/common/safe_io.c b/src/common/safe_io.c
index afee82e..0b31157 100644
--- a/src/common/safe_io.c
+++ b/src/common/safe_io.c
@@ -22,6 +22,7 @@
#include <limits.h>
#include "common/safe_io.h"
+#include "include/compat.h"
ssize_t safe_read(int fd, void *buf, size_t count)
{
@@ -117,6 +118,40 @@ ssize_t safe_pwrite(int fd, const void *buf, size_t count, off_t offset)
return 0;
}
+#ifdef CEPH_HAVE_SPLICE
+ssize_t safe_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out,
+ size_t len, unsigned int flags)
+{
+ size_t cnt = 0;
+
+ while (cnt < len) {
+ ssize_t r = splice(fd_in, off_in, fd_out, off_out, len - cnt, flags);
+ if (r <= 0) {
+ if (r == 0) {
+ // EOF
+ return cnt;
+ }
+ if (errno == EINTR)
+ continue;
+ return -errno;
+ }
+ cnt += r;
+ }
+ return cnt;
+}
+
+ssize_t safe_splice_exact(int fd_in, loff_t *off_in, int fd_out,
+ loff_t *off_out, size_t len, unsigned int flags)
+{
+ ssize_t ret = safe_splice(fd_in, off_in, fd_out, off_out, len, flags);
+ if (ret < 0)
+ return ret;
+ if ((size_t)ret != len)
+ return -EDOM;
+ return 0;
+}
+#endif
+
int safe_write_file(const char *base, const char *file,
const char *val, size_t vallen)
{
diff --git a/src/common/safe_io.h b/src/common/safe_io.h
index a4c9bc7..c45589e 100644
--- a/src/common/safe_io.h
+++ b/src/common/safe_io.h
@@ -15,6 +15,7 @@
#ifndef CEPH_SAFE_IO
#define CEPH_SAFE_IO
+#include "acconfig.h"
#include "common/compiler_extensions.h"
#include <sys/types.h>
@@ -35,6 +36,18 @@ extern "C" {
WARN_UNUSED_RESULT;
ssize_t safe_pwrite(int fd, const void *buf, size_t count, off_t offset)
WARN_UNUSED_RESULT;
+#ifdef CEPH_HAVE_SPLICE
+ /*
+ * Similar to the above (non-exact version) and below (exact version).
+ * See splice(2) for parameter descriptions.
+ */
+ ssize_t safe_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out,
+ size_t len, unsigned int flags)
+ WARN_UNUSED_RESULT;
+ ssize_t safe_splice_exact(int fd_in, loff_t *off_in, int fd_out,
+ loff_t *off_out, size_t len, unsigned int flags)
+ WARN_UNUSED_RESULT;
+#endif
/*
* Same as the above functions, but return -EDOM unless exactly the requested
diff --git a/src/common/sctp_crc32.c b/src/common/sctp_crc32.c
index c02ed85..4acf529 100644
--- a/src/common/sctp_crc32.c
+++ b/src/common/sctp_crc32.c
@@ -42,11 +42,7 @@ __FBSDID("$FreeBSD: src/sys/netinet/sctp_crc32.c,v 1.8 2007/05/08 17:01:10 rrs E
#include <stdint.h>
-#if defined(__FreeBSD__)
-#include <sys/endian.h>
-#else
-#include <endian.h>
-#endif
+#include "include/byteorder.h"
#ifndef SCTP_USE_ADLER32
@@ -541,7 +537,7 @@ sctp_crc32c_sb8_64_bit(uint32_t crc,
crc = sctp_crc_tableil8_o32[(crc ^ *p_buf++) & 0x000000FF] ^
(crc >> 8);
for (li = 0; li < running_length / 8; li++) {
-#if BYTE_ORDER == BIG_ENDIAN
+#ifdef CEPH_BIG_ENDIAN
crc ^= *p_buf++;
crc ^= (*p_buf++) << 8;
crc ^= (*p_buf++) << 16;
@@ -557,7 +553,7 @@ sctp_crc32c_sb8_64_bit(uint32_t crc,
sctp_crc_tableil8_o72[term2 & 0x000000FF] ^
sctp_crc_tableil8_o64[(term2 >> 8) & 0x000000FF];
-#if BYTE_ORDER == BIG_ENDIAN
+#ifdef CEPH_BIG_ENDIAN
crc ^= sctp_crc_tableil8_o56[*p_buf++];
crc ^= sctp_crc_tableil8_o48[*p_buf++];
crc ^= sctp_crc_tableil8_o40[*p_buf++];
@@ -610,7 +606,7 @@ sctp_crc32c_sb8_64_bit_zero(uint32_t crc,
sctp_crc_tableil8_o72[term2 & 0x000000FF] ^
sctp_crc_tableil8_o64[(term2 >> 8) & 0x000000FF];
-#if BYTE_ORDER == BIG_ENDIAN
+#ifdef CEPH_BIG_ENDIAN
crc ^= sctp_crc_tableil8_o56[0];
crc ^= sctp_crc_tableil8_o48[0];
crc ^= sctp_crc_tableil8_o40[0];
diff --git a/src/common/shared_cache.hpp b/src/common/shared_cache.hpp
index 178d100..a435ec4 100644
--- a/src/common/shared_cache.hpp
+++ b/src/common/shared_cache.hpp
@@ -24,8 +24,8 @@
template <class K, class V>
class SharedLRU {
- typedef std::tr1::shared_ptr<V> VPtr;
- typedef std::tr1::weak_ptr<V> WeakVPtr;
+ typedef ceph::shared_ptr<V> VPtr;
+ typedef ceph::weak_ptr<V> WeakVPtr;
Mutex lock;
size_t max_size;
Cond cond;
diff --git a/src/common/sharedptr_registry.hpp b/src/common/sharedptr_registry.hpp
index 83396b8..b8c26ce 100644
--- a/src/common/sharedptr_registry.hpp
+++ b/src/common/sharedptr_registry.hpp
@@ -27,8 +27,8 @@
template <class K, class V>
class SharedPtrRegistry {
public:
- typedef std::tr1::shared_ptr<V> VPtr;
- typedef std::tr1::weak_ptr<V> WeakVPtr;
+ typedef ceph::shared_ptr<V> VPtr;
+ typedef ceph::weak_ptr<V> WeakVPtr;
int waiting;
private:
Mutex lock;
diff --git a/src/common/str_map.cc b/src/common/str_map.cc
new file mode 100644
index 0000000..a17cf77
--- /dev/null
+++ b/src/common/str_map.cc
@@ -0,0 +1,68 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+
+#include "include/str_map.h"
+#include "include/str_list.h"
+
+#include "json_spirit/json_spirit.h"
+
+using namespace std;
+
+int get_str_map(const string &str,
+ stringstream &ss,
+ map<string,string> *str_map)
+{
+ json_spirit::mValue json;
+ try {
+ // try json parsing first
+
+ json_spirit::read_or_throw(str, json);
+
+ if (json.type() != json_spirit::obj_type) {
+ ss << str << " must be a JSON object but is of type "
+ << json.type() << " instead";
+ return -EINVAL;
+ }
+
+ json_spirit::mObject o = json.get_obj();
+
+ for (map<string, json_spirit::mValue>::iterator i = o.begin();
+ i != o.end();
+ ++i) {
+ (*str_map)[i->first] = i->second.get_str();
+ }
+
+ } catch (json_spirit::Error_position &e) {
+ // fallback to key=value format
+
+ list<string> pairs;
+ get_str_list(str, "\t\n ", pairs);
+ for (list<string>::iterator i = pairs.begin(); i != pairs.end(); i++) {
+ size_t equal = i->find('=');
+ if (equal == string::npos)
+ (*str_map)[*i] = string();
+ else {
+ const string key = i->substr(0, equal);
+ equal++;
+ const string value = i->substr(equal);
+ (*str_map)[key] = value;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index 5f92bf7..a954fec 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -227,11 +227,11 @@ int CrushCompiler::decompile(ostream &out)
out << "\truleset " << crush.get_rule_mask_ruleset(i) << "\n";
switch (crush.get_rule_mask_type(i)) {
- case CEPH_PG_TYPE_REP:
+ case CEPH_PG_TYPE_REPLICATED:
out << "\ttype replicated\n";
break;
- case CEPH_PG_TYPE_RAID4:
- out << "\ttype raid4\n";
+ case CEPH_PG_TYPE_ERASURE:
+ out << "\ttype erasure\n";
break;
default:
out << "\ttype " << crush.get_rule_mask_type(i) << "\n";
@@ -253,6 +253,22 @@ int CrushCompiler::decompile(ostream &out)
case CRUSH_RULE_EMIT:
out << "\tstep emit\n";
break;
+ case CRUSH_RULE_SET_CHOOSE_TRIES:
+ out << "\tstep set_choose_tries " << crush.get_rule_arg1(i, j)
+ << "\n";
+ break;
+ case CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES:
+ out << "\tstep set_choose_local_tries " << crush.get_rule_arg1(i, j)
+ << "\n";
+ break;
+ case CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES:
+ out << "\tstep set_choose_local_fallback_tries " << crush.get_rule_arg1(i, j)
+ << "\n";
+ break;
+ case CRUSH_RULE_SET_CHOOSELEAF_TRIES:
+ out << "\tstep set_chooseleaf_tries " << crush.get_rule_arg1(i, j)
+ << "\n";
+ break;
case CRUSH_RULE_CHOOSE_FIRSTN:
out << "\tstep choose firstn "
<< crush.get_rule_arg1(i, j)
@@ -267,14 +283,14 @@ int CrushCompiler::decompile(ostream &out)
print_type_name(out, crush.get_rule_arg2(i, j), crush);
out << "\n";
break;
- case CRUSH_RULE_CHOOSE_LEAF_FIRSTN:
+ case CRUSH_RULE_CHOOSELEAF_FIRSTN:
out << "\tstep chooseleaf firstn "
<< crush.get_rule_arg1(i, j)
<< " type ";
print_type_name(out, crush.get_rule_arg2(i, j), crush);
out << "\n";
break;
- case CRUSH_RULE_CHOOSE_LEAF_INDEP:
+ case CRUSH_RULE_CHOOSELEAF_INDEP:
out << "\tstep chooseleaf indep "
<< crush.get_rule_arg1(i, j)
<< " type ";
@@ -348,10 +364,16 @@ int CrushCompiler::parse_tunable(iter_t const& i)
return -1;
}
+ /*
+
+ current crop of tunables are all now "safe". reenable this when we
+ add new ones that are ... new.
+
if (!unsafe_tunables) {
err << "tunables are NOT FULLY IMPLEMENTED; enable with --enable-unsafe-tunables to enable this feature" << std::endl;
return -1;
}
+ */
if (verbose) err << "tunable " << name << " " << val << std::endl;
return 0;
@@ -561,9 +583,9 @@ int CrushCompiler::parse_rule(iter_t const& i)
string tname = string_node(i->children[start+2]);
int type;
if (tname == "replicated")
- type = CEPH_PG_TYPE_REP;
- else if (tname == "raid4")
- type = CEPH_PG_TYPE_RAID4;
+ type = CEPH_PG_TYPE_REPLICATED;
+ else if (tname == "erasure")
+ type = CEPH_PG_TYPE_ERASURE;
else
assert(0);
@@ -595,6 +617,34 @@ int CrushCompiler::parse_rule(iter_t const& i)
}
break;
+ case crush_grammar::_step_set_choose_tries:
+ {
+ int val = int_node(s->children[1]);
+ crush.set_rule_step_set_choose_tries(ruleno, step++, val);
+ }
+ break;
+
+ case crush_grammar::_step_set_choose_local_tries:
+ {
+ int val = int_node(s->children[1]);
+ crush.set_rule_step_set_choose_local_tries(ruleno, step++, val);
+ }
+ break;
+
+ case crush_grammar::_step_set_choose_local_fallback_tries:
+ {
+ int val = int_node(s->children[1]);
+ crush.set_rule_step_set_choose_local_fallback_tries(ruleno, step++, val);
+ }
+ break;
+
+ case crush_grammar::_step_set_chooseleaf_tries:
+ {
+ int val = int_node(s->children[1]);
+ crush.set_rule_step_set_chooseleaf_tries(ruleno, step++, val);
+ }
+ break;
+
case crush_grammar::_step_choose:
case crush_grammar::_step_chooseleaf:
{
@@ -728,6 +778,10 @@ int CrushCompiler::compile(istream& in, const char *infn)
if (!infn)
infn = "<input>";
+ // always start with legacy tunables, so that the compiled result of
+ // a given crush file is fixed for all time.
+ crush.set_tunables_legacy();
+
string big;
string str;
int line = 1;
diff --git a/src/crush/CrushTester.cc b/src/crush/CrushTester.cc
index 0a649d8..23f1a7d 100644
--- a/src/crush/CrushTester.cc
+++ b/src/crush/CrushTester.cc
@@ -1,6 +1,7 @@
#include "CrushTester.h"
+#include <algorithm>
#include <stdlib.h>
@@ -203,7 +204,7 @@ bool CrushTester::check_valid_placement(int ruleno, vector<int> in, const vector
// check that we don't have any duplicate id's
for (vector<int>::iterator it = included_devices.begin(); it != included_devices.end(); ++it) {
- int num_copies = count(included_devices.begin(), included_devices.end(), (*it) );
+ int num_copies = std::count(included_devices.begin(), included_devices.end(), (*it) );
if (num_copies > 1) {
valid_placement = false;
}
@@ -502,15 +503,22 @@ int CrushTester::test()
if (output_data_file)
write_integer_indexed_vector_data_string(tester_data.placement_information, x, out);
+ bool has_item_none = false;
for (unsigned i = 0; i < out.size(); i++) {
- per[out[i]]++;
- temporary_per[out[i]]++;
+ if (out[i] != CRUSH_ITEM_NONE) {
+ per[out[i]]++;
+ temporary_per[out[i]]++;
+ } else {
+ has_item_none = true;
+ }
}
batch_per[current_batch] = temporary_per;
sizes[out.size()]++;
- if (output_bad_mappings && out.size() != (unsigned)nr) {
- cout << "bad mapping rule " << r << " x " << x << " num_rep " << nr << " result " << out << std::endl;
+ if (output_bad_mappings &&
+ (out.size() != (unsigned)nr ||
+ has_item_none)) {
+ err << "bad mapping rule " << r << " x " << x << " num_rep " << nr << " result " << out << std::endl;
}
}
diff --git a/src/crush/CrushTester.h b/src/crush/CrushTester.h
index 0cbab0f..df5a157 100644
--- a/src/crush/CrushTester.h
+++ b/src/crush/CrushTester.h
@@ -12,7 +12,6 @@
class CrushTester {
CrushWrapper& crush;
ostream& err;
- int verbose;
map<int, int> device_weight;
int min_rule, max_rule;
@@ -165,8 +164,8 @@ class CrushTester {
void write_integer_indexed_scalar_data_string(vector<string> &dst, int index, float scalar_data);
public:
- CrushTester(CrushWrapper& c, ostream& eo, int verbosity=0)
- : crush(c), err(eo), verbose(verbosity),
+ CrushTester(CrushWrapper& c, ostream& eo)
+ : crush(c), err(eo),
min_rule(-1), max_rule(-1),
min_x(-1), max_x(-1),
min_rep(-1), max_rep(-1),
@@ -188,48 +187,103 @@ public:
void set_output_data_file_name(string name) {
output_data_file_name = name;
}
+ string get_output_data_file_name() const {
+ return output_data_file_name;
+ }
+
void set_output_data_file(bool b) {
output_data_file = b;
}
+ bool get_output_data_file() const {
+ return output_data_file;
+ }
+
void set_output_csv(bool b) {
output_csv = b;
}
+ bool get_output_csv() const {
+ return output_csv;
+ }
+
void set_output_utilization(bool b) {
output_utilization = b;
}
+ bool get_output_utilization() const {
+ return output_utilization;
+ }
+
void set_output_utilization_all(bool b) {
output_utilization_all = b;
}
+ bool get_output_utilization_all() const {
+ return output_utilization_all;
+ }
+
void set_output_statistics(bool b) {
output_statistics = b;
}
+ bool get_output_statistics() const {
+ return output_statistics;
+ }
+
void set_output_bad_mappings(bool b) {
output_bad_mappings = b;
}
+ bool get_output_bad_mappings() const {
+ return output_bad_mappings;
+ }
+
void set_output_choose_tries(bool b) {
output_choose_tries = b;
}
+ bool get_output_choose_tries() const {
+ return output_choose_tries;
+ }
void set_batches(int b) {
num_batches = b;
}
+ int get_batches() const {
+ return num_batches;
+ }
+
void set_random_placement() {
use_crush = false;
}
+ bool get_random_placement() const {
+ return use_crush == false;
+ }
+
void set_bucket_down_ratio(float bucket_ratio) {
mark_down_bucket_ratio = bucket_ratio;
}
+ float get_bucket_down_ratio() const {
+ return mark_down_bucket_ratio;
+ }
+
void set_device_down_ratio(float device_ratio) {
mark_down_device_ratio = device_ratio;
}
+ float set_device_down_ratio() const {
+ return mark_down_device_ratio;
+ }
+
void set_device_weight(int dev, float f);
void set_min_rep(int r) {
min_rep = r;
}
+ int get_min_rep() const {
+ return min_rep;
+ }
+
void set_max_rep(int r) {
max_rep = r;
}
+ int get_max_rep() const {
+ return max_rep;
+ }
+
void set_num_rep(int r) {
min_rep = max_rep = r;
}
@@ -237,9 +291,17 @@ public:
void set_min_x(int x) {
min_x = x;
}
+ int get_min_x() const {
+ return min_x;
+ }
+
void set_max_x(int x) {
max_x = x;
}
+ int get_max_x() const {
+ return max_x;
+ }
+
void set_x(int x) {
min_x = max_x = x;
}
@@ -247,9 +309,17 @@ public:
void set_min_rule(int rule) {
min_rule = rule;
}
+ int get_min_rule() const {
+ return min_rule;
+ }
+
void set_max_rule(int rule) {
max_rule = rule;
}
+ int get_max_rule() const {
+ return max_rule;
+ }
+
void set_rule(int rule) {
min_rule = max_rule = rule;
}
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index d17166b..ae4ec1c 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -6,6 +6,23 @@
#define dout_subsys ceph_subsys_crush
+bool CrushWrapper::has_v2_rules() const
+{
+ // check rules for use of indep or new SET_* rule steps
+ for (unsigned i=0; i<crush->max_rules; i++) {
+ crush_rule *r = crush->rules[i];
+ if (!r)
+ continue;
+ for (unsigned j=0; j<r->len; j++) {
+ if (r->steps[j].op == CRUSH_RULE_CHOOSE_INDEP ||
+ r->steps[j].op == CRUSH_RULE_CHOOSELEAF_INDEP ||
+ r->steps[j].op == CRUSH_RULE_SET_CHOOSE_TRIES ||
+ r->steps[j].op == CRUSH_RULE_SET_CHOOSELEAF_TRIES)
+ return true;
+ }
+ }
+ return false;
+}
void CrushWrapper::find_takes(set<int>& roots) const
{
@@ -176,6 +193,71 @@ int CrushWrapper::remove_item_under(CephContext *cct, int item, int ancestor, bo
return ret;
}
+int CrushWrapper::get_common_ancestor_distance(CephContext *cct, int id,
+ const std::multimap<string,string>& loc)
+{
+ ldout(cct, 5) << __func__ << " " << id << " " << loc << dendl;
+ if (!item_exists(id))
+ return -ENOENT;
+ map<string,string> id_loc = get_full_location(id);
+ ldout(cct, 20) << " id is at " << id_loc << dendl;
+
+ for (map<int,string>::const_iterator p = type_map.begin();
+ p != type_map.end();
+ ++p) {
+ map<string,string>::iterator ip = id_loc.find(p->second);
+ if (ip == id_loc.end())
+ continue;
+ for (std::multimap<string,string>::const_iterator q = loc.find(p->second);
+ q != loc.end();
+ ++q) {
+ if (q->first != p->second)
+ break;
+ if (q->second == ip->second)
+ return p->first;
+ }
+ }
+ return -ERANGE;
+}
+
+int CrushWrapper::parse_loc_map(const std::vector<string>& args,
+ std::map<string,string> *ploc)
+{
+ ploc->clear();
+ for (unsigned i = 0; i < args.size(); ++i) {
+ const char *s = args[i].c_str();
+ const char *pos = strchr(s, '=');
+ if (!pos)
+ return -EINVAL;
+ string key(s, 0, pos-s);
+ string value(pos+1);
+ if (value.length())
+ (*ploc)[key] = value;
+ else
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int CrushWrapper::parse_loc_multimap(const std::vector<string>& args,
+ std::multimap<string,string> *ploc)
+{
+ ploc->clear();
+ for (unsigned i = 0; i < args.size(); ++i) {
+ const char *s = args[i].c_str();
+ const char *pos = strchr(s, '=');
+ if (!pos)
+ return -EINVAL;
+ string key(s, 0, pos-s);
+ string value(pos+1);
+ if (value.length())
+ ploc->insert(make_pair(key, value));
+ else
+ return -EINVAL;
+ }
+ return 0;
+}
+
bool CrushWrapper::check_item_loc(CephContext *cct, int item, const map<string,string>& loc,
int *weight)
{
@@ -241,32 +323,18 @@ map<string, string> CrushWrapper::get_full_location(int id)
int CrushWrapper::get_full_location_ordered(int id, vector<pair<string, string> >& path)
{
- int parent_id, ret;
- pair<string, string> parent_coord;
- parent_coord = get_immediate_parent(id, &ret);
-
- // read the type map and get the name of the type with the largest ID
- int high_type = 0;
- for (map<int, string>::iterator it = type_map.begin(); it != type_map.end(); ++it){
- if ( (*it).first > high_type )
- high_type = (*it).first;
- }
-
- string high_type_name = type_map[high_type];
-
- path.push_back(parent_coord);
- parent_id = get_item_id(parent_coord.second);
-
-
- while (parent_coord.first != high_type_name) {
- parent_coord = get_immediate_parent(parent_id);
+ if (!item_exists(id))
+ return -ENOENT;
+ int cur = id;
+ int ret;
+ while (true) {
+ pair<string, string> parent_coord = get_immediate_parent(cur, &ret);
+ if (ret != 0)
+ break;
path.push_back(parent_coord);
- if ( parent_coord.first != high_type_name ){
- parent_id = get_item_id(parent_coord.second);
- }
+ cur = get_item_id(parent_coord.second);
}
-
- return ret;
+ return 0;
}
@@ -336,6 +404,9 @@ int CrushWrapper::insert_item(CephContext *cct, int item, float weight, string n
if (!is_valid_crush_name(name))
return -EINVAL;
+ if (!is_valid_crush_loc(cct, loc))
+ return -EINVAL;
+
if (name_exists(name)) {
if (get_item_id(name) != item) {
ldout(cct, 10) << "device name '" << name << "' already exists as id "
@@ -371,6 +442,7 @@ int CrushWrapper::insert_item(CephContext *cct, int item, float weight, string n
return r;
}
set_item_name(newid, q->second);
+
cur = newid;
continue;
}
@@ -398,14 +470,6 @@ int CrushWrapper::insert_item(CephContext *cct, int item, float weight, string n
return -EINVAL;
}
-
- // make sure the item doesn't already exist in this bucket
- for (unsigned j=0; j<b->size; j++)
- if (b->items[j] == cur) {
- ldout(cct, 1) << "insert_item " << cur << " already exists in bucket " << b->id << dendl;
- return -EEXIST;
- }
-
// are we forming a loop?
if (subtree_contains(cur, b->id)) {
ldout(cct, 1) << "insert_item " << cur << " already contains " << b->id
@@ -448,11 +512,6 @@ int CrushWrapper::move_bucket(CephContext *cct, int id, const map<string,string>
// detach the bucket
int bucket_weight = detach_bucket(cct, id);
- // un-set the device name so we can use add_item later
- build_rmap(name_map, name_rmap);
- name_map.erase(id);
- name_rmap.erase(id_name);
-
// insert the bucket back into the hierarchy
return insert_item(cct, id, bucket_weight / (float)0x10000, id_name, loc);
}
@@ -469,11 +528,9 @@ int CrushWrapper::link_bucket(CephContext *cct, int id, const map<string,string>
// get the name of the bucket we are trying to move for later
string id_name = get_item_name(id);
- // detach the bucket
crush_bucket *b = get_bucket(id);
unsigned bucket_weight = b->weight;
- // insert the bucket back into the hierarchy
return insert_item(cct, id, bucket_weight / (float)0x10000, id_name, loc);
}
@@ -513,6 +570,9 @@ int CrushWrapper::update_item(CephContext *cct, int item, float weight, string n
if (!is_valid_crush_name(name))
return -EINVAL;
+ if (!is_valid_crush_loc(cct, loc))
+ return -EINVAL;
+
// compare quantized (fixed-point integer) weights!
int iweight = (int)(weight * (float)0x10000);
int old_iweight;
@@ -647,8 +707,11 @@ void CrushWrapper::reweight(CephContext *cct)
}
}
-int CrushWrapper::add_simple_rule(string name, string root_name, string failure_domain_name,
- ostream *err)
+int CrushWrapper::add_simple_ruleset(string name, string root_name,
+ string failure_domain_name,
+ string mode,
+ int rule_type,
+ ostream *err)
{
if (rule_exists(name)) {
if (err)
@@ -670,6 +733,11 @@ int CrushWrapper::add_simple_rule(string name, string root_name, string failure_
return -EINVAL;
}
}
+ if (mode != "firstn" && mode != "indep") {
+ if (err)
+ *err << "unknown mode " << mode;
+ return -EINVAL;
+ }
int ruleset = 0;
for (int i = 0; i < get_max_rules(); i++) {
@@ -679,20 +747,30 @@ int CrushWrapper::add_simple_rule(string name, string root_name, string failure_
}
}
- crush_rule *rule = crush_make_rule(3, ruleset, 1 /* pg_pool_t::TYPE_REP */, 1, 10);
+ int steps = 3;
+ if (mode == "indep")
+ steps = 4;
+ int min_rep = mode == "firstn" ? 1 : 3;
+ int max_rep = mode == "firstn" ? 10 : 20;
+ crush_rule *rule = crush_make_rule(steps, ruleset, rule_type, min_rep, max_rep);
assert(rule);
- crush_rule_set_step(rule, 0, CRUSH_RULE_TAKE, root, 0);
+ int step = 0;
+ if (mode == "indep")
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 5, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
if (type)
- crush_rule_set_step(rule, 1,
- CRUSH_RULE_CHOOSE_LEAF_FIRSTN,
+ crush_rule_set_step(rule, step++,
+ mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
+ CRUSH_RULE_CHOOSELEAF_INDEP,
CRUSH_CHOOSE_N,
type);
else
- crush_rule_set_step(rule, 1,
- CRUSH_RULE_CHOOSE_FIRSTN,
+ crush_rule_set_step(rule, step++,
+ mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
+ CRUSH_RULE_CHOOSE_INDEP,
CRUSH_CHOOSE_N,
0);
- crush_rule_set_step(rule, 2, CRUSH_RULE_EMIT, 0, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
int rno = crush_add_rule(crush, rule, -1);
set_rule_name(rno, name);
have_rmaps = false;
@@ -828,6 +906,9 @@ void CrushWrapper::decode(bufferlist::iterator& blp)
::decode(crush->max_rules, blp);
::decode(crush->max_devices, blp);
+ // legacy tunables, unless we decode something newer
+ set_tunables_legacy();
+
try {
// buckets
crush->buckets = (crush_bucket**)calloc(1, crush->max_buckets * sizeof(crush_bucket*));
@@ -1044,11 +1125,29 @@ void CrushWrapper::dump(Formatter *f) const
f->close_section();
f->open_object_section("tunables");
+ dump_tunables(f);
+ f->close_section();
+}
+
+void CrushWrapper::dump_tunables(Formatter *f) const
+{
f->dump_int("choose_local_tries", get_choose_local_tries());
f->dump_int("choose_local_fallback_tries", get_choose_local_fallback_tries());
f->dump_int("choose_total_tries", get_choose_total_tries());
f->dump_int("chooseleaf_descend_once", get_chooseleaf_descend_once());
- f->close_section();
+
+ // be helpful about it
+ if (has_bobtail_tunables())
+ f->dump_string("profile", "bobtail");
+ else if (has_argonaut_tunables())
+ f->dump_string("profile", "argonaut");
+ else
+ f->dump_string("profile", "unknown");
+ f->dump_int("optimal_tunables", (int)has_optimal_tunables());
+ f->dump_int("legacy_tunables", (int)has_legacy_tunables());
+
+ f->dump_int("require_feature_tunables", (int)has_nondefault_tunables());
+ f->dump_int("require_feature_tunables2", (int)has_nondefault_tunables2());
}
void CrushWrapper::dump_rules(Formatter *f) const
@@ -1088,16 +1187,24 @@ void CrushWrapper::dump_rules(Formatter *f) const
f->dump_int("num", get_rule_arg1(i, j));
f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
break;
- case CRUSH_RULE_CHOOSE_LEAF_FIRSTN:
+ case CRUSH_RULE_CHOOSELEAF_FIRSTN:
f->dump_string("op", "chooseleaf_firstn");
f->dump_int("num", get_rule_arg1(i, j));
f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
break;
- case CRUSH_RULE_CHOOSE_LEAF_INDEP:
+ case CRUSH_RULE_CHOOSELEAF_INDEP:
f->dump_string("op", "chooseleaf_indep");
f->dump_int("num", get_rule_arg1(i, j));
f->dump_string("type", get_type_name(get_rule_arg2(i, j)));
break;
+ case CRUSH_RULE_SET_CHOOSE_TRIES:
+ f->dump_string("op", "set_choose_tries");
+ f->dump_int("num", get_rule_arg1(i, j));
+ break;
+ case CRUSH_RULE_SET_CHOOSELEAF_TRIES:
+ f->dump_string("op", "set_chooseleaf_tries");
+ f->dump_int("num", get_rule_arg1(i, j));
+ break;
default:
f->dump_int("opcode", get_rule_op(i, j));
f->dump_int("arg1", get_rule_arg1(i, j));
@@ -1119,12 +1226,137 @@ void CrushWrapper::list_rules(Formatter *f) const
}
}
+struct qi {
+ int item;
+ int depth;
+ float weight;
+ qi() : item(0), depth(0), weight(0) {}
+ qi(int i, int d, float w) : item(i), depth(d), weight(w) {}
+};
+
+void CrushWrapper::dump_tree(const vector<__u32>& w, ostream *out, Formatter *f) const
+{
+ if (out)
+ *out << "# id\tweight\ttype name\treweight\n";
+ if (f)
+ f->open_array_section("nodes");
+ set<int> touched;
+ set<int> roots;
+ find_roots(roots);
+ for (set<int>::iterator p = roots.begin(); p != roots.end(); ++p) {
+ list<qi> q;
+ q.push_back(qi(*p, 0, get_bucket_weight(*p) / (float)0x10000));
+ while (!q.empty()) {
+ int cur = q.front().item;
+ int depth = q.front().depth;
+ float weight = q.front().weight;
+ q.pop_front();
+
+ if (out) {
+ *out << cur << "\t";
+ int oldprecision = out->precision();
+ *out << std::setprecision(4) << weight << std::setprecision(oldprecision) << "\t";
+
+ for (int k=0; k<depth; k++)
+ *out << "\t";
+ }
+ if (f) {
+ f->open_object_section("item");
+ }
+ if (cur >= 0) {
+
+ if (f) {
+ f->dump_unsigned("id", cur);
+ f->dump_stream("name") << "osd." << cur;
+ f->dump_string("type", get_type_name(0));
+ f->dump_int("type_id", 0);
+ }
+ if (out)
+ *out << "osd." << cur << "\t";
+
+ double wf = (double)w[cur] / (double)0x10000;
+ if (out) {
+ std::streamsize p = out->precision();
+ *out << std::setprecision(4)
+ << wf
+ << std::setprecision(p)
+ << "\t";
+ }
+ if (f) {
+ f->dump_float("reweight", wf);
+ }
+
+ if (out)
+ *out << "\n";
+ if (f) {
+ f->dump_float("crush_weight", weight);
+ f->dump_unsigned("depth", depth);
+ f->close_section();
+ }
+ touched.insert(cur);
+ }
+ if (cur >= 0) {
+ continue;
+ }
+
+ // queue bucket contents...
+ int type = get_bucket_type(cur);
+ int s = get_bucket_size(cur);
+ if (f) {
+ f->dump_int("id", cur);
+ f->dump_string("name", get_item_name(cur));
+ f->dump_string("type", get_type_name(type));
+ f->dump_int("type_id", type);
+ f->open_array_section("children");
+ }
+ for (int k=s-1; k>=0; k--) {
+ int item = get_bucket_item(cur, k);
+ q.push_front(qi(item, depth+1, (float)get_bucket_item_weight(cur, k) / (float)0x10000));
+ if (f)
+ f->dump_int("child", item);
+ }
+ if (f)
+ f->close_section();
+
+ if (out)
+ *out << get_type_name(type) << " " << get_item_name(cur) << "\n";
+ if (f) {
+ f->close_section();
+ }
+
+ }
+ }
+ if (f) {
+ f->close_section();
+ f->open_array_section("stray");
+ }
+
+ if (f)
+ f->close_section();
+}
+
void CrushWrapper::generate_test_instances(list<CrushWrapper*>& o)
{
o.push_back(new CrushWrapper);
// fixme
}
+int CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(CephContext *cct)
+{
+ int crush_ruleset = cct->_conf->osd_pool_default_crush_replicated_ruleset;
+ if (cct->_conf->osd_pool_default_crush_rule != -1) {
+ ldout(cct, 0) << "osd_pool_default_crush_rule is deprecated "
+ << "use osd_pool_default_crush_replicated_ruleset instead"
+ << dendl;
+ ldout(cct, 0) << "osd_pool_default_crush_rule = "
+ << cct->_conf-> osd_pool_default_crush_rule << " overrides "
+ << "osd_pool_default_crush_replicated_ruleset = "
+ << cct->_conf->osd_pool_default_crush_replicated_ruleset
+ << dendl;
+ crush_ruleset = cct->_conf->osd_pool_default_crush_rule;
+ }
+ return crush_ruleset;
+}
bool CrushWrapper::is_valid_crush_name(const string& s)
{
@@ -1141,3 +1373,19 @@ bool CrushWrapper::is_valid_crush_name(const string& s)
}
return true;
}
+
+bool CrushWrapper::is_valid_crush_loc(CephContext *cct,
+ const map<string,string> loc)
+{
+ for (map<string,string>::const_iterator l = loc.begin(); l != loc.end(); l++) {
+ if (!is_valid_crush_name(l->first) ||
+ !is_valid_crush_name(l->second)) {
+ ldout(cct, 1) << "loc["
+ << l->first << "] = '"
+ << l->second << "' not a valid crush name ([A-Za-z0-9_-.]+)"
+ << dendl;
+ return false;
+ }
+ }
+ return true;
+}
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index b4bb67b..d0f34f0 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -95,29 +95,32 @@ public:
crush = crush_create();
assert(crush);
have_rmaps = false;
+
+ set_tunables_default();
}
// tunables
- void set_tunables_legacy() {
+ void set_tunables_argonaut() {
crush->choose_local_tries = 2;
crush->choose_local_fallback_tries = 5;
crush->choose_total_tries = 19;
crush->chooseleaf_descend_once = 0;
}
- void set_tunables_optimal() {
+ void set_tunables_bobtail() {
crush->choose_local_tries = 0;
crush->choose_local_fallback_tries = 0;
crush->choose_total_tries = 50;
crush->chooseleaf_descend_once = 1;
}
- void set_tunables_argonaut() {
- set_tunables_legacy();
+
+ void set_tunables_legacy() {
+ set_tunables_argonaut();
}
- void set_tunables_bobtail() {
- set_tunables_optimal();
+ void set_tunables_optimal() {
+ set_tunables_bobtail();
}
void set_tunables_default() {
- set_tunables_legacy();
+ set_tunables_bobtail();
}
int get_choose_local_tries() const {
@@ -148,6 +151,28 @@ public:
crush->chooseleaf_descend_once = !!n;
}
+ bool has_argonaut_tunables() const {
+ return
+ crush->choose_local_tries == 2 &&
+ crush->choose_local_fallback_tries == 5 &&
+ crush->choose_total_tries == 19 &&
+ crush->chooseleaf_descend_once == 0;
+ }
+ bool has_bobtail_tunables() const {
+ return
+ crush->choose_local_tries == 0 &&
+ crush->choose_local_fallback_tries == 0 &&
+ crush->choose_total_tries == 50 &&
+ crush->chooseleaf_descend_once == 1;
+ }
+
+ bool has_optimal_tunables() const {
+ return has_bobtail_tunables();
+ }
+ bool has_legacy_tunables() const {
+ return has_argonaut_tunables();
+ }
+
bool has_nondefault_tunables() const {
return
(crush->choose_local_tries != 2 ||
@@ -158,6 +183,8 @@ public:
return
crush->chooseleaf_descend_once != 0;
}
+ bool has_v2_rules() const;
+
// bucket types
int get_num_type_names() const {
@@ -296,6 +323,8 @@ public:
/**
* returns the (type, name) of the parent bucket of id
+ *
+ * FIXME: ambiguous for items that occur multiple times in the map
*/
pair<string,string> get_immediate_parent(int id, int *ret = NULL);
int get_immediate_parent_id(int id, int *parent);
@@ -337,7 +366,7 @@ public:
* insert an item into the map at a specific position
*
* Add an item as a specific location of the hierarchy.
- * Specifically, we look for the most specific location constriant
+ * Specifically, we look for the most specific location constraint
* for which a bucket already exists, and then create intervening
* buckets beneath that in order to place the item.
*
@@ -446,6 +475,32 @@ public:
int remove_item_under(CephContext *cct, int id, int ancestor, bool unlink_only);
/**
+ * calculate the locality/distance from a given id to a crush location map
+ *
+ * Specifically, we look for the lowest-valued type for which the
+ * location of id matches that described in loc.
+ *
+ * @param cct cct
+ * @param id the existing id in the map
+ * @param loc a set of key=value pairs describing a location in the hierarchy
+ */
+ int get_common_ancestor_distance(CephContext *cct, int id,
+ const std::multimap<string,string>& loc);
+
+ /**
+ * parse a set of key/value pairs out of a string vector
+ *
+ * These are used to describe a location in the CRUSH hierarchy.
+ *
+ * @param args list of strings (each key= or key=value)
+ * @param ploc pointer to a resulting location map or multimap
+ */
+ static int parse_loc_map(const std::vector<string>& args,
+ std::map<string,string> *ploc);
+ static int parse_loc_multimap(const std::vector<string>& args,
+ std::multimap<string,string> *ploc);
+
+ /**
* get an item's weight
*
* Will return the weight for the first instance it finds.
@@ -563,6 +618,18 @@ public:
int set_rule_step_take(unsigned ruleno, unsigned step, int val) {
return set_rule_step(ruleno, step, CRUSH_RULE_TAKE, val, 0);
}
+ int set_rule_step_set_choose_tries(unsigned ruleno, unsigned step, int val) {
+ return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSE_TRIES, val, 0);
+ }
+ int set_rule_step_set_choose_local_tries(unsigned ruleno, unsigned step, int val) {
+ return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES, val, 0);
+ }
+ int set_rule_step_set_choose_local_fallback_tries(unsigned ruleno, unsigned step, int val) {
+ return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES, val, 0);
+ }
+ int set_rule_step_set_chooseleaf_tries(unsigned ruleno, unsigned step, int val) {
+ return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSELEAF_TRIES, val, 0);
+ }
int set_rule_step_choose_firstn(unsigned ruleno, unsigned step, int val, int type) {
return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_FIRSTN, val, type);
}
@@ -570,17 +637,17 @@ public:
return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_INDEP, val, type);
}
int set_rule_step_choose_leaf_firstn(unsigned ruleno, unsigned step, int val, int type) {
- return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_LEAF_FIRSTN, val, type);
+ return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSELEAF_FIRSTN, val, type);
}
int set_rule_step_choose_leaf_indep(unsigned ruleno, unsigned step, int val, int type) {
- return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_LEAF_INDEP, val, type);
+ return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSELEAF_INDEP, val, type);
}
int set_rule_step_emit(unsigned ruleno, unsigned step) {
return set_rule_step(ruleno, step, CRUSH_RULE_EMIT, 0, 0);
}
- int add_simple_rule(string name, string root_name, string failure_domain_type,
- ostream *err = 0);
+ int add_simple_ruleset(string name, string root_name, string failure_domain_type,
+ string mode, int rule_type, ostream *err = 0);
int remove_rule(int ruleno);
@@ -620,7 +687,7 @@ private:
if (!crush)
return (-EINVAL);
- if (item > 0)
+ if (item >= 0)
return (-EINVAL);
// check that the bucket that we want to detach exists
@@ -764,7 +831,8 @@ public:
const vector<__u32>& weight) const {
Mutex::Locker l(mapper_lock);
int rawout[maxout];
- int numrep = crush_do_rule(crush, rule, x, rawout, maxout, &weight[0], weight.size());
+ int scratch[maxout * 3];
+ int numrep = crush_do_rule(crush, rule, x, rawout, maxout, &weight[0], weight.size(), scratch);
if (numrep < 0)
numrep = 0;
out.resize(numrep);
@@ -792,11 +860,16 @@ public:
void decode_crush_bucket(crush_bucket** bptr, bufferlist::iterator &blp);
void dump(Formatter *f) const;
void dump_rules(Formatter *f) const;
+ void dump_tunables(Formatter *f) const;
void list_rules(Formatter *f) const;
+ void dump_tree(const vector<__u32>& w, ostream *out, Formatter *f) const;
static void generate_test_instances(list<CrushWrapper*>& o);
+ static int get_osd_pool_default_crush_replicated_ruleset(CephContext *cct);
static bool is_valid_crush_name(const string& s);
+ static bool is_valid_crush_loc(CephContext *cct,
+ const map<string,string> loc);
};
WRITE_CLASS_ENCODER(CrushWrapper)
diff --git a/src/crush/builder.c b/src/crush/builder.c
index 9bfde0b..c524cfc 100644
--- a/src/crush/builder.c
+++ b/src/crush/builder.c
@@ -6,6 +6,8 @@
#include <assert.h>
#include <errno.h>
+#include "include/int_types.h"
+
#include "builder.h"
#include "hash.h"
diff --git a/src/crush/crush.c b/src/crush/crush.c
index 1e83eb8..519793a 100644
--- a/src/crush/crush.c
+++ b/src/crush/crush.c
@@ -6,6 +6,7 @@
# include <assert.h>
# define kfree(x) do { if (x) free(x); } while (0)
# define BUG_ON(x) assert(!(x))
+# include "include/int_types.h"
#endif
#include "crush.h"
@@ -23,8 +24,8 @@ const char *crush_bucket_alg_name(int alg)
/**
* crush_get_bucket_item_weight - Get weight of an item in given bucket
- * @param b bucket pointer
- * @param p item index in bucket
+ * @b: bucket pointer
+ * @p: item index in bucket
*/
int crush_get_bucket_item_weight(const struct crush_bucket *b, int p)
{
@@ -97,7 +98,7 @@ void crush_destroy_bucket(struct crush_bucket *b)
/**
* crush_destroy - Destroy a crush_map
- * @param map crush_map pointer
+ * @map: crush_map pointer
*/
void crush_destroy(struct crush_map *map)
{
diff --git a/src/crush/crush.h b/src/crush/crush.h
index 4adabcb..0da7180 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -25,13 +25,14 @@
#define CRUSH_MAGIC 0x00010000ul /* for detecting algorithm revisions */
-
#define CRUSH_MAX_DEPTH 10 /* max crush hierarchy depth */
-#define CRUSH_MAX_SET 10 /* max size of a mapping result */
#define CRUSH_MAX_DEVICE_WEIGHT (100u * 0x10000u)
#define CRUSH_MAX_BUCKET_WEIGHT (65535u * 0x10000u)
+#define CRUSH_ITEM_UNDEF 0x7ffffffe /* undefined result (internal use only) */
+#define CRUSH_ITEM_NONE 0x7fffffff /* no result */
+
/*
* CRUSH uses user-defined "rules" to describe how inputs should be
* mapped to devices. A rule consists of sequence of steps to perform
@@ -51,8 +52,13 @@ enum {
/* arg2 = type */
CRUSH_RULE_CHOOSE_INDEP = 3, /* same */
CRUSH_RULE_EMIT = 4, /* no args */
- CRUSH_RULE_CHOOSE_LEAF_FIRSTN = 6,
- CRUSH_RULE_CHOOSE_LEAF_INDEP = 7,
+ CRUSH_RULE_CHOOSELEAF_FIRSTN = 6,
+ CRUSH_RULE_CHOOSELEAF_INDEP = 7,
+
+ CRUSH_RULE_SET_CHOOSE_TRIES = 8, /* override choose_total_tries */
+ CRUSH_RULE_SET_CHOOSELEAF_TRIES = 9, /* override chooseleaf_descend_once */
+ CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES = 10,
+ CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES = 11,
};
/*
@@ -170,7 +176,10 @@ struct crush_map {
__u32 choose_local_fallback_tries;
/* choose attempts before giving up */
__u32 choose_total_tries;
- /* attempt chooseleaf inner descent once; on failure retry outer descent */
+ /* attempt chooseleaf inner descent once for firstn mode; on
+ * reject retry outer descent. Note that this does *not*
+ * apply to a collision: in that case we will retry as we used
+ * to. */
__u32 chooseleaf_descend_once;
__u32 *choose_tries;
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index bb37fed..b1e1aef 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -44,6 +44,10 @@ struct crush_grammar : public grammar<crush_grammar>
_bucket_item,
_bucket,
_step_take,
+ _step_set_chooseleaf_tries,
+ _step_set_choose_tries,
+ _step_set_choose_local_tries,
+ _step_set_choose_local_fallback_tries,
_step_choose,
_step_chooseleaf,
_step_emit,
@@ -74,6 +78,10 @@ struct crush_grammar : public grammar<crush_grammar>
rule<ScannerT, parser_context<>, parser_tag<_bucket> > bucket;
rule<ScannerT, parser_context<>, parser_tag<_step_take> > step_take;
+ rule<ScannerT, parser_context<>, parser_tag<_step_set_choose_tries> > step_set_choose_tries;
+ rule<ScannerT, parser_context<>, parser_tag<_step_set_choose_local_tries> > step_set_choose_local_tries;
+ rule<ScannerT, parser_context<>, parser_tag<_step_set_choose_local_fallback_tries> > step_set_choose_local_fallback_tries;
+ rule<ScannerT, parser_context<>, parser_tag<_step_set_chooseleaf_tries> > step_set_chooseleaf_tries;
rule<ScannerT, parser_context<>, parser_tag<_step_choose> > step_choose;
rule<ScannerT, parser_context<>, parser_tag<_step_chooseleaf> > step_chooseleaf;
rule<ScannerT, parser_context<>, parser_tag<_step_emit> > step_emit;
@@ -116,6 +124,10 @@ struct crush_grammar : public grammar<crush_grammar>
// rules
step_take = str_p("take") >> name;
+ step_set_choose_tries = str_p("set_choose_tries") >> posint;
+ step_set_choose_local_tries = str_p("set_choose_local_tries") >> posint;
+ step_set_choose_local_fallback_tries = str_p("set_choose_local_fallback_tries") >> posint;
+ step_set_chooseleaf_tries = str_p("set_chooseleaf_tries") >> posint;
step_choose = str_p("choose")
>> ( str_p("indep") | str_p("firstn") )
>> integer
@@ -126,12 +138,16 @@ struct crush_grammar : public grammar<crush_grammar>
>> str_p("type") >> name;
step_emit = str_p("emit");
step = str_p("step") >> ( step_take |
+ step_set_choose_tries |
+ step_set_choose_local_tries |
+ step_set_choose_local_fallback_tries |
+ step_set_chooseleaf_tries |
step_choose |
step_chooseleaf |
step_emit );
crushrule = str_p("rule") >> !name >> '{'
>> str_p("ruleset") >> posint
- >> str_p("type") >> ( str_p("replicated") | str_p("raid4") )
+ >> str_p("type") >> ( str_p("replicated") | str_p("erasure") )
>> str_p("min_size") >> posint
>> str_p("max_size") >> posint
>> +step
diff --git a/src/crush/mapper.c b/src/crush/mapper.c
index ce23ef7..0b31844 100644
--- a/src/crush/mapper.c
+++ b/src/crush/mapper.c
@@ -16,6 +16,8 @@
# define dprintk(args...) /* printf(args) */
# define kmalloc(x, f) malloc(x)
# define kfree(x) free(x)
+/*# define DEBUG_INDEP*/
+# include "include/int_types.h"
#endif
#include "crush.h"
@@ -27,10 +29,10 @@
/**
* crush_find_rule - find a crush_rule id for a given ruleset, type, and size.
- * @param map the crush_map
- * @param ruleset the storage ruleset id (user defined)
- * @param type storage ruleset type (user defined)
- * @param size output set size
+ * @map: the crush_map
+ * @ruleset: the storage ruleset id (user defined)
+ * @type: storage ruleset type (user defined)
+ * @size: output set size
*/
int crush_find_rule(const struct crush_map *map, int ruleset, int type, int size)
{
@@ -68,8 +70,8 @@ int crush_find_rule(const struct crush_map *map, int ruleset, int type, int size
static int bucket_perm_choose(struct crush_bucket *bucket,
int x, int r)
{
- unsigned pr = r % bucket->size;
- unsigned i, s;
+ unsigned int pr = r % bucket->size;
+ unsigned int i, s;
/* start a new permutation if @x has changed */
if (bucket->perm_x != (__u32)x || bucket->perm_n == 0) {
@@ -100,13 +102,13 @@ static int bucket_perm_choose(struct crush_bucket *bucket,
for (i = 0; i < bucket->perm_n; i++)
dprintk(" perm_choose have %d: %d\n", i, bucket->perm[i]);
while (bucket->perm_n <= pr) {
- unsigned p = bucket->perm_n;
+ unsigned int p = bucket->perm_n;
/* no point in swapping the final entry */
if (p < bucket->size - 1) {
i = crush_hash32_3(bucket->hash, x, bucket->id, p) %
(bucket->size - p);
if (i) {
- unsigned t = bucket->perm[p + i];
+ unsigned int t = bucket->perm[p + i];
bucket->perm[p + i] = bucket->perm[p];
bucket->perm[p] = t;
}
@@ -264,7 +266,9 @@ static int crush_bucket_choose(struct crush_bucket *in, int x, int r)
* true if device is marked "out" (failed, fully offloaded)
* of the cluster
*/
-static int is_out(const struct crush_map *map, const __u32 *weight, int weight_max, int item, int x)
+static int is_out(const struct crush_map *map,
+ const __u32 *weight, int weight_max,
+ int item, int x)
{
if (item >= weight_max)
return 1;
@@ -279,26 +283,32 @@ static int is_out(const struct crush_map *map, const __u32 *weight, int weight_m
}
/**
- * crush_choose - choose numrep distinct items of given type
- * @param map the crush_map
- * @param bucket the bucket we are choose an item from
- * @param x crush input value
- * @param numrep the number of items to choose
- * @param type the type of item to choose
- * @param out pointer to output vector
- * @param outpos our position in that vector
- * @param firstn true if choosing "first n" items, false if choosing "indep"
- * @param recurse_to_leaf: true if we want one device under each item of given type
- * @descend_once: true if we should only try one descent before giving up
- * @param out2 second output vector for leaf items (if @a recurse_to_leaf)
+ * crush_choose_firstn - choose numrep distinct items of given type
+ * @map: the crush_map
+ * @bucket: the bucket we are choose an item from
+ * @x: crush input value
+ * @numrep: the number of items to choose
+ * @type: the type of item to choose
+ * @out: pointer to output vector
+ * @outpos: our position in that vector
+ * @tries: number of attempts to make
+ * @recurse_tries: number of attempts to have recursive chooseleaf make
+ * @local_retries: localized retries
+ * @local_fallback_retries: localized fallback retries
+ * @recurse_to_leaf: true if we want one device under each item of given type (chooseleaf instead of choose)
+ * @out2: second output vector for leaf items (if @recurse_to_leaf)
*/
-static int crush_choose(const struct crush_map *map,
- struct crush_bucket *bucket,
- const __u32 *weight, int weight_max,
- int x, int numrep, int type,
- int *out, int outpos,
- int firstn, int recurse_to_leaf,
- int descend_once, int *out2)
+static int crush_choose_firstn(const struct crush_map *map,
+ struct crush_bucket *bucket,
+ const __u32 *weight, int weight_max,
+ int x, int numrep, int type,
+ int *out, int outpos,
+ unsigned int tries,
+ unsigned int recurse_tries,
+ unsigned int local_retries,
+ unsigned int local_fallback_retries,
+ int recurse_to_leaf,
+ int *out2)
{
int rep;
unsigned int ftotal, flocal;
@@ -327,35 +337,17 @@ static int crush_choose(const struct crush_map *map,
collide = 0;
retry_bucket = 0;
r = rep;
- if (in->alg == CRUSH_BUCKET_UNIFORM) {
- /* be careful */
- if (firstn || (__u32)numrep >= in->size)
- /* r' = r + f_total */
- r += ftotal;
- else if (in->size % numrep == 0)
- /* r'=r+(n+1)*f_local */
- r += (numrep+1) *
- (flocal+ftotal);
- else
- /* r' = r + n*f_local */
- r += numrep * (flocal+ftotal);
- } else {
- if (firstn)
- /* r' = r + f_total */
- r += ftotal;
- else
- /* r' = r + n*f_local */
- r += numrep * (flocal+ftotal);
- }
+ /* r' = r + f_total */
+ r += ftotal;
/* bucket choose */
if (in->size == 0) {
reject = 1;
goto reject;
}
- if (map->choose_local_fallback_tries > 0 &&
+ if (local_fallback_retries > 0 &&
flocal >= (in->size>>1) &&
- flocal > map->choose_local_fallback_tries)
+ flocal > local_fallback_retries)
item = bucket_perm_choose(in, x, r);
else
item = crush_bucket_choose(in, x, r);
@@ -396,13 +388,15 @@ static int crush_choose(const struct crush_map *map,
reject = 0;
if (!collide && recurse_to_leaf) {
if (item < 0) {
- if (crush_choose(map,
+ if (crush_choose_firstn(map,
map->buckets[-1-item],
weight, weight_max,
x, outpos+1, 0,
out2, outpos,
- firstn, 0,
- map->chooseleaf_descend_once,
+ recurse_tries, 0,
+ local_retries,
+ local_fallback_retries,
+ 0,
NULL) <= outpos)
/* didn't get leaf */
reject = 1;
@@ -415,7 +409,8 @@ static int crush_choose(const struct crush_map *map,
if (!reject) {
/* out? */
if (itemtype == 0)
- reject = is_out(map, weight, weight_max,
+ reject = is_out(map, weight,
+ weight_max,
item, x);
else
reject = 0;
@@ -426,17 +421,14 @@ reject:
ftotal++;
flocal++;
- if (reject && descend_once)
- /* let outer call try again */
- skip_rep = 1;
- else if (collide && flocal <= map->choose_local_tries)
+ if (collide && flocal <= local_retries)
/* retry locally a few times */
retry_bucket = 1;
- else if (map->choose_local_fallback_tries > 0 &&
- flocal <= in->size + map->choose_local_fallback_tries)
+ else if (local_fallback_retries > 0 &&
+ flocal <= in->size + local_fallback_retries)
/* exhaustive bucket search */
retry_bucket = 1;
- else if (ftotal <= map->choose_total_tries)
+ else if (ftotal < tries)
/* then retry descent */
retry_descent = 1;
else
@@ -459,7 +451,7 @@ reject:
out[outpos] = item;
outpos++;
- if (map->choose_tries && ftotal <= map->choose_local_tries)
+ if (map->choose_tries && ftotal <= map->choose_total_tries)
map->choose_tries[ftotal]++;
}
@@ -469,21 +461,207 @@ reject:
/**
+ * crush_choose_indep: alternative breadth-first positionally stable mapping
+ *
+ */
+static void crush_choose_indep(const struct crush_map *map,
+ struct crush_bucket *bucket,
+ const __u32 *weight, int weight_max,
+ int x, int left, int numrep, int type,
+ int *out, int outpos,
+ unsigned int tries,
+ unsigned int recurse_tries,
+ int recurse_to_leaf,
+ int *out2,
+ int parent_r)
+{
+ struct crush_bucket *in = bucket;
+ int endpos = outpos + left;
+ int rep;
+ unsigned int ftotal;
+ int r;
+ int i;
+ int item = 0;
+ int itemtype;
+ int collide;
+
+ dprintk("CHOOSE%s INDEP bucket %d x %d outpos %d numrep %d\n", recurse_to_leaf ? "_LEAF" : "",
+ bucket->id, x, outpos, numrep);
+
+ /* initially my result is undefined */
+ for (rep = outpos; rep < endpos; rep++) {
+ out[rep] = CRUSH_ITEM_UNDEF;
+ if (out2)
+ out2[rep] = CRUSH_ITEM_UNDEF;
+ }
+
+ for (ftotal = 0; left > 0 && ftotal < tries; ftotal++) {
+#ifdef DEBUG_INDEP
+ if (out2 && ftotal) {
+ printf("%d %d a: ", ftotal, left);
+ for (rep = outpos; rep < endpos; rep++) {
+ printf(" %d", out[rep]);
+ }
+ printf("\n");
+ printf("%d %d b: ", ftotal, left);
+ for (rep = outpos; rep < endpos; rep++) {
+ printf(" %d", out2[rep]);
+ }
+ printf("\n");
+ }
+#endif
+ for (rep = outpos; rep < endpos; rep++) {
+ if (out[rep] != CRUSH_ITEM_UNDEF)
+ continue;
+
+ in = bucket; /* initial bucket */
+
+ /* choose through intervening buckets */
+ for (;;) {
+ /* note: we base the choice on the position
+ * even in the nested call. that means that
+ * if the first layer chooses the same bucket
+ * in a different position, we will tend to
+ * choose a different item in that bucket.
+ * this will involve more devices in data
+ * movement and tend to distribute the load.
+ */
+ r = rep + parent_r;
+
+ /* be careful */
+ if (in->alg == CRUSH_BUCKET_UNIFORM &&
+ in->size % numrep == 0)
+ /* r'=r+(n+1)*f_total */
+ r += (numrep+1) * ftotal;
+ else
+ /* r' = r + n*f_total */
+ r += numrep * ftotal;
+
+ /* bucket choose */
+ if (in->size == 0) {
+ dprintk(" empty bucket\n");
+ break;
+ }
+
+ item = crush_bucket_choose(in, x, r);
+ if (item >= map->max_devices) {
+ dprintk(" bad item %d\n", item);
+ out[rep] = CRUSH_ITEM_NONE;
+ if (out2)
+ out2[rep] = CRUSH_ITEM_NONE;
+ left--;
+ break;
+ }
+
+ /* desired type? */
+ if (item < 0)
+ itemtype = map->buckets[-1-item]->type;
+ else
+ itemtype = 0;
+ dprintk(" item %d type %d\n", item, itemtype);
+
+ /* keep going? */
+ if (itemtype != type) {
+ if (item >= 0 ||
+ (-1-item) >= map->max_buckets) {
+ dprintk(" bad item type %d\n", type);
+ out[rep] = CRUSH_ITEM_NONE;
+ if (out2)
+ out2[rep] =
+ CRUSH_ITEM_NONE;
+ left--;
+ break;
+ }
+ in = map->buckets[-1-item];
+ continue;
+ }
+
+ /* collision? */
+ collide = 0;
+ for (i = outpos; i < endpos; i++) {
+ if (out[i] == item) {
+ collide = 1;
+ break;
+ }
+ }
+ if (collide)
+ break;
+
+ if (recurse_to_leaf) {
+ if (item < 0) {
+ crush_choose_indep(map,
+ map->buckets[-1-item],
+ weight, weight_max,
+ x, 1, numrep, 0,
+ out2, rep,
+ recurse_tries, 0,
+ 0, NULL, r);
+ if (out2[rep] == CRUSH_ITEM_NONE) {
+ /* placed nothing; no leaf */
+ break;
+ }
+ } else {
+ /* we already have a leaf! */
+ out2[rep] = item;
+ }
+ }
+
+ /* out? */
+ if (itemtype == 0 &&
+ is_out(map, weight, weight_max, item, x))
+ break;
+
+ /* yay! */
+ out[rep] = item;
+ left--;
+ break;
+ }
+ }
+ }
+ for (rep = outpos; rep < endpos; rep++) {
+ if (out[rep] == CRUSH_ITEM_UNDEF) {
+ out[rep] = CRUSH_ITEM_NONE;
+ }
+ if (out2 && out2[rep] == CRUSH_ITEM_UNDEF) {
+ out2[rep] = CRUSH_ITEM_NONE;
+ }
+ }
+#ifdef DEBUG_INDEP
+ if (out2) {
+ printf("%d %d a: ", ftotal, left);
+ for (rep = outpos; rep < endpos; rep++) {
+ printf(" %d", out[rep]);
+ }
+ printf("\n");
+ printf("%d %d b: ", ftotal, left);
+ for (rep = outpos; rep < endpos; rep++) {
+ printf(" %d", out2[rep]);
+ }
+ printf("\n");
+ }
+#endif
+}
+
+/**
* crush_do_rule - calculate a mapping with the given input and rule
- * @param map the crush_map
- * @param ruleno the rule id
- * @param x hash input
- * @param result pointer to result vector
- * @param resultmax: maximum result size
+ * @map: the crush_map
+ * @ruleno: the rule id
+ * @x: hash input
+ * @result: pointer to result vector
+ * @result_max: maximum result size
+ * @weight: weight vector (for map leaves)
+ * @weight_max: size of weight vector
+ * @scratch: scratch vector for private use; must be >= 3 * result_max
*/
int crush_do_rule(const struct crush_map *map,
int ruleno, int x, int *result, int result_max,
- const __u32 *weight, int weight_max)
+ const __u32 *weight, int weight_max,
+ int *scratch)
{
int result_len;
- int a[CRUSH_MAX_SET];
- int b[CRUSH_MAX_SET];
- int c[CRUSH_MAX_SET];
+ int *a = scratch;
+ int *b = scratch + result_max;
+ int *c = scratch + result_max*2;
int recurse_to_leaf;
int *w;
int wsize = 0;
@@ -494,7 +672,18 @@ int crush_do_rule(const struct crush_map *map,
__u32 step;
int i, j;
int numrep;
- const int descend_once = 0;
+ /*
+ * the original choose_total_tries value was off by one (it
+ * counted "retries" and not "tries"). add one.
+ */
+ int choose_tries = map->choose_total_tries + 1;
+ int choose_leaf_tries = 0;
+ /*
+ * the local tries values were counted as "retries", though,
+ * and need no adjustment
+ */
+ int choose_local_retries = map->choose_local_tries;
+ int choose_local_fallback_retries = map->choose_local_fallback_tries;
if ((__u32)ruleno >= map->max_rules) {
dprintk(" bad ruleno %d\n", ruleno);
@@ -516,20 +705,40 @@ int crush_do_rule(const struct crush_map *map,
wsize = 1;
break;
- case CRUSH_RULE_CHOOSE_LEAF_FIRSTN:
+ case CRUSH_RULE_SET_CHOOSE_TRIES:
+ if (curstep->arg1 > 0)
+ choose_tries = curstep->arg1;
+ break;
+
+ case CRUSH_RULE_SET_CHOOSELEAF_TRIES:
+ if (curstep->arg1 > 0)
+ choose_leaf_tries = curstep->arg1;
+ break;
+
+ case CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES:
+ if (curstep->arg1 >= 0)
+ choose_local_retries = curstep->arg1;
+ break;
+
+ case CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES:
+ if (curstep->arg1 >= 0)
+ choose_local_fallback_retries = curstep->arg1;
+ break;
+
+ case CRUSH_RULE_CHOOSELEAF_FIRSTN:
case CRUSH_RULE_CHOOSE_FIRSTN:
firstn = 1;
/* fall through */
- case CRUSH_RULE_CHOOSE_LEAF_INDEP:
+ case CRUSH_RULE_CHOOSELEAF_INDEP:
case CRUSH_RULE_CHOOSE_INDEP:
if (wsize == 0)
break;
recurse_to_leaf =
curstep->op ==
- CRUSH_RULE_CHOOSE_LEAF_FIRSTN ||
+ CRUSH_RULE_CHOOSELEAF_FIRSTN ||
curstep->op ==
- CRUSH_RULE_CHOOSE_LEAF_INDEP;
+ CRUSH_RULE_CHOOSELEAF_INDEP;
/* reset output */
osize = 0;
@@ -547,15 +756,44 @@ int crush_do_rule(const struct crush_map *map,
continue;
}
j = 0;
- osize += crush_choose(map,
- map->buckets[-1-w[i]],
- weight, weight_max,
- x, numrep,
- curstep->arg2,
- o+osize, j,
- firstn,
- recurse_to_leaf,
- descend_once, c+osize);
+ if (firstn) {
+ int recurse_tries;
+ if (choose_leaf_tries)
+ recurse_tries =
+ choose_leaf_tries;
+ else if (map->chooseleaf_descend_once)
+ recurse_tries = 1;
+ else
+ recurse_tries = choose_tries;
+ osize += crush_choose_firstn(
+ map,
+ map->buckets[-1-w[i]],
+ weight, weight_max,
+ x, numrep,
+ curstep->arg2,
+ o+osize, j,
+ choose_tries,
+ recurse_tries,
+ choose_local_retries,
+ choose_local_fallback_retries,
+ recurse_to_leaf,
+ c+osize);
+ } else {
+ crush_choose_indep(
+ map,
+ map->buckets[-1-w[i]],
+ weight, weight_max,
+ x, numrep, numrep,
+ curstep->arg2,
+ o+osize, j,
+ choose_tries,
+ choose_leaf_tries ?
+ choose_leaf_tries : 1,
+ recurse_to_leaf,
+ c+osize,
+ 0);
+ osize += numrep;
+ }
}
if (recurse_to_leaf)
diff --git a/src/crush/mapper.h b/src/crush/mapper.h
index 73f3feb..5dfd5b1 100644
--- a/src/crush/mapper.h
+++ b/src/crush/mapper.h
@@ -14,6 +14,7 @@ extern int crush_find_rule(const struct crush_map *map, int ruleset, int type, i
extern int crush_do_rule(const struct crush_map *map,
int ruleno,
int x, int *result, int result_max,
- const __u32 *weights, int weight_max);
+ const __u32 *weights, int weight_max,
+ int *scratch);
#endif
diff --git a/src/crush/sample.txt b/src/crush/sample.txt
index aef3962..f7e0ac3 100644
--- a/src/crush/sample.txt
+++ b/src/crush/sample.txt
@@ -38,7 +38,7 @@ rule normal {
rule {
pool 1
- type raid4
+ type erasure
min_size 3
max_size 6
step take root
diff --git a/src/gtest/Makefile.am b/src/gtest/Makefile.am
index 8622e0a..435177d 100644
--- a/src/gtest/Makefile.am
+++ b/src/gtest/Makefile.am
@@ -169,7 +169,7 @@ EXTRA_DIST += $(m4data_DATA)
# We define the global AM_CPPFLAGS as everything we compile includes from these
# directories.
-AM_CPPFLAGS = -I$(srcdir) -I$(srcdir)/include
+AM_CPPFLAGS = -I$(srcdir) -I$(srcdir)/include -DGTEST_HAS_TR1_TUPLE=0
# Modifies compiler and linker flags for pthreads compatibility.
if HAVE_PTHREADS
diff --git a/src/gtest/Makefile.in b/src/gtest/Makefile.in
index d1095d9..b561abf 100644
--- a/src/gtest/Makefile.in
+++ b/src/gtest/Makefile.in
@@ -433,7 +433,7 @@ m4data_DATA = m4/gtest.m4
# We define the global AM_CPPFLAGS as everything we compile includes from these
# directories.
-AM_CPPFLAGS = -I$(srcdir) -I$(srcdir)/include
+AM_CPPFLAGS = -I$(srcdir) -I$(srcdir)/include -DGTEST_HAS_TR1_TUPLE=0
@HAVE_PTHREADS_FALSE at AM_CXXFLAGS = -DGTEST_HAS_PTHREAD=0
# Modifies compiler and linker flags for pthreads compatibility.
diff --git a/src/include/Context.h b/src/include/Context.h
index 663313c..38af843 100644
--- a/src/include/Context.h
+++ b/src/include/Context.h
@@ -23,7 +23,7 @@
#include <set>
#include <iostream>
-#include <tr1/memory>
+#include "include/memory.h"
#define mydout(cct, v) lgeneric_subdout(cct, context, v)
@@ -95,7 +95,7 @@ struct RunOnDelete {
to_run->complete(0);
}
};
-typedef std::tr1::shared_ptr<RunOnDelete> RunOnDeleteRef;
+typedef ceph::shared_ptr<RunOnDelete> RunOnDeleteRef;
/*
* finish and destroy a list of Contexts
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index 34976a6..62cd62c 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -15,7 +15,8 @@ rados_include_DATA = \
$(srcdir)/include/rados/librados.hpp \
$(srcdir)/include/buffer.h \
$(srcdir)/include/page.h \
- $(srcdir)/include/crc32c.h
+ $(srcdir)/include/crc32c.h \
+ $(srcdir)/include/memory.h
noinst_HEADERS += \
include/Context.h \
@@ -59,6 +60,7 @@ noinst_HEADERS += \
include/rbd_types.h \
include/statlite.h \
include/str_list.h \
+ include/str_map.h \
include/stringify.h \
include/triple.h \
include/types.h \
@@ -78,4 +80,11 @@ noinst_HEADERS += \
include/rbd/features.h \
include/rbd/librbd.h \
include/rbd/librbd.hpp\
- include/util.h
+ include/util.h\
+ include/stat.h \
+ include/on_exit.h \
+ include/memory.h \
+ include/rados/memory.h \
+ include/hash_namespace.h \
+ include/unordered_set.h \
+ include/unordered_map.h
diff --git a/src/include/Spinlock.h b/src/include/Spinlock.h
index 6154ae1..d0584c5 100644
--- a/src/include/Spinlock.h
+++ b/src/include/Spinlock.h
@@ -16,17 +16,73 @@
#ifndef CEPH_SPINLOCK_H
#define CEPH_SPINLOCK_H
+#include "acconfig.h"
+
#include <pthread.h>
+typedef struct {
+#ifdef HAVE_PTHREAD_SPINLOCK
+ pthread_spinlock_t lock;
+#else
+ pthread_mutex_t lock;
+#endif
+} ceph_spinlock_t;
+
+#ifdef HAVE_PTHREAD_SPINLOCK
+
+static inline int ceph_spin_init(ceph_spinlock_t *l)
+{
+ return pthread_spin_init(&l->lock, PTHREAD_PROCESS_PRIVATE);
+}
+
+static inline int ceph_spin_destroy(ceph_spinlock_t *l)
+{
+ return pthread_spin_destroy(&l->lock);
+}
+
+static inline int ceph_spin_lock(ceph_spinlock_t *l)
+{
+ return pthread_spin_lock(&l->lock);
+}
+
+static inline int ceph_spin_unlock(ceph_spinlock_t *l)
+{
+ return pthread_spin_unlock(&l->lock);
+}
+
+#else /* !HAVE_PTHREAD_SPINLOCK */
+
+static inline int ceph_spin_init(ceph_spinlock_t *l)
+{
+ return pthread_mutex_init(&l->lock, NULL);
+}
+
+static inline int ceph_spin_destroy(ceph_spinlock_t *l)
+{
+ return pthread_mutex_destroy(&l->lock);
+}
+
+static inline int ceph_spin_lock(ceph_spinlock_t *l)
+{
+ return pthread_mutex_lock(&l->lock);
+}
+
+static inline int ceph_spin_unlock(ceph_spinlock_t *l)
+{
+ return pthread_mutex_unlock(&l->lock);
+}
+
+#endif
+
class Spinlock {
- mutable pthread_spinlock_t _lock;
+ mutable ceph_spinlock_t _lock;
public:
Spinlock() {
- pthread_spin_init(&_lock, PTHREAD_PROCESS_PRIVATE);
+ ceph_spin_init(&_lock);
}
~Spinlock() {
- pthread_spin_destroy(&_lock);
+ ceph_spin_destroy(&_lock);
}
// don't allow copying.
@@ -35,11 +91,11 @@ public:
/// acquire spinlock
void lock() const {
- pthread_spin_lock(&_lock);
+ ceph_spin_lock(&_lock);
}
/// release spinlock
void unlock() const {
- pthread_spin_unlock(&_lock);
+ ceph_spin_unlock(&_lock);
}
class Locker {
diff --git a/src/include/assert.h b/src/include/assert.h
index 38c0eeb..5ff41ba 100644
--- a/src/include/assert.h
+++ b/src/include/assert.h
@@ -27,27 +27,44 @@ struct FailedAssertion {
#endif
-#if defined __cplusplus && __GNUC_PREREQ (2,95)
+#ifdef HAVE_STATIC_CAST
# define __CEPH_ASSERT_VOID_CAST static_cast<void>
#else
# define __CEPH_ASSERT_VOID_CAST (void)
#endif
-/* Version 2.4 and later of GCC define a magical variable `__PRETTY_FUNCTION__'
- which contains the name of the function currently being defined.
- This is broken in G++ before version 2.6.
- C9x has a similar variable called __func__, but prefer the GCC one since
- it demangles C++ function names. */
-# if defined __cplusplus ? __GNUC_PREREQ (2, 6) : __GNUC_PREREQ (2, 4)
-# define __CEPH_ASSERT_FUNCTION __PRETTY_FUNCTION__
-# else
-# if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L
-# define __CEPH_ASSERT_FUNCTION __func__
-# else
-# define __CEPH_ASSERT_FUNCTION ((__const char *) 0)
-# endif
+/*
+ * For GNU, test specific version features. Otherwise (e.g. LLVM) we'll use
+ * the defaults selected below.
+ */
+#ifdef __GNUC_PREREQ
+
+/*
+ * Version 2.4 and later of GCC define a magical variable
+ * `__PRETTY_FUNCTION__' which contains the name of the function currently
+ * being defined. This is broken in G++ before version 2.6. C9x has a
+ * similar variable called __func__, but prefer the GCC one since it demangles
+ * C++ function names. We define __CEPH_NO_PRETTY_FUNC if we want to avoid
+ * broken versions of G++.
+ */
+# if defined __cplusplus ? !__GNUC_PREREQ (2, 6) : !__GNUC_PREREQ (2, 4)
+# define __CEPH_NO_PRETTY_FUNC
# endif
+#endif
+
+/*
+ * Select a function-name variable based on compiler tests, and any compiler
+ * specific overrides.
+ */
+#if defined(HAVE_PRETTY_FUNC) && !defined(__CEPH_NO_PRETTY_FUNC)
+# define __CEPH_ASSERT_FUNCTION __PRETTY_FUNCTION__
+#elif defined(HAVE_FUNC)
+# define __CEPH_ASSERT_FUNCTION __func__
+#else
+# define __CEPH_ASSERT_FUNCTION ((__const char *) 0)
+#endif
+
extern void register_assert_context(CephContext *cct);
extern void __ceph_assert_fail(const char *assertion, const char *file, int line, const char *function)
__attribute__ ((__noreturn__));
diff --git a/src/include/atomic.h b/src/include/atomic.h
index 3ecbd28..537fa98 100644
--- a/src/include/atomic.h
+++ b/src/include/atomic.h
@@ -67,53 +67,52 @@ namespace ceph {
/*
* crappy slow implementation that uses a pthreads spinlock.
*/
-#include <pthread.h>
-#include "include/assert.h"
+#include "include/Spinlock.h"
namespace ceph {
class atomic_t {
- mutable pthread_spinlock_t lock;
+ mutable ceph_spinlock_t lock;
signed long val;
public:
atomic_t(int i=0)
: val(i) {
- pthread_spin_init(&lock, PTHREAD_PROCESS_PRIVATE);
+ ceph_spin_init(&lock);
}
~atomic_t() {
- pthread_spin_destroy(&lock);
+ ceph_spin_destroy(&lock);
}
void set(size_t v) {
- pthread_spin_lock(&lock);
+ ceph_spin_lock(&lock);
val = v;
- pthread_spin_unlock(&lock);
+ ceph_spin_unlock(&lock);
}
int inc() {
- pthread_spin_lock(&lock);
+ ceph_spin_lock(&lock);
int r = ++val;
- pthread_spin_unlock(&lock);
+ ceph_spin_unlock(&lock);
return r;
}
int dec() {
- pthread_spin_lock(&lock);
+ ceph_spin_lock(&lock);
int r = --val;
- pthread_spin_unlock(&lock);
+ ceph_spin_unlock(&lock);
return r;
}
void add(int d) {
- pthread_spin_lock(&lock);
+ ceph_spin_lock(&lock);
val += d;
- pthread_spin_unlock(&lock);
+ ceph_spin_unlock(&lock);
}
void sub(int d) {
- pthread_spin_lock(&lock);
+ ceph_spin_lock(&lock);
val -= d;
- pthread_spin_unlock(&lock);
+ ceph_spin_unlock(&lock);
}
int read() const {
signed long ret;
- pthread_spin_lock(&lock);
+ ceph_spin_lock(&lock);
ret = val;
- pthread_spin_unlock(&lock);
+ ceph_spin_unlock(&lock);
return ret;
}
private:
diff --git a/src/include/buffer.h b/src/include/buffer.h
index 0b497a7..4c275ca 100644
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -28,18 +28,6 @@
#include <stdio.h>
-#ifdef DARWIN
-
-#ifndef MAP_ANON
-#define MAP_ANON 0x1000
-#endif
-#ifndef O_DIRECTORY
-#define O_DIRECTORY 0x100000
-void *valloc(size_t);
-#endif
-
-#endif
-
#if defined(__linux__) // For malloc(2).
#include <malloc.h>
#endif
@@ -101,6 +89,10 @@ public:
private:
char buf[256];
};
+ struct error_code : public malformed_input {
+ explicit error_code(int error);
+ int code;
+ };
/// total bytes allocated
@@ -116,6 +108,10 @@ public:
/// enable/disable tracking of cached crcs
static void track_cached_crc(bool b);
+ /// count of calls to buffer::ptr::c_str()
+ static int get_c_str_accesses();
+ /// enable/disable tracking of buffer::ptr::c_str() calls
+ static void track_c_str(bool b);
private:
@@ -133,6 +129,7 @@ private:
class raw_posix_aligned;
class raw_hack_aligned;
class raw_char;
+ class raw_pipe;
friend std::ostream& operator<<(std::ostream& out, const raw &r);
@@ -148,8 +145,8 @@ public:
static raw* claim_malloc(unsigned len, char *buf);
static raw* create_static(unsigned len, char *buf);
static raw* create_page_aligned(unsigned len);
-
-
+ static raw* create_zero_copy(unsigned len, int fd, int64_t *offset);
+
/*
* a buffer pointer. references (a subsequence of) a raw buffer.
*/
@@ -206,9 +203,12 @@ public:
memcpy(dest, c_str()+o, l);
}
+ bool can_zero_copy() const;
+ int zero_copy_to_fd(int fd, int64_t *offset) const;
+
unsigned wasted();
- int cmp(const ptr& o);
+ int cmp(const ptr& o) const;
bool is_zero() const;
// modifiers
@@ -305,6 +305,7 @@ public:
private:
mutable iterator last_p;
+ int zero_copy_to_fd(int fd) const;
public:
// cons/des
@@ -342,6 +343,7 @@ public:
}
bool contents_equal(buffer::list& other);
+ bool can_zero_copy() const;
bool is_page_aligned() const;
bool is_n_page_sized() const;
@@ -379,6 +381,7 @@ public:
bool is_contiguous();
void rebuild();
+ void rebuild(ptr& nb);
void rebuild_page_aligned();
// sort-of-like-assignment-op
@@ -429,8 +432,10 @@ public:
void hexdump(std::ostream &out) const;
int read_file(const char *fn, std::string *error);
ssize_t read_fd(int fd, size_t len);
+ int read_fd_zero_copy(int fd, size_t len);
int write_file(const char *fn, int mode=0644);
int write_fd(int fd) const;
+ int write_fd_zero_copy(int fd) const;
uint32_t crc32c(uint32_t crc) const;
};
diff --git a/src/include/byteorder.h b/src/include/byteorder.h
index f8c7499..191d242 100644
--- a/src/include/byteorder.h
+++ b/src/include/byteorder.h
@@ -7,18 +7,31 @@
#ifndef CEPH_BYTEORDER_H
#define CEPH_BYTEORDER_H
-#if defined(__linux__)
-#include <endian.h>
-#elif defined(__FreeBSD__)
-#include <sys/endian.h>
-#else
-#error "Your platform is not yet supported."
+#include <sys/param.h>
+#include "int_types.h"
+
+#if defined(__APPLE__)
+# if __DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN
+# define CEPH_LITTLE_ENDIAN
+# elif __DARWIN_BYTE_ORDER == __DARWIN_BIG_ENDIAN
+# define CEPH_BIG_ENDIAN
+# endif
#endif
#if defined(__FreeBSD__)
-#define __BYTE_ORDER _BYTE_ORDER
-#define __BIG_ENDIAN _BIG_ENDIAN
-#define __LITTLE_ENDIAN _LITTLE_ENDIAN
+# if _BYTE_ORDER == _LITTLE_ENDIAN
+# define CEPH_LITTLE_ENDIAN
+# elif _BYTE_ORDER == _BIG_ENDIAN
+# define CEPH_BIG_ENDIAN
+# endif
+#endif
+
+#if defined(__linux__)
+# if BYTE_ORDER == LITTLE_ENDIAN
+# define CEPH_LITTLE_ENDIAN
+# elif BYTE_ORDER == BIG_ENDIAN
+# define CEPH_BIG_ENDIAN
+# endif
#endif
static __inline__ __u16 swab16(__u16 val)
@@ -44,24 +57,20 @@ static __inline__ uint64_t swab64(uint64_t val)
((val << 56)));
}
-#if !defined(__BYTE_ORDER) || !defined(__BIG_ENDIAN) || !defined(__LITTLE_ENDIAN)
-#error "Endianess is unknown!"
-#endif
-
// mswab == maybe swab (if not LE)
-#if __BYTE_ORDER == __BIG_ENDIAN
+#ifdef CEPH_BIG_ENDIAN
# define mswab64(a) swab64(a)
# define mswab32(a) swab32(a)
# define mswab16(a) swab16(a)
-#else
-# if __BYTE_ORDER != __LITTLE_ENDIAN
-# warning __BYTE_ORDER is not defined, assuming little endian
-# endif
+#elif defined(CEPH_LITTLE_ENDIAN)
# define mswab64(a) (a)
# define mswab32(a) (a)
# define mswab16(a) (a)
+#else
+# error "Could not determine endianess"
#endif
+#ifdef __cplusplus
#define MAKE_LE_CLASS(bits) \
struct ceph_le##bits { \
@@ -81,6 +90,8 @@ MAKE_LE_CLASS(32)
MAKE_LE_CLASS(16)
#undef MAKE_LE_CLASS
+#endif /* __cplusplus */
+
#define init_le64(x) { (__u64)mswab64(x) }
#define init_le32(x) { (__u32)mswab32(x) }
#define init_le16(x) { (__u16)mswab16(x) }
diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index c0f01cc..fc9b63d 100644
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -40,6 +40,14 @@
#define CEPH_FEATURE_MON_SCRUB (1ULL<<33)
#define CEPH_FEATURE_OSD_PACKED_RECOVERY (1ULL<<34)
#define CEPH_FEATURE_OSD_CACHEPOOL (1ULL<<35)
+#define CEPH_FEATURE_CRUSH_V2 (1ULL<<36) /* new indep; SET_* steps */
+#define CEPH_FEATURE_EXPORT_PEER (1ULL<<37)
+#define CEPH_FEATURE_OSD_ERASURE_CODES (1ULL<<38)
+#define CEPH_FEATURE_OSD_TMAP2OMAP (1ULL<<38) /* overlap with EC */
+/* The process supports new-style OSDMap encoding. Monitors also use
+ this bit to determine if peers support NAK messages. */
+#define CEPH_FEATURE_OSDMAP_ENC (1ULL<<39)
+#define CEPH_FEATURE_MDS_INLINE_DATA (1ULL<<40)
/*
* The introduction of CEPH_FEATURE_OSD_SNAPMAPPER caused the feature
@@ -102,7 +110,12 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) {
CEPH_FEATURE_OSD_SNAPMAPPER | \
CEPH_FEATURE_MON_SCRUB | \
CEPH_FEATURE_OSD_PACKED_RECOVERY | \
- CEPH_FEATURE_OSD_CACHEPOOL | \
+ CEPH_FEATURE_OSD_CACHEPOOL | \
+ CEPH_FEATURE_CRUSH_V2 | \
+ CEPH_FEATURE_EXPORT_PEER | \
+ CEPH_FEATURE_OSD_ERASURE_CODES | \
+ CEPH_FEATURE_OSDMAP_ENC | \
+ CEPH_FEATURE_MDS_INLINE_DATA | \
0ULL)
#define CEPH_FEATURES_SUPPORTED_DEFAULT CEPH_FEATURES_ALL
@@ -112,6 +125,7 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) {
*/
#define CEPH_FEATURES_CRUSH \
(CEPH_FEATURE_CRUSH_TUNABLES | \
- CEPH_FEATURE_CRUSH_TUNABLES2)
+ CEPH_FEATURE_CRUSH_TUNABLES2 | \
+ CEPH_FEATURE_CRUSH_V2)
#endif
diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 47ec1f1..003f03c 100644
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -283,6 +283,8 @@ enum {
CEPH_SESSION_RENEWCAPS,
CEPH_SESSION_STALE,
CEPH_SESSION_RECALL_STATE,
+ CEPH_SESSION_FLUSHMSG,
+ CEPH_SESSION_FLUSHMSG_ACK,
};
extern const char *ceph_session_op_name(int op);
@@ -462,7 +464,8 @@ struct ceph_mds_reply_cap {
__u8 flags; /* CEPH_CAP_FLAG_* */
} __attribute__ ((packed));
-#define CEPH_CAP_FLAG_AUTH 1 /* cap is issued by auth mds */
+#define CEPH_CAP_FLAG_AUTH (1 << 0) /* cap is issued by auth mds */
+#define CEPH_CAP_FLAG_RELEASE (1 << 1) /* ask client to release the cap */
/* inode record, for bundling with mds reply */
struct ceph_mds_reply_inode {
@@ -526,6 +529,9 @@ struct ceph_filelock {
int ceph_flags_to_mode(int flags);
+/* inline data state */
+#define CEPH_INLINE_NONE ((__u64)-1)
+#define CEPH_INLINE_MAX_SIZE CEPH_MIN_STRIPE_UNIT
/* capability bits */
#define CEPH_CAP_PIN 1 /* no specific capabilities beyond the pin */
@@ -632,6 +638,15 @@ enum {
extern const char *ceph_cap_op_name(int op);
+/* extra info for cap import/export */
+struct ceph_mds_cap_peer {
+ __le64 cap_id;
+ __le32 seq;
+ __le32 mseq;
+ __le32 mds;
+ __u8 flags;
+} __attribute__ ((packed));
+
/*
* caps message, used for capability callbacks, acks, requests, etc.
*/
@@ -655,12 +670,19 @@ struct ceph_mds_caps {
__le32 xattr_len;
__le64 xattr_version;
- /* filelock */
- __le64 size, max_size, truncate_size;
- __le32 truncate_seq;
- struct ceph_timespec mtime, atime, ctime;
- struct ceph_file_layout layout;
- __le32 time_warp_seq;
+ union {
+ /* all except export */
+ struct {
+ /* filelock */
+ __le64 size, max_size, truncate_size;
+ __le32 truncate_seq;
+ struct ceph_timespec mtime, atime, ctime;
+ struct ceph_file_layout layout;
+ __le32 time_warp_seq;
+ };
+ /* export message */
+ struct ceph_mds_cap_peer peer;
+ };
} __attribute__ ((packed));
/* cap release msg head */
diff --git a/src/include/cephfs/libcephfs.h b/src/include/cephfs/libcephfs.h
index 2f69ba7..1370021 100644
--- a/src/include/cephfs/libcephfs.h
+++ b/src/include/cephfs/libcephfs.h
@@ -32,15 +32,16 @@ extern "C" {
#endif
/*
- * On FreeBSD the offset is 64 bit, but libc doesn't announce it in the way glibc does.
+ * On FreeBSD and Apple the offset is 64 bit, but libc doesn't announce it in
+ * the way glibc does.
*/
-#if !defined(__FreeBSD__) && !defined(__USE_FILE_OFFSET64)
+#if !defined(__FreeBSD__) && !defined(__APPLE__) && !defined(__USE_FILE_OFFSET64)
# error libceph: must define __USE_FILE_OFFSET64 or readdir results will be corrupted
#endif
-class ceph_mount_info;
+struct ceph_mount_info;
struct ceph_dir_result;
-class CephContext;
+struct CephContext;
/* setattr mask bits */
#ifndef CEPH_SETATTR_MODE
@@ -82,7 +83,7 @@ const char *ceph_version(int *major, int *minor, int *patch);
* pass in NULL, and the id will be the process id of the client.
* @returns 0 on success, negative error code on failure
*/
-int ceph_create(class ceph_mount_info **cmount, const char * const id);
+int ceph_create(struct ceph_mount_info **cmount, const char * const id);
/**
* Create a mount handle from a CephContext, which holds the configuration
@@ -94,7 +95,7 @@ int ceph_create(class ceph_mount_info **cmount, const char * const id);
* @param conf reuse this pre-existing CephContext config
* @returns 0 on success, negative error code on failure
*/
-int ceph_create_with_context(class ceph_mount_info **cmount, class CephContext *conf);
+int ceph_create_with_context(struct ceph_mount_info **cmount, struct CephContext *conf);
/**
* Perform a mount using the path for the root of the mount.
@@ -105,7 +106,7 @@ int ceph_create_with_context(class ceph_mount_info **cmount, class CephContext *
* be "/". Passing in NULL is equivalent to "/".
* @returns 0 on success, negative error code on failure
*/
-int ceph_mount(class ceph_mount_info *cmount, const char *root);
+int ceph_mount(struct ceph_mount_info *cmount, const char *root);
/**
* Unmount a mount handle.
@@ -113,7 +114,7 @@ int ceph_mount(class ceph_mount_info *cmount, const char *root);
* @param cmount the mount handle
* @return 0 on success, negative error code on failure
*/
-int ceph_unmount(class ceph_mount_info *cmount);
+int ceph_unmount(struct ceph_mount_info *cmount);
/**
* Destroy the mount handle.
@@ -124,7 +125,7 @@ int ceph_unmount(class ceph_mount_info *cmount);
* @param cmount the mount handle
* @return 0 on success, negative error code on failure.
*/
-int ceph_release(class ceph_mount_info *cmount);
+int ceph_release(struct ceph_mount_info *cmount);
/**
* Deprecated. Unmount and destroy the ceph mount handle. This should be
@@ -134,7 +135,7 @@ int ceph_release(class ceph_mount_info *cmount);
*
* @param cmount the mount handle to shutdown
*/
-void ceph_shutdown(class ceph_mount_info *cmount);
+void ceph_shutdown(struct ceph_mount_info *cmount);
/**
* Extract the CephContext from the mount point handle.
@@ -142,14 +143,14 @@ void ceph_shutdown(class ceph_mount_info *cmount);
* @param cmount the ceph mount handle to get the context from.
* @returns the CephContext associated with the mount handle.
*/
-class CephContext *ceph_get_mount_context(class ceph_mount_info *cmount);
+struct CephContext *ceph_get_mount_context(struct ceph_mount_info *cmount);
/*
* Check mount status.
*
* Return non-zero value if mounted. Otherwise, zero.
*/
-int ceph_is_mounted(class ceph_mount_info *cmount);
+int ceph_is_mounted(struct ceph_mount_info *cmount);
/** @} init */
@@ -167,7 +168,7 @@ int ceph_is_mounted(class ceph_mount_info *cmount);
* @param path_list the configuration file path
* @returns 0 on success, negative error code on failure
*/
-int ceph_conf_read_file(class ceph_mount_info *cmount, const char *path_list);
+int ceph_conf_read_file(struct ceph_mount_info *cmount, const char *path_list);
/**
* Parse the command line arguments and load the configuration parameters.
@@ -177,7 +178,7 @@ int ceph_conf_read_file(class ceph_mount_info *cmount, const char *path_list);
* @param argv the argument list
* @returns 0 on success, negative error code on failure
*/
-int ceph_conf_parse_argv(class ceph_mount_info *cmount, int argc, const char **argv);
+int ceph_conf_parse_argv(struct ceph_mount_info *cmount, int argc, const char **argv);
/**
* Configure the cluster handle based on an environment variable
@@ -194,7 +195,7 @@ int ceph_conf_parse_argv(class ceph_mount_info *cmount, int argc, const char **a
* @param var name of the environment variable to read
* @returns 0 on success, negative error code on failure
*/
-int ceph_conf_parse_env(class ceph_mount_info *cmount, const char *var);
+int ceph_conf_parse_env(struct ceph_mount_info *cmount, const char *var);
/** Sets a configuration value from a string.
*
@@ -204,7 +205,7 @@ int ceph_conf_parse_env(class ceph_mount_info *cmount, const char *var);
*
* @returns 0 on success, negative error code otherwise.
*/
-int ceph_conf_set(class ceph_mount_info *cmount, const char *option, const char *value);
+int ceph_conf_set(struct ceph_mount_info *cmount, const char *option, const char *value);
/**
* Gets the configuration value as a string.
@@ -215,7 +216,7 @@ int ceph_conf_set(class ceph_mount_info *cmount, const char *option, const char
* @param len the length of the buffer.
* @returns the size of the buffer filled in with the value, or negative error code on failure
*/
-int ceph_conf_get(class ceph_mount_info *cmount, const char *option, char *buf, size_t len);
+int ceph_conf_get(struct ceph_mount_info *cmount, const char *option, char *buf, size_t len);
/** @} config */
@@ -236,7 +237,7 @@ int ceph_conf_get(class ceph_mount_info *cmount, const char *option, char *buf,
* @param stbuf the file system statistics filled in by this function.
* @return 0 on success, negative error code otherwise.
*/
-int ceph_statfs(class ceph_mount_info *cmount, const char *path, struct statvfs *stbuf);
+int ceph_statfs(struct ceph_mount_info *cmount, const char *path, struct statvfs *stbuf);
/**
* Synchronize all filesystem data to persistent media.
@@ -244,7 +245,7 @@ int ceph_statfs(class ceph_mount_info *cmount, const char *path, struct statvfs
* @param cmount the ceph mount handle to use for performing the sync_fs.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_sync_fs(class ceph_mount_info *cmount);
+int ceph_sync_fs(struct ceph_mount_info *cmount);
/**
* Get the current working directory.
@@ -252,7 +253,7 @@ int ceph_sync_fs(class ceph_mount_info *cmount);
* @param cmount the ceph mount to get the current working directory for.
* @returns the path to the current working directory
*/
-const char* ceph_getcwd(class ceph_mount_info *cmount);
+const char* ceph_getcwd(struct ceph_mount_info *cmount);
/**
* Change the current working directory.
@@ -261,7 +262,7 @@ const char* ceph_getcwd(class ceph_mount_info *cmount);
* @param path the path to the working directory to change into.
* @returns 0 on success, negative error code otherwise.
*/
-int ceph_chdir(class ceph_mount_info *cmount, const char *s);
+int ceph_chdir(struct ceph_mount_info *cmount, const char *s);
/** @} fsops */
@@ -281,7 +282,7 @@ int ceph_chdir(class ceph_mount_info *cmount, const char *s);
* @param dirpp the directory result pointer structure to fill in.
* @returns 0 on success or negative error code otherwise.
*/
-int ceph_opendir(class ceph_mount_info *cmount, const char *name, struct ceph_dir_result **dirpp);
+int ceph_opendir(struct ceph_mount_info *cmount, const char *name, struct ceph_dir_result **dirpp);
/**
* Close the open directory.
@@ -290,7 +291,7 @@ int ceph_opendir(class ceph_mount_info *cmount, const char *name, struct ceph_di
* @param dirp the directory result pointer (set by ceph_opendir) to close
* @returns 0 on success or negative error code on failure.
*/
-int ceph_closedir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp);
+int ceph_closedir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
/**
* Get the next entry in an open directory.
@@ -302,7 +303,7 @@ int ceph_closedir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp);
* is empty. This pointer should not be freed by the caller, and is only safe to
* access between return and the next call to ceph_readdir or ceph_closedir.
*/
-struct dirent * ceph_readdir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp);
+struct dirent * ceph_readdir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
/**
* A safe version of ceph_readdir, where the directory entry struct is allocated by the caller.
@@ -314,7 +315,7 @@ struct dirent * ceph_readdir(class ceph_mount_info *cmount, struct ceph_dir_resu
* @returns 1 if the next entry was filled in, 0 if the end of the directory stream was reached,
* and a negative error code on failure.
*/
-int ceph_readdir_r(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de);
+int ceph_readdir_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de);
/**
* A safe version of ceph_readdir that also returns the file statistics (readdir+stat).
@@ -328,7 +329,7 @@ int ceph_readdir_r(class ceph_mount_info *cmount, struct ceph_dir_result *dirp,
* @returns 1 if the next entry was filled in, 0 if the end of the directory stream was reached,
* and a negative error code on failure.
*/
-int ceph_readdirplus_r(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de,
+int ceph_readdirplus_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de,
struct stat *st, int *stmask);
/**
@@ -342,7 +343,7 @@ int ceph_readdirplus_r(class ceph_mount_info *cmount, struct ceph_dir_result *di
* @returns the length of the buffer that was filled in, will always be multiples of sizeof(struct dirent), or a
* negative error code. If the buffer is not large enough for a single entry, -ERANGE is returned.
*/
-int ceph_getdents(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, char *name, int buflen);
+int ceph_getdents(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, char *name, int buflen);
/**
* Gets multiple directory names.
@@ -355,7 +356,7 @@ int ceph_getdents(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, c
* @returns the length of the buffer filled in with entry names, or a negative error code on failure.
* If the buffer isn't large enough for a single entry, -ERANGE is returned.
*/
-int ceph_getdnames(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, char *name, int buflen);
+int ceph_getdnames(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, char *name, int buflen);
/**
* Rewind the directory stream to the beginning of the directory.
@@ -363,7 +364,7 @@ int ceph_getdnames(class ceph_mount_info *cmount, struct ceph_dir_result *dirp,
* @param cmount the ceph mount handle to use for performing the rewinddir.
* @param dirp the directory stream pointer to rewind.
*/
-void ceph_rewinddir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp);
+void ceph_rewinddir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
/**
* Get the current position of a directory stream.
@@ -374,7 +375,7 @@ void ceph_rewinddir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp)
* by ceph_telldir do not have a particular order (cannot be compared with
* inequality).
*/
-loff_t ceph_telldir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp);
+int64_t ceph_telldir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
/**
* Move the directory stream to a position specified by the given offset.
@@ -385,7 +386,7 @@ loff_t ceph_telldir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp)
* a value returned by seekdir. Note that this value does not refer to the nth
* entry in a directory, and can not be manipulated with plus or minus.
*/
-void ceph_seekdir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, loff_t offset);
+void ceph_seekdir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, int64_t offset);
/**
* Create a directory.
@@ -396,7 +397,7 @@ void ceph_seekdir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, l
* @param mode the permissions the directory should have once created.
* @returns 0 on success or a negative return code on error.
*/
-int ceph_mkdir(class ceph_mount_info *cmount, const char *path, mode_t mode);
+int ceph_mkdir(struct ceph_mount_info *cmount, const char *path, mode_t mode);
/**
* Create multiple directories at once.
@@ -407,7 +408,7 @@ int ceph_mkdir(class ceph_mount_info *cmount, const char *path, mode_t mode);
* @param mode the permissions the directory should have once created.
* @returns 0 on success or a negative return code on error.
*/
-int ceph_mkdirs(class ceph_mount_info *cmount, const char *path, mode_t mode);
+int ceph_mkdirs(struct ceph_mount_info *cmount, const char *path, mode_t mode);
/**
* Remove a directory.
@@ -416,7 +417,7 @@ int ceph_mkdirs(class ceph_mount_info *cmount, const char *path, mode_t mode);
* @param path the path of the directory to remove.
* @returns 0 on success or a negative return code on error.
*/
-int ceph_rmdir(class ceph_mount_info *cmount, const char *path);
+int ceph_rmdir(struct ceph_mount_info *cmount, const char *path);
/** @} dir */
@@ -435,7 +436,7 @@ int ceph_rmdir(class ceph_mount_info *cmount, const char *path);
* @param newname the path to the new file/directory to link from.
* @returns 0 on success or a negative return code on error.
*/
-int ceph_link(class ceph_mount_info *cmount, const char *existing, const char *newname);
+int ceph_link(struct ceph_mount_info *cmount, const char *existing, const char *newname);
/**
* Read a symbolic link.
@@ -446,7 +447,7 @@ int ceph_link(class ceph_mount_info *cmount, const char *existing, const char *n
* @param size the length of the buffer
* @returns 0 on success or negative error code on failure
*/
-int ceph_readlink(class ceph_mount_info *cmount, const char *path, char *buf, loff_t size);
+int ceph_readlink(struct ceph_mount_info *cmount, const char *path, char *buf, int64_t size);
/**
* Creates a symbolic link.
@@ -456,7 +457,7 @@ int ceph_readlink(class ceph_mount_info *cmount, const char *path, char *buf, lo
* @param newname the path to the new file/directory to link from.
* @returns 0 on success or a negative return code on failure.
*/
-int ceph_symlink(class ceph_mount_info *cmount, const char *existing, const char *newname);
+int ceph_symlink(struct ceph_mount_info *cmount, const char *existing, const char *newname);
/** @} links */
@@ -475,7 +476,7 @@ int ceph_symlink(class ceph_mount_info *cmount, const char *existing, const char
* @param path the path of the file or link to unlink.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_unlink(class ceph_mount_info *cmount, const char *path);
+int ceph_unlink(struct ceph_mount_info *cmount, const char *path);
/**
* Rename a file or directory.
@@ -485,7 +486,7 @@ int ceph_unlink(class ceph_mount_info *cmount, const char *path);
* @param to the new name of the file or directory
* @returns 0 on success or negative error code on failure.
*/
-int ceph_rename(class ceph_mount_info *cmount, const char *from, const char *to);
+int ceph_rename(struct ceph_mount_info *cmount, const char *from, const char *to);
/**
* Get a file's statistics and attributes.
@@ -495,7 +496,7 @@ int ceph_rename(class ceph_mount_info *cmount, const char *from, const char *to)
* @param stbuf the stat struct that will be filled in with the file's statistics.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_stat(class ceph_mount_info *cmount, const char *path, struct stat *stbuf);
+int ceph_stat(struct ceph_mount_info *cmount, const char *path, struct stat *stbuf);
/**
* Get a file's statistics and attributes, without following symlinks.
@@ -505,7 +506,7 @@ int ceph_stat(class ceph_mount_info *cmount, const char *path, struct stat *stbu
* @param stbuf the stat struct that will be filled in with the file's statistics.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_lstat(class ceph_mount_info *cmount, const char *path, struct stat *stbuf);
+int ceph_lstat(struct ceph_mount_info *cmount, const char *path, struct stat *stbuf);
/**
* Set a file's attributes.
@@ -516,7 +517,7 @@ int ceph_lstat(class ceph_mount_info *cmount, const char *path, struct stat *stb
* @param mask a mask of all the stat values that have been set on the stat struct.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_setattr(class ceph_mount_info *cmount, const char *relpath, struct stat *attr, int mask);
+int ceph_setattr(struct ceph_mount_info *cmount, const char *relpath, struct stat *attr, int mask);
/**
* Change the mode bits (permissions) of a file/directory.
@@ -526,7 +527,7 @@ int ceph_setattr(class ceph_mount_info *cmount, const char *relpath, struct stat
* @param mode the new permissions to set.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_chmod(class ceph_mount_info *cmount, const char *path, mode_t mode);
+int ceph_chmod(struct ceph_mount_info *cmount, const char *path, mode_t mode);
/**
* Change the mode bits (permissions) of an open file.
@@ -536,7 +537,7 @@ int ceph_chmod(class ceph_mount_info *cmount, const char *path, mode_t mode);
* @param mode the new permissions to set.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_fchmod(class ceph_mount_info *cmount, int fd, mode_t mode);
+int ceph_fchmod(struct ceph_mount_info *cmount, int fd, mode_t mode);
/**
* Change the ownership of a file/directory.
@@ -547,7 +548,7 @@ int ceph_fchmod(class ceph_mount_info *cmount, int fd, mode_t mode);
* @param gid the group id to set on the file/directory.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_chown(class ceph_mount_info *cmount, const char *path, int uid, int gid);
+int ceph_chown(struct ceph_mount_info *cmount, const char *path, int uid, int gid);
/**
* Change the ownership of a file from an open file descriptor.
@@ -558,7 +559,7 @@ int ceph_chown(class ceph_mount_info *cmount, const char *path, int uid, int gid
* @param gid the group id to set on the file/directory.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_fchown(class ceph_mount_info *cmount, int fd, int uid, int gid);
+int ceph_fchown(struct ceph_mount_info *cmount, int fd, int uid, int gid);
/**
* Change the ownership of a file/directory, don't follow symlinks.
@@ -569,7 +570,7 @@ int ceph_fchown(class ceph_mount_info *cmount, int fd, int uid, int gid);
* @param gid the group id to set on the file/directory.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_lchown(class ceph_mount_info *cmount, const char *path, int uid, int gid);
+int ceph_lchown(struct ceph_mount_info *cmount, const char *path, int uid, int gid);
/**
* Change file/directory last access and modification times.
@@ -579,7 +580,7 @@ int ceph_lchown(class ceph_mount_info *cmount, const char *path, int uid, int gi
* @param buf holding the access and modification times to set on the file.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_utime(class ceph_mount_info *cmount, const char *path, struct utimbuf *buf);
+int ceph_utime(struct ceph_mount_info *cmount, const char *path, struct utimbuf *buf);
/**
* Truncate the file to the given size. If this operation causes the
@@ -590,7 +591,7 @@ int ceph_utime(class ceph_mount_info *cmount, const char *path, struct utimbuf *
* @param size the new size of the file.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_truncate(class ceph_mount_info *cmount, const char *path, loff_t size);
+int ceph_truncate(struct ceph_mount_info *cmount, const char *path, int64_t size);
/**
* Make a block or character special file.
@@ -604,7 +605,7 @@ int ceph_truncate(class ceph_mount_info *cmount, const char *path, loff_t size);
* it is ignored.
* @returns 0 on success or negative error code on failure.
*/
-int ceph_mknod(class ceph_mount_info *cmount, const char *path, mode_t mode, dev_t rdev);
+int ceph_mknod(struct ceph_mount_info *cmount, const char *path, mode_t mode, dev_t rdev);
/**
* Create and/or open a file.
*
@@ -616,7 +617,7 @@ int ceph_mknod(class ceph_mount_info *cmount, const char *path, mode_t mode, dev
* is specified in the flags.
* @returns a non-negative file descriptor number on success or a negative error code on failure.
*/
-int ceph_open(class ceph_mount_info *cmount, const char *path, int flags, mode_t mode);
+int ceph_open(struct ceph_mount_info *cmount, const char *path, int flags, mode_t mode);
/**
* Create and/or open a file with a specific file layout.
@@ -633,7 +634,7 @@ int ceph_open(class ceph_mount_info *cmount, const char *path, int flags, mode_t
* @param data_pool name of target data pool name (optional, NULL or empty string for default)
* @returns a non-negative file descriptor number on success or a negative error code on failure.
*/
-int ceph_open_layout(class ceph_mount_info *cmount, const char *path, int flags,
+int ceph_open_layout(struct ceph_mount_info *cmount, const char *path, int flags,
mode_t mode, int stripe_unit, int stripe_count, int object_size,
const char *data_pool);
@@ -644,7 +645,7 @@ int ceph_open_layout(class ceph_mount_info *cmount, const char *path, int flags,
* @param fd the file descriptor referring to the open file.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_close(class ceph_mount_info *cmount, int fd);
+int ceph_close(struct ceph_mount_info *cmount, int fd);
/**
* Reposition the open file stream based on the given offset.
@@ -659,7 +660,7 @@ int ceph_close(class ceph_mount_info *cmount, int fd);
* SEEK_END: the offset is set to the end of the file plus @ref offset bytes.
* @returns 0 on success or a negative error code on failure.
*/
-loff_t ceph_lseek(class ceph_mount_info *cmount, int fd, loff_t offset, int whence);
+int64_t ceph_lseek(struct ceph_mount_info *cmount, int fd, int64_t offset, int whence);
/**
* Read data from the file.
*
@@ -671,7 +672,7 @@ loff_t ceph_lseek(class ceph_mount_info *cmount, int fd, loff_t offset, int when
* function reads from the current offset of the file descriptor.
* @returns the number of bytes read into buf, or a negative error code on failure.
*/
-int ceph_read(class ceph_mount_info *cmount, int fd, char *buf, loff_t size, loff_t offset);
+int ceph_read(struct ceph_mount_info *cmount, int fd, char *buf, int64_t size, int64_t offset);
/**
* Write data to a file.
@@ -684,8 +685,8 @@ int ceph_read(class ceph_mount_info *cmount, int fd, char *buf, loff_t size, lof
* function writes to the current offset of the file descriptor.
* @returns the number of bytes written, or a negative error code
*/
-int ceph_write(class ceph_mount_info *cmount, int fd, const char *buf, loff_t size,
- loff_t offset);
+int ceph_write(struct ceph_mount_info *cmount, int fd, const char *buf, int64_t size,
+ int64_t offset);
/**
* Truncate a file to the given size.
@@ -695,7 +696,7 @@ int ceph_write(class ceph_mount_info *cmount, int fd, const char *buf, loff_t si
* @param size the new size of the file
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_ftruncate(class ceph_mount_info *cmount, int fd, loff_t size);
+int ceph_ftruncate(struct ceph_mount_info *cmount, int fd, int64_t size);
/**
* Synchronize an open file to persistent media.
@@ -706,7 +707,7 @@ int ceph_ftruncate(class ceph_mount_info *cmount, int fd, loff_t size);
* or just data (1).
* @return 0 on success or a negative error code on failure.
*/
-int ceph_fsync(class ceph_mount_info *cmount, int fd, int syncdataonly);
+int ceph_fsync(struct ceph_mount_info *cmount, int fd, int syncdataonly);
/**
* Preallocate or release disk space for the file for the byte range.
@@ -724,7 +725,7 @@ int ceph_fsync(class ceph_mount_info *cmount, int fd, int syncdataonly);
* @return 0 on success or a negative error code on failure.
*/
int ceph_fallocate(struct ceph_mount_info *cmount, int fd, int mode,
- loff_t offset, loff_t length);
+ int64_t offset, int64_t length);
/**
* Get the open file's statistics.
@@ -735,7 +736,7 @@ int ceph_fallocate(struct ceph_mount_info *cmount, int fd, int mode,
* function.
* @returns 0 on success or a negative error code on failure
*/
-int ceph_fstat(class ceph_mount_info *cmount, int fd, struct stat *stbuf);
+int ceph_fstat(struct ceph_mount_info *cmount, int fd, struct stat *stbuf);
/** @} file */
@@ -756,7 +757,7 @@ int ceph_fstat(class ceph_mount_info *cmount, int fd, struct stat *stbuf);
* @param size the size of the pre-allocated buffer
* @returns the size of the value or a negative error code on failure.
*/
-int ceph_getxattr(class ceph_mount_info *cmount, const char *path, const char *name,
+int ceph_getxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
void *value, size_t size);
/**
@@ -772,7 +773,7 @@ int ceph_getxattr(class ceph_mount_info *cmount, const char *path, const char *n
* @param size the size of the pre-allocated buffer
* @returns the size of the value or a negative error code on failure.
*/
-int ceph_lgetxattr(class ceph_mount_info *cmount, const char *path, const char *name,
+int ceph_lgetxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
void *value, size_t size);
/**
@@ -784,7 +785,7 @@ int ceph_lgetxattr(class ceph_mount_info *cmount, const char *path, const char *
* @param size the size of the list buffer.
* @returns the size of the resulting list filled in.
*/
-int ceph_listxattr(class ceph_mount_info *cmount, const char *path, char *list, size_t size);
+int ceph_listxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size);
/**
* Get the list of extended attribute keys on a file, but do not follow symbolic links.
@@ -795,7 +796,7 @@ int ceph_listxattr(class ceph_mount_info *cmount, const char *path, char *list,
* @param size the size of the list buffer.
* @returns the size of the resulting list filled in.
*/
-int ceph_llistxattr(class ceph_mount_info *cmount, const char *path, char *list, size_t size);
+int ceph_llistxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size);
/**
* Remove an extended attribute from a file.
@@ -805,7 +806,7 @@ int ceph_llistxattr(class ceph_mount_info *cmount, const char *path, char *list,
* @param name the name of the extended attribute to remove.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_removexattr(class ceph_mount_info *cmount, const char *path, const char *name);
+int ceph_removexattr(struct ceph_mount_info *cmount, const char *path, const char *name);
/**
* Remove the extended attribute from a file, do not follow symbolic links.
@@ -815,7 +816,7 @@ int ceph_removexattr(class ceph_mount_info *cmount, const char *path, const char
* @param name the name of the extended attribute to remove.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_lremovexattr(class ceph_mount_info *cmount, const char *path, const char *name);
+int ceph_lremovexattr(struct ceph_mount_info *cmount, const char *path, const char *name);
/**
* Set an extended attribute on a file.
@@ -830,7 +831,7 @@ int ceph_lremovexattr(class ceph_mount_info *cmount, const char *path, const cha
* CEPH_XATTR_REPLACE: replace the extended attribute, Must already exist.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_setxattr(class ceph_mount_info *cmount, const char *path, const char *name,
+int ceph_setxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
const void *value, size_t size, int flags);
/**
@@ -846,7 +847,7 @@ int ceph_setxattr(class ceph_mount_info *cmount, const char *path, const char *n
* CEPH_XATTR_REPLACE: replace the extended attribute, Must already exist.
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_lsetxattr(class ceph_mount_info *cmount, const char *path, const char *name,
+int ceph_lsetxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
const void *value, size_t size, int flags);
/** @} xattr */
@@ -865,7 +866,7 @@ int ceph_lsetxattr(class ceph_mount_info *cmount, const char *path, const char *
* @param fh the open file descriptor referring to the file to get the striping unit of.
* @returns the striping unit of the file or a negative error code on failure.
*/
-int ceph_get_file_stripe_unit(class ceph_mount_info *cmount, int fh);
+int ceph_get_file_stripe_unit(struct ceph_mount_info *cmount, int fh);
/**
* Get the file striping unit.
@@ -874,7 +875,7 @@ int ceph_get_file_stripe_unit(class ceph_mount_info *cmount, int fh);
* @param path the path of the file/directory get the striping unit of.
* @returns the striping unit of the file or a negative error code on failure.
*/
-int ceph_get_path_stripe_unit(class ceph_mount_info *cmount, const char *path);
+int ceph_get_path_stripe_unit(struct ceph_mount_info *cmount, const char *path);
/**
* Get the file striping count from an open file descriptor.
@@ -883,7 +884,7 @@ int ceph_get_path_stripe_unit(class ceph_mount_info *cmount, const char *path);
* @param fh the open file descriptor referring to the file to get the striping count of.
* @returns the striping count of the file or a negative error code on failure.
*/
-int ceph_get_file_stripe_count(class ceph_mount_info *cmount, int fh);
+int ceph_get_file_stripe_count(struct ceph_mount_info *cmount, int fh);
/**
* Get the file striping count.
@@ -892,7 +893,7 @@ int ceph_get_file_stripe_count(class ceph_mount_info *cmount, int fh);
* @param path the path of the file/directory get the striping count of.
* @returns the striping count of the file or a negative error code on failure.
*/
-int ceph_get_path_stripe_count(class ceph_mount_info *cmount, const char *path);
+int ceph_get_path_stripe_count(struct ceph_mount_info *cmount, const char *path);
/**
* Get the file object size from an open file descriptor.
@@ -901,7 +902,7 @@ int ceph_get_path_stripe_count(class ceph_mount_info *cmount, const char *path);
* @param fh the open file descriptor referring to the file to get the object size of.
* @returns the object size of the file or a negative error code on failure.
*/
-int ceph_get_file_object_size(class ceph_mount_info *cmount, int fh);
+int ceph_get_file_object_size(struct ceph_mount_info *cmount, int fh);
/**
* Get the file object size.
@@ -910,7 +911,7 @@ int ceph_get_file_object_size(class ceph_mount_info *cmount, int fh);
* @param path the path of the file/directory get the object size of.
* @returns the object size of the file or a negative error code on failure.
*/
-int ceph_get_path_object_size(class ceph_mount_info *cmount, const char *path);
+int ceph_get_path_object_size(struct ceph_mount_info *cmount, const char *path);
/**
* Get the file pool information from an open file descriptor.
@@ -919,7 +920,7 @@ int ceph_get_path_object_size(class ceph_mount_info *cmount, const char *path);
* @param fh the open file descriptor referring to the file to get the pool information of.
* @returns the ceph pool id that the file is in
*/
-int ceph_get_file_pool(class ceph_mount_info *cmount, int fh);
+int ceph_get_file_pool(struct ceph_mount_info *cmount, int fh);
/**
* Get the file pool information.
@@ -928,7 +929,7 @@ int ceph_get_file_pool(class ceph_mount_info *cmount, int fh);
* @param path the path of the file/directory get the pool information of.
* @returns the ceph pool id that the file is in
*/
-int ceph_get_path_pool(class ceph_mount_info *cmount, const char *path);
+int ceph_get_path_pool(struct ceph_mount_info *cmount, const char *path);
/**
* Get the name of the pool a opened file is stored in,
@@ -942,7 +943,7 @@ int ceph_get_path_pool(class ceph_mount_info *cmount, const char *path);
* @param buflen size of the buffer
* @returns length in bytes of the pool name, or -ERANGE if the buffer is not large enough.
*/
-int ceph_get_file_pool_name(class ceph_mount_info *cmount, int fh, char *buf, size_t buflen);
+int ceph_get_file_pool_name(struct ceph_mount_info *cmount, int fh, char *buf, size_t buflen);
/**
* get the name of a pool by id
@@ -955,7 +956,7 @@ int ceph_get_file_pool_name(class ceph_mount_info *cmount, int fh, char *buf, si
* @param buflen size of the buffer
* @returns length in bytes of the pool name, or -ERANGE if the buffer is not large enough
*/
-int ceph_get_pool_name(class ceph_mount_info *cmount, int pool, char *buf, size_t buflen);
+int ceph_get_pool_name(struct ceph_mount_info *cmount, int pool, char *buf, size_t buflen);
/**
* Get the name of the pool a file is stored in
@@ -969,7 +970,7 @@ int ceph_get_pool_name(class ceph_mount_info *cmount, int pool, char *buf, size_
* @param buflen size of the buffer
* @returns length in bytes of the pool name, or -ERANGE if the buffer is not large enough.
*/
-int ceph_get_path_pool_name(class ceph_mount_info *cmount, const char *path, char *buf, size_t buflen);
+int ceph_get_path_pool_name(struct ceph_mount_info *cmount, const char *path, char *buf, size_t buflen);
/**
* Get the file layout from an open file descriptor.
@@ -982,7 +983,7 @@ int ceph_get_path_pool_name(class ceph_mount_info *cmount, const char *path, cha
* @param pg_pool where to store the ceph pool id that the file is in
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_get_file_layout(class ceph_mount_info *cmount, int fh, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool);
+int ceph_get_file_layout(struct ceph_mount_info *cmount, int fh, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool);
/**
* Get the file layout.
@@ -995,7 +996,7 @@ int ceph_get_file_layout(class ceph_mount_info *cmount, int fh, int *stripe_unit
* @param pg_pool where to store the ceph pool id that the file is in
* @returns 0 on success or a negative error code on failure.
*/
-int ceph_get_path_layout(class ceph_mount_info *cmount, const char *path, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool);
+int ceph_get_path_layout(struct ceph_mount_info *cmount, const char *path, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool);
/**
* Get the file replication information from an open file descriptor.
@@ -1004,7 +1005,7 @@ int ceph_get_path_layout(class ceph_mount_info *cmount, const char *path, int *s
* @param fh the open file descriptor referring to the file to get the replication information of.
* @returns the replication factor of the file.
*/
-int ceph_get_file_replication(class ceph_mount_info *cmount, int fh);
+int ceph_get_file_replication(struct ceph_mount_info *cmount, int fh);
/**
* Get the file replication information.
@@ -1013,7 +1014,7 @@ int ceph_get_file_replication(class ceph_mount_info *cmount, int fh);
* @param path the path of the file/directory get the replication information of.
* @returns the replication factor of the file.
*/
-int ceph_get_path_replication(class ceph_mount_info *cmount, const char *path);
+int ceph_get_path_replication(struct ceph_mount_info *cmount, const char *path);
/**
* Get the id of the named pool.
@@ -1022,7 +1023,7 @@ int ceph_get_path_replication(class ceph_mount_info *cmount, const char *path);
* @param pool_name the name of the pool.
* @returns the pool id, or a negative error code on failure.
*/
-int ceph_get_pool_id(class ceph_mount_info *cmount, const char *pool_name);
+int ceph_get_pool_id(struct ceph_mount_info *cmount, const char *pool_name);
/**
* Get the pool replication factor.
@@ -1031,7 +1032,7 @@ int ceph_get_pool_id(class ceph_mount_info *cmount, const char *pool_name);
* @param pool_id the pool id to look up
* @returns the replication factor, or a negative error code on failure.
*/
-int ceph_get_pool_replication(class ceph_mount_info *cmount, int pool_id);
+int ceph_get_pool_replication(struct ceph_mount_info *cmount, int pool_id);
/**
* Get the OSD address where the primary copy of a file stripe is located.
@@ -1045,7 +1046,7 @@ int ceph_get_pool_replication(class ceph_mount_info *cmount, int pool_id);
* @returns the size of the addressed filled into the @ref addr parameter, or a negative
* error code on failure.
*/
-int ceph_get_file_stripe_address(class ceph_mount_info *cmount, int fd, loff_t offset,
+int ceph_get_file_stripe_address(struct ceph_mount_info *cmount, int fd, int64_t offset,
struct sockaddr_storage *addr, int naddr);
/**
@@ -1061,8 +1062,8 @@ int ceph_get_file_stripe_address(class ceph_mount_info *cmount, int fd, loff_t o
* @returns the number of items stored in the output array, or -ERANGE if the
* array is not large enough.
*/
-int ceph_get_file_extent_osds(class ceph_mount_info *cmount, int fh,
- loff_t offset, loff_t *length, int *osds, int nosds);
+int ceph_get_file_extent_osds(struct ceph_mount_info *cmount, int fh,
+ int64_t offset, int64_t *length, int *osds, int nosds);
/**
* Get the fully qualified CRUSH location of an OSD.
@@ -1078,7 +1079,7 @@ int ceph_get_file_extent_osds(class ceph_mount_info *cmount, int fh,
* @returns the amount of bytes written into the buffer, or -ERANGE if the
* array is not large enough.
*/
-int ceph_get_osd_crush_location(class ceph_mount_info *cmount,
+int ceph_get_osd_crush_location(struct ceph_mount_info *cmount,
int osd, char *path, size_t len);
/**
@@ -1089,7 +1090,7 @@ int ceph_get_osd_crush_location(class ceph_mount_info *cmount,
* @param addr the OSD network address.
* @returns zero on success, other returns a negative error code.
*/
-int ceph_get_osd_addr(class ceph_mount_info *cmount, int osd,
+int ceph_get_osd_addr(struct ceph_mount_info *cmount, int osd,
struct sockaddr_storage *addr);
/**
@@ -1097,7 +1098,7 @@ int ceph_get_osd_addr(class ceph_mount_info *cmount, int osd,
* @param cmount the ceph mount handle.
* @returns the stripe unit granularity or a negative error code on failure.
*/
-int ceph_get_stripe_unit_granularity(class ceph_mount_info *cmount);
+int ceph_get_stripe_unit_granularity(struct ceph_mount_info *cmount);
/** @} filelayout */
@@ -1105,11 +1106,11 @@ int ceph_get_stripe_unit_granularity(class ceph_mount_info *cmount);
* No longer available. Do not use.
* These functions will return -EOPNOTSUPP.
*/
-int ceph_set_default_file_stripe_unit(class ceph_mount_info *cmount, int stripe);
-int ceph_set_default_file_stripe_count(class ceph_mount_info *cmount, int count);
-int ceph_set_default_object_size(class ceph_mount_info *cmount, int size);
-int ceph_set_default_preferred_pg(class ceph_mount_info *cmount, int osd);
-int ceph_set_default_file_replication(class ceph_mount_info *cmount, int replication);
+int ceph_set_default_file_stripe_unit(struct ceph_mount_info *cmount, int stripe);
+int ceph_set_default_file_stripe_count(struct ceph_mount_info *cmount, int count);
+int ceph_set_default_object_size(struct ceph_mount_info *cmount, int size);
+int ceph_set_default_preferred_pg(struct ceph_mount_info *cmount, int osd);
+int ceph_set_default_file_replication(struct ceph_mount_info *cmount, int replication);
/**
* Read from local replicas when possible.
@@ -1119,7 +1120,7 @@ int ceph_set_default_file_replication(class ceph_mount_info *cmount, int replica
* for reads.
* @returns 0
*/
-int ceph_localize_reads(class ceph_mount_info *cmount, int val);
+int ceph_localize_reads(struct ceph_mount_info *cmount, int val);
/**
* Get the osd id of the local osd (if any)
@@ -1128,7 +1129,7 @@ int ceph_localize_reads(class ceph_mount_info *cmount, int val);
* @returns the osd (if any) local to the node where this call is made, otherwise
* -1 is returned.
*/
-int ceph_get_local_osd(class ceph_mount_info *cmount);
+int ceph_get_local_osd(struct ceph_mount_info *cmount);
/** @} default_filelayout */
@@ -1140,7 +1141,7 @@ int ceph_get_local_osd(class ceph_mount_info *cmount);
* @returns the current capabilities issued to this client
* for the open file
*/
-int ceph_debug_get_fd_caps(class ceph_mount_info *cmount, int fd);
+int ceph_debug_get_fd_caps(struct ceph_mount_info *cmount, int fd);
/**
* Get the capabilities currently issued to the client.
@@ -1150,7 +1151,7 @@ int ceph_debug_get_fd_caps(class ceph_mount_info *cmount, int fd);
* @returns the current capabilities issued to this client
* for the file
*/
-int ceph_debug_get_file_caps(class ceph_mount_info *cmount, const char *path);
+int ceph_debug_get_file_caps(struct ceph_mount_info *cmount, const char *path);
#ifdef __cplusplus
}
diff --git a/src/include/compat.h b/src/include/compat.h
index a410acb..dc15533 100644
--- a/src/include/compat.h
+++ b/src/include/compat.h
@@ -13,10 +13,21 @@
#define CEPH_COMPAT_H
#if defined(__FreeBSD__)
-#define lseek64(fd, offset, whence) lseek(fd, offset, whence)
#define ENODATA 61
-#define TEMP_FAILURE_RETRY
#define MSG_MORE 0
#endif /* !__FreeBSD__ */
+#ifndef TEMP_FAILURE_RETRY
+#define TEMP_FAILURE_RETRY(expression) ({ \
+ typeof(expression) __result; \
+ do { \
+ __result = (expression); \
+ } while (__result == -1 && errno == EINTR); \
+ __result; })
+#endif
+
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define lseek64(fd, offset, whence) lseek(fd, offset, whence)
+#endif
+
#endif /* !CEPH_COMPAT_H */
diff --git a/src/include/encoding.h b/src/include/encoding.h
index fedc8b3..ddb94ec 100644
--- a/src/include/encoding.h
+++ b/src/include/encoding.h
@@ -16,7 +16,7 @@
#include "include/int_types.h"
-#include <tr1/memory>
+#include "include/memory.h"
#include "byteorder.h"
#include "buffer.h"
@@ -198,13 +198,13 @@ template<class A>
inline void encode_array_nohead(const A a[], int n, bufferlist &bl)
{
for (int i=0; i<n; i++)
- encode(a[n], bl);
+ encode(a[i], bl);
}
template<class A>
inline void decode_array_nohead(A a[], int n, bufferlist::iterator &p)
{
for (int i=0; i<n; i++)
- decode(a[n], p);
+ decode(a[i], p);
}
@@ -286,15 +286,38 @@ inline void decode(T &o, bufferlist& bl)
#include <deque>
#include <vector>
#include <string>
+#include <boost/optional.hpp>
#ifndef _BACKWARD_BACKWARD_WARNING_H
#define _BACKWARD_BACKWARD_WARNING_H // make gcc 4.3 shut up about hash_*
#endif
-#include <ext/hash_map>
-#include <ext/hash_set>
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
#include "triple.h"
+// boost optional
+template<typename T>
+inline void encode(const boost::optional<T> &p, bufferlist &bl)
+{
+ __u8 present = static_cast<bool>(p);
+ ::encode(present, bl);
+ if (p)
+ ::encode(p.get(), bl);
+}
+
+template<typename T>
+inline void decode(boost::optional<T> &p, bufferlist::iterator &bp)
+{
+ __u8 present;
+ ::decode(present, bp);
+ if (present) {
+ T t;
+ p = t;
+ ::decode(p.get(), bp);
+ }
+}
+
// pair
template<class A, class B>
inline void encode(const std::pair<A,B> &p, bufferlist &bl)
@@ -363,14 +386,14 @@ inline void decode(std::list<T>& ls, bufferlist::iterator& p)
}
template<class T>
-inline void encode(const std::list<std::tr1::shared_ptr<T> >& ls, bufferlist& bl)
+inline void encode(const std::list<ceph::shared_ptr<T> >& ls, bufferlist& bl)
{
// should i pre- or post- count?
if (!ls.empty()) {
unsigned pos = bl.length();
unsigned n = 0;
encode(n, bl);
- for (typename std::list<std::tr1::shared_ptr<T> >::const_iterator p = ls.begin(); p != ls.end(); ++p) {
+ for (typename std::list<ceph::shared_ptr<T> >::const_iterator p = ls.begin(); p != ls.end(); ++p) {
n++;
encode(**p, bl);
}
@@ -380,18 +403,18 @@ inline void encode(const std::list<std::tr1::shared_ptr<T> >& ls, bufferlist& bl
} else {
__u32 n = ls.size(); // FIXME: this is slow on a list.
encode(n, bl);
- for (typename std::list<std::tr1::shared_ptr<T> >::const_iterator p = ls.begin(); p != ls.end(); ++p)
+ for (typename std::list<ceph::shared_ptr<T> >::const_iterator p = ls.begin(); p != ls.end(); ++p)
encode(**p, bl);
}
}
template<class T>
-inline void decode(std::list<std::tr1::shared_ptr<T> >& ls, bufferlist::iterator& p)
+inline void decode(std::list<ceph::shared_ptr<T> >& ls, bufferlist::iterator& p)
{
__u32 n;
decode(n, p);
ls.clear();
while (n--) {
- std::tr1::shared_ptr<T> v(new T);
+ ceph::shared_ptr<T> v(new T);
decode(*v, p);
ls.push_back(v);
}
@@ -481,18 +504,18 @@ inline void decode_nohead(int len, std::vector<T>& v, bufferlist::iterator& p)
// vector (shared_ptr)
template<class T>
-inline void encode(const std::vector<std::tr1::shared_ptr<T> >& v, bufferlist& bl)
+inline void encode(const std::vector<ceph::shared_ptr<T> >& v, bufferlist& bl)
{
__u32 n = v.size();
encode(n, bl);
- for (typename std::vector<std::tr1::shared_ptr<T> >::const_iterator p = v.begin(); p != v.end(); ++p)
+ for (typename std::vector<ceph::shared_ptr<T> >::const_iterator p = v.begin(); p != v.end(); ++p)
if (*p)
encode(**p, bl);
else
encode(T(), bl);
}
template<class T>
-inline void decode(std::vector<std::tr1::shared_ptr<T> >& v, bufferlist::iterator& p)
+inline void decode(std::vector<ceph::shared_ptr<T> >& v, bufferlist::iterator& p)
{
__u32 n;
decode(n, p);
@@ -624,19 +647,19 @@ inline void decode(std::multimap<T,U>& m, bufferlist::iterator& p)
}
}
-// hash_map
+// ceph::unordered_map
template<class T, class U>
-inline void encode(const __gnu_cxx::hash_map<T,U>& m, bufferlist& bl)
+inline void encode(const unordered_map<T,U>& m, bufferlist& bl)
{
__u32 n = m.size();
encode(n, bl);
- for (typename __gnu_cxx::hash_map<T,U>::const_iterator p = m.begin(); p != m.end(); ++p) {
+ for (typename unordered_map<T,U>::const_iterator p = m.begin(); p != m.end(); ++p) {
encode(p->first, bl);
encode(p->second, bl);
}
}
template<class T, class U>
-inline void decode(__gnu_cxx::hash_map<T,U>& m, bufferlist::iterator& p)
+inline void decode(unordered_map<T,U>& m, bufferlist::iterator& p)
{
__u32 n;
decode(n, p);
@@ -648,17 +671,17 @@ inline void decode(__gnu_cxx::hash_map<T,U>& m, bufferlist::iterator& p)
}
}
-// hash_set
+// ceph::unordered_set
template<class T>
-inline void encode(const __gnu_cxx::hash_set<T>& m, bufferlist& bl)
+inline void encode(const ceph::unordered_set<T>& m, bufferlist& bl)
{
__u32 n = m.size();
encode(n, bl);
- for (typename __gnu_cxx::hash_set<T>::const_iterator p = m.begin(); p != m.end(); ++p)
+ for (typename ceph::unordered_set<T>::const_iterator p = m.begin(); p != m.end(); ++p)
encode(*p, bl);
}
template<class T>
-inline void decode(__gnu_cxx::hash_set<T>& m, bufferlist::iterator& p)
+inline void decode(ceph::unordered_set<T>& m, bufferlist::iterator& p)
{
__u32 n;
decode(n, p);
@@ -708,6 +731,8 @@ inline void decode(std::deque<T>& ls, bufferlist::iterator& p)
__u8 struct_v = v, struct_compat = compat; \
::encode(struct_v, bl); \
::encode(struct_compat, bl); \
+ buffer::list::iterator struct_compat_it = bl.end(); \
+ struct_compat_it.advance(-1); \
ceph_le32 struct_len; \
struct_len = 0; \
::encode(struct_len, bl); \
@@ -719,11 +744,18 @@ inline void decode(std::deque<T>& ls, bufferlist::iterator& p)
* finish encoding block
*
* @param bl bufferlist we were encoding to
+ * @param new_struct_compat struct-compat value to use
*/
-#define ENCODE_FINISH(bl) \
+#define ENCODE_FINISH_NEW_COMPAT(bl, new_struct_compat) \
} while (false); \
struct_len = bl.length() - struct_len_it.get_off() - sizeof(struct_len); \
- struct_len_it.copy_in(4, (char *)&struct_len);
+ struct_len_it.copy_in(4, (char *)&struct_len); \
+ if (new_struct_compat) { \
+ struct_compat = new_struct_compat; \
+ struct_compat_it.copy_in(1, (char *)&struct_compat); \
+ }
+
+#define ENCODE_FINISH(bl) ENCODE_FINISH_NEW_COMPAT(bl, 0)
#define DECODE_ERR_VERSION(func, v) \
"" #func " unknown encoding version > " #v
diff --git a/src/include/hash_namespace.h b/src/include/hash_namespace.h
new file mode 100644
index 0000000..59a4dff
--- /dev/null
+++ b/src/include/hash_namespace.h
@@ -0,0 +1,24 @@
+#ifndef CEPH_HASH_NAMESPACE_H
+#define CEPH_HASH_NAMESPACE_H
+
+#include <ciso646>
+
+#ifdef _LIBCPP_VERSION
+
+#include <functional>
+
+#define CEPH_HASH_NAMESPACE_START namespace std {
+#define CEPH_HASH_NAMESPACE_END }
+#define CEPH_HASH_NAMESPACE std
+
+#else
+
+#include <tr1/functional>
+
+#define CEPH_HASH_NAMESPACE_START namespace std { namespace tr1 {
+#define CEPH_HASH_NAMESPACE_END }}
+#define CEPH_HASH_NAMESPACE std::tr1
+
+#endif
+
+#endif
diff --git a/src/include/int_types.h b/src/include/int_types.h
index f290f9d..98220e9 100644
--- a/src/include/int_types.h
+++ b/src/include/int_types.h
@@ -3,7 +3,7 @@
#include "acconfig.h"
-#if defined(__linux__)
+#ifdef HAVE_LINUX_TYPES_H
#include <linux/types.h>
#endif
@@ -50,17 +50,41 @@
# endif
#endif
-#if defined(__FreeBSD__)
+#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
+#endif
-typedef int8_t __s8;
+#ifndef HAVE___U8
typedef uint8_t __u8;
-typedef int16_t __s16;
+#endif
+
+#ifndef HAVE___S8
+typedef int8_t __s8;
+#endif
+
+#ifndef HAVE___U16
typedef uint16_t __u16;
-typedef int32_t __s32;
+#endif
+
+#ifndef HAVE___S16
+typedef int16_t __s16;
+#endif
+
+#ifndef HAVE___U32
typedef uint32_t __u32;
-typedef int64_t __s64;
+#endif
+
+#ifndef HAVE___S32
+typedef int32_t __s32;
+#endif
+
+#ifndef HAVE___U64
typedef uint64_t __u64;
+#endif
+
+#ifndef HAVE___S64
+typedef int64_t __s64;
+#endif
#define __bitwise__
@@ -72,4 +96,3 @@ typedef __u64 __bitwise__ __le64;
typedef __u64 __bitwise__ __be64;
#endif
-#endif
diff --git a/src/include/linux_fiemap.h b/src/include/linux_fiemap.h
index 352208b..1811048 100644
--- a/src/include/linux_fiemap.h
+++ b/src/include/linux_fiemap.h
@@ -18,6 +18,8 @@
#include <sys/types.h>
#endif
+#include "include/int_types.h"
+
struct fiemap_extent {
__u64 fe_logical; /* logical offset in bytes for the start of
* the extent from the beginning of the file */
diff --git a/src/include/memory.h b/src/include/memory.h
new file mode 100644
index 0000000..ee28d69
--- /dev/null
+++ b/src/include/memory.h
@@ -0,0 +1,26 @@
+#ifndef CEPH_MEMORY_H
+#define CEPH_MEMORY_H
+
+#include <ciso646>
+
+#ifdef _LIBCPP_VERSION
+
+#include <memory>
+
+namespace ceph {
+ using std::shared_ptr;
+ using std::weak_ptr;
+}
+
+#else
+
+#include <tr1/memory>
+
+namespace ceph {
+ using std::tr1::shared_ptr;
+ using std::tr1::weak_ptr;
+}
+
+#endif
+
+#endif
diff --git a/src/include/object.h b/src/include/object.h
index f81683f..d9fc275 100644
--- a/src/include/object.h
+++ b/src/include/object.h
@@ -22,8 +22,8 @@
#include <iomanip>
using namespace std;
-#include <ext/hash_map>
-using namespace __gnu_cxx;
+#include "include/unordered_map.h"
+#include "include/hash_namespace.h"
#include "hash.h"
#include "encoding.h"
@@ -78,7 +78,7 @@ inline ostream& operator<<(ostream& out, const object_t& o) {
return out << o.name;
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<object_t> {
size_t operator()(const object_t& r) const {
//static hash<string> H;
@@ -86,7 +86,7 @@ namespace __gnu_cxx {
return ceph_str_hash_linux(r.name.c_str(), r.name.length());
}
};
-}
+CEPH_HASH_NAMESPACE_END
struct file_object_t {
@@ -179,7 +179,7 @@ inline bool operator<=(const sobject_t &l, const sobject_t &r) {
inline ostream& operator<<(ostream& out, const sobject_t &o) {
return out << o.oid << "/" << o.snap;
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<sobject_t> {
size_t operator()(const sobject_t &r) const {
static hash<object_t> H;
@@ -187,6 +187,6 @@ namespace __gnu_cxx {
return H(r.oid) ^ I(r.snap);
}
};
-}
+CEPH_HASH_NAMESPACE_END
#endif
diff --git a/src/include/on_exit.h b/src/include/on_exit.h
new file mode 100644
index 0000000..6510fef
--- /dev/null
+++ b/src/include/on_exit.h
@@ -0,0 +1,49 @@
+#ifndef CEPH_ON_EXIT_H
+#define CEPH_ON_EXIT_H
+
+#include <pthread.h>
+#include <assert.h>
+#include <vector>
+
+/*
+ * Create a static instance at the file level to get callbacks called when the
+ * process exits via main() or exit().
+ */
+
+class OnExitManager {
+ public:
+ typedef void (*callback_t)(void *arg);
+
+ OnExitManager() {
+ int ret = pthread_mutex_init(&lock_, NULL);
+ assert(ret == 0);
+ }
+
+ ~OnExitManager() {
+ pthread_mutex_lock(&lock_);
+ std::vector<struct cb>::iterator it;
+ for (it = funcs_.begin(); it != funcs_.end(); it++) {
+ it->func(it->arg);
+ }
+ funcs_.clear();
+ pthread_mutex_unlock(&lock_);
+ }
+
+ void add_callback(callback_t func, void *arg) {
+ pthread_mutex_lock(&lock_);
+ struct cb callback = { func, arg };
+ funcs_.push_back(callback);
+ pthread_mutex_unlock(&lock_);
+ }
+
+ private:
+ struct cb {
+ callback_t func;
+ void *arg;
+ };
+
+ std::vector<struct cb> funcs_;
+ pthread_mutex_t lock_;
+};
+
+#endif
diff --git a/src/include/rados.h b/src/include/rados.h
index c410f80..59cc77b 100644
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -64,9 +64,13 @@ struct ceph_pg {
/*
* pg pool types
+ *
+ * NOTE: These map 1:1 on to the pg_pool_t::TYPE_* values. They are
+ * duplicated here only for CrushCompiler's benefit.
*/
-#define CEPH_PG_TYPE_REP 1
-#define CEPH_PG_TYPE_RAID4 2
+#define CEPH_PG_TYPE_REPLICATED 1
+/* #define CEPH_PG_TYPE_RAID4 2 never implemented */
+#define CEPH_PG_TYPE_ERASURE 3
/*
* stable_mod func is used to control number of placement groups.
@@ -153,6 +157,7 @@ extern const char *ceph_osd_state_name(int s);
#define CEPH_OSD_OP_MODE_WR 0x2000
#define CEPH_OSD_OP_MODE_RMW 0x3000
#define CEPH_OSD_OP_MODE_SUB 0x4000
+#define CEPH_OSD_OP_MODE_CACHE 0x8000
#define CEPH_OSD_OP_TYPE 0x0f00
#define CEPH_OSD_OP_TYPE_LOCK 0x0100
@@ -183,6 +188,9 @@ enum {
CEPH_OSD_OP_LIST_SNAPS = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 10,
+ /* sync */
+ CEPH_OSD_OP_SYNC_READ = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 11,
+
/* write */
CEPH_OSD_OP_WRITE = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 1,
CEPH_OSD_OP_WRITEFULL = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 2,
@@ -217,11 +225,18 @@ enum {
CEPH_OSD_OP_OMAPRMKEYS = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 24,
CEPH_OSD_OP_OMAP_CMP = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 25,
+ /* tiering */
CEPH_OSD_OP_COPY_FROM = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 26,
CEPH_OSD_OP_COPY_GET_CLASSIC = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 27,
CEPH_OSD_OP_UNDIRTY = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 28,
CEPH_OSD_OP_ISDIRTY = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 29,
CEPH_OSD_OP_COPY_GET = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 30,
+ CEPH_OSD_OP_CACHE_FLUSH = CEPH_OSD_OP_MODE_CACHE | CEPH_OSD_OP_TYPE_DATA | 31,
+ CEPH_OSD_OP_CACHE_EVICT = CEPH_OSD_OP_MODE_CACHE | CEPH_OSD_OP_TYPE_DATA | 32,
+ CEPH_OSD_OP_CACHE_TRY_FLUSH = CEPH_OSD_OP_MODE_CACHE | CEPH_OSD_OP_TYPE_DATA | 33,
+
+ /* convert tmap to omap */
+ CEPH_OSD_OP_TMAP2OMAP = CEPH_OSD_OP_MODE_RMW | CEPH_OSD_OP_TYPE_DATA | 34,
/** multi **/
CEPH_OSD_OP_CLONERANGE = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_MULTI | 1,
@@ -266,6 +281,8 @@ enum {
/** pg **/
CEPH_OSD_OP_PGLS = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_PG | 1,
CEPH_OSD_OP_PGLS_FILTER = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_PG | 2,
+ CEPH_OSD_OP_PG_HITSET_LS = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_PG | 3,
+ CEPH_OSD_OP_PG_HITSET_GET = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_PG | 4,
};
static inline int ceph_osd_op_type_lock(int op)
@@ -306,6 +323,10 @@ static inline int ceph_osd_op_mode_modify(int op)
{
return op & CEPH_OSD_OP_MODE_WR;
}
+static inline int ceph_osd_op_mode_cache(int op)
+{
+ return op & CEPH_OSD_OP_MODE_CACHE;
+}
/*
* note that the following tmap stuff is also defined in the ceph librados.h
@@ -340,6 +361,10 @@ enum {
CEPH_OSD_FLAG_EXEC_PUBLIC = 0x1000, /* DEPRECATED op may exec (public) */
CEPH_OSD_FLAG_LOCALIZE_READS = 0x2000, /* read from nearby replica, if any */
CEPH_OSD_FLAG_RWORDERED = 0x4000, /* order wrt concurrent reads */
+ CEPH_OSD_FLAG_IGNORE_CACHE = 0x8000, /* ignore cache logic */
+ CEPH_OSD_FLAG_SKIPRWLOCKS = 0x10000, /* skip rw locks */
+ CEPH_OSD_FLAG_IGNORE_OVERLAY =0x20000, /* ignore pool overlay */
+ CEPH_OSD_FLAG_FLUSH = 0x40000, /* this is part of flush */
};
enum {
@@ -366,6 +391,16 @@ enum {
CEPH_OSD_CMPXATTR_MODE_U64 = 2
};
+enum {
+ CEPH_OSD_COPY_FROM_FLAG_FLUSH = 1, /* part of a flush operation */
+ CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY = 2, /* ignore pool overlay */
+ CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE = 4, /* ignore osd cache logic */
+};
+
+enum {
+ CEPH_OSD_TMAP2OMAP_NULLOK = 1,
+};
+
/*
* an individual object operation. each may be accompanied by some data
* payload
@@ -417,7 +452,14 @@ struct ceph_osd_op {
struct {
__le64 snapid;
__le64 src_version;
+ __u8 flags;
} __attribute__ ((packed)) copy_from;
+ struct {
+ struct ceph_timespec stamp;
+ } __attribute__ ((packed)) hit_set_get;
+ struct {
+ __u8 flags;
+ } __attribute__ ((packed)) tmap2omap;
};
__le32 payload_len;
} __attribute__ ((packed));
diff --git a/src/include/rados/buffer.h b/src/include/rados/buffer.h
index 0b497a7..4c275ca 100644
--- a/src/include/rados/buffer.h
+++ b/src/include/rados/buffer.h
@@ -28,18 +28,6 @@
#include <stdio.h>
-#ifdef DARWIN
-
-#ifndef MAP_ANON
-#define MAP_ANON 0x1000
-#endif
-#ifndef O_DIRECTORY
-#define O_DIRECTORY 0x100000
-void *valloc(size_t);
-#endif
-
-#endif
-
#if defined(__linux__) // For malloc(2).
#include <malloc.h>
#endif
@@ -101,6 +89,10 @@ public:
private:
char buf[256];
};
+ struct error_code : public malformed_input {
+ explicit error_code(int error);
+ int code;
+ };
/// total bytes allocated
@@ -116,6 +108,10 @@ public:
/// enable/disable tracking of cached crcs
static void track_cached_crc(bool b);
+ /// count of calls to buffer::ptr::c_str()
+ static int get_c_str_accesses();
+ /// enable/disable tracking of buffer::ptr::c_str() calls
+ static void track_c_str(bool b);
private:
@@ -133,6 +129,7 @@ private:
class raw_posix_aligned;
class raw_hack_aligned;
class raw_char;
+ class raw_pipe;
friend std::ostream& operator<<(std::ostream& out, const raw &r);
@@ -148,8 +145,8 @@ public:
static raw* claim_malloc(unsigned len, char *buf);
static raw* create_static(unsigned len, char *buf);
static raw* create_page_aligned(unsigned len);
-
-
+ static raw* create_zero_copy(unsigned len, int fd, int64_t *offset);
+
/*
* a buffer pointer. references (a subsequence of) a raw buffer.
*/
@@ -206,9 +203,12 @@ public:
memcpy(dest, c_str()+o, l);
}
+ bool can_zero_copy() const;
+ int zero_copy_to_fd(int fd, int64_t *offset) const;
+
unsigned wasted();
- int cmp(const ptr& o);
+ int cmp(const ptr& o) const;
bool is_zero() const;
// modifiers
@@ -305,6 +305,7 @@ public:
private:
mutable iterator last_p;
+ int zero_copy_to_fd(int fd) const;
public:
// cons/des
@@ -342,6 +343,7 @@ public:
}
bool contents_equal(buffer::list& other);
+ bool can_zero_copy() const;
bool is_page_aligned() const;
bool is_n_page_sized() const;
@@ -379,6 +381,7 @@ public:
bool is_contiguous();
void rebuild();
+ void rebuild(ptr& nb);
void rebuild_page_aligned();
// sort-of-like-assignment-op
@@ -429,8 +432,10 @@ public:
void hexdump(std::ostream &out) const;
int read_file(const char *fn, std::string *error);
ssize_t read_fd(int fd, size_t len);
+ int read_fd_zero_copy(int fd, size_t len);
int write_file(const char *fn, int mode=0644);
int write_fd(int fd) const;
+ int write_fd_zero_copy(int fd) const;
uint32_t crc32c(uint32_t crc) const;
};
diff --git a/src/include/rados/librados.h b/src/include/rados/librados.h
index a67b85e..7dd1748 100644
--- a/src/include/rados/librados.h
+++ b/src/include/rados/librados.h
@@ -1,3 +1,17 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2012 Sage Weil <sage at newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
#ifndef CEPH_LIBRADOS_H
#define CEPH_LIBRADOS_H
@@ -14,6 +28,8 @@ extern "C" {
#include <string.h>
#include "rados_types.h"
+#include <sys/time.h>
+
#ifndef CEPH_OSD_TMAP_SET
/* These are also defined in rados.h and objclass.h. Keep them in sync! */
#define CEPH_OSD_TMAP_HDR 'h'
@@ -37,9 +53,11 @@ extern "C" {
*/
#define LIBRADOS_LOCK_FLAG_RENEW 0x1
+#define LIBRADOS_CREATE_EXCLUSIVE 1
+#define LIBRADOS_CREATE_IDEMPOTENT 0
+
/**
* @defgroup librados_h_xattr_comp xattr comparison operations
- * @note BUG: there's no way to use these in the C api
* @{
*/
/** @cond TODO_enums_not_yet_in_asphyxiate */
@@ -164,6 +182,23 @@ struct rados_cluster_stat_t {
};
/**
+ * @typedef rados_write_op_t
+ *
+ * An object write operation stores a number of operations which can be
+ * executed atomically. For usage, see:
+ * - Creation and deletion: rados_create_write_op() rados_release_write_op()
+ * - Extended attribute manipulation: rados_write_op_cmpxattr()
+ * rados_write_op_cmpxattr(), rados_write_op_setxattr(),
+ * rados_write_op_rmxattr()
+ * - Creating objects: rados_write_op_create()
+ * - IO on objects: rados_write_op_append(), rados_write_op_write(), rados_write_op_zero
+ * rados_write_op_write_full(), rados_write_op_remove, rados_write_op_truncate(),
+ * rados_write_op_zero()
+ * - Performing the operation: rados_write_op_operate(), rados_aio_write_op_operate()
+ */
+typedef void *rados_write_op_t;
+
+/**
* Get the version of librados.
*
* The version number is major.minor.extra. Note that this is
@@ -702,6 +737,23 @@ void rados_ioctx_set_namespace(rados_ioctx_t io, const char *nspace);
int rados_objects_list_open(rados_ioctx_t io, rados_list_ctx_t *ctx);
/**
+ * Return hash position of iterator, rounded to the current PG
+ *
+ * @param ctx iterator marking where you are in the listing
+ * @returns current hash position, rounded to the current pg
+ */
+uint32_t rados_objects_list_get_pg_hash_position(rados_list_ctx_t ctx);
+
+/**
+ * Reposition object iterator to a different hash position
+ *
+ * @param ctx iterator marking where you are in the listing
+ * @param pos hash position to move to
+ * @returns actual (rounded) position we moved to
+ */
+uint32_t rados_objects_list_seek(rados_list_ctx_t ctx, uint32_t pos);
+
+/**
* Get the next object name and locator in the pool
*
* *entry and *key are valid until next call to rados_objects_list_*
@@ -927,6 +979,7 @@ uint64_t rados_get_last_version(rados_ioctx_t io);
/**
* Write data to an object
*
+ * @note This will never return a positive value not equal to len.
* @param io the io context in which the write will occur
* @param oid name of the object
* @param buf data to write
@@ -1216,7 +1269,7 @@ int rados_tmap_get(rados_ioctx_t io, const char *o, char *buf, size_t buflen);
* operations on an object atomically. These plugins are called
* classes. This function allows librados users to call the custom
* methods. The input and output formats are defined by the class.
- * Classes in ceph.git can be found in src/cls_*.cc
+ * Classes in ceph.git can be found in src/cls subdirectories
*
* @param io the context in which to call the method
* @param oid the object to call the method on
@@ -1630,6 +1683,153 @@ int rados_unwatch(rados_ioctx_t io, const char *o, uint64_t handle);
*/
int rados_notify(rados_ioctx_t io, const char *o, uint64_t ver, const char *buf, int buf_len);
+/** @} Atomic write operations */
+
+/**
+ * Create a new rados_write_op_t write operation. This will store all actions
+ * to be performed atomically. You must call rados_release_write_op when you are
+ * finished with it.
+ *
+ * @returns non-NULL on success, NULL on memory allocation error.
+ */
+rados_write_op_t rados_create_write_op();
+
+/**
+ * Free a rados_write_op_t, must be called when you're done with it.
+ * @param write_op operation to deallocate, created with rados_create_write_op
+ */
+void rados_release_write_op(rados_write_op_t write_op);
+
+/**
+ * Ensure that the object exists before writing
+ * @param write_op operation to add this action to
+ */
+void rados_write_op_assert_exists(rados_write_op_t write_op);
+
+/**
+ * Ensure that given xattr satisfies comparison
+ * @param write_op operation to add this action to
+ * @param name name of the xattr to look up
+ * @param comparison_operator currently undocumented, look for
+ * LIBRADOS_CMPXATTR_OP_EQ in librados.h
+ * @param value buffer to compare actual xattr value to
+ * @param value_len length of buffer to compare actual xattr value to
+ */
+void rados_write_op_cmpxattr(rados_write_op_t write_op,
+ const char *name,
+ uint8_t comparison_operator,
+ const char *value,
+ size_t value_len);
+
+/**
+ * Set an xattr
+ * @param write_op operation to add this action to
+ * @param name name of the xattr
+ * @param value buffer to set xattr to
+ * @param value_len length of buffer to set xattr to
+ */
+void rados_write_op_setxattr(rados_write_op_t write_op,
+ const char *name,
+ const char *value,
+ size_t value_len);
+
+/**
+ * Remove an xattr
+ * @param write_op operation to add this action to
+ * @param name name of the xattr to remove
+ */
+void rados_write_op_rmxattr(rados_write_op_t write_op, const char *name);
+
+/**
+ * Create the object
+ * @param write_op operation to add this action to
+ * @param exclusive set to either LIBRADOS_CREATE_EXCLUSIVE or
+ LIBRADOS_CREATE_IDEMPOTENT
+ * will error if the object already exists.
+ */
+void rados_write_op_create(rados_write_op_t write_op,
+ int exclusive,
+ const char* category);
+
+/**
+ * Write to offset
+ * @param write_op operation to add this action to
+ * @param offset offset to write to
+ * @param buffer bytes to write
+ * @param len length of buffer
+ */
+void rados_write_op_write(rados_write_op_t write_op,
+ const char *buffer,
+ size_t len,
+ uint64_t offset);
+
+/**
+ * Write whole object, atomically replacing it.
+ * @param write_op operation to add this action to
+ * @param buffer bytes to write
+ * @param len length of buffer
+ */
+void rados_write_op_write_full(rados_write_op_t write_op,
+ const char *buffer,
+ size_t len);
+
+/**
+ * Append to end of object.
+ * @param write_op operation to add this action to
+ * @param buffer bytes to write
+ * @param len length of buffer
+ */
+void rados_write_op_append(rados_write_op_t write_op,
+ const char *buffer,
+ size_t len);
+/**
+ * Remove object
+ * @param write_op operation to add this action to
+ */
+void rados_write_op_remove(rados_write_op_t write_op);
+
+/**
+ * Truncate an object
+ * @param write_op operation to add this action to
+ * @offset Offset to truncate to
+ */
+void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset);
+
+/**
+ * Zero part of an object
+ * @param write_op operation to add this action to
+ * @offset Offset to zero
+ * @len length to zero
+ */
+void rados_write_op_zero(rados_write_op_t write_op,
+ uint64_t offset,
+ uint64_t len);
+/**
+ * Perform a write operation synchronously
+ * @param write_op operation to perform
+ * @io the ioctx that the object is in
+ * @oid the object id
+ * @mtime the time to set the mtime to, NULL for the current time
+ */
+int rados_write_op_operate(rados_write_op_t write_op,
+ rados_ioctx_t io,
+ const char *oid,
+ time_t *mtime);
+/**
+ * Perform a write operation asynchronously
+ * @param write_op operation to perform
+ * @io the ioctx that the object is in
+ * @param completion what to do when operation has been attempted
+ * @oid the object id
+ * @mtime the time to set the mtime to, NULL for the current time
+ */
+int rados_aio_write_op_operate(rados_write_op_t write_op,
+ rados_ioctx_t io,
+ rados_completion_t completion,
+ const char *oid,
+ time_t *mtime);
+
+
/** @} Watch/Notify */
/**
diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index c8de9f9..6b059fe 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -6,9 +6,9 @@
#include <list>
#include <map>
#include <set>
-#include <tr1/memory>
#include <vector>
#include <utility>
+#include "memory.h"
#include "buffer.h"
#include "librados.h"
@@ -71,9 +71,16 @@ namespace librados
ObjectIterator &operator++(); // Preincrement
ObjectIterator operator++(int); // Postincrement
friend class IoCtx;
+
+ /// get current hash position of the iterator, rounded to the current pg
+ uint32_t get_pg_hash_position() const;
+
+ /// move the iterator to a given hash position. this may (will!) be rounded to the nearest pg.
+ uint32_t seek(uint32_t pos);
+
private:
void get_next();
- std::tr1::shared_ptr < ObjListCtx > ctx;
+ ceph::shared_ptr < ObjListCtx > ctx;
std::pair<std::string, std::string> cur_obj;
};
@@ -96,7 +103,7 @@ namespace librados
bool is_complete_and_cb();
bool is_safe_and_cb();
int get_return_value();
- int get_version(); ///< DEPRECATED get_version() only returns 32-bits
+ int get_version() __attribute__ ((deprecated));
uint64_t get_version64();
void release();
AioCompletionImpl *pc;
@@ -137,12 +144,24 @@ namespace librados
* ORDER_READS_WRITES will order reads the same way writes are
* ordered (e.g., waiting for degraded objects). In particular, it
* will make a write followed by a read sequence be preserved.
+ *
+ * IGNORE_CACHE will skip the caching logic on the OSD that normally
+ * handles promotion of objects between tiers. This allows an operation
+ * to operate (or read) the cached (or uncached) object, even if it is
+ * not coherent.
+ *
+ * IGNORE_OVERLAY will ignore the pool overlay tiering metadata and
+ * process the op directly on the destination pool. This is useful
+ * for CACHE_FLUSH and CACHE_EVICT operations.
*/
enum ObjectOperationGlobalFlags {
OPERATION_NOFLAG = 0,
OPERATION_BALANCE_READS = 1,
OPERATION_LOCALIZE_READS = 2,
OPERATION_ORDER_READS_WRITES = 4,
+ OPERATION_IGNORE_CACHE = 8,
+ OPERATION_SKIPRWLOCKS = 16,
+ OPERATION_IGNORE_OVERLAY = 32,
};
/*
@@ -281,7 +300,8 @@ namespace librados
* @param src_ioctx ioctx for the source object
* @param version current version of the source object
*/
- void copy_from(const std::string& src, const IoCtx& src_ioctx, uint64_t src_version);
+ void copy_from(const std::string& src, const IoCtx& src_ioctx,
+ uint64_t src_version);
/**
* undirty an object
@@ -416,6 +436,31 @@ namespace librados
*/
void is_dirty(bool *isdirty, int *prval);
+ /**
+ * flush a cache tier object to backing tier; will block racing
+ * updates.
+ *
+ * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
+ * triggering a promotion.
+ */
+ void cache_flush();
+
+ /**
+ * Flush a cache tier object to backing tier; will EAGAIN if we race
+ * with an update. Must be used with the SKIPRWLOCKS flag.
+ *
+ * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
+ * triggering a promotion.
+ */
+ void cache_try_flush();
+
+ /**
+ * evict a clean cache tier object
+ *
+ * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
+ * triggering a promote on the OSD (that is then evicted).
+ */
+ void cache_evict();
};
/* IoCtx : This is a context in which we can perform I/O.
@@ -505,6 +550,7 @@ namespace librados
*/
int tmap_put(const std::string& oid, bufferlist& bl);
int tmap_get(const std::string& oid, bufferlist& bl);
+ int tmap_to_omap(const std::string& oid, bool nullok=false);
int omap_get_vals(const std::string& oid,
const std::string& start_after,
@@ -584,9 +630,34 @@ namespace librados
std::list<librados::locker_t> *lockers);
+ /// Start enumerating objects for a pool
ObjectIterator objects_begin();
+ /// Start enumerating objects for a pool starting from a hash position
+ ObjectIterator objects_begin(uint32_t start_hash_position);
+ /// Iterator indicating the end of a pool
const ObjectIterator& objects_end() const;
+ /**
+ * List available hit set objects
+ *
+ * @param uint32_t [in] hash position to query
+ * @param c [in] completion
+ * @param pls [out] list of available intervals
+ */
+ int hit_set_list(uint32_t hash, AioCompletion *c,
+ std::list< std::pair<time_t, time_t> > *pls);
+
+ /**
+ * Retrieve hit set for a given hash, and time
+ *
+ * @param uint32_t [in] hash position
+ * @param c [in] completion
+ * @param stamp [in] time interval that falls within the hit set's interval
+ * @param pbl [out] buffer to store the result in
+ */
+ int hit_set_get(uint32_t hash, AioCompletion *c, time_t stamp,
+ bufferlist *pbl);
+
uint64_t get_last_version();
int aio_read(const std::string& oid, AioCompletion *c,
@@ -661,6 +732,11 @@ namespace librados
*/
int aio_remove(const std::string& oid, AioCompletion *c);
+ /**
+ * Wait for all currently pending aio writes to be safe.
+ *
+ * @returns 0 on success, negative error code on failure
+ */
int aio_flush();
/**
@@ -682,6 +758,7 @@ namespace librados
int operate(const std::string& oid, ObjectWriteOperation *op);
int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl);
int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op);
+ int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags);
/**
* Schedule an async write operation with explicit snapshot parameters
*
@@ -701,8 +778,14 @@ namespace librados
std::vector<snap_t>& snaps);
int aio_operate(const std::string& oid, AioCompletion *c,
ObjectReadOperation *op, bufferlist *pbl);
+
int aio_operate(const std::string& oid, AioCompletion *c,
ObjectReadOperation *op, snap_t snapid, int flags,
+ bufferlist *pbl)
+ __attribute__ ((deprecated));
+
+ int aio_operate(const std::string& oid, AioCompletion *c,
+ ObjectReadOperation *op, int flags,
bufferlist *pbl);
// watch/notify
@@ -725,6 +808,9 @@ namespace librados
int64_t get_id();
+ uint32_t get_object_hash_position(const std::string& oid);
+ uint32_t get_object_pg_hash_position(const std::string& oid);
+
config_t cct();
private:
@@ -763,10 +849,10 @@ namespace librados
int pool_create(const char *name);
int pool_create(const char *name, uint64_t auid);
- int pool_create(const char *name, uint64_t auid, __u8 crush_rule);
+ int pool_create(const char *name, uint64_t auid, uint8_t crush_rule);
int pool_create_async(const char *name, PoolAsyncCompletion *c);
int pool_create_async(const char *name, uint64_t auid, PoolAsyncCompletion *c);
- int pool_create_async(const char *name, uint64_t auid, __u8 crush_rule, PoolAsyncCompletion *c);
+ int pool_create_async(const char *name, uint64_t auid, uint8_t crush_rule, PoolAsyncCompletion *c);
int pool_delete(const char *name);
int pool_delete_async(const char *name, PoolAsyncCompletion *c);
int64_t pool_lookup(const char *name);
@@ -774,6 +860,9 @@ namespace librados
uint64_t get_instance_id();
+ int mon_command(std::string cmd, const bufferlist& inbl,
+ bufferlist *outbl, std::string *outs);
+
int ioctx_create(const char *name, IoCtx &pioctx);
// Features useful for test cases
@@ -789,6 +878,9 @@ namespace librados
int cluster_stat(cluster_stat_t& result);
int cluster_fsid(std::string *fsid);
+ /// get/wait for the most recent osdmap
+ int wait_for_latest_osdmap();
+
/*
* pool aio
*
diff --git a/src/include/rados/memory.h b/src/include/rados/memory.h
new file mode 100644
index 0000000..ee28d69
--- /dev/null
+++ b/src/include/rados/memory.h
@@ -0,0 +1,26 @@
+#ifndef CEPH_MEMORY_H
+#define CEPH_MEMORY_H
+
+#include <ciso646>
+
+#ifdef _LIBCPP_VERSION
+
+#include <memory>
+
+namespace ceph {
+ using std::shared_ptr;
+ using std::weak_ptr;
+}
+
+#else
+
+#include <tr1/memory>
+
+namespace ceph {
+ using std::tr1::shared_ptr;
+ using std::tr1::weak_ptr;
+}
+
+#endif
+
+#endif
diff --git a/src/include/rados/rados_types.hpp b/src/include/rados/rados_types.hpp
index 4a233e6..ec70f9d 100644
--- a/src/include/rados/rados_types.hpp
+++ b/src/include/rados/rados_types.hpp
@@ -19,11 +19,13 @@ struct clone_info_t {
std::vector<snap_t> snaps; // ascending
std::vector< std::pair<uint64_t,uint64_t> > overlap; // with next newest
uint64_t size;
+ clone_info_t() : cloneid(0), size(0) {}
};
struct snap_set_t {
std::vector<clone_info_t> clones; // ascending
snap_t seq; // newest snapid seen by the object
+ snap_set_t() : seq(0) {}
};
}
diff --git a/src/include/stat.h b/src/include/stat.h
new file mode 100644
index 0000000..1939875
--- /dev/null
+++ b/src/include/stat.h
@@ -0,0 +1,145 @@
+#ifndef CEPH_STAT_H
+#define CEPH_STAT_H
+
+#include <acconfig.h>
+
+#include <sys/stat.h>
+
+/*
+ * Access time-related `struct stat` members.
+ *
+ * Note that for each of the stat member get/set functions below, setting a
+ * high-res value (stat_set_*_nsec) on a platform without high-res support is
+ * a no-op.
+ */
+
+#ifdef HAVE_STAT_ST_MTIM_TV_NSEC
+
+static inline uint32_t stat_get_mtime_nsec(struct stat *st)
+{
+ return st->st_mtim.tv_nsec;
+}
+
+static inline void stat_set_mtime_nsec(struct stat *st, uint32_t nsec)
+{
+ st->st_mtim.tv_nsec = nsec;
+}
+
+static inline uint32_t stat_get_atime_nsec(struct stat *st)
+{
+ return st->st_atim.tv_nsec;
+}
+
+static inline void stat_set_atime_nsec(struct stat *st, uint32_t nsec)
+{
+ st->st_atim.tv_nsec = nsec;
+}
+
+static inline uint32_t stat_get_ctime_nsec(struct stat *st)
+{
+ return st->st_ctim.tv_nsec;
+}
+
+static inline void stat_set_ctime_nsec(struct stat *st, uint32_t nsec)
+{
+ st->st_ctim.tv_nsec = nsec;
+}
+
+#elif defined(HAVE_STAT_ST_MTIMESPEC_TV_NSEC)
+
+static inline uint32_t stat_get_mtime_nsec(struct stat *st)
+{
+ return st->st_mtimespec.tv_nsec;
+}
+
+static inline void stat_set_mtime_nsec(struct stat *st, uint32_t nsec)
+{
+ st->st_mtimespec.tv_nsec = nsec;
+}
+
+static inline uint32_t stat_get_atime_nsec(struct stat *st)
+{
+ return st->st_atimespec.tv_nsec;
+}
+
+static inline void stat_set_atime_nsec(struct stat *st, uint32_t nsec)
+{
+ st->st_atimespec.tv_nsec = nsec;
+}
+
+static inline uint32_t stat_get_ctime_nsec(struct stat *st)
+{
+ return st->st_ctimespec.tv_nsec;
+}
+
+static inline void stat_set_ctime_nsec(struct stat *st, uint32_t nsec)
+{
+ st->st_ctimespec.tv_nsec = nsec;
+}
+
+#else
+
+static inline uint32_t stat_get_mtime_nsec(struct stat *st)
+{
+ return 0;
+}
+
+static inline void stat_set_mtime_nsec(struct stat *st, uint32_t nsec)
+{
+}
+
+static inline uint32_t stat_get_atime_nsec(struct stat *st)
+{
+ return 0;
+}
+
+static inline void stat_set_atime_nsec(struct stat *st, uint32_t nsec)
+{
+}
+
+static inline uint32_t stat_get_ctime_nsec(struct stat *st)
+{
+ return 0;
+}
+
+static inline void stat_set_ctime_nsec(struct stat *st, uint32_t nsec)
+{
+}
+
+#endif
+
+/*
+ * Access second-resolution `struct stat` members.
+ */
+
+static inline uint32_t stat_get_mtime_sec(struct stat *st)
+{
+ return st->st_mtime;
+}
+
+static inline void stat_set_mtime_sec(struct stat *st, uint32_t sec)
+{
+ st->st_mtime = sec;
+}
+
+static inline uint32_t stat_get_atime_sec(struct stat *st)
+{
+ return st->st_atime;
+}
+
+static inline void stat_set_atime_sec(struct stat *st, uint32_t sec)
+{
+ st->st_atime = sec;
+}
+
+static inline uint32_t stat_get_ctime_sec(struct stat *st)
+{
+ return st->st_ctime;
+}
+
+static inline void stat_set_ctime_sec(struct stat *st, uint32_t sec)
+{
+ st->st_ctime = sec;
+}
+
+#endif
diff --git a/src/include/str_map.h b/src/include/str_map.h
new file mode 100644
index 0000000..efae903
--- /dev/null
+++ b/src/include/str_map.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_STRMAP_H
+#define CEPH_STRMAP_H
+
+#include <map>
+#include <string>
+#include <sstream>
+
+/**
+ * Parse **str** and set **str_map** with the key/value pairs read
+ * from it. The format of **str** is either a well formed JSON object
+ * or a custom key[=value] plain text format.
+ *
+ * JSON is tried first. If successfully parsed into a JSON object, it
+ * is copied into **str_map** verbatim. If it is not a JSON object ( a
+ * string, integer etc. ), -EINVAL is returned and **ss** is set to
+ * a human readable error message.
+ *
+ * If **str** is no valid JSON, it is assumed to be a string
+ * containing white space separated key=value pairs. A white space is
+ * either space, tab or newline. The value is optional, in which case
+ * it defaults to an empty string. For example:
+ *
+ * insert your own=political statement=here
+ *
+ * will be parsed into:
+ *
+ * { "insert": "",
+ * "your": "",
+ * "own": "policital",
+ * "statement": "here" }
+ *
+ * Returns 0 on success.
+ *
+ * @param [in] str JSON or plain text key/value pairs
+ * @param [out] ss human readable message on error
+ * @param [out] str_map key/value pairs read from str
+ * @return **0** on success or a -EINVAL on error.
+ */
+extern int get_str_map(const std::string &str,
+ std::stringstream &ss,
+ std::map<std::string,std::string> *str_map);
+
+#endif
diff --git a/src/include/types.h b/src/include/types.h
index 5a9e6f6..974edee 100644
--- a/src/include/types.h
+++ b/src/include/types.h
@@ -61,9 +61,8 @@ extern "C" {
using namespace std;
-#include <ext/hash_map>
-using namespace __gnu_cxx;
-
+#include "include/unordered_map.h"
+#include "include/hash_namespace.h"
#include "object.h"
#include "intarith.h"
@@ -85,37 +84,6 @@ typedef off_t loff_t;
typedef off_t off64_t;
#endif
-// -- stl crap --
-
-namespace __gnu_cxx {
- template<> struct hash< std::string >
- {
- size_t operator()( const std::string& x ) const
- {
- static hash<const char*> H;
- return H(x.c_str());
- }
- };
-
-#ifndef __LP64__
- template<> struct hash<int64_t> {
- size_t operator()(int64_t __x) const {
- static hash<int32_t> H;
- return H((__x >> 32) ^ (__x & 0xffffffff));
- }
- };
- template<> struct hash<uint64_t> {
- size_t operator()(uint64_t __x) const {
- static hash<uint32_t> H;
- return H((__x >> 32) ^ (__x & 0xffffffff));
- }
- };
-#endif
-
-}
-
-
-
// -- io helpers --
template<class A, class B>
@@ -211,8 +179,8 @@ inline ostream& operator<<(ostream& out, const multimap<A,B>& m)
/*
* comparators for stl containers
*/
-// for hash_map:
-// hash_map<const char*, long, hash<const char*>, eqstr> vals;
+// for ceph::unordered_map:
+// ceph::unordered_map<const char*, long, hash<const char*>, eqstr> vals;
struct eqstr
{
bool operator()(const char* s1, const char* s2) const
@@ -242,6 +210,7 @@ WRITE_RAW_ENCODER(ceph_mds_request_head)
WRITE_RAW_ENCODER(ceph_mds_request_release)
WRITE_RAW_ENCODER(ceph_filelock)
WRITE_RAW_ENCODER(ceph_mds_caps)
+WRITE_RAW_ENCODER(ceph_mds_cap_peer)
WRITE_RAW_ENCODER(ceph_mds_cap_release)
WRITE_RAW_ENCODER(ceph_mds_cap_item)
WRITE_RAW_ENCODER(ceph_mds_lease)
@@ -328,7 +297,7 @@ inline ostream& operator<<(ostream& out, inodeno_t ino) {
return out << hex << ino.val << dec;
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash< inodeno_t >
{
size_t operator()( const inodeno_t& x ) const
@@ -337,7 +306,7 @@ namespace __gnu_cxx {
return H(x.val);
}
};
-}
+CEPH_HASH_NAMESPACE_END
// file modes
diff --git a/src/include/unordered_map.h b/src/include/unordered_map.h
new file mode 100644
index 0000000..adcb89a
--- /dev/null
+++ b/src/include/unordered_map.h
@@ -0,0 +1,24 @@
+#ifndef CEPH_UNORDERED_MAP_H
+#define CEPH_UNORDERED_MAP_H
+
+#include <ciso646>
+
+#ifdef _LIBCPP_VERSION
+
+#include <unordered_map>
+
+namespace ceph {
+ using std::unordered_map;
+}
+
+#else
+
+#include <tr1/unordered_map>
+
+namespace ceph {
+ using std::tr1::unordered_map;
+}
+
+#endif
+
+#endif
diff --git a/src/include/unordered_set.h b/src/include/unordered_set.h
new file mode 100644
index 0000000..9b3b70b
--- /dev/null
+++ b/src/include/unordered_set.h
@@ -0,0 +1,24 @@
+#ifndef CEPH_UNORDERED_SET_H
+#define CEPH_UNORDERED_SET_H
+
+#include <ciso646>
+
+#ifdef _LIBCPP_VERSION
+
+#include <unordered_set>
+
+namespace ceph {
+ using std::unordered_set;
+}
+
+#else
+
+#include <tr1/unordered_set>
+
+namespace ceph {
+ using std::tr1::unordered_set;
+}
+
+#endif
+
+#endif
diff --git a/src/include/utime.h b/src/include/utime.h
index e134605..5bebc70 100644
--- a/src/include/utime.h
+++ b/src/include/utime.h
@@ -138,8 +138,9 @@ public:
}
void sleep() {
- struct timespec ts = { (__time_t)tv.tv_sec, (long)tv.tv_nsec };
- nanosleep(&ts, &ts);
+ struct timespec ts;
+ to_timespec(&ts);
+ nanosleep(&ts, NULL);
}
// output
diff --git a/src/init-ceph.in b/src/init-ceph.in
index 1a80a42..925047f 100644
--- a/src/init-ceph.in
+++ b/src/init-ceph.in
@@ -317,22 +317,21 @@ for name in $what; do
if [ "$type" = "osd" ]; then
get_conf update_crush "" "osd crush update on start"
- if [ "${update_crush:-1}" = "1" -o "{$update_crush:-1}" = "true" ]; then
- # update location in crush; put in some suitable defaults on the
- # command line, ceph.conf can override what it wants
- get_conf osd_location "" "osd crush location"
+ if [ "${update_crush:-1}" = "1" -o "${update_crush:-1}" = "true" ]; then
+ # update location in crush
+ get_conf osd_location_hook "$BINDIR/ceph-crush-location" "osd crush location hook"
+ osd_location=`$osd_location_hook --cluster ceph --id $id --type osd`
get_conf osd_weight "" "osd crush initial weight"
- defaultweight="$(do_cmd "df $osd_data/. | tail -1 | awk '{ d= \$2/1073741824 ; r = sprintf(\"%.2f\", d); print r }'")"
+ defaultweight="$(do_cmd "df -P -k $osd_data/. | tail -1 | awk '{ d= \$2/1073741824 ; r = sprintf(\"%.2f\", d); print r }'")"
get_conf osd_keyring "$osd_data/keyring" "keyring"
do_cmd "timeout 10 $BINDIR/ceph \
+ -c $conf \
--name=osd.$id \
--keyring=$osd_keyring \
osd crush create-or-move \
-- \
$id \
${osd_weight:-${defaultweight:-1}} \
- root=default \
- host=$host \
$osd_location"
fi
fi
diff --git a/src/init-radosgw b/src/init-radosgw
index de1e01e..caa05cb 100644
--- a/src/init-radosgw
+++ b/src/init-radosgw
@@ -12,23 +12,34 @@ PATH=/sbin:/bin:/usr/bin
. /lib/lsb/init-functions
+daemon_is_running() {
+ daemon=$1
+ if pidof $daemon >/dev/null; then
+ echo "$daemon is running."
+ exit 0
+ else
+ echo "$daemon is not running."
+ exit 1
+ fi
+}
+
+VERBOSE=0
+for opt in $*; do
+ if [ "$opt" = "-v" ] || [ "$opt" = "--verbose" ]; then
+ VERBOSE=1
+ fi
+done
+
# prefix for radosgw instances in ceph.conf
PREFIX='client.radosgw.'
# user to run radosgw as (it not specified in ceph.conf)
DEFAULT_USER='www-data'
-# directory to write logs to
-LOGDIR='/var/log/radosgw'
-
RADOSGW=`which radosgw`
if [ ! -x "$RADOSGW" ]; then
- exit 0
-fi
-
-# make sure log dir exists
-if [ ! -d "$LOGDIR" ]; then
- mkdir -p $LOGDIR
+ [ $VERBOSE -eq 1 ] && echo "$RADOSGW could not start, it is not executable."
+ exit 1
fi
case "$1" in
@@ -48,7 +59,9 @@ case "$1" in
# mapped to this host?
host=`ceph-conf -n $name host`
- if [ "$host" != `hostname` ]; then
+ hostname=`hostname -s`
+ if [ "$host" != "$hostname" ]; then
+ [ $VERBOSE -eq 1 ] && echo "hostname $hostname could not be found in ceph.conf:[$name], not starting."
continue
fi
@@ -66,6 +79,7 @@ case "$1" in
echo "Starting $name..."
start-stop-daemon --start -u $user -x $RADOSGW -- -n $name
done
+ daemon_is_running $RADOSGW
;;
reload)
echo "Reloading $name..."
@@ -79,15 +93,10 @@ case "$1" in
start-stop-daemon --stop -x $RADOSGW --oknodo
;;
status)
- if pidof $RADOSGW >/dev/null; then
- echo "$RADOSGW is running."
- else
- echo "$RADOSGW is not running."
- exit 1
- fi
+ daemon_is_running $RADOSGW
;;
*)
- echo "Usage: $0 start|stop|restart|force-reload|reload|status" >&2
+ echo "Usage: $0 {start|stop|restart|force-reload|reload|status} [-v|--verbose]" >&2
exit 3
;;
esac
diff --git a/src/init-radosgw.sysv b/src/init-radosgw.sysv
index 017e3f9..ab6b250 100644
--- a/src/init-radosgw.sysv
+++ b/src/init-radosgw.sysv
@@ -13,6 +13,24 @@ PATH=/sbin:/bin:/usr/bin
#. /lib/lsb/init-functions
. /etc/rc.d/init.d/functions
+daemon_is_running() {
+ daemon=$1
+ if pidof $daemon >/dev/null; then
+ echo "$daemon is running."
+ exit 0
+ else
+ echo "$daemon is not running."
+ exit 1
+ fi
+}
+
+VERBOSE=0
+for opt in $*; do
+ if [ "$opt" = "-v" ] || [ "$opt" = "--verbose" ]; then
+ VERBOSE=1
+ fi
+done
+
# prefix for radosgw instances in ceph.conf
PREFIX='client.radosgw.'
@@ -20,17 +38,10 @@ PREFIX='client.radosgw.'
#DEFAULT_USER='www-data'
DEFAULT_USER='apache'
-# directory to write logs to
-LOGDIR='/var/log/radosgw'
-
RADOSGW=`which radosgw`
if [ ! -x "$RADOSGW" ]; then
- exit 0
-fi
-
-# make sure log dir exists
-if [ ! -d "$LOGDIR" ]; then
- mkdir -p $LOGDIR
+ [ $VERBOSE -eq 1 ] && echo "$RADOSGW could not start, it is not executable."
+ exit 1
fi
case "$1" in
@@ -51,7 +62,9 @@ case "$1" in
# mapped to this host?
host=`ceph-conf -n $name host`
- if [ "$host" != `hostname` ]; then
+ hostname=`hostname -s`
+ if [ "$host" != "$hostname" ]; then
+ [ $VERBOSE -eq 1 ] && echo "hostname $hostname could not be found in ceph.conf:[$name], not starting."
continue
fi
@@ -67,9 +80,10 @@ case "$1" in
fi
#start-stop-daemon --start -u $user -x $RADOSGW -- -n $name
- daemon --user="$user" "$RADOSGW -n $name"
+ daemon --user="$user" "ulimit -n 32768; $RADOSGW -n $name"
echo "Starting $name..."
done
+ daemon_is_running $RADOSGW
;;
reload)
#start-stop-daemon --signal HUP -x $RADOSGW --oknodo
@@ -86,15 +100,10 @@ case "$1" in
echo "Stopping radosgw instance(s)..."
;;
status)
- if pidof $RADOSGW >/dev/null; then
- echo "$RADOSGW is running."
- else
- echo "$RADOSGW is not running."
- exit 1
- fi
+ daemon_is_running $RADOSGW
;;
*)
- echo "Usage: $0 start|stop|restart|force-reload|reload|status" >&2
+ echo "Usage: $0 {start|stop|restart|force-reload|reload|status} [-v|--verbose]" >&2
exit 3
;;
esac
diff --git a/src/java/Makefile.in b/src/java/Makefile.in
index fc99311..67670b9 100644
--- a/src/java/Makefile.in
+++ b/src/java/Makefile.in
@@ -48,7 +48,10 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
$(top_srcdir)/m4/ac_prog_javac_works.m4 \
$(top_srcdir)/m4/ac_prog_javah.m4 \
$(top_srcdir)/m4/acx_pthread.m4 \
+ $(top_srcdir)/m4/ax_c_pretty_func.m4 \
+ $(top_srcdir)/m4/ax_c_var_func.m4 \
$(top_srcdir)/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/m4/ax_cxx_static_cast.m4 \
$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/pkg.m4 \
@@ -106,6 +109,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
+BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@
CC = @CC@
CCAS = @CCAS@
CCASDEPMODE = @CCASDEPMODE@
@@ -147,6 +151,7 @@ JAR = @JAR@
JAVAC = @JAVAC@
JAVAH = @JAVAH@
JDK_CPPFLAGS = @JDK_CPPFLAGS@
+KEYUTILS_LIB = @KEYUTILS_LIB@
LD = @LD@
LDFLAGS = @LDFLAGS@
LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@
@@ -192,6 +197,7 @@ PYTHON_PLATFORM = @PYTHON_PLATFORM@
PYTHON_PREFIX = @PYTHON_PREFIX@
PYTHON_VERSION = @PYTHON_VERSION@
RANLIB = @RANLIB@
+RESOLV_LIBS = @RESOLV_LIBS@
RPM_RELEASE = @RPM_RELEASE@
SED = @SED@
SET_MAKE = @SET_MAKE@
diff --git a/src/json_spirit/Makefile.am b/src/json_spirit/Makefile.am
index 43025be..9b82ec4 100644
--- a/src/json_spirit/Makefile.am
+++ b/src/json_spirit/Makefile.am
@@ -1,7 +1,6 @@
libjson_spirit_la_SOURCES = \
json_spirit/json_spirit_reader.cpp \
- json_spirit/json_spirit_writer.cpp \
- json_spirit/json_spirit_value.cpp
+ json_spirit/json_spirit_writer.cpp
noinst_LTLIBRARIES += libjson_spirit.la
noinst_HEADERS += \
diff --git a/src/json_spirit/json_spirit_value.cpp b/src/json_spirit/json_spirit_value.cpp
deleted file mode 100644
index dd5b50e..0000000
--- a/src/json_spirit/json_spirit_value.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-/* Copyright (c) 2007 John W Wilkinson
-
- This source code can be used for any purpose as long as
- this comment is retained. */
-
-// json spirit version 2.00
-
-#include "json_spirit_value.h"
diff --git a/src/key_value_store/Makefile.am b/src/key_value_store/Makefile.am
index e5bae00..c4b854f 100644
--- a/src/key_value_store/Makefile.am
+++ b/src/key_value_store/Makefile.am
@@ -1,7 +1,9 @@
+if LINUX
libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc
libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
+libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
radoslib_LTLIBRARIES += libcls_kvs.la
+endif
noinst_HEADERS += \
key_value_store/key_value_structure.h \
diff --git a/src/key_value_store/kv_flat_btree_async.cc b/src/key_value_store/kv_flat_btree_async.cc
index 7faf077..709d880 100644
--- a/src/key_value_store/kv_flat_btree_async.cc
+++ b/src/key_value_store/kv_flat_btree_async.cc
@@ -672,7 +672,7 @@ int KvFlatBtreeAsync::read_object(const string &obj, object_data * odata) {
return err;
}
odata->unwritable = string(unw_bl.c_str(), unw_bl.length()) == "1";
- odata->version = obj_aioc->get_version();
+ odata->version = obj_aioc->get_version64();
odata->size = odata->omap.size();
obj_aioc->release();
return 0;
@@ -697,7 +697,7 @@ int KvFlatBtreeAsync::read_object(const string &obj, rebalance_args * args) {
bufferlist::iterator it = outbl.begin();
args->decode(it);
args->odata.name = obj;
- args->odata.version = a->get_version();
+ args->odata.version = a->get_version64();
a->release();
return err;
}
@@ -2095,7 +2095,7 @@ bool KvFlatBtreeAsync::is_consistent() {
}
}
if (atoi(string(un.c_str(), un.length()).c_str()) != 1 &&
- aioc->get_version() != (int)dit->version) {
+ aioc->get_version64() != dit->version) {
cerr << "Not consistent! object " << dit->obj << " has been "
<< " modified since the client died was not cleaned up."
<< std::endl;
@@ -2251,7 +2251,7 @@ string KvFlatBtreeAsync::str() {
//return ret.str();
}
all_sizes[indexer] = all_maps[indexer].size();
- all_versions[indexer] = aioc->get_version();
+ all_versions[indexer] = aioc->get_version64();
indexer++;
aioc->release();
}
diff --git a/src/libcephfs.cc b/src/libcephfs.cc
index cdd06f4..cc24a31 100644
--- a/src/libcephfs.cc
+++ b/src/libcephfs.cc
@@ -31,7 +31,7 @@
#include "msg/Messenger.h"
#include "include/assert.h"
-class ceph_mount_info
+struct ceph_mount_info
{
public:
ceph_mount_info(uint64_t msgr_nonce_, CephContext *cct_)
@@ -240,7 +240,7 @@ extern "C" const char *ceph_version(int *pmajor, int *pminor, int *ppatch)
return VERSION;
}
-extern "C" int ceph_create_with_context(class ceph_mount_info **cmount, CephContext *cct)
+extern "C" int ceph_create_with_context(struct ceph_mount_info **cmount, CephContext *cct)
{
uint64_t nonce = 0;
@@ -249,11 +249,11 @@ extern "C" int ceph_create_with_context(class ceph_mount_info **cmount, CephCont
nonce &= ~0xffff;
nonce |= (uint64_t)getpid();
- *cmount = new class ceph_mount_info(nonce, cct);
+ *cmount = new struct ceph_mount_info(nonce, cct);
return 0;
}
-extern "C" int ceph_create(class ceph_mount_info **cmount, const char * const id)
+extern "C" int ceph_create(struct ceph_mount_info **cmount, const char * const id)
{
CephInitParameters iparams(CEPH_ENTITY_TYPE_CLIENT);
if (id) {
@@ -266,12 +266,12 @@ extern "C" int ceph_create(class ceph_mount_info **cmount, const char * const id
return ceph_create_with_context(cmount, cct);
}
-extern "C" int ceph_unmount(class ceph_mount_info *cmount)
+extern "C" int ceph_unmount(struct ceph_mount_info *cmount)
{
return cmount->unmount();
}
-extern "C" int ceph_release(class ceph_mount_info *cmount)
+extern "C" int ceph_release(struct ceph_mount_info *cmount)
{
if (cmount->is_mounted())
return -EISCONN;
@@ -279,35 +279,35 @@ extern "C" int ceph_release(class ceph_mount_info *cmount)
return 0;
}
-extern "C" void ceph_shutdown(class ceph_mount_info *cmount)
+extern "C" void ceph_shutdown(struct ceph_mount_info *cmount)
{
cmount->shutdown();
delete cmount;
}
-extern "C" int ceph_conf_read_file(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_conf_read_file(struct ceph_mount_info *cmount, const char *path)
{
return cmount->conf_read_file(path);
}
-extern "C" int ceph_conf_parse_argv(class ceph_mount_info *cmount, int argc,
+extern "C" int ceph_conf_parse_argv(struct ceph_mount_info *cmount, int argc,
const char **argv)
{
return cmount->conf_parse_argv(argc, argv);
}
-extern "C" int ceph_conf_parse_env(class ceph_mount_info *cmount, const char *name)
+extern "C" int ceph_conf_parse_env(struct ceph_mount_info *cmount, const char *name)
{
return cmount->conf_parse_env(name);
}
-extern "C" int ceph_conf_set(class ceph_mount_info *cmount, const char *option,
+extern "C" int ceph_conf_set(struct ceph_mount_info *cmount, const char *option,
const char *value)
{
return cmount->conf_set(option, value);
}
-extern "C" int ceph_conf_get(class ceph_mount_info *cmount, const char *option,
+extern "C" int ceph_conf_get(struct ceph_mount_info *cmount, const char *option,
char *buf, size_t len)
{
if (buf == NULL) {
@@ -316,7 +316,7 @@ extern "C" int ceph_conf_get(class ceph_mount_info *cmount, const char *option,
return cmount->conf_get(option, buf, len);
}
-extern "C" int ceph_mount(class ceph_mount_info *cmount, const char *root)
+extern "C" int ceph_mount(struct ceph_mount_info *cmount, const char *root)
{
std::string mount_root;
if (root)
@@ -324,12 +324,12 @@ extern "C" int ceph_mount(class ceph_mount_info *cmount, const char *root)
return cmount->mount(mount_root);
}
-extern "C" int ceph_is_mounted(class ceph_mount_info *cmount)
+extern "C" int ceph_is_mounted(struct ceph_mount_info *cmount)
{
return cmount->is_mounted() ? 1 : 0;
}
-extern "C" int ceph_statfs(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_statfs(struct ceph_mount_info *cmount, const char *path,
struct statvfs *stbuf)
{
if (!cmount->is_mounted())
@@ -337,26 +337,26 @@ extern "C" int ceph_statfs(class ceph_mount_info *cmount, const char *path,
return cmount->get_client()->statfs(path, stbuf);
}
-extern "C" int ceph_get_local_osd(class ceph_mount_info *cmount)
+extern "C" int ceph_get_local_osd(struct ceph_mount_info *cmount)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->get_local_osd();
}
-extern "C" const char* ceph_getcwd(class ceph_mount_info *cmount)
+extern "C" const char* ceph_getcwd(struct ceph_mount_info *cmount)
{
return cmount->get_cwd();
}
-extern "C" int ceph_chdir (class ceph_mount_info *cmount, const char *s)
+extern "C" int ceph_chdir (struct ceph_mount_info *cmount, const char *s)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->chdir(s);
}
-extern "C" int ceph_opendir(class ceph_mount_info *cmount,
+extern "C" int ceph_opendir(struct ceph_mount_info *cmount,
const char *name, struct ceph_dir_result **dirpp)
{
if (!cmount->is_mounted())
@@ -364,14 +364,14 @@ extern "C" int ceph_opendir(class ceph_mount_info *cmount,
return cmount->get_client()->opendir(name, (dir_result_t **)dirpp);
}
-extern "C" int ceph_closedir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp)
+extern "C" int ceph_closedir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->closedir((dir_result_t*)dirp);
}
-extern "C" struct dirent * ceph_readdir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp)
+extern "C" struct dirent * ceph_readdir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp)
{
if (!cmount->is_mounted()) {
/* Client::readdir also sets errno to signal errors. */
@@ -381,14 +381,14 @@ extern "C" struct dirent * ceph_readdir(class ceph_mount_info *cmount, struct ce
return cmount->get_client()->readdir((dir_result_t*)dirp);
}
-extern "C" int ceph_readdir_r(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de)
+extern "C" int ceph_readdir_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->readdir_r((dir_result_t*)dirp, de);
}
-extern "C" int ceph_readdirplus_r(class ceph_mount_info *cmount, struct ceph_dir_result *dirp,
+extern "C" int ceph_readdirplus_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp,
struct dirent *de, struct stat *st, int *stmask)
{
if (!cmount->is_mounted())
@@ -396,7 +396,7 @@ extern "C" int ceph_readdirplus_r(class ceph_mount_info *cmount, struct ceph_dir
return cmount->get_client()->readdirplus_r((dir_result_t*)dirp, de, st, stmask);
}
-extern "C" int ceph_getdents(class ceph_mount_info *cmount, struct ceph_dir_result *dirp,
+extern "C" int ceph_getdents(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp,
char *buf, int buflen)
{
if (!cmount->is_mounted())
@@ -404,7 +404,7 @@ extern "C" int ceph_getdents(class ceph_mount_info *cmount, struct ceph_dir_resu
return cmount->get_client()->getdents((dir_result_t*)dirp, buf, buflen);
}
-extern "C" int ceph_getdnames(class ceph_mount_info *cmount, struct ceph_dir_result *dirp,
+extern "C" int ceph_getdnames(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp,
char *buf, int buflen)
{
if (!cmount->is_mounted())
@@ -412,28 +412,28 @@ extern "C" int ceph_getdnames(class ceph_mount_info *cmount, struct ceph_dir_res
return cmount->get_client()->getdnames((dir_result_t*)dirp, buf, buflen);
}
-extern "C" void ceph_rewinddir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp)
+extern "C" void ceph_rewinddir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp)
{
if (!cmount->is_mounted())
return;
cmount->get_client()->rewinddir((dir_result_t*)dirp);
}
-extern "C" loff_t ceph_telldir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp)
+extern "C" int64_t ceph_telldir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->telldir((dir_result_t*)dirp);
}
-extern "C" void ceph_seekdir(class ceph_mount_info *cmount, struct ceph_dir_result *dirp, loff_t offset)
+extern "C" void ceph_seekdir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, int64_t offset)
{
if (!cmount->is_mounted())
return;
cmount->get_client()->seekdir((dir_result_t*)dirp, offset);
}
-extern "C" int ceph_link (class ceph_mount_info *cmount, const char *existing,
+extern "C" int ceph_link (struct ceph_mount_info *cmount, const char *existing,
const char *newname)
{
if (!cmount->is_mounted())
@@ -441,14 +441,14 @@ extern "C" int ceph_link (class ceph_mount_info *cmount, const char *existing,
return cmount->get_client()->link(existing, newname);
}
-extern "C" int ceph_unlink(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_unlink(struct ceph_mount_info *cmount, const char *path)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->unlink(path);
}
-extern "C" int ceph_rename(class ceph_mount_info *cmount, const char *from,
+extern "C" int ceph_rename(struct ceph_mount_info *cmount, const char *from,
const char *to)
{
if (!cmount->is_mounted())
@@ -457,21 +457,21 @@ extern "C" int ceph_rename(class ceph_mount_info *cmount, const char *from,
}
// dirs
-extern "C" int ceph_mkdir(class ceph_mount_info *cmount, const char *path, mode_t mode)
+extern "C" int ceph_mkdir(struct ceph_mount_info *cmount, const char *path, mode_t mode)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->mkdir(path, mode);
}
-extern "C" int ceph_mkdirs(class ceph_mount_info *cmount, const char *path, mode_t mode)
+extern "C" int ceph_mkdirs(struct ceph_mount_info *cmount, const char *path, mode_t mode)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->mkdirs(path, mode);
}
-extern "C" int ceph_rmdir(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_rmdir(struct ceph_mount_info *cmount, const char *path)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -479,15 +479,15 @@ extern "C" int ceph_rmdir(class ceph_mount_info *cmount, const char *path)
}
// symlinks
-extern "C" int ceph_readlink(class ceph_mount_info *cmount, const char *path,
- char *buf, loff_t size)
+extern "C" int ceph_readlink(struct ceph_mount_info *cmount, const char *path,
+ char *buf, int64_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->readlink(path, buf, size);
}
-extern "C" int ceph_symlink(class ceph_mount_info *cmount, const char *existing,
+extern "C" int ceph_symlink(struct ceph_mount_info *cmount, const char *existing,
const char *newname)
{
if (!cmount->is_mounted())
@@ -496,7 +496,7 @@ extern "C" int ceph_symlink(class ceph_mount_info *cmount, const char *existing,
}
// inode stuff
-extern "C" int ceph_stat(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_stat(struct ceph_mount_info *cmount, const char *path,
struct stat *stbuf)
{
if (!cmount->is_mounted())
@@ -504,7 +504,7 @@ extern "C" int ceph_stat(class ceph_mount_info *cmount, const char *path,
return cmount->get_client()->stat(path, stbuf);
}
-extern "C" int ceph_lstat(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_lstat(struct ceph_mount_info *cmount, const char *path,
struct stat *stbuf)
{
if (!cmount->is_mounted())
@@ -512,7 +512,7 @@ extern "C" int ceph_lstat(class ceph_mount_info *cmount, const char *path,
return cmount->get_client()->lstat(path, stbuf);
}
-extern "C" int ceph_setattr(class ceph_mount_info *cmount, const char *relpath,
+extern "C" int ceph_setattr(struct ceph_mount_info *cmount, const char *relpath,
struct stat *attr, int mask)
{
if (!cmount->is_mounted())
@@ -521,56 +521,56 @@ extern "C" int ceph_setattr(class ceph_mount_info *cmount, const char *relpath,
}
// *xattr() calls supporting samba/vfs
-extern "C" int ceph_getxattr(class ceph_mount_info *cmount, const char *path, const char *name, void *value, size_t size)
+extern "C" int ceph_getxattr(struct ceph_mount_info *cmount, const char *path, const char *name, void *value, size_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->getxattr(path, name, value, size);
}
-extern "C" int ceph_lgetxattr(class ceph_mount_info *cmount, const char *path, const char *name, void *value, size_t size)
+extern "C" int ceph_lgetxattr(struct ceph_mount_info *cmount, const char *path, const char *name, void *value, size_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->lgetxattr(path, name, value, size);
}
-extern "C" int ceph_listxattr(class ceph_mount_info *cmount, const char *path, char *list, size_t size)
+extern "C" int ceph_listxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->listxattr(path, list, size);
}
-extern "C" int ceph_llistxattr(class ceph_mount_info *cmount, const char *path, char *list, size_t size)
+extern "C" int ceph_llistxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->llistxattr(path, list, size);
}
-extern "C" int ceph_removexattr(class ceph_mount_info *cmount, const char *path, const char *name)
+extern "C" int ceph_removexattr(struct ceph_mount_info *cmount, const char *path, const char *name)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->removexattr(path, name);
}
-extern "C" int ceph_lremovexattr(class ceph_mount_info *cmount, const char *path, const char *name)
+extern "C" int ceph_lremovexattr(struct ceph_mount_info *cmount, const char *path, const char *name)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->lremovexattr(path, name);
}
-extern "C" int ceph_setxattr(class ceph_mount_info *cmount, const char *path, const char *name, const void *value, size_t size, int flags)
+extern "C" int ceph_setxattr(struct ceph_mount_info *cmount, const char *path, const char *name, const void *value, size_t size, int flags)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->setxattr(path, name, value, size, flags);
}
-extern "C" int ceph_lsetxattr(class ceph_mount_info *cmount, const char *path, const char *name, const void *value, size_t size, int flags)
+extern "C" int ceph_lsetxattr(struct ceph_mount_info *cmount, const char *path, const char *name, const void *value, size_t size, int flags)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -578,33 +578,33 @@ extern "C" int ceph_lsetxattr(class ceph_mount_info *cmount, const char *path, c
}
/* end xattr support */
-extern "C" int ceph_chmod(class ceph_mount_info *cmount, const char *path, mode_t mode)
+extern "C" int ceph_chmod(struct ceph_mount_info *cmount, const char *path, mode_t mode)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->chmod(path, mode);
}
-extern "C" int ceph_fchmod(class ceph_mount_info *cmount, int fd, mode_t mode)
+extern "C" int ceph_fchmod(struct ceph_mount_info *cmount, int fd, mode_t mode)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->fchmod(fd, mode);
}
-extern "C" int ceph_chown(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_chown(struct ceph_mount_info *cmount, const char *path,
int uid, int gid)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->chown(path, uid, gid);
}
-extern "C" int ceph_fchown(class ceph_mount_info *cmount, int fd,
+extern "C" int ceph_fchown(struct ceph_mount_info *cmount, int fd,
int uid, int gid)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->fchown(fd, uid, gid);
}
-extern "C" int ceph_lchown(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_lchown(struct ceph_mount_info *cmount, const char *path,
int uid, int gid)
{
if (!cmount->is_mounted())
@@ -613,7 +613,7 @@ extern "C" int ceph_lchown(class ceph_mount_info *cmount, const char *path,
}
-extern "C" int ceph_utime(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_utime(struct ceph_mount_info *cmount, const char *path,
struct utimbuf *buf)
{
if (!cmount->is_mounted())
@@ -621,8 +621,8 @@ extern "C" int ceph_utime(class ceph_mount_info *cmount, const char *path,
return cmount->get_client()->utime(path, buf);
}
-extern "C" int ceph_truncate(class ceph_mount_info *cmount, const char *path,
- loff_t size)
+extern "C" int ceph_truncate(struct ceph_mount_info *cmount, const char *path,
+ int64_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -630,7 +630,7 @@ extern "C" int ceph_truncate(class ceph_mount_info *cmount, const char *path,
}
// file ops
-extern "C" int ceph_mknod(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_mknod(struct ceph_mount_info *cmount, const char *path,
mode_t mode, dev_t rdev)
{
if (!cmount->is_mounted())
@@ -638,7 +638,7 @@ extern "C" int ceph_mknod(class ceph_mount_info *cmount, const char *path,
return cmount->get_client()->mknod(path, mode, rdev);
}
-extern "C" int ceph_open(class ceph_mount_info *cmount, const char *path,
+extern "C" int ceph_open(struct ceph_mount_info *cmount, const char *path,
int flags, mode_t mode)
{
if (!cmount->is_mounted())
@@ -646,7 +646,7 @@ extern "C" int ceph_open(class ceph_mount_info *cmount, const char *path,
return cmount->get_client()->open(path, flags, mode);
}
-extern "C" int ceph_open_layout(class ceph_mount_info *cmount, const char *path, int flags,
+extern "C" int ceph_open_layout(struct ceph_mount_info *cmount, const char *path, int flags,
mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool)
{
if (!cmount->is_mounted())
@@ -655,67 +655,67 @@ extern "C" int ceph_open_layout(class ceph_mount_info *cmount, const char *path,
stripe_count, object_size, data_pool);
}
-extern "C" int ceph_close(class ceph_mount_info *cmount, int fd)
+extern "C" int ceph_close(struct ceph_mount_info *cmount, int fd)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->close(fd);
}
-extern "C" loff_t ceph_lseek(class ceph_mount_info *cmount, int fd,
- loff_t offset, int whence)
+extern "C" int64_t ceph_lseek(struct ceph_mount_info *cmount, int fd,
+ int64_t offset, int whence)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->lseek(fd, offset, whence);
}
-extern "C" int ceph_read(class ceph_mount_info *cmount, int fd, char *buf,
- loff_t size, loff_t offset)
+extern "C" int ceph_read(struct ceph_mount_info *cmount, int fd, char *buf,
+ int64_t size, int64_t offset)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->read(fd, buf, size, offset);
}
-extern "C" int ceph_write(class ceph_mount_info *cmount, int fd, const char *buf,
- loff_t size, loff_t offset)
+extern "C" int ceph_write(struct ceph_mount_info *cmount, int fd, const char *buf,
+ int64_t size, int64_t offset)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->write(fd, buf, size, offset);
}
-extern "C" int ceph_ftruncate(class ceph_mount_info *cmount, int fd, loff_t size)
+extern "C" int ceph_ftruncate(struct ceph_mount_info *cmount, int fd, int64_t size)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->ftruncate(fd, size);
}
-extern "C" int ceph_fsync(class ceph_mount_info *cmount, int fd, int syncdataonly)
+extern "C" int ceph_fsync(struct ceph_mount_info *cmount, int fd, int syncdataonly)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->fsync(fd, syncdataonly);
}
-extern "C" int ceph_fallocate(class ceph_mount_info *cmount, int fd, int mode,
- loff_t offset, loff_t length)
+extern "C" int ceph_fallocate(struct ceph_mount_info *cmount, int fd, int mode,
+ int64_t offset, int64_t length)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->fallocate(fd, mode, offset, length);
}
-extern "C" int ceph_fstat(class ceph_mount_info *cmount, int fd, struct stat *stbuf)
+extern "C" int ceph_fstat(struct ceph_mount_info *cmount, int fd, struct stat *stbuf)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->fstat(fd, stbuf);
}
-extern "C" int ceph_sync_fs(class ceph_mount_info *cmount)
+extern "C" int ceph_sync_fs(struct ceph_mount_info *cmount)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -723,7 +723,7 @@ extern "C" int ceph_sync_fs(class ceph_mount_info *cmount)
}
-extern "C" int ceph_get_file_stripe_unit(class ceph_mount_info *cmount, int fh)
+extern "C" int ceph_get_file_stripe_unit(struct ceph_mount_info *cmount, int fh)
{
struct ceph_file_layout l;
int r;
@@ -736,7 +736,7 @@ extern "C" int ceph_get_file_stripe_unit(class ceph_mount_info *cmount, int fh)
return l.fl_stripe_unit;
}
-extern "C" int ceph_get_path_stripe_unit(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_get_path_stripe_unit(struct ceph_mount_info *cmount, const char *path)
{
struct ceph_file_layout l;
int r;
@@ -749,7 +749,7 @@ extern "C" int ceph_get_path_stripe_unit(class ceph_mount_info *cmount, const ch
return l.fl_stripe_unit;
}
-extern "C" int ceph_get_file_stripe_count(class ceph_mount_info *cmount, int fh)
+extern "C" int ceph_get_file_stripe_count(struct ceph_mount_info *cmount, int fh)
{
struct ceph_file_layout l;
int r;
@@ -762,7 +762,7 @@ extern "C" int ceph_get_file_stripe_count(class ceph_mount_info *cmount, int fh)
return l.fl_stripe_count;
}
-extern "C" int ceph_get_path_stripe_count(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_get_path_stripe_count(struct ceph_mount_info *cmount, const char *path)
{
struct ceph_file_layout l;
int r;
@@ -775,7 +775,7 @@ extern "C" int ceph_get_path_stripe_count(class ceph_mount_info *cmount, const c
return l.fl_stripe_count;
}
-extern "C" int ceph_get_file_object_size(class ceph_mount_info *cmount, int fh)
+extern "C" int ceph_get_file_object_size(struct ceph_mount_info *cmount, int fh)
{
struct ceph_file_layout l;
int r;
@@ -788,7 +788,7 @@ extern "C" int ceph_get_file_object_size(class ceph_mount_info *cmount, int fh)
return l.fl_object_size;
}
-extern "C" int ceph_get_path_object_size(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_get_path_object_size(struct ceph_mount_info *cmount, const char *path)
{
struct ceph_file_layout l;
int r;
@@ -801,7 +801,7 @@ extern "C" int ceph_get_path_object_size(class ceph_mount_info *cmount, const ch
return l.fl_object_size;
}
-extern "C" int ceph_get_file_pool(class ceph_mount_info *cmount, int fh)
+extern "C" int ceph_get_file_pool(struct ceph_mount_info *cmount, int fh)
{
struct ceph_file_layout l;
int r;
@@ -814,7 +814,7 @@ extern "C" int ceph_get_file_pool(class ceph_mount_info *cmount, int fh)
return l.fl_pg_pool;
}
-extern "C" int ceph_get_path_pool(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_get_path_pool(struct ceph_mount_info *cmount, const char *path)
{
struct ceph_file_layout l;
int r;
@@ -827,7 +827,7 @@ extern "C" int ceph_get_path_pool(class ceph_mount_info *cmount, const char *pat
return l.fl_pg_pool;
}
-extern "C" int ceph_get_file_pool_name(class ceph_mount_info *cmount, int fh, char *buf, size_t len)
+extern "C" int ceph_get_file_pool_name(struct ceph_mount_info *cmount, int fh, char *buf, size_t len)
{
struct ceph_file_layout l;
int r;
@@ -846,7 +846,7 @@ extern "C" int ceph_get_file_pool_name(class ceph_mount_info *cmount, int fh, ch
return name.length();
}
-extern "C" int ceph_get_pool_name(class ceph_mount_info *cmount, int pool, char *buf, size_t len)
+extern "C" int ceph_get_pool_name(struct ceph_mount_info *cmount, int pool, char *buf, size_t len)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -859,7 +859,7 @@ extern "C" int ceph_get_pool_name(class ceph_mount_info *cmount, int pool, char
return name.length();
}
-extern "C" int ceph_get_path_pool_name(class ceph_mount_info *cmount, const char *path, char *buf, size_t len)
+extern "C" int ceph_get_path_pool_name(struct ceph_mount_info *cmount, const char *path, char *buf, size_t len)
{
struct ceph_file_layout l;
int r;
@@ -878,7 +878,7 @@ extern "C" int ceph_get_path_pool_name(class ceph_mount_info *cmount, const char
return name.length();
}
-extern "C" int ceph_get_file_layout(class ceph_mount_info *cmount, int fh, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool)
+extern "C" int ceph_get_file_layout(struct ceph_mount_info *cmount, int fh, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool)
{
struct ceph_file_layout l;
int r;
@@ -899,7 +899,7 @@ extern "C" int ceph_get_file_layout(class ceph_mount_info *cmount, int fh, int *
return 0;
}
-extern "C" int ceph_get_path_layout(class ceph_mount_info *cmount, const char *path, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool)
+extern "C" int ceph_get_path_layout(struct ceph_mount_info *cmount, const char *path, int *stripe_unit, int *stripe_count, int *object_size, int *pg_pool)
{
struct ceph_file_layout l;
int r;
@@ -920,7 +920,7 @@ extern "C" int ceph_get_path_layout(class ceph_mount_info *cmount, const char *p
return 0;
}
-extern "C" int ceph_get_file_replication(class ceph_mount_info *cmount, int fh)
+extern "C" int ceph_get_file_replication(struct ceph_mount_info *cmount, int fh)
{
struct ceph_file_layout l;
int r;
@@ -934,7 +934,7 @@ extern "C" int ceph_get_file_replication(class ceph_mount_info *cmount, int fh)
return rep;
}
-extern "C" int ceph_get_path_replication(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_get_path_replication(struct ceph_mount_info *cmount, const char *path)
{
struct ceph_file_layout l;
int r;
@@ -948,41 +948,41 @@ extern "C" int ceph_get_path_replication(class ceph_mount_info *cmount, const ch
return rep;
}
-extern "C" int ceph_set_default_file_stripe_unit(class ceph_mount_info *cmount,
+extern "C" int ceph_set_default_file_stripe_unit(struct ceph_mount_info *cmount,
int stripe)
{
// this option no longer exists
return -EOPNOTSUPP;
}
-extern "C" int ceph_set_default_file_stripe_count(class ceph_mount_info *cmount,
+extern "C" int ceph_set_default_file_stripe_count(struct ceph_mount_info *cmount,
int count)
{
// this option no longer exists
return -EOPNOTSUPP;
}
-extern "C" int ceph_set_default_object_size(class ceph_mount_info *cmount, int size)
+extern "C" int ceph_set_default_object_size(struct ceph_mount_info *cmount, int size)
{
// this option no longer exists
return -EOPNOTSUPP;
}
-extern "C" int ceph_set_default_file_replication(class ceph_mount_info *cmount,
+extern "C" int ceph_set_default_file_replication(struct ceph_mount_info *cmount,
int replication)
{
// this option no longer exists
return -EOPNOTSUPP;
}
-extern "C" int ceph_set_default_preferred_pg(class ceph_mount_info *cmount, int osd)
+extern "C" int ceph_set_default_preferred_pg(struct ceph_mount_info *cmount, int osd)
{
// this option no longer exists
return -EOPNOTSUPP;
}
-extern "C" int ceph_get_file_extent_osds(class ceph_mount_info *cmount, int fh,
- loff_t offset, loff_t *length, int *osds, int nosds)
+extern "C" int ceph_get_file_extent_osds(struct ceph_mount_info *cmount, int fh,
+ int64_t offset, int64_t *length, int *osds, int nosds)
{
if (nosds < 0)
return -EINVAL;
@@ -1007,7 +1007,7 @@ extern "C" int ceph_get_file_extent_osds(class ceph_mount_info *cmount, int fh,
return vosds.size();
}
-extern "C" int ceph_get_osd_crush_location(class ceph_mount_info *cmount,
+extern "C" int ceph_get_osd_crush_location(struct ceph_mount_info *cmount,
int osd, char *path, size_t len)
{
if (!cmount->is_mounted())
@@ -1045,7 +1045,7 @@ extern "C" int ceph_get_osd_crush_location(class ceph_mount_info *cmount,
return needed;
}
-extern "C" int ceph_get_osd_addr(class ceph_mount_info *cmount, int osd,
+extern "C" int ceph_get_osd_addr(struct ceph_mount_info *cmount, int osd,
struct sockaddr_storage *addr)
{
if (!cmount->is_mounted())
@@ -1064,8 +1064,8 @@ extern "C" int ceph_get_osd_addr(class ceph_mount_info *cmount, int osd,
return 0;
}
-extern "C" int ceph_get_file_stripe_address(class ceph_mount_info *cmount, int fh,
- loff_t offset, struct sockaddr_storage *addr, int naddr)
+extern "C" int ceph_get_file_stripe_address(struct ceph_mount_info *cmount, int fh,
+ int64_t offset, struct sockaddr_storage *addr, int naddr)
{
vector<entity_addr_t> address;
unsigned i;
@@ -1091,7 +1091,7 @@ extern "C" int ceph_get_file_stripe_address(class ceph_mount_info *cmount, int f
return address.size();
}
-extern "C" int ceph_localize_reads(class ceph_mount_info *cmount, int val)
+extern "C" int ceph_localize_reads(struct ceph_mount_info *cmount, int val)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -1102,33 +1102,33 @@ extern "C" int ceph_localize_reads(class ceph_mount_info *cmount, int val)
return 0;
}
-extern "C" CephContext *ceph_get_mount_context(class ceph_mount_info *cmount)
+extern "C" CephContext *ceph_get_mount_context(struct ceph_mount_info *cmount)
{
return cmount->get_ceph_context();
}
-extern "C" int ceph_debug_get_fd_caps(class ceph_mount_info *cmount, int fd)
+extern "C" int ceph_debug_get_fd_caps(struct ceph_mount_info *cmount, int fd)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->get_caps_issued(fd);
}
-extern "C" int ceph_debug_get_file_caps(class ceph_mount_info *cmount, const char *path)
+extern "C" int ceph_debug_get_file_caps(struct ceph_mount_info *cmount, const char *path)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->get_caps_issued(path);
}
-extern "C" int ceph_get_stripe_unit_granularity(class ceph_mount_info *cmount)
+extern "C" int ceph_get_stripe_unit_granularity(struct ceph_mount_info *cmount)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return CEPH_MIN_STRIPE_UNIT;
}
-extern "C" int ceph_get_pool_id(class ceph_mount_info *cmount, const char *pool_name)
+extern "C" int ceph_get_pool_id(struct ceph_mount_info *cmount, const char *pool_name)
{
if (!cmount->is_mounted())
return -ENOTCONN;
@@ -1145,7 +1145,7 @@ extern "C" int ceph_get_pool_id(class ceph_mount_info *cmount, const char *pool_
return (int)pool_id;
}
-extern "C" int ceph_get_pool_replication(class ceph_mount_info *cmount, int pool_id)
+extern "C" int ceph_get_pool_replication(struct ceph_mount_info *cmount, int pool_id)
{
if (!cmount->is_mounted())
return -ENOTCONN;
diff --git a/src/librados/AioCompletionImpl.h b/src/librados/AioCompletionImpl.h
index 63a56db..48941bb 100644
--- a/src/librados/AioCompletionImpl.h
+++ b/src/librados/AioCompletionImpl.h
@@ -39,8 +39,7 @@ struct librados::AioCompletionImpl {
// for read
bool is_read;
- bufferlist bl, *pbl;
- char *buf;
+ bufferlist bl;
unsigned maxlen;
IoCtxImpl *io;
@@ -54,7 +53,7 @@ struct librados::AioCompletionImpl {
callback_safe(0),
callback_complete_arg(0),
callback_safe_arg(0),
- is_read(false), pbl(0), buf(0), maxlen(0),
+ is_read(false), maxlen(0),
io(NULL), aio_write_seq(0), aio_write_list_item(this) { }
int set_complete_callback(void *cb_arg, rados_callback_t cb) {
diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index aaa9873..929e105 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -64,6 +64,18 @@ int librados::IoCtxImpl::set_snap_write_context(snapid_t seq, vector<snapid_t>&
return 0;
}
+uint32_t librados::IoCtxImpl::get_object_hash_position(const std::string& oid)
+{
+ Mutex::Locker l(*lock);
+ return objecter->get_object_hash_position(poolid, oid, oloc.nspace);
+}
+
+uint32_t librados::IoCtxImpl::get_object_pg_hash_position(const std::string& oid)
+{
+ Mutex::Locker l(*lock);
+ return objecter->get_object_pg_hash_position(poolid, oid, oloc.nspace);
+}
+
void librados::IoCtxImpl::queue_aio_write(AioCompletionImpl *c)
{
get();
@@ -370,7 +382,7 @@ int librados::IoCtxImpl::list(Objecter::ListContext *context, int max_entries)
object_t oid;
Mutex mylock("IoCtxImpl::list::mylock");
- if (context->at_end)
+ if (context->at_end())
return 0;
context->max_entries = max_entries;
@@ -388,6 +400,14 @@ int librados::IoCtxImpl::list(Objecter::ListContext *context, int max_entries)
return r;
}
+uint32_t librados::IoCtxImpl::list_seek(Objecter::ListContext *context,
+ uint32_t pos)
+{
+ Mutex::Locker l(*lock);
+ context->list.clear();
+ return objecter->list_objects_seek(context, pos);
+}
+
int librados::IoCtxImpl::create(const object_t& oid, bool exclusive)
{
::ObjectOperation op;
@@ -508,10 +528,11 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
int op = o->ops[0].op.op;
ldout(client->cct, 10) << ceph_osd_op_name(op) << " oid=" << oid << " nspace=" << oloc.nspace << dendl;
+ Objecter::Op *objecter_op = objecter->prepare_mutate_op(oid, oloc,
+ *o, snapc, ut, 0,
+ NULL, oncommit, &ver);
lock->Lock();
- objecter->mutate(oid, oloc,
- *o, snapc, ut, 0,
- NULL, oncommit, &ver);
+ objecter->op_submit(objecter_op);
lock->Unlock();
mylock.Lock();
@@ -542,10 +563,11 @@ int librados::IoCtxImpl::operate_read(const object_t& oid,
int op = o->ops[0].op.op;
ldout(client->cct, 10) << ceph_osd_op_name(op) << " oid=" << oid << " nspace=" << oloc.nspace << dendl;
+ Objecter::Op *objecter_op = objecter->prepare_read_op(oid, oloc,
+ *o, snap_seq, pbl, 0,
+ onack, &ver);
lock->Lock();
- objecter->read(oid, oloc,
- *o, snap_seq, pbl, 0,
- onack, &ver);
+ objecter->op_submit(objecter_op);
lock->Unlock();
mylock.Lock();
@@ -570,18 +592,18 @@ int librados::IoCtxImpl::aio_operate_read(const object_t &oid,
c->is_read = true;
c->io = this;
- c->pbl = pbl;
- Mutex::Locker l(*lock);
- objecter->read(oid, oloc,
+ Objecter::Op *objecter_op = objecter->prepare_read_op(oid, oloc,
*o, snap_seq, pbl, flags,
onack, &c->objver);
+ Mutex::Locker l(*lock);
+ objecter->op_submit(objecter_op);
return 0;
}
int librados::IoCtxImpl::aio_operate(const object_t& oid,
::ObjectOperation *o, AioCompletionImpl *c,
- const SnapContext& snap_context)
+ const SnapContext& snap_context, int flags)
{
utime_t ut = ceph_clock_now(client->cct);
/* can't write to a snapshot */
@@ -595,7 +617,7 @@ int librados::IoCtxImpl::aio_operate(const object_t& oid,
queue_aio_write(c);
Mutex::Locker l(*lock);
- objecter->mutate(oid, oloc, *o, snap_context, ut, 0, onack, oncommit,
+ objecter->mutate(oid, oloc, *o, snap_context, ut, flags, onack, oncommit,
&c->objver);
return 0;
@@ -612,11 +634,10 @@ int librados::IoCtxImpl::aio_read(const object_t oid, AioCompletionImpl *c,
c->is_read = true;
c->io = this;
- c->pbl = pbl;
Mutex::Locker l(*lock);
objecter->read(oid, oloc,
- off, len, snapid, &c->bl, 0,
+ off, len, snapid, pbl, 0,
onack, &c->objver);
return 0;
}
@@ -632,8 +653,9 @@ int librados::IoCtxImpl::aio_read(const object_t oid, AioCompletionImpl *c,
c->is_read = true;
c->io = this;
- c->buf = buf;
c->maxlen = len;
+ c->bl.clear();
+ c->bl.push_back(buffer::create_static(len, buf));
Mutex::Locker l(*lock);
objecter->read(oid, oloc,
@@ -668,7 +690,6 @@ int librados::IoCtxImpl::aio_sparse_read(const object_t oid,
c->is_read = true;
c->io = this;
- c->pbl = NULL;
onack->m_ops.sparse_read(off, len, m, data_bl, NULL);
@@ -788,6 +809,37 @@ int librados::IoCtxImpl::aio_stat(const object_t& oid, AioCompletionImpl *c,
return 0;
}
+int librados::IoCtxImpl::hit_set_list(uint32_t hash, AioCompletionImpl *c,
+ std::list< std::pair<time_t, time_t> > *pls)
+{
+ Context *onack = new C_aio_Ack(c);
+ c->is_read = true;
+ c->io = this;
+
+ Mutex::Locker l(*lock);
+ ::ObjectOperation rd;
+ rd.hit_set_ls(pls, NULL);
+ object_locator_t oloc(poolid);
+ objecter->pg_read(hash, oloc, rd, NULL, 0, onack, NULL);
+ return 0;
+}
+
+int librados::IoCtxImpl::hit_set_get(uint32_t hash, AioCompletionImpl *c,
+ time_t stamp,
+ bufferlist *pbl)
+{
+ Context *onack = new C_aio_Ack(c);
+ c->is_read = true;
+ c->io = this;
+
+ Mutex::Locker l(*lock);
+ ::ObjectOperation rd;
+ rd.hit_set_get(utime_t(stamp, 0), pbl, 0);
+ object_locator_t oloc(poolid);
+ objecter->pg_read(hash, oloc, rd, NULL, 0, onack, NULL);
+ return 0;
+}
+
int librados::IoCtxImpl::remove(const object_t& oid)
{
::ObjectOperation op;
@@ -828,6 +880,13 @@ int librados::IoCtxImpl::tmap_get(const object_t& oid, bufferlist& bl)
return operate_read(oid, &rd, NULL);
}
+int librados::IoCtxImpl::tmap_to_omap(const object_t& oid, bool nullok)
+{
+ ::ObjectOperation wr;
+ prepare_assert_ops(&wr);
+ wr.tmap_to_omap(nullok);
+ return operate(oid, &wr, NULL);
+}
int librados::IoCtxImpl::exec(const object_t& oid,
const char *cls, const char *method,
@@ -1179,14 +1238,9 @@ void librados::IoCtxImpl::C_aio_Ack::finish(int r)
c->safe = true;
c->cond.Signal();
- if (c->buf && c->bl.length() > 0) {
- unsigned l = MIN(c->bl.length(), c->maxlen);
- c->bl.copy(0, l, c->buf);
+ if (c->bl.length() > 0) {
c->rval = c->bl.length();
}
- if (c->pbl) {
- *c->pbl = c->bl;
- }
if (c->callback_complete) {
c->io->client->finisher.queue(new C_AioComplete(c));
diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h
index ccecd4e..8693d6c 100644
--- a/src/librados/IoCtxImpl.h
+++ b/src/librados/IoCtxImpl.h
@@ -91,6 +91,9 @@ struct librados::IoCtxImpl {
return poolid;
}
+ uint32_t get_object_hash_position(const std::string& oid);
+ uint32_t get_object_pg_hash_position(const std::string& oid);
+
::ObjectOperation *prepare_assert_ops(::ObjectOperation *op);
// snaps
@@ -108,6 +111,7 @@ struct librados::IoCtxImpl {
// io
int list(Objecter::ListContext *context, int max_entries);
+ uint32_t list_seek(Objecter::ListContext *context, uint32_t pos);
int create(const object_t& oid, bool exclusive);
int create(const object_t& oid, bool exclusive, const std::string& category);
int write(const object_t& oid, bufferlist& bl, size_t len, uint64_t off);
@@ -127,6 +131,7 @@ struct librados::IoCtxImpl {
int tmap_update(const object_t& oid, bufferlist& cmdbl);
int tmap_put(const object_t& oid, bufferlist& bl);
int tmap_get(const object_t& oid, bufferlist& bl);
+ int tmap_to_omap(const object_t& oid, bool nullok=false);
int exec(const object_t& oid, const char *cls, const char *method, bufferlist& inbl, bufferlist& outbl);
@@ -138,7 +143,8 @@ struct librados::IoCtxImpl {
int operate(const object_t& oid, ::ObjectOperation *o, time_t *pmtime);
int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl);
int aio_operate(const object_t& oid, ::ObjectOperation *o,
- AioCompletionImpl *c, const SnapContext& snap_context);
+ AioCompletionImpl *c, const SnapContext& snap_context,
+ int flags);
int aio_operate_read(const object_t& oid, ::ObjectOperation *o,
AioCompletionImpl *c, int flags, bufferlist *pbl);
@@ -183,6 +189,11 @@ struct librados::IoCtxImpl {
int pool_change_auid(unsigned long long auid);
int pool_change_auid_async(unsigned long long auid, PoolAsyncCompletionImpl *c);
+ int hit_set_list(uint32_t hash, AioCompletionImpl *c,
+ std::list< std::pair<time_t, time_t> > *pls);
+ int hit_set_get(uint32_t hash, AioCompletionImpl *c, time_t stamp,
+ bufferlist *pbl);
+
void set_sync_op_version(version_t ver);
int watch(const object_t& oid, uint64_t ver, uint64_t *cookie, librados::WatchCtx *ctx);
int unwatch(const object_t& oid, uint64_t cookie);
diff --git a/src/librados/Makefile.am b/src/librados/Makefile.am
index 570aa91..23e9167 100644
--- a/src/librados/Makefile.am
+++ b/src/librados/Makefile.am
@@ -5,11 +5,14 @@ librados_la_SOURCES = \
librados/snap_set_diff.cc
# We need this to avoid basename conflicts with the librados build tests in test/Makefile.am
-librados_la_CFLAGS = ${AM_CFLAGS}
+librados_la_CXXFLAGS = ${AM_CXXFLAGS}
LIBRADOS_DEPS += libcls_lock_client.la $(LIBOSDC) $(LIBCOMMON)
librados_la_LIBADD = $(LIBRADOS_DEPS) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS)
-librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 -export-symbols-regex '^rados_.*'
+librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0
+if LINUX
+librados_la_LDFLAGS += -export-symbols-regex '^rados_.*'
+endif
lib_LTLIBRARIES += librados.la
noinst_HEADERS += \
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index d6700c8..add1bb0 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -72,8 +72,12 @@ librados::RadosClient::RadosClient(CephContext *cct_)
state(DISCONNECTED),
monclient(cct_),
messenger(NULL),
+ instance_id(0),
objecter(NULL),
+ osdmap_epoch(0),
+ pool_cache_epoch(0),
lock("librados::RadosClient::lock"),
+ pool_cache_rwl("librados::RadosClient::pool_cache_rwl"),
timer(cct, lock),
refcnt(1),
log_last_version(0), log_cb(NULL), log_cb_arg(NULL),
@@ -84,11 +88,35 @@ librados::RadosClient::RadosClient(CephContext *cct_)
int64_t librados::RadosClient::lookup_pool(const char *name)
{
- Mutex::Locker l(lock);
+ pool_cache_rwl.get_read();
+ if (pool_cache_epoch && pool_cache_epoch == osdmap_epoch) {
+ map<string, int64_t>::iterator iter = pool_cache.find(name);
+ if (iter != pool_cache.end()) {
+ uint64_t val = iter->second;
+ pool_cache_rwl.unlock();
+ return val;
+ }
+ }
+
+ pool_cache_rwl.unlock();
+
+ lock.Lock();
+
wait_for_osdmap();
int64_t ret = osdmap.lookup_pg_pool_name(name);
- if (ret < 0)
+ pool_cache_rwl.get_write();
+ lock.Unlock();
+ if (ret < 0) {
+ pool_cache_rwl.unlock();
return -ENOENT;
+ }
+
+ if (pool_cache_epoch != osdmap_epoch) {
+ pool_cache.clear();
+ pool_cache_epoch = osdmap_epoch;
+ }
+ pool_cache[name] = ret;
+ pool_cache_rwl.unlock();
return ret;
}
@@ -227,6 +255,7 @@ int librados::RadosClient::connect()
finisher.start();
state = CONNECTED;
+ instance_id = monclient.get_global_id();
lock.Unlock();
@@ -255,6 +284,7 @@ void librados::RadosClient::shutdown()
objecter->shutdown_locked();
}
state = DISCONNECTED;
+ instance_id = 0;
timer.shutdown(); // will drop+retake lock
lock.Unlock();
monclient.shutdown();
@@ -269,11 +299,7 @@ void librados::RadosClient::shutdown()
uint64_t librados::RadosClient::get_instance_id()
{
- Mutex::Locker l(lock);
- if (state == DISCONNECTED)
- return 0;
- uint64_t id = monclient.get_global_id();
- return id;
+ return instance_id;
}
librados::RadosClient::~RadosClient()
@@ -341,6 +367,9 @@ bool librados::RadosClient::_dispatch(Message *m)
break;
case CEPH_MSG_OSD_MAP:
objecter->handle_osd_map(static_cast<MOSDMap*>(m));
+ pool_cache_rwl.get_write();
+ osdmap_epoch = osdmap.get_epoch();
+ pool_cache_rwl.unlock();
cond.Signal();
break;
case MSG_GETPOOLSTATSREPLY:
@@ -388,6 +417,24 @@ void librados::RadosClient::wait_for_osdmap()
}
}
+int librados::RadosClient::wait_for_latest_osdmap()
+{
+ Mutex mylock("RadosClient::wait_for_latest_osdmap");
+ Cond cond;
+ bool done;
+
+ lock.Lock();
+ objecter->wait_for_latest_osdmap(new C_SafeCond(&mylock, &cond, &done));
+ lock.Unlock();
+
+ mylock.Lock();
+ while (!done)
+ cond.Wait(mylock);
+ mylock.Unlock();
+
+ return 0;
+}
+
int librados::RadosClient::pool_list(std::list<std::string>& v)
{
Mutex::Locker l(lock);
@@ -592,8 +639,8 @@ void librados::RadosClient::watch_notify(MWatchNotify *m)
}
int librados::RadosClient::mon_command(const vector<string>& cmd,
- bufferlist &inbl,
- bufferlist *outbl, string *outs)
+ const bufferlist &inbl,
+ bufferlist *outbl, string *outs)
{
Mutex mylock("RadosClient::mon_command::mylock");
Cond cond;
@@ -611,8 +658,8 @@ int librados::RadosClient::mon_command(const vector<string>& cmd,
}
int librados::RadosClient::mon_command(int rank, const vector<string>& cmd,
- bufferlist &inbl,
- bufferlist *outbl, string *outs)
+ const bufferlist &inbl,
+ bufferlist *outbl, string *outs)
{
Mutex mylock("RadosClient::mon_command::mylock");
Cond cond;
@@ -630,8 +677,8 @@ int librados::RadosClient::mon_command(int rank, const vector<string>& cmd,
}
int librados::RadosClient::mon_command(string name, const vector<string>& cmd,
- bufferlist &inbl,
- bufferlist *outbl, string *outs)
+ const bufferlist &inbl,
+ bufferlist *outbl, string *outs)
{
Mutex mylock("RadosClient::mon_command::mylock");
Cond cond;
@@ -649,8 +696,8 @@ int librados::RadosClient::mon_command(string name, const vector<string>& cmd,
}
int librados::RadosClient::osd_command(int osd, vector<string>& cmd,
- bufferlist& inbl,
- bufferlist *poutbl, string *prs)
+ const bufferlist& inbl,
+ bufferlist *poutbl, string *prs)
{
Mutex mylock("RadosClient::osd_command::mylock");
Cond cond;
@@ -676,8 +723,8 @@ int librados::RadosClient::osd_command(int osd, vector<string>& cmd,
}
int librados::RadosClient::pg_command(pg_t pgid, vector<string>& cmd,
- bufferlist& inbl,
- bufferlist *poutbl, string *prs)
+ const bufferlist& inbl,
+ bufferlist *poutbl, string *prs)
{
Mutex mylock("RadosClient::pg_command::mylock");
Cond cond;
diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index 7c5c8af..f9cc1f7 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -16,6 +16,7 @@
#include "common/Cond.h"
#include "common/Mutex.h"
+#include "common/RWLock.h"
#include "common/Timer.h"
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
@@ -50,6 +51,8 @@ private:
MonClient monclient;
SimpleMessenger *messenger;
+ uint64_t instance_id;
+
bool _dispatch(Message *m);
bool ms_dispatch(Message *m);
@@ -60,7 +63,13 @@ private:
Objecter *objecter;
+ map<string, int64_t> pool_cache;
+
+ epoch_t osdmap_epoch;
+ epoch_t pool_cache_epoch;
+
Mutex lock;
+ RWLock pool_cache_rwl;
Cond cond;
SafeTimer timer;
int refcnt;
@@ -83,6 +92,8 @@ public:
uint64_t get_instance_id();
+ int wait_for_latest_osdmap();
+
int create_ioctx(const char *name, IoCtxImpl **io);
int get_fsid(std::string *s);
@@ -109,17 +120,17 @@ public:
void register_watcher(librados::WatchContext *wc, uint64_t *cookie);
void unregister_watcher(uint64_t cookie);
void watch_notify(MWatchNotify *m);
- int mon_command(const vector<string>& cmd, bufferlist &inbl,
+ int mon_command(const vector<string>& cmd, const bufferlist &inbl,
bufferlist *outbl, string *outs);
int mon_command(int rank,
- const vector<string>& cmd, bufferlist &inbl,
+ const vector<string>& cmd, const bufferlist &inbl,
bufferlist *outbl, string *outs);
int mon_command(string name,
- const vector<string>& cmd, bufferlist &inbl,
+ const vector<string>& cmd, const bufferlist &inbl,
bufferlist *outbl, string *outs);
- int osd_command(int osd, vector<string>& cmd, bufferlist& inbl,
+ int osd_command(int osd, vector<string>& cmd, const bufferlist& inbl,
bufferlist *poutbl, string *prs);
- int pg_command(pg_t pgid, vector<string>& cmd, bufferlist& inbl,
+ int pg_command(pg_t pgid, vector<string>& cmd, const bufferlist& inbl,
bufferlist *poutbl, string *prs);
void handle_log(MLog *m);
diff --git a/src/librados/librados.cc b/src/librados/librados.cc
index 95abbc2..4925d5e 100644
--- a/src/librados/librados.cc
+++ b/src/librados/librados.cc
@@ -275,8 +275,6 @@ void librados::ObjectReadOperation::is_dirty(bool *is_dirty, int *prval)
o->is_dirty(is_dirty, prval);
}
-
-
int librados::IoCtx::omap_get_vals(const std::string& oid,
const std::string& start_after,
const std::string& filter_prefix,
@@ -395,7 +393,8 @@ void librados::ObjectWriteOperation::copy_from(const std::string& src,
uint64_t src_version)
{
::ObjectOperation *o = (::ObjectOperation *)impl;
- o->copy_from(object_t(src), src_ioctx.io_ctx_impl->snap_seq, src_ioctx.io_ctx_impl->oloc, src_version);
+ o->copy_from(object_t(src), src_ioctx.io_ctx_impl->snap_seq,
+ src_ioctx.io_ctx_impl->oloc, src_version, 0);
}
void librados::ObjectWriteOperation::undirty()
@@ -404,6 +403,24 @@ void librados::ObjectWriteOperation::undirty()
o->undirty();
}
+void librados::ObjectReadOperation::cache_flush()
+{
+ ::ObjectOperation *o = (::ObjectOperation *)impl;
+ o->cache_flush();
+}
+
+void librados::ObjectReadOperation::cache_try_flush()
+{
+ ::ObjectOperation *o = (::ObjectOperation *)impl;
+ o->cache_try_flush();
+}
+
+void librados::ObjectReadOperation::cache_evict()
+{
+ ::ObjectOperation *o = (::ObjectOperation *)impl;
+ o->cache_evict();
+}
+
void librados::ObjectWriteOperation::tmap_put(const bufferlist &bl)
{
::ObjectOperation *o = (::ObjectOperation *)impl;
@@ -488,6 +505,13 @@ librados::ObjectIterator librados::ObjectIterator::operator++(int)
return ret;
}
+uint32_t librados::ObjectIterator::seek(uint32_t pos)
+{
+ uint32_t r = rados_objects_list_seek(ctx.get(), pos);
+ get_next();
+ return r;
+}
+
void librados::ObjectIterator::get_next()
{
const char *entry, *key;
@@ -506,6 +530,11 @@ void librados::ObjectIterator::get_next()
cur_obj = make_pair(entry, key ? key : string());
}
+uint32_t librados::ObjectIterator::get_pg_hash_position() const
+{
+ return ctx->lc->get_pg_hash_position();
+}
+
const librados::ObjectIterator librados::ObjectIterator::__EndObjectIterator(NULL);
///////////////////////////// PoolAsyncCompletion //////////////////////////////
@@ -827,6 +856,12 @@ int librados::IoCtx::tmap_get(const std::string& oid, bufferlist& bl)
return io_ctx_impl->tmap_get(obj, bl);
}
+int librados::IoCtx::tmap_to_omap(const std::string& oid, bool nullok)
+{
+ object_t obj(oid);
+ return io_ctx_impl->tmap_to_omap(obj, nullok);
+}
+
int librados::IoCtx::omap_get_vals(const std::string& oid,
const std::string& start_after,
uint64_t max_return,
@@ -919,6 +954,27 @@ int librados::IoCtx::omap_rm_keys(const std::string& oid,
return operate(oid, &op);
}
+
+
+static int translate_flags(int flags)
+{
+ int op_flags = 0;
+ if (flags & librados::OPERATION_BALANCE_READS)
+ op_flags |= CEPH_OSD_FLAG_BALANCE_READS;
+ if (flags & librados::OPERATION_LOCALIZE_READS)
+ op_flags |= CEPH_OSD_FLAG_LOCALIZE_READS;
+ if (flags & librados::OPERATION_ORDER_READS_WRITES)
+ op_flags |= CEPH_OSD_FLAG_RWORDERED;
+ if (flags & librados::OPERATION_IGNORE_CACHE)
+ op_flags |= CEPH_OSD_FLAG_IGNORE_CACHE;
+ if (flags & librados::OPERATION_SKIPRWLOCKS)
+ op_flags |= CEPH_OSD_FLAG_SKIPRWLOCKS;
+ if (flags & librados::OPERATION_IGNORE_OVERLAY)
+ op_flags |= CEPH_OSD_FLAG_IGNORE_OVERLAY;
+
+ return op_flags;
+}
+
int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperation *o)
{
object_t obj(oid);
@@ -936,7 +992,15 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
{
object_t obj(oid);
return io_ctx_impl->aio_operate(obj, (::ObjectOperation*)o->impl, c->pc,
- io_ctx_impl->snapc);
+ io_ctx_impl->snapc, 0);
+}
+int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
+ ObjectWriteOperation *o, int flags)
+{
+ object_t obj(oid);
+ return io_ctx_impl->aio_operate(obj, (::ObjectOperation*)o->impl, c->pc,
+ io_ctx_impl->snapc,
+ translate_flags(flags));
}
int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
@@ -950,7 +1014,7 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
snv[i] = snaps[i];
SnapContext snapc(snap_seq, snv);
return io_ctx_impl->aio_operate(obj, (::ObjectOperation*)o->impl, c->pc,
- snapc);
+ snapc, 0);
}
int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
@@ -962,9 +1026,11 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
0, pbl);
}
+// deprecated
int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
librados::ObjectReadOperation *o,
- snap_t snapid, int flags, bufferlist *pbl)
+ snap_t snapid_unused_deprecated,
+ int flags, bufferlist *pbl)
{
object_t obj(oid);
int op_flags = 0;
@@ -979,6 +1045,16 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
op_flags, pbl);
}
+int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
+ librados::ObjectReadOperation *o,
+ int flags, bufferlist *pbl)
+{
+ object_t obj(oid);
+ return io_ctx_impl->aio_operate_read(obj, (::ObjectOperation*)o->impl, c->pc,
+ translate_flags(flags), pbl);
+}
+
+
void librados::IoCtx::snap_set_read(snap_t seq)
{
io_ctx_impl->set_snap_read(seq);
@@ -1131,11 +1207,35 @@ librados::ObjectIterator librados::IoCtx::objects_begin()
return iter;
}
+librados::ObjectIterator librados::IoCtx::objects_begin(uint32_t pos)
+{
+ rados_list_ctx_t listh;
+ rados_objects_list_open(io_ctx_impl, &listh);
+ ObjectIterator iter((ObjListCtx*)listh);
+ iter.seek(pos);
+ iter.get_next();
+ return iter;
+}
+
const librados::ObjectIterator& librados::IoCtx::objects_end() const
{
return ObjectIterator::__EndObjectIterator;
}
+int librados::IoCtx::hit_set_list(uint32_t hash, AioCompletion *c,
+ std::list< std::pair<time_t, time_t> > *pls)
+{
+ return io_ctx_impl->hit_set_list(hash, c->pc, pls);
+}
+
+int librados::IoCtx::hit_set_get(uint32_t hash, AioCompletion *c, time_t stamp,
+ bufferlist *pbl)
+{
+ return io_ctx_impl->hit_set_get(hash, c->pc, stamp, pbl);
+}
+
+
+
uint64_t librados::IoCtx::get_last_version()
{
return io_ctx_impl->last_version();
@@ -1311,6 +1411,16 @@ int64_t librados::IoCtx::get_id()
return io_ctx_impl->get_id();
}
+uint32_t librados::IoCtx::get_object_hash_position(const std::string& oid)
+{
+ return io_ctx_impl->get_object_hash_position(oid);
+}
+
+uint32_t librados::IoCtx::get_object_pg_hash_position(const std::string& oid)
+{
+ return io_ctx_impl->get_object_pg_hash_position(oid);
+}
+
librados::config_t librados::IoCtx::cct()
{
return (config_t)io_ctx_impl->client->cct;
@@ -1487,6 +1597,14 @@ int librados::Rados::pool_reverse_lookup(int64_t id, std::string *name)
return client->pool_get_name(id, name);
}
+int librados::Rados::mon_command(string cmd, const bufferlist& inbl,
+ bufferlist *outbl, string *outs)
+{
+ vector<string> cmdvec;
+ cmdvec.push_back(cmd);
+ return client->mon_command(cmdvec, inbl, outbl, outs);
+}
+
int librados::Rados::ioctx_create(const char *name, IoCtx &io)
{
rados_ioctx_t p;
@@ -1579,6 +1697,11 @@ int librados::Rados::cluster_fsid(string *fsid)
return client->get_fsid(fsid);
}
+int librados::Rados::wait_for_latest_osdmap()
+{
+ return client->wait_for_latest_osdmap();
+}
+
librados::PoolAsyncCompletion *librados::Rados::pool_async_create_completion()
{
PoolAsyncCompletionImpl *c = new PoolAsyncCompletionImpl;
@@ -2506,6 +2629,13 @@ extern "C" int rados_tmap_get(rados_ioctx_t io, const char *o, char *buf, size_t
return bl.length();
}
+extern "C" int rados_tmap_to_omap(rados_ioctx_t io, const char *o, bool nullok)
+{
+ librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
+ object_t oid(o);
+ return ctx->tmap_to_omap(oid, nullok);
+}
+
extern "C" int rados_exec(rados_ioctx_t io, const char *o, const char *cls, const char *method,
const char *inbuf, size_t in_len, char *buf, size_t out_len)
{
@@ -2545,6 +2675,21 @@ extern "C" void rados_objects_list_close(rados_list_ctx_t h)
delete lh;
}
+extern "C" uint32_t rados_objects_list_seek(rados_list_ctx_t listctx,
+ uint32_t pos)
+{
+ librados::ObjListCtx *lh = (librados::ObjListCtx *)listctx;
+ uint32_t r = lh->ctx->list_seek(lh->lc, pos);
+ return r;
+}
+
+extern "C" uint32_t rados_objects_list_get_pg_hash_position(
+ rados_list_ctx_t listctx)
+{
+ librados::ObjListCtx *lh = (librados::ObjListCtx *)listctx;
+ return lh->lc->get_pg_hash_position();
+}
+
extern "C" int rados_objects_list_next(rados_list_ctx_t listctx, const char **entry, const char **key)
{
librados::ObjListCtx *lh = (librados::ObjListCtx *)listctx;
@@ -2866,3 +3011,132 @@ extern "C" int rados_break_lock(rados_ioctx_t io, const char *o,
return ctx.break_lock(o, name, client, cookie);
}
+
+extern "C" rados_write_op_t rados_create_write_op()
+{
+ return new (std::nothrow)::ObjectOperation;
+}
+
+extern "C" void rados_release_write_op(rados_write_op_t write_op)
+{
+ delete (::ObjectOperation*)write_op;
+}
+
+extern "C" void rados_write_op_assert_exists(rados_write_op_t write_op)
+{
+ ((::ObjectOperation *)write_op)->stat(NULL, (utime_t *)NULL, NULL);
+}
+
+extern "C" void rados_write_op_cmpxattr(rados_write_op_t write_op,
+ const char *name,
+ uint8_t comparison_operator,
+ const char *value,
+ size_t value_len)
+{
+ bufferlist bl;
+ bl.append(value, value_len);
+ ((::ObjectOperation *)write_op)->cmpxattr(name,
+ comparison_operator,
+ CEPH_OSD_CMPXATTR_MODE_STRING,
+ bl);
+}
+
+extern "C" void rados_write_op_setxattr(rados_write_op_t write_op,
+ const char *name,
+ const char *value,
+ size_t value_len)
+{
+ bufferlist bl;
+ bl.append(value, value_len);
+ ((::ObjectOperation *)write_op)->setxattr(name, bl);
+}
+
+extern "C" void rados_write_op_rmxattr(rados_write_op_t write_op,
+ const char *name)
+{
+ bufferlist bl;
+ ((::ObjectOperation *)write_op)->rmxattr(name);
+}
+
+extern "C" void rados_write_op_create(rados_write_op_t write_op,
+ int exclusive,
+ const char* category)
+{
+ ::ObjectOperation *oo = (::ObjectOperation *) write_op;
+ if(category) {
+ std::string cpp_category = category;
+ oo->create(exclusive, category);
+ } else {
+ oo->create(!!exclusive);
+ }
+}
+
+extern "C" void rados_write_op_write(rados_write_op_t write_op,
+ const char *buffer,
+ size_t len,
+ uint64_t offset)
+{
+ bufferlist bl;
+ bl.append(buffer,len);
+ ((::ObjectOperation *)write_op)->write(offset, bl);
+}
+
+extern "C" void rados_write_op_write_full(rados_write_op_t write_op,
+ const char *buffer,
+ size_t len)
+{
+ bufferlist bl;
+ bl.append(buffer,len);
+ ((::ObjectOperation *)write_op)->write_full(bl);
+}
+
+extern "C" void rados_write_op_append(rados_write_op_t write_op,
+ const char *buffer,
+ size_t len)
+{
+ bufferlist bl;
+ bl.append(buffer,len);
+ ((::ObjectOperation *)write_op)->append(bl);
+}
+
+extern "C" void rados_write_op_remove(rados_write_op_t write_op)
+{
+ ((::ObjectOperation *)write_op)->remove();
+}
+
+extern "C" void rados_write_op_truncate(rados_write_op_t write_op,
+ uint64_t offset)
+{
+ ((::ObjectOperation *)write_op)->truncate(offset);
+}
+
+extern "C" void rados_write_op_zero(rados_write_op_t write_op,
+ uint64_t offset,
+ uint64_t len)
+{
+ ((::ObjectOperation *)write_op)->zero(offset, len);
+}
+
+extern "C" int rados_write_op_operate(rados_write_op_t write_op,
+ rados_ioctx_t io,
+ const char *oid,
+ time_t *mtime)
+{
+ object_t obj(oid);
+ ::ObjectOperation *oo = (::ObjectOperation *) write_op;
+ librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
+ return ctx->operate(obj, oo, mtime);
+}
+
+extern "C" int rados_aio_write_op_operate(rados_write_op_t write_op,
+ rados_ioctx_t io,
+ rados_completion_t completion,
+ const char *oid,
+ time_t *mtime)
+{
+ object_t obj(oid);
+ ::ObjectOperation *oo = (::ObjectOperation *) write_op;
+ librados::IoCtxImpl *ctx = (librados::IoCtxImpl *)io;
+ librados::AioCompletionImpl *c = (librados::AioCompletionImpl*)completion;
+ return ctx->aio_operate(obj, oo, c, ctx->snapc, 0);
+}
diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h
index 899586d..aaccefe 100644
--- a/src/librbd/AioCompletion.h
+++ b/src/librbd/AioCompletion.h
@@ -101,9 +101,6 @@ namespace librbd {
utime_t elapsed;
assert(lock.is_locked());
elapsed = ceph_clock_now(ictx->cct) - start_time;
- if (complete_cb) {
- complete_cb(rbd_comp, complete_arg);
- }
switch (aio_type) {
case AIO_TYPE_READ:
ictx->perfcounter->tinc(l_librbd_aio_rd_latency, elapsed); break;
@@ -117,6 +114,9 @@ namespace librbd {
lderr(ictx->cct) << "completed invalid aio_type: " << aio_type << dendl;
break;
}
+ if (complete_cb) {
+ complete_cb(rbd_comp, complete_arg);
+ }
done = true;
cond.Signal();
}
@@ -196,12 +196,10 @@ namespace librbd {
class C_CacheRead : public Context {
public:
- C_CacheRead(Context *completion, AioRead *req)
- : m_completion(completion), m_req(req) {}
+ explicit C_CacheRead(AioRead *req) : m_req(req) {}
virtual ~C_CacheRead() {}
virtual void finish(int r);
private:
- Context *m_completion;
AioRead *m_req;
};
}
diff --git a/src/librbd/AioRequest.cc b/src/librbd/AioRequest.cc
index 11e111f..6e4a05e 100644
--- a/src/librbd/AioRequest.cc
+++ b/src/librbd/AioRequest.cc
@@ -99,7 +99,7 @@ namespace librbd {
} else {
op.read(m_object_off, m_object_len, &m_read_data, NULL);
}
- r = m_ioctx->aio_operate(m_oid, rados_completion, &op, m_snap_id, flags, NULL);
+ r = m_ioctx->aio_operate(m_oid, rados_completion, &op, flags, NULL);
rados_completion->release();
return r;
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 41518b6..6477e8d 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -45,6 +45,7 @@ namespace librbd {
snap_lock("librbd::ImageCtx::snap_lock"),
parent_lock("librbd::ImageCtx::parent_lock"),
refresh_lock("librbd::ImageCtx::refresh_lock"),
+ extra_read_flags(0),
old_format(true),
order(0), size(0), features(0),
format_string(NULL),
@@ -238,8 +239,12 @@ namespace librbd {
delete perfcounter;
}
+ void ImageCtx::set_read_flag(unsigned flag) {
+ extra_read_flags |= flag;
+ }
+
int ImageCtx::get_read_flags(snap_t snap_id) {
- int flags = librados::OPERATION_NOFLAG;
+ int flags = librados::OPERATION_NOFLAG | extra_read_flags;
if (snap_id == LIBRADOS_SNAP_HEAD)
return flags;
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index c9e7439..026a3e0 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -69,6 +69,8 @@ namespace librbd {
RWLock parent_lock; // protects parent_md and parent
Mutex refresh_lock; // protects refresh_seq and last_refresh
+ unsigned extra_read_flags;
+
bool old_format;
uint8_t order;
uint64_t size;
@@ -99,6 +101,7 @@ namespace librbd {
void init_layout();
void perf_start(std::string name);
void perf_stop();
+ void set_read_flag(unsigned flag);
int get_read_flags(librados::snap_t snap_id);
int snap_set(std::string in_snap_name);
void snap_unset();
diff --git a/src/librbd/LibrbdWriteback.cc b/src/librbd/LibrbdWriteback.cc
index da02a34..ddb882d 100644
--- a/src/librbd/LibrbdWriteback.cc
+++ b/src/librbd/LibrbdWriteback.cc
@@ -103,7 +103,7 @@ namespace librbd {
op.read(off, len, pbl, NULL);
int flags = m_ictx->get_read_flags(snapid);
int r = m_ictx->data_ctx.aio_operate(oid.name, rados_completion, &op,
- snapid, flags, NULL);
+ flags, NULL);
rados_completion->release();
assert(r >= 0);
}
diff --git a/src/librbd/LibrbdWriteback.h b/src/librbd/LibrbdWriteback.h
index 4f35aa4..0967421 100644
--- a/src/librbd/LibrbdWriteback.h
+++ b/src/librbd/LibrbdWriteback.h
@@ -55,7 +55,7 @@ namespace librbd {
tid_t m_tid;
Mutex& m_lock;
librbd::ImageCtx *m_ictx;
- hash_map<std::string, std::queue<write_result_d*> > m_writes;
+ ceph::unordered_map<std::string, std::queue<write_result_d*> > m_writes;
friend class C_OrderedWrite;
};
}
diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am
index d4e2455..0a3600d 100644
--- a/src/librbd/Makefile.am
+++ b/src/librbd/Makefile.am
@@ -10,7 +10,10 @@ librbd_la_LIBADD = \
$(LIBRADOS) $(LIBOSDC) \
libcls_rbd_client.la libcls_lock_client.la \
$(PTHREAD_LIBS) $(EXTRALIBS)
-librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^rbd_.*'
+librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0
+if LINUX
+librbd_la_LDFLAGS += -export-symbols-regex '^rbd_.*'
+endif
lib_LTLIBRARIES += librbd.la
noinst_HEADERS += \
diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 4185f36..61e910b 100644
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -28,7 +28,7 @@
#undef dout_prefix
#define dout_prefix *_dout << "librbd: "
-#define howmany(x, y) (((x) + (y) - 1) / (y))
+#define rbd_howmany(x, y) (((x) + (y) - 1) / (y))
using std::map;
using std::pair;
@@ -119,7 +119,7 @@ namespace librbd {
ictx->snap_lock.put_read();
ictx->md_lock.put_read();
info.obj_size = 1ULL << obj_order;
- info.num_objs = howmany(info.size, ictx->get_object_size());
+ info.num_objs = rbd_howmany(info.size, ictx->get_object_size());
info.order = obj_order;
memcpy(&info.block_name_prefix, ictx->object_prefix.c_str(),
min((size_t)RBD_MAX_BLOCK_NAME_SIZE,
@@ -616,8 +616,11 @@ namespace librbd {
if (r < 0)
return r;
- if (is_unprotected)
+ if (is_unprotected) {
+ lderr(ictx->cct) << "snap_unprotect: snapshot is already unprotected"
+ << dendl;
return -EINVAL;
+ }
r = cls_client::set_protection_status(&ictx->md_ctx,
ictx->header_oid,
@@ -1246,6 +1249,13 @@ reprotect_and_return_err:
// since we don't know the image and snapshot name, set their ids and
// reset the snap_name and snap_exists fields after we read the header
ictx->parent = new ImageCtx("", parent_image_id, NULL, p_ioctx, true);
+
+ // set rados flags for reading the parent image
+ if (ictx->cct->_conf->rbd_balance_parent_reads)
+ ictx->parent->set_read_flag(librados::OPERATION_BALANCE_READS);
+ else if (ictx->cct->_conf->rbd_localize_parent_reads)
+ ictx->parent->set_read_flag(librados::OPERATION_LOCALIZE_READS);
+
r = open_image(ictx->parent);
if (r < 0) {
lderr(ictx->cct) << "error opening parent image: " << cpp_strerror(r)
@@ -1343,15 +1353,31 @@ reprotect_and_return_err:
if (r < 0) {
ldout(cct, 2) << "error opening image: " << cpp_strerror(-r) << dendl;
} else {
+ string header_oid = ictx->header_oid;
+ old_format = ictx->old_format;
+ unknown_format = false;
+ id = ictx->id;
+
if (ictx->snaps.size()) {
lderr(cct) << "image has snapshots - not removing" << dendl;
close_image(ictx);
return -ENOTEMPTY;
}
- string header_oid = ictx->header_oid;
- old_format = ictx->old_format;
- unknown_format = false;
- id = ictx->id;
+
+ std::list<obj_watch_t> watchers;
+ r = io_ctx.list_watchers(header_oid, &watchers);
+ if (r < 0) {
+ lderr(cct) << "error listing watchers" << dendl;
+ close_image(ictx);
+ return r;
+ }
+ if (watchers.size() > 1) {
+ lderr(cct) << "image has watchers - not removing" << dendl;
+ close_image(ictx);
+ return -EBUSY;
+ }
+ assert(watchers.size() == 1);
+
ictx->md_lock.get_read();
trim_image(ictx, 0, prog_ctx);
ictx->md_lock.put_read();
@@ -1365,6 +1391,7 @@ reprotect_and_return_err:
parent_info.spec, id);
if (r < 0 && r != -ENOENT) {
lderr(cct) << "error removing child from children list" << dendl;
+ close_image(ictx);
return r;
}
close_image(ictx);
@@ -3080,7 +3107,7 @@ reprotect_and_return_err:
c->add_request();
if (ictx->object_cacher) {
- C_CacheRead *cache_comp = new C_CacheRead(req_comp, req);
+ C_CacheRead *cache_comp = new C_CacheRead(req);
ictx->aio_read_from_cache(q->oid, &req->data(),
q->length, q->offset,
cache_comp);
diff --git a/src/libs3/debian/changelog b/src/libs3/debian/changelog
deleted file mode 100644
index 520b2b9..0000000
--- a/src/libs3/debian/changelog
+++ /dev/null
@@ -1,5 +0,0 @@
-libs3 (all) unstable; urgency=low
-
- * This file is not maintained. See project source code for changes.
-
- -- Bryan Ischo <bryan at ischo.com> Wed, 06 Aug 2008 09:36:43 -0400
diff --git a/src/libs3/debian/changelog.Debian b/src/libs3/debian/changelog.Debian
deleted file mode 100644
index 81072be..0000000
--- a/src/libs3/debian/changelog.Debian
+++ /dev/null
@@ -1,6 +0,0 @@
-libs3 (all) unstable; urgency=low
-
- * libs3 Debian maintainer and upstream author are identical.
- Therefore see normal changelog file for Debian changes.
-
- -- Bryan Ischo <bryan at ischo.com> Wed, 06 Aug 2008 09:36:43 -0400
diff --git a/src/libs3/debian/control b/src/libs3/debian/control
deleted file mode 100644
index 28ddc6b..0000000
--- a/src/libs3/debian/control
+++ /dev/null
@@ -1,12 +0,0 @@
-Package: libs3
-Source: THIS LINE WILL BE REMOVED, dpkg-shlibdepends NEEDS IT
-Version: LIBS3_VERSION
-Architecture: DEBIAN_ARCHITECTURE
-Section: net
-Priority: extra
-Maintainer: Bryan Ischo <bryan at ischo.com>
-Homepage: http://libs3.ischo.com/index.html
-Description: C Library and Tools for Amazon S3 Access
- This package includes the libs3 shared object library, needed to run
- applications compiled against libs3, and additionally contains the s3
- utility for accessing Amazon S3.
diff --git a/src/libs3/debian/control.dev b/src/libs3/debian/control.dev
deleted file mode 100644
index 5ee5ae7..0000000
--- a/src/libs3/debian/control.dev
+++ /dev/null
@@ -1,26 +0,0 @@
-Package: libs3-dev
-Version: LIBS3_VERSION
-Architecture: DEBIAN_ARCHITECTURE
-Section: libdevel
-Priority: extra
-Depends: libs3 (>= LIBS3_VERSION)
-Maintainer: Bryan Ischo <bryan at ischo.com>
-Homepage: http://libs3.ischo.com/index.html
-Description: C Development Library for Amazon S3 Access
- This library provides an API for using Amazon's S3 service (see
- http://s3.amazonaws.com). Its design goals are:
- .
- - To provide a simple and straightforward API for accessing all of S3's
- functionality
- - To not require the developer using libs3 to need to know anything about:
- - HTTP
- - XML
- - SSL
- In other words, this API is meant to stand on its own, without requiring
- any implicit knowledge of how S3 services are accessed using HTTP
- protocols.
- - To be usable from multithreaded code
- - To be usable by code which wants to process multiple S3 requests
- simultaneously from a single thread
- - To be usable in the simple, straightforward way using sequentialized
- blocking requests
diff --git a/src/libs3/debian/postinst b/src/libs3/debian/postinst
deleted file mode 100755
index 2d1871b..0000000
--- a/src/libs3/debian/postinst
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-ldconfig
diff --git a/src/log/Log.cc b/src/log/Log.cc
index afeb120..0087057 100644
--- a/src/log/Log.cc
+++ b/src/log/Log.cc
@@ -13,6 +13,8 @@
#include "common/safe_io.h"
#include "common/Clock.h"
#include "include/assert.h"
+#include "include/compat.h"
+#include "include/on_exit.h"
#define DEFAULT_MAX_NEW 100
#define DEFAULT_MAX_RECENT 10000
@@ -22,7 +24,9 @@
namespace ceph {
namespace log {
-static void log_on_exit(int r, void *p)
+static OnExitManager exit_callbacks;
+
+static void log_on_exit(void *p)
{
Log *l = *(Log **)p;
if (l)
@@ -87,7 +91,7 @@ void Log::set_flush_on_exit()
// assume that exit() won't race with ~Log().
if (m_indirect_this == NULL) {
m_indirect_this = new (Log*)(this);
- on_exit(log_on_exit, m_indirect_this);
+ exit_callbacks.add_callback(log_on_exit, m_indirect_this);
}
}
diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h
index e40854a..39a4c55 100644
--- a/src/mds/CDentry.h
+++ b/src/mds/CDentry.h
@@ -97,7 +97,7 @@ public:
void add_waiter(uint64_t tag, Context *c);
- static const int EXPORT_NONCE = 1;
+ static const unsigned EXPORT_NONCE = 1;
bool is_lt(const MDSCacheObject *r) const {
return *this < *static_cast<const CDentry*>(r);
diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 2c985e4..3697929 100644
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -964,10 +964,10 @@ void CDir::merge(list<CDir*>& subs, list<Context*>& waiters, bool replay)
steal_dentry(dir->items.begin()->second);
// merge replica map
- for (map<int,int>::iterator p = dir->replica_map.begin();
+ for (map<int,unsigned>::iterator p = dir->replicas_begin();
p != dir->replica_map.end();
++p) {
- int cur = replica_map[p->first];
+ unsigned cur = replica_map[p->first];
if (p->second > cur)
replica_map[p->first] = p->second;
}
@@ -1145,11 +1145,14 @@ void CDir::take_sub_waiting(list<Context*>& ls)
waiting_on_dentry.clear();
put(PIN_DNWAITER);
}
- for (map<inodeno_t, list<Context*> >::iterator p = waiting_on_ino.begin();
- p != waiting_on_ino.end();
- ++p)
- ls.splice(ls.end(), p->second);
- waiting_on_ino.clear();
+ if (!waiting_on_ino.empty()) {
+ for (map<inodeno_t, list<Context*> >::iterator p = waiting_on_ino.begin();
+ p != waiting_on_ino.end();
+ ++p)
+ ls.splice(ls.end(), p->second);
+ waiting_on_ino.clear();
+ put(PIN_INOWAITER);
+ }
}
@@ -1336,20 +1339,6 @@ void CDir::last_put()
// -----------------------
// FETCH
-
-class C_Dir_Fetch : public Context {
- protected:
- CDir *dir;
- string want_dn;
- public:
- bufferlist bl;
-
- C_Dir_Fetch(CDir *d, const string& w) : dir(d), want_dn(w) { }
- void finish(int result) {
- dir->_fetched(bl, want_dn);
- }
-};
-
void CDir::fetch(Context *c, bool ignore_authpinnability)
{
string want;
@@ -1385,8 +1374,26 @@ void CDir::fetch(Context *c, const string& want_dn, bool ignore_authpinnability)
if (cache->mds->logger) cache->mds->logger->inc(l_mds_dir_f);
+ _omap_fetch(want_dn);
+}
+
+class C_Dir_TMAP_Fetched : public Context {
+ protected:
+ CDir *dir;
+ string want_dn;
+ public:
+ bufferlist bl;
+
+ C_Dir_TMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
+ void finish(int r) {
+ dir->_tmap_fetched(bl, want_dn, r);
+ }
+};
+
+void CDir::_tmap_fetch(const string& want_dn)
+{
// start by reading the first hunk of it
- C_Dir_Fetch *fin = new C_Dir_Fetch(this, want_dn);
+ C_Dir_TMAP_Fetched *fin = new C_Dir_TMAP_Fetched(this, want_dn);
object_t oid = get_ondisk_object();
object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
ObjectOperation rd;
@@ -1394,22 +1401,85 @@ void CDir::fetch(Context *c, const string& want_dn, bool ignore_authpinnability)
cache->mds->objecter->read(oid, oloc, rd, CEPH_NOSNAP, NULL, 0, fin);
}
-void CDir::_fetched(bufferlist &bl, const string& want_dn)
+void CDir::_tmap_fetched(bufferlist& bl, const string& want_dn, int r)
{
LogClient &clog = cache->mds->clog;
- dout(10) << "_fetched " << bl.length()
- << " bytes for " << *this
- << " want_dn=" << want_dn
- << dendl;
-
+ dout(10) << "_tmap_fetched " << bl.length() << " bytes for " << *this
+ << " want_dn=" << want_dn << dendl;
+
+ assert(r == 0 || r == -ENOENT);
assert(is_auth());
assert(!is_frozen());
- // empty?!?
+ bufferlist header;
+ map<string, bufferlist> omap;
+
if (bl.length() == 0) {
+ r = -ENODATA;
+ } else {
+ bufferlist::iterator p = bl.begin();
+ ::decode(header, p);
+ ::decode(omap, p);
+
+ if (!p.end()) {
+ clog.warn() << "tmap buffer of dir " << dirfrag() << " has "
+ << bl.length() - p.get_off() << " extra bytes\n";
+ }
+ bl.clear();
+ }
+
+ _omap_fetched(header, omap, want_dn, r);
+}
+
+class C_Dir_OMAP_Fetched : public Context {
+ protected:
+ CDir *dir;
+ string want_dn;
+ public:
+ bufferlist hdrbl;
+ map<string, bufferlist> omap;
+ int ret1, ret2;
+
+ C_Dir_OMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
+ void finish(int r) {
+ if (r >= 0) r = ret1;
+ if (r >= 0) r = ret2;
+ dir->_omap_fetched(hdrbl, omap, want_dn, r);
+ }
+};
+
+void CDir::_omap_fetch(const string& want_dn)
+{
+ C_Dir_OMAP_Fetched *fin = new C_Dir_OMAP_Fetched(this, want_dn);
+ object_t oid = get_ondisk_object();
+ object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
+ ObjectOperation rd;
+ rd.omap_get_header(&fin->hdrbl, &fin->ret1);
+ rd.omap_get_vals("", "", (uint64_t)-1, &fin->omap, &fin->ret2);
+ cache->mds->objecter->read(oid, oloc, rd, CEPH_NOSNAP, NULL, 0, fin);
+}
+
+void CDir::_omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
+ const string& want_dn, int r)
+{
+ LogClient &clog = cache->mds->clog;
+ dout(10) << "_fetched header " << hdrbl.length() << " bytes "
+ << omap.size() << " keys for " << *this
+ << " want_dn=" << want_dn << dendl;
+
+ assert(r == 0 || r == -ENOENT || r == -ENODATA);
+ assert(is_auth());
+ assert(!is_frozen());
+
+ if (hdrbl.length() == 0) {
+ if (r != -ENODATA) { // called by _tmap_fetched() ?
+ dout(10) << "_fetched 0 byte from omap, retry tmap" << dendl;
+ _tmap_fetch(want_dn);
+ return;
+ }
+
dout(0) << "_fetched missing object for " << *this << dendl;
- clog.error() << "dir " << ino() << "." << dirfrag()
- << " object missing on disk; some files may be lost\n";
+ clog.error() << "dir " << dirfrag() << " object missing on disk; some files may be lost\n";
log_mark_dirty();
@@ -1423,24 +1493,18 @@ void CDir::_fetched(bufferlist &bl, const string& want_dn)
return;
}
- // decode trivialmap.
- int len = bl.length();
- bufferlist::iterator p = bl.begin();
-
- bufferlist header;
- ::decode(header, p);
- bufferlist::iterator hp = header.begin();
fnode_t got_fnode;
- ::decode(got_fnode, hp);
-
- __u32 n;
- ::decode(n, p);
+ {
+ bufferlist::iterator p = hdrbl.begin();
+ ::decode(got_fnode, p);
+ if (!p.end()) {
+ clog.warn() << "header buffer of dir " << dirfrag() << " has "
+ << hdrbl.length() - p.get_off() << " extra bytes\n";
+ }
+ }
- dout(10) << "_fetched version " << got_fnode.version
- << ", " << len << " bytes, " << n << " keys"
- << dendl;
+ dout(10) << "_fetched version " << got_fnode.version << dendl;
-
// take the loaded fnode?
// only if we are a fresh CDir* with no prior state.
if (get_version() == 0) {
@@ -1457,8 +1521,9 @@ void CDir::_fetched(bufferlist &bl, const string& want_dn)
}
// purge stale snaps?
- // * only if we have past_parents open!
- const set<snapid_t> *snaps = 0;
+ // only if we have past_parents open!
+ bool purged_any = false;
+ const set<snapid_t> *snaps = NULL;
SnapRealm *realm = inode->find_snaprealm();
if (!realm->have_past_parents_open()) {
dout(10) << " no snap purge, one or more past parents NOT open" << dendl;
@@ -1469,30 +1534,26 @@ void CDir::_fetched(bufferlist &bl, const string& want_dn)
<< ", snap purge based on " << *snaps << dendl;
fnode.snap_purged_thru = realm->get_last_destroyed();
}
- bool purged_any = false;
bool stray = inode->is_stray();
- //int num_new_inodes_loaded = 0;
- loff_t baseoff = p.get_off();
- for (unsigned i=0; i<n; i++) {
- loff_t dn_offset = p.get_off() - baseoff;
-
+ unsigned pos = 0;
+ for (map<string, bufferlist>::iterator p = omap.begin();
+ p != omap.end();
+ ++p, ++pos) {
// dname
string dname;
snapid_t first, last;
- dentry_key_t::decode_helper(p, dname, last);
+ dentry_key_t::decode_helper(p->first, dname, last);
- bufferlist dndata;
- ::decode(dndata, p);
- bufferlist::iterator q = dndata.begin();
+ bufferlist::iterator q = p->second.begin();
::decode(first, q);
// marker
char type;
::decode(type, q);
- dout(24) << "_fetched pos " << dn_offset << " marker '" << type << "' dname '" << dname
+ dout(20) << "_fetched pos " << pos << " marker '" << type << "' dname '" << dname
<< " [" << first << "," << last << "]"
<< dendl;
@@ -1512,7 +1573,7 @@ void CDir::_fetched(bufferlist &bl, const string& want_dn)
* doesn't exist) but for which no explicit negative dentry is in
* the cache.
*/
- CDentry *dn = 0;
+ CDentry *dn = NULL;
if (!stale)
dn = lookup(dname, last);
@@ -1641,8 +1702,7 @@ void CDir::_fetched(bufferlist &bl, const string& want_dn)
}
}
} else {
- dout(1) << "corrupt directory, i got tag char '" << type << "' val " << (int)(type)
- << " at offset " << p.get_off() << dendl;
+ dout(1) << "corrupt directory, i got tag char '" << type << "' pos " << pos << dendl;
assert(0);
}
@@ -1677,10 +1737,6 @@ void CDir::_fetched(bufferlist &bl, const string& want_dn)
}
}
}
- if (!p.end()) {
- clog.warn() << "dir " << dirfrag() << " has "
- << bl.length() - p.get_off() << " extra bytes\n";
- }
//cache->mds->logger->inc("newin", num_new_inodes_loaded);
//hack_num_accessed = 0;
@@ -1733,125 +1789,64 @@ void CDir::commit(version_t want, Context *c, bool ignore_authpinnability)
_commit(want);
}
-
-class C_Dir_RetryCommit : public Context {
- CDir *dir;
- version_t want;
-public:
- C_Dir_RetryCommit(CDir *d, version_t v) :
- dir(d), want(v) { }
- void finish(int r) {
- dir->_commit(want);
- }
-};
-
class C_Dir_Committed : public Context {
CDir *dir;
version_t version;
public:
C_Dir_Committed(CDir *d, version_t v) : dir(d), version(v) { }
void finish(int r) {
+ assert(r == 0);
dir->_committed(version);
}
};
/**
- * Try and write out the full directory to disk.
- *
- * If the bufferlist we're using exceeds max_write_size, bail out
- * and switch to _commit_partial -- it can safely break itself into
- * multiple non-atomic writes.
+ * Flush out the modified dentries in this dir. Keep the bufferlist
+ * below max_write_size;
*/
-CDir::map_t::iterator CDir::_commit_full(ObjectOperation& m, const set<snapid_t> *snaps,
- unsigned max_write_size)
+void CDir::_omap_commit()
{
- dout(10) << "_commit_full" << dendl;
-
- // encode
- bufferlist bl;
- __u32 n = 0;
+ dout(10) << "_omap_commit" << dendl;
- bufferlist header;
- ::encode(fnode, header);
- max_write_size -= header.length();
-
- map_t::iterator p = items.begin();
- while (p != items.end() && bl.length() < max_write_size) {
- CDentry *dn = p->second;
- ++p;
-
- if (dn->linkage.is_null())
- continue; // skip negative entries
-
- if (snaps && dn->last != CEPH_NOSNAP &&
- try_trim_snap_dentry(dn, *snaps))
- continue;
-
- n++;
-
- _encode_dentry(dn, bl, snaps);
- }
+ unsigned max_write_size = cache->max_dir_commit_size;
+ unsigned write_size = 0;
- if (p != items.end()) {
- assert(bl.length() > max_write_size);
- return _commit_partial(m, snaps, max_write_size);
+ // snap purge?
+ const set<snapid_t> *snaps = NULL;
+ SnapRealm *realm = inode->find_snaprealm();
+ if (!realm->have_past_parents_open()) {
+ dout(10) << " no snap purge, one or more past parents NOT open" << dendl;
+ } else if (fnode.snap_purged_thru < realm->get_last_destroyed()) {
+ snaps = &realm->get_snaps();
+ dout(10) << " snap_purged_thru " << fnode.snap_purged_thru
+ << " < " << realm->get_last_destroyed()
+ << ", snap purge based on " << *snaps << dendl;
}
- // encode final trivialmap
- bufferlist finalbl;
- ::encode(header, finalbl);
- assert(num_head_items + num_head_null + num_snap_items + num_snap_null == items.size());
- assert(n == (num_head_items + num_snap_items));
- ::encode(n, finalbl);
- finalbl.claim_append(bl);
+ set<string> to_remove;
+ map<string, bufferlist> to_set;
- // write out the full blob
- m.tmap_put(finalbl);
- return p;
-}
+ C_GatherBuilder gather(g_ceph_context, new C_Dir_Committed(this, get_version()));
-/**
- * Flush out the modified dentries in this dir. Keep the bufferlist
- * below max_write_size; if we exceed that size then return the last
- * dentry that got committed into the bufferlist. (Note that the
- * bufferlist might be larger than requested by the size of that
- * last dentry as encoded.)
- *
- * If we're passed a last_committed_dn, skip to the next dentry after that.
- * Also, don't encode the header again -- we don't want to update it
- * on-disk until all the updates have made it through, so keep the header
- * in only the first changeset -- our caller is responsible for making sure
- * that changeset doesn't go through until after all the others do, if it's
- * necessary.
- */
-CDir::map_t::iterator CDir::_commit_partial(ObjectOperation& m,
- const set<snapid_t> *snaps,
- unsigned max_write_size,
- map_t::iterator last_committed_dn)
-{
- dout(10) << "_commit_partial" << dendl;
- bufferlist finalbl;
-
- // header
- if (last_committed_dn == map_t::iterator()) {
- bufferlist header;
- ::encode(fnode, header);
- finalbl.append(CEPH_OSD_TMAP_HDR);
- ::encode(header, finalbl);
- }
-
- // updated dentries
- map_t::iterator p = items.begin();
- if(last_committed_dn != map_t::iterator())
- p = last_committed_dn;
+ SnapContext snapc;
+ object_t oid = get_ondisk_object();
+ object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
- while (p != items.end() && finalbl.length() < max_write_size) {
+ for (map_t::iterator p = items.begin();
+ p != items.end(); ) {
CDentry *dn = p->second;
++p;
-
- if (snaps && dn->last != CEPH_NOSNAP &&
- try_trim_snap_dentry(dn, *snaps))
+
+ string key;
+ dn->key().encode(key);
+
+ if (dn->last != CEPH_NOSNAP &&
+ snaps && try_trim_snap_dentry(dn, *snaps)) {
+ dout(10) << " rm " << dn->name << " " << *dn << dendl;
+ write_size += key.length();
+ to_remove.insert(key);
continue;
+ }
if (!dn->is_dirty() &&
(!dn->state_test(CDentry::STATE_FRAGMENTING) || dn->get_linkage()->is_null()))
@@ -1859,18 +1854,60 @@ CDir::map_t::iterator CDir::_commit_partial(ObjectOperation& m,
if (dn->get_linkage()->is_null()) {
dout(10) << " rm " << dn->name << " " << *dn << dendl;
- finalbl.append(CEPH_OSD_TMAP_RMSLOPPY);
- dn->key().encode(finalbl);
+ write_size += key.length();
+ to_remove.insert(key);
} else {
dout(10) << " set " << dn->name << " " << *dn << dendl;
- finalbl.append(CEPH_OSD_TMAP_SET);
- _encode_dentry(dn, finalbl, snaps);
+ bufferlist dnbl;
+ _encode_dentry(dn, dnbl, snaps);
+ write_size += key.length() + dnbl.length();
+ to_set[key].swap(dnbl);
+ }
+
+ if (write_size >= max_write_size) {
+ ObjectOperation op;
+ op.priority = CEPH_MSG_PRIO_LOW; // set priority lower than journal!
+ op.tmap_to_omap(true); // convert tmap to omap
+
+ if (!to_set.empty())
+ op.omap_set(to_set);
+ if (!to_remove.empty())
+ op.omap_rm_keys(to_remove);
+
+ cache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+ 0, NULL, gather.new_sub());
+
+ write_size = 0;
+ to_set.clear();
+ to_remove.clear();
}
}
- // update the trivialmap at the osd
- m.tmap_update(finalbl);
- return p;
+ ObjectOperation op;
+ op.priority = CEPH_MSG_PRIO_LOW; // set priority lower than journal!
+ op.tmap_to_omap(true); // convert tmap to omap
+
+ /*
+ * save the header at the last moment.. If we were to send it off before other
+ * updates, but die before sending them all, we'd think that the on-disk state
+ * was fully committed even though it wasn't! However, since the messages are
+ * strictly ordered between the MDS and the OSD, and since messages to a given
+ * PG are strictly ordered, if we simply send the message containing the header
+ * off last, we cannot get our header into an incorrect state.
+ */
+ bufferlist header;
+ ::encode(fnode, header);
+ op.omap_set_header(header);
+
+ if (!to_set.empty())
+ op.omap_set(to_set);
+ if (!to_remove.empty())
+ op.omap_rm_keys(to_remove);
+
+ cache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+ 0, NULL, gather.new_sub());
+
+ gather.activate();
}
void CDir::_encode_dentry(CDentry *dn, bufferlist& bl,
@@ -1879,12 +1916,6 @@ void CDir::_encode_dentry(CDentry *dn, bufferlist& bl,
// clear dentry NEW flag, if any. we can no longer silently drop it.
dn->clear_new();
- dn->key().encode(bl);
-
- ceph_le32 plen = init_le32(0);
- unsigned plen_off = bl.length();
- ::encode(plen, bl);
-
::encode(dn->first, bl);
// primary or remote?
@@ -1924,15 +1955,8 @@ void CDir::_encode_dentry(CDentry *dn, bufferlist& bl,
in->purge_stale_snap_data(*snaps);
::encode(in->old_inodes, bl);
}
-
- plen = bl.length() - plen_off - sizeof(__u32);
-
- ceph_le32 eplen;
- eplen = plen;
- bl.copy_in(plen_off, sizeof(eplen), (char*)&eplen);
}
-
void CDir::_commit(version_t want)
{
dout(10) << "_commit want " << want << " on " << *this << dendl;
@@ -1962,14 +1986,6 @@ void CDir::_commit(version_t want)
return;
}
- // complete first? (only if we're not using TMAPUP osd op)
- if (!g_conf->mds_use_tmap && !is_complete()) {
- dout(7) << "commit not complete, fetching first" << dendl;
- if (cache->mds->logger) cache->mds->logger->inc(l_mds_dir_ffc);
- fetch(new C_Dir_RetryCommit(this, want));
- return;
- }
-
// commit.
committing_version = get_version();
@@ -1981,62 +1997,7 @@ void CDir::_commit(version_t want)
if (cache->mds->logger) cache->mds->logger->inc(l_mds_dir_c);
- // snap purge?
- SnapRealm *realm = inode->find_snaprealm();
- const set<snapid_t> *snaps = 0;
- if (!realm->have_past_parents_open()) {
- dout(10) << " no snap purge, one or more past parents NOT open" << dendl;
- } else if (fnode.snap_purged_thru < realm->get_last_destroyed()) {
- snaps = &realm->get_snaps();
- dout(10) << " snap_purged_thru " << fnode.snap_purged_thru
- << " < " << realm->get_last_destroyed()
- << ", snap purge based on " << *snaps << dendl;
- }
-
- ObjectOperation m;
- map_t::iterator committed_dn;
- unsigned max_write_size = cache->max_dir_commit_size;
-
- if (is_complete() &&
- ((num_dirty > (num_head_items*g_conf->mds_dir_commit_ratio)) ||
- state_test(CDir::STATE_FRAGMENTING))) {
- fnode.snap_purged_thru = realm->get_last_destroyed();
- committed_dn = _commit_full(m, snaps, max_write_size);
- } else {
- committed_dn = _commit_partial(m, snaps, max_write_size);
- }
-
- SnapContext snapc;
- object_t oid = get_ondisk_object();
- object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
-
- m.priority = CEPH_MSG_PRIO_LOW; // set priority lower than journal!
-
- if (committed_dn == items.end())
- cache->mds->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(g_ceph_context), 0, NULL,
- new C_Dir_Committed(this, get_version()));
- else { // send in a different Context
- C_GatherBuilder gather(g_ceph_context, new C_Dir_Committed(this, get_version()));
- while (committed_dn != items.end()) {
- ObjectOperation n = ObjectOperation();
- committed_dn = _commit_partial(n, snaps, max_write_size, committed_dn);
- cache->mds->objecter->mutate(oid, oloc, n, snapc, ceph_clock_now(g_ceph_context), 0, NULL,
- gather.new_sub());
- }
- /*
- * save the original object for last -- it contains the new header,
- * which will be committed on-disk. If we were to send it off before
- * the other commits, but die before sending them all, we'd think
- * that the on-disk state was fully committed even though it wasn't!
- * However, since the messages are strictly ordered between the MDS and
- * the OSD, and since messages to a given PG are strictly ordered, if
- * we simply send the message containing the header off last, we cannot
- * get our header into an incorrect state.
- */
- cache->mds->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(g_ceph_context), 0, NULL,
- gather.new_sub());
- gather.activate();
- }
+ _omap_commit();
}
diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index f131d83..db11c06 100644
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -140,7 +140,7 @@ public:
static const int REP_LIST = 2;
- static const int NONCE_EXPORT = 1;
+ static const unsigned EXPORT_NONCE = 1;
// -- wait masks --
@@ -278,6 +278,9 @@ protected:
friend class CDirDiscover;
friend class CDirExport;
+ friend class C_Dir_TMAP_Fetched;
+ friend class C_Dir_OMAP_Fetched;
+ friend class C_Dir_Committed;
bloom_filter *bloom;
/* If you set up the bloom filter, you must keep it accurate!
@@ -415,7 +418,7 @@ private:
// for giving to clients
void get_dist_spec(set<int>& ls, int auth) {
if (is_rep()) {
- for (map<int,int>::iterator p = replicas_begin();
+ for (map<int,unsigned>::iterator p = replicas_begin();
p != replicas_end();
++p)
ls.insert(p->first);
@@ -483,22 +486,23 @@ private:
}
void fetch(Context *c, bool ignore_authpinnability=false);
void fetch(Context *c, const string& want_dn, bool ignore_authpinnability=false);
- void _fetched(bufferlist &bl, const string& want_dn);
+protected:
+ void _omap_fetch(const string& want_dn);
+ void _omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
+ const string& want_dn, int r);
+ void _tmap_fetch(const string& want_dn);
+ void _tmap_fetched(bufferlist &bl, const string& want_dn, int r);
// -- commit --
map<version_t, list<Context*> > waiting_for_commit;
-
- void commit_to(version_t want);
- void commit(version_t want, Context *c, bool ignore_authpinnability=false);
void _commit(version_t want);
- map_t::iterator _commit_full(ObjectOperation& m, const set<snapid_t> *snaps,
- unsigned max_write_size=-1);
- map_t::iterator _commit_partial(ObjectOperation& m, const set<snapid_t> *snaps,
- unsigned max_write_size=-1,
- map_t::iterator last_committed_dn=map_t::iterator());
+ void _omap_commit();
void _encode_dentry(CDentry *dn, bufferlist& bl, const set<snapid_t> *snaps);
void _committed(version_t v);
+public:
void wait_for_commit(Context *c, version_t v=0);
+ void commit_to(version_t want);
+ void commit(version_t want, Context *c, bool ignore_authpinnability=false);
// -- dirtyness --
version_t get_committing_version() { return committing_version; }
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 1fc57fe..1933dea 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -873,6 +873,7 @@ struct C_Inode_Stored : public Context {
Context *fin;
C_Inode_Stored(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
void finish(int r) {
+ assert(r == 0);
in->_stored(version, fin);
}
};
@@ -1002,6 +1003,7 @@ struct C_Inode_StoredBacktrace : public Context {
Context *fin;
C_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
void finish(int r) {
+ assert(r == 0);
in->_stored_backtrace(version, fin);
}
};
@@ -1033,7 +1035,7 @@ void CInode::store_backtrace(Context *fin)
object_locator_t oloc(pool);
Context *fin2 = new C_Inode_StoredBacktrace(this, inode.backtrace_version, fin);
- if (!state_test(STATE_DIRTYPOOL)) {
+ if (!state_test(STATE_DIRTYPOOL) || inode.old_pools.empty()) {
mdcache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
0, NULL, fin2);
return;
@@ -1153,6 +1155,7 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
switch (type) {
case CEPH_LOCK_IAUTH:
+ ::encode(inode.version, bl);
::encode(inode.ctime, bl);
::encode(inode.mode, bl);
::encode(inode.uid, bl);
@@ -1160,13 +1163,16 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
break;
case CEPH_LOCK_ILINK:
+ ::encode(inode.version, bl);
::encode(inode.ctime, bl);
::encode(inode.nlink, bl);
::encode(inode.anchored, bl);
break;
case CEPH_LOCK_IDFT:
- if (!is_auth()) {
+ if (is_auth()) {
+ ::encode(inode.version, bl);
+ } else {
bool dirty = dirfragtreelock.is_dirty();
::encode(dirty, bl);
}
@@ -1189,6 +1195,7 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
case CEPH_LOCK_IFILE:
if (is_auth()) {
+ ::encode(inode.version, bl);
::encode(inode.mtime, bl);
::encode(inode.atime, bl);
::encode(inode.time_warp_seq, bl);
@@ -1231,7 +1238,9 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
break;
case CEPH_LOCK_INEST:
- if (!is_auth()) {
+ if (is_auth()) {
+ ::encode(inode.version, bl);
+ } else {
bool dirty = nestlock.is_dirty();
::encode(dirty, bl);
}
@@ -1266,20 +1275,24 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
break;
case CEPH_LOCK_IXATTR:
+ ::encode(inode.version, bl);
::encode(xattrs, bl);
break;
case CEPH_LOCK_ISNAP:
+ ::encode(inode.version, bl);
encode_snap(bl);
break;
case CEPH_LOCK_IFLOCK:
+ ::encode(inode.version, bl);
::encode(fcntl_locks, bl);
::encode(flock_locks, bl);
break;
case CEPH_LOCK_IPOLICY:
if (inode.is_dir()) {
+ ::encode(inode.version, bl);
::encode(inode.layout, bl);
}
break;
@@ -1312,6 +1325,7 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
switch (type) {
case CEPH_LOCK_IAUTH:
+ ::decode(inode.version, p);
::decode(tm, p);
if (inode.ctime < tm) inode.ctime = tm;
::decode(inode.mode, p);
@@ -1320,6 +1334,7 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
break;
case CEPH_LOCK_ILINK:
+ ::decode(inode.version, p);
::decode(tm, p);
if (inode.ctime < tm) inode.ctime = tm;
::decode(inode.nlink, p);
@@ -1339,6 +1354,8 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
dout(10) << "decode_lock_state setting dftlock dirty flag" << dendl;
dirfragtreelock.mark_dirty(); // ok bc we're auth and caller will handle
}
+ } else {
+ ::decode(inode.version, p);
}
{
fragtree_t temp;
@@ -1374,6 +1391,7 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
case CEPH_LOCK_IFILE:
if (!is_auth()) {
+ ::decode(inode.version, p);
::decode(inode.mtime, p);
::decode(inode.atime, p);
::decode(inode.time_warp_seq, p);
@@ -1456,6 +1474,8 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
dout(10) << "decode_lock_state setting nestlock dirty flag" << dendl;
nestlock.mark_dirty(); // ok bc we're auth and caller will handle
}
+ } else {
+ ::decode(inode.version, p);
}
{
nest_info_t rstat;
@@ -1518,11 +1538,13 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
break;
case CEPH_LOCK_IXATTR:
+ ::decode(inode.version, p);
::decode(xattrs, p);
break;
case CEPH_LOCK_ISNAP:
{
+ ::decode(inode.version, p);
snapid_t seq = 0;
if (snaprealm)
seq = snaprealm->srnode.seq;
@@ -1533,12 +1555,14 @@ void CInode::decode_lock_state(int type, bufferlist& bl)
break;
case CEPH_LOCK_IFLOCK:
+ ::decode(inode.version, p);
::decode(fcntl_locks, p);
::decode(flock_locks, p);
break;
case CEPH_LOCK_IPOLICY:
if (inode.is_dir()) {
+ ::decode(inode.version, p);
::decode(inode.layout, p);
}
break;
@@ -2462,8 +2486,10 @@ Capability *CInode::add_client_cap(client_t client, Session *session, SnapRealm
Capability *cap = new Capability(this, ++mdcache->last_cap_id, client);
assert(client_caps.count(client) == 0);
client_caps[client] = cap;
- if (session)
- session->add_cap(cap);
+
+ session->add_cap(cap);
+ if (session->is_stale())
+ cap->mark_stale();
cap->client_follows = first-1;
@@ -2531,8 +2557,8 @@ Capability *CInode::reconnect_cap(client_t client, ceph_mds_cap_reconnect& icr,
cap->set_wanted(icr.wanted);
cap->issue_norevoke(icr.issued);
cap->reset_seq();
+ cap->set_cap_id(icr.cap_id);
}
- cap->set_cap_id(icr.cap_id);
cap->set_last_issue_stamp(ceph_clock_now(g_ceph_context));
return cap;
}
@@ -2618,6 +2644,9 @@ int CInode::get_caps_allowed_for_client(client_t client)
} else {
allowed = get_caps_allowed_by_type(CAP_ANY);
}
+ if (inode.inline_version != CEPH_INLINE_NONE &&
+ !mdcache->mds->get_session(client)->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA))
+ allowed &= ~(CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR);
return allowed;
}
@@ -2822,6 +2851,16 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
e.files = i->dirstat.nfiles;
e.subdirs = i->dirstat.nsubdirs;
+ // inline data
+ version_t inline_version = 0;
+ bufferlist inline_data;
+ if (!cap || (cap->client_inline_version < i->inline_version)) {
+ inline_version = i->inline_version;
+ inline_data = i->inline_data;
+ if (cap)
+ cap->client_inline_version = i->inline_version;
+ }
+
// nest (do same as file... :/)
i->rstat.rctime.encode_timeval(&e.rctime);
e.rbytes = i->rstat.rbytes;
@@ -2860,6 +2899,7 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
bytes += (sizeof(__u32) + sizeof(__u32)) * dirfragtree._splits.size();
bytes += sizeof(__u32) + symlink.length();
bytes += sizeof(__u32) + xbl.length();
+ bytes += sizeof(version_t) + sizeof(__u32) + inline_data.length();
if (bytes > max_bytes)
return -ENOSPC;
}
@@ -2955,6 +2995,10 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
::encode(i->dir_layout, bl);
}
::encode(xbl, bl);
+ if (session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA)) {
+ ::encode(inline_version, bl);
+ ::encode(inline_data, bl);
+ }
return valid;
}
@@ -2987,6 +3031,13 @@ void CInode::encode_cap_message(MClientCaps *m, Capability *cap)
i->atime.encode_timeval(&m->head.atime);
m->head.time_warp_seq = i->time_warp_seq;
+ if (cap->client_inline_version < i->inline_version) {
+ m->inline_version = cap->client_inline_version = i->inline_version;
+ m->inline_data = i->inline_data;
+ } else {
+ m->inline_version = 0;
+ }
+
// max_size is min of projected, actual.
uint64_t oldms = oi->client_ranges.count(client) ? oi->client_ranges[client].range.last : 0;
uint64_t newms = pi->client_ranges.count(client) ? pi->client_ranges[client].range.last : 0;
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 1c2a933..3762e9d 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -175,7 +175,7 @@ public:
static const uint64_t WAIT_ANY_MASK = (uint64_t)(-1);
// misc
- static const int EXPORT_NONCE = 1; // nonce given to replicas created by export
+ static const unsigned EXPORT_NONCE = 1; // nonce given to replicas created by export
ostream& print_db_line_prefix(ostream& out);
diff --git a/src/mds/Capability.cc b/src/mds/Capability.cc
index f139430..6a68fd7 100644
--- a/src/mds/Capability.cc
+++ b/src/mds/Capability.cc
@@ -24,10 +24,12 @@
void Capability::Export::encode(bufferlist &bl) const
{
ENCODE_START(2, 2, bl);
+ ::encode(cap_id, bl);
::encode(wanted, bl);
::encode(issued, bl);
::encode(pending, bl);
::encode(client_follows, bl);
+ ::encode(seq, bl);
::encode(mseq, bl);
::encode(last_issue_stamp, bl);
ENCODE_FINISH(bl);
@@ -36,10 +38,12 @@ void Capability::Export::encode(bufferlist &bl) const
void Capability::Export::decode(bufferlist::iterator &p)
{
DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, p);
+ ::decode(cap_id, p);
::decode(wanted, p);
::decode(issued, p);
::decode(pending, p);
::decode(client_follows, p);
+ ::decode(seq, p);
::decode(mseq, p);
::decode(last_issue_stamp, p);
DECODE_FINISH(p);
@@ -47,10 +51,12 @@ void Capability::Export::decode(bufferlist::iterator &p)
void Capability::Export::dump(Formatter *f) const
{
+ f->dump_unsigned("cap_id", cap_id);
f->dump_unsigned("wanted", wanted);
f->dump_unsigned("issued", issued);
f->dump_unsigned("pending", pending);
f->dump_unsigned("client_follows", client_follows);
+ f->dump_unsigned("seq", seq);
f->dump_unsigned("migrate_seq", mseq);
f->dump_stream("last_issue_stamp") << last_issue_stamp;
}
@@ -67,6 +73,30 @@ void Capability::Export::generate_test_instances(list<Capability::Export*>& ls)
ls.back()->last_issue_stamp = utime_t(6, 7);
}
+void Capability::Import::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(cap_id, bl);
+ ::encode(issue_seq, bl);
+ ::encode(mseq, bl);
+ ENCODE_FINISH(bl);
+}
+
+void Capability::Import::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(cap_id, bl);
+ ::decode(issue_seq, bl);
+ ::decode(mseq, bl);
+ DECODE_FINISH(bl);
+}
+
+void Capability::Import::dump(Formatter *f) const
+{
+ f->dump_unsigned("cap_id", cap_id);
+ f->dump_unsigned("issue_seq", issue_seq);
+ f->dump_unsigned("migrate_seq", mseq);
+}
/*
* Capability::revoke_info
diff --git a/src/mds/Capability.h b/src/mds/Capability.h
index fb6b3dc..7e04de0 100644
--- a/src/mds/Capability.h
+++ b/src/mds/Capability.h
@@ -78,20 +78,33 @@ public:
}
public:
struct Export {
+ int64_t cap_id;
int32_t wanted;
int32_t issued;
int32_t pending;
snapid_t client_follows;
+ ceph_seq_t seq;
ceph_seq_t mseq;
utime_t last_issue_stamp;
Export() {}
- Export(int w, int i, int p, snapid_t cf, ceph_seq_t s, utime_t lis) :
- wanted(w), issued(i), pending(p), client_follows(cf), mseq(s), last_issue_stamp(lis) {}
+ Export(int64_t id, int w, int i, int p, snapid_t cf, ceph_seq_t s, ceph_seq_t m, utime_t lis) :
+ cap_id(id), wanted(w), issued(i), pending(p), client_follows(cf),
+ seq(s), mseq(m), last_issue_stamp(lis) {}
void encode(bufferlist &bl) const;
void decode(bufferlist::iterator &p);
void dump(Formatter *f) const;
static void generate_test_instances(list<Export*>& ls);
};
+ struct Import {
+ int64_t cap_id;
+ ceph_seq_t issue_seq;
+ ceph_seq_t mseq;
+ Import() {}
+ Import(int64_t i, ceph_seq_t s, ceph_seq_t m) : cap_id(i), issue_seq(s), mseq(m) {}
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &p);
+ void dump(Formatter *f) const;
+ };
private:
CInode *inode;
@@ -204,11 +217,15 @@ private:
ceph_seq_t mseq;
int suppress;
- bool stale;
+ unsigned state;
+
+ const static unsigned STATE_STALE = (1<<0);
+ const static unsigned STATE_NEW = (1<<1);
public:
snapid_t client_follows;
version_t client_xattr_version;
+ version_t client_inline_version;
xlist<Capability*>::item item_session_caps;
xlist<Capability*>::item item_snaprealm_caps;
@@ -221,8 +238,9 @@ public:
last_sent(0),
last_issue(0),
mseq(0),
- suppress(0), stale(false),
+ suppress(0), state(0),
client_follows(0), client_xattr_version(0),
+ client_inline_version(0),
item_session_caps(this), item_snaprealm_caps(this) {
g_num_cap++;
g_num_capa++;
@@ -231,8 +249,12 @@ public:
g_num_cap--;
g_num_caps++;
}
+
+ Capability(const Capability& other); // no copying
+ const Capability& operator=(const Capability& other); // no copying
ceph_seq_t get_mseq() { return mseq; }
+ void inc_mseq() { mseq++; }
ceph_seq_t get_last_sent() { return last_sent; }
utime_t get_last_issue_stamp() { return last_issue_stamp; }
@@ -249,8 +271,12 @@ public:
void inc_suppress() { suppress++; }
void dec_suppress() { suppress--; }
- bool is_stale() { return stale; }
- void set_stale(bool b) { stale = b; }
+ bool is_stale() { return state & STATE_STALE; }
+ void mark_stale() { state |= STATE_STALE; }
+ void clear_stale() { state &= ~STATE_STALE; }
+ bool is_new() { return state & STATE_NEW; }
+ void mark_new() { state |= STATE_NEW; }
+ void clear_new() { state &= ~STATE_NEW; }
CInode *get_inode() { return inode; }
client_t get_client() { return client; }
@@ -262,6 +288,7 @@ public:
//check_rdcaps_list();
}
+ void inc_last_seq() { last_sent++; };
ceph_seq_t get_last_seq() { return last_sent; }
ceph_seq_t get_last_issue() { return last_issue; }
@@ -272,17 +299,20 @@ public:
// -- exports --
Export make_export() {
- return Export(_wanted, issued(), pending(), client_follows, mseq+1, last_issue_stamp);
+ return Export(cap_id, _wanted, issued(), pending(), client_follows, last_sent, mseq+1, last_issue_stamp);
}
- void rejoin_import() { mseq++; }
void merge(Export& other, bool auth_cap) {
- // issued + pending
- int newpending = other.pending | pending();
- if (other.issued & ~newpending)
- issue(other.issued | newpending);
- else
- issue(newpending);
- last_issue_stamp = other.last_issue_stamp;
+ if (!is_stale()) {
+ // issued + pending
+ int newpending = other.pending | pending();
+ if (other.issued & ~newpending)
+ issue(other.issued | newpending);
+ else
+ issue(newpending);
+ last_issue_stamp = other.last_issue_stamp;
+ } else {
+ inc_last_seq();
+ }
client_follows = other.client_follows;
@@ -292,21 +322,25 @@ public:
mseq = other.mseq;
}
void merge(int otherwanted, int otherissued) {
- // issued + pending
- int newpending = pending();
- if (otherissued & ~newpending)
- issue(otherissued | newpending);
- else
- issue(newpending);
+ if (!is_stale()) {
+ // issued + pending
+ int newpending = pending();
+ if (otherissued & ~newpending)
+ issue(otherissued | newpending);
+ else
+ issue(newpending);
+ } else {
+ inc_last_seq();
+ }
// wanted
_wanted = _wanted | otherwanted;
}
void revoke() {
- if (pending())
- issue(0);
- confirm_receipt(last_sent, 0);
+ if (pending() & ~CEPH_CAP_PIN)
+ issue(CEPH_CAP_PIN);
+ confirm_receipt(last_sent, CEPH_CAP_PIN);
}
// serializers
@@ -318,6 +352,7 @@ public:
};
WRITE_CLASS_ENCODER(Capability::Export)
+WRITE_CLASS_ENCODER(Capability::Import)
WRITE_CLASS_ENCODER(Capability::revoke_info)
WRITE_CLASS_ENCODER(Capability)
diff --git a/src/mds/Dumper.cc b/src/mds/Dumper.cc
index 9b83b4d..fc9b481 100644
--- a/src/mds/Dumper.cc
+++ b/src/mds/Dumper.cc
@@ -49,7 +49,7 @@ bool Dumper::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer,
void Dumper::init(int rank)
{
inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
- unsigned pg_pool = CEPH_METADATA_RULE;
+ unsigned pg_pool = MDS_METADATA_POOL;
osdmap = new OSDMap();
objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer);
journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC,
@@ -194,7 +194,7 @@ void Dumper::undump(const char *dump_file)
cout << "start " << start << " len " << len << std::endl;
inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
- unsigned pg_pool = CEPH_METADATA_RULE;
+ unsigned pg_pool = MDS_METADATA_POOL;
Journaler::Header h;
h.trimmed_pos = start;
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 7f85251..4f399e3 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -105,7 +105,7 @@ void Locker::dispatch(Message *m)
void Locker::send_lock_message(SimpleLock *lock, int msg)
{
- for (map<int,int>::iterator it = lock->get_parent()->replicas_begin();
+ for (map<int,unsigned>::iterator it = lock->get_parent()->replicas_begin();
it != lock->get_parent()->replicas_end();
++it) {
if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN)
@@ -117,7 +117,7 @@ void Locker::send_lock_message(SimpleLock *lock, int msg)
void Locker::send_lock_message(SimpleLock *lock, int msg, const bufferlist &data)
{
- for (map<int,int>::iterator it = lock->get_parent()->replicas_begin();
+ for (map<int,unsigned>::iterator it = lock->get_parent()->replicas_begin();
it != lock->get_parent()->replicas_end();
++it) {
if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN)
@@ -823,7 +823,7 @@ void Locker::eval_gather(SimpleLock *lock, bool first, bool *pneed_issue, list<C
bool Locker::eval(CInode *in, int mask, bool caps_imported)
{
- bool need_issue = false;
+ bool need_issue = caps_imported;
list<Context*> finishers;
dout(10) << "eval " << mask << " " << *in << dendl;
@@ -976,8 +976,8 @@ void Locker::try_eval(SimpleLock *lock, bool *pneed_issue)
}
}
- if (p->is_freezing()) {
- dout(7) << "try_eval " << *lock << " frozen, waiting on " << *p << dendl;
+ if (lock->get_type() != CEPH_LOCK_DN && p->is_freezing()) {
+ dout(7) << "try_eval " << *lock << " freezing, waiting on " << *p << dendl;
p->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_Locker_Eval(this, p, lock->get_type()));
return;
}
@@ -1606,8 +1606,6 @@ void Locker::file_update_finish(CInode *in, Mutation *mut, bool share, client_t
set<CInode*> need_issue;
drop_locks(mut, &need_issue);
- mut->cleanup();
- delete mut;
if (!in->is_head() && !in->client_snap_caps.empty()) {
dout(10) << " client_snap_caps " << in->client_snap_caps << dendl;
@@ -1640,6 +1638,10 @@ void Locker::file_update_finish(CInode *in, Mutation *mut, bool share, client_t
share_inode_max_size(in);
}
issue_caps_set(need_issue);
+
+ // auth unpin after issuing caps
+ mut->cleanup();
+ delete mut;
}
Capability* Locker::issue_new_caps(CInode *in,
@@ -1759,6 +1761,10 @@ bool Locker::issue_caps(CInode *in, Capability *only_cap)
// add in any xlocker-only caps (for locks this client is the xlocker for)
allowed |= xlocker_allowed & in->get_xlocker_mask(it->first);
+ if (in->inode.inline_version != CEPH_INLINE_NONE &&
+ !mds->get_session(it->first)->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA))
+ allowed &= ~(CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR);
+
int pending = cap->pending();
int wanted = cap->wanted();
@@ -1850,10 +1856,10 @@ void Locker::revoke_stale_caps(Session *session)
for (xlist<Capability*>::iterator p = session->caps.begin(); !p.end(); ++p) {
Capability *cap = *p;
- cap->set_stale(true);
+ cap->mark_stale();
CInode *in = cap->get_inode();
int issued = cap->issued();
- if (issued) {
+ if (issued & ~CEPH_CAP_PIN) {
dout(10) << " revoking " << ccap_string(issued) << " on " << *in << dendl;
cap->revoke();
@@ -1887,7 +1893,7 @@ void Locker::resume_stale_caps(Session *session)
assert(in->is_head());
if (cap->is_stale()) {
dout(10) << " clearing stale flag on " << *in << dendl;
- cap->set_stale(false);
+ cap->clear_stale();
if (!in->is_auth() || !eval(in, CEPH_CAP_LOCKS))
issue_caps(in, cap);
}
@@ -1926,7 +1932,7 @@ void Locker::request_inode_file_caps(CInode *in)
{
assert(!in->is_auth());
- int wanted = in->get_caps_wanted();
+ int wanted = in->get_caps_wanted() & ~CEPH_CAP_PIN;
if (wanted != in->replica_caps_wanted) {
// wait for single auth
if (in->is_ambiguous_auth()) {
@@ -2174,6 +2180,7 @@ void Locker::share_inode_max_size(CInode *in, Capability *only_cap)
continue;
if (cap->pending() & (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_BUFFER)) {
dout(10) << "share_inode_max_size with client." << client << dendl;
+ cap->inc_last_seq();
MClientCaps *m = new MClientCaps(CEPH_CAP_OP_GRANT,
in->ino(),
in->find_snaprealm()->inode->ino(),
@@ -2518,13 +2525,16 @@ void Locker::process_request_cap_release(MDRequest *mdr, client_t client, const
if (dn) {
ClientLease *l = dn->get_client_lease(client);
if (l) {
- dout(10) << " removing lease on " << *dn << dendl;
+ dout(10) << "process_cap_release removing lease on " << *dn << dendl;
dn->remove_client_lease(l, this);
+ } else {
+ dout(7) << "process_cap_release client." << client
+ << " doesn't have lease on " << *dn << dendl;
}
} else {
- mds->clog.warn() << "client." << client << " released lease on dn "
- << dir->dirfrag() << "/" << dname << " which dne\n";
- }
+ dout(7) << "process_cap_release client." << client << " released lease on dn "
+ << dir->dirfrag() << "/" << dname << " which dne" << dendl;
+ }
}
}
@@ -2532,7 +2542,7 @@ void Locker::process_request_cap_release(MDRequest *mdr, client_t client, const
if (!cap)
return;
- dout(10) << "process_cap_update client." << client << " " << ccap_string(caps) << " on " << *in
+ dout(10) << "process_cap_release client." << client << " " << ccap_string(caps) << " on " << *in
<< (mdr ? "" : " (DEFERRED, no mdr)")
<< dendl;
@@ -2570,6 +2580,38 @@ void Locker::process_request_cap_release(MDRequest *mdr, client_t client, const
mdr->cap_releases[in->vino()] = cap->get_last_seq();
}
+class C_Locker_RetryKickIssueCaps : public Context {
+ Locker *locker;
+ CInode *in;
+ client_t client;
+ ceph_seq_t seq;
+public:
+ C_Locker_RetryKickIssueCaps(Locker *l, CInode *i, client_t c, ceph_seq_t s) :
+ locker(l), in(i), client(c), seq(s) {
+ in->get(CInode::PIN_PTRWAITER);
+ }
+ void finish(int r) {
+ locker->kick_issue_caps(in, client, seq);
+ in->put(CInode::PIN_PTRWAITER);
+ }
+};
+
+void Locker::kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq)
+{
+ Capability *cap = in->get_client_cap(client);
+ if (!cap || cap->get_last_sent() != seq)
+ return;
+ if (in->is_frozen()) {
+ dout(10) << "kick_issue_caps waiting for unfreeze on " << *in << dendl;
+ in->add_waiter(CInode::WAIT_UNFREEZE,
+ new C_Locker_RetryKickIssueCaps(this, in, client, seq));
+ return;
+ }
+ dout(10) << "kick_issue_caps released at current seq " << seq
+ << ", reissuing" << dendl;
+ issue_caps(in, cap);
+}
+
void Locker::kick_cap_releases(MDRequest *mdr)
{
client_t client = mdr->get_client();
@@ -2579,18 +2621,10 @@ void Locker::kick_cap_releases(MDRequest *mdr)
CInode *in = mdcache->get_inode(p->first);
if (!in)
continue;
- Capability *cap = in->get_client_cap(client);
- if (!cap)
- continue;
- if (cap->get_last_sent() == p->second) {
- dout(10) << "kick_cap_releases released at current seq " << p->second
- << ", reissuing" << dendl;
- issue_caps(in, cap);
- }
+ kick_issue_caps(in, client, p->second);
}
}
-
static uint64_t calc_bounding(uint64_t t)
{
t |= t >> 1;
@@ -2688,6 +2722,7 @@ void Locker::_update_cap_fields(CInode *in, int dirty, MClientCaps *m, inode_t *
utime_t mtime = m->get_mtime();
utime_t ctime = m->get_ctime();
uint64_t size = m->get_size();
+ version_t inline_version = m->inline_version;
if (((dirty & CEPH_CAP_FILE_WR) && mtime > pi->mtime) ||
((dirty & CEPH_CAP_FILE_EXCL) && mtime != pi->mtime)) {
@@ -2707,6 +2742,12 @@ void Locker::_update_cap_fields(CInode *in, int dirty, MClientCaps *m, inode_t *
pi->size = size;
pi->rstat.rbytes = size;
}
+ if (in->inode.is_file() &&
+ (dirty & CEPH_CAP_FILE_WR) &&
+ inline_version > pi->inline_version) {
+ pi->inline_version = inline_version;
+ pi->inline_data = m->inline_data;
+ }
if ((dirty & CEPH_CAP_FILE_EXCL) && atime != pi->atime) {
dout(7) << " atime " << pi->atime << " -> " << atime
<< " for " << *in << dendl;
@@ -2900,6 +2941,10 @@ bool Locker::_do_cap_update(CInode *in, Capability *cap,
wrlock_force(&in->xattrlock, mut);
}
+
+ // update backtrace for old format inode. (see inode_t::decode)
+ if (pi->backtrace_version == 0)
+ pi->update_backtrace();
mut->auth_pin(in);
mdcache->predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY, 0, follows);
@@ -3396,8 +3441,13 @@ void Locker::simple_eval(SimpleLock *lock, bool *need_issue)
assert(lock->get_parent()->is_auth());
assert(lock->is_stable());
- if (lock->get_parent()->is_freezing_or_frozen())
- return;
+ if (lock->get_parent()->is_freezing_or_frozen()) {
+ // dentry lock in unreadable state can block path traverse
+ if ((lock->get_type() != CEPH_LOCK_DN ||
+ lock->get_state() == LOCK_SYNC ||
+ lock->get_parent()->is_frozen()))
+ return;
+ }
CInode *in = 0;
int wanted = 0;
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 0c9931a..466f7c9 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -213,6 +213,7 @@ public:
const string &dname);
void kick_cap_releases(MDRequest *mdr);
+ void kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq);
void remove_client_cap(CInode *in, client_t client);
diff --git a/src/mds/LogSegment.h b/src/mds/LogSegment.h
index 624c3bc..c9fdfd7 100644
--- a/src/mds/LogSegment.h
+++ b/src/mds/LogSegment.h
@@ -24,8 +24,8 @@
#include "CDentry.h"
#include "CDir.h"
-#include <ext/hash_set>
-using __gnu_cxx::hash_set;
+#include "include/unordered_set.h"
+using ceph::unordered_set;
class CDir;
class CInode;
@@ -54,7 +54,7 @@ class LogSegment {
set<CInode*> truncating_inodes;
- map<int, hash_set<version_t> > pending_commit_tids; // mdstable
+ map<int, ceph::unordered_set<version_t> > pending_commit_tids; // mdstable
set<metareqid_t> uncommitted_masters;
set<dirfrag_t> uncommitted_fragments;
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index ae59c26..eb20c06 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -2503,7 +2503,7 @@ void MDCache::send_slave_resolves()
} else {
set<int> resolve_set;
mds->mdsmap->get_mds_set(resolve_set, MDSMap::STATE_RESOLVE);
- for (hash_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
+ for (ceph::unordered_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
p != active_requests.end();
++p) {
if (!p->second->is_slave() || !p->second->slave_did_prepare())
@@ -2513,7 +2513,18 @@ void MDCache::send_slave_resolves()
dout(10) << " including uncommitted " << *p->second << dendl;
if (!resolves.count(master))
resolves[master] = new MMDSResolve;
- resolves[master]->add_slave_request(p->first);
+ if (p->second->has_more() && p->second->more()->is_inode_exporter) {
+ // re-send cap exports
+ CInode *in = p->second->more()->rename_inode;
+ map<client_t, Capability::Export> cap_map;
+ in->export_client_caps(cap_map);
+ bufferlist bl;
+ ::encode(in->ino(), bl);
+ ::encode(cap_map, bl);
+ resolves[master]->add_slave_request(p->first, bl);
+ } else {
+ resolves[master]->add_slave_request(p->first);
+ }
}
}
}
@@ -2646,7 +2657,7 @@ void MDCache::handle_mds_failure(int who)
// clean up any requests slave to/from this node
list<MDRequest*> finish;
- for (hash_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
+ for (ceph::unordered_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
p != active_requests.end();
++p) {
// slave to the failed node?
@@ -2677,7 +2688,7 @@ void MDCache::handle_mds_failure(int who)
dout(10) << " slave request " << *p->second << " no longer need rename notity ack from mds."
<< who << dendl;
p->second->more()->waiting_on_slave.erase(who);
- if (p->second->more()->waiting_on_slave.empty())
+ if (p->second->more()->waiting_on_slave.empty() && p->second->slave_request)
mds->queue_waiter(new C_MDS_RetryRequest(this, p->second));
}
@@ -2854,11 +2865,11 @@ void MDCache::handle_resolve(MMDSResolve *m)
// ambiguous slave requests?
if (!m->slave_requests.empty()) {
- for (vector<metareqid_t>::iterator p = m->slave_requests.begin();
+ for (map<metareqid_t, bufferlist>::iterator p = m->slave_requests.begin();
p != m->slave_requests.end();
++p) {
- if (uncommitted_masters.count(*p) && !uncommitted_masters[*p].safe)
- pending_masters.insert(*p);
+ if (uncommitted_masters.count(p->first) && !uncommitted_masters[p->first].safe)
+ pending_masters.insert(p->first);
}
if (!pending_masters.empty()) {
@@ -2868,18 +2879,47 @@ void MDCache::handle_resolve(MMDSResolve *m)
}
MMDSResolveAck *ack = new MMDSResolveAck;
- for (vector<metareqid_t>::iterator p = m->slave_requests.begin();
+ for (map<metareqid_t, bufferlist>::iterator p = m->slave_requests.begin();
p != m->slave_requests.end();
++p) {
- if (uncommitted_masters.count(*p)) { //mds->sessionmap.have_completed_request(*p)) {
+ if (uncommitted_masters.count(p->first)) { //mds->sessionmap.have_completed_request(p->first)) {
// COMMIT
dout(10) << " ambiguous slave request " << *p << " will COMMIT" << dendl;
- ack->add_commit(*p);
- uncommitted_masters[*p].slaves.insert(from); // wait for slave OP_COMMITTED before we log ECommitted
+ ack->add_commit(p->first);
+ uncommitted_masters[p->first].slaves.insert(from); // wait for slave OP_COMMITTED before we log ECommitted
+
+ if (p->second.length() > 0) {
+ // slave wants to export caps (rename)
+ assert(mds->is_resolve());
+
+ inodeno_t ino;
+ map<client_t,Capability::Export> cap_exports;
+ bufferlist::iterator q = p->second.begin();
+ ::decode(ino, q);
+ ::decode(cap_exports, q);
+
+ assert(get_inode(ino));
+
+ for (map<client_t,Capability::Export>::iterator q = cap_exports.begin();
+ q != cap_exports.end();
+ ++q) {
+ Capability::Import& im = rejoin_imported_caps[from][ino][q->first];
+ im.cap_id = ++last_cap_id; // assign a new cap ID
+ im.issue_seq = 1;
+ im.mseq = q->second.mseq;
+ }
+
+ // will process these caps in rejoin stage
+ rejoin_slave_exports[ino].first = from;
+ rejoin_slave_exports[ino].second.swap(cap_exports);
+
+ // send information of imported caps back to slave
+ ::encode(rejoin_imported_caps[from][ino], ack->commit[p->first]);
+ }
} else {
// ABORT
dout(10) << " ambiguous slave request " << *p << " will ABORT" << dendl;
- ack->add_abort(*p);
+ ack->add_abort(p->first);
}
}
mds->send_message(ack, m->get_connection());
@@ -3037,30 +3077,34 @@ void MDCache::handle_resolve_ack(MMDSResolveAck *ack)
assert(mds->is_clientreplay() || mds->is_active() || mds->is_stopping());
}
- for (vector<metareqid_t>::iterator p = ack->commit.begin();
+ for (map<metareqid_t, bufferlist>::iterator p = ack->commit.begin();
p != ack->commit.end();
++p) {
- dout(10) << " commit on slave " << *p << dendl;
+ dout(10) << " commit on slave " << p->first << dendl;
if (ambiguous_slave_updates.count(from)) {
- remove_ambiguous_slave_update(*p, from);
+ remove_ambiguous_slave_update(p->first, from);
continue;
}
if (mds->is_resolve()) {
// replay
- MDSlaveUpdate *su = get_uncommitted_slave_update(*p, from);
+ MDSlaveUpdate *su = get_uncommitted_slave_update(p->first, from);
assert(su);
// log commit
- mds->mdlog->start_submit_entry(new ESlaveUpdate(mds->mdlog, "unknown", *p, from,
+ mds->mdlog->start_submit_entry(new ESlaveUpdate(mds->mdlog, "unknown", p->first, from,
ESlaveUpdate::OP_COMMIT, su->origop));
- mds->mdlog->wait_for_safe(new C_MDC_SlaveCommit(this, from, *p));
+ mds->mdlog->wait_for_safe(new C_MDC_SlaveCommit(this, from, p->first));
mds->mdlog->flush();
- finish_uncommitted_slave_update(*p, from);
+ finish_uncommitted_slave_update(p->first, from);
} else {
- MDRequest *mdr = request_get(*p);
+ MDRequest *mdr = request_get(p->first);
+ // information about master imported caps
+ if (p->second.length() > 0)
+ mdr->more()->inode_import.claim(p->second);
+
assert(mdr->slave_request == 0); // shouldn't be doing anything!
request_finish(mdr);
}
@@ -3345,7 +3389,7 @@ void MDCache::trim_unlinked_inodes()
{
dout(7) << "trim_unlinked_inodes" << dendl;
list<CInode*> q;
- for (hash_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
p != inode_map.end();
++p) {
CInode *in = p->second;
@@ -3632,7 +3676,7 @@ void MDCache::rejoin_send_rejoins()
if (!mds->is_rejoin()) {
// i am survivor. send strong rejoin.
// note request remote_auth_pins, xlocks
- for (hash_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
+ for (ceph::unordered_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
p != active_requests.end();
++p) {
if ( p->second->is_slave())
@@ -3876,6 +3920,8 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
dout(10) << "i am a surivivor, and will ack immediately" << dendl;
ack = new MMDSCacheRejoin(MMDSCacheRejoin::OP_ACK);
+ map<inodeno_t,map<client_t,Capability::Import> > imported_caps;
+
// check cap exports
for (map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> >::iterator p = weak->cap_exports.begin();
p != weak->cap_exports.end();
@@ -3886,13 +3932,23 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
q != p->second.end();
++q) {
dout(10) << " claiming cap import " << p->first << " client." << q->first << " on " << *in << dendl;
- rejoin_import_cap(in, q->first, q->second, from);
+ Capability *cap = rejoin_import_cap(in, q->first, q->second, from);
+ Capability::Import& im = imported_caps[p->first][q->first];
+ im.cap_id = cap->get_cap_id();
+ im.issue_seq = cap->get_last_seq();
+ im.mseq = cap->get_mseq();
}
mds->locker->eval(in, CEPH_CAP_LOCKS, true);
}
+
+ ::encode(imported_caps, ack->imported_caps);
} else {
assert(mds->is_rejoin());
+ // we may have already received a strong rejoin from the sender.
+ rejoin_scour_survivor_replicas(from, NULL, acked_inodes, gather_locks);
+ assert(gather_locks.empty());
+
// check cap exports.
for (map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> >::iterator p = weak->cap_exports.begin();
p != weak->cap_exports.end();
@@ -3949,7 +4005,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
dout(10) << " already have " << p->frag << " -> " << fg << " " << *dir << dendl;
} else {
dirs_to_share.insert(dir);
- int nonce = dir->add_replica(from);
+ unsigned nonce = dir->add_replica(from);
dout(10) << " have " << p->frag << " -> " << fg << " " << *dir << dendl;
if (ack)
ack->add_strong_dirfrag(dir->dirfrag(), nonce, dir->dir_rep);
@@ -3988,7 +4044,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
if (survivor && dn->is_replica(from))
dentry_remove_replica(dn, from, gather_locks);
- int dnonce = dn->add_replica(from);
+ unsigned dnonce = dn->add_replica(from);
dout(10) << " have " << *dn << dendl;
if (ack)
ack->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
@@ -4001,7 +4057,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
if (survivor && in->is_replica(from))
inode_remove_replica(in, from, gather_locks);
- int inonce = in->add_replica(from);
+ unsigned inonce = in->add_replica(from);
dout(10) << " have " << *in << dendl;
// scatter the dirlock, just in case?
@@ -4024,7 +4080,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
assert(in); // hmm fixme wrt stray?
if (survivor && in->is_replica(from))
inode_remove_replica(in, from, gather_locks);
- int inonce = in->add_replica(from);
+ unsigned inonce = in->add_replica(from);
dout(10) << " have base " << *in << dendl;
if (ack) {
@@ -4048,11 +4104,13 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
ack->add_inode_base(in);
}
- rejoin_scour_survivor_replicas(from, ack, gather_locks, acked_inodes);
+ rejoin_scour_survivor_replicas(from, ack, acked_inodes, gather_locks);
mds->send_message(ack, weak->get_connection());
- for (set<SimpleLock*>::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p)
- mds->locker->eval_gather(*p);
+ for (set<SimpleLock*>::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p) {
+ if (!(*p)->is_stable())
+ mds->locker->eval_gather(*p);
+ }
} else {
// done?
if (rejoin_gather.empty()) {
@@ -4200,14 +4258,12 @@ bool MDCache::parallel_fetch_traverse_dir(inodeno_t ino, filepath& path,
* ack, the replica dne, and we can remove it from our replica maps.
*/
void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
- set<SimpleLock *>& gather_locks,
- set<vinodeno_t>& acked_inodes)
+ set<vinodeno_t>& acked_inodes,
+ set<SimpleLock *>& gather_locks)
{
dout(10) << "rejoin_scour_survivor_replicas from mds." << from << dendl;
- // FIXME: what about root and stray inodes.
-
- for (hash_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
p != inode_map.end();
++p) {
CInode *in = p->second;
@@ -4215,7 +4271,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
// inode?
if (in->is_auth() &&
in->is_replica(from) &&
- acked_inodes.count(p->second->vino()) == 0) {
+ (ack == NULL || acked_inodes.count(p->second->vino()) == 0)) {
inode_remove_replica(in, from, gather_locks);
dout(10) << " rem " << *in << dendl;
}
@@ -4231,7 +4287,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
if (dir->is_auth() &&
dir->is_replica(from) &&
- ack->strong_dirfrags.count(dir->dirfrag()) == 0) {
+ (ack == NULL || ack->strong_dirfrags.count(dir->dirfrag()) == 0)) {
dir->remove_replica(from);
dout(10) << " rem " << *dir << dendl;
}
@@ -4243,7 +4299,8 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
CDentry *dn = p->second;
if (dn->is_replica(from) &&
- (ack->strong_dentries.count(dir->dirfrag()) == 0 ||
+ (ack == NULL ||
+ ack->strong_dentries.count(dir->dirfrag()) == 0 ||
ack->strong_dentries[dir->dirfrag()].count(string_snap_t(dn->name, dn->last)) == 0)) {
dentry_remove_replica(dn, from, gather_locks);
dout(10) << " rem " << *dn << dendl;
@@ -4592,8 +4649,9 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
}
// barebones dirfrag; the full dirfrag loop below will clean up.
dir = diri->add_dirfrag(new CDir(diri, p->first.frag, this, false));
- if (dir->authority() != CDIR_AUTH_UNDEF &&
- dir->authority().first != from)
+ if (MDS_INO_MDSDIR(from) == p->first.ino ||
+ (dir->authority() != CDIR_AUTH_UNDEF &&
+ dir->authority().first != from))
adjust_subtree_auth(dir, from);
dout(10) << " add dirfrag " << *dir << dendl;
}
@@ -4724,6 +4782,36 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
// belongs to, were trimmed between sending cache rejoin and receiving rejoin ack.
assert(isolated_inodes.empty());
+ map<inodeno_t,map<client_t,Capability::Import> > peer_imported;
+ bufferlist::iterator bp = ack->imported_caps.begin();
+ ::decode(peer_imported, bp);
+
+ for (map<inodeno_t,map<client_t,Capability::Import> >::iterator p = peer_imported.begin();
+ p != peer_imported.end();
+ ++p) {
+ assert(cap_exports.count(p->first));
+ assert(cap_export_targets.count(p->first));
+ assert(cap_export_targets[p->first] == from);
+ for (map<client_t,Capability::Import>::iterator q = p->second.begin();
+ q != p->second.end();
+ ++q) {
+ assert(cap_exports[p->first].count(q->first));
+
+ dout(10) << " exporting caps for client." << q->first << " ino " << p->first << dendl;
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+ assert(session);
+
+ // mark client caps stale.
+ MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, p->first, 0,
+ cap_exports[p->first][q->first].cap_id, 0);
+ m->set_cap_peer(q->second.cap_id, q->second.issue_seq, q->second.mseq, from, 0);
+ mds->send_message_client_counted(m, session);
+
+ cap_exports[p->first].erase(q->first);
+ }
+ assert(cap_exports[p->first].empty());
+ }
+
// done?
assert(rejoin_ack_gather.count(from));
rejoin_ack_gather.erase(from);
@@ -4957,8 +5045,9 @@ bool MDCache::process_imported_caps()
{
dout(10) << "process_imported_caps" << dendl;
- map<inodeno_t,map<client_t, map<int,ceph_mds_cap_reconnect> > >::iterator p;
- for (p = cap_imports.begin(); p != cap_imports.end(); ++p) {
+ for (map<inodeno_t,map<client_t, map<int,ceph_mds_cap_reconnect> > >::iterator p = cap_imports.begin();
+ p != cap_imports.end();
+ ++p) {
CInode *in = get_inode(p->first);
if (in) {
assert(in->is_auth());
@@ -4978,10 +5067,39 @@ bool MDCache::process_imported_caps()
// called by rejoin_gather_finish() ?
if (rejoin_gather.count(mds->get_nodeid()) == 0) {
+ // process caps that were exported by slave rename
+ for (map<inodeno_t,pair<int,map<client_t,Capability::Export> > >::iterator p = rejoin_slave_exports.begin();
+ p != rejoin_slave_exports.end();
+ ++p) {
+ CInode *in = get_inode(p->first);
+ assert(in);
+ for (map<client_t,Capability::Export>::iterator q = p->second.second.begin();
+ q != p->second.second.end();
+ ++q) {
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+ assert(session);
+
+ Capability *cap = in->get_client_cap(q->first);
+ if (!cap)
+ cap = in->add_client_cap(q->first, session);
+ cap->merge(q->second, true);
+
+ Capability::Import& im = rejoin_imported_caps[p->second.first][p->first][q->first];
+ assert(cap->get_last_seq() == im.issue_seq);
+ assert(cap->get_mseq() == im.mseq);
+ cap->set_cap_id(im.cap_id);
+ // send cap import because we assigned a new cap ID
+ do_cap_import(session, in, cap, q->second.cap_id, q->second.seq, q->second.mseq - 1,
+ p->second.first, CEPH_CAP_FLAG_AUTH);
+ }
+ }
+ rejoin_slave_exports.clear();
+ rejoin_imported_caps.clear();
+
// process cap imports
// ino -> client -> frommds -> capex
- p = cap_imports.begin();
- while (p != cap_imports.end()) {
+ for (map<inodeno_t,map<client_t, map<int,ceph_mds_cap_reconnect> > >::iterator p = cap_imports.begin();
+ p != cap_imports.end(); ) {
CInode *in = get_inode(p->first);
if (!in) {
dout(10) << " still missing ino " << p->first
@@ -4991,33 +5109,30 @@ bool MDCache::process_imported_caps()
}
assert(in->is_auth());
for (map<client_t,map<int,ceph_mds_cap_reconnect> >::iterator q = p->second.begin();
- q != p->second.end();
- ++q)
+ q != p->second.end();
+ ++q) {
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+ assert(session);
for (map<int,ceph_mds_cap_reconnect>::iterator r = q->second.begin();
- r != q->second.end();
- ++r) {
- dout(20) << " add_reconnected_cap " << in->ino() << " client." << q->first << dendl;
+ r != q->second.end();
+ ++r) {
add_reconnected_cap(in, q->first, inodeno_t(r->second.snaprealm));
- rejoin_import_cap(in, q->first, r->second, r->first);
+ Capability *cap = in->reconnect_cap(q->first, r->second, session);
+ if (r->first >= 0) {
+ if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists
+ cap->inc_mseq();
+ do_cap_import(session, in, cap, r->second.cap_id, 0, 0, r->first, 0);
+
+ Capability::Import& im = rejoin_imported_caps[r->first][p->first][q->first];
+ im.cap_id = cap->get_cap_id();
+ im.issue_seq = cap->get_last_seq();
+ im.mseq = cap->get_mseq();
+ }
}
+ }
cap_imports.erase(p++); // remove and move on
}
} else {
- for (map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> >::iterator q = cap_exports.begin();
- q != cap_exports.end();
- ++q) {
- for (map<client_t,ceph_mds_cap_reconnect>::iterator r = q->second.begin();
- r != q->second.end();
- ++r) {
- dout(10) << " exporting caps for client." << r->first << " ino " << q->first << dendl;
- Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(r->first.v));
- assert(session);
- // mark client caps stale.
- MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, q->first, 0, 0, 0);
- mds->send_message_client_counted(m, session);
- }
- }
-
trim_non_auth();
rejoin_gather.erase(mds->get_nodeid());
@@ -5057,7 +5172,7 @@ void MDCache::choose_lock_states_and_reconnect_caps()
map<client_t,MClientSnap*> splits;
- for (hash_map<vinodeno_t,CInode*>::iterator i = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator i = inode_map.begin();
i != inode_map.end();
++i) {
CInode *in = i->second;
@@ -5169,7 +5284,7 @@ void MDCache::clean_open_file_lists()
-void MDCache::rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, int frommds)
+Capability* MDCache::rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, int frommds)
{
dout(10) << "rejoin_import_cap for client." << client << " from mds." << frommds
<< " on " << *in << dendl;
@@ -5179,9 +5294,12 @@ void MDCache::rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconn
Capability *cap = in->reconnect_cap(client, icr, session);
if (frommds >= 0) {
- cap->rejoin_import();
- do_cap_import(session, in, cap);
+ if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists
+ cap->inc_mseq();
+ do_cap_import(session, in, cap, icr.cap_id, 0, 0, frommds, 0);
}
+
+ return cap;
}
void MDCache::export_remaining_imported_caps()
@@ -5201,6 +5319,7 @@ void MDCache::export_remaining_imported_caps()
if (session) {
// mark client caps stale.
MClientCaps *stale = new MClientCaps(CEPH_CAP_OP_EXPORT, p->first, 0, 0, 0);
+ stale->set_cap_peer(0, 0, 0, -1, 0);
mds->send_message_client_counted(stale, q->first);
}
}
@@ -5240,14 +5359,16 @@ void MDCache::try_reconnect_cap(CInode *in, Session *session)
// -------
// cap imports and delayed snap parent opens
-void MDCache::do_cap_import(Session *session, CInode *in, Capability *cap)
+void MDCache::do_cap_import(Session *session, CInode *in, Capability *cap,
+ uint64_t p_cap_id, ceph_seq_t p_seq, ceph_seq_t p_mseq,
+ int peer, int p_flags)
{
client_t client = session->info.inst.name.num();
SnapRealm *realm = in->find_snaprealm();
if (realm->have_past_parents_open()) {
dout(10) << "do_cap_import " << session->info.inst.name << " mseq " << cap->get_mseq() << " on " << *in << dendl;
- if (cap->get_last_seq() == 0)
- cap->issue_norevoke(cap->issued()); // reconnected cap
+ if (cap->get_last_seq() == 0) // reconnected cap
+ cap->inc_last_seq();
cap->set_last_issue();
MClientCaps *reap = new MClientCaps(CEPH_CAP_OP_IMPORT,
in->ino(),
@@ -5257,6 +5378,7 @@ void MDCache::do_cap_import(Session *session, CInode *in, Capability *cap)
cap->get_mseq());
in->encode_cap_message(reap, cap);
realm->build_snap_trace(reap->snapbl);
+ reap->set_cap_peer(p_cap_id, p_seq, p_mseq, peer, p_flags);
mds->send_message_client_counted(reap, session);
} else {
dout(10) << "do_cap_import missing past snap parents, delaying " << session->info.inst.name << " mseq "
@@ -5272,6 +5394,8 @@ void MDCache::do_delayed_cap_imports()
{
dout(10) << "do_delayed_cap_imports" << dendl;
+ assert(delayed_imported_caps.empty());
+#if 0
map<client_t,set<CInode*> > d;
d.swap(delayed_imported_caps);
@@ -5296,6 +5420,7 @@ void MDCache::do_delayed_cap_imports()
mds->locker->issue_caps(in);
}
}
+#endif
}
struct C_MDC_OpenSnapParents : public Context {
@@ -5510,7 +5635,7 @@ void MDCache::rejoin_send_acks()
dq.pop_front();
// dir
- for (map<int,int>::iterator r = dir->replicas_begin();
+ for (map<int,unsigned>::iterator r = dir->replicas_begin();
r != dir->replicas_end();
++r) {
ack[r->first]->add_strong_dirfrag(dir->dirfrag(), ++r->second, dir->dir_rep);
@@ -5529,7 +5654,7 @@ void MDCache::rejoin_send_acks()
in = dnl->get_inode();
// dentry
- for (map<int,int>::iterator r = dn->replicas_begin();
+ for (map<int,unsigned>::iterator r = dn->replicas_begin();
r != dn->replicas_end();
++r) {
ack[r->first]->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
@@ -5546,7 +5671,7 @@ void MDCache::rejoin_send_acks()
if (!in)
continue;
- for (map<int,int>::iterator r = in->replicas_begin();
+ for (map<int,unsigned>::iterator r = in->replicas_begin();
r != in->replicas_end();
++r) {
ack[r->first]->add_inode_base(in);
@@ -5561,14 +5686,14 @@ void MDCache::rejoin_send_acks()
// base inodes too
if (root && root->is_auth())
- for (map<int,int>::iterator r = root->replicas_begin();
+ for (map<int,unsigned>::iterator r = root->replicas_begin();
r != root->replicas_end();
++r) {
ack[r->first]->add_inode_base(root);
ack[r->first]->add_inode_locks(root, ++r->second);
}
if (myin)
- for (map<int,int>::iterator r = myin->replicas_begin();
+ for (map<int,unsigned>::iterator r = myin->replicas_begin();
r != myin->replicas_end();
++r) {
ack[r->first]->add_inode_base(myin);
@@ -5580,7 +5705,7 @@ void MDCache::rejoin_send_acks()
p != rejoin_potential_updated_scatterlocks.end();
++p) {
CInode *in = *p;
- for (map<int,int>::iterator r = in->replicas_begin();
+ for (map<int,unsigned>::iterator r = in->replicas_begin();
r != in->replicas_end();
++r)
ack[r->first]->add_inode_base(in);
@@ -5589,9 +5714,12 @@ void MDCache::rejoin_send_acks()
// send acks
for (map<int,MMDSCacheRejoin*>::iterator p = ack.begin();
p != ack.end();
- ++p)
+ ++p) {
+ ::encode(rejoin_imported_caps[p->first], p->second->imported_caps);
mds->send_message_mds(p->second, p->first);
-
+ }
+
+ rejoin_imported_caps.clear();
}
@@ -5599,7 +5727,7 @@ void MDCache::reissue_all_caps()
{
dout(10) << "reissue_all_caps" << dendl;
- for (hash_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
p != inode_map.end();
++p) {
CInode *in = p->second;
@@ -5701,7 +5829,7 @@ void MDCache::unqueue_file_recover(CInode *in)
void MDCache::identify_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q)
{
dout(10) << "identify_files_to_recover" << dendl;
- for (hash_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
p != inode_map.end();
++p) {
CInode *in = p->second;
@@ -5866,7 +5994,7 @@ struct C_MDC_TruncateFinish : public Context {
C_MDC_TruncateFinish(MDCache *c, CInode *i, LogSegment *l) :
mdc(c), in(i), ls(l) {}
void finish(int r) {
- assert(r != -EINVAL);
+ assert(r == 0 || r == -ENOENT);
mdc->truncate_inode_finish(in, ls);
}
};
@@ -6378,9 +6506,9 @@ void MDCache::trim_non_auth()
if (lru.lru_get_size() == 0) {
// root, stray, etc.?
- hash_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
+ ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
while (p != inode_map.end()) {
- hash_map<vinodeno_t,CInode*>::iterator next = p;
+ ceph::unordered_map<vinodeno_t,CInode*>::iterator next = p;
++next;
CInode *in = p->second;
if (!in->is_auth()) {
@@ -6596,11 +6724,11 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
}
// INODES
- for (map<vinodeno_t,int>::iterator it = p->second.inodes.begin();
+ for (map<vinodeno_t,uint32_t>::iterator it = p->second.inodes.begin();
it != p->second.inodes.end();
++it) {
CInode *in = get_inode(it->first);
- int nonce = it->second;
+ unsigned nonce = it->second;
if (!in) {
dout(0) << " inode expire on " << it->first << " from " << from
@@ -6626,11 +6754,11 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
}
// DIRS
- for (map<dirfrag_t,int>::iterator it = p->second.dirs.begin();
+ for (map<dirfrag_t,uint32_t>::iterator it = p->second.dirs.begin();
it != p->second.dirs.end();
++it) {
CDir *dir = get_dirfrag(it->first);
- int nonce = it->second;
+ unsigned nonce = it->second;
if (!dir) {
dout(0) << " dir expire on " << it->first << " from " << from
@@ -6655,7 +6783,7 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
}
// DENTRIES
- for (map<dirfrag_t, map<pair<string,snapid_t>,int> >::iterator pd = p->second.dentries.begin();
+ for (map<dirfrag_t, map<pair<string,snapid_t>,uint32_t> >::iterator pd = p->second.dentries.begin();
pd != p->second.dentries.end();
++pd) {
dout(10) << " dn expires in dir " << pd->first << dendl;
@@ -6670,10 +6798,10 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
assert(dir->is_auth());
}
- for (map<pair<string,snapid_t>,int>::iterator p = pd->second.begin();
+ for (map<pair<string,snapid_t>,uint32_t>::iterator p = pd->second.begin();
p != pd->second.end();
++p) {
- int nonce = p->second;
+ unsigned nonce = p->second;
CDentry *dn;
if (dir) {
@@ -6703,11 +6831,13 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
}
}
- for (set<SimpleLock*>::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p)
- mds->locker->eval_gather(*p);
-
// done
m->put();
+
+ for (set<SimpleLock*>::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p) {
+ if (!(*p)->is_stable())
+ mds->locker->eval_gather(*p);
+ }
}
void MDCache::process_delayed_expire(CDir *dir)
@@ -8165,8 +8295,10 @@ void MDCache::open_ino_finish(inodeno_t ino, open_ino_info_t& info, int ret)
{
dout(10) << "open_ino_finish ino " << ino << " ret " << ret << dendl;
- finish_contexts(g_ceph_context, info.waiters, ret);
+ list<Context*> waiters;
+ waiters.swap(info.waiters);
opening_inodes.erase(ino);
+ finish_contexts(g_ceph_context, waiters, ret);
}
void MDCache::do_open_ino(inodeno_t ino, open_ino_info_t& info, int err)
@@ -8347,8 +8479,18 @@ void MDCache::open_ino(inodeno_t ino, int64_t pool, Context* fin,
open_ino_info_t& info = opening_inodes[ino];
if (want_replica) {
info.want_replica = true;
- if (want_xlocked)
+ if (want_xlocked && !info.want_xlocked) {
+ if (!info.ancestors.empty()) {
+ CInode *diri = get_inode(info.ancestors[0].dirino);
+ if (diri) {
+ frag_t fg = diri->pick_dirfrag(info.ancestors[0].dname);
+ CDir *dir = diri->get_dirfrag(fg);
+ if (dir && !dir->is_auth())
+ discover_ino(dir, ino, NULL, true);
+ }
+ }
info.want_xlocked = true;
+ }
}
info.waiters.push_back(fin);
} else {
@@ -8357,7 +8499,7 @@ void MDCache::open_ino(inodeno_t ino, int64_t pool, Context* fin,
info.want_replica = want_replica;
info.want_xlocked = want_xlocked;
info.tid = ++open_ino_last_tid;
- info.pool = pool >= 0 ? pool : mds->mdsmap->get_first_data_pool();
+ info.pool = pool >= 0 ? pool : default_file_layout.fl_pg_pool;
info.waiters.push_back(fin);
do_open_ino(ino, info, 0);
}
@@ -8502,61 +8644,10 @@ void MDCache::kick_find_ino_peers(int who)
/* ---------------------------- */
-struct C_MDS_FindInoDir : public Context {
- MDCache *mdcache;
- inodeno_t ino;
- Context *fin;
- bufferlist bl;
- C_MDS_FindInoDir(MDCache *m, inodeno_t i, Context *f) : mdcache(m), ino(i), fin(f) {}
- void finish(int r) {
- mdcache->_find_ino_dir(ino, fin, bl, r);
- }
-};
-
-void MDCache::find_ino_dir(inodeno_t ino, Context *fin)
-{
- dout(10) << "find_ino_dir " << ino << dendl;
- assert(!have_inode(ino));
-
- // get the backtrace from the dir
- object_t oid = CInode::get_object_name(ino, frag_t(), "");
- object_locator_t oloc(mds->mdsmap->get_metadata_pool());
-
- C_MDS_FindInoDir *c = new C_MDS_FindInoDir(this, ino, fin);
- mds->objecter->getxattr(oid, oloc, "path", CEPH_NOSNAP, &c->bl, 0, c);
-}
-
-void MDCache::_find_ino_dir(inodeno_t ino, Context *fin, bufferlist& bl, int r)
-{
- dout(10) << "_find_ino_dir " << ino << " got " << r << " " << bl.length() << " bytes" << dendl;
- if (r < 0) {
- fin->complete(r);
- return;
- }
-
- string s(bl.c_str(), bl.length());
- filepath path(s.c_str());
- vector<CDentry*> trace;
-
- dout(10) << "_find_ino_dir traversing to path " << path << dendl;
-
- C_MDS_FindInoDir *c = new C_MDS_FindInoDir(this, ino, fin);
- c->bl = bl;
- r = path_traverse(NULL, NULL, c, path, &trace, NULL, MDS_TRAVERSE_DISCOVER);
- if (r > 0)
- return;
- delete c; // path_traverse doesn't clean it up for us for r <= 0
-
- fin->complete(r);
-}
-
-
-/* ---------------------------- */
-
int MDCache::get_num_client_requests()
{
int count = 0;
- for (hash_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
+ for (ceph::unordered_map<metareqid_t, MDRequest*>::iterator p = active_requests.begin();
p != active_requests.end();
++p) {
if (p->second->reqid.name.is_client() && !p->second->is_slave())
@@ -8572,8 +8663,8 @@ MDRequest *MDCache::request_start(MClientRequest *req)
if (active_requests.count(req->get_reqid())) {
MDRequest *mdr = active_requests[req->get_reqid()];
if (mdr->is_slave()) {
- dout(10) << "request_start already had " << *mdr << ", forward new msg" << dendl;
- mds->forward_message_mds(req, mdr->slave_to_mds);
+ dout(10) << "request_start already had " << *mdr << ", waiting for finish" << dendl;
+ mdr->more()->waiting_for_finish.push_back(new C_MDS_RetryMessage(mds, req));
} else {
dout(10) << "request_start already processing " << *mdr << ", dropping new msg" << dendl;
req->put();
@@ -8623,7 +8714,7 @@ void MDCache::request_finish(MDRequest *mdr)
dout(7) << "request_finish " << *mdr << dendl;
// slave finisher?
- if (mdr->more()->slave_commit) {
+ if (mdr->has_more() && mdr->more()->slave_commit) {
Context *fin = mdr->more()->slave_commit;
mdr->more()->slave_commit = 0;
fin->complete(0); // this must re-call request_finish.
@@ -8674,6 +8765,9 @@ void MDCache::dispatch_request(MDRequest *mdr)
void MDCache::request_drop_foreign_locks(MDRequest *mdr)
{
+ if (!mdr->has_more())
+ return;
+
// clean up slaves
// (will implicitly drop remote dn pins)
for (set<int>::iterator p = mdr->more()->slaves.begin();
@@ -8681,6 +8775,12 @@ void MDCache::request_drop_foreign_locks(MDRequest *mdr)
++p) {
MMDSSlaveRequest *r = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
MMDSSlaveRequest::OP_FINISH);
+
+ // information about rename imported caps
+ if (mdr->more()->srcdn_auth_mds == *p &&
+ mdr->more()->inode_import.length() > 0)
+ r->inode_export.claim(mdr->more()->inode_import);
+
mds->send_message_mds(r, *p);
}
@@ -8731,8 +8831,12 @@ void MDCache::request_cleanup(MDRequest *mdr)
{
dout(15) << "request_cleanup " << *mdr << dendl;
- if (mdr->has_more() && mdr->more()->is_ambiguous_auth)
- mdr->clear_ambiguous_auth();
+ if (mdr->has_more()) {
+ if (mdr->more()->is_ambiguous_auth)
+ mdr->clear_ambiguous_auth();
+ if (!mdr->more()->waiting_for_finish.empty())
+ mds->queue_waiters(mdr->more()->waiting_for_finish);
+ }
request_drop_locks(mdr);
@@ -9298,6 +9402,7 @@ public:
C_MDC_PurgeStrayPurged(MDCache *c, CDentry *d) :
cache(c), dn(d) { }
void finish(int r) {
+ assert(r == 0 || r == -ENOENT);
cache->_purge_stray_purged(dn, r);
}
};
@@ -9666,7 +9771,12 @@ void MDCache::discover_path(CInode *base,
onfinish = new C_MDC_RetryDiscoverPath(this, base, snap, want_path, from);
base->add_waiter(CInode::WAIT_SINGLEAUTH, onfinish);
return;
- }
+ } else if (from == mds->get_nodeid()) {
+ list<Context*> finished;
+ base->take_waiting(CInode::WAIT_DIR, finished);
+ mds->queue_waiters(finished);
+ return;
+ }
if ((want_xlocked && want_path.depth() == 1) ||
!base->is_waiter_for(CInode::WAIT_DIR) || !onfinish) { // FIXME: weak!
@@ -9714,6 +9824,11 @@ void MDCache::discover_path(CDir *base,
onfinish = new C_MDC_RetryDiscoverPath2(this, base, snap, want_path);
base->add_waiter(CDir::WAIT_SINGLEAUTH, onfinish);
return;
+ } else if (from == mds->get_nodeid()) {
+ list<Context*> finished;
+ base->take_sub_waiting(finished);
+ mds->queue_waiters(finished);
+ return;
}
if ((want_xlocked && want_path.depth() == 1) ||
@@ -9761,7 +9876,12 @@ void MDCache::discover_ino(CDir *base,
onfinish = new C_MDC_RetryDiscoverIno(this, base, want_ino);
base->add_waiter(CDir::WAIT_SINGLEAUTH, onfinish);
return;
- }
+ } else if (from == mds->get_nodeid()) {
+ list<Context*> finished;
+ base->take_sub_waiting(finished);
+ mds->queue_waiters(finished);
+ return;
+ }
if (want_xlocked || !base->is_waiting_for_ino(want_ino) || !onfinish) {
discover_info_t& d = _create_discover(from);
@@ -9943,7 +10063,9 @@ void MDCache::handle_discover(MDiscover *dis)
}
// add dir
- if (reply->is_empty() && !dis->wants_base_dir()) {
+ if (curdir->get_version() == 0) {
+ // fetch newly opened dir
+ } else if (reply->is_empty() && !dis->wants_base_dir()) {
dout(7) << "handle_discover not adding unwanted base dir " << *curdir << dendl;
// make sure the base frag is correct, though, in there was a refragment since the
// original request was sent.
@@ -9958,7 +10080,9 @@ void MDCache::handle_discover(MDiscover *dis)
// lookup
CDentry *dn = 0;
- if (dis->get_want_ino()) {
+ if (curdir->get_version() == 0) {
+ // fetch newly opened dir
+ } else if (dis->get_want_ino()) {
// lookup by ino
CInode *in = get_inode(dis->get_want_ino(), snapid);
if (in && in->is_auth() && in->get_parent_dn()->get_dir() == curdir) {
@@ -10417,7 +10541,7 @@ int MDCache::send_dir_updates(CDir *dir, bool bcast)
if (bcast) {
mds->get_mds_map()->get_active_mds_set(who);
} else {
- for (map<int,int>::iterator p = dir->replicas_begin();
+ for (map<int,unsigned>::iterator p = dir->replicas_begin();
p != dir->replicas_end();
++p)
who.insert(p->first);
@@ -10497,7 +10621,7 @@ void MDCache::send_dentry_link(CDentry *dn)
dout(7) << "send_dentry_link " << *dn << dendl;
CDir *subtree = get_subtree_root(dn->get_dir());
- for (map<int,int>::iterator p = dn->replicas_begin();
+ for (map<int,unsigned>::iterator p = dn->replicas_begin();
p != dn->replicas_end();
++p) {
if (mds->mdsmap->get_state(p->first) < MDSMap::STATE_REJOIN ||
@@ -10583,7 +10707,7 @@ void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn, MDRequest *mdr)
{
dout(10) << "send_dentry_unlink " << *dn << dendl;
// share unlink news with replicas
- for (map<int,int>::iterator it = dn->replicas_begin();
+ for (map<int,unsigned>::iterator it = dn->replicas_begin();
it != dn->replicas_end();
++it) {
// don't tell (rmdir) witnesses; they already know
@@ -11100,6 +11224,7 @@ public:
resultfrags.swap(l);
}
virtual void finish(int r) {
+ assert(r == 0 || r == -ENOENT);
mdcache->_fragment_finish(basedirfrag, resultfrags);
}
};
@@ -11252,7 +11377,7 @@ void MDCache::_fragment_stored(MDRequest *mdr)
// tell peers
CDir *first = *info.resultfrags.begin();
- for (map<int,int>::iterator p = first->replica_map.begin();
+ for (map<int,unsigned>::iterator p = first->replicas_begin();
p != first->replica_map.end();
++p) {
if (mds->mdsmap->get_state(p->first) <= MDSMap::STATE_REJOIN)
@@ -11322,6 +11447,7 @@ void MDCache::_fragment_committed(dirfrag_t basedirfrag, list<CDir*>& resultfrag
// backtrace object
dout(10) << " truncate orphan dirfrag " << oid << dendl;
op.truncate(0);
+ op.omap_clear();
} else {
dout(10) << " removing orphan dirfrag " << oid << dendl;
op.remove();
@@ -11677,7 +11803,7 @@ void MDCache::show_cache()
{
dout(7) << "show_cache" << dendl;
- for (hash_map<vinodeno_t,CInode*>::iterator it = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator it = inode_map.begin();
it != inode_map.end();
++it) {
// unlinked?
@@ -11722,7 +11848,7 @@ void MDCache::dump_cache(const char *fn)
return;
}
- for (hash_map<vinodeno_t,CInode*>::iterator it = inode_map.begin();
+ for (ceph::unordered_map<vinodeno_t,CInode*>::iterator it = inode_map.begin();
it != inode_map.end();
++it) {
CInode *in = it->second;
@@ -11731,7 +11857,7 @@ void MDCache::dump_cache(const char *fn)
std::string s = ss.str();
r = safe_write(fd, s.c_str(), s.length());
if (r < 0)
- return;
+ goto out;
list<CDir*> dfs;
in->get_dirfrags(dfs);
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 87b1098..ad0f5ff 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -84,7 +84,7 @@ class MDCache {
// -- my cache --
LRU lru; // dentry lru for expiring items from cache
protected:
- hash_map<vinodeno_t,CInode*> inode_map; // map of inodes by ino
+ ceph::unordered_map<vinodeno_t,CInode*> inode_map; // map of inodes by ino
CInode *root; // root inode
CInode *myin; // .ceph/mds%d dir
@@ -240,7 +240,7 @@ protected:
// -- requests --
protected:
- hash_map<metareqid_t, MDRequest*> active_requests;
+ ceph::unordered_map<metareqid_t, MDRequest*> active_requests;
public:
int get_num_client_requests();
@@ -411,6 +411,8 @@ protected:
set<int> rejoin_gather; // nodes from whom i need a rejoin
set<int> rejoin_sent; // nodes i sent a rejoin to
set<int> rejoin_ack_gather; // nodes from whom i need a rejoin ack
+ map<int,map<inodeno_t,map<client_t,Capability::Import> > > rejoin_imported_caps;
+ map<inodeno_t,pair<int,map<client_t,Capability::Export> > > rejoin_slave_exports;
map<inodeno_t,map<client_t,ceph_mds_cap_reconnect> > cap_exports; // ino -> client -> capex
map<inodeno_t,int> cap_export_targets; // ino -> auth mds
@@ -435,8 +437,8 @@ protected:
CDir* rejoin_invent_dirfrag(dirfrag_t df);
void handle_cache_rejoin_strong(MMDSCacheRejoin *m);
void rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
- set<SimpleLock *>& gather_locks,
- set<vinodeno_t>& acked_inodes);
+ set<vinodeno_t>& acked_inodes,
+ set<SimpleLock *>& gather_locks);
void handle_cache_rejoin_ack(MMDSCacheRejoin *m);
void handle_cache_rejoin_purge(MMDSCacheRejoin *m);
void handle_cache_rejoin_missing(MMDSCacheRejoin *m);
@@ -494,7 +496,7 @@ public:
map<client_t,MClientSnap*>& splits);
void do_realm_invalidate_and_update_notify(CInode *in, int snapop, bool nosend=false);
void send_snaps(map<client_t,MClientSnap*>& splits);
- void rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, int frommds);
+ Capability* rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, int frommds);
void finish_snaprealm_reconnect(client_t client, SnapRealm *realm, snapid_t seq);
void try_reconnect_cap(CInode *in, Session *session);
void export_remaining_imported_caps();
@@ -504,7 +506,9 @@ public:
map<CInode*,map<client_t, set<inodeno_t> > > missing_snap_parents;
map<client_t,set<CInode*> > delayed_imported_caps;
- void do_cap_import(Session *session, CInode *in, Capability *cap);
+ void do_cap_import(Session *session, CInode *in, Capability *cap,
+ uint64_t p_cap_id, ceph_seq_t p_seq, ceph_seq_t p_mseq,
+ int peer, int p_flags);
void do_delayed_cap_imports();
void check_realm_past_parents(SnapRealm *realm);
void open_snap_parents();
@@ -840,15 +844,6 @@ public:
void handle_find_ino_reply(MMDSFindInoReply *m);
void kick_find_ino_peers(int who);
- // -- find_ino_dir --
- struct find_ino_dir_info_t {
- inodeno_t ino;
- Context *fin;
- };
-
- void find_ino_dir(inodeno_t ino, Context *c);
- void _find_ino_dir(inodeno_t ino, Context *c, bufferlist& bl, int r);
-
// -- anchors --
public:
void anchor_create_prep_locks(MDRequest *mdr, CInode *in, set<SimpleLock*>& rdlocks,
@@ -889,7 +884,7 @@ protected:
void _purge_stray_purged(CDentry *dn, int r=0);
void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls);
void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls);
- friend class C_MDC_RetryScanStray;
+ friend struct C_MDC_RetryScanStray;
friend class C_MDC_FetchedBacktrace;
friend class C_MDC_PurgeStrayLogged;
friend class C_MDC_PurgeStrayLoggedTruncate;
@@ -1030,7 +1025,7 @@ public:
CInode *hack_pick_random_inode() {
assert(!inode_map.empty());
int n = rand() % inode_map.size();
- hash_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
+ ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
while (n--) ++p;
return p->second;
}
diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 8372227..b8ae455 100644
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -479,6 +479,29 @@ int MDS::init(int wanted_state)
return 0;
}
+ objecter->init_locked();
+
+ monc->sub_want("mdsmap", 0, 0);
+ monc->renew_subs();
+
+ mds_lock.Unlock();
+
+ // verify that osds support tmap2omap
+ while (true) {
+ objecter->maybe_request_map();
+ objecter->wait_for_osd_map();
+ uint64_t osd_features = objecter->osdmap->get_up_osd_features();
+ if (osd_features & CEPH_FEATURE_OSD_TMAP2OMAP)
+ break;
+ derr << "*** one or more OSDs do not support TMAP2OMAP; upgrade OSDs before starting MDS (or downgrade MDS) ***" << dendl;
+ sleep(10);
+ }
+
+ mds_lock.Lock();
+ if (want_state == CEPH_MDS_STATE_DNE) {
+ suicide(); // we could do something more graceful here
+ }
+
timer.init();
if (wanted_state==MDSMap::STATE_BOOT && g_conf->mds_standby_replay)
@@ -516,11 +539,6 @@ int MDS::init(int wanted_state)
whoami = -1;
messenger->set_myname(entity_name_t::MDS(whoami));
- objecter->init_locked();
-
- monc->sub_want("mdsmap", 0, 0);
- monc->renew_subs();
-
// schedule tick
reset_tick();
@@ -588,6 +606,7 @@ void MDS::tick()
if (is_active()) {
balancer->tick();
+ mdcache->migrator->find_stale_export_freeze();
if (snapserver)
snapserver->check_osd_map(false);
}
@@ -622,7 +641,8 @@ void MDS::beacon_send()
beacon->set_standby_for_name(standby_for_name);
// include _my_ feature set
- CompatSet mdsmap_compat(get_mdsmap_compat_set());
+ CompatSet mdsmap_compat(get_mdsmap_compat_set_default());
+ mdsmap_compat.merge(mdsmap->compat);
beacon->set_compat(mdsmap_compat);
monc->send_mon_message(beacon);
@@ -845,7 +865,7 @@ void MDS::handle_mds_map(MMDSMap *m)
monc->sub_got("mdsmap", mdsmap->get_epoch());
// verify compatset
- CompatSet mdsmap_compat(get_mdsmap_compat_set());
+ CompatSet mdsmap_compat(get_mdsmap_compat_set_all());
dout(10) << " my compat " << mdsmap_compat << dendl;
dout(10) << " mdsmap compat " << mdsmap->compat << dendl;
if (!mdsmap_compat.writeable(mdsmap->compat)) {
@@ -995,7 +1015,8 @@ void MDS::handle_mds_map(MMDSMap *m)
// RESOLVE
// is someone else newly resolving?
- if (is_resolve() || is_rejoin() || is_clientreplay() || is_active() || is_stopping()) {
+ if (is_resolve() || is_reconnect() || is_rejoin() ||
+ is_clientreplay() || is_active() || is_stopping()) {
if (!oldmap->is_resolving() && mdsmap->is_resolving()) {
set<int> resolve;
mdsmap->get_mds_set(resolve, MDSMap::STATE_RESOLVE);
diff --git a/src/mds/MDS.h b/src/mds/MDS.h
index 764b4e2..d6426de 100644
--- a/src/mds/MDS.h
+++ b/src/mds/MDS.h
@@ -35,7 +35,7 @@
#include "SessionMap.h"
-#define CEPH_MDS_PROTOCOL 19 /* cluster internal */
+#define CEPH_MDS_PROTOCOL 20 /* cluster internal */
enum {
@@ -316,6 +316,9 @@ class MDS : public Dispatcher {
public:
int get_req_rate() { return req_rate; }
+ Session *get_session(client_t client) {
+ return sessionmap.get_session(entity_name_t::CLIENT(client.v));
+ }
private:
bool ms_dispatch(Message *m);
diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 05b0c42..c641358 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -20,7 +20,7 @@ using std::stringstream;
// features
-CompatSet get_mdsmap_compat_set() {
+CompatSet get_mdsmap_compat_set_all() {
CompatSet::FeatureSet feature_compat;
CompatSet::FeatureSet feature_ro_compat;
CompatSet::FeatureSet feature_incompat;
@@ -29,6 +29,22 @@ CompatSet get_mdsmap_compat_set() {
feature_incompat.insert(MDS_FEATURE_INCOMPAT_FILELAYOUT);
feature_incompat.insert(MDS_FEATURE_INCOMPAT_DIRINODE);
feature_incompat.insert(MDS_FEATURE_INCOMPAT_ENCODING);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_OMAPDIRFRAG);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_INLINE);
+
+ return CompatSet(feature_compat, feature_ro_compat, feature_incompat);
+}
+
+CompatSet get_mdsmap_compat_set_default() {
+ CompatSet::FeatureSet feature_compat;
+ CompatSet::FeatureSet feature_ro_compat;
+ CompatSet::FeatureSet feature_incompat;
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_BASE);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_CLIENTRANGES);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_FILELAYOUT);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_DIRINODE);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_ENCODING);
+ feature_incompat.insert(MDS_FEATURE_INCOMPAT_OMAPDIRFRAG);
return CompatSet(feature_compat, feature_ro_compat, feature_incompat);
}
@@ -135,7 +151,7 @@ void MDSMap::generate_test_instances(list<MDSMap*>& ls)
m->data_pools.insert(0);
m->metadata_pool = 1;
m->cas_pool = 2;
- m->compat = get_mdsmap_compat_set();
+ m->compat = get_mdsmap_compat_set_all();
// these aren't the defaults, just in case anybody gets confused
m->session_timeout = 61;
@@ -154,6 +170,7 @@ void MDSMap::print(ostream& out)
out << "root\t" << root << "\n";
out << "session_timeout\t" << session_timeout << "\n"
<< "session_autoclose\t" << session_autoclose << "\n";
+ out << "max_file_size\t" << max_file_size << "\n";
out << "last_failure\t" << last_failure << "\n"
<< "last_failure_osd_epoch\t" << last_failure_osd_epoch << "\n";
out << "compat\t" << compat << "\n";
@@ -164,6 +181,7 @@ void MDSMap::print(ostream& out)
<< "stopped\t" << stopped << "\n";
out << "data_pools\t" << data_pools << "\n";
out << "metadata_pool\t" << metadata_pool << "\n";
+ out << "inline_data\t" << (inline_data_enabled ? "enabled" : "disabled") << "\n";
multimap< pair<unsigned,unsigned>, uint64_t > foo;
for (map<uint64_t,mds_info_t>::iterator p = mds_info.begin();
@@ -470,7 +488,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
::encode(cas_pool, bl);
// kclient ignores everything from here
- __u16 ev = 6;
+ __u16 ev = 7;
::encode(ev, bl);
::encode(compat, bl);
::encode(metadata_pool, bl);
@@ -485,6 +503,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
::encode(last_failure_osd_epoch, bl);
::encode(ever_allowed_snaps, bl);
::encode(explicitly_allowed_snaps, bl);
+ ::encode(inline_data_enabled, bl);
ENCODE_FINISH(bl);
}
}
@@ -549,5 +568,7 @@ void MDSMap::decode(bufferlist::iterator& p)
ever_allowed_snaps = true;
explicitly_allowed_snaps = false;
}
+ if (ev >= 7)
+ ::decode(inline_data_enabled, p);
DECODE_FINISH(p);
}
diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 5eadf15..004a699 100644
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -58,7 +58,8 @@ using namespace std;
class CephContext;
-extern CompatSet get_mdsmap_compat_set();
+extern CompatSet get_mdsmap_compat_set_all();
+extern CompatSet get_mdsmap_compat_set_default();
extern CompatSet get_mdsmap_compat_set_base(); // pre v0.20
#define MDS_FEATURE_INCOMPAT_BASE CompatSet::Feature(1, "base v0.20")
@@ -66,6 +67,8 @@ extern CompatSet get_mdsmap_compat_set_base(); // pre v0.20
#define MDS_FEATURE_INCOMPAT_FILELAYOUT CompatSet::Feature(3, "default file layouts on dirs")
#define MDS_FEATURE_INCOMPAT_DIRINODE CompatSet::Feature(4, "dir inode in separate object")
#define MDS_FEATURE_INCOMPAT_ENCODING CompatSet::Feature(5, "mds uses versioned encoding")
+#define MDS_FEATURE_INCOMPAT_OMAPDIRFRAG CompatSet::Feature(6, "dirfrag is stored in omap")
+#define MDS_FEATURE_INCOMPAT_INLINE CompatSet::Feature(7, "mds uses inline data")
class MDSMap {
public:
@@ -178,6 +181,8 @@ protected:
bool ever_allowed_snaps; //< the cluster has ever allowed snap creation
bool explicitly_allowed_snaps; //< the user has explicitly enabled snap creation
+ bool inline_data_enabled;
+
public:
CompatSet compat;
@@ -193,9 +198,13 @@ public:
metadata_pool(0),
max_mds(0),
ever_allowed_snaps(false),
- explicitly_allowed_snaps(false)
+ explicitly_allowed_snaps(false),
+ inline_data_enabled(false)
{ }
+ bool get_inline_data_enabled() { return inline_data_enabled; }
+ void set_inline_data_enabled(bool enabled) { inline_data_enabled = enabled; }
+
utime_t get_session_timeout() {
return utime_t(session_timeout,0);
}
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 0647448..117852c 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -124,9 +124,6 @@ void Migrator::dispatch(Message *m)
case MSG_MDS_EXPORTCAPS:
handle_export_caps(static_cast<MExportCaps*>(m));
break;
- case MSG_MDS_EXPORTCAPSACK:
- handle_export_caps_ack(static_cast<MExportCapsAck*>(m));
- break;
default:
assert(0);
@@ -182,8 +179,149 @@ void Migrator::export_empty_import(CDir *dir)
export_dir( dir, dest );
}
+void Migrator::find_stale_export_freeze()
+{
+ utime_t now = ceph_clock_now(g_ceph_context);
+ utime_t cutoff = now;
+ cutoff -= g_conf->mds_freeze_tree_timeout;
+ /*
+ * We could have situations like:
+ *
+ * - mds.0 authpins an item in subtree A
+ * - mds.0 sends request to mds.1 to authpin an item in subtree B
+ * - mds.0 freezes subtree A
+ * - mds.1 authpins an item in subtree B
+ * - mds.1 sends request to mds.0 to authpin an item in subtree A
+ * - mds.1 freezes subtree B
+ * - mds.1 receives the remote authpin request from mds.0
+ * (wait because subtree B is freezing)
+ * - mds.0 receives the remote authpin request from mds.1
+ * (wait because subtree A is freezing)
+ *
+ *
+ * - client request authpins items in subtree B
+ * - freeze subtree B
+ * - import subtree A which is parent of subtree B
+ * (authpins parent inode of subtree B, see CDir::set_dir_auth())
+ * - freeze subtree A
+ * - client request tries authpinning items in subtree A
+ * (wait because subtree A is freezing)
+ */
+ for (set<pair<utime_t,CDir*> >::iterator p = export_freezing_dirs.begin();
+ p != export_freezing_dirs.end(); ) {
+ if (p->first >= cutoff)
+ break;
+ CDir *dir = p->second;
+ ++p;
+ if (export_freezing_state[dir].num_waiters > 0 ||
+ (!dir->inode->is_root() && dir->get_parent_dir()->is_freezing())) {
+ assert(get_export_state(dir) == EXPORT_DISCOVERING ||
+ get_export_state(dir) == EXPORT_FREEZING);
+ export_try_cancel(dir);
+ }
+ }
+}
+
+void Migrator::export_try_cancel(CDir *dir)
+{
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+
+ int state = it->second.state;
+ switch (state) {
+ case EXPORT_DISCOVERING:
+ dout(10) << "export state=discovering : canceling freeze and removing auth_pin" << dendl;
+ it->second.state = EXPORT_CANCELLED;
+ dir->unfreeze_tree(); // cancel the freeze
+ dir->auth_unpin(this);
+ export_unlock(dir);
+ export_freeze_finish(dir);
+ dir->state_clear(CDir::STATE_EXPORTING);
+ if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+ mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+ break;
+
+ case EXPORT_FREEZING:
+ dout(10) << "export state=freezing : canceling freeze" << dendl;
+ it->second.state = EXPORT_CANCELLED;
+ dir->unfreeze_tree(); // cancel the freeze
+ export_freeze_finish(dir);
+ dir->state_clear(CDir::STATE_EXPORTING);
+ if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+ mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+ break;
+
+ // NOTE: state order reversal, warning comes after prepping
+ case EXPORT_WARNING:
+ dout(10) << "export state=warning : unpinning bounds, unfreezing, notifying" << dendl;
+ // fall-thru
+
+ case EXPORT_PREPPING:
+ if (state != EXPORT_WARNING)
+ dout(10) << "export state=prepping : unpinning bounds, unfreezing" << dendl;
+
+ it->second.state = EXPORT_CANCELLED;
+ {
+ // unpin bounds
+ set<CDir*> bounds;
+ cache->get_subtree_bounds(dir, bounds);
+ for (set<CDir*>::iterator q = bounds.begin();
+ q != bounds.end();
+ ++q) {
+ CDir *bd = *q;
+ bd->put(CDir::PIN_EXPORTBOUND);
+ bd->state_clear(CDir::STATE_EXPORTBOUND);
+ }
+ if (state == EXPORT_WARNING) {
+ // notify bystanders
+ export_notify_abort(dir, bounds);
+ // process delayed expires
+ cache->process_delayed_expire(dir);
+ }
+ }
+ dir->unfreeze_tree();
+ cache->adjust_subtree_auth(dir, mds->get_nodeid());
+ cache->try_subtree_merge(dir); // NOTE: this may journal subtree_map as side effect
+ export_unlock(dir);
+ dir->state_clear(CDir::STATE_EXPORTING);
+ if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+ mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+ break;
+
+ case EXPORT_EXPORTING:
+ dout(10) << "export state=exporting : reversing, and unfreezing" << dendl;
+ it->second.state = EXPORT_CANCELLED;
+ export_reverse(dir);
+ dir->state_clear(CDir::STATE_EXPORTING);
+ break;
+
+ case EXPORT_LOGGINGFINISH:
+ case EXPORT_NOTIFYING:
+ dout(10) << "export state=loggingfinish|notifying : ignoring dest failure, we were successful." << dendl;
+ // leave export_state, don't clean up now.
+ break;
+
+ default:
+ assert(0);
+ }
+
+ // finish clean-up?
+ if (it->second.state == EXPORT_CANCELLED) {
+ // wake up any waiters
+ mds->queue_waiters(it->second.waiting_for_finish);
+
+ export_state.erase(it);
+
+ // send pending import_maps? (these need to go out when all exports have finished.)
+ cache->maybe_send_pending_resolves();
+
+ cache->show_subtrees();
+
+ maybe_do_queued_export();
+ }
+}
// ==========================================================
// mds failure handling
@@ -200,10 +338,10 @@ void Migrator::handle_mds_failure_or_stop(int who)
// freeze. this way no freeze completions run before we want them
// to.
list<CDir*> pinned_dirs;
- for (map<CDir*,int>::iterator p = export_state.begin();
+ for (map<CDir*,export_state_t>::iterator p = export_state.begin();
p != export_state.end();
++p) {
- if (p->second == EXPORT_FREEZING) {
+ if (p->second.state == EXPORT_FREEZING) {
CDir *dir = p->first;
dout(10) << "adding temp auth_pin on freezing " << *dir << dendl;
dir->auth_pin(this);
@@ -211,9 +349,9 @@ void Migrator::handle_mds_failure_or_stop(int who)
}
}
- map<CDir*,int>::iterator p = export_state.begin();
+ map<CDir*,export_state_t>::iterator p = export_state.begin();
while (p != export_state.end()) {
- map<CDir*,int>::iterator next = p;
+ map<CDir*,export_state_t>::iterator next = p;
++next;
CDir *dir = p->first;
@@ -221,129 +359,36 @@ void Migrator::handle_mds_failure_or_stop(int who)
// - that are going to the failed node
// - that aren't frozen yet (to avoid auth_pin deadlock)
// - they havne't prepped yet (they may need to discover bounds to do that)
- if (export_peer[dir] == who ||
- p->second == EXPORT_DISCOVERING ||
- p->second == EXPORT_FREEZING ||
- p->second == EXPORT_PREPPING) {
+ if (p->second.peer == who ||
+ p->second.state == EXPORT_DISCOVERING ||
+ p->second.state == EXPORT_FREEZING ||
+ p->second.state == EXPORT_PREPPING) {
// the guy i'm exporting to failed, or we're just freezing.
- dout(10) << "cleaning up export state (" << p->second << ")" << get_export_statename(p->second)
- << " of " << *dir << dendl;
-
- switch (p->second) {
- case EXPORT_DISCOVERING:
- dout(10) << "export state=discovering : canceling freeze and removing auth_pin" << dendl;
- dir->unfreeze_tree(); // cancel the freeze
- dir->auth_unpin(this);
- export_state.erase(dir); // clean up
- export_unlock(dir);
- export_locks.erase(dir);
- dir->state_clear(CDir::STATE_EXPORTING);
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(export_peer[dir])) // tell them.
- mds->send_message_mds(new MExportDirCancel(dir->dirfrag()), export_peer[dir]);
- break;
-
- case EXPORT_FREEZING:
- dout(10) << "export state=freezing : canceling freeze" << dendl;
- dir->unfreeze_tree(); // cancel the freeze
- export_state.erase(dir); // clean up
- dir->state_clear(CDir::STATE_EXPORTING);
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(export_peer[dir])) // tell them.
- mds->send_message_mds(new MExportDirCancel(dir->dirfrag()), export_peer[dir]);
- break;
-
- // NOTE: state order reversal, warning comes after prepping
- case EXPORT_WARNING:
- dout(10) << "export state=warning : unpinning bounds, unfreezing, notifying" << dendl;
- // fall-thru
-
- case EXPORT_PREPPING:
- if (p->second != EXPORT_WARNING)
- dout(10) << "export state=prepping : unpinning bounds, unfreezing" << dendl;
- {
- // unpin bounds
- set<CDir*> bounds;
- cache->get_subtree_bounds(dir, bounds);
- for (set<CDir*>::iterator q = bounds.begin();
- q != bounds.end();
- ++q) {
- CDir *bd = *q;
- bd->put(CDir::PIN_EXPORTBOUND);
- bd->state_clear(CDir::STATE_EXPORTBOUND);
- }
- // notify bystanders
- if (p->second == EXPORT_WARNING)
- export_notify_abort(dir, bounds);
- }
- dir->unfreeze_tree();
- export_state.erase(dir); // clean up
- cache->adjust_subtree_auth(dir, mds->get_nodeid());
- cache->try_subtree_merge(dir); // NOTE: this may journal subtree_map as side effect
- export_unlock(dir);
- export_locks.erase(dir);
- dir->state_clear(CDir::STATE_EXPORTING);
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(export_peer[dir])) // tell them.
- mds->send_message_mds(new MExportDirCancel(dir->dirfrag()), export_peer[dir]);
- break;
-
- case EXPORT_EXPORTING:
- dout(10) << "export state=exporting : reversing, and unfreezing" << dendl;
- export_reverse(dir);
- export_state.erase(dir); // clean up
- export_locks.erase(dir);
- dir->state_clear(CDir::STATE_EXPORTING);
- break;
-
- case EXPORT_LOGGINGFINISH:
- case EXPORT_NOTIFYING:
- dout(10) << "export state=loggingfinish|notifying : ignoring dest failure, we were successful." << dendl;
- // leave export_state, don't clean up now.
- break;
-
- default:
- assert(0);
- }
-
- // finish clean-up?
- if (export_state.count(dir) == 0) {
- export_peer.erase(dir);
- export_warning_ack_waiting.erase(dir);
- export_notify_ack_waiting.erase(dir);
-
- // wake up any waiters
- mds->queue_waiters(export_finish_waiters[dir]);
- export_finish_waiters.erase(dir);
-
- // send pending import_maps? (these need to go out when all exports have finished.)
- cache->maybe_send_pending_resolves();
-
- cache->show_subtrees();
-
- maybe_do_queued_export();
- }
+ dout(10) << "cleaning up export state (" << p->second.state << ")"
+ << get_export_statename(p->second.state) << " of " << *dir << dendl;
+ export_try_cancel(dir);
} else {
// bystander failed.
- if (export_warning_ack_waiting.count(dir) &&
- export_warning_ack_waiting[dir].count(who)) {
- export_warning_ack_waiting[dir].erase(who);
- export_notify_ack_waiting[dir].erase(who); // they won't get a notify either.
- if (p->second == EXPORT_WARNING) {
+ if (p->second.warning_ack_waiting.count(who)) {
+ p->second.warning_ack_waiting.erase(who);
+ p->second.notify_ack_waiting.erase(who); // they won't get a notify either.
+ if (p->second.state == EXPORT_WARNING) {
// exporter waiting for warning acks, let's fake theirs.
dout(10) << "faking export_warning_ack from mds." << who
- << " on " << *dir << " to mds." << export_peer[dir]
+ << " on " << *dir << " to mds." << p->second.peer
<< dendl;
- if (export_warning_ack_waiting[dir].empty())
+ if (p->second.warning_ack_waiting.empty())
export_go(dir);
}
}
- if (export_notify_ack_waiting.count(dir) &&
- export_notify_ack_waiting[dir].count(who)) {
- export_notify_ack_waiting[dir].erase(who);
- if (p->second == EXPORT_NOTIFYING) {
+ if (p->second.notify_ack_waiting.count(who)) {
+ p->second.notify_ack_waiting.erase(who);
+ if (p->second.state == EXPORT_NOTIFYING) {
// exporter is waiting for notify acks, fake it
dout(10) << "faking export_notify_ack from mds." << who
- << " on " << *dir << " to mds." << export_peer[dir]
+ << " on " << *dir << " to mds." << p->second.peer
<< dendl;
- if (export_notify_ack_waiting[dir].empty())
+ if (p->second.notify_ack_waiting.empty())
export_finish(dir);
}
}
@@ -355,23 +400,23 @@ void Migrator::handle_mds_failure_or_stop(int who)
// check my imports
- map<dirfrag_t,int>::iterator q = import_state.begin();
+ map<dirfrag_t,import_state_t>::iterator q = import_state.begin();
while (q != import_state.end()) {
- map<dirfrag_t,int>::iterator next = q;
+ map<dirfrag_t,import_state_t>::iterator next = q;
++next;
dirfrag_t df = q->first;
CInode *diri = mds->mdcache->get_inode(df.ino);
CDir *dir = mds->mdcache->get_dirfrag(df);
- if (import_peer[df] == who) {
+ if (q->second.peer == who) {
if (dir)
- dout(10) << "cleaning up import state (" << q->second << ")" << get_import_statename(q->second)
- << " of " << *dir << dendl;
+ dout(10) << "cleaning up import state (" << q->second.state << ")"
+ << get_import_statename(q->second.state) << " of " << *dir << dendl;
else
- dout(10) << "cleaning up import state (" << q->second << ")" << get_import_statename(q->second)
- << " of " << df << dendl;
+ dout(10) << "cleaning up import state (" << q->second.state << ")"
+ << get_import_statename(q->second.state) << " of " << df << dendl;
- switch (q->second) {
+ switch (q->second.state) {
case IMPORT_DISCOVERING:
dout(10) << "import state=discovering : clearing state" << dendl;
import_reverse_discovering(df);
@@ -398,16 +443,16 @@ void Migrator::handle_mds_failure_or_stop(int who)
import_remove_pins(dir, bounds);
// adjust auth back to the exporter
- cache->adjust_subtree_auth(dir, import_peer[df]);
+ cache->adjust_subtree_auth(dir, q->second.peer);
cache->try_subtree_merge(dir); // NOTE: may journal subtree_map as side-effect
// bystanders?
- if (import_bystanders[dir].empty()) {
+ if (q->second.bystanders.empty()) {
import_reverse_unfreeze(dir);
} else {
// notify them; wait in aborting state
import_notify_abort(dir, bounds);
- import_state[df] = IMPORT_ABORTING;
+ import_state[df].state = IMPORT_ABORTING;
assert(g_conf->mds_kill_import_at != 10);
}
}
@@ -430,21 +475,26 @@ void Migrator::handle_mds_failure_or_stop(int who)
}
break;
+ case IMPORT_FINISHING:
+ assert(dir);
+ dout(10) << "import state=finishing : finishing import on " << *dir << dendl;
+ import_finish(dir, true);
+ break;
+
case IMPORT_ABORTING:
assert(dir);
dout(10) << "import state=aborting : ignoring repeat failure " << *dir << dendl;
break;
}
} else {
- if (q->second == IMPORT_ABORTING &&
- import_bystanders[dir].count(who)) {
+ if (q->second.state == IMPORT_ABORTING &&
+ q->second.bystanders.count(who)) {
assert(dir);
dout(10) << "faking export_notify_ack from mds." << who
- << " on aborting import " << *dir << " from mds." << import_peer[df]
+ << " on aborting import " << *dir << " from mds." << q->second.peer
<< dendl;
- import_bystanders[dir].erase(who);
- if (import_bystanders[dir].empty()) {
- import_bystanders.erase(dir);
+ q->second.bystanders.erase(who);
+ if (q->second.bystanders.empty()) {
import_reverse_unfreeze(dir);
}
}
@@ -467,21 +517,18 @@ void Migrator::handle_mds_failure_or_stop(int who)
void Migrator::show_importing()
{
dout(10) << "show_importing" << dendl;
- for (map<dirfrag_t,int>::iterator p = import_state.begin();
+ for (map<dirfrag_t,import_state_t>::iterator p = import_state.begin();
p != import_state.end();
++p) {
CDir *dir = mds->mdcache->get_dirfrag(p->first);
if (dir) {
- dout(10) << " importing from " << import_peer[p->first]
- << ": (" << p->second << ") " << get_import_statename(p->second)
- << " " << p->first
- << " " << *dir
- << dendl;
+ dout(10) << " importing from " << p->second.peer
+ << ": (" << p->second.state << ") " << get_import_statename(p->second.state)
+ << " " << p->first << " " << *dir << dendl;
} else {
- dout(10) << " importing from " << import_peer[p->first]
- << ": (" << p->second << ") " << get_import_statename(p->second)
- << " " << p->first
- << dendl;
+ dout(10) << " importing from " << p->second.peer
+ << ": (" << p->second.state << ") " << get_import_statename(p->second.state)
+ << " " << p->first << dendl;
}
}
}
@@ -489,14 +536,12 @@ void Migrator::show_importing()
void Migrator::show_exporting()
{
dout(10) << "show_exporting" << dendl;
- for (map<CDir*,int>::iterator p = export_state.begin();
+ for (map<CDir*,export_state_t>::iterator p = export_state.begin();
p != export_state.end();
++p)
- dout(10) << " exporting to " << export_peer[p->first]
- << ": (" << p->second << ") " << get_export_statename(p->second)
- << " " << p->first->dirfrag()
- << " " << *p->first
- << dendl;
+ dout(10) << " exporting to " << p->second.peer
+ << ": (" << p->second.state << ") " << get_export_statename(p->second.state)
+ << " " << p->first->dirfrag() << " " << *p->first << dendl;
}
@@ -508,21 +553,21 @@ void Migrator::audit()
// import_state
show_importing();
- for (map<dirfrag_t,int>::iterator p = import_state.begin();
+ for (map<dirfrag_t,import_state_t>::iterator p = import_state.begin();
p != import_state.end();
++p) {
- if (p->second == IMPORT_DISCOVERING)
+ if (p->second.state == IMPORT_DISCOVERING)
continue;
- if (p->second == IMPORT_DISCOVERED) {
+ if (p->second.state == IMPORT_DISCOVERED) {
CInode *in = cache->get_inode(p->first.ino);
assert(in);
continue;
}
CDir *dir = cache->get_dirfrag(p->first);
assert(dir);
- if (p->second == IMPORT_PREPPING)
+ if (p->second.state == IMPORT_PREPPING)
continue;
- if (p->second == IMPORT_ABORTING) {
+ if (p->second.state == IMPORT_ABORTING) {
assert(!dir->is_ambiguous_dir_auth());
assert(dir->get_dir_auth().first != mds->get_nodeid());
continue;
@@ -534,12 +579,12 @@ void Migrator::audit()
// export_state
show_exporting();
- for (map<CDir*,int>::iterator p = export_state.begin();
+ for (map<CDir*,export_state_t>::iterator p = export_state.begin();
p != export_state.end();
++p) {
CDir *dir = p->first;
- if (p->second == EXPORT_DISCOVERING ||
- p->second == EXPORT_FREEZING) continue;
+ if (p->second.state == EXPORT_DISCOVERING ||
+ p->second.state == EXPORT_FREEZING) continue;
assert(dir->is_ambiguous_dir_auth());
assert(dir->authority().first == mds->get_nodeid() ||
dir->authority().second == mds->get_nodeid());
@@ -670,13 +715,16 @@ void Migrator::export_dir(CDir *dir, int dest)
dout(7) << "export_dir can't rdlock needed locks, failing." << dendl;
return;
}
- mds->locker->rdlock_take_set(locks);
- export_locks[dir].swap(locks);
// ok.
+ mds->locker->rdlock_take_set(locks);
+
assert(export_state.count(dir) == 0);
- export_state[dir] = EXPORT_DISCOVERING;
- export_peer[dir] = dest;
+ export_state_t& stat = export_state[dir];
+ stat.state = EXPORT_DISCOVERING;
+ stat.peer = dest;
+ stat.tid = ++last_export_tid;
+ stat.locks.swap(locks);
dir->state_set(CDir::STATE_EXPORTING);
assert(g_conf->mds_kill_export_at != 1);
@@ -684,10 +732,16 @@ void Migrator::export_dir(CDir *dir, int dest)
// send ExportDirDiscover (ask target)
filepath path;
dir->inode->make_path(path);
- mds->send_message_mds(new MExportDirDiscover(mds->get_nodeid(), path, dir->dirfrag()), dest);
+ MExportDirDiscover *discover = new MExportDirDiscover(dir->dirfrag(), path,
+ mds->get_nodeid(), stat.tid);
+ mds->send_message_mds(discover, dest);
assert(g_conf->mds_kill_export_at != 2);
// start the freeze, but hold it up with an auth_pin.
+ utime_t now = ceph_clock_now(g_ceph_context);
+ export_freezing_dirs.insert(make_pair(now, dir));
+ export_freezing_state[dir].start_time = now;
+
dir->auth_pin(this);
dir->freeze_tree();
assert(dir->is_freezing_tree());
@@ -709,16 +763,17 @@ void Migrator::handle_export_discover_ack(MExportDirDiscoverAck *m)
dout(7) << "export_discover_ack from " << m->get_source()
<< " on " << *dir << dendl;
- if (export_state.count(dir) == 0 ||
- export_state[dir] != EXPORT_DISCOVERING ||
- export_peer[dir] != m->get_source().num()) {
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ if (it == export_state.end() ||
+ it->second.tid != m->get_tid() ||
+ it->second.peer != m->get_source().num()) {
dout(7) << "must have aborted" << dendl;
} else {
+ assert(it->second.state == EXPORT_DISCOVERING);
// release locks to avoid deadlock
export_unlock(dir);
- export_locks.erase(dir);
// freeze the subtree
- export_state[dir] = EXPORT_FREEZING;
+ it->second.state = EXPORT_FREEZING;
dir->auth_unpin(this);
assert(g_conf->mds_kill_export_at != 3);
}
@@ -726,13 +781,48 @@ void Migrator::handle_export_discover_ack(MExportDirDiscoverAck *m)
m->put(); // done
}
+class C_M_ExportSessionsFlushed : public Context {
+ Migrator *migrator;
+ CDir *dir;
+ uint64_t tid;
+public:
+ C_M_ExportSessionsFlushed(Migrator *m, CDir *d, uint64_t t) :
+ migrator(m), dir(d), tid(t) {}
+ void finish(int r) {
+ migrator->export_sessions_flushed(dir, tid);
+ }
+};
+
+void Migrator::export_sessions_flushed(CDir *dir, uint64_t tid)
+{
+ dout(7) << "export_sessions_flushed " << *dir << dendl;
+
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ if (it == export_state.end() || it->second.tid != tid) {
+ // export must have aborted.
+ dout(7) << "export must have aborted on " << dir << dendl;
+ return;
+ }
+
+ assert(it->second.state == EXPORT_PREPPING || it->second.state == EXPORT_WARNING);
+ assert(it->second.warning_ack_waiting.count(-1) > 0);
+ it->second.warning_ack_waiting.erase(-1);
+ if (it->second.state == EXPORT_WARNING && it->second.warning_ack_waiting.empty())
+ export_go(dir); // start export.
+}
+
void Migrator::export_frozen(CDir *dir)
{
dout(7) << "export_frozen on " << *dir << dendl;
assert(dir->is_frozen());
assert(dir->get_cum_auth_pins() == 0);
- int dest = export_peer[dir];
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ assert(it->second.state == EXPORT_FREEZING);
+
+ export_freeze_finish(dir);
+
CInode *diri = dir->inode;
// ok, try to grab all my locks.
@@ -743,19 +833,18 @@ void Migrator::export_frozen(CDir *dir)
<< *diri << dendl;
// .. unwind ..
- export_peer.erase(dir);
- export_state.erase(dir);
dir->unfreeze_tree();
dir->state_clear(CDir::STATE_EXPORTING);
+ mds->queue_waiters(it->second.waiting_for_finish);
- mds->queue_waiters(export_finish_waiters[dir]);
- export_finish_waiters.erase(dir);
+ mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
- mds->send_message_mds(new MExportDirCancel(dir->dirfrag()), dest);
+ export_state.erase(it);
return;
}
+
mds->locker->rdlock_take_set(locks);
- export_locks[dir].swap(locks);
+ it->second.locks.swap(locks);
cache->show_subtrees();
@@ -766,20 +855,20 @@ void Migrator::export_frozen(CDir *dir)
cache->get_subtree_bounds(dir, bounds);
// generate prep message, log entry.
- MExportDirPrep *prep = new MExportDirPrep(dir->dirfrag());
+ MExportDirPrep *prep = new MExportDirPrep(dir->dirfrag(), it->second.tid);
// include list of bystanders
- for (map<int,int>::iterator p = dir->replicas_begin();
+ for (map<int,unsigned>::iterator p = dir->replicas_begin();
p != dir->replicas_end();
++p) {
- if (p->first != dest) {
+ if (p->first != it->second.peer) {
dout(10) << "bystander mds." << p->first << dendl;
prep->add_bystander(p->first);
}
}
// include base dirfrag
- cache->replicate_dir(dir, dest, prep->basedir);
+ cache->replicate_dir(dir, it->second.peer, prep->basedir);
/*
* include spanning tree for all nested exports.
@@ -794,10 +883,10 @@ void Migrator::export_frozen(CDir *dir)
set<dirfrag_t> dirfrags_added;
// check bounds
- for (set<CDir*>::iterator it = bounds.begin();
- it != bounds.end();
- ++it) {
- CDir *bound = *it;
+ for (set<CDir*>::iterator p = bounds.begin();
+ p != bounds.end();
+ ++p) {
+ CDir *bound = *p;
// pin it.
bound->get(CDir::PIN_EXPORTBOUND);
@@ -820,9 +909,9 @@ void Migrator::export_frozen(CDir *dir)
// prepend dentry + inode
assert(cur->inode->is_auth());
bufferlist bl;
- cache->replicate_dentry(cur->inode->parent, dest, bl);
+ cache->replicate_dentry(cur->inode->parent, it->second.peer, bl);
dout(7) << " added " << *cur->inode->parent << dendl;
- cache->replicate_inode(cur->inode, dest, bl);
+ cache->replicate_inode(cur->inode, it->second.peer, bl);
dout(7) << " added " << *cur->inode << dendl;
bl.claim_append(tracebl);
tracebl.claim(bl);
@@ -838,7 +927,7 @@ void Migrator::export_frozen(CDir *dir)
dirfrags_added.insert(cur->dirfrag());
// prepend dir
- cache->replicate_dir(cur, dest, bl);
+ cache->replicate_dir(cur, it->second.peer, bl);
dout(7) << " added " << *cur << dendl;
bl.claim_append(tracebl);
tracebl.claim(bl);
@@ -854,9 +943,63 @@ void Migrator::export_frozen(CDir *dir)
}
// send.
- export_state[dir] = EXPORT_PREPPING;
- mds->send_message_mds(prep, dest);
+ it->second.state = EXPORT_PREPPING;
+ mds->send_message_mds(prep, it->second.peer);
assert (g_conf->mds_kill_export_at != 4);
+
+ // make sure any new instantiations of caps are flushed out
+ assert(it->second.warning_ack_waiting.empty());
+
+ set<client_t> export_client_set;
+ get_export_client_set(dir, export_client_set);
+
+ C_GatherBuilder gather(g_ceph_context);
+ mds->server->flush_client_sessions(export_client_set, gather);
+ if (gather.has_subs()) {
+ it->second.warning_ack_waiting.insert(-1);
+ gather.set_finisher(new C_M_ExportSessionsFlushed(this, dir, it->second.tid));
+ gather.activate();
+ }
+}
+
+void Migrator::get_export_client_set(CDir *dir, set<client_t>& client_set)
+{
+ list<CDir*> dfs;
+ dfs.push_back(dir);
+ while (!dfs.empty()) {
+ CDir *dir = dfs.front();
+ dfs.pop_front();
+ for (CDir::map_t::iterator p = dir->begin(); p != dir->end(); ++p) {
+ CDentry *dn = p->second;
+ if (!dn->get_linkage()->is_primary())
+ continue;
+ CInode *in = dn->get_linkage()->get_inode();
+ if (in->is_dir()) {
+ // directory?
+ list<CDir*> ls;
+ in->get_dirfrags(ls);
+ for (list<CDir*>::iterator q = ls.begin(); q != ls.end(); ++q) {
+ if (!(*q)->state_test(CDir::STATE_EXPORTBOUND)) {
+ // include nested dirfrag
+ assert((*q)->get_dir_auth().first == CDIR_AUTH_PARENT);
+ dfs.push_back(*q); // it's ours, recurse (later)
+ }
+ }
+ }
+ for (map<client_t, Capability*>::iterator q = in->client_caps.begin();
+ q != in->client_caps.end();
+ ++q)
+ client_set.insert(q->first);
+ }
+ }
+}
+
+void Migrator::get_export_client_set(CInode *in, set<client_t>& client_set)
+{
+ for (map<client_t, Capability*>::iterator q = in->client_caps.begin();
+ q != in->client_caps.end();
+ ++q)
+ client_set.insert(q->first);
}
/* This function DOES put the passed message before returning*/
@@ -867,46 +1010,50 @@ void Migrator::handle_export_prep_ack(MExportDirPrepAck *m)
dout(7) << "export_prep_ack " << *dir << dendl;
- if (export_state.count(dir) == 0 ||
- export_state[dir] != EXPORT_PREPPING) {
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ if (it == export_state.end() ||
+ it->second.tid != m->get_tid() ||
+ it->second.peer != m->get_source().num()) {
// export must have aborted.
dout(7) << "export must have aborted" << dendl;
m->put();
return;
}
+ assert(it->second.state == EXPORT_PREPPING);
assert (g_conf->mds_kill_export_at != 5);
// send warnings
- int dest = export_peer[dir];
set<CDir*> bounds;
cache->get_subtree_bounds(dir, bounds);
- assert(export_peer.count(dir));
- assert(export_warning_ack_waiting.count(dir) == 0);
- assert(export_notify_ack_waiting.count(dir) == 0);
+ assert(it->second.warning_ack_waiting.empty() ||
+ (it->second.warning_ack_waiting.size() == 1 &&
+ it->second.warning_ack_waiting.count(-1) > 0));
+ assert(it->second.notify_ack_waiting.empty());
- for (map<int,int>::iterator p = dir->replicas_begin();
+ for (map<int,unsigned>::iterator p = dir->replicas_begin();
p != dir->replicas_end();
++p) {
- if (p->first == dest) continue;
+ if (p->first == it->second.peer) continue;
if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(p->first))
continue; // only if active
- export_warning_ack_waiting[dir].insert(p->first);
- export_notify_ack_waiting[dir].insert(p->first); // we'll eventually get a notifyack, too!
+ it->second.warning_ack_waiting.insert(p->first);
+ it->second.notify_ack_waiting.insert(p->first); // we'll eventually get a notifyack, too!
- MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), true,
+ MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), it->second.tid, true,
pair<int,int>(mds->get_nodeid(),CDIR_AUTH_UNKNOWN),
- pair<int,int>(mds->get_nodeid(),export_peer[dir]));
- for (set<CDir*>::iterator i = bounds.begin(); i != bounds.end(); ++i)
- notify->get_bounds().push_back((*i)->dirfrag());
+ pair<int,int>(mds->get_nodeid(),it->second.peer));
+ for (set<CDir*>::iterator q = bounds.begin(); q != bounds.end(); ++q)
+ notify->get_bounds().push_back((*q)->dirfrag());
mds->send_message_mds(notify, p->first);
}
- export_state[dir] = EXPORT_WARNING;
+
+ it->second.state = EXPORT_WARNING;
assert(g_conf->mds_kill_export_at != 6);
// nobody to warn?
- if (export_warning_ack_waiting.count(dir) == 0)
+ if (it->second.warning_ack_waiting.empty())
export_go(dir); // start export.
// done.
@@ -917,54 +1064,55 @@ void Migrator::handle_export_prep_ack(MExportDirPrepAck *m)
class C_M_ExportGo : public Context {
Migrator *migrator;
CDir *dir;
+ uint64_t tid;
public:
- C_M_ExportGo(Migrator *m, CDir *d) : migrator(m), dir(d) {}
+ C_M_ExportGo(Migrator *m, CDir *d, uint64_t t) :
+ migrator(m), dir(d), tid(t) {}
void finish(int r) {
- migrator->export_go_synced(dir);
+ migrator->export_go_synced(dir, tid);
}
};
void Migrator::export_go(CDir *dir)
{
- assert(export_peer.count(dir));
- int dest = export_peer[dir];
- dout(7) << "export_go " << *dir << " to " << dest << dendl;
+ assert(export_state.count(dir));
+ dout(7) << "export_go " << *dir << " to " << export_state[dir].peer << dendl;
// first sync log to flush out e.g. any cap imports
- mds->mdlog->wait_for_safe(new C_M_ExportGo(this, dir));
+ mds->mdlog->wait_for_safe(new C_M_ExportGo(this, dir, export_state[dir].tid));
mds->mdlog->flush();
}
-void Migrator::export_go_synced(CDir *dir)
-{
- if (export_state.count(dir) == 0 ||
- export_state[dir] != EXPORT_WARNING) {
+void Migrator::export_go_synced(CDir *dir, uint64_t tid)
+{
+
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ if (it == export_state.end() ||
+ it->second.tid != tid) {
// export must have aborted.
dout(7) << "export must have aborted on " << dir << dendl;
return;
}
+ assert(it->second.state == EXPORT_WARNING);
- assert(export_peer.count(dir));
- int dest = export_peer[dir];
- dout(7) << "export_go_synced " << *dir << " to " << dest << dendl;
+ dout(7) << "export_go_synced " << *dir << " to " << it->second.peer << dendl;
cache->show_subtrees();
- export_warning_ack_waiting.erase(dir);
- export_state[dir] = EXPORT_EXPORTING;
+ it->second.state = EXPORT_EXPORTING;
assert(g_conf->mds_kill_export_at != 7);
assert(dir->get_cum_auth_pins() == 0);
// set ambiguous auth
- cache->adjust_subtree_auth(dir, mds->get_nodeid(), dest);
+ cache->adjust_subtree_auth(dir, mds->get_nodeid(), it->second.peer);
// take away the popularity we're sending.
utime_t now = ceph_clock_now(g_ceph_context);
mds->balancer->subtract_export(dir, now);
// fill export message with cache data
- MExportDir *req = new MExportDir(dir->dirfrag());
+ MExportDir *req = new MExportDir(dir->dirfrag(), it->second.tid);
map<client_t,entity_inst_t> exported_client_map;
int num_exported_inodes = encode_export_dir(req->export_data,
dir, // recur start point
@@ -981,7 +1129,7 @@ void Migrator::export_go_synced(CDir *dir)
req->add_export((*p)->dirfrag());
// send
- mds->send_message_mds(req, dest);
+ mds->send_message_mds(req, it->second.peer);
assert(g_conf->mds_kill_export_at != 8);
// stats
@@ -1017,10 +1165,10 @@ void Migrator::encode_export_inode(CInode *in, bufferlist& enc_state,
in->encode_export(enc_state);
// caps
- encode_export_inode_caps(in, enc_state, exported_client_map);
+ encode_export_inode_caps(in, true, enc_state, exported_client_map);
}
-void Migrator::encode_export_inode_caps(CInode *in, bufferlist& bl,
+void Migrator::encode_export_inode_caps(CInode *in, bool auth_cap, bufferlist& bl,
map<client_t,entity_inst_t>& exported_client_map)
{
dout(20) << "encode_export_inode_caps " << *in << dendl;
@@ -1029,10 +1177,12 @@ void Migrator::encode_export_inode_caps(CInode *in, bufferlist& bl,
map<client_t,Capability::Export> cap_map;
in->export_client_caps(cap_map);
::encode(cap_map, bl);
- ::encode(in->get_mds_caps_wanted(), bl);
+ if (auth_cap) {
+ ::encode(in->get_mds_caps_wanted(), bl);
- in->state_set(CInode::STATE_EXPORTINGCAPS);
- in->get(CInode::PIN_EXPORTINGCAPS);
+ in->state_set(CInode::STATE_EXPORTINGCAPS);
+ in->get(CInode::PIN_EXPORTINGCAPS);
+ }
// make note of clients named by exported capabilities
for (map<client_t, Capability*>::iterator it = in->client_caps.begin();
@@ -1041,7 +1191,8 @@ void Migrator::encode_export_inode_caps(CInode *in, bufferlist& bl,
exported_client_map[it->first] = mds->sessionmap.get_inst(entity_name_t::CLIENT(it->first.v));
}
-void Migrator::finish_export_inode_caps(CInode *in)
+void Migrator::finish_export_inode_caps(CInode *in, int peer,
+ map<client_t,Capability::Import>& peer_imported)
{
dout(20) << "finish_export_inode_caps " << *in << dendl;
@@ -1053,21 +1204,23 @@ void Migrator::finish_export_inode_caps(CInode *in)
it != in->client_caps.end();
++it) {
Capability *cap = it->second;
- dout(7) << "finish_export_inode telling client." << it->first
+ dout(7) << "finish_export_inode_caps telling client." << it->first
<< " exported caps on " << *in << dendl;
- MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT,
- in->ino(),
- in->find_snaprealm()->inode->ino(),
- cap->get_cap_id(), cap->get_last_seq(),
- cap->pending(), cap->wanted(), 0,
- cap->get_mseq());
+ MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, in->ino(), 0,
+ cap->get_cap_id(), cap->get_mseq());
+
+ map<client_t,Capability::Import>::iterator q = peer_imported.find(it->first);
+ assert(q != peer_imported.end());
+ m->set_cap_peer(q->second.cap_id, q->second.issue_seq, q->second.mseq, peer, 0);
mds->send_message_client_counted(m, it->first);
}
in->clear_client_caps_after_export();
mds->locker->eval(in, CEPH_CAP_LOCKS);
}
-void Migrator::finish_export_inode(CInode *in, utime_t now, list<Context*>& finished)
+void Migrator::finish_export_inode(CInode *in, utime_t now, int peer,
+ map<client_t,Capability::Import>& peer_imported,
+ list<Context*>& finished)
{
dout(12) << "finish_export_inode " << *in << dendl;
@@ -1109,7 +1262,7 @@ void Migrator::finish_export_inode(CInode *in, utime_t now, list<Context*>& fini
in->finish_export(now);
- finish_export_inode_caps(in);
+ finish_export_inode_caps(in, peer, peer_imported);
// *** other state too?
@@ -1210,7 +1363,9 @@ int Migrator::encode_export_dir(bufferlist& exportbl,
return num_exported;
}
-void Migrator::finish_export_dir(CDir *dir, list<Context*>& finished, utime_t now)
+void Migrator::finish_export_dir(CDir *dir, utime_t now, int peer,
+ map<inodeno_t,map<client_t,Capability::Import> >& peer_imported,
+ list<Context*>& finished)
{
dout(10) << "finish_export_dir " << *dir << dendl;
@@ -1221,7 +1376,7 @@ void Migrator::finish_export_dir(CDir *dir, list<Context*>& finished, utime_t no
assert(dir->is_auth());
dir->state_clear(CDir::STATE_AUTH);
dir->remove_bloom();
- dir->replica_nonce = CDir::NONCE_EXPORT;
+ dir->replica_nonce = CDir::EXPORT_NONCE;
if (dir->is_dirty())
dir->mark_clean();
@@ -1244,7 +1399,7 @@ void Migrator::finish_export_dir(CDir *dir, list<Context*>& finished, utime_t no
// inode?
if (dn->get_linkage()->is_primary()) {
- finish_export_inode(in, now, finished);
+ finish_export_inode(in, now, peer, peer_imported[in->ino()], finished);
// subdirs?
in->get_nested_dirfrags(subdirs);
@@ -1253,7 +1408,7 @@ void Migrator::finish_export_dir(CDir *dir, list<Context*>& finished, utime_t no
// subdirs
for (list<CDir*>::iterator it = subdirs.begin(); it != subdirs.end(); ++it)
- finish_export_dir(*it, finished, now);
+ finish_export_dir(*it, now, peer, peer_imported, finished);
}
class C_MDS_ExportFinishLogged : public Context {
@@ -1281,17 +1436,22 @@ void Migrator::handle_export_ack(MExportDirAck *m)
// yay!
dout(7) << "handle_export_ack " << *dir << dendl;
- export_warning_ack_waiting.erase(dir);
-
- export_state[dir] = EXPORT_LOGGINGFINISH;
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ assert(it->second.state == EXPORT_EXPORTING);
+ assert(it->second.tid == m->get_tid());
+
+ bufferlist::iterator bp = m->imported_caps.begin();
+ ::decode(it->second.peer_imported, bp);
+
+ it->second.state = EXPORT_LOGGINGFINISH;
assert (g_conf->mds_kill_export_at != 9);
set<CDir*> bounds;
cache->get_subtree_bounds(dir, bounds);
// list us second, them first.
// this keeps authority().first in sync with subtree auth state in the journal.
- int target = export_peer[dir];
- cache->adjust_subtree_auth(dir, target, mds->get_nodeid());
+ cache->adjust_subtree_auth(dir, it->second.peer, mds->get_nodeid());
// log completion.
// include export bounds, to ensure they're in the journal.
@@ -1322,11 +1482,13 @@ void Migrator::export_notify_abort(CDir *dir, set<CDir*>& bounds)
{
dout(7) << "export_notify_abort " << *dir << dendl;
- for (set<int>::iterator p = export_notify_ack_waiting[dir].begin();
- p != export_notify_ack_waiting[dir].end();
+ export_state_t& stat = export_state[dir];
+
+ for (set<int>::iterator p = stat.notify_ack_waiting.begin();
+ p != stat.notify_ack_waiting.end();
++p) {
- MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), false,
- pair<int,int>(mds->get_nodeid(),export_peer[dir]),
+ MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(),stat.tid,false,
+ pair<int,int>(mds->get_nodeid(),stat.peer),
pair<int,int>(mds->get_nodeid(),CDIR_AUTH_UNKNOWN));
for (set<CDir*>::iterator i = bounds.begin(); i != bounds.end(); ++i)
notify->get_bounds().push_back((*i)->dirfrag());
@@ -1342,9 +1504,7 @@ void Migrator::export_notify_abort(CDir *dir, set<CDir*>& bounds)
void Migrator::export_reverse(CDir *dir)
{
dout(7) << "export_reverse " << *dir << dendl;
-
- assert(export_state[dir] == EXPORT_EXPORTING);
-
+
set<CDir*> bounds;
cache->get_subtree_bounds(dir, bounds);
@@ -1385,10 +1545,6 @@ void Migrator::export_reverse(CDir *dir)
// process delayed expires
cache->process_delayed_expire(dir);
- // some clean up
- export_warning_ack_waiting.erase(dir);
- export_notify_ack_waiting.erase(dir);
-
// unfreeze
dir->unfreeze_tree();
@@ -1407,18 +1563,18 @@ void Migrator::export_logged_finish(CDir *dir)
{
dout(7) << "export_logged_finish " << *dir << dendl;
- // send notifies
- int dest = export_peer[dir];
+ export_state_t& stat = export_state[dir];
+ // send notifies
set<CDir*> bounds;
cache->get_subtree_bounds(dir, bounds);
- for (set<int>::iterator p = export_notify_ack_waiting[dir].begin();
- p != export_notify_ack_waiting[dir].end();
+ for (set<int>::iterator p = stat.notify_ack_waiting.begin();
+ p != stat.notify_ack_waiting.end();
++p) {
- MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), true,
- pair<int,int>(mds->get_nodeid(), dest),
- pair<int,int>(dest, CDIR_AUTH_UNKNOWN));
+ MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), stat.tid, true,
+ pair<int,int>(mds->get_nodeid(), stat.peer),
+ pair<int,int>(stat.peer, CDIR_AUTH_UNKNOWN));
for (set<CDir*>::iterator i = bounds.begin(); i != bounds.end(); ++i)
notify->get_bounds().push_back((*i)->dirfrag());
@@ -1427,12 +1583,20 @@ void Migrator::export_logged_finish(CDir *dir)
}
// wait for notifyacks
- export_state[dir] = EXPORT_NOTIFYING;
+ stat.state = EXPORT_NOTIFYING;
assert (g_conf->mds_kill_export_at != 11);
// no notifies to wait for?
- if (export_notify_ack_waiting[dir].empty())
+ if (stat.notify_ack_waiting.empty()) {
export_finish(dir); // skip notify/notify_ack stage.
+ } else {
+ // notify peer to send cap import messages to clients
+ if (mds->mdsmap->is_clientreplay_or_active_or_stopping(stat.peer)) {
+ mds->send_message_mds(new MExportDirFinish(dir->dirfrag(), false, stat.tid), stat.peer);
+ } else {
+ dout(7) << "not sending MExportDirFinish, dest has failed" << dendl;
+ }
+ }
}
/*
@@ -1450,39 +1614,37 @@ void Migrator::handle_export_notify_ack(MExportDirNotifyAck *m)
CDir *dir = cache->get_dirfrag(m->get_dirfrag());
assert(dir);
int from = m->get_source().num();
-
- if (export_state.count(dir) && export_state[dir] == EXPORT_WARNING) {
- // exporting. process warning.
- dout(7) << "handle_export_notify_ack from " << m->get_source()
- << ": exporting, processing warning on "
- << *dir << dendl;
- assert(export_warning_ack_waiting.count(dir));
- export_warning_ack_waiting[dir].erase(from);
-
- if (export_warning_ack_waiting[dir].empty())
- export_go(dir); // start export.
- }
- else if (export_state.count(dir) && export_state[dir] == EXPORT_NOTIFYING) {
- // exporting. process notify.
- dout(7) << "handle_export_notify_ack from " << m->get_source()
- << ": exporting, processing notify on "
- << *dir << dendl;
- assert(export_notify_ack_waiting.count(dir));
- export_notify_ack_waiting[dir].erase(from);
-
- if (export_notify_ack_waiting[dir].empty())
- export_finish(dir);
- }
- else if (import_state.count(dir->dirfrag()) && import_state[dir->dirfrag()] == IMPORT_ABORTING) {
- // reversing import
- dout(7) << "handle_export_notify_ack from " << m->get_source()
- << ": aborting import on "
- << *dir << dendl;
- assert(import_bystanders[dir].count(from));
- import_bystanders[dir].erase(from);
- if (import_bystanders[dir].empty()) {
- import_bystanders.erase(dir);
- import_reverse_unfreeze(dir);
+
+ if (export_state.count(dir)) {
+ export_state_t& stat = export_state[dir];
+ if (stat.state == EXPORT_WARNING) {
+ // exporting. process warning.
+ dout(7) << "handle_export_notify_ack from " << m->get_source()
+ << ": exporting, processing warning on " << *dir << dendl;
+ stat.warning_ack_waiting.erase(from);
+
+ if (stat.warning_ack_waiting.empty())
+ export_go(dir); // start export.
+ } else if (stat.state == EXPORT_NOTIFYING) {
+ // exporting. process notify.
+ dout(7) << "handle_export_notify_ack from " << m->get_source()
+ << ": exporting, processing notify on " << *dir << dendl;
+ stat.notify_ack_waiting.erase(from);
+
+ if (stat.notify_ack_waiting.empty())
+ export_finish(dir);
+ }
+ }
+ else if (import_state.count(dir->dirfrag())) {
+ import_state_t& stat = import_state[dir->dirfrag()];
+ if (stat.state == IMPORT_ABORTING) {
+ // reversing import
+ dout(7) << "handle_export_notify_ack from " << m->get_source()
+ << ": aborting import on " << *dir << dendl;
+ assert(stat.bystanders.count(from));
+ stat.bystanders.erase(from);
+ if (stat.bystanders.empty())
+ import_reverse_unfreeze(dir);
}
}
@@ -1493,7 +1655,8 @@ void Migrator::export_unlock(CDir *dir)
{
dout(10) << "export_unlock " << *dir << dendl;
- mds->locker->rdlock_finish_set(export_locks[dir]);
+ mds->locker->rdlock_finish_set(export_state[dir].locks);
+ export_state[dir].locks.clear();
list<Context*> ls;
mds->queue_waiters(ls);
@@ -1504,22 +1667,24 @@ void Migrator::export_finish(CDir *dir)
dout(5) << "export_finish " << *dir << dendl;
assert (g_conf->mds_kill_export_at != 12);
- if (export_state.count(dir) == 0) {
+ map<CDir*,export_state_t>::iterator it = export_state.find(dir);
+ if (it == export_state.end()) {
dout(7) << "target must have failed, not sending final commit message. export succeeded anyway." << dendl;
return;
}
// send finish/commit to new auth
- if (mds->mdsmap->is_clientreplay_or_active_or_stopping(export_peer[dir])) {
- mds->send_message_mds(new MExportDirFinish(dir->dirfrag()), export_peer[dir]);
+ if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) {
+ mds->send_message_mds(new MExportDirFinish(dir->dirfrag(), true, it->second.tid), it->second.peer);
} else {
- dout(7) << "not sending MExportDirFinish, dest has failed" << dendl;
+ dout(7) << "not sending MExportDirFinish last, dest has failed" << dendl;
}
assert(g_conf->mds_kill_export_at != 13);
// finish export (adjust local cache state)
C_Contexts *fin = new C_Contexts(g_ceph_context);
- finish_export_dir(dir, fin->contexts, ceph_clock_now(g_ceph_context));
+ finish_export_dir(dir, ceph_clock_now(g_ceph_context),
+ it->second.peer, it->second.peer_imported, fin->contexts);
dir->add_waiter(CDir::WAIT_UNFREEZE, fin);
// unfreeze
@@ -1539,7 +1704,7 @@ void Migrator::export_finish(CDir *dir)
// adjust auth, with possible subtree merge.
// (we do this _after_ removing EXPORTBOUND pins, to allow merges)
- cache->adjust_subtree_auth(dir, export_peer[dir]);
+ cache->adjust_subtree_auth(dir, it->second.peer);
cache->try_subtree_merge(dir); // NOTE: may journal subtree_map as sideeffect
// no more auth subtree? clear scatter dirty
@@ -1555,14 +1720,11 @@ void Migrator::export_finish(CDir *dir)
// remove from exporting list, clean up state
dir->state_clear(CDir::STATE_EXPORTING);
- export_state.erase(dir);
- export_locks.erase(dir);
- export_peer.erase(dir);
- export_notify_ack_waiting.erase(dir);
// queue finishers
- mds->queue_waiters(export_finish_waiters[dir]);
- export_finish_waiters.erase(dir);
+ mds->queue_waiters(it->second.waiting_for_finish);
+
+ export_state.erase(it);
cache->show_subtrees();
audit();
@@ -1593,18 +1755,23 @@ void Migrator::handle_export_discover(MExportDirDiscover *m)
// note import state
dirfrag_t df = m->get_dirfrag();
// only start discovering on this message once.
+ map<dirfrag_t,import_state_t>::iterator it = import_state.find(df);
if (!m->started) {
+ assert(it == import_state.end());
m->started = true;
- import_pending_msg[df] = m;
- import_state[df] = IMPORT_DISCOVERING;
- import_peer[df] = from;
+ import_state[df].state = IMPORT_DISCOVERING;
+ import_state[df].peer = from;
+ import_state[df].tid = m->get_tid();
} else {
// am i retrying after ancient path_traverse results?
- if (import_pending_msg.count(df) == 0 || import_pending_msg[df] != m) {
+ if (it == import_state.end() ||
+ it->second.peer != from ||
+ it->second.tid != m->get_tid()) {
dout(7) << " dropping obsolete message" << dendl;
m->put();
return;
}
+ assert(it->second.state == IMPORT_DISCOVERING);
}
if (!mds->mdcache->is_open()) {
@@ -1634,8 +1801,7 @@ void Migrator::handle_export_discover(MExportDirDiscover *m)
// yay
dout(7) << "handle_export_discover have " << df << " inode " << *in << dendl;
- import_state[m->get_dirfrag()] = IMPORT_DISCOVERED;
- import_pending_msg.erase(m->get_dirfrag());
+ import_state[df].state = IMPORT_DISCOVERED;
// pin inode in the cache (for now)
assert(in->is_dir());
@@ -1643,16 +1809,14 @@ void Migrator::handle_export_discover(MExportDirDiscover *m)
// reply
dout(7) << " sending export_discover_ack on " << *in << dendl;
- mds->send_message_mds(new MExportDirDiscoverAck(df), import_peer[df]);
+ mds->send_message_mds(new MExportDirDiscoverAck(df, m->get_tid()), import_state[df].peer);
m->put();
assert (g_conf->mds_kill_import_at != 2);
}
void Migrator::import_reverse_discovering(dirfrag_t df)
{
- import_pending_msg.erase(df);
import_state.erase(df);
- import_peer.erase(df);
}
void Migrator::import_reverse_discovered(dirfrag_t df, CInode *diri)
@@ -1660,14 +1824,12 @@ void Migrator::import_reverse_discovered(dirfrag_t df, CInode *diri)
// unpin base
diri->put(CInode::PIN_IMPORTING);
import_state.erase(df);
- import_peer.erase(df);
}
void Migrator::import_reverse_prepping(CDir *dir)
{
- import_pending_msg.erase(dir->dirfrag());
set<CDir*> bounds;
- cache->map_dirfrag_set(import_bound_ls[dir], bounds);
+ cache->map_dirfrag_set(import_state[dir->dirfrag()].bound_ls, bounds);
import_remove_pins(dir, bounds);
import_reverse_final(dir);
}
@@ -1677,24 +1839,27 @@ void Migrator::handle_export_cancel(MExportDirCancel *m)
{
dout(7) << "handle_export_cancel on " << m->get_dirfrag() << dendl;
dirfrag_t df = m->get_dirfrag();
- if (import_state[df] == IMPORT_DISCOVERING) {
+ map<dirfrag_t,import_state_t>::iterator it = import_state.find(df);
+ if (it == import_state.end()) {
+ assert(0 == "got export_cancel in weird state");
+ } else if (it->second.state == IMPORT_DISCOVERING) {
import_reverse_discovering(df);
- } else if (import_state[df] == IMPORT_DISCOVERED) {
+ } else if (it->second.state == IMPORT_DISCOVERED) {
CInode *in = cache->get_inode(df.ino);
assert(in);
import_reverse_discovered(df, in);
- } else if (import_state[df] == IMPORT_PREPPING) {
+ } else if (it->second.state == IMPORT_PREPPING) {
CDir *dir = mds->mdcache->get_dirfrag(df);
assert(dir);
import_reverse_prepping(dir);
- } else if (import_state[df] == IMPORT_PREPPED) {
+ } else if (it->second.state == IMPORT_PREPPED) {
CDir *dir = mds->mdcache->get_dirfrag(df);
assert(dir);
set<CDir*> bounds;
cache->get_subtree_bounds(dir, bounds);
import_remove_pins(dir, bounds);
// adjust auth back to the exportor
- cache->adjust_subtree_auth(dir, import_peer[df]);
+ cache->adjust_subtree_auth(dir, it->second.peer);
cache->try_subtree_merge(dir);
import_reverse_unfreeze(dir);
} else {
@@ -1714,19 +1879,24 @@ void Migrator::handle_export_prep(MExportDirPrep *m)
list<Context*> finished;
// assimilate root dir.
+ map<dirfrag_t,import_state_t>::iterator it = import_state.find(m->get_dirfrag());
if (!m->did_assim()) {
+ assert(it != import_state.end());
+ assert(it->second.state == IMPORT_DISCOVERED);
diri = cache->get_inode(m->get_dirfrag().ino);
assert(diri);
bufferlist::iterator p = m->basedir.begin();
dir = cache->add_replica_dir(p, diri, oldauth, finished);
dout(7) << "handle_export_prep on " << *dir << " (first pass)" << dendl;
} else {
- if (import_pending_msg.count(m->get_dirfrag()) == 0 ||
- import_pending_msg[m->get_dirfrag()] != m) {
+ if (it == import_state.end() ||
+ it->second.peer != oldauth ||
+ it->second.tid != m->get_tid()) {
dout(7) << "handle_export_prep obsolete message, dropping" << dendl;
m->put();
return;
}
+ assert(it->second.state == IMPORT_PREPPING);
dir = cache->get_dirfrag(m->get_dirfrag());
assert(dir);
@@ -1750,21 +1920,20 @@ void Migrator::handle_export_prep(MExportDirPrep *m)
if (!m->did_assim()) {
dout(7) << "doing assim on " << *dir << dendl;
m->mark_assim(); // only do this the first time!
- import_pending_msg[dir->dirfrag()] = m;
// change import state
- import_state[dir->dirfrag()] = IMPORT_PREPPING;
- import_bound_ls[dir] = m->get_bounds();
+ it->second.state = IMPORT_PREPPING;
+ it->second.bound_ls = m->get_bounds();
+ it->second.bystanders = m->get_bystanders();
assert(g_conf->mds_kill_import_at != 3);
+ // bystander list
+ dout(7) << "bystanders are " << it->second.bystanders << dendl;
+
// move pin to dir
diri->put(CInode::PIN_IMPORTING);
dir->get(CDir::PIN_IMPORTING);
dir->state_set(CDir::STATE_IMPORTING);
-
- // bystander list
- import_bystanders[dir] = m->get_bystanders();
- dout(7) << "bystanders are " << import_bystanders[dir] << dendl;
// assimilate traces to exports
// each trace is: df ('-' | ('f' dir | 'd') dentry inode (dir dentry inode)*)
@@ -1873,11 +2042,11 @@ void Migrator::handle_export_prep(MExportDirPrep *m)
// ok!
dout(7) << " sending export_prep_ack on " << *dir << dendl;
- mds->send_message(new MExportDirPrepAck(dir->dirfrag()), m->get_connection());
+ mds->send_message(new MExportDirPrepAck(dir->dirfrag(), m->get_tid()), m->get_connection());
// note new state
- import_state[dir->dirfrag()] = IMPORT_PREPPED;
- import_pending_msg.erase(dir->dirfrag());
+ it->second.state = IMPORT_PREPPED;
+
assert(g_conf->mds_kill_import_at != 4);
// done
m->put();
@@ -1911,6 +2080,11 @@ void Migrator::handle_export_dir(MExportDir *m)
CDir *dir = cache->get_dirfrag(m->dirfrag);
assert(dir);
+ map<dirfrag_t,import_state_t>::iterator it = import_state.find(m->dirfrag);
+ assert(it != import_state.end());
+ assert(it->second.state == IMPORT_PREPPED);
+ assert(it->second.tid == m->get_tid());
+
utime_t now = ceph_clock_now(g_ceph_context);
int oldauth = m->get_source().num();
dout(7) << "handle_export_dir importing " << *dir << " from " << oldauth << dendl;
@@ -1946,8 +2120,8 @@ void Migrator::handle_export_dir(MExportDir *m)
dir, // import root
le,
mds->mdlog->get_current_segment(),
- import_caps[dir],
- import_updated_scatterlocks[dir],
+ it->second.peer_exports,
+ it->second.updated_scatterlocks,
now);
}
dout(10) << " " << m->bounds.size() << " imported bounds" << dendl;
@@ -1966,7 +2140,7 @@ void Migrator::handle_export_dir(MExportDir *m)
dout(7) << "handle_export_dir did " << *dir << dendl;
// note state
- import_state[dir->dirfrag()] = IMPORT_LOGGINGSTART;
+ it->second.state = IMPORT_LOGGINGSTART;
assert (g_conf->mds_kill_import_at != 6);
// log it
@@ -1990,14 +2164,15 @@ void Migrator::handle_export_dir(MExportDir *m)
*/
void Migrator::import_remove_pins(CDir *dir, set<CDir*>& bounds)
{
+ import_state_t& stat = import_state[dir->dirfrag()];
// root
dir->put(CDir::PIN_IMPORTING);
dir->state_clear(CDir::STATE_IMPORTING);
// bounding inodes
set<inodeno_t> did;
- for (list<dirfrag_t>::iterator p = import_bound_ls[dir].begin();
- p != import_bound_ls[dir].end();
+ for (list<dirfrag_t>::iterator p = stat.bound_ls.begin();
+ p != stat.bound_ls.end();
++p) {
if (did.count(p->ino))
continue;
@@ -2007,7 +2182,7 @@ void Migrator::import_remove_pins(CDir *dir, set<CDir*>& bounds)
in->put_stickydirs();
}
- if (import_state[dir->dirfrag()] >= IMPORT_PREPPED) {
+ if (stat.state >= IMPORT_PREPPED) {
// bounding dirfrags
for (set<CDir*>::iterator it = bounds.begin();
it != bounds.end();
@@ -2030,6 +2205,8 @@ void Migrator::import_reverse(CDir *dir)
{
dout(7) << "import_reverse " << *dir << dendl;
+ import_state_t& stat = import_state[dir->dirfrag()];
+
set<CDir*> bounds;
cache->get_subtree_bounds(dir, bounds);
@@ -2040,7 +2217,8 @@ void Migrator::import_reverse(CDir *dir)
assert(dir->is_subtree_root());
if (mds->is_resolve())
cache->trim_non_auth_subtree(dir);
- cache->adjust_subtree_auth(dir, import_peer[dir->dirfrag()]);
+
+ cache->adjust_subtree_auth(dir, stat.peer);
if (!dir->get_inode()->is_auth() &&
!dir->get_inode()->has_subtree_root_dirfrag(mds->get_nodeid()))
@@ -2099,20 +2277,28 @@ void Migrator::import_reverse(CDir *dir)
}
}
- // reexport caps
- for (map<CInode*, map<client_t,Capability::Export> >::iterator p = import_caps[dir].begin();
- p != import_caps[dir].end();
- ++p) {
- CInode *in = p->first;
- dout(20) << " reexporting caps on " << *in << dendl;
- /*
- * bleh.. just export all caps for this inode. the auth mds
- * will pick them up during recovery.
- */
- bufferlist bl; // throw this away
- map<client_t,entity_inst_t> exported_client_map; // throw this away too
- encode_export_inode_caps(in, bl, exported_client_map);
- finish_export_inode_caps(in);
+ if (stat.state == IMPORT_ACKING) {
+ // remove imported caps
+ for (map<CInode*,map<client_t,Capability::Export> >::iterator p = stat.peer_exports.begin();
+ p != stat.peer_exports.end();
+ ++p) {
+ CInode *in = p->first;
+ for (map<client_t,Capability::Export>::iterator q = p->second.begin();
+ q != p->second.end();
+ ++q) {
+ Capability *cap = in->get_client_cap(q->first);
+ assert(cap);
+ if (cap->is_new())
+ in->remove_client_cap(q->first);
+ }
+ }
+ for (map<client_t,entity_inst_t>::iterator p = stat.client_map.begin();
+ p != stat.client_map.end();
+ ++p) {
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->first.v));
+ assert(session);
+ session->dec_importing();
+ }
}
// log our failure
@@ -2121,14 +2307,14 @@ void Migrator::import_reverse(CDir *dir)
cache->try_subtree_merge(dir); // NOTE: this may journal subtree map as side effect
// bystanders?
- if (import_bystanders[dir].empty()) {
+ if (stat.bystanders.empty()) {
dout(7) << "no bystanders, finishing reverse now" << dendl;
import_reverse_unfreeze(dir);
} else {
// notify them; wait in aborting state
dout(7) << "notifying bystanders of abort" << dendl;
import_notify_abort(dir, bounds);
- import_state[dir->dirfrag()] = IMPORT_ABORTING;
+ stat.state = IMPORT_ABORTING;
assert (g_conf->mds_kill_import_at != 10);
}
}
@@ -2137,12 +2323,13 @@ void Migrator::import_notify_finish(CDir *dir, set<CDir*>& bounds)
{
dout(7) << "import_notify_finish " << *dir << dendl;
- for (set<int>::iterator p = import_bystanders[dir].begin();
- p != import_bystanders[dir].end();
+ import_state_t& stat = import_state[dir->dirfrag()];
+ for (set<int>::iterator p = stat.bystanders.begin();
+ p != stat.bystanders.end();
++p) {
MExportDirNotify *notify =
- new MExportDirNotify(dir->dirfrag(), false,
- pair<int,int>(import_peer[dir->dirfrag()], mds->get_nodeid()),
+ new MExportDirNotify(dir->dirfrag(), stat.tid, false,
+ pair<int,int>(stat.peer, mds->get_nodeid()),
pair<int,int>(mds->get_nodeid(), CDIR_AUTH_UNKNOWN));
for (set<CDir*>::iterator i = bounds.begin(); i != bounds.end(); ++i)
notify->get_bounds().push_back((*i)->dirfrag());
@@ -2154,13 +2341,14 @@ void Migrator::import_notify_abort(CDir *dir, set<CDir*>& bounds)
{
dout(7) << "import_notify_abort " << *dir << dendl;
- for (set<int>::iterator p = import_bystanders[dir].begin();
- p != import_bystanders[dir].end();
+ import_state_t& stat = import_state[dir->dirfrag()];
+ for (set<int>::iterator p = stat.bystanders.begin();
+ p != stat.bystanders.end();
++p) {
MExportDirNotify *notify =
- new MExportDirNotify(dir->dirfrag(), true,
- pair<int,int>(import_peer[dir->dirfrag()], mds->get_nodeid()),
- pair<int,int>(import_peer[dir->dirfrag()], CDIR_AUTH_UNKNOWN));
+ new MExportDirNotify(dir->dirfrag(), stat.tid, true,
+ pair<int,int>(stat.peer, mds->get_nodeid()),
+ pair<int,int>(stat.peer, CDIR_AUTH_UNKNOWN));
for (set<CDir*>::iterator i = bounds.begin(); i != bounds.end(); ++i)
notify->get_bounds().push_back((*i)->dirfrag());
mds->send_message_mds(notify, *p);
@@ -2184,11 +2372,6 @@ void Migrator::import_reverse_final(CDir *dir)
// clean up
import_state.erase(dir->dirfrag());
- import_peer.erase(dir->dirfrag());
- import_bystanders.erase(dir);
- import_bound_ls.erase(dir);
- import_updated_scatterlocks.erase(dir);
- import_caps.erase(dir);
// send pending import_maps?
mds->mdcache->maybe_send_pending_resolves();
@@ -2204,26 +2387,31 @@ void Migrator::import_logged_start(dirfrag_t df, CDir *dir, int from,
map<client_t,entity_inst_t>& imported_client_map,
map<client_t,uint64_t>& sseqmap)
{
- if (import_state.count(df) == 0 ||
- import_state[df] != IMPORT_LOGGINGSTART) {
+ map<dirfrag_t, import_state_t>::iterator it = import_state.find(dir->dirfrag());
+ if (it == import_state.end() ||
+ it->second.state != IMPORT_LOGGINGSTART) {
dout(7) << "import " << df << " must have aborted" << dendl;
+ mds->server->finish_force_open_sessions(imported_client_map, sseqmap);
return;
}
dout(7) << "import_logged " << *dir << dendl;
// note state
- import_state[dir->dirfrag()] = IMPORT_ACKING;
+ it->second.state = IMPORT_ACKING;
assert (g_conf->mds_kill_import_at != 7);
// force open client sessions and finish cap import
- mds->server->finish_force_open_sessions(imported_client_map, sseqmap);
+ mds->server->finish_force_open_sessions(imported_client_map, sseqmap, false);
+ it->second.client_map.swap(imported_client_map);
- for (map<CInode*, map<client_t,Capability::Export> >::iterator p = import_caps[dir].begin();
- p != import_caps[dir].end();
+ map<inodeno_t,map<client_t,Capability::Import> > imported_caps;
+ for (map<CInode*, map<client_t,Capability::Export> >::iterator p = it->second.peer_exports.begin();
+ p != it->second.peer_exports.end();
++p) {
- finish_import_inode_caps(p->first, true, p->second);
+ // parameter 'peer' is -1, delay sending cap import messages to client
+ finish_import_inode_caps(p->first, -1, true, p->second, imported_caps[p->first->ino()]);
}
// send notify's etc.
@@ -2232,7 +2420,10 @@ void Migrator::import_logged_start(dirfrag_t df, CDir *dir, int from,
// test surviving observer of a failed migration that did not complete
//assert(dir->replica_map.size() < 2 || mds->whoami != 0);
- mds->send_message_mds(new MExportDirAck(dir->dirfrag()), from);
+ MExportDirAck *ack = new MExportDirAck(dir->dirfrag(), it->second.tid);
+ ::encode(imported_caps, ack->imported_caps);
+
+ mds->send_message_mds(ack, from);
assert (g_conf->mds_kill_import_at != 8);
cache->show_subtrees();
@@ -2243,18 +2434,63 @@ void Migrator::handle_export_finish(MExportDirFinish *m)
{
CDir *dir = cache->get_dirfrag(m->get_dirfrag());
assert(dir);
- dout(7) << "handle_export_finish on " << *dir << dendl;
- import_finish(dir, false);
+ dout(7) << "handle_export_finish on " << *dir << (m->is_last() ? " last" : "") << dendl;
+
+ map<dirfrag_t,import_state_t>::iterator it = import_state.find(m->get_dirfrag());
+ assert(it != import_state.end());
+ assert(it->second.tid == m->get_tid());
+
+ import_finish(dir, false, m->is_last());
+
m->put();
}
-void Migrator::import_finish(CDir *dir, bool notify)
+void Migrator::import_finish(CDir *dir, bool notify, bool last)
{
dout(7) << "import_finish on " << *dir << dendl;
+ map<dirfrag_t,import_state_t>::iterator it = import_state.find(dir->dirfrag());
+ assert(it != import_state.end());
+ assert(it->second.state == IMPORT_ACKING || it->second.state == IMPORT_FINISHING);
+
// log finish
assert(g_conf->mds_kill_import_at != 9);
+ if (it->second.state == IMPORT_ACKING) {
+ for (map<CInode*, map<client_t,Capability::Export> >::iterator p = it->second.peer_exports.begin();
+ p != it->second.peer_exports.end();
+ ++p) {
+ CInode *in = p->first;
+ assert(in->is_auth());
+ for (map<client_t,Capability::Export>::iterator q = p->second.begin();
+ q != p->second.end();
+ ++q) {
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+ assert(session);
+ Capability *cap = in->get_client_cap(q->first);
+ assert(cap);
+ cap->clear_new();
+ cap->merge(q->second, true);
+ mds->mdcache->do_cap_import(session, in, cap, q->second.cap_id, q->second.seq,
+ q->second.mseq - 1, it->second.peer, CEPH_CAP_FLAG_AUTH);
+ }
+ p->second.clear();
+ }
+ for (map<client_t,entity_inst_t>::iterator p = it->second.client_map.begin();
+ p != it->second.client_map.end();
+ ++p) {
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->first.v));
+ assert(session);
+ session->dec_importing();
+ }
+ }
+
+ if (!last) {
+ assert(it->second.state == IMPORT_ACKING);
+ it->second.state = IMPORT_FINISHING;
+ return;
+ }
+
// clear updated scatterlocks
/*
for (list<ScatterLock*>::iterator p = import_updated_scatterlocks[dir].begin();
@@ -2272,16 +2508,11 @@ void Migrator::import_finish(CDir *dir, bool notify)
import_remove_pins(dir, bounds);
- map<CInode*, map<client_t,Capability::Export> > cap_imports;
- import_caps[dir].swap(cap_imports);
+ map<CInode*, map<client_t,Capability::Export> > peer_exports;
+ it->second.peer_exports.swap(peer_exports);
// clear import state (we're done!)
- import_state.erase(dir->dirfrag());
- import_peer.erase(dir->dirfrag());
- import_bystanders.erase(dir);
- import_bound_ls.erase(dir);
- import_caps.erase(dir);
- import_updated_scatterlocks.erase(dir);
+ import_state.erase(it);
mds->mdlog->start_submit_entry(new EImportFinish(dir, true));
@@ -2301,8 +2532,8 @@ void Migrator::import_finish(CDir *dir, bool notify)
mds->queue_waiters(ls);
// re-eval imported caps
- for (map<CInode*, map<client_t,Capability::Export> >::iterator p = cap_imports.begin();
- p != cap_imports.end();
+ for (map<CInode*, map<client_t,Capability::Export> >::iterator p = peer_exports.begin();
+ p != peer_exports.end();
++p)
if (p->first->is_auth())
mds->locker->eval(p->first, CEPH_CAP_LOCKS, true);
@@ -2325,7 +2556,7 @@ void Migrator::import_finish(CDir *dir, bool notify)
void Migrator::decode_import_inode(CDentry *dn, bufferlist::iterator& blp, int oldauth,
LogSegment *ls, uint64_t log_offset,
- map<CInode*, map<client_t,Capability::Export> >& cap_imports,
+ map<CInode*, map<client_t,Capability::Export> >& peer_exports,
list<ScatterLock*>& updated_scatterlocks)
{
dout(15) << "decode_import_inode on " << *dn << dendl;
@@ -2351,7 +2582,7 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::iterator& blp, int o
in->last_journaled = log_offset;
// caps
- decode_import_inode_caps(in, blp, cap_imports);
+ decode_import_inode_caps(in, true, blp, peer_exports);
// link before state -- or not! -sage
if (dn->get_linkage()->get_inode() != in) {
@@ -2385,24 +2616,27 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::iterator& blp, int o
}
-void Migrator::decode_import_inode_caps(CInode *in,
+void Migrator::decode_import_inode_caps(CInode *in, bool auth_cap,
bufferlist::iterator &blp,
- map<CInode*, map<client_t,Capability::Export> >& cap_imports)
+ map<CInode*, map<client_t,Capability::Export> >& peer_exports)
{
map<client_t,Capability::Export> cap_map;
::decode(cap_map, blp);
- ::decode(in->get_mds_caps_wanted(), blp);
- if (!cap_map.empty() || !in->get_mds_caps_wanted().empty()) {
- cap_imports[in].swap(cap_map);
+ if (auth_cap)
+ ::decode(in->get_mds_caps_wanted(), blp);
+ if (!cap_map.empty() ||
+ (auth_cap && !in->get_mds_caps_wanted().empty())) {
+ peer_exports[in].swap(cap_map);
in->get(CInode::PIN_IMPORTINGCAPS);
}
}
-void Migrator::finish_import_inode_caps(CInode *in, bool auth_cap,
- map<client_t,Capability::Export> &cap_map)
+void Migrator::finish_import_inode_caps(CInode *in, int peer, bool auth_cap,
+ map<client_t,Capability::Export> &export_map,
+ map<client_t,Capability::Import> &import_map)
{
- for (map<client_t,Capability::Export>::iterator it = cap_map.begin();
- it != cap_map.end();
+ for (map<client_t,Capability::Export>::iterator it = export_map.begin();
+ it != export_map.end();
++it) {
dout(10) << "finish_import_inode_caps for client." << it->first << " on " << *in << dendl;
Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(it->first.v));
@@ -2411,10 +2645,21 @@ void Migrator::finish_import_inode_caps(CInode *in, bool auth_cap,
Capability *cap = in->get_client_cap(it->first);
if (!cap) {
cap = in->add_client_cap(it->first, session);
+ if (peer < 0)
+ cap->mark_new();
}
- cap->merge(it->second, auth_cap);
- mds->mdcache->do_cap_import(session, in, cap);
+ Capability::Import& im = import_map[it->first];
+ im.cap_id = cap->get_cap_id();
+ im.mseq = auth_cap ? it->second.mseq : cap->get_mseq();
+ im.issue_seq = cap->get_last_seq() + 1;
+
+ if (peer >= 0) {
+ cap->merge(it->second, auth_cap);
+ mds->mdcache->do_cap_import(session, in, cap, it->second.cap_id,
+ it->second.seq, it->second.mseq - 1, peer,
+ auth_cap ? CEPH_CAP_FLAG_AUTH : CEPH_CAP_FLAG_RELEASE);
+ }
}
in->replica_caps_wanted = 0;
@@ -2426,7 +2671,7 @@ int Migrator::decode_import_dir(bufferlist::iterator& blp,
CDir *import_root,
EImportStart *le,
LogSegment *ls,
- map<CInode*, map<client_t,Capability::Export> >& cap_imports,
+ map<CInode*,map<client_t,Capability::Export> >& peer_exports,
list<ScatterLock*>& updated_scatterlocks, utime_t now)
{
// set up dir
@@ -2495,6 +2740,10 @@ int Migrator::decode_import_dir(bufferlist::iterator& blp,
if (dn->is_replica(mds->get_nodeid()))
dn->remove_replica(mds->get_nodeid());
+ // dentry lock in unreadable state can block path traverse
+ if (dn->lock.get_state() != LOCK_SYNC)
+ mds->locker->try_eval(&dn->lock, NULL);
+
dout(15) << "decode_import_dir got " << *dn << dendl;
// points to...
@@ -2522,7 +2771,7 @@ int Migrator::decode_import_dir(bufferlist::iterator& blp,
else if (icode == 'I') {
// inode
assert(le);
- decode_import_inode(dn, blp, oldauth, ls, le->get_start_off(), cap_imports, updated_scatterlocks);
+ decode_import_inode(dn, blp, oldauth, ls, le->get_start_off(), peer_exports, updated_scatterlocks);
}
// add dentry to journal entry
@@ -2575,7 +2824,7 @@ void Migrator::handle_export_notify(MExportDirNotify *m)
// send ack
if (m->wants_ack()) {
- mds->send_message_mds(new MExportDirNotifyAck(m->get_dirfrag()), from);
+ mds->send_message_mds(new MExportDirNotifyAck(m->get_dirfrag(), m->get_tid()), from);
} else {
// aborted. no ack.
dout(7) << "handle_export_notify no ack requested" << dendl;
@@ -2584,17 +2833,7 @@ void Migrator::handle_export_notify(MExportDirNotify *m)
m->put();
}
-
-
-
-
-
-
-
/** cap exports **/
-
-
-
void Migrator::export_caps(CInode *in)
{
int dest = in->authority().first;
@@ -2608,37 +2847,23 @@ void Migrator::export_caps(CInode *in)
MExportCaps *ex = new MExportCaps;
ex->ino = in->ino();
- encode_export_inode_caps(in, ex->cap_bl, ex->client_map);
+ encode_export_inode_caps(in, false, ex->cap_bl, ex->client_map);
mds->send_message_mds(ex, dest);
}
-/* This function DOES put the passed message before returning*/
-void Migrator::handle_export_caps_ack(MExportCapsAck *ack)
-{
- CInode *in = cache->get_inode(ack->ino);
- assert(in);
- dout(10) << "handle_export_caps_ack " << *ack << " from " << ack->get_source()
- << " on " << *in
- << dendl;
-
- finish_export_inode_caps(in);
- ack->put();
-}
-
-
class C_M_LoggedImportCaps : public Context {
Migrator *migrator;
CInode *in;
int from;
public:
- map<CInode*, map<client_t,Capability::Export> > cap_imports;
+ map<CInode*, map<client_t,Capability::Export> > peer_exports;
map<client_t,entity_inst_t> client_map;
map<client_t,uint64_t> sseqmap;
C_M_LoggedImportCaps(Migrator *m, CInode *i, int f) : migrator(m), in(i), from(f) {}
void finish(int r) {
- migrator->logged_import_caps(in, from, cap_imports, client_map, sseqmap);
+ migrator->logged_import_caps(in, from, peer_exports, client_map, sseqmap);
}
};
@@ -2660,8 +2885,8 @@ void Migrator::handle_export_caps(MExportCaps *ex)
// decode new caps
bufferlist::iterator blp = ex->cap_bl.begin();
- decode_import_inode_caps(in, blp, finish->cap_imports);
- assert(!finish->cap_imports.empty()); // thus, inode is pinned.
+ decode_import_inode_caps(in, false, blp, finish->peer_exports);
+ assert(!finish->peer_exports.empty()); // thus, inode is pinned.
// journal open client sessions
version_t pv = mds->server->prepare_force_open_sessions(finish->client_map, finish->sseqmap);
@@ -2678,7 +2903,7 @@ void Migrator::handle_export_caps(MExportCaps *ex)
void Migrator::logged_import_caps(CInode *in,
int from,
- map<CInode*, map<client_t,Capability::Export> >& cap_imports,
+ map<CInode*, map<client_t,Capability::Export> >& peer_exports,
map<client_t,entity_inst_t>& client_map,
map<client_t,uint64_t>& sseqmap)
{
@@ -2687,10 +2912,10 @@ void Migrator::logged_import_caps(CInode *in,
// force open client sessions and finish cap import
mds->server->finish_force_open_sessions(client_map, sseqmap);
- assert(cap_imports.count(in));
- finish_import_inode_caps(in, false, cap_imports[in]);
- mds->locker->eval(in, CEPH_CAP_LOCKS, true);
+ map<client_t,Capability::Import> imported_caps;
- mds->send_message_mds(new MExportCapsAck(in->ino()), from);
+ assert(peer_exports.count(in));
+ // clients will release caps from the exporter when they receive the cap import message.
+ finish_import_inode_caps(in, from, false, peer_exports[in], imported_caps);
+ mds->locker->eval(in, CEPH_CAP_LOCKS, true);
}
-
diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h
index afe2e6c..39542a3 100644
--- a/src/mds/Migrator.h
+++ b/src/mds/Migrator.h
@@ -54,9 +54,11 @@ private:
MDS *mds;
MDCache *cache;
+ uint64_t last_export_tid;
// -- exports --
public:
// export stages. used to clean up intelligently if there's a failure.
+ const static int EXPORT_CANCELLED = 0; // cancelled
const static int EXPORT_DISCOVERING = 1; // dest is disovering export dir
const static int EXPORT_FREEZING = 2; // we're freezing the dir tree
const static int EXPORT_PREPPING = 3; // sending dest spanning tree to export bounds
@@ -79,17 +81,29 @@ public:
protected:
// export fun
- map<CDir*,int> export_state;
- map<CDir*,int> export_peer;
- map<CDir*,set<SimpleLock*> > export_locks;
-
- //map<CDir*,list<bufferlist> > export_data; // only during EXPORTING state
- map<CDir*,set<int> > export_warning_ack_waiting;
- map<CDir*,set<int> > export_notify_ack_waiting;
-
- map<CDir*,list<Context*> > export_finish_waiters;
+ struct export_state_t {
+ int state;
+ int peer;
+ uint64_t tid;
+ set<SimpleLock*> locks;
+ set<int> warning_ack_waiting;
+ set<int> notify_ack_waiting;
+ map<inodeno_t,map<client_t,Capability::Import> > peer_imported;
+ list<Context*> waiting_for_finish;
+ };
+
+ map<CDir*, export_state_t> export_state;
- list< pair<dirfrag_t,int> > export_queue;
+ list<pair<dirfrag_t,int> > export_queue;
+
+ // for deadlock detection
+ struct freezing_state_t {
+ utime_t start_time;
+ int num_waiters; // number of remote authpin waiters
+ freezing_state_t() : num_waiters(0) {}
+ };
+ map<CDir*,freezing_state_t > export_freezing_state;
+ set<pair<utime_t,CDir*> > export_freezing_dirs;
// -- imports --
public:
@@ -99,7 +113,8 @@ public:
const static int IMPORT_PREPPED = 4; // opened bounds, waiting for import
const static int IMPORT_LOGGINGSTART = 5; // got import, logging EImportStart
const static int IMPORT_ACKING = 6; // logged EImportStart, sent ack, waiting for finish
- const static int IMPORT_ABORTING = 7; // notifying bystanders of an abort before unfreezing
+ const static int IMPORT_FINISHING = 7; // sent cap imports, waiting for finish
+ const static int IMPORT_ABORTING = 8; // notifying bystanders of an abort before unfreezing
static const char *get_import_statename(int s) {
switch (s) {
case IMPORT_DISCOVERING: return "discovering";
@@ -108,24 +123,29 @@ public:
case IMPORT_PREPPED: return "prepped";
case IMPORT_LOGGINGSTART: return "loggingstart";
case IMPORT_ACKING: return "acking";
+ case IMPORT_FINISHING: return "finishing";
case IMPORT_ABORTING: return "aborting";
default: assert(0); return 0;
}
}
protected:
- map<dirfrag_t,int> import_state; // FIXME make these dirfrags
- map<dirfrag_t,int> import_peer;
- map<dirfrag_t,Message*> import_pending_msg;
- map<CDir*,set<int> > import_bystanders;
- map<CDir*,list<dirfrag_t> > import_bound_ls;
- map<CDir*,list<ScatterLock*> > import_updated_scatterlocks;
- map<CDir*, map<CInode*, map<client_t,Capability::Export> > > import_caps;
-
+ struct import_state_t {
+ int state;
+ int peer;
+ uint64_t tid;
+ set<int> bystanders;
+ list<dirfrag_t> bound_ls;
+ list<ScatterLock*> updated_scatterlocks;
+ map<client_t,entity_inst_t> client_map;
+ map<CInode*, map<client_t,Capability::Export> > peer_exports;
+ };
+
+ map<dirfrag_t, import_state_t> import_state;
public:
// -- cons --
- Migrator(MDS *m, MDCache *c) : mds(m), cache(c) {}
+ Migrator(MDS *m, MDCache *c) : mds(m), cache(c), last_export_tid(0) {}
void dispatch(Message*);
@@ -134,54 +154,68 @@ public:
// -- status --
int is_exporting(CDir *dir) {
- if (export_state.count(dir)) return export_state[dir];
+ map<CDir*, export_state_t>::iterator it = export_state.find(dir);
+ if (it != export_state.end()) return it->second.state;
return 0;
}
bool is_exporting() { return !export_state.empty(); }
int is_importing(dirfrag_t df) {
- if (import_state.count(df)) return import_state[df];
+ map<dirfrag_t, import_state_t>::iterator it = import_state.find(df);
+ if (it != import_state.end()) return it->second.state;
return 0;
}
bool is_importing() { return !import_state.empty(); }
bool is_ambiguous_import(dirfrag_t df) {
- map<dirfrag_t,int>::iterator p = import_state.find(df);
+ map<dirfrag_t, import_state_t>::iterator p = import_state.find(df);
if (p == import_state.end())
return false;
- if (p->second >= IMPORT_LOGGINGSTART &&
- p->second < IMPORT_ABORTING)
+ if (p->second.state >= IMPORT_LOGGINGSTART &&
+ p->second.state < IMPORT_ABORTING)
return true;
return false;
}
int get_import_state(dirfrag_t df) {
- assert(import_state.count(df));
- return import_state[df];
+ map<dirfrag_t, import_state_t>::iterator it = import_state.find(df);
+ assert(it != import_state.end());
+ return it->second.state;
}
int get_import_peer(dirfrag_t df) {
- assert(import_peer.count(df));
- return import_peer[df];
+ map<dirfrag_t, import_state_t>::iterator it = import_state.find(df);
+ assert(it != import_state.end());
+ return it->second.peer;
}
int get_export_state(CDir *dir) {
- assert(export_state.count(dir));
- return export_state[dir];
+ map<CDir*, export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ return it->second.state;
}
// this returns true if we are export @dir,
// and are not waiting for @who to be
// be warned of ambiguous auth.
// only returns meaningful results during EXPORT_WARNING state.
bool export_has_warned(CDir *dir, int who) {
- assert(is_exporting(dir));
- assert(export_state[dir] == EXPORT_WARNING);
- return (export_warning_ack_waiting[dir].count(who) == 0);
+ map<CDir*, export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ assert(it->second.state == EXPORT_WARNING);
+ return (it->second.warning_ack_waiting.count(who) == 0);
}
bool export_has_notified(CDir *dir, int who) {
+ map<CDir*, export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ assert(it->second.state == EXPORT_NOTIFYING);
+ return (it->second.notify_ack_waiting.count(who) == 0);
+ }
+
+ void export_freeze_inc_num_waiters(CDir *dir) {
assert(is_exporting(dir));
- assert(export_state[dir] == EXPORT_NOTIFYING);
- return (export_notify_ack_waiting[dir].count(who) == 0);
+ export_freezing_state[dir].num_waiters++;
}
+ void find_stale_export_freeze();
+
// -- misc --
void handle_mds_failure_or_stop(int who);
@@ -200,22 +234,32 @@ public:
}
void get_export_lock_set(CDir *dir, set<SimpleLock*>& locks);
+ void get_export_client_set(CDir *dir, set<client_t> &client_set);
+ void get_export_client_set(CInode *in, set<client_t> &client_set);
void encode_export_inode(CInode *in, bufferlist& bl,
map<client_t,entity_inst_t>& exported_client_map);
- void encode_export_inode_caps(CInode *in, bufferlist& bl,
+ void encode_export_inode_caps(CInode *in, bool auth_cap, bufferlist& bl,
map<client_t,entity_inst_t>& exported_client_map);
- void finish_export_inode(CInode *in, utime_t now, list<Context*>& finished);
- void finish_export_inode_caps(CInode *in);
+ void finish_export_inode(CInode *in, utime_t now, int target,
+ map<client_t,Capability::Import>& peer_imported,
+ list<Context*>& finished);
+ void finish_export_inode_caps(CInode *in, int target,
+ map<client_t,Capability::Import>& peer_imported);
+
int encode_export_dir(bufferlist& exportbl,
CDir *dir,
map<client_t,entity_inst_t>& exported_client_map,
utime_t now);
- void finish_export_dir(CDir *dir, list<Context*>& finished, utime_t now);
+ void finish_export_dir(CDir *dir, utime_t now, int target,
+ map<inodeno_t,map<client_t,Capability::Import> >& peer_imported,
+ list<Context*>& finished);
void add_export_finish_waiter(CDir *dir, Context *c) {
- export_finish_waiters[dir].push_back(c);
+ map<CDir*, export_state_t>::iterator it = export_state.find(dir);
+ assert(it != export_state.end());
+ it->second.waiting_for_finish.push_back(c);
}
void clear_export_proxy_pins(CDir *dir);
@@ -225,8 +269,10 @@ public:
void handle_export_discover_ack(MExportDirDiscoverAck *m);
void export_frozen(CDir *dir);
void handle_export_prep_ack(MExportDirPrepAck *m);
+ void export_sessions_flushed(CDir *dir, uint64_t tid);
void export_go(CDir *dir);
- void export_go_synced(CDir *dir);
+ void export_go_synced(CDir *dir, uint64_t tid);
+ void export_try_cancel(CDir *dir);
void export_reverse(CDir *dir);
void export_notify_abort(CDir *dir, set<CDir*>& bounds);
void handle_export_ack(MExportDirAck *m);
@@ -235,12 +281,16 @@ public:
void export_unlock(CDir *dir);
void export_finish(CDir *dir);
- void handle_export_caps_ack(MExportCapsAck *m);
-
+ void export_freeze_finish(CDir *dir) {
+ utime_t start = export_freezing_state[dir].start_time;
+ export_freezing_dirs.erase(make_pair(start, dir));
+ export_freezing_state.erase(dir);
+ }
friend class C_MDC_ExportFreeze;
friend class C_MDS_ExportFinishLogged;
friend class C_M_ExportGo;
+ friend class C_M_ExportSessionsFlushed;
// importer
void handle_export_discover(MExportDirDiscover *m);
@@ -253,11 +303,11 @@ public:
LogSegment *ls, uint64_t log_offset,
map<CInode*, map<client_t,Capability::Export> >& cap_imports,
list<ScatterLock*>& updated_scatterlocks);
- void decode_import_inode_caps(CInode *in,
- bufferlist::iterator &blp,
+ void decode_import_inode_caps(CInode *in, bool auth_cap, bufferlist::iterator &blp,
map<CInode*, map<client_t,Capability::Export> >& cap_imports);
- void finish_import_inode_caps(CInode *in, bool auth_cap,
- map<client_t,Capability::Export> &cap_map);
+ void finish_import_inode_caps(CInode *in, int from, bool auth_cap,
+ map<client_t,Capability::Export> &export_map,
+ map<client_t,Capability::Import> &import_map);
int decode_import_dir(bufferlist::iterator& blp,
int oldauth,
CDir *import_root,
@@ -282,7 +332,7 @@ protected:
map<client_t,uint64_t>& sseqmap);
void handle_export_finish(MExportDirFinish *m);
public:
- void import_finish(CDir *dir, bool notify);
+ void import_finish(CDir *dir, bool notify, bool last=true);
protected:
void handle_export_caps(MExportCaps *m);
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index b50a03c..3d6963b 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -234,6 +234,8 @@ struct MDRequest : public Mutation {
Context *slave_commit;
bufferlist rollback_bl;
+ list<Context*> waiting_for_finish;
+
More() :
srcdn_auth_mds(-1),
src_reanchor_atid(0), dst_reanchor_atid(0), inode_import_v(0),
diff --git a/src/mds/Resetter.cc b/src/mds/Resetter.cc
index e968cdc..cbf8f89 100644
--- a/src/mds/Resetter.cc
+++ b/src/mds/Resetter.cc
@@ -57,7 +57,7 @@ bool Resetter::ms_dispatch(Message *m)
void Resetter::init(int rank)
{
inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
- unsigned pg_pool = CEPH_METADATA_RULE;
+ unsigned pg_pool = MDS_METADATA_POOL;
osdmap = new OSDMap();
objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer);
journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC,
@@ -79,9 +79,9 @@ void Resetter::init(int rank)
objecter->init_unlocked();
lock.Lock();
objecter->init_locked();
+ lock.Unlock();
objecter->wait_for_osd_map();
timer.init();
- lock.Unlock();
}
void Resetter::shutdown()
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 0c500cd..0e37727 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -256,12 +256,37 @@ void Server::handle_client_session(MClientSession *m)
}
break;
+ case CEPH_SESSION_FLUSHMSG_ACK:
+ finish_flush_session(session, m->get_seq());
+ break;
+
default:
assert(0);
}
m->put();
}
+void Server::flush_client_sessions(set<client_t>& client_set, C_GatherBuilder& gather)
+{
+ for (set<client_t>::iterator p = client_set.begin(); p != client_set.end(); ++p) {
+ Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->v));
+ assert(session);
+ if (session->is_stale() ||
+ !session->connection.get() ||
+ !session->connection->has_feature(CEPH_FEATURE_EXPORT_PEER))
+ continue;
+ version_t seq = session->wait_for_flush(gather.new_sub());
+ mds->send_message_client(new MClientSession(CEPH_SESSION_FLUSHMSG, seq), session);
+ }
+}
+
+void Server::finish_flush_session(Session *session, version_t seq)
+{
+ list<Context*> finished;
+ session->finish_flush(seq, finished);
+ mds->queue_waiters(finished);
+}
+
void Server::_session_logged(Session *session, uint64_t state_seq, bool open, version_t pv,
interval_set<inodeno_t>& inos, version_t piv)
{
@@ -343,13 +368,14 @@ version_t Server::prepare_force_open_sessions(map<client_t,entity_inst_t>& cm,
session->is_opening() ||
session->is_stale());
session->inc_importing();
- mds->sessionmap.touch_session(session);
+// mds->sessionmap.touch_session(session);
}
return pv;
}
void Server::finish_force_open_sessions(map<client_t,entity_inst_t>& cm,
- map<client_t,uint64_t>& sseqmap)
+ map<client_t,uint64_t>& sseqmap,
+ bool dec_import)
{
/*
* FIXME: need to carefully consider the race conditions between a
@@ -380,7 +406,8 @@ void Server::finish_force_open_sessions(map<client_t,entity_inst_t>& cm,
dout(10) << "force_open_sessions skipping already-open " << session->info.inst << dendl;
assert(session->is_open() || session->is_stale());
}
- session->dec_importing();
+ if (dec_import)
+ session->dec_importing();
}
mds->sessionmap.version++;
}
@@ -439,6 +466,7 @@ void Server::find_idle_sessions()
mds->locker->revoke_stale_caps(session);
mds->locker->remove_stale_leases(session);
mds->send_message_client(new MClientSession(CEPH_SESSION_STALE, session->get_push_seq()), session);
+ finish_flush_session(session, session->get_push_seq());
}
// autoclose
@@ -523,6 +551,8 @@ void Server::journal_close_session(Session *session, int state)
++p;
mdcache->request_kill(mdr);
}
+
+ finish_flush_session(session, session->get_push_seq());
}
void Server::reconnect_clients()
@@ -675,7 +705,9 @@ void Server::reconnect_tick()
p != client_reconnect_gather.end();
++p) {
Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->v));
+ assert(session);
dout(1) << "reconnect gave up on " << session->info.inst << dendl;
+ kill_session(session);
failed_reconnects++;
}
client_reconnect_gather.clear();
@@ -1118,10 +1150,6 @@ void Server::handle_client_request(MClientRequest *req)
session->trim_completed_requests(req->get_oldest_client_tid());
}
- // request_start may drop the request, get a reference for cap release
- if (!req->releases.empty() && req->get_source().is_client() && !req->is_replay())
- req->get();
-
// register + dispatch
MDRequest *mdr = mdcache->request_start(req);
if (mdr) {
@@ -1139,7 +1167,7 @@ void Server::handle_client_request(MClientRequest *req)
p != req->releases.end();
++p)
mds->locker->process_request_cap_release(mdr, client, p->item, p->dname);
- req->put();
+ req->releases.clear();
}
if (mdr)
@@ -1563,6 +1591,9 @@ void Server::dispatch_slave_request(MDRequest *mdr)
break;
case MMDSSlaveRequest::OP_FINISH:
+ // information about rename imported caps
+ if (mdr->slave_request->inode_export.length() > 0)
+ mdr->more()->inode_import.claim(mdr->slave_request->inode_export);
// finish off request.
mdcache->request_finish(mdr);
break;
@@ -1612,6 +1643,22 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
dout(10) << " waiting for authpinnable on " << **p << dendl;
(*p)->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
mdr->drop_local_auth_pins();
+
+ CDir *dir = NULL;
+ if (CInode *in = dynamic_cast<CInode*>(*p)) {
+ if (!in->is_root())
+ dir = in->get_parent_dir();
+ } else if (CDentry *dn = dynamic_cast<CDentry*>(*p)) {
+ dir = dn->get_dir();
+ } else {
+ assert(0);
+ }
+ if (dir && dir->is_freezing_tree()) {
+ while (!dir->is_freezing_tree_root())
+ dir = dir->get_parent_dir();
+ mdcache->migrator->export_freeze_inc_num_waiters(dir);
+ }
+
return;
}
}
@@ -1866,15 +1913,14 @@ CInode* Server::prepare_new_inode(MDRequest *mdr, CDir *dir, inodeno_t useino, u
in->inode.mode = mode;
memset(&in->inode.dir_layout, 0, sizeof(in->inode.dir_layout));
- if (in->inode.is_dir())
+ if (in->inode.is_dir()) {
in->inode.dir_layout.dl_dir_hash = g_conf->mds_default_dir_hash;
-
- if (layout)
- in->inode.layout = *layout;
- else if (in->inode.is_dir())
memset(&in->inode.layout, 0, sizeof(in->inode.layout));
- else
+ } else if (layout) {
+ in->inode.layout = *layout;
+ } else {
in->inode.layout = mds->mdcache->default_file_layout;
+ }
in->inode.truncate_size = -1ull; // not truncated, yet!
in->inode.truncate_seq = 1; /* starting with 1, 0 is kept for no-truncation logic */
@@ -1910,6 +1956,10 @@ CInode* Server::prepare_new_inode(MDRequest *mdr, CDir *dir, inodeno_t useino, u
}
}
+ if (!mds->mdsmap->get_inline_data_enabled() ||
+ !mdr->session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA))
+ in->inode.inline_version = CEPH_INLINE_NONE;
+
mdcache->add_inode(in); // add
dout(10) << "prepare_new_inode " << *in << dendl;
return in;
@@ -2423,6 +2473,13 @@ void Server::handle_client_open(MDRequest *mdr)
reply_request(mdr, -EINVAL);
return;
}
+
+ if (cur->inode.inline_version != CEPH_INLINE_NONE &&
+ !mdr->session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA)) {
+ dout(7) << "old client cannot open inline data file " << *cur << dendl;
+ reply_request(mdr, -EPERM);
+ return;
+ }
// snapped data is read only
if (mdr->snapid != CEPH_NOSNAP &&
@@ -2535,7 +2592,7 @@ public:
// dirty inode, dn, dir
newi->inode.version--; // a bit hacky, see C_MDS_mknod_finish
newi->mark_dirty(newi->inode.version+1, mdr->ls);
- newi->_mark_dirty_parent(mdr->ls);
+ newi->_mark_dirty_parent(mdr->ls, true);
mdr->apply();
@@ -2667,6 +2724,8 @@ void Server::handle_client_openc(MDRequest *mdr)
dn->push_projected_linkage(in);
in->inode.version = dn->pre_dirty();
+ if (layout.fl_pg_pool != mdcache->default_file_layout.fl_pg_pool)
+ in->inode.add_old_pool(mdcache->default_file_layout.fl_pg_pool);
in->inode.update_backtrace();
if (cmode & CEPH_FILE_MODE_WR) {
in->inode.client_ranges[client].range.first = 0;
@@ -2686,7 +2745,7 @@ void Server::handle_client_openc(MDRequest *mdr)
le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
journal_allocated_inos(mdr, &le->metablob);
mdcache->predirty_journal_parents(mdr, &le->metablob, in, dn->get_dir(), PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
- le->metablob.add_primary_dentry(dn, in, true, true);
+ le->metablob.add_primary_dentry(dn, in, true, true, true);
// do the open
mds->locker->issue_new_caps(in, cmode, mdr->session, realm, req->is_replay());
@@ -2785,11 +2844,6 @@ void Server::handle_client_readdir(MDRequest *mdr)
assert(snapid == CEPH_NOSNAP || snaps->count(snapid)); // just checkin'!
}
- // build dir contents
- bufferlist dnbl;
-
- CDir::map_t::iterator it = dir->begin();
-
unsigned max = req->head.args.readdir.max_entries;
if (!max)
max = dir->get_num_any(); // whatever, something big.
@@ -2807,8 +2861,13 @@ void Server::handle_client_readdir(MDRequest *mdr)
int bytes_left = max_bytes - front_bytes;
bytes_left -= realm->get_snap_trace().length();
+ // build dir contents
+ bufferlist dnbl;
__u32 numfiles = 0;
- while (it != dir->end() && numfiles < max) {
+ __u8 end = (dir->begin() == dir->end());
+ for (CDir::map_t::iterator it = dir->begin();
+ !end && numfiles < max;
+ end = (it == dir->end())) {
CDentry *dn = it->second;
++it;
@@ -2897,7 +2956,6 @@ void Server::handle_client_readdir(MDRequest *mdr)
mdcache->lru.lru_touch(dn);
}
- __u8 end = (it == dir->end());
__u8 complete = (end && offset_str.empty()); // FIXME: what purpose does this serve
// finish final blob
@@ -3206,6 +3264,11 @@ void Server::handle_client_setattr(MDRequest *mdr)
// log + wait
le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+
+ // update backtrace for old format inode. (see inode_t::decode)
+ if (pi->backtrace_version == 0)
+ pi->update_backtrace();
+
mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
mdcache->journal_dirty_inode(mdr, &le->metablob, cur);
@@ -3282,11 +3345,10 @@ void Server::handle_client_setlayout(MDRequest *mdr)
reply_request(mdr, -EROFS);
return;
}
- if (cur->is_dir()) {
- reply_request(mdr, -EISDIR);
+ if (!cur->is_file()) {
+ reply_request(mdr, -EINVAL);
return;
}
-
if (cur->get_projected_inode()->size ||
cur->get_projected_inode()->truncate_seq > 1) {
reply_request(mdr, -ENOTEMPTY);
@@ -3842,7 +3904,7 @@ public:
// a new version of hte inode since it's just been created)
newi->inode.version--;
newi->mark_dirty(newi->inode.version + 1, mdr->ls);
- newi->_mark_dirty_parent(mdr->ls);
+ newi->_mark_dirty_parent(mdr->ls, true);
// mkdir?
if (newi->inode.is_dir()) {
@@ -3889,34 +3951,32 @@ void Server::handle_client_mknod(MDRequest *mdr)
if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
return;
+ unsigned mode = req->head.args.mknod.mode;
+ if ((mode & S_IFMT) == 0)
+ mode |= S_IFREG;
+
// set layout
ceph_file_layout layout;
- if (dir_layout)
+ if (dir_layout && S_ISREG(mode))
layout = *dir_layout;
else
layout = mds->mdcache->default_file_layout;
- if (!ceph_file_layout_is_valid(&layout)) {
- dout(10) << " invalid initial file layout" << dendl;
- reply_request(mdr, -EINVAL);
- return;
- }
-
SnapRealm *realm = dn->get_dir()->inode->find_snaprealm();
snapid_t follows = realm->get_newest_seq();
mdr->now = ceph_clock_now(g_ceph_context);
CInode *newi = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino),
- req->head.args.mknod.mode, &layout);
+ mode, &layout);
assert(newi);
dn->push_projected_linkage(newi);
newi->inode.rdev = req->head.args.mknod.rdev;
- if ((newi->inode.mode & S_IFMT) == 0)
- newi->inode.mode |= S_IFREG;
newi->inode.version = dn->pre_dirty();
newi->inode.rstat.rfiles = 1;
+ if (layout.fl_pg_pool != mdcache->default_file_layout.fl_pg_pool)
+ newi->inode.add_old_pool(mdcache->default_file_layout.fl_pg_pool);
newi->inode.update_backtrace();
// if the client created a _regular_ file via MKNOD, it's highly likely they'll
@@ -3958,7 +4018,7 @@ void Server::handle_client_mknod(MDRequest *mdr)
mdcache->predirty_journal_parents(mdr, &le->metablob, newi, dn->get_dir(),
PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
- le->metablob.add_primary_dentry(dn, newi, true, true);
+ le->metablob.add_primary_dentry(dn, newi, true, true, true);
journal_and_reply(mdr, newi, dn, le, new C_MDS_mknod_finish(mds, mdr, dn, newi, follows));
}
@@ -5269,15 +5329,10 @@ bool Server::_dir_is_nonempty_unlocked(MDRequest *mdr, CInode *in)
if (in->snaprealm && in->snaprealm->srnode.snaps.size())
return true; // in a snapshot!
- list<frag_t> frags;
- in->dirfragtree.get_leaves(frags);
-
- for (list<frag_t>::iterator p = frags.begin();
- p != frags.end();
- ++p) {
- CDir *dir = in->get_or_open_dirfrag(mdcache, *p);
- assert(dir);
-
+ list<CDir*> ls;
+ in->get_dirfrags(ls);
+ for (list<CDir*>::iterator p = ls.begin(); p != ls.end(); ++p) {
+ CDir *dir = *p;
// is the frag obviously non-empty?
if (dir->is_auth()) {
if (dir->get_projected_fnode()->fragstat.size()) {
@@ -5295,19 +5350,29 @@ bool Server::_dir_is_nonempty(MDRequest *mdr, CInode *in)
{
dout(10) << "dir_is_nonempty " << *in << dendl;
assert(in->is_auth());
+ assert(in->filelock.can_read(-1));
- if (in->snaprealm && in->snaprealm->srnode.snaps.size())
- return true; // in a snapshot!
+ frag_info_t dirstat;
+ version_t dirstat_version = in->get_projected_inode()->dirstat.version;
- if (in->get_projected_inode()->dirstat.size() > 0) {
- dout(10) << "dir_is_nonempty projected dir size still "
- << in->get_projected_inode()->dirstat.size()
- << " on " << *in
- << dendl;
- return true;
+ list<CDir*> ls;
+ in->get_dirfrags(ls);
+ for (list<CDir*>::iterator p = ls.begin(); p != ls.end(); ++p) {
+ CDir *dir = *p;
+ fnode_t *pf = dir->get_projected_fnode();
+ if (pf->fragstat.size()) {
+ dout(10) << "dir_is_nonempty_unlocked dirstat has "
+ << pf->fragstat.size() << " items " << *dir << dendl;
+ return true;
+ }
+
+ if (pf->accounted_fragstat.version == dirstat_version)
+ dirstat.add(pf->accounted_fragstat);
+ else
+ dirstat.add(pf->fragstat);
}
- return false;
+ return dirstat.size() != in->get_projected_inode()->dirstat.size();
}
@@ -6283,13 +6348,21 @@ void Server::_rename_apply(MDRequest *mdr, CDentry *srcdn, CDentry *destdn, CDen
// srcdn inode import?
if (!srcdn->is_auth() && destdn->is_auth()) {
assert(mdr->more()->inode_import.length() > 0);
+
+ map<client_t,Capability::Import> imported_caps;
// finish cap imports
finish_force_open_sessions(mdr->more()->imported_client_map, mdr->more()->sseq_map);
if (mdr->more()->cap_imports.count(destdnl->get_inode())) {
- mds->mdcache->migrator->finish_import_inode_caps(destdnl->get_inode(), srcdn->authority().first,
- mdr->more()->cap_imports[destdnl->get_inode()]);
+ mds->mdcache->migrator->finish_import_inode_caps(destdnl->get_inode(),
+ mdr->more()->srcdn_auth_mds, true,
+ mdr->more()->cap_imports[destdnl->get_inode()],
+ imported_caps);
}
+
+ mdr->more()->inode_import.clear();
+ ::encode(imported_caps, mdr->more()->inode_import);
+
/* hack: add an auth pin for each xlock we hold. These were
* remote xlocks previously but now they're local and
* we're going to try and unpin when we xlock_finish. */
@@ -6366,6 +6439,20 @@ public:
}
};
+class C_MDS_SlaveRenameSessionsFlushed : public Context {
+ Server *server;
+ MDRequest *mdr;
+public:
+ C_MDS_SlaveRenameSessionsFlushed(Server *s, MDRequest *r) :
+ server(s), mdr(r) {
+ mdr->get();
+ }
+ void finish(int r) {
+ server->_slave_rename_sessions_flushed(mdr);
+ mdr->put();
+ }
+};
+
/* This function DOES put the mdr->slave_request before returning*/
void Server::handle_slave_rename_prep(MDRequest *mdr)
{
@@ -6454,15 +6541,29 @@ void Server::handle_slave_rename_prep(MDRequest *mdr)
if (mdr->slave_request->witnesses.size() > 1) {
dout(10) << " set srci ambiguous auth; providing srcdn replica list" << dendl;
reply_witness = true;
- for (set<int>::iterator p = srcdnrep.begin(); p != srcdnrep.end(); ++p) {
- if (*p == mdr->slave_to_mds ||
- !mds->mdsmap->is_clientreplay_or_active_or_stopping(*p))
- continue;
- MMDSSlaveRequest *notify = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
- MMDSSlaveRequest::OP_RENAMENOTIFY);
- mds->send_message_mds(notify, *p);
- mdr->more()->waiting_on_slave.insert(*p);
- }
+ }
+
+ // make sure bystanders have received all lock related messages
+ for (set<int>::iterator p = srcdnrep.begin(); p != srcdnrep.end(); ++p) {
+ if (*p == mdr->slave_to_mds ||
+ !mds->mdsmap->is_clientreplay_or_active_or_stopping(*p))
+ continue;
+ MMDSSlaveRequest *notify = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+ MMDSSlaveRequest::OP_RENAMENOTIFY);
+ mds->send_message_mds(notify, *p);
+ mdr->more()->waiting_on_slave.insert(*p);
+ }
+
+ // make sure clients have received all cap related messages
+ set<client_t> export_client_set;
+ mdcache->migrator->get_export_client_set(srcdnl->get_inode(), export_client_set);
+
+ C_GatherBuilder gather(g_ceph_context);
+ flush_client_sessions(export_client_set, gather);
+ if (gather.has_subs()) {
+ mdr->more()->waiting_on_slave.insert(-1);
+ gather.set_finisher(new C_MDS_SlaveRenameSessionsFlushed(this, mdr));
+ gather.activate();
}
}
@@ -6643,8 +6744,13 @@ void Server::_commit_slave_rename(MDRequest *mdr, int r,
mds->locker->xlock_export(lock, mdr);
}
+ map<client_t,Capability::Import> peer_imported;
+ bufferlist::iterator bp = mdr->more()->inode_import.begin();
+ ::decode(peer_imported, bp);
+
dout(10) << " finishing inode export on " << *destdnl->get_inode() << dendl;
- mdcache->migrator->finish_export_inode(destdnl->get_inode(), mdr->now, finished);
+ mdcache->migrator->finish_export_inode(destdnl->get_inode(), mdr->now,
+ mdr->slave_to_mds, peer_imported, finished);
mds->queue_waiters(finished); // this includes SINGLEAUTH waiters.
// unfreeze
@@ -7078,7 +7184,21 @@ void Server::handle_slave_rename_notify_ack(MDRequest *mdr, MMDSSlaveRequest *ac
}
}
+void Server::_slave_rename_sessions_flushed(MDRequest *mdr)
+{
+ dout(10) << "_slave_rename_sessions_flushed " << *mdr << dendl;
+
+ if (mdr->more()->waiting_on_slave.count(-1)) {
+ mdr->more()->waiting_on_slave.erase(-1);
+ if (mdr->more()->waiting_on_slave.empty()) {
+ if (mdr->slave_request)
+ dispatch_slave_request(mdr);
+ } else
+ dout(10) << " still waiting for rename notify acks from "
+ << mdr->more()->waiting_on_slave << dendl;
+ }
+}
// snaps
/* This function takes responsibility for the passed mdr*/
diff --git a/src/mds/Server.h b/src/mds/Server.h
index cf1ef5b..d7253f1 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -78,7 +78,10 @@ public:
version_t prepare_force_open_sessions(map<client_t,entity_inst_t> &cm,
map<client_t,uint64_t>& sseqmap);
void finish_force_open_sessions(map<client_t,entity_inst_t> &cm,
- map<client_t,uint64_t>& sseqmap);
+ map<client_t,uint64_t>& sseqmap,
+ bool dec_import=true);
+ void flush_client_sessions(set<client_t>& client_set, C_GatherBuilder& gather);
+ void finish_flush_session(Session *session, version_t seq);
void terminate_sessions();
void find_idle_sessions();
void kill_session(Session *session);
@@ -238,6 +241,7 @@ public:
void handle_slave_rename_prep(MDRequest *mdr);
void handle_slave_rename_prep_ack(MDRequest *mdr, MMDSSlaveRequest *m);
void handle_slave_rename_notify_ack(MDRequest *mdr, MMDSSlaveRequest *m);
+ void _slave_rename_sessions_flushed(MDRequest *mdr);
void _logged_slave_rename(MDRequest *mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
void _commit_slave_rename(MDRequest *mdr, int r, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
void do_rename_rollback(bufferlist &rbl, int master, MDRequest *mdr, bool finish_mdr=false);
diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index f17fa9e..1b27029 100644
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -29,7 +29,7 @@
void SessionMap::dump()
{
dout(10) << "dump" << dendl;
- for (hash_map<entity_name_t,Session*>::iterator p = session_map.begin();
+ for (ceph::unordered_map<entity_name_t,Session*>::iterator p = session_map.begin();
p != session_map.end();
++p)
dout(10) << p->first << " " << p->second
@@ -103,7 +103,8 @@ class C_SM_Save : public Context {
public:
C_SM_Save(SessionMap *cm, version_t v) : sessionmap(cm), version(v) {}
void finish(int r) {
- sessionmap->_save_finish(version);
+ assert(r == 0);
+ sessionmap->_save_finish(version);
}
};
@@ -153,7 +154,7 @@ void SessionMap::encode(bufferlist& bl) const
ENCODE_START(3, 3, bl);
::encode(version, bl);
- for (hash_map<entity_name_t,Session*>::const_iterator p = session_map.begin();
+ for (ceph::unordered_map<entity_name_t,Session*>::const_iterator p = session_map.begin();
p != session_map.end();
++p) {
if (p->second->is_open() ||
@@ -220,7 +221,7 @@ void SessionMap::decode(bufferlist::iterator& p)
void SessionMap::dump(Formatter *f) const
{
f->open_array_section("Sessions");
- for (hash_map<entity_name_t,Session*>::const_iterator p = session_map.begin();
+ for (ceph::unordered_map<entity_name_t,Session*>::const_iterator p = session_map.begin();
p != session_map.end();
++p) {
f->open_object_section("Session");
@@ -257,7 +258,7 @@ void SessionMap::wipe()
void SessionMap::wipe_ino_prealloc()
{
- for (hash_map<entity_name_t,Session*>::iterator p = session_map.begin();
+ for (ceph::unordered_map<entity_name_t,Session*>::iterator p = session_map.begin();
p != session_map.end();
++p) {
p->second->pending_prealloc_inos.clear();
diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index e960ac5..8ddbacc 100644
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -18,8 +18,7 @@
#include <set>
using std::set;
-#include <ext/hash_map>
-using __gnu_cxx::hash_map;
+#include "include/unordered_map.h"
#include "include/Context.h"
#include "include/xlist.h"
@@ -144,6 +143,7 @@ public:
// -- caps --
private:
version_t cap_push_seq; // cap push seq #
+ map<version_t, list<Context*> > waitfor_flush; // flush session messages
public:
xlist<Capability*> caps; // inodes with caps; front=most recently used
xlist<ClientLease*> leases; // metadata leases to clients
@@ -153,6 +153,19 @@ public:
version_t inc_push_seq() { return ++cap_push_seq; }
version_t get_push_seq() const { return cap_push_seq; }
+ version_t wait_for_flush(Context* c) {
+ waitfor_flush[get_push_seq()].push_back(c);
+ return get_push_seq();
+ }
+ void finish_flush(version_t seq, list<Context*>& ls) {
+ while (!waitfor_flush.empty()) {
+ if (waitfor_flush.begin()->first > seq)
+ break;
+ ls.splice(ls.end(), waitfor_flush.begin()->second);
+ waitfor_flush.erase(waitfor_flush.begin());
+ }
+ }
+
void add_cap(Capability *cap) {
caps.push_back(&cap->item_session_caps);
}
@@ -221,7 +234,7 @@ class MDS;
class SessionMap {
private:
MDS *mds;
- hash_map<entity_name_t, Session*> session_map;
+ ceph::unordered_map<entity_name_t, Session*> session_map;
public:
map<int,xlist<Session*>* > by_state;
@@ -267,12 +280,13 @@ public:
}
Session* get_or_add_session(const entity_inst_t& i) {
Session *s;
- if (session_map.count(i.name))
+ if (session_map.count(i.name)) {
s = session_map[i.name];
- else
+ } else {
s = session_map[i.name] = new Session;
- s->info.inst = i;
- s->last_cap_renew = ceph_clock_now(g_ceph_context);
+ s->info.inst = i;
+ s->last_cap_renew = ceph_clock_now(g_ceph_context);
+ }
return s;
}
void add_session(Session *s) {
@@ -317,14 +331,14 @@ public:
void dump();
void get_client_set(set<client_t>& s) {
- for (hash_map<entity_name_t,Session*>::iterator p = session_map.begin();
+ for (ceph::unordered_map<entity_name_t,Session*>::iterator p = session_map.begin();
p != session_map.end();
++p)
if (p->second->info.inst.name.is_client())
s.insert(p->second->info.inst.name.num());
}
void get_client_session_set(set<Session*>& s) {
- for (hash_map<entity_name_t,Session*>::iterator p = session_map.begin();
+ for (ceph::unordered_map<entity_name_t,Session*>::iterator p = session_map.begin();
p != session_map.end();
++p)
if (p->second->info.inst.name.is_client())
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 7684e9a..37c8235 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -368,7 +368,7 @@ public:
}
void init_gather() {
- for (map<int,int>::const_iterator p = parent->replicas_begin();
+ for (map<int,unsigned>::const_iterator p = parent->replicas_begin();
p != parent->replicas_end();
++p)
more()->gather_set.insert(p->first);
diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h
index 6edcf96..685f5ff 100644
--- a/src/mds/events/EMetaBlob.h
+++ b/src/mds/events/EMetaBlob.h
@@ -234,7 +234,7 @@ public:
private:
mutable bufferlist dnbl;
bool dn_decoded;
- list<std::tr1::shared_ptr<fullbit> > dfull;
+ list<ceph::shared_ptr<fullbit> > dfull;
list<remotebit> dremote;
list<nullbit> dnull;
@@ -250,7 +250,7 @@ public:
bool is_importing() { return state & STATE_IMPORTING; }
void mark_importing() { state |= STATE_IMPORTING; }
- list<std::tr1::shared_ptr<fullbit> > &get_dfull() { return dfull; }
+ list<ceph::shared_ptr<fullbit> > &get_dfull() { return dfull; }
list<remotebit> &get_dremote() { return dremote; }
list<nullbit> &get_dnull() { return dnull; }
@@ -260,7 +260,7 @@ public:
<< " num " << nfull << "/" << nremote << "/" << nnull
<< std::endl;
_decode_bits();
- for (list<std::tr1::shared_ptr<fullbit> >::iterator p = dfull.begin(); p != dfull.end(); ++p)
+ for (list<ceph::shared_ptr<fullbit> >::iterator p = dfull.begin(); p != dfull.end(); ++p)
(*p)->print(out);
for (list<remotebit>::iterator p = dremote.begin(); p != dremote.end(); ++p)
p->print(out);
@@ -312,7 +312,7 @@ private:
// my lumps. preserve the order we added them in a list.
list<dirfrag_t> lump_order;
map<dirfrag_t, dirlump> lump_map;
- list<std::tr1::shared_ptr<fullbit> > roots;
+ list<ceph::shared_ptr<fullbit> > roots;
list<pair<__u8,version_t> > table_tids; // tableclient transactions
@@ -454,7 +454,7 @@ private:
sr->encode(snapbl);
lump.nfull++;
- lump.get_dfull().push_back(std::tr1::shared_ptr<fullbit>(new fullbit(dn->get_name(),
+ lump.get_dfull().push_back(ceph::shared_ptr<fullbit>(new fullbit(dn->get_name(),
dn->first, dn->last,
dn->get_projected_version(),
*pi, in->dirfragtree,
@@ -507,7 +507,7 @@ private:
else
in->encode_snap_blob(snapbl);
- for (list<std::tr1::shared_ptr<fullbit> >::iterator p = roots.begin(); p != roots.end(); ++p) {
+ for (list<ceph::shared_ptr<fullbit> >::iterator p = roots.begin(); p != roots.end(); ++p) {
if ((*p)->inode.ino == in->ino()) {
roots.erase(p);
break;
@@ -515,7 +515,7 @@ private:
}
string empty;
- roots.push_back(std::tr1::shared_ptr<fullbit>(new fullbit(empty, in->first, in->last, 0, *pi,
+ roots.push_back(ceph::shared_ptr<fullbit>(new fullbit(empty, in->first, in->last, 0, *pi,
*pdft, *px, in->symlink, snapbl,
dirty ? fullbit::STATE_DIRTY : 0,
&in->old_inodes)));
@@ -532,7 +532,7 @@ private:
dirlump& add_import_dir(CDir *dir) {
// dirty=false would be okay in some cases
return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
- true, dir->is_complete(), false, true);
+ dir->is_dirty(), dir->is_complete(), false, true);
}
dirlump& add_dir(dirfrag_t df, fnode_t *pf, version_t pv, bool dirty,
bool complete=false, bool isnew=false, bool importing=false) {
diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 41a79f9..e6d530f 100644
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -237,11 +237,11 @@ void LogSegment::try_to_expire(MDS *mds, C_GatherBuilder &gather_bld)
}
// pending commit atids
- for (map<int, hash_set<version_t> >::iterator p = pending_commit_tids.begin();
+ for (map<int, ceph::unordered_set<version_t> >::iterator p = pending_commit_tids.begin();
p != pending_commit_tids.end();
++p) {
MDSTableClient *client = mds->get_table_client(p->first);
- for (hash_set<version_t>::iterator q = p->second.begin();
+ for (ceph::unordered_set<version_t>::iterator q = p->second.begin();
q != p->second.end();
++q) {
dout(10) << "try_to_expire " << get_mdstable_name(p->first) << " transaction " << *q
@@ -715,7 +715,7 @@ void EMetaBlob::dirlump::dump(Formatter *f) const
f->dump_int("nnull", nnull);
f->open_array_section("full bits");
- for (list<std::tr1::shared_ptr<fullbit> >::const_iterator
+ for (list<ceph::shared_ptr<fullbit> >::const_iterator
iter = dfull.begin(); iter != dfull.end(); ++iter) {
f->open_object_section("fullbit");
(*iter)->dump(f);
@@ -789,7 +789,7 @@ void EMetaBlob::decode(bufferlist::iterator &bl)
::decode(rootbl, bl);
if (rootbl.length()) {
bufferlist::iterator p = rootbl.begin();
- roots.push_back(std::tr1::shared_ptr<fullbit>(new fullbit(p)));
+ roots.push_back(ceph::shared_ptr<fullbit>(new fullbit(p)));
}
}
::decode(table_tids, bl);
@@ -844,7 +844,7 @@ void EMetaBlob::dump(Formatter *f) const
f->close_section(); // lumps
f->open_array_section("roots");
- for (list<std::tr1::shared_ptr<fullbit> >::const_iterator i = roots.begin();
+ for (list<ceph::shared_ptr<fullbit> >::const_iterator i = roots.begin();
i != roots.end(); ++i) {
f->open_object_section("root");
(*i)->dump(f);
@@ -929,7 +929,7 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
assert(g_conf->mds_kill_journal_replay_at != 1);
- for (list<std::tr1::shared_ptr<fullbit> >::iterator p = roots.begin(); p != roots.end(); ++p) {
+ for (list<ceph::shared_ptr<fullbit> >::iterator p = roots.begin(); p != roots.end(); ++p) {
CInode *in = mds->mdcache->get_inode((*p)->inode.ino);
bool isnew = in ? false:true;
if (!in)
@@ -1032,10 +1032,10 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
lump._decode_bits();
// full dentry+inode pairs
- for (list<std::tr1::shared_ptr<fullbit> >::iterator pp = lump.get_dfull().begin();
+ for (list<ceph::shared_ptr<fullbit> >::iterator pp = lump.get_dfull().begin();
pp != lump.get_dfull().end();
++pp) {
- std::tr1::shared_ptr<fullbit> p = *pp;
+ ceph::shared_ptr<fullbit> p = *pp;
CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast);
if (!dn) {
dn = dir->add_null_dentry(p->dn, p->dnfirst, p->dnlast);
diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index 362f747..e39a723 100644
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -204,7 +204,7 @@ ostream& operator<<(ostream& out, const client_writeable_range_t& r)
*/
void inode_t::encode(bufferlist &bl) const
{
- ENCODE_START(8, 6, bl);
+ ENCODE_START(9, 6, bl);
::encode(ino, bl);
::encode(rdev, bl);
@@ -239,13 +239,15 @@ void inode_t::encode(bufferlist &bl) const
::encode(backtrace_version, bl);
::encode(old_pools, bl);
::encode(max_size_ever, bl);
+ ::encode(inline_version, bl);
+ ::encode(inline_data, bl);
ENCODE_FINISH(bl);
}
void inode_t::decode(bufferlist::iterator &p)
{
- DECODE_START_LEGACY_COMPAT_LEN(7, 6, 6, p);
+ DECODE_START_LEGACY_COMPAT_LEN(9, 6, 6, p);
::decode(ino, p);
::decode(rdev, p);
@@ -295,8 +297,16 @@ void inode_t::decode(bufferlist::iterator &p)
::decode(backtrace_version, p);
if (struct_v >= 7)
::decode(old_pools, p);
+ else
+ backtrace_version = 0; // note inode which has no backtrace
if (struct_v >= 8)
::decode(max_size_ever, p);
+ if (struct_v >= 9) {
+ ::decode(inline_version, p);
+ ::decode(inline_data, p);
+ } else {
+ inline_version = CEPH_INLINE_NONE;
+ }
DECODE_FINISH(p);
}
diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index bd53c85..22eba07 100644
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -24,6 +24,7 @@ using namespace std;
#include <boost/pool/pool.hpp>
#include "include/assert.h"
+#include "include/hash_namespace.h"
#define CEPH_FS_ONDISK_MAGIC "ceph fs volume v011"
@@ -36,6 +37,9 @@ using namespace std;
#define MDS_PORT_LOCKER 0x300
#define MDS_PORT_MIGRATOR 0x400
+// FIXME: this should not be hardcoded
+#define MDS_DATA_POOL 0
+#define MDS_METADATA_POOL 1
#define MAX_MDS 0x100
#define NUM_STRAY 10
@@ -245,7 +249,7 @@ inline bool operator<(const vinodeno_t &l, const vinodeno_t &r) {
(l.ino == r.ino && l.snapid < r.snapid);
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<vinodeno_t> {
size_t operator()(const vinodeno_t &vino) const {
hash<inodeno_t> H;
@@ -253,7 +257,7 @@ namespace __gnu_cxx {
return H(vino.ino) ^ I(vino.snapid);
}
};
-}
+CEPH_HASH_NAMESPACE_END
@@ -336,6 +340,8 @@ struct inode_t {
utime_t mtime; // file data modify time.
utime_t atime; // file data access time.
uint32_t time_warp_seq; // count of (potential) mtime/atime timewarps (i.e., utimes())
+ bufferlist inline_data;
+ version_t inline_version;
map<client_t,client_writeable_range_t> client_ranges; // client(s) can write to these ranges
@@ -358,6 +364,7 @@ struct inode_t {
truncate_seq(0), truncate_size(0), truncate_from(0),
truncate_pending(0),
time_warp_seq(0),
+ inline_version(1),
version(0), file_data_version(0), xattr_version(0), backtrace_version(0) {
clear_layout();
memset(&dir_layout, 0, sizeof(dir_layout));
@@ -538,43 +545,40 @@ struct dentry_key_t {
// encode into something that can be decoded as a string.
// name_ (head) or name_%x (!head)
void encode(bufferlist& bl) const {
- __u32 l = strlen(name) + 1;
+ string key;
+ encode(key);
+ ::encode(key, bl);
+ }
+ void encode(string& key) const {
char b[20];
if (snapid != CEPH_NOSNAP) {
uint64_t val(snapid);
snprintf(b, sizeof(b), "%" PRIx64, val);
- l += strlen(b);
} else {
snprintf(b, sizeof(b), "%s", "head");
- l += 4;
}
- ::encode(l, bl);
- bl.append(name, strlen(name));
- bl.append("_", 1);
- bl.append(b);
+ ostringstream oss;
+ oss << name << "_" << b;
+ key = oss.str();
}
static void decode_helper(bufferlist::iterator& bl, string& nm, snapid_t& sn) {
- string foo;
- ::decode(foo, bl);
-
- int i = foo.length()-1;
- while (foo[i] != '_' && i)
- i--;
- assert(i);
- if (i+5 == (int)foo.length() &&
- foo[i+1] == 'h' &&
- foo[i+2] == 'e' &&
- foo[i+3] == 'a' &&
- foo[i+4] == 'd') {
+ string key;
+ ::decode(key, bl);
+ decode_helper(key, nm, sn);
+ }
+ static void decode_helper(const string& key, string& nm, snapid_t& sn) {
+ size_t i = key.find_last_of('_');
+ assert(i != string::npos);
+ if (key.compare(i+1, string::npos, "head") == 0) {
// name_head
sn = CEPH_NOSNAP;
} else {
// name_%x
long long unsigned x = 0;
- sscanf(foo.c_str() + i + 1, "%llx", &x);
+ sscanf(key.c_str() + i + 1, "%llx", &x);
sn = x;
}
- nm = string(foo.c_str(), i);
+ nm = string(key.c_str(), i);
}
};
@@ -680,14 +684,14 @@ inline bool operator<=(const metareqid_t& l, const metareqid_t& r) {
inline bool operator>(const metareqid_t& l, const metareqid_t& r) { return !(l <= r); }
inline bool operator>=(const metareqid_t& l, const metareqid_t& r) { return !(l < r); }
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<metareqid_t> {
size_t operator()(const metareqid_t &r) const {
hash<uint64_t> H;
return H(r.name.num()) ^ H(r.name.type()) ^ H(r.tid);
}
};
-}
+CEPH_HASH_NAMESPACE_END
// cap info for client reconnect
@@ -803,7 +807,7 @@ inline bool operator==(dirfrag_t l, dirfrag_t r) {
return l.ino == r.ino && l.frag == r.frag;
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<dirfrag_t> {
size_t operator()(const dirfrag_t &df) const {
static rjhash<uint64_t> H;
@@ -811,7 +815,7 @@ namespace __gnu_cxx {
return H(df.ino) ^ I(df.frag);
}
};
-}
+CEPH_HASH_NAMESPACE_END
@@ -1294,26 +1298,26 @@ protected:
// --------------------------------------------
// replication (across mds cluster)
protected:
- __s16 replica_nonce; // [replica] defined on replica
- map<int,int> replica_map; // [auth] mds -> nonce
+ unsigned replica_nonce; // [replica] defined on replica
+ map<int,unsigned> replica_map; // [auth] mds -> nonce
public:
bool is_replicated() { return !replica_map.empty(); }
bool is_replica(int mds) { return replica_map.count(mds); }
int num_replicas() { return replica_map.size(); }
- int add_replica(int mds) {
+ unsigned add_replica(int mds) {
if (replica_map.count(mds))
return ++replica_map[mds]; // inc nonce
if (replica_map.empty())
get(PIN_REPLICATED);
return replica_map[mds] = 1;
}
- void add_replica(int mds, int nonce) {
+ void add_replica(int mds, unsigned nonce) {
if (replica_map.empty())
get(PIN_REPLICATED);
replica_map[mds] = nonce;
}
- int get_replica_nonce(int mds) {
+ unsigned get_replica_nonce(int mds) {
assert(replica_map.count(mds));
return replica_map[mds];
}
@@ -1328,18 +1332,18 @@ protected:
put(PIN_REPLICATED);
replica_map.clear();
}
- map<int,int>::iterator replicas_begin() { return replica_map.begin(); }
- map<int,int>::iterator replicas_end() { return replica_map.end(); }
- const map<int,int>& get_replicas() { return replica_map; }
+ map<int,unsigned>::iterator replicas_begin() { return replica_map.begin(); }
+ map<int,unsigned>::iterator replicas_end() { return replica_map.end(); }
+ const map<int,unsigned>& get_replicas() { return replica_map; }
void list_replicas(set<int>& ls) {
- for (map<int,int>::const_iterator p = replica_map.begin();
+ for (map<int,unsigned>::const_iterator p = replica_map.begin();
p != replica_map.end();
++p)
ls.insert(p->first);
}
- int get_replica_nonce() { return replica_nonce;}
- void set_replica_nonce(int n) { replica_nonce = n; }
+ unsigned get_replica_nonce() { return replica_nonce; }
+ void set_replica_nonce(unsigned n) { replica_nonce = n; }
// ---------------------------------------------
diff --git a/src/messages/MCacheExpire.h b/src/messages/MCacheExpire.h
index aa40407..b71ada7 100644
--- a/src/messages/MCacheExpire.h
+++ b/src/messages/MCacheExpire.h
@@ -26,9 +26,22 @@ public:
that makes it less work to process when exports are in progress.
*/
struct realm {
- map<vinodeno_t, __s32> inodes;
- map<dirfrag_t, __s32> dirs;
- map<dirfrag_t, map<pair<string,snapid_t>,__s32> > dentries;
+ map<vinodeno_t, uint32_t> inodes;
+ map<dirfrag_t, uint32_t> dirs;
+ map<dirfrag_t, map<pair<string,snapid_t>,uint32_t> > dentries;
+
+ void merge(realm& o) {
+ inodes.insert(o.inodes.begin(), o.inodes.end());
+ dirs.insert(o.dirs.begin(), o.dirs.end());
+ for (map<dirfrag_t,map<pair<string,snapid_t>,uint32_t> >::iterator p = o.dentries.begin();
+ p != o.dentries.end();
+ ++p) {
+ if (dentries.count(p->first) == 0)
+ dentries[p->first] = p->second;
+ else
+ dentries[p->first].insert(p->second.begin(), p->second.end());
+ }
+ }
void encode(bufferlist &bl) const {
::encode(inodes, bl);
@@ -57,18 +70,21 @@ private:
public:
virtual const char *get_type_name() const { return "cache_expire";}
- void add_inode(dirfrag_t r, vinodeno_t vino, int nonce) {
+ void add_inode(dirfrag_t r, vinodeno_t vino, unsigned nonce) {
realms[r].inodes[vino] = nonce;
}
- void add_dir(dirfrag_t r, dirfrag_t df, int nonce) {
+ void add_dir(dirfrag_t r, dirfrag_t df, unsigned nonce) {
realms[r].dirs[df] = nonce;
}
- void add_dentry(dirfrag_t r, dirfrag_t df, const string& dn, snapid_t last, int nonce) {
+ void add_dentry(dirfrag_t r, dirfrag_t df, const string& dn, snapid_t last, unsigned nonce) {
realms[r].dentries[df][pair<string,snapid_t>(dn,last)] = nonce;
}
void add_realm(dirfrag_t df, realm& r) {
- realms[df] = r;
+ if (realms.count(df) == 0)
+ realms[df] = r;
+ else
+ realms[df].merge(r);
}
void decode_payload() {
diff --git a/src/messages/MClientCaps.h b/src/messages/MClientCaps.h
index 117f241..11c8068 100644
--- a/src/messages/MClientCaps.h
+++ b/src/messages/MClientCaps.h
@@ -21,14 +21,17 @@
class MClientCaps : public Message {
- static const int HEAD_VERSION = 2; // added flock metadata
+ static const int HEAD_VERSION = 4; // added flock metadata, inline data
static const int COMPAT_VERSION = 1;
public:
struct ceph_mds_caps head;
+ struct ceph_mds_cap_peer peer;
bufferlist snapbl;
bufferlist xattrbl;
bufferlist flockbl;
+ version_t inline_version;
+ bufferlist inline_data;
int get_caps() { return head.caps; }
int get_wanted() { return head.wanted; }
@@ -73,8 +76,18 @@ class MClientCaps : public Message {
void set_mtime(const utime_t &t) { t.encode_timeval(&head.mtime); }
void set_atime(const utime_t &t) { t.encode_timeval(&head.atime); }
+ void set_cap_peer(uint64_t id, ceph_seq_t seq, ceph_seq_t mseq, int mds, int flags) {
+ peer.cap_id = id;
+ peer.seq = seq;
+ peer.mseq = mseq;
+ peer.mds = mds;
+ peer.flags = flags;
+ }
+
MClientCaps()
- : Message(CEPH_MSG_CLIENT_CAPS, HEAD_VERSION, COMPAT_VERSION) { }
+ : Message(CEPH_MSG_CLIENT_CAPS, HEAD_VERSION, COMPAT_VERSION) {
+ inline_version = 0;
+ }
MClientCaps(int op,
inodeno_t ino,
inodeno_t realm,
@@ -95,6 +108,8 @@ class MClientCaps : public Message {
head.wanted = wanted;
head.dirty = dirty;
head.migrate_seq = mseq;
+ peer.cap_id = 0;
+ inline_version = 0;
}
MClientCaps(int op,
inodeno_t ino, inodeno_t realm,
@@ -106,6 +121,8 @@ class MClientCaps : public Message {
head.realm = realm;
head.cap_id = id;
head.migrate_seq = mseq;
+ peer.cap_id = 0;
+ inline_version = 0;
}
private:
~MClientCaps() {}
@@ -151,10 +168,29 @@ public:
// conditionally decode flock metadata
if (header.version >= 2)
::decode(flockbl, p);
+
+ if (header.version >= 3) {
+ if (head.op == CEPH_CAP_OP_IMPORT)
+ ::decode(peer, p);
+ else if (head.op == CEPH_CAP_OP_EXPORT)
+ memcpy(&peer, &head.peer, sizeof(peer));
+ }
+
+ if (header.version >= 4) {
+ ::decode(inline_version, p);
+ ::decode(inline_data, p);
+ } else {
+ inline_version = CEPH_INLINE_NONE;
+ }
}
void encode_payload(uint64_t features) {
head.snap_trace_len = snapbl.length();
head.xattr_len = xattrbl.length();
+
+ // record peer in unused fields of cap export message
+ if ((features & CEPH_FEATURE_EXPORT_PEER) && head.op == CEPH_CAP_OP_EXPORT)
+ memcpy(&head.peer, &peer, sizeof(peer));
+
::encode(head, payload);
::encode_nohead(snapbl, payload);
@@ -164,7 +200,24 @@ public:
if (features & CEPH_FEATURE_FLOCK) {
::encode(flockbl, payload);
} else {
- header.version = 1; // old
+ header.version = 1;
+ return;
+ }
+
+ if (features & CEPH_FEATURE_EXPORT_PEER) {
+ if (head.op == CEPH_CAP_OP_IMPORT)
+ ::encode(peer, payload);
+ } else {
+ header.version = 2;
+ return;
+ }
+
+ if (features & CEPH_FEATURE_MDS_INLINE_DATA) {
+ ::encode(inline_version, payload);
+ ::encode(inline_data, payload);
+ } else {
+ header.version = 3;
+ return;
}
}
};
diff --git a/src/messages/MClientReply.h b/src/messages/MClientReply.h
index 896245f..e9abb85 100644
--- a/src/messages/MClientReply.h
+++ b/src/messages/MClientReply.h
@@ -108,6 +108,8 @@ struct InodeStat {
uint64_t truncate_size;
utime_t ctime, mtime, atime;
version_t time_warp_seq;
+ bufferlist inline_data;
+ version_t inline_version;
frag_info_t dirstat;
nest_info_t rstat;
@@ -174,6 +176,13 @@ struct InodeStat {
xattr_version = e.xattr_version;
::decode(xattrbl, p);
+
+ if (features & CEPH_FEATURE_MDS_INLINE_DATA) {
+ ::decode(inline_version, p);
+ ::decode(inline_data, p);
+ } else {
+ inline_version = CEPH_INLINE_NONE;
+ }
}
// see CInode::encode_inodestat for encoder.
diff --git a/src/messages/MExportDir.h b/src/messages/MExportDir.h
index 688526d..2ff85d4 100644
--- a/src/messages/MExportDir.h
+++ b/src/messages/MExportDir.h
@@ -27,9 +27,9 @@ class MExportDir : public Message {
bufferlist client_map;
MExportDir() : Message(MSG_MDS_EXPORTDIR) {}
- MExportDir(dirfrag_t df) :
- Message(MSG_MDS_EXPORTDIR),
- dirfrag(df) {
+ MExportDir(dirfrag_t df, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIR), dirfrag(df) {
+ set_tid(tid);
}
private:
~MExportDir() {}
diff --git a/src/messages/MExportDirAck.h b/src/messages/MExportDirAck.h
index 42c5c5d..eb23789 100644
--- a/src/messages/MExportDirAck.h
+++ b/src/messages/MExportDirAck.h
@@ -18,14 +18,17 @@
#include "MExportDir.h"
class MExportDirAck : public Message {
+public:
dirfrag_t dirfrag;
+ bufferlist imported_caps;
- public:
dirfrag_t get_dirfrag() { return dirfrag; }
MExportDirAck() : Message(MSG_MDS_EXPORTDIRACK) {}
- MExportDirAck(dirfrag_t i) :
- Message(MSG_MDS_EXPORTDIRACK), dirfrag(i) { }
+ MExportDirAck(dirfrag_t df, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIRACK), dirfrag(df) {
+ set_tid(tid);
+ }
private:
~MExportDirAck() {}
@@ -38,9 +41,11 @@ public:
void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(dirfrag, p);
+ ::decode(imported_caps, p);
}
void encode_payload(uint64_t features) {
::encode(dirfrag, payload);
+ ::encode(imported_caps, payload);
}
};
diff --git a/src/messages/MExportDirCancel.h b/src/messages/MExportDirCancel.h
index 9cd0602..b3b2c58 100644
--- a/src/messages/MExportDirCancel.h
+++ b/src/messages/MExportDirCancel.h
@@ -25,9 +25,10 @@ class MExportDirCancel : public Message {
dirfrag_t get_dirfrag() { return dirfrag; }
MExportDirCancel() : Message(MSG_MDS_EXPORTDIRCANCEL) {}
- MExportDirCancel(dirfrag_t df) :
- Message(MSG_MDS_EXPORTDIRCANCEL),
- dirfrag(df) { }
+ MExportDirCancel(dirfrag_t df, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIRCANCEL), dirfrag(df) {
+ set_tid(tid);
+ }
private:
~MExportDirCancel() {}
diff --git a/src/messages/MExportDirDiscover.h b/src/messages/MExportDirDiscover.h
index 7602bd9..0b29daf 100644
--- a/src/messages/MExportDirDiscover.h
+++ b/src/messages/MExportDirDiscover.h
@@ -34,13 +34,11 @@ class MExportDirDiscover : public Message {
MExportDirDiscover() :
Message(MSG_MDS_EXPORTDIRDISCOVER),
started(false) { }
- MExportDirDiscover(int f, filepath& p, dirfrag_t df) :
+ MExportDirDiscover(dirfrag_t df, filepath& p, int f, uint64_t tid) :
Message(MSG_MDS_EXPORTDIRDISCOVER),
- from(f),
- dirfrag(df),
- path(p),
- started(false)
- { }
+ from(f), dirfrag(df), path(p), started(false) {
+ set_tid(tid);
+ }
private:
~MExportDirDiscover() {}
diff --git a/src/messages/MExportDirDiscoverAck.h b/src/messages/MExportDirDiscoverAck.h
index 82eeb5e..8e20be6 100644
--- a/src/messages/MExportDirDiscoverAck.h
+++ b/src/messages/MExportDirDiscoverAck.h
@@ -28,10 +28,11 @@ class MExportDirDiscoverAck : public Message {
bool is_success() { return success; }
MExportDirDiscoverAck() : Message(MSG_MDS_EXPORTDIRDISCOVERACK) {}
- MExportDirDiscoverAck(dirfrag_t df, bool s=true) :
+ MExportDirDiscoverAck(dirfrag_t df, uint64_t tid, bool s=true) :
Message(MSG_MDS_EXPORTDIRDISCOVERACK),
- dirfrag(df),
- success(s) { }
+ dirfrag(df), success(s) {
+ set_tid(tid);
+ }
private:
~MExportDirDiscoverAck() {}
diff --git a/src/messages/MExportDirFinish.h b/src/messages/MExportDirFinish.h
index 6d05885..dd78dda 100644
--- a/src/messages/MExportDirFinish.h
+++ b/src/messages/MExportDirFinish.h
@@ -19,14 +19,16 @@
class MExportDirFinish : public Message {
dirfrag_t dirfrag;
+ bool last;
public:
dirfrag_t get_dirfrag() { return dirfrag; }
+ bool is_last() { return last; }
MExportDirFinish() {}
- MExportDirFinish(dirfrag_t dirfrag) :
- Message(MSG_MDS_EXPORTDIRFINISH) {
- this->dirfrag = dirfrag;
+ MExportDirFinish(dirfrag_t df, bool l, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIRFINISH), dirfrag(df), last(l) {
+ set_tid(tid);
}
private:
~MExportDirFinish() {}
@@ -34,15 +36,17 @@ private:
public:
const char *get_type_name() const { return "ExFin"; }
void print(ostream& o) const {
- o << "export_finish(" << dirfrag << ")";
+ o << "export_finish(" << dirfrag << (last ? " last" : "") << ")";
}
void encode_payload(uint64_t features) {
::encode(dirfrag, payload);
+ ::encode(last, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
::decode(dirfrag, p);
+ ::decode(last, p);
}
};
diff --git a/src/messages/MExportDirNotify.h b/src/messages/MExportDirNotify.h
index a209901..745cf95 100644
--- a/src/messages/MExportDirNotify.h
+++ b/src/messages/MExportDirNotify.h
@@ -33,9 +33,11 @@ class MExportDirNotify : public Message {
list<dirfrag_t>& get_bounds() { return bounds; }
MExportDirNotify() {}
- MExportDirNotify(dirfrag_t i, bool a, pair<__s32,__s32> oa, pair<__s32,__s32> na) :
+ MExportDirNotify(dirfrag_t i, uint64_t tid, bool a, pair<__s32,__s32> oa, pair<__s32,__s32> na) :
Message(MSG_MDS_EXPORTDIRNOTIFY),
- base(i), ack(a), old_auth(oa), new_auth(na) { }
+ base(i), ack(a), old_auth(oa), new_auth(na) {
+ set_tid(tid);
+ }
private:
~MExportDirNotify() {}
diff --git a/src/messages/MExportDirNotifyAck.h b/src/messages/MExportDirNotifyAck.h
index 35a6956..7b52c4f 100644
--- a/src/messages/MExportDirNotifyAck.h
+++ b/src/messages/MExportDirNotifyAck.h
@@ -26,9 +26,9 @@ class MExportDirNotifyAck : public Message {
dirfrag_t get_dirfrag() { return dirfrag; }
MExportDirNotifyAck() {}
- MExportDirNotifyAck(dirfrag_t dirfrag) :
- Message(MSG_MDS_EXPORTDIRNOTIFYACK) {
- this->dirfrag = dirfrag;
+ MExportDirNotifyAck(dirfrag_t df, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIRNOTIFYACK), dirfrag(df) {
+ set_tid(tid);
}
private:
~MExportDirNotifyAck() {}
diff --git a/src/messages/MExportDirPrep.h b/src/messages/MExportDirPrep.h
index 12f358e..791a98a 100644
--- a/src/messages/MExportDirPrep.h
+++ b/src/messages/MExportDirPrep.h
@@ -40,10 +40,11 @@ public:
MExportDirPrep() {
b_did_assim = false;
}
- MExportDirPrep(dirfrag_t df) :
+ MExportDirPrep(dirfrag_t df, uint64_t tid) :
Message(MSG_MDS_EXPORTDIRPREP),
- dirfrag(df),
- b_did_assim(false) { }
+ dirfrag(df), b_did_assim(false) {
+ set_tid(tid);
+ }
private:
~MExportDirPrep() {}
diff --git a/src/messages/MExportDirPrepAck.h b/src/messages/MExportDirPrepAck.h
index b5d180d..c3d7b49 100644
--- a/src/messages/MExportDirPrepAck.h
+++ b/src/messages/MExportDirPrepAck.h
@@ -25,9 +25,10 @@ class MExportDirPrepAck : public Message {
dirfrag_t get_dirfrag() { return dirfrag; }
MExportDirPrepAck() {}
- MExportDirPrepAck(dirfrag_t df) :
- Message(MSG_MDS_EXPORTDIRPREPACK),
- dirfrag(df) { }
+ MExportDirPrepAck(dirfrag_t df, uint64_t tid) :
+ Message(MSG_MDS_EXPORTDIRPREPACK), dirfrag(df) {
+ set_tid(tid);
+ }
private:
~MExportDirPrepAck() {}
diff --git a/src/messages/MForward.h b/src/messages/MForward.h
index 7c88aba..3128197 100644
--- a/src/messages/MForward.h
+++ b/src/messages/MForward.h
@@ -28,17 +28,27 @@ struct MForward : public Message {
PaxosServiceMessage *msg;
entity_inst_t client;
MonCap client_caps;
-
- MForward() : Message(MSG_FORWARD), tid(0), msg(NULL) {}
+ uint64_t con_features;
+
+ static const int HEAD_VERSION = 2;
+ static const int COMPAT_VERSION = 0;
+
+ MForward() : Message(MSG_FORWARD, HEAD_VERSION, COMPAT_VERSION),
+ tid(0), msg(NULL), con_features(0) {}
//the message needs to have caps filled in!
- MForward(uint64_t t, PaxosServiceMessage *m) :
- Message(MSG_FORWARD), tid(t), msg(m) {
+ MForward(uint64_t t, PaxosServiceMessage *m, uint64_t feat) :
+ Message(MSG_FORWARD, HEAD_VERSION, COMPAT_VERSION),
+ tid(t), msg(m) {
client = m->get_source_inst();
client_caps = m->get_session()->caps;
+ con_features = feat;
}
- MForward(uint64_t t, PaxosServiceMessage *m, const MonCap& caps) :
- Message(MSG_FORWARD), tid(t), msg(m), client_caps(caps) {
+ MForward(uint64_t t, PaxosServiceMessage *m, uint64_t feat,
+ const MonCap& caps) :
+ Message(MSG_FORWARD, HEAD_VERSION, COMPAT_VERSION),
+ tid(t), msg(m), client_caps(caps) {
client = m->get_source_inst();
+ con_features = feat;
}
private:
~MForward() {
@@ -51,6 +61,7 @@ public:
::encode(client, payload);
::encode(client_caps, payload, features);
encode_message(msg, features, payload);
+ ::encode(con_features, payload);
}
void decode_payload() {
@@ -59,12 +70,20 @@ public:
::decode(client, p);
::decode(client_caps, p);
msg = (PaxosServiceMessage *)decode_message(NULL, p);
+ if (header.version >= 2) {
+ ::decode(con_features, p);
+ } else {
+ con_features = 0;
+ }
+
}
const char *get_type_name() const { return "forward"; }
void print(ostream& o) const {
if (msg)
- o << "forward(" << *msg << " caps " << client_caps << ") to leader";
+ o << "forward(" << *msg << " caps " << client_caps
+ << " tid " << tid
+ << " con_features " << con_features << ") to leader";
else o << "forward(??? ) to leader";
}
};
diff --git a/src/messages/MMDSCacheRejoin.h b/src/messages/MMDSCacheRejoin.h
index 3ae8355..d48cd67 100644
--- a/src/messages/MMDSCacheRejoin.h
+++ b/src/messages/MMDSCacheRejoin.h
@@ -48,7 +48,7 @@ class MMDSCacheRejoin : public Message {
// -- types --
struct inode_strong {
- int32_t nonce;
+ uint32_t nonce;
int32_t caps_wanted;
int32_t filelock, nestlock, dftlock;
inode_strong() {}
@@ -73,7 +73,7 @@ class MMDSCacheRejoin : public Message {
WRITE_CLASS_ENCODER(inode_strong)
struct dirfrag_strong {
- int32_t nonce;
+ uint32_t nonce;
int8_t dir_rep;
dirfrag_strong() {}
dirfrag_strong(int n, int dr) : nonce(n), dir_rep(dr) {}
@@ -93,7 +93,7 @@ class MMDSCacheRejoin : public Message {
inodeno_t ino;
inodeno_t remote_ino;
unsigned char remote_d_type;
- int32_t nonce;
+ uint32_t nonce;
int32_t lock;
dn_strong() :
ino(0), remote_ino(0), remote_d_type(0), nonce(0), lock(0) {}
@@ -168,6 +168,7 @@ class MMDSCacheRejoin : public Message {
// open
map<inodeno_t,map<client_t, ceph_mds_cap_reconnect> > cap_exports;
+ bufferlist imported_caps;
// full
bufferlist inode_base;
@@ -299,6 +300,7 @@ public:
::encode(xlocked_inodes, payload);
::encode(wrlocked_inodes, payload);
::encode(cap_exports, payload);
+ ::encode(imported_caps, payload);
::encode(strong_dirfrags, payload);
::encode(dirfrag_bases, payload);
::encode(weak, payload);
@@ -320,6 +322,7 @@ public:
::decode(xlocked_inodes, p);
::decode(wrlocked_inodes, p);
::decode(cap_exports, p);
+ ::decode(imported_caps, p);
::decode(strong_dirfrags, p);
::decode(dirfrag_bases, p);
::decode(weak, p);
diff --git a/src/messages/MMDSResolve.h b/src/messages/MMDSResolve.h
index 551ca99..263bd54 100644
--- a/src/messages/MMDSResolve.h
+++ b/src/messages/MMDSResolve.h
@@ -23,7 +23,7 @@ class MMDSResolve : public Message {
public:
map<dirfrag_t, vector<dirfrag_t> > subtrees;
map<dirfrag_t, vector<dirfrag_t> > ambiguous_imports;
- vector<metareqid_t> slave_requests;
+ map<metareqid_t, bufferlist> slave_requests;
MMDSResolve() : Message(MSG_MDS_RESOLVE) {}
private:
@@ -50,7 +50,11 @@ public:
}
void add_slave_request(metareqid_t reqid) {
- slave_requests.push_back(reqid);
+ slave_requests[reqid].clear();
+ }
+
+ void add_slave_request(metareqid_t reqid, bufferlist& bl) {
+ slave_requests[reqid].claim(bl);
}
void encode_payload(uint64_t features) {
diff --git a/src/messages/MMDSResolveAck.h b/src/messages/MMDSResolveAck.h
index 2118aba..743353f 100644
--- a/src/messages/MMDSResolveAck.h
+++ b/src/messages/MMDSResolveAck.h
@@ -22,7 +22,7 @@
class MMDSResolveAck : public Message {
public:
- vector<metareqid_t> commit;
+ map<metareqid_t, bufferlist> commit;
vector<metareqid_t> abort;
MMDSResolveAck() : Message(MSG_MDS_RESOLVEACK) {}
@@ -39,7 +39,7 @@ public:
*/
void add_commit(metareqid_t r) {
- commit.push_back(r);
+ commit[r].clear();
}
void add_abort(metareqid_t r) {
abort.push_back(r);
diff --git a/src/messages/MMonElection.h b/src/messages/MMonElection.h
index 3d7dd4e..e3ccbf3 100644
--- a/src/messages/MMonElection.h
+++ b/src/messages/MMonElection.h
@@ -21,7 +21,7 @@
class MMonElection : public Message {
- static const int HEAD_VERSION = 4;
+ static const int HEAD_VERSION = 5;
static const int COMPAT_VERSION = 2;
public:
@@ -45,6 +45,7 @@ public:
bufferlist monmap_bl;
set<int> quorum;
uint64_t quorum_features;
+ bufferlist sharing_bl;
/* the following were both used in the next branch for a while
* on user cluster, so we've left them in for compatibility. */
version_t defunct_one;
@@ -74,7 +75,7 @@ public:
}
void encode_payload(uint64_t features) {
- if (monmap_bl.length() && (features & CEPH_FEATURE_MONENC) == 0) {
+ if (monmap_bl.length() && (features != CEPH_FEATURES_ALL)) {
// reencode old-format monmap
MonMap t;
t.decode(monmap_bl);
@@ -90,6 +91,7 @@ public:
::encode(quorum_features, payload);
::encode(defunct_one, payload);
::encode(defunct_two, payload);
+ ::encode(sharing_bl, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
@@ -109,6 +111,8 @@ public:
::decode(defunct_one, p);
::decode(defunct_two, p);
}
+ if (header.version >= 5)
+ ::decode(sharing_bl, p);
}
};
diff --git a/src/messages/MOSDBoot.h b/src/messages/MOSDBoot.h
index d18d56c..bfe7775 100644
--- a/src/messages/MOSDBoot.h
+++ b/src/messages/MOSDBoot.h
@@ -22,7 +22,7 @@
class MOSDBoot : public PaxosServiceMessage {
- static const int HEAD_VERSION = 4;
+ static const int HEAD_VERSION = 5;
static const int COMPAT_VERSION = 2;
public:
@@ -30,6 +30,7 @@ class MOSDBoot : public PaxosServiceMessage {
entity_addr_t hb_back_addr, hb_front_addr;
entity_addr_t cluster_addr;
epoch_t boot_epoch; // last epoch this daemon was added to the map (if any)
+ map<string,string> metadata; ///< misc metadata about this osd
MOSDBoot()
: PaxosServiceMessage(MSG_OSD_BOOT, 0, HEAD_VERSION, COMPAT_VERSION),
@@ -63,6 +64,7 @@ public:
::encode(cluster_addr, payload);
::encode(boot_epoch, payload);
::encode(hb_front_addr, payload);
+ ::encode(metadata, payload);
}
void decode_payload() {
bufferlist::iterator p = payload.begin();
@@ -75,6 +77,8 @@ public:
::decode(boot_epoch, p);
if (header.version >= 4)
::decode(hb_front_addr, p);
+ if (header.version >= 5)
+ ::decode(metadata, p);
}
};
diff --git a/src/messages/MOSDMap.h b/src/messages/MOSDMap.h
index cd4a951..7ed601c 100644
--- a/src/messages/MOSDMap.h
+++ b/src/messages/MOSDMap.h
@@ -86,11 +86,12 @@ public:
::encode(fsid, payload);
if ((features & CEPH_FEATURE_PGID64) == 0 ||
(features & CEPH_FEATURE_PGPOOL3) == 0 ||
- (features & CEPH_FEATURE_OSDENC) == 0) {
+ (features & CEPH_FEATURE_OSDENC) == 0 ||
+ (features & CEPH_FEATURE_OSDMAP_ENC) == 0) {
if ((features & CEPH_FEATURE_PGID64) == 0 ||
(features & CEPH_FEATURE_PGPOOL3) == 0)
header.version = 1; // old old_client version
- else
+ else if ((features & CEPH_FEATURE_OSDENC) == 0)
header.version = 2; // old pg_pool_t
// reencode maps using old format
diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 3855670..32408be 100644
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -365,16 +365,7 @@ struct ceph_osd_request_head {
out << " RETRY=" << get_retry_attempt();
if (get_snap_seq())
out << " snapc " << get_snap_seq() << "=" << snaps;
- if (get_flags() & CEPH_OSD_FLAG_ORDERSNAP)
- out << " ordersnap";
- if (get_flags() & CEPH_OSD_FLAG_BALANCE_READS)
- out << " balance_reads";
- if (get_flags() & CEPH_OSD_FLAG_PARALLELEXEC)
- out << " parallelexec";
- if (get_flags() & CEPH_OSD_FLAG_LOCALIZE_READS)
- out << " localize_reads";
- if (get_flags() & CEPH_OSD_FLAG_RWORDERED)
- out << " rwordered";
+ out << " " << ceph_osd_flag_string(get_flags());
out << " e" << osdmap_epoch;
out << ")";
}
diff --git a/src/messages/MOSDPGLog.h b/src/messages/MOSDPGLog.h
index 53358d6..906a859 100644
--- a/src/messages/MOSDPGLog.h
+++ b/src/messages/MOSDPGLog.h
@@ -55,6 +55,7 @@ public:
const char *get_type_name() const { return "PGlog"; }
void print(ostream& out) const {
out << "pg_log(" << info.pgid << " epoch " << epoch
+ << " log " << log
<< " query_epoch " << query_epoch << ")";
}
diff --git a/src/messages/MOSDPGScan.h b/src/messages/MOSDPGScan.h
index 7df3a96..4c86a3c 100644
--- a/src/messages/MOSDPGScan.h
+++ b/src/messages/MOSDPGScan.h
@@ -47,9 +47,9 @@ public:
::decode(end, p);
// handle hobject_t format upgrade
- if (begin.pool == -1)
+ if (!begin.is_max() && begin.pool == -1)
begin.pool = pgid.pool();
- if (end.pool == -1)
+ if (!end.is_max() && end.pool == -1)
end.pool = pgid.pool();
}
diff --git a/src/messages/MOSDSubOp.h b/src/messages/MOSDSubOp.h
index 4169e01..7e9f087 100644
--- a/src/messages/MOSDSubOp.h
+++ b/src/messages/MOSDSubOp.h
@@ -149,7 +149,7 @@ public:
if (header.version < 7) {
// Handle hobject_t format change
- if (poid.pool == -1)
+ if (!poid.is_max() && poid.pool == -1)
poid.pool = pgid.pool();
hobject_incorrect_pool = true;
}
diff --git a/src/messages/MOSDSubOpReply.h b/src/messages/MOSDSubOpReply.h
index d577dd3..6b0738a 100644
--- a/src/messages/MOSDSubOpReply.h
+++ b/src/messages/MOSDSubOpReply.h
@@ -69,7 +69,7 @@ public:
::decode(peer_stat, p);
::decode(attrset, p);
- if (poid.pool == -1)
+ if (!poid.is_max() && poid.pool == -1)
poid.pool = pgid.pool();
}
virtual void encode_payload(uint64_t features) {
diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc
index 9f88558..bf51306 100644
--- a/src/mon/AuthMonitor.cc
+++ b/src/mon/AuthMonitor.cc
@@ -95,14 +95,16 @@ void AuthMonitor::create_initial()
check_rotate();
assert(pending_auth.size() == 1);
- KeyRing keyring;
- bufferlist bl;
- int ret = mon->store->get("mkfs", "keyring", bl);
- assert(ret == 0);
- bufferlist::iterator p = bl.begin();
- ::decode(keyring, p);
+ if (mon->is_keyring_required()) {
+ KeyRing keyring;
+ bufferlist bl;
+ int ret = mon->store->get("mkfs", "keyring", bl);
+ assert(ret == 0);
+ bufferlist::iterator p = bl.begin();
+ ::decode(keyring, p);
- import_keyring(keyring);
+ import_keyring(keyring);
+ }
max_global_id = MIN_GLOBAL_ID;
@@ -187,7 +189,7 @@ void AuthMonitor::update_from_paxos(bool *need_bootstrap)
keys_ver++;
mon->key_server.set_ver(keys_ver);
- if (keys_ver == 1) {
+ if (keys_ver == 1 && mon->is_keyring_required()) {
MonitorDBStore::Transaction t;
t.erase("mkfs", "keyring");
mon->store->apply_transaction(t);
@@ -726,7 +728,8 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
ss << "imported keyring";
getline(ss, rs);
err = 0;
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "auth add" && !entity_name.empty()) {
/* expected behavior:
@@ -763,7 +766,7 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
if (inc.op == KeyServerData::AUTH_INC_ADD &&
inc.name == entity) {
wait_for_finished_proposal(
- new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ new Monitor::C_Command(mon, m, 0, rs, get_last_committed() + 1));
return true;
}
}
@@ -848,7 +851,8 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
ss << "added key for " << auth_inc.name;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if ((prefix == "auth get-or-create-key" ||
prefix == "auth get-or-create") &&
@@ -900,7 +904,8 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
::decode(auth_inc, q);
if (auth_inc.op == KeyServerData::AUTH_INC_ADD &&
auth_inc.name == entity) {
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
}
@@ -935,7 +940,8 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
rdata.append(ds);
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, rdata, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, rdata,
+ get_last_committed() + 1));
return true;
} else if (prefix == "auth caps" && !entity_name.empty()) {
KeyServerData::Incremental auth_inc;
@@ -957,7 +963,8 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
ss << "updated caps for " << auth_inc.name;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "auth del" && !entity_name.empty()) {
KeyServerData::Incremental auth_inc;
@@ -972,7 +979,8 @@ bool AuthMonitor::prepare_command(MMonCommand *m)
ss << "updated";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
diff --git a/src/mon/DataHealthService.cc b/src/mon/DataHealthService.cc
index 5fc745c..9b707cd 100644
--- a/src/mon/DataHealthService.cc
+++ b/src/mon/DataHealthService.cc
@@ -12,13 +12,26 @@
*
*/
#include <memory>
-#include <tr1/memory>
+#include "include/memory.h"
#include <errno.h>
#include <map>
#include <list>
#include <string>
#include <sstream>
+
+#include "acconfig.h"
+
+#ifdef HAVE_SYS_VFS_H
#include <sys/vfs.h>
+#endif
+
+#ifdef HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
#include "messages/MMonHealth.h"
#include "include/types.h"
@@ -205,7 +218,7 @@ void DataHealthService::service_tick()
DataStats &ours = stats[mon->messenger->get_myinst()];
if (ours.latest_avail_percent <= g_conf->mon_data_avail_crit) {
- derr << "reached critical levels of available space on data store"
+ derr << "reached critical levels of available space on local monitor storage"
<< " -- shutdown!" << dendl;
force_shutdown();
return;
@@ -218,7 +231,7 @@ void DataHealthService::service_tick()
if (ours.latest_avail_percent <= g_conf->mon_data_avail_warn) {
if (ours.latest_avail_percent != last_warned_percent)
mon->clog.warn()
- << "reached concerning levels of available space on data store"
+ << "reached concerning levels of available space on local monitor storage"
<< " (" << ours.latest_avail_percent << "\% free)\n";
last_warned_percent = ours.latest_avail_percent;
} else {
diff --git a/src/mon/MonCommands.h b/src/mon/DumplingMonCommands.h
similarity index 89%
copy from src/mon/MonCommands.h
copy to src/mon/DumplingMonCommands.h
index 881c5a2..8e9c2bb 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/DumplingMonCommands.h
@@ -59,7 +59,7 @@
* CephString: optional badchars
* CephSocketpath: validation involves "is it S_ISSOCK"
* CephIPAddr: v4 or v6 addr with optional port, syntax validated
- * CephEntityAddr: CephIPAddr + optional '/nonce'
+ * CephEntityAddr: CephIPAddr + '/nonce'
* CephPoolname: Plainold string
* CephObjectname: Another plainold string
* CephPgid: n.xxx where n is an int > 0, xxx is a hex number > 0
@@ -111,8 +111,8 @@ COMMAND("pg getmap", "get binary pg map to -o/stdout", "pg", "r", "cli,rest")
COMMAND("pg send_pg_creates", "trigger pg creates to be issued",\
"pg", "rw", "cli,rest")
COMMAND("pg dump " \
- "name=dumpcontents,type=CephChoices,strings=all|summary|sum|delta|pools|osds|pgs|pgs_brief,n=N,req=false", \
- "show human-readable versions of pg map (only 'all' valid with plain)", "pg", "r", "cli,rest")
+ "name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs|pgs_brief,n=N,req=false", \
+ "show human-readable versions of pg map", "pg", "r", "cli,rest")
COMMAND("pg dump_json " \
"name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs,n=N,req=false", \
"show human-readable version of pg map in json only",\
@@ -210,8 +210,8 @@ COMMAND("quorum_status", "report status of monitor quorum", \
"mon", "r", "cli,rest")
COMMAND("mon_status", "report status of monitors", "mon", "r", "cli,rest")
COMMAND("sync force " \
- "name=validate1,type=CephChoices,strings=--yes-i-really-mean-it,req=false " \
- "name=validate2,type=CephChoices,strings=--i-know-what-i-am-doing,req=false", \
+ "name=validate1,type=CephChoices,strings=--yes-i-really-mean-it " \
+ "name=validate2,type=CephChoices,strings=--i-know-what-i-am-doing", \
"force sync of and clear monitor store", "mon", "rw", "cli,rest")
COMMAND("heap " \
"name=heapcmd,type=CephChoices,strings=dump|start_profiler|stop_profiler|release|stats", \
@@ -274,32 +274,23 @@ COMMAND("mds compat rm_compat " \
COMMAND("mds compat rm_incompat " \
"name=feature,type=CephInt,range=0", \
"remove incompatible feature", "mds", "rw", "cli,rest")
-COMMAND("mds set " \
- "name=key,type=CephChoices,strings=allow_new_snaps " \
- "name=sure,type=CephString,req=false", \
- "set <key>", \
- "mds", "w", "cli,rest")
-COMMAND("mds unset " \
- "name=key,type=CephChoices,strings=allow_new_snaps " \
- "name=sure,type=CephString,req=false", \
- "unset <key>", "mds", "w", "cli,rest")
COMMAND("mds add_data_pool " \
- "name=pool,type=CephString", \
- "add data pool <pool>", "mds", "rw", "cli,rest")
+ "name=poolid,type=CephInt,range=0", \
+ "add data pool <poolid>", "mds", "rw", "cli,rest")
COMMAND("mds remove_data_pool " \
- "name=pool,type=CephString", \
- "remove data pool <pool>", "mds", "rw", "cli,rest")
+ "name=poolid,type=CephInt,range=0", \
+ "remove data pool <poolid>", "mds", "rw", "cli,rest")
COMMAND("mds newfs " \
"name=metadata,type=CephInt,range=0 " \
"name=data,type=CephInt,range=0 " \
- "name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+ "name=sure,type=CephChoices,strings=--yes-i-really-mean-it", \
"make new filesystom using pools <metadata> and <data>", \
"mds", "rw", "cli,rest")
/*
* Monmap commands
*/
COMMAND("mon dump " \
- "name=epoch,type=CephInt,range=0,req=false", \
+ "name=epoch,type=CephInt,req=false", \
"dump formatted monmap (optionally from epoch)", \
"mon", "r", "cli,rest")
COMMAND("mon stat", "summarize monitor status", "mon", "r", "cli,rest")
@@ -334,11 +325,6 @@ COMMAND("osd getmap " \
COMMAND("osd getcrushmap " \
"name=epoch,type=CephInt,range=0,req=false", \
"get CRUSH map", "osd", "r", "cli,rest")
-COMMAND("osd perf", \
- "print dump of OSD perf summary stats", \
- "osd", \
- "r", \
- "cli,rest")
COMMAND("osd getmaxosd", "show largest OSD id", "osd", "r", "cli,rest")
COMMAND("osd find " \
"name=id,type=CephInt,range=0", \
@@ -378,13 +364,13 @@ COMMAND("osd crush set " \
"name=id,type=CephOsdName " \
"name=weight,type=CephFloat,range=0.0 " \
"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
- "update crushmap position and weight for <name> to <weight> with location <args>", \
+ "set crushmap entry for <name> to <weight> with location <args>", \
"osd", "rw", "cli,rest")
COMMAND("osd crush add " \
"name=id,type=CephOsdName " \
"name=weight,type=CephFloat,range=0.0 " \
"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
- "add or update crushmap position and weight for <name> with <weight> and location <args>", \
+ "add crushmap entry for <name> with <weight> and location <args>", \
"osd", "rw", "cli,rest")
COMMAND("osd crush create-or-move " \
"name=id,type=CephOsdName " \
@@ -393,7 +379,7 @@ COMMAND("osd crush create-or-move " \
"create entry or move existing entry for <name> <weight> at/to location <args>", \
"osd", "rw", "cli,rest")
COMMAND("osd crush move " \
- "name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+ "name=id,type=CephOsdName " \
"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
"move existing entry for <name> to location <args>", \
"osd", "rw", "cli,rest")
@@ -465,7 +451,7 @@ COMMAND("osd reweight " \
"reweight osd to 0.0 < <weight> < 1.0", "osd", "rw", "cli,rest")
COMMAND("osd lost " \
"name=id,type=CephInt,range=0 " \
- "name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+ "name=sure,type=CephChoices,strings=--yes-i-really-mean-it", \
"mark osd as permanently lost. THIS DESTROYS DATA IF NO MORE REPLICAS EXIST, BE CAREFUL", \
"osd", "rw", "cli,rest")
COMMAND("osd create " \
@@ -488,14 +474,13 @@ COMMAND("osd pool rmsnap " \
COMMAND("osd pool create " \
"name=pool,type=CephPoolname " \
"name=pg_num,type=CephInt,range=0 " \
- "name=pgp_num,type=CephInt,range=0,req=false " \
- "name=properties,type=CephString,n=N,req=false,goodchars=[A-Za-z0-9-_.=]", \
+ "name=pgp_num,type=CephInt,range=0,req=false", \
"create pool", "osd", "rw", "cli,rest")
COMMAND("osd pool delete " \
"name=pool,type=CephPoolname " \
- "name=pool2,type=CephPoolname,req=false " \
- "name=sure,type=CephChoices,strings=--yes-i-really-really-mean-it,req=false", \
- "delete pool", \
+ "name=pool2,type=CephPoolname " \
+ "name=sure,type=CephChoices,strings=--yes-i-really-really-mean-it", \
+ "delete pool (say pool twice, add --yes-i-really-really-mean-it)", \
"osd", "rw", "cli,rest")
COMMAND("osd pool rename " \
"name=srcpool,type=CephPoolname " \
@@ -507,7 +492,7 @@ COMMAND("osd pool get " \
"get pool parameter <var>", "osd", "r", "cli,rest")
COMMAND("osd pool set " \
"name=pool,type=CephPoolname " \
- "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool " \
+ "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset " \
"name=val,type=CephInt", \
"set pool parameter <var> to <val>", "osd", "rw", "cli,rest")
// 'val' is a CephString because it can include a unit. Perhaps
@@ -518,10 +503,6 @@ COMMAND("osd pool set-quota " \
"name=field,type=CephChoices,strings=max_objects|max_bytes " \
"name=val,type=CephString",
"set object or byte limit on pool", "osd", "rw", "cli,rest")
-COMMAND("osd pool stats " \
- "name=name,type=CephString,req=false",
- "obtain stats from all pools, or from specified pool",
- "osd", "r", "cli,rest")
COMMAND("osd reweight-by-utilization " \
"name=oload,type=CephInt,range=100,req=false", \
"reweight OSDs by utilization [overload-percentage-for-consideration, default 120]", \
@@ -530,27 +511,6 @@ COMMAND("osd thrash " \
"name=num_epochs,type=CephInt,range=0", \
"thrash OSDs for <num_epochs>", "osd", "rw", "cli,rest")
-// tiering
-COMMAND("osd tier add " \
- "name=pool,type=CephPoolname " \
- "name=tierpool,type=CephPoolname",
- "add the tier <tierpool> to base pool <pool>", "osd", "rw", "cli,rest")
-COMMAND("osd tier remove " \
- "name=pool,type=CephPoolname " \
- "name=tierpool,type=CephPoolname",
- "remove the tier <tierpool> from base pool <pool>", "osd", "rw", "cli,rest")
-COMMAND("osd tier cache-mode " \
- "name=pool,type=CephPoolname " \
- "name=mode,type=CephChoices,strings=none|writeback|invalidate+forward|readonly", \
- "specify the caching mode for cache tier <pool>", "osd", "rw", "cli,rest")
-COMMAND("osd tier set-overlay " \
- "name=pool,type=CephPoolname " \
- "name=overlaypool,type=CephPoolname", \
- "set the overlay pool for base pool <pool> to be <overlaypool>", "osd", "rw", "cli,rest")
-COMMAND("osd tier remove-overlay " \
- "name=pool,type=CephPoolname ", \
- "remove the overlay pool for base pool <pool>", "osd", "rw", "cli,rest")
-
/*
* mon/ConfigKeyService.cc
*/
diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc
index 1650a99..511d714 100644
--- a/src/mon/Elector.cc
+++ b/src/mon/Elector.cc
@@ -61,6 +61,7 @@ void Elector::bump_epoch(epoch_t e)
// clear up some state
electing_me = false;
acked_me.clear();
+ classic_mons.clear();
}
@@ -73,6 +74,8 @@ void Elector::start()
dout(5) << "start -- can i be leader?" << dendl;
acked_me.clear();
+ classic_mons.clear();
+ required_features = mon->apply_compatset_features_to_quorum_requirements();
init();
// start by trying to elect me
@@ -100,14 +103,16 @@ void Elector::defer(int who)
if (electing_me) {
// drop out
acked_me.clear();
+ classic_mons.clear();
electing_me = false;
}
// ack them
leader_acked = who;
ack_stamp = ceph_clock_now(g_ceph_context);
- mon->messenger->send_message(new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap),
- mon->monmap->get_inst(who));
+ MMonElection *m = new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap);
+ m->sharing_bl = mon->get_supported_commands_bl();
+ mon->messenger->send_message(m, mon->monmap->get_inst(who));
// set a timer
reset_timer(1.0); // give the leader some extra time to declare victory
@@ -159,14 +164,31 @@ void Elector::victory()
++p) {
quorum.insert(p->first);
features &= p->second;
- }
+ }
+
+ // decide what command set we're supporting
+ bool use_classic_commands = !classic_mons.empty();
+ // keep a copy to share with the monitor; we clear classic_mons in bump_epoch
+ set<int> copy_classic_mons = classic_mons;
cancel_timer();
assert(epoch % 2 == 1); // election
bump_epoch(epoch+1); // is over!
+
+ // decide my supported commands for peons to advertise
+ const bufferlist *cmds_bl = NULL;
+ const MonCommand *cmds;
+ int cmdsize;
+ if (use_classic_commands) {
+ mon->get_classic_monitor_commands(&cmds, &cmdsize);
+ cmds_bl = &mon->get_classic_commands_bl();
+ } else {
+ mon->get_locally_supported_monitor_commands(&cmds, &cmdsize);
+ cmds_bl = &mon->get_supported_commands_bl();
+ }
- // tell everyone
+ // tell everyone!
for (set<int>::iterator p = quorum.begin();
p != quorum.end();
++p) {
@@ -174,11 +196,12 @@ void Elector::victory()
MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, epoch, mon->monmap);
m->quorum = quorum;
m->quorum_features = features;
+ m->sharing_bl = *cmds_bl;
mon->messenger->send_message(m, mon->monmap->get_inst(*p));
}
// tell monitor
- mon->win_election(epoch, quorum, features);
+ mon->win_election(epoch, quorum, features, cmds, cmdsize, ©_classic_mons);
}
@@ -188,10 +211,15 @@ void Elector::handle_propose(MMonElection *m)
int from = m->get_source().num();
assert(m->epoch % 2 == 1); // election
- if (m->epoch > epoch) {
+ if ((required_features ^ m->get_connection()->get_features()) &
+ required_features) {
+ dout(5) << " ignoring propose from mon" << from
+ << " without required features" << dendl;
+ nak_old_peer(m);
+ return;
+ } else if (m->epoch > epoch) {
bump_epoch(m->epoch);
- }
- else if (m->epoch < epoch) {
+ } else if (m->epoch < epoch) {
// got an "old" propose,
if (epoch % 2 == 0 && // in a non-election cycle
mon->quorum.count(from) == 0) { // from someone outside the quorum
@@ -251,6 +279,8 @@ void Elector::handle_ack(MMonElection *m)
if (electing_me) {
// thanks
acked_me[from] = m->get_connection()->get_features();
+ if (!m->sharing_bl.length())
+ classic_mons.insert(from);
dout(5) << " so far i have " << acked_me << dendl;
// is that _everyone_?
@@ -292,12 +322,60 @@ void Elector::handle_victory(MMonElection *m)
mon->lose_election(epoch, m->quorum, from, m->quorum_features);
// cancel my timer
- cancel_timer();
+ cancel_timer();
+
+ // stash leader's commands
+ if (m->sharing_bl.length()) {
+ MonCommand *new_cmds;
+ int cmdsize;
+ bufferlist::iterator bi = m->sharing_bl.begin();
+ MonCommand::decode_array(&new_cmds, &cmdsize, bi);
+ mon->set_leader_supported_commands(new_cmds, cmdsize);
+ } else { // they are a legacy monitor; use known legacy command set
+ const MonCommand *new_cmds;
+ int cmdsize;
+ mon->get_classic_monitor_commands(&new_cmds, &cmdsize);
+ mon->set_leader_supported_commands(new_cmds, cmdsize);
+ }
+
m->put();
}
+void Elector::nak_old_peer(MMonElection *m)
+{
+ uint64_t supported_features = m->get_connection()->get_features();
+ if (supported_features & CEPH_FEATURE_OSDMAP_ENC) {
+ uint64_t required_features = mon->apply_compatset_features_to_quorum_requirements();
+ dout(10) << "sending nak to peer " << m->get_source()
+ << " that only supports " << supported_features
+ << " of the required " << required_features << dendl;
+
+ MMonElection *reply = new MMonElection(MMonElection::OP_NAK, m->epoch,
+ mon->monmap);
+ reply->quorum_features = required_features;
+ mon->features.encode(reply->sharing_bl);
+ mon->messenger->send_message(reply, m->get_connection());
+ }
+ m->put();
+}
+void Elector::handle_nak(MMonElection *m)
+{
+ dout(1) << "handle_nak from " << m->get_source()
+ << " quorum_features " << m->quorum_features << dendl;
+
+ CompatSet other;
+ bufferlist::iterator bi = m->sharing_bl.begin();
+ other.decode(bi);
+ CompatSet diff = Monitor::get_supported_features().unsupported(other);
+
+ derr << "Shutting down because I do not support required monitor features: { "
+ << diff << " }" << dendl;
+
+ exit(0);
+ // the end!
+}
void Elector::dispatch(Message *m)
{
@@ -378,6 +456,9 @@ void Elector::dispatch(Message *m)
case MMonElection::OP_VICTORY:
handle_victory(em);
return;
+ case MMonElection::OP_NAK:
+ handle_nak(em);
+ return;
default:
assert(0);
}
diff --git a/src/mon/Elector.h b/src/mon/Elector.h
index f1f19b4..b88e830 100644
--- a/src/mon/Elector.h
+++ b/src/mon/Elector.h
@@ -114,6 +114,9 @@ class Elector {
* victory. Also note each peer's feature set.
*/
map<int, uint64_t> acked_me;
+ set<int> classic_mons;
+ /// features which a monitor must hold for us to defer to them
+ uint64_t required_features;
/**
* @}
*/
@@ -242,7 +245,7 @@ class Elector {
* @post We sent a message of type OP_VICTORY to each quorum member.
*/
void victory();
-
+
/**
* Handle a message from some other node proposing himself to become him
* the Leader.
@@ -314,6 +317,31 @@ class Elector {
* @param m A message with an operation type of OP_VICTORY
*/
void handle_victory(class MMonElection *m);
+ /**
+ * Send a nak to a peer who's out of date, containing information about why.
+ *
+ * If we get a message from a peer who can't support the required quorum
+ * features, we have to ignore them. This function will at least send
+ * them a message about *why* they're being ignored -- if they're new
+ * enough to support such a message.
+ *
+ * @param m A message from a monitor not supporting required features. We
+ * take ownership of the reference.
+ */
+ void nak_old_peer(class MMonElection *m);
+ /**
+ * Handle a message from some other participant declaring
+ * we cannot join the quorum.
+ *
+ * Apparently the quorum requires some feature that we do not implement. Shut
+ * down gracefully.
+ *
+ * @pre Election is on-going.
+ * @post We've shut down.
+ *
+ * @param m A message with an operation type of OP_NAK
+ */
+ void handle_nak(class MMonElection *m);
public:
/**
@@ -322,11 +350,12 @@ class Elector {
* @param m A Monitor instance
*/
Elector(Monitor *m) : mon(m),
- expire_event(0),
- epoch(0),
- participating(true),
- electing_me(false),
- leader_acked(-1) { }
+ expire_event(0),
+ epoch(0),
+ participating(true),
+ electing_me(false),
+ required_features(0),
+ leader_acked(-1) { }
/**
* Initiate the Elector class.
diff --git a/src/mon/LogMonitor.cc b/src/mon/LogMonitor.cc
index a545242..8ed5ed9 100644
--- a/src/mon/LogMonitor.cc
+++ b/src/mon/LogMonitor.cc
@@ -30,6 +30,7 @@
#include "common/config.h"
#include "include/assert.h"
#include "include/str_list.h"
+#include "include/compat.h"
#define dout_subsys ceph_subsys_mon
#undef dout_prefix
@@ -44,7 +45,7 @@ ostream& operator<<(ostream& out, LogMonitor& pm)
{
/*
std::stringstream ss;
- for (hash_map<int,int>::iterator p = pm.pg_map.num_pg_by_state.begin();
+ for (ceph::unordered_map<int,int>::iterator p = pm.pg_map.num_pg_by_state.begin();
p != pm.pg_map.num_pg_by_state.end();
++p) {
if (p != pm.pg_map.num_pg_by_state.begin())
@@ -381,7 +382,8 @@ bool LogMonitor::prepare_command(MMonCommand *m)
le.msg = str_join(logtext, " ");
pending_summary.add(le);
pending_log.insert(pair<utime_t,LogEntry>(le.stamp, le));
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, string(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, string(),
+ get_last_committed() + 1));
return true;
}
diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index b865c37..0f7bf27 100644
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -39,6 +39,8 @@
#include "common/cmdparse.h"
#include "include/str_list.h"
+#include "mds/mdstypes.h"
+
#define dout_subsys ceph_subsys_mon
#undef dout_prefix
#define dout_prefix _prefix(_dout, mon, mdsmap)
@@ -54,7 +56,7 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon, MDSMap& mdsmap) {
void MDSMonitor::print_map(MDSMap &m, int dbl)
{
- dout(7) << "print_map\n";
+ dout(dbl) << "print_map\n";
m.print(*_dout);
*_dout << dendl;
}
@@ -66,7 +68,7 @@ void MDSMonitor::create_new_fs(MDSMap &m, int metadata_pool, int data_pool)
m.data_pools.insert(data_pool);
m.metadata_pool = metadata_pool;
m.cas_pool = -1;
- m.compat = get_mdsmap_compat_set();
+ m.compat = get_mdsmap_compat_set_default();
m.session_timeout = g_conf->mds_session_timeout;
m.session_autoclose = g_conf->mds_session_autoclose;
@@ -80,7 +82,7 @@ void MDSMonitor::create_new_fs(MDSMap &m, int metadata_pool, int data_pool)
void MDSMonitor::create_initial()
{
dout(10) << "create_initial" << dendl;
- create_new_fs(pending_mdsmap, CEPH_METADATA_RULE, CEPH_DATA_RULE);
+ create_new_fs(pending_mdsmap, MDS_METADATA_POOL, MDS_DATA_POOL);
}
@@ -124,7 +126,8 @@ void MDSMonitor::encode_pending(MonitorDBStore::Transaction *t)
pending_mdsmap.modified = ceph_clock_now(g_ceph_context);
- //print_map(pending_mdsmap);
+ // print map iff 'debug mon = 30' or higher
+ print_map(pending_mdsmap, 30);
// apply to paxos
assert(get_last_committed() + 1 == pending_mdsmap.epoch);
@@ -136,6 +139,24 @@ void MDSMonitor::encode_pending(MonitorDBStore::Transaction *t)
put_last_committed(t, pending_mdsmap.epoch);
}
+version_t MDSMonitor::get_trim_to()
+{
+ version_t floor = 0;
+ if (g_conf->mon_mds_force_trim_to > 0 &&
+ g_conf->mon_mds_force_trim_to < (int)get_last_committed()) {
+ floor = g_conf->mon_mds_force_trim_to;
+ dout(10) << __func__ << " explicit mon_mds_force_trim_to = "
+ << floor << dendl;
+ }
+
+ unsigned max = g_conf->mon_max_mdsmap_epochs;
+ version_t last = get_last_committed();
+
+ if (last - get_first_committed() > max && floor < last - max)
+ return last - max;
+ return floor;
+}
+
void MDSMonitor::update_logger()
{
dout(10) << "update_logger" << dendl;
@@ -491,8 +512,8 @@ bool MDSMonitor::prepare_offload_targets(MMDSLoadTargets *m)
bool MDSMonitor::should_propose(double& delay)
{
- delay = 0.0;
- return true;
+ // delegate to PaxosService to assess whether we should propose
+ return PaxosService::should_propose(delay);
}
void MDSMonitor::_updated(MMDSBeacon *m)
@@ -531,6 +552,7 @@ void MDSMonitor::dump_info(Formatter *f)
f->close_section();
f->dump_unsigned("mdsmap_first_committed", get_first_committed());
+ f->dump_unsigned("mdsmap_last_committed", get_last_committed());
}
bool MDSMonitor::preprocess_command(MMonCommand *m)
@@ -570,10 +592,8 @@ bool MDSMonitor::preprocess_command(MMonCommand *m)
}
r = 0;
} else if (prefix == "mds dump") {
- string val;
int64_t epocharg;
epoch_t epoch;
- epoch = epocharg;
MDSMap *p = &mdsmap;
if (cmd_getval(g_ceph_context, cmdmap, "epoch", epocharg)) {
@@ -634,7 +654,6 @@ bool MDSMonitor::preprocess_command(MMonCommand *m)
} else if (prefix == "mds tell") {
string whostr;
cmd_getval(g_ceph_context, cmdmap, "who", whostr);
- string args;
vector<string>args_vec;
cmd_getval(g_ceph_context, cmdmap, "args", args_vec);
@@ -800,13 +819,85 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
}
} else if (prefix == "mds set_max_mds") {
+ // NOTE: see also "mds set max_mds", which can modify the same field.
int64_t maxmds;
if (!cmd_getval(g_ceph_context, cmdmap, "maxmds", maxmds) || maxmds < 0)
goto out;
pending_mdsmap.max_mds = maxmds;
r = 0;
ss << "max_mds = " << pending_mdsmap.max_mds;
-
+ } else if (prefix == "mds set") {
+ string var;
+ if (!cmd_getval(g_ceph_context, cmdmap, "var", var) < 0)
+ goto out;
+ string val;
+ string interr;
+ int64_t n = 0;
+ if (!cmd_getval(g_ceph_context, cmdmap, "val", val))
+ goto out;
+ // we got a string. see if it contains an int.
+ n = strict_strtoll(val.c_str(), 10, &interr);
+ if (var == "max_mds") {
+ // NOTE: see also "mds set_max_mds", which can modify the same field.
+ if (interr.length())
+ goto out;
+ pending_mdsmap.max_mds = n;
+ } else if (var == "inline_data") {
+ if (val == "true" || val == "yes" || (!interr.length() && n == 1)) {
+ string confirm;
+ if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+ confirm != "--yes-i-really-mean-it") {
+ ss << "inline data is new and experimental; you must specify --yes-i-really-mean-it";
+ r = -EPERM;
+ goto out;
+ }
+ ss << "inline data enabled";
+ pending_mdsmap.set_inline_data_enabled(true);
+ pending_mdsmap.compat.incompat.insert(MDS_FEATURE_INCOMPAT_INLINE);
+ } else if (val == "false" || val == "no" ||
+ (!interr.length() && n == 0)) {
+ ss << "inline data disabled";
+ pending_mdsmap.set_inline_data_enabled(false);
+ } else {
+ ss << "value must be false|no|0 or true|yes|1";
+ r = -EINVAL;
+ goto out;
+ }
+ } else if (var == "max_file_size") {
+ if (interr.length()) {
+ ss << var << " requires an integer value";
+ goto out;
+ }
+ if (n < CEPH_MIN_STRIPE_UNIT) {
+ r = -ERANGE;
+ ss << var << " must at least " << CEPH_MIN_STRIPE_UNIT;
+ goto out;
+ }
+ pending_mdsmap.max_file_size = n;
+ } else if (var == "allow_new_snaps") {
+ if (val == "false" || val == "no" || (interr.length() == 0 && n == 0)) {
+ pending_mdsmap.clear_snaps_allowed();
+ ss << "disabled new snapshots";
+ } else if (val == "true" || val == "yes" || (interr.length() == 0 && n == 1)) {
+ string confirm;
+ if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+ confirm != "--yes-i-really-mean-it") {
+ ss << "Snapshots are unstable and will probably break your FS! Set to --yes-i-really-mean-it if you are sure you want to enable them";
+ r = -EPERM;
+ goto out;
+ }
+ pending_mdsmap.set_snaps_allowed();
+ ss << "enabled new snapshots";
+ } else {
+ ss << "value must be true|yes|1 or false|no|0";
+ r = -EINVAL;
+ goto out;
+ }
+ } else {
+ ss << "unknown variable " << var;
+ goto out;
+ }
+ r = 0;
} else if (prefix == "mds setmap") {
MDSMap map;
map.decode(m->get_data());
@@ -819,7 +910,8 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
map.epoch = pending_mdsmap.epoch; // make sure epoch is correct
pending_mdsmap = map;
string rs = "set mds map";
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else {
ss << "next mdsmap epoch " << pending_mdsmap.epoch << " != " << e;
@@ -838,7 +930,8 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
ss << "set mds gid " << gid << " to state " << state << " " << ceph_mds_state_name(state);
string rs;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
@@ -868,7 +961,8 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
ss << "removed mds gid " << gid;
string rs;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} else if (prefix == "mds rmfailed") {
@@ -879,7 +973,8 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
ss << "removed failed mds." << w;
string rs;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "mds cluster_down") {
if (pending_mdsmap.test_flag(CEPH_MDSMAP_DOWN)) {
@@ -920,36 +1015,6 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
r = 0;
}
- } else if (prefix == "mds set") {
- string key;
- cmd_getval(g_ceph_context, cmdmap, "key", key);
- string sure;
- cmd_getval(g_ceph_context, cmdmap, "sure", sure);
- if (key == "allow_new_snaps") {
- if (sure != "--yes-i-really-mean-it") {
- ss << "Snapshots are unstable and will probably break your FS! Add --yes-i-really-mean-it if you are sure";
- r = -EPERM;
- } else {
- pending_mdsmap.set_snaps_allowed();
- ss << "turned on snaps";
- r = 0;
- }
- }
- } else if (prefix == "mds unset") {
- string key;
- cmd_getval(g_ceph_context, cmdmap, "key", key);
- string sure;
- cmd_getval(g_ceph_context, cmdmap, "sure", sure);
- if (key == "allow_new_snaps") {
- if (sure != "--yes-i-really-mean-it") {
- ss << "this won't get rid of snapshots or restore the cluster if it's broken. Add --yes-i-really-mean-it if you are sure";
- r = -EPERM;
- } else {
- pending_mdsmap.clear_snaps_allowed();
- ss << "disabled new snapshots";
- r = 0;
- }
- }
} else if (prefix == "mds add_data_pool") {
string poolname;
cmd_getval(g_ceph_context, cmdmap, "pool", poolname);
@@ -1007,7 +1072,8 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
ss << "new fs with metadata pool " << metadata << " and data pool " << data;
string rs;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} else {
@@ -1019,7 +1085,8 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
if (r >= 0) {
// success.. delay reply
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, r, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, r, rs,
+ get_last_committed() + 1));
return true;
} else {
// reply immediately
@@ -1199,14 +1266,15 @@ void MDSMonitor::tick()
<< " " << ceph_mds_state_name(info.state)
<< " (laggy)" << dendl;
pending_mdsmap.mds_info.erase(gid);
+ do_propose = true;
} else if (!info.laggy()) {
dout(10) << " marking " << gid << " " << info.addr << " mds." << info.rank << "." << info.inc
<< " " << ceph_mds_state_name(info.state)
<< " laggy" << dendl;
info.laggy_since = now;
+ do_propose = true;
}
last_beacon.erase(gid);
- do_propose = true;
}
}
diff --git a/src/mon/MDSMonitor.h b/src/mon/MDSMonitor.h
index e75da1f..901e93e 100644
--- a/src/mon/MDSMonitor.h
+++ b/src/mon/MDSMonitor.h
@@ -65,6 +65,7 @@ class MDSMonitor : public PaxosService {
void create_new_fs(MDSMap &m, int metadata_pool, int data_pool);
+ version_t get_trim_to();
// service methods
void create_initial();
@@ -74,8 +75,6 @@ class MDSMonitor : public PaxosService {
// we don't require full versions; don't encode any.
virtual void encode_full(MonitorDBStore::Transaction *t) { }
- void encode_trim(MonitorDBStore::Transaction *t) { }
-
void update_logger();
void _updated(MMDSBeacon *m);
diff --git a/src/mon/Makefile.am b/src/mon/Makefile.am
index d9094a8..68c6503 100644
--- a/src/mon/Makefile.am
+++ b/src/mon/Makefile.am
@@ -30,6 +30,7 @@ noinst_HEADERS += \
mon/MonCap.h \
mon/MonClient.h \
mon/MonCommands.h \
+ mon/DumplingMonCommands.h \
mon/MonMap.h \
mon/Monitor.h \
mon/MonitorStore.h \
diff --git a/src/mon/MonCap.cc b/src/mon/MonCap.cc
index 644d614..e8d3f7e 100644
--- a/src/mon/MonCap.cc
+++ b/src/mon/MonCap.cc
@@ -349,7 +349,7 @@ struct MonCapParser : qi::grammar<Iterator, MonCap()>
unquoted_word %= +char_("a-zA-Z0-9_.-");
str %= quoted_string | unquoted_word;
- spaces = +lit(' ');
+ spaces = +(lit(' ') | lit('\n') | lit('\t'));
// command := command[=]cmd [k1=v1 k2=v2 ...]
str_match = '=' >> str >> qi::attr(string());
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index f35a0da..a77a5f8 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -895,7 +895,8 @@ void MonClient::_finish_command(MonCommand *r, int ret, string rs)
delete r;
}
-int MonClient::start_mon_command(const vector<string>& cmd, bufferlist& inbl,
+int MonClient::start_mon_command(const vector<string>& cmd,
+ const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish)
{
@@ -913,7 +914,8 @@ int MonClient::start_mon_command(const vector<string>& cmd, bufferlist& inbl,
}
int MonClient::start_mon_command(string name,
- const vector<string>& cmd, bufferlist& inbl,
+ const vector<string>& cmd,
+ const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish)
{
@@ -932,7 +934,8 @@ int MonClient::start_mon_command(string name,
}
int MonClient::start_mon_command(int rank,
- const vector<string>& cmd, bufferlist& inbl,
+ const vector<string>& cmd,
+ const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish)
{
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 0246050..1404f35 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -360,15 +360,15 @@ private:
void handle_mon_command_ack(MMonCommandAck *ack);
public:
- int start_mon_command(const vector<string>& cmd, bufferlist& inbl,
+ int start_mon_command(const vector<string>& cmd, const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish);
int start_mon_command(int mon_rank,
- const vector<string>& cmd, bufferlist& inbl,
+ const vector<string>& cmd, const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish);
int start_mon_command(const string mon_name, ///< mon name, with mon. prefix
- const vector<string>& cmd, bufferlist& inbl,
+ const vector<string>& cmd, const bufferlist& inbl,
bufferlist *outbl, string *outs,
Context *onfinish);
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 881c5a2..bff1f2e 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -248,6 +248,11 @@ COMMAND("mds deactivate name=who,type=CephString", "stop mds", \
COMMAND("mds set_max_mds " \
"name=maxmds,type=CephInt,range=0", \
"set max MDS index", "mds", "rw", "cli,rest")
+COMMAND("mds set " \
+ "name=var,type=CephChoices,strings=max_mds|max_file_size|allow_new_snaps|inline_data " \
+ "name=val,type=CephString " \
+ "name=confirm,type=CephString,req=false", \
+ "set mds parameter <var> to <val>", "mds", "rw", "cli,rest")
COMMAND("mds setmap " \
"name=epoch,type=CephInt,range=0", \
"set mds map; must supply correct epoch number", "mds", "rw", "cli,rest")
@@ -274,15 +279,6 @@ COMMAND("mds compat rm_compat " \
COMMAND("mds compat rm_incompat " \
"name=feature,type=CephInt,range=0", \
"remove incompatible feature", "mds", "rw", "cli,rest")
-COMMAND("mds set " \
- "name=key,type=CephChoices,strings=allow_new_snaps " \
- "name=sure,type=CephString,req=false", \
- "set <key>", \
- "mds", "w", "cli,rest")
-COMMAND("mds unset " \
- "name=key,type=CephChoices,strings=allow_new_snaps " \
- "name=sure,type=CephString,req=false", \
- "unset <key>", "mds", "w", "cli,rest")
COMMAND("mds add_data_pool " \
"name=pool,type=CephString", \
"add data pool <pool>", "mds", "rw", "cli,rest")
@@ -344,6 +340,10 @@ COMMAND("osd find " \
"name=id,type=CephInt,range=0", \
"find osd <id> in the CRUSH map and show its location", \
"osd", "r", "cli,rest")
+COMMAND("osd metadata " \
+ "name=id,type=CephInt,range=0", \
+ "fetch metadata for osd <id>", \
+ "osd", "r", "cli,rest")
COMMAND("osd map " \
"name=pool,type=CephPoolname " \
"name=object,type=CephObjectname", \
@@ -405,17 +405,17 @@ COMMAND("osd crush link " \
COMMAND("osd crush rm " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
- "remove <name> from crush map (everywhere, or just at <ancestor>",\
+ "remove <name> from crush map (everywhere, or just at <ancestor>)",\
"osd", "rw", "cli,rest")
COMMAND("osd crush remove " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
- "remove <name> from crush map (everywhere, or just at <ancestor>", \
+ "remove <name> from crush map (everywhere, or just at <ancestor>)", \
"osd", "rw", "cli,rest")
COMMAND("osd crush unlink " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
- "unlink <name> from crush map (everywhere, or just at <ancestor>", \
+ "unlink <name> from crush map (everywhere, or just at <ancestor>)", \
"osd", "rw", "cli,rest")
COMMAND("osd crush reweight " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
@@ -425,11 +425,14 @@ COMMAND("osd crush reweight " \
COMMAND("osd crush tunables " \
"name=profile,type=CephChoices,strings=legacy|argonaut|bobtail|optimal|default", \
"set crush tunables values to <profile>", "osd", "rw", "cli,rest")
+COMMAND("osd crush show-tunables", \
+ "show current crush tunables", "osd", "r", "cli,rest")
COMMAND("osd crush rule create-simple " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=root,type=CephString,goodchars=[A-Za-z0-9-_.] " \
- "name=type,type=CephString,goodchars=[A-Za-z0-9-_.]",
- "create crush rule <name> in <root> of type <type>", \
+ "name=type,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+ "name=mode,type=CephChoices,strings=firstn|indep,req=false",
+ "create crush rule <name> to start from <root>, replicate across buckets of type <type>, using a choose mode of <firstn|indep> (default firstn; indep best for erasure pools)", \
"osd", "rw", "cli,rest")
COMMAND("osd crush rule rm " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] ", \
@@ -489,6 +492,7 @@ COMMAND("osd pool create " \
"name=pool,type=CephPoolname " \
"name=pg_num,type=CephInt,range=0 " \
"name=pgp_num,type=CephInt,range=0,req=false " \
+ "name=pool_type,type=CephChoices,strings=replicated|erasure,req=false " \
"name=properties,type=CephString,n=N,req=false,goodchars=[A-Za-z0-9-_.=]", \
"create pool", "osd", "rw", "cli,rest")
COMMAND("osd pool delete " \
@@ -507,8 +511,8 @@ COMMAND("osd pool get " \
"get pool parameter <var>", "osd", "r", "cli,rest")
COMMAND("osd pool set " \
"name=pool,type=CephPoolname " \
- "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool " \
- "name=val,type=CephInt", \
+ "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|debug_fake_ec_pool " \
+ "name=val,type=CephString", \
"set pool parameter <var> to <val>", "osd", "rw", "cli,rest")
// 'val' is a CephString because it can include a unit. Perhaps
// there should be a Python type for validation/conversion of strings
@@ -541,7 +545,7 @@ COMMAND("osd tier remove " \
"remove the tier <tierpool> from base pool <pool>", "osd", "rw", "cli,rest")
COMMAND("osd tier cache-mode " \
"name=pool,type=CephPoolname " \
- "name=mode,type=CephChoices,strings=none|writeback|invalidate+forward|readonly", \
+ "name=mode,type=CephChoices,strings=none|writeback|forward|readonly", \
"specify the caching mode for cache tier <pool>", "osd", "rw", "cli,rest")
COMMAND("osd tier set-overlay " \
"name=pool,type=CephPoolname " \
diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc
index 7012e05..cf48123 100644
--- a/src/mon/MonMap.cc
+++ b/src/mon/MonMap.cc
@@ -1,6 +1,7 @@
#include "MonMap.h"
+#include <algorithm>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index cd541f6..35d880c 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -94,6 +94,18 @@ static ostream& _prefix(std::ostream *_dout, const Monitor *mon) {
const string Monitor::MONITOR_NAME = "monitor";
const string Monitor::MONITOR_STORE_PREFIX = "monitor_store";
+
+#undef COMMAND
+MonCommand mon_commands[] = {
+#define COMMAND(parsesig, helptext, modulename, req_perms, avail) \
+ {parsesig, helptext, modulename, req_perms, avail},
+#include <mon/MonCommands.h>
+};
+MonCommand classic_mon_commands[] = {
+#include <mon/DumplingMonCommands.h>
+};
+
+
long parse_pos_long(const char *s, ostream *pss)
{
if (*s == '-' || *s == '+') {
@@ -137,6 +149,8 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
auth_service_required(cct,
cct->_conf->auth_supported.length() ?
cct->_conf->auth_supported : cct->_conf->auth_service_required),
+ leader_supported_mon_commands(NULL),
+ leader_supported_mon_commands_size(0),
store(s),
state(STATE_PROBING),
@@ -213,6 +227,9 @@ Monitor::~Monitor()
delete paxos;
assert(session_map.sessions.empty());
delete mon_caps;
+ if (leader_supported_mon_commands != mon_commands &&
+ leader_supported_mon_commands != classic_mon_commands)
+ delete[] leader_supported_mon_commands;
}
@@ -258,9 +275,17 @@ void Monitor::do_admin_command(string command, cmdmap_t& cmdmap, string format,
return;
}
sync_force(f.get(), ss);
- } else if (command.find("add_bootstrap_peer_hint") == 0)
+ } else if (command.find("add_bootstrap_peer_hint") == 0) {
_add_bootstrap_peer_hint(command, cmdmap, ss);
- else
+ } else if (command == "quorum enter") {
+ elector.start_participating();
+ start_election();
+ ss << "started responding to quorum, initiated new election";
+ } else if (command == "quorum exit") {
+ start_election();
+ elector.stop_participating();
+ ss << "stopped responding to quorum, initiated new election";
+ } else
assert(0 == "bad AdminSocket command binding");
}
@@ -278,6 +303,8 @@ CompatSet Monitor::get_supported_features()
CompatSet::FeatureSet ceph_mon_feature_incompat;
ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_BASE);
ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS);
+ ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
+ ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat,
ceph_mon_feature_incompat);
}
@@ -297,43 +324,43 @@ int Monitor::check_features(MonitorDBStore *store)
CompatSet required = get_supported_features();
CompatSet ondisk;
- bufferlist features;
- store->get(MONITOR_NAME, COMPAT_SET_LOC, features);
- if (features.length() == 0) {
+ read_features_off_disk(store, &ondisk);
+
+ if (!required.writeable(ondisk)) {
+ CompatSet diff = required.unsupported(ondisk);
+ generic_derr << "ERROR: on disk data includes unsupported features: " << diff << dendl;
+ return -EPERM;
+ }
+
+ return 0;
+}
+
+void Monitor::read_features_off_disk(MonitorDBStore *store, CompatSet *features)
+{
+ bufferlist featuresbl;
+ store->get(MONITOR_NAME, COMPAT_SET_LOC, featuresbl);
+ if (featuresbl.length() == 0) {
generic_dout(0) << "WARNING: mon fs missing feature list.\n"
- << "Assuming it is old-style and introducing one." << dendl;
+ << "Assuming it is old-style and introducing one." << dendl;
//we only want the baseline ~v.18 features assumed to be on disk.
//If new features are introduced this code needs to disappear or
//be made smarter.
- ondisk = get_legacy_features();
+ *features = get_legacy_features();
bufferlist bl;
- ondisk.encode(bl);
+ features->encode(bl);
MonitorDBStore::Transaction t;
t.put(MONITOR_NAME, COMPAT_SET_LOC, bl);
store->apply_transaction(t);
} else {
- bufferlist::iterator it = features.begin();
- ondisk.decode(it);
+ bufferlist::iterator it = featuresbl.begin();
+ features->decode(it);
}
-
- if (!required.writeable(ondisk)) {
- CompatSet diff = required.unsupported(ondisk);
- generic_derr << "ERROR: on disk data includes unsupported features: " << diff << dendl;
- return -EPERM;
- }
-
- return 0;
}
void Monitor::read_features()
{
- bufferlist bl;
- store->get(MONITOR_NAME, COMPAT_SET_LOC, bl);
- assert(bl.length());
-
- bufferlist::iterator p = bl.begin();
- ::decode(features, p);
+ read_features_off_disk(store, &features);
dout(10) << "features " << features << dendl;
}
@@ -447,35 +474,39 @@ int Monitor::preinit()
init_paxos();
health_monitor->init();
- // we need to bootstrap authentication keys so we can form an
- // initial quorum.
- if (authmon()->get_last_committed() == 0) {
- dout(10) << "loading initial keyring to bootstrap authentication for mkfs" << dendl;
- bufferlist bl;
- store->get("mkfs", "keyring", bl);
- KeyRing keyring;
- bufferlist::iterator p = bl.begin();
- ::decode(keyring, p);
- extract_save_mon_key(keyring);
- }
-
- string keyring_loc = g_conf->mon_data + "/keyring";
+ int r;
- int r = keyring.load(cct, keyring_loc);
- if (r < 0) {
- EntityName mon_name;
- mon_name.set_type(CEPH_ENTITY_TYPE_MON);
- EntityAuth mon_key;
- if (key_server.get_auth(mon_name, mon_key)) {
- dout(1) << "copying mon. key from old db to external keyring" << dendl;
- keyring.add(mon_name, mon_key);
+ if (is_keyring_required()) {
+ // we need to bootstrap authentication keys so we can form an
+ // initial quorum.
+ if (authmon()->get_last_committed() == 0) {
+ dout(10) << "loading initial keyring to bootstrap authentication for mkfs" << dendl;
bufferlist bl;
- keyring.encode_plaintext(bl);
- write_default_keyring(bl);
- } else {
- derr << "unable to load initial keyring " << g_conf->keyring << dendl;
- lock.Unlock();
- return r;
+ store->get("mkfs", "keyring", bl);
+ KeyRing keyring;
+ bufferlist::iterator p = bl.begin();
+ ::decode(keyring, p);
+ extract_save_mon_key(keyring);
+ }
+
+ string keyring_loc = g_conf->mon_data + "/keyring";
+
+ r = keyring.load(cct, keyring_loc);
+ if (r < 0) {
+ EntityName mon_name;
+ mon_name.set_type(CEPH_ENTITY_TYPE_MON);
+ EntityAuth mon_key;
+ if (key_server.get_auth(mon_name, mon_key)) {
+ dout(1) << "copying mon. key from old db to external keyring" << dendl;
+ keyring.add(mon_name, mon_key);
+ bufferlist bl;
+ keyring.encode_plaintext(bl);
+ write_default_keyring(bl);
+ } else {
+ derr << "unable to load initial keyring " << g_conf->keyring << dendl;
+ lock.Unlock();
+ return r;
+ }
}
}
@@ -504,6 +535,14 @@ int Monitor::preinit()
"add peer address as potential bootstrap"
" peer for cluster bringup");
assert(r == 0);
+ r = admin_socket->register_command("quorum enter", "quorum enter",
+ admin_hook,
+ "force monitor back into quorum");
+ assert(r == 0);
+ r = admin_socket->register_command("quorum exit", "quorum exit",
+ admin_hook,
+ "force monitor out of the quorum");
+ assert(r == 0);
lock.Lock();
lock.Unlock();
@@ -524,6 +563,14 @@ int Monitor::init()
bootstrap();
+ // encode command sets
+ const MonCommand *cmds;
+ int cmdsize;
+ get_locally_supported_monitor_commands(&cmds, &cmdsize);
+ MonCommand::encode_array(cmds, cmdsize, supported_commands_bl);
+ get_classic_monitor_commands(&cmds, &cmdsize);
+ MonCommand::encode_array(cmds, cmdsize, classic_commands_bl);
+
lock.Unlock();
return 0;
}
@@ -821,7 +868,7 @@ void Monitor::sync_obtain_latest_monmap(bufferlist &bl)
dout(1) << __func__ << " obtained monmap e" << latest_monmap.epoch << dendl;
- latest_monmap.encode(bl, CEPH_FEATURES_ALL);
+ latest_monmap.encode(bl, quorum_features);
}
void Monitor::sync_reset_requester()
@@ -1473,7 +1520,11 @@ void Monitor::win_standalone_election()
assert(rank == 0);
set<int> q;
q.insert(rank);
- win_election(1, q, CEPH_FEATURES_ALL);
+
+ const MonCommand *my_cmds;
+ int cmdsize;
+ get_locally_supported_monitor_commands(&my_cmds, &cmdsize);
+ win_election(1, q, CEPH_FEATURES_ALL, my_cmds, cmdsize, NULL);
}
const utime_t& Monitor::get_leader_since() const
@@ -1487,7 +1538,9 @@ epoch_t Monitor::get_epoch()
return elector.get_epoch();
}
-void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features)
+void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,
+ const MonCommand *cmdset, int cmdsize,
+ const set<int> *classic_monitors)
{
dout(10) << __func__ << " epoch " << epoch << " quorum " << active
<< " features " << features << dendl;
@@ -1502,6 +1555,10 @@ void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features)
clog.info() << "mon." << name << "@" << rank
<< " won leader election with quorum " << quorum << "\n";
+ set_leader_supported_commands(cmdset, cmdsize);
+ if (classic_monitors)
+ classic_mons = *classic_monitors;
+
paxos->leader_init();
for (vector<PaxosService*>::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)
(*p)->election_finished();
@@ -1533,6 +1590,7 @@ void Monitor::lose_election(epoch_t epoch, set<int> &q, int l, uint64_t features
void Monitor::finish_election()
{
+ apply_quorum_to_compatset_features();
timecheck_finish();
exited_quorum = utime_t();
finish_contexts(g_ceph_context, waitfor_quorum);
@@ -1550,6 +1608,37 @@ void Monitor::finish_election()
}
}
+void Monitor::apply_quorum_to_compatset_features()
+{
+ CompatSet new_features(features);
+ if (quorum_features & CEPH_FEATURE_OSD_ERASURE_CODES) {
+ new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
+ }
+ if (quorum_features & CEPH_FEATURE_OSDMAP_ENC) {
+ new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
+ }
+
+ if (new_features.compare(features) != 0) {
+ CompatSet diff = features.unsupported(new_features);
+ dout(1) << "Enabling new quorum features: " << diff << dendl;
+ features = new_features;
+ MonitorDBStore::Transaction t;
+ write_features(t);
+ store->apply_transaction(t);
+ }
+}
+
+uint64_t Monitor::apply_compatset_features_to_quorum_requirements()
+{
+ uint64_t required_features = 0;
+ if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES)) {
+ required_features |= CEPH_FEATURE_OSD_ERASURE_CODES;
+ }
+ if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC)) {
+ required_features |= CEPH_FEATURE_OSDMAP_ENC;
+ }
+ return required_features;
+}
void Monitor::sync_force(Formatter *f, ostream& ss)
{
@@ -1703,17 +1792,17 @@ void Monitor::get_health(string& status, bufferlist *detailbl, Formatter *f)
stringstream ss;
health_status_t overall = HEALTH_OK;
if (!summary.empty()) {
- if (f) {
- f->open_object_section("item");
- f->dump_stream("severity") << summary.front().first;
- f->dump_string("summary", summary.front().second);
- f->close_section();
- }
ss << ' ';
while (!summary.empty()) {
if (overall > summary.front().first)
overall = summary.front().first;
ss << summary.front().second;
+ if (f) {
+ f->open_object_section("item");
+ f->dump_stream("severity") << summary.front().first;
+ f->dump_string("summary", summary.front().second);
+ f->close_section();
+ }
summary.pop_front();
if (!summary.empty())
ss << "; ";
@@ -1858,17 +1947,9 @@ void Monitor::get_status(stringstream &ss, Formatter *f)
}
}
-#undef COMMAND
-MonCommand mon_commands[] = {
-#define COMMAND(parsesig, helptext, modulename, req_perms, avail) \
- {parsesig, helptext, modulename, req_perms, avail},
-#include <mon/MonCommands.h>
-};
-
-bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix,
- map<string,cmd_vartype>& cmdmap) {
-
- map<string,string> strmap;
+void Monitor::_generate_command_map(map<string,cmd_vartype>& cmdmap,
+ map<string,string> ¶m_str_map)
+{
for (map<string,cmd_vartype>::const_iterator p = cmdmap.begin();
p != cmdmap.end(); ++p) {
if (p->first == "prefix")
@@ -1879,39 +1960,51 @@ bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix,
cv.size() % 2 == 0) {
for (unsigned i = 0; i < cv.size(); i += 2) {
string k = string("caps_") + cv[i];
- strmap[k] = cv[i + 1];
+ param_str_map[k] = cv[i + 1];
}
continue;
}
}
- strmap[p->first] = cmd_vartype_stringify(p->second);
+ param_str_map[p->first] = cmd_vartype_stringify(p->second);
}
+}
+const MonCommand *Monitor::_get_moncommand(const string &cmd_prefix,
+ MonCommand *cmds, int cmds_size)
+{
MonCommand *this_cmd = NULL;
- for (MonCommand *cp = mon_commands;
- cp < &mon_commands[ARRAY_SIZE(mon_commands)]; cp++) {
- if (cp->cmdstring.find(prefix) != string::npos) {
+ for (MonCommand *cp = cmds;
+ cp < &cmds[cmds_size]; cp++) {
+ if (cp->cmdstring.find(cmd_prefix) != string::npos) {
this_cmd = cp;
break;
}
}
- assert(this_cmd != NULL);
+ return this_cmd;
+}
+
+bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix,
+ const map<string,cmd_vartype>& cmdmap,
+ const map<string,string> param_str_map,
+ const MonCommand *this_cmd) {
+
bool cmd_r = (this_cmd->req_perms.find('r') != string::npos);
bool cmd_w = (this_cmd->req_perms.find('w') != string::npos);
bool cmd_x = (this_cmd->req_perms.find('x') != string::npos);
bool capable = s->caps.is_capable(g_ceph_context, s->inst.name,
- module, prefix, strmap,
+ module, prefix, param_str_map,
cmd_r, cmd_w, cmd_x);
dout(10) << __func__ << " " << (capable ? "" : "not ") << "capable" << dendl;
return capable;
}
-void get_command_descriptions(const MonCommand *commands,
- unsigned commands_size,
- Formatter *f,
- bufferlist *rdata) {
+void Monitor::format_command_descriptions(const MonCommand *commands,
+ unsigned commands_size,
+ Formatter *f,
+ bufferlist *rdata)
+{
int cmdnum = 0;
f->open_object_section("command_descriptions");
for (const MonCommand *cp = commands;
@@ -1929,6 +2022,42 @@ void get_command_descriptions(const MonCommand *commands,
f->flush(*rdata);
}
+void Monitor::get_locally_supported_monitor_commands(const MonCommand **cmds,
+ int *count)
+{
+ *cmds = mon_commands;
+ *count = ARRAY_SIZE(mon_commands);
+}
+void Monitor::get_leader_supported_commands(const MonCommand **cmds, int *count)
+{
+ *cmds = leader_supported_mon_commands;
+ *count = leader_supported_mon_commands_size;
+}
+void Monitor::get_classic_monitor_commands(const MonCommand **cmds, int *count)
+{
+ *cmds = classic_mon_commands;
+ *count = ARRAY_SIZE(classic_mon_commands);
+}
+void Monitor::set_leader_supported_commands(const MonCommand *cmds, int size)
+{
+ if (leader_supported_mon_commands != mon_commands &&
+ leader_supported_mon_commands != classic_mon_commands)
+ delete[] leader_supported_mon_commands;
+ leader_supported_mon_commands = cmds;
+ leader_supported_mon_commands_size = size;
+}
+
+bool Monitor::is_keyring_required()
+{
+ string auth_cluster_required = g_conf->auth_supported.length() ?
+ g_conf->auth_supported : g_conf->auth_cluster_required;
+ string auth_service_required = g_conf->auth_supported.length() ?
+ g_conf->auth_supported : g_conf->auth_service_required;
+
+ return auth_service_required == "cephx" ||
+ auth_cluster_required == "cephx";
+}
+
void Monitor::handle_command(MMonCommand *m)
{
if (m->fsid != monmap->fsid) {
@@ -1974,7 +2103,8 @@ void Monitor::handle_command(MMonCommand *m)
if (prefix == "get_command_descriptions") {
bufferlist rdata;
Formatter *f = new_formatter("json");
- get_command_descriptions(mon_commands, ARRAY_SIZE(mon_commands), f, &rdata);
+ format_command_descriptions(leader_supported_mon_commands,
+ leader_supported_mon_commands_size, f, &rdata);
delete f;
reply_command(m, 0, "", rdata, 0);
return;
@@ -1992,7 +2122,31 @@ void Monitor::handle_command(MMonCommand *m)
get_str_vec(prefix, fullcmd);
module = fullcmd[0];
- if (!_allowed_command(session, module, prefix, cmdmap)) {
+ // validate command is in leader map
+
+ const MonCommand *leader_cmd;
+ leader_cmd = _get_moncommand(prefix,
+ // the boost underlying this isn't const for some reason
+ const_cast<MonCommand*>(leader_supported_mon_commands),
+ leader_supported_mon_commands_size);
+ if (!leader_cmd) {
+ reply_command(m, -EINVAL, "command not known", 0);
+ return;
+ }
+ // validate command is in our map & matches, or forward
+ const MonCommand *mon_cmd = _get_moncommand(prefix, mon_commands,
+ ARRAY_SIZE(mon_commands));
+ if (!is_leader() && (!mon_cmd ||
+ (*leader_cmd != *mon_cmd))) {
+ dout(10) << "We don't match leader, forwarding request " << m << dendl;
+ forward_request_leader(m);
+ return;
+ }
+ // validate user's permissions for requested command
+ map<string,string> param_str_map;
+ _generate_command_map(cmdmap, param_str_map);
+ if (!_allowed_command(session, module, prefix, cmdmap,
+ param_str_map, mon_cmd)) {
dout(1) << __func__ << " access denied" << dendl;
reply_command(m, -EACCES, "access denied", 0);
return;
@@ -2279,6 +2433,7 @@ void Monitor::forward_request_leader(PaxosServiceMessage *req)
rr->tid = ++routed_request_tid;
rr->client_inst = req->get_source_inst();
rr->con = req->get_connection();
+ rr->con_features = rr->con->get_features();
encode_message(req, CEPH_FEATURES_ALL, rr->request_bl); // for my use only; use all features
rr->session = static_cast<MonSession *>(session->get());
routed_requests[rr->tid] = rr;
@@ -2286,7 +2441,9 @@ void Monitor::forward_request_leader(PaxosServiceMessage *req)
dout(10) << "forward_request " << rr->tid << " request " << *req << dendl;
- MForward *forward = new MForward(rr->tid, req, rr->session->caps);
+ MForward *forward = new MForward(rr->tid, req,
+ rr->con_features,
+ rr->session->caps);
forward->set_priority(req->get_priority());
messenger->send_message(forward, monmap->get_inst(mon));
} else {
@@ -2314,6 +2471,7 @@ void Monitor::handle_forward(MForward *m)
c->set_priv(s);
c->set_peer_addr(m->client.addr);
c->set_peer_type(m->client.name.type());
+ c->set_features(m->con_features);
s->caps = m->client_caps;
dout(10) << " caps are " << s->caps << dendl;
@@ -2356,7 +2514,7 @@ void Monitor::try_send_message(Message *m, const entity_inst_t& to)
dout(10) << "try_send_message " << *m << " to " << to << dendl;
bufferlist bl;
- encode_message(m, CEPH_FEATURES_ALL, bl); // fixme: assume peers have all features we do.
+ encode_message(m, quorum_features, bl);
messenger->send_message(m, to);
@@ -2477,7 +2635,8 @@ void Monitor::resend_routed_requests()
delete rr;
} else {
dout(10) << " resend to mon." << mon << " tid " << rr->tid << " " << *req << dendl;
- MForward *forward = new MForward(rr->tid, req, rr->session->caps);
+ MForward *forward = new MForward(rr->tid, req, rr->con_features,
+ rr->session->caps);
forward->client = rr->client_inst;
forward->set_priority(req->get_priority());
messenger->send_message(forward, monmap->get_inst(mon));
@@ -3661,25 +3820,43 @@ int Monitor::mkfs(bufferlist& osdmapbl)
t.put("mkfs", "osdmap", osdmapbl);
}
- KeyRing keyring;
- string keyring_filename;
- if (!ceph_resolve_file_search(g_conf->keyring, keyring_filename)) {
- derr << "unable to find a keyring file on " << g_conf->keyring << dendl;
- return -ENOENT;
- }
-
- r = keyring.load(g_ceph_context, keyring_filename);
- if (r < 0) {
- derr << "unable to load initial keyring " << g_conf->keyring << dendl;
- return r;
- }
+ if (is_keyring_required()) {
+ KeyRing keyring;
+ string keyring_filename;
+ if (!ceph_resolve_file_search(g_conf->keyring, keyring_filename)) {
+ derr << "unable to find a keyring file on " << g_conf->keyring << dendl;
+ if (g_conf->key != "") {
+ string keyring_plaintext = "[mon.]\n\tkey = " + g_conf->key +
+ "\n\tcaps mon = \"allow *\"\n";
+ bufferlist bl;
+ bl.append(keyring_plaintext);
+ try {
+ bufferlist::iterator i = bl.begin();
+ keyring.decode_plaintext(i);
+ }
+ catch (const buffer::error& e) {
+ derr << "error decoding keyring " << keyring_plaintext
+ << ": " << e.what() << dendl;
+ return -EINVAL;
+ }
+ } else {
+ return -ENOENT;
+ }
+ } else {
+ r = keyring.load(g_ceph_context, keyring_filename);
+ if (r < 0) {
+ derr << "unable to load initial keyring " << g_conf->keyring << dendl;
+ return r;
+ }
+ }
- // put mon. key in external keyring; seed with everything else.
- extract_save_mon_key(keyring);
+ // put mon. key in external keyring; seed with everything else.
+ extract_save_mon_key(keyring);
- bufferlist keyringbl;
- keyring.encode_plaintext(keyringbl);
- t.put("mkfs", "keyring", keyringbl);
+ bufferlist keyringbl;
+ keyring.encode_plaintext(keyringbl);
+ t.put("mkfs", "keyring", keyringbl);
+ }
write_fsid(t);
store->apply_transaction(t);
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 2c066f6..47fa658 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -51,7 +51,7 @@
#include "mon/MonitorDBStore.h"
#include <memory>
-#include <tr1/memory>
+#include "include/memory.h"
#include <errno.h>
@@ -102,6 +102,7 @@ struct MRoute;
struct MForward;
struct MTimeCheck;
struct MMonHealth;
+struct MonCommand;
#define COMPAT_SET_LOC "feature_set"
@@ -139,6 +140,9 @@ public:
CompatSet features;
+ const MonCommand *leader_supported_mon_commands;
+ int leader_supported_mon_commands_size;
+
private:
void new_tick();
friend class C_Mon_Tick;
@@ -197,6 +201,9 @@ private:
utime_t leader_since; // when this monitor became the leader, if it is the leader
utime_t exited_quorum; // time detected as not in quorum; 0 if in
uint64_t quorum_features; ///< intersection of quorum member feature bits
+ bufferlist supported_commands_bl; // encoded MonCommands we support
+ bufferlist classic_commands_bl; // encoded MonCommands supported by Dumpling
+ set<int> classic_mons; // set of "classic" monitors; only valid on leader
set<string> outside_quorum;
@@ -518,6 +525,8 @@ public:
uint64_t get_quorum_features() const {
return quorum_features;
}
+ void apply_quorum_to_compatset_features();
+ uint64_t apply_compatset_features_to_quorum_requirements();
private:
void _reset(); ///< called from bootstrap, start_, or join_election
@@ -526,12 +535,25 @@ public:
void join_election();
void start_election();
void win_standalone_election();
+ // end election (called by Elector)
void win_election(epoch_t epoch, set<int>& q,
- uint64_t features); // end election (called by Elector)
+ uint64_t features,
+ const MonCommand *cmdset, int cmdsize,
+ const set<int> *classic_monitors);
void lose_election(epoch_t epoch, set<int>& q, int l,
uint64_t features); // end election (called by Elector)
void finish_election();
+ const bufferlist& get_supported_commands_bl() {
+ return supported_commands_bl;
+ }
+ const bufferlist& get_classic_commands_bl() {
+ return classic_commands_bl;
+ }
+ const set<int>& get_classic_mons() {
+ return classic_mons;
+ }
+
void update_logger();
/**
@@ -588,8 +610,14 @@ public:
void handle_get_version(MMonGetVersion *m);
void handle_subscribe(MMonSubscribe *m);
void handle_mon_get_map(MMonGetMap *m);
- bool _allowed_command(MonSession *s, string &module, string& prefix,
- map<string,cmd_vartype>& cmdmap);
+ static void _generate_command_map(map<string,cmd_vartype>& cmdmap,
+ map<string,string> ¶m_str_map);
+ static const MonCommand *_get_moncommand(const string &cmd_prefix,
+ MonCommand *cmds, int cmds_size);
+ bool _allowed_command(MonSession *s, string &module, string &prefix,
+ const map<string,cmd_vartype>& cmdmap,
+ const map<string,string> param_str_map,
+ const MonCommand *this_cmd);
void _mon_status(Formatter *f, ostream& ss);
void _quorum_status(Formatter *f, ostream& ss);
void _add_bootstrap_peer_hint(string cmd, cmdmap_t& cmdmap, ostream& ss);
@@ -633,8 +661,10 @@ public:
bufferlist request_bl;
MonSession *session;
ConnectionRef con;
+ uint64_t con_features;
entity_inst_t client_inst;
+ RoutedRequest() : tid(0), session(NULL), con_features(0) {}
~RoutedRequest() {
if (session)
session->put();
@@ -722,6 +752,8 @@ public:
// features
static CompatSet get_supported_features();
static CompatSet get_legacy_features();
+ /// read the ondisk features into the CompatSet pointed to by read_features
+ static void read_features_off_disk(MonitorDBStore *store, CompatSet *read_features);
void read_features();
void write_features(MonitorDBStore::Transaction &t);
@@ -843,11 +875,25 @@ public:
void _convert_machines();
void _convert_paxos();
};
+
+ static void format_command_descriptions(const MonCommand *commands,
+ unsigned commands_size,
+ Formatter *f,
+ bufferlist *rdata);
+ void get_locally_supported_monitor_commands(const MonCommand **cmds, int *count);
+ void get_classic_monitor_commands(const MonCommand **cmds, int *count);
+ void get_leader_supported_commands(const MonCommand **cmds, int *count);
+ /// the Monitor owns this pointer once you pass it in
+ void set_leader_supported_commands(const MonCommand *cmds, int size);
+ static bool is_keyring_required();
};
#define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")
#define CEPH_MON_FEATURE_INCOMPAT_GV CompatSet::Feature (2, "global version sequencing (v0.52)")
#define CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS CompatSet::Feature (3, "single paxos with k/v store (v0.\?)")
+#define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools")
+#define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding")
+// make sure you add your feature to Monitor::get_supported_features
long parse_pos_long(const char *s, ostream *pss = NULL);
@@ -857,11 +903,53 @@ struct MonCommand {
string module;
string req_perms;
string availability;
-};
-void get_command_descriptions(const MonCommand *commands,
- unsigned commands_size,
- Formatter *f,
- bufferlist *rdata);
+ void encode(bufferlist &bl) const {
+ /*
+ * very naughty: deliberately unversioned because individual commands
+ * shouldn't be encoded standalone, only as a full set (which we do
+ * version, see encode_array() below).
+ */
+ ::encode(cmdstring, bl);
+ ::encode(helpstring, bl);
+ ::encode(module, bl);
+ ::encode(req_perms, bl);
+ ::encode(availability, bl);
+ }
+ void decode(bufferlist::iterator &bl) {
+ ::decode(cmdstring, bl);
+ ::decode(helpstring, bl);
+ ::decode(module, bl);
+ ::decode(req_perms, bl);
+ ::decode(availability, bl);
+ }
+ bool operator==(const MonCommand& o) const {
+ return cmdstring == o.cmdstring && helpstring == o.helpstring &&
+ module == o.module && req_perms == o.req_perms &&
+ availability == o.availability;
+ }
+ bool operator!=(const MonCommand& o) const {
+ return !(*this == o);
+ }
+
+ static void encode_array(const MonCommand *cmds, int size, bufferlist &bl) {
+ ENCODE_START(1, 1, bl);
+ uint16_t s = size;
+ ::encode(s, bl);
+ ::encode_array_nohead(cmds, size, bl);
+ ENCODE_FINISH(bl);
+ }
+ static void decode_array(MonCommand **cmds, int *size,
+ bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ uint16_t s = 0;
+ ::decode(s, bl);
+ *size = s;
+ *cmds = new MonCommand[*size];
+ ::decode_array_nohead(*cmds, *size, bl);
+ DECODE_FINISH(bl);
+ }
+};
+WRITE_CLASS_ENCODER(MonCommand);
#endif
diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index eda5aaf..88c4f93 100644
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -322,7 +322,7 @@ class MonitorDBStore
virtual void get_chunk_tx(Transaction &tx, uint64_t max) = 0;
virtual pair<string,string> get_next_key() = 0;
};
- typedef std::tr1::shared_ptr<StoreIteratorImpl> Synchronizer;
+ typedef ceph::shared_ptr<StoreIteratorImpl> Synchronizer;
class WholeStoreIteratorImpl : public StoreIteratorImpl {
KeyValueDB::WholeSpaceIterator iter;
@@ -389,7 +389,7 @@ class MonitorDBStore
else
iter->seek_to_first();
- return std::tr1::shared_ptr<StoreIteratorImpl>(
+ return ceph::shared_ptr<StoreIteratorImpl>(
new WholeStoreIteratorImpl(iter, prefixes)
);
}
@@ -486,19 +486,33 @@ class MonitorDBStore
db->submit_transaction_sync(dbt);
}
+ void init_options() {
+ db->init();
+ if (g_conf->mon_leveldb_write_buffer_size)
+ db->options.write_buffer_size = g_conf->mon_leveldb_write_buffer_size;
+ if (g_conf->mon_leveldb_cache_size)
+ db->options.cache_size = g_conf->mon_leveldb_cache_size;
+ if (g_conf->mon_leveldb_block_size)
+ db->options.block_size = g_conf->mon_leveldb_block_size;
+ if (g_conf->mon_leveldb_bloom_size)
+ db->options.bloom_size = g_conf->mon_leveldb_bloom_size;
+ if (g_conf->mon_leveldb_compression)
+ db->options.compression_enabled = g_conf->mon_leveldb_compression;
+ if (g_conf->mon_leveldb_max_open_files)
+ db->options.max_open_files = g_conf->mon_leveldb_max_open_files;
+ if (g_conf->mon_leveldb_paranoid)
+ db->options.paranoid_checks = g_conf->mon_leveldb_paranoid;
+ if (g_conf->mon_leveldb_log.length())
+ db->options.log_file = g_conf->mon_leveldb_log;
+ }
+
int open(ostream &out) {
- db->options.write_buffer_size = g_conf->mon_leveldb_write_buffer_size;
- db->options.cache_size = g_conf->mon_leveldb_cache_size;
- db->options.block_size = g_conf->mon_leveldb_block_size;
- db->options.bloom_size = g_conf->mon_leveldb_bloom_size;
- db->options.compression_enabled = g_conf->mon_leveldb_compression;
- db->options.max_open_files = g_conf->mon_leveldb_max_open_files;
- db->options.paranoid_checks = g_conf->mon_leveldb_paranoid;
- db->options.log_file = g_conf->mon_leveldb_log;
+ init_options();
return db->open(out);
}
int create_and_open(ostream &out) {
+ init_options();
return db->create_and_open(out);
}
diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc
index ca85559..6b4abd5 100644
--- a/src/mon/MonmapMonitor.cc
+++ b/src/mon/MonmapMonitor.cc
@@ -138,6 +138,7 @@ bool MonmapMonitor::preprocess_query(PaxosServiceMessage *m)
void MonmapMonitor::dump_info(Formatter *f)
{
f->dump_unsigned("monmap_first_committed", get_first_committed());
+ f->dump_unsigned("monmap_last_committed", get_last_committed());
f->open_object_section("monmap");
mon->monmap->dump(f);
f->close_section();
@@ -177,58 +178,67 @@ bool MonmapMonitor::preprocess_command(MMonCommand *m)
ss.str("");
r = 0;
- } else if (prefix == "mon getmap") {
- mon->monmap->encode(rdata, CEPH_FEATURES_ALL);
- r = 0;
- ss << "got latest monmap";
+ } else if (prefix == "mon getmap" ||
+ prefix == "mon dump") {
- } else if (prefix == "mon dump") {
- string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
- epoch_t epoch = 0;
- int64_t epochval;
- if (cmd_getval(g_ceph_context, cmdmap, "epoch", epochval))
- epoch = epochval;
+ epoch_t epoch;
+ int64_t epochnum;
+ cmd_getval(g_ceph_context, cmdmap, "epoch", epochnum, (int64_t)0);
+ epoch = epochnum;
MonMap *p = mon->monmap;
if (epoch) {
- bufferlist b;
- r = get_version(epoch, b);
+ bufferlist bl;
+ r = get_version(epoch, bl);
if (r == -ENOENT) {
- p = 0;
- } else {
- p = new MonMap;
- p->decode(b);
+ ss << "there is no map for epoch " << epoch;
+ goto reply;
}
+ assert(r == 0);
+ assert(bl.length() > 0);
+ p = new MonMap;
+ p->decode(bl);
}
- if (p) {
+
+ assert(p != NULL);
+
+ if (prefix == "mon getmap") {
+ p->encode(rdata, m->get_connection()->get_features());
+ r = 0;
+ ss << "got monmap epoch " << p->get_epoch();
+ } else if (prefix == "mon dump") {
+ string format;
+ cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
stringstream ds;
boost::scoped_ptr<Formatter> f(new_formatter(format));
if (f) {
- f->open_object_section("monmap");
- p->dump(f.get());
- f->open_array_section("quorum");
- for (set<int>::iterator q = mon->get_quorum().begin(); q != mon->get_quorum().end(); ++q)
- f->dump_int("mon", *q);
- f->close_section();
- f->close_section();
- f->flush(ds);
- r = 0;
+ f->open_object_section("monmap");
+ p->dump(f.get());
+ f->open_array_section("quorum");
+ for (set<int>::iterator q = mon->get_quorum().begin();
+ q != mon->get_quorum().end(); ++q) {
+ f->dump_int("mon", *q);
+ }
+ f->close_section();
+ f->close_section();
+ f->flush(ds);
+ r = 0;
} else {
- p->print(ds);
- r = 0;
- }
+ p->print(ds);
+ r = 0;
+ }
rdata.append(ds);
ss << "dumped monmap epoch " << p->get_epoch();
- if (p != mon->monmap)
- delete p;
}
+ if (p != mon->monmap)
+ delete p;
}
else if (prefix == "mon add")
return false;
else if (prefix == "mon remove")
return false;
+reply:
if (r != -1) {
string rs;
getline(ss, rs);
@@ -336,7 +346,7 @@ bool MonmapMonitor::prepare_command(MMonCommand *m)
pending_map.last_changed = ceph_clock_now(g_ceph_context);
getline(ss, rs);
wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
- get_last_committed()));
+ get_last_committed() + 1));
return true;
} else if (prefix == "mon remove") {
@@ -440,6 +450,21 @@ void MonmapMonitor::get_health(list<pair<health_status_t, string> >& summary,
}
}
}
+ if (g_conf->mon_warn_on_old_mons && !mon->get_classic_mons().empty()) {
+ ostringstream ss;
+ ss << "some monitors are running older code";
+ summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+ if (detail) {
+ for (set<int>::const_iterator i = mon->get_classic_mons().begin();
+ i != mon->get_classic_mons().end();
+ ++i) {
+ ostringstream ss;
+ ss << "mon." << mon->monmap->get_name(*i)
+ << " only supports the \"classic\" command set";
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
+ }
}
int MonmapMonitor::get_monmap(bufferlist &bl)
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 8d13387..8bf63cc 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -50,6 +50,7 @@
#include "include/util.h"
#include "common/cmdparse.h"
#include "include/str_list.h"
+#include "include/str_map.h"
#define dout_subsys ceph_subsys_mon
#undef dout_prefix
@@ -102,7 +103,7 @@ void OSDMonitor::create_initial()
newmap.created = newmap.modified = ceph_clock_now(g_ceph_context);
// encode into pending incremental
- newmap.encode(pending_inc.fullmap);
+ newmap.encode(pending_inc.fullmap, mon->quorum_features);
}
void OSDMonitor::update_from_paxos(bool *need_bootstrap)
@@ -202,7 +203,7 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap)
// write out the full map for all past epochs
bufferlist full_bl;
- osdmap.encode(full_bl);
+ osdmap.encode(full_bl, inc.encode_features);
tx_size += full_bl.length();
put_version_full(t, osdmap.epoch, full_bl);
@@ -247,6 +248,10 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap)
osd_epoch.erase(p++);
}
+ /** we don't have any of the feature bit infrastructure in place for
+ * supporting primary_temp mappings without breaking old clients/OSDs.*/
+ assert(osdmap.primary_temp->empty());
+
if (mon->is_leader()) {
// kick pgmon, make sure it's seen the latest map
mon->pgmon()->check_osd_map(osdmap.epoch);
@@ -331,10 +336,10 @@ bool OSDMonitor::thrash()
}
// generate some pg_temp entries.
- // let's assume the hash_map iterates in a random-ish order.
+ // let's assume the ceph::unordered_map iterates in a random-ish order.
int n = rand() % mon->pgmon()->pg_map.pg_stat.size();
- hash_map<pg_t,pg_stat_t>::iterator p = mon->pgmon()->pg_map.pg_stat.begin();
- hash_map<pg_t,pg_stat_t>::iterator e = mon->pgmon()->pg_map.pg_stat.end();
+ ceph::unordered_map<pg_t,pg_stat_t>::iterator p = mon->pgmon()->pg_map.pg_stat.begin();
+ ceph::unordered_map<pg_t,pg_stat_t>::iterator e = mon->pgmon()->pg_map.pg_stat.end();
while (n--)
++p;
for (int i=0; i<50; i++) {
@@ -409,45 +414,6 @@ void OSDMonitor::update_logger()
mon->cluster_logger->set(l_cluster_osd_epoch, osdmap.get_epoch());
}
-void OSDMonitor::remove_redundant_pg_temp()
-{
- dout(10) << "remove_redundant_pg_temp" << dendl;
-
- for (map<pg_t,vector<int> >::iterator p = osdmap.pg_temp->begin();
- p != osdmap.pg_temp->end();
- ++p) {
- if (pending_inc.new_pg_temp.count(p->first) == 0) {
- vector<int> raw_up;
- osdmap.pg_to_raw_up(p->first, raw_up);
- if (raw_up == p->second) {
- dout(10) << " removing unnecessary pg_temp " << p->first << " -> " << p->second << dendl;
- pending_inc.new_pg_temp[p->first].clear();
- }
- }
- }
-}
-
-void OSDMonitor::remove_down_pg_temp()
-{
- dout(10) << "remove_down_pg_temp" << dendl;
- OSDMap tmpmap(osdmap);
- tmpmap.apply_incremental(pending_inc);
-
- for (map<pg_t,vector<int> >::iterator p = tmpmap.pg_temp->begin();
- p != tmpmap.pg_temp->end();
- ++p) {
- unsigned num_up = 0;
- for (vector<int>::iterator i = p->second.begin();
- i != p->second.end();
- ++i) {
- if (!tmpmap.is_down(*i))
- ++num_up;
- }
- if (num_up == 0)
- pending_inc.new_pg_temp[p->first].clear();
- }
-}
-
/* Assign a lower weight to overloaded OSDs.
*
* The osds that will get a lower weight are those with with a utilization
@@ -499,7 +465,7 @@ int OSDMonitor::reweight_by_utilization(int oload, std::string& out_str)
std::string sep;
oss << "overloaded osds: ";
bool changed = false;
- for (hash_map<int,osd_stat_t>::const_iterator p = pgm.osd_stat.begin();
+ for (ceph::unordered_map<int,osd_stat_t>::const_iterator p = pgm.osd_stat.begin();
p != pgm.osd_stat.end();
++p) {
float util = p->second.kb_used;
@@ -536,10 +502,10 @@ void OSDMonitor::create_pending()
dout(10) << "create_pending e " << pending_inc.epoch << dendl;
// drop any redundant pg_temp entries
- remove_redundant_pg_temp();
+ OSDMap::remove_redundant_temporaries(g_ceph_context, osdmap, &pending_inc);
- // drop any pg_temp entries with no up entries
- remove_down_pg_temp();
+ // drop any pg or primary_temp entries with no up entries
+ OSDMap::remove_down_temps(g_ceph_context, osdmap, &pending_inc);
}
/**
@@ -554,6 +520,9 @@ void OSDMonitor::encode_pending(MonitorDBStore::Transaction *t)
// finalize up pending_inc
pending_inc.modified = ceph_clock_now(g_ceph_context);
+ int r = pending_inc.propagate_snaps_to_tiers(g_ceph_context, osdmap);
+ assert(r == 0);
+
bufferlist bl;
// tell me about it
@@ -586,11 +555,44 @@ void OSDMonitor::encode_pending(MonitorDBStore::Transaction *t)
// encode
assert(get_last_committed() + 1 == pending_inc.epoch);
- ::encode(pending_inc, bl, CEPH_FEATURES_ALL);
+ ::encode(pending_inc, bl, mon->quorum_features);
/* put everything in the transaction */
put_version(t, pending_inc.epoch, bl);
put_last_committed(t, pending_inc.epoch);
+
+ // metadata, too!
+ for (map<int,bufferlist>::iterator p = pending_metadata.begin();
+ p != pending_metadata.end();
+ ++p)
+ t->put(OSD_METADATA_PREFIX, stringify(p->first), p->second);
+ for (set<int>::iterator p = pending_metadata_rm.begin();
+ p != pending_metadata_rm.end();
+ ++p)
+ t->erase(OSD_METADATA_PREFIX, stringify(*p));
+ pending_metadata.clear();
+ pending_metadata_rm.clear();
+}
+
+int OSDMonitor::dump_osd_metadata(int osd, Formatter *f, ostream *err)
+{
+ bufferlist bl;
+ int r = mon->store->get(OSD_METADATA_PREFIX, stringify(osd), bl);
+ if (r < 0)
+ return r;
+ map<string,string> m;
+ try {
+ bufferlist::iterator p = bl.begin();
+ ::decode(m, p);
+ }
+ catch (buffer::error& e) {
+ if (err)
+ *err << "osd." << osd << " metadata is corrupt";
+ return -EIO;
+ }
+ for (map<string,string>::iterator p = m.begin(); p != m.end(); ++p)
+ f->dump_string(p->first.c_str(), p->second);
+ return 0;
}
void OSDMonitor::share_map_with_random_osd()
@@ -1149,6 +1151,15 @@ bool OSDMonitor::preprocess_boot(MOSDBoot *m)
}
assert(m->get_orig_source_inst().name.is_osd());
+
+ // check if osd has required features to boot
+ if ((osdmap.get_features(NULL) & CEPH_FEATURE_OSD_ERASURE_CODES) &&
+ !(m->get_connection()->get_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+ dout(0) << __func__ << " osdmap requires Erasure Codes but osd at "
+ << m->get_orig_source_inst()
+ << " doesn't announce support -- ignore" << dendl;
+ goto ignore;
+ }
// already booted?
if (osdmap.is_up(from) &&
@@ -1160,6 +1171,16 @@ bool OSDMonitor::preprocess_boot(MOSDBoot *m)
return true;
}
+ if (osdmap.exists(from) &&
+ !osdmap.get_uuid(from).is_zero() &&
+ osdmap.get_uuid(from) != m->sb.osd_fsid) {
+ dout(7) << __func__ << " from " << m->get_orig_source_inst()
+ << " clashes with existing osd: different fsid"
+ << " (ours: " << osdmap.get_uuid(from)
+ << " ; theirs: " << m->sb.osd_fsid << ")" << dendl;
+ goto ignore;
+ }
+
// noup?
if (!can_mark_up(from)) {
dout(7) << "preprocess_boot ignoring boot from " << m->get_orig_source_inst() << dendl;
@@ -1185,7 +1206,7 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
assert(m->get_orig_source().is_osd());
int from = m->get_orig_source().num();
-
+
// does this osd exist?
if (from >= osdmap.get_max_osd()) {
dout(1) << "boot from osd." << from << " >= max_osd " << osdmap.get_max_osd() << dendl;
@@ -1200,7 +1221,9 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
// already up? mark down first?
if (osdmap.is_up(from)) {
dout(7) << "prepare_boot was up, first marking down " << osdmap.get_inst(from) << dendl;
- assert(osdmap.get_inst(from) != m->get_orig_source_inst()); // preproces should have caught it
+ // preprocess should have caught these; if not, assert.
+ assert(osdmap.get_inst(from) != m->get_orig_source_inst());
+ assert(osdmap.get_uuid(from) == m->sb.osd_fsid);
if (pending_inc.new_state.count(from) == 0 ||
(pending_inc.new_state[from] & CEPH_OSD_UP) == 0) {
@@ -1239,8 +1262,11 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
// set uuid?
dout(10) << " setting osd." << from << " uuid to " << m->sb.osd_fsid << dendl;
- if (!osdmap.exists(from) || osdmap.get_uuid(from) != m->sb.osd_fsid)
+ if (!osdmap.exists(from) || osdmap.get_uuid(from) != m->sb.osd_fsid) {
+ // preprocess should have caught this; if not, assert.
+ assert(!osdmap.exists(from) || osdmap.get_uuid(from).is_zero());
pending_inc.new_uuid[from] = m->sb.osd_fsid;
+ }
// fresh osd?
if (m->sb.newest_map == 0 && osdmap.exists(from)) {
@@ -1251,6 +1277,11 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
}
}
+ // metadata
+ bufferlist osd_metadata;
+ ::encode(m->metadata, osd_metadata);
+ pending_metadata[from] = osd_metadata;
+
// adjust last clean unmount epoch?
const osd_info_t& info = osdmap.get_info(from);
dout(10) << " old osd_info: " << info << dendl;
@@ -1284,6 +1315,8 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
xi.laggy_probability * (1.0 - g_conf->mon_osd_laggy_weight);
dout(10) << " laggy, now xi " << xi << dendl;
}
+ // set features shared by the osd
+ xi.features = m->get_connection()->get_features();
pending_inc.new_xinfo[from] = xi;
// wait
@@ -1375,6 +1408,7 @@ bool OSDMonitor::preprocess_pgtemp(MOSDPGTemp *m)
dout(10) << "preprocess_pgtemp " << *m << dendl;
vector<int> empty;
int from = m->get_orig_source().num();
+ size_t ignore_cnt = 0;
// check caps
MonSession *session = m->get_session();
@@ -1395,7 +1429,25 @@ bool OSDMonitor::preprocess_pgtemp(MOSDPGTemp *m)
for (map<pg_t,vector<int> >::iterator p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
dout(20) << " " << p->first
<< (osdmap.pg_temp->count(p->first) ? (*osdmap.pg_temp)[p->first] : empty)
- << " -> " << p->second << dendl;
+ << " -> " << p->second << dendl;
+
+ // does the pool exist?
+ if (!osdmap.have_pg_pool(p->first.pool())) {
+ /*
+ * 1. If the osdmap does not have the pool, it means the pool has been
+ * removed in-between the osd sending this message and us handling it.
+ * 2. If osdmap doesn't have the pool, it is safe to assume the pool does
+ * not exist in the pending either, as the osds would not send a
+ * message about a pool they know nothing about (yet).
+ * 3. However, if the pool does exist in the pending, then it must be a
+ * new pool, and not relevant to this message (see 1).
+ */
+ dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+ << ": pool has been removed" << dendl;
+ ignore_cnt++;
+ continue;
+ }
+
// removal?
if (p->second.empty() && osdmap.pg_temp->count(p->first))
return false;
@@ -1405,6 +1457,10 @@ bool OSDMonitor::preprocess_pgtemp(MOSDPGTemp *m)
return false;
}
+ // should we ignore all the pgs?
+ if (ignore_cnt == m->pg_temp.size())
+ goto ignore;
+
dout(7) << "preprocess_pgtemp e" << m->map_epoch << " no changes from " << m->get_orig_source_inst() << dendl;
_reply_map(m, m->map_epoch);
return true;
@@ -1418,8 +1474,20 @@ bool OSDMonitor::prepare_pgtemp(MOSDPGTemp *m)
{
int from = m->get_orig_source().num();
dout(7) << "prepare_pgtemp e" << m->map_epoch << " from " << m->get_orig_source_inst() << dendl;
- for (map<pg_t,vector<int> >::iterator p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p)
+ for (map<pg_t,vector<int> >::iterator p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
+ uint64_t pool = p->first.pool();
+ if (pending_inc.old_pools.count(pool)) {
+ dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+ << ": pool pending removal" << dendl;
+ continue;
+ }
+ if (!osdmap.have_pg_pool(pool)) {
+ dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+ << ": pool has been removed" << dendl;
+ continue;
+ }
pending_inc.new_pg_temp[p->first] = p->second;
+ }
pending_inc.new_up_thru[from] = m->map_epoch; // set up_thru too, so the osd doesn't have to ask again
wait_for_finished_proposal(new C_ReplyMap(this, m, m->map_epoch));
return true;
@@ -1783,7 +1851,7 @@ void OSDMonitor::tick()
}
// expire blacklisted items?
- for (hash_map<entity_addr_t,utime_t>::iterator p = osdmap.blacklist.begin();
+ for (ceph::unordered_map<entity_addr_t,utime_t>::iterator p = osdmap.blacklist.begin();
p != osdmap.blacklist.end();
++p) {
if (p->second < now) {
@@ -1818,7 +1886,7 @@ void OSDMonitor::tick()
ps_t numps = osdmap.get_pg_num();
for (int64_t pool=0; pool<1; pool++)
for (ps_t ps = 0; ps < numps; ++ps) {
- pg_t pgid = pg_t(pg_t::TYPE_REP, ps, pool, -1);
+ pg_t pgid = pg_t(pg_t::TYPE_REPLICATED, ps, pool, -1);
vector<int> osds;
osdmap.pg_to_osds(pgid, osds);
if (osds[0] == 0) {
@@ -1937,6 +2005,19 @@ void OSDMonitor::get_health(list<pair<health_status_t,string> >& summary,
detail->push_back(make_pair(HEALTH_WARN, ss.str()));
}
+ // old crush tunables?
+ if (g_conf->mon_warn_on_legacy_crush_tunables) {
+ if (!osdmap.crush->has_optimal_tunables()) {
+ ostringstream ss;
+ ss << "crush map has non-optimal tunables";
+ summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+ if (detail) {
+ ss << "; see http://ceph.com/docs/master/rados/operations/crush-map/#tunables";
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
+ }
+
get_pools_health(summary, detail);
}
}
@@ -1947,7 +2028,19 @@ void OSDMonitor::dump_info(Formatter *f)
osdmap.dump(f);
f->close_section();
+ f->open_array_section("osd_metadata");
+ for (int i=0; i<osdmap.get_max_osd(); ++i) {
+ if (osdmap.exists(i)) {
+ f->open_object_section("osd");
+ f->dump_unsigned("id", i);
+ dump_osd_metadata(i, f, NULL);
+ f->close_section();
+ }
+ }
+ f->close_section();
+
f->dump_unsigned("osdmap_first_committed", get_first_committed());
+ f->dump_unsigned("osdmap_last_committed", get_last_committed());
f->open_object_section("crushmap");
osdmap.crush->dump(f);
@@ -2006,6 +2099,7 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
int err = get_version_full(epoch, b);
if (err == -ENOENT) {
r = -ENOENT;
+ ss << "there is no map for epoch " << epoch;
goto reply;
}
assert(err == 0);
@@ -2059,7 +2153,7 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
}
rdata.append(ds);
} else if (prefix == "osd getmap") {
- p->encode(rdata);
+ p->encode(rdata, m->get_connection()->get_features());
ss << "got osdmap epoch " << p->get_epoch();
} else if (prefix == "osd getcrushmap") {
p->crush->encode(rdata);
@@ -2111,6 +2205,23 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
f->close_section();
f->close_section();
f->flush(rdata);
+ } else if (prefix == "osd metadata") {
+ int64_t osd;
+ cmd_getval(g_ceph_context, cmdmap, "id", osd);
+ if (!osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist";
+ r = -ENOENT;
+ goto reply;
+ }
+ string format;
+ cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+ boost::scoped_ptr<Formatter> f(new_formatter(format));
+ f->open_object_section("osd_metadata");
+ r = dump_osd_metadata(osd, f.get(), &ss);
+ if (r < 0)
+ goto reply;
+ f->close_section();
+ f->flush(rdata);
} else if (prefix == "osd map") {
string poolstr, objstr, namespacestr;
cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
@@ -2128,7 +2239,8 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
pg_t pgid = osdmap.object_locator_to_pg(oid, oloc);
pg_t mpgid = osdmap.raw_pg_to_pg(pgid);
vector<int> up, acting;
- osdmap.pg_to_up_acting_osds(mpgid, up, acting);
+ int up_p, acting_p;
+ osdmap.pg_to_up_acting_osds(mpgid, &up, &up_p, &acting, &acting_p);
string fullobjname;
if (!namespacestr.empty())
@@ -2144,7 +2256,9 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
f->dump_stream("raw_pgid") << pgid;
f->dump_stream("pgid") << mpgid;
f->dump_stream("up") << up;
+ f->dump_int("up_primary", up_p);
f->dump_stream("acting") << acting;
+ f->dump_int("acting_primary", acting_p);
f->close_section(); // osd_map
f->flush(rdata);
} else {
@@ -2152,7 +2266,8 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
<< " pool '" << poolstr << "' (" << pool << ")"
<< " object '" << fullobjname << "' ->"
<< " pg " << pgid << " (" << mpgid << ")"
- << " -> up " << up << " acting " << acting;
+ << " -> up (" << up << ", p" << up_p << ") acting ("
+ << acting << ", p" << acting_p << ")";
rdata.append(ds);
}
} else if ((prefix == "osd scrub" ||
@@ -2219,7 +2334,7 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
if (f)
f->open_array_section("blacklist");
- for (hash_map<entity_addr_t,utime_t>::iterator p = osdmap.blacklist.begin();
+ for (ceph::unordered_map<entity_addr_t,utime_t>::iterator p = osdmap.blacklist.begin();
p != osdmap.blacklist.end();
++p) {
if (f) {
@@ -2399,7 +2514,10 @@ stats_out:
prefix == "osd crush rule ls") {
string format;
cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
- boost::scoped_ptr<Formatter> f(new_formatter(format));
+ Formatter *fp = new_formatter(format);
+ if (!fp)
+ fp = new_formatter("json-pretty");
+ boost::scoped_ptr<Formatter> f(fp);
f->open_array_section("rules");
osdmap.crush->list_rules(f.get());
f->close_section();
@@ -2410,7 +2528,10 @@ stats_out:
} else if (prefix == "osd crush rule dump") {
string format;
cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
- boost::scoped_ptr<Formatter> f(new_formatter(format));
+ Formatter *fp = new_formatter(format);
+ if (!fp)
+ fp = new_formatter("json-pretty");
+ boost::scoped_ptr<Formatter> f(fp);
f->open_array_section("rules");
osdmap.crush->dump_rules(f.get());
f->close_section();
@@ -2421,7 +2542,10 @@ stats_out:
} else if (prefix == "osd crush dump") {
string format;
cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
- boost::scoped_ptr<Formatter> f(new_formatter(format));
+ Formatter *fp = new_formatter(format);
+ if (!fp)
+ fp = new_formatter("json-pretty");
+ boost::scoped_ptr<Formatter> f(fp);
f->open_object_section("crush_map");
osdmap.crush->dump(f.get());
f->close_section();
@@ -2429,6 +2553,20 @@ stats_out:
f->flush(rs);
rs << "\n";
rdata.append(rs.str());
+ } else if (prefix == "osd crush show-tunables") {
+ string format;
+ cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+ Formatter *fp = new_formatter(format);
+ if (!fp)
+ fp = new_formatter("json-pretty");
+ boost::scoped_ptr<Formatter> f(fp);
+ f->open_object_section("crush_map_tunables");
+ osdmap.crush->dump_tunables(f.get());
+ f->close_section();
+ ostringstream rs;
+ f->flush(rs);
+ rs << "\n";
+ rdata.append(rs.str());
} else {
// try prepare update
return false;
@@ -2594,10 +2732,13 @@ int OSDMonitor::prepare_new_pool(MPoolOp *m)
if (!session)
return -EPERM;
vector<string> properties;
+ stringstream ss;
if (m->auid)
- return prepare_new_pool(m->name, m->auid, m->crush_rule, 0, 0, properties);
+ return prepare_new_pool(m->name, m->auid, m->crush_rule, 0, 0,
+ properties, pg_pool_t::TYPE_REPLICATED, ss);
else
- return prepare_new_pool(m->name, session->auid, m->crush_rule, 0, 0, properties);
+ return prepare_new_pool(m->name, session->auid, m->crush_rule, 0, 0,
+ properties, pg_pool_t::TYPE_REPLICATED, ss);
}
/**
@@ -2607,13 +2748,28 @@ int OSDMonitor::prepare_new_pool(MPoolOp *m)
* @param pg_num The pg_num to use. If set to 0, will use the system default
* @param pgp_num The pgp_num to use. If set to 0, will use the system default
* @param properties An opaque list of key[=value] pairs for pool configuration
+ * @param pool_type TYPE_ERASURE, TYPE_REP or TYPE_RAID4
+ * @param ss human readable error message, if any.
*
- * @return 0 in all cases. That's silly.
+ * @return 0 on success, negative errno on failure.
*/
-int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_rule,
+int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_ruleset,
unsigned pg_num, unsigned pgp_num,
- const vector<string> &properties)
+ const vector<string> &properties,
+ const unsigned pool_type,
+ stringstream &ss)
{
+ map<string,string> default_properties;
+ if (pool_type == pg_pool_t::TYPE_ERASURE) {
+ int r = get_str_map(g_conf->osd_pool_default_erasure_code_properties,
+ ss,
+ &default_properties);
+ if (r)
+ return r;
+ default_properties["erasure-code-directory"] =
+ g_conf->osd_pool_default_erasure_code_directory;
+ }
+
for (map<int64_t,string>::iterator p = pending_inc.new_pool_names.begin();
p != pending_inc.new_pool_names.end();
++p) {
@@ -2626,30 +2782,41 @@ int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_rule,
int64_t pool = ++pending_inc.new_pool_max;
pg_pool_t empty;
pg_pool_t *pi = pending_inc.get_new_pool(pool, &empty);
- pi->type = pg_pool_t::TYPE_REP;
+ pi->type = pool_type;
pi->flags = g_conf->osd_pool_default_flags;
if (g_conf->osd_pool_default_flag_hashpspool)
pi->flags |= pg_pool_t::FLAG_HASHPSPOOL;
pi->size = g_conf->osd_pool_default_size;
pi->min_size = g_conf->get_osd_pool_default_min_size();
- if (crush_rule >= 0)
- pi->crush_ruleset = crush_rule;
- else
- pi->crush_ruleset = g_conf->osd_pool_default_crush_rule;
+ if (crush_ruleset >= 0) {
+ pi->crush_ruleset = crush_ruleset;
+ } else {
+ switch(pool_type) {
+ case pg_pool_t::TYPE_REPLICATED:
+ pi->crush_ruleset =
+ CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(g_ceph_context);
+ break;
+ case pg_pool_t::TYPE_ERASURE:
+ pi->crush_ruleset = g_conf->osd_pool_default_crush_erasure_ruleset;
+ break;
+ }
+ }
pi->object_hash = CEPH_STR_HASH_RJENKINS;
pi->set_pg_num(pg_num ? pg_num : g_conf->osd_pool_default_pg_num);
pi->set_pgp_num(pgp_num ? pgp_num : g_conf->osd_pool_default_pgp_num);
pi->last_change = pending_inc.epoch;
pi->auid = auid;
+ pi->properties = default_properties;
for (vector<string>::const_iterator i = properties.begin();
i != properties.end();
- i++) {
+ ++i) {
size_t equal = i->find('=');
if (equal == string::npos)
pi->properties[*i] = string();
else {
const string key = i->substr(0, equal);
+ equal++;
const string value = i->substr(equal);
pi->properties[key] = value;
}
@@ -2665,7 +2832,8 @@ bool OSDMonitor::prepare_set_flag(MMonCommand *m, int flag)
pending_inc.new_flags = osdmap.get_flags();
pending_inc.new_flags |= flag;
ss << "set " << OSDMap::get_flag_string(flag);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
}
@@ -2676,7 +2844,8 @@ bool OSDMonitor::prepare_unset_flag(MMonCommand *m, int flag)
pending_inc.new_flags = osdmap.get_flags();
pending_inc.new_flags &= ~flag;
ss << "unset " << OSDMap::get_flag_string(flag);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
}
@@ -2701,22 +2870,6 @@ int OSDMonitor::parse_osd_id(const char *s, stringstream *pss)
return id;
}
-void OSDMonitor::parse_loc_map(const vector<string>& args, map<string,string> *ploc)
-{
- for (unsigned i = 0; i < args.size(); ++i) {
- const char *s = args[i].c_str();
- const char *pos = strchr(s, '=');
- if (!pos)
- break;
- string key(s, 0, pos-s);
- string value(pos+1);
- if (value.length())
- (*ploc)[key] = value;
- else
- ploc->erase(key);
- }
-}
-
int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
stringstream& ss)
{
@@ -2734,19 +2887,29 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
if (pending_inc.new_pools.count(pool))
p = pending_inc.new_pools[pool];
- // accept val as a json string or int, and parse out int
- // values from the string as needed
- string raw_val = cmd_vartype_stringify(cmdmap["val"]);
+ // accept val as a json string in the normal case (current
+ // generation monitor). parse out int or float values from the
+ // string as needed. however, if it is not a string, try to pull
+ // out an int, in case an older monitor with an older json schema is
+ // forwarding a request.
string val;
- cmd_getval(g_ceph_context, cmdmap, "val", val);
- string interr;
+ string interr, floaterr;
int64_t n = 0;
- if (!cmd_getval(g_ceph_context, cmdmap, "val", n))
+ double f = 0;
+ if (!cmd_getval(g_ceph_context, cmdmap, "val", val)) {
+ // wasn't a string; maybe an older mon forwarded json with an int?
+ if (!cmd_getval(g_ceph_context, cmdmap, "val", n))
+ return -EINVAL; // no value!
+ } else {
+ // we got a string. see if it contains an int.
n = strict_strtoll(val.c_str(), 10, &interr);
+ // or a float
+ f = strict_strtod(val.c_str(), &floaterr);
+ }
if (var == "size") {
if (interr.length()) {
- ss << "error parsing integer value '" << raw_val << "': " << interr;
+ ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
}
if (n == 0 || n > 10) {
@@ -2756,92 +2919,138 @@ int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
p.size = n;
if (n < p.min_size)
p.min_size = n;
- ss << "set pool " << pool << " size to " << n;
} else if (var == "min_size") {
if (interr.length()) {
- ss << "error parsing integer value '" << raw_val << "': " << interr;
+ ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
}
p.min_size = n;
- ss << "set pool " << pool << " min_size to " << n;
} else if (var == "crash_replay_interval") {
if (interr.length()) {
- ss << "error parsing integer value '" << raw_val << "': " << interr;
+ ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
}
p.crash_replay_interval = n;
- ss << "set pool " << pool << " to crash_replay_interval to " << n;
} else if (var == "pg_num") {
if (interr.length()) {
- ss << "error parsing integer value '" << raw_val << "': " << interr;
+ ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
}
if (n <= (int)p.get_pg_num()) {
ss << "specified pg_num " << n << " <= current " << p.get_pg_num();
- } else {
- for(set<pg_t>::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
- i != mon->pgmon()->pg_map.creating_pgs.end();
- ++i) {
- if (i->m_pool == static_cast<uint64_t>(pool)) {
- ss << "currently creating pgs, wait";
- return -EAGAIN;
- }
+ if (n < (int)p.get_pg_num())
+ return -EEXIST;
+ return 0;
+ }
+ int expected_osds = MIN(p.get_pg_num(), osdmap.get_num_osds());
+ int64_t new_pgs = n - p.get_pg_num();
+ int64_t pgs_per_osd = new_pgs / expected_osds;
+ if (pgs_per_osd > g_conf->mon_osd_max_split_count) {
+ ss << "specified pg_num " << n << " is too large (creating "
+ << new_pgs << " new PGs on ~" << expected_osds
+ << " OSDs exceeds per-OSD max of" << g_conf->mon_osd_max_split_count
+ << ')';
+ return -E2BIG;
+ }
+ for(set<pg_t>::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
+ i != mon->pgmon()->pg_map.creating_pgs.end();
+ ++i) {
+ if (i->m_pool == static_cast<uint64_t>(pool)) {
+ ss << "currently creating pgs, wait";
+ return -EAGAIN;
}
- p.set_pg_num(n);
- ss << "set pool " << pool << " pg_num to " << n;
}
+ p.set_pg_num(n);
} else if (var == "pgp_num") {
if (interr.length()) {
- ss << "error parsing integer value '" << raw_val << "': " << interr;
+ ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
}
if (n <= 0) {
ss << "specified pgp_num must > 0, but you set to " << n;
- } else if (n > (int)p.get_pg_num()) {
+ return -EINVAL;
+ }
+ if (n > (int)p.get_pg_num()) {
ss << "specified pgp_num " << n << " > pg_num " << p.get_pg_num();
- } else {
- for(set<pg_t>::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
- i != mon->pgmon()->pg_map.creating_pgs.end();
- ++i) {
- if (i->m_pool == static_cast<uint64_t>(pool)) {
- ss << "currently creating pgs, wait";
- return -EAGAIN;
- }
+ return -EINVAL;
+ }
+ for(set<pg_t>::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
+ i != mon->pgmon()->pg_map.creating_pgs.end();
+ ++i) {
+ if (i->m_pool == static_cast<uint64_t>(pool)) {
+ ss << "currently creating pgs, wait";
+ return -EAGAIN;
}
- p.set_pgp_num(n);
- ss << "set pool " << pool << " pgp_num to " << n;
}
+ p.set_pgp_num(n);
} else if (var == "crush_ruleset") {
if (interr.length()) {
- ss << "error parsing integer value '" << raw_val << "': " << interr;
+ ss << "error parsing integer value '" << val << "': " << interr;
return -EINVAL;
}
- if (osdmap.crush->rule_exists(n)) {
- p.crush_ruleset = n;
- ss << "set pool " << pool << " crush_ruleset to " << n;
- } else {
+ if (!osdmap.crush->rule_exists(n)) {
ss << "crush ruleset " << n << " does not exist";
return -ENOENT;
}
+ p.crush_ruleset = n;
} else if (var == "hashpspool") {
// make sure we only compare against 'n' if we didn't receive a string
- int flag_val = (!val.empty() ? -1 : n);
- if (val == "true" || flag_val == 1) {
+ if (val == "true" || (interr.empty() && n == 1)) {
p.flags |= pg_pool_t::FLAG_HASHPSPOOL;
- ss << "set";
- } else if (val == "false" || flag_val == 0) {
+ } else if (val == "false" || (interr.empty() && n == 0)) {
p.flags ^= pg_pool_t::FLAG_HASHPSPOOL;
- ss << "unset";
} else {
- ss << "expecting value 0 or 1";
+ ss << "expecting value 'true', 'false', '0', or '1'";
return -EINVAL;
}
- ss << " pool " << pool << " flag hashpspool";
+ } else if (var == "hit_set_type") {
+ if (val == "none")
+ p.hit_set_params = HitSet::Params();
+ else if (val == "bloom") {
+ BloomHitSet::Params *bsp = new BloomHitSet::Params;
+ bsp->set_fpp(.01);
+ p.hit_set_params = HitSet::Params(bsp);
+ } else if (val == "explicit_hash")
+ p.hit_set_params = HitSet::Params(new ExplicitHashHitSet::Params);
+ else if (val == "explicit_object")
+ p.hit_set_params = HitSet::Params(new ExplicitObjectHitSet::Params);
+ else {
+ ss << "unrecognized hit_set type '" << val << "'";
+ return -EINVAL;
+ }
+ } else if (var == "hit_set_period") {
+ if (interr.length()) {
+ ss << "error parsing integer value '" << val << "': " << interr;
+ return -EINVAL;
+ }
+ p.hit_set_period = n;
+ } else if (var == "hit_set_count") {
+ if (interr.length()) {
+ ss << "error parsing integer value '" << val << "': " << interr;
+ return -EINVAL;
+ }
+ p.hit_set_count = n;
+ } else if (var == "hit_set_fpp") {
+ if (floaterr.length()) {
+ ss << "error parsing floating point value '" << val << "': " << floaterr;
+ return -EINVAL;
+ }
+ if (p.hit_set_params.get_type() != HitSet::TYPE_BLOOM) {
+ ss << "hit set is not of type Bloom; invalid to set a false positive rate!";
+ return -EINVAL;
+ }
+ BloomHitSet::Params *bloomp = static_cast<BloomHitSet::Params*>(p.hit_set_params.impl.get());
+ bloomp->set_fpp(f);
+ } else if (var == "debug_fake_ec_pool") {
+ if (val == "true" || (interr.empty() && n == 1)) {
+ p.flags |= pg_pool_t::FLAG_DEBUG_FAKE_EC_POOL;
+ }
+ ss << " pool " << pool << " set debug_fake_ec_pool";
} else {
ss << "unrecognized variable '" << var << "'";
return -EINVAL;
}
-
+ ss << "set pool " << pool << " " << var << " to " << val;
p.last_change = pending_inc.epoch;
pending_inc.new_pools[pool] = p;
return 0;
@@ -2932,7 +3141,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
// sanity check: test some inputs to make sure this map isn't totally broken
dout(10) << " testing map" << dendl;
stringstream ess;
- CrushTester tester(crush, ess, 1);
+ CrushTester tester(crush, ess);
tester.test();
dout(10) << " result " << ess.str() << dendl;
@@ -3003,7 +3212,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
vector<string> argvec;
cmd_getval(g_ceph_context, cmdmap, "args", argvec);
map<string,string> loc;
- parse_loc_map(argvec, &loc);
+ CrushWrapper::parse_loc_map(argvec, &loc);
if (prefix == "osd crush set"
&& !_get_stable_crush().item_exists(id)) {
@@ -3046,7 +3255,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
ss << action << " item id " << id << " name '" << name << "' weight "
<< weight << " at location " << loc << " to crush map";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd crush create-or-move") {
@@ -3065,7 +3275,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
vector<string> argvec;
cmd_getval(g_ceph_context, cmdmap, "args", argvec);
map<string,string> loc;
- parse_loc_map(argvec, &loc);
+ CrushWrapper::parse_loc_map(argvec, &loc);
dout(0) << "create-or-move crush item name '" << name << "' initial_weight " << weight
<< " at location " << loc << dendl;
@@ -3085,7 +3295,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
ss << "create-or-move updating item name '" << name << "' weight " << weight
<< " at location " << loc << " to crush map";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} while (false);
@@ -3099,7 +3310,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
cmd_getval(g_ceph_context, cmdmap, "name", name);
cmd_getval(g_ceph_context, cmdmap, "args", argvec);
map<string,string> loc;
- parse_loc_map(argvec, &loc);
+ CrushWrapper::parse_loc_map(argvec, &loc);
dout(0) << "moving crush item name '" << name << "' to location " << loc << dendl;
CrushWrapper newcrush;
@@ -3119,7 +3330,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush);
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} else {
@@ -3135,7 +3347,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
vector<string> argvec;
cmd_getval(g_ceph_context, cmdmap, "args", argvec);
map<string,string> loc;
- parse_loc_map(argvec, &loc);
+ CrushWrapper::parse_loc_map(argvec, &loc);
if (!osdmap.crush->name_exists(name)) {
err = -ENOENT;
@@ -3172,7 +3384,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
err = 0;
}
}
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, err, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, err, ss.str(),
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd crush rm" ||
prefix == "osd crush remove" ||
@@ -3194,7 +3407,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
err = 0;
ss << "device '" << name << "' does not appear in the crush map";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
int id = newcrush.get_item_id(name);
@@ -3223,7 +3437,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
newcrush.encode(pending_inc.crush);
ss << "removed item id " << id << " name '" << name << "' from crush map";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} while (false);
@@ -3257,7 +3472,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
ss << "reweighted item id " << id << " name '" << name << "' to " << w
<< " in crush map";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} while (false);
@@ -3282,14 +3498,18 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
newcrush.encode(pending_inc.crush);
ss << "adjusted tunables profile to " << profile;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd crush rule create-simple") {
- string name, root, type;
+ string name, root, type, mode;
cmd_getval(g_ceph_context, cmdmap, "name", name);
cmd_getval(g_ceph_context, cmdmap, "root", root);
cmd_getval(g_ceph_context, cmdmap, "type", type);
+ cmd_getval(g_ceph_context, cmdmap, "mode", mode);
+ if (mode == "")
+ mode = "firstn";
if (osdmap.crush->rule_exists(name)) {
ss << "rule " << name << " already exists";
@@ -3304,7 +3524,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
ss << "rule " << name << " already exists";
err = 0;
} else {
- int rule = newcrush.add_simple_rule(name, root, type, &ss);
+ int rule = newcrush.add_simple_ruleset(name, root, type, mode,
+ pg_pool_t::TYPE_REPLICATED, &ss);
if (rule < 0) {
err = rule;
goto reply;
@@ -3314,7 +3535,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
newcrush.encode(pending_inc.crush);
}
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd crush rule rm") {
@@ -3356,7 +3578,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
newcrush.encode(pending_inc.crush);
}
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd setmaxosd") {
@@ -3373,7 +3596,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
pending_inc.new_max_osd = newmax;
ss << "set new max_osd = " << pending_inc.new_max_osd;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pause") {
@@ -3444,17 +3668,15 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
long osd = parse_osd_id(idvec[j].c_str(), &ss);
if (osd < 0) {
ss << "invalid osd id" << osd;
- // XXX -EINVAL here?
+ err = -EINVAL;
continue;
} else if (!osdmap.exists(osd)) {
ss << "osd." << osd << " does not exist. ";
- err = 0;
continue;
}
if (prefix == "osd down") {
if (osdmap.is_down(osd)) {
ss << "osd." << osd << " is already down. ";
- err = 0;
} else {
pending_inc.new_state[osd] = CEPH_OSD_UP;
ss << "marked down osd." << osd << ". ";
@@ -3463,7 +3685,6 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "osd out") {
if (osdmap.is_out(osd)) {
ss << "osd." << osd << " is already out. ";
- err = 0;
} else {
pending_inc.new_weight[osd] = CEPH_OSD_OUT;
ss << "marked out osd." << osd << ". ";
@@ -3472,7 +3693,6 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
} else if (prefix == "osd in") {
if (osdmap.is_in(osd)) {
ss << "osd." << osd << " is already in. ";
- err = 0;
} else {
pending_inc.new_weight[osd] = CEPH_OSD_IN;
ss << "marked in osd." << osd << ". ";
@@ -3480,20 +3700,27 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
}
} else if (prefix == "osd rm") {
if (osdmap.is_up(osd)) {
- ss << "osd." << osd << " is still up; must be down before removal. ";
+ if (any)
+ ss << ", ";
+ ss << "osd." << osd << " is still up; must be down before removal. ";
+ err = -EBUSY;
} else {
pending_inc.new_state[osd] = osdmap.get_state(osd);
- if (any)
+ pending_inc.new_uuid[osd] = uuid_d();
+ pending_metadata_rm.insert(osd);
+ if (any) {
ss << ", osd." << osd;
- else
+ } else {
ss << "removed osd." << osd;
+ }
any = true;
}
}
}
if (any) {
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, err, rs,
+ get_last_committed() + 1));
return true;
}
} else if (prefix == "osd reweight") {
@@ -3511,7 +3738,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
pending_inc.new_weight[id] = ww;
ss << "reweighted osd." << id << " to " << w << " (" << ios::hex << ww << ios::dec << ")";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
@@ -3531,7 +3759,8 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
pending_inc.new_lost[id] = e;
ss << "marked osd lost in epoch " << e;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
@@ -3600,7 +3829,8 @@ done:
ss << i;
rdata.append(ss);
}
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, rdata, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, rdata,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd blacklist") {
@@ -3622,7 +3852,8 @@ done:
pending_inc.new_blacklist[addr] = expires;
ss << "blacklisting " << addr << " until " << expires << " (" << d << " sec)";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (blacklistop == "rm") {
if (osdmap.is_blacklisted(addr) ||
@@ -3633,7 +3864,8 @@ done:
pending_inc.new_blacklist.erase(addr);
ss << "un-blacklisting " << addr;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
ss << addr << " isn't blacklisted";
@@ -3677,7 +3909,8 @@ done:
ss << "created pool " << poolstr << " snap " << snapname;
}
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pool rmsnap") {
string poolstr;
@@ -3716,7 +3949,8 @@ done:
ss << "already removed pool " << poolstr << " snap " << snapname;
}
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pool create") {
int64_t pg_num;
@@ -3738,21 +3972,74 @@ done:
goto reply;
}
+ string pool_type_str;
+ cmd_getval(g_ceph_context, cmdmap, "pool_type", pool_type_str);
+ if (pool_type_str.empty())
+ pool_type_str = pg_pool_t::get_default_type();
+
string poolstr;
cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
- if (osdmap.name_pool.count(poolstr)) {
- ss << "pool '" << poolstr << "' already exists";
- err = 0;
+ int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+ if (pool_id >= 0) {
+ const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+ if (pool_type_str != p->get_type_name()) {
+ ss << "pool '" << poolstr << "' cannot change to type " << pool_type_str;
+ err = -EINVAL;
+ } else {
+ ss << "pool '" << poolstr << "' already exists";
+ err = 0;
+ }
goto reply;
}
vector<string> properties;
cmd_getval(g_ceph_context, cmdmap, "properties", properties);
+ int pool_type;
+ if (pool_type_str == "replicated") {
+ pool_type = pg_pool_t::TYPE_REPLICATED;
+ } else if (pool_type_str == "erasure") {
+
+ // make sure all the daemons support erasure coding
+ stringstream ec_unsupported_ss;
+ int ec_unsupported_count = 0;
+ if (!(mon->get_quorum_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+ ec_unsupported_ss << "the monitor cluster";
+ ++ec_unsupported_count;
+ }
+
+ set<int32_t> up_osds;
+ osdmap.get_up_osds(up_osds);
+ for (set<int32_t>::iterator it = up_osds.begin();
+ it != up_osds.end(); it ++) {
+ const osd_xinfo_t &xi = osdmap.get_xinfo(*it);
+ if (!(xi.features & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+ if (ec_unsupported_count > 0)
+ ec_unsupported_ss << ", ";
+ ec_unsupported_ss << "osd." << *it;
+ ec_unsupported_count ++;
+ }
+ }
+
+ if (ec_unsupported_count > 0) {
+ ss << "unable to create erasure pool; unsupported by: "
+ << ec_unsupported_ss.str();
+ err = -ENOTSUP;
+ goto reply;
+ }
+
+ pool_type = pg_pool_t::TYPE_ERASURE;
+ } else {
+ ss << "unknown pool type '" << pool_type_str << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+
err = prepare_new_pool(poolstr, 0, // auid=0 for admin created pool
-1, // default crush rule
pg_num, pgp_num,
- properties);
+ properties, pool_type,
+ ss);
if (err < 0 && err != -EEXIST) {
goto reply;
}
@@ -3762,7 +4049,8 @@ done:
ss << "pool '" << poolstr << "' created";
}
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pool delete") {
@@ -3789,7 +4077,8 @@ done:
if (ret == 0)
ss << "pool '" << poolstr << "' deleted";
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, ret, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, ret, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pool rename") {
string srcpoolstr, destpoolstr;
@@ -3829,7 +4118,8 @@ done:
<< cpp_strerror(ret);
}
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, ret, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, ret, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pool set") {
@@ -3838,7 +4128,8 @@ done:
goto reply;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd tier add") {
string poolstr;
@@ -3875,9 +4166,10 @@ done:
}
// go
pending_inc.get_new_pool(pool_id, p)->tiers.insert(tierpool_id);
- pending_inc.get_new_pool(tierpool_id, p)->tier_of = pool_id;
+ pending_inc.get_new_pool(tierpool_id, tp)->tier_of = pool_id;
ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd tier remove") {
string poolstr;
@@ -3919,7 +4211,8 @@ done:
pending_inc.get_new_pool(pool_id, p)->tiers.erase(tierpool_id);
pending_inc.get_new_pool(tierpool_id, tp)->clear_tier();
ss << "pool '" << tierpoolstr << "' is now (or already was) not a tier of '" << poolstr << "'";
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd tier set-overlay") {
string poolstr;
@@ -3961,7 +4254,8 @@ done:
pending_inc.get_new_pool(pool_id, p)->read_tier = overlaypool_id;
pending_inc.get_new_pool(pool_id, p)->write_tier = overlaypool_id;
ss << "overlay for '" << poolstr << "' is now (or already was) '" << overlaypoolstr << "'";
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd tier remove-overlay") {
string poolstr;
@@ -3983,7 +4277,8 @@ done:
pending_inc.get_new_pool(pool_id, p)->clear_read_tier();
pending_inc.get_new_pool(pool_id, p)->clear_write_tier();
ss << "there is now (or already was) no overlay for '" << poolstr << "'";
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd tier cache-mode") {
string poolstr;
@@ -4013,7 +4308,8 @@ done:
pending_inc.get_new_pool(pool_id, p)->cache_mode = mode;
ss << "set cache-mode for pool '" << poolstr
<< "' to " << pg_pool_t::get_cache_mode_name(mode);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(), get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd pool set-quota") {
string poolstr;
@@ -4054,7 +4350,8 @@ done:
}
ss << "set-quota " << field << " = " << value << " for pool " << poolstr;
rs = ss.str();
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd reweight-by-utilization") {
@@ -4070,7 +4367,8 @@ done:
} else {
ss << "SUCCESSFUL reweight-by-utilization: " << out_str;
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
} else if (prefix == "osd thrash") {
@@ -4094,7 +4392,8 @@ reply:
update:
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+ get_last_committed() + 1));
return true;
}
@@ -4346,6 +4645,15 @@ int OSDMonitor::_prepare_remove_pool(uint64_t pool)
pending_inc.new_pg_temp[p->first].clear();
}
}
+ for (map<pg_t,int>::iterator p = osdmap.primary_temp->begin();
+ p != osdmap.primary_temp->end();
+ ++p) {
+ if (p->first.pool() == pool) {
+ dout(10) << "_prepare_remove_pool " << pool
+ << " removing obsolete primary_temp" << p->first << dendl;
+ pending_inc.new_primary_temp[p->first] = -1;
+ }
+ }
return 0;
}
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 25a2061..81ea432 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -37,6 +37,8 @@ class Monitor;
#include "messages/MOSDFailure.h"
#include "messages/MPoolOp.h"
+#define OSD_METADATA_PREFIX "osd_metadata"
+
/// information about a particular peer's failure reports for one osd
struct failure_reporter_t {
int num_reports; ///< reports from this reporter
@@ -120,6 +122,8 @@ public:
private:
// [leader]
OSDMap::Incremental pending_inc;
+ map<int, bufferlist> pending_metadata;
+ set<int> pending_metadata_rm;
map<int, failure_info_t> failure_info;
map<int,utime_t> down_pending_out; // osd down -> out
@@ -200,8 +204,6 @@ private:
void send_incremental(PaxosServiceMessage *m, epoch_t first);
void send_incremental(epoch_t first, entity_inst_t& dest, bool onetime);
- void remove_redundant_pg_temp();
- void remove_down_pg_temp();
int reweight_by_utilization(int oload, std::string& out_str);
bool check_source(PaxosServiceMessage *m, uuid_d fsid);
@@ -234,9 +236,11 @@ private:
bool prepare_pool_op (MPoolOp *m);
bool prepare_pool_op_create (MPoolOp *m);
bool prepare_pool_op_delete(MPoolOp *m);
- int prepare_new_pool(string& name, uint64_t auid, int crush_rule,
+ int prepare_new_pool(string& name, uint64_t auid, int crush_ruleset,
unsigned pg_num, unsigned pgp_num,
- const vector<string> &properties);
+ const vector<string> &properties,
+ const unsigned pool_type,
+ stringstream &ss);
int prepare_new_pool(MPoolOp *m);
void update_pool_flags(int64_t pool_id, uint64_t flags);
@@ -317,7 +321,6 @@ private:
void tick(); // check state, take actions
int parse_osd_id(const char *s, stringstream *pss);
- void parse_loc_map(const vector<string>& args, map<string,string> *ploc);
void get_health(list<pair<health_status_t,string> >& summary,
list<pair<health_status_t,string> > *detail) const;
@@ -339,6 +342,7 @@ private:
epoch_t blacklist(const entity_addr_t& a, utime_t until);
void dump_info(Formatter *f);
+ int dump_osd_metadata(int osd, Formatter *f, ostream *err);
void check_subs();
void check_sub(Subscription *sub);
diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 2ba8402..56376d8 100644
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -47,7 +47,7 @@ void PGMap::Incremental::encode(bufferlist &bl, uint64_t features) const
void PGMap::Incremental::decode(bufferlist::iterator &bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
::decode(version, bl);
if (struct_v < 3) {
pg_stat_updates.clear();
@@ -180,7 +180,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
stamp = inc.stamp;
pool_stat_t pg_sum_old = pg_sum;
- hash_map<uint64_t, pool_stat_t> pg_pool_sum_old;
+ ceph::unordered_map<uint64_t, pool_stat_t> pg_pool_sum_old;
bool ratios_changed = false;
if (inc.full_ratio != full_ratio && inc.full_ratio != -1) {
@@ -203,9 +203,9 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
if (pg_pool_sum_old.count(update_pg.pool()) == 0)
pg_pool_sum_old[update_pg.pool()] = pg_pool_sum[update_pg.pool()];
- hash_map<pg_t,pg_stat_t>::iterator t = pg_stat.find(update_pg);
+ ceph::unordered_map<pg_t,pg_stat_t>::iterator t = pg_stat.find(update_pg);
if (t == pg_stat.end()) {
- hash_map<pg_t,pg_stat_t>::value_type v(update_pg, update_stat);
+ ceph::unordered_map<pg_t,pg_stat_t>::value_type v(update_pg, update_stat);
pg_stat.insert(v);
} else {
stat_pg_sub(update_pg, t->second);
@@ -221,9 +221,9 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
int osd = p->first;
const osd_stat_t &new_stats(p->second);
- hash_map<int32_t,osd_stat_t>::iterator t = osd_stat.find(osd);
+ ceph::unordered_map<int32_t,osd_stat_t>::iterator t = osd_stat.find(osd);
if (t == osd_stat.end()) {
- hash_map<int32_t,osd_stat_t>::value_type v(osd, new_stats);
+ ceph::unordered_map<int32_t,osd_stat_t>::value_type v(osd, new_stats);
osd_stat.insert(v);
} else {
stat_osd_sub(t->second);
@@ -241,7 +241,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
p != inc.pg_remove.end();
++p) {
const pg_t &removed_pg(*p);
- hash_map<pg_t,pg_stat_t>::iterator s = pg_stat.find(removed_pg);
+ ceph::unordered_map<pg_t,pg_stat_t>::iterator s = pg_stat.find(removed_pg);
if (s != pg_stat.end()) {
stat_pg_sub(removed_pg, s->second);
pg_stat.erase(s);
@@ -251,7 +251,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
for (set<int>::iterator p = inc.get_osd_stat_rm().begin();
p != inc.get_osd_stat_rm().end();
++p) {
- hash_map<int32_t,osd_stat_t>::iterator t = osd_stat.find(*p);
+ ceph::unordered_map<int32_t,osd_stat_t>::iterator t = osd_stat.find(*p);
if (t != osd_stat.end()) {
stat_osd_sub(t->second);
osd_stat.erase(t);
@@ -287,7 +287,7 @@ void PGMap::redo_full_sets()
{
full_osds.clear();
nearfull_osds.clear();
- for (hash_map<int32_t, osd_stat_t>::iterator i = osd_stat.begin();
+ for (ceph::unordered_map<int32_t, osd_stat_t>::iterator i = osd_stat.begin();
i != osd_stat.end();
++i) {
register_nearfull_status(i->first, i->second);
@@ -322,12 +322,12 @@ void PGMap::calc_stats()
pg_sum = pool_stat_t();
osd_sum = osd_stat_t();
- for (hash_map<pg_t,pg_stat_t>::iterator p = pg_stat.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::iterator p = pg_stat.begin();
p != pg_stat.end();
++p) {
stat_pg_add(p->first, p->second);
}
- for (hash_map<int32_t,osd_stat_t>::iterator p = osd_stat.begin();
+ for (ceph::unordered_map<int32_t,osd_stat_t>::iterator p = osd_stat.begin();
p != osd_stat.end();
++p)
stat_osd_add(p->second);
@@ -338,7 +338,7 @@ void PGMap::calc_stats()
void PGMap::update_pg(pg_t pgid, bufferlist& bl)
{
bufferlist::iterator p = bl.begin();
- hash_map<pg_t,pg_stat_t>::iterator s = pg_stat.find(pgid);
+ ceph::unordered_map<pg_t,pg_stat_t>::iterator s = pg_stat.find(pgid);
if (s != pg_stat.end())
stat_pg_sub(pgid, s->second);
pg_stat_t& r = pg_stat[pgid];
@@ -348,7 +348,7 @@ void PGMap::update_pg(pg_t pgid, bufferlist& bl)
void PGMap::remove_pg(pg_t pgid)
{
- hash_map<pg_t,pg_stat_t>::iterator s = pg_stat.find(pgid);
+ ceph::unordered_map<pg_t,pg_stat_t>::iterator s = pg_stat.find(pgid);
if (s != pg_stat.end()) {
stat_pg_sub(pgid, s->second);
pg_stat.erase(s);
@@ -358,7 +358,7 @@ void PGMap::remove_pg(pg_t pgid)
void PGMap::update_osd(int osd, bufferlist& bl)
{
bufferlist::iterator p = bl.begin();
- hash_map<int32_t,osd_stat_t>::iterator o = osd_stat.find(osd);
+ ceph::unordered_map<int32_t,osd_stat_t>::iterator o = osd_stat.find(osd);
if (o != osd_stat.end())
stat_osd_sub(o->second);
osd_stat_t& r = osd_stat[osd];
@@ -371,7 +371,7 @@ void PGMap::update_osd(int osd, bufferlist& bl)
void PGMap::remove_osd(int osd)
{
- hash_map<int32_t,osd_stat_t>::iterator o = osd_stat.find(osd);
+ ceph::unordered_map<int32_t,osd_stat_t>::iterator o = osd_stat.find(osd);
if (o != osd_stat.end()) {
stat_osd_sub(o->second);
osd_stat.erase(o);
@@ -433,7 +433,7 @@ epoch_t PGMap::calc_min_last_epoch_clean() const
{
if (pg_stat.empty())
return 0;
- hash_map<pg_t,pg_stat_t>::const_iterator p = pg_stat.begin();
+ ceph::unordered_map<pg_t,pg_stat_t>::const_iterator p = pg_stat.begin();
epoch_t min = p->second.get_effective_last_epoch_clean();
for (++p; p != pg_stat.end(); ++p) {
epoch_t lec = p->second.get_effective_last_epoch_clean();
@@ -442,7 +442,7 @@ epoch_t PGMap::calc_min_last_epoch_clean() const
}
// also scan osd epochs
// don't trim past the oldest reported osd epoch
- for (hash_map<int32_t, epoch_t>::const_iterator i = osd_epochs.begin();
+ for (ceph::unordered_map<int32_t, epoch_t>::const_iterator i = osd_epochs.begin();
i != osd_epochs.end();
++i) {
if (i->second < min)
@@ -481,7 +481,7 @@ void PGMap::encode(bufferlist &bl, uint64_t features) const
void PGMap::decode(bufferlist::iterator &bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(5, 4, 4, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(6, 4, 4, bl);
::decode(version, bl);
if (struct_v < 3) {
pg_stat.clear();
@@ -508,7 +508,7 @@ void PGMap::decode(bufferlist::iterator &bl)
if (struct_v >= 6) {
::decode(osd_epochs, bl);
} else {
- for (hash_map<int32_t, osd_stat_t>::iterator i = osd_stat.begin();
+ for (ceph::unordered_map<int32_t, osd_stat_t>::iterator i = osd_stat.begin();
i != osd_stat.end();
++i) {
// This isn't accurate, but will cause trimming to behave like
@@ -528,10 +528,10 @@ void PGMap::dirty_all(Incremental& inc)
inc.full_ratio = full_ratio;
inc.nearfull_ratio = nearfull_ratio;
- for (hash_map<pg_t,pg_stat_t>::const_iterator p = pg_stat.begin(); p != pg_stat.end(); ++p) {
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator p = pg_stat.begin(); p != pg_stat.end(); ++p) {
inc.pg_stat_updates[p->first] = p->second;
}
- for (hash_map<int32_t, osd_stat_t>::const_iterator p = osd_stat.begin(); p != osd_stat.end(); ++p) {
+ for (ceph::unordered_map<int32_t, osd_stat_t>::const_iterator p = osd_stat.begin(); p != osd_stat.end(); ++p) {
assert(osd_epochs.count(p->first));
inc.update_stat(p->first,
inc.get_osd_epochs().find(p->first)->second,
@@ -577,7 +577,7 @@ void PGMap::dump_delta(Formatter *f) const
void PGMap::dump_pg_stats(Formatter *f, bool brief) const
{
f->open_array_section("pg_stats");
- for (hash_map<pg_t,pg_stat_t>::const_iterator i = pg_stat.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator i = pg_stat.begin();
i != pg_stat.end();
++i) {
f->open_object_section("pg_stat");
@@ -594,7 +594,7 @@ void PGMap::dump_pg_stats(Formatter *f, bool brief) const
void PGMap::dump_pool_stats(Formatter *f) const
{
f->open_array_section("pool_stats");
- for (hash_map<int,pool_stat_t>::const_iterator p = pg_pool_sum.begin();
+ for (ceph::unordered_map<int,pool_stat_t>::const_iterator p = pg_pool_sum.begin();
p != pg_pool_sum.end();
++p) {
f->open_object_section("pool_stat");
@@ -608,7 +608,7 @@ void PGMap::dump_pool_stats(Formatter *f) const
void PGMap::dump_osd_stats(Formatter *f) const
{
f->open_array_section("osd_stats");
- for (hash_map<int32_t,osd_stat_t>::const_iterator q = osd_stat.begin();
+ for (ceph::unordered_map<int32_t,osd_stat_t>::const_iterator q = osd_stat.begin();
q != osd_stat.end();
++q) {
f->open_object_section("osd_stat");
@@ -620,10 +620,10 @@ void PGMap::dump_osd_stats(Formatter *f) const
}
void PGMap::dump_pg_stats_plain(ostream& ss,
- const hash_map<pg_t, pg_stat_t>& pg_stats) const
+ const ceph::unordered_map<pg_t, pg_stat_t>& pg_stats) const
{
ss << "pg_stat\tobjects\tmip\tdegr\tunf\tbytes\tlog\tdisklog\tstate\tstate_stamp\tv\treported\tup\tacting\tlast_scrub\tscrub_stamp\tlast_deep_scrub\tdeep_scrub_stamp" << std::endl;
- for (hash_map<pg_t, pg_stat_t>::const_iterator i = pg_stats.begin();
+ for (ceph::unordered_map<pg_t, pg_stat_t>::const_iterator i = pg_stats.begin();
i != pg_stats.end(); ++i) {
const pg_stat_t &st(i->second);
ss << i->first
@@ -656,7 +656,7 @@ void PGMap::dump(ostream& ss) const
ss << "full_ratio " << full_ratio << std::endl;
ss << "nearfull_ratio " << nearfull_ratio << std::endl;
dump_pg_stats_plain(ss, pg_stat);
- for (hash_map<int,pool_stat_t>::const_iterator p = pg_pool_sum.begin();
+ for (ceph::unordered_map<int,pool_stat_t>::const_iterator p = pg_pool_sum.begin();
p != pg_pool_sum.end();
++p)
ss << "pool " << p->first
@@ -679,7 +679,7 @@ void PGMap::dump(ostream& ss) const
<< "\t" << pg_sum.ondisk_log_size
<< std::endl;
ss << "osdstat\tkbused\tkbavail\tkb\thb in\thb out" << std::endl;
- for (hash_map<int32_t,osd_stat_t>::const_iterator p = osd_stat.begin();
+ for (ceph::unordered_map<int32_t,osd_stat_t>::const_iterator p = osd_stat.begin();
p != osd_stat.end();
++p)
ss << p->first
@@ -696,9 +696,9 @@ void PGMap::dump(ostream& ss) const
}
void PGMap::get_stuck_stats(PGMap::StuckPG type, utime_t cutoff,
- hash_map<pg_t, pg_stat_t>& stuck_pgs) const
+ ceph::unordered_map<pg_t, pg_stat_t>& stuck_pgs) const
{
- for (hash_map<pg_t, pg_stat_t>::const_iterator i = pg_stat.begin();
+ for (ceph::unordered_map<pg_t, pg_stat_t>::const_iterator i = pg_stat.begin();
i != pg_stat.end();
++i) {
utime_t val;
@@ -730,10 +730,10 @@ void PGMap::get_stuck_stats(PGMap::StuckPG type, utime_t cutoff,
void PGMap::dump_stuck(Formatter *f, PGMap::StuckPG type, utime_t cutoff) const
{
- hash_map<pg_t, pg_stat_t> stuck_pg_stats;
+ ceph::unordered_map<pg_t, pg_stat_t> stuck_pg_stats;
get_stuck_stats(type, cutoff, stuck_pg_stats);
f->open_array_section("stuck_pg_stats");
- for (hash_map<pg_t,pg_stat_t>::const_iterator i = stuck_pg_stats.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator i = stuck_pg_stats.begin();
i != stuck_pg_stats.end();
++i) {
f->open_object_section("pg_stat");
@@ -746,7 +746,7 @@ void PGMap::dump_stuck(Formatter *f, PGMap::StuckPG type, utime_t cutoff) const
void PGMap::dump_stuck_plain(ostream& ss, PGMap::StuckPG type, utime_t cutoff) const
{
- hash_map<pg_t, pg_stat_t> stuck_pg_stats;
+ ceph::unordered_map<pg_t, pg_stat_t> stuck_pg_stats;
get_stuck_stats(type, cutoff, stuck_pg_stats);
if (!stuck_pg_stats.empty())
dump_pg_stats_plain(ss, stuck_pg_stats);
@@ -755,7 +755,7 @@ void PGMap::dump_stuck_plain(ostream& ss, PGMap::StuckPG type, utime_t cutoff) c
void PGMap::dump_osd_perf_stats(Formatter *f) const
{
f->open_array_section("osd_perf_infos");
- for (hash_map<int32_t, osd_stat_t>::const_iterator i = osd_stat.begin();
+ for (ceph::unordered_map<int32_t, osd_stat_t>::const_iterator i = osd_stat.begin();
i != osd_stat.end();
++i) {
f->open_object_section("osd");
@@ -775,7 +775,7 @@ void PGMap::print_osd_perf_stats(std::ostream *ss) const
tab.define_column("osdid", TextTable::LEFT, TextTable::RIGHT);
tab.define_column("fs_commit_latency(ms)", TextTable::LEFT, TextTable::RIGHT);
tab.define_column("fs_apply_latency(ms)", TextTable::LEFT, TextTable::RIGHT);
- for (hash_map<int32_t, osd_stat_t>::const_iterator i = osd_stat.begin();
+ for (ceph::unordered_map<int32_t, osd_stat_t>::const_iterator i = osd_stat.begin();
i != osd_stat.end();
++i) {
tab << i->first;
@@ -798,7 +798,7 @@ void PGMap::recovery_summary(Formatter *f, ostream *out,
if (f) {
f->dump_unsigned("degraded_objects", delta_sum.stats.sum.num_objects_degraded);
f->dump_unsigned("degraded_total", delta_sum.stats.sum.num_object_copies);
- f->dump_string("degrated_ratio", b);
+ f->dump_string("degraded_ratio", b);
} else {
*out << delta_sum.stats.sum.num_objects_degraded
<< "/" << delta_sum.stats.sum.num_object_copies << " objects degraded (" << b << "%)";
@@ -865,11 +865,11 @@ void PGMap::overall_recovery_summary(Formatter *f, ostream *out) const
void PGMap::pool_recovery_rate_summary(Formatter *f, ostream *out,
uint64_t poolid) const
{
- hash_map<uint64_t,pair<pool_stat_t,utime_t> >::const_iterator p =
+ ceph::unordered_map<uint64_t,pair<pool_stat_t,utime_t> >::const_iterator p =
per_pool_sum_delta.find(poolid);
if (p == per_pool_sum_delta.end())
return;
- hash_map<uint64_t,utime_t>::const_iterator ts =
+ ceph::unordered_map<uint64_t,utime_t>::const_iterator ts =
per_pool_sum_deltas_stamps.find(p->first);
assert(ts != per_pool_sum_deltas_stamps.end());
recovery_rate_summary(f, out, p->second.first, ts->second);
@@ -878,7 +878,7 @@ void PGMap::pool_recovery_rate_summary(Formatter *f, ostream *out,
void PGMap::pool_recovery_summary(Formatter *f, ostream *out,
uint64_t poolid) const
{
- hash_map<uint64_t,pair<pool_stat_t,utime_t> >::const_iterator p =
+ ceph::unordered_map<uint64_t,pair<pool_stat_t,utime_t> >::const_iterator p =
per_pool_sum_delta.find(poolid);
if (p == per_pool_sum_delta.end())
return;
@@ -926,11 +926,11 @@ void PGMap::overall_client_io_rate_summary(Formatter *f, ostream *out) const
void PGMap::pool_client_io_rate_summary(Formatter *f, ostream *out,
uint64_t poolid) const
{
- hash_map<uint64_t,pair<pool_stat_t,utime_t> >::const_iterator p =
+ ceph::unordered_map<uint64_t,pair<pool_stat_t,utime_t> >::const_iterator p =
per_pool_sum_delta.find(poolid);
if (p == per_pool_sum_delta.end())
return;
- hash_map<uint64_t,utime_t>::const_iterator ts =
+ ceph::unordered_map<uint64_t,utime_t>::const_iterator ts =
per_pool_sum_deltas_stamps.find(p->first);
assert(ts != per_pool_sum_deltas_stamps.end());
client_io_rate_summary(f, out, p->second.first, ts->second);
@@ -1035,9 +1035,9 @@ void PGMap::update_one_pool_delta(CephContext *cct,
* @param pg_pool_sum_old Map of pool stats for delta calcs.
*/
void PGMap::update_pool_deltas(CephContext *cct, const utime_t ts,
- const hash_map<uint64_t,pool_stat_t>& pg_pool_sum_old)
+ const ceph::unordered_map<uint64_t,pool_stat_t>& pg_pool_sum_old)
{
- for (hash_map<uint64_t,pool_stat_t>::const_iterator it = pg_pool_sum_old.begin();
+ for (ceph::unordered_map<uint64_t,pool_stat_t>::const_iterator it = pg_pool_sum_old.begin();
it != pg_pool_sum_old.end(); ++it) {
update_one_pool_delta(cct, ts, it->first, it->second);
}
@@ -1056,7 +1056,7 @@ void PGMap::print_summary(Formatter *f, ostream *out) const
if (f)
f->open_array_section("pgs_by_state");
- for (hash_map<int,int>::const_iterator p = num_pg_by_state.begin();
+ for (ceph::unordered_map<int,int>::const_iterator p = num_pg_by_state.begin();
p != num_pg_by_state.end();
++p) {
if (f) {
@@ -1119,7 +1119,7 @@ void PGMap::print_oneline_summary(ostream *out) const
{
std::stringstream ss;
- for (hash_map<int,int>::const_iterator p = num_pg_by_state.begin();
+ for (ceph::unordered_map<int,int>::const_iterator p = num_pg_by_state.begin();
p != num_pg_by_state.end();
++p) {
if (p != num_pg_by_state.begin())
diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
index 8a931ec..9c34640 100644
--- a/src/mon/PGMap.h
+++ b/src/mon/PGMap.h
@@ -36,15 +36,15 @@ public:
version_t version;
epoch_t last_osdmap_epoch; // last osdmap epoch i applied to the pgmap
epoch_t last_pg_scan; // osdmap epoch
- hash_map<pg_t,pg_stat_t> pg_stat;
- hash_map<int32_t,osd_stat_t> osd_stat;
+ ceph::unordered_map<pg_t,pg_stat_t> pg_stat;
+ ceph::unordered_map<int32_t,osd_stat_t> osd_stat;
set<int32_t> full_osds;
set<int32_t> nearfull_osds;
float full_ratio;
float nearfull_ratio;
// mapping of osd to most recently reported osdmap epoch
- hash_map<int32_t,epoch_t> osd_epochs;
+ ceph::unordered_map<int32_t,epoch_t> osd_epochs;
class Incremental {
public:
@@ -100,9 +100,9 @@ public:
// aggregate stats (soft state), generated by calc_stats()
- hash_map<int,int> num_pg_by_state;
+ ceph::unordered_map<int,int> num_pg_by_state;
int64_t num_pg, num_osd;
- hash_map<int,pool_stat_t> pg_pool_sum;
+ ceph::unordered_map<int,pool_stat_t> pg_pool_sum;
pool_stat_t pg_sum;
osd_stat_t osd_sum;
@@ -113,19 +113,19 @@ public:
* keep track of last deltas for each pool, calculated using
* @p pg_pool_sum as baseline.
*/
- hash_map<uint64_t, list< pair<pool_stat_t, utime_t> > > per_pool_sum_deltas;
+ ceph::unordered_map<uint64_t, list< pair<pool_stat_t, utime_t> > > per_pool_sum_deltas;
/**
* keep track of per-pool timestamp deltas, according to last update on
* each pool.
*/
- hash_map<uint64_t, utime_t> per_pool_sum_deltas_stamps;
+ ceph::unordered_map<uint64_t, utime_t> per_pool_sum_deltas_stamps;
/**
* keep track of sum deltas, per-pool, taking into account any previous
* deltas existing in @p per_pool_sum_deltas. The utime_t as second member
* of the pair is the timestamp refering to the last update (i.e., the first
* member of the pair) for a given pool.
*/
- hash_map<uint64_t, pair<pool_stat_t,utime_t> > per_pool_sum_delta;
+ ceph::unordered_map<uint64_t, pair<pool_stat_t,utime_t> > per_pool_sum_delta;
list< pair<pool_stat_t, utime_t> > pg_sum_deltas;
pool_stat_t pg_sum_delta;
@@ -135,7 +135,7 @@ public:
const utime_t ts, const pool_stat_t& pg_sum_old);
void update_pool_deltas(CephContext *cct,
const utime_t ts,
- const hash_map<uint64_t, pool_stat_t>& pg_pool_sum_old);
+ const ceph::unordered_map<uint64_t, pool_stat_t>& pg_pool_sum_old);
void clear_delta();
private:
@@ -232,9 +232,9 @@ public:
void dump_delta(Formatter *f) const;
void dump_pg_stats_plain(ostream& ss,
- const hash_map<pg_t, pg_stat_t>& pg_stats) const;
+ const ceph::unordered_map<pg_t, pg_stat_t>& pg_stats) const;
void get_stuck_stats(StuckPG type, utime_t cutoff,
- hash_map<pg_t, pg_stat_t>& stuck_pgs) const;
+ ceph::unordered_map<pg_t, pg_stat_t>& stuck_pgs) const;
void dump_stuck(Formatter *f, StuckPG type, utime_t cutoff) const;
void dump_stuck_plain(ostream& ss, StuckPG type, utime_t cutoff) const;
diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index 0ba7970..cdfc83a 100644
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -95,7 +95,7 @@ void PGMonitor::update_logger()
mon->cluster_logger->set(l_cluster_num_pg, pg_map.pg_stat.size());
unsigned active = 0, active_clean = 0, peering = 0;
- for (hash_map<int,int>::iterator p = pg_map.num_pg_by_state.begin();
+ for (ceph::unordered_map<int,int>::iterator p = pg_map.num_pg_by_state.begin();
p != pg_map.num_pg_by_state.end();
++p) {
if (p->first & PG_STATE_ACTIVE) {
@@ -149,7 +149,7 @@ void PGMonitor::tick()
* obtained -- the timestamp IS NOT a delta itself.
*/
if (!pg_map.per_pool_sum_deltas.empty()) {
- hash_map<uint64_t,pair<pool_stat_t,utime_t> >::iterator it;
+ ceph::unordered_map<uint64_t,pair<pool_stat_t,utime_t> >::iterator it;
for (it = pg_map.per_pool_sum_delta.begin();
it != pg_map.per_pool_sum_delta.end(); ) {
utime_t age = ceph_clock_now(g_ceph_context) - it->second.second;
@@ -427,7 +427,7 @@ void PGMonitor::apply_pgmap_delta(bufferlist& bl)
}
pool_stat_t pg_sum_old = pg_map.pg_sum;
- hash_map<uint64_t, pool_stat_t> pg_pool_sum_old;
+ ceph::unordered_map<uint64_t, pool_stat_t> pg_pool_sum_old;
// pgs
bufferlist::iterator p = dirty_pgs.begin();
@@ -706,7 +706,7 @@ bool PGMonitor::preprocess_pg_stats(MPGStats *stats)
bool PGMonitor::pg_stats_have_changed(int from, const MPGStats *stats) const
{
// any new osd info?
- hash_map<int,osd_stat_t>::const_iterator s = pg_map.osd_stat.find(from);
+ ceph::unordered_map<int,osd_stat_t>::const_iterator s = pg_map.osd_stat.find(from);
if (s == pg_map.osd_stat.end())
return true;
if (s->second != stats->osd_stat)
@@ -715,7 +715,7 @@ bool PGMonitor::pg_stats_have_changed(int from, const MPGStats *stats) const
// any new pg info?
for (map<pg_t,pg_stat_t>::const_iterator p = stats->pg_stat.begin();
p != stats->pg_stat.end(); ++p) {
- hash_map<pg_t,pg_stat_t>::const_iterator t = pg_map.pg_stat.find(p->first);
+ ceph::unordered_map<pg_t,pg_stat_t>::const_iterator t = pg_map.pg_stat.find(p->first);
if (t == pg_map.pg_stat.end())
return true;
if (t->second.reported_epoch != p->second.reported_epoch ||
@@ -1028,7 +1028,7 @@ bool PGMonitor::register_new_pgs()
}
// deleted pools?
- for (hash_map<pg_t,pg_stat_t>::const_iterator p = pg_map.pg_stat.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator p = pg_map.pg_stat.begin();
p != pg_map.pg_stat.end(); ++p) {
if (!osdmap->have_pg_pool(p->first.pool())) {
dout(20) << " removing pg_stat " << p->first << " because "
@@ -1155,7 +1155,7 @@ bool PGMonitor::check_down_pgs()
OSDMap *osdmap = &mon->osdmon()->osdmap;
bool ret = false;
- for (hash_map<pg_t,pg_stat_t>::iterator p = pg_map.pg_stat.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::iterator p = pg_map.pg_stat.begin();
p != pg_map.pg_stat.end();
++p) {
if ((p->second.state & PG_STATE_STALE) == 0 &&
@@ -1339,6 +1339,7 @@ void PGMonitor::dump_info(Formatter *f)
f->close_section();
f->dump_unsigned("pgmap_first_committed", get_first_committed());
+ f->dump_unsigned("pgmap_last_committed", get_last_committed());
}
bool PGMonitor::preprocess_command(MMonCommand *m)
@@ -1549,8 +1550,8 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
cmd_getval(g_ceph_context, cmdmap, "debugop", debugop, string("unfound_objects_exist"));
if (debugop == "unfound_objects_exist") {
bool unfound_objects_exist = false;
- hash_map<pg_t,pg_stat_t>::const_iterator end = pg_map.pg_stat.end();
- for (hash_map<pg_t,pg_stat_t>::const_iterator s = pg_map.pg_stat.begin();
+ ceph::unordered_map<pg_t,pg_stat_t>::const_iterator end = pg_map.pg_stat.end();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator s = pg_map.pg_stat.begin();
s != end; ++s) {
if (s->second.stats.sum.num_objects_unfound > 0) {
unfound_objects_exist = true;
@@ -1564,8 +1565,8 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
r = 0;
} else if (debugop == "degraded_pgs_exist") {
bool degraded_pgs_exist = false;
- hash_map<pg_t,pg_stat_t>::const_iterator end = pg_map.pg_stat.end();
- for (hash_map<pg_t,pg_stat_t>::const_iterator s = pg_map.pg_stat.begin();
+ ceph::unordered_map<pg_t,pg_stat_t>::const_iterator end = pg_map.pg_stat.end();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator s = pg_map.pg_stat.begin();
s != end; ++s) {
if (s->second.stats.sum.num_objects_degraded > 0) {
degraded_pgs_exist = true;
@@ -1666,15 +1667,16 @@ bool PGMonitor::prepare_command(MMonCommand *m)
update:
getline(ss, rs);
- wait_for_finished_proposal(new Monitor::C_Command(mon, m, r, rs, get_last_committed()));
+ wait_for_finished_proposal(new Monitor::C_Command(mon, m, r, rs,
+ get_last_committed() + 1));
return true;
}
static void note_stuck_detail(enum PGMap::StuckPG what,
- hash_map<pg_t,pg_stat_t>& stuck_pgs,
+ ceph::unordered_map<pg_t,pg_stat_t>& stuck_pgs,
list<pair<health_status_t,string> > *detail)
{
- for (hash_map<pg_t,pg_stat_t>::iterator p = stuck_pgs.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::iterator p = stuck_pgs.begin();
p != stuck_pgs.end();
++p) {
ostringstream ss;
@@ -1733,8 +1735,8 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
list<pair<health_status_t,string> > *detail) const
{
map<string,int> note;
- hash_map<int,int>::const_iterator p = pg_map.num_pg_by_state.begin();
- hash_map<int,int>::const_iterator p_end = pg_map.num_pg_by_state.end();
+ ceph::unordered_map<int,int>::const_iterator p = pg_map.num_pg_by_state.begin();
+ ceph::unordered_map<int,int>::const_iterator p_end = pg_map.num_pg_by_state.end();
for (; p != p_end; ++p) {
if (p->first & PG_STATE_STALE)
note["stale"] += p->second;
@@ -1764,7 +1766,7 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
note["backfill_toofull"] += p->second;
}
- hash_map<pg_t, pg_stat_t> stuck_pgs;
+ ceph::unordered_map<pg_t, pg_stat_t> stuck_pgs;
utime_t now(ceph_clock_now(g_ceph_context));
utime_t cutoff = now - utime_t(g_conf->mon_pg_stuck_threshold, 0);
@@ -1798,7 +1800,7 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
summary.push_back(make_pair(HEALTH_WARN, ss.str()));
}
if (detail) {
- for (hash_map<pg_t,pg_stat_t>::const_iterator p = pg_map.pg_stat.begin();
+ for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator p = pg_map.pg_stat.begin();
p != pg_map.pg_stat.end();
++p) {
if ((p->second.state & (PG_STATE_STALE |
@@ -1845,7 +1847,7 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
if (detail) {
unsigned num_slow_osds = 0;
// do per-osd warnings
- for (hash_map<int32_t,osd_stat_t>::const_iterator p = pg_map.osd_stat.begin();
+ for (ceph::unordered_map<int32_t,osd_stat_t>::const_iterator p = pg_map.osd_stat.begin();
p != pg_map.osd_stat.end();
++p) {
if (_warn_slow_request_histogram(p->second.op_queue_age_hist,
@@ -1897,7 +1899,7 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
}
}
if (!pg_map.pg_stat.empty()) {
- for (hash_map<int,pool_stat_t>::const_iterator p = pg_map.pg_pool_sum.begin();
+ for (ceph::unordered_map<int,pool_stat_t>::const_iterator p = pg_map.pg_pool_sum.begin();
p != pg_map.pg_pool_sum.end();
++p) {
const pg_pool_t *pi = mon->osdmon()->osdmap.get_pg_pool(p->first);
diff --git a/src/mon/Session.h b/src/mon/Session.h
index 7df6001..a5f67d9 100644
--- a/src/mon/Session.h
+++ b/src/mon/Session.h
@@ -147,7 +147,8 @@ struct MonSessionMap {
bool backward = true, forward = true;
while (backward || forward) {
if (backward) {
- if (osdmap->is_up(b->first)) {
+ if (osdmap->is_up(b->first) &&
+ osdmap->get_addr(b->first) == b->second->con->get_peer_addr()) {
s = b->second;
break;
}
diff --git a/src/mount/mount.ceph.c b/src/mount/mount.ceph.c
old mode 100755
new mode 100644
index 5bbedd0..563cd07
--- a/src/mount/mount.ceph.c
+++ b/src/mount/mount.ceph.c
@@ -17,6 +17,7 @@
#define MAX_SECRET_OPTION_LEN (MAX_SECRET_LEN + 7)
int verboseflag = 0;
+int skip_mtab_flag = 0;
static const char * const EMPTY_STRING = "";
/* TODO duplicates logic from kernel */
@@ -267,6 +268,8 @@ static int parse_arguments(int argc, char *const *const argv,
for (i = 3; i < argc; ++i) {
if (!strcmp("-h", argv[i]))
return 1;
+ else if (!strcmp("-n", argv[i]))
+ skip_mtab_flag = 1;
else if (!strcmp("-v", argv[i]))
verboseflag = 1;
else if (!strcmp("-o", argv[i])) {
@@ -313,10 +316,11 @@ static void modprobe(void) {
static void usage(const char *prog_name)
{
- printf("usage: %s [src] [mount-point] [-v] [-o ceph-options]\n",
+ printf("usage: %s [src] [mount-point] [-n] [-v] [-o ceph-options]\n",
prog_name);
printf("options:\n");
printf("\t-h: Print this help\n");
+ printf("\t-n: Do not update /etc/mtab\n");
printf("\t-v: Verbose\n");
printf("\tceph-options: refer to mount.ceph(8)\n");
printf("\n");
@@ -362,7 +366,9 @@ int main(int argc, char *argv[])
printf("mount error %d = %s\n",errno,strerror(errno));
}
} else {
- update_mtab_entry(rsrc, node, "ceph", popts, flags, 0, 0);
+ if (!skip_mtab_flag) {
+ update_mtab_entry(rsrc, node, "ceph", popts, flags, 0, 0);
+ }
}
block_signals(SIG_UNBLOCK);
diff --git a/src/msg/Accepter.cc b/src/msg/Accepter.cc
index d6e94d1..5eebb2e 100644
--- a/src/msg/Accepter.cc
+++ b/src/msg/Accepter.cc
@@ -155,8 +155,6 @@ int Accepter::rebind(const set<int>& avoid_ports)
{
ldout(msgr->cct,1) << "accepter.rebind avoid " << avoid_ports << dendl;
- stop();
-
// invalidate our previously learned address.
msgr->unlearn_addr();
diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index 66b64d0..6389e20 100644
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -48,7 +48,25 @@ ostream& Pipe::_pipe_prefix(std::ostream *_dout) {
<< ").";
}
+/*
+ * This optimization may not be available on all platforms (e.g. OSX).
+ * Apparently a similar approach based on TCP_CORK can be used.
+ */
+#ifndef MSG_MORE
+# define MSG_MORE 0
+#endif
+/*
+ * On BSD SO_NOSIGPIPE can be set via setsockopt to block SIGPIPE.
+ */
+#ifndef MSG_NOSIGNAL
+# define MSG_NOSIGNAL 0
+# ifdef SO_NOSIGPIPE
+# define CEPH_USE_SO_NOSIGPIPE
+# else
+# error "Cannot block SIGPIPE!"
+# endif
+#endif
/**************************************
* Pipe
@@ -754,6 +772,16 @@ void Pipe::set_socket_options()
ldout(msgr->cct,0) << "couldn't set SO_RCVBUF to " << size << ": " << cpp_strerror(r) << dendl;
}
}
+
+ // block ESIGPIPE
+#ifdef CEPH_USE_SO_NOSIGPIPE
+ int val = 1;
+ int r = ::setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, (void*)&val, sizeof(val));
+ if (r) {
+ r = -errno;
+ ldout(msgr->cct,0) << "couldn't set SO_NOSIGPIPE: " << cpp_strerror(r) << dendl;
+ }
+#endif
}
int Pipe::connect()
@@ -1126,7 +1154,7 @@ void Pipe::register_pipe()
void Pipe::unregister_pipe()
{
assert(msgr->lock.is_locked());
- hash_map<entity_addr_t,Pipe*>::iterator p = msgr->rank_pipe.find(peer_addr);
+ ceph::unordered_map<entity_addr_t,Pipe*>::iterator p = msgr->rank_pipe.find(peer_addr);
if (p != msgr->rank_pipe.end() && p->second == this) {
ldout(msgr->cct,10) << "unregister_pipe" << dendl;
msgr->rank_pipe.erase(p);
diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index 441ed43..ef97b01 100644
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -23,6 +23,7 @@
#include "common/Timer.h"
#include "common/errno.h"
#include "auth/Crypto.h"
+#include "include/Spinlock.h"
#define dout_subsys ceph_subsys_ms
#undef dout_prefix
@@ -53,7 +54,7 @@ SimpleMessenger::SimpleMessenger(CephContext *cct, entity_name_t name,
timeout(0),
local_connection(new Connection(this))
{
- pthread_spin_init(&global_seq_lock, PTHREAD_PROCESS_PRIVATE);
+ ceph_spin_init(&global_seq_lock);
init_local_connection();
}
@@ -277,9 +278,9 @@ int SimpleMessenger::rebind(const set<int>& avoid_ports)
{
ldout(cct,1) << "rebind avoid " << avoid_ports << dendl;
assert(did_bind);
- int r = accepter.rebind(avoid_ports);
+ accepter.stop();
mark_down_all();
- return r;
+ return accepter.rebind(avoid_ports);
}
int SimpleMessenger::start()
@@ -574,7 +575,7 @@ void SimpleMessenger::mark_down_all()
accepting_pipes.clear();
while (!rank_pipe.empty()) {
- hash_map<entity_addr_t,Pipe*>::iterator it = rank_pipe.begin();
+ ceph::unordered_map<entity_addr_t,Pipe*>::iterator it = rank_pipe.begin();
Pipe *p = it->second;
ldout(cct,5) << "mark_down_all " << it->first << " " << p << dendl;
rank_pipe.erase(it);
diff --git a/src/msg/SimpleMessenger.h b/src/msg/SimpleMessenger.h
index 6860c6c..e6e1fb1 100644
--- a/src/msg/SimpleMessenger.h
+++ b/src/msg/SimpleMessenger.h
@@ -21,9 +21,8 @@
#include <list>
#include <map>
using namespace std;
-#include <ext/hash_map>
-#include <ext/hash_set>
-using namespace __gnu_cxx;
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
#include "common/Mutex.h"
#include "include/atomic.h"
@@ -38,6 +37,7 @@ using namespace __gnu_cxx;
#include "Pipe.h"
#include "Accepter.h"
+#include "include/Spinlock.h"
/*
* This class handles transmission and reception of messages. Generally
@@ -308,7 +308,7 @@ private:
/// counter for the global seq our connection protocol uses
__u32 global_seq;
/// lock to protect the global_seq
- pthread_spinlock_t global_seq_lock;
+ ceph_spinlock_t global_seq_lock;
/**
* hash map of addresses to Pipes
@@ -316,7 +316,7 @@ private:
* NOTE: a Pipe* with state CLOSED may still be in the map but is considered
* invalid and can be replaced by anyone holding the msgr lock
*/
- hash_map<entity_addr_t, Pipe*> rank_pipe;
+ ceph::unordered_map<entity_addr_t, Pipe*> rank_pipe;
/**
* list of pipes are in teh process of accepting
*
@@ -350,7 +350,7 @@ private:
friend class Pipe;
Pipe *_lookup_pipe(const entity_addr_t& k) {
- hash_map<entity_addr_t, Pipe*>::iterator p = rank_pipe.find(k);
+ ceph::unordered_map<entity_addr_t, Pipe*>::iterator p = rank_pipe.find(k);
if (p == rank_pipe.end())
return NULL;
// see lock cribbing in Pipe::fault()
@@ -388,11 +388,11 @@ public:
* @return a global sequence ID that nobody else has seen.
*/
__u32 get_global_seq(__u32 old=0) {
- pthread_spin_lock(&global_seq_lock);
+ ceph_spin_lock(&global_seq_lock);
if (old > global_seq)
global_seq = old;
__u32 ret = ++global_seq;
- pthread_spin_unlock(&global_seq_lock);
+ ceph_spin_unlock(&global_seq_lock);
return ret;
}
/**
diff --git a/src/msg/msg_types.h b/src/msg/msg_types.h
index 40b3f0b..5d86b3a 100644
--- a/src/msg/msg_types.h
+++ b/src/msg/msg_types.h
@@ -20,6 +20,7 @@
#include "include/types.h"
#include "include/blobhash.h"
#include "include/encoding.h"
+#include "include/hash_namespace.h"
namespace ceph {
class Formatter;
@@ -139,7 +140,7 @@ inline std::ostream& operator<<(std::ostream& out, const ceph_entity_name& addr)
return out << *(const entity_name_t*)&addr;
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash< entity_name_t >
{
size_t operator()( const entity_name_t &m ) const
@@ -147,7 +148,7 @@ namespace __gnu_cxx {
return rjhash32(m.type() ^ m.num());
}
};
-}
+CEPH_HASH_NAMESPACE_END
@@ -359,7 +360,7 @@ inline bool operator<=(const entity_addr_t& a, const entity_addr_t& b) { return
inline bool operator>(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) > 0; }
inline bool operator>=(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) >= 0; }
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash< entity_addr_t >
{
size_t operator()( const entity_addr_t& x ) const
@@ -368,7 +369,7 @@ namespace __gnu_cxx {
return H((const char*)&x, sizeof(x));
}
};
-}
+CEPH_HASH_NAMESPACE_END
/*
@@ -413,7 +414,7 @@ inline bool operator<=(const entity_inst_t& a, const entity_inst_t& b) {
inline bool operator>(const entity_inst_t& a, const entity_inst_t& b) { return b < a; }
inline bool operator>=(const entity_inst_t& a, const entity_inst_t& b) { return b <= a; }
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash< entity_inst_t >
{
size_t operator()( const entity_inst_t& x ) const
@@ -423,7 +424,7 @@ namespace __gnu_cxx {
return H(x.name) ^ I(x.addr);
}
};
-}
+CEPH_HASH_NAMESPACE_END
inline ostream& operator<<(ostream& out, const entity_inst_t &i)
diff --git a/src/objclass/class_api.cc b/src/objclass/class_api.cc
index bb26c75..46a8ff5 100644
--- a/src/objclass/class_api.cc
+++ b/src/objclass/class_api.cc
@@ -157,7 +157,7 @@ int cls_read(cls_method_context_t hctx, int ofs, int len,
{
ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
vector<OSDOp> ops(1);
- ops[0].op.op = CEPH_OSD_OP_READ;
+ ops[0].op.op = CEPH_OSD_OP_SYNC_READ;
ops[0].op.extent.offset = ofs;
ops[0].op.extent.length = len;
int r = (*pctx)->pg->do_osd_ops(*pctx, ops);
@@ -228,7 +228,7 @@ int cls_cxx_read(cls_method_context_t hctx, int ofs, int len, bufferlist *outbl)
ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
vector<OSDOp> ops(1);
int ret;
- ops[0].op.op = CEPH_OSD_OP_READ;
+ ops[0].op.op = CEPH_OSD_OP_SYNC_READ;
ops[0].op.extent.offset = ofs;
ops[0].op.extent.length = len;
ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
diff --git a/src/ocf/Makefile.in b/src/ocf/Makefile.in
index 032e475..5362dae 100644
--- a/src/ocf/Makefile.in
+++ b/src/ocf/Makefile.in
@@ -45,7 +45,10 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
$(top_srcdir)/m4/ac_prog_javac_works.m4 \
$(top_srcdir)/m4/ac_prog_javah.m4 \
$(top_srcdir)/m4/acx_pthread.m4 \
+ $(top_srcdir)/m4/ax_c_pretty_func.m4 \
+ $(top_srcdir)/m4/ax_c_var_func.m4 \
$(top_srcdir)/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/m4/ax_cxx_static_cast.m4 \
$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/pkg.m4 \
@@ -103,6 +106,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
+BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@
CC = @CC@
CCAS = @CCAS@
CCASDEPMODE = @CCASDEPMODE@
@@ -144,6 +148,7 @@ JAR = @JAR@
JAVAC = @JAVAC@
JAVAH = @JAVAH@
JDK_CPPFLAGS = @JDK_CPPFLAGS@
+KEYUTILS_LIB = @KEYUTILS_LIB@
LD = @LD@
LDFLAGS = @LDFLAGS@
LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@
@@ -189,6 +194,7 @@ PYTHON_PLATFORM = @PYTHON_PLATFORM@
PYTHON_PREFIX = @PYTHON_PREFIX@
PYTHON_VERSION = @PYTHON_VERSION@
RANLIB = @RANLIB@
+RESOLV_LIBS = @RESOLV_LIBS@
RPM_RELEASE = @RPM_RELEASE@
SED = @SED@
SET_MAKE = @SET_MAKE@
diff --git a/src/os/CollectionIndex.h b/src/os/CollectionIndex.h
index 89b7b86..529db01 100644
--- a/src/os/CollectionIndex.h
+++ b/src/os/CollectionIndex.h
@@ -17,7 +17,7 @@
#include <string>
#include <vector>
-#include <tr1/memory>
+#include "include/memory.h"
#include "osd/osd_types.h"
#include "include/object.h"
@@ -43,14 +43,14 @@ protected:
/// Returned path
string full_path;
/// Ref to parent Index
- std::tr1::shared_ptr<CollectionIndex> parent_ref;
+ ceph::shared_ptr<CollectionIndex> parent_ref;
/// coll_t for parent Index
coll_t parent_coll;
/// Normal Constructor
Path(
string path, ///< [in] Path to return.
- std::tr1::weak_ptr<CollectionIndex> ref) ///< [in] weak_ptr to parent.
+ ceph::weak_ptr<CollectionIndex> ref) ///< [in] weak_ptr to parent.
: full_path(path), parent_ref(ref), parent_coll(parent_ref->coll()) {}
/// Debugging Constructor
@@ -66,13 +66,13 @@ protected:
coll_t coll() const { return parent_coll; }
/// Getter for parent
- std::tr1::shared_ptr<CollectionIndex> get_index() const {
+ ceph::shared_ptr<CollectionIndex> get_index() const {
return parent_ref;
}
};
public:
/// Type of returned paths
- typedef std::tr1::shared_ptr<Path> IndexedPath;
+ typedef ceph::shared_ptr<Path> IndexedPath;
static IndexedPath get_testing_path(string path, coll_t collection) {
return IndexedPath(new Path(path, collection));
@@ -99,7 +99,7 @@ protected:
*
* @see IndexManager
*/
- virtual void set_ref(std::tr1::shared_ptr<CollectionIndex> ref) = 0;
+ virtual void set_ref(ceph::shared_ptr<CollectionIndex> ref) = 0;
/**
* Initializes the index.
@@ -161,7 +161,7 @@ protected:
virtual int split(
uint32_t match, //< [in] value to match
uint32_t bits, //< [in] bits to check
- std::tr1::shared_ptr<CollectionIndex> dest //< [in] destination index
+ ceph::shared_ptr<CollectionIndex> dest //< [in] destination index
) { assert(0); return 0; }
diff --git a/src/os/DBObjectMap.cc b/src/os/DBObjectMap.cc
index 635870b..886658f 100644
--- a/src/os/DBObjectMap.cc
+++ b/src/os/DBObjectMap.cc
@@ -7,7 +7,7 @@
#include <set>
#include <map>
#include <string>
-#include <tr1/memory>
+#include "include/memory.h"
#include <vector>
#include "ObjectMap.h"
@@ -769,6 +769,42 @@ int DBObjectMap::rm_keys(const ghobject_t &oid,
return db->submit_transaction(t);
}
+int DBObjectMap::clear_keys_header(const ghobject_t &oid,
+ const SequencerPosition *spos)
+{
+ KeyValueDB::Transaction t = db->get_transaction();
+ Header header = lookup_map_header(oid);
+ if (!header)
+ return -ENOENT;
+ if (check_spos(oid, header, spos))
+ return 0;
+
+ // save old attrs
+ KeyValueDB::Iterator iter = db->get_iterator(xattr_prefix(header));
+ if (!iter)
+ return -EINVAL;
+ map<string, bufferlist> attrs;
+ for (iter->seek_to_first(); !iter->status() && iter->valid(); iter->next())
+ attrs.insert(make_pair(iter->key(), iter->value()));
+ if (iter->status())
+ return iter->status();
+
+ // remove current header
+ remove_map_header(oid, header, t);
+ assert(header->num_children > 0);
+ header->num_children--;
+ int r = _clear(header, t);
+ if (r < 0)
+ return r;
+
+ // create new header
+ Header newheader = generate_new_header(oid, Header());
+ set_map_header(oid, *newheader, t);
+ if (attrs.size())
+ t->set(xattr_prefix(newheader), attrs);
+ return db->submit_transaction(t);
+}
+
int DBObjectMap::get(const ghobject_t &oid,
bufferlist *_header,
map<string, bufferlist> *out)
diff --git a/src/os/DBObjectMap.h b/src/os/DBObjectMap.h
index 459447f..a71c369 100644
--- a/src/os/DBObjectMap.h
+++ b/src/os/DBObjectMap.h
@@ -8,7 +8,7 @@
#include <string>
#include <vector>
-#include <tr1/memory>
+#include "include/memory.h"
#include <boost/scoped_ptr.hpp>
#include "ObjectMap.h"
@@ -94,6 +94,11 @@ public:
const SequencerPosition *spos=0
);
+ int clear_keys_header(
+ const ghobject_t &oid,
+ const SequencerPosition *spos=0
+ );
+
int rm_keys(
const ghobject_t &oid,
const set<string> &to_clear,
@@ -275,7 +280,7 @@ public:
coll_t *c, ghobject_t *oid);
private:
/// Implicit lock on Header->seq
- typedef std::tr1::shared_ptr<_Header> Header;
+ typedef ceph::shared_ptr<_Header> Header;
string map_header_key(const ghobject_t &oid);
string header_key(uint64_t seq);
@@ -311,12 +316,12 @@ private:
Header header;
/// parent_iter == NULL iff no parent
- std::tr1::shared_ptr<DBObjectMapIteratorImpl> parent_iter;
+ ceph::shared_ptr<DBObjectMapIteratorImpl> parent_iter;
KeyValueDB::Iterator key_iter;
KeyValueDB::Iterator complete_iter;
/// cur_iter points to currently valid iterator
- std::tr1::shared_ptr<ObjectMapIteratorImpl> cur_iter;
+ ceph::shared_ptr<ObjectMapIteratorImpl> cur_iter;
int r;
/// init() called, key_iter, complete_iter, parent_iter filled in
@@ -355,7 +360,7 @@ private:
int adjust();
};
- typedef std::tr1::shared_ptr<DBObjectMapIteratorImpl> DBObjectMapIterator;
+ typedef ceph::shared_ptr<DBObjectMapIteratorImpl> DBObjectMapIterator;
DBObjectMapIterator _get_iterator(Header header) {
return DBObjectMapIterator(new DBObjectMapIteratorImpl(this, header));
}
diff --git a/src/os/FDCache.h b/src/os/FDCache.h
index 93557d4..13d7445 100644
--- a/src/os/FDCache.h
+++ b/src/os/FDCache.h
@@ -61,7 +61,7 @@ public:
~FDCache() {
cct->_conf->remove_observer(this);
}
- typedef std::tr1::shared_ptr<FD> FDRef;
+ typedef ceph::shared_ptr<FD> FDRef;
FDRef lookup(const ghobject_t &hoid) {
return registry.lookup(hoid);
diff --git a/src/os/FileJournal.cc b/src/os/FileJournal.cc
index 4a2af08..3a344fa 100644
--- a/src/os/FileJournal.cc
+++ b/src/os/FileJournal.cc
@@ -11,6 +11,7 @@
* Foundation. See file COPYING.
*
*/
+#include "acconfig.h"
#include "common/debug.h"
#include "common/errno.h"
@@ -18,7 +19,7 @@
#include "FileJournal.h"
#include "include/color.h"
#include "common/perf_counters.h"
-#include "os/ObjectStore.h"
+#include "os/FileStore.h"
#include "include/compat.h"
@@ -31,7 +32,7 @@
#include <sys/mount.h>
#include "common/blkdev.h"
-
+#include "common/linux_version.h"
#define dout_subsys ceph_subsys_journal
#undef dout_prefix
@@ -158,49 +159,10 @@ int FileJournal::_open_block_device()
return 0;
}
-static int get_kernel_version(int *a, int *b, int *c)
-{
- int ret;
- char buf[128];
- memset(buf, 0, sizeof(buf));
- int fd = TEMP_FAILURE_RETRY(::open("/proc/version", O_RDONLY));
- if (fd < 0) {
- ret = errno;
- derr << "get_kernel_version: failed to open /proc/version: "
- << cpp_strerror(ret) << dendl;
- goto out;
- }
- ret = safe_read(fd, buf, sizeof(buf) - 1);
- if (ret < 0) {
- derr << "get_kernel_version: failed to read from /proc/version: "
- << cpp_strerror(ret) << dendl;
- goto close_fd;
- }
-
- if (sscanf(buf, "Linux version %d.%d.%d", a, b, c) != 3) {
- if (sscanf(buf, "Linux version %d.%d", a, b) != 2) {
- derr << "get_kernel_version: failed to parse string: '"
- << buf << "'" << dendl;
- ret = EIO;
- goto close_fd;
- }
- *c = 0;
- }
-
- dout(0) << " kernel version is " << *a <<"." << *b << "." << *c << dendl;
- ret = 0;
-
-close_fd:
- TEMP_FAILURE_RETRY(::close(fd));
-out:
- return ret;
-}
-
void FileJournal::_check_disk_write_cache() const
{
ostringstream hdparm_cmd;
FILE *fp = NULL;
- int a, b, c;
if (geteuid() != 0) {
dout(10) << "_check_disk_write_cache: not root, NOT checking disk write "
@@ -243,11 +205,10 @@ void FileJournal::_check_disk_write_cache() const
}
// is our kernel new enough?
- if (get_kernel_version(&a, &b, &c)) {
- dout(10) << "_check_disk_write_cache: failed to get kernel version."
- << dendl;
- }
- else if ((a >= 2 && b >= 6 && c >= 33) || a >= 3) {
+ int ver = get_linux_version();
+ if (ver == 0) {
+ dout(10) << "_check_disk_write_cache: get_linux_version failed" << dendl;
+ } else if (ver >= KERNEL_VERSION(2, 6, 33)) {
dout(20) << "_check_disk_write_cache: disk write cache is on, but your "
<< "kernel is new enough to handle it correctly. (fn:"
<< fn << ")" << dendl;
@@ -264,9 +225,9 @@ void FileJournal::_check_disk_write_cache() const
}
close_f:
- if (::fclose(fp)) {
+ if (pclose(fp)) {
int ret = -errno;
- derr << "_check_disk_write_cache: fclose error: " << cpp_strerror(ret)
+ derr << "_check_disk_write_cache: pclose failed: " << cpp_strerror(ret)
<< dendl;
}
done:
@@ -298,6 +259,7 @@ int FileJournal::_open_file(int64_t oldsize, blksize_t blksize,
<< newsize << " bytes: " << cpp_strerror(err) << dendl;
return -err;
}
+#ifdef HAVE_POSIX_FALLOCATE
ret = ::posix_fallocate(fd, 0, newsize);
if (ret) {
derr << "FileJournal::_open_file : unable to preallocation journal to "
@@ -305,6 +267,24 @@ int FileJournal::_open_file(int64_t oldsize, blksize_t blksize,
return -ret;
}
max_size = newsize;
+#elif defined(__APPLE__)
+ fstore_t store;
+ store.fst_flags = F_ALLOCATECONTIG;
+ store.fst_posmode = F_PEOFPOSMODE;
+ store.fst_offset = 0;
+ store.fst_length = newsize;
+
+ ret = ::fcntl(fd, F_PREALLOCATE, &store);
+ if (ret == -1) {
+ ret = -errno;
+ derr << "FileJournal::_open_file : unable to preallocation journal to "
+ << newsize << " bytes: " << cpp_strerror(ret) << dendl;
+ return ret;
+ }
+ max_size = newsize;
+#else
+# error "Journal pre-allocation not supported on platform."
+#endif
}
else {
max_size = oldsize;
@@ -1627,11 +1607,7 @@ void FileJournal::wrap_read_bl(
else
len = olen; // rest
-#ifdef DARWIN
- int64_t actual = ::lseek(fd, pos, SEEK_SET);
-#else
int64_t actual = ::lseek64(fd, pos, SEEK_SET);
-#endif
assert(actual == pos);
bufferptr bp = buffer::create(len);
@@ -1835,22 +1811,14 @@ void FileJournal::corrupt(
if (corrupt_at >= header.max_size)
corrupt_at = corrupt_at + get_top() - header.max_size;
-#ifdef DARWIN
- int64_t actual = ::lseek(fd, corrupt_at, SEEK_SET);
-#else
int64_t actual = ::lseek64(fd, corrupt_at, SEEK_SET);
-#endif
assert(actual == corrupt_at);
char buf[10];
int r = safe_read_exact(fd, buf, 1);
assert(r == 0);
-#ifdef DARWIN
- actual = ::lseek(wfd, corrupt_at, SEEK_SET);
-#else
actual = ::lseek64(wfd, corrupt_at, SEEK_SET);
-#endif
assert(actual == corrupt_at);
buf[0]++;
diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 89a55b3..db55ccc 100644
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -86,6 +86,13 @@ using ceph::crypto::SHA1;
#define REPLAY_GUARD_XATTR "user.cephos.seq"
#define GLOBAL_REPLAY_GUARD_XATTR "user.cephos.gseq"
+// XATTR_SPILL_OUT_NAME as a xattr is used to maintain that indicates whether
+// xattrs spill over into DBObjectMap, if XATTR_SPILL_OUT_NAME exists in file
+// xattrs and the value is "no", it indicates no xattrs in DBObjectMap
+#define XATTR_SPILL_OUT_NAME "user.cephos.spill_out"
+#define XATTR_NO_SPILL_OUT "0"
+#define XATTR_SPILL_OUT "1"
+
//Initial features in new superblock.
static CompatSet get_fs_initial_compat_set() {
CompatSet::FeatureSet ceph_osd_feature_compat;
@@ -104,6 +111,15 @@ static CompatSet get_fs_supported_compat_set() {
}
+int FileStore::peek_journal_fsid(uuid_d *fsid)
+{
+ // make sure we don't try to use aio or direct_io (and get annoying
+ // error messages from failing to do so); performance implications
+ // should be irrelevant for this use
+ FileJournal j(*fsid, 0, 0, journalpath.c_str(), false, false);
+ return j.peek_fsid(*fsid);
+}
+
void FileStore::FSPerfTracker::update_from_perfcounters(
PerfCounters &logger)
{
@@ -216,50 +232,63 @@ int FileStore::lfn_open(coll_t cid,
if (!(*index)) {
r = get_index(cid, index);
}
- Mutex::Locker l(fdcache_lock);
- if (!replaying)
- *outfd = fdcache.lookup(oid);
- if (*outfd) {
- return 0;
- }
- IndexedPath path2;
- if (!path)
- path = &path2;
+
int fd, exist;
- if (r < 0) {
- derr << "error getting collection index for " << cid
- << ": " << cpp_strerror(-r) << dendl;
- goto fail;
- }
- r = (*index)->lookup(oid, path, &exist);
- if (r < 0) {
- derr << "could not find " << oid << " in index: "
- << cpp_strerror(-r) << dendl;
- goto fail;
+ if (!replaying) {
+ Mutex::Locker l(fdcache_lock);
+ *outfd = fdcache.lookup(oid);
+ if (*outfd)
+ return 0;
}
- r = ::open((*path)->path(), flags, 0644);
- if (r < 0) {
- r = -errno;
- dout(10) << "error opening file " << (*path)->path() << " with flags="
- << flags << ": " << cpp_strerror(-r) << dendl;
- goto fail;
- }
- fd = r;
+ {
+ IndexedPath path2;
+ if (!path)
+ path = &path2;
+ if (r < 0) {
+ derr << "error getting collection index for " << cid
+ << ": " << cpp_strerror(-r) << dendl;
+ goto fail;
+ }
+ r = (*index)->lookup(oid, path, &exist);
+ if (r < 0) {
+ derr << "could not find " << oid << " in index: "
+ << cpp_strerror(-r) << dendl;
+ goto fail;
+ }
- if (create && (!exist)) {
- r = (*index)->created(oid, (*path)->path());
+ r = ::open((*path)->path(), flags, 0644);
if (r < 0) {
- TEMP_FAILURE_RETRY(::close(fd));
- derr << "error creating " << oid << " (" << (*path)->path()
- << ") in index: " << cpp_strerror(-r) << dendl;
+ r = -errno;
+ dout(10) << "error opening file " << (*path)->path() << " with flags="
+ << flags << ": " << cpp_strerror(-r) << dendl;
goto fail;
}
+ fd = r;
+
+ if (create && (!exist)) {
+ r = (*index)->created(oid, (*path)->path());
+ if (r < 0) {
+ TEMP_FAILURE_RETRY(::close(fd));
+ derr << "error creating " << oid << " (" << (*path)->path()
+ << ") in index: " << cpp_strerror(-r) << dendl;
+ goto fail;
+ }
+ }
}
- if (!replaying)
- *outfd = fdcache.add(oid, fd);
- else
+
+ if (!replaying) {
+ Mutex::Locker l(fdcache_lock);
+ *outfd = fdcache.lookup(oid);
+ if (*outfd) {
+ TEMP_FAILURE_RETRY(::close(fd));
+ return 0;
+ } else {
+ *outfd = fdcache.add(oid, fd);
+ }
+ } else {
*outfd = FDRef(new FDCache::FD(fd));
+ }
return 0;
fail:
@@ -284,6 +313,11 @@ int FileStore::lfn_link(coll_t c, coll_t newcid, const ghobject_t& o, const ghob
r = get_index(c, &index_old);
if (r < 0)
return r;
+ } else if (c == newcid) {
+ r = get_index(c, &index_old);
+ if (r < 0)
+ return r;
+ index_new = index_old;
} else {
r = get_index(c, &index_old);
if (r < 0)
@@ -377,6 +411,7 @@ int FileStore::lfn_unlink(coll_t cid, const ghobject_t& o,
}
FileStore::FileStore(const std::string &base, const std::string &jdev, const char *name, bool do_update) :
+ JournalingObjectStore(base),
internal_name(name),
basedir(base), journalpath(jdev),
blk_size(0),
@@ -840,12 +875,12 @@ int FileStore::_detect_fs()
m_fs_type = FS_TYPE_BTRFS;
} else if (st.f_type == XFS_SUPER_MAGIC) {
dout(1) << "mount detected xfs" << dendl;
+ m_fs_type = FS_TYPE_XFS;
if (m_filestore_replica_fadvise) {
dout(1) << " disabling 'filestore replica fadvise' due to known issues with fadvise(DONTNEED) on xfs" << dendl;
g_conf->set_val("filestore_replica_fadvise", "false");
g_conf->apply_changes(NULL);
assert(m_filestore_replica_fadvise == false);
- m_fs_type = FS_TYPE_XFS;
}
}
#endif
@@ -995,7 +1030,8 @@ void FileStore::set_allow_sharded_objects()
bool FileStore::get_allow_sharded_objects()
{
- return superblock.compat_features.incompat.contains(CEPH_FS_FEATURE_INCOMPAT_SHARDS);
+ return g_conf->filestore_debug_disable_sharded_check ||
+ superblock.compat_features.incompat.contains(CEPH_FS_FEATURE_INCOMPAT_SHARDS);
}
int FileStore::update_version_stamp()
@@ -1296,14 +1332,23 @@ int FileStore::mount()
{
LevelDBStore *omap_store = new LevelDBStore(g_ceph_context, omap_dir);
- omap_store->options.write_buffer_size = g_conf->osd_leveldb_write_buffer_size;
- omap_store->options.cache_size = g_conf->osd_leveldb_cache_size;
- omap_store->options.block_size = g_conf->osd_leveldb_block_size;
- omap_store->options.bloom_size = g_conf->osd_leveldb_bloom_size;
- omap_store->options.compression_enabled = g_conf->osd_leveldb_compression;
- omap_store->options.paranoid_checks = g_conf->osd_leveldb_paranoid;
- omap_store->options.max_open_files = g_conf->osd_leveldb_max_open_files;
- omap_store->options.log_file = g_conf->osd_leveldb_log;
+ omap_store->init();
+ if (g_conf->osd_leveldb_write_buffer_size)
+ omap_store->options.write_buffer_size = g_conf->osd_leveldb_write_buffer_size;
+ if (g_conf->osd_leveldb_cache_size)
+ omap_store->options.cache_size = g_conf->osd_leveldb_cache_size;
+ if (g_conf->osd_leveldb_block_size)
+ omap_store->options.block_size = g_conf->osd_leveldb_block_size;
+ if (g_conf->osd_leveldb_bloom_size)
+ omap_store->options.bloom_size = g_conf->osd_leveldb_bloom_size;
+ if (g_conf->osd_leveldb_compression)
+ omap_store->options.compression_enabled = g_conf->osd_leveldb_compression;
+ if (g_conf->osd_leveldb_paranoid)
+ omap_store->options.paranoid_checks = g_conf->osd_leveldb_paranoid;
+ if (g_conf->osd_leveldb_max_open_files)
+ omap_store->options.max_open_files = g_conf->osd_leveldb_max_open_files;
+ if (g_conf->osd_leveldb_log.length())
+ omap_store->options.log_file = g_conf->osd_leveldb_log;
stringstream err;
if (omap_store->create_and_open(err)) {
@@ -1393,6 +1438,7 @@ int FileStore::mount()
}
}
+ wbthrottle.start();
sync_thread.create();
ret = journal_replay(initial_op_seq);
@@ -1410,6 +1456,8 @@ int FileStore::mount()
lock.Unlock();
sync_thread.join();
+ wbthrottle.stop();
+
goto close_current_fd;
}
@@ -1459,6 +1507,7 @@ int FileStore::umount()
sync_cond.Signal();
lock.Unlock();
sync_thread.join();
+ wbthrottle.stop();
op_tp.stop();
journal_stop();
@@ -1564,7 +1613,7 @@ void FileStore::queue_op(OpSequencer *osr, Op *o)
op_wq.queue(osr);
}
-void FileStore::op_queue_reserve_throttle(Op *o)
+void FileStore::op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle)
{
// Do not call while holding the journal lock!
uint64_t max_ops = m_filestore_queue_max_ops;
@@ -1586,7 +1635,11 @@ void FileStore::op_queue_reserve_throttle(Op *o)
&& (op_queue_bytes + o->bytes) > max_bytes)) {
dout(2) << "waiting " << op_queue_len + 1 << " > " << max_ops << " ops || "
<< op_queue_bytes + o->bytes << " > " << max_bytes << dendl;
+ if (handle)
+ handle->suspend_tp_timeout();
op_throttle_cond.Wait(op_throttle_lock);
+ if (handle)
+ handle->reset_tp_timeout();
}
op_queue_len++;
@@ -1671,7 +1724,8 @@ struct C_JournaledAhead : public Context {
};
int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
- TrackedOpRef osd_op)
+ TrackedOpRef osd_op,
+ ThreadPool::TPHandle *handle)
{
Context *onreadable;
Context *ondisk;
@@ -1699,7 +1753,7 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
if (journal && journal->is_writeable() && !m_filestore_journal_trailing) {
Op *o = build_op(tls, onreadable, onreadable_sync, osd_op);
- op_queue_reserve_throttle(o);
+ op_queue_reserve_throttle(o, handle);
journal->throttle();
uint64_t op_num = submit_manager.op_submit_start();
o->op = op_num;
@@ -3130,6 +3184,7 @@ void FileStore::sync_entry()
goto again;
}
}
+ stop = false;
lock.Unlock();
}
@@ -3444,15 +3499,29 @@ int FileStore::getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>
Index index;
dout(15) << "getattrs " << cid << "/" << oid << dendl;
FDRef fd;
+ bool spill_out = true;
+ char buf[2];
+
int r = lfn_open(cid, oid, false, &fd);
if (r < 0) {
goto out;
}
+
+ r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+ if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT)))
+ spill_out = false;
+
r = _fgetattrs(**fd, aset, user_only);
if (r < 0) {
goto out;
}
lfn_close(fd);
+
+ if (!spill_out) {
+ dout(10) << __func__ << " no xattr exists in object_map r = " << r << dendl;
+ goto out;
+ }
+
r = get_index(cid, &index);
if (r < 0) {
dout(10) << __func__ << " could not get index r = " << r << dendl;
@@ -3463,6 +3532,7 @@ int FileStore::getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>
dout(10) << __func__ << " could not get omap_attrs r = " << r << dendl;
goto out;
}
+
r = object_map->get_xattrs(oid, omap_attrs, &omap_aset);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not get omap_attrs r = " << r << dendl;
@@ -3507,14 +3577,25 @@ int FileStore::_setattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr
map<string, bufferptr> inline_set;
map<string, bufferptr> inline_to_set;
FDRef fd;
+ int spill_out = -1;
+
int r = lfn_open(cid, oid, false, &fd);
if (r < 0) {
goto out;
}
+
+ char buf[2];
+ r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+ if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT)))
+ spill_out = 0;
+ else
+ spill_out = 1;
+
r = _fgetattrs(**fd, inline_set, false);
assert(!m_filestore_fail_eio || r != -EIO);
dout(15) << "setattrs " << cid << "/" << oid << dendl;
r = 0;
+
for (map<string,bufferptr>::iterator p = aset.begin();
p != aset.end();
++p) {
@@ -3547,14 +3628,18 @@ int FileStore::_setattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr
inline_set.insert(*p);
inline_to_set.insert(*p);
+ }
+ if (spill_out != 1 && omap_set.size()) {
+ chain_fsetxattr(**fd, XATTR_SPILL_OUT_NAME, XATTR_SPILL_OUT,
+ sizeof(XATTR_SPILL_OUT));
}
r = _fsetattrs(**fd, inline_to_set);
if (r < 0)
goto out_close;
- if (!omap_remove.empty()) {
+ if (spill_out && !omap_remove.empty()) {
r = object_map->remove_xattrs(oid, omap_remove, &spos);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not remove_xattrs r = " << r << dendl;
@@ -3564,7 +3649,7 @@ int FileStore::_setattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr
r = 0; // don't confuse the debug output
}
}
-
+
if (!omap_set.empty()) {
r = object_map->set_xattrs(oid, omap_set, &spos);
if (r < 0) {
@@ -3586,14 +3671,24 @@ int FileStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name,
{
dout(15) << "rmattr " << cid << "/" << oid << " '" << name << "'" << dendl;
FDRef fd;
+ bool spill_out = true;
+ bufferptr bp;
+
int r = lfn_open(cid, oid, false, &fd);
if (r < 0) {
goto out;
}
+
+ char buf[2];
+ r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+ if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT))) {
+ spill_out = false;
+ }
+
char n[CHAIN_XATTR_MAX_NAME_LEN];
get_attrname(name, n, CHAIN_XATTR_MAX_NAME_LEN);
r = chain_fremovexattr(**fd, n);
- if (r == -ENODATA) {
+ if (r == -ENODATA && spill_out) {
Index index;
r = get_index(cid, &index);
if (r < 0) {
@@ -3625,10 +3720,19 @@ int FileStore::_rmattrs(coll_t cid, const ghobject_t& oid,
FDRef fd;
set<string> omap_attrs;
Index index;
+ bool spill_out = true;
+
int r = lfn_open(cid, oid, false, &fd);
if (r < 0) {
goto out;
}
+
+ char buf[2];
+ r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+ if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT))) {
+ spill_out = false;
+ }
+
r = _fgetattrs(**fd, aset, false);
if (r >= 0) {
for (map<string,bufferptr>::iterator p = aset.begin(); p != aset.end(); ++p) {
@@ -3639,26 +3743,36 @@ int FileStore::_rmattrs(coll_t cid, const ghobject_t& oid,
break;
}
}
- lfn_close(fd);
+
+ if (!spill_out) {
+ dout(10) << __func__ << " no xattr exists in object_map r = " << r << dendl;
+ goto out_close;
+ }
r = get_index(cid, &index);
if (r < 0) {
dout(10) << __func__ << " could not get index r = " << r << dendl;
- return r;
+ goto out_close;
}
r = object_map->get_all_xattrs(oid, &omap_attrs);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not get omap_attrs r = " << r << dendl;
assert(!m_filestore_fail_eio || r != -EIO);
- return r;
+ goto out_close;
}
r = object_map->remove_xattrs(oid, omap_attrs, &spos);
if (r < 0 && r != -ENOENT) {
dout(10) << __func__ << " could not remove omap_attrs r = " << r << dendl;
- return r;
+ goto out_close;
}
if (r == -ENOENT)
r = 0;
+
+ chain_fsetxattr(**fd, XATTR_SPILL_OUT_NAME, XATTR_NO_SPILL_OUT,
+ sizeof(XATTR_NO_SPILL_OUT));
+
+ out_close:
+ lfn_close(fd);
out:
dout(10) << "rmattrs " << cid << "/" << oid << " = " << r << dendl;
return r;
@@ -4322,13 +4436,26 @@ int FileStore::_collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
_inject_failure();
+ lfn_close(fd);
+ fd = FDRef();
+
+ if (r == 0)
+ r = lfn_unlink(oldcid, oldoid, spos, true);
+
+ if (r == 0)
+ r = lfn_open(c, o, 0, &fd);
+
// close guard on object so we don't do this again
- if (r == 0) {
+ if (r == 0)
_close_replay_guard(**fd, spos);
- }
+
lfn_close(fd);
}
+ dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/" << oldoid
+ << " = " << r << dendl;
+ return r;
+
out_rm_src:
// remove source
if (_check_replay_guard(oldcid, oldoid, spos) > 0) {
@@ -4360,7 +4487,7 @@ int FileStore::_omap_clear(coll_t cid, const ghobject_t &hoid,
int r = lfn_find(cid, hoid, &path);
if (r < 0)
return r;
- r = object_map->clear(hoid, &spos);
+ r = object_map->clear_keys_header(hoid, &spos);
if (r < 0 && r != -ENOENT)
return r;
return 0;
diff --git a/src/os/FileStore.h b/src/os/FileStore.h
index c489fdd..d30eeba 100644
--- a/src/os/FileStore.h
+++ b/src/os/FileStore.h
@@ -24,8 +24,7 @@
#include <fstream>
using namespace std;
-#include <ext/hash_map>
-using namespace __gnu_cxx;
+#include "include/unordered_map.h"
#include "include/assert.h"
@@ -64,6 +63,34 @@ static const __SWORD_TYPE XFS_SUPER_MAGIC(0x58465342);
static const __SWORD_TYPE ZFS_SUPER_MAGIC(0x2fc12fc1);
#endif
+enum {
+ l_os_first = 84000,
+ l_os_jq_max_ops,
+ l_os_jq_ops,
+ l_os_j_ops,
+ l_os_jq_max_bytes,
+ l_os_jq_bytes,
+ l_os_j_bytes,
+ l_os_j_lat,
+ l_os_j_wr,
+ l_os_j_wr_bytes,
+ l_os_oq_max_ops,
+ l_os_oq_ops,
+ l_os_ops,
+ l_os_oq_max_bytes,
+ l_os_oq_bytes,
+ l_os_bytes,
+ l_os_apply_lat,
+ l_os_committing,
+ l_os_commit,
+ l_os_commit_len,
+ l_os_commit_lat,
+ l_os_j_full,
+ l_os_queue_lat,
+ l_os_last,
+};
+
+
enum fs_types {
FS_TYPE_NONE = 0,
FS_TYPE_XFS,
@@ -97,14 +124,20 @@ inline ostream& operator<<(ostream& out, const FSSuperblock& sb)
class FileStore : public JournalingObjectStore,
public md_config_obs_t
{
+ static const uint32_t target_version = 3;
public:
+ uint32_t get_target_version() {
+ return target_version;
+ }
+
+ int peek_journal_fsid(uuid_d *fsid);
struct FSPerfTracker {
PerfCounters::avg_tracker<uint64_t> os_commit_latency;
PerfCounters::avg_tracker<uint64_t> os_apply_latency;
- filestore_perf_stat_t get_cur_stats() const {
- filestore_perf_stat_t ret;
+ objectstore_perf_stat_t get_cur_stats() const {
+ objectstore_perf_stat_t ret;
ret.filestore_commit_latency = os_commit_latency.avg();
ret.filestore_apply_latency = os_apply_latency.avg();
return ret;
@@ -112,12 +145,11 @@ public:
void update_from_perfcounters(PerfCounters &logger);
} perf_tracker;
- filestore_perf_stat_t get_cur_stats() {
+ objectstore_perf_stat_t get_cur_stats() {
perf_tracker.update_from_perfcounters(*logger);
return perf_tracker.get_cur_stats();
}
- static const uint32_t target_version = 3;
private:
string internal_name; ///< internal name, used to name the perfcounter instance
string basedir, journalpath;
@@ -305,7 +337,7 @@ private:
Context *onreadable, Context *onreadable_sync,
TrackedOpRef osd_op);
void queue_op(OpSequencer *osr, Op *o);
- void op_queue_reserve_throttle(Op *o);
+ void op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle = NULL);
void op_queue_release_throttle(Op *o);
void _journaled_ahead(OpSequencer *osr, Op *o, Context *ondisk);
friend struct C_JournaledAhead;
@@ -378,7 +410,8 @@ public:
ThreadPool::TPHandle *handle);
int queue_transactions(Sequencer *osr, list<Transaction*>& tls,
- TrackedOpRef op = TrackedOpRef());
+ TrackedOpRef op = TrackedOpRef(),
+ ThreadPool::TPHandle *handle = NULL);
/**
* set replay guard xattr on given file
diff --git a/src/os/FlatIndex.cc b/src/os/FlatIndex.cc
index d4644ab..2f1a1d1 100644
--- a/src/os/FlatIndex.cc
+++ b/src/os/FlatIndex.cc
@@ -49,7 +49,7 @@ using ceph::crypto::SHA1;
#define FILENAME_PREFIX_LEN (FILENAME_SHORT_LEN - FILENAME_HASH_LEN - (sizeof(FILENAME_COOKIE) - 1) - FILENAME_EXTRA)
-void FlatIndex::set_ref(std::tr1::shared_ptr<CollectionIndex> ref) {
+void FlatIndex::set_ref(ceph::shared_ptr<CollectionIndex> ref) {
self_ref = ref;
}
diff --git a/src/os/FlatIndex.h b/src/os/FlatIndex.h
index 657c273..e55bd14 100644
--- a/src/os/FlatIndex.h
+++ b/src/os/FlatIndex.h
@@ -19,7 +19,7 @@
#include <map>
#include <set>
#include <vector>
-#include <tr1/memory>
+#include "include/memory.h"
#include "CollectionIndex.h"
@@ -29,7 +29,7 @@
* This class should only be used for converting old filestores.
*/
class FlatIndex : public CollectionIndex {
- std::tr1::weak_ptr<CollectionIndex> self_ref;
+ ceph::weak_ptr<CollectionIndex> self_ref;
string base_path;
coll_t collection;
public:
@@ -42,7 +42,7 @@ public:
coll_t coll() const { return collection; }
/// @see CollectionIndex
- void set_ref(std::tr1::shared_ptr<CollectionIndex> ref);
+ void set_ref(ceph::shared_ptr<CollectionIndex> ref);
/// @see CollectionIndex
int cleanup();
diff --git a/src/os/GenericObjectMap.cc b/src/os/GenericObjectMap.cc
new file mode 100644
index 0000000..3be9b76
--- /dev/null
+++ b/src/os/GenericObjectMap.cc
@@ -0,0 +1,1112 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack <haomai at unitedstack.com>
+ *
+ * Author: Haomai Wang <haomaiwang at gmail.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+#include "include/buffer.h"
+
+#include <iostream>
+#include <set>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <errno.h>
+
+#include "GenericObjectMap.h"
+#include "common/debug.h"
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_keyvaluestore
+const string GenericObjectMap::GLOBAL_STATE_KEY = "HEADER";
+
+const string GenericObjectMap::USER_PREFIX = "_SEQ_";
+const string GenericObjectMap::INTERN_PREFIX = "_INTERN_";
+const string GenericObjectMap::COMPLETE_PREFIX = "_COMPLETE_";
+const string GenericObjectMap::GHOBJECT_TO_SEQ_PREFIX = "_GHOBJTOSEQ_";
+const string GenericObjectMap::PARENT_KEY = "_PARENT_HEADER_";
+
+// In order to make right ordering for leveldb matching with hobject_t,
+// so use "!" to separated
+const string GenericObjectMap::GHOBJECT_KEY_SEP_S = "!";
+const char GenericObjectMap::GHOBJECT_KEY_SEP_C = '!';
+
+// ============== GenericObjectMap Key Function =================
+
+static void append_escaped(const string &in, string *out)
+{
+ for (string::const_iterator i = in.begin(); i != in.end(); ++i) {
+ if (*i == '%') {
+ out->push_back('%');
+ out->push_back('p');
+ } else if (*i == '.') {
+ out->push_back('%');
+ out->push_back('e');
+ } else if (*i == GenericObjectMap::GHOBJECT_KEY_SEP_C) {
+ out->push_back('%');
+ out->push_back('u');
+ } else if (*i == '!') {
+ out->push_back('%');
+ out->push_back('s');
+ } else {
+ out->push_back(*i);
+ }
+ }
+}
+
+static bool append_unescaped(string::const_iterator begin,
+ string::const_iterator end,
+ string *out)
+{
+ for (string::const_iterator i = begin; i != end; ++i) {
+ if (*i == '%') {
+ ++i;
+ if (*i == 'p')
+ out->push_back('%');
+ else if (*i == 'e')
+ out->push_back('.');
+ else if (*i == 'u')
+ out->push_back(GenericObjectMap::GHOBJECT_KEY_SEP_C);
+ else if (*i == 's')
+ out->push_back('!');
+ else
+ return false;
+ } else {
+ out->push_back(*i);
+ }
+ }
+ return true;
+}
+
+string GenericObjectMap::header_key(const coll_t &cid)
+{
+ string full_name;
+
+ append_escaped(cid.to_str(), &full_name);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+ return full_name;
+}
+
+string GenericObjectMap::header_key(const coll_t &cid, const ghobject_t &oid)
+{
+ string full_name;
+
+ append_escaped(cid.to_str(), &full_name);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ char buf[PATH_MAX];
+ char *t = buf;
+ char *end = t + sizeof(buf);
+
+ // make field ordering match with hobject_t compare operations
+ snprintf(t, end - t, "%.*X", (int)(sizeof(oid.hobj.hash)*2),
+ (uint32_t)oid.get_filestore_key_u32());
+ full_name += string(buf);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ append_escaped(oid.hobj.nspace, &full_name);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ t = buf;
+ if (oid.hobj.pool == -1)
+ t += snprintf(t, end - t, "none");
+ else
+ t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.pool);
+ full_name += string(buf);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ append_escaped(oid.hobj.get_key(), &full_name);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ append_escaped(oid.hobj.oid.name, &full_name);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ t = buf;
+ if (oid.hobj.snap == CEPH_NOSNAP)
+ t += snprintf(t, end - t, "head");
+ else if (oid.hobj.snap == CEPH_SNAPDIR)
+ t += snprintf(t, end - t, "snapdir");
+ else
+ // Keep length align
+ t += snprintf(t, end - t, "%016llx", (long long unsigned)oid.hobj.snap);
+ full_name += string(buf);
+
+ if (oid.generation != ghobject_t::NO_GEN) {
+ assert(oid.shard_id != ghobject_t::NO_SHARD);
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ t = buf;
+ end = t + sizeof(buf);
+ t += snprintf(t, end - t, "%llx", (long long unsigned)oid.generation);
+ full_name += string(buf);
+
+ full_name.append(GHOBJECT_KEY_SEP_S);
+
+ t = buf;
+ end = t + sizeof(buf);
+ t += snprintf(t, end - t, "%x", (int)oid.shard_id);
+ full_name += string(buf);
+ }
+
+ return full_name;
+}
+
+bool GenericObjectMap::parse_header_key(const string &long_name,
+ coll_t *out_coll, ghobject_t *out)
+{
+ string coll;
+ string name;
+ string key;
+ string ns;
+ uint32_t hash;
+ snapid_t snap;
+ uint64_t pool;
+ gen_t generation = ghobject_t::NO_GEN;
+ shard_t shard_id = ghobject_t::NO_SHARD;
+
+ string::const_iterator current = long_name.begin();
+ string::const_iterator end;
+
+ for (end = current; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (!append_unescaped(current, end, &coll))
+ return false;
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end == long_name.end())
+ return false;
+ string hash_str(current, end);
+ sscanf(hash_str.c_str(), "%X", &hash);
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end == long_name.end())
+ return false;
+ if (!append_unescaped(current, end, &ns))
+ return false;
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end == long_name.end())
+ return false;
+ string pstring(current, end);
+ if (pstring == "none")
+ pool = (uint64_t)-1;
+ else
+ pool = strtoull(pstring.c_str(), NULL, 16);
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end == long_name.end())
+ return false;
+ if (!append_unescaped(current, end, &key))
+ return false;
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end == long_name.end())
+ return false;
+ if (!append_unescaped(current, end, &name))
+ return false;
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ string snap_str(current, end);
+ if (snap_str == "head")
+ snap = CEPH_NOSNAP;
+ else if (snap_str == "snapdir")
+ snap = CEPH_SNAPDIR;
+ else
+ snap = strtoull(snap_str.c_str(), NULL, 16);
+
+ // Optional generation/shard_id
+ string genstring, shardstring;
+ if (end != long_name.end()) {
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end == long_name.end())
+ return false;
+ genstring = string(current, end);
+
+ generation = (gen_t)strtoull(genstring.c_str(), NULL, 16);
+
+ current = ++end;
+ for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+ if (end != long_name.end())
+ return false;
+ shardstring = string(current, end);
+
+ shard_id = (shard_t)strtoul(shardstring.c_str(), NULL, 16);
+ }
+
+ if (out) {
+ (*out) = ghobject_t(hobject_t(name, key, snap, hash, (int64_t)pool, ns),
+ generation, shard_id);
+ // restore reversed hash. see calculate_key
+ out->hobj.hash = out->get_filestore_key();
+ }
+
+ if (out_coll)
+ *out_coll = coll_t(coll);
+
+ return true;
+}
+
+
+// ============== GenericObjectMap Prefix =================
+
+string GenericObjectMap::user_prefix(Header header, const string &prefix)
+{
+ return USER_PREFIX + seq_key(header->seq) + prefix;
+}
+
+string GenericObjectMap::complete_prefix(Header header)
+{
+ return INTERN_PREFIX + seq_key(header->seq) + COMPLETE_PREFIX;
+}
+
+string GenericObjectMap::parent_seq_prefix(uint64_t seq)
+{
+ return INTERN_PREFIX + seq_key(seq) + PARENT_KEY;
+}
+
+
+// ============== GenericObjectMapIteratorImpl =================
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::init()
+{
+ invalid = false;
+ if (ready) {
+ return 0;
+ }
+
+ assert(!parent_iter);
+ if (header->parent) {
+ Header parent = map->lookup_parent(header);
+ if (!parent) {
+ assert(0);
+ return -EINVAL;
+ }
+ parent_iter.reset(new GenericObjectMapIteratorImpl(map, parent, prefix));
+ }
+
+ key_iter = map->db->get_iterator(map->user_prefix(header, prefix));
+ assert(key_iter);
+ complete_iter = map->db->get_iterator(map->complete_prefix(header));
+ assert(complete_iter);
+ cur_iter = key_iter;
+ assert(cur_iter);
+ ready = true;
+ return 0;
+}
+
+ObjectMap::ObjectMapIterator GenericObjectMap::get_iterator(
+ const coll_t &cid, const ghobject_t &oid, const string &prefix)
+{
+ Header header = lookup_header(cid, oid);
+ if (!header)
+ return ObjectMap::ObjectMapIterator(new EmptyIteratorImpl());
+ return _get_iterator(header, prefix);
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::seek_to_first()
+{
+ init();
+ r = 0;
+ if (parent_iter) {
+ r = parent_iter->seek_to_first();
+ if (r < 0)
+ return r;
+ }
+ r = key_iter->seek_to_first();
+ if (r < 0)
+ return r;
+ return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::seek_to_last()
+{
+ init();
+ r = 0;
+ if (parent_iter) {
+ r = parent_iter->seek_to_last();
+ if (r < 0)
+ return r;
+ if (parent_iter->valid())
+ r = parent_iter->next();
+ if (r < 0)
+ return r;
+ }
+ r = key_iter->seek_to_last();
+ if (r < 0)
+ return r;
+ if (key_iter->valid())
+ r = key_iter->next();
+ if (r < 0)
+ return r;
+ return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::lower_bound(const string &to)
+{
+ init();
+ r = 0;
+ if (parent_iter) {
+ r = parent_iter->lower_bound(to);
+ if (r < 0)
+ return r;
+ }
+ r = key_iter->lower_bound(to);
+ if (r < 0)
+ return r;
+ return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::upper_bound(const string &after)
+{
+ init();
+ r = 0;
+ if (parent_iter) {
+ r = parent_iter->upper_bound(after);
+ if (r < 0)
+ return r;
+ }
+ r = key_iter->upper_bound(after);
+ if (r < 0)
+ return r;
+ return adjust();
+}
+
+bool GenericObjectMap::GenericObjectMapIteratorImpl::valid()
+{
+ bool valid = !invalid && ready;
+ assert(!valid || cur_iter->valid());
+ return valid;
+}
+
+bool GenericObjectMap::GenericObjectMapIteratorImpl::valid_parent()
+{
+ if (parent_iter && parent_iter->valid() &&
+ (!key_iter->valid() || key_iter->key() > parent_iter->key()))
+ return true;
+ return false;
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::next()
+{
+ assert(cur_iter->valid());
+ assert(valid());
+ cur_iter->next();
+ return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::next_parent()
+{
+ if (!parent_iter || !parent_iter->valid()) {
+ invalid = true;
+ return 0;
+ }
+ r = next();
+ if (r < 0)
+ return r;
+ if (!valid() || on_parent() || !parent_iter->valid())
+ return 0;
+
+ return lower_bound(parent_iter->key());
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::in_complete_region(
+ const string &to_test, string *begin, string *end)
+{
+ complete_iter->upper_bound(to_test);
+ if (complete_iter->valid())
+ complete_iter->prev();
+ else
+ complete_iter->seek_to_last();
+
+ if (!complete_iter->valid())
+ return false;
+
+ string _end;
+ if (begin)
+ *begin = complete_iter->key();
+ _end = string(complete_iter->value().c_str());
+ if (end)
+ *end = _end;
+ return (to_test >= complete_iter->key()) && (!_end.size() || _end > to_test);
+}
+
+/**
+ * Moves parent_iter to the next position both out of the complete_region and
+ * not equal to key_iter. Then, we set cur_iter to parent_iter if valid and
+ * less than key_iter and key_iter otherwise.
+ */
+int GenericObjectMap::GenericObjectMapIteratorImpl::adjust()
+{
+ string begin, end;
+ while (parent_iter && parent_iter->valid()) {
+ if (in_complete_region(parent_iter->key(), &begin, &end)) {
+ if (end.size() == 0) {
+ parent_iter->seek_to_last();
+ if (parent_iter->valid())
+ parent_iter->next();
+ } else {
+ parent_iter->lower_bound(end);
+ }
+ } else if (key_iter->valid() && key_iter->key() == parent_iter->key()) {
+ parent_iter->next();
+ } else {
+ break;
+ }
+ }
+ if (valid_parent()) {
+ cur_iter = parent_iter;
+ } else if (key_iter->valid()) {
+ cur_iter = key_iter;
+ } else {
+ invalid = true;
+ }
+ assert(invalid || cur_iter->valid());
+ return 0;
+}
+
+string GenericObjectMap::GenericObjectMapIteratorImpl::key()
+{
+ return cur_iter->key();
+}
+
+bufferlist GenericObjectMap::GenericObjectMapIteratorImpl::value()
+{
+ return cur_iter->value();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::status()
+{
+ return r;
+}
+
+
+// ============== GenericObjectMap Public API =================
+
+void GenericObjectMap::set_keys(const Header header,
+ const string &prefix,
+ const map<string, bufferlist> &set,
+ KeyValueDB::Transaction t)
+{
+ t->set(user_prefix(header, prefix), set);
+}
+
+int GenericObjectMap::clear(const Header header,
+ KeyValueDB::Transaction t)
+{
+ remove_header(header->cid, header->oid, header, t);
+ assert(header->num_children > 0);
+ header->num_children--;
+ int r = _clear(header, t);
+ if (r < 0)
+ return r;
+ return 0;
+}
+
+int GenericObjectMap::rm_keys(const Header header,
+ const string &prefix,
+ const set<string> &to_clear,
+ KeyValueDB::Transaction t)
+{
+ t->rmkeys(user_prefix(header, prefix), to_clear);
+ if (!header->parent) {
+ return 0;
+ }
+
+ // Copy up keys from parent around to_clear
+ int keep_parent;
+ {
+ GenericObjectMapIterator iter = _get_iterator(header, prefix);
+ iter->seek_to_first();
+ map<string, string> new_complete;
+ map<string, bufferlist> to_write;
+ for(set<string>::const_iterator i = to_clear.begin();
+ i != to_clear.end(); ) {
+ unsigned copied = 0;
+ iter->lower_bound(*i);
+ ++i;
+ if (!iter->valid())
+ break;
+ string begin = iter->key();
+ if (!iter->on_parent())
+ iter->next_parent();
+ if (new_complete.size() && new_complete.rbegin()->second == begin) {
+ begin = new_complete.rbegin()->first;
+ }
+ while (iter->valid() && copied < 20) {
+ if (!to_clear.count(iter->key()))
+ to_write[iter->key()].append(iter->value());
+ if (i != to_clear.end() && *i <= iter->key()) {
+ ++i;
+ copied = 0;
+ }
+
+ iter->next_parent();
+ copied++;
+ }
+ if (iter->valid()) {
+ new_complete[begin] = iter->key();
+ } else {
+ new_complete[begin] = "";
+ break;
+ }
+ }
+ t->set(user_prefix(header, prefix), to_write);
+ merge_new_complete(header, new_complete, iter, t);
+ keep_parent = need_parent(iter);
+ if (keep_parent < 0)
+ return keep_parent;
+ }
+
+ if (!keep_parent) {
+ Header parent = lookup_parent(header);
+ if (!parent)
+ return -EINVAL;
+ parent->num_children--;
+ _clear(parent, t);
+ header->parent = 0;
+ set_header(header->cid, header->oid, *header, t);
+ t->rmkeys_by_prefix(complete_prefix(header));
+ }
+
+ return 0;
+}
+
+int GenericObjectMap::get(const coll_t &cid, const ghobject_t &oid,
+ const string &prefix,
+ map<string, bufferlist> *out)
+{
+ Header header = lookup_header(cid, oid);
+ if (!header)
+ return -ENOENT;
+
+ ObjectMap::ObjectMapIterator iter = _get_iterator(header, prefix);
+ for (iter->seek_to_first(); iter->valid(); iter->next()) {
+ if (iter->status())
+ return iter->status();
+ out->insert(make_pair(iter->key(), iter->value()));
+ }
+
+ return 0;
+}
+
+int GenericObjectMap::get_keys(const coll_t &cid, const ghobject_t &oid,
+ const string &prefix,
+ set<string> *keys)
+{
+ Header header = lookup_header(cid, oid);
+ if (!header)
+ return -ENOENT;
+
+ ObjectMap::ObjectMapIterator iter = _get_iterator(header, prefix);
+ for (; iter->valid(); iter->next()) {
+ if (iter->status())
+ return iter->status();
+ keys->insert(iter->key());
+ }
+ return 0;
+}
+int GenericObjectMap::get_values(const coll_t &cid, const ghobject_t &oid,
+ const string &prefix,
+ const set<string> &keys,
+ map<string, bufferlist> *out)
+{
+ Header header = lookup_header(cid, oid);
+ if (!header)
+ return -ENOENT;
+ return scan(header, prefix, keys, 0, out);
+}
+
+int GenericObjectMap::check_keys(const coll_t &cid, const ghobject_t &oid,
+ const string &prefix,
+ const set<string> &keys,
+ set<string> *out)
+{
+ Header header = lookup_header(cid, oid);
+ if (!header)
+ return -ENOENT;
+ return scan(header, prefix, keys, out, 0);
+}
+
+void GenericObjectMap::clone(const Header parent, const coll_t &cid,
+ const ghobject_t &target,
+ KeyValueDB::Transaction t,
+ Header *old_header, Header *new_header)
+{
+ {
+ Header destination = lookup_header(cid, target);
+ if (destination) {
+ remove_header(cid, target, destination, t);
+ destination->num_children--;
+ _clear(destination, t);
+ }
+ }
+
+ Header source = generate_new_header(parent->cid, parent->oid, parent, t);
+ Header destination = generate_new_header(cid, target, parent, t);
+
+ destination->data = parent->data;
+ source->data = parent->data;
+
+ parent->num_children = 2;
+ set_parent_header(parent, t);
+ set_header(parent->cid, parent->oid, *source, t);
+ set_header(cid, target, *destination, t);
+
+ if (new_header)
+ *old_header = source;
+ if (new_header)
+ *new_header = destination;
+}
+
+void GenericObjectMap::rename(const Header old_header, const coll_t &cid,
+ const ghobject_t &target,
+ KeyValueDB::Transaction t)
+{
+ if (old_header->oid == target && old_header->cid == cid)
+ return ;
+
+ remove_header(old_header->cid, old_header->oid, old_header, t);
+ old_header->cid = cid;
+ old_header->oid = target;
+ set_header(cid, target, *old_header, t);
+
+ // "in_use" still hold the "seq"
+}
+
+int GenericObjectMap::init(bool do_upgrade)
+{
+ map<string, bufferlist> result;
+ set<string> to_get;
+ to_get.insert(GLOBAL_STATE_KEY);
+ int r = db->get(INTERN_PREFIX, to_get, &result);
+ if (r < 0)
+ return r;
+ if (!result.empty()) {
+ bufferlist::iterator bliter = result.begin()->second.begin();
+ state.decode(bliter);
+ if (state.v < 1) { // Needs upgrade
+ if (!do_upgrade) {
+ dout(1) << "GenericObjbectMap requires an upgrade,"
+ << " set filestore_update_to"
+ << dendl;
+ return -ENOTSUP;
+ } else {
+ r = upgrade();
+ if (r < 0)
+ return r;
+ }
+ }
+ } else {
+ // New store
+ state.v = 1;
+ state.seq = 1;
+ }
+ dout(20) << "(init)genericobjectmap: seq is " << state.seq << dendl;
+ return 0;
+}
+
+int GenericObjectMap::sync(const Header header, KeyValueDB::Transaction t)
+{
+ write_state(t);
+ if (header) {
+ set_header(header->cid, header->oid, *header, t);
+ }
+ return 0;
+}
+
+bool GenericObjectMap::check(std::ostream &out)
+{
+ bool retval = true;
+ map<uint64_t, uint64_t> parent_to_num_children;
+ map<uint64_t, uint64_t> parent_to_actual_num_children;
+ KeyValueDB::Iterator iter = db->get_iterator(GHOBJECT_TO_SEQ_PREFIX);
+
+ for (iter->seek_to_first(); iter->valid(); iter->next()) {
+ _Header header;
+ assert(header.num_children == 1);
+ header.num_children = 0; // Hack for leaf node
+ bufferlist bl = iter->value();
+ while (true) {
+ bufferlist::iterator bliter = bl.begin();
+ header.decode(bliter);
+ if (header.seq != 0)
+ parent_to_actual_num_children[header.seq] = header.num_children;
+ if (header.parent == 0)
+ break;
+
+ if (!parent_to_num_children.count(header.parent))
+ parent_to_num_children[header.parent] = 0;
+ parent_to_num_children[header.parent]++;
+ if (parent_to_actual_num_children.count(header.parent))
+ break;
+
+ set<string> to_get;
+ map<string, bufferlist> got;
+ to_get.insert(PARENT_KEY);
+ db->get(parent_seq_prefix(header.parent), to_get, &got);
+ if (got.empty()) {
+ out << "Missing: seq " << header.parent << std::endl;
+ retval = false;
+ break;
+ } else {
+ bl = got.begin()->second;
+ }
+ }
+ }
+
+ for (map<uint64_t, uint64_t>::iterator i = parent_to_num_children.begin();
+ i != parent_to_num_children.end();
+ parent_to_num_children.erase(i++)) {
+ if (!parent_to_actual_num_children.count(i->first))
+ continue;
+ if (parent_to_actual_num_children[i->first] != i->second) {
+ out << "Invalid: seq " << i->first << " recorded children: "
+ << parent_to_actual_num_children[i->first] << " found: "
+ << i->second << std::endl;
+ retval = false;
+ }
+ parent_to_actual_num_children.erase(i->first);
+ }
+ return retval;
+}
+
+
+// ============== GenericObjectMap Intern Implementation =================
+
+int GenericObjectMap::scan(Header header,
+ const string &prefix,
+ const set<string> &in_keys,
+ set<string> *out_keys,
+ map<string, bufferlist> *out_values)
+{
+ ObjectMap::ObjectMapIterator db_iter = _get_iterator(header, prefix);
+ for (set<string>::const_iterator key_iter = in_keys.begin();
+ key_iter != in_keys.end();
+ ++key_iter) {
+ db_iter->lower_bound(*key_iter);
+ if (db_iter->status())
+ return db_iter->status();
+
+ if (db_iter->valid() && db_iter->key() == *key_iter) {
+ if (out_keys)
+ out_keys->insert(*key_iter);
+ if (out_values)
+ out_values->insert(make_pair(db_iter->key(), db_iter->value()));
+ }
+ }
+ return 0;
+}
+
+int GenericObjectMap::_clear(Header header, KeyValueDB::Transaction t)
+{
+ while (1) {
+ if (header->num_children) {
+ set_parent_header(header, t);
+ break;
+ }
+
+ clear_header(header, t);
+ if (!header->parent)
+ break;
+
+ Header parent = lookup_parent(header);
+ if (!parent) {
+ return -EINVAL;
+ }
+ assert(parent->num_children > 0);
+ parent->num_children--;
+ header.swap(parent);
+ }
+ return 0;
+}
+
+int GenericObjectMap::merge_new_complete(
+ Header header, const map<string, string> &new_complete,
+ GenericObjectMapIterator iter, KeyValueDB::Transaction t)
+{
+ KeyValueDB::Iterator complete_iter = db->get_iterator(
+ complete_prefix(header));
+ map<string, string>::const_iterator i = new_complete.begin();
+ set<string> to_remove;
+ map<string, bufferlist> to_add;
+
+ string begin, end;
+ while (i != new_complete.end()) {
+ string new_begin = i->first;
+ string new_end = i->second;
+ int r = iter->in_complete_region(new_begin, &begin, &end);
+ if (r < 0)
+ return r;
+ if (r) {
+ to_remove.insert(begin);
+ new_begin = begin;
+ }
+ ++i;
+ while (i != new_complete.end()) {
+ if (!new_end.size() || i->first <= new_end) {
+ if (!new_end.size() && i->second > new_end) {
+ new_end = i->second;
+ }
+ ++i;
+ continue;
+ }
+
+ r = iter->in_complete_region(new_end, &begin, &end);
+ if (r < 0)
+ return r;
+ if (r) {
+ to_remove.insert(begin);
+ new_end = end;
+ continue;
+ }
+ break;
+ }
+ bufferlist bl;
+ bl.append(bufferptr(new_end.c_str(), new_end.size() + 1));
+ to_add.insert(make_pair(new_begin, bl));
+ }
+ t->rmkeys(complete_prefix(header), to_remove);
+ t->set(complete_prefix(header), to_add);
+ return 0;
+}
+
+int GenericObjectMap::need_parent(GenericObjectMapIterator iter)
+{
+ int r = iter->seek_to_first();
+ if (r < 0)
+ return r;
+
+ if (!iter->valid())
+ return 0;
+
+ string begin, end;
+ if (iter->in_complete_region(iter->key(), &begin, &end) && end == "") {
+ return 0;
+ }
+ return 1;
+}
+
+int GenericObjectMap::write_state(KeyValueDB::Transaction t)
+{
+ dout(20) << __func__ << " seq is " << state.seq << dendl;
+ bufferlist bl;
+ state.encode(bl);
+ map<string, bufferlist> to_write;
+ to_write[GLOBAL_STATE_KEY] = bl;
+ t->set(INTERN_PREFIX, to_write);
+ return 0;
+}
+
+GenericObjectMap::Header GenericObjectMap::_lookup_header(
+ const coll_t &cid, const ghobject_t &oid)
+{
+ // FIXME
+ while (map_header_in_use.count(oid))
+ header_cond.Wait(header_lock);
+
+ map<string, bufferlist> out;
+ set<string> to_get;
+ to_get.insert(header_key(cid, oid));
+ int r = db->get(GHOBJECT_TO_SEQ_PREFIX, to_get, &out);
+ if (r < 0)
+ return Header();
+ if (out.empty())
+ return Header();
+
+ Header ret(new _Header(), RemoveMapHeaderOnDelete(this, cid, oid));
+ bufferlist::iterator iter = out.begin()->second.begin();
+ ret->decode(iter);
+ return ret;
+}
+
+GenericObjectMap::Header GenericObjectMap::_generate_new_header(
+ const coll_t &cid, const ghobject_t &oid, Header parent,
+ KeyValueDB::Transaction t)
+{
+ Header header = Header(new _Header(), RemoveOnDelete(this));
+ header->seq = state.seq++;
+ if (parent) {
+ header->parent = parent->seq;
+ }
+ header->num_children = 1;
+ header->oid = oid;
+ header->cid = cid;
+ assert(!in_use.count(header->seq));
+ in_use.insert(header->seq);
+
+ write_state(t);
+ return header;
+}
+
+GenericObjectMap::Header GenericObjectMap::lookup_parent(Header input)
+{
+ Mutex::Locker l(header_lock);
+ while (in_use.count(input->parent))
+ header_cond.Wait(header_lock);
+ map<string, bufferlist> out;
+ set<string> keys;
+ keys.insert(PARENT_KEY);
+
+ dout(20) << "lookup_parent: parent " << input->parent
+ << " for seq " << input->seq << dendl;
+ int r = db->get(parent_seq_prefix(input->parent), keys, &out);
+ if (r < 0) {
+ assert(0);
+ return Header();
+ }
+ if (out.empty()) {
+ assert(0);
+ return Header();
+ }
+
+ Header header = Header(new _Header(), RemoveOnDelete(this));
+ header->seq = input->parent;
+ bufferlist::iterator iter = out.begin()->second.begin();
+ header->decode(iter);
+ dout(20) << "lookup_parent: parent seq is " << header->seq << " with parent "
+ << header->parent << dendl;
+ in_use.insert(header->seq);
+ return header;
+}
+
+GenericObjectMap::Header GenericObjectMap::lookup_create_header(
+ const coll_t &cid, const ghobject_t &oid, KeyValueDB::Transaction t)
+{
+ Mutex::Locker l(header_lock);
+ Header header = _lookup_header(cid, oid);
+ if (!header) {
+ header = _generate_new_header(cid, oid, Header(), t);
+ set_header(cid, oid, *header, t);
+ }
+ return header;
+}
+
+void GenericObjectMap::set_parent_header(Header header, KeyValueDB::Transaction t)
+{
+ dout(20) << __func__ << " setting seq " << header->seq << dendl;
+ map<string, bufferlist> to_write;
+ header->encode(to_write[PARENT_KEY]);
+ t->set(parent_seq_prefix(header->seq), to_write);
+}
+
+void GenericObjectMap::clear_header(Header header, KeyValueDB::Transaction t)
+{
+ dout(20) << __func__ << " clearing seq " << header->seq << dendl;
+ t->rmkeys_by_prefix(user_prefix(header, string()));
+ t->rmkeys_by_prefix(complete_prefix(header));
+ set<string> keys;
+ keys.insert(PARENT_KEY);
+ t->rmkeys(parent_seq_prefix(header->seq), keys);
+}
+
+void GenericObjectMap::remove_header(const coll_t &cid,
+ const ghobject_t &oid, Header header,
+ KeyValueDB::Transaction t)
+{
+ dout(20) << __func__ << " removing " << header->seq
+ << " cid " << cid << " oid " << oid << dendl;
+ set<string> to_remove;
+ to_remove.insert(header_key(cid, oid));
+ t->rmkeys(GHOBJECT_TO_SEQ_PREFIX, to_remove);
+}
+
+void GenericObjectMap::set_header(const coll_t &cid, const ghobject_t &oid,
+ _Header header, KeyValueDB::Transaction t)
+{
+ dout(20) << __func__ << " setting " << header.seq
+ << " cid " << cid << " oid " << oid << " parent seq "
+ << header.parent << dendl;
+ map<string, bufferlist> to_set;
+ header.encode(to_set[header_key(cid, oid)]);
+ t->set(GHOBJECT_TO_SEQ_PREFIX, to_set);
+}
+
+int GenericObjectMap::list_objects(const coll_t &cid, ghobject_t start, int max,
+ vector<ghobject_t> *out, ghobject_t *next)
+{
+ Mutex::Locker l(header_lock);
+
+ if (start.is_max())
+ return 0;
+
+ if (start.hobj.is_min()) {
+ vector<ghobject_t> oids;
+
+ KeyValueDB::Iterator iter = db->get_iterator(GHOBJECT_TO_SEQ_PREFIX);
+ for (iter->lower_bound(header_key(cid)); iter->valid(); iter->next()) {
+ bufferlist bl = iter->value();
+ bufferlist::iterator bliter = bl.begin();
+ _Header header;
+ header.decode(bliter);
+
+ if (header.cid == cid)
+ oids.push_back(header.oid);
+
+ break;
+ }
+
+ if (oids.empty()) {
+ if (next)
+ *next = ghobject_t::get_max();
+ return 0;
+ }
+ start = oids[0];
+ }
+
+ int size = 0;
+ KeyValueDB::Iterator iter = db->get_iterator(GHOBJECT_TO_SEQ_PREFIX);
+ for (iter->lower_bound(header_key(cid, start)); iter->valid(); iter->next()) {
+ bufferlist bl = iter->value();
+ bufferlist::iterator bliter = bl.begin();
+ _Header header;
+ header.decode(bliter);
+
+ if (header.cid != cid) {
+ if (next)
+ *next = ghobject_t::get_max();
+ break;
+ }
+
+ if (max && size >= max) {
+ if (next)
+ *next = header.oid;
+ break;
+ }
+
+ assert(start <= header.oid);
+
+
+ size++;
+ if (out)
+ out->push_back(header.oid);
+ start = header.oid;
+ }
+
+ if (out->size())
+ dout(20) << "objects: " << *out << dendl;
+
+ if (!iter->valid())
+ if (next)
+ *next = ghobject_t::get_max();
+
+ return 0;
+}
diff --git a/src/os/GenericObjectMap.h b/src/os/GenericObjectMap.h
new file mode 100644
index 0000000..fddbe53
--- /dev/null
+++ b/src/os/GenericObjectMap.h
@@ -0,0 +1,468 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack <haomai at unitedstack.com>
+ *
+ * Author: Haomai Wang <haomaiwang at gmail.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_GENERICOBJECTMAP_H
+#define CEPH_GENERICOBJECTMAP_H
+
+#include "include/buffer.h"
+#include <set>
+#include <map>
+#include <string>
+#include <vector>
+#include <boost/scoped_ptr.hpp>
+
+#include "include/memory.h"
+#include "ObjectMap.h"
+#include "KeyValueDB.h"
+#include "osd/osd_types.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+
+/**
+ * Genericobjectmap: Provide with key/value associated to ghobject_t APIs to caller
+ * and avoid concerning too much. Wrap and combine KeyValueDB/ObjectMap APIs
+ * with ghobject_t and adding clone capacity.
+ *
+ * Prefix space structure:
+ *
+ * - GHOBJECT_TO_SEQ: Contains leaf mapping from ghobject_t->Header(including
+ * hobj.seq and related metadata)
+ * - INTERN_PREFIX: GLOBAL_STATE_KEY - contains the global state
+ * @see State
+ * @see write_state
+ * @see init
+ * @see generate_new_header
+ * - INTERN_PREFIX + header_key(header->seq) + COMPLETE_PREFIX: see below
+ * - INTERN_PREFIX + header_key(header->seq) + PARENT_KEY
+ * : used to store parent header(same as headers in GHOBJECT_TO_SEQ)
+ * - USER_PREFIX + header_key(header->seq) + [CUSTOM_PREFIX]
+ * : key->value which set by callers
+ *
+ * For each node (represented by a header), we
+ * store three mappings: the key mapping, the complete mapping, and the parent.
+ * The complete mapping (COMPLETE_PREFIX space) is key->key. Each x->y entry in
+ * this mapping indicates that the key mapping contains all entries on [x,y).
+ * Note, max string is represented by "", so ""->"" indicates that the parent
+ * is unnecessary (@see rm_keys). When looking up a key not contained in the
+ * the complete set, we have to check the parent if we don't find it in the
+ * key set. During rm_keys, we copy keys from the parent and update the
+ * complete set to reflect the change @see rm_keys.
+ */
+
+// This class only provide basic read capacity, suggest inherit it to
+// implement write transaction to use it. @see StripObjectMap
+class GenericObjectMap {
+ public:
+ boost::scoped_ptr<KeyValueDB> db;
+
+ /**
+ * Serializes access to next_seq as well as the in_use set
+ */
+ Mutex header_lock;
+ Cond header_cond;
+ Cond map_header_cond;
+
+ /**
+ * Set of headers currently in use
+ */
+ set<uint64_t> in_use;
+ set<ghobject_t> map_header_in_use;
+
+ GenericObjectMap(KeyValueDB *db) : db(db), header_lock("GenericObjectMap") {}
+
+ int get(
+ const coll_t &cid,
+ const ghobject_t &oid,
+ const string &prefix,
+ map<string, bufferlist> *out
+ );
+
+ int get_keys(
+ const coll_t &cid,
+ const ghobject_t &oid,
+ const string &prefix,
+ set<string> *keys
+ );
+
+ int get_values(
+ const coll_t &cid,
+ const ghobject_t &oid,
+ const string &prefix,
+ const set<string> &keys,
+ map<string, bufferlist> *out
+ );
+
+ int check_keys(
+ const coll_t &cid,
+ const ghobject_t &oid,
+ const string &prefix,
+ const set<string> &keys,
+ set<string> *out
+ );
+
+ /// Read initial state from backing store
+ int init(bool upgrade = false);
+
+ /// Upgrade store to current version
+ int upgrade() {return 0;}
+
+ /// Consistency check, debug, there must be no parallel writes
+ bool check(std::ostream &out);
+
+ /// Util, list all objects, there must be no other concurrent access
+ int list_objects(const coll_t &cid, ghobject_t start, int max,
+ vector<ghobject_t> *objs, ///< [out] objects
+ ghobject_t *next);
+
+ ObjectMap::ObjectMapIterator get_iterator(const coll_t &cid,
+ const ghobject_t &oid,
+ const string &prefix);
+
+ KeyValueDB::Transaction get_transaction() { return db->get_transaction(); }
+ int submit_transaction(KeyValueDB::Transaction t) {
+ return db->submit_transaction(t);
+ }
+
+ /// persistent state for store @see generate_header
+ struct State {
+ __u8 v;
+ uint64_t seq;
+ State() : v(0), seq(1) {}
+ State(uint64_t seq) : v(0), seq(seq) {}
+
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(v, bl);
+ ::encode(seq, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(v, bl);
+ ::decode(seq, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_unsigned("seq", seq);
+ }
+
+ static void generate_test_instances(list<State*> &o) {
+ o.push_back(new State(0));
+ o.push_back(new State(20));
+ }
+ } state;
+
+ struct _Header {
+ uint64_t seq;
+ uint64_t parent;
+ uint64_t num_children;
+
+ coll_t cid;
+ ghobject_t oid;
+
+ // Used by successor
+ bufferlist data;
+
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(seq, bl);
+ ::encode(parent, bl);
+ ::encode(num_children, bl);
+ ::encode(cid, bl);
+ ::encode(oid, bl);
+ ::encode(data, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(seq, bl);
+ ::decode(parent, bl);
+ ::decode(num_children, bl);
+ ::decode(cid, bl);
+ ::decode(oid, bl);
+ ::decode(data, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_unsigned("seq", seq);
+ f->dump_unsigned("parent", parent);
+ f->dump_unsigned("num_children", num_children);
+ f->dump_stream("coll") << cid;
+ f->dump_stream("oid") << oid;
+ }
+
+ _Header() : seq(0), parent(0), num_children(1) {}
+ };
+
+ typedef ceph::shared_ptr<_Header> Header;
+
+ Header lookup_header(const coll_t &cid, const ghobject_t &oid) {
+ Mutex::Locker l(header_lock);
+ return _lookup_header(cid, oid);
+ }
+
+ /// Lookup or create header for c oid
+ Header lookup_create_header(const coll_t &cid, const ghobject_t &oid,
+ KeyValueDB::Transaction t);
+
+ /// Set leaf node for c and oid to the value of header
+ void set_header(const coll_t &cid, const ghobject_t &oid, _Header header,
+ KeyValueDB::Transaction t);
+
+ // Move all modify member function to "protect", in order to indicate these
+ // should be made use of by sub-class
+ void set_keys(
+ const Header header,
+ const string &prefix,
+ const map<string, bufferlist> &set,
+ KeyValueDB::Transaction t
+ );
+
+ int clear(
+ const Header header,
+ KeyValueDB::Transaction t
+ );
+
+ int rm_keys(
+ const Header header,
+ const string &prefix,
+ const set<string> &to_clear,
+ KeyValueDB::Transaction t
+ );
+
+ void clone(
+ const Header origin_header,
+ const coll_t &cid,
+ const ghobject_t &target,
+ KeyValueDB::Transaction t,
+ Header *old_header,
+ Header *new_header
+ );
+
+ void rename(
+ const Header header,
+ const coll_t &cid,
+ const ghobject_t &target,
+ KeyValueDB::Transaction t
+ );
+
+ /// Ensure that all previous operations are durable
+ int sync(const Header header, KeyValueDB::Transaction t);
+
+ static const string GLOBAL_STATE_KEY;
+ static const string PARENT_KEY;
+
+ static const string USER_PREFIX;
+ static const string INTERN_PREFIX;
+ static const string PARENT_PREFIX;
+ static const string COMPLETE_PREFIX;
+ static const string GHOBJECT_TO_SEQ_PREFIX;
+
+ static const string GHOBJECT_KEY_SEP_S;
+ static const char GHOBJECT_KEY_SEP_C;
+
+private:
+ /// Implicit lock on Header->seq
+
+ static string header_key(const coll_t &cid);
+ static string header_key(const coll_t &cid, const ghobject_t &oid);
+ static bool parse_header_key(const string &in, coll_t *c, ghobject_t *oid);
+
+ string seq_key(uint64_t seq) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%.*" PRId64, (int)(2*sizeof(seq)), seq);
+ return string(buf);
+ }
+
+ string user_prefix(Header header, const string &prefix);
+ string complete_prefix(Header header);
+ string parent_seq_prefix(uint64_t seq);
+
+ class EmptyIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+ public:
+ int seek_to_first() { return 0; }
+ int seek_to_last() { return 0; }
+ int upper_bound(const string &after) { return 0; }
+ int lower_bound(const string &to) { return 0; }
+ bool valid() { return false; }
+ int next() { assert(0); return 0; }
+ string key() { assert(0); return ""; }
+ bufferlist value() { assert(0); return bufferlist(); }
+ int status() { return 0; }
+ };
+
+
+ /// Iterator
+ class GenericObjectMapIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+ public:
+ GenericObjectMap *map;
+
+ /// NOTE: implicit lock on header->seq AND for all ancestors
+ Header header;
+
+ /// parent_iter == NULL iff no parent
+ ceph::shared_ptr<GenericObjectMapIteratorImpl> parent_iter;
+ KeyValueDB::Iterator key_iter;
+ KeyValueDB::Iterator complete_iter;
+
+ /// cur_iter points to currently valid iterator
+ ceph::shared_ptr<ObjectMap::ObjectMapIteratorImpl> cur_iter;
+ int r;
+
+ /// init() called, key_iter, complete_iter, parent_iter filled in
+ bool ready;
+ /// past end
+ bool invalid;
+
+ string prefix;
+
+ GenericObjectMapIteratorImpl(GenericObjectMap *map, Header header,
+ const string &_prefix) : map(map), header(header), r(0), ready(false),
+ invalid(true), prefix(_prefix) { }
+ int seek_to_first();
+ int seek_to_last();
+ int upper_bound(const string &after);
+ int lower_bound(const string &to);
+ bool valid();
+ int next();
+ string key();
+ bufferlist value();
+ int status();
+
+ bool on_parent() {
+ return cur_iter == parent_iter;
+ }
+
+ /// skips to next valid parent entry
+ int next_parent();
+
+ /// Tests whether to_test is in complete region
+ int in_complete_region(const string &to_test, ///[in] key to test
+ string *begin, ///[out] beginning of region
+ string *end ///[out] end of region
+ ); ///< @returns true if to_test is in the complete region, else false
+
+ private:
+ int init();
+ bool valid_parent();
+ int adjust();
+ };
+
+ typedef ceph::shared_ptr<GenericObjectMapIteratorImpl> GenericObjectMapIterator;
+ GenericObjectMapIterator _get_iterator(Header header, string prefix) {
+ return GenericObjectMapIterator(new GenericObjectMapIteratorImpl(this, header, prefix));
+ }
+
+ /// Removes node corresponding to header
+ void clear_header(Header header, KeyValueDB::Transaction t);
+
+ /// Set node containing input to new contents
+ void set_parent_header(Header input, KeyValueDB::Transaction t);
+
+ /// Remove leaf node corresponding to oid in c
+ void remove_header(const coll_t &cid, const ghobject_t &oid, Header header,
+ KeyValueDB::Transaction t);
+
+ /**
+ * Generate new header for c oid with new seq number
+ *
+ * Has the side effect of syncronously saving the new GenericObjectMap state
+ */
+ Header _generate_new_header(const coll_t &cid, const ghobject_t &oid,
+ Header parent, KeyValueDB::Transaction t);
+ Header generate_new_header(const coll_t &cid, const ghobject_t &oid,
+ Header parent, KeyValueDB::Transaction t) {
+ Mutex::Locker l(header_lock);
+ return _generate_new_header(cid, oid, parent, t);
+ }
+
+ // Lookup leaf header for c oid
+ Header _lookup_header(const coll_t &cid, const ghobject_t &oid);
+
+ // Lookup header node for input
+ Header lookup_parent(Header input);
+
+ // Scan keys in header into out_keys and out_values (if nonnull)
+ int scan(Header header, const string &prefix, const set<string> &in_keys,
+ set<string> *out_keys, map<string, bufferlist> *out_values);
+
+ // Remove header and all related prefixes
+ int _clear(Header header, KeyValueDB::Transaction t);
+
+ // Adds to t operations necessary to add new_complete to the complete set
+ int merge_new_complete(Header header, const map<string, string> &new_complete,
+ GenericObjectMapIterator iter, KeyValueDB::Transaction t);
+
+ // Writes out State (mainly next_seq)
+ int write_state(KeyValueDB::Transaction _t);
+
+ // 0 if the complete set now contains all of key space, < 0 on error, 1 else
+ int need_parent(GenericObjectMapIterator iter);
+
+ // Copies header entry from parent @see rm_keys
+ int copy_up_header(Header header, KeyValueDB::Transaction t);
+
+ // Sets header @see set_header
+ void _set_header(Header header, const bufferlist &bl,
+ KeyValueDB::Transaction t);
+
+ /**
+ * Removes map header lock once Header is out of scope
+ * @see lookup_header
+ */
+ class RemoveMapHeaderOnDelete {
+ public:
+ GenericObjectMap *db;
+ coll_t cid;
+ ghobject_t oid;
+ RemoveMapHeaderOnDelete(GenericObjectMap *db, const coll_t &cid,
+ const ghobject_t &oid) :
+ db(db), cid(cid), oid(oid) {}
+ void operator() (_Header *header) {
+ Mutex::Locker l(db->header_lock);
+ db->map_header_in_use.erase(oid);
+ db->map_header_cond.Signal();
+ delete header;
+ }
+ };
+
+ /**
+ * Removes header seq lock once Header is out of scope
+ * @see lookup_parent
+ * @see generate_new_header
+ */
+ class RemoveOnDelete {
+ public:
+ GenericObjectMap *db;
+ RemoveOnDelete(GenericObjectMap *db) :
+ db(db) {}
+ void operator() (_Header *header) {
+ Mutex::Locker l(db->header_lock);
+ db->in_use.erase(header->seq);
+ db->header_cond.Signal();
+ delete header;
+ }
+ };
+ friend class RemoveOnDelete;
+};
+WRITE_CLASS_ENCODER(GenericObjectMap::_Header)
+WRITE_CLASS_ENCODER(GenericObjectMap::State)
+
+#endif
diff --git a/src/os/HashIndex.cc b/src/os/HashIndex.cc
index ea50cd0..134deb9 100644
--- a/src/os/HashIndex.cc
+++ b/src/os/HashIndex.cc
@@ -224,7 +224,7 @@ int HashIndex::col_split_level(
int HashIndex::_split(
uint32_t match,
uint32_t bits,
- std::tr1::shared_ptr<CollectionIndex> dest) {
+ ceph::shared_ptr<CollectionIndex> dest) {
assert(collection_version() == dest->collection_version());
unsigned mkdirred = 0;
return col_split_level(
diff --git a/src/os/HashIndex.h b/src/os/HashIndex.h
index 6f5bca0..bdfee2e 100644
--- a/src/os/HashIndex.h
+++ b/src/os/HashIndex.h
@@ -155,7 +155,7 @@ public:
int _split(
uint32_t match,
uint32_t bits,
- std::tr1::shared_ptr<CollectionIndex> dest
+ ceph::shared_ptr<CollectionIndex> dest
);
protected:
diff --git a/src/os/IndexManager.cc b/src/os/IndexManager.cc
index 83bbfc9..e8a3785 100644
--- a/src/os/IndexManager.cc
+++ b/src/os/IndexManager.cc
@@ -12,7 +12,7 @@
*
*/
-#include <tr1/memory>
+#include "include/memory.h"
#include <map>
#if defined(__FreeBSD__)
diff --git a/src/os/IndexManager.h b/src/os/IndexManager.h
index 43de7ee..2aa7c66 100644
--- a/src/os/IndexManager.h
+++ b/src/os/IndexManager.h
@@ -14,7 +14,7 @@
#ifndef OS_INDEXMANAGER_H
#define OS_INDEXMANAGER_H
-#include <tr1/memory>
+#include "include/memory.h"
#include <map>
#include "common/Mutex.h"
@@ -28,7 +28,7 @@
/// Public type for Index
-typedef std::tr1::shared_ptr<CollectionIndex> Index;
+typedef ceph::shared_ptr<CollectionIndex> Index;
/**
* Encapsulates mutual exclusion for CollectionIndexes.
*
@@ -50,7 +50,7 @@ class IndexManager {
bool upgrade;
/// Currently in use CollectionIndices
- map<coll_t,std::tr1::weak_ptr<CollectionIndex> > col_indices;
+ map<coll_t,ceph::weak_ptr<CollectionIndex> > col_indices;
/// Cleans up state for c @see RemoveOnDelete
void put_index(
diff --git a/src/os/JournalingObjectStore.h b/src/os/JournalingObjectStore.h
index 338ac87..946ab7c 100644
--- a/src/os/JournalingObjectStore.h
+++ b/src/os/JournalingObjectStore.h
@@ -120,9 +120,12 @@ public:
}
public:
- JournalingObjectStore() : journal(NULL), finisher(g_ceph_context),
- apply_manager(journal, finisher),
- replaying(false) {}
+ JournalingObjectStore(const std::string& path)
+ : ObjectStore(path),
+ journal(NULL),
+ finisher(g_ceph_context),
+ apply_manager(journal, finisher),
+ replaying(false) {}
};
diff --git a/src/os/KeyValueDB.h b/src/os/KeyValueDB.h
index e98463a..c581aa5 100644
--- a/src/os/KeyValueDB.h
+++ b/src/os/KeyValueDB.h
@@ -7,7 +7,7 @@
#include <set>
#include <map>
#include <string>
-#include <tr1/memory>
+#include "include/memory.h"
#include <boost/scoped_ptr.hpp>
#include "ObjectMap.h"
@@ -62,7 +62,11 @@ public:
virtual ~TransactionImpl() {};
};
- typedef std::tr1::shared_ptr< TransactionImpl > Transaction;
+ typedef ceph::shared_ptr< TransactionImpl > Transaction;
+
+ virtual int init() = 0;
+ virtual int open(ostream &out) = 0;
+ virtual int create_and_open(ostream &out) = 0;
virtual Transaction get_transaction() = 0;
virtual int submit_transaction(Transaction) = 0;
@@ -94,7 +98,7 @@ public:
virtual int status() = 0;
virtual ~WholeSpaceIteratorImpl() { }
};
- typedef std::tr1::shared_ptr< WholeSpaceIteratorImpl > WholeSpaceIterator;
+ typedef ceph::shared_ptr< WholeSpaceIteratorImpl > WholeSpaceIterator;
class IteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
const string prefix;
@@ -143,14 +147,14 @@ public:
}
};
- typedef std::tr1::shared_ptr< IteratorImpl > Iterator;
+ typedef ceph::shared_ptr< IteratorImpl > Iterator;
WholeSpaceIterator get_iterator() {
return _get_iterator();
}
Iterator get_iterator(const string &prefix) {
- return std::tr1::shared_ptr<IteratorImpl>(
+ return ceph::shared_ptr<IteratorImpl>(
new IteratorImpl(prefix, get_iterator())
);
}
@@ -160,7 +164,7 @@ public:
}
Iterator get_snapshot_iterator(const string &prefix) {
- return std::tr1::shared_ptr<IteratorImpl>(
+ return ceph::shared_ptr<IteratorImpl>(
new IteratorImpl(prefix, get_snapshot_iterator())
);
}
diff --git a/src/os/KeyValueStore.cc b/src/os/KeyValueStore.cc
new file mode 100644
index 0000000..e209dbc
--- /dev/null
+++ b/src/os/KeyValueStore.cc
@@ -0,0 +1,2978 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack <haomai at unitedstack.com>
+ *
+ * Author: Haomai Wang <haomaiwang at gmail.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/file.h>
+#include <errno.h>
+#include <dirent.h>
+
+#include <iostream>
+#include <map>
+
+#include "include/compat.h"
+
+#include <fstream>
+#include <sstream>
+
+#include "KeyValueStore.h"
+#include "common/BackTrace.h"
+#include "include/types.h"
+
+#include "osd/osd_types.h"
+#include "include/color.h"
+#include "include/buffer.h"
+
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/run_cmd.h"
+#include "common/safe_io.h"
+#include "common/perf_counters.h"
+#include "common/sync_filesystem.h"
+#include "LevelDBStore.h"
+
+#include "common/ceph_crypto.h"
+using ceph::crypto::SHA1;
+
+#include "include/assert.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_keyvaluestore
+
+const string KeyValueStore::OBJECT_STRIP_PREFIX = "_STRIP_";
+const string KeyValueStore::OBJECT_XATTR = "__OBJATTR__";
+const string KeyValueStore::OBJECT_OMAP = "__OBJOMAP__";
+const string KeyValueStore::OBJECT_OMAP_HEADER = "__OBJOMAP_HEADER__";
+const string KeyValueStore::OBJECT_OMAP_HEADER_KEY = "__OBJOMAP_HEADER__KEY_";
+const string KeyValueStore::COLLECTION = "__COLLECTION__";
+const string KeyValueStore::COLLECTION_ATTR = "__COLL_ATTR__";
+
+// ============== StripObjectMap Implementation =================
+
+void StripObjectMap::sync_wrap(StripObjectHeader &strip_header,
+ KeyValueDB::Transaction t,
+ const SequencerPosition &spos)
+{
+ dout(10) << __func__ << " cid: " << strip_header.cid << "oid: "
+ << strip_header.oid << " setting spos to " << strip_header.spos
+ << dendl;
+ strip_header.spos = spos;
+ strip_header.header->data.clear();
+ ::encode(strip_header, strip_header.header->data);
+
+ sync(strip_header.header, t);
+}
+
+bool StripObjectMap::check_spos(const StripObjectHeader &header,
+ const SequencerPosition *spos)
+{
+ if (!spos || *spos > header.spos) {
+ stringstream out;
+ if (spos)
+ dout(10) << "cid: " << "oid: " << header.oid
+ << " not skipping op, *spos " << *spos << dendl;
+ else
+ dout(10) << "cid: " << "oid: " << header.oid
+ << " not skipping op, *spos " << "empty" << dendl;
+ dout(10) << " > header.spos " << header.spos << dendl;
+ return false;
+ } else {
+ dout(10) << "cid: " << "oid: " << header.oid << " skipping op, *spos "
+ << *spos << " <= header.spos " << header.spos << dendl;
+ return true;
+ }
+}
+
+int StripObjectMap::save_strip_header(StripObjectHeader &strip_header,
+ KeyValueDB::Transaction t)
+{
+ strip_header.header->data.clear();
+ ::encode(strip_header, strip_header.header->data);
+
+ set_header(strip_header.cid, strip_header.oid, *(strip_header.header), t);
+ return 0;
+}
+
+int StripObjectMap::create_strip_header(const coll_t &cid,
+ const ghobject_t &oid,
+ StripObjectHeader &strip_header,
+ KeyValueDB::Transaction t)
+{
+ Header header = lookup_create_header(cid, oid, t);
+ if (!header)
+ return -EINVAL;
+
+ strip_header.oid = oid;
+ strip_header.cid = cid;
+ strip_header.header = header;
+
+ return 0;
+}
+
+int StripObjectMap::lookup_strip_header(const coll_t &cid,
+ const ghobject_t &oid,
+ StripObjectHeader &strip_header)
+{
+ Header header = lookup_header(cid, oid);
+
+ if (!header) {
+ dout(20) << "lookup_strip_header failed to get strip_header "
+ << " cid " << cid <<" oid " << oid << dendl;
+ return -ENOENT;
+ }
+
+ if (header->data.length()) {
+ bufferlist::iterator bliter = header->data.begin();
+ ::decode(strip_header, bliter);
+ }
+
+ if (strip_header.strip_size == 0)
+ strip_header.strip_size = default_strip_size;
+
+ strip_header.oid = oid;
+ strip_header.cid = cid;
+ strip_header.header = header;
+
+ dout(10) << "lookup_strip_header done " << " cid " << cid << " oid "
+ << oid << dendl;
+ return 0;
+}
+
+int StripObjectMap::file_to_extents(uint64_t offset, size_t len,
+ uint64_t strip_size,
+ vector<StripExtent> &extents)
+{
+ if (len == 0)
+ return 0;
+
+ uint64_t start, end, strip_offset, extent_offset, extent_len;
+ start = offset / strip_size;
+ end = (offset + len) / strip_size;
+ strip_offset = start * strip_size;
+
+ // "offset" may in the middle of first strip object
+ if (offset > strip_offset) {
+ extent_offset = offset - strip_offset;
+ if (extent_offset + len <= strip_size)
+ extent_len = len;
+ else
+ extent_len = strip_size - extent_offset;
+ extents.push_back(StripExtent(start, extent_offset, extent_len));
+ start++;
+ strip_offset += strip_size;
+ }
+
+ for (; start < end; ++start) {
+ extents.push_back(StripExtent(start, 0, strip_size));
+ strip_offset += strip_size;
+ }
+
+ // The end of strip object may be partial
+ if (offset + len > strip_offset)
+ extents.push_back(StripExtent(start, 0, offset+len-strip_offset));
+
+ assert(extents.size());
+ dout(10) << "file_to_extents done " << dendl;
+ return 0;
+}
+
+void StripObjectMap::clone_wrap(StripObjectHeader &old_header,
+ const coll_t &cid, const ghobject_t &oid,
+ KeyValueDB::Transaction t,
+ const SequencerPosition &spos,
+ StripObjectHeader *origin_header,
+ StripObjectHeader *target_header)
+{
+ Header new_origin_header;
+
+ if (target_header)
+ *target_header = old_header;
+
+ clone(old_header.header, cid, oid, t, &new_origin_header,
+ &target_header->header);
+
+ old_header.header = new_origin_header;
+
+ if (origin_header)
+ origin_header->spos = spos;
+
+ if (target_header) {
+ target_header->oid = oid;
+ target_header->cid = cid;
+ target_header->spos = spos;
+ }
+}
+
+void StripObjectMap::rename_wrap(const coll_t &cid, const ghobject_t &oid,
+ KeyValueDB::Transaction t,
+ const SequencerPosition &spos,
+ StripObjectHeader *header)
+{
+ assert(header);
+ rename(header->header, cid, oid, t);
+
+ if (header) {
+ header->oid = oid;
+ header->cid = cid;
+ header->spos = spos;
+ }
+}
+
+
+// =========== KeyValueStore::SubmitManager Implementation ==============
+
+uint64_t KeyValueStore::SubmitManager::op_submit_start()
+{
+ lock.Lock();
+ uint64_t op = ++op_seq;
+ dout(10) << "op_submit_start " << op << dendl;
+ return op;
+}
+
+void KeyValueStore::SubmitManager::op_submit_finish(uint64_t op)
+{
+ dout(10) << "op_submit_finish " << op << dendl;
+ if (op != op_submitted + 1) {
+ dout(0) << "op_submit_finish " << op << " expected " << (op_submitted + 1)
+ << ", OUT OF ORDER" << dendl;
+ assert(0 == "out of order op_submit_finish");
+ }
+ op_submitted = op;
+ lock.Unlock();
+}
+
+
+// ========= KeyValueStore::BufferTransaction Implementation ============
+
+int KeyValueStore::BufferTransaction::check_coll(const coll_t &cid)
+{
+ int r = store->_check_coll(cid);
+ if (r == 0)
+ return r;
+
+ StripHeaderMap::iterator it = strip_headers.find(
+ make_pair(get_coll_for_coll(), make_ghobject_for_coll(cid)));
+ if (it != strip_headers.end() && !it->second.deleted) {
+ return 0;
+ }
+ return -ENOENT;
+}
+
+int KeyValueStore::BufferTransaction::lookup_cached_header(
+ const coll_t &cid, const ghobject_t &oid,
+ StripObjectMap::StripObjectHeader **strip_header,
+ bool create_if_missing)
+{
+ if (check_coll(cid) < 0)
+ return -ENOENT;
+
+ StripObjectMap::StripObjectHeader header;
+ int r = 0;
+
+ StripHeaderMap::iterator it = strip_headers.find(make_pair(cid, oid));
+ if (it != strip_headers.end()) {
+ if (it->second.deleted)
+ return -ENOENT;
+
+ if (strip_header)
+ *strip_header = &it->second;
+ return 0;
+ }
+
+ r = store->backend->lookup_strip_header(cid, oid, header);
+ if (r < 0 && create_if_missing) {
+ r = store->backend->create_strip_header(cid, oid, header, t);
+ }
+
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " "
+ << " r = " << r << dendl;
+ return r;
+ }
+
+ strip_headers[make_pair(cid, oid)] = header;
+ if (strip_header)
+ *strip_header = &strip_headers[make_pair(cid, oid)];
+ return r;
+}
+
+int KeyValueStore::BufferTransaction::get_buffer_key(
+ StripObjectMap::StripObjectHeader *strip_header, const string &prefix,
+ const string &key, bufferlist &bl)
+{
+ if (strip_header->buffers.count(make_pair(prefix, key))) {
+ bl.swap(strip_header->buffers[make_pair(prefix, key)]);
+ return 0;
+ }
+
+ set<string> keys;
+ map<string, bufferlist> out;
+ keys.insert(key);
+ int r = store->backend->get_values(strip_header->cid, strip_header->oid,
+ prefix, keys, &out);
+ if (r < 0) {
+ dout(10) << __func__ << " " << strip_header->cid << "/"
+ << strip_header->oid << " " << " r = " << r << dendl;
+ return r;
+ }
+
+ assert(out.size() == 1);
+ bl.swap(out.begin()->second);
+ return 0;
+}
+
+void KeyValueStore::BufferTransaction::set_buffer_keys(
+ const string &prefix, StripObjectMap::StripObjectHeader *strip_header,
+ map<string, bufferlist> &values)
+{
+ if (store->backend->check_spos(*strip_header, &spos))
+ return ;
+
+ store->backend->set_keys(strip_header->header, prefix, values, t);
+
+ for (map<string, bufferlist>::iterator iter = values.begin();
+ iter != values.end(); ++iter) {
+ strip_header->buffers[make_pair(prefix, iter->first)].swap(iter->second);
+ }
+}
+
+int KeyValueStore::BufferTransaction::remove_buffer_keys(
+ const string &prefix, StripObjectMap::StripObjectHeader *strip_header,
+ const set<string> &keys)
+{
+ if (store->backend->check_spos(*strip_header, &spos))
+ return 0;
+
+ for (set<string>::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
+ strip_header->buffers[make_pair(prefix, *iter)] = bufferlist();
+ }
+
+ return store->backend->rm_keys(strip_header->header, prefix, keys, t);
+}
+
+void KeyValueStore::BufferTransaction::clear_buffer_keys(
+ const string &prefix, StripObjectMap::StripObjectHeader *strip_header)
+{
+ for (map<pair<string, string>, bufferlist>::iterator iter = strip_header->buffers.begin();
+ iter != strip_header->buffers.end(); ++iter) {
+ if (iter->first.first == prefix)
+ iter->second = bufferlist();
+ }
+}
+
+int KeyValueStore::BufferTransaction::clear_buffer(
+ StripObjectMap::StripObjectHeader *strip_header)
+{
+ if (store->backend->check_spos(*strip_header, &spos))
+ return 0;
+
+ strip_header->deleted = true;
+
+ return store->backend->clear(strip_header->header, t);
+}
+
+void KeyValueStore::BufferTransaction::clone_buffer(
+ StripObjectMap::StripObjectHeader *old_header,
+ const coll_t &cid, const ghobject_t &oid)
+{
+ if (store->backend->check_spos(*old_header, &spos))
+ return ;
+
+ // Remove target ahead to avoid dead lock
+ strip_headers.erase(make_pair(cid, oid));
+
+ StripObjectMap::StripObjectHeader new_origin_header, new_target_header;
+
+ store->backend->clone_wrap(*old_header, cid, oid, t, spos,
+ &new_origin_header, &new_target_header);
+
+ // FIXME: Lacking of lock for origin header(now become parent), it will
+ // cause other operation can get the origin header while submitting
+ // transactions
+ strip_headers[make_pair(cid, old_header->oid)] = new_origin_header;
+ strip_headers[make_pair(cid, oid)] = new_target_header;
+}
+
+void KeyValueStore::BufferTransaction::rename_buffer(
+ StripObjectMap::StripObjectHeader *old_header,
+ const coll_t &cid, const ghobject_t &oid)
+{
+ if (store->backend->check_spos(*old_header, &spos))
+ return ;
+
+ // FIXME: Lacking of lock for origin header, it will cause other operation
+ // can get the origin header while submitting transactions
+ store->backend->rename_wrap(cid, oid, t, spos, old_header);
+
+ strip_headers.erase(make_pair(old_header->cid, old_header->oid));
+ strip_headers[make_pair(cid, oid)] = *old_header;
+}
+
+int KeyValueStore::BufferTransaction::submit_transaction()
+{
+ int r = 0;
+
+ for (StripHeaderMap::iterator header_iter = strip_headers.begin();
+ header_iter != strip_headers.end(); ++header_iter) {
+ StripObjectMap::StripObjectHeader header = header_iter->second;
+
+ if (store->backend->check_spos(header, &spos))
+ continue;
+
+ if (header.deleted)
+ continue;
+
+ r = store->backend->save_strip_header(header, t);
+ if (r < 0) {
+ dout(10) << __func__ << " save strip header failed " << dendl;
+ goto out;
+ }
+ }
+
+out:
+
+ dout(5) << __func__ << " r = " << r << dendl;
+ return store->backend->submit_transaction(t);
+}
+
+// =========== KeyValueStore Intern Helper Implementation ==============
+
+ostream& operator<<(ostream& out, const KeyValueStore::OpSequencer& s)
+{
+ assert(&out);
+ return out << *s.parent;
+}
+
+int KeyValueStore::_create_current()
+{
+ struct stat st;
+ int ret = ::stat(current_fn.c_str(), &st);
+ if (ret == 0) {
+ // current/ exists
+ if (!S_ISDIR(st.st_mode)) {
+ dout(0) << "_create_current: current/ exists but is not a directory" << dendl;
+ ret = -EINVAL;
+ }
+ } else {
+ ret = ::mkdir(current_fn.c_str(), 0755);
+ if (ret < 0) {
+ ret = -errno;
+ dout(0) << "_create_current: mkdir " << current_fn << " failed: "<< cpp_strerror(ret) << dendl;
+ }
+ }
+
+ return ret;
+}
+
+
+
+// =========== KeyValueStore API Implementation ==============
+
+KeyValueStore::KeyValueStore(const std::string &base,
+ const char *name, bool do_update) :
+ ObjectStore(base),
+ internal_name(name),
+ basedir(base),
+ fsid_fd(-1), op_fd(-1), current_fd(-1),
+ kv_type(KV_TYPE_NONE),
+ backend(NULL),
+ ondisk_finisher(g_ceph_context),
+ lock("KeyValueStore::lock"),
+ default_osr("default"),
+ op_queue_len(0), op_queue_bytes(0),
+ op_finisher(g_ceph_context),
+ op_tp(g_ceph_context, "KeyValueStore::op_tp",
+ g_conf->filestore_op_threads, "keyvaluestore_op_threads"),
+ op_wq(this, g_conf->filestore_op_thread_timeout,
+ g_conf->filestore_op_thread_suicide_timeout, &op_tp),
+ logger(NULL),
+ read_error_lock("KeyValueStore::read_error_lock"),
+ m_fail_eio(g_conf->filestore_fail_eio),
+ do_update(do_update)
+{
+ ostringstream oss;
+ oss << basedir << "/current";
+ current_fn = oss.str();
+
+ ostringstream sss;
+ sss << basedir << "/current/commit_op_seq";
+ current_op_seq_fn = sss.str();
+
+ // initialize logger
+ PerfCountersBuilder plb(g_ceph_context, internal_name, 0, 1);
+ logger = plb.create_perf_counters();
+
+ g_ceph_context->get_perfcounters_collection()->add(logger);
+ g_ceph_context->_conf->add_observer(this);
+}
+
+KeyValueStore::~KeyValueStore()
+{
+ g_ceph_context->_conf->remove_observer(this);
+ g_ceph_context->get_perfcounters_collection()->remove(logger);
+
+ delete logger;
+}
+
+int KeyValueStore::statfs(struct statfs *buf)
+{
+ if (::statfs(basedir.c_str(), buf) < 0) {
+ int r = -errno;
+ assert(!m_fail_eio || r != -EIO);
+ return r;
+ }
+ return 0;
+}
+
+int KeyValueStore::mkfs()
+{
+ int ret = 0;
+ char fsid_fn[PATH_MAX];
+ uuid_d old_fsid;
+
+ dout(1) << "mkfs in " << basedir << dendl;
+
+ // open+lock fsid
+ snprintf(fsid_fn, sizeof(fsid_fn), "%s/fsid", basedir.c_str());
+ fsid_fd = ::open(fsid_fn, O_RDWR|O_CREAT, 0644);
+ if (fsid_fd < 0) {
+ ret = -errno;
+ derr << "mkfs: failed to open " << fsid_fn << ": " << cpp_strerror(ret) << dendl;
+ return ret;
+ }
+
+ if (lock_fsid() < 0) {
+ ret = -EBUSY;
+ goto close_fsid_fd;
+ }
+
+ if (read_fsid(fsid_fd, &old_fsid) < 0 || old_fsid.is_zero()) {
+ if (fsid.is_zero()) {
+ fsid.generate_random();
+ dout(1) << "mkfs generated fsid " << fsid << dendl;
+ } else {
+ dout(1) << "mkfs using provided fsid " << fsid << dendl;
+ }
+
+ char fsid_str[40];
+ fsid.print(fsid_str);
+ strcat(fsid_str, "\n");
+ ret = ::ftruncate(fsid_fd, 0);
+ if (ret < 0) {
+ ret = -errno;
+ derr << "mkfs: failed to truncate fsid: " << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+ ret = safe_write(fsid_fd, fsid_str, strlen(fsid_str));
+ if (ret < 0) {
+ derr << "mkfs: failed to write fsid: " << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+ if (::fsync(fsid_fd) < 0) {
+ ret = errno;
+ derr << "mkfs: close failed: can't write fsid: "
+ << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+ dout(10) << "mkfs fsid is " << fsid << dendl;
+ } else {
+ if (!fsid.is_zero() && fsid != old_fsid) {
+ derr << "mkfs on-disk fsid " << old_fsid << " != provided " << fsid << dendl;
+ ret = -EINVAL;
+ goto close_fsid_fd;
+ }
+ fsid = old_fsid;
+ dout(1) << "mkfs fsid is already set to " << fsid << dendl;
+ }
+
+ // version stamp
+ ret = write_version_stamp();
+ if (ret < 0) {
+ derr << "mkfs: write_version_stamp() failed: "
+ << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+
+ ret = _create_current();
+ if (ret < 0) {
+ derr << "mkfs: failed to create current/ " << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+
+ if (_detect_backend()) {
+ derr << "KeyValueStore::mkfs error in _detect_backend" << dendl;
+ ret = -1;
+ goto close_fsid_fd;
+ }
+
+ {
+ KeyValueDB *store;
+ if (kv_type == KV_TYPE_LEVELDB) {
+ store = new LevelDBStore(g_ceph_context, current_fn);
+ } else {
+ derr << "KeyValueStore::mkfs error: unknown backend type" << kv_type << dendl;
+ ret = -1;
+ goto close_fsid_fd;
+ }
+
+ store->init();
+ stringstream err;
+ if (store->create_and_open(err)) {
+ derr << "KeyValueStore::mkfs failed to create keyvaluestore backend: "
+ << err.str() << dendl;
+ ret = -1;
+ delete store;
+ goto close_fsid_fd;
+ } else {
+ delete store;
+ dout(1) << "keyvaluestore backend exists/created" << dendl;
+ }
+ }
+
+ dout(1) << "mkfs done in " << basedir << dendl;
+ ret = 0;
+
+ close_fsid_fd:
+ TEMP_FAILURE_RETRY(::close(fsid_fd));
+ fsid_fd = -1;
+ return ret;
+}
+
+int KeyValueStore::read_fsid(int fd, uuid_d *uuid)
+{
+ char fsid_str[40];
+ int ret = safe_read(fd, fsid_str, sizeof(fsid_str));
+ if (ret < 0)
+ return ret;
+ if (ret == 8) {
+ // old 64-bit fsid... mirror it.
+ *(uint64_t*)&uuid->uuid[0] = *(uint64_t*)fsid_str;
+ *(uint64_t*)&uuid->uuid[8] = *(uint64_t*)fsid_str;
+ return 0;
+ }
+
+ if (ret > 36)
+ fsid_str[36] = 0;
+ if (!uuid->parse(fsid_str))
+ return -EINVAL;
+ return 0;
+}
+
+int KeyValueStore::lock_fsid()
+{
+ struct flock l;
+ memset(&l, 0, sizeof(l));
+ l.l_type = F_WRLCK;
+ l.l_whence = SEEK_SET;
+ l.l_start = 0;
+ l.l_len = 0;
+ int r = ::fcntl(fsid_fd, F_SETLK, &l);
+ if (r < 0) {
+ int err = errno;
+ dout(0) << "lock_fsid failed to lock " << basedir
+ << "/fsid, is another ceph-osd still running? "
+ << cpp_strerror(err) << dendl;
+ return -err;
+ }
+ return 0;
+}
+
+bool KeyValueStore::test_mount_in_use()
+{
+ dout(5) << "test_mount basedir " << basedir << dendl;
+ char fn[PATH_MAX];
+ snprintf(fn, sizeof(fn), "%s/fsid", basedir.c_str());
+
+ // verify fs isn't in use
+
+ fsid_fd = ::open(fn, O_RDWR, 0644);
+ if (fsid_fd < 0)
+ return 0; // no fsid, ok.
+ bool inuse = lock_fsid() < 0;
+ TEMP_FAILURE_RETRY(::close(fsid_fd));
+ fsid_fd = -1;
+ return inuse;
+}
+
+int KeyValueStore::update_version_stamp()
+{
+ return write_version_stamp();
+}
+
+int KeyValueStore::version_stamp_is_valid(uint32_t *version)
+{
+ bufferptr bp(PATH_MAX);
+ int ret = safe_read_file(basedir.c_str(), "store_version",
+ bp.c_str(), bp.length());
+ if (ret < 0) {
+ if (ret == -ENOENT)
+ return 0;
+ return ret;
+ }
+ bufferlist bl;
+ bl.push_back(bp);
+ bufferlist::iterator i = bl.begin();
+ ::decode(*version, i);
+ if (*version == target_version)
+ return 1;
+ else
+ return 0;
+}
+
+int KeyValueStore::write_version_stamp()
+{
+ bufferlist bl;
+ ::encode(target_version, bl);
+
+ return safe_write_file(basedir.c_str(), "store_version",
+ bl.c_str(), bl.length());
+}
+
+int KeyValueStore::mount()
+{
+ int ret;
+ char buf[PATH_MAX];
+
+ dout(5) << "basedir " << basedir << dendl;
+
+ // make sure global base dir exists
+ if (::access(basedir.c_str(), R_OK | W_OK)) {
+ ret = -errno;
+ derr << "KeyValueStore::mount: unable to access basedir '" << basedir
+ << "': " << cpp_strerror(ret) << dendl;
+ goto done;
+ }
+
+ // get fsid
+ snprintf(buf, sizeof(buf), "%s/fsid", basedir.c_str());
+ fsid_fd = ::open(buf, O_RDWR, 0644);
+ if (fsid_fd < 0) {
+ ret = -errno;
+ derr << "KeyValueStore::mount: error opening '" << buf << "': "
+ << cpp_strerror(ret) << dendl;
+ goto done;
+ }
+
+ ret = read_fsid(fsid_fd, &fsid);
+ if (ret < 0) {
+ derr << "KeyValueStore::mount: error reading fsid_fd: "
+ << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+
+ if (lock_fsid() < 0) {
+ derr << "KeyValueStore::mount: lock_fsid failed" << dendl;
+ ret = -EBUSY;
+ goto close_fsid_fd;
+ }
+
+ dout(10) << "mount fsid is " << fsid << dendl;
+
+ uint32_t version_stamp;
+ ret = version_stamp_is_valid(&version_stamp);
+ if (ret < 0) {
+ derr << "KeyValueStore::mount : error in version_stamp_is_valid: "
+ << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ } else if (ret == 0) {
+ if (do_update) {
+ derr << "KeyValueStore::mount : stale version stamp detected: "
+ << version_stamp << ". Proceeding, do_update "
+ << "is set, performing disk format upgrade." << dendl;
+ } else {
+ ret = -EINVAL;
+ derr << "KeyValueStore::mount : stale version stamp " << version_stamp
+ << ". Please run the KeyValueStore update script before starting "
+ << "the OSD, or set keyvaluestore_update_to to " << target_version
+ << dendl;
+ goto close_fsid_fd;
+ }
+ }
+
+ current_fd = ::open(current_fn.c_str(), O_RDONLY);
+ if (current_fd < 0) {
+ ret = -errno;
+ derr << "KeyValueStore::mount: error opening: " << current_fn << ": "
+ << cpp_strerror(ret) << dendl;
+ goto close_fsid_fd;
+ }
+
+ assert(current_fd >= 0);
+
+ if (_detect_backend()) {
+ derr << "KeyValueStore::mount error in _detect_backend" << dendl;
+ ret = -1;
+ goto close_current_fd;
+ }
+
+ {
+ KeyValueDB *store;
+ if (kv_type == KV_TYPE_LEVELDB) {
+ store = new LevelDBStore(g_ceph_context, current_fn);
+ } else {
+ derr << "KeyValueStore::mount error: unknown backend type" << kv_type
+ << dendl;
+ ret = -1;
+ goto close_current_fd;
+ }
+
+ store->init();
+ stringstream err;
+ if (store->open(err)) {
+ derr << "KeyValueStore::mount Error initializing keyvaluestore backend: "
+ << err.str() << dendl;
+ ret = -1;
+ delete store;
+ goto close_current_fd;
+ }
+
+ StripObjectMap *dbomap = new StripObjectMap(store);
+ ret = dbomap->init(do_update);
+ if (ret < 0) {
+ delete dbomap;
+ derr << "Error initializing StripObjectMap: " << ret << dendl;
+ goto close_current_fd;
+ }
+ stringstream err2;
+
+ if (g_conf->filestore_debug_omap_check && !dbomap->check(err2)) {
+ derr << err2.str() << dendl;;
+ delete dbomap;
+ ret = -EINVAL;
+ goto close_current_fd;
+ }
+
+ backend.reset(dbomap);
+ }
+
+ op_tp.start();
+ op_finisher.start();
+ ondisk_finisher.start();
+
+ // all okay.
+ return 0;
+
+close_current_fd:
+ TEMP_FAILURE_RETRY(::close(current_fd));
+ current_fd = -1;
+close_fsid_fd:
+ TEMP_FAILURE_RETRY(::close(fsid_fd));
+ fsid_fd = -1;
+done:
+ assert(!m_fail_eio || ret != -EIO);
+ return ret;
+}
+
+int KeyValueStore::umount()
+{
+ dout(5) << "umount " << basedir << dendl;
+
+ op_tp.stop();
+ op_finisher.stop();
+ ondisk_finisher.stop();
+
+ if (fsid_fd >= 0) {
+ TEMP_FAILURE_RETRY(::close(fsid_fd));
+ fsid_fd = -1;
+ }
+ if (op_fd >= 0) {
+ TEMP_FAILURE_RETRY(::close(op_fd));
+ op_fd = -1;
+ }
+ if (current_fd >= 0) {
+ TEMP_FAILURE_RETRY(::close(current_fd));
+ current_fd = -1;
+ }
+
+ backend.reset();
+
+ // nothing
+ return 0;
+}
+
+int KeyValueStore::get_max_object_name_length()
+{
+ lock.Lock();
+ int ret = pathconf(basedir.c_str(), _PC_NAME_MAX);
+ if (ret < 0) {
+ int err = errno;
+ lock.Unlock();
+ if (err == 0)
+ return -EDOM;
+ return -err;
+ }
+ lock.Unlock();
+ return ret;
+}
+
+int KeyValueStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
+ TrackedOpRef osd_op,
+ ThreadPool::TPHandle *handle)
+{
+ Context *onreadable;
+ Context *ondisk;
+ Context *onreadable_sync;
+ ObjectStore::Transaction::collect_contexts(
+ tls, &onreadable, &ondisk, &onreadable_sync);
+
+ // set up the sequencer
+ OpSequencer *osr;
+ if (!posr)
+ posr = &default_osr;
+ if (posr->p) {
+ osr = static_cast<OpSequencer *>(posr->p);
+ dout(5) << "queue_transactions existing " << *osr << "/" << osr->parent
+ << dendl; //<< " w/ q " << osr->q << dendl;
+ } else {
+ osr = new OpSequencer;
+ osr->parent = posr;
+ posr->p = osr;
+ dout(5) << "queue_transactions new " << *osr << "/" << osr->parent << dendl;
+ }
+
+ Op *o = build_op(tls, ondisk, onreadable, onreadable_sync, osd_op);
+ uint64_t op = submit_manager.op_submit_start();
+ o->op = op;
+ dout(5) << "queue_transactions (trailing journal) " << op << " "
+ << tls <<dendl;
+ queue_op(osr, o);
+
+ submit_manager.op_submit_finish(op);
+
+ return 0;
+}
+
+
+// ============== KeyValueStore Op Handler =================
+
+KeyValueStore::Op *KeyValueStore::build_op(list<Transaction*>& tls,
+ Context *ondisk, Context *onreadable, Context *onreadable_sync,
+ TrackedOpRef osd_op)
+{
+ uint64_t bytes = 0, ops = 0;
+ for (list<Transaction*>::iterator p = tls.begin();
+ p != tls.end();
+ ++p) {
+ bytes += (*p)->get_num_bytes();
+ ops += (*p)->get_num_ops();
+ }
+
+ Op *o = new Op;
+ o->start = ceph_clock_now(g_ceph_context);
+ o->tls.swap(tls);
+ o->ondisk = ondisk;
+ o->onreadable = onreadable;
+ o->onreadable_sync = onreadable_sync;
+ o->ops = ops;
+ o->bytes = bytes;
+ o->osd_op = osd_op;
+ return o;
+}
+
+void KeyValueStore::queue_op(OpSequencer *osr, Op *o)
+{
+ // queue op on sequencer, then queue sequencer for the threadpool,
+ // so that regardless of which order the threads pick up the
+ // sequencer, the op order will be preserved.
+
+ osr->queue(o);
+
+ dout(5) << "queue_op " << o << " seq " << o->op << " " << *osr << " "
+ << o->bytes << " bytes" << " (queue has " << op_queue_len
+ << " ops and " << op_queue_bytes << " bytes)" << dendl;
+ op_wq.queue(osr);
+}
+
+void KeyValueStore::_do_op(OpSequencer *osr, ThreadPool::TPHandle &handle)
+{
+ // inject a stall?
+ if (g_conf->filestore_inject_stall) {
+ int orig = g_conf->filestore_inject_stall;
+ dout(5) << "_do_op filestore_inject_stall " << orig << ", sleeping" << dendl;
+ for (int n = 0; n < g_conf->filestore_inject_stall; n++)
+ sleep(1);
+ g_conf->set_val("filestore_inject_stall", "0");
+ dout(5) << "_do_op done stalling" << dendl;
+ }
+
+ // FIXME: Suppose the collection of transaction only affect objects in the
+ // one PG, so this lock will ensure no other concurrent write operation
+ osr->apply_lock.Lock();
+ Op *o = osr->peek_queue();
+ dout(5) << "_do_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << " start" << dendl;
+ int r = _do_transactions(o->tls, o->op, &handle);
+ dout(10) << "_do_op " << o << " seq " << o->op << " r = " << r
+ << ", finisher " << o->onreadable << " " << o->onreadable_sync << dendl;
+
+ if (o->ondisk) {
+ if (r < 0) {
+ delete o->ondisk;
+ o->ondisk = 0;
+ } else {
+ ondisk_finisher.queue(o->ondisk, r);
+ }
+ }
+}
+
+void KeyValueStore::_finish_op(OpSequencer *osr)
+{
+ Op *o = osr->dequeue();
+
+ dout(10) << "_finish_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << dendl;
+ osr->apply_lock.Unlock(); // locked in _do_op
+
+ utime_t lat = ceph_clock_now(g_ceph_context);
+ lat -= o->start;
+
+ if (o->onreadable_sync) {
+ o->onreadable_sync->complete(0);
+ }
+ op_finisher.queue(o->onreadable);
+ delete o;
+}
+
+// Combine all the ops in the same transaction using "BufferTransaction" and
+// cache the middle results in order to make visible to the following ops.
+//
+// Lock: KeyValueStore use "in_use" in GenericObjectMap to avoid concurrent
+// operation on the same object. Not sure ReadWrite lock should be applied to
+// improve concurrent performance. In the future, I'd like to remove apply_lock
+// on "osr" and introduce PG RWLock.
+int KeyValueStore::_do_transactions(list<Transaction*> &tls, uint64_t op_seq,
+ ThreadPool::TPHandle *handle)
+{
+ int r = 0;
+
+ uint64_t bytes = 0, ops = 0;
+ for (list<Transaction*>::iterator p = tls.begin();
+ p != tls.end();
+ ++p) {
+ bytes += (*p)->get_num_bytes();
+ ops += (*p)->get_num_ops();
+ }
+
+ int trans_num = 0;
+ SequencerPosition spos(op_seq, trans_num, 0);
+ BufferTransaction bt(this, spos);
+
+ for (list<Transaction*>::iterator p = tls.begin();
+ p != tls.end();
+ ++p, trans_num++) {
+ r = _do_transaction(**p, bt, spos, handle);
+ if (r < 0)
+ break;
+ if (handle)
+ handle->reset_tp_timeout();
+ }
+
+ r = bt.submit_transaction();
+ if (r < 0) {
+ assert(0 == "unexpected error"); // FIXME
+ }
+
+ return r;
+}
+
+unsigned KeyValueStore::_do_transaction(Transaction& transaction,
+ BufferTransaction &t,
+ SequencerPosition& spos,
+ ThreadPool::TPHandle *handle)
+{
+ dout(10) << "_do_transaction on " << &transaction << dendl;
+
+ Transaction::iterator i = transaction.begin();
+
+ while (i.have_op()) {
+ if (handle)
+ handle->reset_tp_timeout();
+
+ int op = i.get_op();
+ int r = 0;
+
+ switch (op) {
+ case Transaction::OP_NOP:
+ break;
+
+ case Transaction::OP_TOUCH:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _touch(cid, oid, t);
+ }
+ break;
+
+ case Transaction::OP_WRITE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t off = i.get_length();
+ uint64_t len = i.get_length();
+ bool replica = i.get_replica();
+ bufferlist bl;
+ i.get_bl(bl);
+ r = _write(cid, oid, off, len, bl, t, replica);
+ }
+ break;
+
+ case Transaction::OP_ZERO:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t off = i.get_length();
+ uint64_t len = i.get_length();
+ r = _zero(cid, oid, off, len, t);
+ }
+ break;
+
+ case Transaction::OP_TRIMCACHE:
+ {
+ i.get_cid();
+ i.get_oid();
+ i.get_length();
+ i.get_length();
+ // deprecated, no-op
+ }
+ break;
+
+ case Transaction::OP_TRUNCATE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t off = i.get_length();
+ r = _truncate(cid, oid, off, t);
+ }
+ break;
+
+ case Transaction::OP_REMOVE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _remove(cid, oid, t);
+ }
+ break;
+
+ case Transaction::OP_SETATTR:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ string name = i.get_attrname();
+ bufferlist bl;
+ i.get_bl(bl);
+ map<string, bufferptr> to_set;
+ to_set[name] = bufferptr(bl.c_str(), bl.length());
+ r = _setattrs(cid, oid, to_set, t);
+ if (r == -ENOSPC)
+ dout(0) << " ENOSPC on setxattr on " << cid << "/" << oid
+ << " name " << name << " size " << bl.length() << dendl;
+ }
+ break;
+
+ case Transaction::OP_SETATTRS:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ map<string, bufferptr> aset;
+ i.get_attrset(aset);
+ r = _setattrs(cid, oid, aset, t);
+ if (r == -ENOSPC)
+ dout(0) << " ENOSPC on setxattrs on " << cid << "/" << oid << dendl;
+ }
+ break;
+
+ case Transaction::OP_RMATTR:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ string name = i.get_attrname();
+ r = _rmattr(cid, oid, name.c_str(), t);
+ }
+ break;
+
+ case Transaction::OP_RMATTRS:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _rmattrs(cid, oid, t);
+ }
+ break;
+
+ case Transaction::OP_CLONE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ ghobject_t noid = i.get_oid();
+ r = _clone(cid, oid, noid, t);
+ }
+ break;
+
+ case Transaction::OP_CLONERANGE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ ghobject_t noid = i.get_oid();
+ uint64_t off = i.get_length();
+ uint64_t len = i.get_length();
+ r = _clone_range(cid, oid, noid, off, len, off, t);
+ }
+ break;
+
+ case Transaction::OP_CLONERANGE2:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ ghobject_t noid = i.get_oid();
+ uint64_t srcoff = i.get_length();
+ uint64_t len = i.get_length();
+ uint64_t dstoff = i.get_length();
+ r = _clone_range(cid, oid, noid, srcoff, len, dstoff, t);
+ }
+ break;
+
+ case Transaction::OP_MKCOLL:
+ {
+ coll_t cid = i.get_cid();
+ r = _create_collection(cid, t);
+ }
+ break;
+
+ case Transaction::OP_RMCOLL:
+ {
+ coll_t cid = i.get_cid();
+ r = _destroy_collection(cid, t);
+ }
+ break;
+
+ case Transaction::OP_COLL_ADD:
+ {
+ coll_t ncid = i.get_cid();
+ coll_t ocid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _collection_add(ncid, ocid, oid, t);
+ }
+ break;
+
+ case Transaction::OP_COLL_REMOVE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _remove(cid, oid, t);
+ }
+ break;
+
+ case Transaction::OP_COLL_MOVE:
+ {
+ // WARNING: this is deprecated and buggy; only here to replay old journals.
+ coll_t ocid = i.get_cid();
+ coll_t ncid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _collection_add(ocid, ncid, oid, t);
+ if (r == 0)
+ r = _remove(ocid, oid, t);
+ }
+ break;
+
+ case Transaction::OP_COLL_MOVE_RENAME:
+ {
+ coll_t oldcid = i.get_cid();
+ ghobject_t oldoid = i.get_oid();
+ coll_t newcid = i.get_cid();
+ ghobject_t newoid = i.get_oid();
+ r = _collection_move_rename(oldcid, oldoid, newcid, newoid, t);
+ }
+ break;
+
+ case Transaction::OP_COLL_SETATTR:
+ {
+ coll_t cid = i.get_cid();
+ string name = i.get_attrname();
+ bufferlist bl;
+ i.get_bl(bl);
+ r = _collection_setattr(cid, name.c_str(), bl.c_str(), bl.length(), t);
+ }
+ break;
+
+ case Transaction::OP_COLL_RMATTR:
+ {
+ coll_t cid = i.get_cid();
+ string name = i.get_attrname();
+ r = _collection_rmattr(cid, name.c_str(), t);
+ }
+ break;
+
+ case Transaction::OP_STARTSYNC:
+ {
+ start_sync();
+ break;
+ }
+
+ case Transaction::OP_COLL_RENAME:
+ {
+ coll_t cid(i.get_cid());
+ coll_t ncid(i.get_cid());
+ r = _collection_rename(cid, ncid, t);
+ }
+ break;
+
+ case Transaction::OP_OMAP_CLEAR:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ r = _omap_clear(cid, oid, t);
+ }
+ break;
+ case Transaction::OP_OMAP_SETKEYS:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ map<string, bufferlist> aset;
+ i.get_attrset(aset);
+ r = _omap_setkeys(cid, oid, aset, t);
+ }
+ break;
+ case Transaction::OP_OMAP_RMKEYS:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ set<string> keys;
+ i.get_keyset(keys);
+ r = _omap_rmkeys(cid, oid, keys, t);
+ }
+ break;
+ case Transaction::OP_OMAP_RMKEYRANGE:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ string first, last;
+ first = i.get_key();
+ last = i.get_key();
+ r = _omap_rmkeyrange(cid, oid, first, last, t);
+ }
+ break;
+ case Transaction::OP_OMAP_SETHEADER:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ bufferlist bl;
+ i.get_bl(bl);
+ r = _omap_setheader(cid, oid, bl, t);
+ }
+ break;
+ case Transaction::OP_SPLIT_COLLECTION:
+ {
+ coll_t cid(i.get_cid());
+ uint32_t bits(i.get_u32());
+ uint32_t rem(i.get_u32());
+ coll_t dest(i.get_cid());
+ r = _split_collection_create(cid, bits, rem, dest, t);
+ }
+ break;
+ case Transaction::OP_SPLIT_COLLECTION2:
+ {
+ coll_t cid(i.get_cid());
+ uint32_t bits(i.get_u32());
+ uint32_t rem(i.get_u32());
+ coll_t dest(i.get_cid());
+ r = _split_collection(cid, bits, rem, dest, t);
+ }
+ break;
+
+ default:
+ derr << "bad op " << op << dendl;
+ assert(0);
+ }
+
+ if (r < 0) {
+ bool ok = false;
+
+ if (r == -ENOENT && !(op == Transaction::OP_CLONERANGE ||
+ op == Transaction::OP_CLONE ||
+ op == Transaction::OP_CLONERANGE2))
+ // -ENOENT is normally okay
+ // ...including on a replayed OP_RMCOLL with checkpoint mode
+ ok = true;
+ if (r == -ENODATA)
+ ok = true;
+
+ if (!ok) {
+ const char *msg = "unexpected error code";
+
+ if (r == -ENOENT && (op == Transaction::OP_CLONERANGE ||
+ op == Transaction::OP_CLONE ||
+ op == Transaction::OP_CLONERANGE2))
+ msg = "ENOENT on clone suggests osd bug";
+
+ if (r == -ENOSPC)
+ // For now, if we hit _any_ ENOSPC, crash, before we do any damage
+ // by partially applying transactions.
+ msg = "ENOSPC handling not implemented";
+
+ if (r == -ENOTEMPTY) {
+ msg = "ENOTEMPTY suggests garbage data in osd data dir";
+ }
+
+ dout(0) << " error " << cpp_strerror(r) << " not handled on operation "
+ << op << " (" << spos << ", or op " << spos.op
+ << ", counting from 0)" << dendl;
+ dout(0) << msg << dendl;
+ dout(0) << " transaction dump:\n";
+ JSONFormatter f(true);
+ f.open_object_section("transaction");
+ transaction.dump(&f);
+ f.close_section();
+ f.flush(*_dout);
+ *_dout << dendl;
+ assert(0 == "unexpected error");
+
+ if (r == -EMFILE) {
+ dump_open_fds(g_ceph_context);
+ }
+ }
+ }
+
+ spos.op++;
+ }
+
+ return 0; // FIXME count errors
+}
+
+
+// =========== KeyValueStore Op Implementation ==============
+// objects
+
+int KeyValueStore::_check_coll(const coll_t &cid)
+{
+ if (is_coll_obj(cid))
+ return 0;
+
+ StripObjectMap::StripObjectHeader header;
+ int r = backend->lookup_strip_header(get_coll_for_coll(),
+ make_ghobject_for_coll(cid), header);
+ if (r < 0) {
+ dout(10) << __func__ << " could not find header r = " << r << dendl;
+ return -ENOENT;
+ }
+
+ return 0;
+}
+
+bool KeyValueStore::exists(coll_t cid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << "collection: " << cid << " object: " << oid
+ << dendl;
+ int r;
+ StripObjectMap::StripObjectHeader header;
+
+ r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->lookup_strip_header(cid, oid, header);
+ if (r < 0) {
+ return false;
+ }
+
+ return true;
+}
+
+int KeyValueStore::stat(coll_t cid, const ghobject_t& oid,
+ struct stat *st, bool allow_eio)
+{
+ dout(10) << "stat " << cid << "/" << oid << dendl;
+
+ StripObjectMap::StripObjectHeader header;
+
+ int r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->lookup_strip_header(cid, oid, header);
+ if (r < 0) {
+ dout(10) << "stat " << cid << "/" << oid << "=" << r << dendl;
+ return -ENOENT;
+ }
+
+ st->st_blocks = header.max_size / header.strip_size;
+ if (header.max_size % header.strip_size)
+ st->st_blocks++;
+ st->st_nlink = 1;
+ st->st_size = header.max_size;
+ st->st_blksize = header.strip_size;
+
+ return r;
+}
+
+int KeyValueStore::_generic_read(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len, bufferlist& bl,
+ bool allow_eio, BufferTransaction *bt)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << dendl;
+
+ int r;
+ StripObjectMap::StripObjectHeader header;
+
+ r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ // use strip_header buffer
+ if (bt) {
+ StripObjectMap::StripObjectHeader *cache_header;
+ r = bt->lookup_cached_header(cid, oid, &cache_header, false);
+ if (r == 0) {
+ header = *cache_header;
+ }
+ } else {
+ r = backend->lookup_strip_header(cid, oid, header);
+ }
+
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << " header isn't exist: r = " << r << dendl;
+ return r;
+ }
+
+ if (header.max_size < offset) {
+ r = -EINVAL;
+ dout(10) << __func__ << " " << cid << "/" << oid << ")"
+ << " offset exceed the length of bl"<< dendl;
+ return r;
+ }
+
+ if (len == 0)
+ len = header.max_size - offset;
+
+ if (offset + len > header.max_size)
+ len = header.max_size - offset;
+
+ vector<StripObjectMap::StripExtent> extents;
+ StripObjectMap::file_to_extents(offset, len, header.strip_size,
+ extents);
+ map<string, bufferlist> out;
+ set<string> keys;
+
+ for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
+ iter != extents.end(); ++iter) {
+ bufferlist old;
+ string key = strip_object_key(iter->no);
+
+ if (bt && header.buffers.count(make_pair(OBJECT_STRIP_PREFIX, key))) {
+ assert(header.bits[iter->no]);
+ out[key] = header.buffers[make_pair(OBJECT_STRIP_PREFIX, key)];
+ } else if (header.bits[iter->no]) {
+ keys.insert(key);
+ }
+ }
+
+ r = backend->get_values(cid, oid, OBJECT_STRIP_PREFIX, keys, &out);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << " = " << r << dendl;
+ return r;
+ }
+ if (out.size() != keys.size()) {
+ r = -EINVAL;
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << " get incorrect key/value pairs " << dendl;
+ return r;
+ }
+
+ uint64_t readed = 0;
+
+ for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
+ iter != extents.end(); ++iter) {
+ string key = strip_object_key(iter->no);
+ if (readed + header.strip_size > header.max_size) {
+ if (header.bits[iter->no]) {
+ out[key].copy(0, iter->len, bl);
+ } else {
+ bl.append_zero(iter->len);
+ }
+
+ break;
+ }
+
+ if (header.bits[iter->no]) {
+ bl.append(out[key]);
+ } else {
+ bl.append_zero(header.strip_size);
+ }
+ readed += header.strip_size;
+ }
+
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset
+ << "~" << bl.length() << "/" << len << " r = " << r << dendl;
+
+ return bl.length();
+}
+
+
+int KeyValueStore::read(coll_t cid, const ghobject_t& oid, uint64_t offset,
+ size_t len, bufferlist& bl, bool allow_eio)
+{
+ return _generic_read(cid, oid, offset, len, bl, allow_eio);
+}
+
+int KeyValueStore::fiemap(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len, bufferlist& bl)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << offset << "~"
+ << len << dendl;
+ int r;
+ StripObjectMap::StripObjectHeader header;
+
+ r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->lookup_strip_header(cid, oid, header);
+ if (r < 0) {
+ dout(10) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ vector<StripObjectMap::StripExtent> extents;
+ StripObjectMap::file_to_extents(offset, len, header.strip_size,
+ extents);
+
+ map<uint64_t, uint64_t> m;
+ for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
+ iter != extents.end(); ++iter) {
+ m[iter->offset] = iter->len;
+ }
+ ::encode(m, bl);
+ return 0;
+}
+
+int KeyValueStore::_remove(coll_t cid, const ghobject_t& oid,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " "
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ r = t.clear_buffer(header);
+
+ dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " size " << size
+ << dendl;
+
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << size
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ if (header->max_size == size)
+ return 0;
+
+ if (header->max_size > size) {
+ vector<StripObjectMap::StripExtent> extents;
+ StripObjectMap::file_to_extents(size, header->max_size,
+ header->strip_size, extents);
+ assert(extents.size());
+
+ vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
+ if (iter->offset != 0) {
+ bufferlist value;
+ bufferlist old;
+ map<string, bufferlist> values;
+ r = t.get_buffer_key(header, OBJECT_STRIP_PREFIX,
+ strip_object_key(iter->no), old);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " "
+ << size << " = " << r << dendl;
+ return r;
+ }
+
+ old.copy(0, iter->offset, value);
+ value.append_zero(header->strip_size-iter->offset);
+ assert(value.length() == header->strip_size);
+ ++iter;
+
+ values[strip_object_key(iter->no)] = value;
+ t.set_buffer_keys(OBJECT_STRIP_PREFIX, header, values);
+ }
+
+ set<string> keys;
+ for (; iter != extents.end(); ++iter) {
+ if (header->bits[iter->no]) {
+ keys.insert(strip_object_key(iter->no));
+ header->bits[iter->no] = 0;
+ }
+ }
+ r = t.remove_buffer_keys(OBJECT_STRIP_PREFIX, header, keys);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " "
+ << size << " = " << r << dendl;
+ return r;
+ }
+ }
+
+ header->bits.resize(size/header->strip_size+1);
+ header->max_size = size;
+
+ dout(10) << __func__ << " " << cid << "/" << oid << " size " << size << " = "
+ << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_touch(coll_t cid, const ghobject_t& oid,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, true);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " "
+ << " failed to get header: r = " << r << dendl;
+ r = -EINVAL;
+ return r;
+ }
+
+ dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_write(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len, const bufferlist& bl,
+ BufferTransaction &t, bool replica)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << dendl;
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, true);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset
+ << "~" << len << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ if (len > bl.length())
+ len = bl.length();
+
+ if (len + offset > header->max_size) {
+ header->max_size = len + offset;
+ header->bits.resize(header->max_size/header->strip_size+1);
+ }
+
+ vector<StripObjectMap::StripExtent> extents;
+ StripObjectMap::file_to_extents(offset, len, header->strip_size,
+ extents);
+ uint64_t bl_offset = 0;
+ map<string, bufferlist> values;
+ for (vector<StripObjectMap::StripExtent>::iterator iter = extents.begin();
+ iter != extents.end(); ++iter) {
+ bufferlist value;
+ string key = strip_object_key(iter->no);
+ if (header->bits[iter->no]) {
+ if (iter->offset == 0 && iter->len == header->strip_size) {
+ bl.copy(bl_offset, iter->len, value);
+ bl_offset += iter->len;
+ } else {
+ bufferlist old;
+ r = t.get_buffer_key(header, OBJECT_STRIP_PREFIX, key, old);
+ if (r < 0) {
+ dout(10) << __func__ << " failed to get value " << cid << "/" << oid
+ << " " << offset << "~" << len << " = " << r << dendl;
+ return r;
+ }
+
+ old.copy(0, iter->offset, value);
+ bl.copy(bl_offset, iter->len, value);
+ bl_offset += iter->len;
+
+ if (value.length() != header->strip_size)
+ old.copy(value.length(), header->strip_size-value.length(), value);
+ }
+ } else {
+ if (iter->offset)
+ value.append_zero(iter->offset);
+ bl.copy(bl_offset, iter->len, value);
+ bl_offset += iter->len;
+
+ if (value.length() < header->strip_size)
+ value.append_zero(header->strip_size-value.length());
+
+ header->bits[iter->no] = 1;
+ }
+ assert(value.length() == header->strip_size);
+ values[key].swap(value);
+ }
+ assert(bl_offset == len);
+
+ t.set_buffer_keys(OBJECT_STRIP_PREFIX, header, values);
+ dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~" << len
+ << " = " << r << dendl;
+
+ return r;
+}
+
+int KeyValueStore::_zero(coll_t cid, const ghobject_t& oid, uint64_t offset,
+ size_t len, BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+
+ bufferptr bp(len);
+ bp.zero();
+ bufferlist bl;
+ bl.push_back(bp);
+ int r = _write(cid, oid, offset, len, bl, t);
+
+ dout(20) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+ << len << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_clone(coll_t cid, const ghobject_t& oldoid,
+ const ghobject_t& newoid, BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << dendl;
+
+ if (oldoid == newoid)
+ return 0;
+
+ int r;
+ StripObjectMap::StripObjectHeader *old_header;
+
+ r = t.lookup_cached_header(cid, oldoid, &old_header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << " = " << r << dendl;
+ return r;
+ }
+
+ t.clone_buffer(old_header, cid, newoid);
+
+ dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_clone_range(coll_t cid, const ghobject_t& oldoid,
+ const ghobject_t& newoid, uint64_t srcoff,
+ uint64_t len, uint64_t dstoff,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << " " << srcoff << "~" << len << " to " << dstoff
+ << dendl;
+
+ int r;
+ bufferlist bl;
+
+ r = _generic_read(cid, oldoid, srcoff, len, bl, &t);
+ if (r < 0)
+ goto out;
+
+ r = _write(cid, newoid, dstoff, len, bl, t);
+
+ out:
+ dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+ << newoid << " " << srcoff << "~" << len << " to " << dstoff
+ << " = " << r << dendl;
+ return r;
+}
+
+// attrs
+
+int KeyValueStore::getattr(coll_t cid, const ghobject_t& oid, const char *name,
+ bufferptr &bp)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " '" << name << "'"
+ << dendl;
+
+ int r;
+ map<string, bufferlist> got;
+ set<string> to_get;
+
+ r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ to_get.insert(string(name));
+ r = backend->get_values(cid, oid, OBJECT_XATTR, to_get, &got);
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << __func__ << " get_xattrs err r =" << r << dendl;
+ goto out;
+ }
+ if (got.empty()) {
+ dout(10) << __func__ << " got.size() is 0" << dendl;
+ return -ENODATA;
+ }
+ bp = bufferptr(got.begin()->second.c_str(),
+ got.begin()->second.length());
+ r = 0;
+
+ out:
+ dout(10) << __func__ << " " << cid << "/" << oid << " '" << name << "' = "
+ << r << dendl;
+ return r;
+}
+
+int KeyValueStore::getattrs(coll_t cid, const ghobject_t& oid,
+ map<string,bufferptr>& aset, bool user_only)
+{
+ int r;
+ map<string, bufferlist> attr_aset;
+
+ r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->get(cid, oid, OBJECT_XATTR, &attr_aset);
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << __func__ << " could not get attrs r = " << r << dendl;
+ goto out;
+ }
+
+ if (r == -ENOENT)
+ r = 0;
+
+ for (map<string, bufferlist>::iterator i = attr_aset.begin();
+ i != attr_aset.end(); ++i) {
+ string key;
+ if (user_only) {
+ if (i->first[0] != '_')
+ continue;
+ if (i->first == "_")
+ continue;
+ key = i->first.substr(1, i->first.size());
+ } else {
+ key = i->first;
+ }
+ aset.insert(make_pair(key,
+ bufferptr(i->second.c_str(), i->second.length())));
+ }
+
+ out:
+ dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+
+ return r;
+}
+
+int KeyValueStore::_setattrs(coll_t cid, const ghobject_t& oid,
+ map<string, bufferptr>& aset,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+ int r;
+
+ StripObjectMap::StripObjectHeader *header;
+ map<string, bufferlist> attrs;
+
+ r = t.lookup_cached_header(cid, oid, &header, false);
+ if (r < 0)
+ goto out;
+
+ for (map<string, bufferptr>::iterator it = aset.begin();
+ it != aset.end(); ++it) {
+ attrs[it->first].push_back(it->second);
+ }
+
+ t.set_buffer_keys(OBJECT_XATTR, header, attrs);
+
+out:
+ dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+ return r;
+}
+
+
+int KeyValueStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << " '" << name << "'"
+ << dendl;
+
+ int r;
+ set<string> to_remove;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " could not find header r = " << r
+ << dendl;
+ return r;
+ }
+
+ to_remove.insert(string(name));
+ r = t.remove_buffer_keys(OBJECT_XATTR, header, to_remove);
+
+ dout(10) << __func__ << " " << cid << "/" << oid << " '" << name << "' = "
+ << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_rmattrs(coll_t cid, const ghobject_t& oid,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+ int r;
+ set<string> attrs;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(cid, oid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " could not find header r = " << r
+ << dendl;
+ return r;
+ }
+
+ r = backend->get_keys(cid, oid, OBJECT_XATTR, &attrs);
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << __func__ << " could not get attrs r = " << r << dendl;
+ assert(!m_fail_eio || r != -EIO);
+ return r;
+ }
+
+ r = t.remove_buffer_keys(OBJECT_XATTR, header, attrs);
+ t.clear_buffer_keys(OBJECT_XATTR, header);
+
+ dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+ return r;
+}
+
+// collection attrs
+
+int KeyValueStore::collection_getattr(coll_t c, const char *name,
+ void *value, size_t size)
+{
+ dout(15) << __func__ << " " << c.to_str() << " '" << name << "' len "
+ << size << dendl;
+
+ bufferlist bl;
+ int r;
+
+ r = collection_getattr(c, name, bl);
+ if (r < 0)
+ goto out;
+
+ if (bl.length() < size) {
+ r = bl.length();
+ bl.copy(0, bl.length(), static_cast<char*>(value));
+ } else {
+ r = size;
+ bl.copy(0, size, static_cast<char*>(value));
+ }
+
+out:
+ dout(10) << __func__ << " " << c.to_str() << " '" << name << "' len "
+ << size << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::collection_getattr(coll_t c, const char *name,
+ bufferlist& bl)
+{
+ dout(15) << __func__ << " " << c.to_str() << " '" << name
+ << "'" << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ set<string> keys;
+ map<string, bufferlist> out;
+ keys.insert(string(name));
+
+ r = backend->get_values(get_coll_for_coll(), make_ghobject_for_coll(c),
+ COLLECTION_ATTR, keys, &out);
+ if (r < 0) {
+ dout(10) << __func__ << " could not get key" << string(name) << dendl;
+ r = -EINVAL;
+ }
+
+ assert(out.size());
+ bl.swap(out.begin()->second);
+
+ dout(10) << __func__ << " " << c.to_str() << " '" << name << "' len "
+ << bl.length() << " = " << r << dendl;
+ return bl.length();
+}
+
+int KeyValueStore::collection_getattrs(coll_t cid,
+ map<string, bufferptr> &aset)
+{
+ dout(10) << __func__ << " " << cid.to_str() << dendl;
+
+ int r = _check_coll(cid);
+ if (r < 0) {
+ return r;
+ }
+
+ map<string, bufferlist> out;
+ set<string> keys;
+
+ for (map<string, bufferptr>::iterator it = aset.begin();
+ it != aset.end(); it++) {
+ keys.insert(it->first);
+ }
+
+ r = backend->get_values(get_coll_for_coll(), make_ghobject_for_coll(cid),
+ COLLECTION_ATTR, keys, &out);
+ if (r < 0) {
+ dout(10) << __func__ << " could not get keys" << dendl;
+ r = -EINVAL;
+ goto out;
+ }
+
+ for (map<string, bufferlist>::iterator it = out.begin(); it != out.end();
+ ++it) {
+ bufferptr ptr(it->second.c_str(), it->second.length());
+ aset.insert(make_pair(it->first, ptr));
+ }
+
+ out:
+ dout(10) << __func__ << " " << cid.to_str() << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_collection_setattr(coll_t c, const char *name,
+ const void *value, size_t size,
+ BufferTransaction &t)
+{
+ dout(10) << __func__ << " " << c << " '" << name << "' len "
+ << size << dendl;
+
+ int r;
+ bufferlist bl;
+ map<string, bufferlist> out;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(c),
+ &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " could not find header r = " << r << dendl;
+ return r;
+ }
+
+ bl.append(reinterpret_cast<const char*>(value), size);
+ out.insert(make_pair(string(name), bl));
+
+ t.set_buffer_keys(COLLECTION_ATTR, header, out);
+
+ dout(10) << __func__ << " " << c << " '"
+ << name << "' len " << size << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_collection_rmattr(coll_t c, const char *name,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << c << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ bufferlist bl;
+ set<string> out;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(c),
+ &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " could not find header r = " << r << dendl;
+ return r;
+ }
+
+ out.insert(string(name));
+ r = t.remove_buffer_keys(COLLECTION_ATTR, header, out);
+
+ dout(10) << __func__ << " " << c << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_collection_setattrs(coll_t cid,
+ map<string,bufferptr>& aset,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << dendl;
+
+ map<string, bufferlist> attrs;
+ StripObjectMap::StripObjectHeader *header;
+ int r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(cid),
+ &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " could not find header r = " << r << dendl;
+ return r;
+ }
+
+ for (map<string, bufferptr>::iterator it = aset.begin(); it != aset.end();
+ ++it) {
+ attrs[it->first].push_back(it->second);
+ }
+
+ t.set_buffer_keys(COLLECTION_ATTR, header, attrs);
+
+ dout(10) << __func__ << " " << cid << " = " << r << dendl;
+ return r;
+}
+
+
+// collections
+
+int KeyValueStore::_create_collection(coll_t c, BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << c << dendl;
+
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+ bufferlist bl;
+
+ r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(c), &header,
+ false);
+ if (r == 0) {
+ r = -EEXIST;
+ return r;
+ }
+
+ r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(c), &header,
+ true);
+
+ dout(10) << __func__ << " cid " << c << " r = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_destroy_collection(coll_t c, BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << c << dendl;
+
+ int r;
+ uint64_t modified_object = 0;
+ StripObjectMap::StripObjectHeader *header;
+ vector<ghobject_t> oids;
+
+ r = t.lookup_cached_header(get_coll_for_coll(), make_ghobject_for_coll(c),
+ &header, false);
+ if (r < 0) {
+ goto out;
+ }
+
+ // All modified objects are marked deleted
+ for (BufferTransaction::StripHeaderMap::iterator iter = t.strip_headers.begin();
+ iter != t.strip_headers.end(); iter++) {
+ // sum the total modified object in this PG
+ if (iter->first.first != c)
+ continue;
+
+ modified_object++;
+ if (!iter->second.deleted) {
+ r = -ENOTEMPTY;
+ goto out;
+ }
+ }
+
+ r = backend->list_objects(c, ghobject_t(), modified_object+1, &oids,
+ 0);
+ // No other object
+ if (oids.size() != modified_object && oids.size() != 0) {
+ r = -ENOTEMPTY;
+ goto out;
+ }
+
+ for(vector<ghobject_t>::iterator iter = oids.begin();
+ iter != oids.end(); ++iter) {
+ if (!t.strip_headers.count(make_pair(c, *iter))) {
+ r = -ENOTEMPTY;
+ goto out;
+ }
+ }
+
+ r = t.clear_buffer(header);
+
+out:
+ dout(10) << __func__ << " " << c << " = " << r << dendl;
+ return r;
+}
+
+
+int KeyValueStore::_collection_add(coll_t c, coll_t oldcid,
+ const ghobject_t& o,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+ << o << dendl;
+
+ bufferlist bl;
+ StripObjectMap::StripObjectHeader *header, *old_header;
+
+ int r = t.lookup_cached_header(oldcid, o, &old_header, false);
+ if (r < 0) {
+ goto out;
+ }
+
+ r = t.lookup_cached_header(c, o, &header, false);
+ if (r == 0) {
+ r = -EEXIST;
+ dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+ << o << " already exist " << dendl;
+ goto out;
+ }
+
+ r = _generic_read(oldcid, o, 0, old_header->max_size, bl, &t);
+ if (r < 0) {
+ r = -EINVAL;
+ goto out;
+ }
+
+ r = _write(c, o, 0, bl.length(), bl, t);
+ if (r < 0) {
+ r = -EINVAL;
+ }
+
+out:
+ dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+ << o << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_collection_move_rename(coll_t oldcid,
+ const ghobject_t& oldoid,
+ coll_t c, const ghobject_t& o,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+ << oldoid << dendl;
+ int r;
+ StripObjectMap::StripObjectHeader *header;
+
+ r = t.lookup_cached_header(c, o, &header, false);
+ if (r == 0) {
+ dout(10) << __func__ << " " << oldcid << "/" << oldoid << " -> " << c
+ << "/" << o << " = " << r << dendl;
+ return -EEXIST;
+ }
+
+ r = t.lookup_cached_header(oldcid, oldoid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << oldcid << "/" << oldoid << " -> " << c
+ << "/" << o << " = " << r << dendl;
+ return r;
+ }
+
+ t.rename_buffer(header, c, o);
+
+ dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+ << oldoid << " = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_collection_remove_recursive(const coll_t &cid,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << dendl;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(cid),
+ &header, false);
+ if (r < 0) {
+ return 0;
+ }
+
+ vector<ghobject_t> objects;
+ ghobject_t max;
+ while (!max.is_max()) {
+ r = collection_list_partial(cid, max, 200, 300, 0, &objects, &max);
+ if (r < 0)
+ return r;
+
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end(); ++i) {
+ r = _remove(cid, *i, t);
+
+ if (r < 0)
+ return r;
+ }
+ }
+
+ r = t.clear_buffer(header);
+
+ dout(10) << __func__ << " " << cid << " r = " << r << dendl;
+ return 0;
+}
+
+int KeyValueStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
+ BufferTransaction &t)
+{
+ dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
+ << dendl;
+ return -EOPNOTSUPP;
+}
+
+int KeyValueStore::list_collections(vector<coll_t>& ls)
+{
+ dout(10) << __func__ << " " << dendl;
+
+ vector<ghobject_t> oids;
+ ghobject_t next;
+ backend->list_objects(get_coll_for_coll(), ghobject_t(), 0, &oids, &next);
+ assert(next == ghobject_t::get_max());
+
+ for (vector<ghobject_t>::const_iterator iter = oids.begin();
+ iter != oids.end(); ++iter) {
+ ls.push_back(coll_t(iter->hobj.oid.name));
+ }
+
+ return 0;
+}
+
+bool KeyValueStore::collection_exists(coll_t c)
+{
+ dout(10) << __func__ << " " << dendl;
+
+ StripObjectMap::StripObjectHeader header;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return false;
+ }
+ return true;
+}
+
+bool KeyValueStore::collection_empty(coll_t c)
+{
+ dout(10) << __func__ << " " << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return false;
+ }
+
+ vector<ghobject_t> oids;
+ backend->list_objects(c, ghobject_t(), 1, &oids, 0);
+
+ return oids.empty();
+}
+
+int KeyValueStore::collection_list_range(coll_t c, ghobject_t start,
+ ghobject_t end, snapid_t seq,
+ vector<ghobject_t> *ls)
+{
+ bool done = false;
+ ghobject_t next = start;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ while (!done) {
+ vector<ghobject_t> next_objects;
+ r = collection_list_partial(c, next, get_ideal_list_min(),
+ get_ideal_list_max(), seq,
+ &next_objects, &next);
+ if (r < 0)
+ return r;
+
+ ls->insert(ls->end(), next_objects.begin(), next_objects.end());
+
+ // special case for empty collection
+ if (ls->empty()) {
+ break;
+ }
+
+ while (!ls->empty() && ls->back() >= end) {
+ ls->pop_back();
+ done = true;
+ }
+
+ if (next >= end) {
+ done = true;
+ }
+ }
+
+ return 0;
+}
+
+int KeyValueStore::collection_list_partial(coll_t c, ghobject_t start,
+ int min, int max, snapid_t seq,
+ vector<ghobject_t> *ls,
+ ghobject_t *next)
+{
+ dout(10) << __func__ << " " << c << " start:" << start << " is_max:"
+ << start.is_max() << dendl;
+
+ if (min < 0 || max < 0)
+ return -EINVAL;
+
+ if (start.is_max())
+ return 0;
+
+ return backend->list_objects(c, start, max, ls, next);
+}
+
+int KeyValueStore::collection_list(coll_t c, vector<ghobject_t>& ls)
+{
+ return collection_list_partial(c, ghobject_t(), 0, 0, 0, &ls, 0);
+}
+
+int KeyValueStore::collection_version_current(coll_t c, uint32_t *version)
+{
+ *version = COLLECTION_VERSION;
+ if (*version == target_version)
+ return 1;
+ else
+ return 0;
+}
+
+// omap
+
+int KeyValueStore::omap_get(coll_t c, const ghobject_t &hoid,
+ bufferlist *header, map<string, bufferlist> *out)
+{
+ dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->get(c, hoid, OBJECT_OMAP, out);
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << __func__ << " err r =" << r << dendl;
+ return r;
+ }
+
+ return omap_get_header(c, hoid, header, false);
+}
+
+int KeyValueStore::omap_get_header(coll_t c, const ghobject_t &hoid,
+ bufferlist *bl, bool allow_eio)
+{
+ dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ set<string> keys;
+ map<string, bufferlist> got;
+
+ keys.insert(OBJECT_OMAP_HEADER_KEY);
+ r = backend->get_values(c, hoid, OBJECT_OMAP_HEADER, keys, &got);
+ if (r < 0 && r != -ENOENT) {
+ assert(allow_eio || !m_fail_eio || r != -EIO);
+ dout(10) << __func__ << " err r =" << r << dendl;
+ return r;
+ }
+
+ if (got.size()) {
+ assert(got.size() == 1);
+ bl->swap(got.begin()->second);
+ }
+
+ return 0;
+}
+
+int KeyValueStore::omap_get_keys(coll_t c, const ghobject_t &hoid, set<string> *keys)
+{
+ dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->get_keys(c, hoid, OBJECT_OMAP, keys);
+ if (r < 0 && r != -ENOENT) {
+ assert(!m_fail_eio || r != -EIO);
+ return r;
+ }
+ return 0;
+}
+
+int KeyValueStore::omap_get_values(coll_t c, const ghobject_t &hoid,
+ const set<string> &keys,
+ map<string, bufferlist> *out)
+{
+ dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->get_values(c, hoid, OBJECT_OMAP, keys, out);
+ if (r < 0 && r != -ENOENT) {
+ assert(!m_fail_eio || r != -EIO);
+ return r;
+ }
+ return 0;
+}
+
+int KeyValueStore::omap_check_keys(coll_t c, const ghobject_t &hoid,
+ const set<string> &keys, set<string> *out)
+{
+ dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+ int r = _check_coll(c);
+ if (r < 0) {
+ return r;
+ }
+
+ r = backend->check_keys(c, hoid, OBJECT_OMAP, keys, out);
+ if (r < 0 && r != -ENOENT) {
+ assert(!m_fail_eio || r != -EIO);
+ return r;
+ }
+ return 0;
+}
+
+ObjectMap::ObjectMapIterator KeyValueStore::get_omap_iterator(
+ coll_t c, const ghobject_t &hoid)
+{
+ dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+ return backend->get_iterator(c, hoid, OBJECT_OMAP);
+}
+
+int KeyValueStore::_omap_clear(coll_t cid, const ghobject_t &hoid,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(cid, hoid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << hoid << " "
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ set<string> keys;
+ r = backend->get_keys(cid, hoid, OBJECT_OMAP, &keys);
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << __func__ << " could not get omap_keys r = " << r << dendl;
+ assert(!m_fail_eio || r != -EIO);
+ return r;
+ }
+
+ r = t.remove_buffer_keys(OBJECT_OMAP, header, keys);
+ if (r < 0) {
+ dout(10) << __func__ << " could not remove keys r = " << r << dendl;
+ return r;
+ }
+
+ keys.clear();
+ keys.insert(OBJECT_OMAP_HEADER_KEY);
+ r = t.remove_buffer_keys(OBJECT_OMAP_HEADER, header, keys);
+ if (r < 0) {
+ dout(10) << __func__ << " could not remove keys r = " << r << dendl;
+ return r;
+ }
+
+ t.clear_buffer_keys(OBJECT_OMAP_HEADER, header);
+
+ dout(10) << __func__ << " " << cid << "/" << hoid << " r = " << r << dendl;
+ return 0;
+}
+
+int KeyValueStore::_omap_setkeys(coll_t cid, const ghobject_t &hoid,
+ map<string, bufferlist> &aset,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(cid, hoid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << hoid << " "
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ t.set_buffer_keys(OBJECT_OMAP, header, aset);
+
+ return 0;
+}
+
+int KeyValueStore::_omap_rmkeys(coll_t cid, const ghobject_t &hoid,
+ const set<string> &keys,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(cid, hoid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << hoid << " "
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ r = t.remove_buffer_keys(OBJECT_OMAP, header, keys);
+
+ dout(10) << __func__ << " " << cid << "/" << hoid << " r = " << r << dendl;
+ return r;
+}
+
+int KeyValueStore::_omap_rmkeyrange(coll_t cid, const ghobject_t &hoid,
+ const string& first, const string& last,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << hoid << " [" << first << ","
+ << last << "]" << dendl;
+
+ set<string> keys;
+ {
+ ObjectMap::ObjectMapIterator iter = get_omap_iterator(cid, hoid);
+ if (!iter)
+ return -ENOENT;
+
+ for (iter->lower_bound(first); iter->valid() && iter->key() < last;
+ iter->next()) {
+ keys.insert(iter->key());
+ }
+ }
+ return _omap_rmkeys(cid, hoid, keys, t);
+}
+
+int KeyValueStore::_omap_setheader(coll_t cid, const ghobject_t &hoid,
+ const bufferlist &bl,
+ BufferTransaction &t)
+{
+ dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+ map<string, bufferlist> sets;
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(cid, hoid, &header, false);
+ if (r < 0) {
+ dout(10) << __func__ << " " << cid << "/" << hoid << " "
+ << " failed to get header: r = " << r << dendl;
+ return r;
+ }
+
+ sets[OBJECT_OMAP_HEADER_KEY] = bl;
+ t.set_buffer_keys(OBJECT_OMAP_HEADER, header, sets);
+ return 0;
+}
+
+int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
+ coll_t dest, BufferTransaction &t)
+{
+ {
+ dout(15) << __func__ << " " << cid << " bits: " << bits << dendl;
+
+ StripObjectMap::StripObjectHeader *header;
+
+ int r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(cid),
+ &header, false);
+ if (r < 0) {
+ dout(2) << __func__ << ": " << cid << " DNE" << dendl;
+ return 0;
+ }
+
+ r = t.lookup_cached_header(get_coll_for_coll(),
+ make_ghobject_for_coll(dest),
+ &header, false);
+ if (r < 0) {
+ dout(2) << __func__ << ": " << dest << " DNE" << dendl;
+ return 0;
+ }
+
+ vector<ghobject_t> objects;
+ ghobject_t next, current;
+ int move_size = 0;
+ while (1) {
+ collection_list_partial(cid, current, get_ideal_list_min(),
+ get_ideal_list_max(), 0, &objects, &next);
+
+ dout(20) << __func__ << cid << "objects size: " << objects.size()
+ << dendl;
+
+ if (objects.empty())
+ break;
+
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end(); ++i) {
+ if (i->match(bits, rem)) {
+ if (_collection_add(dest, cid, *i, t) < 0) {
+ return -1;
+ }
+ _remove(cid, *i, t);
+ move_size++;
+ }
+ }
+
+ objects.clear();
+ current = next;
+ }
+
+ dout(20) << __func__ << "move" << move_size << " object from " << cid
+ << "to " << dest << dendl;
+ }
+
+ if (g_conf->filestore_debug_verify_split) {
+ vector<ghobject_t> objects;
+ ghobject_t next;
+ while (1) {
+ collection_list_partial(cid, next, get_ideal_list_min(),
+ get_ideal_list_max(), 0, &objects, &next);
+ if (objects.empty())
+ break;
+
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end(); ++i) {
+ dout(20) << __func__ << ": " << *i << " still in source "
+ << cid << dendl;
+ assert(!i->match(bits, rem));
+ }
+ objects.clear();
+ }
+
+ next = ghobject_t();
+ while (1) {
+ collection_list_partial(dest, next, get_ideal_list_min(),
+ get_ideal_list_max(), 0, &objects, &next);
+ if (objects.empty())
+ break;
+
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end(); ++i) {
+ dout(20) << __func__ << ": " << *i << " now in dest "
+ << *i << dendl;
+ assert(i->match(bits, rem));
+ }
+ objects.clear();
+ }
+ }
+ return 0;
+}
+
+const char** KeyValueStore::get_tracked_conf_keys() const
+{
+ static const char* KEYS[] = {
+ "filestore_min_sync_interval",
+ "filestore_max_sync_interval",
+ "filestore_queue_max_ops",
+ "filestore_queue_max_bytes",
+ "filestore_queue_committing_max_ops",
+ "filestore_queue_committing_max_bytes",
+ "filestore_commit_timeout",
+ "filestore_dump_file",
+ "filestore_kill_at",
+ "filestore_fail_eio",
+ "filestore_replica_fadvise",
+ "filestore_sloppy_crc",
+ "filestore_sloppy_crc_block_size",
+ NULL
+ };
+ return KEYS;
+}
+
+void KeyValueStore::handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed)
+{
+}
+
+void KeyValueStore::dump_transactions(list<ObjectStore::Transaction*>& ls, uint64_t seq, OpSequencer *osr)
+{
+}
+
+// ============== KeyValueStore Debug EIO Injection =================
+
+void KeyValueStore::inject_data_error(const ghobject_t &oid) {
+ Mutex::Locker l(read_error_lock);
+ dout(10) << __func__ << ": init error on " << oid << dendl;
+ data_error_set.insert(oid);
+}
+
+void KeyValueStore::inject_mdata_error(const ghobject_t &oid) {
+ Mutex::Locker l(read_error_lock);
+ dout(10) << __func__ << ": init error on " << oid << dendl;
+ mdata_error_set.insert(oid);
+}
+
+void KeyValueStore::debug_obj_on_delete(const ghobject_t &oid) {
+ Mutex::Locker l(read_error_lock);
+ dout(10) << __func__ << ": clear error on " << oid << dendl;
+ data_error_set.erase(oid);
+ mdata_error_set.erase(oid);
+}
+
+bool KeyValueStore::debug_data_eio(const ghobject_t &oid) {
+ Mutex::Locker l(read_error_lock);
+ if (data_error_set.count(oid)) {
+ dout(10) << __func__ << ": inject error on " << oid << dendl;
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool KeyValueStore::debug_mdata_eio(const ghobject_t &oid) {
+ Mutex::Locker l(read_error_lock);
+ if (mdata_error_set.count(oid)) {
+ dout(10) << __func__ << ": inject error on " << oid << dendl;
+ return true;
+ } else {
+ return false;
+ }
+}
diff --git a/src/os/KeyValueStore.h b/src/os/KeyValueStore.h
new file mode 100644
index 0000000..5a1d0f0
--- /dev/null
+++ b/src/os/KeyValueStore.h
@@ -0,0 +1,594 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack <haomai at unitedstack.com>
+ *
+ * Author: Haomai Wang <haomaiwang at gmail.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+
+#ifndef CEPH_KEYVALUESTORE_H
+#define CEPH_KEYVALUESTORE_H
+
+#include "include/types.h"
+
+#include <map>
+#include <deque>
+#include <boost/scoped_ptr.hpp>
+#include <fstream>
+using namespace std;
+
+#include <ext/hash_map>
+using namespace __gnu_cxx;
+
+#include "include/assert.h"
+
+#include "ObjectStore.h"
+
+#include "common/WorkQueue.h"
+#include "common/Finisher.h"
+#include "common/fd.h"
+
+#include "common/Mutex.h"
+#include "GenericObjectMap.h"
+#include "SequencerPosition.h"
+#include "KeyValueDB.h"
+
+#include "include/uuid.h"
+
+enum kvstore_types {
+ KV_TYPE_NONE = 0,
+ KV_TYPE_LEVELDB,
+ KV_TYPE_OTHER
+};
+
+
+class StripObjectMap: public GenericObjectMap {
+ public:
+
+ struct StripExtent {
+ uint64_t no;
+ uint64_t offset; // in key
+ uint64_t len; // in key
+ StripExtent(uint64_t n, uint64_t off, size_t len):
+ no(n), offset(off), len(len) {}
+ };
+
+ // -- strip object --
+ struct StripObjectHeader {
+ // Persistent state
+ uint64_t strip_size;
+ uint64_t max_size;
+ vector<char> bits;
+ SequencerPosition spos;
+
+ // soft state
+ Header header; // FIXME: Hold lock to avoid concurrent operations, it will
+ // also block read operation which not should be permitted.
+ coll_t cid;
+ ghobject_t oid;
+ bool deleted;
+ map<pair<string, string>, bufferlist> buffers; // pair(prefix, key)
+
+ StripObjectHeader(): strip_size(default_strip_size), max_size(0), deleted(false) {}
+
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(strip_size, bl);
+ ::encode(max_size, bl);
+ ::encode(bits, bl);
+ ::encode(spos, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(strip_size, bl);
+ ::decode(max_size, bl);
+ ::decode(bits, bl);
+ ::decode(spos, bl);
+ DECODE_FINISH(bl);
+ }
+ };
+
+ bool check_spos(const StripObjectHeader &header,
+ const SequencerPosition *spos);
+ void sync_wrap(StripObjectHeader &strip_header, KeyValueDB::Transaction t,
+ const SequencerPosition &spos);
+
+ static int file_to_extents(uint64_t offset, size_t len, uint64_t strip_size,
+ vector<StripExtent> &extents);
+ int lookup_strip_header(const coll_t & cid, const ghobject_t &oid,
+ StripObjectHeader &header);
+ int save_strip_header(StripObjectHeader &header, KeyValueDB::Transaction t);
+ int create_strip_header(const coll_t &cid, const ghobject_t &oid,
+ StripObjectHeader &strip_header,
+ KeyValueDB::Transaction t);
+ void clone_wrap(StripObjectHeader &old_header,
+ const coll_t &cid, const ghobject_t &oid,
+ KeyValueDB::Transaction t,
+ const SequencerPosition &spos,
+ StripObjectHeader *origin_header,
+ StripObjectHeader *target_header);
+ void rename_wrap(const coll_t &cid, const ghobject_t &oid,
+ KeyValueDB::Transaction t,
+ const SequencerPosition &spos,
+ StripObjectHeader *header);
+
+
+ StripObjectMap(KeyValueDB *db): GenericObjectMap(db) {}
+
+ static const uint64_t default_strip_size = 1024;
+};
+
+
+class KeyValueStore : public ObjectStore,
+ public md_config_obs_t {
+ public:
+ objectstore_perf_stat_t get_cur_stats() {
+ objectstore_perf_stat_t ret;
+ return ret;
+ }
+
+ static const uint32_t target_version = 1;
+
+ private:
+ string internal_name; // internal name, used to name the perfcounter instance
+ string basedir;
+ std::string current_fn;
+ std::string current_op_seq_fn;
+ uuid_d fsid;
+
+ int fsid_fd, op_fd, current_fd;
+
+ enum kvstore_types kv_type;
+
+ deque<uint64_t> snaps;
+
+ // ObjectMap
+ boost::scoped_ptr<StripObjectMap> backend;
+
+ Finisher ondisk_finisher;
+
+ Mutex lock;
+
+ int _create_current();
+
+ /// read a uuid from fd
+ int read_fsid(int fd, uuid_d *uuid);
+
+ /// lock fsid_fd
+ int lock_fsid();
+
+ string strip_object_key(uint64_t no) {
+ char n[100];
+ snprintf(n, 100, "%lld", (long long)no);
+ return string(n);
+ }
+
+ // A special coll used by store collection info, each obj in this coll
+ // represent a coll_t
+ static bool is_coll_obj(coll_t c) {
+ return c == coll_t("COLLECTIONS");
+ }
+ static coll_t get_coll_for_coll() {
+ return coll_t("COLLECTIONS");
+ }
+ static ghobject_t make_ghobject_for_coll(const coll_t &col) {
+ return ghobject_t(hobject_t(sobject_t(col.to_str(), CEPH_NOSNAP)));
+ }
+
+ // Each transaction has side effect which may influent the following
+ // operations, we need to make it visible for the following within
+ // transaction by caching middle result.
+ // Side effects contains:
+ // 1. Creating/Deleting collection
+ // 2. Creating/Deleting object
+ // 3. Object modify(including omap, xattr)
+ // 4. Clone or rename
+ struct BufferTransaction {
+ typedef pair<coll_t, ghobject_t> uniq_id;
+ typedef map<uniq_id, StripObjectMap::StripObjectHeader> StripHeaderMap;
+
+ //Dirty records
+ StripHeaderMap strip_headers;
+
+ KeyValueStore *store;
+
+ SequencerPosition spos;
+ KeyValueDB::Transaction t;
+
+ int check_coll(const coll_t &cid);
+ int lookup_cached_header(const coll_t &cid, const ghobject_t &oid,
+ StripObjectMap::StripObjectHeader **strip_header,
+ bool create_if_missing);
+ int get_buffer_key(StripObjectMap::StripObjectHeader *strip_header,
+ const string &prefix, const string &key,
+ bufferlist &out);
+ void set_buffer_keys(const string &prefix,
+ StripObjectMap::StripObjectHeader *strip_header,
+ map<string, bufferlist> &bl);
+ int remove_buffer_keys(const string &prefix,
+ StripObjectMap::StripObjectHeader *strip_header,
+ const set<string> &keys);
+ void clear_buffer_keys(const string &prefix,
+ StripObjectMap::StripObjectHeader *strip_header);
+ int clear_buffer(StripObjectMap::StripObjectHeader *strip_header);
+ void clone_buffer(StripObjectMap::StripObjectHeader *old_header,
+ const coll_t &cid, const ghobject_t &oid);
+ void rename_buffer(StripObjectMap::StripObjectHeader *old_header,
+ const coll_t &cid, const ghobject_t &oid);
+ int submit_transaction();
+
+ BufferTransaction(KeyValueStore *store,
+ SequencerPosition &spos): store(store), spos(spos) {
+ t = store->backend->get_transaction();
+ }
+ };
+
+ // -- op workqueue --
+ struct Op {
+ utime_t start;
+ uint64_t op;
+ list<Transaction*> tls;
+ Context *ondisk, *onreadable, *onreadable_sync;
+ uint64_t ops, bytes;
+ TrackedOpRef osd_op;
+ };
+ class OpSequencer : public Sequencer_impl {
+ Mutex qlock; // to protect q, for benefit of flush (peek/dequeue also protected by lock)
+ list<Op*> q;
+ list<uint64_t> jq;
+ Cond cond;
+ public:
+ Sequencer *parent;
+ Mutex apply_lock; // for apply mutual exclusion
+
+ void queue(Op *o) {
+ Mutex::Locker l(qlock);
+ q.push_back(o);
+ }
+ Op *peek_queue() {
+ assert(apply_lock.is_locked());
+ return q.front();
+ }
+ Op *dequeue() {
+ assert(apply_lock.is_locked());
+ Mutex::Locker l(qlock);
+ Op *o = q.front();
+ q.pop_front();
+ cond.Signal();
+ return o;
+ }
+ void flush() {
+ Mutex::Locker l(qlock);
+
+ // get max for journal _or_ op queues
+ uint64_t seq = 0;
+ if (!q.empty())
+ seq = q.back()->op;
+ if (!jq.empty() && jq.back() > seq)
+ seq = jq.back();
+
+ if (seq) {
+ // everything prior to our watermark to drain through either/both
+ // queues
+ while ((!q.empty() && q.front()->op <= seq) ||
+ (!jq.empty() && jq.front() <= seq))
+ cond.Wait(qlock);
+ }
+ }
+
+ OpSequencer()
+ : qlock("KeyValueStore::OpSequencer::qlock", false, false),
+ parent(0),
+ apply_lock("KeyValueStore::OpSequencer::apply_lock", false, false) {}
+ ~OpSequencer() {
+ assert(q.empty());
+ }
+
+ const string& get_name() const {
+ return parent->get_name();
+ }
+ };
+
+ friend ostream& operator<<(ostream& out, const OpSequencer& s);
+
+ Sequencer default_osr;
+ deque<OpSequencer*> op_queue;
+ uint64_t op_queue_len, op_queue_bytes;
+ Finisher op_finisher;
+
+ ThreadPool op_tp;
+ struct OpWQ : public ThreadPool::WorkQueue<OpSequencer> {
+ KeyValueStore *store;
+ OpWQ(KeyValueStore *fs, time_t timeout, time_t suicide_timeout,
+ ThreadPool *tp) :
+ ThreadPool::WorkQueue<OpSequencer>("KeyValueStore::OpWQ",
+ timeout, suicide_timeout, tp),
+ store(fs) {}
+
+ bool _enqueue(OpSequencer *osr) {
+ store->op_queue.push_back(osr);
+ return true;
+ }
+ void _dequeue(OpSequencer *o) {
+ assert(0);
+ }
+ bool _empty() {
+ return store->op_queue.empty();
+ }
+ OpSequencer *_dequeue() {
+ if (store->op_queue.empty())
+ return NULL;
+ OpSequencer *osr = store->op_queue.front();
+ store->op_queue.pop_front();
+ return osr;
+ }
+ void _process(OpSequencer *osr, ThreadPool::TPHandle &handle) {
+ store->_do_op(osr, handle);
+ }
+ void _process_finish(OpSequencer *osr) {
+ store->_finish_op(osr);
+ }
+ void _clear() {
+ assert(store->op_queue.empty());
+ }
+ } op_wq;
+
+ void _do_op(OpSequencer *osr, ThreadPool::TPHandle &handle);
+ void _finish_op(OpSequencer *osr);
+ Op *build_op(list<Transaction*>& tls, Context *ondisk, Context *onreadable,
+ Context *onreadable_sync, TrackedOpRef osd_op);
+ void queue_op(OpSequencer *osr, Op *o);
+
+ PerfCounters *logger;
+
+ public:
+
+ KeyValueStore(const std::string &base,
+ const char *internal_name = "keyvaluestore-dev",
+ bool update_to=false);
+ ~KeyValueStore();
+
+ int _detect_backend() { kv_type = KV_TYPE_LEVELDB; return 0; }
+ bool test_mount_in_use();
+ int version_stamp_is_valid(uint32_t *version);
+ int update_version_stamp();
+ uint32_t get_target_version() {
+ return target_version;
+ }
+ int peek_journal_fsid(uuid_d *id) {
+ *id = fsid;
+ return 0;
+ }
+
+ int write_version_stamp();
+ int mount();
+ int umount();
+ int get_max_object_name_length();
+ int mkfs();
+ int mkjournal() {return 0;}
+
+ /**
+ ** set_allow_sharded_objects()
+ **
+ ** Before sharded ghobject_t can be specified this function must be called
+ **/
+ void set_allow_sharded_objects() {}
+
+ /**
+ ** get_allow_sharded_objects()
+ **
+ ** return value: true if set_allow_sharded_objects() called, otherwise false
+ **/
+ bool get_allow_sharded_objects() {return false;}
+
+ int statfs(struct statfs *buf);
+
+ int _do_transactions(
+ list<Transaction*> &tls, uint64_t op_seq,
+ ThreadPool::TPHandle *handle);
+ int do_transactions(list<Transaction*> &tls, uint64_t op_seq) {
+ return _do_transactions(tls, op_seq, 0);
+ }
+ unsigned _do_transaction(Transaction& transaction,
+ BufferTransaction &bt,
+ SequencerPosition& spos,
+ ThreadPool::TPHandle *handle);
+
+ int queue_transactions(Sequencer *osr, list<Transaction*>& tls,
+ TrackedOpRef op = TrackedOpRef(),
+ ThreadPool::TPHandle *handle = NULL);
+
+
+ // ------------------
+ // objects
+
+ // Read operation need call "check_coll", checking "coll_t" in write
+ // operation is done by lookup_cached_header
+ int _check_coll(const coll_t &cid);
+
+ int _generic_read(coll_t cid, const ghobject_t& oid, uint64_t offset,
+ size_t len, bufferlist& bl, bool allow_eio = false,
+ BufferTransaction *bt = 0);
+ bool exists(coll_t cid, const ghobject_t& oid);
+ int stat(coll_t cid, const ghobject_t& oid, struct stat *st,
+ bool allow_eio = false);
+ int read(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+ bufferlist& bl, bool allow_eio = false);
+ int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+ bufferlist& bl);
+
+ int _touch(coll_t cid, const ghobject_t& oid, BufferTransaction &t);
+ int _write(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+ const bufferlist& bl, BufferTransaction &t, bool replica = false);
+ int _zero(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+ BufferTransaction &t);
+ int _truncate(coll_t cid, const ghobject_t& oid, uint64_t size,
+ BufferTransaction &t);
+ int _clone(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid,
+ BufferTransaction &t);
+ int _clone_range(coll_t cid, const ghobject_t& oldoid,
+ const ghobject_t& newoid, uint64_t srcoff,
+ uint64_t len, uint64_t dstoff, BufferTransaction &t);
+ int _remove(coll_t cid, const ghobject_t& oid, BufferTransaction &t);
+
+
+ void start_sync() {}
+ void sync() {}
+ void flush() {}
+ void sync_and_flush() {}
+
+ void set_fsid(uuid_d u) { fsid = u; }
+ uuid_d get_fsid() { return fsid; }
+
+ // DEBUG read error injection, an object is removed from both on delete()
+ Mutex read_error_lock;
+ set<ghobject_t> data_error_set; // read() will return -EIO
+ set<ghobject_t> mdata_error_set; // getattr(),stat() will return -EIO
+ void inject_data_error(const ghobject_t &oid);
+ void inject_mdata_error(const ghobject_t &oid);
+ void debug_obj_on_delete(const ghobject_t &oid);
+ bool debug_data_eio(const ghobject_t &oid);
+ bool debug_mdata_eio(const ghobject_t &oid);
+
+ // attrs
+ int getattr(coll_t cid, const ghobject_t& oid, const char *name,
+ bufferptr &bp);
+ int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>& aset,
+ bool user_only = false);
+
+ int _setattrs(coll_t cid, const ghobject_t& oid,
+ map<string, bufferptr>& aset, BufferTransaction &t);
+ int _rmattr(coll_t cid, const ghobject_t& oid, const char *name,
+ BufferTransaction &t);
+ int _rmattrs(coll_t cid, const ghobject_t& oid, BufferTransaction &t);
+
+ int collection_getattr(coll_t c, const char *name, void *value, size_t size);
+ int collection_getattr(coll_t c, const char *name, bufferlist& bl);
+ int collection_getattrs(coll_t cid, map<string,bufferptr> &aset);
+
+ int _collection_setattr(coll_t c, const char *name, const void *value,
+ size_t size, BufferTransaction &t);
+ int _collection_rmattr(coll_t c, const char *name, BufferTransaction &t);
+ int _collection_setattrs(coll_t cid, map<string,bufferptr> &aset,
+ BufferTransaction &t);
+
+ // collections
+ int _create_collection(coll_t c, BufferTransaction &t);
+ int _destroy_collection(coll_t c, BufferTransaction &t);
+ int _collection_add(coll_t c, coll_t ocid, const ghobject_t& oid,
+ BufferTransaction &t);
+ int _collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+ coll_t c, const ghobject_t& o,
+ BufferTransaction &t);
+ int _collection_remove_recursive(const coll_t &cid,
+ BufferTransaction &t);
+ int _collection_rename(const coll_t &cid, const coll_t &ncid,
+ BufferTransaction &t);
+ int list_collections(vector<coll_t>& ls);
+ bool collection_exists(coll_t c);
+ bool collection_empty(coll_t c);
+ int collection_list(coll_t c, vector<ghobject_t>& oid);
+ int collection_list_partial(coll_t c, ghobject_t start,
+ int min, int max, snapid_t snap,
+ vector<ghobject_t> *ls, ghobject_t *next);
+ int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
+ snapid_t seq, vector<ghobject_t> *ls);
+ int collection_version_current(coll_t c, uint32_t *version);
+
+ // omap (see ObjectStore.h for documentation)
+ int omap_get(coll_t c, const ghobject_t &oid, bufferlist *header,
+ map<string, bufferlist> *out);
+ int omap_get_header(
+ coll_t c,
+ const ghobject_t &oid,
+ bufferlist *out,
+ bool allow_eio = false);
+ int omap_get_keys(coll_t c, const ghobject_t &oid, set<string> *keys);
+ int omap_get_values(coll_t c, const ghobject_t &oid, const set<string> &keys,
+ map<string, bufferlist> *out);
+ int omap_check_keys(coll_t c, const ghobject_t &oid, const set<string> &keys,
+ set<string> *out);
+ ObjectMap::ObjectMapIterator get_omap_iterator(coll_t c,
+ const ghobject_t &oid);
+
+ void dump_transactions(list<ObjectStore::Transaction*>& ls, uint64_t seq,
+ OpSequencer *osr);
+
+ private:
+ void _inject_failure() {}
+
+ // omap
+ int _omap_clear(coll_t cid, const ghobject_t &oid,
+ BufferTransaction &t);
+ int _omap_setkeys(coll_t cid, const ghobject_t &oid,
+ map<string, bufferlist> &aset,
+ BufferTransaction &t);
+ int _omap_rmkeys(coll_t cid, const ghobject_t &oid, const set<string> &keys,
+ BufferTransaction &t);
+ int _omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+ const string& first, const string& last,
+ BufferTransaction &t);
+ int _omap_setheader(coll_t cid, const ghobject_t &oid, const bufferlist &bl,
+ BufferTransaction &t);
+ int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest,
+ BufferTransaction &t);
+ int _split_collection_create(coll_t cid, uint32_t bits, uint32_t rem,
+ coll_t dest, BufferTransaction &t){
+ return 0;
+ }
+
+ virtual const char** get_tracked_conf_keys() const;
+ virtual void handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed);
+
+ std::string m_osd_rollback_to_cluster_snap;
+ bool m_osd_use_stale_snap;
+ bool m_fail_eio;
+
+ int do_update;
+
+
+ static const string OBJECT_STRIP_PREFIX;
+ static const string OBJECT_XATTR;
+ static const string OBJECT_OMAP;
+ static const string OBJECT_OMAP_HEADER;
+ static const string OBJECT_OMAP_HEADER_KEY;
+ static const string COLLECTION;
+ static const string COLLECTION_ATTR;
+ static const uint32_t COLLECTION_VERSION = 1;
+
+ class SubmitManager {
+ Mutex lock;
+ uint64_t op_seq;
+ uint64_t op_submitted;
+ public:
+ SubmitManager() :
+ lock("JOS::SubmitManager::lock", false, true, false, g_ceph_context),
+ op_seq(0), op_submitted(0)
+ {}
+ uint64_t op_submit_start();
+ void op_submit_finish(uint64_t op);
+ void set_op_seq(uint64_t seq) {
+ Mutex::Locker l(lock);
+ op_submitted = op_seq = seq;
+ }
+ uint64_t get_op_seq() {
+ return op_seq;
+ }
+ } submit_manager;
+};
+
+WRITE_CLASS_ENCODER(StripObjectMap::StripObjectHeader)
+
+#endif
diff --git a/src/os/LFNIndex.cc b/src/os/LFNIndex.cc
index 83e1c14..a250016 100644
--- a/src/os/LFNIndex.cc
+++ b/src/os/LFNIndex.cc
@@ -63,7 +63,7 @@ void LFNIndex::maybe_inject_failure()
/* Public methods */
-void LFNIndex::set_ref(std::tr1::shared_ptr<CollectionIndex> ref)
+void LFNIndex::set_ref(ceph::shared_ptr<CollectionIndex> ref)
{
self_ref = ref;
}
diff --git a/src/os/LFNIndex.h b/src/os/LFNIndex.h
index f436446..c9c7f5e 100644
--- a/src/os/LFNIndex.h
+++ b/src/os/LFNIndex.h
@@ -20,7 +20,7 @@
#include <map>
#include <set>
#include <vector>
-#include <tr1/memory>
+#include "include/memory.h"
#include <exception>
#include "osd/osd_types.h"
@@ -99,7 +99,7 @@ class LFNIndex : public CollectionIndex {
/// Path to Index base.
const string base_path;
/// For reference counting the collection @see Path
- std::tr1::weak_ptr<CollectionIndex> self_ref;
+ ceph::weak_ptr<CollectionIndex> self_ref;
protected:
const uint32_t index_version;
@@ -155,7 +155,7 @@ public:
virtual ~LFNIndex() {}
/// @see CollectionIndex
- void set_ref(std::tr1::shared_ptr<CollectionIndex> ref);
+ void set_ref(ceph::shared_ptr<CollectionIndex> ref);
/// @see CollectionIndex
int init();
@@ -199,14 +199,14 @@ public:
virtual int _split(
uint32_t match, //< [in] value to match
uint32_t bits, //< [in] bits to check
- std::tr1::shared_ptr<CollectionIndex> dest //< [in] destination index
+ ceph::shared_ptr<CollectionIndex> dest //< [in] destination index
) = 0;
/// @see CollectionIndex
int split(
uint32_t match,
uint32_t bits,
- std::tr1::shared_ptr<CollectionIndex> dest
+ ceph::shared_ptr<CollectionIndex> dest
) {
WRAP_RETRY(
r = _split(match, bits, dest);
diff --git a/src/os/LevelDBStore.cc b/src/os/LevelDBStore.cc
index bd4dcdc..326862f 100644
--- a/src/os/LevelDBStore.cc
+++ b/src/os/LevelDBStore.cc
@@ -5,12 +5,27 @@
#include <set>
#include <map>
#include <string>
-#include <tr1/memory>
+#include "include/memory.h"
#include <errno.h>
using std::string;
#include "common/perf_counters.h"
-int LevelDBStore::init(ostream &out, bool create_if_missing)
+int LevelDBStore::init()
+{
+ // init defaults. caller can override these if they want
+ // prior to calling open.
+ options.write_buffer_size = g_conf->leveldb_write_buffer_size;
+ options.cache_size = g_conf->leveldb_cache_size;
+ options.block_size = g_conf->leveldb_block_size;
+ options.bloom_size = g_conf->leveldb_bloom_size;
+ options.compression_enabled = g_conf->leveldb_compression;
+ options.paranoid_checks = g_conf->leveldb_paranoid;
+ options.max_open_files = g_conf->leveldb_max_open_files;
+ options.log_file = g_conf->leveldb_log;
+ return 0;
+}
+
+int LevelDBStore::do_open(ostream &out, bool create_if_missing)
{
leveldb::Options ldoptions;
@@ -59,6 +74,12 @@ int LevelDBStore::init(ostream &out, bool create_if_missing)
return -EINVAL;
}
+ if (g_conf->leveldb_compact_on_mount) {
+ derr << "Compacting leveldb store..." << dendl;
+ compact();
+ derr << "Finished compacting leveldb store" << dendl;
+ }
+
PerfCountersBuilder plb(g_ceph_context, "leveldb", l_leveldb_first, l_leveldb_last);
plb.add_u64_counter(l_leveldb_gets, "leveldb_get");
plb.add_u64_counter(l_leveldb_txns, "leveldb_transaction");
@@ -75,6 +96,9 @@ LevelDBStore::~LevelDBStore()
{
close();
delete logger;
+
+ // Ensure db is destroyed before dependent db_cache and filterpolicy
+ db.reset();
}
void LevelDBStore::close()
diff --git a/src/os/LevelDBStore.h b/src/os/LevelDBStore.h
index bc5b612..26e7bbe 100644
--- a/src/os/LevelDBStore.h
+++ b/src/os/LevelDBStore.h
@@ -9,7 +9,7 @@
#include <set>
#include <map>
#include <string>
-#include <tr1/memory>
+#include "include/memory.h"
#include <boost/scoped_ptr.hpp>
#include "leveldb/db.h"
#include "leveldb/env.h"
@@ -48,13 +48,13 @@ class LevelDBStore : public KeyValueDB {
CephContext *cct;
PerfCounters *logger;
string path;
- boost::scoped_ptr<leveldb::DB> db;
boost::scoped_ptr<leveldb::Cache> db_cache;
#ifdef HAVE_LEVELDB_FILTER_POLICY
boost::scoped_ptr<const leveldb::FilterPolicy> filterpolicy;
#endif
+ boost::scoped_ptr<leveldb::DB> db;
- int init(ostream &out, bool create_if_missing);
+ int do_open(ostream &out, bool create_if_missing);
// manage async compactions
Mutex compact_queue_lock;
@@ -154,13 +154,15 @@ public:
~LevelDBStore();
+ int init();
+
/// Opens underlying db
int open(ostream &out) {
- return init(out, false);
+ return do_open(out, false);
}
/// Creates underlying db if missing and opens it
int create_and_open(ostream &out) {
- return init(out, true);
+ return do_open(out, true);
}
void close();
@@ -186,7 +188,7 @@ public:
};
KeyValueDB::Transaction get_transaction() {
- return std::tr1::shared_ptr< LevelDBTransactionImpl >(
+ return ceph::shared_ptr< LevelDBTransactionImpl >(
new LevelDBTransactionImpl(this));
}
@@ -372,7 +374,7 @@ err:
protected:
WholeSpaceIterator _get_iterator() {
- return std::tr1::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
+ return ceph::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
new LevelDBWholeSpaceIteratorImpl(
db->NewIterator(leveldb::ReadOptions())
)
@@ -386,7 +388,7 @@ protected:
snapshot = db->GetSnapshot();
options.snapshot = snapshot;
- return std::tr1::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
+ return ceph::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
new LevelDBSnapshotIteratorImpl(db.get(), snapshot,
db->NewIterator(options))
);
diff --git a/src/os/Makefile.am b/src/os/Makefile.am
index 4f12a6a..2bba7aa 100644
--- a/src/os/Makefile.am
+++ b/src/os/Makefile.am
@@ -1,45 +1,58 @@
libos_la_SOURCES = \
+ os/chain_xattr.cc \
+ os/DBObjectMap.cc \
+ os/GenericObjectMap.cc \
os/FileJournal.cc \
os/FileStore.cc \
- os/chain_xattr.cc \
- os/ObjectStore.cc \
- os/JournalingObjectStore.cc \
- os/LFNIndex.cc \
+ os/FlatIndex.cc \
+ os/GenericFileStoreBackend.cc \
os/HashIndex.cc \
os/IndexManager.cc \
- os/FlatIndex.cc \
- os/DBObjectMap.cc \
+ os/JournalingObjectStore.cc \
os/LevelDBStore.cc \
+ os/LFNIndex.cc \
+ os/MemStore.cc \
+ os/KeyValueStore.cc \
+ os/ObjectStore.cc \
os/WBThrottle.cc \
- os/BtrfsFileStoreBackend.cc \
- os/GenericFileStoreBackend.cc \
- os/ZFSFileStoreBackend.cc \
common/TrackedOp.cc
+
+if LINUX
+libos_la_SOURCES += os/BtrfsFileStoreBackend.cc
+endif
+
+if WITH_LIBZFS
+libos_la_SOURCES += os/ZFSFileStoreBackend.cc
+endif
+
noinst_LTLIBRARIES += libos.la
noinst_HEADERS += \
os/btrfs_ioctl.h \
os/chain_xattr.h \
+ os/BtrfsFileStoreBackend.h \
os/CollectionIndex.h \
+ os/DBObjectMap.h \
+ os/GenericObjectMap.h \
os/FileJournal.h \
os/FileStore.h \
- os/BtrfsFileStoreBackend.h \
- os/GenericFileStoreBackend.h \
- os/ZFSFileStoreBackend.h \
os/FlatIndex.h \
- os/HashIndex.h \
os/FDCache.h \
- os/WBThrottle.h \
+ os/GenericFileStoreBackend.h \
+ os/HashIndex.h \
os/IndexManager.h \
os/Journal.h \
os/JournalingObjectStore.h \
+ os/KeyValueDB.h \
+ os/LevelDBStore.h \
os/LFNIndex.h \
+ os/MemStore.h \
+ os/KeyValueStore.h \
+ os/ObjectMap.h \
os/ObjectStore.h \
os/SequencerPosition.h \
- os/ObjectMap.h \
- os/DBObjectMap.h \
- os/KeyValueDB.h \
- os/LevelDBStore.h
+ os/WBThrottle.h \
+ os/ZFSFileStoreBackend.h
if WITH_LIBZFS
libos_zfs_a_SOURCES = os/ZFS.cc
diff --git a/src/os/MemStore.cc b/src/os/MemStore.cc
new file mode 100644
index 0000000..c736187
--- /dev/null
+++ b/src/os/MemStore.cc
@@ -0,0 +1,1478 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "acconfig.h"
+
+#ifdef HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+#include "include/types.h"
+#include "include/stringify.h"
+#include "include/unordered_map.h"
+#include "include/memory.h"
+#include "common/errno.h"
+#include "MemStore.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "memstore(" << path << ") "
+
+// for comparing collections for lock ordering
+bool operator>(const MemStore::CollectionRef& l,
+ const MemStore::CollectionRef& r)
+{
+ return (unsigned long)l.get() > (unsigned long)r.get();
+}
+
+
+int MemStore::peek_journal_fsid(uuid_d *fsid)
+{
+ *fsid = uuid_d();
+ return 0;
+}
+
+int MemStore::mount()
+{
+ int r = _load();
+ if (r < 0)
+ return r;
+ finisher.start();
+ return 0;
+}
+
+int MemStore::umount()
+{
+ finisher.stop();
+ return _save();
+}
+
+int MemStore::_save()
+{
+ dout(10) << __func__ << dendl;
+ Mutex::Locker l(apply_lock); // block any writer
+ dump_all();
+ set<coll_t> collections;
+ for (ceph::unordered_map<coll_t,CollectionRef>::iterator p = coll_map.begin();
+ p != coll_map.end();
+ ++p) {
+ dout(20) << __func__ << " coll " << p->first << " " << p->second << dendl;
+ collections.insert(p->first);
+ bufferlist bl;
+ assert(p->second);
+ p->second->encode(bl);
+ string fn = path + "/" + stringify(p->first);
+ int r = bl.write_file(fn.c_str());
+ if (r < 0)
+ return r;
+ }
+
+ string fn = path + "/collections";
+ bufferlist bl;
+ ::encode(collections, bl);
+ int r = bl.write_file(fn.c_str());
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+void MemStore::dump_all()
+{
+ Formatter *f = new_formatter("json-pretty");
+ f->open_object_section("store");
+ dump(f);
+ f->close_section();
+ dout(0) << "dump:";
+ f->flush(*_dout);
+ *_dout << dendl;
+ delete f;
+}
+
+void MemStore::dump(Formatter *f)
+{
+ f->open_array_section("collections");
+ for (ceph::unordered_map<coll_t,CollectionRef>::iterator p = coll_map.begin();
+ p != coll_map.end();
+ ++p) {
+ f->open_object_section("collection");
+ f->dump_string("name", stringify(p->first));
+
+ f->open_array_section("xattrs");
+ for (map<string,bufferptr>::iterator q = p->second->xattr.begin();
+ q != p->second->xattr.end();
+ ++q) {
+ f->open_object_section("xattr");
+ f->dump_string("name", q->first);
+ f->dump_int("length", q->second.length());
+ f->close_section();
+ }
+ f->close_section();
+
+ f->open_array_section("objects");
+ for (map<ghobject_t,ObjectRef>::iterator q = p->second->object_map.begin();
+ q != p->second->object_map.end();
+ ++q) {
+ f->open_object_section("object");
+ f->dump_string("name", stringify(q->first));
+ if (q->second)
+ q->second->dump(f);
+ f->close_section();
+ }
+ f->close_section();
+
+ f->close_section();
+ }
+ f->close_section();
+}
+
+int MemStore::_load()
+{
+ dout(10) << __func__ << dendl;
+ bufferlist bl;
+ string fn = path + "/collections";
+ string err;
+ int r = bl.read_file(fn.c_str(), &err);
+ if (r < 0)
+ return r;
+
+ set<coll_t> collections;
+ bufferlist::iterator p = bl.begin();
+ ::decode(collections, p);
+
+ for (set<coll_t>::iterator q = collections.begin();
+ q != collections.end();
+ ++q) {
+ string fn = path + "/" + stringify(*q);
+ bufferlist cbl;
+ int r = cbl.read_file(fn.c_str(), &err);
+ if (r < 0)
+ return r;
+ CollectionRef c(new Collection);
+ bufferlist::iterator p = cbl.begin();
+ c->decode(p);
+ coll_map[*q] = c;
+ }
+
+ dump_all();
+
+ return 0;
+}
+
+void MemStore::set_fsid(uuid_d u)
+{
+ int r = write_meta("fs_fsid", stringify(u));
+ assert(r >= 0);
+}
+
+uuid_d MemStore::get_fsid()
+{
+ string fsid_str;
+ int r = read_meta("fs_fsid", &fsid_str);
+ assert(r >= 0);
+ uuid_d uuid;
+ bool b = uuid.parse(fsid_str.c_str());
+ assert(b);
+ return uuid;
+}
+
+int MemStore::mkfs()
+{
+ string fsid_str;
+ int r = read_meta("fs_fsid", &fsid_str);
+ if (r == -ENOENT) {
+ uuid_d fsid;
+ fsid.generate_random();
+ fsid_str = stringify(fsid);
+ r = write_meta("fs_fsid", fsid_str);
+ if (r < 0)
+ return r;
+ dout(1) << __func__ << " new fsid " << fsid_str << dendl;
+ } else {
+ dout(1) << __func__ << " had fsid " << fsid_str << dendl;
+ }
+
+ string fn = path + "/collections";
+ derr << path << dendl;
+ bufferlist bl;
+ set<coll_t> collections;
+ ::encode(collections, bl);
+ r = bl.write_file(fn.c_str());
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+int MemStore::statfs(struct statfs *st)
+{
+ dout(10) << __func__ << dendl;
+ // make some shit up. these are the only fields that matter.
+ st->f_bsize = 1024;
+ st->f_blocks = 1000000;
+ st->f_bfree = 1000000;
+ st->f_bavail = 1000000;
+ return 0;
+}
+
+objectstore_perf_stat_t MemStore::get_cur_stats()
+{
+ // fixme
+ return objectstore_perf_stat_t();
+}
+
+MemStore::CollectionRef MemStore::get_collection(coll_t cid)
+{
+ RWLock::RLocker l(coll_lock);
+ ceph::unordered_map<coll_t,CollectionRef>::iterator cp = coll_map.find(cid);
+ if (cp == coll_map.end())
+ return CollectionRef();
+ return cp->second;
+}
+
+
+// ---------------
+// read operations
+
+bool MemStore::exists(coll_t cid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return false;
+ RWLock::RLocker l(c->lock);
+
+ // Perform equivalent of c->get_object_(oid) != NULL. In C++11 the
+ // shared_ptr needs to be compared to nullptr.
+ return (bool)c->get_object(oid);
+}
+
+int MemStore::stat(
+ coll_t cid,
+ const ghobject_t& oid,
+ struct stat *st,
+ bool allow_eio)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ st->st_size = o->data.length();
+ st->st_blksize = 4096;
+ st->st_blocks = (st->st_size + st->st_blksize - 1) / st->st_blksize;
+ st->st_nlink = 1;
+ return 0;
+}
+
+int MemStore::read(
+ coll_t cid,
+ const ghobject_t& oid,
+ uint64_t offset,
+ size_t len,
+ bufferlist& bl,
+ bool allow_eio)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " "
+ << offset << "~" << len << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker lc(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ if (offset >= o->data.length())
+ return 0;
+ size_t l = len;
+ if (l == 0) // note: len == 0 means read the entire object
+ l = o->data.length();
+ else if (offset + l > o->data.length())
+ l = o->data.length() - offset;
+ bl.clear();
+ bl.substr_of(o->data, offset, l);
+ return bl.length();
+}
+
+int MemStore::fiemap(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len, bufferlist& bl)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << offset << "~"
+ << len << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker lc(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ if (offset >= o->data.length())
+ return 0;
+ size_t l = len;
+ if (offset + l > o->data.length())
+ l = o->data.length() - offset;
+ map<uint64_t, uint64_t> m;
+ m[offset] = l;
+ ::encode(m, bl);
+ return 0;
+}
+
+int MemStore::getattr(coll_t cid, const ghobject_t& oid,
+ const char *name, bufferptr& value)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << name << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ string k(name);
+ if (!o->xattr.count(k)) {
+ return -ENODATA;
+ }
+ value = o->xattr[k];
+ return 0;
+}
+
+int MemStore::getattrs(coll_t cid, const ghobject_t& oid,
+ map<string,bufferptr>& aset, bool user_only)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ if (user_only) {
+ for (map<string,bufferptr>::iterator p = o->xattr.begin();
+ p != o->xattr.end();
+ ++p) {
+ if (p->first.length() > 1 && p->first[0] == '_') {
+ aset[p->first.substr(1)] = p->second;
+ }
+ }
+ } else {
+ aset = o->xattr;
+ }
+ return 0;
+}
+
+int MemStore::list_collections(vector<coll_t>& ls)
+{
+ dout(10) << __func__ << dendl;
+ RWLock::RLocker l(coll_lock);
+ for (ceph::unordered_map<coll_t,CollectionRef>::iterator p = coll_map.begin();
+ p != coll_map.end();
+ ++p) {
+ ls.push_back(p->first);
+ }
+ return 0;
+}
+
+bool MemStore::collection_exists(coll_t cid)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ RWLock::RLocker l(coll_lock);
+ return coll_map.count(cid);
+}
+
+int MemStore::collection_getattr(coll_t cid, const char *name,
+ void *value, size_t size)
+{
+ dout(10) << __func__ << " " << cid << " " << name << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker lc(c->lock);
+
+ if (!c->xattr.count(name))
+ return -ENOENT;
+ bufferlist bl;
+ bl.append(c->xattr[name]);
+ size_t l = MIN(size, bl.length());
+ bl.copy(0, size, (char *)value);
+ return l;
+}
+
+int MemStore::collection_getattr(coll_t cid, const char *name, bufferlist& bl)
+{
+ dout(10) << __func__ << " " << cid << " " << name << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ if (!c->xattr.count(name))
+ return -ENOENT;
+ bl.clear();
+ bl.append(c->xattr[name]);
+ return bl.length();
+}
+
+int MemStore::collection_getattrs(coll_t cid, map<string,bufferptr> &aset)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ aset = c->xattr;
+ return 0;
+}
+
+bool MemStore::collection_empty(coll_t cid)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ return c->object_map.empty();
+}
+
+int MemStore::collection_list(coll_t cid, vector<ghobject_t>& o)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ for (map<ghobject_t,ObjectRef>::iterator p = c->object_map.begin();
+ p != c->object_map.end();
+ ++p)
+ o.push_back(p->first);
+ return 0;
+}
+
+int MemStore::collection_list_partial(coll_t cid, ghobject_t start,
+ int min, int max, snapid_t snap,
+ vector<ghobject_t> *ls, ghobject_t *next)
+{
+ dout(10) << __func__ << " " << cid << " " << start << " " << min << "-"
+ << max << " " << snap << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ map<ghobject_t,ObjectRef>::iterator p = c->object_map.lower_bound(start);
+ while (p != c->object_map.end() &&
+ ls->size() < (unsigned)max) {
+ ls->push_back(p->first);
+ ++p;
+ }
+ if (p == c->object_map.end())
+ *next = ghobject_t::get_max();
+ else
+ *next = p->first;
+ return 0;
+}
+
+int MemStore::collection_list_range(coll_t cid,
+ ghobject_t start, ghobject_t end,
+ snapid_t seq, vector<ghobject_t> *ls)
+{
+ dout(10) << __func__ << " " << cid << " " << start << " " << end
+ << " " << seq << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ map<ghobject_t,ObjectRef>::iterator p = c->object_map.lower_bound(start);
+ while (p != c->object_map.end() &&
+ p->first < end) {
+ ls->push_back(p->first);
+ ++p;
+ }
+ return 0;
+}
+
+int MemStore::omap_get(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ bufferlist *header, ///< [out] omap header
+ map<string, bufferlist> *out /// < [out] Key to value map
+ )
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ *header = o->omap_header;
+ *out = o->omap;
+ return 0;
+}
+
+int MemStore::omap_get_header(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ bufferlist *header, ///< [out] omap header
+ bool allow_eio ///< [in] don't assert on eio
+ )
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ *header = o->omap_header;
+ return 0;
+}
+
+int MemStore::omap_get_keys(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ set<string> *keys ///< [out] Keys defined on oid
+ )
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ for (map<string,bufferlist>::iterator p = o->omap.begin();
+ p != o->omap.end();
+ ++p)
+ keys->insert(p->first);
+ return 0;
+}
+
+int MemStore::omap_get_values(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ const set<string> &keys, ///< [in] Keys to get
+ map<string, bufferlist> *out ///< [out] Returned keys and values
+ )
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ for (set<string>::const_iterator p = keys.begin();
+ p != keys.end();
+ ++p) {
+ map<string,bufferlist>::iterator q = o->omap.find(*p);
+ if (q != o->omap.end())
+ out->insert(*q);
+ }
+ return 0;
+}
+
+int MemStore::omap_check_keys(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ const set<string> &keys, ///< [in] Keys to check
+ set<string> *out ///< [out] Subset of keys defined on oid
+ )
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ for (set<string>::const_iterator p = keys.begin();
+ p != keys.end();
+ ++p) {
+ map<string,bufferlist>::iterator q = o->omap.find(*p);
+ if (q != o->omap.end())
+ out->insert(*p);
+ }
+ return 0;
+}
+
+ObjectMap::ObjectMapIterator MemStore::get_omap_iterator(coll_t cid,
+ const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return ObjectMap::ObjectMapIterator();
+ RWLock::RLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return ObjectMap::ObjectMapIterator();
+ return ObjectMap::ObjectMapIterator(new OmapIteratorImpl(c, o));
+}
+
+
+// ---------------
+// write operations
+
+int MemStore::queue_transactions(Sequencer *osr,
+ list<Transaction*>& tls,
+ TrackedOpRef op,
+ ThreadPool::TPHandle *handle)
+{
+ // fixme: ignore the Sequencer and serialize everything.
+ Mutex::Locker l(apply_lock);
+
+ for (list<Transaction*>::iterator p = tls.begin(); p != tls.end(); ++p) {
+ // poke the TPHandle heartbeat just to exercise that code path
+ if (handle)
+ handle->reset_tp_timeout();
+
+ _do_transaction(**p);
+ }
+
+ Context *on_apply = NULL, *on_apply_sync = NULL, *on_commit = NULL;
+ ObjectStore::Transaction::collect_contexts(tls, &on_apply, &on_commit,
+ &on_apply_sync);
+ if (on_apply_sync)
+ on_apply_sync->complete(0);
+ if (on_apply)
+ finisher.queue(on_apply);
+ if (on_commit)
+ finisher.queue(on_commit);
+ return 0;
+}
+
+void MemStore::_do_transaction(Transaction& t)
+{
+ Transaction::iterator i = t.begin();
+ int pos = 0;
+
+ while (i.have_op()) {
+ int op = i.get_op();
+ int r = 0;
+
+ switch (op) {
+ case Transaction::OP_NOP:
+ break;
+ case Transaction::OP_TOUCH:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _touch(cid, oid);
+ }
+ break;
+
+ case Transaction::OP_WRITE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t off = i.get_length();
+ uint64_t len = i.get_length();
+ bool replica = i.get_replica();
+ bufferlist bl;
+ i.get_bl(bl);
+ r = _write(cid, oid, off, len, bl, replica);
+ }
+ break;
+
+ case Transaction::OP_ZERO:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t off = i.get_length();
+ uint64_t len = i.get_length();
+ r = _zero(cid, oid, off, len);
+ }
+ break;
+
+ case Transaction::OP_TRIMCACHE:
+ {
+ i.get_cid();
+ i.get_oid();
+ i.get_length();
+ i.get_length();
+ // deprecated, no-op
+ }
+ break;
+
+ case Transaction::OP_TRUNCATE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ uint64_t off = i.get_length();
+ r = _truncate(cid, oid, off);
+ }
+ break;
+
+ case Transaction::OP_REMOVE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _remove(cid, oid);
+ }
+ break;
+
+ case Transaction::OP_SETATTR:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ string name = i.get_attrname();
+ bufferlist bl;
+ i.get_bl(bl);
+ map<string, bufferptr> to_set;
+ to_set[name] = bufferptr(bl.c_str(), bl.length());
+ r = _setattrs(cid, oid, to_set);
+ }
+ break;
+
+ case Transaction::OP_SETATTRS:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ map<string, bufferptr> aset;
+ i.get_attrset(aset);
+ r = _setattrs(cid, oid, aset);
+ }
+ break;
+
+ case Transaction::OP_RMATTR:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ string name = i.get_attrname();
+ r = _rmattr(cid, oid, name.c_str());
+ }
+ break;
+
+ case Transaction::OP_RMATTRS:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _rmattrs(cid, oid);
+ }
+ break;
+
+ case Transaction::OP_CLONE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ ghobject_t noid = i.get_oid();
+ r = _clone(cid, oid, noid);
+ }
+ break;
+
+ case Transaction::OP_CLONERANGE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ ghobject_t noid = i.get_oid();
+ uint64_t off = i.get_length();
+ uint64_t len = i.get_length();
+ r = _clone_range(cid, oid, noid, off, len, off);
+ }
+ break;
+
+ case Transaction::OP_CLONERANGE2:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ ghobject_t noid = i.get_oid();
+ uint64_t srcoff = i.get_length();
+ uint64_t len = i.get_length();
+ uint64_t dstoff = i.get_length();
+ r = _clone_range(cid, oid, noid, srcoff, len, dstoff);
+ }
+ break;
+
+ case Transaction::OP_MKCOLL:
+ {
+ coll_t cid = i.get_cid();
+ r = _create_collection(cid);
+ }
+ break;
+
+ case Transaction::OP_RMCOLL:
+ {
+ coll_t cid = i.get_cid();
+ r = _destroy_collection(cid);
+ }
+ break;
+
+ case Transaction::OP_COLL_ADD:
+ {
+ coll_t ncid = i.get_cid();
+ coll_t ocid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _collection_add(ncid, ocid, oid);
+ }
+ break;
+
+ case Transaction::OP_COLL_REMOVE:
+ {
+ coll_t cid = i.get_cid();
+ ghobject_t oid = i.get_oid();
+ r = _remove(cid, oid);
+ }
+ break;
+
+ case Transaction::OP_COLL_MOVE:
+ assert(0 == "deprecated");
+ break;
+
+ case Transaction::OP_COLL_MOVE_RENAME:
+ {
+ coll_t oldcid = i.get_cid();
+ ghobject_t oldoid = i.get_oid();
+ coll_t newcid = i.get_cid();
+ ghobject_t newoid = i.get_oid();
+ r = _collection_move_rename(oldcid, oldoid, newcid, newoid);
+ }
+ break;
+
+ case Transaction::OP_COLL_SETATTR:
+ {
+ coll_t cid = i.get_cid();
+ string name = i.get_attrname();
+ bufferlist bl;
+ i.get_bl(bl);
+ r = _collection_setattr(cid, name.c_str(), bl.c_str(), bl.length());
+ }
+ break;
+
+ case Transaction::OP_COLL_RMATTR:
+ {
+ coll_t cid = i.get_cid();
+ string name = i.get_attrname();
+ r = _collection_rmattr(cid, name.c_str());
+ }
+ break;
+
+ case Transaction::OP_COLL_RENAME:
+ {
+ coll_t cid(i.get_cid());
+ coll_t ncid(i.get_cid());
+ r = _collection_rename(cid, ncid);
+ }
+ break;
+
+ case Transaction::OP_OMAP_CLEAR:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ r = _omap_clear(cid, oid);
+ }
+ break;
+ case Transaction::OP_OMAP_SETKEYS:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ map<string, bufferlist> aset;
+ i.get_attrset(aset);
+ r = _omap_setkeys(cid, oid, aset);
+ }
+ break;
+ case Transaction::OP_OMAP_RMKEYS:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ set<string> keys;
+ i.get_keyset(keys);
+ r = _omap_rmkeys(cid, oid, keys);
+ }
+ break;
+ case Transaction::OP_OMAP_RMKEYRANGE:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ string first, last;
+ first = i.get_key();
+ last = i.get_key();
+ r = _omap_rmkeyrange(cid, oid, first, last);
+ }
+ break;
+ case Transaction::OP_OMAP_SETHEADER:
+ {
+ coll_t cid(i.get_cid());
+ ghobject_t oid = i.get_oid();
+ bufferlist bl;
+ i.get_bl(bl);
+ r = _omap_setheader(cid, oid, bl);
+ }
+ break;
+ case Transaction::OP_SPLIT_COLLECTION:
+ assert(0 == "deprecated");
+ break;
+ case Transaction::OP_SPLIT_COLLECTION2:
+ {
+ coll_t cid(i.get_cid());
+ uint32_t bits(i.get_u32());
+ uint32_t rem(i.get_u32());
+ coll_t dest(i.get_cid());
+ r = _split_collection(cid, bits, rem, dest);
+ }
+ break;
+
+ default:
+ derr << "bad op " << op << dendl;
+ assert(0);
+ }
+
+ if (r < 0) {
+ bool ok = false;
+
+ if (r == -ENOENT && !(op == Transaction::OP_CLONERANGE ||
+ op == Transaction::OP_CLONE ||
+ op == Transaction::OP_CLONERANGE2 ||
+ op == Transaction::OP_COLL_ADD))
+ // -ENOENT is usually okay
+ ok = true;
+ if (r == -ENODATA)
+ ok = true;
+
+ if (!ok) {
+ const char *msg = "unexpected error code";
+
+ if (r == -ENOENT && (op == Transaction::OP_CLONERANGE ||
+ op == Transaction::OP_CLONE ||
+ op == Transaction::OP_CLONERANGE2))
+ msg = "ENOENT on clone suggests osd bug";
+
+ if (r == -ENOSPC)
+ // For now, if we hit _any_ ENOSPC, crash, before we do any damage
+ // by partially applying transactions.
+ msg = "ENOSPC handling not implemented";
+
+ if (r == -ENOTEMPTY) {
+ msg = "ENOTEMPTY suggests garbage data in osd data dir";
+ dump_all();
+ }
+
+ dout(0) << " error " << cpp_strerror(r) << " not handled on operation " << op
+ << " (op " << pos << ", counting from 0)" << dendl;
+ dout(0) << msg << dendl;
+ dout(0) << " transaction dump:\n";
+ JSONFormatter f(true);
+ f.open_object_section("transaction");
+ t.dump(&f);
+ f.close_section();
+ f.flush(*_dout);
+ *_dout << dendl;
+ assert(0 == "unexpected error");
+ }
+ }
+
+ ++pos;
+ }
+}
+
+int MemStore::_touch(coll_t cid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o) {
+ o.reset(new Object);
+ c->object_map[oid] = o;
+ c->object_hash[oid] = o;
+ }
+ return 0;
+}
+
+int MemStore::_write(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len, const bufferlist& bl,
+ bool replica)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " "
+ << offset << "~" << len << dendl;
+ assert(len == bl.length());
+
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o) {
+ // write implicitly creates a missing object
+ o.reset(new Object);
+ c->object_map[oid] = o;
+ c->object_hash[oid] = o;
+ }
+
+ _write_into_bl(bl, offset, &o->data);
+ return 0;
+}
+
+void MemStore::_write_into_bl(const bufferlist& src, unsigned offset,
+ bufferlist *dst)
+{
+ unsigned len = src.length();
+
+ // before
+ bufferlist newdata;
+ if (dst->length() >= offset) {
+ newdata.substr_of(*dst, 0, offset);
+ } else {
+ newdata.substr_of(*dst, 0, dst->length());
+ bufferptr bp(offset - dst->length());
+ bp.zero();
+ newdata.append(bp);
+ }
+
+ newdata.append(src);
+
+ // after
+ if (dst->length() > offset + len) {
+ bufferlist tail;
+ tail.substr_of(*dst, offset + len, dst->length() - (offset + len));
+ newdata.append(tail);
+ }
+
+ dst->claim(newdata);
+}
+
+int MemStore::_zero(coll_t cid, const ghobject_t& oid,
+ uint64_t offset, size_t len)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << offset << "~"
+ << len << dendl;
+ bufferptr bp(len);
+ bp.zero();
+ bufferlist bl;
+ bl.push_back(bp);
+ return _write(cid, oid, offset, len, bl);
+}
+
+int MemStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << size << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ if (o->data.length() > size) {
+ bufferlist bl;
+ bl.substr_of(o->data, 0, size);
+ o->data.claim(bl);
+ } else if (o->data.length() == size) {
+ // do nothing
+ } else {
+ bufferptr bp(size - o->data.length());
+ bp.zero();
+ o->data.append(bp);
+ }
+ return 0;
+}
+
+int MemStore::_remove(coll_t cid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ c->object_map.erase(oid);
+ c->object_hash.erase(oid);
+ return 0;
+}
+
+int MemStore::_setattrs(coll_t cid, const ghobject_t& oid,
+ map<string,bufferptr>& aset)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ for (map<string,bufferptr>::const_iterator p = aset.begin(); p != aset.end(); ++p)
+ o->xattr[p->first] = p->second;
+ return 0;
+}
+
+int MemStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << name << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ if (!o->xattr.count(name))
+ return -ENODATA;
+ o->xattr.erase(name);
+ return 0;
+}
+
+int MemStore::_rmattrs(coll_t cid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ o->xattr.clear();
+ return 0;
+}
+
+int MemStore::_clone(coll_t cid, const ghobject_t& oldoid,
+ const ghobject_t& newoid)
+{
+ dout(10) << __func__ << " " << cid << " " << oldoid
+ << " -> " << newoid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef oo = c->get_object(oldoid);
+ if (!oo)
+ return -ENOENT;
+ ObjectRef no = c->get_object(newoid);
+ if (!no) {
+ no.reset(new Object);
+ c->object_map[newoid] = no;
+ c->object_hash[newoid] = no;
+ }
+ no->data = oo->data;
+ no->omap_header = oo->omap_header;
+ no->omap = oo->omap;
+ return 0;
+}
+
+int MemStore::_clone_range(coll_t cid, const ghobject_t& oldoid,
+ const ghobject_t& newoid,
+ uint64_t srcoff, uint64_t len, uint64_t dstoff)
+{
+ dout(10) << __func__ << " " << cid << " "
+ << oldoid << " " << srcoff << "~" << len << " -> "
+ << newoid << " " << dstoff << "~" << len
+ << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef oo = c->get_object(oldoid);
+ if (!oo)
+ return -ENOENT;
+ ObjectRef no = c->get_object(newoid);
+ if (!no) {
+ no.reset(new Object);
+ c->object_map[newoid] = no;
+ c->object_hash[newoid] = no;
+ }
+ if (srcoff >= oo->data.length())
+ return 0;
+ if (srcoff + len >= oo->data.length())
+ len = oo->data.length() - srcoff;
+ bufferlist bl;
+ bl.substr_of(oo->data, srcoff, len);
+ _write_into_bl(bl, dstoff, &no->data);
+ return len;
+}
+
+int MemStore::_omap_clear(coll_t cid, const ghobject_t &oid)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ o->omap.clear();
+ return 0;
+}
+
+int MemStore::_omap_setkeys(coll_t cid, const ghobject_t &oid,
+ const map<string, bufferlist> &aset)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ for (map<string,bufferlist>::const_iterator p = aset.begin(); p != aset.end(); ++p)
+ o->omap[p->first] = p->second;
+ return 0;
+}
+
+int MemStore::_omap_rmkeys(coll_t cid, const ghobject_t &oid,
+ const set<string> &keys)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ for (set<string>::const_iterator p = keys.begin(); p != keys.end(); ++p)
+ o->omap.erase(*p);
+ return 0;
+}
+
+int MemStore::_omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+ const string& first, const string& last)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << " " << first
+ << " " << last << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ map<string,bufferlist>::iterator p = o->omap.upper_bound(first);
+ map<string,bufferlist>::iterator e = o->omap.lower_bound(last);
+ while (p != e)
+ o->omap.erase(p++);
+ return 0;
+}
+
+int MemStore::_omap_setheader(coll_t cid, const ghobject_t &oid,
+ const bufferlist &bl)
+{
+ dout(10) << __func__ << " " << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ RWLock::WLocker l(c->lock);
+
+ ObjectRef o = c->get_object(oid);
+ if (!o)
+ return -ENOENT;
+ o->omap_header = bl;
+ return 0;
+}
+
+int MemStore::_create_collection(coll_t cid)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ RWLock::WLocker l(coll_lock);
+ ceph::unordered_map<coll_t,CollectionRef>::iterator cp = coll_map.find(cid);
+ if (cp != coll_map.end())
+ return -EEXIST;
+ coll_map[cid].reset(new Collection);
+ return 0;
+}
+
+int MemStore::_destroy_collection(coll_t cid)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ RWLock::WLocker l(coll_lock);
+ ceph::unordered_map<coll_t,CollectionRef>::iterator cp = coll_map.find(cid);
+ if (cp == coll_map.end())
+ return -ENOENT;
+ {
+ RWLock::RLocker l2(cp->second->lock);
+ if (!cp->second->object_map.empty())
+ return -ENOTEMPTY;
+ }
+ coll_map.erase(cp);
+ return 0;
+}
+
+int MemStore::_collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << cid << " " << ocid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ CollectionRef oc = get_collection(ocid);
+ if (!oc)
+ return -ENOENT;
+ RWLock::WLocker l1(MIN(c, oc)->lock);
+ RWLock::WLocker l2(MAX(c, oc)->lock);
+
+ if (c->object_hash.count(oid))
+ return -EEXIST;
+ if (oc->object_hash.count(oid) == 0)
+ return -ENOENT;
+ ObjectRef o = oc->object_hash[oid];
+ c->object_map[oid] = o;
+ c->object_hash[oid] = o;
+ return 0;
+}
+
+int MemStore::_collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+ coll_t cid, const ghobject_t& oid)
+{
+ dout(10) << __func__ << " " << oldcid << " " << oldoid << " -> "
+ << cid << " " << oid << dendl;
+ CollectionRef c = get_collection(cid);
+ if (!c)
+ return -ENOENT;
+ CollectionRef oc = get_collection(oldcid);
+ if (!oc)
+ return -ENOENT;
+ RWLock::WLocker l1(MIN(c, oc)->lock);
+ RWLock::WLocker l2(MAX(c, oc)->lock);
+
+ if (c->object_hash.count(oid))
+ return -EEXIST;
+ if (oc->object_hash.count(oldoid) == 0)
+ return -ENOENT;
+ ObjectRef o = oc->object_hash[oldoid];
+ c->object_map[oid] = o;
+ c->object_hash[oid] = o;
+ oc->object_map.erase(oldoid);
+ oc->object_hash.erase(oldoid);
+ return 0;
+}
+
+int MemStore::_collection_setattr(coll_t cid, const char *name,
+ const void *value, size_t size)
+{
+ dout(10) << __func__ << " " << cid << " " << name << dendl;
+ ceph::unordered_map<coll_t,CollectionRef>::iterator cp = coll_map.find(cid);
+ if (cp == coll_map.end())
+ return -ENOENT;
+ RWLock::WLocker l(cp->second->lock);
+
+ cp->second->xattr[name] = bufferptr((const char *)value, size);
+ return 0;
+}
+
+int MemStore::_collection_setattrs(coll_t cid, map<string,bufferptr> &aset)
+{
+ dout(10) << __func__ << " " << cid << dendl;
+ ceph::unordered_map<coll_t,CollectionRef>::iterator cp = coll_map.find(cid);
+ if (cp == coll_map.end())
+ return -ENOENT;
+ RWLock::WLocker l(cp->second->lock);
+
+ for (map<string,bufferptr>::const_iterator p = aset.begin();
+ p != aset.end();
+ ++p) {
+ cp->second->xattr[p->first] = p->second;
+ }
+ return 0;
+}
+
+int MemStore::_collection_rmattr(coll_t cid, const char *name)
+{
+ dout(10) << __func__ << " " << cid << " " << name << dendl;
+ ceph::unordered_map<coll_t,CollectionRef>::iterator cp = coll_map.find(cid);
+ if (cp == coll_map.end())
+ return -ENOENT;
+ RWLock::WLocker l(cp->second->lock);
+
+ if (cp->second->xattr.count(name) == 0)
+ return -ENODATA;
+ cp->second->xattr.erase(name);
+ return 0;
+}
+
+int MemStore::_collection_rename(const coll_t &cid, const coll_t &ncid)
+{
+ dout(10) << __func__ << " " << cid << " -> " << ncid << dendl;
+ RWLock::WLocker l(coll_lock);
+ if (coll_map.count(cid) == 0)
+ return -ENOENT;
+ if (coll_map.count(ncid))
+ return -EEXIST;
+ coll_map[ncid] = coll_map[cid];
+ coll_map.erase(cid);
+ return 0;
+}
+
+int MemStore::_split_collection(coll_t cid, uint32_t bits, uint32_t match,
+ coll_t dest)
+{
+ dout(10) << __func__ << " " << cid << " " << bits << " " << match << " "
+ << dest << dendl;
+ CollectionRef sc = get_collection(cid);
+ if (!sc)
+ return -ENOENT;
+ CollectionRef dc = get_collection(dest);
+ if (!dc)
+ return -ENOENT;
+ RWLock::WLocker l1(MIN(sc, dc)->lock);
+ RWLock::WLocker l2(MAX(sc, dc)->lock);
+
+ map<ghobject_t,ObjectRef>::iterator p = sc->object_map.begin();
+ while (p != sc->object_map.end()) {
+ if (p->first.match(bits, match)) {
+ dout(20) << " moving " << p->first << dendl;
+ dc->object_map.insert(make_pair(p->first, p->second));
+ dc->object_hash.insert(make_pair(p->first, p->second));
+ sc->object_hash.erase(p->first);
+ sc->object_map.erase(p++);
+ } else {
+ ++p;
+ }
+ }
+
+ return 0;
+}
diff --git a/src/os/MemStore.h b/src/os/MemStore.h
new file mode 100644
index 0000000..c33dd04
--- /dev/null
+++ b/src/os/MemStore.h
@@ -0,0 +1,357 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013- Sage Weil <sage at inktank.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+
+#ifndef CEPH_MEMSTORE_H
+#define CEPH_MEMSTORE_H
+
+#include "include/assert.h"
+#include "include/unordered_map.h"
+#include "include/memory.h"
+#include "common/Finisher.h"
+#include "common/RWLock.h"
+#include "ObjectStore.h"
+
+class MemStore : public ObjectStore {
+public:
+ struct Object {
+ bufferlist data;
+ map<string,bufferptr> xattr;
+ bufferlist omap_header;
+ map<string,bufferlist> omap;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(data, bl);
+ ::encode(xattr, bl);
+ ::encode(omap_header, bl);
+ ::encode(omap, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& p) {
+ DECODE_START(1, p);
+ ::decode(data, p);
+ ::decode(xattr, p);
+ ::decode(omap_header, p);
+ ::decode(omap, p);
+ DECODE_FINISH(p);
+ }
+ void dump(Formatter *f) const {
+ f->dump_int("data_len", data.length());
+ f->dump_int("omap_header_len", omap_header.length());
+
+ f->open_array_section("xattrs");
+ for (map<string,bufferptr>::const_iterator p = xattr.begin();
+ p != xattr.end();
+ ++p) {
+ f->open_object_section("xattr");
+ f->dump_string("name", p->first);
+ f->dump_int("length", p->second.length());
+ f->close_section();
+ }
+ f->close_section();
+
+ f->open_array_section("omap");
+ for (map<string,bufferlist>::const_iterator p = omap.begin();
+ p != omap.end();
+ ++p) {
+ f->open_object_section("pair");
+ f->dump_string("key", p->first);
+ f->dump_int("length", p->second.length());
+ f->close_section();
+ }
+ f->close_section();
+ }
+ };
+ typedef ceph::shared_ptr<Object> ObjectRef;
+
+ struct Collection {
+ ceph::unordered_map<ghobject_t, ObjectRef> object_hash; ///< for lookup
+ map<ghobject_t, ObjectRef> object_map; ///< for iteration
+ map<string,bufferptr> xattr;
+ RWLock lock; ///< for object_{map,hash}
+
+ // NOTE: The lock only needs to protect the object_map/hash, not the
+ // contents of individual objects. The osd is already sequencing
+ // reads and writes, so we will never see them concurrently at this
+ // level.
+
+ ObjectRef get_object(ghobject_t oid) {
+ ceph::unordered_map<ghobject_t,ObjectRef>::iterator o = object_hash.find(oid);
+ if (o == object_hash.end())
+ return ObjectRef();
+ return o->second;
+ }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(xattr, bl);
+ uint32_t s = object_map.size();
+ ::encode(s, bl);
+ for (map<ghobject_t, ObjectRef>::const_iterator p = object_map.begin();
+ p != object_map.end();
+ ++p) {
+ ::encode(p->first, bl);
+ p->second->encode(bl);
+ }
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& p) {
+ DECODE_START(1, p);
+ ::decode(xattr, p);
+ uint32_t s;
+ ::decode(s, p);
+ while (s--) {
+ ghobject_t k;
+ ::decode(k, p);
+ ObjectRef o(new Object);
+ o->decode(p);
+ object_map.insert(make_pair(k, o));
+ object_hash.insert(make_pair(k, o));
+ }
+ DECODE_FINISH(p);
+ }
+
+ Collection() : lock("MemStore::Collection::lock") {}
+ };
+ typedef ceph::shared_ptr<Collection> CollectionRef;
+
+private:
+ class OmapIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+ CollectionRef c;
+ ObjectRef o;
+ map<string,bufferlist>::iterator it;
+ public:
+ OmapIteratorImpl(CollectionRef c, ObjectRef o)
+ : c(c), o(o), it(o->omap.begin()) {}
+
+ int seek_to_first() {
+ RWLock::RLocker l(c->lock);
+ it = o->omap.begin();
+ return 0;
+ }
+ int upper_bound(const string &after) {
+ RWLock::RLocker l(c->lock);
+ it = o->omap.upper_bound(after);
+ return 0;
+ }
+ int lower_bound(const string &to) {
+ RWLock::RLocker l(c->lock);
+ it = o->omap.lower_bound(to);
+ return 0;
+ }
+ bool valid() {
+ RWLock::RLocker l(c->lock);
+ return it != o->omap.end();
+ }
+ int next() {
+ RWLock::RLocker l(c->lock);
+ ++it;
+ return 0;
+ }
+ string key() {
+ RWLock::RLocker l(c->lock);
+ return it->first;
+ }
+ bufferlist value() {
+ RWLock::RLocker l(c->lock);
+ return it->second;
+ }
+ int status() {
+ return 0;
+ }
+ };
+
+
+ ceph::unordered_map<coll_t, CollectionRef> coll_map;
+ RWLock coll_lock; ///< rwlock to protect coll_map
+ Mutex apply_lock; ///< serialize all updates
+
+ CollectionRef get_collection(coll_t cid);
+
+ Finisher finisher;
+
+ void _do_transaction(Transaction& t);
+
+ void _write_into_bl(const bufferlist& src, unsigned offset, bufferlist *dst);
+
+ int _touch(coll_t cid, const ghobject_t& oid);
+ int _write(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, const bufferlist& bl,
+ bool replica = false);
+ int _zero(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len);
+ int _truncate(coll_t cid, const ghobject_t& oid, uint64_t size);
+ int _remove(coll_t cid, const ghobject_t& oid);
+ int _setattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>& aset);
+ int _rmattr(coll_t cid, const ghobject_t& oid, const char *name);
+ int _rmattrs(coll_t cid, const ghobject_t& oid);
+ int _clone(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid);
+ int _clone_range(coll_t cid, const ghobject_t& oldoid,
+ const ghobject_t& newoid,
+ uint64_t srcoff, uint64_t len, uint64_t dstoff);
+ int _omap_clear(coll_t cid, const ghobject_t &oid);
+ int _omap_setkeys(coll_t cid, const ghobject_t &oid,
+ const map<string, bufferlist> &aset);
+ int _omap_rmkeys(coll_t cid, const ghobject_t &oid, const set<string> &keys);
+ int _omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+ const string& first, const string& last);
+ int _omap_setheader(coll_t cid, const ghobject_t &oid, const bufferlist &bl);
+
+ int _create_collection(coll_t c);
+ int _destroy_collection(coll_t c);
+ int _collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid);
+ int _collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+ coll_t cid, const ghobject_t& o);
+ int _collection_setattr(coll_t cid, const char *name, const void *value,
+ size_t size);
+ int _collection_setattrs(coll_t cid, map<string,bufferptr> &aset);
+ int _collection_rmattr(coll_t cid, const char *name);
+ int _collection_rename(const coll_t &cid, const coll_t &ncid);
+ int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest);
+
+ int _save();
+ int _load();
+
+ void dump(Formatter *f);
+ void dump_all();
+
+public:
+ MemStore(CephContext *cct, const string& path)
+ : ObjectStore(path),
+ coll_lock("MemStore::coll_lock"),
+ apply_lock("MemStore::apply_lock"),
+ finisher(cct) { }
+ ~MemStore() { }
+
+ int update_version_stamp() {
+ return 0;
+ }
+ uint32_t get_target_version() {
+ return 1;
+ }
+
+ int peek_journal_fsid(uuid_d *fsid);
+
+ bool test_mount_in_use() {
+ return false;
+ }
+
+ int mount();
+ int umount();
+
+ int get_max_object_name_length() {
+ return 4096;
+ }
+
+ int mkfs();
+ int mkjournal() {
+ return 0;
+ }
+
+ void set_allow_sharded_objects() {
+ }
+ bool get_allow_sharded_objects() {
+ return true;
+ }
+
+ int statfs(struct statfs *buf);
+
+ bool exists(coll_t cid, const ghobject_t& oid);
+ int stat(
+ coll_t cid,
+ const ghobject_t& oid,
+ struct stat *st,
+ bool allow_eio = false); // struct stat?
+ int read(
+ coll_t cid,
+ const ghobject_t& oid,
+ uint64_t offset,
+ size_t len,
+ bufferlist& bl,
+ bool allow_eio = false);
+ int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, bufferlist& bl);
+ int getattr(coll_t cid, const ghobject_t& oid, const char *name, bufferptr& value);
+ int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>& aset, bool user_only = false);
+
+ int list_collections(vector<coll_t>& ls);
+ bool collection_exists(coll_t c);
+ int collection_getattr(coll_t cid, const char *name,
+ void *value, size_t size);
+ int collection_getattr(coll_t cid, const char *name, bufferlist& bl);
+ int collection_getattrs(coll_t cid, map<string,bufferptr> &aset);
+ bool collection_empty(coll_t c);
+ int collection_list(coll_t cid, vector<ghobject_t>& o);
+ int collection_list_partial(coll_t cid, ghobject_t start,
+ int min, int max, snapid_t snap,
+ vector<ghobject_t> *ls, ghobject_t *next);
+ int collection_list_range(coll_t cid, ghobject_t start, ghobject_t end,
+ snapid_t seq, vector<ghobject_t> *ls);
+
+ int omap_get(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ bufferlist *header, ///< [out] omap header
+ map<string, bufferlist> *out /// < [out] Key to value map
+ );
+
+ /// Get omap header
+ int omap_get_header(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ bufferlist *header, ///< [out] omap header
+ bool allow_eio = false ///< [in] don't assert on eio
+ );
+
+ /// Get keys defined on oid
+ int omap_get_keys(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ set<string> *keys ///< [out] Keys defined on oid
+ );
+
+ /// Get key values
+ int omap_get_values(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ const set<string> &keys, ///< [in] Keys to get
+ map<string, bufferlist> *out ///< [out] Returned keys and values
+ );
+
+ /// Filters keys into out which are defined on oid
+ int omap_check_keys(
+ coll_t cid, ///< [in] Collection containing oid
+ const ghobject_t &oid, ///< [in] Object containing omap
+ const set<string> &keys, ///< [in] Keys to check
+ set<string> *out ///< [out] Subset of keys defined on oid
+ );
+
+ ObjectMap::ObjectMapIterator get_omap_iterator(
+ coll_t cid, ///< [in] collection
+ const ghobject_t &oid ///< [in] object
+ );
+
+ void set_fsid(uuid_d u);
+ uuid_d get_fsid();
+
+ objectstore_perf_stat_t get_cur_stats();
+
+ int queue_transactions(
+ Sequencer *osr, list<Transaction*>& tls,
+ TrackedOpRef op = TrackedOpRef(),
+ ThreadPool::TPHandle *handle = NULL);
+};
+
+
+
+
+#endif
diff --git a/src/os/ObjectMap.h b/src/os/ObjectMap.h
index 7717aac..86f9e3e 100644
--- a/src/os/ObjectMap.h
+++ b/src/os/ObjectMap.h
@@ -19,7 +19,7 @@
#include "SequencerPosition.h"
#include <string>
#include <vector>
-#include <tr1/memory>
+#include "include/memory.h"
/**
* Encapsulates the FileStore key value store
@@ -54,13 +54,19 @@ public:
const SequencerPosition *spos=0 ///< [in] sequencer position
) = 0;
- /// Clear all map keys and values from oid
+ /// Clear all map keys and values in to_clear from oid
virtual int rm_keys(
const ghobject_t &oid, ///< [in] object containing map
const set<string> &to_clear, ///< [in] Keys to clear
const SequencerPosition *spos=0 ///< [in] sequencer position
) = 0;
+ /// Clear all omap keys and the header
+ virtual int clear_keys_header(
+ const ghobject_t &oid, ///< [in] oid to clear
+ const SequencerPosition *spos=0 ///< [in] sequencer position
+ ) = 0;
+
/// Get all keys and values
virtual int get(
const ghobject_t &oid, ///< [in] object containing map
@@ -98,7 +104,7 @@ public:
/// Get all xattrs
virtual int get_all_xattrs(
const ghobject_t &oid, ///< [in] object
- set<string> *out ///< [out] attrs and values
+ set<string> *out ///< [out] attrs and values
) = 0;
/// set xattrs in to_set
@@ -143,7 +149,7 @@ public:
virtual int status() = 0;
virtual ~ObjectMapIteratorImpl() {}
};
- typedef std::tr1::shared_ptr<ObjectMapIteratorImpl> ObjectMapIterator;
+ typedef ceph::shared_ptr<ObjectMapIteratorImpl> ObjectMapIterator;
virtual ObjectMapIterator get_iterator(const ghobject_t &oid) {
return ObjectMapIterator();
}
diff --git a/src/os/ObjectStore.cc b/src/os/ObjectStore.cc
index 1a1bbcb..7496c45 100644
--- a/src/os/ObjectStore.cc
+++ b/src/os/ObjectStore.cc
@@ -11,11 +11,63 @@
* Foundation. See file COPYING.
*
*/
+#include <ctype.h>
#include <sstream>
-#include <tr1/memory>
+#include "include/memory.h"
#include "ObjectStore.h"
#include "common/Formatter.h"
#include "FileStore.h"
+#include "MemStore.h"
+#include "KeyValueStore.h"
+#include "common/safe_io.h"
+
+ObjectStore *ObjectStore::create(CephContext *cct,
+ const string& type,
+ const string& data,
+ const string& journal)
+{
+ if (type == "filestore") {
+ return new FileStore(data, journal);
+ }
+ if (type == "memstore") {
+ return new MemStore(cct, data);
+ }
+ if (type == "keyvaluestore-dev") {
+ return new KeyValueStore(data);
+ }
+ return NULL;
+}
+
+int ObjectStore::write_meta(const std::string& key,
+ const std::string& value)
+{
+ string v = value;
+ v += "\n";
+ int r = safe_write_file(path.c_str(), key.c_str(),
+ v.c_str(), v.length());
+ if (r < 0)
+ return r;
+ return 0;
+}
+
+int ObjectStore::read_meta(const std::string& key,
+ std::string *value)
+{
+ char buf[4096];
+ int r = safe_read_file(path.c_str(), key.c_str(),
+ buf, sizeof(buf));
+ if (r <= 0)
+ return r;
+ // drop trailing newlines
+ while (r && isspace(buf[r-1])) {
+ --r;
+ }
+ *value = string(buf, r);
+ return 0;
+}
+
+
+
ostream& operator<<(ostream& out, const ObjectStore::Sequencer& s)
{
@@ -504,7 +556,7 @@ int ObjectStore::collection_list(coll_t c, vector<hobject_t>& o)
int ret = collection_list(c, go);
if (ret == 0) {
o.reserve(go.size());
- for (vector<ghobject_t>::iterator i = go.begin(); i != go.end() ; i++)
+ for (vector<ghobject_t>::iterator i = go.begin(); i != go.end() ; ++i)
o.push_back(i->hobj);
}
return ret;
@@ -520,7 +572,7 @@ int ObjectStore::collection_list_partial(coll_t c, hobject_t start,
if (ret == 0) {
*next = gnext.hobj;
ls->reserve(go.size());
- for (vector<ghobject_t>::iterator i = go.begin(); i != go.end() ; i++)
+ for (vector<ghobject_t>::iterator i = go.begin(); i != go.end() ; ++i)
ls->push_back(i->hobj);
}
return ret;
@@ -534,7 +586,7 @@ int ObjectStore::collection_list_range(coll_t c, hobject_t start, hobject_t end,
int ret = collection_list_range(c, gstart, gend, seq, &go);
if (ret == 0) {
ls->reserve(go.size());
- for (vector<ghobject_t>::iterator i = go.begin(); i != go.end() ; i++)
+ for (vector<ghobject_t>::iterator i = go.begin(); i != go.end() ; ++i)
ls->push_back(i->hobj);
}
return ret;
diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index 6494290..ef17db4 100644
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -19,6 +19,7 @@
#include "include/types.h"
#include "osd/osd_types.h"
#include "common/TrackedOp.h"
+#include "common/WorkQueue.h"
#include "ObjectMap.h"
#include <errno.h>
@@ -31,6 +32,8 @@
#include <sys/vfs.h> /* or <sys/statfs.h> */
#endif /* DARWIN */
+class CephContext;
+
using std::vector;
using std::string;
@@ -44,44 +47,31 @@ namespace ceph {
class Logger;
-enum {
- l_os_first = 84000,
- l_os_jq_max_ops,
- l_os_jq_ops,
- l_os_j_ops,
- l_os_jq_max_bytes,
- l_os_jq_bytes,
- l_os_j_bytes,
- l_os_j_lat,
- l_os_j_wr,
- l_os_j_wr_bytes,
- l_os_oq_max_ops,
- l_os_oq_ops,
- l_os_ops,
- l_os_oq_max_bytes,
- l_os_oq_bytes,
- l_os_bytes,
- l_os_apply_lat,
- l_os_committing,
- l_os_commit,
- l_os_commit_len,
- l_os_commit_lat,
- l_os_j_full,
- l_os_queue_lat,
- l_os_last,
-};
-
static inline void encode(const map<string,bufferptr> *attrset, bufferlist &bl) {
::encode(*attrset, bl);
}
class ObjectStore {
+protected:
+ string path;
+
public:
+ /**
+ * create - create an ObjectStore instance
+ *
+ * @param type type of store
+ * @param data path (or other descriptor) for data
+ * @param journal path (or other descriptor) for journal (optional)
+ */
+ static ObjectStore *create(CephContext *cct,
+ const string& type,
+ const string& data,
+ const string& journal);
Logger *logger;
- virtual filestore_perf_stat_t get_cur_stats() = 0;
+ virtual objectstore_perf_stat_t get_cur_stats() = 0;
/**
* a sequencer orders transactions
@@ -352,7 +342,7 @@ public:
} else {
::decode(oid, p);
if (use_pool_override && pool_override != -1 &&
- oid.hobj.pool == -1) {
+ !oid.hobj.is_max() && oid.hobj.pool == -1) {
oid.hobj.pool = pool_override;
}
}
@@ -565,6 +555,7 @@ public:
::encode(oldoid, tbl);
::encode(cid, tbl);
::encode(oid, tbl);
+ ops++;
}
void collection_setattr(coll_t cid, const char* name, bufferlist& val) {
@@ -793,34 +784,40 @@ public:
}
unsigned apply_transactions(Sequencer *osr, list<Transaction*>& tls, Context *ondisk=0);
- int queue_transaction(Sequencer *osr, Transaction* t) {
+ int queue_transaction_and_cleanup(Sequencer *osr, Transaction* t,
+ ThreadPool::TPHandle *handle = NULL) {
list<Transaction *> tls;
tls.push_back(t);
- return queue_transactions(osr, tls, new C_DeleteTransaction(t));
+ return queue_transactions(osr, tls, new C_DeleteTransaction(t),
+ NULL, NULL, TrackedOpRef(), handle);
}
int queue_transaction(Sequencer *osr, Transaction *t, Context *onreadable, Context *ondisk=0,
Context *onreadable_sync=0,
- TrackedOpRef op = TrackedOpRef()) {
+ TrackedOpRef op = TrackedOpRef(),
+ ThreadPool::TPHandle *handle = NULL) {
list<Transaction*> tls;
tls.push_back(t);
- return queue_transactions(osr, tls, onreadable, ondisk, onreadable_sync, op);
+ return queue_transactions(osr, tls, onreadable, ondisk, onreadable_sync,
+ op, handle);
}
int queue_transactions(Sequencer *osr, list<Transaction*>& tls,
Context *onreadable, Context *ondisk=0,
Context *onreadable_sync=0,
- TrackedOpRef op = TrackedOpRef()) {
+ TrackedOpRef op = TrackedOpRef(),
+ ThreadPool::TPHandle *handle = NULL) {
assert(!tls.empty());
tls.back()->register_on_applied(onreadable);
tls.back()->register_on_commit(ondisk);
tls.back()->register_on_applied_sync(onreadable_sync);
- return queue_transactions(osr, tls, op);
+ return queue_transactions(osr, tls, op, handle);
}
virtual int queue_transactions(
Sequencer *osr, list<Transaction*>& tls,
- TrackedOpRef op = TrackedOpRef()) = 0;
+ TrackedOpRef op = TrackedOpRef(),
+ ThreadPool::TPHandle *handle = NULL) = 0;
int queue_transactions(
@@ -847,9 +844,13 @@ public:
}
public:
- ObjectStore() : logger(NULL) {}
+ ObjectStore(const std::string& path_) : path(path_), logger(NULL) {}
virtual ~ObjectStore() {}
+ // no copying
+ ObjectStore(const ObjectStore& o);
+ const ObjectStore& operator=(const ObjectStore& o);
+
// mgmt
virtual int version_stamp_is_valid(uint32_t *version) { return 1; }
virtual int update_version_stamp() = 0;
@@ -865,6 +866,47 @@ public:
virtual int statfs(struct statfs *buf) = 0;
/**
+ * get the most recent "on-disk format version" supported
+ */
+ virtual uint32_t get_target_version() = 0;
+
+ /**
+ * check the journal uuid/fsid, without opening
+ */
+ virtual int peek_journal_fsid(uuid_d *fsid) = 0;
+
+ /**
+ * write_meta - write a simple configuration key out-of-band
+ *
+ * Write a simple key/value pair for basic store configuration
+ * (e.g., a uuid or magic number) to an unopened/unmounted store.
+ * The default implementation writes this to a plaintext file in the
+ * path.
+ *
+ * A newline is appended.
+ *
+ * @param key key name (e.g., "fsid")
+ * @param value value (e.g., a uuid rendered as a string)
+ * @returns 0 for success, or an error code
+ */
+ virtual int write_meta(const std::string& key,
+ const std::string& value);
+
+ /**
+ * read_meta - read a simple configuration key out-of-band
+ *
+ * Read a simple key value to an unopened/mounted store.
+ *
+ * Trailing whitespace is stripped off.
+ *
+ * @param key key name
+ * @param value pointer to value string
+ * @returns 0 for success, or an error code
+ */
+ virtual int read_meta(const std::string& key,
+ std::string *value);
+
+ /**
* get ideal min value for collection_list_partial()
*
* default to some arbitrary values; the implementation will override.
@@ -904,7 +946,7 @@ public:
value.push_back(bp);
return r;
}
- virtual int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>& aset, bool user_only = false) {return 0;};
+ virtual int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferptr>& aset, bool user_only = false) = 0;
int getattrs(coll_t cid, const ghobject_t& oid, map<string,bufferlist>& aset, bool user_only = false) {
map<string,bufferptr> bmap;
int r = getattrs(cid, oid, bmap, user_only);
@@ -1040,8 +1082,6 @@ public:
virtual void inject_data_error(const ghobject_t &oid) {}
virtual void inject_mdata_error(const ghobject_t &oid) {}
};
-
-
WRITE_CLASS_ENCODER(ObjectStore::Transaction)
ostream& operator<<(ostream& out, const ObjectStore::Sequencer& s);
diff --git a/src/os/WBThrottle.cc b/src/os/WBThrottle.cc
index e02c176..0354ceb 100644
--- a/src/os/WBThrottle.cc
+++ b/src/os/WBThrottle.cc
@@ -1,6 +1,8 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
+#include "acconfig.h"
+
#include "os/WBThrottle.h"
#include "common/perf_counters.h"
@@ -8,7 +10,7 @@ WBThrottle::WBThrottle(CephContext *cct) :
cur_ios(0), cur_size(0),
cct(cct),
logger(NULL),
- stopping(false),
+ stopping(true),
lock("WBThrottle::lock", false, true, false, cct),
fs(XFS)
{
@@ -32,20 +34,33 @@ WBThrottle::WBThrottle(CephContext *cct) :
logger->set(i, 0);
cct->_conf->add_observer(this);
- create();
}
WBThrottle::~WBThrottle() {
assert(cct);
+ cct->get_perfcounters_collection()->remove(logger);
+ delete logger;
+ cct->_conf->remove_observer(this);
+}
+
+void WBThrottle::start()
+{
+ {
+ Mutex::Locker l(lock);
+ stopping = false;
+ }
+ create();
+}
+
+void WBThrottle::stop()
+{
{
Mutex::Locker l(lock);
stopping = true;
cond.Signal();
}
+
join();
- cct->get_perfcounters_collection()->remove(logger);
- delete logger;
- cct->_conf->remove_observer(this);
}
const char** WBThrottle::get_tracked_conf_keys() const
@@ -145,9 +160,17 @@ void *WBThrottle::entry()
while (get_next_should_flush(&wb)) {
clearing = wb.get<0>();
lock.Unlock();
+#ifdef HAVE_FDATASYNC
::fdatasync(**wb.get<1>());
- if (wb.get<2>().nocache)
- posix_fadvise(**wb.get<1>(), 0, 0, POSIX_FADV_DONTNEED);
+#else
+ ::fsync(**wb.get<1>());
+#endif
+#ifdef HAVE_POSIX_FADVISE
+ if (wb.get<2>().nocache) {
+ int fa_r = posix_fadvise(**wb.get<1>(), 0, 0, POSIX_FADV_DONTNEED);
+ assert(fa_r == 0);
+ }
+#endif
lock.Lock();
clearing = ghobject_t();
cur_ios -= wb.get<2>().ios;
diff --git a/src/os/WBThrottle.h b/src/os/WBThrottle.h
index e418cf9..f643e39 100644
--- a/src/os/WBThrottle.h
+++ b/src/os/WBThrottle.h
@@ -17,7 +17,7 @@
#include <map>
#include <boost/tuple/tuple.hpp>
-#include <tr1/memory>
+#include "include/memory.h"
#include "include/buffer.h"
#include "common/Formatter.h"
#include "common/hobject.h"
@@ -134,6 +134,8 @@ public:
WBThrottle(CephContext *cct);
~WBThrottle();
+ void start();
+ void stop();
/// Set fs as XFS or BTRFS
void set_fs(FS new_fs) {
Mutex::Locker l(lock);
diff --git a/src/osd/ErasureCodeInterface.h b/src/osd/ErasureCodeInterface.h
index 656ee91..9a44be7 100644
--- a/src/osd/ErasureCodeInterface.h
+++ b/src/osd/ErasureCodeInterface.h
@@ -25,12 +25,57 @@
are systematic (i.e. the data is not mangled and can be
reconstructed by concatenating chunks ).
- All methods return **0** on success and a negative value on
- error. If the value returned on error is not explained in
- **ErasureCodeInterface**, the sources or the documentation of the
- interface implementer (i.e. the plugin ) must be read to figure
- out what it means. It is recommended that each error code matches
- an *errno* value that relates to the cause of the error.
+ Methods returning an **int** return **0** on success and a
+ negative value on error. If the value returned on error is not
+ explained in **ErasureCodeInterface**, the sources or the
+ documentation of the interface implementer (i.e. the plugin ) must
+ be read to figure out what it means. It is recommended that each
+ error code matches an *errno* value that relates to the cause of
+ the error.
+
+ If an object is small enough, the caller can process it with
+ one call to the **encode** or **decode** method.
+
+ +---------------- coded object O -------------------------+
+ |+----------------+ +----------------+ +----------------+ |
+ || chunk 0 | | chunk 1 | | chunk 2 | |
+ || [0,N) | | [N,2N) | | [2N,3N) | |
+ |+----------------+ +----------------+ +----------------+ |
+ +------^--------------------------------------------------+
+ |
+ chunk B / C | offset B % C ( where C is the chunk size )
+ |
+ +-----^---- raw object O ----+------+
+ | B [0,X) | pad |
+ +----------------------------+------+
+
+ The object size is paded so that each chunks are of the same size.
+ In the example above, if the actual object size was X, then it
+ will be padded to 2N >= X assuming there are two data chunks (0
+ and 1) and one coding chunk (2).
+
+ For chunks of size C, byte B of the object is found in chunk number
+ B / C at offset B % C.
+
+ If an object is too large to be encoded in memory, the caller
+ should divide it in smaller units named **stripes**.
+
+ +---------------------- object O -------------------------+
+ |+----------------+ +----------------+ +----------------+ |
+ stripe || chunk 0 | | chunk 1 | | chunk 2 | |
+ 0 || [0,N) | | [N,2N) | | [2N,3N) | |
+ |+----------------+ +----------------+ +----------------+ |
+ |+----------------+ +----------------+ +----------------+ |
+ stripe || chunk 0 | | chunk 1 | | chunk 2 | |
+ 1 || [X,M) | | [X+M,X+2M) | | [X+2M,X+3M) | |
+ || | | | | | |
+ |+----------------+ +----------------+ +----------------+ |
+ | ... |
+ +---------------------------------------------------------+
+
+ The interface does not concern itself with stripes nor does it
+ impose constraints on the size of each stripe. Variable names in
+ the interface always use **object** and never use **stripe**.
Assuming the interface implementer provides three data chunks ( K
= 3 ) and two coding chunks ( M = 2 ), a buffer could be encoded as
@@ -97,7 +142,7 @@
#include <map>
#include <set>
-#include <tr1/memory>
+#include "include/memory.h"
#include "include/buffer.h"
using namespace std;
@@ -109,6 +154,48 @@ namespace ceph {
virtual ~ErasureCodeInterface() {}
/**
+ * Return the number of chunks created by a call to the **encode**
+ * method.
+ *
+ * In the simplest case it can be K + M, i.e. the number
+ * of data chunks (K) plus the number of parity chunks
+ * (M). However, if the implementation provides local parity there
+ * could be an additional overhead.
+ *
+ * @return the number of chunks created by encode()
+ */
+ virtual unsigned int get_chunk_count() const = 0;
+
+ /**
+ * Return the number of data chunks created by a call to the
+ * **encode** method. The data chunks contain the buffer provided
+ * to **encode**, verbatim, with padding at the end of the last
+ * chunk.
+ *
+ * @return the number of data chunks created by encode()
+ */
+ virtual unsigned int get_data_chunk_count() const = 0;
+
+ /**
+ * Return the size (in bytes) of a single chunk created by a call
+ * to the **decode** method. The returned size multiplied by
+ * **get_chunk_count()** is greater or equal to **object_size**.
+ *
+ * If the object size is properly aligned, the chunk size is
+ * **object_size / get_chunk_count()**. However, if
+ * **object_size** is not a multiple of **get_chunk_count** or if
+ * the implementation imposes additional alignment constraints,
+ * the chunk size may be larger.
+ *
+ * The byte found at offset **B** of the original object is mapped
+ * to chunk **B / get_chunk_size()** at offset **B % get_chunk_size()**.
+ *
+ * @param [in] object_size the number of bytes of the object to **encode()**
+ * @return the size (in bytes) of a single chunk created by **encode()**
+ */
+ virtual unsigned int get_chunk_size(unsigned int object_size) const = 0;
+
+ /**
* Compute the smallest subset of **available** chunks that needs
* to be retrieved in order to successfully decode
* **want_to_read** chunks.
@@ -231,9 +318,32 @@ namespace ceph {
virtual int decode(const set<int> &want_to_read,
const map<int, bufferlist> &chunks,
map<int, bufferlist> *decoded) = 0;
+
+ /**
+ * Decode the first **get_data_chunk_count()** **chunks** and
+ * concatenate them them into **decoded**.
+ *
+ * Returns 0 on success.
+ *
+ * @param [in] chunks map chunk indexes to chunk data
+ * @param [out] decoded concatenante of the data chunks
+ * @return **0** on success or a negative errno on error.
+ */
+ int decode_concat(const map<int, bufferlist> &chunks,
+ bufferlist *decoded) {
+ set<int> want_to_read;
+ for (unsigned int i = 0; i < get_data_chunk_count(); i++)
+ want_to_read.insert(i);
+ map<int, bufferlist> decoded_map;
+ int r = decode(want_to_read, chunks, &decoded_map);
+ if (r == 0)
+ for (unsigned int i = 0; i < get_data_chunk_count(); i++)
+ decoded->claim_append(decoded_map[i]);
+ return r;
+ }
};
- typedef std::tr1::shared_ptr<ErasureCodeInterface> ErasureCodeInterfaceRef;
+ typedef ceph::shared_ptr<ErasureCodeInterface> ErasureCodeInterfaceRef;
}
diff --git a/src/osd/ErasureCodePlugin.cc b/src/osd/ErasureCodePlugin.cc
index d8b9ae0..d31b197 100644
--- a/src/osd/ErasureCodePlugin.cc
+++ b/src/osd/ErasureCodePlugin.cc
@@ -37,15 +37,19 @@ ErasureCodePluginRegistry ErasureCodePluginRegistry::singleton;
ErasureCodePluginRegistry::ErasureCodePluginRegistry() :
lock("ErasureCodePluginRegistry::lock"),
- loading(false)
+ loading(false),
+ disable_dlclose(false)
{
}
ErasureCodePluginRegistry::~ErasureCodePluginRegistry()
{
+ if (disable_dlclose)
+ return;
+
for (std::map<std::string,ErasureCodePlugin*>::iterator i = plugins.begin();
i != plugins.end();
- i++) {
+ ++i) {
void *library = i->second->library;
delete i->second;
dlclose(library);
@@ -112,6 +116,7 @@ int ErasureCodePluginRegistry::load(const std::string &plugin_name,
if (r != 0) {
derr << "erasure_code_init(" << plugin_name
<< "): " << strerror(-r) << dendl;
+ dlclose(library);
return r;
}
} else {
diff --git a/src/osd/ErasureCodePlugin.h b/src/osd/ErasureCodePlugin.h
index a2feb71..1f5e46c 100644
--- a/src/osd/ErasureCodePlugin.h
+++ b/src/osd/ErasureCodePlugin.h
@@ -42,6 +42,7 @@ namespace ceph {
public:
Mutex lock;
bool loading;
+ bool disable_dlclose;
std::map<std::string,ErasureCodePlugin*> plugins;
static ErasureCodePluginRegistry singleton;
diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
index f2be1ed..f1de0e4 100644
--- a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
+++ b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc
@@ -18,6 +18,7 @@
#include <algorithm>
#include "common/debug.h"
#include "ErasureCodeJerasure.h"
+#include "vectorop.h"
extern "C" {
#include "jerasure.h"
#include "reed_sol.h"
@@ -42,6 +43,15 @@ void ErasureCodeJerasure::init(const map<std::string,std::string> ¶meters)
prepare();
}
+unsigned int ErasureCodeJerasure::get_chunk_size(unsigned int object_size) const
+{
+ unsigned alignment = get_alignment();
+ unsigned tail = object_size % alignment;
+ unsigned padded_length = object_size + ( tail ? ( alignment - tail ) : 0 );
+ assert(padded_length % k == 0);
+ return padded_length / k;
+}
+
int ErasureCodeJerasure::minimum_to_decode(const set<int> &want_to_read,
const set<int> &available_chunks,
set<int> *minimum)
@@ -54,7 +64,7 @@ int ErasureCodeJerasure::minimum_to_decode(const set<int> &want_to_read,
return -EIO;
set<int>::iterator i;
unsigned j;
- for (i = available_chunks.begin(), j = 0; j < (unsigned)k; i++, j++)
+ for (i = available_chunks.begin(), j = 0; j < (unsigned)k; ++i, j++)
minimum->insert(*i);
}
return 0;
@@ -67,7 +77,7 @@ int ErasureCodeJerasure::minimum_to_decode_with_cost(const set<int> &want_to_rea
set <int> available_chunks;
for (map<int, int>::const_iterator i = available.begin();
i != available.end();
- i++)
+ ++i)
available_chunks.insert(i->first);
return minimum_to_decode(want_to_read, available_chunks, minimum);
}
@@ -76,18 +86,21 @@ int ErasureCodeJerasure::encode(const set<int> &want_to_encode,
const bufferlist &in,
map<int, bufferlist> *encoded)
{
- unsigned alignment = get_alignment();
- unsigned tail = in.length() % alignment;
- unsigned padded_length = in.length() + ( tail ? ( alignment - tail ) : 0 );
+ unsigned blocksize = get_chunk_size(in.length());
+ unsigned padded_length = blocksize * k;
dout(10) << "encode adjusted buffer length from " << in.length()
<< " to " << padded_length << dendl;
assert(padded_length % k == 0);
- unsigned blocksize = padded_length / k;
- unsigned length = blocksize * ( k + m );
bufferlist out(in);
- bufferptr pad(length - in.length());
- pad.zero(0, padded_length - in.length());
- out.push_back(pad);
+ if (padded_length - in.length() > 0) {
+ bufferptr pad(padded_length - in.length());
+ pad.zero();
+ out.push_back(pad);
+ out.rebuild_page_aligned();
+ }
+ unsigned coding_length = blocksize * m;
+ bufferptr coding(buffer::create_page_aligned(coding_length));
+ out.push_back(coding);
char *chunks[k + m];
for (int i = 0; i < k + m; i++) {
bufferlist &chunk = (*encoded)[i];
@@ -190,9 +203,13 @@ int ErasureCodeJerasureReedSolomonVandermonde::jerasure_decode(int *erasures,
erasures, data, coding, blocksize);
}
-unsigned ErasureCodeJerasureReedSolomonVandermonde::get_alignment()
+unsigned ErasureCodeJerasureReedSolomonVandermonde::get_alignment() const
{
- return k*w*sizeof(int);
+ unsigned alignment = k*w*sizeof(int);
+ if ( ((w*sizeof(int))%LARGEST_VECTOR_WORDSIZE) )
+ alignment = k*w*LARGEST_VECTOR_WORDSIZE;
+ return alignment;
+
}
void ErasureCodeJerasureReedSolomonVandermonde::parse(const map<std::string,std::string> ¶meters)
@@ -230,9 +247,12 @@ int ErasureCodeJerasureReedSolomonRAID6::jerasure_decode(int *erasures,
return jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, blocksize);
}
-unsigned ErasureCodeJerasureReedSolomonRAID6::get_alignment()
+unsigned ErasureCodeJerasureReedSolomonRAID6::get_alignment() const
{
- return k*w*sizeof(int);
+ unsigned alignment = k*w*sizeof(int);
+ if ( ((w*sizeof(int))%LARGEST_VECTOR_WORDSIZE) )
+ alignment = k*w*LARGEST_VECTOR_WORDSIZE;
+ return alignment;
}
void ErasureCodeJerasureReedSolomonRAID6::parse(const map<std::string,std::string> ¶meters)
@@ -272,9 +292,12 @@ int ErasureCodeJerasureCauchy::jerasure_decode(int *erasures,
erasures, data, coding, blocksize, packetsize, 1);
}
-unsigned ErasureCodeJerasureCauchy::get_alignment()
+unsigned ErasureCodeJerasureCauchy::get_alignment() const
{
- return k*w*packetsize*sizeof(int);
+ unsigned alignment = k*w*packetsize*sizeof(int);
+ if ( ((w*packetsize*sizeof(int))%LARGEST_VECTOR_WORDSIZE) )
+ alignment = k*w*packetsize*LARGEST_VECTOR_WORDSIZE;
+ return alignment;
}
void ErasureCodeJerasureCauchy::parse(const map<std::string,std::string> ¶meters)
@@ -339,9 +362,12 @@ int ErasureCodeJerasureLiberation::jerasure_decode(int *erasures,
coding, blocksize, packetsize, 1);
}
-unsigned ErasureCodeJerasureLiberation::get_alignment()
+unsigned ErasureCodeJerasureLiberation::get_alignment() const
{
- return k*w*packetsize*sizeof(int);
+ unsigned alignment = k*w*packetsize*sizeof(int);
+ if ( ((w*packetsize*sizeof(int))%LARGEST_VECTOR_WORDSIZE) )
+ alignment = k*w*packetsize*LARGEST_VECTOR_WORDSIZE;
+ return alignment;
}
void ErasureCodeJerasureLiberation::parse(const map<std::string,std::string> ¶meters)
diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h
index fc76ed7..a0db6a0 100644
--- a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h
+++ b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h
@@ -32,6 +32,16 @@ public:
virtual ~ErasureCodeJerasure() {}
+ virtual unsigned int get_chunk_count() const {
+ return k + m;
+ }
+
+ virtual unsigned int get_data_chunk_count() const {
+ return k;
+ }
+
+ virtual unsigned int get_chunk_size(unsigned int object_size) const;
+
virtual int minimum_to_decode(const set<int> &want_to_read,
const set<int> &available_chunks,
set<int> *minimum);
@@ -56,7 +66,7 @@ public:
char **data,
char **coding,
int blocksize) = 0;
- virtual unsigned get_alignment() = 0;
+ virtual unsigned get_alignment() const = 0;
virtual void parse(const map<std::string,std::string> ¶meters) = 0;
virtual void prepare() = 0;
static int to_int(const std::string &name,
@@ -88,7 +98,7 @@ public:
char **data,
char **coding,
int blocksize);
- virtual unsigned get_alignment();
+ virtual unsigned get_alignment() const;
virtual void parse(const map<std::string,std::string> ¶meters);
virtual void prepare();
};
@@ -115,7 +125,7 @@ public:
char **data,
char **coding,
int blocksize);
- virtual unsigned get_alignment();
+ virtual unsigned get_alignment() const;
virtual void parse(const map<std::string,std::string> ¶meters);
virtual void prepare();
};
@@ -125,7 +135,7 @@ public:
static const int DEFAULT_K = 7;
static const int DEFAULT_M = 3;
static const int DEFAULT_W = 8;
- static const int DEFAULT_PACKETSIZE = 8;
+ static const int DEFAULT_PACKETSIZE = 2048;
int *bitmatrix;
int **schedule;
int packetsize;
@@ -149,7 +159,7 @@ public:
char **data,
char **coding,
int blocksize);
- virtual unsigned get_alignment();
+ virtual unsigned get_alignment() const;
virtual void parse(const map<std::string,std::string> ¶meters);
void prepare_schedule(int *matrix);
};
@@ -177,7 +187,7 @@ public:
static const int DEFAULT_K = 2;
static const int DEFAULT_M = 2;
static const int DEFAULT_W = 7;
- static const int DEFAULT_PACKETSIZE = 8;
+ static const int DEFAULT_PACKETSIZE = 2048;
int *bitmatrix;
int **schedule;
int packetsize;
@@ -196,7 +206,7 @@ public:
char **data,
char **coding,
int blocksize);
- virtual unsigned get_alignment();
+ virtual unsigned get_alignment() const;
virtual void parse(const map<std::string,std::string> ¶meters);
virtual void prepare();
};
diff --git a/src/osd/ErasureCodePluginJerasure/Makefile.am b/src/osd/ErasureCodePluginJerasure/Makefile.am
index b31fb1c..51ef83b 100644
--- a/src/osd/ErasureCodePluginJerasure/Makefile.am
+++ b/src/osd/ErasureCodePluginJerasure/Makefile.am
@@ -7,16 +7,22 @@ libec_jerasure_la_SOURCES = \
osd/ErasureCodePluginJerasure/jerasure.c \
osd/ErasureCodePluginJerasure/liberation.c \
osd/ErasureCodePluginJerasure/reed_sol.c
+
noinst_HEADERS += \
osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \
osd/ErasureCodePluginJerasure/cauchy.h \
osd/ErasureCodePluginJerasure/galois.h \
osd/ErasureCodePluginJerasure/jerasure.h \
osd/ErasureCodePluginJerasure/liberation.h \
- osd/ErasureCodePluginJerasure/reed_sol.h
+ osd/ErasureCodePluginJerasure/reed_sol.h \
+ osd/ErasureCodePluginJerasure/vectorop.h
+
libec_jerasure_la_CFLAGS = ${AM_CFLAGS}
libec_jerasure_la_CXXFLAGS= ${AM_CXXFLAGS}
libec_jerasure_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__erasure_code_.*'
+libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0
+if LINUX
+libec_jerasure_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
+endif
erasure_codelib_LTLIBRARIES += libec_jerasure.la
diff --git a/src/osd/ErasureCodePluginJerasure/cauchy.c b/src/osd/ErasureCodePluginJerasure/cauchy.c
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/cauchy.h b/src/osd/ErasureCodePluginJerasure/cauchy.h
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/galois.c b/src/osd/ErasureCodePluginJerasure/galois.c
old mode 100755
new mode 100644
index 0de6fbd..9981353
--- a/src/osd/ErasureCodePluginJerasure/galois.c
+++ b/src/osd/ErasureCodePluginJerasure/galois.c
@@ -49,8 +49,11 @@ POSSIBILITY OF SUCH DAMAGE.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <assert.h>
+#include <stdint.h>
#include "galois.h"
+#include "vectorop.h"
#define NONE (10)
#define TABLE (11)
@@ -737,31 +740,40 @@ void galois_w32_region_multiply(char *region, /* Region to multiply */
}
}
return;
-
}
+#define is_aligned(POINTER, BYTE_COUNT) \
+ (((uintptr_t)(const void *)(POINTER)) % (BYTE_COUNT) == 0)
+
void galois_region_xor( char *r1, /* Region 1 */
char *r2, /* Region 2 */
char *r3, /* Sum region (r3 = r1 ^ r2) -- can be r1 or r2 */
int nbytes) /* Number of bytes in region */
{
- long *l1;
- long *l2;
- long *l3;
- long *ltop;
- char *ctop;
-
- ctop = r1 + nbytes;
- ltop = (long *) ctop;
- l1 = (long *) r1;
- l2 = (long *) r2;
- l3 = (long *) r3;
-
- while (l1 < ltop) {
- *l3 = ((*l1) ^ (*l2));
- l1++;
- l2++;
- l3++;
+ if (nbytes%VECTOR_WORDSIZE) {
+ assert(is_aligned(r1, sizeof(long)));
+ assert(is_aligned(r2, sizeof(long)));
+ assert(is_aligned(r3, sizeof(long)));
+ long* l1 = (long*)r1;
+ long* l2 = (long*)r2;
+ long* l3 = (long*)r3;
+ char *ctop = r1 + nbytes;
+ long* ltop = (long*)ctop;
+ while (l1 < ltop) {
+ *l3++ = ((*l1++) ^ (*l2++));
+ }
+ } else {
+ assert(is_aligned(r1, VECTOR_WORDSIZE));
+ assert(is_aligned(r2, VECTOR_WORDSIZE));
+ assert(is_aligned(r3, VECTOR_WORDSIZE));
+ vector_op_t* l1 = (vector_op_t*)r1;
+ vector_op_t* l2 = (vector_op_t*)r2;
+ vector_op_t* l3 = (vector_op_t*)r3;
+ char *ctop = r1 + nbytes;
+ vector_op_t* ltop = (vector_op_t*)ctop;
+ while (l1 < ltop) {
+ *l3++ = ((*l1++) ^ (*l2++));
+ }
}
}
diff --git a/src/osd/ErasureCodePluginJerasure/galois.h b/src/osd/ErasureCodePluginJerasure/galois.h
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/jerasure.c b/src/osd/ErasureCodePluginJerasure/jerasure.c
old mode 100755
new mode 100644
index d5752a8..e5df475
--- a/src/osd/ErasureCodePluginJerasure/jerasure.c
+++ b/src/osd/ErasureCodePluginJerasure/jerasure.c
@@ -839,7 +839,11 @@ static int **jerasure_generate_decoding_schedule(int k, int m, int w, int *bitma
row_ids = talloc(int, k+m);
ind_to_row = talloc(int, k+m);
- if (set_up_ids_for_scheduled_decoding(k, m, erasures, row_ids, ind_to_row) < 0) return NULL;
+ if (set_up_ids_for_scheduled_decoding(k, m, erasures, row_ids, ind_to_row) < 0) {
+ free(row_ids);
+ free(ind_to_row);
+ return NULL;
+ }
/* Now, we're going to create one decoding matrix which is going to
decode everything with one call. The hope is that the scheduler
diff --git a/src/osd/ErasureCodePluginJerasure/jerasure.h b/src/osd/ErasureCodePluginJerasure/jerasure.h
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/liberation.c b/src/osd/ErasureCodePluginJerasure/liberation.c
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/liberation.h b/src/osd/ErasureCodePluginJerasure/liberation.h
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/reed_sol.c b/src/osd/ErasureCodePluginJerasure/reed_sol.c
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/reed_sol.h b/src/osd/ErasureCodePluginJerasure/reed_sol.h
old mode 100755
new mode 100644
diff --git a/src/osd/ErasureCodePluginJerasure/vectorop.h b/src/osd/ErasureCodePluginJerasure/vectorop.h
new file mode 100644
index 0000000..5135f82
--- /dev/null
+++ b/src/osd/ErasureCodePluginJerasure/vectorop.h
@@ -0,0 +1,51 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 CERN/Switzerland
+ *
+ *
+ * Authors: Andreas-Joachim Peters <andreas.joachim.peters at cern.ch>
+ *
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_VECTOROP_H
+#define CEPH_VECTOROP_H
+
+// -------------------------------------------------------------------------
+// constant used in the block alignment function to allow for vector ops
+// -------------------------------------------------------------------------
+#define LARGEST_VECTOR_WORDSIZE 16
+
+// -------------------------------------------------------------------------
+// switch to 128-bit XOR operations if possible
+// -------------------------------------------------------------------------
+#if __GNUC__ > 4 || \
+ (__GNUC__ == 4 && (__GNUC_MINOR__ >= 4) ) || \
+ (__clang__ == 1 )
+
+#ifdef VECTOROP_DEBUG
+#pragma message "* using 128-bit vector operations in " __FILE__
+#endif
+// -------------------------------------------------------------------------
+// use 128-bit pointer
+// -------------------------------------------------------------------------
+typedef long vector_op_t __attribute__ ((vector_size (16)));
+#define VECTOR_WORDSIZE 16
+#else
+// -------------------------------------------------------------------------
+// use 64-bit pointer
+// -------------------------------------------------------------------------
+typedef unsigned long long vector_op_t;
+#define VECTOR_WORDSIZE 8
+#endif
+
+#endif /* CEPH_VECTOROP_H */
+
diff --git a/src/osd/HitSet.cc b/src/osd/HitSet.cc
new file mode 100644
index 0000000..700da5d
--- /dev/null
+++ b/src/osd/HitSet.cc
@@ -0,0 +1,218 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank <info at inktank.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "HitSet.h"
+
+// -- HitSet --
+
+HitSet::HitSet(const HitSet::Params& params)
+ : sealed(false)
+{
+ switch (params.get_type()) {
+ case TYPE_BLOOM:
+ {
+ BloomHitSet::Params *p =
+ static_cast<BloomHitSet::Params*>(params.impl.get());
+ impl.reset(new BloomHitSet(p));
+ }
+ break;
+
+ case TYPE_EXPLICIT_HASH:
+ impl.reset(new ExplicitHashHitSet(static_cast<ExplicitHashHitSet::Params*>(params.impl.get())));
+ break;
+
+ case TYPE_EXPLICIT_OBJECT:
+ impl.reset(new ExplicitObjectHitSet(static_cast<ExplicitObjectHitSet::Params*>(params.impl.get())));
+ break;
+
+ case TYPE_NONE:
+ break;
+
+ default:
+ assert (0 == "unknown HitSet type");
+ }
+}
+
+void HitSet::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(sealed, bl);
+ if (impl) {
+ ::encode((__u8)impl->get_type(), bl);
+ impl->encode(bl);
+ } else {
+ ::encode((__u8)TYPE_NONE, bl);
+ }
+ ENCODE_FINISH(bl);
+}
+
+void HitSet::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(sealed, bl);
+ __u8 type;
+ ::decode(type, bl);
+ switch ((impl_type_t)type) {
+ case TYPE_EXPLICIT_HASH:
+ impl.reset(new ExplicitHashHitSet);
+ break;
+ case TYPE_EXPLICIT_OBJECT:
+ impl.reset(new ExplicitObjectHitSet);
+ break;
+ case TYPE_BLOOM:
+ impl.reset(new BloomHitSet);
+ break;
+ case TYPE_NONE:
+ impl.reset(NULL);
+ break;
+ default:
+ throw buffer::malformed_input("unrecognized HitMap type");
+ }
+ if (impl)
+ impl->decode(bl);
+ DECODE_FINISH(bl);
+}
+
+void HitSet::dump(Formatter *f) const
+{
+ f->dump_string("type", get_type_name());
+ f->dump_string("sealed", sealed ? "yes" : "no");
+ if (impl)
+ impl->dump(f);
+}
+
+void HitSet::generate_test_instances(list<HitSet*>& o)
+{
+ o.push_back(new HitSet);
+ o.push_back(new HitSet(new BloomHitSet(10, .1, 1)));
+ o.back()->insert(hobject_t());
+ o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+ o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+ o.push_back(new HitSet(new ExplicitHashHitSet));
+ o.back()->insert(hobject_t());
+ o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+ o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+ o.push_back(new HitSet(new ExplicitObjectHitSet));
+ o.back()->insert(hobject_t());
+ o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+ o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+}
+
+HitSet::Params::Params(const Params& o)
+{
+ if (o.get_type() != TYPE_NONE) {
+ create_impl(o.get_type());
+ // it's annoying to write virtual operator= methods; use encode/decode
+ // instead.
+ bufferlist bl;
+ o.impl->encode(bl);
+ bufferlist::iterator p = bl.begin();
+ impl->decode(p);
+ } // else we don't need to do anything
+}
+
+const HitSet::Params& HitSet::Params::operator=(const Params& o)
+{
+ create_impl(o.get_type());
+ if (o.impl) {
+ // it's annoying to write virtual operator= methods; use encode/decode
+ // instead.
+ bufferlist bl;
+ o.impl->encode(bl);
+ bufferlist::iterator p = bl.begin();
+ impl->decode(p);
+ }
+ return *this;
+}
+
+void HitSet::Params::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ if (impl) {
+ ::encode((__u8)impl->get_type(), bl);
+ impl->encode(bl);
+ } else {
+ ::encode((__u8)TYPE_NONE, bl);
+ }
+ ENCODE_FINISH(bl);
+}
+
+bool HitSet::Params::create_impl(impl_type_t type)
+{
+ switch ((impl_type_t)type) {
+ case TYPE_EXPLICIT_HASH:
+ impl.reset(new ExplicitHashHitSet::Params);
+ break;
+ case TYPE_EXPLICIT_OBJECT:
+ impl.reset(new ExplicitObjectHitSet::Params);
+ break;
+ case TYPE_BLOOM:
+ impl.reset(new BloomHitSet::Params);
+ break;
+ case TYPE_NONE:
+ impl.reset(NULL);
+ break;
+ default:
+ return false;
+ }
+ return true;
+}
+
+void HitSet::Params::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ __u8 type;
+ ::decode(type, bl);
+ if (!create_impl((impl_type_t)type))
+ throw buffer::malformed_input("unrecognized HitMap type");
+ if (impl)
+ impl->decode(bl);
+ DECODE_FINISH(bl);
+}
+
+void HitSet::Params::dump(Formatter *f) const
+{
+ f->dump_string("type", HitSet::get_type_name(get_type()));
+ if (impl)
+ impl->dump(f);
+}
+
+void HitSet::Params::generate_test_instances(list<HitSet::Params*>& o)
+{
+#define loop_hitset_params(kind) \
+{ \
+ list<kind::Params*> params; \
+ kind::Params::generate_test_instances(params); \
+ for (list<kind::Params*>::iterator i = params.begin(); \
+ i != params.end(); ++i) \
+ o.push_back(new Params(*i)); \
+}
+ o.push_back(new Params);
+ o.push_back(new Params(new BloomHitSet::Params));
+ loop_hitset_params(BloomHitSet);
+ o.push_back(new Params(new ExplicitHashHitSet::Params));
+ loop_hitset_params(ExplicitHashHitSet);
+ o.push_back(new Params(new ExplicitObjectHitSet::Params));
+ loop_hitset_params(ExplicitObjectHitSet);
+}
+
+ostream& operator<<(ostream& out, const HitSet::Params& p) {
+ out << HitSet::get_type_name(p.get_type());
+ if (p.impl) {
+ out << "{";
+ p.impl->dump_stream(out);
+ }
+ out << "}";
+ return out;
+}
diff --git a/src/osd/HitSet.h b/src/osd/HitSet.h
new file mode 100644
index 0000000..b6acce9
--- /dev/null
+++ b/src/osd/HitSet.h
@@ -0,0 +1,475 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank <info at inktank.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_OSD_HITSET_H
+#define CEPH_OSD_HITSET_H
+
+#include <boost/scoped_ptr.hpp>
+
+#include "include/encoding.h"
+#include "include/unordered_set.h"
+#include "common/bloom_filter.hpp"
+#include "common/hobject.h"
+#include "common/Formatter.h"
+
+/**
+ * generic container for a HitSet
+ *
+ * Encapsulate a HitSetImpl of any type. Expose a generic interface
+ * to users and wrap the encoded object with a type so that it can be
+ * safely decoded later.
+ */
+
+class HitSet {
+public:
+ typedef enum {
+ TYPE_NONE = 0,
+ TYPE_EXPLICIT_HASH = 1,
+ TYPE_EXPLICIT_OBJECT = 2,
+ TYPE_BLOOM = 3
+ } impl_type_t;
+
+ static const char *get_type_name(impl_type_t t) {
+ switch (t) {
+ case TYPE_NONE: return "none";
+ case TYPE_EXPLICIT_HASH: return "explicit_hash";
+ case TYPE_EXPLICIT_OBJECT: return "explicit_object";
+ case TYPE_BLOOM: return "bloom";
+ default: return "???";
+ }
+ }
+ const char *get_type_name() const {
+ if (impl)
+ return get_type_name(impl->get_type());
+ return get_type_name(TYPE_NONE);
+ }
+
+ /// abstract interface for a HitSet implementation
+ class Impl {
+ public:
+ virtual impl_type_t get_type() const = 0;
+ virtual bool is_full() const = 0;
+ virtual void insert(const hobject_t& o) = 0;
+ virtual bool contains(const hobject_t& o) const = 0;
+ virtual unsigned insert_count() const = 0;
+ virtual unsigned approx_unique_insert_count() const = 0;
+ virtual void encode(bufferlist &bl) const = 0;
+ virtual void decode(bufferlist::iterator& p) = 0;
+ virtual void dump(Formatter *f) const = 0;
+ virtual Impl* clone() const = 0;
+ virtual void seal() {}
+ virtual ~Impl() {}
+ };
+
+ boost::scoped_ptr<Impl> impl;
+ bool sealed;
+
+ class Params {
+ /// create an Impl* of the given type
+ bool create_impl(impl_type_t t);
+
+ public:
+ class Impl {
+ public:
+ virtual impl_type_t get_type() const = 0;
+ virtual HitSet::Impl *get_new_impl() const = 0;
+ virtual void encode(bufferlist &bl) const {}
+ virtual void decode(bufferlist::iterator& p) {}
+ virtual void dump(Formatter *f) const {}
+ virtual void dump_stream(ostream& o) const {}
+ virtual ~Impl() {}
+ };
+
+ Params() {}
+ Params(Impl *i) : impl(i) {}
+ virtual ~Params() {}
+
+ boost::scoped_ptr<Params::Impl> impl;
+
+ impl_type_t get_type() const {
+ if (impl)
+ return impl->get_type();
+ return TYPE_NONE;
+ }
+
+ Params(const Params& o);
+ const Params& operator=(const Params& o);
+
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<HitSet::Params*>& o);
+
+ friend ostream& operator<<(ostream& out, const HitSet::Params& p);
+ };
+
+ HitSet() : impl(NULL), sealed(false) {}
+ HitSet(Impl *i) : impl(i), sealed(false) {}
+ HitSet(const HitSet::Params& params);
+
+ HitSet(const HitSet& o) {
+ sealed = o.sealed;
+ if (o.impl)
+ impl.reset(o.impl->clone());
+ else
+ impl.reset(NULL);
+ }
+ const HitSet& operator=(const HitSet& o) {
+ sealed = o.sealed;
+ if (o.impl)
+ impl.reset(o.impl->clone());
+ else
+ impl.reset(NULL);
+ return *this;
+ }
+
+
+ bool is_full() const {
+ return impl->is_full();
+ }
+ /// insert a hash into the set
+ void insert(const hobject_t& o) {
+ impl->insert(o);
+ }
+ /// query whether a hash is in the set
+ bool contains(const hobject_t& o) const {
+ return impl->contains(o);
+ }
+
+ unsigned insert_count() const {
+ return impl->insert_count();
+ }
+ unsigned approx_unique_insert_count() const {
+ return impl->approx_unique_insert_count();
+ }
+ void seal() {
+ assert(!sealed);
+ sealed = true;
+ impl->seal();
+ }
+
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<HitSet*>& o);
+
+private:
+ void reset_to_type(impl_type_t type);
+};
+WRITE_CLASS_ENCODER(HitSet);
+WRITE_CLASS_ENCODER(HitSet::Params);
+
+ostream& operator<<(ostream& out, const HitSet::Params& p);
+
+/**
+ * explicitly enumerate hash hits in the set
+ */
+class ExplicitHashHitSet : public HitSet::Impl {
+ uint64_t count;
+ ceph::unordered_set<uint32_t> hits;
+public:
+ class Params : public HitSet::Params::Impl {
+ public:
+ virtual HitSet::impl_type_t get_type() const {
+ return HitSet::TYPE_EXPLICIT_HASH;
+ }
+ virtual HitSet::Impl *get_new_impl() const {
+ return new ExplicitHashHitSet;
+ }
+ static void generate_test_instances(list<Params*>& o) {
+ o.push_back(new Params);
+ }
+ };
+
+ ExplicitHashHitSet() : count(0) {}
+ ExplicitHashHitSet(const ExplicitHashHitSet::Params *p) : count(0) {}
+ ExplicitHashHitSet(const ExplicitHashHitSet &o) : count(o.count),
+ hits(o.hits) {}
+
+ HitSet::Impl *clone() const {
+ return new ExplicitHashHitSet(*this);
+ }
+
+ HitSet::impl_type_t get_type() const {
+ return HitSet::TYPE_EXPLICIT_HASH;
+ }
+ bool is_full() const {
+ return false;
+ }
+ void insert(const hobject_t& o) {
+ hits.insert(o.hash);
+ ++count;
+ }
+ bool contains(const hobject_t& o) const {
+ return hits.count(o.hash);
+ }
+ unsigned insert_count() const {
+ return count;
+ }
+ unsigned approx_unique_insert_count() const {
+ return hits.size();
+ }
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(count, bl);
+ ::encode(hits, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(count, bl);
+ ::decode(hits, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const {
+ f->dump_unsigned("insert_count", count);
+ f->open_array_section("hash_set");
+ for (ceph::unordered_set<uint32_t>::const_iterator p = hits.begin(); p != hits.end(); ++p)
+ f->dump_unsigned("hash", *p);
+ f->close_section();
+ }
+ static void generate_test_instances(list<ExplicitHashHitSet*>& o) {
+ o.push_back(new ExplicitHashHitSet);
+ o.push_back(new ExplicitHashHitSet);
+ o.back()->insert(hobject_t());
+ o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+ o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+ }
+};
+WRITE_CLASS_ENCODER(ExplicitHashHitSet)
+
+/**
+ * explicitly enumerate objects in the set
+ */
+class ExplicitObjectHitSet : public HitSet::Impl {
+ uint64_t count;
+ ceph::unordered_set<hobject_t> hits;
+public:
+ class Params : public HitSet::Params::Impl {
+ public:
+ virtual HitSet::impl_type_t get_type() const {
+ return HitSet::TYPE_EXPLICIT_OBJECT;
+ }
+ virtual HitSet::Impl *get_new_impl() const {
+ return new ExplicitObjectHitSet;
+ }
+ static void generate_test_instances(list<Params*>& o) {
+ o.push_back(new Params);
+ }
+ };
+
+ ExplicitObjectHitSet() : count(0) {}
+ ExplicitObjectHitSet(const ExplicitObjectHitSet::Params *p) : count(0) {}
+ ExplicitObjectHitSet(const ExplicitObjectHitSet &o) : count(o.count),
+ hits(o.hits) {}
+
+ HitSet::Impl *clone() const {
+ return new ExplicitObjectHitSet(*this);
+ }
+
+ HitSet::impl_type_t get_type() const {
+ return HitSet::TYPE_EXPLICIT_OBJECT;
+ }
+ bool is_full() const {
+ return false;
+ }
+ void insert(const hobject_t& o) {
+ hits.insert(o);
+ ++count;
+ }
+ bool contains(const hobject_t& o) const {
+ return hits.count(o);
+ }
+ unsigned insert_count() const {
+ return count;
+ }
+ unsigned approx_unique_insert_count() const {
+ return hits.size();
+ }
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(count, bl);
+ ::encode(hits, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(count, bl);
+ ::decode(hits, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const {
+ f->dump_unsigned("insert_count", count);
+ f->open_array_section("set");
+ for (ceph::unordered_set<hobject_t>::const_iterator p = hits.begin(); p != hits.end(); ++p) {
+ f->open_object_section("object");
+ p->dump(f);
+ f->close_section();
+ }
+ f->close_section();
+ }
+ static void generate_test_instances(list<ExplicitObjectHitSet*>& o) {
+ o.push_back(new ExplicitObjectHitSet);
+ o.push_back(new ExplicitObjectHitSet);
+ o.back()->insert(hobject_t());
+ o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+ o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+ }
+};
+WRITE_CLASS_ENCODER(ExplicitObjectHitSet)
+
+/**
+ * use a bloom_filter to track hits to the set
+ */
+class BloomHitSet : public HitSet::Impl {
+ compressible_bloom_filter bloom;
+
+public:
+ HitSet::impl_type_t get_type() const {
+ return HitSet::TYPE_BLOOM;
+ }
+
+ class Params : public HitSet::Params::Impl {
+ public:
+ virtual HitSet::impl_type_t get_type() const {
+ return HitSet::TYPE_BLOOM;
+ }
+ virtual HitSet::Impl *get_new_impl() const {
+ return new BloomHitSet;
+ }
+
+ uint32_t fpp_micro; ///< false positive probability / 1M
+ uint64_t target_size; ///< number of unique insertions we expect to this HitSet
+ uint64_t seed; ///< seed to use when initializing the bloom filter
+
+ Params()
+ : fpp_micro(0), target_size(0), seed(0) {}
+ Params(double fpp, uint64_t t, uint64_t s)
+ : fpp_micro(fpp * 1000000.0), target_size(t), seed(s) {}
+ Params(const Params &o)
+ : fpp_micro(o.fpp_micro),
+ target_size(o.target_size),
+ seed(o.seed) {}
+ ~Params() {}
+
+ double get_fpp() const {
+ return (double)fpp_micro / 1000000.0;
+ }
+ void set_fpp(double f) {
+ fpp_micro = (unsigned)(f * 1000000.0);
+ }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(fpp_micro, bl);
+ ::encode(target_size, bl);
+ ::encode(seed, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(fpp_micro, bl);
+ ::decode(target_size, bl);
+ ::decode(seed, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const {
+ f->dump_float("false_positive_probability", get_fpp());
+ f->dump_int("target_size", target_size);
+ f->dump_int("seed", seed);
+ }
+ void dump_stream(ostream& o) const {
+ o << "false_positive_probability: "
+ << get_fpp() << ", target_size: " << target_size
+ << ", seed: " << seed;
+ }
+ static void generate_test_instances(list<Params*>& o) {
+ o.push_back(new Params);
+ o.push_back(new Params);
+ (*o.rbegin())->fpp_micro = 123456;
+ (*o.rbegin())->target_size = 300;
+ (*o.rbegin())->seed = 99;
+ }
+ };
+
+ BloomHitSet() {}
+ BloomHitSet(unsigned inserts, double fpp, int seed)
+ : bloom(inserts, fpp, seed)
+ {}
+ BloomHitSet(const BloomHitSet::Params *p) : bloom(p->target_size,
+ p->get_fpp(),
+ p->seed)
+ {}
+
+ BloomHitSet(const BloomHitSet &o) {
+ // oh god
+ bufferlist bl;
+ o.encode(bl);
+ bufferlist::iterator bli = bl.begin();
+ this->decode(bli);
+ }
+
+ HitSet::Impl *clone() const {
+ return new BloomHitSet(*this);
+ }
+
+ bool is_full() const {
+ return bloom.is_full();
+ }
+
+ void insert(const hobject_t& o) {
+ bloom.insert(o.hash);
+ }
+ bool contains(const hobject_t& o) const {
+ return bloom.contains(o.hash);
+ }
+ unsigned insert_count() const {
+ return bloom.element_count();
+ }
+ unsigned approx_unique_insert_count() const {
+ return bloom.approx_unique_element_count();
+ }
+ void seal() {
+ // aim for a density of .5 (50% of bit set)
+ double pc = (double)bloom.density() * 2.0;
+ if (pc < 1.0)
+ bloom.compress(pc);
+ }
+
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(bloom, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator &bl) {
+ DECODE_START(1, bl);
+ ::decode(bloom, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const {
+ f->open_object_section("bloom_filter");
+ bloom.dump(f);
+ f->close_section();
+ }
+ static void generate_test_instances(list<BloomHitSet*>& o) {
+ o.push_back(new BloomHitSet);
+ o.push_back(new BloomHitSet(10, .1, 1));
+ o.back()->insert(hobject_t());
+ o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+ o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+ }
+};
+WRITE_CLASS_ENCODER(BloomHitSet)
+
+#endif
diff --git a/src/osd/Makefile.am b/src/osd/Makefile.am
index cae0201..9bbc7e4 100644
--- a/src/osd/Makefile.am
+++ b/src/osd/Makefile.am
@@ -1,5 +1,5 @@
## erasure code plugins
-erasure_codelibdir = $(libdir)/erasure-code
+erasure_codelibdir = $(pkglibdir)/erasure-code
erasure_codelib_LTLIBRARIES =
include osd/ErasureCodePluginJerasure/Makefile.am
@@ -10,7 +10,9 @@ libosd_la_SOURCES = \
osd/PGLog.cc \
osd/ReplicatedPG.cc \
osd/ReplicatedBackend.cc \
+ osd/PGBackend.cc \
osd/Ager.cc \
+ osd/HitSet.cc \
osd/OSD.cc \
osd/OSDCap.cc \
osd/Watch.cc \
@@ -28,6 +30,7 @@ noinst_HEADERS += \
osd/ClassHandler.h \
osd/ErasureCodeInterface.h \
osd/ErasureCodePlugin.h \
+ osd/HitSet.h \
osd/OSD.h \
osd/OSDCap.h \
osd/OSDMap.h \
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 3f767fe..38bb171 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -11,19 +11,24 @@
* Foundation. See file COPYING.
*
*/
+#include "acconfig.h"
#include <fstream>
#include <iostream>
#include <errno.h>
#include <sys/stat.h>
+#include <sys/utsname.h>
#include <signal.h>
#include <ctype.h>
#include <boost/scoped_ptr.hpp>
-#if defined(DARWIN) || defined(__FreeBSD__)
+#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
+#endif
+
+#ifdef HAVE_SYS_MOUNT_H
#include <sys/mount.h>
-#endif // DARWIN || __FreeBSD__
+#endif
#include "osd/PG.h"
@@ -37,8 +42,8 @@
#include "common/ceph_argparse.h"
#include "common/version.h"
-#include "os/FileStore.h"
-#include "os/FileJournal.h"
+
+#include "os/ObjectStore.h"
#include "ReplicatedPG.h"
@@ -98,7 +103,6 @@
#include "common/perf_counters.h"
#include "common/Timer.h"
#include "common/LogClient.h"
-#include "common/safe_io.h"
#include "common/HeartbeatMap.h"
#include "common/admin_socket.h"
@@ -181,6 +185,7 @@ OSDService::OSDService(OSD *osd) :
scrub_finalize_wq(osd->scrub_finalize_wq),
rep_scrub_wq(osd->rep_scrub_wq),
push_wq("push_wq", cct->_conf->osd_recovery_thread_timeout, &osd->recovery_tp),
+ gen_wq("gen_wq", cct->_conf->osd_recovery_thread_timeout, &osd->recovery_tp),
class_handler(osd->class_handler),
publish_lock("OSDService::publish_lock"),
pre_publish_lock("OSDService::pre_publish_lock"),
@@ -434,26 +439,10 @@ void OSDService::init()
objecter_timer.init();
objecter->set_client_incarnation(0);
objecter->init_locked();
- objecter->unset_honor_cache_redirects();
}
watch_timer.init();
}
-ObjectStore *OSD::create_object_store(CephContext *cct, const std::string &dev, const std::string &jdev)
-{
- struct stat st;
- if (::stat(dev.c_str(), &st) != 0)
- return 0;
-
- if (cct->_conf->filestore)
- return new FileStore(dev, jdev);
-
- if (S_ISDIR(st.st_mode))
- return new FileStore(dev, jdev);
- else
- return 0;
-}
-
#undef dout_prefix
#define dout_prefix *_dout
@@ -526,7 +515,7 @@ int OSD::do_convertfs(ObjectStore *store)
if (r == 1)
return store->umount();
- derr << "FileStore is old at version " << version << ". Updating..." << dendl;
+ derr << "ObjectStore is old at version " << version << ". Updating..." << dendl;
derr << "Removing tmp pgs" << dendl;
vector<coll_t> collections;
@@ -583,39 +572,24 @@ int OSD::do_convertfs(ObjectStore *store)
return store->umount();
}
-int OSD::convertfs(const std::string &dev, const std::string &jdev)
-{
- boost::scoped_ptr<ObjectStore> store(
- new FileStore(dev, jdev, "filestore",
- true));
- int r = do_convertfs(store.get());
- return r;
-}
-
-int OSD::mkfs(CephContext *cct, const std::string &dev, const std::string &jdev, uuid_d fsid, int whoami)
+int OSD::mkfs(CephContext *cct, ObjectStore *store, const string &dev,
+ uuid_d fsid, int whoami)
{
int ret;
- ObjectStore *store = NULL;
try {
- store = create_object_store(cct, dev, jdev);
- if (!store) {
- ret = -ENOENT;
- goto out;
- }
-
// if we are fed a uuid for this osd, use it.
store->set_fsid(cct->_conf->osd_uuid);
ret = store->mkfs();
if (ret) {
- derr << "OSD::mkfs: FileStore::mkfs failed with error " << ret << dendl;
+ derr << "OSD::mkfs: ObjectStore::mkfs failed with error " << ret << dendl;
goto free_store;
}
ret = store->mount();
if (ret) {
- derr << "OSD::mkfs: couldn't mount FileStore: error " << ret << dendl;
+ derr << "OSD::mkfs: couldn't mount ObjectStore: error " << ret << dendl;
goto free_store;
}
@@ -672,7 +646,7 @@ int OSD::mkfs(CephContext *cct, const std::string &dev, const std::string &jdev,
for (int i=0; i<1000; i++) {
ObjectStore::Transaction *t = new ObjectStore::Transaction;
t->write(coll_t::META_COLL, hobject_t(sobject_t(oid, 0)), i*bl.length(), bl.length(), bl);
- store->queue_transaction(NULL, t);
+ store->queue_transaction_and_cleanup(NULL, t);
}
store->sync();
utime_t end = ceph_clock_now(cct);
@@ -707,18 +681,12 @@ int OSD::mkfs(CephContext *cct, const std::string &dev, const std::string &jdev,
store->sync_and_flush();
- ret = write_meta(dev, sb.cluster_fsid, sb.osd_fsid, whoami);
+ ret = write_meta(store, sb.cluster_fsid, sb.osd_fsid, whoami);
if (ret) {
derr << "OSD::mkfs: failed to write fsid file: error " << ret << dendl;
goto umount_store;
}
- ret = safe_write_file(dev.c_str(), "ready", "ready\n", 6);
- if (ret) {
- derr << "OSD::mkfs: failed to write ready file: error " << ret << dendl;
- goto umount_store;
- }
-
}
catch (const std::exception &se) {
derr << "OSD::mkfs: caught exception " << se.what() << dendl;
@@ -733,108 +701,78 @@ umount_store:
store->umount();
free_store:
delete store;
-out:
return ret;
}
-int OSD::mkjournal(CephContext *cct, const std::string &dev, const std::string &jdev)
-{
- ObjectStore *store = create_object_store(cct, dev, jdev);
- if (!store)
- return -ENOENT;
- return store->mkjournal();
-}
-
-int OSD::flushjournal(CephContext *cct, const std::string &dev, const std::string &jdev)
-{
- ObjectStore *store = create_object_store(cct, dev, jdev);
- if (!store)
- return -ENOENT;
- int err = store->mount();
- if (!err) {
- store->sync_and_flush();
- store->umount();
- }
- delete store;
- return err;
-}
-
-int OSD::dump_journal(CephContext *cct, const std::string &dev, const std::string &jdev, ostream& out)
-{
- ObjectStore *store = create_object_store(cct, dev, jdev);
- if (!store)
- return -ENOENT;
- int err = store->dump_journal(out);
- delete store;
- return err;
-}
-
-int OSD::write_meta(const std::string &base, uuid_d& cluster_fsid, uuid_d& osd_fsid, int whoami)
+int OSD::write_meta(ObjectStore *store, uuid_d& cluster_fsid, uuid_d& osd_fsid, int whoami)
{
char val[80];
+ int r;
- snprintf(val, sizeof(val), "%s\n", CEPH_OSD_ONDISK_MAGIC);
- safe_write_file(base.c_str(), "magic", val, strlen(val));
+ snprintf(val, sizeof(val), "%s", CEPH_OSD_ONDISK_MAGIC);
+ r = store->write_meta("magic", val);
+ if (r < 0)
+ return r;
- snprintf(val, sizeof(val), "%d\n", whoami);
- safe_write_file(base.c_str(), "whoami", val, strlen(val));
+ snprintf(val, sizeof(val), "%d", whoami);
+ r = store->write_meta("whoami", val);
+ if (r < 0)
+ return r;
cluster_fsid.print(val);
- strcat(val, "\n");
- safe_write_file(base.c_str(), "ceph_fsid", val, strlen(val));
+ r = store->write_meta("ceph_fsid", val);
+ if (r < 0)
+ return r;
+
+ r = store->write_meta("ready", "ready");
+ if (r < 0)
+ return r;
return 0;
}
-int OSD::peek_meta(const std::string &dev, std::string& magic,
+int OSD::peek_meta(ObjectStore *store, std::string& magic,
uuid_d& cluster_fsid, uuid_d& osd_fsid, int& whoami)
{
- char val[80] = { 0 };
+ string val;
- if (safe_read_file(dev.c_str(), "magic", val, sizeof(val)) < 0)
- return -errno;
- int l = strlen(val);
- if (l && val[l-1] == '\n')
- val[l-1] = 0;
+ int r = store->read_meta("magic", &val);
+ if (r < 0)
+ return r;
magic = val;
- if (safe_read_file(dev.c_str(), "whoami", val, sizeof(val)) < 0)
- return -errno;
- whoami = atoi(val);
+ r = store->read_meta("whoami", &val);
+ if (r < 0)
+ return r;
+ whoami = atoi(val.c_str());
- if (safe_read_file(dev.c_str(), "ceph_fsid", val, sizeof(val)) < 0)
- return -errno;
- if (strlen(val) > 36)
- val[36] = 0;
- cluster_fsid.parse(val);
+ r = store->read_meta("ceph_fsid", &val);
+ if (r < 0)
+ return r;
+ r = cluster_fsid.parse(val.c_str());
+ if (r < 0)
+ return r;
- if (safe_read_file(dev.c_str(), "fsid", val, sizeof(val)) < 0)
+ r = store->read_meta("fsid", &val);
+ if (r < 0) {
osd_fsid = uuid_d();
- else {
- if (strlen(val) > 36)
- val[36] = 0;
- osd_fsid.parse(val);
+ } else {
+ r = osd_fsid.parse(val.c_str());
+ if (r < 0)
+ return r;
}
return 0;
}
-int OSD::peek_journal_fsid(string path, uuid_d& fsid)
-{
- // make sure we don't try to use aio or direct_io (and get annoying
- // error messages from failing to do so); performance implications
- // should be irrelevant for this use
- FileJournal j(fsid, 0, 0, path.c_str(), false, false);
- return j.peek_fsid(fsid);
-}
-
#undef dout_prefix
#define dout_prefix _prefix(_dout, whoami, osdmap)
// cons/des
-OSD::OSD(CephContext *cct_, int id, Messenger *internal_messenger, Messenger *external_messenger,
+OSD::OSD(CephContext *cct_, ObjectStore *store_,
+ int id, Messenger *internal_messenger, Messenger *external_messenger,
Messenger *hb_clientm,
Messenger *hb_front_serverm,
Messenger *hb_back_serverm,
@@ -858,7 +796,7 @@ OSD::OSD(CephContext *cct_, int id, Messenger *internal_messenger, Messenger *ex
monc(mc),
logger(NULL),
recoverystate_perf(NULL),
- store(NULL),
+ store(store_),
clog(cct, client_messenger, &mc->monmap, LogClient::NO_FLAGS),
whoami(id),
dev_path(dev), journal_path(jdev),
@@ -900,7 +838,7 @@ OSD::OSD(CephContext *cct_, int id, Messenger *internal_messenger, Messenger *ex
replay_queue_lock("OSD::replay_queue_lock"),
snap_trim_wq(this, cct->_conf->osd_snap_trim_thread_timeout, &disk_tp),
scrub_wq(this, cct->_conf->osd_scrub_thread_timeout, &disk_tp),
- scrub_finalize_wq(this, cct->_conf->osd_scrub_finalize_thread_timeout, &op_tp),
+ scrub_finalize_wq(cct->_conf->osd_scrub_finalize_thread_timeout, &op_tp),
rep_scrub_wq(this, cct->_conf->osd_scrub_thread_timeout, &disk_tp),
remove_wq(store, cct->_conf->osd_remove_thread_timeout, &disk_tp),
next_removal_seq(0),
@@ -941,13 +879,6 @@ int OSD::pre_init()
if (is_stopping())
return 0;
- assert(!store);
- store = create_object_store(cct, dev_path, journal_path);
- if (!store) {
- derr << "OSD::pre_init: unable to create object store" << dendl;
- return -ENODEV;
- }
-
if (store->test_mount_in_use()) {
derr << "OSD::pre_init: object store '" << dev_path << "' is "
<< "currently in use. (Is ceph-osd already running?)" << dendl;
@@ -1007,7 +938,7 @@ bool OSD::asok_command(string command, cmdmap_t& cmdmap, string format,
list<obj_watch_item_t> watchers;
osd_lock.Lock();
// scan pg's
- for (hash_map<pg_t,PG*>::iterator it = pg_map.begin();
+ for (ceph::unordered_map<pg_t,PG*>::iterator it = pg_map.begin();
it != pg_map.end();
++it) {
@@ -1177,7 +1108,7 @@ int OSD::init()
goto out;
}
osdmap = get_map(superblock.current_epoch);
- check_osdmap_features();
+ check_osdmap_features(store);
create_recoverystate_perf();
@@ -1369,19 +1300,23 @@ void OSD::create_logger()
osd_plb.add_u64_counter(l_osd_op_inb, "op_in_bytes"); // client op in bytes (writes)
osd_plb.add_u64_counter(l_osd_op_outb, "op_out_bytes"); // client op out bytes (reads)
osd_plb.add_time_avg(l_osd_op_lat, "op_latency"); // client op latency
+ osd_plb.add_time_avg(l_osd_op_process_lat, "op_process_latency"); // client op process latency
osd_plb.add_u64_counter(l_osd_op_r, "op_r"); // client reads
osd_plb.add_u64_counter(l_osd_op_r_outb, "op_r_out_bytes"); // client read out bytes
osd_plb.add_time_avg(l_osd_op_r_lat, "op_r_latency"); // client read latency
+ osd_plb.add_time_avg(l_osd_op_r_process_lat, "op_r_process_latency"); // client read process latency
osd_plb.add_u64_counter(l_osd_op_w, "op_w"); // client writes
osd_plb.add_u64_counter(l_osd_op_w_inb, "op_w_in_bytes"); // client write in bytes
osd_plb.add_time_avg(l_osd_op_w_rlat, "op_w_rlat"); // client write readable/applied latency
osd_plb.add_time_avg(l_osd_op_w_lat, "op_w_latency"); // client write latency
+ osd_plb.add_time_avg(l_osd_op_w_process_lat, "op_w_process_latency"); // client write process latency
osd_plb.add_u64_counter(l_osd_op_rw, "op_rw"); // client rmw
osd_plb.add_u64_counter(l_osd_op_rw_inb, "op_rw_in_bytes"); // client rmw in bytes
osd_plb.add_u64_counter(l_osd_op_rw_outb,"op_rw_out_bytes"); // client rmw out bytes
osd_plb.add_time_avg(l_osd_op_rw_rlat,"op_rw_rlat"); // client rmw readable/applied latency
osd_plb.add_time_avg(l_osd_op_rw_lat, "op_rw_latency"); // client rmw latency
+ osd_plb.add_time_avg(l_osd_op_rw_process_lat, "op_rw_process_latency"); // client rmw process latency
osd_plb.add_u64_counter(l_osd_sop, "subop"); // subops
osd_plb.add_u64_counter(l_osd_sop_inb, "subop_in_bytes"); // subop in bytes
@@ -1517,7 +1452,7 @@ int OSD::shutdown()
cct->_conf->apply_changes(NULL);
// Shutdown PGs
- for (hash_map<pg_t, PG*>::iterator p = pg_map.begin();
+ for (ceph::unordered_map<pg_t, PG*>::iterator p = pg_map.begin();
p != pg_map.end();
++p) {
dout(20) << " kicking pg " << p->first << dendl;
@@ -1613,7 +1548,7 @@ int OSD::shutdown()
#ifdef PG_DEBUG_REFS
service.dump_live_pgids();
#endif
- for (hash_map<pg_t, PG*>::iterator p = pg_map.begin();
+ for (ceph::unordered_map<pg_t, PG*>::iterator p = pg_map.begin();
p != pg_map.end();
++p) {
dout(20) << " kicking pg " << p->first << dendl;
@@ -1766,7 +1701,7 @@ PG* OSD::_make_pg(
PG *pg;
hobject_t logoid = make_pg_log_oid(pgid);
hobject_t infooid = make_pg_biginfo_oid(pgid);
- if (createmap->get_pg_type(pgid) == pg_pool_t::TYPE_REP)
+ if (createmap->get_pg_type(pgid) == pg_pool_t::TYPE_REPLICATED)
pg = new ReplicatedPG(&service, createmap, pool, pgid, logoid, infooid);
else
assert(0);
@@ -1783,7 +1718,7 @@ void OSD::add_newly_split_pg(PG *pg, PG::RecoveryCtx *rctx)
dout(10) << "Adding newly split pg " << *pg << dendl;
vector<int> up, acting;
pg->get_osdmap()->pg_to_up_acting_osds(pg->info.pgid, up, acting);
- int role = pg->get_osdmap()->calc_pg_role(service.whoami, acting);
+ int role = OSDMap::calc_pg_role(service.whoami, acting);
pg->set_role(role);
pg->reg_next_scrub();
pg->handle_loaded(rctx);
@@ -2037,7 +1972,7 @@ void OSD::load_pgs()
// generate state for PG's current mapping
pg->get_osdmap()->pg_to_up_acting_osds(pgid, pg->up, pg->acting);
- int role = pg->get_osdmap()->calc_pg_role(whoami, pg->acting);
+ int role = OSDMap::calc_pg_role(whoami, pg->acting);
pg->set_role(role);
PG::RecoveryCtx rctx(0, 0, 0, 0, 0, 0);
@@ -2074,7 +2009,7 @@ void OSD::build_past_intervals_parallel()
// calculate untion of map range
epoch_t end_epoch = superblock.oldest_map;
epoch_t cur_epoch = superblock.newest_map;
- for (hash_map<pg_t, PG*>::iterator i = pg_map.begin();
+ for (ceph::unordered_map<pg_t, PG*>::iterator i = pg_map.begin();
i != pg_map.end();
++i) {
PG *pg = i->second;
@@ -2216,8 +2151,6 @@ void OSD::handle_pg_peering_evt(
bool create = false;
if (primary) {
- assert(role == 0); // otherwise, probably bug in project_pg_history.
-
// DNE on source?
if (info.dne()) {
// is there a creation pending on this pg?
@@ -2235,8 +2168,7 @@ void OSD::handle_pg_peering_evt(
}
creating_pgs.erase(info.pgid);
} else {
- assert(role != 0); // i should be replica
- assert(!info.dne()); // and pg exists if we are hearing about it
+ assert(!info.dne()); // pg exists if we are hearing about it
}
// do we need to resurrect a deleting pg?
@@ -2655,7 +2587,7 @@ void OSD::maybe_update_heartbeat_peers()
// build heartbeat from set
if (is_active()) {
- for (hash_map<pg_t, PG*>::iterator i = pg_map.begin();
+ for (ceph::unordered_map<pg_t, PG*>::iterator i = pg_map.begin();
i != pg_map.end();
++i) {
PG *pg = i->second;
@@ -3567,9 +3499,111 @@ void OSD::_send_boot()
<< ", hb_back_addr " << hb_back_addr
<< ", hb_front_addr " << hb_front_addr
<< dendl;
+ _collect_metadata(&mboot->metadata);
monc->send_mon_message(mboot);
}
+void OSD::_collect_metadata(map<string,string> *pm)
+{
+ (*pm)["ceph_version"] = pretty_version_to_str();
+
+ // config info
+ (*pm)["osd_data"] = dev_path;
+ (*pm)["osd_journal"] = journal_path;
+ (*pm)["front_addr"] = stringify(client_messenger->get_myaddr());
+ (*pm)["back_addr"] = stringify(cluster_messenger->get_myaddr());
+ (*pm)["hb_front_addr"] = stringify(hb_front_server_messenger->get_myaddr());
+ (*pm)["hb_back_addr"] = stringify(hb_back_server_messenger->get_myaddr());
+
+ // kernel info
+ struct utsname u;
+ int r = uname(&u);
+ if (r >= 0) {
+ (*pm)["os"] = u.sysname;
+ (*pm)["kernel_version"] = u.release;
+ (*pm)["kernel_description"] = u.version;
+ (*pm)["hostname"] = u.nodename;
+ (*pm)["arch"] = u.machine;
+ }
+
+ // memory
+ FILE *f = fopen("/proc/meminfo", "r");
+ if (f) {
+ char buf[100];
+ while (!feof(f)) {
+ char *line = fgets(buf, sizeof(buf), f);
+ if (!line)
+ break;
+ char key[40];
+ long long value;
+ int r = sscanf(line, "%s %lld", key, &value);
+ if (r == 2) {
+ if (strcmp(key, "MemTotal:") == 0)
+ (*pm)["mem_total_kb"] = stringify(value);
+ else if (strcmp(key, "SwapTotal:") == 0)
+ (*pm)["mem_swap_kb"] = stringify(value);
+ }
+ }
+ fclose(f);
+ }
+
+ // processor
+ f = fopen("/proc/cpuinfo", "r");
+ if (f) {
+ char buf[100];
+ while (!feof(f)) {
+ char *line = fgets(buf, sizeof(buf), f);
+ if (!line)
+ break;
+ if (strncmp(line, "model name", 10) == 0) {
+ char *c = strchr(buf, ':');
+ c++;
+ while (*c == ' ')
+ ++c;
+ char *nl = c;
+ while (*nl != '\n')
+ ++nl;
+ *nl = '\0';
+ (*pm)["cpu"] = c;
+ break;
+ }
+ }
+ fclose(f);
+ }
+
+ // distro info
+ f = fopen("/etc/lsb-release", "r");
+ if (f) {
+ char buf[100];
+ while (!feof(f)) {
+ char *line = fgets(buf, sizeof(buf), f);
+ if (!line)
+ break;
+ char *eq = strchr(buf, '=');
+ if (!eq)
+ break;
+ *eq = '\0';
+ ++eq;
+ while (*eq == '\"')
+ ++eq;
+ while (*eq && (eq[strlen(eq)-1] == '\n' ||
+ eq[strlen(eq)-1] == '\"'))
+ eq[strlen(eq)-1] = '\0';
+ if (strcmp(buf, "DISTRIB_ID") == 0)
+ (*pm)["distro"] = eq;
+ else if (strcmp(buf, "DISTRIB_RELEASE") == 0)
+ (*pm)["distro_version"] = eq;
+ else if (strcmp(buf, "DISTRIB_CODENAME") == 0)
+ (*pm)["distro_codename"] = eq;
+ else if (strcmp(buf, "DISTRIB_DESCRIPTION") == 0)
+ (*pm)["distro_description"] = eq;
+ }
+ fclose(f);
+ }
+
+ dout(10) << __func__ << " " << *pm << dendl;
+}
+
void OSD::queue_want_up_thru(epoch_t want)
{
map_lock.get_read();
@@ -4087,14 +4121,14 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
hobject_t soid(sobject_t(oid, 0));
ObjectStore::Transaction *t = new ObjectStore::Transaction;
t->write(coll_t::META_COLL, soid, 0, bsize, bl);
- store->queue_transaction(NULL, t);
+ store->queue_transaction_and_cleanup(NULL, t);
cleanupt->remove(coll_t::META_COLL, soid);
}
store->sync_and_flush();
utime_t end = ceph_clock_now(cct);
// clean up
- store->queue_transaction(NULL, cleanupt);
+ store->queue_transaction_and_cleanup(NULL, cleanupt);
uint64_t rate = (double)count / (end - start);
if (f) {
@@ -4140,7 +4174,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
}
std::set <pg_t> keys;
- for (hash_map<pg_t, PG*>::const_iterator pg_map_e = pg_map.begin();
+ for (ceph::unordered_map<pg_t, PG*>::const_iterator pg_map_e = pg_map.begin();
pg_map_e != pg_map.end(); ++pg_map_e) {
keys.insert(pg_map_e->first);
}
@@ -4148,7 +4182,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist
fout << "*** osd " << whoami << ": dump_missing ***" << std::endl;
for (std::set <pg_t>::iterator p = keys.begin();
p != keys.end(); ++p) {
- hash_map<pg_t, PG*>::iterator q = pg_map.find(*p);
+ ceph::unordered_map<pg_t, PG*>::iterator q = pg_map.find(*p);
assert(q != pg_map.end());
PG *pg = q->second;
pg->lock();
@@ -4720,7 +4754,7 @@ void OSD::handle_scrub(MOSDScrub *m)
}
if (m->scrub_pgs.empty()) {
- for (hash_map<pg_t, PG*>::iterator p = pg_map.begin();
+ for (ceph::unordered_map<pg_t, PG*>::iterator p = pg_map.begin();
p != pg_map.end();
++p) {
PG *pg = p->second;
@@ -4822,7 +4856,7 @@ void OSD::sched_scrub()
PG *pg = _lookup_lock_pg(pgid);
if (pg) {
- if (pg->is_active() &&
+ if (pg->get_pgbackend()->scrub_supported() && pg->is_active() &&
(load_is_low ||
(double)diff >= cct->_conf->osd_scrub_max_interval ||
pg->scrubber.must_scrub)) {
@@ -5001,6 +5035,11 @@ void OSD::handle_osd_map(MOSDMap *m)
m->put();
return;
}
+ if (is_initializing()) {
+ dout(0) << "ignoring osdmap until we have initialized" << dendl;
+ m->put();
+ return;
+ }
Session *session = static_cast<Session *>(m->get_connection()->get_priv());
if (session && !(session->entity_name.is_mon() || session->entity_name.is_osd())) {
@@ -5066,6 +5105,7 @@ void OSD::handle_osd_map(MOSDMap *m)
ObjectStore::Transaction &t = *_t;
// store new maps: queue for disk and put in the osdmap cache
+ epoch_t last_marked_full = 0;
epoch_t start = MAX(osdmap->get_epoch() + 1, first);
for (epoch_t e = start; e <= last; e++) {
map<epoch_t,bufferlist>::iterator p;
@@ -5076,6 +5116,8 @@ void OSD::handle_osd_map(MOSDMap *m)
bufferlist& bl = p->second;
o->decode(bl);
+ if (o->test_flag(CEPH_OSDMAP_FULL))
+ last_marked_full = e;
pinned_maps.push_back(add_map(o));
hobject_t fulloid = get_osdmap_pobject_name(e);
@@ -5108,6 +5150,8 @@ void OSD::handle_osd_map(MOSDMap *m)
assert(0 == "bad fsid");
}
+ if (o->test_flag(CEPH_OSDMAP_FULL))
+ last_marked_full = e;
pinned_maps.push_back(add_map(o));
bufferlist fbl;
@@ -5143,6 +5187,8 @@ void OSD::handle_osd_map(MOSDMap *m)
superblock.oldest_map = first;
superblock.newest_map = last;
+ if (last_marked_full > superblock.last_map_marked_full)
+ superblock.last_map_marked_full = last_marked_full;
map_lock.get_write();
@@ -5274,7 +5320,7 @@ void OSD::handle_osd_map(MOSDMap *m)
map_lock.put_write();
- check_osdmap_features();
+ check_osdmap_features(store);
// yay!
consume_map();
@@ -5306,7 +5352,7 @@ void OSD::handle_osd_map(MOSDMap *m)
m->put();
}
-void OSD::check_osdmap_features()
+void OSD::check_osdmap_features(ObjectStore *fs)
{
// adjust required feature bits?
@@ -5338,6 +5384,17 @@ void OSD::check_osdmap_features()
cluster_messenger->set_policy(entity_name_t::TYPE_OSD, p);
}
}
+
+ if ((features & CEPH_FEATURE_OSD_ERASURE_CODES) &&
+ !fs->get_allow_sharded_objects()) {
+ dout(0) << __func__ << " enabling on-disk ERASURE CODES compat feature" << dendl;
+ superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+ ObjectStore::Transaction t;
+ write_superblock(t);
+ int err = store->apply_transaction(t);
+ assert(err == 0);
+ fs->set_allow_sharded_objects();
+ }
}
void OSD::advance_pg(
@@ -5407,16 +5464,16 @@ void OSD::advance_map(ObjectStore::Transaction& t, C_Contexts *tfin)
}
// scan pg creations
- hash_map<pg_t, create_pg_info>::iterator n = creating_pgs.begin();
+ ceph::unordered_map<pg_t, create_pg_info>::iterator n = creating_pgs.begin();
while (n != creating_pgs.end()) {
- hash_map<pg_t, create_pg_info>::iterator p = n++;
+ ceph::unordered_map<pg_t, create_pg_info>::iterator p = n++;
pg_t pgid = p->first;
// am i still primary?
vector<int> acting;
- int nrep = osdmap->pg_to_acting_osds(pgid, acting);
- int role = osdmap->calc_pg_role(whoami, acting, nrep);
- if (role != 0) {
+ int primary;
+ osdmap->pg_to_acting_osds(pgid, &acting, &primary);
+ if (primary != whoami) {
dout(10) << " no longer primary for " << pgid << ", stopping creation" << dendl;
creating_pgs.erase(p);
} else {
@@ -5433,7 +5490,6 @@ void OSD::advance_map(ObjectStore::Transaction& t, C_Contexts *tfin)
while (p != waiting_for_pg.end()) {
pg_t pgid = p->first;
- // am i still primary?
vector<int> acting;
int nrep = osdmap->pg_to_acting_osds(pgid, acting);
int role = osdmap->calc_pg_role(whoami, acting, nrep);
@@ -5458,7 +5514,7 @@ void OSD::consume_map()
list<PGRef> to_remove;
// scan pg's
- for (hash_map<pg_t,PG*>::iterator it = pg_map.begin();
+ for (ceph::unordered_map<pg_t,PG*>::iterator it = pg_map.begin();
it != pg_map.end();
++it) {
PG *pg = it->second;
@@ -5495,7 +5551,7 @@ void OSD::consume_map()
service.publish_map(osdmap);
// scan pg's
- for (hash_map<pg_t,PG*>::iterator it = pg_map.begin();
+ for (ceph::unordered_map<pg_t,PG*>::iterator it = pg_map.begin();
it != pg_map.end();
++it) {
PG *pg = it->second;
@@ -5891,15 +5947,20 @@ void OSD::handle_pg_create(OpRequestRef op)
dout(20) << "ignoring localized pg " << pgid << dendl;
continue;
}
+ if (!osdmap->have_pg_pool(pgid.pool())) {
+ dout(20) << "ignoring pg on deleted pool " << pgid << dendl;
+ continue;
+ }
dout(20) << "mkpg " << pgid << " e" << created << dendl;
// is it still ours?
vector<int> up, acting;
- osdmap->pg_to_up_acting_osds(on, up, acting);
+ int up_primary, acting_primary;
+ osdmap->pg_to_up_acting_osds(on, &up, &up_primary, &acting, &acting_primary);
int role = osdmap->calc_pg_role(whoami, acting, acting.size());
- if (role != 0) {
+ if (up_primary != whoami) {
dout(10) << "mkpg " << pgid << " not primary (role=" << role << "), skipping" << dendl;
continue;
}
@@ -5994,13 +6055,15 @@ PG::RecoveryCtx OSD::create_context()
return rctx;
}
-void OSD::dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg)
+void OSD::dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg,
+ ThreadPool::TPHandle *handle)
{
if (!ctx.transaction->empty()) {
ctx.on_applied->add(new ObjectStore::C_DeleteTransaction(ctx.transaction));
int tr = store->queue_transaction(
pg->osr.get(),
- ctx.transaction, ctx.on_applied, ctx.on_safe);
+ ctx.transaction, ctx.on_applied, ctx.on_safe, NULL,
+ TrackedOpRef(), handle);
assert(tr == 0);
ctx.transaction = new ObjectStore::Transaction;
ctx.on_applied = new C_Contexts(cct);
@@ -6011,8 +6074,11 @@ void OSD::dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg)
bool OSD::compat_must_dispatch_immediately(PG *pg)
{
assert(pg->is_locked());
- for (vector<int>::iterator i = pg->acting.begin();
- i != pg->acting.end();
+ vector<int> *tmpacting = &pg->acting;
+ if (pg->actingbackfill.size() > 0)
+ tmpacting = &pg->actingbackfill;
+ for (vector<int>::iterator i = tmpacting->begin();
+ i != tmpacting->end();
++i) {
if (*i == whoami)
continue;
@@ -6025,9 +6091,11 @@ bool OSD::compat_must_dispatch_immediately(PG *pg)
return false;
}
-void OSD::dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap)
+void OSD::dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap,
+ ThreadPool::TPHandle *handle)
{
- if (service.get_osdmap()->is_up(whoami)) {
+ if (service.get_osdmap()->is_up(whoami) &&
+ is_active()) {
do_notifies(*ctx.notify_list, curmap);
do_queries(*ctx.query_map, curmap);
do_infos(*ctx.info_map, curmap);
@@ -6045,7 +6113,8 @@ void OSD::dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap)
ctx.on_applied->add(new ObjectStore::C_DeleteTransaction(ctx.transaction));
int tr = store->queue_transaction(
pg->osr.get(),
- ctx.transaction, ctx.on_applied, ctx.on_safe);
+ ctx.transaction, ctx.on_applied, ctx.on_safe, NULL, TrackedOpRef(),
+ handle);
assert(tr == 0);
}
}
@@ -6315,11 +6384,14 @@ void OSD::handle_pg_trim(OpRequestRef op)
} else {
// primary is instructing us to trim
ObjectStore::Transaction *t = new ObjectStore::Transaction;
- pg->pg_log.trim(m->trim_to, pg->info);
+ PG::PGLogEntryHandler handler;
+ pg->pg_log.trim(&handler, m->trim_to, pg->info);
+ handler.apply(pg, t);
pg->dirty_info = true;
pg->write_if_dirty(*t);
- int tr = store->queue_transaction(pg->osr.get(), t,
- new ObjectStore::C_DeleteTransaction(t));
+ int tr = store->queue_transaction(
+ pg->osr.get(), t,
+ new ObjectStore::C_DeleteTransaction(t));
assert(tr == 0);
}
pg->unlock();
@@ -6538,7 +6610,6 @@ void OSD::handle_pg_query(OpRequestRef op)
// get active crush mapping
vector<int> up, acting;
osdmap->pg_to_up_acting_osds(pgid, up, acting);
- int role = osdmap->calc_pg_role(whoami, acting, acting.size());
// same primary?
pg_history_t history = it->second.history;
@@ -6553,7 +6624,6 @@ void OSD::handle_pg_query(OpRequestRef op)
continue;
}
- assert(role != 0);
dout(10) << " pg " << pgid << " dne" << dendl;
pg_info_t empty(pgid);
if (it->second.type == pg_query_t::LOG ||
@@ -6640,6 +6710,13 @@ void OSD::_remove_pg(PG *pg)
service.cancel_pending_splits_for_parent(pg->info.pgid);
+ store->queue_transaction(
+ pg->osr.get(), rmt,
+ new ObjectStore::C_DeleteTransactionHolder<
+ SequencerRef>(rmt, pg->osr),
+ new ContainerContext<
+ SequencerRef>(pg->osr));
+
DeletingStateRef deleting = service.deleting_pgs.lookup_or_create(
pg->info.pgid,
make_pair(
@@ -6648,13 +6725,6 @@ void OSD::_remove_pg(PG *pg)
);
remove_wq.queue(make_pair(PGRef(pg), deleting));
- store->queue_transaction(
- pg->osr.get(), rmt,
- new ObjectStore::C_DeleteTransactionHolder<
- SequencerRef>(rmt, pg->osr),
- new ContainerContext<
- SequencerRef>(pg->osr));
-
// remove from map
pg_map.erase(pg->info.pgid);
pg->put("PGMap"); // since we've taken it out of map
@@ -6688,7 +6758,7 @@ void OSD::check_replay_queue()
dout(10) << "check_replay_queue " << *pg << dendl;
if (pg->is_active() &&
pg->is_replay() &&
- pg->get_role() == 0 &&
+ pg->is_primary() &&
pg->replay_until == p->second) {
pg->replay_queued_ops();
}
@@ -6937,9 +7007,11 @@ void OSD::handle_op(OpRequestRef op)
if (op->may_write()) {
// full?
if ((service.check_failsafe_full() ||
- osdmap->test_flag(CEPH_OSDMAP_FULL)) &&
+ osdmap->test_flag(CEPH_OSDMAP_FULL) ||
+ m->get_map_epoch() < superblock.last_map_marked_full) &&
!m->get_source().is_mds()) { // FIXME: we'll exclude mds writes for now.
- service.reply_op_error(op, -ENOSPC);
+ // Drop the request, since the client will retry when the full
+ // flag is unset.
return;
}
@@ -7024,7 +7096,7 @@ void OSD::handle_replica_op(OpRequestRef op)
T *m = static_cast<T *>(op->get_req());
assert(m->get_header().type == MSGTYPE);
- dout(10) << __func__ << *m << " epoch " << m->map_epoch << dendl;
+ dout(10) << __func__ << " " << *m << " epoch " << m->map_epoch << dendl;
if (m->map_epoch < up_epoch) {
dout(3) << "replica op from before up" << dendl;
return;
@@ -7166,7 +7238,9 @@ void OSD::dequeue_op(
PGRef pg, OpRequestRef op,
ThreadPool::TPHandle &handle)
{
- utime_t latency = ceph_clock_now(cct) - op->get_req()->get_recv_stamp();
+ utime_t now = ceph_clock_now(cct);
+ op->set_dequeued_time(now);
+ utime_t latency = now - op->get_req()->get_recv_stamp();
dout(10) << "dequeue_op " << op << " prio " << op->get_req()->get_priority()
<< " cost " << op->get_req()->get_cost()
<< " latency " << latency
@@ -7250,17 +7324,17 @@ void OSD::process_peering_events(
split_pgs.clear();
}
if (compat_must_dispatch_immediately(pg)) {
- dispatch_context(rctx, pg, curmap);
+ dispatch_context(rctx, pg, curmap, &handle);
rctx = create_context();
} else {
- dispatch_context_transaction(rctx, pg);
+ dispatch_context_transaction(rctx, pg, &handle);
}
pg->unlock();
handle.reset_tp_timeout();
}
if (need_up_thru)
queue_want_up_thru(same_interval_since);
- dispatch_context(rctx, 0, curmap);
+ dispatch_context(rctx, 0, curmap, &handle);
service.send_pg_temp();
}
@@ -7322,6 +7396,9 @@ int OSD::init_op_flags(OpRequestRef op)
if (ceph_osd_op_type_pg(iter->op.op))
op->set_pg_op();
+ if (ceph_osd_op_mode_cache(iter->op.op))
+ op->set_cache();
+
switch (iter->op.op) {
case CEPH_OSD_OP_CALL:
{
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index b9fb615..cebceb7 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -42,12 +42,11 @@
#include <map>
#include <memory>
-#include <tr1/memory>
+#include "include/memory.h"
using namespace std;
-#include <ext/hash_map>
-#include <ext/hash_set>
-using namespace __gnu_cxx;
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
#include "Watch.h"
#include "common/shared_cache.hpp"
@@ -66,18 +65,22 @@ enum {
l_osd_op_inb,
l_osd_op_outb,
l_osd_op_lat,
+ l_osd_op_process_lat,
l_osd_op_r,
l_osd_op_r_outb,
l_osd_op_r_lat,
+ l_osd_op_r_process_lat,
l_osd_op_w,
l_osd_op_w_inb,
l_osd_op_w_rlat,
l_osd_op_w_lat,
+ l_osd_op_w_process_lat,
l_osd_op_rw,
l_osd_op_rw_inb,
l_osd_op_rw_outb,
l_osd_op_rw_rlat,
l_osd_op_rw_lat,
+ l_osd_op_rw_process_lat,
l_osd_sop,
l_osd_sop_inb,
@@ -179,7 +182,7 @@ class HistoricOpsSocketHook;
class TestOpsSocketHook;
struct C_CompleteSplits;
-typedef std::tr1::shared_ptr<ObjectStore::Sequencer> SequencerRef;
+typedef ceph::shared_ptr<ObjectStore::Sequencer> SequencerRef;
class DeletingState {
Mutex lock;
@@ -279,7 +282,7 @@ public:
return status != DELETED_DIR;
} ///< @return true if we don't need to recreate the collection
};
-typedef std::tr1::shared_ptr<DeletingState> DeletingStateRef;
+typedef ceph::shared_ptr<DeletingState> DeletingStateRef;
class OSD;
class OSDService {
@@ -308,6 +311,7 @@ public:
ThreadPool::WorkQueue<PG> &scrub_finalize_wq;
ThreadPool::WorkQueue<MOSDRepScrub> &rep_scrub_wq;
GenContextWQ push_wq;
+ GenContextWQ gen_wq;
ClassHandler *&class_handler;
void dequeue_pg(PG *pg, list<OpRequestRef> *dequeued);
@@ -440,7 +444,7 @@ public:
bool force_new);
ObjecterDispatcher(OSDService *o) : Dispatcher(cct), osd(o) {}
} objecter_dispatcher;
- friend class ObjecterDispatcher;
+ friend struct ObjecterDispatcher;
// -- Watch --
@@ -697,7 +701,7 @@ protected:
void _dispatch(Message *m);
void dispatch_op(OpRequestRef op);
- void check_osdmap_features();
+ void check_osdmap_features(ObjectStore *store);
// asok
friend class OSDSocketHook;
@@ -1142,7 +1146,7 @@ private:
protected:
// -- placement groups --
- hash_map<pg_t, PG*> pg_map;
+ ceph::unordered_map<pg_t, PG*> pg_map;
map<pg_t, list<OpRequestRef> > waiting_for_pg;
map<pg_t, list<PG::CephPeeringEvtRef> > peering_wait_for_split;
PGRecoveryStats pg_recovery_stats;
@@ -1220,7 +1224,7 @@ protected:
set<int> prior;
pg_t parent;
};
- hash_map<pg_t, create_pg_info> creating_pgs;
+ ceph::unordered_map<pg_t, create_pg_info> creating_pgs;
double debug_drop_pg_create_probability;
int debug_drop_pg_create_duration;
int debug_drop_pg_create_left; // 0 if we just dropped the last one, -1 if we can drop more
@@ -1254,6 +1258,7 @@ protected:
void start_boot();
void _maybe_boot(epoch_t oldest, epoch_t newest);
void _send_boot();
+ void _collect_metadata(map<string,string> *pmeta);
void start_waiting_for_healthy();
bool _is_healthy();
@@ -1318,8 +1323,10 @@ protected:
// -- generic pg peering --
PG::RecoveryCtx create_context();
bool compat_must_dispatch_immediately(PG *pg);
- void dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap);
- void dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg);
+ void dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap,
+ ThreadPool::TPHandle *handle = NULL);
+ void dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg,
+ ThreadPool::TPHandle *handle = NULL);
void do_notifies(map< int,vector<pair<pg_notify_t, pg_interval_map_t> > >& notify_list,
OSDMapRef map);
void do_queries(map< int, map<pg_t,pg_query_t> >& query_map,
@@ -1559,12 +1566,11 @@ protected:
struct ScrubFinalizeWQ : public ThreadPool::WorkQueue<PG> {
private:
- OSD *osd;
xlist<PG*> scrub_finalize_queue;
public:
- ScrubFinalizeWQ(OSD *o, time_t ti, ThreadPool *tp)
- : ThreadPool::WorkQueue<PG>("OSD::ScrubFinalizeWQ", ti, ti*10, tp), osd(o) {}
+ ScrubFinalizeWQ(time_t ti, ThreadPool *tp)
+ : ThreadPool::WorkQueue<PG>("OSD::ScrubFinalizeWQ", ti, ti*10, tp) {}
bool _empty() {
return scrub_finalize_queue.empty();
@@ -1709,6 +1715,7 @@ protected:
/* internal and external can point to the same messenger, they will still
* be cleaned up properly*/
OSD(CephContext *cct_,
+ ObjectStore *store_,
int id,
Messenger *internal,
Messenger *external,
@@ -1721,15 +1728,11 @@ protected:
// static bits
static int find_osd_dev(char *result, int whoami);
- static ObjectStore *create_object_store(CephContext *cct, const std::string &dev, const std::string &jdev);
- static int convertfs(const std::string &dev, const std::string &jdev);
static int do_convertfs(ObjectStore *store);
static int convert_collection(ObjectStore *store, coll_t cid);
- static int mkfs(CephContext *cct, const std::string &dev, const std::string &jdev,
+ static int mkfs(CephContext *cct, ObjectStore *store,
+ const string& dev,
uuid_d fsid, int whoami);
- static int mkjournal(CephContext *cct, const std::string &dev, const std::string &jdev);
- static int flushjournal(CephContext *cct, const std::string &dev, const std::string &jdev);
- static int dump_journal(CephContext *cct, const std::string &dev, const std::string &jdev, ostream& out);
/* remove any non-user xattrs from a map of them */
void filter_xattrs(map<string, bufferptr>& attrs) {
for (map<string, bufferptr>::iterator iter = attrs.begin();
@@ -1742,12 +1745,11 @@ protected:
}
private:
- static int write_meta(const std::string &base,
+ static int write_meta(ObjectStore *store,
uuid_d& cluster_fsid, uuid_d& osd_fsid, int whoami);
public:
- static int peek_meta(const std::string &dev, string& magic,
+ static int peek_meta(ObjectStore *store, string& magic,
uuid_d& cluster_fsid, uuid_d& osd_fsid, int& whoami);
- static int peek_journal_fsid(std::string jpath, uuid_d& fsid);
// startup/shutdown
diff --git a/src/osd/OSDCap.cc b/src/osd/OSDCap.cc
index e315835..383674f 100644
--- a/src/osd/OSDCap.cc
+++ b/src/osd/OSDCap.cc
@@ -182,7 +182,8 @@ struct OSDCapParser : qi::grammar<Iterator, OSDCap()>
str %= quoted_string | unquoted_word;
estr %= equoted_string | unquoted_word;
- spaces = +lit(' ');
+ spaces = +(lit(' ') | lit('\n') | lit('\t'));
+
// match := [pool[=]<poolname> [namespace[=]<namespace>] | auid <123>] [object_prefix <prefix>]
pool_name %= -(spaces >> lit("pool") >> (lit('=') | spaces) >> str);
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 8007d61..902717a 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -90,15 +90,17 @@ void osd_xinfo_t::dump(Formatter *f) const
f->dump_stream("down_stamp") << down_stamp;
f->dump_float("laggy_probability", laggy_probability);
f->dump_int("laggy_interval", laggy_interval);
+ f->dump_int("features", features);
}
void osd_xinfo_t::encode(bufferlist& bl) const
{
- ENCODE_START(1, 1, bl);
+ ENCODE_START(2, 1, bl);
::encode(down_stamp, bl);
__u32 lp = laggy_probability * 0xfffffffful;
::encode(lp, bl);
::encode(laggy_interval, bl);
+ ::encode(features, bl);
ENCODE_FINISH(bl);
}
@@ -110,6 +112,10 @@ void osd_xinfo_t::decode(bufferlist::iterator& bl)
::decode(lp, bl);
laggy_probability = (float)lp / (float)0xffffffff;
::decode(laggy_interval, bl);
+ if (struct_v >= 2)
+ ::decode(features, bl);
+ else
+ features = 0;
DECODE_FINISH(bl);
}
@@ -172,6 +178,47 @@ int OSDMap::Incremental::identify_osd(uuid_d u) const
return -1;
}
+int OSDMap::Incremental::propagate_snaps_to_tiers(CephContext *cct,
+ const OSDMap& osdmap)
+{
+ assert(epoch == osdmap.get_epoch() + 1);
+ for (map<int64_t,pg_pool_t>::iterator p = new_pools.begin();
+ p != new_pools.end(); ++p) {
+ if (!p->second.tiers.empty()) {
+ pg_pool_t& base = p->second;
+ for (set<uint64_t>::const_iterator q = base.tiers.begin();
+ q != base.tiers.end();
+ ++q) {
+ map<int64_t,pg_pool_t>::iterator r = new_pools.find(*q);
+ pg_pool_t *tier = 0;
+ if (r == new_pools.end()) {
+ const pg_pool_t *orig = osdmap.get_pg_pool(*q);
+ if (!orig) {
+ lderr(cct) << __func__ << " no pool " << *q << dendl;
+ return -EIO;
+ }
+ tier = get_new_pool(*q, orig);
+ } else {
+ tier = &r->second;
+ }
+ if (tier->tier_of != p->first) {
+ lderr(cct) << __func__ << " " << r->first << " tier_of != " << p->first << dendl;
+ return -EIO;
+ }
+
+ ldout(cct, 10) << __func__ << " from " << p->first << " to "
+ << r->first << dendl;
+ tier->snap_seq = base.snap_seq;
+ tier->snap_epoch = base.snap_epoch;
+ tier->snaps = base.snaps;
+ tier->removed_snaps = base.removed_snaps;
+ }
+ }
+ }
+ return 0;
+}
+
+
bool OSDMap::subtree_is_down(int id, set<int> *down_cache) const
{
if (id >= 0)
@@ -287,7 +334,7 @@ void OSDMap::Incremental::encode_client_old(bufferlist& bl) const
}
}
-void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
+void OSDMap::Incremental::encode_classic(bufferlist& bl, uint64_t features) const
{
if ((features & CEPH_FEATURE_PGID64) == 0) {
encode_client_old(bl);
@@ -330,7 +377,60 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
::encode(new_hb_front_up, bl);
}
-void OSDMap::Incremental::decode(bufferlist::iterator &p)
+void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
+{
+ if ((features & CEPH_FEATURE_OSDMAP_ENC) == 0) {
+ encode_classic(bl, features);
+ return;
+ }
+
+ // meta-encoding: how we include client-used and osd-specific data
+ ENCODE_START(7, 7, bl);
+
+ {
+ ENCODE_START(1, 1, bl); // client-usable data
+ ::encode(fsid, bl);
+ ::encode(epoch, bl);
+ ::encode(modified, bl);
+ ::encode(new_pool_max, bl);
+ ::encode(new_flags, bl);
+ ::encode(fullmap, bl);
+ ::encode(crush, bl);
+
+ ::encode(new_max_osd, bl);
+ ::encode(new_pools, bl, features);
+ ::encode(new_pool_names, bl);
+ ::encode(old_pools, bl);
+ ::encode(new_up_client, bl);
+ ::encode(new_state, bl);
+ ::encode(new_weight, bl);
+ ::encode(new_pg_temp, bl);
+ ::encode(new_primary_temp, bl);
+ ENCODE_FINISH(bl); // client-usable data
+ }
+
+ {
+ ENCODE_START(2, 1, bl); // extended, osd-only data
+ ::encode(new_hb_back_up, bl);
+ ::encode(new_up_thru, bl);
+ ::encode(new_last_clean_interval, bl);
+ ::encode(new_lost, bl);
+ ::encode(new_blacklist, bl);
+ ::encode(old_blacklist, bl);
+ ::encode(new_up_cluster, bl);
+ ::encode(cluster_snapshot, bl);
+ ::encode(new_uuid, bl);
+ ::encode(new_xinfo, bl);
+ ::encode(new_hb_front_up, bl);
+ ::encode(features, bl); // NOTE: features arg, not the member
+ ENCODE_FINISH(bl); // osd-only data
+ }
+
+ ENCODE_FINISH(bl); // meta-encoding wrapper
+
+}
+
+void OSDMap::Incremental::decode_classic(bufferlist::iterator &p)
{
__u32 n, t;
// base
@@ -423,6 +523,68 @@ void OSDMap::Incremental::decode(bufferlist::iterator &p)
::decode(new_hb_front_up, p);
}
+void OSDMap::Incremental::decode(bufferlist::iterator& bl)
+{
+ /**
+ * Older encodings of the Incremental had a single struct_v which
+ * covered the whole encoding, and was prior to our modern
+ * stuff which includes a compatv and a size. So if we see
+ * a struct_v < 7, we must rewind to the beginning and use our
+ * classic decoder.
+ */
+ DECODE_START_LEGACY_COMPAT_LEN(7, 7, 7, bl); // wrapper
+ if (struct_v < 7) {
+ int struct_v_size = sizeof(struct_v);
+ bl.advance(-struct_v_size);
+ decode_classic(bl);
+ encode_features = 0;
+ return;
+ }
+ {
+ DECODE_START(1, bl); // client-usable data
+ ::decode(fsid, bl);
+ ::decode(epoch, bl);
+ ::decode(modified, bl);
+ ::decode(new_pool_max, bl);
+ ::decode(new_flags, bl);
+ ::decode(fullmap, bl);
+ ::decode(crush, bl);
+
+ ::decode(new_max_osd, bl);
+ ::decode(new_pools, bl);
+ ::decode(new_pool_names, bl);
+ ::decode(old_pools, bl);
+ ::decode(new_up_client, bl);
+ ::decode(new_state, bl);
+ ::decode(new_weight, bl);
+ ::decode(new_pg_temp, bl);
+ ::decode(new_primary_temp, bl);
+ DECODE_FINISH(bl); // client-usable data
+ }
+
+ {
+ DECODE_START(2, bl); // extended, osd-only data
+ ::decode(new_hb_back_up, bl);
+ ::decode(new_up_thru, bl);
+ ::decode(new_last_clean_interval, bl);
+ ::decode(new_lost, bl);
+ ::decode(new_blacklist, bl);
+ ::decode(old_blacklist, bl);
+ ::decode(new_up_cluster, bl);
+ ::decode(cluster_snapshot, bl);
+ ::decode(new_uuid, bl);
+ ::decode(new_xinfo, bl);
+ ::decode(new_hb_front_up, bl);
+ if (struct_v >= 2)
+ ::decode(encode_features, bl);
+ else
+ encode_features = 0;
+ DECODE_FINISH(bl); // osd-only data
+ }
+
+ DECODE_FINISH(bl); // wrapper
+}
+
void OSDMap::Incremental::dump(Formatter *f) const
{
f->dump_int("epoch", epoch);
@@ -523,6 +685,15 @@ void OSDMap::Incremental::dump(Formatter *f) const
}
f->close_section();
+ f->open_array_section("primary_temp");
+ for (map<pg_t, int>::const_iterator p = new_primary_temp.begin();
+ p != new_primary_temp.end();
+ ++p) {
+ f->dump_stream("pgid") << p->first;
+ f->dump_int("osd", p->second);
+ }
+ f->close_section(); // primary_temp
+
f->open_array_section("new_up_thru");
for (map<int32_t,uint32_t>::const_iterator p = new_up_thru.begin(); p != new_up_thru.end(); ++p) {
f->open_object_section("osd");
@@ -630,7 +801,7 @@ bool OSDMap::is_blacklisted(const entity_addr_t& a) const
void OSDMap::get_blacklist(list<pair<entity_addr_t,utime_t> > *bl) const
{
- for (hash_map<entity_addr_t,utime_t>::const_iterator it = blacklist.begin() ;
+ for (ceph::unordered_map<entity_addr_t,utime_t>::const_iterator it = blacklist.begin() ;
it != blacklist.end(); ++it) {
bl->push_back(*it);
}
@@ -673,18 +844,26 @@ void OSDMap::get_all_osds(set<int32_t>& ls) const
ls.insert(i);
}
-int OSDMap::get_num_up_osds() const
+void OSDMap::get_up_osds(set<int32_t>& ls) const
{
- int n = 0;
+ for (int i = 0; i < max_osd; i++) {
+ if (is_up(i))
+ ls.insert(i);
+ }
+}
+
+unsigned OSDMap::get_num_up_osds() const
+{
+ unsigned n = 0;
for (int i=0; i<max_osd; i++)
if ((osd_state[i] & CEPH_OSD_EXISTS) &&
(osd_state[i] & CEPH_OSD_UP)) n++;
return n;
}
-int OSDMap::get_num_in_osds() const
+unsigned OSDMap::get_num_in_osds() const
{
- int n = 0;
+ unsigned n = 0;
for (int i=0; i<max_osd; i++)
if ((osd_state[i] & CEPH_OSD_EXISTS) &&
get_weight(i) != CEPH_OSD_OUT) n++;
@@ -744,31 +923,55 @@ bool OSDMap::find_osd_on_ip(const entity_addr_t& ip) const
uint64_t OSDMap::get_features(uint64_t *pmask) const
{
- uint64_t features = 0;
- uint64_t mask = 0;
+ uint64_t features = 0; // things we actually have
+ uint64_t mask = 0; // things we could have
if (crush->has_nondefault_tunables())
features |= CEPH_FEATURE_CRUSH_TUNABLES;
if (crush->has_nondefault_tunables2())
features |= CEPH_FEATURE_CRUSH_TUNABLES2;
+ if (crush->has_v2_rules())
+ features |= CEPH_FEATURE_CRUSH_V2;
mask |= CEPH_FEATURES_CRUSH;
for (map<int64_t,pg_pool_t>::const_iterator p = pools.begin(); p != pools.end(); ++p) {
if (p->second.flags & pg_pool_t::FLAG_HASHPSPOOL) {
features |= CEPH_FEATURE_OSDHASHPSPOOL;
}
+ if (p->second.is_erasure()) {
+ features |= CEPH_FEATURE_OSD_ERASURE_CODES;
+ }
if (!p->second.tiers.empty() ||
p->second.is_tier()) {
features |= CEPH_FEATURE_OSD_CACHEPOOL;
}
}
- mask |= CEPH_FEATURE_OSDHASHPSPOOL;
+ mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL |
+ CEPH_FEATURE_OSD_ERASURE_CODES;
if (pmask)
*pmask = mask;
return features;
}
+uint64_t OSDMap::get_up_osd_features() const
+{
+ bool first = true;
+ uint64_t features = 0;
+ for (int osd = 0; osd < max_osd; ++osd) {
+ if (!is_up(osd))
+ continue;
+ const osd_xinfo_t &xi = get_xinfo(osd);
+ if (first) {
+ features = xi.features;
+ first = false;
+ } else {
+ features &= xi.features;
+ }
+ }
+ return features;
+}
+
void OSDMap::dedup(const OSDMap *o, OSDMap *n)
{
if (o->epoch == n->epoch)
@@ -820,12 +1023,87 @@ void OSDMap::dedup(const OSDMap *o, OSDMap *n)
n->pg_temp = o->pg_temp;
}
+ // does primary_temp match?
+ if (o->primary_temp->size() == n->primary_temp->size()) {
+ if (*o->primary_temp == *n->primary_temp)
+ n->primary_temp = o->primary_temp;
+ }
+
// do uuids match?
if (o->osd_uuid->size() == n->osd_uuid->size() &&
*o->osd_uuid == *n->osd_uuid)
n->osd_uuid = o->osd_uuid;
}
+void OSDMap::remove_redundant_temporaries(CephContext *cct, const OSDMap& osdmap,
+ OSDMap::Incremental *pending_inc)
+{
+ ldout(cct, 10) << "remove_redundant_temporaries" << dendl;
+
+ for (map<pg_t,vector<int> >::iterator p = osdmap.pg_temp->begin();
+ p != osdmap.pg_temp->end();
+ ++p) {
+ if (pending_inc->new_pg_temp.count(p->first) == 0) {
+ vector<int> raw_up;
+ int primary;
+ osdmap.pg_to_raw_up(p->first, &raw_up, &primary);
+ if (raw_up == p->second) {
+ ldout(cct, 10) << " removing unnecessary pg_temp " << p->first << " -> " << p->second << dendl;
+ pending_inc->new_pg_temp[p->first].clear();
+ }
+ }
+ }
+ if (!osdmap.primary_temp->empty()) {
+ OSDMap templess;
+ templess.deepish_copy_from(osdmap);
+ templess.primary_temp->clear();
+ for (map<pg_t,int>::iterator p = osdmap.primary_temp->begin();
+ p != osdmap.primary_temp->end();
+ ++p) {
+ if (pending_inc->new_primary_temp.count(p->first) == 0) {
+ vector<int> real_up, templess_up;
+ int real_primary, templess_primary;
+ osdmap.pg_to_acting_osds(p->first, &real_up, &real_primary);
+ templess.pg_to_acting_osds(p->first, &templess_up, &templess_primary);
+ if (real_primary == templess_primary){
+ ldout(cct, 10) << " removing unnecessary primary_temp "
+ << p->first << " -> " << p->second << dendl;
+ pending_inc->new_primary_temp[p->first] = -1;
+ }
+ }
+ }
+ }
+}
+
+void OSDMap::remove_down_temps(CephContext *cct,
+ const OSDMap& osdmap, Incremental *pending_inc)
+{
+ ldout(cct, 10) << "remove_down_pg_temp" << dendl;
+ OSDMap tmpmap;
+ tmpmap.deepish_copy_from(osdmap);
+ tmpmap.apply_incremental(*pending_inc);
+
+ for (map<pg_t,vector<int> >::iterator p = tmpmap.pg_temp->begin();
+ p != tmpmap.pg_temp->end();
+ ++p) {
+ unsigned num_up = 0;
+ for (vector<int>::iterator i = p->second.begin();
+ i != p->second.end();
+ ++i) {
+ if (!tmpmap.is_down(*i))
+ ++num_up;
+ }
+ if (num_up == 0)
+ pending_inc->new_pg_temp[p->first].clear();
+ }
+ for (map<pg_t,int>::iterator p = tmpmap.primary_temp->begin();
+ p != tmpmap.primary_temp->end();
+ ++p) {
+ if (tmpmap.is_down(p->second))
+ pending_inc->new_primary_temp[p->first] = -1;
+ }
+}
+
int OSDMap::apply_incremental(const Incremental &inc)
{
new_blacklist_entries = false;
@@ -955,6 +1233,15 @@ int OSDMap::apply_incremental(const Incremental &inc)
(*pg_temp)[p->first] = p->second;
}
+ for (map<pg_t,int>::const_iterator p = inc.new_primary_temp.begin();
+ p != inc.new_primary_temp.end();
+ ++p) {
+ if (p->second == -1)
+ primary_temp->erase(p->first);
+ else
+ (*primary_temp)[p->first] = p->second;
+ }
+
// blacklist
for (map<entity_addr_t,utime_t>::const_iterator p = inc.new_blacklist.begin();
p != inc.new_blacklist.end();
@@ -988,13 +1275,6 @@ int OSDMap::apply_incremental(const Incremental &inc)
return 0;
}
-static string make_hash_str(const string &inkey, const string &nspace)
-{
- if (nspace.empty())
- return inkey;
- return nspace + '\037' + inkey;
-}
-
// mapping
int OSDMap::object_locator_to_pg(
const object_t& oid,
@@ -1006,13 +1286,14 @@ int OSDMap::object_locator_to_pg(
if (!pool)
return -ENOENT;
ps_t ps;
- string key;
- if (!loc.key.empty())
- key = make_hash_str(loc.key, loc.nspace);
- else
- key = make_hash_str(oid.name, loc.nspace);
-
- ps = ceph_str_hash(pool->object_hash, key.c_str(), key.length());
+ if (loc.hash >= 0) {
+ ps = loc.hash;
+ } else {
+ if (!loc.key.empty())
+ ps = pool->hash_key(loc.key, loc.nspace);
+ else
+ ps = pool->hash_key(oid.name, loc.nspace);
+ }
pg = pg_t(ps, loc.get_pool(), -1);
return 0;
}
@@ -1029,23 +1310,32 @@ ceph_object_layout OSDMap::make_object_layout(
return ol;
}
-void OSDMap::_remove_nonexistent_osds(vector<int>& osds) const
+void OSDMap::_remove_nonexistent_osds(const pg_pool_t& pool,
+ vector<int>& osds) const
{
- unsigned removed = 0;
- for (unsigned i = 0; i < osds.size(); i++) {
- if (!exists(osds[i])) {
- removed++;
- continue;
+ if (pool.can_shift_osds()) {
+ unsigned removed = 0;
+ for (unsigned i = 0; i < osds.size(); i++) {
+ if (!exists(osds[i])) {
+ removed++;
+ continue;
+ }
+ if (removed) {
+ osds[i - removed] = osds[i];
+ }
}
- if (removed) {
- osds[i - removed] = osds[i];
+ if (removed)
+ osds.resize(osds.size() - removed);
+ } else {
+ for (vector<int>::iterator p = osds.begin(); p != osds.end(); ++p) {
+ if (!exists(*p))
+ *p = CRUSH_ITEM_NONE;
}
}
- if (removed)
- osds.resize(osds.size() - removed);
}
-int OSDMap::_pg_to_osds(const pg_pool_t& pool, pg_t pg, vector<int>& osds) const
+int OSDMap::_pg_to_osds(const pg_pool_t& pool, pg_t pg,
+ vector<int> *osds, int *primary) const
{
// map to osds[]
ps_t pps = pool.raw_pg_to_pps(pg); // placement ps
@@ -1054,80 +1344,110 @@ int OSDMap::_pg_to_osds(const pg_pool_t& pool, pg_t pg, vector<int>& osds) const
// what crush rule?
int ruleno = crush->find_rule(pool.get_crush_ruleset(), pool.get_type(), size);
if (ruleno >= 0)
- crush->do_rule(ruleno, pps, osds, size, osd_weight);
+ crush->do_rule(ruleno, pps, *osds, size, osd_weight);
- _remove_nonexistent_osds(osds);
+ _remove_nonexistent_osds(pool, *osds);
- return osds.size();
+ *primary = (osds->empty() ? -1 : osds->front());
+
+ return osds->size();
}
// pg -> (up osd list)
-void OSDMap::_raw_to_up_osds(pg_t pg, vector<int>& raw, vector<int>& up) const
+void OSDMap::_raw_to_up_osds(pg_t pg, const vector<int>& raw,
+ vector<int> *up, int *primary) const
{
- up.clear();
+ up->clear();
for (unsigned i=0; i<raw.size(); i++) {
- if (!exists(raw[i]) || is_down(raw[i]))
+ if (!exists(raw[i]) || is_down(raw[i]))
continue;
- up.push_back(raw[i]);
+ up->push_back(raw[i]);
}
+ *primary = (up->empty() ? -1 : up->front());
}
-bool OSDMap::_raw_to_temp_osds(const pg_pool_t& pool, pg_t pg, vector<int>& raw, vector<int>& temp) const
+void OSDMap::_get_temp_osds(const pg_pool_t& pool, pg_t pg,
+ vector<int> *temp_pg, int *temp_primary) const
{
pg = pool.raw_pg_to_pg(pg);
map<pg_t,vector<int> >::const_iterator p = pg_temp->find(pg);
+ temp_pg->clear();
if (p != pg_temp->end()) {
- temp.clear();
for (unsigned i=0; i<p->second.size(); i++) {
if (!exists(p->second[i]) || is_down(p->second[i]))
continue;
- temp.push_back(p->second[i]);
+ temp_pg->push_back(p->second[i]);
}
- return true;
}
- return false;
+ map<pg_t,int>::const_iterator pp = primary_temp->find(pg);
+ *temp_primary = -1;
+ if (pp != primary_temp->end())
+ *temp_primary = pp->second;
+ else if (!temp_pg->empty()) // apply pg_temp's primary
+ *temp_primary = temp_pg->front();
}
-int OSDMap::pg_to_osds(pg_t pg, vector<int>& raw) const
+int OSDMap::pg_to_osds(pg_t pg, vector<int> *raw, int *primary) const
{
+ *primary = -1;
+ raw->clear();
const pg_pool_t *pool = get_pg_pool(pg.pool());
if (!pool)
return 0;
- return _pg_to_osds(*pool, pg, raw);
-}
-
-int OSDMap::pg_to_acting_osds(pg_t pg, vector<int>& acting) const
-{
- const pg_pool_t *pool = get_pg_pool(pg.pool());
- if (!pool)
- return 0;
- vector<int> raw;
- _pg_to_osds(*pool, pg, raw);
- if (!_raw_to_temp_osds(*pool, pg, raw, acting))
- _raw_to_up_osds(pg, raw, acting);
- return acting.size();
+ int r = _pg_to_osds(*pool, pg, raw, primary);
+ return r;
}
-void OSDMap::pg_to_raw_up(pg_t pg, vector<int>& up) const
+void OSDMap::pg_to_raw_up(pg_t pg, vector<int> *up, int *primary) const
{
const pg_pool_t *pool = get_pg_pool(pg.pool());
- if (!pool)
+ if (!pool) {
+ if (primary)
+ *primary = -1;
+ if (up)
+ up->clear();
return;
+ }
vector<int> raw;
- _pg_to_osds(*pool, pg, raw);
- _raw_to_up_osds(pg, raw, up);
+ _pg_to_osds(*pool, pg, &raw, primary);
+ _raw_to_up_osds(pg, raw, up, primary);
}
-void OSDMap::pg_to_up_acting_osds(pg_t pg, vector<int>& up, vector<int>& acting) const
+void OSDMap::_pg_to_up_acting_osds(pg_t pg, vector<int> *up, int *up_primary,
+ vector<int> *acting, int *acting_primary) const
{
const pg_pool_t *pool = get_pg_pool(pg.pool());
- if (!pool)
+ if (!pool) {
+ if (up)
+ up->clear();
+ if (up_primary)
+ *up_primary = -1;
+ if (acting)
+ acting->clear();
+ if (acting_primary)
+ *acting_primary = -1;
return;
+ }
vector<int> raw;
- _pg_to_osds(*pool, pg, raw);
- _raw_to_up_osds(pg, raw, up);
- if (!_raw_to_temp_osds(*pool, pg, raw, acting))
- acting = up;
+ vector<int> _up;
+ vector<int> _acting;
+ int _up_primary;
+ int _acting_primary;
+ _pg_to_osds(*pool, pg, &raw, &_up_primary);
+ _raw_to_up_osds(pg, raw, &_up, &_up_primary);
+ _get_temp_osds(*pool, pg, &_acting, &_acting_primary);
+ if (_acting.empty())
+ _acting = _up;
+ if (_acting_primary == -1)
+ _acting_primary = _up_primary;
+ if (up)
+ up->swap(_up);
+ if (up_primary)
+ *up_primary = _up_primary;
+ if (acting)
+ acting->swap(_acting);
+ if (acting_primary)
+ *acting_primary = _acting_primary;
}
int OSDMap::calc_pg_rank(int osd, vector<int>& acting, int nrep)
@@ -1208,7 +1528,7 @@ void OSDMap::encode_client_old(bufferlist& bl) const
::encode(cbl, bl);
}
-void OSDMap::encode(bufferlist& bl, uint64_t features) const
+void OSDMap::encode_classic(bufferlist& bl, uint64_t features) const
{
if ((features & CEPH_FEATURE_PGID64) == 0) {
encode_client_old(bl);
@@ -1256,13 +1576,68 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
::encode(osd_addrs->hb_front_addr, bl);
}
+void OSDMap::encode(bufferlist& bl, uint64_t features) const
+{
+ if ((features & CEPH_FEATURE_OSDMAP_ENC) == 0) {
+ encode_classic(bl, features);
+ return;
+ }
+ // meta-encoding: how we include client-used and osd-specific data
+ ENCODE_START(7, 7, bl);
+
+ {
+ ENCODE_START(1, 1, bl); // client-usable data
+ // base
+ ::encode(fsid, bl);
+ ::encode(epoch, bl);
+ ::encode(created, bl);
+ ::encode(modified, bl);
+
+ ::encode(pools, bl, features);
+ ::encode(pool_name, bl);
+ ::encode(pool_max, bl);
+
+ ::encode(flags, bl);
+
+ ::encode(max_osd, bl);
+ ::encode(osd_state, bl);
+ ::encode(osd_weight, bl);
+ ::encode(osd_addrs->client_addr, bl);
+
+ ::encode(*pg_temp, bl);
+ ::encode(*primary_temp, bl);
+
+ // crush
+ bufferlist cbl;
+ crush->encode(cbl);
+ ::encode(cbl, bl);
+ ENCODE_FINISH(bl); // client-usable data
+ }
+
+ {
+ ENCODE_START(1, 1, bl); // extended, osd-only data
+ ::encode(osd_addrs->hb_back_addr, bl);
+ ::encode(osd_info, bl);
+ ::encode(blacklist, bl);
+ ::encode(osd_addrs->cluster_addr, bl);
+ ::encode(cluster_snapshot_epoch, bl);
+ ::encode(cluster_snapshot, bl);
+ ::encode(*osd_uuid, bl);
+ ::encode(osd_xinfo, bl);
+ ::encode(osd_addrs->hb_front_addr, bl);
+ ENCODE_FINISH(bl); // osd-only data
+ }
+
+ ENCODE_FINISH(bl); // meta-encoding wrapper
+}
+
void OSDMap::decode(bufferlist& bl)
{
bufferlist::iterator p = bl.begin();
decode(p);
}
-void OSDMap::decode(bufferlist::iterator& p)
+void OSDMap::decode_classic(bufferlist::iterator& p)
{
__u32 n, t;
__u16 v;
@@ -1368,16 +1743,89 @@ void OSDMap::decode(bufferlist::iterator& p)
else
osd_addrs->hb_front_addr.resize(osd_addrs->hb_back_addr.size());
+ post_decode();
+}
+
+void OSDMap::decode(bufferlist::iterator& bl)
+{
+ /**
+ * Older encodings of the OSDMap had a single struct_v which
+ * covered the whole encoding, and was prior to our modern
+ * stuff which includes a compatv and a size. So if we see
+ * a struct_v < 7, we must rewind to the beginning and use our
+ * classic decoder.
+ */
+ DECODE_START_LEGACY_COMPAT_LEN(7, 7, 7, bl); // wrapper
+ if (struct_v < 7) {
+ int struct_v_size = sizeof(struct_v);
+ bl.advance(-struct_v_size);
+ decode_classic(bl);
+ return;
+ }
+ /**
+ * Since we made it past that hurdle, we can use our normal paths.
+ */
+ {
+ DECODE_START(1, bl); // client-usable data
+ // base
+ ::decode(fsid, bl);
+ ::decode(epoch, bl);
+ ::decode(created, bl);
+ ::decode(modified, bl);
+
+ ::decode(pools, bl);
+ ::decode(pool_name, bl);
+ ::decode(pool_max, bl);
+
+ ::decode(flags, bl);
+
+ ::decode(max_osd, bl);
+ ::decode(osd_state, bl);
+ ::decode(osd_weight, bl);
+ ::decode(osd_addrs->client_addr, bl);
+
+ ::decode(*pg_temp, bl);
+ ::decode(*primary_temp, bl);
+
+ // crush
+ bufferlist cbl;
+ ::decode(cbl, bl);
+ bufferlist::iterator cblp = cbl.begin();
+ crush->decode(cblp);
+ DECODE_FINISH(bl); // client-usable data
+ }
+
+ {
+ DECODE_START(1, bl); // extended, osd-only data
+ ::decode(osd_addrs->hb_back_addr, bl);
+ ::decode(osd_info, bl);
+ ::decode(blacklist, bl);
+ ::decode(osd_addrs->cluster_addr, bl);
+ ::decode(cluster_snapshot_epoch, bl);
+ ::decode(cluster_snapshot, bl);
+ ::decode(*osd_uuid, bl);
+ ::decode(osd_xinfo, bl);
+ ::decode(osd_addrs->hb_front_addr, bl);
+ DECODE_FINISH(bl); // osd-only data
+ }
+
+ DECODE_FINISH(bl); // wrapper
+
+ post_decode();
+}
+
+void OSDMap::post_decode()
+{
// index pool names
name_pool.clear();
- for (map<int64_t,string>::iterator i = pool_name.begin(); i != pool_name.end(); ++i)
+ for (map<int64_t,string>::iterator i = pool_name.begin();
+ i != pool_name.end(); ++i) {
name_pool[i->second] = i->first;
+ }
calc_num_osds();
}
-
-
void OSDMap::dump_json(ostream& out) const
{
JSONFormatter jsf(true);
@@ -1462,8 +1910,17 @@ void OSDMap::dump(Formatter *f) const
}
f->close_section();
+ f->open_array_section("primary_temp");
+ for (map<pg_t, int>::const_iterator p = primary_temp->begin();
+ p != primary_temp->end();
+ ++p) {
+ f->dump_stream("pgid") << p->first;
+ f->dump_int("osd", p->second);
+ }
+ f->close_section(); // primary_temp
+
f->open_array_section("blacklist");
- for (hash_map<entity_addr_t,utime_t>::const_iterator p = blacklist.begin();
+ for (ceph::unordered_map<entity_addr_t,utime_t>::const_iterator p = blacklist.begin();
p != blacklist.end();
++p) {
stringstream ss;
@@ -1587,7 +2044,12 @@ void OSDMap::print(ostream& out) const
++p)
out << "pg_temp " << p->first << " " << p->second << "\n";
- for (hash_map<entity_addr_t,utime_t>::const_iterator p = blacklist.begin();
+ for (map<pg_t,int>::const_iterator p = primary_temp->begin();
+ p != primary_temp->end();
+ ++p)
+ out << "primary_temp " << p->first << " " << p->second << "\n";
+
+ for (ceph::unordered_map<entity_addr_t,utime_t>::const_iterator p = blacklist.begin();
p != blacklist.end();
++p)
out << "blacklist " << p->first << " expires " << p->second << "\n";
@@ -1741,7 +2203,7 @@ void OSDMap::print_summary(Formatter *f, ostream& out) const
f->dump_int("epoch", get_epoch());
f->dump_int("num_osds", get_num_osds());
f->dump_int("num_up_osds", get_num_up_osds());
- f->dump_stream("num_in_osds") << get_num_in_osds();
+ f->dump_int("num_in_osds", get_num_in_osds());
f->dump_string("full", test_flag(CEPH_OSDMAP_FULL) ? "true" : "false");
f->dump_string("nearfull", test_flag(CEPH_OSDMAP_NEARFULL) ?
"true" : "false");
@@ -1777,7 +2239,7 @@ bool OSDMap::crush_ruleset_in_use(int ruleset) const
return false;
}
-void OSDMap::build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
+int OSDMap::build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
int nosd, int pg_bits, int pgp_bits)
{
ldout(cct, 10) << "build_simple on " << num_osd
@@ -1787,66 +2249,112 @@ void OSDMap::build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
set_fsid(fsid);
created = modified = ceph_clock_now(cct);
- set_max_osd(nosd);
+ if (nosd >= 0) {
+ set_max_osd(nosd);
+ } else {
+ // count osds
+ int maxosd = 0, numosd = 0;
+ const md_config_t *conf = cct->_conf;
+ vector<string> sections;
+ conf->get_all_sections(sections);
+ for (vector<string>::iterator i = sections.begin(); i != sections.end(); ++i) {
+ if (i->find("osd.") != 0)
+ continue;
+
+ const char *begin = i->c_str() + 4;
+ char *end = (char*)begin;
+ int o = strtol(begin, &end, 10);
+ if (*end != '\0')
+ continue;
+
+ if (o > cct->_conf->mon_max_osd) {
+ lderr(cct) << "[osd." << o << "] in config has id > mon_max_osd " << cct->_conf->mon_max_osd << dendl;
+ return -ERANGE;
+ }
+ numosd++;
+ if (o > maxosd)
+ maxosd = o;
+ }
+
+ set_max_osd(maxosd + 1);
+ }
// pgp_num <= pg_num
if (pgp_bits > pg_bits)
pgp_bits = pg_bits;
- // crush map
- map<int, const char*> rulesets;
- rulesets[CEPH_DATA_RULE] = "data";
- rulesets[CEPH_METADATA_RULE] = "metadata";
- rulesets[CEPH_RBD_RULE] = "rbd";
+ vector<string> pool_names;
+ pool_names.push_back("data");
+ pool_names.push_back("metadata");
+ pool_names.push_back("rbd");
- int poolbase = nosd ? nosd : 1;
+ int poolbase = get_max_osd() ? get_max_osd() : 1;
- for (map<int,const char*>::iterator p = rulesets.begin(); p != rulesets.end(); ++p) {
+ for (vector<string>::iterator p = pool_names.begin();
+ p != pool_names.end(); ++p) {
int64_t pool = ++pool_max;
- pools[pool].type = pg_pool_t::TYPE_REP;
+ pools[pool].type = pg_pool_t::TYPE_REPLICATED;
pools[pool].flags = cct->_conf->osd_pool_default_flags;
if (cct->_conf->osd_pool_default_flag_hashpspool)
pools[pool].flags |= pg_pool_t::FLAG_HASHPSPOOL;
pools[pool].size = cct->_conf->osd_pool_default_size;
pools[pool].min_size = cct->_conf->get_osd_pool_default_min_size();
- pools[pool].crush_ruleset = p->first;
+ pools[pool].crush_ruleset =
+ CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(cct);
pools[pool].object_hash = CEPH_STR_HASH_RJENKINS;
pools[pool].set_pg_num(poolbase << pg_bits);
pools[pool].set_pgp_num(poolbase << pgp_bits);
pools[pool].last_change = epoch;
- if (p->first == CEPH_DATA_RULE)
+ if (*p == "data")
pools[pool].crash_replay_interval = cct->_conf->osd_default_data_pool_replay_window;
- pool_name[pool] = p->second;
- name_pool[p->second] = pool;
+ pool_name[pool] = *p;
+ name_pool[*p] = pool;
}
- build_simple_crush_map(cct, *crush, rulesets, nosd);
+ stringstream ss;
+ int r;
+ if (nosd >= 0)
+ r = build_simple_crush_map(cct, *crush, nosd, &ss);
+ else
+ r = build_simple_crush_map_from_conf(cct, *crush, &ss);
- for (int i=0; i<nosd; i++) {
+ if (r < 0)
+ lderr(cct) << ss.str() << dendl;
+
+ for (int i=0; i<get_max_osd(); i++) {
set_state(i, 0);
set_weight(i, CEPH_OSD_OUT);
}
-}
+ return r;
+}
-void OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
- map<int, const char*>& rulesets, int nosd)
+int OSDMap::_build_crush_types(CrushWrapper& crush)
{
- const md_config_t *conf = cct->_conf;
-
- crush.create();
-
crush.set_type_name(0, "osd");
crush.set_type_name(1, "host");
- crush.set_type_name(2, "rack");
- crush.set_type_name(3, "row");
- crush.set_type_name(4, "room");
- crush.set_type_name(5, "datacenter");
- crush.set_type_name(6, "root");
+ crush.set_type_name(2, "chassis");
+ crush.set_type_name(3, "rack");
+ crush.set_type_name(4, "row");
+ crush.set_type_name(5, "pdu");
+ crush.set_type_name(6, "pod");
+ crush.set_type_name(7, "room");
+ crush.set_type_name(8, "datacenter");
+ crush.set_type_name(9, "region");
+ crush.set_type_name(10, "root");
+ return 10;
+}
+
+int OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
+ int nosd, ostream *ss)
+{
+ crush.create();
// root
+ int root_type = _build_crush_types(crush);
int rootid;
- int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, 6 /* pool */, 0, NULL, NULL, &rootid);
+ int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT,
+ root_type, 0, NULL, NULL, &rootid);
assert(r == 0);
crush.set_item_name(rootid, "default");
@@ -1861,124 +2369,28 @@ void OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
crush.insert_item(cct, o, 1.0, name, loc);
}
- // rules
- int minrep = conf->osd_min_rep;
- int maxrep = conf->osd_max_rep;
- assert(maxrep >= minrep);
- for (map<int,const char*>::iterator p = rulesets.begin(); p != rulesets.end(); ++p) {
- int ruleset = p->first;
- crush_rule *rule = crush_make_rule(3, ruleset, pg_pool_t::TYPE_REP, minrep, maxrep);
- assert(rule);
- crush_rule_set_step(rule, 0, CRUSH_RULE_TAKE, rootid, 0);
- crush_rule_set_step(rule, 1,
- cct->_conf->osd_crush_chooseleaf_type ? CRUSH_RULE_CHOOSE_LEAF_FIRSTN : CRUSH_RULE_CHOOSE_FIRSTN,
- CRUSH_CHOOSE_N,
- cct->_conf->osd_crush_chooseleaf_type);
- crush_rule_set_step(rule, 2, CRUSH_RULE_EMIT, 0, 0);
- int rno = crush_add_rule(crush.crush, rule, -1);
- crush.set_rule_name(rno, p->second);
- }
+ build_simple_crush_rulesets(cct, crush, "default", ss);
crush.finalize();
-}
-
-int OSDMap::build_simple_from_conf(CephContext *cct, epoch_t e, uuid_d &fsid,
- int pg_bits, int pgp_bits)
-{
- ldout(cct, 10) << "build_simple_from_conf with "
- << pg_bits << " pg bits per osd, "
- << dendl;
- epoch = e;
- set_fsid(fsid);
- created = modified = ceph_clock_now(cct);
-
- const md_config_t *conf = cct->_conf;
-
- // count osds
- int maxosd = 0, numosd = 0;
-
- vector<string> sections;
- conf->get_all_sections(sections);
- for (vector<string>::iterator i = sections.begin(); i != sections.end(); ++i) {
- if (i->find("osd.") != 0)
- continue;
-
- const char *begin = i->c_str() + 4;
- char *end = (char*)begin;
- int o = strtol(begin, &end, 10);
- if (*end != '\0')
- continue;
-
- if (o > cct->_conf->mon_max_osd) {
- lderr(cct) << "[osd." << o << "] in config has id > mon_max_osd " << cct->_conf->mon_max_osd << dendl;
- return -ERANGE;
- }
- numosd++;
- if (o > maxosd)
- maxosd = o;
- }
-
- set_max_osd(maxosd + 1);
-
- // pgp_num <= pg_num
- if (pgp_bits > pg_bits)
- pgp_bits = pg_bits;
-
- // crush map
- map<int, const char*> rulesets;
- rulesets[CEPH_DATA_RULE] = "data";
- rulesets[CEPH_METADATA_RULE] = "metadata";
- rulesets[CEPH_RBD_RULE] = "rbd";
-
- for (map<int,const char*>::iterator p = rulesets.begin(); p != rulesets.end(); ++p) {
- int64_t pool = ++pool_max;
- pools[pool].type = pg_pool_t::TYPE_REP;
- pools[pool].flags = cct->_conf->osd_pool_default_flags;
- if (cct->_conf->osd_pool_default_flag_hashpspool)
- pools[pool].flags |= pg_pool_t::FLAG_HASHPSPOOL;
- pools[pool].size = cct->_conf->osd_pool_default_size;
- pools[pool].min_size = cct->_conf->get_osd_pool_default_min_size();
- pools[pool].crush_ruleset = p->first;
- pools[pool].object_hash = CEPH_STR_HASH_RJENKINS;
- pools[pool].set_pg_num((numosd + 1) << pg_bits);
- pools[pool].set_pgp_num((numosd + 1) << pgp_bits);
- pools[pool].last_change = epoch;
- if (p->first == CEPH_DATA_RULE)
- pools[pool].crash_replay_interval = cct->_conf->osd_default_data_pool_replay_window;
- pool_name[pool] = p->second;
- name_pool[p->second] = pool;
- }
-
- build_simple_crush_map_from_conf(cct, *crush, rulesets);
-
- for (int i=0; i<=maxosd; i++) {
- set_state(i, 0);
- set_weight(i, CEPH_OSD_OUT);
- }
return 0;
}
-void OSDMap::build_simple_crush_map_from_conf(CephContext *cct, CrushWrapper& crush,
- map<int, const char*>& rulesets)
+int OSDMap::build_simple_crush_map_from_conf(CephContext *cct,
+ CrushWrapper& crush,
+ ostream *ss)
{
const md_config_t *conf = cct->_conf;
crush.create();
- crush.set_type_name(0, "osd");
- crush.set_type_name(1, "host");
- crush.set_type_name(2, "rack");
- crush.set_type_name(3, "row");
- crush.set_type_name(4, "room");
- crush.set_type_name(5, "datacenter");
- crush.set_type_name(6, "root");
-
set<string> hosts, racks;
// root
+ int root_type = _build_crush_types(crush);
int rootid;
- int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, 6 /* pool */, 0, NULL, NULL, &rootid);
+ int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT,
+ root_type, 0, NULL, NULL, &rootid);
assert(r == 0);
crush.set_item_name(rootid, "default");
@@ -2029,31 +2441,28 @@ void OSDMap::build_simple_crush_map_from_conf(CephContext *cct, CrushWrapper& cr
crush.insert_item(cct, o, 1.0, *i, loc);
}
- // rules
- int minrep = conf->osd_min_rep;
- int maxrep = conf->osd_max_rep;
- for (map<int,const char*>::iterator p = rulesets.begin(); p != rulesets.end(); ++p) {
- int ruleset = p->first;
- crush_rule *rule = crush_make_rule(3, ruleset, pg_pool_t::TYPE_REP, minrep, maxrep);
- assert(rule);
- crush_rule_set_step(rule, 0, CRUSH_RULE_TAKE, rootid, 0);
-
- if (racks.size() > 3) {
- // spread replicas across hosts
- crush_rule_set_step(rule, 1, CRUSH_RULE_CHOOSE_LEAF_FIRSTN, CRUSH_CHOOSE_N, 2);
- } else if (hosts.size() > 1) {
- // spread replicas across hosts
- crush_rule_set_step(rule, 1, CRUSH_RULE_CHOOSE_LEAF_FIRSTN, CRUSH_CHOOSE_N, 1);
- } else {
- // just spread across osds
- crush_rule_set_step(rule, 1, CRUSH_RULE_CHOOSE_FIRSTN, CRUSH_CHOOSE_N, 0);
- }
- crush_rule_set_step(rule, 2, CRUSH_RULE_EMIT, 0, 0);
- int rno = crush_add_rule(crush.crush, rule, -1);
- crush.set_rule_name(rno, p->second);
- }
+ build_simple_crush_rulesets(cct, crush, "default", ss);
crush.finalize();
+
+ return 0;
}
+int OSDMap::build_simple_crush_rulesets(CephContext *cct,
+ CrushWrapper& crush,
+ const string& root,
+ ostream *ss)
+{
+ string failure_domain =
+ crush.get_type_name(cct->_conf->osd_crush_chooseleaf_type);
+
+ int r;
+ r = crush.add_simple_ruleset("replicated_ruleset", root, failure_domain,
+ "firstn", pg_pool_t::TYPE_REPLICATED, ss);
+ if (r < 0)
+ return r;
+ // do not add an erasure rule by default or else we will implicitly
+ // require the crush_v2 feature of clients
+ return 0;
+}
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index bd8f09b..a3acc02 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -38,11 +38,10 @@
#include <list>
#include <set>
#include <map>
-#include <tr1/memory>
+#include "include/memory.h"
using namespace std;
-#include <ext/hash_set>
-using __gnu_cxx::hash_set;
+#include "include/unordered_set.h"
/*
* we track up to two intervals during which the osd was alive and
@@ -86,13 +85,14 @@ WRITE_CLASS_ENCODER(osd_info_t)
ostream& operator<<(ostream& out, const osd_info_t& info);
-
struct osd_xinfo_t {
utime_t down_stamp; ///< timestamp when we were last marked down
float laggy_probability; ///< encoded as __u32: 0 = definitely not laggy, 0xffffffff definitely laggy
__u32 laggy_interval; ///< average interval between being marked laggy and recovering
+ uint64_t features; ///< features supported by this osd we should know about
- osd_xinfo_t() : laggy_probability(0), laggy_interval(0) {}
+ osd_xinfo_t() : laggy_probability(0), laggy_interval(0),
+ features(0) {}
void dump(Formatter *f) const;
void encode(bufferlist& bl) const;
@@ -111,6 +111,9 @@ class OSDMap {
public:
class Incremental {
public:
+ /// feature bits we were encoded with. the subsequent OSDMap
+ /// encoding should match.
+ uint64_t encode_features;
uuid_d fsid;
epoch_t epoch; // new epoch; we are a diff from epoch-1 to epoch
utime_t modified;
@@ -131,6 +134,7 @@ public:
map<int32_t,uint8_t> new_state; // XORed onto previous state.
map<int32_t,uint32_t> new_weight;
map<pg_t,vector<int32_t> > new_pg_temp; // [] to remove
+ map<pg_t, int> new_primary_temp; // [-1] to remove
map<int32_t,epoch_t> new_up_thru;
map<int32_t,pair<epoch_t,epoch_t> > new_last_clean_interval;
map<int32_t,epoch_t> new_lost;
@@ -149,12 +153,15 @@ public:
int identify_osd(uuid_d u) const;
void encode_client_old(bufferlist& bl) const;
+ void encode_classic(bufferlist& bl, uint64_t features) const;
void encode(bufferlist& bl, uint64_t features=CEPH_FEATURES_ALL) const;
- void decode(bufferlist::iterator &p);
+ void decode_classic(bufferlist::iterator &p);
+ void decode(bufferlist::iterator &bl);
void dump(Formatter *f) const;
static void generate_test_instances(list<Incremental*>& o);
Incremental(epoch_t e=0) :
+ encode_features(0),
epoch(e), new_pool_max(-1), new_flags(-1), new_max_osd(-1) {
memset(&fsid, 0, sizeof(fsid));
}
@@ -171,6 +178,9 @@ public:
new_pools[pool] = *orig;
return &new_pools[pool];
}
+
+ /// propage update pools' snap metadata to any of their tiers
+ int propagate_snaps_to_tiers(CephContext *cct, const OSDMap &base);
};
private:
@@ -186,33 +196,34 @@ private:
vector<uint8_t> osd_state;
struct addrs_s {
- vector<std::tr1::shared_ptr<entity_addr_t> > client_addr;
- vector<std::tr1::shared_ptr<entity_addr_t> > cluster_addr;
- vector<std::tr1::shared_ptr<entity_addr_t> > hb_back_addr;
- vector<std::tr1::shared_ptr<entity_addr_t> > hb_front_addr;
+ vector<ceph::shared_ptr<entity_addr_t> > client_addr;
+ vector<ceph::shared_ptr<entity_addr_t> > cluster_addr;
+ vector<ceph::shared_ptr<entity_addr_t> > hb_back_addr;
+ vector<ceph::shared_ptr<entity_addr_t> > hb_front_addr;
entity_addr_t blank;
};
- std::tr1::shared_ptr<addrs_s> osd_addrs;
+ ceph::shared_ptr<addrs_s> osd_addrs;
vector<__u32> osd_weight; // 16.16 fixed point, 0x10000 = "in", 0 = "out"
vector<osd_info_t> osd_info;
- std::tr1::shared_ptr< map<pg_t,vector<int> > > pg_temp; // temp pg mapping (e.g. while we rebuild)
+ ceph::shared_ptr< map<pg_t,vector<int> > > pg_temp; // temp pg mapping (e.g. while we rebuild)
+ ceph::shared_ptr< map<pg_t,int > > primary_temp; // temp primary mapping (e.g. while we rebuild)
map<int64_t,pg_pool_t> pools;
map<int64_t,string> pool_name;
map<string,int64_t> name_pool;
- std::tr1::shared_ptr< vector<uuid_d> > osd_uuid;
+ ceph::shared_ptr< vector<uuid_d> > osd_uuid;
vector<osd_xinfo_t> osd_xinfo;
- hash_map<entity_addr_t,utime_t> blacklist;
+ ceph::unordered_map<entity_addr_t,utime_t> blacklist;
epoch_t cluster_snapshot_epoch;
string cluster_snapshot;
bool new_blacklist_entries;
public:
- std::tr1::shared_ptr<CrushWrapper> crush; // hierarchical map
+ ceph::shared_ptr<CrushWrapper> crush; // hierarchical map
friend class OSDMonitor;
friend class PGMonitor;
@@ -225,6 +236,7 @@ private:
num_osd(0), max_osd(0),
osd_addrs(new addrs_s),
pg_temp(new map<pg_t,vector<int> >),
+ primary_temp(new map<pg_t,int>),
osd_uuid(new vector<uuid_d>),
cluster_snapshot_epoch(0),
new_blacklist_entries(false),
@@ -232,6 +244,27 @@ private:
memset(&fsid, 0, sizeof(fsid));
}
+ // no copying
+ /* oh, how i long for c++11...
+private:
+ OSDMap(const OSDMap& other) = default;
+ const OSDMap& operator=(const OSDMap& other) = default;
+public:
+ */
+
+ void deepish_copy_from(const OSDMap& o) {
+ *this = o;
+ primary_temp.reset(new map<pg_t,int>(*o.primary_temp));
+ pg_temp.reset(new map<pg_t,vector<int> >(*o.pg_temp));
+ osd_uuid.reset(new vector<uuid_d>(*o.osd_uuid));
+
+ // NOTE: this still references shared entity_addr_t's.
+ osd_addrs.reset(new addrs_s(*o.osd_addrs));
+
+ // NOTE: we do not copy crush. note that apply_incremental will
+ // allocate a new CrushWrapper, though.
+ }
+
// map info
const uuid_d& get_fsid() const { return fsid; }
void set_fsid(uuid_d& f) { fsid = f; }
@@ -259,14 +292,15 @@ private:
int get_max_osd() const { return max_osd; }
void set_max_osd(int m);
- int get_num_osds() const {
+ unsigned get_num_osds() const {
return num_osd;
}
int calc_num_osds();
void get_all_osds(set<int32_t>& ls) const;
- int get_num_up_osds() const;
- int get_num_in_osds() const;
+ void get_up_osds(set<int32_t>& ls) const;
+ unsigned get_num_up_osds() const;
+ unsigned get_num_in_osds() const;
int get_flags() const { return flags; }
int test_flag(int f) const { return flags & f; }
@@ -445,18 +479,31 @@ private:
*/
uint64_t get_features(uint64_t *mask) const;
+ /**
+ * get intersection of features supported by up osds
+ */
+ uint64_t get_up_osd_features() const;
+
int apply_incremental(const Incremental &inc);
/// try to re-use/reference addrs in oldmap from newmap
static void dedup(const OSDMap *oldmap, OSDMap *newmap);
+ static void remove_redundant_temporaries(CephContext *cct, const OSDMap& osdmap,
+ Incremental *pending_inc);
+ static void remove_down_temps(CephContext *cct, const OSDMap& osdmap,
+ Incremental *pending_inc);
+
// serialize, unserialize
private:
void encode_client_old(bufferlist& bl) const;
+ void encode_classic(bufferlist& bl, uint64_t features) const;
+ void decode_classic(bufferlist::iterator& p);
+ void post_decode();
public:
void encode(bufferlist& bl, uint64_t features=CEPH_FEATURES_ALL) const;
void decode(bufferlist& bl);
- void decode(bufferlist::iterator& p);
+ void decode(bufferlist::iterator& bl);
/**** mapping facilities ****/
@@ -488,19 +535,71 @@ public:
private:
/// pg -> (raw osd list)
- int _pg_to_osds(const pg_pool_t& pool, pg_t pg, vector<int>& osds) const;
- void _remove_nonexistent_osds(vector<int>& osds) const;
+ int _pg_to_osds(const pg_pool_t& pool, pg_t pg,
+ vector<int> *osds, int *primary) const;
+ void _remove_nonexistent_osds(const pg_pool_t& pool, vector<int>& osds) const;
/// pg -> (up osd list)
- void _raw_to_up_osds(pg_t pg, vector<int>& raw, vector<int>& up) const;
+ void _raw_to_up_osds(pg_t pg, const vector<int>& raw,
+ vector<int> *up, int *primary) const;
- bool _raw_to_temp_osds(const pg_pool_t& pool, pg_t pg, vector<int>& raw, vector<int>& temp) const;
+ /**
+ * Get the pg and primary temp, if they are specified.
+ * @param temp_pg [out] Will be empty or contain the temp PG mapping on return
+ * @param temp_primary [out] Will be the value in primary_temp, or a value derived
+ * from the pg_temp (if specified), or -1 if you should use the calculated (up_)primary.
+ */
+ void _get_temp_osds(const pg_pool_t& pool, pg_t pg,
+ vector<int> *temp_pg, int *temp_primary) const;
+
+ /**
+ * map to up and acting. Fills in whatever fields are non-NULL.
+ */
+ void _pg_to_up_acting_osds(pg_t pg, vector<int> *up, int *up_primary,
+ vector<int> *acting, int *acting_primary) const;
public:
- int pg_to_osds(pg_t pg, vector<int>& raw) const;
- int pg_to_acting_osds(pg_t pg, vector<int>& acting) const;
- void pg_to_raw_up(pg_t pg, vector<int>& up) const;
- void pg_to_up_acting_osds(pg_t pg, vector<int>& up, vector<int>& acting) const;
+ /***
+ * This is suitable only for looking at raw CRUSH outputs. It skips
+ * applying the temp and up checks and should not be used
+ * by anybody for data mapping purposes.
+ * raw and primary must be non-NULL
+ */
+ int pg_to_osds(pg_t pg, vector<int> *raw, int *primary) const;
+ /// map a pg to its acting set. @return acting set size
+ int pg_to_acting_osds(pg_t pg, vector<int> *acting,
+ int *acting_primary) const {
+ _pg_to_up_acting_osds(pg, NULL, NULL, acting, acting_primary);
+ return acting->size();
+ }
+ int pg_to_acting_osds(pg_t pg, vector<int>& acting) const {
+ int primary;
+ int r = pg_to_acting_osds(pg, &acting, &primary);
+ assert(acting.empty() || primary == acting.front());
+ return r;
+ }
+ /**
+ * This does not apply temp overrides and should not be used
+ * by anybody for data mapping purposes. Specify both pointers.
+ */
+ void pg_to_raw_up(pg_t pg, vector<int> *up, int *primary) const;
+ /**
+ * map a pg to its acting set as well as its up set. You must use
+ * the acting set for data mapping purposes, but some users will
+ * also find the up set useful for things like deciding what to
+ * set as pg_temp.
+ * Each of these pointers must be non-NULL.
+ */
+ void pg_to_up_acting_osds(pg_t pg, vector<int> *up, int *up_primary,
+ vector<int> *acting, int *acting_primary) const {
+ _pg_to_up_acting_osds(pg, up, up_primary, acting, acting_primary);
+ }
+ void pg_to_up_acting_osds(pg_t pg, vector<int>& up, vector<int>& acting) const {
+ int up_primary, acting_primary;
+ pg_to_up_acting_osds(pg, &up, &up_primary, &acting, &acting_primary);
+ assert(up.empty() || up_primary == up.front());
+ assert(acting.empty() || acting_primary == acting.front());
+ }
int64_t lookup_pg_pool_name(const string& name) {
if (name_pool.count(name))
@@ -549,15 +648,6 @@ public:
return pools.find(pg.pool())->second.raw_pg_to_pg(pg);
}
- // pg -> primary osd
- int get_pg_primary(pg_t pg) const {
- vector<int> group;
- int nrep = pg_to_osds(pg, group);
- if (nrep)
- return group[0];
- return -1; // we fail!
- }
-
// pg -> acting primary osd
int get_pg_acting_primary(pg_t pg) const {
vector<int> group;
@@ -596,14 +686,29 @@ public:
/*
* handy helpers to build simple maps...
*/
- void build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
- int num_osd, int pg_bits, int pgp_bits);
- int build_simple_from_conf(CephContext *cct, epoch_t e, uuid_d &fsid,
- int pg_bits, int pgp_bits);
- static void build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
- map<int, const char*>& poolsets, int num_osd);
- static void build_simple_crush_map_from_conf(CephContext *cct, CrushWrapper& crush,
- map<int, const char*>& rulesets);
+ /**
+ * Build an OSD map suitable for basic usage. If **num_osd** is >= 0
+ * it will be initialized with the specified number of OSDs in a
+ * single host. If **num_osd** is < 0 the layout of the OSD map will
+ * be built by reading the content of the configuration file.
+ *
+ * @param cct [in] in core ceph context
+ * @param e [in] initial epoch
+ * @param fsid [in] id of the cluster
+ * @param num_osd [in] number of OSDs if >= 0 or read from conf if < 0
+ * @return **0** on success, negative errno on error.
+ */
+ int build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
+ int num_osd, int pg_bits, int pgp_bits);
+ static int _build_crush_types(CrushWrapper& crush);
+ static int build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
+ int num_osd, ostream *ss);
+ static int build_simple_crush_map_from_conf(CephContext *cct,
+ CrushWrapper& crush,
+ ostream *ss);
+ static int build_simple_crush_rulesets(CephContext *cct, CrushWrapper& crush,
+ const string& root,
+ ostream *ss);
bool crush_ruleset_in_use(int ruleset) const;
@@ -625,7 +730,7 @@ public:
WRITE_CLASS_ENCODER_FEATURES(OSDMap)
WRITE_CLASS_ENCODER_FEATURES(OSDMap::Incremental)
-typedef std::tr1::shared_ptr<const OSDMap> OSDMapRef;
+typedef ceph::shared_ptr<const OSDMap> OSDMapRef;
inline ostream& operator<<(ostream& out, const OSDMap& m) {
m.print_oneline_summary(out);
diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc
index 0730ce8..33e7fbd 100644
--- a/src/osd/OpRequest.cc
+++ b/src/osd/OpRequest.cc
@@ -64,6 +64,7 @@ bool OpRequest::check_rmw(int flag) {
}
bool OpRequest::may_read() { return need_read_cap() || need_class_read_cap(); }
bool OpRequest::may_write() { return need_write_cap() || need_class_write_cap(); }
+bool OpRequest::may_cache() { return check_rmw(CEPH_OSD_RMW_FLAG_CACHE); }
bool OpRequest::includes_pg_op() { return check_rmw(CEPH_OSD_RMW_FLAG_PGOP); }
bool OpRequest::need_read_cap() {
return check_rmw(CEPH_OSD_RMW_FLAG_READ);
@@ -82,3 +83,4 @@ void OpRequest::set_write() { rmw_flags |= CEPH_OSD_RMW_FLAG_WRITE; }
void OpRequest::set_class_read() { rmw_flags |= CEPH_OSD_RMW_FLAG_CLASS_READ; }
void OpRequest::set_class_write() { rmw_flags |= CEPH_OSD_RMW_FLAG_CLASS_WRITE; }
void OpRequest::set_pg_op() { rmw_flags |= CEPH_OSD_RMW_FLAG_PGOP; }
+void OpRequest::set_cache() { rmw_flags |= CEPH_OSD_RMW_FLAG_CACHE; }
diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h
index 6d367db..7e07340 100644
--- a/src/osd/OpRequest.h
+++ b/src/osd/OpRequest.h
@@ -21,7 +21,7 @@
#include "common/Mutex.h"
#include "include/xlist.h"
#include "msg/Message.h"
-#include <tr1/memory>
+#include "include/memory.h"
#include "common/TrackedOp.h"
/**
@@ -59,6 +59,7 @@ struct OpRequest : public TrackedOp {
bool check_rmw(int flag);
bool may_read();
bool may_write();
+ bool may_cache();
bool includes_pg_op();
bool need_read_cap();
bool need_write_cap();
@@ -66,6 +67,7 @@ struct OpRequest : public TrackedOp {
bool need_class_write_cap();
void set_read();
void set_write();
+ void set_cache();
void set_class_read();
void set_class_write();
void set_pg_op();
@@ -76,6 +78,7 @@ private:
osd_reqid_t reqid;
uint8_t hit_flag_points;
uint8_t latest_flag_point;
+ utime_t dequeued_time;
static const uint8_t flag_queued_for_pg=1 << 0;
static const uint8_t flag_reached_pg = 1 << 1;
static const uint8_t flag_delayed = 1 << 2;
@@ -149,13 +152,20 @@ public:
latest_flag_point = flag_commit_sent;
}
+ utime_t get_dequeued_time() const {
+ return dequeued_time;
+ }
+ void set_dequeued_time(utime_t deq_time) {
+ dequeued_time = deq_time;
+ }
+
osd_reqid_t get_reqid() const {
return reqid;
}
void init_from_message();
- typedef std::tr1::shared_ptr<OpRequest> Ref;
+ typedef ceph::shared_ptr<OpRequest> Ref;
};
typedef OpRequest::Ref OpRequestRef;
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 8207a67..4c1d909 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -162,18 +162,15 @@ PG::PG(OSDService *o, OSDMapRef curmap,
coll(p), pg_log(cct), log_oid(loid), biginfo_oid(ioid),
recovery_item(this), scrub_item(this), scrub_finalize_item(this), snap_trim_item(this), stat_queue_item(this),
recovery_ops_active(0),
- waiting_on_backfill(0),
role(0),
state(0),
send_notify(false),
need_up_thru(false),
- need_flush(false),
last_peering_reset(0),
heartbeat_peer_lock("PG::heartbeat_peer_lock"),
- backfill_target(-1),
backfill_reserved(0),
backfill_reserving(0),
- flushed(false),
+ flushes_in_progress(0),
pg_stats_publish_lock("PG::pg_stats_publish_lock"),
pg_stats_publish_valid(false),
osr(osd->osr_registry.lookup_or_create(p, (stringify(p)))),
@@ -304,10 +301,16 @@ bool PG::proc_replica_info(int from, const pg_info_t &oinfo)
}
void PG::remove_snap_mapped_object(
- ObjectStore::Transaction& t, const hobject_t& soid)
+ ObjectStore::Transaction &t, const hobject_t &soid)
{
t.remove(coll, soid);
- OSDriver::OSTransaction _t(osdriver.get_transaction(&t));
+ clear_object_snap_mapping(&t, soid);
+}
+
+void PG::clear_object_snap_mapping(
+ ObjectStore::Transaction *t, const hobject_t &soid)
+{
+ OSDriver::OSTransaction _t(osdriver.get_transaction(t));
if (soid.snap < CEPH_MAXSNAP) {
int r = snap_mapper.remove_oid(
soid,
@@ -319,24 +322,39 @@ void PG::remove_snap_mapped_object(
}
}
-void PG::merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from)
+void PG::update_object_snap_mapping(
+ ObjectStore::Transaction *t, const hobject_t &soid, const set<snapid_t> &snaps)
{
- list<hobject_t> to_remove;
- pg_log.merge_log(t, oinfo, olog, from, info, to_remove, dirty_info, dirty_big_info);
- for(list<hobject_t>::iterator i = to_remove.begin();
- i != to_remove.end();
- ++i)
- remove_snap_mapped_object(t, *i);
+ OSDriver::OSTransaction _t(osdriver.get_transaction(t));
+ assert(soid.snap < CEPH_MAXSNAP);
+ int r = snap_mapper.remove_oid(
+ soid,
+ &_t);
+ if (!(r == 0 || r == -ENOENT)) {
+ derr << __func__ << ": remove_oid returned " << cpp_strerror(r) << dendl;
+ assert(0);
+ }
+ snap_mapper.add_oid(
+ soid,
+ snaps,
+ &_t);
+}
+
+void PG::merge_log(
+ ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from)
+{
+ PGLogEntryHandler rollbacker;
+ pg_log.merge_log(
+ t, oinfo, olog, from, info, &rollbacker, dirty_info, dirty_big_info);
+ rollbacker.apply(this, &t);
}
void PG::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead)
{
- list<hobject_t> to_remove;
- pg_log.rewind_divergent_log(t, newhead, info, to_remove, dirty_info, dirty_big_info);
- for(list<hobject_t>::iterator i = to_remove.begin();
- i != to_remove.end();
- ++i)
- remove_snap_mapped_object(t, *i);
+ PGLogEntryHandler rollbacker;
+ pg_log.rewind_divergent_log(
+ t, newhead, info, &rollbacker, dirty_info, dirty_big_info);
+ rollbacker.apply(this, &t);
}
/*
@@ -442,6 +460,13 @@ void PG::discover_all_missing(map< int, map<pg_t,pg_query_t> > &query_map)
continue;
}
+ map<int, pg_info_t>::const_iterator iter = peer_info.find(peer);
+ if (iter != peer_info.end() &&
+ (iter->second.is_empty() || iter->second.dne())) {
+ // ignore empty peers
+ continue;
+ }
+
// If we've requested any of this stuff, the pg_missing_t information
// should be on its way.
// TODO: coalsce requested_* into a single data structure
@@ -484,8 +509,9 @@ bool PG::needs_recovery() const
ret = true;
}
- vector<int>::const_iterator end = acting.end();
- vector<int>::const_iterator a = acting.begin();
+ assert(actingbackfill.size() > 0);
+ vector<int>::const_iterator end = actingbackfill.end();
+ vector<int>::const_iterator a = actingbackfill.begin();
assert(a != end);
++a;
for (; a != end; ++a) {
@@ -513,14 +539,14 @@ bool PG::needs_backfill() const
bool ret = false;
- vector<int>::const_iterator end = acting.end();
- vector<int>::const_iterator a = acting.begin();
- assert(a != end);
- ++a;
+ // We can assume that only possible osds that need backfill
+ // are on the backfill_targets vector nodes.
+ vector<int>::const_iterator end = backfill_targets.end();
+ vector<int>::const_iterator a = backfill_targets.begin();
for (; a != end; ++a) {
int peer = *a;
map<int,pg_info_t>::const_iterator pi = peer_info.find(peer);
- if (pi->second.last_backfill != hobject_t::get_max()) {
+ if (!pi->second.last_backfill.is_max()) {
dout(10) << __func__ << " osd." << peer << " has last_backfill " << pi->second.last_backfill << dendl;
ret = true;
}
@@ -674,6 +700,10 @@ bool PG::all_unfound_are_queried_or_lost(const OSDMapRef osdmap) const
for (; peer != mend; ++peer) {
if (peer_missing.count(*peer))
continue;
+ map<int, pg_info_t>::const_iterator iter = peer_info.find(*peer);
+ if (iter != peer_info.end() &&
+ (iter->second.is_empty() || iter->second.dne()))
+ continue;
const osd_info_t &osd_info(osdmap->get_info(*peer));
if (osd_info.lost_at <= osd_info.up_from) {
// If there is even one OSD in might_have_unfound that isn't lost, we
@@ -785,7 +815,8 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
assert(min_last_update_acceptable != eversion_t::max());
map<int, pg_info_t>::const_iterator best = infos.end();
- // find osd with newest last_update. if there are multiples, prefer
+ // find osd with newest last_update (oldest for ec_pool).
+ // if there are multiples, prefer
// - a longer tail, if it brings another peer into log contiguity
// - the current primary
for (map<int, pg_info_t>::const_iterator p = infos.begin();
@@ -802,11 +833,20 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
continue;
}
// Prefer newer last_update
- if (p->second.last_update < best->second.last_update)
- continue;
- if (p->second.last_update > best->second.last_update) {
- best = p;
- continue;
+ if (pool.info.ec_pool()) {
+ if (p->second.last_update > best->second.last_update)
+ continue;
+ if (p->second.last_update < best->second.last_update) {
+ best = p;
+ continue;
+ }
+ } else {
+ if (p->second.last_update < best->second.last_update)
+ continue;
+ if (p->second.last_update > best->second.last_update) {
+ best = p;
+ continue;
+ }
}
// Prefer longer tail if it brings another peer into contiguity
for (map<int, pg_info_t>::const_iterator q = infos.begin();
@@ -844,7 +884,7 @@ map<int, pg_info_t>::const_iterator PG::find_best_info(const map<int, pg_info_t>
* incomplete, or another osd has a longer tail that allows us to
* bring other up nodes up to date.
*/
-bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want) const
+bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want, vector<int>& backfill) const
{
map<int, pg_info_t> all_info(peer_info.begin(), peer_info.end());
all_info[osd->whoami] = info;
@@ -908,7 +948,6 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want) const
<< " with " << primary->second << dendl;
want.push_back(primary->first);
unsigned usable = 1;
- unsigned backfill = 0;
// select replicas that have log contiguity with primary.
// prefer up, then acting, then any peer_info osds
@@ -918,14 +957,17 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want) const
if (*i == primary->first)
continue;
const pg_info_t &cur_info = all_info.find(*i)->second;
- if (cur_info.is_incomplete() || cur_info.last_update < primary->second.log_tail) {
- if (backfill < 1) {
- dout(10) << " osd." << *i << " (up) accepted (backfill) " << cur_info << dendl;
- want.push_back(*i);
- backfill++;
- } else {
- dout(10) << " osd." << *i << " (up) rejected" << cur_info << dendl;
- }
+ if (cur_info.is_incomplete() ||
+ cur_info.last_update < MIN(
+ primary->second.log_tail,
+ newest_update_osd->second.log_tail)) {
+ /* We include newest_update_osd->second.log_tail because in GetLog,
+ * we will request logs back to the min last_update over our
+ * acting_backfill set, which will result in our log being extended
+ * as far backwards as necessary to pick up any peers which can
+ * be log recovered by newest_update_osd's log */
+ dout(10) << " osd." << *i << " (up) backfill " << cur_info << dendl;
+ backfill.push_back(*i);
} else {
want.push_back(*i);
usable++;
@@ -933,6 +975,7 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want) const
}
}
+ // This no longer has backfill OSDs, but they are covered above.
for (vector<int>::const_iterator i = acting.begin();
i != acting.end();
++i) {
@@ -992,33 +1035,93 @@ bool PG::calc_acting(int& newest_update_osd_id, vector<int>& want) const
*/
bool PG::choose_acting(int& newest_update_osd)
{
- vector<int> want;
+ vector<int> want, backfill;
- if (!calc_acting(newest_update_osd, want)) {
+ if (!calc_acting(newest_update_osd, want, backfill)) {
dout(10) << "choose_acting failed" << dendl;
assert(want_acting.empty());
return false;
}
- if (want.size() < pool.info.min_size) {
+ // Determine if compatibility needed
+ bool compat_mode = !cct->_conf->osd_debug_override_acting_compat;
+
+ if (compat_mode) {
+ bool all_support = true;
+ OSDMapRef osdmap = get_osdmap();
+ vector<int> allpeers;
+
+ allpeers = want;
+ allpeers.insert(allpeers.end(), backfill.begin(), backfill.end());
+ for (vector<int>::iterator it = allpeers.begin();
+ it != allpeers.end(); ++it) {
+ int peer = *it;
+
+ const osd_xinfo_t& xi = osdmap->get_xinfo(peer);
+ if (!(xi.features & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+ all_support = false;
+ break;
+ }
+ }
+ if (all_support)
+ compat_mode = false;
+ }
+
+ if (compat_mode && !backfill.empty()) {
+ backfill.resize(1);
+ }
+
+ // This might cause a problem if min_size is large
+ // and we need to backfill more than 1 osd. Older
+ // code would only include 1 backfill osd and now we
+ // have the resize above.
+ if (want.size() + backfill.size() < pool.info.min_size) {
want_acting.clear();
return false;
}
+ if (compat_mode) {
+ want.insert(want.end(), backfill.begin(), backfill.end());
+ }
+
if (want != acting) {
dout(10) << "choose_acting want " << want << " != acting " << acting
<< ", requesting pg_temp change" << dendl;
want_acting = want;
+
if (want == up) {
+ // There can't be any pending backfill if
+ // want is the same as crush map up OSDs.
+ assert(compat_mode || backfill.empty());
vector<int> empty;
osd->queue_want_pg_temp(info.pgid, empty);
} else
osd->queue_want_pg_temp(info.pgid, want);
return false;
+ }
+ want_acting.clear();
+ // We can only get here when new interval has arrived and
+ // we've accepted the acting set. Now we can create
+ // actingbackfill and backfill_targets vectors.
+ actingbackfill = acting;
+ if (!compat_mode)
+ actingbackfill.insert(actingbackfill.end(), backfill.begin(), backfill.end());
+ assert(backfill_targets.empty() || backfill_targets == backfill);
+ if (backfill_targets.empty()) {
+ backfill_targets = backfill;
+ for (unsigned i = 0; i < backfill.size() ; ++i) {
+ stray_set.erase(backfill[i]);
+ }
} else {
- want_acting.clear();
+ // Will not change if already set because up would have had to change
+ assert(backfill_targets == backfill);
+ // Verify that nothing in backfill is in stray_set
+ for (unsigned i = 0; i < backfill.size() ; ++i) {
+ assert(stray_set.find(backfill[i]) == stray_set.end());
+ }
}
- dout(10) << "choose_acting want " << want << " (== acting)" << dendl;
+ dout(10) << "choose_acting want " << want << " (== acting) backfill_targets "
+ << backfill << dendl;
return true;
}
@@ -1127,7 +1230,7 @@ void PG::activate(ObjectStore::Transaction& t,
}
}
- if (role == 0) { // primary state
+ if (is_primary()) {
last_update_ondisk = info.last_update;
min_last_complete_ondisk = eversion_t(0,0); // we don't know (yet)!
}
@@ -1179,11 +1282,9 @@ void PG::activate(ObjectStore::Transaction& t,
if (is_primary()) {
// start up replicas
- // count replicas that are not backfilling
- unsigned active = 1;
-
- for (unsigned i=1; i<acting.size(); i++) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++) {
+ int peer = actingbackfill[i];
assert(peer_info.count(peer));
pg_info_t& pi = peer_info[peer];
@@ -1243,9 +1344,6 @@ void PG::activate(ObjectStore::Transaction& t,
if (needs_past_intervals)
m->past_intervals = past_intervals;
- if (pi.last_backfill == hobject_t::get_max())
- active++;
-
// update local version of peer's missing list!
if (m && pi.last_backfill != hobject_t()) {
for (list<pg_log_entry_t>::iterator p = m->log.log.begin();
@@ -1274,7 +1372,7 @@ void PG::activate(ObjectStore::Transaction& t,
}
// degraded?
- if (get_osdmap()->get_pg_size(info.pgid) > active)
+ if (get_osdmap()->get_pg_size(info.pgid) > acting.size())
state_set(PG_STATE_DEGRADED);
// all clean?
@@ -1307,28 +1405,14 @@ void PG::activate(ObjectStore::Transaction& t,
publish_stats_to_osd();
}
- // we need to flush this all out before doing anything else..
- need_flush = true;
-
// waiters
- if (!is_replay()) {
+ if (!is_replay() && flushes_in_progress == 0) {
requeue_ops(waiting_for_active);
}
on_activate();
}
-void PG::do_pending_flush()
-{
- assert(is_locked());
- if (need_flush) {
- dout(10) << "do_pending_flush doing pending flush" << dendl;
- osr->flush();
- need_flush = false;
- dout(10) << "do_pending_flush done" << dendl;
- }
-}
-
bool PG::op_has_sufficient_caps(OpRequestRef op)
{
// only check MOSDOp
@@ -1437,7 +1521,8 @@ void PG::_activate_committed(epoch_t e)
dout(10) << "_activate_committed " << e << " peer_activated now " << peer_activated
<< " last_epoch_started " << info.history.last_epoch_started
<< " same_interval_since " << info.history.same_interval_since << dendl;
- if (peer_activated.size() == acting.size())
+ assert(actingbackfill.size() > 0);
+ if (peer_activated.size() == actingbackfill.size())
all_activated_and_committed();
} else {
dout(10) << "_activate_committed " << e << " telling primary" << dendl;
@@ -1453,7 +1538,7 @@ void PG::_activate_committed(epoch_t e)
if (dirty_info) {
ObjectStore::Transaction *t = new ObjectStore::Transaction;
write_if_dirty(*t);
- int tr = osd->store->queue_transaction(osr.get(), t);
+ int tr = osd->store->queue_transaction_and_cleanup(osr.get(), t);
assert(tr == 0);
}
@@ -1469,7 +1554,8 @@ void PG::all_activated_and_committed()
{
dout(10) << "all_activated_and_committed" << dendl;
assert(is_primary());
- assert(peer_activated.size() == acting.size());
+ assert(peer_activated.size() == actingbackfill.size());
+ assert(actingbackfill.size() > 0);
// info.last_epoch_started is set during activate()
info.history.last_epoch_started = info.last_epoch_started;
@@ -1704,8 +1790,9 @@ void PG::split_into(pg_t child_pgid, PG *child, unsigned split_bits)
child->snap_trimq = snap_trimq;
+ // There can't be recovery/backfill going on now
get_osdmap()->pg_to_up_acting_osds(child->info.pgid, child->up, child->acting);
- child->role = get_osdmap()->calc_pg_role(osd->whoami, child->acting);
+ child->role = OSDMap::calc_pg_role(osd->whoami, child->acting);
if (get_primary() != child->get_primary())
child->info.history.same_primary_since = get_osdmap()->get_epoch();
@@ -1736,10 +1823,10 @@ void PG::clear_recovery_state()
finish_recovery_op(soid, true);
}
- backfill_target = -1;
+ backfill_targets.clear();
backfill_info.clear();
peer_backfill_info.clear();
- waiting_on_backfill = false;
+ waiting_on_backfill.clear();
_clear_recovery_state(); // pg impl specific hook
}
@@ -1804,7 +1891,7 @@ void PG::update_heartbeat_peers()
assert(is_locked());
set<int> new_peers;
- if (role == 0) {
+ if (is_primary()) {
for (unsigned i=0; i<acting.size(); i++)
new_peers.insert(acting[i]);
for (unsigned i=0; i<up.size(); i++)
@@ -1828,6 +1915,58 @@ void PG::update_heartbeat_peers()
osd->need_heartbeat_peer_update();
}
+void PG::_update_calc_stats()
+{
+ info.stats.version = info.last_update;
+ info.stats.created = info.history.epoch_created;
+ info.stats.last_scrub = info.history.last_scrub;
+ info.stats.last_scrub_stamp = info.history.last_scrub_stamp;
+ info.stats.last_deep_scrub = info.history.last_deep_scrub;
+ info.stats.last_deep_scrub_stamp = info.history.last_deep_scrub_stamp;
+ info.stats.last_clean_scrub_stamp = info.history.last_clean_scrub_stamp;
+ info.stats.last_epoch_clean = info.history.last_epoch_clean;
+
+ info.stats.log_size = pg_log.get_head().version - pg_log.get_tail().version;
+ info.stats.ondisk_log_size =
+ pg_log.get_head().version - pg_log.get_tail().version;
+ info.stats.log_start = pg_log.get_tail();
+ info.stats.ondisk_log_start = pg_log.get_tail();
+
+ // calc copies, degraded
+ unsigned target = MAX(get_osdmap()->get_pg_size(info.pgid), actingbackfill.size());
+ info.stats.stats.calc_copies(target);
+ info.stats.stats.sum.num_objects_degraded = 0;
+ if ((is_degraded() || !is_clean()) && is_active()) {
+ // NOTE: we only generate copies, degraded, unfound values for
+ // the summation, not individual stat categories.
+ uint64_t num_objects = info.stats.stats.sum.num_objects;
+
+ uint64_t degraded = 0;
+
+ // if the actingbackfill set is smaller than we want, add in those missing replicas
+ if (actingbackfill.size() < target)
+ degraded += (target - actingbackfill.size()) * num_objects;
+
+ // missing on primary
+ info.stats.stats.sum.num_objects_missing_on_primary =
+ pg_log.get_missing().num_missing();
+ degraded += pg_log.get_missing().num_missing();
+
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++) {
+ assert(peer_missing.count(actingbackfill[i]));
+
+ // in missing set
+ degraded += peer_missing[actingbackfill[i]].num_missing();
+
+ // not yet backfilled
+ degraded += num_objects - peer_info[actingbackfill[i]].stats.stats.sum.num_objects;
+ }
+ info.stats.stats.sum.num_objects_degraded = degraded;
+ info.stats.stats.sum.num_objects_unfound = get_num_unfound();
+ }
+}
+
void PG::publish_stats_to_osd()
{
pg_stats_publish_lock.Lock();
@@ -1835,14 +1974,6 @@ void PG::publish_stats_to_osd()
// update our stat summary
info.stats.reported_epoch = get_osdmap()->get_epoch();
++info.stats.reported_seq;
- info.stats.version = info.last_update;
- info.stats.created = info.history.epoch_created;
- info.stats.last_scrub = info.history.last_scrub;
- info.stats.last_scrub_stamp = info.history.last_scrub_stamp;
- info.stats.last_deep_scrub = info.history.last_deep_scrub;
- info.stats.last_deep_scrub_stamp = info.history.last_deep_scrub_stamp;
- info.stats.last_clean_scrub_stamp = info.history.last_clean_scrub_stamp;
- info.stats.last_epoch_clean = info.history.last_epoch_clean;
if (info.stats.stats.sum.num_scrub_errors)
state_set(PG_STATE_INCONSISTENT);
@@ -1864,49 +1995,12 @@ void PG::publish_stats_to_osd()
info.stats.last_active = now;
info.stats.last_unstale = now;
- info.stats.log_size = pg_log.get_head().version - pg_log.get_tail().version;
- info.stats.ondisk_log_size =
- pg_log.get_head().version - pg_log.get_tail().version;
- info.stats.log_start = pg_log.get_tail();
- info.stats.ondisk_log_start = pg_log.get_tail();
+ _update_calc_stats();
pg_stats_publish_valid = true;
pg_stats_publish = info.stats;
pg_stats_publish.stats.add(unstable_stats);
- // calc copies, degraded
- unsigned target = MAX(get_osdmap()->get_pg_size(info.pgid), acting.size());
- pg_stats_publish.stats.calc_copies(target);
- pg_stats_publish.stats.sum.num_objects_degraded = 0;
- if ((is_degraded() || !is_clean()) && is_active()) {
- // NOTE: we only generate copies, degraded, unfound values for
- // the summation, not individual stat categories.
- uint64_t num_objects = pg_stats_publish.stats.sum.num_objects;
-
- uint64_t degraded = 0;
-
- // if the acting set is smaller than we want, add in those missing replicas
- if (acting.size() < target)
- degraded += (target - acting.size()) * num_objects;
-
- // missing on primary
- pg_stats_publish.stats.sum.num_objects_missing_on_primary =
- pg_log.get_missing().num_missing();
- degraded += pg_log.get_missing().num_missing();
-
- for (unsigned i=1; i<acting.size(); i++) {
- assert(peer_missing.count(acting[i]));
-
- // in missing set
- degraded += peer_missing[acting[i]].num_missing();
-
- // not yet backfilled
- degraded += num_objects - peer_info[acting[i]].stats.stats.sum.num_objects;
- }
- pg_stats_publish.stats.sum.num_objects_degraded = degraded;
- pg_stats_publish.stats.sum.num_objects_unfound = get_num_unfound();
- }
-
dout(15) << "publish_stats_to_osd " << pg_stats_publish.reported_epoch
<< ":" << pg_stats_publish.reported_seq << dendl;
} else {
@@ -2213,11 +2307,13 @@ void PG::write_if_dirty(ObjectStore::Transaction& t)
void PG::trim_peers()
{
+ assert(is_primary());
calc_trim_to();
dout(10) << "trim_peers " << pg_trim_to << dendl;
if (pg_trim_to != eversion_t()) {
- for (unsigned i=1; i<acting.size(); i++)
- osd->send_message_osd_cluster(acting[i],
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++)
+ osd->send_message_osd_cluster(actingbackfill[i],
new MOSDPGTrim(get_osdmap()->get_epoch(), info.pgid,
pg_trim_to),
get_osdmap()->get_epoch());
@@ -2248,8 +2344,11 @@ void PG::add_log_entry(pg_log_entry_t& e, bufferlist& log_bl)
void PG::append_log(
- vector<pg_log_entry_t>& logv, eversion_t trim_to, ObjectStore::Transaction &t)
+ vector<pg_log_entry_t>& logv, eversion_t trim_to, ObjectStore::Transaction &t,
+ bool transaction_applied)
{
+ if (transaction_applied)
+ update_snap_map(logv, t);
dout(10) << "append_log " << pg_log.get_log() << " " << logv << dendl;
map<string,bufferlist> keys;
@@ -2259,11 +2358,14 @@ void PG::append_log(
p->offset = 0;
add_log_entry(*p, keys[p->get_key_name()]);
}
+ if (!transaction_applied)
+ pg_log.clear_can_rollback_to();
dout(10) << "append_log adding " << keys.size() << " keys" << dendl;
t.omap_setkeys(coll_t::META_COLL, log_oid, keys);
-
- pg_log.trim(trim_to, info);
+ PGLogEntryHandler handler;
+ pg_log.trim(&handler, trim_to, info);
+ handler.apply(this, &t);
// update the local pg, pg log
dirty_info = true;
@@ -2444,12 +2546,12 @@ void PG::update_snap_map(
}
set<snapid_t> _snaps(snaps.begin(), snaps.end());
- if (i->is_clone()) {
+ if (i->is_clone() || i->is_promote()) {
snap_mapper.add_oid(
i->soid,
_snaps,
&_t);
- } else {
+ } else if (i->is_modify()) {
assert(i->is_modify());
int r = snap_mapper.update_snaps(
i->soid,
@@ -2457,6 +2559,8 @@ void PG::update_snap_map(
0,
&_t);
assert(r == 0);
+ } else {
+ assert(i->is_clean());
}
}
}
@@ -2672,111 +2776,6 @@ void PG::sub_op_scrub_map(OpRequestRef op)
}
}
-/*
- * pg lock may or may not be held
- */
-void PG::_scan_list(
- ScrubMap &map, vector<hobject_t> &ls, bool deep,
- ThreadPool::TPHandle &handle)
-{
- dout(10) << "_scan_list scanning " << ls.size() << " objects"
- << (deep ? " deeply" : "") << dendl;
- int i = 0;
- for (vector<hobject_t>::iterator p = ls.begin();
- p != ls.end();
- ++p, i++) {
- handle.reset_tp_timeout();
- hobject_t poid = *p;
-
- struct stat st;
- int r = osd->store->stat(coll, poid, &st, true);
- if (r == 0) {
- ScrubMap::object &o = map.objects[poid];
- o.size = st.st_size;
- assert(!o.negative);
- osd->store->getattrs(coll, poid, o.attrs);
-
- // calculate the CRC32 on deep scrubs
- if (deep) {
- bufferhash h, oh;
- bufferlist bl, hdrbl;
- int r;
- __u64 pos = 0;
- while ( (r = osd->store->read(coll, poid, pos,
- cct->_conf->osd_deep_scrub_stride, bl,
- true)) > 0) {
- handle.reset_tp_timeout();
- h << bl;
- pos += bl.length();
- bl.clear();
- }
- if (r == -EIO) {
- dout(25) << "_scan_list " << poid << " got "
- << r << " on read, read_error" << dendl;
- o.read_error = true;
- }
- o.digest = h.digest();
- o.digest_present = true;
-
- bl.clear();
- r = osd->store->omap_get_header(coll, poid, &hdrbl, true);
- if (r == 0) {
- dout(25) << "CRC header " << string(hdrbl.c_str(), hdrbl.length())
- << dendl;
- ::encode(hdrbl, bl);
- oh << bl;
- bl.clear();
- } else if (r == -EIO) {
- dout(25) << "_scan_list " << poid << " got "
- << r << " on omap header read, read_error" << dendl;
- o.read_error = true;
- }
-
- ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(
- coll, poid);
- assert(iter);
- uint64_t keys_scanned = 0;
- for (iter->seek_to_first(); iter->valid() ; iter->next()) {
- if (cct->_conf->osd_scan_list_ping_tp_interval &&
- (keys_scanned % cct->_conf->osd_scan_list_ping_tp_interval == 0)) {
- handle.reset_tp_timeout();
- }
- ++keys_scanned;
-
- dout(25) << "CRC key " << iter->key() << " value "
- << string(iter->value().c_str(), iter->value().length()) << dendl;
-
- ::encode(iter->key(), bl);
- ::encode(iter->value(), bl);
- oh << bl;
- bl.clear();
- }
- if (iter->status() == -EIO) {
- dout(25) << "_scan_list " << poid << " got "
- << r << " on omap scan, read_error" << dendl;
- o.read_error = true;
- break;
- }
-
- //Store final calculated CRC32 of omap header & key/values
- o.omap_digest = oh.digest();
- o.omap_digest_present = true;
- }
-
- dout(25) << "_scan_list " << poid << dendl;
- } else if (r == -ENOENT) {
- dout(25) << "_scan_list " << poid << " got " << r << ", skipping" << dendl;
- } else if (r == -EIO) {
- dout(25) << "_scan_list " << poid << " got " << r << ", read_error" << dendl;
- ScrubMap::object &o = map.objects[poid];
- o.read_error = true;
- } else {
- derr << "_scan_list got: " << cpp_strerror(r) << dendl;
- assert(0);
- }
- }
-}
-
// send scrub v2-compatible messages (classic scrub)
void PG::_request_scrub_map_classic(int replica, eversion_t version)
{
@@ -2914,6 +2913,7 @@ void PG::clear_scrub_reserved()
void PG::scrub_reserve_replicas()
{
+ assert(backfill_targets.empty());
for (unsigned i=1; i<acting.size(); i++) {
dout(10) << "scrub requesting reserve from osd." << acting[i] << dendl;
vector<OSDOp> scrub(1);
@@ -2930,6 +2930,7 @@ void PG::scrub_reserve_replicas()
void PG::scrub_unreserve_replicas()
{
+ assert(backfill_targets.empty());
for (unsigned i=1; i<acting.size(); i++) {
dout(10) << "scrub requesting unreserve from osd." << acting[i] << dendl;
vector<OSDOp> scrub(1);
@@ -3037,7 +3038,7 @@ int PG::build_scrub_map_chunk(
return ret;
}
- _scan_list(map, ls, deep, handle);
+ get_pgbackend()->be_scan_list(map, ls, deep, handle);
_scan_snaps(map);
// pg attrs
@@ -3068,7 +3069,7 @@ void PG::build_scrub_map(ScrubMap &map, ThreadPool::TPHandle &handle)
vector<hobject_t> ls;
osd->store->collection_list(coll, ls);
- _scan_list(map, ls, false, handle);
+ get_pgbackend()->be_scan_list(map, ls, false, handle);
lock();
_scan_snaps(map);
@@ -3117,7 +3118,7 @@ void PG::build_inc_scrub_map(
}
}
- _scan_list(map, ls, false, handle);
+ get_pgbackend()->be_scan_list(map, ls, false, handle);
// pg attrs
osd->store->collection_getattrs(coll, map.attrs);
}
@@ -3268,6 +3269,7 @@ void PG::scrub(ThreadPool::TPHandle &handle)
if (!scrubber.active) {
OSDMapRef curmap = osd->get_osdmap();
scrubber.is_chunky = true;
+ assert(backfill_targets.empty());
for (unsigned i=1; i<acting.size(); i++) {
ConnectionRef con = osd->get_con_osd_cluster(acting[i], get_osdmap()->get_epoch());
if (!con)
@@ -3340,6 +3342,7 @@ void PG::scrub(ThreadPool::TPHandle &handle)
*/
void PG::classic_scrub(ThreadPool::TPHandle &handle)
{
+ assert(pool.info.type == pg_pool_t::TYPE_REPLICATED);
if (!scrubber.active) {
dout(10) << "scrub start" << dendl;
scrubber.active = true;
@@ -3747,213 +3750,8 @@ bool PG::scrub_gather_replica_maps()
}
}
-enum PG::error_type PG::_compare_scrub_objects(ScrubMap::object &auth,
- ScrubMap::object &candidate,
- ostream &errorstream)
-{
- enum PG::error_type error = CLEAN;
- if (candidate.read_error) {
- // This can occur on stat() of a shallow scrub, but in that case size will
- // be invalid, and this will be over-ridden below.
- error = DEEP_ERROR;
- errorstream << "candidate had a read error";
- }
- if (auth.digest_present && candidate.digest_present) {
- if (auth.digest != candidate.digest) {
- if (error != CLEAN)
- errorstream << ", ";
- error = DEEP_ERROR;
-
- errorstream << "digest " << candidate.digest
- << " != known digest " << auth.digest;
- }
- }
- if (auth.omap_digest_present && candidate.omap_digest_present) {
- if (auth.omap_digest != candidate.omap_digest) {
- if (error != CLEAN)
- errorstream << ", ";
- error = DEEP_ERROR;
-
- errorstream << "omap_digest " << candidate.omap_digest
- << " != known omap_digest " << auth.omap_digest;
- }
- }
- // Shallow error takes precendence because this will be seen by
- // both types of scrubs.
- if (auth.size != candidate.size) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "size " << candidate.size
- << " != known size " << auth.size;
- }
- for (map<string,bufferptr>::const_iterator i = auth.attrs.begin();
- i != auth.attrs.end();
- ++i) {
- if (!candidate.attrs.count(i->first)) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "missing attr " << i->first;
- } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "attr value mismatch " << i->first;
- }
- }
- for (map<string,bufferptr>::const_iterator i = candidate.attrs.begin();
- i != candidate.attrs.end();
- ++i) {
- if (!auth.attrs.count(i->first)) {
- if (error != CLEAN)
- errorstream << ", ";
- error = SHALLOW_ERROR;
- errorstream << "extra attr " << i->first;
- }
- }
- return error;
-}
-
-map<int, ScrubMap *>::const_iterator PG::_select_auth_object(
- const hobject_t &obj,
- const map<int,ScrubMap*> &maps)
-{
- map<int, ScrubMap *>::const_iterator auth = maps.end();
- for (map<int, ScrubMap *>::const_iterator j = maps.begin();
- j != maps.end();
- ++j) {
- map<hobject_t, ScrubMap::object>::iterator i =
- j->second->objects.find(obj);
- if (i == j->second->objects.end()) {
- continue;
- }
- if (auth == maps.end()) {
- // Something is better than nothing
- // TODO: something is NOT better than nothing, do something like
- // unfound_lost if no valid copies can be found, or just mark unfound
- auth = j;
- dout(10) << __func__ << ": selecting osd " << j->first
- << " for obj " << obj
- << ", auth == maps.end()"
- << dendl;
- continue;
- }
- if (i->second.read_error) {
- // scrub encountered read error, probably corrupt
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", read_error"
- << dendl;
- continue;
- }
- map<string, bufferptr>::iterator k = i->second.attrs.find(OI_ATTR);
- if (k == i->second.attrs.end()) {
- // no object info on object, probably corrupt
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", no oi attr"
- << dendl;
- continue;
- }
- bufferlist bl;
- bl.push_back(k->second);
- object_info_t oi;
- try {
- bufferlist::iterator bliter = bl.begin();
- ::decode(oi, bliter);
- } catch (...) {
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", corrupt oi attr"
- << dendl;
- // invalid object info, probably corrupt
- continue;
- }
- if (oi.size != i->second.size) {
- // invalid size, probably corrupt
- dout(10) << __func__ << ": rejecting osd " << j->first
- << " for obj " << obj
- << ", size mismatch"
- << dendl;
- // invalid object info, probably corrupt
- continue;
- }
- dout(10) << __func__ << ": selecting osd " << j->first
- << " for obj " << obj
- << dendl;
- auth = j;
- }
- return auth;
-}
-
-void PG::_compare_scrubmaps(const map<int,ScrubMap*> &maps,
- map<hobject_t, set<int> > &missing,
- map<hobject_t, set<int> > &inconsistent,
- map<hobject_t, int> &authoritative,
- map<hobject_t, set<int> > &invalid_snapcolls,
- ostream &errorstream)
-{
- map<hobject_t,ScrubMap::object>::const_iterator i;
- map<int, ScrubMap *>::const_iterator j;
- set<hobject_t> master_set;
-
- // Construct master set
- for (j = maps.begin(); j != maps.end(); ++j) {
- for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) {
- master_set.insert(i->first);
- }
- }
-
- // Check maps against master set and each other
- for (set<hobject_t>::const_iterator k = master_set.begin();
- k != master_set.end();
- ++k) {
- map<int, ScrubMap *>::const_iterator auth = _select_auth_object(*k, maps);
- assert(auth != maps.end());
- set<int> cur_missing;
- set<int> cur_inconsistent;
- for (j = maps.begin(); j != maps.end(); ++j) {
- if (j == auth)
- continue;
- if (j->second->objects.count(*k)) {
- // Compare
- stringstream ss;
- enum PG::error_type error = _compare_scrub_objects(auth->second->objects[*k],
- j->second->objects[*k],
- ss);
- if (error != CLEAN) {
- cur_inconsistent.insert(j->first);
- if (error == SHALLOW_ERROR)
- ++scrubber.shallow_errors;
- else
- ++scrubber.deep_errors;
- errorstream << info.pgid << " osd." << acting[j->first]
- << ": soid " << *k << " " << ss.str() << std::endl;
- }
- } else {
- cur_missing.insert(j->first);
- ++scrubber.shallow_errors;
- errorstream << info.pgid
- << " osd." << acting[j->first]
- << " missing " << *k << std::endl;
- }
- }
- assert(auth != maps.end());
- if (!cur_missing.empty()) {
- missing[*k] = cur_missing;
- }
- if (!cur_inconsistent.empty()) {
- inconsistent[*k] = cur_inconsistent;
- }
- if (!cur_inconsistent.empty() || !cur_missing.empty()) {
- authoritative[*k] = auth->first;
- }
- }
-}
-
void PG::scrub_compare_maps()
{
dout(10) << "scrub_compare_maps has maps, analyzing" << dendl;
@@ -3979,12 +3777,15 @@ void PG::scrub_compare_maps()
maps[i] = &scrubber.received_maps[acting[i]];
}
- _compare_scrubmaps(
+ get_pgbackend()->be_compare_scrubmaps(
maps,
scrubber.missing,
scrubber.inconsistent,
authoritative,
scrubber.inconsistent_snapcolls,
+ scrubber.shallow_errors,
+ scrubber.deep_errors,
+ info.pgid, acting,
ss);
dout(2) << ss.str() << dendl;
@@ -4179,7 +3980,7 @@ void PG::scrub_finish()
ObjectStore::Transaction *t = new ObjectStore::Transaction;
dirty_info = true;
write_if_dirty(*t);
- int tr = osd->store->queue_transaction(osr.get(), t);
+ int tr = osd->store->queue_transaction_and_cleanup(osr.get(), t);
assert(tr == 0);
}
@@ -4206,8 +4007,9 @@ void PG::share_pg_info()
dout(10) << "share_pg_info" << dendl;
// share new pg_info_t with replicas
- for (unsigned i=1; i<acting.size(); i++) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++) {
+ int peer = actingbackfill[i];
if (peer_info.count(i)) {
peer_info[i].last_epoch_started = info.last_epoch_started;
peer_info[i].history.merge(info.history);
@@ -4234,9 +4036,9 @@ void PG::share_pg_log()
dout(10) << __func__ << dendl;
assert(is_primary());
- vector<int>::const_iterator a = acting.begin();
- assert(a != acting.end());
- vector<int>::const_iterator end = acting.end();
+ vector<int>::const_iterator a = actingbackfill.begin();
+ assert(a != actingbackfill.end());
+ vector<int>::const_iterator end = actingbackfill.end();
while (++a != end) {
int peer(*a);
pg_missing_t& pmissing(peer_missing[peer]);
@@ -4438,7 +4240,7 @@ struct FlushState {
pg->unlock();
}
};
-typedef std::tr1::shared_ptr<FlushState> FlushStateRef;
+typedef ceph::shared_ptr<FlushState> FlushStateRef;
void PG::start_flush(ObjectStore::Transaction *t,
list<Context *> *on_applied,
@@ -4448,7 +4250,7 @@ void PG::start_flush(ObjectStore::Transaction *t,
FlushStateRef flush_trigger(
new FlushState(this, get_osdmap()->get_epoch()));
t->nop();
- assert(!flushed);
+ flushes_in_progress++;
on_applied->push_back(new ContainerContext<FlushStateRef>(flush_trigger));
on_safe->push_back(new ContainerContext<FlushStateRef>(flush_trigger));
}
@@ -4466,6 +4268,7 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
vector<int> oldacting, oldup;
int oldrole = get_role();
int oldprimary = get_primary();
+ bool was_old_primary = is_primary();
acting.swap(oldacting);
up.swap(oldup);
@@ -4479,6 +4282,8 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
info.stats.mapping_epoch = info.history.same_interval_since;
}
+ // This will now be remapped during a backfill in cases
+ // that it would not have been before.
if (up != acting)
state_set(PG_STATE_REMAPPED);
else
@@ -4536,9 +4341,10 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
peer_missing.clear();
peer_purged.clear();
+ actingbackfill.clear();
// reset primary state?
- if (oldrole == 0 || get_role() == 0)
+ if (was_old_primary || is_primary())
clear_primary_state();
@@ -4547,9 +4353,10 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
assert(!deleting);
- if (role != oldrole) {
- // old primary?
- if (oldrole == 0) {
+ if (role != oldrole ||
+ was_old_primary != is_primary()) {
+ // did primary change?
+ if (was_old_primary != is_primary()) {
state_clear(PG_STATE_CLEAN);
clear_publish_stats();
@@ -4569,7 +4376,7 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
requeue_ops(waiting_for_active);
// should we tell the primary we are here?
- send_notify = (role != 0);
+ send_notify = !is_primary();
} else {
// no role change.
@@ -4584,7 +4391,7 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
<< dendl;
} else {
// primary is the same.
- if (role == 0) {
+ if (is_primary()) {
// i am (still) primary. but my replica set changed.
state_clear(PG_STATE_CLEAN);
@@ -4682,13 +4489,14 @@ ostream& operator<<(ostream& out, const PG& pg)
}
}
- if (pg.get_backfill_target() >= 0)
- out << " bft=" << pg.get_backfill_target();
+ if (!pg.backfill_targets.empty())
+ out << " bft=" << pg.backfill_targets;
+ out << " crt=" << pg.pg_log.get_log().can_rollback_to;
if (pg.last_complete_ondisk != pg.info.last_complete)
out << " lcod " << pg.last_complete_ondisk;
- if (pg.get_role() == 0) {
+ if (pg.is_primary()) {
out << " mlcod " << pg.min_last_complete_ondisk;
}
@@ -4727,7 +4535,7 @@ bool PG::can_discard_op(OpRequestRef op)
if (OSD::op_is_discardable(m)) {
dout(20) << " discard " << *m << dendl;
return true;
- } else if (op->may_write() &&
+ } else if ((op->may_write() || op->may_cache()) &&
(!is_primary() ||
!same_for_modify_since(m->get_map_epoch()))) {
osd->handle_misdirected_op(this, op);
@@ -4754,6 +4562,15 @@ bool PG::can_discard_replica_op(OpRequestRef op)
T *m = static_cast<T *>(op->get_req());
assert(m->get_header().type == MSGTYPE);
+ /* Mostly, this overlaps with the old_peering_msg
+ * condition. An important exception is pushes
+ * sent by replicas not in the acting set, since
+ * if such a replica goes down it does not cause
+ * a new interval. */
+ int from = m->get_source().num();
+ if (get_osdmap()->get_down_at(from) >= m->map_epoch)
+ return true;
+
// same pg?
// if pg changes _at all_, we reset and repeer!
if (old_peering_msg(m->map_epoch, m->map_epoch)) {
@@ -4971,6 +4788,8 @@ void PG::handle_advance_map(OSDMapRef osdmap, OSDMapRef lastmap,
dout(10) << "handle_advance_map " << newup << "/" << newacting << dendl;
update_osdmap_ref(osdmap);
pool.update(osdmap);
+ if (pool.info.last_change == osdmap_ref->get_epoch())
+ on_pool_change();
AdvMap evt(osdmap, lastmap, newup, newacting);
recovery_state.handle_event(evt, rctx);
}
@@ -5056,7 +4875,7 @@ boost::statechart::result PG::RecoveryState::Initial::react(const Load& l)
PG *pg = context< RecoveryMachine >().pg;
// do we tell someone we're here?
- pg->send_notify = (pg->role != 0);
+ pg->send_notify = (!pg->is_primary());
return transit< Reset >();
}
@@ -5106,9 +4925,7 @@ boost::statechart::result
PG::RecoveryState::Started::react(const FlushedEvt&)
{
PG *pg = context< RecoveryMachine >().pg;
- pg->flushed = true;
pg->on_flushed();
- pg->requeue_ops(pg->waiting_for_active);
return discard_event();
}
@@ -5151,7 +4968,7 @@ PG::RecoveryState::Reset::Reset(my_context ctx)
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
- pg->flushed = false;
+ pg->flushes_in_progress = 0;
pg->set_last_peering_reset();
}
@@ -5159,9 +4976,7 @@ boost::statechart::result
PG::RecoveryState::Reset::react(const FlushedEvt&)
{
PG *pg = context< RecoveryMachine >().pg;
- pg->flushed = true;
pg->on_flushed();
- pg->requeue_ops(pg->waiting_for_active);
return discard_event();
}
@@ -5296,8 +5111,7 @@ void PG::RecoveryState::Primary::exit()
/*---------Peering--------*/
PG::RecoveryState::Peering::Peering(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering"),
- flushed(false)
+ NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering")
{
context< RecoveryMachine >().log_enter(state_name);
@@ -5392,6 +5206,7 @@ PG::RecoveryState::Backfilling::Backfilling(my_context ctx)
PG *pg = context< RecoveryMachine >().pg;
pg->backfill_reserved = true;
pg->osd->queue_for_recovery(pg);
+ pg->state_clear(PG_STATE_BACKFILL_TOOFULL);
pg->state_clear(PG_STATE_BACKFILL_WAIT);
pg->state_set(PG_STATE_BACKFILL);
}
@@ -5424,27 +5239,43 @@ void PG::RecoveryState::Backfilling::exit()
PG::RecoveryState::WaitRemoteBackfillReserved::WaitRemoteBackfillReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteBackfillReserved")
+ NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteBackfillReserved"),
+ backfill_osd_it(context< Active >().sorted_backfill_set.begin())
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
pg->state_set(PG_STATE_BACKFILL_WAIT);
- ConnectionRef con = pg->osd->get_con_osd_cluster(
- pg->backfill_target, pg->get_osdmap()->get_epoch());
- if (con) {
- if (con->has_feature(CEPH_FEATURE_BACKFILL_RESERVATION)) {
- unsigned priority = pg->is_degraded() ? OSDService::BACKFILL_HIGH
+ post_event(RemoteBackfillReserved());
+}
+
+boost::statechart::result
+PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteBackfillReserved &evt)
+{
+ PG *pg = context< RecoveryMachine >().pg;
+
+ if (backfill_osd_it != context< Active >().sorted_backfill_set.end()) {
+ //The primary never backfills itself
+ assert(*backfill_osd_it != pg->osd->whoami);
+ ConnectionRef con = pg->osd->get_con_osd_cluster(*backfill_osd_it, pg->get_osdmap()->get_epoch());
+ if (con) {
+ if (con->has_feature(CEPH_FEATURE_BACKFILL_RESERVATION)) {
+ unsigned priority = pg->is_degraded() ? OSDService::BACKFILL_HIGH
: OSDService::BACKFILL_LOW;
- pg->osd->send_message_osd_cluster(
- new MBackfillReserve(
+ pg->osd->send_message_osd_cluster(
+ new MBackfillReserve(
MBackfillReserve::REQUEST,
pg->info.pgid,
pg->get_osdmap()->get_epoch(), priority),
con.get());
- } else {
- post_event(RemoteBackfillReserved());
+ } else {
+ post_event(RemoteBackfillReserved());
+ }
}
+ ++backfill_osd_it;
+ } else {
+ post_event(AllBackfillsReserved());
}
+ return discard_event();
}
void PG::RecoveryState::WaitRemoteBackfillReserved::exit()
@@ -5456,14 +5287,6 @@ void PG::RecoveryState::WaitRemoteBackfillReserved::exit()
}
boost::statechart::result
-PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteBackfillReserved &evt)
-{
- PG *pg = context< RecoveryMachine >().pg;
- pg->state_clear(PG_STATE_BACKFILL_TOOFULL);
- return transit<Backfilling>();
-}
-
-boost::statechart::result
PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteReservationRejected &evt)
{
PG *pg = context< RecoveryMachine >().pg;
@@ -5755,7 +5578,6 @@ void PG::RecoveryState::Recovering::release_reservations()
{
PG *pg = context< RecoveryMachine >().pg;
assert(!pg->pg_log.get_missing().have_missing());
- pg->state_clear(PG_STATE_RECOVERING);
// release remote reservations
for (set<int>::const_iterator i = context< Active >().sorted_acting_set.begin();
@@ -5815,7 +5637,8 @@ PG::RecoveryState::Recovered::Recovered(my_context ctx)
// if we finished backfill, all acting are active; recheck if
// DEGRADED is appropriate.
- if (pg->get_osdmap()->get_pg_size(pg->info.pgid) <= pg->acting.size())
+ assert(pg->actingbackfill.size() > 0);
+ if (pg->get_osdmap()->get_pg_size(pg->info.pgid) <= pg->actingbackfill.size())
pg->state_clear(PG_STATE_DEGRADED);
// adjust acting set? (e.g. because backfill completed...)
@@ -5868,8 +5691,10 @@ void PG::RecoveryState::Clean::exit()
PG::RecoveryState::Active::Active(my_context ctx)
: my_base(ctx),
NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active"),
- sorted_acting_set(context< RecoveryMachine >().pg->acting.begin(),
- context< RecoveryMachine >().pg->acting.end()),
+ sorted_acting_set(context< RecoveryMachine >().pg->actingbackfill.begin(),
+ context< RecoveryMachine >().pg->actingbackfill.end()),
+ sorted_backfill_set(context< RecoveryMachine >().pg->backfill_targets.begin(),
+ context< RecoveryMachine >().pg->backfill_targets.end()),
all_replicas_activated(false)
{
context< RecoveryMachine >().log_enter(state_name);
@@ -5880,6 +5705,10 @@ PG::RecoveryState::Active::Active(my_context ctx)
assert(!pg->backfill_reserved);
assert(pg->is_primary());
dout(10) << "In Active, about to call activate" << dendl;
+ pg->start_flush(
+ context< RecoveryMachine >().get_cur_transaction(),
+ context< RecoveryMachine >().get_on_applied_context_list(),
+ context< RecoveryMachine >().get_on_safe_context_list());
pg->activate(*context< RecoveryMachine >().get_cur_transaction(),
pg->get_osdmap()->get_epoch(),
*context< RecoveryMachine >().get_on_safe_context_list(),
@@ -5914,10 +5743,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const AdvMap& advmap)
* this does not matter) */
if (advmap.lastmap->get_pg_size(pg->info.pgid) !=
pg->get_osdmap()->get_pg_size(pg->info.pgid)) {
- unsigned active = pg->acting.size();
- if (pg->backfill_target != -1)
- --active;
- if (pg->get_osdmap()->get_pg_size(pg->info.pgid) <= active)
+ if (pg->get_osdmap()->get_pg_size(pg->info.pgid) <= pg->acting.size())
pg->state_clear(PG_STATE_DEGRADED);
else
pg->state_set(PG_STATE_DEGRADED);
@@ -6004,10 +5830,11 @@ boost::statechart::result PG::RecoveryState::Active::react(const MInfoRec& infoe
assert(pg->is_active());
assert(pg->is_primary());
+ assert(pg->actingbackfill.size() > 0);
// don't update history (yet) if we are active and primary; the replica
// may be telling us they have activated (and committed) but we can't
// share that until _everyone_ does the same.
- if (pg->is_acting(infoevt.from)) {
+ if (pg->is_actingbackfill(infoevt.from)) {
assert(pg->info.history.last_epoch_started <
pg->info.history.same_interval_since);
assert(infoevt.info.history.last_epoch_started >=
@@ -6017,7 +5844,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const MInfoRec& infoe
pg->peer_activated.insert(infoevt.from);
}
- if (pg->peer_activated.size() == pg->acting.size()) {
+ if (pg->peer_activated.size() == pg->actingbackfill.size()) {
pg->all_activated_and_committed();
}
return discard_event();
@@ -6218,11 +6045,10 @@ PG::RecoveryState::Stray::Stray(my_context ctx)
assert(!pg->is_active());
assert(!pg->is_peering());
assert(!pg->is_primary());
- if (!pg->is_replica()) // stray, need to flush for pulls
- pg->start_flush(
- context< RecoveryMachine >().get_cur_transaction(),
- context< RecoveryMachine >().get_on_applied_context_list(),
- context< RecoveryMachine >().get_on_safe_context_list());
+ pg->start_flush(
+ context< RecoveryMachine >().get_cur_transaction(),
+ context< RecoveryMachine >().get_on_applied_context_list(),
+ context< RecoveryMachine >().get_on_safe_context_list());
}
boost::statechart::result PG::RecoveryState::Stray::react(const MLogRec& logevt)
@@ -6522,7 +6348,9 @@ PG::RecoveryState::GetLog::GetLog(my_context ctx)
// how much log to request?
eversion_t request_log_from = pg->info.last_update;
- for (vector<int>::iterator p = pg->acting.begin() + 1; p != pg->acting.end(); ++p) {
+ assert(pg->actingbackfill.size() > 0);
+ for (vector<int>::iterator p = pg->actingbackfill.begin() + 1;
+ p != pg->actingbackfill.end(); ++p) {
pg_info_t& ri = pg->peer_info[*p];
if (ri.last_update >= best.log_tail && ri.last_update < request_log_from)
request_log_from = ri.last_update;
@@ -6688,6 +6516,21 @@ boost::statechart::result PG::RecoveryState::Incomplete::react(const AdvMap &adv
return forward_event();
}
+boost::statechart::result PG::RecoveryState::Incomplete::react(const MNotifyRec& notevt) {
+ dout(7) << "handle_pg_notify from osd." << notevt.from << dendl;
+ PG *pg = context< RecoveryMachine >().pg;
+ if (pg->peer_info.count(notevt.from) &&
+ pg->peer_info[notevt.from].last_update == notevt.notify.info.last_update) {
+ dout(10) << *pg << " got dup osd." << notevt.from << " info " << notevt.notify.info
+ << ", identical to ours" << dendl;
+ return discard_event();
+ } else {
+ pg->proc_replica_info(notevt.from, notevt.notify.info);
+ // try again!
+ return transit< GetLog >();
+ }
+}
+
void PG::RecoveryState::Incomplete::exit()
{
context< RecoveryMachine >().log_exit(state_name, enter_time);
@@ -6706,8 +6549,9 @@ PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
- for (vector<int>::iterator i = pg->acting.begin() + 1;
- i != pg->acting.end();
+ assert(pg->actingbackfill.size() > 0);
+ for (vector<int>::iterator i = pg->actingbackfill.begin() + 1;
+ i != pg->actingbackfill.end();
++i) {
const pg_info_t& pi = pg->peer_info[*i];
@@ -6830,7 +6674,7 @@ PG::RecoveryState::WaitFlushedPeering::WaitFlushedPeering(my_context ctx)
{
PG *pg = context< RecoveryMachine >().pg;
context< RecoveryMachine >().log_enter(state_name);
- if (context< RecoveryMachine >().pg->flushed)
+ if (context< RecoveryMachine >().pg->flushes_in_progress == 0)
post_event(Activate(pg->get_osdmap()->get_epoch()));
}
@@ -6838,8 +6682,6 @@ boost::statechart::result
PG::RecoveryState::WaitFlushedPeering::react(const FlushedEvt &evt)
{
PG *pg = context< RecoveryMachine >().pg;
- pg->flushed = true;
- pg->requeue_ops(pg->waiting_for_active);
pg->on_flushed();
return transit< WaitFlushedPeering >();
}
diff --git a/src/osd/PG.h b/src/osd/PG.h
index a1f1d2c..23c84fd 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -23,7 +23,7 @@
#include <boost/statechart/transition.hpp>
#include <boost/statechart/event_base.hpp>
#include <boost/scoped_ptr.hpp>
-#include <tr1/memory>
+#include "include/memory.h"
// re-include our assert to clobber boost's
#include "include/assert.h"
@@ -55,9 +55,8 @@
#include <string>
using namespace std;
-#include <ext/hash_map>
-#include <ext/hash_set>
-using namespace __gnu_cxx;
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
//#define DEBUG_RECOVERY_OIDS // track set of recovering oids explicitly, to find counting bugs
@@ -313,7 +312,7 @@ public:
* (if they have one) */
xlist<PG*>::item recovery_item, scrub_item, scrub_finalize_item, snap_trim_item, stat_queue_item;
int recovery_ops_active;
- bool waiting_on_backfill;
+ set<int> waiting_on_backfill;
#ifdef DEBUG_RECOVERY_OIDS
set<hobject_t> recovering_oids;
#endif
@@ -333,7 +332,7 @@ public:
// primary state
public:
- vector<int> up, acting, want_acting;
+ vector<int> up, acting, want_acting, actingbackfill;
map<int,eversion_t> peer_last_complete_ondisk;
eversion_t min_last_complete_ondisk; // up: min over last_complete_ondisk, peer_last_complete_ondisk
eversion_t pg_trim_to;
@@ -417,8 +416,6 @@ protected:
// primary-only, recovery-only state
set<int> might_have_unfound; // These osds might have objects on them
// which are unfound on the primary
- bool need_flush; // need to flush before any new activity
-
epoch_t last_peering_reset;
@@ -458,13 +455,13 @@ protected:
}
/// true if there are no objects in this interval
- bool empty() {
+ bool empty() const {
return objects.empty();
}
/// true if interval extends to the end of the range
- bool extends_to_end() {
- return end == hobject_t::get_max();
+ bool extends_to_end() const {
+ return end.is_max();
}
/// removes items <= soid and adjusts begin to the first object
@@ -508,23 +505,28 @@ protected:
};
BackfillInterval backfill_info;
- BackfillInterval peer_backfill_info;
- int backfill_target;
+ map<int, BackfillInterval> peer_backfill_info;
bool backfill_reserved;
bool backfill_reserving;
friend class OSD;
public:
- int get_backfill_target() const {
- return backfill_target;
+ vector<int> backfill_targets;
+
+ bool is_backfill_targets(int osd) {
+ if (std::find(backfill_targets.begin(), backfill_targets.end(), osd)
+ != backfill_targets.end())
+ return true;
+ else
+ return false;
}
protected:
// pg waiters
- bool flushed;
+ unsigned flushes_in_progress;
// Ops waiting on backfill_pos to change
list<OpRequestRef> waiting_for_active;
@@ -551,8 +553,9 @@ protected:
pg_stat_t pg_stats_publish;
// for ordering writes
- std::tr1::shared_ptr<ObjectStore::Sequencer> osr;
+ ceph::shared_ptr<ObjectStore::Sequencer> osr;
+ void _update_calc_stats();
void publish_stats_to_osd();
void clear_publish_stats();
@@ -570,6 +573,11 @@ public:
if (up[i] == osd) return true;
return false;
}
+ bool is_actingbackfill(int osd) const {
+ for (unsigned i=0; i<actingbackfill.size(); i++)
+ if (actingbackfill[i] == osd) return true;
+ return false;
+ }
bool needs_recovery() const;
bool needs_backfill() const;
@@ -591,10 +599,11 @@ public:
bool calc_min_last_complete_ondisk() {
eversion_t min = last_complete_ondisk;
- for (unsigned i=1; i<acting.size(); i++) {
- if (peer_last_complete_ondisk.count(acting[i]) == 0)
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++) {
+ if (peer_last_complete_ondisk.count(actingbackfill[i]) == 0)
return false; // we don't have complete info
- eversion_t a = peer_last_complete_ondisk[acting[i]];
+ eversion_t a = peer_last_complete_ondisk[actingbackfill[i]];
if (a < min)
min = a;
}
@@ -611,6 +620,97 @@ public:
void proc_master_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
pg_missing_t& omissing, int from);
bool proc_replica_info(int from, const pg_info_t &info);
+
+
+ struct LogEntryTrimmer : public ObjectModDesc::Visitor {
+ const hobject_t &soid;
+ PG *pg;
+ ObjectStore::Transaction *t;
+ LogEntryTrimmer(const hobject_t &soid, PG *pg, ObjectStore::Transaction *t)
+ : soid(soid), pg(pg), t(t) {}
+ void rmobject(version_t old_version) {
+ pg->get_pgbackend()->trim_stashed_object(
+ soid,
+ old_version,
+ t);
+ }
+ };
+
+ struct SnapRollBacker : public ObjectModDesc::Visitor {
+ const hobject_t &soid;
+ PG *pg;
+ ObjectStore::Transaction *t;
+ SnapRollBacker(const hobject_t &soid, PG *pg, ObjectStore::Transaction *t)
+ : soid(soid), pg(pg), t(t) {}
+ void update_snaps(set<snapid_t> &snaps) {
+ pg->update_object_snap_mapping(t, soid, snaps);
+ }
+ void create() {
+ pg->clear_object_snap_mapping(
+ t,
+ soid);
+ }
+ };
+
+ struct PGLogEntryHandler : public PGLog::LogEntryHandler {
+ map<hobject_t, list<pg_log_entry_t> > to_rollback;
+ set<hobject_t> cannot_rollback;
+ set<hobject_t> to_remove;
+ list<pg_log_entry_t> to_trim;
+
+ // LogEntryHandler
+ void remove(const hobject_t &hoid) {
+ to_remove.insert(hoid);
+ }
+ void rollback(const pg_log_entry_t &entry) {
+ assert(!cannot_rollback.count(entry.soid));
+ to_rollback[entry.soid].push_back(entry);
+ }
+ void cant_rollback(const pg_log_entry_t &entry) {
+ to_rollback.erase(entry.soid);
+ cannot_rollback.insert(entry.soid);
+ }
+ void trim(const pg_log_entry_t &entry) {
+ to_trim.push_back(entry);
+ }
+
+ void apply(PG *pg, ObjectStore::Transaction *t) {
+ for (map<hobject_t, list<pg_log_entry_t> >::iterator i =
+ to_rollback.begin();
+ i != to_rollback.end();
+ ++i) {
+ for (list<pg_log_entry_t>::reverse_iterator j = i->second.rbegin();
+ j != i->second.rend();
+ ++j) {
+ assert(j->mod_desc.can_rollback());
+ pg->get_pgbackend()->rollback(j->soid, j->mod_desc, t);
+ SnapRollBacker rollbacker(j->soid, pg, t);
+ j->mod_desc.visit(&rollbacker);
+ }
+ }
+ for (set<hobject_t>::iterator i = to_remove.begin();
+ i != to_remove.end();
+ ++i) {
+ pg->get_pgbackend()->rollback_create(*i, t);
+ pg->remove_snap_mapped_object(*t, *i);
+ }
+ for (list<pg_log_entry_t>::reverse_iterator i = to_trim.rbegin();
+ i != to_trim.rend();
+ ++i) {
+ LogEntryTrimmer trimmer(i->soid, pg, t);
+ i->mod_desc.visit(&trimmer);
+ }
+ }
+ };
+
+ friend struct SnapRollBacker;
+ friend struct PGLogEntryHandler;
+ friend struct LogEntryTrimmer;
+ void update_object_snap_mapping(
+ ObjectStore::Transaction *t, const hobject_t &soid,
+ const set<snapid_t> &snaps);
+ void clear_object_snap_mapping(
+ ObjectStore::Transaction *t, const hobject_t &soid);
void remove_snap_mapped_object(
ObjectStore::Transaction& t, const hobject_t& soid);
void merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from);
@@ -626,7 +726,7 @@ public:
void trim_write_ahead();
map<int, pg_info_t>::const_iterator find_best_info(const map<int, pg_info_t> &infos) const;
- bool calc_acting(int& newest_update_osd, vector<int>& want) const;
+ bool calc_acting(int& newest_update_osd, vector<int>& want, vector<int>& backfill) const;
bool choose_acting(int& newest_update_osd);
void build_might_have_unfound();
void replay_queued_ops();
@@ -839,24 +939,6 @@ public:
int active_pushes;
void repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer);
- map<int, ScrubMap *>::const_iterator _select_auth_object(
- const hobject_t &obj,
- const map<int,ScrubMap*> &maps);
-
- enum error_type {
- CLEAN,
- DEEP_ERROR,
- SHALLOW_ERROR
- };
- enum error_type _compare_scrub_objects(ScrubMap::object &auth,
- ScrubMap::object &candidate,
- ostream &errorstream);
- void _compare_scrubmaps(const map<int,ScrubMap*> &maps,
- map<hobject_t, set<int> > &missing,
- map<hobject_t, set<int> > &inconsistent,
- map<hobject_t, int> &authoritative,
- map<hobject_t, set<int> > &inconsistent_snapcolls,
- ostream &errorstream);
void scrub(ThreadPool::TPHandle &handle);
void classic_scrub(ThreadPool::TPHandle &handle);
void chunky_scrub(ThreadPool::TPHandle &handle);
@@ -866,9 +948,6 @@ public:
void scrub_finish();
void scrub_clear_state();
bool scrub_gather_replica_maps();
- void _scan_list(
- ScrubMap &map, vector<hobject_t> &ls, bool deep,
- ThreadPool::TPHandle &handle);
void _scan_snaps(ScrubMap &map);
void _request_scrub_map_classic(int replica, eversion_t version);
void _request_scrub_map(int replica, eversion_t version,
@@ -957,7 +1036,7 @@ public:
const boost::statechart::event_base &get_event() { return *evt; }
string get_desc() { return desc; }
};
- typedef std::tr1::shared_ptr<CephPeeringEvt> CephPeeringEvtRef;
+ typedef ceph::shared_ptr<CephPeeringEvt> CephPeeringEvtRef;
list<CephPeeringEvtRef> peering_queue; // op queue
list<CephPeeringEvtRef> peering_waiters;
@@ -1074,8 +1153,8 @@ public:
TrivialEvent(LocalRecoveryReserved)
TrivialEvent(RemoteRecoveryReserved)
TrivialEvent(AllRemotesReserved)
+ TrivialEvent(AllBackfillsReserved)
TrivialEvent(Recovering)
- TrivialEvent(WaitRemoteBackfillReserved)
TrivialEvent(GoClean)
TrivialEvent(AllReplicasActivated)
@@ -1281,7 +1360,6 @@ public:
struct Peering : boost::statechart::state< Peering, Primary, GetInfo >, NamedState {
std::auto_ptr< PriorSet > prior_set;
- bool flushed;
Peering(my_context ctx);
void exit();
@@ -1302,6 +1380,7 @@ public:
void exit();
const set<int> sorted_acting_set;
+ const set<int> sorted_backfill_set;
bool all_replicas_activated;
typedef boost::mpl::list <
@@ -1360,8 +1439,10 @@ public:
struct WaitRemoteBackfillReserved : boost::statechart::state< WaitRemoteBackfillReserved, Active >, NamedState {
typedef boost::mpl::list<
boost::statechart::custom_reaction< RemoteBackfillReserved >,
- boost::statechart::custom_reaction< RemoteReservationRejected >
+ boost::statechart::custom_reaction< RemoteReservationRejected >,
+ boost::statechart::transition< AllBackfillsReserved, Backfilling >
> reactions;
+ set<int>::const_iterator backfill_osd_it;
WaitRemoteBackfillReserved(my_context ctx);
void exit();
boost::statechart::result react(const RemoteBackfillReserved& evt);
@@ -1600,10 +1681,12 @@ public:
struct Incomplete : boost::statechart::state< Incomplete, Peering>, NamedState {
typedef boost::mpl::list <
- boost::statechart::custom_reaction< AdvMap >
+ boost::statechart::custom_reaction< AdvMap >,
+ boost::statechart::custom_reaction< MNotifyRec >
> reactions;
Incomplete(my_context ctx);
boost::statechart::result react(const AdvMap &advmap);
+ boost::statechart::result react(const MNotifyRec& infoevt);
void exit();
};
@@ -1702,9 +1785,18 @@ public:
__u8 info_struct_v, bool dirty_big_info, bool force_ver = false);
void write_if_dirty(ObjectStore::Transaction& t);
+ eversion_t get_next_version() const {
+ eversion_t at_version(get_osdmap()->get_epoch(),
+ pg_log.get_head().version+1);
+ assert(at_version > info.last_update);
+ assert(at_version > pg_log.get_head());
+ return at_version;
+ }
+
void add_log_entry(pg_log_entry_t& e, bufferlist& log_bl);
void append_log(
- vector<pg_log_entry_t>& logv, eversion_t trim_to, ObjectStore::Transaction &t);
+ vector<pg_log_entry_t>& logv, eversion_t trim_to, ObjectStore::Transaction &t,
+ bool transaction_applied = true);
bool check_log_for_corruption(ObjectStore *store);
void trim_peers();
@@ -1829,6 +1921,7 @@ public:
virtual bool same_for_rep_modify_since(epoch_t e) = 0;
virtual void on_role_change() = 0;
+ virtual void on_pool_change() = 0;
virtual void on_change(ObjectStore::Transaction *t) = 0;
virtual void on_activate() = 0;
virtual void on_flushed() = 0;
diff --git a/src/osd/PGBackend.cc b/src/osd/PGBackend.cc
new file mode 100644
index 0000000..80e289e
--- /dev/null
+++ b/src/osd/PGBackend.cc
@@ -0,0 +1,66 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+
+#include "PGBackend.h"
+
+// -- ObjectModDesc --
+struct RollbackVisitor : public ObjectModDesc::Visitor {
+ const hobject_t &hoid;
+ PGBackend *pg;
+ ObjectStore::Transaction t;
+ RollbackVisitor(
+ const hobject_t &hoid,
+ PGBackend *pg) : hoid(hoid), pg(pg) {}
+ void append(uint64_t old_size) {
+ ObjectStore::Transaction temp;
+ pg->rollback_append(hoid, old_size, &temp);
+ temp.append(t);
+ temp.swap(t);
+ }
+ void setattrs(map<string, boost::optional<bufferlist> > &attrs) {
+ ObjectStore::Transaction temp;
+ pg->rollback_setattrs(hoid, attrs, &temp);
+ temp.append(t);
+ temp.swap(t);
+ }
+ void rmobject(version_t old_version) {
+ ObjectStore::Transaction temp;
+ pg->rollback_stash(hoid, old_version, &temp);
+ temp.append(t);
+ temp.swap(t);
+ }
+ void create() {
+ ObjectStore::Transaction temp;
+ pg->rollback_create(hoid, &temp);
+ temp.append(t);
+ temp.swap(t);
+ }
+ void update_snaps(set<snapid_t> &snaps) {
+ // pass
+ }
+};
+
+void PGBackend::rollback(
+ const hobject_t &hoid,
+ const ObjectModDesc &desc,
+ ObjectStore::Transaction *t)
+{
+ assert(desc.can_rollback());
+ RollbackVisitor vis(hoid, this);
+ desc.visit(&vis);
+ t->append(vis.t);
+}
+
+
diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h
index 408c589..b887b14 100644
--- a/src/osd/PGBackend.h
+++ b/src/osd/PGBackend.h
@@ -15,8 +15,13 @@
#ifndef PGBACKEND_H
#define PGBACKEND_H
+#include "OSDMap.h"
+#include "PGLog.h"
+#include "osd_types.h"
+#include "common/WorkQueue.h"
#include "osd_types.h"
#include "include/Context.h"
+#include "os/ObjectStore.h"
#include <string>
/**
@@ -29,6 +34,7 @@
* 1) Handling client operations
* 2) Handling object recovery
* 3) Handling object access
+ * 4) Handling scrub, deep-scrub, repair
*/
class PGBackend {
public:
@@ -93,9 +99,12 @@
GenContext<ThreadPool::TPHandle&> *c) = 0;
virtual void send_message(int to_osd, Message *m) = 0;
- virtual void queue_transaction(ObjectStore::Transaction *t) = 0;
+ virtual void queue_transaction(
+ ObjectStore::Transaction *t,
+ OpRequestRef op = OpRequestRef()
+ ) = 0;
virtual epoch_t get_epoch() = 0;
- virtual const vector<int> &get_acting() = 0;
+ virtual const vector<int> &get_actingbackfill() = 0;
virtual std::string gen_dbg_prefix() const = 0;
virtual const map<hobject_t, set<int> > &get_missing_loc() = 0;
@@ -111,6 +120,29 @@
const hobject_t &hoid,
map<string, bufferptr> &attrs) = 0;
+ virtual void op_applied(
+ const eversion_t &applied_version) = 0;
+
+ virtual bool should_send_op(
+ int peer,
+ const hobject_t &hoid) = 0;
+
+ virtual void log_operation(
+ vector<pg_log_entry_t> &logv,
+ const eversion_t &trim_to,
+ bool transaction_applied,
+ ObjectStore::Transaction *t) = 0;
+
+ virtual void update_peer_last_complete_ondisk(
+ int fromosd,
+ eversion_t lcod) = 0;
+
+ virtual void update_last_complete_ondisk(
+ eversion_t lcod) = 0;
+
+ virtual void update_stats(
+ const pg_stat_t &stat) = 0;
+
virtual ~Listener() {}
};
Listener *parent;
@@ -173,6 +205,13 @@
RecoveryHandle *h ///< [in,out] handle to attach recovery op to
) = 0;
+ /**
+ * true if PGBackend can handle this message while inactive
+ *
+ * If it returns true, handle_message *must* also return true
+ */
+ virtual bool can_handle_while_inactive(OpRequestRef op) = 0;
+
/// gives PGBackend a crack at an incoming message
virtual bool handle_message(
OpRequestRef op ///< [in] message received
@@ -206,6 +245,155 @@
virtual ~PGBackend() {}
+ /**
+ * Client IO Interface
+ */
+ class PGTransaction {
+ public:
+ /// Write
+ virtual void touch(
+ const hobject_t &hoid ///< [in] obj to touch
+ ) = 0;
+ virtual void stash(
+ const hobject_t &hoid, ///< [in] obj to remove
+ version_t former_version ///< [in] former object version
+ ) = 0;
+ virtual void remove(
+ const hobject_t &hoid ///< [in] obj to remove
+ ) = 0;
+ virtual void setattrs(
+ const hobject_t &hoid, ///< [in] object to write
+ map<string, bufferlist> &attrs ///< [in] attrs, may be cleared
+ ) = 0;
+ virtual void setattr(
+ const hobject_t &hoid, ///< [in] object to write
+ const string &attrname, ///< [in] attr to write
+ bufferlist &bl ///< [in] val to write, may be claimed
+ ) = 0;
+ virtual void rmattr(
+ const hobject_t &hoid, ///< [in] object to write
+ const string &attrname ///< [in] attr to remove
+ ) = 0;
+ virtual void clone(
+ const hobject_t &from,
+ const hobject_t &to
+ ) = 0;
+ virtual void rename(
+ const hobject_t &from,
+ const hobject_t &to
+ ) = 0;
+
+ /// Optional, not supported on ec-pool
+ virtual void write(
+ const hobject_t &hoid, ///< [in] object to write
+ uint64_t off, ///< [in] off at which to write
+ uint64_t len, ///< [in] len to write from bl
+ bufferlist &bl ///< [in] bl to write will be claimed to len
+ ) { assert(0); }
+ virtual void omap_setkeys(
+ const hobject_t &hoid, ///< [in] object to write
+ map<string, bufferlist> &keys ///< [in] omap keys, may be cleared
+ ) { assert(0); }
+ virtual void omap_rmkeys(
+ const hobject_t &hoid, ///< [in] object to write
+ set<string> &keys ///< [in] omap keys, may be cleared
+ ) { assert(0); }
+ virtual void omap_clear(
+ const hobject_t &hoid ///< [in] object to clear omap
+ ) { assert(0); }
+ virtual void omap_setheader(
+ const hobject_t &hoid, ///< [in] object to write
+ bufferlist &header ///< [in] header
+ ) { assert(0); }
+ virtual void clone_range(
+ const hobject_t &from, ///< [in] from
+ const hobject_t &to, ///< [in] to
+ uint64_t fromoff, ///< [in] offset
+ uint64_t len, ///< [in] len
+ uint64_t tooff ///< [in] offset
+ ) { assert(0); }
+ virtual void truncate(
+ const hobject_t &hoid,
+ uint64_t off
+ ) { assert(0); }
+ virtual void zero(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len
+ ) { assert(0); }
+
+ /// Supported on all backends
+
+ /// off must be the current object size
+ virtual void append(
+ const hobject_t &hoid, ///< [in] object to write
+ uint64_t off, ///< [in] off at which to write
+ uint64_t len, ///< [in] len to write from bl
+ bufferlist &bl ///< [in] bl to write will be claimed to len
+ ) { write(hoid, off, len, bl); }
+
+ /// to_append *must* have come from the same PGBackend (same concrete type)
+ virtual void append(
+ PGTransaction *to_append ///< [in] trans to append, to_append is cleared
+ ) = 0;
+ virtual void nop() = 0;
+ virtual bool empty() const = 0;
+ virtual uint64_t get_bytes_written() const = 0;
+ virtual ~PGTransaction() {}
+ };
+ /// Get implementation specific empty transaction
+ virtual PGTransaction *get_transaction() = 0;
+
+ /// execute implementation specific transaction
+ virtual void submit_transaction(
+ const hobject_t &hoid, ///< [in] object
+ const eversion_t &at_version, ///< [in] version
+ PGTransaction *t, ///< [in] trans to execute
+ const eversion_t &trim_to, ///< [in] trim log to here
+ vector<pg_log_entry_t> &log_entries, ///< [in] log entries for t
+ Context *on_local_applied_sync, ///< [in] called when applied locally
+ Context *on_all_applied, ///< [in] called when all acked
+ Context *on_all_commit, ///< [in] called when all commit
+ tid_t tid, ///< [in] tid
+ osd_reqid_t reqid, ///< [in] reqid
+ OpRequestRef op ///< [in] op
+ ) = 0;
+
+
+ void rollback(
+ const hobject_t &hoid,
+ const ObjectModDesc &desc,
+ ObjectStore::Transaction *t);
+
+ /// Reapply old attributes
+ virtual void rollback_setattrs(
+ const hobject_t &hoid,
+ map<string, boost::optional<bufferlist> > &old_attrs,
+ ObjectStore::Transaction *t) = 0;
+
+ /// Truncate object to rollback append
+ virtual void rollback_append(
+ const hobject_t &hoid,
+ uint64_t old_size,
+ ObjectStore::Transaction *t) = 0;
+
+ /// Unstash object to rollback stash
+ virtual void rollback_stash(
+ const hobject_t &hoid,
+ version_t old_version,
+ ObjectStore::Transaction *t) = 0;
+
+ /// Delete object to rollback create
+ virtual void rollback_create(
+ const hobject_t &hoid,
+ ObjectStore::Transaction *t) = 0;
+
+ /// Trim object stashed at stashed_version
+ virtual void trim_stashed_object(
+ const hobject_t &hoid,
+ version_t stashed_version,
+ ObjectStore::Transaction *t) = 0;
+
/// List objects in collection
virtual int objects_list_partial(
const hobject_t &begin,
@@ -225,6 +413,42 @@
const hobject_t &hoid,
const string &attr,
bufferlist *out) = 0;
+
+ virtual int objects_get_attrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> *out) = 0;
+
+ virtual int objects_read_sync(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist *bl) = 0;
+
+ virtual void objects_read_async(
+ const hobject_t &hoid,
+ const list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > &to_read,
+ Context *on_complete) = 0;
+
+ virtual bool scrub_supported() { return false; }
+ virtual void be_scan_list(ScrubMap &map, const vector<hobject_t> &ls, bool deep,
+ ThreadPool::TPHandle &handle) { assert(0); }
+ virtual enum scrub_error_type be_compare_scrub_objects(
+ const ScrubMap::object &auth,
+ const ScrubMap::object &candidate,
+ ostream &errorstream) { assert(0); }
+ virtual map<int, ScrubMap *>::const_iterator be_select_auth_object(
+ const hobject_t &obj,
+ const map<int,ScrubMap*> &maps) { assert(0); }
+ virtual void be_compare_scrubmaps(const map<int,ScrubMap*> &maps,
+ map<hobject_t, set<int> > &missing,
+ map<hobject_t, set<int> > &inconsistent,
+ map<hobject_t, int> &authoritative,
+ map<hobject_t, set<int> > &invalid_snapcolls,
+ int &shallow_errors, int &deep_errors,
+ const pg_t pgid,
+ const vector<int> &acting,
+ ostream &errorstream) { assert(0); }
};
#endif
diff --git a/src/osd/PGLog.cc b/src/osd/PGLog.cc
index 9f6ca1f..e845bd6 100644
--- a/src/osd/PGLog.cc
+++ b/src/osd/PGLog.cc
@@ -18,6 +18,7 @@
#include "PGLog.h"
#include "PG.h"
#include "SnapMapper.h"
+#include "../include/unordered_map.h"
#define dout_subsys ceph_subsys_osd
@@ -40,27 +41,21 @@ void PGLog::IndexedLog::split_into(
) {
if ((i->soid.hash & mask) == child_pgid.m_seed) {
olog->log.push_back(*i);
- if (log.empty())
- tail = i->version;
} else {
log.push_back(*i);
- if (olog->empty())
- olog->tail = i->version;
}
oldlog.erase(i++);
}
- if (log.empty())
- tail = head;
-
- if (olog->empty())
- olog->tail = olog->head;
-
olog->index();
index();
+ olog->can_rollback_to = can_rollback_to;
}
-void PGLog::IndexedLog::trim(eversion_t s, set<eversion_t> *trimmed)
+void PGLog::IndexedLog::trim(
+ LogEntryHandler *handler,
+ eversion_t s,
+ set<eversion_t> *trimmed)
{
if (complete_to != log.end() &&
complete_to->version <= s) {
@@ -75,6 +70,7 @@ void PGLog::IndexedLog::trim(eversion_t s, set<eversion_t> *trimmed)
generic_dout(20) << "trim " << e << dendl;
if (trimmed)
trimmed->insert(e.version);
+ handler->trim(e);
unindex(e); // remove from index,
log.pop_front(); // from log
}
@@ -128,7 +124,10 @@ void PGLog::clear_info_log(
t->omap_rmkeys(coll_t::META_COLL, infos_oid, keys_to_remove);
}
-void PGLog::trim(eversion_t trim_to, pg_info_t &info)
+void PGLog::trim(
+ LogEntryHandler *handler,
+ eversion_t trim_to,
+ pg_info_t &info)
{
// trim?
if (trim_to > log.tail) {
@@ -140,7 +139,7 @@ void PGLog::trim(eversion_t trim_to, pg_info_t &info)
assert(trim_to <= info.last_complete);
dout(10) << "trim " << log << " to " << trim_to << dendl;
- log.trim(trim_to, &trimmed);
+ log.trim(handler, trim_to, &trimmed);
info.log_tail = log.tail;
}
}
@@ -168,68 +167,48 @@ void PGLog::proc_replica_log(ObjectStore::Transaction& t,
<< " have " << i->second.have << dendl;
}
- list<pg_log_entry_t>::const_reverse_iterator pp = olog.log.rbegin();
+ list<pg_log_entry_t> divergent;
+ list<pg_log_entry_t>::const_iterator pp = olog.log.end();
eversion_t lu(oinfo.last_update);
while (true) {
- if (pp == olog.log.rend()) {
- if (pp != olog.log.rbegin()) // no last_update adjustment if we discard nothing!
+ if (pp == olog.log.begin()) {
+ if (pp != olog.log.end()) // no last_update adjustment if we discard nothing!
lu = olog.tail;
break;
}
+ --pp;
const pg_log_entry_t& oe = *pp;
// don't continue past the tail of our log.
- if (oe.version <= log.tail)
+ if (oe.version <= log.tail) {
break;
-
- if (!log.objects.count(oe.soid)) {
- dout(10) << " had " << oe << " new dne : divergent, ignoring" << dendl;
- ++pp;
- continue;
}
-
- const pg_log_entry_t& ne = *(log.objects.find(oe.soid)->second);
- if (ne.version == oe.version) {
- dout(10) << " had " << oe << " new " << ne << " : match, stopping" << dendl;
+
+ ceph::unordered_map<hobject_t, pg_log_entry_t*>::const_iterator i =
+ log.objects.find(oe.soid);
+ if (i != log.objects.end() && i->second->version == oe.version) {
+ dout(10) << " had " << oe << " new " << *(i->second)
+ << " : match, stopping" << dendl;
lu = pp->version;
break;
}
- if (oe.soid > oinfo.last_backfill) {
- // past backfill line, don't care
- dout(10) << " had " << oe << " beyond last_backfill : skipping" << dendl;
- ++pp;
- continue;
- }
-
- if (ne.version > oe.version) {
- dout(10) << " had " << oe << " new " << ne << " : new will supercede" << dendl;
- } else {
- if (oe.is_delete()) {
- if (ne.is_delete()) {
- // old and new are delete
- dout(10) << " had " << oe << " new " << ne << " : both deletes" << dendl;
- } else {
- // old delete, new update.
- dout(10) << " had " << oe << " new " << ne << " : missing" << dendl;
- omissing.add(ne.soid, ne.version, eversion_t());
- }
- } else {
- if (ne.is_delete()) {
- // old update, new delete
- dout(10) << " had " << oe << " new " << ne << " : new will supercede" << dendl;
- omissing.rm(oe.soid, oe.version);
- } else {
- // old update, new update
- dout(10) << " had " << oe << " new " << ne << " : new will supercede" << dendl;
- omissing.revise_need(ne.soid, ne.version);
- }
- }
- }
-
- ++pp;
+ divergent.push_front(oe);
}
+ for (list<pg_log_entry_t>::iterator i = divergent.begin();
+ i != divergent.end();
+ ++i) {
+ _merge_old_entry(
+ t,
+ *i,
+ oinfo,
+ omissing,
+ olog.can_rollback_to,
+ 0,
+ 0);
+ }
+
if (lu < oinfo.last_update) {
dout(10) << " peer osd." << from << " last_update now " << lu << dendl;
oinfo.last_update = lu;
@@ -261,42 +240,73 @@ void PGLog::proc_replica_log(ObjectStore::Transaction& t,
*
* return true if entry is not divergent.
*/
-bool PGLog::merge_old_entry(ObjectStore::Transaction& t, const pg_log_entry_t& oe, const pg_info_t& info, list<hobject_t>& remove_snap)
+bool PGLog::_merge_old_entry(
+ ObjectStore::Transaction& t,
+ const pg_log_entry_t &oe,
+ const pg_info_t& info,
+ pg_missing_t &missing,
+ eversion_t olog_can_rollback_to,
+ boost::optional<pair<eversion_t, hobject_t> > *new_divergent_prior,
+ LogEntryHandler *rollbacker) const
{
if (oe.soid > info.last_backfill) {
- dout(20) << "merge_old_entry had " << oe << " : beyond last_backfill" << dendl;
+ dout(20) << "merge_old_entry had " << oe
+ << " : beyond last_backfill" << dendl;
return false;
}
- if (log.objects.count(oe.soid)) {
- pg_log_entry_t &ne = *log.objects[oe.soid]; // new(er?) entry
+ ceph::unordered_map<hobject_t, pg_log_entry_t*>::const_iterator objiter =
+ log.objects.find(oe.soid);
+ if (objiter != log.objects.end()) {
+ pg_log_entry_t &ne = *(objiter->second); // new(er?) entry
if (ne.version > oe.version) {
- dout(20) << "merge_old_entry had " << oe << " new " << ne << " : older, missing" << dendl;
- assert(ne.is_delete() || missing.is_missing(ne.soid));
+ dout(20) << "merge_old_entry had " << oe
+ << " new " << ne << " : older, missing" << dendl;
return false;
}
if (ne.version == oe.version) {
- dout(20) << "merge_old_entry had " << oe << " new " << ne << " : same" << dendl;
+ dout(20) << "merge_old_entry had " << oe
+ << " new " << ne << " : same" << dendl;
return true;
}
if (oe.is_delete()) {
if (ne.is_delete()) {
// old and new are delete
- dout(20) << "merge_old_entry had " << oe << " new " << ne << " : both deletes" << dendl;
+ dout(20) << "merge_old_entry had " << oe
+ << " new " << ne << " : both deletes" << dendl;
} else {
// old delete, new update.
- dout(20) << "merge_old_entry had " << oe << " new " << ne << " : missing" << dendl;
+ dout(20) << "merge_old_entry had " << oe
+ << " new " << ne << " : missing" << dendl;
missing.revise_need(ne.soid, ne.version);
+ if (rollbacker)
+ rollbacker->cant_rollback(oe);
}
} else {
if (ne.is_delete()) {
// old update, new delete
- dout(20) << "merge_old_entry had " << oe << " new " << ne << " : new delete supercedes" << dendl;
- missing.rm(oe.soid, oe.version);
+ dout(20) << "merge_old_entry had " << oe
+ << " new " << ne << " : new delete supercedes" << dendl;
+ if (rollbacker) {
+ rollbacker->remove(oe.soid);
+ rollbacker->cant_rollback(oe);
+ }
+ if (missing.is_missing(oe.soid))
+ missing.rm(oe.soid, oe.version);
} else {
// old update, new update
- dout(20) << "merge_old_entry had " << oe << " new " << ne << " : new item supercedes" << dendl;
- missing.revise_need(ne.soid, ne.version);
+ dout(20) << "merge_old_entry had " << oe
+ << " new " << ne << " : new item supercedes" << dendl;
+ if (oe.mod_desc.can_rollback() && oe.version > olog_can_rollback_to) {
+ dout(20) << __func__ << ": ne.version < oe.version && can rollback, "
+ << "rolling back " << oe << dendl;
+ if (rollbacker)
+ rollbacker->rollback(oe);
+ } else {
+ missing.revise_need(ne.soid, ne.version);
+ if (rollbacker)
+ rollbacker->cant_rollback(oe);
+ }
}
}
} else if (oe.op == pg_log_entry_t::CLONE) {
@@ -304,10 +314,20 @@ bool PGLog::merge_old_entry(ObjectStore::Transaction& t, const pg_log_entry_t& o
dout(20) << "merge_old_entry had " << oe
<< ", clone with no non-divergent log entries, "
<< "deleting" << dendl;
- remove_snap.push_back(oe.soid);
- if (missing.is_missing(oe.soid))
+ if (missing.is_missing(oe.soid)) {
missing.rm(oe.soid, missing.missing[oe.soid].need);
- } else if (oe.prior_version > info.log_tail) {
+ }
+
+ if (oe.mod_desc.can_rollback() && oe.version > olog_can_rollback_to) {
+ dout(20) << __func__ << ": rolling back " << oe << dendl;
+ if (rollbacker)
+ rollbacker->rollback(oe);
+ } else {
+ dout(20) << __func__ << ": had " << oe << " deleting" << dendl;
+ if (rollbacker)
+ rollbacker->remove(oe.soid);
+ }
+ } else if (oe.prior_version > info.log_tail && missing.is_missing(oe.soid)) {
/**
* oe.prior_version is a previously divergent log entry
* oe.soid must have already been handled and the missing
@@ -318,17 +338,27 @@ bool PGLog::merge_old_entry(ObjectStore::Transaction& t, const pg_log_entry_t& o
<< " oe.soid " << oe.soid
<< " must already have been merged" << dendl;
} else {
- if (!oe.is_delete()) {
- dout(20) << "merge_old_entry had " << oe << " deleting" << dendl;
- remove_snap.push_back(oe.soid);
- }
- dout(20) << "merge_old_entry had " << oe << " updating missing to "
- << oe.prior_version << dendl;
- if (oe.prior_version > eversion_t()) {
- add_divergent_prior(oe.prior_version, oe.soid);
- missing.revise_need(oe.soid, oe.prior_version);
- } else if (missing.is_missing(oe.soid)) {
- missing.rm(oe.soid, missing.missing[oe.soid].need);
+ if (oe.mod_desc.can_rollback() && oe.version > olog_can_rollback_to) {
+ dout(20) << __func__ << ": rolling back " << oe << dendl;
+ if (rollbacker)
+ rollbacker->rollback(oe);
+ } else {
+ if (!oe.is_delete()) {
+ if (rollbacker)
+ rollbacker->remove(oe.soid);
+ dout(20) << __func__ << ": had " << oe << " deleting" << dendl;
+ }
+ dout(20) << "merge_old_entry had " << oe << " updating missing to "
+ << oe.prior_version << dendl;
+ if (oe.prior_version > eversion_t()) {
+ if (new_divergent_prior)
+ *new_divergent_prior = make_pair(oe.prior_version, oe.soid);
+ missing.revise_need(oe.soid, oe.prior_version);
+ if (rollbacker)
+ rollbacker->cant_rollback(oe);
+ } else if (missing.is_missing(oe.soid)) {
+ missing.rm(oe.soid, missing.missing[oe.soid].need);
+ }
}
}
return false;
@@ -344,11 +374,11 @@ bool PGLog::merge_old_entry(ObjectStore::Transaction& t, const pg_log_entry_t& o
* @param newhead new head to rewind to
*/
void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead,
- pg_info_t &info, list<hobject_t>& remove_snap,
- bool &dirty_info, bool &dirty_big_info)
+ pg_info_t &info, LogEntryHandler *rollbacker,
+ bool &dirty_info, bool &dirty_big_info)
{
dout(10) << "rewind_divergent_log truncate divergent future " << newhead << dendl;
- assert(newhead > log.tail);
+ assert(newhead >= log.tail);
list<pg_log_entry_t>::iterator p = log.log.end();
list<pg_log_entry_t> divergent;
@@ -375,8 +405,15 @@ void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead
info.last_complete = newhead;
log.index();
- for (list<pg_log_entry_t>::iterator d = divergent.begin(); d != divergent.end(); ++d)
- merge_old_entry(t, *d, info, remove_snap);
+ for (list<pg_log_entry_t>::iterator d = divergent.begin();
+ d != divergent.end();
+ ++d) {
+ merge_old_entry(t, *d, info, rollbacker);
+ rollbacker->trim(*d);
+ }
+
+ if (info.last_update < log.can_rollback_to)
+ log.can_rollback_to = info.last_update;
dirty_info = true;
dirty_big_info = true;
@@ -384,7 +421,7 @@ void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead
void PGLog::merge_log(ObjectStore::Transaction& t,
pg_info_t &oinfo, pg_log_t &olog, int fromosd,
- pg_info_t &info, list<hobject_t>& remove_snap,
+ pg_info_t &info, LogEntryHandler *rollbacker,
bool &dirty_info, bool &dirty_big_info)
{
dout(10) << "merge_log " << olog << " from osd." << fromosd
@@ -441,7 +478,7 @@ void PGLog::merge_log(ObjectStore::Transaction& t,
// do we have divergent entries to throw out?
if (olog.head < log.head) {
- rewind_divergent_log(t, olog.head, info, remove_snap, dirty_info, dirty_big_info);
+ rewind_divergent_log(t, olog.head, info, rollbacker, dirty_info, dirty_big_info);
changed = true;
}
@@ -475,7 +512,7 @@ void PGLog::merge_log(ObjectStore::Transaction& t,
if (ne.soid <= info.last_backfill) {
missing.add_next_event(ne);
if (ne.is_delete())
- remove_snap.push_back(ne.soid);
+ rollbacker->remove(ne.soid);
}
}
@@ -504,15 +541,21 @@ void PGLog::merge_log(ObjectStore::Transaction& t,
log.index();
info.last_update = log.head = olog.head;
+
info.last_user_version = oinfo.last_user_version;
info.purged_snaps = oinfo.purged_snaps;
// process divergent items
- if (!divergent.empty()) {
- for (list<pg_log_entry_t>::iterator d = divergent.begin(); d != divergent.end(); ++d)
- merge_old_entry(t, *d, info, remove_snap);
+ for (list<pg_log_entry_t>::iterator d = divergent.begin();
+ d != divergent.end();
+ ++d) {
+ merge_old_entry(t, *d, info, rollbacker);
+ rollbacker->trim(*d);
}
+ // We cannot rollback into the new log entries
+ log.can_rollback_to = log.head;
+
changed = true;
}
@@ -632,6 +675,7 @@ void PGLog::_write_log(
//dout(10) << "write_log: writing divergent_priors" << dendl;
::encode(divergent_priors, keys["divergent_priors"]);
}
+ ::encode(log.can_rollback_to, keys["can_rollback_to"]);
t.omap_rmkeys(coll_t::META_COLL, log_oid, to_remove);
t.omap_setkeys(coll_t::META_COLL, log_oid, keys);
@@ -656,6 +700,8 @@ bool PGLog::read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
rewrite_log = true;
} else {
log.tail = info.log_tail;
+ // will get overridden below if it had been recorded
+ log.can_rollback_to = info.last_update;
ObjectMap::ObjectMapIterator p = store->get_omap_iterator(coll_t::META_COLL, log_oid);
if (p) for (p->seek_to_first(); p->valid() ; p->next()) {
bufferlist bl = p->value();//Copy bufferlist before creating iterator
@@ -663,6 +709,10 @@ bool PGLog::read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
if (p->key() == "divergent_priors") {
::decode(divergent_priors, bp);
dout(20) << "read_log " << divergent_priors.size() << " divergent_priors" << dendl;
+ } else if (p->key() == "can_rollback_to") {
+ bufferlist bl = p->value();
+ bufferlist::iterator bp = bl.begin();
+ ::decode(log.can_rollback_to, bp);
} else {
pg_log_entry_t e;
e.decode_with_checksum(bp);
diff --git a/src/osd/PGLog.h b/src/osd/PGLog.h
index 792191b..beafdbf 100644
--- a/src/osd/PGLog.h
+++ b/src/osd/PGLog.h
@@ -27,6 +27,17 @@ using namespace std;
struct PGLog {
////////////////////////////// sub classes //////////////////////////////
+ struct LogEntryHandler {
+ virtual void rollback(
+ const pg_log_entry_t &entry) = 0;
+ virtual void remove(
+ const hobject_t &hoid) = 0;
+ virtual void trim(
+ const pg_log_entry_t &entry) = 0;
+ virtual void cant_rollback(
+ const pg_log_entry_t &entry) = 0;
+ virtual ~LogEntryHandler() {}
+ };
/* Exceptions */
class read_log_error : public buffer::error {
@@ -46,8 +57,8 @@ struct PGLog {
* plus some methods to manipulate it all.
*/
struct IndexedLog : public pg_log_t {
- hash_map<hobject_t,pg_log_entry_t*> objects; // ptrs into log. be careful!
- hash_map<osd_reqid_t,pg_log_entry_t*> caller_ops;
+ ceph::unordered_map<hobject_t,pg_log_entry_t*> objects; // ptrs into log. be careful!
+ ceph::unordered_map<osd_reqid_t,pg_log_entry_t*> caller_ops;
// recovery pointers
list<pg_log_entry_t>::iterator complete_to; // not inclusive of referenced item
@@ -85,7 +96,7 @@ struct PGLog {
return caller_ops.count(r);
}
const pg_log_entry_t *get_request(const osd_reqid_t &r) const {
- hash_map<osd_reqid_t,pg_log_entry_t*>::const_iterator p = caller_ops.find(r);
+ ceph::unordered_map<osd_reqid_t,pg_log_entry_t*>::const_iterator p = caller_ops.find(r);
if (p == caller_ops.end())
return NULL;
return p->second;
@@ -142,7 +153,10 @@ struct PGLog {
caller_ops[e.reqid] = &(log.back());
}
- void trim(eversion_t s, set<eversion_t> *trimmed);
+ void trim(
+ LogEntryHandler *handler,
+ eversion_t s,
+ set<eversion_t> *trimmed);
ostream& print(ostream& out) const;
};
@@ -304,7 +318,14 @@ public:
const hobject_t &log_oid,
ObjectStore::Transaction *t);
- void trim(eversion_t trim_to, pg_info_t &info);
+ void trim(
+ LogEntryHandler *handler,
+ eversion_t trim_to,
+ pg_info_t &info);
+
+ void clear_can_rollback_to() {
+ log.can_rollback_to = log.head;
+ }
//////////////////// get or set log & missing ////////////////////
@@ -342,6 +363,9 @@ public:
log.complete_to++;
}
}
+
+ if (log.can_rollback_to < v)
+ log.can_rollback_to = v;
}
void activate_not_complete(pg_info_t &info) {
@@ -364,16 +388,39 @@ public:
pg_missing_t& omissing, int from) const;
protected:
- bool merge_old_entry(ObjectStore::Transaction& t, const pg_log_entry_t& oe,
- const pg_info_t& info, list<hobject_t>& remove_snap);
+ bool _merge_old_entry(
+ ObjectStore::Transaction& t,
+ const pg_log_entry_t &oe,
+ const pg_info_t& info,
+ pg_missing_t &missing,
+ eversion_t olog_can_rollback_to,
+ boost::optional<pair<eversion_t, hobject_t> > *new_divergent_prior,
+ LogEntryHandler *rollbacker) const;
+ bool merge_old_entry(
+ ObjectStore::Transaction& t,
+ const pg_log_entry_t& oe,
+ const pg_info_t& info,
+ LogEntryHandler *rollbacker) {
+ boost::optional<pair<eversion_t, hobject_t> > new_divergent_prior;
+ bool merged = _merge_old_entry(
+ t, oe, info, missing,
+ log.can_rollback_to,
+ &new_divergent_prior,
+ rollbacker);
+ if (new_divergent_prior)
+ add_divergent_prior(
+ (*new_divergent_prior).first,
+ (*new_divergent_prior).second);
+ return merged;
+ }
public:
void rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead,
- pg_info_t &info, list<hobject_t>& remove_snap,
+ pg_info_t &info, LogEntryHandler *rollbacker,
bool &dirty_info, bool &dirty_big_info);
void merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, int from,
- pg_info_t &info, list<hobject_t>& remove_snap,
- bool &dirty_info, bool &dirty_big_info);
+ pg_info_t &info, LogEntryHandler *rollbacker,
+ bool &dirty_info, bool &dirty_big_info);
void write_log(ObjectStore::Transaction& t, const hobject_t &log_oid);
diff --git a/src/osd/ReplicatedBackend.cc b/src/osd/ReplicatedBackend.cc
index 9529e15..6d7d2e5 100644
--- a/src/osd/ReplicatedBackend.cc
+++ b/src/osd/ReplicatedBackend.cc
@@ -11,7 +11,9 @@
* Foundation. See file COPYING.
*
*/
+#include "common/errno.h"
#include "ReplicatedBackend.h"
+#include "messages/MOSDOp.h"
#include "messages/MOSDSubOp.h"
#include "messages/MOSDSubOpReply.h"
#include "messages/MOSDPGPush.h"
@@ -91,6 +93,31 @@ void ReplicatedBackend::check_recovery_sources(const OSDMapRef osdmap)
}
}
+bool ReplicatedBackend::can_handle_while_inactive(OpRequestRef op)
+{
+ dout(10) << __func__ << ": " << op << dendl;
+ switch (op->get_req()->get_type()) {
+ case MSG_OSD_PG_PULL:
+ return true;
+ case MSG_OSD_SUBOP: {
+ MOSDSubOp *m = static_cast<MOSDSubOp*>(op->get_req());
+ if (m->ops.size() >= 1) {
+ OSDOp *first = &m->ops[0];
+ switch (first->op.op) {
+ case CEPH_OSD_OP_PULL:
+ return true;
+ default:
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+ default:
+ return false;
+ }
+}
+
bool ReplicatedBackend::handle_message(
OpRequestRef op
)
@@ -98,7 +125,6 @@ bool ReplicatedBackend::handle_message(
dout(10) << __func__ << ": " << op << dendl;
switch (op->get_req()->get_type()) {
case MSG_OSD_PG_PUSH:
- // TODOXXX: needs to be active possibly
do_push(op);
return true;
@@ -119,12 +145,13 @@ bool ReplicatedBackend::handle_message(
sub_op_pull(op);
return true;
case CEPH_OSD_OP_PUSH:
- // TODOXXX: needs to be active possibly
sub_op_push(op);
return true;
default:
break;
}
+ } else {
+ sub_op_modify(op);
}
break;
}
@@ -139,6 +166,8 @@ bool ReplicatedBackend::handle_message(
sub_op_push_reply(op);
return true;
}
+ } else {
+ sub_op_modify_reply(op);
}
break;
}
@@ -169,6 +198,14 @@ void ReplicatedBackend::on_change(ObjectStore::Transaction *t)
t->remove(get_temp_coll(t), *i);
}
temp_contents.clear();
+ for (map<tid_t, InProgressOp>::iterator i = in_progress_ops.begin();
+ i != in_progress_ops.end();
+ in_progress_ops.erase(i++)) {
+ if (i->second.on_commit)
+ delete i->second.on_commit;
+ if (i->second.on_applied)
+ delete i->second.on_applied;
+ }
clear_state();
}
@@ -204,25 +241,33 @@ int ReplicatedBackend::objects_list_partial(
vector<hobject_t> *ls,
hobject_t *next)
{
- vector<ghobject_t> objects;
- ghobject_t _next;
- int r = osd->store->collection_list_partial(
- coll,
- begin,
- min,
- max,
- seq,
- &objects,
- &_next);
- ls->reserve(objects.size());
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- assert(i->is_degenerate());
- ls->push_back(i->hobj);
+ assert(ls);
+ ghobject_t _next(begin);
+ ls->reserve(max);
+ int r = 0;
+ while (!_next.is_max() && ls->size() < (unsigned)min) {
+ vector<ghobject_t> objects;
+ int r = osd->store->collection_list_partial(
+ coll,
+ _next,
+ min - ls->size(),
+ max - ls->size(),
+ seq,
+ &objects,
+ &_next);
+ if (r != 0)
+ break;
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ assert(i->is_no_shard());
+ if (i->is_no_gen()) {
+ ls->push_back(i->hobj);
+ }
+ }
}
- assert(_next.is_degenerate());
- *next = _next.hobj;
+ if (r == 0)
+ *next = _next.hobj;
return r;
}
@@ -232,6 +277,7 @@ int ReplicatedBackend::objects_list_range(
snapid_t seq,
vector<hobject_t> *ls)
{
+ assert(ls);
vector<ghobject_t> objects;
int r = osd->store->collection_list_range(
coll,
@@ -243,8 +289,10 @@ int ReplicatedBackend::objects_list_range(
for (vector<ghobject_t>::iterator i = objects.begin();
i != objects.end();
++i) {
- assert(i->is_degenerate());
- ls->push_back(i->hobj);
+ assert(i->is_no_shard());
+ if (i->is_no_gen()) {
+ ls->push_back(i->hobj);
+ }
}
return r;
}
@@ -266,3 +314,766 @@ int ReplicatedBackend::objects_get_attr(
}
return r;
}
+
+int ReplicatedBackend::objects_get_attrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> *out)
+{
+ return osd->store->getattrs(
+ coll,
+ hoid,
+ *out);
+}
+
+int ReplicatedBackend::objects_read_sync(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist *bl)
+{
+ return osd->store->read(coll, hoid, off, len, *bl);
+}
+
+struct AsyncReadCallback : public GenContext<ThreadPool::TPHandle&> {
+ int r;
+ Context *c;
+ AsyncReadCallback(int r, Context *c) : r(r), c(c) {}
+ void finish(ThreadPool::TPHandle&) {
+ c->complete(r);
+ c = NULL;
+ }
+ ~AsyncReadCallback() {
+ delete c;
+ }
+};
+void ReplicatedBackend::objects_read_async(
+ const hobject_t &hoid,
+ const list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > &to_read,
+ Context *on_complete)
+{
+ int r = 0;
+ for (list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > >::const_iterator i =
+ to_read.begin();
+ i != to_read.end() && r >= 0;
+ ++i) {
+ int _r = osd->store->read(coll, hoid, i->first.first,
+ i->first.second, *(i->second.first));
+ if (i->second.second) {
+ osd->gen_wq.queue(
+ get_parent()->bless_gencontext(
+ new AsyncReadCallback(_r, i->second.second)));
+ }
+ if (_r < 0)
+ r = _r;
+ }
+ osd->gen_wq.queue(
+ get_parent()->bless_gencontext(
+ new AsyncReadCallback(r, on_complete)));
+}
+
+
+class RPGTransaction : public PGBackend::PGTransaction {
+ coll_t coll;
+ coll_t temp_coll;
+ set<hobject_t> temp_added;
+ set<hobject_t> temp_cleared;
+ ObjectStore::Transaction *t;
+ const coll_t &get_coll_ct(const hobject_t &hoid) {
+ if (hoid.is_temp()) {
+ temp_cleared.erase(hoid);
+ temp_added.insert(hoid);
+ }
+ return get_coll(hoid);
+ }
+ const coll_t &get_coll_rm(const hobject_t &hoid) {
+ if (hoid.is_temp()) {
+ temp_added.erase(hoid);
+ temp_cleared.insert(hoid);
+ }
+ return get_coll(hoid);
+ }
+ const coll_t &get_coll(const hobject_t &hoid) {
+ if (hoid.is_temp())
+ return temp_coll;
+ else
+ return coll;
+ }
+public:
+ RPGTransaction(coll_t coll, coll_t temp_coll)
+ : coll(coll), temp_coll(temp_coll), t(new ObjectStore::Transaction)
+ {}
+
+ /// Yields ownership of contained transaction
+ ObjectStore::Transaction *get_transaction() {
+ ObjectStore::Transaction *_t = t;
+ t = 0;
+ return _t;
+ }
+ const set<hobject_t> &get_temp_added() {
+ return temp_added;
+ }
+ const set<hobject_t> &get_temp_cleared() {
+ return temp_cleared;
+ }
+
+ void write(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist &bl
+ ) {
+ t->write(get_coll_ct(hoid), hoid, off, len, bl);
+ }
+ void remove(
+ const hobject_t &hoid
+ ) {
+ t->remove(get_coll_rm(hoid), hoid);
+ }
+ void stash(
+ const hobject_t &hoid,
+ version_t former_version) {
+ t->collection_move_rename(
+ coll, hoid, coll,
+ ghobject_t(hoid, former_version, 0));
+ }
+ void setattrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> &attrs
+ ) {
+ t->setattrs(get_coll(hoid), hoid, attrs);
+ }
+ void setattr(
+ const hobject_t &hoid,
+ const string &attrname,
+ bufferlist &bl
+ ) {
+ t->setattr(get_coll(hoid), hoid, attrname, bl);
+ }
+ void rmattr(
+ const hobject_t &hoid,
+ const string &attrname
+ ) {
+ t->rmattr(get_coll(hoid), hoid, attrname);
+ }
+ void omap_setkeys(
+ const hobject_t &hoid,
+ map<string, bufferlist> &keys
+ ) {
+ return t->omap_setkeys(get_coll(hoid), hoid, keys);
+ }
+ void omap_rmkeys(
+ const hobject_t &hoid,
+ set<string> &keys
+ ) {
+ t->omap_rmkeys(get_coll(hoid), hoid, keys);
+ }
+ void omap_clear(
+ const hobject_t &hoid
+ ) {
+ t->omap_clear(get_coll(hoid), hoid);
+ }
+ void omap_setheader(
+ const hobject_t &hoid,
+ bufferlist &header
+ ) {
+ t->omap_setheader(get_coll(hoid), hoid, header);
+ }
+ void clone_range(
+ const hobject_t &from,
+ const hobject_t &to,
+ uint64_t fromoff,
+ uint64_t len,
+ uint64_t tooff
+ ) {
+ assert(get_coll(from) == get_coll_ct(to) && get_coll(from) == coll);
+ t->clone_range(coll, from, to, fromoff, len, tooff);
+ }
+ void clone(
+ const hobject_t &from,
+ const hobject_t &to
+ ) {
+ assert(get_coll(from) == get_coll_ct(to) && get_coll(from) == coll);
+ t->clone(coll, from, to);
+ }
+ void rename(
+ const hobject_t &from,
+ const hobject_t &to
+ ) {
+ t->collection_move_rename(
+ get_coll_rm(from),
+ from,
+ get_coll_ct(to),
+ to);
+ }
+
+ void touch(
+ const hobject_t &hoid
+ ) {
+ t->touch(get_coll_ct(hoid), hoid);
+ }
+
+ void truncate(
+ const hobject_t &hoid,
+ uint64_t off
+ ) {
+ t->truncate(get_coll(hoid), hoid, off);
+ }
+ void zero(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len
+ ) {
+ t->zero(get_coll(hoid), hoid, off, len);
+ }
+
+ void append(
+ PGTransaction *_to_append
+ ) {
+ RPGTransaction *to_append = dynamic_cast<RPGTransaction*>(_to_append);
+ t->append(*(to_append->t));
+ for (set<hobject_t>::iterator i = to_append->temp_added.begin();
+ i != to_append->temp_added.end();
+ ++i) {
+ temp_cleared.erase(*i);
+ temp_added.insert(*i);
+ }
+ for (set<hobject_t>::iterator i = to_append->temp_cleared.begin();
+ i != to_append->temp_cleared.end();
+ ++i) {
+ temp_added.erase(*i);
+ temp_cleared.insert(*i);
+ }
+ }
+ void nop() {
+ t->nop();
+ }
+ bool empty() const {
+ return t->empty();
+ }
+ uint64_t get_bytes_written() const {
+ return t->get_encoded_bytes();
+ }
+ ~RPGTransaction() { delete t; }
+};
+
+PGBackend::PGTransaction *ReplicatedBackend::get_transaction()
+{
+ return new RPGTransaction(coll, get_temp_coll());
+}
+
+class C_OSD_OnOpCommit : public Context {
+ ReplicatedBackend *pg;
+ ReplicatedBackend::InProgressOp *op;
+public:
+ C_OSD_OnOpCommit(ReplicatedBackend *pg, ReplicatedBackend::InProgressOp *op)
+ : pg(pg), op(op) {}
+ void finish(int) {
+ pg->op_commit(op);
+ }
+};
+
+class C_OSD_OnOpApplied : public Context {
+ ReplicatedBackend *pg;
+ ReplicatedBackend::InProgressOp *op;
+public:
+ C_OSD_OnOpApplied(ReplicatedBackend *pg, ReplicatedBackend::InProgressOp *op)
+ : pg(pg), op(op) {}
+ void finish(int) {
+ pg->op_applied(op);
+ }
+};
+
+void ReplicatedBackend::submit_transaction(
+ const hobject_t &soid,
+ const eversion_t &at_version,
+ PGTransaction *_t,
+ const eversion_t &trim_to,
+ vector<pg_log_entry_t> &log_entries,
+ Context *on_local_applied_sync,
+ Context *on_all_acked,
+ Context *on_all_commit,
+ tid_t tid,
+ osd_reqid_t reqid,
+ OpRequestRef orig_op)
+{
+ RPGTransaction *t = dynamic_cast<RPGTransaction*>(_t);
+ ObjectStore::Transaction *op_t = t->get_transaction();
+
+ assert(t->get_temp_added().size() <= 1);
+ assert(t->get_temp_cleared().size() <= 1);
+
+ assert(!in_progress_ops.count(tid));
+ InProgressOp &op = in_progress_ops.insert(
+ make_pair(
+ tid,
+ InProgressOp(
+ tid, on_all_commit, on_all_acked,
+ orig_op, at_version)
+ )
+ ).first->second;
+
+ issue_op(
+ soid,
+ at_version,
+ tid,
+ reqid,
+ trim_to,
+ t->get_temp_added().size() ? *(t->get_temp_added().begin()) : hobject_t(),
+ t->get_temp_cleared().size() ?
+ *(t->get_temp_cleared().begin()) :hobject_t(),
+ log_entries,
+ &op,
+ op_t);
+
+ // add myself to gather set
+ op.waiting_for_applied.insert(osd->whoami);
+ op.waiting_for_commit.insert(osd->whoami);
+
+ ObjectStore::Transaction local_t;
+ if (t->get_temp_added().size()) {
+ get_temp_coll(&local_t);
+ temp_contents.insert(t->get_temp_added().begin(), t->get_temp_added().end());
+ }
+ for (set<hobject_t>::const_iterator i = t->get_temp_cleared().begin();
+ i != t->get_temp_cleared().end();
+ ++i) {
+ temp_contents.erase(*i);
+ }
+ parent->log_operation(log_entries, trim_to, true, &local_t);
+ local_t.append(*op_t);
+ local_t.swap(*op_t);
+
+ op_t->register_on_applied_sync(on_local_applied_sync);
+ op_t->register_on_applied(
+ parent->bless_context(
+ new C_OSD_OnOpApplied(this, &op)));
+ op_t->register_on_applied(
+ new ObjectStore::C_DeleteTransaction(op_t));
+ op_t->register_on_commit(
+ parent->bless_context(
+ new C_OSD_OnOpCommit(this, &op)));
+
+ parent->queue_transaction(op_t, op.op);
+ delete t;
+}
+
+void ReplicatedBackend::op_applied(
+ InProgressOp *op)
+{
+ dout(10) << __func__ << ": " << op->tid << dendl;
+ if (op->op)
+ op->op->mark_event("op_applied");
+
+ op->waiting_for_applied.erase(osd->whoami);
+ parent->op_applied(op->v);
+
+ if (op->waiting_for_applied.empty()) {
+ op->on_applied->complete(0);
+ op->on_applied = 0;
+ }
+ if (op->done()) {
+ assert(!op->on_commit && !op->on_applied);
+ in_progress_ops.erase(op->tid);
+ }
+}
+
+void ReplicatedBackend::op_commit(
+ InProgressOp *op)
+{
+ dout(10) << __func__ << ": " << op->tid << dendl;
+ if (op->op)
+ op->op->mark_event("op_commit");
+
+ op->waiting_for_commit.erase(osd->whoami);
+
+ if (op->waiting_for_commit.empty()) {
+ op->on_commit->complete(0);
+ op->on_commit = 0;
+ }
+ if (op->done()) {
+ assert(!op->on_commit && !op->on_applied);
+ in_progress_ops.erase(op->tid);
+ }
+}
+
+void ReplicatedBackend::sub_op_modify_reply(OpRequestRef op)
+{
+ MOSDSubOpReply *r = static_cast<MOSDSubOpReply*>(op->get_req());
+ assert(r->get_header().type == MSG_OSD_SUBOPREPLY);
+
+ op->mark_started();
+
+ // must be replication.
+ tid_t rep_tid = r->get_tid();
+ int fromosd = r->get_source().num();
+
+ if (in_progress_ops.count(rep_tid)) {
+ map<tid_t, InProgressOp>::iterator iter =
+ in_progress_ops.find(rep_tid);
+ InProgressOp &ip_op = iter->second;
+ MOSDOp *m = NULL;
+ if (ip_op.op)
+ m = static_cast<MOSDOp *>(ip_op.op->get_req());
+
+ if (m)
+ dout(7) << __func__ << ": tid " << ip_op.tid << " op " //<< *m
+ << " ack_type " << r->ack_type
+ << " from osd." << fromosd
+ << dendl;
+ else
+ dout(7) << __func__ << ": tid " << ip_op.tid << " (no op) "
+ << " ack_type " << r->ack_type
+ << " from osd." << fromosd
+ << dendl;
+
+ // oh, good.
+
+ if (r->ack_type & CEPH_OSD_FLAG_ONDISK) {
+ assert(ip_op.waiting_for_commit.count(fromosd));
+ ip_op.waiting_for_commit.erase(fromosd);
+ if (ip_op.op)
+ ip_op.op->mark_event("sub_op_commit_rec");
+ } else {
+ assert(ip_op.waiting_for_applied.count(fromosd));
+ if (ip_op.op)
+ ip_op.op->mark_event("sub_op_applied_rec");
+ }
+ ip_op.waiting_for_applied.erase(fromosd);
+
+ parent->update_peer_last_complete_ondisk(
+ fromosd,
+ r->get_last_complete_ondisk());
+
+ if (ip_op.waiting_for_applied.empty() &&
+ ip_op.on_applied) {
+ ip_op.on_applied->complete(0);
+ ip_op.on_applied = 0;
+ }
+ if (ip_op.waiting_for_commit.empty() &&
+ ip_op.on_commit) {
+ ip_op.on_commit->complete(0);
+ ip_op.on_commit= 0;
+ }
+ if (ip_op.done()) {
+ assert(!ip_op.on_commit && !ip_op.on_applied);
+ in_progress_ops.erase(iter);
+ }
+ }
+}
+
+/*
+ * pg lock may or may not be held
+ */
+void ReplicatedBackend::be_scan_list(
+ ScrubMap &map, const vector<hobject_t> &ls, bool deep,
+ ThreadPool::TPHandle &handle)
+{
+ dout(10) << "_scan_list scanning " << ls.size() << " objects"
+ << (deep ? " deeply" : "") << dendl;
+ int i = 0;
+ for (vector<hobject_t>::const_iterator p = ls.begin();
+ p != ls.end();
+ ++p, i++) {
+ handle.reset_tp_timeout();
+ hobject_t poid = *p;
+
+ struct stat st;
+ int r = osd->store->stat(coll, poid, &st, true);
+ if (r == 0) {
+ ScrubMap::object &o = map.objects[poid];
+ o.size = st.st_size;
+ assert(!o.negative);
+ osd->store->getattrs(coll, poid, o.attrs);
+
+ // calculate the CRC32 on deep scrubs
+ if (deep) {
+ bufferhash h, oh;
+ bufferlist bl, hdrbl;
+ int r;
+ __u64 pos = 0;
+ while ( (r = osd->store->read(coll, poid, pos,
+ cct->_conf->osd_deep_scrub_stride, bl,
+ true)) > 0) {
+ handle.reset_tp_timeout();
+ h << bl;
+ pos += bl.length();
+ bl.clear();
+ }
+ if (r == -EIO) {
+ dout(25) << "_scan_list " << poid << " got "
+ << r << " on read, read_error" << dendl;
+ o.read_error = true;
+ }
+ o.digest = h.digest();
+ o.digest_present = true;
+
+ bl.clear();
+ r = osd->store->omap_get_header(coll, poid, &hdrbl, true);
+ if (r == 0) {
+ dout(25) << "CRC header " << string(hdrbl.c_str(), hdrbl.length())
+ << dendl;
+ ::encode(hdrbl, bl);
+ oh << bl;
+ bl.clear();
+ } else if (r == -EIO) {
+ dout(25) << "_scan_list " << poid << " got "
+ << r << " on omap header read, read_error" << dendl;
+ o.read_error = true;
+ }
+
+ ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(
+ coll, poid);
+ assert(iter);
+ uint64_t keys_scanned = 0;
+ for (iter->seek_to_first(); iter->valid() ; iter->next()) {
+ if (cct->_conf->osd_scan_list_ping_tp_interval &&
+ (keys_scanned % cct->_conf->osd_scan_list_ping_tp_interval == 0)) {
+ handle.reset_tp_timeout();
+ }
+ ++keys_scanned;
+
+ dout(25) << "CRC key " << iter->key() << " value "
+ << string(iter->value().c_str(), iter->value().length()) << dendl;
+
+ ::encode(iter->key(), bl);
+ ::encode(iter->value(), bl);
+ oh << bl;
+ bl.clear();
+ }
+ if (iter->status() == -EIO) {
+ dout(25) << "_scan_list " << poid << " got "
+ << r << " on omap scan, read_error" << dendl;
+ o.read_error = true;
+ break;
+ }
+
+ //Store final calculated CRC32 of omap header & key/values
+ o.omap_digest = oh.digest();
+ o.omap_digest_present = true;
+ }
+
+ dout(25) << "_scan_list " << poid << dendl;
+ } else if (r == -ENOENT) {
+ dout(25) << "_scan_list " << poid << " got " << r << ", skipping" << dendl;
+ } else if (r == -EIO) {
+ dout(25) << "_scan_list " << poid << " got " << r << ", read_error" << dendl;
+ ScrubMap::object &o = map.objects[poid];
+ o.read_error = true;
+ } else {
+ derr << "_scan_list got: " << cpp_strerror(r) << dendl;
+ assert(0);
+ }
+ }
+}
+
+enum scrub_error_type ReplicatedBackend::be_compare_scrub_objects(
+ const ScrubMap::object &auth,
+ const ScrubMap::object &candidate,
+ ostream &errorstream)
+{
+ enum scrub_error_type error = CLEAN;
+ if (candidate.read_error) {
+ // This can occur on stat() of a shallow scrub, but in that case size will
+ // be invalid, and this will be over-ridden below.
+ error = DEEP_ERROR;
+ errorstream << "candidate had a read error";
+ }
+ if (auth.digest_present && candidate.digest_present) {
+ if (auth.digest != candidate.digest) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = DEEP_ERROR;
+
+ errorstream << "digest " << candidate.digest
+ << " != known digest " << auth.digest;
+ }
+ }
+ if (auth.omap_digest_present && candidate.omap_digest_present) {
+ if (auth.omap_digest != candidate.omap_digest) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = DEEP_ERROR;
+
+ errorstream << "omap_digest " << candidate.omap_digest
+ << " != known omap_digest " << auth.omap_digest;
+ }
+ }
+ // Shallow error takes precendence because this will be seen by
+ // both types of scrubs.
+ if (auth.size != candidate.size) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "size " << candidate.size
+ << " != known size " << auth.size;
+ }
+ for (map<string,bufferptr>::const_iterator i = auth.attrs.begin();
+ i != auth.attrs.end();
+ ++i) {
+ if (!candidate.attrs.count(i->first)) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "missing attr " << i->first;
+ } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "attr value mismatch " << i->first;
+ }
+ }
+ for (map<string,bufferptr>::const_iterator i = candidate.attrs.begin();
+ i != candidate.attrs.end();
+ ++i) {
+ if (!auth.attrs.count(i->first)) {
+ if (error != CLEAN)
+ errorstream << ", ";
+ error = SHALLOW_ERROR;
+ errorstream << "extra attr " << i->first;
+ }
+ }
+ return error;
+}
+
+map<int, ScrubMap *>::const_iterator ReplicatedBackend::be_select_auth_object(
+ const hobject_t &obj,
+ const map<int,ScrubMap*> &maps)
+{
+ map<int, ScrubMap *>::const_iterator auth = maps.end();
+ for (map<int, ScrubMap *>::const_iterator j = maps.begin();
+ j != maps.end();
+ ++j) {
+ map<hobject_t, ScrubMap::object>::iterator i =
+ j->second->objects.find(obj);
+ if (i == j->second->objects.end()) {
+ continue;
+ }
+ if (auth == maps.end()) {
+ // Something is better than nothing
+ // TODO: something is NOT better than nothing, do something like
+ // unfound_lost if no valid copies can be found, or just mark unfound
+ auth = j;
+ dout(10) << __func__ << ": selecting osd " << j->first
+ << " for obj " << obj
+ << ", auth == maps.end()"
+ << dendl;
+ continue;
+ }
+ if (i->second.read_error) {
+ // scrub encountered read error, probably corrupt
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", read_error"
+ << dendl;
+ continue;
+ }
+ map<string, bufferptr>::iterator k = i->second.attrs.find(OI_ATTR);
+ if (k == i->second.attrs.end()) {
+ // no object info on object, probably corrupt
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", no oi attr"
+ << dendl;
+ continue;
+ }
+ bufferlist bl;
+ bl.push_back(k->second);
+ object_info_t oi;
+ try {
+ bufferlist::iterator bliter = bl.begin();
+ ::decode(oi, bliter);
+ } catch (...) {
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", corrupt oi attr"
+ << dendl;
+ // invalid object info, probably corrupt
+ continue;
+ }
+ if (oi.size != i->second.size) {
+ // invalid size, probably corrupt
+ dout(10) << __func__ << ": rejecting osd " << j->first
+ << " for obj " << obj
+ << ", size mismatch"
+ << dendl;
+ // invalid object info, probably corrupt
+ continue;
+ }
+ dout(10) << __func__ << ": selecting osd " << j->first
+ << " for obj " << obj
+ << dendl;
+ auth = j;
+ }
+ return auth;
+}
+
+void ReplicatedBackend::be_compare_scrubmaps(const map<int,ScrubMap*> &maps,
+ map<hobject_t, set<int> > &missing,
+ map<hobject_t, set<int> > &inconsistent,
+ map<hobject_t, int> &authoritative,
+ map<hobject_t, set<int> > &invalid_snapcolls,
+ int &shallow_errors,
+ int &deep_errors,
+ const pg_t pgid,
+ const vector<int> &acting,
+ ostream &errorstream)
+{
+ map<hobject_t,ScrubMap::object>::const_iterator i;
+ map<int, ScrubMap *>::const_iterator j;
+ set<hobject_t> master_set;
+
+ // Construct master set
+ for (j = maps.begin(); j != maps.end(); ++j) {
+ for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) {
+ master_set.insert(i->first);
+ }
+ }
+
+ // Check maps against master set and each other
+ for (set<hobject_t>::const_iterator k = master_set.begin();
+ k != master_set.end();
+ ++k) {
+ map<int, ScrubMap *>::const_iterator auth = be_select_auth_object(*k, maps);
+ assert(auth != maps.end());
+ set<int> cur_missing;
+ set<int> cur_inconsistent;
+ for (j = maps.begin(); j != maps.end(); ++j) {
+ if (j == auth)
+ continue;
+ if (j->second->objects.count(*k)) {
+ // Compare
+ stringstream ss;
+ enum scrub_error_type error = be_compare_scrub_objects(auth->second->objects[*k],
+ j->second->objects[*k],
+ ss);
+ if (error != CLEAN) {
+ cur_inconsistent.insert(j->first);
+ if (error == SHALLOW_ERROR)
+ ++shallow_errors;
+ else
+ ++deep_errors;
+ errorstream << pgid << " osd." << acting[j->first]
+ << ": soid " << *k << " " << ss.str() << std::endl;
+ }
+ } else {
+ cur_missing.insert(j->first);
+ ++shallow_errors;
+ errorstream << pgid
+ << " osd." << acting[j->first]
+ << " missing " << *k << std::endl;
+ }
+ }
+ assert(auth != maps.end());
+ if (!cur_missing.empty()) {
+ missing[*k] = cur_missing;
+ }
+ if (!cur_inconsistent.empty()) {
+ inconsistent[*k] = cur_inconsistent;
+ }
+ if (!cur_inconsistent.empty() || !cur_missing.empty()) {
+ authoritative[*k] = auth->first;
+ }
+ }
+}
diff --git a/src/osd/ReplicatedBackend.h b/src/osd/ReplicatedBackend.h
index a95a77b..0f9cf56 100644
--- a/src/osd/ReplicatedBackend.h
+++ b/src/osd/ReplicatedBackend.h
@@ -18,6 +18,7 @@
#include "OSD.h"
#include "PGBackend.h"
#include "osd_types.h"
+#include "../include/memory.h"
struct C_ReplicatedBackend_OnPullComplete;
class ReplicatedBackend : public PGBackend {
@@ -66,6 +67,9 @@ public:
void check_recovery_sources(const OSDMapRef osdmap);
+ /// @see PGBackend::delay_message_until_active
+ bool can_handle_while_inactive(OpRequestRef op);
+
/// @see PGBackend::handle_message
bool handle_message(
OpRequestRef op
@@ -168,6 +172,23 @@ public:
const hobject_t &hoid,
const string &attr,
bufferlist *out);
+
+ int objects_get_attrs(
+ const hobject_t &hoid,
+ map<string, bufferlist> *out);
+
+ int objects_read_sync(
+ const hobject_t &hoid,
+ uint64_t off,
+ uint64_t len,
+ bufferlist *bl);
+
+ void objects_read_async(
+ const hobject_t &hoid,
+ const list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > &to_read,
+ Context *on_complete);
+
private:
// push
struct PushInfo {
@@ -324,6 +345,169 @@ private:
const ObjectRecoveryInfo& recovery_info,
SnapSetContext *ssc
);
+
+ /**
+ * Client IO
+ */
+ struct InProgressOp {
+ tid_t tid;
+ set<int> waiting_for_commit;
+ set<int> waiting_for_applied;
+ Context *on_commit;
+ Context *on_applied;
+ OpRequestRef op;
+ eversion_t v;
+ InProgressOp(
+ tid_t tid, Context *on_commit, Context *on_applied,
+ OpRequestRef op, eversion_t v)
+ : tid(tid), on_commit(on_commit), on_applied(on_applied),
+ op(op), v(v) {}
+ bool done() const {
+ return waiting_for_commit.empty() &&
+ waiting_for_applied.empty();
+ }
+ };
+ map<tid_t, InProgressOp> in_progress_ops;
+public:
+ PGTransaction *get_transaction();
+ friend class C_OSD_OnOpCommit;
+ friend class C_OSD_OnOpApplied;
+ void submit_transaction(
+ const hobject_t &hoid,
+ const eversion_t &at_version,
+ PGTransaction *t,
+ const eversion_t &trim_to,
+ vector<pg_log_entry_t> &log_entries,
+ Context *on_local_applied_sync,
+ Context *on_all_applied,
+ Context *on_all_commit,
+ tid_t tid,
+ osd_reqid_t reqid,
+ OpRequestRef op
+ );
+
+ void rollback_setattrs(
+ const hobject_t &hoid,
+ map<string, boost::optional<bufferlist> > &old_attrs,
+ ObjectStore::Transaction *t) {
+ map<string, bufferlist> to_set;
+ set<string> to_remove;
+ for (map<string, boost::optional<bufferlist> >::iterator i = old_attrs.begin();
+ i != old_attrs.end();
+ ++i) {
+ if (i->second) {
+ to_set[i->first] = i->second.get();
+ } else {
+ t->rmattr(coll, hoid, i->first);
+ }
+ }
+ t->setattrs(coll, hoid, to_set);
+ }
+
+ void rollback_append(
+ const hobject_t &hoid,
+ uint64_t old_size,
+ ObjectStore::Transaction *t) {
+ t->truncate(coll, hoid, old_size);
+ }
+
+ void rollback_stash(
+ const hobject_t &hoid,
+ version_t old_version,
+ ObjectStore::Transaction *t) {
+ t->remove(coll, hoid);
+ t->collection_move_rename(
+ coll,
+ ghobject_t(hoid, old_version, 0),
+ coll,
+ hoid);
+ }
+
+ void rollback_create(
+ const hobject_t &hoid,
+ ObjectStore::Transaction *t) {
+ t->remove(coll, hoid);
+ }
+
+ void trim_stashed_object(
+ const hobject_t &hoid,
+ version_t old_version,
+ ObjectStore::Transaction *t) {
+ t->remove(coll, ghobject_t(hoid, old_version, 0));
+ }
+
+private:
+ void issue_op(
+ const hobject_t &soid,
+ const eversion_t &at_version,
+ tid_t tid,
+ osd_reqid_t reqid,
+ eversion_t pg_trim_to,
+ hobject_t new_temp_oid,
+ hobject_t discard_temp_oid,
+ vector<pg_log_entry_t> &log_entries,
+ InProgressOp *op,
+ ObjectStore::Transaction *op_t);
+ void op_applied(InProgressOp *op);
+ void op_commit(InProgressOp *op);
+ void sub_op_modify_reply(OpRequestRef op);
+ void sub_op_modify(OpRequestRef op);
+
+ struct RepModify {
+ OpRequestRef op;
+ bool applied, committed;
+ int ackerosd;
+ eversion_t last_complete;
+ epoch_t epoch_started;
+
+ uint64_t bytes_written;
+
+ ObjectStore::Transaction opt, localt;
+
+ RepModify() : applied(false), committed(false), ackerosd(-1),
+ epoch_started(0), bytes_written(0) {}
+ };
+ typedef ceph::shared_ptr<RepModify> RepModifyRef;
+
+ struct C_OSD_RepModifyApply : public Context {
+ ReplicatedBackend *pg;
+ RepModifyRef rm;
+ C_OSD_RepModifyApply(ReplicatedBackend *pg, RepModifyRef r)
+ : pg(pg), rm(r) {}
+ void finish(int r) {
+ pg->sub_op_modify_applied(rm);
+ }
+ };
+ struct C_OSD_RepModifyCommit : public Context {
+ ReplicatedBackend *pg;
+ RepModifyRef rm;
+ C_OSD_RepModifyCommit(ReplicatedBackend *pg, RepModifyRef r)
+ : pg(pg), rm(r) {}
+ void finish(int r) {
+ pg->sub_op_modify_commit(rm);
+ }
+ };
+ void sub_op_modify_applied(RepModifyRef rm);
+ void sub_op_modify_commit(RepModifyRef rm);
+ bool scrub_supported() { return true; }
+ void be_scan_list(ScrubMap &map, const vector<hobject_t> &ls, bool deep,
+ ThreadPool::TPHandle &handle);
+ enum scrub_error_type be_compare_scrub_objects(
+ const ScrubMap::object &auth,
+ const ScrubMap::object &candidate,
+ ostream &errorstream);
+ map<int, ScrubMap *>::const_iterator be_select_auth_object(
+ const hobject_t &obj,
+ const map<int,ScrubMap*> &maps);
+ void be_compare_scrubmaps(const map<int,ScrubMap*> &maps,
+ map<hobject_t, set<int> > &missing,
+ map<hobject_t, set<int> > &inconsistent,
+ map<hobject_t, int> &authoritative,
+ map<hobject_t, set<int> > &invalid_snapcolls,
+ int &shallow_errors, int &deep_errors,
+ const pg_t pgid,
+ const vector<int> &acting,
+ ostream &errorstream);
};
#endif
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 0a28d15..2068af7 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -101,6 +101,86 @@ static void log_subop_stats(
osd->logger->tinc(tag_lat, latency);
}
+struct OnReadComplete : public Context {
+ ReplicatedPG *pg;
+ ReplicatedPG::OpContext *opcontext;
+ OnReadComplete(
+ ReplicatedPG *pg,
+ ReplicatedPG::OpContext *ctx) : pg(pg), opcontext(ctx) {}
+ void finish(int r) {
+ if (r < 0)
+ opcontext->async_read_result = r;
+ opcontext->finish_read(pg);
+ }
+ ~OnReadComplete() {}
+};
+
+// OpContext
+void ReplicatedPG::OpContext::start_async_reads(ReplicatedPG *pg)
+{
+ inflightreads = 1;
+ pg->pgbackend->objects_read_async(
+ obc->obs.oi.soid,
+ pending_async_reads,
+ new OnReadComplete(pg, this));
+ pending_async_reads.clear();
+}
+void ReplicatedPG::OpContext::finish_read(ReplicatedPG *pg)
+{
+ assert(inflightreads > 0);
+ --inflightreads;
+ if (async_reads_complete()) {
+ assert(pg->in_progress_async_reads.size());
+ assert(pg->in_progress_async_reads.front().second == this);
+ pg->in_progress_async_reads.pop_front();
+ pg->complete_read_ctx(async_read_result, this);
+ }
+}
+
+class CopyFromCallback: public ReplicatedPG::CopyCallback {
+public:
+ ReplicatedPG::CopyResults *results;
+ int retval;
+ ReplicatedPG::OpContext *ctx;
+ CopyFromCallback(ReplicatedPG::OpContext *ctx_)
+ : results(NULL),
+ retval(0),
+ ctx(ctx_) {}
+ ~CopyFromCallback() {}
+
+ virtual void finish(ReplicatedPG::CopyCallbackResults results_) {
+ results = results_.get<1>();
+ int r = results_.get<0>();
+ retval = r;
+
+ // for finish_copyfrom
+ ctx->user_at_version = results->user_version;
+
+ if (r >= 0) {
+ ctx->pg->execute_ctx(ctx);
+ }
+ ctx->copy_cb = NULL;
+ if (r < 0) {
+ if (r != -ECANCELED) { // on cancel just toss it out; client resends
+ ctx->pg->osd->reply_op_error(ctx->op, r);
+ } else if (results->should_requeue) {
+ ctx->pg->requeue_op(ctx->op);
+ }
+ ctx->pg->close_op_ctx(ctx);
+ }
+ }
+
+ bool is_temp_obj_used() {
+ return results->started_temp_obj;
+ }
+ uint64_t get_data_size() {
+ return results->object_size;
+ }
+ int get_result() {
+ return retval;
+ }
+};
+
// ======================
// PGBackend::Listener
@@ -150,6 +230,8 @@ void ReplicatedPG::on_local_recover(
bufferlist bl;
::encode(recovery_info.oi, bl);
t->setattr(coll, recovery_info.soid, OI_ATTR, bl);
+ if (obc)
+ obc->attr_cache[OI_ATTR] = bl;
}
}
@@ -224,7 +306,7 @@ void ReplicatedPG::on_peer_recover(
publish_stats_to_osd();
// done!
peer_missing[peer].got(soid, recovery_info.version);
- if (peer == backfill_target && backfills_in_flight.count(soid)) {
+ if (is_backfill_targets(peer) && backfills_in_flight.count(soid)) {
map<hobject_t, ObjectContextRef>::iterator i = recovering.find(soid);
assert(i != recovering.end());
list<OpRequestRef> requeue_list;
@@ -305,15 +387,16 @@ bool ReplicatedPG::is_degraded_object(const hobject_t& soid)
{
if (pg_log.get_missing().missing.count(soid))
return true;
- for (unsigned i = 1; i < acting.size(); i++) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i = 1; i < actingbackfill.size(); i++) {
+ int peer = actingbackfill[i];
if (peer_missing.count(peer) &&
peer_missing[peer].missing.count(soid))
return true;
// Object is degraded if after last_backfill AND
// we are backfilling it
- if (peer == backfill_target &&
+ if (is_backfill_targets(peer) &&
peer_info[peer].last_backfill <= soid &&
last_backfill_started >= soid &&
backfills_in_flight.count(soid))
@@ -338,8 +421,9 @@ void ReplicatedPG::wait_for_degraded_object(const hobject_t& soid, OpRequestRef
<< ", recovering"
<< dendl;
eversion_t v;
- for (unsigned i = 1; i < acting.size(); i++) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i = 1; i < actingbackfill.size(); i++) {
+ int peer = actingbackfill[i];
if (peer_missing.count(peer) &&
peer_missing[peer].missing.count(soid)) {
v = peer_missing[peer].missing[soid].need;
@@ -462,10 +546,34 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss,
for (vector<int>::iterator p = acting.begin(); p != acting.end(); ++p)
f->dump_unsigned("osd", *p);
f->close_section();
+ if (backfill_targets.size() > 0) {
+ f->open_array_section("backfill_targets");
+ for (vector<int>::iterator p = backfill_targets.begin(); p != backfill_targets.end(); ++p)
+ f->dump_unsigned("osd", *p);
+ f->close_section();
+ }
+ if (actingbackfill.size() > 0) {
+ f->open_array_section("actingbackfill");
+ for (vector<int>::iterator p = actingbackfill.begin(); p != actingbackfill.end(); ++p)
+ f->dump_unsigned("osd", *p);
+ f->close_section();
+ }
f->open_object_section("info");
+ _update_calc_stats();
info.dump(f.get());
f->close_section();
+ f->open_array_section("peer_info");
+ for (map<int,pg_info_t>::iterator p = peer_info.begin();
+ p != peer_info.end();
+ ++p) {
+ f->open_object_section("info");
+ f->dump_unsigned("peer", p->first);
+ p->second.dump(f.get());
+ f->close_section();
+ }
+ f->close_section();
+
f->open_array_section("recovery_state");
handle_query_state(f.get());
f->close_section();
@@ -587,7 +695,6 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
op->mark_started();
- bufferlist outdata;
int result = 0;
string cname, mname;
PGLSFilter *filter = NULL;
@@ -595,7 +702,10 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
snapid_t snapid = m->get_snapid();
- for (vector<OSDOp>::iterator p = m->ops.begin(); p != m->ops.end(); ++p) {
+ vector<OSDOp> ops = m->ops;
+
+ for (vector<OSDOp>::iterator p = ops.begin(); p != ops.end(); ++p) {
+ OSDOp& osd_op = *p;
bufferlist::iterator bp = p->indata.begin();
switch (p->op.op) {
case CEPH_OSD_OP_PGLS_FILTER:
@@ -738,14 +848,64 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
result = 1;
}
response.handle = next;
- ::encode(response, outdata);
+ ::encode(response, osd_op.outdata);
if (filter)
- ::encode(filter_out, outdata);
+ ::encode(filter_out, osd_op.outdata);
dout(10) << " pgls result=" << result << " outdata.length()="
- << outdata.length() << dendl;
+ << osd_op.outdata.length() << dendl;
+ }
+ break;
+
+ case CEPH_OSD_OP_PG_HITSET_LS:
+ {
+ list< pair<utime_t,utime_t> > ls;
+ for (list<pg_hit_set_info_t>::const_iterator p = info.hit_set.history.begin();
+ p != info.hit_set.history.end();
+ ++p)
+ ls.push_back(make_pair(p->begin, p->end));
+ if (info.hit_set.current_info.begin)
+ ls.push_back(make_pair(info.hit_set.current_info.begin, utime_t()));
+ else if (hit_set)
+ ls.push_back(make_pair(hit_set_start_stamp, utime_t()));
+ ::encode(ls, osd_op.outdata);
+ }
+ break;
+
+ case CEPH_OSD_OP_PG_HITSET_GET:
+ {
+ utime_t stamp(osd_op.op.hit_set_get.stamp);
+ if ((info.hit_set.current_info.begin &&
+ stamp >= info.hit_set.current_info.begin) ||
+ stamp >= hit_set_start_stamp) {
+ // read the current in-memory HitSet, not the version we've
+ // checkpointed.
+ if (!hit_set) {
+ result= -ENOENT;
+ break;
+ }
+ ::encode(*hit_set, osd_op.outdata);
+ result = osd_op.outdata.length();
+ } else {
+ // read an archived HitSet.
+ hobject_t oid;
+ for (list<pg_hit_set_info_t>::const_iterator p = info.hit_set.history.begin();
+ p != info.hit_set.history.end();
+ ++p) {
+ if (stamp >= p->begin && stamp <= p->end) {
+ oid = get_hit_set_archive_object(p->begin, p->end);
+ break;
+ }
+ }
+ if (oid == hobject_t()) {
+ result = -ENOENT;
+ break;
+ }
+ result = osd->store->read(coll, oid, 0, 0, osd_op.outdata);
+ }
}
break;
+
default:
result = -EINVAL;
break;
@@ -755,7 +915,7 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
// reply
MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(),
CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
- reply->set_data(outdata);
+ reply->claim_op_out_data(ops);
reply->set_result(result);
reply->set_reply_versions(info.last_update, info.last_user_version);
osd->send_message_osd_client(reply, m->get_connection());
@@ -825,9 +985,6 @@ void ReplicatedPG::do_request(
OpRequestRef op,
ThreadPool::TPHandle &handle)
{
- // do any pending flush
- do_pending_flush();
-
if (!op_has_sufficient_caps(op)) {
osd->reply_op_error(op, -EPERM);
return;
@@ -836,18 +993,33 @@ void ReplicatedPG::do_request(
if (can_discard_request(op)) {
return;
}
- if (!flushed) {
- dout(20) << " !flushed, waiting for active on " << op << dendl;
+ if (flushes_in_progress > 0) {
+ dout(20) << flushes_in_progress
+ << " flushes_in_progress pending "
+ << "waiting for active on " << op << dendl;
waiting_for_active.push_back(op);
return;
}
+ if (!is_active()) {
+ // Delay unless PGBackend says it's ok
+ if (pgbackend->can_handle_while_inactive(op)) {
+ bool handled = pgbackend->handle_message(op);
+ assert(handled);
+ return;
+ } else {
+ waiting_for_active.push_back(op);
+ return;
+ }
+ }
+
+ assert(is_active() && flushes_in_progress == 0);
if (pgbackend->handle_message(op))
return;
switch (op->get_req()->get_type()) {
case CEPH_MSG_OSD_OP:
- if (is_replay() || !is_active()) {
+ if (is_replay()) {
dout(20) << " replay, waiting for active on " << op << dendl;
waiting_for_active.push_back(op);
return;
@@ -876,6 +1048,40 @@ void ReplicatedPG::do_request(
}
}
+hobject_t ReplicatedPG::earliest_backfill() const
+{
+ hobject_t e = hobject_t::get_max();
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ map<int, pg_info_t>::const_iterator iter = peer_info.find(bt);
+ assert(iter != peer_info.end());
+ if (iter->second.last_backfill < e)
+ e = iter->second.last_backfill;
+ }
+ return e;
+}
+
+// if we have src_oids, we need to be careful of the target being
+// before and a src being after the last_backfill line, or else the
+// operation won't apply properly on the backfill_target. (the
+// opposite is not a problem; if the target is after the line, we
+// don't apply on the backfill_target and it doesn't matter.)
+// With multi-backfill some backfill targets can be ahead of
+// last_backfill_started. We consider each replica individually and
+// take the larger of last_backfill_started and the replicas last_backfill.
+bool ReplicatedPG::check_src_targ(const hobject_t& soid, const hobject_t& toid) const
+{
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ map<int, pg_info_t>::const_iterator iter = peer_info.find(bt);
+ assert(iter != peer_info.end());
+
+ if (toid <= MAX(last_backfill_started, iter->second.last_backfill) &&
+ soid > MAX(last_backfill_started, iter->second.last_backfill))
+ return true;
+ }
+ return false;
+}
/** do_op - do an op
* pg lock will be held (if multithreaded)
@@ -894,12 +1100,17 @@ void ReplicatedPG::do_op(OpRequestRef op)
}
// order this op as a write?
- bool write_ordered = op->may_write() || (m->get_flags() & CEPH_OSD_FLAG_RWORDERED);
+ bool write_ordered =
+ op->may_write() ||
+ op->may_cache() ||
+ (m->get_flags() & CEPH_OSD_FLAG_RWORDERED);
dout(10) << "do_op " << *m
<< (op->may_write() ? " may_write" : "")
<< (op->may_read() ? " may_read" : "")
+ << (op->may_cache() ? " may_cache" : "")
<< " -> " << (write_ordered ? "write-ordered" : "read-ordered")
+ << " flags " << ceph_osd_flag_string(m->get_flags())
<< dendl;
hobject_t head(m->get_oid(), m->get_object_locator().key,
@@ -948,16 +1159,15 @@ void ReplicatedPG::do_op(OpRequestRef op)
}
ObjectContextRef obc;
- bool can_create = op->may_write();
- snapid_t snapid;
- int r = find_object_context(
- hobject_t(m->get_oid(),
- m->get_object_locator().key,
- m->get_snapid(),
- m->get_pg().ps(),
- m->get_object_locator().get_pool(),
- m->get_object_locator().nspace),
- &obc, can_create, &snapid);
+ bool can_create = op->may_write() || op->may_cache();
+ hobject_t missing_oid;
+ hobject_t oid(m->get_oid(),
+ m->get_object_locator().key,
+ m->get_snapid(),
+ m->get_pg().ps(),
+ m->get_object_locator().get_pool(),
+ m->get_object_locator().nspace);
+ int r = find_object_context(oid, &obc, can_create, &missing_oid);
if (r == -EAGAIN) {
// If we're not the primary of this OSD, and we have
@@ -968,15 +1178,21 @@ void ReplicatedPG::do_op(OpRequestRef op)
!(m->get_flags() & CEPH_OSD_FLAG_LOCALIZE_READS))) {
// missing the specific snap we need; requeue and wait.
assert(!can_create); // only happens on a read
- hobject_t soid(m->get_oid(), m->get_object_locator().key,
- snapid, m->get_pg().ps(),
- info.pgid.pool(), m->get_object_locator().nspace);
- wait_for_missing_object(soid, op);
+ wait_for_missing_object(missing_oid, op);
return;
}
}
- if (maybe_handle_cache(op, obc, r))
+ if (hit_set) {
+ hit_set->insert(oid);
+ if (hit_set->is_full() ||
+ hit_set_start_stamp + pool.info.hit_set_period <= m->get_recv_stamp()) {
+ hit_set_persist();
+ }
+ }
+
+ if ((m->get_flags() & CEPH_OSD_FLAG_IGNORE_CACHE) == 0 &&
+ maybe_handle_cache(op, obc, r, missing_oid))
return;
if (r) {
@@ -995,13 +1211,13 @@ void ReplicatedPG::do_op(OpRequestRef op)
}
// io blocked on obc?
- if (obc->is_blocked()) {
+ if (obc->is_blocked() &&
+ (m->get_flags() & CEPH_OSD_FLAG_FLUSH) == 0) {
wait_for_blocked_object(obc->obs.oi.soid, op);
return;
}
- dout(25) << __func__ << ": object " << obc->obs.oi.soid
- << " has oi of " << obc->obs.oi << dendl;
+ dout(25) << __func__ << " oi " << obc->obs.oi << dendl;
// are writes blocked by another object?
if (obc->blocked_by) {
@@ -1011,18 +1227,6 @@ void ReplicatedPG::do_op(OpRequestRef op)
return;
}
- // if we have src_oids, we need to be careful of the target being
- // before and a src being after the last_backfill line, or else the
- // operation won't apply properly on the backfill_target. (the
- // opposite is not a problem; if the target is after the line, we
- // don't apply on the backfill_target and it doesn't matter.)
- pg_info_t *backfill_target_info = NULL;
- bool before_backfill = false;
- if (backfill_target >= 0) {
- backfill_target_info = &peer_info[backfill_target];
- before_backfill = obc->obs.oi.soid < backfill_target_info->last_backfill;
- }
-
// src_oids
map<hobject_t,ObjectContextRef> src_obc;
for (vector<OSDOp>::iterator p = m->ops.begin(); p != m->ops.end(); ++p) {
@@ -1045,19 +1249,18 @@ void ReplicatedPG::do_op(OpRequestRef op)
info.pgid.pool(), m->get_object_locator().nspace);
if (!src_obc.count(src_oid)) {
ObjectContextRef sobc;
- snapid_t ssnapid;
+ hobject_t wait_oid;
int r;
if (src_oid.is_head() && is_missing_object(src_oid)) {
wait_for_missing_object(src_oid, op);
} else if ((r = find_object_context(
- src_oid, &sobc, false, &ssnapid)) == -EAGAIN) {
+ src_oid, &sobc, false, &wait_oid)) == -EAGAIN) {
// missing the specific snap we need; requeue and wait.
- hobject_t wait_oid(osd_op.soid.oid, src_oloc.key, ssnapid, m->get_pg().ps(),
- info.pgid.pool(), m->get_object_locator().nspace);
wait_for_missing_object(wait_oid, op);
} else if (r) {
- osd->reply_op_error(op, r);
+ if (!maybe_handle_cache(op, sobc, r, wait_oid, true))
+ osd->reply_op_error(op, r);
} else if (sobc->obs.oi.is_whiteout()) {
osd->reply_op_error(op, -ENOENT);
} else {
@@ -1068,8 +1271,13 @@ void ReplicatedPG::do_op(OpRequestRef op)
<< obc->obs.oi.soid << dendl;
osd->reply_op_error(op, -EINVAL);
} else if (is_degraded_object(sobc->obs.oi.soid) ||
- (before_backfill && sobc->obs.oi.soid > backfill_target_info->last_backfill)) {
- wait_for_degraded_object(sobc->obs.oi.soid, op);
+ (check_src_targ(sobc->obs.oi.soid, obc->obs.oi.soid))) {
+ if (is_degraded_object(sobc->obs.oi.soid)) {
+ wait_for_degraded_object(sobc->obs.oi.soid, op);
+ } else {
+ waiting_for_degraded_object[sobc->obs.oi.soid].push_back(op);
+ op->mark_delayed("waiting for degraded object");
+ }
dout(10) << " writes for " << obc->obs.oi.soid << " now blocked by "
<< sobc->obs.oi.soid << dendl;
obc->blocked_by = sobc;
@@ -1092,7 +1300,7 @@ void ReplicatedPG::do_op(OpRequestRef op)
return;
}
- // any SNAPDIR op needs to hvae all clones present. treat them as
+ // any SNAPDIR op needs to have all clones present. treat them as
// src_obc's so that we track references properly and clean up later.
if (m->get_snapid() == CEPH_SNAPDIR) {
for (vector<snapid_t>::iterator p = obc->ssc->snapset.clones.begin();
@@ -1104,18 +1312,15 @@ void ReplicatedPG::do_op(OpRequestRef op)
clone_oid.snap = *p;
if (!src_obc.count(clone_oid)) {
ObjectContextRef sobc;
- snapid_t ssnapid;
+ hobject_t wait_oid;
- int r = find_object_context(clone_oid, &sobc, false, &ssnapid);
+ int r = find_object_context(clone_oid, &sobc, false, &wait_oid);
if (r == -EAGAIN) {
// missing the specific snap we need; requeue and wait.
- hobject_t wait_oid(clone_oid.oid, src_oloc.key, ssnapid, m->get_pg().ps(),
- info.pgid.pool(), clone_oid.get_namespace());
wait_for_missing_object(wait_oid, op);
} else if (r) {
- osd->reply_op_error(op, r);
- } else if (sobc->obs.oi.is_whiteout()) {
- osd->reply_op_error(op, -ENOENT);
+ if (!maybe_handle_cache(op, sobc, r, wait_oid, true))
+ osd->reply_op_error(op, r);
} else {
dout(10) << " clone_oid " << clone_oid << " obc " << sobc << dendl;
src_obc[clone_oid] = sobc;
@@ -1131,8 +1336,24 @@ void ReplicatedPG::do_op(OpRequestRef op)
OpContext *ctx = new OpContext(op, m->get_reqid(), m->ops,
&obc->obs, obc->ssc,
this);
+ ctx->op_t = pgbackend->get_transaction();
ctx->obc = obc;
- if (!get_rw_locks(ctx)) {
+ if (m->get_flags() & CEPH_OSD_FLAG_SKIPRWLOCKS) {
+ dout(20) << __func__ << ": skipping rw locks" << dendl;
+ } else if (m->get_flags() & CEPH_OSD_FLAG_FLUSH) {
+ dout(20) << __func__ << ": part of flush, will ignore write lock" << dendl;
+
+ // verify there is in fact a flush in progress
+ // FIXME: we could make this a stronger test.
+ map<hobject_t,FlushOpRef>::iterator p = flush_ops.find(obc->obs.oi.soid);
+ if (p == flush_ops.end()) {
+ dout(10) << __func__ << " no flush in progress, aborting" << dendl;
+ close_op_ctx(ctx);
+ osd->reply_op_error(op, -EINVAL);
+ return;
+ }
+ } else if (!get_rw_locks(ctx)) {
+ dout(20) << __func__ << " waiting for rw locks " << dendl;
op->mark_delayed("waiting for rw locks");
close_op_ctx(ctx);
return;
@@ -1146,8 +1367,8 @@ void ReplicatedPG::do_op(OpRequestRef op)
osd->reply_op_error(op, -ENFILE);
return;
}
- if (!op->may_write() && (!obc->obs.exists ||
- obc->obs.oi.is_whiteout())) {
+ if (!op->may_write() && !op->may_cache() && (!obc->obs.exists ||
+ obc->obs.oi.is_whiteout())) {
close_op_ctx(ctx);
osd->reply_op_error(op, -ENOENT);
return;
@@ -1160,42 +1381,91 @@ void ReplicatedPG::do_op(OpRequestRef op)
}
bool ReplicatedPG::maybe_handle_cache(OpRequestRef op, ObjectContextRef obc,
- int r)
+ int r, const hobject_t& missing_oid,
+ bool must_promote)
{
+ if (obc)
+ dout(25) << __func__ << " " << obc->obs.oi << " "
+ << (obc->obs.exists ? "exists" : "DNE")
+ << " missing_oid " << missing_oid
+ << dendl;
+ else
+ dout(25) << __func__ << " (no obc)"
+ << " missing_oid " << missing_oid
+ << dendl;
+
if (obc.get() && obc->is_blocked()) {
// we're already doing something with this object
+ dout(20) << __func__ << " blocked on " << obc->obs.oi.soid << dendl;
+ return false;
+ }
+
+ if (r == -ENOENT && missing_oid == hobject_t()) {
+ // we know this object is logically absent (e.g., an undefined clone)
return false;
}
- switch(pool.info.cache_mode) {
+
+ switch (pool.info.cache_mode) {
case pg_pool_t::CACHEMODE_NONE:
return false;
- break;
+
case pg_pool_t::CACHEMODE_WRITEBACK:
- if (obc.get()) {
+ if (obc.get() && obc->obs.exists) {
return false;
- } else {
- do_cache_redirect(op, obc);
- return true;
}
- break;
- case pg_pool_t::CACHEMODE_INVALIDATE_FORWARD:
- do_cache_redirect(op, obc);
+ if (!must_promote && can_skip_promote(op, obc)) {
+ return false;
+ }
+ promote_object(op, obc, missing_oid);
return true;
- break;
+
+ case pg_pool_t::CACHEMODE_FORWARD:
+ if (obc.get() && obc->obs.exists) {
+ return false;
+ }
+ if (must_promote)
+ promote_object(op, obc, missing_oid);
+ else
+ do_cache_redirect(op, obc);
+ return true;
+
case pg_pool_t::CACHEMODE_READONLY:
- if (obc.get() && !r) {
+ // TODO: clean this case up
+ if (obc.get() && obc->obs.exists) {
return false;
- } else {
+ }
+ if (!obc.get() && r == -ENOENT) {
+ // we don't have the object and op's a read
+ promote_object(op, obc, missing_oid);
+ return true;
+ }
+ if (!r) { // it must be a write
do_cache_redirect(op, obc);
return true;
}
- break;
+ // crap, there was a failure of some kind
+ return false;
+
default:
- assert(0);
+ assert(0 == "unrecognized cache_mode");
}
return false;
}
+bool ReplicatedPG::can_skip_promote(OpRequestRef op, ObjectContextRef obc)
+{
+ MOSDOp *m = static_cast<MOSDOp*>(op->get_req());
+ if (m->ops.empty())
+ return false;
+ // if we get a delete with FAILOK we can skip promote. without
+ // FAILOK we still need to promote (or do something smarter) to
+ // determine whether to return ENOENT or 0.
+ if (m->ops[0].op.op == CEPH_OSD_OP_DELETE &&
+ (m->ops[0].op.flags & CEPH_OSD_OP_FLAG_FAILOK))
+ return true;
+ return false;
+}
+
void ReplicatedPG::do_cache_redirect(OpRequestRef op, ObjectContextRef obc)
{
MOSDOp *m = static_cast<MOSDOp*>(op->get_req());
@@ -1210,6 +1480,46 @@ void ReplicatedPG::do_cache_redirect(OpRequestRef op, ObjectContextRef obc)
return;
}
+class PromoteCallback: public ReplicatedPG::CopyCallback {
+ OpRequestRef op;
+ ObjectContextRef obc;
+ ReplicatedPG *pg;
+public:
+ PromoteCallback(OpRequestRef op_, ObjectContextRef obc_,
+ ReplicatedPG *pg_)
+ : op(op_),
+ obc(obc_),
+ pg(pg_) {}
+
+ virtual void finish(ReplicatedPG::CopyCallbackResults results) {
+ ReplicatedPG::CopyResults *results_data = results.get<1>();
+ int r = results.get<0>();
+ pg->finish_promote(r, op, results_data, obc);
+ }
+};
+
+void ReplicatedPG::promote_object(OpRequestRef op, ObjectContextRef obc,
+ const hobject_t& missing_oid)
+{
+ MOSDOp *m = static_cast<MOSDOp*>(op->get_req());
+ if (!obc) { // we need to create an ObjectContext
+ assert(missing_oid != hobject_t());
+ obc = get_object_context(missing_oid, true);
+ }
+ dout(10) << __func__ << " " << obc->obs.oi.soid << dendl;
+
+ PromoteCallback *cb = new PromoteCallback(op, obc, this);
+ object_locator_t oloc(m->get_object_locator());
+ oloc.pool = pool.info.tier_of;
+ start_copy(cb, obc, obc->obs.oi.soid, oloc, 0,
+ CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY |
+ CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE,
+ obc->obs.oi.soid.snap == CEPH_NOSNAP);
+
+ assert(obc->is_blocked());
+ wait_for_blocked_object(obc->obs.oi.soid, op);
+}
+
void ReplicatedPG::execute_ctx(OpContext *ctx)
{
dout(10) << __func__ << " " << ctx << dendl;
@@ -1222,11 +1532,11 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
// this method must be idempotent since we may call it several times
// before we finally apply the resulting transaction.
- ctx->op_t = ObjectStore::Transaction();
- ctx->local_t = ObjectStore::Transaction();
+ delete ctx->op_t;
+ ctx->op_t = pgbackend->get_transaction();
- // dup/replay?
- if (op->may_write()) {
+ if (op->may_write() || op->may_cache()) {
+ // dup/replay?
const pg_log_entry_t *entry = pg_log.get_log().get_request(ctx->reqid);
if (entry) {
const eversion_t& oldv = entry->version;
@@ -1275,13 +1585,7 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
}
// version
- ctx->at_version = pg_log.get_head();
-
- ctx->at_version.epoch = get_osdmap()->get_epoch();
- ctx->at_version.version++;
- assert(ctx->at_version > info.last_update);
- assert(ctx->at_version > pg_log.get_head());
-
+ ctx->at_version = get_next_version();
ctx->mtime = m->get_mtime();
dout(10) << "do_op " << soid << " " << ctx->ops
@@ -1295,7 +1599,9 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
<< dendl;
}
- ctx->user_at_version = obc->obs.oi.user_version;
+ if (!ctx->user_at_version)
+ ctx->user_at_version = obc->obs.oi.user_version;
+ dout(30) << __func__ << " user_at_version " << ctx->user_at_version << dendl;
// note my stats
utime_t now = ceph_clock_now(cct);
@@ -1348,11 +1654,7 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
// possible to construct an operation that does a read, does a guard
// check (e.g., CMPXATTR), and then a write. Then we either succeed
// with the write, or return a CMPXATTR and the read value.
- if ((ctx->op_t.empty() && !ctx->modify) || result < 0) {
- // read.
- ctx->reply->claim_op_out_data(ctx->ops);
- ctx->reply->get_header().data_off = ctx->data_off;
- } else {
+ if (!((ctx->op_t->empty() && !ctx->modify) || result < 0)) {
// write. normalize the result code.
if (result > 0) {
dout(20) << " zeroing write result code " << result << dendl;
@@ -1362,36 +1664,23 @@ void ReplicatedPG::execute_ctx(OpContext *ctx)
ctx->reply->set_result(result);
// read or error?
- if (ctx->op_t.empty() || result < 0) {
- MOSDOpReply *reply = ctx->reply;
- ctx->reply = NULL;
-
- if (result >= 0) {
- log_op_stats(ctx);
- publish_stats_to_osd();
-
- // on read, return the current object version
- reply->set_reply_versions(eversion_t(), ctx->obs->oi.user_version);
- } else if (result == -ENOENT) {
- // on ENOENT, set a floor for what the next user version will be.
- reply->set_enoent_reply_versions(info.last_update, info.last_user_version);
+ if (ctx->op_t->empty() || result < 0) {
+ if (ctx->pending_async_reads.empty()) {
+ complete_read_ctx(result, ctx);
+ } else {
+ in_progress_async_reads.push_back(make_pair(op, ctx));
+ ctx->start_async_reads(this);
}
-
- reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
- osd->send_message_osd_client(reply, m->get_connection());
- close_op_ctx(ctx);
return;
}
ctx->reply->set_reply_versions(ctx->at_version, ctx->user_at_version);
- assert(op->may_write());
+ assert(op->may_write() || op->may_cache());
// trim log?
calc_trim_to();
- append_log(ctx->log, pg_trim_to, ctx->local_t);
-
// verify that we are doing this in order?
if (cct->_conf->osd_debug_op_order && m->get_source().is_client()) {
map<client_t,tid_t>& cm = debug_op_order[obc->obs.oi.soid];
@@ -1444,6 +1733,8 @@ void ReplicatedPG::log_op_stats(OpContext *ctx)
utime_t now = ceph_clock_now(cct);
utime_t latency = now;
latency -= ctx->op->get_req()->get_recv_stamp();
+ utime_t process_latency = now;
+ process_latency -= ctx->op->get_dequeued_time();
utime_t rlatency;
if (ctx->readable_stamp != utime_t()) {
@@ -1459,6 +1750,7 @@ void ReplicatedPG::log_op_stats(OpContext *ctx)
osd->logger->inc(l_osd_op_outb, outb);
osd->logger->inc(l_osd_op_inb, inb);
osd->logger->tinc(l_osd_op_lat, latency);
+ osd->logger->tinc(l_osd_op_process_lat, process_latency);
if (op->may_read() && op->may_write()) {
osd->logger->inc(l_osd_op_rw);
@@ -1466,15 +1758,18 @@ void ReplicatedPG::log_op_stats(OpContext *ctx)
osd->logger->inc(l_osd_op_rw_outb, outb);
osd->logger->tinc(l_osd_op_rw_rlat, rlatency);
osd->logger->tinc(l_osd_op_rw_lat, latency);
+ osd->logger->tinc(l_osd_op_rw_process_lat, process_latency);
} else if (op->may_read()) {
osd->logger->inc(l_osd_op_r);
osd->logger->inc(l_osd_op_r_outb, outb);
osd->logger->tinc(l_osd_op_r_lat, latency);
- } else if (op->may_write()) {
+ osd->logger->tinc(l_osd_op_r_process_lat, process_latency);
+ } else if (op->may_write() || op->may_cache()) {
osd->logger->inc(l_osd_op_w);
osd->logger->inc(l_osd_op_w_inb, inb);
osd->logger->tinc(l_osd_op_w_rlat, rlatency);
osd->logger->tinc(l_osd_op_w_lat, latency);
+ osd->logger->tinc(l_osd_op_w_process_lat, process_latency);
} else
assert(0);
@@ -1522,8 +1817,6 @@ void ReplicatedPG::do_sub_op(OpRequestRef op)
return;
}
}
-
- sub_op_modify(op);
}
void ReplicatedPG::do_sub_op_reply(OpRequestRef op)
@@ -1538,8 +1831,6 @@ void ReplicatedPG::do_sub_op_reply(OpRequestRef op)
return;
}
}
-
- sub_op_modify_reply(op);
}
void ReplicatedPG::do_scan(
@@ -1588,8 +1879,11 @@ void ReplicatedPG::do_scan(
case MOSDPGScan::OP_SCAN_DIGEST:
{
int from = m->get_source().num();
- assert(from == backfill_target);
- BackfillInterval& bi = peer_backfill_info;
+
+ // Check that from is in backfill_targets vector
+ assert(is_backfill_targets(from));
+
+ BackfillInterval bi;
bi.begin = m->begin;
bi.end = m->end;
bufferlist::iterator p = m->get_data().begin();
@@ -1603,15 +1897,20 @@ void ReplicatedPG::do_scan(
i != tmp.end();
++i) {
hobject_t first(i->first);
- if (first.pool == -1)
+ if (!first.is_max() && first.pool == -1)
first.pool = info.pgid.pool();
bi.objects[first] = i->second;
}
}
+ peer_backfill_info[from] = bi;
+
+ assert(waiting_on_backfill.find(from) != waiting_on_backfill.end());
+ waiting_on_backfill.erase(from);
- assert(waiting_on_backfill);
- waiting_on_backfill = false;
- finish_recovery_op(bi.begin);
+ if (waiting_on_backfill.empty()) {
+ assert(peer_backfill_info.size() == backfill_targets.size());
+ finish_recovery_op(hobject_t::get_max());
+ }
}
break;
}
@@ -1643,6 +1942,8 @@ void ReplicatedBackend::_do_push(OpRequestRef op)
new C_OSD_SendMessageOnConn(
osd, reply, m->get_connection()));
+ t->register_on_applied(
+ new ObjectStore::C_DeleteTransaction(t));
get_parent()->queue_transaction(t);
}
@@ -1715,6 +2016,8 @@ void ReplicatedBackend::_do_pull_response(OpRequestRef op)
osd, reply, m->get_connection()));
}
+ t->register_on_applied(
+ new ObjectStore::C_DeleteTransaction(t));
get_parent()->queue_transaction(t);
}
@@ -1766,7 +2069,6 @@ void ReplicatedPG::do_backfill(OpRequestRef op)
switch (m->op) {
case MOSDPGBackfill::OP_BACKFILL_FINISH:
{
- assert(is_replica());
assert(cct->_conf->osd_kill_backfill_at != 1);
MOSDPGBackfill *reply = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_FINISH_ACK,
@@ -1785,7 +2087,6 @@ void ReplicatedPG::do_backfill(OpRequestRef op)
case MOSDPGBackfill::OP_BACKFILL_PROGRESS:
{
- assert(is_replica());
assert(cct->_conf->osd_kill_backfill_at != 2);
info.last_backfill = m->last_backfill;
@@ -1798,7 +2099,7 @@ void ReplicatedPG::do_backfill(OpRequestRef op)
ObjectStore::Transaction *t = new ObjectStore::Transaction;
dirty_info = true;
write_if_dirty(*t);
- int tr = osd->store->queue_transaction(osr.get(), t);
+ int tr = osd->store->queue_transaction_and_cleanup(osr.get(), t);
assert(tr == 0);
}
break;
@@ -1817,13 +2118,11 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
{
// load clone info
bufferlist bl;
- ObjectContextRef obc;
- int r = find_object_context(coid, &obc, false, NULL);
- if (r == -ENOENT || coid.snap != obc->obs.oi.soid.snap) {
+ ObjectContextRef obc = get_object_context(coid, false, NULL);
+ if (!obc) {
derr << __func__ << "could not find coid " << coid << dendl;
assert(0);
}
- assert(r == 0);
object_info_t &coi = obc->obs.oi;
set<snapid_t> old_snaps(coi.snaps.begin(), coi.snaps.end());
@@ -1845,26 +2144,11 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
<< " old snapset " << snapset << dendl;
assert(snapset.seq);
- vector<OSDOp> ops;
- tid_t rep_tid = osd->get_tid();
- osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
- OpContext *ctx = new OpContext(
- OpRequestRef(),
- reqid,
- ops,
- &obc->obs,
- obc->ssc,
- this);
- ctx->mtime = ceph_clock_now(cct);
-
- ctx->at_version.epoch = get_osdmap()->get_epoch();
- ctx->at_version.version = pg_log.get_head().version + 1;
-
- RepGather *repop = new_repop(ctx, obc, rep_tid);
+ RepGather *repop = simple_repop_create(obc);
+ OpContext *ctx = repop->ctx;
+ ctx->at_version = get_next_version();
- ObjectStore::Transaction *t = &ctx->op_t;
- OSDriver::OSTransaction os_t(osdriver.get_transaction(t));
-
+ PGBackend::PGTransaction *t = ctx->op_t;
set<snapid_t> new_snaps;
for (set<snapid_t>::iterator i = old_snaps.begin();
i != old_snaps.end();
@@ -1873,22 +2157,10 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
new_snaps.insert(*i);
}
- r = snap_mapper.update_snaps(
- coid,
- new_snaps,
- &old_snaps, // debug
- &os_t);
- if (r != 0) {
- derr << __func__ << ": snap_mapper.update_snap returned "
- << cpp_strerror(r) << dendl;
- assert(0);
- }
-
if (new_snaps.empty()) {
// remove clone
dout(10) << coid << " snaps " << old_snaps << " -> "
<< new_snaps << " ... deleting" << dendl;
- t->remove(coll, coid);
// ...from snapset
snapid_t last = coid.snap;
@@ -1914,9 +2186,14 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
delta); // not deallocated
}
delta.num_objects--;
+ if (coi.is_dirty())
+ delta.num_objects_dirty--;
+ if (coi.is_whiteout())
+ delta.num_whiteouts--;
delta.num_object_clones--;
delta.num_bytes -= snapset.clone_size[last];
info.stats.stats.add(delta, obc->obs.oi.category);
+ obc->obs.exists = false;
snapset.clones.erase(p);
snapset.clone_overlap.erase(last);
@@ -1932,6 +2209,20 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
osd_reqid_t(),
ctx->mtime)
);
+ if (pool.info.ec_pool()) {
+ set<snapid_t> snaps(
+ ctx->obc->obs.oi.snaps.begin(),
+ ctx->obc->obs.oi.snaps.end());
+ ctx->log.back().mod_desc.update_snaps(snaps);
+ if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+ t->stash(coid, ctx->at_version.version);
+ } else {
+ t->remove(coid);
+ }
+ } else {
+ t->remove(coid);
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
ctx->at_version.version++;
} else {
// save adjusted snaps for this object
@@ -1943,7 +2234,7 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
coi.version = ctx->at_version;
bl.clear();
::encode(coi, bl);
- t->setattr(coll, coid, OI_ATTR, bl);
+ setattr_maybe_cache(ctx->obc, ctx, t, OI_ATTR, bl);
ctx->log.push_back(
pg_log_entry_t(
@@ -1955,6 +2246,18 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
osd_reqid_t(),
ctx->mtime)
);
+ if (pool.info.ec_pool()) {
+ set<string> changing;
+ changing.insert(OI_ATTR);
+ ctx->obc->fill_in_setattrs(changing, &(ctx->log.back().mod_desc));
+ set<snapid_t> snaps(
+ ctx->obc->obs.oi.snaps.begin(),
+ ctx->obc->obs.oi.snaps.end());
+ ctx->log.back().mod_desc.update_snaps(snaps);
+ } else {
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
+
::encode(coi.snaps, ctx->log.back().snaps);
ctx->at_version.version++;
}
@@ -1980,9 +2283,19 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
osd_reqid_t(),
ctx->mtime)
);
- ctx->snapset_obc->obs.exists = false;
- t->remove(coll, snapoid);
+ ctx->snapset_obc->obs.exists = false;
+
+ if (pool.info.ec_pool()) {
+ if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+ t->stash(snapoid, ctx->at_version.version);
+ } else {
+ t->remove(snapoid);
+ }
+ } else {
+ t->remove(snapoid);
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
} else {
dout(10) << coid << " updating snapset on " << snapoid << dendl;
ctx->log.push_back(
@@ -2002,11 +2315,20 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
bl.clear();
::encode(snapset, bl);
- t->setattr(coll, snapoid, SS_ATTR, bl);
+ setattr_maybe_cache(ctx->snapset_obc, ctx, t, SS_ATTR, bl);
bl.clear();
::encode(ctx->snapset_obc->obs.oi, bl);
- t->setattr(coll, snapoid, OI_ATTR, bl);
+ setattr_maybe_cache(ctx->snapset_obc, ctx, t, OI_ATTR, bl);
+
+ if (pool.info.ec_pool()) {
+ set<string> changing;
+ changing.insert(OI_ATTR);
+ changing.insert(SS_ATTR);
+ ctx->snapset_obc->fill_in_setattrs(changing, &(ctx->log.back().mod_desc));
+ } else {
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
}
return repop;
@@ -2112,6 +2434,32 @@ int ReplicatedPG::do_xattr_cmp_str(int op, string& v1s, bufferlist& xattr)
// ========================================================================
// low level osd ops
+int ReplicatedPG::do_tmap2omap(OpContext *ctx, unsigned flags)
+{
+ dout(20) << " convert tmap to omap for " << ctx->new_obs.oi.soid << dendl;
+ bufferlist header, vals;
+ int r = _get_tmap(ctx, &header, &vals);
+ if (r < 0) {
+ if (r == -ENODATA && (flags & CEPH_OSD_TMAP2OMAP_NULLOK))
+ r = 0;
+ return r;
+ }
+
+ vector<OSDOp> ops(3);
+
+ ops[0].op.op = CEPH_OSD_OP_TRUNCATE;
+ ops[0].op.extent.offset = 0;
+ ops[0].op.extent.length = 0;
+
+ ops[1].op.op = CEPH_OSD_OP_OMAPSETHEADER;
+ ops[1].indata.claim(header);
+
+ ops[2].op.op = CEPH_OSD_OP_OMAPSETVALS;
+ ops[2].indata.claim(vals);
+
+ return do_osd_ops(ctx, ops);
+}
+
int ReplicatedPG::do_tmapup_slow(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op,
bufferlist& bl)
{
@@ -2387,6 +2735,16 @@ static int check_offset_and_length(uint64_t offset, uint64_t length, uint64_t ma
return 0;
}
+struct FillInExtent : public Context {
+ ceph_le64 *r;
+ FillInExtent(ceph_le64 *r) : r(r) {}
+ void finish(int _r) {
+ if (_r >= 0) {
+ *r = _r;
+ }
+ }
+};
+
int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
{
int result = 0;
@@ -2397,7 +2755,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
bool first_read = true;
- ObjectStore::Transaction& t = ctx->op_t;
+ PGBackend::PGTransaction* t = ctx->op_t;
dout(10) << "do_osd_op " << soid << " " << ops << dendl;
@@ -2413,6 +2771,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
switch (op.op) {
// non user-visible modifications
case CEPH_OSD_OP_WATCH:
+ case CEPH_OSD_OP_CACHE_EVICT:
+ case CEPH_OSD_OP_CACHE_FLUSH:
+ case CEPH_OSD_OP_CACHE_TRY_FLUSH:
+ case CEPH_OSD_OP_UNDIRTY:
+ case CEPH_OSD_OP_COPY_FROM: // we handle user_version update explicitly
break;
default:
if (op.op & CEPH_OSD_OP_MODE_WR)
@@ -2457,27 +2820,15 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// --- READS ---
+ case CEPH_OSD_OP_SYNC_READ:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+ // fall through
case CEPH_OSD_OP_READ:
++ctx->num_read;
{
- // read into a buffer
- bufferlist bl;
- int r = osd->store->read(coll, soid, op.extent.offset, op.extent.length, bl);
- if (first_read) {
- first_read = false;
- ctx->data_off = op.extent.offset;
- }
- osd_op.outdata.claim_append(bl);
- if (r >= 0)
- op.extent.length = r;
- else {
- result = r;
- op.extent.length = 0;
- }
- ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(op.extent.length, 10);
- ctx->delta_stats.num_rd++;
- dout(10) << " read got " << r << " / " << op.extent.length << " bytes from obj " << soid << dendl;
-
__u32 seq = oi.truncate_seq;
// are we beyond truncate_size?
if ( (seq < op.extent.truncate_seq) &&
@@ -2489,20 +2840,44 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
unsigned trim = to-from;
op.extent.length = op.extent.length - trim;
+ }
- bufferlist keep;
-
- // keep first part of osd_op.outdata; trim at truncation point
- dout(10) << " obj " << soid << " seq " << seq
- << ": trimming overlap " << from << "~" << trim << dendl;
- keep.substr_of(osd_op.outdata, 0, osd_op.outdata.length() - trim);
- osd_op.outdata.claim(keep);
+ // read into a buffer
+ bufferlist bl;
+ if (pool.info.ec_pool()) {
+ ctx->pending_async_reads.push_back(
+ make_pair(
+ make_pair(op.extent.offset, op.extent.length),
+ make_pair(&osd_op.outdata, new FillInExtent(&op.extent.length))));
+ dout(10) << " async_read noted for " << soid << dendl;
+ } else {
+ int r = pgbackend->objects_read_sync(
+ soid, op.extent.offset, op.extent.length, &osd_op.outdata);
+ if (r >= 0)
+ op.extent.length = r;
+ else {
+ result = r;
+ op.extent.length = 0;
+ }
+ dout(10) << " read got " << r << " / " << op.extent.length
+ << " bytes from obj " << soid << dendl;
+ }
+ if (first_read) {
+ first_read = false;
+ ctx->data_off = op.extent.offset;
}
+ ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(op.extent.length, 10);
+ ctx->delta_stats.num_rd++;
+
}
break;
/* map extents */
case CEPH_OSD_OP_MAPEXT:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
// read into a buffer
@@ -2519,6 +2894,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
/* map extents */
case CEPH_OSD_OP_SPARSE_READ:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
if (op.extent.truncate_seq) {
@@ -2546,7 +2925,8 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
last < miter->first) {
bufferlist t;
uint64_t len = miter->first - last;
- r = osd->store->read(coll, soid, last, len, t);
+ r = pgbackend->objects_read_sync(
+ soid, last, len, &t);
if (!t.is_zero()) {
osd->clog.error() << coll << " " << soid << " sparse-read found data in hole "
<< last << "~" << len << "\n";
@@ -2554,7 +2934,8 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
}
bufferlist tmpbl;
- r = osd->store->read(coll, soid, miter->first, miter->second, tmpbl);
+ r = pgbackend->objects_read_sync(
+ soid, miter->first, miter->second, &tmpbl);
if (r < 0)
break;
@@ -2572,7 +2953,8 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
if (last < end) {
bufferlist t;
uint64_t len = end - last;
- r = osd->store->read(coll, soid, last, len, t);
+ r = pgbackend->objects_read_sync(
+ soid, last, len, &t);
if (!t.is_zero()) {
osd->clog.error() << coll << " " << soid << " sparse-read found data in hole "
<< last << "~" << len << "\n";
@@ -2658,7 +3040,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
case CEPH_OSD_OP_STAT:
// note: stat does not require RD
{
- if (obs.exists) {
+ if (obs.exists && !oi.is_whiteout()) {
::encode(oi.size, osd_op.outdata);
::encode(oi.mtime, osd_op.outdata);
dout(10) << "stat oi has " << oi.size << " " << oi.mtime << dendl;
@@ -2684,27 +3066,118 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
case CEPH_OSD_OP_UNDIRTY:
++ctx->num_write;
{
- ctx->undirty = true; // see make_writeable()
- ctx->modify = true;
- ctx->delta_stats.num_wr++;
+ if (oi.is_dirty()) {
+ ctx->undirty = true; // see make_writeable()
+ ctx->modify = true;
+ ctx->delta_stats.num_wr++;
+ }
+ result = 0;
}
break;
- case CEPH_OSD_OP_GETXATTR:
- ++ctx->num_read;
+ case CEPH_OSD_OP_CACHE_TRY_FLUSH:
+ ++ctx->num_write;
{
- string aname;
- bp.copy(op.xattr.name_len, aname);
- string name = "_" + aname;
- int r = pgbackend->objects_get_attr(
- soid,
- name,
- &(osd_op.outdata));
- if (r >= 0) {
- op.xattr.value_len = r;
- result = 0;
- ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(r, 10);
- ctx->delta_stats.num_rd++;
+ if (ctx->lock_to_release != OpContext::NONE) {
+ dout(10) << "cache-try-flush without SKIPRWLOCKS flag set" << dendl;
+ result = -EINVAL;
+ break;
+ }
+ if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE) {
+ result = -EINVAL;
+ break;
+ }
+ if (!obs.exists) {
+ result = 0;
+ break;
+ }
+ if (oi.is_dirty()) {
+ result = start_flush(ctx, false);
+ } else {
+ result = 0;
+ }
+ }
+ break;
+
+ case CEPH_OSD_OP_CACHE_FLUSH:
+ ++ctx->num_write;
+ {
+ if (ctx->lock_to_release == OpContext::NONE) {
+ dout(10) << "cache-flush with SKIPRWLOCKS flag set" << dendl;
+ result = -EINVAL;
+ break;
+ }
+ if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE) {
+ result = -EINVAL;
+ break;
+ }
+ if (!obs.exists) {
+ result = 0;
+ break;
+ }
+ if (oi.is_dirty()) {
+ result = start_flush(ctx, true);
+ } else {
+ result = 0;
+ }
+ }
+ break;
+
+ case CEPH_OSD_OP_CACHE_EVICT:
+ ++ctx->num_write;
+ {
+ if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE) {
+ result = -EINVAL;
+ break;
+ }
+ if (oi.is_dirty()) {
+ result = -EBUSY;
+ break;
+ }
+ if (!oi.watchers.empty()) {
+ result = -EBUSY;
+ break;
+ }
+ if (soid.snap == CEPH_NOSNAP) {
+ // verify that all clones have been evicted
+ dout(20) << __func__ << " verifying clones are absent "
+ << ctx->new_snapset << dendl;
+ result = 0;
+ for (vector<snapid_t>::iterator p = ctx->new_snapset.clones.begin();
+ p != ctx->new_snapset.clones.end();
+ ++p) {
+ hobject_t clone_oid = soid;
+ clone_oid.snap = *p;
+ ObjectContextRef clone_obc = get_object_context(clone_oid, false);
+ if (clone_obc && clone_obc->obs.exists) {
+ dout(10) << __func__ << " cannot evict head before clone "
+ << clone_oid << dendl;
+ result = -EBUSY;
+ break;
+ }
+ }
+ if (result < 0)
+ break;
+ }
+ result = _delete_head(ctx, true);
+ }
+ break;
+
+ case CEPH_OSD_OP_GETXATTR:
+ ++ctx->num_read;
+ {
+ string aname;
+ bp.copy(op.xattr.name_len, aname);
+ string name = "_" + aname;
+ int r = getattr_maybe_cache(
+ ctx->obc,
+ name,
+ &(osd_op.outdata));
+ if (r >= 0) {
+ op.xattr.value_len = r;
+ result = 0;
+ ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(r, 10);
+ ctx->delta_stats.num_rd++;
} else
result = r;
}
@@ -2713,18 +3186,24 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
case CEPH_OSD_OP_GETXATTRS:
++ctx->num_read;
{
- map<string,bufferptr> attrset;
- result = osd->store->getattrs(coll, soid, attrset, true);
- map<string, bufferptr>::iterator iter;
- map<string, bufferlist> newattrs;
- for (iter = attrset.begin(); iter != attrset.end(); ++iter) {
- bufferlist bl;
- bl.append(iter->second);
- newattrs[iter->first] = bl;
- }
+ map<string,bufferlist> attrset;
+ result = getattrs_maybe_cache(
+ ctx->obc,
+ &attrset);
+ map<string, bufferlist> out;
+ for (map<string, bufferlist>::iterator i = attrset.begin();
+ i != attrset.end();
+ ++i) {
+ if (i->first[0] != '_')
+ continue;
+ if (i->first == "_")
+ continue;
+ out[i->first.substr(1, i->first.size())].claim(
+ i->second);
+ }
bufferlist bl;
- ::encode(newattrs, bl);
+ ::encode(out, bl);
ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(bl.length(), 10);
ctx->delta_stats.num_rd++;
osd_op.outdata.claim_append(bl);
@@ -2742,13 +3221,13 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
bufferlist xattr;
if (op.op == CEPH_OSD_OP_CMPXATTR)
- result = pgbackend->objects_get_attr(
- soid,
+ result = getattr_maybe_cache(
+ ctx->obc,
name,
&xattr);
else
- result = pgbackend->objects_get_attr(
- src_obc->obs.oi.soid,
+ result = getattr_maybe_cache(
+ src_obc,
name,
&xattr);
if (result < 0 && result != -EEXIST && result != -ENODATA)
@@ -2991,6 +3470,19 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = -EINVAL;
break;
}
+
+ if (!obs.exists) {
+ ctx->mod_desc.create();
+ } else if (op.extent.offset == oi.size) {
+ ctx->mod_desc.append(oi.size);
+ } else {
+ ctx->mod_desc.mark_unrollbackable();
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+ }
+
__u32 seq = oi.truncate_seq;
if (seq && (seq > op.extent.truncate_seq) &&
(op.extent.offset + op.extent.length > oi.size)) {
@@ -3004,10 +3496,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
}
if (op.extent.truncate_seq > seq) {
// write arrives before trimtrunc
- if (obs.exists) {
+ if (obs.exists && !oi.is_whiteout()) {
dout(10) << " truncate_seq " << op.extent.truncate_seq << " > current " << seq
<< ", truncating to " << op.extent.truncate_size << dendl;
- t.truncate(coll, soid, op.extent.truncate_size);
+ t->truncate(soid, op.extent.truncate_size);
oi.truncate_seq = op.extent.truncate_seq;
oi.truncate_size = op.extent.truncate_size;
if (op.extent.truncate_size != oi.size) {
@@ -3025,7 +3517,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
if (result < 0)
break;
- t.write(coll, soid, op.extent.offset, op.extent.length, osd_op.indata);
+ if (pool.info.ec_pool()) {
+ t->append(soid, op.extent.offset, op.extent.length, osd_op.indata);
+ } else {
+ t->write(soid, op.extent.offset, op.extent.length, osd_op.indata);
+ }
write_update_size_and_usage(ctx->delta_stats, oi, ssc->snapset, ctx->modified_ranges,
op.extent.offset, op.extent.length, true);
if (!obs.exists) {
@@ -3045,13 +3541,31 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
if (result < 0)
break;
- if (obs.exists) {
- t.truncate(coll, soid, 0);
+
+ if (pool.info.ec_pool()) {
+ if (obs.exists) {
+ if (ctx->mod_desc.rmobject(oi.version.version)) {
+ t->stash(soid, oi.version.version);
+ } else {
+ t->remove(soid);
+ }
+ }
+ ctx->mod_desc.create();
+ t->append(soid, op.extent.offset, op.extent.length, osd_op.indata);
+ if (obs.exists) {
+ t->setattrs(soid, ctx->obc->attr_cache);
+ }
} else {
+ ctx->mod_desc.mark_unrollbackable();
+ if (obs.exists) {
+ t->truncate(soid, 0);
+ }
+ t->write(soid, op.extent.offset, op.extent.length, osd_op.indata);
+ }
+ if (!obs.exists) {
ctx->delta_stats.num_objects++;
obs.exists = true;
}
- t.write(coll, soid, op.extent.offset, op.extent.length, osd_op.indata);
interval_set<uint64_t> ch;
if (oi.size > 0)
ch.insert(0, oi.size);
@@ -3072,14 +3586,19 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_ZERO:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_write;
{ // zero
result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
if (result < 0)
break;
assert(op.extent.length);
- if (obs.exists) {
- t.zero(coll, soid, op.extent.offset, op.extent.length);
+ if (obs.exists && !oi.is_whiteout()) {
+ ctx->mod_desc.mark_unrollbackable();
+ t->zero(soid, op.extent.offset, op.extent.length);
interval_set<uint64_t> ch;
ch.insert(op.extent.offset, op.extent.length);
ctx->modified_ranges.union_of(ch);
@@ -3093,7 +3612,8 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
++ctx->num_write;
{
int flags = le32_to_cpu(op.flags);
- if (obs.exists && (flags & CEPH_OSD_OP_FLAG_EXCL)) {
+ if (obs.exists && !oi.is_whiteout() &&
+ (flags & CEPH_OSD_OP_FLAG_EXCL)) {
result = -EEXIST; /* this is an exclusive create */
} else {
if (osd_op.indata.length()) {
@@ -3107,7 +3627,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
goto fail;
}
if (category.size()) {
- if (obs.exists) {
+ if (obs.exists && !oi.is_whiteout()) {
if (obs.oi.category != category)
result = -EEXIST; // category cannot be reset
} else {
@@ -3116,7 +3636,8 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
}
}
if (result >= 0 && !obs.exists) {
- t.touch(coll, soid);
+ ctx->mod_desc.create();
+ t->touch(soid);
ctx->delta_stats.num_objects++;
obs.exists = true;
}
@@ -3129,10 +3650,15 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// falling through
case CEPH_OSD_OP_TRUNCATE:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_write;
+ ctx->mod_desc.mark_unrollbackable();
{
// truncate
- if (!obs.exists) {
+ if (!obs.exists || oi.is_whiteout()) {
dout(10) << " object dne, truncate is a no-op" << dendl;
break;
}
@@ -3155,7 +3681,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
oi.truncate_size = op.extent.truncate_size;
}
- t.truncate(coll, soid, op.extent.offset);
+ t->truncate(soid, op.extent.offset);
if (oi.size > op.extent.offset) {
interval_set<uint64_t> trim;
trim.insert(op.extent.offset, oi.size-op.extent.offset);
@@ -3177,16 +3703,21 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// Cannot delete an object with watchers
result = -EBUSY;
} else {
- result = _delete_head(ctx);
+ result = _delete_head(ctx, false);
}
break;
case CEPH_OSD_OP_CLONERANGE:
+ ctx->mod_desc.mark_unrollbackable();
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
++ctx->num_write;
{
if (!obs.exists) {
- t.touch(coll, obs.oi.soid);
+ t->touch(obs.oi.soid);
ctx->delta_stats.num_objects++;
obs.exists = true;
}
@@ -3197,7 +3728,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = -EINVAL;
break;
}
- t.clone_range(coll, src_obc->obs.oi.soid,
+ t->clone_range(src_obc->obs.oi.soid,
obs.oi.soid, op.clonerange.src_offset,
op.clonerange.length, op.clonerange.offset);
@@ -3229,7 +3760,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
} else {
dout(10) << " registered new watch " << w << " by " << entity << dendl;
oi.watchers[make_pair(cookie, entity)] = w;
- t.nop(); // make sure update the object_info on disk!
+ t->nop(); // make sure update the object_info on disk!
}
ctx->watch_connects.push_back(w);
} else {
@@ -3239,7 +3770,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
dout(10) << " removed watch " << oi_iter->second << " by "
<< entity << dendl;
oi.watchers.erase(oi_iter);
- t.nop(); // update oi on disk
+ t->nop(); // update oi on disk
ctx->watch_disconnects.push_back(w);
} else {
dout(10) << " can't remove: no watch by " << entity << dendl;
@@ -3260,16 +3791,30 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
}
if (!obs.exists) {
- t.touch(coll, soid);
+ ctx->mod_desc.create();
+ t->touch(soid);
ctx->delta_stats.num_objects++;
obs.exists = true;
}
string aname;
bp.copy(op.xattr.name_len, aname);
string name = "_" + aname;
+ if (pool.info.ec_pool()) {
+ map<string, boost::optional<bufferlist> > to_set;
+ bufferlist old;
+ int r = getattr_maybe_cache(ctx->obc, name, &old);
+ if (r == 0) {
+ to_set[name] = old;
+ } else {
+ to_set[name];
+ }
+ ctx->mod_desc.setattrs(to_set);
+ } else {
+ ctx->mod_desc.mark_unrollbackable();
+ }
bufferlist bl;
bp.copy(op.xattr.value_len, bl);
- t.setattr(coll, soid, name, bl);
+ setattr_maybe_cache(ctx->obc, ctx, t, name, bl);
ctx->delta_stats.num_wr++;
}
break;
@@ -3280,7 +3825,20 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
string aname;
bp.copy(op.xattr.name_len, aname);
string name = "_" + aname;
- t.rmattr(coll, soid, name);
+ if (pool.info.ec_pool()) {
+ map<string, boost::optional<bufferlist> > to_set;
+ bufferlist old;
+ int r = getattr_maybe_cache(ctx->obc, name, &old);
+ if (r == 0) {
+ to_set[name] = old;
+ } else {
+ to_set[name];
+ }
+ ctx->mod_desc.setattrs(to_set);
+ } else {
+ ctx->mod_desc.mark_unrollbackable();
+ }
+ rmattr_maybe_cache(ctx->obc, ctx, t, name);
ctx->delta_stats.num_wr++;
}
break;
@@ -3305,17 +3863,21 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_STARTSYNC:
- t.start_sync();
+ t->nop();
break;
// -- trivial map --
case CEPH_OSD_OP_TMAPGET:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
vector<OSDOp> nops(1);
OSDOp& newop = nops[0];
- newop.op.op = CEPH_OSD_OP_READ;
+ newop.op.op = CEPH_OSD_OP_SYNC_READ;
newop.op.extent.offset = 0;
newop.op.extent.length = 0;
do_osd_ops(ctx, nops);
@@ -3324,6 +3886,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_TMAPPUT:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_write;
{
//_dout_lock.Lock();
@@ -3378,12 +3944,25 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_TMAPUP:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_write;
result = do_tmapup(ctx, bp, osd_op);
break;
+ case CEPH_OSD_OP_TMAP2OMAP:
+ ++ctx->num_write;
+ result = do_tmap2omap(ctx, op.tmap2omap.flags);
+ break;
+
// OMAP Read ops
case CEPH_OSD_OP_OMAPGETKEYS:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
string start_after;
@@ -3417,6 +3996,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAPGETVALS:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
string start_after;
@@ -3458,6 +4041,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAPGETHEADER:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
osd->store->omap_get_header(coll, soid, &osd_op.outdata);
@@ -3467,6 +4054,10 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAPGETVALSBYKEYS:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
set<string> keys_to_get;
@@ -3486,9 +4077,13 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
break;
case CEPH_OSD_OP_OMAP_CMP:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
++ctx->num_read;
{
- if (!obs.exists) {
+ if (!obs.exists || oi.is_whiteout()) {
result = -ENOENT;
break;
}
@@ -3553,13 +4148,18 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
// OMAP Write ops
case CEPH_OSD_OP_OMAPSETVALS:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+ ctx->mod_desc.mark_unrollbackable();
++ctx->num_write;
{
if (!obs.exists) {
ctx->delta_stats.num_objects++;
obs.exists = true;
}
- t.touch(coll, soid);
+ t->touch(soid);
map<string, bufferlist> to_set;
try {
::decode(to_set, bp);
@@ -3574,45 +4174,60 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
++i) {
dout(20) << "\t" << i->first << dendl;
}
- t.omap_setkeys(coll, soid, to_set);
+ t->omap_setkeys(soid, to_set);
ctx->delta_stats.num_wr++;
}
break;
case CEPH_OSD_OP_OMAPSETHEADER:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+ ctx->mod_desc.mark_unrollbackable();
++ctx->num_write;
{
if (!obs.exists) {
ctx->delta_stats.num_objects++;
obs.exists = true;
}
- t.touch(coll, soid);
- t.omap_setheader(coll, soid, osd_op.indata);
+ t->touch(soid);
+ t->omap_setheader(soid, osd_op.indata);
ctx->delta_stats.num_wr++;
}
break;
case CEPH_OSD_OP_OMAPCLEAR:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+ ctx->mod_desc.mark_unrollbackable();
++ctx->num_write;
{
- if (!obs.exists) {
+ if (!obs.exists || oi.is_whiteout()) {
result = -ENOENT;
break;
}
- t.touch(coll, soid);
- t.omap_clear(coll, soid);
+ t->touch(soid);
+ t->omap_clear(soid);
ctx->delta_stats.num_wr++;
}
break;
case CEPH_OSD_OP_OMAPRMKEYS:
+ if (pool.info.ec_pool()) {
+ result = -EOPNOTSUPP;
+ break;
+ }
+ ctx->mod_desc.mark_unrollbackable();
++ctx->num_write;
{
- if (!obs.exists) {
+ if (!obs.exists || oi.is_whiteout()) {
result = -ENOENT;
break;
}
- t.touch(coll, soid);
+ t->touch(soid);
set<string> to_rm;
try {
::decode(to_rm, bp);
@@ -3621,21 +4236,21 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = -EINVAL;
goto fail;
}
- t.omap_rmkeys(coll, soid, to_rm);
+ t->omap_rmkeys(soid, to_rm);
ctx->delta_stats.num_wr++;
}
break;
case CEPH_OSD_OP_COPY_GET_CLASSIC:
++ctx->num_read;
- result = fill_in_copy_get(bp, osd_op, oi, true);
+ result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, true);
if (result == -EINVAL)
goto fail;
break;
case CEPH_OSD_OP_COPY_GET:
++ctx->num_read;
- result = fill_in_copy_get(bp, osd_op, oi, false);
+ result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, false);
if (result == -EINVAL)
goto fail;
break;
@@ -3667,16 +4282,17 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
result = -EINVAL;
break;
}
- hobject_t temp_target = generate_temp_object();
- CopyFromCallback *cb = new CopyFromCallback(ctx, temp_target);
+ CopyFromCallback *cb = new CopyFromCallback(ctx);
ctx->copy_cb = cb;
start_copy(cb, ctx->obc, src, src_oloc, src_version,
- temp_target);
+ op.copy_from.flags,
+ false);
result = -EINPROGRESS;
} else {
// finish
assert(ctx->copy_cb->get_result() >= 0);
- result = finish_copyfrom(ctx);
+ finish_copyfrom(ctx);
+ result = 0;
}
}
break;
@@ -3700,10 +4316,12 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
return result;
}
-int ReplicatedPG::_get_tmap(OpContext *ctx,
- map<string, bufferlist> *out,
- bufferlist *header)
+int ReplicatedPG::_get_tmap(OpContext *ctx, bufferlist *header, bufferlist *vals)
{
+ if (ctx->new_obs.oi.size == 0) {
+ dout(20) << "unable to get tmap for zero sized " << ctx->new_obs.oi.soid << dendl;
+ return -ENODATA;
+ }
vector<OSDOp> nops(1);
OSDOp &newop = nops[0];
newop.op.op = CEPH_OSD_OP_TMAPGET;
@@ -3711,7 +4329,7 @@ int ReplicatedPG::_get_tmap(OpContext *ctx,
try {
bufferlist::iterator i = newop.outdata.begin();
::decode(*header, i);
- ::decode(*out, i);
+ (*vals).substr_of(newop.outdata, i.get_off(), i.get_remaining());
} catch (...) {
dout(20) << "unsuccessful at decoding tmap for " << ctx->new_obs.oi.soid
<< dendl;
@@ -3722,18 +4340,27 @@ int ReplicatedPG::_get_tmap(OpContext *ctx,
return 0;
}
-inline int ReplicatedPG::_delete_head(OpContext *ctx)
+inline int ReplicatedPG::_delete_head(OpContext *ctx, bool no_whiteout)
{
SnapSet& snapset = ctx->new_snapset;
ObjectState& obs = ctx->new_obs;
object_info_t& oi = obs.oi;
const hobject_t& soid = oi.soid;
- ObjectStore::Transaction& t = ctx->op_t;
+ PGBackend::PGTransaction* t = ctx->op_t;
- if (!obs.exists)
+ if (!obs.exists || (obs.oi.is_whiteout() && !no_whiteout))
return -ENOENT;
-
- t.remove(coll, soid);
+
+ if (pool.info.ec_pool()) {
+ if (ctx->mod_desc.rmobject(oi.version.version)) {
+ t->stash(soid, oi.version.version);
+ } else {
+ t->remove(soid);
+ }
+ } else {
+ ctx->mod_desc.mark_unrollbackable();
+ t->remove(soid);
+ }
if (oi.size > 0) {
interval_set<uint64_t> ch;
@@ -3741,14 +4368,26 @@ inline int ReplicatedPG::_delete_head(OpContext *ctx)
ctx->modified_ranges.union_of(ch);
}
- ctx->delta_stats.num_objects--;
+ ctx->delta_stats.num_wr++;
ctx->delta_stats.num_bytes -= oi.size;
-
oi.size = 0;
+
+ // cache: writeback: set whiteout on delete?
+ if (pool.info.cache_mode == pg_pool_t::CACHEMODE_WRITEBACK && !no_whiteout) {
+ dout(20) << __func__ << " setting whiteout on " << soid << dendl;
+ oi.set_flag(object_info_t::FLAG_WHITEOUT);
+ ctx->delta_stats.num_whiteouts++;
+ t->touch(soid);
+ return 0;
+ }
+
+ ctx->delta_stats.num_objects--;
+ if (oi.is_dirty())
+ ctx->delta_stats.num_objects_dirty--;
+ if (oi.is_whiteout())
+ ctx->delta_stats.num_whiteouts--;
snapset.head_exists = false;
obs.exists = false;
-
- ctx->delta_stats.num_wr++;
return 0;
}
@@ -3758,16 +4397,31 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
ObjectState& obs = ctx->new_obs;
object_info_t& oi = obs.oi;
const hobject_t& soid = oi.soid;
- ObjectStore::Transaction& t = ctx->op_t;
+ PGBackend::PGTransaction* t = ctx->op_t;
snapid_t snapid = (uint64_t)op.snap.snapid;
- snapid_t cloneid = 0;
+ hobject_t missing_oid;
dout(10) << "_rollback_to " << soid << " snapid " << snapid << dendl;
ObjectContextRef rollback_to;
int ret = find_object_context(
- hobject_t(soid.oid, soid.get_key(), snapid, soid.hash, info.pgid.pool(), soid.get_namespace()),
- &rollback_to, false, &cloneid);
+ hobject_t(soid.oid, soid.get_key(), snapid, soid.hash, info.pgid.pool(),
+ soid.get_namespace()),
+ &rollback_to, false, &missing_oid);
+ if (ret == -EAGAIN) {
+ /* a different problem, like degraded pool
+ * with not-yet-restored object. We shouldn't have been able
+ * to get here; recovery should have completed first! */
+ assert(is_missing_object(missing_oid));
+ dout(20) << "_rollback_to attempted to roll back to a missing object "
+ << missing_oid << " (requested snapid: ) " << snapid << dendl;
+ wait_for_missing_object(missing_oid, ctx->op);
+ return ret;
+ }
+ if (maybe_handle_cache(ctx->op, rollback_to, ret, missing_oid, true)) {
+ // promoting the rollback src, presumably
+ return -EAGAIN;
+ }
if (ret == -ENOENT || (rollback_to && rollback_to->obs.oi.is_whiteout())) {
// there's no snapshot here, or there's no object.
// if there's no snapshot, we delete the object; otherwise, do nothing.
@@ -3777,19 +4431,9 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
// Cannot delete an object with watchers
ret = -EBUSY;
} else {
- _delete_head(ctx);
+ _delete_head(ctx, false);
ret = 0;
}
- } else if (-EAGAIN == ret) {
- /* a different problem, like degraded pool
- * with not-yet-restored object. We shouldn't have been able
- * to get here; recovery should have completed first! */
- hobject_t rollback_target(soid.oid, soid.get_key(), cloneid, soid.hash,
- info.pgid.pool(), soid.get_namespace());
- assert(is_missing_object(rollback_target));
- dout(20) << "_rollback_to attempted to roll back to a missing object "
- << rollback_target << " (requested snapid: ) " << snapid << dendl;
- wait_for_missing_object(rollback_target, ctx->op);
} else if (ret) {
// ummm....huh? It *can't* return anything else at time of writing.
assert(0 == "unexpected error code in _rollback_to");
@@ -3811,11 +4455,22 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
dout(10) << "_rollback_to deleting " << soid.oid
<< " and rolling back to old snap" << dendl;
- if (obs.exists)
- t.remove(coll, soid);
-
- t.clone(coll,
- rollback_to_sobject, soid);
+ if (pool.info.ec_pool()) {
+ if (obs.exists) {
+ if (ctx->mod_desc.rmobject(oi.version.version)) {
+ t->stash(soid, oi.version.version);
+ } else {
+ t->remove(soid);
+ }
+ }
+ } else {
+ if (obs.exists) {
+ ctx->mod_desc.mark_unrollbackable();
+ t->remove(soid);
+ }
+ }
+ ctx->mod_desc.create();
+ t->clone(rollback_to_sobject, soid);
snapset.head_exists = true;
map<snapid_t, interval_set<uint64_t> >::iterator iter =
@@ -3839,6 +4494,7 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
if (!obs.exists) {
obs.exists = true; //we're about to recreate it
ctx->delta_stats.num_objects++;
+ ctx->delta_stats.num_objects_dirty++;
}
ctx->delta_stats.num_bytes -= obs.oi.size;
ctx->delta_stats.num_bytes += rollback_to->obs.oi.size;
@@ -3849,36 +4505,46 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
return ret;
}
-void ReplicatedPG::_make_clone(ObjectStore::Transaction& t,
- const hobject_t& head, const hobject_t& coid,
- object_info_t *poi)
+void ReplicatedPG::_make_clone(
+ OpContext *ctx,
+ PGBackend::PGTransaction* t,
+ ObjectContextRef obc,
+ const hobject_t& head, const hobject_t& coid,
+ object_info_t *poi)
{
bufferlist bv;
::encode(*poi, bv);
- t.clone(coll, head, coid);
- t.setattr(coll, coid, OI_ATTR, bv);
- t.rmattr(coll, coid, SS_ATTR);
+ t->clone(head, coid);
+ setattr_maybe_cache(obc, ctx, t, OI_ATTR, bv);
+ rmattr_maybe_cache(obc, ctx, t, SS_ATTR);
}
void ReplicatedPG::make_writeable(OpContext *ctx)
{
const hobject_t& soid = ctx->obs->oi.soid;
SnapContext& snapc = ctx->snapc;
- ObjectStore::Transaction t;
+ PGBackend::PGTransaction *t = pgbackend->get_transaction();
// clone?
assert(soid.snap == CEPH_NOSNAP);
dout(20) << "make_writeable " << soid << " snapset=" << ctx->snapset
<< " snapc=" << snapc << dendl;;
- // we will mark the object dirty
- if (ctx->undirty) {
- dout(20) << " clearing DIRTY flag" << dendl;
- ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
- } else {
- dout(20) << " setting DIRTY flag" << dendl;
- ctx->new_obs.oi.set_flag(object_info_t::FLAG_DIRTY);
+ bool was_dirty = ctx->new_obs.oi.is_dirty();
+
+ if (ctx->new_obs.exists) {
+ // we will mark the object dirty
+ if (ctx->undirty) {
+ dout(20) << " clearing DIRTY flag" << dendl;
+ assert(ctx->new_obs.oi.is_dirty());
+ ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
+ --ctx->delta_stats.num_objects_dirty;
+ } else if (!ctx->new_obs.oi.test_flag(object_info_t::FLAG_DIRTY)) {
+ dout(20) << " setting DIRTY flag" << dendl;
+ ctx->new_obs.oi.set_flag(object_info_t::FLAG_DIRTY);
+ ++ctx->delta_stats.num_objects_dirty;
+ }
}
// use newer snapc?
@@ -3913,6 +4579,8 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
ctx->clone_obc->destructor_callback = new C_PG_ObjectContext(this, ctx->clone_obc.get());
ctx->clone_obc->obs.oi = static_snap_oi;
ctx->clone_obc->obs.exists = true;
+ if (pool.info.ec_pool())
+ ctx->clone_obc->attr_cache = ctx->obc->attr_cache;
snap_oi = &ctx->clone_obc->obs.oi;
} else {
snap_oi = &static_snap_oi;
@@ -3921,13 +4589,15 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
snap_oi->prior_version = ctx->obs->oi.version;
snap_oi->copy_user_bits(ctx->obs->oi);
snap_oi->snaps = snaps;
- _make_clone(t, soid, coid, snap_oi);
-
- OSDriver::OSTransaction _t(osdriver.get_transaction(&(ctx->local_t)));
- set<snapid_t> _snaps(snaps.begin(), snaps.end());
- snap_mapper.add_oid(coid, _snaps, &_t);
+ if (was_dirty)
+ snap_oi->set_flag(object_info_t::FLAG_DIRTY);
+ _make_clone(ctx, t, ctx->clone_obc, soid, coid, snap_oi);
ctx->delta_stats.num_objects++;
+ if (snap_oi->is_dirty())
+ ctx->delta_stats.num_objects_dirty++;
+ if (snap_oi->is_whiteout())
+ ctx->delta_stats.num_whiteouts++;
ctx->delta_stats.num_object_clones++;
ctx->new_snapset.clones.push_back(coid.snap);
ctx->new_snapset.clone_size[coid.snap] = ctx->obs->oi.size;
@@ -3947,6 +4617,7 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
ctx->obs->oi.user_version,
osd_reqid_t(), ctx->new_obs.oi.mtime));
::encode(snaps, ctx->log.back().snaps);
+ ctx->log.back().mod_desc.create();
ctx->at_version.version++;
}
@@ -3961,8 +4632,9 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
}
// prepend transaction to op_t
- t.append(ctx->op_t);
- t.swap(ctx->op_t);
+ t->append(ctx->op_t);
+ delete ctx->op_t;
+ ctx->op_t = t;
// update snapset with latest snap context
ctx->new_snapset.seq = snapc.seq;
@@ -4100,8 +4772,7 @@ hobject_t ReplicatedPG::generate_temp_object()
{
ostringstream ss;
ss << "temp_" << info.pgid << "_" << get_role() << "_" << osd->monc->get_global_id() << "_" << (++temp_seq);
- hobject_t hoid(object_t(ss.str()), "", CEPH_NOSNAP, 0, -1, "");
- pgbackend->add_temp_obj(hoid);
+ hobject_t hoid = hobject_t::make_temp(ss.str());
dout(20) << __func__ << " " << hoid << dendl;
return hoid;
}
@@ -4112,11 +4783,6 @@ int ReplicatedPG::prepare_transaction(OpContext *ctx)
const hobject_t& soid = ctx->obs->oi.soid;
- // we'll need this to log
- eversion_t old_version = ctx->obs->oi.version;
-
- bool head_existed = ctx->obs->exists;
-
// valid snap context?
if (!ctx->snapc.is_valid()) {
dout(10) << " invalid snapc " << ctx->snapc << dendl;
@@ -4133,59 +4799,112 @@ int ReplicatedPG::prepare_transaction(OpContext *ctx)
do_osd_op_effects(ctx);
// read-op? done?
- if (ctx->op_t.empty() && !ctx->modify) {
+ if (ctx->op_t->empty() && !ctx->modify) {
unstable_stats.add(ctx->delta_stats, ctx->obc->obs.oi.category);
return result;
}
+ // cache: clear whiteout?
+ if (pool.info.cache_mode == pg_pool_t::CACHEMODE_WRITEBACK) {
+ if (ctx->user_modify &&
+ ctx->obc->obs.oi.is_whiteout()) {
+ dout(10) << __func__ << " clearing whiteout on " << soid << dendl;
+ ctx->new_obs.oi.clear_flag(object_info_t::FLAG_WHITEOUT);
+ --ctx->delta_stats.num_whiteouts;
+ }
+ }
// clone, if necessary
- make_writeable(ctx);
+ if (soid.snap == CEPH_NOSNAP)
+ make_writeable(ctx);
+
+ finish_ctx(ctx,
+ ctx->new_obs.exists ? pg_log_entry_t::MODIFY :
+ pg_log_entry_t::DELETE);
+
+ return result;
+}
+
+void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type)
+{
+ const hobject_t& soid = ctx->obs->oi.soid;
+ dout(20) << __func__ << " " << soid << " " << ctx
+ << " op " << pg_log_entry_t::get_op_name(log_op_type)
+ << dendl;
// snapset
bufferlist bss;
- ::encode(ctx->new_snapset, bss);
- assert(ctx->new_obs.exists == ctx->new_snapset.head_exists);
-
- if (ctx->new_obs.exists) {
- if (!head_existed) {
- // if we logically recreated the head, remove old _snapdir object
- hobject_t snapoid(soid.oid, soid.get_key(), CEPH_SNAPDIR, soid.hash,
- info.pgid.pool(), soid.get_namespace());
- ctx->snapset_obc = get_object_context(snapoid, false);
- if (ctx->snapset_obc && ctx->snapset_obc->obs.exists) {
- ctx->op_t.remove(coll, snapoid);
- dout(10) << " removing old " << snapoid << dendl;
+ if (soid.snap == CEPH_NOSNAP) {
+ ::encode(ctx->new_snapset, bss);
+ assert(ctx->new_obs.exists == ctx->new_snapset.head_exists);
+
+ if (ctx->new_obs.exists) {
+ if (!ctx->obs->exists) {
+ // if we logically recreated the head, remove old _snapdir object
+ hobject_t snapoid(soid.oid, soid.get_key(), CEPH_SNAPDIR, soid.hash,
+ info.pgid.pool(), soid.get_namespace());
+
+ ctx->snapset_obc = get_object_context(snapoid, false);
+ if (ctx->snapset_obc && ctx->snapset_obc->obs.exists) {
+ ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::DELETE, snapoid,
+ ctx->at_version,
+ ctx->obs->oi.version,
+ 0, osd_reqid_t(), ctx->mtime));
+ if (pool.info.ec_pool()) {
+ if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+ ctx->op_t->stash(snapoid, ctx->at_version.version);
+ } else {
+ ctx->op_t->remove(snapoid);
+ }
+ } else {
+ ctx->op_t->remove(snapoid);
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
+ dout(10) << " removing old " << snapoid << dendl;
- ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::DELETE, snapoid, ctx->at_version, old_version,
- 0, osd_reqid_t(), ctx->mtime));
- ctx->at_version.version++;
+ ctx->at_version.version++;
- ctx->snapset_obc->obs.exists = false;
+ ctx->snapset_obc->obs.exists = false;
+ }
+ }
+ } else if (ctx->new_snapset.clones.size()) {
+ // save snapset on _snap
+ hobject_t snapoid(soid.oid, soid.get_key(), CEPH_SNAPDIR, soid.hash,
+ info.pgid.pool(), soid.get_namespace());
+ dout(10) << " final snapset " << ctx->new_snapset
+ << " in " << snapoid << dendl;
+ ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::MODIFY, snapoid,
+ ctx->at_version,
+ ctx->obs->oi.version,
+ 0, osd_reqid_t(), ctx->mtime));
+
+ ctx->snapset_obc = get_object_context(snapoid, true);
+ if (pool.info.ec_pool() && !ctx->snapset_obc->obs.exists) {
+ ctx->log.back().mod_desc.create();
+ } else if (!pool.info.ec_pool()) {
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
+ ctx->snapset_obc->obs.exists = true;
+ ctx->snapset_obc->obs.oi.version = ctx->at_version;
+ ctx->snapset_obc->obs.oi.last_reqid = ctx->reqid;
+ ctx->snapset_obc->obs.oi.mtime = ctx->mtime;
+
+ bufferlist bv(sizeof(ctx->new_obs.oi));
+ ::encode(ctx->snapset_obc->obs.oi, bv);
+ ctx->op_t->touch(snapoid);
+ setattr_maybe_cache(ctx->snapset_obc, ctx, ctx->op_t, OI_ATTR, bv);
+ setattr_maybe_cache(ctx->snapset_obc, ctx, ctx->op_t, SS_ATTR, bss);
+ if (pool.info.ec_pool()) {
+ map<string, boost::optional<bufferlist> > to_set;
+ to_set[SS_ATTR];
+ to_set[OI_ATTR];
+ ctx->log.back().mod_desc.setattrs(to_set);
+ } else {
+ ctx->log.back().mod_desc.mark_unrollbackable();
}
+ ctx->at_version.version++;
}
- } else if (ctx->new_snapset.clones.size()) {
- // save snapset on _snap
- hobject_t snapoid(soid.oid, soid.get_key(), CEPH_SNAPDIR, soid.hash,
- info.pgid.pool(), soid.get_namespace());
- dout(10) << " final snapset " << ctx->new_snapset
- << " in " << snapoid << dendl;
- ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::MODIFY, snapoid, ctx->at_version, old_version,
- 0, osd_reqid_t(), ctx->mtime));
-
- ctx->snapset_obc = get_object_context(snapoid, true);
- ctx->snapset_obc->obs.exists = true;
- ctx->snapset_obc->obs.oi.version = ctx->at_version;
- ctx->snapset_obc->obs.oi.last_reqid = ctx->reqid;
- ctx->snapset_obc->obs.oi.mtime = ctx->mtime;
-
- bufferlist bv(sizeof(ctx->new_obs.oi));
- ::encode(ctx->snapset_obc->obs.oi, bv);
- ctx->op_t.touch(coll, snapoid);
- ctx->op_t.setattr(coll, snapoid, OI_ATTR, bv);
- ctx->op_t.setattr(coll, snapoid, SS_ATTR, bss);
- ctx->at_version.version++;
}
// finish and log the op.
@@ -4200,12 +4919,12 @@ int ReplicatedPG::prepare_transaction(OpContext *ctx)
ctx->user_at_version = ctx->at_version.version;
ctx->new_obs.oi.user_version = ctx->user_at_version;
}
- ctx->bytes_written = ctx->op_t.get_encoded_bytes();
+ ctx->bytes_written = ctx->op_t->get_bytes_written();
if (ctx->new_obs.exists) {
// on the head object
ctx->new_obs.oi.version = ctx->at_version;
- ctx->new_obs.oi.prior_version = old_version;
+ ctx->new_obs.oi.prior_version = ctx->obs->oi.version;
ctx->new_obs.oi.last_reqid = ctx->reqid;
if (ctx->mtime != utime_t()) {
ctx->new_obs.oi.mtime = ctx->mtime;
@@ -4216,40 +4935,97 @@ int ReplicatedPG::prepare_transaction(OpContext *ctx)
bufferlist bv(sizeof(ctx->new_obs.oi));
::encode(ctx->new_obs.oi, bv);
- ctx->op_t.setattr(coll, soid, OI_ATTR, bv);
+ setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, OI_ATTR, bv);
- dout(10) << " final snapset " << ctx->new_snapset
- << " in " << soid << dendl;
- ctx->op_t.setattr(coll, soid, SS_ATTR, bss);
+ if (soid.snap == CEPH_NOSNAP) {
+ dout(10) << " final snapset " << ctx->new_snapset
+ << " in " << soid << dendl;
+ setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, SS_ATTR, bss);
+
+ if (pool.info.ec_pool()) {
+ set<string> changing;
+ changing.insert(OI_ATTR);
+ changing.insert(SS_ATTR);
+ ctx->obc->fill_in_setattrs(changing, &(ctx->mod_desc));
+ }
+ } else {
+ dout(10) << " no snapset (this is a clone)" << dendl;
+ }
}
// append to log
- int logopcode = pg_log_entry_t::MODIFY;
- if (!ctx->new_obs.exists)
- logopcode = pg_log_entry_t::DELETE;
- ctx->log.push_back(pg_log_entry_t(logopcode, soid, ctx->at_version, old_version,
- ctx->user_at_version, ctx->reqid, ctx->mtime));
+ ctx->log.push_back(pg_log_entry_t(log_op_type, soid, ctx->at_version,
+ ctx->obs->oi.version,
+ ctx->user_at_version, ctx->reqid,
+ ctx->mtime));
+ if (soid.snap < CEPH_NOSNAP) {
+ set<snapid_t> _snaps(ctx->new_obs.oi.snaps.begin(),
+ ctx->new_obs.oi.snaps.end());
+ switch (log_op_type) {
+ case pg_log_entry_t::MODIFY:
+ case pg_log_entry_t::PROMOTE:
+ dout(20) << __func__ << " encoding snaps " << ctx->new_obs.oi.snaps
+ << dendl;
+ ::encode(ctx->new_obs.oi.snaps, ctx->log.back().snaps);
+ break;
+ case pg_log_entry_t::CLEAN:
+ dout(20) << __func__ << " encoding snaps " << ctx->new_obs.oi.snaps
+ << dendl;
+ ::encode(ctx->new_obs.oi.snaps, ctx->log.back().snaps);
+ break;
+ default:
+ break;
+ }
+ }
+
+ ctx->log.back().mod_desc.claim(ctx->mod_desc);
// apply new object state.
ctx->obc->obs = ctx->new_obs;
ctx->obc->ssc->snapset = ctx->new_snapset;
- info.stats.stats.add(ctx->delta_stats, ctx->obc->obs.oi.category);
+ info.stats.stats.add(ctx->delta_stats, ctx->obs->oi.category);
- if (backfill_target >= 0) {
- pg_info_t& pinfo = peer_info[backfill_target];
+ for (unsigned i = 0; i < backfill_targets.size() ; ++i) {
+ int bt = backfill_targets[i];
+ pg_info_t& pinfo = peer_info[bt];
if (soid <= pinfo.last_backfill)
- pinfo.stats.stats.add(ctx->delta_stats, ctx->obc->obs.oi.category);
+ pinfo.stats.stats.add(ctx->delta_stats, ctx->obs->oi.category);
else if (soid <= last_backfill_started)
- pending_backfill_updates[soid].stats.add(ctx->delta_stats, ctx->obc->obs.oi.category);
+ pending_backfill_updates[soid].stats.add(ctx->delta_stats,
+ ctx->obs->oi.category);
}
if (scrubber.active && scrubber.is_chunky) {
assert(soid < scrubber.start || soid >= scrubber.end);
if (soid < scrubber.start)
- scrub_cstat.add(ctx->delta_stats, ctx->obc->obs.oi.category);
+ scrub_cstat.add(ctx->delta_stats, ctx->obs->oi.category);
}
+}
- return result;
+void ReplicatedPG::complete_read_ctx(int result, OpContext *ctx)
+{
+ MOSDOp *m = static_cast<MOSDOp*>(ctx->op->get_req());
+ assert(ctx->async_reads_complete());
+ ctx->reply->claim_op_out_data(ctx->ops);
+ ctx->reply->get_header().data_off = ctx->data_off;
+
+ MOSDOpReply *reply = ctx->reply;
+ ctx->reply = NULL;
+
+ if (result >= 0) {
+ log_op_stats(ctx);
+ publish_stats_to_osd();
+
+ // on read, return the current object version
+ reply->set_reply_versions(eversion_t(), ctx->obs->oi.user_version);
+ } else if (result == -ENOENT) {
+ // on ENOENT, set a floor for what the next user version will be.
+ reply->set_enoent_reply_versions(info.last_update, info.last_user_version);
+ }
+
+ reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
+ osd->send_message_osd_client(reply, m->get_connection());
+ close_op_ctx(ctx);
}
// ========================================================================
@@ -4265,6 +5041,8 @@ struct C_Copyfrom : public Context {
tid(0)
{}
void finish(int r) {
+ if (r == -ECANCELED)
+ return;
pg->lock();
if (last_peering_reset == pg->get_last_peering_reset()) {
pg->process_copy_chunk(oid, tid, r);
@@ -4273,9 +5051,29 @@ struct C_Copyfrom : public Context {
}
};
-int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
- object_info_t& oi, bool classic)
+struct C_CopyFrom_AsyncReadCb : public Context {
+ OSDOp *osd_op;
+ object_copy_data_t reply_obj;
+ bool classic;
+ C_CopyFrom_AsyncReadCb(OSDOp *osd_op, bool classic) :
+ osd_op(osd_op), classic(classic) {}
+ void finish(int r) {
+ if (classic) {
+ reply_obj.encode_classic(osd_op->outdata);
+ } else {
+ ::encode(reply_obj, osd_op->outdata);
+ }
+ }
+};
+
+int ReplicatedPG::fill_in_copy_get(
+ OpContext *ctx,
+ bufferlist::iterator& bp,
+ OSDOp& osd_op,
+ ObjectContextRef &obc,
+ bool classic)
{
+ object_info_t& oi = obc->obs.oi;
hobject_t& soid = oi.soid;
int result = 0;
object_copy_cursor_t cursor;
@@ -4289,11 +5087,23 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
return result;
}
- object_copy_data_t reply_obj;
+ bool async_read_started = false;
+ object_copy_data_t _reply_obj;
+ C_CopyFrom_AsyncReadCb *cb = NULL;
+ if (pool.info.ec_pool()) {
+ cb = new C_CopyFrom_AsyncReadCb(&osd_op, classic);
+ }
+ object_copy_data_t &reply_obj = cb ? cb->reply_obj : _reply_obj;
// size, mtime
reply_obj.size = oi.size;
reply_obj.mtime = oi.mtime;
reply_obj.category = oi.category;
+ if (soid.snap < CEPH_NOSNAP) {
+ reply_obj.snaps = oi.snaps;
+ } else {
+ assert(obc->ssc);
+ reply_obj.snap_seq = obc->ssc->snapset.seq;
+ }
// attrs
map<string,bufferlist>& out_attrs = reply_obj.attrs;
@@ -4311,10 +5121,20 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
bufferlist& bl = reply_obj.data;
if (left > 0 && !cursor.data_complete) {
if (cursor.data_offset < oi.size) {
- result = osd->store->read(coll, oi.soid, cursor.data_offset, left, bl);
- if (result < 0)
- return result;
- assert(result <= left);
+ if (cb) {
+ async_read_started = true;
+ ctx->pending_async_reads.push_back(
+ make_pair(
+ make_pair(cursor.data_offset, left),
+ make_pair(&bl, cb)));
+ result = MIN(oi.size - cursor.data_offset, (uint64_t)left);
+ } else {
+ result = pgbackend->objects_read_sync(
+ oi.soid, cursor.data_offset, left, &bl);
+ if (result < 0)
+ return result;
+ }
+ assert(result <= left);
left -= result;
cursor.data_offset += result;
}
@@ -4322,52 +5142,74 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
cursor.data_complete = true;
dout(20) << " got data" << dendl;
}
+ assert(cursor.data_offset <= oi.size);
}
// omap
std::map<std::string,bufferlist>& out_omap = reply_obj.omap;
- if (left > 0 && !cursor.omap_complete) {
- ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(coll, oi.soid);
- assert(iter);
- if (iter->valid()) {
+ if (pool.info.ec_pool()) {
+ cursor.omap_complete = true;
+ } else {
+ if (left > 0 && !cursor.omap_complete) {
+ assert(cursor.data_complete);
+ if (cursor.omap_offset.empty()) {
+ osd->store->omap_get_header(coll, oi.soid, &reply_obj.omap_header);
+ }
+ ObjectMap::ObjectMapIterator iter =
+ osd->store->get_omap_iterator(coll, oi.soid);
+ assert(iter);
iter->upper_bound(cursor.omap_offset);
- for (; left > 0 && iter->valid(); iter->next()) {
- out_omap.insert(make_pair(iter->key(), iter->value()));
- left -= iter->key().length() + 4 + iter->value().length() + 4;
+ if (iter->valid()) {
+ for (; left > 0 && iter->valid(); iter->next()) {
+ out_omap.insert(make_pair(iter->key(), iter->value()));
+ left -= iter->key().length() + 4 + iter->value().length() + 4;
+ }
+ }
+ if (iter->valid()) {
+ cursor.omap_offset = iter->key();
+ } else {
+ cursor.omap_complete = true;
+ dout(20) << " got omap" << dendl;
}
- }
- if (iter->valid()) {
- cursor.omap_offset = iter->key();
- } else {
- cursor.omap_complete = true;
- dout(20) << " got omap" << dendl;
}
}
dout(20) << " cursor.is_complete=" << cursor.is_complete()
- << " " << out_attrs.size() << " attrs"
- << " " << bl.length() << " bytes"
- << " " << out_omap.size() << " keys"
- << dendl;
+ << " " << out_attrs.size() << " attrs"
+ << " " << bl.length() << " bytes"
+ << " " << reply_obj.omap_header.length() << " omap header bytes"
+ << " " << out_omap.size() << " keys"
+ << dendl;
reply_obj.cursor = cursor;
- if (classic) {
- reply_obj.encode_classic(osd_op.outdata);
- } else {
- ::encode(reply_obj, osd_op.outdata);
+ if (!async_read_started) {
+ if (classic) {
+ reply_obj.encode_classic(osd_op.outdata);
+ } else {
+ ::encode(reply_obj, osd_op.outdata);
+ }
+ }
+ if (cb && !async_read_started) {
+ delete cb;
}
result = 0;
return result;
}
void ReplicatedPG::start_copy(CopyCallback *cb, ObjectContextRef obc,
- hobject_t src, object_locator_t oloc, version_t version,
- const hobject_t& temp_dest_oid)
+ hobject_t src, object_locator_t oloc,
+ version_t version, unsigned flags,
+ bool mirror_snapset)
{
const hobject_t& dest = obc->obs.oi.soid;
dout(10) << __func__ << " " << dest
<< " from " << src << " " << oloc << " v" << version
+ << " flags " << flags
+ << (mirror_snapset ? " mirror_snapset" : "")
<< dendl;
+ assert(!mirror_snapset || (src.snap == CEPH_NOSNAP ||
+ src.snap == CEPH_SNAPDIR));
+
// cancel a previous in-progress copy?
if (copy_ops.count(dest)) {
// FIXME: if the src etc match, we could avoid restarting from the
@@ -4376,9 +5218,10 @@ void ReplicatedPG::start_copy(CopyCallback *cb, ObjectContextRef obc,
cancel_copy(cop, false);
}
- CopyOpRef cop(new CopyOp(cb, obc, src, oloc, version, temp_dest_oid));
+ CopyOpRef cop(new CopyOp(cb, obc, src, oloc, version, flags,
+ mirror_snapset));
copy_ops[dest] = cop;
- ++obc->copyfrom_readside;
+ obc->start_block();
_copy_some(obc, cop);
}
@@ -4386,36 +5229,67 @@ void ReplicatedPG::start_copy(CopyCallback *cb, ObjectContextRef obc,
void ReplicatedPG::_copy_some(ObjectContextRef obc, CopyOpRef cop)
{
dout(10) << __func__ << " " << obc << " " << cop << dendl;
+
+ unsigned flags = 0;
+ if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_FLUSH)
+ flags |= CEPH_OSD_FLAG_FLUSH;
+ if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE)
+ flags |= CEPH_OSD_FLAG_IGNORE_CACHE;
+ if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY)
+ flags |= CEPH_OSD_FLAG_IGNORE_OVERLAY;
+
+ C_GatherBuilder gather(g_ceph_context);
+
+ if (cop->cursor.is_initial() && cop->mirror_snapset) {
+ // list snaps too.
+ assert(cop->src.snap == CEPH_NOSNAP);
+ ObjectOperation op;
+ op.list_snaps(&cop->results.snapset, NULL);
+ osd->objecter_lock.Lock();
+ tid_t tid = osd->objecter->read(cop->src.oid, cop->oloc, op,
+ CEPH_SNAPDIR, NULL,
+ flags, gather.new_sub(), NULL);
+ cop->objecter_tid2 = tid;
+ osd->objecter_lock.Unlock();
+ }
+
ObjectOperation op;
- if (cop->user_version) {
- op.assert_version(cop->user_version);
+ if (cop->results.user_version) {
+ op.assert_version(cop->results.user_version);
} else {
// we should learn the version after the first chunk, if we didn't know
// it already!
assert(cop->cursor.is_initial());
}
op.copy_get(&cop->cursor, cct->_conf->osd_copyfrom_max_chunk,
- &cop->size, &cop->mtime, &cop->category,
- &cop->attrs, &cop->data, &cop->omap,
+ &cop->results.object_size, &cop->results.mtime,
+ &cop->results.category,
+ &cop->attrs, &cop->data, &cop->omap_header, &cop->omap,
+ &cop->results.snaps, &cop->results.snap_seq,
&cop->rval);
C_Copyfrom *fin = new C_Copyfrom(this, obc->obs.oi.soid,
get_last_peering_reset());
+ gather.set_finisher(new C_OnFinisher(fin,
+ &osd->objecter_finisher));
+
osd->objecter_lock.Lock();
tid_t tid = osd->objecter->read(cop->src.oid, cop->oloc, op,
- cop->src.snap, NULL, 0,
- new C_OnFinisher(fin,
- &osd->objecter_finisher),
+ cop->src.snap, NULL,
+ flags,
+ gather.new_sub(),
// discover the object version if we don't know it yet
- cop->user_version ? NULL : &cop->user_version);
+ cop->results.user_version ? NULL : &cop->results.user_version);
fin->tid = tid;
cop->objecter_tid = tid;
+ gather.activate();
osd->objecter_lock.Unlock();
}
void ReplicatedPG::process_copy_chunk(hobject_t oid, tid_t tid, int r)
{
- dout(10) << __func__ << " tid " << tid << " " << cpp_strerror(r) << dendl;
+ dout(10) << __func__ << " " << oid << " tid " << tid
+ << " " << cpp_strerror(r) << dendl;
map<hobject_t,CopyOpRef>::iterator p = copy_ops.find(oid);
if (p == copy_ops.end()) {
dout(10) << __func__ << " no copy_op found" << dendl;
@@ -4427,52 +5301,79 @@ void ReplicatedPG::process_copy_chunk(hobject_t oid, tid_t tid, int r)
<< " tid " << cop->objecter_tid << dendl;
return;
}
- ObjectContextRef obc = cop->obc;
- cop->objecter_tid = 0;
- CopyResults results;
- if (r >= 0) {
- assert(cop->rval >= 0);
-
- if (!cop->cursor.is_complete()) {
- // write out what we have so far
- vector<OSDOp> ops;
- tid_t rep_tid = osd->get_tid();
- osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
- OpContext *tctx = new OpContext(OpRequestRef(), reqid, ops, &obc->obs, obc->ssc, this);
- tctx->mtime = ceph_clock_now(g_ceph_context);
- RepGather *repop = new_repop(tctx, obc, rep_tid);
-
- if (cop->temp_cursor.is_initial()) {
- cop->temp_coll = get_temp_coll(&tctx->local_t);
- repop->ctx->new_temp_oid = cop->temp_oid;
+ if (r >= 0 && pool.info.ec_pool() && cop->omap.size()) {
+ r = -EOPNOTSUPP;
+ }
+ cop->objecter_tid = 0;
+ cop->objecter_tid2 = 0; // assume this ordered before us (if it happened)
+ ObjectContextRef& cobc = cop->obc;
+
+ if (r < 0)
+ goto out;
+
+ assert(cop->rval >= 0);
+
+ if (oid.snap < CEPH_NOSNAP) {
+ // verify snap hasn't been deleted
+ vector<snapid_t>::iterator p = cop->results.snaps.begin();
+ while (p != cop->results.snaps.end()) {
+ if (pool.info.is_removed_snap(*p)) {
+ dout(10) << __func__ << " clone snap " << *p << " has been deleted"
+ << dendl;
+ for (vector<snapid_t>::iterator q = p + 1;
+ q != cop->results.snaps.end();
+ ++q)
+ *(q - 1) = *q;
+ cop->results.snaps.resize(cop->results.snaps.size() - 1);
+ } else {
+ ++p;
}
+ }
+ if (cop->results.snaps.empty()) {
+ dout(10) << __func__ << " no more snaps for " << oid << dendl;
+ r = -ENOENT;
+ goto out;
+ }
+ }
- _write_copy_chunk(cop, &tctx->op_t);
-
- issue_repop(repop, repop->ctx->mtime);
- eval_repop(repop);
- repop->put();
+ assert(cop->rval >= 0);
- dout(10) << __func__ << " fetching more" << dendl;
- _copy_some(obc, cop);
- return;
+ if (!cop->cursor.is_complete()) {
+ // write out what we have so far
+ if (cop->temp_cursor.is_initial()) {
+ assert(!cop->results.started_temp_obj);
+ cop->results.started_temp_obj = true;
+ cop->results.temp_oid = generate_temp_object();
+ dout(20) << __func__ << " using temp " << cop->results.temp_oid << dendl;
}
- _build_finish_copy_transaction(cop, results.get<3>());
- results.get<1>() = cop->temp_cursor.data_offset;
+ ObjectContextRef tempobc = get_object_context(cop->results.temp_oid, true);
+ RepGather *repop = simple_repop_create(tempobc);
+ if (cop->temp_cursor.is_initial()) {
+ repop->ctx->new_temp_oid = cop->results.temp_oid;
+ }
+ _write_copy_chunk(cop, repop->ctx->op_t);
+ simple_repop_submit(repop);
+ dout(10) << __func__ << " fetching more" << dendl;
+ _copy_some(cobc, cop);
+ return;
}
- dout(20) << __func__ << " complete; committing" << dendl;
- results.get<0>() = r;
- results.get<4>() = false;
+ dout(20) << __func__ << " success; committing" << dendl;
+ cop->results.final_tx = pgbackend->get_transaction();
+ _build_finish_copy_transaction(cop, cop->results.final_tx);
+
+ out:
+ dout(20) << __func__ << " complete r = " << cpp_strerror(r) << dendl;
+ CopyCallbackResults results(r, &cop->results);
cop->cb->complete(results);
- copy_ops.erase(obc->obs.oi.soid);
- --obc->copyfrom_readside;
- kick_object_context_blocked(obc);
+ copy_ops.erase(cobc->obs.oi.soid);
+ cobc->stop_block();
+ kick_object_context_blocked(cobc);
}
-void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, ObjectStore::Transaction *t)
+void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t)
{
dout(20) << __func__ << " " << cop
<< " " << cop->attrs.size() << " attrs"
@@ -4480,59 +5381,93 @@ void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, ObjectStore::Transaction *t)
<< " " << cop->omap.size() << " keys"
<< dendl;
if (!cop->temp_cursor.attr_complete) {
- t->touch(cop->temp_coll, cop->temp_oid);
- for (map<string,bufferlist>::iterator p = cop->attrs.begin(); p != cop->attrs.end(); ++p)
- t->setattr(cop->temp_coll, cop->temp_oid, string("_") + p->first, p->second);
+ t->touch(cop->results.temp_oid);
+ for (map<string,bufferlist>::iterator p = cop->attrs.begin();
+ p != cop->attrs.end();
+ ++p)
+ t->setattr(
+ cop->results.temp_oid,
+ string("_") + p->first, p->second);
cop->attrs.clear();
}
if (!cop->temp_cursor.data_complete) {
- t->write(cop->temp_coll, cop->temp_oid, cop->temp_cursor.data_offset, cop->data.length(), cop->data);
+ t->append(
+ cop->results.temp_oid,
+ cop->temp_cursor.data_offset,
+ cop->data.length(),
+ cop->data);
cop->data.clear();
}
if (!cop->temp_cursor.omap_complete) {
- t->omap_setkeys(cop->temp_coll, cop->temp_oid, cop->omap);
+ if (cop->omap_header.length()) {
+ t->omap_setheader(
+ cop->results.temp_oid,
+ cop->omap_header);
+ cop->omap_header.clear();
+ }
+ t->omap_setkeys(cop->results.temp_oid, cop->omap);
cop->omap.clear();
}
cop->temp_cursor = cop->cursor;
}
void ReplicatedPG::_build_finish_copy_transaction(CopyOpRef cop,
- ObjectStore::Transaction& t)
+ PGBackend::PGTransaction* t)
{
ObjectState& obs = cop->obc->obs;
-
- if (obs.exists) {
- t.remove(coll, obs.oi.soid);
- }
-
if (cop->temp_cursor.is_initial()) {
// write directly to final object
- cop->temp_coll = coll;
- cop->temp_oid = obs.oi.soid;
- _write_copy_chunk(cop, &t);
+ cop->results.temp_oid = obs.oi.soid;
+ _write_copy_chunk(cop, t);
} else {
// finish writing to temp object, then move into place
- _write_copy_chunk(cop, &t);
- t.collection_move_rename(cop->temp_coll, cop->temp_oid, coll, obs.oi.soid);
- pgbackend->clear_temp_obj(cop->temp_oid);
+ _write_copy_chunk(cop, t);
+ t->rename(cop->results.temp_oid, obs.oi.soid);
}
}
-int ReplicatedPG::finish_copyfrom(OpContext *ctx)
+void ReplicatedPG::finish_copyfrom(OpContext *ctx)
{
dout(20) << "finish_copyfrom on " << ctx->obs->oi.soid << dendl;
ObjectState& obs = ctx->new_obs;
CopyFromCallback *cb = static_cast<CopyFromCallback*>(ctx->copy_cb);
- if (!ctx->obs->exists) {
+ if (pool.info.ec_pool()) {
+ if (obs.exists) {
+ if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+ ctx->op_t->stash(obs.oi.soid, ctx->at_version.version);
+ } else {
+ ctx->op_t->remove(obs.oi.soid);
+ }
+ }
+ ctx->mod_desc.create();
+ } else {
+ if (obs.exists) {
+ ctx->op_t->remove(obs.oi.soid);
+ }
+ ctx->mod_desc.mark_unrollbackable();
+ }
+
+ if (!obs.exists) {
ctx->delta_stats.num_objects++;
obs.exists = true;
}
if (cb->is_temp_obj_used()) {
- ctx->discard_temp_oid = cb->temp_obj;
+ ctx->discard_temp_oid = cb->results->temp_oid;
+ }
+ ctx->op_t->append(cb->results->final_tx);
+ delete cb->results->final_tx;
+ cb->results->final_tx = NULL;
+
+ // CopyFromCallback fills this in for us
+ obs.oi.user_version = ctx->user_at_version;
+
+ // cache: clear whiteout?
+ if (obs.oi.is_whiteout()) {
+ dout(10) << __func__ << " clearing whiteout on " << obs.oi.soid << dendl;
+ obs.oi.clear_flag(object_info_t::FLAG_WHITEOUT);
+ --ctx->delta_stats.num_whiteouts;
}
- ctx->op_t.swap(cb->results.get<3>());
- ctx->op_t.append(cb->results.get<3>());
interval_set<uint64_t> ch;
if (obs.oi.size > 0)
@@ -4546,29 +5481,142 @@ int ReplicatedPG::finish_copyfrom(OpContext *ctx)
}
ctx->delta_stats.num_wr++;
ctx->delta_stats.num_wr_kb += SHIFT_ROUND_UP(obs.oi.size, 10);
+}
- return 0;
+void ReplicatedPG::finish_promote(int r, OpRequestRef op,
+ CopyResults *results, ObjectContextRef obc)
+{
+ const hobject_t& soid = obc->obs.oi.soid;
+ dout(10) << __func__ << " " << soid << " r=" << r
+ << " uv" << results->user_version << dendl;
+
+ if (r == -ECANCELED) {
+ return;
+ }
+
+ if (r == -ENOENT && results->started_temp_obj) {
+ dout(10) << __func__ << " abort; will clean up partial work" << dendl;
+ ObjectContextRef tempobc = get_object_context(results->temp_oid, true);
+ RepGather *repop = simple_repop_create(tempobc);
+ repop->ctx->op_t->remove(results->temp_oid);
+ simple_repop_submit(repop);
+ results->started_temp_obj = false;
+ }
+
+ bool whiteout = false;
+ if (r == -ENOENT &&
+ soid.snap == CEPH_NOSNAP &&
+ (pool.info.cache_mode == pg_pool_t::CACHEMODE_WRITEBACK ||
+ pool.info.cache_mode == pg_pool_t::CACHEMODE_READONLY)) {
+ dout(10) << __func__ << " whiteout " << soid << dendl;
+ whiteout = true;
+ }
+
+ if (r < 0 && !whiteout) {
+ // we need to get rid of the op in the blocked queue
+ map<hobject_t,list<OpRequestRef> >::iterator blocked_iter =
+ waiting_for_blocked_object.find(soid);
+ assert(blocked_iter != waiting_for_blocked_object.end());
+ assert(blocked_iter->second.begin()->get() == op.get());
+ blocked_iter->second.pop_front();
+ if (blocked_iter->second.empty()) {
+ waiting_for_blocked_object.erase(blocked_iter);
+ }
+ osd->reply_op_error(op, r);
+ return;
+ }
+
+ RepGather *repop = simple_repop_create(obc);
+ OpContext *tctx = repop->ctx;
+ tctx->at_version = get_next_version();
+
+ ++tctx->delta_stats.num_objects;
+ if (soid.snap < CEPH_NOSNAP)
+ ++tctx->delta_stats.num_object_clones;
+ tctx->new_obs.exists = true;
+
+ if (whiteout) {
+ // create a whiteout
+ tctx->op_t->touch(soid);
+ tctx->new_obs.oi.set_flag(object_info_t::FLAG_WHITEOUT);
+ ++tctx->delta_stats.num_whiteouts;
+ dout(20) << __func__ << " creating whiteout" << dendl;
+ } else {
+ tctx->op_t->append(results->final_tx);
+ delete results->final_tx;
+ results->final_tx = NULL;
+ if (results->started_temp_obj) {
+ tctx->discard_temp_oid = results->temp_oid;
+ }
+ tctx->new_obs.oi.size = results->object_size;
+ tctx->delta_stats.num_bytes += results->object_size;
+ tctx->new_obs.oi.category = results->category;
+ tctx->new_obs.oi.user_version = results->user_version;
+
+ if (soid.snap != CEPH_NOSNAP) {
+ if (!results->snaps.empty()) {
+ tctx->new_obs.oi.snaps = results->snaps;
+ } else {
+ // we must have read "snap" content from the head object in
+ // the base pool. use snap_seq to construct what snaps should
+ // be for this clone (what is was before we evicted the clean
+ // clone from this pool, and what it will be when we flush and
+ // the clone eventually happens in the base pool).
+ SnapSet& snapset = obc->ssc->snapset;
+ vector<snapid_t>::iterator p = snapset.snaps.begin();
+ while (p != snapset.snaps.end() && *p > soid.snap)
+ ++p;
+ assert(p != snapset.snaps.end() && *p == soid.snap);
+ do {
+ tctx->new_obs.oi.snaps.push_back(*p);
+ ++p;
+ } while (p != snapset.snaps.end() && *p > results->snap_seq);
+ }
+ dout(20) << __func__ << " snaps " << tctx->new_obs.oi.snaps << dendl;
+ assert(!tctx->new_obs.oi.snaps.empty());
+ }
+ }
+
+ if (results->mirror_snapset) {
+ assert(tctx->new_obs.oi.soid.snap == CEPH_NOSNAP);
+ tctx->new_snapset.from_snap_set(results->snapset);
+ }
+ tctx->new_snapset.head_exists = true;
+ dout(20) << __func__ << " new_snapset " << tctx->new_snapset << dendl;
+
+ // take RWWRITE lock for duration of our local write
+ if (!obc->rwstate.get_write_lock()) {
+ assert(0 == "problem!");
+ }
+ tctx->lock_to_release = OpContext::W_LOCK;
+ dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl;
+
+ finish_ctx(tctx, pg_log_entry_t::PROMOTE);
+
+ simple_repop_submit(repop);
}
void ReplicatedPG::cancel_copy(CopyOpRef cop, bool requeue)
{
dout(10) << __func__ << " " << cop->obc->obs.oi.soid
- << " from " << cop->src << " " << cop->oloc << " v" << cop->user_version
- << dendl;
+ << " from " << cop->src << " " << cop->oloc
+ << " v" << cop->results.user_version << dendl;
// cancel objecter op, if we can
if (cop->objecter_tid) {
Mutex::Locker l(osd->objecter_lock);
osd->objecter->op_cancel(cop->objecter_tid);
+ if (cop->objecter_tid2) {
+ osd->objecter->op_cancel(cop->objecter_tid2);
+ }
}
copy_ops.erase(cop->obc->obs.oi.soid);
- --cop->obc->copyfrom_readside;
+ cop->obc->stop_block();
kick_object_context_blocked(cop->obc);
- bool temp_obj_created = !cop->cursor.is_initial();
- CopyResults result(-ECANCELED, 0, temp_obj_created,
- ObjectStore::Transaction(), requeue);
+ cop->results.should_requeue = requeue;
+ CopyCallbackResults result(-ECANCELED, &cop->results);
cop->cb->complete(result);
}
@@ -4584,111 +5632,379 @@ void ReplicatedPG::cancel_copy_ops(bool requeue)
// ========================================================================
-// rep op gather
-
-class C_OSD_OpApplied : public Context {
-public:
+// flush
+//
+// Flush a dirty object in the cache tier by writing it back to the
+// base tier. The sequence looks like:
+//
+// * send a copy-from operation to the base tier to copy the current
+// version of the object
+// * base tier will pull the object via (perhaps multiple) copy-get(s)
+// * on completion, we check if the object has been modified. if so,
+// just reply with -EAGAIN.
+// * try to take a write lock so we can clear the dirty flag. if this
+// fails, wait and retry
+// * start a repop that clears the bit.
+//
+// If we have to wait, we will retry by coming back through the
+// start_flush method. We check if a flush is already in progress
+// and, if so, try to finish it by rechecking the version and trying
+// to clear the dirty bit.
+//
+// In order for the cache-flush (a write op) to not block the copy-get
+// from reading the object, the client *must* set the SKIPRWLOCKS
+// flag.
+//
+// NOTE: normally writes are strictly ordered for the client, but
+// flushes are special in that they can be reordered with respect to
+// other writes. In particular, we can't have a flush request block
+// an update to the cache pool object!
+
+struct C_Flush : public Context {
ReplicatedPGRef pg;
- ReplicatedPG::RepGather *repop;
-
- C_OSD_OpApplied(ReplicatedPG *p, ReplicatedPG::RepGather *rg) :
- pg(p), repop(rg) {
- repop->get();
- }
+ hobject_t oid;
+ epoch_t last_peering_reset;
+ tid_t tid;
+ C_Flush(ReplicatedPG *p, hobject_t o, epoch_t lpr)
+ : pg(p), oid(o), last_peering_reset(lpr),
+ tid(0)
+ {}
void finish(int r) {
- pg->op_applied(repop);
+ if (r == -ECANCELED)
+ return;
+ pg->lock();
+ if (last_peering_reset == pg->get_last_peering_reset()) {
+ pg->finish_flush(oid, tid, r);
+ }
+ pg->unlock();
}
};
-class C_OSD_OpCommit : public Context {
-public:
- ReplicatedPGRef pg;
- ReplicatedPG::RepGather *repop;
+int ReplicatedPG::start_flush(OpContext *ctx, bool blocking)
+{
+ const object_info_t& oi = ctx->obc->obs.oi;
+ const hobject_t& soid = oi.soid;
+ dout(10) << __func__ << " " << soid
+ << " v" << oi.version
+ << " uv" << oi.user_version
+ << " " << (blocking ? "blocking" : "non-blocking/best-effort")
+ << dendl;
- C_OSD_OpCommit(ReplicatedPG *p, ReplicatedPG::RepGather *rg) :
- pg(p), repop(rg) {
- repop->get();
+ // verify there are no (older) check for dirty clones
+ SnapSet& snapset = ctx->obc->ssc->snapset;
+ {
+ dout(20) << " snapset " << snapset << dendl;
+ vector<snapid_t>::reverse_iterator p = snapset.clones.rbegin();
+ while (p != snapset.clones.rend() && *p >= soid.snap)
+ ++p;
+ if (p != snapset.clones.rend()) {
+ hobject_t next = soid;
+ next.snap = *p;
+ assert(next.snap < soid.snap);
+ ObjectContextRef older_obc = get_object_context(next, false);
+ if (older_obc) {
+ dout(20) << __func__ << " next oldest clone is " << older_obc->obs.oi
+ << dendl;
+ if (older_obc->obs.oi.is_dirty()) {
+ dout(10) << __func__ << " next oldest clone is dirty: "
+ << older_obc->obs.oi << dendl;
+ return -EBUSY;
+ }
+ } else {
+ dout(20) << __func__ << " next oldest clone " << next
+ << " is not present; implicitly clean" << dendl;
+ }
+ } else {
+ dout(20) << __func__ << " no older clones" << dendl;
+ }
}
- void finish(int r) {
- pg->op_commit(repop);
+
+ if (blocking)
+ ctx->obc->start_block();
+
+ map<hobject_t,FlushOpRef>::iterator p = flush_ops.find(soid);
+ if (p != flush_ops.end()) {
+ FlushOpRef fop = p->second;
+ if (fop->ctx->op == ctx->op) {
+ // we couldn't take the write lock on a cache-try-flush before;
+ // now we are trying again for the lock.
+ close_op_ctx(fop->ctx); // clean up the previous ctx and use the new one.
+ fop->ctx = ctx;
+ return try_flush_mark_clean(fop);
+ }
+ if (fop->flushed_version == ctx->obc->obs.oi.user_version &&
+ (fop->blocking || !blocking)) {
+ // nonblocking can join anything
+ // blocking can only join a blocking flush
+ dout(20) << __func__ << " piggybacking on existing flush " << dendl;
+ fop->dup_ops.push_back(ctx->op);
+ return -EAGAIN; // clean up this ctx; op will retry later
+ }
+
+ // cancel current flush since it will fail anyway, or because we
+ // are blocking and the existing flush is nonblocking.
+ dout(20) << __func__ << " canceling previous flush; it will fail" << dendl;
+ if (fop->ctx->op)
+ osd->reply_op_error(fop->ctx->op, -EBUSY);
+ while (!fop->dup_ops.empty()) {
+ osd->reply_op_error(fop->dup_ops.front(), -EBUSY);
+ fop->dup_ops.pop_front();
+ }
+ cancel_flush(fop, false);
}
-};
-void ReplicatedPG::apply_repop(RepGather *repop)
+ FlushOpRef fop(new FlushOp);
+ fop->ctx = ctx;
+ fop->flushed_version = oi.user_version;
+ fop->blocking = blocking;
+
+ ObjectOperation o;
+ if (oi.is_whiteout()) {
+ fop->removal = true;
+ o.remove();
+ } else {
+ object_locator_t oloc(soid);
+ o.copy_from(soid.oid.name, soid.snap, oloc, oi.user_version,
+ CEPH_OSD_COPY_FROM_FLAG_FLUSH |
+ CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY |
+ CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE);
+ }
+ C_Flush *fin = new C_Flush(this, soid, get_last_peering_reset());
+ object_locator_t base_oloc(soid);
+ base_oloc.pool = pool.info.tier_of;
+
+ // construct a SnapContext appropriate for this clone/head
+ SnapContext snapc;
+ if (soid.snap == CEPH_NOSNAP) {
+ snapc.seq = snapset.seq;
+ snapc.snaps = snapset.snaps;
+ } else {
+ // we want to only include snaps that are older than the oldest
+ // snap for which we are defined, so that the object appears to
+ // have been written before that.
+ vector<snapid_t>::iterator p = snapset.snaps.begin();
+ while (p != snapset.snaps.end() && *p >= oi.snaps.back())
+ p++;
+ snapc.snaps = vector<snapid_t>(p, snapset.snaps.end());
+ snapc.seq = oi.snaps.back() - 1;
+ }
+
+ osd->objecter_lock.Lock();
+ tid_t tid = osd->objecter->mutate(soid.oid, base_oloc, o, snapc, oi.mtime,
+ CEPH_OSD_FLAG_IGNORE_OVERLAY,
+ NULL,
+ new C_OnFinisher(fin,
+ &osd->objecter_finisher));
+ fin->tid = tid;
+ fop->objecter_tid = tid;
+ osd->objecter_lock.Unlock();
+
+ flush_ops[soid] = fop;
+ return -EINPROGRESS;
+}
+
+void ReplicatedPG::finish_flush(hobject_t oid, tid_t tid, int r)
{
- dout(10) << "apply_repop applying update on " << *repop << dendl;
- assert(!repop->applying);
- assert(!repop->applied);
+ dout(10) << __func__ << " " << oid << " tid " << tid
+ << " " << cpp_strerror(r) << dendl;
+ map<hobject_t,FlushOpRef>::iterator p = flush_ops.find(oid);
+ if (p == flush_ops.end()) {
+ dout(10) << __func__ << " no flush_op found" << dendl;
+ return;
+ }
+ FlushOpRef fop = p->second;
+ if (tid != fop->objecter_tid) {
+ dout(10) << __func__ << " tid " << tid << " != fop " << fop
+ << " tid " << fop->objecter_tid << dendl;
+ return;
+ }
+ ObjectContextRef obc = fop->ctx->obc;
+ fop->objecter_tid = 0;
- repop->applying = true;
+ if (r < 0 && !(r == -ENOENT && fop->removal)) {
+ reply_ctx(fop->ctx, -EBUSY, obc->obs.oi.version,
+ obc->obs.oi.user_version);
+ if (!fop->dup_ops.empty()) {
+ dout(20) << __func__ << " requeueing dups" << dendl;
+ requeue_ops(fop->dup_ops);
+ }
+ flush_ops.erase(oid);
+ return;
+ }
- repop->tls.push_back(&repop->ctx->local_t);
- repop->tls.push_back(&repop->ctx->op_t);
+ r = try_flush_mark_clean(fop);
+ if (r == -EBUSY) {
+ reply_ctx(fop->ctx, -EBUSY, obc->obs.oi.version,
+ obc->obs.oi.user_version);
+ }
+}
- repop->obc->ondisk_write_lock();
- if (repop->ctx->clone_obc)
- repop->ctx->clone_obc->ondisk_write_lock();
+int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
+{
+ ObjectContextRef obc = fop->ctx->obc;
+ const hobject_t& oid = obc->obs.oi.soid;
- bool unlock_snapset_obc = false;
- if (repop->ctx->snapset_obc && repop->ctx->snapset_obc->obs.oi.soid !=
- repop->obc->obs.oi.soid) {
- repop->ctx->snapset_obc->ondisk_write_lock();
- unlock_snapset_obc = true;
+ if (fop->flushed_version != obc->obs.oi.user_version ||
+ !obc->obs.exists) {
+ if (obc->obs.exists)
+ dout(10) << __func__ << " flushed_version " << fop->flushed_version
+ << " != current " << obc->obs.oi.user_version
+ << dendl;
+ else
+ dout(10) << __func__ << " object no longer exists" << dendl;
+
+ if (!fop->dup_ops.empty()) {
+ dout(20) << __func__ << " requeueing dups" << dendl;
+ requeue_ops(fop->dup_ops);
+ }
+ if (fop->blocking) {
+ obc->stop_block();
+ kick_object_context_blocked(obc);
+ }
+ flush_ops.erase(oid);
+ return -EBUSY;
}
- Context *oncommit = new C_OSD_OpCommit(this, repop);
- Context *onapplied = new C_OSD_OpApplied(this, repop);
- Context *onapplied_sync = new C_OSD_OndiskWriteUnlock(
- repop->obc,
- repop->ctx->clone_obc,
- unlock_snapset_obc ? repop->ctx->snapset_obc : ObjectContextRef());
- int r = osd->store->queue_transactions(osr.get(), repop->tls, onapplied, oncommit, onapplied_sync, repop->ctx->op);
- if (r) {
- derr << "apply_repop queue_transactions returned " << r << " on " << *repop << dendl;
- assert(0);
+ // successfully flushed; can we clear the dirty bit?
+ if (!fop->blocking) {
+ // non-blocking: try to take the lock manually, since we don't
+ // have a ctx yet.
+ dout(20) << __func__ << " taking write lock" << dendl;
+ if (!obc->get_write(fop->ctx->op)) {
+ dout(10) << __func__ << " waiting on lock" << dendl;
+ return -EINPROGRESS; // will retry. this ctx is still alive!
+ }
+ } else {
+ dout(20) << __func__ << " already holding write lock: "
+ << obc->rwstate << dendl;
+ assert(obc->rwstate.state == ObjectContext::RWState::RWWRITE);
+ assert(fop->ctx->lock_to_release == OpContext::W_LOCK);
+
+ // let other writes continue
+ obc->stop_block();
+ kick_object_context_blocked(obc);
}
+
+ dout(10) << __func__ << " clearing DIRTY flag for " << oid << dendl;
+ tid_t rep_tid = osd->get_tid();
+ RepGather *repop = new_repop(fop->ctx, obc, rep_tid);
+ OpContext *ctx = fop->ctx;
+ if (!fop->blocking) {
+ ctx->lock_to_release = OpContext::W_LOCK; // we took it above
+ }
+ ctx->at_version = get_next_version();
+
+ ctx->new_obs = obc->obs;
+ ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
+ --ctx->delta_stats.num_objects_dirty;
+
+ finish_ctx(ctx, pg_log_entry_t::CLEAN);
+
+ if (!fop->dup_ops.empty()) {
+ dout(20) << __func__ << " queueing dups for " << ctx->at_version << dendl;
+ list<OpRequestRef>& ls = waiting_for_ondisk[ctx->at_version];
+ ls.splice(ls.end(), fop->dup_ops);
+ }
+
+ simple_repop_submit(repop);
+
+ flush_ops.erase(oid);
+ return -EINPROGRESS;
}
-void ReplicatedPG::op_applied(RepGather *repop)
+void ReplicatedPG::cancel_flush(FlushOpRef fop, bool requeue)
{
- lock();
- dout(10) << "op_applied " << *repop << dendl;
- if (repop->ctx->op)
- repop->ctx->op->mark_event("op_applied");
-
- repop->applying = false;
- repop->applied = true;
+ dout(10) << __func__ << " " << fop->ctx->obc->obs.oi.soid << " tid "
+ << fop->objecter_tid << dendl;
+ if (fop->objecter_tid) {
+ Mutex::Locker l(osd->objecter_lock);
+ osd->objecter->op_cancel(fop->objecter_tid);
+ }
+ if (fop->ctx->op && requeue) {
+ requeue_op(fop->ctx->op);
+ requeue_ops(fop->dup_ops);
+ }
+ if (fop->blocking) {
+ fop->ctx->obc->stop_block();
+ kick_object_context_blocked(fop->ctx->obc);
+ }
+ flush_ops.erase(fop->ctx->obc->obs.oi.soid);
+ close_op_ctx(fop->ctx);
+}
+
+void ReplicatedPG::cancel_flush_ops(bool requeue)
+{
+ dout(10) << __func__ << dendl;
+ map<hobject_t,FlushOpRef>::iterator p = flush_ops.begin();
+ while (p != flush_ops.end()) {
+ cancel_flush((p++)->second, requeue);
+ }
+}
- // (logical) local ack.
- int whoami = osd->get_nodeid();
+// ========================================================================
+// rep op gather
- if (repop->ctx->clone_obc) {
- repop->ctx->clone_obc = ObjectContextRef();
+class C_OSD_RepopApplied : public Context {
+ ReplicatedPGRef pg;
+ boost::intrusive_ptr<ReplicatedPG::RepGather> repop;
+public:
+ C_OSD_RepopApplied(ReplicatedPG *pg, ReplicatedPG::RepGather *repop)
+ : pg(pg), repop(repop) {}
+ void finish(int) {
+ pg->repop_all_applied(repop.get());
}
- if (repop->ctx->snapset_obc) {
- repop->ctx->snapset_obc = ObjectContextRef();
+};
+
+
+void ReplicatedPG::repop_all_applied(RepGather *repop)
+{
+ dout(10) << __func__ << ": repop tid " << repop->rep_tid << " all applied "
+ << dendl;
+ repop->all_applied = true;
+ if (!repop->rep_aborted) {
+ eval_repop(repop);
}
+}
- repop->src_obc.clear();
+class C_OSD_RepopCommit : public Context {
+ ReplicatedPGRef pg;
+ boost::intrusive_ptr<ReplicatedPG::RepGather> repop;
+public:
+ C_OSD_RepopCommit(ReplicatedPG *pg, ReplicatedPG::RepGather *repop)
+ : pg(pg), repop(repop) {}
+ void finish(int) {
+ pg->repop_all_committed(repop.get());
+ }
+};
- if (!repop->aborted) {
- assert(repop->waitfor_ack.count(whoami) ||
- repop->waitfor_disk.count(whoami) == 0); // commit before ondisk
- repop->waitfor_ack.erase(whoami);
+void ReplicatedPG::repop_all_committed(RepGather *repop)
+{
+ dout(10) << __func__ << ": repop tid " << repop->rep_tid << " all committed "
+ << dendl;
+ repop->all_committed = true;
+ if (!repop->rep_aborted) {
if (repop->v != eversion_t()) {
- assert(info.last_update >= repop->v);
- assert(last_update_applied < repop->v);
- last_update_applied = repop->v;
+ last_update_ondisk = repop->v;
+ last_complete_ondisk = repop->pg_local_last_complete;
}
+ eval_repop(repop);
+ }
+}
- // chunky scrub
+void ReplicatedPG::op_applied(const eversion_t &applied_version)
+{
+ dout(10) << "op_applied on primary on version " << applied_version << dendl;
+ if (applied_version == eversion_t())
+ return;
+ assert(applied_version > last_update_applied);
+ assert(applied_version <= info.last_update);
+ last_update_applied = applied_version;
+ if (is_primary()) {
if (scrubber.active && scrubber.is_chunky) {
if (last_update_applied == scrubber.subset_last_update) {
osd->scrub_wq.queue(this);
}
-
- // classic scrub
} else if (last_update_applied == info.last_update && scrubber.block_writes) {
dout(10) << "requeueing scrub for cleanup" << dendl;
scrubber.finalizing = true;
@@ -4697,49 +6013,17 @@ void ReplicatedPG::op_applied(RepGather *repop)
scrubber.waiting_on_whom.insert(osd->whoami);
osd->scrub_wq.queue(this);
}
- }
-
- if (!repop->aborted)
- eval_repop(repop);
-
- repop->put();
- unlock();
-}
-
-void ReplicatedPG::op_commit(RepGather *repop)
-{
- lock();
- if (repop->ctx->op)
- repop->ctx->op->mark_event("op_commit");
-
- if (repop->aborted) {
- dout(10) << "op_commit " << *repop << " -- aborted" << dendl;
- } else if (repop->waitfor_disk.count(osd->get_nodeid()) == 0) {
- dout(10) << "op_commit " << *repop << " -- already marked ondisk" << dendl;
} else {
- dout(10) << "op_commit " << *repop << dendl;
- int whoami = osd->get_nodeid();
-
- repop->waitfor_disk.erase(whoami);
-
- // remove from ack waitfor list too. sub_op_modify_commit()
- // behaves the same in that the COMMIT implies and ACK and there
- // is no separate reply sent.
- repop->waitfor_ack.erase(whoami);
-
- if (repop->v != eversion_t()) {
- last_update_ondisk = repop->v;
- last_complete_ondisk = repop->pg_local_last_complete;
+ dout(10) << "op_applied on replica on version " << applied_version << dendl;
+ if (scrubber.active_rep_scrub) {
+ if (last_update_applied == scrubber.active_rep_scrub->scrub_to) {
+ osd->rep_scrub_wq.queue(scrubber.active_rep_scrub);
+ scrubber.active_rep_scrub = 0;
+ }
}
- eval_repop(repop);
}
-
- repop->put();
- unlock();
}
-
-
void ReplicatedPG::eval_repop(RepGather *repop)
{
MOSDOp *m = NULL;
@@ -4749,27 +6033,23 @@ void ReplicatedPG::eval_repop(RepGather *repop)
if (m)
dout(10) << "eval_repop " << *repop
<< " wants=" << (m->wants_ack() ? "a":"") << (m->wants_ondisk() ? "d":"")
- << (repop->done() ? " DONE" : "")
+ << (repop->rep_done ? " DONE" : "")
<< dendl;
else
dout(10) << "eval_repop " << *repop << " (no op)"
- << (repop->done() ? " DONE" : "")
+ << (repop->rep_done ? " DONE" : "")
<< dendl;
- if (repop->done())
+ if (repop->rep_done)
return;
- // apply?
- if (!repop->applied && !repop->applying)
- apply_repop(repop);
-
if (m) {
// an 'ondisk' reply implies 'ack'. so, prefer to send just one
// ondisk instead of ack followed by ondisk.
// ondisk?
- if (repop->waitfor_disk.empty()) {
+ if (repop->all_committed) {
release_op_ctx_locks(repop->ctx);
@@ -4806,7 +6086,6 @@ void ReplicatedPG::eval_repop(RepGather *repop)
}
reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
dout(10) << " sending commit on " << *repop << " " << reply << dendl;
- assert(entity_name_t::TYPE_OSD != m->get_connection()->peer_type);
osd->send_message_osd_client(reply, m->get_connection());
repop->sent_disk = true;
repop->ctx->op->mark_commit_sent();
@@ -4814,7 +6093,7 @@ void ReplicatedPG::eval_repop(RepGather *repop)
}
// applied?
- if (repop->waitfor_ack.empty()) {
+ if (repop->all_applied) {
// send dup acks, in order
if (waiting_for_ack.count(repop->v)) {
@@ -4859,9 +6138,8 @@ void ReplicatedPG::eval_repop(RepGather *repop)
}
// done.
- if (repop->waitfor_ack.empty() && repop->waitfor_disk.empty() &&
- repop->applied) {
- repop->mark_done();
+ if (repop->all_applied && repop->all_committed) {
+ repop->rep_done = true;
calc_min_last_complete_ondisk();
@@ -4887,74 +6165,135 @@ void ReplicatedPG::issue_repop(RepGather *repop, utime_t now)
{
OpContext *ctx = repop->ctx;
const hobject_t& soid = ctx->obs->oi.soid;
-
+ if (ctx->op &&
+ ((static_cast<MOSDOp *>(
+ ctx->op->get_req()))->get_flags() & CEPH_OSD_FLAG_PARALLELEXEC)) {
+ // replicate original op for parallel execution on replica
+ assert(0 == "broken implementation, do not use");
+ }
dout(7) << "issue_repop rep_tid " << repop->rep_tid
<< " o " << soid
<< dendl;
repop->v = ctx->at_version;
- // add myself to gather set
- repop->waitfor_ack.insert(acting[0]);
- repop->waitfor_disk.insert(acting[0]);
+ for (vector<int>::iterator i = actingbackfill.begin() + 1;
+ i != actingbackfill.end();
+ ++i) {
+ pg_info_t &pinfo = peer_info[*i];
+ // keep peer_info up to date
+ if (pinfo.last_complete == pinfo.last_update)
+ pinfo.last_complete = ctx->at_version;
+ pinfo.last_update = ctx->at_version;
+ }
+ repop->obc->ondisk_write_lock();
+ if (repop->ctx->clone_obc)
+ repop->ctx->clone_obc->ondisk_write_lock();
+
+ bool unlock_snapset_obc = false;
+ if (repop->ctx->snapset_obc && repop->ctx->snapset_obc->obs.oi.soid !=
+ repop->obc->obs.oi.soid) {
+ repop->ctx->snapset_obc->ondisk_write_lock();
+ unlock_snapset_obc = true;
+ }
+
+ repop->ctx->apply_pending_attrs();
+
+ if (pool.info.ec_pool()) {
+ for (vector<pg_log_entry_t>::iterator i = repop->ctx->log.begin();
+ i != repop->ctx->log.end();
+ ++i) {
+ assert(i->mod_desc.can_rollback());
+ assert(!i->mod_desc.empty());
+ }
+ }
+
+ Context *on_all_commit = new C_OSD_RepopCommit(this, repop);
+ Context *on_all_applied = new C_OSD_RepopApplied(this, repop);
+ Context *onapplied_sync = new C_OSD_OndiskWriteUnlock(
+ repop->obc,
+ repop->ctx->clone_obc,
+ unlock_snapset_obc ? repop->ctx->snapset_obc : ObjectContextRef());
+ pgbackend->submit_transaction(
+ soid,
+ repop->ctx->at_version,
+ repop->ctx->op_t,
+ pg_trim_to,
+ repop->ctx->log,
+ onapplied_sync,
+ on_all_applied,
+ on_all_commit,
+ repop->rep_tid,
+ repop->ctx->reqid,
+ repop->ctx->op);
+ repop->ctx->op_t = NULL;
+}
+
+void ReplicatedBackend::issue_op(
+ const hobject_t &soid,
+ const eversion_t &at_version,
+ tid_t tid,
+ osd_reqid_t reqid,
+ eversion_t pg_trim_to,
+ hobject_t new_temp_oid,
+ hobject_t discard_temp_oid,
+ vector<pg_log_entry_t> &log_entries,
+ InProgressOp *op,
+ ObjectStore::Transaction *op_t)
+{
int acks_wanted = CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK;
- if (ctx->op && acting.size() > 1) {
+ if (parent->get_actingbackfill().size() > 1) {
ostringstream ss;
- ss << "waiting for subops from " << vector<int>(acting.begin() + 1, acting.end());
- ctx->op->mark_sub_op_sent(ss.str());
+ ss << "waiting for subops from " <<
+ vector<int>(
+ parent->get_actingbackfill().begin() + 1,
+ parent->get_actingbackfill().end());
+ if (op->op)
+ op->op->mark_sub_op_sent(ss.str());
}
- for (unsigned i=1; i<acting.size(); i++) {
- int peer = acting[i];
- pg_info_t &pinfo = peer_info[peer];
+ for (unsigned i=1; i<parent->get_actingbackfill().size(); i++) {
+ int peer = parent->get_actingbackfill()[i];
+ const pg_info_t &pinfo = parent->get_peer_info().find(peer)->second;
- repop->waitfor_ack.insert(peer);
- repop->waitfor_disk.insert(peer);
+ op->waiting_for_applied.insert(peer);
+ op->waiting_for_commit.insert(peer);
// forward the write/update/whatever
- MOSDSubOp *wr = new MOSDSubOp(repop->ctx->reqid, info.pgid, soid,
- false, acks_wanted,
- get_osdmap()->get_epoch(),
- repop->rep_tid, repop->ctx->at_version);
- if (ctx->op &&
- ((static_cast<MOSDOp *>(ctx->op->get_req()))->get_flags() & CEPH_OSD_FLAG_PARALLELEXEC)) {
- // replicate original op for parallel execution on replica
- assert(0 == "broken implementation, do not use");
- }
+ MOSDSubOp *wr = new MOSDSubOp(
+ reqid, get_info().pgid, soid,
+ false, acks_wanted,
+ get_osdmap()->get_epoch(),
+ tid, at_version);
// ship resulting transaction, log entries, and pg_stats
- if (peer == backfill_target && soid > last_backfill_started &&
- // only skip normal (not temp pool=-1) objects
- soid.pool == (int64_t)info.pgid.pool()) {
+ if (!parent->should_send_op(peer, soid)) {
dout(10) << "issue_repop shipping empty opt to osd." << peer
- <<", object beyond last_backfill_started"
- << last_backfill_started << ", last_backfill is "
- << pinfo.last_backfill << dendl;
+ <<", object " << soid
+ << " beyond MAX(last_backfill_started "
+ << ", pinfo.last_backfill "
+ << pinfo.last_backfill << ")" << dendl;
ObjectStore::Transaction t;
::encode(t, wr->get_data());
} else {
- ::encode(repop->ctx->op_t, wr->get_data());
+ ::encode(*op_t, wr->get_data());
}
- ::encode(repop->ctx->log, wr->logbl);
+ ::encode(log_entries, wr->logbl);
- if (backfill_target >= 0 && backfill_target == peer)
+ if (pinfo.is_incomplete())
wr->pg_stats = pinfo.stats; // reflects backfill progress
else
- wr->pg_stats = info.stats;
+ wr->pg_stats = get_info().stats;
wr->pg_trim_to = pg_trim_to;
- wr->new_temp_oid = repop->ctx->new_temp_oid;
- wr->discard_temp_oid = repop->ctx->discard_temp_oid;
+ wr->new_temp_oid = new_temp_oid;
+ wr->discard_temp_oid = discard_temp_oid;
osd->send_message_osd_cluster(peer, wr, get_osdmap()->get_epoch());
- // keep peer_info up to date
- if (pinfo.last_complete == pinfo.last_update)
- pinfo.last_update = ctx->at_version;
- pinfo.last_update = ctx->at_version;
}
}
@@ -4981,6 +6320,7 @@ ReplicatedPG::RepGather *ReplicatedPG::new_repop(OpContext *ctx, ObjectContextRe
void ReplicatedPG::remove_repop(RepGather *repop)
{
+ dout(20) << __func__ << " " << *repop << dendl;
release_op_ctx_locks(repop->ctx);
repop_map.erase(repop->rep_tid);
repop->put();
@@ -4988,57 +6328,28 @@ void ReplicatedPG::remove_repop(RepGather *repop)
osd->logger->set(l_osd_op_wip, repop_map.size());
}
-void ReplicatedPG::repop_ack(RepGather *repop, int result, int ack_type,
- int fromosd, eversion_t peer_lcod)
+ReplicatedPG::RepGather *ReplicatedPG::simple_repop_create(ObjectContextRef obc)
{
- MOSDOp *m = NULL;
-
- if (repop->ctx->op)
- m = static_cast<MOSDOp *>(repop->ctx->op->get_req());
-
- if (m)
- dout(7) << "repop_ack rep_tid " << repop->rep_tid << " op " << *m
- << " result " << result
- << " ack_type " << ack_type
- << " from osd." << fromosd
- << dendl;
- else
- dout(7) << "repop_ack rep_tid " << repop->rep_tid << " (no op) "
- << " result " << result
- << " ack_type " << ack_type
- << " from osd." << fromosd
- << dendl;
-
- if (ack_type & CEPH_OSD_FLAG_ONDISK) {
- if (repop->ctx->op)
- repop->ctx->op->mark_event("sub_op_commit_rec");
- // disk
- if (repop->waitfor_disk.count(fromosd)) {
- repop->waitfor_disk.erase(fromosd);
- //repop->waitfor_nvram.erase(fromosd);
- repop->waitfor_ack.erase(fromosd);
- peer_last_complete_ondisk[fromosd] = peer_lcod;
- }
-/*} else if (ack_type & CEPH_OSD_FLAG_ONNVRAM) {
- // nvram
- repop->waitfor_nvram.erase(fromosd);
- repop->waitfor_ack.erase(fromosd);*/
- } else {
- // ack
- if (repop->ctx->op)
- repop->ctx->op->mark_event("sub_op_applied_rec");
- repop->waitfor_ack.erase(fromosd);
- }
-
- if (!repop->aborted)
- eval_repop(repop);
+ dout(20) << __func__ << " " << obc->obs.oi.soid << dendl;
+ vector<OSDOp> ops;
+ tid_t rep_tid = osd->get_tid();
+ osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
+ OpContext *ctx = new OpContext(OpRequestRef(), reqid, ops,
+ &obc->obs, obc->ssc, this);
+ ctx->op_t = pgbackend->get_transaction();
+ ctx->mtime = ceph_clock_now(g_ceph_context);
+ ctx->obc = obc;
+ RepGather *repop = new_repop(ctx, obc, rep_tid);
+ return repop;
}
-
-
-
-
-
+void ReplicatedPG::simple_repop_submit(RepGather *repop)
+{
+ dout(20) << __func__ << " " << repop << dendl;
+ issue_repop(repop, repop->ctx->mtime);
+ eval_repop(repop);
+ repop->put();
+}
// -------------------------------------------------------
@@ -5169,16 +6480,15 @@ void ReplicatedPG::handle_watch_timeout(WatchRef watch)
osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
OpContext *ctx = new OpContext(OpRequestRef(), reqid, ops,
&obc->obs, obc->ssc, this);
+ ctx->op_t = pgbackend->get_transaction();
ctx->mtime = ceph_clock_now(cct);
-
- ctx->at_version.epoch = get_osdmap()->get_epoch();
- ctx->at_version.version = pg_log.get_head().version + 1;
+ ctx->at_version = get_next_version();
entity_inst_t nobody;
RepGather *repop = new_repop(ctx, obc, rep_tid);
- ObjectStore::Transaction *t = &ctx->op_t;
+ PGBackend::PGTransaction *t = ctx->op_t;
ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::MODIFY, obc->obs.oi.soid,
ctx->at_version,
@@ -5190,9 +6500,15 @@ void ReplicatedPG::handle_watch_timeout(WatchRef watch)
obc->obs.oi.version = ctx->at_version;
bufferlist bl;
::encode(obc->obs.oi, bl);
- t->setattr(coll, obc->obs.oi.soid, OI_ATTR, bl);
+ setattr_maybe_cache(obc, repop->ctx, t, OI_ATTR, bl);
- append_log(repop->ctx->log, eversion_t(), repop->ctx->local_t);
+ if (pool.info.ec_pool()) {
+ map<string, boost::optional<bufferlist> > to_set;
+ to_set[OI_ATTR] = bl;
+ ctx->log.back().mod_desc.setattrs(to_set);
+ } else {
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
// obc ref swallowed by repop!
issue_repop(repop, repop->ctx->mtime);
@@ -5228,7 +6544,8 @@ ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
pg_log_entry_t::LOST_REVERT));
ObjectContextRef obc = object_contexts.lookup(soid);
if (obc) {
- dout(10) << "get_object_context " << obc << " " << soid << dendl;
+ dout(10) << "get_object_context " << obc << " " << soid
+ << " " << obc->rwstate << dendl;
} else {
// check disk
bufferlist bv;
@@ -5266,7 +6583,27 @@ ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
register_snapset_context(obc->ssc);
populate_obc_watchers(obc);
- dout(10) << "get_object_context " << obc << " " << soid << " 0 -> 1 read " << obc->obs.oi << dendl;
+
+ if (pool.info.ec_pool()) {
+ if (attrs) {
+ for (map<string, bufferptr>::iterator i = attrs->begin();
+ i != attrs->end();
+ ++i) {
+ bufferlist bl;
+ bl.append(i->second);
+ obc->attr_cache.insert(make_pair(i->first, bl));
+ }
+ } else {
+ int r = pgbackend->objects_get_attrs(
+ soid,
+ &obc->attr_cache);
+ assert(r == 0);
+ }
+ }
+
+ dout(10) << "get_object_context " << obc << " " << soid
+ << " " << obc->rwstate
+ << " 0 -> 1 read " << obc->obs.oi << dendl;
}
return obc;
}
@@ -5288,10 +6625,20 @@ void ReplicatedPG::context_registry_on_change()
}
+/*
+ * If we return an error, and set *pmissing, then promoting that
+ * object may help.
+ *
+ * If we return -EAGAIN, we will always set *pmissing to the missing
+ * object to wait for.
+ *
+ * If we return an error but do not set *pmissing, then we know the
+ * object does not exist.
+ */
int ReplicatedPG::find_object_context(const hobject_t& oid,
ObjectContextRef *pobc,
bool can_create,
- snapid_t *psnapid)
+ hobject_t *pmissing)
{
hobject_t head(oid.oid, oid.get_key(), CEPH_NOSNAP, oid.hash,
info.pgid.pool(), oid.get_namespace());
@@ -5302,42 +6649,56 @@ int ReplicatedPG::find_object_context(const hobject_t& oid,
if (oid.snap == CEPH_SNAPDIR) {
// return head or snapdir, whichever exists.
ObjectContextRef obc = get_object_context(head, can_create);
- if (obc && !obc->obs.exists) {
- // ignore it if the obc exists but the object doesn't
- obc = ObjectContextRef();
- }
- if (!obc) {
+ if (!obc || !obc->obs.exists)
obc = get_object_context(snapdir, can_create);
- }
- if (!obc)
+ if (!obc || !obc->obs.exists) {
+ // if we have neither, we would want to promote the head.
+ if (pmissing)
+ *pmissing = head;
return -ENOENT;
+ }
dout(10) << "find_object_context " << oid << " @" << oid.snap << dendl;
*pobc = obc;
// always populate ssc for SNAPDIR...
if (!obc->ssc)
- obc->ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, true, oid.get_namespace());
+ obc->ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, true,
+ oid.get_namespace());
return 0;
}
// want the head?
if (oid.snap == CEPH_NOSNAP) {
ObjectContextRef obc = get_object_context(head, can_create);
- if (!obc)
+ if (!obc) {
+ if (pmissing)
+ *pmissing = head;
return -ENOENT;
+ }
dout(10) << "find_object_context " << oid << " @" << oid.snap << dendl;
*pobc = obc;
if (can_create && !obc->ssc)
- obc->ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, true, oid.get_namespace());
+ obc->ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, true,
+ oid.get_namespace());
return 0;
}
// we want a snap
- SnapSetContext *ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash, can_create, oid.get_namespace());
- if (!ssc)
+ if (pool.info.is_removed_snap(oid.snap)) {
+ dout(10) << __func__ << " snap " << oid.snap << " is removed" << dendl;
return -ENOENT;
+ }
+
+ SnapSetContext *ssc = get_snapset_context(oid.oid, oid.get_key(), oid.hash,
+ can_create, oid.get_namespace());
+ if (!ssc) {
+ dout(20) << __func__ << " " << oid << " no snapset" << dendl;
+ if (pmissing)
+ *pmissing = head; // start by getting the head
+ return -ENOENT;
+ }
dout(10) << "find_object_context " << oid << " @" << oid.snap
<< " snapset " << ssc->snapset << dendl;
@@ -5373,28 +6734,43 @@ int ReplicatedPG::find_object_context(const hobject_t& oid,
ssc->snapset.clones[k] < oid.snap)
k++;
if (k == ssc->snapset.clones.size()) {
- dout(10) << "find_object_context no clones with last >= oid.snap " << oid.snap << " -- DNE" << dendl;
+ dout(10) << "find_object_context no clones with last >= oid.snap "
+ << oid.snap << " -- DNE" << dendl;
put_snapset_context(ssc);
return -ENOENT;
}
hobject_t soid(oid.oid, oid.get_key(), ssc->snapset.clones[k], oid.hash,
info.pgid.pool(), oid.get_namespace());
- put_snapset_context(ssc); // we're done with ssc
- ssc = 0;
-
if (pg_log.get_missing().is_missing(soid)) {
- dout(20) << "find_object_context " << soid << " missing, try again later" << dendl;
- if (psnapid)
- *psnapid = soid.snap;
+ dout(20) << "find_object_context " << soid << " missing, try again later"
+ << dendl;
+ if (pmissing)
+ *pmissing = soid;
+ put_snapset_context(ssc);
return -EAGAIN;
}
- ObjectContextRef obc = get_object_context(soid, false);
- assert(obc);
+ ObjectContextRef obc = get_object_context(soid, false);
+ if (!obc) {
+ dout(20) << __func__ << " missing clone " << soid << dendl;
+ if (pmissing)
+ *pmissing = soid;
+ put_snapset_context(ssc);
+ return -ENOENT;
+ }
+
+ if (!obc->ssc) {
+ obc->ssc = ssc;
+ } else {
+ assert(obc->ssc == ssc);
+ put_snapset_context(ssc);
+ }
+ ssc = 0;
// clone
- dout(20) << "find_object_context " << soid << " snaps " << obc->obs.oi.snaps << dendl;
+ dout(20) << "find_object_context " << soid << " snaps " << obc->obs.oi.snaps
+ << dendl;
snapid_t first = obc->obs.oi.snaps[obc->obs.oi.snaps.size()-1];
snapid_t last = obc->obs.oi.snaps[0];
if (first <= oid.snap) {
@@ -5426,6 +6802,10 @@ void ReplicatedPG::add_object_context_to_pg_stat(ObjectContextRef obc, pg_stat_t
if (oi.soid.snap != CEPH_SNAPDIR)
stat.num_objects++;
+ if (oi.is_dirty())
+ stat.num_objects_dirty++;
+ if (oi.is_whiteout())
+ stat.num_whiteouts++;
if (oi.soid.snap && oi.soid.snap != CEPH_NOSNAP && oi.soid.snap != CEPH_SNAPDIR) {
stat.num_object_clones++;
@@ -5538,7 +6918,7 @@ void ReplicatedPG::put_snapset_context(SnapSetContext *ssc)
// sub op modify
-void ReplicatedPG::sub_op_modify(OpRequestRef op)
+void ReplicatedBackend::sub_op_modify(OpRequestRef op)
{
MOSDSubOp *m = static_cast<MOSDSubOp*>(op->get_req());
assert(m->get_header().type == MSG_OSD_SUBOP);
@@ -5562,24 +6942,19 @@ void ReplicatedPG::sub_op_modify(OpRequestRef op)
<< dendl;
// sanity checks
- assert(m->map_epoch >= info.history.same_interval_since);
- assert(is_active());
- assert(is_replica());
+ assert(m->map_epoch >= get_info().history.same_interval_since);
// we better not be missing this.
- assert(!pg_log.get_missing().is_missing(soid));
+ assert(!parent->get_log().get_missing().is_missing(soid));
- int ackerosd = acting[0];
+ int ackerosd = m->get_source().num();
op->mark_started();
- RepModify *rm = new RepModify;
- rm->pg = this;
- get("RepModify");
+ RepModifyRef rm(new RepModify);
rm->op = op;
- rm->ctx = 0;
rm->ackerosd = ackerosd;
- rm->last_complete = info.last_complete;
+ rm->last_complete = get_info().last_complete;
rm->epoch_started = get_osdmap()->get_epoch();
if (!m->noop) {
@@ -5591,12 +6966,12 @@ void ReplicatedPG::sub_op_modify(OpRequestRef op)
if (m->new_temp_oid != hobject_t()) {
dout(20) << __func__ << " start tracking temp " << m->new_temp_oid << dendl;
- pgbackend->add_temp_obj(m->new_temp_oid);
+ add_temp_obj(m->new_temp_oid);
get_temp_coll(&rm->localt);
}
if (m->discard_temp_oid != hobject_t()) {
dout(20) << __func__ << " stop tracking temp " << m->discard_temp_oid << dendl;
- pgbackend->clear_temp_obj(m->discard_temp_oid);
+ clear_temp_obj(m->discard_temp_oid);
}
::decode(rm->opt, p);
@@ -5608,158 +6983,96 @@ void ReplicatedPG::sub_op_modify(OpRequestRef op)
for (vector<pg_log_entry_t>::iterator i = log.begin();
i != log.end();
++i) {
- if (i->soid.pool == -1)
- i->soid.pool = info.pgid.pool();
+ if (!i->soid.is_max() && i->soid.pool == -1)
+ i->soid.pool = get_info().pgid.pool();
}
- rm->opt.set_pool_override(info.pgid.pool());
+ rm->opt.set_pool_override(get_info().pgid.pool());
}
rm->opt.set_replica();
- info.stats = m->pg_stats;
+ bool update_snaps = false;
if (!rm->opt.empty()) {
// If the opt is non-empty, we infer we are before
// last_backfill (according to the primary, not our
// not-quite-accurate value), and should update the
// collections now. Otherwise, we do it later on push.
- update_snap_map(log, rm->localt);
- }
- append_log(log, m->pg_trim_to, rm->localt);
-
- rm->tls.push_back(&rm->localt);
- rm->tls.push_back(&rm->opt);
-
+ update_snaps = true;
+ }
+ parent->update_stats(m->pg_stats);
+ parent->log_operation(
+ log,
+ m->pg_trim_to,
+ update_snaps,
+ &(rm->localt));
+
rm->bytes_written = rm->opt.get_encoded_bytes();
} else {
+ assert(0);
+ #if 0
// just trim the log
if (m->pg_trim_to != eversion_t()) {
pg_log.trim(m->pg_trim_to, info);
dirty_info = true;
write_if_dirty(rm->localt);
- rm->tls.push_back(&rm->localt);
}
+ #endif
}
op->mark_started();
- Context *oncommit = new C_OSD_RepModifyCommit(rm);
- Context *onapply = new C_OSD_RepModifyApply(rm);
- int r = osd->store->queue_transactions(osr.get(), rm->tls, onapply, oncommit, 0, op);
- if (r) {
- dout(0) << "error applying transaction: r = " << r << dendl;
- assert(0);
- }
+ rm->localt.append(rm->opt);
+ rm->localt.register_on_commit(
+ parent->bless_context(
+ new C_OSD_RepModifyCommit(this, rm)));
+ rm->localt.register_on_applied(
+ parent->bless_context(
+ new C_OSD_RepModifyApply(this, rm)));
+ parent->queue_transaction(&(rm->localt), op);
// op is cleaned up by oncommit/onapply when both are executed
}
-void ReplicatedPG::sub_op_modify_applied(RepModify *rm)
+void ReplicatedBackend::sub_op_modify_applied(RepModifyRef rm)
{
- lock();
rm->op->mark_event("sub_op_applied");
rm->applied = true;
- if (!pg_has_reset_since(rm->epoch_started)) {
- dout(10) << "sub_op_modify_applied on " << rm << " op " << *rm->op->get_req() << dendl;
- MOSDSubOp *m = static_cast<MOSDSubOp*>(rm->op->get_req());
- assert(m->get_header().type == MSG_OSD_SUBOP);
-
- if (!rm->committed) {
- // send ack to acker only if we haven't sent a commit already
- MOSDSubOpReply *ack = new MOSDSubOpReply(m, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
- ack->set_priority(CEPH_MSG_PRIO_HIGH); // this better match commit priority!
- osd->send_message_osd_cluster(rm->ackerosd, ack, get_osdmap()->get_epoch());
- }
-
- if (m->version != eversion_t()) {
- assert(info.last_update >= m->version);
- assert(last_update_applied < m->version);
- last_update_applied = m->version;
- }
- if (scrubber.active_rep_scrub) {
- if (last_update_applied == scrubber.active_rep_scrub->scrub_to) {
- osd->rep_scrub_wq.queue(scrubber.active_rep_scrub);
- scrubber.active_rep_scrub = 0;
- }
- }
- } else {
- dout(10) << "sub_op_modify_applied on " << rm << " op " << *rm->op->get_req()
- << " from epoch " << rm->epoch_started << " < last_peering_reset "
- << last_peering_reset << dendl;
- }
-
- bool done = rm->applied && rm->committed;
- unlock();
- if (done) {
- delete rm->ctx;
- delete rm;
- put("RepModify");
+ dout(10) << "sub_op_modify_applied on " << rm << " op "
+ << *rm->op->get_req() << dendl;
+ MOSDSubOp *m = static_cast<MOSDSubOp*>(rm->op->get_req());
+ assert(m->get_header().type == MSG_OSD_SUBOP);
+
+ if (!rm->committed) {
+ // send ack to acker only if we haven't sent a commit already
+ MOSDSubOpReply *ack = new MOSDSubOpReply(m, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+ ack->set_priority(CEPH_MSG_PRIO_HIGH); // this better match commit priority!
+ osd->send_message_osd_cluster(rm->ackerosd, ack, get_osdmap()->get_epoch());
}
+
+ parent->op_applied(m->version);
}
-void ReplicatedPG::sub_op_modify_commit(RepModify *rm)
+void ReplicatedBackend::sub_op_modify_commit(RepModifyRef rm)
{
- lock();
rm->op->mark_commit_sent();
rm->committed = true;
- if (!pg_has_reset_since(rm->epoch_started)) {
- // send commit.
- dout(10) << "sub_op_modify_commit on op " << *rm->op->get_req()
- << ", sending commit to osd." << rm->ackerosd
- << dendl;
-
- if (get_osdmap()->is_up(rm->ackerosd)) {
- last_complete_ondisk = rm->last_complete;
- MOSDSubOpReply *commit = new MOSDSubOpReply(static_cast<MOSDSubOp*>(rm->op->get_req()), 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ONDISK);
- commit->set_last_complete_ondisk(rm->last_complete);
- commit->set_priority(CEPH_MSG_PRIO_HIGH); // this better match ack priority!
- osd->send_message_osd_cluster(rm->ackerosd, commit, get_osdmap()->get_epoch());
- }
- } else {
- dout(10) << "sub_op_modify_commit " << rm << " op " << *rm->op->get_req()
- << " from epoch " << rm->epoch_started << " < last_peering_reset "
- << last_peering_reset << dendl;
- }
+ // send commit.
+ dout(10) << "sub_op_modify_commit on op " << *rm->op->get_req()
+ << ", sending commit to osd." << rm->ackerosd
+ << dendl;
- log_subop_stats(osd, rm->op, l_osd_sop_w_inb, l_osd_sop_w_lat);
- bool done = rm->applied && rm->committed;
- unlock();
- if (done) {
- delete rm->ctx;
- delete rm;
- put("RepModify");
- }
-}
-
-void ReplicatedPG::sub_op_modify_reply(OpRequestRef op)
-{
- MOSDSubOpReply *r = static_cast<MOSDSubOpReply*>(op->get_req());
- assert(r->get_header().type == MSG_OSD_SUBOPREPLY);
-
- op->mark_started();
-
- // must be replication.
- tid_t rep_tid = r->get_tid();
- int fromosd = r->get_source().num();
+ assert(get_osdmap()->is_up(rm->ackerosd));
+ get_parent()->update_last_complete_ondisk(rm->last_complete);
+ MOSDSubOpReply *commit = new MOSDSubOpReply(static_cast<MOSDSubOp*>(rm->op->get_req()), 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ONDISK);
+ commit->set_last_complete_ondisk(rm->last_complete);
+ commit->set_priority(CEPH_MSG_PRIO_HIGH); // this better match ack priority!
+ osd->send_message_osd_cluster(rm->ackerosd, commit, get_osdmap()->get_epoch());
- if (repop_map.count(rep_tid)) {
- // oh, good.
- repop_ack(repop_map[rep_tid],
- r->get_result(), r->ack_type,
- fromosd,
- r->get_last_complete_ondisk());
- }
+ log_subop_stats(osd, rm->op, l_osd_sop_w_inb, l_osd_sop_w_lat);
}
-
-
-
-
-
-
-
-
// ===========================================================
void ReplicatedBackend::calc_head_subsets(
@@ -6738,6 +8051,7 @@ bool ReplicatedBackend::handle_push_reply(int peer, PushReplyOp &op, PushOp *rep
if (pushing[soid].empty()) {
get_parent()->on_global_recover(soid);
+ pushing.erase(soid);
} else {
dout(10) << "pushed " << soid << ", still waiting for push ack from "
<< pushing[soid].size() << " others" << dendl;
@@ -6843,13 +8157,14 @@ void ReplicatedPG::_committed_pushed_object(
last_complete_ondisk = last_complete;
if (last_complete_ondisk == info.last_update) {
- if (is_replica()) {
+ if (!is_primary()) {
+ // Either we are a replica or backfill target.
// we are fully up to date. tell the primary!
osd->send_message_osd_cluster(get_primary(),
new MOSDPGTrim(get_osdmap()->get_epoch(), info.pgid,
last_complete_ondisk),
get_osdmap()->get_epoch());
- } else if (is_primary()) {
+ } else {
// we are the primary. tell replicas to trim?
if (calc_min_last_complete_ondisk())
trim_peers();
@@ -7020,6 +8335,8 @@ void ReplicatedBackend::sub_op_push(OpRequestRef op)
t->register_on_complete(new C_OSD_SendMessageOnConn(
osd, reply, m->get_connection()));
}
+ t->register_on_applied(
+ new ObjectStore::C_DeleteTransaction(t));
get_parent()->queue_transaction(t);
return;
}
@@ -7062,7 +8379,7 @@ void ReplicatedPG::sub_op_remove(OpRequestRef op)
ObjectStore::Transaction *t = new ObjectStore::Transaction;
remove_snap_mapped_object(*t, m->poid);
- int r = osd->store->queue_transaction(osr.get(), t);
+ int r = osd->store->queue_transaction_and_cleanup(osr.get(), t);
assert(r == 0);
}
@@ -7075,10 +8392,11 @@ eversion_t ReplicatedPG::pick_newest_available(const hobject_t& oid)
v = pg_log.get_missing().missing.find(oid)->second.have;
dout(10) << "pick_newest_available " << oid << " " << v << " on osd." << osd->whoami << " (local)" << dendl;
- for (unsigned i=1; i<acting.size(); ++i) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); ++i) {
+ int peer = actingbackfill[i];
if (!peer_missing[peer].is_missing(oid)) {
- assert(peer == backfill_target);
+ assert(is_backfill_targets(peer));
continue;
}
eversion_t h = peer_missing[peer].missing[oid].have;
@@ -7271,9 +8589,7 @@ void ReplicatedPG::apply_and_flush_repops(bool requeue)
RepGather *repop = repop_queue.front();
repop_queue.pop_front();
dout(10) << " applying repop tid " << repop->rep_tid << dendl;
- if (!repop->applied && !repop->applying)
- apply_repop(repop);
- repop->aborted = true;
+ repop->rep_aborted = true;
if (requeue) {
if (repop->ctx->op) {
@@ -7305,11 +8621,18 @@ void ReplicatedPG::apply_and_flush_repops(bool requeue)
void ReplicatedPG::on_flushed()
{
- pair<hobject_t, ObjectContextRef> i;
- while (object_contexts.get_next(i.first, &i)) {
- derr << "on_flushed: object " << i.first << " obc still alive" << dendl;
+ assert(flushes_in_progress > 0);
+ flushes_in_progress--;
+ if (flushes_in_progress == 0) {
+ requeue_ops(waiting_for_active);
+ }
+ if (!is_active() || !is_primary()) {
+ pair<hobject_t, ObjectContextRef> i;
+ while (object_contexts.get_next(i.first, &i)) {
+ derr << "on_flushed: object " << i.first << " obc still alive" << dendl;
+ }
+ assert(object_contexts.empty());
}
- assert(object_contexts.empty());
pgbackend->on_flushed();
}
@@ -7343,6 +8666,7 @@ void ReplicatedPG::on_shutdown()
unreg_next_scrub();
cancel_copy_ops(false);
+ cancel_flush_ops(false);
apply_and_flush_repops(false);
context_registry_on_change();
@@ -7356,21 +8680,31 @@ void ReplicatedPG::on_shutdown()
void ReplicatedPG::on_activate()
{
- for (unsigned i = 1; i<acting.size(); i++) {
- if (peer_info[acting[i]].last_backfill != hobject_t::get_max()) {
- assert(backfill_target == -1);
- backfill_target = acting[i];
- last_backfill_started = peer_info[acting[i]].last_backfill;
- dout(10) << " chose backfill target osd." << backfill_target
- << " from " << last_backfill_started << dendl;
+ if (!backfill_targets.empty()) {
+ last_backfill_started = earliest_backfill();
+ new_backfill = true;
+ assert(!last_backfill_started.is_max());
+ dout(5) << "on activate: bft=" << backfill_targets
+ << " from " << last_backfill_started << dendl;
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ dout(5) << "target osd." << backfill_targets[i]
+ << " from " << peer_info[backfill_targets[i]].last_backfill
+ << dendl;
}
}
+
+ hit_set_setup();
}
void ReplicatedPG::on_change(ObjectStore::Transaction *t)
{
dout(10) << "on_change" << dendl;
+ if (hit_set && hit_set->insert_count() == 0) {
+ dout(20) << " discarding empty hit_set" << dendl;
+ hit_set_clear();
+ }
+
// requeue everything in the reverse order they should be
// reexamined.
@@ -7379,7 +8713,16 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
context_registry_on_change();
+ for (list<pair<OpRequestRef, OpContext*> >::iterator i =
+ in_progress_async_reads.begin();
+ i != in_progress_async_reads.end();
+ in_progress_async_reads.erase(i++)) {
+ close_op_ctx(i->second);
+ requeue_op(i->first);
+ }
+
cancel_copy_ops(is_primary());
+ cancel_flush_ops(is_primary());
// requeue object waiters
if (is_primary()) {
@@ -7426,8 +8769,17 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
void ReplicatedPG::on_role_change()
{
dout(10) << "on_role_change" << dendl;
+ if (get_role() != 0 && hit_set) {
+ dout(10) << " clearing hit set" << dendl;
+ hit_set_clear();
+ }
}
+void ReplicatedPG::on_pool_change()
+{
+ dout(10) << __func__ << dendl;
+ hit_set_setup();
+}
// clear state. called on recovery completion AND cancellation.
void ReplicatedPG::_clear_recovery_state()
@@ -7575,9 +8927,9 @@ bool ReplicatedPG::start_recovery_ops(
bool deferred_backfill = false;
if (recovering.empty() &&
state_test(PG_STATE_BACKFILL) &&
- backfill_target >= 0 && started < max &&
+ !backfill_targets.empty() && started < max &&
missing.num_missing() == 0 &&
- !waiting_on_backfill) {
+ waiting_on_backfill.empty()) {
if (get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL)) {
dout(10) << "deferring backfill due to NOBACKFILL" << dendl;
deferred_backfill = true;
@@ -7659,7 +9011,7 @@ bool ReplicatedPG::start_recovery_ops(
Backfilled())));
}
- return 0;
+ return false;
}
/**
@@ -7827,6 +9179,7 @@ int ReplicatedPG::prep_object_replica_pushes(
const hobject_t& soid, eversion_t v,
PGBackend::RecoveryHandle *h)
{
+ assert(is_primary());
dout(10) << __func__ << ": on " << soid << dendl;
// NOTE: we know we will get a valid oloc off of disk here.
@@ -7834,8 +9187,9 @@ int ReplicatedPG::prep_object_replica_pushes(
if (!obc) {
pg_log.missing_add(soid, v, eversion_t());
bool uhoh = true;
- for (unsigned i=1; i<acting.size(); i++) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++) {
+ int peer = actingbackfill[i];
if (!peer_missing[peer].is_missing(soid, v)) {
missing_loc[soid].insert(peer);
missing_loc_sources.insert(peer);
@@ -7878,8 +9232,9 @@ int ReplicatedBackend::start_pushes(
{
int pushes = 0;
// who needs it?
- for (unsigned i=1; i<get_parent()->get_acting().size(); i++) {
- int peer = get_parent()->get_acting()[i];
+ assert(get_parent()->get_actingbackfill().size() > 0);
+ for (unsigned i=1; i<get_parent()->get_actingbackfill().size(); i++) {
+ int peer = get_parent()->get_actingbackfill()[i];
map<int, pg_missing_t>::const_iterator j =
get_parent()->get_peer_missing().find(peer);
assert(j != get_parent()->get_peer_missing().end());
@@ -7902,8 +9257,9 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
// this is FAR from an optimal recovery order. pretty lame, really.
- for (unsigned i=1; i<acting.size(); i++) {
- int peer = acting[i];
+ assert(actingbackfill.size() > 0);
+ for (unsigned i=1; i<actingbackfill.size(); i++) {
+ int peer = actingbackfill[i];
map<int, pg_missing_t>::const_iterator pm = peer_missing.find(peer);
assert(pm != peer_missing.end());
map<int, pg_info_t>::const_iterator pi = peer_info.find(peer);
@@ -7954,6 +9310,38 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
return started;
}
+hobject_t ReplicatedPG::earliest_peer_backfill() const
+{
+ hobject_t e = hobject_t::get_max();
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int peer = backfill_targets[i];
+ map<int, BackfillInterval>::const_iterator iter =
+ peer_backfill_info.find(peer);
+ assert(iter != peer_backfill_info.end());
+ if (iter->second.begin < e)
+ e = iter->second.begin;
+ }
+ return e;
+}
+
+bool ReplicatedPG::all_peer_done() const
+{
+ // Primary hasn't got any more objects
+ assert(backfill_info.empty());
+
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ map<int, BackfillInterval>::const_iterator piter =
+ peer_backfill_info.find(bt);
+ assert(piter != peer_backfill_info.end());
+ const BackfillInterval& pbi = piter->second;
+ // See if peer has more to process
+ if (!pbi.extends_to_end() || !pbi.empty())
+ return false;
+ }
+ return true;
+}
+
/**
* recover_backfill
*
@@ -7962,21 +9350,23 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
* backfilled: fully pushed to replica or present in replica's missing set (both
* our copy and theirs).
*
- * All objects on backfill_target in [MIN,peer_backfill_info.begin) are either
+ * All objects on a backfill_target in
+ * [MIN,peer_backfill_info[backfill_target].begin) are either
* not present or backfilled (all removed objects have been removed).
* There may be PG objects in this interval yet to be backfilled.
*
- * All objects in PG in [MIN,backfill_info.begin) have been backfilled to
- * backfill_target. There may be objects on backfill_target yet to be deleted.
+ * All objects in PG in [MIN,backfill_info.begin) have been backfilled to all
+ * backfill_targets. There may be objects on backfill_target(s) yet to be deleted.
*
- * All objects < MIN(peer_backfill_info.begin, backfill_info.begin) in PG are
- * backfilled. No deleted objects in this interval remain on backfill_target.
+ * For a backfill target, all objects < MIN(peer_backfill_info[target].begin,
+ * backfill_info.begin) in PG are backfilled. No deleted objects in this
+ * interval remain on the backfill target.
*
- * All objects <= peer_info[backfill_target].last_backfill have been backfilled
- * to backfill_target
+ * For a backfill target, all objects <= peer_info[target].last_backfill
+ * have been backfilled to target
*
* There *MAY* be objects between last_backfill_started and
- * MIN(peer_backfill_info.begin, backfill_info.begin) in the event that client
+ * MIN(peer_backfill_info[*].begin, backfill_info.begin) in the event that client
* io created objects since the last scan. For this reason, we call
* update_range() again before continuing backfill.
*/
@@ -7984,40 +9374,52 @@ int ReplicatedPG::recover_backfill(
int max,
ThreadPool::TPHandle &handle, bool *work_started)
{
- dout(10) << "recover_backfill (" << max << ")" << dendl;
- assert(backfill_target >= 0);
-
- pg_info_t& pinfo = peer_info[backfill_target];
- BackfillInterval& pbi = peer_backfill_info;
+ dout(10) << "recover_backfill (" << max << ")"
+ << " bft=" << backfill_targets
+ << " last_backfill_started " << last_backfill_started << dendl;
+ assert(!backfill_targets.empty());
// Initialize from prior backfill state
- if (pbi.begin < pinfo.last_backfill) {
- pbi.reset(pinfo.last_backfill);
- backfill_info.reset(pinfo.last_backfill);
+ if (new_backfill) {
+ // on_activate() was called prior to getting here
+ assert(last_backfill_started == earliest_backfill());
+ new_backfill = false;
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ peer_backfill_info[bt].reset(peer_info[bt].last_backfill);
+ }
+ backfill_info.reset(last_backfill_started);
+ }
+
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ dout(10) << "peer osd." << bt
+ << " info " << peer_info[bt]
+ << " interval " << peer_backfill_info[bt].begin
+ << "-" << peer_backfill_info[bt].end
+ << " " << peer_backfill_info[bt].objects.size() << " objects"
+ << dendl;
}
- dout(10) << " peer osd." << backfill_target
- << " last_backfill_started " << last_backfill_started
- << " info " << pinfo
- << " interval " << pbi.begin << "-" << pbi.end
- << " " << pbi.objects.size() << " objects" << dendl;
-
// update our local interval to cope with recent changes
backfill_info.begin = last_backfill_started;
update_range(&backfill_info, handle);
int ops = 0;
- map<hobject_t,
- boost::tuple<eversion_t, eversion_t, ObjectContextRef> > to_push;
- map<hobject_t, eversion_t> to_remove;
+ vector<boost::tuple<hobject_t, eversion_t,
+ ObjectContextRef, vector<int> > > to_push;
+ vector<boost::tuple<hobject_t, eversion_t, int> > to_remove;
set<hobject_t> add_to_stat;
- pbi.trim_to(last_backfill_started);
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ peer_backfill_info[bt].trim_to(last_backfill_started);
+ }
backfill_info.trim_to(last_backfill_started);
- hobject_t backfill_pos = MIN(backfill_info.begin, pbi.begin);
+ hobject_t backfill_pos = MIN(backfill_info.begin, earliest_peer_backfill());
while (ops < max) {
- if (backfill_info.begin <= pbi.begin &&
+ if (backfill_info.begin <= earliest_peer_backfill() &&
!backfill_info.extends_to_end() && backfill_info.empty()) {
hobject_t next = backfill_info.end;
backfill_info.clear();
@@ -8026,55 +9428,130 @@ int ReplicatedPG::recover_backfill(
update_range(&backfill_info, handle);
backfill_info.trim();
}
- backfill_pos = MIN(backfill_info.begin, pbi.begin);
+ backfill_pos = MIN(backfill_info.begin, earliest_peer_backfill());
- dout(20) << " my backfill " << backfill_info.begin << "-" << backfill_info.end
- << " " << backfill_info.objects << dendl;
- dout(20) << " peer backfill " << pbi.begin << "-" << pbi.end << " " << pbi.objects << dendl;
+ dout(20) << " my backfill interval " << backfill_info.begin << "-" << backfill_info.end
+ << " " << backfill_info.objects.size() << " objects"
+ << " " << backfill_info.objects
+ << dendl;
- if (pbi.begin <= backfill_info.begin &&
- !pbi.extends_to_end() && pbi.empty()) {
- dout(10) << " scanning peer osd." << backfill_target << " from " << pbi.end << dendl;
- epoch_t e = get_osdmap()->get_epoch();
- MOSDPGScan *m = new MOSDPGScan(MOSDPGScan::OP_SCAN_GET_DIGEST, e, e, info.pgid,
+ bool sent_scan = false;
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ BackfillInterval& pbi = peer_backfill_info[bt];
+
+ dout(20) << " peer osd." << bt << " backfill " << pbi.begin << "-"
+ << pbi.end << " " << pbi.objects << dendl;
+ if (pbi.begin <= backfill_info.begin &&
+ !pbi.extends_to_end() && pbi.empty()) {
+ dout(10) << " scanning peer osd." << bt << " from " << pbi.end << dendl;
+ epoch_t e = get_osdmap()->get_epoch();
+ MOSDPGScan *m = new MOSDPGScan(MOSDPGScan::OP_SCAN_GET_DIGEST, e, e, info.pgid,
pbi.end, hobject_t());
- osd->send_message_osd_cluster(backfill_target, m, get_osdmap()->get_epoch());
- waiting_on_backfill = true;
- start_recovery_op(pbi.end);
- ops++;
- break;
+ osd->send_message_osd_cluster(bt, m, get_osdmap()->get_epoch());
+ assert(waiting_on_backfill.find(bt) == waiting_on_backfill.end());
+ waiting_on_backfill.insert(bt);
+ sent_scan = true;
+ }
+ }
+
+ // Count simultaneous scans as a single op and let those complete
+ if (sent_scan) {
+ ops++;
+ start_recovery_op(hobject_t::get_max()); // XXX: was pbi.end
+ break;
}
- if (backfill_info.empty() && pbi.empty()) {
- dout(10) << " reached end for both local and peer" << dendl;
+ if (backfill_info.empty() && all_peer_done()) {
+ dout(10) << " reached end for both local and all peers" << dendl;
break;
}
- if (pbi.begin < backfill_info.begin) {
- dout(20) << " removing peer " << pbi.begin << dendl;
- to_remove[pbi.begin] = pbi.objects.begin()->second;
- // Object was degraded, but won't be recovered
- if (waiting_for_degraded_object.count(pbi.begin)) {
- requeue_ops(
- waiting_for_degraded_object[pbi.begin]);
- waiting_for_degraded_object.erase(pbi.begin);
+ // Get object within set of peers to operate on and
+ // the set of targets for which that object applies.
+ hobject_t check = earliest_peer_backfill();
+
+ if (check < backfill_info.begin) {
+
+ vector<int> check_targets;
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ BackfillInterval& pbi = peer_backfill_info[bt];
+ if (pbi.begin == check)
+ check_targets.push_back(bt);
+ }
+ assert(!check_targets.empty());
+
+ dout(20) << " BACKFILL removing " << check
+ << " from peers " << check_targets << dendl;
+ for (unsigned i = 0; i < check_targets.size(); ++i) {
+ int bt = check_targets[i];
+ BackfillInterval& pbi = peer_backfill_info[bt];
+ assert(pbi.begin == check);
+
+ to_remove.push_back(boost::make_tuple(check, pbi.objects.begin()->second, bt));
+ pbi.pop_front();
}
- last_backfill_started = pbi.begin;
- pbi.pop_front();
+ last_backfill_started = check;
// Don't increment ops here because deletions
// are cheap and not replied to unlike real recovery_ops,
// and we can't increment ops without requeueing ourself
// for recovery.
- } else if (pbi.begin == backfill_info.begin) {
+ } else {
eversion_t& obj_v = backfill_info.objects.begin()->second;
- if (pbi.objects.begin()->second != obj_v) {
+
+ vector<int> need_ver_targs, missing_targs, keep_ver_targs, skip_targs;
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ BackfillInterval& pbi = peer_backfill_info[bt];
+ // Find all check peers that have the wrong version
+ if (check == backfill_info.begin && check == pbi.begin) {
+ if (pbi.objects.begin()->second != obj_v) {
+ need_ver_targs.push_back(bt);
+ } else {
+ keep_ver_targs.push_back(bt);
+ }
+ } else {
+ pg_info_t& pinfo = peer_info[bt];
+
+ // Only include peers that we've caught up to their backfill line
+ // otherwise, they only appear to be missing this object
+ // because their pbi.begin > backfill_info.begin.
+ if (backfill_info.begin > pinfo.last_backfill)
+ missing_targs.push_back(bt);
+ else
+ skip_targs.push_back(bt);
+ }
+ }
+
+ if (!keep_ver_targs.empty()) {
+ // These peers have version obj_v
+ dout(20) << " BACKFILL keeping " << check
+ << " with ver " << obj_v
+ << " on peers " << keep_ver_targs << dendl;
+ //assert(!waiting_for_degraded_object.count(check));
+ }
+ if (!need_ver_targs.empty() || !missing_targs.empty()) {
ObjectContextRef obc = get_object_context(backfill_info.begin, false);
assert(obc);
if (obc->get_backfill_read()) {
- dout(20) << " replacing peer " << pbi.begin << " with local "
- << obj_v << dendl;
- to_push[pbi.begin] = boost::make_tuple(
- obj_v, pbi.objects.begin()->second, obc);
+ if (!need_ver_targs.empty()) {
+ dout(20) << " BACKFILL replacing " << check
+ << " with ver " << obj_v
+ << " to peers " << need_ver_targs << dendl;
+ }
+ if (!missing_targs.empty()) {
+ dout(20) << " BACKFILL pushing " << backfill_info.begin
+ << " with ver " << obj_v
+ << " to peers " << missing_targs << dendl;
+ }
+ vector<int> all_push = need_ver_targs;
+ all_push.insert(all_push.end(), missing_targs.begin(), missing_targs.end());
+
+ to_push.push_back(
+ boost::tuple<hobject_t, eversion_t, ObjectContextRef, vector<int> >
+ (backfill_info.begin, obj_v, obc, all_push));
+ // Count all simultaneous pushes of the same object as a single op
ops++;
} else {
*work_started = true;
@@ -8082,44 +9559,26 @@ int ReplicatedPG::recover_backfill(
<< "; could not get rw_manager lock" << dendl;
break;
}
- } else {
- dout(20) << " keeping peer " << pbi.begin << " "
- << pbi.objects.begin()->second << dendl;
- // Object was degraded, but won't be recovered
- if (waiting_for_degraded_object.count(pbi.begin)) {
- requeue_ops(waiting_for_degraded_object[pbi.begin]);
- waiting_for_degraded_object.erase(pbi.begin);
- }
}
- last_backfill_started = pbi.begin;
- add_to_stat.insert(pbi.begin);
+ dout(20) << "need_ver_targs=" << need_ver_targs
+ << " keep_ver_targs=" << keep_ver_targs << dendl;
+ dout(20) << "backfill_targets=" << backfill_targets
+ << " missing_targs=" << missing_targs
+ << " skip_targs=" << skip_targs << dendl;
+
+ last_backfill_started = backfill_info.begin;
+ add_to_stat.insert(backfill_info.begin); // XXX: Only one for all pushes?
backfill_info.pop_front();
- pbi.pop_front();
- } else {
- ObjectContextRef obc = get_object_context(backfill_info.begin, false);
- assert(obc);
- if (obc->get_backfill_read()) {
- dout(20) << " pushing local " << backfill_info.begin << " "
- << backfill_info.objects.begin()->second
- << " to peer osd." << backfill_target << dendl;
- to_push[backfill_info.begin] =
- boost::make_tuple(
- backfill_info.objects.begin()->second,
- eversion_t(),
- obc);
- add_to_stat.insert(backfill_info.begin);
- last_backfill_started = backfill_info.begin;
- backfill_info.pop_front();
- ops++;
- } else {
- *work_started = true;
- dout(20) << "backfill blocking on " << backfill_info.begin
- << "; could not get rw_manager lock" << dendl;
- break;
+ vector<int> check_targets = need_ver_targs;
+ check_targets.insert(check_targets.end(), keep_ver_targs.begin(), keep_ver_targs.end());
+ for (unsigned i = 0; i < check_targets.size(); ++i) {
+ int bt = check_targets[i];
+ BackfillInterval& pbi = peer_backfill_info[bt];
+ pbi.pop_front();
}
}
}
- backfill_pos = MIN(backfill_info.begin, pbi.begin);
+ backfill_pos = MIN(backfill_info.begin, earliest_peer_backfill());
for (set<hobject_t>::iterator i = add_to_stat.begin();
i != add_to_stat.end();
@@ -8129,33 +9588,24 @@ int ReplicatedPG::recover_backfill(
add_object_context_to_pg_stat(obc, &stat);
pending_backfill_updates[*i] = stat;
}
- for (map<hobject_t, eversion_t>::iterator i = to_remove.begin();
- i != to_remove.end();
- ++i) {
+ for (unsigned i = 0; i < to_remove.size(); ++i) {
handle.reset_tp_timeout();
// ordered before any subsequent updates
- send_remove_op(i->first, i->second, backfill_target);
+ send_remove_op(to_remove[i].get<0>(), to_remove[i].get<1>(), to_remove[i].get<2>());
- pending_backfill_updates[i->first]; // add empty stat!
+ pending_backfill_updates[to_remove[i].get<0>()]; // add empty stat!
}
PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
- map<int, vector<PushOp> > pushes;
- for (map<hobject_t,
- boost::tuple<eversion_t, eversion_t, ObjectContextRef> >::iterator i =
- to_push.begin();
- i != to_push.end();
- ++i) {
+ for (unsigned i = 0; i < to_push.size(); ++i) {
handle.reset_tp_timeout();
- prep_backfill_object_push(
- i->first, i->second.get<0>(), i->second.get<1>(), i->second.get<2>(),
- backfill_target, h);
+ prep_backfill_object_push(to_push[i].get<0>(), to_push[i].get<1>(),
+ to_push[i].get<2>(), to_push[i].get<3>(), h);
}
pgbackend->run_recovery_op(h, cct->_conf->osd_recovery_op_priority);
- dout(5) << "backfill_pos is " << backfill_pos << " and pinfo.last_backfill is "
- << pinfo.last_backfill << dendl;
+ dout(5) << "backfill_pos is " << backfill_pos << dendl;
for (set<hobject_t>::iterator i = backfills_in_flight.begin();
i != backfills_in_flight.end();
++i) {
@@ -8164,15 +9614,23 @@ int ReplicatedPG::recover_backfill(
hobject_t next_backfill_to_complete = backfills_in_flight.size() ?
*(backfills_in_flight.begin()) : backfill_pos;
- hobject_t new_last_backfill = pinfo.last_backfill;
+ hobject_t new_last_backfill = earliest_backfill();
+ dout(10) << "starting new_last_backfill at " << new_last_backfill << dendl;
for (map<hobject_t, pg_stat_t>::iterator i = pending_backfill_updates.begin();
i != pending_backfill_updates.end() &&
i->first < next_backfill_to_complete;
pending_backfill_updates.erase(i++)) {
- pinfo.stats.add(i->second);
assert(i->first > new_last_backfill);
+ for (unsigned j = 0; j < backfill_targets.size(); ++j) {
+ int bt = backfill_targets[j];
+ pg_info_t& pinfo = peer_info[bt];
+ //Add stats to all peers that were missing object
+ if (i->first > pinfo.last_backfill)
+ pinfo.stats.add(i->second);
+ }
new_last_backfill = i->first;
}
+ dout(10) << "possible new_last_backfill at " << new_last_backfill << dendl;
/* If last_backfill is snapdir, we know that head necessarily cannot exist,
* therefore it's safe to bump the snap up to NOSNAP. This is necessary
@@ -8192,46 +9650,60 @@ int ReplicatedPG::recover_backfill(
new_last_backfill = backfill_pos;
last_backfill_started = backfill_pos;
}
- if (new_last_backfill > pinfo.last_backfill) {
- pinfo.last_backfill = new_last_backfill;
- epoch_t e = get_osdmap()->get_epoch();
- MOSDPGBackfill *m = NULL;
- if (pinfo.last_backfill.is_max()) {
- m = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_FINISH, e, e, info.pgid);
- // Use default priority here, must match sub_op priority
- /* pinfo.stats might be wrong if we did log-based recovery on the
- * backfilled portion in addition to continuing backfill.
- */
- pinfo.stats = info.stats;
- start_recovery_op(hobject_t::get_max());
- } else {
- m = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_PROGRESS, e, e, info.pgid);
- // Use default priority here, must match sub_op priority
+ dout(10) << "final new_last_backfill at " << new_last_backfill << dendl;
+
+ // If new_last_backfill == MAX, then we will send OP_BACKFILL_FINISH to
+ // all the backfill targets. Otherwise, we will move last_backfill up on
+ // those targets need it and send OP_BACKFILL_PROGRESS to them.
+ for (unsigned i = 0; i < backfill_targets.size(); ++i) {
+ int bt = backfill_targets[i];
+ pg_info_t& pinfo = peer_info[bt];
+
+ if (new_last_backfill > pinfo.last_backfill) {
+ pinfo.last_backfill = new_last_backfill;
+ epoch_t e = get_osdmap()->get_epoch();
+ MOSDPGBackfill *m = NULL;
+ if (pinfo.last_backfill.is_max()) {
+ m = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_FINISH, e, e, info.pgid);
+ // Use default priority here, must match sub_op priority
+ /* pinfo.stats might be wrong if we did log-based recovery on the
+ * backfilled portion in addition to continuing backfill.
+ */
+ pinfo.stats = info.stats;
+ start_recovery_op(hobject_t::get_max());
+ } else {
+ m = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_PROGRESS, e, e, info.pgid);
+ // Use default priority here, must match sub_op priority
+ }
+ m->last_backfill = pinfo.last_backfill;
+ m->stats = pinfo.stats;
+ osd->send_message_osd_cluster(bt, m, get_osdmap()->get_epoch());
+ dout(10) << " peer osd." << bt
+ << " num_objects now " << pinfo.stats.stats.sum.num_objects
+ << " / " << info.stats.stats.sum.num_objects << dendl;
}
- m->last_backfill = pinfo.last_backfill;
- m->stats = pinfo.stats;
- osd->send_message_osd_cluster(backfill_target, m, get_osdmap()->get_epoch());
}
- dout(10) << " peer num_objects now " << pinfo.stats.stats.sum.num_objects
- << " / " << info.stats.stats.sum.num_objects << dendl;
if (ops)
*work_started = true;
return ops;
}
void ReplicatedPG::prep_backfill_object_push(
- hobject_t oid, eversion_t v, eversion_t have,
+ hobject_t oid, eversion_t v,
ObjectContextRef obc,
- int peer,
+ vector<int> peers,
PGBackend::RecoveryHandle *h)
{
- dout(10) << "push_backfill_object " << oid << " v " << v << " to osd." << peer << dendl;
+ dout(10) << "push_backfill_object " << oid << " v " << v << " to peers " << peers << dendl;
+ assert(!peers.empty());
backfills_in_flight.insert(oid);
- map<int, pg_missing_t>::iterator bpm = peer_missing.find(backfill_target);
- assert(bpm != peer_missing.end());
- bpm->second.add(oid, eversion_t(), eversion_t());
+ for (unsigned int i = 0 ; i < peers.size(); ++i) {
+ map<int, pg_missing_t>::iterator bpm = peer_missing.find(peers[i]);
+ assert(bpm != peer_missing.end());
+ bpm->second.add(oid, eversion_t(), eversion_t());
+ }
assert(!recovering.count(oid));
@@ -8271,6 +9743,16 @@ void ReplicatedPG::update_range(
dout(10) << __func__<< ": bi is current " << dendl;
assert(bi->version == info.last_update);
} else if (bi->version >= info.log_tail) {
+ if (pg_log.get_log().empty()) {
+ /* Because we don't move log_tail on split, the log might be
+ * empty even if log_tail != last_update. However, the only
+ * way to get here with an empty log is if log_tail is actually
+ * eversion_t(), because otherwise the entry which changed
+ * last_update since the last scan would have to be present.
+ */
+ assert(bi->version == eversion_t());
+ return;
+ }
assert(!pg_log.get_log().empty());
dout(10) << __func__<< ": bi is old, (" << bi->version
<< ") can be updated with log" << dendl;
@@ -8373,8 +9855,8 @@ void ReplicatedPG::check_local()
struct stat st;
int r = osd->store->stat(coll, p->soid, &st);
if (r != -ENOENT) {
- dout(10) << "Object " << p->soid << " exists, but should have been "
- << "deleted" << dendl;
+ derr << __func__ << " " << p->soid << " exists, but should have been "
+ << "deleted" << dendl;
assert(0 == "erroneously present object");
}
} else {
@@ -8385,6 +9867,283 @@ void ReplicatedPG::check_local()
+// ===========================
+// hit sets
+
+hobject_t ReplicatedPG::get_hit_set_current_object(utime_t stamp)
+{
+ ostringstream ss;
+ ss << "hit_set_" << info.pgid << "_current_" << stamp;
+ hobject_t hoid(sobject_t(ss.str(), CEPH_NOSNAP), "",
+ info.pgid.ps(), info.pgid.pool(),
+ cct->_conf->osd_hit_set_namespace);
+ dout(20) << __func__ << " " << hoid << dendl;
+ return hoid;
+}
+
+hobject_t ReplicatedPG::get_hit_set_archive_object(utime_t start, utime_t end)
+{
+ ostringstream ss;
+ ss << "hit_set_" << info.pgid << "_archive_" << start << "_" << end;
+ hobject_t hoid(sobject_t(ss.str(), CEPH_NOSNAP), "",
+ info.pgid.ps(), info.pgid.pool(),
+ cct->_conf->osd_hit_set_namespace);
+ dout(20) << __func__ << " " << hoid << dendl;
+ return hoid;
+}
+
+void ReplicatedPG::hit_set_clear()
+{
+ dout(20) << __func__ << dendl;
+ hit_set.reset(NULL);
+ hit_set_start_stamp = utime_t();
+}
+
+void ReplicatedPG::hit_set_setup()
+{
+ if (!is_primary() ||
+ !pool.info.hit_set_count ||
+ !pool.info.hit_set_period ||
+ pool.info.hit_set_params.get_type() == HitSet::TYPE_NONE) {
+ hit_set_clear();
+ //hit_set_remove_all(); // FIXME: implement me soon
+ return;
+ }
+
+ // FIXME: discard any previous data for now
+ hit_set_create();
+
+ // include any writes we know about from the pg log. this doesn't
+ // capture reads, but it is better than nothing!
+ hit_set_apply_log();
+}
+
+void ReplicatedPG::hit_set_create()
+{
+ utime_t now = ceph_clock_now(NULL);
+ // make a copy of the params to modify
+ HitSet::Params params(pool.info.hit_set_params);
+
+ dout(20) << __func__ << " " << params << dendl;
+ if (pool.info.hit_set_params.get_type() == HitSet::TYPE_BLOOM) {
+ BloomHitSet::Params *p =
+ static_cast<BloomHitSet::Params*>(params.impl.get());
+
+ // convert false positive rate so it holds up across the full period
+ p->set_fpp(p->get_fpp() / pool.info.hit_set_count);
+ if (p->get_fpp() <= 0.0)
+ p->set_fpp(.01); // fpp cannot be zero!
+
+ // if we don't have specified size, estimate target size based on the
+ // previous bin!
+ if (p->target_size == 0 && hit_set) {
+ utime_t dur = now - hit_set_start_stamp;
+ unsigned unique = hit_set->approx_unique_insert_count();
+ dout(20) << __func__ << " previous set had approx " << unique
+ << " unique items over " << dur << " seconds" << dendl;
+ p->target_size = (double)unique * (double)pool.info.hit_set_period
+ / (double)dur;
+ }
+ if (p->target_size < static_cast<uint64_t>(g_conf->osd_hit_set_min_size))
+ p->target_size = g_conf->osd_hit_set_min_size;
+
+ p->seed = now.sec();
+
+ dout(10) << __func__ << " target_size " << p->target_size
+ << " fpp " << p->get_fpp() << dendl;
+ }
+ hit_set.reset(new HitSet(params));
+ hit_set_start_stamp = now;
+}
+
+/**
+ * apply log entries to set
+ *
+ * this would only happen after peering, to at least capture writes
+ * during an interval that was potentially lost.
+ */
+bool ReplicatedPG::hit_set_apply_log()
+{
+ if (!hit_set)
+ return false;
+
+ eversion_t to = info.last_update;
+ eversion_t from = info.hit_set.current_last_update;
+ if (to <= from) {
+ dout(20) << __func__ << " no update" << dendl;
+ return false;
+ }
+
+ dout(20) << __func__ << " " << to << " .. " << info.last_update << dendl;
+ list<pg_log_entry_t>::const_reverse_iterator p = pg_log.get_log().log.rbegin();
+ while (p != pg_log.get_log().log.rend() && p->version > to)
+ ++p;
+ while (p != pg_log.get_log().log.rend() && p->version > from) {
+ hit_set->insert(p->soid);
+ ++p;
+ }
+
+ return true;
+}
+
+void ReplicatedPG::hit_set_persist()
+{
+ dout(10) << __func__ << dendl;
+ bufferlist bl;
+
+ utime_t now = ceph_clock_now(cct);
+ RepGather *repop;
+ hobject_t oid;
+ bool reset = false;
+
+ if (!info.hit_set.current_info.begin)
+ info.hit_set.current_info.begin = hit_set_start_stamp;
+ if (hit_set->is_full() ||
+ hit_set_start_stamp + pool.info.hit_set_period <= now) {
+ // archive
+ hit_set->seal();
+ ::encode(*hit_set, bl);
+ info.hit_set.current_info.end = now;
+ oid = get_hit_set_archive_object(info.hit_set.current_info.begin,
+ info.hit_set.current_info.end);
+ dout(20) << __func__ << " archive " << oid << dendl;
+ reset = true;
+ } else {
+ // persist snapshot of current hitset
+ ::encode(*hit_set, bl);
+ oid = get_hit_set_current_object(now);
+ dout(20) << __func__ << " checkpoint " << oid << dendl;
+ }
+
+ ObjectContextRef obc = get_object_context(oid, true);
+ repop = simple_repop_create(obc);
+ OpContext *ctx = repop->ctx;
+ ctx->at_version = get_next_version();
+
+ if (info.hit_set.current_last_stamp != utime_t()) {
+ // FIXME: we cheat slightly here by bundling in a remove on a object
+ // other the RepGather object. we aren't carrying an ObjectContext for
+ // the deleted object over this period.
+ hobject_t old_obj =
+ get_hit_set_current_object(info.hit_set.current_last_stamp);
+ ctx->log.push_back(
+ pg_log_entry_t(pg_log_entry_t::DELETE,
+ old_obj,
+ ctx->at_version,
+ info.hit_set.current_last_update,
+ 0,
+ osd_reqid_t(),
+ ctx->mtime));
+ if (pool.info.ec_pool()) {
+ if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+ ctx->op_t->stash(old_obj, ctx->at_version.version);
+ } else {
+ ctx->op_t->remove(old_obj);
+ }
+ } else {
+ ctx->op_t->remove(old_obj);
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
+ ++ctx->at_version.version;
+
+ struct stat st;
+ int r = osd->store->stat(coll, old_obj, &st);
+ assert(r == 0);
+ --ctx->delta_stats.num_objects;
+ ctx->delta_stats.num_bytes -= st.st_size;
+ }
+
+ info.hit_set.current_last_update = info.last_update; // *after* above remove!
+ info.hit_set.current_info.version = ctx->at_version;
+ if (reset) {
+ info.hit_set.history.push_back(info.hit_set.current_info);
+ hit_set_create();
+ info.hit_set.current_info = pg_hit_set_info_t();
+ info.hit_set.current_last_stamp = utime_t();
+ } else {
+ info.hit_set.current_last_stamp = now;
+ }
+
+ // fabricate an object_info_t and SnapSet
+ obc->obs.oi.version = ctx->at_version;
+ obc->obs.oi.mtime = now;
+ obc->obs.oi.size = bl.length();
+ obc->obs.exists = true;
+
+ ctx->new_obs = obc->obs;
+ ctx->new_snapset.head_exists = true;
+
+ ctx->delta_stats.num_objects++;
+ ctx->delta_stats.num_bytes += bl.length();
+
+ bufferlist bss;
+ ::encode(ctx->new_snapset, bss);
+ bufferlist boi(sizeof(ctx->new_obs.oi));
+ ::encode(ctx->new_obs.oi, boi);
+
+ ctx->op_t->append(oid, 0, bl.length(), bl);
+ setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, OI_ATTR, boi);
+ setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, SS_ATTR, bss);
+ ctx->log.push_back(
+ pg_log_entry_t(
+ pg_log_entry_t::MODIFY,
+ oid,
+ ctx->at_version,
+ ctx->obs->oi.version,
+ 0,
+ osd_reqid_t(),
+ ctx->mtime)
+ );
+ if (pool.info.ec_pool()) {
+ ctx->log.back().mod_desc.create();
+ } else {
+ ctx->log.back().mod_desc.mark_unrollbackable();
+ }
+
+ hit_set_trim(repop, pool.info.hit_set_count);
+
+ info.stats.stats.add(ctx->delta_stats, string());
+
+ simple_repop_submit(repop);
+}
+
+void ReplicatedPG::hit_set_trim(RepGather *repop, unsigned max)
+{
+ for (unsigned num = info.hit_set.history.size(); num > max; --num) {
+ list<pg_hit_set_info_t>::iterator p = info.hit_set.history.begin();
+ assert(p != info.hit_set.history.end());
+ hobject_t oid = get_hit_set_archive_object(p->begin, p->end);
+ dout(20) << __func__ << " removing " << oid << dendl;
+ ++repop->ctx->at_version.version;
+ repop->ctx->log.push_back(
+ pg_log_entry_t(pg_log_entry_t::DELETE,
+ oid,
+ repop->ctx->at_version,
+ p->version,
+ 0,
+ osd_reqid_t(),
+ repop->ctx->mtime));
+ if (pool.info.ec_pool()) {
+ if (repop->ctx->log.back().mod_desc.rmobject(
+ repop->ctx->at_version.version)) {
+ repop->ctx->op_t->stash(oid, repop->ctx->at_version.version);
+ } else {
+ repop->ctx->op_t->remove(oid);
+ }
+ } else {
+ repop->ctx->op_t->remove(oid);
+ repop->ctx->log.back().mod_desc.mark_unrollbackable();
+ }
+ info.hit_set.history.pop_front();
+
+ struct stat st;
+ int r = osd->store->stat(coll, oid, &st);
+ assert(r == 0);
+ --repop->ctx->delta_stats.num_objects;
+ repop->ctx->delta_stats.num_bytes -= st.st_size;
+ }
+}
+
// ==========================================================================================
// SCRUB
@@ -8483,6 +10242,11 @@ void ReplicatedPG::_scrub(ScrubMap& scrubmap)
stat.num_bytes += p->second.size;
+ if (oi.is_dirty())
+ ++stat.num_objects_dirty;
+ if (oi.is_whiteout())
+ ++stat.num_whiteouts;
+
//bufferlist data;
//osd->store->read(c, poid, 0, 0, data);
//assert(data.length() == p->size);
@@ -8555,22 +10319,29 @@ void ReplicatedPG::_scrub_finish()
dout(10) << mode << " got "
<< scrub_cstat.sum.num_objects << "/" << info.stats.stats.sum.num_objects << " objects, "
<< scrub_cstat.sum.num_object_clones << "/" << info.stats.stats.sum.num_object_clones << " clones, "
+ << scrub_cstat.sum.num_objects_dirty << "/" << info.stats.stats.sum.num_objects_dirty << " dirty, "
<< scrub_cstat.sum.num_bytes << "/" << info.stats.stats.sum.num_bytes << " bytes."
<< dendl;
if (scrub_cstat.sum.num_objects != info.stats.stats.sum.num_objects ||
scrub_cstat.sum.num_object_clones != info.stats.stats.sum.num_object_clones ||
+ (scrub_cstat.sum.num_objects_dirty != info.stats.stats.sum.num_objects_dirty &&
+ !info.stats.dirty_stats_invalid) ||
+ scrub_cstat.sum.num_whiteouts != info.stats.stats.sum.num_whiteouts ||
scrub_cstat.sum.num_bytes != info.stats.stats.sum.num_bytes) {
osd->clog.error() << info.pgid << " " << mode
<< " stat mismatch, got "
<< scrub_cstat.sum.num_objects << "/" << info.stats.stats.sum.num_objects << " objects, "
<< scrub_cstat.sum.num_object_clones << "/" << info.stats.stats.sum.num_object_clones << " clones, "
+ << scrub_cstat.sum.num_objects_dirty << "/" << info.stats.stats.sum.num_objects_dirty << " dirty, "
+ << scrub_cstat.sum.num_whiteouts << "/" << info.stats.stats.sum.num_whiteouts << " whiteouts, "
<< scrub_cstat.sum.num_bytes << "/" << info.stats.stats.sum.num_bytes << " bytes.\n";
++scrubber.shallow_errors;
if (repair) {
++scrubber.fixed;
info.stats.stats = scrub_cstat;
+ info.stats.dirty_stats_invalid = false;
publish_stats_to_osd();
share_pg_info();
}
@@ -8692,11 +10463,8 @@ boost::statechart::result ReplicatedPG::TrimmingObjects::react(const SnapTrim&)
assert(repop);
repop->queue_snap_trimmer = true;
- pg->append_log(repop->ctx->log, eversion_t(), repop->ctx->local_t);
- pg->issue_repop(repop, repop->ctx->mtime);
- pg->eval_repop(repop);
-
- repops.insert(repop);
+ repops.insert(repop->get());
+ pg->simple_repop_submit(repop);
return discard_event();
}
/* WaitingOnReplicasObjects */
@@ -8730,7 +10498,7 @@ boost::statechart::result ReplicatedPG::WaitingOnReplicas::react(const SnapTrim&
for (set<RepGather *>::iterator i = repops.begin();
i != repops.end();
repops.erase(i++)) {
- if (!(*i)->applied || !(*i)->waitfor_ack.empty()) {
+ if (!(*i)->all_applied) {
return discard_event();
} else {
(*i)->put();
@@ -8749,7 +10517,7 @@ boost::statechart::result ReplicatedPG::WaitingOnReplicas::react(const SnapTrim&
ObjectStore::Transaction *t = new ObjectStore::Transaction;
pg->dirty_big_info = true;
pg->write_if_dirty(*t);
- int tr = pg->osd->store->queue_transaction(pg->osr.get(), t);
+ int tr = pg->osd->store->queue_transaction_and_cleanup(pg->osr.get(), t);
assert(tr == 0);
context<SnapTrimmer>().need_share_pg_info = true;
@@ -8759,6 +10527,61 @@ boost::statechart::result ReplicatedPG::WaitingOnReplicas::react(const SnapTrim&
return transit< NotTrimming >();
}
+void ReplicatedPG::setattr_maybe_cache(
+ ObjectContextRef obc,
+ OpContext *op,
+ PGBackend::PGTransaction *t,
+ const string &key,
+ bufferlist &val)
+{
+ if (pool.info.ec_pool()) {
+ op->pending_attrs[obc][key] = val;
+ }
+ t->setattr(obc->obs.oi.soid, key, val);
+}
+
+void ReplicatedPG::rmattr_maybe_cache(
+ ObjectContextRef obc,
+ OpContext *op,
+ PGBackend::PGTransaction *t,
+ const string &key)
+{
+ if (pool.info.ec_pool()) {
+ op->pending_attrs[obc][key] = boost::optional<bufferlist>();
+ }
+ t->rmattr(obc->obs.oi.soid, key);
+}
+
+int ReplicatedPG::getattr_maybe_cache(
+ ObjectContextRef obc,
+ const string &key,
+ bufferlist *val)
+{
+ if (pool.info.ec_pool()) {
+ map<string, bufferlist>::iterator i = obc->attr_cache.find(key);
+ if (i != obc->attr_cache.end()) {
+ if (val)
+ *val = i->second;
+ return 0;
+ } else {
+ return -ENODATA;
+ }
+ }
+ return pgbackend->objects_get_attr(obc->obs.oi.soid, key, val);
+}
+
+int ReplicatedPG::getattrs_maybe_cache(
+ ObjectContextRef obc,
+ map<string, bufferlist> *out)
+{
+ if (pool.info.ec_pool()) {
+ if (out)
+ *out = obc->attr_cache;
+ return 0;
+ }
+ return pgbackend->objects_get_attrs(obc->obs.oi.soid, out);
+}
+
void intrusive_ptr_add_ref(ReplicatedPG *pg) { pg->get("intptr"); }
void intrusive_ptr_release(ReplicatedPG *pg) { pg->put("intptr"); }
@@ -8766,3 +10589,6 @@ void intrusive_ptr_release(ReplicatedPG *pg) { pg->put("intptr"); }
uint64_t get_with_id(ReplicatedPG *pg) { return pg->get_with_id(); }
void put_with_id(ReplicatedPG *pg, uint64_t id) { return pg->put_with_id(id); }
#endif
+
+void intrusive_ptr_add_ref(ReplicatedPG::RepGather *repop) { repop->get(); }
+void intrusive_ptr_release(ReplicatedPG::RepGather *repop) { repop->put(); }
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index d1779d1..208ee40 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -23,8 +23,9 @@
#include "include/assert.h"
#include "common/cmdparse.h"
-#include "PG.h"
+#include "HitSet.h"
#include "OSD.h"
+#include "PG.h"
#include "Watch.h"
#include "OpRequest.h"
@@ -39,6 +40,9 @@
class MOSDSubOpReply;
+class CopyFromCallback;
+class PromoteCallback;
+
class ReplicatedPG;
void intrusive_ptr_add_ref(ReplicatedPG *pg);
void intrusive_ptr_release(ReplicatedPG *pg);
@@ -96,36 +100,67 @@ public:
struct OpContext;
class CopyCallback;
+ /**
+ * CopyResults stores the object metadata of interest to a copy initiator.
+ */
+ struct CopyResults {
+ utime_t mtime; ///< the copy source's mtime
+ uint64_t object_size; ///< the copied object's size
+ bool started_temp_obj; ///< true if the callback needs to delete temp object
+ hobject_t temp_oid; ///< temp object (if any)
+ /**
+ * Final transaction; if non-empty the callback must execute it before any
+ * other accesses to the object (in order to complete the copy).
+ */
+ PGBackend::PGTransaction *final_tx;
+ string category; ///< The copy source's category
+ version_t user_version; ///< The copy source's user version
+ bool should_requeue; ///< op should be requeued on cancel
+ vector<snapid_t> snaps; ///< src's snaps (if clone)
+ snapid_t snap_seq; ///< src's snap_seq (if head)
+ librados::snap_set_t snapset; ///< src snapset (if head)
+ bool mirror_snapset;
+ CopyResults() : object_size(0), started_temp_obj(false),
+ user_version(0), should_requeue(false),
+ mirror_snapset(false) {}
+ };
+
struct CopyOp {
CopyCallback *cb;
ObjectContextRef obc;
hobject_t src;
object_locator_t oloc;
- version_t user_version;
+ unsigned flags;
+ bool mirror_snapset;
+
+ CopyResults results;
tid_t objecter_tid;
+ tid_t objecter_tid2;
object_copy_cursor_t cursor;
- uint64_t size;
- utime_t mtime;
- string category;
map<string,bufferlist> attrs;
bufferlist data;
+ bufferlist omap_header;
map<string,bufferlist> omap;
int rval;
- coll_t temp_coll;
- hobject_t temp_oid;
object_copy_cursor_t temp_cursor;
- CopyOp(CopyCallback *cb_, ObjectContextRef _obc, hobject_t s, object_locator_t l,
- version_t v, const hobject_t& dest)
- : cb(cb_), obc(_obc), src(s), oloc(l), user_version(v),
+ CopyOp(CopyCallback *cb_, ObjectContextRef _obc, hobject_t s,
+ object_locator_t l,
+ version_t v,
+ unsigned f,
+ bool ms)
+ : cb(cb_), obc(_obc), src(s), oloc(l), flags(f),
+ mirror_snapset(ms),
objecter_tid(0),
- size(0),
- rval(-1),
- temp_oid(dest)
- {}
+ objecter_tid2(0),
+ rval(-1)
+ {
+ results.user_version = v;
+ results.mirror_snapset = mirror_snapset;
+ }
};
typedef boost::shared_ptr<CopyOp> CopyOpRef;
@@ -135,69 +170,42 @@ public:
* one and give an instance of the class to start_copy.
*
* The implementer is responsible for making sure that the CopyCallback
- * can associate itself with the correct copy operation. The presence
- * of the closing Transaction ensures that write operations can be performed
- * atomically with the copy being completed (which doing them in separate
- * transactions would not allow); if you are doing the copy for a read
- * op you will have to generate a separate op to finish the copy with.
+ * can associate itself with the correct copy operation.
*/
- /// return code, total object size, data in temp object?, final Transaction, should requeue Op
- typedef boost::tuple<int, size_t, bool, ObjectStore::Transaction, bool> CopyResults;
- class CopyCallback : public GenContext<CopyResults&> {
+ typedef boost::tuple<int, CopyResults*> CopyCallbackResults;
+ class CopyCallback : public GenContext<CopyCallbackResults> {
protected:
CopyCallback() {}
/**
* results.get<0>() is the return code: 0 for success; -ECANCELLED if
* the operation was cancelled by the local OSD; -errno for other issues.
- * results.get<1>() is the total size of the object (for updating pg stats)
- * results.get<2>() indicates whether we have already written data to
- * the temp object (so it needs to get cleaned up, if the return code
- * indicates a failure)
- * results.get<3>() is a Transaction; if non-empty you need to perform
- * its results before any other accesses to the object in order to
- * complete the copy.
- * results.get<4>() is a bool; if true you must requeue the client Op
- * after processing the rest of the results (this will only be true
- * in conjunction with an ECANCELED return code).
+ * results.get<1>() is a pointer to a CopyResults object, which you are
+ * responsible for deleting.
*/
- virtual void finish(CopyResults& results_) = 0;
+ virtual void finish(CopyCallbackResults results_) = 0;
public:
/// Provide the final size of the copied object to the CopyCallback
virtual ~CopyCallback() {};
};
- class CopyFromCallback: public CopyCallback {
- public:
- CopyResults results;
- OpContext *ctx;
- hobject_t temp_obj;
- CopyFromCallback(OpContext *ctx_, const hobject_t& temp_obj_) :
- ctx(ctx_), temp_obj(temp_obj_) {}
- ~CopyFromCallback() {}
-
- virtual void finish(CopyResults& results_) {
- results = results_;
- int r = results.get<0>();
- if (r >= 0) {
- ctx->pg->execute_ctx(ctx);
- }
- ctx->copy_cb = NULL;
- if (r < 0) {
- if (r != -ECANCELED) { // on cancel just toss it out; client resends
- ctx->pg->osd->reply_op_error(ctx->op, r);
- } else if (results_.get<4>()) {
- ctx->pg->requeue_op(ctx->op);
- }
- ctx->pg->close_op_ctx(ctx);
- }
- }
-
- bool is_temp_obj_used() { return results.get<2>(); }
- uint64_t get_data_size() { return results.get<1>(); }
- int get_result() { return results.get<0>(); }
- };
friend class CopyFromCallback;
+ friend class PromoteCallback;
+
+ struct FlushOp {
+ OpContext *ctx; ///< the parent OpContext
+ list<OpRequestRef> dup_ops; ///< dup flush requests
+ version_t flushed_version; ///< user version we are flushing
+ tid_t objecter_tid; ///< copy-from request tid
+ int rval; ///< copy-from result
+ bool blocking; ///< whether we are blocking updates
+ bool removal; ///< we are removing the backend object
+
+ FlushOp()
+ : ctx(NULL), objecter_tid(0), rval(0),
+ blocking(false), removal(false) {}
+ };
+ typedef boost::shared_ptr<FlushOp> FlushOpRef;
boost::scoped_ptr<PGBackend> pgbackend;
PGBackend *get_pgbackend() {
@@ -231,7 +239,7 @@ public:
template <typename T>
class BlessedGenContext : public GenContext<T> {
- ReplicatedPG *pg;
+ ReplicatedPGRef pg;
GenContext<T> *c;
epoch_t e;
public:
@@ -247,7 +255,7 @@ public:
}
};
class BlessedContext : public Context {
- ReplicatedPG *pg;
+ ReplicatedPGRef pg;
Context *c;
epoch_t e;
public:
@@ -274,14 +282,16 @@ public:
void send_message(int to_osd, Message *m) {
osd->send_message_osd_cluster(to_osd, m, get_osdmap()->get_epoch());
}
- void queue_transaction(ObjectStore::Transaction *t) {
- osd->store->queue_transaction(osr.get(), t);
+ void queue_transaction(ObjectStore::Transaction *t, OpRequestRef op) {
+ list<ObjectStore::Transaction *> tls;
+ tls.push_back(t);
+ osd->store->queue_transaction(osr.get(), t, 0, 0, 0, op);
}
epoch_t get_epoch() {
return get_osdmap()->get_epoch();
}
- const vector<int> &get_acting() {
- return acting;
+ const vector<int> &get_actingbackfill() {
+ return actingbackfill;
}
std::string gen_dbg_prefix() const { return gen_prefix(); }
@@ -314,6 +324,43 @@ public:
map<string, bufferptr> &attrs) {
return get_object_context(hoid, true, &attrs);
}
+ void log_operation(
+ vector<pg_log_entry_t> &logv,
+ const eversion_t &trim_to,
+ bool transaction_applied,
+ ObjectStore::Transaction *t) {
+ append_log(logv, trim_to, *t, transaction_applied);
+ }
+
+ void op_applied(
+ const eversion_t &applied_version);
+
+ bool should_send_op(
+ int peer,
+ const hobject_t &hoid) {
+ assert(peer_info.count(peer));
+ bool should_send = hoid.pool != (int64_t)info.pgid.pool() ||
+ hoid <= MAX(last_backfill_started, peer_info[peer].last_backfill);
+ if (!should_send)
+ assert(is_backfill_targets(peer));
+ return should_send;
+ }
+
+ void update_peer_last_complete_ondisk(
+ int fromosd,
+ eversion_t lcod) {
+ peer_last_complete_ondisk[fromosd] = lcod;
+ }
+
+ void update_last_complete_ondisk(
+ eversion_t lcod) {
+ last_complete_ondisk = lcod;
+ }
+
+ void update_stats(
+ const pg_stat_t &stat) {
+ info.stats = stat;
+ }
/*
* Capture all object state associated with an in-progress read or write.
@@ -357,7 +404,7 @@ public:
int current_osd_subop_num;
- ObjectStore::Transaction op_t, local_t;
+ PGBackend::PGTransaction *op_t;
vector<pg_log_entry_t> log;
interval_set<uint64_t> modified_ranges;
@@ -380,6 +427,42 @@ public:
hobject_t new_temp_oid, discard_temp_oid; ///< temp objects we should start/stop tracking
+ // pending xattr updates
+ map<ObjectContextRef,
+ map<string, boost::optional<bufferlist> > > pending_attrs;
+ void apply_pending_attrs() {
+ for (map<ObjectContextRef,
+ map<string, boost::optional<bufferlist> > >::iterator i =
+ pending_attrs.begin();
+ i != pending_attrs.end();
+ ++i) {
+ for (map<string, boost::optional<bufferlist> >::iterator j =
+ i->second.begin();
+ j != i->second.end();
+ ++j) {
+ if (j->second)
+ i->first->attr_cache[j->first] = j->second.get();
+ else
+ i->first->attr_cache.erase(j->first);
+ }
+ }
+ pending_attrs.clear();
+ }
+
+ // pending async reads <off, len> -> <outbl, outr>
+ list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > > pending_async_reads;
+ int async_read_result;
+ unsigned inflightreads;
+ friend struct OnReadComplete;
+ void start_async_reads(ReplicatedPG *pg);
+ void finish_read(ReplicatedPG *pg);
+ bool async_reads_complete() {
+ return inflightreads == 0;
+ }
+
+ ObjectModDesc mod_desc;
+
enum { W_LOCK, R_LOCK, NONE } lock_to_release;
OpContext(const OpContext& other);
@@ -393,10 +476,13 @@ public:
modify(false), user_modify(false), undirty(false),
bytes_written(0), bytes_read(0), user_at_version(0),
current_osd_subop_num(0),
+ op_t(NULL),
data_off(0), reply(NULL), pg(_pg),
num_read(0),
num_write(0),
copy_cb(NULL),
+ async_read_result(0),
+ inflightreads(0),
lock_to_release(NONE) {
if (_ssc) {
new_snapset = _ssc->snapset;
@@ -411,18 +497,25 @@ public:
}
}
~OpContext() {
- assert(!clone_obc);
+ assert(!op_t);
assert(lock_to_release == NONE);
if (reply)
reply->put();
+ for (list<pair<pair<uint64_t, uint64_t>,
+ pair<bufferlist*, Context*> > >::iterator i =
+ pending_async_reads.begin();
+ i != pending_async_reads.end();
+ pending_async_reads.erase(i++)) {
+ delete i->second.second;
+ }
}
};
+ friend class OpContext;
/*
* State on the PG primary associated with the replicated mutation
*/
class RepGather {
- bool is_done;
public:
xlist<RepGather*>::item queue_item;
int nref;
@@ -435,63 +528,48 @@ public:
tid_t rep_tid;
- bool applying, applied, aborted;
+ bool rep_aborted, rep_done;
- set<int> waitfor_ack;
- //set<int> waitfor_nvram;
- set<int> waitfor_disk;
+ bool all_applied;
+ bool all_committed;
bool sent_ack;
//bool sent_nvram;
bool sent_disk;
- Context *ondone; ///< if set, this Context will be activated when repop is done
-
utime_t start;
eversion_t pg_local_last_complete;
- list<ObjectStore::Transaction*> tls;
bool queue_snap_trimmer;
RepGather(OpContext *c, ObjectContextRef pi, tid_t rt,
eversion_t lc) :
- is_done(false),
queue_item(this),
nref(1),
ctx(c), obc(pi),
rep_tid(rt),
- applying(false), applied(false), aborted(false),
- sent_ack(false),
+ rep_aborted(false), rep_done(false),
+ all_applied(false), all_committed(false), sent_ack(false),
//sent_nvram(false),
sent_disk(false),
- ondone(NULL),
pg_local_last_complete(lc),
queue_snap_trimmer(false) { }
- void get() {
+ RepGather *get() {
nref++;
+ return this;
}
void put() {
assert(nref > 0);
if (--nref == 0) {
- assert(src_obc.empty());
delete ctx; // must already be unlocked
delete this;
//generic_dout(0) << "deleting " << this << dendl;
}
}
- void mark_done() {
- is_done = true;
- if (ondone)
- ondone->complete(0);
- }
- bool done() {
- return is_done;
- }
};
-
protected:
/**
@@ -501,7 +579,7 @@ protected:
* @return true on success, false if we are queued
*/
bool get_rw_locks(OpContext *ctx) {
- if (ctx->op->may_write()) {
+ if (ctx->op->may_write() || ctx->op->may_cache()) {
if (ctx->obc->get_write(ctx->op)) {
ctx->lock_to_release = OpContext::W_LOCK;
return true;
@@ -526,6 +604,8 @@ protected:
*/
void close_op_ctx(OpContext *ctx) {
release_op_ctx_locks(ctx);
+ delete ctx->op_t;
+ ctx->op_t = NULL;
delete ctx;
}
@@ -560,16 +640,31 @@ protected:
xlist<RepGather*> repop_queue;
map<tid_t, RepGather*> repop_map;
- void apply_repop(RepGather *repop);
- void op_applied(RepGather *repop);
- void op_commit(RepGather *repop);
+ friend class C_OSD_RepopApplied;
+ friend class C_OSD_RepopCommit;
+ void repop_all_applied(RepGather *repop);
+ void repop_all_committed(RepGather *repop);
void eval_repop(RepGather*);
void issue_repop(RepGather *repop, utime_t now);
RepGather *new_repop(OpContext *ctx, ObjectContextRef obc, tid_t rep_tid);
void remove_repop(RepGather *repop);
- void repop_ack(RepGather *repop,
- int result, int ack_type,
- int fromosd, eversion_t pg_complete_thru=eversion_t(0,0));
+
+ RepGather *simple_repop_create(ObjectContextRef obc);
+ void simple_repop_submit(RepGather *repop);
+
+ // hot/cold tracking
+ boost::scoped_ptr<HitSet> hit_set; ///< currently accumulating HitSet
+ utime_t hit_set_start_stamp; ///< time the current HitSet started recording
+
+ void hit_set_clear(); ///< discard any HitSet state
+ void hit_set_setup(); ///< initialize HitSet state
+ void hit_set_create(); ///< create a new HitSet
+ void hit_set_persist(); ///< persist hit info
+ bool hit_set_apply_log(); ///< apply log entries to update in-memory HitSet
+ void hit_set_trim(RepGather *repop, unsigned max); ///< discard old HitSets
+
+ hobject_t get_hit_set_current_object(utime_t stamp);
+ hobject_t get_hit_set_archive_object(utime_t start, utime_t end);
/// true if we can send an ondisk/commit for v
bool already_complete(eversion_t v) {
@@ -578,7 +673,7 @@ protected:
++i) {
if ((*i)->v > v)
break;
- if (!(*i)->waitfor_disk.empty())
+ if (!(*i)->all_committed)
return false;
}
return true;
@@ -590,14 +685,12 @@ protected:
++i) {
if ((*i)->v > v)
break;
- if (!(*i)->waitfor_ack.empty())
+ if (!(*i)->all_applied)
return false;
}
return true;
}
- friend class C_OSD_OpCommit;
- friend class C_OSD_OpApplied;
friend struct C_OnPushCommit;
// projected object info
@@ -638,7 +731,8 @@ protected:
int find_object_context(const hobject_t& oid,
ObjectContextRef *pobc,
- bool can_create, snapid_t *psnapid=NULL);
+ bool can_create,
+ hobject_t *missing_oid=NULL);
void add_object_context_to_pg_stat(ObjectContextRef obc, pg_stat_t *stat);
@@ -684,8 +778,16 @@ protected:
map<hobject_t, pg_stat_t> pending_backfill_updates;
void dump_recovery_info(Formatter *f) const {
- f->dump_int("backfill_target", get_backfill_target());
- f->dump_int("waiting_on_backfill", waiting_on_backfill);
+ f->open_array_section("backfill_targets");
+ for (vector<int>::const_iterator p = backfill_targets.begin();
+ p != backfill_targets.end(); ++p)
+ f->dump_int("osd", *p);
+ f->close_section();
+ f->open_array_section("waiting_on_backfill");
+ for (set<int>::const_iterator p = waiting_on_backfill.begin();
+ p != waiting_on_backfill.end(); ++p)
+ f->dump_int("osd", *p);
+ f->close_section();
f->dump_stream("last_backfill_started") << last_backfill_started;
{
f->open_object_section("backfill_info");
@@ -693,8 +795,14 @@ protected:
f->close_section();
}
{
- f->open_object_section("peer_backfill_info");
- peer_backfill_info.dump(f);
+ f->open_array_section("peer_backfill_info");
+ for (map<int, BackfillInterval>::const_iterator pbi = peer_backfill_info.begin();
+ pbi != peer_backfill_info.end(); ++pbi) {
+ f->dump_int("osd", pbi->first);
+ f->open_object_section("BackfillInterval");
+ pbi->second.dump(f);
+ f->close_section();
+ }
f->close_section();
}
{
@@ -724,6 +832,7 @@ protected:
/// last backfill operation started
hobject_t last_backfill_started;
+ bool new_backfill;
int prep_object_replica_pushes(const hobject_t& soid, eversion_t v,
PGBackend::RecoveryHandle *h);
@@ -741,10 +850,14 @@ protected:
// low level ops
- void _make_clone(ObjectStore::Transaction& t,
- const hobject_t& head, const hobject_t& coid,
- object_info_t *poi);
+ void _make_clone(
+ OpContext *ctx,
+ PGBackend::PGTransaction* t,
+ ObjectContextRef obc,
+ const hobject_t& head, const hobject_t& coid,
+ object_info_t *poi);
void execute_ctx(OpContext *ctx);
+ void finish_ctx(OpContext *ctx, int log_op_type);
void reply_ctx(OpContext *ctx, int err);
void reply_ctx(OpContext *ctx, int err, eversion_t v, version_t uv);
void make_writeable(OpContext *ctx);
@@ -755,10 +868,31 @@ protected:
uint64_t offset, uint64_t length, bool count_bytes);
void add_interval_usage(interval_set<uint64_t>& s, object_stat_sum_t& st);
- inline bool maybe_handle_cache(OpRequestRef op, ObjectContextRef obc, int r);
+ /**
+ * This helper function is called from do_op if the ObjectContext lookup fails.
+ * @returns true if the caching code is handling the Op, false otherwise.
+ */
+ inline bool maybe_handle_cache(OpRequestRef op, ObjectContextRef obc, int r,
+ const hobject_t& missing_oid,
+ bool must_promote = false);
+ /**
+ * This helper function tells the client to redirect their request elsewhere.
+ */
void do_cache_redirect(OpRequestRef op, ObjectContextRef obc);
+ /**
+ * This function starts up a copy from
+ */
+ void promote_object(OpRequestRef op, ObjectContextRef obc,
+ const hobject_t& missing_object);
+
+ /**
+ * Check if the op is such that we can skip promote (e.g., DELETE)
+ */
+ bool can_skip_promote(OpRequestRef op, ObjectContextRef obc);
int prepare_transaction(OpContext *ctx);
+ list<pair<OpRequestRef, OpContext*> > in_progress_async_reads;
+ void complete_read_ctx(int result, OpContext *ctx);
// pg on-disk content
void check_local();
@@ -772,6 +906,8 @@ protected:
int recover_primary(int max, ThreadPool::TPHandle &handle);
int recover_replicas(int max, ThreadPool::TPHandle &handle);
+ hobject_t earliest_peer_backfill() const;
+ bool all_peer_done() const;
/**
* @param work_started will be set to true if recover_backfill got anywhere
* @returns the number of operations started
@@ -799,44 +935,12 @@ protected:
);
void prep_backfill_object_push(
- hobject_t oid, eversion_t v, eversion_t have, ObjectContextRef obc,
- int peer,
+ hobject_t oid, eversion_t v, ObjectContextRef obc,
+ vector<int> peer,
PGBackend::RecoveryHandle *h);
void send_remove_op(const hobject_t& oid, eversion_t v, int peer);
- struct RepModify {
- ReplicatedPG *pg;
- OpRequestRef op;
- OpContext *ctx;
- bool applied, committed;
- int ackerosd;
- eversion_t last_complete;
- epoch_t epoch_started;
-
- uint64_t bytes_written;
-
- ObjectStore::Transaction opt, localt;
- list<ObjectStore::Transaction*> tls;
-
- RepModify() : pg(NULL), ctx(NULL), applied(false), committed(false), ackerosd(-1),
- epoch_started(0), bytes_written(0) {}
- };
-
- struct C_OSD_RepModifyApply : public Context {
- RepModify *rm;
- C_OSD_RepModifyApply(RepModify *r) : rm(r) { }
- void finish(int r) {
- rm->pg->sub_op_modify_applied(rm);
- }
- };
- struct C_OSD_RepModifyCommit : public Context {
- RepModify *rm;
- C_OSD_RepModifyCommit(RepModify *r) : rm(r) { }
- void finish(int r) {
- rm->pg->sub_op_modify_commit(rm);
- }
- };
struct C_OSD_OndiskWriteUnlock : public Context {
ObjectContextRef obc, obc2, obc3;
C_OSD_OndiskWriteUnlock(
@@ -891,11 +995,6 @@ protected:
void sub_op_remove(OpRequestRef op);
- void sub_op_modify(OpRequestRef op);
- void sub_op_modify_applied(RepModify *rm);
- void sub_op_modify_commit(RepModify *rm);
-
- void sub_op_modify_reply(OpRequestRef op);
void _applied_recovered_object(ObjectContextRef obc);
void _applied_recovered_object_replica();
void _committed_pushed_object(epoch_t epoch, eversion_t lc);
@@ -904,8 +1003,13 @@ protected:
// -- copyfrom --
map<hobject_t, CopyOpRef> copy_ops;
- int fill_in_copy_get(bufferlist::iterator& bp, OSDOp& op,
- object_info_t& oi, bool classic);
+ int fill_in_copy_get(
+ OpContext *ctx,
+ bufferlist::iterator& bp,
+ OSDOp& op,
+ ObjectContextRef& obc,
+ bool classic);
+
/**
* To copy an object, call start_copy.
*
@@ -917,18 +1021,31 @@ protected:
* @param temp_dest_oid: the temporary object to use for large objects
*/
void start_copy(CopyCallback *cb, ObjectContextRef obc, hobject_t src,
- object_locator_t oloc, version_t version,
- const hobject_t& temp_dest_oid);
+ object_locator_t oloc, version_t version, unsigned flags,
+ bool mirror_snapset);
void process_copy_chunk(hobject_t oid, tid_t tid, int r);
- void _write_copy_chunk(CopyOpRef cop, ObjectStore::Transaction *t);
+ void _write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t);
void _copy_some(ObjectContextRef obc, CopyOpRef cop);
void _build_finish_copy_transaction(CopyOpRef cop,
- ObjectStore::Transaction& t);
- int finish_copyfrom(OpContext *ctx);
+ PGBackend::PGTransaction *t);
+ void finish_copyfrom(OpContext *ctx);
+ void finish_promote(int r, OpRequestRef op,
+ CopyResults *results, ObjectContextRef obc);
void cancel_copy(CopyOpRef cop, bool requeue);
void cancel_copy_ops(bool requeue);
- friend class C_Copyfrom;
+ friend struct C_Copyfrom;
+
+ // -- flush --
+ map<hobject_t, FlushOpRef> flush_ops;
+
+ int start_flush(OpContext *ctx, bool blocking);
+ void finish_flush(hobject_t oid, tid_t tid, int r);
+ int try_flush_mark_clean(FlushOpRef fop);
+ void cancel_flush(FlushOpRef fop, bool requeue);
+ void cancel_flush_ops(bool requeue);
+
+ friend struct C_Flush;
// -- scrub --
virtual void _scrub(ScrubMap& map);
@@ -972,11 +1089,15 @@ public:
void snap_trimmer();
int do_osd_ops(OpContext *ctx, vector<OSDOp>& ops);
+ int _get_tmap(OpContext *ctx, bufferlist *header, bufferlist *vals);
+ int do_tmap2omap(OpContext *ctx, unsigned flags);
int do_tmapup(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op);
int do_tmapup_slow(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op, bufferlist& bl);
void do_osd_op_effects(OpContext *ctx);
private:
+ hobject_t earliest_backfill() const;
+ bool check_src_targ(const hobject_t& soid, const hobject_t& toid) const;
uint64_t temp_seq; ///< last id for naming temp objects
coll_t get_temp_coll(ObjectStore::Transaction *t);
hobject_t generate_temp_object(); ///< generate a new temp object name
@@ -1051,9 +1172,7 @@ private:
boost::statechart::result react(const SnapTrim&);
};
- int _get_tmap(OpContext *ctx, map<string, bufferlist> *out,
- bufferlist *header);
- int _delete_head(OpContext *ctx);
+ int _delete_head(OpContext *ctx, bool no_whiteout);
int _rollback_to(OpContext *ctx, ceph_osd_op& op);
public:
bool same_for_read_since(epoch_t e);
@@ -1070,17 +1189,6 @@ public:
void wait_for_blocked_object(const hobject_t& soid, OpRequestRef op);
void kick_object_context_blocked(ObjectContextRef obc);
- struct C_KickBlockedObject : public Context {
- ObjectContextRef obc;
- ReplicatedPG *pg;
- C_KickBlockedObject(ObjectContextRef obc_, ReplicatedPG *pg_) :
- obc(obc_), pg(pg_) {}
- protected:
- void finish(int r) {
- pg->kick_object_context_blocked(obc);
- }
- };
-
void mark_all_unfound_lost(int what);
eversion_t pick_newest_available(const hobject_t& oid);
ObjectContextRef mark_object_lost(ObjectStore::Transaction *t,
@@ -1089,27 +1197,51 @@ public:
void _finish_mark_all_unfound_lost(list<ObjectContextRef>& obcs);
void on_role_change();
+ void on_pool_change();
void on_change(ObjectStore::Transaction *t);
void on_activate();
void on_flushed();
void on_removal(ObjectStore::Transaction *t);
void on_shutdown();
+
+ // attr cache handling
+ void setattr_maybe_cache(
+ ObjectContextRef obc,
+ OpContext *op,
+ PGBackend::PGTransaction *t,
+ const string &key,
+ bufferlist &val);
+ void rmattr_maybe_cache(
+ ObjectContextRef obc,
+ OpContext *op,
+ PGBackend::PGTransaction *t,
+ const string &key);
+ int getattr_maybe_cache(
+ ObjectContextRef obc,
+ const string &key,
+ bufferlist *val);
+ int getattrs_maybe_cache(
+ ObjectContextRef obc,
+ map<string, bufferlist> *out);
};
inline ostream& operator<<(ostream& out, ReplicatedPG::RepGather& repop)
{
out << "repgather(" << &repop
- << (repop.applying ? " applying" : "")
- << (repop.applied ? " applied" : "")
<< " " << repop.v
<< " rep_tid=" << repop.rep_tid
- << " wfack=" << repop.waitfor_ack
- //<< " wfnvram=" << repop.waitfor_nvram
- << " wfdisk=" << repop.waitfor_disk;
+ << " committed?=" << repop.all_committed
+ << " applied?=" << repop.all_applied;
+ if (repop.ctx->lock_to_release != ReplicatedPG::OpContext::NONE)
+ out << " lock=" << (int)repop.ctx->lock_to_release;
if (repop.ctx->op)
out << " op=" << *(repop.ctx->op->get_req());
out << ")";
return out;
}
+void intrusive_ptr_add_ref(ReplicatedPG::RepGather *repop);
+void intrusive_ptr_release(ReplicatedPG::RepGather *repop);
+
+
#endif
diff --git a/src/osd/Watch.h b/src/osd/Watch.h
index ecb61ad..8e98c9d 100644
--- a/src/osd/Watch.h
+++ b/src/osd/Watch.h
@@ -15,7 +15,7 @@
#define CEPH_WATCH_H
#include <boost/intrusive_ptr.hpp>
-#include <tr1/memory>
+#include "include/memory.h"
#include <set>
#include "msg/Messenger.h"
@@ -35,12 +35,12 @@ struct ObjectContext;
class MWatchNotify;
class Watch;
-typedef std::tr1::shared_ptr<Watch> WatchRef;
-typedef std::tr1::weak_ptr<Watch> WWatchRef;
+typedef ceph::shared_ptr<Watch> WatchRef;
+typedef ceph::weak_ptr<Watch> WWatchRef;
class Notify;
-typedef std::tr1::shared_ptr<Notify> NotifyRef;
-typedef std::tr1::weak_ptr<Notify> WNotifyRef;
+typedef ceph::shared_ptr<Notify> NotifyRef;
+typedef ceph::weak_ptr<Notify> WNotifyRef;
struct CancelableContext;
@@ -151,7 +151,7 @@ class Watch {
OSDService *osd;
boost::intrusive_ptr<ReplicatedPG> pg;
- std::tr1::shared_ptr<ObjectContext> obc;
+ ceph::shared_ptr<ObjectContext> obc;
std::map<uint64_t, NotifyRef> in_progress_notifies;
@@ -165,7 +165,7 @@ class Watch {
Watch(
ReplicatedPG *pg, OSDService *osd,
- std::tr1::shared_ptr<ObjectContext> obc, uint32_t timeout,
+ ceph::shared_ptr<ObjectContext> obc, uint32_t timeout,
uint64_t cookie, entity_name_t entity,
entity_addr_t addr);
@@ -187,7 +187,7 @@ public:
string gen_dbg_prefix();
static WatchRef makeWatchRef(
ReplicatedPG *pg, OSDService *osd,
- std::tr1::shared_ptr<ObjectContext> obc, uint32_t timeout, uint64_t cookie, entity_name_t entity, entity_addr_t addr);
+ ceph::shared_ptr<ObjectContext> obc, uint32_t timeout, uint64_t cookie, entity_name_t entity, entity_addr_t addr);
void set_self(WatchRef _self) {
self = _self;
}
@@ -195,7 +195,7 @@ public:
/// Does not grant a ref count!
boost::intrusive_ptr<ReplicatedPG> get_pg() { return pg; }
- std::tr1::shared_ptr<ObjectContext> get_obc() { return obc; }
+ ceph::shared_ptr<ObjectContext> get_obc() { return obc; }
uint64_t get_cookie() const { return cookie; }
entity_name_t get_entity() const { return entity; }
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 4a9a96c..c1ab20b 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -19,6 +19,48 @@ extern "C" {
}
#include "PG.h"
#include "OSDMap.h"
+#include "PGBackend.h"
+
+const char *ceph_osd_flag_name(unsigned flag)
+{
+ switch (flag) {
+ case CEPH_OSD_FLAG_ACK: return "ack";
+ case CEPH_OSD_FLAG_ONNVRAM: return "onnvram";
+ case CEPH_OSD_FLAG_ONDISK: return "ondisk";
+ case CEPH_OSD_FLAG_RETRY: return "retry";
+ case CEPH_OSD_FLAG_READ: return "read";
+ case CEPH_OSD_FLAG_WRITE: return "write";
+ case CEPH_OSD_FLAG_ORDERSNAP: return "ordersnap";
+ case CEPH_OSD_FLAG_PEERSTAT_OLD: return "peerstat_old";
+ case CEPH_OSD_FLAG_BALANCE_READS: return "balance_reads";
+ case CEPH_OSD_FLAG_PARALLELEXEC: return "parallelexec";
+ case CEPH_OSD_FLAG_PGOP: return "pgop";
+ case CEPH_OSD_FLAG_EXEC: return "exec";
+ case CEPH_OSD_FLAG_EXEC_PUBLIC: return "exec_public";
+ case CEPH_OSD_FLAG_LOCALIZE_READS: return "localize_reads";
+ case CEPH_OSD_FLAG_RWORDERED: return "rwordered";
+ case CEPH_OSD_FLAG_IGNORE_CACHE: return "ignore_cache";
+ case CEPH_OSD_FLAG_SKIPRWLOCKS: return "skiprwlocks";
+ case CEPH_OSD_FLAG_IGNORE_OVERLAY: return "ignore_overlay";
+ case CEPH_OSD_FLAG_FLUSH: return "flush";
+ default: return "???";
+ }
+}
+
+string ceph_osd_flag_string(unsigned flags)
+{
+ string s;
+ for (unsigned i=0; i<32; ++i) {
+ if (flags & (1u<<i)) {
+ if (s.length())
+ s += "+";
+ s += ceph_osd_flag_name(1u << i);
+ }
+ }
+ if (s.length())
+ return s;
+ return string("-");
+}
// -- osd_reqid_t --
void osd_reqid_t::encode(bufferlist &bl) const
@@ -56,18 +98,24 @@ void osd_reqid_t::generate_test_instances(list<osd_reqid_t*>& o)
void object_locator_t::encode(bufferlist& bl) const
{
- ENCODE_START(5, 3, bl);
+ // verify that nobody's corrupted the locator
+ assert(hash == -1 || key.empty());
+ __u8 encode_compat = 3;
+ ENCODE_START(6, encode_compat, bl);
::encode(pool, bl);
int32_t preferred = -1; // tell old code there is no preferred osd (-1).
::encode(preferred, bl);
::encode(key, bl);
::encode(nspace, bl);
- ENCODE_FINISH(bl);
+ ::encode(hash, bl);
+ if (hash != -1)
+ encode_compat = MAX(encode_compat, 6); // need to interpret the hash
+ ENCODE_FINISH_NEW_COMPAT(bl, encode_compat);
}
void object_locator_t::decode(bufferlist::iterator& p)
{
- DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, p);
+ DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, p);
if (struct_v < 2) {
int32_t op;
::decode(op, p);
@@ -82,7 +130,13 @@ void object_locator_t::decode(bufferlist::iterator& p)
::decode(key, p);
if (struct_v >= 5)
::decode(nspace, p);
+ if (struct_v >= 6)
+ ::decode(hash, p);
+ else
+ hash = -1;
DECODE_FINISH(p);
+ // verify that nobody's corrupted the locator
+ assert(hash == -1 || key.empty());
}
void object_locator_t::dump(Formatter *f) const
@@ -90,12 +144,14 @@ void object_locator_t::dump(Formatter *f) const
f->dump_int("pool", pool);
f->dump_string("key", key);
f->dump_string("namespace", nspace);
+ f->dump_int("hash", hash);
}
void object_locator_t::generate_test_instances(list<object_locator_t*>& o)
{
o.push_back(new object_locator_t);
o.push_back(new object_locator_t(123));
+ o.push_back(new object_locator_t(123, 876));
o.push_back(new object_locator_t(1, "n2"));
o.push_back(new object_locator_t(1234, "", "key"));
o.push_back(new object_locator_t(12, "n1", "key2"));
@@ -171,13 +227,13 @@ void pow2_hist_t::generate_test_instances(list<pow2_hist_t*>& ls)
ls.back()->h.push_back(2);
}
-void filestore_perf_stat_t::dump(Formatter *f) const
+void objectstore_perf_stat_t::dump(Formatter *f) const
{
f->dump_unsigned("commit_latency_ms", filestore_commit_latency);
f->dump_unsigned("apply_latency_ms", filestore_apply_latency);
}
-void filestore_perf_stat_t::encode(bufferlist &bl) const
+void objectstore_perf_stat_t::encode(bufferlist &bl) const
{
ENCODE_START(1, 1, bl);
::encode(filestore_commit_latency, bl);
@@ -185,7 +241,7 @@ void filestore_perf_stat_t::encode(bufferlist &bl) const
ENCODE_FINISH(bl);
}
-void filestore_perf_stat_t::decode(bufferlist::iterator &bl)
+void objectstore_perf_stat_t::decode(bufferlist::iterator &bl)
{
DECODE_START(1, bl);
::decode(filestore_commit_latency, bl);
@@ -193,10 +249,10 @@ void filestore_perf_stat_t::decode(bufferlist::iterator &bl)
DECODE_FINISH(bl);
}
-void filestore_perf_stat_t::generate_test_instances(std::list<filestore_perf_stat_t*>& o)
+void objectstore_perf_stat_t::generate_test_instances(std::list<objectstore_perf_stat_t*>& o)
{
- o.push_back(new filestore_perf_stat_t());
- o.push_back(new filestore_perf_stat_t());
+ o.push_back(new objectstore_perf_stat_t());
+ o.push_back(new objectstore_perf_stat_t());
o.back()->filestore_commit_latency = 20;
o.back()->filestore_apply_latency = 30;
}
@@ -669,7 +725,7 @@ void pg_pool_t::dump(Formatter *f) const
f->dump_string("snap_mode", is_pool_snaps_mode() ? "pool" : "selfmanaged");
f->dump_unsigned("snap_seq", get_snap_seq());
f->dump_unsigned("snap_epoch", get_snap_epoch());
- f->open_object_section("pool_snaps");
+ f->open_array_section("pool_snaps");
for (map<snapid_t, pool_snap_info_t>::const_iterator p = snaps.begin(); p != snaps.end(); ++p) {
f->open_object_section("pool_snap_info");
p->second.dump(f);
@@ -687,7 +743,7 @@ void pg_pool_t::dump(Formatter *f) const
f->dump_int("read_tier", read_tier);
f->dump_int("write_tier", write_tier);
f->dump_string("cache_mode", get_cache_mode_name());
- f->open_array_section("properties");
+ f->open_object_section("properties");
for (map<string,string>::const_iterator i = properties.begin();
i != properties.end();
++i) {
@@ -695,6 +751,11 @@ void pg_pool_t::dump(Formatter *f) const
f->dump_string(name.c_str(), i->second);
}
f->close_section();
+ f->open_object_section("hit_set_params");
+ hit_set_params.dump(f);
+ f->close_section(); // hit_set_params
+ f->dump_unsigned("hit_set_period", hit_set_period);
+ f->dump_unsigned("hit_set_count", hit_set_count);
}
@@ -813,6 +874,24 @@ SnapContext pg_pool_t::get_snap_context() const
return SnapContext(get_snap_seq(), s);
}
+static string make_hash_str(const string &inkey, const string &nspace)
+{
+ if (nspace.empty())
+ return inkey;
+ return nspace + '\037' + inkey;
+}
+
+uint32_t pg_pool_t::hash_key(const string& key, const string& ns) const
+{
+ string n = make_hash_str(key, ns);
+ return ceph_str_hash(object_hash, n.c_str(), n.length());
+}
+
+uint32_t pg_pool_t::raw_hash_to_pg(uint32_t v) const
+{
+ return ceph_stable_mod(v, pg_num, pg_num_mask);
+}
+
/*
* map a raw pg (with full precision ps) into an actual pg, for storage
*/
@@ -899,7 +978,8 @@ void pg_pool_t::encode(bufferlist& bl, uint64_t features) const
return;
}
- ENCODE_START(10, 5, bl);
+ __u8 encode_compat = 5;
+ ENCODE_START(11, encode_compat, bl);
::encode(type, bl);
::encode(size, bl);
::encode(crush_ruleset, bl);
@@ -927,12 +1007,17 @@ void pg_pool_t::encode(bufferlist& bl, uint64_t features) const
::encode(read_tier, bl);
::encode(write_tier, bl);
::encode(properties, bl);
- ENCODE_FINISH(bl);
+ if (hit_set_params.get_type() != HitSet::TYPE_NONE)
+ encode_compat = MAX(encode_compat, 11); // need to be able to understand all the data!
+ ::encode(hit_set_params, bl);
+ ::encode(hit_set_period, bl);
+ ::encode(hit_set_count, bl);
+ ENCODE_FINISH_NEW_COMPAT(bl, encode_compat);
}
void pg_pool_t::decode(bufferlist::iterator& bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(11, 5, 5, bl);
::decode(type, bl);
::decode(size, bl);
::decode(crush_ruleset, bl);
@@ -997,6 +1082,15 @@ void pg_pool_t::decode(bufferlist::iterator& bl)
if (struct_v >= 10) {
::decode(properties, bl);
}
+ if (struct_v >= 11) {
+ ::decode(hit_set_params, bl);
+ ::decode(hit_set_period, bl);
+ ::decode(hit_set_count, bl);
+ } else {
+ pg_pool_t def;
+ hit_set_period = def.hit_set_period;
+ hit_set_count = def.hit_set_count;
+ }
DECODE_FINISH(bl);
calc_pg_masks();
}
@@ -1006,7 +1100,7 @@ void pg_pool_t::generate_test_instances(list<pg_pool_t*>& o)
pg_pool_t a;
o.push_back(new pg_pool_t(a));
- a.type = TYPE_REP;
+ a.type = TYPE_REPLICATED;
a.size = 2;
a.crush_ruleset = 3;
a.object_hash = 4;
@@ -1040,6 +1134,9 @@ void pg_pool_t::generate_test_instances(list<pg_pool_t*>& o)
a.write_tier = 1;
a.properties["p-1"] = "v-1";
a.properties["empty"] = string();
+ a.hit_set_params = HitSet::Params(new BloomHitSet::Params);
+ a.hit_set_period = 3600;
+ a.hit_set_count = 8;
o.push_back(new pg_pool_t(a));
}
@@ -1062,7 +1159,7 @@ ostream& operator<<(ostream& out, const pg_pool_t& p)
out << " max_bytes " << p.quota_max_bytes;
if (p.quota_max_objects)
out << " max_objects " << p.quota_max_objects;
- if (p.tiers.size())
+ if (!p.tiers.empty())
out << " tiers " << p.tiers;
if (p.is_tier())
out << " tier_of " << p.tier_of;
@@ -1072,6 +1169,11 @@ ostream& operator<<(ostream& out, const pg_pool_t& p)
out << " write_tier " << p.write_tier;
if (p.cache_mode)
out << " cache_mode " << p.get_cache_mode_name();
+ if (p.hit_set_params.get_type() != HitSet::TYPE_NONE) {
+ out << " hit_set " << p.hit_set_params
+ << " " << p.hit_set_period << "s"
+ << " x" << p.hit_set_count;
+ }
return out;
}
@@ -1087,6 +1189,8 @@ void object_stat_sum_t::dump(Formatter *f) const
f->dump_int("num_objects_missing_on_primary", num_objects_missing_on_primary);
f->dump_int("num_objects_degraded", num_objects_degraded);
f->dump_int("num_objects_unfound", num_objects_unfound);
+ f->dump_int("num_objects_dirty", num_objects_dirty);
+ f->dump_int("num_whiteouts", num_whiteouts);
f->dump_int("num_read", num_rd);
f->dump_int("num_read_kb", num_rd_kb);
f->dump_int("num_write", num_wr);
@@ -1101,7 +1205,7 @@ void object_stat_sum_t::dump(Formatter *f) const
void object_stat_sum_t::encode(bufferlist& bl) const
{
- ENCODE_START(6, 3, bl);
+ ENCODE_START(7, 3, bl);
::encode(num_bytes, bl);
::encode(num_objects, bl);
::encode(num_object_clones, bl);
@@ -1119,12 +1223,14 @@ void object_stat_sum_t::encode(bufferlist& bl) const
::encode(num_keys_recovered, bl);
::encode(num_shallow_scrub_errors, bl);
::encode(num_deep_scrub_errors, bl);
+ ::encode(num_objects_dirty, bl);
+ ::encode(num_whiteouts, bl);
ENCODE_FINISH(bl);
}
void object_stat_sum_t::decode(bufferlist::iterator& bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(7, 3, 3, bl);
::decode(num_bytes, bl);
if (struct_v < 3) {
uint64_t num_kb;
@@ -1161,6 +1267,13 @@ void object_stat_sum_t::decode(bufferlist::iterator& bl)
num_shallow_scrub_errors = 0;
num_deep_scrub_errors = 0;
}
+ if (struct_v >= 7) {
+ ::decode(num_objects_dirty, bl);
+ ::decode(num_whiteouts, bl);
+ } else {
+ num_objects_dirty = 0;
+ num_whiteouts = 0;
+ }
DECODE_FINISH(bl);
}
@@ -1184,6 +1297,8 @@ void object_stat_sum_t::generate_test_instances(list<object_stat_sum_t*>& o)
a.num_deep_scrub_errors = 17;
a.num_shallow_scrub_errors = 18;
a.num_scrub_errors = a.num_deep_scrub_errors + a.num_shallow_scrub_errors;
+ a.num_objects_dirty = 21;
+ a.num_whiteouts = 22;
o.push_back(new object_stat_sum_t(a));
}
@@ -1206,6 +1321,8 @@ void object_stat_sum_t::add(const object_stat_sum_t& o)
num_objects_recovered += o.num_objects_recovered;
num_bytes_recovered += o.num_bytes_recovered;
num_keys_recovered += o.num_keys_recovered;
+ num_objects_dirty += o.num_objects_dirty;
+ num_whiteouts += o.num_whiteouts;
}
void object_stat_sum_t::sub(const object_stat_sum_t& o)
@@ -1227,6 +1344,8 @@ void object_stat_sum_t::sub(const object_stat_sum_t& o)
num_objects_recovered -= o.num_objects_recovered;
num_bytes_recovered -= o.num_bytes_recovered;
num_keys_recovered -= o.num_keys_recovered;
+ num_objects_dirty -= o.num_objects_dirty;
+ num_whiteouts -= o.num_whiteouts;
}
@@ -1332,7 +1451,7 @@ void pg_stat_t::dump_brief(Formatter *f) const
void pg_stat_t::encode(bufferlist &bl) const
{
- ENCODE_START(13, 8, bl);
+ ENCODE_START(14, 8, bl);
::encode(version, bl);
::encode(reported_seq, bl);
::encode(reported_epoch, bl);
@@ -1361,12 +1480,13 @@ void pg_stat_t::encode(bufferlist &bl) const
::encode(stats_invalid, bl);
::encode(last_clean_scrub_stamp, bl);
::encode(last_became_active, bl);
+ ::encode(dirty_stats_invalid, bl);
ENCODE_FINISH(bl);
}
void pg_stat_t::decode(bufferlist::iterator &bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(13, 8, 8, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(14, 8, 8, bl);
::decode(version, bl);
::decode(reported_seq, bl);
::decode(reported_epoch, bl);
@@ -1446,6 +1566,13 @@ void pg_stat_t::decode(bufferlist::iterator &bl)
} else {
last_became_active = last_active;
}
+ if (struct_v >= 14) {
+ ::decode(dirty_stats_invalid, bl);
+ } else {
+ // if we are decoding an old encoding of this object, then the
+ // encoder may not have supported num_objects_dirty accounting.
+ dirty_stats_invalid = true;
+ }
DECODE_FINISH(bl);
}
@@ -1644,7 +1771,7 @@ void pg_history_t::generate_test_instances(list<pg_history_t*>& o)
void pg_info_t::encode(bufferlist &bl) const
{
- ENCODE_START(28, 26, bl);
+ ENCODE_START(29, 26, bl);
::encode(pgid, bl);
::encode(last_update, bl);
::encode(last_complete, bl);
@@ -1655,12 +1782,13 @@ void pg_info_t::encode(bufferlist &bl) const
::encode(purged_snaps, bl);
::encode(last_epoch_started, bl);
::encode(last_user_version, bl);
+ ::encode(hit_set, bl);
ENCODE_FINISH(bl);
}
void pg_info_t::decode(bufferlist::iterator &bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(28, 26, 26, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(29, 26, 26, bl);
if (struct_v < 23) {
old_pg_t opgid;
::decode(opgid, bl);
@@ -1694,6 +1822,8 @@ void pg_info_t::decode(bufferlist::iterator &bl)
::decode(last_user_version, bl);
else
last_user_version = last_update.version;
+ if (struct_v >= 29)
+ ::decode(hit_set, bl);
DECODE_FINISH(bl);
}
@@ -1719,6 +1849,10 @@ void pg_info_t::dump(Formatter *f) const
f->dump_int("dne", dne());
f->dump_int("incomplete", is_incomplete());
f->dump_int("last_epoch_started", last_epoch_started);
+
+ f->open_object_section("hit_set_history");
+ hit_set.dump(f);
+ f->close_section();
}
void pg_info_t::generate_test_instances(list<pg_info_t*>& o)
@@ -1734,9 +1868,16 @@ void pg_info_t::generate_test_instances(list<pg_info_t*>& o)
o.back()->last_user_version = 2;
o.back()->log_tail = eversion_t(7, 8);
o.back()->last_backfill = hobject_t(object_t("objname"), "key", 123, 456, -1, "");
- list<pg_stat_t*> s;
- pg_stat_t::generate_test_instances(s);
- o.back()->stats = *s.back();
+ {
+ list<pg_stat_t*> s;
+ pg_stat_t::generate_test_instances(s);
+ o.back()->stats = *s.back();
+ }
+ {
+ list<pg_hit_set_history_t*> s;
+ pg_hit_set_history_t::generate_test_instances(s);
+ o.back()->hit_set = *s.back();
+ }
}
// -- pg_notify_t --
@@ -1979,6 +2120,145 @@ void pg_query_t::generate_test_instances(list<pg_query_t*>& o)
o.push_back(new pg_query_t(pg_query_t::FULLLOG, *h.back(), 5));
}
+// -- ObjectModDesc --
+void ObjectModDesc::visit(Visitor *visitor) const
+{
+ bufferlist::iterator bp = bl.begin();
+ try {
+ while (!bp.end()) {
+ DECODE_START(1, bp);
+ uint8_t code;
+ ::decode(code, bp);
+ switch (code) {
+ case APPEND: {
+ uint64_t size;
+ ::decode(size, bp);
+ visitor->append(size);
+ break;
+ }
+ case SETATTRS: {
+ map<string, boost::optional<bufferlist> > attrs;
+ ::decode(attrs, bp);
+ visitor->setattrs(attrs);
+ break;
+ }
+ case DELETE: {
+ version_t old_version;
+ ::decode(old_version, bp);
+ visitor->rmobject(old_version);
+ break;
+ }
+ case CREATE: {
+ visitor->create();
+ break;
+ }
+ case UPDATE_SNAPS: {
+ set<snapid_t> snaps;
+ ::decode(snaps, bp);
+ visitor->update_snaps(snaps);
+ break;
+ }
+ default:
+ assert(0 == "Invalid rollback code");
+ }
+ DECODE_FINISH(bp);
+ }
+ } catch (...) {
+ assert(0 == "Invalid encoding");
+ }
+}
+
+struct DumpVisitor : public ObjectModDesc::Visitor {
+ Formatter *f;
+ DumpVisitor(Formatter *f) : f(f) {}
+ void append(uint64_t old_size) {
+ f->open_object_section("op");
+ f->dump_string("code", "APPEND");
+ f->dump_unsigned("old_size", old_size);
+ f->close_section();
+ }
+ void setattrs(map<string, boost::optional<bufferlist> > &attrs) {
+ f->open_object_section("op");
+ f->dump_string("code", "SETATTRS");
+ f->open_array_section("attrs");
+ for (map<string, boost::optional<bufferlist> >::iterator i = attrs.begin();
+ i != attrs.end();
+ ++i) {
+ f->dump_string("attr_name", i->first);
+ }
+ f->close_section();
+ f->close_section();
+ }
+ void rmobject(version_t old_version) {
+ f->open_object_section("op");
+ f->dump_string("code", "RMOBJECT");
+ f->dump_unsigned("old_version", old_version);
+ f->close_section();
+ }
+ void create() {
+ f->open_object_section("op");
+ f->dump_string("code", "CREATE");
+ f->close_section();
+ }
+ void update_snaps(set<snapid_t> &snaps) {
+ f->open_object_section("op");
+ f->dump_string("code", "UPDATE_SNAPS");
+ f->dump_stream("snaps") << snaps;
+ f->close_section();
+ }
+};
+
+void ObjectModDesc::dump(Formatter *f) const
+{
+ f->open_object_section("object_mod_desc");
+ f->dump_stream("can_local_rollback") << can_local_rollback;
+ f->dump_stream("stashed") << stashed;
+ {
+ f->open_array_section("ops");
+ DumpVisitor vis(f);
+ visit(&vis);
+ f->close_section();
+ }
+ f->close_section();
+}
+
+void ObjectModDesc::generate_test_instances(list<ObjectModDesc*>& o)
+{
+ map<string, boost::optional<bufferlist> > attrs;
+ attrs[OI_ATTR];
+ attrs[SS_ATTR];
+ attrs["asdf"];
+ o.push_back(new ObjectModDesc());
+ o.back()->append(100);
+ o.back()->setattrs(attrs);
+ o.push_back(new ObjectModDesc());
+ o.back()->rmobject(1001);
+ o.push_back(new ObjectModDesc());
+ o.back()->create();
+ o.back()->setattrs(attrs);
+ o.push_back(new ObjectModDesc());
+ o.back()->create();
+ o.back()->setattrs(attrs);
+ o.back()->mark_unrollbackable();
+ o.back()->append(1000);
+}
+
+void ObjectModDesc::encode(bufferlist &_bl) const
+{
+ ENCODE_START(1, 1, _bl);
+ ::encode(can_local_rollback, _bl);
+ ::encode(stashed, _bl);
+ ::encode(bl, _bl);
+ ENCODE_FINISH(_bl);
+}
+void ObjectModDesc::decode(bufferlist::iterator &_bl)
+{
+ DECODE_START(1, _bl);
+ ::decode(can_local_rollback, _bl);
+ ::decode(stashed, _bl);
+ ::decode(bl, _bl);
+ DECODE_FINISH(_bl);
+}
// -- pg_log_entry_t --
@@ -2010,7 +2290,7 @@ void pg_log_entry_t::decode_with_checksum(bufferlist::iterator& p)
void pg_log_entry_t::encode(bufferlist &bl) const
{
- ENCODE_START(8, 4, bl);
+ ENCODE_START(9, 4, bl);
::encode(op, bl);
::encode(soid, bl);
::encode(version, bl);
@@ -2033,6 +2313,7 @@ void pg_log_entry_t::encode(bufferlist &bl) const
::encode(prior_version, bl);
::encode(snaps, bl);
::encode(user_version, bl);
+ ::encode(mod_desc, bl);
ENCODE_FINISH(bl);
}
@@ -2080,6 +2361,11 @@ void pg_log_entry_t::decode(bufferlist::iterator &bl)
else
user_version = version.version;
+ if (struct_v >= 9)
+ ::decode(mod_desc, bl);
+ else
+ mod_desc.mark_unrollbackable();
+
DECODE_FINISH(bl);
}
@@ -2105,6 +2391,11 @@ void pg_log_entry_t::dump(Formatter *f) const
f->dump_unsigned("snap", *p);
f->close_section();
}
+ {
+ f->open_object_section("mod_desc");
+ mod_desc.dump(f);
+ f->close_section();
+ }
}
void pg_log_entry_t::generate_test_instances(list<pg_log_entry_t*>& o)
@@ -2139,16 +2430,17 @@ ostream& operator<<(ostream& out, const pg_log_entry_t& e)
void pg_log_t::encode(bufferlist& bl) const
{
- ENCODE_START(4, 3, bl);
+ ENCODE_START(5, 3, bl);
::encode(head, bl);
::encode(tail, bl);
::encode(log, bl);
+ ::encode(can_rollback_to, bl);
ENCODE_FINISH(bl);
}
void pg_log_t::decode(bufferlist::iterator &bl, int64_t pool)
{
- DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
::decode(head, bl);
::decode(tail, bl);
if (struct_v < 2) {
@@ -2156,6 +2448,8 @@ void pg_log_t::decode(bufferlist::iterator &bl, int64_t pool)
::decode(backlog, bl);
}
::decode(log, bl);
+ if (struct_v >= 5)
+ ::decode(can_rollback_to, bl);
DECODE_FINISH(bl);
// handle hobject_t format change
@@ -2163,7 +2457,7 @@ void pg_log_t::decode(bufferlist::iterator &bl, int64_t pool)
for (list<pg_log_entry_t>::iterator i = log.begin();
i != log.end();
++i) {
- if (i->soid.pool == -1)
+ if (!i->soid.is_max() && i->soid.pool == -1)
i->soid.pool = pool;
}
}
@@ -2198,6 +2492,7 @@ void pg_log_t::generate_test_instances(list<pg_log_t*>& o)
void pg_log_t::copy_after(const pg_log_t &other, eversion_t v)
{
+ can_rollback_to = other.can_rollback_to;
head = other.head;
tail = other.tail;
for (list<pg_log_entry_t>::const_reverse_iterator i = other.log.rbegin();
@@ -2215,6 +2510,7 @@ void pg_log_t::copy_after(const pg_log_t &other, eversion_t v)
void pg_log_t::copy_range(const pg_log_t &other, eversion_t from, eversion_t to)
{
+ can_rollback_to = other.can_rollback_to;
list<pg_log_entry_t>::const_reverse_iterator i = other.log.rbegin();
assert(i != other.log.rend());
while (i->version > to) {
@@ -2234,6 +2530,7 @@ void pg_log_t::copy_range(const pg_log_t &other, eversion_t from, eversion_t to)
void pg_log_t::copy_up_to(const pg_log_t &other, int max)
{
+ can_rollback_to = other.can_rollback_to;
int n = 0;
head = other.head;
tail = other.tail;
@@ -2280,7 +2577,7 @@ void pg_missing_t::decode(bufferlist::iterator &bl, int64_t pool)
for (map<hobject_t, item>::iterator i = missing.begin();
i != missing.end();
) {
- if (i->first.pool == -1) {
+ if (!i->first.is_max() && i->first.pool == -1) {
hobject_t to_insert(i->first);
to_insert.pool = pool;
tmp[to_insert] = i->second;
@@ -2547,7 +2844,7 @@ void object_copy_data_t::decode_classic(bufferlist::iterator& bl)
void object_copy_data_t::encode(bufferlist& bl) const
{
- ENCODE_START(1, 1, bl);
+ ENCODE_START(3, 1, bl);
::encode(size, bl);
::encode(mtime, bl);
::encode(category, bl);
@@ -2555,12 +2852,15 @@ void object_copy_data_t::encode(bufferlist& bl) const
::encode(data, bl);
::encode(omap, bl);
::encode(cursor, bl);
+ ::encode(omap_header, bl);
+ ::encode(snaps, bl);
+ ::encode(snap_seq, bl);
ENCODE_FINISH(bl);
}
void object_copy_data_t::decode(bufferlist::iterator& bl)
{
- DECODE_START(1, bl);
+ DECODE_START(2, bl);
::decode(size, bl);
::decode(mtime, bl);
::decode(category, bl);
@@ -2568,6 +2868,15 @@ void object_copy_data_t::decode(bufferlist::iterator& bl)
::decode(data, bl);
::decode(omap, bl);
::decode(cursor, bl);
+ if (struct_v >= 2)
+ ::decode(omap_header, bl);
+ if (struct_v >= 3) {
+ ::decode(snaps, bl);
+ ::decode(snap_seq, bl);
+ } else {
+ snaps.clear();
+ snap_seq = 0;
+ }
DECODE_FINISH(bl);
}
@@ -2596,6 +2905,8 @@ void object_copy_data_t::generate_test_instances(list<object_copy_data_t*>& o)
o.back()->omap["why"] = bl2;
bufferptr databp("iamsomedatatocontain", 20);
o.back()->data.push_back(databp);
+ o.back()->omap_header.append("this is an omap header");
+ o.back()->snaps.push_back(123);
}
void object_copy_data_t::dump(Formatter *f) const
@@ -2609,7 +2920,13 @@ void object_copy_data_t::dump(Formatter *f) const
const-correctness prents that */
f->dump_int("attrs_size", attrs.size());
f->dump_int("omap_size", omap.size());
+ f->dump_int("omap_header_length", omap_header.length());
f->dump_int("data_length", data.length());
+ f->open_array_section("snaps");
+ for (vector<snapid_t>::const_iterator p = snaps.begin();
+ p != snaps.end(); ++p)
+ f->dump_unsigned("snap", *p);
+ f->close_section();
}
// -- pg_create_t --
@@ -2646,6 +2963,93 @@ void pg_create_t::generate_test_instances(list<pg_create_t*>& o)
}
+// -- pg_hit_set_info_t --
+
+void pg_hit_set_info_t::encode(bufferlist& bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(begin, bl);
+ ::encode(end, bl);
+ ::encode(version, bl);
+ ENCODE_FINISH(bl);
+}
+
+void pg_hit_set_info_t::decode(bufferlist::iterator& p)
+{
+ DECODE_START(1, p);
+ ::decode(begin, p);
+ ::decode(end, p);
+ ::decode(version, p);
+ DECODE_FINISH(p);
+}
+
+void pg_hit_set_info_t::dump(Formatter *f) const
+{
+ f->dump_stream("begin") << begin;
+ f->dump_stream("end") << end;
+ f->dump_stream("version") << version;
+}
+
+void pg_hit_set_info_t::generate_test_instances(list<pg_hit_set_info_t*>& ls)
+{
+ ls.push_back(new pg_hit_set_info_t);
+ ls.push_back(new pg_hit_set_info_t);
+ ls.back()->begin = utime_t(1, 2);
+ ls.back()->end = utime_t(3, 4);
+}
+
+
+// -- pg_hit_set_history_t --
+
+void pg_hit_set_history_t::encode(bufferlist& bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(current_last_update, bl);
+ ::encode(current_last_stamp, bl);
+ ::encode(current_info, bl);
+ ::encode(history, bl);
+ ENCODE_FINISH(bl);
+}
+
+void pg_hit_set_history_t::decode(bufferlist::iterator& p)
+{
+ DECODE_START(1, p);
+ ::decode(current_last_update, p);
+ ::decode(current_last_stamp, p);
+ ::decode(current_info, p);
+ ::decode(history, p);
+ DECODE_FINISH(p);
+}
+
+void pg_hit_set_history_t::dump(Formatter *f) const
+{
+ f->dump_stream("current_last_update") << current_last_update;
+ f->dump_stream("current_last_stamp") << current_last_stamp;
+ f->open_object_section("current_info");
+ current_info.dump(f);
+ f->close_section();
+ f->open_array_section("history");
+ for (list<pg_hit_set_info_t>::const_iterator p = history.begin();
+ p != history.end(); ++p) {
+ f->open_object_section("info");
+ p->dump(f);
+ f->close_section();
+ }
+ f->close_section();
+}
+
+void pg_hit_set_history_t::generate_test_instances(list<pg_hit_set_history_t*>& ls)
+{
+ ls.push_back(new pg_hit_set_history_t);
+ ls.push_back(new pg_hit_set_history_t);
+ ls.back()->current_last_update = eversion_t(1, 2);
+ ls.back()->current_last_stamp = utime_t(100, 123);
+ ls.back()->current_info.begin = utime_t(2, 4);
+ ls.back()->current_info.end = utime_t(62, 24);
+ ls.back()->history.push_back(ls.back()->current_info);
+ ls.back()->history.push_back(pg_hit_set_info_t());
+}
+
// -- osd_peer_stat_t --
void osd_peer_stat_t::encode(bufferlist& bl) const
@@ -2684,7 +3088,7 @@ ostream& operator<<(ostream& out, const osd_peer_stat_t &stat)
void OSDSuperblock::encode(bufferlist &bl) const
{
- ENCODE_START(5, 5, bl);
+ ENCODE_START(6, 5, bl);
::encode(cluster_fsid, bl);
::encode(whoami, bl);
::encode(current_epoch, bl);
@@ -2695,12 +3099,13 @@ void OSDSuperblock::encode(bufferlist &bl) const
::encode(clean_thru, bl);
::encode(mounted, bl);
::encode(osd_fsid, bl);
+ ::encode(last_map_marked_full, bl);
ENCODE_FINISH(bl);
}
void OSDSuperblock::decode(bufferlist::iterator &bl)
{
- DECODE_START_LEGACY_COMPAT_LEN(5, 5, 5, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl);
if (struct_v < 3) {
string magic;
::decode(magic, bl);
@@ -2720,6 +3125,8 @@ void OSDSuperblock::decode(bufferlist::iterator &bl)
::decode(mounted, bl);
if (struct_v >= 4)
::decode(osd_fsid, bl);
+ if (struct_v >= 6)
+ ::decode(last_map_marked_full, bl);
DECODE_FINISH(bl);
}
@@ -2737,6 +3144,7 @@ void OSDSuperblock::dump(Formatter *f) const
f->close_section();
f->dump_int("clean_thru", clean_thru);
f->dump_int("last_epoch_mounted", mounted);
+ f->dump_int("last_map_marked_full", last_map_marked_full);
}
void OSDSuperblock::generate_test_instances(list<OSDSuperblock*>& o)
@@ -2752,6 +3160,8 @@ void OSDSuperblock::generate_test_instances(list<OSDSuperblock*>& o)
z.mounted = 8;
z.clean_thru = 7;
o.push_back(new OSDSuperblock(z));
+ z.last_map_marked_full = 7;
+ o.push_back(new OSDSuperblock(z));
}
// -- SnapSet --
@@ -2823,6 +3233,49 @@ ostream& operator<<(ostream& out, const SnapSet& cs)
<< (cs.head_exists ? "+head":"");
}
+void SnapSet::from_snap_set(const librados::snap_set_t& ss)
+{
+ // NOTE: our reconstruction of snaps (and the snapc) is not strictly
+ // correct: it will not include snaps that still logically exist
+ // but for which there was no clone that is defined. For all
+ // practical purposes this doesn't matter, since we only use that
+ // information to clone on the OSD, and we have already moved
+ // forward past that part of the object history.
+
+ seq = ss.seq;
+ set<snapid_t> _snaps;
+ set<snapid_t> _clones;
+ head_exists = false;
+ for (vector<librados::clone_info_t>::const_iterator p = ss.clones.begin();
+ p != ss.clones.end();
+ ++p) {
+ if (p->cloneid == librados::SNAP_HEAD) {
+ head_exists = true;
+ } else {
+ _clones.insert(p->cloneid);
+ _snaps.insert(p->snaps.begin(), p->snaps.end());
+ clone_size[p->cloneid] = p->size;
+ clone_overlap[p->cloneid]; // the entry must exist, even if it's empty.
+ for (vector<pair<uint64_t, uint64_t> >::const_iterator q =
+ p->overlap.begin(); q != p->overlap.end(); ++q)
+ clone_overlap[p->cloneid].insert(q->first, q->second);
+ }
+ }
+
+ // ascending
+ clones.clear();
+ clones.reserve(_clones.size());
+ for (set<snapid_t>::iterator p = _clones.begin(); p != _clones.end(); ++p)
+ clones.push_back(*p);
+
+ // descending
+ snaps.clear();
+ snaps.reserve(_snaps.size());
+ for (set<snapid_t>::reverse_iterator p = _snaps.rbegin();
+ p != _snaps.rend(); ++p)
+ snaps.push_back(*p);
+}
+
// -- watch_info_t --
void watch_info_t::encode(bufferlist& bl) const
@@ -2888,6 +3341,7 @@ void object_info_t::copy_user_bits(const object_info_t& other)
truncate_size = other.truncate_size;
flags = other.flags;
category = other.category;
+ user_version = other.user_version;
}
ps_t object_info_t::legacy_object_locator_to_ps(const object_t &oid,
@@ -3028,6 +3482,7 @@ void object_info_t::dump(Formatter *f) const
f->dump_stream("version") << version;
f->dump_stream("prior_version") << prior_version;
f->dump_stream("last_reqid") << last_reqid;
+ f->dump_unsigned("user_version", user_version);
f->dump_unsigned("size", size);
f->dump_stream("mtime") << mtime;
f->dump_unsigned("lost", (int)is_lost());
@@ -3069,6 +3524,8 @@ ostream& operator<<(ostream& out, const object_info_t& oi)
out << " " << oi.snaps;
if (oi.flags)
out << " " << oi.get_flag_string();
+ out << " s " << oi.size;
+ out << " uv" << oi.user_version;
out << ")";
return out;
}
@@ -3164,7 +3621,7 @@ void ObjectRecoveryInfo::decode(bufferlist::iterator &bl,
DECODE_FINISH(bl);
if (struct_v < 2) {
- if (soid.pool == -1)
+ if (!soid.is_max() && soid.pool == -1)
soid.pool = pool;
map<hobject_t, interval_set<uint64_t> > tmp;
tmp.swap(clone_subset);
@@ -3172,7 +3629,7 @@ void ObjectRecoveryInfo::decode(bufferlist::iterator &bl,
i != tmp.end();
++i) {
hobject_t first(i->first);
- if (first.pool == -1)
+ if (!first.is_max() && first.pool == -1)
first.pool = pool;
clone_subset[first].swap(i->second);
}
@@ -3492,7 +3949,7 @@ void ScrubMap::decode(bufferlist::iterator& bl, int64_t pool)
i != tmp.end();
++i) {
hobject_t first(i->first);
- if (first.pool == -1)
+ if (!first.is_max() && first.pool == -1)
first.pool = pool;
objects[first] = i->second;
}
@@ -3624,6 +4081,9 @@ ostream& operator<<(ostream& out, const OSDOp& op)
case CEPH_OSD_OP_LIST_SNAPS:
case CEPH_OSD_OP_UNDIRTY:
case CEPH_OSD_OP_ISDIRTY:
+ case CEPH_OSD_OP_CACHE_FLUSH:
+ case CEPH_OSD_OP_CACHE_TRY_FLUSH:
+ case CEPH_OSD_OP_CACHE_EVICT:
break;
case CEPH_OSD_OP_ASSERT_VER:
out << " v" << op.op.assert_ver.ver;
@@ -3642,6 +4102,7 @@ ostream& operator<<(ostream& out, const OSDOp& op)
out << (op.op.watch.flag ? " add":" remove")
<< " cookie " << op.op.watch.cookie << " ver " << op.op.watch.ver;
break;
+ case CEPH_OSD_OP_COPY_GET:
case CEPH_OSD_OP_COPY_GET_CLASSIC:
out << " max " << op.op.copy_get.max;
case CEPH_OSD_OP_COPY_FROM:
@@ -3676,6 +4137,11 @@ ostream& operator<<(ostream& out, const OSDOp& op)
case CEPH_OSD_OP_PGLS_FILTER:
out << " start_epoch " << op.op.pgls.start_epoch;
break;
+ case CEPH_OSD_OP_PG_HITSET_LS:
+ break;
+ case CEPH_OSD_OP_PG_HITSET_GET:
+ out << " " << utime_t(op.op.hit_set_get.stamp);
+ break;
}
} else if (ceph_osd_op_type_multi(op.op.op)) {
switch (op.op.op) {
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 351d050..e2a4822 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -18,6 +18,10 @@
#include <sstream>
#include <stdio.h>
#include <memory>
+#include <boost/scoped_ptr.hpp>
+#include <boost/optional.hpp>
+
+#include "include/rados/rados_types.hpp"
#include "msg/msg_types.h"
#include "include/types.h"
@@ -25,11 +29,14 @@
#include "include/CompatSet.h"
#include "include/histogram.h"
#include "include/interval_set.h"
-#include "common/snap_types.h"
#include "common/Formatter.h"
+#include "common/bloom_filter.hpp"
#include "common/hobject.h"
+#include "common/snap_types.h"
+#include "HitSet.h"
#include "Watch.h"
#include "OpRequest.h"
+#include "include/hash_namespace.h"
#define CEPH_OSD_ONDISK_MAGIC "ceph osd volume v026"
@@ -50,6 +57,11 @@ typedef hobject_t collection_list_handle_t;
typedef uint8_t shard_id_t;
+/// convert a single CPEH_OSD_FLAG_* to a string
+const char *ceph_osd_flag_name(unsigned flag);
+
+/// convert CEPH_OSD_FLAG_* op flags to a string
+string ceph_osd_flag_string(unsigned flags);
inline ostream& operator<<(ostream& out, const osd_reqid_t& r) {
return out << r.name << "." << r.inc << ":" << r.tid;
@@ -72,14 +84,14 @@ inline bool operator<=(const osd_reqid_t& l, const osd_reqid_t& r) {
inline bool operator>(const osd_reqid_t& l, const osd_reqid_t& r) { return !(l <= r); }
inline bool operator>=(const osd_reqid_t& l, const osd_reqid_t& r) { return !(l < r); }
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<osd_reqid_t> {
size_t operator()(const osd_reqid_t &r) const {
static hash<uint64_t> H;
return H(r.name.num() ^ r.tid ^ r.inc);
}
};
-}
+CEPH_HASH_NAMESPACE_END
// -----
@@ -87,20 +99,26 @@ namespace __gnu_cxx {
// a locator constrains the placement of an object. mainly, which pool
// does it go in.
struct object_locator_t {
- int64_t pool;
- string key;
- string nspace;
+ // You specify either the hash or the key -- not both
+ int64_t pool; ///< pool id
+ string key; ///< key string (if non-empty)
+ string nspace; ///< namespace
+ int64_t hash; ///< hash position (if >= 0)
explicit object_locator_t()
- : pool(-1) {}
+ : pool(-1), hash(-1) {}
explicit object_locator_t(int64_t po)
- : pool(po) {}
+ : pool(po), hash(-1) {}
+ explicit object_locator_t(int64_t po, int64_t ps)
+ : pool(po), hash(ps) {}
explicit object_locator_t(int64_t po, string ns)
- : pool(po), nspace(ns) {}
+ : pool(po), nspace(ns), hash(-1) {}
+ explicit object_locator_t(int64_t po, string ns, int64_t ps)
+ : pool(po), nspace(ns), hash(ps) {}
explicit object_locator_t(int64_t po, string ns, string s)
- : pool(po), key(s), nspace(ns) {}
+ : pool(po), key(s), nspace(ns), hash(-1) {}
explicit object_locator_t(const hobject_t& soid)
- : pool(soid.pool), key(soid.get_key()), nspace(soid.nspace) {}
+ : pool(soid.pool), key(soid.get_key()), nspace(soid.nspace), hash(-1) {}
int64_t get_pool() const {
return pool;
@@ -110,6 +128,7 @@ struct object_locator_t {
pool = -1;
key = "";
nspace = "";
+ hash = -1;
}
bool empty() const {
@@ -124,7 +143,7 @@ struct object_locator_t {
WRITE_CLASS_ENCODER(object_locator_t)
inline bool operator==(const object_locator_t& l, const object_locator_t& r) {
- return l.pool == r.pool && l.key == r.key && l.nspace == r.nspace;
+ return l.pool == r.pool && l.key == r.key && l.nspace == r.nspace && l.hash == r.hash;
}
inline bool operator!=(const object_locator_t& l, const object_locator_t& r) {
return !(l == r);
@@ -189,6 +208,7 @@ enum {
CEPH_OSD_RMW_FLAG_CLASS_READ = (1 << 3),
CEPH_OSD_RMW_FLAG_CLASS_WRITE = (1 << 4),
CEPH_OSD_RMW_FLAG_PGOP = (1 << 5),
+ CEPH_OSD_RMW_FLAG_CACHE = (1 << 6),
};
@@ -200,13 +220,6 @@ enum {
#define CEPH_CAS_NS 3
#define CEPH_OSDMETADATA_NS 0xff
-// poolsets
-enum {
- CEPH_DATA_RULE,
- CEPH_METADATA_RULE,
- CEPH_RBD_RULE,
-};
-
#define OSD_SUPERBLOCK_POBJECT hobject_t(sobject_t(object_t("osd_superblock"), 0))
// placement seed (a hash value)
@@ -231,7 +244,7 @@ struct pg_t {
int32_t m_preferred;
pg_t() : m_pool(0), m_seed(0), m_preferred(-1) {}
- pg_t(ps_t seed, uint64_t pool, int pref) {
+ pg_t(ps_t seed, uint64_t pool, int pref=-1) {
m_seed = seed;
m_pool = pool;
m_preferred = pref;
@@ -344,7 +357,7 @@ inline bool operator>=(const pg_t& l, const pg_t& r) {
ostream& operator<<(ostream& out, const pg_t &pg);
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash< pg_t >
{
size_t operator()( const pg_t& x ) const
@@ -353,7 +366,7 @@ namespace __gnu_cxx {
return H((x.pool() & 0xffffffff) ^ (x.pool() >> 32) ^ x.ps() ^ x.preferred());
}
};
-}
+CEPH_HASH_NAMESPACE_END
// ----------------------
@@ -437,7 +450,7 @@ inline ostream& operator<<(ostream& out, const coll_t& c) {
return out;
}
-namespace __gnu_cxx {
+CEPH_HASH_NAMESPACE_START
template<> struct hash<coll_t> {
size_t operator()(const coll_t &c) const {
size_t h = 0;
@@ -454,7 +467,7 @@ namespace __gnu_cxx {
return h;
}
};
-}
+CEPH_HASH_NAMESPACE_END
inline ostream& operator<<(ostream& out, const ceph_object_layout &ol)
{
@@ -537,32 +550,32 @@ inline ostream& operator<<(ostream& out, const eversion_t e) {
}
/**
- * filestore_perf_stat_t
+ * objectstore_perf_stat_t
*
* current perf information about the osd
*/
-struct filestore_perf_stat_t {
+struct objectstore_perf_stat_t {
// cur_op_latency is in ms since double add/sub are not associative
uint32_t filestore_commit_latency;
uint32_t filestore_apply_latency;
- filestore_perf_stat_t() :
+ objectstore_perf_stat_t() :
filestore_commit_latency(0), filestore_apply_latency(0) {}
- void add(const filestore_perf_stat_t &o) {
+ void add(const objectstore_perf_stat_t &o) {
filestore_commit_latency += o.filestore_commit_latency;
filestore_apply_latency += o.filestore_apply_latency;
}
- void sub(const filestore_perf_stat_t &o) {
+ void sub(const objectstore_perf_stat_t &o) {
filestore_commit_latency -= o.filestore_commit_latency;
filestore_apply_latency -= o.filestore_apply_latency;
}
void dump(Formatter *f) const;
void encode(bufferlist &bl) const;
void decode(bufferlist::iterator &bl);
- static void generate_test_instances(std::list<filestore_perf_stat_t*>& o);
+ static void generate_test_instances(std::list<objectstore_perf_stat_t*>& o);
};
-WRITE_CLASS_ENCODER(filestore_perf_stat_t)
+WRITE_CLASS_ENCODER(objectstore_perf_stat_t)
/** osd_stat
* aggregate stats for an osd
@@ -574,7 +587,7 @@ struct osd_stat_t {
pow2_hist_t op_queue_age_hist;
- filestore_perf_stat_t fs_perf_stat;
+ objectstore_perf_stat_t fs_perf_stat;
osd_stat_t() : kb(0), kb_used(0), kb_avail(0),
snap_trim_queue_len(0), num_snap_trimming(0) {}
@@ -686,29 +699,36 @@ inline ostream& operator<<(ostream& out, const pool_snap_info_t& si) {
*/
struct pg_pool_t {
enum {
- TYPE_REP = 1, // replication
- TYPE_RAID4 = 2, // raid4 (never implemented)
+ TYPE_REPLICATED = 1, // replication
+ //TYPE_RAID4 = 2, // raid4 (never implemented)
+ TYPE_ERASURE = 3, // erasure-coded
};
static const char *get_type_name(int t) {
switch (t) {
- case TYPE_REP: return "rep";
- case TYPE_RAID4: return "raid4";
+ case TYPE_REPLICATED: return "replicated";
+ //case TYPE_RAID4: return "raid4";
+ case TYPE_ERASURE: return "erasure";
default: return "???";
}
}
const char *get_type_name() const {
return get_type_name(type);
}
+ static const char* get_default_type() {
+ return "replicated";
+ }
enum {
FLAG_HASHPSPOOL = 1, // hash pg seed and pool together (instead of adding)
FLAG_FULL = 2, // pool is full
+ FLAG_DEBUG_FAKE_EC_POOL = 1<<2, // require ReplicatedPG to act like an EC pg
};
static const char *get_flag_name(int f) {
switch (f) {
case FLAG_HASHPSPOOL: return "hashpspool";
case FLAG_FULL: return "full";
+ case FLAG_DEBUG_FAKE_EC_POOL: return "require_local_rollback";
default: return "???";
}
}
@@ -730,14 +750,14 @@ struct pg_pool_t {
typedef enum {
CACHEMODE_NONE = 0, ///< no caching
CACHEMODE_WRITEBACK = 1, ///< write to cache, flush later
- CACHEMODE_INVALIDATE_FORWARD = 2, ///< delete from cache, forward write
+ CACHEMODE_FORWARD = 2, ///< forward if not in cache
CACHEMODE_READONLY = 3, ///< handle reads, forward writes [not strongly consistent]
} cache_mode_t;
static const char *get_cache_mode_name(cache_mode_t m) {
switch (m) {
case CACHEMODE_NONE: return "none";
case CACHEMODE_WRITEBACK: return "writeback";
- case CACHEMODE_INVALIDATE_FORWARD: return "invalidate+forward";
+ case CACHEMODE_FORWARD: return "forward";
case CACHEMODE_READONLY: return "readonly";
default: return "unknown";
}
@@ -747,8 +767,8 @@ struct pg_pool_t {
return CACHEMODE_NONE;
if (s == "writeback")
return CACHEMODE_WRITEBACK;
- if (s == "invalidate+forward")
- return CACHEMODE_INVALIDATE_FORWARD;
+ if (s == "forward")
+ return CACHEMODE_FORWARD;
if (s == "readonly")
return CACHEMODE_READONLY;
return (cache_mode_t)-1;
@@ -757,25 +777,25 @@ struct pg_pool_t {
return get_cache_mode_name(cache_mode);
}
- uint64_t flags; /// FLAG_*
- __u8 type; /// TYPE_*
- __u8 size, min_size; /// number of osds in each pg
- __u8 crush_ruleset; /// crush placement rule set
- __u8 object_hash; /// hash mapping object name to ps
+ uint64_t flags; ///< FLAG_*
+ __u8 type; ///< TYPE_*
+ __u8 size, min_size; ///< number of osds in each pg
+ __u8 crush_ruleset; ///< crush placement rule set
+ __u8 object_hash; ///< hash mapping object name to ps
private:
- __u32 pg_num, pgp_num; /// number of pgs
+ __u32 pg_num, pgp_num; ///< number of pgs
public:
- map<string,string> properties; /// interpreted according to the pool type
- epoch_t last_change; /// most recent epoch changed, exclusing snapshot changes
- snapid_t snap_seq; /// seq for per-pool snapshot
- epoch_t snap_epoch; /// osdmap epoch of last snap
- uint64_t auid; /// who owns the pg
- __u32 crash_replay_interval; /// seconds to allow clients to replay ACKed but unCOMMITted requests
+ map<string,string> properties; ///< interpreted according to the pool type
+ epoch_t last_change; ///< most recent epoch changed, exclusing snapshot changes
+ snapid_t snap_seq; ///< seq for per-pool snapshot
+ epoch_t snap_epoch; ///< osdmap epoch of last snap
+ uint64_t auid; ///< who owns the pg
+ __u32 crash_replay_interval; ///< seconds to allow clients to replay ACKed but unCOMMITted requests
- uint64_t quota_max_bytes; /// maximum number of bytes for this pool
- uint64_t quota_max_objects; /// maximum number of objects for this pool
+ uint64_t quota_max_bytes; ///< maximum number of bytes for this pool
+ uint64_t quota_max_objects; ///< maximum number of objects for this pool
/*
* Pool snaps (global to this pool). These define a SnapContext for
@@ -800,7 +820,6 @@ public:
int64_t write_tier; ///< pool/tier for objecter to direct writes to
cache_mode_t cache_mode; ///< cache pool mode
-
bool is_tier() const { return tier_of >= 0; }
void clear_tier() { tier_of = -1; }
bool has_read_tier() const { return read_tier >= 0; }
@@ -808,6 +827,10 @@ public:
bool has_write_tier() const { return write_tier >= 0; }
void clear_write_tier() { write_tier = -1; }
+ HitSet::Params hit_set_params; ///< The HitSet params to use on this pool
+ uint32_t hit_set_period; ///< periodicity of HitSet segments (seconds)
+ uint32_t hit_set_count; ///< number of periods to retain
+
pg_pool_t()
: flags(0), type(0), size(0), min_size(0),
crush_ruleset(0), object_hash(0),
@@ -819,12 +842,21 @@ public:
quota_max_bytes(0), quota_max_objects(0),
pg_num_mask(0), pgp_num_mask(0),
tier_of(-1), read_tier(-1), write_tier(-1),
- cache_mode(CACHEMODE_NONE)
+ cache_mode(CACHEMODE_NONE),
+ hit_set_params(),
+ hit_set_period(0),
+ hit_set_count(0)
{ }
void dump(Formatter *f) const;
uint64_t get_flags() const { return flags; }
+
+ /// This method will later return true for ec pools as well
+ bool ec_pool() const {
+ return flags & FLAG_DEBUG_FAKE_EC_POOL;
+ }
+
unsigned get_type() const { return type; }
unsigned get_size() const { return size; }
unsigned get_min_size() const { return min_size; }
@@ -842,8 +874,19 @@ public:
void set_snap_seq(snapid_t s) { snap_seq = s; }
void set_snap_epoch(epoch_t e) { snap_epoch = e; }
- bool is_rep() const { return get_type() == TYPE_REP; }
- bool is_raid4() const { return get_type() == TYPE_RAID4; }
+ bool is_replicated() const { return get_type() == TYPE_REPLICATED; }
+ bool is_erasure() const { return get_type() == TYPE_ERASURE; }
+
+ bool can_shift_osds() const {
+ switch (get_type()) {
+ case TYPE_REPLICATED:
+ return true;
+ case TYPE_ERASURE:
+ return false;
+ default:
+ assert(0 == "unhandled pool type");
+ }
+ }
unsigned get_pg_num() const { return pg_num; }
unsigned get_pgp_num() const { return pgp_num; }
@@ -904,6 +947,12 @@ public:
SnapContext get_snap_context() const;
+ /// hash a object name+namespace key to a hash position
+ uint32_t hash_key(const string& key, const string& ns) const;
+
+ /// round a hash position down to a pg num
+ uint32_t raw_hash_to_pg(uint32_t v) const;
+
/*
* map a raw pg (with full precision ps) into an actual pg, for storage
*/
@@ -947,6 +996,8 @@ struct object_stat_sum_t {
int64_t num_objects_recovered;
int64_t num_bytes_recovered;
int64_t num_keys_recovered;
+ int64_t num_objects_dirty;
+ int64_t num_whiteouts;
object_stat_sum_t()
: num_bytes(0),
@@ -957,7 +1008,9 @@ struct object_stat_sum_t {
num_deep_scrub_errors(0),
num_objects_recovered(0),
num_bytes_recovered(0),
- num_keys_recovered(0)
+ num_keys_recovered(0),
+ num_objects_dirty(0),
+ num_whiteouts(0)
{}
void floor(int64_t f) {
@@ -979,6 +1032,8 @@ struct object_stat_sum_t {
FLOOR(num_objects_recovered);
FLOOR(num_bytes_recovered);
FLOOR(num_keys_recovered);
+ FLOOR(num_objects_dirty);
+ FLOOR(num_whiteouts);
#undef FLOOR
}
@@ -1106,6 +1161,10 @@ struct pg_stat_t {
utime_t last_became_active;
+ /// true if num_objects_dirty is not accurate (because it was not
+ /// maintained starting from pool creation)
+ bool dirty_stats_invalid;
+
pg_stat_t()
: reported_seq(0),
reported_epoch(0),
@@ -1114,7 +1173,8 @@ struct pg_stat_t {
parent_split_bits(0),
stats_invalid(false),
log_size(0), ondisk_log_size(0),
- mapping_epoch(0)
+ mapping_epoch(0),
+ dirty_stats_invalid(false)
{ }
epoch_t get_effective_last_epoch_clean() const {
@@ -1202,6 +1262,51 @@ struct pool_stat_t {
WRITE_CLASS_ENCODER_FEATURES(pool_stat_t)
+// -----------------------------------------
+
+/**
+ * pg_hit_set_info_t - information about a single recorded HitSet
+ *
+ * Track basic metadata about a HitSet, like the nubmer of insertions
+ * and the time range it covers.
+ */
+struct pg_hit_set_info_t {
+ utime_t begin, end; ///< time interval
+ eversion_t version; ///< version this HitSet object was written
+
+ pg_hit_set_info_t() {}
+ pg_hit_set_info_t(utime_t b)
+ : begin(b) {}
+
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<pg_hit_set_info_t*>& o);
+};
+WRITE_CLASS_ENCODER(pg_hit_set_info_t)
+
+/**
+ * pg_hit_set_history_t - information about a history of hitsets
+ *
+ * Include information about the currently accumulating hit set as well
+ * as archived/historical ones.
+ */
+struct pg_hit_set_history_t {
+ eversion_t current_last_update; ///< last version inserted into current set
+ utime_t current_last_stamp; ///< timestamp of last insert
+ pg_hit_set_info_t current_info; ///< metadata about the current set
+ list<pg_hit_set_info_t> history; ///< archived sets, sorted oldest -> newest
+
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<pg_hit_set_history_t*>& o);
+};
+WRITE_CLASS_ENCODER(pg_hit_set_history_t)
+
+
+// -----------------------------------------
+
/**
* pg_history_t - information about recent pg peering/mapping history
*
@@ -1318,6 +1423,7 @@ struct pg_info_t {
pg_stat_t stats;
pg_history_t history;
+ pg_hit_set_history_t hit_set;
pg_info_t()
: last_epoch_started(0), last_user_version(0),
@@ -1332,7 +1438,7 @@ struct pg_info_t {
bool is_empty() const { return last_update.version == 0; }
bool dne() const { return history.epoch_created == 0; }
- bool is_incomplete() const { return last_backfill != hobject_t::get_max(); }
+ bool is_incomplete() const { return !last_backfill.is_max(); }
void encode(bufferlist& bl) const;
void decode(bufferlist::iterator& p);
@@ -1411,8 +1517,8 @@ struct pg_interval_t {
const vector<int> &new_up, ///< [in] up as of osdmap
epoch_t same_interval_since, ///< [in] as of osdmap
epoch_t last_epoch_clean, ///< [in] current
- std::tr1::shared_ptr<const OSDMap> osdmap, ///< [in] current map
- std::tr1::shared_ptr<const OSDMap> lastmap, ///< [in] last map
+ ceph::shared_ptr<const OSDMap> osdmap, ///< [in] current map
+ ceph::shared_ptr<const OSDMap> lastmap, ///< [in] last map
int64_t poolid, ///< [in] pool for pg
pg_t pgid, ///< [in] pgid for pg
map<epoch_t, pg_interval_t> *past_intervals,///< [out] intervals
@@ -1483,6 +1589,100 @@ inline ostream& operator<<(ostream& out, const pg_query_t& q) {
return out;
}
+class PGBackend;
+class ObjectModDesc {
+ bool can_local_rollback;
+ bool stashed;
+public:
+ class Visitor {
+ public:
+ virtual void append(uint64_t old_offset) {}
+ virtual void setattrs(map<string, boost::optional<bufferlist> > &attrs) {}
+ virtual void rmobject(version_t old_version) {}
+ virtual void create() {}
+ virtual void update_snaps(set<snapid_t> &old_snaps) {}
+ virtual ~Visitor() {}
+ };
+ void visit(Visitor *visitor) const;
+ mutable bufferlist bl;
+ enum ModID {
+ APPEND = 1,
+ SETATTRS = 2,
+ DELETE = 3,
+ CREATE = 4,
+ UPDATE_SNAPS = 5
+ };
+ ObjectModDesc() : can_local_rollback(true), stashed(false) {}
+ void claim(ObjectModDesc &other) {
+ bl.clear();
+ bl.claim(other.bl);
+ can_local_rollback = other.can_local_rollback;
+ stashed = other.stashed;
+ }
+ void append_id(ModID id) {
+ uint8_t _id(id);
+ ::encode(_id, bl);
+ }
+ void append(uint64_t old_size) {
+ if (!can_local_rollback || stashed)
+ return;
+ ENCODE_START(1, 1, bl);
+ append_id(APPEND);
+ ::encode(old_size, bl);
+ ENCODE_FINISH(bl);
+ }
+ void setattrs(map<string, boost::optional<bufferlist> > &old_attrs) {
+ if (!can_local_rollback || stashed)
+ return;
+ ENCODE_START(1, 1, bl);
+ append_id(SETATTRS);
+ ::encode(old_attrs, bl);
+ ENCODE_FINISH(bl);
+ }
+ bool rmobject(version_t deletion_version) {
+ if (!can_local_rollback || stashed)
+ return false;
+ ENCODE_START(1, 1, bl);
+ append_id(DELETE);
+ ::encode(deletion_version, bl);
+ ENCODE_FINISH(bl);
+ stashed = true;
+ return true;
+ }
+ void create() {
+ if (!can_local_rollback || stashed)
+ return;
+ ENCODE_START(1, 1, bl);
+ append_id(CREATE);
+ ENCODE_FINISH(bl);
+ }
+ void update_snaps(set<snapid_t> &old_snaps) {
+ if (!can_local_rollback || stashed)
+ return;
+ ENCODE_START(1, 1, bl);
+ append_id(UPDATE_SNAPS);
+ ::encode(old_snaps, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ // cannot be rolled back
+ void mark_unrollbackable() {
+ can_local_rollback = false;
+ bl.clear();
+ }
+ bool can_rollback() const {
+ return can_local_rollback;
+ }
+ bool empty() const {
+ return can_local_rollback && (bl.length() == 0);
+ }
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bl);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<ObjectModDesc*>& o);
+};
+WRITE_CLASS_ENCODER(ObjectModDesc)
+
/**
* pg_log_entry_t - single entry/event in pg log
@@ -1490,18 +1690,22 @@ inline ostream& operator<<(ostream& out, const pg_query_t& q) {
*/
struct pg_log_entry_t {
enum {
- MODIFY = 1,
- CLONE = 2,
- DELETE = 3,
+ MODIFY = 1, // some unspecified modification (but not *all* modifications)
+ CLONE = 2, // cloned object from head
+ DELETE = 3, // deleted object
BACKLOG = 4, // event invented by generate_backlog [deprecated]
LOST_REVERT = 5, // lost new version, revert to an older version.
LOST_DELETE = 6, // lost new version, revert to no object (deleted).
LOST_MARK = 7, // lost new version, now EIO
+ PROMOTE = 8, // promoted object from another tier
+ CLEAN = 9, // mark an object clean
};
static const char *get_op_name(int op) {
switch (op) {
case MODIFY:
return "modify ";
+ case PROMOTE:
+ return "promote ";
case CLONE:
return "clone ";
case DELETE:
@@ -1514,6 +1718,8 @@ struct pg_log_entry_t {
return "l_delete";
case LOST_MARK:
return "l_mark ";
+ case CLEAN:
+ return "clean ";
default:
return "unknown ";
}
@@ -1533,6 +1739,9 @@ struct pg_log_entry_t {
bool invalid_pool; // only when decoding pool-less hobject based entries
uint64_t offset; // [soft state] my offset on disk
+
+ /// describes state for a locally-rollbackable entry
+ ObjectModDesc mod_desc;
pg_log_entry_t()
: op(0), user_version(0),
@@ -1548,13 +1757,17 @@ struct pg_log_entry_t {
bool is_clone() const { return op == CLONE; }
bool is_modify() const { return op == MODIFY; }
+ bool is_promote() const { return op == PROMOTE; }
+ bool is_clean() const { return op == CLEAN; }
bool is_backlog() const { return op == BACKLOG; }
bool is_lost_revert() const { return op == LOST_REVERT; }
bool is_lost_delete() const { return op == LOST_DELETE; }
bool is_lost_mark() const { return op == LOST_MARK; }
bool is_update() const {
- return is_clone() || is_modify() || is_backlog() || is_lost_revert() || is_lost_mark();
+ return
+ is_clone() || is_modify() || is_promote() || is_clean() ||
+ is_backlog() || is_lost_revert() || is_lost_mark();
}
bool is_delete() const {
return op == DELETE || op == LOST_DELETE;
@@ -1595,13 +1808,16 @@ struct pg_log_t {
eversion_t head; // newest entry
eversion_t tail; // version prior to oldest
+ // We can rollback rollback-able entries > can_rollback_to
+ eversion_t can_rollback_to;
+
list<pg_log_entry_t> log; // the actual log.
pg_log_t() {}
void clear() {
eversion_t z;
- head = tail = z;
+ can_rollback_to = head = tail = z;
log.clear();
}
@@ -1689,7 +1905,8 @@ WRITE_CLASS_ENCODER(pg_log_t)
inline ostream& operator<<(ostream& out, const pg_log_t& log)
{
- out << "log(" << log.tail << "," << log.head << "]";
+ out << "log((" << log.tail << "," << log.head << "], crt="
+ << log.can_rollback_to << ")";
return out;
}
@@ -1857,8 +2074,14 @@ struct object_copy_data_t {
utime_t mtime;
map<string, bufferlist> attrs;
bufferlist data;
+ bufferlist omap_header;
map<string, bufferlist> omap;
string category;
+
+ /// which snaps we are defined for (if a snap and not the head)
+ vector<snapid_t> snaps;
+ ///< latest snap seq for the object (if head)
+ snapid_t snap_seq;
public:
object_copy_data_t() : size((uint64_t)-1) {}
@@ -1956,11 +2179,12 @@ public:
// last interval over which i mounted and was then active
epoch_t mounted; // last epoch i mounted
epoch_t clean_thru; // epoch i was active and clean thru
+ epoch_t last_map_marked_full; // last epoch osdmap was marked full
OSDSuperblock() :
whoami(-1),
current_epoch(0), oldest_map(0), newest_map(0), weight(0),
- mounted(0), clean_thru(0) {
+ mounted(0), clean_thru(0), last_map_marked_full(0) {
}
void encode(bufferlist &bl) const;
@@ -2007,6 +2231,9 @@ struct SnapSet {
bufferlist::iterator p = bl.begin();
decode(p);
}
+
+ /// populate SnapSet from a librados::snap_set_t
+ void from_snap_set(const librados::snap_set_t& ss);
void encode(bufferlist& bl) const;
void decode(bufferlist::iterator& bl);
@@ -2185,7 +2412,7 @@ struct SnapSetContext {
struct ObjectContext;
-typedef std::tr1::shared_ptr<ObjectContext> ObjectContextRef;
+typedef ceph::shared_ptr<ObjectContext> ObjectContextRef;
struct ObjectContext {
ObjectState obs;
@@ -2201,7 +2428,7 @@ public:
int unstable_writes, readers, writers_waiting, readers_waiting;
/// in-progress copyfrom ops for this object
- int copyfrom_readside;
+ bool blocked;
// set if writes for this object are blocked on another objects recovery
ObjectContextRef blocked_by; // object blocking our writes
@@ -2216,9 +2443,21 @@ public:
RWREAD,
RWWRITE
};
- State state; /// rw state
- uint64_t count; /// number of readers or writers
- list<OpRequestRef> waiters; /// ops waiting on state change
+ static const char *get_state_name(State s) {
+ switch (s) {
+ case RWNONE: return "none";
+ case RWREAD: return "read";
+ case RWWRITE: return "write";
+ default: return "???";
+ }
+ }
+ const char *get_state_name() const {
+ return get_state_name(state);
+ }
+
+ State state; ///< rw state
+ uint64_t count; ///< number of readers or writers
+ list<OpRequestRef> waiters; ///< ops waiting on state change
/// if set, restart backfill when we can get a read lock
bool backfill_read_marker;
@@ -2337,7 +2576,7 @@ public:
destructor_callback(0),
lock("ReplicatedPG::ObjectContext::lock"),
unstable_writes(0), readers(0), writers_waiting(0), readers_waiting(0),
- copyfrom_readside(0) {}
+ blocked(false) {}
~ObjectContext() {
assert(rwstate.empty());
@@ -2345,8 +2584,16 @@ public:
destructor_callback->complete(0);
}
+ void start_block() {
+ assert(!blocked);
+ blocked = true;
+ }
+ void stop_block() {
+ assert(blocked);
+ blocked = false;
+ }
bool is_blocked() const {
- return copyfrom_readside > 0;
+ return blocked;
}
// do simple synchronous mutual exclusion, for now. now waitqueues or anything fancy.
@@ -2384,9 +2631,27 @@ public:
cond.Signal();
lock.Unlock();
}
+
+ // attr cache
+ map<string, bufferlist> attr_cache;
+
+ void fill_in_setattrs(const set<string> &changing, ObjectModDesc *mod) {
+ map<string, boost::optional<bufferlist> > to_set;
+ for (set<string>::const_iterator i = changing.begin();
+ i != changing.end();
+ ++i) {
+ map<string, bufferlist>::iterator iter = attr_cache.find(*i);
+ if (iter != attr_cache.end()) {
+ to_set[*i] = iter->second;
+ } else {
+ to_set[*i];
+ }
+ }
+ mod->setattrs(to_set);
+ }
};
-inline ostream& operator<<(ostream& out, ObjectState& obs)
+inline ostream& operator<<(ostream& out, const ObjectState& obs)
{
out << obs.oi.soid;
if (!obs.exists)
@@ -2394,9 +2659,17 @@ inline ostream& operator<<(ostream& out, ObjectState& obs)
return out;
}
-inline ostream& operator<<(ostream& out, ObjectContext& obc)
+inline ostream& operator<<(ostream& out, const ObjectContext::RWState& rw)
+{
+ return out << "rwstate(" << rw.get_state_name()
+ << " n=" << rw.count
+ << " w=" << rw.waiters.size()
+ << ")";
+}
+
+inline ostream& operator<<(ostream& out, const ObjectContext& obc)
{
- return out << "obc(" << obc.obs << ")";
+ return out << "obc(" << obc.obs << " " << obc.rwstate << ")";
}
@@ -2809,4 +3082,9 @@ struct obj_list_snap_response_t {
WRITE_CLASS_ENCODER(obj_list_snap_response_t)
+enum scrub_error_type {
+ CLEAN,
+ DEEP_ERROR,
+ SHALLOW_ERROR
+};
#endif
diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index 81335b7..6160d70 100644
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -522,7 +522,7 @@ ObjectCacher::~ObjectCacher()
finisher.stop();
perf_stop();
// we should be empty.
- for (vector<hash_map<sobject_t, Object *> >::iterator i = objects.begin();
+ for (vector<ceph::unordered_map<sobject_t, Object *> >::iterator i = objects.begin();
i != objects.end();
++i)
assert(i->empty());
@@ -1457,6 +1457,12 @@ void ObjectCacher::flusher_entry()
ldout(cct, 10) << "flusher flushing aged dirty bh " << *bh << dendl;
bh_write(bh);
}
+ if (!max) {
+ // back off the lock to avoid starving other threads
+ lock.Unlock();
+ lock.Lock();
+ continue;
+ }
}
if (flusher_stop)
break;
@@ -1758,11 +1764,11 @@ uint64_t ObjectCacher::release_all()
ldout(cct, 10) << "release_all" << dendl;
uint64_t unclean = 0;
- vector<hash_map<sobject_t, Object*> >::iterator i = objects.begin();
+ vector<ceph::unordered_map<sobject_t, Object*> >::iterator i = objects.begin();
while (i != objects.end()) {
- hash_map<sobject_t, Object*>::iterator p = i->begin();
+ ceph::unordered_map<sobject_t, Object*>::iterator p = i->begin();
while (p != i->end()) {
- hash_map<sobject_t, Object*>::iterator n = p;
+ ceph::unordered_map<sobject_t, Object*>::iterator n = p;
++n;
Object *ob = p->second;
@@ -1847,10 +1853,10 @@ void ObjectCacher::verify_stats() const
ldout(cct, 10) << "verify_stats" << dendl;
loff_t clean = 0, zero = 0, dirty = 0, rx = 0, tx = 0, missing = 0, error = 0;
- for (vector<hash_map<sobject_t, Object*> >::const_iterator i = objects.begin();
+ for (vector<ceph::unordered_map<sobject_t, Object*> >::const_iterator i = objects.begin();
i != objects.end();
++i) {
- for (hash_map<sobject_t, Object*>::const_iterator p = i->begin();
+ for (ceph::unordered_map<sobject_t, Object*>::const_iterator p = i->begin();
p != i->end();
++p) {
Object *ob = p->second;
diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index a62a41f..d7ba9d8 100644
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -339,7 +339,7 @@ class ObjectCacher {
flush_set_callback_t flush_set_callback;
void *flush_set_callback_arg;
- vector<hash_map<sobject_t, Object*> > objects; // indexed by pool_id
+ vector<ceph::unordered_map<sobject_t, Object*> > objects; // indexed by pool_id
tid_t last_read_tid;
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index d2c574d..3da9be9 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -44,6 +44,7 @@
#include "common/config.h"
#include "common/perf_counters.h"
+#include "include/str_list.h"
#define dout_subsys ceph_subsys_objecter
@@ -127,6 +128,35 @@ enum {
};
+// config obs ----------------------------
+
+static const char *config_keys[] = {
+ "crush_location",
+ NULL
+};
+
+const char** Objecter::get_tracked_conf_keys() const
+{
+ return config_keys;
+}
+
+
+void Objecter::handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed)
+{
+ if (changed.count("crush_location")) {
+ crush_location.clear();
+ vector<string> lvec;
+ get_str_vec(cct->_conf->crush_location, ";, \t", lvec);
+ int r = CrushWrapper::parse_loc_multimap(lvec, &crush_location);
+ if (r < 0) {
+ lderr(cct) << "warning: crush_location '" << cct->_conf->crush_location
+ << "' does not parse" << dendl;
+ }
+ }
+}
+
+
// messages ------------------------------
void Objecter::init_unlocked()
@@ -447,7 +477,8 @@ void Objecter::dispatch(Message *m)
}
}
-void Objecter::scan_requests(bool skipped_map,
+void Objecter::scan_requests(bool force_resend,
+ bool force_resend_writes,
map<tid_t, Op*>& need_resend,
list<LingerOp*>& need_resend_linger,
map<tid_t, CommandOp*>& need_resend_command)
@@ -461,8 +492,7 @@ void Objecter::scan_requests(bool skipped_map,
int r = recalc_linger_op_target(op);
switch (r) {
case RECALC_OP_TARGET_NO_ACTION:
- // resend if skipped map; otherwise do nothing.
- if (!skipped_map)
+ if (!force_resend && !force_resend_writes)
break;
// -- fall-thru --
case RECALC_OP_TARGET_NEED_RESEND:
@@ -484,8 +514,8 @@ void Objecter::scan_requests(bool skipped_map,
int r = recalc_op_target(op);
switch (r) {
case RECALC_OP_TARGET_NO_ACTION:
- // resend if skipped map; otherwise do nothing.
- if (!skipped_map)
+ if (!force_resend &&
+ (!force_resend_writes || !(op->flags & CEPH_OSD_FLAG_WRITE)))
break;
// -- fall-thru --
case RECALC_OP_TARGET_NEED_RESEND:
@@ -508,7 +538,7 @@ void Objecter::scan_requests(bool skipped_map,
switch (r) {
case RECALC_OP_TARGET_NO_ACTION:
// resend if skipped map; otherwise do nothing.
- if (!skipped_map)
+ if (!force_resend && !force_resend_writes)
break;
// -- fall-thru --
case RECALC_OP_TARGET_NEED_RESEND:
@@ -537,8 +567,9 @@ void Objecter::handle_osd_map(MOSDMap *m)
}
bool was_pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
- bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || osdmap->test_flag(CEPH_OSDMAP_FULL);
-
+ bool was_full = osdmap->test_flag(CEPH_OSDMAP_FULL);
+ bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || was_full;
+
list<LingerOp*> need_resend_linger;
map<tid_t, Op*> need_resend;
map<tid_t, CommandOp*> need_resend_command;
@@ -586,8 +617,10 @@ void Objecter::handle_osd_map(MOSDMap *m)
continue;
}
logger->set(l_osdc_map_epoch, osdmap->get_epoch());
-
- scan_requests(skipped_map, need_resend, need_resend_linger, need_resend_command);
+
+ was_full = was_full || osdmap->test_flag(CEPH_OSDMAP_FULL);
+ scan_requests(skipped_map, was_full, need_resend, need_resend_linger,
+ need_resend_command);
// osd addr changes?
for (map<int,OSDSession*>::iterator p = osd_sessions.begin();
@@ -611,7 +644,8 @@ void Objecter::handle_osd_map(MOSDMap *m)
ldout(cct, 3) << "handle_osd_map decoding full epoch " << m->get_last() << dendl;
osdmap->decode(m->maps[m->get_last()]);
- scan_requests(false, need_resend, need_resend_linger, need_resend_command);
+ scan_requests(false, false, need_resend, need_resend_linger,
+ need_resend_command);
} else {
ldout(cct, 3) << "handle_osd_map hmm, i want a full map, requesting" << dendl;
monc->sub_want("osdmap", 0, CEPH_SUBSCRIBE_ONETIME);
@@ -627,34 +661,11 @@ void Objecter::handle_osd_map(MOSDMap *m)
if (was_pauserd || was_pausewr || pauserd || pausewr)
maybe_request_map();
- // unpause requests?
- if ((was_pauserd && !pauserd) ||
- (was_pausewr && !pausewr)) {
- for (map<tid_t,Op*>::iterator p = ops.begin();
- p != ops.end();
- ++p) {
- Op *op = p->second;
- if (op->paused &&
- !((op->flags & CEPH_OSD_FLAG_READ) && pauserd) && // not still paused as a read
- !((op->flags & CEPH_OSD_FLAG_WRITE) && pausewr)) // not still paused as a write
- need_resend[op->tid] = op;
- }
- for (map<tid_t, LingerOp*>::iterator lp = linger_ops.begin();
- lp != linger_ops.end();
- ++lp) {
- LingerOp *op = lp->second;
- if (!op->registered &&
- !pauserd && // not still paused as a read
- !((op->flags & CEPH_OSD_FLAG_WRITE) && pausewr)) // not still paused as a write
- need_resend_linger.push_back(op);
- }
- }
-
// resend requests
for (map<tid_t, Op*>::iterator p = need_resend.begin(); p != need_resend.end(); ++p) {
Op *op = p->second;
if (op->should_resend) {
- if (op->session) {
+ if (op->session && !op->paused) {
logger->inc(l_osdc_op_resend);
send_op(op);
}
@@ -969,6 +980,43 @@ void Objecter::wait_for_osd_map()
lock.Unlock();
}
+struct C_Objecter_GetVersion : public Context {
+ Objecter *objecter;
+ uint64_t oldest, newest;
+ Context *fin;
+ C_Objecter_GetVersion(Objecter *o, Context *c)
+ : objecter(o), oldest(0), newest(0), fin(c) {}
+ void finish(int r) {
+ if (r >= 0)
+ objecter->_get_latest_version(oldest, newest, fin);
+ else if (r == -EAGAIN) { // try again as instructed
+ objecter->wait_for_latest_osdmap(fin);
+ } else {
+ // it doesn't return any other error codes!
+ assert(0);
+ }
+ }
+};
+
+void Objecter::wait_for_latest_osdmap(Context *fin)
+{
+ ldout(cct, 10) << __func__ << dendl;
+ C_Objecter_GetVersion *c = new C_Objecter_GetVersion(this, fin);
+ monc->get_version("osdmap", &c->newest, &c->oldest, c);
+}
+
+void Objecter::_get_latest_version(epoch_t oldest, epoch_t newest, Context *fin)
+{
+ if (osdmap->get_epoch() >= newest) {
+ ldout(cct, 10) << __func__ << " latest " << newest << ", have it" << dendl;
+ if (fin)
+ fin->complete(0);
+ return;
+ }
+
+ ldout(cct, 10) << __func__ << " latest " << newest << ", waiting" << dendl;
+ wait_for_new_map(fin, newest, 0);
+}
void Objecter::maybe_request_map()
{
@@ -1001,7 +1049,8 @@ void Objecter::kick_requests(OSDSession *session)
++p;
logger->inc(l_osdc_op_resend);
if (op->should_resend) {
- resend[op->tid] = op;
+ if (!op->paused)
+ resend[op->tid] = op;
} else {
cancel_linger_op(op);
}
@@ -1320,11 +1369,35 @@ bool Objecter::is_pg_changed(vector<int>& o, vector<int>& n, bool any_change)
return false; // same primary (tho replicas may have changed)
}
-int Objecter::recalc_op_target(Op *op)
+bool Objecter::op_should_be_paused(Op *op)
{
- vector<int> acting;
- pg_t pgid = op->pgid;
+ bool pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
+ bool pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || osdmap->test_flag(CEPH_OSDMAP_FULL);
+
+ return (op->flags & CEPH_OSD_FLAG_READ && pauserd) ||
+ (op->flags & CEPH_OSD_FLAG_WRITE && pausewr);
+}
+
+int64_t Objecter::get_object_hash_position(int64_t pool, const string& key,
+ const string& ns)
+{
+ const pg_pool_t *p = osdmap->get_pg_pool(pool);
+ if (!p)
+ return -ENOENT;
+ return p->hash_key(key, ns);
+}
+int64_t Objecter::get_object_pg_hash_position(int64_t pool, const string& key,
+ const string& ns)
+{
+ const pg_pool_t *p = osdmap->get_pg_pool(pool);
+ if (!p)
+ return -ENOENT;
+ return p->raw_hash_to_pg(p->hash_key(key, ns));
+}
+
+int Objecter::recalc_op_target(Op *op)
+{
bool is_read = op->flags & CEPH_OSD_FLAG_READ;
bool is_write = op->flags & CEPH_OSD_FLAG_WRITE;
@@ -1338,7 +1411,8 @@ int Objecter::recalc_op_target(Op *op)
need_check_tiering = true;
}
- if (honor_cache_redirects && need_check_tiering) {
+ if (need_check_tiering &&
+ (op->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) {
const pg_pool_t *pi = osdmap->get_pg_pool(op->base_oloc.pool);
if (pi) {
if (is_read && pi->has_read_tier())
@@ -1348,18 +1422,31 @@ int Objecter::recalc_op_target(Op *op)
}
}
+ pg_t pgid;
if (op->precalc_pgid) {
assert(op->base_oid.name.empty()); // make sure this is a listing op
- ldout(cct, 10) << "recalc_op_target have " << pgid << " pool " << osdmap->have_pg_pool(pgid.pool()) << dendl;
- if (!osdmap->have_pg_pool(pgid.pool()))
+ ldout(cct, 10) << "recalc_op_target have " << op->base_pgid << " pool "
+ << osdmap->have_pg_pool(op->base_pgid.pool()) << dendl;
+ if (!osdmap->have_pg_pool(op->base_pgid.pool()))
return RECALC_OP_TARGET_POOL_DNE;
+ pgid = osdmap->raw_pg_to_pg(op->base_pgid);
} else {
- int ret = osdmap->object_locator_to_pg(op->target_oid, op->target_oloc, pgid);
+ int ret = osdmap->object_locator_to_pg(op->target_oid, op->target_oloc,
+ pgid);
if (ret == -ENOENT)
return RECALC_OP_TARGET_POOL_DNE;
}
+ vector<int> acting;
osdmap->pg_to_acting_osds(pgid, acting);
+ bool need_resend = false;
+
+ bool paused = op_should_be_paused(op);
+ if (!paused && paused != op->paused) {
+ op->paused = false;
+ need_resend = true;
+ }
+
if (op->pgid != pgid || is_pg_changed(op->acting, acting, op->used_replica)) {
op->pgid = pgid;
op->acting = acting;
@@ -1377,23 +1464,33 @@ int Objecter::recalc_op_target(Op *op)
op->used_replica = true;
osd = acting[p];
ldout(cct, 10) << " chose random osd." << osd << " of " << acting << dendl;
- } else if (read && (op->flags & CEPH_OSD_FLAG_LOCALIZE_READS)) {
- // look for a local replica
- int i;
- /* loop through the OSD replicas and see if any are local to read from.
- * We don't need to check the primary since we default to it. (Be
- * careful to preserve that default, which is why we iterate in reverse
- * order.) */
- for (i = acting.size()-1; i > 0; --i) {
- if (osdmap->get_addr(acting[i]).is_same_host(messenger->get_myaddr())) {
- op->used_replica = true;
- ldout(cct, 10) << " chose local osd." << acting[i] << " of " << acting << dendl;
- break;
+ } else if (read && (op->flags & CEPH_OSD_FLAG_LOCALIZE_READS) &&
+ acting.size() > 1) {
+ // look for a local replica. prefer the primary if the
+ // distance is the same.
+ int best;
+ int best_locality;
+ for (unsigned i = 0; i < acting.size(); ++i) {
+ int locality = osdmap->crush->get_common_ancestor_distance(
+ cct, acting[i], crush_location);
+ ldout(cct, 20) << __func__ << " localize: rank " << i
+ << " osd." << acting[i]
+ << " locality " << locality << dendl;
+ if (i == 0 ||
+ (locality >= 0 && best_locality >= 0 &&
+ locality < best_locality) ||
+ (best_locality < 0 && locality >= 0)) {
+ best = i;
+ best_locality = locality;
+ if (i)
+ op->used_replica = true;
}
}
- osd = acting[i];
- } else
+ assert(best >= 0);
+ osd = acting[best];
+ } else {
osd = acting[0];
+ }
s = get_session(osd);
}
@@ -1407,6 +1504,9 @@ int Objecter::recalc_op_target(Op *op)
else
num_homeless_ops++;
}
+ need_resend = true;
+ }
+ if (need_resend) {
return RECALC_OP_TARGET_NEED_RESEND;
}
return RECALC_OP_TARGET_NO_ACTION;
@@ -1615,7 +1715,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
ldout(cct, 5) << " got redirect reply; redirecting" << dendl;
unregister_op(op);
m->get_redirect().combine_with_locator(op->target_oloc, op->target_oid.name);
- op_submit(op);
+ _op_submit(op);
m->put();
return;
}
@@ -1623,7 +1723,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
if (rc == -EAGAIN) {
ldout(cct, 7) << " got -EAGAIN, resubmitting" << dendl;
unregister_op(op);
- op_submit(op);
+ _op_submit(op);
m->put();
return;
}
@@ -1711,18 +1811,46 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
}
-void Objecter::list_objects(ListContext *list_context, Context *onfinish) {
+uint32_t Objecter::list_objects_seek(ListContext *list_context,
+ uint32_t pos)
+{
+ assert(client_lock.is_locked());
+ pg_t actual = osdmap->raw_pg_to_pg(pg_t(pos, list_context->pool_id));
+ ldout(cct, 10) << "list_objects_seek " << list_context
+ << " pos " << pos << " -> " << actual << dendl;
+ list_context->current_pg = actual.ps();
+ list_context->cookie = collection_list_handle_t();
+ list_context->at_end_of_pg = false;
+ list_context->at_end_of_pool = false;
+ list_context->current_pg_epoch = 0;
+ return list_context->current_pg;
+}
+void Objecter::list_objects(ListContext *list_context, Context *onfinish)
+{
+ assert(client_lock.is_locked());
ldout(cct, 10) << "list_objects" << dendl;
- ldout(cct, 20) << "pool_id " << list_context->pool_id
- << "\npool_snap_seq " << list_context->pool_snap_seq
- << "\nmax_entries " << list_context->max_entries
- << "\nlist_context " << list_context
- << "\nonfinish " << onfinish
- << "\nlist_context->current_pg" << list_context->current_pg
- << "\nlist_context->cookie" << list_context->cookie << dendl;
-
- if (list_context->at_end) {
+ ldout(cct, 20) << " pool_id " << list_context->pool_id
+ << " pool_snap_seq " << list_context->pool_snap_seq
+ << " max_entries " << list_context->max_entries
+ << " list_context " << list_context
+ << " onfinish " << onfinish
+ << " list_context->current_pg " << list_context->current_pg
+ << " list_context->cookie " << list_context->cookie << dendl;
+
+ if (list_context->at_end_of_pg) {
+ list_context->at_end_of_pg = false;
+ ++list_context->current_pg;
+ list_context->current_pg_epoch = 0;
+ list_context->cookie = collection_list_handle_t();
+ if (list_context->current_pg >= list_context->starting_pg_num) {
+ list_context->at_end_of_pool = true;
+ ldout(cct, 20) << " no more pgs; reached end of pool" << dendl;
+ } else {
+ ldout(cct, 20) << " move to next pg " << list_context->current_pg << dendl;
+ }
+ }
+ if (list_context->at_end_of_pool) {
onfinish->complete(0);
return;
}
@@ -1736,46 +1864,30 @@ void Objecter::list_objects(ListContext *list_context, Context *onfinish) {
}
if (list_context->starting_pg_num != pg_num) {
// start reading from the beginning; the pgs have changed
- ldout(cct, 10) << "The placement groups have changed, restarting with " << pg_num << dendl;
+ ldout(cct, 10) << " pg_num changed; restarting with " << pg_num << dendl;
list_context->current_pg = 0;
list_context->cookie = collection_list_handle_t();
list_context->current_pg_epoch = 0;
list_context->starting_pg_num = pg_num;
}
- if (list_context->current_pg == pg_num){ //this context got all the way through
- onfinish->complete(0);
- return;
- }
+ assert(list_context->current_pg <= pg_num);
ObjectOperation op;
op.pg_ls(list_context->max_entries, list_context->filter, list_context->cookie,
list_context->current_pg_epoch);
-
- bufferlist *bl = new bufferlist();
- C_List *onack = new C_List(list_context, onfinish, bl, this);
-
- object_t oid;
+ list_context->bl.clear();
+ C_List *onack = new C_List(list_context, onfinish, this);
object_locator_t oloc(list_context->pool_id, list_context->nspace);
-
- //
- Op *o = new Op(oid, oloc, op.ops, CEPH_OSD_FLAG_READ, onack, NULL, NULL);
- o->priority = op.priority;
- o->snapid = list_context->pool_snap_seq;
- o->outbl = bl;
- o->reply_epoch = &onack->epoch;
-
- o->pgid = pg_t(list_context->current_pg, list_context->pool_id, -1);
- o->precalc_pgid = true;
-
- op_submit(o);
+ pg_read(list_context->current_pg, oloc, op,
+ &list_context->bl, 0, onack, &onack->epoch);
}
-void Objecter::_list_reply(ListContext *list_context, int r, bufferlist *bl,
+void Objecter::_list_reply(ListContext *list_context, int r,
Context *final_finish, epoch_t reply_epoch)
{
ldout(cct, 10) << "_list_reply" << dendl;
- bufferlist::iterator iter = bl->begin();
+ bufferlist::iterator iter = list_context->bl.begin();
pg_ls_response_t response;
bufferlist extra_info;
::decode(response, iter);
@@ -1785,56 +1897,44 @@ void Objecter::_list_reply(ListContext *list_context, int r, bufferlist *bl,
list_context->cookie = response.handle;
if (!list_context->current_pg_epoch) {
// first pgls result, set epoch marker
- ldout(cct, 20) << "first pgls piece, reply_epoch is " << reply_epoch << dendl;
+ ldout(cct, 20) << " first pgls piece, reply_epoch is "
+ << reply_epoch << dendl;
list_context->current_pg_epoch = reply_epoch;
}
int response_size = response.entries.size();
- ldout(cct, 20) << "response.entries.size " << response_size
- << ", response.entries " << response.entries << dendl;
+ ldout(cct, 20) << " response.entries.size " << response_size
+ << ", response.entries " << response.entries << dendl;
list_context->extra_info.append(extra_info);
if (response_size) {
- ldout(cct, 20) << "got a response with objects, proceeding" << dendl;
list_context->list.merge(response.entries);
- if (response_size >= list_context->max_entries) {
- final_finish->complete(0);
- delete bl;
- return;
- }
-
- // ask for fewer objects next time around
- list_context->max_entries -= response_size;
+ }
- // if the osd returns 1 (newer code), or no entries, it means we
- // hit the end of the pg.
- if (r == 0 && response_size > 0) {
- // not yet done with this pg
- delete bl;
+ // if the osd returns 1 (newer code), or no entries, it means we
+ // hit the end of the pg.
+ if (response_size == 0 || r == 1) {
+ ldout(cct, 20) << " at end of pg" << dendl;
+ list_context->at_end_of_pg = true;
+ } else {
+ // there is more for this pg; get it?
+ if (response_size < list_context->max_entries) {
+ list_context->max_entries -= response_size;
list_objects(list_context, final_finish);
return;
}
}
-
- // if we make this this far, there are no objects left in the current pg, but we want more!
- ++list_context->current_pg;
- list_context->current_pg_epoch = 0;
- ldout(cct, 20) << "emptied current pg, moving on to next one:" << list_context->current_pg << dendl;
- if (list_context->current_pg < list_context->starting_pg_num){ // we have more pgs to go through
- list_context->cookie = collection_list_handle_t();
- delete bl;
- list_objects(list_context, final_finish);
+ if (!list_context->list.empty()) {
+ ldout(cct, 20) << " returning results so far" << dendl;
+ final_finish->complete(0);
return;
}
-
- // if we make it this far, there are no more pgs
- ldout(cct, 20) << "out of pgs, returning to" << final_finish << dendl;
- list_context->at_end = true;
- delete bl;
- final_finish->complete(0);
- return;
+
+ // continue!
+ list_objects(list_context, final_finish);
}
+
//snapshots
int Objecter::create_pool_snap(int64_t pool, string& snap_name, Context *onfinish)
@@ -2406,7 +2506,6 @@ Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) :
bool Objecter::RequestStateHook::call(std::string command, cmdmap_t& cmdmap,
std::string format, bufferlist& out)
{
- stringstream ss;
Formatter *f = new_formatter(format);
m_objecter->client_lock.Lock();
m_objecter->dump_requests(f);
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 41973dd..dd0a471 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -32,7 +32,6 @@
#include <memory>
#include <sstream>
using namespace std;
-using namespace __gnu_cxx;
class Context;
class Messenger;
@@ -279,8 +278,16 @@ struct ObjectOperation {
out_handler[p] = h;
out_rval[p] = prval;
}
- void write(uint64_t off, bufferlist& bl) {
+ void write(uint64_t off, bufferlist& bl,
+ uint64_t truncate_size,
+ uint32_t truncate_seq) {
add_data(CEPH_OSD_OP_WRITE, off, bl.length(), bl);
+ OSDOp& o = *ops.rbegin();
+ o.op.extent.truncate_size = truncate_size;
+ o.op.extent.truncate_seq = truncate_seq;
+ }
+ void write(uint64_t off, bufferlist& bl) {
+ write(off, bl, 0, 0);
}
void write_full(bufferlist& bl) {
add_data(CEPH_OSD_OP_WRITEFULL, 0, bl.length(), bl);
@@ -483,6 +490,12 @@ struct ObjectOperation {
void tmap_get() {
add_op(CEPH_OSD_OP_TMAPGET);
}
+ void tmap_to_omap(bool nullok=false) {
+ OSDOp& osd_op = add_op(CEPH_OSD_OP_TMAP2OMAP);
+ osd_op.op.op = CEPH_OSD_OP_TMAP2OMAP;
+ if (nullok)
+ osd_op.op.tmap2omap.flags = CEPH_OSD_TMAP2OMAP_NULLOK;
+ }
// objectmap
void omap_get_keys(const string &start_after,
@@ -571,20 +584,26 @@ struct ObjectOperation {
utime_t *out_mtime;
string *out_category;
std::map<std::string,bufferlist> *out_attrs;
- bufferlist *out_data;
+ bufferlist *out_data, *out_omap_header;
std::map<std::string,bufferlist> *out_omap;
+ vector<snapid_t> *out_snaps;
+ snapid_t *out_snap_seq;
int *prval;
C_ObjectOperation_copyget(object_copy_cursor_t *c,
uint64_t *s,
utime_t *m,
string *cat,
std::map<std::string,bufferlist> *a,
- bufferlist *d,
+ bufferlist *d, bufferlist *oh,
std::map<std::string,bufferlist> *o,
+ std::vector<snapid_t> *osnaps,
+ snapid_t *osnap_seq,
int *r)
: cursor(c),
out_size(s), out_mtime(m), out_category(cat),
- out_attrs(a), out_data(d), out_omap(o), prval(r) {}
+ out_attrs(a), out_data(d), out_omap_header(oh),
+ out_omap(o), out_snaps(osnaps), out_snap_seq(osnap_seq),
+ prval(r) {}
void finish(int r) {
if (r < 0)
return;
@@ -602,8 +621,14 @@ struct ObjectOperation {
*out_attrs = copy_reply.attrs;
if (out_data)
out_data->claim_append(copy_reply.data);
+ if (out_omap_header)
+ out_omap_header->claim_append(copy_reply.omap_header);
if (out_omap)
*out_omap = copy_reply.omap;
+ if (out_snaps)
+ *out_snaps = copy_reply.snaps;
+ if (out_snap_seq)
+ *out_snap_seq = copy_reply.snap_seq;
*cursor = copy_reply.cursor;
} catch (buffer::error& e) {
if (prval)
@@ -619,7 +644,10 @@ struct ObjectOperation {
string *out_category,
std::map<std::string,bufferlist> *out_attrs,
bufferlist *out_data,
+ bufferlist *out_omap_header,
std::map<std::string,bufferlist> *out_omap,
+ vector<snapid_t> *out_snaps,
+ snapid_t *out_snap_seq,
int *prval) {
OSDOp& osd_op = add_op(CEPH_OSD_OP_COPY_GET);
osd_op.op.copy_get.max = max;
@@ -629,7 +657,8 @@ struct ObjectOperation {
out_rval[p] = prval;
C_ObjectOperation_copyget *h =
new C_ObjectOperation_copyget(cursor, out_size, out_mtime, out_category,
- out_attrs, out_data, out_omap, prval);
+ out_attrs, out_data, out_omap_header,
+ out_omap, out_snaps, out_snap_seq, prval);
out_bl[p] = &h->bl;
out_handler[p] = h;
}
@@ -670,6 +699,85 @@ struct ObjectOperation {
out_handler[p] = h;
}
+ struct C_ObjectOperation_hit_set_ls : public Context {
+ bufferlist bl;
+ std::list< std::pair<time_t, time_t> > *ptls;
+ std::list< std::pair<utime_t, utime_t> > *putls;
+ int *prval;
+ C_ObjectOperation_hit_set_ls(std::list< std::pair<time_t, time_t> > *t,
+ std::list< std::pair<utime_t, utime_t> > *ut,
+ int *r)
+ : ptls(t), putls(ut), prval(r) {}
+ void finish(int r) {
+ if (r < 0)
+ return;
+ try {
+ bufferlist::iterator p = bl.begin();
+ std::list< std::pair<utime_t, utime_t> > ls;
+ ::decode(ls, p);
+ if (ptls) {
+ ptls->clear();
+ for (list< pair<utime_t,utime_t> >::iterator p = ls.begin(); p != ls.end(); ++p)
+ // round initial timestamp up to the next full second to keep this a valid interval.
+ ptls->push_back(make_pair(p->first.usec() ? p->first.sec() + 1 : p->first.sec(), p->second.sec()));
+ }
+ if (putls)
+ putls->swap(ls);
+ } catch (buffer::error& e) {
+ r = -EIO;
+ }
+ if (prval)
+ *prval = r;
+ }
+ };
+
+ /**
+ * list available HitSets.
+ *
+ * We will get back a list of time intervals. Note that the most recent range may have
+ * an empty end timestamp if it is still accumulating.
+ *
+ * @param pls [out] list of time intervals
+ * @param prval [out] return value
+ */
+ void hit_set_ls(std::list< std::pair<time_t, time_t> > *pls, int *prval) {
+ add_op(CEPH_OSD_OP_PG_HITSET_LS);
+ unsigned p = ops.size() - 1;
+ out_rval[p] = prval;
+ C_ObjectOperation_hit_set_ls *h =
+ new C_ObjectOperation_hit_set_ls(pls, NULL, prval);
+ out_bl[p] = &h->bl;
+ out_handler[p] = h;
+ }
+ void hit_set_ls(std::list< std::pair<utime_t, utime_t> > *pls, int *prval) {
+ add_op(CEPH_OSD_OP_PG_HITSET_LS);
+ unsigned p = ops.size() - 1;
+ out_rval[p] = prval;
+ C_ObjectOperation_hit_set_ls *h =
+ new C_ObjectOperation_hit_set_ls(NULL, pls, prval);
+ out_bl[p] = &h->bl;
+ out_handler[p] = h;
+ }
+
+ /**
+ * get HitSet
+ *
+ * Return an encoded HitSet that includes the provided time
+ * interval.
+ *
+ * @param stamp [in] timestamp
+ * @param pbl [out] target buffer for encoded HitSet
+ * @param prval [out] return value
+ */
+ void hit_set_get(utime_t stamp, bufferlist *pbl, int *prval) {
+ OSDOp& op = add_op(CEPH_OSD_OP_PG_HITSET_GET);
+ op.op.hit_set_get.stamp.tv_sec = stamp.sec();
+ op.op.hit_set_get.stamp.tv_nsec = stamp.nsec();
+ unsigned p = ops.size() - 1;
+ out_rval[p] = prval;
+ out_bl[p] = pbl;
+ }
+
void omap_get_header(bufferlist *bl, int *prval) {
add_op(CEPH_OSD_OP_OMAPGETHEADER);
unsigned p = ops.size() - 1;
@@ -781,25 +889,75 @@ struct ObjectOperation {
osd_op.op.snap.snapid = snapid;
}
- void copy_from(object_t src, snapid_t snapid, object_locator_t src_oloc, version_t src_version) {
+ void copy_from(object_t src, snapid_t snapid, object_locator_t src_oloc,
+ version_t src_version, unsigned flags) {
OSDOp& osd_op = add_op(CEPH_OSD_OP_COPY_FROM);
osd_op.op.copy_from.snapid = snapid;
osd_op.op.copy_from.src_version = src_version;
+ osd_op.op.copy_from.flags = flags;
::encode(src, osd_op.indata);
::encode(src_oloc, osd_op.indata);
}
+
+ /**
+ * writeback content to backing tier
+ *
+ * If object is marked dirty in the cache tier, write back content
+ * to backing tier. If the object is clean this is a no-op.
+ *
+ * If writeback races with an update, the update will block.
+ *
+ * use with IGNORE_CACHE to avoid triggering promote.
+ */
+ void cache_flush() {
+ add_op(CEPH_OSD_OP_CACHE_FLUSH);
+ }
+
+ /**
+ * writeback content to backing tier
+ *
+ * If object is marked dirty in the cache tier, write back content
+ * to backing tier. If the object is clean this is a no-op.
+ *
+ * If writeback races with an update, return EAGAIN. Requires that
+ * the SKIPRWLOCKS flag be set.
+ *
+ * use with IGNORE_CACHE to avoid triggering promote.
+ */
+ void cache_try_flush() {
+ add_op(CEPH_OSD_OP_CACHE_TRY_FLUSH);
+ }
+
+ /**
+ * evict object from cache tier
+ *
+ * If object is marked clean, remove the object from the cache tier.
+ * Otherwise, return EBUSY.
+ *
+ * use with IGNORE_CACHE to avoid triggering promote.
+ */
+ void cache_evict() {
+ add_op(CEPH_OSD_OP_CACHE_EVICT);
+ }
};
// ----------------
-class Objecter {
- public:
+class Objecter : public md_config_obs_t {
+public:
+ // config observer bits
+ virtual const char** get_tracked_conf_keys() const;
+ virtual void handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed);
+
+public:
Messenger *messenger;
MonClient *monc;
OSDMap *osdmap;
CephContext *cct;
+ std::multimap<string,string> crush_location;
bool initialized;
@@ -812,9 +970,10 @@ class Objecter {
int global_op_flags; // flags which are applied to each IO op
bool keep_balanced_budget;
bool honor_osdmap_full;
- bool honor_cache_redirects;
+public:
void maybe_request_map();
+private:
version_t last_seen_osdmap_version;
version_t last_seen_pgmap_version;
@@ -861,8 +1020,11 @@ public:
object_t target_oid;
object_locator_t target_oloc;
- pg_t pgid;
- vector<int> acting;
+ bool precalc_pgid; ///< true if we are directed at base_pgid, not base_oid
+ pg_t base_pgid; ///< explciti pg target, if any
+
+ pg_t pgid; ///< last pg we mapped to
+ vector<int> acting; ///< acting for last pg we mapped to
bool used_replica;
ConnectionRef con; // for rx buffer only
@@ -892,7 +1054,6 @@ public:
utime_t stamp;
- bool precalc_pgid;
epoch_t map_dne_bound;
bool budgeted;
@@ -904,12 +1065,13 @@ public:
int f, Context *ac, Context *co, version_t *ov) :
session(NULL), session_item(this), incarnation(0),
base_oid(o), base_oloc(ol),
+ precalc_pgid(false),
used_replica(false), con(NULL),
snapid(CEPH_NOSNAP),
outbl(NULL),
flags(f), priority(0), onack(ac), oncommit(co),
tid(0), attempts(0),
- paused(false), objver(ov), reply_epoch(NULL), precalc_pgid(false),
+ paused(false), objver(ov), reply_epoch(NULL),
map_dne_bound(0),
budgeted(false),
should_resend(true) {
@@ -1000,12 +1162,15 @@ public:
collection_list_handle_t cookie;
epoch_t current_pg_epoch;
int starting_pg_num;
- bool at_end;
+ bool at_end_of_pool;
+ bool at_end_of_pg;
int64_t pool_id;
int pool_snap_seq;
int max_entries;
string nspace;
+
+ bufferlist bl; // raw data read to here
std::list<pair<object_t, string> > list;
bufferlist filter;
@@ -1013,21 +1178,30 @@ public:
bufferlist extra_info;
ListContext() : current_pg(0), current_pg_epoch(0), starting_pg_num(0),
- at_end(false), pool_id(0),
+ at_end_of_pool(false),
+ at_end_of_pg(false),
+ pool_id(0),
pool_snap_seq(0), max_entries(0) {}
+
+ bool at_end() const {
+ return at_end_of_pool;
+ }
+
+ uint32_t get_pg_hash_position() const {
+ return current_pg;
+ }
};
struct C_List : public Context {
ListContext *list_context;
Context *final_finish;
- bufferlist *bl;
Objecter *objecter;
epoch_t epoch;
- C_List(ListContext *lc, Context * finish, bufferlist *b, Objecter *ob) :
- list_context(lc), final_finish(finish), bl(b), objecter(ob), epoch(0) {}
+ C_List(ListContext *lc, Context * finish, Objecter *ob) :
+ list_context(lc), final_finish(finish), objecter(ob), epoch(0) {}
void finish(int r) {
if (r >= 0) {
- objecter->_list_reply(list_context, r, bl, final_finish, epoch);
+ objecter->_list_reply(list_context, r, final_finish, epoch);
} else {
final_finish->complete(r);
}
@@ -1225,6 +1399,7 @@ public:
RECALC_OP_TARGET_OSD_DNE,
RECALC_OP_TARGET_OSD_DOWN,
};
+ bool op_should_be_paused(Op *op);
int recalc_op_target(Op *op);
bool recalc_linger_op_target(LingerOp *op);
@@ -1248,7 +1423,7 @@ public:
void reopen_session(OSDSession *session);
void close_session(OSDSession *session);
- void _list_reply(ListContext *list_context, int r, bufferlist *bl, Context *final_finish,
+ void _list_reply(ListContext *list_context, int r, Context *final_finish,
epoch_t reply_epoch);
void resend_mon_ops();
@@ -1288,7 +1463,6 @@ public:
num_unacked(0), num_uncommitted(0),
global_op_flags(0),
keep_balanced_budget(false), honor_osdmap_full(true),
- honor_cache_redirects(true),
last_seen_osdmap_version(0),
last_seen_pgmap_version(0),
client_lock(l), timer(t),
@@ -1322,14 +1496,15 @@ public:
void set_honor_osdmap_full() { honor_osdmap_full = true; }
void unset_honor_osdmap_full() { honor_osdmap_full = false; }
- void set_honor_cache_redirects() { honor_cache_redirects = true; }
- void unset_honor_cache_redirects() { honor_cache_redirects = false; }
-
- void scan_requests(bool skipped_map,
+ void scan_requests(bool force_resend,
+ bool force_resend_writes,
map<tid_t, Op*>& need_resend,
list<LingerOp*>& need_resend_linger,
map<tid_t, CommandOp*>& need_resend_command);
+ int64_t get_object_hash_position(int64_t pool, const string& key, const string& ns);
+ int64_t get_object_pg_hash_position(int64_t pool, const string& key, const string& ns);
+
// messages
public:
void dispatch(Message *m);
@@ -1339,12 +1514,12 @@ public:
private:
// low-level
- tid_t op_submit(Op *op);
tid_t _op_submit(Op *op);
inline void unregister_op(Op *op);
// public interface
- public:
+public:
+ tid_t op_submit(Op *op);
bool is_active() {
return !(ops.empty() && linger_ops.empty() && poolstat_ops.empty() && statfs_ops.empty());
}
@@ -1365,6 +1540,8 @@ private:
void set_client_incarnation(int inc) { client_inc = inc; }
void wait_for_new_map(Context *c, epoch_t epoch, int err=0);
+ void wait_for_latest_osdmap(Context *fin);
+ void _get_latest_version(epoch_t oldest, epoch_t neweset, Context *fin);
/** Get the current set of global op flags */
int get_global_op_flags() { return global_op_flags; }
@@ -1377,8 +1554,9 @@ private:
int op_cancel(tid_t tid);
// commands
- int osd_command(int osd, vector<string>& cmd, bufferlist& inbl, tid_t *ptid,
- bufferlist *poutbl, string *prs, Context *onfinish) {
+ int osd_command(int osd, vector<string>& cmd,
+ const bufferlist& inbl, tid_t *ptid,
+ bufferlist *poutbl, string *prs, Context *onfinish) {
assert(osd >= 0);
CommandOp *c = new CommandOp;
c->cmd = cmd;
@@ -1389,8 +1567,9 @@ private:
c->target_osd = osd;
return _submit_command(c, ptid);
}
- int pg_command(pg_t pgid, vector<string>& cmd, bufferlist& inbl, tid_t *ptid,
- bufferlist *poutbl, string *prs, Context *onfinish) {
+ int pg_command(pg_t pgid, vector<string>& cmd,
+ const bufferlist& inbl, tid_t *ptid,
+ bufferlist *poutbl, string *prs, Context *onfinish) {
CommandOp *c = new CommandOp;
c->cmd = cmd;
c->inbl = inbl;
@@ -1402,7 +1581,7 @@ private:
}
// mid-level helpers
- tid_t mutate(const object_t& oid, const object_locator_t& oloc,
+ Op *prepare_mutate_op(const object_t& oid, const object_locator_t& oloc,
ObjectOperation& op,
const SnapContext& snapc, utime_t mtime, int flags,
Context *onack, Context *oncommit, version_t *objver = NULL) {
@@ -1410,9 +1589,16 @@ private:
o->priority = op.priority;
o->mtime = mtime;
o->snapc = snapc;
+ return o;
+ }
+ tid_t mutate(const object_t& oid, const object_locator_t& oloc,
+ ObjectOperation& op,
+ const SnapContext& snapc, utime_t mtime, int flags,
+ Context *onack, Context *oncommit, version_t *objver = NULL) {
+ Op *o = prepare_mutate_op(oid, oloc, op, snapc, mtime, flags, onack, oncommit, objver);
return op_submit(o);
}
- tid_t read(const object_t& oid, const object_locator_t& oloc,
+ Op *prepare_read_op(const object_t& oid, const object_locator_t& oloc,
ObjectOperation& op,
snapid_t snapid, bufferlist *pbl, int flags,
Context *onack, version_t *objver = NULL) {
@@ -1423,6 +1609,32 @@ private:
o->out_bl.swap(op.out_bl);
o->out_handler.swap(op.out_handler);
o->out_rval.swap(op.out_rval);
+ return o;
+ }
+ tid_t read(const object_t& oid, const object_locator_t& oloc,
+ ObjectOperation& op,
+ snapid_t snapid, bufferlist *pbl, int flags,
+ Context *onack, version_t *objver = NULL) {
+ Op *o = prepare_read_op(oid, oloc, op, snapid, pbl, flags, onack, objver);
+ return op_submit(o);
+ }
+ tid_t pg_read(uint32_t hash, object_locator_t oloc,
+ ObjectOperation& op,
+ bufferlist *pbl, int flags,
+ Context *onack,
+ epoch_t *reply_epoch) {
+ Op *o = new Op(object_t(), oloc,
+ op.ops, flags | global_op_flags | CEPH_OSD_FLAG_READ,
+ onack, NULL, NULL);
+ o->precalc_pgid = true;
+ o->base_pgid = pg_t(hash, oloc.pool);
+ o->priority = op.priority;
+ o->snapid = CEPH_NOSNAP;
+ o->outbl = pbl;
+ o->out_bl.swap(op.out_bl);
+ o->out_handler.swap(op.out_handler);
+ o->out_rval.swap(op.out_rval);
+ o->reply_epoch = reply_epoch;
return op_submit(o);
}
tid_t linger_mutate(const object_t& oid, const object_locator_t& oloc,
@@ -1767,6 +1979,7 @@ private:
}
void list_objects(ListContext *p, Context *onfinish);
+ uint32_t list_objects_seek(ListContext *p, uint32_t pos);
// -------------------------
// pool ops
diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py
index 75e6106..46d971e 100755
--- a/src/pybind/ceph_rest_api.py
+++ b/src/pybind/ceph_rest_api.py
@@ -56,7 +56,7 @@ def find_up_osd(app):
raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump')
osds = [osd['osd'] for osd in osddump['osds'] if osd['up']]
if not osds:
- raise EnvironmentError(errno.ENOENT, 'No up OSDs found')
+ return None
return int(osds[-1])
@@ -139,7 +139,7 @@ def api_setup(app, conf, cluster, clientname, clientid, args):
app.ceph_sigdict = get_command_descriptions(app.ceph_cluster)
osdid = find_up_osd(app)
- if osdid:
+ if osdid is not None:
osd_sigdict = get_command_descriptions(app.ceph_cluster,
target=('osd', int(osdid)))
diff --git a/src/pybind/cephfs.py b/src/pybind/cephfs.py
index 80b7e4b..13a8262 100644
--- a/src/pybind/cephfs.py
+++ b/src/pybind/cephfs.py
@@ -3,6 +3,7 @@ This module is a thin wrapper around libcephfs.
"""
from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \
create_string_buffer, byref, Structure
+from ctypes.util import find_library
import errno
class Error(Exception):
@@ -114,6 +115,21 @@ class cephfs_stat(Structure):
('__unused2', c_long),
('__unused3', c_long) ]
+def load_libcephfs():
+ """
+ Load the libcephfs shared library.
+ """
+ libcephfs_path = find_library('cephfs')
+ if libcephfs_path:
+ return CDLL(libcephfs_path)
+
+ # try harder, find_library() doesn't search LD_LIBRARY_PATH
+ # in addition, it doesn't seem work on centos 6.4 (see e46d2ca067b5)
+ try:
+ return CDLL('libcephfs.so.1')
+ except OSError as e:
+ raise EnvironmentError("Unable to load libcephfs: %s" % e)
+
class LibCephFS(object):
"""libcephfs python wrapper"""
def require_state(self, *args):
@@ -124,7 +140,7 @@ class LibCephFS(object):
"CephFS object in state %s." % (self.state))
def __init__(self, conf=None, conffile=None):
- self.libcephfs = CDLL('libcephfs.so.1')
+ self.libcephfs = load_libcephfs()
self.cluster = c_void_p()
if conffile is not None and not isinstance(conffile, str):
diff --git a/src/pybind/rados.py b/src/pybind/rados.py
index fecf4bb..0989ebb 100644
--- a/src/pybind/rados.py
+++ b/src/pybind/rados.py
@@ -6,6 +6,7 @@ Copyright 2011, Hannu Valtonen <hannu.valtonen at ormod.com>
from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_char, c_int, c_long, \
c_ulong, create_string_buffer, byref, Structure, c_uint64, c_ubyte, \
pointer, CFUNCTYPE
+from ctypes.util import find_library
import ctypes
import errno
import threading
@@ -194,7 +195,17 @@ Rados object in state %s." % (self.state))
def __init__(self, rados_id=None, name=None, clustername=None,
conf_defaults=None, conffile=None, conf=None, flags=0):
- self.librados = CDLL('librados.so.2')
+ librados_path = find_library('rados')
+ if not librados_path:
+ #maybe find_library can not find it correctly on all platforms.
+ try:
+ self.librados = CDLL('librados.so.2')
+ except OSError as e:
+ raise EnvironmentError("Unable to load librados: %s" % e)
+ except:
+ raise Error("Unexpected error")
+ else:
+ self.librados = CDLL(librados_path)
self.cluster = c_void_p()
self.rados_id = rados_id
if rados_id is not None and not isinstance(rados_id, str):
diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py
index 6d2cd9a..97fa9ab 100644
--- a/src/pybind/rbd.py
+++ b/src/pybind/rbd.py
@@ -18,6 +18,7 @@ methods, a :class:`TypeError` will be raised.
from ctypes import CDLL, c_char, c_char_p, c_size_t, c_void_p, c_int, \
create_string_buffer, byref, Structure, c_uint64, c_int64, c_uint8, \
CFUNCTYPE
+from ctypes.util import find_library
import ctypes
import errno
@@ -116,12 +117,27 @@ class rbd_snap_info_t(Structure):
("size", c_uint64),
("name", c_char_p)]
+def load_librbd():
+ """
+ Load the librbd shared library.
+ """
+ librbd_path = find_library('rbd')
+ if librbd_path:
+ return CDLL(librbd_path)
+
+ # try harder, find_library() doesn't search LD_LIBRARY_PATH
+ # in addition, it doesn't seem work on centos 6.4 (see e46d2ca067b5)
+ try:
+ return CDLL('librbd.so.1')
+ except OSError as e:
+ raise EnvironmentError("Unable to load librbd: %s" % e)
+
class RBD(object):
"""
This class wraps librbd CRUD functions.
"""
def __init__(self):
- self.librbd = CDLL('librbd.so.1')
+ self.librbd = load_librbd()
def version(self):
"""
@@ -330,7 +346,7 @@ class Image(object):
:type read_only: bool
"""
self.closed = True
- self.librbd = CDLL('librbd.so.1')
+ self.librbd = load_librbd()
self.image = c_void_p()
self.name = name
if not isinstance(name, str):
diff --git a/src/rbd.cc b/src/rbd.cc
index 41cd243..c6a3dcf 100644
--- a/src/rbd.cc
+++ b/src/rbd.cc
@@ -40,7 +40,7 @@
#include <stdlib.h>
#include <sys/types.h>
#include <time.h>
-#include <tr1/memory>
+#include "include/memory.h"
#include <sys/ioctl.h>
#include "include/rbd_types.h"
@@ -57,6 +57,8 @@
#include <sys/param.h>
#endif
+#include <blkid/blkid.h>
+
#define MAX_SECRET_LEN 1000
#define MAX_POOL_NAME_SIZE 128
@@ -68,7 +70,8 @@ static string dir_info_oid = RBD_INFO;
bool udevadm_settle = true;
bool progress = true;
bool resize_allow_shrink = false;
-bool read_only = false;
+
+map<string, string> map_options; // -o / --options map
#define dout_subsys ceph_subsys_rbd
@@ -152,6 +155,7 @@ void usage()
" --pretty-format make json or xml output more readable\n"
" --no-settle do not wait for udevadm to settle on map/unmap\n"
" --no-progress do not show progress for long-running commands\n"
+" -o, --options <map-options> options to use when mapping an image\n"
" --read-only set device readonly when mapping image\n"
" --allow-shrink allow shrinking of an image when resizing\n";
}
@@ -247,6 +251,8 @@ static int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag,
{
std::vector<string> names;
int r = rbd.list(io_ctx, names);
+ if (r == -ENOENT)
+ r = 0;
if (r < 0)
return r;
@@ -867,6 +873,11 @@ void rbd_bencher_completion(void *vc, void *pc)
librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc;
rbd_bencher *b = static_cast<rbd_bencher *>(pc);
//cout << "complete " << c << std::endl;
+ int ret = c->get_return_value();
+ if (ret != 0) {
+ cout << "write error: " << cpp_strerror(ret) << std::endl;
+ assert(0 == ret);
+ }
b->lock.Lock();
b->in_flight--;
b->cond.Signal();
@@ -904,19 +915,34 @@ static int do_bench_write(librbd::Image& image, uint64_t io_size,
uint64_t size = 0;
image.size(&size);
+ vector<uint64_t> thread_offset;
+ uint64_t i;
+ uint64_t start_pos;
+
+ // disturb all thread's offset, used by seq write
+ for (i = 0; i < io_threads; i++) {
+ start_pos = (rand() % (size / io_size)) * io_size;
+ thread_offset.push_back(start_pos);
+ }
+
printf(" SEC OPS OPS/SEC BYTES/SEC\n");
uint64_t off;
for (off = 0; off < io_bytes; off += io_size) {
b.wait_for(io_threads - 1);
- uint64_t i = 0;
- uint64_t real_off = off;
- if (pattern == "rand") {
- real_off = (rand() % (size / io_size)) * io_size;
- }
- while (i < io_threads &&
- b.start_write(io_threads, real_off, io_size, bl)) {
+ i = 0;
+ while (i < io_threads && off < io_bytes &&
+ b.start_write(io_threads, thread_offset[i], io_size, bl)) {
++i;
++ios;
+ off += io_size;
+
+ if (pattern == "rand") {
+ thread_offset[i] = (rand() % (size / io_size)) * io_size;
+ } else {
+ thread_offset[i] += io_size;
+ if (thread_offset[i] + io_size > size)
+ thread_offset[i] = 0;
+ }
}
utime_t now = ceph_clock_now(NULL);
@@ -1125,8 +1151,10 @@ static int do_export_diff(librbd::Image& image, const char *fromsnapname,
::encode(endsize, bl);
r = bl.write_fd(fd);
- if (r < 0)
+ if (r < 0) {
+ close(fd);
return r;
+ }
}
ExportContext ec(&image, fd, info.size);
@@ -1139,8 +1167,6 @@ static int do_export_diff(librbd::Image& image, const char *fromsnapname,
bufferlist bl;
::encode(tag, bl);
r = bl.write_fd(fd);
- if (r < 0)
- return r;
}
out:
@@ -1642,13 +1668,8 @@ static int do_kernel_add(const char *poolname, const char *imgname,
oss << ",";
}
- if (read_only)
- oss << " ro,";
- else
- oss << " ";
-
const char *user = g_conf->name.get_id().c_str();
- oss << "name=" << user;
+ oss << " name=" << user;
char key_name[strlen(user) + strlen("client.") + 1];
snprintf(key_name, sizeof(key_name), "client.%s", user);
@@ -1684,6 +1705,18 @@ static int do_kernel_add(const char *poolname, const char *imgname,
oss << ",key=" << key_name;
}
+ for (map<string, string>::const_iterator it = map_options.begin();
+ it != map_options.end();
+ ++it) {
+ // for compatibility with < 3.7 kernels, assume that rw is on by
+ // default and omit it even if it was specified by the user
+ // (see ceph.git commit fb0f1986449b)
+ if (it->first == "rw" && it->second == "rw")
+ continue;
+
+ oss << "," << it->second;
+ }
+
oss << " " << poolname << " " << imgname;
if (snapname) {
@@ -1693,7 +1726,17 @@ static int do_kernel_add(const char *poolname, const char *imgname,
// modprobe the rbd module if /sys/bus/rbd doesn't exist
struct stat sb;
if ((stat("/sys/bus/rbd", &sb) < 0) || (!S_ISDIR(sb.st_mode))) {
- r = system("/sbin/modprobe rbd");
+ // turn on single-major device number allocation scheme if the
+ // kernel supports it
+ const char *cmd = "/sbin/modprobe rbd";
+ r = system("/sbin/modinfo -F parm rbd | /bin/grep -q ^single_major:");
+ if (r == 0) {
+ cmd = "/sbin/modprobe rbd single_major=Y";
+ } else if (r < 0) {
+ cerr << "rbd: error executing modinfo as shell command!" << std::endl;
+ }
+
+ r = system(cmd);
if (r) {
if (r < 0)
cerr << "rbd: error executing modprobe as shell command!" << std::endl;
@@ -1703,16 +1746,29 @@ static int do_kernel_add(const char *poolname, const char *imgname,
}
}
- // write to /sys/bus/rbd/add
- int fd = open("/sys/bus/rbd/add", O_WRONLY);
+ // 'add' interface is deprecated, use 'add_single_major' if it's
+ // available
+ //
+ // ('add' and 'add_single_major' interfaces are identical, except
+ // that if rbd kernel module is new enough and is configured to use
+ // single-major scheme, 'add' is disabled in order to prevent old
+ // userspace from doing weird things at unmap time)
+ int fd = open("/sys/bus/rbd/add_single_major", O_WRONLY);
if (fd < 0) {
- r = -errno;
- if (r == -ENOENT) {
- cerr << "rbd: /sys/bus/rbd/add does not exist!" << std::endl
- << "Did you run 'modprobe rbd' or is your rbd module too old?"
- << std::endl;
+ if (errno == ENOENT) {
+ fd = open("/sys/bus/rbd/add", O_WRONLY);
+ if (fd < 0) {
+ r = -errno;
+ if (r == -ENOENT) {
+ cerr << "rbd: /sys/bus/rbd/add does not exist!" << std::endl
+ << "Did you run 'modprobe rbd' or is your rbd module too old?"
+ << std::endl;
+ }
+ return r;
+ }
+ } else {
+ return -errno;
}
- return r;
}
string add = oss.str();
@@ -1771,7 +1827,7 @@ static int do_kernel_showmapped(Formatter *f)
TextTable tbl;
const char *devices_path = "/sys/bus/rbd/devices";
- std::tr1::shared_ptr<DIR> device_dir(opendir(devices_path), do_closedir);
+ ceph::shared_ptr<DIR> device_dir(opendir(devices_path), do_closedir);
if (!device_dir.get()) {
r = -errno;
cerr << "rbd: could not open " << devices_path << ": "
@@ -1859,9 +1915,19 @@ static int do_kernel_showmapped(Formatter *f)
return 0;
}
-static int get_rbd_seq(int major_num, string &seq)
+static int get_rbd_seq(dev_t devno, string &seq)
{
- int r;
+ // convert devno, which might be a partition major:minor pair, into
+ // a whole disk major:minor pair
+ dev_t wholediskno;
+ int r = blkid_devno_to_wholedisk(devno, NULL, 0, &wholediskno);
+ if (r) {
+ cerr << "rbd: could not compute wholediskno: " << r << std::endl;
+ // ignore the error: devno == wholediskno most of the time, and if
+ // it turns out it's not we will fail with -ENOENT later anyway
+ wholediskno = devno;
+ }
+
const char *devices_path = "/sys/bus/rbd/devices";
DIR *device_dir = opendir(devices_path);
if (!device_dir) {
@@ -1881,28 +1947,66 @@ static int get_rbd_seq(int major_num, string &seq)
return r;
}
- char major[32];
+ int match_minor = -1;
do {
+ char fn[strlen(devices_path) + strlen(dent->d_name) + strlen("//major") + 1];
+ char buf[32];
+
if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)
continue;
- char fn[strlen(devices_path) + strlen(dent->d_name) + strlen("//major") + 1];
-
snprintf(fn, sizeof(fn), "%s/%s/major", devices_path, dent->d_name);
- r = read_file(fn, major, sizeof(major));
+ r = read_file(fn, buf, sizeof(buf));
if (r < 0) {
cerr << "rbd: could not read major number from " << fn << ": "
<< cpp_strerror(-r) << std::endl;
continue;
}
+ string err;
+ int cur_major = strict_strtol(buf, 10, &err);
+ if (!err.empty()) {
+ cerr << err << std::endl;
+ cerr << "rbd: could not parse major number read from " << fn << ": "
+ << cpp_strerror(-r) << std::endl;
+ continue;
+ }
+ if (cur_major != (int)major(wholediskno))
+ continue;
- int cur_major = atoi(major);
- if (cur_major == major_num) {
- seq = string(dent->d_name);
- closedir(device_dir);
- return 0;
+ if (match_minor == -1) {
+ // matching minors in addition to majors is not necessary unless
+ // single-major scheme is turned on, but, if the kernel supports
+ // it, do it anyway (blkid stuff above ensures that we always have
+ // the correct minor to match with)
+ struct stat sbuf;
+ snprintf(fn, sizeof(fn), "%s/%s/minor", devices_path, dent->d_name);
+ match_minor = (stat(fn, &sbuf) == 0);
+ }
+
+ if (match_minor == 1) {
+ snprintf(fn, sizeof(fn), "%s/%s/minor", devices_path, dent->d_name);
+ r = read_file(fn, buf, sizeof(buf));
+ if (r < 0) {
+ cerr << "rbd: could not read minor number from " << fn << ": "
+ << cpp_strerror(-r) << std::endl;
+ continue;
+ }
+ int cur_minor = strict_strtol(buf, 10, &err);
+ if (!err.empty()) {
+ cerr << err << std::endl;
+ cerr << "rbd: could not parse minor number read from " << fn << ": "
+ << cpp_strerror(-r) << std::endl;
+ continue;
+ }
+ if (cur_minor != (int)minor(wholediskno))
+ continue;
+ } else {
+ assert(match_minor == 0);
}
+ seq = string(dent->d_name);
+ closedir(device_dir);
+ return 0;
} while ((dent = readdir(device_dir)));
closedir(device_dir);
@@ -1911,16 +2015,14 @@ static int get_rbd_seq(int major_num, string &seq)
static int do_kernel_rm(const char *dev)
{
- struct stat dev_stat;
- int r = stat(dev, &dev_stat);
- if (!S_ISBLK(dev_stat.st_mode)) {
+ struct stat sbuf;
+ if (stat(dev, &sbuf) || !S_ISBLK(sbuf.st_mode)) {
cerr << "rbd: " << dev << " is not a block device" << std::endl;
return -EINVAL;
}
- int major = major(dev_stat.st_rdev);
string seq_num;
- r = get_rbd_seq(major, seq_num);
+ int r = get_rbd_seq(sbuf.st_rdev, seq_num);
if (r == -ENOENT) {
cerr << "rbd: " << dev << " is not an rbd device" << std::endl;
return -EINVAL;
@@ -1940,9 +2042,17 @@ static int do_kernel_rm(const char *dev)
}
}
- int fd = open("/sys/bus/rbd/remove", O_WRONLY);
+ // see comment in do_kernel_add(), same goes for 'remove' vs
+ // 'remove_single_major'
+ int fd = open("/sys/bus/rbd/remove_single_major", O_WRONLY);
if (fd < 0) {
- return -errno;
+ if (errno == ENOENT) {
+ fd = open("/sys/bus/rbd/remove", O_WRONLY);
+ if (fd < 0)
+ return -errno;
+ } else {
+ return -errno;
+ }
}
r = safe_write(fd, seq_num.c_str(), seq_num.size());
@@ -1972,6 +2082,112 @@ static int do_kernel_rm(const char *dev)
return r;
}
+static string map_option_uuid_cb(const char *value_char)
+{
+ uuid_d u;
+ if (!u.parse(value_char))
+ return "";
+
+ ostringstream oss;
+ oss << u;
+ return oss.str();
+}
+
+static string map_option_ip_cb(const char *value_char)
+{
+ entity_addr_t a;
+ const char *endptr;
+ if (!a.parse(value_char, &endptr) ||
+ endptr != value_char + strlen(value_char)) {
+ return "";
+ }
+
+ ostringstream oss;
+ oss << a.addr;
+ return oss.str();
+}
+
+static string map_option_int_cb(const char *value_char)
+{
+ string err;
+ int d = strict_strtol(value_char, 10, &err);
+ if (!err.empty() || d < 0)
+ return "";
+
+ ostringstream oss;
+ oss << d;
+ return oss.str();
+}
+
+static void put_map_option(const string key, string val)
+{
+ map<string, string>::const_iterator it = map_options.find(key);
+ if (it != map_options.end()) {
+ cerr << "rbd: warning: redefining map option " << key << ": '"
+ << it->second << "' -> '" << val << "'" << std::endl;
+ }
+ map_options[key] = val;
+}
+
+static int put_map_option_value(const string opt, const char *value_char,
+ string (*parse_cb)(const char *))
+{
+ if (!value_char || *value_char == '\0') {
+ cerr << "rbd: " << opt << " option requires a value" << std::endl;
+ return 1;
+ }
+
+ string value = parse_cb(value_char);
+ if (value.empty()) {
+ cerr << "rbd: invalid " << opt << " value '" << value_char << "'"
+ << std::endl;
+ return 1;
+ }
+
+ put_map_option(opt, opt + "=" + value);
+ return 0;
+}
+
+static int parse_map_options(char *options)
+{
+ for (char *this_char = strtok(options, ", ");
+ this_char != NULL;
+ this_char = strtok(NULL, ",")) {
+ char *value_char;
+
+ if ((value_char = strchr(this_char, '=')) != NULL)
+ *value_char++ = '\0';
+
+ if (!strcmp(this_char, "fsid")) {
+ if (put_map_option_value("fsid", value_char, map_option_uuid_cb))
+ return 1;
+ } else if (!strcmp(this_char, "ip")) {
+ if (put_map_option_value("ip", value_char, map_option_ip_cb))
+ return 1;
+ } else if (!strcmp(this_char, "share") || !strcmp(this_char, "noshare")) {
+ put_map_option("share", this_char);
+ } else if (!strcmp(this_char, "crc") || !strcmp(this_char, "nocrc")) {
+ put_map_option("crc", this_char);
+ } else if (!strcmp(this_char, "mount_timeout")) {
+ if (put_map_option_value("mount_timeout", value_char, map_option_int_cb))
+ return 1;
+ } else if (!strcmp(this_char, "osdkeepalive")) {
+ if (put_map_option_value("osdkeepalive", value_char, map_option_int_cb))
+ return 1;
+ } else if (!strcmp(this_char, "osd_idle_ttl")) {
+ if (put_map_option_value("osd_idle_ttl", value_char, map_option_int_cb))
+ return 1;
+ } else if (!strcmp(this_char, "rw") || !strcmp(this_char, "ro")) {
+ put_map_option("rw", this_char);
+ } else {
+ cerr << "rbd: unknown map option '" << this_char << "'" << std::endl;
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
enum {
OPT_NO_CMD = 0,
OPT_LIST,
@@ -2207,8 +2423,15 @@ int main(int argc, const char **argv)
lock_tag = strdup(val.c_str());
} else if (ceph_argparse_flag(args, i, "--no-settle", (char *)NULL)) {
udevadm_settle = false;
+ } else if (ceph_argparse_witharg(args, i, &val, "-o", "--options", (char*)NULL)) {
+ char *map_options = strdup(val.c_str());
+ if (parse_map_options(map_options)) {
+ cerr << "rbd: couldn't parse map options" << std::endl;
+ return EXIT_FAILURE;
+ }
} else if (ceph_argparse_flag(args, i, "--read-only", (char *)NULL)) {
- read_only = true;
+ // --read-only is equivalent to -o ro
+ put_map_option("rw", "ro");
} else if (ceph_argparse_flag(args, i, "--no-progress", (char *)NULL)) {
progress = false;
} else if (ceph_argparse_flag(args, i , "--allow-shrink", (char *)NULL)) {
@@ -2559,14 +2782,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \
case OPT_LIST:
r = do_list(rbd, io_ctx, lflag, formatter.get());
if (r < 0) {
- switch (r) {
- case -ENOENT:
- cerr << "rbd: pool " << poolname << " doesn't contain rbd images"
- << std::endl;
- break;
- default:
- cerr << "rbd: list: " << cpp_strerror(-r) << std::endl;
- }
+ cerr << "rbd: list: " << cpp_strerror(-r) << std::endl;
return -r;
}
break;
diff --git a/src/rgw/Makefile.am b/src/rgw/Makefile.am
index b92c35e..b93af6d 100644
--- a/src/rgw/Makefile.am
+++ b/src/rgw/Makefile.am
@@ -41,6 +41,7 @@ LIBRGW_DEPS += \
libcls_rgw_client.la \
libcls_log_client.a \
libcls_statelog_client.a \
+ libcls_user_client.a \
libcls_replica_log_client.a \
libcls_lock_client.la \
libcls_refcount_client.la \
@@ -48,7 +49,8 @@ LIBRGW_DEPS += \
-lcurl \
-lexpat \
-lm \
- -lfcgi
+ -lfcgi \
+ -ldl
radosgw_SOURCES = \
rgw/rgw_resolve.cc \
@@ -67,8 +69,12 @@ radosgw_SOURCES = \
rgw/rgw_http_client.cc \
rgw/rgw_swift.cc \
rgw/rgw_swift_auth.cc \
+ rgw/rgw_loadgen.cc \
+ rgw/rgw_civetweb.cc \
+ civetweb/src/civetweb.c \
rgw/rgw_main.cc
-radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) -lresolv $(CEPH_GLOBAL)
+radosgw_CFLAGS = -Icivetweb/include
+radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(RESOLV_LIBS) $(CEPH_GLOBAL)
bin_PROGRAMS += radosgw
radosgw_admin_SOURCES = rgw/rgw_admin.cc
@@ -116,6 +122,7 @@ noinst_HEADERS += \
rgw/rgw_formats.h \
rgw/rgw_http_errors.h \
rgw/rgw_log.h \
+ rgw/rgw_loadgen.h \
rgw/rgw_multi.h \
rgw/rgw_policy_s3.h \
rgw/rgw_gc.h \
@@ -148,5 +155,9 @@ noinst_HEADERS += \
rgw/rgw_usage.h \
rgw/rgw_user.h \
rgw/rgw_bucket.h \
- rgw/rgw_keystone.h
+ rgw/rgw_keystone.h \
+ rgw/rgw_civetweb.h \
+ civetweb/civetweb.h \
+ civetweb/include/civetweb.h \
+ civetweb/src/md5.h
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index cb6c302..26da02e 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -47,6 +47,7 @@ void _usage()
cerr << " user suspend suspend a user\n";
cerr << " user enable reenable user after suspension\n";
cerr << " user check check user info\n";
+ cerr << " user stats show user stats as accounted by quota subsystem\n";
cerr << " caps add add user capabilities\n";
cerr << " caps rm remove user capabilities\n";
cerr << " subuser create create a new subuser\n" ;
@@ -114,6 +115,7 @@ void _usage()
cerr << " --gen-access-key generate random access key (for S3)\n";
cerr << " --gen-secret generate random secret key\n";
cerr << " --key-type=<type> key type, options are: swift, s3\n";
+ cerr << " --temp-url-key[-2]=<key> temp url key\n";
cerr << " --access=<access> Set access permissions for sub-user, should be one\n";
cerr << " of read, write, readwrite, full\n";
cerr << " --display-name=<name>\n";
@@ -144,6 +146,8 @@ void _usage()
cerr << " subuser keys\n";
cerr << " --purge-objects remove a bucket's objects before deleting it\n";
cerr << " (NOTE: required to delete a non-empty bucket)\n";
+ cerr << " --sync-stats option to 'user stats', update user stats with current\n";
+ cerr << " stats reported by user's buckets indexes\n";
cerr << " --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
cerr << " --show-log-sum=<flag> enable/disable dump of log summation on log show\n";
cerr << " --skip-zero-entries log show only dumps entries that don't have zero value\n";
@@ -187,6 +191,7 @@ enum {
OPT_USER_SUSPEND,
OPT_USER_ENABLE,
OPT_USER_CHECK,
+ OPT_USER_STATS,
OPT_SUBUSER_CREATE,
OPT_SUBUSER_MODIFY,
OPT_SUBUSER_RM,
@@ -300,6 +305,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
return OPT_USER_ENABLE;
if (strcmp(cmd, "check") == 0)
return OPT_USER_CHECK;
+ if (strcmp(cmd, "stats") == 0)
+ return OPT_USER_STATS;
} else if (strcmp(prev_cmd, "subuser") == 0) {
if (strcmp(cmd, "create") == 0)
return OPT_SUBUSER_CREATE;
@@ -475,13 +482,13 @@ static void show_user_info(RGWUserInfo& info, Formatter *formatter)
cout << std::endl;
}
-static void dump_bucket_usage(map<RGWObjCategory, RGWBucketStats>& stats, Formatter *formatter)
+static void dump_bucket_usage(map<RGWObjCategory, RGWStorageStats>& stats, Formatter *formatter)
{
- map<RGWObjCategory, RGWBucketStats>::iterator iter;
+ map<RGWObjCategory, RGWStorageStats>::iterator iter;
formatter->open_object_section("usage");
for (iter = stats.begin(); iter != stats.end(); ++iter) {
- RGWBucketStats& s = iter->second;
+ RGWStorageStats& s = iter->second;
const char *cat_name = rgw_obj_category_name(iter->first);
formatter->open_object_section(cat_name);
formatter->dump_int("size_kb", s.num_kb);
@@ -501,7 +508,7 @@ int bucket_stats(rgw_bucket& bucket, Formatter *formatter)
if (r < 0)
return r;
- map<RGWObjCategory, RGWBucketStats> stats;
+ map<RGWObjCategory, RGWStorageStats> stats;
uint64_t bucket_ver, master_ver;
string max_marker;
int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, &max_marker);
@@ -536,10 +543,9 @@ public:
}
};
-static int init_bucket(string& bucket_name, rgw_bucket& bucket)
+static int init_bucket(string& bucket_name, RGWBucketInfo& bucket_info, rgw_bucket& bucket)
{
if (!bucket_name.empty()) {
- RGWBucketInfo bucket_info;
int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
if (r < 0) {
cerr << "could not get bucket info for bucket=" << bucket_name << std::endl;
@@ -720,6 +726,24 @@ int set_user_bucket_quota(int opt_cmd, RGWUser& user, RGWUserAdminOpState& op_st
return 0;
}
+int set_user_quota(int opt_cmd, RGWUser& user, RGWUserAdminOpState& op_state, int64_t max_size, int64_t max_objects,
+ bool have_max_size, bool have_max_objects)
+{
+ RGWUserInfo& user_info = op_state.get_user_info();
+
+ set_quota_info(user_info.user_quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
+
+ op_state.set_user_quota(user_info.user_quota);
+
+ string err;
+ int r = user.modify(op_state, &err);
+ if (r < 0) {
+ cerr << "ERROR: failed updating user info: " << cpp_strerror(-r) << ": " << err << std::endl;
+ return -r;
+ }
+ return 0;
+}
+
int main(int argc, char **argv)
{
vector<const char*> args;
@@ -743,6 +767,8 @@ int main(int argc, char **argv)
int gen_access_key = 0;
int gen_secret_key = 0;
bool set_perm = false;
+ bool set_temp_url_key = false;
+ map<int, string> temp_url_keys;
string bucket_id;
Formatter *formatter = NULL;
int purge_data = false;
@@ -787,6 +813,8 @@ int main(int argc, char **argv)
bool have_max_objects = false;
bool have_max_size = false;
+ int sync_stats = false;
+
std::string val;
std::ostringstream errs;
string err;
@@ -886,6 +914,12 @@ int main(int argc, char **argv)
access = val;
perm_mask = rgw_str_to_perm(access.c_str());
set_perm = true;
+ } else if (ceph_argparse_witharg(args, i, &val, "--temp-url-key", (char*)NULL)) {
+ temp_url_keys[0] = val;
+ set_temp_url_key = true;
+ } else if (ceph_argparse_witharg(args, i, &val, "--temp-url-key2", "--temp-url-key-2", (char*)NULL)) {
+ temp_url_keys[1] = val;
+ set_temp_url_key = true;
} else if (ceph_argparse_witharg(args, i, &val, "--bucket-id", (char*)NULL)) {
bucket_id = val;
if (bucket_id.empty()) {
@@ -916,6 +950,8 @@ int main(int argc, char **argv)
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &check_objects, NULL, "--check-objects", (char*)NULL)) {
// do nothing
+ } else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) {
+ // do nothing
} else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
caps = val;
} else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) {
@@ -1010,7 +1046,7 @@ int main(int argc, char **argv)
if (raw_storage_op) {
store = RGWStoreManager::get_raw_storage(g_ceph_context);
} else {
- store = RGWStoreManager::get_storage(g_ceph_context, false);
+ store = RGWStoreManager::get_storage(g_ceph_context, false, false);
}
if (!store) {
cerr << "couldn't init storage provider" << std::endl;
@@ -1263,6 +1299,13 @@ int main(int argc, char **argv)
if (set_perm)
user_op.set_perm(perm_mask);
+ if (set_temp_url_key) {
+ map<int, string>::iterator iter = temp_url_keys.begin();
+ for (; iter != temp_url_keys.end(); ++iter) {
+ user_op.set_temp_url_key(iter->second, iter->first);
+ }
+ }
+
if (!op_mask_str.empty()) {
uint32_t op_mask;
int ret = rgw_parse_op_type_list(op_mask_str, &op_mask);
@@ -1425,7 +1468,8 @@ int main(int argc, char **argv)
if (bucket_name.empty()) {
RGWBucketAdminOp::info(store, bucket_op, f);
} else {
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -1749,12 +1793,13 @@ next:
}
if (opt_cmd == OPT_OBJECT_RM) {
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
}
- ret = rgw_remove_object(store, bucket, object);
+ ret = rgw_remove_object(store, bucket_info.owner, bucket, object);
if (ret < 0) {
cerr << "ERROR: object remove returned: " << cpp_strerror(-ret) << std::endl;
@@ -1763,7 +1808,8 @@ next:
}
if (opt_cmd == OPT_OBJECT_UNLINK) {
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -1778,7 +1824,8 @@ next:
}
if (opt_cmd == OPT_OBJECT_STAT) {
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -1885,6 +1932,38 @@ next:
check_bad_user_bucket_mapping(store, user_id, fix);
}
+ if (opt_cmd == OPT_USER_STATS) {
+ if (sync_stats) {
+ if (!bucket_name.empty()) {
+ int ret = rgw_bucket_sync_user_stats(store, bucket_name);
+ if (ret < 0) {
+ cerr << "ERROR: could not sync bucket stats: " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+ } else {
+ int ret = rgw_user_sync_all_stats(store, user_id);
+ if (ret < 0) {
+ cerr << "ERROR: failed to sync user stats: " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+ }
+ }
+
+ if (user_id.empty()) {
+ cerr << "ERROR: uid not specified" << std::endl;
+ return EINVAL;
+ }
+ cls_user_header header;
+ int ret = store->cls_user_get_header(user_id, &header);
+ if (ret < 0) {
+ cerr << "ERROR: can't read user header: " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+
+ encode_json("header", header, formatter);
+ formatter->flush(cout);
+ }
+
if (opt_cmd == OPT_METADATA_GET) {
int ret = store->meta_mgr->get(metadata_key, formatter);
if (ret < 0) {
@@ -2029,7 +2108,8 @@ next:
cerr << "ERROR: bucket not specified" << std::endl;
return -EINVAL;
}
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -2068,7 +2148,8 @@ next:
cerr << "ERROR: bucket not specified" << std::endl;
return -EINVAL;
}
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -2248,7 +2329,8 @@ next:
cerr << "ERROR: bucket not specified" << std::endl;
return -EINVAL;
}
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -2298,7 +2380,8 @@ next:
cerr << "ERROR: bucket not specified" << std::endl;
return -EINVAL;
}
- int ret = init_bucket(bucket_name, bucket);
+ RGWBucketInfo bucket_info;
+ int ret = init_bucket(bucket_name, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
@@ -2326,12 +2409,16 @@ next:
}
set_bucket_quota(store, opt_cmd, bucket_name, max_size, max_objects, have_max_size, have_max_objects);
} else if (!user_id.empty()) {
- if (quota_scope != "bucket") {
- cerr << "ERROR: only bucket-level user quota can be handled. Please specify --quota-scope=bucket" << std::endl;
+ if (quota_scope == "bucket") {
+ set_user_bucket_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
+ } else if (quota_scope == "user") {
+ set_user_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
+ } else {
+ cerr << "ERROR: invalid quota scope specification. Please specify either --quota-scope=bucket, or --quota-scope=user" << std::endl;
return EINVAL;
}
- set_user_bucket_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
}
}
+
return 0;
}
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 31afebe..9827bad 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -17,6 +17,8 @@
// until everything is moved from rgw_common
#include "rgw_common.h"
+#include "cls/user/cls_user_types.h"
+
#define dout_subsys ceph_subsys_rgw
#define BUCKET_TAG_TIMEOUT 30
@@ -27,7 +29,7 @@ static RGWMetadataHandler *bucket_meta_handler = NULL;
static RGWMetadataHandler *bucket_instance_meta_handler = NULL;
// define as static when RGWBucket implementation compete
-void rgw_get_buckets_obj(string& user_id, string& buckets_obj_id)
+void rgw_get_buckets_obj(const string& user_id, string& buckets_obj_id)
{
buckets_obj_id = user_id;
buckets_obj_id += RGW_BUCKETS_OBJ_SUFFIX;
@@ -47,21 +49,28 @@ int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& bucke
bufferlist bl;
rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
bufferlist header;
- map<string,bufferlist> m;
+ list<cls_user_bucket_entry> entries;
- ret = store->omap_get_vals(obj, header, marker, max, m);
- if (ret == -ENOENT)
- ret = 0;
+ bool truncated = false;
+ string m = marker;
- if (ret < 0)
- return ret;
+ uint64_t total = 0;
- for (map<string,bufferlist>::iterator q = m.begin(); q != m.end(); ++q) {
- bufferlist::iterator iter = q->second.begin();
- RGWBucketEnt bucket;
- ::decode(bucket, iter);
- buckets.add(bucket);
- }
+ do {
+ ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated);
+ if (ret == -ENOENT)
+ ret = 0;
+
+ if (ret < 0)
+ return ret;
+
+ for (list<cls_user_bucket_entry>::iterator q = entries.begin(); q != entries.end(); ++q) {
+ RGWBucketEnt e(*q);
+ buckets.add(e);
+ total++;
+ }
+
+ } while (truncated && total < max);
if (need_stats) {
map<string, RGWBucketEnt>& m = buckets.get_buckets();
@@ -73,25 +82,49 @@ int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& bucke
return 0;
}
+int rgw_bucket_sync_user_stats(RGWRados *store, const string& user_id, rgw_bucket& bucket)
+{
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
+ rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+
+ return store->cls_user_sync_bucket_stats(obj, bucket);
+}
+
+int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name)
+{
+ RGWBucketInfo bucket_info;
+ int ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: could not fetch bucket info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ ret = rgw_bucket_sync_user_stats(store, bucket_info.owner, bucket_info.bucket);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: could not sync user stats for bucket " << bucket_name << ": ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint)
{
int ret;
string& bucket_name = bucket.name;
- bufferlist bl;
-
- RGWBucketEnt new_bucket;
+ cls_user_bucket_entry new_bucket;
RGWBucketEntryPoint ep;
RGWObjVersionTracker ot;
- new_bucket.bucket = bucket;
+ bucket.convert(&new_bucket.bucket);
new_bucket.size = 0;
if (!creation_time)
time(&new_bucket.creation_time);
else
new_bucket.creation_time = creation_time;
- ::encode(new_bucket, bl);
map<string, bufferlist> attrs;
@@ -109,7 +142,7 @@ int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t
rgw_get_buckets_obj(user_id, buckets_obj_id);
rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
- ret = store->omap_set(obj, bucket_name, bl);
+ ret = store->cls_user_add_bucket(obj, new_bucket);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: error adding bucket to directory: "
<< cpp_strerror(-ret)<< dendl;
@@ -143,8 +176,10 @@ int rgw_unlink_bucket(RGWRados *store, string user_id, const string& bucket_name
string buckets_obj_id;
rgw_get_buckets_obj(user_id, buckets_obj_id);
+ cls_user_bucket bucket;
+ bucket.name = bucket_name;
rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
- ret = store->omap_del(obj, bucket_name);
+ ret = store->cls_user_remove_bucket(obj, bucket);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: error removing bucket from directory: "
<< cpp_strerror(-ret)<< dendl;
@@ -194,11 +229,21 @@ int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersi
return store->meta_mgr->remove_entry(bucket_instance_meta_handler, entry, objv_tracker);
}
-int rgw_bucket_set_attrs(RGWRados *store, rgw_bucket& bucket,
+int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
map<string, bufferlist>& attrs,
map<string, bufferlist>* rmattrs,
RGWObjVersionTracker *objv_tracker)
{
+ rgw_bucket& bucket = bucket_info.bucket;
+
+ if (!bucket_info.has_instance_obj) {
+ /* an old bucket object, need to convert it */
+ int ret = store->convert_old_bucket_info(NULL, bucket.name);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: failed converting old bucket info: " << ret << dendl;
+ return ret;
+ }
+ }
string oid;
store->get_bucket_meta_oid(bucket, oid);
rgw_obj obj(store->zone.domain_root, oid);
@@ -287,21 +332,21 @@ static bool bucket_object_check_filter(const string& name)
return rgw_obj::translate_raw_obj_to_obj_in_ns(obj, ns);
}
-int rgw_remove_object(RGWRados *store, rgw_bucket& bucket, std::string& object)
+int rgw_remove_object(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, std::string& object)
{
RGWRadosCtx rctx(store);
- rgw_obj obj(bucket,object);
+ rgw_obj obj(bucket, object);
- int ret = store->delete_obj((void *)&rctx, obj);
+ int ret = store->delete_obj((void *)&rctx, bucket_owner, obj);
return ret;
}
-int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
+int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, bool delete_children)
{
int ret;
- map<RGWObjCategory, RGWBucketStats> stats;
+ map<RGWObjCategory, RGWStorageStats> stats;
std::vector<RGWObjEnt> objs;
std::string prefix, delim, marker, ns;
map<string, bool> common_prefixes;
@@ -333,7 +378,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
while (!objs.empty()) {
std::vector<RGWObjEnt>::iterator it = objs.begin();
for (it = objs.begin(); it != objs.end(); ++it) {
- ret = rgw_remove_object(store, bucket, (*it).name);
+ ret = rgw_remove_object(store, bucket_owner, bucket, (*it).name);
if (ret < 0)
return ret;
}
@@ -380,8 +425,6 @@ int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
store = storage;
- RGWBucketInfo bucket_info;
-
string user_id = op_state.get_user_id();
bucket_name = op_state.get_bucket_name();
RGWUserBuckets user_buckets;
@@ -505,7 +548,7 @@ int RGWBucket::remove(RGWBucketAdminOpState& op_state, std::string *err_msg)
bool delete_children = op_state.will_delete_children();
rgw_bucket bucket = op_state.get_bucket();
- int ret = rgw_remove_bucket(store, bucket, delete_children);
+ int ret = rgw_remove_bucket(store, bucket_info.owner, bucket, delete_children);
if (ret < 0) {
set_err_msg(err_msg, "unable to remove bucket" + cpp_strerror(-ret));
return ret;
@@ -519,7 +562,7 @@ int RGWBucket::remove_object(RGWBucketAdminOpState& op_state, std::string *err_m
rgw_bucket bucket = op_state.get_bucket();
std::string object_name = op_state.get_object_name();
- int ret = rgw_remove_object(store, bucket, object_name);
+ int ret = rgw_remove_object(store, bucket_info.owner, bucket, object_name);
if (ret < 0) {
set_err_msg(err_msg, "unable to remove object" + cpp_strerror(-ret));
return ret;
@@ -536,13 +579,13 @@ static void dump_bucket_index(map<string, RGWObjEnt> result, Formatter *f)
}
}
-static void dump_bucket_usage(map<RGWObjCategory, RGWBucketStats>& stats, Formatter *formatter)
+static void dump_bucket_usage(map<RGWObjCategory, RGWStorageStats>& stats, Formatter *formatter)
{
- map<RGWObjCategory, RGWBucketStats>::iterator iter;
+ map<RGWObjCategory, RGWStorageStats>::iterator iter;
formatter->open_object_section("usage");
for (iter = stats.begin(); iter != stats.end(); ++iter) {
- RGWBucketStats& s = iter->second;
+ RGWStorageStats& s = iter->second;
const char *cat_name = rgw_obj_category_name(iter->first);
formatter->open_object_section(cat_name);
formatter->dump_int("size_kb", s.num_kb);
@@ -553,8 +596,8 @@ static void dump_bucket_usage(map<RGWObjCategory, RGWBucketStats>& stats, Format
formatter->close_section();
}
-static void dump_index_check(map<RGWObjCategory, RGWBucketStats> existing_stats,
- map<RGWObjCategory, RGWBucketStats> calculated_stats,
+static void dump_index_check(map<RGWObjCategory, RGWStorageStats> existing_stats,
+ map<RGWObjCategory, RGWStorageStats> calculated_stats,
Formatter *formatter)
{
formatter->open_object_section("check_result");
@@ -693,8 +736,8 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
int RGWBucket::check_index(RGWBucketAdminOpState& op_state,
- map<RGWObjCategory, RGWBucketStats>& existing_stats,
- map<RGWObjCategory, RGWBucketStats>& calculated_stats,
+ map<RGWObjCategory, RGWStorageStats>& existing_stats,
+ map<RGWObjCategory, RGWStorageStats>& calculated_stats,
std::string *err_msg)
{
rgw_bucket bucket = op_state.get_bucket();
@@ -717,17 +760,9 @@ int RGWBucket::check_index(RGWBucketAdminOpState& op_state,
return 0;
}
-int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, ostream& o)
-{
- std::string object_name = op_state.get_object_name();
- rgw_bucket bucket = op_state.get_bucket();
-
- bufferlist bl;
- rgw_obj obj(bucket, object_name);
- int ret = store->get_attr(NULL, obj, RGW_ATTR_ACL, bl);
- if (ret < 0)
- return ret;
+int RGWBucket::policy_bl_to_stream(bufferlist& bl, ostream& o)
+{
RGWAccessControlPolicy_S3 policy(g_ceph_context);
bufferlist::iterator iter = bl.begin();
try {
@@ -737,10 +772,40 @@ int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, ostream& o)
return -EIO;
}
policy.to_xml(o);
-
return 0;
}
+int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, ostream& o)
+{
+ std::string object_name = op_state.get_object_name();
+ rgw_bucket bucket = op_state.get_bucket();
+
+ if (!object_name.empty()) {
+ bufferlist bl;
+ rgw_obj obj(bucket, object_name);
+ int ret = store->get_attr(NULL, obj, RGW_ATTR_ACL, bl);
+ if (ret < 0)
+ return ret;
+
+ return policy_bl_to_stream(bl, o);
+ }
+
+
+ RGWBucketInfo bucket_info;
+ map<string, bufferlist> attrs;
+ int ret = store->get_bucket_info(NULL, bucket.name, bucket_info, NULL, &attrs);
+ if (ret < 0) {
+ return ret;
+ }
+
+ map<string, bufferlist>::iterator aiter = attrs.find(RGW_ATTR_ACL);
+ if (aiter == attrs.end()) {
+ return -ENOENT;
+ }
+
+ return policy_bl_to_stream(aiter->second, o);
+}
+
int RGWBucketAdminOp::get_policy(RGWRados *store, RGWBucketAdminOpState& op_state,
ostream& os)
@@ -809,8 +874,8 @@ int RGWBucketAdminOp::check_index(RGWRados *store, RGWBucketAdminOpState& op_sta
{
int ret;
map<string, RGWObjEnt> result;
- map<RGWObjCategory, RGWBucketStats> existing_stats;
- map<RGWObjCategory, RGWBucketStats> calculated_stats;
+ map<RGWObjCategory, RGWStorageStats> existing_stats;
+ map<RGWObjCategory, RGWStorageStats> calculated_stats;
list<std::string> objs_to_unlink;
RGWBucket bucket;
@@ -872,7 +937,7 @@ static int bucket_stats(RGWRados *store, std::string& bucket_name, Formatter *f
{
RGWBucketInfo bucket_info;
rgw_bucket bucket;
- map<RGWObjCategory, RGWBucketStats> stats;
+ map<RGWObjCategory, RGWStorageStats> stats;
time_t mtime;
int r = store->get_bucket_info(NULL, bucket_name, bucket_info, &mtime);
diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 4779540..f113590 100644
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -20,7 +20,7 @@
using namespace std;
// define as static when RGWBucket implementation compete
-extern void rgw_get_buckets_obj(string& user_id, string& buckets_obj_id);
+extern void rgw_get_buckets_obj(const string& user_id, string& buckets_obj_id);
extern int rgw_bucket_store_info(RGWRados *store, const string& bucket_name, bufferlist& bl, bool exclusive,
map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker,
@@ -33,6 +33,9 @@ extern int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWO
extern int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker);
+extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& user_id, rgw_bucket& bucket);
+extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name);
+
/**
* Store a list of the user's buckets, with associated functinos.
*/
@@ -102,10 +105,10 @@ extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets
extern int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true);
extern int rgw_unlink_bucket(RGWRados *store, string user_id, const string& bucket_name, bool update_entrypoint = true);
-extern int rgw_remove_object(RGWRados *store, rgw_bucket& bucket, std::string& object);
-extern int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children);
+extern int rgw_remove_object(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, std::string& object);
+extern int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, bool delete_children);
-extern int rgw_bucket_set_attrs(RGWRados *store, rgw_bucket& obj,
+extern int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
map<string, bufferlist>& attrs,
map<string, bufferlist>* rmattrs,
RGWObjVersionTracker *objv_tracker);
@@ -183,6 +186,8 @@ class RGWBucket
bool failure;
+ RGWBucketInfo bucket_info;
+
private:
public:
@@ -196,8 +201,8 @@ public:
map<string, RGWObjEnt> result, std::string *err_msg = NULL);
int check_index(RGWBucketAdminOpState& op_state,
- map<RGWObjCategory, RGWBucketStats>& existing_stats,
- map<RGWObjCategory, RGWBucketStats>& calculated_stats,
+ map<RGWObjCategory, RGWStorageStats>& existing_stats,
+ map<RGWObjCategory, RGWStorageStats>& calculated_stats,
std::string *err_msg = NULL);
int remove(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
@@ -205,6 +210,7 @@ public:
int unlink(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
int remove_object(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
+ int policy_bl_to_stream(bufferlist& bl, ostream& o);
int get_policy(RGWBucketAdminOpState& op_state, ostream& o);
void clear_failure() { failure = false; };
@@ -298,7 +304,7 @@ class RGWDataChangesLog {
}
};
- typedef std::tr1::shared_ptr<ChangeStatus> ChangeStatusPtr;
+ typedef ceph::shared_ptr<ChangeStatus> ChangeStatusPtr;
lru_map<string, ChangeStatusPtr> changes;
diff --git a/src/rgw/rgw_cache.cc b/src/rgw/rgw_cache.cc
index d0afdcd..94b3d04 100644
--- a/src/rgw/rgw_cache.cc
+++ b/src/rgw/rgw_cache.cc
@@ -8,7 +8,7 @@ using namespace std;
int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
{
- Mutex::Locker l(lock);
+ RWLock::RLocker l(lock);
map<string, ObjectCacheEntry>::iterator iter = cache_map.find(name);
if (iter == cache_map.end()) {
@@ -17,7 +17,18 @@ int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
return -ENOENT;
}
- touch_lru(name, iter->second.lru_iter);
+ ObjectCacheEntry& entry = iter->second;
+
+ if (lru_counter - entry.lru_promotion_ts > lru_window) {
+ ldout(cct, 20) << "cache get: touching lru, lru_counter=" << lru_counter << " promotion_ts=" << entry.lru_promotion_ts << dendl;
+ lock.unlock();
+ lock.get_write(); /* promote lock to writer */
+
+ /* check again, we might have lost a race here */
+ if (lru_counter - entry.lru_promotion_ts > lru_window) {
+ touch_lru(name, entry, iter->second.lru_iter);
+ }
+ }
ObjectCacheInfo& src = iter->second.info;
if ((src.flags & mask) != mask) {
@@ -35,7 +46,7 @@ int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
void ObjectCache::put(string& name, ObjectCacheInfo& info)
{
- Mutex::Locker l(lock);
+ RWLock::WLocker l(lock);
ldout(cct, 10) << "cache put: name=" << name << dendl;
map<string, ObjectCacheEntry>::iterator iter = cache_map.find(name);
@@ -48,7 +59,7 @@ void ObjectCache::put(string& name, ObjectCacheInfo& info)
ObjectCacheEntry& entry = iter->second;
ObjectCacheInfo& target = entry.info;
- touch_lru(name, entry.lru_iter);
+ touch_lru(name, entry, entry.lru_iter);
target.status = info.status;
@@ -93,7 +104,7 @@ void ObjectCache::put(string& name, ObjectCacheInfo& info)
void ObjectCache::remove(string& name)
{
- Mutex::Locker l(lock);
+ RWLock::WLocker l(lock);
map<string, ObjectCacheEntry>::iterator iter = cache_map.find(name);
if (iter == cache_map.end())
@@ -105,7 +116,7 @@ void ObjectCache::remove(string& name)
cache_map.erase(iter);
}
-void ObjectCache::touch_lru(string& name, std::list<string>::iterator& lru_iter)
+void ObjectCache::touch_lru(string& name, ObjectCacheEntry& entry, std::list<string>::iterator& lru_iter)
{
while (lru_size > (size_t)cct->_conf->rgw_cache_lru_size) {
list<string>::iterator iter = lru.begin();
@@ -136,6 +147,9 @@ void ObjectCache::touch_lru(string& name, std::list<string>::iterator& lru_iter)
lru_iter = lru.end();
--lru_iter;
}
+
+ lru_counter++;
+ entry.lru_promotion_ts = lru_counter;
}
void ObjectCache::remove_lru(string& name, std::list<string>::iterator& lru_iter)
diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h
index 68720d0..3b793f1 100644
--- a/src/rgw/rgw_cache.h
+++ b/src/rgw/rgw_cache.h
@@ -7,6 +7,7 @@
#include "include/types.h"
#include "include/utime.h"
#include "include/assert.h"
+#include "common/RWLock.h"
enum {
UPDATE_OBJ,
@@ -126,23 +127,31 @@ WRITE_CLASS_ENCODER(RGWCacheNotifyInfo)
struct ObjectCacheEntry {
ObjectCacheInfo info;
std::list<string>::iterator lru_iter;
+ uint64_t lru_promotion_ts;
+
+ ObjectCacheEntry() : lru_promotion_ts(0) {}
};
class ObjectCache {
std::map<string, ObjectCacheEntry> cache_map;
std::list<string> lru;
unsigned long lru_size;
- Mutex lock;
+ unsigned long lru_counter;
+ unsigned long lru_window;
+ RWLock lock;
CephContext *cct;
- void touch_lru(string& name, std::list<string>::iterator& lru_iter);
+ void touch_lru(string& name, ObjectCacheEntry& entry, std::list<string>::iterator& lru_iter);
void remove_lru(string& name, std::list<string>::iterator& lru_iter);
public:
- ObjectCache() : lru_size(0), lock("ObjectCache"), cct(NULL) { }
+ ObjectCache() : lru_size(0), lru_counter(0), lru_window(0), lock("ObjectCache"), cct(NULL) { }
int get(std::string& name, ObjectCacheInfo& bl, uint32_t mask);
void put(std::string& name, ObjectCacheInfo& bl);
void remove(std::string& name);
- void set_ctx(CephContext *_cct) { cct = _cct; }
+ void set_ctx(CephContext *_cct) {
+ cct = _cct;
+ lru_window = cct->_conf->rgw_cache_lru_size / 2;
+ }
};
template <class T>
@@ -199,7 +208,8 @@ public:
map<std::string, bufferlist>& attrs, RGWObjCategory category, int flags,
map<std::string, bufferlist>* rmattrs, const bufferlist *data,
RGWObjManifest *manifest, const string *ptag, list<string> *remove_objs,
- bool modify_version, RGWObjVersionTracker *objv_tracker, time_t set_mtime);
+ bool modify_version, RGWObjVersionTracker *objv_tracker, time_t set_mtime,
+ const string& owner);
int put_obj_data(void *ctx, rgw_obj& obj, const char *data,
off_t ofs, size_t len, bool exclusive);
@@ -208,7 +218,7 @@ public:
int obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime, uint64_t *epoch, map<string, bufferlist> *attrs,
bufferlist *first_chunk, RGWObjVersionTracker *objv_tracker);
- int delete_obj(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_tracker);
+ int delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker);
};
template <class T>
@@ -224,13 +234,13 @@ void RGWCache<T>::normalize_bucket_and_obj(rgw_bucket& src_bucket, string& src_o
}
template <class T>
-int RGWCache<T>::delete_obj(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+int RGWCache<T>::delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
{
rgw_bucket bucket;
string oid;
normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
if (bucket.name[0] != '.')
- return T::delete_obj(ctx, obj, objv_tracker);
+ return T::delete_obj_impl(ctx, bucket_owner, obj, objv_tracker);
string name = normal_name(obj);
cache.remove(name);
@@ -238,7 +248,7 @@ int RGWCache<T>::delete_obj(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_
ObjectCacheInfo info;
distribute_cache(name, obj, info, REMOVE_OBJ);
- return T::delete_obj(ctx, obj, objv_tracker);
+ return T::delete_obj_impl(ctx, bucket_owner, obj, objv_tracker);
}
template <class T>
@@ -379,7 +389,8 @@ int RGWCache<T>::put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size, time_
map<std::string, bufferlist>& attrs, RGWObjCategory category, int flags,
map<std::string, bufferlist>* rmattrs, const bufferlist *data,
RGWObjManifest *manifest, const string *ptag, list<string> *remove_objs,
- bool modify_version, RGWObjVersionTracker *objv_tracker, time_t set_mtime)
+ bool modify_version, RGWObjVersionTracker *objv_tracker, time_t set_mtime,
+ const string& owner)
{
rgw_bucket bucket;
string oid;
@@ -401,7 +412,7 @@ int RGWCache<T>::put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size, time_
}
}
int ret = T::put_obj_meta_impl(ctx, obj, size, mtime, attrs, category, flags, rmattrs, data, manifest, ptag, remove_objs,
- modify_version, objv_tracker, set_mtime);
+ modify_version, objv_tracker, set_mtime, owner);
if (cacheable) {
string name = normal_name(bucket, oid);
if (ret >= 0) {
diff --git a/src/rgw/rgw_civetweb.cc b/src/rgw/rgw_civetweb.cc
new file mode 100644
index 0000000..a31177f
--- /dev/null
+++ b/src/rgw/rgw_civetweb.cc
@@ -0,0 +1,172 @@
+
+#include <string.h>
+
+#include "civetweb/civetweb.h"
+#include "rgw_civetweb.h"
+
+
+#define dout_subsys ceph_subsys_rgw
+
+int RGWMongoose::write_data(const char *buf, int len)
+{
+ if (!header_done) {
+ header_data.append(buf, len);
+ return 0;
+ }
+ if (!sent_header) {
+ data.append(buf, len);
+ return 0;
+ }
+ return mg_write(conn, buf, len);
+}
+
+RGWMongoose::RGWMongoose(mg_connection *_conn, int _port) : conn(_conn), port(_port), header_done(false), sent_header(false), has_content_length(false),
+ explicit_keepalive(false)
+{
+}
+
+int RGWMongoose::read_data(char *buf, int len)
+{
+ return mg_read(conn, buf, len);
+}
+
+void RGWMongoose::flush()
+{
+}
+
+int RGWMongoose::complete_request()
+{
+ if (!sent_header) {
+ if (!has_content_length) {
+ header_done = false; /* let's go back to writing the header */
+
+ if (0 && data.length() == 0) {
+ has_content_length = true;
+ print("Transfer-Enconding: %s\n", "chunked");
+ data.append("0\r\n\r\n", sizeof("0\r\n\r\n")-1);
+ } else {
+ int r = send_content_length(data.length());
+ if (r < 0)
+ return r;
+ }
+ }
+
+ complete_header();
+ }
+
+ if (data.length()) {
+ int r = write_data(data.c_str(), data.length());
+ if (r < 0)
+ return r;
+ data.clear();
+ }
+
+ return 0;
+}
+
+void RGWMongoose::init_env(CephContext *cct)
+{
+ env.init(cct);
+ struct mg_request_info *info = mg_get_request_info(conn);
+ if (!info)
+ return;
+
+ for (int i = 0; i < info->num_headers; i++) {
+ struct mg_request_info::mg_header *header = &info->http_headers[i];
+
+ if (strcasecmp(header->name, "content-length") == 0) {
+ env.set("CONTENT_LENGTH", header->value);
+ continue;
+ }
+
+ if (strcasecmp(header->name, "content-type") == 0) {
+ env.set("CONTENT_TYPE", header->value);
+ continue;
+ }
+
+ if (strcasecmp(header->name, "connection") == 0) {
+ explicit_keepalive = (strcasecmp(header->value, "keep-alive") == 0);
+ }
+
+ int len = strlen(header->name) + 5; /* HTTP_ prepended */
+ char buf[len + 1];
+ memcpy(buf, "HTTP_", 5);
+ const char *src = header->name;
+ char *dest = &buf[5];
+ for (; *src; src++, dest++) {
+ char c = *src;
+ switch (c) {
+ case '-':
+ c = '_';
+ break;
+ default:
+ c = toupper(c);
+ break;
+ }
+ *dest = c;
+ }
+ *dest = '\0';
+
+ env.set(buf, header->value);
+ }
+
+ env.set("REQUEST_METHOD", info->request_method);
+ env.set("REQUEST_URI", info->uri);
+ env.set("QUERY_STRING", info->query_string);
+ env.set("REMOTE_USER", info->remote_user);
+ env.set("SCRIPT_URI", info->uri); /* FIXME */
+
+ char port_buf[16];
+ snprintf(port_buf, sizeof(port_buf), "%d", port);
+ env.set("SERVER_PORT", port_buf);
+}
+
+int RGWMongoose::send_status(const char *status, const char *status_name)
+{
+ char buf[128];
+
+ if (!status_name)
+ status_name = "";
+
+ snprintf(buf, sizeof(buf), "HTTP/1.1 %s %s\n", status, status_name);
+
+ bufferlist bl;
+ bl.append(buf);
+ bl.append(header_data);
+ header_data = bl;
+
+ return 0;
+}
+
+int RGWMongoose::send_100_continue()
+{
+ char buf[] = "HTTP/1.1 100 CONTINUE\r\n\r\n";
+
+ return mg_write(conn, buf, sizeof(buf) - 1);
+}
+
+int RGWMongoose::complete_header()
+{
+ header_done = true;
+
+ if (!has_content_length) {
+ return 0;
+ }
+
+ if (explicit_keepalive)
+ header_data.append("Connection: Keep-Alive\r\n");
+
+ header_data.append("\r\n");
+
+ sent_header = true;
+
+ return write_data(header_data.c_str(), header_data.length());
+}
+
+int RGWMongoose::send_content_length(uint64_t len)
+{
+ has_content_length = true;
+ char buf[21];
+ snprintf(buf, sizeof(buf), "%"PRIu64, len);
+ return print("Content-Length: %s\n", buf);
+}
diff --git a/src/rgw/rgw_civetweb.h b/src/rgw/rgw_civetweb.h
new file mode 100644
index 0000000..c1df9f0
--- /dev/null
+++ b/src/rgw/rgw_civetweb.h
@@ -0,0 +1,41 @@
+#ifndef CEPH_RGW_MONGOOSE_H
+#define CEPH_RGW_MONGOOSE_H
+
+#include "rgw_client_io.h"
+
+
+struct mg_connection;
+
+
+class RGWMongoose : public RGWClientIO
+{
+ mg_connection *conn;
+
+ bufferlist header_data;
+ bufferlist data;
+
+ int port;
+
+ bool header_done;
+ bool sent_header;
+ bool has_content_length;
+ bool explicit_keepalive;
+
+public:
+ void init_env(CephContext *cct);
+
+ int write_data(const char *buf, int len);
+ int read_data(char *buf, int len);
+
+ int send_status(const char *status, const char *status_name);
+ int send_100_continue();
+ int complete_header();
+ int complete_request();
+ int send_content_length(uint64_t len);
+
+ RGWMongoose(mg_connection *_conn, int _port);
+ void flush();
+};
+
+
+#endif
diff --git a/src/rgw/rgw_client_io.cc b/src/rgw/rgw_client_io.cc
index 46385f4..193f44e 100644
--- a/src/rgw/rgw_client_io.cc
+++ b/src/rgw/rgw_client_io.cc
@@ -5,6 +5,21 @@
#include "rgw_client_io.h"
+#define dout_subsys ceph_subsys_rgw
+
+void RGWClientIO::init(CephContext *cct) {
+ init_env(cct);
+
+ if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
+ std::map<string, string, ltstr_nocase>& env_map = env.get_map();
+ std::map<string, string, ltstr_nocase>::iterator iter = env_map.begin();
+
+ for (iter = env_map.begin(); iter != env_map.end(); ++iter) {
+ ldout(cct, 20) << iter->first << "=" << iter->second << dendl;
+ }
+ }
+}
+
int RGWClientIO::print(const char *format, ...)
{
@@ -57,3 +72,4 @@ int RGWClientIO::read(char *buf, int max, int *actual)
return 0;
}
+
diff --git a/src/rgw/rgw_client_io.h b/src/rgw/rgw_client_io.h
index 546b16d..2e8720e 100644
--- a/src/rgw/rgw_client_io.h
+++ b/src/rgw/rgw_client_io.h
@@ -5,6 +5,8 @@
#include "include/types.h"
+#include "rgw_common.h"
+
class RGWClientIO {
bool account;
@@ -12,6 +14,10 @@ class RGWClientIO {
size_t bytes_received;
protected:
+ RGWEnv env;
+
+ virtual void init_env(CephContext *cct) = 0;
+
virtual int write_data(const char *buf, int len) = 0;
virtual int read_data(char *buf, int max) = 0;
@@ -19,12 +25,19 @@ public:
virtual ~RGWClientIO() {}
RGWClientIO() : account(false), bytes_sent(0), bytes_received(0) {}
+ void init(CephContext *cct);
int print(const char *format, ...);
int write(const char *buf, int len);
virtual void flush() = 0;
int read(char *buf, int max, int *actual);
- virtual const char **envp() = 0;
+ virtual int send_status(const char *status, const char *status_name) = 0;
+ virtual int send_100_continue() = 0;
+ virtual int complete_header() = 0;
+ virtual int complete_request() = 0;
+ virtual int send_content_length(uint64_t len) = 0;
+
+ RGWEnv& get_env() { return env; }
void set_account(bool _account) {
account = _account;
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 317dd88..57b8ceb 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -101,6 +101,8 @@ req_info::req_info(CephContext *cct, class RGWEnv *e) : env(e) {
if (pos >= 0) {
request_params = request_uri.substr(pos + 1);
request_uri = request_uri.substr(0, pos);
+ } else {
+ request_params = env->get("QUERY_STRING", "");
}
host = env->get("HTTP_HOST");
}
@@ -128,6 +130,7 @@ req_state::req_state(CephContext *_cct, class RGWEnv *e) : cct(_cct), cio(NULL),
{
enable_ops_log = e->conf->enable_ops_log;
enable_usage_log = e->conf->enable_usage_log;
+ defer_to_bucket_acls = e->conf->defer_to_bucket_acls;
content_started = false;
format = 0;
formatter = NULL;
@@ -184,8 +187,8 @@ void req_info::init_meta_info(bool *found_bad_meta)
{
x_meta_map.clear();
- map<string, string>& m = env->get_map();
- map<string, string>::iterator iter;
+ map<string, string, ltstr_nocase>& m = env->get_map();
+ map<string, string, ltstr_nocase>::iterator iter;
for (iter = m.begin(); iter != m.end(); ++iter) {
const char *prefix;
const string& header_name = iter->first;
@@ -541,6 +544,7 @@ int XMLArgs::parse()
(name.compare("caps") == 0) ||
(name.compare("index") == 0) ||
(name.compare("policy") == 0) ||
+ (name.compare("quota") == 0) ||
(name.compare("object") == 0)) {
if (!admin_subresource_added) {
@@ -615,8 +619,18 @@ bool verify_bucket_permission(struct req_state *s, int perm)
return s->bucket_acl->verify_permission(s->user.user_id, perm, perm);
}
+static inline bool check_deferred_bucket_acl(struct req_state *s, uint8_t deferred_check, int perm)
+{
+ return (s->defer_to_bucket_acls == deferred_check && verify_bucket_permission(s, perm));
+}
+
bool verify_object_permission(struct req_state *s, RGWAccessControlPolicy *bucket_acl, RGWAccessControlPolicy *object_acl, int perm)
{
+ if (check_deferred_bucket_acl(s, RGW_DEFER_TO_BUCKET_ACLS_RECURSE, perm) ||
+ check_deferred_bucket_acl(s, RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL, RGW_PERM_FULL_CONTROL)) {
+ return true;
+ }
+
if (!object_acl)
return false;
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index a7ef250..30f8e95 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -30,7 +30,9 @@
#include "rgw_acl.h"
#include "rgw_cors.h"
#include "rgw_quota.h"
+#include "rgw_string.h"
#include "cls/version/cls_version_types.h"
+#include "cls/user/cls_user_types.h"
#include "include/rados/librados.hpp"
using namespace std;
@@ -96,6 +98,9 @@ using ceph::crypto::MD5;
#define RGW_DEFAULT_MAX_BUCKETS 1000
+#define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
+#define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
+
#define STATUS_CREATED 1900
#define STATUS_ACCEPTED 1901
#define STATUS_NO_CONTENT 1902
@@ -273,13 +278,15 @@ class XMLArgs
class RGWConf;
class RGWEnv {
- std::map<string, string> env_map;
+ std::map<string, string, ltstr_nocase> env_map;
public:
RGWConf *conf;
RGWEnv();
~RGWEnv();
+ void init(CephContext *cct);
void init(CephContext *cct, char **envp);
+ void set(const char *name, const char *val);
const char *get(const char *name, const char *def_val = NULL);
int get_int(const char *name, int def_val = 0);
bool get_bool(const char *name, bool def_val = 0);
@@ -288,8 +295,8 @@ public:
bool exists_prefix(const char *prefix);
void remove(const char *name);
- void set(const char *name, const char *val);
- std::map<string, string>& get_map() { return env_map; }
+
+ std::map<string, string, ltstr_nocase>& get_map() { return env_map; }
};
class RGWConf {
@@ -298,10 +305,11 @@ protected:
void init(CephContext *cct, RGWEnv * env);
public:
RGWConf() :
- enable_ops_log(1), enable_usage_log(1) {}
+ enable_ops_log(1), enable_usage_log(1), defer_to_bucket_acls(0) {}
int enable_ops_log;
int enable_usage_log;
+ uint8_t defer_to_bucket_acls;
};
enum http_op {
@@ -319,8 +327,8 @@ class RGWAccessControlPolicy;
class JSONObj;
struct RGWAccessKey {
- string id;
- string key;
+ string id; // AccessKey
+ string key; // SecretKey
string subuser;
RGWAccessKey() {}
@@ -427,11 +435,13 @@ struct RGWUserInfo
string default_placement;
list<string> placement_tags;
RGWQuotaInfo bucket_quota;
+ map<int, string> temp_url_keys;
+ RGWQuotaInfo user_quota;
RGWUserInfo() : auid(0), suspended(0), max_buckets(RGW_DEFAULT_MAX_BUCKETS), op_mask(RGW_OP_TYPE_ALL), system(0) {}
void encode(bufferlist& bl) const {
- ENCODE_START(14, 9, bl);
+ ENCODE_START(16, 9, bl);
::encode(auid, bl);
string access_key;
string secret_key;
@@ -467,10 +477,12 @@ struct RGWUserInfo
::encode(default_placement, bl);
::encode(placement_tags, bl);
::encode(bucket_quota, bl);
+ ::encode(temp_url_keys, bl);
+ ::encode(user_quota, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
- DECODE_START_LEGACY_COMPAT_LEN_32(13, 9, 9, bl);
+ DECODE_START_LEGACY_COMPAT_LEN_32(16, 9, 9, bl);
if (struct_v >= 2) ::decode(auid, bl);
else auid = CEPH_AUTH_UID_DEFAULT;
string access_key;
@@ -526,6 +538,12 @@ struct RGWUserInfo
if (struct_v >= 14) {
::decode(bucket_quota, bl);
}
+ if (struct_v >= 15) {
+ ::decode(temp_url_keys, bl);
+ }
+ if (struct_v >= 16) {
+ ::decode(user_quota, bl);
+ }
DECODE_FINISH(bl);
}
void dump(Formatter *f) const;
@@ -556,6 +574,13 @@ struct rgw_bucket {
*/
rgw_bucket() { }
+ rgw_bucket(const cls_user_bucket& b) {
+ name = b.name;
+ data_pool = b.data_pool;
+ index_pool = b.index_pool;
+ marker = b.marker;
+ bucket_id = b.bucket_id;
+ }
rgw_bucket(const char *n) : name(n) {
assert(*n == '.'); // only rgw private buckets should be initialized without pool
data_pool = index_pool = n;
@@ -564,6 +589,14 @@ struct rgw_bucket {
rgw_bucket(const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) :
name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
+ void convert(cls_user_bucket *b) {
+ b->name = name;
+ b->data_pool = data_pool;
+ b->index_pool = index_pool;
+ b->marker = marker;
+ b->bucket_id = bucket_id;
+ }
+
void clear() {
name = "";
data_pool = "";
@@ -764,14 +797,14 @@ struct RGWBucketEntryPoint
};
WRITE_CLASS_ENCODER(RGWBucketEntryPoint)
-struct RGWBucketStats
+struct RGWStorageStats
{
RGWObjCategory category;
uint64_t num_kb;
uint64_t num_kb_rounded;
uint64_t num_objects;
- RGWBucketStats() : num_kb(0), num_kb_rounded(0), num_objects(0) {}
+ RGWStorageStats() : category(RGW_OBJ_CATEGORY_NONE), num_kb(0), num_kb_rounded(0), num_objects(0) {}
};
struct req_state;
@@ -816,6 +849,7 @@ struct req_state {
uint64_t obj_size;
bool enable_ops_log;
bool enable_usage_log;
+ uint8_t defer_to_bucket_acls;
uint32_t perm_mask;
utime_t header_time;
@@ -896,6 +930,22 @@ struct RGWBucketEnt {
RGWBucketEnt() : size(0), size_rounded(0), creation_time(0), count(0) {}
+ RGWBucketEnt(const cls_user_bucket_entry& e) {
+ bucket = e.bucket;
+ size = e.size;
+ size_rounded = e.size_rounded;
+ creation_time = e.creation_time;
+ count = e.count;
+ }
+
+ void convert(cls_user_bucket_entry *b) {
+ bucket.convert(&b->bucket);
+ b->size = size;
+ b->size_rounded = size_rounded;
+ b->creation_time = creation_time;
+ b->count = count;
+ }
+
void encode(bufferlist& bl) const {
ENCODE_START(5, 5, bl);
uint64_t s = size;
@@ -1237,6 +1287,11 @@ static inline uint64_t rgw_rounded_kb(uint64_t bytes)
return (bytes + 1023) / 1024;
}
+static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
+{
+ return ((bytes + 4095) & ~4095) / 1024;
+}
+
extern string rgw_string_unquote(const string& s);
extern void parse_csv_string(const string& ival, vector<string>& ovals);
extern int parse_key_value(string& in_str, string& key, string& val);
diff --git a/src/rgw/rgw_cors.cc b/src/rgw/rgw_cors.cc
index 4be8360..a120a68 100644
--- a/src/rgw/rgw_cors.cc
+++ b/src/rgw/rgw_cors.cc
@@ -51,6 +51,33 @@ void RGWCORSRule::erase_origin_if_present(string& origin, bool *rule_empty) {
}
}
+/*
+ * make attrs look-like-this
+ * does not convert underscores or dashes
+ *
+ * Per CORS specification, section 3:
+ * ===
+ * "Converting a string to ASCII lowercase" means replacing all characters in the
+ * range U+0041 LATIN CAPITAL LETTER A to U+005A LATIN CAPITAL LETTER Z with
+ * the corresponding characters in the range U+0061 LATIN SMALL LETTER A to
+ * U+007A LATIN SMALL LETTER Z).
+ * ===
+ *
+ * @todo When UTF-8 is allowed in HTTP headers, this function will need to change
+ */
+string lowercase_http_attr(const string& orig)
+{
+ const char *s = orig.c_str();
+ char buf[orig.size() + 1];
+ buf[orig.size()] = '\0';
+
+ for (size_t i = 0; i < orig.size(); ++i, ++s) {
+ buf[i] = tolower(*s);
+ }
+ return string(buf);
+}
+
+
static bool is_string_in_set(set<string>& s, string h) {
if ((s.find("*") != s.end()) ||
(s.find(h) != s.end())) {
@@ -96,7 +123,13 @@ bool RGWCORSRule::is_origin_present(const char *o) {
bool RGWCORSRule::is_header_allowed(const char *h, size_t len) {
string hdr(h, len);
- return is_string_in_set(allowed_hdrs, hdr);
+ if(lowercase_allowed_hdrs.empty()) {
+ set<string>::iterator iter;
+ for (iter = allowed_hdrs.begin(); iter != allowed_hdrs.end(); ++iter) {
+ lowercase_allowed_hdrs.insert(lowercase_http_attr(*iter));
+ }
+ }
+ return is_string_in_set(lowercase_allowed_hdrs, lowercase_http_attr(hdr));
}
void RGWCORSRule::format_exp_headers(string& s) {
diff --git a/src/rgw/rgw_cors.h b/src/rgw/rgw_cors.h
index 1e0ec3b..124ebf9 100644
--- a/src/rgw/rgw_cors.h
+++ b/src/rgw/rgw_cors.h
@@ -41,7 +41,8 @@ protected:
uint32_t max_age;
uint8_t allowed_methods;
std::string id;
- std::set<string> allowed_hdrs;
+ std::set<string> allowed_hdrs; /* If you change this, you need to discard lowercase_allowed_hdrs */
+ std::set<string> lowercase_allowed_hdrs; /* Not built until needed in RGWCORSRule::is_header_allowed */
std::set<string> allowed_origins;
std::list<string> exposable_hdrs;
diff --git a/src/rgw/rgw_env.cc b/src/rgw/rgw_env.cc
index 78ac0d4..faa8d8f 100644
--- a/src/rgw/rgw_env.cc
+++ b/src/rgw/rgw_env.cc
@@ -16,6 +16,20 @@ RGWEnv::~RGWEnv()
delete conf;
}
+void RGWEnv::init(CephContext *cct)
+{
+ conf->init(cct, this);
+}
+
+void RGWEnv::set(const char *name, const char *val)
+{
+ if (!val)
+ val = "";
+ env_map[name] = val;
+
+ dout(20) << "RGWEnv::set(): " << name << ": " << val << dendl;
+}
+
void RGWEnv::init(CephContext *cct, char **envp)
{
const char *p;
@@ -32,12 +46,12 @@ void RGWEnv::init(CephContext *cct, char **envp)
env_map[name] = val;
}
- conf->init(cct, this);
+ init(cct);
}
const char *RGWEnv::get(const char *name, const char *def_val)
{
- map<string, string>::iterator iter = env_map.find(name);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.find(name);
if (iter == env_map.end())
return def_val;
@@ -46,7 +60,7 @@ const char *RGWEnv::get(const char *name, const char *def_val)
int RGWEnv::get_int(const char *name, int def_val)
{
- map<string, string>::iterator iter = env_map.find(name);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.find(name);
if (iter == env_map.end())
return def_val;
@@ -56,7 +70,7 @@ int RGWEnv::get_int(const char *name, int def_val)
bool RGWEnv::get_bool(const char *name, bool def_val)
{
- map<string, string>::iterator iter = env_map.find(name);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.find(name);
if (iter == env_map.end())
return def_val;
@@ -66,7 +80,7 @@ bool RGWEnv::get_bool(const char *name, bool def_val)
size_t RGWEnv::get_size(const char *name, size_t def_val)
{
- map<string, string>::iterator iter = env_map.find(name);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.find(name);
if (iter == env_map.end())
return def_val;
@@ -76,7 +90,7 @@ size_t RGWEnv::get_size(const char *name, size_t def_val)
bool RGWEnv::exists(const char *name)
{
- map<string, string>::iterator iter = env_map.find(name);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.find(name);
return (iter != env_map.end());
}
@@ -85,21 +99,16 @@ bool RGWEnv::exists_prefix(const char *prefix)
if (env_map.empty() || prefix == NULL)
return false;
- map<string, string>::iterator iter = env_map.lower_bound(prefix);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.lower_bound(prefix);
if (iter == env_map.end())
return false;
return (strncmp(iter->first.c_str(), prefix, strlen(prefix)) == 0);
}
-void RGWEnv::set(const char *name, const char *val)
-{
- env_map[name] = val;
-}
-
void RGWEnv::remove(const char *name)
{
- map<string, string>::iterator iter = env_map.find(name);
+ map<string, string, ltstr_nocase>::iterator iter = env_map.find(name);
if (iter != env_map.end())
env_map.erase(iter);
}
@@ -108,4 +117,11 @@ void RGWConf::init(CephContext *cct, RGWEnv *env)
{
enable_ops_log = cct->_conf->rgw_enable_ops_log;
enable_usage_log = cct->_conf->rgw_enable_usage_log;
+
+ defer_to_bucket_acls = 0; // default
+ if (cct->_conf->rgw_defer_to_bucket_acls == "recurse") {
+ defer_to_bucket_acls = RGW_DEFER_TO_BUCKET_ACLS_RECURSE;
+ } else if (cct->_conf->rgw_defer_to_bucket_acls == "full_control") {
+ defer_to_bucket_acls = RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL;
+ }
}
diff --git a/src/rgw/rgw_fcgi.cc b/src/rgw/rgw_fcgi.cc
index 70ed996..4b24dab 100644
--- a/src/rgw/rgw_fcgi.cc
+++ b/src/rgw/rgw_fcgi.cc
@@ -25,7 +25,34 @@ void RGWFCGX::flush()
FCGX_FFlush(fcgx->out);
}
-const char **RGWFCGX::envp()
+void RGWFCGX::init_env(CephContext *cct)
{
- return (const char **)fcgx->envp;
+ env.init(cct, (char **)fcgx->envp);
}
+
+int RGWFCGX::send_status(const char *status, const char *status_name)
+{
+ return print("Status: %s\n", status);
+}
+
+int RGWFCGX::send_100_continue()
+{
+ int r = send_status("100", "Continue");
+ if (r >= 0) {
+ flush();
+ }
+ return r;
+}
+
+int RGWFCGX::send_content_length(uint64_t len)
+{
+ char buf[21];
+ snprintf(buf, sizeof(buf), "%"PRIu64, len);
+ return print("Content-Length: %s\n", buf);
+}
+
+int RGWFCGX::complete_header()
+{
+ return print("\r\n");
+}
+
diff --git a/src/rgw/rgw_fcgi.h b/src/rgw/rgw_fcgi.h
index ccf48f5..fabc77a 100644
--- a/src/rgw/rgw_fcgi.h
+++ b/src/rgw/rgw_fcgi.h
@@ -11,13 +11,18 @@ class RGWFCGX : public RGWClientIO
{
FCGX_Request *fcgx;
protected:
+ void init_env(CephContext *cct);
int write_data(const char *buf, int len);
int read_data(char *buf, int len);
+ int send_status(const char *status, const char *status_name);
+ int send_100_continue();
+ int complete_header();
+ int complete_request() { return 0; }
+ int send_content_length(uint64_t len);
public:
RGWFCGX(FCGX_Request *_fcgx) : fcgx(_fcgx) {}
void flush();
- const char **envp();
};
diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 353afd5..1b2b8ca 100644
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -17,11 +17,17 @@ using namespace librados;
static string gc_oid_prefix = "gc";
static string gc_index_lock_name = "gc_process";
+
+#define HASH_PRIME 7877
+
void RGWGC::initialize(CephContext *_cct, RGWRados *_store) {
cct = _cct;
store = _store;
max_objs = cct->_conf->rgw_gc_max_objs;
+ if (max_objs > HASH_PRIME)
+ max_objs = HASH_PRIME;
+
obj_names = new string[max_objs];
for (int i = 0; i < max_objs; i++) {
@@ -39,7 +45,7 @@ void RGWGC::finalize()
int RGWGC::tag_index(const string& tag)
{
- return ceph_str_hash_linux(tag.c_str(), tag.size()) % max_objs;
+ return ceph_str_hash_linux(tag.c_str(), tag.size()) % HASH_PRIME % max_objs;
}
void RGWGC::add_chain(ObjectWriteOperation& op, cls_rgw_obj_chain& chain, const string& tag)
diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc
index 314e80b..1c6b6d4 100644
--- a/src/rgw/rgw_http_client.cc
+++ b/src/rgw/rgw_http_client.cc
@@ -234,7 +234,7 @@ static int do_curl_wait(CephContext *cct, CURLM *handle)
int RGWHTTPClient::process_request(void *handle, bool wait_for_data, bool *done)
{
- multi_req_data *req_data = (multi_req_data *)handle;
+ multi_req_data *req_data = static_cast<multi_req_data *>(handle);
int still_running;
int mstatus;
@@ -282,7 +282,7 @@ int RGWHTTPClient::complete_request(void *handle)
do {
ret = process_request(handle, true, &done);
} while (!done && !ret);
- multi_req_data *req_data = (multi_req_data *)handle;
+ multi_req_data *req_data = static_cast<multi_req_data *>(handle);
delete req_data;
return ret;
diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h
index ba3e522..0ab19ad 100644
--- a/src/rgw/rgw_http_errors.h
+++ b/src/rgw/rgw_http_errors.h
@@ -60,6 +60,52 @@ const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = {
{ ERR_BAD_URL, 412, "Bad URL" },
};
+struct rgw_http_status_code {
+ int code;
+ const char *name;
+};
+
+const static struct rgw_http_status_code http_codes[] = {
+ { 100, "Continue" },
+ { 200, "OK" },
+ { 201, "Created" },
+ { 202, "Accepted" },
+ { 204, "No Content" },
+ { 205, "Reset Content" },
+ { 206, "Partial Content" },
+ { 207, "Multi Status" },
+ { 208, "Already Reported" },
+ { 300, "Multiple Choices" },
+ { 302, "Found" },
+ { 303, "See Other" },
+ { 304, "Not Modified" },
+ { 305, "User Proxy" },
+ { 306, "Switch Proxy" },
+ { 307, "Temporary Redirect" },
+ { 308, "Permanent Redirect" },
+ { 400, "Bad Request" },
+ { 401, "Unauthorized" },
+ { 402, "Payment Required" },
+ { 403, "Forbidden" },
+ { 404, "Not Found" },
+ { 405, "Method Not Allowed" },
+ { 406, "Not Acceptable" },
+ { 407, "Proxy Authentication Required" },
+ { 408, "Request Timeout" },
+ { 409, "Conflict" },
+ { 410, "Gone" },
+ { 411, "Length Required" },
+ { 412, "Precondition Failed" },
+ { 413, "Request Entity Too Large" },
+ { 414, "Request-URI Too Long" },
+ { 415, "Unsupported Media Type" },
+ { 416, "Requested Range Not Satisfiable" },
+ { 417, "Expectation Failed" },
+ { 422, "Unprocessable Entity" },
+ { 500, "Internal Server Error" },
+ { 0, NULL },
+};
+
#define ARRAY_LEN(arr) (sizeof(arr) / sizeof(arr[0]))
static inline const struct rgw_http_errors *search_err(int err_no, const struct rgw_http_errors *errs, int len)
diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 4d6b253..231475b 100644
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -397,6 +397,8 @@ void RGWUserInfo::dump(Formatter *f) const
encode_json("default_placement", default_placement, f);
encode_json("placement_tags", placement_tags, f);
encode_json("bucket_quota", bucket_quota, f);
+ encode_json("user_quota", user_quota, f);
+ encode_json("temp_url_keys", temp_url_keys, f);
}
@@ -448,6 +450,8 @@ void RGWUserInfo::decode_json(JSONObj *obj)
JSONDecoder::decode_json("default_placement", default_placement, obj);
JSONDecoder::decode_json("placement_tags", placement_tags, obj);
JSONDecoder::decode_json("bucket_quota", bucket_quota, obj);
+ JSONDecoder::decode_json("user_quota", user_quota, obj);
+ JSONDecoder::decode_json("temp_url_keys", temp_url_keys, obj);
}
void RGWQuotaInfo::dump(Formatter *f) const
@@ -692,13 +696,14 @@ void RGWRegionMap::dump(Formatter *f) const
encode_json("regions", regions, f);
encode_json("master_region", master_region, f);
encode_json("bucket_quota", bucket_quota, f);
+ encode_json("user_quota", user_quota, f);
}
void RGWRegionMap::decode_json(JSONObj *obj)
{
JSONDecoder::decode_json("regions", regions, obj);
JSONDecoder::decode_json("master_region", master_region, obj);
- JSONDecoder::decode_json("bucket_quota", bucket_quota, obj);
+ JSONDecoder::decode_json("user_quota", user_quota, obj);
}
void RGWMetadataLogInfo::dump(Formatter *f) const
diff --git a/src/rgw/rgw_loadgen.cc b/src/rgw/rgw_loadgen.cc
new file mode 100644
index 0000000..01cf1a3
--- /dev/null
+++ b/src/rgw/rgw_loadgen.cc
@@ -0,0 +1,111 @@
+
+#include <string.h>
+
+#include "rgw_loadgen.h"
+#include "rgw_auth_s3.h"
+
+
+#define dout_subsys ceph_subsys_rgw
+
+void RGWLoadGenRequestEnv::set_date(utime_t& tm)
+{
+ stringstream s;
+ tm.asctime(s);
+ date_str = s.str();
+}
+
+int RGWLoadGenRequestEnv::sign(RGWAccessKey& access_key)
+{
+ map<string, string> meta_map;
+ map<string, string> sub_resources;
+
+ string canonical_header;
+ string digest;
+
+ rgw_create_s3_canonical_header(request_method.c_str(),
+ NULL, /* const char *content_md5 */
+ content_type.c_str(),
+ date_str.c_str(),
+ meta_map,
+ uri.c_str(),
+ sub_resources,
+ canonical_header);
+
+ int ret = rgw_get_s3_header_digest(canonical_header, access_key.key, digest);
+ if (ret < 0) {
+ return ret;
+ }
+
+ headers["HTTP_DATE"] = date_str;
+ headers["HTTP_AUTHORIZATION"] = string("AWS ") + access_key.id + ":" + digest;
+
+ return 0;
+}
+
+int RGWLoadGenIO::write_data(const char *buf, int len)
+{
+ return len;
+}
+
+int RGWLoadGenIO::read_data(char *buf, int len)
+{
+ int read_len = MIN(left_to_read, (uint64_t)len);
+ left_to_read -= read_len;
+ return read_len;
+}
+
+void RGWLoadGenIO::flush()
+{
+}
+
+int RGWLoadGenIO::complete_request()
+{
+ return 0;
+}
+
+void RGWLoadGenIO::init_env(CephContext *cct)
+{
+ env.init(cct);
+
+ left_to_read = req->content_length;
+
+ char buf[32];
+ snprintf(buf, sizeof(buf), "%lld", (long long)req->content_length);
+ env.set("CONTENT_LENGTH", buf);
+
+ env.set("CONTENT_TYPE", req->content_type.c_str());
+ env.set("HTTP_DATE", req->date_str.c_str());
+
+ for (map<string, string>::iterator iter = req->headers.begin(); iter != req->headers.end(); ++iter) {
+ env.set(iter->first.c_str(), iter->second.c_str());
+ }
+
+ env.set("REQUEST_METHOD", req->request_method.c_str());
+ env.set("REQUEST_URI", req->uri.c_str());
+ env.set("QUERY_STRING", req->query_string.c_str());
+ env.set("SCRIPT_URI", req->uri.c_str());
+
+ char port_buf[16];
+ snprintf(port_buf, sizeof(port_buf), "%d", req->port);
+ env.set("SERVER_PORT", port_buf);
+}
+
+int RGWLoadGenIO::send_status(const char *status, const char *status_name)
+{
+ return 0;
+}
+
+int RGWLoadGenIO::send_100_continue()
+{
+ return 0;
+}
+
+int RGWLoadGenIO::complete_header()
+{
+ return 0;
+}
+
+int RGWLoadGenIO::send_content_length(uint64_t len)
+{
+ return 0;
+}
diff --git a/src/rgw/rgw_loadgen.h b/src/rgw/rgw_loadgen.h
new file mode 100644
index 0000000..5459330
--- /dev/null
+++ b/src/rgw/rgw_loadgen.h
@@ -0,0 +1,45 @@
+#ifndef CEPH_RGW_LOADGEN_H
+#define CEPH_RGW_LOADGEN_H
+
+#include "rgw_client_io.h"
+
+
+struct RGWLoadGenRequestEnv {
+ int port;
+ uint64_t content_length;
+ string content_type;
+ string request_method;
+ string uri;
+ string query_string;
+ string date_str;
+
+ map<string, string> headers;
+
+ RGWLoadGenRequestEnv() : port(0), content_length(0) {}
+
+ void set_date(utime_t& tm);
+ int sign(RGWAccessKey& access_key);
+};
+
+class RGWLoadGenIO : public RGWClientIO
+{
+ uint64_t left_to_read;
+ RGWLoadGenRequestEnv *req;
+public:
+ void init_env(CephContext *cct);
+
+ int write_data(const char *buf, int len);
+ int read_data(char *buf, int len);
+
+ int send_status(const char *status, const char *status_name);
+ int send_100_continue();
+ int complete_header();
+ int complete_request();
+ int send_content_length(uint64_t len);
+
+ RGWLoadGenIO(RGWLoadGenRequestEnv *_re) : left_to_read(0), req(_re) {}
+ void flush();
+};
+
+
+#endif
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 5fbecf8..f104109 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -27,6 +27,7 @@
#include "common/WorkQueue.h"
#include "common/Timer.h"
#include "common/Throttle.h"
+#include "common/safe_io.h"
#include "include/str_list.h"
#include "rgw_common.h"
#include "rgw_rados.h"
@@ -51,6 +52,10 @@
#include "rgw_log.h"
#include "rgw_tools.h"
#include "rgw_resolve.h"
+#include "rgw_loadgen.h"
+#include "rgw_civetweb.h"
+
+#include "civetweb/civetweb.h"
#include <map>
#include <string>
@@ -65,18 +70,20 @@
using namespace std;
-static sighandler_t sighandler_alrm;
+static sig_t sighandler_alrm;
class RGWProcess;
-static RGWProcess *pprocess = NULL;
+static int signal_fd[2] = {0, 0};
+static atomic_t disable_signal_fd;
+
+static void signal_shutdown();
#define SOCKET_BACKLOG 1024
struct RGWRequest
{
- FCGX_Request fcgx;
uint64_t id;
struct req_state *s;
string req_str;
@@ -123,13 +130,46 @@ struct RGWRequest
}
};
-class RGWProcess {
+class RGWFrontendConfig {
+ string config;
+ map<string, string> config_map;
+ int parse_config(const string& config, map<string, string>& config_map);
+ string framework;
+public:
+ RGWFrontendConfig(const string& _conf) : config(_conf) {}
+ int init() {
+ int ret = parse_config(config, config_map);
+ if (ret < 0)
+ return ret;
+ return 0;
+ }
+ bool get_val(const string& key, const string& def_val, string *out);
+ bool get_val(const string& key, int def_val, int *out);
+
+ string get_framework() { return framework; }
+};
+
+
+struct RGWFCGXRequest : public RGWRequest {
+ FCGX_Request fcgx;
+};
+
+struct RGWProcessEnv {
RGWRados *store;
+ RGWREST *rest;
OpsLogSocket *olog;
+ int port;
+};
+
+class RGWProcess {
deque<RGWRequest *> m_req_queue;
+protected:
+ RGWRados *store;
+ OpsLogSocket *olog;
ThreadPool m_tp;
Throttle req_throttle;
RGWREST *rest;
+ RGWFrontendConfig *conf;
int sock_fd;
struct RGWWQ : public ThreadPool::WorkQueue<RGWRequest> {
@@ -185,27 +225,49 @@ class RGWProcess {
uint64_t max_req_id;
public:
- RGWProcess(CephContext *cct, RGWRados *rgwstore, OpsLogSocket *_olog, int num_threads, RGWREST *_rest)
- : store(rgwstore), olog(_olog), m_tp(cct, "RGWProcess::m_tp", num_threads),
+ RGWProcess(CephContext *cct, RGWProcessEnv *pe, int num_threads, RGWFrontendConfig *_conf)
+ : store(pe->store), olog(pe->olog), m_tp(cct, "RGWProcess::m_tp", num_threads),
req_throttle(cct, "rgw_ops", num_threads * 2),
- rest(_rest), sock_fd(-1),
+ rest(pe->rest),
+ conf(_conf),
+ sock_fd(-1),
req_wq(this, g_conf->rgw_op_thread_timeout,
g_conf->rgw_op_thread_suicide_timeout, &m_tp),
max_req_id(0) {}
- void run();
- void handle_request(RGWRequest *req);
+ virtual ~RGWProcess() {}
+ virtual void run() = 0;
+ virtual void handle_request(RGWRequest *req) = 0;
void close_fd() {
- if (sock_fd >= 0)
- close(sock_fd);
+ if (sock_fd >= 0) {
+ ::close(sock_fd);
+ sock_fd = -1;
+ }
}
};
-void RGWProcess::run()
+
+class RGWFCGXProcess : public RGWProcess {
+public:
+ RGWFCGXProcess(CephContext *cct, RGWProcessEnv *pe, int num_threads, RGWFrontendConfig *_conf) :
+ RGWProcess(cct, pe, num_threads, _conf) {}
+ void run();
+ void handle_request(RGWRequest *req);
+};
+
+void RGWFCGXProcess::run()
{
- sock_fd = 0;
- if (!g_conf->rgw_socket_path.empty()) {
- string path_str = g_conf->rgw_socket_path;
+ string socket_path;
+ string socket_port;
+ string socket_host;
+
+ conf->get_val("socket_path", g_conf->rgw_socket_path, &socket_path);
+ conf->get_val("socket_port", g_conf->rgw_port, &socket_port);
+ conf->get_val("socket_host", g_conf->rgw_host, &socket_host);
+
+
+ if (!socket_path.empty()) {
+ string path_str = socket_path;
/* this is necessary, as FCGX_OpenSocket might not return an error, but rather ungracefully exit */
int fd = open(path_str.c_str(), O_CREAT, 0644);
@@ -229,8 +291,8 @@ void RGWProcess::run()
if (chmod(path, 0777) < 0) {
dout(0) << "WARNING: couldn't set permissions on unix domain socket" << dendl;
}
- } else if (!g_conf->rgw_port.empty()) {
- string bind = g_conf->rgw_host + ":" + g_conf->rgw_port;
+ } else if (!socket_port.empty()) {
+ string bind = socket_host + ":" + socket_port;
sock_fd = FCGX_OpenSocket(bind.c_str(), SOCKET_BACKLOG);
if (sock_fd < 0) {
dout(0) << "ERROR: FCGX_OpenSocket (" << bind.c_str() << ") returned " << sock_fd << dendl;
@@ -241,7 +303,7 @@ void RGWProcess::run()
m_tp.start();
for (;;) {
- RGWRequest *req = new RGWRequest;
+ RGWFCGXRequest *req = new RGWFCGXRequest;
req->id = ++max_req_id;
dout(10) << "allocated request req=" << hex << req << dec << dendl;
FCGX_InitRequest(&req->fcgx, sock_fd, 0);
@@ -257,23 +319,168 @@ void RGWProcess::run()
req_wq.queue(req);
}
- m_tp.drain();
+ m_tp.drain(&req_wq);
m_tp.stop();
}
+struct RGWLoadGenRequest : public RGWRequest {
+ string method;
+ string resource;
+ int content_length;
+ atomic_t *fail_flag;
+
+
+ RGWLoadGenRequest(const string& _m, const string& _r, int _cl,
+ atomic_t *ff) : method(_m), resource(_r), content_length(_cl), fail_flag(ff) {}
+};
+
+class RGWLoadGenProcess : public RGWProcess {
+ RGWAccessKey access_key;
+public:
+ RGWLoadGenProcess(CephContext *cct, RGWProcessEnv *pe, int num_threads, RGWFrontendConfig *_conf) :
+ RGWProcess(cct, pe, num_threads, _conf) {}
+ void run();
+ void checkpoint();
+ void handle_request(RGWRequest *req);
+ void gen_request(const string& method, const string& resource, int content_length, atomic_t *fail_flag);
+
+ void set_access_key(RGWAccessKey& key) { access_key = key; }
+};
+
+void RGWLoadGenProcess::checkpoint()
+{
+ m_tp.drain(&req_wq);
+}
+
+void RGWLoadGenProcess::run()
+{
+ m_tp.start(); /* start thread pool */
+
+ int i;
+
+ int num_objs;
+
+ conf->get_val("num_objs", 1000, &num_objs);
+
+ int num_buckets;
+ conf->get_val("num_buckets", 1, &num_buckets);
+
+ string buckets[num_buckets];
+
+ atomic_t failed;
+
+ for (i = 0; i < num_buckets; i++) {
+ buckets[i] = "/loadgen";
+ string& bucket = buckets[i];
+ append_rand_alpha(NULL, bucket, bucket, 16);
+
+ /* first create a bucket */
+ gen_request("PUT", bucket, 0, &failed);
+ checkpoint();
+ }
+
+ string *objs = new string[num_objs];
+
+ if (failed.read()) {
+ derr << "ERROR: bucket creation failed" << dendl;
+ goto done;
+ }
+
+ for (i = 0; i < num_objs; i++) {
+ char buf[16 + 1];
+ gen_rand_alphanumeric(NULL, buf, sizeof(buf));
+ buf[16] = '\0';
+ objs[i] = buckets[i % num_buckets] + "/" + buf;
+ }
+
+ for (i = 0; i < num_objs; i++) {
+ gen_request("PUT", objs[i], 4096, &failed);
+ }
+
+ checkpoint();
+
+ if (failed.read()) {
+ derr << "ERROR: bucket creation failed" << dendl;
+ goto done;
+ }
+
+ for (i = 0; i < num_objs; i++) {
+ gen_request("GET", objs[i], 4096, NULL);
+ }
+
+ checkpoint();
+
+ for (i = 0; i < num_objs; i++) {
+ gen_request("DELETE", objs[i], 0, NULL);
+ }
+
+ checkpoint();
+
+ for (i = 0; i < num_buckets; i++) {
+ gen_request("DELETE", buckets[i], 0, NULL);
+ }
+
+done:
+ checkpoint();
+
+ m_tp.stop();
+
+ delete[] objs;
+
+ signal_shutdown();
+}
+
+void RGWLoadGenProcess::gen_request(const string& method, const string& resource, int content_length, atomic_t *fail_flag)
+{
+ RGWLoadGenRequest *req = new RGWLoadGenRequest(method, resource, content_length, fail_flag);
+ req->id = ++max_req_id;
+ dout(10) << "allocated request req=" << hex << req << dec << dendl;
+ req_throttle.get(1);
+ req_wq.queue(req);
+}
+
+static void signal_shutdown()
+{
+ if (!disable_signal_fd.read()) {
+ int val = 0;
+ int ret = write(signal_fd[0], (char *)&val, sizeof(val));
+ if (ret < 0) {
+ int err = -errno;
+ derr << "ERROR: " << __func__ << ": write() returned " << cpp_strerror(-err) << dendl;
+ }
+ }
+}
+
+static void wait_shutdown()
+{
+ int val;
+ int r = safe_read_exact(signal_fd[1], &val, sizeof(val));
+ if (r < 0) {
+ derr << "safe_read_exact returned with error" << dendl;
+ }
+}
+
+int signal_fd_init()
+{
+ return socketpair(AF_UNIX, SOCK_STREAM, 0, signal_fd);
+}
+
+static void signal_fd_finalize()
+{
+ close(signal_fd[0]);
+ close(signal_fd[1]);
+}
+
static void handle_sigterm(int signum)
{
dout(1) << __func__ << dendl;
FCGX_ShutdownPending();
- // close the fd, so that accept can't start again.
- pprocess->close_fd();
-
// send a signal to make fcgi's accept(2) wake up. unfortunately the
// initial signal often isn't sufficient because we race with accept's
// check of the flag wet by ShutdownPending() above.
if (signum != SIGUSR1) {
- kill(getpid(), SIGUSR1);
+ signal_shutdown();
// safety net in case we get stuck doing an orderly shutdown.
uint64_t secs = g_ceph_context->_conf->rgw_exit_timeout_secs;
@@ -295,19 +502,19 @@ static int call_log_intent(RGWRados *store, void *ctx, rgw_obj& obj, RGWIntentEv
return rgw_log_intent(store, s, obj, intent);
}
-void RGWProcess::handle_request(RGWRequest *req)
+
+static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWClientIO *client_io, OpsLogSocket *olog)
{
- FCGX_Request *fcgx = &req->fcgx;
- int ret;
- RGWEnv rgw_env;
- RGWFCGX client_io(fcgx);
+ int ret = 0;
+
+ client_io->init(g_ceph_context);
req->log_init();
dout(1) << "====== starting new request req=" << hex << req << dec << " =====" << dendl;
perfcounter->inc(l_rgw_req);
- rgw_env.init(g_ceph_context, fcgx->envp);
+ RGWEnv& rgw_env = client_io->get_env();
struct req_state *s = req->init_state(g_ceph_context, &rgw_env);
s->obj_ctx = store->create_context(s);
@@ -321,7 +528,7 @@ void RGWProcess::handle_request(RGWRequest *req)
int init_error = 0;
bool should_log = false;
RGWRESTMgr *mgr;
- RGWHandler *handler = rest->get_handler(store, s, &client_io, &mgr, &init_error);
+ RGWHandler *handler = rest->get_handler(store, s, client_io, &mgr, &init_error);
if (init_error != 0) {
abort_early(s, NULL, init_error);
goto done;
@@ -389,13 +596,15 @@ void RGWProcess::handle_request(RGWRequest *req)
goto done;
}
- if (s->expect_cont)
- dump_continue(s);
-
req->log(s, "executing");
+ op->pre_exec();
op->execute();
op->complete();
done:
+ int r = client_io->complete_request();
+ if (r < 0) {
+ dout(0) << "ERROR: client_io->complete_request() returned " << r << dendl;
+ }
if (should_log) {
rgw_log_op(store, s, (op ? op->name() : "unknown"), olog);
}
@@ -408,12 +617,87 @@ done:
handler->put_op(op);
rest->put_handler(handler);
store->destroy_context(s->obj_ctx);
- FCGX_Finish_r(fcgx);
dout(1) << "====== req done req=" << hex << req << dec << " http_status=" << http_ret << " ======" << dendl;
+
+ return (ret < 0 ? ret : s->err.ret);
+}
+
+void RGWFCGXProcess::handle_request(RGWRequest *r)
+{
+ RGWFCGXRequest *req = static_cast<RGWFCGXRequest *>(r);
+ FCGX_Request *fcgx = &req->fcgx;
+ RGWFCGX client_io(fcgx);
+
+
+ int ret = process_request(store, rest, req, &client_io, olog);
+ if (ret < 0) {
+ /* we don't really care about return code */
+ dout(20) << "process_request() returned " << ret << dendl;
+ }
+
+ FCGX_Finish_r(fcgx);
+
delete req;
}
+void RGWLoadGenProcess::handle_request(RGWRequest *r)
+{
+ RGWLoadGenRequest *req = static_cast<RGWLoadGenRequest *>(r);
+
+ RGWLoadGenRequestEnv env;
+
+ utime_t tm = ceph_clock_now(NULL);
+
+ env.port = 80;
+ env.content_length = req->content_length;
+ env.content_type = "binary/octet-stream";
+ env.request_method = req->method;
+ env.uri = req->resource;
+ env.set_date(tm);
+ env.sign(access_key);
+
+ RGWLoadGenIO client_io(&env);
+
+ int ret = process_request(store, rest, req, &client_io, olog);
+ if (ret < 0) {
+ /* we don't really care about return code */
+ dout(20) << "process_request() returned " << ret << dendl;
+
+ if (req->fail_flag) {
+ req->fail_flag->inc();
+ }
+ }
+
+ delete req;
+}
+
+
+static int civetweb_callback(struct mg_connection *conn) {
+ struct mg_request_info *req_info = mg_get_request_info(conn);
+ RGWProcessEnv *pe = (RGWProcessEnv *)req_info->user_data;
+ RGWRados *store = pe->store;
+ RGWREST *rest = pe->rest;
+ OpsLogSocket *olog = pe->olog;
+
+ RGWRequest *req = new RGWRequest;
+ RGWMongoose client_io(conn, pe->port);
+
+ client_io.init(g_ceph_context);
+
+
+ int ret = process_request(store, rest, req, &client_io, olog);
+ if (ret < 0) {
+ /* we don't really care about return code */
+ dout(20) << "process_request() returned " << ret << dendl;
+ }
+
+ delete req;
+
+// Mark as processed
+ return 1;
+}
+
#ifdef HAVE_CURL_MULTI_WAIT
static void check_curl()
{
@@ -451,6 +735,219 @@ static RGWRESTMgr *set_logging(RGWRESTMgr *mgr)
return mgr;
}
+
+int RGWFrontendConfig::parse_config(const string& config, map<string, string>& config_map)
+{
+ list<string> config_list;
+ get_str_list(config, " ", config_list);
+
+ list<string>::iterator iter;
+ for (iter = config_list.begin(); iter != config_list.end(); ++iter) {
+ string& entry = *iter;
+ string key;
+ string val;
+
+ if (framework.empty()) {
+ framework = entry;
+ dout(0) << "framework: " << framework << dendl;
+ continue;
+ }
+
+ ssize_t pos = entry.find('=');
+ if (pos < 0) {
+ dout(0) << "framework conf key: " << entry << dendl;
+ config_map[entry] = "";
+ continue;
+ }
+
+ int ret = parse_key_value(entry, key, val);
+ if (ret < 0) {
+ cerr << "ERROR: can't parse " << entry << std::endl;
+ return ret;
+ }
+
+ dout(0) << "framework conf key: " << key << ", val: " << val << dendl;
+ config_map[key] = val;
+ }
+
+ return 0;
+}
+
+
+bool RGWFrontendConfig::get_val(const string& key, const string& def_val, string *out)
+{
+ map<string, string>::iterator iter = config_map.find(key);
+ if (iter == config_map.end()) {
+ *out = def_val;
+ return false;
+ }
+
+ *out = iter->second;
+ return true;
+}
+
+
+bool RGWFrontendConfig::get_val(const string& key, int def_val, int *out)
+{
+ string str;
+ bool found = get_val(key, "", &str);
+ if (!found) {
+ *out = def_val;
+ return false;
+ }
+ string err;
+ *out = strict_strtol(str.c_str(), 10, &err);
+ if (!err.empty()) {
+ cerr << "error parsing int: " << str << ": " << err << std::endl;
+ return -EINVAL;
+ }
+ return 0;
+}
+
+class RGWFrontend {
+public:
+ virtual ~RGWFrontend() {}
+
+ virtual int init() = 0;
+
+ virtual int run() = 0;
+ virtual void stop() = 0;
+ virtual void join() = 0;
+};
+
+class RGWProcessControlThread : public Thread {
+ RGWProcess *pprocess;
+public:
+ RGWProcessControlThread(RGWProcess *_pprocess) : pprocess(_pprocess) {}
+
+ void *entry() {
+ pprocess->run();
+ return NULL;
+ };
+};
+
+class RGWProcessFrontend : public RGWFrontend {
+protected:
+ RGWFrontendConfig *conf;
+ RGWProcess *pprocess;
+ RGWProcessEnv env;
+ RGWProcessControlThread *thread;
+
+public:
+ RGWProcessFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : conf(_conf), pprocess(NULL), env(pe), thread(NULL) {
+ }
+
+ ~RGWProcessFrontend() {
+ delete thread;
+ delete pprocess;
+ }
+
+ int run() {
+ assert(pprocess); /* should have initialized by init() */
+ thread = new RGWProcessControlThread(pprocess);
+ thread->create();
+ return 0;
+ }
+
+ void stop() {
+ pprocess->close_fd();
+ thread->kill(SIGUSR1);
+ }
+
+ void join() {
+ thread->join();
+ }
+};
+
+class RGWFCGXFrontend : public RGWProcessFrontend {
+public:
+ RGWFCGXFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : RGWProcessFrontend(pe, _conf) {}
+
+ int init() {
+ pprocess = new RGWFCGXProcess(g_ceph_context, &env, g_conf->rgw_thread_pool_size, conf);
+ return 0;
+ }
+};
+
+class RGWLoadGenFrontend : public RGWProcessFrontend {
+public:
+ RGWLoadGenFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : RGWProcessFrontend(pe, _conf) {}
+
+ int init() {
+ int num_threads;
+ conf->get_val("num_threads", g_conf->rgw_thread_pool_size, &num_threads);
+ RGWLoadGenProcess *pp = new RGWLoadGenProcess(g_ceph_context, &env, num_threads, conf);
+
+ pprocess = pp;
+
+ string uid;
+ conf->get_val("uid", "", &uid);
+ if (uid.empty()) {
+ derr << "ERROR: uid param must be specified for loadgen frontend" << dendl;
+ return EINVAL;
+ }
+
+ RGWUserInfo user_info;
+ int ret = rgw_get_user_info_by_uid(env.store, uid, user_info, NULL);
+ if (ret < 0) {
+ derr << "ERROR: failed reading user info: uid=" << uid << " ret=" << ret << dendl;
+ return ret;
+ }
+
+ map<string, RGWAccessKey>::iterator aiter = user_info.access_keys.begin();
+ if (aiter == user_info.access_keys.end()) {
+ derr << "ERROR: user has no S3 access keys set" << dendl;
+ return -EINVAL;
+ }
+
+ pp->set_access_key(aiter->second);
+
+ return 0;
+ }
+};
+
+class RGWMongooseFrontend : public RGWFrontend {
+ RGWFrontendConfig *conf;
+ struct mg_context *ctx;
+ RGWProcessEnv env;
+
+public:
+ RGWMongooseFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : conf(_conf), ctx(NULL), env(pe) {
+ }
+
+ int init() {
+ return 0;
+ }
+
+ int run() {
+ char thread_pool_buf[32];
+ snprintf(thread_pool_buf, sizeof(thread_pool_buf), "%d", (int)g_conf->rgw_thread_pool_size);
+ string port_str;
+ conf->get_val("port", "80", &port_str);
+ const char *options[] = {"listening_ports", port_str.c_str(), "enable_keep_alive", "yes", "num_threads", thread_pool_buf, NULL};
+
+ struct mg_callbacks cb;
+ memset((void *)&cb, 0, sizeof(cb));
+ cb.begin_request = civetweb_callback;
+ ctx = mg_start(&cb, &env, (const char **)&options);
+
+ if (!ctx) {
+ return -EIO;
+ }
+
+ return 0;
+ }
+
+ void stop() {
+ if (ctx) {
+ mg_stop(ctx);
+ }
+ }
+
+ void join() {
+ }
+};
+
/*
* start up the RADOS connection and then handle HTTP messages as they come in
*/
@@ -488,11 +985,6 @@ int main(int argc, const char **argv)
check_curl();
if (g_conf->daemonize) {
- if (g_conf->rgw_socket_path.empty() and g_conf->rgw_port.empty()) {
- cerr << "radosgw: must specify 'rgw socket path' or 'rgw port' to run as a daemon" << std::endl;
- exit(1);
- }
-
global_init_daemonize(g_ceph_context, 0);
}
Mutex mutex("main");
@@ -514,7 +1006,7 @@ int main(int argc, const char **argv)
FCGX_Init();
int r = 0;
- RGWRados *store = RGWStoreManager::get_storage(g_ceph_context, true);
+ RGWRados *store = RGWStoreManager::get_storage(g_ceph_context, true, true);
if (!store) {
derr << "Couldn't init storage provider (RADOS)" << dendl;
r = EIO;
@@ -580,27 +1072,113 @@ int main(int argc, const char **argv)
olog->init(g_conf->rgw_ops_log_socket_path);
}
- pprocess = new RGWProcess(g_ceph_context, store, olog, g_conf->rgw_thread_pool_size, &rest);
+ r = signal_fd_init();
+ if (r < 0) {
+ derr << "ERROR: unable to initialize signal fds" << dendl;
+ exit(1);
+ }
init_async_signal_handler();
register_async_signal_handler(SIGHUP, sighup_handler);
register_async_signal_handler(SIGTERM, handle_sigterm);
register_async_signal_handler(SIGINT, handle_sigterm);
register_async_signal_handler(SIGUSR1, handle_sigterm);
-
sighandler_alrm = signal(SIGALRM, godown_alarm);
- pprocess->run();
+ string frontend_frameworks = g_conf->rgw_frontends;
+
+ list<string> frontends;
+
+ get_str_list(g_conf->rgw_frontends, ",", frontends);
+
+ multimap<string, RGWFrontendConfig *> fe_map;
+ list<RGWFrontendConfig *> configs;
+ if (frontends.empty()) {
+ frontends.push_back("fastcgi");
+ }
+ for (list<string>::iterator iter = frontends.begin(); iter != frontends.end(); ++iter) {
+ string& f = *iter;
+
+ RGWFrontendConfig *config = new RGWFrontendConfig(f);
+ int r = config->init();
+ if (r < 0) {
+ cerr << "ERROR: failed to init config: " << f << std::endl;
+ return EINVAL;
+ }
+
+ configs.push_back(config);
+
+ string framework = config->get_framework();
+ fe_map.insert(make_pair<string, RGWFrontendConfig *>(framework, config));
+ }
+
+ list<RGWFrontend *> fes;
+
+ for (multimap<string, RGWFrontendConfig *>::iterator fiter = fe_map.begin(); fiter != fe_map.end(); ++fiter) {
+ RGWFrontendConfig *config = fiter->second;
+ string framework = config->get_framework();
+ RGWFrontend *fe;
+ if (framework == "fastcgi" || framework == "fcgi") {
+ RGWProcessEnv fcgi_pe = { store, &rest, olog, 0 };
+
+ fe = new RGWFCGXFrontend(fcgi_pe, config);
+ } else if (framework == "civetweb" || framework == "mongoose") {
+ string err;
+
+ int port;
+ config->get_val("port", 80, &port);
+
+ RGWProcessEnv env = { store, &rest, olog, port };
+
+ fe = new RGWMongooseFrontend(env, config);
+ } else if (framework == "loadgen") {
+ int port;
+ config->get_val("port", 80, &port);
+
+ RGWProcessEnv env = { store, &rest, olog, port };
+
+ fe = new RGWLoadGenFrontend(env, config);
+ } else {
+ dout(0) << "WARNING: skipping unknown framework: " << framework << dendl;
+ continue;
+ }
+ dout(0) << "starting handler: " << fiter->first << dendl;
+ int r = fe->init();
+ if (r < 0) {
+ derr << "ERROR: failed initializing frontend" << dendl;
+ return -r;
+ }
+ fe->run();
+
+ fes.push_back(fe);
+ }
+
+ wait_shutdown();
+
derr << "shutting down" << dendl;
+ for (list<RGWFrontend *>::iterator liter = fes.begin(); liter != fes.end(); ++liter) {
+ RGWFrontend *fe = *liter;
+ fe->stop();
+ }
+
+ for (list<RGWFrontend *>::iterator liter = fes.begin(); liter != fes.end(); ++liter) {
+ RGWFrontend *fe = *liter;
+ fe->join();
+ delete fe;
+ }
+
+ for (list<RGWFrontendConfig *>::iterator liter = configs.begin(); liter != configs.end(); ++liter) {
+ RGWFrontendConfig *fec = *liter;
+ delete fec;
+ }
+
unregister_async_signal_handler(SIGHUP, sighup_handler);
unregister_async_signal_handler(SIGTERM, handle_sigterm);
unregister_async_signal_handler(SIGINT, handle_sigterm);
unregister_async_signal_handler(SIGUSR1, handle_sigterm);
shutdown_async_signal_handler();
- delete pprocess;
-
if (do_swift) {
swift_finalize();
}
@@ -622,6 +1200,8 @@ int main(int argc, const char **argv)
ceph::crypto::shutdown();
+ signal_fd_finalize();
+
return 0;
}
diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index 1dd6107..4307ce9 100644
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -600,7 +600,7 @@ int RGWMetadataManager::remove_entry(RGWMetadataHandler *handler, string& key, R
rgw_obj obj(bucket, oid);
- ret = store->delete_obj(NULL, obj, objv_tracker);
+ ret = store->delete_system_obj(NULL, obj, objv_tracker);
/* cascading ret into post_modify() */
ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index bd73a23..a713285 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -33,6 +33,8 @@ using ceph::crypto::MD5;
static string mp_ns = RGW_OBJ_NS_MULTIPART;
static string shadow_ns = RGW_OBJ_NS_SHADOW;
+#define MULTIPART_UPLOAD_ID_PREFIX "2/" // must contain a unique char that may not come up in gen_rand_alpha()
+
class MultipartMetaFilter : public RGWAccessListFilter {
public:
MultipartMetaFilter() {}
@@ -395,6 +397,14 @@ static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bu
return ret;
}
+static void rgw_bucket_object_pre_exec(struct req_state *s)
+{
+ if (s->expect_cont)
+ dump_continue(s);
+
+ dump_bucket_from_state(s);
+}
+
int RGWGetObj::verify_permission()
{
obj.init(s->bucket, s->object_str);
@@ -442,28 +452,32 @@ int RGWOp::init_quota()
return 0;
}
- if (s->bucket_info.quota.enabled) {
- bucket_quota = s->bucket_info.quota;
- return 0;
- }
+ RGWUserInfo owner_info;
+ RGWUserInfo *uinfo;
+
if (s->user.user_id == s->bucket_owner.get_id()) {
- if (s->user.bucket_quota.enabled) {
- bucket_quota = s->user.bucket_quota;
- return 0;
- }
+ uinfo = &s->user;
} else {
- RGWUserInfo owner_info;
int r = rgw_get_user_info_by_uid(store, s->bucket_info.owner, owner_info);
if (r < 0)
return r;
+ uinfo = &owner_info;
+ }
- if (owner_info.bucket_quota.enabled) {
- bucket_quota = owner_info.bucket_quota;
- return 0;
- }
+ if (s->bucket_info.quota.enabled) {
+ bucket_quota = s->bucket_info.quota;
+ } else if (uinfo->bucket_quota.enabled) {
+ bucket_quota = uinfo->bucket_quota;
+ } else {
+ bucket_quota = store->region_map.bucket_quota;
+ }
+
+ if (uinfo->user_quota.enabled) {
+ user_quota = uinfo->user_quota;
+ } else {
+ user_quota = store->region_map.user_quota;
}
- bucket_quota = store->region_map.bucket_quota;
return 0;
}
@@ -516,6 +530,11 @@ int RGWOp::read_bucket_cors()
return 0;
}
+/** CORS 6.2.6.
+ * If any of the header field-names is not a ASCII case-insensitive match for
+ * any of the values in list of headers do not set any additional headers and
+ * terminate this set of steps.
+ * */
static void get_cors_response_headers(RGWCORSRule *rule, const char *req_hdrs, string& hdrs, string& exp_hdrs, unsigned *max_age) {
if (req_hdrs) {
list<string> hl;
@@ -533,12 +552,20 @@ static void get_cors_response_headers(RGWCORSRule *rule, const char *req_hdrs, s
*max_age = rule->get_max_age();
}
+/**
+ * Generate the CORS header response
+ *
+ * This is described in the CORS standard, section 6.2.
+ */
bool RGWOp::generate_cors_headers(string& origin, string& method, string& headers, string& exp_headers, unsigned *max_age)
{
+ /* CORS 6.2.1. */
const char *orig = s->info.env->get("HTTP_ORIGIN");
if (!orig) {
return false;
}
+
+ /* Custom: */
origin = orig;
int ret = read_bucket_cors();
if (ret < 0) {
@@ -550,10 +577,12 @@ bool RGWOp::generate_cors_headers(string& origin, string& method, string& header
return false;
}
+ /* CORS 6.2.2. */
RGWCORSRule *rule = bucket_cors.host_name_rule(orig);
if (!rule)
return false;
+ /* CORS 6.2.3. */
const char *req_meth = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
if (!req_meth) {
req_meth = s->info.method;
@@ -561,13 +590,15 @@ bool RGWOp::generate_cors_headers(string& origin, string& method, string& header
if (req_meth)
method = req_meth;
-
+ /* CORS 6.2.5. */
if (!validate_cors_rule_method(rule, req_meth)) {
return false;
}
- const char *req_hdrs = s->info.env->get("HTTP_ACCESS_CONTROL_ALLOW_HEADERS");
+ /* CORS 6.2.4. */
+ const char *req_hdrs = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS");
+ /* CORS 6.2.6. */
get_cors_response_headers(rule, req_hdrs, headers, exp_headers, max_age);
return true;
@@ -769,7 +800,7 @@ class RGWGetObj_CB : public RGWGetDataCB
public:
RGWGetObj_CB(RGWGetObj *_op) : op(_op) {}
virtual ~RGWGetObj_CB() {}
-
+
int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) {
return op->get_data_cb(bl, bl_ofs, bl_len);
}
@@ -790,6 +821,11 @@ int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len)
return send_response_data(bl, bl_ofs, bl_len);
}
+void RGWGetObj::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWGetObj::execute()
{
void *handle = NULL;
@@ -983,6 +1019,11 @@ int RGWStatBucket::verify_permission()
return 0;
}
+void RGWStatBucket::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWStatBucket::execute()
{
RGWUserBuckets buckets;
@@ -1030,6 +1071,11 @@ int RGWListBucket::parse_max_keys()
return 0;
}
+void RGWListBucket::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWListBucket::execute()
{
string no_ns;
@@ -1094,6 +1140,11 @@ static int forward_request_to_master(struct req_state *s, obj_version *objv, RGW
return 0;
}
+void RGWCreateBucket::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWCreateBucket::execute()
{
RGWAccessControlPolicy old_policy(s->cct);
@@ -1179,7 +1230,7 @@ void RGWCreateBucket::execute()
* from a partial create by retrying it. */
ldout(s->cct, 20) << "rgw_create_bucket returned ret=" << ret << " bucket=" << s->bucket << dendl;
- if (ret && ret != -EEXIST)
+ if (ret && ret != -EEXIST)
return;
existed = (ret == -EEXIST);
@@ -1198,8 +1249,12 @@ void RGWCreateBucket::execute()
}
ret = rgw_link_bucket(store, s->user.user_id, s->bucket, info.creation_time, false);
- if (ret && !existed && ret != -EEXIST) /* if it exists (or previously existed), don't remove it! */
- rgw_unlink_bucket(store, s->user.user_id, s->bucket.name);
+ if (ret && !existed && ret != -EEXIST) { /* if it exists (or previously existed), don't remove it! */
+ ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.name);
+ if (ret < 0) {
+ ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl;
+ }
+ }
if (ret == -EEXIST)
ret = -ERR_BUCKET_EXISTS;
@@ -1213,6 +1268,11 @@ int RGWDeleteBucket::verify_permission()
return 0;
}
+void RGWDeleteBucket::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWDeleteBucket::execute()
{
ret = -EINVAL;
@@ -1281,6 +1341,7 @@ class RGWPutObjProcessor_Multipart : public RGWPutObjProcessor_Atomic
string part_num;
RGWMPObj mp;
req_state *s;
+ string upload_id;
protected:
bool immutable_head() { return true; }
@@ -1288,7 +1349,8 @@ protected:
int do_complete(string& etag, time_t *mtime, time_t set_mtime, map<string, bufferlist>& attrs);
public:
- RGWPutObjProcessor_Multipart(uint64_t _p, req_state *_s) : RGWPutObjProcessor_Atomic(_s->bucket, _s->object_str, _p, _s->req_id), s(_s) {}
+ RGWPutObjProcessor_Multipart(const string& bucket_owner, uint64_t _p, req_state *_s) :
+ RGWPutObjProcessor_Atomic(bucket_owner, _s->bucket, _s->object_str, _p, _s->req_id), s(_s) {}
};
int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, void *obj_ctx)
@@ -1296,7 +1358,6 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, void *obj_ctx)
RGWPutObjProcessor::prepare(store, obj_ctx);
string oid = obj_str;
- string upload_id;
upload_id = s->info.args.get("uploadId");
mp.init(oid, upload_id);
@@ -1315,6 +1376,13 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, void *obj_ctx)
return 0;
}
+static bool is_v2_upload_id(const string& upload_id)
+{
+ const char *uid = upload_id.c_str();
+
+ return (strncmp(uid, MULTIPART_UPLOAD_ID_PREFIX, sizeof(MULTIPART_UPLOAD_ID_PREFIX) - 1) == 0);
+}
+
int RGWPutObjProcessor_Multipart::do_complete(string& etag, time_t *mtime, time_t set_mtime, map<string, bufferlist>& attrs)
{
complete_writing_data();
@@ -1322,6 +1390,7 @@ int RGWPutObjProcessor_Multipart::do_complete(string& etag, time_t *mtime, time_
RGWRados::PutObjMetaExtraParams params;
params.set_mtime = set_mtime;
params.mtime = mtime;
+ params.owner = s->owner.get_id();
int r = store->put_obj_meta(obj_ctx, head_obj, s->obj_size, attrs, RGW_OBJ_CATEGORY_MAIN, 0, params);
if (r < 0)
@@ -1330,7 +1399,21 @@ int RGWPutObjProcessor_Multipart::do_complete(string& etag, time_t *mtime, time_
bufferlist bl;
RGWUploadPartInfo info;
string p = "part.";
- p.append(part_num);
+ bool sorted_omap = is_v2_upload_id(upload_id);
+
+ if (sorted_omap) {
+ string err;
+ int part_num_int = strict_strtol(part_num.c_str(), 10, &err);
+ if (!err.empty()) {
+ dout(10) << "bad part number specified: " << part_num << dendl;
+ return -EINVAL;
+ }
+ char buf[32];
+ snprintf(buf, sizeof(buf), "%08d", part_num_int);
+ p.append(buf);
+ } else {
+ p.append(part_num);
+ }
info.num = atoi(part_num.c_str());
info.etag = etag;
info.size = s->obj_size;
@@ -1357,10 +1440,12 @@ RGWPutObjProcessor *RGWPutObj::select_processor()
uint64_t part_size = s->cct->_conf->rgw_obj_stripe_size;
+ const string& bucket_owner = s->bucket_owner.get_id();
+
if (!multipart) {
- processor = new RGWPutObjProcessor_Atomic(s->bucket, s->object_str, part_size, s->req_id);
+ processor = new RGWPutObjProcessor_Atomic(bucket_owner, s->bucket, s->object_str, part_size, s->req_id);
} else {
- processor = new RGWPutObjProcessor_Multipart(part_size, s);
+ processor = new RGWPutObjProcessor_Multipart(bucket_owner, part_size, s);
}
return processor;
@@ -1371,6 +1456,11 @@ void RGWPutObj::dispose_processor(RGWPutObjProcessor *processor)
delete processor;
}
+void RGWPutObj::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWPutObj::execute()
{
RGWPutObjProcessor *processor = NULL;
@@ -1411,7 +1501,8 @@ void RGWPutObj::execute()
if (!chunked_upload) { /* with chunked upload we don't know how big is the upload.
we also check sizes at the end anyway */
- ret = store->check_quota(s->bucket, bucket_quota, s->content_length);
+ ret = store->check_quota(s->bucket_owner.get_id(), s->bucket,
+ user_quota, bucket_quota, s->content_length);
if (ret < 0) {
goto done;
}
@@ -1461,7 +1552,8 @@ void RGWPutObj::execute()
s->obj_size = ofs;
perfcounter->inc(l_rgw_put_b, s->obj_size);
- ret = store->check_quota(s->bucket, bucket_quota, s->obj_size);
+ ret = store->check_quota(s->bucket_owner.get_id(), s->bucket,
+ user_quota, bucket_quota, s->obj_size);
if (ret < 0) {
goto done;
}
@@ -1517,7 +1609,7 @@ RGWPutObjProcessor *RGWPostObj::select_processor()
uint64_t part_size = s->cct->_conf->rgw_obj_stripe_size;
- processor = new RGWPutObjProcessor_Atomic(s->bucket, s->object_str, part_size, s->req_id);
+ processor = new RGWPutObjProcessor_Atomic(s->bucket_owner.get_id(), s->bucket, s->object_str, part_size, s->req_id);
return processor;
}
@@ -1527,6 +1619,11 @@ void RGWPostObj::dispose_processor(RGWPutObjProcessor *processor)
delete processor;
}
+void RGWPostObj::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWPostObj::execute()
{
RGWPutObjProcessor *processor = NULL;
@@ -1536,7 +1633,7 @@ void RGWPostObj::execute()
bufferlist bl, aclbl;
int len = 0;
- // read in the data from the POST form
+ // read in the data from the POST form
ret = get_params();
if (ret < 0)
goto done;
@@ -1627,6 +1724,11 @@ int RGWPutMetadata::verify_permission()
return 0;
}
+void RGWPutMetadata::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWPutMetadata::execute()
{
const char *meta_prefix = RGW_ATTR_META_PREFIX;
@@ -1681,10 +1783,46 @@ void RGWPutMetadata::execute()
if (s->object) {
ret = store->set_attrs(s->obj_ctx, obj, attrs, &rmattrs, ptracker);
} else {
- ret = rgw_bucket_set_attrs(store, obj.bucket, attrs, &rmattrs, ptracker);
+ ret = rgw_bucket_set_attrs(store, s->bucket_info, attrs, &rmattrs, ptracker);
+ }
+}
+
+int RGWSetTempUrl::verify_permission()
+{
+ if (s->perm_mask != RGW_PERM_FULL_CONTROL)
+ return -EACCES;
+
+ return 0;
+}
+
+void RGWSetTempUrl::execute()
+{
+ ret = get_params();
+ if (ret < 0)
+ return;
+
+ RGWUserAdminOpState user_op;
+ user_op.set_user_id(s->user.user_id);
+ map<int, string>::iterator iter;
+ for (iter = temp_url_keys.begin(); iter != temp_url_keys.end(); ++iter) {
+ user_op.set_temp_url_key(iter->second, iter->first);
+ }
+
+ RGWUser user;
+ ret = user.init(store, user_op);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: could not init user ret=" << ret << dendl;
+ return;
+ }
+ string err_msg;
+ ret = user.modify(user_op, &err_msg);
+ if (ret < 0) {
+ ldout(store->ctx(), 10) << "user.modify() returned " << ret << ": " << err_msg << dendl;
+ return;
}
}
+
int RGWDeleteObj::verify_permission()
{
if (!verify_bucket_permission(s, RGW_PERM_WRITE))
@@ -1693,13 +1831,18 @@ int RGWDeleteObj::verify_permission()
return 0;
}
+void RGWDeleteObj::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWDeleteObj::execute()
{
ret = -EINVAL;
rgw_obj obj(s->bucket, s->object_str);
if (s->object) {
store->set_atomic(s->obj_ctx, obj);
- ret = store->delete_obj(s->obj_ctx, obj);
+ ret = store->delete_obj(s->obj_ctx, s->bucket_owner.get_id(), obj);
}
}
@@ -1845,6 +1988,11 @@ void RGWCopyObj::progress_cb(off_t ofs)
last_ofs = ofs;
}
+void RGWCopyObj::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWCopyObj::execute()
{
rgw_obj src_obj, dst_obj;
@@ -1895,13 +2043,18 @@ int RGWGetACLs::verify_permission()
return 0;
}
+void RGWGetACLs::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWGetACLs::execute()
{
stringstream ss;
RGWAccessControlPolicy *acl = (s->object ? s->object_acl : s->bucket_acl);
RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(acl);
s3policy->to_xml(ss);
- acls = ss.str();
+ acls = ss.str();
}
@@ -1920,6 +2073,11 @@ int RGWPutACLs::verify_permission()
return 0;
}
+void RGWPutACLs::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWPutACLs::execute()
{
bufferlist bl;
@@ -2000,7 +2158,7 @@ void RGWPutACLs::execute()
if (s->object) {
ret = store->set_attrs(s->obj_ctx, obj, attrs, NULL, ptracker);
} else {
- ret = rgw_bucket_set_attrs(store, obj.bucket, attrs, NULL, ptracker);
+ ret = rgw_bucket_set_attrs(store, s->bucket_info, attrs, NULL, ptracker);
}
}
@@ -2119,7 +2277,7 @@ void RGWOptionsCORS::execute()
origin = s->info.env->get("HTTP_ORIGIN");
if (!origin) {
- dout(0) <<
+ dout(0) <<
"Preflight request without mandatory Origin header"
<< dendl;
ret = -EINVAL;
@@ -2127,7 +2285,7 @@ void RGWOptionsCORS::execute()
}
req_meth = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
if (!req_meth) {
- dout(0) <<
+ dout(0) <<
"Preflight request without mandatory Access-control-request-method header"
<< dendl;
ret = -EINVAL;
@@ -2138,7 +2296,7 @@ void RGWOptionsCORS::execute()
ret = -ENOENT;
return;
}
- req_hdrs = s->info.env->get("HTTP_ACCESS_CONTROL_ALLOW_HEADERS");
+ req_hdrs = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS");
ret = validate_cors_request(&bucket_cors);
if (!rule) {
origin = req_meth = NULL;
@@ -2155,6 +2313,11 @@ int RGWInitMultipart::verify_permission()
return 0;
}
+void RGWInitMultipart::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWInitMultipart::execute()
{
bufferlist aclbl;
@@ -2183,7 +2346,8 @@ void RGWInitMultipart::execute()
do {
char buf[33];
gen_rand_alphanumeric(s->cct, buf, sizeof(buf) - 1);
- upload_id = buf;
+ upload_id = "2/"; /* v2 upload id */
+ upload_id.append(buf);
string tmp_obj_name;
RGWMPObj mp(s->object_str, upload_id);
@@ -2191,12 +2355,12 @@ void RGWInitMultipart::execute()
obj.init_ns(s->bucket, tmp_obj_name, mp_ns);
// the meta object will be indexed with 0 size, we c
- ret = store->put_obj_meta(s->obj_ctx, obj, 0, NULL, attrs, RGW_OBJ_CATEGORY_MULTIMETA, PUT_OBJ_CREATE_EXCL);
+ ret = store->put_obj_meta(s->obj_ctx, obj, 0, NULL, attrs, RGW_OBJ_CATEGORY_MULTIMETA, PUT_OBJ_CREATE_EXCL, s->owner.get_id());
} while (ret == -EEXIST);
}
-static int get_multiparts_info(RGWRados *store, struct req_state *s, string& meta_oid, map<uint32_t, RGWUploadPartInfo>& parts,
- RGWAccessControlPolicy& policy, map<string, bufferlist>& attrs)
+static int get_multipart_info(RGWRados *store, struct req_state *s, string& meta_oid,
+ RGWAccessControlPolicy *policy, map<string, bufferlist>& attrs)
{
map<string, bufferlist> parts_map;
map<string, bufferlist>::iterator iter;
@@ -2209,27 +2373,67 @@ static int get_multiparts_info(RGWRados *store, struct req_state *s, string& met
if (ret < 0)
return ret;
- ret = store->omap_get_all(obj, header, parts_map);
- if (ret < 0)
- return ret;
-
- for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
- string name = iter->first;
- if (name.compare(RGW_ATTR_ACL) == 0) {
- bufferlist& bl = iter->second;
- bufferlist::iterator bli = bl.begin();
- try {
- ::decode(policy, bli);
- } catch (buffer::error& err) {
- ldout(s->cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
- return -EIO;
+ if (policy) {
+ for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+ string name = iter->first;
+ if (name.compare(RGW_ATTR_ACL) == 0) {
+ bufferlist& bl = iter->second;
+ bufferlist::iterator bli = bl.begin();
+ try {
+ ::decode(*policy, bli);
+ } catch (buffer::error& err) {
+ ldout(s->cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+ return -EIO;
+ }
+ break;
}
- break;
}
}
+ return 0;
+}
- for (iter = parts_map.begin(); iter != parts_map.end(); ++iter) {
+static int list_multipart_parts(RGWRados *store, struct req_state *s,
+ const string& upload_id,
+ string& meta_oid, int num_parts,
+ int marker, map<uint32_t, RGWUploadPartInfo>& parts,
+ int *next_marker, bool *truncated,
+ bool assume_unsorted = false)
+{
+ map<string, bufferlist> parts_map;
+ map<string, bufferlist>::iterator iter;
+ bufferlist header;
+
+ rgw_obj obj;
+ obj.init_ns(s->bucket, meta_oid, mp_ns);
+
+ bool sorted_omap = is_v2_upload_id(upload_id) && !assume_unsorted;
+
+ int ret;
+
+ parts.clear();
+
+ if (sorted_omap) {
+ string p;
+ p = "part.";
+ char buf[32];
+
+ snprintf(buf, sizeof(buf), "%08d", marker);
+ p.append(buf);
+
+ ret = store->omap_get_vals(obj, header, p, num_parts + 1, parts_map);
+ } else {
+ ret = store->omap_get_all(obj, header, parts_map);
+ }
+ if (ret < 0)
+ return ret;
+
+ int i;
+ int last_num = 0;
+
+ uint32_t expected_next = marker + 1;
+
+ for (i = 0, iter = parts_map.begin(); (i < num_parts || !sorted_omap) && iter != parts_map.end(); ++iter, ++i) {
bufferlist& bl = iter->second;
bufferlist::iterator bli = bl.begin();
RGWUploadPartInfo info;
@@ -2239,8 +2443,48 @@ static int get_multiparts_info(RGWRados *store, struct req_state *s, string& met
ldout(s->cct, 0) << "ERROR: could not part info, caught buffer::error" << dendl;
return -EIO;
}
- parts[info.num] = info;
+ if (sorted_omap) {
+ if (info.num != expected_next) {
+ /* ouch, we expected a specific part num here, but we got a different one. Either
+ * a part is missing, or it could be a case of mixed rgw versions working on the same
+ * upload, where one gateway doesn't support correctly sorted omap keys for multipart
+ * upload just assume data is unsorted.
+ */
+ return list_multipart_parts(store, s, upload_id, meta_oid, num_parts, marker, parts, next_marker, truncated, true);
+ }
+ expected_next++;
+ }
+ if (sorted_omap ||
+ (int)info.num > marker) {
+ parts[info.num] = info;
+ last_num = info.num;
+ }
+ }
+
+ if (sorted_omap) {
+ if (truncated)
+ *truncated = (iter != parts_map.end());
+ } else {
+ /* rebuild a map with only num_parts entries */
+
+ map<uint32_t, RGWUploadPartInfo> new_parts;
+ map<uint32_t, RGWUploadPartInfo>::iterator piter;
+
+ for (i = 0, piter = parts.begin(); i < num_parts && piter != parts.end(); ++i, ++piter) {
+ new_parts[piter->first] = piter->second;
+ last_num = piter->first;
+ }
+
+ if (truncated)
+ *truncated = (piter != parts.end());
+
+ parts.swap(new_parts);
}
+
+ if (next_marker) {
+ *next_marker = last_num;
+ }
+
return 0;
}
@@ -2252,6 +2496,11 @@ int RGWCompleteMultipart::verify_permission()
return 0;
}
+void RGWCompleteMultipart::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWCompleteMultipart::execute()
{
RGWMultiCompleteUpload *parts;
@@ -2260,7 +2509,6 @@ void RGWCompleteMultipart::execute()
string meta_oid;
map<uint32_t, RGWUploadPartInfo> obj_parts;
map<uint32_t, RGWUploadPartInfo>::iterator obj_iter;
- RGWAccessControlPolicy policy(s->cct);
map<string, bufferlist> attrs;
off_t ofs = 0;
MD5 hash;
@@ -2297,44 +2545,81 @@ void RGWCompleteMultipart::execute()
return;
}
- // ensure that each part if of the minimum size
- for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
- if ((obj_iter->second).size < min_part_size) {
- ret = -ERR_TOO_SMALL;
- return;
- }
- }
-
mp.init(s->object_str, upload_id);
meta_oid = mp.get_meta();
- ret = get_multiparts_info(store, s, meta_oid, obj_parts, policy, attrs);
- if (ret == -ENOENT)
- ret = -ERR_NO_SUCH_UPLOAD;
- if (parts->parts.size() != obj_parts.size())
- ret = -ERR_INVALID_PART;
- if (ret < 0)
- return;
+ int total_parts = 0;
+ int handled_parts = 0;
+ int max_parts = 1000;
+ int marker = 0;
+ bool truncated;
- for (iter = parts->parts.begin(), obj_iter = obj_parts.begin();
- iter != parts->parts.end() && obj_iter != obj_parts.end();
- ++iter, ++obj_iter) {
- char etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
- if (iter->first != (int)obj_iter->first) {
- ldout(s->cct, 0) << "NOTICE: parts num mismatch: next requested: " << iter->first << " next uploaded: " << obj_iter->first << dendl;
- ret = -ERR_INVALID_PART;
- return;
+ uint64_t min_part_size = s->cct->_conf->rgw_multipart_min_part_size;
+
+ list<string> remove_objs; /* objects to be removed from index listing */
+
+ iter = parts->parts.begin();
+
+ do {
+ ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, marker, obj_parts, &marker, &truncated);
+ if (ret == -ENOENT) {
+ ret = -ERR_NO_SUCH_UPLOAD;
}
- string part_etag = rgw_string_unquote(iter->second);
- if (part_etag.compare(obj_iter->second.etag) != 0) {
- ldout(s->cct, 0) << "NOTICE: etag mismatch: part: " << iter->first << " etag: " << iter->second << dendl;
+ if (ret < 0)
+ return;
+
+ total_parts += obj_parts.size();
+ if (!truncated && total_parts != (int)parts->parts.size()) {
ret = -ERR_INVALID_PART;
return;
}
- hex_to_buf(obj_iter->second.etag.c_str(), etag, CEPH_CRYPTO_MD5_DIGESTSIZE);
- hash.Update((const byte *)etag, sizeof(etag));
- }
+ for (obj_iter = obj_parts.begin(); iter != parts->parts.end() && obj_iter != obj_parts.end(); ++iter, ++obj_iter, ++handled_parts) {
+ uint64_t part_size = obj_iter->second.size;
+ if (handled_parts < (int)parts->parts.size() - 1 &&
+ part_size < min_part_size) {
+ ret = -ERR_TOO_SMALL;
+ return;
+ }
+
+ char etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
+ if (iter->first != (int)obj_iter->first) {
+ ldout(s->cct, 0) << "NOTICE: parts num mismatch: next requested: " << iter->first << " next uploaded: " << obj_iter->first << dendl;
+ ret = -ERR_INVALID_PART;
+ return;
+ }
+ string part_etag = rgw_string_unquote(iter->second);
+ if (part_etag.compare(obj_iter->second.etag) != 0) {
+ ldout(s->cct, 0) << "NOTICE: etag mismatch: part: " << iter->first << " etag: " << iter->second << dendl;
+ ret = -ERR_INVALID_PART;
+ return;
+ }
+
+ hex_to_buf(obj_iter->second.etag.c_str(), etag, CEPH_CRYPTO_MD5_DIGESTSIZE);
+ hash.Update((const byte *)etag, sizeof(etag));
+
+ RGWUploadPartInfo& obj_part = obj_iter->second;
+
+ /* update manifest for part */
+ string oid = mp.get_part(obj_iter->second.num);
+ rgw_obj src_obj;
+ src_obj.init_ns(s->bucket, oid, mp_ns);
+
+ if (obj_part.manifest.empty()) {
+ RGWObjManifestPart& part = manifest.objs[ofs];
+
+ part.loc = src_obj;
+ part.loc_ofs = 0;
+ part.size = part_size;
+ } else {
+ manifest.append(obj_part.manifest);
+ }
+
+ remove_objs.push_back(src_obj.object);
+
+ ofs += obj_part.size;
+ }
+ } while (truncated);
hash.Final((byte *)final_etag);
buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str);
@@ -2348,29 +2633,6 @@ void RGWCompleteMultipart::execute()
target_obj.init(s->bucket, s->object_str);
- list<string> remove_objs; /* objects to be removed from index listing */
-
- for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
- RGWUploadPartInfo& obj_part = obj_iter->second;
- string oid = mp.get_part(obj_iter->second.num);
- rgw_obj src_obj;
- src_obj.init_ns(s->bucket, oid, mp_ns);
-
- if (obj_part.manifest.empty()) {
- RGWObjManifestPart& part = manifest.objs[ofs];
-
- part.loc = src_obj;
- part.loc_ofs = 0;
- part.size = obj_iter->second.size;
- } else {
- manifest.append(obj_part.manifest);
- }
-
- remove_objs.push_back(src_obj.object);
-
- ofs += obj_part.size;
- }
-
manifest.obj_size = ofs;
store->set_atomic(s->obj_ctx, target_obj);
@@ -2381,6 +2643,7 @@ void RGWCompleteMultipart::execute()
extra_params.remove_objs = &remove_objs;
extra_params.ptag = &s->req_id; /* use req_id as operation tag */
+ extra_params.owner = s->owner.get_id();
ret = store->put_obj_meta(s->obj_ctx, target_obj, ofs, attrs,
RGW_OBJ_CATEGORY_MAIN, PUT_OBJ_CREATE,
@@ -2390,7 +2653,7 @@ void RGWCompleteMultipart::execute()
// remove the upload obj
meta_obj.init_ns(s->bucket, meta_oid, mp_ns);
- store->delete_obj(s->obj_ctx, meta_obj);
+ store->delete_obj(s->obj_ctx, s->bucket_owner.get_id(), meta_obj);
}
int RGWAbortMultipart::verify_permission()
@@ -2401,6 +2664,11 @@ int RGWAbortMultipart::verify_permission()
return 0;
}
+void RGWAbortMultipart::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWAbortMultipart::execute()
{
ret = -EINVAL;
@@ -2409,45 +2677,56 @@ void RGWAbortMultipart::execute()
upload_id = s->info.args.get("uploadId");
map<uint32_t, RGWUploadPartInfo> obj_parts;
map<uint32_t, RGWUploadPartInfo>::iterator obj_iter;
- RGWAccessControlPolicy policy(s->cct);
map<string, bufferlist> attrs;
rgw_obj meta_obj;
RGWMPObj mp;
+ const string& owner = s->bucket_owner.get_id();
if (upload_id.empty() || s->object_str.empty())
return;
- mp.init(s->object_str, upload_id);
+ mp.init(s->object_str, upload_id);
meta_oid = mp.get_meta();
- ret = get_multiparts_info(store, s, meta_oid, obj_parts, policy, attrs);
+ ret = get_multipart_info(store, s, meta_oid, NULL, attrs);
if (ret < 0)
return;
- for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
- RGWUploadPartInfo& obj_part = obj_iter->second;
+ bool truncated;
+ int marker = 0;
+ int max_parts = 1000;
- if (obj_part.manifest.empty()) {
- string oid = mp.get_part(obj_iter->second.num);
- rgw_obj obj;
- obj.init_ns(s->bucket, oid, mp_ns);
- ret = store->delete_obj(s->obj_ctx, obj);
- if (ret < 0 && ret != -ENOENT)
- return;
- } else {
- RGWObjManifest& manifest = obj_part.manifest;
- map<uint64_t, RGWObjManifestPart>::iterator oiter;
- for (oiter = manifest.objs.begin(); oiter != manifest.objs.end(); ++oiter) {
- RGWObjManifestPart& part = oiter->second;
- ret = store->delete_obj(s->obj_ctx, part.loc);
+ do {
+ ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, marker, obj_parts, &marker, &truncated);
+ if (ret < 0)
+ return;
+
+ for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
+ RGWUploadPartInfo& obj_part = obj_iter->second;
+
+ if (obj_part.manifest.empty()) {
+ string oid = mp.get_part(obj_iter->second.num);
+ rgw_obj obj;
+ obj.init_ns(s->bucket, oid, mp_ns);
+ ret = store->delete_obj(s->obj_ctx, owner, obj);
if (ret < 0 && ret != -ENOENT)
return;
+ } else {
+ RGWObjManifest& manifest = obj_part.manifest;
+ map<uint64_t, RGWObjManifestPart>::iterator oiter;
+ for (oiter = manifest.objs.begin(); oiter != manifest.objs.end(); ++oiter) {
+ RGWObjManifestPart& part = oiter->second;
+ ret = store->delete_obj(s->obj_ctx, owner, part.loc);
+ if (ret < 0 && ret != -ENOENT)
+ return;
+ }
}
}
- }
+ } while (truncated);
+
// and also remove the metadata obj
meta_obj.init_ns(s->bucket, meta_oid, mp_ns);
- ret = store->delete_obj(s->obj_ctx, meta_obj);
+ ret = store->delete_obj(s->obj_ctx, owner, meta_obj);
if (ret == -ENOENT) {
ret = -ERR_NO_SUCH_BUCKET;
}
@@ -2461,6 +2740,11 @@ int RGWListMultipart::verify_permission()
return 0;
}
+void RGWListMultipart::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWListMultipart::execute()
{
map<string, bufferlist> xattrs;
@@ -2474,7 +2758,11 @@ void RGWListMultipart::execute()
mp.init(s->object_str, upload_id);
meta_oid = mp.get_meta();
- ret = get_multiparts_info(store, s, meta_oid, parts, policy, xattrs);
+ ret = get_multipart_info(store, s, meta_oid, &policy, xattrs);
+ if (ret < 0)
+ return;
+
+ ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, marker, parts, NULL, &truncated);
}
int RGWListBucketMultiparts::verify_permission()
@@ -2485,6 +2773,11 @@ int RGWListBucketMultiparts::verify_permission()
return 0;
}
+void RGWListBucketMultiparts::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWListBucketMultiparts::execute()
{
vector<RGWObjEnt> objs;
@@ -2531,6 +2824,11 @@ int RGWDeleteMultiObj::verify_permission()
return 0;
}
+void RGWDeleteMultiObj::pre_exec()
+{
+ rgw_bucket_object_pre_exec(s);
+}
+
void RGWDeleteMultiObj::execute()
{
RGWMultiDelDelete *multi_delete;
@@ -2579,7 +2877,10 @@ void RGWDeleteMultiObj::execute()
rgw_obj obj(bucket,(*iter));
store->set_atomic(s->obj_ctx, obj);
- ret = store->delete_obj(s->obj_ctx, obj);
+ ret = store->delete_obj(s->obj_ctx, s->bucket_owner.get_id(), obj);
+ if (ret == -ENOENT) {
+ ret = 0;
+ }
result = make_pair(*iter, ret);
send_partial_response(result);
@@ -2611,14 +2912,6 @@ int RGWHandler::init(RGWRados *_store, struct req_state *_s, RGWClientIO *cio)
store = _store;
s = _s;
- if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
- const char *p;
- const char **envp = cio->envp();
- for (int i=0; (p = envp[i]); ++i) {
- ldout(s->cct, 20) << p << dendl;
- }
- }
-
return 0;
}
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 41ee2e8..b85f0e2 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -38,6 +38,7 @@ protected:
RGWCORSConfiguration bucket_cors;
bool cors_exist;
RGWQuotaInfo bucket_quota;
+ RGWQuotaInfo user_quota;
virtual int init_quota();
public:
@@ -64,9 +65,12 @@ public:
virtual bool prefetch_data() { return false; }
virtual int verify_permission() = 0;
virtual int verify_op_mask();
+ virtual void pre_exec() {}
virtual void execute() = 0;
virtual void send_response() {}
- virtual void complete() { send_response(); }
+ virtual void complete() {
+ send_response();
+ }
virtual const string name() = 0;
virtual uint32_t op_mask() { return 0; }
@@ -123,6 +127,7 @@ public:
this->get_data = get_data;
}
int verify_permission();
+ void pre_exec();
void execute();
int read_user_manifest_part(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs);
int iterate_user_manifest_parts(rgw_bucket& bucket, string& obj_prefix, RGWAccessControlPolicy *bucket_policy,
@@ -217,6 +222,7 @@ public:
is_truncated = false;
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -246,6 +252,7 @@ public:
~RGWStatBucket() {}
int verify_permission();
+ void pre_exec();
void execute();
virtual void send_response() = 0;
@@ -268,6 +275,7 @@ public:
RGWCreateBucket() : ret(0) {}
int verify_permission();
+ void pre_exec();
void execute();
virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
RGWOp::init(store, s, h);
@@ -289,6 +297,7 @@ public:
RGWDeleteBucket() : ret(0) {}
int verify_permission();
+ void pre_exec();
void execute();
virtual void send_response() = 0;
@@ -331,6 +340,7 @@ public:
void dispose_processor(RGWPutObjProcessor *processor);
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -370,6 +380,7 @@ public:
}
int verify_permission();
+ void pre_exec();
void execute();
RGWPutObjProcessor *select_processor();
@@ -402,6 +413,7 @@ public:
policy.set_ctx(s->cct);
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -410,6 +422,21 @@ public:
virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
};
+class RGWSetTempUrl : public RGWOp {
+protected:
+ int ret;
+ map<int, string> temp_url_keys;
+public:
+ RGWSetTempUrl() : ret(0) {}
+
+ int verify_permission();
+ void execute();
+
+ virtual int get_params() = 0;
+ virtual void send_response() = 0;
+ virtual const string name() { return "set_temp_url"; }
+};
+
class RGWDeleteObj : public RGWOp {
protected:
int ret;
@@ -418,6 +445,7 @@ public:
RGWDeleteObj() : ret(0) {}
int verify_permission();
+ void pre_exec();
void execute();
virtual void send_response() = 0;
@@ -486,6 +514,7 @@ public:
dest_policy.set_ctx(s->cct);
}
int verify_permission();
+ void pre_exec();
void execute();
void progress_cb(off_t ofs);
@@ -506,6 +535,7 @@ public:
RGWGetACLs() : ret(0) {}
int verify_permission();
+ void pre_exec();
void execute();
virtual void send_response() = 0;
@@ -530,6 +560,7 @@ public:
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_policy_from_state(RGWRados *store, struct req_state *s, stringstream& ss) { return 0; }
@@ -625,6 +656,7 @@ public:
policy.set_ctx(s->cct);
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -640,20 +672,19 @@ protected:
string etag;
char *data;
int len;
- uint64_t min_part_size;
public:
RGWCompleteMultipart() {
ret = 0;
data = NULL;
len = 0;
- min_part_size = RGW_MIN_MULTIPART_SIZE;
}
virtual ~RGWCompleteMultipart() {
free(data);
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -670,6 +701,7 @@ public:
RGWAbortMultipart() : ret(0) {}
int verify_permission();
+ void pre_exec();
void execute();
virtual void send_response() = 0;
@@ -685,12 +717,14 @@ protected:
int max_parts;
int marker;
RGWAccessControlPolicy policy;
+ bool truncated;
public:
RGWListMultipart() {
ret = 0;
max_parts = 1000;
marker = 0;
+ truncated = false;
}
virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
@@ -698,6 +732,7 @@ public:
policy = RGWAccessControlPolicy(s->cct);
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -801,6 +836,7 @@ public:
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
@@ -831,6 +867,7 @@ public:
status_dumped = false;
}
int verify_permission();
+ void pre_exec();
void execute();
virtual int get_params() = 0;
diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index 66609ca..eab7e37 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -16,46 +16,90 @@
#include "include/utime.h"
#include "common/lru_map.h"
#include "common/RefCountedObj.h"
+#include "common/Thread.h"
+#include "common/Mutex.h"
+#include "common/RWLock.h"
#include "rgw_common.h"
#include "rgw_rados.h"
#include "rgw_quota.h"
+#include "rgw_bucket.h"
+#include "rgw_user.h"
#define dout_subsys ceph_subsys_rgw
-struct RGWQuotaBucketStats {
- RGWBucketStats stats;
+struct RGWQuotaCacheStats {
+ RGWStorageStats stats;
utime_t expiration;
utime_t async_refresh_time;
};
-class RGWBucketStatsCache {
+template<class T>
+class RGWQuotaCache {
+protected:
RGWRados *store;
- lru_map<rgw_bucket, RGWQuotaBucketStats> stats_map;
+ lru_map<T, RGWQuotaCacheStats> stats_map;
RefCountedWaitObject *async_refcount;
- int fetch_bucket_totals(rgw_bucket& bucket, RGWBucketStats& stats);
+ class StatsAsyncTestSet : public lru_map<T, RGWQuotaCacheStats>::UpdateContext {
+ int objs_delta;
+ uint64_t added_bytes;
+ uint64_t removed_bytes;
+ public:
+ StatsAsyncTestSet() {}
+ bool update(RGWQuotaCacheStats *entry) {
+ if (entry->async_refresh_time.sec() == 0)
+ return false;
+ entry->async_refresh_time = utime_t(0, 0);
+
+ return true;
+ }
+ };
+
+ virtual int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats) = 0;
+
+ virtual bool map_find(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) = 0;
+
+ virtual bool map_find_and_update(const string& user, rgw_bucket& bucket, typename lru_map<T, RGWQuotaCacheStats>::UpdateContext *ctx) = 0;
+ virtual void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) = 0;
+
+ virtual void data_modified(const string& user, rgw_bucket& bucket) {}
public:
- RGWBucketStatsCache(RGWRados *_store) : store(_store), stats_map(store->ctx()->_conf->rgw_bucket_quota_cache_size) {
+ RGWQuotaCache(RGWRados *_store, int size) : store(_store), stats_map(size) {
async_refcount = new RefCountedWaitObject;
}
- ~RGWBucketStatsCache() {
+ virtual ~RGWQuotaCache() {
async_refcount->put_wait(); /* wait for all pending async requests to complete */
}
- int get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& stats, RGWQuotaInfo& quota);
- void adjust_bucket_stats(rgw_bucket& bucket, int objs_delta, uint64_t added_bytes, uint64_t removed_bytes);
+ int get_stats(const string& user, rgw_bucket& bucket, RGWStorageStats& stats, RGWQuotaInfo& quota);
+ void adjust_stats(const string& user, rgw_bucket& bucket, int objs_delta, uint64_t added_bytes, uint64_t removed_bytes);
+
+ virtual bool can_use_cached_stats(RGWQuotaInfo& quota, RGWStorageStats& stats);
+
+ void set_stats(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs, RGWStorageStats& stats);
+ int async_refresh(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs);
+ void async_refresh_response(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
- bool can_use_cached_stats(RGWQuotaInfo& quota, RGWBucketStats& stats);
+ class AsyncRefreshHandler {
+ protected:
+ RGWRados *store;
+ RGWQuotaCache<T> *cache;
+ public:
+ AsyncRefreshHandler(RGWRados *_store, RGWQuotaCache<T> *_cache) : store(_store), cache(_cache) {}
+ virtual ~AsyncRefreshHandler() {}
- void set_stats(rgw_bucket& bucket, RGWQuotaBucketStats& qs, RGWBucketStats& stats);
- int async_refresh(rgw_bucket& bucket, RGWQuotaBucketStats& qs);
- void async_refresh_response(rgw_bucket& bucket, RGWBucketStats& stats);
+ virtual int init_fetch() = 0;
+ virtual void drop_reference() = 0;
+ };
+
+ virtual AsyncRefreshHandler *allocate_refresh_handler(const string& user, rgw_bucket& bucket) = 0;
};
-bool RGWBucketStatsCache::can_use_cached_stats(RGWQuotaInfo& quota, RGWBucketStats& cached_stats)
+template<class T>
+bool RGWQuotaCache<T>::can_use_cached_stats(RGWQuotaInfo& quota, RGWStorageStats& cached_stats)
{
if (quota.max_size_kb >= 0) {
if (quota.max_size_soft_threshold < 0) {
@@ -84,49 +128,136 @@ bool RGWBucketStatsCache::can_use_cached_stats(RGWQuotaInfo& quota, RGWBucketSta
return true;
}
-int RGWBucketStatsCache::fetch_bucket_totals(rgw_bucket& bucket, RGWBucketStats& stats)
+template<class T>
+int RGWQuotaCache<T>::async_refresh(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs)
{
- RGWBucketInfo bucket_info;
+ /* protect against multiple updates */
+ StatsAsyncTestSet test_update;
+ if (!map_find_and_update(user, bucket, &test_update)) {
+ /* most likely we just raced with another update */
+ return 0;
+ }
- uint64_t bucket_ver;
- uint64_t master_ver;
+ async_refcount->get();
- map<RGWObjCategory, RGWBucketStats> bucket_stats;
- int r = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, bucket_stats, NULL);
- if (r < 0) {
- ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
- return r;
+
+ AsyncRefreshHandler *handler = allocate_refresh_handler(user, bucket);
+
+ int ret = handler->init_fetch();
+ if (ret < 0) {
+ async_refcount->put();
+ handler->drop_reference();
+ return ret;
}
- stats = RGWBucketStats();
+ return 0;
+}
- map<RGWObjCategory, RGWBucketStats>::iterator iter;
- for (iter = bucket_stats.begin(); iter != bucket_stats.end(); ++iter) {
- RGWBucketStats& s = iter->second;
- stats.num_kb += s.num_kb;
- stats.num_kb_rounded += s.num_kb_rounded;
- stats.num_objects += s.num_objects;
+template<class T>
+void RGWQuotaCache<T>::async_refresh_response(const string& user, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+ ldout(store->ctx(), 20) << "async stats refresh response for bucket=" << bucket << dendl;
+
+ RGWQuotaCacheStats qs;
+
+ map_find(user, bucket, qs);
+
+ set_stats(user, bucket, qs, stats);
+
+ async_refcount->put();
+}
+
+template<class T>
+void RGWQuotaCache<T>::set_stats(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs, RGWStorageStats& stats)
+{
+ qs.stats = stats;
+ qs.expiration = ceph_clock_now(store->ctx());
+ qs.async_refresh_time = qs.expiration;
+ qs.expiration += store->ctx()->_conf->rgw_bucket_quota_ttl;
+ qs.async_refresh_time += store->ctx()->_conf->rgw_bucket_quota_ttl / 2;
+
+ map_add(user, bucket, qs);
+}
+
+template<class T>
+int RGWQuotaCache<T>::get_stats(const string& user, rgw_bucket& bucket, RGWStorageStats& stats, RGWQuotaInfo& quota) {
+ RGWQuotaCacheStats qs;
+ utime_t now = ceph_clock_now(store->ctx());
+ if (map_find(user, bucket, qs)) {
+ if (qs.async_refresh_time.sec() > 0 && now >= qs.async_refresh_time) {
+ int r = async_refresh(user, bucket, qs);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "ERROR: quota async refresh returned ret=" << r << dendl;
+
+ /* continue processing, might be a transient error, async refresh is just optimization */
+ }
+ }
+
+ if (can_use_cached_stats(quota, qs.stats) && qs.expiration > ceph_clock_now(store->ctx())) {
+ stats = qs.stats;
+ return 0;
+ }
}
+ int ret = fetch_stats_from_storage(user, bucket, stats);
+ if (ret < 0 && ret != -ENOENT)
+ return ret;
+
+ set_stats(user, bucket, qs, stats);
+
return 0;
}
-class AsyncRefreshHandler : public RGWGetBucketStats_CB {
- RGWRados *store;
- RGWBucketStatsCache *cache;
+
+template<class T>
+class RGWQuotaStatsUpdate : public lru_map<T, RGWQuotaCacheStats>::UpdateContext {
+ int objs_delta;
+ uint64_t added_bytes;
+ uint64_t removed_bytes;
public:
- AsyncRefreshHandler(RGWRados *_store, RGWBucketStatsCache *_cache, rgw_bucket& _bucket) : RGWGetBucketStats_CB(_bucket), store(_store), cache(_cache) {}
+ RGWQuotaStatsUpdate(int _objs_delta, uint64_t _added_bytes, uint64_t _removed_bytes) :
+ objs_delta(_objs_delta), added_bytes(_added_bytes), removed_bytes(_removed_bytes) {}
+ bool update(RGWQuotaCacheStats *entry) {
+ uint64_t rounded_kb_added = rgw_rounded_objsize_kb(added_bytes);
+ uint64_t rounded_kb_removed = rgw_rounded_objsize_kb(removed_bytes);
- int init_fetch();
+ entry->stats.num_kb_rounded += (rounded_kb_added - rounded_kb_removed);
+ entry->stats.num_kb += (added_bytes - removed_bytes) / 1024;
+ entry->stats.num_objects += objs_delta;
- void handle_response(int r);
+ return true;
+ }
};
-int AsyncRefreshHandler::init_fetch()
+template<class T>
+void RGWQuotaCache<T>::adjust_stats(const string& user, rgw_bucket& bucket, int objs_delta,
+ uint64_t added_bytes, uint64_t removed_bytes)
+{
+ RGWQuotaStatsUpdate<T> update(objs_delta, added_bytes, removed_bytes);
+ map_find_and_update(user, bucket, &update);
+
+ data_modified(user, bucket);
+}
+
+class BucketAsyncRefreshHandler : public RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler,
+ public RGWGetBucketStats_CB {
+ string user;
+public:
+ BucketAsyncRefreshHandler(RGWRados *_store, RGWQuotaCache<rgw_bucket> *_cache,
+ const string& _user, rgw_bucket& _bucket) :
+ RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler(_store, _cache),
+ RGWGetBucketStats_CB(_bucket), user(_user) {}
+
+ void drop_reference() { put(); }
+ void handle_response(int r);
+ int init_fetch();
+};
+
+int BucketAsyncRefreshHandler::init_fetch()
{
ldout(store->ctx(), 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
- map<RGWObjCategory, RGWBucketStats> bucket_stats;
+
int r = store->get_bucket_stats_async(bucket, this);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
@@ -138,192 +269,480 @@ int AsyncRefreshHandler::init_fetch()
return 0;
}
-void AsyncRefreshHandler::handle_response(int r)
+void BucketAsyncRefreshHandler::handle_response(int r)
{
if (r < 0) {
ldout(store->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
return; /* nothing to do here */
}
- RGWBucketStats bs;
+ RGWStorageStats bs;
- map<RGWObjCategory, RGWBucketStats>::iterator iter;
+ map<RGWObjCategory, RGWStorageStats>::iterator iter;
for (iter = stats->begin(); iter != stats->end(); ++iter) {
- RGWBucketStats& s = iter->second;
+ RGWStorageStats& s = iter->second;
bs.num_kb += s.num_kb;
bs.num_kb_rounded += s.num_kb_rounded;
bs.num_objects += s.num_objects;
}
- cache->async_refresh_response(bucket, bs);
+ cache->async_refresh_response(user, bucket, bs);
}
-class RGWBucketStatsAsyncTestSet : public lru_map<rgw_bucket, RGWQuotaBucketStats>::UpdateContext {
- int objs_delta;
- uint64_t added_bytes;
- uint64_t removed_bytes;
-public:
- RGWBucketStatsAsyncTestSet() {}
- bool update(RGWQuotaBucketStats *entry) {
- if (entry->async_refresh_time.sec() == 0)
- return false;
+class RGWBucketStatsCache : public RGWQuotaCache<rgw_bucket> {
+protected:
+ bool map_find(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+ return stats_map.find(bucket, qs);
+ }
- entry->async_refresh_time = utime_t(0, 0);
+ bool map_find_and_update(const string& user, rgw_bucket& bucket, lru_map<rgw_bucket, RGWQuotaCacheStats>::UpdateContext *ctx) {
+ return stats_map.find_and_update(bucket, NULL, ctx);
+ }
- return true;
+ void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+ stats_map.add(bucket, qs);
+ }
+
+ int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
+
+public:
+ RGWBucketStatsCache(RGWRados *_store) : RGWQuotaCache<rgw_bucket>(_store, _store->ctx()->_conf->rgw_bucket_quota_cache_size) {
+ }
+
+ AsyncRefreshHandler *allocate_refresh_handler(const string& user, rgw_bucket& bucket) {
+ return new BucketAsyncRefreshHandler(store, this, user, bucket);
}
};
-int RGWBucketStatsCache::async_refresh(rgw_bucket& bucket, RGWQuotaBucketStats& qs)
+int RGWBucketStatsCache::fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats)
{
- /* protect against multiple updates */
- RGWBucketStatsAsyncTestSet test_update;
- if (!stats_map.find_and_update(bucket, NULL, &test_update)) {
- /* most likely we just raced with another update */
- return 0;
- }
+ RGWBucketInfo bucket_info;
- async_refcount->get();
+ uint64_t bucket_ver;
+ uint64_t master_ver;
- AsyncRefreshHandler *handler = new AsyncRefreshHandler(store, this, bucket);
+ map<RGWObjCategory, RGWStorageStats> bucket_stats;
+ int r = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, bucket_stats, NULL);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
+ return r;
+ }
- int ret = handler->init_fetch();
- if (ret < 0) {
- async_refcount->put();
- handler->put();
- return ret;
+ stats = RGWStorageStats();
+
+ map<RGWObjCategory, RGWStorageStats>::iterator iter;
+ for (iter = bucket_stats.begin(); iter != bucket_stats.end(); ++iter) {
+ RGWStorageStats& s = iter->second;
+ stats.num_kb += s.num_kb;
+ stats.num_kb_rounded += s.num_kb_rounded;
+ stats.num_objects += s.num_objects;
}
return 0;
}
-void RGWBucketStatsCache::async_refresh_response(rgw_bucket& bucket, RGWBucketStats& stats)
-{
- ldout(store->ctx(), 20) << "async stats refresh response for bucket=" << bucket << dendl;
+class UserAsyncRefreshHandler : public RGWQuotaCache<string>::AsyncRefreshHandler,
+ public RGWGetUserStats_CB {
+ rgw_bucket bucket;
+public:
+ UserAsyncRefreshHandler(RGWRados *_store, RGWQuotaCache<string> *_cache,
+ const string& _user, rgw_bucket& _bucket) :
+ RGWQuotaCache<string>::AsyncRefreshHandler(_store, _cache),
+ RGWGetUserStats_CB(_user),
+ bucket(_bucket) {}
- RGWQuotaBucketStats qs;
+ void drop_reference() { put(); }
+ int init_fetch();
+ void handle_response(int r);
+};
- stats_map.find(bucket, qs);
+int UserAsyncRefreshHandler::init_fetch()
+{
+ ldout(store->ctx(), 20) << "initiating async quota refresh for user=" << user << dendl;
+ int r = store->get_user_stats_async(user, this);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get bucket info for user=" << user << dendl;
- set_stats(bucket, qs, stats);
+ /* get_bucket_stats_async() dropped our reference already */
+ return r;
+ }
- async_refcount->put();
+ return 0;
}
-void RGWBucketStatsCache::set_stats(rgw_bucket& bucket, RGWQuotaBucketStats& qs, RGWBucketStats& stats)
+void UserAsyncRefreshHandler::handle_response(int r)
{
- qs.stats = stats;
- qs.expiration = ceph_clock_now(store->ctx());
- qs.async_refresh_time = qs.expiration;
- qs.expiration += store->ctx()->_conf->rgw_bucket_quota_ttl;
- qs.async_refresh_time += store->ctx()->_conf->rgw_bucket_quota_ttl / 2;
+ if (r < 0) {
+ ldout(store->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
+ return; /* nothing to do here */
+ }
- stats_map.add(bucket, qs);
+ cache->async_refresh_response(user, bucket, stats);
}
-int RGWBucketStatsCache::get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& stats, RGWQuotaInfo& quota) {
- RGWQuotaBucketStats qs;
- utime_t now = ceph_clock_now(store->ctx());
- if (stats_map.find(bucket, qs)) {
- if (qs.async_refresh_time.sec() > 0 && now >= qs.async_refresh_time) {
- int r = async_refresh(bucket, qs);
- if (r < 0) {
- ldout(store->ctx(), 0) << "ERROR: quota async refresh returned ret=" << r << dendl;
+class RGWUserStatsCache : public RGWQuotaCache<string> {
+ atomic_t down_flag;
+ RWLock rwlock;
+ map<rgw_bucket, string> modified_buckets;
+
+ /* thread, sync recent modified buckets info */
+ class BucketsSyncThread : public Thread {
+ CephContext *cct;
+ RGWUserStatsCache *stats;
+
+ Mutex lock;
+ Cond cond;
+ public:
+
+ BucketsSyncThread(CephContext *_cct, RGWUserStatsCache *_s) : cct(_cct), stats(_s), lock("RGWUserStatsCache::BucketsSyncThread") {}
+
+ void *entry() {
+ ldout(cct, 20) << "BucketsSyncThread: start" << dendl;
+ do {
+ map<rgw_bucket, string> buckets;
+
+ stats->swap_modified_buckets(buckets);
+
+ for (map<rgw_bucket, string>::iterator iter = buckets.begin(); iter != buckets.end(); ++iter) {
+ rgw_bucket bucket = iter->first;
+ string& user = iter->second;
+ ldout(cct, 20) << "BucketsSyncThread: sync user=" << user << " bucket=" << bucket << dendl;
+ int r = stats->sync_bucket(user, bucket);
+ if (r < 0) {
+ ldout(cct, 0) << "WARNING: sync_bucket() returned r=" << r << dendl;
+ }
+ }
+
+ if (stats->going_down())
+ break;
+
+ lock.Lock();
+ cond.WaitInterval(cct, lock, utime_t(cct->_conf->rgw_user_quota_bucket_sync_interval, 0));
+ lock.Unlock();
+ } while (!stats->going_down());
+ ldout(cct, 20) << "BucketsSyncThread: done" << dendl;
+
+ return NULL;
+ }
- /* continue processing, might be a transient error, async refresh is just optimization */
- }
+ void stop() {
+ Mutex::Locker l(lock);
+ cond.Signal();
}
+ };
- if (can_use_cached_stats(quota, qs.stats) && qs.expiration > ceph_clock_now(store->ctx())) {
- stats = qs.stats;
- return 0;
+ /*
+ * thread, full sync all users stats periodically
+ *
+ * only sync non idle users or ones that never got synced before, this is needed so that
+ * users that didn't have quota turned on before (or existed before the user objclass
+ * tracked stats) need to get their backend stats up to date.
+ */
+ class UserSyncThread : public Thread {
+ CephContext *cct;
+ RGWUserStatsCache *stats;
+
+ Mutex lock;
+ Cond cond;
+ public:
+
+ UserSyncThread(CephContext *_cct, RGWUserStatsCache *_s) : cct(_cct), stats(_s), lock("RGWUserStatsCache::UserSyncThread") {}
+
+ void *entry() {
+ ldout(cct, 20) << "UserSyncThread: start" << dendl;
+ do {
+
+ string key = "user";
+
+ int ret = stats->sync_all_users();
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: sync_all_users() returned ret=" << ret << dendl;
+ }
+
+ lock.Lock();
+ cond.WaitInterval(cct, lock, utime_t(cct->_conf->rgw_user_quota_sync_interval, 0));
+ lock.Unlock();
+ } while (!stats->going_down());
+ ldout(cct, 20) << "UserSyncThread: done" << dendl;
+
+ return NULL;
}
+
+ void stop() {
+ Mutex::Locker l(lock);
+ cond.Signal();
+ }
+ };
+
+ BucketsSyncThread *buckets_sync_thread;
+ UserSyncThread *user_sync_thread;
+protected:
+ bool map_find(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+ return stats_map.find(user, qs);
}
- int ret = fetch_bucket_totals(bucket, stats);
- if (ret < 0 && ret != -ENOENT)
- return ret;
+ bool map_find_and_update(const string& user, rgw_bucket& bucket, lru_map<string, RGWQuotaCacheStats>::UpdateContext *ctx) {
+ return stats_map.find_and_update(user, NULL, ctx);
+ }
- set_stats(bucket, qs, stats);
+ void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+ stats_map.add(user, qs);
+ }
- return 0;
-}
+ int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
+ int sync_bucket(const string& user, rgw_bucket& bucket);
+ int sync_user(const string& user);
+ int sync_all_users();
+ void data_modified(const string& user, rgw_bucket& bucket);
+
+ void swap_modified_buckets(map<rgw_bucket, string>& out) {
+ rwlock.get_write();
+ modified_buckets.swap(out);
+ rwlock.unlock();
+ }
+
+ template<class T> /* easier doing it as a template, Thread doesn't have ->stop() */
+ void stop_thread(T **pthr) {
+ T *thread = *pthr;
+ if (!thread)
+ return;
+
+ thread->stop();
+ thread->join();
+ delete thread;
+ *pthr = NULL;
+ }
-class RGWBucketStatsUpdate : public lru_map<rgw_bucket, RGWQuotaBucketStats>::UpdateContext {
- int objs_delta;
- uint64_t added_bytes;
- uint64_t removed_bytes;
public:
- RGWBucketStatsUpdate(int _objs_delta, uint64_t _added_bytes, uint64_t _removed_bytes) :
- objs_delta(_objs_delta), added_bytes(_added_bytes), removed_bytes(_removed_bytes) {}
- bool update(RGWQuotaBucketStats *entry) {
- uint64_t rounded_kb_added = rgw_rounded_kb(added_bytes);
- uint64_t rounded_kb_removed = rgw_rounded_kb(removed_bytes);
+ RGWUserStatsCache(RGWRados *_store, bool quota_threads) : RGWQuotaCache<string>(_store, _store->ctx()->_conf->rgw_bucket_quota_cache_size),
+ rwlock("RGWUserStatsCache::rwlock") {
+ if (quota_threads) {
+ buckets_sync_thread = new BucketsSyncThread(store->ctx(), this);
+ buckets_sync_thread->create();
+ user_sync_thread = new UserSyncThread(store->ctx(), this);
+ user_sync_thread->create();
+ } else {
+ buckets_sync_thread = NULL;
+ user_sync_thread = NULL;
+ }
+ }
+ ~RGWUserStatsCache() {
+ stop();
+ }
- entry->stats.num_kb_rounded += (rounded_kb_added - rounded_kb_removed);
- entry->stats.num_kb += (added_bytes - removed_bytes) / 1024;
- entry->stats.num_objects += objs_delta;
+ AsyncRefreshHandler *allocate_refresh_handler(const string& user, rgw_bucket& bucket) {
+ return new UserAsyncRefreshHandler(store, this, user, bucket);
+ }
+ bool can_use_cached_stats(RGWQuotaInfo& quota, RGWStorageStats& stats) {
+ /* in the user case, the cached stats may contain a better estimation of the totals, as
+ * the backend is only periodically getting updated.
+ */
return true;
}
+
+ bool going_down() {
+ return (down_flag.read() != 0);
+ }
+
+ void stop() {
+ down_flag.set(1);
+ rwlock.get_write();
+ stop_thread(&buckets_sync_thread);
+ rwlock.unlock();
+ stop_thread(&user_sync_thread);
+ }
};
+int RGWUserStatsCache::fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+ int r = store->get_user_stats(user, stats);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get user stats for user=" << user << dendl;
+ return r;
+ }
+
+ return 0;
+}
-void RGWBucketStatsCache::adjust_bucket_stats(rgw_bucket& bucket, int objs_delta, uint64_t added_bytes, uint64_t removed_bytes)
+int RGWUserStatsCache::sync_bucket(const string& user, rgw_bucket& bucket)
{
- RGWBucketStatsUpdate update(objs_delta, added_bytes, removed_bytes);
- stats_map.find_and_update(bucket, NULL, &update);
+ int r = rgw_bucket_sync_user_stats(store, user, bucket);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "ERROR: rgw_bucket_sync_user_stats() for user=" << user << ", bucket=" << bucket << " returned " << r << dendl;
+ return r;
+ }
+
+ return 0;
+}
+
+int RGWUserStatsCache::sync_user(const string& user)
+{
+ cls_user_header header;
+ int ret = store->cls_user_get_header(user, &header);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: can't read user header: ret=" << ret << dendl;
+ return ret;
+ }
+
+ if (!store->ctx()->_conf->rgw_user_quota_sync_idle_users &&
+ header.last_stats_update < header.last_stats_sync) {
+ ldout(store->ctx(), 20) << "user is idle, not doing a full sync (user=" << user << ")" << dendl;
+ return 0;
+ }
+
+ utime_t when_need_full_sync = header.last_stats_sync;
+ when_need_full_sync += store->ctx()->_conf->rgw_user_quota_sync_wait_time;
+
+ // check if enough time passed since last full sync
+
+ ret = rgw_user_sync_all_stats(store, user);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: failed user stats sync, ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int RGWUserStatsCache::sync_all_users()
+{
+ string key = "user";
+ void *handle;
+
+ int ret = store->meta_mgr->list_keys_init(key, &handle);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: can't get key: ret=" << ret << dendl;
+ return ret;
+ }
+
+ bool truncated;
+ int max = 1000;
+
+ do {
+ list<string> keys;
+ ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: lists_keys_next(): ret=" << ret << dendl;
+ goto done;
+ }
+ for (list<string>::iterator iter = keys.begin();
+ iter != keys.end() && !going_down();
+ ++iter) {
+ string& user = *iter;
+ ldout(store->ctx(), 20) << "RGWUserStatsCache: sync user=" << user << dendl;
+ int ret = sync_user(user);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: sync_user() failed, user=" << user << " ret=" << ret << dendl;
+
+ /* continuing to next user */
+ continue;
+ }
+ }
+ } while (truncated);
+
+ ret = 0;
+done:
+ store->meta_mgr->list_keys_complete(handle);
+ return ret;
+}
+
+void RGWUserStatsCache::data_modified(const string& user, rgw_bucket& bucket)
+{
+ /* racy, but it's ok */
+ rwlock.get_read();
+ bool need_update = modified_buckets.find(bucket) == modified_buckets.end();
+ rwlock.unlock();
+
+ if (need_update) {
+ rwlock.get_write();
+ modified_buckets[bucket] = user;
+ rwlock.unlock();
+ }
}
class RGWQuotaHandlerImpl : public RGWQuotaHandler {
RGWRados *store;
- RGWBucketStatsCache stats_cache;
+ RGWBucketStatsCache bucket_stats_cache;
+ RGWUserStatsCache user_stats_cache;
+
+ int check_quota(const char *entity, RGWQuotaInfo& quota, RGWStorageStats& stats,
+ uint64_t num_objs, uint64_t size_kb) {
+ ldout(store->ctx(), 20) << entity << " quota: max_objects=" << quota.max_objects
+ << " max_size_kb=" << quota.max_size_kb << dendl;
+
+ if (quota.max_objects >= 0 &&
+ stats.num_objects + num_objs > (uint64_t)quota.max_objects) {
+ ldout(store->ctx(), 10) << "quota exceeded: stats.num_objects=" << stats.num_objects
+ << " " << entity << "_quota.max_objects=" << quota.max_objects << dendl;
+
+ return -ERR_QUOTA_EXCEEDED;
+ }
+ if (quota.max_size_kb >= 0 &&
+ stats.num_kb_rounded + size_kb > (uint64_t)quota.max_size_kb) {
+ ldout(store->ctx(), 10) << "quota exceeded: stats.num_kb_rounded=" << stats.num_kb_rounded << " size_kb=" << size_kb
+ << " " << entity << "_quota.max_size_kb=" << quota.max_size_kb << dendl;
+ return -ERR_QUOTA_EXCEEDED;
+ }
+
+ return 0;
+ }
public:
- RGWQuotaHandlerImpl(RGWRados *_store) : store(_store), stats_cache(_store) {}
- virtual int check_quota(rgw_bucket& bucket, RGWQuotaInfo& bucket_quota,
+ RGWQuotaHandlerImpl(RGWRados *_store, bool quota_threads) : store(_store), bucket_stats_cache(_store), user_stats_cache(_store, quota_threads) {}
+ virtual int check_quota(const string& user, rgw_bucket& bucket,
+ RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota,
uint64_t num_objs, uint64_t size) {
- uint64_t size_kb = rgw_rounded_kb(size);
- if (!bucket_quota.enabled) {
+
+ if (!bucket_quota.enabled && !user_quota.enabled)
return 0;
- }
- RGWBucketStats stats;
+ uint64_t size_kb = rgw_rounded_objsize_kb(size);
+
+ RGWStorageStats bucket_stats;
- int ret = stats_cache.get_bucket_stats(bucket, stats, bucket_quota);
+ /*
+ * we need to fetch bucket stats if the user quota is enabled, because the whole system relies
+ * on us periodically updating the user's bucket stats in the user's header, this happens in
+ * get_stats() if we actually fetch that info and not rely on cached data
+ */
+
+ int ret = bucket_stats_cache.get_stats(user, bucket, bucket_stats, bucket_quota);
if (ret < 0)
return ret;
- ldout(store->ctx(), 20) << "bucket quota: max_objects=" << bucket_quota.max_objects
- << " max_size_kb=" << bucket_quota.max_size_kb << dendl;
+ if (bucket_quota.enabled) {
+ ret = check_quota("bucket", bucket_quota, bucket_stats, num_objs, size_kb);
+ if (ret < 0)
+ return ret;
+ }
- if (bucket_quota.max_objects >= 0 &&
- stats.num_objects + num_objs > (uint64_t)bucket_quota.max_objects) {
- ldout(store->ctx(), 10) << "quota exceeded: stats.num_objects=" << stats.num_objects
- << " bucket_quota.max_objects=" << bucket_quota.max_objects << dendl;
+ if (user_quota.enabled) {
+ RGWStorageStats user_stats;
- return -ERR_QUOTA_EXCEEDED;
- }
- if (bucket_quota.max_size_kb >= 0 &&
- stats.num_kb_rounded + size_kb > (uint64_t)bucket_quota.max_size_kb) {
- ldout(store->ctx(), 10) << "quota exceeded: stats.num_kb_rounded=" << stats.num_kb_rounded << " size_kb=" << size_kb
- << " bucket_quota.max_size_kb=" << bucket_quota.max_size_kb << dendl;
- return -ERR_QUOTA_EXCEEDED;
+ ret = user_stats_cache.get_stats(user, bucket, user_stats, user_quota);
+ if (ret < 0)
+ return ret;
+
+ ret = check_quota("user", user_quota, user_stats, num_objs, size_kb);
+ if (ret < 0)
+ return ret;
}
return 0;
}
- virtual void update_stats(rgw_bucket& bucket, int obj_delta, uint64_t added_bytes, uint64_t removed_bytes) {
- stats_cache.adjust_bucket_stats(bucket, obj_delta, added_bytes, removed_bytes);
+ virtual void update_stats(const string& user, rgw_bucket& bucket, int obj_delta, uint64_t added_bytes, uint64_t removed_bytes) {
+ bucket_stats_cache.adjust_stats(user, bucket, obj_delta, added_bytes, removed_bytes);
+ user_stats_cache.adjust_stats(user, bucket, obj_delta, added_bytes, removed_bytes);
};
};
-RGWQuotaHandler *RGWQuotaHandler::generate_handler(RGWRados *store)
+RGWQuotaHandler *RGWQuotaHandler::generate_handler(RGWRados *store, bool quota_threads)
{
- return new RGWQuotaHandlerImpl(store);
+ return new RGWQuotaHandlerImpl(store, quota_threads);
};
void RGWQuotaHandler::free_handler(RGWQuotaHandler *handler)
diff --git a/src/rgw/rgw_quota.h b/src/rgw/rgw_quota.h
index 2f8f28e..abdb62e 100644
--- a/src/rgw/rgw_quota.h
+++ b/src/rgw/rgw_quota.h
@@ -62,12 +62,13 @@ public:
RGWQuotaHandler() {}
virtual ~RGWQuotaHandler() {
}
- virtual int check_quota(rgw_bucket& bucket, RGWQuotaInfo& bucket_quota,
+ virtual int check_quota(const string& bucket_owner, rgw_bucket& bucket,
+ RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota,
uint64_t num_objs, uint64_t size) = 0;
- virtual void update_stats(rgw_bucket& bucket, int obj_delta, uint64_t added_bytes, uint64_t removed_bytes) = 0;
+ virtual void update_stats(const string& bucket_owner, rgw_bucket& bucket, int obj_delta, uint64_t added_bytes, uint64_t removed_bytes) = 0;
- static RGWQuotaHandler *generate_handler(RGWRados *store);
+ static RGWQuotaHandler *generate_handler(RGWRados *store, bool quota_threads);
static void free_handler(RGWQuotaHandler *handler);
};
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index ac9503a..cce463b 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -22,6 +22,7 @@
#include "cls/log/cls_log_client.h"
#include "cls/statelog/cls_statelog_client.h"
#include "cls/lock/cls_lock_client.h"
+#include "cls/user/cls_user_client.h"
#include "rgw_tools.h"
@@ -389,20 +390,23 @@ int RGWZoneParams::store_info(CephContext *cct, RGWRados *store, RGWRegion& regi
}
void RGWRegionMap::encode(bufferlist& bl) const {
- ENCODE_START(2, 1, bl);
+ ENCODE_START(3, 1, bl);
::encode(regions, bl);
::encode(master_region, bl);
::encode(bucket_quota, bl);
+ ::encode(user_quota, bl);
ENCODE_FINISH(bl);
}
void RGWRegionMap::decode(bufferlist::iterator& bl) {
- DECODE_START(2, bl);
+ DECODE_START(3, bl);
::decode(regions, bl);
::decode(master_region, bl);
if (struct_v >= 2)
::decode(bucket_quota, bl);
+ if (struct_v >= 3)
+ ::decode(user_quota, bl);
DECODE_FINISH(bl);
regions_by_api.clear();
@@ -561,7 +565,7 @@ RGWPutObjProcessor::~RGWPutObjProcessor()
list<rgw_obj>::iterator iter;
for (iter = objs.begin(); iter != objs.end(); ++iter) {
rgw_obj& obj = *iter;
- int r = store->delete_obj(obj_ctx, obj);
+ int r = store->delete_obj(obj_ctx, bucket_owner, obj);
if (r < 0 && r != -ENOENT) {
ldout(store->ctx(), 0) << "WARNING: failed to remove obj (" << obj << "), leaked" << dendl;
}
@@ -594,6 +598,7 @@ int RGWPutObjProcessor_Plain::do_complete(string& etag, time_t *mtime, time_t se
params.set_mtime = set_mtime;
params.mtime = mtime;
params.data = &data;
+ params.owner = bucket_owner;
int r = store->put_obj_meta(obj_ctx, obj, data.length(), attrs,
RGW_OBJ_CATEGORY_MAIN, PUT_OBJ_CREATE,
@@ -819,6 +824,7 @@ int RGWPutObjProcessor_Atomic::do_complete(string& etag, time_t *mtime, time_t s
extra_params.ptag = &unique_tag; /* use req_id as operation tag */
extra_params.mtime = mtime;
extra_params.set_mtime = set_mtime;
+ extra_params.owner = bucket_owner;
r = store->put_obj_meta(obj_ctx, head_obj, obj_len, attrs,
RGW_OBJ_CATEGORY_MAIN, PUT_OBJ_CREATE,
@@ -999,7 +1005,7 @@ int RGWRados::init_complete()
if (use_gc_thread)
gc->start_processor();
- quota_handler = RGWQuotaHandler::generate_handler(this);
+ quota_handler = RGWQuotaHandler::generate_handler(this, quota_threads);
return ret;
}
@@ -2261,7 +2267,8 @@ int RGWRados::put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size,
list<string> *remove_objs,
bool modify_version,
RGWObjVersionTracker *objv_tracker,
- time_t set_mtime)
+ time_t set_mtime,
+ const string& bucket_owner)
{
rgw_bucket bucket;
std::string oid, key;
@@ -2367,7 +2374,9 @@ int RGWRados::put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size,
return r;
r = io_ctx.operate(oid, &op);
- if (r < 0)
+ if (r < 0) /* we can expect to get -ECANCELED if object was replaced under,
+ or -ENOENT if was removed, or -EEXIST if it did not exist
+ before and now it does */
goto done_cancel;
if (objv_tracker) {
@@ -2393,7 +2402,7 @@ int RGWRados::put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size,
if (state) {
/* update quota cache */
- quota_handler->update_stats(bucket, (state->exists ? 0 : 1), size, state->size);
+ quota_handler->update_stats(bucket_owner, bucket, (state->exists ? 0 : 1), size, state->size);
}
return 0;
@@ -2403,11 +2412,17 @@ done_cancel:
if (ret < 0) {
ldout(cct, 0) << "ERROR: complete_update_index_cancel() returned ret=" << ret << dendl;
}
- /* we lost in a race, object was already overwritten, we
+ /* we lost in a race. There are a few options:
+ * - existing object was rewritten (ECANCELED)
+ * - non existing object was created (EEXIST)
+ * - object was removed (ENOENT)
* should treat it as a success
*/
- if (r == -ECANCELED)
+ if ((r == -ECANCELED || r == -ENOENT) ||
+ (!(flags & PUT_OBJ_EXCL) && r == -EEXIST)) {
r = 0;
+ }
+
return r;
}
@@ -2638,7 +2653,7 @@ int RGWRados::copy_obj(void *ctx,
string tag;
append_rand_alpha(cct, tag, tag, 32);
- RGWPutObjProcessor_Atomic processor(dest_obj.bucket, dest_obj.object,
+ RGWPutObjProcessor_Atomic processor(dest_bucket_info.owner, dest_obj.bucket, dest_obj.object,
cct->_conf->rgw_obj_stripe_size, tag);
ret = processor.prepare(this, ctx);
if (ret < 0)
@@ -2773,7 +2788,7 @@ set_err_state:
return 0;
} else if (copy_data) { /* refcounting tail wouldn't work here, just copy the data */
- return copy_obj_data(ctx, &handle, end, dest_obj, src_obj, mtime, src_attrs, category, ptag, err);
+ return copy_obj_data(ctx, dest_bucket_info.owner, &handle, end, dest_obj, src_obj, mtime, src_attrs, category, ptag, err);
}
map<uint64_t, RGWObjManifestPart>::iterator miter = astate->manifest.objs.begin();
@@ -2847,6 +2862,7 @@ set_err_state:
ep.data = &first_chunk;
ep.manifest = pmanifest;
ep.ptag = &tag;
+ ep.owner = dest_bucket_info.owner;
ret = put_obj_meta(ctx, dest_obj, end + 1, src_attrs, category, PUT_OBJ_CREATE, ep);
@@ -2878,6 +2894,7 @@ done_ret:
int RGWRados::copy_obj_data(void *ctx,
+ const string& owner,
void **handle, off_t end,
rgw_obj& dest_obj,
rgw_obj& src_obj,
@@ -2946,6 +2963,7 @@ int RGWRados::copy_obj_data(void *ctx,
ep.data = &first_chunk;
ep.manifest = &manifest;
ep.ptag = ptag;
+ ep.owner = owner;
ret = put_obj_meta(ctx, dest_obj, end + 1, attrs, category, PUT_OBJ_CREATE, ep);
if (mtime)
@@ -2953,7 +2971,7 @@ int RGWRados::copy_obj_data(void *ctx,
return ret;
done_err:
- delete_obj(ctx, shadow_obj);
+ delete_obj(ctx, owner, shadow_obj);
return r;
}
@@ -3114,12 +3132,12 @@ int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
return 0;
}
-static void translate_raw_stats(rgw_bucket_dir_header& header, map<RGWObjCategory, RGWBucketStats>& stats)
+static void translate_raw_stats(rgw_bucket_dir_header& header, map<RGWObjCategory, RGWStorageStats>& stats)
{
map<uint8_t, struct rgw_bucket_category_stats>::iterator iter = header.stats.begin();
for (; iter != header.stats.end(); ++iter) {
RGWObjCategory category = (RGWObjCategory)iter->first;
- RGWBucketStats& s = stats[category];
+ RGWStorageStats& s = stats[category];
struct rgw_bucket_category_stats& header_stats = iter->second;
s.category = (RGWObjCategory)iter->first;
s.num_kb = ((header_stats.total_size + 1023) / 1024);
@@ -3129,8 +3147,8 @@ static void translate_raw_stats(rgw_bucket_dir_header& header, map<RGWObjCategor
}
int RGWRados::bucket_check_index(rgw_bucket& bucket,
- map<RGWObjCategory, RGWBucketStats> *existing_stats,
- map<RGWObjCategory, RGWBucketStats> *calculated_stats)
+ map<RGWObjCategory, RGWStorageStats> *existing_stats,
+ map<RGWObjCategory, RGWStorageStats> *calculated_stats)
{
librados::IoCtx index_ctx;
string oid;
@@ -3204,7 +3222,7 @@ int RGWRados::defer_gc(void *ctx, rgw_obj& obj)
* obj: name of the object to delete
* Returns: 0 on success, -ERR# otherwise.
*/
-int RGWRados::delete_obj_impl(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+int RGWRados::delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
{
rgw_bucket bucket;
std::string oid, key;
@@ -3267,17 +3285,29 @@ int RGWRados::delete_obj_impl(void *ctx, rgw_obj& obj, RGWObjVersionTracker *obj
if (state) {
/* update quota cache */
- quota_handler->update_stats(bucket, -1, 0, state->size);
+ quota_handler->update_stats(bucket_owner, bucket, -1, 0, state->size);
}
return 0;
}
-int RGWRados::delete_obj(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+int RGWRados::delete_obj(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+{
+ int r;
+
+ r = delete_obj_impl(ctx, bucket_owner, obj, objv_tracker);
+ if (r == -ECANCELED)
+ r = 0;
+
+ return r;
+}
+
+int RGWRados::delete_system_obj(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
{
int r;
- r = delete_obj_impl(ctx, obj, objv_tracker);
+ string no_owner;
+ r = delete_obj_impl(ctx, no_owner, obj, objv_tracker);
if (r == -ECANCELED)
r = 0;
@@ -3497,8 +3527,12 @@ int RGWRados::prepare_atomic_for_write_impl(RGWRadosCtx *rctx, rgw_obj& obj,
}
if (reset_obj) {
- op.create(false);
- op.remove();
+ if (state->exists) {
+ op.create(false);
+ op.remove();
+ } else {
+ op.create(true);
+ }
}
if (ptag) {
@@ -4672,7 +4706,7 @@ int RGWRados::obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime,
return 0;
}
-int RGWRados::get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWBucketStats>& stats,
+int RGWRados::get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWStorageStats>& stats,
string *max_marker)
{
rgw_bucket_dir_header header;
@@ -4699,7 +4733,7 @@ class RGWGetBucketStatsContext : public RGWGetDirHeader_CB {
public:
RGWGetBucketStatsContext(RGWGetBucketStats_CB *_cb) : cb(_cb) {}
void handle_response(int r, rgw_bucket_dir_header& header) {
- map<RGWObjCategory, RGWBucketStats> stats;
+ map<RGWObjCategory, RGWStorageStats> stats;
if (r >= 0) {
translate_raw_stats(header, stats);
@@ -4725,6 +4759,58 @@ int RGWRados::get_bucket_stats_async(rgw_bucket& bucket, RGWGetBucketStats_CB *c
return 0;
}
+class RGWGetUserStatsContext : public RGWGetUserHeader_CB {
+ RGWGetUserStats_CB *cb;
+
+public:
+ RGWGetUserStatsContext(RGWGetUserStats_CB *_cb) : cb(_cb) {}
+ void handle_response(int r, cls_user_header& header) {
+ cls_user_stats& hs = header.stats;
+ if (r >= 0) {
+ RGWStorageStats stats;
+
+ stats.num_kb = (hs.total_bytes + 1023) / 1024;
+ stats.num_kb_rounded = (hs.total_bytes_rounded + 1023) / 1024;
+ stats.num_objects = hs.total_entries;
+
+ cb->set_response(stats);
+ }
+
+ cb->handle_response(r);
+
+ cb->put();
+ }
+};
+
+int RGWRados::get_user_stats(const string& user, RGWStorageStats& stats)
+{
+ cls_user_header header;
+ int r = cls_user_get_header(user, &header);
+ if (r < 0)
+ return r;
+
+ cls_user_stats& hs = header.stats;
+
+ stats.num_kb = (hs.total_bytes + 1023) / 1024;
+ stats.num_kb_rounded = (hs.total_bytes_rounded + 1023) / 1024;
+ stats.num_objects = hs.total_entries;
+
+ return 0;
+}
+
+int RGWRados::get_user_stats_async(const string& user, RGWGetUserStats_CB *ctx)
+{
+ RGWGetUserStatsContext *get_ctx = new RGWGetUserStatsContext(ctx);
+ int r = cls_user_get_header_async(user, get_ctx);
+ if (r < 0) {
+ ctx->put();
+ delete get_ctx;
+ return r;
+ }
+
+ return 0;
+}
+
void RGWRados::get_bucket_instance_entry(rgw_bucket& bucket, string& entry)
{
entry = bucket.name + ":" + bucket.bucket_id;
@@ -4819,7 +4905,42 @@ int RGWRados::get_bucket_entrypoint_info(void *ctx, const string& bucket_name,
return 0;
}
-int RGWRados::get_bucket_info(void *ctx, string& bucket_name, RGWBucketInfo& info,
+int RGWRados::convert_old_bucket_info(void *ctx, string& bucket_name)
+{
+ RGWBucketEntryPoint entry_point;
+ time_t ep_mtime;
+ RGWObjVersionTracker ot;
+ map<string, bufferlist> attrs;
+ RGWBucketInfo info;
+
+ ldout(cct, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket_name << dendl;
+
+ int ret = get_bucket_entrypoint_info(ctx, bucket_name, entry_point, &ot, &ep_mtime, &attrs);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: get_bucket_entrypont_info() returned " << ret << " bucket=" << bucket_name << dendl;
+ return ret;
+ }
+
+ if (!entry_point.has_bucket_info) {
+ /* already converted! */
+ return 0;
+ }
+
+ info = entry_point.old_bucket_info;
+ info.bucket.oid = bucket_name;
+ info.ep_objv = ot.read_version;
+
+ ot.generate_new_write_ver(cct);
+
+ ret = put_linked_bucket_info(info, false, ep_mtime, &ot.write_version, &attrs, true);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret << dendl;
+ }
+
+ return 0;
+}
+
+int RGWRados::get_bucket_info(void *ctx, const string& bucket_name, RGWBucketInfo& info,
time_t *pmtime, map<string, bufferlist> *pattrs)
{
bufferlist bl;
@@ -5622,9 +5743,218 @@ int RGWRados::cls_bucket_head_async(rgw_bucket& bucket, RGWGetDirHeader_CB *ctx)
return 0;
}
-int RGWRados::check_quota(rgw_bucket& bucket, RGWQuotaInfo& quota_info, uint64_t obj_size)
+int RGWRados::cls_user_get_header(const string& user_id, cls_user_header *header)
+{
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
+ rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+ librados::IoCtx io_ctx;
+ rgw_bucket bucket;
+ std::string oid, key;
+ get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+ int r = open_bucket_data_ctx(bucket, io_ctx);
+ if (r < 0)
+ return r;
+
+ librados::ObjectReadOperation op;
+ int rc;
+ ::cls_user_get_header(op, header, &rc);
+ bufferlist ibl;
+ r = io_ctx.operate(oid, &op, &ibl);
+ if (r < 0)
+ return r;
+ if (rc < 0)
+ return rc;
+
+ return 0;
+}
+
+int RGWRados::cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx)
+{
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
+ rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+ librados::IoCtx io_ctx;
+ rgw_bucket bucket;
+ std::string oid, key;
+ get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+ int r = open_bucket_data_ctx(bucket, io_ctx);
+ if (r < 0)
+ return r;
+
+ r = ::cls_user_get_header_async(io_ctx, oid, ctx);
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+int RGWRados::cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket)
+{
+ rgw_bucket_dir_header header;
+ int r = cls_bucket_head(bucket, header);
+ if (r < 0) {
+ ldout(cct, 20) << "cls_bucket_header() returned " << r << dendl;
+ return r;
+ }
+
+ cls_user_bucket_entry entry;
+
+ bucket.convert(&entry.bucket);
+
+ map<uint8_t, struct rgw_bucket_category_stats>::iterator iter = header.stats.begin();
+ for (; iter != header.stats.end(); ++iter) {
+ struct rgw_bucket_category_stats& header_stats = iter->second;
+ entry.size += header_stats.total_size;
+ entry.size_rounded += header_stats.total_size_rounded;
+ entry.count += header_stats.num_entries;
+ }
+
+ list<cls_user_bucket_entry> entries;
+ entries.push_back(entry);
+
+ r = cls_user_update_buckets(user_obj, entries, false);
+ if (r < 0) {
+ ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
+ return r;
+ }
+
+ return 0;
+}
+
+int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+ cls_user_bucket_entry entry;
+
+ entry.size = stats.num_kb * 1024;
+ entry.size_rounded = stats.num_kb_rounded * 1024;
+ entry.count += stats.num_objects;
+
+ list<cls_user_bucket_entry> entries;
+ entries.push_back(entry);
+
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
+ rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+ int r = cls_user_update_buckets(obj, entries, false);
+ if (r < 0) {
+ ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
+ return r;
+ }
+
+ return 0;
+}
+
+int RGWRados::cls_user_list_buckets(rgw_obj& obj,
+ const string& in_marker, int max_entries,
+ list<cls_user_bucket_entry>& entries,
+ string *out_marker, bool *truncated)
+{
+ librados::IoCtx io_ctx;
+ rgw_bucket bucket;
+ std::string oid, key;
+ get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+ int r = open_bucket_data_ctx(bucket, io_ctx);
+ if (r < 0)
+ return r;
+
+ librados::ObjectReadOperation op;
+ int rc;
+
+ cls_user_bucket_list(op, in_marker, max_entries, entries, out_marker, truncated, &rc);
+ bufferlist ibl;
+ r = io_ctx.operate(oid, &op, &ibl);
+ if (r < 0)
+ return r;
+ if (rc < 0)
+ return rc;
+
+ return 0;
+}
+
+int RGWRados::cls_user_update_buckets(rgw_obj& obj, list<cls_user_bucket_entry>& entries, bool add)
+{
+ bufferlist bl;
+ librados::IoCtx io_ctx;
+ rgw_bucket bucket;
+ std::string oid, key;
+ get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+ int r = open_bucket_data_ctx(bucket, io_ctx);
+ if (r < 0)
+ return r;
+
+ librados::ObjectWriteOperation op;
+ cls_user_set_buckets(op, entries, add);
+ r = io_ctx.operate(oid, &op);
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+int RGWRados::complete_sync_user_stats(const string& user_id)
+{
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
+ rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+ return cls_user_complete_stats_sync(obj);
+}
+
+int RGWRados::cls_user_complete_stats_sync(rgw_obj& obj)
+{
+ bufferlist bl;
+ librados::IoCtx io_ctx;
+ rgw_bucket bucket;
+ std::string oid, key;
+ get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+ int r = open_bucket_data_ctx(bucket, io_ctx);
+ if (r < 0)
+ return r;
+
+ librados::ObjectWriteOperation op;
+ ::cls_user_complete_stats_sync(op);
+ r = io_ctx.operate(oid, &op);
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+int RGWRados::cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry)
{
- return quota_handler->check_quota(bucket, quota_info, 1, obj_size);
+ list<cls_user_bucket_entry> l;
+ l.push_back(entry);
+
+ return cls_user_update_buckets(obj, l, true);
+}
+
+int RGWRados::cls_user_remove_bucket(rgw_obj& obj, const cls_user_bucket& bucket)
+{
+ bufferlist bl;
+ librados::IoCtx io_ctx;
+ rgw_bucket b;
+ std::string oid, key;
+ get_obj_bucket_and_oid_key(obj, b, oid, key);
+ int r = open_bucket_data_ctx(b, io_ctx);
+ if (r < 0)
+ return r;
+
+ librados::ObjectWriteOperation op;
+ ::cls_user_remove_bucket(op, bucket);
+ r = io_ctx.operate(oid, &op);
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+int RGWRados::check_quota(const string& bucket_owner, rgw_bucket& bucket,
+ RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size)
+{
+ return quota_handler->check_quota(bucket_owner, bucket, user_quota, bucket_quota, 1, obj_size);
}
class IntentLogNameFilter : public RGWAccessListFilter
@@ -5722,6 +6052,8 @@ int RGWRados::process_intent_log(rgw_bucket& bucket, string& oid,
bufferlist::iterator iter;
off_t off;
+ string no_owner;
+
while (!eof || !iter.end()) {
off = iter.get_off();
if (!eof && (bl.length() - off) < chunk / 2) {
@@ -5764,7 +6096,7 @@ int RGWRados::process_intent_log(rgw_bucket& bucket, string& oid,
complete = false;
break;
}
- r = delete_obj(NULL, entry.obj);
+ r = delete_obj(NULL, no_owner, entry.obj);
if (r < 0 && r != -ENOENT) {
cerr << "failed to remove obj: " << entry.obj << std::endl;
complete = false;
@@ -5801,7 +6133,7 @@ int RGWRados::process_intent_log(rgw_bucket& bucket, string& oid,
rgw_obj obj(bucket, oid);
cout << "completed intent log: " << obj << (purge ? ", purging it" : "") << std::endl;
if (purge) {
- r = delete_obj(NULL, obj);
+ r = delete_system_obj(NULL, obj);
if (r < 0)
cerr << "failed to remove obj: " << obj << std::endl;
}
@@ -6086,7 +6418,7 @@ uint64_t RGWRados::next_bucket_id()
return ++max_bucket_id;
}
-RGWRados *RGWStoreManager::init_storage_provider(CephContext *cct, bool use_gc_thread)
+RGWRados *RGWStoreManager::init_storage_provider(CephContext *cct, bool use_gc_thread, bool quota_threads)
{
int use_cache = cct->_conf->rgw_cache_enabled;
RGWRados *store = NULL;
@@ -6096,7 +6428,7 @@ RGWRados *RGWStoreManager::init_storage_provider(CephContext *cct, bool use_gc_t
store = new RGWCache<RGWRados>;
}
- if (store->initialize(cct, use_gc_thread) < 0) {
+ if (store->initialize(cct, use_gc_thread, quota_threads) < 0) {
delete store;
return NULL;
}
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index b0b39ec..64b9ef0 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -186,6 +186,7 @@ protected:
RGWRados *store;
void *obj_ctx;
bool is_complete;
+ string bucket_owner;
virtual int do_complete(string& etag, time_t *mtime, time_t set_mtime, map<string, bufferlist>& attrs) = 0;
@@ -195,7 +196,7 @@ protected:
objs.push_back(obj);
}
public:
- RGWPutObjProcessor() : store(NULL), obj_ctx(NULL), is_complete(false) {}
+ RGWPutObjProcessor(const string& _bo) : store(NULL), obj_ctx(NULL), is_complete(false), bucket_owner(_bo) {}
virtual ~RGWPutObjProcessor();
virtual int prepare(RGWRados *_store, void *_o) {
store = _store;
@@ -223,7 +224,8 @@ protected:
public:
int throttle_data(void *handle) { return 0; }
- RGWPutObjProcessor_Plain(rgw_bucket& b, const string& o) : bucket(b), obj_str(o), ofs(0) {}
+ RGWPutObjProcessor_Plain(const string& bucket_owner, rgw_bucket& b, const string& o) : RGWPutObjProcessor(bucket_owner),
+ bucket(b), obj_str(o), ofs(0) {}
};
struct put_obj_aio_info {
@@ -248,7 +250,7 @@ protected:
public:
int throttle_data(void *handle);
- RGWPutObjProcessor_Aio() : max_chunks(RGW_MAX_PENDING_CHUNKS), obj_len(0) {}
+ RGWPutObjProcessor_Aio(const string& bucket_owner) : RGWPutObjProcessor(bucket_owner), max_chunks(RGW_MAX_PENDING_CHUNKS), obj_len(0) {}
virtual ~RGWPutObjProcessor_Aio() {
drain_pending();
}
@@ -288,7 +290,9 @@ protected:
public:
~RGWPutObjProcessor_Atomic() {}
- RGWPutObjProcessor_Atomic(rgw_bucket& _b, const string& _o, uint64_t _p, const string& _t) : part_size(_p),
+ RGWPutObjProcessor_Atomic(const string& bucket_owner, rgw_bucket& _b, const string& _o, uint64_t _p, const string& _t) :
+ RGWPutObjProcessor_Aio(bucket_owner),
+ part_size(_p),
cur_part_ofs(0),
next_part_ofs(_p),
cur_part_id(0),
@@ -640,6 +644,7 @@ struct RGWRegionMap {
string master_region;
RGWQuotaInfo bucket_quota;
+ RGWQuotaInfo user_quota;
RGWRegionMap() : lock("RGWRegionMap") {}
@@ -769,14 +774,14 @@ protected:
rgw_bucket bucket;
uint64_t bucket_ver;
uint64_t master_ver;
- map<RGWObjCategory, RGWBucketStats> *stats;
+ map<RGWObjCategory, RGWStorageStats> *stats;
string max_marker;
public:
RGWGetBucketStats_CB(rgw_bucket& _bucket) : bucket(_bucket), stats(NULL) {}
virtual ~RGWGetBucketStats_CB() {}
virtual void handle_response(int r) = 0;
virtual void set_response(uint64_t _bucket_ver, uint64_t _master_ver,
- map<RGWObjCategory, RGWBucketStats> *_stats,
+ map<RGWObjCategory, RGWStorageStats> *_stats,
const string &_max_marker) {
bucket_ver = _bucket_ver;
master_ver = _master_ver;
@@ -785,7 +790,21 @@ public:
}
};
+class RGWGetUserStats_CB : public RefCountedObject {
+protected:
+ string user;
+ RGWStorageStats stats;
+public:
+ RGWGetUserStats_CB(const string& _user) : user(_user) {}
+ virtual ~RGWGetUserStats_CB() {}
+ virtual void handle_response(int r) = 0;
+ virtual void set_response(RGWStorageStats& _stats) {
+ stats = _stats;
+ }
+};
+
class RGWGetDirHeader_CB;
+class RGWGetUserHeader_CB;
class RGWRados
@@ -824,6 +843,7 @@ class RGWRados
RGWGC *gc;
bool use_gc_thread;
+ bool quota_threads;
int num_watchers;
RGWWatcher **watchers;
@@ -874,13 +894,14 @@ class RGWRados
v.push_back(info);
return clone_objs(ctx, dst_obj, v, attrs, category, pmtime, true, false);
}
- int delete_obj_impl(void *ctx, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker);
int complete_atomic_overwrite(RGWRadosCtx *rctx, RGWObjState *state, rgw_obj& obj);
int update_placement_map();
int store_bucket_info(RGWBucketInfo& info, map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker, bool exclusive);
protected:
+ virtual int delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker);
+
CephContext *cct;
librados::Rados *rados;
librados::IoCtx gc_pool_ctx; // .rgw.gc
@@ -894,7 +915,7 @@ protected:
public:
RGWRados() : lock("rados_timer_lock"), timer(NULL),
- gc(NULL), use_gc_thread(false),
+ gc(NULL), use_gc_thread(false), quota_threads(false),
num_watchers(0), watchers(NULL), watch_handles(NULL),
watch_initialized(false),
bucket_id_lock("rados_bucket_id"), max_bucket_id(0),
@@ -947,9 +968,10 @@ public:
CephContext *ctx() { return cct; }
/** do all necessary setup of the storage device */
- int initialize(CephContext *_cct, bool _use_gc_thread) {
+ int initialize(CephContext *_cct, bool _use_gc_thread, bool _quota_threads) {
set_context(_cct);
use_gc_thread = _use_gc_thread;
+ quota_threads = _quota_threads;
return initialize();
}
/** Initialize the RADOS instance and prepare to do other ops */
@@ -1038,6 +1060,7 @@ public:
bool modify_version;
RGWObjVersionTracker *objv_tracker;
time_t set_mtime;
+ string owner;
PutObjMetaExtraParams() : mtime(NULL), rmattrs(NULL),
data(NULL), manifest(NULL), ptag(NULL),
@@ -1051,21 +1074,22 @@ public:
map<std::string, bufferlist>* rmattrs, const bufferlist *data,
RGWObjManifest *manifest, const string *ptag, list<string> *remove_objs,
bool modify_version, RGWObjVersionTracker *objv_tracker,
- time_t set_mtime /* 0 for don't set */);
+ time_t set_mtime /* 0 for don't set */,
+ const string& owner);
virtual int put_obj_meta(void *ctx, rgw_obj& obj, uint64_t size, time_t *mtime,
map<std::string, bufferlist>& attrs, RGWObjCategory category, int flags,
- const bufferlist *data = NULL) {
+ const string& owner, const bufferlist *data = NULL) {
return put_obj_meta_impl(ctx, obj, size, mtime, attrs, category, flags,
NULL, data, NULL, NULL, NULL,
- false, NULL, 0);
+ false, NULL, 0, owner);
}
virtual int put_obj_meta(void *ctx, rgw_obj& obj, uint64_t size, map<std::string, bufferlist>& attrs,
RGWObjCategory category, int flags, PutObjMetaExtraParams& params) {
return put_obj_meta_impl(ctx, obj, size, params.mtime, attrs, category, flags,
params.rmattrs, params.data, params.manifest, params.ptag, params.remove_objs,
- params.modify_version, params.objv_tracker, params.set_mtime);
+ params.modify_version, params.objv_tracker, params.set_mtime, params.owner);
}
virtual int put_obj_data(void *ctx, rgw_obj& obj, const char *data,
@@ -1161,6 +1185,7 @@ public:
void *progress_data);
int copy_obj_data(void *ctx,
+ const string& owner,
void **handle, off_t end,
rgw_obj& dest_obj,
rgw_obj& src_obj,
@@ -1181,7 +1206,8 @@ public:
int bucket_suspended(rgw_bucket& bucket, bool *suspended);
/** Delete an object.*/
- virtual int delete_obj(void *ctx, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker = NULL);
+ virtual int delete_obj(void *ctx, const string& bucket_owner, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker = NULL);
+ virtual int delete_system_obj(void *ctx, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker = NULL);
/** Remove an object from the bucket index */
int delete_obj_index(rgw_obj& obj);
@@ -1321,9 +1347,11 @@ public:
}
int decode_policy(bufferlist& bl, ACLOwner *owner);
- int get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWBucketStats>& stats,
+ int get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWStorageStats>& stats,
string *max_marker);
int get_bucket_stats_async(rgw_bucket& bucket, RGWGetBucketStats_CB *cb);
+ int get_user_stats(const string& user, RGWStorageStats& stats);
+ int get_user_stats_async(const string& user, RGWGetUserStats_CB *cb);
void get_bucket_instance_obj(rgw_bucket& bucket, rgw_obj& obj);
void get_bucket_instance_entry(rgw_bucket& bucket, string& entry);
void get_bucket_meta_oid(rgw_bucket& bucket, string& oid);
@@ -1337,7 +1365,8 @@ public:
int get_bucket_instance_info(void *ctx, rgw_bucket& bucket, RGWBucketInfo& info, time_t *pmtime, map<string, bufferlist> *pattrs);
int get_bucket_instance_from_oid(void *ctx, string& oid, RGWBucketInfo& info, time_t *pmtime, map<string, bufferlist> *pattrs);
- virtual int get_bucket_info(void *ctx, string& bucket_name, RGWBucketInfo& info,
+ int convert_old_bucket_info(void *ctx, string& bucket_name);
+ virtual int get_bucket_info(void *ctx, const string& bucket_name, RGWBucketInfo& info,
time_t *pmtime, map<string, bufferlist> *pattrs = NULL);
virtual int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, time_t mtime, obj_version *pep_objv,
map<string, bufferlist> *pattrs, bool create_entry_point);
@@ -1407,12 +1436,28 @@ public:
int defer_gc(void *ctx, rgw_obj& obj);
int bucket_check_index(rgw_bucket& bucket,
- map<RGWObjCategory, RGWBucketStats> *existing_stats,
- map<RGWObjCategory, RGWBucketStats> *calculated_stats);
+ map<RGWObjCategory, RGWStorageStats> *existing_stats,
+ map<RGWObjCategory, RGWStorageStats> *calculated_stats);
int bucket_rebuild_index(rgw_bucket& bucket);
int remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list);
- int check_quota(rgw_bucket& bucket, RGWQuotaInfo& quota_info, uint64_t obj_size);
+ int cls_user_get_header(const string& user_id, cls_user_header *header);
+ int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);
+ int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket);
+ int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
+ int cls_user_list_buckets(rgw_obj& obj,
+ const string& in_marker, int max_entries,
+ list<cls_user_bucket_entry>& entries,
+ string *out_marker, bool *truncated);
+ int cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry);
+ int cls_user_update_buckets(rgw_obj& obj, list<cls_user_bucket_entry>& entries, bool add);
+ int cls_user_complete_stats_sync(rgw_obj& obj);
+ int complete_sync_user_stats(const string& user_id);
+ int cls_user_add_bucket(rgw_obj& obj, list<cls_user_bucket_entry>& entries);
+ int cls_user_remove_bucket(rgw_obj& obj, const cls_user_bucket& bucket);
+
+ int check_quota(const string& bucket_owner, rgw_bucket& bucket,
+ RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size);
string unique_id(uint64_t unique_num) {
char buf[32];
@@ -1487,15 +1532,15 @@ public:
class RGWStoreManager {
public:
RGWStoreManager() {}
- static RGWRados *get_storage(CephContext *cct, bool use_gc_thread) {
- RGWRados *store = init_storage_provider(cct, use_gc_thread);
+ static RGWRados *get_storage(CephContext *cct, bool use_gc_thread, bool quota_threads) {
+ RGWRados *store = init_storage_provider(cct, use_gc_thread, quota_threads);
return store;
}
static RGWRados *get_raw_storage(CephContext *cct) {
RGWRados *store = init_raw_storage_provider(cct);
return store;
}
- static RGWRados *init_storage_provider(CephContext *cct, bool use_gc_thread);
+ static RGWRados *init_storage_provider(CephContext *cct, bool use_gc_thread, bool quota_threads);
static RGWRados *init_raw_storage_provider(CephContext *cct);
static void close_storage(RGWRados *store);
diff --git a/src/rgw/rgw_resolve.cc b/src/rgw/rgw_resolve.cc
index 9a8b14b..471ac3f 100644
--- a/src/rgw/rgw_resolve.cc
+++ b/src/rgw/rgw_resolve.cc
@@ -3,6 +3,12 @@
#include <arpa/nameser.h>
#include <resolv.h>
+#include "acconfig.h"
+
+#ifdef HAVE_ARPA_NAMESER_COMPAT_H
+#include <arpa/nameser_compat.h>
+#endif
+
#include "rgw_common.h"
#include "rgw_resolve.h"
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 30cb0d9..2201d4f 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -13,6 +13,7 @@
#include "rgw_rest_s3.h"
#include "rgw_swift_auth.h"
#include "rgw_cors_s3.h"
+#include "rgw_http_errors.h"
#include "rgw_client_io.h"
#include "rgw_resolve.h"
@@ -59,11 +60,13 @@ struct generic_attr generic_attrs[] = {
map<string, string> rgw_to_http_attrs;
static map<string, string> generic_attrs_map;
+map<int, const char *> http_status_names;
/*
* make attrs look_like_this
+ * converts dashes to underscores
*/
-string lowercase_http_attr(const string& orig)
+string lowercase_underscore_http_attr(const string& orig)
{
const char *s = orig.c_str();
char buf[orig.size() + 1];
@@ -83,8 +86,9 @@ string lowercase_http_attr(const string& orig)
/*
* make attrs LOOK_LIKE_THIS
+ * converts dashes to underscores
*/
-string uppercase_http_attr(const string& orig)
+string uppercase_underscore_http_attr(const string& orig)
{
const char *s = orig.c_str();
char buf[orig.size() + 1];
@@ -104,6 +108,7 @@ string uppercase_http_attr(const string& orig)
/*
* make attrs Look-Like-This
+ * converts underscores to dashes
*/
string camelcase_dash_http_attr(const string& orig)
{
@@ -146,22 +151,26 @@ void rgw_rest_init(CephContext *cct)
list<string>::iterator iter;
for (iter = extended_http_attrs.begin(); iter != extended_http_attrs.end(); ++iter) {
string rgw_attr = RGW_ATTR_PREFIX;
- rgw_attr.append(lowercase_http_attr(*iter));
+ rgw_attr.append(lowercase_underscore_http_attr(*iter));
rgw_to_http_attrs[rgw_attr] = camelcase_dash_http_attr(*iter);
string http_header = "HTTP_";
- http_header.append(uppercase_http_attr(*iter));
+ http_header.append(uppercase_underscore_http_attr(*iter));
generic_attrs_map[http_header] = rgw_attr;
}
+
+ for (const struct rgw_http_status_code *h = http_codes; h->code; h++) {
+ http_status_names[h->code] = h->name;
+ }
}
-static void dump_status(struct req_state *s, const char *status)
+static void dump_status(struct req_state *s, const char *status, const char *status_name)
{
- int r = s->cio->print("Status: %s\n", status);
+ int r = s->cio->send_status(status, status_name);
if (r < 0) {
- ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+ ldout(s->cct, 0) << "ERROR: s->cio->send_status() returned err=" << r << dendl;
}
}
@@ -193,7 +202,7 @@ void set_req_state_err(struct req_state *s, int err_no)
if (err_no < 0)
err_no = -err_no;
- s->err.ret = err_no;
+ s->err.ret = -err_no;
if (s->prot_flags & RGW_REST_SWIFT) {
r = search_err(err_no, RGW_HTTP_SWIFT_ERRORS, ARRAY_LEN(RGW_HTTP_SWIFT_ERRORS));
if (r) {
@@ -218,21 +227,19 @@ void dump_errno(struct req_state *s)
{
char buf[32];
snprintf(buf, sizeof(buf), "%d", s->err.http_ret);
- dump_status(s, buf);
+ dump_status(s, buf, http_status_names[s->err.http_ret]);
}
void dump_errno(struct req_state *s, int err)
{
char buf[32];
snprintf(buf, sizeof(buf), "%d", err);
- dump_status(s, buf);
+ dump_status(s, buf, http_status_names[s->err.http_ret]);
}
void dump_content_length(struct req_state *s, uint64_t len)
{
- char buf[21];
- snprintf(buf, sizeof(buf), "%"PRIu64, len);
- int r = s->cio->print("Content-Length: %s\n", buf);
+ int r = s->cio->send_content_length(len);
if (r < 0) {
ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
}
@@ -262,8 +269,11 @@ void dump_pair(struct req_state *s, const char *key, const char *value)
void dump_bucket_from_state(struct req_state *s)
{
- if (!s->bucket_name_str.empty())
- s->cio->print("Bucket: \"%s\"\n", s->bucket_name_str.c_str());
+ int expose_bucket = g_conf->rgw_expose_bucket;
+ if (expose_bucket) {
+ if (!s->bucket_name_str.empty())
+ s->cio->print("Bucket: \"%s\"\n", s->bucket_name_str.c_str());
+ }
}
void dump_object_from_state(struct req_state *s)
@@ -434,10 +444,15 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type)
s->formatter->close_section();
dump_content_length(s, s->formatter->get_len());
}
- int r = s->cio->print("Content-type: %s\r\n\r\n", content_type);
+ int r = s->cio->print("Content-type: %s\r\n", content_type);
if (r < 0) {
ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
}
+ r = s->cio->complete_header();
+ if (r < 0) {
+ ldout(s->cct, 0) << "ERROR: s->cio->complete_header() returned err=" << r << dendl;
+ }
+
s->cio->set_account(true);
rgw_flush_formatter_and_reset(s, s->formatter);
}
@@ -450,6 +465,7 @@ void abort_early(struct req_state *s, RGWOp *op, int err_no)
}
set_req_state_err(s, err_no);
dump_errno(s);
+ dump_bucket_from_state(s);
end_header(s, op);
rgw_flush_formatter_and_reset(s, s->formatter);
perfcounter->inc(l_rgw_failed_req);
@@ -457,8 +473,7 @@ void abort_early(struct req_state *s, RGWOp *op, int err_no)
void dump_continue(struct req_state *s)
{
- dump_status(s, "100");
- s->cio->flush();
+ s->cio->send_100_continue();
}
void dump_range(struct req_state *s, uint64_t ofs, uint64_t end, uint64_t total)
@@ -875,11 +890,19 @@ int RGWListMultipart_ObjStore::get_params()
if (upload_id.empty()) {
ret = -ENOTSUP;
}
- string str = s->info.args.get("part-number-marker");
- if (!str.empty())
- marker = atoi(str.c_str());
+ string marker_str = s->info.args.get("part-number-marker");
+
+ if (!marker_str.empty()) {
+ string err;
+ marker = strict_strtol(marker_str.c_str(), 10, &err);
+ if (!err.empty()) {
+ ldout(s->cct, 20) << "bad marker: " << marker << dendl;
+ ret = -EINVAL;
+ return ret;
+ }
+ }
- str = s->info.args.get("max-parts");
+ string str = s->info.args.get("max-parts");
if (!str.empty())
max_parts = atoi(str.c_str());
diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 15ac863..38ffd8c 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -2,6 +2,8 @@
#define CEPH_RGW_REST_H
#define TIME_BUF_SIZE 128
+#include "common/ceph_json.h"
+#include "include/assert.h" /* needed because of common/ceph_json.h */
#include "rgw_op.h"
#include "rgw_formats.h"
@@ -18,6 +20,41 @@ extern void rgw_flush_formatter(struct req_state *s,
extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen, int max_len);
+template <class T>
+int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, int max_len, bool *empty)
+{
+ int rv, data_len;
+ char *data;
+
+ if (empty)
+ *empty = false;
+
+ if ((rv = rgw_rest_read_all_input(s, &data, &data_len, max_len)) < 0) {
+ return rv;
+ }
+
+ if (!data_len) {
+ if (empty) {
+ *empty = true;
+ }
+
+ return -EINVAL;
+ }
+
+ JSONParser parser;
+
+ if (!parser.parse(data, data_len)) {
+ free(data);
+ return -EINVAL;
+ }
+
+ decode_json_obj(out, &parser);
+
+ free(data);
+ return 0;
+}
+
+
class RESTArgs {
public:
static int get_string(struct req_state *s, const string& name, const string& def_val, string *val, bool *existed = NULL);
@@ -128,6 +165,12 @@ public:
~RGWPutMetadata_ObjStore() {}
};
+class RGWSetTempUrl_ObjStore : public RGWSetTempUrl {
+public:
+ RGWSetTempUrl_ObjStore() {}
+ ~RGWSetTempUrl_ObjStore() {}
+};
+
class RGWDeleteObj_ObjStore : public RGWDeleteObj {
public:
RGWDeleteObj_ObjStore() {}
diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc
index 7f4e2b6..256db77 100644
--- a/src/rgw/rgw_rest_client.cc
+++ b/src/rgw/rgw_rest_client.cc
@@ -171,7 +171,7 @@ void RGWRESTSimpleRequest::get_params_str(map<string, string>& extra_args, strin
int RGWRESTSimpleRequest::sign_request(RGWAccessKey& key, RGWEnv& env, req_info& info)
{
- map<string, string>& m = env.get_map();
+ map<string, string, ltstr_nocase>& m = env.get_map();
map<string, string>::iterator i;
for (i = m.begin(); i != m.end(); ++i) {
@@ -218,7 +218,7 @@ int RGWRESTSimpleRequest::forward_request(RGWAccessKey& key, req_info& info, siz
return ret;
}
- map<string, string>& m = new_env.get_map();
+ map<string, string, ltstr_nocase>& m = new_env.get_map();
map<string, string>::iterator iter;
for (iter = m.begin(); iter != m.end(); ++iter) {
headers.push_back(make_pair<string, string>(iter->first, iter->second));
@@ -364,7 +364,7 @@ static void grants_by_type_add_perm(map<int, string>& grants_by_type, int perm,
}
}
-static void add_grants_headers(map<int, string>& grants, map<string, string>& attrs, map<string, string>& meta_map)
+static void add_grants_headers(map<int, string>& grants, map<string, string, ltstr_nocase>& attrs, map<string, string>& meta_map)
{
struct grant_type_to_header *t;
@@ -405,7 +405,7 @@ int RGWRESTStreamWriteRequest::put_obj_init(RGWAccessKey& key, rgw_obj& obj, uin
new_info.request_uri = new_info.script_uri;
new_info.effective_uri = new_info.effective_uri;
- map<string, string>& m = new_env.get_map();
+ map<string, string, ltstr_nocase>& m = new_env.get_map();
map<string, bufferlist>::iterator bliter;
/* merge send headers */
@@ -582,7 +582,7 @@ int RGWRESTStreamReadRequest::get_obj(RGWAccessKey& key, map<string, string>& ex
return ret;
}
- map<string, string>& m = new_env.get_map();
+ map<string, string, ltstr_nocase>& m = new_env.get_map();
map<string, string>::iterator iter;
for (iter = m.begin(); iter != m.end(); ++iter) {
headers.push_back(make_pair<string, string>(iter->first, iter->second));
diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index 35a8ac2..32bf0ea 100644
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -92,7 +92,7 @@ int RGWRESTConn::get_obj(const string& uid, req_info *info /* optional */, rgw_o
*req = new RGWRESTStreamReadRequest(cct, url, cb, NULL, ¶ms);
map<string, string> extra_headers;
if (info) {
- map<string, string>& orig_map = info->env->get_map();
+ map<string, string, ltstr_nocase>& orig_map = info->env->get_map();
/* add original headers that start with HTTP_X_AMZ_ */
#define SEARCH_AMZ_PREFIX "HTTP_X_AMZ_"
diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 2de424f..9f32fc9 100644
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -379,7 +379,7 @@ void RGWOp_BILog_Info::execute() {
return;
}
}
- map<RGWObjCategory, RGWBucketStats> stats;
+ map<RGWObjCategory, RGWStorageStats> stats;
int ret = store->get_bucket_stats(bucket_info.bucket, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0 && ret != -ENOENT) {
http_ret = ret;
diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc
index 5036235..afd5c7b 100644
--- a/src/rgw/rgw_rest_metadata.cc
+++ b/src/rgw/rgw_rest_metadata.cc
@@ -31,7 +31,6 @@ static inline void frame_metadata_key(req_state *s, string& out) {
bool exists;
string key = s->info.args.get("key", &exists);
- string metadata_key;
string section;
if (!s->bucket_name_str.empty()) {
section = s->bucket_name_str;
diff --git a/src/rgw/rgw_rest_replica_log.cc b/src/rgw/rgw_rest_replica_log.cc
index 2543f32..e7dd962 100644
--- a/src/rgw/rgw_rest_replica_log.cc
+++ b/src/rgw/rgw_rest_replica_log.cc
@@ -38,29 +38,6 @@ static int parse_to_utime(string& in, utime_t& out) {
}
-template <class T>
-static int get_input(req_state *s, T& out) {
- int rv, data_len;
- char *data;
-
- if ((rv = rgw_rest_read_all_input(s, &data, &data_len, REPLICA_INPUT_MAX_LEN)) < 0) {
- dout(5) << "Error - reading input data - " << rv << dendl;
- return rv;
- }
-
- JSONParser parser;
-
- if (!parser.parse(data, data_len)) {
- free(data);
- return -EINVAL;
- }
-
- decode_json_obj(out, &parser);
-
- free(data);
- return 0;
-}
-
void RGWOp_OBJLog_SetBounds::execute() {
string id_str = s->info.args.get("id"),
marker = s->info.args.get("marker"),
@@ -97,7 +74,8 @@ void RGWOp_OBJLog_SetBounds::execute() {
bufferlist bl;
list<RGWReplicaItemMarker> markers;
- if ((http_ret = get_input(s, markers)) < 0) {
+ if ((http_ret = rgw_rest_get_json_input(store->ctx(), s, markers, REPLICA_INPUT_MAX_LEN, NULL)) < 0) {
+ dout(5) << "Error - retrieving input data - " << http_ret << dendl;
return;
}
@@ -211,7 +189,8 @@ void RGWOp_BILog_SetBounds::execute() {
bufferlist bl;
list<RGWReplicaItemMarker> markers;
- if ((http_ret = get_input(s, markers)) < 0) {
+ if ((http_ret = rgw_rest_get_json_input(store->ctx(), s, markers, REPLICA_INPUT_MAX_LEN, NULL)) < 0) {
+ dout(5) << "Error - retrieving input data - " << http_ret << dendl;
return;
}
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 12bd377..9dfede4 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -946,8 +946,8 @@ int RGWPostObj_ObjStore_S3::get_policy()
err_msg = "Missing access key";
return -EINVAL;
}
- string signature_str;
- if (!part_str("signature", &signature_str)) {
+ string received_signature_str;
+ if (!part_str("signature", &received_signature_str)) {
ldout(s->cct, 0) << "No signature found!" << dendl;
err_msg = "Missing signature";
return -EINVAL;
@@ -964,22 +964,29 @@ int RGWPostObj_ObjStore_S3::get_policy()
map<string, RGWAccessKey> access_keys = user_info.access_keys;
- map<string, RGWAccessKey>::const_iterator iter = access_keys.begin();
+ map<string, RGWAccessKey>::const_iterator iter = access_keys.find(s3_access_key);
+ // We know the key must exist, since the user was returned by
+ // rgw_get_user_info_by_access_key, but it doesn't hurt to check!
+ if (iter == access_keys.end()) {
+ ldout(s->cct, 0) << "Secret key lookup failed!" << dendl;
+ err_msg = "No secret key for matching access key";
+ return -EACCES;
+ }
string s3_secret_key = (iter->second).key;
- char calc_signature[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+ char expected_signature_char[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
- calc_hmac_sha1(s3_secret_key.c_str(), s3_secret_key.size(), encoded_policy.c_str(), encoded_policy.length(), calc_signature);
- bufferlist encoded_hmac;
- bufferlist raw_hmac;
- raw_hmac.append(calc_signature, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
- raw_hmac.encode_base64(encoded_hmac);
- encoded_hmac.append((char)0); /* null terminate */
+ calc_hmac_sha1(s3_secret_key.c_str(), s3_secret_key.size(), encoded_policy.c_str(), encoded_policy.length(), expected_signature_char);
+ bufferlist expected_signature_hmac_raw;
+ bufferlist expected_signature_hmac_encoded;
+ expected_signature_hmac_raw.append(expected_signature_char, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+ expected_signature_hmac_raw.encode_base64(expected_signature_hmac_encoded);
+ expected_signature_hmac_encoded.append((char)0); /* null terminate */
- if (signature_str.compare(encoded_hmac.c_str()) != 0) {
+ if (received_signature_str.compare(expected_signature_hmac_encoded.c_str()) != 0) {
ldout(s->cct, 0) << "Signature verification failed!" << dendl;
- ldout(s->cct, 0) << "expected: " << signature_str.c_str() << dendl;
- ldout(s->cct, 0) << "got: " << encoded_hmac.c_str() << dendl;
+ ldout(s->cct, 0) << "received: " << received_signature_str.c_str() << dendl;
+ ldout(s->cct, 0) << "expected: " << expected_signature_hmac_encoded.c_str() << dendl;
err_msg = "Bad access key / signature";
return -EACCES;
}
@@ -1157,7 +1164,9 @@ done:
s->err.message = err_msg;
set_req_state_err(s, ret);
dump_errno(s);
- dump_content_length(s, s->formatter->get_len());
+ if (ret >= 0) {
+ dump_content_length(s, s->formatter->get_len());
+ }
end_header(s, this);
if (ret != STATUS_CREATED)
return;
@@ -1515,11 +1524,13 @@ void RGWListMultipart_ObjStore_S3::send_response()
dump_start(s);
s->formatter->open_object_section_in_ns("ListMultipartUploadResult",
"http://s3.amazonaws.com/doc/2006-03-01/");
- map<uint32_t, RGWUploadPartInfo>::iterator iter, test_iter;
- int i, cur_max = 0;
+ map<uint32_t, RGWUploadPartInfo>::iterator iter;
+ map<uint32_t, RGWUploadPartInfo>::reverse_iterator test_iter;
+ int cur_max = 0;
- iter = parts.upper_bound(marker);
- for (i = 0, test_iter = iter; test_iter != parts.end() && i < max_parts; ++test_iter, ++i) {
+ iter = parts.begin();
+ test_iter = parts.rbegin();
+ if (test_iter != parts.rend()) {
cur_max = test_iter->first;
}
s->formatter->dump_string("Bucket", s->bucket_name_str);
@@ -1527,9 +1538,9 @@ void RGWListMultipart_ObjStore_S3::send_response()
s->formatter->dump_string("UploadId", upload_id);
s->formatter->dump_string("StorageClass", "STANDARD");
s->formatter->dump_int("PartNumberMarker", marker);
- s->formatter->dump_int("NextPartNumberMarker", cur_max + 1);
+ s->formatter->dump_int("NextPartNumberMarker", cur_max);
s->formatter->dump_int("MaxParts", max_parts);
- s->formatter->dump_string("IsTruncated", (test_iter == parts.end() ? "false" : "true"));
+ s->formatter->dump_string("IsTruncated", (truncated ? "true" : "false"));
ACLOwner& owner = policy.get_owner();
dump_owner(s, owner.get_id(), owner.get_display_name());
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 95d54ae..6524db8 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -425,6 +425,36 @@ void RGWPutMetadata_ObjStore_SWIFT::send_response()
rgw_flush_formatter_and_reset(s, s->formatter);
}
+int RGWSetTempUrl_ObjStore_SWIFT::get_params()
+{
+ const char *temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY");
+ if (temp_url) {
+ temp_url_keys[0] = temp_url;
+ }
+
+ temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY_2");
+ if (temp_url) {
+ temp_url_keys[1] = temp_url;
+ }
+
+ if (temp_url_keys.size() == 0)
+ return -EINVAL;
+
+ return 0;
+}
+
+void RGWSetTempUrl_ObjStore_SWIFT::send_response()
+{
+ int r = ret;
+ if (!r)
+ r = STATUS_NO_CONTENT;
+
+ set_req_state_err(s, r);
+ dump_errno(s);
+ end_header(s, this);
+ rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
void RGWDeleteObj_ObjStore_SWIFT::send_response()
{
int r = ret;
@@ -600,6 +630,19 @@ RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_head()
return new RGWStatAccount_ObjStore_SWIFT;
}
+RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_post()
+{
+ const char *temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY");
+ if (temp_url) {
+ return new RGWSetTempUrl_ObjStore_SWIFT;
+ }
+ temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY_2");
+ if (temp_url) {
+ return new RGWSetTempUrl_ObjStore_SWIFT;
+ }
+ return NULL;
+}
+
RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::get_obj_op(bool get_data)
{
if (is_acl_op()) {
@@ -711,7 +754,7 @@ RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_options()
int RGWHandler_ObjStore_SWIFT::authorize()
{
- if (!s->os_auth_token) {
+ if (!s->os_auth_token && s->info.args.get("temp_url_sig").empty()) {
/* anonymous access */
rgw_get_anon_user(s->user);
s->perm_mask = RGW_PERM_FULL_CONTROL;
@@ -816,9 +859,15 @@ int RGWHandler_ObjStore_SWIFT::init_from_header(struct req_state *s)
first = req;
}
+ string tenant_path;
+ if (!g_conf->rgw_swift_tenant_name.empty()) {
+ tenant_path = "/AUTH_";
+ tenant_path.append(g_conf->rgw_swift_tenant_name);
+ }
+
/* verify that the request_uri conforms with what's expected */
- char buf[g_conf->rgw_swift_url_prefix.length() + 16];
- int blen = sprintf(buf, "/%s/v1", g_conf->rgw_swift_url_prefix.c_str());
+ char buf[g_conf->rgw_swift_url_prefix.length() + 16 + tenant_path.length()];
+ int blen = sprintf(buf, "/%s/v1%s", g_conf->rgw_swift_url_prefix.c_str(), tenant_path.c_str());
if (s->decoded_uri[0] != '/' ||
s->decoded_uri.compare(0, blen, buf) != 0) {
return -ENOENT;
@@ -831,6 +880,12 @@ int RGWHandler_ObjStore_SWIFT::init_from_header(struct req_state *s)
string ver;
next_tok(req, ver, '/');
+
+ string tenant;
+ if (!tenant_path.empty()) {
+ next_tok(req, tenant, '/');
+ }
+
s->os_auth_token = s->info.env->get("HTTP_X_AUTH_TOKEN");
next_tok(req, first, '/');
diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h
index 1c23ab2..27a31aa 100644
--- a/src/rgw/rgw_rest_swift.h
+++ b/src/rgw/rgw_rest_swift.h
@@ -91,6 +91,15 @@ public:
void send_response();
};
+class RGWSetTempUrl_ObjStore_SWIFT : public RGWSetTempUrl_ObjStore {
+public:
+ RGWSetTempUrl_ObjStore_SWIFT() {}
+ ~RGWSetTempUrl_ObjStore_SWIFT() {}
+
+ int get_params();
+ void send_response();
+};
+
class RGWDeleteObj_ObjStore_SWIFT : public RGWDeleteObj_ObjStore {
public:
RGWDeleteObj_ObjStore_SWIFT() {}
@@ -160,6 +169,7 @@ class RGWHandler_ObjStore_Service_SWIFT : public RGWHandler_ObjStore_SWIFT {
protected:
RGWOp *op_get();
RGWOp *op_head();
+ RGWOp *op_post();
public:
RGWHandler_ObjStore_Service_SWIFT() {}
virtual ~RGWHandler_ObjStore_Service_SWIFT() {}
diff --git a/src/rgw/rgw_rest_user.cc b/src/rgw/rgw_rest_user.cc
index 3d08e40..0613c49 100644
--- a/src/rgw/rgw_rest_user.cc
+++ b/src/rgw/rgw_rest_user.cc
@@ -1,3 +1,5 @@
+#include "common/ceph_json.h"
+
#include "rgw_op.h"
#include "rgw_user.h"
#include "rgw_rest_user.h"
@@ -609,8 +611,268 @@ void RGWOp_Caps_Remove::execute()
http_ret = RGWUserAdminOp_Caps::remove(store, op_state, flusher);
}
+struct UserQuotas {
+ RGWQuotaInfo bucket_quota;
+ RGWQuotaInfo user_quota;
+
+ UserQuotas() {}
+
+ UserQuotas(RGWUserInfo& info) {
+ bucket_quota = info.bucket_quota;
+ user_quota = info.user_quota;
+ }
+ void dump(Formatter *f) const {
+ encode_json("bucket_quota", bucket_quota, f);
+ encode_json("user_quota", user_quota, f);
+ }
+ void decode_json(JSONObj *obj) {
+ JSONDecoder::decode_json("bucket_quota", bucket_quota, obj);
+ JSONDecoder::decode_json("user_quota", user_quota, obj);
+ }
+};
+
+class RGWOp_Quota_Info : public RGWRESTOp {
+
+public:
+ RGWOp_Quota_Info() {}
+
+ int check_caps(RGWUserCaps& caps) {
+ return caps.check_cap("users", RGW_CAP_READ);
+ }
+
+ void execute();
+
+ virtual const string name() { return "get_quota_info"; }
+};
+
+
+void RGWOp_Quota_Info::execute()
+{
+ RGWUserAdminOpState op_state;
+
+ std::string uid;
+ std::string quota_type;
+
+ RESTArgs::get_string(s, "uid", uid, &uid);
+ RESTArgs::get_string(s, "quota-type", quota_type, "a_type);
+
+ if (uid.empty()) {
+ http_ret = -EINVAL;
+ return;
+ }
+
+ bool show_all = quota_type.empty();
+ bool show_bucket = show_all || (quota_type == "bucket");
+ bool show_user = show_all || (quota_type == "user");
+
+ if (!(show_all || show_bucket || show_user)) {
+ http_ret = -EINVAL;
+ return;
+ }
+
+ op_state.set_user_id(uid);
+
+ RGWUser user;
+ http_ret = user.init(store, op_state);
+ if (http_ret < 0)
+ return;
+
+ RGWUserInfo info;
+ string err_msg;
+ http_ret = user.info(info, &err_msg);
+ if (http_ret < 0)
+ return;
+
+ if (show_all) {
+ UserQuotas quotas(info);
+ encode_json("quota", quotas, s->formatter);
+ } else if (show_user) {
+ encode_json("user_quota", info.user_quota, s->formatter);
+ } else {
+ encode_json("bucket_quota", info.bucket_quota, s->formatter);
+ }
+
+ flusher.flush();
+}
+
+class RGWOp_Quota_Set : public RGWRESTOp {
+
+public:
+ RGWOp_Quota_Set() {}
+
+ int check_caps(RGWUserCaps& caps) {
+ return caps.check_cap("users", RGW_CAP_WRITE);
+ }
+
+ void execute();
+
+ virtual const string name() { return "set_quota_info"; }
+};
+
+/**
+ * set quota
+ *
+ * two different ways to set the quota info: as json struct in the message body or via http params.
+ *
+ * as json:
+ *
+ * PUT /admin/user?uid=<uid>["a-type=<type>]
+ *
+ * whereas quota-type is optional and is either user, or bucket
+ *
+ * if quota-type is not specified then we expect to get a structure that contains both quotas,
+ * otherwise we'll only get the relevant configuration.
+ *
+ * E.g., if quota type not specified:
+ * {
+ * "user_quota" : {
+ * "max_size_kb" : 4096,
+ * "max_objects" : -1,
+ * "enabled" : false
+ * },
+ * "bucket_quota" : {
+ * "max_size_kb" : 1024,
+ * "max_objects" : -1,
+ * "enabled" : true
+ * }
+ * }
+ *
+ *
+ * or if quota type is specified:
+ * {
+ * "max_size_kb" : 4096,
+ * "max_objects" : -1,
+ * "enabled" : false
+ * }
+ *
+ * Another option is not to pass any body and set the following http params:
+ *
+ *
+ * max-size-kb=<size>
+ * max-objects=<max objects>
+ * enabled[={true,false}]
+ *
+ * all params are optionals and default to the current settings. With this type of configuration the
+ * quota-type param is mandatory.
+ *
+ */
+
+void RGWOp_Quota_Set::execute()
+{
+ RGWUserAdminOpState op_state;
+
+ std::string uid;
+ std::string quota_type;
+
+ RESTArgs::get_string(s, "uid", uid, &uid);
+ RESTArgs::get_string(s, "quota-type", quota_type, "a_type);
+
+ if (uid.empty()) {
+ http_ret = -EINVAL;
+ return;
+ }
+
+ bool set_all = quota_type.empty();
+ bool set_bucket = set_all || (quota_type == "bucket");
+ bool set_user = set_all || (quota_type == "user");
+
+ if (!(set_all || set_bucket || set_user)) {
+ ldout(store->ctx(), 20) << "invalid quota type" << dendl;
+ http_ret = -EINVAL;
+ return;
+ }
+
+ bool use_http_params;
+
+ if (s->length > 0) {
+ use_http_params = false;
+ } else {
+ const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
+ use_http_params = (!encoding || strcmp(encoding, "chunked") != 0);
+ }
+
+ if (use_http_params && set_all) {
+ ldout(store->ctx(), 20) << "quota type was not specified, can't set all quotas via http headers" << dendl;
+ http_ret = -EINVAL;
+ return;
+ }
+
+ op_state.set_user_id(uid);
+
+ RGWUser user;
+ http_ret = user.init(store, op_state);
+ if (http_ret < 0) {
+ ldout(store->ctx(), 20) << "failed initializing user info: " << http_ret << dendl;
+ return;
+ }
+
+#define QUOTA_INPUT_MAX_LEN 1024
+ if (set_all) {
+ UserQuotas quotas;
+
+ if ((http_ret = rgw_rest_get_json_input(store->ctx(), s, quotas, QUOTA_INPUT_MAX_LEN, NULL)) < 0) {
+ ldout(store->ctx(), 20) << "failed to retrieve input" << dendl;
+ return;
+ }
+
+ op_state.set_user_quota(quotas.user_quota);
+ op_state.set_bucket_quota(quotas.bucket_quota);
+ } else {
+ RGWQuotaInfo quota;
+
+ if (!use_http_params) {
+ bool empty;
+ http_ret = rgw_rest_get_json_input(store->ctx(), s, quota, QUOTA_INPUT_MAX_LEN, &empty);
+ if (http_ret < 0) {
+ ldout(store->ctx(), 20) << "failed to retrieve input" << dendl;
+ if (!empty)
+ return;
+
+ /* was probably chunked input, but no content provided, configure via http params */
+ use_http_params = true;
+ }
+ }
+
+ if (use_http_params) {
+ RGWUserInfo info;
+ string err_msg;
+ http_ret = user.info(info, &err_msg);
+ if (http_ret < 0) {
+ ldout(store->ctx(), 20) << "failed to get user info: " << http_ret << dendl;
+ return;
+ }
+ RGWQuotaInfo *old_quota;
+ if (set_user) {
+ old_quota = &info.user_quota;
+ } else {
+ old_quota = &info.bucket_quota;
+ }
+
+ RESTArgs::get_int64(s, "max-objects", old_quota->max_objects, "a.max_objects);
+ RESTArgs::get_int64(s, "max-size-kb", old_quota->max_size_kb, "a.max_size_kb);
+ RESTArgs::get_bool(s, "enabled", old_quota->enabled, "a.enabled);
+ }
+
+ if (set_user) {
+ op_state.set_user_quota(quota);
+ } else {
+ op_state.set_bucket_quota(quota);
+ }
+ }
+
+ string err;
+ http_ret = user.modify(op_state, &err);
+ if (http_ret < 0) {
+ ldout(store->ctx(), 20) << "failed updating user info: " << http_ret << ": " << err << dendl;
+ return;
+ }
+}
+
RGWOp *RGWHandler_User::op_get()
{
+ if (s->info.args.sub_resource_exists("quota"))
+ return new RGWOp_Quota_Info;
+
return new RGWOp_User_Info;
};
@@ -625,6 +887,9 @@ RGWOp *RGWHandler_User::op_put()
if (s->info.args.sub_resource_exists("caps"))
return new RGWOp_Caps_Add;
+ if (s->info.args.sub_resource_exists("quota"))
+ return new RGWOp_Quota_Set;
+
return new RGWOp_User_Create;
};
diff --git a/src/rgw/rgw_swift.cc b/src/rgw/rgw_swift.cc
index 24e0905..1ae8f94 100644
--- a/src/rgw/rgw_swift.cc
+++ b/src/rgw/rgw_swift.cc
@@ -104,32 +104,44 @@ int RGWSwift::validate_token(const char *token, struct rgw_swift_auth_info *info
return 0;
}
-
-
-class RGWValidateKeystoneToken : public RGWHTTPClient {
+class RGWPostHTTPData : public RGWHTTPClient {
bufferlist *bl;
+ std::string post_data;
+ size_t post_data_index;
public:
- RGWValidateKeystoneToken(CephContext *_cct, bufferlist *_bl) : RGWHTTPClient(_cct), bl(_bl) {}
+ RGWPostHTTPData(CephContext *_cct, bufferlist *_bl) : RGWHTTPClient(_cct), bl(_bl), post_data_index(0) {}
+
+ void set_post_data(const std::string& _post_data) {
+ this->post_data = _post_data;
+ }
+
+ int send_data(void* ptr, size_t len) {
+ int length_to_copy = 0;
+ if (post_data_index < post_data.length()) {
+ length_to_copy = min(post_data.length() - post_data_index, len);
+ memcpy(ptr, post_data.data() + post_data_index, length_to_copy);
+ post_data_index += length_to_copy;
+ }
+ return length_to_copy;
+ }
int receive_data(void *ptr, size_t len) {
bl->append((char *)ptr, len);
return 0;
}
+
int receive_header(void *ptr, size_t len) {
return 0;
}
- int send_data(void *ptr, size_t len) {
- return 0;
- }
-
};
-static RGWKeystoneTokenCache *keystone_token_cache = NULL;
+typedef RGWPostHTTPData RGWGetKeystoneAdminToken;
+typedef RGWPostHTTPData RGWGetRevokedTokens;
-class RGWGetRevokedTokens : public RGWHTTPClient {
+class RGWValidateKeystoneToken : public RGWHTTPClient {
bufferlist *bl;
public:
- RGWGetRevokedTokens(CephContext *_cct, bufferlist *_bl) : RGWHTTPClient(_cct), bl(_bl) {}
+ RGWValidateKeystoneToken(CephContext *_cct, bufferlist *_bl) : RGWHTTPClient(_cct), bl(_bl) {}
int receive_data(void *ptr, size_t len) {
bl->append((char *)ptr, len);
@@ -141,8 +153,11 @@ public:
int send_data(void *ptr, size_t len) {
return 0;
}
+
};
+static RGWKeystoneTokenCache *keystone_token_cache = NULL;
+
static int open_cms_envelope(CephContext *cct, string& src, string& dst)
{
#define BEGIN_CMS "-----BEGIN CMS-----"
@@ -211,23 +226,70 @@ static int decode_b64_cms(CephContext *cct, const string& signed_b64, bufferlist
return 0;
}
-
-int RGWSwift::check_revoked()
+int RGWSwift::get_keystone_url(std::string& url)
{
bufferlist bl;
RGWGetRevokedTokens req(cct, &bl);
- string url = g_conf->rgw_keystone_url;
+ url = cct->_conf->rgw_keystone_url;
if (url.empty()) {
ldout(cct, 0) << "ERROR: keystone url is not configured" << dendl;
return -EINVAL;
}
if (url[url.size() - 1] != '/')
url.append("/");
- url.append("v2.0/tokens/revoked");
+ return 0;
+}
+
+int RGWSwift::get_keystone_admin_token(std::string& token)
+{
+ std::string token_url;
+
+ if (get_keystone_url(token_url) < 0)
+ return -EINVAL;
+ if (cct->_conf->rgw_keystone_admin_token.empty()) {
+ token_url.append("v2.0/tokens");
+ KeystoneToken t;
+ bufferlist token_bl;
+ RGWGetKeystoneAdminToken token_req(cct, &token_bl);
+ JSONFormatter jf;
+ jf.open_object_section("auth");
+ jf.open_object_section("passwordCredentials");
+ encode_json("username", cct->_conf->rgw_keystone_admin_user, &jf);
+ encode_json("password", cct->_conf->rgw_keystone_admin_password, &jf);
+ jf.close_section();
+ encode_json("tenantName", cct->_conf->rgw_keystone_admin_tenant, &jf);
+ jf.close_section();
+ std::stringstream ss;
+ jf.flush(ss);
+ token_req.set_post_data(ss.str());
+ int ret = token_req.process(token_url.c_str());
+ if (ret < 0)
+ return ret;
+ if (t.parse(cct, token_bl) != 0)
+ return -EINVAL;
+ token = t.token.id;
+ } else {
+ token = cct->_conf->rgw_keystone_admin_token;
+ }
+ return 0;
+}
+
- req.append_header("X-Auth-Token", g_conf->rgw_keystone_admin_token);
+int RGWSwift::check_revoked()
+{
+ string url;
+ string token;
+ bufferlist bl;
+ RGWGetRevokedTokens req(cct, &bl);
+
+ if (get_keystone_admin_token(token) < 0)
+ return -EINVAL;
+ if (get_keystone_url(url) < 0)
+ return -EINVAL;
+ url.append("v2.0/tokens/revoked");
+ req.append_header("X-Auth-Token", token);
int ret = req.process(url.c_str());
if (ret < 0)
return ret;
@@ -429,10 +491,16 @@ int RGWSwift::validate_keystone_token(RGWRados *store, const string& token, stru
}
if (url[url.size() - 1] != '/')
url.append("/");
+ std::string admin_token;
+ if (get_keystone_admin_token(admin_token) < 0)
+ return -EINVAL;
+ if (get_keystone_url(url) < 0)
+ return -EINVAL;
+
url.append("v2.0/tokens/");
url.append(token);
- validate.append_header("X-Auth-Token", g_conf->rgw_keystone_admin_token);
+ validate.append_header("X-Auth-Token", admin_token);
int ret = validate.process(url.c_str());
if (ret < 0)
@@ -454,11 +522,94 @@ int RGWSwift::validate_keystone_token(RGWRados *store, const string& token, stru
return 0;
}
+int authenticate_temp_url(RGWRados *store, req_state *s)
+{
+ /* temp url requires bucket and object specified in the requets */
+ if (s->bucket_name_str.empty())
+ return -EPERM;
+
+ if (s->object_str.empty())
+ return -EPERM;
+
+ string temp_url_sig = s->info.args.get("temp_url_sig");
+ if (temp_url_sig.empty())
+ return -EPERM;
+
+ string temp_url_expires = s->info.args.get("temp_url_expires");
+ if (temp_url_expires.empty())
+ return -EPERM;
+
+ /* need to get user info of bucket owner */
+ RGWBucketInfo bucket_info;
+
+ int ret = store->get_bucket_info(NULL, s->bucket_name_str, bucket_info, NULL);
+ if (ret < 0)
+ return -EPERM;
+
+ dout(20) << "temp url user (bucket owner): " << bucket_info.owner << dendl;
+ if (rgw_get_user_info_by_uid(store, bucket_info.owner, s->user) < 0) {
+ return -EPERM;
+ }
+
+ if (s->user.temp_url_keys.empty()) {
+ dout(5) << "user does not have temp url key set, aborting" << dendl;
+ return -EPERM;
+ }
+
+ if (!s->info.method)
+ return -EPERM;
+
+ utime_t now = ceph_clock_now(g_ceph_context);
+
+ string err;
+ uint64_t expiration = (uint64_t)strict_strtoll(temp_url_expires.c_str(), 10, &err);
+ if (!err.empty()) {
+ dout(5) << "failed to parse temp_url_expires: " << err << dendl;
+ return -EPERM;
+ }
+ if (expiration <= (uint64_t)now.sec()) {
+ dout(5) << "temp url expired: " << expiration << " <= " << now.sec() << dendl;
+ return -EPERM;
+ }
+
+ /* strip the swift prefix from the uri */
+ int pos = g_conf->rgw_swift_url_prefix.find_last_not_of('/') + 1;
+ string object_path = s->info.request_uri.substr(pos + 1);
+ string str = string(s->info.method) + "\n" + temp_url_expires + "\n" + object_path;
+
+ dout(20) << "temp url signature (plain text): " << str << dendl;
+
+ map<int, string>::iterator iter;
+ for (iter = s->user.temp_url_keys.begin(); iter != s->user.temp_url_keys.end(); ++iter) {
+ string& temp_url_key = iter->second;
+
+ if (temp_url_key.empty())
+ continue;
+
+ char dest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+ calc_hmac_sha1(temp_url_key.c_str(), temp_url_key.size(),
+ str.c_str(), str.size(), dest);
+
+ char dest_str[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2 + 1];
+ buf_to_hex((const unsigned char *)dest, sizeof(dest), dest_str);
+ dout(20) << "temp url signature [" << iter->first << "] (calculated): " << dest_str << dendl;
+
+ if (dest_str != temp_url_sig) {
+ dout(5) << "temp url signature mismatch: " << dest_str << " != " << temp_url_sig << dendl;
+ } else {
+ return 0;
+ }
+ }
+
+ return -EPERM;
+}
bool RGWSwift::verify_swift_token(RGWRados *store, req_state *s)
{
- if (!s->os_auth_token)
- return false;
+ if (!s->os_auth_token) {
+ int ret = authenticate_temp_url(store, s);
+ return (ret >= 0);
+ }
if (strncmp(s->os_auth_token, "AUTH_rgwtk", 10) == 0) {
int ret = rgw_swift_verify_signed_token(s->cct, store, s->os_auth_token, s->user);
diff --git a/src/rgw/rgw_swift.h b/src/rgw/rgw_swift.h
index 3f0bd16..97347e8 100644
--- a/src/rgw/rgw_swift.h
+++ b/src/rgw/rgw_swift.h
@@ -29,6 +29,8 @@ class RGWSwift {
int parse_keystone_token_response(const string& token, bufferlist& bl, struct rgw_swift_auth_info *info,
KeystoneToken& t);
int update_user_info(RGWRados *store, struct rgw_swift_auth_info *info, RGWUserInfo& user_info);
+ int get_keystone_url(std::string& url);
+ int get_keystone_admin_token(std::string& token);
class KeystoneRevokeThread : public Thread {
CephContext *cct;
diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc
index b42be89..9c800c4 100644
--- a/src/rgw/rgw_swift_auth.cc
+++ b/src/rgw/rgw_swift_auth.cc
@@ -142,6 +142,7 @@ void RGW_SWIFT_Auth_Get::execute()
string swift_url = g_conf->rgw_swift_url;
string swift_prefix = g_conf->rgw_swift_url_prefix;
+ string tenant_path;
if (swift_prefix.size() == 0) {
swift_prefix = DEFAULT_SWIFT_PREFIX;
@@ -199,8 +200,13 @@ void RGW_SWIFT_Auth_Get::execute()
goto done;
}
- s->cio->print("X-Storage-Url: %s/%s/v1\n", swift_url.c_str(),
- swift_prefix.c_str());
+ if (!g_conf->rgw_swift_tenant_name.empty()) {
+ tenant_path = "/AUTH_";
+ tenant_path.append(g_conf->rgw_swift_tenant_name);
+ }
+
+ s->cio->print("X-Storage-Url: %s/%s/v1%s\n", swift_url.c_str(),
+ swift_prefix.c_str(), tenant_path.c_str());
if ((ret = encode_token(s->cct, swift_key->id, swift_key->key, bl)) < 0)
goto done;
diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index dc529e3..f620f9b 100644
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -41,6 +41,46 @@ bool rgw_user_is_authenticated(RGWUserInfo& info)
return (info.user_id != RGW_USER_ANON_ID);
}
+int rgw_user_sync_all_stats(RGWRados *store, const string& user_id)
+{
+ CephContext *cct = store->ctx();
+ size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+ bool done;
+ string marker;
+ int ret;
+
+ do {
+ RGWUserBuckets user_buckets;
+ ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+ if (ret < 0) {
+ ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl;
+ return ret;
+ }
+ map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+ for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+ i != buckets.end();
+ ++i) {
+ marker = i->first;
+
+ RGWBucketEnt& bucket_ent = i->second;
+ ret = rgw_bucket_sync_user_stats(store, user_id, bucket_ent.bucket);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: could not sync bucket stats: ret=" << ret << dendl;
+ return ret;
+ }
+ }
+ done = (buckets.size() < max_entries);
+ } while (!done);
+
+ ret = store->complete_sync_user_stats(user_id);
+ if (ret < 0) {
+ cerr << "ERROR: failed to complete syncing user stats: ret=" << ret << std::endl;
+ return ret;
+ }
+
+ return 0;
+}
+
/**
* Save the given user information to storage.
* Returns: 0 on success, -ERR# on failure.
@@ -239,7 +279,7 @@ extern int rgw_get_user_info_by_access_key(RGWRados *store, string& access_key,
int rgw_remove_key_index(RGWRados *store, RGWAccessKey& access_key)
{
rgw_obj obj(store->zone.user_keys_pool, access_key.id);
- int ret = store->delete_obj(NULL, obj);
+ int ret = store->delete_system_obj(NULL, obj);
return ret;
}
@@ -261,14 +301,14 @@ int rgw_remove_uid_index(RGWRados *store, string& uid)
int rgw_remove_email_index(RGWRados *store, string& email)
{
rgw_obj obj(store->zone.user_email_pool, email);
- int ret = store->delete_obj(NULL, obj);
+ int ret = store->delete_system_obj(NULL, obj);
return ret;
}
int rgw_remove_swift_name_index(RGWRados *store, string& swift_name)
{
rgw_obj obj(store->zone.user_swift_pool, swift_name);
- int ret = store->delete_obj(NULL, obj);
+ int ret = store->delete_system_obj(NULL, obj);
return ret;
}
@@ -330,7 +370,7 @@ int rgw_delete_user(RGWRados *store, RGWUserInfo& info, RGWObjVersionTracker& ob
rgw_obj email_obj(store->zone.user_email_pool, info.user_email);
ldout(store->ctx(), 10) << "removing email index: " << info.user_email << dendl;
- ret = store->delete_obj(NULL, email_obj);
+ ret = store->delete_system_obj(NULL, email_obj);
if (ret < 0 && ret != -ENOENT) {
ldout(store->ctx(), 0) << "ERROR: could not remove " << info.user_id << ":" << email_obj << ", should be fixed (err=" << ret << ")" << dendl;
return ret;
@@ -340,7 +380,7 @@ int rgw_delete_user(RGWRados *store, RGWUserInfo& info, RGWObjVersionTracker& ob
rgw_get_buckets_obj(info.user_id, buckets_obj_id);
rgw_obj uid_bucks(store->zone.user_uid_pool, buckets_obj_id);
ldout(store->ctx(), 10) << "removing user buckets index" << dendl;
- ret = store->delete_obj(NULL, uid_bucks);
+ ret = store->delete_system_obj(NULL, uid_bucks);
if (ret < 0 && ret != -ENOENT) {
ldout(store->ctx(), 0) << "ERROR: could not remove " << info.user_id << ":" << uid_bucks << ", should be fixed (err=" << ret << ")" << dendl;
return ret;
@@ -1685,6 +1725,17 @@ int RGWUser::execute_add(RGWUserAdminOpState& op_state, std::string *err_msg)
if (op_state.has_bucket_quota())
user_info.bucket_quota = op_state.get_bucket_quota();
+ if (op_state.temp_url_key_specified) {
+ map<int, string>::iterator iter;
+ for (iter = op_state.temp_url_keys.begin();
+ iter != op_state.temp_url_keys.end(); ++iter) {
+ user_info.temp_url_keys[iter->first] = iter->second;
+ }
+ }
+
+ if (op_state.has_user_quota())
+ user_info.user_quota = op_state.get_user_quota();
+
// update the request
op_state.set_user_info(user_info);
op_state.set_populated();
@@ -1774,7 +1825,7 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg)
std::map<std::string, RGWBucketEnt>::iterator it;
for (it = m.begin(); it != m.end(); ++it) {
- ret = rgw_remove_bucket(store, ((*it).second).bucket, true);
+ ret = rgw_remove_bucket(store, uid, ((*it).second).bucket, true);
if (ret < 0) {
set_err_msg(err_msg, "unable to delete user data");
return ret;
@@ -1884,12 +1935,23 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg)
if (op_state.system_specified)
user_info.system = op_state.system;
+ if (op_state.temp_url_key_specified) {
+ map<int, string>::iterator iter;
+ for (iter = op_state.temp_url_keys.begin();
+ iter != op_state.temp_url_keys.end(); ++iter) {
+ user_info.temp_url_keys[iter->first] = iter->second;
+ }
+ }
+
if (op_state.op_mask_specified)
user_info.op_mask = op_state.get_op_mask();
if (op_state.has_bucket_quota())
user_info.bucket_quota = op_state.get_bucket_quota();
+ if (op_state.has_user_quota())
+ user_info.user_quota = op_state.get_user_quota();
+
if (op_state.has_suspension_op()) {
__u8 suspended = op_state.get_suspension_status();
user_info.suspended = suspended;
@@ -2358,7 +2420,7 @@ public:
}
int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated) {
- list_keys_info *info = (list_keys_info *)handle;
+ list_keys_info *info = static_cast<list_keys_info *>(handle);
string no_filter;
@@ -2387,7 +2449,7 @@ public:
}
void list_keys_complete(void *handle) {
- list_keys_info *info = (list_keys_info *)handle;
+ list_keys_info *info = static_cast<list_keys_info *>(handle);
delete info;
}
};
diff --git a/src/rgw/rgw_user.h b/src/rgw/rgw_user.h
index e71b8f8..3cd08d3 100644
--- a/src/rgw/rgw_user.h
+++ b/src/rgw/rgw_user.h
@@ -38,6 +38,7 @@ struct RGWUID
};
WRITE_CLASS_ENCODER(RGWUID)
+extern int rgw_user_sync_all_stats(RGWRados *store, const string& user_id);
/**
* Get the anonymous (ie, unauthenticated) user info.
*/
@@ -131,6 +132,7 @@ struct RGWUserAdminOpState {
std::string caps;
RGWObjVersionTracker objv;
uint32_t op_mask;
+ map<int, string> temp_url_keys;
// subuser attributes
std::string subuser;
@@ -164,6 +166,7 @@ struct RGWUserAdminOpState {
bool suspension_op;
bool system_specified;
bool key_op;
+ bool temp_url_key_specified;
// req parameters
bool populated;
@@ -173,8 +176,10 @@ struct RGWUserAdminOpState {
bool user_params_checked;
bool bucket_quota_specified;
+ bool user_quota_specified;
RGWQuotaInfo bucket_quota;
+ RGWQuotaInfo user_quota;
void set_access_key(std::string& access_key) {
if (access_key.empty())
@@ -245,6 +250,10 @@ struct RGWUserAdminOpState {
op_mask = mask;
op_mask_specified = true;
}
+ void set_temp_url_key(const string& key, int index) {
+ temp_url_keys[index] = key;
+ temp_url_key_specified = true;
+ }
void set_key_type(int32_t type) {
key_type = type;
type_specified = true;
@@ -289,12 +298,16 @@ struct RGWUserAdminOpState {
key_op = true;
}
- void set_bucket_quota(RGWQuotaInfo& quota)
- {
+ void set_bucket_quota(RGWQuotaInfo& quota) {
bucket_quota = quota;
bucket_quota_specified = true;
}
+ void set_user_quota(RGWQuotaInfo& quota) {
+ user_quota = quota;
+ user_quota_specified = true;
+ }
+
bool is_populated() { return populated; };
bool is_initialized() { return initialized; };
bool has_existing_user() { return existing_user; };
@@ -314,6 +327,7 @@ struct RGWUserAdminOpState {
bool will_purge_data() { return purge_data; };
bool will_generate_subuser() { return gen_subuser; };
bool has_bucket_quota() { return bucket_quota_specified; }
+ bool has_user_quota() { return user_quota_specified; }
void set_populated() { populated = true; };
void clear_populated() { populated = false; };
void set_initialized() { initialized = true; };
@@ -329,6 +343,7 @@ struct RGWUserAdminOpState {
uint32_t get_max_buckets() { return max_buckets; };
uint32_t get_op_mask() { return op_mask; };
RGWQuotaInfo& get_bucket_quota() { return bucket_quota; }
+ RGWQuotaInfo& get_user_quota() { return user_quota; }
std::string get_user_id() { return user_id; };
std::string get_subuser() { return subuser; };
@@ -337,6 +352,7 @@ struct RGWUserAdminOpState {
std::string get_caps() { return caps; };
std::string get_user_email() { return user_email; };
std::string get_display_name() { return display_name; };
+ map<int, std::string>& get_temp_url_keys() { return temp_url_keys; };
RGWUserInfo& get_user_info() { return info; };
@@ -409,6 +425,7 @@ struct RGWUserAdminOpState {
perm_specified = false;
op_mask_specified = false;
suspension_op = false;
+ system_specified = false;
key_op = false;
populated = false;
initialized = false;
@@ -416,6 +433,8 @@ struct RGWUserAdminOpState {
subuser_params_checked = false;
user_params_checked = false;
bucket_quota_specified = false;
+ temp_url_key_specified = false;
+ user_quota_specified = false;
}
};
diff --git a/src/sample.ceph.conf b/src/sample.ceph.conf
index 62450be..139a05c 100644
--- a/src/sample.ceph.conf
+++ b/src/sample.ceph.conf
@@ -1,163 +1,364 @@
-;
-; Sample ceph ceph.conf file.
-;
-; This file defines cluster membership, the various locations
-; that Ceph stores data, and any other runtime options.
-
-; If a 'host' is defined for a daemon, the init.d start/stop script will
-; verify that it matches the hostname (or else ignore it). If it is
-; not defined, it is assumed that the daemon is intended to start on
-; the current host (e.g., in a setup with a startup.conf on each
-; node).
-
-; The variables $type, $id and $name are available to use in paths
-; $type = The type of daemon, possible values: mon, mds and osd
-; $id = The ID of the daemon, for mon.alpha, $id will be alpha
-; $name = $type.$id
-
-; For example:
-; osd.0
-; $type = osd
-; $id = 0
-; $name = osd.0
-
-; mon.beta
-; $type = mon
-; $id = beta
-; $name = mon.beta
-
-; global
+##
+# Sample ceph ceph.conf file.
+##
+# This file defines cluster membership, the various locations
+# that Ceph stores data, and any other runtime options.
+
+# If a 'host' is defined for a daemon, the init.d start/stop script will
+# verify that it matches the hostname (or else ignore it). If it is
+# not defined, it is assumed that the daemon is intended to start on
+# the current host (e.g., in a setup with a startup.conf on each
+# node).
+
+## Metavariables
+# $cluster ; Expands to the Ceph Storage Cluster name. Useful
+# ; when running multiple Ceph Storage Clusters
+# ; on the same hardware.
+# ; Example: /etc/ceph/$cluster.keyring
+# ; (Default: ceph)
+#
+# $type ; Expands to one of mds, osd, or mon, depending on
+# ; the type of the instant daemon.
+# ; Example: /var/lib/ceph/$type
+#
+# $id ; Expands to the daemon identifier. For osd.0, this
+# ; would be 0; for mds.a, it would be a.
+# ; Example: /var/lib/ceph/$type/$cluster-$id
+#
+# $host ; Expands to the host name of the instant daemon.
+#
+# $name ; Expands to $type.$id.
+# ; Example: /var/run/ceph/$cluster-$name.asok
+
[global]
- ; enable secure authentication
- auth supported = cephx
+### http://ceph.com/docs/master/rados/configuration/general-config-ref/
+
+ ;fsid = {UUID} # use `uuidgen` to generate your own UUID
+ ;public network = 192.168.0.0/24
+ ;cluster network = 192.168.0.0/24
+
+ # Each running Ceph daemon has a running process identifier (PID) file.
+ # The PID file is generated upon start-up.
+ # Type: String (optional)
+ # (Default: N/A). The default path is /var/run/$cluster/$name.pid.
+ pid file = /var/run/ceph/$name.pid
+
+ # If set, when the Ceph Storage Cluster starts, Ceph sets the max open fds
+ # at the OS level (i.e., the max # of file descriptors).
+ # It helps prevents Ceph OSD Daemons from running out of file descriptors.
+ # Type: 64-bit Integer (optional)
+ # (Default: 0)
+ ;max open files = 131072
+
+
+### http://ceph.com/docs/master/rados/operations/authentication
+### http://ceph.com/docs/master/rados/configuration/auth-config-ref/
- ; allow ourselves to open a lot of files
- max open files = 131072
+ # If enabled, the Ceph Storage Cluster daemons (i.e., ceph-mon, ceph-osd,
+ # and ceph-mds) must authenticate with each other.
+ # Type: String (optional); Valid settings are "cephx" or "none".
+ # (Default: cephx)
+ auth cluster required = cephx
+
+ # If enabled, the Ceph Storage Cluster daemons require Ceph Clients to
+ # authenticate with the Ceph Storage Cluster in order to access Ceph
+ # services.
+ # Type: String (optional); Valid settings are "cephx" or "none".
+ # (Default: cephx)
+ auth service required = cephx
+
+ # If enabled, the Ceph Client requires the Ceph Storage Cluster to
+ # authenticate with the Ceph Client.
+ # Type: String (optional); Valid settings are "cephx" or "none".
+ # (Default: cephx)
+ auth client required = cephx
+
+ # If set to true, Ceph requires signatures on all message traffic between
+ # the Ceph Client and the Ceph Storage Cluster, and between daemons
+ # comprising the Ceph Storage Cluster.
+ # Type: Boolean (optional)
+ # (Default: false)
+ cephx require signatures = true ; everywhere possible
+
+ # The path to the keyring file.
+ # Type: String (optional)
+ # Default: /etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin
+ ;keyring = /etc/ceph/$cluster.$name.keyring
+
+
+### http://ceph.com/docs/master/rados/configuration/pool-pg-config-ref/
+
+
+ ## Replication level, number of data copies.
+ # Type: 32-bit Integer
+ # (Default: 2)
+ ;osd pool default size = 2
+
+ ## Replication level in degraded state, less than 'osd pool default size' value.
+ # Sets the minimum number of written replicas for objects in the
+ # pool in order to acknowledge a write operation to the client. If
+ # minimum is not met, Ceph will not acknowledge the write to the
+ # client. This setting ensures a minimum number of replicas when
+ # operating in degraded mode.
+ # Type: 32-bit Integer
+ # (Default: 0), which means no particular minimum. If 0, minimum is size - (size / 2).
+ ;osd pool default min size = 1
+
+ ## Ensure you have a realistic number of placement groups. We recommend
+ ## approximately 100 per OSD. E.g., total number of OSDs multiplied by 100
+ ## divided by the number of replicas (i.e., osd pool default size). So for
+ ## 10 OSDs and osd pool default size = 3, we'd recommend approximately
+ ## (100 * 10) / 3 = 333
+
+ # Description: The default number of placement groups for a pool. The
+ # default value is the same as pg_num with mkpool.
+ # Type: 32-bit Integer
+ # (Default: 8)
+ ;osd pool default pg num = 100
+
+ # Description: The default number of placement groups for placement for a
+ # pool. The default value is the same as pgp_num with mkpool.
+ # PG and PGP should be equal (for now).
+ # Type: 32-bit Integer
+ # (Default: 8)
+ ;osd pool default pgp num = 100
+
+ # The default CRUSH ruleset to use when creating a pool
+ # Type: 32-bit Integer
+ # (Default: 0)
+ ;osd pool default crush rule = 0
+
+ # The bucket type to use for chooseleaf in a CRUSH rule.
+ # Uses ordinal rank rather than name.
+ # Type: 32-bit Integer
+ # (Default: 1) Typically a host containing one or more Ceph OSD Daemons.
+ ;osd crush chooseleaf type = 1
+
+
+### http://ceph.com/docs/bobtail/rados/configuration/log-and-debug-ref/
+
+ # Default: /var/log/ceph/$cluster-$name.log
+ ;log file = /var/log/ceph/$cluster-$name.log
+
+ ;log to syslog = true
+
+
+### http://ceph.com/docs/master/rados/configuration/ms-ref/
+
+ # Enable if you want your daemons to bind to IPv6 address instead of
+ # IPv4 ones. (Not required if you specify a daemon or cluster IP.)
+ # Type: Boolean
+ # (Default: false)
+ ;ms bind ipv6 = true
+
+
+### http://ceph.com/docs/master/rados/configuration/filestore-config-ref/
+
+ # The maximum interval in seconds for synchronizing the filestore.
+ # Type: Double (optional)
+ # (Default: 5)
+ ;filestore max sync interval = 5
+
+ # Use object map for XATTRS. Set to true for ext4 file systems only.
+ # Type: Boolean (optional)
+ # (Default: false)
+ ;filestore xattr use omap = true
- ; set log file
- log file = /var/log/ceph/$name.log
- ; log_to_syslog = true ; uncomment this line to log to syslog
+### http://ceph.com/docs/master/rados/configuration/journal-ref/
- ; set up pid files
- pid file = /var/run/ceph/$name.pid
+##################
+## Monitors
+## You need at least one. You need at least three if you want to
+## tolerate any node failures. Always create an odd number.
+[mon]
+### http://ceph.com/docs/argonaut/config-ref/mon-config/
+### http://ceph.com/docs/master/rados/configuration/mon-config-ref/
+### http://ceph.com/docs/dumpling/rados/configuration/mon-osd-interaction/
- ; If you want to run a IPv6 cluster, set this to true. Dual-stack isn't possible
- ;ms bind ipv6 = true
+ # The IDs of initial monitors in a cluster during startup.
+ # If specified, Ceph requires an odd number of monitors to form an
+ # initial quorum (e.g., 3).
+ # Type: String
+ # (Default: None)
+ ;mon initial members = mycephhost
-; monitors
-; You need at least one. You need at least three if you want to
-; tolerate any node failures. Always create an odd number.
-[mon]
- mon data = /data/$name
+ ;mon host = cephhost01,cephhost02
+ ;mon addr = 192.168.0.101,192.168.0.102
+
+ # The monitor’s data location
+ # Default: /var/lib/ceph/mon/$cluster-$id
+ ;mon data = /var/lib/ceph/mon/$name
+
+ # The clock drift in seconds allowed between monitors.
+ # Type: Float
+ # (Default: .050)
+ ;mon clock drift allowed = .15
- ; If you are using for example the RADOS Gateway and want to have your newly created
- ; pools a higher replication level, you can set a default
- ;osd pool default size = 3
+ # Exponential backoff for clock drift warnings
+ # Type: Float
+ # (Default: 5)
+ ;mon clock drift warn backoff = 30 ; Tell the monitor to backoff from this warning for 30 seconds
- ; You can also specify a CRUSH rule for new pools
- ; Wiki: http://ceph.newdream.net/wiki/Custom_data_placement_with_CRUSH
- ;osd pool default crush rule = 0
+ # The percentage of disk space used before an OSD is considered full.
+ # Type: Float
+ # (Default: .95)
+ ;mon osd full ratio = .95
- ; Timing is critical for monitors, but if you want to allow the clocks to drift a
- ; bit more, you can specify the max drift.
- ;mon clock drift allowed = 1
+ # The percentage of disk space used before an OSD is considered nearfull.
+ # Type: Float
+ # (Default: .85)
+ ;mon osd nearfull ratio = .85
- ; Tell the monitor to backoff from this warning for 30 seconds
- ;mon clock drift warn backoff = 30
- ; logging, for debugging monitor crashes, in order of
- ; their likelihood of being helpful :)
- ;debug ms = 1
- ;debug mon = 20
- ;debug paxos = 20
- ;debug auth = 20
+### http://ceph.com/docs/next/rados/troubleshooting/log-and-debug/
-[mon.alpha]
- host = alpha
- mon addr = 192.168.0.10:6789
+ # logging, for debugging monitor crashes, in order of
+ # their likelihood of being helpful :)
+ ;debug ms = 1
+ ;debug mon = 20
+ ;debug paxos = 20
+ ;debug auth = 20
-[mon.beta]
- host = beta
- mon addr = 192.168.0.11:6789
-[mon.gamma]
- host = gamma
- mon addr = 192.168.0.12:6789
+;[mon.alpha]
+; host = alpha
+; mon addr = 192.168.0.10:6789
-; mds
-; You need at least one. Define two to get a standby.
+;[mon.beta]
+; host = beta
+; mon addr = 192.168.0.11:6789
+
+;[mon.gamma]
+; host = gamma
+; mon addr = 192.168.0.12:6789
+
+
+##################
+## Metadata servers
+# You must deploy at least one metadata server to use CephFS. There is
+# experimental support for running multiple metadata servers. Do not run
+# multiple metadata servers in production.
[mds]
- ; where the mds keeps it's secret encryption keys
- keyring = /data/keyring.$name
+### http://ceph.com/docs/argonaut/config-ref/mds-config/
+### http://ceph.com/docs/master/cephfs/mds-config-ref/
+
+ # where the mds keeps it's secret encryption keys
+ ;keyring = /var/lib/ceph/mds/$name/keyring
+
+ ; mds logging to debug issues.
+ ;debug ms = 1
+ ;debug mds = 20
- ; mds logging to debug issues.
- ;debug ms = 1
- ;debug mds = 20
-[mds.alpha]
- host = alpha
+;[mds.alpha]
+; host = alpha
-[mds.beta]
- host = beta
+;[mds.beta]
+; host = beta
-; osd
-; You need at least one. Two if you want data to be replicated.
-; Define as many as you like.
+##################
+## osd
+# You need at least one. Two or more if you want data to be replicated.
+# Define as many as you like.
[osd]
- ; This is where the osd expects its data
- osd data = /data/$name
-
- ; Ideally, make the journal a separate disk or partition.
- ; 1-10GB should be enough; more if you have fast or many
- ; disks. You can use a file under the osd data dir if need be
- ; (e.g. /data/$name/journal), but it will be slower than a
- ; separate disk or partition.
- ; This is an example of a file-based journal.
- osd journal = /data/$name/journal
- osd journal size = 1000 ; journal size, in megabytes
-
- ; If you want to run the journal on a tmpfs (don't), disable DirectIO
- ;journal dio = false
-
- ; You can change the number of recovery operations to speed up recovery
- ; or slow it down if your machines can't handle it
- ; osd recovery max active = 3
-
- ; osd logging to debug osd issues, in order of likelihood of being
- ; helpful
- ;debug ms = 1
- ;debug osd = 20
- ;debug filestore = 20
- ;debug journal = 20
-
-
- ; ### The below options only apply if you're using mkcephfs
- ; ### and the devs options
- ; The filesystem used on the volumes
- osd mkfs type = btrfs
- ; If you want to specify some other mount options, you can do so.
- ; for other filesystems use 'osd mount options $fstype'
- osd mount options btrfs = rw,noatime
- ; The options used to format the filesystem via mkfs.$fstype
- ; for other filesystems use 'osd mkfs options $fstype'
- ; osd mkfs options btrfs =
-
-
-[osd.0]
- host = delta
-
- ; if 'devs' is not specified, you're responsible for
- ; setting up the 'osd data' dir.
- devs = /dev/sdx
-
-[osd.1]
- host = epsilon
- devs = /dev/sdy
-
-[osd.2]
- host = zeta
- devs = /dev/sdx
-
-[osd.3]
- host = eta
- devs = /dev/sdy
+### http://ceph.com/docs/argonaut/config-ref/osd-config/
+### http://ceph.com/docs/bobtail/rados/configuration/osd-config-ref/
+
+ # The path to the OSDs data.
+ # You must create the directory when deploying Ceph.
+ # You should mount a drive for OSD data at this mount point.
+ # We do not recommend changing the default.
+ # Type: String
+ # Default: /var/lib/ceph/osd/$cluster-$id
+ ;osd data = /var/lib/ceph/osd/$name
+
+ ## You can change the number of recovery operations to speed up recovery
+ ## or slow it down if your machines can't handle it
+
+ # The number of active recovery requests per OSD at one time.
+ # More requests will accelerate recovery, but the requests
+ # places an increased load on the cluster.
+ # Type: 32-bit Integer
+ # (Default: 5)
+ ;osd recovery max active = 3
+
+
+ # You may add settings for mkcephfs so that it will create and mount
+ # the file system for you. Remove the comment `#` character for
+ # the following settings and replace the values in parenthesis
+ # with appropriate values, or leave the following settings commented
+ # out to accept the default values. You must specify the --mkfs
+ # option with mkcephfs in order for the deployment script to
+ # utilize the following settings, and you must define the 'devs'
+ # option for each osd instance; see below.
+
+ #osd mkfs type = {fs-type}
+ #osd mkfs options {fs-type} = {mkfs options} # default for xfs is "-f"
+ #osd mount options {fs-type} = {mount options} # default mount option is "rw, noatime"
+ ;osd mkfs type = btrfs
+ ;osd mount options btrfs = noatime,nodiratime
+
+ ## Ideally, make this a separate disk or partition. A few
+ ## hundred MB should be enough; more if you have fast or many
+ ## disks. You can use a file under the osd data dir if need be
+ ## (e.g. /data/$name/journal), but it will be slower than a
+ ## separate disk or partition.
+ # The path to the OSD’s journal. This may be a path to a file or a block
+ # device (such as a partition of an SSD). If it is a file, you must
+ # create the directory to contain it.
+ # We recommend using a drive separate from the osd data drive.
+ # Type: String
+ # Default: /var/lib/ceph/osd/$cluster-$id/journal
+ ;osd journal = /var/lib/ceph/osd/$name/journal
+
+### http://ceph.com/docs/master/rados/configuration/journal-ref/
+
+ # The size of the journal in megabytes. If this is 0,
+ # and the journal is a block device, the entire block device is used.
+ # Since v0.54, this is ignored if the journal is a block device,
+ # and the entire block device is used.
+ # Type: 32-bit Integer
+ # (Default: 5120)
+ # Recommended: Begin with 1GB. Should be at least twice the product
+ # of the expected speed multiplied by "filestore max sync interval".
+ ;osd journal size = 1000 ; journal size, in megabytes
+
+ ## If you want to run the journal on a tmpfs, disable DirectIO
+ # Enables direct i/o to the journal.
+ # Requires journal block align set to true.
+ # Type: Boolean
+ # Required: Yes when using aio.
+ # (Default: true)
+ ;journal dio = false
+
+ # Check log files for corruption. Can be computationally expensive.
+ # Type: Boolean
+ # (Default: false)
+ ;osd check for log corruption = true
+
+ # osd logging to debug osd issues, in order of likelihood of being helpful
+ ;debug ms = 1
+ ;debug osd = 20
+ ;debug filestore = 20
+ ;debug journal = 20
+
+
+
+;[osd.0]
+; host = delta
+
+ # If 'devs' is not specified, you're responsible for
+ # setting up the 'osd data' dir (e.g. `mkdir /var/lib/ceph/osd/ceph-0`)
+ ;devs = /dev/sdx
+
+;[osd.1]
+; host = epsilon
+; devs = /dev/sdy
+
+;[osd.2]
+; host = zeta
+; devs = /dev/sdx
+
+;[osd.3]
+; host = eta
+; devs = /dev/sdy
diff --git a/src/stop.sh b/src/stop.sh
index 3a3ed16..1a498ba 100755
--- a/src/stop.sh
+++ b/src/stop.sh
@@ -1,4 +1,20 @@
-#!/bin/sh
+#!/bin/bash -x
+#
+# Copyright (C) 2013 Inktank <info at inktank.com>
+# Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+#
+# Author: Loic Dachary <loic at dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library Public License for more details.
+#
test -d dev/osd0/. && test -e dev/sudo && SUDO="sudo"
@@ -41,7 +57,14 @@ while [ $# -ge 1 ]; do
done
if [ $stop_all -eq 1 ]; then
- killall ceph-mon ceph-mds ceph-osd radosgw lt-radosgw apache2
+ for p in ceph-mon ceph-mds ceph-osd radosgw lt-radosgw apache2 ; do
+ for try in 0 1 1 1 1 ; do
+ if ! pkill $p ; then
+ break
+ fi
+ sleep $try
+ done
+ done
pkill -f valgrind.bin.\*ceph-mon
$SUDO pkill -f valgrind.bin.\*ceph-osd
pkill -f valgrind.bin.\*ceph-mds
diff --git a/src/test/Makefile.am b/src/test/Makefile.am
index 572d5d5..83b83f4 100644
--- a/src/test/Makefile.am
+++ b/src/test/Makefile.am
@@ -130,7 +130,7 @@ ceph_smalliobench_SOURCES = \
test/bench/rados_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobench_LDADD = $(LIBRADOS) -lboost_program_options $(CEPH_GLOBAL)
+ceph_smalliobench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_smalliobench
ceph_smalliobenchfs_SOURCES = \
@@ -138,7 +138,7 @@ ceph_smalliobenchfs_SOURCES = \
test/bench/testfilestore_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobenchfs_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL)
+ceph_smalliobenchfs_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_smalliobenchfs
ceph_smalliobenchdumb_SOURCES = \
@@ -146,7 +146,7 @@ ceph_smalliobenchdumb_SOURCES = \
test/bench/dumb_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobenchdumb_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL)
+ceph_smalliobenchdumb_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_smalliobenchdumb
ceph_smalliobenchrbd_SOURCES = \
@@ -154,24 +154,26 @@ ceph_smalliobenchrbd_SOURCES = \
test/bench/rbd_backend.cc \
test/bench/detailed_stat_collector.cc \
test/bench/bencher.cc
-ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) -lboost_program_options $(CEPH_GLOBAL)
+ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_smalliobenchrbd
ceph_tpbench_SOURCES = \
test/bench/tp_bench.cc \
test/bench/detailed_stat_collector.cc
-ceph_tpbench_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL)
+ceph_tpbench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_tpbench
ceph_omapbench_SOURCES = test/omap_bench.cc
ceph_omapbench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_omapbench
+if LINUX
ceph_kvstorebench_SOURCES = \
test/kv_store_bench.cc \
key_value_store/kv_flat_btree_async.cc
ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_kvstorebench
+endif
ceph_multi_stress_watch_SOURCES = \
test/multi_stress_watch.cc \
@@ -179,13 +181,18 @@ ceph_multi_stress_watch_SOURCES = \
ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_multi_stress_watch
-
-
-
+ceph_erasure_code_benchmark_SOURCES = \
+ test/osd/ceph_erasure_code_benchmark.cc
+ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+if LINUX
+ceph_erasure_code_benchmark_LDADD += -ldl
+endif
+bin_DEBUGPROGRAMS += ceph_erasure_code_benchmark
## System tests
+if LINUX
libsystest_la_SOURCES = \
test/system/cross_process_sem.cc \
test/system/systest_runnable.cc \
@@ -223,6 +230,7 @@ ceph_test_rados_watch_notify_SOURCES = \
test/system/st_rados_notify.cc
ceph_test_rados_watch_notify_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
bin_DEBUGPROGRAMS += ceph_test_rados_watch_notify
+endif
ceph_bench_log_SOURCES = test/bench_log.cc
ceph_bench_log_LDADD = $(CEPH_GLOBAL)
@@ -234,7 +242,12 @@ bin_DEBUGPROGRAMS += ceph_bench_log
check_SCRIPTS += \
unittest_bufferlist.sh \
- test/encoding/check-generated.sh
+ test/encoding/check-generated.sh \
+ test/mon/osd-pool-create.sh \
+ test/mon/mkfs.sh \
+ test/ceph-disk.sh \
+ test/mon/mon-handle-forward.sh \
+ test/vstart_wrapped_tests.sh
# target to build but not run the unit tests
unittests:: $(check_PROGRAMS)
@@ -263,6 +276,16 @@ unittest_bloom_filter_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_bloom_filter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
check_PROGRAMS += unittest_bloom_filter
+unittest_str_map_SOURCES = test/common/test_str_map.cc
+unittest_str_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_str_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_str_map
+
+unittest_crushwrapper_SOURCES = test/test_crushwrapper.cc
+unittest_crushwrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_crushwrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
+check_PROGRAMS += unittest_crushwrapper
+
unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc
unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -278,6 +301,16 @@ unittest_util_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_util_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CRYPTO_LIBS) $(EXTRALIBS)
check_PROGRAMS += unittest_util
+unittest_crush_indep_SOURCES = test/crush/indep.cc
+unittest_crush_indep_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_crush_indep_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CEPH_CRUSH) $(EXTRALIBS) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_crush_indep
+
+unittest_osdmap_SOURCES = test/osd/TestOSDMap.cc
+unittest_osdmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_osdmap_LDADD = $(UNITTEST_LDADD) $(LIBCOMMON) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_osdmap
+
unittest_workqueue_SOURCES = test/test_workqueue.cc
unittest_workqueue_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_workqueue_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
@@ -308,6 +341,11 @@ unittest_throttle_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_throttle_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
check_PROGRAMS += unittest_throttle
+unittest_crush_wrapper_SOURCES = test/crush/TestCrushWrapper.cc
+unittest_crush_wrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
+unittest_crush_wrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
+check_PROGRAMS += unittest_crush_wrapper
+
unittest_base64_SOURCES = test/base64.cc
unittest_base64_LDADD = $(LIBCEPHFS) -lm $(UNITTEST_LDADD)
unittest_base64_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -401,6 +439,11 @@ unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
check_PROGRAMS += unittest_pglog
+unittest_hitset_SOURCES = test/osd/hitset.cc
+unittest_hitset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_hitset_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_hitset
+
if LINUX
unittest_pglog_LDADD += -ldl
endif # LINUX
@@ -440,7 +483,7 @@ unittest_crc32c_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_crc32c_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_crc32c
-unittest_arch_SOURCES = test/test_arch.c
+unittest_arch_SOURCES = test/test_arch.cc
unittest_arch_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_arch_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_arch
@@ -500,6 +543,11 @@ unittest_confutils_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_confutils
+unittest_config_SOURCES = test/common/test_config.cc
+unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_config
+
unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc
unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
@@ -567,6 +615,10 @@ unittest_texttable_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD)
unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_texttable
+unittest_on_exit_SOURCES = test/on_exit.cc
+unittest_on_exit_LDADD = $(PTHREAD_LIBS)
+check_PROGRAMS += unittest_on_exit
+
check_SCRIPTS += test/pybind/test_ceph_argparse.py
if WITH_RADOSGW
@@ -585,7 +637,7 @@ ceph_test_cls_rgw_meta_LDADD = \
-lcurl -luuid -lexpat \
libcls_version_client.a libcls_log_client.a \
libcls_statelog_client.a libcls_refcount_client.la \
- libcls_rgw_client.la libcls_lock_client.la
+ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
ceph_test_cls_rgw_meta_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_rgw_meta
@@ -596,7 +648,7 @@ ceph_test_cls_rgw_log_LDADD = \
-lcurl -luuid -lexpat \
libcls_version_client.a libcls_log_client.a \
libcls_statelog_client.a libcls_refcount_client.la \
- libcls_rgw_client.la libcls_lock_client.la
+ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
ceph_test_cls_rgw_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_rgw_log
@@ -607,7 +659,7 @@ ceph_test_cls_rgw_opstate_LDADD = \
-lcurl -luuid -lexpat \
libcls_version_client.a libcls_log_client.a \
libcls_statelog_client.a libcls_refcount_client.la \
- libcls_rgw_client.la libcls_lock_client.la
+ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_cls_rgw_opstate
endif # WITH_RADOSGW
@@ -619,10 +671,12 @@ ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_librbd
+if LINUX
ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c
ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm
ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format
bin_DEBUGPROGRAMS += ceph_test_librbd_fsx
+endif
ceph_test_cls_rbd_SOURCES = \
test/cls_rbd/test_cls_rbd.cc \
@@ -711,6 +765,13 @@ ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_io
+ceph_test_rados_api_c_write_operations_SOURCES = \
+ test/librados/c_write_operations.cc \
+ test/librados/test.cc
+ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+ceph_test_rados_api_c_write_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_c_write_operations
+
ceph_test_rados_api_aio_SOURCES = \
test/librados/aio.cc \
test/librados/test.cc
@@ -767,6 +828,14 @@ ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_rados_api_misc
+ceph_test_rados_api_tier_SOURCES = \
+ test/librados/tier.cc \
+ test/librados/test.cc \
+ osd/HitSet.cc
+ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_rados_api_tier_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_tier
+
ceph_test_rados_api_lock_SOURCES = \
test/librados/lock.cc \
test/librados/test.cc
@@ -783,10 +852,12 @@ ceph_test_libcephfs_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_libcephfs
+if LINUX
ceph_test_filestore_SOURCES = test/filestore/store_test.cc
ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph_test_filestore
+endif
ceph_test_filestore_workloadgen_SOURCES = \
test/filestore/workload_generator.cc \
@@ -859,40 +930,45 @@ bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_iterators
ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc
bin_DEBUGPROGRAMS += ceph_test_cfuse_cache_invalidate
+ceph_test_c_headers_SOURCES = test/test_c_headers.c
+ceph_test_c_headers_LDADD = $(LIBRADOS) $(LIBCEPHFS)
+bin_DEBUGPROGRAMS += ceph_test_c_headers
+
+ceph_test_get_blkdev_size_SOURCES = test/test_get_blkdev_size.cc
+ceph_test_get_blkdev_size_LDADD = $(LIBCOMMON)
+bin_DEBUGPROGRAMS += ceph_test_get_blkdev_size
noinst_HEADERS += \
- test/osd/RadosModel.h \
- test/osd/Object.h \
- test/osd/TestOpStat.h \
+ test/bench/backend.h \
+ test/bench/bencher.h \
+ test/bench/detailed_stat_collector.h \
test/bench/distribution.h \
+ test/bench/dumb_backend.h \
test/bench/rados_backend.h \
test/bench/rbd_backend.h \
- test/bench/bencher.h \
- test/bench/backend.h \
- test/bench/dumb_backend.h \
test/bench/stat_collector.h \
- test/bench/detailed_stat_collector.h \
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h \
test/encoding/types.h \
test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreTracker.h \
test/filestore/FileStoreDiff.h \
+ test/filestore/FileStoreTracker.h \
test/filestore/TestFileStoreState.h \
test/filestore/workload_generator.h \
test/kv_store_bench.h \
test/librados/test.h \
test/ObjectMap/KeyValueDBMemory.h \
test/omap_bench.h \
+ test/osd/ceph_erasure_code_benchmark.h \
+ test/osdc/FakeWriteback.h \
test/osd/Object.h \
test/osd/RadosModel.h \
test/osd/TestOpStat.h \
- test/osdc/FakeWriteback.h \
test/system/cross_process_sem.h \
test/system/st_rados_create_pool.h \
- test/system/st_rados_list_objects.h \
test/system/st_rados_delete_objs.h \
test/system/st_rados_delete_pool.h \
+ test/system/st_rados_list_objects.h \
test/system/st_rados_notify.h \
test/system/st_rados_watch.h \
test/system/systest_runnable.h \
diff --git a/src/test/ObjectMap/KeyValueDBMemory.cc b/src/test/ObjectMap/KeyValueDBMemory.cc
index 43fa60c..c03e169 100644
--- a/src/test/ObjectMap/KeyValueDBMemory.cc
+++ b/src/test/ObjectMap/KeyValueDBMemory.cc
@@ -4,7 +4,7 @@
#include "KeyValueDBMemory.h"
#include <map>
#include <set>
-#include <tr1/memory>
+#include "include/memory.h"
#include <iostream>
using namespace std;
@@ -214,7 +214,7 @@ int KeyValueDBMemory::rmkeys_by_prefix(const string &prefix) {
}
KeyValueDB::WholeSpaceIterator KeyValueDBMemory::_get_iterator() {
- return std::tr1::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
+ return ceph::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
new WholeSpaceMemIterator(this)
);
}
@@ -243,7 +243,7 @@ public:
KeyValueDB::WholeSpaceIterator KeyValueDBMemory::_get_snapshot_iterator() {
KeyValueDBMemory *snap_db = new KeyValueDBMemory(this);
- return std::tr1::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
+ return ceph::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
new WholeSpaceSnapshotMemIterator(snap_db)
);
}
diff --git a/src/test/ObjectMap/KeyValueDBMemory.h b/src/test/ObjectMap/KeyValueDBMemory.h
index 5cffce3..8df3966 100644
--- a/src/test/ObjectMap/KeyValueDBMemory.h
+++ b/src/test/ObjectMap/KeyValueDBMemory.h
@@ -3,7 +3,7 @@
#include <map>
#include <set>
#include <string>
-#include <tr1/memory>
+#include "include/memory.h"
#include "os/KeyValueDB.h"
#include "include/buffer.h"
@@ -19,6 +19,16 @@ public:
KeyValueDBMemory(KeyValueDBMemory *db) : db(db->db) { }
virtual ~KeyValueDBMemory() { }
+ virtual int init() {
+ return 0;
+ }
+ virtual int open(ostream &out) {
+ return 0;
+ }
+ virtual int create_and_open(ostream &out) {
+ return 0;
+ }
+
int get(
const string &prefix,
const std::set<string> &key,
diff --git a/src/test/ObjectMap/test_keyvaluedb_atomicity.cc b/src/test/ObjectMap/test_keyvaluedb_atomicity.cc
index e06a9ff..04cef95 100644
--- a/src/test/ObjectMap/test_keyvaluedb_atomicity.cc
+++ b/src/test/ObjectMap/test_keyvaluedb_atomicity.cc
@@ -6,7 +6,7 @@
#include <dirent.h>
#include <string>
#include <vector>
-#include <tr1/memory>
+#include "include/memory.h"
#include <boost/scoped_ptr.hpp>
#include <sstream>
#include "stdlib.h"
diff --git a/src/test/ObjectMap/test_keyvaluedb_iterators.cc b/src/test/ObjectMap/test_keyvaluedb_iterators.cc
index c4a5e38..cbe2ab1 100644
--- a/src/test/ObjectMap/test_keyvaluedb_iterators.cc
+++ b/src/test/ObjectMap/test_keyvaluedb_iterators.cc
@@ -10,7 +10,7 @@
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*/
-#include <tr1/memory>
+#include "include/memory.h"
#include <map>
#include <set>
#include <deque>
diff --git a/src/test/ObjectMap/test_object_map.cc b/src/test/ObjectMap/test_object_map.cc
index 23f220d..eecd9ea 100644
--- a/src/test/ObjectMap/test_object_map.cc
+++ b/src/test/ObjectMap/test_object_map.cc
@@ -1,5 +1,5 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#include <tr1/memory>
+#include "include/memory.h"
#include <map>
#include <set>
#include <boost/scoped_ptr.hpp>
@@ -196,6 +196,14 @@ public:
db->clear(hoid);
}
+ void clear_omap(const string &objname) {
+ clear_omap(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))));
+ }
+
+ void clear_omap(const ghobject_t &objname) {
+ db->clear_keys_header(objname);
+ }
+
void def_init() {
for (unsigned i = 0; i < 1000; ++i) {
key_space.insert("key_" + num_str(i));
@@ -462,6 +470,14 @@ public:
xattrs.erase(*object);
}
+ void auto_clear_omap(ostream &out) {
+ set<string>::iterator object = rand_choose(object_name_space);
+ out << "auto_clear_object " << *object << std::endl;
+ clear_omap(*object);
+ omap.erase(*object);
+ hmap.erase(*object);
+ }
+
void auto_write_header(ostream &out) {
set<string>::iterator object = rand_choose(object_name_space);
string header = val_from_key(*object, "HEADER");
@@ -573,9 +589,28 @@ TEST_F(ObjectMapTest, CreateOneObject) {
db->get(hoid, &header, &got);
ASSERT_EQ(got.size(), (unsigned)0);
+ map<string, bufferlist> attrs;
+ attrs["attr1"] = bl;
+ db->set_xattrs(hoid, attrs);
+
+ db->set_header(hoid, bl);
+
+ db->clear_keys_header(hoid);
+ set<string> attrs_got;
+ db->get_all_xattrs(hoid, &attrs_got);
+ ASSERT_EQ(attrs_got.size(), 1U);
+ ASSERT_EQ(*(attrs_got.begin()), "attr1");
+ db->get(hoid, &header, &got);
+ ASSERT_EQ(got.size(), (unsigned)0);
+ ASSERT_EQ(header.length(), 0U);
+ got.clear();
+
db->clear(hoid);
db->get(hoid, &header, &got);
ASSERT_EQ(got.size(), (unsigned)0);
+ attrs_got.clear();
+ db->get_all_xattrs(hoid, &attrs_got);
+ ASSERT_EQ(attrs_got.size(), 0U);
}
TEST_F(ObjectMapTest, CloneOneObject) {
@@ -734,8 +769,10 @@ TEST_F(ObjectMapTest, RandomTest) {
ASSERT_TRUE(tester.auto_check_present_xattr(std::cerr));
} else if (val < 70) {
ASSERT_TRUE(tester.auto_check_absent_key(std::cerr));
- } else if (val < 73) {
+ } else if (val < 72) {
ASSERT_TRUE(tester.auto_check_absent_xattr(std::cerr));
+ } else if (val < 73) {
+ tester.auto_clear_omap(std::cerr);
} else if (val < 76) {
tester.auto_delete_object(std::cerr);
} else if (val < 85) {
diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc
index 6f2a215..78384a8 100644
--- a/src/test/admin_socket.cc
+++ b/src/test/admin_socket.cc
@@ -12,11 +12,15 @@
*
*/
+#include <gtest/gtest.h>
+
#include "common/Mutex.h"
+#include "common/Cond.h"
#include "common/admin_socket.h"
#include "common/admin_socket_client.h"
-#include "common/ceph_context.h"
-#include "test/unit.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
#include <stdint.h>
#include <string.h>
@@ -33,6 +37,9 @@ public:
bool init(const std::string &uri) {
return m_asokc->init(uri);
}
+ string bind_and_listen(const std::string &sock_path, int *fd) {
+ return m_asokc->bind_and_listen(sock_path, fd);
+ }
bool shutdown() {
m_asokc->shutdown();
return true;
@@ -146,3 +153,127 @@ TEST(AdminSocket, RegisterCommandPrefixes) {
ASSERT_EQ("test| this thing", result);
ASSERT_EQ(true, asoct.shutdown());
}
+
+class BlockingHook : public AdminSocketHook {
+public:
+ Mutex _lock;
+ Cond _cond;
+
+ BlockingHook() : _lock("BlockingHook::_lock") {}
+
+ bool call(std::string command, cmdmap_t& cmdmap, std::string format, bufferlist& result) {
+ Mutex::Locker l(_lock);
+ _cond.Wait(_lock);
+ return true;
+ }
+};
+
+TEST(AdminSocketClient, Ping) {
+ string path = get_rand_socket_path();
+ std::auto_ptr<AdminSocket>
+ asokc(new AdminSocket(g_ceph_context));
+ AdminSocketClient client(path);
+ // no socket
+ {
+ bool ok;
+ std::string result = client.ping(&ok);
+ EXPECT_NE(std::string::npos, result.find("No such file or directory"));
+ ASSERT_FALSE(ok);
+ }
+ // file exists but does not allow connections (no process, wrong type...)
+ ASSERT_TRUE(::creat(path.c_str(), 0777));
+ {
+ bool ok;
+ std::string result = client.ping(&ok);
+ EXPECT_NE(std::string::npos, result.find("Connection refused"));
+ ASSERT_FALSE(ok);
+ }
+ // a daemon is connected to the socket
+ {
+ AdminSocketTest asoct(asokc.get());
+ ASSERT_TRUE(asoct.init(path));
+ bool ok;
+ std::string result = client.ping(&ok);
+ EXPECT_EQ("", result);
+ ASSERT_TRUE(ok);
+ ASSERT_TRUE(asoct.shutdown());
+ }
+ // hardcoded five seconds timeout prevents infinite blockage
+ {
+ AdminSocketTest asoct(asokc.get());
+ BlockingHook *blocking = new BlockingHook();
+ ASSERT_EQ(0, asoct.m_asokc->register_command("0", "0", blocking, ""));
+ ASSERT_TRUE(asoct.init(path));
+ bool ok;
+ std::string result = client.ping(&ok);
+ EXPECT_NE(std::string::npos, result.find("Resource temporarily unavailable"));
+ ASSERT_FALSE(ok);
+ {
+ Mutex::Locker l(blocking->_lock);
+ blocking->_cond.Signal();
+ }
+ ASSERT_TRUE(asoct.shutdown());
+ delete blocking;
+ }
+}
+
+TEST(AdminSocket, bind_and_listen) {
+ string path = get_rand_socket_path();
+ std::auto_ptr<AdminSocket>
+ asokc(new AdminSocket(g_ceph_context));
+
+ AdminSocketTest asoct(asokc.get());
+ // successfull bind
+ {
+ int fd = 0;
+ string message;
+ message = asoct.bind_and_listen(path, &fd);
+ ASSERT_NE(0, fd);
+ ASSERT_EQ("", message);
+ ASSERT_EQ(0, ::close(fd));
+ ASSERT_EQ(0, ::unlink(path.c_str()));
+ }
+ // silently discard an existing file
+ {
+ int fd = 0;
+ string message;
+ ASSERT_TRUE(::creat(path.c_str(), 0777));
+ message = asoct.bind_and_listen(path, &fd);
+ ASSERT_NE(0, fd);
+ ASSERT_EQ("", message);
+ ASSERT_EQ(0, ::close(fd));
+ ASSERT_EQ(0, ::unlink(path.c_str()));
+ }
+ // do not take over a live socket
+ {
+ ASSERT_TRUE(asoct.init(path));
+ int fd = 0;
+ string message;
+ message = asoct.bind_and_listen(path, &fd);
+ EXPECT_NE(std::string::npos, message.find("File exists"));
+ ASSERT_TRUE(asoct.shutdown());
+ }
+}
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ vector<const char*> def_args;
+ global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd .. ;
+ * make unittest_admin_socket &&
+ * valgrind \
+ * --max-stackframe=20000000 --tool=memcheck \
+ * ./unittest_admin_socket --debug-asok 20 # --gtest_filter=AdminSocket*.*
+ * "
+ * End:
+ */
+
diff --git a/src/test/bench/bencher.cc b/src/test/bench/bencher.cc
index 32c3538..c178146 100644
--- a/src/test/bench/bencher.cc
+++ b/src/test/bench/bencher.cc
@@ -3,7 +3,7 @@
#include "bencher.h"
#include "include/utime.h"
#include <unistd.h>
-#include <tr1/memory>
+#include "include/memory.h"
#include "common/Mutex.h"
#include "common/Cond.h"
@@ -35,10 +35,10 @@ struct Cleanup : public Context {
struct OnWriteApplied : public Context {
Bencher *bench;
uint64_t seq;
- std::tr1::shared_ptr<OnDelete> on_delete;
+ ceph::shared_ptr<OnDelete> on_delete;
OnWriteApplied(
Bencher *bench, uint64_t seq,
- std::tr1::shared_ptr<OnDelete> on_delete
+ ceph::shared_ptr<OnDelete> on_delete
) : bench(bench), seq(seq), on_delete(on_delete) {}
void finish(int r) {
bench->stat_collector->write_applied(seq);
@@ -48,10 +48,10 @@ struct OnWriteApplied : public Context {
struct OnWriteCommit : public Context {
Bencher *bench;
uint64_t seq;
- std::tr1::shared_ptr<OnDelete> on_delete;
+ ceph::shared_ptr<OnDelete> on_delete;
OnWriteCommit(
Bencher *bench, uint64_t seq,
- std::tr1::shared_ptr<OnDelete> on_delete
+ ceph::shared_ptr<OnDelete> on_delete
) : bench(bench), seq(seq), on_delete(on_delete) {}
void finish(int r) {
bench->stat_collector->write_committed(seq);
@@ -120,7 +120,7 @@ void Bencher::init(
Cond cond;
bool done = 0;
{
- std::tr1::shared_ptr<OnFinish> on_finish(
+ ceph::shared_ptr<OnFinish> on_finish(
new OnFinish(&done, &lock, &cond));
uint64_t num = 0;
for (set<std::string>::const_iterator i = objects.begin();
@@ -132,8 +132,8 @@ void Bencher::init(
*i,
0,
bl,
- new C_Holder<std::tr1::shared_ptr<OnFinish> >(on_finish),
- new C_Holder<std::tr1::shared_ptr<OnFinish> >(on_finish)
+ new C_Holder<ceph::shared_ptr<OnFinish> >(on_finish),
+ new C_Holder<ceph::shared_ptr<OnFinish> >(on_finish)
);
}
}
@@ -162,7 +162,7 @@ void Bencher::run_bench()
OpType op_type = next.get<3>();
switch (op_type) {
case WRITE: {
- std::tr1::shared_ptr<OnDelete> on_delete(
+ ceph::shared_ptr<OnDelete> on_delete(
new OnDelete(new Cleanup(this)));
stat_collector->start_write(seq, length);
while (bl.length() < length) {
diff --git a/src/test/bench/bencher.h b/src/test/bench/bencher.h
index 6660d1d..a4f6321 100644
--- a/src/test/bench/bencher.h
+++ b/src/test/bench/bencher.h
@@ -12,10 +12,10 @@
#include "common/Cond.h"
#include "common/Thread.h"
-class OnWriteApplied;
-class OnWriteCommit;
-class OnReadComplete;
-class Clenaup;
+struct OnWriteApplied;
+struct OnWriteCommit;
+struct OnReadComplete;
+struct Cleanup;
class Bencher : public Thread {
public:
@@ -27,7 +27,7 @@ public:
private:
boost::scoped_ptr<
Distribution<boost::tuple<std::string,uint64_t,uint64_t, OpType> > > op_dist;
- std::tr1::shared_ptr<StatCollector> stat_collector;
+ ceph::shared_ptr<StatCollector> stat_collector;
boost::scoped_ptr<Backend> backend;
const uint64_t max_in_flight;
const uint64_t max_duration;
@@ -42,7 +42,7 @@ private:
public:
Bencher(
Distribution<boost::tuple<std::string, uint64_t, uint64_t, OpType> > *op_gen,
- std::tr1::shared_ptr<StatCollector> stat_collector,
+ ceph::shared_ptr<StatCollector> stat_collector,
Backend *backend,
uint64_t max_in_flight,
uint64_t max_duration,
@@ -105,10 +105,10 @@ public:
run_bench();
return 0;
}
- friend class OnWriteApplied;
- friend class OnWriteCommit;
- friend class OnReadComplete;
- friend class Cleanup;
+ friend struct OnWriteApplied;
+ friend struct OnWriteCommit;
+ friend struct OnReadComplete;
+ friend struct Cleanup;
};
class SequentialLoad :
diff --git a/src/test/bench/dumb_backend.cc b/src/test/bench/dumb_backend.cc
index c36dce3..bf78ab2 100644
--- a/src/test/bench/dumb_backend.cc
+++ b/src/test/bench/dumb_backend.cc
@@ -1,5 +1,7 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "acconfig.h"
+
#include <unistd.h>
#include "dumb_backend.h"
@@ -34,15 +36,23 @@ void DumbBackend::_write(
on_applied->complete(0);
if (do_fsync)
::fsync(fd);
+#ifdef HAVE_SYNC_FILE_RANGE
if (do_sync_file_range)
::sync_file_range(fd, offset, bl.length(),
SYNC_FILE_RANGE_WAIT_AFTER);
+#else
+# warning "sync_file_range not supported!"
+#endif
+#ifdef HAVE_POSIX_FADVISE
if (do_fadvise) {
int fa_r = ::posix_fadvise(fd, offset, bl.length(), POSIX_FADV_DONTNEED);
if (fa_r) {
std::cout << "posix_fadvise failed, errno is: " << fa_r << std::endl;
}
}
+#else
+# warning "posix_fadvise not supported!"
+#endif
::close(fd);
{
Mutex::Locker l(pending_commit_mutex);
diff --git a/src/test/bench/rbd_backend.cc b/src/test/bench/rbd_backend.cc
index 51df9b9..7928896 100644
--- a/src/test/bench/rbd_backend.cc
+++ b/src/test/bench/rbd_backend.cc
@@ -26,7 +26,7 @@ void RBDBackend::write(
Context *on_commit)
{
bufferlist &bl_non_const = const_cast<bufferlist&>(bl);
- std::tr1::shared_ptr<librbd::Image> image = (*m_images)[oid];
+ ceph::shared_ptr<librbd::Image> image = (*m_images)[oid];
void *arg = static_cast<void *>(new arg_type(on_commit, on_write_applied));
librbd::RBD::AioCompletion *completion =
new librbd::RBD::AioCompletion(arg, on_complete);
@@ -42,7 +42,7 @@ void RBDBackend::read(
bufferlist *bl,
Context *on_read_complete)
{
- std::tr1::shared_ptr<librbd::Image> image = (*m_images)[oid];
+ ceph::shared_ptr<librbd::Image> image = (*m_images)[oid];
void *arg = static_cast<void *>(new arg_type(on_read_complete, NULL));
librbd::RBD::AioCompletion *completion =
new librbd::RBD::AioCompletion(arg, on_complete);
diff --git a/src/test/bench/rbd_backend.h b/src/test/bench/rbd_backend.h
index 981c8d7..9c3bc8b 100644
--- a/src/test/bench/rbd_backend.h
+++ b/src/test/bench/rbd_backend.h
@@ -8,9 +8,9 @@
#include "include/rbd/librbd.hpp"
class RBDBackend : public Backend {
- map<string, std::tr1::shared_ptr<librbd::Image> > *m_images;
+ map<string, ceph::shared_ptr<librbd::Image> > *m_images;
public:
- RBDBackend(map<string, std::tr1::shared_ptr<librbd::Image> > *images)
+ RBDBackend(map<string, ceph::shared_ptr<librbd::Image> > *images)
: m_images(images) {}
void write(
const string &oid,
diff --git a/src/test/bench/small_io_bench_dumb.cc b/src/test/bench/small_io_bench_dumb.cc
index 6d79fdd..73841c3 100644
--- a/src/test/bench/small_io_bench_dumb.cc
+++ b/src/test/bench/small_io_bench_dumb.cc
@@ -1,5 +1,7 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "acconfig.h"
+
#include <boost/scoped_ptr.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/program_options/option.hpp>
@@ -191,6 +193,16 @@ int main(int argc, char **argv)
);
}
+#ifndef HAVE_SYNC_FILE_RANGE
+ if (vm["sync-file-range"].as<bool>())
+ std::cerr << "Warning: sync_file_range(2) not supported!" << std::endl;
+#endif
+
+#ifndef HAVE_POSIX_FADVISE
+ if (vm["fadvise"].as<bool>())
+ std::cerr << "Warning: posix_fadvise(2) not supported!" << std::endl;
+#endif
+
Bencher bencher(
gen,
new DetailedStatCollector(1, new JSONFormatter, detailed_ops, &cout),
diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc
index 6f4ac07..7f03f5a 100644
--- a/src/test/bench/small_io_bench_fs.cc
+++ b/src/test/bench/small_io_bench_fs.cc
@@ -150,7 +150,7 @@ int main(int argc, char **argv)
detailed_ops = &cerr;
}
- std::tr1::shared_ptr<StatCollector> col(
+ ceph::shared_ptr<StatCollector> col(
new DetailedStatCollector(
1, new JSONFormatter, detailed_ops, &cout,
new MorePrinting(g_ceph_context)));
@@ -175,9 +175,9 @@ int main(int argc, char **argv)
fs.apply_transaction(t);
}
- vector<std::tr1::shared_ptr<Bencher> > benchers(
+ vector<ceph::shared_ptr<Bencher> > benchers(
vm["num-writers"].as<unsigned>());
- for (vector<std::tr1::shared_ptr<Bencher> >::iterator i = benchers.begin();
+ for (vector<ceph::shared_ptr<Bencher> >::iterator i = benchers.begin();
i != benchers.end();
++i) {
set<string> objects;
@@ -227,12 +227,12 @@ int main(int argc, char **argv)
(*i).reset(bencher);
}
- for (vector<std::tr1::shared_ptr<Bencher> >::iterator i = benchers.begin();
+ for (vector<ceph::shared_ptr<Bencher> >::iterator i = benchers.begin();
i != benchers.end();
++i) {
(*i)->create();
}
- for (vector<std::tr1::shared_ptr<Bencher> >::iterator i = benchers.begin();
+ for (vector<ceph::shared_ptr<Bencher> >::iterator i = benchers.begin();
i != benchers.end();
++i) {
(*i)->join();
diff --git a/src/test/bench/small_io_bench_rbd.cc b/src/test/bench/small_io_bench_rbd.cc
index 4c56ffd..ba7071e 100644
--- a/src/test/bench/small_io_bench_rbd.cc
+++ b/src/test/bench/small_io_bench_rbd.cc
@@ -132,7 +132,7 @@ int main(int argc, char **argv)
librbd::RBD rbd;
{
- map<string, std::tr1::shared_ptr<librbd::Image> > images;
+ map<string, ceph::shared_ptr<librbd::Image> > images;
int order = vm["order"].as<unsigned>();
uint64_t image_size = ((uint64_t)vm["image-size"].as<unsigned>()) << 20;
for (set<string>::const_iterator i = image_names.begin();
@@ -142,7 +142,7 @@ int main(int argc, char **argv)
cerr << "error creating image " << *i << " r=" << r << std::endl;
return -r;
}
- std::tr1::shared_ptr<librbd::Image> image(new librbd::Image());
+ ceph::shared_ptr<librbd::Image> image(new librbd::Image());
r = rbd.open(ioctx, *image, i->c_str());
if (r < 0) {
cerr << "error opening image " << *i << " r=" << r << std::endl;
diff --git a/src/test/bufferlist.cc b/src/test/bufferlist.cc
index 8b6ca26..64253c9 100644
--- a/src/test/bufferlist.cc
+++ b/src/test/bufferlist.cc
@@ -19,7 +19,7 @@
*
*/
-#include <tr1/memory>
+#include "include/memory.h"
#include <limits.h>
#include <errno.h>
#include <sys/uio.h>
@@ -29,6 +29,7 @@
#include "include/encoding.h"
#include "common/environment.h"
#include "common/Clock.h"
+#include "common/safe_io.h"
#include "gtest/gtest.h"
#include "stdlib.h"
@@ -141,6 +142,25 @@ TEST(Buffer, constructors) {
bufferptr clone = ptr.clone();
EXPECT_EQ(0, ::memcmp(clone.c_str(), ptr.c_str(), len));
}
+#ifdef CEPH_HAVE_SPLICE
+ if (ceph_buffer_track)
+ EXPECT_EQ(0, buffer::get_total_alloc());
+ {
+ // no fd
+ EXPECT_THROW(buffer::create_zero_copy(len, -1, NULL), buffer::error_code);
+
+ unsigned zc_len = 4;
+ ::unlink("testfile");
+ ::system("echo ABC > testfile");
+ int fd = ::open("testfile", O_RDONLY);
+ bufferptr ptr(buffer::create_zero_copy(zc_len, fd, NULL));
+ EXPECT_EQ(zc_len, ptr.length());
+ if (ceph_buffer_track)
+ EXPECT_EQ(zc_len, (unsigned)buffer::get_total_alloc());
+ ::close(fd);
+ ::unlink("testfile");
+ }
+#endif
if (ceph_buffer_track)
EXPECT_EQ(0, buffer::get_total_alloc());
}
@@ -153,6 +173,146 @@ TEST(BufferRaw, ostream) {
EXPECT_GT(stream.str().size(), stream.str().find("len 1 nref 1)"));
}
+#ifdef CEPH_HAVE_SPLICE
+class TestRawPipe : public ::testing::Test {
+protected:
+ virtual void SetUp() {
+ len = 4;
+ ::unlink("testfile");
+ ::system("echo ABC > testfile");
+ fd = ::open("testfile", O_RDONLY);
+ assert(fd >= 0);
+ }
+ virtual void TearDown() {
+ ::close(fd);
+ ::unlink("testfile");
+ }
+ int fd;
+ unsigned len;
+};
+
+TEST_F(TestRawPipe, create_zero_copy) {
+ bufferptr ptr(buffer::create_zero_copy(len, fd, NULL));
+ EXPECT_EQ(len, ptr.length());
+ if (get_env_bool("CEPH_BUFFER_TRACK"))
+ EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+}
+
+TEST_F(TestRawPipe, c_str_no_fd) {
+ EXPECT_THROW(bufferptr ptr(buffer::create_zero_copy(len, -1, NULL)),
+ buffer::error_code);
+}
+
+TEST_F(TestRawPipe, c_str_basic) {
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, NULL));
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+ EXPECT_EQ(len, ptr.length());
+}
+
+TEST_F(TestRawPipe, c_str_twice) {
+ // make sure we're creating a copy of the data and not consuming it
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, NULL));
+ EXPECT_EQ(len, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+}
+
+TEST_F(TestRawPipe, c_str_basic_offset) {
+ loff_t offset = len - 1;
+ ::lseek(fd, offset, SEEK_SET);
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len - offset, fd, NULL));
+ EXPECT_EQ(len - offset, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "\n", len - offset));
+}
+
+TEST_F(TestRawPipe, c_str_dest_short) {
+ ::lseek(fd, 1, SEEK_SET);
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(2, fd, NULL));
+ EXPECT_EQ(2u, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "BC", 2));
+}
+
+TEST_F(TestRawPipe, c_str_source_short) {
+ ::lseek(fd, 1, SEEK_SET);
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, NULL));
+ EXPECT_EQ(len - 1, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "BC\n", len - 1));
+}
+
+TEST_F(TestRawPipe, c_str_explicit_zero_offset) {
+ int64_t offset = 0;
+ ::lseek(fd, 1, SEEK_SET);
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, &offset));
+ EXPECT_EQ(len, offset);
+ EXPECT_EQ(len, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+}
+
+TEST_F(TestRawPipe, c_str_explicit_positive_offset) {
+ int64_t offset = 1;
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len - offset, fd,
+ &offset));
+ EXPECT_EQ(len, offset);
+ EXPECT_EQ(len - 1, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "BC\n", len - 1));
+}
+
+TEST_F(TestRawPipe, c_str_explicit_positive_empty_result) {
+ int64_t offset = len;
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len - offset, fd,
+ &offset));
+ EXPECT_EQ(len, offset);
+ EXPECT_EQ(0u, ptr.length());
+}
+
+TEST_F(TestRawPipe, c_str_source_short_explicit_offset) {
+ int64_t offset = 1;
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, &offset));
+ EXPECT_EQ(len, offset);
+ EXPECT_EQ(len - 1, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "BC\n", len - 1));
+}
+
+TEST_F(TestRawPipe, c_str_dest_short_explicit_offset) {
+ int64_t offset = 1;
+ bufferptr ptr = bufferptr(buffer::create_zero_copy(2, fd, &offset));
+ EXPECT_EQ(3, offset);
+ EXPECT_EQ(2u, ptr.length());
+ EXPECT_EQ(0, memcmp(ptr.c_str(), "BC", 2));
+}
+
+TEST_F(TestRawPipe, buffer_list_read_fd_zero_copy) {
+ bufferlist bl;
+ EXPECT_EQ(-EBADF, bl.read_fd_zero_copy(-1, len));
+ bl = bufferlist();
+ EXPECT_EQ(0, bl.read_fd_zero_copy(fd, len));
+ EXPECT_EQ(len, bl.length());
+ EXPECT_EQ(0u, bl.buffers().front().unused_tail_length());
+ EXPECT_EQ(1u, bl.buffers().size());
+ EXPECT_EQ(len, bl.buffers().front().raw_length());
+ EXPECT_EQ(0, memcmp(bl.c_str(), "ABC\n", len));
+ EXPECT_TRUE(bl.can_zero_copy());
+}
+
+TEST_F(TestRawPipe, buffer_list_write_fd_zero_copy) {
+ ::unlink("testfile_out");
+ bufferlist bl;
+ EXPECT_EQ(0, bl.read_fd_zero_copy(fd, len));
+ EXPECT_TRUE(bl.can_zero_copy());
+ int out_fd = ::open("testfile_out", O_RDWR|O_CREAT|O_TRUNC, 0600);
+ EXPECT_EQ(0, bl.write_fd_zero_copy(out_fd));
+ struct stat st;
+ memset(&st, 0, sizeof(st));
+ EXPECT_EQ(0, ::stat("testfile_out", &st));
+ EXPECT_EQ(len, st.st_size);
+ char buf[len + 1];
+ EXPECT_EQ(len, safe_read(out_fd, buf, len + 1));
+ EXPECT_EQ(0, memcmp(buf, "ABC\n", len));
+ ::close(out_fd);
+ ::unlink("testfile_out");
+}
+#endif // CEPH_HAVE_SPLICE
+
//
// +-----------+ +-----+
// | | | |
@@ -969,17 +1129,17 @@ TEST(BufferList, is_page_aligned) {
}
{
bufferlist bl;
- bufferptr ptr(2);
+ bufferptr ptr(buffer::create_page_aligned(2));
ptr.set_offset(1);
ptr.set_length(1);
bl.append(ptr);
EXPECT_FALSE(bl.is_page_aligned());
bl.rebuild_page_aligned();
- EXPECT_FALSE(bl.is_page_aligned());
+ EXPECT_TRUE(bl.is_page_aligned());
}
{
bufferlist bl;
- bufferptr ptr(CEPH_PAGE_SIZE + 1);
+ bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
ptr.set_offset(1);
ptr.set_length(CEPH_PAGE_SIZE);
bl.append(ptr);
@@ -1126,7 +1286,7 @@ TEST(BufferList, is_contiguous) {
TEST(BufferList, rebuild) {
{
bufferlist bl;
- bufferptr ptr(2);
+ bufferptr ptr(buffer::create_page_aligned(2));
ptr.set_offset(1);
ptr.set_length(1);
bl.append(ptr);
@@ -1151,7 +1311,7 @@ TEST(BufferList, rebuild_page_aligned) {
{
bufferlist bl;
{
- bufferptr ptr(CEPH_PAGE_SIZE + 1);
+ bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
ptr.set_offset(1);
ptr.set_length(CEPH_PAGE_SIZE);
bl.append(ptr);
@@ -1166,30 +1326,42 @@ TEST(BufferList, rebuild_page_aligned) {
bufferlist bl;
{
bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE));
+ EXPECT_TRUE(ptr.is_page_aligned());
+ EXPECT_TRUE(ptr.is_n_page_sized());
bl.append(ptr);
}
{
- bufferptr ptr(CEPH_PAGE_SIZE + 1);
+ bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
+ EXPECT_TRUE(ptr.is_page_aligned());
+ EXPECT_FALSE(ptr.is_n_page_sized());
bl.append(ptr);
}
{
- bufferptr ptr(2);
+ bufferptr ptr(buffer::create_page_aligned(2));
ptr.set_offset(1);
ptr.set_length(1);
+ EXPECT_FALSE(ptr.is_page_aligned());
+ EXPECT_FALSE(ptr.is_n_page_sized());
bl.append(ptr);
}
{
- bufferptr ptr(CEPH_PAGE_SIZE - 2);
+ bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE - 2));
+ EXPECT_TRUE(ptr.is_page_aligned());
+ EXPECT_FALSE(ptr.is_n_page_sized());
bl.append(ptr);
}
{
bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE));
+ EXPECT_TRUE(ptr.is_page_aligned());
+ EXPECT_TRUE(ptr.is_n_page_sized());
bl.append(ptr);
}
{
- bufferptr ptr(CEPH_PAGE_SIZE + 1);
+ bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
ptr.set_offset(1);
ptr.set_length(CEPH_PAGE_SIZE);
+ EXPECT_FALSE(ptr.is_page_aligned());
+ EXPECT_TRUE(ptr.is_n_page_sized());
bl.append(ptr);
}
EXPECT_EQ((unsigned)6, bl.buffers().size());
@@ -1444,7 +1616,7 @@ TEST(BufferList, append) {
//
{
bufferlist bl;
- std::string expected("ABC\n\nDEF\n");
+ std::string expected("ABC\nDEF\n");
std::istringstream is("ABC\n\nDEF");
bl.append(is);
EXPECT_EQ(0, ::memcmp(expected.c_str(), bl.c_str(), expected.size()));
@@ -1522,7 +1694,7 @@ TEST(BufferList, splice) {
bl.push_back(ptr);
}
EXPECT_EQ((unsigned)4, bl.buffers().size());
- EXPECT_THROW(bl.splice(0, 0), FailedAssertion);
+ bl.splice(0, 0);
bufferlist other;
other.append('X');
@@ -1604,8 +1776,8 @@ TEST(BufferList, read_fd) {
EXPECT_EQ(-EBADF, bl.read_fd(fd, len));
fd = ::open("testfile", O_RDONLY);
EXPECT_EQ(len, (unsigned)bl.read_fd(fd, len));
- EXPECT_EQ(len, bl.length());
EXPECT_EQ(CEPH_PAGE_SIZE - len, bl.buffers().front().unused_tail_length());
+ EXPECT_EQ(len, bl.length());
::close(fd);
::unlink("testfile");
}
@@ -1972,7 +2144,7 @@ TEST(BufferList, TestDirectAppend) {
TEST(BufferList, TestCopyAll) {
const static size_t BIG_SZ = 10737414;
- std::tr1::shared_ptr <unsigned char> big(
+ ceph::shared_ptr <unsigned char> big(
(unsigned char*)malloc(BIG_SZ), free);
unsigned char c = 0;
for (size_t i = 0; i < BIG_SZ; ++i) {
@@ -1984,7 +2156,7 @@ TEST(BufferList, TestCopyAll) {
bufferlist bl2;
i.copy_all(bl2);
ASSERT_EQ(bl2.length(), BIG_SZ);
- std::tr1::shared_ptr <unsigned char> big2(
+ ceph::shared_ptr <unsigned char> big2(
(unsigned char*)malloc(BIG_SZ), free);
bl2.copy(0, BIG_SZ, (char*)big2.get());
ASSERT_EQ(memcmp(big.get(), big2.get(), BIG_SZ), 0);
@@ -2012,6 +2184,12 @@ TEST(BufferHash, all) {
}
}
-// Local Variables:
-// compile-command: "cd .. ; make unittest_bufferlist ; ulimit -s unlimited ; CEPH_BUFFER_TRACK=true valgrind --max-stackframe=20000000 --tool=memcheck ./unittest_bufferlist # --gtest_filter=BufferList.constructors"
-// End:
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make unittest_bufferlist &&
+ * ulimit -s unlimited ; CEPH_BUFFER_TRACK=true valgrind \
+ * --max-stackframe=20000000 --tool=memcheck \
+ * ./unittest_bufferlist # --gtest_filter=BufferList.constructors"
+ * End:
+ */
+
diff --git a/src/test/ceph_crypto.cc b/src/test/ceph_crypto.cc
index 2c934fa..11d4101 100644
--- a/src/test/ceph_crypto.cc
+++ b/src/test/ceph_crypto.cc
@@ -135,6 +135,8 @@ void do_simple_crypto() {
exit(0);
}
+#if GTEST_HAS_DEATH_TEST
TEST_F(ForkDeathTest, MD5) {
ASSERT_EXIT(do_simple_crypto(), ::testing::ExitedWithCode(0), "^$");
}
+#endif //GTEST_HAS_DEATH_TEST
diff --git a/src/test/cli/ceph-conf/show-config-value.t b/src/test/cli/ceph-conf/show-config-value.t
index 03d7ed8..f2f8204 100644
--- a/src/test/cli/ceph-conf/show-config-value.t
+++ b/src/test/cli/ceph-conf/show-config-value.t
@@ -3,3 +3,18 @@
$ ceph-conf -n osd.0 --show-config-value log_file -c /dev/null
/var/log/ceph/ceph-osd.0.log
+ $ CEPH_ARGS="--fsid 96a3abe6-7552-4635-a79b-f3c096ff8b95" ceph-conf -n osd.0 --show-config-value fsid -c /dev/null
+ 96a3abe6-7552-4635-a79b-f3c096ff8b95
+ $ ceph-conf -n osd.0 --show-config-value INVALID -c /dev/null
+ failed to get config option 'INVALID': option not found
+ [1]
+ $ echo '[global]' > $TESTDIR/ceph.conf
+ $ echo 'mon_host=$public_network' >> $TESTDIR/ceph.conf
+ $ echo 'public_network=$mon_host' >> $TESTDIR/ceph.conf
+ $ ceph-conf --show-config-value mon_host -c $TESTDIR/ceph.conf
+ variable expansion loop at public_network=$mon_host
+ expansion stack:
+ mon_host=$public_network
+ public_network=$mon_host
+ $mon_host
+ $ rm $TESTDIR/ceph.conf
diff --git a/src/test/cli/ceph-conf/show-config.t b/src/test/cli/ceph-conf/show-config.t
new file mode 100644
index 0000000..cfd7239
--- /dev/null
+++ b/src/test/cli/ceph-conf/show-config.t
@@ -0,0 +1,6 @@
+ $ ceph-conf -n osd.0 --show-config -c /dev/null | grep ceph-osd
+ admin_socket = /var/run/ceph/ceph-osd.0.asok
+ log_file = /var/log/ceph/ceph-osd.0.log
+ mon_debug_dump_location = /var/log/ceph/ceph-osd.0.tdump
+ $ CEPH_ARGS="--fsid 96a3abe6-7552-4635-a79b-f3c096ff8b95" ceph-conf -n osd.0 --show-config -c /dev/null | grep fsid
+ fsid = 96a3abe6-7552-4635-a79b-f3c096ff8b95
diff --git a/src/test/cli/crushtool/bad-mappings.crushmap.txt b/src/test/cli/crushtool/bad-mappings.crushmap.txt
new file mode 100644
index 0000000..a3d5dfe
--- /dev/null
+++ b/src/test/cli/crushtool/bad-mappings.crushmap.txt
@@ -0,0 +1,39 @@
+device 0 device0
+device 1 device1
+device 2 device2
+device 3 device3
+device 4 device4
+
+type 0 osd
+type 1 domain
+
+domain root {
+ id -1
+ alg straw
+ hash 0
+ item device0 weight 1.000
+ item device1 weight 1.000
+ item device2 weight 1.000
+ item device3 weight 1.000
+ item device4 weight 1.000
+}
+
+rule rule-firstn {
+ ruleset 0
+ type replicated
+ min_size 1
+ max_size 10
+ step take root
+ step choose firstn 0 type osd
+ step emit
+}
+
+rule rule-indep {
+ ruleset 1
+ type erasure
+ min_size 1
+ max_size 10
+ step take root
+ step choose indep 0 type osd
+ step emit
+}
diff --git a/src/test/cli/crushtool/bad-mappings.t b/src/test/cli/crushtool/bad-mappings.t
new file mode 100644
index 0000000..dbdcd57
--- /dev/null
+++ b/src/test/cli/crushtool/bad-mappings.t
@@ -0,0 +1,6 @@
+ $ crushtool -c "$TESTDIR/bad-mappings.crushmap.txt" -o "$TESTDIR/bad-mappings.crushmap"
+ $ crushtool -i "$TESTDIR/bad-mappings.crushmap" --test --show-bad-mappings --rule 0 --x 1 --num-rep 10
+ bad mapping rule 0 x 1 num_rep 10 result [4,0,2,3,1]
+ $ crushtool -i "$TESTDIR/bad-mappings.crushmap" --test --show-bad-mappings --rule 1 --x 1 --num-rep 10
+ bad mapping rule 1 x 1 num_rep 10 result [4,0,2,1,3,2147483647,2147483647,2147483647,2147483647,2147483647]
+ $ rm -f "$TESTDIR/bad-mappings.crushmap"
diff --git a/src/test/cli/crushtool/build.t b/src/test/cli/crushtool/build.t
new file mode 100644
index 0000000..ca0804d
--- /dev/null
+++ b/src/test/cli/crushtool/build.t
@@ -0,0 +1,104 @@
+ $ map="$TESTDIR/build.crushmap"
+
+#
+# display the crush tree by default
+#
+ $ crushtool --outfn "$map" --build --num_osds 5 node straw 2 rack straw 1 root straw 0
+ .* (re)
+ # id\tweight\ttype name\treweight (esc)
+ -7\t5\troot root (esc)
+ -4\t2\t\track rack0 (esc)
+ -1\t2\t\t\tnode node0 (esc)
+ 0\t1\t\t\t\tosd.0\t1\t (esc)
+ 1\t1\t\t\t\tosd.1\t1\t (esc)
+ -5\t2\t\track rack1 (esc)
+ -2\t2\t\t\tnode node1 (esc)
+ 2\t1\t\t\t\tosd.2\t1\t (esc)
+ 3\t1\t\t\t\tosd.3\t1\t (esc)
+ -6\t1\t\track rack2 (esc)
+ -3\t1\t\t\tnode node2 (esc)
+ 4\t1\t\t\t\tosd.4\t1\t (esc)
+
+
+#
+# silence all messages with --debug-crush 0
+#
+ $ CEPH_ARGS="--debug-crush 0" crushtool --outfn "$map" --build --num_osds 5 node straw 2 rack straw 1 root straw 0
+
+#
+# display a warning if there is more than one root
+#
+ $ crushtool --outfn "$map" --build --num_osds 5 node straw 2 rack straw 1
+ .* (re)
+ # id\tweight\ttype name\treweight (esc)
+ -6\t1\track rack2 (esc)
+ -3\t1\t\tnode node2 (esc)
+ 4\t1\t\t\tosd.4\t1\t (esc)
+ -5\t2\track rack1 (esc)
+ -2\t2\t\tnode node1 (esc)
+ 2\t1\t\t\tosd.2\t1\t (esc)
+ 3\t1\t\t\tosd.3\t1\t (esc)
+ -4\t2\track rack0 (esc)
+ -1\t2\t\tnode node0 (esc)
+ 0\t1\t\t\tosd.0\t1\t (esc)
+ 1\t1\t\t\tosd.1\t1\t (esc)
+
+ .* The crush rulesets will use the root rack0 (re)
+ and ignore the others.
+ There are 3 roots, they can be
+ grouped into a single root by appending something like:
+ root straw 0
+
+#
+# crush rulesets are generated using the OSDMap helpers
+#
+ $ CEPH_ARGS="--debug-crush 0" crushtool --outfn "$map" --build --num_osds 1 root straw 0
+ $ crushtool -o "$map.txt" -d "$map"
+ $ cat "$map.txt"
+ # begin crush map
+ tunable choose_local_tries 0
+ tunable choose_local_fallback_tries 0
+ tunable choose_total_tries 50
+ tunable chooseleaf_descend_once 1
+
+ # devices
+ device 0 device0
+
+ # types
+ type 0 device
+ type 1 root
+
+ # buckets
+ root root {
+ \tid -1\t\t# do not change unnecessarily (esc)
+ \t# weight 1.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem device0 weight 1.000 (esc)
+ }
+
+ # rules
+ rule replicated_ruleset {
+ \truleset 0 (esc)
+ \ttype replicated (esc)
+ \tmin_size 1 (esc)
+ \tmax_size 10 (esc)
+ \tstep take root (esc)
+ \tstep chooseleaf firstn 0 type root (esc)
+ \tstep emit (esc)
+ }
+
+ # end crush map
+ $ rm "$map" "$map.txt"
+
+#
+# Wrong number of arguments
+#
+ $ crushtool --outfn "$map" --debug-crush 0 --build --num_osds 5 node straw 0
+ remaining args: [--debug-crush,0,node,straw,0]
+ layers must be specified with 3-tuples of (name, buckettype, size)
+ [1]
+
+# Local Variables:
+# compile-command: "cd ../../.. ; make crushtool && test/run-cli-tests"
+# End:
diff --git a/src/test/cli/crushtool/set-choose.crushmap.txt b/src/test/cli/crushtool/set-choose.crushmap.txt
new file mode 100644
index 0000000..bc890e0
--- /dev/null
+++ b/src/test/cli/crushtool/set-choose.crushmap.txt
@@ -0,0 +1,144 @@
+# begin crush map
+tunable choose_local_tries 1
+tunable choose_local_fallback_tries 2
+tunable choose_total_tries 35
+tunable chooseleaf_descend_once 1
+
+# devices
+device 0 device0
+device 1 device1
+device 2 device2
+device 3 device3
+device 4 device4
+device 5 device5
+device 6 device6
+device 7 device7
+device 8 device8
+
+# types
+type 0 device
+type 1 host
+type 2 rack
+type 3 root
+
+# buckets
+host host0 {
+ id -1 # do not change unnecessarily
+ # weight 3.000
+ alg straw
+ hash 0 # rjenkins1
+ item device0 weight 1.000
+ item device1 weight 1.000
+ item device2 weight 1.000
+}
+host host1 {
+ id -2 # do not change unnecessarily
+ # weight 3.000
+ alg straw
+ hash 0 # rjenkins1
+ item device3 weight 1.000
+ item device4 weight 1.000
+ item device5 weight 1.000
+}
+host host2 {
+ id -3 # do not change unnecessarily
+ # weight 3.000
+ alg straw
+ hash 0 # rjenkins1
+ item device6 weight 1.000
+ item device7 weight 1.000
+ item device8 weight 1.000
+}
+rack rack0 {
+ id -4 # do not change unnecessarily
+ # weight 9.000
+ alg straw
+ hash 0 # rjenkins1
+ item host0 weight 3.000
+ item host1 weight 3.000
+ item host2 weight 3.000
+}
+root root0 {
+ id -5 # do not change unnecessarily
+ # weight 9.000
+ alg straw
+ hash 0 # rjenkins1
+ item rack0 weight 9.000
+}
+
+# rules
+rule choose {
+ ruleset 1
+ type replicated
+ min_size 2
+ max_size 3
+ step take root0
+ step choose firstn 0 type host
+ step choose firstn 1 type device
+ step emit
+}
+
+rule choose-two {
+ ruleset 2
+ type replicated
+ min_size 2
+ max_size 3
+ step take root0
+ step choose firstn 0 type device
+ step emit
+}
+
+rule chooseleaf {
+ ruleset 3
+ type replicated
+ min_size 2
+ max_size 3
+ step take root0
+ step chooseleaf firstn 0 type host
+ step emit
+}
+
+rule choose-set {
+ ruleset 4
+ type replicated
+ min_size 2
+ max_size 3
+ step set_choose_tries 3
+ step set_choose_local_tries 2
+ step set_choose_local_fallback_tries 2
+ step set_chooseleaf_tries 3
+ step take root0
+ step choose firstn 0 type host
+ step choose firstn 1 type device
+ step emit
+}
+
+rule choose-set-two {
+ ruleset 5
+ type replicated
+ min_size 2
+ max_size 3
+ step set_choose_tries 3
+ step set_choose_local_tries 2
+ step set_choose_local_fallback_tries 2
+ step set_chooseleaf_tries 3
+ step take root0
+ step choose firstn 0 type device
+ step emit
+}
+
+rule chooseleaf-set {
+ ruleset 6
+ type replicated
+ min_size 2
+ max_size 3
+ step set_choose_tries 3
+ step set_choose_local_tries 2
+ step set_choose_local_fallback_tries 2
+ step set_chooseleaf_tries 3
+ step take root0
+ step chooseleaf firstn 0 type host
+ step emit
+}
+
+# end crush map
diff --git a/src/test/cli/crushtool/set-choose.t b/src/test/cli/crushtool/set-choose.t
new file mode 100644
index 0000000..e160ad7
--- /dev/null
+++ b/src/test/cli/crushtool/set-choose.t
@@ -0,0 +1,36941 @@
+ $ crushtool -c "$TESTDIR/set-choose.crushmap.txt" -o set-choose.crushmap
+ $ crushtool -i set-choose.crushmap --test --show-statistics
+ rule 0 (choose), x = 0..1023, numrep = 2..3
+ CRUSH rule 0 x 0 [0,3]
+ CRUSH rule 0 x 1 [0,8]
+ CRUSH rule 0 x 2 [1,4]
+ CRUSH rule 0 x 3 [8,0]
+ CRUSH rule 0 x 4 [5,1]
+ CRUSH rule 0 x 5 [7,0]
+ CRUSH rule 0 x 6 [2,6]
+ CRUSH rule 0 x 7 [5,6]
+ CRUSH rule 0 x 8 [5,7]
+ CRUSH rule 0 x 9 [2,4]
+ CRUSH rule 0 x 10 [0,8]
+ CRUSH rule 0 x 11 [0,6]
+ CRUSH rule 0 x 12 [0,3]
+ CRUSH rule 0 x 13 [3,8]
+ CRUSH rule 0 x 14 [7,1]
+ CRUSH rule 0 x 15 [7,1]
+ CRUSH rule 0 x 16 [3,7]
+ CRUSH rule 0 x 17 [5,1]
+ CRUSH rule 0 x 18 [1,3]
+ CRUSH rule 0 x 19 [7,5]
+ CRUSH rule 0 x 20 [2,4]
+ CRUSH rule 0 x 21 [3,6]
+ CRUSH rule 0 x 22 [8,5]
+ CRUSH rule 0 x 23 [3,7]
+ CRUSH rule 0 x 24 [1,6]
+ CRUSH rule 0 x 25 [3,8]
+ CRUSH rule 0 x 26 [2,7]
+ CRUSH rule 0 x 27 [3,1]
+ CRUSH rule 0 x 28 [6,2]
+ CRUSH rule 0 x 29 [8,5]
+ CRUSH rule 0 x 30 [5,6]
+ CRUSH rule 0 x 31 [8,2]
+ CRUSH rule 0 x 32 [3,7]
+ CRUSH rule 0 x 33 [2,7]
+ CRUSH rule 0 x 34 [2,3]
+ CRUSH rule 0 x 35 [0,6]
+ CRUSH rule 0 x 36 [3,6]
+ CRUSH rule 0 x 37 [0,4]
+ CRUSH rule 0 x 38 [4,6]
+ CRUSH rule 0 x 39 [3,7]
+ CRUSH rule 0 x 40 [7,2]
+ CRUSH rule 0 x 41 [0,7]
+ CRUSH rule 0 x 42 [4,7]
+ CRUSH rule 0 x 43 [0,3]
+ CRUSH rule 0 x 44 [1,8]
+ CRUSH rule 0 x 45 [8,2]
+ CRUSH rule 0 x 46 [2,4]
+ CRUSH rule 0 x 47 [4,1]
+ CRUSH rule 0 x 48 [4,6]
+ CRUSH rule 0 x 49 [5,6]
+ CRUSH rule 0 x 50 [3,1]
+ CRUSH rule 0 x 51 [3,6]
+ CRUSH rule 0 x 52 [8,2]
+ CRUSH rule 0 x 53 [3,6]
+ CRUSH rule 0 x 54 [7,4]
+ CRUSH rule 0 x 55 [8,0]
+ CRUSH rule 0 x 56 [6,5]
+ CRUSH rule 0 x 57 [5,8]
+ CRUSH rule 0 x 58 [1,8]
+ CRUSH rule 0 x 59 [4,0]
+ CRUSH rule 0 x 60 [3,1]
+ CRUSH rule 0 x 61 [4,8]
+ CRUSH rule 0 x 62 [7,0]
+ CRUSH rule 0 x 63 [5,6]
+ CRUSH rule 0 x 64 [4,1]
+ CRUSH rule 0 x 65 [7,4]
+ CRUSH rule 0 x 66 [5,6]
+ CRUSH rule 0 x 67 [5,0]
+ CRUSH rule 0 x 68 [0,3]
+ CRUSH rule 0 x 69 [5,2]
+ CRUSH rule 0 x 70 [7,2]
+ CRUSH rule 0 x 71 [2,7]
+ CRUSH rule 0 x 72 [6,2]
+ CRUSH rule 0 x 73 [2,7]
+ CRUSH rule 0 x 74 [0,8]
+ CRUSH rule 0 x 75 [3,0]
+ CRUSH rule 0 x 76 [5,2]
+ CRUSH rule 0 x 77 [7,0]
+ CRUSH rule 0 x 78 [1,4]
+ CRUSH rule 0 x 79 [5,0]
+ CRUSH rule 0 x 80 [0,4]
+ CRUSH rule 0 x 81 [0,5]
+ CRUSH rule 0 x 82 [7,2]
+ CRUSH rule 0 x 83 [2,6]
+ CRUSH rule 0 x 84 [7,0]
+ CRUSH rule 0 x 85 [3,7]
+ CRUSH rule 0 x 86 [0,6]
+ CRUSH rule 0 x 87 [0,6]
+ CRUSH rule 0 x 88 [1,8]
+ CRUSH rule 0 x 89 [3,1]
+ CRUSH rule 0 x 90 [6,3]
+ CRUSH rule 0 x 91 [3,6]
+ CRUSH rule 0 x 92 [1,6]
+ CRUSH rule 0 x 93 [7,4]
+ CRUSH rule 0 x 94 [0,3]
+ CRUSH rule 0 x 95 [7,3]
+ CRUSH rule 0 x 96 [3,8]
+ CRUSH rule 0 x 97 [8,4]
+ CRUSH rule 0 x 98 [2,7]
+ CRUSH rule 0 x 99 [0,8]
+ CRUSH rule 0 x 100 [1,6]
+ CRUSH rule 0 x 101 [3,8]
+ CRUSH rule 0 x 102 [4,0]
+ CRUSH rule 0 x 103 [4,7]
+ CRUSH rule 0 x 104 [7,4]
+ CRUSH rule 0 x 105 [2,3]
+ CRUSH rule 0 x 106 [1,6]
+ CRUSH rule 0 x 107 [3,1]
+ CRUSH rule 0 x 108 [7,1]
+ CRUSH rule 0 x 109 [1,3]
+ CRUSH rule 0 x 110 [3,2]
+ CRUSH rule 0 x 111 [2,4]
+ CRUSH rule 0 x 112 [2,6]
+ CRUSH rule 0 x 113 [6,1]
+ CRUSH rule 0 x 114 [7,5]
+ CRUSH rule 0 x 115 [8,2]
+ CRUSH rule 0 x 116 [1,7]
+ CRUSH rule 0 x 117 [7,5]
+ CRUSH rule 0 x 118 [0,3]
+ CRUSH rule 0 x 119 [5,7]
+ CRUSH rule 0 x 120 [0,5]
+ CRUSH rule 0 x 121 [2,8]
+ CRUSH rule 0 x 122 [8,4]
+ CRUSH rule 0 x 123 [2,4]
+ CRUSH rule 0 x 124 [3,0]
+ CRUSH rule 0 x 125 [0,7]
+ CRUSH rule 0 x 126 [4,1]
+ CRUSH rule 0 x 127 [3,6]
+ CRUSH rule 0 x 128 [3,8]
+ CRUSH rule 0 x 129 [0,4]
+ CRUSH rule 0 x 130 [3,8]
+ CRUSH rule 0 x 131 [1,5]
+ CRUSH rule 0 x 132 [1,3]
+ CRUSH rule 0 x 133 [3,6]
+ CRUSH rule 0 x 134 [1,7]
+ CRUSH rule 0 x 135 [5,7]
+ CRUSH rule 0 x 136 [2,3]
+ CRUSH rule 0 x 137 [7,3]
+ CRUSH rule 0 x 138 [8,3]
+ CRUSH rule 0 x 139 [3,0]
+ CRUSH rule 0 x 140 [1,8]
+ CRUSH rule 0 x 141 [6,0]
+ CRUSH rule 0 x 142 [3,2]
+ CRUSH rule 0 x 143 [5,7]
+ CRUSH rule 0 x 144 [8,2]
+ CRUSH rule 0 x 145 [8,4]
+ CRUSH rule 0 x 146 [2,8]
+ CRUSH rule 0 x 147 [2,6]
+ CRUSH rule 0 x 148 [3,0]
+ CRUSH rule 0 x 149 [4,6]
+ CRUSH rule 0 x 150 [1,8]
+ CRUSH rule 0 x 151 [3,8]
+ CRUSH rule 0 x 152 [8,3]
+ CRUSH rule 0 x 153 [8,3]
+ CRUSH rule 0 x 154 [3,0]
+ CRUSH rule 0 x 155 [3,6]
+ CRUSH rule 0 x 156 [4,2]
+ CRUSH rule 0 x 157 [4,1]
+ CRUSH rule 0 x 158 [2,6]
+ CRUSH rule 0 x 159 [7,0]
+ CRUSH rule 0 x 160 [2,8]
+ CRUSH rule 0 x 161 [1,3]
+ CRUSH rule 0 x 162 [0,8]
+ CRUSH rule 0 x 163 [5,8]
+ CRUSH rule 0 x 164 [7,2]
+ CRUSH rule 0 x 165 [7,2]
+ CRUSH rule 0 x 166 [2,3]
+ CRUSH rule 0 x 167 [0,6]
+ CRUSH rule 0 x 168 [4,0]
+ CRUSH rule 0 x 169 [2,7]
+ CRUSH rule 0 x 170 [1,3]
+ CRUSH rule 0 x 171 [7,3]
+ CRUSH rule 0 x 172 [0,8]
+ CRUSH rule 0 x 173 [8,5]
+ CRUSH rule 0 x 174 [1,4]
+ CRUSH rule 0 x 175 [6,0]
+ CRUSH rule 0 x 176 [4,2]
+ CRUSH rule 0 x 177 [5,1]
+ CRUSH rule 0 x 178 [3,0]
+ CRUSH rule 0 x 179 [4,1]
+ CRUSH rule 0 x 180 [3,7]
+ CRUSH rule 0 x 181 [6,0]
+ CRUSH rule 0 x 182 [8,3]
+ CRUSH rule 0 x 183 [7,5]
+ CRUSH rule 0 x 184 [5,6]
+ CRUSH rule 0 x 185 [6,0]
+ CRUSH rule 0 x 186 [2,4]
+ CRUSH rule 0 x 187 [1,6]
+ CRUSH rule 0 x 188 [1,6]
+ CRUSH rule 0 x 189 [0,6]
+ CRUSH rule 0 x 190 [4,1]
+ CRUSH rule 0 x 191 [7,0]
+ CRUSH rule 0 x 192 [5,2]
+ CRUSH rule 0 x 193 [4,0]
+ CRUSH rule 0 x 194 [1,3]
+ CRUSH rule 0 x 195 [6,5]
+ CRUSH rule 0 x 196 [6,1]
+ CRUSH rule 0 x 197 [6,5]
+ CRUSH rule 0 x 198 [2,3]
+ CRUSH rule 0 x 199 [0,5]
+ CRUSH rule 0 x 200 [0,3]
+ CRUSH rule 0 x 201 [7,1]
+ CRUSH rule 0 x 202 [6,4]
+ CRUSH rule 0 x 203 [4,6]
+ CRUSH rule 0 x 204 [2,4]
+ CRUSH rule 0 x 205 [0,7]
+ CRUSH rule 0 x 206 [0,8]
+ CRUSH rule 0 x 207 [3,2]
+ CRUSH rule 0 x 208 [7,2]
+ CRUSH rule 0 x 209 [1,8]
+ CRUSH rule 0 x 210 [1,4]
+ CRUSH rule 0 x 211 [5,2]
+ CRUSH rule 0 x 212 [7,5]
+ CRUSH rule 0 x 213 [8,4]
+ CRUSH rule 0 x 214 [4,7]
+ CRUSH rule 0 x 215 [8,0]
+ CRUSH rule 0 x 216 [5,2]
+ CRUSH rule 0 x 217 [0,7]
+ CRUSH rule 0 x 218 [0,7]
+ CRUSH rule 0 x 219 [4,6]
+ CRUSH rule 0 x 220 [5,8]
+ CRUSH rule 0 x 221 [3,7]
+ CRUSH rule 0 x 222 [6,4]
+ CRUSH rule 0 x 223 [1,3]
+ CRUSH rule 0 x 224 [1,3]
+ CRUSH rule 0 x 225 [8,0]
+ CRUSH rule 0 x 226 [7,0]
+ CRUSH rule 0 x 227 [3,0]
+ CRUSH rule 0 x 228 [5,6]
+ CRUSH rule 0 x 229 [3,7]
+ CRUSH rule 0 x 230 [4,6]
+ CRUSH rule 0 x 231 [4,7]
+ CRUSH rule 0 x 232 [2,8]
+ CRUSH rule 0 x 233 [3,6]
+ CRUSH rule 0 x 234 [0,4]
+ CRUSH rule 0 x 235 [3,6]
+ CRUSH rule 0 x 236 [5,0]
+ CRUSH rule 0 x 237 [4,8]
+ CRUSH rule 0 x 238 [4,2]
+ CRUSH rule 0 x 239 [8,3]
+ CRUSH rule 0 x 240 [5,8]
+ CRUSH rule 0 x 241 [3,1]
+ CRUSH rule 0 x 242 [3,2]
+ CRUSH rule 0 x 243 [4,8]
+ CRUSH rule 0 x 244 [4,6]
+ CRUSH rule 0 x 245 [7,2]
+ CRUSH rule 0 x 246 [1,3]
+ CRUSH rule 0 x 247 [6,1]
+ CRUSH rule 0 x 248 [8,0]
+ CRUSH rule 0 x 249 [2,5]
+ CRUSH rule 0 x 250 [2,3]
+ CRUSH rule 0 x 251 [2,3]
+ CRUSH rule 0 x 252 [3,7]
+ CRUSH rule 0 x 253 [3,0]
+ CRUSH rule 0 x 254 [3,2]
+ CRUSH rule 0 x 255 [1,7]
+ CRUSH rule 0 x 256 [5,6]
+ CRUSH rule 0 x 257 [2,6]
+ CRUSH rule 0 x 258 [5,2]
+ CRUSH rule 0 x 259 [4,6]
+ CRUSH rule 0 x 260 [3,8]
+ CRUSH rule 0 x 261 [8,5]
+ CRUSH rule 0 x 262 [5,6]
+ CRUSH rule 0 x 263 [6,1]
+ CRUSH rule 0 x 264 [3,6]
+ CRUSH rule 0 x 265 [8,4]
+ CRUSH rule 0 x 266 [8,0]
+ CRUSH rule 0 x 267 [2,4]
+ CRUSH rule 0 x 268 [0,6]
+ CRUSH rule 0 x 269 [0,6]
+ CRUSH rule 0 x 270 [5,1]
+ CRUSH rule 0 x 271 [7,4]
+ CRUSH rule 0 x 272 [2,6]
+ CRUSH rule 0 x 273 [3,2]
+ CRUSH rule 0 x 274 [6,3]
+ CRUSH rule 0 x 275 [4,8]
+ CRUSH rule 0 x 276 [7,0]
+ CRUSH rule 0 x 277 [6,4]
+ CRUSH rule 0 x 278 [6,2]
+ CRUSH rule 0 x 279 [8,5]
+ CRUSH rule 0 x 280 [0,7]
+ CRUSH rule 0 x 281 [8,1]
+ CRUSH rule 0 x 282 [3,2]
+ CRUSH rule 0 x 283 [8,1]
+ CRUSH rule 0 x 284 [6,3]
+ CRUSH rule 0 x 285 [5,7]
+ CRUSH rule 0 x 286 [2,8]
+ CRUSH rule 0 x 287 [0,3]
+ CRUSH rule 0 x 288 [8,1]
+ CRUSH rule 0 x 289 [4,6]
+ CRUSH rule 0 x 290 [1,5]
+ CRUSH rule 0 x 291 [0,5]
+ CRUSH rule 0 x 292 [8,1]
+ CRUSH rule 0 x 293 [6,2]
+ CRUSH rule 0 x 294 [7,5]
+ CRUSH rule 0 x 295 [4,6]
+ CRUSH rule 0 x 296 [3,1]
+ CRUSH rule 0 x 297 [6,0]
+ CRUSH rule 0 x 298 [1,5]
+ CRUSH rule 0 x 299 [2,8]
+ CRUSH rule 0 x 300 [8,4]
+ CRUSH rule 0 x 301 [0,7]
+ CRUSH rule 0 x 302 [3,1]
+ CRUSH rule 0 x 303 [7,4]
+ CRUSH rule 0 x 304 [2,6]
+ CRUSH rule 0 x 305 [5,6]
+ CRUSH rule 0 x 306 [0,8]
+ CRUSH rule 0 x 307 [0,7]
+ CRUSH rule 0 x 308 [0,8]
+ CRUSH rule 0 x 309 [7,4]
+ CRUSH rule 0 x 310 [4,0]
+ CRUSH rule 0 x 311 [3,8]
+ CRUSH rule 0 x 312 [2,6]
+ CRUSH rule 0 x 313 [5,1]
+ CRUSH rule 0 x 314 [4,0]
+ CRUSH rule 0 x 315 [2,3]
+ CRUSH rule 0 x 316 [6,4]
+ CRUSH rule 0 x 317 [2,7]
+ CRUSH rule 0 x 318 [8,1]
+ CRUSH rule 0 x 319 [5,2]
+ CRUSH rule 0 x 320 [3,8]
+ CRUSH rule 0 x 321 [1,5]
+ CRUSH rule 0 x 322 [2,6]
+ CRUSH rule 0 x 323 [4,7]
+ CRUSH rule 0 x 324 [7,2]
+ CRUSH rule 0 x 325 [4,6]
+ CRUSH rule 0 x 326 [3,1]
+ CRUSH rule 0 x 327 [0,8]
+ CRUSH rule 0 x 328 [7,3]
+ CRUSH rule 0 x 329 [5,7]
+ CRUSH rule 0 x 330 [3,7]
+ CRUSH rule 0 x 331 [2,7]
+ CRUSH rule 0 x 332 [2,3]
+ CRUSH rule 0 x 333 [6,4]
+ CRUSH rule 0 x 334 [8,3]
+ CRUSH rule 0 x 335 [7,2]
+ CRUSH rule 0 x 336 [4,6]
+ CRUSH rule 0 x 337 [7,0]
+ CRUSH rule 0 x 338 [5,8]
+ CRUSH rule 0 x 339 [7,5]
+ CRUSH rule 0 x 340 [2,6]
+ CRUSH rule 0 x 341 [5,1]
+ CRUSH rule 0 x 342 [0,8]
+ CRUSH rule 0 x 343 [6,5]
+ CRUSH rule 0 x 344 [6,1]
+ CRUSH rule 0 x 345 [4,7]
+ CRUSH rule 0 x 346 [8,0]
+ CRUSH rule 0 x 347 [3,0]
+ CRUSH rule 0 x 348 [8,0]
+ CRUSH rule 0 x 349 [1,7]
+ CRUSH rule 0 x 350 [8,5]
+ CRUSH rule 0 x 351 [3,8]
+ CRUSH rule 0 x 352 [1,8]
+ CRUSH rule 0 x 353 [6,5]
+ CRUSH rule 0 x 354 [0,5]
+ CRUSH rule 0 x 355 [3,8]
+ CRUSH rule 0 x 356 [3,1]
+ CRUSH rule 0 x 357 [6,1]
+ CRUSH rule 0 x 358 [2,8]
+ CRUSH rule 0 x 359 [6,0]
+ CRUSH rule 0 x 360 [5,0]
+ CRUSH rule 0 x 361 [8,5]
+ CRUSH rule 0 x 362 [4,0]
+ CRUSH rule 0 x 363 [4,2]
+ CRUSH rule 0 x 364 [2,5]
+ CRUSH rule 0 x 365 [6,4]
+ CRUSH rule 0 x 366 [7,0]
+ CRUSH rule 0 x 367 [4,2]
+ CRUSH rule 0 x 368 [7,3]
+ CRUSH rule 0 x 369 [3,7]
+ CRUSH rule 0 x 370 [8,2]
+ CRUSH rule 0 x 371 [1,5]
+ CRUSH rule 0 x 372 [3,1]
+ CRUSH rule 0 x 373 [0,6]
+ CRUSH rule 0 x 374 [3,8]
+ CRUSH rule 0 x 375 [6,5]
+ CRUSH rule 0 x 376 [7,1]
+ CRUSH rule 0 x 377 [1,4]
+ CRUSH rule 0 x 378 [0,6]
+ CRUSH rule 0 x 379 [8,3]
+ CRUSH rule 0 x 380 [2,3]
+ CRUSH rule 0 x 381 [0,3]
+ CRUSH rule 0 x 382 [1,3]
+ CRUSH rule 0 x 383 [4,7]
+ CRUSH rule 0 x 384 [7,0]
+ CRUSH rule 0 x 385 [7,4]
+ CRUSH rule 0 x 386 [0,4]
+ CRUSH rule 0 x 387 [1,5]
+ CRUSH rule 0 x 388 [5,2]
+ CRUSH rule 0 x 389 [1,4]
+ CRUSH rule 0 x 390 [5,8]
+ CRUSH rule 0 x 391 [5,6]
+ CRUSH rule 0 x 392 [1,7]
+ CRUSH rule 0 x 393 [4,0]
+ CRUSH rule 0 x 394 [4,8]
+ CRUSH rule 0 x 395 [4,0]
+ CRUSH rule 0 x 396 [4,2]
+ CRUSH rule 0 x 397 [2,4]
+ CRUSH rule 0 x 398 [2,3]
+ CRUSH rule 0 x 399 [8,5]
+ CRUSH rule 0 x 400 [8,1]
+ CRUSH rule 0 x 401 [0,3]
+ CRUSH rule 0 x 402 [7,4]
+ CRUSH rule 0 x 403 [0,4]
+ CRUSH rule 0 x 404 [4,0]
+ CRUSH rule 0 x 405 [6,4]
+ CRUSH rule 0 x 406 [2,6]
+ CRUSH rule 0 x 407 [2,7]
+ CRUSH rule 0 x 408 [4,1]
+ CRUSH rule 0 x 409 [7,5]
+ CRUSH rule 0 x 410 [8,3]
+ CRUSH rule 0 x 411 [2,7]
+ CRUSH rule 0 x 412 [0,3]
+ CRUSH rule 0 x 413 [5,2]
+ CRUSH rule 0 x 414 [4,0]
+ CRUSH rule 0 x 415 [0,6]
+ CRUSH rule 0 x 416 [2,8]
+ CRUSH rule 0 x 417 [8,0]
+ CRUSH rule 0 x 418 [7,1]
+ CRUSH rule 0 x 419 [8,5]
+ CRUSH rule 0 x 420 [1,4]
+ CRUSH rule 0 x 421 [8,3]
+ CRUSH rule 0 x 422 [6,3]
+ CRUSH rule 0 x 423 [0,3]
+ CRUSH rule 0 x 424 [8,5]
+ CRUSH rule 0 x 425 [1,4]
+ CRUSH rule 0 x 426 [6,2]
+ CRUSH rule 0 x 427 [0,8]
+ CRUSH rule 0 x 428 [5,8]
+ CRUSH rule 0 x 429 [4,8]
+ CRUSH rule 0 x 430 [3,7]
+ CRUSH rule 0 x 431 [5,0]
+ CRUSH rule 0 x 432 [7,0]
+ CRUSH rule 0 x 433 [6,4]
+ CRUSH rule 0 x 434 [5,0]
+ CRUSH rule 0 x 435 [0,4]
+ CRUSH rule 0 x 436 [4,0]
+ CRUSH rule 0 x 437 [7,3]
+ CRUSH rule 0 x 438 [0,5]
+ CRUSH rule 0 x 439 [1,3]
+ CRUSH rule 0 x 440 [2,6]
+ CRUSH rule 0 x 441 [5,8]
+ CRUSH rule 0 x 442 [2,3]
+ CRUSH rule 0 x 443 [6,0]
+ CRUSH rule 0 x 444 [7,1]
+ CRUSH rule 0 x 445 [6,4]
+ CRUSH rule 0 x 446 [4,0]
+ CRUSH rule 0 x 447 [2,4]
+ CRUSH rule 0 x 448 [7,0]
+ CRUSH rule 0 x 449 [7,4]
+ CRUSH rule 0 x 450 [4,0]
+ CRUSH rule 0 x 451 [6,5]
+ CRUSH rule 0 x 452 [8,4]
+ CRUSH rule 0 x 453 [6,5]
+ CRUSH rule 0 x 454 [6,5]
+ CRUSH rule 0 x 455 [2,8]
+ CRUSH rule 0 x 456 [6,2]
+ CRUSH rule 0 x 457 [7,1]
+ CRUSH rule 0 x 458 [2,8]
+ CRUSH rule 0 x 459 [2,6]
+ CRUSH rule 0 x 460 [6,5]
+ CRUSH rule 0 x 461 [6,4]
+ CRUSH rule 0 x 462 [8,0]
+ CRUSH rule 0 x 463 [6,0]
+ CRUSH rule 0 x 464 [7,4]
+ CRUSH rule 0 x 465 [7,1]
+ CRUSH rule 0 x 466 [5,6]
+ CRUSH rule 0 x 467 [6,5]
+ CRUSH rule 0 x 468 [7,2]
+ CRUSH rule 0 x 469 [7,2]
+ CRUSH rule 0 x 470 [3,0]
+ CRUSH rule 0 x 471 [0,6]
+ CRUSH rule 0 x 472 [5,0]
+ CRUSH rule 0 x 473 [1,4]
+ CRUSH rule 0 x 474 [6,1]
+ CRUSH rule 0 x 475 [6,2]
+ CRUSH rule 0 x 476 [4,7]
+ CRUSH rule 0 x 477 [5,6]
+ CRUSH rule 0 x 478 [6,1]
+ CRUSH rule 0 x 479 [0,3]
+ CRUSH rule 0 x 480 [1,6]
+ CRUSH rule 0 x 481 [2,5]
+ CRUSH rule 0 x 482 [4,8]
+ CRUSH rule 0 x 483 [0,6]
+ CRUSH rule 0 x 484 [1,8]
+ CRUSH rule 0 x 485 [4,8]
+ CRUSH rule 0 x 486 [4,0]
+ CRUSH rule 0 x 487 [5,0]
+ CRUSH rule 0 x 488 [5,7]
+ CRUSH rule 0 x 489 [2,8]
+ CRUSH rule 0 x 490 [6,4]
+ CRUSH rule 0 x 491 [1,6]
+ CRUSH rule 0 x 492 [6,5]
+ CRUSH rule 0 x 493 [0,8]
+ CRUSH rule 0 x 494 [1,6]
+ CRUSH rule 0 x 495 [3,0]
+ CRUSH rule 0 x 496 [7,5]
+ CRUSH rule 0 x 497 [5,7]
+ CRUSH rule 0 x 498 [0,4]
+ CRUSH rule 0 x 499 [8,5]
+ CRUSH rule 0 x 500 [3,6]
+ CRUSH rule 0 x 501 [0,7]
+ CRUSH rule 0 x 502 [7,1]
+ CRUSH rule 0 x 503 [2,3]
+ CRUSH rule 0 x 504 [5,8]
+ CRUSH rule 0 x 505 [0,7]
+ CRUSH rule 0 x 506 [5,1]
+ CRUSH rule 0 x 507 [6,0]
+ CRUSH rule 0 x 508 [0,4]
+ CRUSH rule 0 x 509 [7,4]
+ CRUSH rule 0 x 510 [6,2]
+ CRUSH rule 0 x 511 [5,6]
+ CRUSH rule 0 x 512 [7,2]
+ CRUSH rule 0 x 513 [7,2]
+ CRUSH rule 0 x 514 [4,7]
+ CRUSH rule 0 x 515 [8,3]
+ CRUSH rule 0 x 516 [4,1]
+ CRUSH rule 0 x 517 [7,0]
+ CRUSH rule 0 x 518 [4,6]
+ CRUSH rule 0 x 519 [7,3]
+ CRUSH rule 0 x 520 [2,8]
+ CRUSH rule 0 x 521 [8,0]
+ CRUSH rule 0 x 522 [6,0]
+ CRUSH rule 0 x 523 [4,1]
+ CRUSH rule 0 x 524 [0,4]
+ CRUSH rule 0 x 525 [0,3]
+ CRUSH rule 0 x 526 [1,3]
+ CRUSH rule 0 x 527 [0,4]
+ CRUSH rule 0 x 528 [5,2]
+ CRUSH rule 0 x 529 [5,6]
+ CRUSH rule 0 x 530 [6,4]
+ CRUSH rule 0 x 531 [6,0]
+ CRUSH rule 0 x 532 [6,5]
+ CRUSH rule 0 x 533 [5,8]
+ CRUSH rule 0 x 534 [7,4]
+ CRUSH rule 0 x 535 [8,0]
+ CRUSH rule 0 x 536 [6,2]
+ CRUSH rule 0 x 537 [3,8]
+ CRUSH rule 0 x 538 [6,4]
+ CRUSH rule 0 x 539 [8,4]
+ CRUSH rule 0 x 540 [0,7]
+ CRUSH rule 0 x 541 [2,5]
+ CRUSH rule 0 x 542 [3,0]
+ CRUSH rule 0 x 543 [6,2]
+ CRUSH rule 0 x 544 [3,7]
+ CRUSH rule 0 x 545 [5,7]
+ CRUSH rule 0 x 546 [6,2]
+ CRUSH rule 0 x 547 [8,1]
+ CRUSH rule 0 x 548 [5,1]
+ CRUSH rule 0 x 549 [5,7]
+ CRUSH rule 0 x 550 [0,5]
+ CRUSH rule 0 x 551 [7,4]
+ CRUSH rule 0 x 552 [5,7]
+ CRUSH rule 0 x 553 [4,2]
+ CRUSH rule 0 x 554 [0,6]
+ CRUSH rule 0 x 555 [5,0]
+ CRUSH rule 0 x 556 [3,6]
+ CRUSH rule 0 x 557 [7,4]
+ CRUSH rule 0 x 558 [3,2]
+ CRUSH rule 0 x 559 [4,1]
+ CRUSH rule 0 x 560 [8,4]
+ CRUSH rule 0 x 561 [6,4]
+ CRUSH rule 0 x 562 [3,0]
+ CRUSH rule 0 x 563 [2,7]
+ CRUSH rule 0 x 564 [5,1]
+ CRUSH rule 0 x 565 [3,8]
+ CRUSH rule 0 x 566 [4,6]
+ CRUSH rule 0 x 567 [3,7]
+ CRUSH rule 0 x 568 [7,4]
+ CRUSH rule 0 x 569 [3,1]
+ CRUSH rule 0 x 570 [1,4]
+ CRUSH rule 0 x 571 [3,6]
+ CRUSH rule 0 x 572 [3,0]
+ CRUSH rule 0 x 573 [3,1]
+ CRUSH rule 0 x 574 [2,3]
+ CRUSH rule 0 x 575 [8,1]
+ CRUSH rule 0 x 576 [4,7]
+ CRUSH rule 0 x 577 [8,2]
+ CRUSH rule 0 x 578 [6,0]
+ CRUSH rule 0 x 579 [3,2]
+ CRUSH rule 0 x 580 [3,0]
+ CRUSH rule 0 x 581 [7,1]
+ CRUSH rule 0 x 582 [2,8]
+ CRUSH rule 0 x 583 [6,2]
+ CRUSH rule 0 x 584 [8,0]
+ CRUSH rule 0 x 585 [7,0]
+ CRUSH rule 0 x 586 [0,8]
+ CRUSH rule 0 x 587 [2,5]
+ CRUSH rule 0 x 588 [3,8]
+ CRUSH rule 0 x 589 [7,1]
+ CRUSH rule 0 x 590 [6,2]
+ CRUSH rule 0 x 591 [5,2]
+ CRUSH rule 0 x 592 [2,4]
+ CRUSH rule 0 x 593 [0,8]
+ CRUSH rule 0 x 594 [0,6]
+ CRUSH rule 0 x 595 [7,1]
+ CRUSH rule 0 x 596 [4,0]
+ CRUSH rule 0 x 597 [3,1]
+ CRUSH rule 0 x 598 [3,0]
+ CRUSH rule 0 x 599 [5,1]
+ CRUSH rule 0 x 600 [7,0]
+ CRUSH rule 0 x 601 [0,6]
+ CRUSH rule 0 x 602 [3,8]
+ CRUSH rule 0 x 603 [5,0]
+ CRUSH rule 0 x 604 [7,4]
+ CRUSH rule 0 x 605 [3,2]
+ CRUSH rule 0 x 606 [2,7]
+ CRUSH rule 0 x 607 [0,5]
+ CRUSH rule 0 x 608 [5,1]
+ CRUSH rule 0 x 609 [5,0]
+ CRUSH rule 0 x 610 [3,8]
+ CRUSH rule 0 x 611 [1,8]
+ CRUSH rule 0 x 612 [2,8]
+ CRUSH rule 0 x 613 [7,1]
+ CRUSH rule 0 x 614 [7,2]
+ CRUSH rule 0 x 615 [6,2]
+ CRUSH rule 0 x 616 [0,7]
+ CRUSH rule 0 x 617 [6,2]
+ CRUSH rule 0 x 618 [7,3]
+ CRUSH rule 0 x 619 [5,0]
+ CRUSH rule 0 x 620 [4,1]
+ CRUSH rule 0 x 621 [5,6]
+ CRUSH rule 0 x 622 [0,3]
+ CRUSH rule 0 x 623 [0,8]
+ CRUSH rule 0 x 624 [3,2]
+ CRUSH rule 0 x 625 [2,5]
+ CRUSH rule 0 x 626 [7,2]
+ CRUSH rule 0 x 627 [2,6]
+ CRUSH rule 0 x 628 [8,1]
+ CRUSH rule 0 x 629 [2,6]
+ CRUSH rule 0 x 630 [2,6]
+ CRUSH rule 0 x 631 [0,6]
+ CRUSH rule 0 x 632 [7,0]
+ CRUSH rule 0 x 633 [8,4]
+ CRUSH rule 0 x 634 [0,5]
+ CRUSH rule 0 x 635 [5,6]
+ CRUSH rule 0 x 636 [1,3]
+ CRUSH rule 0 x 637 [4,0]
+ CRUSH rule 0 x 638 [6,2]
+ CRUSH rule 0 x 639 [4,0]
+ CRUSH rule 0 x 640 [3,2]
+ CRUSH rule 0 x 641 [7,2]
+ CRUSH rule 0 x 642 [2,8]
+ CRUSH rule 0 x 643 [3,0]
+ CRUSH rule 0 x 644 [8,0]
+ CRUSH rule 0 x 645 [5,7]
+ CRUSH rule 0 x 646 [8,0]
+ CRUSH rule 0 x 647 [7,0]
+ CRUSH rule 0 x 648 [0,8]
+ CRUSH rule 0 x 649 [4,7]
+ CRUSH rule 0 x 650 [7,5]
+ CRUSH rule 0 x 651 [3,6]
+ CRUSH rule 0 x 652 [3,6]
+ CRUSH rule 0 x 653 [8,3]
+ CRUSH rule 0 x 654 [7,4]
+ CRUSH rule 0 x 655 [0,5]
+ CRUSH rule 0 x 656 [4,7]
+ CRUSH rule 0 x 657 [6,0]
+ CRUSH rule 0 x 658 [5,8]
+ CRUSH rule 0 x 659 [4,7]
+ CRUSH rule 0 x 660 [7,3]
+ CRUSH rule 0 x 661 [1,7]
+ CRUSH rule 0 x 662 [4,2]
+ CRUSH rule 0 x 663 [1,3]
+ CRUSH rule 0 x 664 [1,3]
+ CRUSH rule 0 x 665 [5,6]
+ CRUSH rule 0 x 666 [2,7]
+ CRUSH rule 0 x 667 [1,3]
+ CRUSH rule 0 x 668 [3,7]
+ CRUSH rule 0 x 669 [6,3]
+ CRUSH rule 0 x 670 [4,1]
+ CRUSH rule 0 x 671 [0,8]
+ CRUSH rule 0 x 672 [4,2]
+ CRUSH rule 0 x 673 [5,0]
+ CRUSH rule 0 x 674 [3,0]
+ CRUSH rule 0 x 675 [0,8]
+ CRUSH rule 0 x 676 [0,3]
+ CRUSH rule 0 x 677 [4,1]
+ CRUSH rule 0 x 678 [2,3]
+ CRUSH rule 0 x 679 [6,0]
+ CRUSH rule 0 x 680 [0,3]
+ CRUSH rule 0 x 681 [4,6]
+ CRUSH rule 0 x 682 [0,4]
+ CRUSH rule 0 x 683 [0,3]
+ CRUSH rule 0 x 684 [7,2]
+ CRUSH rule 0 x 685 [7,2]
+ CRUSH rule 0 x 686 [1,5]
+ CRUSH rule 0 x 687 [3,7]
+ CRUSH rule 0 x 688 [5,6]
+ CRUSH rule 0 x 689 [6,5]
+ CRUSH rule 0 x 690 [8,0]
+ CRUSH rule 0 x 691 [3,1]
+ CRUSH rule 0 x 692 [7,1]
+ CRUSH rule 0 x 693 [6,5]
+ CRUSH rule 0 x 694 [6,4]
+ CRUSH rule 0 x 695 [0,6]
+ CRUSH rule 0 x 696 [1,5]
+ CRUSH rule 0 x 697 [6,0]
+ CRUSH rule 0 x 698 [6,0]
+ CRUSH rule 0 x 699 [1,8]
+ CRUSH rule 0 x 700 [0,4]
+ CRUSH rule 0 x 701 [4,0]
+ CRUSH rule 0 x 702 [3,0]
+ CRUSH rule 0 x 703 [8,4]
+ CRUSH rule 0 x 704 [0,4]
+ CRUSH rule 0 x 705 [8,0]
+ CRUSH rule 0 x 706 [1,5]
+ CRUSH rule 0 x 707 [7,3]
+ CRUSH rule 0 x 708 [3,7]
+ CRUSH rule 0 x 709 [6,5]
+ CRUSH rule 0 x 710 [8,5]
+ CRUSH rule 0 x 711 [2,4]
+ CRUSH rule 0 x 712 [2,3]
+ CRUSH rule 0 x 713 [6,3]
+ CRUSH rule 0 x 714 [3,0]
+ CRUSH rule 0 x 715 [1,3]
+ CRUSH rule 0 x 716 [3,6]
+ CRUSH rule 0 x 717 [8,0]
+ CRUSH rule 0 x 718 [3,7]
+ CRUSH rule 0 x 719 [2,6]
+ CRUSH rule 0 x 720 [6,0]
+ CRUSH rule 0 x 721 [5,7]
+ CRUSH rule 0 x 722 [5,7]
+ CRUSH rule 0 x 723 [5,2]
+ CRUSH rule 0 x 724 [0,7]
+ CRUSH rule 0 x 725 [0,4]
+ CRUSH rule 0 x 726 [3,7]
+ CRUSH rule 0 x 727 [4,7]
+ CRUSH rule 0 x 728 [2,6]
+ CRUSH rule 0 x 729 [5,6]
+ CRUSH rule 0 x 730 [3,8]
+ CRUSH rule 0 x 731 [4,1]
+ CRUSH rule 0 x 732 [1,4]
+ CRUSH rule 0 x 733 [5,6]
+ CRUSH rule 0 x 734 [6,5]
+ CRUSH rule 0 x 735 [4,6]
+ CRUSH rule 0 x 736 [3,8]
+ CRUSH rule 0 x 737 [1,8]
+ CRUSH rule 0 x 738 [5,1]
+ CRUSH rule 0 x 739 [0,7]
+ CRUSH rule 0 x 740 [0,7]
+ CRUSH rule 0 x 741 [7,2]
+ CRUSH rule 0 x 742 [8,2]
+ CRUSH rule 0 x 743 [7,0]
+ CRUSH rule 0 x 744 [4,6]
+ CRUSH rule 0 x 745 [3,0]
+ CRUSH rule 0 x 746 [4,1]
+ CRUSH rule 0 x 747 [6,2]
+ CRUSH rule 0 x 748 [2,6]
+ CRUSH rule 0 x 749 [4,7]
+ CRUSH rule 0 x 750 [1,7]
+ CRUSH rule 0 x 751 [2,7]
+ CRUSH rule 0 x 752 [8,0]
+ CRUSH rule 0 x 753 [7,4]
+ CRUSH rule 0 x 754 [8,5]
+ CRUSH rule 0 x 755 [1,6]
+ CRUSH rule 0 x 756 [5,8]
+ CRUSH rule 0 x 757 [8,0]
+ CRUSH rule 0 x 758 [6,1]
+ CRUSH rule 0 x 759 [8,5]
+ CRUSH rule 0 x 760 [1,5]
+ CRUSH rule 0 x 761 [4,2]
+ CRUSH rule 0 x 762 [2,8]
+ CRUSH rule 0 x 763 [8,3]
+ CRUSH rule 0 x 764 [1,7]
+ CRUSH rule 0 x 765 [6,4]
+ CRUSH rule 0 x 766 [8,3]
+ CRUSH rule 0 x 767 [1,8]
+ CRUSH rule 0 x 768 [8,5]
+ CRUSH rule 0 x 769 [6,2]
+ CRUSH rule 0 x 770 [6,1]
+ CRUSH rule 0 x 771 [7,2]
+ CRUSH rule 0 x 772 [8,4]
+ CRUSH rule 0 x 773 [3,2]
+ CRUSH rule 0 x 774 [4,7]
+ CRUSH rule 0 x 775 [6,5]
+ CRUSH rule 0 x 776 [7,0]
+ CRUSH rule 0 x 777 [3,0]
+ CRUSH rule 0 x 778 [1,6]
+ CRUSH rule 0 x 779 [2,6]
+ CRUSH rule 0 x 780 [0,5]
+ CRUSH rule 0 x 781 [6,5]
+ CRUSH rule 0 x 782 [5,2]
+ CRUSH rule 0 x 783 [7,0]
+ CRUSH rule 0 x 784 [0,5]
+ CRUSH rule 0 x 785 [6,1]
+ CRUSH rule 0 x 786 [7,3]
+ CRUSH rule 0 x 787 [1,8]
+ CRUSH rule 0 x 788 [6,0]
+ CRUSH rule 0 x 789 [0,5]
+ CRUSH rule 0 x 790 [8,3]
+ CRUSH rule 0 x 791 [3,6]
+ CRUSH rule 0 x 792 [5,6]
+ CRUSH rule 0 x 793 [6,2]
+ CRUSH rule 0 x 794 [2,8]
+ CRUSH rule 0 x 795 [0,4]
+ CRUSH rule 0 x 796 [3,7]
+ CRUSH rule 0 x 797 [2,3]
+ CRUSH rule 0 x 798 [6,0]
+ CRUSH rule 0 x 799 [5,2]
+ CRUSH rule 0 x 800 [5,2]
+ CRUSH rule 0 x 801 [3,7]
+ CRUSH rule 0 x 802 [1,6]
+ CRUSH rule 0 x 803 [0,4]
+ CRUSH rule 0 x 804 [6,0]
+ CRUSH rule 0 x 805 [3,8]
+ CRUSH rule 0 x 806 [1,5]
+ CRUSH rule 0 x 807 [5,7]
+ CRUSH rule 0 x 808 [4,7]
+ CRUSH rule 0 x 809 [1,3]
+ CRUSH rule 0 x 810 [5,7]
+ CRUSH rule 0 x 811 [8,4]
+ CRUSH rule 0 x 812 [8,3]
+ CRUSH rule 0 x 813 [6,4]
+ CRUSH rule 0 x 814 [3,8]
+ CRUSH rule 0 x 815 [3,1]
+ CRUSH rule 0 x 816 [2,6]
+ CRUSH rule 0 x 817 [4,6]
+ CRUSH rule 0 x 818 [3,1]
+ CRUSH rule 0 x 819 [5,0]
+ CRUSH rule 0 x 820 [3,7]
+ CRUSH rule 0 x 821 [4,8]
+ CRUSH rule 0 x 822 [2,3]
+ CRUSH rule 0 x 823 [4,7]
+ CRUSH rule 0 x 824 [3,7]
+ CRUSH rule 0 x 825 [2,8]
+ CRUSH rule 0 x 826 [7,1]
+ CRUSH rule 0 x 827 [0,6]
+ CRUSH rule 0 x 828 [2,5]
+ CRUSH rule 0 x 829 [5,6]
+ CRUSH rule 0 x 830 [2,4]
+ CRUSH rule 0 x 831 [1,6]
+ CRUSH rule 0 x 832 [4,8]
+ CRUSH rule 0 x 833 [2,6]
+ CRUSH rule 0 x 834 [3,0]
+ CRUSH rule 0 x 835 [8,5]
+ CRUSH rule 0 x 836 [3,8]
+ CRUSH rule 0 x 837 [6,4]
+ CRUSH rule 0 x 838 [6,0]
+ CRUSH rule 0 x 839 [5,2]
+ CRUSH rule 0 x 840 [7,3]
+ CRUSH rule 0 x 841 [4,8]
+ CRUSH rule 0 x 842 [2,5]
+ CRUSH rule 0 x 843 [6,4]
+ CRUSH rule 0 x 844 [4,8]
+ CRUSH rule 0 x 845 [3,6]
+ CRUSH rule 0 x 846 [3,0]
+ CRUSH rule 0 x 847 [0,8]
+ CRUSH rule 0 x 848 [2,6]
+ CRUSH rule 0 x 849 [4,8]
+ CRUSH rule 0 x 850 [1,5]
+ CRUSH rule 0 x 851 [6,5]
+ CRUSH rule 0 x 852 [7,4]
+ CRUSH rule 0 x 853 [6,2]
+ CRUSH rule 0 x 854 [7,0]
+ CRUSH rule 0 x 855 [5,7]
+ CRUSH rule 0 x 856 [6,3]
+ CRUSH rule 0 x 857 [8,4]
+ CRUSH rule 0 x 858 [6,5]
+ CRUSH rule 0 x 859 [6,0]
+ CRUSH rule 0 x 860 [4,2]
+ CRUSH rule 0 x 861 [8,4]
+ CRUSH rule 0 x 862 [6,0]
+ CRUSH rule 0 x 863 [8,1]
+ CRUSH rule 0 x 864 [5,6]
+ CRUSH rule 0 x 865 [8,0]
+ CRUSH rule 0 x 866 [3,6]
+ CRUSH rule 0 x 867 [6,3]
+ CRUSH rule 0 x 868 [6,4]
+ CRUSH rule 0 x 869 [8,4]
+ CRUSH rule 0 x 870 [0,5]
+ CRUSH rule 0 x 871 [3,1]
+ CRUSH rule 0 x 872 [5,0]
+ CRUSH rule 0 x 873 [4,6]
+ CRUSH rule 0 x 874 [2,6]
+ CRUSH rule 0 x 875 [2,6]
+ CRUSH rule 0 x 876 [5,8]
+ CRUSH rule 0 x 877 [6,4]
+ CRUSH rule 0 x 878 [5,0]
+ CRUSH rule 0 x 879 [7,3]
+ CRUSH rule 0 x 880 [3,1]
+ CRUSH rule 0 x 881 [5,8]
+ CRUSH rule 0 x 882 [4,1]
+ CRUSH rule 0 x 883 [2,5]
+ CRUSH rule 0 x 884 [6,0]
+ CRUSH rule 0 x 885 [5,1]
+ CRUSH rule 0 x 886 [3,7]
+ CRUSH rule 0 x 887 [7,5]
+ CRUSH rule 0 x 888 [6,1]
+ CRUSH rule 0 x 889 [2,8]
+ CRUSH rule 0 x 890 [7,1]
+ CRUSH rule 0 x 891 [1,7]
+ CRUSH rule 0 x 892 [6,0]
+ CRUSH rule 0 x 893 [2,3]
+ CRUSH rule 0 x 894 [7,4]
+ CRUSH rule 0 x 895 [5,1]
+ CRUSH rule 0 x 896 [1,6]
+ CRUSH rule 0 x 897 [4,0]
+ CRUSH rule 0 x 898 [0,3]
+ CRUSH rule 0 x 899 [1,6]
+ CRUSH rule 0 x 900 [4,0]
+ CRUSH rule 0 x 901 [5,2]
+ CRUSH rule 0 x 902 [8,4]
+ CRUSH rule 0 x 903 [5,6]
+ CRUSH rule 0 x 904 [5,7]
+ CRUSH rule 0 x 905 [6,0]
+ CRUSH rule 0 x 906 [1,7]
+ CRUSH rule 0 x 907 [7,2]
+ CRUSH rule 0 x 908 [5,6]
+ CRUSH rule 0 x 909 [2,3]
+ CRUSH rule 0 x 910 [6,5]
+ CRUSH rule 0 x 911 [5,7]
+ CRUSH rule 0 x 912 [0,8]
+ CRUSH rule 0 x 913 [7,1]
+ CRUSH rule 0 x 914 [6,4]
+ CRUSH rule 0 x 915 [8,2]
+ CRUSH rule 0 x 916 [3,0]
+ CRUSH rule 0 x 917 [1,3]
+ CRUSH rule 0 x 918 [8,0]
+ CRUSH rule 0 x 919 [6,2]
+ CRUSH rule 0 x 920 [7,4]
+ CRUSH rule 0 x 921 [1,4]
+ CRUSH rule 0 x 922 [6,3]
+ CRUSH rule 0 x 923 [5,6]
+ CRUSH rule 0 x 924 [3,1]
+ CRUSH rule 0 x 925 [5,6]
+ CRUSH rule 0 x 926 [3,0]
+ CRUSH rule 0 x 927 [1,6]
+ CRUSH rule 0 x 928 [8,1]
+ CRUSH rule 0 x 929 [4,2]
+ CRUSH rule 0 x 930 [2,5]
+ CRUSH rule 0 x 931 [5,2]
+ CRUSH rule 0 x 932 [4,2]
+ CRUSH rule 0 x 933 [8,4]
+ CRUSH rule 0 x 934 [5,8]
+ CRUSH rule 0 x 935 [6,3]
+ CRUSH rule 0 x 936 [0,7]
+ CRUSH rule 0 x 937 [5,8]
+ CRUSH rule 0 x 938 [6,4]
+ CRUSH rule 0 x 939 [2,8]
+ CRUSH rule 0 x 940 [8,5]
+ CRUSH rule 0 x 941 [5,0]
+ CRUSH rule 0 x 942 [1,6]
+ CRUSH rule 0 x 943 [8,2]
+ CRUSH rule 0 x 944 [4,8]
+ CRUSH rule 0 x 945 [7,0]
+ CRUSH rule 0 x 946 [2,8]
+ CRUSH rule 0 x 947 [4,2]
+ CRUSH rule 0 x 948 [7,4]
+ CRUSH rule 0 x 949 [6,2]
+ CRUSH rule 0 x 950 [3,7]
+ CRUSH rule 0 x 951 [4,6]
+ CRUSH rule 0 x 952 [2,7]
+ CRUSH rule 0 x 953 [1,3]
+ CRUSH rule 0 x 954 [4,0]
+ CRUSH rule 0 x 955 [8,1]
+ CRUSH rule 0 x 956 [1,7]
+ CRUSH rule 0 x 957 [7,0]
+ CRUSH rule 0 x 958 [8,3]
+ CRUSH rule 0 x 959 [5,1]
+ CRUSH rule 0 x 960 [3,6]
+ CRUSH rule 0 x 961 [4,1]
+ CRUSH rule 0 x 962 [7,5]
+ CRUSH rule 0 x 963 [0,5]
+ CRUSH rule 0 x 964 [3,2]
+ CRUSH rule 0 x 965 [7,3]
+ CRUSH rule 0 x 966 [3,6]
+ CRUSH rule 0 x 967 [8,4]
+ CRUSH rule 0 x 968 [7,0]
+ CRUSH rule 0 x 969 [8,0]
+ CRUSH rule 0 x 970 [0,8]
+ CRUSH rule 0 x 971 [1,8]
+ CRUSH rule 0 x 972 [1,7]
+ CRUSH rule 0 x 973 [1,8]
+ CRUSH rule 0 x 974 [5,1]
+ CRUSH rule 0 x 975 [3,8]
+ CRUSH rule 0 x 976 [4,7]
+ CRUSH rule 0 x 977 [8,3]
+ CRUSH rule 0 x 978 [7,0]
+ CRUSH rule 0 x 979 [7,2]
+ CRUSH rule 0 x 980 [6,2]
+ CRUSH rule 0 x 981 [7,5]
+ CRUSH rule 0 x 982 [4,1]
+ CRUSH rule 0 x 983 [3,6]
+ CRUSH rule 0 x 984 [0,8]
+ CRUSH rule 0 x 985 [2,4]
+ CRUSH rule 0 x 986 [8,4]
+ CRUSH rule 0 x 987 [0,4]
+ CRUSH rule 0 x 988 [1,4]
+ CRUSH rule 0 x 989 [0,8]
+ CRUSH rule 0 x 990 [1,6]
+ CRUSH rule 0 x 991 [0,4]
+ CRUSH rule 0 x 992 [7,0]
+ CRUSH rule 0 x 993 [0,6]
+ CRUSH rule 0 x 994 [3,6]
+ CRUSH rule 0 x 995 [7,0]
+ CRUSH rule 0 x 996 [6,5]
+ CRUSH rule 0 x 997 [6,3]
+ CRUSH rule 0 x 998 [8,0]
+ CRUSH rule 0 x 999 [0,7]
+ CRUSH rule 0 x 1000 [8,4]
+ CRUSH rule 0 x 1001 [2,3]
+ CRUSH rule 0 x 1002 [1,3]
+ CRUSH rule 0 x 1003 [2,7]
+ CRUSH rule 0 x 1004 [6,0]
+ CRUSH rule 0 x 1005 [6,1]
+ CRUSH rule 0 x 1006 [1,8]
+ CRUSH rule 0 x 1007 [1,3]
+ CRUSH rule 0 x 1008 [1,7]
+ CRUSH rule 0 x 1009 [6,5]
+ CRUSH rule 0 x 1010 [3,1]
+ CRUSH rule 0 x 1011 [3,0]
+ CRUSH rule 0 x 1012 [3,1]
+ CRUSH rule 0 x 1013 [5,2]
+ CRUSH rule 0 x 1014 [2,8]
+ CRUSH rule 0 x 1015 [6,3]
+ CRUSH rule 0 x 1016 [2,5]
+ CRUSH rule 0 x 1017 [6,1]
+ CRUSH rule 0 x 1018 [5,1]
+ CRUSH rule 0 x 1019 [5,8]
+ CRUSH rule 0 x 1020 [5,0]
+ CRUSH rule 0 x 1021 [5,2]
+ CRUSH rule 0 x 1022 [1,7]
+ CRUSH rule 0 x 1023 [3,0]
+ rule 0 (choose) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [0,3,7]
+ CRUSH rule 0 x 1 [0,8,5]
+ CRUSH rule 0 x 2 [1,4,8]
+ CRUSH rule 0 x 3 [8,0,4]
+ CRUSH rule 0 x 4 [5,1,8]
+ CRUSH rule 0 x 5 [7,0,3]
+ CRUSH rule 0 x 6 [2,6,3]
+ CRUSH rule 0 x 7 [5,6,1]
+ CRUSH rule 0 x 8 [5,7,2]
+ CRUSH rule 0 x 9 [2,4,8]
+ CRUSH rule 0 x 10 [0,8,4]
+ CRUSH rule 0 x 11 [0,6,3]
+ CRUSH rule 0 x 12 [0,3,7]
+ CRUSH rule 0 x 13 [3,8,0]
+ CRUSH rule 0 x 14 [7,1,5]
+ CRUSH rule 0 x 15 [7,1,3]
+ CRUSH rule 0 x 16 [3,7,1]
+ CRUSH rule 0 x 17 [5,1,7]
+ CRUSH rule 0 x 18 [1,3,6]
+ CRUSH rule 0 x 19 [7,5,2]
+ CRUSH rule 0 x 20 [2,4,7]
+ CRUSH rule 0 x 21 [3,6,0]
+ CRUSH rule 0 x 22 [8,5,1]
+ CRUSH rule 0 x 23 [3,7,2]
+ CRUSH rule 0 x 24 [1,6,3]
+ CRUSH rule 0 x 25 [3,8,1]
+ CRUSH rule 0 x 26 [2,7,3]
+ CRUSH rule 0 x 27 [3,1,7]
+ CRUSH rule 0 x 28 [6,2,4]
+ CRUSH rule 0 x 29 [8,5,2]
+ CRUSH rule 0 x 30 [5,6,0]
+ CRUSH rule 0 x 31 [8,2,5]
+ CRUSH rule 0 x 32 [3,7,2]
+ CRUSH rule 0 x 33 [2,7,4]
+ CRUSH rule 0 x 34 [2,3,7]
+ CRUSH rule 0 x 35 [0,6,3]
+ CRUSH rule 0 x 36 [3,6,1]
+ CRUSH rule 0 x 37 [0,4,7]
+ CRUSH rule 0 x 38 [4,6,1]
+ CRUSH rule 0 x 39 [3,7,2]
+ CRUSH rule 0 x 40 [7,2,5]
+ CRUSH rule 0 x 41 [0,7,4]
+ CRUSH rule 0 x 42 [4,7,2]
+ CRUSH rule 0 x 43 [0,3,8]
+ CRUSH rule 0 x 44 [1,8,4]
+ CRUSH rule 0 x 45 [8,2,3]
+ CRUSH rule 0 x 46 [2,4,8]
+ CRUSH rule 0 x 47 [4,1,7]
+ CRUSH rule 0 x 48 [4,6,0]
+ CRUSH rule 0 x 49 [5,6,0]
+ CRUSH rule 0 x 50 [3,1,7]
+ CRUSH rule 0 x 51 [3,6,1]
+ CRUSH rule 0 x 52 [8,2,3]
+ CRUSH rule 0 x 53 [3,6,2]
+ CRUSH rule 0 x 54 [7,4,0]
+ CRUSH rule 0 x 55 [8,0,3]
+ CRUSH rule 0 x 56 [6,5,1]
+ CRUSH rule 0 x 57 [5,8,0]
+ CRUSH rule 0 x 58 [1,8,5]
+ CRUSH rule 0 x 59 [4,0,7]
+ CRUSH rule 0 x 60 [3,1,8]
+ CRUSH rule 0 x 61 [4,8,0]
+ CRUSH rule 0 x 62 [7,0,4]
+ CRUSH rule 0 x 63 [5,6,2]
+ CRUSH rule 0 x 64 [4,1,8]
+ CRUSH rule 0 x 65 [7,4,2]
+ CRUSH rule 0 x 66 [5,6,0]
+ CRUSH rule 0 x 67 [5,0,6]
+ CRUSH rule 0 x 68 [0,3,8]
+ CRUSH rule 0 x 69 [5,2,7]
+ CRUSH rule 0 x 70 [7,2,3]
+ CRUSH rule 0 x 71 [2,7,4]
+ CRUSH rule 0 x 72 [6,2,4]
+ CRUSH rule 0 x 73 [2,7,3]
+ CRUSH rule 0 x 74 [0,8,3]
+ CRUSH rule 0 x 75 [3,0,7]
+ CRUSH rule 0 x 76 [5,2,6]
+ CRUSH rule 0 x 77 [7,0,5]
+ CRUSH rule 0 x 78 [1,4,7]
+ CRUSH rule 0 x 79 [5,0,7]
+ CRUSH rule 0 x 80 [0,4,6]
+ CRUSH rule 0 x 81 [0,5,8]
+ CRUSH rule 0 x 82 [7,2,5]
+ CRUSH rule 0 x 83 [2,6,4]
+ CRUSH rule 0 x 84 [7,0,3]
+ CRUSH rule 0 x 85 [3,7,0]
+ CRUSH rule 0 x 86 [0,6,4]
+ CRUSH rule 0 x 87 [0,6,5]
+ CRUSH rule 0 x 88 [1,8,4]
+ CRUSH rule 0 x 89 [3,1,8]
+ CRUSH rule 0 x 90 [6,3,2]
+ CRUSH rule 0 x 91 [3,6,2]
+ CRUSH rule 0 x 92 [1,6,3]
+ CRUSH rule 0 x 93 [7,4,0]
+ CRUSH rule 0 x 94 [0,3,7]
+ CRUSH rule 0 x 95 [7,3,1]
+ CRUSH rule 0 x 96 [3,8,2]
+ CRUSH rule 0 x 97 [8,4,1]
+ CRUSH rule 0 x 98 [2,7,4]
+ CRUSH rule 0 x 99 [0,8,5]
+ CRUSH rule 0 x 100 [1,6,3]
+ CRUSH rule 0 x 101 [3,8,1]
+ CRUSH rule 0 x 102 [4,0,7]
+ CRUSH rule 0 x 103 [4,7,0]
+ CRUSH rule 0 x 104 [7,4,0]
+ CRUSH rule 0 x 105 [2,3,8]
+ CRUSH rule 0 x 106 [1,6,5]
+ CRUSH rule 0 x 107 [3,1,7]
+ CRUSH rule 0 x 108 [7,1,3]
+ CRUSH rule 0 x 109 [1,3,8]
+ CRUSH rule 0 x 110 [3,2,6]
+ CRUSH rule 0 x 111 [2,4,7]
+ CRUSH rule 0 x 112 [2,6,3]
+ CRUSH rule 0 x 113 [6,1,4]
+ CRUSH rule 0 x 114 [7,5,0]
+ CRUSH rule 0 x 115 [8,2,3]
+ CRUSH rule 0 x 116 [1,7,4]
+ CRUSH rule 0 x 117 [7,5,1]
+ CRUSH rule 0 x 118 [0,3,6]
+ CRUSH rule 0 x 119 [5,7,1]
+ CRUSH rule 0 x 120 [0,5,6]
+ CRUSH rule 0 x 121 [2,8,4]
+ CRUSH rule 0 x 122 [8,4,2]
+ CRUSH rule 0 x 123 [2,4,7]
+ CRUSH rule 0 x 124 [3,0,8]
+ CRUSH rule 0 x 125 [0,7,3]
+ CRUSH rule 0 x 126 [4,1,7]
+ CRUSH rule 0 x 127 [3,6,1]
+ CRUSH rule 0 x 128 [3,8,1]
+ CRUSH rule 0 x 129 [0,4,7]
+ CRUSH rule 0 x 130 [3,8,1]
+ CRUSH rule 0 x 131 [1,5,6]
+ CRUSH rule 0 x 132 [1,3,8]
+ CRUSH rule 0 x 133 [3,6,1]
+ CRUSH rule 0 x 134 [1,7,4]
+ CRUSH rule 0 x 135 [5,7,0]
+ CRUSH rule 0 x 136 [2,3,8]
+ CRUSH rule 0 x 137 [7,3,1]
+ CRUSH rule 0 x 138 [8,3,0]
+ CRUSH rule 0 x 139 [3,0,8]
+ CRUSH rule 0 x 140 [1,8,3]
+ CRUSH rule 0 x 141 [6,0,4]
+ CRUSH rule 0 x 142 [3,2,8]
+ CRUSH rule 0 x 143 [5,7,0]
+ CRUSH rule 0 x 144 [8,2,5]
+ CRUSH rule 0 x 145 [8,4,2]
+ CRUSH rule 0 x 146 [2,8,4]
+ CRUSH rule 0 x 147 [2,6,4]
+ CRUSH rule 0 x 148 [3,0,8]
+ CRUSH rule 0 x 149 [4,6,2]
+ CRUSH rule 0 x 150 [1,8,4]
+ CRUSH rule 0 x 151 [3,8,1]
+ CRUSH rule 0 x 152 [8,3,0]
+ CRUSH rule 0 x 153 [8,3,0]
+ CRUSH rule 0 x 154 [3,0,6]
+ CRUSH rule 0 x 155 [3,6,0]
+ CRUSH rule 0 x 156 [4,2,7]
+ CRUSH rule 0 x 157 [4,1,7]
+ CRUSH rule 0 x 158 [2,6,3]
+ CRUSH rule 0 x 159 [7,0,3]
+ CRUSH rule 0 x 160 [2,8,5]
+ CRUSH rule 0 x 161 [1,3,6]
+ CRUSH rule 0 x 162 [0,8,5]
+ CRUSH rule 0 x 163 [5,8,2]
+ CRUSH rule 0 x 164 [7,2,4]
+ CRUSH rule 0 x 165 [7,2,4]
+ CRUSH rule 0 x 166 [2,3,7]
+ CRUSH rule 0 x 167 [0,6,3]
+ CRUSH rule 0 x 168 [4,0,7]
+ CRUSH rule 0 x 169 [2,7,3]
+ CRUSH rule 0 x 170 [1,3,8]
+ CRUSH rule 0 x 171 [7,3,2]
+ CRUSH rule 0 x 172 [0,8,4]
+ CRUSH rule 0 x 173 [8,5,1]
+ CRUSH rule 0 x 174 [1,4,8]
+ CRUSH rule 0 x 175 [6,0,3]
+ CRUSH rule 0 x 176 [4,2,7]
+ CRUSH rule 0 x 177 [5,1,8]
+ CRUSH rule 0 x 178 [3,0,6]
+ CRUSH rule 0 x 179 [4,1,7]
+ CRUSH rule 0 x 180 [3,7,2]
+ CRUSH rule 0 x 181 [6,0,5]
+ CRUSH rule 0 x 182 [8,3,1]
+ CRUSH rule 0 x 183 [7,5,1]
+ CRUSH rule 0 x 184 [5,6,1]
+ CRUSH rule 0 x 185 [6,0,3]
+ CRUSH rule 0 x 186 [2,4,8]
+ CRUSH rule 0 x 187 [1,6,3]
+ CRUSH rule 0 x 188 [1,6,3]
+ CRUSH rule 0 x 189 [0,6,5]
+ CRUSH rule 0 x 190 [4,1,8]
+ CRUSH rule 0 x 191 [7,0,5]
+ CRUSH rule 0 x 192 [5,2,6]
+ CRUSH rule 0 x 193 [4,0,6]
+ CRUSH rule 0 x 194 [1,3,7]
+ CRUSH rule 0 x 195 [6,5,2]
+ CRUSH rule 0 x 196 [6,1,3]
+ CRUSH rule 0 x 197 [6,5,1]
+ CRUSH rule 0 x 198 [2,3,6]
+ CRUSH rule 0 x 199 [0,5,7]
+ CRUSH rule 0 x 200 [0,3,8]
+ CRUSH rule 0 x 201 [7,1,5]
+ CRUSH rule 0 x 202 [6,4,1]
+ CRUSH rule 0 x 203 [4,6,1]
+ CRUSH rule 0 x 204 [2,4,8]
+ CRUSH rule 0 x 205 [0,7,4]
+ CRUSH rule 0 x 206 [0,8,4]
+ CRUSH rule 0 x 207 [3,2,7]
+ CRUSH rule 0 x 208 [7,2,4]
+ CRUSH rule 0 x 209 [1,8,3]
+ CRUSH rule 0 x 210 [1,4,6]
+ CRUSH rule 0 x 211 [5,2,7]
+ CRUSH rule 0 x 212 [7,5,0]
+ CRUSH rule 0 x 213 [8,4,0]
+ CRUSH rule 0 x 214 [4,7,1]
+ CRUSH rule 0 x 215 [8,0,5]
+ CRUSH rule 0 x 216 [5,2,8]
+ CRUSH rule 0 x 217 [0,7,5]
+ CRUSH rule 0 x 218 [0,7,4]
+ CRUSH rule 0 x 219 [4,6,0]
+ CRUSH rule 0 x 220 [5,8,0]
+ CRUSH rule 0 x 221 [3,7,0]
+ CRUSH rule 0 x 222 [6,4,1]
+ CRUSH rule 0 x 223 [1,3,6]
+ CRUSH rule 0 x 224 [1,3,8]
+ CRUSH rule 0 x 225 [8,0,3]
+ CRUSH rule 0 x 226 [7,0,4]
+ CRUSH rule 0 x 227 [3,0,7]
+ CRUSH rule 0 x 228 [5,6,1]
+ CRUSH rule 0 x 229 [3,7,0]
+ CRUSH rule 0 x 230 [4,6,1]
+ CRUSH rule 0 x 231 [4,7,1]
+ CRUSH rule 0 x 232 [2,8,4]
+ CRUSH rule 0 x 233 [3,6,0]
+ CRUSH rule 0 x 234 [0,4,6]
+ CRUSH rule 0 x 235 [3,6,1]
+ CRUSH rule 0 x 236 [5,0,8]
+ CRUSH rule 0 x 237 [4,8,0]
+ CRUSH rule 0 x 238 [4,2,6]
+ CRUSH rule 0 x 239 [8,3,2]
+ CRUSH rule 0 x 240 [5,8,2]
+ CRUSH rule 0 x 241 [3,1,7]
+ CRUSH rule 0 x 242 [3,2,6]
+ CRUSH rule 0 x 243 [4,8,2]
+ CRUSH rule 0 x 244 [4,6,0]
+ CRUSH rule 0 x 245 [7,2,3]
+ CRUSH rule 0 x 246 [1,3,7]
+ CRUSH rule 0 x 247 [6,1,4]
+ CRUSH rule 0 x 248 [8,0,3]
+ CRUSH rule 0 x 249 [2,5,8]
+ CRUSH rule 0 x 250 [2,3,8]
+ CRUSH rule 0 x 251 [2,3,7]
+ CRUSH rule 0 x 252 [3,7,2]
+ CRUSH rule 0 x 253 [3,0,7]
+ CRUSH rule 0 x 254 [3,2,8]
+ CRUSH rule 0 x 255 [1,7,4]
+ CRUSH rule 0 x 256 [5,6,1]
+ CRUSH rule 0 x 257 [2,6,4]
+ CRUSH rule 0 x 258 [5,2,8]
+ CRUSH rule 0 x 259 [4,6,1]
+ CRUSH rule 0 x 260 [3,8,1]
+ CRUSH rule 0 x 261 [8,5,2]
+ CRUSH rule 0 x 262 [5,6,2]
+ CRUSH rule 0 x 263 [6,1,5]
+ CRUSH rule 0 x 264 [3,6,1]
+ CRUSH rule 0 x 265 [8,4,0]
+ CRUSH rule 0 x 266 [8,0,4]
+ CRUSH rule 0 x 267 [2,4,6]
+ CRUSH rule 0 x 268 [0,6,4]
+ CRUSH rule 0 x 269 [0,6,5]
+ CRUSH rule 0 x 270 [5,1,8]
+ CRUSH rule 0 x 271 [7,4,2]
+ CRUSH rule 0 x 272 [2,6,5]
+ CRUSH rule 0 x 273 [3,2,7]
+ CRUSH rule 0 x 274 [6,3,2]
+ CRUSH rule 0 x 275 [4,8,1]
+ CRUSH rule 0 x 276 [7,0,5]
+ CRUSH rule 0 x 277 [6,4,1]
+ CRUSH rule 0 x 278 [6,2,3]
+ CRUSH rule 0 x 279 [8,5,0]
+ CRUSH rule 0 x 280 [0,7,4]
+ CRUSH rule 0 x 281 [8,1,5]
+ CRUSH rule 0 x 282 [3,2,8]
+ CRUSH rule 0 x 283 [8,1,3]
+ CRUSH rule 0 x 284 [6,3,1]
+ CRUSH rule 0 x 285 [5,7,2]
+ CRUSH rule 0 x 286 [2,8,4]
+ CRUSH rule 0 x 287 [0,3,8]
+ CRUSH rule 0 x 288 [8,1,4]
+ CRUSH rule 0 x 289 [4,6,0]
+ CRUSH rule 0 x 290 [1,5,6]
+ CRUSH rule 0 x 291 [0,5,8]
+ CRUSH rule 0 x 292 [8,1,4]
+ CRUSH rule 0 x 293 [6,2,3]
+ CRUSH rule 0 x 294 [7,5,2]
+ CRUSH rule 0 x 295 [4,6,0]
+ CRUSH rule 0 x 296 [3,1,6]
+ CRUSH rule 0 x 297 [6,0,5]
+ CRUSH rule 0 x 298 [1,5,6]
+ CRUSH rule 0 x 299 [2,8,4]
+ CRUSH rule 0 x 300 [8,4,1]
+ CRUSH rule 0 x 301 [0,7,3]
+ CRUSH rule 0 x 302 [3,1,6]
+ CRUSH rule 0 x 303 [7,4,1]
+ CRUSH rule 0 x 304 [2,6,4]
+ CRUSH rule 0 x 305 [5,6,1]
+ CRUSH rule 0 x 306 [0,8,3]
+ CRUSH rule 0 x 307 [0,7,4]
+ CRUSH rule 0 x 308 [0,8,5]
+ CRUSH rule 0 x 309 [7,4,2]
+ CRUSH rule 0 x 310 [4,0,6]
+ CRUSH rule 0 x 311 [3,8,2]
+ CRUSH rule 0 x 312 [2,6,5]
+ CRUSH rule 0 x 313 [5,1,7]
+ CRUSH rule 0 x 314 [4,0,6]
+ CRUSH rule 0 x 315 [2,3,7]
+ CRUSH rule 0 x 316 [6,4,1]
+ CRUSH rule 0 x 317 [2,7,4]
+ CRUSH rule 0 x 318 [8,1,4]
+ CRUSH rule 0 x 319 [5,2,6]
+ CRUSH rule 0 x 320 [3,8,0]
+ CRUSH rule 0 x 321 [1,5,8]
+ CRUSH rule 0 x 322 [2,6,3]
+ CRUSH rule 0 x 323 [4,7,0]
+ CRUSH rule 0 x 324 [7,2,5]
+ CRUSH rule 0 x 325 [4,6,0]
+ CRUSH rule 0 x 326 [3,1,7]
+ CRUSH rule 0 x 327 [0,8,3]
+ CRUSH rule 0 x 328 [7,3,1]
+ CRUSH rule 0 x 329 [5,7,0]
+ CRUSH rule 0 x 330 [3,7,0]
+ CRUSH rule 0 x 331 [2,7,3]
+ CRUSH rule 0 x 332 [2,3,8]
+ CRUSH rule 0 x 333 [6,4,1]
+ CRUSH rule 0 x 334 [8,3,2]
+ CRUSH rule 0 x 335 [7,2,3]
+ CRUSH rule 0 x 336 [4,6,2]
+ CRUSH rule 0 x 337 [7,0,4]
+ CRUSH rule 0 x 338 [5,8,2]
+ CRUSH rule 0 x 339 [7,5,0]
+ CRUSH rule 0 x 340 [2,6,5]
+ CRUSH rule 0 x 341 [5,1,7]
+ CRUSH rule 0 x 342 [0,8,5]
+ CRUSH rule 0 x 343 [6,5,0]
+ CRUSH rule 0 x 344 [6,1,4]
+ CRUSH rule 0 x 345 [4,7,0]
+ CRUSH rule 0 x 346 [8,0,3]
+ CRUSH rule 0 x 347 [3,0,8]
+ CRUSH rule 0 x 348 [8,0,3]
+ CRUSH rule 0 x 349 [1,7,3]
+ CRUSH rule 0 x 350 [8,5,1]
+ CRUSH rule 0 x 351 [3,8,0]
+ CRUSH rule 0 x 352 [1,8,4]
+ CRUSH rule 0 x 353 [6,5,1]
+ CRUSH rule 0 x 354 [0,5,6]
+ CRUSH rule 0 x 355 [3,8,0]
+ CRUSH rule 0 x 356 [3,1,8]
+ CRUSH rule 0 x 357 [6,1,3]
+ CRUSH rule 0 x 358 [2,8,5]
+ CRUSH rule 0 x 359 [6,0,5]
+ CRUSH rule 0 x 360 [5,0,8]
+ CRUSH rule 0 x 361 [8,5,0]
+ CRUSH rule 0 x 362 [4,0,8]
+ CRUSH rule 0 x 363 [4,2,8]
+ CRUSH rule 0 x 364 [2,5,7]
+ CRUSH rule 0 x 365 [6,4,2]
+ CRUSH rule 0 x 366 [7,0,3]
+ CRUSH rule 0 x 367 [4,2,7]
+ CRUSH rule 0 x 368 [7,3,1]
+ CRUSH rule 0 x 369 [3,7,2]
+ CRUSH rule 0 x 370 [8,2,4]
+ CRUSH rule 0 x 371 [1,5,8]
+ CRUSH rule 0 x 372 [3,1,8]
+ CRUSH rule 0 x 373 [0,6,4]
+ CRUSH rule 0 x 374 [3,8,1]
+ CRUSH rule 0 x 375 [6,5,2]
+ CRUSH rule 0 x 376 [7,1,3]
+ CRUSH rule 0 x 377 [1,4,7]
+ CRUSH rule 0 x 378 [0,6,4]
+ CRUSH rule 0 x 379 [8,3,0]
+ CRUSH rule 0 x 380 [2,3,8]
+ CRUSH rule 0 x 381 [0,3,7]
+ CRUSH rule 0 x 382 [1,3,7]
+ CRUSH rule 0 x 383 [4,7,1]
+ CRUSH rule 0 x 384 [7,0,4]
+ CRUSH rule 0 x 385 [7,4,0]
+ CRUSH rule 0 x 386 [0,4,6]
+ CRUSH rule 0 x 387 [1,5,8]
+ CRUSH rule 0 x 388 [5,2,7]
+ CRUSH rule 0 x 389 [1,4,8]
+ CRUSH rule 0 x 390 [5,8,1]
+ CRUSH rule 0 x 391 [5,6,0]
+ CRUSH rule 0 x 392 [1,7,5]
+ CRUSH rule 0 x 393 [4,0,6]
+ CRUSH rule 0 x 394 [4,8,2]
+ CRUSH rule 0 x 395 [4,0,8]
+ CRUSH rule 0 x 396 [4,2,6]
+ CRUSH rule 0 x 397 [2,4,7]
+ CRUSH rule 0 x 398 [2,3,6]
+ CRUSH rule 0 x 399 [8,5,2]
+ CRUSH rule 0 x 400 [8,1,3]
+ CRUSH rule 0 x 401 [0,3,6]
+ CRUSH rule 0 x 402 [7,4,2]
+ CRUSH rule 0 x 403 [0,4,7]
+ CRUSH rule 0 x 404 [4,0,6]
+ CRUSH rule 0 x 405 [6,4,0]
+ CRUSH rule 0 x 406 [2,6,4]
+ CRUSH rule 0 x 407 [2,7,5]
+ CRUSH rule 0 x 408 [4,1,7]
+ CRUSH rule 0 x 409 [7,5,0]
+ CRUSH rule 0 x 410 [8,3,1]
+ CRUSH rule 0 x 411 [2,7,4]
+ CRUSH rule 0 x 412 [0,3,7]
+ CRUSH rule 0 x 413 [5,2,8]
+ CRUSH rule 0 x 414 [4,0,8]
+ CRUSH rule 0 x 415 [0,6,3]
+ CRUSH rule 0 x 416 [2,8,5]
+ CRUSH rule 0 x 417 [8,0,3]
+ CRUSH rule 0 x 418 [7,1,3]
+ CRUSH rule 0 x 419 [8,5,2]
+ CRUSH rule 0 x 420 [1,4,7]
+ CRUSH rule 0 x 421 [8,3,0]
+ CRUSH rule 0 x 422 [6,3,1]
+ CRUSH rule 0 x 423 [0,3,7]
+ CRUSH rule 0 x 424 [8,5,1]
+ CRUSH rule 0 x 425 [1,4,8]
+ CRUSH rule 0 x 426 [6,2,4]
+ CRUSH rule 0 x 427 [0,8,3]
+ CRUSH rule 0 x 428 [5,8,1]
+ CRUSH rule 0 x 429 [4,8,0]
+ CRUSH rule 0 x 430 [3,7,0]
+ CRUSH rule 0 x 431 [5,0,7]
+ CRUSH rule 0 x 432 [7,0,4]
+ CRUSH rule 0 x 433 [6,4,0]
+ CRUSH rule 0 x 434 [5,0,7]
+ CRUSH rule 0 x 435 [0,4,6]
+ CRUSH rule 0 x 436 [4,0,7]
+ CRUSH rule 0 x 437 [7,3,1]
+ CRUSH rule 0 x 438 [0,5,8]
+ CRUSH rule 0 x 439 [1,3,8]
+ CRUSH rule 0 x 440 [2,6,5]
+ CRUSH rule 0 x 441 [5,8,0]
+ CRUSH rule 0 x 442 [2,3,6]
+ CRUSH rule 0 x 443 [6,0,3]
+ CRUSH rule 0 x 444 [7,1,4]
+ CRUSH rule 0 x 445 [6,4,0]
+ CRUSH rule 0 x 446 [4,0,8]
+ CRUSH rule 0 x 447 [2,4,6]
+ CRUSH rule 0 x 448 [7,0,5]
+ CRUSH rule 0 x 449 [7,4,2]
+ CRUSH rule 0 x 450 [4,0,6]
+ CRUSH rule 0 x 451 [6,5,1]
+ CRUSH rule 0 x 452 [8,4,0]
+ CRUSH rule 0 x 453 [6,5,0]
+ CRUSH rule 0 x 454 [6,5,0]
+ CRUSH rule 0 x 455 [2,8,4]
+ CRUSH rule 0 x 456 [6,2,5]
+ CRUSH rule 0 x 457 [7,1,5]
+ CRUSH rule 0 x 458 [2,8,5]
+ CRUSH rule 0 x 459 [2,6,5]
+ CRUSH rule 0 x 460 [6,5,2]
+ CRUSH rule 0 x 461 [6,4,2]
+ CRUSH rule 0 x 462 [8,0,4]
+ CRUSH rule 0 x 463 [6,0,4]
+ CRUSH rule 0 x 464 [7,4,1]
+ CRUSH rule 0 x 465 [7,1,5]
+ CRUSH rule 0 x 466 [5,6,2]
+ CRUSH rule 0 x 467 [6,5,1]
+ CRUSH rule 0 x 468 [7,2,3]
+ CRUSH rule 0 x 469 [7,2,4]
+ CRUSH rule 0 x 470 [3,0,8]
+ CRUSH rule 0 x 471 [0,6,4]
+ CRUSH rule 0 x 472 [5,0,8]
+ CRUSH rule 0 x 473 [1,4,7]
+ CRUSH rule 0 x 474 [6,1,4]
+ CRUSH rule 0 x 475 [6,2,3]
+ CRUSH rule 0 x 476 [4,7,1]
+ CRUSH rule 0 x 477 [5,6,1]
+ CRUSH rule 0 x 478 [6,1,3]
+ CRUSH rule 0 x 479 [0,3,6]
+ CRUSH rule 0 x 480 [1,6,4]
+ CRUSH rule 0 x 481 [2,5,7]
+ CRUSH rule 0 x 482 [4,8,0]
+ CRUSH rule 0 x 483 [0,6,4]
+ CRUSH rule 0 x 484 [1,8,5]
+ CRUSH rule 0 x 485 [4,8,1]
+ CRUSH rule 0 x 486 [4,0,8]
+ CRUSH rule 0 x 487 [5,0,8]
+ CRUSH rule 0 x 488 [5,7,1]
+ CRUSH rule 0 x 489 [2,8,4]
+ CRUSH rule 0 x 490 [6,4,0]
+ CRUSH rule 0 x 491 [1,6,5]
+ CRUSH rule 0 x 492 [6,5,2]
+ CRUSH rule 0 x 493 [0,8,3]
+ CRUSH rule 0 x 494 [1,6,4]
+ CRUSH rule 0 x 495 [3,0,6]
+ CRUSH rule 0 x 496 [7,5,1]
+ CRUSH rule 0 x 497 [5,7,2]
+ CRUSH rule 0 x 498 [0,4,6]
+ CRUSH rule 0 x 499 [8,5,2]
+ CRUSH rule 0 x 500 [3,6,1]
+ CRUSH rule 0 x 501 [0,7,5]
+ CRUSH rule 0 x 502 [7,1,3]
+ CRUSH rule 0 x 503 [2,3,7]
+ CRUSH rule 0 x 504 [5,8,2]
+ CRUSH rule 0 x 505 [0,7,5]
+ CRUSH rule 0 x 506 [5,1,7]
+ CRUSH rule 0 x 507 [6,0,3]
+ CRUSH rule 0 x 508 [0,4,7]
+ CRUSH rule 0 x 509 [7,4,0]
+ CRUSH rule 0 x 510 [6,2,5]
+ CRUSH rule 0 x 511 [5,6,2]
+ CRUSH rule 0 x 512 [7,2,4]
+ CRUSH rule 0 x 513 [7,2,4]
+ CRUSH rule 0 x 514 [4,7,0]
+ CRUSH rule 0 x 515 [8,3,0]
+ CRUSH rule 0 x 516 [4,1,7]
+ CRUSH rule 0 x 517 [7,0,4]
+ CRUSH rule 0 x 518 [4,6,0]
+ CRUSH rule 0 x 519 [7,3,1]
+ CRUSH rule 0 x 520 [2,8,5]
+ CRUSH rule 0 x 521 [8,0,3]
+ CRUSH rule 0 x 522 [6,0,4]
+ CRUSH rule 0 x 523 [4,1,7]
+ CRUSH rule 0 x 524 [0,4,7]
+ CRUSH rule 0 x 525 [0,3,8]
+ CRUSH rule 0 x 526 [1,3,8]
+ CRUSH rule 0 x 527 [0,4,6]
+ CRUSH rule 0 x 528 [5,2,7]
+ CRUSH rule 0 x 529 [5,6,1]
+ CRUSH rule 0 x 530 [6,4,1]
+ CRUSH rule 0 x 531 [6,0,3]
+ CRUSH rule 0 x 532 [6,5,1]
+ CRUSH rule 0 x 533 [5,8,0]
+ CRUSH rule 0 x 534 [7,4,2]
+ CRUSH rule 0 x 535 [8,0,3]
+ CRUSH rule 0 x 536 [6,2,3]
+ CRUSH rule 0 x 537 [3,8,0]
+ CRUSH rule 0 x 538 [6,4,2]
+ CRUSH rule 0 x 539 [8,4,2]
+ CRUSH rule 0 x 540 [0,7,4]
+ CRUSH rule 0 x 541 [2,5,7]
+ CRUSH rule 0 x 542 [3,0,8]
+ CRUSH rule 0 x 543 [6,2,4]
+ CRUSH rule 0 x 544 [3,7,1]
+ CRUSH rule 0 x 545 [5,7,1]
+ CRUSH rule 0 x 546 [6,2,3]
+ CRUSH rule 0 x 547 [8,1,5]
+ CRUSH rule 0 x 548 [5,1,8]
+ CRUSH rule 0 x 549 [5,7,1]
+ CRUSH rule 0 x 550 [0,5,6]
+ CRUSH rule 0 x 551 [7,4,0]
+ CRUSH rule 0 x 552 [5,7,2]
+ CRUSH rule 0 x 553 [4,2,8]
+ CRUSH rule 0 x 554 [0,6,5]
+ CRUSH rule 0 x 555 [5,0,8]
+ CRUSH rule 0 x 556 [3,6,1]
+ CRUSH rule 0 x 557 [7,4,0]
+ CRUSH rule 0 x 558 [3,2,6]
+ CRUSH rule 0 x 559 [4,1,8]
+ CRUSH rule 0 x 560 [8,4,2]
+ CRUSH rule 0 x 561 [6,4,1]
+ CRUSH rule 0 x 562 [3,0,8]
+ CRUSH rule 0 x 563 [2,7,4]
+ CRUSH rule 0 x 564 [5,1,7]
+ CRUSH rule 0 x 565 [3,8,0]
+ CRUSH rule 0 x 566 [4,6,0]
+ CRUSH rule 0 x 567 [3,7,0]
+ CRUSH rule 0 x 568 [7,4,0]
+ CRUSH rule 0 x 569 [3,1,6]
+ CRUSH rule 0 x 570 [1,4,7]
+ CRUSH rule 0 x 571 [3,6,0]
+ CRUSH rule 0 x 572 [3,0,7]
+ CRUSH rule 0 x 573 [3,1,7]
+ CRUSH rule 0 x 574 [2,3,6]
+ CRUSH rule 0 x 575 [8,1,5]
+ CRUSH rule 0 x 576 [4,7,1]
+ CRUSH rule 0 x 577 [8,2,3]
+ CRUSH rule 0 x 578 [6,0,4]
+ CRUSH rule 0 x 579 [3,2,6]
+ CRUSH rule 0 x 580 [3,0,8]
+ CRUSH rule 0 x 581 [7,1,5]
+ CRUSH rule 0 x 582 [2,8,5]
+ CRUSH rule 0 x 583 [6,2,3]
+ CRUSH rule 0 x 584 [8,0,5]
+ CRUSH rule 0 x 585 [7,0,5]
+ CRUSH rule 0 x 586 [0,8,5]
+ CRUSH rule 0 x 587 [2,5,6]
+ CRUSH rule 0 x 588 [3,8,0]
+ CRUSH rule 0 x 589 [7,1,5]
+ CRUSH rule 0 x 590 [6,2,5]
+ CRUSH rule 0 x 591 [5,2,7]
+ CRUSH rule 0 x 592 [2,4,6]
+ CRUSH rule 0 x 593 [0,8,4]
+ CRUSH rule 0 x 594 [0,6,3]
+ CRUSH rule 0 x 595 [7,1,5]
+ CRUSH rule 0 x 596 [4,0,6]
+ CRUSH rule 0 x 597 [3,1,7]
+ CRUSH rule 0 x 598 [3,0,7]
+ CRUSH rule 0 x 599 [5,1,6]
+ CRUSH rule 0 x 600 [7,0,4]
+ CRUSH rule 0 x 601 [0,6,3]
+ CRUSH rule 0 x 602 [3,8,2]
+ CRUSH rule 0 x 603 [5,0,8]
+ CRUSH rule 0 x 604 [7,4,0]
+ CRUSH rule 0 x 605 [3,2,6]
+ CRUSH rule 0 x 606 [2,7,3]
+ CRUSH rule 0 x 607 [0,5,8]
+ CRUSH rule 0 x 608 [5,1,8]
+ CRUSH rule 0 x 609 [5,0,6]
+ CRUSH rule 0 x 610 [3,8,2]
+ CRUSH rule 0 x 611 [1,8,3]
+ CRUSH rule 0 x 612 [2,8,3]
+ CRUSH rule 0 x 613 [7,1,3]
+ CRUSH rule 0 x 614 [7,2,3]
+ CRUSH rule 0 x 615 [6,2,3]
+ CRUSH rule 0 x 616 [0,7,5]
+ CRUSH rule 0 x 617 [6,2,3]
+ CRUSH rule 0 x 618 [7,3,1]
+ CRUSH rule 0 x 619 [5,0,6]
+ CRUSH rule 0 x 620 [4,1,8]
+ CRUSH rule 0 x 621 [5,6,0]
+ CRUSH rule 0 x 622 [0,3,8]
+ CRUSH rule 0 x 623 [0,8,4]
+ CRUSH rule 0 x 624 [3,2,8]
+ CRUSH rule 0 x 625 [2,5,8]
+ CRUSH rule 0 x 626 [7,2,3]
+ CRUSH rule 0 x 627 [2,6,5]
+ CRUSH rule 0 x 628 [8,1,3]
+ CRUSH rule 0 x 629 [2,6,3]
+ CRUSH rule 0 x 630 [2,6,4]
+ CRUSH rule 0 x 631 [0,6,5]
+ CRUSH rule 0 x 632 [7,0,4]
+ CRUSH rule 0 x 633 [8,4,2]
+ CRUSH rule 0 x 634 [0,5,6]
+ CRUSH rule 0 x 635 [5,6,2]
+ CRUSH rule 0 x 636 [1,3,7]
+ CRUSH rule 0 x 637 [4,0,8]
+ CRUSH rule 0 x 638 [6,2,5]
+ CRUSH rule 0 x 639 [4,0,6]
+ CRUSH rule 0 x 640 [3,2,7]
+ CRUSH rule 0 x 641 [7,2,3]
+ CRUSH rule 0 x 642 [2,8,4]
+ CRUSH rule 0 x 643 [3,0,8]
+ CRUSH rule 0 x 644 [8,0,4]
+ CRUSH rule 0 x 645 [5,7,2]
+ CRUSH rule 0 x 646 [8,0,3]
+ CRUSH rule 0 x 647 [7,0,5]
+ CRUSH rule 0 x 648 [0,8,3]
+ CRUSH rule 0 x 649 [4,7,1]
+ CRUSH rule 0 x 650 [7,5,1]
+ CRUSH rule 0 x 651 [3,6,1]
+ CRUSH rule 0 x 652 [3,6,1]
+ CRUSH rule 0 x 653 [8,3,0]
+ CRUSH rule 0 x 654 [7,4,0]
+ CRUSH rule 0 x 655 [0,5,6]
+ CRUSH rule 0 x 656 [4,7,0]
+ CRUSH rule 0 x 657 [6,0,5]
+ CRUSH rule 0 x 658 [5,8,0]
+ CRUSH rule 0 x 659 [4,7,0]
+ CRUSH rule 0 x 660 [7,3,0]
+ CRUSH rule 0 x 661 [1,7,3]
+ CRUSH rule 0 x 662 [4,2,8]
+ CRUSH rule 0 x 663 [1,3,7]
+ CRUSH rule 0 x 664 [1,3,6]
+ CRUSH rule 0 x 665 [5,6,1]
+ CRUSH rule 0 x 666 [2,7,4]
+ CRUSH rule 0 x 667 [1,3,8]
+ CRUSH rule 0 x 668 [3,7,0]
+ CRUSH rule 0 x 669 [6,3,0]
+ CRUSH rule 0 x 670 [4,1,6]
+ CRUSH rule 0 x 671 [0,8,5]
+ CRUSH rule 0 x 672 [4,2,8]
+ CRUSH rule 0 x 673 [5,0,7]
+ CRUSH rule 0 x 674 [3,0,7]
+ CRUSH rule 0 x 675 [0,8,5]
+ CRUSH rule 0 x 676 [0,3,7]
+ CRUSH rule 0 x 677 [4,1,6]
+ CRUSH rule 0 x 678 [2,3,8]
+ CRUSH rule 0 x 679 [6,0,5]
+ CRUSH rule 0 x 680 [0,3,8]
+ CRUSH rule 0 x 681 [4,6,1]
+ CRUSH rule 0 x 682 [0,4,8]
+ CRUSH rule 0 x 683 [0,3,8]
+ CRUSH rule 0 x 684 [7,2,5]
+ CRUSH rule 0 x 685 [7,2,5]
+ CRUSH rule 0 x 686 [1,5,8]
+ CRUSH rule 0 x 687 [3,7,2]
+ CRUSH rule 0 x 688 [5,6,2]
+ CRUSH rule 0 x 689 [6,5,1]
+ CRUSH rule 0 x 690 [8,0,4]
+ CRUSH rule 0 x 691 [3,1,6]
+ CRUSH rule 0 x 692 [7,1,4]
+ CRUSH rule 0 x 693 [6,5,2]
+ CRUSH rule 0 x 694 [6,4,2]
+ CRUSH rule 0 x 695 [0,6,3]
+ CRUSH rule 0 x 696 [1,5,8]
+ CRUSH rule 0 x 697 [6,0,4]
+ CRUSH rule 0 x 698 [6,0,3]
+ CRUSH rule 0 x 699 [1,8,3]
+ CRUSH rule 0 x 700 [0,4,6]
+ CRUSH rule 0 x 701 [4,0,6]
+ CRUSH rule 0 x 702 [3,0,8]
+ CRUSH rule 0 x 703 [8,4,0]
+ CRUSH rule 0 x 704 [0,4,6]
+ CRUSH rule 0 x 705 [8,0,4]
+ CRUSH rule 0 x 706 [1,5,8]
+ CRUSH rule 0 x 707 [7,3,0]
+ CRUSH rule 0 x 708 [3,7,1]
+ CRUSH rule 0 x 709 [6,5,1]
+ CRUSH rule 0 x 710 [8,5,1]
+ CRUSH rule 0 x 711 [2,4,7]
+ CRUSH rule 0 x 712 [2,3,6]
+ CRUSH rule 0 x 713 [6,3,1]
+ CRUSH rule 0 x 714 [3,0,6]
+ CRUSH rule 0 x 715 [1,3,6]
+ CRUSH rule 0 x 716 [3,6,1]
+ CRUSH rule 0 x 717 [8,0,5]
+ CRUSH rule 0 x 718 [3,7,0]
+ CRUSH rule 0 x 719 [2,6,3]
+ CRUSH rule 0 x 720 [6,0,4]
+ CRUSH rule 0 x 721 [5,7,0]
+ CRUSH rule 0 x 722 [5,7,1]
+ CRUSH rule 0 x 723 [5,2,7]
+ CRUSH rule 0 x 724 [0,7,3]
+ CRUSH rule 0 x 725 [0,4,6]
+ CRUSH rule 0 x 726 [3,7,2]
+ CRUSH rule 0 x 727 [4,7,2]
+ CRUSH rule 0 x 728 [2,6,3]
+ CRUSH rule 0 x 729 [5,6,1]
+ CRUSH rule 0 x 730 [3,8,2]
+ CRUSH rule 0 x 731 [4,1,6]
+ CRUSH rule 0 x 732 [1,4,8]
+ CRUSH rule 0 x 733 [5,6,2]
+ CRUSH rule 0 x 734 [6,5,0]
+ CRUSH rule 0 x 735 [4,6,1]
+ CRUSH rule 0 x 736 [3,8,1]
+ CRUSH rule 0 x 737 [1,8,3]
+ CRUSH rule 0 x 738 [5,1,6]
+ CRUSH rule 0 x 739 [0,7,4]
+ CRUSH rule 0 x 740 [0,7,3]
+ CRUSH rule 0 x 741 [7,2,3]
+ CRUSH rule 0 x 742 [8,2,3]
+ CRUSH rule 0 x 743 [7,0,5]
+ CRUSH rule 0 x 744 [4,6,1]
+ CRUSH rule 0 x 745 [3,0,8]
+ CRUSH rule 0 x 746 [4,1,7]
+ CRUSH rule 0 x 747 [6,2,4]
+ CRUSH rule 0 x 748 [2,6,5]
+ CRUSH rule 0 x 749 [4,7,2]
+ CRUSH rule 0 x 750 [1,7,4]
+ CRUSH rule 0 x 751 [2,7,3]
+ CRUSH rule 0 x 752 [8,0,3]
+ CRUSH rule 0 x 753 [7,4,2]
+ CRUSH rule 0 x 754 [8,5,2]
+ CRUSH rule 0 x 755 [1,6,5]
+ CRUSH rule 0 x 756 [5,8,1]
+ CRUSH rule 0 x 757 [8,0,4]
+ CRUSH rule 0 x 758 [6,1,5]
+ CRUSH rule 0 x 759 [8,5,0]
+ CRUSH rule 0 x 760 [1,5,7]
+ CRUSH rule 0 x 761 [4,2,6]
+ CRUSH rule 0 x 762 [2,8,3]
+ CRUSH rule 0 x 763 [8,3,1]
+ CRUSH rule 0 x 764 [1,7,4]
+ CRUSH rule 0 x 765 [6,4,0]
+ CRUSH rule 0 x 766 [8,3,0]
+ CRUSH rule 0 x 767 [1,8,5]
+ CRUSH rule 0 x 768 [8,5,0]
+ CRUSH rule 0 x 769 [6,2,4]
+ CRUSH rule 0 x 770 [6,1,4]
+ CRUSH rule 0 x 771 [7,2,4]
+ CRUSH rule 0 x 772 [8,4,2]
+ CRUSH rule 0 x 773 [3,2,6]
+ CRUSH rule 0 x 774 [4,7,1]
+ CRUSH rule 0 x 775 [6,5,0]
+ CRUSH rule 0 x 776 [7,0,3]
+ CRUSH rule 0 x 777 [3,0,6]
+ CRUSH rule 0 x 778 [1,6,3]
+ CRUSH rule 0 x 779 [2,6,5]
+ CRUSH rule 0 x 780 [0,5,8]
+ CRUSH rule 0 x 781 [6,5,0]
+ CRUSH rule 0 x 782 [5,2,8]
+ CRUSH rule 0 x 783 [7,0,3]
+ CRUSH rule 0 x 784 [0,5,8]
+ CRUSH rule 0 x 785 [6,1,3]
+ CRUSH rule 0 x 786 [7,3,0]
+ CRUSH rule 0 x 787 [1,8,5]
+ CRUSH rule 0 x 788 [6,0,4]
+ CRUSH rule 0 x 789 [0,5,8]
+ CRUSH rule 0 x 790 [8,3,1]
+ CRUSH rule 0 x 791 [3,6,2]
+ CRUSH rule 0 x 792 [5,6,1]
+ CRUSH rule 0 x 793 [6,2,3]
+ CRUSH rule 0 x 794 [2,8,3]
+ CRUSH rule 0 x 795 [0,4,6]
+ CRUSH rule 0 x 796 [3,7,1]
+ CRUSH rule 0 x 797 [2,3,6]
+ CRUSH rule 0 x 798 [6,0,3]
+ CRUSH rule 0 x 799 [5,2,7]
+ CRUSH rule 0 x 800 [5,2,8]
+ CRUSH rule 0 x 801 [3,7,2]
+ CRUSH rule 0 x 802 [1,6,4]
+ CRUSH rule 0 x 803 [0,4,7]
+ CRUSH rule 0 x 804 [6,0,5]
+ CRUSH rule 0 x 805 [3,8,0]
+ CRUSH rule 0 x 806 [1,5,6]
+ CRUSH rule 0 x 807 [5,7,2]
+ CRUSH rule 0 x 808 [4,7,2]
+ CRUSH rule 0 x 809 [1,3,6]
+ CRUSH rule 0 x 810 [5,7,2]
+ CRUSH rule 0 x 811 [8,4,1]
+ CRUSH rule 0 x 812 [8,3,1]
+ CRUSH rule 0 x 813 [6,4,2]
+ CRUSH rule 0 x 814 [3,8,0]
+ CRUSH rule 0 x 815 [3,1,6]
+ CRUSH rule 0 x 816 [2,6,5]
+ CRUSH rule 0 x 817 [4,6,2]
+ CRUSH rule 0 x 818 [3,1,7]
+ CRUSH rule 0 x 819 [5,0,7]
+ CRUSH rule 0 x 820 [3,7,1]
+ CRUSH rule 0 x 821 [4,8,0]
+ CRUSH rule 0 x 822 [2,3,7]
+ CRUSH rule 0 x 823 [4,7,2]
+ CRUSH rule 0 x 824 [3,7,1]
+ CRUSH rule 0 x 825 [2,8,4]
+ CRUSH rule 0 x 826 [7,1,5]
+ CRUSH rule 0 x 827 [0,6,3]
+ CRUSH rule 0 x 828 [2,5,8]
+ CRUSH rule 0 x 829 [5,6,0]
+ CRUSH rule 0 x 830 [2,4,7]
+ CRUSH rule 0 x 831 [1,6,4]
+ CRUSH rule 0 x 832 [4,8,2]
+ CRUSH rule 0 x 833 [2,6,5]
+ CRUSH rule 0 x 834 [3,0,8]
+ CRUSH rule 0 x 835 [8,5,0]
+ CRUSH rule 0 x 836 [3,8,1]
+ CRUSH rule 0 x 837 [6,4,0]
+ CRUSH rule 0 x 838 [6,0,4]
+ CRUSH rule 0 x 839 [5,2,8]
+ CRUSH rule 0 x 840 [7,3,2]
+ CRUSH rule 0 x 841 [4,8,0]
+ CRUSH rule 0 x 842 [2,5,8]
+ CRUSH rule 0 x 843 [6,4,2]
+ CRUSH rule 0 x 844 [4,8,2]
+ CRUSH rule 0 x 845 [3,6,2]
+ CRUSH rule 0 x 846 [3,0,6]
+ CRUSH rule 0 x 847 [0,8,5]
+ CRUSH rule 0 x 848 [2,6,5]
+ CRUSH rule 0 x 849 [4,8,1]
+ CRUSH rule 0 x 850 [1,5,7]
+ CRUSH rule 0 x 851 [6,5,1]
+ CRUSH rule 0 x 852 [7,4,2]
+ CRUSH rule 0 x 853 [6,2,5]
+ CRUSH rule 0 x 854 [7,0,3]
+ CRUSH rule 0 x 855 [5,7,2]
+ CRUSH rule 0 x 856 [6,3,1]
+ CRUSH rule 0 x 857 [8,4,0]
+ CRUSH rule 0 x 858 [6,5,2]
+ CRUSH rule 0 x 859 [6,0,5]
+ CRUSH rule 0 x 860 [4,2,8]
+ CRUSH rule 0 x 861 [8,4,1]
+ CRUSH rule 0 x 862 [6,0,5]
+ CRUSH rule 0 x 863 [8,1,5]
+ CRUSH rule 0 x 864 [5,6,1]
+ CRUSH rule 0 x 865 [8,0,3]
+ CRUSH rule 0 x 866 [3,6,2]
+ CRUSH rule 0 x 867 [6,3,1]
+ CRUSH rule 0 x 868 [6,4,2]
+ CRUSH rule 0 x 869 [8,4,2]
+ CRUSH rule 0 x 870 [0,5,7]
+ CRUSH rule 0 x 871 [3,1,6]
+ CRUSH rule 0 x 872 [5,0,6]
+ CRUSH rule 0 x 873 [4,6,1]
+ CRUSH rule 0 x 874 [2,6,4]
+ CRUSH rule 0 x 875 [2,6,5]
+ CRUSH rule 0 x 876 [5,8,1]
+ CRUSH rule 0 x 877 [6,4,0]
+ CRUSH rule 0 x 878 [5,0,7]
+ CRUSH rule 0 x 879 [7,3,1]
+ CRUSH rule 0 x 880 [3,1,7]
+ CRUSH rule 0 x 881 [5,8,2]
+ CRUSH rule 0 x 882 [4,1,6]
+ CRUSH rule 0 x 883 [2,5,6]
+ CRUSH rule 0 x 884 [6,0,3]
+ CRUSH rule 0 x 885 [5,1,6]
+ CRUSH rule 0 x 886 [3,7,2]
+ CRUSH rule 0 x 887 [7,5,2]
+ CRUSH rule 0 x 888 [6,1,4]
+ CRUSH rule 0 x 889 [2,8,3]
+ CRUSH rule 0 x 890 [7,1,4]
+ CRUSH rule 0 x 891 [1,7,4]
+ CRUSH rule 0 x 892 [6,0,5]
+ CRUSH rule 0 x 893 [2,3,6]
+ CRUSH rule 0 x 894 [7,4,1]
+ CRUSH rule 0 x 895 [5,1,6]
+ CRUSH rule 0 x 896 [1,6,3]
+ CRUSH rule 0 x 897 [4,0,8]
+ CRUSH rule 0 x 898 [0,3,8]
+ CRUSH rule 0 x 899 [1,6,4]
+ CRUSH rule 0 x 900 [4,0,7]
+ CRUSH rule 0 x 901 [5,2,6]
+ CRUSH rule 0 x 902 [8,4,2]
+ CRUSH rule 0 x 903 [5,6,1]
+ CRUSH rule 0 x 904 [5,7,0]
+ CRUSH rule 0 x 905 [6,0,3]
+ CRUSH rule 0 x 906 [1,7,4]
+ CRUSH rule 0 x 907 [7,2,3]
+ CRUSH rule 0 x 908 [5,6,1]
+ CRUSH rule 0 x 909 [2,3,8]
+ CRUSH rule 0 x 910 [6,5,0]
+ CRUSH rule 0 x 911 [5,7,1]
+ CRUSH rule 0 x 912 [0,8,5]
+ CRUSH rule 0 x 913 [7,1,4]
+ CRUSH rule 0 x 914 [6,4,0]
+ CRUSH rule 0 x 915 [8,2,3]
+ CRUSH rule 0 x 916 [3,0,6]
+ CRUSH rule 0 x 917 [1,3,6]
+ CRUSH rule 0 x 918 [8,0,5]
+ CRUSH rule 0 x 919 [6,2,3]
+ CRUSH rule 0 x 920 [7,4,2]
+ CRUSH rule 0 x 921 [1,4,8]
+ CRUSH rule 0 x 922 [6,3,1]
+ CRUSH rule 0 x 923 [5,6,0]
+ CRUSH rule 0 x 924 [3,1,8]
+ CRUSH rule 0 x 925 [5,6,0]
+ CRUSH rule 0 x 926 [3,0,7]
+ CRUSH rule 0 x 927 [1,6,5]
+ CRUSH rule 0 x 928 [8,1,4]
+ CRUSH rule 0 x 929 [4,2,6]
+ CRUSH rule 0 x 930 [2,5,8]
+ CRUSH rule 0 x 931 [5,2,8]
+ CRUSH rule 0 x 932 [4,2,7]
+ CRUSH rule 0 x 933 [8,4,2]
+ CRUSH rule 0 x 934 [5,8,0]
+ CRUSH rule 0 x 935 [6,3,0]
+ CRUSH rule 0 x 936 [0,7,3]
+ CRUSH rule 0 x 937 [5,8,1]
+ CRUSH rule 0 x 938 [6,4,0]
+ CRUSH rule 0 x 939 [2,8,3]
+ CRUSH rule 0 x 940 [8,5,2]
+ CRUSH rule 0 x 941 [5,0,6]
+ CRUSH rule 0 x 942 [1,6,4]
+ CRUSH rule 0 x 943 [8,2,4]
+ CRUSH rule 0 x 944 [4,8,1]
+ CRUSH rule 0 x 945 [7,0,3]
+ CRUSH rule 0 x 946 [2,8,3]
+ CRUSH rule 0 x 947 [4,2,6]
+ CRUSH rule 0 x 948 [7,4,1]
+ CRUSH rule 0 x 949 [6,2,5]
+ CRUSH rule 0 x 950 [3,7,0]
+ CRUSH rule 0 x 951 [4,6,2]
+ CRUSH rule 0 x 952 [2,7,4]
+ CRUSH rule 0 x 953 [1,3,6]
+ CRUSH rule 0 x 954 [4,0,8]
+ CRUSH rule 0 x 955 [8,1,4]
+ CRUSH rule 0 x 956 [1,7,4]
+ CRUSH rule 0 x 957 [7,0,3]
+ CRUSH rule 0 x 958 [8,3,2]
+ CRUSH rule 0 x 959 [5,1,6]
+ CRUSH rule 0 x 960 [3,6,2]
+ CRUSH rule 0 x 961 [4,1,6]
+ CRUSH rule 0 x 962 [7,5,0]
+ CRUSH rule 0 x 963 [0,5,8]
+ CRUSH rule 0 x 964 [3,2,7]
+ CRUSH rule 0 x 965 [7,3,1]
+ CRUSH rule 0 x 966 [3,6,0]
+ CRUSH rule 0 x 967 [8,4,2]
+ CRUSH rule 0 x 968 [7,0,4]
+ CRUSH rule 0 x 969 [8,0,4]
+ CRUSH rule 0 x 970 [0,8,3]
+ CRUSH rule 0 x 971 [1,8,5]
+ CRUSH rule 0 x 972 [1,7,5]
+ CRUSH rule 0 x 973 [1,8,3]
+ CRUSH rule 0 x 974 [5,1,7]
+ CRUSH rule 0 x 975 [3,8,0]
+ CRUSH rule 0 x 976 [4,7,2]
+ CRUSH rule 0 x 977 [8,3,2]
+ CRUSH rule 0 x 978 [7,0,3]
+ CRUSH rule 0 x 979 [7,2,5]
+ CRUSH rule 0 x 980 [6,2,3]
+ CRUSH rule 0 x 981 [7,5,1]
+ CRUSH rule 0 x 982 [4,1,8]
+ CRUSH rule 0 x 983 [3,6,0]
+ CRUSH rule 0 x 984 [0,8,4]
+ CRUSH rule 0 x 985 [2,4,8]
+ CRUSH rule 0 x 986 [8,4,2]
+ CRUSH rule 0 x 987 [0,4,8]
+ CRUSH rule 0 x 988 [1,4,6]
+ CRUSH rule 0 x 989 [0,8,5]
+ CRUSH rule 0 x 990 [1,6,4]
+ CRUSH rule 0 x 991 [0,4,8]
+ CRUSH rule 0 x 992 [7,0,4]
+ CRUSH rule 0 x 993 [0,6,4]
+ CRUSH rule 0 x 994 [3,6,1]
+ CRUSH rule 0 x 995 [7,0,4]
+ CRUSH rule 0 x 996 [6,5,1]
+ CRUSH rule 0 x 997 [6,3,1]
+ CRUSH rule 0 x 998 [8,0,4]
+ CRUSH rule 0 x 999 [0,7,5]
+ CRUSH rule 0 x 1000 [8,4,1]
+ CRUSH rule 0 x 1001 [2,3,6]
+ CRUSH rule 0 x 1002 [1,3,8]
+ CRUSH rule 0 x 1003 [2,7,3]
+ CRUSH rule 0 x 1004 [6,0,3]
+ CRUSH rule 0 x 1005 [6,1,4]
+ CRUSH rule 0 x 1006 [1,8,4]
+ CRUSH rule 0 x 1007 [1,3,8]
+ CRUSH rule 0 x 1008 [1,7,4]
+ CRUSH rule 0 x 1009 [6,5,1]
+ CRUSH rule 0 x 1010 [3,1,6]
+ CRUSH rule 0 x 1011 [3,0,8]
+ CRUSH rule 0 x 1012 [3,1,8]
+ CRUSH rule 0 x 1013 [5,2,8]
+ CRUSH rule 0 x 1014 [2,8,3]
+ CRUSH rule 0 x 1015 [6,3,2]
+ CRUSH rule 0 x 1016 [2,5,6]
+ CRUSH rule 0 x 1017 [6,1,5]
+ CRUSH rule 0 x 1018 [5,1,6]
+ CRUSH rule 0 x 1019 [5,8,1]
+ CRUSH rule 0 x 1020 [5,0,7]
+ CRUSH rule 0 x 1021 [5,2,7]
+ CRUSH rule 0 x 1022 [1,7,4]
+ CRUSH rule 0 x 1023 [3,0,7]
+ rule 0 (choose) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 1 (choose-two), x = 0..1023, numrep = 2..3
+ CRUSH rule 1 x 0 [0,2]
+ CRUSH rule 1 x 1 [0,8]
+ CRUSH rule 1 x 2 [1,3]
+ CRUSH rule 1 x 3 [8,0]
+ CRUSH rule 1 x 4 [5,4]
+ CRUSH rule 1 x 5 [7,8]
+ CRUSH rule 1 x 6 [2,6]
+ CRUSH rule 1 x 7 [5,4]
+ CRUSH rule 1 x 8 [5,3]
+ CRUSH rule 1 x 9 [2,3]
+ CRUSH rule 1 x 10 [0,2]
+ CRUSH rule 1 x 11 [0,7]
+ CRUSH rule 1 x 12 [0,2]
+ CRUSH rule 1 x 13 [3,8]
+ CRUSH rule 1 x 14 [7,6]
+ CRUSH rule 1 x 15 [7,2]
+ CRUSH rule 1 x 16 [3,6]
+ CRUSH rule 1 x 17 [5,4]
+ CRUSH rule 1 x 18 [1,4]
+ CRUSH rule 1 x 19 [7,5]
+ CRUSH rule 1 x 20 [2,4]
+ CRUSH rule 1 x 21 [3,7]
+ CRUSH rule 1 x 22 [8,3]
+ CRUSH rule 1 x 23 [3,6]
+ CRUSH rule 1 x 24 [1,0]
+ CRUSH rule 1 x 25 [3,7]
+ CRUSH rule 1 x 26 [2,8]
+ CRUSH rule 1 x 27 [3,1]
+ CRUSH rule 1 x 28 [6,0]
+ CRUSH rule 1 x 29 [8,5]
+ CRUSH rule 1 x 30 [5,7]
+ CRUSH rule 1 x 31 [8,7]
+ CRUSH rule 1 x 32 [3,6]
+ CRUSH rule 1 x 33 [2,7]
+ CRUSH rule 1 x 34 [2,5]
+ CRUSH rule 1 x 35 [0,8]
+ CRUSH rule 1 x 36 [3,8]
+ CRUSH rule 1 x 37 [0,4]
+ CRUSH rule 1 x 38 [4,8]
+ CRUSH rule 1 x 39 [3,7]
+ CRUSH rule 1 x 40 [7,8]
+ CRUSH rule 1 x 41 [0,2]
+ CRUSH rule 1 x 42 [4,3]
+ CRUSH rule 1 x 43 [0,3]
+ CRUSH rule 1 x 44 [1,6]
+ CRUSH rule 1 x 45 [8,0]
+ CRUSH rule 1 x 46 [2,4]
+ CRUSH rule 1 x 47 [4,5]
+ CRUSH rule 1 x 48 [4,6]
+ CRUSH rule 1 x 49 [5,4]
+ CRUSH rule 1 x 50 [3,4]
+ CRUSH rule 1 x 51 [3,6]
+ CRUSH rule 1 x 52 [8,6]
+ CRUSH rule 1 x 53 [3,5]
+ CRUSH rule 1 x 54 [7,6]
+ CRUSH rule 1 x 55 [8,7]
+ CRUSH rule 1 x 56 [6,4]
+ CRUSH rule 1 x 57 [5,3]
+ CRUSH rule 1 x 58 [1,0]
+ CRUSH rule 1 x 59 [4,2]
+ CRUSH rule 1 x 60 [3,5]
+ CRUSH rule 1 x 61 [4,6]
+ CRUSH rule 1 x 62 [7,0]
+ CRUSH rule 1 x 63 [5,6]
+ CRUSH rule 1 x 64 [4,5]
+ CRUSH rule 1 x 65 [7,3]
+ CRUSH rule 1 x 66 [5,4]
+ CRUSH rule 1 x 67 [5,0]
+ CRUSH rule 1 x 68 [0,5]
+ CRUSH rule 1 x 69 [5,1]
+ CRUSH rule 1 x 70 [7,0]
+ CRUSH rule 1 x 71 [2,8]
+ CRUSH rule 1 x 72 [6,1]
+ CRUSH rule 1 x 73 [2,7]
+ CRUSH rule 1 x 74 [0,7]
+ CRUSH rule 1 x 75 [3,2]
+ CRUSH rule 1 x 76 [5,1]
+ CRUSH rule 1 x 77 [7,2]
+ CRUSH rule 1 x 78 [1,4]
+ CRUSH rule 1 x 79 [5,4]
+ CRUSH rule 1 x 80 [0,3]
+ CRUSH rule 1 x 81 [0,2]
+ CRUSH rule 1 x 82 [7,1]
+ CRUSH rule 1 x 83 [2,6]
+ CRUSH rule 1 x 84 [7,2]
+ CRUSH rule 1 x 85 [3,4]
+ CRUSH rule 1 x 86 [0,2]
+ CRUSH rule 1 x 87 [0,7]
+ CRUSH rule 1 x 88 [1,6]
+ CRUSH rule 1 x 89 [3,0]
+ CRUSH rule 1 x 90 [6,7]
+ CRUSH rule 1 x 91 [3,8]
+ CRUSH rule 1 x 92 [1,8]
+ CRUSH rule 1 x 93 [7,4]
+ CRUSH rule 1 x 94 [0,4]
+ CRUSH rule 1 x 95 [7,5]
+ CRUSH rule 1 x 96 [3,6]
+ CRUSH rule 1 x 97 [8,7]
+ CRUSH rule 1 x 98 [2,0]
+ CRUSH rule 1 x 99 [0,7]
+ CRUSH rule 1 x 100 [1,7]
+ CRUSH rule 1 x 101 [3,7]
+ CRUSH rule 1 x 102 [4,2]
+ CRUSH rule 1 x 103 [4,7]
+ CRUSH rule 1 x 104 [7,4]
+ CRUSH rule 1 x 105 [2,4]
+ CRUSH rule 1 x 106 [1,6]
+ CRUSH rule 1 x 107 [3,2]
+ CRUSH rule 1 x 108 [7,2]
+ CRUSH rule 1 x 109 [1,2]
+ CRUSH rule 1 x 110 [3,2]
+ CRUSH rule 1 x 111 [2,1]
+ CRUSH rule 1 x 112 [2,0]
+ CRUSH rule 1 x 113 [6,2]
+ CRUSH rule 1 x 114 [7,6]
+ CRUSH rule 1 x 115 [8,2]
+ CRUSH rule 1 x 116 [1,2]
+ CRUSH rule 1 x 117 [7,3]
+ CRUSH rule 1 x 118 [0,3]
+ CRUSH rule 1 x 119 [5,6]
+ CRUSH rule 1 x 120 [0,2]
+ CRUSH rule 1 x 121 [2,0]
+ CRUSH rule 1 x 122 [8,5]
+ CRUSH rule 1 x 123 [2,5]
+ CRUSH rule 1 x 124 [3,5]
+ CRUSH rule 1 x 125 [0,7]
+ CRUSH rule 1 x 126 [4,2]
+ CRUSH rule 1 x 127 [3,6]
+ CRUSH rule 1 x 128 [3,5]
+ CRUSH rule 1 x 129 [0,2]
+ CRUSH rule 1 x 130 [3,8]
+ CRUSH rule 1 x 131 [1,2]
+ CRUSH rule 1 x 132 [1,2]
+ CRUSH rule 1 x 133 [3,6]
+ CRUSH rule 1 x 134 [1,8]
+ CRUSH rule 1 x 135 [5,6]
+ CRUSH rule 1 x 136 [2,1]
+ CRUSH rule 1 x 137 [7,3]
+ CRUSH rule 1 x 138 [8,7]
+ CRUSH rule 1 x 139 [3,0]
+ CRUSH rule 1 x 140 [1,6]
+ CRUSH rule 1 x 141 [6,8]
+ CRUSH rule 1 x 142 [3,5]
+ CRUSH rule 1 x 143 [5,8]
+ CRUSH rule 1 x 144 [8,1]
+ CRUSH rule 1 x 145 [8,5]
+ CRUSH rule 1 x 146 [2,6]
+ CRUSH rule 1 x 147 [2,8]
+ CRUSH rule 1 x 148 [3,4]
+ CRUSH rule 1 x 149 [4,8]
+ CRUSH rule 1 x 150 [1,6]
+ CRUSH rule 1 x 151 [3,4]
+ CRUSH rule 1 x 152 [8,4]
+ CRUSH rule 1 x 153 [8,6]
+ CRUSH rule 1 x 154 [3,2]
+ CRUSH rule 1 x 155 [3,5]
+ CRUSH rule 1 x 156 [4,5]
+ CRUSH rule 1 x 157 [4,1]
+ CRUSH rule 1 x 158 [2,8]
+ CRUSH rule 1 x 159 [7,0]
+ CRUSH rule 1 x 160 [2,8]
+ CRUSH rule 1 x 161 [1,5]
+ CRUSH rule 1 x 162 [0,6]
+ CRUSH rule 1 x 163 [5,6]
+ CRUSH rule 1 x 164 [7,8]
+ CRUSH rule 1 x 165 [7,0]
+ CRUSH rule 1 x 166 [2,4]
+ CRUSH rule 1 x 167 [0,1]
+ CRUSH rule 1 x 168 [4,2]
+ CRUSH rule 1 x 169 [2,6]
+ CRUSH rule 1 x 170 [1,0]
+ CRUSH rule 1 x 171 [7,5]
+ CRUSH rule 1 x 172 [0,7]
+ CRUSH rule 1 x 173 [8,5]
+ CRUSH rule 1 x 174 [1,4]
+ CRUSH rule 1 x 175 [6,0]
+ CRUSH rule 1 x 176 [4,3]
+ CRUSH rule 1 x 177 [5,3]
+ CRUSH rule 1 x 178 [3,0]
+ CRUSH rule 1 x 179 [4,1]
+ CRUSH rule 1 x 180 [3,8]
+ CRUSH rule 1 x 181 [6,2]
+ CRUSH rule 1 x 182 [8,5]
+ CRUSH rule 1 x 183 [7,8]
+ CRUSH rule 1 x 184 [5,7]
+ CRUSH rule 1 x 185 [6,8]
+ CRUSH rule 1 x 186 [2,1]
+ CRUSH rule 1 x 187 [1,6]
+ CRUSH rule 1 x 188 [1,8]
+ CRUSH rule 1 x 189 [0,7]
+ CRUSH rule 1 x 190 [4,0]
+ CRUSH rule 1 x 191 [7,6]
+ CRUSH rule 1 x 192 [5,0]
+ CRUSH rule 1 x 193 [4,2]
+ CRUSH rule 1 x 194 [1,3]
+ CRUSH rule 1 x 195 [6,4]
+ CRUSH rule 1 x 196 [6,7]
+ CRUSH rule 1 x 197 [6,5]
+ CRUSH rule 1 x 198 [2,5]
+ CRUSH rule 1 x 199 [0,5]
+ CRUSH rule 1 x 200 [0,5]
+ CRUSH rule 1 x 201 [7,1]
+ CRUSH rule 1 x 202 [6,3]
+ CRUSH rule 1 x 203 [4,8]
+ CRUSH rule 1 x 204 [2,1]
+ CRUSH rule 1 x 205 [0,7]
+ CRUSH rule 1 x 206 [0,1]
+ CRUSH rule 1 x 207 [3,0]
+ CRUSH rule 1 x 208 [7,1]
+ CRUSH rule 1 x 209 [1,2]
+ CRUSH rule 1 x 210 [1,2]
+ CRUSH rule 1 x 211 [5,4]
+ CRUSH rule 1 x 212 [7,5]
+ CRUSH rule 1 x 213 [8,4]
+ CRUSH rule 1 x 214 [4,5]
+ CRUSH rule 1 x 215 [8,1]
+ CRUSH rule 1 x 216 [5,0]
+ CRUSH rule 1 x 217 [0,1]
+ CRUSH rule 1 x 218 [0,7]
+ CRUSH rule 1 x 219 [4,8]
+ CRUSH rule 1 x 220 [5,7]
+ CRUSH rule 1 x 221 [3,6]
+ CRUSH rule 1 x 222 [6,8]
+ CRUSH rule 1 x 223 [1,3]
+ CRUSH rule 1 x 224 [1,5]
+ CRUSH rule 1 x 225 [8,6]
+ CRUSH rule 1 x 226 [7,2]
+ CRUSH rule 1 x 227 [3,4]
+ CRUSH rule 1 x 228 [5,4]
+ CRUSH rule 1 x 229 [3,4]
+ CRUSH rule 1 x 230 [4,7]
+ CRUSH rule 1 x 231 [4,3]
+ CRUSH rule 1 x 232 [2,7]
+ CRUSH rule 1 x 233 [3,4]
+ CRUSH rule 1 x 234 [0,1]
+ CRUSH rule 1 x 235 [3,8]
+ CRUSH rule 1 x 236 [5,2]
+ CRUSH rule 1 x 237 [4,7]
+ CRUSH rule 1 x 238 [4,3]
+ CRUSH rule 1 x 239 [8,7]
+ CRUSH rule 1 x 240 [5,7]
+ CRUSH rule 1 x 241 [3,1]
+ CRUSH rule 1 x 242 [3,5]
+ CRUSH rule 1 x 243 [4,7]
+ CRUSH rule 1 x 244 [4,6]
+ CRUSH rule 1 x 245 [7,6]
+ CRUSH rule 1 x 246 [1,5]
+ CRUSH rule 1 x 247 [6,0]
+ CRUSH rule 1 x 248 [8,0]
+ CRUSH rule 1 x 249 [2,0]
+ CRUSH rule 1 x 250 [2,1]
+ CRUSH rule 1 x 251 [2,3]
+ CRUSH rule 1 x 252 [3,7]
+ CRUSH rule 1 x 253 [3,2]
+ CRUSH rule 1 x 254 [3,5]
+ CRUSH rule 1 x 255 [1,7]
+ CRUSH rule 1 x 256 [5,7]
+ CRUSH rule 1 x 257 [2,8]
+ CRUSH rule 1 x 258 [5,3]
+ CRUSH rule 1 x 259 [4,3]
+ CRUSH rule 1 x 260 [3,6]
+ CRUSH rule 1 x 261 [8,7]
+ CRUSH rule 1 x 262 [5,4]
+ CRUSH rule 1 x 263 [6,8]
+ CRUSH rule 1 x 264 [3,6]
+ CRUSH rule 1 x 265 [8,6]
+ CRUSH rule 1 x 266 [8,2]
+ CRUSH rule 1 x 267 [2,3]
+ CRUSH rule 1 x 268 [0,7]
+ CRUSH rule 1 x 269 [0,8]
+ CRUSH rule 1 x 270 [5,0]
+ CRUSH rule 1 x 271 [7,5]
+ CRUSH rule 1 x 272 [2,8]
+ CRUSH rule 1 x 273 [3,5]
+ CRUSH rule 1 x 274 [6,8]
+ CRUSH rule 1 x 275 [4,3]
+ CRUSH rule 1 x 276 [7,1]
+ CRUSH rule 1 x 277 [6,4]
+ CRUSH rule 1 x 278 [6,8]
+ CRUSH rule 1 x 279 [8,3]
+ CRUSH rule 1 x 280 [0,6]
+ CRUSH rule 1 x 281 [8,0]
+ CRUSH rule 1 x 282 [3,1]
+ CRUSH rule 1 x 283 [8,2]
+ CRUSH rule 1 x 284 [6,3]
+ CRUSH rule 1 x 285 [5,3]
+ CRUSH rule 1 x 286 [2,1]
+ CRUSH rule 1 x 287 [0,4]
+ CRUSH rule 1 x 288 [8,0]
+ CRUSH rule 1 x 289 [4,6]
+ CRUSH rule 1 x 290 [1,3]
+ CRUSH rule 1 x 291 [0,1]
+ CRUSH rule 1 x 292 [8,0]
+ CRUSH rule 1 x 293 [6,0]
+ CRUSH rule 1 x 294 [7,4]
+ CRUSH rule 1 x 295 [4,8]
+ CRUSH rule 1 x 296 [3,1]
+ CRUSH rule 1 x 297 [6,2]
+ CRUSH rule 1 x 298 [1,2]
+ CRUSH rule 1 x 299 [2,0]
+ CRUSH rule 1 x 300 [8,7]
+ CRUSH rule 1 x 301 [0,8]
+ CRUSH rule 1 x 302 [3,0]
+ CRUSH rule 1 x 303 [7,5]
+ CRUSH rule 1 x 304 [2,7]
+ CRUSH rule 1 x 305 [5,8]
+ CRUSH rule 1 x 306 [0,7]
+ CRUSH rule 1 x 307 [0,2]
+ CRUSH rule 1 x 308 [0,8]
+ CRUSH rule 1 x 309 [7,4]
+ CRUSH rule 1 x 310 [4,3]
+ CRUSH rule 1 x 311 [3,4]
+ CRUSH rule 1 x 312 [2,1]
+ CRUSH rule 1 x 313 [5,3]
+ CRUSH rule 1 x 314 [4,5]
+ CRUSH rule 1 x 315 [2,0]
+ CRUSH rule 1 x 316 [6,3]
+ CRUSH rule 1 x 317 [2,6]
+ CRUSH rule 1 x 318 [8,6]
+ CRUSH rule 1 x 319 [5,0]
+ CRUSH rule 1 x 320 [3,7]
+ CRUSH rule 1 x 321 [1,3]
+ CRUSH rule 1 x 322 [2,7]
+ CRUSH rule 1 x 323 [4,7]
+ CRUSH rule 1 x 324 [7,0]
+ CRUSH rule 1 x 325 [4,6]
+ CRUSH rule 1 x 326 [3,4]
+ CRUSH rule 1 x 327 [0,6]
+ CRUSH rule 1 x 328 [7,4]
+ CRUSH rule 1 x 329 [5,6]
+ CRUSH rule 1 x 330 [3,7]
+ CRUSH rule 1 x 331 [2,6]
+ CRUSH rule 1 x 332 [2,0]
+ CRUSH rule 1 x 333 [6,8]
+ CRUSH rule 1 x 334 [8,3]
+ CRUSH rule 1 x 335 [7,1]
+ CRUSH rule 1 x 336 [4,6]
+ CRUSH rule 1 x 337 [7,2]
+ CRUSH rule 1 x 338 [5,6]
+ CRUSH rule 1 x 339 [7,5]
+ CRUSH rule 1 x 340 [2,0]
+ CRUSH rule 1 x 341 [5,1]
+ CRUSH rule 1 x 342 [0,7]
+ CRUSH rule 1 x 343 [6,7]
+ CRUSH rule 1 x 344 [6,0]
+ CRUSH rule 1 x 345 [4,3]
+ CRUSH rule 1 x 346 [8,0]
+ CRUSH rule 1 x 347 [3,1]
+ CRUSH rule 1 x 348 [8,0]
+ CRUSH rule 1 x 349 [1,6]
+ CRUSH rule 1 x 350 [8,5]
+ CRUSH rule 1 x 351 [3,6]
+ CRUSH rule 1 x 352 [1,0]
+ CRUSH rule 1 x 353 [6,4]
+ CRUSH rule 1 x 354 [0,3]
+ CRUSH rule 1 x 355 [3,4]
+ CRUSH rule 1 x 356 [3,5]
+ CRUSH rule 1 x 357 [6,1]
+ CRUSH rule 1 x 358 [2,1]
+ CRUSH rule 1 x 359 [6,7]
+ CRUSH rule 1 x 360 [5,3]
+ CRUSH rule 1 x 361 [8,4]
+ CRUSH rule 1 x 362 [4,5]
+ CRUSH rule 1 x 363 [4,0]
+ CRUSH rule 1 x 364 [2,5]
+ CRUSH rule 1 x 365 [6,7]
+ CRUSH rule 1 x 366 [7,2]
+ CRUSH rule 1 x 367 [4,5]
+ CRUSH rule 1 x 368 [7,4]
+ CRUSH rule 1 x 369 [3,7]
+ CRUSH rule 1 x 370 [8,7]
+ CRUSH rule 1 x 371 [1,3]
+ CRUSH rule 1 x 372 [3,1]
+ CRUSH rule 1 x 373 [0,6]
+ CRUSH rule 1 x 374 [3,8]
+ CRUSH rule 1 x 375 [6,4]
+ CRUSH rule 1 x 376 [7,1]
+ CRUSH rule 1 x 377 [1,2]
+ CRUSH rule 1 x 378 [0,1]
+ CRUSH rule 1 x 379 [8,5]
+ CRUSH rule 1 x 380 [2,5]
+ CRUSH rule 1 x 381 [0,4]
+ CRUSH rule 1 x 382 [1,5]
+ CRUSH rule 1 x 383 [4,3]
+ CRUSH rule 1 x 384 [7,0]
+ CRUSH rule 1 x 385 [7,4]
+ CRUSH rule 1 x 386 [0,3]
+ CRUSH rule 1 x 387 [1,3]
+ CRUSH rule 1 x 388 [5,0]
+ CRUSH rule 1 x 389 [1,5]
+ CRUSH rule 1 x 390 [5,6]
+ CRUSH rule 1 x 391 [5,6]
+ CRUSH rule 1 x 392 [1,8]
+ CRUSH rule 1 x 393 [4,2]
+ CRUSH rule 1 x 394 [4,7]
+ CRUSH rule 1 x 395 [4,0]
+ CRUSH rule 1 x 396 [4,2]
+ CRUSH rule 1 x 397 [2,1]
+ CRUSH rule 1 x 398 [2,4]
+ CRUSH rule 1 x 399 [8,7]
+ CRUSH rule 1 x 400 [8,1]
+ CRUSH rule 1 x 401 [0,1]
+ CRUSH rule 1 x 402 [7,8]
+ CRUSH rule 1 x 403 [0,1]
+ CRUSH rule 1 x 404 [4,3]
+ CRUSH rule 1 x 405 [6,5]
+ CRUSH rule 1 x 406 [2,0]
+ CRUSH rule 1 x 407 [2,8]
+ CRUSH rule 1 x 408 [4,1]
+ CRUSH rule 1 x 409 [7,3]
+ CRUSH rule 1 x 410 [8,6]
+ CRUSH rule 1 x 411 [2,0]
+ CRUSH rule 1 x 412 [0,5]
+ CRUSH rule 1 x 413 [5,0]
+ CRUSH rule 1 x 414 [4,1]
+ CRUSH rule 1 x 415 [0,6]
+ CRUSH rule 1 x 416 [2,1]
+ CRUSH rule 1 x 417 [8,7]
+ CRUSH rule 1 x 418 [7,6]
+ CRUSH rule 1 x 419 [8,3]
+ CRUSH rule 1 x 420 [1,4]
+ CRUSH rule 1 x 421 [8,6]
+ CRUSH rule 1 x 422 [6,7]
+ CRUSH rule 1 x 423 [0,5]
+ CRUSH rule 1 x 424 [8,4]
+ CRUSH rule 1 x 425 [1,3]
+ CRUSH rule 1 x 426 [6,7]
+ CRUSH rule 1 x 427 [0,7]
+ CRUSH rule 1 x 428 [5,4]
+ CRUSH rule 1 x 429 [4,6]
+ CRUSH rule 1 x 430 [3,6]
+ CRUSH rule 1 x 431 [5,3]
+ CRUSH rule 1 x 432 [7,1]
+ CRUSH rule 1 x 433 [6,5]
+ CRUSH rule 1 x 434 [5,2]
+ CRUSH rule 1 x 435 [0,5]
+ CRUSH rule 1 x 436 [4,0]
+ CRUSH rule 1 x 437 [7,5]
+ CRUSH rule 1 x 438 [0,3]
+ CRUSH rule 1 x 439 [1,3]
+ CRUSH rule 1 x 440 [2,7]
+ CRUSH rule 1 x 441 [5,7]
+ CRUSH rule 1 x 442 [2,4]
+ CRUSH rule 1 x 443 [6,8]
+ CRUSH rule 1 x 444 [7,0]
+ CRUSH rule 1 x 445 [6,3]
+ CRUSH rule 1 x 446 [4,3]
+ CRUSH rule 1 x 447 [2,1]
+ CRUSH rule 1 x 448 [7,2]
+ CRUSH rule 1 x 449 [7,8]
+ CRUSH rule 1 x 450 [4,5]
+ CRUSH rule 1 x 451 [6,8]
+ CRUSH rule 1 x 452 [8,3]
+ CRUSH rule 1 x 453 [6,8]
+ CRUSH rule 1 x 454 [6,7]
+ CRUSH rule 1 x 455 [2,7]
+ CRUSH rule 1 x 456 [6,8]
+ CRUSH rule 1 x 457 [7,2]
+ CRUSH rule 1 x 458 [2,8]
+ CRUSH rule 1 x 459 [2,0]
+ CRUSH rule 1 x 460 [6,5]
+ CRUSH rule 1 x 461 [6,5]
+ CRUSH rule 1 x 462 [8,1]
+ CRUSH rule 1 x 463 [6,7]
+ CRUSH rule 1 x 464 [7,4]
+ CRUSH rule 1 x 465 [7,6]
+ CRUSH rule 1 x 466 [5,8]
+ CRUSH rule 1 x 467 [6,4]
+ CRUSH rule 1 x 468 [7,8]
+ CRUSH rule 1 x 469 [7,0]
+ CRUSH rule 1 x 470 [3,0]
+ CRUSH rule 1 x 471 [0,1]
+ CRUSH rule 1 x 472 [5,4]
+ CRUSH rule 1 x 473 [1,0]
+ CRUSH rule 1 x 474 [6,0]
+ CRUSH rule 1 x 475 [6,7]
+ CRUSH rule 1 x 476 [4,3]
+ CRUSH rule 1 x 477 [5,8]
+ CRUSH rule 1 x 478 [6,7]
+ CRUSH rule 1 x 479 [0,5]
+ CRUSH rule 1 x 480 [1,8]
+ CRUSH rule 1 x 481 [2,4]
+ CRUSH rule 1 x 482 [4,3]
+ CRUSH rule 1 x 483 [0,2]
+ CRUSH rule 1 x 484 [1,2]
+ CRUSH rule 1 x 485 [4,7]
+ CRUSH rule 1 x 486 [4,1]
+ CRUSH rule 1 x 487 [5,0]
+ CRUSH rule 1 x 488 [5,7]
+ CRUSH rule 1 x 489 [2,8]
+ CRUSH rule 1 x 490 [6,4]
+ CRUSH rule 1 x 491 [1,0]
+ CRUSH rule 1 x 492 [6,5]
+ CRUSH rule 1 x 493 [0,2]
+ CRUSH rule 1 x 494 [1,0]
+ CRUSH rule 1 x 495 [3,4]
+ CRUSH rule 1 x 496 [7,5]
+ CRUSH rule 1 x 497 [5,7]
+ CRUSH rule 1 x 498 [0,5]
+ CRUSH rule 1 x 499 [8,4]
+ CRUSH rule 1 x 500 [3,6]
+ CRUSH rule 1 x 501 [0,7]
+ CRUSH rule 1 x 502 [7,1]
+ CRUSH rule 1 x 503 [2,3]
+ CRUSH rule 1 x 504 [5,6]
+ CRUSH rule 1 x 505 [0,7]
+ CRUSH rule 1 x 506 [5,3]
+ CRUSH rule 1 x 507 [6,0]
+ CRUSH rule 1 x 508 [0,1]
+ CRUSH rule 1 x 509 [7,5]
+ CRUSH rule 1 x 510 [6,0]
+ CRUSH rule 1 x 511 [5,8]
+ CRUSH rule 1 x 512 [7,6]
+ CRUSH rule 1 x 513 [7,2]
+ CRUSH rule 1 x 514 [4,3]
+ CRUSH rule 1 x 515 [8,5]
+ CRUSH rule 1 x 516 [4,0]
+ CRUSH rule 1 x 517 [7,8]
+ CRUSH rule 1 x 518 [4,6]
+ CRUSH rule 1 x 519 [7,3]
+ CRUSH rule 1 x 520 [2,6]
+ CRUSH rule 1 x 521 [8,7]
+ CRUSH rule 1 x 522 [6,8]
+ CRUSH rule 1 x 523 [4,2]
+ CRUSH rule 1 x 524 [0,4]
+ CRUSH rule 1 x 525 [0,4]
+ CRUSH rule 1 x 526 [1,5]
+ CRUSH rule 1 x 527 [0,2]
+ CRUSH rule 1 x 528 [5,3]
+ CRUSH rule 1 x 529 [5,7]
+ CRUSH rule 1 x 530 [6,7]
+ CRUSH rule 1 x 531 [6,1]
+ CRUSH rule 1 x 532 [6,3]
+ CRUSH rule 1 x 533 [5,6]
+ CRUSH rule 1 x 534 [7,3]
+ CRUSH rule 1 x 535 [8,6]
+ CRUSH rule 1 x 536 [6,7]
+ CRUSH rule 1 x 537 [3,7]
+ CRUSH rule 1 x 538 [6,8]
+ CRUSH rule 1 x 539 [8,3]
+ CRUSH rule 1 x 540 [0,6]
+ CRUSH rule 1 x 541 [2,3]
+ CRUSH rule 1 x 542 [3,5]
+ CRUSH rule 1 x 543 [6,0]
+ CRUSH rule 1 x 544 [3,7]
+ CRUSH rule 1 x 545 [5,7]
+ CRUSH rule 1 x 546 [6,1]
+ CRUSH rule 1 x 547 [8,2]
+ CRUSH rule 1 x 548 [5,2]
+ CRUSH rule 1 x 549 [5,8]
+ CRUSH rule 1 x 550 [0,5]
+ CRUSH rule 1 x 551 [7,5]
+ CRUSH rule 1 x 552 [5,4]
+ CRUSH rule 1 x 553 [4,2]
+ CRUSH rule 1 x 554 [0,8]
+ CRUSH rule 1 x 555 [5,0]
+ CRUSH rule 1 x 556 [3,4]
+ CRUSH rule 1 x 557 [7,4]
+ CRUSH rule 1 x 558 [3,1]
+ CRUSH rule 1 x 559 [4,2]
+ CRUSH rule 1 x 560 [8,3]
+ CRUSH rule 1 x 561 [6,3]
+ CRUSH rule 1 x 562 [3,4]
+ CRUSH rule 1 x 563 [2,6]
+ CRUSH rule 1 x 564 [5,1]
+ CRUSH rule 1 x 565 [3,6]
+ CRUSH rule 1 x 566 [4,7]
+ CRUSH rule 1 x 567 [3,6]
+ CRUSH rule 1 x 568 [7,4]
+ CRUSH rule 1 x 569 [3,4]
+ CRUSH rule 1 x 570 [1,5]
+ CRUSH rule 1 x 571 [3,7]
+ CRUSH rule 1 x 572 [3,4]
+ CRUSH rule 1 x 573 [3,0]
+ CRUSH rule 1 x 574 [2,0]
+ CRUSH rule 1 x 575 [8,6]
+ CRUSH rule 1 x 576 [4,6]
+ CRUSH rule 1 x 577 [8,2]
+ CRUSH rule 1 x 578 [6,8]
+ CRUSH rule 1 x 579 [3,1]
+ CRUSH rule 1 x 580 [3,0]
+ CRUSH rule 1 x 581 [7,2]
+ CRUSH rule 1 x 582 [2,8]
+ CRUSH rule 1 x 583 [6,0]
+ CRUSH rule 1 x 584 [8,1]
+ CRUSH rule 1 x 585 [7,0]
+ CRUSH rule 1 x 586 [0,1]
+ CRUSH rule 1 x 587 [2,5]
+ CRUSH rule 1 x 588 [3,4]
+ CRUSH rule 1 x 589 [7,1]
+ CRUSH rule 1 x 590 [6,2]
+ CRUSH rule 1 x 591 [5,2]
+ CRUSH rule 1 x 592 [2,0]
+ CRUSH rule 1 x 593 [0,8]
+ CRUSH rule 1 x 594 [0,7]
+ CRUSH rule 1 x 595 [7,1]
+ CRUSH rule 1 x 596 [4,3]
+ CRUSH rule 1 x 597 [3,1]
+ CRUSH rule 1 x 598 [3,2]
+ CRUSH rule 1 x 599 [5,2]
+ CRUSH rule 1 x 600 [7,0]
+ CRUSH rule 1 x 601 [0,7]
+ CRUSH rule 1 x 602 [3,7]
+ CRUSH rule 1 x 603 [5,1]
+ CRUSH rule 1 x 604 [7,5]
+ CRUSH rule 1 x 605 [3,0]
+ CRUSH rule 1 x 606 [2,0]
+ CRUSH rule 1 x 607 [0,4]
+ CRUSH rule 1 x 608 [5,3]
+ CRUSH rule 1 x 609 [5,2]
+ CRUSH rule 1 x 610 [3,7]
+ CRUSH rule 1 x 611 [1,0]
+ CRUSH rule 1 x 612 [2,0]
+ CRUSH rule 1 x 613 [7,2]
+ CRUSH rule 1 x 614 [7,8]
+ CRUSH rule 1 x 615 [6,8]
+ CRUSH rule 1 x 616 [0,8]
+ CRUSH rule 1 x 617 [6,1]
+ CRUSH rule 1 x 618 [7,6]
+ CRUSH rule 1 x 619 [5,1]
+ CRUSH rule 1 x 620 [4,1]
+ CRUSH rule 1 x 621 [5,8]
+ CRUSH rule 1 x 622 [0,4]
+ CRUSH rule 1 x 623 [0,6]
+ CRUSH rule 1 x 624 [3,5]
+ CRUSH rule 1 x 625 [2,3]
+ CRUSH rule 1 x 626 [7,8]
+ CRUSH rule 1 x 627 [2,7]
+ CRUSH rule 1 x 628 [8,0]
+ CRUSH rule 1 x 629 [2,6]
+ CRUSH rule 1 x 630 [2,7]
+ CRUSH rule 1 x 631 [0,7]
+ CRUSH rule 1 x 632 [7,0]
+ CRUSH rule 1 x 633 [8,6]
+ CRUSH rule 1 x 634 [0,4]
+ CRUSH rule 1 x 635 [5,6]
+ CRUSH rule 1 x 636 [1,4]
+ CRUSH rule 1 x 637 [4,1]
+ CRUSH rule 1 x 638 [6,8]
+ CRUSH rule 1 x 639 [4,2]
+ CRUSH rule 1 x 640 [3,1]
+ CRUSH rule 1 x 641 [7,2]
+ CRUSH rule 1 x 642 [2,0]
+ CRUSH rule 1 x 643 [3,5]
+ CRUSH rule 1 x 644 [8,1]
+ CRUSH rule 1 x 645 [5,4]
+ CRUSH rule 1 x 646 [8,0]
+ CRUSH rule 1 x 647 [7,1]
+ CRUSH rule 1 x 648 [0,6]
+ CRUSH rule 1 x 649 [4,7]
+ CRUSH rule 1 x 650 [7,8]
+ CRUSH rule 1 x 651 [3,7]
+ CRUSH rule 1 x 652 [3,4]
+ CRUSH rule 1 x 653 [8,5]
+ CRUSH rule 1 x 654 [7,5]
+ CRUSH rule 1 x 655 [0,3]
+ CRUSH rule 1 x 656 [4,3]
+ CRUSH rule 1 x 657 [6,1]
+ CRUSH rule 1 x 658 [5,4]
+ CRUSH rule 1 x 659 [4,6]
+ CRUSH rule 1 x 660 [7,8]
+ CRUSH rule 1 x 661 [1,8]
+ CRUSH rule 1 x 662 [4,5]
+ CRUSH rule 1 x 663 [1,3]
+ CRUSH rule 1 x 664 [1,4]
+ CRUSH rule 1 x 665 [5,7]
+ CRUSH rule 1 x 666 [2,8]
+ CRUSH rule 1 x 667 [1,3]
+ CRUSH rule 1 x 668 [3,7]
+ CRUSH rule 1 x 669 [6,4]
+ CRUSH rule 1 x 670 [4,0]
+ CRUSH rule 1 x 671 [0,2]
+ CRUSH rule 1 x 672 [4,3]
+ CRUSH rule 1 x 673 [5,3]
+ CRUSH rule 1 x 674 [3,1]
+ CRUSH rule 1 x 675 [0,8]
+ CRUSH rule 1 x 676 [0,2]
+ CRUSH rule 1 x 677 [4,1]
+ CRUSH rule 1 x 678 [2,3]
+ CRUSH rule 1 x 679 [6,0]
+ CRUSH rule 1 x 680 [0,4]
+ CRUSH rule 1 x 681 [4,7]
+ CRUSH rule 1 x 682 [0,5]
+ CRUSH rule 1 x 683 [0,1]
+ CRUSH rule 1 x 684 [7,1]
+ CRUSH rule 1 x 685 [7,1]
+ CRUSH rule 1 x 686 [1,4]
+ CRUSH rule 1 x 687 [3,5]
+ CRUSH rule 1 x 688 [5,7]
+ CRUSH rule 1 x 689 [6,5]
+ CRUSH rule 1 x 690 [8,1]
+ CRUSH rule 1 x 691 [3,0]
+ CRUSH rule 1 x 692 [7,2]
+ CRUSH rule 1 x 693 [6,7]
+ CRUSH rule 1 x 694 [6,5]
+ CRUSH rule 1 x 695 [0,8]
+ CRUSH rule 1 x 696 [1,4]
+ CRUSH rule 1 x 697 [6,1]
+ CRUSH rule 1 x 698 [6,2]
+ CRUSH rule 1 x 699 [1,6]
+ CRUSH rule 1 x 700 [0,3]
+ CRUSH rule 1 x 701 [4,3]
+ CRUSH rule 1 x 702 [3,5]
+ CRUSH rule 1 x 703 [8,3]
+ CRUSH rule 1 x 704 [0,3]
+ CRUSH rule 1 x 705 [8,6]
+ CRUSH rule 1 x 706 [1,2]
+ CRUSH rule 1 x 707 [7,8]
+ CRUSH rule 1 x 708 [3,5]
+ CRUSH rule 1 x 709 [6,3]
+ CRUSH rule 1 x 710 [8,4]
+ CRUSH rule 1 x 711 [2,3]
+ CRUSH rule 1 x 712 [2,3]
+ CRUSH rule 1 x 713 [6,7]
+ CRUSH rule 1 x 714 [3,2]
+ CRUSH rule 1 x 715 [1,2]
+ CRUSH rule 1 x 716 [3,6]
+ CRUSH rule 1 x 717 [8,7]
+ CRUSH rule 1 x 718 [3,7]
+ CRUSH rule 1 x 719 [2,6]
+ CRUSH rule 1 x 720 [6,8]
+ CRUSH rule 1 x 721 [5,4]
+ CRUSH rule 1 x 722 [5,4]
+ CRUSH rule 1 x 723 [5,1]
+ CRUSH rule 1 x 724 [0,6]
+ CRUSH rule 1 x 725 [0,1]
+ CRUSH rule 1 x 726 [3,8]
+ CRUSH rule 1 x 727 [4,6]
+ CRUSH rule 1 x 728 [2,1]
+ CRUSH rule 1 x 729 [5,3]
+ CRUSH rule 1 x 730 [3,7]
+ CRUSH rule 1 x 731 [4,1]
+ CRUSH rule 1 x 732 [1,5]
+ CRUSH rule 1 x 733 [5,4]
+ CRUSH rule 1 x 734 [6,4]
+ CRUSH rule 1 x 735 [4,8]
+ CRUSH rule 1 x 736 [3,5]
+ CRUSH rule 1 x 737 [1,0]
+ CRUSH rule 1 x 738 [5,2]
+ CRUSH rule 1 x 739 [0,1]
+ CRUSH rule 1 x 740 [0,1]
+ CRUSH rule 1 x 741 [7,8]
+ CRUSH rule 1 x 742 [8,2]
+ CRUSH rule 1 x 743 [7,0]
+ CRUSH rule 1 x 744 [4,7]
+ CRUSH rule 1 x 745 [3,4]
+ CRUSH rule 1 x 746 [4,1]
+ CRUSH rule 1 x 747 [6,0]
+ CRUSH rule 1 x 748 [2,7]
+ CRUSH rule 1 x 749 [4,5]
+ CRUSH rule 1 x 750 [1,6]
+ CRUSH rule 1 x 751 [2,1]
+ CRUSH rule 1 x 752 [8,1]
+ CRUSH rule 1 x 753 [7,8]
+ CRUSH rule 1 x 754 [8,6]
+ CRUSH rule 1 x 755 [1,2]
+ CRUSH rule 1 x 756 [5,6]
+ CRUSH rule 1 x 757 [8,6]
+ CRUSH rule 1 x 758 [6,0]
+ CRUSH rule 1 x 759 [8,5]
+ CRUSH rule 1 x 760 [1,5]
+ CRUSH rule 1 x 761 [4,1]
+ CRUSH rule 1 x 762 [2,7]
+ CRUSH rule 1 x 763 [8,6]
+ CRUSH rule 1 x 764 [1,7]
+ CRUSH rule 1 x 765 [6,5]
+ CRUSH rule 1 x 766 [8,5]
+ CRUSH rule 1 x 767 [1,2]
+ CRUSH rule 1 x 768 [8,3]
+ CRUSH rule 1 x 769 [6,2]
+ CRUSH rule 1 x 770 [6,0]
+ CRUSH rule 1 x 771 [7,0]
+ CRUSH rule 1 x 772 [8,3]
+ CRUSH rule 1 x 773 [3,1]
+ CRUSH rule 1 x 774 [4,6]
+ CRUSH rule 1 x 775 [6,8]
+ CRUSH rule 1 x 776 [7,2]
+ CRUSH rule 1 x 777 [3,1]
+ CRUSH rule 1 x 778 [1,8]
+ CRUSH rule 1 x 779 [2,7]
+ CRUSH rule 1 x 780 [0,2]
+ CRUSH rule 1 x 781 [6,3]
+ CRUSH rule 1 x 782 [5,4]
+ CRUSH rule 1 x 783 [7,1]
+ CRUSH rule 1 x 784 [0,1]
+ CRUSH rule 1 x 785 [6,1]
+ CRUSH rule 1 x 786 [7,6]
+ CRUSH rule 1 x 787 [1,0]
+ CRUSH rule 1 x 788 [6,0]
+ CRUSH rule 1 x 789 [0,4]
+ CRUSH rule 1 x 790 [8,4]
+ CRUSH rule 1 x 791 [3,8]
+ CRUSH rule 1 x 792 [5,8]
+ CRUSH rule 1 x 793 [6,1]
+ CRUSH rule 1 x 794 [2,6]
+ CRUSH rule 1 x 795 [0,3]
+ CRUSH rule 1 x 796 [3,4]
+ CRUSH rule 1 x 797 [2,3]
+ CRUSH rule 1 x 798 [6,8]
+ CRUSH rule 1 x 799 [5,1]
+ CRUSH rule 1 x 800 [5,0]
+ CRUSH rule 1 x 801 [3,6]
+ CRUSH rule 1 x 802 [1,8]
+ CRUSH rule 1 x 803 [0,5]
+ CRUSH rule 1 x 804 [6,2]
+ CRUSH rule 1 x 805 [3,6]
+ CRUSH rule 1 x 806 [1,3]
+ CRUSH rule 1 x 807 [5,4]
+ CRUSH rule 1 x 808 [4,6]
+ CRUSH rule 1 x 809 [1,4]
+ CRUSH rule 1 x 810 [5,7]
+ CRUSH rule 1 x 811 [8,4]
+ CRUSH rule 1 x 812 [8,5]
+ CRUSH rule 1 x 813 [6,4]
+ CRUSH rule 1 x 814 [3,6]
+ CRUSH rule 1 x 815 [3,1]
+ CRUSH rule 1 x 816 [2,1]
+ CRUSH rule 1 x 817 [4,3]
+ CRUSH rule 1 x 818 [3,5]
+ CRUSH rule 1 x 819 [5,1]
+ CRUSH rule 1 x 820 [3,5]
+ CRUSH rule 1 x 821 [4,5]
+ CRUSH rule 1 x 822 [2,0]
+ CRUSH rule 1 x 823 [4,8]
+ CRUSH rule 1 x 824 [3,7]
+ CRUSH rule 1 x 825 [2,8]
+ CRUSH rule 1 x 826 [7,0]
+ CRUSH rule 1 x 827 [0,8]
+ CRUSH rule 1 x 828 [2,3]
+ CRUSH rule 1 x 829 [5,6]
+ CRUSH rule 1 x 830 [2,3]
+ CRUSH rule 1 x 831 [1,6]
+ CRUSH rule 1 x 832 [4,5]
+ CRUSH rule 1 x 833 [2,1]
+ CRUSH rule 1 x 834 [3,4]
+ CRUSH rule 1 x 835 [8,4]
+ CRUSH rule 1 x 836 [3,4]
+ CRUSH rule 1 x 837 [6,3]
+ CRUSH rule 1 x 838 [6,7]
+ CRUSH rule 1 x 839 [5,0]
+ CRUSH rule 1 x 840 [7,8]
+ CRUSH rule 1 x 841 [4,8]
+ CRUSH rule 1 x 842 [2,4]
+ CRUSH rule 1 x 843 [6,4]
+ CRUSH rule 1 x 844 [4,8]
+ CRUSH rule 1 x 845 [3,8]
+ CRUSH rule 1 x 846 [3,2]
+ CRUSH rule 1 x 847 [0,2]
+ CRUSH rule 1 x 848 [2,6]
+ CRUSH rule 1 x 849 [4,5]
+ CRUSH rule 1 x 850 [1,0]
+ CRUSH rule 1 x 851 [6,8]
+ CRUSH rule 1 x 852 [7,3]
+ CRUSH rule 1 x 853 [6,8]
+ CRUSH rule 1 x 854 [7,6]
+ CRUSH rule 1 x 855 [5,7]
+ CRUSH rule 1 x 856 [6,7]
+ CRUSH rule 1 x 857 [8,5]
+ CRUSH rule 1 x 858 [6,4]
+ CRUSH rule 1 x 859 [6,0]
+ CRUSH rule 1 x 860 [4,1]
+ CRUSH rule 1 x 861 [8,7]
+ CRUSH rule 1 x 862 [6,1]
+ CRUSH rule 1 x 863 [8,7]
+ CRUSH rule 1 x 864 [5,6]
+ CRUSH rule 1 x 865 [8,1]
+ CRUSH rule 1 x 866 [3,4]
+ CRUSH rule 1 x 867 [6,5]
+ CRUSH rule 1 x 868 [6,3]
+ CRUSH rule 1 x 869 [8,7]
+ CRUSH rule 1 x 870 [0,4]
+ CRUSH rule 1 x 871 [3,4]
+ CRUSH rule 1 x 872 [5,1]
+ CRUSH rule 1 x 873 [4,6]
+ CRUSH rule 1 x 874 [2,6]
+ CRUSH rule 1 x 875 [2,6]
+ CRUSH rule 1 x 876 [5,8]
+ CRUSH rule 1 x 877 [6,4]
+ CRUSH rule 1 x 878 [5,4]
+ CRUSH rule 1 x 879 [7,4]
+ CRUSH rule 1 x 880 [3,5]
+ CRUSH rule 1 x 881 [5,6]
+ CRUSH rule 1 x 882 [4,0]
+ CRUSH rule 1 x 883 [2,1]
+ CRUSH rule 1 x 884 [6,0]
+ CRUSH rule 1 x 885 [5,1]
+ CRUSH rule 1 x 886 [3,6]
+ CRUSH rule 1 x 887 [7,4]
+ CRUSH rule 1 x 888 [6,8]
+ CRUSH rule 1 x 889 [2,1]
+ CRUSH rule 1 x 890 [7,2]
+ CRUSH rule 1 x 891 [1,8]
+ CRUSH rule 1 x 892 [6,2]
+ CRUSH rule 1 x 893 [2,3]
+ CRUSH rule 1 x 894 [7,5]
+ CRUSH rule 1 x 895 [5,3]
+ CRUSH rule 1 x 896 [1,8]
+ CRUSH rule 1 x 897 [4,2]
+ CRUSH rule 1 x 898 [0,5]
+ CRUSH rule 1 x 899 [1,7]
+ CRUSH rule 1 x 900 [4,1]
+ CRUSH rule 1 x 901 [5,0]
+ CRUSH rule 1 x 902 [8,5]
+ CRUSH rule 1 x 903 [5,7]
+ CRUSH rule 1 x 904 [5,6]
+ CRUSH rule 1 x 905 [6,2]
+ CRUSH rule 1 x 906 [1,2]
+ CRUSH rule 1 x 907 [7,1]
+ CRUSH rule 1 x 908 [5,8]
+ CRUSH rule 1 x 909 [2,3]
+ CRUSH rule 1 x 910 [6,4]
+ CRUSH rule 1 x 911 [5,8]
+ CRUSH rule 1 x 912 [0,1]
+ CRUSH rule 1 x 913 [7,6]
+ CRUSH rule 1 x 914 [6,4]
+ CRUSH rule 1 x 915 [8,2]
+ CRUSH rule 1 x 916 [3,1]
+ CRUSH rule 1 x 917 [1,5]
+ CRUSH rule 1 x 918 [8,2]
+ CRUSH rule 1 x 919 [6,2]
+ CRUSH rule 1 x 920 [7,6]
+ CRUSH rule 1 x 921 [1,4]
+ CRUSH rule 1 x 922 [6,7]
+ CRUSH rule 1 x 923 [5,3]
+ CRUSH rule 1 x 924 [3,5]
+ CRUSH rule 1 x 925 [5,7]
+ CRUSH rule 1 x 926 [3,4]
+ CRUSH rule 1 x 927 [1,6]
+ CRUSH rule 1 x 928 [8,1]
+ CRUSH rule 1 x 929 [4,5]
+ CRUSH rule 1 x 930 [2,4]
+ CRUSH rule 1 x 931 [5,0]
+ CRUSH rule 1 x 932 [4,3]
+ CRUSH rule 1 x 933 [8,5]
+ CRUSH rule 1 x 934 [5,3]
+ CRUSH rule 1 x 935 [6,3]
+ CRUSH rule 1 x 936 [0,6]
+ CRUSH rule 1 x 937 [5,4]
+ CRUSH rule 1 x 938 [6,5]
+ CRUSH rule 1 x 939 [2,7]
+ CRUSH rule 1 x 940 [8,7]
+ CRUSH rule 1 x 941 [5,2]
+ CRUSH rule 1 x 942 [1,0]
+ CRUSH rule 1 x 943 [8,2]
+ CRUSH rule 1 x 944 [4,3]
+ CRUSH rule 1 x 945 [7,2]
+ CRUSH rule 1 x 946 [2,0]
+ CRUSH rule 1 x 947 [4,5]
+ CRUSH rule 1 x 948 [7,8]
+ CRUSH rule 1 x 949 [6,1]
+ CRUSH rule 1 x 950 [3,5]
+ CRUSH rule 1 x 951 [4,5]
+ CRUSH rule 1 x 952 [2,0]
+ CRUSH rule 1 x 953 [1,3]
+ CRUSH rule 1 x 954 [4,2]
+ CRUSH rule 1 x 955 [8,6]
+ CRUSH rule 1 x 956 [1,0]
+ CRUSH rule 1 x 957 [7,6]
+ CRUSH rule 1 x 958 [8,7]
+ CRUSH rule 1 x 959 [5,2]
+ CRUSH rule 1 x 960 [3,6]
+ CRUSH rule 1 x 961 [4,0]
+ CRUSH rule 1 x 962 [7,4]
+ CRUSH rule 1 x 963 [0,5]
+ CRUSH rule 1 x 964 [3,1]
+ CRUSH rule 1 x 965 [7,6]
+ CRUSH rule 1 x 966 [3,8]
+ CRUSH rule 1 x 967 [8,6]
+ CRUSH rule 1 x 968 [7,2]
+ CRUSH rule 1 x 969 [8,0]
+ CRUSH rule 1 x 970 [0,6]
+ CRUSH rule 1 x 971 [1,7]
+ CRUSH rule 1 x 972 [1,8]
+ CRUSH rule 1 x 973 [1,2]
+ CRUSH rule 1 x 974 [5,3]
+ CRUSH rule 1 x 975 [3,7]
+ CRUSH rule 1 x 976 [4,3]
+ CRUSH rule 1 x 977 [8,3]
+ CRUSH rule 1 x 978 [7,2]
+ CRUSH rule 1 x 979 [7,6]
+ CRUSH rule 1 x 980 [6,0]
+ CRUSH rule 1 x 981 [7,3]
+ CRUSH rule 1 x 982 [4,2]
+ CRUSH rule 1 x 983 [3,5]
+ CRUSH rule 1 x 984 [0,2]
+ CRUSH rule 1 x 985 [2,5]
+ CRUSH rule 1 x 986 [8,7]
+ CRUSH rule 1 x 987 [0,5]
+ CRUSH rule 1 x 988 [1,3]
+ CRUSH rule 1 x 989 [0,6]
+ CRUSH rule 1 x 990 [1,0]
+ CRUSH rule 1 x 991 [0,4]
+ CRUSH rule 1 x 992 [7,1]
+ CRUSH rule 1 x 993 [0,6]
+ CRUSH rule 1 x 994 [3,4]
+ CRUSH rule 1 x 995 [7,6]
+ CRUSH rule 1 x 996 [6,7]
+ CRUSH rule 1 x 997 [6,4]
+ CRUSH rule 1 x 998 [8,1]
+ CRUSH rule 1 x 999 [0,7]
+ CRUSH rule 1 x 1000 [8,5]
+ CRUSH rule 1 x 1001 [2,0]
+ CRUSH rule 1 x 1002 [1,3]
+ CRUSH rule 1 x 1003 [2,8]
+ CRUSH rule 1 x 1004 [6,1]
+ CRUSH rule 1 x 1005 [6,1]
+ CRUSH rule 1 x 1006 [1,0]
+ CRUSH rule 1 x 1007 [1,2]
+ CRUSH rule 1 x 1008 [1,7]
+ CRUSH rule 1 x 1009 [6,8]
+ CRUSH rule 1 x 1010 [3,4]
+ CRUSH rule 1 x 1011 [3,0]
+ CRUSH rule 1 x 1012 [3,0]
+ CRUSH rule 1 x 1013 [5,1]
+ CRUSH rule 1 x 1014 [2,8]
+ CRUSH rule 1 x 1015 [6,8]
+ CRUSH rule 1 x 1016 [2,0]
+ CRUSH rule 1 x 1017 [6,0]
+ CRUSH rule 1 x 1018 [5,4]
+ CRUSH rule 1 x 1019 [5,3]
+ CRUSH rule 1 x 1020 [5,1]
+ CRUSH rule 1 x 1021 [5,2]
+ CRUSH rule 1 x 1022 [1,6]
+ CRUSH rule 1 x 1023 [3,2]
+ rule 1 (choose-two) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [0,2,3]
+ CRUSH rule 1 x 1 [0,8,2]
+ CRUSH rule 1 x 2 [1,3,5]
+ CRUSH rule 1 x 3 [8,0,6]
+ CRUSH rule 1 x 4 [5,4,0]
+ CRUSH rule 1 x 5 [7,8,0]
+ CRUSH rule 1 x 6 [2,6,7]
+ CRUSH rule 1 x 7 [5,4,3]
+ CRUSH rule 1 x 8 [5,3,8]
+ CRUSH rule 1 x 9 [2,3,1]
+ CRUSH rule 1 x 10 [0,2,8]
+ CRUSH rule 1 x 11 [0,7,2]
+ CRUSH rule 1 x 12 [0,2,4]
+ CRUSH rule 1 x 13 [3,8,5]
+ CRUSH rule 1 x 14 [7,6,8]
+ CRUSH rule 1 x 15 [7,2,4]
+ CRUSH rule 1 x 16 [3,6,0]
+ CRUSH rule 1 x 17 [5,4,0]
+ CRUSH rule 1 x 18 [1,4,5]
+ CRUSH rule 1 x 19 [7,5,4]
+ CRUSH rule 1 x 20 [2,4,7]
+ CRUSH rule 1 x 21 [3,7,2]
+ CRUSH rule 1 x 22 [8,3,7]
+ CRUSH rule 1 x 23 [3,6,8]
+ CRUSH rule 1 x 24 [1,0,6]
+ CRUSH rule 1 x 25 [3,7,0]
+ CRUSH rule 1 x 26 [2,8,7]
+ CRUSH rule 1 x 27 [3,1,5]
+ CRUSH rule 1 x 28 [6,0,3]
+ CRUSH rule 1 x 29 [8,5,7]
+ CRUSH rule 1 x 30 [5,7,3]
+ CRUSH rule 1 x 31 [8,7,6]
+ CRUSH rule 1 x 32 [3,6,8]
+ CRUSH rule 1 x 33 [2,7,8]
+ CRUSH rule 1 x 34 [2,5,4]
+ CRUSH rule 1 x 35 [0,8,1]
+ CRUSH rule 1 x 36 [3,8,2]
+ CRUSH rule 1 x 37 [0,4,5]
+ CRUSH rule 1 x 38 [4,8,7]
+ CRUSH rule 1 x 39 [3,7,0]
+ CRUSH rule 1 x 40 [7,8,1]
+ CRUSH rule 1 x 41 [0,2,6]
+ CRUSH rule 1 x 42 [4,3,6]
+ CRUSH rule 1 x 43 [0,3,7]
+ CRUSH rule 1 x 44 [1,6,7]
+ CRUSH rule 1 x 45 [8,0,4]
+ CRUSH rule 1 x 46 [2,4,3]
+ CRUSH rule 1 x 47 [4,5,3]
+ CRUSH rule 1 x 48 [4,6,5]
+ CRUSH rule 1 x 49 [5,4,8]
+ CRUSH rule 1 x 50 [3,4,1]
+ CRUSH rule 1 x 51 [3,6,0]
+ CRUSH rule 1 x 52 [8,6,1]
+ CRUSH rule 1 x 53 [3,5,4]
+ CRUSH rule 1 x 54 [7,6,8]
+ CRUSH rule 1 x 55 [8,7,0]
+ CRUSH rule 1 x 56 [6,4,8]
+ CRUSH rule 1 x 57 [5,3,6]
+ CRUSH rule 1 x 58 [1,0,2]
+ CRUSH rule 1 x 59 [4,2,1]
+ CRUSH rule 1 x 60 [3,5,0]
+ CRUSH rule 1 x 61 [4,6,8]
+ CRUSH rule 1 x 62 [7,0,1]
+ CRUSH rule 1 x 63 [5,6,8]
+ CRUSH rule 1 x 64 [4,5,1]
+ CRUSH rule 1 x 65 [7,3,6]
+ CRUSH rule 1 x 66 [5,4,3]
+ CRUSH rule 1 x 67 [5,0,8]
+ CRUSH rule 1 x 68 [0,5,8]
+ CRUSH rule 1 x 69 [5,1,6]
+ CRUSH rule 1 x 70 [7,0,2]
+ CRUSH rule 1 x 71 [2,8,6]
+ CRUSH rule 1 x 72 [6,1,5]
+ CRUSH rule 1 x 73 [2,7,3]
+ CRUSH rule 1 x 74 [0,7,1]
+ CRUSH rule 1 x 75 [3,2,0]
+ CRUSH rule 1 x 76 [5,1,7]
+ CRUSH rule 1 x 77 [7,2,6]
+ CRUSH rule 1 x 78 [1,4,2]
+ CRUSH rule 1 x 79 [5,4,0]
+ CRUSH rule 1 x 80 [0,3,5]
+ CRUSH rule 1 x 81 [0,2,5]
+ CRUSH rule 1 x 82 [7,1,6]
+ CRUSH rule 1 x 83 [2,6,8]
+ CRUSH rule 1 x 84 [7,2,1]
+ CRUSH rule 1 x 85 [3,4,8]
+ CRUSH rule 1 x 86 [0,2,1]
+ CRUSH rule 1 x 87 [0,7,4]
+ CRUSH rule 1 x 88 [1,6,0]
+ CRUSH rule 1 x 89 [3,0,2]
+ CRUSH rule 1 x 90 [6,7,5]
+ CRUSH rule 1 x 91 [3,8,7]
+ CRUSH rule 1 x 92 [1,8,4]
+ CRUSH rule 1 x 93 [7,4,5]
+ CRUSH rule 1 x 94 [0,4,3]
+ CRUSH rule 1 x 95 [7,5,1]
+ CRUSH rule 1 x 96 [3,6,4]
+ CRUSH rule 1 x 97 [8,7,4]
+ CRUSH rule 1 x 98 [2,0,7]
+ CRUSH rule 1 x 99 [0,7,2]
+ CRUSH rule 1 x 100 [1,7,4]
+ CRUSH rule 1 x 101 [3,7,2]
+ CRUSH rule 1 x 102 [4,2,7]
+ CRUSH rule 1 x 103 [4,7,6]
+ CRUSH rule 1 x 104 [7,4,3]
+ CRUSH rule 1 x 105 [2,4,1]
+ CRUSH rule 1 x 106 [1,6,2]
+ CRUSH rule 1 x 107 [3,2,1]
+ CRUSH rule 1 x 108 [7,2,8]
+ CRUSH rule 1 x 109 [1,2,5]
+ CRUSH rule 1 x 110 [3,2,7]
+ CRUSH rule 1 x 111 [2,1,0]
+ CRUSH rule 1 x 112 [2,0,6]
+ CRUSH rule 1 x 113 [6,2,1]
+ CRUSH rule 1 x 114 [7,6,3]
+ CRUSH rule 1 x 115 [8,2,3]
+ CRUSH rule 1 x 116 [1,2,6]
+ CRUSH rule 1 x 117 [7,3,6]
+ CRUSH rule 1 x 118 [0,3,8]
+ CRUSH rule 1 x 119 [5,6,1]
+ CRUSH rule 1 x 120 [0,2,5]
+ CRUSH rule 1 x 121 [2,0,1]
+ CRUSH rule 1 x 122 [8,5,0]
+ CRUSH rule 1 x 123 [2,5,1]
+ CRUSH rule 1 x 124 [3,5,1]
+ CRUSH rule 1 x 125 [0,7,3]
+ CRUSH rule 1 x 126 [4,2,6]
+ CRUSH rule 1 x 127 [3,6,7]
+ CRUSH rule 1 x 128 [3,5,8]
+ CRUSH rule 1 x 129 [0,2,1]
+ CRUSH rule 1 x 130 [3,8,5]
+ CRUSH rule 1 x 131 [1,2,0]
+ CRUSH rule 1 x 132 [1,2,3]
+ CRUSH rule 1 x 133 [3,6,7]
+ CRUSH rule 1 x 134 [1,8,2]
+ CRUSH rule 1 x 135 [5,6,4]
+ CRUSH rule 1 x 136 [2,1,5]
+ CRUSH rule 1 x 137 [7,3,4]
+ CRUSH rule 1 x 138 [8,7,6]
+ CRUSH rule 1 x 139 [3,0,5]
+ CRUSH rule 1 x 140 [1,6,7]
+ CRUSH rule 1 x 141 [6,8,1]
+ CRUSH rule 1 x 142 [3,5,0]
+ CRUSH rule 1 x 143 [5,8,7]
+ CRUSH rule 1 x 144 [8,1,0]
+ CRUSH rule 1 x 145 [8,5,6]
+ CRUSH rule 1 x 146 [2,6,4]
+ CRUSH rule 1 x 147 [2,8,7]
+ CRUSH rule 1 x 148 [3,4,0]
+ CRUSH rule 1 x 149 [4,8,6]
+ CRUSH rule 1 x 150 [1,6,5]
+ CRUSH rule 1 x 151 [3,4,5]
+ CRUSH rule 1 x 152 [8,4,6]
+ CRUSH rule 1 x 153 [8,6,7]
+ CRUSH rule 1 x 154 [3,2,0]
+ CRUSH rule 1 x 155 [3,5,8]
+ CRUSH rule 1 x 156 [4,5,3]
+ CRUSH rule 1 x 157 [4,1,2]
+ CRUSH rule 1 x 158 [2,8,1]
+ CRUSH rule 1 x 159 [7,0,5]
+ CRUSH rule 1 x 160 [2,8,1]
+ CRUSH rule 1 x 161 [1,5,2]
+ CRUSH rule 1 x 162 [0,6,2]
+ CRUSH rule 1 x 163 [5,6,1]
+ CRUSH rule 1 x 164 [7,8,1]
+ CRUSH rule 1 x 165 [7,0,5]
+ CRUSH rule 1 x 166 [2,4,1]
+ CRUSH rule 1 x 167 [0,1,6]
+ CRUSH rule 1 x 168 [4,2,3]
+ CRUSH rule 1 x 169 [2,6,1]
+ CRUSH rule 1 x 170 [1,0,5]
+ CRUSH rule 1 x 171 [7,5,4]
+ CRUSH rule 1 x 172 [0,7,8]
+ CRUSH rule 1 x 173 [8,5,3]
+ CRUSH rule 1 x 174 [1,4,7]
+ CRUSH rule 1 x 175 [6,0,4]
+ CRUSH rule 1 x 176 [4,3,5]
+ CRUSH rule 1 x 177 [5,3,0]
+ CRUSH rule 1 x 178 [3,0,2]
+ CRUSH rule 1 x 179 [4,1,2]
+ CRUSH rule 1 x 180 [3,8,7]
+ CRUSH rule 1 x 181 [6,2,0]
+ CRUSH rule 1 x 182 [8,5,3]
+ CRUSH rule 1 x 183 [7,8,6]
+ CRUSH rule 1 x 184 [5,7,3]
+ CRUSH rule 1 x 185 [6,8,0]
+ CRUSH rule 1 x 186 [2,1,4]
+ CRUSH rule 1 x 187 [1,6,7]
+ CRUSH rule 1 x 188 [1,8,7]
+ CRUSH rule 1 x 189 [0,7,8]
+ CRUSH rule 1 x 190 [4,0,1]
+ CRUSH rule 1 x 191 [7,6,8]
+ CRUSH rule 1 x 192 [5,0,1]
+ CRUSH rule 1 x 193 [4,2,6]
+ CRUSH rule 1 x 194 [1,3,4]
+ CRUSH rule 1 x 195 [6,4,3]
+ CRUSH rule 1 x 196 [6,7,2]
+ CRUSH rule 1 x 197 [6,5,2]
+ CRUSH rule 1 x 198 [2,5,6]
+ CRUSH rule 1 x 199 [0,5,7]
+ CRUSH rule 1 x 200 [0,5,1]
+ CRUSH rule 1 x 201 [7,1,0]
+ CRUSH rule 1 x 202 [6,3,4]
+ CRUSH rule 1 x 203 [4,8,7]
+ CRUSH rule 1 x 204 [2,1,5]
+ CRUSH rule 1 x 205 [0,7,6]
+ CRUSH rule 1 x 206 [0,1,7]
+ CRUSH rule 1 x 207 [3,0,5]
+ CRUSH rule 1 x 208 [7,1,0]
+ CRUSH rule 1 x 209 [1,2,0]
+ CRUSH rule 1 x 210 [1,2,3]
+ CRUSH rule 1 x 211 [5,4,2]
+ CRUSH rule 1 x 212 [7,5,0]
+ CRUSH rule 1 x 213 [8,4,0]
+ CRUSH rule 1 x 214 [4,5,7]
+ CRUSH rule 1 x 215 [8,1,7]
+ CRUSH rule 1 x 216 [5,0,2]
+ CRUSH rule 1 x 217 [0,1,7]
+ CRUSH rule 1 x 218 [0,7,8]
+ CRUSH rule 1 x 219 [4,8,2]
+ CRUSH rule 1 x 220 [5,7,3]
+ CRUSH rule 1 x 221 [3,6,5]
+ CRUSH rule 1 x 222 [6,8,3]
+ CRUSH rule 1 x 223 [1,3,6]
+ CRUSH rule 1 x 224 [1,5,0]
+ CRUSH rule 1 x 225 [8,6,2]
+ CRUSH rule 1 x 226 [7,2,3]
+ CRUSH rule 1 x 227 [3,4,0]
+ CRUSH rule 1 x 228 [5,4,8]
+ CRUSH rule 1 x 229 [3,4,7]
+ CRUSH rule 1 x 230 [4,7,2]
+ CRUSH rule 1 x 231 [4,3,5]
+ CRUSH rule 1 x 232 [2,7,4]
+ CRUSH rule 1 x 233 [3,4,5]
+ CRUSH rule 1 x 234 [0,1,2]
+ CRUSH rule 1 x 235 [3,8,0]
+ CRUSH rule 1 x 236 [5,2,7]
+ CRUSH rule 1 x 237 [4,7,6]
+ CRUSH rule 1 x 238 [4,3,5]
+ CRUSH rule 1 x 239 [8,7,6]
+ CRUSH rule 1 x 240 [5,7,4]
+ CRUSH rule 1 x 241 [3,1,5]
+ CRUSH rule 1 x 242 [3,5,4]
+ CRUSH rule 1 x 243 [4,7,3]
+ CRUSH rule 1 x 244 [4,6,0]
+ CRUSH rule 1 x 245 [7,6,1]
+ CRUSH rule 1 x 246 [1,5,2]
+ CRUSH rule 1 x 247 [6,0,2]
+ CRUSH rule 1 x 248 [8,0,7]
+ CRUSH rule 1 x 249 [2,0,1]
+ CRUSH rule 1 x 250 [2,1,0]
+ CRUSH rule 1 x 251 [2,3,6]
+ CRUSH rule 1 x 252 [3,7,8]
+ CRUSH rule 1 x 253 [3,2,4]
+ CRUSH rule 1 x 254 [3,5,4]
+ CRUSH rule 1 x 255 [1,7,8]
+ CRUSH rule 1 x 256 [5,7,0]
+ CRUSH rule 1 x 257 [2,8,4]
+ CRUSH rule 1 x 258 [5,3,0]
+ CRUSH rule 1 x 259 [4,3,6]
+ CRUSH rule 1 x 260 [3,6,2]
+ CRUSH rule 1 x 261 [8,7,4]
+ CRUSH rule 1 x 262 [5,4,8]
+ CRUSH rule 1 x 263 [6,8,2]
+ CRUSH rule 1 x 264 [3,6,5]
+ CRUSH rule 1 x 265 [8,6,7]
+ CRUSH rule 1 x 266 [8,2,1]
+ CRUSH rule 1 x 267 [2,3,0]
+ CRUSH rule 1 x 268 [0,7,2]
+ CRUSH rule 1 x 269 [0,8,2]
+ CRUSH rule 1 x 270 [5,0,1]
+ CRUSH rule 1 x 271 [7,5,4]
+ CRUSH rule 1 x 272 [2,8,3]
+ CRUSH rule 1 x 273 [3,5,0]
+ CRUSH rule 1 x 274 [6,8,3]
+ CRUSH rule 1 x 275 [4,3,6]
+ CRUSH rule 1 x 276 [7,1,2]
+ CRUSH rule 1 x 277 [6,4,0]
+ CRUSH rule 1 x 278 [6,8,0]
+ CRUSH rule 1 x 279 [8,3,6]
+ CRUSH rule 1 x 280 [0,6,2]
+ CRUSH rule 1 x 281 [8,0,2]
+ CRUSH rule 1 x 282 [3,1,6]
+ CRUSH rule 1 x 283 [8,2,0]
+ CRUSH rule 1 x 284 [6,3,0]
+ CRUSH rule 1 x 285 [5,3,7]
+ CRUSH rule 1 x 286 [2,1,6]
+ CRUSH rule 1 x 287 [0,4,1]
+ CRUSH rule 1 x 288 [8,0,7]
+ CRUSH rule 1 x 289 [4,6,2]
+ CRUSH rule 1 x 290 [1,3,7]
+ CRUSH rule 1 x 291 [0,1,4]
+ CRUSH rule 1 x 292 [8,0,2]
+ CRUSH rule 1 x 293 [6,0,8]
+ CRUSH rule 1 x 294 [7,4,5]
+ CRUSH rule 1 x 295 [4,8,7]
+ CRUSH rule 1 x 296 [3,1,5]
+ CRUSH rule 1 x 297 [6,2,8]
+ CRUSH rule 1 x 298 [1,2,3]
+ CRUSH rule 1 x 299 [2,0,7]
+ CRUSH rule 1 x 300 [8,7,3]
+ CRUSH rule 1 x 301 [0,8,1]
+ CRUSH rule 1 x 302 [3,0,6]
+ CRUSH rule 1 x 303 [7,5,8]
+ CRUSH rule 1 x 304 [2,7,5]
+ CRUSH rule 1 x 305 [5,8,2]
+ CRUSH rule 1 x 306 [0,7,1]
+ CRUSH rule 1 x 307 [0,2,8]
+ CRUSH rule 1 x 308 [0,8,4]
+ CRUSH rule 1 x 309 [7,4,5]
+ CRUSH rule 1 x 310 [4,3,1]
+ CRUSH rule 1 x 311 [3,4,5]
+ CRUSH rule 1 x 312 [2,1,0]
+ CRUSH rule 1 x 313 [5,3,4]
+ CRUSH rule 1 x 314 [4,5,0]
+ CRUSH rule 1 x 315 [2,0,1]
+ CRUSH rule 1 x 316 [6,3,8]
+ CRUSH rule 1 x 317 [2,6,0]
+ CRUSH rule 1 x 318 [8,6,7]
+ CRUSH rule 1 x 319 [5,0,8]
+ CRUSH rule 1 x 320 [3,7,1]
+ CRUSH rule 1 x 321 [1,3,0]
+ CRUSH rule 1 x 322 [2,7,3]
+ CRUSH rule 1 x 323 [4,7,0]
+ CRUSH rule 1 x 324 [7,0,8]
+ CRUSH rule 1 x 325 [4,6,0]
+ CRUSH rule 1 x 326 [3,4,1]
+ CRUSH rule 1 x 327 [0,6,7]
+ CRUSH rule 1 x 328 [7,4,8]
+ CRUSH rule 1 x 329 [5,6,3]
+ CRUSH rule 1 x 330 [3,7,4]
+ CRUSH rule 1 x 331 [2,6,3]
+ CRUSH rule 1 x 332 [2,0,3]
+ CRUSH rule 1 x 333 [6,8,5]
+ CRUSH rule 1 x 334 [8,3,5]
+ CRUSH rule 1 x 335 [7,1,2]
+ CRUSH rule 1 x 336 [4,6,0]
+ CRUSH rule 1 x 337 [7,2,6]
+ CRUSH rule 1 x 338 [5,6,4]
+ CRUSH rule 1 x 339 [7,5,2]
+ CRUSH rule 1 x 340 [2,0,1]
+ CRUSH rule 1 x 341 [5,1,7]
+ CRUSH rule 1 x 342 [0,7,4]
+ CRUSH rule 1 x 343 [6,7,3]
+ CRUSH rule 1 x 344 [6,0,5]
+ CRUSH rule 1 x 345 [4,3,5]
+ CRUSH rule 1 x 346 [8,0,5]
+ CRUSH rule 1 x 347 [3,1,5]
+ CRUSH rule 1 x 348 [8,0,2]
+ CRUSH rule 1 x 349 [1,6,7]
+ CRUSH rule 1 x 350 [8,5,7]
+ CRUSH rule 1 x 351 [3,6,4]
+ CRUSH rule 1 x 352 [1,0,2]
+ CRUSH rule 1 x 353 [6,4,5]
+ CRUSH rule 1 x 354 [0,3,1]
+ CRUSH rule 1 x 355 [3,4,6]
+ CRUSH rule 1 x 356 [3,5,4]
+ CRUSH rule 1 x 357 [6,1,2]
+ CRUSH rule 1 x 358 [2,1,8]
+ CRUSH rule 1 x 359 [6,7,8]
+ CRUSH rule 1 x 360 [5,3,1]
+ CRUSH rule 1 x 361 [8,4,5]
+ CRUSH rule 1 x 362 [4,5,3]
+ CRUSH rule 1 x 363 [4,0,3]
+ CRUSH rule 1 x 364 [2,5,0]
+ CRUSH rule 1 x 365 [6,7,8]
+ CRUSH rule 1 x 366 [7,2,8]
+ CRUSH rule 1 x 367 [4,5,0]
+ CRUSH rule 1 x 368 [7,4,6]
+ CRUSH rule 1 x 369 [3,7,1]
+ CRUSH rule 1 x 370 [8,7,6]
+ CRUSH rule 1 x 371 [1,3,5]
+ CRUSH rule 1 x 372 [3,1,8]
+ CRUSH rule 1 x 373 [0,6,8]
+ CRUSH rule 1 x 374 [3,8,4]
+ CRUSH rule 1 x 375 [6,4,5]
+ CRUSH rule 1 x 376 [7,1,2]
+ CRUSH rule 1 x 377 [1,2,4]
+ CRUSH rule 1 x 378 [0,1,2]
+ CRUSH rule 1 x 379 [8,5,4]
+ CRUSH rule 1 x 380 [2,5,0]
+ CRUSH rule 1 x 381 [0,4,7]
+ CRUSH rule 1 x 382 [1,5,2]
+ CRUSH rule 1 x 383 [4,3,5]
+ CRUSH rule 1 x 384 [7,0,6]
+ CRUSH rule 1 x 385 [7,4,6]
+ CRUSH rule 1 x 386 [0,3,5]
+ CRUSH rule 1 x 387 [1,3,6]
+ CRUSH rule 1 x 388 [5,0,3]
+ CRUSH rule 1 x 389 [1,5,4]
+ CRUSH rule 1 x 390 [5,6,0]
+ CRUSH rule 1 x 391 [5,6,2]
+ CRUSH rule 1 x 392 [1,8,6]
+ CRUSH rule 1 x 393 [4,2,6]
+ CRUSH rule 1 x 394 [4,7,6]
+ CRUSH rule 1 x 395 [4,0,1]
+ CRUSH rule 1 x 396 [4,2,5]
+ CRUSH rule 1 x 397 [2,1,5]
+ CRUSH rule 1 x 398 [2,4,5]
+ CRUSH rule 1 x 399 [8,7,3]
+ CRUSH rule 1 x 400 [8,1,4]
+ CRUSH rule 1 x 401 [0,1,2]
+ CRUSH rule 1 x 402 [7,8,6]
+ CRUSH rule 1 x 403 [0,1,4]
+ CRUSH rule 1 x 404 [4,3,2]
+ CRUSH rule 1 x 405 [6,5,2]
+ CRUSH rule 1 x 406 [2,0,1]
+ CRUSH rule 1 x 407 [2,8,7]
+ CRUSH rule 1 x 408 [4,1,6]
+ CRUSH rule 1 x 409 [7,3,6]
+ CRUSH rule 1 x 410 [8,6,3]
+ CRUSH rule 1 x 411 [2,0,6]
+ CRUSH rule 1 x 412 [0,5,8]
+ CRUSH rule 1 x 413 [5,0,8]
+ CRUSH rule 1 x 414 [4,1,3]
+ CRUSH rule 1 x 415 [0,6,1]
+ CRUSH rule 1 x 416 [2,1,7]
+ CRUSH rule 1 x 417 [8,7,2]
+ CRUSH rule 1 x 418 [7,6,8]
+ CRUSH rule 1 x 419 [8,3,0]
+ CRUSH rule 1 x 420 [1,4,2]
+ CRUSH rule 1 x 421 [8,6,7]
+ CRUSH rule 1 x 422 [6,7,8]
+ CRUSH rule 1 x 423 [0,5,6]
+ CRUSH rule 1 x 424 [8,4,7]
+ CRUSH rule 1 x 425 [1,3,5]
+ CRUSH rule 1 x 426 [6,7,2]
+ CRUSH rule 1 x 427 [0,7,5]
+ CRUSH rule 1 x 428 [5,4,3]
+ CRUSH rule 1 x 429 [4,6,7]
+ CRUSH rule 1 x 430 [3,6,5]
+ CRUSH rule 1 x 431 [5,3,0]
+ CRUSH rule 1 x 432 [7,1,2]
+ CRUSH rule 1 x 433 [6,5,1]
+ CRUSH rule 1 x 434 [5,2,1]
+ CRUSH rule 1 x 435 [0,5,3]
+ CRUSH rule 1 x 436 [4,0,5]
+ CRUSH rule 1 x 437 [7,5,3]
+ CRUSH rule 1 x 438 [0,3,8]
+ CRUSH rule 1 x 439 [1,3,4]
+ CRUSH rule 1 x 440 [2,7,1]
+ CRUSH rule 1 x 441 [5,7,2]
+ CRUSH rule 1 x 442 [2,4,5]
+ CRUSH rule 1 x 443 [6,8,0]
+ CRUSH rule 1 x 444 [7,0,1]
+ CRUSH rule 1 x 445 [6,3,5]
+ CRUSH rule 1 x 446 [4,3,5]
+ CRUSH rule 1 x 447 [2,1,4]
+ CRUSH rule 1 x 448 [7,2,5]
+ CRUSH rule 1 x 449 [7,8,3]
+ CRUSH rule 1 x 450 [4,5,2]
+ CRUSH rule 1 x 451 [6,8,3]
+ CRUSH rule 1 x 452 [8,3,5]
+ CRUSH rule 1 x 453 [6,8,7]
+ CRUSH rule 1 x 454 [6,7,5]
+ CRUSH rule 1 x 455 [2,7,5]
+ CRUSH rule 1 x 456 [6,8,7]
+ CRUSH rule 1 x 457 [7,2,8]
+ CRUSH rule 1 x 458 [2,8,1]
+ CRUSH rule 1 x 459 [2,0,6]
+ CRUSH rule 1 x 460 [6,5,7]
+ CRUSH rule 1 x 461 [6,5,4]
+ CRUSH rule 1 x 462 [8,1,5]
+ CRUSH rule 1 x 463 [6,7,2]
+ CRUSH rule 1 x 464 [7,4,2]
+ CRUSH rule 1 x 465 [7,6,0]
+ CRUSH rule 1 x 466 [5,8,4]
+ CRUSH rule 1 x 467 [6,4,7]
+ CRUSH rule 1 x 468 [7,8,0]
+ CRUSH rule 1 x 469 [7,0,1]
+ CRUSH rule 1 x 470 [3,0,5]
+ CRUSH rule 1 x 471 [0,1,2]
+ CRUSH rule 1 x 472 [5,4,0]
+ CRUSH rule 1 x 473 [1,0,4]
+ CRUSH rule 1 x 474 [6,0,7]
+ CRUSH rule 1 x 475 [6,7,8]
+ CRUSH rule 1 x 476 [4,3,5]
+ CRUSH rule 1 x 477 [5,8,6]
+ CRUSH rule 1 x 478 [6,7,2]
+ CRUSH rule 1 x 479 [0,5,8]
+ CRUSH rule 1 x 480 [1,8,6]
+ CRUSH rule 1 x 481 [2,4,5]
+ CRUSH rule 1 x 482 [4,3,5]
+ CRUSH rule 1 x 483 [0,2,6]
+ CRUSH rule 1 x 484 [1,2,7]
+ CRUSH rule 1 x 485 [4,7,3]
+ CRUSH rule 1 x 486 [4,1,7]
+ CRUSH rule 1 x 487 [5,0,1]
+ CRUSH rule 1 x 488 [5,7,1]
+ CRUSH rule 1 x 489 [2,8,5]
+ CRUSH rule 1 x 490 [6,4,1]
+ CRUSH rule 1 x 491 [1,0,6]
+ CRUSH rule 1 x 492 [6,5,4]
+ CRUSH rule 1 x 493 [0,2,7]
+ CRUSH rule 1 x 494 [1,0,8]
+ CRUSH rule 1 x 495 [3,4,5]
+ CRUSH rule 1 x 496 [7,5,4]
+ CRUSH rule 1 x 497 [5,7,3]
+ CRUSH rule 1 x 498 [0,5,8]
+ CRUSH rule 1 x 499 [8,4,3]
+ CRUSH rule 1 x 500 [3,6,5]
+ CRUSH rule 1 x 501 [0,7,2]
+ CRUSH rule 1 x 502 [7,1,4]
+ CRUSH rule 1 x 503 [2,3,5]
+ CRUSH rule 1 x 504 [5,6,3]
+ CRUSH rule 1 x 505 [0,7,6]
+ CRUSH rule 1 x 506 [5,3,0]
+ CRUSH rule 1 x 507 [6,0,8]
+ CRUSH rule 1 x 508 [0,1,2]
+ CRUSH rule 1 x 509 [7,5,8]
+ CRUSH rule 1 x 510 [6,0,2]
+ CRUSH rule 1 x 511 [5,8,7]
+ CRUSH rule 1 x 512 [7,6,2]
+ CRUSH rule 1 x 513 [7,2,1]
+ CRUSH rule 1 x 514 [4,3,5]
+ CRUSH rule 1 x 515 [8,5,6]
+ CRUSH rule 1 x 516 [4,0,1]
+ CRUSH rule 1 x 517 [7,8,6]
+ CRUSH rule 1 x 518 [4,6,7]
+ CRUSH rule 1 x 519 [7,3,5]
+ CRUSH rule 1 x 520 [2,6,3]
+ CRUSH rule 1 x 521 [8,7,6]
+ CRUSH rule 1 x 522 [6,8,0]
+ CRUSH rule 1 x 523 [4,2,5]
+ CRUSH rule 1 x 524 [0,4,5]
+ CRUSH rule 1 x 525 [0,4,1]
+ CRUSH rule 1 x 526 [1,5,8]
+ CRUSH rule 1 x 527 [0,2,4]
+ CRUSH rule 1 x 528 [5,3,0]
+ CRUSH rule 1 x 529 [5,7,0]
+ CRUSH rule 1 x 530 [6,7,8]
+ CRUSH rule 1 x 531 [6,1,0]
+ CRUSH rule 1 x 532 [6,3,7]
+ CRUSH rule 1 x 533 [5,6,3]
+ CRUSH rule 1 x 534 [7,3,5]
+ CRUSH rule 1 x 535 [8,6,0]
+ CRUSH rule 1 x 536 [6,7,0]
+ CRUSH rule 1 x 537 [3,7,5]
+ CRUSH rule 1 x 538 [6,8,3]
+ CRUSH rule 1 x 539 [8,3,7]
+ CRUSH rule 1 x 540 [0,6,1]
+ CRUSH rule 1 x 541 [2,3,1]
+ CRUSH rule 1 x 542 [3,5,4]
+ CRUSH rule 1 x 543 [6,0,8]
+ CRUSH rule 1 x 544 [3,7,6]
+ CRUSH rule 1 x 545 [5,7,0]
+ CRUSH rule 1 x 546 [6,1,7]
+ CRUSH rule 1 x 547 [8,2,0]
+ CRUSH rule 1 x 548 [5,2,0]
+ CRUSH rule 1 x 549 [5,8,2]
+ CRUSH rule 1 x 550 [0,5,4]
+ CRUSH rule 1 x 551 [7,5,4]
+ CRUSH rule 1 x 552 [5,4,3]
+ CRUSH rule 1 x 553 [4,2,3]
+ CRUSH rule 1 x 554 [0,8,5]
+ CRUSH rule 1 x 555 [5,0,8]
+ CRUSH rule 1 x 556 [3,4,5]
+ CRUSH rule 1 x 557 [7,4,6]
+ CRUSH rule 1 x 558 [3,1,0]
+ CRUSH rule 1 x 559 [4,2,6]
+ CRUSH rule 1 x 560 [8,3,5]
+ CRUSH rule 1 x 561 [6,3,7]
+ CRUSH rule 1 x 562 [3,4,1]
+ CRUSH rule 1 x 563 [2,6,0]
+ CRUSH rule 1 x 564 [5,1,7]
+ CRUSH rule 1 x 565 [3,6,5]
+ CRUSH rule 1 x 566 [4,7,2]
+ CRUSH rule 1 x 567 [3,6,1]
+ CRUSH rule 1 x 568 [7,4,1]
+ CRUSH rule 1 x 569 [3,4,2]
+ CRUSH rule 1 x 570 [1,5,4]
+ CRUSH rule 1 x 571 [3,7,5]
+ CRUSH rule 1 x 572 [3,4,0]
+ CRUSH rule 1 x 573 [3,0,7]
+ CRUSH rule 1 x 574 [2,0,1]
+ CRUSH rule 1 x 575 [8,6,0]
+ CRUSH rule 1 x 576 [4,6,8]
+ CRUSH rule 1 x 577 [8,2,6]
+ CRUSH rule 1 x 578 [6,8,7]
+ CRUSH rule 1 x 579 [3,1,6]
+ CRUSH rule 1 x 580 [3,0,7]
+ CRUSH rule 1 x 581 [7,2,6]
+ CRUSH rule 1 x 582 [2,8,7]
+ CRUSH rule 1 x 583 [6,0,8]
+ CRUSH rule 1 x 584 [8,1,2]
+ CRUSH rule 1 x 585 [7,0,5]
+ CRUSH rule 1 x 586 [0,1,7]
+ CRUSH rule 1 x 587 [2,5,0]
+ CRUSH rule 1 x 588 [3,4,7]
+ CRUSH rule 1 x 589 [7,1,4]
+ CRUSH rule 1 x 590 [6,2,3]
+ CRUSH rule 1 x 591 [5,2,4]
+ CRUSH rule 1 x 592 [2,0,3]
+ CRUSH rule 1 x 593 [0,8,1]
+ CRUSH rule 1 x 594 [0,7,4]
+ CRUSH rule 1 x 595 [7,1,8]
+ CRUSH rule 1 x 596 [4,3,5]
+ CRUSH rule 1 x 597 [3,1,2]
+ CRUSH rule 1 x 598 [3,2,4]
+ CRUSH rule 1 x 599 [5,2,1]
+ CRUSH rule 1 x 600 [7,0,8]
+ CRUSH rule 1 x 601 [0,7,8]
+ CRUSH rule 1 x 602 [3,7,5]
+ CRUSH rule 1 x 603 [5,1,6]
+ CRUSH rule 1 x 604 [7,5,4]
+ CRUSH rule 1 x 605 [3,0,5]
+ CRUSH rule 1 x 606 [2,0,1]
+ CRUSH rule 1 x 607 [0,4,8]
+ CRUSH rule 1 x 608 [5,3,2]
+ CRUSH rule 1 x 609 [5,2,4]
+ CRUSH rule 1 x 610 [3,7,6]
+ CRUSH rule 1 x 611 [1,0,2]
+ CRUSH rule 1 x 612 [2,0,8]
+ CRUSH rule 1 x 613 [7,2,0]
+ CRUSH rule 1 x 614 [7,8,6]
+ CRUSH rule 1 x 615 [6,8,2]
+ CRUSH rule 1 x 616 [0,8,1]
+ CRUSH rule 1 x 617 [6,1,2]
+ CRUSH rule 1 x 618 [7,6,5]
+ CRUSH rule 1 x 619 [5,1,8]
+ CRUSH rule 1 x 620 [4,1,0]
+ CRUSH rule 1 x 621 [5,8,3]
+ CRUSH rule 1 x 622 [0,4,3]
+ CRUSH rule 1 x 623 [0,6,2]
+ CRUSH rule 1 x 624 [3,5,1]
+ CRUSH rule 1 x 625 [2,3,7]
+ CRUSH rule 1 x 626 [7,8,0]
+ CRUSH rule 1 x 627 [2,7,1]
+ CRUSH rule 1 x 628 [8,0,5]
+ CRUSH rule 1 x 629 [2,6,5]
+ CRUSH rule 1 x 630 [2,7,1]
+ CRUSH rule 1 x 631 [0,7,8]
+ CRUSH rule 1 x 632 [7,0,2]
+ CRUSH rule 1 x 633 [8,6,5]
+ CRUSH rule 1 x 634 [0,4,2]
+ CRUSH rule 1 x 635 [5,6,4]
+ CRUSH rule 1 x 636 [1,4,2]
+ CRUSH rule 1 x 637 [4,1,2]
+ CRUSH rule 1 x 638 [6,8,1]
+ CRUSH rule 1 x 639 [4,2,1]
+ CRUSH rule 1 x 640 [3,1,0]
+ CRUSH rule 1 x 641 [7,2,0]
+ CRUSH rule 1 x 642 [2,0,1]
+ CRUSH rule 1 x 643 [3,5,1]
+ CRUSH rule 1 x 644 [8,1,5]
+ CRUSH rule 1 x 645 [5,4,6]
+ CRUSH rule 1 x 646 [8,0,3]
+ CRUSH rule 1 x 647 [7,1,3]
+ CRUSH rule 1 x 648 [0,6,3]
+ CRUSH rule 1 x 649 [4,7,5]
+ CRUSH rule 1 x 650 [7,8,6]
+ CRUSH rule 1 x 651 [3,7,6]
+ CRUSH rule 1 x 652 [3,4,8]
+ CRUSH rule 1 x 653 [8,5,2]
+ CRUSH rule 1 x 654 [7,5,2]
+ CRUSH rule 1 x 655 [0,3,4]
+ CRUSH rule 1 x 656 [4,3,8]
+ CRUSH rule 1 x 657 [6,1,2]
+ CRUSH rule 1 x 658 [5,4,6]
+ CRUSH rule 1 x 659 [4,6,7]
+ CRUSH rule 1 x 660 [7,8,6]
+ CRUSH rule 1 x 661 [1,8,3]
+ CRUSH rule 1 x 662 [4,5,3]
+ CRUSH rule 1 x 663 [1,3,8]
+ CRUSH rule 1 x 664 [1,4,0]
+ CRUSH rule 1 x 665 [5,7,6]
+ CRUSH rule 1 x 666 [2,8,4]
+ CRUSH rule 1 x 667 [1,3,5]
+ CRUSH rule 1 x 668 [3,7,6]
+ CRUSH rule 1 x 669 [6,4,5]
+ CRUSH rule 1 x 670 [4,0,3]
+ CRUSH rule 1 x 671 [0,2,1]
+ CRUSH rule 1 x 672 [4,3,2]
+ CRUSH rule 1 x 673 [5,3,0]
+ CRUSH rule 1 x 674 [3,1,8]
+ CRUSH rule 1 x 675 [0,8,6]
+ CRUSH rule 1 x 676 [0,2,4]
+ CRUSH rule 1 x 677 [4,1,3]
+ CRUSH rule 1 x 678 [2,3,1]
+ CRUSH rule 1 x 679 [6,0,2]
+ CRUSH rule 1 x 680 [0,4,6]
+ CRUSH rule 1 x 681 [4,7,3]
+ CRUSH rule 1 x 682 [0,5,2]
+ CRUSH rule 1 x 683 [0,1,5]
+ CRUSH rule 1 x 684 [7,1,6]
+ CRUSH rule 1 x 685 [7,1,6]
+ CRUSH rule 1 x 686 [1,4,3]
+ CRUSH rule 1 x 687 [3,5,7]
+ CRUSH rule 1 x 688 [5,7,2]
+ CRUSH rule 1 x 689 [6,5,0]
+ CRUSH rule 1 x 690 [8,1,7]
+ CRUSH rule 1 x 691 [3,0,5]
+ CRUSH rule 1 x 692 [7,2,8]
+ CRUSH rule 1 x 693 [6,7,3]
+ CRUSH rule 1 x 694 [6,5,1]
+ CRUSH rule 1 x 695 [0,8,7]
+ CRUSH rule 1 x 696 [1,4,3]
+ CRUSH rule 1 x 697 [6,1,2]
+ CRUSH rule 1 x 698 [6,2,0]
+ CRUSH rule 1 x 699 [1,6,8]
+ CRUSH rule 1 x 700 [0,3,1]
+ CRUSH rule 1 x 701 [4,3,2]
+ CRUSH rule 1 x 702 [3,5,0]
+ CRUSH rule 1 x 703 [8,3,4]
+ CRUSH rule 1 x 704 [0,3,8]
+ CRUSH rule 1 x 705 [8,6,0]
+ CRUSH rule 1 x 706 [1,2,4]
+ CRUSH rule 1 x 707 [7,8,6]
+ CRUSH rule 1 x 708 [3,5,8]
+ CRUSH rule 1 x 709 [6,3,0]
+ CRUSH rule 1 x 710 [8,4,3]
+ CRUSH rule 1 x 711 [2,3,8]
+ CRUSH rule 1 x 712 [2,3,7]
+ CRUSH rule 1 x 713 [6,7,8]
+ CRUSH rule 1 x 714 [3,2,0]
+ CRUSH rule 1 x 715 [1,2,4]
+ CRUSH rule 1 x 716 [3,6,0]
+ CRUSH rule 1 x 717 [8,7,2]
+ CRUSH rule 1 x 718 [3,7,8]
+ CRUSH rule 1 x 719 [2,6,3]
+ CRUSH rule 1 x 720 [6,8,2]
+ CRUSH rule 1 x 721 [5,4,7]
+ CRUSH rule 1 x 722 [5,4,6]
+ CRUSH rule 1 x 723 [5,1,0]
+ CRUSH rule 1 x 724 [0,6,2]
+ CRUSH rule 1 x 725 [0,1,5]
+ CRUSH rule 1 x 726 [3,8,4]
+ CRUSH rule 1 x 727 [4,6,8]
+ CRUSH rule 1 x 728 [2,1,0]
+ CRUSH rule 1 x 729 [5,3,7]
+ CRUSH rule 1 x 730 [3,7,5]
+ CRUSH rule 1 x 731 [4,1,5]
+ CRUSH rule 1 x 732 [1,5,3]
+ CRUSH rule 1 x 733 [5,4,7]
+ CRUSH rule 1 x 734 [6,4,2]
+ CRUSH rule 1 x 735 [4,8,3]
+ CRUSH rule 1 x 736 [3,5,6]
+ CRUSH rule 1 x 737 [1,0,8]
+ CRUSH rule 1 x 738 [5,2,3]
+ CRUSH rule 1 x 739 [0,1,2]
+ CRUSH rule 1 x 740 [0,1,7]
+ CRUSH rule 1 x 741 [7,8,0]
+ CRUSH rule 1 x 742 [8,2,1]
+ CRUSH rule 1 x 743 [7,0,1]
+ CRUSH rule 1 x 744 [4,7,3]
+ CRUSH rule 1 x 745 [3,4,1]
+ CRUSH rule 1 x 746 [4,1,3]
+ CRUSH rule 1 x 747 [6,0,3]
+ CRUSH rule 1 x 748 [2,7,0]
+ CRUSH rule 1 x 749 [4,5,8]
+ CRUSH rule 1 x 750 [1,6,3]
+ CRUSH rule 1 x 751 [2,1,6]
+ CRUSH rule 1 x 752 [8,1,5]
+ CRUSH rule 1 x 753 [7,8,3]
+ CRUSH rule 1 x 754 [8,6,7]
+ CRUSH rule 1 x 755 [1,2,0]
+ CRUSH rule 1 x 756 [5,6,1]
+ CRUSH rule 1 x 757 [8,6,1]
+ CRUSH rule 1 x 758 [6,0,3]
+ CRUSH rule 1 x 759 [8,5,3]
+ CRUSH rule 1 x 760 [1,5,4]
+ CRUSH rule 1 x 761 [4,1,2]
+ CRUSH rule 1 x 762 [2,7,8]
+ CRUSH rule 1 x 763 [8,6,7]
+ CRUSH rule 1 x 764 [1,7,0]
+ CRUSH rule 1 x 765 [6,5,2]
+ CRUSH rule 1 x 766 [8,5,7]
+ CRUSH rule 1 x 767 [1,2,0]
+ CRUSH rule 1 x 768 [8,3,2]
+ CRUSH rule 1 x 769 [6,2,8]
+ CRUSH rule 1 x 770 [6,0,7]
+ CRUSH rule 1 x 771 [7,0,3]
+ CRUSH rule 1 x 772 [8,3,7]
+ CRUSH rule 1 x 773 [3,1,5]
+ CRUSH rule 1 x 774 [4,6,5]
+ CRUSH rule 1 x 775 [6,8,4]
+ CRUSH rule 1 x 776 [7,2,1]
+ CRUSH rule 1 x 777 [3,1,6]
+ CRUSH rule 1 x 778 [1,8,0]
+ CRUSH rule 1 x 779 [2,7,3]
+ CRUSH rule 1 x 780 [0,2,3]
+ CRUSH rule 1 x 781 [6,3,7]
+ CRUSH rule 1 x 782 [5,4,0]
+ CRUSH rule 1 x 783 [7,1,8]
+ CRUSH rule 1 x 784 [0,1,5]
+ CRUSH rule 1 x 785 [6,1,2]
+ CRUSH rule 1 x 786 [7,6,5]
+ CRUSH rule 1 x 787 [1,0,6]
+ CRUSH rule 1 x 788 [6,0,8]
+ CRUSH rule 1 x 789 [0,4,3]
+ CRUSH rule 1 x 790 [8,4,7]
+ CRUSH rule 1 x 791 [3,8,7]
+ CRUSH rule 1 x 792 [5,8,4]
+ CRUSH rule 1 x 793 [6,1,3]
+ CRUSH rule 1 x 794 [2,6,8]
+ CRUSH rule 1 x 795 [0,3,2]
+ CRUSH rule 1 x 796 [3,4,5]
+ CRUSH rule 1 x 797 [2,3,4]
+ CRUSH rule 1 x 798 [6,8,7]
+ CRUSH rule 1 x 799 [5,1,4]
+ CRUSH rule 1 x 800 [5,0,3]
+ CRUSH rule 1 x 801 [3,6,8]
+ CRUSH rule 1 x 802 [1,8,0]
+ CRUSH rule 1 x 803 [0,5,7]
+ CRUSH rule 1 x 804 [6,2,5]
+ CRUSH rule 1 x 805 [3,6,7]
+ CRUSH rule 1 x 806 [1,3,4]
+ CRUSH rule 1 x 807 [5,4,7]
+ CRUSH rule 1 x 808 [4,6,2]
+ CRUSH rule 1 x 809 [1,4,5]
+ CRUSH rule 1 x 810 [5,7,3]
+ CRUSH rule 1 x 811 [8,4,5]
+ CRUSH rule 1 x 812 [8,5,4]
+ CRUSH rule 1 x 813 [6,4,8]
+ CRUSH rule 1 x 814 [3,6,8]
+ CRUSH rule 1 x 815 [3,1,2]
+ CRUSH rule 1 x 816 [2,1,8]
+ CRUSH rule 1 x 817 [4,3,7]
+ CRUSH rule 1 x 818 [3,5,0]
+ CRUSH rule 1 x 819 [5,1,4]
+ CRUSH rule 1 x 820 [3,5,4]
+ CRUSH rule 1 x 821 [4,5,8]
+ CRUSH rule 1 x 822 [2,0,5]
+ CRUSH rule 1 x 823 [4,8,2]
+ CRUSH rule 1 x 824 [3,7,4]
+ CRUSH rule 1 x 825 [2,8,7]
+ CRUSH rule 1 x 826 [7,0,5]
+ CRUSH rule 1 x 827 [0,8,2]
+ CRUSH rule 1 x 828 [2,3,1]
+ CRUSH rule 1 x 829 [5,6,7]
+ CRUSH rule 1 x 830 [2,3,5]
+ CRUSH rule 1 x 831 [1,6,0]
+ CRUSH rule 1 x 832 [4,5,3]
+ CRUSH rule 1 x 833 [2,1,7]
+ CRUSH rule 1 x 834 [3,4,2]
+ CRUSH rule 1 x 835 [8,4,3]
+ CRUSH rule 1 x 836 [3,4,5]
+ CRUSH rule 1 x 837 [6,3,1]
+ CRUSH rule 1 x 838 [6,7,2]
+ CRUSH rule 1 x 839 [5,0,6]
+ CRUSH rule 1 x 840 [7,8,5]
+ CRUSH rule 1 x 841 [4,8,5]
+ CRUSH rule 1 x 842 [2,4,0]
+ CRUSH rule 1 x 843 [6,4,7]
+ CRUSH rule 1 x 844 [4,8,1]
+ CRUSH rule 1 x 845 [3,8,6]
+ CRUSH rule 1 x 846 [3,2,0]
+ CRUSH rule 1 x 847 [0,2,7]
+ CRUSH rule 1 x 848 [2,6,1]
+ CRUSH rule 1 x 849 [4,5,3]
+ CRUSH rule 1 x 850 [1,0,5]
+ CRUSH rule 1 x 851 [6,8,7]
+ CRUSH rule 1 x 852 [7,3,8]
+ CRUSH rule 1 x 853 [6,8,1]
+ CRUSH rule 1 x 854 [7,6,0]
+ CRUSH rule 1 x 855 [5,7,2]
+ CRUSH rule 1 x 856 [6,7,3]
+ CRUSH rule 1 x 857 [8,5,0]
+ CRUSH rule 1 x 858 [6,4,1]
+ CRUSH rule 1 x 859 [6,0,7]
+ CRUSH rule 1 x 860 [4,1,2]
+ CRUSH rule 1 x 861 [8,7,6]
+ CRUSH rule 1 x 862 [6,1,7]
+ CRUSH rule 1 x 863 [8,7,0]
+ CRUSH rule 1 x 864 [5,6,8]
+ CRUSH rule 1 x 865 [8,1,0]
+ CRUSH rule 1 x 866 [3,4,8]
+ CRUSH rule 1 x 867 [6,5,1]
+ CRUSH rule 1 x 868 [6,3,0]
+ CRUSH rule 1 x 869 [8,7,3]
+ CRUSH rule 1 x 870 [0,4,8]
+ CRUSH rule 1 x 871 [3,4,5]
+ CRUSH rule 1 x 872 [5,1,3]
+ CRUSH rule 1 x 873 [4,6,5]
+ CRUSH rule 1 x 874 [2,6,1]
+ CRUSH rule 1 x 875 [2,6,4]
+ CRUSH rule 1 x 876 [5,8,1]
+ CRUSH rule 1 x 877 [6,4,2]
+ CRUSH rule 1 x 878 [5,4,0]
+ CRUSH rule 1 x 879 [7,4,8]
+ CRUSH rule 1 x 880 [3,5,0]
+ CRUSH rule 1 x 881 [5,6,1]
+ CRUSH rule 1 x 882 [4,0,2]
+ CRUSH rule 1 x 883 [2,1,0]
+ CRUSH rule 1 x 884 [6,0,4]
+ CRUSH rule 1 x 885 [5,1,4]
+ CRUSH rule 1 x 886 [3,6,4]
+ CRUSH rule 1 x 887 [7,4,0]
+ CRUSH rule 1 x 888 [6,8,0]
+ CRUSH rule 1 x 889 [2,1,7]
+ CRUSH rule 1 x 890 [7,2,0]
+ CRUSH rule 1 x 891 [1,8,0]
+ CRUSH rule 1 x 892 [6,2,3]
+ CRUSH rule 1 x 893 [2,3,7]
+ CRUSH rule 1 x 894 [7,5,0]
+ CRUSH rule 1 x 895 [5,3,2]
+ CRUSH rule 1 x 896 [1,8,2]
+ CRUSH rule 1 x 897 [4,2,6]
+ CRUSH rule 1 x 898 [0,5,4]
+ CRUSH rule 1 x 899 [1,7,6]
+ CRUSH rule 1 x 900 [4,1,0]
+ CRUSH rule 1 x 901 [5,0,1]
+ CRUSH rule 1 x 902 [8,5,7]
+ CRUSH rule 1 x 903 [5,7,3]
+ CRUSH rule 1 x 904 [5,6,8]
+ CRUSH rule 1 x 905 [6,2,5]
+ CRUSH rule 1 x 906 [1,2,0]
+ CRUSH rule 1 x 907 [7,1,0]
+ CRUSH rule 1 x 908 [5,8,1]
+ CRUSH rule 1 x 909 [2,3,4]
+ CRUSH rule 1 x 910 [6,4,0]
+ CRUSH rule 1 x 911 [5,8,4]
+ CRUSH rule 1 x 912 [0,1,7]
+ CRUSH rule 1 x 913 [7,6,8]
+ CRUSH rule 1 x 914 [6,4,7]
+ CRUSH rule 1 x 915 [8,2,6]
+ CRUSH rule 1 x 916 [3,1,4]
+ CRUSH rule 1 x 917 [1,5,3]
+ CRUSH rule 1 x 918 [8,2,1]
+ CRUSH rule 1 x 919 [6,2,8]
+ CRUSH rule 1 x 920 [7,6,4]
+ CRUSH rule 1 x 921 [1,4,5]
+ CRUSH rule 1 x 922 [6,7,8]
+ CRUSH rule 1 x 923 [5,3,6]
+ CRUSH rule 1 x 924 [3,5,4]
+ CRUSH rule 1 x 925 [5,7,3]
+ CRUSH rule 1 x 926 [3,4,5]
+ CRUSH rule 1 x 927 [1,6,3]
+ CRUSH rule 1 x 928 [8,1,2]
+ CRUSH rule 1 x 929 [4,5,1]
+ CRUSH rule 1 x 930 [2,4,6]
+ CRUSH rule 1 x 931 [5,0,1]
+ CRUSH rule 1 x 932 [4,3,0]
+ CRUSH rule 1 x 933 [8,5,4]
+ CRUSH rule 1 x 934 [5,3,8]
+ CRUSH rule 1 x 935 [6,3,4]
+ CRUSH rule 1 x 936 [0,6,7]
+ CRUSH rule 1 x 937 [5,4,3]
+ CRUSH rule 1 x 938 [6,5,8]
+ CRUSH rule 1 x 939 [2,7,0]
+ CRUSH rule 1 x 940 [8,7,6]
+ CRUSH rule 1 x 941 [5,2,0]
+ CRUSH rule 1 x 942 [1,0,2]
+ CRUSH rule 1 x 943 [8,2,4]
+ CRUSH rule 1 x 944 [4,3,7]
+ CRUSH rule 1 x 945 [7,2,4]
+ CRUSH rule 1 x 946 [2,0,7]
+ CRUSH rule 1 x 947 [4,5,3]
+ CRUSH rule 1 x 948 [7,8,6]
+ CRUSH rule 1 x 949 [6,1,7]
+ CRUSH rule 1 x 950 [3,5,8]
+ CRUSH rule 1 x 951 [4,5,3]
+ CRUSH rule 1 x 952 [2,0,7]
+ CRUSH rule 1 x 953 [1,3,5]
+ CRUSH rule 1 x 954 [4,2,5]
+ CRUSH rule 1 x 955 [8,6,0]
+ CRUSH rule 1 x 956 [1,0,8]
+ CRUSH rule 1 x 957 [7,6,1]
+ CRUSH rule 1 x 958 [8,7,5]
+ CRUSH rule 1 x 959 [5,2,7]
+ CRUSH rule 1 x 960 [3,6,5]
+ CRUSH rule 1 x 961 [4,0,2]
+ CRUSH rule 1 x 962 [7,4,3]
+ CRUSH rule 1 x 963 [0,5,2]
+ CRUSH rule 1 x 964 [3,1,4]
+ CRUSH rule 1 x 965 [7,6,5]
+ CRUSH rule 1 x 966 [3,8,4]
+ CRUSH rule 1 x 967 [8,6,5]
+ CRUSH rule 1 x 968 [7,2,4]
+ CRUSH rule 1 x 969 [8,0,6]
+ CRUSH rule 1 x 970 [0,6,3]
+ CRUSH rule 1 x 971 [1,7,8]
+ CRUSH rule 1 x 972 [1,8,4]
+ CRUSH rule 1 x 973 [1,2,0]
+ CRUSH rule 1 x 974 [5,3,2]
+ CRUSH rule 1 x 975 [3,7,4]
+ CRUSH rule 1 x 976 [4,3,5]
+ CRUSH rule 1 x 977 [8,3,2]
+ CRUSH rule 1 x 978 [7,2,8]
+ CRUSH rule 1 x 979 [7,6,0]
+ CRUSH rule 1 x 980 [6,0,7]
+ CRUSH rule 1 x 981 [7,3,2]
+ CRUSH rule 1 x 982 [4,2,0]
+ CRUSH rule 1 x 983 [3,5,6]
+ CRUSH rule 1 x 984 [0,2,1]
+ CRUSH rule 1 x 985 [2,5,4]
+ CRUSH rule 1 x 986 [8,7,3]
+ CRUSH rule 1 x 987 [0,5,1]
+ CRUSH rule 1 x 988 [1,3,5]
+ CRUSH rule 1 x 989 [0,6,3]
+ CRUSH rule 1 x 990 [1,0,8]
+ CRUSH rule 1 x 991 [0,4,1]
+ CRUSH rule 1 x 992 [7,1,5]
+ CRUSH rule 1 x 993 [0,6,2]
+ CRUSH rule 1 x 994 [3,4,5]
+ CRUSH rule 1 x 995 [7,6,2]
+ CRUSH rule 1 x 996 [6,7,5]
+ CRUSH rule 1 x 997 [6,4,1]
+ CRUSH rule 1 x 998 [8,1,2]
+ CRUSH rule 1 x 999 [0,7,8]
+ CRUSH rule 1 x 1000 [8,5,0]
+ CRUSH rule 1 x 1001 [2,0,4]
+ CRUSH rule 1 x 1002 [1,3,2]
+ CRUSH rule 1 x 1003 [2,8,7]
+ CRUSH rule 1 x 1004 [6,1,2]
+ CRUSH rule 1 x 1005 [6,1,2]
+ CRUSH rule 1 x 1006 [1,0,2]
+ CRUSH rule 1 x 1007 [1,2,4]
+ CRUSH rule 1 x 1008 [1,7,0]
+ CRUSH rule 1 x 1009 [6,8,5]
+ CRUSH rule 1 x 1010 [3,4,0]
+ CRUSH rule 1 x 1011 [3,0,4]
+ CRUSH rule 1 x 1012 [3,0,7]
+ CRUSH rule 1 x 1013 [5,1,0]
+ CRUSH rule 1 x 1014 [2,8,4]
+ CRUSH rule 1 x 1015 [6,8,4]
+ CRUSH rule 1 x 1016 [2,0,1]
+ CRUSH rule 1 x 1017 [6,0,2]
+ CRUSH rule 1 x 1018 [5,4,3]
+ CRUSH rule 1 x 1019 [5,3,8]
+ CRUSH rule 1 x 1020 [5,1,3]
+ CRUSH rule 1 x 1021 [5,2,1]
+ CRUSH rule 1 x 1022 [1,6,7]
+ CRUSH rule 1 x 1023 [3,2,0]
+ rule 1 (choose-two) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 2 (chooseleaf), x = 0..1023, numrep = 2..3
+ CRUSH rule 2 x 0 [0,5]
+ CRUSH rule 2 x 1 [0,8]
+ CRUSH rule 2 x 2 [1,3]
+ CRUSH rule 2 x 3 [8,0]
+ CRUSH rule 2 x 4 [5,0]
+ CRUSH rule 2 x 5 [7,0]
+ CRUSH rule 2 x 6 [2,6]
+ CRUSH rule 2 x 7 [5,8]
+ CRUSH rule 2 x 8 [5,6]
+ CRUSH rule 2 x 9 [2,3]
+ CRUSH rule 2 x 10 [0,7]
+ CRUSH rule 2 x 11 [0,7]
+ CRUSH rule 2 x 12 [0,4]
+ CRUSH rule 2 x 13 [3,8]
+ CRUSH rule 2 x 14 [7,0]
+ CRUSH rule 2 x 15 [7,2]
+ CRUSH rule 2 x 16 [3,6]
+ CRUSH rule 2 x 17 [5,1]
+ CRUSH rule 2 x 18 [1,4]
+ CRUSH rule 2 x 19 [7,5]
+ CRUSH rule 2 x 20 [2,4]
+ CRUSH rule 2 x 21 [3,7]
+ CRUSH rule 2 x 22 [8,3]
+ CRUSH rule 2 x 23 [3,6]
+ CRUSH rule 2 x 24 [1,7]
+ CRUSH rule 2 x 25 [3,7]
+ CRUSH rule 2 x 26 [2,8]
+ CRUSH rule 2 x 27 [3,1]
+ CRUSH rule 2 x 28 [6,0]
+ CRUSH rule 2 x 29 [8,5]
+ CRUSH rule 2 x 30 [5,7]
+ CRUSH rule 2 x 31 [8,0]
+ CRUSH rule 2 x 32 [3,6]
+ CRUSH rule 2 x 33 [2,7]
+ CRUSH rule 2 x 34 [2,5]
+ CRUSH rule 2 x 35 [0,8]
+ CRUSH rule 2 x 36 [3,8]
+ CRUSH rule 2 x 37 [0,4]
+ CRUSH rule 2 x 38 [4,8]
+ CRUSH rule 2 x 39 [3,7]
+ CRUSH rule 2 x 40 [7,2]
+ CRUSH rule 2 x 41 [0,6]
+ CRUSH rule 2 x 42 [4,6]
+ CRUSH rule 2 x 43 [0,3]
+ CRUSH rule 2 x 44 [1,6]
+ CRUSH rule 2 x 45 [8,0]
+ CRUSH rule 2 x 46 [2,4]
+ CRUSH rule 2 x 47 [4,2]
+ CRUSH rule 2 x 48 [4,6]
+ CRUSH rule 2 x 49 [5,7]
+ CRUSH rule 2 x 50 [3,1]
+ CRUSH rule 2 x 51 [3,6]
+ CRUSH rule 2 x 52 [8,1]
+ CRUSH rule 2 x 53 [3,8]
+ CRUSH rule 2 x 54 [7,3]
+ CRUSH rule 2 x 55 [8,2]
+ CRUSH rule 2 x 56 [6,4]
+ CRUSH rule 2 x 57 [5,8]
+ CRUSH rule 2 x 58 [1,8]
+ CRUSH rule 2 x 59 [4,2]
+ CRUSH rule 2 x 60 [3,2]
+ CRUSH rule 2 x 61 [4,6]
+ CRUSH rule 2 x 62 [7,0]
+ CRUSH rule 2 x 63 [5,6]
+ CRUSH rule 2 x 64 [4,2]
+ CRUSH rule 2 x 65 [7,3]
+ CRUSH rule 2 x 66 [5,6]
+ CRUSH rule 2 x 67 [5,0]
+ CRUSH rule 2 x 68 [0,5]
+ CRUSH rule 2 x 69 [5,1]
+ CRUSH rule 2 x 70 [7,0]
+ CRUSH rule 2 x 71 [2,8]
+ CRUSH rule 2 x 72 [6,1]
+ CRUSH rule 2 x 73 [2,7]
+ CRUSH rule 2 x 74 [0,7]
+ CRUSH rule 2 x 75 [3,2]
+ CRUSH rule 2 x 76 [5,1]
+ CRUSH rule 2 x 77 [7,2]
+ CRUSH rule 2 x 78 [1,4]
+ CRUSH rule 2 x 79 [5,1]
+ CRUSH rule 2 x 80 [0,3]
+ CRUSH rule 2 x 81 [0,3]
+ CRUSH rule 2 x 82 [7,1]
+ CRUSH rule 2 x 83 [2,6]
+ CRUSH rule 2 x 84 [7,2]
+ CRUSH rule 2 x 85 [3,8]
+ CRUSH rule 2 x 86 [0,6]
+ CRUSH rule 2 x 87 [0,7]
+ CRUSH rule 2 x 88 [1,6]
+ CRUSH rule 2 x 89 [3,0]
+ CRUSH rule 2 x 90 [6,4]
+ CRUSH rule 2 x 91 [3,8]
+ CRUSH rule 2 x 92 [1,8]
+ CRUSH rule 2 x 93 [7,4]
+ CRUSH rule 2 x 94 [0,4]
+ CRUSH rule 2 x 95 [7,5]
+ CRUSH rule 2 x 96 [3,6]
+ CRUSH rule 2 x 97 [8,4]
+ CRUSH rule 2 x 98 [2,7]
+ CRUSH rule 2 x 99 [0,7]
+ CRUSH rule 2 x 100 [1,7]
+ CRUSH rule 2 x 101 [3,7]
+ CRUSH rule 2 x 102 [4,2]
+ CRUSH rule 2 x 103 [4,7]
+ CRUSH rule 2 x 104 [7,4]
+ CRUSH rule 2 x 105 [2,4]
+ CRUSH rule 2 x 106 [1,6]
+ CRUSH rule 2 x 107 [3,2]
+ CRUSH rule 2 x 108 [7,2]
+ CRUSH rule 2 x 109 [1,4]
+ CRUSH rule 2 x 110 [3,2]
+ CRUSH rule 2 x 111 [2,3]
+ CRUSH rule 2 x 112 [2,6]
+ CRUSH rule 2 x 113 [6,2]
+ CRUSH rule 2 x 114 [7,3]
+ CRUSH rule 2 x 115 [8,2]
+ CRUSH rule 2 x 116 [1,6]
+ CRUSH rule 2 x 117 [7,3]
+ CRUSH rule 2 x 118 [0,3]
+ CRUSH rule 2 x 119 [5,6]
+ CRUSH rule 2 x 120 [0,3]
+ CRUSH rule 2 x 121 [2,7]
+ CRUSH rule 2 x 122 [8,5]
+ CRUSH rule 2 x 123 [2,5]
+ CRUSH rule 2 x 124 [3,2]
+ CRUSH rule 2 x 125 [0,7]
+ CRUSH rule 2 x 126 [4,2]
+ CRUSH rule 2 x 127 [3,6]
+ CRUSH rule 2 x 128 [3,6]
+ CRUSH rule 2 x 129 [0,3]
+ CRUSH rule 2 x 130 [3,8]
+ CRUSH rule 2 x 131 [1,3]
+ CRUSH rule 2 x 132 [1,4]
+ CRUSH rule 2 x 133 [3,6]
+ CRUSH rule 2 x 134 [1,8]
+ CRUSH rule 2 x 135 [5,6]
+ CRUSH rule 2 x 136 [2,3]
+ CRUSH rule 2 x 137 [7,3]
+ CRUSH rule 2 x 138 [8,4]
+ CRUSH rule 2 x 139 [3,0]
+ CRUSH rule 2 x 140 [1,6]
+ CRUSH rule 2 x 141 [6,2]
+ CRUSH rule 2 x 142 [3,0]
+ CRUSH rule 2 x 143 [5,8]
+ CRUSH rule 2 x 144 [8,1]
+ CRUSH rule 2 x 145 [8,5]
+ CRUSH rule 2 x 146 [2,6]
+ CRUSH rule 2 x 147 [2,8]
+ CRUSH rule 2 x 148 [3,1]
+ CRUSH rule 2 x 149 [4,8]
+ CRUSH rule 2 x 150 [1,6]
+ CRUSH rule 2 x 151 [3,6]
+ CRUSH rule 2 x 152 [8,4]
+ CRUSH rule 2 x 153 [8,4]
+ CRUSH rule 2 x 154 [3,2]
+ CRUSH rule 2 x 155 [3,7]
+ CRUSH rule 2 x 156 [4,2]
+ CRUSH rule 2 x 157 [4,1]
+ CRUSH rule 2 x 158 [2,8]
+ CRUSH rule 2 x 159 [7,0]
+ CRUSH rule 2 x 160 [2,8]
+ CRUSH rule 2 x 161 [1,5]
+ CRUSH rule 2 x 162 [0,6]
+ CRUSH rule 2 x 163 [5,6]
+ CRUSH rule 2 x 164 [7,1]
+ CRUSH rule 2 x 165 [7,0]
+ CRUSH rule 2 x 166 [2,4]
+ CRUSH rule 2 x 167 [0,7]
+ CRUSH rule 2 x 168 [4,2]
+ CRUSH rule 2 x 169 [2,6]
+ CRUSH rule 2 x 170 [1,4]
+ CRUSH rule 2 x 171 [7,5]
+ CRUSH rule 2 x 172 [0,7]
+ CRUSH rule 2 x 173 [8,5]
+ CRUSH rule 2 x 174 [1,4]
+ CRUSH rule 2 x 175 [6,0]
+ CRUSH rule 2 x 176 [4,1]
+ CRUSH rule 2 x 177 [5,1]
+ CRUSH rule 2 x 178 [3,0]
+ CRUSH rule 2 x 179 [4,1]
+ CRUSH rule 2 x 180 [3,8]
+ CRUSH rule 2 x 181 [6,2]
+ CRUSH rule 2 x 182 [8,5]
+ CRUSH rule 2 x 183 [7,5]
+ CRUSH rule 2 x 184 [5,7]
+ CRUSH rule 2 x 185 [6,1]
+ CRUSH rule 2 x 186 [2,5]
+ CRUSH rule 2 x 187 [1,6]
+ CRUSH rule 2 x 188 [1,8]
+ CRUSH rule 2 x 189 [0,7]
+ CRUSH rule 2 x 190 [4,0]
+ CRUSH rule 2 x 191 [7,1]
+ CRUSH rule 2 x 192 [5,0]
+ CRUSH rule 2 x 193 [4,2]
+ CRUSH rule 2 x 194 [1,3]
+ CRUSH rule 2 x 195 [6,4]
+ CRUSH rule 2 x 196 [6,0]
+ CRUSH rule 2 x 197 [6,5]
+ CRUSH rule 2 x 198 [2,5]
+ CRUSH rule 2 x 199 [0,5]
+ CRUSH rule 2 x 200 [0,5]
+ CRUSH rule 2 x 201 [7,1]
+ CRUSH rule 2 x 202 [6,3]
+ CRUSH rule 2 x 203 [4,8]
+ CRUSH rule 2 x 204 [2,3]
+ CRUSH rule 2 x 205 [0,7]
+ CRUSH rule 2 x 206 [0,7]
+ CRUSH rule 2 x 207 [3,0]
+ CRUSH rule 2 x 208 [7,1]
+ CRUSH rule 2 x 209 [1,8]
+ CRUSH rule 2 x 210 [1,4]
+ CRUSH rule 2 x 211 [5,2]
+ CRUSH rule 2 x 212 [7,5]
+ CRUSH rule 2 x 213 [8,4]
+ CRUSH rule 2 x 214 [4,8]
+ CRUSH rule 2 x 215 [8,1]
+ CRUSH rule 2 x 216 [5,0]
+ CRUSH rule 2 x 217 [0,7]
+ CRUSH rule 2 x 218 [0,7]
+ CRUSH rule 2 x 219 [4,8]
+ CRUSH rule 2 x 220 [5,7]
+ CRUSH rule 2 x 221 [3,6]
+ CRUSH rule 2 x 222 [6,4]
+ CRUSH rule 2 x 223 [1,3]
+ CRUSH rule 2 x 224 [1,5]
+ CRUSH rule 2 x 225 [8,2]
+ CRUSH rule 2 x 226 [7,2]
+ CRUSH rule 2 x 227 [3,1]
+ CRUSH rule 2 x 228 [5,6]
+ CRUSH rule 2 x 229 [3,8]
+ CRUSH rule 2 x 230 [4,7]
+ CRUSH rule 2 x 231 [4,7]
+ CRUSH rule 2 x 232 [2,7]
+ CRUSH rule 2 x 233 [3,7]
+ CRUSH rule 2 x 234 [0,3]
+ CRUSH rule 2 x 235 [3,8]
+ CRUSH rule 2 x 236 [5,2]
+ CRUSH rule 2 x 237 [4,7]
+ CRUSH rule 2 x 238 [4,2]
+ CRUSH rule 2 x 239 [8,4]
+ CRUSH rule 2 x 240 [5,7]
+ CRUSH rule 2 x 241 [3,1]
+ CRUSH rule 2 x 242 [3,2]
+ CRUSH rule 2 x 243 [4,7]
+ CRUSH rule 2 x 244 [4,6]
+ CRUSH rule 2 x 245 [7,0]
+ CRUSH rule 2 x 246 [1,5]
+ CRUSH rule 2 x 247 [6,0]
+ CRUSH rule 2 x 248 [8,0]
+ CRUSH rule 2 x 249 [2,4]
+ CRUSH rule 2 x 250 [2,5]
+ CRUSH rule 2 x 251 [2,3]
+ CRUSH rule 2 x 252 [3,7]
+ CRUSH rule 2 x 253 [3,2]
+ CRUSH rule 2 x 254 [3,2]
+ CRUSH rule 2 x 255 [1,7]
+ CRUSH rule 2 x 256 [5,7]
+ CRUSH rule 2 x 257 [2,8]
+ CRUSH rule 2 x 258 [5,0]
+ CRUSH rule 2 x 259 [4,6]
+ CRUSH rule 2 x 260 [3,6]
+ CRUSH rule 2 x 261 [8,5]
+ CRUSH rule 2 x 262 [5,6]
+ CRUSH rule 2 x 263 [6,1]
+ CRUSH rule 2 x 264 [3,6]
+ CRUSH rule 2 x 265 [8,5]
+ CRUSH rule 2 x 266 [8,2]
+ CRUSH rule 2 x 267 [2,3]
+ CRUSH rule 2 x 268 [0,7]
+ CRUSH rule 2 x 269 [0,8]
+ CRUSH rule 2 x 270 [5,0]
+ CRUSH rule 2 x 271 [7,5]
+ CRUSH rule 2 x 272 [2,8]
+ CRUSH rule 2 x 273 [3,1]
+ CRUSH rule 2 x 274 [6,3]
+ CRUSH rule 2 x 275 [4,7]
+ CRUSH rule 2 x 276 [7,1]
+ CRUSH rule 2 x 277 [6,4]
+ CRUSH rule 2 x 278 [6,1]
+ CRUSH rule 2 x 279 [8,3]
+ CRUSH rule 2 x 280 [0,6]
+ CRUSH rule 2 x 281 [8,0]
+ CRUSH rule 2 x 282 [3,1]
+ CRUSH rule 2 x 283 [8,2]
+ CRUSH rule 2 x 284 [6,3]
+ CRUSH rule 2 x 285 [5,7]
+ CRUSH rule 2 x 286 [2,6]
+ CRUSH rule 2 x 287 [0,4]
+ CRUSH rule 2 x 288 [8,0]
+ CRUSH rule 2 x 289 [4,6]
+ CRUSH rule 2 x 290 [1,3]
+ CRUSH rule 2 x 291 [0,3]
+ CRUSH rule 2 x 292 [8,0]
+ CRUSH rule 2 x 293 [6,0]
+ CRUSH rule 2 x 294 [7,4]
+ CRUSH rule 2 x 295 [4,8]
+ CRUSH rule 2 x 296 [3,1]
+ CRUSH rule 2 x 297 [6,2]
+ CRUSH rule 2 x 298 [1,5]
+ CRUSH rule 2 x 299 [2,8]
+ CRUSH rule 2 x 300 [8,3]
+ CRUSH rule 2 x 301 [0,8]
+ CRUSH rule 2 x 302 [3,0]
+ CRUSH rule 2 x 303 [7,5]
+ CRUSH rule 2 x 304 [2,7]
+ CRUSH rule 2 x 305 [5,8]
+ CRUSH rule 2 x 306 [0,7]
+ CRUSH rule 2 x 307 [0,7]
+ CRUSH rule 2 x 308 [0,8]
+ CRUSH rule 2 x 309 [7,4]
+ CRUSH rule 2 x 310 [4,1]
+ CRUSH rule 2 x 311 [3,6]
+ CRUSH rule 2 x 312 [2,6]
+ CRUSH rule 2 x 313 [5,1]
+ CRUSH rule 2 x 314 [4,2]
+ CRUSH rule 2 x 315 [2,4]
+ CRUSH rule 2 x 316 [6,3]
+ CRUSH rule 2 x 317 [2,6]
+ CRUSH rule 2 x 318 [8,1]
+ CRUSH rule 2 x 319 [5,0]
+ CRUSH rule 2 x 320 [3,7]
+ CRUSH rule 2 x 321 [1,3]
+ CRUSH rule 2 x 322 [2,7]
+ CRUSH rule 2 x 323 [4,7]
+ CRUSH rule 2 x 324 [7,0]
+ CRUSH rule 2 x 325 [4,6]
+ CRUSH rule 2 x 326 [3,2]
+ CRUSH rule 2 x 327 [0,6]
+ CRUSH rule 2 x 328 [7,4]
+ CRUSH rule 2 x 329 [5,6]
+ CRUSH rule 2 x 330 [3,7]
+ CRUSH rule 2 x 331 [2,6]
+ CRUSH rule 2 x 332 [2,4]
+ CRUSH rule 2 x 333 [6,5]
+ CRUSH rule 2 x 334 [8,3]
+ CRUSH rule 2 x 335 [7,1]
+ CRUSH rule 2 x 336 [4,6]
+ CRUSH rule 2 x 337 [7,2]
+ CRUSH rule 2 x 338 [5,6]
+ CRUSH rule 2 x 339 [7,5]
+ CRUSH rule 2 x 340 [2,8]
+ CRUSH rule 2 x 341 [5,1]
+ CRUSH rule 2 x 342 [0,7]
+ CRUSH rule 2 x 343 [6,3]
+ CRUSH rule 2 x 344 [6,0]
+ CRUSH rule 2 x 345 [4,7]
+ CRUSH rule 2 x 346 [8,0]
+ CRUSH rule 2 x 347 [3,1]
+ CRUSH rule 2 x 348 [8,0]
+ CRUSH rule 2 x 349 [1,6]
+ CRUSH rule 2 x 350 [8,5]
+ CRUSH rule 2 x 351 [3,6]
+ CRUSH rule 2 x 352 [1,8]
+ CRUSH rule 2 x 353 [6,4]
+ CRUSH rule 2 x 354 [0,3]
+ CRUSH rule 2 x 355 [3,8]
+ CRUSH rule 2 x 356 [3,0]
+ CRUSH rule 2 x 357 [6,1]
+ CRUSH rule 2 x 358 [2,8]
+ CRUSH rule 2 x 359 [6,1]
+ CRUSH rule 2 x 360 [5,2]
+ CRUSH rule 2 x 361 [8,4]
+ CRUSH rule 2 x 362 [4,1]
+ CRUSH rule 2 x 363 [4,0]
+ CRUSH rule 2 x 364 [2,5]
+ CRUSH rule 2 x 365 [6,5]
+ CRUSH rule 2 x 366 [7,2]
+ CRUSH rule 2 x 367 [4,0]
+ CRUSH rule 2 x 368 [7,4]
+ CRUSH rule 2 x 369 [3,7]
+ CRUSH rule 2 x 370 [8,2]
+ CRUSH rule 2 x 371 [1,3]
+ CRUSH rule 2 x 372 [3,1]
+ CRUSH rule 2 x 373 [0,6]
+ CRUSH rule 2 x 374 [3,8]
+ CRUSH rule 2 x 375 [6,4]
+ CRUSH rule 2 x 376 [7,1]
+ CRUSH rule 2 x 377 [1,3]
+ CRUSH rule 2 x 378 [0,8]
+ CRUSH rule 2 x 379 [8,5]
+ CRUSH rule 2 x 380 [2,5]
+ CRUSH rule 2 x 381 [0,4]
+ CRUSH rule 2 x 382 [1,5]
+ CRUSH rule 2 x 383 [4,6]
+ CRUSH rule 2 x 384 [7,0]
+ CRUSH rule 2 x 385 [7,4]
+ CRUSH rule 2 x 386 [0,3]
+ CRUSH rule 2 x 387 [1,3]
+ CRUSH rule 2 x 388 [5,0]
+ CRUSH rule 2 x 389 [1,5]
+ CRUSH rule 2 x 390 [5,6]
+ CRUSH rule 2 x 391 [5,6]
+ CRUSH rule 2 x 392 [1,8]
+ CRUSH rule 2 x 393 [4,2]
+ CRUSH rule 2 x 394 [4,7]
+ CRUSH rule 2 x 395 [4,0]
+ CRUSH rule 2 x 396 [4,2]
+ CRUSH rule 2 x 397 [2,4]
+ CRUSH rule 2 x 398 [2,4]
+ CRUSH rule 2 x 399 [8,4]
+ CRUSH rule 2 x 400 [8,1]
+ CRUSH rule 2 x 401 [0,5]
+ CRUSH rule 2 x 402 [7,5]
+ CRUSH rule 2 x 403 [0,3]
+ CRUSH rule 2 x 404 [4,2]
+ CRUSH rule 2 x 405 [6,5]
+ CRUSH rule 2 x 406 [2,6]
+ CRUSH rule 2 x 407 [2,8]
+ CRUSH rule 2 x 408 [4,1]
+ CRUSH rule 2 x 409 [7,3]
+ CRUSH rule 2 x 410 [8,3]
+ CRUSH rule 2 x 411 [2,8]
+ CRUSH rule 2 x 412 [0,5]
+ CRUSH rule 2 x 413 [5,0]
+ CRUSH rule 2 x 414 [4,1]
+ CRUSH rule 2 x 415 [0,6]
+ CRUSH rule 2 x 416 [2,6]
+ CRUSH rule 2 x 417 [8,2]
+ CRUSH rule 2 x 418 [7,1]
+ CRUSH rule 2 x 419 [8,3]
+ CRUSH rule 2 x 420 [1,4]
+ CRUSH rule 2 x 421 [8,4]
+ CRUSH rule 2 x 422 [6,4]
+ CRUSH rule 2 x 423 [0,5]
+ CRUSH rule 2 x 424 [8,4]
+ CRUSH rule 2 x 425 [1,3]
+ CRUSH rule 2 x 426 [6,0]
+ CRUSH rule 2 x 427 [0,7]
+ CRUSH rule 2 x 428 [5,7]
+ CRUSH rule 2 x 429 [4,6]
+ CRUSH rule 2 x 430 [3,6]
+ CRUSH rule 2 x 431 [5,0]
+ CRUSH rule 2 x 432 [7,1]
+ CRUSH rule 2 x 433 [6,5]
+ CRUSH rule 2 x 434 [5,2]
+ CRUSH rule 2 x 435 [0,5]
+ CRUSH rule 2 x 436 [4,0]
+ CRUSH rule 2 x 437 [7,5]
+ CRUSH rule 2 x 438 [0,3]
+ CRUSH rule 2 x 439 [1,3]
+ CRUSH rule 2 x 440 [2,7]
+ CRUSH rule 2 x 441 [5,7]
+ CRUSH rule 2 x 442 [2,4]
+ CRUSH rule 2 x 443 [6,0]
+ CRUSH rule 2 x 444 [7,0]
+ CRUSH rule 2 x 445 [6,3]
+ CRUSH rule 2 x 446 [4,1]
+ CRUSH rule 2 x 447 [2,3]
+ CRUSH rule 2 x 448 [7,2]
+ CRUSH rule 2 x 449 [7,5]
+ CRUSH rule 2 x 450 [4,1]
+ CRUSH rule 2 x 451 [6,5]
+ CRUSH rule 2 x 452 [8,3]
+ CRUSH rule 2 x 453 [6,5]
+ CRUSH rule 2 x 454 [6,4]
+ CRUSH rule 2 x 455 [2,7]
+ CRUSH rule 2 x 456 [6,2]
+ CRUSH rule 2 x 457 [7,2]
+ CRUSH rule 2 x 458 [2,8]
+ CRUSH rule 2 x 459 [2,7]
+ CRUSH rule 2 x 460 [6,5]
+ CRUSH rule 2 x 461 [6,5]
+ CRUSH rule 2 x 462 [8,1]
+ CRUSH rule 2 x 463 [6,0]
+ CRUSH rule 2 x 464 [7,4]
+ CRUSH rule 2 x 465 [7,2]
+ CRUSH rule 2 x 466 [5,8]
+ CRUSH rule 2 x 467 [6,4]
+ CRUSH rule 2 x 468 [7,0]
+ CRUSH rule 2 x 469 [7,0]
+ CRUSH rule 2 x 470 [3,0]
+ CRUSH rule 2 x 471 [0,7]
+ CRUSH rule 2 x 472 [5,1]
+ CRUSH rule 2 x 473 [1,4]
+ CRUSH rule 2 x 474 [6,0]
+ CRUSH rule 2 x 475 [6,2]
+ CRUSH rule 2 x 476 [4,6]
+ CRUSH rule 2 x 477 [5,8]
+ CRUSH rule 2 x 478 [6,2]
+ CRUSH rule 2 x 479 [0,5]
+ CRUSH rule 2 x 480 [1,8]
+ CRUSH rule 2 x 481 [2,4]
+ CRUSH rule 2 x 482 [4,7]
+ CRUSH rule 2 x 483 [0,6]
+ CRUSH rule 2 x 484 [1,7]
+ CRUSH rule 2 x 485 [4,7]
+ CRUSH rule 2 x 486 [4,1]
+ CRUSH rule 2 x 487 [5,0]
+ CRUSH rule 2 x 488 [5,7]
+ CRUSH rule 2 x 489 [2,8]
+ CRUSH rule 2 x 490 [6,4]
+ CRUSH rule 2 x 491 [1,7]
+ CRUSH rule 2 x 492 [6,5]
+ CRUSH rule 2 x 493 [0,7]
+ CRUSH rule 2 x 494 [1,7]
+ CRUSH rule 2 x 495 [3,1]
+ CRUSH rule 2 x 496 [7,5]
+ CRUSH rule 2 x 497 [5,7]
+ CRUSH rule 2 x 498 [0,5]
+ CRUSH rule 2 x 499 [8,4]
+ CRUSH rule 2 x 500 [3,6]
+ CRUSH rule 2 x 501 [0,7]
+ CRUSH rule 2 x 502 [7,1]
+ CRUSH rule 2 x 503 [2,3]
+ CRUSH rule 2 x 504 [5,6]
+ CRUSH rule 2 x 505 [0,7]
+ CRUSH rule 2 x 506 [5,2]
+ CRUSH rule 2 x 507 [6,0]
+ CRUSH rule 2 x 508 [0,3]
+ CRUSH rule 2 x 509 [7,5]
+ CRUSH rule 2 x 510 [6,0]
+ CRUSH rule 2 x 511 [5,8]
+ CRUSH rule 2 x 512 [7,0]
+ CRUSH rule 2 x 513 [7,2]
+ CRUSH rule 2 x 514 [4,6]
+ CRUSH rule 2 x 515 [8,5]
+ CRUSH rule 2 x 516 [4,0]
+ CRUSH rule 2 x 517 [7,2]
+ CRUSH rule 2 x 518 [4,6]
+ CRUSH rule 2 x 519 [7,3]
+ CRUSH rule 2 x 520 [2,6]
+ CRUSH rule 2 x 521 [8,0]
+ CRUSH rule 2 x 522 [6,0]
+ CRUSH rule 2 x 523 [4,2]
+ CRUSH rule 2 x 524 [0,4]
+ CRUSH rule 2 x 525 [0,4]
+ CRUSH rule 2 x 526 [1,5]
+ CRUSH rule 2 x 527 [0,5]
+ CRUSH rule 2 x 528 [5,0]
+ CRUSH rule 2 x 529 [5,7]
+ CRUSH rule 2 x 530 [6,5]
+ CRUSH rule 2 x 531 [6,1]
+ CRUSH rule 2 x 532 [6,3]
+ CRUSH rule 2 x 533 [5,6]
+ CRUSH rule 2 x 534 [7,3]
+ CRUSH rule 2 x 535 [8,1]
+ CRUSH rule 2 x 536 [6,2]
+ CRUSH rule 2 x 537 [3,7]
+ CRUSH rule 2 x 538 [6,3]
+ CRUSH rule 2 x 539 [8,3]
+ CRUSH rule 2 x 540 [0,6]
+ CRUSH rule 2 x 541 [2,3]
+ CRUSH rule 2 x 542 [3,2]
+ CRUSH rule 2 x 543 [6,0]
+ CRUSH rule 2 x 544 [3,7]
+ CRUSH rule 2 x 545 [5,7]
+ CRUSH rule 2 x 546 [6,1]
+ CRUSH rule 2 x 547 [8,2]
+ CRUSH rule 2 x 548 [5,2]
+ CRUSH rule 2 x 549 [5,8]
+ CRUSH rule 2 x 550 [0,5]
+ CRUSH rule 2 x 551 [7,5]
+ CRUSH rule 2 x 552 [5,8]
+ CRUSH rule 2 x 553 [4,2]
+ CRUSH rule 2 x 554 [0,8]
+ CRUSH rule 2 x 555 [5,0]
+ CRUSH rule 2 x 556 [3,6]
+ CRUSH rule 2 x 557 [7,4]
+ CRUSH rule 2 x 558 [3,1]
+ CRUSH rule 2 x 559 [4,2]
+ CRUSH rule 2 x 560 [8,3]
+ CRUSH rule 2 x 561 [6,3]
+ CRUSH rule 2 x 562 [3,0]
+ CRUSH rule 2 x 563 [2,6]
+ CRUSH rule 2 x 564 [5,1]
+ CRUSH rule 2 x 565 [3,6]
+ CRUSH rule 2 x 566 [4,7]
+ CRUSH rule 2 x 567 [3,6]
+ CRUSH rule 2 x 568 [7,4]
+ CRUSH rule 2 x 569 [3,1]
+ CRUSH rule 2 x 570 [1,5]
+ CRUSH rule 2 x 571 [3,7]
+ CRUSH rule 2 x 572 [3,2]
+ CRUSH rule 2 x 573 [3,0]
+ CRUSH rule 2 x 574 [2,5]
+ CRUSH rule 2 x 575 [8,2]
+ CRUSH rule 2 x 576 [4,6]
+ CRUSH rule 2 x 577 [8,2]
+ CRUSH rule 2 x 578 [6,1]
+ CRUSH rule 2 x 579 [3,1]
+ CRUSH rule 2 x 580 [3,0]
+ CRUSH rule 2 x 581 [7,2]
+ CRUSH rule 2 x 582 [2,8]
+ CRUSH rule 2 x 583 [6,0]
+ CRUSH rule 2 x 584 [8,1]
+ CRUSH rule 2 x 585 [7,0]
+ CRUSH rule 2 x 586 [0,7]
+ CRUSH rule 2 x 587 [2,5]
+ CRUSH rule 2 x 588 [3,7]
+ CRUSH rule 2 x 589 [7,1]
+ CRUSH rule 2 x 590 [6,2]
+ CRUSH rule 2 x 591 [5,2]
+ CRUSH rule 2 x 592 [2,4]
+ CRUSH rule 2 x 593 [0,8]
+ CRUSH rule 2 x 594 [0,7]
+ CRUSH rule 2 x 595 [7,1]
+ CRUSH rule 2 x 596 [4,0]
+ CRUSH rule 2 x 597 [3,1]
+ CRUSH rule 2 x 598 [3,2]
+ CRUSH rule 2 x 599 [5,2]
+ CRUSH rule 2 x 600 [7,0]
+ CRUSH rule 2 x 601 [0,7]
+ CRUSH rule 2 x 602 [3,7]
+ CRUSH rule 2 x 603 [5,1]
+ CRUSH rule 2 x 604 [7,5]
+ CRUSH rule 2 x 605 [3,0]
+ CRUSH rule 2 x 606 [2,7]
+ CRUSH rule 2 x 607 [0,4]
+ CRUSH rule 2 x 608 [5,2]
+ CRUSH rule 2 x 609 [5,2]
+ CRUSH rule 2 x 610 [3,7]
+ CRUSH rule 2 x 611 [1,8]
+ CRUSH rule 2 x 612 [2,6]
+ CRUSH rule 2 x 613 [7,2]
+ CRUSH rule 2 x 614 [7,2]
+ CRUSH rule 2 x 615 [6,0]
+ CRUSH rule 2 x 616 [0,8]
+ CRUSH rule 2 x 617 [6,1]
+ CRUSH rule 2 x 618 [7,4]
+ CRUSH rule 2 x 619 [5,1]
+ CRUSH rule 2 x 620 [4,1]
+ CRUSH rule 2 x 621 [5,8]
+ CRUSH rule 2 x 622 [0,4]
+ CRUSH rule 2 x 623 [0,6]
+ CRUSH rule 2 x 624 [3,2]
+ CRUSH rule 2 x 625 [2,3]
+ CRUSH rule 2 x 626 [7,0]
+ CRUSH rule 2 x 627 [2,7]
+ CRUSH rule 2 x 628 [8,0]
+ CRUSH rule 2 x 629 [2,6]
+ CRUSH rule 2 x 630 [2,7]
+ CRUSH rule 2 x 631 [0,7]
+ CRUSH rule 2 x 632 [7,0]
+ CRUSH rule 2 x 633 [8,3]
+ CRUSH rule 2 x 634 [0,4]
+ CRUSH rule 2 x 635 [5,6]
+ CRUSH rule 2 x 636 [1,4]
+ CRUSH rule 2 x 637 [4,1]
+ CRUSH rule 2 x 638 [6,0]
+ CRUSH rule 2 x 639 [4,2]
+ CRUSH rule 2 x 640 [3,1]
+ CRUSH rule 2 x 641 [7,2]
+ CRUSH rule 2 x 642 [2,7]
+ CRUSH rule 2 x 643 [3,0]
+ CRUSH rule 2 x 644 [8,1]
+ CRUSH rule 2 x 645 [5,7]
+ CRUSH rule 2 x 646 [8,0]
+ CRUSH rule 2 x 647 [7,1]
+ CRUSH rule 2 x 648 [0,6]
+ CRUSH rule 2 x 649 [4,7]
+ CRUSH rule 2 x 650 [7,3]
+ CRUSH rule 2 x 651 [3,7]
+ CRUSH rule 2 x 652 [3,7]
+ CRUSH rule 2 x 653 [8,5]
+ CRUSH rule 2 x 654 [7,5]
+ CRUSH rule 2 x 655 [0,3]
+ CRUSH rule 2 x 656 [4,7]
+ CRUSH rule 2 x 657 [6,1]
+ CRUSH rule 2 x 658 [5,6]
+ CRUSH rule 2 x 659 [4,6]
+ CRUSH rule 2 x 660 [7,4]
+ CRUSH rule 2 x 661 [1,8]
+ CRUSH rule 2 x 662 [4,2]
+ CRUSH rule 2 x 663 [1,3]
+ CRUSH rule 2 x 664 [1,4]
+ CRUSH rule 2 x 665 [5,7]
+ CRUSH rule 2 x 666 [2,8]
+ CRUSH rule 2 x 667 [1,3]
+ CRUSH rule 2 x 668 [3,7]
+ CRUSH rule 2 x 669 [6,4]
+ CRUSH rule 2 x 670 [4,0]
+ CRUSH rule 2 x 671 [0,7]
+ CRUSH rule 2 x 672 [4,2]
+ CRUSH rule 2 x 673 [5,2]
+ CRUSH rule 2 x 674 [3,1]
+ CRUSH rule 2 x 675 [0,8]
+ CRUSH rule 2 x 676 [0,4]
+ CRUSH rule 2 x 677 [4,1]
+ CRUSH rule 2 x 678 [2,3]
+ CRUSH rule 2 x 679 [6,0]
+ CRUSH rule 2 x 680 [0,4]
+ CRUSH rule 2 x 681 [4,7]
+ CRUSH rule 2 x 682 [0,5]
+ CRUSH rule 2 x 683 [0,5]
+ CRUSH rule 2 x 684 [7,1]
+ CRUSH rule 2 x 685 [7,1]
+ CRUSH rule 2 x 686 [1,4]
+ CRUSH rule 2 x 687 [3,6]
+ CRUSH rule 2 x 688 [5,7]
+ CRUSH rule 2 x 689 [6,5]
+ CRUSH rule 2 x 690 [8,1]
+ CRUSH rule 2 x 691 [3,0]
+ CRUSH rule 2 x 692 [7,2]
+ CRUSH rule 2 x 693 [6,3]
+ CRUSH rule 2 x 694 [6,5]
+ CRUSH rule 2 x 695 [0,8]
+ CRUSH rule 2 x 696 [1,4]
+ CRUSH rule 2 x 697 [6,1]
+ CRUSH rule 2 x 698 [6,2]
+ CRUSH rule 2 x 699 [1,6]
+ CRUSH rule 2 x 700 [0,3]
+ CRUSH rule 2 x 701 [4,1]
+ CRUSH rule 2 x 702 [3,2]
+ CRUSH rule 2 x 703 [8,3]
+ CRUSH rule 2 x 704 [0,3]
+ CRUSH rule 2 x 705 [8,0]
+ CRUSH rule 2 x 706 [1,5]
+ CRUSH rule 2 x 707 [7,3]
+ CRUSH rule 2 x 708 [3,7]
+ CRUSH rule 2 x 709 [6,3]
+ CRUSH rule 2 x 710 [8,4]
+ CRUSH rule 2 x 711 [2,3]
+ CRUSH rule 2 x 712 [2,3]
+ CRUSH rule 2 x 713 [6,3]
+ CRUSH rule 2 x 714 [3,2]
+ CRUSH rule 2 x 715 [1,3]
+ CRUSH rule 2 x 716 [3,6]
+ CRUSH rule 2 x 717 [8,2]
+ CRUSH rule 2 x 718 [3,7]
+ CRUSH rule 2 x 719 [2,6]
+ CRUSH rule 2 x 720 [6,1]
+ CRUSH rule 2 x 721 [5,7]
+ CRUSH rule 2 x 722 [5,7]
+ CRUSH rule 2 x 723 [5,1]
+ CRUSH rule 2 x 724 [0,6]
+ CRUSH rule 2 x 725 [0,3]
+ CRUSH rule 2 x 726 [3,8]
+ CRUSH rule 2 x 727 [4,6]
+ CRUSH rule 2 x 728 [2,7]
+ CRUSH rule 2 x 729 [5,6]
+ CRUSH rule 2 x 730 [3,7]
+ CRUSH rule 2 x 731 [4,1]
+ CRUSH rule 2 x 732 [1,5]
+ CRUSH rule 2 x 733 [5,7]
+ CRUSH rule 2 x 734 [6,4]
+ CRUSH rule 2 x 735 [4,8]
+ CRUSH rule 2 x 736 [3,8]
+ CRUSH rule 2 x 737 [1,6]
+ CRUSH rule 2 x 738 [5,2]
+ CRUSH rule 2 x 739 [0,7]
+ CRUSH rule 2 x 740 [0,8]
+ CRUSH rule 2 x 741 [7,1]
+ CRUSH rule 2 x 742 [8,2]
+ CRUSH rule 2 x 743 [7,0]
+ CRUSH rule 2 x 744 [4,7]
+ CRUSH rule 2 x 745 [3,1]
+ CRUSH rule 2 x 746 [4,1]
+ CRUSH rule 2 x 747 [6,0]
+ CRUSH rule 2 x 748 [2,7]
+ CRUSH rule 2 x 749 [4,8]
+ CRUSH rule 2 x 750 [1,6]
+ CRUSH rule 2 x 751 [2,8]
+ CRUSH rule 2 x 752 [8,1]
+ CRUSH rule 2 x 753 [7,3]
+ CRUSH rule 2 x 754 [8,5]
+ CRUSH rule 2 x 755 [1,6]
+ CRUSH rule 2 x 756 [5,6]
+ CRUSH rule 2 x 757 [8,0]
+ CRUSH rule 2 x 758 [6,0]
+ CRUSH rule 2 x 759 [8,5]
+ CRUSH rule 2 x 760 [1,5]
+ CRUSH rule 2 x 761 [4,1]
+ CRUSH rule 2 x 762 [2,7]
+ CRUSH rule 2 x 763 [8,5]
+ CRUSH rule 2 x 764 [1,7]
+ CRUSH rule 2 x 765 [6,5]
+ CRUSH rule 2 x 766 [8,5]
+ CRUSH rule 2 x 767 [1,8]
+ CRUSH rule 2 x 768 [8,3]
+ CRUSH rule 2 x 769 [6,2]
+ CRUSH rule 2 x 770 [6,0]
+ CRUSH rule 2 x 771 [7,0]
+ CRUSH rule 2 x 772 [8,3]
+ CRUSH rule 2 x 773 [3,1]
+ CRUSH rule 2 x 774 [4,6]
+ CRUSH rule 2 x 775 [6,4]
+ CRUSH rule 2 x 776 [7,2]
+ CRUSH rule 2 x 777 [3,1]
+ CRUSH rule 2 x 778 [1,8]
+ CRUSH rule 2 x 779 [2,7]
+ CRUSH rule 2 x 780 [0,5]
+ CRUSH rule 2 x 781 [6,3]
+ CRUSH rule 2 x 782 [5,0]
+ CRUSH rule 2 x 783 [7,1]
+ CRUSH rule 2 x 784 [0,4]
+ CRUSH rule 2 x 785 [6,1]
+ CRUSH rule 2 x 786 [7,3]
+ CRUSH rule 2 x 787 [1,6]
+ CRUSH rule 2 x 788 [6,0]
+ CRUSH rule 2 x 789 [0,4]
+ CRUSH rule 2 x 790 [8,4]
+ CRUSH rule 2 x 791 [3,8]
+ CRUSH rule 2 x 792 [5,8]
+ CRUSH rule 2 x 793 [6,1]
+ CRUSH rule 2 x 794 [2,6]
+ CRUSH rule 2 x 795 [0,3]
+ CRUSH rule 2 x 796 [3,7]
+ CRUSH rule 2 x 797 [2,3]
+ CRUSH rule 2 x 798 [6,1]
+ CRUSH rule 2 x 799 [5,1]
+ CRUSH rule 2 x 800 [5,0]
+ CRUSH rule 2 x 801 [3,6]
+ CRUSH rule 2 x 802 [1,8]
+ CRUSH rule 2 x 803 [0,5]
+ CRUSH rule 2 x 804 [6,2]
+ CRUSH rule 2 x 805 [3,6]
+ CRUSH rule 2 x 806 [1,3]
+ CRUSH rule 2 x 807 [5,7]
+ CRUSH rule 2 x 808 [4,6]
+ CRUSH rule 2 x 809 [1,4]
+ CRUSH rule 2 x 810 [5,7]
+ CRUSH rule 2 x 811 [8,4]
+ CRUSH rule 2 x 812 [8,5]
+ CRUSH rule 2 x 813 [6,4]
+ CRUSH rule 2 x 814 [3,6]
+ CRUSH rule 2 x 815 [3,1]
+ CRUSH rule 2 x 816 [2,7]
+ CRUSH rule 2 x 817 [4,8]
+ CRUSH rule 2 x 818 [3,0]
+ CRUSH rule 2 x 819 [5,1]
+ CRUSH rule 2 x 820 [3,6]
+ CRUSH rule 2 x 821 [4,6]
+ CRUSH rule 2 x 822 [2,5]
+ CRUSH rule 2 x 823 [4,8]
+ CRUSH rule 2 x 824 [3,7]
+ CRUSH rule 2 x 825 [2,8]
+ CRUSH rule 2 x 826 [7,0]
+ CRUSH rule 2 x 827 [0,8]
+ CRUSH rule 2 x 828 [2,3]
+ CRUSH rule 2 x 829 [5,6]
+ CRUSH rule 2 x 830 [2,3]
+ CRUSH rule 2 x 831 [1,6]
+ CRUSH rule 2 x 832 [4,7]
+ CRUSH rule 2 x 833 [2,7]
+ CRUSH rule 2 x 834 [3,1]
+ CRUSH rule 2 x 835 [8,4]
+ CRUSH rule 2 x 836 [3,7]
+ CRUSH rule 2 x 837 [6,3]
+ CRUSH rule 2 x 838 [6,2]
+ CRUSH rule 2 x 839 [5,0]
+ CRUSH rule 2 x 840 [7,3]
+ CRUSH rule 2 x 841 [4,8]
+ CRUSH rule 2 x 842 [2,4]
+ CRUSH rule 2 x 843 [6,4]
+ CRUSH rule 2 x 844 [4,8]
+ CRUSH rule 2 x 845 [3,8]
+ CRUSH rule 2 x 846 [3,2]
+ CRUSH rule 2 x 847 [0,8]
+ CRUSH rule 2 x 848 [2,6]
+ CRUSH rule 2 x 849 [4,6]
+ CRUSH rule 2 x 850 [1,3]
+ CRUSH rule 2 x 851 [6,4]
+ CRUSH rule 2 x 852 [7,3]
+ CRUSH rule 2 x 853 [6,0]
+ CRUSH rule 2 x 854 [7,0]
+ CRUSH rule 2 x 855 [5,7]
+ CRUSH rule 2 x 856 [6,3]
+ CRUSH rule 2 x 857 [8,5]
+ CRUSH rule 2 x 858 [6,4]
+ CRUSH rule 2 x 859 [6,0]
+ CRUSH rule 2 x 860 [4,1]
+ CRUSH rule 2 x 861 [8,3]
+ CRUSH rule 2 x 862 [6,1]
+ CRUSH rule 2 x 863 [8,2]
+ CRUSH rule 2 x 864 [5,6]
+ CRUSH rule 2 x 865 [8,1]
+ CRUSH rule 2 x 866 [3,6]
+ CRUSH rule 2 x 867 [6,5]
+ CRUSH rule 2 x 868 [6,3]
+ CRUSH rule 2 x 869 [8,5]
+ CRUSH rule 2 x 870 [0,4]
+ CRUSH rule 2 x 871 [3,2]
+ CRUSH rule 2 x 872 [5,1]
+ CRUSH rule 2 x 873 [4,6]
+ CRUSH rule 2 x 874 [2,6]
+ CRUSH rule 2 x 875 [2,6]
+ CRUSH rule 2 x 876 [5,8]
+ CRUSH rule 2 x 877 [6,4]
+ CRUSH rule 2 x 878 [5,2]
+ CRUSH rule 2 x 879 [7,4]
+ CRUSH rule 2 x 880 [3,2]
+ CRUSH rule 2 x 881 [5,6]
+ CRUSH rule 2 x 882 [4,0]
+ CRUSH rule 2 x 883 [2,3]
+ CRUSH rule 2 x 884 [6,0]
+ CRUSH rule 2 x 885 [5,1]
+ CRUSH rule 2 x 886 [3,6]
+ CRUSH rule 2 x 887 [7,4]
+ CRUSH rule 2 x 888 [6,2]
+ CRUSH rule 2 x 889 [2,6]
+ CRUSH rule 2 x 890 [7,2]
+ CRUSH rule 2 x 891 [1,8]
+ CRUSH rule 2 x 892 [6,2]
+ CRUSH rule 2 x 893 [2,3]
+ CRUSH rule 2 x 894 [7,5]
+ CRUSH rule 2 x 895 [5,1]
+ CRUSH rule 2 x 896 [1,8]
+ CRUSH rule 2 x 897 [4,2]
+ CRUSH rule 2 x 898 [0,5]
+ CRUSH rule 2 x 899 [1,7]
+ CRUSH rule 2 x 900 [4,1]
+ CRUSH rule 2 x 901 [5,0]
+ CRUSH rule 2 x 902 [8,5]
+ CRUSH rule 2 x 903 [5,7]
+ CRUSH rule 2 x 904 [5,6]
+ CRUSH rule 2 x 905 [6,2]
+ CRUSH rule 2 x 906 [1,6]
+ CRUSH rule 2 x 907 [7,1]
+ CRUSH rule 2 x 908 [5,8]
+ CRUSH rule 2 x 909 [2,3]
+ CRUSH rule 2 x 910 [6,4]
+ CRUSH rule 2 x 911 [5,8]
+ CRUSH rule 2 x 912 [0,7]
+ CRUSH rule 2 x 913 [7,2]
+ CRUSH rule 2 x 914 [6,4]
+ CRUSH rule 2 x 915 [8,2]
+ CRUSH rule 2 x 916 [3,1]
+ CRUSH rule 2 x 917 [1,5]
+ CRUSH rule 2 x 918 [8,2]
+ CRUSH rule 2 x 919 [6,2]
+ CRUSH rule 2 x 920 [7,4]
+ CRUSH rule 2 x 921 [1,4]
+ CRUSH rule 2 x 922 [6,4]
+ CRUSH rule 2 x 923 [5,8]
+ CRUSH rule 2 x 924 [3,1]
+ CRUSH rule 2 x 925 [5,7]
+ CRUSH rule 2 x 926 [3,0]
+ CRUSH rule 2 x 927 [1,6]
+ CRUSH rule 2 x 928 [8,1]
+ CRUSH rule 2 x 929 [4,1]
+ CRUSH rule 2 x 930 [2,4]
+ CRUSH rule 2 x 931 [5,0]
+ CRUSH rule 2 x 932 [4,1]
+ CRUSH rule 2 x 933 [8,5]
+ CRUSH rule 2 x 934 [5,6]
+ CRUSH rule 2 x 935 [6,3]
+ CRUSH rule 2 x 936 [0,6]
+ CRUSH rule 2 x 937 [5,8]
+ CRUSH rule 2 x 938 [6,5]
+ CRUSH rule 2 x 939 [2,7]
+ CRUSH rule 2 x 940 [8,5]
+ CRUSH rule 2 x 941 [5,2]
+ CRUSH rule 2 x 942 [1,8]
+ CRUSH rule 2 x 943 [8,2]
+ CRUSH rule 2 x 944 [4,8]
+ CRUSH rule 2 x 945 [7,2]
+ CRUSH rule 2 x 946 [2,8]
+ CRUSH rule 2 x 947 [4,2]
+ CRUSH rule 2 x 948 [7,5]
+ CRUSH rule 2 x 949 [6,1]
+ CRUSH rule 2 x 950 [3,6]
+ CRUSH rule 2 x 951 [4,8]
+ CRUSH rule 2 x 952 [2,7]
+ CRUSH rule 2 x 953 [1,3]
+ CRUSH rule 2 x 954 [4,2]
+ CRUSH rule 2 x 955 [8,0]
+ CRUSH rule 2 x 956 [1,6]
+ CRUSH rule 2 x 957 [7,1]
+ CRUSH rule 2 x 958 [8,4]
+ CRUSH rule 2 x 959 [5,2]
+ CRUSH rule 2 x 960 [3,6]
+ CRUSH rule 2 x 961 [4,0]
+ CRUSH rule 2 x 962 [7,4]
+ CRUSH rule 2 x 963 [0,5]
+ CRUSH rule 2 x 964 [3,1]
+ CRUSH rule 2 x 965 [7,4]
+ CRUSH rule 2 x 966 [3,8]
+ CRUSH rule 2 x 967 [8,5]
+ CRUSH rule 2 x 968 [7,2]
+ CRUSH rule 2 x 969 [8,0]
+ CRUSH rule 2 x 970 [0,6]
+ CRUSH rule 2 x 971 [1,7]
+ CRUSH rule 2 x 972 [1,8]
+ CRUSH rule 2 x 973 [1,6]
+ CRUSH rule 2 x 974 [5,1]
+ CRUSH rule 2 x 975 [3,7]
+ CRUSH rule 2 x 976 [4,8]
+ CRUSH rule 2 x 977 [8,3]
+ CRUSH rule 2 x 978 [7,2]
+ CRUSH rule 2 x 979 [7,1]
+ CRUSH rule 2 x 980 [6,0]
+ CRUSH rule 2 x 981 [7,3]
+ CRUSH rule 2 x 982 [4,2]
+ CRUSH rule 2 x 983 [3,7]
+ CRUSH rule 2 x 984 [0,7]
+ CRUSH rule 2 x 985 [2,5]
+ CRUSH rule 2 x 986 [8,3]
+ CRUSH rule 2 x 987 [0,5]
+ CRUSH rule 2 x 988 [1,3]
+ CRUSH rule 2 x 989 [0,6]
+ CRUSH rule 2 x 990 [1,6]
+ CRUSH rule 2 x 991 [0,4]
+ CRUSH rule 2 x 992 [7,1]
+ CRUSH rule 2 x 993 [0,6]
+ CRUSH rule 2 x 994 [3,7]
+ CRUSH rule 2 x 995 [7,1]
+ CRUSH rule 2 x 996 [6,5]
+ CRUSH rule 2 x 997 [6,4]
+ CRUSH rule 2 x 998 [8,1]
+ CRUSH rule 2 x 999 [0,7]
+ CRUSH rule 2 x 1000 [8,5]
+ CRUSH rule 2 x 1001 [2,5]
+ CRUSH rule 2 x 1002 [1,3]
+ CRUSH rule 2 x 1003 [2,8]
+ CRUSH rule 2 x 1004 [6,1]
+ CRUSH rule 2 x 1005 [6,1]
+ CRUSH rule 2 x 1006 [1,6]
+ CRUSH rule 2 x 1007 [1,5]
+ CRUSH rule 2 x 1008 [1,7]
+ CRUSH rule 2 x 1009 [6,4]
+ CRUSH rule 2 x 1010 [3,1]
+ CRUSH rule 2 x 1011 [3,0]
+ CRUSH rule 2 x 1012 [3,0]
+ CRUSH rule 2 x 1013 [5,1]
+ CRUSH rule 2 x 1014 [2,8]
+ CRUSH rule 2 x 1015 [6,5]
+ CRUSH rule 2 x 1016 [2,4]
+ CRUSH rule 2 x 1017 [6,0]
+ CRUSH rule 2 x 1018 [5,0]
+ CRUSH rule 2 x 1019 [5,8]
+ CRUSH rule 2 x 1020 [5,1]
+ CRUSH rule 2 x 1021 [5,2]
+ CRUSH rule 2 x 1022 [1,6]
+ CRUSH rule 2 x 1023 [3,2]
+ rule 2 (chooseleaf) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 2 x 0 [0,5,7]
+ CRUSH rule 2 x 1 [0,8,5]
+ CRUSH rule 2 x 2 [1,3,7]
+ CRUSH rule 2 x 3 [8,0,4]
+ CRUSH rule 2 x 4 [5,0,7]
+ CRUSH rule 2 x 5 [7,0,4]
+ CRUSH rule 2 x 6 [2,6,3]
+ CRUSH rule 2 x 7 [5,8,2]
+ CRUSH rule 2 x 8 [5,6,0]
+ CRUSH rule 2 x 9 [2,3,8]
+ CRUSH rule 2 x 10 [0,7,4]
+ CRUSH rule 2 x 11 [0,7,4]
+ CRUSH rule 2 x 12 [0,4,6]
+ CRUSH rule 2 x 13 [3,8,1]
+ CRUSH rule 2 x 14 [7,0,5]
+ CRUSH rule 2 x 15 [7,2,4]
+ CRUSH rule 2 x 16 [3,6,0]
+ CRUSH rule 2 x 17 [5,1,6]
+ CRUSH rule 2 x 18 [1,4,6]
+ CRUSH rule 2 x 19 [7,5,0]
+ CRUSH rule 2 x 20 [2,4,7]
+ CRUSH rule 2 x 21 [3,7,2]
+ CRUSH rule 2 x 22 [8,3,1]
+ CRUSH rule 2 x 23 [3,6,2]
+ CRUSH rule 2 x 24 [1,7,3]
+ CRUSH rule 2 x 25 [3,7,0]
+ CRUSH rule 2 x 26 [2,8,4]
+ CRUSH rule 2 x 27 [3,1,8]
+ CRUSH rule 2 x 28 [6,0,3]
+ CRUSH rule 2 x 29 [8,5,2]
+ CRUSH rule 2 x 30 [5,7,0]
+ CRUSH rule 2 x 31 [8,0,4]
+ CRUSH rule 2 x 32 [3,6,2]
+ CRUSH rule 2 x 33 [2,7,5]
+ CRUSH rule 2 x 34 [2,5,7]
+ CRUSH rule 2 x 35 [0,8,5]
+ CRUSH rule 2 x 36 [3,8,2]
+ CRUSH rule 2 x 37 [0,4,6]
+ CRUSH rule 2 x 38 [4,8,2]
+ CRUSH rule 2 x 39 [3,7,0]
+ CRUSH rule 2 x 40 [7,2,3]
+ CRUSH rule 2 x 41 [0,6,4]
+ CRUSH rule 2 x 42 [4,6,2]
+ CRUSH rule 2 x 43 [0,3,7]
+ CRUSH rule 2 x 44 [1,6,5]
+ CRUSH rule 2 x 45 [8,0,4]
+ CRUSH rule 2 x 46 [2,4,8]
+ CRUSH rule 2 x 47 [4,2,8]
+ CRUSH rule 2 x 48 [4,6,2]
+ CRUSH rule 2 x 49 [5,7,2]
+ CRUSH rule 2 x 50 [3,1,7]
+ CRUSH rule 2 x 51 [3,6,0]
+ CRUSH rule 2 x 52 [8,1,4]
+ CRUSH rule 2 x 53 [3,8,0]
+ CRUSH rule 2 x 54 [7,3,1]
+ CRUSH rule 2 x 55 [8,2,4]
+ CRUSH rule 2 x 56 [6,4,2]
+ CRUSH rule 2 x 57 [5,8,1]
+ CRUSH rule 2 x 58 [1,8,5]
+ CRUSH rule 2 x 59 [4,2,7]
+ CRUSH rule 2 x 60 [3,2,6]
+ CRUSH rule 2 x 61 [4,6,0]
+ CRUSH rule 2 x 62 [7,0,5]
+ CRUSH rule 2 x 63 [5,6,0]
+ CRUSH rule 2 x 64 [4,2,8]
+ CRUSH rule 2 x 65 [7,3,0]
+ CRUSH rule 2 x 66 [5,6,2]
+ CRUSH rule 2 x 67 [5,0,8]
+ CRUSH rule 2 x 68 [0,5,8]
+ CRUSH rule 2 x 69 [5,1,6]
+ CRUSH rule 2 x 70 [7,0,5]
+ CRUSH rule 2 x 71 [2,8,5]
+ CRUSH rule 2 x 72 [6,1,5]
+ CRUSH rule 2 x 73 [2,7,3]
+ CRUSH rule 2 x 74 [0,7,3]
+ CRUSH rule 2 x 75 [3,2,6]
+ CRUSH rule 2 x 76 [5,1,7]
+ CRUSH rule 2 x 77 [7,2,5]
+ CRUSH rule 2 x 78 [1,4,8]
+ CRUSH rule 2 x 79 [5,1,7]
+ CRUSH rule 2 x 80 [0,3,6]
+ CRUSH rule 2 x 81 [0,3,6]
+ CRUSH rule 2 x 82 [7,1,5]
+ CRUSH rule 2 x 83 [2,6,3]
+ CRUSH rule 2 x 84 [7,2,4]
+ CRUSH rule 2 x 85 [3,8,0]
+ CRUSH rule 2 x 86 [0,6,3]
+ CRUSH rule 2 x 87 [0,7,4]
+ CRUSH rule 2 x 88 [1,6,5]
+ CRUSH rule 2 x 89 [3,0,7]
+ CRUSH rule 2 x 90 [6,4,1]
+ CRUSH rule 2 x 91 [3,8,1]
+ CRUSH rule 2 x 92 [1,8,4]
+ CRUSH rule 2 x 93 [7,4,2]
+ CRUSH rule 2 x 94 [0,4,8]
+ CRUSH rule 2 x 95 [7,5,1]
+ CRUSH rule 2 x 96 [3,6,1]
+ CRUSH rule 2 x 97 [8,4,0]
+ CRUSH rule 2 x 98 [2,7,5]
+ CRUSH rule 2 x 99 [0,7,3]
+ CRUSH rule 2 x 100 [1,7,4]
+ CRUSH rule 2 x 101 [3,7,2]
+ CRUSH rule 2 x 102 [4,2,7]
+ CRUSH rule 2 x 103 [4,7,0]
+ CRUSH rule 2 x 104 [7,4,1]
+ CRUSH rule 2 x 105 [2,4,6]
+ CRUSH rule 2 x 106 [1,6,5]
+ CRUSH rule 2 x 107 [3,2,6]
+ CRUSH rule 2 x 108 [7,2,5]
+ CRUSH rule 2 x 109 [1,4,6]
+ CRUSH rule 2 x 110 [3,2,7]
+ CRUSH rule 2 x 111 [2,3,6]
+ CRUSH rule 2 x 112 [2,6,4]
+ CRUSH rule 2 x 113 [6,2,4]
+ CRUSH rule 2 x 114 [7,3,1]
+ CRUSH rule 2 x 115 [8,2,3]
+ CRUSH rule 2 x 116 [1,6,3]
+ CRUSH rule 2 x 117 [7,3,2]
+ CRUSH rule 2 x 118 [0,3,8]
+ CRUSH rule 2 x 119 [5,6,1]
+ CRUSH rule 2 x 120 [0,3,7]
+ CRUSH rule 2 x 121 [2,7,5]
+ CRUSH rule 2 x 122 [8,5,0]
+ CRUSH rule 2 x 123 [2,5,8]
+ CRUSH rule 2 x 124 [3,2,8]
+ CRUSH rule 2 x 125 [0,7,3]
+ CRUSH rule 2 x 126 [4,2,6]
+ CRUSH rule 2 x 127 [3,6,2]
+ CRUSH rule 2 x 128 [3,6,1]
+ CRUSH rule 2 x 129 [0,3,7]
+ CRUSH rule 2 x 130 [3,8,2]
+ CRUSH rule 2 x 131 [1,3,8]
+ CRUSH rule 2 x 132 [1,4,6]
+ CRUSH rule 2 x 133 [3,6,2]
+ CRUSH rule 2 x 134 [1,8,5]
+ CRUSH rule 2 x 135 [5,6,2]
+ CRUSH rule 2 x 136 [2,3,6]
+ CRUSH rule 2 x 137 [7,3,2]
+ CRUSH rule 2 x 138 [8,4,2]
+ CRUSH rule 2 x 139 [3,0,7]
+ CRUSH rule 2 x 140 [1,6,4]
+ CRUSH rule 2 x 141 [6,2,3]
+ CRUSH rule 2 x 142 [3,0,8]
+ CRUSH rule 2 x 143 [5,8,1]
+ CRUSH rule 2 x 144 [8,1,3]
+ CRUSH rule 2 x 145 [8,5,0]
+ CRUSH rule 2 x 146 [2,6,4]
+ CRUSH rule 2 x 147 [2,8,4]
+ CRUSH rule 2 x 148 [3,1,7]
+ CRUSH rule 2 x 149 [4,8,1]
+ CRUSH rule 2 x 150 [1,6,5]
+ CRUSH rule 2 x 151 [3,6,0]
+ CRUSH rule 2 x 152 [8,4,2]
+ CRUSH rule 2 x 153 [8,4,0]
+ CRUSH rule 2 x 154 [3,2,7]
+ CRUSH rule 2 x 155 [3,7,2]
+ CRUSH rule 2 x 156 [4,2,6]
+ CRUSH rule 2 x 157 [4,1,6]
+ CRUSH rule 2 x 158 [2,8,5]
+ CRUSH rule 2 x 159 [7,0,5]
+ CRUSH rule 2 x 160 [2,8,5]
+ CRUSH rule 2 x 161 [1,5,6]
+ CRUSH rule 2 x 162 [0,6,4]
+ CRUSH rule 2 x 163 [5,6,1]
+ CRUSH rule 2 x 164 [7,1,5]
+ CRUSH rule 2 x 165 [7,0,5]
+ CRUSH rule 2 x 166 [2,4,6]
+ CRUSH rule 2 x 167 [0,7,4]
+ CRUSH rule 2 x 168 [4,2,7]
+ CRUSH rule 2 x 169 [2,6,4]
+ CRUSH rule 2 x 170 [1,4,8]
+ CRUSH rule 2 x 171 [7,5,0]
+ CRUSH rule 2 x 172 [0,7,5]
+ CRUSH rule 2 x 173 [8,5,1]
+ CRUSH rule 2 x 174 [1,4,7]
+ CRUSH rule 2 x 175 [6,0,4]
+ CRUSH rule 2 x 176 [4,1,7]
+ CRUSH rule 2 x 177 [5,1,6]
+ CRUSH rule 2 x 178 [3,0,8]
+ CRUSH rule 2 x 179 [4,1,7]
+ CRUSH rule 2 x 180 [3,8,1]
+ CRUSH rule 2 x 181 [6,2,4]
+ CRUSH rule 2 x 182 [8,5,1]
+ CRUSH rule 2 x 183 [7,5,1]
+ CRUSH rule 2 x 184 [5,7,2]
+ CRUSH rule 2 x 185 [6,1,4]
+ CRUSH rule 2 x 186 [2,5,7]
+ CRUSH rule 2 x 187 [1,6,4]
+ CRUSH rule 2 x 188 [1,8,5]
+ CRUSH rule 2 x 189 [0,7,4]
+ CRUSH rule 2 x 190 [4,0,8]
+ CRUSH rule 2 x 191 [7,1,3]
+ CRUSH rule 2 x 192 [5,0,8]
+ CRUSH rule 2 x 193 [4,2,6]
+ CRUSH rule 2 x 194 [1,3,8]
+ CRUSH rule 2 x 195 [6,4,1]
+ CRUSH rule 2 x 196 [6,0,3]
+ CRUSH rule 2 x 197 [6,5,2]
+ CRUSH rule 2 x 198 [2,5,6]
+ CRUSH rule 2 x 199 [0,5,7]
+ CRUSH rule 2 x 200 [0,5,6]
+ CRUSH rule 2 x 201 [7,1,5]
+ CRUSH rule 2 x 202 [6,3,1]
+ CRUSH rule 2 x 203 [4,8,1]
+ CRUSH rule 2 x 204 [2,3,7]
+ CRUSH rule 2 x 205 [0,7,5]
+ CRUSH rule 2 x 206 [0,7,5]
+ CRUSH rule 2 x 207 [3,0,7]
+ CRUSH rule 2 x 208 [7,1,4]
+ CRUSH rule 2 x 209 [1,8,4]
+ CRUSH rule 2 x 210 [1,4,8]
+ CRUSH rule 2 x 211 [5,2,8]
+ CRUSH rule 2 x 212 [7,5,0]
+ CRUSH rule 2 x 213 [8,4,0]
+ CRUSH rule 2 x 214 [4,8,2]
+ CRUSH rule 2 x 215 [8,1,4]
+ CRUSH rule 2 x 216 [5,0,6]
+ CRUSH rule 2 x 217 [0,7,3]
+ CRUSH rule 2 x 218 [0,7,5]
+ CRUSH rule 2 x 219 [4,8,2]
+ CRUSH rule 2 x 220 [5,7,1]
+ CRUSH rule 2 x 221 [3,6,0]
+ CRUSH rule 2 x 222 [6,4,1]
+ CRUSH rule 2 x 223 [1,3,6]
+ CRUSH rule 2 x 224 [1,5,8]
+ CRUSH rule 2 x 225 [8,2,3]
+ CRUSH rule 2 x 226 [7,2,3]
+ CRUSH rule 2 x 227 [3,1,6]
+ CRUSH rule 2 x 228 [5,6,0]
+ CRUSH rule 2 x 229 [3,8,2]
+ CRUSH rule 2 x 230 [4,7,2]
+ CRUSH rule 2 x 231 [4,7,2]
+ CRUSH rule 2 x 232 [2,7,4]
+ CRUSH rule 2 x 233 [3,7,2]
+ CRUSH rule 2 x 234 [0,3,6]
+ CRUSH rule 2 x 235 [3,8,0]
+ CRUSH rule 2 x 236 [5,2,7]
+ CRUSH rule 2 x 237 [4,7,1]
+ CRUSH rule 2 x 238 [4,2,6]
+ CRUSH rule 2 x 239 [8,4,1]
+ CRUSH rule 2 x 240 [5,7,0]
+ CRUSH rule 2 x 241 [3,1,8]
+ CRUSH rule 2 x 242 [3,2,6]
+ CRUSH rule 2 x 243 [4,7,0]
+ CRUSH rule 2 x 244 [4,6,0]
+ CRUSH rule 2 x 245 [7,0,5]
+ CRUSH rule 2 x 246 [1,5,8]
+ CRUSH rule 2 x 247 [6,0,4]
+ CRUSH rule 2 x 248 [8,0,3]
+ CRUSH rule 2 x 249 [2,4,7]
+ CRUSH rule 2 x 250 [2,5,6]
+ CRUSH rule 2 x 251 [2,3,6]
+ CRUSH rule 2 x 252 [3,7,0]
+ CRUSH rule 2 x 253 [3,2,6]
+ CRUSH rule 2 x 254 [3,2,7]
+ CRUSH rule 2 x 255 [1,7,5]
+ CRUSH rule 2 x 256 [5,7,0]
+ CRUSH rule 2 x 257 [2,8,4]
+ CRUSH rule 2 x 258 [5,0,6]
+ CRUSH rule 2 x 259 [4,6,2]
+ CRUSH rule 2 x 260 [3,6,2]
+ CRUSH rule 2 x 261 [8,5,2]
+ CRUSH rule 2 x 262 [5,6,0]
+ CRUSH rule 2 x 263 [6,1,5]
+ CRUSH rule 2 x 264 [3,6,2]
+ CRUSH rule 2 x 265 [8,5,2]
+ CRUSH rule 2 x 266 [8,2,5]
+ CRUSH rule 2 x 267 [2,3,8]
+ CRUSH rule 2 x 268 [0,7,4]
+ CRUSH rule 2 x 269 [0,8,4]
+ CRUSH rule 2 x 270 [5,0,7]
+ CRUSH rule 2 x 271 [7,5,1]
+ CRUSH rule 2 x 272 [2,8,3]
+ CRUSH rule 2 x 273 [3,1,7]
+ CRUSH rule 2 x 274 [6,3,1]
+ CRUSH rule 2 x 275 [4,7,0]
+ CRUSH rule 2 x 276 [7,1,4]
+ CRUSH rule 2 x 277 [6,4,0]
+ CRUSH rule 2 x 278 [6,1,4]
+ CRUSH rule 2 x 279 [8,3,2]
+ CRUSH rule 2 x 280 [0,6,4]
+ CRUSH rule 2 x 281 [8,0,3]
+ CRUSH rule 2 x 282 [3,1,6]
+ CRUSH rule 2 x 283 [8,2,5]
+ CRUSH rule 2 x 284 [6,3,0]
+ CRUSH rule 2 x 285 [5,7,0]
+ CRUSH rule 2 x 286 [2,6,3]
+ CRUSH rule 2 x 287 [0,4,6]
+ CRUSH rule 2 x 288 [8,0,4]
+ CRUSH rule 2 x 289 [4,6,2]
+ CRUSH rule 2 x 290 [1,3,7]
+ CRUSH rule 2 x 291 [0,3,8]
+ CRUSH rule 2 x 292 [8,0,5]
+ CRUSH rule 2 x 293 [6,0,4]
+ CRUSH rule 2 x 294 [7,4,1]
+ CRUSH rule 2 x 295 [4,8,0]
+ CRUSH rule 2 x 296 [3,1,6]
+ CRUSH rule 2 x 297 [6,2,4]
+ CRUSH rule 2 x 298 [1,5,7]
+ CRUSH rule 2 x 299 [2,8,3]
+ CRUSH rule 2 x 300 [8,3,0]
+ CRUSH rule 2 x 301 [0,8,4]
+ CRUSH rule 2 x 302 [3,0,6]
+ CRUSH rule 2 x 303 [7,5,1]
+ CRUSH rule 2 x 304 [2,7,5]
+ CRUSH rule 2 x 305 [5,8,2]
+ CRUSH rule 2 x 306 [0,7,3]
+ CRUSH rule 2 x 307 [0,7,5]
+ CRUSH rule 2 x 308 [0,8,4]
+ CRUSH rule 2 x 309 [7,4,0]
+ CRUSH rule 2 x 310 [4,1,7]
+ CRUSH rule 2 x 311 [3,6,0]
+ CRUSH rule 2 x 312 [2,6,3]
+ CRUSH rule 2 x 313 [5,1,6]
+ CRUSH rule 2 x 314 [4,2,6]
+ CRUSH rule 2 x 315 [2,4,8]
+ CRUSH rule 2 x 316 [6,3,1]
+ CRUSH rule 2 x 317 [2,6,5]
+ CRUSH rule 2 x 318 [8,1,4]
+ CRUSH rule 2 x 319 [5,0,8]
+ CRUSH rule 2 x 320 [3,7,1]
+ CRUSH rule 2 x 321 [1,3,8]
+ CRUSH rule 2 x 322 [2,7,3]
+ CRUSH rule 2 x 323 [4,7,0]
+ CRUSH rule 2 x 324 [7,0,3]
+ CRUSH rule 2 x 325 [4,6,0]
+ CRUSH rule 2 x 326 [3,2,6]
+ CRUSH rule 2 x 327 [0,6,3]
+ CRUSH rule 2 x 328 [7,4,1]
+ CRUSH rule 2 x 329 [5,6,2]
+ CRUSH rule 2 x 330 [3,7,2]
+ CRUSH rule 2 x 331 [2,6,3]
+ CRUSH rule 2 x 332 [2,4,6]
+ CRUSH rule 2 x 333 [6,5,1]
+ CRUSH rule 2 x 334 [8,3,2]
+ CRUSH rule 2 x 335 [7,1,4]
+ CRUSH rule 2 x 336 [4,6,0]
+ CRUSH rule 2 x 337 [7,2,3]
+ CRUSH rule 2 x 338 [5,6,1]
+ CRUSH rule 2 x 339 [7,5,2]
+ CRUSH rule 2 x 340 [2,8,4]
+ CRUSH rule 2 x 341 [5,1,7]
+ CRUSH rule 2 x 342 [0,7,4]
+ CRUSH rule 2 x 343 [6,3,0]
+ CRUSH rule 2 x 344 [6,0,5]
+ CRUSH rule 2 x 345 [4,7,1]
+ CRUSH rule 2 x 346 [8,0,5]
+ CRUSH rule 2 x 347 [3,1,7]
+ CRUSH rule 2 x 348 [8,0,5]
+ CRUSH rule 2 x 349 [1,6,4]
+ CRUSH rule 2 x 350 [8,5,1]
+ CRUSH rule 2 x 351 [3,6,0]
+ CRUSH rule 2 x 352 [1,8,3]
+ CRUSH rule 2 x 353 [6,4,0]
+ CRUSH rule 2 x 354 [0,3,6]
+ CRUSH rule 2 x 355 [3,8,2]
+ CRUSH rule 2 x 356 [3,0,6]
+ CRUSH rule 2 x 357 [6,1,4]
+ CRUSH rule 2 x 358 [2,8,4]
+ CRUSH rule 2 x 359 [6,1,3]
+ CRUSH rule 2 x 360 [5,2,7]
+ CRUSH rule 2 x 361 [8,4,1]
+ CRUSH rule 2 x 362 [4,1,6]
+ CRUSH rule 2 x 363 [4,0,6]
+ CRUSH rule 2 x 364 [2,5,6]
+ CRUSH rule 2 x 365 [6,5,2]
+ CRUSH rule 2 x 366 [7,2,3]
+ CRUSH rule 2 x 367 [4,0,6]
+ CRUSH rule 2 x 368 [7,4,0]
+ CRUSH rule 2 x 369 [3,7,1]
+ CRUSH rule 2 x 370 [8,2,5]
+ CRUSH rule 2 x 371 [1,3,6]
+ CRUSH rule 2 x 372 [3,1,8]
+ CRUSH rule 2 x 373 [0,6,4]
+ CRUSH rule 2 x 374 [3,8,1]
+ CRUSH rule 2 x 375 [6,4,1]
+ CRUSH rule 2 x 376 [7,1,5]
+ CRUSH rule 2 x 377 [1,3,6]
+ CRUSH rule 2 x 378 [0,8,3]
+ CRUSH rule 2 x 379 [8,5,2]
+ CRUSH rule 2 x 380 [2,5,8]
+ CRUSH rule 2 x 381 [0,4,7]
+ CRUSH rule 2 x 382 [1,5,8]
+ CRUSH rule 2 x 383 [4,6,0]
+ CRUSH rule 2 x 384 [7,0,4]
+ CRUSH rule 2 x 385 [7,4,0]
+ CRUSH rule 2 x 386 [0,3,6]
+ CRUSH rule 2 x 387 [1,3,6]
+ CRUSH rule 2 x 388 [5,0,8]
+ CRUSH rule 2 x 389 [1,5,7]
+ CRUSH rule 2 x 390 [5,6,0]
+ CRUSH rule 2 x 391 [5,6,2]
+ CRUSH rule 2 x 392 [1,8,5]
+ CRUSH rule 2 x 393 [4,2,6]
+ CRUSH rule 2 x 394 [4,7,0]
+ CRUSH rule 2 x 395 [4,0,8]
+ CRUSH rule 2 x 396 [4,2,7]
+ CRUSH rule 2 x 397 [2,4,8]
+ CRUSH rule 2 x 398 [2,4,8]
+ CRUSH rule 2 x 399 [8,4,2]
+ CRUSH rule 2 x 400 [8,1,4]
+ CRUSH rule 2 x 401 [0,5,8]
+ CRUSH rule 2 x 402 [7,5,2]
+ CRUSH rule 2 x 403 [0,3,8]
+ CRUSH rule 2 x 404 [4,2,8]
+ CRUSH rule 2 x 405 [6,5,2]
+ CRUSH rule 2 x 406 [2,6,5]
+ CRUSH rule 2 x 407 [2,8,5]
+ CRUSH rule 2 x 408 [4,1,6]
+ CRUSH rule 2 x 409 [7,3,0]
+ CRUSH rule 2 x 410 [8,3,2]
+ CRUSH rule 2 x 411 [2,8,3]
+ CRUSH rule 2 x 412 [0,5,8]
+ CRUSH rule 2 x 413 [5,0,8]
+ CRUSH rule 2 x 414 [4,1,6]
+ CRUSH rule 2 x 415 [0,6,4]
+ CRUSH rule 2 x 416 [2,6,3]
+ CRUSH rule 2 x 417 [8,2,4]
+ CRUSH rule 2 x 418 [7,1,4]
+ CRUSH rule 2 x 419 [8,3,0]
+ CRUSH rule 2 x 420 [1,4,6]
+ CRUSH rule 2 x 421 [8,4,0]
+ CRUSH rule 2 x 422 [6,4,1]
+ CRUSH rule 2 x 423 [0,5,6]
+ CRUSH rule 2 x 424 [8,4,1]
+ CRUSH rule 2 x 425 [1,3,7]
+ CRUSH rule 2 x 426 [6,0,5]
+ CRUSH rule 2 x 427 [0,7,5]
+ CRUSH rule 2 x 428 [5,7,1]
+ CRUSH rule 2 x 429 [4,6,0]
+ CRUSH rule 2 x 430 [3,6,2]
+ CRUSH rule 2 x 431 [5,0,7]
+ CRUSH rule 2 x 432 [7,1,3]
+ CRUSH rule 2 x 433 [6,5,1]
+ CRUSH rule 2 x 434 [5,2,8]
+ CRUSH rule 2 x 435 [0,5,6]
+ CRUSH rule 2 x 436 [4,0,7]
+ CRUSH rule 2 x 437 [7,5,2]
+ CRUSH rule 2 x 438 [0,3,8]
+ CRUSH rule 2 x 439 [1,3,7]
+ CRUSH rule 2 x 440 [2,7,5]
+ CRUSH rule 2 x 441 [5,7,2]
+ CRUSH rule 2 x 442 [2,4,7]
+ CRUSH rule 2 x 443 [6,0,4]
+ CRUSH rule 2 x 444 [7,0,4]
+ CRUSH rule 2 x 445 [6,3,1]
+ CRUSH rule 2 x 446 [4,1,8]
+ CRUSH rule 2 x 447 [2,3,7]
+ CRUSH rule 2 x 448 [7,2,5]
+ CRUSH rule 2 x 449 [7,5,1]
+ CRUSH rule 2 x 450 [4,1,8]
+ CRUSH rule 2 x 451 [6,5,0]
+ CRUSH rule 2 x 452 [8,3,0]
+ CRUSH rule 2 x 453 [6,5,1]
+ CRUSH rule 2 x 454 [6,4,2]
+ CRUSH rule 2 x 455 [2,7,5]
+ CRUSH rule 2 x 456 [6,2,5]
+ CRUSH rule 2 x 457 [7,2,5]
+ CRUSH rule 2 x 458 [2,8,5]
+ CRUSH rule 2 x 459 [2,7,5]
+ CRUSH rule 2 x 460 [6,5,0]
+ CRUSH rule 2 x 461 [6,5,2]
+ CRUSH rule 2 x 462 [8,1,5]
+ CRUSH rule 2 x 463 [6,0,4]
+ CRUSH rule 2 x 464 [7,4,2]
+ CRUSH rule 2 x 465 [7,2,5]
+ CRUSH rule 2 x 466 [5,8,2]
+ CRUSH rule 2 x 467 [6,4,0]
+ CRUSH rule 2 x 468 [7,0,3]
+ CRUSH rule 2 x 469 [7,0,5]
+ CRUSH rule 2 x 470 [3,0,6]
+ CRUSH rule 2 x 471 [0,7,5]
+ CRUSH rule 2 x 472 [5,1,8]
+ CRUSH rule 2 x 473 [1,4,6]
+ CRUSH rule 2 x 474 [6,0,3]
+ CRUSH rule 2 x 475 [6,2,3]
+ CRUSH rule 2 x 476 [4,6,1]
+ CRUSH rule 2 x 477 [5,8,2]
+ CRUSH rule 2 x 478 [6,2,3]
+ CRUSH rule 2 x 479 [0,5,8]
+ CRUSH rule 2 x 480 [1,8,3]
+ CRUSH rule 2 x 481 [2,4,7]
+ CRUSH rule 2 x 482 [4,7,0]
+ CRUSH rule 2 x 483 [0,6,4]
+ CRUSH rule 2 x 484 [1,7,4]
+ CRUSH rule 2 x 485 [4,7,0]
+ CRUSH rule 2 x 486 [4,1,7]
+ CRUSH rule 2 x 487 [5,0,8]
+ CRUSH rule 2 x 488 [5,7,1]
+ CRUSH rule 2 x 489 [2,8,5]
+ CRUSH rule 2 x 490 [6,4,1]
+ CRUSH rule 2 x 491 [1,7,5]
+ CRUSH rule 2 x 492 [6,5,0]
+ CRUSH rule 2 x 493 [0,7,3]
+ CRUSH rule 2 x 494 [1,7,4]
+ CRUSH rule 2 x 495 [3,1,8]
+ CRUSH rule 2 x 496 [7,5,0]
+ CRUSH rule 2 x 497 [5,7,0]
+ CRUSH rule 2 x 498 [0,5,8]
+ CRUSH rule 2 x 499 [8,4,2]
+ CRUSH rule 2 x 500 [3,6,0]
+ CRUSH rule 2 x 501 [0,7,3]
+ CRUSH rule 2 x 502 [7,1,4]
+ CRUSH rule 2 x 503 [2,3,7]
+ CRUSH rule 2 x 504 [5,6,2]
+ CRUSH rule 2 x 505 [0,7,3]
+ CRUSH rule 2 x 506 [5,2,8]
+ CRUSH rule 2 x 507 [6,0,3]
+ CRUSH rule 2 x 508 [0,3,8]
+ CRUSH rule 2 x 509 [7,5,2]
+ CRUSH rule 2 x 510 [6,0,4]
+ CRUSH rule 2 x 511 [5,8,2]
+ CRUSH rule 2 x 512 [7,0,4]
+ CRUSH rule 2 x 513 [7,2,4]
+ CRUSH rule 2 x 514 [4,6,1]
+ CRUSH rule 2 x 515 [8,5,0]
+ CRUSH rule 2 x 516 [4,0,8]
+ CRUSH rule 2 x 517 [7,2,4]
+ CRUSH rule 2 x 518 [4,6,1]
+ CRUSH rule 2 x 519 [7,3,1]
+ CRUSH rule 2 x 520 [2,6,3]
+ CRUSH rule 2 x 521 [8,0,3]
+ CRUSH rule 2 x 522 [6,0,4]
+ CRUSH rule 2 x 523 [4,2,7]
+ CRUSH rule 2 x 524 [0,4,8]
+ CRUSH rule 2 x 525 [0,4,6]
+ CRUSH rule 2 x 526 [1,5,8]
+ CRUSH rule 2 x 527 [0,5,6]
+ CRUSH rule 2 x 528 [5,0,6]
+ CRUSH rule 2 x 529 [5,7,0]
+ CRUSH rule 2 x 530 [6,5,2]
+ CRUSH rule 2 x 531 [6,1,3]
+ CRUSH rule 2 x 532 [6,3,0]
+ CRUSH rule 2 x 533 [5,6,2]
+ CRUSH rule 2 x 534 [7,3,1]
+ CRUSH rule 2 x 535 [8,1,5]
+ CRUSH rule 2 x 536 [6,2,4]
+ CRUSH rule 2 x 537 [3,7,2]
+ CRUSH rule 2 x 538 [6,3,0]
+ CRUSH rule 2 x 539 [8,3,1]
+ CRUSH rule 2 x 540 [0,6,3]
+ CRUSH rule 2 x 541 [2,3,8]
+ CRUSH rule 2 x 542 [3,2,8]
+ CRUSH rule 2 x 543 [6,0,4]
+ CRUSH rule 2 x 544 [3,7,0]
+ CRUSH rule 2 x 545 [5,7,0]
+ CRUSH rule 2 x 546 [6,1,5]
+ CRUSH rule 2 x 547 [8,2,4]
+ CRUSH rule 2 x 548 [5,2,8]
+ CRUSH rule 2 x 549 [5,8,2]
+ CRUSH rule 2 x 550 [0,5,7]
+ CRUSH rule 2 x 551 [7,5,0]
+ CRUSH rule 2 x 552 [5,8,1]
+ CRUSH rule 2 x 553 [4,2,7]
+ CRUSH rule 2 x 554 [0,8,5]
+ CRUSH rule 2 x 555 [5,0,8]
+ CRUSH rule 2 x 556 [3,6,0]
+ CRUSH rule 2 x 557 [7,4,0]
+ CRUSH rule 2 x 558 [3,1,6]
+ CRUSH rule 2 x 559 [4,2,6]
+ CRUSH rule 2 x 560 [8,3,2]
+ CRUSH rule 2 x 561 [6,3,1]
+ CRUSH rule 2 x 562 [3,0,6]
+ CRUSH rule 2 x 563 [2,6,4]
+ CRUSH rule 2 x 564 [5,1,7]
+ CRUSH rule 2 x 565 [3,6,2]
+ CRUSH rule 2 x 566 [4,7,2]
+ CRUSH rule 2 x 567 [3,6,1]
+ CRUSH rule 2 x 568 [7,4,1]
+ CRUSH rule 2 x 569 [3,1,7]
+ CRUSH rule 2 x 570 [1,5,8]
+ CRUSH rule 2 x 571 [3,7,1]
+ CRUSH rule 2 x 572 [3,2,8]
+ CRUSH rule 2 x 573 [3,0,7]
+ CRUSH rule 2 x 574 [2,5,8]
+ CRUSH rule 2 x 575 [8,2,5]
+ CRUSH rule 2 x 576 [4,6,0]
+ CRUSH rule 2 x 577 [8,2,5]
+ CRUSH rule 2 x 578 [6,1,4]
+ CRUSH rule 2 x 579 [3,1,6]
+ CRUSH rule 2 x 580 [3,0,7]
+ CRUSH rule 2 x 581 [7,2,4]
+ CRUSH rule 2 x 582 [2,8,5]
+ CRUSH rule 2 x 583 [6,0,3]
+ CRUSH rule 2 x 584 [8,1,3]
+ CRUSH rule 2 x 585 [7,0,5]
+ CRUSH rule 2 x 586 [0,7,5]
+ CRUSH rule 2 x 587 [2,5,7]
+ CRUSH rule 2 x 588 [3,7,1]
+ CRUSH rule 2 x 589 [7,1,4]
+ CRUSH rule 2 x 590 [6,2,3]
+ CRUSH rule 2 x 591 [5,2,8]
+ CRUSH rule 2 x 592 [2,4,7]
+ CRUSH rule 2 x 593 [0,8,3]
+ CRUSH rule 2 x 594 [0,7,4]
+ CRUSH rule 2 x 595 [7,1,3]
+ CRUSH rule 2 x 596 [4,0,6]
+ CRUSH rule 2 x 597 [3,1,7]
+ CRUSH rule 2 x 598 [3,2,6]
+ CRUSH rule 2 x 599 [5,2,8]
+ CRUSH rule 2 x 600 [7,0,3]
+ CRUSH rule 2 x 601 [0,7,3]
+ CRUSH rule 2 x 602 [3,7,1]
+ CRUSH rule 2 x 603 [5,1,6]
+ CRUSH rule 2 x 604 [7,5,0]
+ CRUSH rule 2 x 605 [3,0,7]
+ CRUSH rule 2 x 606 [2,7,4]
+ CRUSH rule 2 x 607 [0,4,8]
+ CRUSH rule 2 x 608 [5,2,7]
+ CRUSH rule 2 x 609 [5,2,8]
+ CRUSH rule 2 x 610 [3,7,0]
+ CRUSH rule 2 x 611 [1,8,5]
+ CRUSH rule 2 x 612 [2,6,5]
+ CRUSH rule 2 x 613 [7,2,3]
+ CRUSH rule 2 x 614 [7,2,3]
+ CRUSH rule 2 x 615 [6,0,5]
+ CRUSH rule 2 x 616 [0,8,5]
+ CRUSH rule 2 x 617 [6,1,4]
+ CRUSH rule 2 x 618 [7,4,0]
+ CRUSH rule 2 x 619 [5,1,8]
+ CRUSH rule 2 x 620 [4,1,6]
+ CRUSH rule 2 x 621 [5,8,2]
+ CRUSH rule 2 x 622 [0,4,8]
+ CRUSH rule 2 x 623 [0,6,3]
+ CRUSH rule 2 x 624 [3,2,8]
+ CRUSH rule 2 x 625 [2,3,7]
+ CRUSH rule 2 x 626 [7,0,3]
+ CRUSH rule 2 x 627 [2,7,3]
+ CRUSH rule 2 x 628 [8,0,5]
+ CRUSH rule 2 x 629 [2,6,5]
+ CRUSH rule 2 x 630 [2,7,5]
+ CRUSH rule 2 x 631 [0,7,3]
+ CRUSH rule 2 x 632 [7,0,3]
+ CRUSH rule 2 x 633 [8,3,1]
+ CRUSH rule 2 x 634 [0,4,7]
+ CRUSH rule 2 x 635 [5,6,2]
+ CRUSH rule 2 x 636 [1,4,8]
+ CRUSH rule 2 x 637 [4,1,7]
+ CRUSH rule 2 x 638 [6,0,4]
+ CRUSH rule 2 x 639 [4,2,8]
+ CRUSH rule 2 x 640 [3,1,8]
+ CRUSH rule 2 x 641 [7,2,3]
+ CRUSH rule 2 x 642 [2,7,3]
+ CRUSH rule 2 x 643 [3,0,8]
+ CRUSH rule 2 x 644 [8,1,5]
+ CRUSH rule 2 x 645 [5,7,1]
+ CRUSH rule 2 x 646 [8,0,3]
+ CRUSH rule 2 x 647 [7,1,3]
+ CRUSH rule 2 x 648 [0,6,3]
+ CRUSH rule 2 x 649 [4,7,0]
+ CRUSH rule 2 x 650 [7,3,1]
+ CRUSH rule 2 x 651 [3,7,0]
+ CRUSH rule 2 x 652 [3,7,0]
+ CRUSH rule 2 x 653 [8,5,2]
+ CRUSH rule 2 x 654 [7,5,2]
+ CRUSH rule 2 x 655 [0,3,7]
+ CRUSH rule 2 x 656 [4,7,1]
+ CRUSH rule 2 x 657 [6,1,4]
+ CRUSH rule 2 x 658 [5,6,0]
+ CRUSH rule 2 x 659 [4,6,2]
+ CRUSH rule 2 x 660 [7,4,1]
+ CRUSH rule 2 x 661 [1,8,3]
+ CRUSH rule 2 x 662 [4,2,7]
+ CRUSH rule 2 x 663 [1,3,8]
+ CRUSH rule 2 x 664 [1,4,7]
+ CRUSH rule 2 x 665 [5,7,0]
+ CRUSH rule 2 x 666 [2,8,4]
+ CRUSH rule 2 x 667 [1,3,7]
+ CRUSH rule 2 x 668 [3,7,1]
+ CRUSH rule 2 x 669 [6,4,0]
+ CRUSH rule 2 x 670 [4,0,6]
+ CRUSH rule 2 x 671 [0,7,3]
+ CRUSH rule 2 x 672 [4,2,7]
+ CRUSH rule 2 x 673 [5,2,7]
+ CRUSH rule 2 x 674 [3,1,8]
+ CRUSH rule 2 x 675 [0,8,3]
+ CRUSH rule 2 x 676 [0,4,8]
+ CRUSH rule 2 x 677 [4,1,7]
+ CRUSH rule 2 x 678 [2,3,8]
+ CRUSH rule 2 x 679 [6,0,5]
+ CRUSH rule 2 x 680 [0,4,6]
+ CRUSH rule 2 x 681 [4,7,1]
+ CRUSH rule 2 x 682 [0,5,7]
+ CRUSH rule 2 x 683 [0,5,6]
+ CRUSH rule 2 x 684 [7,1,5]
+ CRUSH rule 2 x 685 [7,1,5]
+ CRUSH rule 2 x 686 [1,4,8]
+ CRUSH rule 2 x 687 [3,6,1]
+ CRUSH rule 2 x 688 [5,7,2]
+ CRUSH rule 2 x 689 [6,5,0]
+ CRUSH rule 2 x 690 [8,1,3]
+ CRUSH rule 2 x 691 [3,0,6]
+ CRUSH rule 2 x 692 [7,2,3]
+ CRUSH rule 2 x 693 [6,3,1]
+ CRUSH rule 2 x 694 [6,5,1]
+ CRUSH rule 2 x 695 [0,8,4]
+ CRUSH rule 2 x 696 [1,4,8]
+ CRUSH rule 2 x 697 [6,1,3]
+ CRUSH rule 2 x 698 [6,2,4]
+ CRUSH rule 2 x 699 [1,6,3]
+ CRUSH rule 2 x 700 [0,3,7]
+ CRUSH rule 2 x 701 [4,1,7]
+ CRUSH rule 2 x 702 [3,2,8]
+ CRUSH rule 2 x 703 [8,3,1]
+ CRUSH rule 2 x 704 [0,3,8]
+ CRUSH rule 2 x 705 [8,0,4]
+ CRUSH rule 2 x 706 [1,5,6]
+ CRUSH rule 2 x 707 [7,3,1]
+ CRUSH rule 2 x 708 [3,7,1]
+ CRUSH rule 2 x 709 [6,3,0]
+ CRUSH rule 2 x 710 [8,4,0]
+ CRUSH rule 2 x 711 [2,3,8]
+ CRUSH rule 2 x 712 [2,3,7]
+ CRUSH rule 2 x 713 [6,3,0]
+ CRUSH rule 2 x 714 [3,2,7]
+ CRUSH rule 2 x 715 [1,3,6]
+ CRUSH rule 2 x 716 [3,6,0]
+ CRUSH rule 2 x 717 [8,2,5]
+ CRUSH rule 2 x 718 [3,7,2]
+ CRUSH rule 2 x 719 [2,6,3]
+ CRUSH rule 2 x 720 [6,1,4]
+ CRUSH rule 2 x 721 [5,7,2]
+ CRUSH rule 2 x 722 [5,7,1]
+ CRUSH rule 2 x 723 [5,1,7]
+ CRUSH rule 2 x 724 [0,6,3]
+ CRUSH rule 2 x 725 [0,3,7]
+ CRUSH rule 2 x 726 [3,8,1]
+ CRUSH rule 2 x 727 [4,6,1]
+ CRUSH rule 2 x 728 [2,7,4]
+ CRUSH rule 2 x 729 [5,6,2]
+ CRUSH rule 2 x 730 [3,7,2]
+ CRUSH rule 2 x 731 [4,1,8]
+ CRUSH rule 2 x 732 [1,5,6]
+ CRUSH rule 2 x 733 [5,7,0]
+ CRUSH rule 2 x 734 [6,4,2]
+ CRUSH rule 2 x 735 [4,8,1]
+ CRUSH rule 2 x 736 [3,8,1]
+ CRUSH rule 2 x 737 [1,6,4]
+ CRUSH rule 2 x 738 [5,2,7]
+ CRUSH rule 2 x 739 [0,7,4]
+ CRUSH rule 2 x 740 [0,8,4]
+ CRUSH rule 2 x 741 [7,1,4]
+ CRUSH rule 2 x 742 [8,2,3]
+ CRUSH rule 2 x 743 [7,0,5]
+ CRUSH rule 2 x 744 [4,7,1]
+ CRUSH rule 2 x 745 [3,1,8]
+ CRUSH rule 2 x 746 [4,1,7]
+ CRUSH rule 2 x 747 [6,0,3]
+ CRUSH rule 2 x 748 [2,7,5]
+ CRUSH rule 2 x 749 [4,8,0]
+ CRUSH rule 2 x 750 [1,6,3]
+ CRUSH rule 2 x 751 [2,8,3]
+ CRUSH rule 2 x 752 [8,1,5]
+ CRUSH rule 2 x 753 [7,3,1]
+ CRUSH rule 2 x 754 [8,5,2]
+ CRUSH rule 2 x 755 [1,6,3]
+ CRUSH rule 2 x 756 [5,6,1]
+ CRUSH rule 2 x 757 [8,0,5]
+ CRUSH rule 2 x 758 [6,0,3]
+ CRUSH rule 2 x 759 [8,5,2]
+ CRUSH rule 2 x 760 [1,5,7]
+ CRUSH rule 2 x 761 [4,1,8]
+ CRUSH rule 2 x 762 [2,7,5]
+ CRUSH rule 2 x 763 [8,5,1]
+ CRUSH rule 2 x 764 [1,7,5]
+ CRUSH rule 2 x 765 [6,5,2]
+ CRUSH rule 2 x 766 [8,5,1]
+ CRUSH rule 2 x 767 [1,8,3]
+ CRUSH rule 2 x 768 [8,3,2]
+ CRUSH rule 2 x 769 [6,2,5]
+ CRUSH rule 2 x 770 [6,0,4]
+ CRUSH rule 2 x 771 [7,0,3]
+ CRUSH rule 2 x 772 [8,3,1]
+ CRUSH rule 2 x 773 [3,1,7]
+ CRUSH rule 2 x 774 [4,6,2]
+ CRUSH rule 2 x 775 [6,4,2]
+ CRUSH rule 2 x 776 [7,2,5]
+ CRUSH rule 2 x 777 [3,1,6]
+ CRUSH rule 2 x 778 [1,8,4]
+ CRUSH rule 2 x 779 [2,7,3]
+ CRUSH rule 2 x 780 [0,5,7]
+ CRUSH rule 2 x 781 [6,3,2]
+ CRUSH rule 2 x 782 [5,0,8]
+ CRUSH rule 2 x 783 [7,1,3]
+ CRUSH rule 2 x 784 [0,4,6]
+ CRUSH rule 2 x 785 [6,1,3]
+ CRUSH rule 2 x 786 [7,3,1]
+ CRUSH rule 2 x 787 [1,6,4]
+ CRUSH rule 2 x 788 [6,0,3]
+ CRUSH rule 2 x 789 [0,4,8]
+ CRUSH rule 2 x 790 [8,4,0]
+ CRUSH rule 2 x 791 [3,8,0]
+ CRUSH rule 2 x 792 [5,8,0]
+ CRUSH rule 2 x 793 [6,1,3]
+ CRUSH rule 2 x 794 [2,6,4]
+ CRUSH rule 2 x 795 [0,3,8]
+ CRUSH rule 2 x 796 [3,7,2]
+ CRUSH rule 2 x 797 [2,3,8]
+ CRUSH rule 2 x 798 [6,1,5]
+ CRUSH rule 2 x 799 [5,1,8]
+ CRUSH rule 2 x 800 [5,0,7]
+ CRUSH rule 2 x 801 [3,6,1]
+ CRUSH rule 2 x 802 [1,8,5]
+ CRUSH rule 2 x 803 [0,5,7]
+ CRUSH rule 2 x 804 [6,2,5]
+ CRUSH rule 2 x 805 [3,6,1]
+ CRUSH rule 2 x 806 [1,3,7]
+ CRUSH rule 2 x 807 [5,7,2]
+ CRUSH rule 2 x 808 [4,6,2]
+ CRUSH rule 2 x 809 [1,4,8]
+ CRUSH rule 2 x 810 [5,7,2]
+ CRUSH rule 2 x 811 [8,4,0]
+ CRUSH rule 2 x 812 [8,5,2]
+ CRUSH rule 2 x 813 [6,4,2]
+ CRUSH rule 2 x 814 [3,6,1]
+ CRUSH rule 2 x 815 [3,1,8]
+ CRUSH rule 2 x 816 [2,7,3]
+ CRUSH rule 2 x 817 [4,8,2]
+ CRUSH rule 2 x 818 [3,0,7]
+ CRUSH rule 2 x 819 [5,1,8]
+ CRUSH rule 2 x 820 [3,6,0]
+ CRUSH rule 2 x 821 [4,6,2]
+ CRUSH rule 2 x 822 [2,5,8]
+ CRUSH rule 2 x 823 [4,8,2]
+ CRUSH rule 2 x 824 [3,7,2]
+ CRUSH rule 2 x 825 [2,8,5]
+ CRUSH rule 2 x 826 [7,0,5]
+ CRUSH rule 2 x 827 [0,8,3]
+ CRUSH rule 2 x 828 [2,3,8]
+ CRUSH rule 2 x 829 [5,6,1]
+ CRUSH rule 2 x 830 [2,3,8]
+ CRUSH rule 2 x 831 [1,6,3]
+ CRUSH rule 2 x 832 [4,7,0]
+ CRUSH rule 2 x 833 [2,7,3]
+ CRUSH rule 2 x 834 [3,1,7]
+ CRUSH rule 2 x 835 [8,4,1]
+ CRUSH rule 2 x 836 [3,7,1]
+ CRUSH rule 2 x 837 [6,3,1]
+ CRUSH rule 2 x 838 [6,2,4]
+ CRUSH rule 2 x 839 [5,0,6]
+ CRUSH rule 2 x 840 [7,3,2]
+ CRUSH rule 2 x 841 [4,8,2]
+ CRUSH rule 2 x 842 [2,4,6]
+ CRUSH rule 2 x 843 [6,4,1]
+ CRUSH rule 2 x 844 [4,8,1]
+ CRUSH rule 2 x 845 [3,8,2]
+ CRUSH rule 2 x 846 [3,2,7]
+ CRUSH rule 2 x 847 [0,8,4]
+ CRUSH rule 2 x 848 [2,6,5]
+ CRUSH rule 2 x 849 [4,6,2]
+ CRUSH rule 2 x 850 [1,3,6]
+ CRUSH rule 2 x 851 [6,4,0]
+ CRUSH rule 2 x 852 [7,3,0]
+ CRUSH rule 2 x 853 [6,0,4]
+ CRUSH rule 2 x 854 [7,0,4]
+ CRUSH rule 2 x 855 [5,7,2]
+ CRUSH rule 2 x 856 [6,3,2]
+ CRUSH rule 2 x 857 [8,5,0]
+ CRUSH rule 2 x 858 [6,4,1]
+ CRUSH rule 2 x 859 [6,0,5]
+ CRUSH rule 2 x 860 [4,1,7]
+ CRUSH rule 2 x 861 [8,3,1]
+ CRUSH rule 2 x 862 [6,1,4]
+ CRUSH rule 2 x 863 [8,2,3]
+ CRUSH rule 2 x 864 [5,6,2]
+ CRUSH rule 2 x 865 [8,1,3]
+ CRUSH rule 2 x 866 [3,6,0]
+ CRUSH rule 2 x 867 [6,5,1]
+ CRUSH rule 2 x 868 [6,3,0]
+ CRUSH rule 2 x 869 [8,5,2]
+ CRUSH rule 2 x 870 [0,4,8]
+ CRUSH rule 2 x 871 [3,2,8]
+ CRUSH rule 2 x 872 [5,1,8]
+ CRUSH rule 2 x 873 [4,6,2]
+ CRUSH rule 2 x 874 [2,6,4]
+ CRUSH rule 2 x 875 [2,6,4]
+ CRUSH rule 2 x 876 [5,8,1]
+ CRUSH rule 2 x 877 [6,4,2]
+ CRUSH rule 2 x 878 [5,2,7]
+ CRUSH rule 2 x 879 [7,4,2]
+ CRUSH rule 2 x 880 [3,2,8]
+ CRUSH rule 2 x 881 [5,6,1]
+ CRUSH rule 2 x 882 [4,0,7]
+ CRUSH rule 2 x 883 [2,3,7]
+ CRUSH rule 2 x 884 [6,0,4]
+ CRUSH rule 2 x 885 [5,1,8]
+ CRUSH rule 2 x 886 [3,6,0]
+ CRUSH rule 2 x 887 [7,4,0]
+ CRUSH rule 2 x 888 [6,2,5]
+ CRUSH rule 2 x 889 [2,6,4]
+ CRUSH rule 2 x 890 [7,2,4]
+ CRUSH rule 2 x 891 [1,8,5]
+ CRUSH rule 2 x 892 [6,2,3]
+ CRUSH rule 2 x 893 [2,3,7]
+ CRUSH rule 2 x 894 [7,5,0]
+ CRUSH rule 2 x 895 [5,1,8]
+ CRUSH rule 2 x 896 [1,8,5]
+ CRUSH rule 2 x 897 [4,2,6]
+ CRUSH rule 2 x 898 [0,5,7]
+ CRUSH rule 2 x 899 [1,7,5]
+ CRUSH rule 2 x 900 [4,1,6]
+ CRUSH rule 2 x 901 [5,0,8]
+ CRUSH rule 2 x 902 [8,5,0]
+ CRUSH rule 2 x 903 [5,7,1]
+ CRUSH rule 2 x 904 [5,6,2]
+ CRUSH rule 2 x 905 [6,2,5]
+ CRUSH rule 2 x 906 [1,6,3]
+ CRUSH rule 2 x 907 [7,1,5]
+ CRUSH rule 2 x 908 [5,8,1]
+ CRUSH rule 2 x 909 [2,3,7]
+ CRUSH rule 2 x 910 [6,4,0]
+ CRUSH rule 2 x 911 [5,8,1]
+ CRUSH rule 2 x 912 [0,7,3]
+ CRUSH rule 2 x 913 [7,2,4]
+ CRUSH rule 2 x 914 [6,4,0]
+ CRUSH rule 2 x 915 [8,2,3]
+ CRUSH rule 2 x 916 [3,1,8]
+ CRUSH rule 2 x 917 [1,5,8]
+ CRUSH rule 2 x 918 [8,2,4]
+ CRUSH rule 2 x 919 [6,2,3]
+ CRUSH rule 2 x 920 [7,4,0]
+ CRUSH rule 2 x 921 [1,4,6]
+ CRUSH rule 2 x 922 [6,4,0]
+ CRUSH rule 2 x 923 [5,8,2]
+ CRUSH rule 2 x 924 [3,1,7]
+ CRUSH rule 2 x 925 [5,7,2]
+ CRUSH rule 2 x 926 [3,0,8]
+ CRUSH rule 2 x 927 [1,6,3]
+ CRUSH rule 2 x 928 [8,1,3]
+ CRUSH rule 2 x 929 [4,1,7]
+ CRUSH rule 2 x 930 [2,4,6]
+ CRUSH rule 2 x 931 [5,0,7]
+ CRUSH rule 2 x 932 [4,1,8]
+ CRUSH rule 2 x 933 [8,5,0]
+ CRUSH rule 2 x 934 [5,6,0]
+ CRUSH rule 2 x 935 [6,3,1]
+ CRUSH rule 2 x 936 [0,6,5]
+ CRUSH rule 2 x 937 [5,8,2]
+ CRUSH rule 2 x 938 [6,5,2]
+ CRUSH rule 2 x 939 [2,7,5]
+ CRUSH rule 2 x 940 [8,5,0]
+ CRUSH rule 2 x 941 [5,2,8]
+ CRUSH rule 2 x 942 [1,8,4]
+ CRUSH rule 2 x 943 [8,2,4]
+ CRUSH rule 2 x 944 [4,8,2]
+ CRUSH rule 2 x 945 [7,2,4]
+ CRUSH rule 2 x 946 [2,8,5]
+ CRUSH rule 2 x 947 [4,2,8]
+ CRUSH rule 2 x 948 [7,5,0]
+ CRUSH rule 2 x 949 [6,1,3]
+ CRUSH rule 2 x 950 [3,6,0]
+ CRUSH rule 2 x 951 [4,8,1]
+ CRUSH rule 2 x 952 [2,7,3]
+ CRUSH rule 2 x 953 [1,3,6]
+ CRUSH rule 2 x 954 [4,2,7]
+ CRUSH rule 2 x 955 [8,0,4]
+ CRUSH rule 2 x 956 [1,6,4]
+ CRUSH rule 2 x 957 [7,1,3]
+ CRUSH rule 2 x 958 [8,4,1]
+ CRUSH rule 2 x 959 [5,2,7]
+ CRUSH rule 2 x 960 [3,6,0]
+ CRUSH rule 2 x 961 [4,0,8]
+ CRUSH rule 2 x 962 [7,4,0]
+ CRUSH rule 2 x 963 [0,5,6]
+ CRUSH rule 2 x 964 [3,1,8]
+ CRUSH rule 2 x 965 [7,4,0]
+ CRUSH rule 2 x 966 [3,8,0]
+ CRUSH rule 2 x 967 [8,5,0]
+ CRUSH rule 2 x 968 [7,2,4]
+ CRUSH rule 2 x 969 [8,0,5]
+ CRUSH rule 2 x 970 [0,6,3]
+ CRUSH rule 2 x 971 [1,7,3]
+ CRUSH rule 2 x 972 [1,8,4]
+ CRUSH rule 2 x 973 [1,6,3]
+ CRUSH rule 2 x 974 [5,1,8]
+ CRUSH rule 2 x 975 [3,7,0]
+ CRUSH rule 2 x 976 [4,8,2]
+ CRUSH rule 2 x 977 [8,3,2]
+ CRUSH rule 2 x 978 [7,2,4]
+ CRUSH rule 2 x 979 [7,1,5]
+ CRUSH rule 2 x 980 [6,0,5]
+ CRUSH rule 2 x 981 [7,3,2]
+ CRUSH rule 2 x 982 [4,2,8]
+ CRUSH rule 2 x 983 [3,7,0]
+ CRUSH rule 2 x 984 [0,7,3]
+ CRUSH rule 2 x 985 [2,5,7]
+ CRUSH rule 2 x 986 [8,3,0]
+ CRUSH rule 2 x 987 [0,5,8]
+ CRUSH rule 2 x 988 [1,3,7]
+ CRUSH rule 2 x 989 [0,6,3]
+ CRUSH rule 2 x 990 [1,6,5]
+ CRUSH rule 2 x 991 [0,4,8]
+ CRUSH rule 2 x 992 [7,1,5]
+ CRUSH rule 2 x 993 [0,6,3]
+ CRUSH rule 2 x 994 [3,7,2]
+ CRUSH rule 2 x 995 [7,1,5]
+ CRUSH rule 2 x 996 [6,5,0]
+ CRUSH rule 2 x 997 [6,4,1]
+ CRUSH rule 2 x 998 [8,1,5]
+ CRUSH rule 2 x 999 [0,7,4]
+ CRUSH rule 2 x 1000 [8,5,0]
+ CRUSH rule 2 x 1001 [2,5,6]
+ CRUSH rule 2 x 1002 [1,3,7]
+ CRUSH rule 2 x 1003 [2,8,3]
+ CRUSH rule 2 x 1004 [6,1,3]
+ CRUSH rule 2 x 1005 [6,1,5]
+ CRUSH rule 2 x 1006 [1,6,5]
+ CRUSH rule 2 x 1007 [1,5,7]
+ CRUSH rule 2 x 1008 [1,7,3]
+ CRUSH rule 2 x 1009 [6,4,1]
+ CRUSH rule 2 x 1010 [3,1,7]
+ CRUSH rule 2 x 1011 [3,0,8]
+ CRUSH rule 2 x 1012 [3,0,7]
+ CRUSH rule 2 x 1013 [5,1,7]
+ CRUSH rule 2 x 1014 [2,8,4]
+ CRUSH rule 2 x 1015 [6,5,0]
+ CRUSH rule 2 x 1016 [2,4,7]
+ CRUSH rule 2 x 1017 [6,0,3]
+ CRUSH rule 2 x 1018 [5,0,6]
+ CRUSH rule 2 x 1019 [5,8,2]
+ CRUSH rule 2 x 1020 [5,1,7]
+ CRUSH rule 2 x 1021 [5,2,6]
+ CRUSH rule 2 x 1022 [1,6,4]
+ CRUSH rule 2 x 1023 [3,2,8]
+ rule 2 (chooseleaf) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 3 (choose-set), x = 0..1023, numrep = 2..3
+ CRUSH rule 3 x 0 [0,3]
+ CRUSH rule 3 x 1 [0,8]
+ CRUSH rule 3 x 2 [1,4]
+ CRUSH rule 3 x 3 [8,0]
+ CRUSH rule 3 x 4 [5,1]
+ CRUSH rule 3 x 5 [7,0]
+ CRUSH rule 3 x 6 [2,6]
+ CRUSH rule 3 x 7 [5,6]
+ CRUSH rule 3 x 8 [5,7]
+ CRUSH rule 3 x 9 [2,4]
+ CRUSH rule 3 x 10 [0,8]
+ CRUSH rule 3 x 11 [0,6]
+ CRUSH rule 3 x 12 [0,3]
+ CRUSH rule 3 x 13 [3,8]
+ CRUSH rule 3 x 14 [7,1]
+ CRUSH rule 3 x 15 [7,1]
+ CRUSH rule 3 x 16 [3,7]
+ CRUSH rule 3 x 17 [5,1]
+ CRUSH rule 3 x 18 [1,3]
+ CRUSH rule 3 x 19 [7,5]
+ CRUSH rule 3 x 20 [2,4]
+ CRUSH rule 3 x 21 [3,6]
+ CRUSH rule 3 x 22 [8,5]
+ CRUSH rule 3 x 23 [3,7]
+ CRUSH rule 3 x 24 [1,6]
+ CRUSH rule 3 x 25 [3,8]
+ CRUSH rule 3 x 26 [2,7]
+ CRUSH rule 3 x 27 [3,1]
+ CRUSH rule 3 x 28 [6,2]
+ CRUSH rule 3 x 29 [8,5]
+ CRUSH rule 3 x 30 [5,6]
+ CRUSH rule 3 x 31 [8,2]
+ CRUSH rule 3 x 32 [3,7]
+ CRUSH rule 3 x 33 [2,7]
+ CRUSH rule 3 x 34 [2,3]
+ CRUSH rule 3 x 35 [0,6]
+ CRUSH rule 3 x 36 [3,6]
+ CRUSH rule 3 x 37 [0,4]
+ CRUSH rule 3 x 38 [4,6]
+ CRUSH rule 3 x 39 [3,7]
+ CRUSH rule 3 x 40 [7,2]
+ CRUSH rule 3 x 41 [0,7]
+ CRUSH rule 3 x 42 [4,7]
+ CRUSH rule 3 x 43 [0,3]
+ CRUSH rule 3 x 44 [1,8]
+ CRUSH rule 3 x 45 [8,2]
+ CRUSH rule 3 x 46 [2,4]
+ CRUSH rule 3 x 47 [4,1]
+ CRUSH rule 3 x 48 [4,6]
+ CRUSH rule 3 x 49 [5,6]
+ CRUSH rule 3 x 50 [3,1]
+ CRUSH rule 3 x 51 [3,6]
+ CRUSH rule 3 x 52 [8,2]
+ CRUSH rule 3 x 53 [3,6]
+ CRUSH rule 3 x 54 [7,4]
+ CRUSH rule 3 x 55 [8,0]
+ CRUSH rule 3 x 56 [6,5]
+ CRUSH rule 3 x 57 [5,8]
+ CRUSH rule 3 x 58 [1,8]
+ CRUSH rule 3 x 59 [4,0]
+ CRUSH rule 3 x 60 [3,1]
+ CRUSH rule 3 x 61 [4,8]
+ CRUSH rule 3 x 62 [7,0]
+ CRUSH rule 3 x 63 [5,6]
+ CRUSH rule 3 x 64 [4,1]
+ CRUSH rule 3 x 65 [7,4]
+ CRUSH rule 3 x 66 [5,6]
+ CRUSH rule 3 x 67 [5,0]
+ CRUSH rule 3 x 68 [0,3]
+ CRUSH rule 3 x 69 [5,2]
+ CRUSH rule 3 x 70 [7,2]
+ CRUSH rule 3 x 71 [2,7]
+ CRUSH rule 3 x 72 [6,2]
+ CRUSH rule 3 x 73 [2,7]
+ CRUSH rule 3 x 74 [0,8]
+ CRUSH rule 3 x 75 [3,0]
+ CRUSH rule 3 x 76 [5,2]
+ CRUSH rule 3 x 77 [7,0]
+ CRUSH rule 3 x 78 [1,4]
+ CRUSH rule 3 x 79 [5,0]
+ CRUSH rule 3 x 80 [0,4]
+ CRUSH rule 3 x 81 [0,5]
+ CRUSH rule 3 x 82 [7,2]
+ CRUSH rule 3 x 83 [2,6]
+ CRUSH rule 3 x 84 [7,0]
+ CRUSH rule 3 x 85 [3,7]
+ CRUSH rule 3 x 86 [0,6]
+ CRUSH rule 3 x 87 [0,6]
+ CRUSH rule 3 x 88 [1,8]
+ CRUSH rule 3 x 89 [3,1]
+ CRUSH rule 3 x 90 [6,3]
+ CRUSH rule 3 x 91 [3,6]
+ CRUSH rule 3 x 92 [1,6]
+ CRUSH rule 3 x 93 [7,4]
+ CRUSH rule 3 x 94 [0,3]
+ CRUSH rule 3 x 95 [7,3]
+ CRUSH rule 3 x 96 [3,8]
+ CRUSH rule 3 x 97 [8,4]
+ CRUSH rule 3 x 98 [2,7]
+ CRUSH rule 3 x 99 [0,8]
+ CRUSH rule 3 x 100 [1,6]
+ CRUSH rule 3 x 101 [3,8]
+ CRUSH rule 3 x 102 [4,0]
+ CRUSH rule 3 x 103 [4,7]
+ CRUSH rule 3 x 104 [7,4]
+ CRUSH rule 3 x 105 [2,3]
+ CRUSH rule 3 x 106 [1,6]
+ CRUSH rule 3 x 107 [3,1]
+ CRUSH rule 3 x 108 [7,1]
+ CRUSH rule 3 x 109 [1,3]
+ CRUSH rule 3 x 110 [3,2]
+ CRUSH rule 3 x 111 [2,4]
+ CRUSH rule 3 x 112 [2,6]
+ CRUSH rule 3 x 113 [6,1]
+ CRUSH rule 3 x 114 [7,5]
+ CRUSH rule 3 x 115 [8,2]
+ CRUSH rule 3 x 116 [1,7]
+ CRUSH rule 3 x 117 [7,5]
+ CRUSH rule 3 x 118 [0,3]
+ CRUSH rule 3 x 119 [5,7]
+ CRUSH rule 3 x 120 [0,5]
+ CRUSH rule 3 x 121 [2,8]
+ CRUSH rule 3 x 122 [8,4]
+ CRUSH rule 3 x 123 [2,4]
+ CRUSH rule 3 x 124 [3,0]
+ CRUSH rule 3 x 125 [0,7]
+ CRUSH rule 3 x 126 [4,1]
+ CRUSH rule 3 x 127 [3,6]
+ CRUSH rule 3 x 128 [3,8]
+ CRUSH rule 3 x 129 [0,4]
+ CRUSH rule 3 x 130 [3,8]
+ CRUSH rule 3 x 131 [1,5]
+ CRUSH rule 3 x 132 [1,3]
+ CRUSH rule 3 x 133 [3,6]
+ CRUSH rule 3 x 134 [1,7]
+ CRUSH rule 3 x 135 [5,7]
+ CRUSH rule 3 x 136 [2,3]
+ CRUSH rule 3 x 137 [7,3]
+ CRUSH rule 3 x 138 [8,3]
+ CRUSH rule 3 x 139 [3,0]
+ CRUSH rule 3 x 140 [1,8]
+ CRUSH rule 3 x 141 [6,0]
+ CRUSH rule 3 x 142 [3,2]
+ CRUSH rule 3 x 143 [5,7]
+ CRUSH rule 3 x 144 [8,2]
+ CRUSH rule 3 x 145 [8,4]
+ CRUSH rule 3 x 146 [2,8]
+ CRUSH rule 3 x 147 [2,6]
+ CRUSH rule 3 x 148 [3,0]
+ CRUSH rule 3 x 149 [4,6]
+ CRUSH rule 3 x 150 [1,8]
+ CRUSH rule 3 x 151 [3,8]
+ CRUSH rule 3 x 152 [8,3]
+ CRUSH rule 3 x 153 [8,3]
+ CRUSH rule 3 x 154 [3,0]
+ CRUSH rule 3 x 155 [3,6]
+ CRUSH rule 3 x 156 [4,2]
+ CRUSH rule 3 x 157 [4,1]
+ CRUSH rule 3 x 158 [2,6]
+ CRUSH rule 3 x 159 [7,0]
+ CRUSH rule 3 x 160 [2,8]
+ CRUSH rule 3 x 161 [1,3]
+ CRUSH rule 3 x 162 [0,8]
+ CRUSH rule 3 x 163 [5,8]
+ CRUSH rule 3 x 164 [7,2]
+ CRUSH rule 3 x 165 [7,2]
+ CRUSH rule 3 x 166 [2,3]
+ CRUSH rule 3 x 167 [0,6]
+ CRUSH rule 3 x 168 [4,0]
+ CRUSH rule 3 x 169 [2,7]
+ CRUSH rule 3 x 170 [1,3]
+ CRUSH rule 3 x 171 [7,3]
+ CRUSH rule 3 x 172 [0,8]
+ CRUSH rule 3 x 173 [8,5]
+ CRUSH rule 3 x 174 [1,4]
+ CRUSH rule 3 x 175 [6,0]
+ CRUSH rule 3 x 176 [4,2]
+ CRUSH rule 3 x 177 [5,1]
+ CRUSH rule 3 x 178 [3,0]
+ CRUSH rule 3 x 179 [4,1]
+ CRUSH rule 3 x 180 [3,7]
+ CRUSH rule 3 x 181 [6,0]
+ CRUSH rule 3 x 182 [8,3]
+ CRUSH rule 3 x 183 [7,5]
+ CRUSH rule 3 x 184 [5,6]
+ CRUSH rule 3 x 185 [6,0]
+ CRUSH rule 3 x 186 [2,4]
+ CRUSH rule 3 x 187 [1,6]
+ CRUSH rule 3 x 188 [1,6]
+ CRUSH rule 3 x 189 [0,6]
+ CRUSH rule 3 x 190 [4,1]
+ CRUSH rule 3 x 191 [7,0]
+ CRUSH rule 3 x 192 [5,2]
+ CRUSH rule 3 x 193 [4,0]
+ CRUSH rule 3 x 194 [1,3]
+ CRUSH rule 3 x 195 [6,5]
+ CRUSH rule 3 x 196 [6,1]
+ CRUSH rule 3 x 197 [6,5]
+ CRUSH rule 3 x 198 [2,3]
+ CRUSH rule 3 x 199 [0,5]
+ CRUSH rule 3 x 200 [0,3]
+ CRUSH rule 3 x 201 [7,1]
+ CRUSH rule 3 x 202 [6,4]
+ CRUSH rule 3 x 203 [4,6]
+ CRUSH rule 3 x 204 [2,4]
+ CRUSH rule 3 x 205 [0,7]
+ CRUSH rule 3 x 206 [0,8]
+ CRUSH rule 3 x 207 [3,2]
+ CRUSH rule 3 x 208 [7,2]
+ CRUSH rule 3 x 209 [1,8]
+ CRUSH rule 3 x 210 [1,4]
+ CRUSH rule 3 x 211 [5,2]
+ CRUSH rule 3 x 212 [7,5]
+ CRUSH rule 3 x 213 [8,4]
+ CRUSH rule 3 x 214 [4,7]
+ CRUSH rule 3 x 215 [8,0]
+ CRUSH rule 3 x 216 [5,2]
+ CRUSH rule 3 x 217 [0,7]
+ CRUSH rule 3 x 218 [0,7]
+ CRUSH rule 3 x 219 [4,6]
+ CRUSH rule 3 x 220 [5,8]
+ CRUSH rule 3 x 221 [3,7]
+ CRUSH rule 3 x 222 [6,4]
+ CRUSH rule 3 x 223 [1,3]
+ CRUSH rule 3 x 224 [1,3]
+ CRUSH rule 3 x 225 [8,0]
+ CRUSH rule 3 x 226 [7,0]
+ CRUSH rule 3 x 227 [3,0]
+ CRUSH rule 3 x 228 [5,6]
+ CRUSH rule 3 x 229 [3,7]
+ CRUSH rule 3 x 230 [4,6]
+ CRUSH rule 3 x 231 [4,7]
+ CRUSH rule 3 x 232 [2,8]
+ CRUSH rule 3 x 233 [3,6]
+ CRUSH rule 3 x 234 [0,4]
+ CRUSH rule 3 x 235 [3,6]
+ CRUSH rule 3 x 236 [5,0]
+ CRUSH rule 3 x 237 [4,8]
+ CRUSH rule 3 x 238 [4,2]
+ CRUSH rule 3 x 239 [8,3]
+ CRUSH rule 3 x 240 [5,8]
+ CRUSH rule 3 x 241 [3,1]
+ CRUSH rule 3 x 242 [3,2]
+ CRUSH rule 3 x 243 [4,8]
+ CRUSH rule 3 x 244 [4,6]
+ CRUSH rule 3 x 245 [7,2]
+ CRUSH rule 3 x 246 [1,3]
+ CRUSH rule 3 x 247 [6,1]
+ CRUSH rule 3 x 248 [8,0]
+ CRUSH rule 3 x 249 [2,5]
+ CRUSH rule 3 x 250 [2,3]
+ CRUSH rule 3 x 251 [2,3]
+ CRUSH rule 3 x 252 [3,7]
+ CRUSH rule 3 x 253 [3,0]
+ CRUSH rule 3 x 254 [3,2]
+ CRUSH rule 3 x 255 [1,7]
+ CRUSH rule 3 x 256 [5,6]
+ CRUSH rule 3 x 257 [2,6]
+ CRUSH rule 3 x 258 [5,2]
+ CRUSH rule 3 x 259 [4,6]
+ CRUSH rule 3 x 260 [3,8]
+ CRUSH rule 3 x 261 [8,5]
+ CRUSH rule 3 x 262 [5,6]
+ CRUSH rule 3 x 263 [6,1]
+ CRUSH rule 3 x 264 [3,6]
+ CRUSH rule 3 x 265 [8,4]
+ CRUSH rule 3 x 266 [8,0]
+ CRUSH rule 3 x 267 [2,4]
+ CRUSH rule 3 x 268 [0,6]
+ CRUSH rule 3 x 269 [0,6]
+ CRUSH rule 3 x 270 [5,1]
+ CRUSH rule 3 x 271 [7,4]
+ CRUSH rule 3 x 272 [2,6]
+ CRUSH rule 3 x 273 [3,2]
+ CRUSH rule 3 x 274 [6,3]
+ CRUSH rule 3 x 275 [4,8]
+ CRUSH rule 3 x 276 [7,0]
+ CRUSH rule 3 x 277 [6,4]
+ CRUSH rule 3 x 278 [6,2]
+ CRUSH rule 3 x 279 [8,5]
+ CRUSH rule 3 x 280 [0,7]
+ CRUSH rule 3 x 281 [8,1]
+ CRUSH rule 3 x 282 [3,2]
+ CRUSH rule 3 x 283 [8,1]
+ CRUSH rule 3 x 284 [6,3]
+ CRUSH rule 3 x 285 [5,7]
+ CRUSH rule 3 x 286 [2,8]
+ CRUSH rule 3 x 287 [0,3]
+ CRUSH rule 3 x 288 [8,1]
+ CRUSH rule 3 x 289 [4,6]
+ CRUSH rule 3 x 290 [1,5]
+ CRUSH rule 3 x 291 [0,5]
+ CRUSH rule 3 x 292 [8,1]
+ CRUSH rule 3 x 293 [6,2]
+ CRUSH rule 3 x 294 [7,5]
+ CRUSH rule 3 x 295 [4,6]
+ CRUSH rule 3 x 296 [3,1]
+ CRUSH rule 3 x 297 [6,0]
+ CRUSH rule 3 x 298 [1,5]
+ CRUSH rule 3 x 299 [2,8]
+ CRUSH rule 3 x 300 [8,4]
+ CRUSH rule 3 x 301 [0,7]
+ CRUSH rule 3 x 302 [3,1]
+ CRUSH rule 3 x 303 [7,4]
+ CRUSH rule 3 x 304 [2,6]
+ CRUSH rule 3 x 305 [5,6]
+ CRUSH rule 3 x 306 [0,8]
+ CRUSH rule 3 x 307 [0,7]
+ CRUSH rule 3 x 308 [0,8]
+ CRUSH rule 3 x 309 [7,4]
+ CRUSH rule 3 x 310 [4,0]
+ CRUSH rule 3 x 311 [3,8]
+ CRUSH rule 3 x 312 [2,6]
+ CRUSH rule 3 x 313 [5,1]
+ CRUSH rule 3 x 314 [4,0]
+ CRUSH rule 3 x 315 [2,3]
+ CRUSH rule 3 x 316 [6,4]
+ CRUSH rule 3 x 317 [2,7]
+ CRUSH rule 3 x 318 [8,1]
+ CRUSH rule 3 x 319 [5,2]
+ CRUSH rule 3 x 320 [3,8]
+ CRUSH rule 3 x 321 [1,5]
+ CRUSH rule 3 x 322 [2,6]
+ CRUSH rule 3 x 323 [4,7]
+ CRUSH rule 3 x 324 [7,2]
+ CRUSH rule 3 x 325 [4,6]
+ CRUSH rule 3 x 326 [3,1]
+ CRUSH rule 3 x 327 [0,8]
+ CRUSH rule 3 x 328 [7,3]
+ CRUSH rule 3 x 329 [5,7]
+ CRUSH rule 3 x 330 [3,7]
+ CRUSH rule 3 x 331 [2,7]
+ CRUSH rule 3 x 332 [2,3]
+ CRUSH rule 3 x 333 [6,4]
+ CRUSH rule 3 x 334 [8,3]
+ CRUSH rule 3 x 335 [7,2]
+ CRUSH rule 3 x 336 [4,6]
+ CRUSH rule 3 x 337 [7,0]
+ CRUSH rule 3 x 338 [5,8]
+ CRUSH rule 3 x 339 [7,5]
+ CRUSH rule 3 x 340 [2,6]
+ CRUSH rule 3 x 341 [5,1]
+ CRUSH rule 3 x 342 [0,8]
+ CRUSH rule 3 x 343 [6,5]
+ CRUSH rule 3 x 344 [6,1]
+ CRUSH rule 3 x 345 [4,7]
+ CRUSH rule 3 x 346 [8,0]
+ CRUSH rule 3 x 347 [3,0]
+ CRUSH rule 3 x 348 [8,0]
+ CRUSH rule 3 x 349 [1,7]
+ CRUSH rule 3 x 350 [8,5]
+ CRUSH rule 3 x 351 [3,8]
+ CRUSH rule 3 x 352 [1,8]
+ CRUSH rule 3 x 353 [6,5]
+ CRUSH rule 3 x 354 [0,5]
+ CRUSH rule 3 x 355 [3,8]
+ CRUSH rule 3 x 356 [3,1]
+ CRUSH rule 3 x 357 [6,1]
+ CRUSH rule 3 x 358 [2,8]
+ CRUSH rule 3 x 359 [6,0]
+ CRUSH rule 3 x 360 [5,0]
+ CRUSH rule 3 x 361 [8,5]
+ CRUSH rule 3 x 362 [4,0]
+ CRUSH rule 3 x 363 [4,2]
+ CRUSH rule 3 x 364 [2,5]
+ CRUSH rule 3 x 365 [6,4]
+ CRUSH rule 3 x 366 [7,0]
+ CRUSH rule 3 x 367 [4,2]
+ CRUSH rule 3 x 368 [7,3]
+ CRUSH rule 3 x 369 [3,7]
+ CRUSH rule 3 x 370 [8,2]
+ CRUSH rule 3 x 371 [1,5]
+ CRUSH rule 3 x 372 [3,1]
+ CRUSH rule 3 x 373 [0,6]
+ CRUSH rule 3 x 374 [3,8]
+ CRUSH rule 3 x 375 [6,5]
+ CRUSH rule 3 x 376 [7,1]
+ CRUSH rule 3 x 377 [1,4]
+ CRUSH rule 3 x 378 [0,6]
+ CRUSH rule 3 x 379 [8,3]
+ CRUSH rule 3 x 380 [2,3]
+ CRUSH rule 3 x 381 [0,3]
+ CRUSH rule 3 x 382 [1,3]
+ CRUSH rule 3 x 383 [4,7]
+ CRUSH rule 3 x 384 [7,0]
+ CRUSH rule 3 x 385 [7,4]
+ CRUSH rule 3 x 386 [0,4]
+ CRUSH rule 3 x 387 [1,5]
+ CRUSH rule 3 x 388 [5,2]
+ CRUSH rule 3 x 389 [1,4]
+ CRUSH rule 3 x 390 [5,8]
+ CRUSH rule 3 x 391 [5,6]
+ CRUSH rule 3 x 392 [1,7]
+ CRUSH rule 3 x 393 [4,0]
+ CRUSH rule 3 x 394 [4,8]
+ CRUSH rule 3 x 395 [4,0]
+ CRUSH rule 3 x 396 [4,2]
+ CRUSH rule 3 x 397 [2,4]
+ CRUSH rule 3 x 398 [2,3]
+ CRUSH rule 3 x 399 [8,5]
+ CRUSH rule 3 x 400 [8,1]
+ CRUSH rule 3 x 401 [0,3]
+ CRUSH rule 3 x 402 [7,4]
+ CRUSH rule 3 x 403 [0,4]
+ CRUSH rule 3 x 404 [4,0]
+ CRUSH rule 3 x 405 [6,4]
+ CRUSH rule 3 x 406 [2,6]
+ CRUSH rule 3 x 407 [2,7]
+ CRUSH rule 3 x 408 [4,1]
+ CRUSH rule 3 x 409 [7,5]
+ CRUSH rule 3 x 410 [8,3]
+ CRUSH rule 3 x 411 [2,7]
+ CRUSH rule 3 x 412 [0,3]
+ CRUSH rule 3 x 413 [5,2]
+ CRUSH rule 3 x 414 [4,0]
+ CRUSH rule 3 x 415 [0,6]
+ CRUSH rule 3 x 416 [2,8]
+ CRUSH rule 3 x 417 [8,0]
+ CRUSH rule 3 x 418 [7,1]
+ CRUSH rule 3 x 419 [8,5]
+ CRUSH rule 3 x 420 [1,4]
+ CRUSH rule 3 x 421 [8,3]
+ CRUSH rule 3 x 422 [6,3]
+ CRUSH rule 3 x 423 [0,3]
+ CRUSH rule 3 x 424 [8,5]
+ CRUSH rule 3 x 425 [1,4]
+ CRUSH rule 3 x 426 [6,2]
+ CRUSH rule 3 x 427 [0,8]
+ CRUSH rule 3 x 428 [5,8]
+ CRUSH rule 3 x 429 [4,8]
+ CRUSH rule 3 x 430 [3,7]
+ CRUSH rule 3 x 431 [5,0]
+ CRUSH rule 3 x 432 [7,0]
+ CRUSH rule 3 x 433 [6,4]
+ CRUSH rule 3 x 434 [5,0]
+ CRUSH rule 3 x 435 [0,4]
+ CRUSH rule 3 x 436 [4,0]
+ CRUSH rule 3 x 437 [7,3]
+ CRUSH rule 3 x 438 [0,5]
+ CRUSH rule 3 x 439 [1,3]
+ CRUSH rule 3 x 440 [2,6]
+ CRUSH rule 3 x 441 [5,8]
+ CRUSH rule 3 x 442 [2,3]
+ CRUSH rule 3 x 443 [6,0]
+ CRUSH rule 3 x 444 [7,1]
+ CRUSH rule 3 x 445 [6,4]
+ CRUSH rule 3 x 446 [4,0]
+ CRUSH rule 3 x 447 [2,4]
+ CRUSH rule 3 x 448 [7,0]
+ CRUSH rule 3 x 449 [7,4]
+ CRUSH rule 3 x 450 [4,0]
+ CRUSH rule 3 x 451 [6,5]
+ CRUSH rule 3 x 452 [8,4]
+ CRUSH rule 3 x 453 [6,5]
+ CRUSH rule 3 x 454 [6,5]
+ CRUSH rule 3 x 455 [2,8]
+ CRUSH rule 3 x 456 [6,2]
+ CRUSH rule 3 x 457 [7,1]
+ CRUSH rule 3 x 458 [2,8]
+ CRUSH rule 3 x 459 [2,6]
+ CRUSH rule 3 x 460 [6,5]
+ CRUSH rule 3 x 461 [6,4]
+ CRUSH rule 3 x 462 [8,0]
+ CRUSH rule 3 x 463 [6,0]
+ CRUSH rule 3 x 464 [7,4]
+ CRUSH rule 3 x 465 [7,1]
+ CRUSH rule 3 x 466 [5,6]
+ CRUSH rule 3 x 467 [6,5]
+ CRUSH rule 3 x 468 [7,2]
+ CRUSH rule 3 x 469 [7,2]
+ CRUSH rule 3 x 470 [3,0]
+ CRUSH rule 3 x 471 [0,6]
+ CRUSH rule 3 x 472 [5,0]
+ CRUSH rule 3 x 473 [1,4]
+ CRUSH rule 3 x 474 [6,1]
+ CRUSH rule 3 x 475 [6,2]
+ CRUSH rule 3 x 476 [4,7]
+ CRUSH rule 3 x 477 [5,6]
+ CRUSH rule 3 x 478 [6,1]
+ CRUSH rule 3 x 479 [0,3]
+ CRUSH rule 3 x 480 [1,6]
+ CRUSH rule 3 x 481 [2,5]
+ CRUSH rule 3 x 482 [4,8]
+ CRUSH rule 3 x 483 [0,6]
+ CRUSH rule 3 x 484 [1,8]
+ CRUSH rule 3 x 485 [4,8]
+ CRUSH rule 3 x 486 [4,0]
+ CRUSH rule 3 x 487 [5,0]
+ CRUSH rule 3 x 488 [5,7]
+ CRUSH rule 3 x 489 [2,8]
+ CRUSH rule 3 x 490 [6,4]
+ CRUSH rule 3 x 491 [1,6]
+ CRUSH rule 3 x 492 [6,5]
+ CRUSH rule 3 x 493 [0,8]
+ CRUSH rule 3 x 494 [1,6]
+ CRUSH rule 3 x 495 [3,0]
+ CRUSH rule 3 x 496 [7,5]
+ CRUSH rule 3 x 497 [5,7]
+ CRUSH rule 3 x 498 [0,4]
+ CRUSH rule 3 x 499 [8,5]
+ CRUSH rule 3 x 500 [3,6]
+ CRUSH rule 3 x 501 [0,7]
+ CRUSH rule 3 x 502 [7,1]
+ CRUSH rule 3 x 503 [2,3]
+ CRUSH rule 3 x 504 [5,8]
+ CRUSH rule 3 x 505 [0,7]
+ CRUSH rule 3 x 506 [5,1]
+ CRUSH rule 3 x 507 [6,0]
+ CRUSH rule 3 x 508 [0,4]
+ CRUSH rule 3 x 509 [7,4]
+ CRUSH rule 3 x 510 [6,2]
+ CRUSH rule 3 x 511 [5,6]
+ CRUSH rule 3 x 512 [7,2]
+ CRUSH rule 3 x 513 [7,2]
+ CRUSH rule 3 x 514 [4,7]
+ CRUSH rule 3 x 515 [8,3]
+ CRUSH rule 3 x 516 [4,1]
+ CRUSH rule 3 x 517 [7,0]
+ CRUSH rule 3 x 518 [4,6]
+ CRUSH rule 3 x 519 [7,3]
+ CRUSH rule 3 x 520 [2,8]
+ CRUSH rule 3 x 521 [8,0]
+ CRUSH rule 3 x 522 [6,0]
+ CRUSH rule 3 x 523 [4,1]
+ CRUSH rule 3 x 524 [0,4]
+ CRUSH rule 3 x 525 [0,3]
+ CRUSH rule 3 x 526 [1,3]
+ CRUSH rule 3 x 527 [0,4]
+ CRUSH rule 3 x 528 [5,2]
+ CRUSH rule 3 x 529 [5,6]
+ CRUSH rule 3 x 530 [6,4]
+ CRUSH rule 3 x 531 [6,0]
+ CRUSH rule 3 x 532 [6,5]
+ CRUSH rule 3 x 533 [5,8]
+ CRUSH rule 3 x 534 [7,4]
+ CRUSH rule 3 x 535 [8,0]
+ CRUSH rule 3 x 536 [6,2]
+ CRUSH rule 3 x 537 [3,8]
+ CRUSH rule 3 x 538 [6,4]
+ CRUSH rule 3 x 539 [8,4]
+ CRUSH rule 3 x 540 [0,7]
+ CRUSH rule 3 x 541 [2,5]
+ CRUSH rule 3 x 542 [3,0]
+ CRUSH rule 3 x 543 [6,2]
+ CRUSH rule 3 x 544 [3,7]
+ CRUSH rule 3 x 545 [5,7]
+ CRUSH rule 3 x 546 [6,2]
+ CRUSH rule 3 x 547 [8,1]
+ CRUSH rule 3 x 548 [5,1]
+ CRUSH rule 3 x 549 [5,7]
+ CRUSH rule 3 x 550 [0,5]
+ CRUSH rule 3 x 551 [7,4]
+ CRUSH rule 3 x 552 [5,7]
+ CRUSH rule 3 x 553 [4,2]
+ CRUSH rule 3 x 554 [0,6]
+ CRUSH rule 3 x 555 [5,0]
+ CRUSH rule 3 x 556 [3,6]
+ CRUSH rule 3 x 557 [7,4]
+ CRUSH rule 3 x 558 [3,2]
+ CRUSH rule 3 x 559 [4,1]
+ CRUSH rule 3 x 560 [8,4]
+ CRUSH rule 3 x 561 [6,4]
+ CRUSH rule 3 x 562 [3,0]
+ CRUSH rule 3 x 563 [2,7]
+ CRUSH rule 3 x 564 [5,1]
+ CRUSH rule 3 x 565 [3,8]
+ CRUSH rule 3 x 566 [4,6]
+ CRUSH rule 3 x 567 [3,7]
+ CRUSH rule 3 x 568 [7,4]
+ CRUSH rule 3 x 569 [3,1]
+ CRUSH rule 3 x 570 [1,4]
+ CRUSH rule 3 x 571 [3,6]
+ CRUSH rule 3 x 572 [3,0]
+ CRUSH rule 3 x 573 [3,1]
+ CRUSH rule 3 x 574 [2,3]
+ CRUSH rule 3 x 575 [8,1]
+ CRUSH rule 3 x 576 [4,7]
+ CRUSH rule 3 x 577 [8,2]
+ CRUSH rule 3 x 578 [6,0]
+ CRUSH rule 3 x 579 [3,2]
+ CRUSH rule 3 x 580 [3,0]
+ CRUSH rule 3 x 581 [7,1]
+ CRUSH rule 3 x 582 [2,8]
+ CRUSH rule 3 x 583 [6,2]
+ CRUSH rule 3 x 584 [8,0]
+ CRUSH rule 3 x 585 [7,0]
+ CRUSH rule 3 x 586 [0,8]
+ CRUSH rule 3 x 587 [2,5]
+ CRUSH rule 3 x 588 [3,8]
+ CRUSH rule 3 x 589 [7,1]
+ CRUSH rule 3 x 590 [6,2]
+ CRUSH rule 3 x 591 [5,2]
+ CRUSH rule 3 x 592 [2,4]
+ CRUSH rule 3 x 593 [0,8]
+ CRUSH rule 3 x 594 [0,6]
+ CRUSH rule 3 x 595 [7,1]
+ CRUSH rule 3 x 596 [4,0]
+ CRUSH rule 3 x 597 [3,1]
+ CRUSH rule 3 x 598 [3,0]
+ CRUSH rule 3 x 599 [5,1]
+ CRUSH rule 3 x 600 [7,0]
+ CRUSH rule 3 x 601 [0,6]
+ CRUSH rule 3 x 602 [3,8]
+ CRUSH rule 3 x 603 [5,0]
+ CRUSH rule 3 x 604 [7,4]
+ CRUSH rule 3 x 605 [3,2]
+ CRUSH rule 3 x 606 [2,7]
+ CRUSH rule 3 x 607 [0,5]
+ CRUSH rule 3 x 608 [5,1]
+ CRUSH rule 3 x 609 [5,0]
+ CRUSH rule 3 x 610 [3,8]
+ CRUSH rule 3 x 611 [1,8]
+ CRUSH rule 3 x 612 [2,8]
+ CRUSH rule 3 x 613 [7,1]
+ CRUSH rule 3 x 614 [7,2]
+ CRUSH rule 3 x 615 [6,2]
+ CRUSH rule 3 x 616 [0,7]
+ CRUSH rule 3 x 617 [6,2]
+ CRUSH rule 3 x 618 [7,3]
+ CRUSH rule 3 x 619 [5,0]
+ CRUSH rule 3 x 620 [4,1]
+ CRUSH rule 3 x 621 [5,6]
+ CRUSH rule 3 x 622 [0,3]
+ CRUSH rule 3 x 623 [0,8]
+ CRUSH rule 3 x 624 [3,2]
+ CRUSH rule 3 x 625 [2,5]
+ CRUSH rule 3 x 626 [7,2]
+ CRUSH rule 3 x 627 [2,6]
+ CRUSH rule 3 x 628 [8,1]
+ CRUSH rule 3 x 629 [2,6]
+ CRUSH rule 3 x 630 [2,6]
+ CRUSH rule 3 x 631 [0,6]
+ CRUSH rule 3 x 632 [7,0]
+ CRUSH rule 3 x 633 [8,4]
+ CRUSH rule 3 x 634 [0,5]
+ CRUSH rule 3 x 635 [5,6]
+ CRUSH rule 3 x 636 [1,3]
+ CRUSH rule 3 x 637 [4,0]
+ CRUSH rule 3 x 638 [6,2]
+ CRUSH rule 3 x 639 [4,0]
+ CRUSH rule 3 x 640 [3,2]
+ CRUSH rule 3 x 641 [7,2]
+ CRUSH rule 3 x 642 [2,8]
+ CRUSH rule 3 x 643 [3,0]
+ CRUSH rule 3 x 644 [8,0]
+ CRUSH rule 3 x 645 [5,7]
+ CRUSH rule 3 x 646 [8,0]
+ CRUSH rule 3 x 647 [7,0]
+ CRUSH rule 3 x 648 [0,8]
+ CRUSH rule 3 x 649 [4,7]
+ CRUSH rule 3 x 650 [7,5]
+ CRUSH rule 3 x 651 [3,6]
+ CRUSH rule 3 x 652 [3,6]
+ CRUSH rule 3 x 653 [8,3]
+ CRUSH rule 3 x 654 [7,4]
+ CRUSH rule 3 x 655 [0,5]
+ CRUSH rule 3 x 656 [4,7]
+ CRUSH rule 3 x 657 [6,0]
+ CRUSH rule 3 x 658 [5,8]
+ CRUSH rule 3 x 659 [4,7]
+ CRUSH rule 3 x 660 [7,3]
+ CRUSH rule 3 x 661 [1,7]
+ CRUSH rule 3 x 662 [4,2]
+ CRUSH rule 3 x 663 [1,3]
+ CRUSH rule 3 x 664 [1,3]
+ CRUSH rule 3 x 665 [5,6]
+ CRUSH rule 3 x 666 [2,7]
+ CRUSH rule 3 x 667 [1,3]
+ CRUSH rule 3 x 668 [3,7]
+ CRUSH rule 3 x 669 [6,3]
+ CRUSH rule 3 x 670 [4,1]
+ CRUSH rule 3 x 671 [0,8]
+ CRUSH rule 3 x 672 [4,2]
+ CRUSH rule 3 x 673 [5,0]
+ CRUSH rule 3 x 674 [3,0]
+ CRUSH rule 3 x 675 [0,8]
+ CRUSH rule 3 x 676 [0,3]
+ CRUSH rule 3 x 677 [4,1]
+ CRUSH rule 3 x 678 [2,3]
+ CRUSH rule 3 x 679 [6,0]
+ CRUSH rule 3 x 680 [0,3]
+ CRUSH rule 3 x 681 [4,6]
+ CRUSH rule 3 x 682 [0,4]
+ CRUSH rule 3 x 683 [0,3]
+ CRUSH rule 3 x 684 [7,2]
+ CRUSH rule 3 x 685 [7,2]
+ CRUSH rule 3 x 686 [1,5]
+ CRUSH rule 3 x 687 [3,7]
+ CRUSH rule 3 x 688 [5,6]
+ CRUSH rule 3 x 689 [6,5]
+ CRUSH rule 3 x 690 [8,0]
+ CRUSH rule 3 x 691 [3,1]
+ CRUSH rule 3 x 692 [7,1]
+ CRUSH rule 3 x 693 [6,5]
+ CRUSH rule 3 x 694 [6,4]
+ CRUSH rule 3 x 695 [0,6]
+ CRUSH rule 3 x 696 [1,5]
+ CRUSH rule 3 x 697 [6,0]
+ CRUSH rule 3 x 698 [6,0]
+ CRUSH rule 3 x 699 [1,8]
+ CRUSH rule 3 x 700 [0,4]
+ CRUSH rule 3 x 701 [4,0]
+ CRUSH rule 3 x 702 [3,0]
+ CRUSH rule 3 x 703 [8,4]
+ CRUSH rule 3 x 704 [0,4]
+ CRUSH rule 3 x 705 [8,0]
+ CRUSH rule 3 x 706 [1,5]
+ CRUSH rule 3 x 707 [7,3]
+ CRUSH rule 3 x 708 [3,7]
+ CRUSH rule 3 x 709 [6,5]
+ CRUSH rule 3 x 710 [8,5]
+ CRUSH rule 3 x 711 [2,4]
+ CRUSH rule 3 x 712 [2,3]
+ CRUSH rule 3 x 713 [6,3]
+ CRUSH rule 3 x 714 [3,0]
+ CRUSH rule 3 x 715 [1,3]
+ CRUSH rule 3 x 716 [3,6]
+ CRUSH rule 3 x 717 [8,0]
+ CRUSH rule 3 x 718 [3,7]
+ CRUSH rule 3 x 719 [2,6]
+ CRUSH rule 3 x 720 [6,0]
+ CRUSH rule 3 x 721 [5,7]
+ CRUSH rule 3 x 722 [5,7]
+ CRUSH rule 3 x 723 [5,2]
+ CRUSH rule 3 x 724 [0,7]
+ CRUSH rule 3 x 725 [0,4]
+ CRUSH rule 3 x 726 [3,7]
+ CRUSH rule 3 x 727 [4,7]
+ CRUSH rule 3 x 728 [2,6]
+ CRUSH rule 3 x 729 [5,6]
+ CRUSH rule 3 x 730 [3,8]
+ CRUSH rule 3 x 731 [4,1]
+ CRUSH rule 3 x 732 [1,4]
+ CRUSH rule 3 x 733 [5,6]
+ CRUSH rule 3 x 734 [6,5]
+ CRUSH rule 3 x 735 [4,6]
+ CRUSH rule 3 x 736 [3,8]
+ CRUSH rule 3 x 737 [1,8]
+ CRUSH rule 3 x 738 [5,1]
+ CRUSH rule 3 x 739 [0,7]
+ CRUSH rule 3 x 740 [0,7]
+ CRUSH rule 3 x 741 [7,2]
+ CRUSH rule 3 x 742 [8,2]
+ CRUSH rule 3 x 743 [7,0]
+ CRUSH rule 3 x 744 [4,6]
+ CRUSH rule 3 x 745 [3,0]
+ CRUSH rule 3 x 746 [4,1]
+ CRUSH rule 3 x 747 [6,2]
+ CRUSH rule 3 x 748 [2,6]
+ CRUSH rule 3 x 749 [4,7]
+ CRUSH rule 3 x 750 [1,7]
+ CRUSH rule 3 x 751 [2,7]
+ CRUSH rule 3 x 752 [8,0]
+ CRUSH rule 3 x 753 [7,4]
+ CRUSH rule 3 x 754 [8,5]
+ CRUSH rule 3 x 755 [1,6]
+ CRUSH rule 3 x 756 [5,8]
+ CRUSH rule 3 x 757 [8,0]
+ CRUSH rule 3 x 758 [6,1]
+ CRUSH rule 3 x 759 [8,5]
+ CRUSH rule 3 x 760 [1,5]
+ CRUSH rule 3 x 761 [4,2]
+ CRUSH rule 3 x 762 [2,8]
+ CRUSH rule 3 x 763 [8,3]
+ CRUSH rule 3 x 764 [1,7]
+ CRUSH rule 3 x 765 [6,4]
+ CRUSH rule 3 x 766 [8,3]
+ CRUSH rule 3 x 767 [1,8]
+ CRUSH rule 3 x 768 [8,5]
+ CRUSH rule 3 x 769 [6,2]
+ CRUSH rule 3 x 770 [6,1]
+ CRUSH rule 3 x 771 [7,2]
+ CRUSH rule 3 x 772 [8,4]
+ CRUSH rule 3 x 773 [3,2]
+ CRUSH rule 3 x 774 [4,7]
+ CRUSH rule 3 x 775 [6,5]
+ CRUSH rule 3 x 776 [7,0]
+ CRUSH rule 3 x 777 [3,0]
+ CRUSH rule 3 x 778 [1,6]
+ CRUSH rule 3 x 779 [2,6]
+ CRUSH rule 3 x 780 [0,5]
+ CRUSH rule 3 x 781 [6,5]
+ CRUSH rule 3 x 782 [5,2]
+ CRUSH rule 3 x 783 [7,0]
+ CRUSH rule 3 x 784 [0,5]
+ CRUSH rule 3 x 785 [6,1]
+ CRUSH rule 3 x 786 [7,3]
+ CRUSH rule 3 x 787 [1,8]
+ CRUSH rule 3 x 788 [6,0]
+ CRUSH rule 3 x 789 [0,5]
+ CRUSH rule 3 x 790 [8,3]
+ CRUSH rule 3 x 791 [3,6]
+ CRUSH rule 3 x 792 [5,6]
+ CRUSH rule 3 x 793 [6,2]
+ CRUSH rule 3 x 794 [2,8]
+ CRUSH rule 3 x 795 [0,4]
+ CRUSH rule 3 x 796 [3,7]
+ CRUSH rule 3 x 797 [2,3]
+ CRUSH rule 3 x 798 [6,0]
+ CRUSH rule 3 x 799 [5,2]
+ CRUSH rule 3 x 800 [5,2]
+ CRUSH rule 3 x 801 [3,7]
+ CRUSH rule 3 x 802 [1,6]
+ CRUSH rule 3 x 803 [0,4]
+ CRUSH rule 3 x 804 [6,0]
+ CRUSH rule 3 x 805 [3,8]
+ CRUSH rule 3 x 806 [1,5]
+ CRUSH rule 3 x 807 [5,7]
+ CRUSH rule 3 x 808 [4,7]
+ CRUSH rule 3 x 809 [1,3]
+ CRUSH rule 3 x 810 [5,7]
+ CRUSH rule 3 x 811 [8,4]
+ CRUSH rule 3 x 812 [8,3]
+ CRUSH rule 3 x 813 [6,4]
+ CRUSH rule 3 x 814 [3,8]
+ CRUSH rule 3 x 815 [3,1]
+ CRUSH rule 3 x 816 [2,6]
+ CRUSH rule 3 x 817 [4,6]
+ CRUSH rule 3 x 818 [3,1]
+ CRUSH rule 3 x 819 [5,0]
+ CRUSH rule 3 x 820 [3,7]
+ CRUSH rule 3 x 821 [4,8]
+ CRUSH rule 3 x 822 [2,3]
+ CRUSH rule 3 x 823 [4,7]
+ CRUSH rule 3 x 824 [3,7]
+ CRUSH rule 3 x 825 [2,8]
+ CRUSH rule 3 x 826 [7,1]
+ CRUSH rule 3 x 827 [0,6]
+ CRUSH rule 3 x 828 [2,5]
+ CRUSH rule 3 x 829 [5,6]
+ CRUSH rule 3 x 830 [2,4]
+ CRUSH rule 3 x 831 [1,6]
+ CRUSH rule 3 x 832 [4,8]
+ CRUSH rule 3 x 833 [2,6]
+ CRUSH rule 3 x 834 [3,0]
+ CRUSH rule 3 x 835 [8,5]
+ CRUSH rule 3 x 836 [3,8]
+ CRUSH rule 3 x 837 [6,4]
+ CRUSH rule 3 x 838 [6,0]
+ CRUSH rule 3 x 839 [5,2]
+ CRUSH rule 3 x 840 [7,3]
+ CRUSH rule 3 x 841 [4,8]
+ CRUSH rule 3 x 842 [2,5]
+ CRUSH rule 3 x 843 [6,4]
+ CRUSH rule 3 x 844 [4,8]
+ CRUSH rule 3 x 845 [3,6]
+ CRUSH rule 3 x 846 [3,0]
+ CRUSH rule 3 x 847 [0,8]
+ CRUSH rule 3 x 848 [2,6]
+ CRUSH rule 3 x 849 [4,8]
+ CRUSH rule 3 x 850 [1,5]
+ CRUSH rule 3 x 851 [6,5]
+ CRUSH rule 3 x 852 [7,4]
+ CRUSH rule 3 x 853 [6,2]
+ CRUSH rule 3 x 854 [7,0]
+ CRUSH rule 3 x 855 [5,7]
+ CRUSH rule 3 x 856 [6,3]
+ CRUSH rule 3 x 857 [8,4]
+ CRUSH rule 3 x 858 [6,5]
+ CRUSH rule 3 x 859 [6,0]
+ CRUSH rule 3 x 860 [4,2]
+ CRUSH rule 3 x 861 [8,4]
+ CRUSH rule 3 x 862 [6,0]
+ CRUSH rule 3 x 863 [8,1]
+ CRUSH rule 3 x 864 [5,6]
+ CRUSH rule 3 x 865 [8,0]
+ CRUSH rule 3 x 866 [3,6]
+ CRUSH rule 3 x 867 [6,3]
+ CRUSH rule 3 x 868 [6,4]
+ CRUSH rule 3 x 869 [8,4]
+ CRUSH rule 3 x 870 [0,5]
+ CRUSH rule 3 x 871 [3,1]
+ CRUSH rule 3 x 872 [5,0]
+ CRUSH rule 3 x 873 [4,6]
+ CRUSH rule 3 x 874 [2,6]
+ CRUSH rule 3 x 875 [2,6]
+ CRUSH rule 3 x 876 [5,8]
+ CRUSH rule 3 x 877 [6,4]
+ CRUSH rule 3 x 878 [5,0]
+ CRUSH rule 3 x 879 [7,3]
+ CRUSH rule 3 x 880 [3,1]
+ CRUSH rule 3 x 881 [5,8]
+ CRUSH rule 3 x 882 [4,1]
+ CRUSH rule 3 x 883 [2,5]
+ CRUSH rule 3 x 884 [6,0]
+ CRUSH rule 3 x 885 [5,1]
+ CRUSH rule 3 x 886 [3,7]
+ CRUSH rule 3 x 887 [7,5]
+ CRUSH rule 3 x 888 [6,1]
+ CRUSH rule 3 x 889 [2,8]
+ CRUSH rule 3 x 890 [7,1]
+ CRUSH rule 3 x 891 [1,7]
+ CRUSH rule 3 x 892 [6,0]
+ CRUSH rule 3 x 893 [2,3]
+ CRUSH rule 3 x 894 [7,4]
+ CRUSH rule 3 x 895 [5,1]
+ CRUSH rule 3 x 896 [1,6]
+ CRUSH rule 3 x 897 [4,0]
+ CRUSH rule 3 x 898 [0,3]
+ CRUSH rule 3 x 899 [1,6]
+ CRUSH rule 3 x 900 [4,0]
+ CRUSH rule 3 x 901 [5,2]
+ CRUSH rule 3 x 902 [8,4]
+ CRUSH rule 3 x 903 [5,6]
+ CRUSH rule 3 x 904 [5,7]
+ CRUSH rule 3 x 905 [6,0]
+ CRUSH rule 3 x 906 [1,7]
+ CRUSH rule 3 x 907 [7,2]
+ CRUSH rule 3 x 908 [5,6]
+ CRUSH rule 3 x 909 [2,3]
+ CRUSH rule 3 x 910 [6,5]
+ CRUSH rule 3 x 911 [5,7]
+ CRUSH rule 3 x 912 [0,8]
+ CRUSH rule 3 x 913 [7,1]
+ CRUSH rule 3 x 914 [6,4]
+ CRUSH rule 3 x 915 [8,2]
+ CRUSH rule 3 x 916 [3,0]
+ CRUSH rule 3 x 917 [1,3]
+ CRUSH rule 3 x 918 [8,0]
+ CRUSH rule 3 x 919 [6,2]
+ CRUSH rule 3 x 920 [7,4]
+ CRUSH rule 3 x 921 [1,4]
+ CRUSH rule 3 x 922 [6,3]
+ CRUSH rule 3 x 923 [5,6]
+ CRUSH rule 3 x 924 [3,1]
+ CRUSH rule 3 x 925 [5,6]
+ CRUSH rule 3 x 926 [3,0]
+ CRUSH rule 3 x 927 [1,6]
+ CRUSH rule 3 x 928 [8,1]
+ CRUSH rule 3 x 929 [4,2]
+ CRUSH rule 3 x 930 [2,5]
+ CRUSH rule 3 x 931 [5,2]
+ CRUSH rule 3 x 932 [4,2]
+ CRUSH rule 3 x 933 [8,4]
+ CRUSH rule 3 x 934 [5,8]
+ CRUSH rule 3 x 935 [6,3]
+ CRUSH rule 3 x 936 [0,7]
+ CRUSH rule 3 x 937 [5,8]
+ CRUSH rule 3 x 938 [6,4]
+ CRUSH rule 3 x 939 [2,8]
+ CRUSH rule 3 x 940 [8,5]
+ CRUSH rule 3 x 941 [5,0]
+ CRUSH rule 3 x 942 [1,6]
+ CRUSH rule 3 x 943 [8,2]
+ CRUSH rule 3 x 944 [4,8]
+ CRUSH rule 3 x 945 [7,0]
+ CRUSH rule 3 x 946 [2,8]
+ CRUSH rule 3 x 947 [4,2]
+ CRUSH rule 3 x 948 [7,4]
+ CRUSH rule 3 x 949 [6,2]
+ CRUSH rule 3 x 950 [3,7]
+ CRUSH rule 3 x 951 [4,6]
+ CRUSH rule 3 x 952 [2,7]
+ CRUSH rule 3 x 953 [1,3]
+ CRUSH rule 3 x 954 [4,0]
+ CRUSH rule 3 x 955 [8,1]
+ CRUSH rule 3 x 956 [1,7]
+ CRUSH rule 3 x 957 [7,0]
+ CRUSH rule 3 x 958 [8,3]
+ CRUSH rule 3 x 959 [5,1]
+ CRUSH rule 3 x 960 [3,6]
+ CRUSH rule 3 x 961 [4,1]
+ CRUSH rule 3 x 962 [7,5]
+ CRUSH rule 3 x 963 [0,5]
+ CRUSH rule 3 x 964 [3,2]
+ CRUSH rule 3 x 965 [7,3]
+ CRUSH rule 3 x 966 [3,6]
+ CRUSH rule 3 x 967 [8,4]
+ CRUSH rule 3 x 968 [7,0]
+ CRUSH rule 3 x 969 [8,0]
+ CRUSH rule 3 x 970 [0,8]
+ CRUSH rule 3 x 971 [1,8]
+ CRUSH rule 3 x 972 [1,7]
+ CRUSH rule 3 x 973 [1,8]
+ CRUSH rule 3 x 974 [5,1]
+ CRUSH rule 3 x 975 [3,8]
+ CRUSH rule 3 x 976 [4,7]
+ CRUSH rule 3 x 977 [8,3]
+ CRUSH rule 3 x 978 [7,0]
+ CRUSH rule 3 x 979 [7,2]
+ CRUSH rule 3 x 980 [6,2]
+ CRUSH rule 3 x 981 [7,5]
+ CRUSH rule 3 x 982 [4,1]
+ CRUSH rule 3 x 983 [3,6]
+ CRUSH rule 3 x 984 [0,8]
+ CRUSH rule 3 x 985 [2,4]
+ CRUSH rule 3 x 986 [8,4]
+ CRUSH rule 3 x 987 [0,4]
+ CRUSH rule 3 x 988 [1,4]
+ CRUSH rule 3 x 989 [0,8]
+ CRUSH rule 3 x 990 [1,6]
+ CRUSH rule 3 x 991 [0,4]
+ CRUSH rule 3 x 992 [7,0]
+ CRUSH rule 3 x 993 [0,6]
+ CRUSH rule 3 x 994 [3,6]
+ CRUSH rule 3 x 995 [7,0]
+ CRUSH rule 3 x 996 [6,5]
+ CRUSH rule 3 x 997 [6,3]
+ CRUSH rule 3 x 998 [8,0]
+ CRUSH rule 3 x 999 [0,7]
+ CRUSH rule 3 x 1000 [8,4]
+ CRUSH rule 3 x 1001 [2,3]
+ CRUSH rule 3 x 1002 [1,3]
+ CRUSH rule 3 x 1003 [2,7]
+ CRUSH rule 3 x 1004 [6,0]
+ CRUSH rule 3 x 1005 [6,1]
+ CRUSH rule 3 x 1006 [1,8]
+ CRUSH rule 3 x 1007 [1,3]
+ CRUSH rule 3 x 1008 [1,7]
+ CRUSH rule 3 x 1009 [6,5]
+ CRUSH rule 3 x 1010 [3,1]
+ CRUSH rule 3 x 1011 [3,0]
+ CRUSH rule 3 x 1012 [3,1]
+ CRUSH rule 3 x 1013 [5,2]
+ CRUSH rule 3 x 1014 [2,8]
+ CRUSH rule 3 x 1015 [6,3]
+ CRUSH rule 3 x 1016 [2,5]
+ CRUSH rule 3 x 1017 [6,1]
+ CRUSH rule 3 x 1018 [5,1]
+ CRUSH rule 3 x 1019 [5,8]
+ CRUSH rule 3 x 1020 [5,0]
+ CRUSH rule 3 x 1021 [5,2]
+ CRUSH rule 3 x 1022 [1,7]
+ CRUSH rule 3 x 1023 [3,0]
+ rule 3 (choose-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [0,3,7]
+ CRUSH rule 3 x 1 [0,8,5]
+ CRUSH rule 3 x 2 [1,4,8]
+ CRUSH rule 3 x 3 [8,0,4]
+ CRUSH rule 3 x 4 [5,1,8]
+ CRUSH rule 3 x 5 [7,0,3]
+ CRUSH rule 3 x 6 [2,6,3]
+ CRUSH rule 3 x 7 [5,6,1]
+ CRUSH rule 3 x 8 [5,7,2]
+ CRUSH rule 3 x 9 [2,4,8]
+ CRUSH rule 3 x 10 [0,8,4]
+ CRUSH rule 3 x 11 [0,6,3]
+ CRUSH rule 3 x 12 [0,3,7]
+ CRUSH rule 3 x 13 [3,8,0]
+ CRUSH rule 3 x 14 [7,1,5]
+ CRUSH rule 3 x 15 [7,1,3]
+ CRUSH rule 3 x 16 [3,7,1]
+ CRUSH rule 3 x 17 [5,1,7]
+ CRUSH rule 3 x 18 [1,3,6]
+ CRUSH rule 3 x 19 [7,5,2]
+ CRUSH rule 3 x 20 [2,4,7]
+ CRUSH rule 3 x 21 [3,6,0]
+ CRUSH rule 3 x 22 [8,5,1]
+ CRUSH rule 3 x 23 [3,7,2]
+ CRUSH rule 3 x 24 [1,6,3]
+ CRUSH rule 3 x 25 [3,8,1]
+ CRUSH rule 3 x 26 [2,7,3]
+ CRUSH rule 3 x 27 [3,1,7]
+ CRUSH rule 3 x 28 [6,2,4]
+ CRUSH rule 3 x 29 [8,5,2]
+ CRUSH rule 3 x 30 [5,6,0]
+ CRUSH rule 3 x 31 [8,2,5]
+ CRUSH rule 3 x 32 [3,7,2]
+ CRUSH rule 3 x 33 [2,7,4]
+ CRUSH rule 3 x 34 [2,3,7]
+ CRUSH rule 3 x 35 [0,6,3]
+ CRUSH rule 3 x 36 [3,6,1]
+ CRUSH rule 3 x 37 [0,4,7]
+ CRUSH rule 3 x 38 [4,6,1]
+ CRUSH rule 3 x 39 [3,7,2]
+ CRUSH rule 3 x 40 [7,2,5]
+ CRUSH rule 3 x 41 [0,7,4]
+ CRUSH rule 3 x 42 [4,7,2]
+ CRUSH rule 3 x 43 [0,3,8]
+ CRUSH rule 3 x 44 [1,8,4]
+ CRUSH rule 3 x 45 [8,2,3]
+ CRUSH rule 3 x 46 [2,4,8]
+ CRUSH rule 3 x 47 [4,1,7]
+ CRUSH rule 3 x 48 [4,6,0]
+ CRUSH rule 3 x 49 [5,6,0]
+ CRUSH rule 3 x 50 [3,1,7]
+ CRUSH rule 3 x 51 [3,6,1]
+ CRUSH rule 3 x 52 [8,2,3]
+ CRUSH rule 3 x 53 [3,6,2]
+ CRUSH rule 3 x 54 [7,4,0]
+ CRUSH rule 3 x 55 [8,0,3]
+ CRUSH rule 3 x 56 [6,5,1]
+ CRUSH rule 3 x 57 [5,8,0]
+ CRUSH rule 3 x 58 [1,8,5]
+ CRUSH rule 3 x 59 [4,0,7]
+ CRUSH rule 3 x 60 [3,1,8]
+ CRUSH rule 3 x 61 [4,8,0]
+ CRUSH rule 3 x 62 [7,0,4]
+ CRUSH rule 3 x 63 [5,6,2]
+ CRUSH rule 3 x 64 [4,1,8]
+ CRUSH rule 3 x 65 [7,4,2]
+ CRUSH rule 3 x 66 [5,6,0]
+ CRUSH rule 3 x 67 [5,0,6]
+ CRUSH rule 3 x 68 [0,3,8]
+ CRUSH rule 3 x 69 [5,2,7]
+ CRUSH rule 3 x 70 [7,2,3]
+ CRUSH rule 3 x 71 [2,7,4]
+ CRUSH rule 3 x 72 [6,2,4]
+ CRUSH rule 3 x 73 [2,7,3]
+ CRUSH rule 3 x 74 [0,8,3]
+ CRUSH rule 3 x 75 [3,0,7]
+ CRUSH rule 3 x 76 [5,2,6]
+ CRUSH rule 3 x 77 [7,0,5]
+ CRUSH rule 3 x 78 [1,4,7]
+ CRUSH rule 3 x 79 [5,0,7]
+ CRUSH rule 3 x 80 [0,4,6]
+ CRUSH rule 3 x 81 [0,5,8]
+ CRUSH rule 3 x 82 [7,2,5]
+ CRUSH rule 3 x 83 [2,6,4]
+ CRUSH rule 3 x 84 [7,0,3]
+ CRUSH rule 3 x 85 [3,7,0]
+ CRUSH rule 3 x 86 [0,6,4]
+ CRUSH rule 3 x 87 [0,6,5]
+ CRUSH rule 3 x 88 [1,8,4]
+ CRUSH rule 3 x 89 [3,1,8]
+ CRUSH rule 3 x 90 [6,3,2]
+ CRUSH rule 3 x 91 [3,6,2]
+ CRUSH rule 3 x 92 [1,6,3]
+ CRUSH rule 3 x 93 [7,4,0]
+ CRUSH rule 3 x 94 [0,3,7]
+ CRUSH rule 3 x 95 [7,3,1]
+ CRUSH rule 3 x 96 [3,8,2]
+ CRUSH rule 3 x 97 [8,4,1]
+ CRUSH rule 3 x 98 [2,7,4]
+ CRUSH rule 3 x 99 [0,8,5]
+ CRUSH rule 3 x 100 [1,6,3]
+ CRUSH rule 3 x 101 [3,8,1]
+ CRUSH rule 3 x 102 [4,0,7]
+ CRUSH rule 3 x 103 [4,7,0]
+ CRUSH rule 3 x 104 [7,4,0]
+ CRUSH rule 3 x 105 [2,3,8]
+ CRUSH rule 3 x 106 [1,6,5]
+ CRUSH rule 3 x 107 [3,1,7]
+ CRUSH rule 3 x 108 [7,1,3]
+ CRUSH rule 3 x 109 [1,3,8]
+ CRUSH rule 3 x 110 [3,2,6]
+ CRUSH rule 3 x 111 [2,4,7]
+ CRUSH rule 3 x 112 [2,6,3]
+ CRUSH rule 3 x 113 [6,1,4]
+ CRUSH rule 3 x 114 [7,5,0]
+ CRUSH rule 3 x 115 [8,2,3]
+ CRUSH rule 3 x 116 [1,7,4]
+ CRUSH rule 3 x 117 [7,5,1]
+ CRUSH rule 3 x 118 [0,3,6]
+ CRUSH rule 3 x 119 [5,7,1]
+ CRUSH rule 3 x 120 [0,5,6]
+ CRUSH rule 3 x 121 [2,8,4]
+ CRUSH rule 3 x 122 [8,4,2]
+ CRUSH rule 3 x 123 [2,4,7]
+ CRUSH rule 3 x 124 [3,0,8]
+ CRUSH rule 3 x 125 [0,7,3]
+ CRUSH rule 3 x 126 [4,1,7]
+ CRUSH rule 3 x 127 [3,6,1]
+ CRUSH rule 3 x 128 [3,8,1]
+ CRUSH rule 3 x 129 [0,4,7]
+ CRUSH rule 3 x 130 [3,8,1]
+ CRUSH rule 3 x 131 [1,5,6]
+ CRUSH rule 3 x 132 [1,3,8]
+ CRUSH rule 3 x 133 [3,6,1]
+ CRUSH rule 3 x 134 [1,7,4]
+ CRUSH rule 3 x 135 [5,7,0]
+ CRUSH rule 3 x 136 [2,3,8]
+ CRUSH rule 3 x 137 [7,3,1]
+ CRUSH rule 3 x 138 [8,3,0]
+ CRUSH rule 3 x 139 [3,0,8]
+ CRUSH rule 3 x 140 [1,8,3]
+ CRUSH rule 3 x 141 [6,0,4]
+ CRUSH rule 3 x 142 [3,2,8]
+ CRUSH rule 3 x 143 [5,7,0]
+ CRUSH rule 3 x 144 [8,2,5]
+ CRUSH rule 3 x 145 [8,4,2]
+ CRUSH rule 3 x 146 [2,8,4]
+ CRUSH rule 3 x 147 [2,6,4]
+ CRUSH rule 3 x 148 [3,0,8]
+ CRUSH rule 3 x 149 [4,6,2]
+ CRUSH rule 3 x 150 [1,8,4]
+ CRUSH rule 3 x 151 [3,8,1]
+ CRUSH rule 3 x 152 [8,3,0]
+ CRUSH rule 3 x 153 [8,3,0]
+ CRUSH rule 3 x 154 [3,0,6]
+ CRUSH rule 3 x 155 [3,6,0]
+ CRUSH rule 3 x 156 [4,2,7]
+ CRUSH rule 3 x 157 [4,1,7]
+ CRUSH rule 3 x 158 [2,6,3]
+ CRUSH rule 3 x 159 [7,0,3]
+ CRUSH rule 3 x 160 [2,8,5]
+ CRUSH rule 3 x 161 [1,3,6]
+ CRUSH rule 3 x 162 [0,8,5]
+ CRUSH rule 3 x 163 [5,8,2]
+ CRUSH rule 3 x 164 [7,2,4]
+ CRUSH rule 3 x 165 [7,2,4]
+ CRUSH rule 3 x 166 [2,3,7]
+ CRUSH rule 3 x 167 [0,6,3]
+ CRUSH rule 3 x 168 [4,0,7]
+ CRUSH rule 3 x 169 [2,7,3]
+ CRUSH rule 3 x 170 [1,3,8]
+ CRUSH rule 3 x 171 [7,3,2]
+ CRUSH rule 3 x 172 [0,8,4]
+ CRUSH rule 3 x 173 [8,5,1]
+ CRUSH rule 3 x 174 [1,4,8]
+ CRUSH rule 3 x 175 [6,0,3]
+ CRUSH rule 3 x 176 [4,2,7]
+ CRUSH rule 3 x 177 [5,1,8]
+ CRUSH rule 3 x 178 [3,0,6]
+ CRUSH rule 3 x 179 [4,1,7]
+ CRUSH rule 3 x 180 [3,7,2]
+ CRUSH rule 3 x 181 [6,0,5]
+ CRUSH rule 3 x 182 [8,3,1]
+ CRUSH rule 3 x 183 [7,5,1]
+ CRUSH rule 3 x 184 [5,6,1]
+ CRUSH rule 3 x 185 [6,0,3]
+ CRUSH rule 3 x 186 [2,4,8]
+ CRUSH rule 3 x 187 [1,6,3]
+ CRUSH rule 3 x 188 [1,6,3]
+ CRUSH rule 3 x 189 [0,6,5]
+ CRUSH rule 3 x 190 [4,1,8]
+ CRUSH rule 3 x 191 [7,0,5]
+ CRUSH rule 3 x 192 [5,2,6]
+ CRUSH rule 3 x 193 [4,0,6]
+ CRUSH rule 3 x 194 [1,3,7]
+ CRUSH rule 3 x 195 [6,5,2]
+ CRUSH rule 3 x 196 [6,1,3]
+ CRUSH rule 3 x 197 [6,5,1]
+ CRUSH rule 3 x 198 [2,3,6]
+ CRUSH rule 3 x 199 [0,5,7]
+ CRUSH rule 3 x 200 [0,3,8]
+ CRUSH rule 3 x 201 [7,1,5]
+ CRUSH rule 3 x 202 [6,4,1]
+ CRUSH rule 3 x 203 [4,6,1]
+ CRUSH rule 3 x 204 [2,4,8]
+ CRUSH rule 3 x 205 [0,7,4]
+ CRUSH rule 3 x 206 [0,8,4]
+ CRUSH rule 3 x 207 [3,2,7]
+ CRUSH rule 3 x 208 [7,2,4]
+ CRUSH rule 3 x 209 [1,8,3]
+ CRUSH rule 3 x 210 [1,4,6]
+ CRUSH rule 3 x 211 [5,2,7]
+ CRUSH rule 3 x 212 [7,5,0]
+ CRUSH rule 3 x 213 [8,4,0]
+ CRUSH rule 3 x 214 [4,7,1]
+ CRUSH rule 3 x 215 [8,0,5]
+ CRUSH rule 3 x 216 [5,2,8]
+ CRUSH rule 3 x 217 [0,7,5]
+ CRUSH rule 3 x 218 [0,7,4]
+ CRUSH rule 3 x 219 [4,6,0]
+ CRUSH rule 3 x 220 [5,8,0]
+ CRUSH rule 3 x 221 [3,7,0]
+ CRUSH rule 3 x 222 [6,4,1]
+ CRUSH rule 3 x 223 [1,3,6]
+ CRUSH rule 3 x 224 [1,3,8]
+ CRUSH rule 3 x 225 [8,0,3]
+ CRUSH rule 3 x 226 [7,0,4]
+ CRUSH rule 3 x 227 [3,0,7]
+ CRUSH rule 3 x 228 [5,6,1]
+ CRUSH rule 3 x 229 [3,7,0]
+ CRUSH rule 3 x 230 [4,6,1]
+ CRUSH rule 3 x 231 [4,7,1]
+ CRUSH rule 3 x 232 [2,8,4]
+ CRUSH rule 3 x 233 [3,6,0]
+ CRUSH rule 3 x 234 [0,4,6]
+ CRUSH rule 3 x 235 [3,6,1]
+ CRUSH rule 3 x 236 [5,0,8]
+ CRUSH rule 3 x 237 [4,8,0]
+ CRUSH rule 3 x 238 [4,2,6]
+ CRUSH rule 3 x 239 [8,3,2]
+ CRUSH rule 3 x 240 [5,8,2]
+ CRUSH rule 3 x 241 [3,1,7]
+ CRUSH rule 3 x 242 [3,2,6]
+ CRUSH rule 3 x 243 [4,8,2]
+ CRUSH rule 3 x 244 [4,6,0]
+ CRUSH rule 3 x 245 [7,2,3]
+ CRUSH rule 3 x 246 [1,3,7]
+ CRUSH rule 3 x 247 [6,1,4]
+ CRUSH rule 3 x 248 [8,0,3]
+ CRUSH rule 3 x 249 [2,5,8]
+ CRUSH rule 3 x 250 [2,3,8]
+ CRUSH rule 3 x 251 [2,3,7]
+ CRUSH rule 3 x 252 [3,7,2]
+ CRUSH rule 3 x 253 [3,0,7]
+ CRUSH rule 3 x 254 [3,2,8]
+ CRUSH rule 3 x 255 [1,7,4]
+ CRUSH rule 3 x 256 [5,6,1]
+ CRUSH rule 3 x 257 [2,6,4]
+ CRUSH rule 3 x 258 [5,2,8]
+ CRUSH rule 3 x 259 [4,6,1]
+ CRUSH rule 3 x 260 [3,8,1]
+ CRUSH rule 3 x 261 [8,5,2]
+ CRUSH rule 3 x 262 [5,6,2]
+ CRUSH rule 3 x 263 [6,1,5]
+ CRUSH rule 3 x 264 [3,6,1]
+ CRUSH rule 3 x 265 [8,4,0]
+ CRUSH rule 3 x 266 [8,0,4]
+ CRUSH rule 3 x 267 [2,4,6]
+ CRUSH rule 3 x 268 [0,6,4]
+ CRUSH rule 3 x 269 [0,6,5]
+ CRUSH rule 3 x 270 [5,1,8]
+ CRUSH rule 3 x 271 [7,4,2]
+ CRUSH rule 3 x 272 [2,6,5]
+ CRUSH rule 3 x 273 [3,2,7]
+ CRUSH rule 3 x 274 [6,3,2]
+ CRUSH rule 3 x 275 [4,8,1]
+ CRUSH rule 3 x 276 [7,0,5]
+ CRUSH rule 3 x 277 [6,4,1]
+ CRUSH rule 3 x 278 [6,2,3]
+ CRUSH rule 3 x 279 [8,5,0]
+ CRUSH rule 3 x 280 [0,7,4]
+ CRUSH rule 3 x 281 [8,1,5]
+ CRUSH rule 3 x 282 [3,2,8]
+ CRUSH rule 3 x 283 [8,1,3]
+ CRUSH rule 3 x 284 [6,3,1]
+ CRUSH rule 3 x 285 [5,7,2]
+ CRUSH rule 3 x 286 [2,8,4]
+ CRUSH rule 3 x 287 [0,3,8]
+ CRUSH rule 3 x 288 [8,1,4]
+ CRUSH rule 3 x 289 [4,6,0]
+ CRUSH rule 3 x 290 [1,5,6]
+ CRUSH rule 3 x 291 [0,5,8]
+ CRUSH rule 3 x 292 [8,1,4]
+ CRUSH rule 3 x 293 [6,2,3]
+ CRUSH rule 3 x 294 [7,5,2]
+ CRUSH rule 3 x 295 [4,6,0]
+ CRUSH rule 3 x 296 [3,1,6]
+ CRUSH rule 3 x 297 [6,0,5]
+ CRUSH rule 3 x 298 [1,5,6]
+ CRUSH rule 3 x 299 [2,8,4]
+ CRUSH rule 3 x 300 [8,4,1]
+ CRUSH rule 3 x 301 [0,7,3]
+ CRUSH rule 3 x 302 [3,1,6]
+ CRUSH rule 3 x 303 [7,4,1]
+ CRUSH rule 3 x 304 [2,6,4]
+ CRUSH rule 3 x 305 [5,6,1]
+ CRUSH rule 3 x 306 [0,8,3]
+ CRUSH rule 3 x 307 [0,7,4]
+ CRUSH rule 3 x 308 [0,8,5]
+ CRUSH rule 3 x 309 [7,4,2]
+ CRUSH rule 3 x 310 [4,0,6]
+ CRUSH rule 3 x 311 [3,8,2]
+ CRUSH rule 3 x 312 [2,6,5]
+ CRUSH rule 3 x 313 [5,1,7]
+ CRUSH rule 3 x 314 [4,0,6]
+ CRUSH rule 3 x 315 [2,3,7]
+ CRUSH rule 3 x 316 [6,4,1]
+ CRUSH rule 3 x 317 [2,7,4]
+ CRUSH rule 3 x 318 [8,1,4]
+ CRUSH rule 3 x 319 [5,2,6]
+ CRUSH rule 3 x 320 [3,8,0]
+ CRUSH rule 3 x 321 [1,5,8]
+ CRUSH rule 3 x 322 [2,6,3]
+ CRUSH rule 3 x 323 [4,7,0]
+ CRUSH rule 3 x 324 [7,2,5]
+ CRUSH rule 3 x 325 [4,6,0]
+ CRUSH rule 3 x 326 [3,1,7]
+ CRUSH rule 3 x 327 [0,8,3]
+ CRUSH rule 3 x 328 [7,3,1]
+ CRUSH rule 3 x 329 [5,7,0]
+ CRUSH rule 3 x 330 [3,7,0]
+ CRUSH rule 3 x 331 [2,7,3]
+ CRUSH rule 3 x 332 [2,3,8]
+ CRUSH rule 3 x 333 [6,4,1]
+ CRUSH rule 3 x 334 [8,3,2]
+ CRUSH rule 3 x 335 [7,2,3]
+ CRUSH rule 3 x 336 [4,6,2]
+ CRUSH rule 3 x 337 [7,0,4]
+ CRUSH rule 3 x 338 [5,8,2]
+ CRUSH rule 3 x 339 [7,5,0]
+ CRUSH rule 3 x 340 [2,6,5]
+ CRUSH rule 3 x 341 [5,1,7]
+ CRUSH rule 3 x 342 [0,8,5]
+ CRUSH rule 3 x 343 [6,5,0]
+ CRUSH rule 3 x 344 [6,1,4]
+ CRUSH rule 3 x 345 [4,7,0]
+ CRUSH rule 3 x 346 [8,0,3]
+ CRUSH rule 3 x 347 [3,0,8]
+ CRUSH rule 3 x 348 [8,0,3]
+ CRUSH rule 3 x 349 [1,7,3]
+ CRUSH rule 3 x 350 [8,5,1]
+ CRUSH rule 3 x 351 [3,8,0]
+ CRUSH rule 3 x 352 [1,8,4]
+ CRUSH rule 3 x 353 [6,5,1]
+ CRUSH rule 3 x 354 [0,5,6]
+ CRUSH rule 3 x 355 [3,8,0]
+ CRUSH rule 3 x 356 [3,1,8]
+ CRUSH rule 3 x 357 [6,1,3]
+ CRUSH rule 3 x 358 [2,8,5]
+ CRUSH rule 3 x 359 [6,0,5]
+ CRUSH rule 3 x 360 [5,0,8]
+ CRUSH rule 3 x 361 [8,5,0]
+ CRUSH rule 3 x 362 [4,0,8]
+ CRUSH rule 3 x 363 [4,2,8]
+ CRUSH rule 3 x 364 [2,5,7]
+ CRUSH rule 3 x 365 [6,4,2]
+ CRUSH rule 3 x 366 [7,0,3]
+ CRUSH rule 3 x 367 [4,2,7]
+ CRUSH rule 3 x 368 [7,3,1]
+ CRUSH rule 3 x 369 [3,7,2]
+ CRUSH rule 3 x 370 [8,2,4]
+ CRUSH rule 3 x 371 [1,5,8]
+ CRUSH rule 3 x 372 [3,1,8]
+ CRUSH rule 3 x 373 [0,6,4]
+ CRUSH rule 3 x 374 [3,8,1]
+ CRUSH rule 3 x 375 [6,5,2]
+ CRUSH rule 3 x 376 [7,1,3]
+ CRUSH rule 3 x 377 [1,4,7]
+ CRUSH rule 3 x 378 [0,6,4]
+ CRUSH rule 3 x 379 [8,3,0]
+ CRUSH rule 3 x 380 [2,3,8]
+ CRUSH rule 3 x 381 [0,3,7]
+ CRUSH rule 3 x 382 [1,3,7]
+ CRUSH rule 3 x 383 [4,7,1]
+ CRUSH rule 3 x 384 [7,0,4]
+ CRUSH rule 3 x 385 [7,4,0]
+ CRUSH rule 3 x 386 [0,4,6]
+ CRUSH rule 3 x 387 [1,5,8]
+ CRUSH rule 3 x 388 [5,2,7]
+ CRUSH rule 3 x 389 [1,4,8]
+ CRUSH rule 3 x 390 [5,8,1]
+ CRUSH rule 3 x 391 [5,6,0]
+ CRUSH rule 3 x 392 [1,7,5]
+ CRUSH rule 3 x 393 [4,0,6]
+ CRUSH rule 3 x 394 [4,8,2]
+ CRUSH rule 3 x 395 [4,0,8]
+ CRUSH rule 3 x 396 [4,2,6]
+ CRUSH rule 3 x 397 [2,4,7]
+ CRUSH rule 3 x 398 [2,3,6]
+ CRUSH rule 3 x 399 [8,5,2]
+ CRUSH rule 3 x 400 [8,1,3]
+ CRUSH rule 3 x 401 [0,3,6]
+ CRUSH rule 3 x 402 [7,4,2]
+ CRUSH rule 3 x 403 [0,4,7]
+ CRUSH rule 3 x 404 [4,0,6]
+ CRUSH rule 3 x 405 [6,4,0]
+ CRUSH rule 3 x 406 [2,6,4]
+ CRUSH rule 3 x 407 [2,7,5]
+ CRUSH rule 3 x 408 [4,1,7]
+ CRUSH rule 3 x 409 [7,5,0]
+ CRUSH rule 3 x 410 [8,3,1]
+ CRUSH rule 3 x 411 [2,7,4]
+ CRUSH rule 3 x 412 [0,3,7]
+ CRUSH rule 3 x 413 [5,2,8]
+ CRUSH rule 3 x 414 [4,0,8]
+ CRUSH rule 3 x 415 [0,6,3]
+ CRUSH rule 3 x 416 [2,8,5]
+ CRUSH rule 3 x 417 [8,0,3]
+ CRUSH rule 3 x 418 [7,1,3]
+ CRUSH rule 3 x 419 [8,5,2]
+ CRUSH rule 3 x 420 [1,4,7]
+ CRUSH rule 3 x 421 [8,3,0]
+ CRUSH rule 3 x 422 [6,3,1]
+ CRUSH rule 3 x 423 [0,3,7]
+ CRUSH rule 3 x 424 [8,5,1]
+ CRUSH rule 3 x 425 [1,4,8]
+ CRUSH rule 3 x 426 [6,2,4]
+ CRUSH rule 3 x 427 [0,8,3]
+ CRUSH rule 3 x 428 [5,8,1]
+ CRUSH rule 3 x 429 [4,8,0]
+ CRUSH rule 3 x 430 [3,7,0]
+ CRUSH rule 3 x 431 [5,0,7]
+ CRUSH rule 3 x 432 [7,0,4]
+ CRUSH rule 3 x 433 [6,4,0]
+ CRUSH rule 3 x 434 [5,0,7]
+ CRUSH rule 3 x 435 [0,4,6]
+ CRUSH rule 3 x 436 [4,0,7]
+ CRUSH rule 3 x 437 [7,3,1]
+ CRUSH rule 3 x 438 [0,5,8]
+ CRUSH rule 3 x 439 [1,3,8]
+ CRUSH rule 3 x 440 [2,6,5]
+ CRUSH rule 3 x 441 [5,8,0]
+ CRUSH rule 3 x 442 [2,3,6]
+ CRUSH rule 3 x 443 [6,0,3]
+ CRUSH rule 3 x 444 [7,1,4]
+ CRUSH rule 3 x 445 [6,4,0]
+ CRUSH rule 3 x 446 [4,0,8]
+ CRUSH rule 3 x 447 [2,4,6]
+ CRUSH rule 3 x 448 [7,0,5]
+ CRUSH rule 3 x 449 [7,4,2]
+ CRUSH rule 3 x 450 [4,0,6]
+ CRUSH rule 3 x 451 [6,5,1]
+ CRUSH rule 3 x 452 [8,4,0]
+ CRUSH rule 3 x 453 [6,5,0]
+ CRUSH rule 3 x 454 [6,5,0]
+ CRUSH rule 3 x 455 [2,8,4]
+ CRUSH rule 3 x 456 [6,2,5]
+ CRUSH rule 3 x 457 [7,1,5]
+ CRUSH rule 3 x 458 [2,8,5]
+ CRUSH rule 3 x 459 [2,6,5]
+ CRUSH rule 3 x 460 [6,5,2]
+ CRUSH rule 3 x 461 [6,4,2]
+ CRUSH rule 3 x 462 [8,0,4]
+ CRUSH rule 3 x 463 [6,0,4]
+ CRUSH rule 3 x 464 [7,4,1]
+ CRUSH rule 3 x 465 [7,1,5]
+ CRUSH rule 3 x 466 [5,6,2]
+ CRUSH rule 3 x 467 [6,5,1]
+ CRUSH rule 3 x 468 [7,2,3]
+ CRUSH rule 3 x 469 [7,2,4]
+ CRUSH rule 3 x 470 [3,0,8]
+ CRUSH rule 3 x 471 [0,6,4]
+ CRUSH rule 3 x 472 [5,0,8]
+ CRUSH rule 3 x 473 [1,4,7]
+ CRUSH rule 3 x 474 [6,1,4]
+ CRUSH rule 3 x 475 [6,2,3]
+ CRUSH rule 3 x 476 [4,7,1]
+ CRUSH rule 3 x 477 [5,6,1]
+ CRUSH rule 3 x 478 [6,1,3]
+ CRUSH rule 3 x 479 [0,3,6]
+ CRUSH rule 3 x 480 [1,6,4]
+ CRUSH rule 3 x 481 [2,5,7]
+ CRUSH rule 3 x 482 [4,8,0]
+ CRUSH rule 3 x 483 [0,6,4]
+ CRUSH rule 3 x 484 [1,8,5]
+ CRUSH rule 3 x 485 [4,8,1]
+ CRUSH rule 3 x 486 [4,0,8]
+ CRUSH rule 3 x 487 [5,0,8]
+ CRUSH rule 3 x 488 [5,7,1]
+ CRUSH rule 3 x 489 [2,8,4]
+ CRUSH rule 3 x 490 [6,4,0]
+ CRUSH rule 3 x 491 [1,6,5]
+ CRUSH rule 3 x 492 [6,5,2]
+ CRUSH rule 3 x 493 [0,8,3]
+ CRUSH rule 3 x 494 [1,6,4]
+ CRUSH rule 3 x 495 [3,0,6]
+ CRUSH rule 3 x 496 [7,5,1]
+ CRUSH rule 3 x 497 [5,7,2]
+ CRUSH rule 3 x 498 [0,4,6]
+ CRUSH rule 3 x 499 [8,5,2]
+ CRUSH rule 3 x 500 [3,6,1]
+ CRUSH rule 3 x 501 [0,7,5]
+ CRUSH rule 3 x 502 [7,1,3]
+ CRUSH rule 3 x 503 [2,3,7]
+ CRUSH rule 3 x 504 [5,8,2]
+ CRUSH rule 3 x 505 [0,7,5]
+ CRUSH rule 3 x 506 [5,1,7]
+ CRUSH rule 3 x 507 [6,0,3]
+ CRUSH rule 3 x 508 [0,4,7]
+ CRUSH rule 3 x 509 [7,4,0]
+ CRUSH rule 3 x 510 [6,2,5]
+ CRUSH rule 3 x 511 [5,6,2]
+ CRUSH rule 3 x 512 [7,2,4]
+ CRUSH rule 3 x 513 [7,2,4]
+ CRUSH rule 3 x 514 [4,7,0]
+ CRUSH rule 3 x 515 [8,3,0]
+ CRUSH rule 3 x 516 [4,1,7]
+ CRUSH rule 3 x 517 [7,0,4]
+ CRUSH rule 3 x 518 [4,6,0]
+ CRUSH rule 3 x 519 [7,3,1]
+ CRUSH rule 3 x 520 [2,8,5]
+ CRUSH rule 3 x 521 [8,0,3]
+ CRUSH rule 3 x 522 [6,0,4]
+ CRUSH rule 3 x 523 [4,1,7]
+ CRUSH rule 3 x 524 [0,4,7]
+ CRUSH rule 3 x 525 [0,3,8]
+ CRUSH rule 3 x 526 [1,3,8]
+ CRUSH rule 3 x 527 [0,4,6]
+ CRUSH rule 3 x 528 [5,2,7]
+ CRUSH rule 3 x 529 [5,6,1]
+ CRUSH rule 3 x 530 [6,4,1]
+ CRUSH rule 3 x 531 [6,0,3]
+ CRUSH rule 3 x 532 [6,5,1]
+ CRUSH rule 3 x 533 [5,8,0]
+ CRUSH rule 3 x 534 [7,4,2]
+ CRUSH rule 3 x 535 [8,0,3]
+ CRUSH rule 3 x 536 [6,2,3]
+ CRUSH rule 3 x 537 [3,8,0]
+ CRUSH rule 3 x 538 [6,4,2]
+ CRUSH rule 3 x 539 [8,4,2]
+ CRUSH rule 3 x 540 [0,7,4]
+ CRUSH rule 3 x 541 [2,5,7]
+ CRUSH rule 3 x 542 [3,0,8]
+ CRUSH rule 3 x 543 [6,2,4]
+ CRUSH rule 3 x 544 [3,7,1]
+ CRUSH rule 3 x 545 [5,7,1]
+ CRUSH rule 3 x 546 [6,2,3]
+ CRUSH rule 3 x 547 [8,1,5]
+ CRUSH rule 3 x 548 [5,1,8]
+ CRUSH rule 3 x 549 [5,7,1]
+ CRUSH rule 3 x 550 [0,5,6]
+ CRUSH rule 3 x 551 [7,4,0]
+ CRUSH rule 3 x 552 [5,7,2]
+ CRUSH rule 3 x 553 [4,2,8]
+ CRUSH rule 3 x 554 [0,6,5]
+ CRUSH rule 3 x 555 [5,0,8]
+ CRUSH rule 3 x 556 [3,6,1]
+ CRUSH rule 3 x 557 [7,4,0]
+ CRUSH rule 3 x 558 [3,2,6]
+ CRUSH rule 3 x 559 [4,1,8]
+ CRUSH rule 3 x 560 [8,4,2]
+ CRUSH rule 3 x 561 [6,4,1]
+ CRUSH rule 3 x 562 [3,0,8]
+ CRUSH rule 3 x 563 [2,7,4]
+ CRUSH rule 3 x 564 [5,1,7]
+ CRUSH rule 3 x 565 [3,8,0]
+ CRUSH rule 3 x 566 [4,6,0]
+ CRUSH rule 3 x 567 [3,7,0]
+ CRUSH rule 3 x 568 [7,4,0]
+ CRUSH rule 3 x 569 [3,1,6]
+ CRUSH rule 3 x 570 [1,4,7]
+ CRUSH rule 3 x 571 [3,6,0]
+ CRUSH rule 3 x 572 [3,0,7]
+ CRUSH rule 3 x 573 [3,1,7]
+ CRUSH rule 3 x 574 [2,3,6]
+ CRUSH rule 3 x 575 [8,1,5]
+ CRUSH rule 3 x 576 [4,7,1]
+ CRUSH rule 3 x 577 [8,2,3]
+ CRUSH rule 3 x 578 [6,0,4]
+ CRUSH rule 3 x 579 [3,2,6]
+ CRUSH rule 3 x 580 [3,0,8]
+ CRUSH rule 3 x 581 [7,1,5]
+ CRUSH rule 3 x 582 [2,8,5]
+ CRUSH rule 3 x 583 [6,2,3]
+ CRUSH rule 3 x 584 [8,0,5]
+ CRUSH rule 3 x 585 [7,0,5]
+ CRUSH rule 3 x 586 [0,8,5]
+ CRUSH rule 3 x 587 [2,5,6]
+ CRUSH rule 3 x 588 [3,8,0]
+ CRUSH rule 3 x 589 [7,1,5]
+ CRUSH rule 3 x 590 [6,2,5]
+ CRUSH rule 3 x 591 [5,2,7]
+ CRUSH rule 3 x 592 [2,4,6]
+ CRUSH rule 3 x 593 [0,8,4]
+ CRUSH rule 3 x 594 [0,6,3]
+ CRUSH rule 3 x 595 [7,1,5]
+ CRUSH rule 3 x 596 [4,0,6]
+ CRUSH rule 3 x 597 [3,1,7]
+ CRUSH rule 3 x 598 [3,0,7]
+ CRUSH rule 3 x 599 [5,1,6]
+ CRUSH rule 3 x 600 [7,0,4]
+ CRUSH rule 3 x 601 [0,6,3]
+ CRUSH rule 3 x 602 [3,8,2]
+ CRUSH rule 3 x 603 [5,0,8]
+ CRUSH rule 3 x 604 [7,4,0]
+ CRUSH rule 3 x 605 [3,2,6]
+ CRUSH rule 3 x 606 [2,7,3]
+ CRUSH rule 3 x 607 [0,5,8]
+ CRUSH rule 3 x 608 [5,1,8]
+ CRUSH rule 3 x 609 [5,0,6]
+ CRUSH rule 3 x 610 [3,8,2]
+ CRUSH rule 3 x 611 [1,8,3]
+ CRUSH rule 3 x 612 [2,8,3]
+ CRUSH rule 3 x 613 [7,1,3]
+ CRUSH rule 3 x 614 [7,2,3]
+ CRUSH rule 3 x 615 [6,2,3]
+ CRUSH rule 3 x 616 [0,7,5]
+ CRUSH rule 3 x 617 [6,2,3]
+ CRUSH rule 3 x 618 [7,3,1]
+ CRUSH rule 3 x 619 [5,0,6]
+ CRUSH rule 3 x 620 [4,1,8]
+ CRUSH rule 3 x 621 [5,6,0]
+ CRUSH rule 3 x 622 [0,3,8]
+ CRUSH rule 3 x 623 [0,8,4]
+ CRUSH rule 3 x 624 [3,2,8]
+ CRUSH rule 3 x 625 [2,5,8]
+ CRUSH rule 3 x 626 [7,2,3]
+ CRUSH rule 3 x 627 [2,6,5]
+ CRUSH rule 3 x 628 [8,1,3]
+ CRUSH rule 3 x 629 [2,6,3]
+ CRUSH rule 3 x 630 [2,6,4]
+ CRUSH rule 3 x 631 [0,6,5]
+ CRUSH rule 3 x 632 [7,0,4]
+ CRUSH rule 3 x 633 [8,4,2]
+ CRUSH rule 3 x 634 [0,5,6]
+ CRUSH rule 3 x 635 [5,6,2]
+ CRUSH rule 3 x 636 [1,3,7]
+ CRUSH rule 3 x 637 [4,0,8]
+ CRUSH rule 3 x 638 [6,2,5]
+ CRUSH rule 3 x 639 [4,0,6]
+ CRUSH rule 3 x 640 [3,2,7]
+ CRUSH rule 3 x 641 [7,2,3]
+ CRUSH rule 3 x 642 [2,8,4]
+ CRUSH rule 3 x 643 [3,0,8]
+ CRUSH rule 3 x 644 [8,0,4]
+ CRUSH rule 3 x 645 [5,7,2]
+ CRUSH rule 3 x 646 [8,0,3]
+ CRUSH rule 3 x 647 [7,0,5]
+ CRUSH rule 3 x 648 [0,8,3]
+ CRUSH rule 3 x 649 [4,7,1]
+ CRUSH rule 3 x 650 [7,5,1]
+ CRUSH rule 3 x 651 [3,6,1]
+ CRUSH rule 3 x 652 [3,6,1]
+ CRUSH rule 3 x 653 [8,3,0]
+ CRUSH rule 3 x 654 [7,4,0]
+ CRUSH rule 3 x 655 [0,5,6]
+ CRUSH rule 3 x 656 [4,7,0]
+ CRUSH rule 3 x 657 [6,0,5]
+ CRUSH rule 3 x 658 [5,8,0]
+ CRUSH rule 3 x 659 [4,7,0]
+ CRUSH rule 3 x 660 [7,3,0]
+ CRUSH rule 3 x 661 [1,7,3]
+ CRUSH rule 3 x 662 [4,2,8]
+ CRUSH rule 3 x 663 [1,3,7]
+ CRUSH rule 3 x 664 [1,3,6]
+ CRUSH rule 3 x 665 [5,6,1]
+ CRUSH rule 3 x 666 [2,7,4]
+ CRUSH rule 3 x 667 [1,3,8]
+ CRUSH rule 3 x 668 [3,7,0]
+ CRUSH rule 3 x 669 [6,3,0]
+ CRUSH rule 3 x 670 [4,1,6]
+ CRUSH rule 3 x 671 [0,8,5]
+ CRUSH rule 3 x 672 [4,2,8]
+ CRUSH rule 3 x 673 [5,0,7]
+ CRUSH rule 3 x 674 [3,0,7]
+ CRUSH rule 3 x 675 [0,8,5]
+ CRUSH rule 3 x 676 [0,3,7]
+ CRUSH rule 3 x 677 [4,1,6]
+ CRUSH rule 3 x 678 [2,3,8]
+ CRUSH rule 3 x 679 [6,0,5]
+ CRUSH rule 3 x 680 [0,3,8]
+ CRUSH rule 3 x 681 [4,6,1]
+ CRUSH rule 3 x 682 [0,4,8]
+ CRUSH rule 3 x 683 [0,3,8]
+ CRUSH rule 3 x 684 [7,2,5]
+ CRUSH rule 3 x 685 [7,2,5]
+ CRUSH rule 3 x 686 [1,5,8]
+ CRUSH rule 3 x 687 [3,7,2]
+ CRUSH rule 3 x 688 [5,6,2]
+ CRUSH rule 3 x 689 [6,5,1]
+ CRUSH rule 3 x 690 [8,0,4]
+ CRUSH rule 3 x 691 [3,1,6]
+ CRUSH rule 3 x 692 [7,1,4]
+ CRUSH rule 3 x 693 [6,5,2]
+ CRUSH rule 3 x 694 [6,4,2]
+ CRUSH rule 3 x 695 [0,6,3]
+ CRUSH rule 3 x 696 [1,5,8]
+ CRUSH rule 3 x 697 [6,0,4]
+ CRUSH rule 3 x 698 [6,0,3]
+ CRUSH rule 3 x 699 [1,8,3]
+ CRUSH rule 3 x 700 [0,4,6]
+ CRUSH rule 3 x 701 [4,0,6]
+ CRUSH rule 3 x 702 [3,0,8]
+ CRUSH rule 3 x 703 [8,4,0]
+ CRUSH rule 3 x 704 [0,4,6]
+ CRUSH rule 3 x 705 [8,0,4]
+ CRUSH rule 3 x 706 [1,5,8]
+ CRUSH rule 3 x 707 [7,3,0]
+ CRUSH rule 3 x 708 [3,7,1]
+ CRUSH rule 3 x 709 [6,5,1]
+ CRUSH rule 3 x 710 [8,5,1]
+ CRUSH rule 3 x 711 [2,4,7]
+ CRUSH rule 3 x 712 [2,3,6]
+ CRUSH rule 3 x 713 [6,3,1]
+ CRUSH rule 3 x 714 [3,0,6]
+ CRUSH rule 3 x 715 [1,3,6]
+ CRUSH rule 3 x 716 [3,6,1]
+ CRUSH rule 3 x 717 [8,0,5]
+ CRUSH rule 3 x 718 [3,7,0]
+ CRUSH rule 3 x 719 [2,6,3]
+ CRUSH rule 3 x 720 [6,0,4]
+ CRUSH rule 3 x 721 [5,7,0]
+ CRUSH rule 3 x 722 [5,7,1]
+ CRUSH rule 3 x 723 [5,2,7]
+ CRUSH rule 3 x 724 [0,7,3]
+ CRUSH rule 3 x 725 [0,4,6]
+ CRUSH rule 3 x 726 [3,7,2]
+ CRUSH rule 3 x 727 [4,7,2]
+ CRUSH rule 3 x 728 [2,6,3]
+ CRUSH rule 3 x 729 [5,6,1]
+ CRUSH rule 3 x 730 [3,8,2]
+ CRUSH rule 3 x 731 [4,1,6]
+ CRUSH rule 3 x 732 [1,4,8]
+ CRUSH rule 3 x 733 [5,6,2]
+ CRUSH rule 3 x 734 [6,5,0]
+ CRUSH rule 3 x 735 [4,6,1]
+ CRUSH rule 3 x 736 [3,8,1]
+ CRUSH rule 3 x 737 [1,8,3]
+ CRUSH rule 3 x 738 [5,1,6]
+ CRUSH rule 3 x 739 [0,7,4]
+ CRUSH rule 3 x 740 [0,7,3]
+ CRUSH rule 3 x 741 [7,2,3]
+ CRUSH rule 3 x 742 [8,2,3]
+ CRUSH rule 3 x 743 [7,0,5]
+ CRUSH rule 3 x 744 [4,6,1]
+ CRUSH rule 3 x 745 [3,0,8]
+ CRUSH rule 3 x 746 [4,1,7]
+ CRUSH rule 3 x 747 [6,2,4]
+ CRUSH rule 3 x 748 [2,6,5]
+ CRUSH rule 3 x 749 [4,7,2]
+ CRUSH rule 3 x 750 [1,7,4]
+ CRUSH rule 3 x 751 [2,7,3]
+ CRUSH rule 3 x 752 [8,0,3]
+ CRUSH rule 3 x 753 [7,4,2]
+ CRUSH rule 3 x 754 [8,5,2]
+ CRUSH rule 3 x 755 [1,6,5]
+ CRUSH rule 3 x 756 [5,8,1]
+ CRUSH rule 3 x 757 [8,0,4]
+ CRUSH rule 3 x 758 [6,1,5]
+ CRUSH rule 3 x 759 [8,5,0]
+ CRUSH rule 3 x 760 [1,5,7]
+ CRUSH rule 3 x 761 [4,2,6]
+ CRUSH rule 3 x 762 [2,8,3]
+ CRUSH rule 3 x 763 [8,3,1]
+ CRUSH rule 3 x 764 [1,7,4]
+ CRUSH rule 3 x 765 [6,4,0]
+ CRUSH rule 3 x 766 [8,3,0]
+ CRUSH rule 3 x 767 [1,8,5]
+ CRUSH rule 3 x 768 [8,5,0]
+ CRUSH rule 3 x 769 [6,2,4]
+ CRUSH rule 3 x 770 [6,1,4]
+ CRUSH rule 3 x 771 [7,2,4]
+ CRUSH rule 3 x 772 [8,4,2]
+ CRUSH rule 3 x 773 [3,2,6]
+ CRUSH rule 3 x 774 [4,7,1]
+ CRUSH rule 3 x 775 [6,5,0]
+ CRUSH rule 3 x 776 [7,0,3]
+ CRUSH rule 3 x 777 [3,0,6]
+ CRUSH rule 3 x 778 [1,6,3]
+ CRUSH rule 3 x 779 [2,6,5]
+ CRUSH rule 3 x 780 [0,5,8]
+ CRUSH rule 3 x 781 [6,5,0]
+ CRUSH rule 3 x 782 [5,2,8]
+ CRUSH rule 3 x 783 [7,0,3]
+ CRUSH rule 3 x 784 [0,5,8]
+ CRUSH rule 3 x 785 [6,1,3]
+ CRUSH rule 3 x 786 [7,3,0]
+ CRUSH rule 3 x 787 [1,8,5]
+ CRUSH rule 3 x 788 [6,0,4]
+ CRUSH rule 3 x 789 [0,5,8]
+ CRUSH rule 3 x 790 [8,3,1]
+ CRUSH rule 3 x 791 [3,6,2]
+ CRUSH rule 3 x 792 [5,6,1]
+ CRUSH rule 3 x 793 [6,2,3]
+ CRUSH rule 3 x 794 [2,8,3]
+ CRUSH rule 3 x 795 [0,4,6]
+ CRUSH rule 3 x 796 [3,7,1]
+ CRUSH rule 3 x 797 [2,3,6]
+ CRUSH rule 3 x 798 [6,0,3]
+ CRUSH rule 3 x 799 [5,2,7]
+ CRUSH rule 3 x 800 [5,2,8]
+ CRUSH rule 3 x 801 [3,7,2]
+ CRUSH rule 3 x 802 [1,6,4]
+ CRUSH rule 3 x 803 [0,4,7]
+ CRUSH rule 3 x 804 [6,0,5]
+ CRUSH rule 3 x 805 [3,8,0]
+ CRUSH rule 3 x 806 [1,5,6]
+ CRUSH rule 3 x 807 [5,7,2]
+ CRUSH rule 3 x 808 [4,7,2]
+ CRUSH rule 3 x 809 [1,3,6]
+ CRUSH rule 3 x 810 [5,7,2]
+ CRUSH rule 3 x 811 [8,4,1]
+ CRUSH rule 3 x 812 [8,3,1]
+ CRUSH rule 3 x 813 [6,4,2]
+ CRUSH rule 3 x 814 [3,8,0]
+ CRUSH rule 3 x 815 [3,1,6]
+ CRUSH rule 3 x 816 [2,6,5]
+ CRUSH rule 3 x 817 [4,6,2]
+ CRUSH rule 3 x 818 [3,1,7]
+ CRUSH rule 3 x 819 [5,0,7]
+ CRUSH rule 3 x 820 [3,7,1]
+ CRUSH rule 3 x 821 [4,8,0]
+ CRUSH rule 3 x 822 [2,3,7]
+ CRUSH rule 3 x 823 [4,7,2]
+ CRUSH rule 3 x 824 [3,7,1]
+ CRUSH rule 3 x 825 [2,8,4]
+ CRUSH rule 3 x 826 [7,1,5]
+ CRUSH rule 3 x 827 [0,6,3]
+ CRUSH rule 3 x 828 [2,5,8]
+ CRUSH rule 3 x 829 [5,6,0]
+ CRUSH rule 3 x 830 [2,4,7]
+ CRUSH rule 3 x 831 [1,6,4]
+ CRUSH rule 3 x 832 [4,8,2]
+ CRUSH rule 3 x 833 [2,6,5]
+ CRUSH rule 3 x 834 [3,0,8]
+ CRUSH rule 3 x 835 [8,5,0]
+ CRUSH rule 3 x 836 [3,8,1]
+ CRUSH rule 3 x 837 [6,4,0]
+ CRUSH rule 3 x 838 [6,0,4]
+ CRUSH rule 3 x 839 [5,2,8]
+ CRUSH rule 3 x 840 [7,3,2]
+ CRUSH rule 3 x 841 [4,8,0]
+ CRUSH rule 3 x 842 [2,5,8]
+ CRUSH rule 3 x 843 [6,4,2]
+ CRUSH rule 3 x 844 [4,8,2]
+ CRUSH rule 3 x 845 [3,6,2]
+ CRUSH rule 3 x 846 [3,0,6]
+ CRUSH rule 3 x 847 [0,8,5]
+ CRUSH rule 3 x 848 [2,6,5]
+ CRUSH rule 3 x 849 [4,8,1]
+ CRUSH rule 3 x 850 [1,5,7]
+ CRUSH rule 3 x 851 [6,5,1]
+ CRUSH rule 3 x 852 [7,4,2]
+ CRUSH rule 3 x 853 [6,2,5]
+ CRUSH rule 3 x 854 [7,0,3]
+ CRUSH rule 3 x 855 [5,7,2]
+ CRUSH rule 3 x 856 [6,3,1]
+ CRUSH rule 3 x 857 [8,4,0]
+ CRUSH rule 3 x 858 [6,5,2]
+ CRUSH rule 3 x 859 [6,0,5]
+ CRUSH rule 3 x 860 [4,2,8]
+ CRUSH rule 3 x 861 [8,4,1]
+ CRUSH rule 3 x 862 [6,0,5]
+ CRUSH rule 3 x 863 [8,1,5]
+ CRUSH rule 3 x 864 [5,6,1]
+ CRUSH rule 3 x 865 [8,0,3]
+ CRUSH rule 3 x 866 [3,6,2]
+ CRUSH rule 3 x 867 [6,3,1]
+ CRUSH rule 3 x 868 [6,4,2]
+ CRUSH rule 3 x 869 [8,4,2]
+ CRUSH rule 3 x 870 [0,5,7]
+ CRUSH rule 3 x 871 [3,1,6]
+ CRUSH rule 3 x 872 [5,0,6]
+ CRUSH rule 3 x 873 [4,6,1]
+ CRUSH rule 3 x 874 [2,6,4]
+ CRUSH rule 3 x 875 [2,6,5]
+ CRUSH rule 3 x 876 [5,8,1]
+ CRUSH rule 3 x 877 [6,4,0]
+ CRUSH rule 3 x 878 [5,0,7]
+ CRUSH rule 3 x 879 [7,3,1]
+ CRUSH rule 3 x 880 [3,1,7]
+ CRUSH rule 3 x 881 [5,8,2]
+ CRUSH rule 3 x 882 [4,1,6]
+ CRUSH rule 3 x 883 [2,5,6]
+ CRUSH rule 3 x 884 [6,0,3]
+ CRUSH rule 3 x 885 [5,1,6]
+ CRUSH rule 3 x 886 [3,7,2]
+ CRUSH rule 3 x 887 [7,5,2]
+ CRUSH rule 3 x 888 [6,1,4]
+ CRUSH rule 3 x 889 [2,8,3]
+ CRUSH rule 3 x 890 [7,1,4]
+ CRUSH rule 3 x 891 [1,7,4]
+ CRUSH rule 3 x 892 [6,0,5]
+ CRUSH rule 3 x 893 [2,3,6]
+ CRUSH rule 3 x 894 [7,4,1]
+ CRUSH rule 3 x 895 [5,1,6]
+ CRUSH rule 3 x 896 [1,6,3]
+ CRUSH rule 3 x 897 [4,0,8]
+ CRUSH rule 3 x 898 [0,3,8]
+ CRUSH rule 3 x 899 [1,6,4]
+ CRUSH rule 3 x 900 [4,0,7]
+ CRUSH rule 3 x 901 [5,2,6]
+ CRUSH rule 3 x 902 [8,4,2]
+ CRUSH rule 3 x 903 [5,6,1]
+ CRUSH rule 3 x 904 [5,7,0]
+ CRUSH rule 3 x 905 [6,0,3]
+ CRUSH rule 3 x 906 [1,7,4]
+ CRUSH rule 3 x 907 [7,2,3]
+ CRUSH rule 3 x 908 [5,6,1]
+ CRUSH rule 3 x 909 [2,3,8]
+ CRUSH rule 3 x 910 [6,5,0]
+ CRUSH rule 3 x 911 [5,7,1]
+ CRUSH rule 3 x 912 [0,8,5]
+ CRUSH rule 3 x 913 [7,1,4]
+ CRUSH rule 3 x 914 [6,4,0]
+ CRUSH rule 3 x 915 [8,2,3]
+ CRUSH rule 3 x 916 [3,0,6]
+ CRUSH rule 3 x 917 [1,3,6]
+ CRUSH rule 3 x 918 [8,0,5]
+ CRUSH rule 3 x 919 [6,2,3]
+ CRUSH rule 3 x 920 [7,4,2]
+ CRUSH rule 3 x 921 [1,4,8]
+ CRUSH rule 3 x 922 [6,3,1]
+ CRUSH rule 3 x 923 [5,6,0]
+ CRUSH rule 3 x 924 [3,1,8]
+ CRUSH rule 3 x 925 [5,6,0]
+ CRUSH rule 3 x 926 [3,0,7]
+ CRUSH rule 3 x 927 [1,6,5]
+ CRUSH rule 3 x 928 [8,1,4]
+ CRUSH rule 3 x 929 [4,2,6]
+ CRUSH rule 3 x 930 [2,5,8]
+ CRUSH rule 3 x 931 [5,2,8]
+ CRUSH rule 3 x 932 [4,2,7]
+ CRUSH rule 3 x 933 [8,4,2]
+ CRUSH rule 3 x 934 [5,8,0]
+ CRUSH rule 3 x 935 [6,3,0]
+ CRUSH rule 3 x 936 [0,7,3]
+ CRUSH rule 3 x 937 [5,8,1]
+ CRUSH rule 3 x 938 [6,4,0]
+ CRUSH rule 3 x 939 [2,8,3]
+ CRUSH rule 3 x 940 [8,5,2]
+ CRUSH rule 3 x 941 [5,0,6]
+ CRUSH rule 3 x 942 [1,6,4]
+ CRUSH rule 3 x 943 [8,2,4]
+ CRUSH rule 3 x 944 [4,8,1]
+ CRUSH rule 3 x 945 [7,0,3]
+ CRUSH rule 3 x 946 [2,8,3]
+ CRUSH rule 3 x 947 [4,2,6]
+ CRUSH rule 3 x 948 [7,4,1]
+ CRUSH rule 3 x 949 [6,2,5]
+ CRUSH rule 3 x 950 [3,7,0]
+ CRUSH rule 3 x 951 [4,6,2]
+ CRUSH rule 3 x 952 [2,7,4]
+ CRUSH rule 3 x 953 [1,3,6]
+ CRUSH rule 3 x 954 [4,0,8]
+ CRUSH rule 3 x 955 [8,1,4]
+ CRUSH rule 3 x 956 [1,7,4]
+ CRUSH rule 3 x 957 [7,0,3]
+ CRUSH rule 3 x 958 [8,3,2]
+ CRUSH rule 3 x 959 [5,1,6]
+ CRUSH rule 3 x 960 [3,6,2]
+ CRUSH rule 3 x 961 [4,1,6]
+ CRUSH rule 3 x 962 [7,5,0]
+ CRUSH rule 3 x 963 [0,5,8]
+ CRUSH rule 3 x 964 [3,2,7]
+ CRUSH rule 3 x 965 [7,3,1]
+ CRUSH rule 3 x 966 [3,6,0]
+ CRUSH rule 3 x 967 [8,4,2]
+ CRUSH rule 3 x 968 [7,0,4]
+ CRUSH rule 3 x 969 [8,0,4]
+ CRUSH rule 3 x 970 [0,8,3]
+ CRUSH rule 3 x 971 [1,8,5]
+ CRUSH rule 3 x 972 [1,7,5]
+ CRUSH rule 3 x 973 [1,8,3]
+ CRUSH rule 3 x 974 [5,1,7]
+ CRUSH rule 3 x 975 [3,8,0]
+ CRUSH rule 3 x 976 [4,7,2]
+ CRUSH rule 3 x 977 [8,3,2]
+ CRUSH rule 3 x 978 [7,0,3]
+ CRUSH rule 3 x 979 [7,2,5]
+ CRUSH rule 3 x 980 [6,2,3]
+ CRUSH rule 3 x 981 [7,5,1]
+ CRUSH rule 3 x 982 [4,1,8]
+ CRUSH rule 3 x 983 [3,6,0]
+ CRUSH rule 3 x 984 [0,8,4]
+ CRUSH rule 3 x 985 [2,4,8]
+ CRUSH rule 3 x 986 [8,4,2]
+ CRUSH rule 3 x 987 [0,4,8]
+ CRUSH rule 3 x 988 [1,4,6]
+ CRUSH rule 3 x 989 [0,8,5]
+ CRUSH rule 3 x 990 [1,6,4]
+ CRUSH rule 3 x 991 [0,4,8]
+ CRUSH rule 3 x 992 [7,0,4]
+ CRUSH rule 3 x 993 [0,6,4]
+ CRUSH rule 3 x 994 [3,6,1]
+ CRUSH rule 3 x 995 [7,0,4]
+ CRUSH rule 3 x 996 [6,5,1]
+ CRUSH rule 3 x 997 [6,3,1]
+ CRUSH rule 3 x 998 [8,0,4]
+ CRUSH rule 3 x 999 [0,7,5]
+ CRUSH rule 3 x 1000 [8,4,1]
+ CRUSH rule 3 x 1001 [2,3,6]
+ CRUSH rule 3 x 1002 [1,3,8]
+ CRUSH rule 3 x 1003 [2,7,3]
+ CRUSH rule 3 x 1004 [6,0,3]
+ CRUSH rule 3 x 1005 [6,1,4]
+ CRUSH rule 3 x 1006 [1,8,4]
+ CRUSH rule 3 x 1007 [1,3,8]
+ CRUSH rule 3 x 1008 [1,7,4]
+ CRUSH rule 3 x 1009 [6,5,1]
+ CRUSH rule 3 x 1010 [3,1,6]
+ CRUSH rule 3 x 1011 [3,0,8]
+ CRUSH rule 3 x 1012 [3,1,8]
+ CRUSH rule 3 x 1013 [5,2,8]
+ CRUSH rule 3 x 1014 [2,8,3]
+ CRUSH rule 3 x 1015 [6,3,2]
+ CRUSH rule 3 x 1016 [2,5,6]
+ CRUSH rule 3 x 1017 [6,1,5]
+ CRUSH rule 3 x 1018 [5,1,6]
+ CRUSH rule 3 x 1019 [5,8,1]
+ CRUSH rule 3 x 1020 [5,0,7]
+ CRUSH rule 3 x 1021 [5,2,7]
+ CRUSH rule 3 x 1022 [1,7,4]
+ CRUSH rule 3 x 1023 [3,0,7]
+ rule 3 (choose-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 4 (choose-set-two), x = 0..1023, numrep = 2..3
+ CRUSH rule 4 x 0 [0,2]
+ CRUSH rule 4 x 1 [0,8]
+ CRUSH rule 4 x 2 [1,3]
+ CRUSH rule 4 x 3 [8,0]
+ CRUSH rule 4 x 4 [5,4]
+ CRUSH rule 4 x 5 [7,8]
+ CRUSH rule 4 x 6 [2,6]
+ CRUSH rule 4 x 7 [5,4]
+ CRUSH rule 4 x 8 [5,3]
+ CRUSH rule 4 x 9 [2,3]
+ CRUSH rule 4 x 10 [0,2]
+ CRUSH rule 4 x 11 [0,7]
+ CRUSH rule 4 x 12 [0,2]
+ CRUSH rule 4 x 13 [3,8]
+ CRUSH rule 4 x 14 [7,6]
+ CRUSH rule 4 x 15 [7,2]
+ CRUSH rule 4 x 16 [3,6]
+ CRUSH rule 4 x 17 [5,4]
+ CRUSH rule 4 x 18 [1,4]
+ CRUSH rule 4 x 19 [7,5]
+ CRUSH rule 4 x 20 [2,4]
+ CRUSH rule 4 x 21 [3,7]
+ CRUSH rule 4 x 22 [8,3]
+ CRUSH rule 4 x 23 [3,6]
+ CRUSH rule 4 x 24 [1,0]
+ CRUSH rule 4 x 25 [3,7]
+ CRUSH rule 4 x 26 [2,8]
+ CRUSH rule 4 x 27 [3,1]
+ CRUSH rule 4 x 28 [6,0]
+ CRUSH rule 4 x 29 [8,5]
+ CRUSH rule 4 x 30 [5,7]
+ CRUSH rule 4 x 31 [8,7]
+ CRUSH rule 4 x 32 [3,6]
+ CRUSH rule 4 x 33 [2,7]
+ CRUSH rule 4 x 34 [2,5]
+ CRUSH rule 4 x 35 [0,8]
+ CRUSH rule 4 x 36 [3,8]
+ CRUSH rule 4 x 37 [0,4]
+ CRUSH rule 4 x 38 [4,8]
+ CRUSH rule 4 x 39 [3,7]
+ CRUSH rule 4 x 40 [7,8]
+ CRUSH rule 4 x 41 [0,2]
+ CRUSH rule 4 x 42 [4,3]
+ CRUSH rule 4 x 43 [0,3]
+ CRUSH rule 4 x 44 [1,6]
+ CRUSH rule 4 x 45 [8,0]
+ CRUSH rule 4 x 46 [2,4]
+ CRUSH rule 4 x 47 [4,5]
+ CRUSH rule 4 x 48 [4,6]
+ CRUSH rule 4 x 49 [5,4]
+ CRUSH rule 4 x 50 [3,4]
+ CRUSH rule 4 x 51 [3,6]
+ CRUSH rule 4 x 52 [8,6]
+ CRUSH rule 4 x 53 [3,5]
+ CRUSH rule 4 x 54 [7,6]
+ CRUSH rule 4 x 55 [8,7]
+ CRUSH rule 4 x 56 [6,4]
+ CRUSH rule 4 x 57 [5,3]
+ CRUSH rule 4 x 58 [1,0]
+ CRUSH rule 4 x 59 [4,2]
+ CRUSH rule 4 x 60 [3,5]
+ CRUSH rule 4 x 61 [4,6]
+ CRUSH rule 4 x 62 [7,0]
+ CRUSH rule 4 x 63 [5,6]
+ CRUSH rule 4 x 64 [4,5]
+ CRUSH rule 4 x 65 [7,3]
+ CRUSH rule 4 x 66 [5,4]
+ CRUSH rule 4 x 67 [5,0]
+ CRUSH rule 4 x 68 [0,5]
+ CRUSH rule 4 x 69 [5,1]
+ CRUSH rule 4 x 70 [7,0]
+ CRUSH rule 4 x 71 [2,8]
+ CRUSH rule 4 x 72 [6,1]
+ CRUSH rule 4 x 73 [2,7]
+ CRUSH rule 4 x 74 [0,7]
+ CRUSH rule 4 x 75 [3,2]
+ CRUSH rule 4 x 76 [5,1]
+ CRUSH rule 4 x 77 [7,2]
+ CRUSH rule 4 x 78 [1,4]
+ CRUSH rule 4 x 79 [5,4]
+ CRUSH rule 4 x 80 [0,3]
+ CRUSH rule 4 x 81 [0,2]
+ CRUSH rule 4 x 82 [7,1]
+ CRUSH rule 4 x 83 [2,6]
+ CRUSH rule 4 x 84 [7,2]
+ CRUSH rule 4 x 85 [3,4]
+ CRUSH rule 4 x 86 [0,2]
+ CRUSH rule 4 x 87 [0,7]
+ CRUSH rule 4 x 88 [1,6]
+ CRUSH rule 4 x 89 [3,0]
+ CRUSH rule 4 x 90 [6,7]
+ CRUSH rule 4 x 91 [3,8]
+ CRUSH rule 4 x 92 [1,8]
+ CRUSH rule 4 x 93 [7,4]
+ CRUSH rule 4 x 94 [0,4]
+ CRUSH rule 4 x 95 [7,5]
+ CRUSH rule 4 x 96 [3,6]
+ CRUSH rule 4 x 97 [8,7]
+ CRUSH rule 4 x 98 [2,0]
+ CRUSH rule 4 x 99 [0,7]
+ CRUSH rule 4 x 100 [1,7]
+ CRUSH rule 4 x 101 [3,7]
+ CRUSH rule 4 x 102 [4,2]
+ CRUSH rule 4 x 103 [4,7]
+ CRUSH rule 4 x 104 [7,4]
+ CRUSH rule 4 x 105 [2,4]
+ CRUSH rule 4 x 106 [1,6]
+ CRUSH rule 4 x 107 [3,2]
+ CRUSH rule 4 x 108 [7,2]
+ CRUSH rule 4 x 109 [1,2]
+ CRUSH rule 4 x 110 [3,2]
+ CRUSH rule 4 x 111 [2,1]
+ CRUSH rule 4 x 112 [2,0]
+ CRUSH rule 4 x 113 [6,2]
+ CRUSH rule 4 x 114 [7,6]
+ CRUSH rule 4 x 115 [8,2]
+ CRUSH rule 4 x 116 [1,2]
+ CRUSH rule 4 x 117 [7,3]
+ CRUSH rule 4 x 118 [0,3]
+ CRUSH rule 4 x 119 [5,6]
+ CRUSH rule 4 x 120 [0,2]
+ CRUSH rule 4 x 121 [2,0]
+ CRUSH rule 4 x 122 [8,5]
+ CRUSH rule 4 x 123 [2,5]
+ CRUSH rule 4 x 124 [3,5]
+ CRUSH rule 4 x 125 [0,7]
+ CRUSH rule 4 x 126 [4,2]
+ CRUSH rule 4 x 127 [3,6]
+ CRUSH rule 4 x 128 [3,5]
+ CRUSH rule 4 x 129 [0,2]
+ CRUSH rule 4 x 130 [3,8]
+ CRUSH rule 4 x 131 [1,2]
+ CRUSH rule 4 x 132 [1,2]
+ CRUSH rule 4 x 133 [3,6]
+ CRUSH rule 4 x 134 [1,8]
+ CRUSH rule 4 x 135 [5,6]
+ CRUSH rule 4 x 136 [2,1]
+ CRUSH rule 4 x 137 [7,3]
+ CRUSH rule 4 x 138 [8,7]
+ CRUSH rule 4 x 139 [3,0]
+ CRUSH rule 4 x 140 [1,6]
+ CRUSH rule 4 x 141 [6,8]
+ CRUSH rule 4 x 142 [3,5]
+ CRUSH rule 4 x 143 [5,8]
+ CRUSH rule 4 x 144 [8,1]
+ CRUSH rule 4 x 145 [8,5]
+ CRUSH rule 4 x 146 [2,6]
+ CRUSH rule 4 x 147 [2,8]
+ CRUSH rule 4 x 148 [3,4]
+ CRUSH rule 4 x 149 [4,8]
+ CRUSH rule 4 x 150 [1,6]
+ CRUSH rule 4 x 151 [3,4]
+ CRUSH rule 4 x 152 [8,4]
+ CRUSH rule 4 x 153 [8,6]
+ CRUSH rule 4 x 154 [3,2]
+ CRUSH rule 4 x 155 [3,5]
+ CRUSH rule 4 x 156 [4,5]
+ CRUSH rule 4 x 157 [4,1]
+ CRUSH rule 4 x 158 [2,8]
+ CRUSH rule 4 x 159 [7,0]
+ CRUSH rule 4 x 160 [2,8]
+ CRUSH rule 4 x 161 [1,5]
+ CRUSH rule 4 x 162 [0,6]
+ CRUSH rule 4 x 163 [5,6]
+ CRUSH rule 4 x 164 [7,8]
+ CRUSH rule 4 x 165 [7,0]
+ CRUSH rule 4 x 166 [2,4]
+ CRUSH rule 4 x 167 [0,1]
+ CRUSH rule 4 x 168 [4,2]
+ CRUSH rule 4 x 169 [2,6]
+ CRUSH rule 4 x 170 [1,0]
+ CRUSH rule 4 x 171 [7,5]
+ CRUSH rule 4 x 172 [0,7]
+ CRUSH rule 4 x 173 [8,5]
+ CRUSH rule 4 x 174 [1,4]
+ CRUSH rule 4 x 175 [6,0]
+ CRUSH rule 4 x 176 [4,3]
+ CRUSH rule 4 x 177 [5,3]
+ CRUSH rule 4 x 178 [3,0]
+ CRUSH rule 4 x 179 [4,1]
+ CRUSH rule 4 x 180 [3,8]
+ CRUSH rule 4 x 181 [6,2]
+ CRUSH rule 4 x 182 [8,5]
+ CRUSH rule 4 x 183 [7,8]
+ CRUSH rule 4 x 184 [5,7]
+ CRUSH rule 4 x 185 [6,8]
+ CRUSH rule 4 x 186 [2,1]
+ CRUSH rule 4 x 187 [1,6]
+ CRUSH rule 4 x 188 [1,8]
+ CRUSH rule 4 x 189 [0,7]
+ CRUSH rule 4 x 190 [4,0]
+ CRUSH rule 4 x 191 [7,6]
+ CRUSH rule 4 x 192 [5,0]
+ CRUSH rule 4 x 193 [4,2]
+ CRUSH rule 4 x 194 [1,3]
+ CRUSH rule 4 x 195 [6,4]
+ CRUSH rule 4 x 196 [6,7]
+ CRUSH rule 4 x 197 [6,5]
+ CRUSH rule 4 x 198 [2,5]
+ CRUSH rule 4 x 199 [0,5]
+ CRUSH rule 4 x 200 [0,5]
+ CRUSH rule 4 x 201 [7,1]
+ CRUSH rule 4 x 202 [6,3]
+ CRUSH rule 4 x 203 [4,8]
+ CRUSH rule 4 x 204 [2,1]
+ CRUSH rule 4 x 205 [0,7]
+ CRUSH rule 4 x 206 [0,1]
+ CRUSH rule 4 x 207 [3,0]
+ CRUSH rule 4 x 208 [7,1]
+ CRUSH rule 4 x 209 [1,2]
+ CRUSH rule 4 x 210 [1,2]
+ CRUSH rule 4 x 211 [5,4]
+ CRUSH rule 4 x 212 [7,5]
+ CRUSH rule 4 x 213 [8,4]
+ CRUSH rule 4 x 214 [4,5]
+ CRUSH rule 4 x 215 [8,1]
+ CRUSH rule 4 x 216 [5,0]
+ CRUSH rule 4 x 217 [0,1]
+ CRUSH rule 4 x 218 [0,7]
+ CRUSH rule 4 x 219 [4,8]
+ CRUSH rule 4 x 220 [5,7]
+ CRUSH rule 4 x 221 [3,6]
+ CRUSH rule 4 x 222 [6,8]
+ CRUSH rule 4 x 223 [1,3]
+ CRUSH rule 4 x 224 [1,5]
+ CRUSH rule 4 x 225 [8,6]
+ CRUSH rule 4 x 226 [7,2]
+ CRUSH rule 4 x 227 [3,4]
+ CRUSH rule 4 x 228 [5,4]
+ CRUSH rule 4 x 229 [3,4]
+ CRUSH rule 4 x 230 [4,7]
+ CRUSH rule 4 x 231 [4,3]
+ CRUSH rule 4 x 232 [2,7]
+ CRUSH rule 4 x 233 [3,4]
+ CRUSH rule 4 x 234 [0,1]
+ CRUSH rule 4 x 235 [3,8]
+ CRUSH rule 4 x 236 [5,2]
+ CRUSH rule 4 x 237 [4,7]
+ CRUSH rule 4 x 238 [4,3]
+ CRUSH rule 4 x 239 [8,7]
+ CRUSH rule 4 x 240 [5,7]
+ CRUSH rule 4 x 241 [3,1]
+ CRUSH rule 4 x 242 [3,5]
+ CRUSH rule 4 x 243 [4,7]
+ CRUSH rule 4 x 244 [4,6]
+ CRUSH rule 4 x 245 [7,6]
+ CRUSH rule 4 x 246 [1,5]
+ CRUSH rule 4 x 247 [6,0]
+ CRUSH rule 4 x 248 [8,0]
+ CRUSH rule 4 x 249 [2,0]
+ CRUSH rule 4 x 250 [2,1]
+ CRUSH rule 4 x 251 [2,3]
+ CRUSH rule 4 x 252 [3,7]
+ CRUSH rule 4 x 253 [3,2]
+ CRUSH rule 4 x 254 [3,5]
+ CRUSH rule 4 x 255 [1,7]
+ CRUSH rule 4 x 256 [5,7]
+ CRUSH rule 4 x 257 [2,8]
+ CRUSH rule 4 x 258 [5,3]
+ CRUSH rule 4 x 259 [4,3]
+ CRUSH rule 4 x 260 [3,6]
+ CRUSH rule 4 x 261 [8,7]
+ CRUSH rule 4 x 262 [5,4]
+ CRUSH rule 4 x 263 [6,8]
+ CRUSH rule 4 x 264 [3,6]
+ CRUSH rule 4 x 265 [8,6]
+ CRUSH rule 4 x 266 [8,2]
+ CRUSH rule 4 x 267 [2,3]
+ CRUSH rule 4 x 268 [0,7]
+ CRUSH rule 4 x 269 [0,8]
+ CRUSH rule 4 x 270 [5,0]
+ CRUSH rule 4 x 271 [7,5]
+ CRUSH rule 4 x 272 [2,8]
+ CRUSH rule 4 x 273 [3,5]
+ CRUSH rule 4 x 274 [6,8]
+ CRUSH rule 4 x 275 [4,3]
+ CRUSH rule 4 x 276 [7,1]
+ CRUSH rule 4 x 277 [6,4]
+ CRUSH rule 4 x 278 [6,8]
+ CRUSH rule 4 x 279 [8,3]
+ CRUSH rule 4 x 280 [0,6]
+ CRUSH rule 4 x 281 [8,0]
+ CRUSH rule 4 x 282 [3,1]
+ CRUSH rule 4 x 283 [8,2]
+ CRUSH rule 4 x 284 [6,3]
+ CRUSH rule 4 x 285 [5,3]
+ CRUSH rule 4 x 286 [2,1]
+ CRUSH rule 4 x 287 [0,4]
+ CRUSH rule 4 x 288 [8,0]
+ CRUSH rule 4 x 289 [4,6]
+ CRUSH rule 4 x 290 [1,3]
+ CRUSH rule 4 x 291 [0,1]
+ CRUSH rule 4 x 292 [8,0]
+ CRUSH rule 4 x 293 [6,0]
+ CRUSH rule 4 x 294 [7,4]
+ CRUSH rule 4 x 295 [4,8]
+ CRUSH rule 4 x 296 [3,1]
+ CRUSH rule 4 x 297 [6,2]
+ CRUSH rule 4 x 298 [1,2]
+ CRUSH rule 4 x 299 [2,0]
+ CRUSH rule 4 x 300 [8,7]
+ CRUSH rule 4 x 301 [0,8]
+ CRUSH rule 4 x 302 [3,0]
+ CRUSH rule 4 x 303 [7,5]
+ CRUSH rule 4 x 304 [2,7]
+ CRUSH rule 4 x 305 [5,8]
+ CRUSH rule 4 x 306 [0,7]
+ CRUSH rule 4 x 307 [0,2]
+ CRUSH rule 4 x 308 [0,8]
+ CRUSH rule 4 x 309 [7,4]
+ CRUSH rule 4 x 310 [4,3]
+ CRUSH rule 4 x 311 [3,4]
+ CRUSH rule 4 x 312 [2,1]
+ CRUSH rule 4 x 313 [5,3]
+ CRUSH rule 4 x 314 [4,5]
+ CRUSH rule 4 x 315 [2,0]
+ CRUSH rule 4 x 316 [6,3]
+ CRUSH rule 4 x 317 [2,6]
+ CRUSH rule 4 x 318 [8,6]
+ CRUSH rule 4 x 319 [5,0]
+ CRUSH rule 4 x 320 [3,7]
+ CRUSH rule 4 x 321 [1,3]
+ CRUSH rule 4 x 322 [2,7]
+ CRUSH rule 4 x 323 [4,7]
+ CRUSH rule 4 x 324 [7,0]
+ CRUSH rule 4 x 325 [4,6]
+ CRUSH rule 4 x 326 [3,4]
+ CRUSH rule 4 x 327 [0,6]
+ CRUSH rule 4 x 328 [7,4]
+ CRUSH rule 4 x 329 [5,6]
+ CRUSH rule 4 x 330 [3,7]
+ CRUSH rule 4 x 331 [2,6]
+ CRUSH rule 4 x 332 [2,0]
+ CRUSH rule 4 x 333 [6,8]
+ CRUSH rule 4 x 334 [8,3]
+ CRUSH rule 4 x 335 [7,1]
+ CRUSH rule 4 x 336 [4,6]
+ CRUSH rule 4 x 337 [7,2]
+ CRUSH rule 4 x 338 [5,6]
+ CRUSH rule 4 x 339 [7,5]
+ CRUSH rule 4 x 340 [2,0]
+ CRUSH rule 4 x 341 [5,1]
+ CRUSH rule 4 x 342 [0,7]
+ CRUSH rule 4 x 343 [6,7]
+ CRUSH rule 4 x 344 [6,0]
+ CRUSH rule 4 x 345 [4,3]
+ CRUSH rule 4 x 346 [8,0]
+ CRUSH rule 4 x 347 [3,1]
+ CRUSH rule 4 x 348 [8,0]
+ CRUSH rule 4 x 349 [1,6]
+ CRUSH rule 4 x 350 [8,5]
+ CRUSH rule 4 x 351 [3,6]
+ CRUSH rule 4 x 352 [1,0]
+ CRUSH rule 4 x 353 [6,4]
+ CRUSH rule 4 x 354 [0,3]
+ CRUSH rule 4 x 355 [3,4]
+ CRUSH rule 4 x 356 [3,5]
+ CRUSH rule 4 x 357 [6,1]
+ CRUSH rule 4 x 358 [2,1]
+ CRUSH rule 4 x 359 [6,7]
+ CRUSH rule 4 x 360 [5,3]
+ CRUSH rule 4 x 361 [8,4]
+ CRUSH rule 4 x 362 [4,5]
+ CRUSH rule 4 x 363 [4,0]
+ CRUSH rule 4 x 364 [2,5]
+ CRUSH rule 4 x 365 [6,7]
+ CRUSH rule 4 x 366 [7,2]
+ CRUSH rule 4 x 367 [4,5]
+ CRUSH rule 4 x 368 [7,4]
+ CRUSH rule 4 x 369 [3,7]
+ CRUSH rule 4 x 370 [8,7]
+ CRUSH rule 4 x 371 [1,3]
+ CRUSH rule 4 x 372 [3,1]
+ CRUSH rule 4 x 373 [0,6]
+ CRUSH rule 4 x 374 [3,8]
+ CRUSH rule 4 x 375 [6,4]
+ CRUSH rule 4 x 376 [7,1]
+ CRUSH rule 4 x 377 [1,2]
+ CRUSH rule 4 x 378 [0,1]
+ CRUSH rule 4 x 379 [8,5]
+ CRUSH rule 4 x 380 [2,5]
+ CRUSH rule 4 x 381 [0,4]
+ CRUSH rule 4 x 382 [1,5]
+ CRUSH rule 4 x 383 [4,3]
+ CRUSH rule 4 x 384 [7,0]
+ CRUSH rule 4 x 385 [7,4]
+ CRUSH rule 4 x 386 [0,3]
+ CRUSH rule 4 x 387 [1,3]
+ CRUSH rule 4 x 388 [5,0]
+ CRUSH rule 4 x 389 [1,5]
+ CRUSH rule 4 x 390 [5,6]
+ CRUSH rule 4 x 391 [5,6]
+ CRUSH rule 4 x 392 [1,8]
+ CRUSH rule 4 x 393 [4,2]
+ CRUSH rule 4 x 394 [4,7]
+ CRUSH rule 4 x 395 [4,0]
+ CRUSH rule 4 x 396 [4,2]
+ CRUSH rule 4 x 397 [2,1]
+ CRUSH rule 4 x 398 [2,4]
+ CRUSH rule 4 x 399 [8,7]
+ CRUSH rule 4 x 400 [8,1]
+ CRUSH rule 4 x 401 [0,1]
+ CRUSH rule 4 x 402 [7,8]
+ CRUSH rule 4 x 403 [0,1]
+ CRUSH rule 4 x 404 [4,3]
+ CRUSH rule 4 x 405 [6,5]
+ CRUSH rule 4 x 406 [2,0]
+ CRUSH rule 4 x 407 [2,8]
+ CRUSH rule 4 x 408 [4,1]
+ CRUSH rule 4 x 409 [7,3]
+ CRUSH rule 4 x 410 [8,6]
+ CRUSH rule 4 x 411 [2,0]
+ CRUSH rule 4 x 412 [0,5]
+ CRUSH rule 4 x 413 [5,0]
+ CRUSH rule 4 x 414 [4,1]
+ CRUSH rule 4 x 415 [0,6]
+ CRUSH rule 4 x 416 [2,1]
+ CRUSH rule 4 x 417 [8,7]
+ CRUSH rule 4 x 418 [7,6]
+ CRUSH rule 4 x 419 [8,3]
+ CRUSH rule 4 x 420 [1,4]
+ CRUSH rule 4 x 421 [8,6]
+ CRUSH rule 4 x 422 [6,7]
+ CRUSH rule 4 x 423 [0,5]
+ CRUSH rule 4 x 424 [8,4]
+ CRUSH rule 4 x 425 [1,3]
+ CRUSH rule 4 x 426 [6,7]
+ CRUSH rule 4 x 427 [0,7]
+ CRUSH rule 4 x 428 [5,4]
+ CRUSH rule 4 x 429 [4,6]
+ CRUSH rule 4 x 430 [3,6]
+ CRUSH rule 4 x 431 [5,3]
+ CRUSH rule 4 x 432 [7,1]
+ CRUSH rule 4 x 433 [6,5]
+ CRUSH rule 4 x 434 [5,2]
+ CRUSH rule 4 x 435 [0,5]
+ CRUSH rule 4 x 436 [4,0]
+ CRUSH rule 4 x 437 [7,5]
+ CRUSH rule 4 x 438 [0,3]
+ CRUSH rule 4 x 439 [1,3]
+ CRUSH rule 4 x 440 [2,7]
+ CRUSH rule 4 x 441 [5,7]
+ CRUSH rule 4 x 442 [2,4]
+ CRUSH rule 4 x 443 [6,8]
+ CRUSH rule 4 x 444 [7,0]
+ CRUSH rule 4 x 445 [6,3]
+ CRUSH rule 4 x 446 [4,3]
+ CRUSH rule 4 x 447 [2,1]
+ CRUSH rule 4 x 448 [7,2]
+ CRUSH rule 4 x 449 [7,8]
+ CRUSH rule 4 x 450 [4,5]
+ CRUSH rule 4 x 451 [6,8]
+ CRUSH rule 4 x 452 [8,3]
+ CRUSH rule 4 x 453 [6,8]
+ CRUSH rule 4 x 454 [6,7]
+ CRUSH rule 4 x 455 [2,7]
+ CRUSH rule 4 x 456 [6,8]
+ CRUSH rule 4 x 457 [7,2]
+ CRUSH rule 4 x 458 [2,8]
+ CRUSH rule 4 x 459 [2,0]
+ CRUSH rule 4 x 460 [6,5]
+ CRUSH rule 4 x 461 [6,5]
+ CRUSH rule 4 x 462 [8,1]
+ CRUSH rule 4 x 463 [6,7]
+ CRUSH rule 4 x 464 [7,4]
+ CRUSH rule 4 x 465 [7,6]
+ CRUSH rule 4 x 466 [5,8]
+ CRUSH rule 4 x 467 [6,4]
+ CRUSH rule 4 x 468 [7,8]
+ CRUSH rule 4 x 469 [7,0]
+ CRUSH rule 4 x 470 [3,0]
+ CRUSH rule 4 x 471 [0,1]
+ CRUSH rule 4 x 472 [5,4]
+ CRUSH rule 4 x 473 [1,0]
+ CRUSH rule 4 x 474 [6,0]
+ CRUSH rule 4 x 475 [6,7]
+ CRUSH rule 4 x 476 [4,3]
+ CRUSH rule 4 x 477 [5,8]
+ CRUSH rule 4 x 478 [6,7]
+ CRUSH rule 4 x 479 [0,5]
+ CRUSH rule 4 x 480 [1,8]
+ CRUSH rule 4 x 481 [2,4]
+ CRUSH rule 4 x 482 [4,3]
+ CRUSH rule 4 x 483 [0,2]
+ CRUSH rule 4 x 484 [1,2]
+ CRUSH rule 4 x 485 [4,7]
+ CRUSH rule 4 x 486 [4,1]
+ CRUSH rule 4 x 487 [5,0]
+ CRUSH rule 4 x 488 [5,7]
+ CRUSH rule 4 x 489 [2,8]
+ CRUSH rule 4 x 490 [6,4]
+ CRUSH rule 4 x 491 [1,0]
+ CRUSH rule 4 x 492 [6,5]
+ CRUSH rule 4 x 493 [0,2]
+ CRUSH rule 4 x 494 [1,0]
+ CRUSH rule 4 x 495 [3,4]
+ CRUSH rule 4 x 496 [7,5]
+ CRUSH rule 4 x 497 [5,7]
+ CRUSH rule 4 x 498 [0,5]
+ CRUSH rule 4 x 499 [8,4]
+ CRUSH rule 4 x 500 [3,6]
+ CRUSH rule 4 x 501 [0,7]
+ CRUSH rule 4 x 502 [7,1]
+ CRUSH rule 4 x 503 [2,3]
+ CRUSH rule 4 x 504 [5,6]
+ CRUSH rule 4 x 505 [0,7]
+ CRUSH rule 4 x 506 [5,3]
+ CRUSH rule 4 x 507 [6,0]
+ CRUSH rule 4 x 508 [0,1]
+ CRUSH rule 4 x 509 [7,5]
+ CRUSH rule 4 x 510 [6,0]
+ CRUSH rule 4 x 511 [5,8]
+ CRUSH rule 4 x 512 [7,6]
+ CRUSH rule 4 x 513 [7,2]
+ CRUSH rule 4 x 514 [4,3]
+ CRUSH rule 4 x 515 [8,5]
+ CRUSH rule 4 x 516 [4,0]
+ CRUSH rule 4 x 517 [7,8]
+ CRUSH rule 4 x 518 [4,6]
+ CRUSH rule 4 x 519 [7,3]
+ CRUSH rule 4 x 520 [2,6]
+ CRUSH rule 4 x 521 [8,7]
+ CRUSH rule 4 x 522 [6,8]
+ CRUSH rule 4 x 523 [4,2]
+ CRUSH rule 4 x 524 [0,4]
+ CRUSH rule 4 x 525 [0,4]
+ CRUSH rule 4 x 526 [1,5]
+ CRUSH rule 4 x 527 [0,2]
+ CRUSH rule 4 x 528 [5,3]
+ CRUSH rule 4 x 529 [5,7]
+ CRUSH rule 4 x 530 [6,7]
+ CRUSH rule 4 x 531 [6,1]
+ CRUSH rule 4 x 532 [6,3]
+ CRUSH rule 4 x 533 [5,6]
+ CRUSH rule 4 x 534 [7,3]
+ CRUSH rule 4 x 535 [8,6]
+ CRUSH rule 4 x 536 [6,7]
+ CRUSH rule 4 x 537 [3,7]
+ CRUSH rule 4 x 538 [6,8]
+ CRUSH rule 4 x 539 [8,3]
+ CRUSH rule 4 x 540 [0,6]
+ CRUSH rule 4 x 541 [2,3]
+ CRUSH rule 4 x 542 [3,5]
+ CRUSH rule 4 x 543 [6,0]
+ CRUSH rule 4 x 544 [3,7]
+ CRUSH rule 4 x 545 [5,7]
+ CRUSH rule 4 x 546 [6,1]
+ CRUSH rule 4 x 547 [8,2]
+ CRUSH rule 4 x 548 [5,2]
+ CRUSH rule 4 x 549 [5,8]
+ CRUSH rule 4 x 550 [0,5]
+ CRUSH rule 4 x 551 [7,5]
+ CRUSH rule 4 x 552 [5,4]
+ CRUSH rule 4 x 553 [4,2]
+ CRUSH rule 4 x 554 [0,8]
+ CRUSH rule 4 x 555 [5,0]
+ CRUSH rule 4 x 556 [3,4]
+ CRUSH rule 4 x 557 [7,4]
+ CRUSH rule 4 x 558 [3,1]
+ CRUSH rule 4 x 559 [4,2]
+ CRUSH rule 4 x 560 [8,3]
+ CRUSH rule 4 x 561 [6,3]
+ CRUSH rule 4 x 562 [3,4]
+ CRUSH rule 4 x 563 [2,6]
+ CRUSH rule 4 x 564 [5,1]
+ CRUSH rule 4 x 565 [3,6]
+ CRUSH rule 4 x 566 [4,7]
+ CRUSH rule 4 x 567 [3,6]
+ CRUSH rule 4 x 568 [7,4]
+ CRUSH rule 4 x 569 [3,4]
+ CRUSH rule 4 x 570 [1,5]
+ CRUSH rule 4 x 571 [3,7]
+ CRUSH rule 4 x 572 [3,4]
+ CRUSH rule 4 x 573 [3,0]
+ CRUSH rule 4 x 574 [2,0]
+ CRUSH rule 4 x 575 [8,6]
+ CRUSH rule 4 x 576 [4,6]
+ CRUSH rule 4 x 577 [8,2]
+ CRUSH rule 4 x 578 [6,8]
+ CRUSH rule 4 x 579 [3,1]
+ CRUSH rule 4 x 580 [3,0]
+ CRUSH rule 4 x 581 [7,2]
+ CRUSH rule 4 x 582 [2,8]
+ CRUSH rule 4 x 583 [6,0]
+ CRUSH rule 4 x 584 [8,1]
+ CRUSH rule 4 x 585 [7,0]
+ CRUSH rule 4 x 586 [0,1]
+ CRUSH rule 4 x 587 [2,5]
+ CRUSH rule 4 x 588 [3,4]
+ CRUSH rule 4 x 589 [7,1]
+ CRUSH rule 4 x 590 [6,2]
+ CRUSH rule 4 x 591 [5,2]
+ CRUSH rule 4 x 592 [2,0]
+ CRUSH rule 4 x 593 [0,8]
+ CRUSH rule 4 x 594 [0,7]
+ CRUSH rule 4 x 595 [7,1]
+ CRUSH rule 4 x 596 [4,3]
+ CRUSH rule 4 x 597 [3,1]
+ CRUSH rule 4 x 598 [3,2]
+ CRUSH rule 4 x 599 [5,2]
+ CRUSH rule 4 x 600 [7,0]
+ CRUSH rule 4 x 601 [0,7]
+ CRUSH rule 4 x 602 [3,7]
+ CRUSH rule 4 x 603 [5,1]
+ CRUSH rule 4 x 604 [7,5]
+ CRUSH rule 4 x 605 [3,0]
+ CRUSH rule 4 x 606 [2,0]
+ CRUSH rule 4 x 607 [0,4]
+ CRUSH rule 4 x 608 [5,3]
+ CRUSH rule 4 x 609 [5,2]
+ CRUSH rule 4 x 610 [3,7]
+ CRUSH rule 4 x 611 [1,0]
+ CRUSH rule 4 x 612 [2,0]
+ CRUSH rule 4 x 613 [7,2]
+ CRUSH rule 4 x 614 [7,8]
+ CRUSH rule 4 x 615 [6,8]
+ CRUSH rule 4 x 616 [0,8]
+ CRUSH rule 4 x 617 [6,1]
+ CRUSH rule 4 x 618 [7,6]
+ CRUSH rule 4 x 619 [5,1]
+ CRUSH rule 4 x 620 [4,1]
+ CRUSH rule 4 x 621 [5,8]
+ CRUSH rule 4 x 622 [0,4]
+ CRUSH rule 4 x 623 [0,6]
+ CRUSH rule 4 x 624 [3,5]
+ CRUSH rule 4 x 625 [2,3]
+ CRUSH rule 4 x 626 [7,8]
+ CRUSH rule 4 x 627 [2,7]
+ CRUSH rule 4 x 628 [8,0]
+ CRUSH rule 4 x 629 [2,6]
+ CRUSH rule 4 x 630 [2,7]
+ CRUSH rule 4 x 631 [0,7]
+ CRUSH rule 4 x 632 [7,0]
+ CRUSH rule 4 x 633 [8,6]
+ CRUSH rule 4 x 634 [0,4]
+ CRUSH rule 4 x 635 [5,6]
+ CRUSH rule 4 x 636 [1,4]
+ CRUSH rule 4 x 637 [4,1]
+ CRUSH rule 4 x 638 [6,8]
+ CRUSH rule 4 x 639 [4,2]
+ CRUSH rule 4 x 640 [3,1]
+ CRUSH rule 4 x 641 [7,2]
+ CRUSH rule 4 x 642 [2,0]
+ CRUSH rule 4 x 643 [3,5]
+ CRUSH rule 4 x 644 [8,1]
+ CRUSH rule 4 x 645 [5,4]
+ CRUSH rule 4 x 646 [8,0]
+ CRUSH rule 4 x 647 [7,1]
+ CRUSH rule 4 x 648 [0,6]
+ CRUSH rule 4 x 649 [4,7]
+ CRUSH rule 4 x 650 [7,8]
+ CRUSH rule 4 x 651 [3,7]
+ CRUSH rule 4 x 652 [3,4]
+ CRUSH rule 4 x 653 [8,5]
+ CRUSH rule 4 x 654 [7,5]
+ CRUSH rule 4 x 655 [0,3]
+ CRUSH rule 4 x 656 [4,3]
+ CRUSH rule 4 x 657 [6,1]
+ CRUSH rule 4 x 658 [5,4]
+ CRUSH rule 4 x 659 [4,6]
+ CRUSH rule 4 x 660 [7,8]
+ CRUSH rule 4 x 661 [1,8]
+ CRUSH rule 4 x 662 [4,5]
+ CRUSH rule 4 x 663 [1,3]
+ CRUSH rule 4 x 664 [1,4]
+ CRUSH rule 4 x 665 [5,7]
+ CRUSH rule 4 x 666 [2,8]
+ CRUSH rule 4 x 667 [1,3]
+ CRUSH rule 4 x 668 [3,7]
+ CRUSH rule 4 x 669 [6,4]
+ CRUSH rule 4 x 670 [4,0]
+ CRUSH rule 4 x 671 [0,2]
+ CRUSH rule 4 x 672 [4,3]
+ CRUSH rule 4 x 673 [5,3]
+ CRUSH rule 4 x 674 [3,1]
+ CRUSH rule 4 x 675 [0,8]
+ CRUSH rule 4 x 676 [0,2]
+ CRUSH rule 4 x 677 [4,1]
+ CRUSH rule 4 x 678 [2,3]
+ CRUSH rule 4 x 679 [6,0]
+ CRUSH rule 4 x 680 [0,4]
+ CRUSH rule 4 x 681 [4,7]
+ CRUSH rule 4 x 682 [0,5]
+ CRUSH rule 4 x 683 [0,1]
+ CRUSH rule 4 x 684 [7,1]
+ CRUSH rule 4 x 685 [7,1]
+ CRUSH rule 4 x 686 [1,4]
+ CRUSH rule 4 x 687 [3,5]
+ CRUSH rule 4 x 688 [5,7]
+ CRUSH rule 4 x 689 [6,5]
+ CRUSH rule 4 x 690 [8,1]
+ CRUSH rule 4 x 691 [3,0]
+ CRUSH rule 4 x 692 [7,2]
+ CRUSH rule 4 x 693 [6,7]
+ CRUSH rule 4 x 694 [6,5]
+ CRUSH rule 4 x 695 [0,8]
+ CRUSH rule 4 x 696 [1,4]
+ CRUSH rule 4 x 697 [6,1]
+ CRUSH rule 4 x 698 [6,2]
+ CRUSH rule 4 x 699 [1,6]
+ CRUSH rule 4 x 700 [0,3]
+ CRUSH rule 4 x 701 [4,3]
+ CRUSH rule 4 x 702 [3,5]
+ CRUSH rule 4 x 703 [8,3]
+ CRUSH rule 4 x 704 [0,3]
+ CRUSH rule 4 x 705 [8,6]
+ CRUSH rule 4 x 706 [1,2]
+ CRUSH rule 4 x 707 [7,8]
+ CRUSH rule 4 x 708 [3,5]
+ CRUSH rule 4 x 709 [6,3]
+ CRUSH rule 4 x 710 [8,4]
+ CRUSH rule 4 x 711 [2,3]
+ CRUSH rule 4 x 712 [2,3]
+ CRUSH rule 4 x 713 [6,7]
+ CRUSH rule 4 x 714 [3,2]
+ CRUSH rule 4 x 715 [1,2]
+ CRUSH rule 4 x 716 [3,6]
+ CRUSH rule 4 x 717 [8,7]
+ CRUSH rule 4 x 718 [3,7]
+ CRUSH rule 4 x 719 [2,6]
+ CRUSH rule 4 x 720 [6,8]
+ CRUSH rule 4 x 721 [5,4]
+ CRUSH rule 4 x 722 [5,4]
+ CRUSH rule 4 x 723 [5,1]
+ CRUSH rule 4 x 724 [0,6]
+ CRUSH rule 4 x 725 [0,1]
+ CRUSH rule 4 x 726 [3,8]
+ CRUSH rule 4 x 727 [4,6]
+ CRUSH rule 4 x 728 [2,1]
+ CRUSH rule 4 x 729 [5,3]
+ CRUSH rule 4 x 730 [3,7]
+ CRUSH rule 4 x 731 [4,1]
+ CRUSH rule 4 x 732 [1,5]
+ CRUSH rule 4 x 733 [5,4]
+ CRUSH rule 4 x 734 [6,4]
+ CRUSH rule 4 x 735 [4,8]
+ CRUSH rule 4 x 736 [3,5]
+ CRUSH rule 4 x 737 [1,0]
+ CRUSH rule 4 x 738 [5,2]
+ CRUSH rule 4 x 739 [0,1]
+ CRUSH rule 4 x 740 [0,1]
+ CRUSH rule 4 x 741 [7,8]
+ CRUSH rule 4 x 742 [8,2]
+ CRUSH rule 4 x 743 [7,0]
+ CRUSH rule 4 x 744 [4,7]
+ CRUSH rule 4 x 745 [3,4]
+ CRUSH rule 4 x 746 [4,1]
+ CRUSH rule 4 x 747 [6,0]
+ CRUSH rule 4 x 748 [2,7]
+ CRUSH rule 4 x 749 [4,5]
+ CRUSH rule 4 x 750 [1,6]
+ CRUSH rule 4 x 751 [2,1]
+ CRUSH rule 4 x 752 [8,1]
+ CRUSH rule 4 x 753 [7,8]
+ CRUSH rule 4 x 754 [8,6]
+ CRUSH rule 4 x 755 [1,2]
+ CRUSH rule 4 x 756 [5,6]
+ CRUSH rule 4 x 757 [8,6]
+ CRUSH rule 4 x 758 [6,0]
+ CRUSH rule 4 x 759 [8,5]
+ CRUSH rule 4 x 760 [1,5]
+ CRUSH rule 4 x 761 [4,1]
+ CRUSH rule 4 x 762 [2,7]
+ CRUSH rule 4 x 763 [8,6]
+ CRUSH rule 4 x 764 [1,7]
+ CRUSH rule 4 x 765 [6,5]
+ CRUSH rule 4 x 766 [8,5]
+ CRUSH rule 4 x 767 [1,2]
+ CRUSH rule 4 x 768 [8,3]
+ CRUSH rule 4 x 769 [6,2]
+ CRUSH rule 4 x 770 [6,0]
+ CRUSH rule 4 x 771 [7,0]
+ CRUSH rule 4 x 772 [8,3]
+ CRUSH rule 4 x 773 [3,1]
+ CRUSH rule 4 x 774 [4,6]
+ CRUSH rule 4 x 775 [6,8]
+ CRUSH rule 4 x 776 [7,2]
+ CRUSH rule 4 x 777 [3,1]
+ CRUSH rule 4 x 778 [1,8]
+ CRUSH rule 4 x 779 [2,7]
+ CRUSH rule 4 x 780 [0,2]
+ CRUSH rule 4 x 781 [6,3]
+ CRUSH rule 4 x 782 [5,4]
+ CRUSH rule 4 x 783 [7,1]
+ CRUSH rule 4 x 784 [0,1]
+ CRUSH rule 4 x 785 [6,1]
+ CRUSH rule 4 x 786 [7,6]
+ CRUSH rule 4 x 787 [1,0]
+ CRUSH rule 4 x 788 [6,0]
+ CRUSH rule 4 x 789 [0,4]
+ CRUSH rule 4 x 790 [8,4]
+ CRUSH rule 4 x 791 [3,8]
+ CRUSH rule 4 x 792 [5,8]
+ CRUSH rule 4 x 793 [6,1]
+ CRUSH rule 4 x 794 [2,6]
+ CRUSH rule 4 x 795 [0,3]
+ CRUSH rule 4 x 796 [3,4]
+ CRUSH rule 4 x 797 [2,3]
+ CRUSH rule 4 x 798 [6,8]
+ CRUSH rule 4 x 799 [5,1]
+ CRUSH rule 4 x 800 [5,0]
+ CRUSH rule 4 x 801 [3,6]
+ CRUSH rule 4 x 802 [1,8]
+ CRUSH rule 4 x 803 [0,5]
+ CRUSH rule 4 x 804 [6,2]
+ CRUSH rule 4 x 805 [3,6]
+ CRUSH rule 4 x 806 [1,3]
+ CRUSH rule 4 x 807 [5,4]
+ CRUSH rule 4 x 808 [4,6]
+ CRUSH rule 4 x 809 [1,4]
+ CRUSH rule 4 x 810 [5,7]
+ CRUSH rule 4 x 811 [8,4]
+ CRUSH rule 4 x 812 [8,5]
+ CRUSH rule 4 x 813 [6,4]
+ CRUSH rule 4 x 814 [3,6]
+ CRUSH rule 4 x 815 [3,1]
+ CRUSH rule 4 x 816 [2,1]
+ CRUSH rule 4 x 817 [4,3]
+ CRUSH rule 4 x 818 [3,5]
+ CRUSH rule 4 x 819 [5,1]
+ CRUSH rule 4 x 820 [3,5]
+ CRUSH rule 4 x 821 [4,5]
+ CRUSH rule 4 x 822 [2,0]
+ CRUSH rule 4 x 823 [4,8]
+ CRUSH rule 4 x 824 [3,7]
+ CRUSH rule 4 x 825 [2,8]
+ CRUSH rule 4 x 826 [7,0]
+ CRUSH rule 4 x 827 [0,8]
+ CRUSH rule 4 x 828 [2,3]
+ CRUSH rule 4 x 829 [5,6]
+ CRUSH rule 4 x 830 [2,3]
+ CRUSH rule 4 x 831 [1,6]
+ CRUSH rule 4 x 832 [4,5]
+ CRUSH rule 4 x 833 [2,1]
+ CRUSH rule 4 x 834 [3,4]
+ CRUSH rule 4 x 835 [8,4]
+ CRUSH rule 4 x 836 [3,4]
+ CRUSH rule 4 x 837 [6,3]
+ CRUSH rule 4 x 838 [6,7]
+ CRUSH rule 4 x 839 [5,0]
+ CRUSH rule 4 x 840 [7,8]
+ CRUSH rule 4 x 841 [4,8]
+ CRUSH rule 4 x 842 [2,4]
+ CRUSH rule 4 x 843 [6,4]
+ CRUSH rule 4 x 844 [4,8]
+ CRUSH rule 4 x 845 [3,8]
+ CRUSH rule 4 x 846 [3,2]
+ CRUSH rule 4 x 847 [0,2]
+ CRUSH rule 4 x 848 [2,6]
+ CRUSH rule 4 x 849 [4,5]
+ CRUSH rule 4 x 850 [1,0]
+ CRUSH rule 4 x 851 [6,8]
+ CRUSH rule 4 x 852 [7,3]
+ CRUSH rule 4 x 853 [6,8]
+ CRUSH rule 4 x 854 [7,6]
+ CRUSH rule 4 x 855 [5,7]
+ CRUSH rule 4 x 856 [6,7]
+ CRUSH rule 4 x 857 [8,5]
+ CRUSH rule 4 x 858 [6,4]
+ CRUSH rule 4 x 859 [6,0]
+ CRUSH rule 4 x 860 [4,1]
+ CRUSH rule 4 x 861 [8,7]
+ CRUSH rule 4 x 862 [6,1]
+ CRUSH rule 4 x 863 [8,7]
+ CRUSH rule 4 x 864 [5,6]
+ CRUSH rule 4 x 865 [8,1]
+ CRUSH rule 4 x 866 [3,4]
+ CRUSH rule 4 x 867 [6,5]
+ CRUSH rule 4 x 868 [6,3]
+ CRUSH rule 4 x 869 [8,7]
+ CRUSH rule 4 x 870 [0,4]
+ CRUSH rule 4 x 871 [3,4]
+ CRUSH rule 4 x 872 [5,1]
+ CRUSH rule 4 x 873 [4,6]
+ CRUSH rule 4 x 874 [2,6]
+ CRUSH rule 4 x 875 [2,6]
+ CRUSH rule 4 x 876 [5,8]
+ CRUSH rule 4 x 877 [6,4]
+ CRUSH rule 4 x 878 [5,4]
+ CRUSH rule 4 x 879 [7,4]
+ CRUSH rule 4 x 880 [3,5]
+ CRUSH rule 4 x 881 [5,6]
+ CRUSH rule 4 x 882 [4,0]
+ CRUSH rule 4 x 883 [2,1]
+ CRUSH rule 4 x 884 [6,0]
+ CRUSH rule 4 x 885 [5,1]
+ CRUSH rule 4 x 886 [3,6]
+ CRUSH rule 4 x 887 [7,4]
+ CRUSH rule 4 x 888 [6,8]
+ CRUSH rule 4 x 889 [2,1]
+ CRUSH rule 4 x 890 [7,2]
+ CRUSH rule 4 x 891 [1,8]
+ CRUSH rule 4 x 892 [6,2]
+ CRUSH rule 4 x 893 [2,3]
+ CRUSH rule 4 x 894 [7,5]
+ CRUSH rule 4 x 895 [5,3]
+ CRUSH rule 4 x 896 [1,8]
+ CRUSH rule 4 x 897 [4,2]
+ CRUSH rule 4 x 898 [0,5]
+ CRUSH rule 4 x 899 [1,7]
+ CRUSH rule 4 x 900 [4,1]
+ CRUSH rule 4 x 901 [5,0]
+ CRUSH rule 4 x 902 [8,5]
+ CRUSH rule 4 x 903 [5,7]
+ CRUSH rule 4 x 904 [5,6]
+ CRUSH rule 4 x 905 [6,2]
+ CRUSH rule 4 x 906 [1,2]
+ CRUSH rule 4 x 907 [7,1]
+ CRUSH rule 4 x 908 [5,8]
+ CRUSH rule 4 x 909 [2,3]
+ CRUSH rule 4 x 910 [6,4]
+ CRUSH rule 4 x 911 [5,8]
+ CRUSH rule 4 x 912 [0,1]
+ CRUSH rule 4 x 913 [7,6]
+ CRUSH rule 4 x 914 [6,4]
+ CRUSH rule 4 x 915 [8,2]
+ CRUSH rule 4 x 916 [3,1]
+ CRUSH rule 4 x 917 [1,5]
+ CRUSH rule 4 x 918 [8,2]
+ CRUSH rule 4 x 919 [6,2]
+ CRUSH rule 4 x 920 [7,6]
+ CRUSH rule 4 x 921 [1,4]
+ CRUSH rule 4 x 922 [6,7]
+ CRUSH rule 4 x 923 [5,3]
+ CRUSH rule 4 x 924 [3,5]
+ CRUSH rule 4 x 925 [5,7]
+ CRUSH rule 4 x 926 [3,4]
+ CRUSH rule 4 x 927 [1,6]
+ CRUSH rule 4 x 928 [8,1]
+ CRUSH rule 4 x 929 [4,5]
+ CRUSH rule 4 x 930 [2,4]
+ CRUSH rule 4 x 931 [5,0]
+ CRUSH rule 4 x 932 [4,3]
+ CRUSH rule 4 x 933 [8,5]
+ CRUSH rule 4 x 934 [5,3]
+ CRUSH rule 4 x 935 [6,3]
+ CRUSH rule 4 x 936 [0,6]
+ CRUSH rule 4 x 937 [5,4]
+ CRUSH rule 4 x 938 [6,5]
+ CRUSH rule 4 x 939 [2,7]
+ CRUSH rule 4 x 940 [8,7]
+ CRUSH rule 4 x 941 [5,2]
+ CRUSH rule 4 x 942 [1,0]
+ CRUSH rule 4 x 943 [8,2]
+ CRUSH rule 4 x 944 [4,3]
+ CRUSH rule 4 x 945 [7,2]
+ CRUSH rule 4 x 946 [2,0]
+ CRUSH rule 4 x 947 [4,5]
+ CRUSH rule 4 x 948 [7,8]
+ CRUSH rule 4 x 949 [6,1]
+ CRUSH rule 4 x 950 [3,5]
+ CRUSH rule 4 x 951 [4,5]
+ CRUSH rule 4 x 952 [2,0]
+ CRUSH rule 4 x 953 [1,3]
+ CRUSH rule 4 x 954 [4,2]
+ CRUSH rule 4 x 955 [8,6]
+ CRUSH rule 4 x 956 [1,0]
+ CRUSH rule 4 x 957 [7,6]
+ CRUSH rule 4 x 958 [8,7]
+ CRUSH rule 4 x 959 [5,2]
+ CRUSH rule 4 x 960 [3,6]
+ CRUSH rule 4 x 961 [4,0]
+ CRUSH rule 4 x 962 [7,4]
+ CRUSH rule 4 x 963 [0,5]
+ CRUSH rule 4 x 964 [3,1]
+ CRUSH rule 4 x 965 [7,6]
+ CRUSH rule 4 x 966 [3,8]
+ CRUSH rule 4 x 967 [8,6]
+ CRUSH rule 4 x 968 [7,2]
+ CRUSH rule 4 x 969 [8,0]
+ CRUSH rule 4 x 970 [0,6]
+ CRUSH rule 4 x 971 [1,7]
+ CRUSH rule 4 x 972 [1,8]
+ CRUSH rule 4 x 973 [1,2]
+ CRUSH rule 4 x 974 [5,3]
+ CRUSH rule 4 x 975 [3,7]
+ CRUSH rule 4 x 976 [4,3]
+ CRUSH rule 4 x 977 [8,3]
+ CRUSH rule 4 x 978 [7,2]
+ CRUSH rule 4 x 979 [7,6]
+ CRUSH rule 4 x 980 [6,0]
+ CRUSH rule 4 x 981 [7,3]
+ CRUSH rule 4 x 982 [4,2]
+ CRUSH rule 4 x 983 [3,5]
+ CRUSH rule 4 x 984 [0,2]
+ CRUSH rule 4 x 985 [2,5]
+ CRUSH rule 4 x 986 [8,7]
+ CRUSH rule 4 x 987 [0,5]
+ CRUSH rule 4 x 988 [1,3]
+ CRUSH rule 4 x 989 [0,6]
+ CRUSH rule 4 x 990 [1,0]
+ CRUSH rule 4 x 991 [0,4]
+ CRUSH rule 4 x 992 [7,1]
+ CRUSH rule 4 x 993 [0,6]
+ CRUSH rule 4 x 994 [3,4]
+ CRUSH rule 4 x 995 [7,6]
+ CRUSH rule 4 x 996 [6,7]
+ CRUSH rule 4 x 997 [6,4]
+ CRUSH rule 4 x 998 [8,1]
+ CRUSH rule 4 x 999 [0,7]
+ CRUSH rule 4 x 1000 [8,5]
+ CRUSH rule 4 x 1001 [2,0]
+ CRUSH rule 4 x 1002 [1,3]
+ CRUSH rule 4 x 1003 [2,8]
+ CRUSH rule 4 x 1004 [6,1]
+ CRUSH rule 4 x 1005 [6,1]
+ CRUSH rule 4 x 1006 [1,0]
+ CRUSH rule 4 x 1007 [1,2]
+ CRUSH rule 4 x 1008 [1,7]
+ CRUSH rule 4 x 1009 [6,8]
+ CRUSH rule 4 x 1010 [3,4]
+ CRUSH rule 4 x 1011 [3,0]
+ CRUSH rule 4 x 1012 [3,0]
+ CRUSH rule 4 x 1013 [5,1]
+ CRUSH rule 4 x 1014 [2,8]
+ CRUSH rule 4 x 1015 [6,8]
+ CRUSH rule 4 x 1016 [2,0]
+ CRUSH rule 4 x 1017 [6,0]
+ CRUSH rule 4 x 1018 [5,4]
+ CRUSH rule 4 x 1019 [5,3]
+ CRUSH rule 4 x 1020 [5,1]
+ CRUSH rule 4 x 1021 [5,2]
+ CRUSH rule 4 x 1022 [1,6]
+ CRUSH rule 4 x 1023 [3,2]
+ rule 4 (choose-set-two) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 4 x 0 [0,2,3]
+ CRUSH rule 4 x 1 [0,8,2]
+ CRUSH rule 4 x 2 [1,3,5]
+ CRUSH rule 4 x 3 [8,0,6]
+ CRUSH rule 4 x 4 [5,4,0]
+ CRUSH rule 4 x 5 [7,8,0]
+ CRUSH rule 4 x 6 [2,6,7]
+ CRUSH rule 4 x 7 [5,4,3]
+ CRUSH rule 4 x 8 [5,3,8]
+ CRUSH rule 4 x 9 [2,3,1]
+ CRUSH rule 4 x 10 [0,2,8]
+ CRUSH rule 4 x 11 [0,7,2]
+ CRUSH rule 4 x 12 [0,2,4]
+ CRUSH rule 4 x 13 [3,8,5]
+ CRUSH rule 4 x 14 [7,6,8]
+ CRUSH rule 4 x 15 [7,2,4]
+ CRUSH rule 4 x 16 [3,6,0]
+ CRUSH rule 4 x 17 [5,4,0]
+ CRUSH rule 4 x 18 [1,4,5]
+ CRUSH rule 4 x 19 [7,5,4]
+ CRUSH rule 4 x 20 [2,4,7]
+ CRUSH rule 4 x 21 [3,7,2]
+ CRUSH rule 4 x 22 [8,3,7]
+ CRUSH rule 4 x 23 [3,6,8]
+ CRUSH rule 4 x 24 [1,0,6]
+ CRUSH rule 4 x 25 [3,7,0]
+ CRUSH rule 4 x 26 [2,8,7]
+ CRUSH rule 4 x 27 [3,1,5]
+ CRUSH rule 4 x 28 [6,0,3]
+ CRUSH rule 4 x 29 [8,5,7]
+ CRUSH rule 4 x 30 [5,7,3]
+ CRUSH rule 4 x 31 [8,7,6]
+ CRUSH rule 4 x 32 [3,6,8]
+ CRUSH rule 4 x 33 [2,7,8]
+ CRUSH rule 4 x 34 [2,5,4]
+ CRUSH rule 4 x 35 [0,8,1]
+ CRUSH rule 4 x 36 [3,8,2]
+ CRUSH rule 4 x 37 [0,4,5]
+ CRUSH rule 4 x 38 [4,8,7]
+ CRUSH rule 4 x 39 [3,7,0]
+ CRUSH rule 4 x 40 [7,8,1]
+ CRUSH rule 4 x 41 [0,2,6]
+ CRUSH rule 4 x 42 [4,3,6]
+ CRUSH rule 4 x 43 [0,3,7]
+ CRUSH rule 4 x 44 [1,6,7]
+ CRUSH rule 4 x 45 [8,0,4]
+ CRUSH rule 4 x 46 [2,4,3]
+ CRUSH rule 4 x 47 [4,5,3]
+ CRUSH rule 4 x 48 [4,6,5]
+ CRUSH rule 4 x 49 [5,4,8]
+ CRUSH rule 4 x 50 [3,4,1]
+ CRUSH rule 4 x 51 [3,6,0]
+ CRUSH rule 4 x 52 [8,6,1]
+ CRUSH rule 4 x 53 [3,5,4]
+ CRUSH rule 4 x 54 [7,6,8]
+ CRUSH rule 4 x 55 [8,7,0]
+ CRUSH rule 4 x 56 [6,4,8]
+ CRUSH rule 4 x 57 [5,3,6]
+ CRUSH rule 4 x 58 [1,0,2]
+ CRUSH rule 4 x 59 [4,2,1]
+ CRUSH rule 4 x 60 [3,5,0]
+ CRUSH rule 4 x 61 [4,6,8]
+ CRUSH rule 4 x 62 [7,0,1]
+ CRUSH rule 4 x 63 [5,6,8]
+ CRUSH rule 4 x 64 [4,5,1]
+ CRUSH rule 4 x 65 [7,3,6]
+ CRUSH rule 4 x 66 [5,4,3]
+ CRUSH rule 4 x 67 [5,0,8]
+ CRUSH rule 4 x 68 [0,5,8]
+ CRUSH rule 4 x 69 [5,1,6]
+ CRUSH rule 4 x 70 [7,0,2]
+ CRUSH rule 4 x 71 [2,8,6]
+ CRUSH rule 4 x 72 [6,1,5]
+ CRUSH rule 4 x 73 [2,7,3]
+ CRUSH rule 4 x 74 [0,7,1]
+ CRUSH rule 4 x 75 [3,2,0]
+ CRUSH rule 4 x 76 [5,1,7]
+ CRUSH rule 4 x 77 [7,2,6]
+ CRUSH rule 4 x 78 [1,4,2]
+ CRUSH rule 4 x 79 [5,4,0]
+ CRUSH rule 4 x 80 [0,3,5]
+ CRUSH rule 4 x 81 [0,2,5]
+ CRUSH rule 4 x 82 [7,1,6]
+ CRUSH rule 4 x 83 [2,6,8]
+ CRUSH rule 4 x 84 [7,2,1]
+ CRUSH rule 4 x 85 [3,4,8]
+ CRUSH rule 4 x 86 [0,2,1]
+ CRUSH rule 4 x 87 [0,7,4]
+ CRUSH rule 4 x 88 [1,6,0]
+ CRUSH rule 4 x 89 [3,0,2]
+ CRUSH rule 4 x 90 [6,7,5]
+ CRUSH rule 4 x 91 [3,8,7]
+ CRUSH rule 4 x 92 [1,8,4]
+ CRUSH rule 4 x 93 [7,4,5]
+ CRUSH rule 4 x 94 [0,4,3]
+ CRUSH rule 4 x 95 [7,5,1]
+ CRUSH rule 4 x 96 [3,6,4]
+ CRUSH rule 4 x 97 [8,7,4]
+ CRUSH rule 4 x 98 [2,0,7]
+ CRUSH rule 4 x 99 [0,7,2]
+ CRUSH rule 4 x 100 [1,7,4]
+ CRUSH rule 4 x 101 [3,7,2]
+ CRUSH rule 4 x 102 [4,2,7]
+ CRUSH rule 4 x 103 [4,7,6]
+ CRUSH rule 4 x 104 [7,4,3]
+ CRUSH rule 4 x 105 [2,4,1]
+ CRUSH rule 4 x 106 [1,6,2]
+ CRUSH rule 4 x 107 [3,2,1]
+ CRUSH rule 4 x 108 [7,2,8]
+ CRUSH rule 4 x 109 [1,2,5]
+ CRUSH rule 4 x 110 [3,2,7]
+ CRUSH rule 4 x 111 [2,1,0]
+ CRUSH rule 4 x 112 [2,0,6]
+ CRUSH rule 4 x 113 [6,2,1]
+ CRUSH rule 4 x 114 [7,6,3]
+ CRUSH rule 4 x 115 [8,2,3]
+ CRUSH rule 4 x 116 [1,2,6]
+ CRUSH rule 4 x 117 [7,3,6]
+ CRUSH rule 4 x 118 [0,3,8]
+ CRUSH rule 4 x 119 [5,6,1]
+ CRUSH rule 4 x 120 [0,2,5]
+ CRUSH rule 4 x 121 [2,0,1]
+ CRUSH rule 4 x 122 [8,5,0]
+ CRUSH rule 4 x 123 [2,5,1]
+ CRUSH rule 4 x 124 [3,5,1]
+ CRUSH rule 4 x 125 [0,7,3]
+ CRUSH rule 4 x 126 [4,2,6]
+ CRUSH rule 4 x 127 [3,6,7]
+ CRUSH rule 4 x 128 [3,5,8]
+ CRUSH rule 4 x 129 [0,2,1]
+ CRUSH rule 4 x 130 [3,8,5]
+ CRUSH rule 4 x 131 [1,2,0]
+ CRUSH rule 4 x 132 [1,2,3]
+ CRUSH rule 4 x 133 [3,6,7]
+ CRUSH rule 4 x 134 [1,8,2]
+ CRUSH rule 4 x 135 [5,6,4]
+ CRUSH rule 4 x 136 [2,1,5]
+ CRUSH rule 4 x 137 [7,3,4]
+ CRUSH rule 4 x 138 [8,7,6]
+ CRUSH rule 4 x 139 [3,0,5]
+ CRUSH rule 4 x 140 [1,6,7]
+ CRUSH rule 4 x 141 [6,8,1]
+ CRUSH rule 4 x 142 [3,5,0]
+ CRUSH rule 4 x 143 [5,8,7]
+ CRUSH rule 4 x 144 [8,1,0]
+ CRUSH rule 4 x 145 [8,5,6]
+ CRUSH rule 4 x 146 [2,6,4]
+ CRUSH rule 4 x 147 [2,8,7]
+ CRUSH rule 4 x 148 [3,4,0]
+ CRUSH rule 4 x 149 [4,8,6]
+ CRUSH rule 4 x 150 [1,6,5]
+ CRUSH rule 4 x 151 [3,4,5]
+ CRUSH rule 4 x 152 [8,4,6]
+ CRUSH rule 4 x 153 [8,6,7]
+ CRUSH rule 4 x 154 [3,2,0]
+ CRUSH rule 4 x 155 [3,5,8]
+ CRUSH rule 4 x 156 [4,5,3]
+ CRUSH rule 4 x 157 [4,1,2]
+ CRUSH rule 4 x 158 [2,8,1]
+ CRUSH rule 4 x 159 [7,0,5]
+ CRUSH rule 4 x 160 [2,8,1]
+ CRUSH rule 4 x 161 [1,5,2]
+ CRUSH rule 4 x 162 [0,6,2]
+ CRUSH rule 4 x 163 [5,6,1]
+ CRUSH rule 4 x 164 [7,8,1]
+ CRUSH rule 4 x 165 [7,0,5]
+ CRUSH rule 4 x 166 [2,4,1]
+ CRUSH rule 4 x 167 [0,1,6]
+ CRUSH rule 4 x 168 [4,2,3]
+ CRUSH rule 4 x 169 [2,6,1]
+ CRUSH rule 4 x 170 [1,0,5]
+ CRUSH rule 4 x 171 [7,5,4]
+ CRUSH rule 4 x 172 [0,7,8]
+ CRUSH rule 4 x 173 [8,5,3]
+ CRUSH rule 4 x 174 [1,4,7]
+ CRUSH rule 4 x 175 [6,0,4]
+ CRUSH rule 4 x 176 [4,3,5]
+ CRUSH rule 4 x 177 [5,3,0]
+ CRUSH rule 4 x 178 [3,0,2]
+ CRUSH rule 4 x 179 [4,1,2]
+ CRUSH rule 4 x 180 [3,8,7]
+ CRUSH rule 4 x 181 [6,2,0]
+ CRUSH rule 4 x 182 [8,5,3]
+ CRUSH rule 4 x 183 [7,8,6]
+ CRUSH rule 4 x 184 [5,7,3]
+ CRUSH rule 4 x 185 [6,8,0]
+ CRUSH rule 4 x 186 [2,1,4]
+ CRUSH rule 4 x 187 [1,6,7]
+ CRUSH rule 4 x 188 [1,8,7]
+ CRUSH rule 4 x 189 [0,7,8]
+ CRUSH rule 4 x 190 [4,0,1]
+ CRUSH rule 4 x 191 [7,6,8]
+ CRUSH rule 4 x 192 [5,0,1]
+ CRUSH rule 4 x 193 [4,2,6]
+ CRUSH rule 4 x 194 [1,3,4]
+ CRUSH rule 4 x 195 [6,4,3]
+ CRUSH rule 4 x 196 [6,7,2]
+ CRUSH rule 4 x 197 [6,5,2]
+ CRUSH rule 4 x 198 [2,5,6]
+ CRUSH rule 4 x 199 [0,5,7]
+ CRUSH rule 4 x 200 [0,5,1]
+ CRUSH rule 4 x 201 [7,1,0]
+ CRUSH rule 4 x 202 [6,3,4]
+ CRUSH rule 4 x 203 [4,8,7]
+ CRUSH rule 4 x 204 [2,1,5]
+ CRUSH rule 4 x 205 [0,7,6]
+ CRUSH rule 4 x 206 [0,1,7]
+ CRUSH rule 4 x 207 [3,0,5]
+ CRUSH rule 4 x 208 [7,1,0]
+ CRUSH rule 4 x 209 [1,2,0]
+ CRUSH rule 4 x 210 [1,2,3]
+ CRUSH rule 4 x 211 [5,4,2]
+ CRUSH rule 4 x 212 [7,5,0]
+ CRUSH rule 4 x 213 [8,4,0]
+ CRUSH rule 4 x 214 [4,5,7]
+ CRUSH rule 4 x 215 [8,1,7]
+ CRUSH rule 4 x 216 [5,0,2]
+ CRUSH rule 4 x 217 [0,1,7]
+ CRUSH rule 4 x 218 [0,7,8]
+ CRUSH rule 4 x 219 [4,8,2]
+ CRUSH rule 4 x 220 [5,7,3]
+ CRUSH rule 4 x 221 [3,6,5]
+ CRUSH rule 4 x 222 [6,8,3]
+ CRUSH rule 4 x 223 [1,3,6]
+ CRUSH rule 4 x 224 [1,5,0]
+ CRUSH rule 4 x 225 [8,6,2]
+ CRUSH rule 4 x 226 [7,2,3]
+ CRUSH rule 4 x 227 [3,4,0]
+ CRUSH rule 4 x 228 [5,4,8]
+ CRUSH rule 4 x 229 [3,4,7]
+ CRUSH rule 4 x 230 [4,7,2]
+ CRUSH rule 4 x 231 [4,3,5]
+ CRUSH rule 4 x 232 [2,7,4]
+ CRUSH rule 4 x 233 [3,4,5]
+ CRUSH rule 4 x 234 [0,1,2]
+ CRUSH rule 4 x 235 [3,8,0]
+ CRUSH rule 4 x 236 [5,2,7]
+ CRUSH rule 4 x 237 [4,7,6]
+ CRUSH rule 4 x 238 [4,3,5]
+ CRUSH rule 4 x 239 [8,7,6]
+ CRUSH rule 4 x 240 [5,7,4]
+ CRUSH rule 4 x 241 [3,1,5]
+ CRUSH rule 4 x 242 [3,5,4]
+ CRUSH rule 4 x 243 [4,7,3]
+ CRUSH rule 4 x 244 [4,6,0]
+ CRUSH rule 4 x 245 [7,6,1]
+ CRUSH rule 4 x 246 [1,5,2]
+ CRUSH rule 4 x 247 [6,0,2]
+ CRUSH rule 4 x 248 [8,0,7]
+ CRUSH rule 4 x 249 [2,0,1]
+ CRUSH rule 4 x 250 [2,1,0]
+ CRUSH rule 4 x 251 [2,3,6]
+ CRUSH rule 4 x 252 [3,7,8]
+ CRUSH rule 4 x 253 [3,2,4]
+ CRUSH rule 4 x 254 [3,5,4]
+ CRUSH rule 4 x 255 [1,7,8]
+ CRUSH rule 4 x 256 [5,7,0]
+ CRUSH rule 4 x 257 [2,8,4]
+ CRUSH rule 4 x 258 [5,3,0]
+ CRUSH rule 4 x 259 [4,3,6]
+ CRUSH rule 4 x 260 [3,6,2]
+ CRUSH rule 4 x 261 [8,7,4]
+ CRUSH rule 4 x 262 [5,4,8]
+ CRUSH rule 4 x 263 [6,8,2]
+ CRUSH rule 4 x 264 [3,6,5]
+ CRUSH rule 4 x 265 [8,6,7]
+ CRUSH rule 4 x 266 [8,2,1]
+ CRUSH rule 4 x 267 [2,3,0]
+ CRUSH rule 4 x 268 [0,7,2]
+ CRUSH rule 4 x 269 [0,8,2]
+ CRUSH rule 4 x 270 [5,0,1]
+ CRUSH rule 4 x 271 [7,5,4]
+ CRUSH rule 4 x 272 [2,8,3]
+ CRUSH rule 4 x 273 [3,5,0]
+ CRUSH rule 4 x 274 [6,8,3]
+ CRUSH rule 4 x 275 [4,3,6]
+ CRUSH rule 4 x 276 [7,1,2]
+ CRUSH rule 4 x 277 [6,4,0]
+ CRUSH rule 4 x 278 [6,8,0]
+ CRUSH rule 4 x 279 [8,3,6]
+ CRUSH rule 4 x 280 [0,6,2]
+ CRUSH rule 4 x 281 [8,0,2]
+ CRUSH rule 4 x 282 [3,1,6]
+ CRUSH rule 4 x 283 [8,2,0]
+ CRUSH rule 4 x 284 [6,3,0]
+ CRUSH rule 4 x 285 [5,3,7]
+ CRUSH rule 4 x 286 [2,1,6]
+ CRUSH rule 4 x 287 [0,4,1]
+ CRUSH rule 4 x 288 [8,0,7]
+ CRUSH rule 4 x 289 [4,6,2]
+ CRUSH rule 4 x 290 [1,3,7]
+ CRUSH rule 4 x 291 [0,1,4]
+ CRUSH rule 4 x 292 [8,0,2]
+ CRUSH rule 4 x 293 [6,0,8]
+ CRUSH rule 4 x 294 [7,4,5]
+ CRUSH rule 4 x 295 [4,8,7]
+ CRUSH rule 4 x 296 [3,1,5]
+ CRUSH rule 4 x 297 [6,2,8]
+ CRUSH rule 4 x 298 [1,2,3]
+ CRUSH rule 4 x 299 [2,0,7]
+ CRUSH rule 4 x 300 [8,7,3]
+ CRUSH rule 4 x 301 [0,8,1]
+ CRUSH rule 4 x 302 [3,0,6]
+ CRUSH rule 4 x 303 [7,5,8]
+ CRUSH rule 4 x 304 [2,7,5]
+ CRUSH rule 4 x 305 [5,8,2]
+ CRUSH rule 4 x 306 [0,7,1]
+ CRUSH rule 4 x 307 [0,2,8]
+ CRUSH rule 4 x 308 [0,8,4]
+ CRUSH rule 4 x 309 [7,4,5]
+ CRUSH rule 4 x 310 [4,3,1]
+ CRUSH rule 4 x 311 [3,4,5]
+ CRUSH rule 4 x 312 [2,1,0]
+ CRUSH rule 4 x 313 [5,3,4]
+ CRUSH rule 4 x 314 [4,5,0]
+ CRUSH rule 4 x 315 [2,0,1]
+ CRUSH rule 4 x 316 [6,3,8]
+ CRUSH rule 4 x 317 [2,6,0]
+ CRUSH rule 4 x 318 [8,6,7]
+ CRUSH rule 4 x 319 [5,0,8]
+ CRUSH rule 4 x 320 [3,7,1]
+ CRUSH rule 4 x 321 [1,3,0]
+ CRUSH rule 4 x 322 [2,7,3]
+ CRUSH rule 4 x 323 [4,7,0]
+ CRUSH rule 4 x 324 [7,0,8]
+ CRUSH rule 4 x 325 [4,6,0]
+ CRUSH rule 4 x 326 [3,4,1]
+ CRUSH rule 4 x 327 [0,6,7]
+ CRUSH rule 4 x 328 [7,4,8]
+ CRUSH rule 4 x 329 [5,6,3]
+ CRUSH rule 4 x 330 [3,7,4]
+ CRUSH rule 4 x 331 [2,6,3]
+ CRUSH rule 4 x 332 [2,0,3]
+ CRUSH rule 4 x 333 [6,8,5]
+ CRUSH rule 4 x 334 [8,3,5]
+ CRUSH rule 4 x 335 [7,1,2]
+ CRUSH rule 4 x 336 [4,6,0]
+ CRUSH rule 4 x 337 [7,2,6]
+ CRUSH rule 4 x 338 [5,6,4]
+ CRUSH rule 4 x 339 [7,5,2]
+ CRUSH rule 4 x 340 [2,0,1]
+ CRUSH rule 4 x 341 [5,1,7]
+ CRUSH rule 4 x 342 [0,7,4]
+ CRUSH rule 4 x 343 [6,7,3]
+ CRUSH rule 4 x 344 [6,0,5]
+ CRUSH rule 4 x 345 [4,3,5]
+ CRUSH rule 4 x 346 [8,0,5]
+ CRUSH rule 4 x 347 [3,1,5]
+ CRUSH rule 4 x 348 [8,0,2]
+ CRUSH rule 4 x 349 [1,6,7]
+ CRUSH rule 4 x 350 [8,5,7]
+ CRUSH rule 4 x 351 [3,6,4]
+ CRUSH rule 4 x 352 [1,0,2]
+ CRUSH rule 4 x 353 [6,4,5]
+ CRUSH rule 4 x 354 [0,3,1]
+ CRUSH rule 4 x 355 [3,4,6]
+ CRUSH rule 4 x 356 [3,5,4]
+ CRUSH rule 4 x 357 [6,1,2]
+ CRUSH rule 4 x 358 [2,1,8]
+ CRUSH rule 4 x 359 [6,7,8]
+ CRUSH rule 4 x 360 [5,3,1]
+ CRUSH rule 4 x 361 [8,4,5]
+ CRUSH rule 4 x 362 [4,5,3]
+ CRUSH rule 4 x 363 [4,0,3]
+ CRUSH rule 4 x 364 [2,5,0]
+ CRUSH rule 4 x 365 [6,7,8]
+ CRUSH rule 4 x 366 [7,2,8]
+ CRUSH rule 4 x 367 [4,5,0]
+ CRUSH rule 4 x 368 [7,4,6]
+ CRUSH rule 4 x 369 [3,7,1]
+ CRUSH rule 4 x 370 [8,7,6]
+ CRUSH rule 4 x 371 [1,3,5]
+ CRUSH rule 4 x 372 [3,1,8]
+ CRUSH rule 4 x 373 [0,6,8]
+ CRUSH rule 4 x 374 [3,8,4]
+ CRUSH rule 4 x 375 [6,4,5]
+ CRUSH rule 4 x 376 [7,1,2]
+ CRUSH rule 4 x 377 [1,2,4]
+ CRUSH rule 4 x 378 [0,1,2]
+ CRUSH rule 4 x 379 [8,5,4]
+ CRUSH rule 4 x 380 [2,5,0]
+ CRUSH rule 4 x 381 [0,4,7]
+ CRUSH rule 4 x 382 [1,5,2]
+ CRUSH rule 4 x 383 [4,3,5]
+ CRUSH rule 4 x 384 [7,0,6]
+ CRUSH rule 4 x 385 [7,4,6]
+ CRUSH rule 4 x 386 [0,3,5]
+ CRUSH rule 4 x 387 [1,3,6]
+ CRUSH rule 4 x 388 [5,0,3]
+ CRUSH rule 4 x 389 [1,5,4]
+ CRUSH rule 4 x 390 [5,6,0]
+ CRUSH rule 4 x 391 [5,6,2]
+ CRUSH rule 4 x 392 [1,8,6]
+ CRUSH rule 4 x 393 [4,2,6]
+ CRUSH rule 4 x 394 [4,7,6]
+ CRUSH rule 4 x 395 [4,0,1]
+ CRUSH rule 4 x 396 [4,2,5]
+ CRUSH rule 4 x 397 [2,1,5]
+ CRUSH rule 4 x 398 [2,4,5]
+ CRUSH rule 4 x 399 [8,7,3]
+ CRUSH rule 4 x 400 [8,1,4]
+ CRUSH rule 4 x 401 [0,1,2]
+ CRUSH rule 4 x 402 [7,8,6]
+ CRUSH rule 4 x 403 [0,1,4]
+ CRUSH rule 4 x 404 [4,3,2]
+ CRUSH rule 4 x 405 [6,5,2]
+ CRUSH rule 4 x 406 [2,0,1]
+ CRUSH rule 4 x 407 [2,8,7]
+ CRUSH rule 4 x 408 [4,1,6]
+ CRUSH rule 4 x 409 [7,3,6]
+ CRUSH rule 4 x 410 [8,6,3]
+ CRUSH rule 4 x 411 [2,0,6]
+ CRUSH rule 4 x 412 [0,5,8]
+ CRUSH rule 4 x 413 [5,0,8]
+ CRUSH rule 4 x 414 [4,1,3]
+ CRUSH rule 4 x 415 [0,6,1]
+ CRUSH rule 4 x 416 [2,1,7]
+ CRUSH rule 4 x 417 [8,7,2]
+ CRUSH rule 4 x 418 [7,6,8]
+ CRUSH rule 4 x 419 [8,3,0]
+ CRUSH rule 4 x 420 [1,4,2]
+ CRUSH rule 4 x 421 [8,6,7]
+ CRUSH rule 4 x 422 [6,7,8]
+ CRUSH rule 4 x 423 [0,5,6]
+ CRUSH rule 4 x 424 [8,4,7]
+ CRUSH rule 4 x 425 [1,3,5]
+ CRUSH rule 4 x 426 [6,7,2]
+ CRUSH rule 4 x 427 [0,7,5]
+ CRUSH rule 4 x 428 [5,4,3]
+ CRUSH rule 4 x 429 [4,6,7]
+ CRUSH rule 4 x 430 [3,6,5]
+ CRUSH rule 4 x 431 [5,3,0]
+ CRUSH rule 4 x 432 [7,1,2]
+ CRUSH rule 4 x 433 [6,5,1]
+ CRUSH rule 4 x 434 [5,2,1]
+ CRUSH rule 4 x 435 [0,5,3]
+ CRUSH rule 4 x 436 [4,0,5]
+ CRUSH rule 4 x 437 [7,5,3]
+ CRUSH rule 4 x 438 [0,3,8]
+ CRUSH rule 4 x 439 [1,3,4]
+ CRUSH rule 4 x 440 [2,7,1]
+ CRUSH rule 4 x 441 [5,7,2]
+ CRUSH rule 4 x 442 [2,4,5]
+ CRUSH rule 4 x 443 [6,8,0]
+ CRUSH rule 4 x 444 [7,0,1]
+ CRUSH rule 4 x 445 [6,3,5]
+ CRUSH rule 4 x 446 [4,3,5]
+ CRUSH rule 4 x 447 [2,1,4]
+ CRUSH rule 4 x 448 [7,2,5]
+ CRUSH rule 4 x 449 [7,8,3]
+ CRUSH rule 4 x 450 [4,5,2]
+ CRUSH rule 4 x 451 [6,8,3]
+ CRUSH rule 4 x 452 [8,3,5]
+ CRUSH rule 4 x 453 [6,8,7]
+ CRUSH rule 4 x 454 [6,7,5]
+ CRUSH rule 4 x 455 [2,7,5]
+ CRUSH rule 4 x 456 [6,8,7]
+ CRUSH rule 4 x 457 [7,2,8]
+ CRUSH rule 4 x 458 [2,8,1]
+ CRUSH rule 4 x 459 [2,0,6]
+ CRUSH rule 4 x 460 [6,5,7]
+ CRUSH rule 4 x 461 [6,5,4]
+ CRUSH rule 4 x 462 [8,1,5]
+ CRUSH rule 4 x 463 [6,7,2]
+ CRUSH rule 4 x 464 [7,4,2]
+ CRUSH rule 4 x 465 [7,6,0]
+ CRUSH rule 4 x 466 [5,8,4]
+ CRUSH rule 4 x 467 [6,4,7]
+ CRUSH rule 4 x 468 [7,8,0]
+ CRUSH rule 4 x 469 [7,0,1]
+ CRUSH rule 4 x 470 [3,0,5]
+ CRUSH rule 4 x 471 [0,1,2]
+ CRUSH rule 4 x 472 [5,4,0]
+ CRUSH rule 4 x 473 [1,0,4]
+ CRUSH rule 4 x 474 [6,0,7]
+ CRUSH rule 4 x 475 [6,7,8]
+ CRUSH rule 4 x 476 [4,3,5]
+ CRUSH rule 4 x 477 [5,8,6]
+ CRUSH rule 4 x 478 [6,7,2]
+ CRUSH rule 4 x 479 [0,5,8]
+ CRUSH rule 4 x 480 [1,8,6]
+ CRUSH rule 4 x 481 [2,4,5]
+ CRUSH rule 4 x 482 [4,3,5]
+ CRUSH rule 4 x 483 [0,2,6]
+ CRUSH rule 4 x 484 [1,2,7]
+ CRUSH rule 4 x 485 [4,7,3]
+ CRUSH rule 4 x 486 [4,1,7]
+ CRUSH rule 4 x 487 [5,0,1]
+ CRUSH rule 4 x 488 [5,7,1]
+ CRUSH rule 4 x 489 [2,8,5]
+ CRUSH rule 4 x 490 [6,4,1]
+ CRUSH rule 4 x 491 [1,0,6]
+ CRUSH rule 4 x 492 [6,5,4]
+ CRUSH rule 4 x 493 [0,2,7]
+ CRUSH rule 4 x 494 [1,0,8]
+ CRUSH rule 4 x 495 [3,4,5]
+ CRUSH rule 4 x 496 [7,5,4]
+ CRUSH rule 4 x 497 [5,7,3]
+ CRUSH rule 4 x 498 [0,5,8]
+ CRUSH rule 4 x 499 [8,4,3]
+ CRUSH rule 4 x 500 [3,6,5]
+ CRUSH rule 4 x 501 [0,7,2]
+ CRUSH rule 4 x 502 [7,1,4]
+ CRUSH rule 4 x 503 [2,3,5]
+ CRUSH rule 4 x 504 [5,6,3]
+ CRUSH rule 4 x 505 [0,7,6]
+ CRUSH rule 4 x 506 [5,3,0]
+ CRUSH rule 4 x 507 [6,0,8]
+ CRUSH rule 4 x 508 [0,1,2]
+ CRUSH rule 4 x 509 [7,5,8]
+ CRUSH rule 4 x 510 [6,0,2]
+ CRUSH rule 4 x 511 [5,8,7]
+ CRUSH rule 4 x 512 [7,6,2]
+ CRUSH rule 4 x 513 [7,2,1]
+ CRUSH rule 4 x 514 [4,3,5]
+ CRUSH rule 4 x 515 [8,5,6]
+ CRUSH rule 4 x 516 [4,0,1]
+ CRUSH rule 4 x 517 [7,8,6]
+ CRUSH rule 4 x 518 [4,6,7]
+ CRUSH rule 4 x 519 [7,3,5]
+ CRUSH rule 4 x 520 [2,6,3]
+ CRUSH rule 4 x 521 [8,7,6]
+ CRUSH rule 4 x 522 [6,8,0]
+ CRUSH rule 4 x 523 [4,2,5]
+ CRUSH rule 4 x 524 [0,4,5]
+ CRUSH rule 4 x 525 [0,4,1]
+ CRUSH rule 4 x 526 [1,5,8]
+ CRUSH rule 4 x 527 [0,2,4]
+ CRUSH rule 4 x 528 [5,3,0]
+ CRUSH rule 4 x 529 [5,7,0]
+ CRUSH rule 4 x 530 [6,7,8]
+ CRUSH rule 4 x 531 [6,1,0]
+ CRUSH rule 4 x 532 [6,3,7]
+ CRUSH rule 4 x 533 [5,6,3]
+ CRUSH rule 4 x 534 [7,3,5]
+ CRUSH rule 4 x 535 [8,6,0]
+ CRUSH rule 4 x 536 [6,7,0]
+ CRUSH rule 4 x 537 [3,7,5]
+ CRUSH rule 4 x 538 [6,8,3]
+ CRUSH rule 4 x 539 [8,3,7]
+ CRUSH rule 4 x 540 [0,6,1]
+ CRUSH rule 4 x 541 [2,3,1]
+ CRUSH rule 4 x 542 [3,5,4]
+ CRUSH rule 4 x 543 [6,0,8]
+ CRUSH rule 4 x 544 [3,7,6]
+ CRUSH rule 4 x 545 [5,7,0]
+ CRUSH rule 4 x 546 [6,1,7]
+ CRUSH rule 4 x 547 [8,2,0]
+ CRUSH rule 4 x 548 [5,2,0]
+ CRUSH rule 4 x 549 [5,8,2]
+ CRUSH rule 4 x 550 [0,5,4]
+ CRUSH rule 4 x 551 [7,5,4]
+ CRUSH rule 4 x 552 [5,4,3]
+ CRUSH rule 4 x 553 [4,2,3]
+ CRUSH rule 4 x 554 [0,8,5]
+ CRUSH rule 4 x 555 [5,0,8]
+ CRUSH rule 4 x 556 [3,4,5]
+ CRUSH rule 4 x 557 [7,4,6]
+ CRUSH rule 4 x 558 [3,1,0]
+ CRUSH rule 4 x 559 [4,2,6]
+ CRUSH rule 4 x 560 [8,3,5]
+ CRUSH rule 4 x 561 [6,3,7]
+ CRUSH rule 4 x 562 [3,4,1]
+ CRUSH rule 4 x 563 [2,6,0]
+ CRUSH rule 4 x 564 [5,1,7]
+ CRUSH rule 4 x 565 [3,6,5]
+ CRUSH rule 4 x 566 [4,7,2]
+ CRUSH rule 4 x 567 [3,6,1]
+ CRUSH rule 4 x 568 [7,4,1]
+ CRUSH rule 4 x 569 [3,4,2]
+ CRUSH rule 4 x 570 [1,5,4]
+ CRUSH rule 4 x 571 [3,7,5]
+ CRUSH rule 4 x 572 [3,4,0]
+ CRUSH rule 4 x 573 [3,0,7]
+ CRUSH rule 4 x 574 [2,0,1]
+ CRUSH rule 4 x 575 [8,6,0]
+ CRUSH rule 4 x 576 [4,6,8]
+ CRUSH rule 4 x 577 [8,2,6]
+ CRUSH rule 4 x 578 [6,8,7]
+ CRUSH rule 4 x 579 [3,1,6]
+ CRUSH rule 4 x 580 [3,0,7]
+ CRUSH rule 4 x 581 [7,2,6]
+ CRUSH rule 4 x 582 [2,8,7]
+ CRUSH rule 4 x 583 [6,0,8]
+ CRUSH rule 4 x 584 [8,1,2]
+ CRUSH rule 4 x 585 [7,0,5]
+ CRUSH rule 4 x 586 [0,1,7]
+ CRUSH rule 4 x 587 [2,5,0]
+ CRUSH rule 4 x 588 [3,4,7]
+ CRUSH rule 4 x 589 [7,1,4]
+ CRUSH rule 4 x 590 [6,2,3]
+ CRUSH rule 4 x 591 [5,2,4]
+ CRUSH rule 4 x 592 [2,0,3]
+ CRUSH rule 4 x 593 [0,8,1]
+ CRUSH rule 4 x 594 [0,7,4]
+ CRUSH rule 4 x 595 [7,1,8]
+ CRUSH rule 4 x 596 [4,3,5]
+ CRUSH rule 4 x 597 [3,1,2]
+ CRUSH rule 4 x 598 [3,2,4]
+ CRUSH rule 4 x 599 [5,2,1]
+ CRUSH rule 4 x 600 [7,0,8]
+ CRUSH rule 4 x 601 [0,7,8]
+ CRUSH rule 4 x 602 [3,7,5]
+ CRUSH rule 4 x 603 [5,1,6]
+ CRUSH rule 4 x 604 [7,5,4]
+ CRUSH rule 4 x 605 [3,0,5]
+ CRUSH rule 4 x 606 [2,0,1]
+ CRUSH rule 4 x 607 [0,4,8]
+ CRUSH rule 4 x 608 [5,3,2]
+ CRUSH rule 4 x 609 [5,2,4]
+ CRUSH rule 4 x 610 [3,7,6]
+ CRUSH rule 4 x 611 [1,0,2]
+ CRUSH rule 4 x 612 [2,0,8]
+ CRUSH rule 4 x 613 [7,2,0]
+ CRUSH rule 4 x 614 [7,8,6]
+ CRUSH rule 4 x 615 [6,8,2]
+ CRUSH rule 4 x 616 [0,8,1]
+ CRUSH rule 4 x 617 [6,1,2]
+ CRUSH rule 4 x 618 [7,6,5]
+ CRUSH rule 4 x 619 [5,1,8]
+ CRUSH rule 4 x 620 [4,1,0]
+ CRUSH rule 4 x 621 [5,8,3]
+ CRUSH rule 4 x 622 [0,4,3]
+ CRUSH rule 4 x 623 [0,6,2]
+ CRUSH rule 4 x 624 [3,5,1]
+ CRUSH rule 4 x 625 [2,3,7]
+ CRUSH rule 4 x 626 [7,8,0]
+ CRUSH rule 4 x 627 [2,7,1]
+ CRUSH rule 4 x 628 [8,0,5]
+ CRUSH rule 4 x 629 [2,6,5]
+ CRUSH rule 4 x 630 [2,7,1]
+ CRUSH rule 4 x 631 [0,7,8]
+ CRUSH rule 4 x 632 [7,0,2]
+ CRUSH rule 4 x 633 [8,6,5]
+ CRUSH rule 4 x 634 [0,4,2]
+ CRUSH rule 4 x 635 [5,6,4]
+ CRUSH rule 4 x 636 [1,4,2]
+ CRUSH rule 4 x 637 [4,1,2]
+ CRUSH rule 4 x 638 [6,8,1]
+ CRUSH rule 4 x 639 [4,2,1]
+ CRUSH rule 4 x 640 [3,1,0]
+ CRUSH rule 4 x 641 [7,2,0]
+ CRUSH rule 4 x 642 [2,0,1]
+ CRUSH rule 4 x 643 [3,5,1]
+ CRUSH rule 4 x 644 [8,1,5]
+ CRUSH rule 4 x 645 [5,4,6]
+ CRUSH rule 4 x 646 [8,0,3]
+ CRUSH rule 4 x 647 [7,1,3]
+ CRUSH rule 4 x 648 [0,6,3]
+ CRUSH rule 4 x 649 [4,7,5]
+ CRUSH rule 4 x 650 [7,8,6]
+ CRUSH rule 4 x 651 [3,7,6]
+ CRUSH rule 4 x 652 [3,4,8]
+ CRUSH rule 4 x 653 [8,5,2]
+ CRUSH rule 4 x 654 [7,5,2]
+ CRUSH rule 4 x 655 [0,3,4]
+ CRUSH rule 4 x 656 [4,3,8]
+ CRUSH rule 4 x 657 [6,1,2]
+ CRUSH rule 4 x 658 [5,4,6]
+ CRUSH rule 4 x 659 [4,6,7]
+ CRUSH rule 4 x 660 [7,8,6]
+ CRUSH rule 4 x 661 [1,8,3]
+ CRUSH rule 4 x 662 [4,5,3]
+ CRUSH rule 4 x 663 [1,3,8]
+ CRUSH rule 4 x 664 [1,4,0]
+ CRUSH rule 4 x 665 [5,7,6]
+ CRUSH rule 4 x 666 [2,8,4]
+ CRUSH rule 4 x 667 [1,3,5]
+ CRUSH rule 4 x 668 [3,7,6]
+ CRUSH rule 4 x 669 [6,4,5]
+ CRUSH rule 4 x 670 [4,0,3]
+ CRUSH rule 4 x 671 [0,2,1]
+ CRUSH rule 4 x 672 [4,3,2]
+ CRUSH rule 4 x 673 [5,3,0]
+ CRUSH rule 4 x 674 [3,1,8]
+ CRUSH rule 4 x 675 [0,8,6]
+ CRUSH rule 4 x 676 [0,2,4]
+ CRUSH rule 4 x 677 [4,1,3]
+ CRUSH rule 4 x 678 [2,3,1]
+ CRUSH rule 4 x 679 [6,0,2]
+ CRUSH rule 4 x 680 [0,4,6]
+ CRUSH rule 4 x 681 [4,7,3]
+ CRUSH rule 4 x 682 [0,5,2]
+ CRUSH rule 4 x 683 [0,1,5]
+ CRUSH rule 4 x 684 [7,1,6]
+ CRUSH rule 4 x 685 [7,1,6]
+ CRUSH rule 4 x 686 [1,4,3]
+ CRUSH rule 4 x 687 [3,5,7]
+ CRUSH rule 4 x 688 [5,7,2]
+ CRUSH rule 4 x 689 [6,5,0]
+ CRUSH rule 4 x 690 [8,1,7]
+ CRUSH rule 4 x 691 [3,0,5]
+ CRUSH rule 4 x 692 [7,2,8]
+ CRUSH rule 4 x 693 [6,7,3]
+ CRUSH rule 4 x 694 [6,5,1]
+ CRUSH rule 4 x 695 [0,8,7]
+ CRUSH rule 4 x 696 [1,4,3]
+ CRUSH rule 4 x 697 [6,1,2]
+ CRUSH rule 4 x 698 [6,2,0]
+ CRUSH rule 4 x 699 [1,6,8]
+ CRUSH rule 4 x 700 [0,3,1]
+ CRUSH rule 4 x 701 [4,3,2]
+ CRUSH rule 4 x 702 [3,5,0]
+ CRUSH rule 4 x 703 [8,3,4]
+ CRUSH rule 4 x 704 [0,3,8]
+ CRUSH rule 4 x 705 [8,6,0]
+ CRUSH rule 4 x 706 [1,2,4]
+ CRUSH rule 4 x 707 [7,8,6]
+ CRUSH rule 4 x 708 [3,5,8]
+ CRUSH rule 4 x 709 [6,3,0]
+ CRUSH rule 4 x 710 [8,4,3]
+ CRUSH rule 4 x 711 [2,3,8]
+ CRUSH rule 4 x 712 [2,3,7]
+ CRUSH rule 4 x 713 [6,7,8]
+ CRUSH rule 4 x 714 [3,2,0]
+ CRUSH rule 4 x 715 [1,2,4]
+ CRUSH rule 4 x 716 [3,6,0]
+ CRUSH rule 4 x 717 [8,7,2]
+ CRUSH rule 4 x 718 [3,7,8]
+ CRUSH rule 4 x 719 [2,6,3]
+ CRUSH rule 4 x 720 [6,8,2]
+ CRUSH rule 4 x 721 [5,4,7]
+ CRUSH rule 4 x 722 [5,4,6]
+ CRUSH rule 4 x 723 [5,1,0]
+ CRUSH rule 4 x 724 [0,6,2]
+ CRUSH rule 4 x 725 [0,1,5]
+ CRUSH rule 4 x 726 [3,8,4]
+ CRUSH rule 4 x 727 [4,6,8]
+ CRUSH rule 4 x 728 [2,1,0]
+ CRUSH rule 4 x 729 [5,3,7]
+ CRUSH rule 4 x 730 [3,7,5]
+ CRUSH rule 4 x 731 [4,1,5]
+ CRUSH rule 4 x 732 [1,5,3]
+ CRUSH rule 4 x 733 [5,4,7]
+ CRUSH rule 4 x 734 [6,4,2]
+ CRUSH rule 4 x 735 [4,8,3]
+ CRUSH rule 4 x 736 [3,5,6]
+ CRUSH rule 4 x 737 [1,0,8]
+ CRUSH rule 4 x 738 [5,2,3]
+ CRUSH rule 4 x 739 [0,1,2]
+ CRUSH rule 4 x 740 [0,1,7]
+ CRUSH rule 4 x 741 [7,8,0]
+ CRUSH rule 4 x 742 [8,2,1]
+ CRUSH rule 4 x 743 [7,0,1]
+ CRUSH rule 4 x 744 [4,7,3]
+ CRUSH rule 4 x 745 [3,4,1]
+ CRUSH rule 4 x 746 [4,1,3]
+ CRUSH rule 4 x 747 [6,0,3]
+ CRUSH rule 4 x 748 [2,7,0]
+ CRUSH rule 4 x 749 [4,5,8]
+ CRUSH rule 4 x 750 [1,6,3]
+ CRUSH rule 4 x 751 [2,1,6]
+ CRUSH rule 4 x 752 [8,1,5]
+ CRUSH rule 4 x 753 [7,8,3]
+ CRUSH rule 4 x 754 [8,6,7]
+ CRUSH rule 4 x 755 [1,2,0]
+ CRUSH rule 4 x 756 [5,6,1]
+ CRUSH rule 4 x 757 [8,6,1]
+ CRUSH rule 4 x 758 [6,0,3]
+ CRUSH rule 4 x 759 [8,5,3]
+ CRUSH rule 4 x 760 [1,5,4]
+ CRUSH rule 4 x 761 [4,1,2]
+ CRUSH rule 4 x 762 [2,7,8]
+ CRUSH rule 4 x 763 [8,6,7]
+ CRUSH rule 4 x 764 [1,7,0]
+ CRUSH rule 4 x 765 [6,5,2]
+ CRUSH rule 4 x 766 [8,5,7]
+ CRUSH rule 4 x 767 [1,2,0]
+ CRUSH rule 4 x 768 [8,3,2]
+ CRUSH rule 4 x 769 [6,2,8]
+ CRUSH rule 4 x 770 [6,0,7]
+ CRUSH rule 4 x 771 [7,0,3]
+ CRUSH rule 4 x 772 [8,3,7]
+ CRUSH rule 4 x 773 [3,1,5]
+ CRUSH rule 4 x 774 [4,6,5]
+ CRUSH rule 4 x 775 [6,8,4]
+ CRUSH rule 4 x 776 [7,2,1]
+ CRUSH rule 4 x 777 [3,1,6]
+ CRUSH rule 4 x 778 [1,8,0]
+ CRUSH rule 4 x 779 [2,7,3]
+ CRUSH rule 4 x 780 [0,2,3]
+ CRUSH rule 4 x 781 [6,3,7]
+ CRUSH rule 4 x 782 [5,4,0]
+ CRUSH rule 4 x 783 [7,1,8]
+ CRUSH rule 4 x 784 [0,1,5]
+ CRUSH rule 4 x 785 [6,1,2]
+ CRUSH rule 4 x 786 [7,6,5]
+ CRUSH rule 4 x 787 [1,0,6]
+ CRUSH rule 4 x 788 [6,0,8]
+ CRUSH rule 4 x 789 [0,4,3]
+ CRUSH rule 4 x 790 [8,4,7]
+ CRUSH rule 4 x 791 [3,8,7]
+ CRUSH rule 4 x 792 [5,8,4]
+ CRUSH rule 4 x 793 [6,1,3]
+ CRUSH rule 4 x 794 [2,6,8]
+ CRUSH rule 4 x 795 [0,3,2]
+ CRUSH rule 4 x 796 [3,4,5]
+ CRUSH rule 4 x 797 [2,3,4]
+ CRUSH rule 4 x 798 [6,8,7]
+ CRUSH rule 4 x 799 [5,1,4]
+ CRUSH rule 4 x 800 [5,0,3]
+ CRUSH rule 4 x 801 [3,6,8]
+ CRUSH rule 4 x 802 [1,8,0]
+ CRUSH rule 4 x 803 [0,5,7]
+ CRUSH rule 4 x 804 [6,2,5]
+ CRUSH rule 4 x 805 [3,6,7]
+ CRUSH rule 4 x 806 [1,3,4]
+ CRUSH rule 4 x 807 [5,4,7]
+ CRUSH rule 4 x 808 [4,6,2]
+ CRUSH rule 4 x 809 [1,4,5]
+ CRUSH rule 4 x 810 [5,7,3]
+ CRUSH rule 4 x 811 [8,4,5]
+ CRUSH rule 4 x 812 [8,5,4]
+ CRUSH rule 4 x 813 [6,4,8]
+ CRUSH rule 4 x 814 [3,6,8]
+ CRUSH rule 4 x 815 [3,1,2]
+ CRUSH rule 4 x 816 [2,1,8]
+ CRUSH rule 4 x 817 [4,3,7]
+ CRUSH rule 4 x 818 [3,5,0]
+ CRUSH rule 4 x 819 [5,1,4]
+ CRUSH rule 4 x 820 [3,5,4]
+ CRUSH rule 4 x 821 [4,5,8]
+ CRUSH rule 4 x 822 [2,0,5]
+ CRUSH rule 4 x 823 [4,8,2]
+ CRUSH rule 4 x 824 [3,7,4]
+ CRUSH rule 4 x 825 [2,8,7]
+ CRUSH rule 4 x 826 [7,0,5]
+ CRUSH rule 4 x 827 [0,8,2]
+ CRUSH rule 4 x 828 [2,3,1]
+ CRUSH rule 4 x 829 [5,6,7]
+ CRUSH rule 4 x 830 [2,3,5]
+ CRUSH rule 4 x 831 [1,6,0]
+ CRUSH rule 4 x 832 [4,5,3]
+ CRUSH rule 4 x 833 [2,1,7]
+ CRUSH rule 4 x 834 [3,4,2]
+ CRUSH rule 4 x 835 [8,4,3]
+ CRUSH rule 4 x 836 [3,4,5]
+ CRUSH rule 4 x 837 [6,3,1]
+ CRUSH rule 4 x 838 [6,7,2]
+ CRUSH rule 4 x 839 [5,0,6]
+ CRUSH rule 4 x 840 [7,8,5]
+ CRUSH rule 4 x 841 [4,8,5]
+ CRUSH rule 4 x 842 [2,4,0]
+ CRUSH rule 4 x 843 [6,4,7]
+ CRUSH rule 4 x 844 [4,8,1]
+ CRUSH rule 4 x 845 [3,8,6]
+ CRUSH rule 4 x 846 [3,2,0]
+ CRUSH rule 4 x 847 [0,2,7]
+ CRUSH rule 4 x 848 [2,6,1]
+ CRUSH rule 4 x 849 [4,5,3]
+ CRUSH rule 4 x 850 [1,0,5]
+ CRUSH rule 4 x 851 [6,8,7]
+ CRUSH rule 4 x 852 [7,3,8]
+ CRUSH rule 4 x 853 [6,8,1]
+ CRUSH rule 4 x 854 [7,6,0]
+ CRUSH rule 4 x 855 [5,7,2]
+ CRUSH rule 4 x 856 [6,7,3]
+ CRUSH rule 4 x 857 [8,5,0]
+ CRUSH rule 4 x 858 [6,4,1]
+ CRUSH rule 4 x 859 [6,0,7]
+ CRUSH rule 4 x 860 [4,1,2]
+ CRUSH rule 4 x 861 [8,7,6]
+ CRUSH rule 4 x 862 [6,1,7]
+ CRUSH rule 4 x 863 [8,7,0]
+ CRUSH rule 4 x 864 [5,6,8]
+ CRUSH rule 4 x 865 [8,1,0]
+ CRUSH rule 4 x 866 [3,4,8]
+ CRUSH rule 4 x 867 [6,5,1]
+ CRUSH rule 4 x 868 [6,3,0]
+ CRUSH rule 4 x 869 [8,7,3]
+ CRUSH rule 4 x 870 [0,4,8]
+ CRUSH rule 4 x 871 [3,4,5]
+ CRUSH rule 4 x 872 [5,1,3]
+ CRUSH rule 4 x 873 [4,6,5]
+ CRUSH rule 4 x 874 [2,6,1]
+ CRUSH rule 4 x 875 [2,6,4]
+ CRUSH rule 4 x 876 [5,8,1]
+ CRUSH rule 4 x 877 [6,4,2]
+ CRUSH rule 4 x 878 [5,4,0]
+ CRUSH rule 4 x 879 [7,4,8]
+ CRUSH rule 4 x 880 [3,5,0]
+ CRUSH rule 4 x 881 [5,6,1]
+ CRUSH rule 4 x 882 [4,0,2]
+ CRUSH rule 4 x 883 [2,1,0]
+ CRUSH rule 4 x 884 [6,0,4]
+ CRUSH rule 4 x 885 [5,1,4]
+ CRUSH rule 4 x 886 [3,6,4]
+ CRUSH rule 4 x 887 [7,4,0]
+ CRUSH rule 4 x 888 [6,8,0]
+ CRUSH rule 4 x 889 [2,1,7]
+ CRUSH rule 4 x 890 [7,2,0]
+ CRUSH rule 4 x 891 [1,8,0]
+ CRUSH rule 4 x 892 [6,2,3]
+ CRUSH rule 4 x 893 [2,3,7]
+ CRUSH rule 4 x 894 [7,5,0]
+ CRUSH rule 4 x 895 [5,3,2]
+ CRUSH rule 4 x 896 [1,8,2]
+ CRUSH rule 4 x 897 [4,2,6]
+ CRUSH rule 4 x 898 [0,5,4]
+ CRUSH rule 4 x 899 [1,7,6]
+ CRUSH rule 4 x 900 [4,1,0]
+ CRUSH rule 4 x 901 [5,0,1]
+ CRUSH rule 4 x 902 [8,5,7]
+ CRUSH rule 4 x 903 [5,7,3]
+ CRUSH rule 4 x 904 [5,6,8]
+ CRUSH rule 4 x 905 [6,2,5]
+ CRUSH rule 4 x 906 [1,2,0]
+ CRUSH rule 4 x 907 [7,1,0]
+ CRUSH rule 4 x 908 [5,8,1]
+ CRUSH rule 4 x 909 [2,3,4]
+ CRUSH rule 4 x 910 [6,4,0]
+ CRUSH rule 4 x 911 [5,8,4]
+ CRUSH rule 4 x 912 [0,1,7]
+ CRUSH rule 4 x 913 [7,6,8]
+ CRUSH rule 4 x 914 [6,4,7]
+ CRUSH rule 4 x 915 [8,2,6]
+ CRUSH rule 4 x 916 [3,1,4]
+ CRUSH rule 4 x 917 [1,5,3]
+ CRUSH rule 4 x 918 [8,2,1]
+ CRUSH rule 4 x 919 [6,2,8]
+ CRUSH rule 4 x 920 [7,6,4]
+ CRUSH rule 4 x 921 [1,4,5]
+ CRUSH rule 4 x 922 [6,7,8]
+ CRUSH rule 4 x 923 [5,3,6]
+ CRUSH rule 4 x 924 [3,5,4]
+ CRUSH rule 4 x 925 [5,7,3]
+ CRUSH rule 4 x 926 [3,4,5]
+ CRUSH rule 4 x 927 [1,6,3]
+ CRUSH rule 4 x 928 [8,1,2]
+ CRUSH rule 4 x 929 [4,5,1]
+ CRUSH rule 4 x 930 [2,4,6]
+ CRUSH rule 4 x 931 [5,0,1]
+ CRUSH rule 4 x 932 [4,3,0]
+ CRUSH rule 4 x 933 [8,5,4]
+ CRUSH rule 4 x 934 [5,3,8]
+ CRUSH rule 4 x 935 [6,3,4]
+ CRUSH rule 4 x 936 [0,6,7]
+ CRUSH rule 4 x 937 [5,4,3]
+ CRUSH rule 4 x 938 [6,5,8]
+ CRUSH rule 4 x 939 [2,7,0]
+ CRUSH rule 4 x 940 [8,7,6]
+ CRUSH rule 4 x 941 [5,2,0]
+ CRUSH rule 4 x 942 [1,0,2]
+ CRUSH rule 4 x 943 [8,2,4]
+ CRUSH rule 4 x 944 [4,3,7]
+ CRUSH rule 4 x 945 [7,2,4]
+ CRUSH rule 4 x 946 [2,0,7]
+ CRUSH rule 4 x 947 [4,5,3]
+ CRUSH rule 4 x 948 [7,8,6]
+ CRUSH rule 4 x 949 [6,1,7]
+ CRUSH rule 4 x 950 [3,5,8]
+ CRUSH rule 4 x 951 [4,5,3]
+ CRUSH rule 4 x 952 [2,0,7]
+ CRUSH rule 4 x 953 [1,3,5]
+ CRUSH rule 4 x 954 [4,2,5]
+ CRUSH rule 4 x 955 [8,6,0]
+ CRUSH rule 4 x 956 [1,0,8]
+ CRUSH rule 4 x 957 [7,6,1]
+ CRUSH rule 4 x 958 [8,7,5]
+ CRUSH rule 4 x 959 [5,2,7]
+ CRUSH rule 4 x 960 [3,6,5]
+ CRUSH rule 4 x 961 [4,0,2]
+ CRUSH rule 4 x 962 [7,4,3]
+ CRUSH rule 4 x 963 [0,5,2]
+ CRUSH rule 4 x 964 [3,1,4]
+ CRUSH rule 4 x 965 [7,6,5]
+ CRUSH rule 4 x 966 [3,8,4]
+ CRUSH rule 4 x 967 [8,6,5]
+ CRUSH rule 4 x 968 [7,2,4]
+ CRUSH rule 4 x 969 [8,0,6]
+ CRUSH rule 4 x 970 [0,6,3]
+ CRUSH rule 4 x 971 [1,7,8]
+ CRUSH rule 4 x 972 [1,8,4]
+ CRUSH rule 4 x 973 [1,2,0]
+ CRUSH rule 4 x 974 [5,3,2]
+ CRUSH rule 4 x 975 [3,7,4]
+ CRUSH rule 4 x 976 [4,3,5]
+ CRUSH rule 4 x 977 [8,3,2]
+ CRUSH rule 4 x 978 [7,2,8]
+ CRUSH rule 4 x 979 [7,6,0]
+ CRUSH rule 4 x 980 [6,0,7]
+ CRUSH rule 4 x 981 [7,3,2]
+ CRUSH rule 4 x 982 [4,2,0]
+ CRUSH rule 4 x 983 [3,5,6]
+ CRUSH rule 4 x 984 [0,2,1]
+ CRUSH rule 4 x 985 [2,5,4]
+ CRUSH rule 4 x 986 [8,7,3]
+ CRUSH rule 4 x 987 [0,5,1]
+ CRUSH rule 4 x 988 [1,3,5]
+ CRUSH rule 4 x 989 [0,6,3]
+ CRUSH rule 4 x 990 [1,0,8]
+ CRUSH rule 4 x 991 [0,4,1]
+ CRUSH rule 4 x 992 [7,1,5]
+ CRUSH rule 4 x 993 [0,6,2]
+ CRUSH rule 4 x 994 [3,4,5]
+ CRUSH rule 4 x 995 [7,6,2]
+ CRUSH rule 4 x 996 [6,7,5]
+ CRUSH rule 4 x 997 [6,4,1]
+ CRUSH rule 4 x 998 [8,1,2]
+ CRUSH rule 4 x 999 [0,7,8]
+ CRUSH rule 4 x 1000 [8,5,0]
+ CRUSH rule 4 x 1001 [2,0,4]
+ CRUSH rule 4 x 1002 [1,3,2]
+ CRUSH rule 4 x 1003 [2,8,7]
+ CRUSH rule 4 x 1004 [6,1,2]
+ CRUSH rule 4 x 1005 [6,1,2]
+ CRUSH rule 4 x 1006 [1,0,2]
+ CRUSH rule 4 x 1007 [1,2,4]
+ CRUSH rule 4 x 1008 [1,7,0]
+ CRUSH rule 4 x 1009 [6,8,5]
+ CRUSH rule 4 x 1010 [3,4,0]
+ CRUSH rule 4 x 1011 [3,0,4]
+ CRUSH rule 4 x 1012 [3,0,7]
+ CRUSH rule 4 x 1013 [5,1,0]
+ CRUSH rule 4 x 1014 [2,8,4]
+ CRUSH rule 4 x 1015 [6,8,4]
+ CRUSH rule 4 x 1016 [2,0,1]
+ CRUSH rule 4 x 1017 [6,0,2]
+ CRUSH rule 4 x 1018 [5,4,3]
+ CRUSH rule 4 x 1019 [5,3,8]
+ CRUSH rule 4 x 1020 [5,1,3]
+ CRUSH rule 4 x 1021 [5,2,1]
+ CRUSH rule 4 x 1022 [1,6,7]
+ CRUSH rule 4 x 1023 [3,2,0]
+ rule 4 (choose-set-two) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 5 (chooseleaf-set), x = 0..1023, numrep = 2..3
+ CRUSH rule 5 x 0 [0,5]
+ CRUSH rule 5 x 1 [0,8]
+ CRUSH rule 5 x 2 [1,3]
+ CRUSH rule 5 x 3 [8,0]
+ CRUSH rule 5 x 4 [5,0]
+ CRUSH rule 5 x 5 [7,0]
+ CRUSH rule 5 x 6 [2,6]
+ CRUSH rule 5 x 7 [5,8]
+ CRUSH rule 5 x 8 [5,6]
+ CRUSH rule 5 x 9 [2,3]
+ CRUSH rule 5 x 10 [0,7]
+ CRUSH rule 5 x 11 [0,7]
+ CRUSH rule 5 x 12 [0,4]
+ CRUSH rule 5 x 13 [3,8]
+ CRUSH rule 5 x 14 [7,0]
+ CRUSH rule 5 x 15 [7,2]
+ CRUSH rule 5 x 16 [3,6]
+ CRUSH rule 5 x 17 [5,1]
+ CRUSH rule 5 x 18 [1,4]
+ CRUSH rule 5 x 19 [7,5]
+ CRUSH rule 5 x 20 [2,4]
+ CRUSH rule 5 x 21 [3,7]
+ CRUSH rule 5 x 22 [8,3]
+ CRUSH rule 5 x 23 [3,6]
+ CRUSH rule 5 x 24 [1,7]
+ CRUSH rule 5 x 25 [3,7]
+ CRUSH rule 5 x 26 [2,8]
+ CRUSH rule 5 x 27 [3,1]
+ CRUSH rule 5 x 28 [6,0]
+ CRUSH rule 5 x 29 [8,5]
+ CRUSH rule 5 x 30 [5,7]
+ CRUSH rule 5 x 31 [8,0]
+ CRUSH rule 5 x 32 [3,6]
+ CRUSH rule 5 x 33 [2,7]
+ CRUSH rule 5 x 34 [2,5]
+ CRUSH rule 5 x 35 [0,8]
+ CRUSH rule 5 x 36 [3,8]
+ CRUSH rule 5 x 37 [0,4]
+ CRUSH rule 5 x 38 [4,8]
+ CRUSH rule 5 x 39 [3,7]
+ CRUSH rule 5 x 40 [7,2]
+ CRUSH rule 5 x 41 [0,6]
+ CRUSH rule 5 x 42 [4,6]
+ CRUSH rule 5 x 43 [0,3]
+ CRUSH rule 5 x 44 [1,6]
+ CRUSH rule 5 x 45 [8,0]
+ CRUSH rule 5 x 46 [2,4]
+ CRUSH rule 5 x 47 [4,2]
+ CRUSH rule 5 x 48 [4,6]
+ CRUSH rule 5 x 49 [5,7]
+ CRUSH rule 5 x 50 [3,1]
+ CRUSH rule 5 x 51 [3,6]
+ CRUSH rule 5 x 52 [8,1]
+ CRUSH rule 5 x 53 [3,8]
+ CRUSH rule 5 x 54 [7,3]
+ CRUSH rule 5 x 55 [8,2]
+ CRUSH rule 5 x 56 [6,4]
+ CRUSH rule 5 x 57 [5,8]
+ CRUSH rule 5 x 58 [1,8]
+ CRUSH rule 5 x 59 [4,2]
+ CRUSH rule 5 x 60 [3,2]
+ CRUSH rule 5 x 61 [4,6]
+ CRUSH rule 5 x 62 [7,0]
+ CRUSH rule 5 x 63 [5,6]
+ CRUSH rule 5 x 64 [4,2]
+ CRUSH rule 5 x 65 [7,3]
+ CRUSH rule 5 x 66 [5,6]
+ CRUSH rule 5 x 67 [5,0]
+ CRUSH rule 5 x 68 [0,5]
+ CRUSH rule 5 x 69 [5,1]
+ CRUSH rule 5 x 70 [7,0]
+ CRUSH rule 5 x 71 [2,8]
+ CRUSH rule 5 x 72 [6,1]
+ CRUSH rule 5 x 73 [2,7]
+ CRUSH rule 5 x 74 [0,7]
+ CRUSH rule 5 x 75 [3,2]
+ CRUSH rule 5 x 76 [5,1]
+ CRUSH rule 5 x 77 [7,2]
+ CRUSH rule 5 x 78 [1,4]
+ CRUSH rule 5 x 79 [5,1]
+ CRUSH rule 5 x 80 [0,3]
+ CRUSH rule 5 x 81 [0,3]
+ CRUSH rule 5 x 82 [7,1]
+ CRUSH rule 5 x 83 [2,6]
+ CRUSH rule 5 x 84 [7,2]
+ CRUSH rule 5 x 85 [3,8]
+ CRUSH rule 5 x 86 [0,6]
+ CRUSH rule 5 x 87 [0,7]
+ CRUSH rule 5 x 88 [1,6]
+ CRUSH rule 5 x 89 [3,0]
+ CRUSH rule 5 x 90 [6,4]
+ CRUSH rule 5 x 91 [3,8]
+ CRUSH rule 5 x 92 [1,8]
+ CRUSH rule 5 x 93 [7,4]
+ CRUSH rule 5 x 94 [0,4]
+ CRUSH rule 5 x 95 [7,5]
+ CRUSH rule 5 x 96 [3,6]
+ CRUSH rule 5 x 97 [8,4]
+ CRUSH rule 5 x 98 [2,7]
+ CRUSH rule 5 x 99 [0,7]
+ CRUSH rule 5 x 100 [1,7]
+ CRUSH rule 5 x 101 [3,7]
+ CRUSH rule 5 x 102 [4,2]
+ CRUSH rule 5 x 103 [4,7]
+ CRUSH rule 5 x 104 [7,4]
+ CRUSH rule 5 x 105 [2,4]
+ CRUSH rule 5 x 106 [1,6]
+ CRUSH rule 5 x 107 [3,2]
+ CRUSH rule 5 x 108 [7,2]
+ CRUSH rule 5 x 109 [1,4]
+ CRUSH rule 5 x 110 [3,2]
+ CRUSH rule 5 x 111 [2,3]
+ CRUSH rule 5 x 112 [2,6]
+ CRUSH rule 5 x 113 [6,2]
+ CRUSH rule 5 x 114 [7,3]
+ CRUSH rule 5 x 115 [8,2]
+ CRUSH rule 5 x 116 [1,6]
+ CRUSH rule 5 x 117 [7,3]
+ CRUSH rule 5 x 118 [0,3]
+ CRUSH rule 5 x 119 [5,6]
+ CRUSH rule 5 x 120 [0,3]
+ CRUSH rule 5 x 121 [2,7]
+ CRUSH rule 5 x 122 [8,5]
+ CRUSH rule 5 x 123 [2,5]
+ CRUSH rule 5 x 124 [3,2]
+ CRUSH rule 5 x 125 [0,7]
+ CRUSH rule 5 x 126 [4,2]
+ CRUSH rule 5 x 127 [3,6]
+ CRUSH rule 5 x 128 [3,6]
+ CRUSH rule 5 x 129 [0,3]
+ CRUSH rule 5 x 130 [3,8]
+ CRUSH rule 5 x 131 [1,3]
+ CRUSH rule 5 x 132 [1,4]
+ CRUSH rule 5 x 133 [3,6]
+ CRUSH rule 5 x 134 [1,8]
+ CRUSH rule 5 x 135 [5,6]
+ CRUSH rule 5 x 136 [2,3]
+ CRUSH rule 5 x 137 [7,3]
+ CRUSH rule 5 x 138 [8,4]
+ CRUSH rule 5 x 139 [3,0]
+ CRUSH rule 5 x 140 [1,6]
+ CRUSH rule 5 x 141 [6,2]
+ CRUSH rule 5 x 142 [3,0]
+ CRUSH rule 5 x 143 [5,8]
+ CRUSH rule 5 x 144 [8,1]
+ CRUSH rule 5 x 145 [8,5]
+ CRUSH rule 5 x 146 [2,6]
+ CRUSH rule 5 x 147 [2,8]
+ CRUSH rule 5 x 148 [3,1]
+ CRUSH rule 5 x 149 [4,8]
+ CRUSH rule 5 x 150 [1,6]
+ CRUSH rule 5 x 151 [3,6]
+ CRUSH rule 5 x 152 [8,4]
+ CRUSH rule 5 x 153 [8,4]
+ CRUSH rule 5 x 154 [3,2]
+ CRUSH rule 5 x 155 [3,7]
+ CRUSH rule 5 x 156 [4,2]
+ CRUSH rule 5 x 157 [4,1]
+ CRUSH rule 5 x 158 [2,8]
+ CRUSH rule 5 x 159 [7,0]
+ CRUSH rule 5 x 160 [2,8]
+ CRUSH rule 5 x 161 [1,5]
+ CRUSH rule 5 x 162 [0,6]
+ CRUSH rule 5 x 163 [5,6]
+ CRUSH rule 5 x 164 [7,1]
+ CRUSH rule 5 x 165 [7,0]
+ CRUSH rule 5 x 166 [2,4]
+ CRUSH rule 5 x 167 [0,7]
+ CRUSH rule 5 x 168 [4,2]
+ CRUSH rule 5 x 169 [2,6]
+ CRUSH rule 5 x 170 [1,4]
+ CRUSH rule 5 x 171 [7,5]
+ CRUSH rule 5 x 172 [0,7]
+ CRUSH rule 5 x 173 [8,5]
+ CRUSH rule 5 x 174 [1,4]
+ CRUSH rule 5 x 175 [6,0]
+ CRUSH rule 5 x 176 [4,1]
+ CRUSH rule 5 x 177 [5,1]
+ CRUSH rule 5 x 178 [3,0]
+ CRUSH rule 5 x 179 [4,1]
+ CRUSH rule 5 x 180 [3,8]
+ CRUSH rule 5 x 181 [6,2]
+ CRUSH rule 5 x 182 [8,5]
+ CRUSH rule 5 x 183 [7,5]
+ CRUSH rule 5 x 184 [5,7]
+ CRUSH rule 5 x 185 [6,1]
+ CRUSH rule 5 x 186 [2,5]
+ CRUSH rule 5 x 187 [1,6]
+ CRUSH rule 5 x 188 [1,8]
+ CRUSH rule 5 x 189 [0,7]
+ CRUSH rule 5 x 190 [4,0]
+ CRUSH rule 5 x 191 [7,1]
+ CRUSH rule 5 x 192 [5,0]
+ CRUSH rule 5 x 193 [4,2]
+ CRUSH rule 5 x 194 [1,3]
+ CRUSH rule 5 x 195 [6,4]
+ CRUSH rule 5 x 196 [6,0]
+ CRUSH rule 5 x 197 [6,5]
+ CRUSH rule 5 x 198 [2,5]
+ CRUSH rule 5 x 199 [0,5]
+ CRUSH rule 5 x 200 [0,5]
+ CRUSH rule 5 x 201 [7,1]
+ CRUSH rule 5 x 202 [6,3]
+ CRUSH rule 5 x 203 [4,8]
+ CRUSH rule 5 x 204 [2,3]
+ CRUSH rule 5 x 205 [0,7]
+ CRUSH rule 5 x 206 [0,7]
+ CRUSH rule 5 x 207 [3,0]
+ CRUSH rule 5 x 208 [7,1]
+ CRUSH rule 5 x 209 [1,8]
+ CRUSH rule 5 x 210 [1,4]
+ CRUSH rule 5 x 211 [5,2]
+ CRUSH rule 5 x 212 [7,5]
+ CRUSH rule 5 x 213 [8,4]
+ CRUSH rule 5 x 214 [4,8]
+ CRUSH rule 5 x 215 [8,1]
+ CRUSH rule 5 x 216 [5,0]
+ CRUSH rule 5 x 217 [0,7]
+ CRUSH rule 5 x 218 [0,7]
+ CRUSH rule 5 x 219 [4,8]
+ CRUSH rule 5 x 220 [5,7]
+ CRUSH rule 5 x 221 [3,6]
+ CRUSH rule 5 x 222 [6,4]
+ CRUSH rule 5 x 223 [1,3]
+ CRUSH rule 5 x 224 [1,5]
+ CRUSH rule 5 x 225 [8,2]
+ CRUSH rule 5 x 226 [7,2]
+ CRUSH rule 5 x 227 [3,1]
+ CRUSH rule 5 x 228 [5,6]
+ CRUSH rule 5 x 229 [3,8]
+ CRUSH rule 5 x 230 [4,7]
+ CRUSH rule 5 x 231 [4,7]
+ CRUSH rule 5 x 232 [2,7]
+ CRUSH rule 5 x 233 [3,7]
+ CRUSH rule 5 x 234 [0,3]
+ CRUSH rule 5 x 235 [3,8]
+ CRUSH rule 5 x 236 [5,2]
+ CRUSH rule 5 x 237 [4,7]
+ CRUSH rule 5 x 238 [4,2]
+ CRUSH rule 5 x 239 [8,4]
+ CRUSH rule 5 x 240 [5,7]
+ CRUSH rule 5 x 241 [3,1]
+ CRUSH rule 5 x 242 [3,2]
+ CRUSH rule 5 x 243 [4,7]
+ CRUSH rule 5 x 244 [4,6]
+ CRUSH rule 5 x 245 [7,0]
+ CRUSH rule 5 x 246 [1,5]
+ CRUSH rule 5 x 247 [6,0]
+ CRUSH rule 5 x 248 [8,0]
+ CRUSH rule 5 x 249 [2,4]
+ CRUSH rule 5 x 250 [2,5]
+ CRUSH rule 5 x 251 [2,3]
+ CRUSH rule 5 x 252 [3,7]
+ CRUSH rule 5 x 253 [3,2]
+ CRUSH rule 5 x 254 [3,2]
+ CRUSH rule 5 x 255 [1,7]
+ CRUSH rule 5 x 256 [5,7]
+ CRUSH rule 5 x 257 [2,8]
+ CRUSH rule 5 x 258 [5,0]
+ CRUSH rule 5 x 259 [4,6]
+ CRUSH rule 5 x 260 [3,6]
+ CRUSH rule 5 x 261 [8,5]
+ CRUSH rule 5 x 262 [5,6]
+ CRUSH rule 5 x 263 [6,1]
+ CRUSH rule 5 x 264 [3,6]
+ CRUSH rule 5 x 265 [8,5]
+ CRUSH rule 5 x 266 [8,2]
+ CRUSH rule 5 x 267 [2,3]
+ CRUSH rule 5 x 268 [0,7]
+ CRUSH rule 5 x 269 [0,8]
+ CRUSH rule 5 x 270 [5,0]
+ CRUSH rule 5 x 271 [7,5]
+ CRUSH rule 5 x 272 [2,8]
+ CRUSH rule 5 x 273 [3,1]
+ CRUSH rule 5 x 274 [6,3]
+ CRUSH rule 5 x 275 [4,7]
+ CRUSH rule 5 x 276 [7,1]
+ CRUSH rule 5 x 277 [6,4]
+ CRUSH rule 5 x 278 [6,1]
+ CRUSH rule 5 x 279 [8,3]
+ CRUSH rule 5 x 280 [0,6]
+ CRUSH rule 5 x 281 [8,0]
+ CRUSH rule 5 x 282 [3,1]
+ CRUSH rule 5 x 283 [8,2]
+ CRUSH rule 5 x 284 [6,3]
+ CRUSH rule 5 x 285 [5,7]
+ CRUSH rule 5 x 286 [2,6]
+ CRUSH rule 5 x 287 [0,4]
+ CRUSH rule 5 x 288 [8,0]
+ CRUSH rule 5 x 289 [4,6]
+ CRUSH rule 5 x 290 [1,3]
+ CRUSH rule 5 x 291 [0,3]
+ CRUSH rule 5 x 292 [8,0]
+ CRUSH rule 5 x 293 [6,0]
+ CRUSH rule 5 x 294 [7,4]
+ CRUSH rule 5 x 295 [4,8]
+ CRUSH rule 5 x 296 [3,1]
+ CRUSH rule 5 x 297 [6,2]
+ CRUSH rule 5 x 298 [1,5]
+ CRUSH rule 5 x 299 [2,8]
+ CRUSH rule 5 x 300 [8,3]
+ CRUSH rule 5 x 301 [0,8]
+ CRUSH rule 5 x 302 [3,0]
+ CRUSH rule 5 x 303 [7,5]
+ CRUSH rule 5 x 304 [2,7]
+ CRUSH rule 5 x 305 [5,8]
+ CRUSH rule 5 x 306 [0,7]
+ CRUSH rule 5 x 307 [0,7]
+ CRUSH rule 5 x 308 [0,8]
+ CRUSH rule 5 x 309 [7,4]
+ CRUSH rule 5 x 310 [4,1]
+ CRUSH rule 5 x 311 [3,6]
+ CRUSH rule 5 x 312 [2,6]
+ CRUSH rule 5 x 313 [5,1]
+ CRUSH rule 5 x 314 [4,2]
+ CRUSH rule 5 x 315 [2,4]
+ CRUSH rule 5 x 316 [6,3]
+ CRUSH rule 5 x 317 [2,6]
+ CRUSH rule 5 x 318 [8,1]
+ CRUSH rule 5 x 319 [5,0]
+ CRUSH rule 5 x 320 [3,7]
+ CRUSH rule 5 x 321 [1,3]
+ CRUSH rule 5 x 322 [2,7]
+ CRUSH rule 5 x 323 [4,7]
+ CRUSH rule 5 x 324 [7,0]
+ CRUSH rule 5 x 325 [4,6]
+ CRUSH rule 5 x 326 [3,2]
+ CRUSH rule 5 x 327 [0,6]
+ CRUSH rule 5 x 328 [7,4]
+ CRUSH rule 5 x 329 [5,6]
+ CRUSH rule 5 x 330 [3,7]
+ CRUSH rule 5 x 331 [2,6]
+ CRUSH rule 5 x 332 [2,4]
+ CRUSH rule 5 x 333 [6,5]
+ CRUSH rule 5 x 334 [8,3]
+ CRUSH rule 5 x 335 [7,1]
+ CRUSH rule 5 x 336 [4,6]
+ CRUSH rule 5 x 337 [7,2]
+ CRUSH rule 5 x 338 [5,6]
+ CRUSH rule 5 x 339 [7,5]
+ CRUSH rule 5 x 340 [2,8]
+ CRUSH rule 5 x 341 [5,1]
+ CRUSH rule 5 x 342 [0,7]
+ CRUSH rule 5 x 343 [6,3]
+ CRUSH rule 5 x 344 [6,0]
+ CRUSH rule 5 x 345 [4,7]
+ CRUSH rule 5 x 346 [8,0]
+ CRUSH rule 5 x 347 [3,1]
+ CRUSH rule 5 x 348 [8,0]
+ CRUSH rule 5 x 349 [1,6]
+ CRUSH rule 5 x 350 [8,5]
+ CRUSH rule 5 x 351 [3,6]
+ CRUSH rule 5 x 352 [1,8]
+ CRUSH rule 5 x 353 [6,4]
+ CRUSH rule 5 x 354 [0,3]
+ CRUSH rule 5 x 355 [3,8]
+ CRUSH rule 5 x 356 [3,0]
+ CRUSH rule 5 x 357 [6,1]
+ CRUSH rule 5 x 358 [2,8]
+ CRUSH rule 5 x 359 [6,1]
+ CRUSH rule 5 x 360 [5,2]
+ CRUSH rule 5 x 361 [8,4]
+ CRUSH rule 5 x 362 [4,1]
+ CRUSH rule 5 x 363 [4,0]
+ CRUSH rule 5 x 364 [2,5]
+ CRUSH rule 5 x 365 [6,5]
+ CRUSH rule 5 x 366 [7,2]
+ CRUSH rule 5 x 367 [4,0]
+ CRUSH rule 5 x 368 [7,4]
+ CRUSH rule 5 x 369 [3,7]
+ CRUSH rule 5 x 370 [8,2]
+ CRUSH rule 5 x 371 [1,3]
+ CRUSH rule 5 x 372 [3,1]
+ CRUSH rule 5 x 373 [0,6]
+ CRUSH rule 5 x 374 [3,8]
+ CRUSH rule 5 x 375 [6,4]
+ CRUSH rule 5 x 376 [7,1]
+ CRUSH rule 5 x 377 [1,3]
+ CRUSH rule 5 x 378 [0,8]
+ CRUSH rule 5 x 379 [8,5]
+ CRUSH rule 5 x 380 [2,5]
+ CRUSH rule 5 x 381 [0,4]
+ CRUSH rule 5 x 382 [1,5]
+ CRUSH rule 5 x 383 [4,6]
+ CRUSH rule 5 x 384 [7,0]
+ CRUSH rule 5 x 385 [7,4]
+ CRUSH rule 5 x 386 [0,3]
+ CRUSH rule 5 x 387 [1,3]
+ CRUSH rule 5 x 388 [5,0]
+ CRUSH rule 5 x 389 [1,5]
+ CRUSH rule 5 x 390 [5,6]
+ CRUSH rule 5 x 391 [5,6]
+ CRUSH rule 5 x 392 [1,8]
+ CRUSH rule 5 x 393 [4,2]
+ CRUSH rule 5 x 394 [4,7]
+ CRUSH rule 5 x 395 [4,0]
+ CRUSH rule 5 x 396 [4,2]
+ CRUSH rule 5 x 397 [2,4]
+ CRUSH rule 5 x 398 [2,4]
+ CRUSH rule 5 x 399 [8,4]
+ CRUSH rule 5 x 400 [8,1]
+ CRUSH rule 5 x 401 [0,5]
+ CRUSH rule 5 x 402 [7,5]
+ CRUSH rule 5 x 403 [0,3]
+ CRUSH rule 5 x 404 [4,2]
+ CRUSH rule 5 x 405 [6,5]
+ CRUSH rule 5 x 406 [2,6]
+ CRUSH rule 5 x 407 [2,8]
+ CRUSH rule 5 x 408 [4,1]
+ CRUSH rule 5 x 409 [7,3]
+ CRUSH rule 5 x 410 [8,3]
+ CRUSH rule 5 x 411 [2,8]
+ CRUSH rule 5 x 412 [0,5]
+ CRUSH rule 5 x 413 [5,0]
+ CRUSH rule 5 x 414 [4,1]
+ CRUSH rule 5 x 415 [0,6]
+ CRUSH rule 5 x 416 [2,6]
+ CRUSH rule 5 x 417 [8,2]
+ CRUSH rule 5 x 418 [7,1]
+ CRUSH rule 5 x 419 [8,3]
+ CRUSH rule 5 x 420 [1,4]
+ CRUSH rule 5 x 421 [8,4]
+ CRUSH rule 5 x 422 [6,4]
+ CRUSH rule 5 x 423 [0,5]
+ CRUSH rule 5 x 424 [8,4]
+ CRUSH rule 5 x 425 [1,3]
+ CRUSH rule 5 x 426 [6,0]
+ CRUSH rule 5 x 427 [0,7]
+ CRUSH rule 5 x 428 [5,7]
+ CRUSH rule 5 x 429 [4,6]
+ CRUSH rule 5 x 430 [3,6]
+ CRUSH rule 5 x 431 [5,0]
+ CRUSH rule 5 x 432 [7,1]
+ CRUSH rule 5 x 433 [6,5]
+ CRUSH rule 5 x 434 [5,2]
+ CRUSH rule 5 x 435 [0,5]
+ CRUSH rule 5 x 436 [4,0]
+ CRUSH rule 5 x 437 [7,5]
+ CRUSH rule 5 x 438 [0,3]
+ CRUSH rule 5 x 439 [1,3]
+ CRUSH rule 5 x 440 [2,7]
+ CRUSH rule 5 x 441 [5,7]
+ CRUSH rule 5 x 442 [2,4]
+ CRUSH rule 5 x 443 [6,0]
+ CRUSH rule 5 x 444 [7,0]
+ CRUSH rule 5 x 445 [6,3]
+ CRUSH rule 5 x 446 [4,1]
+ CRUSH rule 5 x 447 [2,3]
+ CRUSH rule 5 x 448 [7,2]
+ CRUSH rule 5 x 449 [7,5]
+ CRUSH rule 5 x 450 [4,1]
+ CRUSH rule 5 x 451 [6,5]
+ CRUSH rule 5 x 452 [8,3]
+ CRUSH rule 5 x 453 [6,5]
+ CRUSH rule 5 x 454 [6,4]
+ CRUSH rule 5 x 455 [2,7]
+ CRUSH rule 5 x 456 [6,2]
+ CRUSH rule 5 x 457 [7,2]
+ CRUSH rule 5 x 458 [2,8]
+ CRUSH rule 5 x 459 [2,7]
+ CRUSH rule 5 x 460 [6,5]
+ CRUSH rule 5 x 461 [6,5]
+ CRUSH rule 5 x 462 [8,1]
+ CRUSH rule 5 x 463 [6,0]
+ CRUSH rule 5 x 464 [7,4]
+ CRUSH rule 5 x 465 [7,2]
+ CRUSH rule 5 x 466 [5,8]
+ CRUSH rule 5 x 467 [6,4]
+ CRUSH rule 5 x 468 [7,0]
+ CRUSH rule 5 x 469 [7,0]
+ CRUSH rule 5 x 470 [3,0]
+ CRUSH rule 5 x 471 [0,7]
+ CRUSH rule 5 x 472 [5,1]
+ CRUSH rule 5 x 473 [1,4]
+ CRUSH rule 5 x 474 [6,0]
+ CRUSH rule 5 x 475 [6,2]
+ CRUSH rule 5 x 476 [4,6]
+ CRUSH rule 5 x 477 [5,8]
+ CRUSH rule 5 x 478 [6,2]
+ CRUSH rule 5 x 479 [0,5]
+ CRUSH rule 5 x 480 [1,8]
+ CRUSH rule 5 x 481 [2,4]
+ CRUSH rule 5 x 482 [4,7]
+ CRUSH rule 5 x 483 [0,6]
+ CRUSH rule 5 x 484 [1,7]
+ CRUSH rule 5 x 485 [4,7]
+ CRUSH rule 5 x 486 [4,1]
+ CRUSH rule 5 x 487 [5,0]
+ CRUSH rule 5 x 488 [5,7]
+ CRUSH rule 5 x 489 [2,8]
+ CRUSH rule 5 x 490 [6,4]
+ CRUSH rule 5 x 491 [1,7]
+ CRUSH rule 5 x 492 [6,5]
+ CRUSH rule 5 x 493 [0,7]
+ CRUSH rule 5 x 494 [1,7]
+ CRUSH rule 5 x 495 [3,1]
+ CRUSH rule 5 x 496 [7,5]
+ CRUSH rule 5 x 497 [5,7]
+ CRUSH rule 5 x 498 [0,5]
+ CRUSH rule 5 x 499 [8,4]
+ CRUSH rule 5 x 500 [3,6]
+ CRUSH rule 5 x 501 [0,7]
+ CRUSH rule 5 x 502 [7,1]
+ CRUSH rule 5 x 503 [2,3]
+ CRUSH rule 5 x 504 [5,6]
+ CRUSH rule 5 x 505 [0,7]
+ CRUSH rule 5 x 506 [5,2]
+ CRUSH rule 5 x 507 [6,0]
+ CRUSH rule 5 x 508 [0,3]
+ CRUSH rule 5 x 509 [7,5]
+ CRUSH rule 5 x 510 [6,0]
+ CRUSH rule 5 x 511 [5,8]
+ CRUSH rule 5 x 512 [7,0]
+ CRUSH rule 5 x 513 [7,2]
+ CRUSH rule 5 x 514 [4,6]
+ CRUSH rule 5 x 515 [8,5]
+ CRUSH rule 5 x 516 [4,0]
+ CRUSH rule 5 x 517 [7,2]
+ CRUSH rule 5 x 518 [4,6]
+ CRUSH rule 5 x 519 [7,3]
+ CRUSH rule 5 x 520 [2,6]
+ CRUSH rule 5 x 521 [8,0]
+ CRUSH rule 5 x 522 [6,0]
+ CRUSH rule 5 x 523 [4,2]
+ CRUSH rule 5 x 524 [0,4]
+ CRUSH rule 5 x 525 [0,4]
+ CRUSH rule 5 x 526 [1,5]
+ CRUSH rule 5 x 527 [0,5]
+ CRUSH rule 5 x 528 [5,0]
+ CRUSH rule 5 x 529 [5,7]
+ CRUSH rule 5 x 530 [6,5]
+ CRUSH rule 5 x 531 [6,1]
+ CRUSH rule 5 x 532 [6,3]
+ CRUSH rule 5 x 533 [5,6]
+ CRUSH rule 5 x 534 [7,3]
+ CRUSH rule 5 x 535 [8,1]
+ CRUSH rule 5 x 536 [6,2]
+ CRUSH rule 5 x 537 [3,7]
+ CRUSH rule 5 x 538 [6,3]
+ CRUSH rule 5 x 539 [8,3]
+ CRUSH rule 5 x 540 [0,6]
+ CRUSH rule 5 x 541 [2,3]
+ CRUSH rule 5 x 542 [3,2]
+ CRUSH rule 5 x 543 [6,0]
+ CRUSH rule 5 x 544 [3,7]
+ CRUSH rule 5 x 545 [5,7]
+ CRUSH rule 5 x 546 [6,1]
+ CRUSH rule 5 x 547 [8,2]
+ CRUSH rule 5 x 548 [5,2]
+ CRUSH rule 5 x 549 [5,8]
+ CRUSH rule 5 x 550 [0,5]
+ CRUSH rule 5 x 551 [7,5]
+ CRUSH rule 5 x 552 [5,8]
+ CRUSH rule 5 x 553 [4,2]
+ CRUSH rule 5 x 554 [0,8]
+ CRUSH rule 5 x 555 [5,0]
+ CRUSH rule 5 x 556 [3,6]
+ CRUSH rule 5 x 557 [7,4]
+ CRUSH rule 5 x 558 [3,1]
+ CRUSH rule 5 x 559 [4,2]
+ CRUSH rule 5 x 560 [8,3]
+ CRUSH rule 5 x 561 [6,3]
+ CRUSH rule 5 x 562 [3,0]
+ CRUSH rule 5 x 563 [2,6]
+ CRUSH rule 5 x 564 [5,1]
+ CRUSH rule 5 x 565 [3,6]
+ CRUSH rule 5 x 566 [4,7]
+ CRUSH rule 5 x 567 [3,6]
+ CRUSH rule 5 x 568 [7,4]
+ CRUSH rule 5 x 569 [3,1]
+ CRUSH rule 5 x 570 [1,5]
+ CRUSH rule 5 x 571 [3,7]
+ CRUSH rule 5 x 572 [3,2]
+ CRUSH rule 5 x 573 [3,0]
+ CRUSH rule 5 x 574 [2,5]
+ CRUSH rule 5 x 575 [8,2]
+ CRUSH rule 5 x 576 [4,6]
+ CRUSH rule 5 x 577 [8,2]
+ CRUSH rule 5 x 578 [6,1]
+ CRUSH rule 5 x 579 [3,1]
+ CRUSH rule 5 x 580 [3,0]
+ CRUSH rule 5 x 581 [7,2]
+ CRUSH rule 5 x 582 [2,8]
+ CRUSH rule 5 x 583 [6,0]
+ CRUSH rule 5 x 584 [8,1]
+ CRUSH rule 5 x 585 [7,0]
+ CRUSH rule 5 x 586 [0,7]
+ CRUSH rule 5 x 587 [2,5]
+ CRUSH rule 5 x 588 [3,7]
+ CRUSH rule 5 x 589 [7,1]
+ CRUSH rule 5 x 590 [6,2]
+ CRUSH rule 5 x 591 [5,2]
+ CRUSH rule 5 x 592 [2,4]
+ CRUSH rule 5 x 593 [0,8]
+ CRUSH rule 5 x 594 [0,7]
+ CRUSH rule 5 x 595 [7,1]
+ CRUSH rule 5 x 596 [4,0]
+ CRUSH rule 5 x 597 [3,1]
+ CRUSH rule 5 x 598 [3,2]
+ CRUSH rule 5 x 599 [5,2]
+ CRUSH rule 5 x 600 [7,0]
+ CRUSH rule 5 x 601 [0,7]
+ CRUSH rule 5 x 602 [3,7]
+ CRUSH rule 5 x 603 [5,1]
+ CRUSH rule 5 x 604 [7,5]
+ CRUSH rule 5 x 605 [3,0]
+ CRUSH rule 5 x 606 [2,7]
+ CRUSH rule 5 x 607 [0,4]
+ CRUSH rule 5 x 608 [5,2]
+ CRUSH rule 5 x 609 [5,2]
+ CRUSH rule 5 x 610 [3,7]
+ CRUSH rule 5 x 611 [1,8]
+ CRUSH rule 5 x 612 [2,6]
+ CRUSH rule 5 x 613 [7,2]
+ CRUSH rule 5 x 614 [7,2]
+ CRUSH rule 5 x 615 [6,0]
+ CRUSH rule 5 x 616 [0,8]
+ CRUSH rule 5 x 617 [6,1]
+ CRUSH rule 5 x 618 [7,4]
+ CRUSH rule 5 x 619 [5,1]
+ CRUSH rule 5 x 620 [4,1]
+ CRUSH rule 5 x 621 [5,8]
+ CRUSH rule 5 x 622 [0,4]
+ CRUSH rule 5 x 623 [0,6]
+ CRUSH rule 5 x 624 [3,2]
+ CRUSH rule 5 x 625 [2,3]
+ CRUSH rule 5 x 626 [7,0]
+ CRUSH rule 5 x 627 [2,7]
+ CRUSH rule 5 x 628 [8,0]
+ CRUSH rule 5 x 629 [2,6]
+ CRUSH rule 5 x 630 [2,7]
+ CRUSH rule 5 x 631 [0,7]
+ CRUSH rule 5 x 632 [7,0]
+ CRUSH rule 5 x 633 [8,3]
+ CRUSH rule 5 x 634 [0,4]
+ CRUSH rule 5 x 635 [5,6]
+ CRUSH rule 5 x 636 [1,4]
+ CRUSH rule 5 x 637 [4,1]
+ CRUSH rule 5 x 638 [6,0]
+ CRUSH rule 5 x 639 [4,2]
+ CRUSH rule 5 x 640 [3,1]
+ CRUSH rule 5 x 641 [7,2]
+ CRUSH rule 5 x 642 [2,7]
+ CRUSH rule 5 x 643 [3,0]
+ CRUSH rule 5 x 644 [8,1]
+ CRUSH rule 5 x 645 [5,7]
+ CRUSH rule 5 x 646 [8,0]
+ CRUSH rule 5 x 647 [7,1]
+ CRUSH rule 5 x 648 [0,6]
+ CRUSH rule 5 x 649 [4,7]
+ CRUSH rule 5 x 650 [7,3]
+ CRUSH rule 5 x 651 [3,7]
+ CRUSH rule 5 x 652 [3,7]
+ CRUSH rule 5 x 653 [8,5]
+ CRUSH rule 5 x 654 [7,5]
+ CRUSH rule 5 x 655 [0,3]
+ CRUSH rule 5 x 656 [4,7]
+ CRUSH rule 5 x 657 [6,1]
+ CRUSH rule 5 x 658 [5,6]
+ CRUSH rule 5 x 659 [4,6]
+ CRUSH rule 5 x 660 [7,4]
+ CRUSH rule 5 x 661 [1,8]
+ CRUSH rule 5 x 662 [4,2]
+ CRUSH rule 5 x 663 [1,3]
+ CRUSH rule 5 x 664 [1,4]
+ CRUSH rule 5 x 665 [5,7]
+ CRUSH rule 5 x 666 [2,8]
+ CRUSH rule 5 x 667 [1,3]
+ CRUSH rule 5 x 668 [3,7]
+ CRUSH rule 5 x 669 [6,4]
+ CRUSH rule 5 x 670 [4,0]
+ CRUSH rule 5 x 671 [0,7]
+ CRUSH rule 5 x 672 [4,2]
+ CRUSH rule 5 x 673 [5,2]
+ CRUSH rule 5 x 674 [3,1]
+ CRUSH rule 5 x 675 [0,8]
+ CRUSH rule 5 x 676 [0,4]
+ CRUSH rule 5 x 677 [4,1]
+ CRUSH rule 5 x 678 [2,3]
+ CRUSH rule 5 x 679 [6,0]
+ CRUSH rule 5 x 680 [0,4]
+ CRUSH rule 5 x 681 [4,7]
+ CRUSH rule 5 x 682 [0,5]
+ CRUSH rule 5 x 683 [0,5]
+ CRUSH rule 5 x 684 [7,1]
+ CRUSH rule 5 x 685 [7,1]
+ CRUSH rule 5 x 686 [1,4]
+ CRUSH rule 5 x 687 [3,6]
+ CRUSH rule 5 x 688 [5,7]
+ CRUSH rule 5 x 689 [6,5]
+ CRUSH rule 5 x 690 [8,1]
+ CRUSH rule 5 x 691 [3,0]
+ CRUSH rule 5 x 692 [7,2]
+ CRUSH rule 5 x 693 [6,3]
+ CRUSH rule 5 x 694 [6,5]
+ CRUSH rule 5 x 695 [0,8]
+ CRUSH rule 5 x 696 [1,4]
+ CRUSH rule 5 x 697 [6,1]
+ CRUSH rule 5 x 698 [6,2]
+ CRUSH rule 5 x 699 [1,6]
+ CRUSH rule 5 x 700 [0,3]
+ CRUSH rule 5 x 701 [4,1]
+ CRUSH rule 5 x 702 [3,2]
+ CRUSH rule 5 x 703 [8,3]
+ CRUSH rule 5 x 704 [0,3]
+ CRUSH rule 5 x 705 [8,0]
+ CRUSH rule 5 x 706 [1,5]
+ CRUSH rule 5 x 707 [7,3]
+ CRUSH rule 5 x 708 [3,7]
+ CRUSH rule 5 x 709 [6,3]
+ CRUSH rule 5 x 710 [8,4]
+ CRUSH rule 5 x 711 [2,3]
+ CRUSH rule 5 x 712 [2,3]
+ CRUSH rule 5 x 713 [6,3]
+ CRUSH rule 5 x 714 [3,2]
+ CRUSH rule 5 x 715 [1,3]
+ CRUSH rule 5 x 716 [3,6]
+ CRUSH rule 5 x 717 [8,2]
+ CRUSH rule 5 x 718 [3,7]
+ CRUSH rule 5 x 719 [2,6]
+ CRUSH rule 5 x 720 [6,1]
+ CRUSH rule 5 x 721 [5,7]
+ CRUSH rule 5 x 722 [5,7]
+ CRUSH rule 5 x 723 [5,1]
+ CRUSH rule 5 x 724 [0,6]
+ CRUSH rule 5 x 725 [0,3]
+ CRUSH rule 5 x 726 [3,8]
+ CRUSH rule 5 x 727 [4,6]
+ CRUSH rule 5 x 728 [2,7]
+ CRUSH rule 5 x 729 [5,6]
+ CRUSH rule 5 x 730 [3,7]
+ CRUSH rule 5 x 731 [4,1]
+ CRUSH rule 5 x 732 [1,5]
+ CRUSH rule 5 x 733 [5,7]
+ CRUSH rule 5 x 734 [6,4]
+ CRUSH rule 5 x 735 [4,8]
+ CRUSH rule 5 x 736 [3,8]
+ CRUSH rule 5 x 737 [1,6]
+ CRUSH rule 5 x 738 [5,2]
+ CRUSH rule 5 x 739 [0,7]
+ CRUSH rule 5 x 740 [0,8]
+ CRUSH rule 5 x 741 [7,1]
+ CRUSH rule 5 x 742 [8,2]
+ CRUSH rule 5 x 743 [7,0]
+ CRUSH rule 5 x 744 [4,7]
+ CRUSH rule 5 x 745 [3,1]
+ CRUSH rule 5 x 746 [4,1]
+ CRUSH rule 5 x 747 [6,0]
+ CRUSH rule 5 x 748 [2,7]
+ CRUSH rule 5 x 749 [4,8]
+ CRUSH rule 5 x 750 [1,6]
+ CRUSH rule 5 x 751 [2,8]
+ CRUSH rule 5 x 752 [8,1]
+ CRUSH rule 5 x 753 [7,3]
+ CRUSH rule 5 x 754 [8,5]
+ CRUSH rule 5 x 755 [1,6]
+ CRUSH rule 5 x 756 [5,6]
+ CRUSH rule 5 x 757 [8,0]
+ CRUSH rule 5 x 758 [6,0]
+ CRUSH rule 5 x 759 [8,5]
+ CRUSH rule 5 x 760 [1,5]
+ CRUSH rule 5 x 761 [4,1]
+ CRUSH rule 5 x 762 [2,7]
+ CRUSH rule 5 x 763 [8,5]
+ CRUSH rule 5 x 764 [1,7]
+ CRUSH rule 5 x 765 [6,5]
+ CRUSH rule 5 x 766 [8,5]
+ CRUSH rule 5 x 767 [1,8]
+ CRUSH rule 5 x 768 [8,3]
+ CRUSH rule 5 x 769 [6,2]
+ CRUSH rule 5 x 770 [6,0]
+ CRUSH rule 5 x 771 [7,0]
+ CRUSH rule 5 x 772 [8,3]
+ CRUSH rule 5 x 773 [3,1]
+ CRUSH rule 5 x 774 [4,6]
+ CRUSH rule 5 x 775 [6,4]
+ CRUSH rule 5 x 776 [7,2]
+ CRUSH rule 5 x 777 [3,1]
+ CRUSH rule 5 x 778 [1,8]
+ CRUSH rule 5 x 779 [2,7]
+ CRUSH rule 5 x 780 [0,5]
+ CRUSH rule 5 x 781 [6,3]
+ CRUSH rule 5 x 782 [5,0]
+ CRUSH rule 5 x 783 [7,1]
+ CRUSH rule 5 x 784 [0,4]
+ CRUSH rule 5 x 785 [6,1]
+ CRUSH rule 5 x 786 [7,3]
+ CRUSH rule 5 x 787 [1,6]
+ CRUSH rule 5 x 788 [6,0]
+ CRUSH rule 5 x 789 [0,4]
+ CRUSH rule 5 x 790 [8,4]
+ CRUSH rule 5 x 791 [3,8]
+ CRUSH rule 5 x 792 [5,8]
+ CRUSH rule 5 x 793 [6,1]
+ CRUSH rule 5 x 794 [2,6]
+ CRUSH rule 5 x 795 [0,3]
+ CRUSH rule 5 x 796 [3,7]
+ CRUSH rule 5 x 797 [2,3]
+ CRUSH rule 5 x 798 [6,1]
+ CRUSH rule 5 x 799 [5,1]
+ CRUSH rule 5 x 800 [5,0]
+ CRUSH rule 5 x 801 [3,6]
+ CRUSH rule 5 x 802 [1,8]
+ CRUSH rule 5 x 803 [0,5]
+ CRUSH rule 5 x 804 [6,2]
+ CRUSH rule 5 x 805 [3,6]
+ CRUSH rule 5 x 806 [1,3]
+ CRUSH rule 5 x 807 [5,7]
+ CRUSH rule 5 x 808 [4,6]
+ CRUSH rule 5 x 809 [1,4]
+ CRUSH rule 5 x 810 [5,7]
+ CRUSH rule 5 x 811 [8,4]
+ CRUSH rule 5 x 812 [8,5]
+ CRUSH rule 5 x 813 [6,4]
+ CRUSH rule 5 x 814 [3,6]
+ CRUSH rule 5 x 815 [3,1]
+ CRUSH rule 5 x 816 [2,7]
+ CRUSH rule 5 x 817 [4,8]
+ CRUSH rule 5 x 818 [3,0]
+ CRUSH rule 5 x 819 [5,1]
+ CRUSH rule 5 x 820 [3,6]
+ CRUSH rule 5 x 821 [4,6]
+ CRUSH rule 5 x 822 [2,5]
+ CRUSH rule 5 x 823 [4,8]
+ CRUSH rule 5 x 824 [3,7]
+ CRUSH rule 5 x 825 [2,8]
+ CRUSH rule 5 x 826 [7,0]
+ CRUSH rule 5 x 827 [0,8]
+ CRUSH rule 5 x 828 [2,3]
+ CRUSH rule 5 x 829 [5,6]
+ CRUSH rule 5 x 830 [2,3]
+ CRUSH rule 5 x 831 [1,6]
+ CRUSH rule 5 x 832 [4,7]
+ CRUSH rule 5 x 833 [2,7]
+ CRUSH rule 5 x 834 [3,1]
+ CRUSH rule 5 x 835 [8,4]
+ CRUSH rule 5 x 836 [3,7]
+ CRUSH rule 5 x 837 [6,3]
+ CRUSH rule 5 x 838 [6,2]
+ CRUSH rule 5 x 839 [5,0]
+ CRUSH rule 5 x 840 [7,3]
+ CRUSH rule 5 x 841 [4,8]
+ CRUSH rule 5 x 842 [2,4]
+ CRUSH rule 5 x 843 [6,4]
+ CRUSH rule 5 x 844 [4,8]
+ CRUSH rule 5 x 845 [3,8]
+ CRUSH rule 5 x 846 [3,2]
+ CRUSH rule 5 x 847 [0,8]
+ CRUSH rule 5 x 848 [2,6]
+ CRUSH rule 5 x 849 [4,6]
+ CRUSH rule 5 x 850 [1,3]
+ CRUSH rule 5 x 851 [6,4]
+ CRUSH rule 5 x 852 [7,3]
+ CRUSH rule 5 x 853 [6,0]
+ CRUSH rule 5 x 854 [7,0]
+ CRUSH rule 5 x 855 [5,7]
+ CRUSH rule 5 x 856 [6,3]
+ CRUSH rule 5 x 857 [8,5]
+ CRUSH rule 5 x 858 [6,4]
+ CRUSH rule 5 x 859 [6,0]
+ CRUSH rule 5 x 860 [4,1]
+ CRUSH rule 5 x 861 [8,3]
+ CRUSH rule 5 x 862 [6,1]
+ CRUSH rule 5 x 863 [8,2]
+ CRUSH rule 5 x 864 [5,6]
+ CRUSH rule 5 x 865 [8,1]
+ CRUSH rule 5 x 866 [3,6]
+ CRUSH rule 5 x 867 [6,5]
+ CRUSH rule 5 x 868 [6,3]
+ CRUSH rule 5 x 869 [8,5]
+ CRUSH rule 5 x 870 [0,4]
+ CRUSH rule 5 x 871 [3,2]
+ CRUSH rule 5 x 872 [5,1]
+ CRUSH rule 5 x 873 [4,6]
+ CRUSH rule 5 x 874 [2,6]
+ CRUSH rule 5 x 875 [2,6]
+ CRUSH rule 5 x 876 [5,8]
+ CRUSH rule 5 x 877 [6,4]
+ CRUSH rule 5 x 878 [5,2]
+ CRUSH rule 5 x 879 [7,4]
+ CRUSH rule 5 x 880 [3,2]
+ CRUSH rule 5 x 881 [5,6]
+ CRUSH rule 5 x 882 [4,0]
+ CRUSH rule 5 x 883 [2,3]
+ CRUSH rule 5 x 884 [6,0]
+ CRUSH rule 5 x 885 [5,1]
+ CRUSH rule 5 x 886 [3,6]
+ CRUSH rule 5 x 887 [7,4]
+ CRUSH rule 5 x 888 [6,2]
+ CRUSH rule 5 x 889 [2,6]
+ CRUSH rule 5 x 890 [7,2]
+ CRUSH rule 5 x 891 [1,8]
+ CRUSH rule 5 x 892 [6,2]
+ CRUSH rule 5 x 893 [2,3]
+ CRUSH rule 5 x 894 [7,5]
+ CRUSH rule 5 x 895 [5,1]
+ CRUSH rule 5 x 896 [1,8]
+ CRUSH rule 5 x 897 [4,2]
+ CRUSH rule 5 x 898 [0,5]
+ CRUSH rule 5 x 899 [1,7]
+ CRUSH rule 5 x 900 [4,1]
+ CRUSH rule 5 x 901 [5,0]
+ CRUSH rule 5 x 902 [8,5]
+ CRUSH rule 5 x 903 [5,7]
+ CRUSH rule 5 x 904 [5,6]
+ CRUSH rule 5 x 905 [6,2]
+ CRUSH rule 5 x 906 [1,6]
+ CRUSH rule 5 x 907 [7,1]
+ CRUSH rule 5 x 908 [5,8]
+ CRUSH rule 5 x 909 [2,3]
+ CRUSH rule 5 x 910 [6,4]
+ CRUSH rule 5 x 911 [5,8]
+ CRUSH rule 5 x 912 [0,7]
+ CRUSH rule 5 x 913 [7,2]
+ CRUSH rule 5 x 914 [6,4]
+ CRUSH rule 5 x 915 [8,2]
+ CRUSH rule 5 x 916 [3,1]
+ CRUSH rule 5 x 917 [1,5]
+ CRUSH rule 5 x 918 [8,2]
+ CRUSH rule 5 x 919 [6,2]
+ CRUSH rule 5 x 920 [7,4]
+ CRUSH rule 5 x 921 [1,4]
+ CRUSH rule 5 x 922 [6,4]
+ CRUSH rule 5 x 923 [5,8]
+ CRUSH rule 5 x 924 [3,1]
+ CRUSH rule 5 x 925 [5,7]
+ CRUSH rule 5 x 926 [3,0]
+ CRUSH rule 5 x 927 [1,6]
+ CRUSH rule 5 x 928 [8,1]
+ CRUSH rule 5 x 929 [4,1]
+ CRUSH rule 5 x 930 [2,4]
+ CRUSH rule 5 x 931 [5,0]
+ CRUSH rule 5 x 932 [4,1]
+ CRUSH rule 5 x 933 [8,5]
+ CRUSH rule 5 x 934 [5,6]
+ CRUSH rule 5 x 935 [6,3]
+ CRUSH rule 5 x 936 [0,6]
+ CRUSH rule 5 x 937 [5,8]
+ CRUSH rule 5 x 938 [6,5]
+ CRUSH rule 5 x 939 [2,7]
+ CRUSH rule 5 x 940 [8,5]
+ CRUSH rule 5 x 941 [5,2]
+ CRUSH rule 5 x 942 [1,8]
+ CRUSH rule 5 x 943 [8,2]
+ CRUSH rule 5 x 944 [4,8]
+ CRUSH rule 5 x 945 [7,2]
+ CRUSH rule 5 x 946 [2,8]
+ CRUSH rule 5 x 947 [4,2]
+ CRUSH rule 5 x 948 [7,5]
+ CRUSH rule 5 x 949 [6,1]
+ CRUSH rule 5 x 950 [3,6]
+ CRUSH rule 5 x 951 [4,8]
+ CRUSH rule 5 x 952 [2,7]
+ CRUSH rule 5 x 953 [1,3]
+ CRUSH rule 5 x 954 [4,2]
+ CRUSH rule 5 x 955 [8,0]
+ CRUSH rule 5 x 956 [1,6]
+ CRUSH rule 5 x 957 [7,1]
+ CRUSH rule 5 x 958 [8,4]
+ CRUSH rule 5 x 959 [5,2]
+ CRUSH rule 5 x 960 [3,6]
+ CRUSH rule 5 x 961 [4,0]
+ CRUSH rule 5 x 962 [7,4]
+ CRUSH rule 5 x 963 [0,5]
+ CRUSH rule 5 x 964 [3,1]
+ CRUSH rule 5 x 965 [7,4]
+ CRUSH rule 5 x 966 [3,8]
+ CRUSH rule 5 x 967 [8,5]
+ CRUSH rule 5 x 968 [7,2]
+ CRUSH rule 5 x 969 [8,0]
+ CRUSH rule 5 x 970 [0,6]
+ CRUSH rule 5 x 971 [1,7]
+ CRUSH rule 5 x 972 [1,8]
+ CRUSH rule 5 x 973 [1,6]
+ CRUSH rule 5 x 974 [5,1]
+ CRUSH rule 5 x 975 [3,7]
+ CRUSH rule 5 x 976 [4,8]
+ CRUSH rule 5 x 977 [8,3]
+ CRUSH rule 5 x 978 [7,2]
+ CRUSH rule 5 x 979 [7,1]
+ CRUSH rule 5 x 980 [6,0]
+ CRUSH rule 5 x 981 [7,3]
+ CRUSH rule 5 x 982 [4,2]
+ CRUSH rule 5 x 983 [3,7]
+ CRUSH rule 5 x 984 [0,7]
+ CRUSH rule 5 x 985 [2,5]
+ CRUSH rule 5 x 986 [8,3]
+ CRUSH rule 5 x 987 [0,5]
+ CRUSH rule 5 x 988 [1,3]
+ CRUSH rule 5 x 989 [0,6]
+ CRUSH rule 5 x 990 [1,6]
+ CRUSH rule 5 x 991 [0,4]
+ CRUSH rule 5 x 992 [7,1]
+ CRUSH rule 5 x 993 [0,6]
+ CRUSH rule 5 x 994 [3,7]
+ CRUSH rule 5 x 995 [7,1]
+ CRUSH rule 5 x 996 [6,5]
+ CRUSH rule 5 x 997 [6,4]
+ CRUSH rule 5 x 998 [8,1]
+ CRUSH rule 5 x 999 [0,7]
+ CRUSH rule 5 x 1000 [8,5]
+ CRUSH rule 5 x 1001 [2,5]
+ CRUSH rule 5 x 1002 [1,3]
+ CRUSH rule 5 x 1003 [2,8]
+ CRUSH rule 5 x 1004 [6,1]
+ CRUSH rule 5 x 1005 [6,1]
+ CRUSH rule 5 x 1006 [1,6]
+ CRUSH rule 5 x 1007 [1,5]
+ CRUSH rule 5 x 1008 [1,7]
+ CRUSH rule 5 x 1009 [6,4]
+ CRUSH rule 5 x 1010 [3,1]
+ CRUSH rule 5 x 1011 [3,0]
+ CRUSH rule 5 x 1012 [3,0]
+ CRUSH rule 5 x 1013 [5,1]
+ CRUSH rule 5 x 1014 [2,8]
+ CRUSH rule 5 x 1015 [6,5]
+ CRUSH rule 5 x 1016 [2,4]
+ CRUSH rule 5 x 1017 [6,0]
+ CRUSH rule 5 x 1018 [5,0]
+ CRUSH rule 5 x 1019 [5,8]
+ CRUSH rule 5 x 1020 [5,1]
+ CRUSH rule 5 x 1021 [5,2]
+ CRUSH rule 5 x 1022 [1,6]
+ CRUSH rule 5 x 1023 [3,2]
+ rule 5 (chooseleaf-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 5 x 0 [0,5,7]
+ CRUSH rule 5 x 1 [0,8,5]
+ CRUSH rule 5 x 2 [1,3,7]
+ CRUSH rule 5 x 3 [8,0,4]
+ CRUSH rule 5 x 4 [5,0,7]
+ CRUSH rule 5 x 5 [7,0,4]
+ CRUSH rule 5 x 6 [2,6,3]
+ CRUSH rule 5 x 7 [5,8,2]
+ CRUSH rule 5 x 8 [5,6,0]
+ CRUSH rule 5 x 9 [2,3,8]
+ CRUSH rule 5 x 10 [0,7,4]
+ CRUSH rule 5 x 11 [0,7,4]
+ CRUSH rule 5 x 12 [0,4,6]
+ CRUSH rule 5 x 13 [3,8,1]
+ CRUSH rule 5 x 14 [7,0,5]
+ CRUSH rule 5 x 15 [7,2,4]
+ CRUSH rule 5 x 16 [3,6,0]
+ CRUSH rule 5 x 17 [5,1,6]
+ CRUSH rule 5 x 18 [1,4,6]
+ CRUSH rule 5 x 19 [7,5,0]
+ CRUSH rule 5 x 20 [2,4,7]
+ CRUSH rule 5 x 21 [3,7,2]
+ CRUSH rule 5 x 22 [8,3,1]
+ CRUSH rule 5 x 23 [3,6,2]
+ CRUSH rule 5 x 24 [1,7,3]
+ CRUSH rule 5 x 25 [3,7,0]
+ CRUSH rule 5 x 26 [2,8,4]
+ CRUSH rule 5 x 27 [3,1,8]
+ CRUSH rule 5 x 28 [6,0,3]
+ CRUSH rule 5 x 29 [8,5,2]
+ CRUSH rule 5 x 30 [5,7,0]
+ CRUSH rule 5 x 31 [8,0,4]
+ CRUSH rule 5 x 32 [3,6,2]
+ CRUSH rule 5 x 33 [2,7,5]
+ CRUSH rule 5 x 34 [2,5,7]
+ CRUSH rule 5 x 35 [0,8,5]
+ CRUSH rule 5 x 36 [3,8,2]
+ CRUSH rule 5 x 37 [0,4,6]
+ CRUSH rule 5 x 38 [4,8,2]
+ CRUSH rule 5 x 39 [3,7,0]
+ CRUSH rule 5 x 40 [7,2,3]
+ CRUSH rule 5 x 41 [0,6,4]
+ CRUSH rule 5 x 42 [4,6,2]
+ CRUSH rule 5 x 43 [0,3,7]
+ CRUSH rule 5 x 44 [1,6,5]
+ CRUSH rule 5 x 45 [8,0,4]
+ CRUSH rule 5 x 46 [2,4,8]
+ CRUSH rule 5 x 47 [4,2,8]
+ CRUSH rule 5 x 48 [4,6,2]
+ CRUSH rule 5 x 49 [5,7,2]
+ CRUSH rule 5 x 50 [3,1,7]
+ CRUSH rule 5 x 51 [3,6,0]
+ CRUSH rule 5 x 52 [8,1,4]
+ CRUSH rule 5 x 53 [3,8,0]
+ CRUSH rule 5 x 54 [7,3,1]
+ CRUSH rule 5 x 55 [8,2,4]
+ CRUSH rule 5 x 56 [6,4,2]
+ CRUSH rule 5 x 57 [5,8,1]
+ CRUSH rule 5 x 58 [1,8,5]
+ CRUSH rule 5 x 59 [4,2,7]
+ CRUSH rule 5 x 60 [3,2,6]
+ CRUSH rule 5 x 61 [4,6,0]
+ CRUSH rule 5 x 62 [7,0,5]
+ CRUSH rule 5 x 63 [5,6,0]
+ CRUSH rule 5 x 64 [4,2,8]
+ CRUSH rule 5 x 65 [7,3,0]
+ CRUSH rule 5 x 66 [5,6,2]
+ CRUSH rule 5 x 67 [5,0,8]
+ CRUSH rule 5 x 68 [0,5,8]
+ CRUSH rule 5 x 69 [5,1,6]
+ CRUSH rule 5 x 70 [7,0,5]
+ CRUSH rule 5 x 71 [2,8,5]
+ CRUSH rule 5 x 72 [6,1,5]
+ CRUSH rule 5 x 73 [2,7,3]
+ CRUSH rule 5 x 74 [0,7,3]
+ CRUSH rule 5 x 75 [3,2,6]
+ CRUSH rule 5 x 76 [5,1,7]
+ CRUSH rule 5 x 77 [7,2,5]
+ CRUSH rule 5 x 78 [1,4,8]
+ CRUSH rule 5 x 79 [5,1,7]
+ CRUSH rule 5 x 80 [0,3,6]
+ CRUSH rule 5 x 81 [0,3,6]
+ CRUSH rule 5 x 82 [7,1,5]
+ CRUSH rule 5 x 83 [2,6,3]
+ CRUSH rule 5 x 84 [7,2,4]
+ CRUSH rule 5 x 85 [3,8,0]
+ CRUSH rule 5 x 86 [0,6,3]
+ CRUSH rule 5 x 87 [0,7,4]
+ CRUSH rule 5 x 88 [1,6,5]
+ CRUSH rule 5 x 89 [3,0,7]
+ CRUSH rule 5 x 90 [6,4,1]
+ CRUSH rule 5 x 91 [3,8,1]
+ CRUSH rule 5 x 92 [1,8,4]
+ CRUSH rule 5 x 93 [7,4,2]
+ CRUSH rule 5 x 94 [0,4,8]
+ CRUSH rule 5 x 95 [7,5,1]
+ CRUSH rule 5 x 96 [3,6,1]
+ CRUSH rule 5 x 97 [8,4,0]
+ CRUSH rule 5 x 98 [2,7,5]
+ CRUSH rule 5 x 99 [0,7,3]
+ CRUSH rule 5 x 100 [1,7,4]
+ CRUSH rule 5 x 101 [3,7,2]
+ CRUSH rule 5 x 102 [4,2,7]
+ CRUSH rule 5 x 103 [4,7,0]
+ CRUSH rule 5 x 104 [7,4,1]
+ CRUSH rule 5 x 105 [2,4,6]
+ CRUSH rule 5 x 106 [1,6,5]
+ CRUSH rule 5 x 107 [3,2,6]
+ CRUSH rule 5 x 108 [7,2,5]
+ CRUSH rule 5 x 109 [1,4,6]
+ CRUSH rule 5 x 110 [3,2,7]
+ CRUSH rule 5 x 111 [2,3,6]
+ CRUSH rule 5 x 112 [2,6,4]
+ CRUSH rule 5 x 113 [6,2,4]
+ CRUSH rule 5 x 114 [7,3,1]
+ CRUSH rule 5 x 115 [8,2,3]
+ CRUSH rule 5 x 116 [1,6,3]
+ CRUSH rule 5 x 117 [7,3,2]
+ CRUSH rule 5 x 118 [0,3,8]
+ CRUSH rule 5 x 119 [5,6,1]
+ CRUSH rule 5 x 120 [0,3,7]
+ CRUSH rule 5 x 121 [2,7,5]
+ CRUSH rule 5 x 122 [8,5,0]
+ CRUSH rule 5 x 123 [2,5,8]
+ CRUSH rule 5 x 124 [3,2,8]
+ CRUSH rule 5 x 125 [0,7,3]
+ CRUSH rule 5 x 126 [4,2,6]
+ CRUSH rule 5 x 127 [3,6,2]
+ CRUSH rule 5 x 128 [3,6,1]
+ CRUSH rule 5 x 129 [0,3,7]
+ CRUSH rule 5 x 130 [3,8,2]
+ CRUSH rule 5 x 131 [1,3,8]
+ CRUSH rule 5 x 132 [1,4,6]
+ CRUSH rule 5 x 133 [3,6,2]
+ CRUSH rule 5 x 134 [1,8,5]
+ CRUSH rule 5 x 135 [5,6,2]
+ CRUSH rule 5 x 136 [2,3,6]
+ CRUSH rule 5 x 137 [7,3,2]
+ CRUSH rule 5 x 138 [8,4,2]
+ CRUSH rule 5 x 139 [3,0,7]
+ CRUSH rule 5 x 140 [1,6,4]
+ CRUSH rule 5 x 141 [6,2,3]
+ CRUSH rule 5 x 142 [3,0,8]
+ CRUSH rule 5 x 143 [5,8,1]
+ CRUSH rule 5 x 144 [8,1,3]
+ CRUSH rule 5 x 145 [8,5,0]
+ CRUSH rule 5 x 146 [2,6,4]
+ CRUSH rule 5 x 147 [2,8,4]
+ CRUSH rule 5 x 148 [3,1,7]
+ CRUSH rule 5 x 149 [4,8,1]
+ CRUSH rule 5 x 150 [1,6,5]
+ CRUSH rule 5 x 151 [3,6,0]
+ CRUSH rule 5 x 152 [8,4,2]
+ CRUSH rule 5 x 153 [8,4,0]
+ CRUSH rule 5 x 154 [3,2,7]
+ CRUSH rule 5 x 155 [3,7,2]
+ CRUSH rule 5 x 156 [4,2,6]
+ CRUSH rule 5 x 157 [4,1,6]
+ CRUSH rule 5 x 158 [2,8,5]
+ CRUSH rule 5 x 159 [7,0,5]
+ CRUSH rule 5 x 160 [2,8,5]
+ CRUSH rule 5 x 161 [1,5,6]
+ CRUSH rule 5 x 162 [0,6,4]
+ CRUSH rule 5 x 163 [5,6,1]
+ CRUSH rule 5 x 164 [7,1,5]
+ CRUSH rule 5 x 165 [7,0,5]
+ CRUSH rule 5 x 166 [2,4,6]
+ CRUSH rule 5 x 167 [0,7,4]
+ CRUSH rule 5 x 168 [4,2,7]
+ CRUSH rule 5 x 169 [2,6,4]
+ CRUSH rule 5 x 170 [1,4,8]
+ CRUSH rule 5 x 171 [7,5,0]
+ CRUSH rule 5 x 172 [0,7,5]
+ CRUSH rule 5 x 173 [8,5,1]
+ CRUSH rule 5 x 174 [1,4,7]
+ CRUSH rule 5 x 175 [6,0,4]
+ CRUSH rule 5 x 176 [4,1,7]
+ CRUSH rule 5 x 177 [5,1,6]
+ CRUSH rule 5 x 178 [3,0,8]
+ CRUSH rule 5 x 179 [4,1,7]
+ CRUSH rule 5 x 180 [3,8,1]
+ CRUSH rule 5 x 181 [6,2,4]
+ CRUSH rule 5 x 182 [8,5,1]
+ CRUSH rule 5 x 183 [7,5,1]
+ CRUSH rule 5 x 184 [5,7,2]
+ CRUSH rule 5 x 185 [6,1,4]
+ CRUSH rule 5 x 186 [2,5,7]
+ CRUSH rule 5 x 187 [1,6,4]
+ CRUSH rule 5 x 188 [1,8,5]
+ CRUSH rule 5 x 189 [0,7,4]
+ CRUSH rule 5 x 190 [4,0,8]
+ CRUSH rule 5 x 191 [7,1,3]
+ CRUSH rule 5 x 192 [5,0,8]
+ CRUSH rule 5 x 193 [4,2,6]
+ CRUSH rule 5 x 194 [1,3,8]
+ CRUSH rule 5 x 195 [6,4,1]
+ CRUSH rule 5 x 196 [6,0,3]
+ CRUSH rule 5 x 197 [6,5,2]
+ CRUSH rule 5 x 198 [2,5,6]
+ CRUSH rule 5 x 199 [0,5,7]
+ CRUSH rule 5 x 200 [0,5,6]
+ CRUSH rule 5 x 201 [7,1,5]
+ CRUSH rule 5 x 202 [6,3,1]
+ CRUSH rule 5 x 203 [4,8,1]
+ CRUSH rule 5 x 204 [2,3,7]
+ CRUSH rule 5 x 205 [0,7,5]
+ CRUSH rule 5 x 206 [0,7,5]
+ CRUSH rule 5 x 207 [3,0,7]
+ CRUSH rule 5 x 208 [7,1,4]
+ CRUSH rule 5 x 209 [1,8,4]
+ CRUSH rule 5 x 210 [1,4,8]
+ CRUSH rule 5 x 211 [5,2,8]
+ CRUSH rule 5 x 212 [7,5,0]
+ CRUSH rule 5 x 213 [8,4,0]
+ CRUSH rule 5 x 214 [4,8,2]
+ CRUSH rule 5 x 215 [8,1,4]
+ CRUSH rule 5 x 216 [5,0,6]
+ CRUSH rule 5 x 217 [0,7,3]
+ CRUSH rule 5 x 218 [0,7,5]
+ CRUSH rule 5 x 219 [4,8,2]
+ CRUSH rule 5 x 220 [5,7,1]
+ CRUSH rule 5 x 221 [3,6,0]
+ CRUSH rule 5 x 222 [6,4,1]
+ CRUSH rule 5 x 223 [1,3,6]
+ CRUSH rule 5 x 224 [1,5,8]
+ CRUSH rule 5 x 225 [8,2,3]
+ CRUSH rule 5 x 226 [7,2,3]
+ CRUSH rule 5 x 227 [3,1,6]
+ CRUSH rule 5 x 228 [5,6,0]
+ CRUSH rule 5 x 229 [3,8,2]
+ CRUSH rule 5 x 230 [4,7,2]
+ CRUSH rule 5 x 231 [4,7,2]
+ CRUSH rule 5 x 232 [2,7,4]
+ CRUSH rule 5 x 233 [3,7,2]
+ CRUSH rule 5 x 234 [0,3,6]
+ CRUSH rule 5 x 235 [3,8,0]
+ CRUSH rule 5 x 236 [5,2,7]
+ CRUSH rule 5 x 237 [4,7,1]
+ CRUSH rule 5 x 238 [4,2,6]
+ CRUSH rule 5 x 239 [8,4,1]
+ CRUSH rule 5 x 240 [5,7,0]
+ CRUSH rule 5 x 241 [3,1,8]
+ CRUSH rule 5 x 242 [3,2,6]
+ CRUSH rule 5 x 243 [4,7,0]
+ CRUSH rule 5 x 244 [4,6,0]
+ CRUSH rule 5 x 245 [7,0,5]
+ CRUSH rule 5 x 246 [1,5,8]
+ CRUSH rule 5 x 247 [6,0,4]
+ CRUSH rule 5 x 248 [8,0,3]
+ CRUSH rule 5 x 249 [2,4,7]
+ CRUSH rule 5 x 250 [2,5,6]
+ CRUSH rule 5 x 251 [2,3,6]
+ CRUSH rule 5 x 252 [3,7,0]
+ CRUSH rule 5 x 253 [3,2,6]
+ CRUSH rule 5 x 254 [3,2,7]
+ CRUSH rule 5 x 255 [1,7,5]
+ CRUSH rule 5 x 256 [5,7,0]
+ CRUSH rule 5 x 257 [2,8,4]
+ CRUSH rule 5 x 258 [5,0,6]
+ CRUSH rule 5 x 259 [4,6,2]
+ CRUSH rule 5 x 260 [3,6,2]
+ CRUSH rule 5 x 261 [8,5,2]
+ CRUSH rule 5 x 262 [5,6,0]
+ CRUSH rule 5 x 263 [6,1,5]
+ CRUSH rule 5 x 264 [3,6,2]
+ CRUSH rule 5 x 265 [8,5,2]
+ CRUSH rule 5 x 266 [8,2,5]
+ CRUSH rule 5 x 267 [2,3,8]
+ CRUSH rule 5 x 268 [0,7,4]
+ CRUSH rule 5 x 269 [0,8,4]
+ CRUSH rule 5 x 270 [5,0,7]
+ CRUSH rule 5 x 271 [7,5,1]
+ CRUSH rule 5 x 272 [2,8,3]
+ CRUSH rule 5 x 273 [3,1,7]
+ CRUSH rule 5 x 274 [6,3,1]
+ CRUSH rule 5 x 275 [4,7,0]
+ CRUSH rule 5 x 276 [7,1,4]
+ CRUSH rule 5 x 277 [6,4,0]
+ CRUSH rule 5 x 278 [6,1,4]
+ CRUSH rule 5 x 279 [8,3,2]
+ CRUSH rule 5 x 280 [0,6,4]
+ CRUSH rule 5 x 281 [8,0,3]
+ CRUSH rule 5 x 282 [3,1,6]
+ CRUSH rule 5 x 283 [8,2,5]
+ CRUSH rule 5 x 284 [6,3,0]
+ CRUSH rule 5 x 285 [5,7,0]
+ CRUSH rule 5 x 286 [2,6,3]
+ CRUSH rule 5 x 287 [0,4,6]
+ CRUSH rule 5 x 288 [8,0,4]
+ CRUSH rule 5 x 289 [4,6,2]
+ CRUSH rule 5 x 290 [1,3,7]
+ CRUSH rule 5 x 291 [0,3,8]
+ CRUSH rule 5 x 292 [8,0,5]
+ CRUSH rule 5 x 293 [6,0,4]
+ CRUSH rule 5 x 294 [7,4,1]
+ CRUSH rule 5 x 295 [4,8,0]
+ CRUSH rule 5 x 296 [3,1,6]
+ CRUSH rule 5 x 297 [6,2,4]
+ CRUSH rule 5 x 298 [1,5,7]
+ CRUSH rule 5 x 299 [2,8,3]
+ CRUSH rule 5 x 300 [8,3,0]
+ CRUSH rule 5 x 301 [0,8,4]
+ CRUSH rule 5 x 302 [3,0,6]
+ CRUSH rule 5 x 303 [7,5,1]
+ CRUSH rule 5 x 304 [2,7,5]
+ CRUSH rule 5 x 305 [5,8,2]
+ CRUSH rule 5 x 306 [0,7,3]
+ CRUSH rule 5 x 307 [0,7,5]
+ CRUSH rule 5 x 308 [0,8,4]
+ CRUSH rule 5 x 309 [7,4,0]
+ CRUSH rule 5 x 310 [4,1,7]
+ CRUSH rule 5 x 311 [3,6,0]
+ CRUSH rule 5 x 312 [2,6,3]
+ CRUSH rule 5 x 313 [5,1,6]
+ CRUSH rule 5 x 314 [4,2,6]
+ CRUSH rule 5 x 315 [2,4,8]
+ CRUSH rule 5 x 316 [6,3,1]
+ CRUSH rule 5 x 317 [2,6,5]
+ CRUSH rule 5 x 318 [8,1,4]
+ CRUSH rule 5 x 319 [5,0,8]
+ CRUSH rule 5 x 320 [3,7,1]
+ CRUSH rule 5 x 321 [1,3,8]
+ CRUSH rule 5 x 322 [2,7,3]
+ CRUSH rule 5 x 323 [4,7,0]
+ CRUSH rule 5 x 324 [7,0,3]
+ CRUSH rule 5 x 325 [4,6,0]
+ CRUSH rule 5 x 326 [3,2,6]
+ CRUSH rule 5 x 327 [0,6,3]
+ CRUSH rule 5 x 328 [7,4,1]
+ CRUSH rule 5 x 329 [5,6,2]
+ CRUSH rule 5 x 330 [3,7,2]
+ CRUSH rule 5 x 331 [2,6,3]
+ CRUSH rule 5 x 332 [2,4,6]
+ CRUSH rule 5 x 333 [6,5,1]
+ CRUSH rule 5 x 334 [8,3,2]
+ CRUSH rule 5 x 335 [7,1,4]
+ CRUSH rule 5 x 336 [4,6,0]
+ CRUSH rule 5 x 337 [7,2,3]
+ CRUSH rule 5 x 338 [5,6,1]
+ CRUSH rule 5 x 339 [7,5,2]
+ CRUSH rule 5 x 340 [2,8,4]
+ CRUSH rule 5 x 341 [5,1,7]
+ CRUSH rule 5 x 342 [0,7,4]
+ CRUSH rule 5 x 343 [6,3,0]
+ CRUSH rule 5 x 344 [6,0,5]
+ CRUSH rule 5 x 345 [4,7,1]
+ CRUSH rule 5 x 346 [8,0,5]
+ CRUSH rule 5 x 347 [3,1,7]
+ CRUSH rule 5 x 348 [8,0,5]
+ CRUSH rule 5 x 349 [1,6,4]
+ CRUSH rule 5 x 350 [8,5,1]
+ CRUSH rule 5 x 351 [3,6,0]
+ CRUSH rule 5 x 352 [1,8,3]
+ CRUSH rule 5 x 353 [6,4,0]
+ CRUSH rule 5 x 354 [0,3,6]
+ CRUSH rule 5 x 355 [3,8,2]
+ CRUSH rule 5 x 356 [3,0,6]
+ CRUSH rule 5 x 357 [6,1,4]
+ CRUSH rule 5 x 358 [2,8,4]
+ CRUSH rule 5 x 359 [6,1,3]
+ CRUSH rule 5 x 360 [5,2,7]
+ CRUSH rule 5 x 361 [8,4,1]
+ CRUSH rule 5 x 362 [4,1,6]
+ CRUSH rule 5 x 363 [4,0,6]
+ CRUSH rule 5 x 364 [2,5,6]
+ CRUSH rule 5 x 365 [6,5,2]
+ CRUSH rule 5 x 366 [7,2,3]
+ CRUSH rule 5 x 367 [4,0,6]
+ CRUSH rule 5 x 368 [7,4,0]
+ CRUSH rule 5 x 369 [3,7,1]
+ CRUSH rule 5 x 370 [8,2,5]
+ CRUSH rule 5 x 371 [1,3,6]
+ CRUSH rule 5 x 372 [3,1,8]
+ CRUSH rule 5 x 373 [0,6,4]
+ CRUSH rule 5 x 374 [3,8,1]
+ CRUSH rule 5 x 375 [6,4,1]
+ CRUSH rule 5 x 376 [7,1,5]
+ CRUSH rule 5 x 377 [1,3,6]
+ CRUSH rule 5 x 378 [0,8,3]
+ CRUSH rule 5 x 379 [8,5,2]
+ CRUSH rule 5 x 380 [2,5,8]
+ CRUSH rule 5 x 381 [0,4,7]
+ CRUSH rule 5 x 382 [1,5,8]
+ CRUSH rule 5 x 383 [4,6,0]
+ CRUSH rule 5 x 384 [7,0,4]
+ CRUSH rule 5 x 385 [7,4,0]
+ CRUSH rule 5 x 386 [0,3,6]
+ CRUSH rule 5 x 387 [1,3,6]
+ CRUSH rule 5 x 388 [5,0,8]
+ CRUSH rule 5 x 389 [1,5,7]
+ CRUSH rule 5 x 390 [5,6,0]
+ CRUSH rule 5 x 391 [5,6,2]
+ CRUSH rule 5 x 392 [1,8,5]
+ CRUSH rule 5 x 393 [4,2,6]
+ CRUSH rule 5 x 394 [4,7,0]
+ CRUSH rule 5 x 395 [4,0,8]
+ CRUSH rule 5 x 396 [4,2,7]
+ CRUSH rule 5 x 397 [2,4,8]
+ CRUSH rule 5 x 398 [2,4,8]
+ CRUSH rule 5 x 399 [8,4,2]
+ CRUSH rule 5 x 400 [8,1,4]
+ CRUSH rule 5 x 401 [0,5,8]
+ CRUSH rule 5 x 402 [7,5,2]
+ CRUSH rule 5 x 403 [0,3,8]
+ CRUSH rule 5 x 404 [4,2,8]
+ CRUSH rule 5 x 405 [6,5,2]
+ CRUSH rule 5 x 406 [2,6,5]
+ CRUSH rule 5 x 407 [2,8,5]
+ CRUSH rule 5 x 408 [4,1,6]
+ CRUSH rule 5 x 409 [7,3,0]
+ CRUSH rule 5 x 410 [8,3,2]
+ CRUSH rule 5 x 411 [2,8,3]
+ CRUSH rule 5 x 412 [0,5,8]
+ CRUSH rule 5 x 413 [5,0,8]
+ CRUSH rule 5 x 414 [4,1,6]
+ CRUSH rule 5 x 415 [0,6,4]
+ CRUSH rule 5 x 416 [2,6,3]
+ CRUSH rule 5 x 417 [8,2,4]
+ CRUSH rule 5 x 418 [7,1,4]
+ CRUSH rule 5 x 419 [8,3,0]
+ CRUSH rule 5 x 420 [1,4,6]
+ CRUSH rule 5 x 421 [8,4,0]
+ CRUSH rule 5 x 422 [6,4,1]
+ CRUSH rule 5 x 423 [0,5,6]
+ CRUSH rule 5 x 424 [8,4,1]
+ CRUSH rule 5 x 425 [1,3,7]
+ CRUSH rule 5 x 426 [6,0,5]
+ CRUSH rule 5 x 427 [0,7,5]
+ CRUSH rule 5 x 428 [5,7,1]
+ CRUSH rule 5 x 429 [4,6,0]
+ CRUSH rule 5 x 430 [3,6,2]
+ CRUSH rule 5 x 431 [5,0,7]
+ CRUSH rule 5 x 432 [7,1,3]
+ CRUSH rule 5 x 433 [6,5,1]
+ CRUSH rule 5 x 434 [5,2,8]
+ CRUSH rule 5 x 435 [0,5,6]
+ CRUSH rule 5 x 436 [4,0,7]
+ CRUSH rule 5 x 437 [7,5,2]
+ CRUSH rule 5 x 438 [0,3,8]
+ CRUSH rule 5 x 439 [1,3,7]
+ CRUSH rule 5 x 440 [2,7,5]
+ CRUSH rule 5 x 441 [5,7,2]
+ CRUSH rule 5 x 442 [2,4,7]
+ CRUSH rule 5 x 443 [6,0,4]
+ CRUSH rule 5 x 444 [7,0,4]
+ CRUSH rule 5 x 445 [6,3,1]
+ CRUSH rule 5 x 446 [4,1,8]
+ CRUSH rule 5 x 447 [2,3,7]
+ CRUSH rule 5 x 448 [7,2,5]
+ CRUSH rule 5 x 449 [7,5,1]
+ CRUSH rule 5 x 450 [4,1,8]
+ CRUSH rule 5 x 451 [6,5,0]
+ CRUSH rule 5 x 452 [8,3,0]
+ CRUSH rule 5 x 453 [6,5,1]
+ CRUSH rule 5 x 454 [6,4,2]
+ CRUSH rule 5 x 455 [2,7,5]
+ CRUSH rule 5 x 456 [6,2,5]
+ CRUSH rule 5 x 457 [7,2,5]
+ CRUSH rule 5 x 458 [2,8,5]
+ CRUSH rule 5 x 459 [2,7,5]
+ CRUSH rule 5 x 460 [6,5,0]
+ CRUSH rule 5 x 461 [6,5,2]
+ CRUSH rule 5 x 462 [8,1,5]
+ CRUSH rule 5 x 463 [6,0,4]
+ CRUSH rule 5 x 464 [7,4,2]
+ CRUSH rule 5 x 465 [7,2,5]
+ CRUSH rule 5 x 466 [5,8,2]
+ CRUSH rule 5 x 467 [6,4,0]
+ CRUSH rule 5 x 468 [7,0,3]
+ CRUSH rule 5 x 469 [7,0,5]
+ CRUSH rule 5 x 470 [3,0,6]
+ CRUSH rule 5 x 471 [0,7,5]
+ CRUSH rule 5 x 472 [5,1,8]
+ CRUSH rule 5 x 473 [1,4,6]
+ CRUSH rule 5 x 474 [6,0,3]
+ CRUSH rule 5 x 475 [6,2,3]
+ CRUSH rule 5 x 476 [4,6,1]
+ CRUSH rule 5 x 477 [5,8,2]
+ CRUSH rule 5 x 478 [6,2,3]
+ CRUSH rule 5 x 479 [0,5,8]
+ CRUSH rule 5 x 480 [1,8,3]
+ CRUSH rule 5 x 481 [2,4,7]
+ CRUSH rule 5 x 482 [4,7,0]
+ CRUSH rule 5 x 483 [0,6,4]
+ CRUSH rule 5 x 484 [1,7,4]
+ CRUSH rule 5 x 485 [4,7,0]
+ CRUSH rule 5 x 486 [4,1,7]
+ CRUSH rule 5 x 487 [5,0,8]
+ CRUSH rule 5 x 488 [5,7,1]
+ CRUSH rule 5 x 489 [2,8,5]
+ CRUSH rule 5 x 490 [6,4,1]
+ CRUSH rule 5 x 491 [1,7,5]
+ CRUSH rule 5 x 492 [6,5,0]
+ CRUSH rule 5 x 493 [0,7,3]
+ CRUSH rule 5 x 494 [1,7,4]
+ CRUSH rule 5 x 495 [3,1,8]
+ CRUSH rule 5 x 496 [7,5,0]
+ CRUSH rule 5 x 497 [5,7,0]
+ CRUSH rule 5 x 498 [0,5,8]
+ CRUSH rule 5 x 499 [8,4,2]
+ CRUSH rule 5 x 500 [3,6,0]
+ CRUSH rule 5 x 501 [0,7,3]
+ CRUSH rule 5 x 502 [7,1,4]
+ CRUSH rule 5 x 503 [2,3,7]
+ CRUSH rule 5 x 504 [5,6,2]
+ CRUSH rule 5 x 505 [0,7,3]
+ CRUSH rule 5 x 506 [5,2,8]
+ CRUSH rule 5 x 507 [6,0,3]
+ CRUSH rule 5 x 508 [0,3,8]
+ CRUSH rule 5 x 509 [7,5,2]
+ CRUSH rule 5 x 510 [6,0,4]
+ CRUSH rule 5 x 511 [5,8,2]
+ CRUSH rule 5 x 512 [7,0,4]
+ CRUSH rule 5 x 513 [7,2,4]
+ CRUSH rule 5 x 514 [4,6,1]
+ CRUSH rule 5 x 515 [8,5,0]
+ CRUSH rule 5 x 516 [4,0,8]
+ CRUSH rule 5 x 517 [7,2,4]
+ CRUSH rule 5 x 518 [4,6,1]
+ CRUSH rule 5 x 519 [7,3,1]
+ CRUSH rule 5 x 520 [2,6,3]
+ CRUSH rule 5 x 521 [8,0,3]
+ CRUSH rule 5 x 522 [6,0,4]
+ CRUSH rule 5 x 523 [4,2,7]
+ CRUSH rule 5 x 524 [0,4,8]
+ CRUSH rule 5 x 525 [0,4,6]
+ CRUSH rule 5 x 526 [1,5,8]
+ CRUSH rule 5 x 527 [0,5,6]
+ CRUSH rule 5 x 528 [5,0,6]
+ CRUSH rule 5 x 529 [5,7,0]
+ CRUSH rule 5 x 530 [6,5,2]
+ CRUSH rule 5 x 531 [6,1,3]
+ CRUSH rule 5 x 532 [6,3,0]
+ CRUSH rule 5 x 533 [5,6,2]
+ CRUSH rule 5 x 534 [7,3,1]
+ CRUSH rule 5 x 535 [8,1,5]
+ CRUSH rule 5 x 536 [6,2,4]
+ CRUSH rule 5 x 537 [3,7,2]
+ CRUSH rule 5 x 538 [6,3,0]
+ CRUSH rule 5 x 539 [8,3,1]
+ CRUSH rule 5 x 540 [0,6,3]
+ CRUSH rule 5 x 541 [2,3,8]
+ CRUSH rule 5 x 542 [3,2,8]
+ CRUSH rule 5 x 543 [6,0,4]
+ CRUSH rule 5 x 544 [3,7,0]
+ CRUSH rule 5 x 545 [5,7,0]
+ CRUSH rule 5 x 546 [6,1,5]
+ CRUSH rule 5 x 547 [8,2,4]
+ CRUSH rule 5 x 548 [5,2,8]
+ CRUSH rule 5 x 549 [5,8,2]
+ CRUSH rule 5 x 550 [0,5,7]
+ CRUSH rule 5 x 551 [7,5,0]
+ CRUSH rule 5 x 552 [5,8,1]
+ CRUSH rule 5 x 553 [4,2,7]
+ CRUSH rule 5 x 554 [0,8,5]
+ CRUSH rule 5 x 555 [5,0,8]
+ CRUSH rule 5 x 556 [3,6,0]
+ CRUSH rule 5 x 557 [7,4,0]
+ CRUSH rule 5 x 558 [3,1,6]
+ CRUSH rule 5 x 559 [4,2,6]
+ CRUSH rule 5 x 560 [8,3,2]
+ CRUSH rule 5 x 561 [6,3,1]
+ CRUSH rule 5 x 562 [3,0,6]
+ CRUSH rule 5 x 563 [2,6,4]
+ CRUSH rule 5 x 564 [5,1,7]
+ CRUSH rule 5 x 565 [3,6,2]
+ CRUSH rule 5 x 566 [4,7,2]
+ CRUSH rule 5 x 567 [3,6,1]
+ CRUSH rule 5 x 568 [7,4,1]
+ CRUSH rule 5 x 569 [3,1,7]
+ CRUSH rule 5 x 570 [1,5,8]
+ CRUSH rule 5 x 571 [3,7,1]
+ CRUSH rule 5 x 572 [3,2,8]
+ CRUSH rule 5 x 573 [3,0,7]
+ CRUSH rule 5 x 574 [2,5,8]
+ CRUSH rule 5 x 575 [8,2,5]
+ CRUSH rule 5 x 576 [4,6,0]
+ CRUSH rule 5 x 577 [8,2,5]
+ CRUSH rule 5 x 578 [6,1,4]
+ CRUSH rule 5 x 579 [3,1,6]
+ CRUSH rule 5 x 580 [3,0,7]
+ CRUSH rule 5 x 581 [7,2,4]
+ CRUSH rule 5 x 582 [2,8,5]
+ CRUSH rule 5 x 583 [6,0,3]
+ CRUSH rule 5 x 584 [8,1,3]
+ CRUSH rule 5 x 585 [7,0,5]
+ CRUSH rule 5 x 586 [0,7,5]
+ CRUSH rule 5 x 587 [2,5,7]
+ CRUSH rule 5 x 588 [3,7,1]
+ CRUSH rule 5 x 589 [7,1,4]
+ CRUSH rule 5 x 590 [6,2,3]
+ CRUSH rule 5 x 591 [5,2,8]
+ CRUSH rule 5 x 592 [2,4,7]
+ CRUSH rule 5 x 593 [0,8,3]
+ CRUSH rule 5 x 594 [0,7,4]
+ CRUSH rule 5 x 595 [7,1,3]
+ CRUSH rule 5 x 596 [4,0,6]
+ CRUSH rule 5 x 597 [3,1,7]
+ CRUSH rule 5 x 598 [3,2,6]
+ CRUSH rule 5 x 599 [5,2,8]
+ CRUSH rule 5 x 600 [7,0,3]
+ CRUSH rule 5 x 601 [0,7,3]
+ CRUSH rule 5 x 602 [3,7,1]
+ CRUSH rule 5 x 603 [5,1,6]
+ CRUSH rule 5 x 604 [7,5,0]
+ CRUSH rule 5 x 605 [3,0,7]
+ CRUSH rule 5 x 606 [2,7,4]
+ CRUSH rule 5 x 607 [0,4,8]
+ CRUSH rule 5 x 608 [5,2,7]
+ CRUSH rule 5 x 609 [5,2,8]
+ CRUSH rule 5 x 610 [3,7,0]
+ CRUSH rule 5 x 611 [1,8,5]
+ CRUSH rule 5 x 612 [2,6,5]
+ CRUSH rule 5 x 613 [7,2,3]
+ CRUSH rule 5 x 614 [7,2,3]
+ CRUSH rule 5 x 615 [6,0,5]
+ CRUSH rule 5 x 616 [0,8,5]
+ CRUSH rule 5 x 617 [6,1,4]
+ CRUSH rule 5 x 618 [7,4,0]
+ CRUSH rule 5 x 619 [5,1,8]
+ CRUSH rule 5 x 620 [4,1,6]
+ CRUSH rule 5 x 621 [5,8,2]
+ CRUSH rule 5 x 622 [0,4,8]
+ CRUSH rule 5 x 623 [0,6,3]
+ CRUSH rule 5 x 624 [3,2,8]
+ CRUSH rule 5 x 625 [2,3,7]
+ CRUSH rule 5 x 626 [7,0,3]
+ CRUSH rule 5 x 627 [2,7,3]
+ CRUSH rule 5 x 628 [8,0,5]
+ CRUSH rule 5 x 629 [2,6,5]
+ CRUSH rule 5 x 630 [2,7,5]
+ CRUSH rule 5 x 631 [0,7,3]
+ CRUSH rule 5 x 632 [7,0,3]
+ CRUSH rule 5 x 633 [8,3,1]
+ CRUSH rule 5 x 634 [0,4,7]
+ CRUSH rule 5 x 635 [5,6,2]
+ CRUSH rule 5 x 636 [1,4,8]
+ CRUSH rule 5 x 637 [4,1,7]
+ CRUSH rule 5 x 638 [6,0,4]
+ CRUSH rule 5 x 639 [4,2,8]
+ CRUSH rule 5 x 640 [3,1,8]
+ CRUSH rule 5 x 641 [7,2,3]
+ CRUSH rule 5 x 642 [2,7,3]
+ CRUSH rule 5 x 643 [3,0,8]
+ CRUSH rule 5 x 644 [8,1,5]
+ CRUSH rule 5 x 645 [5,7,1]
+ CRUSH rule 5 x 646 [8,0,3]
+ CRUSH rule 5 x 647 [7,1,3]
+ CRUSH rule 5 x 648 [0,6,3]
+ CRUSH rule 5 x 649 [4,7,0]
+ CRUSH rule 5 x 650 [7,3,1]
+ CRUSH rule 5 x 651 [3,7,0]
+ CRUSH rule 5 x 652 [3,7,0]
+ CRUSH rule 5 x 653 [8,5,2]
+ CRUSH rule 5 x 654 [7,5,2]
+ CRUSH rule 5 x 655 [0,3,7]
+ CRUSH rule 5 x 656 [4,7,1]
+ CRUSH rule 5 x 657 [6,1,4]
+ CRUSH rule 5 x 658 [5,6,0]
+ CRUSH rule 5 x 659 [4,6,2]
+ CRUSH rule 5 x 660 [7,4,1]
+ CRUSH rule 5 x 661 [1,8,3]
+ CRUSH rule 5 x 662 [4,2,7]
+ CRUSH rule 5 x 663 [1,3,8]
+ CRUSH rule 5 x 664 [1,4,7]
+ CRUSH rule 5 x 665 [5,7,0]
+ CRUSH rule 5 x 666 [2,8,4]
+ CRUSH rule 5 x 667 [1,3,7]
+ CRUSH rule 5 x 668 [3,7,1]
+ CRUSH rule 5 x 669 [6,4,0]
+ CRUSH rule 5 x 670 [4,0,6]
+ CRUSH rule 5 x 671 [0,7,3]
+ CRUSH rule 5 x 672 [4,2,7]
+ CRUSH rule 5 x 673 [5,2,7]
+ CRUSH rule 5 x 674 [3,1,8]
+ CRUSH rule 5 x 675 [0,8,3]
+ CRUSH rule 5 x 676 [0,4,8]
+ CRUSH rule 5 x 677 [4,1,7]
+ CRUSH rule 5 x 678 [2,3,8]
+ CRUSH rule 5 x 679 [6,0,5]
+ CRUSH rule 5 x 680 [0,4,6]
+ CRUSH rule 5 x 681 [4,7,1]
+ CRUSH rule 5 x 682 [0,5,7]
+ CRUSH rule 5 x 683 [0,5,6]
+ CRUSH rule 5 x 684 [7,1,5]
+ CRUSH rule 5 x 685 [7,1,5]
+ CRUSH rule 5 x 686 [1,4,8]
+ CRUSH rule 5 x 687 [3,6,1]
+ CRUSH rule 5 x 688 [5,7,2]
+ CRUSH rule 5 x 689 [6,5,0]
+ CRUSH rule 5 x 690 [8,1,3]
+ CRUSH rule 5 x 691 [3,0,6]
+ CRUSH rule 5 x 692 [7,2,3]
+ CRUSH rule 5 x 693 [6,3,1]
+ CRUSH rule 5 x 694 [6,5,1]
+ CRUSH rule 5 x 695 [0,8,4]
+ CRUSH rule 5 x 696 [1,4,8]
+ CRUSH rule 5 x 697 [6,1,3]
+ CRUSH rule 5 x 698 [6,2,4]
+ CRUSH rule 5 x 699 [1,6,3]
+ CRUSH rule 5 x 700 [0,3,7]
+ CRUSH rule 5 x 701 [4,1,7]
+ CRUSH rule 5 x 702 [3,2,8]
+ CRUSH rule 5 x 703 [8,3,1]
+ CRUSH rule 5 x 704 [0,3,8]
+ CRUSH rule 5 x 705 [8,0,4]
+ CRUSH rule 5 x 706 [1,5,6]
+ CRUSH rule 5 x 707 [7,3,1]
+ CRUSH rule 5 x 708 [3,7,1]
+ CRUSH rule 5 x 709 [6,3,0]
+ CRUSH rule 5 x 710 [8,4,0]
+ CRUSH rule 5 x 711 [2,3,8]
+ CRUSH rule 5 x 712 [2,3,7]
+ CRUSH rule 5 x 713 [6,3,0]
+ CRUSH rule 5 x 714 [3,2,7]
+ CRUSH rule 5 x 715 [1,3,6]
+ CRUSH rule 5 x 716 [3,6,0]
+ CRUSH rule 5 x 717 [8,2,5]
+ CRUSH rule 5 x 718 [3,7,2]
+ CRUSH rule 5 x 719 [2,6,3]
+ CRUSH rule 5 x 720 [6,1,4]
+ CRUSH rule 5 x 721 [5,7,2]
+ CRUSH rule 5 x 722 [5,7,1]
+ CRUSH rule 5 x 723 [5,1,7]
+ CRUSH rule 5 x 724 [0,6,3]
+ CRUSH rule 5 x 725 [0,3,7]
+ CRUSH rule 5 x 726 [3,8,1]
+ CRUSH rule 5 x 727 [4,6,1]
+ CRUSH rule 5 x 728 [2,7,4]
+ CRUSH rule 5 x 729 [5,6,2]
+ CRUSH rule 5 x 730 [3,7,2]
+ CRUSH rule 5 x 731 [4,1,8]
+ CRUSH rule 5 x 732 [1,5,6]
+ CRUSH rule 5 x 733 [5,7,0]
+ CRUSH rule 5 x 734 [6,4,2]
+ CRUSH rule 5 x 735 [4,8,1]
+ CRUSH rule 5 x 736 [3,8,1]
+ CRUSH rule 5 x 737 [1,6,4]
+ CRUSH rule 5 x 738 [5,2,7]
+ CRUSH rule 5 x 739 [0,7,4]
+ CRUSH rule 5 x 740 [0,8,4]
+ CRUSH rule 5 x 741 [7,1,4]
+ CRUSH rule 5 x 742 [8,2,3]
+ CRUSH rule 5 x 743 [7,0,5]
+ CRUSH rule 5 x 744 [4,7,1]
+ CRUSH rule 5 x 745 [3,1,8]
+ CRUSH rule 5 x 746 [4,1,7]
+ CRUSH rule 5 x 747 [6,0,3]
+ CRUSH rule 5 x 748 [2,7,5]
+ CRUSH rule 5 x 749 [4,8,0]
+ CRUSH rule 5 x 750 [1,6,3]
+ CRUSH rule 5 x 751 [2,8,3]
+ CRUSH rule 5 x 752 [8,1,5]
+ CRUSH rule 5 x 753 [7,3,1]
+ CRUSH rule 5 x 754 [8,5,2]
+ CRUSH rule 5 x 755 [1,6,3]
+ CRUSH rule 5 x 756 [5,6,1]
+ CRUSH rule 5 x 757 [8,0,5]
+ CRUSH rule 5 x 758 [6,0,3]
+ CRUSH rule 5 x 759 [8,5,2]
+ CRUSH rule 5 x 760 [1,5,7]
+ CRUSH rule 5 x 761 [4,1,8]
+ CRUSH rule 5 x 762 [2,7,5]
+ CRUSH rule 5 x 763 [8,5,1]
+ CRUSH rule 5 x 764 [1,7,5]
+ CRUSH rule 5 x 765 [6,5,2]
+ CRUSH rule 5 x 766 [8,5,1]
+ CRUSH rule 5 x 767 [1,8,3]
+ CRUSH rule 5 x 768 [8,3,2]
+ CRUSH rule 5 x 769 [6,2,5]
+ CRUSH rule 5 x 770 [6,0,4]
+ CRUSH rule 5 x 771 [7,0,3]
+ CRUSH rule 5 x 772 [8,3,1]
+ CRUSH rule 5 x 773 [3,1,7]
+ CRUSH rule 5 x 774 [4,6,2]
+ CRUSH rule 5 x 775 [6,4,2]
+ CRUSH rule 5 x 776 [7,2,5]
+ CRUSH rule 5 x 777 [3,1,6]
+ CRUSH rule 5 x 778 [1,8,4]
+ CRUSH rule 5 x 779 [2,7,3]
+ CRUSH rule 5 x 780 [0,5,7]
+ CRUSH rule 5 x 781 [6,3,2]
+ CRUSH rule 5 x 782 [5,0,8]
+ CRUSH rule 5 x 783 [7,1,3]
+ CRUSH rule 5 x 784 [0,4,6]
+ CRUSH rule 5 x 785 [6,1,3]
+ CRUSH rule 5 x 786 [7,3,1]
+ CRUSH rule 5 x 787 [1,6,4]
+ CRUSH rule 5 x 788 [6,0,3]
+ CRUSH rule 5 x 789 [0,4,8]
+ CRUSH rule 5 x 790 [8,4,0]
+ CRUSH rule 5 x 791 [3,8,0]
+ CRUSH rule 5 x 792 [5,8,0]
+ CRUSH rule 5 x 793 [6,1,3]
+ CRUSH rule 5 x 794 [2,6,4]
+ CRUSH rule 5 x 795 [0,3,8]
+ CRUSH rule 5 x 796 [3,7,2]
+ CRUSH rule 5 x 797 [2,3,8]
+ CRUSH rule 5 x 798 [6,1,5]
+ CRUSH rule 5 x 799 [5,1,8]
+ CRUSH rule 5 x 800 [5,0,7]
+ CRUSH rule 5 x 801 [3,6,1]
+ CRUSH rule 5 x 802 [1,8,5]
+ CRUSH rule 5 x 803 [0,5,7]
+ CRUSH rule 5 x 804 [6,2,5]
+ CRUSH rule 5 x 805 [3,6,1]
+ CRUSH rule 5 x 806 [1,3,7]
+ CRUSH rule 5 x 807 [5,7,2]
+ CRUSH rule 5 x 808 [4,6,2]
+ CRUSH rule 5 x 809 [1,4,8]
+ CRUSH rule 5 x 810 [5,7,2]
+ CRUSH rule 5 x 811 [8,4,0]
+ CRUSH rule 5 x 812 [8,5,2]
+ CRUSH rule 5 x 813 [6,4,2]
+ CRUSH rule 5 x 814 [3,6,1]
+ CRUSH rule 5 x 815 [3,1,8]
+ CRUSH rule 5 x 816 [2,7,3]
+ CRUSH rule 5 x 817 [4,8,2]
+ CRUSH rule 5 x 818 [3,0,7]
+ CRUSH rule 5 x 819 [5,1,8]
+ CRUSH rule 5 x 820 [3,6,0]
+ CRUSH rule 5 x 821 [4,6,2]
+ CRUSH rule 5 x 822 [2,5,8]
+ CRUSH rule 5 x 823 [4,8,2]
+ CRUSH rule 5 x 824 [3,7,2]
+ CRUSH rule 5 x 825 [2,8,5]
+ CRUSH rule 5 x 826 [7,0,5]
+ CRUSH rule 5 x 827 [0,8,3]
+ CRUSH rule 5 x 828 [2,3,8]
+ CRUSH rule 5 x 829 [5,6,1]
+ CRUSH rule 5 x 830 [2,3,8]
+ CRUSH rule 5 x 831 [1,6,3]
+ CRUSH rule 5 x 832 [4,7,0]
+ CRUSH rule 5 x 833 [2,7,3]
+ CRUSH rule 5 x 834 [3,1,7]
+ CRUSH rule 5 x 835 [8,4,1]
+ CRUSH rule 5 x 836 [3,7,1]
+ CRUSH rule 5 x 837 [6,3,1]
+ CRUSH rule 5 x 838 [6,2,4]
+ CRUSH rule 5 x 839 [5,0,6]
+ CRUSH rule 5 x 840 [7,3,2]
+ CRUSH rule 5 x 841 [4,8,2]
+ CRUSH rule 5 x 842 [2,4,6]
+ CRUSH rule 5 x 843 [6,4,1]
+ CRUSH rule 5 x 844 [4,8,1]
+ CRUSH rule 5 x 845 [3,8,2]
+ CRUSH rule 5 x 846 [3,2,7]
+ CRUSH rule 5 x 847 [0,8,4]
+ CRUSH rule 5 x 848 [2,6,5]
+ CRUSH rule 5 x 849 [4,6,2]
+ CRUSH rule 5 x 850 [1,3,6]
+ CRUSH rule 5 x 851 [6,4,0]
+ CRUSH rule 5 x 852 [7,3,0]
+ CRUSH rule 5 x 853 [6,0,4]
+ CRUSH rule 5 x 854 [7,0,4]
+ CRUSH rule 5 x 855 [5,7,2]
+ CRUSH rule 5 x 856 [6,3,2]
+ CRUSH rule 5 x 857 [8,5,0]
+ CRUSH rule 5 x 858 [6,4,1]
+ CRUSH rule 5 x 859 [6,0,5]
+ CRUSH rule 5 x 860 [4,1,7]
+ CRUSH rule 5 x 861 [8,3,1]
+ CRUSH rule 5 x 862 [6,1,4]
+ CRUSH rule 5 x 863 [8,2,3]
+ CRUSH rule 5 x 864 [5,6,2]
+ CRUSH rule 5 x 865 [8,1,3]
+ CRUSH rule 5 x 866 [3,6,0]
+ CRUSH rule 5 x 867 [6,5,1]
+ CRUSH rule 5 x 868 [6,3,0]
+ CRUSH rule 5 x 869 [8,5,2]
+ CRUSH rule 5 x 870 [0,4,8]
+ CRUSH rule 5 x 871 [3,2,8]
+ CRUSH rule 5 x 872 [5,1,8]
+ CRUSH rule 5 x 873 [4,6,2]
+ CRUSH rule 5 x 874 [2,6,4]
+ CRUSH rule 5 x 875 [2,6,4]
+ CRUSH rule 5 x 876 [5,8,1]
+ CRUSH rule 5 x 877 [6,4,2]
+ CRUSH rule 5 x 878 [5,2,7]
+ CRUSH rule 5 x 879 [7,4,2]
+ CRUSH rule 5 x 880 [3,2,8]
+ CRUSH rule 5 x 881 [5,6,1]
+ CRUSH rule 5 x 882 [4,0,7]
+ CRUSH rule 5 x 883 [2,3,7]
+ CRUSH rule 5 x 884 [6,0,4]
+ CRUSH rule 5 x 885 [5,1,8]
+ CRUSH rule 5 x 886 [3,6,0]
+ CRUSH rule 5 x 887 [7,4,0]
+ CRUSH rule 5 x 888 [6,2,5]
+ CRUSH rule 5 x 889 [2,6,4]
+ CRUSH rule 5 x 890 [7,2,4]
+ CRUSH rule 5 x 891 [1,8,5]
+ CRUSH rule 5 x 892 [6,2,3]
+ CRUSH rule 5 x 893 [2,3,7]
+ CRUSH rule 5 x 894 [7,5,0]
+ CRUSH rule 5 x 895 [5,1,8]
+ CRUSH rule 5 x 896 [1,8,5]
+ CRUSH rule 5 x 897 [4,2,6]
+ CRUSH rule 5 x 898 [0,5,7]
+ CRUSH rule 5 x 899 [1,7,5]
+ CRUSH rule 5 x 900 [4,1,6]
+ CRUSH rule 5 x 901 [5,0,8]
+ CRUSH rule 5 x 902 [8,5,0]
+ CRUSH rule 5 x 903 [5,7,1]
+ CRUSH rule 5 x 904 [5,6,2]
+ CRUSH rule 5 x 905 [6,2,5]
+ CRUSH rule 5 x 906 [1,6,3]
+ CRUSH rule 5 x 907 [7,1,5]
+ CRUSH rule 5 x 908 [5,8,1]
+ CRUSH rule 5 x 909 [2,3,7]
+ CRUSH rule 5 x 910 [6,4,0]
+ CRUSH rule 5 x 911 [5,8,1]
+ CRUSH rule 5 x 912 [0,7,3]
+ CRUSH rule 5 x 913 [7,2,4]
+ CRUSH rule 5 x 914 [6,4,0]
+ CRUSH rule 5 x 915 [8,2,3]
+ CRUSH rule 5 x 916 [3,1,8]
+ CRUSH rule 5 x 917 [1,5,8]
+ CRUSH rule 5 x 918 [8,2,4]
+ CRUSH rule 5 x 919 [6,2,3]
+ CRUSH rule 5 x 920 [7,4,0]
+ CRUSH rule 5 x 921 [1,4,6]
+ CRUSH rule 5 x 922 [6,4,0]
+ CRUSH rule 5 x 923 [5,8,2]
+ CRUSH rule 5 x 924 [3,1,7]
+ CRUSH rule 5 x 925 [5,7,2]
+ CRUSH rule 5 x 926 [3,0,8]
+ CRUSH rule 5 x 927 [1,6,3]
+ CRUSH rule 5 x 928 [8,1,3]
+ CRUSH rule 5 x 929 [4,1,7]
+ CRUSH rule 5 x 930 [2,4,6]
+ CRUSH rule 5 x 931 [5,0,7]
+ CRUSH rule 5 x 932 [4,1,8]
+ CRUSH rule 5 x 933 [8,5,0]
+ CRUSH rule 5 x 934 [5,6,0]
+ CRUSH rule 5 x 935 [6,3,1]
+ CRUSH rule 5 x 936 [0,6,5]
+ CRUSH rule 5 x 937 [5,8,2]
+ CRUSH rule 5 x 938 [6,5,2]
+ CRUSH rule 5 x 939 [2,7,5]
+ CRUSH rule 5 x 940 [8,5,0]
+ CRUSH rule 5 x 941 [5,2,8]
+ CRUSH rule 5 x 942 [1,8,4]
+ CRUSH rule 5 x 943 [8,2,4]
+ CRUSH rule 5 x 944 [4,8,2]
+ CRUSH rule 5 x 945 [7,2,4]
+ CRUSH rule 5 x 946 [2,8,5]
+ CRUSH rule 5 x 947 [4,2,8]
+ CRUSH rule 5 x 948 [7,5,0]
+ CRUSH rule 5 x 949 [6,1,3]
+ CRUSH rule 5 x 950 [3,6,0]
+ CRUSH rule 5 x 951 [4,8,1]
+ CRUSH rule 5 x 952 [2,7,3]
+ CRUSH rule 5 x 953 [1,3,6]
+ CRUSH rule 5 x 954 [4,2,7]
+ CRUSH rule 5 x 955 [8,0,4]
+ CRUSH rule 5 x 956 [1,6,4]
+ CRUSH rule 5 x 957 [7,1,3]
+ CRUSH rule 5 x 958 [8,4,1]
+ CRUSH rule 5 x 959 [5,2,7]
+ CRUSH rule 5 x 960 [3,6,0]
+ CRUSH rule 5 x 961 [4,0,8]
+ CRUSH rule 5 x 962 [7,4,0]
+ CRUSH rule 5 x 963 [0,5,6]
+ CRUSH rule 5 x 964 [3,1,8]
+ CRUSH rule 5 x 965 [7,4,0]
+ CRUSH rule 5 x 966 [3,8,0]
+ CRUSH rule 5 x 967 [8,5,0]
+ CRUSH rule 5 x 968 [7,2,4]
+ CRUSH rule 5 x 969 [8,0,5]
+ CRUSH rule 5 x 970 [0,6,3]
+ CRUSH rule 5 x 971 [1,7,3]
+ CRUSH rule 5 x 972 [1,8,4]
+ CRUSH rule 5 x 973 [1,6,3]
+ CRUSH rule 5 x 974 [5,1,8]
+ CRUSH rule 5 x 975 [3,7,0]
+ CRUSH rule 5 x 976 [4,8,2]
+ CRUSH rule 5 x 977 [8,3,2]
+ CRUSH rule 5 x 978 [7,2,4]
+ CRUSH rule 5 x 979 [7,1,5]
+ CRUSH rule 5 x 980 [6,0,5]
+ CRUSH rule 5 x 981 [7,3,2]
+ CRUSH rule 5 x 982 [4,2,8]
+ CRUSH rule 5 x 983 [3,7,0]
+ CRUSH rule 5 x 984 [0,7,3]
+ CRUSH rule 5 x 985 [2,5,7]
+ CRUSH rule 5 x 986 [8,3,0]
+ CRUSH rule 5 x 987 [0,5,8]
+ CRUSH rule 5 x 988 [1,3,7]
+ CRUSH rule 5 x 989 [0,6,3]
+ CRUSH rule 5 x 990 [1,6,5]
+ CRUSH rule 5 x 991 [0,4,8]
+ CRUSH rule 5 x 992 [7,1,5]
+ CRUSH rule 5 x 993 [0,6,3]
+ CRUSH rule 5 x 994 [3,7,2]
+ CRUSH rule 5 x 995 [7,1,5]
+ CRUSH rule 5 x 996 [6,5,0]
+ CRUSH rule 5 x 997 [6,4,1]
+ CRUSH rule 5 x 998 [8,1,5]
+ CRUSH rule 5 x 999 [0,7,4]
+ CRUSH rule 5 x 1000 [8,5,0]
+ CRUSH rule 5 x 1001 [2,5,6]
+ CRUSH rule 5 x 1002 [1,3,7]
+ CRUSH rule 5 x 1003 [2,8,3]
+ CRUSH rule 5 x 1004 [6,1,3]
+ CRUSH rule 5 x 1005 [6,1,5]
+ CRUSH rule 5 x 1006 [1,6,5]
+ CRUSH rule 5 x 1007 [1,5,7]
+ CRUSH rule 5 x 1008 [1,7,3]
+ CRUSH rule 5 x 1009 [6,4,1]
+ CRUSH rule 5 x 1010 [3,1,7]
+ CRUSH rule 5 x 1011 [3,0,8]
+ CRUSH rule 5 x 1012 [3,0,7]
+ CRUSH rule 5 x 1013 [5,1,7]
+ CRUSH rule 5 x 1014 [2,8,4]
+ CRUSH rule 5 x 1015 [6,5,0]
+ CRUSH rule 5 x 1016 [2,4,7]
+ CRUSH rule 5 x 1017 [6,0,3]
+ CRUSH rule 5 x 1018 [5,0,6]
+ CRUSH rule 5 x 1019 [5,8,2]
+ CRUSH rule 5 x 1020 [5,1,7]
+ CRUSH rule 5 x 1021 [5,2,6]
+ CRUSH rule 5 x 1022 [1,6,4]
+ CRUSH rule 5 x 1023 [3,2,8]
+ rule 5 (chooseleaf-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+ $ crushtool -i set-choose.crushmap --test --show-statistics --weight 0 0 --weight 1 0 --weight 3 0 --weight 4 0
+ rule 0 (choose), x = 0..1023, numrep = 2..3
+ CRUSH rule 0 x 0 [2,5]
+ CRUSH rule 0 x 1 [2,8]
+ CRUSH rule 0 x 2 [2,5]
+ CRUSH rule 0 x 3 [8,2]
+ CRUSH rule 0 x 4 [5,2]
+ CRUSH rule 0 x 5 [7,2]
+ CRUSH rule 0 x 6 [2,6]
+ CRUSH rule 0 x 7 [5,6]
+ CRUSH rule 0 x 8 [5,7]
+ CRUSH rule 0 x 9 [2,5]
+ CRUSH rule 0 x 10 [2,8]
+ CRUSH rule 0 x 11 [2,6]
+ CRUSH rule 0 x 12 [2,5]
+ CRUSH rule 0 x 13 [5,8]
+ CRUSH rule 0 x 14 [7,2]
+ CRUSH rule 0 x 15 [7,2]
+ CRUSH rule 0 x 16 [5,7]
+ CRUSH rule 0 x 17 [5,2]
+ CRUSH rule 0 x 18 [2,5]
+ CRUSH rule 0 x 19 [7,5]
+ CRUSH rule 0 x 20 [2,5]
+ CRUSH rule 0 x 21 [5,6]
+ CRUSH rule 0 x 22 [8,5]
+ CRUSH rule 0 x 23 [5,7]
+ CRUSH rule 0 x 24 [2,6]
+ CRUSH rule 0 x 25 [5,8]
+ CRUSH rule 0 x 26 [2,7]
+ CRUSH rule 0 x 27 [5,2]
+ CRUSH rule 0 x 28 [6,2]
+ CRUSH rule 0 x 29 [8,5]
+ CRUSH rule 0 x 30 [5,6]
+ CRUSH rule 0 x 31 [8,2]
+ CRUSH rule 0 x 32 [5,7]
+ CRUSH rule 0 x 33 [2,7]
+ CRUSH rule 0 x 34 [2,5]
+ CRUSH rule 0 x 35 [2,6]
+ CRUSH rule 0 x 36 [5,6]
+ CRUSH rule 0 x 37 [2,5]
+ CRUSH rule 0 x 38 [5,6]
+ CRUSH rule 0 x 39 [5,7]
+ CRUSH rule 0 x 40 [7,2]
+ CRUSH rule 0 x 41 [2,7]
+ CRUSH rule 0 x 42 [5,7]
+ CRUSH rule 0 x 43 [2,5]
+ CRUSH rule 0 x 44 [2,8]
+ CRUSH rule 0 x 45 [8,2]
+ CRUSH rule 0 x 46 [2,5]
+ CRUSH rule 0 x 47 [5,2]
+ CRUSH rule 0 x 48 [5,6]
+ CRUSH rule 0 x 49 [5,6]
+ CRUSH rule 0 x 50 [5,2]
+ CRUSH rule 0 x 51 [5,6]
+ CRUSH rule 0 x 52 [8,2]
+ CRUSH rule 0 x 53 [5,6]
+ CRUSH rule 0 x 54 [7,5]
+ CRUSH rule 0 x 55 [8,2]
+ CRUSH rule 0 x 56 [6,5]
+ CRUSH rule 0 x 57 [5,8]
+ CRUSH rule 0 x 58 [2,8]
+ CRUSH rule 0 x 59 [5,2]
+ CRUSH rule 0 x 60 [5,2]
+ CRUSH rule 0 x 61 [5,8]
+ CRUSH rule 0 x 62 [7,2]
+ CRUSH rule 0 x 63 [5,6]
+ CRUSH rule 0 x 64 [5,2]
+ CRUSH rule 0 x 65 [7,5]
+ CRUSH rule 0 x 66 [5,6]
+ CRUSH rule 0 x 67 [5,2]
+ CRUSH rule 0 x 68 [2,5]
+ CRUSH rule 0 x 69 [5,2]
+ CRUSH rule 0 x 70 [7,2]
+ CRUSH rule 0 x 71 [2,7]
+ CRUSH rule 0 x 72 [6,2]
+ CRUSH rule 0 x 73 [2,7]
+ CRUSH rule 0 x 74 [2,8]
+ CRUSH rule 0 x 75 [5,2]
+ CRUSH rule 0 x 76 [5,2]
+ CRUSH rule 0 x 77 [7,2]
+ CRUSH rule 0 x 78 [2,5]
+ CRUSH rule 0 x 79 [5,2]
+ CRUSH rule 0 x 80 [2,5]
+ CRUSH rule 0 x 81 [2,5]
+ CRUSH rule 0 x 82 [7,2]
+ CRUSH rule 0 x 83 [2,6]
+ CRUSH rule 0 x 84 [7,2]
+ CRUSH rule 0 x 85 [5,7]
+ CRUSH rule 0 x 86 [2,6]
+ CRUSH rule 0 x 87 [2,6]
+ CRUSH rule 0 x 88 [2,8]
+ CRUSH rule 0 x 89 [5,2]
+ CRUSH rule 0 x 90 [6,5]
+ CRUSH rule 0 x 91 [5,6]
+ CRUSH rule 0 x 92 [2,6]
+ CRUSH rule 0 x 93 [7,5]
+ CRUSH rule 0 x 94 [2,5]
+ CRUSH rule 0 x 95 [7,5]
+ CRUSH rule 0 x 96 [5,8]
+ CRUSH rule 0 x 97 [8,5]
+ CRUSH rule 0 x 98 [2,7]
+ CRUSH rule 0 x 99 [2,8]
+ CRUSH rule 0 x 100 [2,6]
+ CRUSH rule 0 x 101 [5,8]
+ CRUSH rule 0 x 102 [5,2]
+ CRUSH rule 0 x 103 [5,7]
+ CRUSH rule 0 x 104 [7,5]
+ CRUSH rule 0 x 105 [2,5]
+ CRUSH rule 0 x 106 [2,6]
+ CRUSH rule 0 x 107 [5,2]
+ CRUSH rule 0 x 108 [7,2]
+ CRUSH rule 0 x 109 [2,5]
+ CRUSH rule 0 x 110 [5,2]
+ CRUSH rule 0 x 111 [2,5]
+ CRUSH rule 0 x 112 [2,6]
+ CRUSH rule 0 x 113 [6,2]
+ CRUSH rule 0 x 114 [7,5]
+ CRUSH rule 0 x 115 [8,2]
+ CRUSH rule 0 x 116 [2,7]
+ CRUSH rule 0 x 117 [7,5]
+ CRUSH rule 0 x 118 [2,5]
+ CRUSH rule 0 x 119 [5,7]
+ CRUSH rule 0 x 120 [2,5]
+ CRUSH rule 0 x 121 [2,8]
+ CRUSH rule 0 x 122 [8,5]
+ CRUSH rule 0 x 123 [2,5]
+ CRUSH rule 0 x 124 [5,2]
+ CRUSH rule 0 x 125 [2,7]
+ CRUSH rule 0 x 126 [5,2]
+ CRUSH rule 0 x 127 [5,6]
+ CRUSH rule 0 x 128 [5,8]
+ CRUSH rule 0 x 129 [2,5]
+ CRUSH rule 0 x 130 [5,8]
+ CRUSH rule 0 x 131 [2,5]
+ CRUSH rule 0 x 132 [2,5]
+ CRUSH rule 0 x 133 [5,6]
+ CRUSH rule 0 x 134 [2,7]
+ CRUSH rule 0 x 135 [5,7]
+ CRUSH rule 0 x 136 [2,5]
+ CRUSH rule 0 x 137 [7,5]
+ CRUSH rule 0 x 138 [8,5]
+ CRUSH rule 0 x 139 [5,2]
+ CRUSH rule 0 x 140 [2,8]
+ CRUSH rule 0 x 141 [6,2]
+ CRUSH rule 0 x 142 [5,2]
+ CRUSH rule 0 x 143 [5,7]
+ CRUSH rule 0 x 144 [8,2]
+ CRUSH rule 0 x 145 [8,5]
+ CRUSH rule 0 x 146 [2,8]
+ CRUSH rule 0 x 147 [2,6]
+ CRUSH rule 0 x 148 [5,2]
+ CRUSH rule 0 x 149 [5,6]
+ CRUSH rule 0 x 150 [2,8]
+ CRUSH rule 0 x 151 [5,8]
+ CRUSH rule 0 x 152 [8,5]
+ CRUSH rule 0 x 153 [8,5]
+ CRUSH rule 0 x 154 [5,2]
+ CRUSH rule 0 x 155 [5,6]
+ CRUSH rule 0 x 156 [5,2]
+ CRUSH rule 0 x 157 [5,2]
+ CRUSH rule 0 x 158 [2,6]
+ CRUSH rule 0 x 159 [7,2]
+ CRUSH rule 0 x 160 [2,8]
+ CRUSH rule 0 x 161 [2,5]
+ CRUSH rule 0 x 162 [2,8]
+ CRUSH rule 0 x 163 [5,8]
+ CRUSH rule 0 x 164 [7,2]
+ CRUSH rule 0 x 165 [7,2]
+ CRUSH rule 0 x 166 [2,5]
+ CRUSH rule 0 x 167 [2,6]
+ CRUSH rule 0 x 168 [5,2]
+ CRUSH rule 0 x 169 [2,7]
+ CRUSH rule 0 x 170 [2,5]
+ CRUSH rule 0 x 171 [7,5]
+ CRUSH rule 0 x 172 [2,8]
+ CRUSH rule 0 x 173 [8,5]
+ CRUSH rule 0 x 174 [2,5]
+ CRUSH rule 0 x 175 [6,2]
+ CRUSH rule 0 x 176 [5,2]
+ CRUSH rule 0 x 177 [5,2]
+ CRUSH rule 0 x 178 [5,2]
+ CRUSH rule 0 x 179 [5,2]
+ CRUSH rule 0 x 180 [5,7]
+ CRUSH rule 0 x 181 [6,2]
+ CRUSH rule 0 x 182 [8,5]
+ CRUSH rule 0 x 183 [7,5]
+ CRUSH rule 0 x 184 [5,6]
+ CRUSH rule 0 x 185 [6,2]
+ CRUSH rule 0 x 186 [2,5]
+ CRUSH rule 0 x 187 [2,6]
+ CRUSH rule 0 x 188 [2,6]
+ CRUSH rule 0 x 189 [2,6]
+ CRUSH rule 0 x 190 [5,2]
+ CRUSH rule 0 x 191 [7,2]
+ CRUSH rule 0 x 192 [5,2]
+ CRUSH rule 0 x 193 [5,2]
+ CRUSH rule 0 x 194 [2,5]
+ CRUSH rule 0 x 195 [6,5]
+ CRUSH rule 0 x 196 [6,2]
+ CRUSH rule 0 x 197 [6,5]
+ CRUSH rule 0 x 198 [2,5]
+ CRUSH rule 0 x 199 [2,5]
+ CRUSH rule 0 x 200 [2,5]
+ CRUSH rule 0 x 201 [7,2]
+ CRUSH rule 0 x 202 [6,5]
+ CRUSH rule 0 x 203 [5,6]
+ CRUSH rule 0 x 204 [2,5]
+ CRUSH rule 0 x 205 [2,7]
+ CRUSH rule 0 x 206 [2,8]
+ CRUSH rule 0 x 207 [5,2]
+ CRUSH rule 0 x 208 [7,2]
+ CRUSH rule 0 x 209 [2,8]
+ CRUSH rule 0 x 210 [2,5]
+ CRUSH rule 0 x 211 [5,2]
+ CRUSH rule 0 x 212 [7,5]
+ CRUSH rule 0 x 213 [8,5]
+ CRUSH rule 0 x 214 [5,7]
+ CRUSH rule 0 x 215 [8,2]
+ CRUSH rule 0 x 216 [5,2]
+ CRUSH rule 0 x 217 [2,7]
+ CRUSH rule 0 x 218 [2,7]
+ CRUSH rule 0 x 219 [5,6]
+ CRUSH rule 0 x 220 [5,8]
+ CRUSH rule 0 x 221 [5,7]
+ CRUSH rule 0 x 222 [6,5]
+ CRUSH rule 0 x 223 [2,5]
+ CRUSH rule 0 x 224 [2,5]
+ CRUSH rule 0 x 225 [8,2]
+ CRUSH rule 0 x 226 [7,2]
+ CRUSH rule 0 x 227 [5,2]
+ CRUSH rule 0 x 228 [5,6]
+ CRUSH rule 0 x 229 [5,7]
+ CRUSH rule 0 x 230 [5,6]
+ CRUSH rule 0 x 231 [5,7]
+ CRUSH rule 0 x 232 [2,8]
+ CRUSH rule 0 x 233 [5,6]
+ CRUSH rule 0 x 234 [2,5]
+ CRUSH rule 0 x 235 [5,6]
+ CRUSH rule 0 x 236 [5,2]
+ CRUSH rule 0 x 237 [5,8]
+ CRUSH rule 0 x 238 [5,2]
+ CRUSH rule 0 x 239 [8,5]
+ CRUSH rule 0 x 240 [5,8]
+ CRUSH rule 0 x 241 [5,2]
+ CRUSH rule 0 x 242 [5,2]
+ CRUSH rule 0 x 243 [5,8]
+ CRUSH rule 0 x 244 [5,6]
+ CRUSH rule 0 x 245 [7,2]
+ CRUSH rule 0 x 246 [2,5]
+ CRUSH rule 0 x 247 [6,2]
+ CRUSH rule 0 x 248 [8,2]
+ CRUSH rule 0 x 249 [2,5]
+ CRUSH rule 0 x 250 [2,5]
+ CRUSH rule 0 x 251 [2,5]
+ CRUSH rule 0 x 252 [5,7]
+ CRUSH rule 0 x 253 [5,2]
+ CRUSH rule 0 x 254 [5,2]
+ CRUSH rule 0 x 255 [2,7]
+ CRUSH rule 0 x 256 [5,6]
+ CRUSH rule 0 x 257 [2,6]
+ CRUSH rule 0 x 258 [5,2]
+ CRUSH rule 0 x 259 [5,6]
+ CRUSH rule 0 x 260 [5,8]
+ CRUSH rule 0 x 261 [8,5]
+ CRUSH rule 0 x 262 [5,6]
+ CRUSH rule 0 x 263 [6,2]
+ CRUSH rule 0 x 264 [5,6]
+ CRUSH rule 0 x 265 [8,5]
+ CRUSH rule 0 x 266 [8,2]
+ CRUSH rule 0 x 267 [2,5]
+ CRUSH rule 0 x 268 [2,6]
+ CRUSH rule 0 x 269 [2,6]
+ CRUSH rule 0 x 270 [5,2]
+ CRUSH rule 0 x 271 [7,5]
+ CRUSH rule 0 x 272 [2,6]
+ CRUSH rule 0 x 273 [5,2]
+ CRUSH rule 0 x 274 [6,5]
+ CRUSH rule 0 x 275 [5,8]
+ CRUSH rule 0 x 276 [7,2]
+ CRUSH rule 0 x 277 [6,5]
+ CRUSH rule 0 x 278 [6,2]
+ CRUSH rule 0 x 279 [8,5]
+ CRUSH rule 0 x 280 [2,7]
+ CRUSH rule 0 x 281 [8,2]
+ CRUSH rule 0 x 282 [5,2]
+ CRUSH rule 0 x 283 [8,2]
+ CRUSH rule 0 x 284 [6,5]
+ CRUSH rule 0 x 285 [5,7]
+ CRUSH rule 0 x 286 [2,8]
+ CRUSH rule 0 x 287 [2,5]
+ CRUSH rule 0 x 288 [8,2]
+ CRUSH rule 0 x 289 [5,6]
+ CRUSH rule 0 x 290 [2,5]
+ CRUSH rule 0 x 291 [2,5]
+ CRUSH rule 0 x 292 [8,2]
+ CRUSH rule 0 x 293 [6,2]
+ CRUSH rule 0 x 294 [7,5]
+ CRUSH rule 0 x 295 [5,6]
+ CRUSH rule 0 x 296 [5,2]
+ CRUSH rule 0 x 297 [6,2]
+ CRUSH rule 0 x 298 [2,5]
+ CRUSH rule 0 x 299 [2,8]
+ CRUSH rule 0 x 300 [8,5]
+ CRUSH rule 0 x 301 [2,7]
+ CRUSH rule 0 x 302 [5,2]
+ CRUSH rule 0 x 303 [7,5]
+ CRUSH rule 0 x 304 [2,6]
+ CRUSH rule 0 x 305 [5,6]
+ CRUSH rule 0 x 306 [2,8]
+ CRUSH rule 0 x 307 [2,7]
+ CRUSH rule 0 x 308 [2,8]
+ CRUSH rule 0 x 309 [7,5]
+ CRUSH rule 0 x 310 [5,2]
+ CRUSH rule 0 x 311 [5,8]
+ CRUSH rule 0 x 312 [2,6]
+ CRUSH rule 0 x 313 [5,2]
+ CRUSH rule 0 x 314 [5,2]
+ CRUSH rule 0 x 315 [2,5]
+ CRUSH rule 0 x 316 [6,5]
+ CRUSH rule 0 x 317 [2,7]
+ CRUSH rule 0 x 318 [8,2]
+ CRUSH rule 0 x 319 [5,2]
+ CRUSH rule 0 x 320 [5,8]
+ CRUSH rule 0 x 321 [2,5]
+ CRUSH rule 0 x 322 [2,6]
+ CRUSH rule 0 x 323 [5,7]
+ CRUSH rule 0 x 324 [7,2]
+ CRUSH rule 0 x 325 [5,6]
+ CRUSH rule 0 x 326 [5,2]
+ CRUSH rule 0 x 327 [2,8]
+ CRUSH rule 0 x 328 [7,5]
+ CRUSH rule 0 x 329 [5,7]
+ CRUSH rule 0 x 330 [5,7]
+ CRUSH rule 0 x 331 [2,7]
+ CRUSH rule 0 x 332 [2,5]
+ CRUSH rule 0 x 333 [6,5]
+ CRUSH rule 0 x 334 [8,5]
+ CRUSH rule 0 x 335 [7,2]
+ CRUSH rule 0 x 336 [5,6]
+ CRUSH rule 0 x 337 [7,2]
+ CRUSH rule 0 x 338 [5,8]
+ CRUSH rule 0 x 339 [7,5]
+ CRUSH rule 0 x 340 [2,6]
+ CRUSH rule 0 x 341 [5,2]
+ CRUSH rule 0 x 342 [2,8]
+ CRUSH rule 0 x 343 [6,5]
+ CRUSH rule 0 x 344 [6,2]
+ CRUSH rule 0 x 345 [5,7]
+ CRUSH rule 0 x 346 [8,2]
+ CRUSH rule 0 x 347 [5,2]
+ CRUSH rule 0 x 348 [8,2]
+ CRUSH rule 0 x 349 [2,7]
+ CRUSH rule 0 x 350 [8,5]
+ CRUSH rule 0 x 351 [5,8]
+ CRUSH rule 0 x 352 [2,8]
+ CRUSH rule 0 x 353 [6,5]
+ CRUSH rule 0 x 354 [2,5]
+ CRUSH rule 0 x 355 [5,8]
+ CRUSH rule 0 x 356 [5,2]
+ CRUSH rule 0 x 357 [6,2]
+ CRUSH rule 0 x 358 [2,8]
+ CRUSH rule 0 x 359 [6,2]
+ CRUSH rule 0 x 360 [5,2]
+ CRUSH rule 0 x 361 [8,5]
+ CRUSH rule 0 x 362 [5,2]
+ CRUSH rule 0 x 363 [5,2]
+ CRUSH rule 0 x 364 [2,5]
+ CRUSH rule 0 x 365 [6,5]
+ CRUSH rule 0 x 366 [7,2]
+ CRUSH rule 0 x 367 [5,2]
+ CRUSH rule 0 x 368 [7,5]
+ CRUSH rule 0 x 369 [5,7]
+ CRUSH rule 0 x 370 [8,2]
+ CRUSH rule 0 x 371 [2,5]
+ CRUSH rule 0 x 372 [5,2]
+ CRUSH rule 0 x 373 [2,6]
+ CRUSH rule 0 x 374 [5,8]
+ CRUSH rule 0 x 375 [6,5]
+ CRUSH rule 0 x 376 [7,2]
+ CRUSH rule 0 x 377 [2,5]
+ CRUSH rule 0 x 378 [2,6]
+ CRUSH rule 0 x 379 [8,5]
+ CRUSH rule 0 x 380 [2,5]
+ CRUSH rule 0 x 381 [2,5]
+ CRUSH rule 0 x 382 [2,5]
+ CRUSH rule 0 x 383 [5,7]
+ CRUSH rule 0 x 384 [7,2]
+ CRUSH rule 0 x 385 [7,5]
+ CRUSH rule 0 x 386 [2,5]
+ CRUSH rule 0 x 387 [2,5]
+ CRUSH rule 0 x 388 [5,2]
+ CRUSH rule 0 x 389 [2,5]
+ CRUSH rule 0 x 390 [5,8]
+ CRUSH rule 0 x 391 [5,6]
+ CRUSH rule 0 x 392 [2,7]
+ CRUSH rule 0 x 393 [5,2]
+ CRUSH rule 0 x 394 [5,8]
+ CRUSH rule 0 x 395 [5,2]
+ CRUSH rule 0 x 396 [5,2]
+ CRUSH rule 0 x 397 [2,5]
+ CRUSH rule 0 x 398 [2,5]
+ CRUSH rule 0 x 399 [8,5]
+ CRUSH rule 0 x 400 [8,2]
+ CRUSH rule 0 x 401 [2,5]
+ CRUSH rule 0 x 402 [7,5]
+ CRUSH rule 0 x 403 [2,5]
+ CRUSH rule 0 x 404 [5,2]
+ CRUSH rule 0 x 405 [6,5]
+ CRUSH rule 0 x 406 [2,6]
+ CRUSH rule 0 x 407 [2,7]
+ CRUSH rule 0 x 408 [5,2]
+ CRUSH rule 0 x 409 [7,5]
+ CRUSH rule 0 x 410 [8,5]
+ CRUSH rule 0 x 411 [2,7]
+ CRUSH rule 0 x 412 [2,5]
+ CRUSH rule 0 x 413 [5,2]
+ CRUSH rule 0 x 414 [5,2]
+ CRUSH rule 0 x 415 [2,6]
+ CRUSH rule 0 x 416 [2,8]
+ CRUSH rule 0 x 417 [8,2]
+ CRUSH rule 0 x 418 [7,2]
+ CRUSH rule 0 x 419 [8,5]
+ CRUSH rule 0 x 420 [2,5]
+ CRUSH rule 0 x 421 [8,5]
+ CRUSH rule 0 x 422 [6,5]
+ CRUSH rule 0 x 423 [2,5]
+ CRUSH rule 0 x 424 [8,5]
+ CRUSH rule 0 x 425 [2,5]
+ CRUSH rule 0 x 426 [6,2]
+ CRUSH rule 0 x 427 [2,8]
+ CRUSH rule 0 x 428 [5,8]
+ CRUSH rule 0 x 429 [5,8]
+ CRUSH rule 0 x 430 [5,7]
+ CRUSH rule 0 x 431 [5,2]
+ CRUSH rule 0 x 432 [7,2]
+ CRUSH rule 0 x 433 [6,5]
+ CRUSH rule 0 x 434 [5,2]
+ CRUSH rule 0 x 435 [2,5]
+ CRUSH rule 0 x 436 [5,2]
+ CRUSH rule 0 x 437 [7,5]
+ CRUSH rule 0 x 438 [2,5]
+ CRUSH rule 0 x 439 [2,5]
+ CRUSH rule 0 x 440 [2,6]
+ CRUSH rule 0 x 441 [5,8]
+ CRUSH rule 0 x 442 [2,5]
+ CRUSH rule 0 x 443 [6,2]
+ CRUSH rule 0 x 444 [7,2]
+ CRUSH rule 0 x 445 [6,5]
+ CRUSH rule 0 x 446 [5,2]
+ CRUSH rule 0 x 447 [2,5]
+ CRUSH rule 0 x 448 [7,2]
+ CRUSH rule 0 x 449 [7,5]
+ CRUSH rule 0 x 450 [5,2]
+ CRUSH rule 0 x 451 [6,5]
+ CRUSH rule 0 x 452 [8,5]
+ CRUSH rule 0 x 453 [6,5]
+ CRUSH rule 0 x 454 [6,5]
+ CRUSH rule 0 x 455 [2,8]
+ CRUSH rule 0 x 456 [6,2]
+ CRUSH rule 0 x 457 [7,2]
+ CRUSH rule 0 x 458 [2,8]
+ CRUSH rule 0 x 459 [2,6]
+ CRUSH rule 0 x 460 [6,5]
+ CRUSH rule 0 x 461 [6,5]
+ CRUSH rule 0 x 462 [8,2]
+ CRUSH rule 0 x 463 [6,2]
+ CRUSH rule 0 x 464 [7,5]
+ CRUSH rule 0 x 465 [7,2]
+ CRUSH rule 0 x 466 [5,6]
+ CRUSH rule 0 x 467 [6,5]
+ CRUSH rule 0 x 468 [7,2]
+ CRUSH rule 0 x 469 [7,2]
+ CRUSH rule 0 x 470 [5,2]
+ CRUSH rule 0 x 471 [2,6]
+ CRUSH rule 0 x 472 [5,2]
+ CRUSH rule 0 x 473 [2,5]
+ CRUSH rule 0 x 474 [6,2]
+ CRUSH rule 0 x 475 [6,2]
+ CRUSH rule 0 x 476 [5,7]
+ CRUSH rule 0 x 477 [5,6]
+ CRUSH rule 0 x 478 [6,2]
+ CRUSH rule 0 x 479 [2,5]
+ CRUSH rule 0 x 480 [2,6]
+ CRUSH rule 0 x 481 [2,5]
+ CRUSH rule 0 x 482 [5,8]
+ CRUSH rule 0 x 483 [2,6]
+ CRUSH rule 0 x 484 [2,8]
+ CRUSH rule 0 x 485 [5,8]
+ CRUSH rule 0 x 486 [5,2]
+ CRUSH rule 0 x 487 [5,2]
+ CRUSH rule 0 x 488 [5,7]
+ CRUSH rule 0 x 489 [2,8]
+ CRUSH rule 0 x 490 [6,5]
+ CRUSH rule 0 x 491 [2,6]
+ CRUSH rule 0 x 492 [6,5]
+ CRUSH rule 0 x 493 [2,8]
+ CRUSH rule 0 x 494 [2,6]
+ CRUSH rule 0 x 495 [5,2]
+ CRUSH rule 0 x 496 [7,5]
+ CRUSH rule 0 x 497 [5,7]
+ CRUSH rule 0 x 498 [2,5]
+ CRUSH rule 0 x 499 [8,5]
+ CRUSH rule 0 x 500 [5,6]
+ CRUSH rule 0 x 501 [2,7]
+ CRUSH rule 0 x 502 [7,2]
+ CRUSH rule 0 x 503 [2,5]
+ CRUSH rule 0 x 504 [5,8]
+ CRUSH rule 0 x 505 [2,7]
+ CRUSH rule 0 x 506 [5,2]
+ CRUSH rule 0 x 507 [6,2]
+ CRUSH rule 0 x 508 [2,5]
+ CRUSH rule 0 x 509 [7,5]
+ CRUSH rule 0 x 510 [6,2]
+ CRUSH rule 0 x 511 [5,6]
+ CRUSH rule 0 x 512 [7,2]
+ CRUSH rule 0 x 513 [7,2]
+ CRUSH rule 0 x 514 [5,7]
+ CRUSH rule 0 x 515 [8,5]
+ CRUSH rule 0 x 516 [5,2]
+ CRUSH rule 0 x 517 [7,2]
+ CRUSH rule 0 x 518 [5,6]
+ CRUSH rule 0 x 519 [7,5]
+ CRUSH rule 0 x 520 [2,8]
+ CRUSH rule 0 x 521 [8,2]
+ CRUSH rule 0 x 522 [6,2]
+ CRUSH rule 0 x 523 [5,2]
+ CRUSH rule 0 x 524 [2,5]
+ CRUSH rule 0 x 525 [2,5]
+ CRUSH rule 0 x 526 [2,5]
+ CRUSH rule 0 x 527 [2,5]
+ CRUSH rule 0 x 528 [5,2]
+ CRUSH rule 0 x 529 [5,6]
+ CRUSH rule 0 x 530 [6,5]
+ CRUSH rule 0 x 531 [6,2]
+ CRUSH rule 0 x 532 [6,5]
+ CRUSH rule 0 x 533 [5,8]
+ CRUSH rule 0 x 534 [7,5]
+ CRUSH rule 0 x 535 [8,2]
+ CRUSH rule 0 x 536 [6,2]
+ CRUSH rule 0 x 537 [5,8]
+ CRUSH rule 0 x 538 [6,5]
+ CRUSH rule 0 x 539 [8,5]
+ CRUSH rule 0 x 540 [2,7]
+ CRUSH rule 0 x 541 [2,5]
+ CRUSH rule 0 x 542 [5,2]
+ CRUSH rule 0 x 543 [6,2]
+ CRUSH rule 0 x 544 [5,7]
+ CRUSH rule 0 x 545 [5,7]
+ CRUSH rule 0 x 546 [6,2]
+ CRUSH rule 0 x 547 [8,2]
+ CRUSH rule 0 x 548 [5,2]
+ CRUSH rule 0 x 549 [5,7]
+ CRUSH rule 0 x 550 [2,5]
+ CRUSH rule 0 x 551 [7,5]
+ CRUSH rule 0 x 552 [5,7]
+ CRUSH rule 0 x 553 [5,2]
+ CRUSH rule 0 x 554 [2,6]
+ CRUSH rule 0 x 555 [5,2]
+ CRUSH rule 0 x 556 [5,6]
+ CRUSH rule 0 x 557 [7,5]
+ CRUSH rule 0 x 558 [5,2]
+ CRUSH rule 0 x 559 [5,2]
+ CRUSH rule 0 x 560 [8,5]
+ CRUSH rule 0 x 561 [6,5]
+ CRUSH rule 0 x 562 [5,2]
+ CRUSH rule 0 x 563 [2,7]
+ CRUSH rule 0 x 564 [5,2]
+ CRUSH rule 0 x 565 [5,8]
+ CRUSH rule 0 x 566 [5,6]
+ CRUSH rule 0 x 567 [5,7]
+ CRUSH rule 0 x 568 [7,5]
+ CRUSH rule 0 x 569 [5,2]
+ CRUSH rule 0 x 570 [2,5]
+ CRUSH rule 0 x 571 [5,6]
+ CRUSH rule 0 x 572 [5,2]
+ CRUSH rule 0 x 573 [5,2]
+ CRUSH rule 0 x 574 [2,5]
+ CRUSH rule 0 x 575 [8,2]
+ CRUSH rule 0 x 576 [5,7]
+ CRUSH rule 0 x 577 [8,2]
+ CRUSH rule 0 x 578 [6,2]
+ CRUSH rule 0 x 579 [5,2]
+ CRUSH rule 0 x 580 [5,2]
+ CRUSH rule 0 x 581 [7,2]
+ CRUSH rule 0 x 582 [2,8]
+ CRUSH rule 0 x 583 [6,2]
+ CRUSH rule 0 x 584 [8,2]
+ CRUSH rule 0 x 585 [7,2]
+ CRUSH rule 0 x 586 [2,8]
+ CRUSH rule 0 x 587 [2,5]
+ CRUSH rule 0 x 588 [5,8]
+ CRUSH rule 0 x 589 [7,2]
+ CRUSH rule 0 x 590 [6,2]
+ CRUSH rule 0 x 591 [5,2]
+ CRUSH rule 0 x 592 [2,5]
+ CRUSH rule 0 x 593 [2,8]
+ CRUSH rule 0 x 594 [2,6]
+ CRUSH rule 0 x 595 [7,2]
+ CRUSH rule 0 x 596 [5,2]
+ CRUSH rule 0 x 597 [5,2]
+ CRUSH rule 0 x 598 [5,2]
+ CRUSH rule 0 x 599 [5,2]
+ CRUSH rule 0 x 600 [7,2]
+ CRUSH rule 0 x 601 [2,6]
+ CRUSH rule 0 x 602 [5,8]
+ CRUSH rule 0 x 603 [5,2]
+ CRUSH rule 0 x 604 [7,5]
+ CRUSH rule 0 x 605 [5,2]
+ CRUSH rule 0 x 606 [2,7]
+ CRUSH rule 0 x 607 [2,5]
+ CRUSH rule 0 x 608 [5,2]
+ CRUSH rule 0 x 609 [5,2]
+ CRUSH rule 0 x 610 [5,8]
+ CRUSH rule 0 x 611 [2,8]
+ CRUSH rule 0 x 612 [2,8]
+ CRUSH rule 0 x 613 [7,2]
+ CRUSH rule 0 x 614 [7,2]
+ CRUSH rule 0 x 615 [6,2]
+ CRUSH rule 0 x 616 [2,7]
+ CRUSH rule 0 x 617 [6,2]
+ CRUSH rule 0 x 618 [7,5]
+ CRUSH rule 0 x 619 [5,2]
+ CRUSH rule 0 x 620 [5,2]
+ CRUSH rule 0 x 621 [5,6]
+ CRUSH rule 0 x 622 [2,5]
+ CRUSH rule 0 x 623 [2,8]
+ CRUSH rule 0 x 624 [5,2]
+ CRUSH rule 0 x 625 [2,5]
+ CRUSH rule 0 x 626 [7,2]
+ CRUSH rule 0 x 627 [2,6]
+ CRUSH rule 0 x 628 [8,2]
+ CRUSH rule 0 x 629 [2,6]
+ CRUSH rule 0 x 630 [2,6]
+ CRUSH rule 0 x 631 [2,6]
+ CRUSH rule 0 x 632 [7,2]
+ CRUSH rule 0 x 633 [8,5]
+ CRUSH rule 0 x 634 [2,5]
+ CRUSH rule 0 x 635 [5,6]
+ CRUSH rule 0 x 636 [2,5]
+ CRUSH rule 0 x 637 [5,2]
+ CRUSH rule 0 x 638 [6,2]
+ CRUSH rule 0 x 639 [5,2]
+ CRUSH rule 0 x 640 [5,2]
+ CRUSH rule 0 x 641 [7,2]
+ CRUSH rule 0 x 642 [2,8]
+ CRUSH rule 0 x 643 [5,2]
+ CRUSH rule 0 x 644 [8,2]
+ CRUSH rule 0 x 645 [5,7]
+ CRUSH rule 0 x 646 [8,2]
+ CRUSH rule 0 x 647 [7,2]
+ CRUSH rule 0 x 648 [2,8]
+ CRUSH rule 0 x 649 [5,7]
+ CRUSH rule 0 x 650 [7,5]
+ CRUSH rule 0 x 651 [5,6]
+ CRUSH rule 0 x 652 [5,6]
+ CRUSH rule 0 x 653 [8,5]
+ CRUSH rule 0 x 654 [7,5]
+ CRUSH rule 0 x 655 [2,5]
+ CRUSH rule 0 x 656 [5,7]
+ CRUSH rule 0 x 657 [6,2]
+ CRUSH rule 0 x 658 [5,8]
+ CRUSH rule 0 x 659 [5,7]
+ CRUSH rule 0 x 660 [7,5]
+ CRUSH rule 0 x 661 [2,7]
+ CRUSH rule 0 x 662 [5,2]
+ CRUSH rule 0 x 663 [2,5]
+ CRUSH rule 0 x 664 [2,5]
+ CRUSH rule 0 x 665 [5,6]
+ CRUSH rule 0 x 666 [2,7]
+ CRUSH rule 0 x 667 [2,5]
+ CRUSH rule 0 x 668 [5,7]
+ CRUSH rule 0 x 669 [6,5]
+ CRUSH rule 0 x 670 [5,2]
+ CRUSH rule 0 x 671 [2,8]
+ CRUSH rule 0 x 672 [5,2]
+ CRUSH rule 0 x 673 [5,2]
+ CRUSH rule 0 x 674 [5,2]
+ CRUSH rule 0 x 675 [2,8]
+ CRUSH rule 0 x 676 [2,5]
+ CRUSH rule 0 x 677 [5,2]
+ CRUSH rule 0 x 678 [2,5]
+ CRUSH rule 0 x 679 [6,2]
+ CRUSH rule 0 x 680 [2,5]
+ CRUSH rule 0 x 681 [5,6]
+ CRUSH rule 0 x 682 [2,5]
+ CRUSH rule 0 x 683 [2,5]
+ CRUSH rule 0 x 684 [7,2]
+ CRUSH rule 0 x 685 [7,2]
+ CRUSH rule 0 x 686 [2,5]
+ CRUSH rule 0 x 687 [5,7]
+ CRUSH rule 0 x 688 [5,6]
+ CRUSH rule 0 x 689 [6,5]
+ CRUSH rule 0 x 690 [8,2]
+ CRUSH rule 0 x 691 [5,2]
+ CRUSH rule 0 x 692 [7,2]
+ CRUSH rule 0 x 693 [6,5]
+ CRUSH rule 0 x 694 [6,5]
+ CRUSH rule 0 x 695 [2,6]
+ CRUSH rule 0 x 696 [2,5]
+ CRUSH rule 0 x 697 [6,2]
+ CRUSH rule 0 x 698 [6,2]
+ CRUSH rule 0 x 699 [2,8]
+ CRUSH rule 0 x 700 [2,5]
+ CRUSH rule 0 x 701 [5,2]
+ CRUSH rule 0 x 702 [5,2]
+ CRUSH rule 0 x 703 [8,5]
+ CRUSH rule 0 x 704 [2,5]
+ CRUSH rule 0 x 705 [8,2]
+ CRUSH rule 0 x 706 [2,5]
+ CRUSH rule 0 x 707 [7,5]
+ CRUSH rule 0 x 708 [5,7]
+ CRUSH rule 0 x 709 [6,5]
+ CRUSH rule 0 x 710 [8,5]
+ CRUSH rule 0 x 711 [2,5]
+ CRUSH rule 0 x 712 [2,5]
+ CRUSH rule 0 x 713 [6,5]
+ CRUSH rule 0 x 714 [5,2]
+ CRUSH rule 0 x 715 [2,5]
+ CRUSH rule 0 x 716 [5,6]
+ CRUSH rule 0 x 717 [8,2]
+ CRUSH rule 0 x 718 [5,7]
+ CRUSH rule 0 x 719 [2,6]
+ CRUSH rule 0 x 720 [6,2]
+ CRUSH rule 0 x 721 [5,7]
+ CRUSH rule 0 x 722 [5,7]
+ CRUSH rule 0 x 723 [5,2]
+ CRUSH rule 0 x 724 [2,7]
+ CRUSH rule 0 x 725 [2,5]
+ CRUSH rule 0 x 726 [5,7]
+ CRUSH rule 0 x 727 [5,7]
+ CRUSH rule 0 x 728 [2,6]
+ CRUSH rule 0 x 729 [5,6]
+ CRUSH rule 0 x 730 [5,8]
+ CRUSH rule 0 x 731 [5,2]
+ CRUSH rule 0 x 732 [2,5]
+ CRUSH rule 0 x 733 [5,6]
+ CRUSH rule 0 x 734 [6,5]
+ CRUSH rule 0 x 735 [5,6]
+ CRUSH rule 0 x 736 [5,8]
+ CRUSH rule 0 x 737 [2,8]
+ CRUSH rule 0 x 738 [5,2]
+ CRUSH rule 0 x 739 [2,7]
+ CRUSH rule 0 x 740 [2,7]
+ CRUSH rule 0 x 741 [7,2]
+ CRUSH rule 0 x 742 [8,2]
+ CRUSH rule 0 x 743 [7,2]
+ CRUSH rule 0 x 744 [5,6]
+ CRUSH rule 0 x 745 [5,2]
+ CRUSH rule 0 x 746 [5,2]
+ CRUSH rule 0 x 747 [6,2]
+ CRUSH rule 0 x 748 [2,6]
+ CRUSH rule 0 x 749 [5,7]
+ CRUSH rule 0 x 750 [2,7]
+ CRUSH rule 0 x 751 [2,7]
+ CRUSH rule 0 x 752 [8,2]
+ CRUSH rule 0 x 753 [7,5]
+ CRUSH rule 0 x 754 [8,5]
+ CRUSH rule 0 x 755 [2,6]
+ CRUSH rule 0 x 756 [5,8]
+ CRUSH rule 0 x 757 [8,2]
+ CRUSH rule 0 x 758 [6,2]
+ CRUSH rule 0 x 759 [8,5]
+ CRUSH rule 0 x 760 [2,5]
+ CRUSH rule 0 x 761 [5,2]
+ CRUSH rule 0 x 762 [2,8]
+ CRUSH rule 0 x 763 [8,5]
+ CRUSH rule 0 x 764 [2,7]
+ CRUSH rule 0 x 765 [6,5]
+ CRUSH rule 0 x 766 [8,5]
+ CRUSH rule 0 x 767 [2,8]
+ CRUSH rule 0 x 768 [8,5]
+ CRUSH rule 0 x 769 [6,2]
+ CRUSH rule 0 x 770 [6,2]
+ CRUSH rule 0 x 771 [7,2]
+ CRUSH rule 0 x 772 [8,5]
+ CRUSH rule 0 x 773 [5,2]
+ CRUSH rule 0 x 774 [5,7]
+ CRUSH rule 0 x 775 [6,5]
+ CRUSH rule 0 x 776 [7,2]
+ CRUSH rule 0 x 777 [5,2]
+ CRUSH rule 0 x 778 [2,6]
+ CRUSH rule 0 x 779 [2,6]
+ CRUSH rule 0 x 780 [2,5]
+ CRUSH rule 0 x 781 [6,5]
+ CRUSH rule 0 x 782 [5,2]
+ CRUSH rule 0 x 783 [7,2]
+ CRUSH rule 0 x 784 [2,5]
+ CRUSH rule 0 x 785 [6,2]
+ CRUSH rule 0 x 786 [7,5]
+ CRUSH rule 0 x 787 [2,8]
+ CRUSH rule 0 x 788 [6,2]
+ CRUSH rule 0 x 789 [2,5]
+ CRUSH rule 0 x 790 [8,5]
+ CRUSH rule 0 x 791 [5,6]
+ CRUSH rule 0 x 792 [5,6]
+ CRUSH rule 0 x 793 [6,2]
+ CRUSH rule 0 x 794 [2,8]
+ CRUSH rule 0 x 795 [2,5]
+ CRUSH rule 0 x 796 [5,7]
+ CRUSH rule 0 x 797 [2,5]
+ CRUSH rule 0 x 798 [6,2]
+ CRUSH rule 0 x 799 [5,2]
+ CRUSH rule 0 x 800 [5,2]
+ CRUSH rule 0 x 801 [5,7]
+ CRUSH rule 0 x 802 [2,6]
+ CRUSH rule 0 x 803 [2,5]
+ CRUSH rule 0 x 804 [6,2]
+ CRUSH rule 0 x 805 [5,8]
+ CRUSH rule 0 x 806 [2,5]
+ CRUSH rule 0 x 807 [5,7]
+ CRUSH rule 0 x 808 [5,7]
+ CRUSH rule 0 x 809 [2,5]
+ CRUSH rule 0 x 810 [5,7]
+ CRUSH rule 0 x 811 [8,5]
+ CRUSH rule 0 x 812 [8,5]
+ CRUSH rule 0 x 813 [6,5]
+ CRUSH rule 0 x 814 [5,8]
+ CRUSH rule 0 x 815 [5,2]
+ CRUSH rule 0 x 816 [2,6]
+ CRUSH rule 0 x 817 [5,6]
+ CRUSH rule 0 x 818 [5,2]
+ CRUSH rule 0 x 819 [5,2]
+ CRUSH rule 0 x 820 [5,7]
+ CRUSH rule 0 x 821 [5,8]
+ CRUSH rule 0 x 822 [2,5]
+ CRUSH rule 0 x 823 [5,7]
+ CRUSH rule 0 x 824 [5,7]
+ CRUSH rule 0 x 825 [2,8]
+ CRUSH rule 0 x 826 [7,2]
+ CRUSH rule 0 x 827 [2,6]
+ CRUSH rule 0 x 828 [2,5]
+ CRUSH rule 0 x 829 [5,6]
+ CRUSH rule 0 x 830 [2,5]
+ CRUSH rule 0 x 831 [2,6]
+ CRUSH rule 0 x 832 [5,8]
+ CRUSH rule 0 x 833 [2,6]
+ CRUSH rule 0 x 834 [5,2]
+ CRUSH rule 0 x 835 [8,5]
+ CRUSH rule 0 x 836 [5,8]
+ CRUSH rule 0 x 837 [6,5]
+ CRUSH rule 0 x 838 [6,2]
+ CRUSH rule 0 x 839 [5,2]
+ CRUSH rule 0 x 840 [7,5]
+ CRUSH rule 0 x 841 [5,8]
+ CRUSH rule 0 x 842 [2,5]
+ CRUSH rule 0 x 843 [6,5]
+ CRUSH rule 0 x 844 [5,8]
+ CRUSH rule 0 x 845 [5,6]
+ CRUSH rule 0 x 846 [5,2]
+ CRUSH rule 0 x 847 [2,8]
+ CRUSH rule 0 x 848 [2,6]
+ CRUSH rule 0 x 849 [5,8]
+ CRUSH rule 0 x 850 [2,5]
+ CRUSH rule 0 x 851 [6,5]
+ CRUSH rule 0 x 852 [7,5]
+ CRUSH rule 0 x 853 [6,2]
+ CRUSH rule 0 x 854 [7,2]
+ CRUSH rule 0 x 855 [5,7]
+ CRUSH rule 0 x 856 [6,5]
+ CRUSH rule 0 x 857 [8,5]
+ CRUSH rule 0 x 858 [6,5]
+ CRUSH rule 0 x 859 [6,2]
+ CRUSH rule 0 x 860 [5,2]
+ CRUSH rule 0 x 861 [8,5]
+ CRUSH rule 0 x 862 [6,2]
+ CRUSH rule 0 x 863 [8,2]
+ CRUSH rule 0 x 864 [5,6]
+ CRUSH rule 0 x 865 [8,2]
+ CRUSH rule 0 x 866 [5,6]
+ CRUSH rule 0 x 867 [6,5]
+ CRUSH rule 0 x 868 [6,5]
+ CRUSH rule 0 x 869 [8,5]
+ CRUSH rule 0 x 870 [2,5]
+ CRUSH rule 0 x 871 [5,2]
+ CRUSH rule 0 x 872 [5,2]
+ CRUSH rule 0 x 873 [5,6]
+ CRUSH rule 0 x 874 [2,6]
+ CRUSH rule 0 x 875 [2,6]
+ CRUSH rule 0 x 876 [5,8]
+ CRUSH rule 0 x 877 [6,5]
+ CRUSH rule 0 x 878 [5,2]
+ CRUSH rule 0 x 879 [7,5]
+ CRUSH rule 0 x 880 [5,2]
+ CRUSH rule 0 x 881 [5,8]
+ CRUSH rule 0 x 882 [5,2]
+ CRUSH rule 0 x 883 [2,5]
+ CRUSH rule 0 x 884 [6,2]
+ CRUSH rule 0 x 885 [5,2]
+ CRUSH rule 0 x 886 [5,7]
+ CRUSH rule 0 x 887 [7,5]
+ CRUSH rule 0 x 888 [6,2]
+ CRUSH rule 0 x 889 [2,8]
+ CRUSH rule 0 x 890 [7,2]
+ CRUSH rule 0 x 891 [2,7]
+ CRUSH rule 0 x 892 [6,2]
+ CRUSH rule 0 x 893 [2,5]
+ CRUSH rule 0 x 894 [7,5]
+ CRUSH rule 0 x 895 [5,2]
+ CRUSH rule 0 x 896 [2,6]
+ CRUSH rule 0 x 897 [5,2]
+ CRUSH rule 0 x 898 [2,5]
+ CRUSH rule 0 x 899 [2,6]
+ CRUSH rule 0 x 900 [5,2]
+ CRUSH rule 0 x 901 [5,2]
+ CRUSH rule 0 x 902 [8,5]
+ CRUSH rule 0 x 903 [5,6]
+ CRUSH rule 0 x 904 [5,7]
+ CRUSH rule 0 x 905 [6,2]
+ CRUSH rule 0 x 906 [2,7]
+ CRUSH rule 0 x 907 [7,2]
+ CRUSH rule 0 x 908 [5,6]
+ CRUSH rule 0 x 909 [2,5]
+ CRUSH rule 0 x 910 [6,5]
+ CRUSH rule 0 x 911 [5,7]
+ CRUSH rule 0 x 912 [2,8]
+ CRUSH rule 0 x 913 [7,2]
+ CRUSH rule 0 x 914 [6,5]
+ CRUSH rule 0 x 915 [8,2]
+ CRUSH rule 0 x 916 [5,2]
+ CRUSH rule 0 x 917 [2,5]
+ CRUSH rule 0 x 918 [8,2]
+ CRUSH rule 0 x 919 [6,2]
+ CRUSH rule 0 x 920 [7,5]
+ CRUSH rule 0 x 921 [2,5]
+ CRUSH rule 0 x 922 [6,5]
+ CRUSH rule 0 x 923 [5,6]
+ CRUSH rule 0 x 924 [5,2]
+ CRUSH rule 0 x 925 [5,6]
+ CRUSH rule 0 x 926 [5,2]
+ CRUSH rule 0 x 927 [2,6]
+ CRUSH rule 0 x 928 [8,2]
+ CRUSH rule 0 x 929 [5,2]
+ CRUSH rule 0 x 930 [2,5]
+ CRUSH rule 0 x 931 [5,2]
+ CRUSH rule 0 x 932 [5,2]
+ CRUSH rule 0 x 933 [8,5]
+ CRUSH rule 0 x 934 [5,8]
+ CRUSH rule 0 x 935 [6,5]
+ CRUSH rule 0 x 936 [2,7]
+ CRUSH rule 0 x 937 [5,8]
+ CRUSH rule 0 x 938 [6,5]
+ CRUSH rule 0 x 939 [2,8]
+ CRUSH rule 0 x 940 [8,5]
+ CRUSH rule 0 x 941 [5,2]
+ CRUSH rule 0 x 942 [2,6]
+ CRUSH rule 0 x 943 [8,2]
+ CRUSH rule 0 x 944 [5,8]
+ CRUSH rule 0 x 945 [7,2]
+ CRUSH rule 0 x 946 [2,8]
+ CRUSH rule 0 x 947 [5,2]
+ CRUSH rule 0 x 948 [7,5]
+ CRUSH rule 0 x 949 [6,2]
+ CRUSH rule 0 x 950 [5,7]
+ CRUSH rule 0 x 951 [5,6]
+ CRUSH rule 0 x 952 [2,7]
+ CRUSH rule 0 x 953 [2,5]
+ CRUSH rule 0 x 954 [5,2]
+ CRUSH rule 0 x 955 [8,2]
+ CRUSH rule 0 x 956 [2,7]
+ CRUSH rule 0 x 957 [7,2]
+ CRUSH rule 0 x 958 [8,5]
+ CRUSH rule 0 x 959 [5,2]
+ CRUSH rule 0 x 960 [5,6]
+ CRUSH rule 0 x 961 [5,2]
+ CRUSH rule 0 x 962 [7,5]
+ CRUSH rule 0 x 963 [2,5]
+ CRUSH rule 0 x 964 [5,2]
+ CRUSH rule 0 x 965 [7,5]
+ CRUSH rule 0 x 966 [5,6]
+ CRUSH rule 0 x 967 [8,5]
+ CRUSH rule 0 x 968 [7,2]
+ CRUSH rule 0 x 969 [8,2]
+ CRUSH rule 0 x 970 [2,8]
+ CRUSH rule 0 x 971 [2,8]
+ CRUSH rule 0 x 972 [2,7]
+ CRUSH rule 0 x 973 [2,8]
+ CRUSH rule 0 x 974 [5,2]
+ CRUSH rule 0 x 975 [5,8]
+ CRUSH rule 0 x 976 [5,7]
+ CRUSH rule 0 x 977 [8,5]
+ CRUSH rule 0 x 978 [7,2]
+ CRUSH rule 0 x 979 [7,2]
+ CRUSH rule 0 x 980 [6,2]
+ CRUSH rule 0 x 981 [7,5]
+ CRUSH rule 0 x 982 [5,2]
+ CRUSH rule 0 x 983 [5,6]
+ CRUSH rule 0 x 984 [2,8]
+ CRUSH rule 0 x 985 [2,5]
+ CRUSH rule 0 x 986 [8,5]
+ CRUSH rule 0 x 987 [2,5]
+ CRUSH rule 0 x 988 [2,5]
+ CRUSH rule 0 x 989 [2,8]
+ CRUSH rule 0 x 990 [2,6]
+ CRUSH rule 0 x 991 [2,5]
+ CRUSH rule 0 x 992 [7,2]
+ CRUSH rule 0 x 993 [2,6]
+ CRUSH rule 0 x 994 [5,6]
+ CRUSH rule 0 x 995 [7,2]
+ CRUSH rule 0 x 996 [6,5]
+ CRUSH rule 0 x 997 [6,5]
+ CRUSH rule 0 x 998 [8,2]
+ CRUSH rule 0 x 999 [2,7]
+ CRUSH rule 0 x 1000 [8,5]
+ CRUSH rule 0 x 1001 [2,5]
+ CRUSH rule 0 x 1002 [2,5]
+ CRUSH rule 0 x 1003 [2,7]
+ CRUSH rule 0 x 1004 [6,2]
+ CRUSH rule 0 x 1005 [6,2]
+ CRUSH rule 0 x 1006 [2,8]
+ CRUSH rule 0 x 1007 [2,5]
+ CRUSH rule 0 x 1008 [2,7]
+ CRUSH rule 0 x 1009 [6,5]
+ CRUSH rule 0 x 1010 [5,2]
+ CRUSH rule 0 x 1011 [5,2]
+ CRUSH rule 0 x 1012 [5,2]
+ CRUSH rule 0 x 1013 [5,2]
+ CRUSH rule 0 x 1014 [2,8]
+ CRUSH rule 0 x 1015 [6,5]
+ CRUSH rule 0 x 1016 [2,5]
+ CRUSH rule 0 x 1017 [6,2]
+ CRUSH rule 0 x 1018 [5,2]
+ CRUSH rule 0 x 1019 [5,8]
+ CRUSH rule 0 x 1020 [5,2]
+ CRUSH rule 0 x 1021 [5,2]
+ CRUSH rule 0 x 1022 [2,7]
+ CRUSH rule 0 x 1023 [5,2]
+ rule 0 (choose) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [2,5,7]
+ CRUSH rule 0 x 1 [2,8,5]
+ CRUSH rule 0 x 2 [2,5,8]
+ CRUSH rule 0 x 3 [8,2,5]
+ CRUSH rule 0 x 4 [5,2,8]
+ CRUSH rule 0 x 5 [7,2,5]
+ CRUSH rule 0 x 6 [2,6,5]
+ CRUSH rule 0 x 7 [5,6,2]
+ CRUSH rule 0 x 8 [5,7,2]
+ CRUSH rule 0 x 9 [2,5,8]
+ CRUSH rule 0 x 10 [2,8,5]
+ CRUSH rule 0 x 11 [2,6,5]
+ CRUSH rule 0 x 12 [2,5,7]
+ CRUSH rule 0 x 13 [5,8,2]
+ CRUSH rule 0 x 14 [7,2,5]
+ CRUSH rule 0 x 15 [7,2,5]
+ CRUSH rule 0 x 16 [5,7,2]
+ CRUSH rule 0 x 17 [5,2,7]
+ CRUSH rule 0 x 18 [2,5,6]
+ CRUSH rule 0 x 19 [7,5,2]
+ CRUSH rule 0 x 20 [2,5,7]
+ CRUSH rule 0 x 21 [5,6,2]
+ CRUSH rule 0 x 22 [8,5,2]
+ CRUSH rule 0 x 23 [5,7,2]
+ CRUSH rule 0 x 24 [2,6,5]
+ CRUSH rule 0 x 25 [5,8,2]
+ CRUSH rule 0 x 26 [2,7,5]
+ CRUSH rule 0 x 27 [5,2,7]
+ CRUSH rule 0 x 28 [6,2,5]
+ CRUSH rule 0 x 29 [8,5,2]
+ CRUSH rule 0 x 30 [5,6,2]
+ CRUSH rule 0 x 31 [8,2,5]
+ CRUSH rule 0 x 32 [5,7,2]
+ CRUSH rule 0 x 33 [2,7,5]
+ CRUSH rule 0 x 34 [2,5,7]
+ CRUSH rule 0 x 35 [2,6,5]
+ CRUSH rule 0 x 36 [5,6,2]
+ CRUSH rule 0 x 37 [2,5,7]
+ CRUSH rule 0 x 38 [5,6,2]
+ CRUSH rule 0 x 39 [5,7,2]
+ CRUSH rule 0 x 40 [7,2,5]
+ CRUSH rule 0 x 41 [2,7,5]
+ CRUSH rule 0 x 42 [5,7,2]
+ CRUSH rule 0 x 43 [2,5,8]
+ CRUSH rule 0 x 44 [2,8,5]
+ CRUSH rule 0 x 45 [8,2,5]
+ CRUSH rule 0 x 46 [2,5,8]
+ CRUSH rule 0 x 47 [5,2,7]
+ CRUSH rule 0 x 48 [5,6,2]
+ CRUSH rule 0 x 49 [5,6,2]
+ CRUSH rule 0 x 50 [5,2,7]
+ CRUSH rule 0 x 51 [5,6,2]
+ CRUSH rule 0 x 52 [8,2,5]
+ CRUSH rule 0 x 53 [5,6,2]
+ CRUSH rule 0 x 54 [7,5,2]
+ CRUSH rule 0 x 55 [8,2,5]
+ CRUSH rule 0 x 56 [6,5,2]
+ CRUSH rule 0 x 57 [5,8,2]
+ CRUSH rule 0 x 58 [2,8,5]
+ CRUSH rule 0 x 59 [5,2,7]
+ CRUSH rule 0 x 60 [5,2,8]
+ CRUSH rule 0 x 61 [5,8,2]
+ CRUSH rule 0 x 62 [7,2,5]
+ CRUSH rule 0 x 63 [5,6,2]
+ CRUSH rule 0 x 64 [5,2,8]
+ CRUSH rule 0 x 65 [7,5,2]
+ CRUSH rule 0 x 66 [5,6,2]
+ CRUSH rule 0 x 67 [5,2,6]
+ CRUSH rule 0 x 68 [2,5,8]
+ CRUSH rule 0 x 69 [5,2,7]
+ CRUSH rule 0 x 70 [7,2,5]
+ CRUSH rule 0 x 71 [2,7,5]
+ CRUSH rule 0 x 72 [6,2,5]
+ CRUSH rule 0 x 73 [2,7,5]
+ CRUSH rule 0 x 74 [2,8,5]
+ CRUSH rule 0 x 75 [5,2,7]
+ CRUSH rule 0 x 76 [5,2,6]
+ CRUSH rule 0 x 77 [7,2,5]
+ CRUSH rule 0 x 78 [2,5,7]
+ CRUSH rule 0 x 79 [5,2,7]
+ CRUSH rule 0 x 80 [2,5,6]
+ CRUSH rule 0 x 81 [2,5,8]
+ CRUSH rule 0 x 82 [7,2,5]
+ CRUSH rule 0 x 83 [2,6,5]
+ CRUSH rule 0 x 84 [7,2,5]
+ CRUSH rule 0 x 85 [5,7,2]
+ CRUSH rule 0 x 86 [2,6,5]
+ CRUSH rule 0 x 87 [2,6,5]
+ CRUSH rule 0 x 88 [2,8,5]
+ CRUSH rule 0 x 89 [5,2,8]
+ CRUSH rule 0 x 90 [6,5,2]
+ CRUSH rule 0 x 91 [5,6,2]
+ CRUSH rule 0 x 92 [2,6,5]
+ CRUSH rule 0 x 93 [7,5,2]
+ CRUSH rule 0 x 94 [2,5,7]
+ CRUSH rule 0 x 95 [7,5,2]
+ CRUSH rule 0 x 96 [5,8,2]
+ CRUSH rule 0 x 97 [8,5,2]
+ CRUSH rule 0 x 98 [2,7,5]
+ CRUSH rule 0 x 99 [2,8,5]
+ CRUSH rule 0 x 100 [2,6,5]
+ CRUSH rule 0 x 101 [5,8,2]
+ CRUSH rule 0 x 102 [5,2,7]
+ CRUSH rule 0 x 103 [5,7,2]
+ CRUSH rule 0 x 104 [7,5,2]
+ CRUSH rule 0 x 105 [2,5,8]
+ CRUSH rule 0 x 106 [2,6,5]
+ CRUSH rule 0 x 107 [5,2,7]
+ CRUSH rule 0 x 108 [7,2,5]
+ CRUSH rule 0 x 109 [2,5,8]
+ CRUSH rule 0 x 110 [5,2,6]
+ CRUSH rule 0 x 111 [2,5,7]
+ CRUSH rule 0 x 112 [2,6,5]
+ CRUSH rule 0 x 113 [6,2,5]
+ CRUSH rule 0 x 114 [7,5,2]
+ CRUSH rule 0 x 115 [8,2,5]
+ CRUSH rule 0 x 116 [2,7,5]
+ CRUSH rule 0 x 117 [7,5,2]
+ CRUSH rule 0 x 118 [2,5,6]
+ CRUSH rule 0 x 119 [5,7,2]
+ CRUSH rule 0 x 120 [2,5,6]
+ CRUSH rule 0 x 121 [2,8,5]
+ CRUSH rule 0 x 122 [8,5,2]
+ CRUSH rule 0 x 123 [2,5,7]
+ CRUSH rule 0 x 124 [5,2,8]
+ CRUSH rule 0 x 125 [2,7,5]
+ CRUSH rule 0 x 126 [5,2,7]
+ CRUSH rule 0 x 127 [5,6,2]
+ CRUSH rule 0 x 128 [5,8,2]
+ CRUSH rule 0 x 129 [2,5,7]
+ CRUSH rule 0 x 130 [5,8,2]
+ CRUSH rule 0 x 131 [2,5,6]
+ CRUSH rule 0 x 132 [2,5,8]
+ CRUSH rule 0 x 133 [5,6,2]
+ CRUSH rule 0 x 134 [2,7,5]
+ CRUSH rule 0 x 135 [5,7,2]
+ CRUSH rule 0 x 136 [2,5,8]
+ CRUSH rule 0 x 137 [7,5,2]
+ CRUSH rule 0 x 138 [8,5,2]
+ CRUSH rule 0 x 139 [5,2,8]
+ CRUSH rule 0 x 140 [2,8,5]
+ CRUSH rule 0 x 141 [6,2,5]
+ CRUSH rule 0 x 142 [5,2,8]
+ CRUSH rule 0 x 143 [5,7,2]
+ CRUSH rule 0 x 144 [8,2,5]
+ CRUSH rule 0 x 145 [8,5,2]
+ CRUSH rule 0 x 146 [2,8,5]
+ CRUSH rule 0 x 147 [2,6,5]
+ CRUSH rule 0 x 148 [5,2,8]
+ CRUSH rule 0 x 149 [5,6,2]
+ CRUSH rule 0 x 150 [2,8,5]
+ CRUSH rule 0 x 151 [5,8,2]
+ CRUSH rule 0 x 152 [8,5,2]
+ CRUSH rule 0 x 153 [8,5,2]
+ CRUSH rule 0 x 154 [5,2,6]
+ CRUSH rule 0 x 155 [5,6,2]
+ CRUSH rule 0 x 156 [5,2,7]
+ CRUSH rule 0 x 157 [5,2,7]
+ CRUSH rule 0 x 158 [2,6,5]
+ CRUSH rule 0 x 159 [7,2,5]
+ CRUSH rule 0 x 160 [2,8,5]
+ CRUSH rule 0 x 161 [2,5,6]
+ CRUSH rule 0 x 162 [2,8,5]
+ CRUSH rule 0 x 163 [5,8,2]
+ CRUSH rule 0 x 164 [7,2,5]
+ CRUSH rule 0 x 165 [7,2,5]
+ CRUSH rule 0 x 166 [2,5,7]
+ CRUSH rule 0 x 167 [2,6,5]
+ CRUSH rule 0 x 168 [5,2,7]
+ CRUSH rule 0 x 169 [2,7,5]
+ CRUSH rule 0 x 170 [2,5,8]
+ CRUSH rule 0 x 171 [7,5,2]
+ CRUSH rule 0 x 172 [2,8,5]
+ CRUSH rule 0 x 173 [8,5,2]
+ CRUSH rule 0 x 174 [2,5,8]
+ CRUSH rule 0 x 175 [6,2,5]
+ CRUSH rule 0 x 176 [5,2,7]
+ CRUSH rule 0 x 177 [5,2,8]
+ CRUSH rule 0 x 178 [5,2,6]
+ CRUSH rule 0 x 179 [5,2,7]
+ CRUSH rule 0 x 180 [5,7,2]
+ CRUSH rule 0 x 181 [6,2,5]
+ CRUSH rule 0 x 182 [8,5,2]
+ CRUSH rule 0 x 183 [7,5,2]
+ CRUSH rule 0 x 184 [5,6,2]
+ CRUSH rule 0 x 185 [6,2,5]
+ CRUSH rule 0 x 186 [2,5,8]
+ CRUSH rule 0 x 187 [2,6,5]
+ CRUSH rule 0 x 188 [2,6,5]
+ CRUSH rule 0 x 189 [2,6,5]
+ CRUSH rule 0 x 190 [5,2,8]
+ CRUSH rule 0 x 191 [7,2,5]
+ CRUSH rule 0 x 192 [5,2,6]
+ CRUSH rule 0 x 193 [5,2,6]
+ CRUSH rule 0 x 194 [2,5,7]
+ CRUSH rule 0 x 195 [6,5,2]
+ CRUSH rule 0 x 196 [6,2,5]
+ CRUSH rule 0 x 197 [6,5,2]
+ CRUSH rule 0 x 198 [2,5,6]
+ CRUSH rule 0 x 199 [2,5,7]
+ CRUSH rule 0 x 200 [2,5,8]
+ CRUSH rule 0 x 201 [7,2,5]
+ CRUSH rule 0 x 202 [6,5,2]
+ CRUSH rule 0 x 203 [5,6,2]
+ CRUSH rule 0 x 204 [2,5,8]
+ CRUSH rule 0 x 205 [2,7,5]
+ CRUSH rule 0 x 206 [2,8,5]
+ CRUSH rule 0 x 207 [5,2,7]
+ CRUSH rule 0 x 208 [7,2,5]
+ CRUSH rule 0 x 209 [2,8,5]
+ CRUSH rule 0 x 210 [2,5,6]
+ CRUSH rule 0 x 211 [5,2,7]
+ CRUSH rule 0 x 212 [7,5,2]
+ CRUSH rule 0 x 213 [8,5,2]
+ CRUSH rule 0 x 214 [5,7,2]
+ CRUSH rule 0 x 215 [8,2,5]
+ CRUSH rule 0 x 216 [5,2,8]
+ CRUSH rule 0 x 217 [2,7,5]
+ CRUSH rule 0 x 218 [2,7,5]
+ CRUSH rule 0 x 219 [5,6,2]
+ CRUSH rule 0 x 220 [5,8,2]
+ CRUSH rule 0 x 221 [5,7,2]
+ CRUSH rule 0 x 222 [6,5,2]
+ CRUSH rule 0 x 223 [2,5,6]
+ CRUSH rule 0 x 224 [2,5,8]
+ CRUSH rule 0 x 225 [8,2,5]
+ CRUSH rule 0 x 226 [7,2,5]
+ CRUSH rule 0 x 227 [5,2,7]
+ CRUSH rule 0 x 228 [5,6,2]
+ CRUSH rule 0 x 229 [5,7,2]
+ CRUSH rule 0 x 230 [5,6,2]
+ CRUSH rule 0 x 231 [5,7,2]
+ CRUSH rule 0 x 232 [2,8,5]
+ CRUSH rule 0 x 233 [5,6,2]
+ CRUSH rule 0 x 234 [2,5,6]
+ CRUSH rule 0 x 235 [5,6,2]
+ CRUSH rule 0 x 236 [5,2,8]
+ CRUSH rule 0 x 237 [5,8,2]
+ CRUSH rule 0 x 238 [5,2,6]
+ CRUSH rule 0 x 239 [8,5,2]
+ CRUSH rule 0 x 240 [5,8,2]
+ CRUSH rule 0 x 241 [5,2,7]
+ CRUSH rule 0 x 242 [5,2,6]
+ CRUSH rule 0 x 243 [5,8,2]
+ CRUSH rule 0 x 244 [5,6,2]
+ CRUSH rule 0 x 245 [7,2,5]
+ CRUSH rule 0 x 246 [2,5,7]
+ CRUSH rule 0 x 247 [6,2,5]
+ CRUSH rule 0 x 248 [8,2,5]
+ CRUSH rule 0 x 249 [2,5,8]
+ CRUSH rule 0 x 250 [2,5,8]
+ CRUSH rule 0 x 251 [2,5,7]
+ CRUSH rule 0 x 252 [5,7,2]
+ CRUSH rule 0 x 253 [5,2,7]
+ CRUSH rule 0 x 254 [5,2,8]
+ CRUSH rule 0 x 255 [2,7,5]
+ CRUSH rule 0 x 256 [5,6,2]
+ CRUSH rule 0 x 257 [2,6,5]
+ CRUSH rule 0 x 258 [5,2,8]
+ CRUSH rule 0 x 259 [5,6,2]
+ CRUSH rule 0 x 260 [5,8,2]
+ CRUSH rule 0 x 261 [8,5,2]
+ CRUSH rule 0 x 262 [5,6,2]
+ CRUSH rule 0 x 263 [6,2,5]
+ CRUSH rule 0 x 264 [5,6,2]
+ CRUSH rule 0 x 265 [8,5,2]
+ CRUSH rule 0 x 266 [8,2,5]
+ CRUSH rule 0 x 267 [2,5,6]
+ CRUSH rule 0 x 268 [2,6,5]
+ CRUSH rule 0 x 269 [2,6,5]
+ CRUSH rule 0 x 270 [5,2,8]
+ CRUSH rule 0 x 271 [7,5,2]
+ CRUSH rule 0 x 272 [2,6,5]
+ CRUSH rule 0 x 273 [5,2,7]
+ CRUSH rule 0 x 274 [6,5,2]
+ CRUSH rule 0 x 275 [5,8,2]
+ CRUSH rule 0 x 276 [7,2,5]
+ CRUSH rule 0 x 277 [6,5,2]
+ CRUSH rule 0 x 278 [6,2,5]
+ CRUSH rule 0 x 279 [8,5,2]
+ CRUSH rule 0 x 280 [2,7,5]
+ CRUSH rule 0 x 281 [8,2,5]
+ CRUSH rule 0 x 282 [5,2,8]
+ CRUSH rule 0 x 283 [8,2,5]
+ CRUSH rule 0 x 284 [6,5,2]
+ CRUSH rule 0 x 285 [5,7,2]
+ CRUSH rule 0 x 286 [2,8,5]
+ CRUSH rule 0 x 287 [2,5,8]
+ CRUSH rule 0 x 288 [8,2,5]
+ CRUSH rule 0 x 289 [5,6,2]
+ CRUSH rule 0 x 290 [2,5,6]
+ CRUSH rule 0 x 291 [2,5,8]
+ CRUSH rule 0 x 292 [8,2,5]
+ CRUSH rule 0 x 293 [6,2,5]
+ CRUSH rule 0 x 294 [7,5,2]
+ CRUSH rule 0 x 295 [5,6,2]
+ CRUSH rule 0 x 296 [5,2,6]
+ CRUSH rule 0 x 297 [6,2,5]
+ CRUSH rule 0 x 298 [2,5,6]
+ CRUSH rule 0 x 299 [2,8,5]
+ CRUSH rule 0 x 300 [8,5,2]
+ CRUSH rule 0 x 301 [2,7,5]
+ CRUSH rule 0 x 302 [5,2,6]
+ CRUSH rule 0 x 303 [7,5,2]
+ CRUSH rule 0 x 304 [2,6,5]
+ CRUSH rule 0 x 305 [5,6,2]
+ CRUSH rule 0 x 306 [2,8,5]
+ CRUSH rule 0 x 307 [2,7,5]
+ CRUSH rule 0 x 308 [2,8,5]
+ CRUSH rule 0 x 309 [7,5,2]
+ CRUSH rule 0 x 310 [5,2,6]
+ CRUSH rule 0 x 311 [5,8,2]
+ CRUSH rule 0 x 312 [2,6,5]
+ CRUSH rule 0 x 313 [5,2,7]
+ CRUSH rule 0 x 314 [5,2,6]
+ CRUSH rule 0 x 315 [2,5,7]
+ CRUSH rule 0 x 316 [6,5,2]
+ CRUSH rule 0 x 317 [2,7,5]
+ CRUSH rule 0 x 318 [8,2,5]
+ CRUSH rule 0 x 319 [5,2,6]
+ CRUSH rule 0 x 320 [5,8,2]
+ CRUSH rule 0 x 321 [2,5,8]
+ CRUSH rule 0 x 322 [2,6,5]
+ CRUSH rule 0 x 323 [5,7,2]
+ CRUSH rule 0 x 324 [7,2,5]
+ CRUSH rule 0 x 325 [5,6,2]
+ CRUSH rule 0 x 326 [5,2,7]
+ CRUSH rule 0 x 327 [2,8,5]
+ CRUSH rule 0 x 328 [7,5,2]
+ CRUSH rule 0 x 329 [5,7,2]
+ CRUSH rule 0 x 330 [5,7,2]
+ CRUSH rule 0 x 331 [2,7,5]
+ CRUSH rule 0 x 332 [2,5,8]
+ CRUSH rule 0 x 333 [6,5,2]
+ CRUSH rule 0 x 334 [8,5,2]
+ CRUSH rule 0 x 335 [7,2,5]
+ CRUSH rule 0 x 336 [5,6,2]
+ CRUSH rule 0 x 337 [7,2,5]
+ CRUSH rule 0 x 338 [5,8,2]
+ CRUSH rule 0 x 339 [7,5,2]
+ CRUSH rule 0 x 340 [2,6,5]
+ CRUSH rule 0 x 341 [5,2,7]
+ CRUSH rule 0 x 342 [2,8,5]
+ CRUSH rule 0 x 343 [6,5,2]
+ CRUSH rule 0 x 344 [6,2,5]
+ CRUSH rule 0 x 345 [5,7,2]
+ CRUSH rule 0 x 346 [8,2,5]
+ CRUSH rule 0 x 347 [5,2,8]
+ CRUSH rule 0 x 348 [8,2,5]
+ CRUSH rule 0 x 349 [2,7,5]
+ CRUSH rule 0 x 350 [8,5,2]
+ CRUSH rule 0 x 351 [5,8,2]
+ CRUSH rule 0 x 352 [2,8,5]
+ CRUSH rule 0 x 353 [6,5,2]
+ CRUSH rule 0 x 354 [2,5,6]
+ CRUSH rule 0 x 355 [5,8,2]
+ CRUSH rule 0 x 356 [5,2,8]
+ CRUSH rule 0 x 357 [6,2,5]
+ CRUSH rule 0 x 358 [2,8,5]
+ CRUSH rule 0 x 359 [6,2,5]
+ CRUSH rule 0 x 360 [5,2,8]
+ CRUSH rule 0 x 361 [8,5,2]
+ CRUSH rule 0 x 362 [5,2,8]
+ CRUSH rule 0 x 363 [5,2,8]
+ CRUSH rule 0 x 364 [2,5,7]
+ CRUSH rule 0 x 365 [6,5,2]
+ CRUSH rule 0 x 366 [7,2,5]
+ CRUSH rule 0 x 367 [5,2,7]
+ CRUSH rule 0 x 368 [7,5,2]
+ CRUSH rule 0 x 369 [5,7,2]
+ CRUSH rule 0 x 370 [8,2,5]
+ CRUSH rule 0 x 371 [2,5,8]
+ CRUSH rule 0 x 372 [5,2,8]
+ CRUSH rule 0 x 373 [2,6,5]
+ CRUSH rule 0 x 374 [5,8,2]
+ CRUSH rule 0 x 375 [6,5,2]
+ CRUSH rule 0 x 376 [7,2,5]
+ CRUSH rule 0 x 377 [2,5,7]
+ CRUSH rule 0 x 378 [2,6,5]
+ CRUSH rule 0 x 379 [8,5,2]
+ CRUSH rule 0 x 380 [2,5,8]
+ CRUSH rule 0 x 381 [2,5,7]
+ CRUSH rule 0 x 382 [2,5,7]
+ CRUSH rule 0 x 383 [5,7,2]
+ CRUSH rule 0 x 384 [7,2,5]
+ CRUSH rule 0 x 385 [7,5,2]
+ CRUSH rule 0 x 386 [2,5,6]
+ CRUSH rule 0 x 387 [2,5,8]
+ CRUSH rule 0 x 388 [5,2,7]
+ CRUSH rule 0 x 389 [2,5,8]
+ CRUSH rule 0 x 390 [5,8,2]
+ CRUSH rule 0 x 391 [5,6,2]
+ CRUSH rule 0 x 392 [2,7,5]
+ CRUSH rule 0 x 393 [5,2,6]
+ CRUSH rule 0 x 394 [5,8,2]
+ CRUSH rule 0 x 395 [5,2,8]
+ CRUSH rule 0 x 396 [5,2,6]
+ CRUSH rule 0 x 397 [2,5,7]
+ CRUSH rule 0 x 398 [2,5,6]
+ CRUSH rule 0 x 399 [8,5,2]
+ CRUSH rule 0 x 400 [8,2,5]
+ CRUSH rule 0 x 401 [2,5,6]
+ CRUSH rule 0 x 402 [7,5,2]
+ CRUSH rule 0 x 403 [2,5,7]
+ CRUSH rule 0 x 404 [5,2,6]
+ CRUSH rule 0 x 405 [6,5,2]
+ CRUSH rule 0 x 406 [2,6,5]
+ CRUSH rule 0 x 407 [2,7,5]
+ CRUSH rule 0 x 408 [5,2,7]
+ CRUSH rule 0 x 409 [7,5,2]
+ CRUSH rule 0 x 410 [8,5,2]
+ CRUSH rule 0 x 411 [2,7,5]
+ CRUSH rule 0 x 412 [2,5,7]
+ CRUSH rule 0 x 413 [5,2,8]
+ CRUSH rule 0 x 414 [5,2,8]
+ CRUSH rule 0 x 415 [2,6,5]
+ CRUSH rule 0 x 416 [2,8,5]
+ CRUSH rule 0 x 417 [8,2,5]
+ CRUSH rule 0 x 418 [7,2,5]
+ CRUSH rule 0 x 419 [8,5,2]
+ CRUSH rule 0 x 420 [2,5,7]
+ CRUSH rule 0 x 421 [8,5,2]
+ CRUSH rule 0 x 422 [6,5,2]
+ CRUSH rule 0 x 423 [2,5,7]
+ CRUSH rule 0 x 424 [8,5,2]
+ CRUSH rule 0 x 425 [2,5,8]
+ CRUSH rule 0 x 426 [6,2,5]
+ CRUSH rule 0 x 427 [2,8,5]
+ CRUSH rule 0 x 428 [5,8,2]
+ CRUSH rule 0 x 429 [5,8,2]
+ CRUSH rule 0 x 430 [5,7,2]
+ CRUSH rule 0 x 431 [5,2,7]
+ CRUSH rule 0 x 432 [7,2,5]
+ CRUSH rule 0 x 433 [6,5,2]
+ CRUSH rule 0 x 434 [5,2,7]
+ CRUSH rule 0 x 435 [2,5,6]
+ CRUSH rule 0 x 436 [5,2,7]
+ CRUSH rule 0 x 437 [7,5,2]
+ CRUSH rule 0 x 438 [2,5,8]
+ CRUSH rule 0 x 439 [2,5,8]
+ CRUSH rule 0 x 440 [2,6,5]
+ CRUSH rule 0 x 441 [5,8,2]
+ CRUSH rule 0 x 442 [2,5,6]
+ CRUSH rule 0 x 443 [6,2,5]
+ CRUSH rule 0 x 444 [7,2,5]
+ CRUSH rule 0 x 445 [6,5,2]
+ CRUSH rule 0 x 446 [5,2,8]
+ CRUSH rule 0 x 447 [2,5,6]
+ CRUSH rule 0 x 448 [7,2,5]
+ CRUSH rule 0 x 449 [7,5,2]
+ CRUSH rule 0 x 450 [5,2,6]
+ CRUSH rule 0 x 451 [6,5,2]
+ CRUSH rule 0 x 452 [8,5,2]
+ CRUSH rule 0 x 453 [6,5,2]
+ CRUSH rule 0 x 454 [6,5,2]
+ CRUSH rule 0 x 455 [2,8,5]
+ CRUSH rule 0 x 456 [6,2,5]
+ CRUSH rule 0 x 457 [7,2,5]
+ CRUSH rule 0 x 458 [2,8,5]
+ CRUSH rule 0 x 459 [2,6,5]
+ CRUSH rule 0 x 460 [6,5,2]
+ CRUSH rule 0 x 461 [6,5,2]
+ CRUSH rule 0 x 462 [8,2,5]
+ CRUSH rule 0 x 463 [6,2,5]
+ CRUSH rule 0 x 464 [7,5,2]
+ CRUSH rule 0 x 465 [7,2,5]
+ CRUSH rule 0 x 466 [5,6,2]
+ CRUSH rule 0 x 467 [6,5,2]
+ CRUSH rule 0 x 468 [7,2,5]
+ CRUSH rule 0 x 469 [7,2,5]
+ CRUSH rule 0 x 470 [5,2,8]
+ CRUSH rule 0 x 471 [2,6,5]
+ CRUSH rule 0 x 472 [5,2,8]
+ CRUSH rule 0 x 473 [2,5,7]
+ CRUSH rule 0 x 474 [6,2,5]
+ CRUSH rule 0 x 475 [6,2,5]
+ CRUSH rule 0 x 476 [5,7,2]
+ CRUSH rule 0 x 477 [5,6,2]
+ CRUSH rule 0 x 478 [6,2,5]
+ CRUSH rule 0 x 479 [2,5,6]
+ CRUSH rule 0 x 480 [2,6,5]
+ CRUSH rule 0 x 481 [2,5,7]
+ CRUSH rule 0 x 482 [5,8,2]
+ CRUSH rule 0 x 483 [2,6,5]
+ CRUSH rule 0 x 484 [2,8,5]
+ CRUSH rule 0 x 485 [5,8,2]
+ CRUSH rule 0 x 486 [5,2,8]
+ CRUSH rule 0 x 487 [5,2,8]
+ CRUSH rule 0 x 488 [5,7,2]
+ CRUSH rule 0 x 489 [2,8,5]
+ CRUSH rule 0 x 490 [6,5,2]
+ CRUSH rule 0 x 491 [2,6,5]
+ CRUSH rule 0 x 492 [6,5,2]
+ CRUSH rule 0 x 493 [2,8,5]
+ CRUSH rule 0 x 494 [2,6,5]
+ CRUSH rule 0 x 495 [5,2,6]
+ CRUSH rule 0 x 496 [7,5,2]
+ CRUSH rule 0 x 497 [5,7,2]
+ CRUSH rule 0 x 498 [2,5,6]
+ CRUSH rule 0 x 499 [8,5,2]
+ CRUSH rule 0 x 500 [5,6,2]
+ CRUSH rule 0 x 501 [2,7,5]
+ CRUSH rule 0 x 502 [7,2,5]
+ CRUSH rule 0 x 503 [2,5,7]
+ CRUSH rule 0 x 504 [5,8,2]
+ CRUSH rule 0 x 505 [2,7,5]
+ CRUSH rule 0 x 506 [5,2,7]
+ CRUSH rule 0 x 507 [6,2,5]
+ CRUSH rule 0 x 508 [2,5,7]
+ CRUSH rule 0 x 509 [7,5,2]
+ CRUSH rule 0 x 510 [6,2,5]
+ CRUSH rule 0 x 511 [5,6,2]
+ CRUSH rule 0 x 512 [7,2,5]
+ CRUSH rule 0 x 513 [7,2,5]
+ CRUSH rule 0 x 514 [5,7,2]
+ CRUSH rule 0 x 515 [8,5,2]
+ CRUSH rule 0 x 516 [5,2,7]
+ CRUSH rule 0 x 517 [7,2,5]
+ CRUSH rule 0 x 518 [5,6,2]
+ CRUSH rule 0 x 519 [7,5,2]
+ CRUSH rule 0 x 520 [2,8,5]
+ CRUSH rule 0 x 521 [8,2,5]
+ CRUSH rule 0 x 522 [6,2,5]
+ CRUSH rule 0 x 523 [5,2,7]
+ CRUSH rule 0 x 524 [2,5,7]
+ CRUSH rule 0 x 525 [2,5,8]
+ CRUSH rule 0 x 526 [2,5,8]
+ CRUSH rule 0 x 527 [2,5,6]
+ CRUSH rule 0 x 528 [5,2,7]
+ CRUSH rule 0 x 529 [5,6,2]
+ CRUSH rule 0 x 530 [6,5,2]
+ CRUSH rule 0 x 531 [6,2,5]
+ CRUSH rule 0 x 532 [6,5,2]
+ CRUSH rule 0 x 533 [5,8,2]
+ CRUSH rule 0 x 534 [7,5,2]
+ CRUSH rule 0 x 535 [8,2,5]
+ CRUSH rule 0 x 536 [6,2,5]
+ CRUSH rule 0 x 537 [5,8,2]
+ CRUSH rule 0 x 538 [6,5,2]
+ CRUSH rule 0 x 539 [8,5,2]
+ CRUSH rule 0 x 540 [2,7,5]
+ CRUSH rule 0 x 541 [2,5,7]
+ CRUSH rule 0 x 542 [5,2,8]
+ CRUSH rule 0 x 543 [6,2,5]
+ CRUSH rule 0 x 544 [5,7,2]
+ CRUSH rule 0 x 545 [5,7,2]
+ CRUSH rule 0 x 546 [6,2,5]
+ CRUSH rule 0 x 547 [8,2,5]
+ CRUSH rule 0 x 548 [5,2,8]
+ CRUSH rule 0 x 549 [5,7,2]
+ CRUSH rule 0 x 550 [2,5,6]
+ CRUSH rule 0 x 551 [7,5,2]
+ CRUSH rule 0 x 552 [5,7,2]
+ CRUSH rule 0 x 553 [5,2,8]
+ CRUSH rule 0 x 554 [2,6,5]
+ CRUSH rule 0 x 555 [5,2,8]
+ CRUSH rule 0 x 556 [5,6,2]
+ CRUSH rule 0 x 557 [7,5,2]
+ CRUSH rule 0 x 558 [5,2,6]
+ CRUSH rule 0 x 559 [5,2,8]
+ CRUSH rule 0 x 560 [8,5,2]
+ CRUSH rule 0 x 561 [6,5,2]
+ CRUSH rule 0 x 562 [5,2,8]
+ CRUSH rule 0 x 563 [2,7,5]
+ CRUSH rule 0 x 564 [5,2,7]
+ CRUSH rule 0 x 565 [5,8,2]
+ CRUSH rule 0 x 566 [5,6,2]
+ CRUSH rule 0 x 567 [5,7,2]
+ CRUSH rule 0 x 568 [7,5,2]
+ CRUSH rule 0 x 569 [5,2,6]
+ CRUSH rule 0 x 570 [2,5,7]
+ CRUSH rule 0 x 571 [5,6,2]
+ CRUSH rule 0 x 572 [5,2,7]
+ CRUSH rule 0 x 573 [5,2,7]
+ CRUSH rule 0 x 574 [2,5,6]
+ CRUSH rule 0 x 575 [8,2,5]
+ CRUSH rule 0 x 576 [5,7,2]
+ CRUSH rule 0 x 577 [8,2,5]
+ CRUSH rule 0 x 578 [6,2,5]
+ CRUSH rule 0 x 579 [5,2,6]
+ CRUSH rule 0 x 580 [5,2,8]
+ CRUSH rule 0 x 581 [7,2,5]
+ CRUSH rule 0 x 582 [2,8,5]
+ CRUSH rule 0 x 583 [6,2,5]
+ CRUSH rule 0 x 584 [8,2,5]
+ CRUSH rule 0 x 585 [7,2,5]
+ CRUSH rule 0 x 586 [2,8,5]
+ CRUSH rule 0 x 587 [2,5,6]
+ CRUSH rule 0 x 588 [5,8,2]
+ CRUSH rule 0 x 589 [7,2,5]
+ CRUSH rule 0 x 590 [6,2,5]
+ CRUSH rule 0 x 591 [5,2,7]
+ CRUSH rule 0 x 592 [2,5,6]
+ CRUSH rule 0 x 593 [2,8,5]
+ CRUSH rule 0 x 594 [2,6,5]
+ CRUSH rule 0 x 595 [7,2,5]
+ CRUSH rule 0 x 596 [5,2,6]
+ CRUSH rule 0 x 597 [5,2,7]
+ CRUSH rule 0 x 598 [5,2,7]
+ CRUSH rule 0 x 599 [5,2,6]
+ CRUSH rule 0 x 600 [7,2,5]
+ CRUSH rule 0 x 601 [2,6,5]
+ CRUSH rule 0 x 602 [5,8,2]
+ CRUSH rule 0 x 603 [5,2,8]
+ CRUSH rule 0 x 604 [7,5,2]
+ CRUSH rule 0 x 605 [5,2,6]
+ CRUSH rule 0 x 606 [2,7,5]
+ CRUSH rule 0 x 607 [2,5,8]
+ CRUSH rule 0 x 608 [5,2,8]
+ CRUSH rule 0 x 609 [5,2,6]
+ CRUSH rule 0 x 610 [5,8,2]
+ CRUSH rule 0 x 611 [2,8,5]
+ CRUSH rule 0 x 612 [2,8,5]
+ CRUSH rule 0 x 613 [7,2,5]
+ CRUSH rule 0 x 614 [7,2,5]
+ CRUSH rule 0 x 615 [6,2,5]
+ CRUSH rule 0 x 616 [2,7,5]
+ CRUSH rule 0 x 617 [6,2,5]
+ CRUSH rule 0 x 618 [7,5,2]
+ CRUSH rule 0 x 619 [5,2,6]
+ CRUSH rule 0 x 620 [5,2,8]
+ CRUSH rule 0 x 621 [5,6,2]
+ CRUSH rule 0 x 622 [2,5,8]
+ CRUSH rule 0 x 623 [2,8,5]
+ CRUSH rule 0 x 624 [5,2,8]
+ CRUSH rule 0 x 625 [2,5,8]
+ CRUSH rule 0 x 626 [7,2,5]
+ CRUSH rule 0 x 627 [2,6,5]
+ CRUSH rule 0 x 628 [8,2,5]
+ CRUSH rule 0 x 629 [2,6,5]
+ CRUSH rule 0 x 630 [2,6,5]
+ CRUSH rule 0 x 631 [2,6,5]
+ CRUSH rule 0 x 632 [7,2,5]
+ CRUSH rule 0 x 633 [8,5,2]
+ CRUSH rule 0 x 634 [2,5,6]
+ CRUSH rule 0 x 635 [5,6,2]
+ CRUSH rule 0 x 636 [2,5,7]
+ CRUSH rule 0 x 637 [5,2,8]
+ CRUSH rule 0 x 638 [6,2,5]
+ CRUSH rule 0 x 639 [5,2,6]
+ CRUSH rule 0 x 640 [5,2,7]
+ CRUSH rule 0 x 641 [7,2,5]
+ CRUSH rule 0 x 642 [2,8,5]
+ CRUSH rule 0 x 643 [5,2,8]
+ CRUSH rule 0 x 644 [8,2,5]
+ CRUSH rule 0 x 645 [5,7,2]
+ CRUSH rule 0 x 646 [8,2,5]
+ CRUSH rule 0 x 647 [7,2,5]
+ CRUSH rule 0 x 648 [2,8,5]
+ CRUSH rule 0 x 649 [5,7,2]
+ CRUSH rule 0 x 650 [7,5,2]
+ CRUSH rule 0 x 651 [5,6,2]
+ CRUSH rule 0 x 652 [5,6,2]
+ CRUSH rule 0 x 653 [8,5,2]
+ CRUSH rule 0 x 654 [7,5,2]
+ CRUSH rule 0 x 655 [2,5,6]
+ CRUSH rule 0 x 656 [5,7,2]
+ CRUSH rule 0 x 657 [6,2,5]
+ CRUSH rule 0 x 658 [5,8,2]
+ CRUSH rule 0 x 659 [5,7,2]
+ CRUSH rule 0 x 660 [7,5,2]
+ CRUSH rule 0 x 661 [2,7,5]
+ CRUSH rule 0 x 662 [5,2,8]
+ CRUSH rule 0 x 663 [2,5,7]
+ CRUSH rule 0 x 664 [2,5,6]
+ CRUSH rule 0 x 665 [5,6,2]
+ CRUSH rule 0 x 666 [2,7,5]
+ CRUSH rule 0 x 667 [2,5,8]
+ CRUSH rule 0 x 668 [5,7,2]
+ CRUSH rule 0 x 669 [6,5,2]
+ CRUSH rule 0 x 670 [5,2,6]
+ CRUSH rule 0 x 671 [2,8,5]
+ CRUSH rule 0 x 672 [5,2,8]
+ CRUSH rule 0 x 673 [5,2,7]
+ CRUSH rule 0 x 674 [5,2,7]
+ CRUSH rule 0 x 675 [2,8,5]
+ CRUSH rule 0 x 676 [2,5,7]
+ CRUSH rule 0 x 677 [5,2,6]
+ CRUSH rule 0 x 678 [2,5,8]
+ CRUSH rule 0 x 679 [6,2,5]
+ CRUSH rule 0 x 680 [2,5,8]
+ CRUSH rule 0 x 681 [5,6,2]
+ CRUSH rule 0 x 682 [2,5,8]
+ CRUSH rule 0 x 683 [2,5,8]
+ CRUSH rule 0 x 684 [7,2,5]
+ CRUSH rule 0 x 685 [7,2,5]
+ CRUSH rule 0 x 686 [2,5,8]
+ CRUSH rule 0 x 687 [5,7,2]
+ CRUSH rule 0 x 688 [5,6,2]
+ CRUSH rule 0 x 689 [6,5,2]
+ CRUSH rule 0 x 690 [8,2,5]
+ CRUSH rule 0 x 691 [5,2,6]
+ CRUSH rule 0 x 692 [7,2,5]
+ CRUSH rule 0 x 693 [6,5,2]
+ CRUSH rule 0 x 694 [6,5,2]
+ CRUSH rule 0 x 695 [2,6,5]
+ CRUSH rule 0 x 696 [2,5,8]
+ CRUSH rule 0 x 697 [6,2,5]
+ CRUSH rule 0 x 698 [6,2,5]
+ CRUSH rule 0 x 699 [2,8,5]
+ CRUSH rule 0 x 700 [2,5,6]
+ CRUSH rule 0 x 701 [5,2,6]
+ CRUSH rule 0 x 702 [5,2,8]
+ CRUSH rule 0 x 703 [8,5,2]
+ CRUSH rule 0 x 704 [2,5,6]
+ CRUSH rule 0 x 705 [8,2,5]
+ CRUSH rule 0 x 706 [2,5,8]
+ CRUSH rule 0 x 707 [7,5,2]
+ CRUSH rule 0 x 708 [5,7,2]
+ CRUSH rule 0 x 709 [6,5,2]
+ CRUSH rule 0 x 710 [8,5,2]
+ CRUSH rule 0 x 711 [2,5,7]
+ CRUSH rule 0 x 712 [2,5,6]
+ CRUSH rule 0 x 713 [6,5,2]
+ CRUSH rule 0 x 714 [5,2,6]
+ CRUSH rule 0 x 715 [2,5,6]
+ CRUSH rule 0 x 716 [5,6,2]
+ CRUSH rule 0 x 717 [8,2,5]
+ CRUSH rule 0 x 718 [5,7,2]
+ CRUSH rule 0 x 719 [2,6,5]
+ CRUSH rule 0 x 720 [6,2,5]
+ CRUSH rule 0 x 721 [5,7,2]
+ CRUSH rule 0 x 722 [5,7,2]
+ CRUSH rule 0 x 723 [5,2,7]
+ CRUSH rule 0 x 724 [2,7,5]
+ CRUSH rule 0 x 725 [2,5,6]
+ CRUSH rule 0 x 726 [5,7,2]
+ CRUSH rule 0 x 727 [5,7,2]
+ CRUSH rule 0 x 728 [2,6,5]
+ CRUSH rule 0 x 729 [5,6,2]
+ CRUSH rule 0 x 730 [5,8,2]
+ CRUSH rule 0 x 731 [5,2,6]
+ CRUSH rule 0 x 732 [2,5,8]
+ CRUSH rule 0 x 733 [5,6,2]
+ CRUSH rule 0 x 734 [6,5,2]
+ CRUSH rule 0 x 735 [5,6,2]
+ CRUSH rule 0 x 736 [5,8,2]
+ CRUSH rule 0 x 737 [2,8,5]
+ CRUSH rule 0 x 738 [5,2,6]
+ CRUSH rule 0 x 739 [2,7,5]
+ CRUSH rule 0 x 740 [2,7,5]
+ CRUSH rule 0 x 741 [7,2,5]
+ CRUSH rule 0 x 742 [8,2,5]
+ CRUSH rule 0 x 743 [7,2,5]
+ CRUSH rule 0 x 744 [5,6,2]
+ CRUSH rule 0 x 745 [5,2,8]
+ CRUSH rule 0 x 746 [5,2,7]
+ CRUSH rule 0 x 747 [6,2,5]
+ CRUSH rule 0 x 748 [2,6,5]
+ CRUSH rule 0 x 749 [5,7,2]
+ CRUSH rule 0 x 750 [2,7,5]
+ CRUSH rule 0 x 751 [2,7,5]
+ CRUSH rule 0 x 752 [8,2,5]
+ CRUSH rule 0 x 753 [7,5,2]
+ CRUSH rule 0 x 754 [8,5,2]
+ CRUSH rule 0 x 755 [2,6,5]
+ CRUSH rule 0 x 756 [5,8,2]
+ CRUSH rule 0 x 757 [8,2,5]
+ CRUSH rule 0 x 758 [6,2,5]
+ CRUSH rule 0 x 759 [8,5,2]
+ CRUSH rule 0 x 760 [2,5,7]
+ CRUSH rule 0 x 761 [5,2,6]
+ CRUSH rule 0 x 762 [2,8,5]
+ CRUSH rule 0 x 763 [8,5,2]
+ CRUSH rule 0 x 764 [2,7,5]
+ CRUSH rule 0 x 765 [6,5,2]
+ CRUSH rule 0 x 766 [8,5,2]
+ CRUSH rule 0 x 767 [2,8,5]
+ CRUSH rule 0 x 768 [8,5,2]
+ CRUSH rule 0 x 769 [6,2,5]
+ CRUSH rule 0 x 770 [6,2,5]
+ CRUSH rule 0 x 771 [7,2,5]
+ CRUSH rule 0 x 772 [8,5,2]
+ CRUSH rule 0 x 773 [5,2,6]
+ CRUSH rule 0 x 774 [5,7,2]
+ CRUSH rule 0 x 775 [6,5,2]
+ CRUSH rule 0 x 776 [7,2,5]
+ CRUSH rule 0 x 777 [5,2,6]
+ CRUSH rule 0 x 778 [2,6,5]
+ CRUSH rule 0 x 779 [2,6,5]
+ CRUSH rule 0 x 780 [2,5,8]
+ CRUSH rule 0 x 781 [6,5,2]
+ CRUSH rule 0 x 782 [5,2,8]
+ CRUSH rule 0 x 783 [7,2,5]
+ CRUSH rule 0 x 784 [2,5,8]
+ CRUSH rule 0 x 785 [6,2,5]
+ CRUSH rule 0 x 786 [7,5,2]
+ CRUSH rule 0 x 787 [2,8,5]
+ CRUSH rule 0 x 788 [6,2,5]
+ CRUSH rule 0 x 789 [2,5,8]
+ CRUSH rule 0 x 790 [8,5,2]
+ CRUSH rule 0 x 791 [5,6,2]
+ CRUSH rule 0 x 792 [5,6,2]
+ CRUSH rule 0 x 793 [6,2,5]
+ CRUSH rule 0 x 794 [2,8,5]
+ CRUSH rule 0 x 795 [2,5,6]
+ CRUSH rule 0 x 796 [5,7,2]
+ CRUSH rule 0 x 797 [2,5,6]
+ CRUSH rule 0 x 798 [6,2,5]
+ CRUSH rule 0 x 799 [5,2,7]
+ CRUSH rule 0 x 800 [5,2,8]
+ CRUSH rule 0 x 801 [5,7,2]
+ CRUSH rule 0 x 802 [2,6,5]
+ CRUSH rule 0 x 803 [2,5,7]
+ CRUSH rule 0 x 804 [6,2,5]
+ CRUSH rule 0 x 805 [5,8,2]
+ CRUSH rule 0 x 806 [2,5,6]
+ CRUSH rule 0 x 807 [5,7,2]
+ CRUSH rule 0 x 808 [5,7,2]
+ CRUSH rule 0 x 809 [2,5,6]
+ CRUSH rule 0 x 810 [5,7,2]
+ CRUSH rule 0 x 811 [8,5,2]
+ CRUSH rule 0 x 812 [8,5,2]
+ CRUSH rule 0 x 813 [6,5,2]
+ CRUSH rule 0 x 814 [5,8,2]
+ CRUSH rule 0 x 815 [5,2,6]
+ CRUSH rule 0 x 816 [2,6,5]
+ CRUSH rule 0 x 817 [5,6,2]
+ CRUSH rule 0 x 818 [5,2,7]
+ CRUSH rule 0 x 819 [5,2,7]
+ CRUSH rule 0 x 820 [5,7,2]
+ CRUSH rule 0 x 821 [5,8,2]
+ CRUSH rule 0 x 822 [2,5,7]
+ CRUSH rule 0 x 823 [5,7,2]
+ CRUSH rule 0 x 824 [5,7,2]
+ CRUSH rule 0 x 825 [2,8,5]
+ CRUSH rule 0 x 826 [7,2,5]
+ CRUSH rule 0 x 827 [2,6,5]
+ CRUSH rule 0 x 828 [2,5,8]
+ CRUSH rule 0 x 829 [5,6,2]
+ CRUSH rule 0 x 830 [2,5,7]
+ CRUSH rule 0 x 831 [2,6,5]
+ CRUSH rule 0 x 832 [5,8,2]
+ CRUSH rule 0 x 833 [2,6,5]
+ CRUSH rule 0 x 834 [5,2,8]
+ CRUSH rule 0 x 835 [8,5,2]
+ CRUSH rule 0 x 836 [5,8,2]
+ CRUSH rule 0 x 837 [6,5,2]
+ CRUSH rule 0 x 838 [6,2,5]
+ CRUSH rule 0 x 839 [5,2,8]
+ CRUSH rule 0 x 840 [7,5,2]
+ CRUSH rule 0 x 841 [5,8,2]
+ CRUSH rule 0 x 842 [2,5,8]
+ CRUSH rule 0 x 843 [6,5,2]
+ CRUSH rule 0 x 844 [5,8,2]
+ CRUSH rule 0 x 845 [5,6,2]
+ CRUSH rule 0 x 846 [5,2,6]
+ CRUSH rule 0 x 847 [2,8,5]
+ CRUSH rule 0 x 848 [2,6,5]
+ CRUSH rule 0 x 849 [5,8,2]
+ CRUSH rule 0 x 850 [2,5,7]
+ CRUSH rule 0 x 851 [6,5,2]
+ CRUSH rule 0 x 852 [7,5,2]
+ CRUSH rule 0 x 853 [6,2,5]
+ CRUSH rule 0 x 854 [7,2,5]
+ CRUSH rule 0 x 855 [5,7,2]
+ CRUSH rule 0 x 856 [6,5,2]
+ CRUSH rule 0 x 857 [8,5,2]
+ CRUSH rule 0 x 858 [6,5,2]
+ CRUSH rule 0 x 859 [6,2,5]
+ CRUSH rule 0 x 860 [5,2,8]
+ CRUSH rule 0 x 861 [8,5,2]
+ CRUSH rule 0 x 862 [6,2,5]
+ CRUSH rule 0 x 863 [8,2,5]
+ CRUSH rule 0 x 864 [5,6,2]
+ CRUSH rule 0 x 865 [8,2,5]
+ CRUSH rule 0 x 866 [5,6,2]
+ CRUSH rule 0 x 867 [6,5,2]
+ CRUSH rule 0 x 868 [6,5,2]
+ CRUSH rule 0 x 869 [8,5,2]
+ CRUSH rule 0 x 870 [2,5,7]
+ CRUSH rule 0 x 871 [5,2,6]
+ CRUSH rule 0 x 872 [5,2,6]
+ CRUSH rule 0 x 873 [5,6,2]
+ CRUSH rule 0 x 874 [2,6,5]
+ CRUSH rule 0 x 875 [2,6,5]
+ CRUSH rule 0 x 876 [5,8,2]
+ CRUSH rule 0 x 877 [6,5,2]
+ CRUSH rule 0 x 878 [5,2,7]
+ CRUSH rule 0 x 879 [7,5,2]
+ CRUSH rule 0 x 880 [5,2,7]
+ CRUSH rule 0 x 881 [5,8,2]
+ CRUSH rule 0 x 882 [5,2,6]
+ CRUSH rule 0 x 883 [2,5,6]
+ CRUSH rule 0 x 884 [6,2,5]
+ CRUSH rule 0 x 885 [5,2,6]
+ CRUSH rule 0 x 886 [5,7,2]
+ CRUSH rule 0 x 887 [7,5,2]
+ CRUSH rule 0 x 888 [6,2,5]
+ CRUSH rule 0 x 889 [2,8,5]
+ CRUSH rule 0 x 890 [7,2,5]
+ CRUSH rule 0 x 891 [2,7,5]
+ CRUSH rule 0 x 892 [6,2,5]
+ CRUSH rule 0 x 893 [2,5,6]
+ CRUSH rule 0 x 894 [7,5,2]
+ CRUSH rule 0 x 895 [5,2,6]
+ CRUSH rule 0 x 896 [2,6,5]
+ CRUSH rule 0 x 897 [5,2,8]
+ CRUSH rule 0 x 898 [2,5,8]
+ CRUSH rule 0 x 899 [2,6,5]
+ CRUSH rule 0 x 900 [5,2,7]
+ CRUSH rule 0 x 901 [5,2,6]
+ CRUSH rule 0 x 902 [8,5,2]
+ CRUSH rule 0 x 903 [5,6,2]
+ CRUSH rule 0 x 904 [5,7,2]
+ CRUSH rule 0 x 905 [6,2,5]
+ CRUSH rule 0 x 906 [2,7,5]
+ CRUSH rule 0 x 907 [7,2,5]
+ CRUSH rule 0 x 908 [5,6,2]
+ CRUSH rule 0 x 909 [2,5,8]
+ CRUSH rule 0 x 910 [6,5,2]
+ CRUSH rule 0 x 911 [5,7,2]
+ CRUSH rule 0 x 912 [2,8,5]
+ CRUSH rule 0 x 913 [7,2,5]
+ CRUSH rule 0 x 914 [6,5,2]
+ CRUSH rule 0 x 915 [8,2,5]
+ CRUSH rule 0 x 916 [5,2,6]
+ CRUSH rule 0 x 917 [2,5,6]
+ CRUSH rule 0 x 918 [8,2,5]
+ CRUSH rule 0 x 919 [6,2,5]
+ CRUSH rule 0 x 920 [7,5,2]
+ CRUSH rule 0 x 921 [2,5,8]
+ CRUSH rule 0 x 922 [6,5,2]
+ CRUSH rule 0 x 923 [5,6,2]
+ CRUSH rule 0 x 924 [5,2,8]
+ CRUSH rule 0 x 925 [5,6,2]
+ CRUSH rule 0 x 926 [5,2,7]
+ CRUSH rule 0 x 927 [2,6,5]
+ CRUSH rule 0 x 928 [8,2,5]
+ CRUSH rule 0 x 929 [5,2,6]
+ CRUSH rule 0 x 930 [2,5,8]
+ CRUSH rule 0 x 931 [5,2,8]
+ CRUSH rule 0 x 932 [5,2,7]
+ CRUSH rule 0 x 933 [8,5,2]
+ CRUSH rule 0 x 934 [5,8,2]
+ CRUSH rule 0 x 935 [6,5,2]
+ CRUSH rule 0 x 936 [2,7,5]
+ CRUSH rule 0 x 937 [5,8,2]
+ CRUSH rule 0 x 938 [6,5,2]
+ CRUSH rule 0 x 939 [2,8,5]
+ CRUSH rule 0 x 940 [8,5,2]
+ CRUSH rule 0 x 941 [5,2,6]
+ CRUSH rule 0 x 942 [2,6,5]
+ CRUSH rule 0 x 943 [8,2,5]
+ CRUSH rule 0 x 944 [5,8,2]
+ CRUSH rule 0 x 945 [7,2,5]
+ CRUSH rule 0 x 946 [2,8,5]
+ CRUSH rule 0 x 947 [5,2,6]
+ CRUSH rule 0 x 948 [7,5,2]
+ CRUSH rule 0 x 949 [6,2,5]
+ CRUSH rule 0 x 950 [5,7,2]
+ CRUSH rule 0 x 951 [5,6,2]
+ CRUSH rule 0 x 952 [2,7,5]
+ CRUSH rule 0 x 953 [2,5,6]
+ CRUSH rule 0 x 954 [5,2,8]
+ CRUSH rule 0 x 955 [8,2,5]
+ CRUSH rule 0 x 956 [2,7,5]
+ CRUSH rule 0 x 957 [7,2,5]
+ CRUSH rule 0 x 958 [8,5,2]
+ CRUSH rule 0 x 959 [5,2,6]
+ CRUSH rule 0 x 960 [5,6,2]
+ CRUSH rule 0 x 961 [5,2,6]
+ CRUSH rule 0 x 962 [7,5,2]
+ CRUSH rule 0 x 963 [2,5,8]
+ CRUSH rule 0 x 964 [5,2,7]
+ CRUSH rule 0 x 965 [7,5,2]
+ CRUSH rule 0 x 966 [5,6,2]
+ CRUSH rule 0 x 967 [8,5,2]
+ CRUSH rule 0 x 968 [7,2,5]
+ CRUSH rule 0 x 969 [8,2,5]
+ CRUSH rule 0 x 970 [2,8,5]
+ CRUSH rule 0 x 971 [2,8,5]
+ CRUSH rule 0 x 972 [2,7,5]
+ CRUSH rule 0 x 973 [2,8,5]
+ CRUSH rule 0 x 974 [5,2,7]
+ CRUSH rule 0 x 975 [5,8,2]
+ CRUSH rule 0 x 976 [5,7,2]
+ CRUSH rule 0 x 977 [8,5,2]
+ CRUSH rule 0 x 978 [7,2,5]
+ CRUSH rule 0 x 979 [7,2,5]
+ CRUSH rule 0 x 980 [6,2,5]
+ CRUSH rule 0 x 981 [7,5,2]
+ CRUSH rule 0 x 982 [5,2,8]
+ CRUSH rule 0 x 983 [5,6,2]
+ CRUSH rule 0 x 984 [2,8,5]
+ CRUSH rule 0 x 985 [2,5,8]
+ CRUSH rule 0 x 986 [8,5,2]
+ CRUSH rule 0 x 987 [2,5,8]
+ CRUSH rule 0 x 988 [2,5,6]
+ CRUSH rule 0 x 989 [2,8,5]
+ CRUSH rule 0 x 990 [2,6,5]
+ CRUSH rule 0 x 991 [2,5,8]
+ CRUSH rule 0 x 992 [7,2,5]
+ CRUSH rule 0 x 993 [2,6,5]
+ CRUSH rule 0 x 994 [5,6,2]
+ CRUSH rule 0 x 995 [7,2,5]
+ CRUSH rule 0 x 996 [6,5,2]
+ CRUSH rule 0 x 997 [6,5,2]
+ CRUSH rule 0 x 998 [8,2,5]
+ CRUSH rule 0 x 999 [2,7,5]
+ CRUSH rule 0 x 1000 [8,5,2]
+ CRUSH rule 0 x 1001 [2,5,6]
+ CRUSH rule 0 x 1002 [2,5,8]
+ CRUSH rule 0 x 1003 [2,7,5]
+ CRUSH rule 0 x 1004 [6,2,5]
+ CRUSH rule 0 x 1005 [6,2,5]
+ CRUSH rule 0 x 1006 [2,8,5]
+ CRUSH rule 0 x 1007 [2,5,8]
+ CRUSH rule 0 x 1008 [2,7,5]
+ CRUSH rule 0 x 1009 [6,5,2]
+ CRUSH rule 0 x 1010 [5,2,6]
+ CRUSH rule 0 x 1011 [5,2,8]
+ CRUSH rule 0 x 1012 [5,2,8]
+ CRUSH rule 0 x 1013 [5,2,8]
+ CRUSH rule 0 x 1014 [2,8,5]
+ CRUSH rule 0 x 1015 [6,5,2]
+ CRUSH rule 0 x 1016 [2,5,6]
+ CRUSH rule 0 x 1017 [6,2,5]
+ CRUSH rule 0 x 1018 [5,2,6]
+ CRUSH rule 0 x 1019 [5,8,2]
+ CRUSH rule 0 x 1020 [5,2,7]
+ CRUSH rule 0 x 1021 [5,2,7]
+ CRUSH rule 0 x 1022 [2,7,5]
+ CRUSH rule 0 x 1023 [5,2,7]
+ rule 0 (choose) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 1 (choose-two), x = 0..1023, numrep = 2..3
+ CRUSH rule 1 x 0 [2,8]
+ CRUSH rule 1 x 1 [2,8]
+ CRUSH rule 1 x 2 [2,5]
+ CRUSH rule 1 x 3 [8,2]
+ CRUSH rule 1 x 4 [5,7]
+ CRUSH rule 1 x 5 [7,8]
+ CRUSH rule 1 x 6 [2,6]
+ CRUSH rule 1 x 7 [5,7]
+ CRUSH rule 1 x 8 [5,2]
+ CRUSH rule 1 x 9 [2,5]
+ CRUSH rule 1 x 10 [2,8]
+ CRUSH rule 1 x 11 [2,7]
+ CRUSH rule 1 x 12 [2,7]
+ CRUSH rule 1 x 13 [5,8]
+ CRUSH rule 1 x 14 [7,6]
+ CRUSH rule 1 x 15 [7,2]
+ CRUSH rule 1 x 16 [5,6]
+ CRUSH rule 1 x 17 [5,7]
+ CRUSH rule 1 x 18 [2,5]
+ CRUSH rule 1 x 19 [7,5]
+ CRUSH rule 1 x 20 [2,5]
+ CRUSH rule 1 x 21 [5,7]
+ CRUSH rule 1 x 22 [8,5]
+ CRUSH rule 1 x 23 [5,6]
+ CRUSH rule 1 x 24 [2,8]
+ CRUSH rule 1 x 25 [5,7]
+ CRUSH rule 1 x 26 [2,8]
+ CRUSH rule 1 x 27 [5,2]
+ CRUSH rule 1 x 28 [6,2]
+ CRUSH rule 1 x 29 [8,5]
+ CRUSH rule 1 x 30 [5,7]
+ CRUSH rule 1 x 31 [8,7]
+ CRUSH rule 1 x 32 [5,6]
+ CRUSH rule 1 x 33 [2,7]
+ CRUSH rule 1 x 34 [2,5]
+ CRUSH rule 1 x 35 [2,8]
+ CRUSH rule 1 x 36 [5,8]
+ CRUSH rule 1 x 37 [2,5]
+ CRUSH rule 1 x 38 [5,8]
+ CRUSH rule 1 x 39 [5,7]
+ CRUSH rule 1 x 40 [7,8]
+ CRUSH rule 1 x 41 [2,6]
+ CRUSH rule 1 x 42 [5,2]
+ CRUSH rule 1 x 43 [2,5]
+ CRUSH rule 1 x 44 [2,6]
+ CRUSH rule 1 x 45 [8,2]
+ CRUSH rule 1 x 46 [2,5]
+ CRUSH rule 1 x 47 [5,2]
+ CRUSH rule 1 x 48 [5,6]
+ CRUSH rule 1 x 49 [5,2]
+ CRUSH rule 1 x 50 [5,6]
+ CRUSH rule 1 x 51 [5,6]
+ CRUSH rule 1 x 52 [8,6]
+ CRUSH rule 1 x 53 [5,2]
+ CRUSH rule 1 x 54 [7,6]
+ CRUSH rule 1 x 55 [8,7]
+ CRUSH rule 1 x 56 [6,5]
+ CRUSH rule 1 x 57 [5,2]
+ CRUSH rule 1 x 58 [2,5]
+ CRUSH rule 1 x 59 [5,2]
+ CRUSH rule 1 x 60 [5,7]
+ CRUSH rule 1 x 61 [5,6]
+ CRUSH rule 1 x 62 [7,2]
+ CRUSH rule 1 x 63 [5,6]
+ CRUSH rule 1 x 64 [5,2]
+ CRUSH rule 1 x 65 [7,5]
+ CRUSH rule 1 x 66 [5,2]
+ CRUSH rule 1 x 67 [5,2]
+ CRUSH rule 1 x 68 [2,5]
+ CRUSH rule 1 x 69 [5,2]
+ CRUSH rule 1 x 70 [7,2]
+ CRUSH rule 1 x 71 [2,8]
+ CRUSH rule 1 x 72 [6,2]
+ CRUSH rule 1 x 73 [2,7]
+ CRUSH rule 1 x 74 [2,7]
+ CRUSH rule 1 x 75 [5,2]
+ CRUSH rule 1 x 76 [5,2]
+ CRUSH rule 1 x 77 [7,2]
+ CRUSH rule 1 x 78 [2,5]
+ CRUSH rule 1 x 79 [5,2]
+ CRUSH rule 1 x 80 [2,5]
+ CRUSH rule 1 x 81 [2]
+ CRUSH rule 1 x 82 [7,2]
+ CRUSH rule 1 x 83 [2,6]
+ CRUSH rule 1 x 84 [7,2]
+ CRUSH rule 1 x 85 [5,8]
+ CRUSH rule 1 x 86 [2,7]
+ CRUSH rule 1 x 87 [2,7]
+ CRUSH rule 1 x 88 [2,6]
+ CRUSH rule 1 x 89 [5,2]
+ CRUSH rule 1 x 90 [6,7]
+ CRUSH rule 1 x 91 [5,8]
+ CRUSH rule 1 x 92 [2,8]
+ CRUSH rule 1 x 93 [7,5]
+ CRUSH rule 1 x 94 [2,5]
+ CRUSH rule 1 x 95 [7,5]
+ CRUSH rule 1 x 96 [5,6]
+ CRUSH rule 1 x 97 [8,7]
+ CRUSH rule 1 x 98 [2,6]
+ CRUSH rule 1 x 99 [2,7]
+ CRUSH rule 1 x 100 [2,7]
+ CRUSH rule 1 x 101 [5,7]
+ CRUSH rule 1 x 102 [5,2]
+ CRUSH rule 1 x 103 [5,7]
+ CRUSH rule 1 x 104 [7,5]
+ CRUSH rule 1 x 105 [2,5]
+ CRUSH rule 1 x 106 [2,6]
+ CRUSH rule 1 x 107 [5,2]
+ CRUSH rule 1 x 108 [7,2]
+ CRUSH rule 1 x 109 [2,5]
+ CRUSH rule 1 x 110 [5,2]
+ CRUSH rule 1 x 111 [2,5]
+ CRUSH rule 1 x 112 [2,5]
+ CRUSH rule 1 x 113 [6,2]
+ CRUSH rule 1 x 114 [7,6]
+ CRUSH rule 1 x 115 [8,2]
+ CRUSH rule 1 x 116 [2,6]
+ CRUSH rule 1 x 117 [7,5]
+ CRUSH rule 1 x 118 [2,5]
+ CRUSH rule 1 x 119 [5,6]
+ CRUSH rule 1 x 120 [2,5]
+ CRUSH rule 1 x 121 [2,5]
+ CRUSH rule 1 x 122 [8,5]
+ CRUSH rule 1 x 123 [2,5]
+ CRUSH rule 1 x 124 [5,8]
+ CRUSH rule 1 x 125 [2,7]
+ CRUSH rule 1 x 126 [5,2]
+ CRUSH rule 1 x 127 [5,6]
+ CRUSH rule 1 x 128 [5,8]
+ CRUSH rule 1 x 129 [2,7]
+ CRUSH rule 1 x 130 [5,8]
+ CRUSH rule 1 x 131 [2,8]
+ CRUSH rule 1 x 132 [2,7]
+ CRUSH rule 1 x 133 [5,6]
+ CRUSH rule 1 x 134 [2,8]
+ CRUSH rule 1 x 135 [5,6]
+ CRUSH rule 1 x 136 [2,5]
+ CRUSH rule 1 x 137 [7,5]
+ CRUSH rule 1 x 138 [8,7]
+ CRUSH rule 1 x 139 [5,2]
+ CRUSH rule 1 x 140 [2,6]
+ CRUSH rule 1 x 141 [6,8]
+ CRUSH rule 1 x 142 [5,6]
+ CRUSH rule 1 x 143 [5,8]
+ CRUSH rule 1 x 144 [8,2]
+ CRUSH rule 1 x 145 [8,5]
+ CRUSH rule 1 x 146 [2,6]
+ CRUSH rule 1 x 147 [2,8]
+ CRUSH rule 1 x 148 [5,2]
+ CRUSH rule 1 x 149 [5,8]
+ CRUSH rule 1 x 150 [2,6]
+ CRUSH rule 1 x 151 [5,2]
+ CRUSH rule 1 x 152 [8,5]
+ CRUSH rule 1 x 153 [8,6]
+ CRUSH rule 1 x 154 [5,2]
+ CRUSH rule 1 x 155 [5,7]
+ CRUSH rule 1 x 156 [5,8]
+ CRUSH rule 1 x 157 [5,2]
+ CRUSH rule 1 x 158 [2,8]
+ CRUSH rule 1 x 159 [7,2]
+ CRUSH rule 1 x 160 [2,8]
+ CRUSH rule 1 x 161 [2,5]
+ CRUSH rule 1 x 162 [2,6]
+ CRUSH rule 1 x 163 [5,6]
+ CRUSH rule 1 x 164 [7,8]
+ CRUSH rule 1 x 165 [7,2]
+ CRUSH rule 1 x 166 [2,5]
+ CRUSH rule 1 x 167 [2,6]
+ CRUSH rule 1 x 168 [5,2]
+ CRUSH rule 1 x 169 [2,6]
+ CRUSH rule 1 x 170 [2,8]
+ CRUSH rule 1 x 171 [7,5]
+ CRUSH rule 1 x 172 [2,7]
+ CRUSH rule 1 x 173 [8,5]
+ CRUSH rule 1 x 174 [2,5]
+ CRUSH rule 1 x 175 [6,2]
+ CRUSH rule 1 x 176 [5,2]
+ CRUSH rule 1 x 177 [5,8]
+ CRUSH rule 1 x 178 [5,2]
+ CRUSH rule 1 x 179 [5,2]
+ CRUSH rule 1 x 180 [5,8]
+ CRUSH rule 1 x 181 [6,2]
+ CRUSH rule 1 x 182 [8,5]
+ CRUSH rule 1 x 183 [7,8]
+ CRUSH rule 1 x 184 [5,7]
+ CRUSH rule 1 x 185 [6,8]
+ CRUSH rule 1 x 186 [2,8]
+ CRUSH rule 1 x 187 [2,6]
+ CRUSH rule 1 x 188 [2,8]
+ CRUSH rule 1 x 189 [2,7]
+ CRUSH rule 1 x 190 [5,2]
+ CRUSH rule 1 x 191 [7,6]
+ CRUSH rule 1 x 192 [5,2]
+ CRUSH rule 1 x 193 [5,2]
+ CRUSH rule 1 x 194 [2,5]
+ CRUSH rule 1 x 195 [6,5]
+ CRUSH rule 1 x 196 [6,7]
+ CRUSH rule 1 x 197 [6,5]
+ CRUSH rule 1 x 198 [2,5]
+ CRUSH rule 1 x 199 [2,5]
+ CRUSH rule 1 x 200 [2,5]
+ CRUSH rule 1 x 201 [7,2]
+ CRUSH rule 1 x 202 [6,5]
+ CRUSH rule 1 x 203 [5,8]
+ CRUSH rule 1 x 204 [2,5]
+ CRUSH rule 1 x 205 [2,7]
+ CRUSH rule 1 x 206 [2,5]
+ CRUSH rule 1 x 207 [5,2]
+ CRUSH rule 1 x 208 [7,2]
+ CRUSH rule 1 x 209 [2,5]
+ CRUSH rule 1 x 210 [2,8]
+ CRUSH rule 1 x 211 [5,6]
+ CRUSH rule 1 x 212 [7,5]
+ CRUSH rule 1 x 213 [8,5]
+ CRUSH rule 1 x 214 [5,7]
+ CRUSH rule 1 x 215 [8,2]
+ CRUSH rule 1 x 216 [5,2]
+ CRUSH rule 1 x 217 [2,5]
+ CRUSH rule 1 x 218 [2,7]
+ CRUSH rule 1 x 219 [5,8]
+ CRUSH rule 1 x 220 [5,7]
+ CRUSH rule 1 x 221 [5,6]
+ CRUSH rule 1 x 222 [6,8]
+ CRUSH rule 1 x 223 [2,5]
+ CRUSH rule 1 x 224 [2,5]
+ CRUSH rule 1 x 225 [8,6]
+ CRUSH rule 1 x 226 [7,2]
+ CRUSH rule 1 x 227 [5,6]
+ CRUSH rule 1 x 228 [5,6]
+ CRUSH rule 1 x 229 [5,2]
+ CRUSH rule 1 x 230 [5,7]
+ CRUSH rule 1 x 231 [5,6]
+ CRUSH rule 1 x 232 [2,7]
+ CRUSH rule 1 x 233 [5,8]
+ CRUSH rule 1 x 234 [2,7]
+ CRUSH rule 1 x 235 [5,8]
+ CRUSH rule 1 x 236 [5,2]
+ CRUSH rule 1 x 237 [5,7]
+ CRUSH rule 1 x 238 [5,6]
+ CRUSH rule 1 x 239 [8,7]
+ CRUSH rule 1 x 240 [5,7]
+ CRUSH rule 1 x 241 [5,2]
+ CRUSH rule 1 x 242 [5,8]
+ CRUSH rule 1 x 243 [5,7]
+ CRUSH rule 1 x 244 [5,6]
+ CRUSH rule 1 x 245 [7,6]
+ CRUSH rule 1 x 246 [2,5]
+ CRUSH rule 1 x 247 [6,2]
+ CRUSH rule 1 x 248 [8,2]
+ CRUSH rule 1 x 249 [2,5]
+ CRUSH rule 1 x 250 [2,7]
+ CRUSH rule 1 x 251 [2,5]
+ CRUSH rule 1 x 252 [5,7]
+ CRUSH rule 1 x 253 [5,2]
+ CRUSH rule 1 x 254 [5,6]
+ CRUSH rule 1 x 255 [2,7]
+ CRUSH rule 1 x 256 [5,7]
+ CRUSH rule 1 x 257 [2,8]
+ CRUSH rule 1 x 258 [5]
+ CRUSH rule 1 x 259 [5,7]
+ CRUSH rule 1 x 260 [5,6]
+ CRUSH rule 1 x 261 [8,7]
+ CRUSH rule 1 x 262 [5,2]
+ CRUSH rule 1 x 263 [6,8]
+ CRUSH rule 1 x 264 [5,6]
+ CRUSH rule 1 x 265 [8,6]
+ CRUSH rule 1 x 266 [8,2]
+ CRUSH rule 1 x 267 [2,5]
+ CRUSH rule 1 x 268 [2,7]
+ CRUSH rule 1 x 269 [2,8]
+ CRUSH rule 1 x 270 [5,2]
+ CRUSH rule 1 x 271 [7,5]
+ CRUSH rule 1 x 272 [2,8]
+ CRUSH rule 1 x 273 [5,2]
+ CRUSH rule 1 x 274 [6,8]
+ CRUSH rule 1 x 275 [5,8]
+ CRUSH rule 1 x 276 [7,2]
+ CRUSH rule 1 x 277 [6,5]
+ CRUSH rule 1 x 278 [6,8]
+ CRUSH rule 1 x 279 [8,5]
+ CRUSH rule 1 x 280 [2,6]
+ CRUSH rule 1 x 281 [8,2]
+ CRUSH rule 1 x 282 [5,2]
+ CRUSH rule 1 x 283 [8,2]
+ CRUSH rule 1 x 284 [6,5]
+ CRUSH rule 1 x 285 [5,6]
+ CRUSH rule 1 x 286 [2,5]
+ CRUSH rule 1 x 287 [2,5]
+ CRUSH rule 1 x 288 [8,2]
+ CRUSH rule 1 x 289 [5,6]
+ CRUSH rule 1 x 290 [2,5]
+ CRUSH rule 1 x 291 [2,5]
+ CRUSH rule 1 x 292 [8,2]
+ CRUSH rule 1 x 293 [6,2]
+ CRUSH rule 1 x 294 [7,5]
+ CRUSH rule 1 x 295 [5,8]
+ CRUSH rule 1 x 296 [5,2]
+ CRUSH rule 1 x 297 [6,2]
+ CRUSH rule 1 x 298 [2,8]
+ CRUSH rule 1 x 299 [2,5]
+ CRUSH rule 1 x 300 [8,7]
+ CRUSH rule 1 x 301 [2,8]
+ CRUSH rule 1 x 302 [5,2]
+ CRUSH rule 1 x 303 [7,5]
+ CRUSH rule 1 x 304 [2,7]
+ CRUSH rule 1 x 305 [5,8]
+ CRUSH rule 1 x 306 [2,7]
+ CRUSH rule 1 x 307 [2,7]
+ CRUSH rule 1 x 308 [2,8]
+ CRUSH rule 1 x 309 [7,5]
+ CRUSH rule 1 x 310 [5,6]
+ CRUSH rule 1 x 311 [5,2]
+ CRUSH rule 1 x 312 [2,6]
+ CRUSH rule 1 x 313 [5,8]
+ CRUSH rule 1 x 314 [5,2]
+ CRUSH rule 1 x 315 [2,5]
+ CRUSH rule 1 x 316 [6,5]
+ CRUSH rule 1 x 317 [2,6]
+ CRUSH rule 1 x 318 [8,6]
+ CRUSH rule 1 x 319 [5,2]
+ CRUSH rule 1 x 320 [5,7]
+ CRUSH rule 1 x 321 [2,5]
+ CRUSH rule 1 x 322 [2,7]
+ CRUSH rule 1 x 323 [5,7]
+ CRUSH rule 1 x 324 [7,2]
+ CRUSH rule 1 x 325 [5,6]
+ CRUSH rule 1 x 326 [5,6]
+ CRUSH rule 1 x 327 [2,6]
+ CRUSH rule 1 x 328 [7,5]
+ CRUSH rule 1 x 329 [5,6]
+ CRUSH rule 1 x 330 [5,7]
+ CRUSH rule 1 x 331 [2,6]
+ CRUSH rule 1 x 332 [2,5]
+ CRUSH rule 1 x 333 [6,8]
+ CRUSH rule 1 x 334 [8,5]
+ CRUSH rule 1 x 335 [7,2]
+ CRUSH rule 1 x 336 [5,6]
+ CRUSH rule 1 x 337 [7,2]
+ CRUSH rule 1 x 338 [5,6]
+ CRUSH rule 1 x 339 [7,5]
+ CRUSH rule 1 x 340 [2,5]
+ CRUSH rule 1 x 341 [5,2]
+ CRUSH rule 1 x 342 [2,7]
+ CRUSH rule 1 x 343 [6,7]
+ CRUSH rule 1 x 344 [6,2]
+ CRUSH rule 1 x 345 [5,2]
+ CRUSH rule 1 x 346 [8,2]
+ CRUSH rule 1 x 347 [5,2]
+ CRUSH rule 1 x 348 [8,2]
+ CRUSH rule 1 x 349 [2,6]
+ CRUSH rule 1 x 350 [8,5]
+ CRUSH rule 1 x 351 [5,6]
+ CRUSH rule 1 x 352 [2,7]
+ CRUSH rule 1 x 353 [6,5]
+ CRUSH rule 1 x 354 [2,5]
+ CRUSH rule 1 x 355 [5,2]
+ CRUSH rule 1 x 356 [5,8]
+ CRUSH rule 1 x 357 [6,2]
+ CRUSH rule 1 x 358 [2,5]
+ CRUSH rule 1 x 359 [6,7]
+ CRUSH rule 1 x 360 [5,2]
+ CRUSH rule 1 x 361 [8,5]
+ CRUSH rule 1 x 362 [5,7]
+ CRUSH rule 1 x 363 [5,2]
+ CRUSH rule 1 x 364 [2,5]
+ CRUSH rule 1 x 365 [6,7]
+ CRUSH rule 1 x 366 [7,2]
+ CRUSH rule 1 x 367 [5,7]
+ CRUSH rule 1 x 368 [7,5]
+ CRUSH rule 1 x 369 [5,7]
+ CRUSH rule 1 x 370 [8,7]
+ CRUSH rule 1 x 371 [2,5]
+ CRUSH rule 1 x 372 [5,2]
+ CRUSH rule 1 x 373 [2,6]
+ CRUSH rule 1 x 374 [5,8]
+ CRUSH rule 1 x 375 [6,5]
+ CRUSH rule 1 x 376 [7,2]
+ CRUSH rule 1 x 377 [2,5]
+ CRUSH rule 1 x 378 [2,6]
+ CRUSH rule 1 x 379 [8,5]
+ CRUSH rule 1 x 380 [2,5]
+ CRUSH rule 1 x 381 [2,5]
+ CRUSH rule 1 x 382 [2,5]
+ CRUSH rule 1 x 383 [5,2]
+ CRUSH rule 1 x 384 [7,2]
+ CRUSH rule 1 x 385 [7,5]
+ CRUSH rule 1 x 386 [2,5]
+ CRUSH rule 1 x 387 [2,5]
+ CRUSH rule 1 x 388 [5,2]
+ CRUSH rule 1 x 389 [2,5]
+ CRUSH rule 1 x 390 [5,6]
+ CRUSH rule 1 x 391 [5,6]
+ CRUSH rule 1 x 392 [2,8]
+ CRUSH rule 1 x 393 [5,2]
+ CRUSH rule 1 x 394 [5,7]
+ CRUSH rule 1 x 395 [5,2]
+ CRUSH rule 1 x 396 [5,2]
+ CRUSH rule 1 x 397 [2,7]
+ CRUSH rule 1 x 398 [2,5]
+ CRUSH rule 1 x 399 [8,7]
+ CRUSH rule 1 x 400 [8,2]
+ CRUSH rule 1 x 401 [2,5]
+ CRUSH rule 1 x 402 [7,8]
+ CRUSH rule 1 x 403 [2,7]
+ CRUSH rule 1 x 404 [5,2]
+ CRUSH rule 1 x 405 [6,5]
+ CRUSH rule 1 x 406 [2,6]
+ CRUSH rule 1 x 407 [2,8]
+ CRUSH rule 1 x 408 [5,2]
+ CRUSH rule 1 x 409 [7,5]
+ CRUSH rule 1 x 410 [8,6]
+ CRUSH rule 1 x 411 [2,6]
+ CRUSH rule 1 x 412 [2,5]
+ CRUSH rule 1 x 413 [5,2]
+ CRUSH rule 1 x 414 [5,2]
+ CRUSH rule 1 x 415 [2,6]
+ CRUSH rule 1 x 416 [2,5]
+ CRUSH rule 1 x 417 [8,7]
+ CRUSH rule 1 x 418 [7,6]
+ CRUSH rule 1 x 419 [8,5]
+ CRUSH rule 1 x 420 [2,5]
+ CRUSH rule 1 x 421 [8,6]
+ CRUSH rule 1 x 422 [6,7]
+ CRUSH rule 1 x 423 [2,5]
+ CRUSH rule 1 x 424 [8,5]
+ CRUSH rule 1 x 425 [2,5]
+ CRUSH rule 1 x 426 [6,7]
+ CRUSH rule 1 x 427 [2,7]
+ CRUSH rule 1 x 428 [5,6]
+ CRUSH rule 1 x 429 [5,6]
+ CRUSH rule 1 x 430 [5,6]
+ CRUSH rule 1 x 431 [5,2]
+ CRUSH rule 1 x 432 [7,2]
+ CRUSH rule 1 x 433 [6,5]
+ CRUSH rule 1 x 434 [5,2]
+ CRUSH rule 1 x 435 [2,5]
+ CRUSH rule 1 x 436 [5,2]
+ CRUSH rule 1 x 437 [7,5]
+ CRUSH rule 1 x 438 [2,5]
+ CRUSH rule 1 x 439 [2,5]
+ CRUSH rule 1 x 440 [2,7]
+ CRUSH rule 1 x 441 [5,7]
+ CRUSH rule 1 x 442 [2,5]
+ CRUSH rule 1 x 443 [6,8]
+ CRUSH rule 1 x 444 [7,2]
+ CRUSH rule 1 x 445 [6,5]
+ CRUSH rule 1 x 446 [5,7]
+ CRUSH rule 1 x 447 [2,5]
+ CRUSH rule 1 x 448 [7,2]
+ CRUSH rule 1 x 449 [7,8]
+ CRUSH rule 1 x 450 [5,8]
+ CRUSH rule 1 x 451 [6,8]
+ CRUSH rule 1 x 452 [8,5]
+ CRUSH rule 1 x 453 [6,8]
+ CRUSH rule 1 x 454 [6,7]
+ CRUSH rule 1 x 455 [2,7]
+ CRUSH rule 1 x 456 [6,8]
+ CRUSH rule 1 x 457 [7,2]
+ CRUSH rule 1 x 458 [2,8]
+ CRUSH rule 1 x 459 [2,5]
+ CRUSH rule 1 x 460 [6,5]
+ CRUSH rule 1 x 461 [6,5]
+ CRUSH rule 1 x 462 [8,2]
+ CRUSH rule 1 x 463 [6,7]
+ CRUSH rule 1 x 464 [7,5]
+ CRUSH rule 1 x 465 [7,6]
+ CRUSH rule 1 x 466 [5,8]
+ CRUSH rule 1 x 467 [6,5]
+ CRUSH rule 1 x 468 [7,8]
+ CRUSH rule 1 x 469 [7,2]
+ CRUSH rule 1 x 470 [5,2]
+ CRUSH rule 1 x 471 [2,6]
+ CRUSH rule 1 x 472 [5,6]
+ CRUSH rule 1 x 473 [2,5]
+ CRUSH rule 1 x 474 [6,2]
+ CRUSH rule 1 x 475 [6,7]
+ CRUSH rule 1 x 476 [5,2]
+ CRUSH rule 1 x 477 [5,8]
+ CRUSH rule 1 x 478 [6,7]
+ CRUSH rule 1 x 479 [2,5]
+ CRUSH rule 1 x 480 [2,8]
+ CRUSH rule 1 x 481 [2,5]
+ CRUSH rule 1 x 482 [5,7]
+ CRUSH rule 1 x 483 [2,7]
+ CRUSH rule 1 x 484 [2,7]
+ CRUSH rule 1 x 485 [5,7]
+ CRUSH rule 1 x 486 [5,2]
+ CRUSH rule 1 x 487 [5,2]
+ CRUSH rule 1 x 488 [5,7]
+ CRUSH rule 1 x 489 [2,8]
+ CRUSH rule 1 x 490 [6,5]
+ CRUSH rule 1 x 491 [2,6]
+ CRUSH rule 1 x 492 [6,5]
+ CRUSH rule 1 x 493 [2,8]
+ CRUSH rule 1 x 494 [2,7]
+ CRUSH rule 1 x 495 [5,6]
+ CRUSH rule 1 x 496 [7,5]
+ CRUSH rule 1 x 497 [5,7]
+ CRUSH rule 1 x 498 [2,5]
+ CRUSH rule 1 x 499 [8,5]
+ CRUSH rule 1 x 500 [5,6]
+ CRUSH rule 1 x 501 [2,7]
+ CRUSH rule 1 x 502 [7,2]
+ CRUSH rule 1 x 503 [2,5]
+ CRUSH rule 1 x 504 [5,6]
+ CRUSH rule 1 x 505 [2,7]
+ CRUSH rule 1 x 506 [5,2]
+ CRUSH rule 1 x 507 [6,2]
+ CRUSH rule 1 x 508 [2,7]
+ CRUSH rule 1 x 509 [7,5]
+ CRUSH rule 1 x 510 [6,2]
+ CRUSH rule 1 x 511 [5,8]
+ CRUSH rule 1 x 512 [7,6]
+ CRUSH rule 1 x 513 [7,2]
+ CRUSH rule 1 x 514 [5,7]
+ CRUSH rule 1 x 515 [8,5]
+ CRUSH rule 1 x 516 [5,2]
+ CRUSH rule 1 x 517 [7,8]
+ CRUSH rule 1 x 518 [5,6]
+ CRUSH rule 1 x 519 [7,5]
+ CRUSH rule 1 x 520 [2,6]
+ CRUSH rule 1 x 521 [8,7]
+ CRUSH rule 1 x 522 [6,8]
+ CRUSH rule 1 x 523 [5,2]
+ CRUSH rule 1 x 524 [2,5]
+ CRUSH rule 1 x 525 [2,5]
+ CRUSH rule 1 x 526 [2,5]
+ CRUSH rule 1 x 527 [2,5]
+ CRUSH rule 1 x 528 [5,2]
+ CRUSH rule 1 x 529 [5,7]
+ CRUSH rule 1 x 530 [6,7]
+ CRUSH rule 1 x 531 [6,2]
+ CRUSH rule 1 x 532 [6,5]
+ CRUSH rule 1 x 533 [5,6]
+ CRUSH rule 1 x 534 [7,5]
+ CRUSH rule 1 x 535 [8,6]
+ CRUSH rule 1 x 536 [6,7]
+ CRUSH rule 1 x 537 [5,7]
+ CRUSH rule 1 x 538 [6,8]
+ CRUSH rule 1 x 539 [8,5]
+ CRUSH rule 1 x 540 [2,6]
+ CRUSH rule 1 x 541 [2,5]
+ CRUSH rule 1 x 542 [5,2]
+ CRUSH rule 1 x 543 [6,2]
+ CRUSH rule 1 x 544 [5,7]
+ CRUSH rule 1 x 545 [5,7]
+ CRUSH rule 1 x 546 [6,2]
+ CRUSH rule 1 x 547 [8,2]
+ CRUSH rule 1 x 548 [5,2]
+ CRUSH rule 1 x 549 [5,8]
+ CRUSH rule 1 x 550 [2,5]
+ CRUSH rule 1 x 551 [7,5]
+ CRUSH rule 1 x 552 [5,2]
+ CRUSH rule 1 x 553 [5,2]
+ CRUSH rule 1 x 554 [2,8]
+ CRUSH rule 1 x 555 [5,2]
+ CRUSH rule 1 x 556 [5,6]
+ CRUSH rule 1 x 557 [7,5]
+ CRUSH rule 1 x 558 [5,2]
+ CRUSH rule 1 x 559 [5,2]
+ CRUSH rule 1 x 560 [8,5]
+ CRUSH rule 1 x 561 [6,5]
+ CRUSH rule 1 x 562 [5,7]
+ CRUSH rule 1 x 563 [2,6]
+ CRUSH rule 1 x 564 [5,2]
+ CRUSH rule 1 x 565 [5,6]
+ CRUSH rule 1 x 566 [5,7]
+ CRUSH rule 1 x 567 [5,6]
+ CRUSH rule 1 x 568 [7,5]
+ CRUSH rule 1 x 569 [5,2]
+ CRUSH rule 1 x 570 [2,5]
+ CRUSH rule 1 x 571 [5,7]
+ CRUSH rule 1 x 572 [5]
+ CRUSH rule 1 x 573 [5,2]
+ CRUSH rule 1 x 574 [2,8]
+ CRUSH rule 1 x 575 [8,6]
+ CRUSH rule 1 x 576 [5,6]
+ CRUSH rule 1 x 577 [8,2]
+ CRUSH rule 1 x 578 [6,8]
+ CRUSH rule 1 x 579 [5,2]
+ CRUSH rule 1 x 580 [5,2]
+ CRUSH rule 1 x 581 [7,2]
+ CRUSH rule 1 x 582 [2,8]
+ CRUSH rule 1 x 583 [6,2]
+ CRUSH rule 1 x 584 [8,2]
+ CRUSH rule 1 x 585 [7,2]
+ CRUSH rule 1 x 586 [2,6]
+ CRUSH rule 1 x 587 [2,5]
+ CRUSH rule 1 x 588 [5,2]
+ CRUSH rule 1 x 589 [7,2]
+ CRUSH rule 1 x 590 [6,2]
+ CRUSH rule 1 x 591 [5,2]
+ CRUSH rule 1 x 592 [2,5]
+ CRUSH rule 1 x 593 [2,8]
+ CRUSH rule 1 x 594 [2,7]
+ CRUSH rule 1 x 595 [7,2]
+ CRUSH rule 1 x 596 [5,7]
+ CRUSH rule 1 x 597 [5,2]
+ CRUSH rule 1 x 598 [5,2]
+ CRUSH rule 1 x 599 [5,2]
+ CRUSH rule 1 x 600 [7,2]
+ CRUSH rule 1 x 601 [2,7]
+ CRUSH rule 1 x 602 [5,7]
+ CRUSH rule 1 x 603 [5,2]
+ CRUSH rule 1 x 604 [7,5]
+ CRUSH rule 1 x 605 [5,2]
+ CRUSH rule 1 x 606 [2,6]
+ CRUSH rule 1 x 607 [2,5]
+ CRUSH rule 1 x 608 [5,2]
+ CRUSH rule 1 x 609 [5,2]
+ CRUSH rule 1 x 610 [5,7]
+ CRUSH rule 1 x 611 [2,5]
+ CRUSH rule 1 x 612 [2,8]
+ CRUSH rule 1 x 613 [7,2]
+ CRUSH rule 1 x 614 [7,8]
+ CRUSH rule 1 x 615 [6,8]
+ CRUSH rule 1 x 616 [2,8]
+ CRUSH rule 1 x 617 [6,2]
+ CRUSH rule 1 x 618 [7,6]
+ CRUSH rule 1 x 619 [5,2]
+ CRUSH rule 1 x 620 [5,2]
+ CRUSH rule 1 x 621 [5,8]
+ CRUSH rule 1 x 622 [2,5]
+ CRUSH rule 1 x 623 [2,6]
+ CRUSH rule 1 x 624 [5,7]
+ CRUSH rule 1 x 625 [2,5]
+ CRUSH rule 1 x 626 [7,8]
+ CRUSH rule 1 x 627 [2,7]
+ CRUSH rule 1 x 628 [8,2]
+ CRUSH rule 1 x 629 [2,6]
+ CRUSH rule 1 x 630 [2,7]
+ CRUSH rule 1 x 631 [2,7]
+ CRUSH rule 1 x 632 [7,2]
+ CRUSH rule 1 x 633 [8,6]
+ CRUSH rule 1 x 634 [2,5]
+ CRUSH rule 1 x 635 [5,6]
+ CRUSH rule 1 x 636 [2,5]
+ CRUSH rule 1 x 637 [5,2]
+ CRUSH rule 1 x 638 [6,8]
+ CRUSH rule 1 x 639 [5,2]
+ CRUSH rule 1 x 640 [5,2]
+ CRUSH rule 1 x 641 [7,2]
+ CRUSH rule 1 x 642 [2,8]
+ CRUSH rule 1 x 643 [5,7]
+ CRUSH rule 1 x 644 [8,2]
+ CRUSH rule 1 x 645 [5,2]
+ CRUSH rule 1 x 646 [8,2]
+ CRUSH rule 1 x 647 [7,2]
+ CRUSH rule 1 x 648 [2,6]
+ CRUSH rule 1 x 649 [5,7]
+ CRUSH rule 1 x 650 [7,8]
+ CRUSH rule 1 x 651 [5,7]
+ CRUSH rule 1 x 652 [5,6]
+ CRUSH rule 1 x 653 [8,5]
+ CRUSH rule 1 x 654 [7,5]
+ CRUSH rule 1 x 655 [2,5]
+ CRUSH rule 1 x 656 [5,2]
+ CRUSH rule 1 x 657 [6,2]
+ CRUSH rule 1 x 658 [5,2]
+ CRUSH rule 1 x 659 [5,6]
+ CRUSH rule 1 x 660 [7,8]
+ CRUSH rule 1 x 661 [2,8]
+ CRUSH rule 1 x 662 [5,2]
+ CRUSH rule 1 x 663 [2,5]
+ CRUSH rule 1 x 664 [2,5]
+ CRUSH rule 1 x 665 [5,7]
+ CRUSH rule 1 x 666 [2,8]
+ CRUSH rule 1 x 667 [2,5]
+ CRUSH rule 1 x 668 [5,7]
+ CRUSH rule 1 x 669 [6,5]
+ CRUSH rule 1 x 670 [5,2]
+ CRUSH rule 1 x 671 [2,5]
+ CRUSH rule 1 x 672 [5,6]
+ CRUSH rule 1 x 673 [5,2]
+ CRUSH rule 1 x 674 [5,2]
+ CRUSH rule 1 x 675 [2,8]
+ CRUSH rule 1 x 676 [2,5]
+ CRUSH rule 1 x 677 [5,2]
+ CRUSH rule 1 x 678 [2,5]
+ CRUSH rule 1 x 679 [6,2]
+ CRUSH rule 1 x 680 [2,5]
+ CRUSH rule 1 x 681 [5,7]
+ CRUSH rule 1 x 682 [2,5]
+ CRUSH rule 1 x 683 [2,8]
+ CRUSH rule 1 x 684 [7,2]
+ CRUSH rule 1 x 685 [7,2]
+ CRUSH rule 1 x 686 [2,5]
+ CRUSH rule 1 x 687 [5,7]
+ CRUSH rule 1 x 688 [5,7]
+ CRUSH rule 1 x 689 [6,5]
+ CRUSH rule 1 x 690 [8,2]
+ CRUSH rule 1 x 691 [5,2]
+ CRUSH rule 1 x 692 [7,2]
+ CRUSH rule 1 x 693 [6,7]
+ CRUSH rule 1 x 694 [6,5]
+ CRUSH rule 1 x 695 [2,8]
+ CRUSH rule 1 x 696 [2,5]
+ CRUSH rule 1 x 697 [6,2]
+ CRUSH rule 1 x 698 [6,2]
+ CRUSH rule 1 x 699 [2,6]
+ CRUSH rule 1 x 700 [2,5]
+ CRUSH rule 1 x 701 [5,2]
+ CRUSH rule 1 x 702 [5,6]
+ CRUSH rule 1 x 703 [8,5]
+ CRUSH rule 1 x 704 [2,5]
+ CRUSH rule 1 x 705 [8,6]
+ CRUSH rule 1 x 706 [2,7]
+ CRUSH rule 1 x 707 [7,8]
+ CRUSH rule 1 x 708 [5,7]
+ CRUSH rule 1 x 709 [6,5]
+ CRUSH rule 1 x 710 [8,5]
+ CRUSH rule 1 x 711 [2,5]
+ CRUSH rule 1 x 712 [2,5]
+ CRUSH rule 1 x 713 [6,7]
+ CRUSH rule 1 x 714 [5,2]
+ CRUSH rule 1 x 715 [2,5]
+ CRUSH rule 1 x 716 [5,6]
+ CRUSH rule 1 x 717 [8,7]
+ CRUSH rule 1 x 718 [5,7]
+ CRUSH rule 1 x 719 [2,6]
+ CRUSH rule 1 x 720 [6,8]
+ CRUSH rule 1 x 721 [5,6]
+ CRUSH rule 1 x 722 [5,2]
+ CRUSH rule 1 x 723 [5,2]
+ CRUSH rule 1 x 724 [2,6]
+ CRUSH rule 1 x 725 [2,8]
+ CRUSH rule 1 x 726 [5,8]
+ CRUSH rule 1 x 727 [5,6]
+ CRUSH rule 1 x 728 [2,6]
+ CRUSH rule 1 x 729 [5,6]
+ CRUSH rule 1 x 730 [5,7]
+ CRUSH rule 1 x 731 [5,2]
+ CRUSH rule 1 x 732 [2,5]
+ CRUSH rule 1 x 733 [5,2]
+ CRUSH rule 1 x 734 [6,5]
+ CRUSH rule 1 x 735 [5,8]
+ CRUSH rule 1 x 736 [5,8]
+ CRUSH rule 1 x 737 [2,5]
+ CRUSH rule 1 x 738 [5,2]
+ CRUSH rule 1 x 739 [2,8]
+ CRUSH rule 1 x 740 [2,6]
+ CRUSH rule 1 x 741 [7,8]
+ CRUSH rule 1 x 742 [8,2]
+ CRUSH rule 1 x 743 [7,2]
+ CRUSH rule 1 x 744 [5,7]
+ CRUSH rule 1 x 745 [5,6]
+ CRUSH rule 1 x 746 [5,2]
+ CRUSH rule 1 x 747 [6,2]
+ CRUSH rule 1 x 748 [2,7]
+ CRUSH rule 1 x 749 [5,8]
+ CRUSH rule 1 x 750 [2,6]
+ CRUSH rule 1 x 751 [2,8]
+ CRUSH rule 1 x 752 [8,2]
+ CRUSH rule 1 x 753 [7,8]
+ CRUSH rule 1 x 754 [8,6]
+ CRUSH rule 1 x 755 [2,5]
+ CRUSH rule 1 x 756 [5,6]
+ CRUSH rule 1 x 757 [8,6]
+ CRUSH rule 1 x 758 [6,2]
+ CRUSH rule 1 x 759 [8,5]
+ CRUSH rule 1 x 760 [2,5]
+ CRUSH rule 1 x 761 [5,2]
+ CRUSH rule 1 x 762 [2,7]
+ CRUSH rule 1 x 763 [8,6]
+ CRUSH rule 1 x 764 [2,7]
+ CRUSH rule 1 x 765 [6,5]
+ CRUSH rule 1 x 766 [8,5]
+ CRUSH rule 1 x 767 [2,6]
+ CRUSH rule 1 x 768 [8,5]
+ CRUSH rule 1 x 769 [6,2]
+ CRUSH rule 1 x 770 [6,2]
+ CRUSH rule 1 x 771 [7,2]
+ CRUSH rule 1 x 772 [8,5]
+ CRUSH rule 1 x 773 [5,2]
+ CRUSH rule 1 x 774 [5,6]
+ CRUSH rule 1 x 775 [6,8]
+ CRUSH rule 1 x 776 [7,2]
+ CRUSH rule 1 x 777 [5,2]
+ CRUSH rule 1 x 778 [2,8]
+ CRUSH rule 1 x 779 [2,7]
+ CRUSH rule 1 x 780 [2,5]
+ CRUSH rule 1 x 781 [6,5]
+ CRUSH rule 1 x 782 [5,2]
+ CRUSH rule 1 x 783 [7,2]
+ CRUSH rule 1 x 784 [2,7]
+ CRUSH rule 1 x 785 [6,2]
+ CRUSH rule 1 x 786 [7,6]
+ CRUSH rule 1 x 787 [2,7]
+ CRUSH rule 1 x 788 [6,2]
+ CRUSH rule 1 x 789 [2,5]
+ CRUSH rule 1 x 790 [8,5]
+ CRUSH rule 1 x 791 [5,8]
+ CRUSH rule 1 x 792 [5,8]
+ CRUSH rule 1 x 793 [6,2]
+ CRUSH rule 1 x 794 [2,6]
+ CRUSH rule 1 x 795 [2,5]
+ CRUSH rule 1 x 796 [5,6]
+ CRUSH rule 1 x 797 [2,5]
+ CRUSH rule 1 x 798 [6,8]
+ CRUSH rule 1 x 799 [5,2]
+ CRUSH rule 1 x 800 [5,2]
+ CRUSH rule 1 x 801 [5,6]
+ CRUSH rule 1 x 802 [2,8]
+ CRUSH rule 1 x 803 [2,5]
+ CRUSH rule 1 x 804 [6,2]
+ CRUSH rule 1 x 805 [5,6]
+ CRUSH rule 1 x 806 [2,5]
+ CRUSH rule 1 x 807 [5,7]
+ CRUSH rule 1 x 808 [5,6]
+ CRUSH rule 1 x 809 [2,5]
+ CRUSH rule 1 x 810 [5,7]
+ CRUSH rule 1 x 811 [8,5]
+ CRUSH rule 1 x 812 [8,5]
+ CRUSH rule 1 x 813 [6,5]
+ CRUSH rule 1 x 814 [5,6]
+ CRUSH rule 1 x 815 [5,2]
+ CRUSH rule 1 x 816 [2,5]
+ CRUSH rule 1 x 817 [5,7]
+ CRUSH rule 1 x 818 [5,2]
+ CRUSH rule 1 x 819 [5,2]
+ CRUSH rule 1 x 820 [5,2]
+ CRUSH rule 1 x 821 [5,2]
+ CRUSH rule 1 x 822 [2,7]
+ CRUSH rule 1 x 823 [5,8]
+ CRUSH rule 1 x 824 [5,7]
+ CRUSH rule 1 x 825 [2,8]
+ CRUSH rule 1 x 826 [7,2]
+ CRUSH rule 1 x 827 [2,8]
+ CRUSH rule 1 x 828 [2,5]
+ CRUSH rule 1 x 829 [5,6]
+ CRUSH rule 1 x 830 [2,5]
+ CRUSH rule 1 x 831 [2,6]
+ CRUSH rule 1 x 832 [5,7]
+ CRUSH rule 1 x 833 [2,7]
+ CRUSH rule 1 x 834 [5]
+ CRUSH rule 1 x 835 [8,5]
+ CRUSH rule 1 x 836 [5,2]
+ CRUSH rule 1 x 837 [6,5]
+ CRUSH rule 1 x 838 [6,7]
+ CRUSH rule 1 x 839 [5,2]
+ CRUSH rule 1 x 840 [7,8]
+ CRUSH rule 1 x 841 [5,8]
+ CRUSH rule 1 x 842 [2,5]
+ CRUSH rule 1 x 843 [6,5]
+ CRUSH rule 1 x 844 [5,8]
+ CRUSH rule 1 x 845 [5,8]
+ CRUSH rule 1 x 846 [5,2]
+ CRUSH rule 1 x 847 [2,8]
+ CRUSH rule 1 x 848 [2,6]
+ CRUSH rule 1 x 849 [5,8]
+ CRUSH rule 1 x 850 [2,5]
+ CRUSH rule 1 x 851 [6,8]
+ CRUSH rule 1 x 852 [7,5]
+ CRUSH rule 1 x 853 [6,8]
+ CRUSH rule 1 x 854 [7,6]
+ CRUSH rule 1 x 855 [5,7]
+ CRUSH rule 1 x 856 [6,7]
+ CRUSH rule 1 x 857 [8,5]
+ CRUSH rule 1 x 858 [6,5]
+ CRUSH rule 1 x 859 [6,2]
+ CRUSH rule 1 x 860 [5,2]
+ CRUSH rule 1 x 861 [8,7]
+ CRUSH rule 1 x 862 [6,2]
+ CRUSH rule 1 x 863 [8,7]
+ CRUSH rule 1 x 864 [5,6]
+ CRUSH rule 1 x 865 [8,2]
+ CRUSH rule 1 x 866 [5,8]
+ CRUSH rule 1 x 867 [6,5]
+ CRUSH rule 1 x 868 [6,5]
+ CRUSH rule 1 x 869 [8,7]
+ CRUSH rule 1 x 870 [2,5]
+ CRUSH rule 1 x 871 [5,7]
+ CRUSH rule 1 x 872 [5,2]
+ CRUSH rule 1 x 873 [5,6]
+ CRUSH rule 1 x 874 [2,6]
+ CRUSH rule 1 x 875 [2,6]
+ CRUSH rule 1 x 876 [5,8]
+ CRUSH rule 1 x 877 [6,5]
+ CRUSH rule 1 x 878 [5,2]
+ CRUSH rule 1 x 879 [7,5]
+ CRUSH rule 1 x 880 [5,2]
+ CRUSH rule 1 x 881 [5,6]
+ CRUSH rule 1 x 882 [5,2]
+ CRUSH rule 1 x 883 [2,7]
+ CRUSH rule 1 x 884 [6,2]
+ CRUSH rule 1 x 885 [5,2]
+ CRUSH rule 1 x 886 [5,6]
+ CRUSH rule 1 x 887 [7,5]
+ CRUSH rule 1 x 888 [6,8]
+ CRUSH rule 1 x 889 [2,5]
+ CRUSH rule 1 x 890 [7,2]
+ CRUSH rule 1 x 891 [2,8]
+ CRUSH rule 1 x 892 [6,2]
+ CRUSH rule 1 x 893 [2,5]
+ CRUSH rule 1 x 894 [7,5]
+ CRUSH rule 1 x 895 [5,7]
+ CRUSH rule 1 x 896 [2,8]
+ CRUSH rule 1 x 897 [5,2]
+ CRUSH rule 1 x 898 [2,5]
+ CRUSH rule 1 x 899 [2,7]
+ CRUSH rule 1 x 900 [5,2]
+ CRUSH rule 1 x 901 [5,2]
+ CRUSH rule 1 x 902 [8,5]
+ CRUSH rule 1 x 903 [5,7]
+ CRUSH rule 1 x 904 [5,6]
+ CRUSH rule 1 x 905 [6,2]
+ CRUSH rule 1 x 906 [2,5]
+ CRUSH rule 1 x 907 [7,2]
+ CRUSH rule 1 x 908 [5,8]
+ CRUSH rule 1 x 909 [2,5]
+ CRUSH rule 1 x 910 [6,5]
+ CRUSH rule 1 x 911 [5,8]
+ CRUSH rule 1 x 912 [2,6]
+ CRUSH rule 1 x 913 [7,6]
+ CRUSH rule 1 x 914 [6,5]
+ CRUSH rule 1 x 915 [8,2]
+ CRUSH rule 1 x 916 [5,2]
+ CRUSH rule 1 x 917 [2,5]
+ CRUSH rule 1 x 918 [8,2]
+ CRUSH rule 1 x 919 [6,2]
+ CRUSH rule 1 x 920 [7,6]
+ CRUSH rule 1 x 921 [2,5]
+ CRUSH rule 1 x 922 [6,7]
+ CRUSH rule 1 x 923 [5,8]
+ CRUSH rule 1 x 924 [5,6]
+ CRUSH rule 1 x 925 [5,7]
+ CRUSH rule 1 x 926 [5,8]
+ CRUSH rule 1 x 927 [2,6]
+ CRUSH rule 1 x 928 [8,2]
+ CRUSH rule 1 x 929 [5,2]
+ CRUSH rule 1 x 930 [2,5]
+ CRUSH rule 1 x 931 [5,2]
+ CRUSH rule 1 x 932 [5,8]
+ CRUSH rule 1 x 933 [8,5]
+ CRUSH rule 1 x 934 [5,2]
+ CRUSH rule 1 x 935 [6,5]
+ CRUSH rule 1 x 936 [2,6]
+ CRUSH rule 1 x 937 [5,6]
+ CRUSH rule 1 x 938 [6,5]
+ CRUSH rule 1 x 939 [2,7]
+ CRUSH rule 1 x 940 [8,7]
+ CRUSH rule 1 x 941 [5,2]
+ CRUSH rule 1 x 942 [2,8]
+ CRUSH rule 1 x 943 [8,2]
+ CRUSH rule 1 x 944 [5,2]
+ CRUSH rule 1 x 945 [7,2]
+ CRUSH rule 1 x 946 [2,5]
+ CRUSH rule 1 x 947 [5,6]
+ CRUSH rule 1 x 948 [7,8]
+ CRUSH rule 1 x 949 [6,2]
+ CRUSH rule 1 x 950 [5,7]
+ CRUSH rule 1 x 951 [5,8]
+ CRUSH rule 1 x 952 [2,5]
+ CRUSH rule 1 x 953 [2,5]
+ CRUSH rule 1 x 954 [5,2]
+ CRUSH rule 1 x 955 [8,6]
+ CRUSH rule 1 x 956 [2,8]
+ CRUSH rule 1 x 957 [7,6]
+ CRUSH rule 1 x 958 [8,7]
+ CRUSH rule 1 x 959 [5,2]
+ CRUSH rule 1 x 960 [5,6]
+ CRUSH rule 1 x 961 [5,2]
+ CRUSH rule 1 x 962 [7,5]
+ CRUSH rule 1 x 963 [2,5]
+ CRUSH rule 1 x 964 [5,2]
+ CRUSH rule 1 x 965 [7,6]
+ CRUSH rule 1 x 966 [5,8]
+ CRUSH rule 1 x 967 [8,6]
+ CRUSH rule 1 x 968 [7,2]
+ CRUSH rule 1 x 969 [8,2]
+ CRUSH rule 1 x 970 [2,6]
+ CRUSH rule 1 x 971 [2,7]
+ CRUSH rule 1 x 972 [2,8]
+ CRUSH rule 1 x 973 [2,6]
+ CRUSH rule 1 x 974 [5,2]
+ CRUSH rule 1 x 975 [5,7]
+ CRUSH rule 1 x 976 [5,6]
+ CRUSH rule 1 x 977 [8,5]
+ CRUSH rule 1 x 978 [7,2]
+ CRUSH rule 1 x 979 [7,6]
+ CRUSH rule 1 x 980 [6,2]
+ CRUSH rule 1 x 981 [7,5]
+ CRUSH rule 1 x 982 [5,2]
+ CRUSH rule 1 x 983 [5,6]
+ CRUSH rule 1 x 984 [2,8]
+ CRUSH rule 1 x 985 [2,5]
+ CRUSH rule 1 x 986 [8,7]
+ CRUSH rule 1 x 987 [2,5]
+ CRUSH rule 1 x 988 [2,5]
+ CRUSH rule 1 x 989 [2,6]
+ CRUSH rule 1 x 990 [2,5]
+ CRUSH rule 1 x 991 [2,5]
+ CRUSH rule 1 x 992 [7,2]
+ CRUSH rule 1 x 993 [2,6]
+ CRUSH rule 1 x 994 [5,2]
+ CRUSH rule 1 x 995 [7,6]
+ CRUSH rule 1 x 996 [6,7]
+ CRUSH rule 1 x 997 [6,5]
+ CRUSH rule 1 x 998 [8,2]
+ CRUSH rule 1 x 999 [2,7]
+ CRUSH rule 1 x 1000 [8,5]
+ CRUSH rule 1 x 1001 [2,5]
+ CRUSH rule 1 x 1002 [2,5]
+ CRUSH rule 1 x 1003 [2,8]
+ CRUSH rule 1 x 1004 [6,2]
+ CRUSH rule 1 x 1005 [6,2]
+ CRUSH rule 1 x 1006 [2,5]
+ CRUSH rule 1 x 1007 [2,8]
+ CRUSH rule 1 x 1008 [2,7]
+ CRUSH rule 1 x 1009 [6,8]
+ CRUSH rule 1 x 1010 [5,6]
+ CRUSH rule 1 x 1011 [5,2]
+ CRUSH rule 1 x 1012 [5,2]
+ CRUSH rule 1 x 1013 [5,2]
+ CRUSH rule 1 x 1014 [2,8]
+ CRUSH rule 1 x 1015 [6,8]
+ CRUSH rule 1 x 1016 [2,6]
+ CRUSH rule 1 x 1017 [6,2]
+ CRUSH rule 1 x 1018 [5,2]
+ CRUSH rule 1 x 1019 [5,7]
+ CRUSH rule 1 x 1020 [5,2]
+ CRUSH rule 1 x 1021 [5,2]
+ CRUSH rule 1 x 1022 [2,6]
+ CRUSH rule 1 x 1023 [5,2]
+ rule 1 (choose-two) num_rep 2 result size == 1:\t4/1024 (esc)
+ rule 1 (choose-two) num_rep 2 result size == 2:\t1020/1024 (esc)
+ CRUSH rule 1 x 0 [2,8,5]
+ CRUSH rule 1 x 1 [2,8,6]
+ CRUSH rule 1 x 2 [2,5,8]
+ CRUSH rule 1 x 3 [8,2,6]
+ CRUSH rule 1 x 4 [5,7,2]
+ CRUSH rule 1 x 5 [7,8,2]
+ CRUSH rule 1 x 6 [2,6,7]
+ CRUSH rule 1 x 7 [5,7,8]
+ CRUSH rule 1 x 8 [5,2,8]
+ CRUSH rule 1 x 9 [2,5,6]
+ CRUSH rule 1 x 10 [2,8,6]
+ CRUSH rule 1 x 11 [2,7,5]
+ CRUSH rule 1 x 12 [2,7,5]
+ CRUSH rule 1 x 13 [5,8,2]
+ CRUSH rule 1 x 14 [7,6,8]
+ CRUSH rule 1 x 15 [7,2,5]
+ CRUSH rule 1 x 16 [5,6,2]
+ CRUSH rule 1 x 17 [5,7,2]
+ CRUSH rule 1 x 18 [2,5,8]
+ CRUSH rule 1 x 19 [7,5,8]
+ CRUSH rule 1 x 20 [2,5,7]
+ CRUSH rule 1 x 21 [5,7,2]
+ CRUSH rule 1 x 22 [8,5,7]
+ CRUSH rule 1 x 23 [5,6,8]
+ CRUSH rule 1 x 24 [2,8,6]
+ CRUSH rule 1 x 25 [5,7,2]
+ CRUSH rule 1 x 26 [2,8,7]
+ CRUSH rule 1 x 27 [5,2]
+ CRUSH rule 1 x 28 [6,2,5]
+ CRUSH rule 1 x 29 [8,5,7]
+ CRUSH rule 1 x 30 [5,7,2]
+ CRUSH rule 1 x 31 [8,7,6]
+ CRUSH rule 1 x 32 [5,6,8]
+ CRUSH rule 1 x 33 [2,7,8]
+ CRUSH rule 1 x 34 [2,5,7]
+ CRUSH rule 1 x 35 [2,8,6]
+ CRUSH rule 1 x 36 [5,8,2]
+ CRUSH rule 1 x 37 [2,5,8]
+ CRUSH rule 1 x 38 [5,8,7]
+ CRUSH rule 1 x 39 [5,7,2]
+ CRUSH rule 1 x 40 [7,8,2]
+ CRUSH rule 1 x 41 [2,6,8]
+ CRUSH rule 1 x 42 [5,2,6]
+ CRUSH rule 1 x 43 [2,5,7]
+ CRUSH rule 1 x 44 [2,6,7]
+ CRUSH rule 1 x 45 [8,2,5]
+ CRUSH rule 1 x 46 [2,5,7]
+ CRUSH rule 1 x 47 [5,2,8]
+ CRUSH rule 1 x 48 [5,6,7]
+ CRUSH rule 1 x 49 [5,2,8]
+ CRUSH rule 1 x 50 [5,6,2]
+ CRUSH rule 1 x 51 [5,6,2]
+ CRUSH rule 1 x 52 [8,6,2]
+ CRUSH rule 1 x 53 [5,2,8]
+ CRUSH rule 1 x 54 [7,6,8]
+ CRUSH rule 1 x 55 [8,7,2]
+ CRUSH rule 1 x 56 [6,5,8]
+ CRUSH rule 1 x 57 [5,2,6]
+ CRUSH rule 1 x 58 [2,5,6]
+ CRUSH rule 1 x 59 [5,2,7]
+ CRUSH rule 1 x 60 [5,7,2]
+ CRUSH rule 1 x 61 [5,6,8]
+ CRUSH rule 1 x 62 [7,2,6]
+ CRUSH rule 1 x 63 [5,6,8]
+ CRUSH rule 1 x 64 [5,2,8]
+ CRUSH rule 1 x 65 [7,5,6]
+ CRUSH rule 1 x 66 [5,2,6]
+ CRUSH rule 1 x 67 [5,2,8]
+ CRUSH rule 1 x 68 [2,5,8]
+ CRUSH rule 1 x 69 [5,2,6]
+ CRUSH rule 1 x 70 [7,2,5]
+ CRUSH rule 1 x 71 [2,8,6]
+ CRUSH rule 1 x 72 [6,2,5]
+ CRUSH rule 1 x 73 [2,7,5]
+ CRUSH rule 1 x 74 [2,7,5]
+ CRUSH rule 1 x 75 [5,2,7]
+ CRUSH rule 1 x 76 [5,2,7]
+ CRUSH rule 1 x 77 [7,2,6]
+ CRUSH rule 1 x 78 [2,5,8]
+ CRUSH rule 1 x 79 [5,2,7]
+ CRUSH rule 1 x 80 [2,5,7]
+ CRUSH rule 1 x 81 [2,5]
+ CRUSH rule 1 x 82 [7,2,6]
+ CRUSH rule 1 x 83 [2,6,8]
+ CRUSH rule 1 x 84 [7,2,6]
+ CRUSH rule 1 x 85 [5,8,7]
+ CRUSH rule 1 x 86 [2,7,8]
+ CRUSH rule 1 x 87 [2,7,5]
+ CRUSH rule 1 x 88 [2,6,5]
+ CRUSH rule 1 x 89 [5,2,6]
+ CRUSH rule 1 x 90 [6,7,5]
+ CRUSH rule 1 x 91 [5,8,7]
+ CRUSH rule 1 x 92 [2,8,5]
+ CRUSH rule 1 x 93 [7,5,8]
+ CRUSH rule 1 x 94 [2,5,7]
+ CRUSH rule 1 x 95 [7,5,2]
+ CRUSH rule 1 x 96 [5,6,7]
+ CRUSH rule 1 x 97 [8,7,5]
+ CRUSH rule 1 x 98 [2,6,7]
+ CRUSH rule 1 x 99 [2,7,8]
+ CRUSH rule 1 x 100 [2,7,5]
+ CRUSH rule 1 x 101 [5,7,2]
+ CRUSH rule 1 x 102 [5,2,7]
+ CRUSH rule 1 x 103 [5,7,6]
+ CRUSH rule 1 x 104 [7,5,2]
+ CRUSH rule 1 x 105 [2,5,6]
+ CRUSH rule 1 x 106 [2,6,5]
+ CRUSH rule 1 x 107 [5,2,7]
+ CRUSH rule 1 x 108 [7,2,8]
+ CRUSH rule 1 x 109 [2,5,8]
+ CRUSH rule 1 x 110 [5,2,7]
+ CRUSH rule 1 x 111 [2,5,8]
+ CRUSH rule 1 x 112 [2,5,6]
+ CRUSH rule 1 x 113 [6,2,8]
+ CRUSH rule 1 x 114 [7,6,5]
+ CRUSH rule 1 x 115 [8,2,5]
+ CRUSH rule 1 x 116 [2,6,7]
+ CRUSH rule 1 x 117 [7,5,6]
+ CRUSH rule 1 x 118 [2,5,8]
+ CRUSH rule 1 x 119 [5,6,2]
+ CRUSH rule 1 x 120 [2,5,8]
+ CRUSH rule 1 x 121 [2,5,7]
+ CRUSH rule 1 x 122 [8,5,2]
+ CRUSH rule 1 x 123 [2,5]
+ CRUSH rule 1 x 124 [5,8,2]
+ CRUSH rule 1 x 125 [2,7,5]
+ CRUSH rule 1 x 126 [5,2,6]
+ CRUSH rule 1 x 127 [5,6,7]
+ CRUSH rule 1 x 128 [5,8,6]
+ CRUSH rule 1 x 129 [2,7,6]
+ CRUSH rule 1 x 130 [5,8,2]
+ CRUSH rule 1 x 131 [2,8,7]
+ CRUSH rule 1 x 132 [2,7,5]
+ CRUSH rule 1 x 133 [5,6,7]
+ CRUSH rule 1 x 134 [2,8,7]
+ CRUSH rule 1 x 135 [5,6,2]
+ CRUSH rule 1 x 136 [2,5,6]
+ CRUSH rule 1 x 137 [7,5,6]
+ CRUSH rule 1 x 138 [8,7,6]
+ CRUSH rule 1 x 139 [5,2,7]
+ CRUSH rule 1 x 140 [2,6,7]
+ CRUSH rule 1 x 141 [6,8,2]
+ CRUSH rule 1 x 142 [5,6,2]
+ CRUSH rule 1 x 143 [5,8,7]
+ CRUSH rule 1 x 144 [8,2,5]
+ CRUSH rule 1 x 145 [8,5,6]
+ CRUSH rule 1 x 146 [2,6,5]
+ CRUSH rule 1 x 147 [2,8,7]
+ CRUSH rule 1 x 148 [5,2,6]
+ CRUSH rule 1 x 149 [5,8,6]
+ CRUSH rule 1 x 150 [2,6,5]
+ CRUSH rule 1 x 151 [5,2]
+ CRUSH rule 1 x 152 [8,5,6]
+ CRUSH rule 1 x 153 [8,6,7]
+ CRUSH rule 1 x 154 [5,2,6]
+ CRUSH rule 1 x 155 [5,7,8]
+ CRUSH rule 1 x 156 [5,8,7]
+ CRUSH rule 1 x 157 [5,2,7]
+ CRUSH rule 1 x 158 [2,8,7]
+ CRUSH rule 1 x 159 [7,2,5]
+ CRUSH rule 1 x 160 [2,8,7]
+ CRUSH rule 1 x 161 [2,5,6]
+ CRUSH rule 1 x 162 [2,6,7]
+ CRUSH rule 1 x 163 [5,6,2]
+ CRUSH rule 1 x 164 [7,8,2]
+ CRUSH rule 1 x 165 [7,2,5]
+ CRUSH rule 1 x 166 [2,5,8]
+ CRUSH rule 1 x 167 [2,6,8]
+ CRUSH rule 1 x 168 [5,2,6]
+ CRUSH rule 1 x 169 [2,6,8]
+ CRUSH rule 1 x 170 [2,8,5]
+ CRUSH rule 1 x 171 [7,5,8]
+ CRUSH rule 1 x 172 [2,7,8]
+ CRUSH rule 1 x 173 [8,5,6]
+ CRUSH rule 1 x 174 [2,5,7]
+ CRUSH rule 1 x 175 [6,2,5]
+ CRUSH rule 1 x 176 [5,2,6]
+ CRUSH rule 1 x 177 [5,8,2]
+ CRUSH rule 1 x 178 [5,2,7]
+ CRUSH rule 1 x 179 [5,2,8]
+ CRUSH rule 1 x 180 [5,8,7]
+ CRUSH rule 1 x 181 [6,2,8]
+ CRUSH rule 1 x 182 [8,5,2]
+ CRUSH rule 1 x 183 [7,8,6]
+ CRUSH rule 1 x 184 [5,7,6]
+ CRUSH rule 1 x 185 [6,8,2]
+ CRUSH rule 1 x 186 [2,8,5]
+ CRUSH rule 1 x 187 [2,6,7]
+ CRUSH rule 1 x 188 [2,8,7]
+ CRUSH rule 1 x 189 [2,7,8]
+ CRUSH rule 1 x 190 [5,2,8]
+ CRUSH rule 1 x 191 [7,6,8]
+ CRUSH rule 1 x 192 [5,2,7]
+ CRUSH rule 1 x 193 [5,2,6]
+ CRUSH rule 1 x 194 [2,5,6]
+ CRUSH rule 1 x 195 [6,5,8]
+ CRUSH rule 1 x 196 [6,7,2]
+ CRUSH rule 1 x 197 [6,5,2]
+ CRUSH rule 1 x 198 [2,5,6]
+ CRUSH rule 1 x 199 [2,5,7]
+ CRUSH rule 1 x 200 [2,5,7]
+ CRUSH rule 1 x 201 [7,2,8]
+ CRUSH rule 1 x 202 [6,5,2]
+ CRUSH rule 1 x 203 [5,8,7]
+ CRUSH rule 1 x 204 [2,5,8]
+ CRUSH rule 1 x 205 [2,7,6]
+ CRUSH rule 1 x 206 [2,5,7]
+ CRUSH rule 1 x 207 [5,2,8]
+ CRUSH rule 1 x 208 [7,2,8]
+ CRUSH rule 1 x 209 [2,5,7]
+ CRUSH rule 1 x 210 [2,8,5]
+ CRUSH rule 1 x 211 [5,6,2]
+ CRUSH rule 1 x 212 [7,5,2]
+ CRUSH rule 1 x 213 [8,5,2]
+ CRUSH rule 1 x 214 [5,7,8]
+ CRUSH rule 1 x 215 [8,2,7]
+ CRUSH rule 1 x 216 [5,2,6]
+ CRUSH rule 1 x 217 [2,5,7]
+ CRUSH rule 1 x 218 [2,7,8]
+ CRUSH rule 1 x 219 [5,8,2]
+ CRUSH rule 1 x 220 [5,7,8]
+ CRUSH rule 1 x 221 [5,6,8]
+ CRUSH rule 1 x 222 [6,8,5]
+ CRUSH rule 1 x 223 [2,5,6]
+ CRUSH rule 1 x 224 [2,5,8]
+ CRUSH rule 1 x 225 [8,6,2]
+ CRUSH rule 1 x 226 [7,2,5]
+ CRUSH rule 1 x 227 [5,6,2]
+ CRUSH rule 1 x 228 [5,6,8]
+ CRUSH rule 1 x 229 [5,2,7]
+ CRUSH rule 1 x 230 [5,7,2]
+ CRUSH rule 1 x 231 [5,6,2]
+ CRUSH rule 1 x 232 [2,7,5]
+ CRUSH rule 1 x 233 [5,8,7]
+ CRUSH rule 1 x 234 [2,7,5]
+ CRUSH rule 1 x 235 [5,8,2]
+ CRUSH rule 1 x 236 [5,2,7]
+ CRUSH rule 1 x 237 [5,7,6]
+ CRUSH rule 1 x 238 [5,6,2]
+ CRUSH rule 1 x 239 [8,7,6]
+ CRUSH rule 1 x 240 [5,7,2]
+ CRUSH rule 1 x 241 [5,2,8]
+ CRUSH rule 1 x 242 [5,8,2]
+ CRUSH rule 1 x 243 [5,7,8]
+ CRUSH rule 1 x 244 [5,6,2]
+ CRUSH rule 1 x 245 [7,6,2]
+ CRUSH rule 1 x 246 [2,5,7]
+ CRUSH rule 1 x 247 [6,2,7]
+ CRUSH rule 1 x 248 [8,2,7]
+ CRUSH rule 1 x 249 [2,5,8]
+ CRUSH rule 1 x 250 [2,7,5]
+ CRUSH rule 1 x 251 [2,5,6]
+ CRUSH rule 1 x 252 [5,7,8]
+ CRUSH rule 1 x 253 [5,2,8]
+ CRUSH rule 1 x 254 [5,6,7]
+ CRUSH rule 1 x 255 [2,7,8]
+ CRUSH rule 1 x 256 [5,7,2]
+ CRUSH rule 1 x 257 [2,8,5]
+ CRUSH rule 1 x 258 [5,2]
+ CRUSH rule 1 x 259 [5,7,6]
+ CRUSH rule 1 x 260 [5,6,2]
+ CRUSH rule 1 x 261 [8,7,5]
+ CRUSH rule 1 x 262 [5,2,8]
+ CRUSH rule 1 x 263 [6,8,2]
+ CRUSH rule 1 x 264 [5,6,2]
+ CRUSH rule 1 x 265 [8,6,7]
+ CRUSH rule 1 x 266 [8,2,5]
+ CRUSH rule 1 x 267 [2,5,8]
+ CRUSH rule 1 x 268 [2,7,5]
+ CRUSH rule 1 x 269 [2,8,7]
+ CRUSH rule 1 x 270 [5,2,7]
+ CRUSH rule 1 x 271 [7,5,6]
+ CRUSH rule 1 x 272 [2,8,5]
+ CRUSH rule 1 x 273 [5,2]
+ CRUSH rule 1 x 274 [6,8,5]
+ CRUSH rule 1 x 275 [5,8,6]
+ CRUSH rule 1 x 276 [7,2,6]
+ CRUSH rule 1 x 277 [6,5,2]
+ CRUSH rule 1 x 278 [6,8,2]
+ CRUSH rule 1 x 279 [8,5,6]
+ CRUSH rule 1 x 280 [2,6,7]
+ CRUSH rule 1 x 281 [8,2,6]
+ CRUSH rule 1 x 282 [5,2,6]
+ CRUSH rule 1 x 283 [8,2,7]
+ CRUSH rule 1 x 284 [6,5,2]
+ CRUSH rule 1 x 285 [5,6,7]
+ CRUSH rule 1 x 286 [2,5,6]
+ CRUSH rule 1 x 287 [2,5,6]
+ CRUSH rule 1 x 288 [8,2,7]
+ CRUSH rule 1 x 289 [5,6,2]
+ CRUSH rule 1 x 290 [2,5,7]
+ CRUSH rule 1 x 291 [2,5,7]
+ CRUSH rule 1 x 292 [8,2,7]
+ CRUSH rule 1 x 293 [6,2,8]
+ CRUSH rule 1 x 294 [7,5,2]
+ CRUSH rule 1 x 295 [5,8,7]
+ CRUSH rule 1 x 296 [5,2,6]
+ CRUSH rule 1 x 297 [6,2,8]
+ CRUSH rule 1 x 298 [2,8,5]
+ CRUSH rule 1 x 299 [2,5,7]
+ CRUSH rule 1 x 300 [8,7,5]
+ CRUSH rule 1 x 301 [2,8,7]
+ CRUSH rule 1 x 302 [5,2,6]
+ CRUSH rule 1 x 303 [7,5,8]
+ CRUSH rule 1 x 304 [2,7,5]
+ CRUSH rule 1 x 305 [5,8,2]
+ CRUSH rule 1 x 306 [2,7,5]
+ CRUSH rule 1 x 307 [2,7,8]
+ CRUSH rule 1 x 308 [2,8,5]
+ CRUSH rule 1 x 309 [7,5,6]
+ CRUSH rule 1 x 310 [5,6,2]
+ CRUSH rule 1 x 311 [5,2,7]
+ CRUSH rule 1 x 312 [2,6,5]
+ CRUSH rule 1 x 313 [5,8,6]
+ CRUSH rule 1 x 314 [5,2,6]
+ CRUSH rule 1 x 315 [2,5,8]
+ CRUSH rule 1 x 316 [6,5,8]
+ CRUSH rule 1 x 317 [2,6,7]
+ CRUSH rule 1 x 318 [8,6,7]
+ CRUSH rule 1 x 319 [5,2,8]
+ CRUSH rule 1 x 320 [5,7,2]
+ CRUSH rule 1 x 321 [2,5]
+ CRUSH rule 1 x 322 [2,7,5]
+ CRUSH rule 1 x 323 [5,7,2]
+ CRUSH rule 1 x 324 [7,2,8]
+ CRUSH rule 1 x 325 [5,6,2]
+ CRUSH rule 1 x 326 [5,6,2]
+ CRUSH rule 1 x 327 [2,6,7]
+ CRUSH rule 1 x 328 [7,5,8]
+ CRUSH rule 1 x 329 [5,6,2]
+ CRUSH rule 1 x 330 [5,7,2]
+ CRUSH rule 1 x 331 [2,6,5]
+ CRUSH rule 1 x 332 [2,5,8]
+ CRUSH rule 1 x 333 [6,8,5]
+ CRUSH rule 1 x 334 [8,5,6]
+ CRUSH rule 1 x 335 [7,2,5]
+ CRUSH rule 1 x 336 [5,6,2]
+ CRUSH rule 1 x 337 [7,2,6]
+ CRUSH rule 1 x 338 [5,6,8]
+ CRUSH rule 1 x 339 [7,5,2]
+ CRUSH rule 1 x 340 [2,5,6]
+ CRUSH rule 1 x 341 [5,2,7]
+ CRUSH rule 1 x 342 [2,7,5]
+ CRUSH rule 1 x 343 [6,7,5]
+ CRUSH rule 1 x 344 [6,2,5]
+ CRUSH rule 1 x 345 [5,2,7]
+ CRUSH rule 1 x 346 [8,2,5]
+ CRUSH rule 1 x 347 [5,2,6]
+ CRUSH rule 1 x 348 [8,2,7]
+ CRUSH rule 1 x 349 [2,6,7]
+ CRUSH rule 1 x 350 [8,5,7]
+ CRUSH rule 1 x 351 [5,6,2]
+ CRUSH rule 1 x 352 [2,7,5]
+ CRUSH rule 1 x 353 [6,5,8]
+ CRUSH rule 1 x 354 [2,5,8]
+ CRUSH rule 1 x 355 [5,2,6]
+ CRUSH rule 1 x 356 [5,8,2]
+ CRUSH rule 1 x 357 [6,2,7]
+ CRUSH rule 1 x 358 [2,5,8]
+ CRUSH rule 1 x 359 [6,7,8]
+ CRUSH rule 1 x 360 [5,2,6]
+ CRUSH rule 1 x 361 [8,5,2]
+ CRUSH rule 1 x 362 [5,7,6]
+ CRUSH rule 1 x 363 [5,2,6]
+ CRUSH rule 1 x 364 [2,5,7]
+ CRUSH rule 1 x 365 [6,7,8]
+ CRUSH rule 1 x 366 [7,2,8]
+ CRUSH rule 1 x 367 [5,7,2]
+ CRUSH rule 1 x 368 [7,5,6]
+ CRUSH rule 1 x 369 [5,7,2]
+ CRUSH rule 1 x 370 [8,7,6]
+ CRUSH rule 1 x 371 [2,5,8]
+ CRUSH rule 1 x 372 [5,2,8]
+ CRUSH rule 1 x 373 [2,6,8]
+ CRUSH rule 1 x 374 [5,8,7]
+ CRUSH rule 1 x 375 [6,5,7]
+ CRUSH rule 1 x 376 [7,2,5]
+ CRUSH rule 1 x 377 [2,5,6]
+ CRUSH rule 1 x 378 [2,6,7]
+ CRUSH rule 1 x 379 [8,5,7]
+ CRUSH rule 1 x 380 [2,5]
+ CRUSH rule 1 x 381 [2,5,7]
+ CRUSH rule 1 x 382 [2,5,6]
+ CRUSH rule 1 x 383 [5,2]
+ CRUSH rule 1 x 384 [7,2,6]
+ CRUSH rule 1 x 385 [7,5,6]
+ CRUSH rule 1 x 386 [2,5,8]
+ CRUSH rule 1 x 387 [2,5,6]
+ CRUSH rule 1 x 388 [5,2,8]
+ CRUSH rule 1 x 389 [2,5,8]
+ CRUSH rule 1 x 390 [5,6,2]
+ CRUSH rule 1 x 391 [5,6,2]
+ CRUSH rule 1 x 392 [2,8,6]
+ CRUSH rule 1 x 393 [5,2,6]
+ CRUSH rule 1 x 394 [5,7,6]
+ CRUSH rule 1 x 395 [5,2,8]
+ CRUSH rule 1 x 396 [5,2,8]
+ CRUSH rule 1 x 397 [2,7,5]
+ CRUSH rule 1 x 398 [2,5,6]
+ CRUSH rule 1 x 399 [8,7,5]
+ CRUSH rule 1 x 400 [8,2,5]
+ CRUSH rule 1 x 401 [2,5,6]
+ CRUSH rule 1 x 402 [7,8,6]
+ CRUSH rule 1 x 403 [2,7,5]
+ CRUSH rule 1 x 404 [5,2,7]
+ CRUSH rule 1 x 405 [6,5,2]
+ CRUSH rule 1 x 406 [2,6,5]
+ CRUSH rule 1 x 407 [2,8,7]
+ CRUSH rule 1 x 408 [5,2,6]
+ CRUSH rule 1 x 409 [7,5,6]
+ CRUSH rule 1 x 410 [8,6,5]
+ CRUSH rule 1 x 411 [2,6,8]
+ CRUSH rule 1 x 412 [2,5,8]
+ CRUSH rule 1 x 413 [5,2,8]
+ CRUSH rule 1 x 414 [5,2,8]
+ CRUSH rule 1 x 415 [2,6,5]
+ CRUSH rule 1 x 416 [2,5,7]
+ CRUSH rule 1 x 417 [8,7,2]
+ CRUSH rule 1 x 418 [7,6,8]
+ CRUSH rule 1 x 419 [8,5,2]
+ CRUSH rule 1 x 420 [2,5,8]
+ CRUSH rule 1 x 421 [8,6,7]
+ CRUSH rule 1 x 422 [6,7,8]
+ CRUSH rule 1 x 423 [2,5,6]
+ CRUSH rule 1 x 424 [8,5,7]
+ CRUSH rule 1 x 425 [2,5,7]
+ CRUSH rule 1 x 426 [6,7,2]
+ CRUSH rule 1 x 427 [2,7,5]
+ CRUSH rule 1 x 428 [5,6,7]
+ CRUSH rule 1 x 429 [5,6,7]
+ CRUSH rule 1 x 430 [5,6,2]
+ CRUSH rule 1 x 431 [5,2]
+ CRUSH rule 1 x 432 [7,2,8]
+ CRUSH rule 1 x 433 [6,5,2]
+ CRUSH rule 1 x 434 [5,2,6]
+ CRUSH rule 1 x 435 [2,5,8]
+ CRUSH rule 1 x 436 [5,2,6]
+ CRUSH rule 1 x 437 [7,5,2]
+ CRUSH rule 1 x 438 [2,5,8]
+ CRUSH rule 1 x 439 [2,5,7]
+ CRUSH rule 1 x 440 [2,7,6]
+ CRUSH rule 1 x 441 [5,7,2]
+ CRUSH rule 1 x 442 [2,5,7]
+ CRUSH rule 1 x 443 [6,8,2]
+ CRUSH rule 1 x 444 [7,2,8]
+ CRUSH rule 1 x 445 [6,5,7]
+ CRUSH rule 1 x 446 [5,7,6]
+ CRUSH rule 1 x 447 [2,5,8]
+ CRUSH rule 1 x 448 [7,2,5]
+ CRUSH rule 1 x 449 [7,8,5]
+ CRUSH rule 1 x 450 [5,8,2]
+ CRUSH rule 1 x 451 [6,8,5]
+ CRUSH rule 1 x 452 [8,5,6]
+ CRUSH rule 1 x 453 [6,8,7]
+ CRUSH rule 1 x 454 [6,7,5]
+ CRUSH rule 1 x 455 [2,7,5]
+ CRUSH rule 1 x 456 [6,8,7]
+ CRUSH rule 1 x 457 [7,2,8]
+ CRUSH rule 1 x 458 [2,8,7]
+ CRUSH rule 1 x 459 [2,5,6]
+ CRUSH rule 1 x 460 [6,5,7]
+ CRUSH rule 1 x 461 [6,5,8]
+ CRUSH rule 1 x 462 [8,2,5]
+ CRUSH rule 1 x 463 [6,7,2]
+ CRUSH rule 1 x 464 [7,5,2]
+ CRUSH rule 1 x 465 [7,6,2]
+ CRUSH rule 1 x 466 [5,8,7]
+ CRUSH rule 1 x 467 [6,5,7]
+ CRUSH rule 1 x 468 [7,8,2]
+ CRUSH rule 1 x 469 [7,2,6]
+ CRUSH rule 1 x 470 [5,2,6]
+ CRUSH rule 1 x 471 [2,6,8]
+ CRUSH rule 1 x 472 [5,6,2]
+ CRUSH rule 1 x 473 [2,5,6]
+ CRUSH rule 1 x 474 [6,2,7]
+ CRUSH rule 1 x 475 [6,7,8]
+ CRUSH rule 1 x 476 [5,2]
+ CRUSH rule 1 x 477 [5,8,6]
+ CRUSH rule 1 x 478 [6,7,2]
+ CRUSH rule 1 x 479 [2,5,8]
+ CRUSH rule 1 x 480 [2,8,6]
+ CRUSH rule 1 x 481 [2,5,7]
+ CRUSH rule 1 x 482 [5,7,2]
+ CRUSH rule 1 x 483 [2,7,6]
+ CRUSH rule 1 x 484 [2,7,6]
+ CRUSH rule 1 x 485 [5,7,2]
+ CRUSH rule 1 x 486 [5,2,7]
+ CRUSH rule 1 x 487 [5,2]
+ CRUSH rule 1 x 488 [5,7,2]
+ CRUSH rule 1 x 489 [2,8,5]
+ CRUSH rule 1 x 490 [6,5,2]
+ CRUSH rule 1 x 491 [2,6,7]
+ CRUSH rule 1 x 492 [6,5,2]
+ CRUSH rule 1 x 493 [2,8,7]
+ CRUSH rule 1 x 494 [2,7,8]
+ CRUSH rule 1 x 495 [5,6,2]
+ CRUSH rule 1 x 496 [7,5,6]
+ CRUSH rule 1 x 497 [5,7,6]
+ CRUSH rule 1 x 498 [2,5,8]
+ CRUSH rule 1 x 499 [8,5,2]
+ CRUSH rule 1 x 500 [5,6,2]
+ CRUSH rule 1 x 501 [2,7,5]
+ CRUSH rule 1 x 502 [7,2,5]
+ CRUSH rule 1 x 503 [2,5,6]
+ CRUSH rule 1 x 504 [5,6,2]
+ CRUSH rule 1 x 505 [2,7,6]
+ CRUSH rule 1 x 506 [5,2]
+ CRUSH rule 1 x 507 [6,2,8]
+ CRUSH rule 1 x 508 [2,7,8]
+ CRUSH rule 1 x 509 [7,5,8]
+ CRUSH rule 1 x 510 [6,2,5]
+ CRUSH rule 1 x 511 [5,8,7]
+ CRUSH rule 1 x 512 [7,6,2]
+ CRUSH rule 1 x 513 [7,2,5]
+ CRUSH rule 1 x 514 [5,7,8]
+ CRUSH rule 1 x 515 [8,5,6]
+ CRUSH rule 1 x 516 [5,2,6]
+ CRUSH rule 1 x 517 [7,8,6]
+ CRUSH rule 1 x 518 [5,6,7]
+ CRUSH rule 1 x 519 [7,5,8]
+ CRUSH rule 1 x 520 [2,6,5]
+ CRUSH rule 1 x 521 [8,7,6]
+ CRUSH rule 1 x 522 [6,8,2]
+ CRUSH rule 1 x 523 [5,2,6]
+ CRUSH rule 1 x 524 [2,5,8]
+ CRUSH rule 1 x 525 [2,5,7]
+ CRUSH rule 1 x 526 [2,5,8]
+ CRUSH rule 1 x 527 [2,5,8]
+ CRUSH rule 1 x 528 [5,2,7]
+ CRUSH rule 1 x 529 [5,7,2]
+ CRUSH rule 1 x 530 [6,7,8]
+ CRUSH rule 1 x 531 [6,2,5]
+ CRUSH rule 1 x 532 [6,5,7]
+ CRUSH rule 1 x 533 [5,6,2]
+ CRUSH rule 1 x 534 [7,5,2]
+ CRUSH rule 1 x 535 [8,6,2]
+ CRUSH rule 1 x 536 [6,7,2]
+ CRUSH rule 1 x 537 [5,7,6]
+ CRUSH rule 1 x 538 [6,8,5]
+ CRUSH rule 1 x 539 [8,5,7]
+ CRUSH rule 1 x 540 [2,6,5]
+ CRUSH rule 1 x 541 [2,5,7]
+ CRUSH rule 1 x 542 [5,2,8]
+ CRUSH rule 1 x 543 [6,2,8]
+ CRUSH rule 1 x 544 [5,7,6]
+ CRUSH rule 1 x 545 [5,7,2]
+ CRUSH rule 1 x 546 [6,2,7]
+ CRUSH rule 1 x 547 [8,2,5]
+ CRUSH rule 1 x 548 [5,2,7]
+ CRUSH rule 1 x 549 [5,8,2]
+ CRUSH rule 1 x 550 [2,5,8]
+ CRUSH rule 1 x 551 [7,5,6]
+ CRUSH rule 1 x 552 [5,2]
+ CRUSH rule 1 x 553 [5,2,6]
+ CRUSH rule 1 x 554 [2,8,5]
+ CRUSH rule 1 x 555 [5,2,8]
+ CRUSH rule 1 x 556 [5,6,7]
+ CRUSH rule 1 x 557 [7,5,6]
+ CRUSH rule 1 x 558 [5,2,6]
+ CRUSH rule 1 x 559 [5,2,6]
+ CRUSH rule 1 x 560 [8,5,7]
+ CRUSH rule 1 x 561 [6,5,7]
+ CRUSH rule 1 x 562 [5,7,2]
+ CRUSH rule 1 x 563 [2,6,8]
+ CRUSH rule 1 x 564 [5,2,7]
+ CRUSH rule 1 x 565 [5,6,7]
+ CRUSH rule 1 x 566 [5,7,2]
+ CRUSH rule 1 x 567 [5,6,2]
+ CRUSH rule 1 x 568 [7,5,2]
+ CRUSH rule 1 x 569 [5,2,8]
+ CRUSH rule 1 x 570 [2,5]
+ CRUSH rule 1 x 571 [5,7,8]
+ CRUSH rule 1 x 572 [5,2]
+ CRUSH rule 1 x 573 [5,2,7]
+ CRUSH rule 1 x 574 [2,8,6]
+ CRUSH rule 1 x 575 [8,6,2]
+ CRUSH rule 1 x 576 [5,6,8]
+ CRUSH rule 1 x 577 [8,2,6]
+ CRUSH rule 1 x 578 [6,8,7]
+ CRUSH rule 1 x 579 [5,2,6]
+ CRUSH rule 1 x 580 [5,2,7]
+ CRUSH rule 1 x 581 [7,2,6]
+ CRUSH rule 1 x 582 [2,8,7]
+ CRUSH rule 1 x 583 [6,2,8]
+ CRUSH rule 1 x 584 [8,2,6]
+ CRUSH rule 1 x 585 [7,2,5]
+ CRUSH rule 1 x 586 [2,6,7]
+ CRUSH rule 1 x 587 [2,5,6]
+ CRUSH rule 1 x 588 [5,2,7]
+ CRUSH rule 1 x 589 [7,2,5]
+ CRUSH rule 1 x 590 [6,2,5]
+ CRUSH rule 1 x 591 [5,2,8]
+ CRUSH rule 1 x 592 [2,5,7]
+ CRUSH rule 1 x 593 [2,8,6]
+ CRUSH rule 1 x 594 [2,7,5]
+ CRUSH rule 1 x 595 [7,2,8]
+ CRUSH rule 1 x 596 [5,7,2]
+ CRUSH rule 1 x 597 [5,2,7]
+ CRUSH rule 1 x 598 [5,2,8]
+ CRUSH rule 1 x 599 [5,2,6]
+ CRUSH rule 1 x 600 [7,2,8]
+ CRUSH rule 1 x 601 [2,7,8]
+ CRUSH rule 1 x 602 [5,7,8]
+ CRUSH rule 1 x 603 [5,2,6]
+ CRUSH rule 1 x 604 [7,5,2]
+ CRUSH rule 1 x 605 [5,2,8]
+ CRUSH rule 1 x 606 [2,6,8]
+ CRUSH rule 1 x 607 [2,5,8]
+ CRUSH rule 1 x 608 [5,2,6]
+ CRUSH rule 1 x 609 [5,2]
+ CRUSH rule 1 x 610 [5,7,6]
+ CRUSH rule 1 x 611 [2,5,7]
+ CRUSH rule 1 x 612 [2,8,7]
+ CRUSH rule 1 x 613 [7,2,5]
+ CRUSH rule 1 x 614 [7,8,6]
+ CRUSH rule 1 x 615 [6,8,2]
+ CRUSH rule 1 x 616 [2,8,7]
+ CRUSH rule 1 x 617 [6,2,7]
+ CRUSH rule 1 x 618 [7,6,5]
+ CRUSH rule 1 x 619 [5,2,8]
+ CRUSH rule 1 x 620 [5,2,6]
+ CRUSH rule 1 x 621 [5,8,2]
+ CRUSH rule 1 x 622 [2,5,8]
+ CRUSH rule 1 x 623 [2,6,5]
+ CRUSH rule 1 x 624 [5,7,2]
+ CRUSH rule 1 x 625 [2,5,7]
+ CRUSH rule 1 x 626 [7,8,2]
+ CRUSH rule 1 x 627 [2,7,8]
+ CRUSH rule 1 x 628 [8,2,5]
+ CRUSH rule 1 x 629 [2,6,5]
+ CRUSH rule 1 x 630 [2,7,5]
+ CRUSH rule 1 x 631 [2,7,8]
+ CRUSH rule 1 x 632 [7,2,5]
+ CRUSH rule 1 x 633 [8,6,5]
+ CRUSH rule 1 x 634 [2,5,8]
+ CRUSH rule 1 x 635 [5,6,2]
+ CRUSH rule 1 x 636 [2,5,6]
+ CRUSH rule 1 x 637 [5,2,7]
+ CRUSH rule 1 x 638 [6,8,2]
+ CRUSH rule 1 x 639 [5,2,7]
+ CRUSH rule 1 x 640 [5,2,6]
+ CRUSH rule 1 x 641 [7,2,6]
+ CRUSH rule 1 x 642 [2,8,5]
+ CRUSH rule 1 x 643 [5,7,2]
+ CRUSH rule 1 x 644 [8,2,5]
+ CRUSH rule 1 x 645 [5,2,6]
+ CRUSH rule 1 x 646 [8,2,5]
+ CRUSH rule 1 x 647 [7,2,5]
+ CRUSH rule 1 x 648 [2,6,5]
+ CRUSH rule 1 x 649 [5,7,6]
+ CRUSH rule 1 x 650 [7,8,6]
+ CRUSH rule 1 x 651 [5,7,6]
+ CRUSH rule 1 x 652 [5,6,8]
+ CRUSH rule 1 x 653 [8,5,2]
+ CRUSH rule 1 x 654 [7,5,2]
+ CRUSH rule 1 x 655 [2,5,6]
+ CRUSH rule 1 x 656 [5,2,8]
+ CRUSH rule 1 x 657 [6,2,8]
+ CRUSH rule 1 x 658 [5,2,6]
+ CRUSH rule 1 x 659 [5,6,7]
+ CRUSH rule 1 x 660 [7,8,6]
+ CRUSH rule 1 x 661 [2,8,5]
+ CRUSH rule 1 x 662 [5,2]
+ CRUSH rule 1 x 663 [2,5,8]
+ CRUSH rule 1 x 664 [2,5]
+ CRUSH rule 1 x 665 [5,7,6]
+ CRUSH rule 1 x 666 [2,8,5]
+ CRUSH rule 1 x 667 [2,5,6]
+ CRUSH rule 1 x 668 [5,7,6]
+ CRUSH rule 1 x 669 [6,5,2]
+ CRUSH rule 1 x 670 [5,2]
+ CRUSH rule 1 x 671 [2,5,8]
+ CRUSH rule 1 x 672 [5,6,2]
+ CRUSH rule 1 x 673 [5,2,7]
+ CRUSH rule 1 x 674 [5,2,8]
+ CRUSH rule 1 x 675 [2,8,6]
+ CRUSH rule 1 x 676 [2,5]
+ CRUSH rule 1 x 677 [5,2,7]
+ CRUSH rule 1 x 678 [2,5,6]
+ CRUSH rule 1 x 679 [6,2,7]
+ CRUSH rule 1 x 680 [2,5,6]
+ CRUSH rule 1 x 681 [5,7,2]
+ CRUSH rule 1 x 682 [2,5,7]
+ CRUSH rule 1 x 683 [2,8,5]
+ CRUSH rule 1 x 684 [7,2,6]
+ CRUSH rule 1 x 685 [7,2,6]
+ CRUSH rule 1 x 686 [2,5,8]
+ CRUSH rule 1 x 687 [5,7,6]
+ CRUSH rule 1 x 688 [5,7,2]
+ CRUSH rule 1 x 689 [6,5,2]
+ CRUSH rule 1 x 690 [8,2,7]
+ CRUSH rule 1 x 691 [5,2]
+ CRUSH rule 1 x 692 [7,2,8]
+ CRUSH rule 1 x 693 [6,7,5]
+ CRUSH rule 1 x 694 [6,5,2]
+ CRUSH rule 1 x 695 [2,8,7]
+ CRUSH rule 1 x 696 [2,5,8]
+ CRUSH rule 1 x 697 [6,2,7]
+ CRUSH rule 1 x 698 [6,2,5]
+ CRUSH rule 1 x 699 [2,6,8]
+ CRUSH rule 1 x 700 [2,5,7]
+ CRUSH rule 1 x 701 [5,2,7]
+ CRUSH rule 1 x 702 [5,6,2]
+ CRUSH rule 1 x 703 [8,5,2]
+ CRUSH rule 1 x 704 [2,5,8]
+ CRUSH rule 1 x 705 [8,6,2]
+ CRUSH rule 1 x 706 [2,7,5]
+ CRUSH rule 1 x 707 [7,8,6]
+ CRUSH rule 1 x 708 [5,7,8]
+ CRUSH rule 1 x 709 [6,5,2]
+ CRUSH rule 1 x 710 [8,5,2]
+ CRUSH rule 1 x 711 [2,5,8]
+ CRUSH rule 1 x 712 [2,5,7]
+ CRUSH rule 1 x 713 [6,7,8]
+ CRUSH rule 1 x 714 [5,2,6]
+ CRUSH rule 1 x 715 [2,5,6]
+ CRUSH rule 1 x 716 [5,6,2]
+ CRUSH rule 1 x 717 [8,7,2]
+ CRUSH rule 1 x 718 [5,7,8]
+ CRUSH rule 1 x 719 [2,6,5]
+ CRUSH rule 1 x 720 [6,8,2]
+ CRUSH rule 1 x 721 [5,6,7]
+ CRUSH rule 1 x 722 [5,2,6]
+ CRUSH rule 1 x 723 [5,2,6]
+ CRUSH rule 1 x 724 [2,6,5]
+ CRUSH rule 1 x 725 [2,8,5]
+ CRUSH rule 1 x 726 [5,8,2]
+ CRUSH rule 1 x 727 [5,6,8]
+ CRUSH rule 1 x 728 [2,6,8]
+ CRUSH rule 1 x 729 [5,6,7]
+ CRUSH rule 1 x 730 [5,7,6]
+ CRUSH rule 1 x 731 [5,2,6]
+ CRUSH rule 1 x 732 [2,5,6]
+ CRUSH rule 1 x 733 [5,2,7]
+ CRUSH rule 1 x 734 [6,5,2]
+ CRUSH rule 1 x 735 [5,8,2]
+ CRUSH rule 1 x 736 [5,8,6]
+ CRUSH rule 1 x 737 [2,5,8]
+ CRUSH rule 1 x 738 [5,2,7]
+ CRUSH rule 1 x 739 [2,8,5]
+ CRUSH rule 1 x 740 [2,6,7]
+ CRUSH rule 1 x 741 [7,8,2]
+ CRUSH rule 1 x 742 [8,2,5]
+ CRUSH rule 1 x 743 [7,2,8]
+ CRUSH rule 1 x 744 [5,7,2]
+ CRUSH rule 1 x 745 [5,6,2]
+ CRUSH rule 1 x 746 [5,2,8]
+ CRUSH rule 1 x 747 [6,2,5]
+ CRUSH rule 1 x 748 [2,7,8]
+ CRUSH rule 1 x 749 [5,8,7]
+ CRUSH rule 1 x 750 [2,6,5]
+ CRUSH rule 1 x 751 [2,8,6]
+ CRUSH rule 1 x 752 [8,2,5]
+ CRUSH rule 1 x 753 [7,8,5]
+ CRUSH rule 1 x 754 [8,6,7]
+ CRUSH rule 1 x 755 [2,5]
+ CRUSH rule 1 x 756 [5,6,2]
+ CRUSH rule 1 x 757 [8,6,2]
+ CRUSH rule 1 x 758 [6,2,5]
+ CRUSH rule 1 x 759 [8,5,6]
+ CRUSH rule 1 x 760 [2,5,8]
+ CRUSH rule 1 x 761 [5,2,7]
+ CRUSH rule 1 x 762 [2,7,8]
+ CRUSH rule 1 x 763 [8,6,7]
+ CRUSH rule 1 x 764 [2,7,5]
+ CRUSH rule 1 x 765 [6,5,2]
+ CRUSH rule 1 x 766 [8,5,7]
+ CRUSH rule 1 x 767 [2,6,5]
+ CRUSH rule 1 x 768 [8,5,2]
+ CRUSH rule 1 x 769 [6,2,8]
+ CRUSH rule 1 x 770 [6,2,7]
+ CRUSH rule 1 x 771 [7,2,5]
+ CRUSH rule 1 x 772 [8,5,7]
+ CRUSH rule 1 x 773 [5,2,7]
+ CRUSH rule 1 x 774 [5,6,2]
+ CRUSH rule 1 x 775 [6,8,5]
+ CRUSH rule 1 x 776 [7,2,5]
+ CRUSH rule 1 x 777 [5,2,6]
+ CRUSH rule 1 x 778 [2,8,5]
+ CRUSH rule 1 x 779 [2,7,5]
+ CRUSH rule 1 x 780 [2,5,6]
+ CRUSH rule 1 x 781 [6,5,7]
+ CRUSH rule 1 x 782 [5,2,8]
+ CRUSH rule 1 x 783 [7,2,8]
+ CRUSH rule 1 x 784 [2,7,5]
+ CRUSH rule 1 x 785 [6,2,5]
+ CRUSH rule 1 x 786 [7,6,5]
+ CRUSH rule 1 x 787 [2,7,6]
+ CRUSH rule 1 x 788 [6,2,8]
+ CRUSH rule 1 x 789 [2,5,8]
+ CRUSH rule 1 x 790 [8,5,7]
+ CRUSH rule 1 x 791 [5,8,7]
+ CRUSH rule 1 x 792 [5,8,7]
+ CRUSH rule 1 x 793 [6,2,5]
+ CRUSH rule 1 x 794 [2,6,8]
+ CRUSH rule 1 x 795 [2,5,6]
+ CRUSH rule 1 x 796 [5,6,7]
+ CRUSH rule 1 x 797 [2,5,7]
+ CRUSH rule 1 x 798 [6,8,7]
+ CRUSH rule 1 x 799 [5,2,8]
+ CRUSH rule 1 x 800 [5,2,7]
+ CRUSH rule 1 x 801 [5,6,8]
+ CRUSH rule 1 x 802 [2,8,7]
+ CRUSH rule 1 x 803 [2,5,7]
+ CRUSH rule 1 x 804 [6,2,5]
+ CRUSH rule 1 x 805 [5,6,7]
+ CRUSH rule 1 x 806 [2,5,8]
+ CRUSH rule 1 x 807 [5,7,8]
+ CRUSH rule 1 x 808 [5,6,2]
+ CRUSH rule 1 x 809 [2,5,7]
+ CRUSH rule 1 x 810 [5,7,2]
+ CRUSH rule 1 x 811 [8,5,2]
+ CRUSH rule 1 x 812 [8,5,2]
+ CRUSH rule 1 x 813 [6,5,8]
+ CRUSH rule 1 x 814 [5,6,8]
+ CRUSH rule 1 x 815 [5,2,8]
+ CRUSH rule 1 x 816 [2,5,8]
+ CRUSH rule 1 x 817 [5,7,6]
+ CRUSH rule 1 x 818 [5,2,8]
+ CRUSH rule 1 x 819 [5,2,6]
+ CRUSH rule 1 x 820 [5,2,8]
+ CRUSH rule 1 x 821 [5,2,8]
+ CRUSH rule 1 x 822 [2,7,5]
+ CRUSH rule 1 x 823 [5,8,2]
+ CRUSH rule 1 x 824 [5,7,2]
+ CRUSH rule 1 x 825 [2,8,7]
+ CRUSH rule 1 x 826 [7,2,5]
+ CRUSH rule 1 x 827 [2,8,7]
+ CRUSH rule 1 x 828 [2,5,7]
+ CRUSH rule 1 x 829 [5,6,7]
+ CRUSH rule 1 x 830 [2,5,8]
+ CRUSH rule 1 x 831 [2,6,8]
+ CRUSH rule 1 x 832 [5,7,2]
+ CRUSH rule 1 x 833 [2,7,8]
+ CRUSH rule 1 x 834 [5,2]
+ CRUSH rule 1 x 835 [8,5,6]
+ CRUSH rule 1 x 836 [5,2]
+ CRUSH rule 1 x 837 [6,5,2]
+ CRUSH rule 1 x 838 [6,7,2]
+ CRUSH rule 1 x 839 [5,2,6]
+ CRUSH rule 1 x 840 [7,8,5]
+ CRUSH rule 1 x 841 [5,8,7]
+ CRUSH rule 1 x 842 [2,5,7]
+ CRUSH rule 1 x 843 [6,5,7]
+ CRUSH rule 1 x 844 [5,8,2]
+ CRUSH rule 1 x 845 [5,8,6]
+ CRUSH rule 1 x 846 [5,2,7]
+ CRUSH rule 1 x 847 [2,8,7]
+ CRUSH rule 1 x 848 [2,6,8]
+ CRUSH rule 1 x 849 [5,8,2]
+ CRUSH rule 1 x 850 [2,5,6]
+ CRUSH rule 1 x 851 [6,8,7]
+ CRUSH rule 1 x 852 [7,5,8]
+ CRUSH rule 1 x 853 [6,8,2]
+ CRUSH rule 1 x 854 [7,6,2]
+ CRUSH rule 1 x 855 [5,7,2]
+ CRUSH rule 1 x 856 [6,7,5]
+ CRUSH rule 1 x 857 [8,5,2]
+ CRUSH rule 1 x 858 [6,5,2]
+ CRUSH rule 1 x 859 [6,2,7]
+ CRUSH rule 1 x 860 [5,2,6]
+ CRUSH rule 1 x 861 [8,7,6]
+ CRUSH rule 1 x 862 [6,2,7]
+ CRUSH rule 1 x 863 [8,7,2]
+ CRUSH rule 1 x 864 [5,6,8]
+ CRUSH rule 1 x 865 [8,2,6]
+ CRUSH rule 1 x 866 [5,8,7]
+ CRUSH rule 1 x 867 [6,5,2]
+ CRUSH rule 1 x 868 [6,5,2]
+ CRUSH rule 1 x 869 [8,7,5]
+ CRUSH rule 1 x 870 [2,5,8]
+ CRUSH rule 1 x 871 [5,7,8]
+ CRUSH rule 1 x 872 [5,2,7]
+ CRUSH rule 1 x 873 [5,6,7]
+ CRUSH rule 1 x 874 [2,6,7]
+ CRUSH rule 1 x 875 [2,6,5]
+ CRUSH rule 1 x 876 [5,8,2]
+ CRUSH rule 1 x 877 [6,5,2]
+ CRUSH rule 1 x 878 [5,2]
+ CRUSH rule 1 x 879 [7,5,8]
+ CRUSH rule 1 x 880 [5,2,6]
+ CRUSH rule 1 x 881 [5,6,2]
+ CRUSH rule 1 x 882 [5,2]
+ CRUSH rule 1 x 883 [2,7,5]
+ CRUSH rule 1 x 884 [6,2,5]
+ CRUSH rule 1 x 885 [5,2,6]
+ CRUSH rule 1 x 886 [5,6,8]
+ CRUSH rule 1 x 887 [7,5,2]
+ CRUSH rule 1 x 888 [6,8,2]
+ CRUSH rule 1 x 889 [2,5,7]
+ CRUSH rule 1 x 890 [7,2,6]
+ CRUSH rule 1 x 891 [2,8,5]
+ CRUSH rule 1 x 892 [6,2,5]
+ CRUSH rule 1 x 893 [2,5,7]
+ CRUSH rule 1 x 894 [7,5,2]
+ CRUSH rule 1 x 895 [5,7,2]
+ CRUSH rule 1 x 896 [2,8,6]
+ CRUSH rule 1 x 897 [5,2,6]
+ CRUSH rule 1 x 898 [2,5,7]
+ CRUSH rule 1 x 899 [2,7,6]
+ CRUSH rule 1 x 900 [5,2,6]
+ CRUSH rule 1 x 901 [5,2,7]
+ CRUSH rule 1 x 902 [8,5,7]
+ CRUSH rule 1 x 903 [5,7,2]
+ CRUSH rule 1 x 904 [5,6,8]
+ CRUSH rule 1 x 905 [6,2,5]
+ CRUSH rule 1 x 906 [2,5,7]
+ CRUSH rule 1 x 907 [7,2]
+ CRUSH rule 1 x 908 [5,8,2]
+ CRUSH rule 1 x 909 [2,5,8]
+ CRUSH rule 1 x 910 [6,5,2]
+ CRUSH rule 1 x 911 [5,8,2]
+ CRUSH rule 1 x 912 [2,6,7]
+ CRUSH rule 1 x 913 [7,6,8]
+ CRUSH rule 1 x 914 [6,5,7]
+ CRUSH rule 1 x 915 [8,2,6]
+ CRUSH rule 1 x 916 [5,2,7]
+ CRUSH rule 1 x 917 [2,5,6]
+ CRUSH rule 1 x 918 [8,2,7]
+ CRUSH rule 1 x 919 [6,2,8]
+ CRUSH rule 1 x 920 [7,6,5]
+ CRUSH rule 1 x 921 [2,5,7]
+ CRUSH rule 1 x 922 [6,7,8]
+ CRUSH rule 1 x 923 [5,8,6]
+ CRUSH rule 1 x 924 [5,6,2]
+ CRUSH rule 1 x 925 [5,7,2]
+ CRUSH rule 1 x 926 [5,8,2]
+ CRUSH rule 1 x 927 [2,6,5]
+ CRUSH rule 1 x 928 [8,2,5]
+ CRUSH rule 1 x 929 [5,2]
+ CRUSH rule 1 x 930 [2,5,6]
+ CRUSH rule 1 x 931 [5,2]
+ CRUSH rule 1 x 932 [5,8,2]
+ CRUSH rule 1 x 933 [8,5,2]
+ CRUSH rule 1 x 934 [5,2,8]
+ CRUSH rule 1 x 935 [6,5,2]
+ CRUSH rule 1 x 936 [2,6,7]
+ CRUSH rule 1 x 937 [5,6,7]
+ CRUSH rule 1 x 938 [6,5,8]
+ CRUSH rule 1 x 939 [2,7,6]
+ CRUSH rule 1 x 940 [8,7,6]
+ CRUSH rule 1 x 941 [5,2,8]
+ CRUSH rule 1 x 942 [2,8,7]
+ CRUSH rule 1 x 943 [8,2,5]
+ CRUSH rule 1 x 944 [5,2,7]
+ CRUSH rule 1 x 945 [7,2,5]
+ CRUSH rule 1 x 946 [2,5,7]
+ CRUSH rule 1 x 947 [5,6,2]
+ CRUSH rule 1 x 948 [7,8,6]
+ CRUSH rule 1 x 949 [6,2,7]
+ CRUSH rule 1 x 950 [5,7,8]
+ CRUSH rule 1 x 951 [5,8,7]
+ CRUSH rule 1 x 952 [2,5,7]
+ CRUSH rule 1 x 953 [2,5,6]
+ CRUSH rule 1 x 954 [5,2,7]
+ CRUSH rule 1 x 955 [8,6,2]
+ CRUSH rule 1 x 956 [2,8,6]
+ CRUSH rule 1 x 957 [7,6,2]
+ CRUSH rule 1 x 958 [8,7,5]
+ CRUSH rule 1 x 959 [5,2,7]
+ CRUSH rule 1 x 960 [5,6,7]
+ CRUSH rule 1 x 961 [5,2,6]
+ CRUSH rule 1 x 962 [7,5,2]
+ CRUSH rule 1 x 963 [2,5,8]
+ CRUSH rule 1 x 964 [5,2,6]
+ CRUSH rule 1 x 965 [7,6,5]
+ CRUSH rule 1 x 966 [5,8,6]
+ CRUSH rule 1 x 967 [8,6,5]
+ CRUSH rule 1 x 968 [7,2,5]
+ CRUSH rule 1 x 969 [8,2,6]
+ CRUSH rule 1 x 970 [2,6,5]
+ CRUSH rule 1 x 971 [2,7,8]
+ CRUSH rule 1 x 972 [2,8,5]
+ CRUSH rule 1 x 973 [2,6,5]
+ CRUSH rule 1 x 974 [5,2,7]
+ CRUSH rule 1 x 975 [5,7,8]
+ CRUSH rule 1 x 976 [5,6,7]
+ CRUSH rule 1 x 977 [8,5,2]
+ CRUSH rule 1 x 978 [7,2,8]
+ CRUSH rule 1 x 979 [7,6,2]
+ CRUSH rule 1 x 980 [6,2,7]
+ CRUSH rule 1 x 981 [7,5,2]
+ CRUSH rule 1 x 982 [5,2,6]
+ CRUSH rule 1 x 983 [5,6,8]
+ CRUSH rule 1 x 984 [2,8,5]
+ CRUSH rule 1 x 985 [2,5,6]
+ CRUSH rule 1 x 986 [8,7,5]
+ CRUSH rule 1 x 987 [2,5,8]
+ CRUSH rule 1 x 988 [2,5,6]
+ CRUSH rule 1 x 989 [2,6,5]
+ CRUSH rule 1 x 990 [2,5,8]
+ CRUSH rule 1 x 991 [2,5,8]
+ CRUSH rule 1 x 992 [7,2,5]
+ CRUSH rule 1 x 993 [2,6,5]
+ CRUSH rule 1 x 994 [5,2,8]
+ CRUSH rule 1 x 995 [7,6,2]
+ CRUSH rule 1 x 996 [6,7,5]
+ CRUSH rule 1 x 997 [6,5,2]
+ CRUSH rule 1 x 998 [8,2,5]
+ CRUSH rule 1 x 999 [2,7,8]
+ CRUSH rule 1 x 1000 [8,5,2]
+ CRUSH rule 1 x 1001 [2,5]
+ CRUSH rule 1 x 1002 [2,5,7]
+ CRUSH rule 1 x 1003 [2,8,7]
+ CRUSH rule 1 x 1004 [6,2,8]
+ CRUSH rule 1 x 1005 [6,2,8]
+ CRUSH rule 1 x 1006 [2,5]
+ CRUSH rule 1 x 1007 [2,8,5]
+ CRUSH rule 1 x 1008 [2,7,5]
+ CRUSH rule 1 x 1009 [6,8,5]
+ CRUSH rule 1 x 1010 [5,6,2]
+ CRUSH rule 1 x 1011 [5,2,7]
+ CRUSH rule 1 x 1012 [5,2,7]
+ CRUSH rule 1 x 1013 [5,2,7]
+ CRUSH rule 1 x 1014 [2,8,5]
+ CRUSH rule 1 x 1015 [6,8,5]
+ CRUSH rule 1 x 1016 [2,6,5]
+ CRUSH rule 1 x 1017 [6,2,5]
+ CRUSH rule 1 x 1018 [5,2,7]
+ CRUSH rule 1 x 1019 [5,7,8]
+ CRUSH rule 1 x 1020 [5,2,7]
+ CRUSH rule 1 x 1021 [5,2,8]
+ CRUSH rule 1 x 1022 [2,6,7]
+ CRUSH rule 1 x 1023 [5,2,8]
+ rule 1 (choose-two) num_rep 3 result size == 2:\t32/1024 (esc)
+ rule 1 (choose-two) num_rep 3 result size == 3:\t992/1024 (esc)
+ rule 2 (chooseleaf), x = 0..1023, numrep = 2..3
+ CRUSH rule 2 x 0 [2,5]
+ CRUSH rule 2 x 1 [2,8]
+ CRUSH rule 2 x 2 [2,5]
+ CRUSH rule 2 x 3 [8,2]
+ CRUSH rule 2 x 4 [5,2]
+ CRUSH rule 2 x 5 [7,2]
+ CRUSH rule 2 x 6 [2,6]
+ CRUSH rule 2 x 7 [5,8]
+ CRUSH rule 2 x 8 [5,6]
+ CRUSH rule 2 x 9 [2,5]
+ CRUSH rule 2 x 10 [2,7]
+ CRUSH rule 2 x 11 [2,7]
+ CRUSH rule 2 x 12 [2,5]
+ CRUSH rule 2 x 13 [5,8]
+ CRUSH rule 2 x 14 [7,2]
+ CRUSH rule 2 x 15 [7,2]
+ CRUSH rule 2 x 16 [5,6]
+ CRUSH rule 2 x 17 [5,2]
+ CRUSH rule 2 x 18 [2,5]
+ CRUSH rule 2 x 19 [7,5]
+ CRUSH rule 2 x 20 [2,5]
+ CRUSH rule 2 x 21 [5,7]
+ CRUSH rule 2 x 22 [8,5]
+ CRUSH rule 2 x 23 [5,6]
+ CRUSH rule 2 x 24 [2,7]
+ CRUSH rule 2 x 25 [5,7]
+ CRUSH rule 2 x 26 [2,8]
+ CRUSH rule 2 x 27 [5,2]
+ CRUSH rule 2 x 28 [6,2]
+ CRUSH rule 2 x 29 [8,5]
+ CRUSH rule 2 x 30 [5,7]
+ CRUSH rule 2 x 31 [8,2]
+ CRUSH rule 2 x 32 [5,6]
+ CRUSH rule 2 x 33 [2,7]
+ CRUSH rule 2 x 34 [2,5]
+ CRUSH rule 2 x 35 [2,8]
+ CRUSH rule 2 x 36 [5,8]
+ CRUSH rule 2 x 37 [2,5]
+ CRUSH rule 2 x 38 [5,8]
+ CRUSH rule 2 x 39 [5,7]
+ CRUSH rule 2 x 40 [7,2]
+ CRUSH rule 2 x 41 [2,6]
+ CRUSH rule 2 x 42 [5,6]
+ CRUSH rule 2 x 43 [2,5]
+ CRUSH rule 2 x 44 [2,6]
+ CRUSH rule 2 x 45 [8,2]
+ CRUSH rule 2 x 46 [2,5]
+ CRUSH rule 2 x 47 [5,2]
+ CRUSH rule 2 x 48 [5,6]
+ CRUSH rule 2 x 49 [5,7]
+ CRUSH rule 2 x 50 [5,2]
+ CRUSH rule 2 x 51 [5,6]
+ CRUSH rule 2 x 52 [8,2]
+ CRUSH rule 2 x 53 [5,8]
+ CRUSH rule 2 x 54 [7,5]
+ CRUSH rule 2 x 55 [8,2]
+ CRUSH rule 2 x 56 [6,5]
+ CRUSH rule 2 x 57 [5,8]
+ CRUSH rule 2 x 58 [2,8]
+ CRUSH rule 2 x 59 [5,2]
+ CRUSH rule 2 x 60 [5,2]
+ CRUSH rule 2 x 61 [5,6]
+ CRUSH rule 2 x 62 [7,2]
+ CRUSH rule 2 x 63 [5,6]
+ CRUSH rule 2 x 64 [5,2]
+ CRUSH rule 2 x 65 [7,5]
+ CRUSH rule 2 x 66 [5,6]
+ CRUSH rule 2 x 67 [5,2]
+ CRUSH rule 2 x 68 [2,5]
+ CRUSH rule 2 x 69 [5,2]
+ CRUSH rule 2 x 70 [7,2]
+ CRUSH rule 2 x 71 [2,8]
+ CRUSH rule 2 x 72 [6,2]
+ CRUSH rule 2 x 73 [2,7]
+ CRUSH rule 2 x 74 [2,7]
+ CRUSH rule 2 x 75 [5,2]
+ CRUSH rule 2 x 76 [5,2]
+ CRUSH rule 2 x 77 [7,2]
+ CRUSH rule 2 x 78 [2,5]
+ CRUSH rule 2 x 79 [5,2]
+ CRUSH rule 2 x 80 [2,5]
+ CRUSH rule 2 x 81 [2,5]
+ CRUSH rule 2 x 82 [7,2]
+ CRUSH rule 2 x 83 [2,6]
+ CRUSH rule 2 x 84 [7,2]
+ CRUSH rule 2 x 85 [5,8]
+ CRUSH rule 2 x 86 [2,6]
+ CRUSH rule 2 x 87 [2,7]
+ CRUSH rule 2 x 88 [2,6]
+ CRUSH rule 2 x 89 [5,2]
+ CRUSH rule 2 x 90 [6,5]
+ CRUSH rule 2 x 91 [5,8]
+ CRUSH rule 2 x 92 [2,8]
+ CRUSH rule 2 x 93 [7,5]
+ CRUSH rule 2 x 94 [2,5]
+ CRUSH rule 2 x 95 [7,5]
+ CRUSH rule 2 x 96 [5,6]
+ CRUSH rule 2 x 97 [8,5]
+ CRUSH rule 2 x 98 [2,7]
+ CRUSH rule 2 x 99 [2,7]
+ CRUSH rule 2 x 100 [2,7]
+ CRUSH rule 2 x 101 [5,7]
+ CRUSH rule 2 x 102 [5,2]
+ CRUSH rule 2 x 103 [5,7]
+ CRUSH rule 2 x 104 [7,5]
+ CRUSH rule 2 x 105 [2,5]
+ CRUSH rule 2 x 106 [2,6]
+ CRUSH rule 2 x 107 [5,2]
+ CRUSH rule 2 x 108 [7,2]
+ CRUSH rule 2 x 109 [2,5]
+ CRUSH rule 2 x 110 [5,2]
+ CRUSH rule 2 x 111 [2,5]
+ CRUSH rule 2 x 112 [2,6]
+ CRUSH rule 2 x 113 [6,2]
+ CRUSH rule 2 x 114 [7,5]
+ CRUSH rule 2 x 115 [8,2]
+ CRUSH rule 2 x 116 [2,6]
+ CRUSH rule 2 x 117 [7,5]
+ CRUSH rule 2 x 118 [2,5]
+ CRUSH rule 2 x 119 [5,6]
+ CRUSH rule 2 x 120 [2,5]
+ CRUSH rule 2 x 121 [2,7]
+ CRUSH rule 2 x 122 [8,5]
+ CRUSH rule 2 x 123 [2,5]
+ CRUSH rule 2 x 124 [5,2]
+ CRUSH rule 2 x 125 [2,7]
+ CRUSH rule 2 x 126 [5,2]
+ CRUSH rule 2 x 127 [5,6]
+ CRUSH rule 2 x 128 [5,6]
+ CRUSH rule 2 x 129 [2,5]
+ CRUSH rule 2 x 130 [5,8]
+ CRUSH rule 2 x 131 [2,5]
+ CRUSH rule 2 x 132 [2,5]
+ CRUSH rule 2 x 133 [5,6]
+ CRUSH rule 2 x 134 [2,8]
+ CRUSH rule 2 x 135 [5,6]
+ CRUSH rule 2 x 136 [2,5]
+ CRUSH rule 2 x 137 [7,5]
+ CRUSH rule 2 x 138 [8,5]
+ CRUSH rule 2 x 139 [5,2]
+ CRUSH rule 2 x 140 [2,6]
+ CRUSH rule 2 x 141 [6,2]
+ CRUSH rule 2 x 142 [5,2]
+ CRUSH rule 2 x 143 [5,8]
+ CRUSH rule 2 x 144 [8,2]
+ CRUSH rule 2 x 145 [8,5]
+ CRUSH rule 2 x 146 [2,6]
+ CRUSH rule 2 x 147 [2,8]
+ CRUSH rule 2 x 148 [5,2]
+ CRUSH rule 2 x 149 [5,8]
+ CRUSH rule 2 x 150 [2,6]
+ CRUSH rule 2 x 151 [5,6]
+ CRUSH rule 2 x 152 [8,5]
+ CRUSH rule 2 x 153 [8,5]
+ CRUSH rule 2 x 154 [5,2]
+ CRUSH rule 2 x 155 [5,7]
+ CRUSH rule 2 x 156 [5,2]
+ CRUSH rule 2 x 157 [5,2]
+ CRUSH rule 2 x 158 [2,8]
+ CRUSH rule 2 x 159 [7,2]
+ CRUSH rule 2 x 160 [2,8]
+ CRUSH rule 2 x 161 [2,5]
+ CRUSH rule 2 x 162 [2,6]
+ CRUSH rule 2 x 163 [5,6]
+ CRUSH rule 2 x 164 [7,2]
+ CRUSH rule 2 x 165 [7,2]
+ CRUSH rule 2 x 166 [2,5]
+ CRUSH rule 2 x 167 [2,7]
+ CRUSH rule 2 x 168 [5,2]
+ CRUSH rule 2 x 169 [2,6]
+ CRUSH rule 2 x 170 [2,5]
+ CRUSH rule 2 x 171 [7,5]
+ CRUSH rule 2 x 172 [2,7]
+ CRUSH rule 2 x 173 [8,5]
+ CRUSH rule 2 x 174 [2,5]
+ CRUSH rule 2 x 175 [6,2]
+ CRUSH rule 2 x 176 [5,2]
+ CRUSH rule 2 x 177 [5,2]
+ CRUSH rule 2 x 178 [5,2]
+ CRUSH rule 2 x 179 [5,2]
+ CRUSH rule 2 x 180 [5,8]
+ CRUSH rule 2 x 181 [6,2]
+ CRUSH rule 2 x 182 [8,5]
+ CRUSH rule 2 x 183 [7,5]
+ CRUSH rule 2 x 184 [5,7]
+ CRUSH rule 2 x 185 [6,2]
+ CRUSH rule 2 x 186 [2,5]
+ CRUSH rule 2 x 187 [2,6]
+ CRUSH rule 2 x 188 [2,8]
+ CRUSH rule 2 x 189 [2,7]
+ CRUSH rule 2 x 190 [5,2]
+ CRUSH rule 2 x 191 [7,2]
+ CRUSH rule 2 x 192 [5,2]
+ CRUSH rule 2 x 193 [5,2]
+ CRUSH rule 2 x 194 [2,5]
+ CRUSH rule 2 x 195 [6,5]
+ CRUSH rule 2 x 196 [6,2]
+ CRUSH rule 2 x 197 [6,5]
+ CRUSH rule 2 x 198 [2,5]
+ CRUSH rule 2 x 199 [2,5]
+ CRUSH rule 2 x 200 [2,5]
+ CRUSH rule 2 x 201 [7,2]
+ CRUSH rule 2 x 202 [6,5]
+ CRUSH rule 2 x 203 [5,8]
+ CRUSH rule 2 x 204 [2,5]
+ CRUSH rule 2 x 205 [2,7]
+ CRUSH rule 2 x 206 [2,7]
+ CRUSH rule 2 x 207 [5,2]
+ CRUSH rule 2 x 208 [7,2]
+ CRUSH rule 2 x 209 [2,8]
+ CRUSH rule 2 x 210 [2,5]
+ CRUSH rule 2 x 211 [5,2]
+ CRUSH rule 2 x 212 [7,5]
+ CRUSH rule 2 x 213 [8,5]
+ CRUSH rule 2 x 214 [5,8]
+ CRUSH rule 2 x 215 [8,2]
+ CRUSH rule 2 x 216 [5,2]
+ CRUSH rule 2 x 217 [2,7]
+ CRUSH rule 2 x 218 [2,7]
+ CRUSH rule 2 x 219 [5,8]
+ CRUSH rule 2 x 220 [5,7]
+ CRUSH rule 2 x 221 [5,6]
+ CRUSH rule 2 x 222 [6,5]
+ CRUSH rule 2 x 223 [2,5]
+ CRUSH rule 2 x 224 [2,5]
+ CRUSH rule 2 x 225 [8,2]
+ CRUSH rule 2 x 226 [7,2]
+ CRUSH rule 2 x 227 [5,2]
+ CRUSH rule 2 x 228 [5,6]
+ CRUSH rule 2 x 229 [5,8]
+ CRUSH rule 2 x 230 [5,7]
+ CRUSH rule 2 x 231 [5,7]
+ CRUSH rule 2 x 232 [2,7]
+ CRUSH rule 2 x 233 [5,7]
+ CRUSH rule 2 x 234 [2,5]
+ CRUSH rule 2 x 235 [5,8]
+ CRUSH rule 2 x 236 [5,2]
+ CRUSH rule 2 x 237 [5,7]
+ CRUSH rule 2 x 238 [5,2]
+ CRUSH rule 2 x 239 [8,5]
+ CRUSH rule 2 x 240 [5,7]
+ CRUSH rule 2 x 241 [5,2]
+ CRUSH rule 2 x 242 [5,2]
+ CRUSH rule 2 x 243 [5,7]
+ CRUSH rule 2 x 244 [5,6]
+ CRUSH rule 2 x 245 [7,2]
+ CRUSH rule 2 x 246 [2,5]
+ CRUSH rule 2 x 247 [6,2]
+ CRUSH rule 2 x 248 [8,2]
+ CRUSH rule 2 x 249 [2,5]
+ CRUSH rule 2 x 250 [2,5]
+ CRUSH rule 2 x 251 [2,5]
+ CRUSH rule 2 x 252 [5,7]
+ CRUSH rule 2 x 253 [5,2]
+ CRUSH rule 2 x 254 [5,2]
+ CRUSH rule 2 x 255 [2,7]
+ CRUSH rule 2 x 256 [5,7]
+ CRUSH rule 2 x 257 [2,8]
+ CRUSH rule 2 x 258 [5,2]
+ CRUSH rule 2 x 259 [5,6]
+ CRUSH rule 2 x 260 [5,6]
+ CRUSH rule 2 x 261 [8,5]
+ CRUSH rule 2 x 262 [5,6]
+ CRUSH rule 2 x 263 [6,2]
+ CRUSH rule 2 x 264 [5,6]
+ CRUSH rule 2 x 265 [8,5]
+ CRUSH rule 2 x 266 [8,2]
+ CRUSH rule 2 x 267 [2,5]
+ CRUSH rule 2 x 268 [2,7]
+ CRUSH rule 2 x 269 [2,8]
+ CRUSH rule 2 x 270 [5,2]
+ CRUSH rule 2 x 271 [7,5]
+ CRUSH rule 2 x 272 [2,8]
+ CRUSH rule 2 x 273 [5,2]
+ CRUSH rule 2 x 274 [6,5]
+ CRUSH rule 2 x 275 [5,7]
+ CRUSH rule 2 x 276 [7,2]
+ CRUSH rule 2 x 277 [6,5]
+ CRUSH rule 2 x 278 [6,2]
+ CRUSH rule 2 x 279 [8,5]
+ CRUSH rule 2 x 280 [2,6]
+ CRUSH rule 2 x 281 [8,2]
+ CRUSH rule 2 x 282 [5,2]
+ CRUSH rule 2 x 283 [8,2]
+ CRUSH rule 2 x 284 [6,5]
+ CRUSH rule 2 x 285 [5,7]
+ CRUSH rule 2 x 286 [2,6]
+ CRUSH rule 2 x 287 [2,5]
+ CRUSH rule 2 x 288 [8,2]
+ CRUSH rule 2 x 289 [5,6]
+ CRUSH rule 2 x 290 [2,5]
+ CRUSH rule 2 x 291 [2,5]
+ CRUSH rule 2 x 292 [8,2]
+ CRUSH rule 2 x 293 [6,2]
+ CRUSH rule 2 x 294 [7,5]
+ CRUSH rule 2 x 295 [5,8]
+ CRUSH rule 2 x 296 [5,2]
+ CRUSH rule 2 x 297 [6,2]
+ CRUSH rule 2 x 298 [2,5]
+ CRUSH rule 2 x 299 [2,8]
+ CRUSH rule 2 x 300 [8,5]
+ CRUSH rule 2 x 301 [2,8]
+ CRUSH rule 2 x 302 [5,2]
+ CRUSH rule 2 x 303 [7,5]
+ CRUSH rule 2 x 304 [2,7]
+ CRUSH rule 2 x 305 [5,8]
+ CRUSH rule 2 x 306 [2,7]
+ CRUSH rule 2 x 307 [2,7]
+ CRUSH rule 2 x 308 [2,8]
+ CRUSH rule 2 x 309 [7,5]
+ CRUSH rule 2 x 310 [5,2]
+ CRUSH rule 2 x 311 [5,6]
+ CRUSH rule 2 x 312 [2,6]
+ CRUSH rule 2 x 313 [5,2]
+ CRUSH rule 2 x 314 [5,2]
+ CRUSH rule 2 x 315 [2,5]
+ CRUSH rule 2 x 316 [6,5]
+ CRUSH rule 2 x 317 [2,6]
+ CRUSH rule 2 x 318 [8,2]
+ CRUSH rule 2 x 319 [5,2]
+ CRUSH rule 2 x 320 [5,7]
+ CRUSH rule 2 x 321 [2,5]
+ CRUSH rule 2 x 322 [2,7]
+ CRUSH rule 2 x 323 [5,7]
+ CRUSH rule 2 x 324 [7,2]
+ CRUSH rule 2 x 325 [5,6]
+ CRUSH rule 2 x 326 [5,2]
+ CRUSH rule 2 x 327 [2,6]
+ CRUSH rule 2 x 328 [7,5]
+ CRUSH rule 2 x 329 [5,6]
+ CRUSH rule 2 x 330 [5,7]
+ CRUSH rule 2 x 331 [2,6]
+ CRUSH rule 2 x 332 [2,5]
+ CRUSH rule 2 x 333 [6,5]
+ CRUSH rule 2 x 334 [8,5]
+ CRUSH rule 2 x 335 [7,2]
+ CRUSH rule 2 x 336 [5,6]
+ CRUSH rule 2 x 337 [7,2]
+ CRUSH rule 2 x 338 [5,6]
+ CRUSH rule 2 x 339 [7,5]
+ CRUSH rule 2 x 340 [2,8]
+ CRUSH rule 2 x 341 [5,2]
+ CRUSH rule 2 x 342 [2,7]
+ CRUSH rule 2 x 343 [6,5]
+ CRUSH rule 2 x 344 [6,2]
+ CRUSH rule 2 x 345 [5,7]
+ CRUSH rule 2 x 346 [8,2]
+ CRUSH rule 2 x 347 [5,2]
+ CRUSH rule 2 x 348 [8,2]
+ CRUSH rule 2 x 349 [2,6]
+ CRUSH rule 2 x 350 [8,5]
+ CRUSH rule 2 x 351 [5,6]
+ CRUSH rule 2 x 352 [2,8]
+ CRUSH rule 2 x 353 [6,5]
+ CRUSH rule 2 x 354 [2,5]
+ CRUSH rule 2 x 355 [5,8]
+ CRUSH rule 2 x 356 [5,2]
+ CRUSH rule 2 x 357 [6,2]
+ CRUSH rule 2 x 358 [2,8]
+ CRUSH rule 2 x 359 [6,2]
+ CRUSH rule 2 x 360 [5,2]
+ CRUSH rule 2 x 361 [8,5]
+ CRUSH rule 2 x 362 [5,2]
+ CRUSH rule 2 x 363 [5,2]
+ CRUSH rule 2 x 364 [2,5]
+ CRUSH rule 2 x 365 [6,5]
+ CRUSH rule 2 x 366 [7,2]
+ CRUSH rule 2 x 367 [5,2]
+ CRUSH rule 2 x 368 [7,5]
+ CRUSH rule 2 x 369 [5,7]
+ CRUSH rule 2 x 370 [8,2]
+ CRUSH rule 2 x 371 [2,5]
+ CRUSH rule 2 x 372 [5,2]
+ CRUSH rule 2 x 373 [2,6]
+ CRUSH rule 2 x 374 [5,8]
+ CRUSH rule 2 x 375 [6,5]
+ CRUSH rule 2 x 376 [7,2]
+ CRUSH rule 2 x 377 [2,5]
+ CRUSH rule 2 x 378 [2,8]
+ CRUSH rule 2 x 379 [8,5]
+ CRUSH rule 2 x 380 [2,5]
+ CRUSH rule 2 x 381 [2,5]
+ CRUSH rule 2 x 382 [2,5]
+ CRUSH rule 2 x 383 [5,6]
+ CRUSH rule 2 x 384 [7,2]
+ CRUSH rule 2 x 385 [7,5]
+ CRUSH rule 2 x 386 [2,5]
+ CRUSH rule 2 x 387 [2,5]
+ CRUSH rule 2 x 388 [5,2]
+ CRUSH rule 2 x 389 [2,5]
+ CRUSH rule 2 x 390 [5,6]
+ CRUSH rule 2 x 391 [5,6]
+ CRUSH rule 2 x 392 [2,8]
+ CRUSH rule 2 x 393 [5,2]
+ CRUSH rule 2 x 394 [5,7]
+ CRUSH rule 2 x 395 [5,2]
+ CRUSH rule 2 x 396 [5,2]
+ CRUSH rule 2 x 397 [2,5]
+ CRUSH rule 2 x 398 [2,5]
+ CRUSH rule 2 x 399 [8,5]
+ CRUSH rule 2 x 400 [8,2]
+ CRUSH rule 2 x 401 [2,5]
+ CRUSH rule 2 x 402 [7,5]
+ CRUSH rule 2 x 403 [2,5]
+ CRUSH rule 2 x 404 [5,2]
+ CRUSH rule 2 x 405 [6,5]
+ CRUSH rule 2 x 406 [2,6]
+ CRUSH rule 2 x 407 [2,8]
+ CRUSH rule 2 x 408 [5,2]
+ CRUSH rule 2 x 409 [7,5]
+ CRUSH rule 2 x 410 [8,5]
+ CRUSH rule 2 x 411 [2,8]
+ CRUSH rule 2 x 412 [2,5]
+ CRUSH rule 2 x 413 [5,2]
+ CRUSH rule 2 x 414 [5,2]
+ CRUSH rule 2 x 415 [2,6]
+ CRUSH rule 2 x 416 [2,6]
+ CRUSH rule 2 x 417 [8,2]
+ CRUSH rule 2 x 418 [7,2]
+ CRUSH rule 2 x 419 [8,5]
+ CRUSH rule 2 x 420 [2,5]
+ CRUSH rule 2 x 421 [8,5]
+ CRUSH rule 2 x 422 [6,5]
+ CRUSH rule 2 x 423 [2,5]
+ CRUSH rule 2 x 424 [8,5]
+ CRUSH rule 2 x 425 [2,5]
+ CRUSH rule 2 x 426 [6,2]
+ CRUSH rule 2 x 427 [2,7]
+ CRUSH rule 2 x 428 [5,7]
+ CRUSH rule 2 x 429 [5,6]
+ CRUSH rule 2 x 430 [5,6]
+ CRUSH rule 2 x 431 [5,2]
+ CRUSH rule 2 x 432 [7,2]
+ CRUSH rule 2 x 433 [6,5]
+ CRUSH rule 2 x 434 [5,2]
+ CRUSH rule 2 x 435 [2,5]
+ CRUSH rule 2 x 436 [5,2]
+ CRUSH rule 2 x 437 [7,5]
+ CRUSH rule 2 x 438 [2,5]
+ CRUSH rule 2 x 439 [2,5]
+ CRUSH rule 2 x 440 [2,7]
+ CRUSH rule 2 x 441 [5,7]
+ CRUSH rule 2 x 442 [2,5]
+ CRUSH rule 2 x 443 [6,2]
+ CRUSH rule 2 x 444 [7,2]
+ CRUSH rule 2 x 445 [6,5]
+ CRUSH rule 2 x 446 [5,2]
+ CRUSH rule 2 x 447 [2,5]
+ CRUSH rule 2 x 448 [7,2]
+ CRUSH rule 2 x 449 [7,5]
+ CRUSH rule 2 x 450 [5,2]
+ CRUSH rule 2 x 451 [6,5]
+ CRUSH rule 2 x 452 [8,5]
+ CRUSH rule 2 x 453 [6,5]
+ CRUSH rule 2 x 454 [6,5]
+ CRUSH rule 2 x 455 [2,7]
+ CRUSH rule 2 x 456 [6,2]
+ CRUSH rule 2 x 457 [7,2]
+ CRUSH rule 2 x 458 [2,8]
+ CRUSH rule 2 x 459 [2,7]
+ CRUSH rule 2 x 460 [6,5]
+ CRUSH rule 2 x 461 [6,5]
+ CRUSH rule 2 x 462 [8,2]
+ CRUSH rule 2 x 463 [6,2]
+ CRUSH rule 2 x 464 [7,5]
+ CRUSH rule 2 x 465 [7,2]
+ CRUSH rule 2 x 466 [5,8]
+ CRUSH rule 2 x 467 [6,5]
+ CRUSH rule 2 x 468 [7,2]
+ CRUSH rule 2 x 469 [7,2]
+ CRUSH rule 2 x 470 [5,2]
+ CRUSH rule 2 x 471 [2,7]
+ CRUSH rule 2 x 472 [5,2]
+ CRUSH rule 2 x 473 [2,5]
+ CRUSH rule 2 x 474 [6,2]
+ CRUSH rule 2 x 475 [6,2]
+ CRUSH rule 2 x 476 [5,6]
+ CRUSH rule 2 x 477 [5,8]
+ CRUSH rule 2 x 478 [6,2]
+ CRUSH rule 2 x 479 [2,5]
+ CRUSH rule 2 x 480 [2,8]
+ CRUSH rule 2 x 481 [2,5]
+ CRUSH rule 2 x 482 [5,7]
+ CRUSH rule 2 x 483 [2,6]
+ CRUSH rule 2 x 484 [2,7]
+ CRUSH rule 2 x 485 [5,7]
+ CRUSH rule 2 x 486 [5,2]
+ CRUSH rule 2 x 487 [5,2]
+ CRUSH rule 2 x 488 [5,7]
+ CRUSH rule 2 x 489 [2,8]
+ CRUSH rule 2 x 490 [6,5]
+ CRUSH rule 2 x 491 [2,7]
+ CRUSH rule 2 x 492 [6,5]
+ CRUSH rule 2 x 493 [2,7]
+ CRUSH rule 2 x 494 [2,7]
+ CRUSH rule 2 x 495 [5,2]
+ CRUSH rule 2 x 496 [7,5]
+ CRUSH rule 2 x 497 [5,7]
+ CRUSH rule 2 x 498 [2,5]
+ CRUSH rule 2 x 499 [8,5]
+ CRUSH rule 2 x 500 [5,6]
+ CRUSH rule 2 x 501 [2,7]
+ CRUSH rule 2 x 502 [7,2]
+ CRUSH rule 2 x 503 [2,5]
+ CRUSH rule 2 x 504 [5,6]
+ CRUSH rule 2 x 505 [2,7]
+ CRUSH rule 2 x 506 [5,2]
+ CRUSH rule 2 x 507 [6,2]
+ CRUSH rule 2 x 508 [2,5]
+ CRUSH rule 2 x 509 [7,5]
+ CRUSH rule 2 x 510 [6,2]
+ CRUSH rule 2 x 511 [5,8]
+ CRUSH rule 2 x 512 [7,2]
+ CRUSH rule 2 x 513 [7,2]
+ CRUSH rule 2 x 514 [5,6]
+ CRUSH rule 2 x 515 [8,5]
+ CRUSH rule 2 x 516 [5,2]
+ CRUSH rule 2 x 517 [7,2]
+ CRUSH rule 2 x 518 [5,6]
+ CRUSH rule 2 x 519 [7,5]
+ CRUSH rule 2 x 520 [2,6]
+ CRUSH rule 2 x 521 [8,2]
+ CRUSH rule 2 x 522 [6,2]
+ CRUSH rule 2 x 523 [5,2]
+ CRUSH rule 2 x 524 [2,5]
+ CRUSH rule 2 x 525 [2,5]
+ CRUSH rule 2 x 526 [2,5]
+ CRUSH rule 2 x 527 [2,5]
+ CRUSH rule 2 x 528 [5,2]
+ CRUSH rule 2 x 529 [5,7]
+ CRUSH rule 2 x 530 [6,5]
+ CRUSH rule 2 x 531 [6,2]
+ CRUSH rule 2 x 532 [6,5]
+ CRUSH rule 2 x 533 [5,6]
+ CRUSH rule 2 x 534 [7,5]
+ CRUSH rule 2 x 535 [8,2]
+ CRUSH rule 2 x 536 [6,2]
+ CRUSH rule 2 x 537 [5,7]
+ CRUSH rule 2 x 538 [6,5]
+ CRUSH rule 2 x 539 [8,5]
+ CRUSH rule 2 x 540 [2,6]
+ CRUSH rule 2 x 541 [2,5]
+ CRUSH rule 2 x 542 [5,2]
+ CRUSH rule 2 x 543 [6,2]
+ CRUSH rule 2 x 544 [5,7]
+ CRUSH rule 2 x 545 [5,7]
+ CRUSH rule 2 x 546 [6,2]
+ CRUSH rule 2 x 547 [8,2]
+ CRUSH rule 2 x 548 [5,2]
+ CRUSH rule 2 x 549 [5,8]
+ CRUSH rule 2 x 550 [2,5]
+ CRUSH rule 2 x 551 [7,5]
+ CRUSH rule 2 x 552 [5,8]
+ CRUSH rule 2 x 553 [5,2]
+ CRUSH rule 2 x 554 [2,8]
+ CRUSH rule 2 x 555 [5,2]
+ CRUSH rule 2 x 556 [5,6]
+ CRUSH rule 2 x 557 [7,5]
+ CRUSH rule 2 x 558 [5,2]
+ CRUSH rule 2 x 559 [5,2]
+ CRUSH rule 2 x 560 [8,5]
+ CRUSH rule 2 x 561 [6,5]
+ CRUSH rule 2 x 562 [5,2]
+ CRUSH rule 2 x 563 [2,6]
+ CRUSH rule 2 x 564 [5,2]
+ CRUSH rule 2 x 565 [5,6]
+ CRUSH rule 2 x 566 [5,7]
+ CRUSH rule 2 x 567 [5,6]
+ CRUSH rule 2 x 568 [7,5]
+ CRUSH rule 2 x 569 [5,2]
+ CRUSH rule 2 x 570 [2,5]
+ CRUSH rule 2 x 571 [5,7]
+ CRUSH rule 2 x 572 [5,2]
+ CRUSH rule 2 x 573 [5,2]
+ CRUSH rule 2 x 574 [2,5]
+ CRUSH rule 2 x 575 [8,2]
+ CRUSH rule 2 x 576 [5,6]
+ CRUSH rule 2 x 577 [8,2]
+ CRUSH rule 2 x 578 [6,2]
+ CRUSH rule 2 x 579 [5,2]
+ CRUSH rule 2 x 580 [5,2]
+ CRUSH rule 2 x 581 [7,2]
+ CRUSH rule 2 x 582 [2,8]
+ CRUSH rule 2 x 583 [6,2]
+ CRUSH rule 2 x 584 [8,2]
+ CRUSH rule 2 x 585 [7,2]
+ CRUSH rule 2 x 586 [2,7]
+ CRUSH rule 2 x 587 [2,5]
+ CRUSH rule 2 x 588 [5,7]
+ CRUSH rule 2 x 589 [7,2]
+ CRUSH rule 2 x 590 [6,2]
+ CRUSH rule 2 x 591 [5,2]
+ CRUSH rule 2 x 592 [2,5]
+ CRUSH rule 2 x 593 [2,8]
+ CRUSH rule 2 x 594 [2,7]
+ CRUSH rule 2 x 595 [7,2]
+ CRUSH rule 2 x 596 [5,2]
+ CRUSH rule 2 x 597 [5,2]
+ CRUSH rule 2 x 598 [5,2]
+ CRUSH rule 2 x 599 [5,2]
+ CRUSH rule 2 x 600 [7,2]
+ CRUSH rule 2 x 601 [2,7]
+ CRUSH rule 2 x 602 [5,7]
+ CRUSH rule 2 x 603 [5,2]
+ CRUSH rule 2 x 604 [7,5]
+ CRUSH rule 2 x 605 [5,2]
+ CRUSH rule 2 x 606 [2,7]
+ CRUSH rule 2 x 607 [2,5]
+ CRUSH rule 2 x 608 [5,2]
+ CRUSH rule 2 x 609 [5,2]
+ CRUSH rule 2 x 610 [5,7]
+ CRUSH rule 2 x 611 [2,8]
+ CRUSH rule 2 x 612 [2,6]
+ CRUSH rule 2 x 613 [7,2]
+ CRUSH rule 2 x 614 [7,2]
+ CRUSH rule 2 x 615 [6,2]
+ CRUSH rule 2 x 616 [2,8]
+ CRUSH rule 2 x 617 [6,2]
+ CRUSH rule 2 x 618 [7,5]
+ CRUSH rule 2 x 619 [5,2]
+ CRUSH rule 2 x 620 [5,2]
+ CRUSH rule 2 x 621 [5,8]
+ CRUSH rule 2 x 622 [2,5]
+ CRUSH rule 2 x 623 [2,6]
+ CRUSH rule 2 x 624 [5,2]
+ CRUSH rule 2 x 625 [2,5]
+ CRUSH rule 2 x 626 [7,2]
+ CRUSH rule 2 x 627 [2,7]
+ CRUSH rule 2 x 628 [8,2]
+ CRUSH rule 2 x 629 [2,6]
+ CRUSH rule 2 x 630 [2,7]
+ CRUSH rule 2 x 631 [2,7]
+ CRUSH rule 2 x 632 [7,2]
+ CRUSH rule 2 x 633 [8,5]
+ CRUSH rule 2 x 634 [2,5]
+ CRUSH rule 2 x 635 [5,6]
+ CRUSH rule 2 x 636 [2,5]
+ CRUSH rule 2 x 637 [5,2]
+ CRUSH rule 2 x 638 [6,2]
+ CRUSH rule 2 x 639 [5,2]
+ CRUSH rule 2 x 640 [5,2]
+ CRUSH rule 2 x 641 [7,2]
+ CRUSH rule 2 x 642 [2,7]
+ CRUSH rule 2 x 643 [5,2]
+ CRUSH rule 2 x 644 [8,2]
+ CRUSH rule 2 x 645 [5,7]
+ CRUSH rule 2 x 646 [8,2]
+ CRUSH rule 2 x 647 [7,2]
+ CRUSH rule 2 x 648 [2,6]
+ CRUSH rule 2 x 649 [5,7]
+ CRUSH rule 2 x 650 [7,5]
+ CRUSH rule 2 x 651 [5,7]
+ CRUSH rule 2 x 652 [5,7]
+ CRUSH rule 2 x 653 [8,5]
+ CRUSH rule 2 x 654 [7,5]
+ CRUSH rule 2 x 655 [2,5]
+ CRUSH rule 2 x 656 [5,7]
+ CRUSH rule 2 x 657 [6,2]
+ CRUSH rule 2 x 658 [5,6]
+ CRUSH rule 2 x 659 [5,6]
+ CRUSH rule 2 x 660 [7,5]
+ CRUSH rule 2 x 661 [2,8]
+ CRUSH rule 2 x 662 [5,2]
+ CRUSH rule 2 x 663 [2,5]
+ CRUSH rule 2 x 664 [2,5]
+ CRUSH rule 2 x 665 [5,7]
+ CRUSH rule 2 x 666 [2,8]
+ CRUSH rule 2 x 667 [2,5]
+ CRUSH rule 2 x 668 [5,7]
+ CRUSH rule 2 x 669 [6,5]
+ CRUSH rule 2 x 670 [5,2]
+ CRUSH rule 2 x 671 [2,7]
+ CRUSH rule 2 x 672 [5,2]
+ CRUSH rule 2 x 673 [5,2]
+ CRUSH rule 2 x 674 [5,2]
+ CRUSH rule 2 x 675 [2,8]
+ CRUSH rule 2 x 676 [2,5]
+ CRUSH rule 2 x 677 [5,2]
+ CRUSH rule 2 x 678 [2,5]
+ CRUSH rule 2 x 679 [6,2]
+ CRUSH rule 2 x 680 [2,5]
+ CRUSH rule 2 x 681 [5,7]
+ CRUSH rule 2 x 682 [2,5]
+ CRUSH rule 2 x 683 [2,5]
+ CRUSH rule 2 x 684 [7,2]
+ CRUSH rule 2 x 685 [7,2]
+ CRUSH rule 2 x 686 [2,5]
+ CRUSH rule 2 x 687 [5,6]
+ CRUSH rule 2 x 688 [5,7]
+ CRUSH rule 2 x 689 [6,5]
+ CRUSH rule 2 x 690 [8,2]
+ CRUSH rule 2 x 691 [5,2]
+ CRUSH rule 2 x 692 [7,2]
+ CRUSH rule 2 x 693 [6,5]
+ CRUSH rule 2 x 694 [6,5]
+ CRUSH rule 2 x 695 [2,8]
+ CRUSH rule 2 x 696 [2,5]
+ CRUSH rule 2 x 697 [6,2]
+ CRUSH rule 2 x 698 [6,2]
+ CRUSH rule 2 x 699 [2,6]
+ CRUSH rule 2 x 700 [2,5]
+ CRUSH rule 2 x 701 [5,2]
+ CRUSH rule 2 x 702 [5,2]
+ CRUSH rule 2 x 703 [8,5]
+ CRUSH rule 2 x 704 [2,5]
+ CRUSH rule 2 x 705 [8,2]
+ CRUSH rule 2 x 706 [2,5]
+ CRUSH rule 2 x 707 [7,5]
+ CRUSH rule 2 x 708 [5,7]
+ CRUSH rule 2 x 709 [6,5]
+ CRUSH rule 2 x 710 [8,5]
+ CRUSH rule 2 x 711 [2,5]
+ CRUSH rule 2 x 712 [2,5]
+ CRUSH rule 2 x 713 [6,5]
+ CRUSH rule 2 x 714 [5,2]
+ CRUSH rule 2 x 715 [2,5]
+ CRUSH rule 2 x 716 [5,6]
+ CRUSH rule 2 x 717 [8,2]
+ CRUSH rule 2 x 718 [5,7]
+ CRUSH rule 2 x 719 [2,6]
+ CRUSH rule 2 x 720 [6,2]
+ CRUSH rule 2 x 721 [5,7]
+ CRUSH rule 2 x 722 [5,7]
+ CRUSH rule 2 x 723 [5,2]
+ CRUSH rule 2 x 724 [2,6]
+ CRUSH rule 2 x 725 [2,5]
+ CRUSH rule 2 x 726 [5,8]
+ CRUSH rule 2 x 727 [5,6]
+ CRUSH rule 2 x 728 [2,7]
+ CRUSH rule 2 x 729 [5,6]
+ CRUSH rule 2 x 730 [5,7]
+ CRUSH rule 2 x 731 [5,2]
+ CRUSH rule 2 x 732 [2,5]
+ CRUSH rule 2 x 733 [5,7]
+ CRUSH rule 2 x 734 [6,5]
+ CRUSH rule 2 x 735 [5,8]
+ CRUSH rule 2 x 736 [5,8]
+ CRUSH rule 2 x 737 [2,6]
+ CRUSH rule 2 x 738 [5,2]
+ CRUSH rule 2 x 739 [2,7]
+ CRUSH rule 2 x 740 [2,8]
+ CRUSH rule 2 x 741 [7,2]
+ CRUSH rule 2 x 742 [8,2]
+ CRUSH rule 2 x 743 [7,2]
+ CRUSH rule 2 x 744 [5,7]
+ CRUSH rule 2 x 745 [5,2]
+ CRUSH rule 2 x 746 [5,2]
+ CRUSH rule 2 x 747 [6,2]
+ CRUSH rule 2 x 748 [2,7]
+ CRUSH rule 2 x 749 [5,8]
+ CRUSH rule 2 x 750 [2,6]
+ CRUSH rule 2 x 751 [2,8]
+ CRUSH rule 2 x 752 [8,2]
+ CRUSH rule 2 x 753 [7,5]
+ CRUSH rule 2 x 754 [8,5]
+ CRUSH rule 2 x 755 [2,6]
+ CRUSH rule 2 x 756 [5,6]
+ CRUSH rule 2 x 757 [8,2]
+ CRUSH rule 2 x 758 [6,2]
+ CRUSH rule 2 x 759 [8,5]
+ CRUSH rule 2 x 760 [2,5]
+ CRUSH rule 2 x 761 [5,2]
+ CRUSH rule 2 x 762 [2,7]
+ CRUSH rule 2 x 763 [8,5]
+ CRUSH rule 2 x 764 [2,7]
+ CRUSH rule 2 x 765 [6,5]
+ CRUSH rule 2 x 766 [8,5]
+ CRUSH rule 2 x 767 [2,8]
+ CRUSH rule 2 x 768 [8,5]
+ CRUSH rule 2 x 769 [6,2]
+ CRUSH rule 2 x 770 [6,2]
+ CRUSH rule 2 x 771 [7,2]
+ CRUSH rule 2 x 772 [8,5]
+ CRUSH rule 2 x 773 [5,2]
+ CRUSH rule 2 x 774 [5,6]
+ CRUSH rule 2 x 775 [6,5]
+ CRUSH rule 2 x 776 [7,2]
+ CRUSH rule 2 x 777 [5,2]
+ CRUSH rule 2 x 778 [2,8]
+ CRUSH rule 2 x 779 [2,7]
+ CRUSH rule 2 x 780 [2,5]
+ CRUSH rule 2 x 781 [6,5]
+ CRUSH rule 2 x 782 [5,2]
+ CRUSH rule 2 x 783 [7,2]
+ CRUSH rule 2 x 784 [2,5]
+ CRUSH rule 2 x 785 [6,2]
+ CRUSH rule 2 x 786 [7,5]
+ CRUSH rule 2 x 787 [2,6]
+ CRUSH rule 2 x 788 [6,2]
+ CRUSH rule 2 x 789 [2,5]
+ CRUSH rule 2 x 790 [8,5]
+ CRUSH rule 2 x 791 [5,8]
+ CRUSH rule 2 x 792 [5,8]
+ CRUSH rule 2 x 793 [6,2]
+ CRUSH rule 2 x 794 [2,6]
+ CRUSH rule 2 x 795 [2,5]
+ CRUSH rule 2 x 796 [5,7]
+ CRUSH rule 2 x 797 [2,5]
+ CRUSH rule 2 x 798 [6,2]
+ CRUSH rule 2 x 799 [5,2]
+ CRUSH rule 2 x 800 [5,2]
+ CRUSH rule 2 x 801 [5,6]
+ CRUSH rule 2 x 802 [2,8]
+ CRUSH rule 2 x 803 [2,5]
+ CRUSH rule 2 x 804 [6,2]
+ CRUSH rule 2 x 805 [5,6]
+ CRUSH rule 2 x 806 [2,5]
+ CRUSH rule 2 x 807 [5,7]
+ CRUSH rule 2 x 808 [5,6]
+ CRUSH rule 2 x 809 [2,5]
+ CRUSH rule 2 x 810 [5,7]
+ CRUSH rule 2 x 811 [8,5]
+ CRUSH rule 2 x 812 [8,5]
+ CRUSH rule 2 x 813 [6,5]
+ CRUSH rule 2 x 814 [5,6]
+ CRUSH rule 2 x 815 [5,2]
+ CRUSH rule 2 x 816 [2,7]
+ CRUSH rule 2 x 817 [5,8]
+ CRUSH rule 2 x 818 [5,2]
+ CRUSH rule 2 x 819 [5,2]
+ CRUSH rule 2 x 820 [5,6]
+ CRUSH rule 2 x 821 [5,6]
+ CRUSH rule 2 x 822 [2,5]
+ CRUSH rule 2 x 823 [5,8]
+ CRUSH rule 2 x 824 [5,7]
+ CRUSH rule 2 x 825 [2,8]
+ CRUSH rule 2 x 826 [7,2]
+ CRUSH rule 2 x 827 [2,8]
+ CRUSH rule 2 x 828 [2,5]
+ CRUSH rule 2 x 829 [5,6]
+ CRUSH rule 2 x 830 [2,5]
+ CRUSH rule 2 x 831 [2,6]
+ CRUSH rule 2 x 832 [5,7]
+ CRUSH rule 2 x 833 [2,7]
+ CRUSH rule 2 x 834 [5,2]
+ CRUSH rule 2 x 835 [8,5]
+ CRUSH rule 2 x 836 [5,7]
+ CRUSH rule 2 x 837 [6,5]
+ CRUSH rule 2 x 838 [6,2]
+ CRUSH rule 2 x 839 [5,2]
+ CRUSH rule 2 x 840 [7,5]
+ CRUSH rule 2 x 841 [5,8]
+ CRUSH rule 2 x 842 [2,5]
+ CRUSH rule 2 x 843 [6,5]
+ CRUSH rule 2 x 844 [5,8]
+ CRUSH rule 2 x 845 [5,8]
+ CRUSH rule 2 x 846 [5,2]
+ CRUSH rule 2 x 847 [2,8]
+ CRUSH rule 2 x 848 [2,6]
+ CRUSH rule 2 x 849 [5,6]
+ CRUSH rule 2 x 850 [2,5]
+ CRUSH rule 2 x 851 [6,5]
+ CRUSH rule 2 x 852 [7,5]
+ CRUSH rule 2 x 853 [6,2]
+ CRUSH rule 2 x 854 [7,2]
+ CRUSH rule 2 x 855 [5,7]
+ CRUSH rule 2 x 856 [6,5]
+ CRUSH rule 2 x 857 [8,5]
+ CRUSH rule 2 x 858 [6,5]
+ CRUSH rule 2 x 859 [6,2]
+ CRUSH rule 2 x 860 [5,2]
+ CRUSH rule 2 x 861 [8,5]
+ CRUSH rule 2 x 862 [6,2]
+ CRUSH rule 2 x 863 [8,2]
+ CRUSH rule 2 x 864 [5,6]
+ CRUSH rule 2 x 865 [8,2]
+ CRUSH rule 2 x 866 [5,6]
+ CRUSH rule 2 x 867 [6,5]
+ CRUSH rule 2 x 868 [6,5]
+ CRUSH rule 2 x 869 [8,5]
+ CRUSH rule 2 x 870 [2,5]
+ CRUSH rule 2 x 871 [5,2]
+ CRUSH rule 2 x 872 [5,2]
+ CRUSH rule 2 x 873 [5,6]
+ CRUSH rule 2 x 874 [2,6]
+ CRUSH rule 2 x 875 [2,6]
+ CRUSH rule 2 x 876 [5,8]
+ CRUSH rule 2 x 877 [6,5]
+ CRUSH rule 2 x 878 [5,2]
+ CRUSH rule 2 x 879 [7,5]
+ CRUSH rule 2 x 880 [5,2]
+ CRUSH rule 2 x 881 [5,6]
+ CRUSH rule 2 x 882 [5,2]
+ CRUSH rule 2 x 883 [2,5]
+ CRUSH rule 2 x 884 [6,2]
+ CRUSH rule 2 x 885 [5,2]
+ CRUSH rule 2 x 886 [5,6]
+ CRUSH rule 2 x 887 [7,5]
+ CRUSH rule 2 x 888 [6,2]
+ CRUSH rule 2 x 889 [2,6]
+ CRUSH rule 2 x 890 [7,2]
+ CRUSH rule 2 x 891 [2,8]
+ CRUSH rule 2 x 892 [6,2]
+ CRUSH rule 2 x 893 [2,5]
+ CRUSH rule 2 x 894 [7,5]
+ CRUSH rule 2 x 895 [5,2]
+ CRUSH rule 2 x 896 [2,8]
+ CRUSH rule 2 x 897 [5,2]
+ CRUSH rule 2 x 898 [2,5]
+ CRUSH rule 2 x 899 [2,7]
+ CRUSH rule 2 x 900 [5,2]
+ CRUSH rule 2 x 901 [5,2]
+ CRUSH rule 2 x 902 [8,5]
+ CRUSH rule 2 x 903 [5,7]
+ CRUSH rule 2 x 904 [5,6]
+ CRUSH rule 2 x 905 [6,2]
+ CRUSH rule 2 x 906 [2,6]
+ CRUSH rule 2 x 907 [7,2]
+ CRUSH rule 2 x 908 [5,8]
+ CRUSH rule 2 x 909 [2,5]
+ CRUSH rule 2 x 910 [6,5]
+ CRUSH rule 2 x 911 [5,8]
+ CRUSH rule 2 x 912 [2,7]
+ CRUSH rule 2 x 913 [7,2]
+ CRUSH rule 2 x 914 [6,5]
+ CRUSH rule 2 x 915 [8,2]
+ CRUSH rule 2 x 916 [5,2]
+ CRUSH rule 2 x 917 [2,5]
+ CRUSH rule 2 x 918 [8,2]
+ CRUSH rule 2 x 919 [6,2]
+ CRUSH rule 2 x 920 [7,5]
+ CRUSH rule 2 x 921 [2,5]
+ CRUSH rule 2 x 922 [6,5]
+ CRUSH rule 2 x 923 [5,8]
+ CRUSH rule 2 x 924 [5,2]
+ CRUSH rule 2 x 925 [5,7]
+ CRUSH rule 2 x 926 [5,2]
+ CRUSH rule 2 x 927 [2,6]
+ CRUSH rule 2 x 928 [8,2]
+ CRUSH rule 2 x 929 [5,2]
+ CRUSH rule 2 x 930 [2,5]
+ CRUSH rule 2 x 931 [5,2]
+ CRUSH rule 2 x 932 [5,2]
+ CRUSH rule 2 x 933 [8,5]
+ CRUSH rule 2 x 934 [5,6]
+ CRUSH rule 2 x 935 [6,5]
+ CRUSH rule 2 x 936 [2,6]
+ CRUSH rule 2 x 937 [5,8]
+ CRUSH rule 2 x 938 [6,5]
+ CRUSH rule 2 x 939 [2,7]
+ CRUSH rule 2 x 940 [8,5]
+ CRUSH rule 2 x 941 [5,2]
+ CRUSH rule 2 x 942 [2,8]
+ CRUSH rule 2 x 943 [8,2]
+ CRUSH rule 2 x 944 [5,8]
+ CRUSH rule 2 x 945 [7,2]
+ CRUSH rule 2 x 946 [2,8]
+ CRUSH rule 2 x 947 [5,2]
+ CRUSH rule 2 x 948 [7,5]
+ CRUSH rule 2 x 949 [6,2]
+ CRUSH rule 2 x 950 [5,6]
+ CRUSH rule 2 x 951 [5,8]
+ CRUSH rule 2 x 952 [2,7]
+ CRUSH rule 2 x 953 [2,5]
+ CRUSH rule 2 x 954 [5,2]
+ CRUSH rule 2 x 955 [8,2]
+ CRUSH rule 2 x 956 [2,6]
+ CRUSH rule 2 x 957 [7,2]
+ CRUSH rule 2 x 958 [8,5]
+ CRUSH rule 2 x 959 [5,2]
+ CRUSH rule 2 x 960 [5,6]
+ CRUSH rule 2 x 961 [5,2]
+ CRUSH rule 2 x 962 [7,5]
+ CRUSH rule 2 x 963 [2,5]
+ CRUSH rule 2 x 964 [5,2]
+ CRUSH rule 2 x 965 [7,5]
+ CRUSH rule 2 x 966 [5,8]
+ CRUSH rule 2 x 967 [8,5]
+ CRUSH rule 2 x 968 [7,2]
+ CRUSH rule 2 x 969 [8,2]
+ CRUSH rule 2 x 970 [2,6]
+ CRUSH rule 2 x 971 [2,7]
+ CRUSH rule 2 x 972 [2,8]
+ CRUSH rule 2 x 973 [2,6]
+ CRUSH rule 2 x 974 [5,2]
+ CRUSH rule 2 x 975 [5,7]
+ CRUSH rule 2 x 976 [5,8]
+ CRUSH rule 2 x 977 [8,5]
+ CRUSH rule 2 x 978 [7,2]
+ CRUSH rule 2 x 979 [7,2]
+ CRUSH rule 2 x 980 [6,2]
+ CRUSH rule 2 x 981 [7,5]
+ CRUSH rule 2 x 982 [5,2]
+ CRUSH rule 2 x 983 [5,7]
+ CRUSH rule 2 x 984 [2,7]
+ CRUSH rule 2 x 985 [2,5]
+ CRUSH rule 2 x 986 [8,5]
+ CRUSH rule 2 x 987 [2,5]
+ CRUSH rule 2 x 988 [2,5]
+ CRUSH rule 2 x 989 [2,6]
+ CRUSH rule 2 x 990 [2,6]
+ CRUSH rule 2 x 991 [2,5]
+ CRUSH rule 2 x 992 [7,2]
+ CRUSH rule 2 x 993 [2,6]
+ CRUSH rule 2 x 994 [5,7]
+ CRUSH rule 2 x 995 [7,2]
+ CRUSH rule 2 x 996 [6,5]
+ CRUSH rule 2 x 997 [6,5]
+ CRUSH rule 2 x 998 [8,2]
+ CRUSH rule 2 x 999 [2,7]
+ CRUSH rule 2 x 1000 [8,5]
+ CRUSH rule 2 x 1001 [2,5]
+ CRUSH rule 2 x 1002 [2,5]
+ CRUSH rule 2 x 1003 [2,8]
+ CRUSH rule 2 x 1004 [6,2]
+ CRUSH rule 2 x 1005 [6,2]
+ CRUSH rule 2 x 1006 [2,6]
+ CRUSH rule 2 x 1007 [2,5]
+ CRUSH rule 2 x 1008 [2,7]
+ CRUSH rule 2 x 1009 [6,5]
+ CRUSH rule 2 x 1010 [5,2]
+ CRUSH rule 2 x 1011 [5,2]
+ CRUSH rule 2 x 1012 [5,2]
+ CRUSH rule 2 x 1013 [5,2]
+ CRUSH rule 2 x 1014 [2,8]
+ CRUSH rule 2 x 1015 [6,5]
+ CRUSH rule 2 x 1016 [2,5]
+ CRUSH rule 2 x 1017 [6,2]
+ CRUSH rule 2 x 1018 [5,2]
+ CRUSH rule 2 x 1019 [5,8]
+ CRUSH rule 2 x 1020 [5,2]
+ CRUSH rule 2 x 1021 [5,2]
+ CRUSH rule 2 x 1022 [2,6]
+ CRUSH rule 2 x 1023 [5,2]
+ rule 2 (chooseleaf) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 2 x 0 [2,5,7]
+ CRUSH rule 2 x 1 [2,8,5]
+ CRUSH rule 2 x 2 [2,5,7]
+ CRUSH rule 2 x 3 [8,2,5]
+ CRUSH rule 2 x 4 [5,2,7]
+ CRUSH rule 2 x 5 [7,2,5]
+ CRUSH rule 2 x 6 [2,6,5]
+ CRUSH rule 2 x 7 [5,8,2]
+ CRUSH rule 2 x 8 [5,6,2]
+ CRUSH rule 2 x 9 [2,5,8]
+ CRUSH rule 2 x 10 [2,7,5]
+ CRUSH rule 2 x 11 [2,7,5]
+ CRUSH rule 2 x 12 [2,5,6]
+ CRUSH rule 2 x 13 [5,8,2]
+ CRUSH rule 2 x 14 [7,2,5]
+ CRUSH rule 2 x 15 [7,2,5]
+ CRUSH rule 2 x 16 [5,6,2]
+ CRUSH rule 2 x 17 [5,2,6]
+ CRUSH rule 2 x 18 [2,5,6]
+ CRUSH rule 2 x 19 [7,5,2]
+ CRUSH rule 2 x 20 [2,5,7]
+ CRUSH rule 2 x 21 [5,7,2]
+ CRUSH rule 2 x 22 [8,5,2]
+ CRUSH rule 2 x 23 [5,6,2]
+ CRUSH rule 2 x 24 [2,7,5]
+ CRUSH rule 2 x 25 [5,7,2]
+ CRUSH rule 2 x 26 [2,8,5]
+ CRUSH rule 2 x 27 [5,2,8]
+ CRUSH rule 2 x 28 [6,2,5]
+ CRUSH rule 2 x 29 [8,5,2]
+ CRUSH rule 2 x 30 [5,7,2]
+ CRUSH rule 2 x 31 [8,2,5]
+ CRUSH rule 2 x 32 [5,6,2]
+ CRUSH rule 2 x 33 [2,7,5]
+ CRUSH rule 2 x 34 [2,5,7]
+ CRUSH rule 2 x 35 [2,8,5]
+ CRUSH rule 2 x 36 [5,8,2]
+ CRUSH rule 2 x 37 [2,5,6]
+ CRUSH rule 2 x 38 [5,8,2]
+ CRUSH rule 2 x 39 [5,7,2]
+ CRUSH rule 2 x 40 [7,2,5]
+ CRUSH rule 2 x 41 [2,6,5]
+ CRUSH rule 2 x 42 [5,6,2]
+ CRUSH rule 2 x 43 [2,5,7]
+ CRUSH rule 2 x 44 [2,6,5]
+ CRUSH rule 2 x 45 [8,2,5]
+ CRUSH rule 2 x 46 [2,5,8]
+ CRUSH rule 2 x 47 [5,2,8]
+ CRUSH rule 2 x 48 [5,6,2]
+ CRUSH rule 2 x 49 [5,7,2]
+ CRUSH rule 2 x 50 [5,2,7]
+ CRUSH rule 2 x 51 [5,6,2]
+ CRUSH rule 2 x 52 [8,2,5]
+ CRUSH rule 2 x 53 [5,8,2]
+ CRUSH rule 2 x 54 [7,5,2]
+ CRUSH rule 2 x 55 [8,2,5]
+ CRUSH rule 2 x 56 [6,5,2]
+ CRUSH rule 2 x 57 [5,8,2]
+ CRUSH rule 2 x 58 [2,8,5]
+ CRUSH rule 2 x 59 [5,2,7]
+ CRUSH rule 2 x 60 [5,2,6]
+ CRUSH rule 2 x 61 [5,6,2]
+ CRUSH rule 2 x 62 [7,2,5]
+ CRUSH rule 2 x 63 [5,6,2]
+ CRUSH rule 2 x 64 [5,2,8]
+ CRUSH rule 2 x 65 [7,5,2]
+ CRUSH rule 2 x 66 [5,6,2]
+ CRUSH rule 2 x 67 [5,2,8]
+ CRUSH rule 2 x 68 [2,5,8]
+ CRUSH rule 2 x 69 [5,2,6]
+ CRUSH rule 2 x 70 [7,2,5]
+ CRUSH rule 2 x 71 [2,8,5]
+ CRUSH rule 2 x 72 [6,2,5]
+ CRUSH rule 2 x 73 [2,7,5]
+ CRUSH rule 2 x 74 [2,7,5]
+ CRUSH rule 2 x 75 [5,2,6]
+ CRUSH rule 2 x 76 [5,2,7]
+ CRUSH rule 2 x 77 [7,2,5]
+ CRUSH rule 2 x 78 [2,5,8]
+ CRUSH rule 2 x 79 [5,2,7]
+ CRUSH rule 2 x 80 [2,5,6]
+ CRUSH rule 2 x 81 [2,5,6]
+ CRUSH rule 2 x 82 [7,2,5]
+ CRUSH rule 2 x 83 [2,6,5]
+ CRUSH rule 2 x 84 [7,2,5]
+ CRUSH rule 2 x 85 [5,8,2]
+ CRUSH rule 2 x 86 [2,6,5]
+ CRUSH rule 2 x 87 [2,7,5]
+ CRUSH rule 2 x 88 [2,6,5]
+ CRUSH rule 2 x 89 [5,2,7]
+ CRUSH rule 2 x 90 [6,5,2]
+ CRUSH rule 2 x 91 [5,8,2]
+ CRUSH rule 2 x 92 [2,8,5]
+ CRUSH rule 2 x 93 [7,5,2]
+ CRUSH rule 2 x 94 [2,5,8]
+ CRUSH rule 2 x 95 [7,5,2]
+ CRUSH rule 2 x 96 [5,6,2]
+ CRUSH rule 2 x 97 [8,5,2]
+ CRUSH rule 2 x 98 [2,7,5]
+ CRUSH rule 2 x 99 [2,7,5]
+ CRUSH rule 2 x 100 [2,7,5]
+ CRUSH rule 2 x 101 [5,7,2]
+ CRUSH rule 2 x 102 [5,2,7]
+ CRUSH rule 2 x 103 [5,7,2]
+ CRUSH rule 2 x 104 [7,5,2]
+ CRUSH rule 2 x 105 [2,5,6]
+ CRUSH rule 2 x 106 [2,6,5]
+ CRUSH rule 2 x 107 [5,2,6]
+ CRUSH rule 2 x 108 [7,2,5]
+ CRUSH rule 2 x 109 [2,5,6]
+ CRUSH rule 2 x 110 [5,2,7]
+ CRUSH rule 2 x 111 [2,5,6]
+ CRUSH rule 2 x 112 [2,6,5]
+ CRUSH rule 2 x 113 [6,2,5]
+ CRUSH rule 2 x 114 [7,5,2]
+ CRUSH rule 2 x 115 [8,2,5]
+ CRUSH rule 2 x 116 [2,6,5]
+ CRUSH rule 2 x 117 [7,5,2]
+ CRUSH rule 2 x 118 [2,5,8]
+ CRUSH rule 2 x 119 [5,6,2]
+ CRUSH rule 2 x 120 [2,5,7]
+ CRUSH rule 2 x 121 [2,7,5]
+ CRUSH rule 2 x 122 [8,5,2]
+ CRUSH rule 2 x 123 [2,5,8]
+ CRUSH rule 2 x 124 [5,2,8]
+ CRUSH rule 2 x 125 [2,7,5]
+ CRUSH rule 2 x 126 [5,2,6]
+ CRUSH rule 2 x 127 [5,6,2]
+ CRUSH rule 2 x 128 [5,6,2]
+ CRUSH rule 2 x 129 [2,5,7]
+ CRUSH rule 2 x 130 [5,8,2]
+ CRUSH rule 2 x 131 [2,5,8]
+ CRUSH rule 2 x 132 [2,5,6]
+ CRUSH rule 2 x 133 [5,6,2]
+ CRUSH rule 2 x 134 [2,8,5]
+ CRUSH rule 2 x 135 [5,6,2]
+ CRUSH rule 2 x 136 [2,5,6]
+ CRUSH rule 2 x 137 [7,5,2]
+ CRUSH rule 2 x 138 [8,5,2]
+ CRUSH rule 2 x 139 [5,2,7]
+ CRUSH rule 2 x 140 [2,6,5]
+ CRUSH rule 2 x 141 [6,2,5]
+ CRUSH rule 2 x 142 [5,2,8]
+ CRUSH rule 2 x 143 [5,8,2]
+ CRUSH rule 2 x 144 [8,2,5]
+ CRUSH rule 2 x 145 [8,5,2]
+ CRUSH rule 2 x 146 [2,6,5]
+ CRUSH rule 2 x 147 [2,8,5]
+ CRUSH rule 2 x 148 [5,2,7]
+ CRUSH rule 2 x 149 [5,8,2]
+ CRUSH rule 2 x 150 [2,6,5]
+ CRUSH rule 2 x 151 [5,6,2]
+ CRUSH rule 2 x 152 [8,5,2]
+ CRUSH rule 2 x 153 [8,5,2]
+ CRUSH rule 2 x 154 [5,2,7]
+ CRUSH rule 2 x 155 [5,7,2]
+ CRUSH rule 2 x 156 [5,2,6]
+ CRUSH rule 2 x 157 [5,2,6]
+ CRUSH rule 2 x 158 [2,8,5]
+ CRUSH rule 2 x 159 [7,2,5]
+ CRUSH rule 2 x 160 [2,8,5]
+ CRUSH rule 2 x 161 [2,5,6]
+ CRUSH rule 2 x 162 [2,6,5]
+ CRUSH rule 2 x 163 [5,6,2]
+ CRUSH rule 2 x 164 [7,2,5]
+ CRUSH rule 2 x 165 [7,2,5]
+ CRUSH rule 2 x 166 [2,5,6]
+ CRUSH rule 2 x 167 [2,7,5]
+ CRUSH rule 2 x 168 [5,2,7]
+ CRUSH rule 2 x 169 [2,6,5]
+ CRUSH rule 2 x 170 [2,5,8]
+ CRUSH rule 2 x 171 [7,5,2]
+ CRUSH rule 2 x 172 [2,7,5]
+ CRUSH rule 2 x 173 [8,5,2]
+ CRUSH rule 2 x 174 [2,5,7]
+ CRUSH rule 2 x 175 [6,2,5]
+ CRUSH rule 2 x 176 [5,2,7]
+ CRUSH rule 2 x 177 [5,2,6]
+ CRUSH rule 2 x 178 [5,2,8]
+ CRUSH rule 2 x 179 [5,2,7]
+ CRUSH rule 2 x 180 [5,8,2]
+ CRUSH rule 2 x 181 [6,2,5]
+ CRUSH rule 2 x 182 [8,5,2]
+ CRUSH rule 2 x 183 [7,5,2]
+ CRUSH rule 2 x 184 [5,7,2]
+ CRUSH rule 2 x 185 [6,2,5]
+ CRUSH rule 2 x 186 [2,5,7]
+ CRUSH rule 2 x 187 [2,6,5]
+ CRUSH rule 2 x 188 [2,8,5]
+ CRUSH rule 2 x 189 [2,7,5]
+ CRUSH rule 2 x 190 [5,2,8]
+ CRUSH rule 2 x 191 [7,2,5]
+ CRUSH rule 2 x 192 [5,2,8]
+ CRUSH rule 2 x 193 [5,2,6]
+ CRUSH rule 2 x 194 [2,5,8]
+ CRUSH rule 2 x 195 [6,5,2]
+ CRUSH rule 2 x 196 [6,2,5]
+ CRUSH rule 2 x 197 [6,5,2]
+ CRUSH rule 2 x 198 [2,5,6]
+ CRUSH rule 2 x 199 [2,5,7]
+ CRUSH rule 2 x 200 [2,5,6]
+ CRUSH rule 2 x 201 [7,2,5]
+ CRUSH rule 2 x 202 [6,5,2]
+ CRUSH rule 2 x 203 [5,8,2]
+ CRUSH rule 2 x 204 [2,5,7]
+ CRUSH rule 2 x 205 [2,7,5]
+ CRUSH rule 2 x 206 [2,7,5]
+ CRUSH rule 2 x 207 [5,2,7]
+ CRUSH rule 2 x 208 [7,2,5]
+ CRUSH rule 2 x 209 [2,8,5]
+ CRUSH rule 2 x 210 [2,5,8]
+ CRUSH rule 2 x 211 [5,2,8]
+ CRUSH rule 2 x 212 [7,5,2]
+ CRUSH rule 2 x 213 [8,5,2]
+ CRUSH rule 2 x 214 [5,8,2]
+ CRUSH rule 2 x 215 [8,2,5]
+ CRUSH rule 2 x 216 [5,2,6]
+ CRUSH rule 2 x 217 [2,7,5]
+ CRUSH rule 2 x 218 [2,7,5]
+ CRUSH rule 2 x 219 [5,8,2]
+ CRUSH rule 2 x 220 [5,7,2]
+ CRUSH rule 2 x 221 [5,6,2]
+ CRUSH rule 2 x 222 [6,5,2]
+ CRUSH rule 2 x 223 [2,5,6]
+ CRUSH rule 2 x 224 [2,5,8]
+ CRUSH rule 2 x 225 [8,2,5]
+ CRUSH rule 2 x 226 [7,2,5]
+ CRUSH rule 2 x 227 [5,2,6]
+ CRUSH rule 2 x 228 [5,6,2]
+ CRUSH rule 2 x 229 [5,8,2]
+ CRUSH rule 2 x 230 [5,7,2]
+ CRUSH rule 2 x 231 [5,7,2]
+ CRUSH rule 2 x 232 [2,7,5]
+ CRUSH rule 2 x 233 [5,7,2]
+ CRUSH rule 2 x 234 [2,5,6]
+ CRUSH rule 2 x 235 [5,8,2]
+ CRUSH rule 2 x 236 [5,2,7]
+ CRUSH rule 2 x 237 [5,7,2]
+ CRUSH rule 2 x 238 [5,2,6]
+ CRUSH rule 2 x 239 [8,5,2]
+ CRUSH rule 2 x 240 [5,7,2]
+ CRUSH rule 2 x 241 [5,2,8]
+ CRUSH rule 2 x 242 [5,2,6]
+ CRUSH rule 2 x 243 [5,7,2]
+ CRUSH rule 2 x 244 [5,6,2]
+ CRUSH rule 2 x 245 [7,2,5]
+ CRUSH rule 2 x 246 [2,5,8]
+ CRUSH rule 2 x 247 [6,2,5]
+ CRUSH rule 2 x 248 [8,2,5]
+ CRUSH rule 2 x 249 [2,5,7]
+ CRUSH rule 2 x 250 [2,5,6]
+ CRUSH rule 2 x 251 [2,5,6]
+ CRUSH rule 2 x 252 [5,7,2]
+ CRUSH rule 2 x 253 [5,2,6]
+ CRUSH rule 2 x 254 [5,2,7]
+ CRUSH rule 2 x 255 [2,7,5]
+ CRUSH rule 2 x 256 [5,7,2]
+ CRUSH rule 2 x 257 [2,8,5]
+ CRUSH rule 2 x 258 [5,2,6]
+ CRUSH rule 2 x 259 [5,6,2]
+ CRUSH rule 2 x 260 [5,6,2]
+ CRUSH rule 2 x 261 [8,5,2]
+ CRUSH rule 2 x 262 [5,6,2]
+ CRUSH rule 2 x 263 [6,2,5]
+ CRUSH rule 2 x 264 [5,6,2]
+ CRUSH rule 2 x 265 [8,5,2]
+ CRUSH rule 2 x 266 [8,2,5]
+ CRUSH rule 2 x 267 [2,5,8]
+ CRUSH rule 2 x 268 [2,7,5]
+ CRUSH rule 2 x 269 [2,8,5]
+ CRUSH rule 2 x 270 [5,2,7]
+ CRUSH rule 2 x 271 [7,5,2]
+ CRUSH rule 2 x 272 [2,8,5]
+ CRUSH rule 2 x 273 [5,2,7]
+ CRUSH rule 2 x 274 [6,5,2]
+ CRUSH rule 2 x 275 [5,7,2]
+ CRUSH rule 2 x 276 [7,2,5]
+ CRUSH rule 2 x 277 [6,5,2]
+ CRUSH rule 2 x 278 [6,2,5]
+ CRUSH rule 2 x 279 [8,5,2]
+ CRUSH rule 2 x 280 [2,6,5]
+ CRUSH rule 2 x 281 [8,2,5]
+ CRUSH rule 2 x 282 [5,2,6]
+ CRUSH rule 2 x 283 [8,2,5]
+ CRUSH rule 2 x 284 [6,5,2]
+ CRUSH rule 2 x 285 [5,7,2]
+ CRUSH rule 2 x 286 [2,6,5]
+ CRUSH rule 2 x 287 [2,5,6]
+ CRUSH rule 2 x 288 [8,2,5]
+ CRUSH rule 2 x 289 [5,6,2]
+ CRUSH rule 2 x 290 [2,5,7]
+ CRUSH rule 2 x 291 [2,5,8]
+ CRUSH rule 2 x 292 [8,2,5]
+ CRUSH rule 2 x 293 [6,2,5]
+ CRUSH rule 2 x 294 [7,5,2]
+ CRUSH rule 2 x 295 [5,8,2]
+ CRUSH rule 2 x 296 [5,2,6]
+ CRUSH rule 2 x 297 [6,2,5]
+ CRUSH rule 2 x 298 [2,5,7]
+ CRUSH rule 2 x 299 [2,8,5]
+ CRUSH rule 2 x 300 [8,5,2]
+ CRUSH rule 2 x 301 [2,8,5]
+ CRUSH rule 2 x 302 [5,2,6]
+ CRUSH rule 2 x 303 [7,5,2]
+ CRUSH rule 2 x 304 [2,7,5]
+ CRUSH rule 2 x 305 [5,8,2]
+ CRUSH rule 2 x 306 [2,7,5]
+ CRUSH rule 2 x 307 [2,7,5]
+ CRUSH rule 2 x 308 [2,8,5]
+ CRUSH rule 2 x 309 [7,5,2]
+ CRUSH rule 2 x 310 [5,2,7]
+ CRUSH rule 2 x 311 [5,6,2]
+ CRUSH rule 2 x 312 [2,6,5]
+ CRUSH rule 2 x 313 [5,2,6]
+ CRUSH rule 2 x 314 [5,2,6]
+ CRUSH rule 2 x 315 [2,5,8]
+ CRUSH rule 2 x 316 [6,5,2]
+ CRUSH rule 2 x 317 [2,6,5]
+ CRUSH rule 2 x 318 [8,2,5]
+ CRUSH rule 2 x 319 [5,2,8]
+ CRUSH rule 2 x 320 [5,7,2]
+ CRUSH rule 2 x 321 [2,5,8]
+ CRUSH rule 2 x 322 [2,7,5]
+ CRUSH rule 2 x 323 [5,7,2]
+ CRUSH rule 2 x 324 [7,2,5]
+ CRUSH rule 2 x 325 [5,6,2]
+ CRUSH rule 2 x 326 [5,2,6]
+ CRUSH rule 2 x 327 [2,6,5]
+ CRUSH rule 2 x 328 [7,5,2]
+ CRUSH rule 2 x 329 [5,6,2]
+ CRUSH rule 2 x 330 [5,7,2]
+ CRUSH rule 2 x 331 [2,6,5]
+ CRUSH rule 2 x 332 [2,5,6]
+ CRUSH rule 2 x 333 [6,5,2]
+ CRUSH rule 2 x 334 [8,5,2]
+ CRUSH rule 2 x 335 [7,2,5]
+ CRUSH rule 2 x 336 [5,6,2]
+ CRUSH rule 2 x 337 [7,2,5]
+ CRUSH rule 2 x 338 [5,6,2]
+ CRUSH rule 2 x 339 [7,5,2]
+ CRUSH rule 2 x 340 [2,8,5]
+ CRUSH rule 2 x 341 [5,2,7]
+ CRUSH rule 2 x 342 [2,7,5]
+ CRUSH rule 2 x 343 [6,5,2]
+ CRUSH rule 2 x 344 [6,2,5]
+ CRUSH rule 2 x 345 [5,7,2]
+ CRUSH rule 2 x 346 [8,2,5]
+ CRUSH rule 2 x 347 [5,2,7]
+ CRUSH rule 2 x 348 [8,2,5]
+ CRUSH rule 2 x 349 [2,6,5]
+ CRUSH rule 2 x 350 [8,5,2]
+ CRUSH rule 2 x 351 [5,6,2]
+ CRUSH rule 2 x 352 [2,8,5]
+ CRUSH rule 2 x 353 [6,5,2]
+ CRUSH rule 2 x 354 [2,5,6]
+ CRUSH rule 2 x 355 [5,8,2]
+ CRUSH rule 2 x 356 [5,2,6]
+ CRUSH rule 2 x 357 [6,2,5]
+ CRUSH rule 2 x 358 [2,8,5]
+ CRUSH rule 2 x 359 [6,2,5]
+ CRUSH rule 2 x 360 [5,2,7]
+ CRUSH rule 2 x 361 [8,5,2]
+ CRUSH rule 2 x 362 [5,2,6]
+ CRUSH rule 2 x 363 [5,2,6]
+ CRUSH rule 2 x 364 [2,5,6]
+ CRUSH rule 2 x 365 [6,5,2]
+ CRUSH rule 2 x 366 [7,2,5]
+ CRUSH rule 2 x 367 [5,2,6]
+ CRUSH rule 2 x 368 [7,5,2]
+ CRUSH rule 2 x 369 [5,7,2]
+ CRUSH rule 2 x 370 [8,2,5]
+ CRUSH rule 2 x 371 [2,5,6]
+ CRUSH rule 2 x 372 [5,2,8]
+ CRUSH rule 2 x 373 [2,6,5]
+ CRUSH rule 2 x 374 [5,8,2]
+ CRUSH rule 2 x 375 [6,5,2]
+ CRUSH rule 2 x 376 [7,2,5]
+ CRUSH rule 2 x 377 [2,5,6]
+ CRUSH rule 2 x 378 [2,8,5]
+ CRUSH rule 2 x 379 [8,5,2]
+ CRUSH rule 2 x 380 [2,5,8]
+ CRUSH rule 2 x 381 [2,5,7]
+ CRUSH rule 2 x 382 [2,5,8]
+ CRUSH rule 2 x 383 [5,6,2]
+ CRUSH rule 2 x 384 [7,2,5]
+ CRUSH rule 2 x 385 [7,5,2]
+ CRUSH rule 2 x 386 [2,5,6]
+ CRUSH rule 2 x 387 [2,5,6]
+ CRUSH rule 2 x 388 [5,2,8]
+ CRUSH rule 2 x 389 [2,5,7]
+ CRUSH rule 2 x 390 [5,6,2]
+ CRUSH rule 2 x 391 [5,6,2]
+ CRUSH rule 2 x 392 [2,8,5]
+ CRUSH rule 2 x 393 [5,2,6]
+ CRUSH rule 2 x 394 [5,7,2]
+ CRUSH rule 2 x 395 [5,2,8]
+ CRUSH rule 2 x 396 [5,2,7]
+ CRUSH rule 2 x 397 [2,5,8]
+ CRUSH rule 2 x 398 [2,5,8]
+ CRUSH rule 2 x 399 [8,5,2]
+ CRUSH rule 2 x 400 [8,2,5]
+ CRUSH rule 2 x 401 [2,5,8]
+ CRUSH rule 2 x 402 [7,5,2]
+ CRUSH rule 2 x 403 [2,5,8]
+ CRUSH rule 2 x 404 [5,2,8]
+ CRUSH rule 2 x 405 [6,5,2]
+ CRUSH rule 2 x 406 [2,6,5]
+ CRUSH rule 2 x 407 [2,8,5]
+ CRUSH rule 2 x 408 [5,2,6]
+ CRUSH rule 2 x 409 [7,5,2]
+ CRUSH rule 2 x 410 [8,5,2]
+ CRUSH rule 2 x 411 [2,8,5]
+ CRUSH rule 2 x 412 [2,5,8]
+ CRUSH rule 2 x 413 [5,2,8]
+ CRUSH rule 2 x 414 [5,2,6]
+ CRUSH rule 2 x 415 [2,6,5]
+ CRUSH rule 2 x 416 [2,6,5]
+ CRUSH rule 2 x 417 [8,2,5]
+ CRUSH rule 2 x 418 [7,2,5]
+ CRUSH rule 2 x 419 [8,5,2]
+ CRUSH rule 2 x 420 [2,5,6]
+ CRUSH rule 2 x 421 [8,5,2]
+ CRUSH rule 2 x 422 [6,5,2]
+ CRUSH rule 2 x 423 [2,5,6]
+ CRUSH rule 2 x 424 [8,5,2]
+ CRUSH rule 2 x 425 [2,5,7]
+ CRUSH rule 2 x 426 [6,2,5]
+ CRUSH rule 2 x 427 [2,7,5]
+ CRUSH rule 2 x 428 [5,7,2]
+ CRUSH rule 2 x 429 [5,6,2]
+ CRUSH rule 2 x 430 [5,6,2]
+ CRUSH rule 2 x 431 [5,2,7]
+ CRUSH rule 2 x 432 [7,2,5]
+ CRUSH rule 2 x 433 [6,5,2]
+ CRUSH rule 2 x 434 [5,2,8]
+ CRUSH rule 2 x 435 [2,5,6]
+ CRUSH rule 2 x 436 [5,2,7]
+ CRUSH rule 2 x 437 [7,5,2]
+ CRUSH rule 2 x 438 [2,5,8]
+ CRUSH rule 2 x 439 [2,5,7]
+ CRUSH rule 2 x 440 [2,7,5]
+ CRUSH rule 2 x 441 [5,7,2]
+ CRUSH rule 2 x 442 [2,5,7]
+ CRUSH rule 2 x 443 [6,2,5]
+ CRUSH rule 2 x 444 [7,2,5]
+ CRUSH rule 2 x 445 [6,5,2]
+ CRUSH rule 2 x 446 [5,2,8]
+ CRUSH rule 2 x 447 [2,5,7]
+ CRUSH rule 2 x 448 [7,2,5]
+ CRUSH rule 2 x 449 [7,5,2]
+ CRUSH rule 2 x 450 [5,2,8]
+ CRUSH rule 2 x 451 [6,5,2]
+ CRUSH rule 2 x 452 [8,5,2]
+ CRUSH rule 2 x 453 [6,5,2]
+ CRUSH rule 2 x 454 [6,5,2]
+ CRUSH rule 2 x 455 [2,7,5]
+ CRUSH rule 2 x 456 [6,2,5]
+ CRUSH rule 2 x 457 [7,2,5]
+ CRUSH rule 2 x 458 [2,8,5]
+ CRUSH rule 2 x 459 [2,7,5]
+ CRUSH rule 2 x 460 [6,5,2]
+ CRUSH rule 2 x 461 [6,5,2]
+ CRUSH rule 2 x 462 [8,2,5]
+ CRUSH rule 2 x 463 [6,2,5]
+ CRUSH rule 2 x 464 [7,5,2]
+ CRUSH rule 2 x 465 [7,2,5]
+ CRUSH rule 2 x 466 [5,8,2]
+ CRUSH rule 2 x 467 [6,5,2]
+ CRUSH rule 2 x 468 [7,2,5]
+ CRUSH rule 2 x 469 [7,2,5]
+ CRUSH rule 2 x 470 [5,2,6]
+ CRUSH rule 2 x 471 [2,7,5]
+ CRUSH rule 2 x 472 [5,2,8]
+ CRUSH rule 2 x 473 [2,5,6]
+ CRUSH rule 2 x 474 [6,2,5]
+ CRUSH rule 2 x 475 [6,2,5]
+ CRUSH rule 2 x 476 [5,6,2]
+ CRUSH rule 2 x 477 [5,8,2]
+ CRUSH rule 2 x 478 [6,2,5]
+ CRUSH rule 2 x 479 [2,5,8]
+ CRUSH rule 2 x 480 [2,8,5]
+ CRUSH rule 2 x 481 [2,5,7]
+ CRUSH rule 2 x 482 [5,7,2]
+ CRUSH rule 2 x 483 [2,6,5]
+ CRUSH rule 2 x 484 [2,7,5]
+ CRUSH rule 2 x 485 [5,7,2]
+ CRUSH rule 2 x 486 [5,2,7]
+ CRUSH rule 2 x 487 [5,2,8]
+ CRUSH rule 2 x 488 [5,7,2]
+ CRUSH rule 2 x 489 [2,8,5]
+ CRUSH rule 2 x 490 [6,5,2]
+ CRUSH rule 2 x 491 [2,7,5]
+ CRUSH rule 2 x 492 [6,5,2]
+ CRUSH rule 2 x 493 [2,7,5]
+ CRUSH rule 2 x 494 [2,7,5]
+ CRUSH rule 2 x 495 [5,2,8]
+ CRUSH rule 2 x 496 [7,5,2]
+ CRUSH rule 2 x 497 [5,7,2]
+ CRUSH rule 2 x 498 [2,5,8]
+ CRUSH rule 2 x 499 [8,5,2]
+ CRUSH rule 2 x 500 [5,6,2]
+ CRUSH rule 2 x 501 [2,7,5]
+ CRUSH rule 2 x 502 [7,2,5]
+ CRUSH rule 2 x 503 [2,5,7]
+ CRUSH rule 2 x 504 [5,6,2]
+ CRUSH rule 2 x 505 [2,7,5]
+ CRUSH rule 2 x 506 [5,2,8]
+ CRUSH rule 2 x 507 [6,2,5]
+ CRUSH rule 2 x 508 [2,5,8]
+ CRUSH rule 2 x 509 [7,5,2]
+ CRUSH rule 2 x 510 [6,2,5]
+ CRUSH rule 2 x 511 [5,8,2]
+ CRUSH rule 2 x 512 [7,2,5]
+ CRUSH rule 2 x 513 [7,2,5]
+ CRUSH rule 2 x 514 [5,6,2]
+ CRUSH rule 2 x 515 [8,5,2]
+ CRUSH rule 2 x 516 [5,2,8]
+ CRUSH rule 2 x 517 [7,2,5]
+ CRUSH rule 2 x 518 [5,6,2]
+ CRUSH rule 2 x 519 [7,5,2]
+ CRUSH rule 2 x 520 [2,6,5]
+ CRUSH rule 2 x 521 [8,2,5]
+ CRUSH rule 2 x 522 [6,2,5]
+ CRUSH rule 2 x 523 [5,2,7]
+ CRUSH rule 2 x 524 [2,5,8]
+ CRUSH rule 2 x 525 [2,5,6]
+ CRUSH rule 2 x 526 [2,5,8]
+ CRUSH rule 2 x 527 [2,5,6]
+ CRUSH rule 2 x 528 [5,2,6]
+ CRUSH rule 2 x 529 [5,7,2]
+ CRUSH rule 2 x 530 [6,5,2]
+ CRUSH rule 2 x 531 [6,2,5]
+ CRUSH rule 2 x 532 [6,5,2]
+ CRUSH rule 2 x 533 [5,6,2]
+ CRUSH rule 2 x 534 [7,5,2]
+ CRUSH rule 2 x 535 [8,2,5]
+ CRUSH rule 2 x 536 [6,2,5]
+ CRUSH rule 2 x 537 [5,7,2]
+ CRUSH rule 2 x 538 [6,5,2]
+ CRUSH rule 2 x 539 [8,5,2]
+ CRUSH rule 2 x 540 [2,6,5]
+ CRUSH rule 2 x 541 [2,5,8]
+ CRUSH rule 2 x 542 [5,2,8]
+ CRUSH rule 2 x 543 [6,2,5]
+ CRUSH rule 2 x 544 [5,7,2]
+ CRUSH rule 2 x 545 [5,7,2]
+ CRUSH rule 2 x 546 [6,2,5]
+ CRUSH rule 2 x 547 [8,2,5]
+ CRUSH rule 2 x 548 [5,2,8]
+ CRUSH rule 2 x 549 [5,8,2]
+ CRUSH rule 2 x 550 [2,5,7]
+ CRUSH rule 2 x 551 [7,5,2]
+ CRUSH rule 2 x 552 [5,8,2]
+ CRUSH rule 2 x 553 [5,2,7]
+ CRUSH rule 2 x 554 [2,8,5]
+ CRUSH rule 2 x 555 [5,2,8]
+ CRUSH rule 2 x 556 [5,6,2]
+ CRUSH rule 2 x 557 [7,5,2]
+ CRUSH rule 2 x 558 [5,2,6]
+ CRUSH rule 2 x 559 [5,2,6]
+ CRUSH rule 2 x 560 [8,5,2]
+ CRUSH rule 2 x 561 [6,5,2]
+ CRUSH rule 2 x 562 [5,2,6]
+ CRUSH rule 2 x 563 [2,6,5]
+ CRUSH rule 2 x 564 [5,2,7]
+ CRUSH rule 2 x 565 [5,6,2]
+ CRUSH rule 2 x 566 [5,7,2]
+ CRUSH rule 2 x 567 [5,6,2]
+ CRUSH rule 2 x 568 [7,5,2]
+ CRUSH rule 2 x 569 [5,2,7]
+ CRUSH rule 2 x 570 [2,5,8]
+ CRUSH rule 2 x 571 [5,7,2]
+ CRUSH rule 2 x 572 [5,2,8]
+ CRUSH rule 2 x 573 [5,2,7]
+ CRUSH rule 2 x 574 [2,5,8]
+ CRUSH rule 2 x 575 [8,2,5]
+ CRUSH rule 2 x 576 [5,6,2]
+ CRUSH rule 2 x 577 [8,2,5]
+ CRUSH rule 2 x 578 [6,2,5]
+ CRUSH rule 2 x 579 [5,2,6]
+ CRUSH rule 2 x 580 [5,2,7]
+ CRUSH rule 2 x 581 [7,2,5]
+ CRUSH rule 2 x 582 [2,8,5]
+ CRUSH rule 2 x 583 [6,2,5]
+ CRUSH rule 2 x 584 [8,2,5]
+ CRUSH rule 2 x 585 [7,2,5]
+ CRUSH rule 2 x 586 [2,7,5]
+ CRUSH rule 2 x 587 [2,5,7]
+ CRUSH rule 2 x 588 [5,7,2]
+ CRUSH rule 2 x 589 [7,2,5]
+ CRUSH rule 2 x 590 [6,2,5]
+ CRUSH rule 2 x 591 [5,2,8]
+ CRUSH rule 2 x 592 [2,5,7]
+ CRUSH rule 2 x 593 [2,8,5]
+ CRUSH rule 2 x 594 [2,7,5]
+ CRUSH rule 2 x 595 [7,2,5]
+ CRUSH rule 2 x 596 [5,2,6]
+ CRUSH rule 2 x 597 [5,2,7]
+ CRUSH rule 2 x 598 [5,2,6]
+ CRUSH rule 2 x 599 [5,2,8]
+ CRUSH rule 2 x 600 [7,2,5]
+ CRUSH rule 2 x 601 [2,7,5]
+ CRUSH rule 2 x 602 [5,7,2]
+ CRUSH rule 2 x 603 [5,2,6]
+ CRUSH rule 2 x 604 [7,5,2]
+ CRUSH rule 2 x 605 [5,2,7]
+ CRUSH rule 2 x 606 [2,7,5]
+ CRUSH rule 2 x 607 [2,5,8]
+ CRUSH rule 2 x 608 [5,2,7]
+ CRUSH rule 2 x 609 [5,2,8]
+ CRUSH rule 2 x 610 [5,7,2]
+ CRUSH rule 2 x 611 [2,8,5]
+ CRUSH rule 2 x 612 [2,6,5]
+ CRUSH rule 2 x 613 [7,2,5]
+ CRUSH rule 2 x 614 [7,2,5]
+ CRUSH rule 2 x 615 [6,2,5]
+ CRUSH rule 2 x 616 [2,8,5]
+ CRUSH rule 2 x 617 [6,2,5]
+ CRUSH rule 2 x 618 [7,5,2]
+ CRUSH rule 2 x 619 [5,2,8]
+ CRUSH rule 2 x 620 [5,2,6]
+ CRUSH rule 2 x 621 [5,8,2]
+ CRUSH rule 2 x 622 [2,5,8]
+ CRUSH rule 2 x 623 [2,6,5]
+ CRUSH rule 2 x 624 [5,2,8]
+ CRUSH rule 2 x 625 [2,5,7]
+ CRUSH rule 2 x 626 [7,2,5]
+ CRUSH rule 2 x 627 [2,7,5]
+ CRUSH rule 2 x 628 [8,2,5]
+ CRUSH rule 2 x 629 [2,6,5]
+ CRUSH rule 2 x 630 [2,7,5]
+ CRUSH rule 2 x 631 [2,7,5]
+ CRUSH rule 2 x 632 [7,2,5]
+ CRUSH rule 2 x 633 [8,5,2]
+ CRUSH rule 2 x 634 [2,5,7]
+ CRUSH rule 2 x 635 [5,6,2]
+ CRUSH rule 2 x 636 [2,5,8]
+ CRUSH rule 2 x 637 [5,2,7]
+ CRUSH rule 2 x 638 [6,2,5]
+ CRUSH rule 2 x 639 [5,2,8]
+ CRUSH rule 2 x 640 [5,2,8]
+ CRUSH rule 2 x 641 [7,2,5]
+ CRUSH rule 2 x 642 [2,7,5]
+ CRUSH rule 2 x 643 [5,2,8]
+ CRUSH rule 2 x 644 [8,2,5]
+ CRUSH rule 2 x 645 [5,7,2]
+ CRUSH rule 2 x 646 [8,2,5]
+ CRUSH rule 2 x 647 [7,2,5]
+ CRUSH rule 2 x 648 [2,6,5]
+ CRUSH rule 2 x 649 [5,7,2]
+ CRUSH rule 2 x 650 [7,5,2]
+ CRUSH rule 2 x 651 [5,7,2]
+ CRUSH rule 2 x 652 [5,7,2]
+ CRUSH rule 2 x 653 [8,5,2]
+ CRUSH rule 2 x 654 [7,5,2]
+ CRUSH rule 2 x 655 [2,5,7]
+ CRUSH rule 2 x 656 [5,7,2]
+ CRUSH rule 2 x 657 [6,2,5]
+ CRUSH rule 2 x 658 [5,6,2]
+ CRUSH rule 2 x 659 [5,6,2]
+ CRUSH rule 2 x 660 [7,5,2]
+ CRUSH rule 2 x 661 [2,8,5]
+ CRUSH rule 2 x 662 [5,2,7]
+ CRUSH rule 2 x 663 [2,5,8]
+ CRUSH rule 2 x 664 [2,5,7]
+ CRUSH rule 2 x 665 [5,7,2]
+ CRUSH rule 2 x 666 [2,8,5]
+ CRUSH rule 2 x 667 [2,5,7]
+ CRUSH rule 2 x 668 [5,7,2]
+ CRUSH rule 2 x 669 [6,5,2]
+ CRUSH rule 2 x 670 [5,2,6]
+ CRUSH rule 2 x 671 [2,7,5]
+ CRUSH rule 2 x 672 [5,2,7]
+ CRUSH rule 2 x 673 [5,2,7]
+ CRUSH rule 2 x 674 [5,2,8]
+ CRUSH rule 2 x 675 [2,8,5]
+ CRUSH rule 2 x 676 [2,5,8]
+ CRUSH rule 2 x 677 [5,2,7]
+ CRUSH rule 2 x 678 [2,5,8]
+ CRUSH rule 2 x 679 [6,2,5]
+ CRUSH rule 2 x 680 [2,5,6]
+ CRUSH rule 2 x 681 [5,7,2]
+ CRUSH rule 2 x 682 [2,5,7]
+ CRUSH rule 2 x 683 [2,5,6]
+ CRUSH rule 2 x 684 [7,2,5]
+ CRUSH rule 2 x 685 [7,2,5]
+ CRUSH rule 2 x 686 [2,5,8]
+ CRUSH rule 2 x 687 [5,6,2]
+ CRUSH rule 2 x 688 [5,7,2]
+ CRUSH rule 2 x 689 [6,5,2]
+ CRUSH rule 2 x 690 [8,2,5]
+ CRUSH rule 2 x 691 [5,2,6]
+ CRUSH rule 2 x 692 [7,2,5]
+ CRUSH rule 2 x 693 [6,5,2]
+ CRUSH rule 2 x 694 [6,5,2]
+ CRUSH rule 2 x 695 [2,8,5]
+ CRUSH rule 2 x 696 [2,5,8]
+ CRUSH rule 2 x 697 [6,2,5]
+ CRUSH rule 2 x 698 [6,2,5]
+ CRUSH rule 2 x 699 [2,6,5]
+ CRUSH rule 2 x 700 [2,5,7]
+ CRUSH rule 2 x 701 [5,2,7]
+ CRUSH rule 2 x 702 [5,2,8]
+ CRUSH rule 2 x 703 [8,5,2]
+ CRUSH rule 2 x 704 [2,5,8]
+ CRUSH rule 2 x 705 [8,2,5]
+ CRUSH rule 2 x 706 [2,5,6]
+ CRUSH rule 2 x 707 [7,5,2]
+ CRUSH rule 2 x 708 [5,7,2]
+ CRUSH rule 2 x 709 [6,5,2]
+ CRUSH rule 2 x 710 [8,5,2]
+ CRUSH rule 2 x 711 [2,5,8]
+ CRUSH rule 2 x 712 [2,5,7]
+ CRUSH rule 2 x 713 [6,5,2]
+ CRUSH rule 2 x 714 [5,2,7]
+ CRUSH rule 2 x 715 [2,5,6]
+ CRUSH rule 2 x 716 [5,6,2]
+ CRUSH rule 2 x 717 [8,2,5]
+ CRUSH rule 2 x 718 [5,7,2]
+ CRUSH rule 2 x 719 [2,6,5]
+ CRUSH rule 2 x 720 [6,2,5]
+ CRUSH rule 2 x 721 [5,7,2]
+ CRUSH rule 2 x 722 [5,7,2]
+ CRUSH rule 2 x 723 [5,2,7]
+ CRUSH rule 2 x 724 [2,6,5]
+ CRUSH rule 2 x 725 [2,5,7]
+ CRUSH rule 2 x 726 [5,8,2]
+ CRUSH rule 2 x 727 [5,6,2]
+ CRUSH rule 2 x 728 [2,7,5]
+ CRUSH rule 2 x 729 [5,6,2]
+ CRUSH rule 2 x 730 [5,7,2]
+ CRUSH rule 2 x 731 [5,2,8]
+ CRUSH rule 2 x 732 [2,5,6]
+ CRUSH rule 2 x 733 [5,7,2]
+ CRUSH rule 2 x 734 [6,5,2]
+ CRUSH rule 2 x 735 [5,8,2]
+ CRUSH rule 2 x 736 [5,8,2]
+ CRUSH rule 2 x 737 [2,6,5]
+ CRUSH rule 2 x 738 [5,2,7]
+ CRUSH rule 2 x 739 [2,7,5]
+ CRUSH rule 2 x 740 [2,8,5]
+ CRUSH rule 2 x 741 [7,2,5]
+ CRUSH rule 2 x 742 [8,2,5]
+ CRUSH rule 2 x 743 [7,2,5]
+ CRUSH rule 2 x 744 [5,7,2]
+ CRUSH rule 2 x 745 [5,2,8]
+ CRUSH rule 2 x 746 [5,2,7]
+ CRUSH rule 2 x 747 [6,2,5]
+ CRUSH rule 2 x 748 [2,7,5]
+ CRUSH rule 2 x 749 [5,8,2]
+ CRUSH rule 2 x 750 [2,6,5]
+ CRUSH rule 2 x 751 [2,8,5]
+ CRUSH rule 2 x 752 [8,2,5]
+ CRUSH rule 2 x 753 [7,5,2]
+ CRUSH rule 2 x 754 [8,5,2]
+ CRUSH rule 2 x 755 [2,6,5]
+ CRUSH rule 2 x 756 [5,6,2]
+ CRUSH rule 2 x 757 [8,2,5]
+ CRUSH rule 2 x 758 [6,2,5]
+ CRUSH rule 2 x 759 [8,5,2]
+ CRUSH rule 2 x 760 [2,5,7]
+ CRUSH rule 2 x 761 [5,2,8]
+ CRUSH rule 2 x 762 [2,7,5]
+ CRUSH rule 2 x 763 [8,5,2]
+ CRUSH rule 2 x 764 [2,7,5]
+ CRUSH rule 2 x 765 [6,5,2]
+ CRUSH rule 2 x 766 [8,5,2]
+ CRUSH rule 2 x 767 [2,8,5]
+ CRUSH rule 2 x 768 [8,5,2]
+ CRUSH rule 2 x 769 [6,2,5]
+ CRUSH rule 2 x 770 [6,2,5]
+ CRUSH rule 2 x 771 [7,2,5]
+ CRUSH rule 2 x 772 [8,5,2]
+ CRUSH rule 2 x 773 [5,2,7]
+ CRUSH rule 2 x 774 [5,6,2]
+ CRUSH rule 2 x 775 [6,5,2]
+ CRUSH rule 2 x 776 [7,2,5]
+ CRUSH rule 2 x 777 [5,2,6]
+ CRUSH rule 2 x 778 [2,8,5]
+ CRUSH rule 2 x 779 [2,7,5]
+ CRUSH rule 2 x 780 [2,5,7]
+ CRUSH rule 2 x 781 [6,5,2]
+ CRUSH rule 2 x 782 [5,2,8]
+ CRUSH rule 2 x 783 [7,2,5]
+ CRUSH rule 2 x 784 [2,5,6]
+ CRUSH rule 2 x 785 [6,2,5]
+ CRUSH rule 2 x 786 [7,5,2]
+ CRUSH rule 2 x 787 [2,6,5]
+ CRUSH rule 2 x 788 [6,2,5]
+ CRUSH rule 2 x 789 [2,5,8]
+ CRUSH rule 2 x 790 [8,5,2]
+ CRUSH rule 2 x 791 [5,8,2]
+ CRUSH rule 2 x 792 [5,8,2]
+ CRUSH rule 2 x 793 [6,2,5]
+ CRUSH rule 2 x 794 [2,6,5]
+ CRUSH rule 2 x 795 [2,5,8]
+ CRUSH rule 2 x 796 [5,7,2]
+ CRUSH rule 2 x 797 [2,5,8]
+ CRUSH rule 2 x 798 [6,2,5]
+ CRUSH rule 2 x 799 [5,2,8]
+ CRUSH rule 2 x 800 [5,2,7]
+ CRUSH rule 2 x 801 [5,6,2]
+ CRUSH rule 2 x 802 [2,8,5]
+ CRUSH rule 2 x 803 [2,5,7]
+ CRUSH rule 2 x 804 [6,2,5]
+ CRUSH rule 2 x 805 [5,6,2]
+ CRUSH rule 2 x 806 [2,5,7]
+ CRUSH rule 2 x 807 [5,7,2]
+ CRUSH rule 2 x 808 [5,6,2]
+ CRUSH rule 2 x 809 [2,5,8]
+ CRUSH rule 2 x 810 [5,7,2]
+ CRUSH rule 2 x 811 [8,5,2]
+ CRUSH rule 2 x 812 [8,5,2]
+ CRUSH rule 2 x 813 [6,5,2]
+ CRUSH rule 2 x 814 [5,6,2]
+ CRUSH rule 2 x 815 [5,2,8]
+ CRUSH rule 2 x 816 [2,7,5]
+ CRUSH rule 2 x 817 [5,8,2]
+ CRUSH rule 2 x 818 [5,2,7]
+ CRUSH rule 2 x 819 [5,2,8]
+ CRUSH rule 2 x 820 [5,6,2]
+ CRUSH rule 2 x 821 [5,6,2]
+ CRUSH rule 2 x 822 [2,5,8]
+ CRUSH rule 2 x 823 [5,8,2]
+ CRUSH rule 2 x 824 [5,7,2]
+ CRUSH rule 2 x 825 [2,8,5]
+ CRUSH rule 2 x 826 [7,2,5]
+ CRUSH rule 2 x 827 [2,8,5]
+ CRUSH rule 2 x 828 [2,5,8]
+ CRUSH rule 2 x 829 [5,6,2]
+ CRUSH rule 2 x 830 [2,5,8]
+ CRUSH rule 2 x 831 [2,6,5]
+ CRUSH rule 2 x 832 [5,7,2]
+ CRUSH rule 2 x 833 [2,7,5]
+ CRUSH rule 2 x 834 [5,2,7]
+ CRUSH rule 2 x 835 [8,5,2]
+ CRUSH rule 2 x 836 [5,7,2]
+ CRUSH rule 2 x 837 [6,5,2]
+ CRUSH rule 2 x 838 [6,2,5]
+ CRUSH rule 2 x 839 [5,2,6]
+ CRUSH rule 2 x 840 [7,5,2]
+ CRUSH rule 2 x 841 [5,8,2]
+ CRUSH rule 2 x 842 [2,5,6]
+ CRUSH rule 2 x 843 [6,5,2]
+ CRUSH rule 2 x 844 [5,8,2]
+ CRUSH rule 2 x 845 [5,8,2]
+ CRUSH rule 2 x 846 [5,2,7]
+ CRUSH rule 2 x 847 [2,8,5]
+ CRUSH rule 2 x 848 [2,6,5]
+ CRUSH rule 2 x 849 [5,6,2]
+ CRUSH rule 2 x 850 [2,5,6]
+ CRUSH rule 2 x 851 [6,5,2]
+ CRUSH rule 2 x 852 [7,5,2]
+ CRUSH rule 2 x 853 [6,2,5]
+ CRUSH rule 2 x 854 [7,2,5]
+ CRUSH rule 2 x 855 [5,7,2]
+ CRUSH rule 2 x 856 [6,5,2]
+ CRUSH rule 2 x 857 [8,5,2]
+ CRUSH rule 2 x 858 [6,5,2]
+ CRUSH rule 2 x 859 [6,2,5]
+ CRUSH rule 2 x 860 [5,2,7]
+ CRUSH rule 2 x 861 [8,5,2]
+ CRUSH rule 2 x 862 [6,2,5]
+ CRUSH rule 2 x 863 [8,2,5]
+ CRUSH rule 2 x 864 [5,6,2]
+ CRUSH rule 2 x 865 [8,2,5]
+ CRUSH rule 2 x 866 [5,6,2]
+ CRUSH rule 2 x 867 [6,5,2]
+ CRUSH rule 2 x 868 [6,5,2]
+ CRUSH rule 2 x 869 [8,5,2]
+ CRUSH rule 2 x 870 [2,5,8]
+ CRUSH rule 2 x 871 [5,2,8]
+ CRUSH rule 2 x 872 [5,2,8]
+ CRUSH rule 2 x 873 [5,6,2]
+ CRUSH rule 2 x 874 [2,6,5]
+ CRUSH rule 2 x 875 [2,6,5]
+ CRUSH rule 2 x 876 [5,8,2]
+ CRUSH rule 2 x 877 [6,5,2]
+ CRUSH rule 2 x 878 [5,2,7]
+ CRUSH rule 2 x 879 [7,5,2]
+ CRUSH rule 2 x 880 [5,2,8]
+ CRUSH rule 2 x 881 [5,6,2]
+ CRUSH rule 2 x 882 [5,2,7]
+ CRUSH rule 2 x 883 [2,5,7]
+ CRUSH rule 2 x 884 [6,2,5]
+ CRUSH rule 2 x 885 [5,2,8]
+ CRUSH rule 2 x 886 [5,6,2]
+ CRUSH rule 2 x 887 [7,5,2]
+ CRUSH rule 2 x 888 [6,2,5]
+ CRUSH rule 2 x 889 [2,6,5]
+ CRUSH rule 2 x 890 [7,2,5]
+ CRUSH rule 2 x 891 [2,8,5]
+ CRUSH rule 2 x 892 [6,2,5]
+ CRUSH rule 2 x 893 [2,5,7]
+ CRUSH rule 2 x 894 [7,5,2]
+ CRUSH rule 2 x 895 [5,2,8]
+ CRUSH rule 2 x 896 [2,8,5]
+ CRUSH rule 2 x 897 [5,2,6]
+ CRUSH rule 2 x 898 [2,5,7]
+ CRUSH rule 2 x 899 [2,7,5]
+ CRUSH rule 2 x 900 [5,2,6]
+ CRUSH rule 2 x 901 [5,2,8]
+ CRUSH rule 2 x 902 [8,5,2]
+ CRUSH rule 2 x 903 [5,7,2]
+ CRUSH rule 2 x 904 [5,6,2]
+ CRUSH rule 2 x 905 [6,2,5]
+ CRUSH rule 2 x 906 [2,6,5]
+ CRUSH rule 2 x 907 [7,2,5]
+ CRUSH rule 2 x 908 [5,8,2]
+ CRUSH rule 2 x 909 [2,5,7]
+ CRUSH rule 2 x 910 [6,5,2]
+ CRUSH rule 2 x 911 [5,8,2]
+ CRUSH rule 2 x 912 [2,7,5]
+ CRUSH rule 2 x 913 [7,2,5]
+ CRUSH rule 2 x 914 [6,5,2]
+ CRUSH rule 2 x 915 [8,2,5]
+ CRUSH rule 2 x 916 [5,2,8]
+ CRUSH rule 2 x 917 [2,5,8]
+ CRUSH rule 2 x 918 [8,2,5]
+ CRUSH rule 2 x 919 [6,2,5]
+ CRUSH rule 2 x 920 [7,5,2]
+ CRUSH rule 2 x 921 [2,5,6]
+ CRUSH rule 2 x 922 [6,5,2]
+ CRUSH rule 2 x 923 [5,8,2]
+ CRUSH rule 2 x 924 [5,2,7]
+ CRUSH rule 2 x 925 [5,7,2]
+ CRUSH rule 2 x 926 [5,2,8]
+ CRUSH rule 2 x 927 [2,6,5]
+ CRUSH rule 2 x 928 [8,2,5]
+ CRUSH rule 2 x 929 [5,2,7]
+ CRUSH rule 2 x 930 [2,5,6]
+ CRUSH rule 2 x 931 [5,2,7]
+ CRUSH rule 2 x 932 [5,2,8]
+ CRUSH rule 2 x 933 [8,5,2]
+ CRUSH rule 2 x 934 [5,6,2]
+ CRUSH rule 2 x 935 [6,5,2]
+ CRUSH rule 2 x 936 [2,6,5]
+ CRUSH rule 2 x 937 [5,8,2]
+ CRUSH rule 2 x 938 [6,5,2]
+ CRUSH rule 2 x 939 [2,7,5]
+ CRUSH rule 2 x 940 [8,5,2]
+ CRUSH rule 2 x 941 [5,2,8]
+ CRUSH rule 2 x 942 [2,8,5]
+ CRUSH rule 2 x 943 [8,2,5]
+ CRUSH rule 2 x 944 [5,8,2]
+ CRUSH rule 2 x 945 [7,2,5]
+ CRUSH rule 2 x 946 [2,8,5]
+ CRUSH rule 2 x 947 [5,2,8]
+ CRUSH rule 2 x 948 [7,5,2]
+ CRUSH rule 2 x 949 [6,2,5]
+ CRUSH rule 2 x 950 [5,6,2]
+ CRUSH rule 2 x 951 [5,8,2]
+ CRUSH rule 2 x 952 [2,7,5]
+ CRUSH rule 2 x 953 [2,5,6]
+ CRUSH rule 2 x 954 [5,2,7]
+ CRUSH rule 2 x 955 [8,2,5]
+ CRUSH rule 2 x 956 [2,6,5]
+ CRUSH rule 2 x 957 [7,2,5]
+ CRUSH rule 2 x 958 [8,5,2]
+ CRUSH rule 2 x 959 [5,2,7]
+ CRUSH rule 2 x 960 [5,6,2]
+ CRUSH rule 2 x 961 [5,2,8]
+ CRUSH rule 2 x 962 [7,5,2]
+ CRUSH rule 2 x 963 [2,5,6]
+ CRUSH rule 2 x 964 [5,2,8]
+ CRUSH rule 2 x 965 [7,5,2]
+ CRUSH rule 2 x 966 [5,8,2]
+ CRUSH rule 2 x 967 [8,5,2]
+ CRUSH rule 2 x 968 [7,2,5]
+ CRUSH rule 2 x 969 [8,2,5]
+ CRUSH rule 2 x 970 [2,6,5]
+ CRUSH rule 2 x 971 [2,7,5]
+ CRUSH rule 2 x 972 [2,8,5]
+ CRUSH rule 2 x 973 [2,6,5]
+ CRUSH rule 2 x 974 [5,2,8]
+ CRUSH rule 2 x 975 [5,7,2]
+ CRUSH rule 2 x 976 [5,8,2]
+ CRUSH rule 2 x 977 [8,5,2]
+ CRUSH rule 2 x 978 [7,2,5]
+ CRUSH rule 2 x 979 [7,2,5]
+ CRUSH rule 2 x 980 [6,2,5]
+ CRUSH rule 2 x 981 [7,5,2]
+ CRUSH rule 2 x 982 [5,2,8]
+ CRUSH rule 2 x 983 [5,7,2]
+ CRUSH rule 2 x 984 [2,7,5]
+ CRUSH rule 2 x 985 [2,5,7]
+ CRUSH rule 2 x 986 [8,5,2]
+ CRUSH rule 2 x 987 [2,5,8]
+ CRUSH rule 2 x 988 [2,5,7]
+ CRUSH rule 2 x 989 [2,6,5]
+ CRUSH rule 2 x 990 [2,6,5]
+ CRUSH rule 2 x 991 [2,5,8]
+ CRUSH rule 2 x 992 [7,2,5]
+ CRUSH rule 2 x 993 [2,6,5]
+ CRUSH rule 2 x 994 [5,7,2]
+ CRUSH rule 2 x 995 [7,2,5]
+ CRUSH rule 2 x 996 [6,5,2]
+ CRUSH rule 2 x 997 [6,5,2]
+ CRUSH rule 2 x 998 [8,2,5]
+ CRUSH rule 2 x 999 [2,7,5]
+ CRUSH rule 2 x 1000 [8,5,2]
+ CRUSH rule 2 x 1001 [2,5,6]
+ CRUSH rule 2 x 1002 [2,5,7]
+ CRUSH rule 2 x 1003 [2,8,5]
+ CRUSH rule 2 x 1004 [6,2,5]
+ CRUSH rule 2 x 1005 [6,2,5]
+ CRUSH rule 2 x 1006 [2,6,5]
+ CRUSH rule 2 x 1007 [2,5,7]
+ CRUSH rule 2 x 1008 [2,7,5]
+ CRUSH rule 2 x 1009 [6,5,2]
+ CRUSH rule 2 x 1010 [5,2,7]
+ CRUSH rule 2 x 1011 [5,2,8]
+ CRUSH rule 2 x 1012 [5,2,7]
+ CRUSH rule 2 x 1013 [5,2,7]
+ CRUSH rule 2 x 1014 [2,8,5]
+ CRUSH rule 2 x 1015 [6,5,2]
+ CRUSH rule 2 x 1016 [2,5,7]
+ CRUSH rule 2 x 1017 [6,2,5]
+ CRUSH rule 2 x 1018 [5,2,6]
+ CRUSH rule 2 x 1019 [5,8,2]
+ CRUSH rule 2 x 1020 [5,2,7]
+ CRUSH rule 2 x 1021 [5,2,6]
+ CRUSH rule 2 x 1022 [2,6,5]
+ CRUSH rule 2 x 1023 [5,2,8]
+ rule 2 (chooseleaf) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 3 (choose-set), x = 0..1023, numrep = 2..3
+ CRUSH rule 3 x 0 [2,5]
+ CRUSH rule 3 x 1 [2,8]
+ CRUSH rule 3 x 2 [2,5]
+ CRUSH rule 3 x 3 [8,2]
+ CRUSH rule 3 x 4 [5,2]
+ CRUSH rule 3 x 5 [7,2]
+ CRUSH rule 3 x 6 [2,6]
+ CRUSH rule 3 x 7 [5,6]
+ CRUSH rule 3 x 8 [5,7]
+ CRUSH rule 3 x 9 [2,5]
+ CRUSH rule 3 x 10 [2,8]
+ CRUSH rule 3 x 11 [2,6]
+ CRUSH rule 3 x 12 [2,5]
+ CRUSH rule 3 x 13 [5,8]
+ CRUSH rule 3 x 14 [7,2]
+ CRUSH rule 3 x 15 [7,2]
+ CRUSH rule 3 x 16 [5,7]
+ CRUSH rule 3 x 17 [5,2]
+ CRUSH rule 3 x 18 [2,5]
+ CRUSH rule 3 x 19 [7,5]
+ CRUSH rule 3 x 20 [2,5]
+ CRUSH rule 3 x 21 [5,6]
+ CRUSH rule 3 x 22 [8,5]
+ CRUSH rule 3 x 23 [5,7]
+ CRUSH rule 3 x 24 [2,6]
+ CRUSH rule 3 x 25 [5,8]
+ CRUSH rule 3 x 26 [2,7]
+ CRUSH rule 3 x 27 [5,2]
+ CRUSH rule 3 x 28 [6,2]
+ CRUSH rule 3 x 29 [8,5]
+ CRUSH rule 3 x 30 [5,6]
+ CRUSH rule 3 x 31 [8,2]
+ CRUSH rule 3 x 32 [5,7]
+ CRUSH rule 3 x 33 [2,7]
+ CRUSH rule 3 x 34 [2,5]
+ CRUSH rule 3 x 35 [2,6]
+ CRUSH rule 3 x 36 [5,6]
+ CRUSH rule 3 x 37 [2,5]
+ CRUSH rule 3 x 38 [5,6]
+ CRUSH rule 3 x 39 [5,7]
+ CRUSH rule 3 x 40 [7,2]
+ CRUSH rule 3 x 41 [2,7]
+ CRUSH rule 3 x 42 [5,7]
+ CRUSH rule 3 x 43 [2,5]
+ CRUSH rule 3 x 44 [2,8]
+ CRUSH rule 3 x 45 [8,2]
+ CRUSH rule 3 x 46 [2,5]
+ CRUSH rule 3 x 47 [5,2]
+ CRUSH rule 3 x 48 [5,6]
+ CRUSH rule 3 x 49 [5,6]
+ CRUSH rule 3 x 50 [5,2]
+ CRUSH rule 3 x 51 [5,6]
+ CRUSH rule 3 x 52 [8,2]
+ CRUSH rule 3 x 53 [5,6]
+ CRUSH rule 3 x 54 [7,5]
+ CRUSH rule 3 x 55 [8,2]
+ CRUSH rule 3 x 56 [6,5]
+ CRUSH rule 3 x 57 [5,8]
+ CRUSH rule 3 x 58 [2,8]
+ CRUSH rule 3 x 59 [5,2]
+ CRUSH rule 3 x 60 [5,2]
+ CRUSH rule 3 x 61 [5,8]
+ CRUSH rule 3 x 62 [7,2]
+ CRUSH rule 3 x 63 [5,6]
+ CRUSH rule 3 x 64 [5,2]
+ CRUSH rule 3 x 65 [7,5]
+ CRUSH rule 3 x 66 [5,6]
+ CRUSH rule 3 x 67 [5,2]
+ CRUSH rule 3 x 68 [2,5]
+ CRUSH rule 3 x 69 [5,2]
+ CRUSH rule 3 x 70 [7,2]
+ CRUSH rule 3 x 71 [2,7]
+ CRUSH rule 3 x 72 [6,2]
+ CRUSH rule 3 x 73 [2,7]
+ CRUSH rule 3 x 74 [2,8]
+ CRUSH rule 3 x 75 [5,2]
+ CRUSH rule 3 x 76 [5,2]
+ CRUSH rule 3 x 77 [7,2]
+ CRUSH rule 3 x 78 [2,5]
+ CRUSH rule 3 x 79 [5,2]
+ CRUSH rule 3 x 80 [2,5]
+ CRUSH rule 3 x 81 [2,5]
+ CRUSH rule 3 x 82 [7,2]
+ CRUSH rule 3 x 83 [2,6]
+ CRUSH rule 3 x 84 [7,2]
+ CRUSH rule 3 x 85 [5,7]
+ CRUSH rule 3 x 86 [2,6]
+ CRUSH rule 3 x 87 [2,6]
+ CRUSH rule 3 x 88 [2,8]
+ CRUSH rule 3 x 89 [5,2]
+ CRUSH rule 3 x 90 [6,5]
+ CRUSH rule 3 x 91 [5,6]
+ CRUSH rule 3 x 92 [2,6]
+ CRUSH rule 3 x 93 [7,5]
+ CRUSH rule 3 x 94 [2,5]
+ CRUSH rule 3 x 95 [7,5]
+ CRUSH rule 3 x 96 [5,8]
+ CRUSH rule 3 x 97 [8,5]
+ CRUSH rule 3 x 98 [2,7]
+ CRUSH rule 3 x 99 [2,8]
+ CRUSH rule 3 x 100 [2,6]
+ CRUSH rule 3 x 101 [5,8]
+ CRUSH rule 3 x 102 [5,2]
+ CRUSH rule 3 x 103 [5,7]
+ CRUSH rule 3 x 104 [7,5]
+ CRUSH rule 3 x 105 [2,5]
+ CRUSH rule 3 x 106 [2,6]
+ CRUSH rule 3 x 107 [5,2]
+ CRUSH rule 3 x 108 [7,2]
+ CRUSH rule 3 x 109 [2,5]
+ CRUSH rule 3 x 110 [5,2]
+ CRUSH rule 3 x 111 [2,5]
+ CRUSH rule 3 x 112 [2,6]
+ CRUSH rule 3 x 113 [6,2]
+ CRUSH rule 3 x 114 [7,5]
+ CRUSH rule 3 x 115 [8,2]
+ CRUSH rule 3 x 116 [2,7]
+ CRUSH rule 3 x 117 [7,5]
+ CRUSH rule 3 x 118 [2,5]
+ CRUSH rule 3 x 119 [5,7]
+ CRUSH rule 3 x 120 [2,5]
+ CRUSH rule 3 x 121 [2,8]
+ CRUSH rule 3 x 122 [8,5]
+ CRUSH rule 3 x 123 [2,5]
+ CRUSH rule 3 x 124 [5,2]
+ CRUSH rule 3 x 125 [2,7]
+ CRUSH rule 3 x 126 [5,2]
+ CRUSH rule 3 x 127 [5,6]
+ CRUSH rule 3 x 128 [5,8]
+ CRUSH rule 3 x 129 [2,5]
+ CRUSH rule 3 x 130 [5,8]
+ CRUSH rule 3 x 131 [2,5]
+ CRUSH rule 3 x 132 [2,5]
+ CRUSH rule 3 x 133 [5,6]
+ CRUSH rule 3 x 134 [2,7]
+ CRUSH rule 3 x 135 [5,7]
+ CRUSH rule 3 x 136 [2,5]
+ CRUSH rule 3 x 137 [7,5]
+ CRUSH rule 3 x 138 [8,5]
+ CRUSH rule 3 x 139 [5,2]
+ CRUSH rule 3 x 140 [2,8]
+ CRUSH rule 3 x 141 [6,2]
+ CRUSH rule 3 x 142 [5,2]
+ CRUSH rule 3 x 143 [5,7]
+ CRUSH rule 3 x 144 [8,2]
+ CRUSH rule 3 x 145 [8,5]
+ CRUSH rule 3 x 146 [2,8]
+ CRUSH rule 3 x 147 [2,6]
+ CRUSH rule 3 x 148 [5,2]
+ CRUSH rule 3 x 149 [5,6]
+ CRUSH rule 3 x 150 [2,8]
+ CRUSH rule 3 x 151 [5,8]
+ CRUSH rule 3 x 152 [8,5]
+ CRUSH rule 3 x 153 [8,5]
+ CRUSH rule 3 x 154 [5,2]
+ CRUSH rule 3 x 155 [5,6]
+ CRUSH rule 3 x 156 [5,2]
+ CRUSH rule 3 x 157 [5,2]
+ CRUSH rule 3 x 158 [2,6]
+ CRUSH rule 3 x 159 [7,2]
+ CRUSH rule 3 x 160 [2,8]
+ CRUSH rule 3 x 161 [2,5]
+ CRUSH rule 3 x 162 [2,8]
+ CRUSH rule 3 x 163 [5,8]
+ CRUSH rule 3 x 164 [7,2]
+ CRUSH rule 3 x 165 [7,2]
+ CRUSH rule 3 x 166 [2,5]
+ CRUSH rule 3 x 167 [2,6]
+ CRUSH rule 3 x 168 [5,2]
+ CRUSH rule 3 x 169 [2,7]
+ CRUSH rule 3 x 170 [2,5]
+ CRUSH rule 3 x 171 [7,5]
+ CRUSH rule 3 x 172 [2,8]
+ CRUSH rule 3 x 173 [8,5]
+ CRUSH rule 3 x 174 [2,5]
+ CRUSH rule 3 x 175 [6,2]
+ CRUSH rule 3 x 176 [5,2]
+ CRUSH rule 3 x 177 [5,2]
+ CRUSH rule 3 x 178 [5,2]
+ CRUSH rule 3 x 179 [5,2]
+ CRUSH rule 3 x 180 [5,7]
+ CRUSH rule 3 x 181 [6,2]
+ CRUSH rule 3 x 182 [8,5]
+ CRUSH rule 3 x 183 [7,5]
+ CRUSH rule 3 x 184 [5,6]
+ CRUSH rule 3 x 185 [6,2]
+ CRUSH rule 3 x 186 [2,5]
+ CRUSH rule 3 x 187 [2,6]
+ CRUSH rule 3 x 188 [2,6]
+ CRUSH rule 3 x 189 [2,6]
+ CRUSH rule 3 x 190 [5,2]
+ CRUSH rule 3 x 191 [7,2]
+ CRUSH rule 3 x 192 [5,2]
+ CRUSH rule 3 x 193 [5,2]
+ CRUSH rule 3 x 194 [2,5]
+ CRUSH rule 3 x 195 [6,5]
+ CRUSH rule 3 x 196 [6,2]
+ CRUSH rule 3 x 197 [6,5]
+ CRUSH rule 3 x 198 [2,5]
+ CRUSH rule 3 x 199 [2,5]
+ CRUSH rule 3 x 200 [2,5]
+ CRUSH rule 3 x 201 [7,2]
+ CRUSH rule 3 x 202 [6,5]
+ CRUSH rule 3 x 203 [5,6]
+ CRUSH rule 3 x 204 [2,5]
+ CRUSH rule 3 x 205 [2,7]
+ CRUSH rule 3 x 206 [2,8]
+ CRUSH rule 3 x 207 [5,2]
+ CRUSH rule 3 x 208 [7,2]
+ CRUSH rule 3 x 209 [2,8]
+ CRUSH rule 3 x 210 [2,5]
+ CRUSH rule 3 x 211 [5,2]
+ CRUSH rule 3 x 212 [7,5]
+ CRUSH rule 3 x 213 [8,5]
+ CRUSH rule 3 x 214 [5,7]
+ CRUSH rule 3 x 215 [8,2]
+ CRUSH rule 3 x 216 [5,2]
+ CRUSH rule 3 x 217 [2,7]
+ CRUSH rule 3 x 218 [2,7]
+ CRUSH rule 3 x 219 [5,6]
+ CRUSH rule 3 x 220 [5,8]
+ CRUSH rule 3 x 221 [5,7]
+ CRUSH rule 3 x 222 [6,5]
+ CRUSH rule 3 x 223 [2,5]
+ CRUSH rule 3 x 224 [2,5]
+ CRUSH rule 3 x 225 [8,2]
+ CRUSH rule 3 x 226 [7,2]
+ CRUSH rule 3 x 227 [5,2]
+ CRUSH rule 3 x 228 [5,6]
+ CRUSH rule 3 x 229 [5,7]
+ CRUSH rule 3 x 230 [5,6]
+ CRUSH rule 3 x 231 [5,7]
+ CRUSH rule 3 x 232 [2,8]
+ CRUSH rule 3 x 233 [5,6]
+ CRUSH rule 3 x 234 [2,5]
+ CRUSH rule 3 x 235 [5,6]
+ CRUSH rule 3 x 236 [5,2]
+ CRUSH rule 3 x 237 [5,8]
+ CRUSH rule 3 x 238 [5,2]
+ CRUSH rule 3 x 239 [8,5]
+ CRUSH rule 3 x 240 [5,8]
+ CRUSH rule 3 x 241 [5,2]
+ CRUSH rule 3 x 242 [5,2]
+ CRUSH rule 3 x 243 [5,8]
+ CRUSH rule 3 x 244 [5,6]
+ CRUSH rule 3 x 245 [7,2]
+ CRUSH rule 3 x 246 [2,5]
+ CRUSH rule 3 x 247 [6,2]
+ CRUSH rule 3 x 248 [8,2]
+ CRUSH rule 3 x 249 [2,5]
+ CRUSH rule 3 x 250 [2,5]
+ CRUSH rule 3 x 251 [2,5]
+ CRUSH rule 3 x 252 [5,7]
+ CRUSH rule 3 x 253 [5,2]
+ CRUSH rule 3 x 254 [5,2]
+ CRUSH rule 3 x 255 [2,7]
+ CRUSH rule 3 x 256 [5,6]
+ CRUSH rule 3 x 257 [2,6]
+ CRUSH rule 3 x 258 [5,2]
+ CRUSH rule 3 x 259 [5,6]
+ CRUSH rule 3 x 260 [5,8]
+ CRUSH rule 3 x 261 [8,5]
+ CRUSH rule 3 x 262 [5,6]
+ CRUSH rule 3 x 263 [6,2]
+ CRUSH rule 3 x 264 [5,6]
+ CRUSH rule 3 x 265 [8,5]
+ CRUSH rule 3 x 266 [8,2]
+ CRUSH rule 3 x 267 [2,5]
+ CRUSH rule 3 x 268 [2,6]
+ CRUSH rule 3 x 269 [2,6]
+ CRUSH rule 3 x 270 [5,2]
+ CRUSH rule 3 x 271 [7,5]
+ CRUSH rule 3 x 272 [2,6]
+ CRUSH rule 3 x 273 [5,2]
+ CRUSH rule 3 x 274 [6,5]
+ CRUSH rule 3 x 275 [5,8]
+ CRUSH rule 3 x 276 [7,2]
+ CRUSH rule 3 x 277 [6,5]
+ CRUSH rule 3 x 278 [6,2]
+ CRUSH rule 3 x 279 [8,5]
+ CRUSH rule 3 x 280 [2,7]
+ CRUSH rule 3 x 281 [8,2]
+ CRUSH rule 3 x 282 [5,2]
+ CRUSH rule 3 x 283 [8,2]
+ CRUSH rule 3 x 284 [6,5]
+ CRUSH rule 3 x 285 [5,7]
+ CRUSH rule 3 x 286 [2,8]
+ CRUSH rule 3 x 287 [2,5]
+ CRUSH rule 3 x 288 [8,2]
+ CRUSH rule 3 x 289 [5,6]
+ CRUSH rule 3 x 290 [2,5]
+ CRUSH rule 3 x 291 [2,5]
+ CRUSH rule 3 x 292 [8,2]
+ CRUSH rule 3 x 293 [6,2]
+ CRUSH rule 3 x 294 [7,5]
+ CRUSH rule 3 x 295 [5,6]
+ CRUSH rule 3 x 296 [5,2]
+ CRUSH rule 3 x 297 [6,2]
+ CRUSH rule 3 x 298 [2,5]
+ CRUSH rule 3 x 299 [2,8]
+ CRUSH rule 3 x 300 [8,5]
+ CRUSH rule 3 x 301 [2,7]
+ CRUSH rule 3 x 302 [5,2]
+ CRUSH rule 3 x 303 [7,5]
+ CRUSH rule 3 x 304 [2,6]
+ CRUSH rule 3 x 305 [5,6]
+ CRUSH rule 3 x 306 [2,8]
+ CRUSH rule 3 x 307 [2,7]
+ CRUSH rule 3 x 308 [2,8]
+ CRUSH rule 3 x 309 [7,5]
+ CRUSH rule 3 x 310 [5,2]
+ CRUSH rule 3 x 311 [5,8]
+ CRUSH rule 3 x 312 [2,6]
+ CRUSH rule 3 x 313 [5,2]
+ CRUSH rule 3 x 314 [5,2]
+ CRUSH rule 3 x 315 [2,5]
+ CRUSH rule 3 x 316 [6,5]
+ CRUSH rule 3 x 317 [2,7]
+ CRUSH rule 3 x 318 [8,2]
+ CRUSH rule 3 x 319 [5,2]
+ CRUSH rule 3 x 320 [5,8]
+ CRUSH rule 3 x 321 [2,5]
+ CRUSH rule 3 x 322 [2,6]
+ CRUSH rule 3 x 323 [5,7]
+ CRUSH rule 3 x 324 [7,2]
+ CRUSH rule 3 x 325 [5,6]
+ CRUSH rule 3 x 326 [5,2]
+ CRUSH rule 3 x 327 [2,8]
+ CRUSH rule 3 x 328 [7,5]
+ CRUSH rule 3 x 329 [5,7]
+ CRUSH rule 3 x 330 [5,7]
+ CRUSH rule 3 x 331 [2,7]
+ CRUSH rule 3 x 332 [2,5]
+ CRUSH rule 3 x 333 [6,5]
+ CRUSH rule 3 x 334 [8,5]
+ CRUSH rule 3 x 335 [7,2]
+ CRUSH rule 3 x 336 [5,6]
+ CRUSH rule 3 x 337 [7,2]
+ CRUSH rule 3 x 338 [5,8]
+ CRUSH rule 3 x 339 [7,5]
+ CRUSH rule 3 x 340 [2,6]
+ CRUSH rule 3 x 341 [5,2]
+ CRUSH rule 3 x 342 [2,8]
+ CRUSH rule 3 x 343 [6,5]
+ CRUSH rule 3 x 344 [6,2]
+ CRUSH rule 3 x 345 [5,7]
+ CRUSH rule 3 x 346 [8,2]
+ CRUSH rule 3 x 347 [5,2]
+ CRUSH rule 3 x 348 [8,2]
+ CRUSH rule 3 x 349 [2,7]
+ CRUSH rule 3 x 350 [8,5]
+ CRUSH rule 3 x 351 [5,8]
+ CRUSH rule 3 x 352 [2,8]
+ CRUSH rule 3 x 353 [6,5]
+ CRUSH rule 3 x 354 [2,5]
+ CRUSH rule 3 x 355 [5,8]
+ CRUSH rule 3 x 356 [5,2]
+ CRUSH rule 3 x 357 [6,2]
+ CRUSH rule 3 x 358 [2,8]
+ CRUSH rule 3 x 359 [6,2]
+ CRUSH rule 3 x 360 [5,2]
+ CRUSH rule 3 x 361 [8,5]
+ CRUSH rule 3 x 362 [5,2]
+ CRUSH rule 3 x 363 [5,2]
+ CRUSH rule 3 x 364 [2,5]
+ CRUSH rule 3 x 365 [6,5]
+ CRUSH rule 3 x 366 [7,2]
+ CRUSH rule 3 x 367 [5,2]
+ CRUSH rule 3 x 368 [7,5]
+ CRUSH rule 3 x 369 [5,7]
+ CRUSH rule 3 x 370 [8,2]
+ CRUSH rule 3 x 371 [2,5]
+ CRUSH rule 3 x 372 [5,2]
+ CRUSH rule 3 x 373 [2,6]
+ CRUSH rule 3 x 374 [5,8]
+ CRUSH rule 3 x 375 [6,5]
+ CRUSH rule 3 x 376 [7,2]
+ CRUSH rule 3 x 377 [2,5]
+ CRUSH rule 3 x 378 [2,6]
+ CRUSH rule 3 x 379 [8,5]
+ CRUSH rule 3 x 380 [2,5]
+ CRUSH rule 3 x 381 [2,5]
+ CRUSH rule 3 x 382 [2,5]
+ CRUSH rule 3 x 383 [5,7]
+ CRUSH rule 3 x 384 [7,2]
+ CRUSH rule 3 x 385 [7,5]
+ CRUSH rule 3 x 386 [2,5]
+ CRUSH rule 3 x 387 [2,5]
+ CRUSH rule 3 x 388 [5,2]
+ CRUSH rule 3 x 389 [2,5]
+ CRUSH rule 3 x 390 [5,8]
+ CRUSH rule 3 x 391 [5,6]
+ CRUSH rule 3 x 392 [2,7]
+ CRUSH rule 3 x 393 [5,2]
+ CRUSH rule 3 x 394 [5,8]
+ CRUSH rule 3 x 395 [5,2]
+ CRUSH rule 3 x 396 [5,2]
+ CRUSH rule 3 x 397 [2,5]
+ CRUSH rule 3 x 398 [2,5]
+ CRUSH rule 3 x 399 [8,5]
+ CRUSH rule 3 x 400 [8,2]
+ CRUSH rule 3 x 401 [2,5]
+ CRUSH rule 3 x 402 [7,5]
+ CRUSH rule 3 x 403 [2,5]
+ CRUSH rule 3 x 404 [5,2]
+ CRUSH rule 3 x 405 [6,5]
+ CRUSH rule 3 x 406 [2,6]
+ CRUSH rule 3 x 407 [2,7]
+ CRUSH rule 3 x 408 [5,2]
+ CRUSH rule 3 x 409 [7,5]
+ CRUSH rule 3 x 410 [8,5]
+ CRUSH rule 3 x 411 [2,7]
+ CRUSH rule 3 x 412 [2,5]
+ CRUSH rule 3 x 413 [5,2]
+ CRUSH rule 3 x 414 [5,2]
+ CRUSH rule 3 x 415 [2,6]
+ CRUSH rule 3 x 416 [2,8]
+ CRUSH rule 3 x 417 [8,2]
+ CRUSH rule 3 x 418 [7,2]
+ CRUSH rule 3 x 419 [8,5]
+ CRUSH rule 3 x 420 [2,5]
+ CRUSH rule 3 x 421 [8,5]
+ CRUSH rule 3 x 422 [6,5]
+ CRUSH rule 3 x 423 [2,5]
+ CRUSH rule 3 x 424 [8,5]
+ CRUSH rule 3 x 425 [2,5]
+ CRUSH rule 3 x 426 [6,2]
+ CRUSH rule 3 x 427 [2,8]
+ CRUSH rule 3 x 428 [5,8]
+ CRUSH rule 3 x 429 [5,8]
+ CRUSH rule 3 x 430 [5,7]
+ CRUSH rule 3 x 431 [5,2]
+ CRUSH rule 3 x 432 [7,2]
+ CRUSH rule 3 x 433 [6,5]
+ CRUSH rule 3 x 434 [5,2]
+ CRUSH rule 3 x 435 [2,5]
+ CRUSH rule 3 x 436 [5,2]
+ CRUSH rule 3 x 437 [7,5]
+ CRUSH rule 3 x 438 [2,5]
+ CRUSH rule 3 x 439 [2,5]
+ CRUSH rule 3 x 440 [2,6]
+ CRUSH rule 3 x 441 [5,8]
+ CRUSH rule 3 x 442 [2,5]
+ CRUSH rule 3 x 443 [6,2]
+ CRUSH rule 3 x 444 [7,2]
+ CRUSH rule 3 x 445 [6,5]
+ CRUSH rule 3 x 446 [5,2]
+ CRUSH rule 3 x 447 [2,5]
+ CRUSH rule 3 x 448 [7,2]
+ CRUSH rule 3 x 449 [7,5]
+ CRUSH rule 3 x 450 [5,2]
+ CRUSH rule 3 x 451 [6,5]
+ CRUSH rule 3 x 452 [8,5]
+ CRUSH rule 3 x 453 [6,5]
+ CRUSH rule 3 x 454 [6,5]
+ CRUSH rule 3 x 455 [2,8]
+ CRUSH rule 3 x 456 [6,2]
+ CRUSH rule 3 x 457 [7,2]
+ CRUSH rule 3 x 458 [2,8]
+ CRUSH rule 3 x 459 [2,6]
+ CRUSH rule 3 x 460 [6,5]
+ CRUSH rule 3 x 461 [6,5]
+ CRUSH rule 3 x 462 [8,2]
+ CRUSH rule 3 x 463 [6,2]
+ CRUSH rule 3 x 464 [7,5]
+ CRUSH rule 3 x 465 [7,2]
+ CRUSH rule 3 x 466 [5,6]
+ CRUSH rule 3 x 467 [6,5]
+ CRUSH rule 3 x 468 [7,2]
+ CRUSH rule 3 x 469 [7,2]
+ CRUSH rule 3 x 470 [5,2]
+ CRUSH rule 3 x 471 [2,6]
+ CRUSH rule 3 x 472 [5,2]
+ CRUSH rule 3 x 473 [2,5]
+ CRUSH rule 3 x 474 [6,2]
+ CRUSH rule 3 x 475 [6,2]
+ CRUSH rule 3 x 476 [5,7]
+ CRUSH rule 3 x 477 [5,6]
+ CRUSH rule 3 x 478 [6,2]
+ CRUSH rule 3 x 479 [2,5]
+ CRUSH rule 3 x 480 [2,6]
+ CRUSH rule 3 x 481 [2,5]
+ CRUSH rule 3 x 482 [5,8]
+ CRUSH rule 3 x 483 [2,6]
+ CRUSH rule 3 x 484 [2,8]
+ CRUSH rule 3 x 485 [5,8]
+ CRUSH rule 3 x 486 [5,2]
+ CRUSH rule 3 x 487 [5,2]
+ CRUSH rule 3 x 488 [5,7]
+ CRUSH rule 3 x 489 [2,8]
+ CRUSH rule 3 x 490 [6,5]
+ CRUSH rule 3 x 491 [2,6]
+ CRUSH rule 3 x 492 [6,5]
+ CRUSH rule 3 x 493 [2,8]
+ CRUSH rule 3 x 494 [2,6]
+ CRUSH rule 3 x 495 [5,2]
+ CRUSH rule 3 x 496 [7,5]
+ CRUSH rule 3 x 497 [5,7]
+ CRUSH rule 3 x 498 [2,5]
+ CRUSH rule 3 x 499 [8,5]
+ CRUSH rule 3 x 500 [5,6]
+ CRUSH rule 3 x 501 [2,7]
+ CRUSH rule 3 x 502 [7,2]
+ CRUSH rule 3 x 503 [2,5]
+ CRUSH rule 3 x 504 [5,8]
+ CRUSH rule 3 x 505 [2,7]
+ CRUSH rule 3 x 506 [5,2]
+ CRUSH rule 3 x 507 [6,2]
+ CRUSH rule 3 x 508 [2,5]
+ CRUSH rule 3 x 509 [7,5]
+ CRUSH rule 3 x 510 [6,2]
+ CRUSH rule 3 x 511 [5,6]
+ CRUSH rule 3 x 512 [7,2]
+ CRUSH rule 3 x 513 [7,2]
+ CRUSH rule 3 x 514 [5,7]
+ CRUSH rule 3 x 515 [8,5]
+ CRUSH rule 3 x 516 [5,2]
+ CRUSH rule 3 x 517 [7,2]
+ CRUSH rule 3 x 518 [5,6]
+ CRUSH rule 3 x 519 [7,5]
+ CRUSH rule 3 x 520 [2,8]
+ CRUSH rule 3 x 521 [8,2]
+ CRUSH rule 3 x 522 [6,2]
+ CRUSH rule 3 x 523 [5,2]
+ CRUSH rule 3 x 524 [2,5]
+ CRUSH rule 3 x 525 [2,5]
+ CRUSH rule 3 x 526 [2,5]
+ CRUSH rule 3 x 527 [2,5]
+ CRUSH rule 3 x 528 [5,2]
+ CRUSH rule 3 x 529 [5,6]
+ CRUSH rule 3 x 530 [6,5]
+ CRUSH rule 3 x 531 [6,2]
+ CRUSH rule 3 x 532 [6,5]
+ CRUSH rule 3 x 533 [5,8]
+ CRUSH rule 3 x 534 [7,5]
+ CRUSH rule 3 x 535 [8,2]
+ CRUSH rule 3 x 536 [6,2]
+ CRUSH rule 3 x 537 [5,8]
+ CRUSH rule 3 x 538 [6,5]
+ CRUSH rule 3 x 539 [8,5]
+ CRUSH rule 3 x 540 [2,7]
+ CRUSH rule 3 x 541 [2,5]
+ CRUSH rule 3 x 542 [5,2]
+ CRUSH rule 3 x 543 [6,2]
+ CRUSH rule 3 x 544 [5,7]
+ CRUSH rule 3 x 545 [5,7]
+ CRUSH rule 3 x 546 [6,2]
+ CRUSH rule 3 x 547 [8,2]
+ CRUSH rule 3 x 548 [5,2]
+ CRUSH rule 3 x 549 [5,7]
+ CRUSH rule 3 x 550 [2,5]
+ CRUSH rule 3 x 551 [7,5]
+ CRUSH rule 3 x 552 [5,7]
+ CRUSH rule 3 x 553 [5,2]
+ CRUSH rule 3 x 554 [2,6]
+ CRUSH rule 3 x 555 [5,2]
+ CRUSH rule 3 x 556 [5,6]
+ CRUSH rule 3 x 557 [7,5]
+ CRUSH rule 3 x 558 [5,2]
+ CRUSH rule 3 x 559 [5,2]
+ CRUSH rule 3 x 560 [8,5]
+ CRUSH rule 3 x 561 [6,5]
+ CRUSH rule 3 x 562 [5,2]
+ CRUSH rule 3 x 563 [2,7]
+ CRUSH rule 3 x 564 [5,2]
+ CRUSH rule 3 x 565 [5,8]
+ CRUSH rule 3 x 566 [5,6]
+ CRUSH rule 3 x 567 [5,7]
+ CRUSH rule 3 x 568 [7,5]
+ CRUSH rule 3 x 569 [5,2]
+ CRUSH rule 3 x 570 [2,5]
+ CRUSH rule 3 x 571 [5,6]
+ CRUSH rule 3 x 572 [5,2]
+ CRUSH rule 3 x 573 [5,2]
+ CRUSH rule 3 x 574 [2,5]
+ CRUSH rule 3 x 575 [8,2]
+ CRUSH rule 3 x 576 [5,7]
+ CRUSH rule 3 x 577 [8,2]
+ CRUSH rule 3 x 578 [6,2]
+ CRUSH rule 3 x 579 [5,2]
+ CRUSH rule 3 x 580 [5,2]
+ CRUSH rule 3 x 581 [7,2]
+ CRUSH rule 3 x 582 [2,8]
+ CRUSH rule 3 x 583 [6,2]
+ CRUSH rule 3 x 584 [8,2]
+ CRUSH rule 3 x 585 [7,2]
+ CRUSH rule 3 x 586 [2,8]
+ CRUSH rule 3 x 587 [2,5]
+ CRUSH rule 3 x 588 [5,8]
+ CRUSH rule 3 x 589 [7,2]
+ CRUSH rule 3 x 590 [6,2]
+ CRUSH rule 3 x 591 [5,2]
+ CRUSH rule 3 x 592 [2,5]
+ CRUSH rule 3 x 593 [2,8]
+ CRUSH rule 3 x 594 [2,6]
+ CRUSH rule 3 x 595 [7,2]
+ CRUSH rule 3 x 596 [5,2]
+ CRUSH rule 3 x 597 [5,2]
+ CRUSH rule 3 x 598 [5,2]
+ CRUSH rule 3 x 599 [5,2]
+ CRUSH rule 3 x 600 [7,2]
+ CRUSH rule 3 x 601 [2,6]
+ CRUSH rule 3 x 602 [5,8]
+ CRUSH rule 3 x 603 [5,2]
+ CRUSH rule 3 x 604 [7,5]
+ CRUSH rule 3 x 605 [5,2]
+ CRUSH rule 3 x 606 [2,7]
+ CRUSH rule 3 x 607 [2,5]
+ CRUSH rule 3 x 608 [5,2]
+ CRUSH rule 3 x 609 [5,2]
+ CRUSH rule 3 x 610 [5,8]
+ CRUSH rule 3 x 611 [2,8]
+ CRUSH rule 3 x 612 [2,8]
+ CRUSH rule 3 x 613 [7,2]
+ CRUSH rule 3 x 614 [7,2]
+ CRUSH rule 3 x 615 [6,2]
+ CRUSH rule 3 x 616 [2,7]
+ CRUSH rule 3 x 617 [6,2]
+ CRUSH rule 3 x 618 [7,5]
+ CRUSH rule 3 x 619 [5,2]
+ CRUSH rule 3 x 620 [5,2]
+ CRUSH rule 3 x 621 [5,6]
+ CRUSH rule 3 x 622 [2,5]
+ CRUSH rule 3 x 623 [2,8]
+ CRUSH rule 3 x 624 [5,2]
+ CRUSH rule 3 x 625 [2,5]
+ CRUSH rule 3 x 626 [7,2]
+ CRUSH rule 3 x 627 [2,6]
+ CRUSH rule 3 x 628 [8,2]
+ CRUSH rule 3 x 629 [2,6]
+ CRUSH rule 3 x 630 [2,6]
+ CRUSH rule 3 x 631 [2,6]
+ CRUSH rule 3 x 632 [7,2]
+ CRUSH rule 3 x 633 [8,5]
+ CRUSH rule 3 x 634 [2,5]
+ CRUSH rule 3 x 635 [5,6]
+ CRUSH rule 3 x 636 [2,5]
+ CRUSH rule 3 x 637 [5,2]
+ CRUSH rule 3 x 638 [6,2]
+ CRUSH rule 3 x 639 [5,2]
+ CRUSH rule 3 x 640 [5,2]
+ CRUSH rule 3 x 641 [7,2]
+ CRUSH rule 3 x 642 [2,8]
+ CRUSH rule 3 x 643 [5,2]
+ CRUSH rule 3 x 644 [8,2]
+ CRUSH rule 3 x 645 [5,7]
+ CRUSH rule 3 x 646 [8,2]
+ CRUSH rule 3 x 647 [7,2]
+ CRUSH rule 3 x 648 [2,8]
+ CRUSH rule 3 x 649 [5,7]
+ CRUSH rule 3 x 650 [7,5]
+ CRUSH rule 3 x 651 [5,6]
+ CRUSH rule 3 x 652 [5,6]
+ CRUSH rule 3 x 653 [8,5]
+ CRUSH rule 3 x 654 [7,5]
+ CRUSH rule 3 x 655 [2,5]
+ CRUSH rule 3 x 656 [5,7]
+ CRUSH rule 3 x 657 [6,2]
+ CRUSH rule 3 x 658 [5,8]
+ CRUSH rule 3 x 659 [5,7]
+ CRUSH rule 3 x 660 [7,5]
+ CRUSH rule 3 x 661 [2,7]
+ CRUSH rule 3 x 662 [5,2]
+ CRUSH rule 3 x 663 [2,5]
+ CRUSH rule 3 x 664 [2,5]
+ CRUSH rule 3 x 665 [5,6]
+ CRUSH rule 3 x 666 [2,7]
+ CRUSH rule 3 x 667 [2,5]
+ CRUSH rule 3 x 668 [5,7]
+ CRUSH rule 3 x 669 [6,5]
+ CRUSH rule 3 x 670 [5,2]
+ CRUSH rule 3 x 671 [2,8]
+ CRUSH rule 3 x 672 [5,2]
+ CRUSH rule 3 x 673 [5,2]
+ CRUSH rule 3 x 674 [5,2]
+ CRUSH rule 3 x 675 [2,8]
+ CRUSH rule 3 x 676 [2,5]
+ CRUSH rule 3 x 677 [5,2]
+ CRUSH rule 3 x 678 [2,5]
+ CRUSH rule 3 x 679 [6,2]
+ CRUSH rule 3 x 680 [2,5]
+ CRUSH rule 3 x 681 [5,6]
+ CRUSH rule 3 x 682 [2,5]
+ CRUSH rule 3 x 683 [2,5]
+ CRUSH rule 3 x 684 [7,2]
+ CRUSH rule 3 x 685 [7,2]
+ CRUSH rule 3 x 686 [2,5]
+ CRUSH rule 3 x 687 [5,7]
+ CRUSH rule 3 x 688 [5,6]
+ CRUSH rule 3 x 689 [6,5]
+ CRUSH rule 3 x 690 [8,2]
+ CRUSH rule 3 x 691 [5,2]
+ CRUSH rule 3 x 692 [7,2]
+ CRUSH rule 3 x 693 [6,5]
+ CRUSH rule 3 x 694 [6,5]
+ CRUSH rule 3 x 695 [2,6]
+ CRUSH rule 3 x 696 [2,5]
+ CRUSH rule 3 x 697 [6,2]
+ CRUSH rule 3 x 698 [6,2]
+ CRUSH rule 3 x 699 [2,8]
+ CRUSH rule 3 x 700 [2,5]
+ CRUSH rule 3 x 701 [5,2]
+ CRUSH rule 3 x 702 [5,2]
+ CRUSH rule 3 x 703 [8,5]
+ CRUSH rule 3 x 704 [2,5]
+ CRUSH rule 3 x 705 [8,2]
+ CRUSH rule 3 x 706 [2,5]
+ CRUSH rule 3 x 707 [7,5]
+ CRUSH rule 3 x 708 [5,7]
+ CRUSH rule 3 x 709 [6,5]
+ CRUSH rule 3 x 710 [8,5]
+ CRUSH rule 3 x 711 [2,5]
+ CRUSH rule 3 x 712 [2,5]
+ CRUSH rule 3 x 713 [6,5]
+ CRUSH rule 3 x 714 [5,2]
+ CRUSH rule 3 x 715 [2,5]
+ CRUSH rule 3 x 716 [5,6]
+ CRUSH rule 3 x 717 [8,2]
+ CRUSH rule 3 x 718 [5,7]
+ CRUSH rule 3 x 719 [2,6]
+ CRUSH rule 3 x 720 [6,2]
+ CRUSH rule 3 x 721 [5,7]
+ CRUSH rule 3 x 722 [5,7]
+ CRUSH rule 3 x 723 [5,2]
+ CRUSH rule 3 x 724 [2,7]
+ CRUSH rule 3 x 725 [2,5]
+ CRUSH rule 3 x 726 [5,7]
+ CRUSH rule 3 x 727 [5,7]
+ CRUSH rule 3 x 728 [2,6]
+ CRUSH rule 3 x 729 [5,6]
+ CRUSH rule 3 x 730 [5,8]
+ CRUSH rule 3 x 731 [5,2]
+ CRUSH rule 3 x 732 [2,5]
+ CRUSH rule 3 x 733 [5,6]
+ CRUSH rule 3 x 734 [6,5]
+ CRUSH rule 3 x 735 [5,6]
+ CRUSH rule 3 x 736 [5,8]
+ CRUSH rule 3 x 737 [2,8]
+ CRUSH rule 3 x 738 [5,2]
+ CRUSH rule 3 x 739 [2,7]
+ CRUSH rule 3 x 740 [2,7]
+ CRUSH rule 3 x 741 [7,2]
+ CRUSH rule 3 x 742 [8,2]
+ CRUSH rule 3 x 743 [7,2]
+ CRUSH rule 3 x 744 [5,6]
+ CRUSH rule 3 x 745 [5,2]
+ CRUSH rule 3 x 746 [5,2]
+ CRUSH rule 3 x 747 [6,2]
+ CRUSH rule 3 x 748 [2,6]
+ CRUSH rule 3 x 749 [5,7]
+ CRUSH rule 3 x 750 [2,7]
+ CRUSH rule 3 x 751 [2,7]
+ CRUSH rule 3 x 752 [8,2]
+ CRUSH rule 3 x 753 [7,5]
+ CRUSH rule 3 x 754 [8,5]
+ CRUSH rule 3 x 755 [2,6]
+ CRUSH rule 3 x 756 [5,8]
+ CRUSH rule 3 x 757 [8,2]
+ CRUSH rule 3 x 758 [6,2]
+ CRUSH rule 3 x 759 [8,5]
+ CRUSH rule 3 x 760 [2,5]
+ CRUSH rule 3 x 761 [5,2]
+ CRUSH rule 3 x 762 [2,8]
+ CRUSH rule 3 x 763 [8,5]
+ CRUSH rule 3 x 764 [2,7]
+ CRUSH rule 3 x 765 [6,5]
+ CRUSH rule 3 x 766 [8,5]
+ CRUSH rule 3 x 767 [2,8]
+ CRUSH rule 3 x 768 [8,5]
+ CRUSH rule 3 x 769 [6,2]
+ CRUSH rule 3 x 770 [6,2]
+ CRUSH rule 3 x 771 [7,2]
+ CRUSH rule 3 x 772 [8,5]
+ CRUSH rule 3 x 773 [5,2]
+ CRUSH rule 3 x 774 [5,7]
+ CRUSH rule 3 x 775 [6,5]
+ CRUSH rule 3 x 776 [7,2]
+ CRUSH rule 3 x 777 [5,2]
+ CRUSH rule 3 x 778 [2,6]
+ CRUSH rule 3 x 779 [2,6]
+ CRUSH rule 3 x 780 [2,5]
+ CRUSH rule 3 x 781 [6,5]
+ CRUSH rule 3 x 782 [5,2]
+ CRUSH rule 3 x 783 [7,2]
+ CRUSH rule 3 x 784 [2,5]
+ CRUSH rule 3 x 785 [6,2]
+ CRUSH rule 3 x 786 [7,5]
+ CRUSH rule 3 x 787 [2,8]
+ CRUSH rule 3 x 788 [6,2]
+ CRUSH rule 3 x 789 [2,5]
+ CRUSH rule 3 x 790 [8,5]
+ CRUSH rule 3 x 791 [5,6]
+ CRUSH rule 3 x 792 [5,6]
+ CRUSH rule 3 x 793 [6,2]
+ CRUSH rule 3 x 794 [2,8]
+ CRUSH rule 3 x 795 [2,5]
+ CRUSH rule 3 x 796 [5,7]
+ CRUSH rule 3 x 797 [2,5]
+ CRUSH rule 3 x 798 [6,2]
+ CRUSH rule 3 x 799 [5,2]
+ CRUSH rule 3 x 800 [5,2]
+ CRUSH rule 3 x 801 [5,7]
+ CRUSH rule 3 x 802 [2,6]
+ CRUSH rule 3 x 803 [2,5]
+ CRUSH rule 3 x 804 [6,2]
+ CRUSH rule 3 x 805 [5,8]
+ CRUSH rule 3 x 806 [2,5]
+ CRUSH rule 3 x 807 [5,7]
+ CRUSH rule 3 x 808 [5,7]
+ CRUSH rule 3 x 809 [2,5]
+ CRUSH rule 3 x 810 [5,7]
+ CRUSH rule 3 x 811 [8,5]
+ CRUSH rule 3 x 812 [8,5]
+ CRUSH rule 3 x 813 [6,5]
+ CRUSH rule 3 x 814 [5,8]
+ CRUSH rule 3 x 815 [5,2]
+ CRUSH rule 3 x 816 [2,6]
+ CRUSH rule 3 x 817 [5,6]
+ CRUSH rule 3 x 818 [5,2]
+ CRUSH rule 3 x 819 [5,2]
+ CRUSH rule 3 x 820 [5,7]
+ CRUSH rule 3 x 821 [5,8]
+ CRUSH rule 3 x 822 [2,5]
+ CRUSH rule 3 x 823 [5,7]
+ CRUSH rule 3 x 824 [5,7]
+ CRUSH rule 3 x 825 [2,8]
+ CRUSH rule 3 x 826 [7,2]
+ CRUSH rule 3 x 827 [2,6]
+ CRUSH rule 3 x 828 [2,5]
+ CRUSH rule 3 x 829 [5,6]
+ CRUSH rule 3 x 830 [2,5]
+ CRUSH rule 3 x 831 [2,6]
+ CRUSH rule 3 x 832 [5,8]
+ CRUSH rule 3 x 833 [2,6]
+ CRUSH rule 3 x 834 [5,2]
+ CRUSH rule 3 x 835 [8,5]
+ CRUSH rule 3 x 836 [5,8]
+ CRUSH rule 3 x 837 [6,5]
+ CRUSH rule 3 x 838 [6,2]
+ CRUSH rule 3 x 839 [5,2]
+ CRUSH rule 3 x 840 [7,5]
+ CRUSH rule 3 x 841 [5,8]
+ CRUSH rule 3 x 842 [2,5]
+ CRUSH rule 3 x 843 [6,5]
+ CRUSH rule 3 x 844 [5,8]
+ CRUSH rule 3 x 845 [5,6]
+ CRUSH rule 3 x 846 [5,2]
+ CRUSH rule 3 x 847 [2,8]
+ CRUSH rule 3 x 848 [2,6]
+ CRUSH rule 3 x 849 [5,8]
+ CRUSH rule 3 x 850 [2,5]
+ CRUSH rule 3 x 851 [6,5]
+ CRUSH rule 3 x 852 [7,5]
+ CRUSH rule 3 x 853 [6,2]
+ CRUSH rule 3 x 854 [7,2]
+ CRUSH rule 3 x 855 [5,7]
+ CRUSH rule 3 x 856 [6,5]
+ CRUSH rule 3 x 857 [8,5]
+ CRUSH rule 3 x 858 [6,5]
+ CRUSH rule 3 x 859 [6,2]
+ CRUSH rule 3 x 860 [5,2]
+ CRUSH rule 3 x 861 [8,5]
+ CRUSH rule 3 x 862 [6,2]
+ CRUSH rule 3 x 863 [8,2]
+ CRUSH rule 3 x 864 [5,6]
+ CRUSH rule 3 x 865 [8,2]
+ CRUSH rule 3 x 866 [5,6]
+ CRUSH rule 3 x 867 [6,5]
+ CRUSH rule 3 x 868 [6,5]
+ CRUSH rule 3 x 869 [8,5]
+ CRUSH rule 3 x 870 [2,5]
+ CRUSH rule 3 x 871 [5,2]
+ CRUSH rule 3 x 872 [5,2]
+ CRUSH rule 3 x 873 [5,6]
+ CRUSH rule 3 x 874 [2,6]
+ CRUSH rule 3 x 875 [2,6]
+ CRUSH rule 3 x 876 [5,8]
+ CRUSH rule 3 x 877 [6,5]
+ CRUSH rule 3 x 878 [5,2]
+ CRUSH rule 3 x 879 [7,5]
+ CRUSH rule 3 x 880 [5,2]
+ CRUSH rule 3 x 881 [5,8]
+ CRUSH rule 3 x 882 [5,2]
+ CRUSH rule 3 x 883 [2,5]
+ CRUSH rule 3 x 884 [6,2]
+ CRUSH rule 3 x 885 [5,2]
+ CRUSH rule 3 x 886 [5,7]
+ CRUSH rule 3 x 887 [7,5]
+ CRUSH rule 3 x 888 [6,2]
+ CRUSH rule 3 x 889 [2,8]
+ CRUSH rule 3 x 890 [7,2]
+ CRUSH rule 3 x 891 [2,7]
+ CRUSH rule 3 x 892 [6,2]
+ CRUSH rule 3 x 893 [2,5]
+ CRUSH rule 3 x 894 [7,5]
+ CRUSH rule 3 x 895 [5,2]
+ CRUSH rule 3 x 896 [2,6]
+ CRUSH rule 3 x 897 [5,2]
+ CRUSH rule 3 x 898 [2,5]
+ CRUSH rule 3 x 899 [2,6]
+ CRUSH rule 3 x 900 [5,2]
+ CRUSH rule 3 x 901 [5,2]
+ CRUSH rule 3 x 902 [8,5]
+ CRUSH rule 3 x 903 [5,6]
+ CRUSH rule 3 x 904 [5,7]
+ CRUSH rule 3 x 905 [6,2]
+ CRUSH rule 3 x 906 [2,7]
+ CRUSH rule 3 x 907 [7,2]
+ CRUSH rule 3 x 908 [5,6]
+ CRUSH rule 3 x 909 [2,5]
+ CRUSH rule 3 x 910 [6,5]
+ CRUSH rule 3 x 911 [5,7]
+ CRUSH rule 3 x 912 [2,8]
+ CRUSH rule 3 x 913 [7,2]
+ CRUSH rule 3 x 914 [6,5]
+ CRUSH rule 3 x 915 [8,2]
+ CRUSH rule 3 x 916 [5,2]
+ CRUSH rule 3 x 917 [2,5]
+ CRUSH rule 3 x 918 [8,2]
+ CRUSH rule 3 x 919 [6,2]
+ CRUSH rule 3 x 920 [7,5]
+ CRUSH rule 3 x 921 [2,5]
+ CRUSH rule 3 x 922 [6,5]
+ CRUSH rule 3 x 923 [5,6]
+ CRUSH rule 3 x 924 [5,2]
+ CRUSH rule 3 x 925 [5,6]
+ CRUSH rule 3 x 926 [5,2]
+ CRUSH rule 3 x 927 [2,6]
+ CRUSH rule 3 x 928 [8,2]
+ CRUSH rule 3 x 929 [5,2]
+ CRUSH rule 3 x 930 [2,5]
+ CRUSH rule 3 x 931 [5,2]
+ CRUSH rule 3 x 932 [5,2]
+ CRUSH rule 3 x 933 [8,5]
+ CRUSH rule 3 x 934 [5,8]
+ CRUSH rule 3 x 935 [6,5]
+ CRUSH rule 3 x 936 [2,7]
+ CRUSH rule 3 x 937 [5,8]
+ CRUSH rule 3 x 938 [6,5]
+ CRUSH rule 3 x 939 [2,8]
+ CRUSH rule 3 x 940 [8,5]
+ CRUSH rule 3 x 941 [5,2]
+ CRUSH rule 3 x 942 [2,6]
+ CRUSH rule 3 x 943 [8,2]
+ CRUSH rule 3 x 944 [5,8]
+ CRUSH rule 3 x 945 [7,2]
+ CRUSH rule 3 x 946 [2,8]
+ CRUSH rule 3 x 947 [5,2]
+ CRUSH rule 3 x 948 [7,5]
+ CRUSH rule 3 x 949 [6,2]
+ CRUSH rule 3 x 950 [5,7]
+ CRUSH rule 3 x 951 [5,6]
+ CRUSH rule 3 x 952 [2,7]
+ CRUSH rule 3 x 953 [2,5]
+ CRUSH rule 3 x 954 [5,2]
+ CRUSH rule 3 x 955 [8,2]
+ CRUSH rule 3 x 956 [2,7]
+ CRUSH rule 3 x 957 [7,2]
+ CRUSH rule 3 x 958 [8,5]
+ CRUSH rule 3 x 959 [5,2]
+ CRUSH rule 3 x 960 [5,6]
+ CRUSH rule 3 x 961 [5,2]
+ CRUSH rule 3 x 962 [7,5]
+ CRUSH rule 3 x 963 [2,5]
+ CRUSH rule 3 x 964 [5,2]
+ CRUSH rule 3 x 965 [7,5]
+ CRUSH rule 3 x 966 [5,6]
+ CRUSH rule 3 x 967 [8,5]
+ CRUSH rule 3 x 968 [7,2]
+ CRUSH rule 3 x 969 [8,2]
+ CRUSH rule 3 x 970 [2,8]
+ CRUSH rule 3 x 971 [2,8]
+ CRUSH rule 3 x 972 [2,7]
+ CRUSH rule 3 x 973 [2,8]
+ CRUSH rule 3 x 974 [5,2]
+ CRUSH rule 3 x 975 [5,8]
+ CRUSH rule 3 x 976 [5,7]
+ CRUSH rule 3 x 977 [8,5]
+ CRUSH rule 3 x 978 [7,2]
+ CRUSH rule 3 x 979 [7,2]
+ CRUSH rule 3 x 980 [6,2]
+ CRUSH rule 3 x 981 [7,5]
+ CRUSH rule 3 x 982 [5,2]
+ CRUSH rule 3 x 983 [5,6]
+ CRUSH rule 3 x 984 [2,8]
+ CRUSH rule 3 x 985 [2,5]
+ CRUSH rule 3 x 986 [8,5]
+ CRUSH rule 3 x 987 [2,5]
+ CRUSH rule 3 x 988 [2,5]
+ CRUSH rule 3 x 989 [2,8]
+ CRUSH rule 3 x 990 [2,6]
+ CRUSH rule 3 x 991 [2,5]
+ CRUSH rule 3 x 992 [7,2]
+ CRUSH rule 3 x 993 [2,6]
+ CRUSH rule 3 x 994 [5,6]
+ CRUSH rule 3 x 995 [7,2]
+ CRUSH rule 3 x 996 [6,5]
+ CRUSH rule 3 x 997 [6,5]
+ CRUSH rule 3 x 998 [8,2]
+ CRUSH rule 3 x 999 [2,7]
+ CRUSH rule 3 x 1000 [8,5]
+ CRUSH rule 3 x 1001 [2,5]
+ CRUSH rule 3 x 1002 [2,5]
+ CRUSH rule 3 x 1003 [2,7]
+ CRUSH rule 3 x 1004 [6,2]
+ CRUSH rule 3 x 1005 [6,2]
+ CRUSH rule 3 x 1006 [2,8]
+ CRUSH rule 3 x 1007 [2,5]
+ CRUSH rule 3 x 1008 [2,7]
+ CRUSH rule 3 x 1009 [6,5]
+ CRUSH rule 3 x 1010 [5,2]
+ CRUSH rule 3 x 1011 [5,2]
+ CRUSH rule 3 x 1012 [5,2]
+ CRUSH rule 3 x 1013 [5,2]
+ CRUSH rule 3 x 1014 [2,8]
+ CRUSH rule 3 x 1015 [6,5]
+ CRUSH rule 3 x 1016 [2,5]
+ CRUSH rule 3 x 1017 [6,2]
+ CRUSH rule 3 x 1018 [5,2]
+ CRUSH rule 3 x 1019 [5,8]
+ CRUSH rule 3 x 1020 [5,2]
+ CRUSH rule 3 x 1021 [5,2]
+ CRUSH rule 3 x 1022 [2,7]
+ CRUSH rule 3 x 1023 [5,2]
+ rule 3 (choose-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 3 x 0 [2,5,7]
+ CRUSH rule 3 x 1 [2,8,5]
+ CRUSH rule 3 x 2 [2,5,8]
+ CRUSH rule 3 x 3 [8,2,5]
+ CRUSH rule 3 x 4 [5,2,8]
+ CRUSH rule 3 x 5 [7,2,5]
+ CRUSH rule 3 x 6 [2,6,5]
+ CRUSH rule 3 x 7 [5,6,2]
+ CRUSH rule 3 x 8 [5,7,2]
+ CRUSH rule 3 x 9 [2,5,8]
+ CRUSH rule 3 x 10 [2,8,5]
+ CRUSH rule 3 x 11 [2,6,5]
+ CRUSH rule 3 x 12 [2,5,7]
+ CRUSH rule 3 x 13 [5,8,2]
+ CRUSH rule 3 x 14 [7,2,5]
+ CRUSH rule 3 x 15 [7,2,5]
+ CRUSH rule 3 x 16 [5,7,2]
+ CRUSH rule 3 x 17 [5,2,7]
+ CRUSH rule 3 x 18 [2,5,6]
+ CRUSH rule 3 x 19 [7,5,2]
+ CRUSH rule 3 x 20 [2,5,7]
+ CRUSH rule 3 x 21 [5,6,2]
+ CRUSH rule 3 x 22 [8,5,2]
+ CRUSH rule 3 x 23 [5,7,2]
+ CRUSH rule 3 x 24 [2,6,5]
+ CRUSH rule 3 x 25 [5,8,2]
+ CRUSH rule 3 x 26 [2,7,5]
+ CRUSH rule 3 x 27 [5,2,7]
+ CRUSH rule 3 x 28 [6,2,5]
+ CRUSH rule 3 x 29 [8,5,2]
+ CRUSH rule 3 x 30 [5,6,2]
+ CRUSH rule 3 x 31 [8,2,5]
+ CRUSH rule 3 x 32 [5,7,2]
+ CRUSH rule 3 x 33 [2,7,5]
+ CRUSH rule 3 x 34 [2,5,7]
+ CRUSH rule 3 x 35 [2,6,5]
+ CRUSH rule 3 x 36 [5,6,2]
+ CRUSH rule 3 x 37 [2,5,7]
+ CRUSH rule 3 x 38 [5,6,2]
+ CRUSH rule 3 x 39 [5,7,2]
+ CRUSH rule 3 x 40 [7,2,5]
+ CRUSH rule 3 x 41 [2,7,5]
+ CRUSH rule 3 x 42 [5,7,2]
+ CRUSH rule 3 x 43 [2,5,8]
+ CRUSH rule 3 x 44 [2,8,5]
+ CRUSH rule 3 x 45 [8,2,5]
+ CRUSH rule 3 x 46 [2,5,8]
+ CRUSH rule 3 x 47 [5,2,7]
+ CRUSH rule 3 x 48 [5,6,2]
+ CRUSH rule 3 x 49 [5,6,2]
+ CRUSH rule 3 x 50 [5,2,7]
+ CRUSH rule 3 x 51 [5,6,2]
+ CRUSH rule 3 x 52 [8,2,5]
+ CRUSH rule 3 x 53 [5,6,2]
+ CRUSH rule 3 x 54 [7,5,2]
+ CRUSH rule 3 x 55 [8,2,5]
+ CRUSH rule 3 x 56 [6,5,2]
+ CRUSH rule 3 x 57 [5,8,2]
+ CRUSH rule 3 x 58 [2,8,5]
+ CRUSH rule 3 x 59 [5,2,7]
+ CRUSH rule 3 x 60 [5,2,8]
+ CRUSH rule 3 x 61 [5,8,2]
+ CRUSH rule 3 x 62 [7,2,5]
+ CRUSH rule 3 x 63 [5,6,2]
+ CRUSH rule 3 x 64 [5,2,8]
+ CRUSH rule 3 x 65 [7,5,2]
+ CRUSH rule 3 x 66 [5,6,2]
+ CRUSH rule 3 x 67 [5,2,6]
+ CRUSH rule 3 x 68 [2,5,8]
+ CRUSH rule 3 x 69 [5,2,7]
+ CRUSH rule 3 x 70 [7,2,5]
+ CRUSH rule 3 x 71 [2,7,5]
+ CRUSH rule 3 x 72 [6,2,5]
+ CRUSH rule 3 x 73 [2,7,5]
+ CRUSH rule 3 x 74 [2,8,5]
+ CRUSH rule 3 x 75 [5,2,7]
+ CRUSH rule 3 x 76 [5,2,6]
+ CRUSH rule 3 x 77 [7,2,5]
+ CRUSH rule 3 x 78 [2,5,7]
+ CRUSH rule 3 x 79 [5,2,7]
+ CRUSH rule 3 x 80 [2,5,6]
+ CRUSH rule 3 x 81 [2,5,8]
+ CRUSH rule 3 x 82 [7,2,5]
+ CRUSH rule 3 x 83 [2,6,5]
+ CRUSH rule 3 x 84 [7,2,5]
+ CRUSH rule 3 x 85 [5,7,2]
+ CRUSH rule 3 x 86 [2,6,5]
+ CRUSH rule 3 x 87 [2,6,5]
+ CRUSH rule 3 x 88 [2,8,5]
+ CRUSH rule 3 x 89 [5,2,8]
+ CRUSH rule 3 x 90 [6,5,2]
+ CRUSH rule 3 x 91 [5,6,2]
+ CRUSH rule 3 x 92 [2,6,5]
+ CRUSH rule 3 x 93 [7,5,2]
+ CRUSH rule 3 x 94 [2,5,7]
+ CRUSH rule 3 x 95 [7,5,2]
+ CRUSH rule 3 x 96 [5,8,2]
+ CRUSH rule 3 x 97 [8,5,2]
+ CRUSH rule 3 x 98 [2,7,5]
+ CRUSH rule 3 x 99 [2,8,5]
+ CRUSH rule 3 x 100 [2,6,5]
+ CRUSH rule 3 x 101 [5,8,2]
+ CRUSH rule 3 x 102 [5,2,7]
+ CRUSH rule 3 x 103 [5,7,2]
+ CRUSH rule 3 x 104 [7,5,2]
+ CRUSH rule 3 x 105 [2,5,8]
+ CRUSH rule 3 x 106 [2,6,5]
+ CRUSH rule 3 x 107 [5,2,7]
+ CRUSH rule 3 x 108 [7,2,5]
+ CRUSH rule 3 x 109 [2,5,8]
+ CRUSH rule 3 x 110 [5,2,6]
+ CRUSH rule 3 x 111 [2,5,7]
+ CRUSH rule 3 x 112 [2,6,5]
+ CRUSH rule 3 x 113 [6,2,5]
+ CRUSH rule 3 x 114 [7,5,2]
+ CRUSH rule 3 x 115 [8,2,5]
+ CRUSH rule 3 x 116 [2,7,5]
+ CRUSH rule 3 x 117 [7,5,2]
+ CRUSH rule 3 x 118 [2,5,6]
+ CRUSH rule 3 x 119 [5,7,2]
+ CRUSH rule 3 x 120 [2,5,6]
+ CRUSH rule 3 x 121 [2,8,5]
+ CRUSH rule 3 x 122 [8,5,2]
+ CRUSH rule 3 x 123 [2,5,7]
+ CRUSH rule 3 x 124 [5,2,8]
+ CRUSH rule 3 x 125 [2,7,5]
+ CRUSH rule 3 x 126 [5,2,7]
+ CRUSH rule 3 x 127 [5,6,2]
+ CRUSH rule 3 x 128 [5,8,2]
+ CRUSH rule 3 x 129 [2,5,7]
+ CRUSH rule 3 x 130 [5,8,2]
+ CRUSH rule 3 x 131 [2,5,6]
+ CRUSH rule 3 x 132 [2,5,8]
+ CRUSH rule 3 x 133 [5,6,2]
+ CRUSH rule 3 x 134 [2,7,5]
+ CRUSH rule 3 x 135 [5,7,2]
+ CRUSH rule 3 x 136 [2,5,8]
+ CRUSH rule 3 x 137 [7,5,2]
+ CRUSH rule 3 x 138 [8,5,2]
+ CRUSH rule 3 x 139 [5,2,8]
+ CRUSH rule 3 x 140 [2,8,5]
+ CRUSH rule 3 x 141 [6,2,5]
+ CRUSH rule 3 x 142 [5,2,8]
+ CRUSH rule 3 x 143 [5,7,2]
+ CRUSH rule 3 x 144 [8,2,5]
+ CRUSH rule 3 x 145 [8,5,2]
+ CRUSH rule 3 x 146 [2,8,5]
+ CRUSH rule 3 x 147 [2,6,5]
+ CRUSH rule 3 x 148 [5,2,8]
+ CRUSH rule 3 x 149 [5,6,2]
+ CRUSH rule 3 x 150 [2,8,5]
+ CRUSH rule 3 x 151 [5,8,2]
+ CRUSH rule 3 x 152 [8,5,2]
+ CRUSH rule 3 x 153 [8,5,2]
+ CRUSH rule 3 x 154 [5,2,6]
+ CRUSH rule 3 x 155 [5,6,2]
+ CRUSH rule 3 x 156 [5,2,7]
+ CRUSH rule 3 x 157 [5,2,7]
+ CRUSH rule 3 x 158 [2,6,5]
+ CRUSH rule 3 x 159 [7,2,5]
+ CRUSH rule 3 x 160 [2,8,5]
+ CRUSH rule 3 x 161 [2,5,6]
+ CRUSH rule 3 x 162 [2,8,5]
+ CRUSH rule 3 x 163 [5,8,2]
+ CRUSH rule 3 x 164 [7,2,5]
+ CRUSH rule 3 x 165 [7,2,5]
+ CRUSH rule 3 x 166 [2,5,7]
+ CRUSH rule 3 x 167 [2,6,5]
+ CRUSH rule 3 x 168 [5,2,7]
+ CRUSH rule 3 x 169 [2,7,5]
+ CRUSH rule 3 x 170 [2,5,8]
+ CRUSH rule 3 x 171 [7,5,2]
+ CRUSH rule 3 x 172 [2,8,5]
+ CRUSH rule 3 x 173 [8,5,2]
+ CRUSH rule 3 x 174 [2,5,8]
+ CRUSH rule 3 x 175 [6,2,5]
+ CRUSH rule 3 x 176 [5,2,7]
+ CRUSH rule 3 x 177 [5,2,8]
+ CRUSH rule 3 x 178 [5,2,6]
+ CRUSH rule 3 x 179 [5,2,7]
+ CRUSH rule 3 x 180 [5,7,2]
+ CRUSH rule 3 x 181 [6,2,5]
+ CRUSH rule 3 x 182 [8,5,2]
+ CRUSH rule 3 x 183 [7,5,2]
+ CRUSH rule 3 x 184 [5,6,2]
+ CRUSH rule 3 x 185 [6,2,5]
+ CRUSH rule 3 x 186 [2,5,8]
+ CRUSH rule 3 x 187 [2,6,5]
+ CRUSH rule 3 x 188 [2,6,5]
+ CRUSH rule 3 x 189 [2,6,5]
+ CRUSH rule 3 x 190 [5,2,8]
+ CRUSH rule 3 x 191 [7,2,5]
+ CRUSH rule 3 x 192 [5,2,6]
+ CRUSH rule 3 x 193 [5,2,6]
+ CRUSH rule 3 x 194 [2,5,7]
+ CRUSH rule 3 x 195 [6,5,2]
+ CRUSH rule 3 x 196 [6,2,5]
+ CRUSH rule 3 x 197 [6,5,2]
+ CRUSH rule 3 x 198 [2,5,6]
+ CRUSH rule 3 x 199 [2,5,7]
+ CRUSH rule 3 x 200 [2,5,8]
+ CRUSH rule 3 x 201 [7,2,5]
+ CRUSH rule 3 x 202 [6,5,2]
+ CRUSH rule 3 x 203 [5,6,2]
+ CRUSH rule 3 x 204 [2,5,8]
+ CRUSH rule 3 x 205 [2,7,5]
+ CRUSH rule 3 x 206 [2,8,5]
+ CRUSH rule 3 x 207 [5,2,7]
+ CRUSH rule 3 x 208 [7,2,5]
+ CRUSH rule 3 x 209 [2,8,5]
+ CRUSH rule 3 x 210 [2,5,6]
+ CRUSH rule 3 x 211 [5,2,7]
+ CRUSH rule 3 x 212 [7,5,2]
+ CRUSH rule 3 x 213 [8,5,2]
+ CRUSH rule 3 x 214 [5,7,2]
+ CRUSH rule 3 x 215 [8,2,5]
+ CRUSH rule 3 x 216 [5,2,8]
+ CRUSH rule 3 x 217 [2,7,5]
+ CRUSH rule 3 x 218 [2,7,5]
+ CRUSH rule 3 x 219 [5,6,2]
+ CRUSH rule 3 x 220 [5,8,2]
+ CRUSH rule 3 x 221 [5,7,2]
+ CRUSH rule 3 x 222 [6,5,2]
+ CRUSH rule 3 x 223 [2,5,6]
+ CRUSH rule 3 x 224 [2,5,8]
+ CRUSH rule 3 x 225 [8,2,5]
+ CRUSH rule 3 x 226 [7,2,5]
+ CRUSH rule 3 x 227 [5,2,7]
+ CRUSH rule 3 x 228 [5,6,2]
+ CRUSH rule 3 x 229 [5,7,2]
+ CRUSH rule 3 x 230 [5,6,2]
+ CRUSH rule 3 x 231 [5,7,2]
+ CRUSH rule 3 x 232 [2,8,5]
+ CRUSH rule 3 x 233 [5,6,2]
+ CRUSH rule 3 x 234 [2,5,6]
+ CRUSH rule 3 x 235 [5,6,2]
+ CRUSH rule 3 x 236 [5,2,8]
+ CRUSH rule 3 x 237 [5,8,2]
+ CRUSH rule 3 x 238 [5,2,6]
+ CRUSH rule 3 x 239 [8,5,2]
+ CRUSH rule 3 x 240 [5,8,2]
+ CRUSH rule 3 x 241 [5,2,7]
+ CRUSH rule 3 x 242 [5,2,6]
+ CRUSH rule 3 x 243 [5,8,2]
+ CRUSH rule 3 x 244 [5,6,2]
+ CRUSH rule 3 x 245 [7,2,5]
+ CRUSH rule 3 x 246 [2,5,7]
+ CRUSH rule 3 x 247 [6,2,5]
+ CRUSH rule 3 x 248 [8,2,5]
+ CRUSH rule 3 x 249 [2,5,8]
+ CRUSH rule 3 x 250 [2,5,8]
+ CRUSH rule 3 x 251 [2,5,7]
+ CRUSH rule 3 x 252 [5,7,2]
+ CRUSH rule 3 x 253 [5,2,7]
+ CRUSH rule 3 x 254 [5,2,8]
+ CRUSH rule 3 x 255 [2,7,5]
+ CRUSH rule 3 x 256 [5,6,2]
+ CRUSH rule 3 x 257 [2,6,5]
+ CRUSH rule 3 x 258 [5,2,8]
+ CRUSH rule 3 x 259 [5,6,2]
+ CRUSH rule 3 x 260 [5,8,2]
+ CRUSH rule 3 x 261 [8,5,2]
+ CRUSH rule 3 x 262 [5,6,2]
+ CRUSH rule 3 x 263 [6,2,5]
+ CRUSH rule 3 x 264 [5,6,2]
+ CRUSH rule 3 x 265 [8,5,2]
+ CRUSH rule 3 x 266 [8,2,5]
+ CRUSH rule 3 x 267 [2,5,6]
+ CRUSH rule 3 x 268 [2,6,5]
+ CRUSH rule 3 x 269 [2,6,5]
+ CRUSH rule 3 x 270 [5,2,8]
+ CRUSH rule 3 x 271 [7,5,2]
+ CRUSH rule 3 x 272 [2,6,5]
+ CRUSH rule 3 x 273 [5,2,7]
+ CRUSH rule 3 x 274 [6,5,2]
+ CRUSH rule 3 x 275 [5,8,2]
+ CRUSH rule 3 x 276 [7,2,5]
+ CRUSH rule 3 x 277 [6,5,2]
+ CRUSH rule 3 x 278 [6,2,5]
+ CRUSH rule 3 x 279 [8,5,2]
+ CRUSH rule 3 x 280 [2,7,5]
+ CRUSH rule 3 x 281 [8,2,5]
+ CRUSH rule 3 x 282 [5,2,8]
+ CRUSH rule 3 x 283 [8,2,5]
+ CRUSH rule 3 x 284 [6,5,2]
+ CRUSH rule 3 x 285 [5,7,2]
+ CRUSH rule 3 x 286 [2,8,5]
+ CRUSH rule 3 x 287 [2,5,8]
+ CRUSH rule 3 x 288 [8,2,5]
+ CRUSH rule 3 x 289 [5,6,2]
+ CRUSH rule 3 x 290 [2,5,6]
+ CRUSH rule 3 x 291 [2,5,8]
+ CRUSH rule 3 x 292 [8,2,5]
+ CRUSH rule 3 x 293 [6,2,5]
+ CRUSH rule 3 x 294 [7,5,2]
+ CRUSH rule 3 x 295 [5,6,2]
+ CRUSH rule 3 x 296 [5,2,6]
+ CRUSH rule 3 x 297 [6,2,5]
+ CRUSH rule 3 x 298 [2,5,6]
+ CRUSH rule 3 x 299 [2,8,5]
+ CRUSH rule 3 x 300 [8,5,2]
+ CRUSH rule 3 x 301 [2,7,5]
+ CRUSH rule 3 x 302 [5,2,6]
+ CRUSH rule 3 x 303 [7,5,2]
+ CRUSH rule 3 x 304 [2,6,5]
+ CRUSH rule 3 x 305 [5,6,2]
+ CRUSH rule 3 x 306 [2,8,5]
+ CRUSH rule 3 x 307 [2,7,5]
+ CRUSH rule 3 x 308 [2,8,5]
+ CRUSH rule 3 x 309 [7,5,2]
+ CRUSH rule 3 x 310 [5,2,6]
+ CRUSH rule 3 x 311 [5,8,2]
+ CRUSH rule 3 x 312 [2,6,5]
+ CRUSH rule 3 x 313 [5,2,7]
+ CRUSH rule 3 x 314 [5,2,6]
+ CRUSH rule 3 x 315 [2,5,7]
+ CRUSH rule 3 x 316 [6,5,2]
+ CRUSH rule 3 x 317 [2,7,5]
+ CRUSH rule 3 x 318 [8,2,5]
+ CRUSH rule 3 x 319 [5,2,6]
+ CRUSH rule 3 x 320 [5,8,2]
+ CRUSH rule 3 x 321 [2,5,8]
+ CRUSH rule 3 x 322 [2,6,5]
+ CRUSH rule 3 x 323 [5,7,2]
+ CRUSH rule 3 x 324 [7,2,5]
+ CRUSH rule 3 x 325 [5,6,2]
+ CRUSH rule 3 x 326 [5,2,7]
+ CRUSH rule 3 x 327 [2,8,5]
+ CRUSH rule 3 x 328 [7,5,2]
+ CRUSH rule 3 x 329 [5,7,2]
+ CRUSH rule 3 x 330 [5,7,2]
+ CRUSH rule 3 x 331 [2,7,5]
+ CRUSH rule 3 x 332 [2,5,8]
+ CRUSH rule 3 x 333 [6,5,2]
+ CRUSH rule 3 x 334 [8,5,2]
+ CRUSH rule 3 x 335 [7,2,5]
+ CRUSH rule 3 x 336 [5,6,2]
+ CRUSH rule 3 x 337 [7,2,5]
+ CRUSH rule 3 x 338 [5,8,2]
+ CRUSH rule 3 x 339 [7,5,2]
+ CRUSH rule 3 x 340 [2,6,5]
+ CRUSH rule 3 x 341 [5,2,7]
+ CRUSH rule 3 x 342 [2,8,5]
+ CRUSH rule 3 x 343 [6,5,2]
+ CRUSH rule 3 x 344 [6,2,5]
+ CRUSH rule 3 x 345 [5,7,2]
+ CRUSH rule 3 x 346 [8,2,5]
+ CRUSH rule 3 x 347 [5,2,8]
+ CRUSH rule 3 x 348 [8,2,5]
+ CRUSH rule 3 x 349 [2,7,5]
+ CRUSH rule 3 x 350 [8,5,2]
+ CRUSH rule 3 x 351 [5,8,2]
+ CRUSH rule 3 x 352 [2,8,5]
+ CRUSH rule 3 x 353 [6,5,2]
+ CRUSH rule 3 x 354 [2,5,6]
+ CRUSH rule 3 x 355 [5,8,2]
+ CRUSH rule 3 x 356 [5,2,8]
+ CRUSH rule 3 x 357 [6,2,5]
+ CRUSH rule 3 x 358 [2,8,5]
+ CRUSH rule 3 x 359 [6,2,5]
+ CRUSH rule 3 x 360 [5,2,8]
+ CRUSH rule 3 x 361 [8,5,2]
+ CRUSH rule 3 x 362 [5,2,8]
+ CRUSH rule 3 x 363 [5,2,8]
+ CRUSH rule 3 x 364 [2,5,7]
+ CRUSH rule 3 x 365 [6,5,2]
+ CRUSH rule 3 x 366 [7,2,5]
+ CRUSH rule 3 x 367 [5,2,7]
+ CRUSH rule 3 x 368 [7,5,2]
+ CRUSH rule 3 x 369 [5,7,2]
+ CRUSH rule 3 x 370 [8,2,5]
+ CRUSH rule 3 x 371 [2,5,8]
+ CRUSH rule 3 x 372 [5,2,8]
+ CRUSH rule 3 x 373 [2,6,5]
+ CRUSH rule 3 x 374 [5,8,2]
+ CRUSH rule 3 x 375 [6,5,2]
+ CRUSH rule 3 x 376 [7,2,5]
+ CRUSH rule 3 x 377 [2,5,7]
+ CRUSH rule 3 x 378 [2,6,5]
+ CRUSH rule 3 x 379 [8,5,2]
+ CRUSH rule 3 x 380 [2,5,8]
+ CRUSH rule 3 x 381 [2,5,7]
+ CRUSH rule 3 x 382 [2,5,7]
+ CRUSH rule 3 x 383 [5,7,2]
+ CRUSH rule 3 x 384 [7,2,5]
+ CRUSH rule 3 x 385 [7,5,2]
+ CRUSH rule 3 x 386 [2,5,6]
+ CRUSH rule 3 x 387 [2,5,8]
+ CRUSH rule 3 x 388 [5,2,7]
+ CRUSH rule 3 x 389 [2,5,8]
+ CRUSH rule 3 x 390 [5,8,2]
+ CRUSH rule 3 x 391 [5,6,2]
+ CRUSH rule 3 x 392 [2,7,5]
+ CRUSH rule 3 x 393 [5,2,6]
+ CRUSH rule 3 x 394 [5,8,2]
+ CRUSH rule 3 x 395 [5,2,8]
+ CRUSH rule 3 x 396 [5,2,6]
+ CRUSH rule 3 x 397 [2,5,7]
+ CRUSH rule 3 x 398 [2,5,6]
+ CRUSH rule 3 x 399 [8,5,2]
+ CRUSH rule 3 x 400 [8,2,5]
+ CRUSH rule 3 x 401 [2,5,6]
+ CRUSH rule 3 x 402 [7,5,2]
+ CRUSH rule 3 x 403 [2,5,7]
+ CRUSH rule 3 x 404 [5,2,6]
+ CRUSH rule 3 x 405 [6,5,2]
+ CRUSH rule 3 x 406 [2,6,5]
+ CRUSH rule 3 x 407 [2,7,5]
+ CRUSH rule 3 x 408 [5,2,7]
+ CRUSH rule 3 x 409 [7,5,2]
+ CRUSH rule 3 x 410 [8,5,2]
+ CRUSH rule 3 x 411 [2,7,5]
+ CRUSH rule 3 x 412 [2,5,7]
+ CRUSH rule 3 x 413 [5,2,8]
+ CRUSH rule 3 x 414 [5,2,8]
+ CRUSH rule 3 x 415 [2,6,5]
+ CRUSH rule 3 x 416 [2,8,5]
+ CRUSH rule 3 x 417 [8,2,5]
+ CRUSH rule 3 x 418 [7,2,5]
+ CRUSH rule 3 x 419 [8,5,2]
+ CRUSH rule 3 x 420 [2,5,7]
+ CRUSH rule 3 x 421 [8,5,2]
+ CRUSH rule 3 x 422 [6,5,2]
+ CRUSH rule 3 x 423 [2,5,7]
+ CRUSH rule 3 x 424 [8,5,2]
+ CRUSH rule 3 x 425 [2,5,8]
+ CRUSH rule 3 x 426 [6,2,5]
+ CRUSH rule 3 x 427 [2,8,5]
+ CRUSH rule 3 x 428 [5,8,2]
+ CRUSH rule 3 x 429 [5,8,2]
+ CRUSH rule 3 x 430 [5,7,2]
+ CRUSH rule 3 x 431 [5,2,7]
+ CRUSH rule 3 x 432 [7,2,5]
+ CRUSH rule 3 x 433 [6,5,2]
+ CRUSH rule 3 x 434 [5,2,7]
+ CRUSH rule 3 x 435 [2,5,6]
+ CRUSH rule 3 x 436 [5,2,7]
+ CRUSH rule 3 x 437 [7,5,2]
+ CRUSH rule 3 x 438 [2,5,8]
+ CRUSH rule 3 x 439 [2,5,8]
+ CRUSH rule 3 x 440 [2,6,5]
+ CRUSH rule 3 x 441 [5,8,2]
+ CRUSH rule 3 x 442 [2,5,6]
+ CRUSH rule 3 x 443 [6,2,5]
+ CRUSH rule 3 x 444 [7,2,5]
+ CRUSH rule 3 x 445 [6,5,2]
+ CRUSH rule 3 x 446 [5,2,8]
+ CRUSH rule 3 x 447 [2,5,6]
+ CRUSH rule 3 x 448 [7,2,5]
+ CRUSH rule 3 x 449 [7,5,2]
+ CRUSH rule 3 x 450 [5,2,6]
+ CRUSH rule 3 x 451 [6,5,2]
+ CRUSH rule 3 x 452 [8,5,2]
+ CRUSH rule 3 x 453 [6,5,2]
+ CRUSH rule 3 x 454 [6,5,2]
+ CRUSH rule 3 x 455 [2,8,5]
+ CRUSH rule 3 x 456 [6,2,5]
+ CRUSH rule 3 x 457 [7,2,5]
+ CRUSH rule 3 x 458 [2,8,5]
+ CRUSH rule 3 x 459 [2,6,5]
+ CRUSH rule 3 x 460 [6,5,2]
+ CRUSH rule 3 x 461 [6,5,2]
+ CRUSH rule 3 x 462 [8,2,5]
+ CRUSH rule 3 x 463 [6,2,5]
+ CRUSH rule 3 x 464 [7,5,2]
+ CRUSH rule 3 x 465 [7,2,5]
+ CRUSH rule 3 x 466 [5,6,2]
+ CRUSH rule 3 x 467 [6,5,2]
+ CRUSH rule 3 x 468 [7,2,5]
+ CRUSH rule 3 x 469 [7,2,5]
+ CRUSH rule 3 x 470 [5,2,8]
+ CRUSH rule 3 x 471 [2,6,5]
+ CRUSH rule 3 x 472 [5,2,8]
+ CRUSH rule 3 x 473 [2,5,7]
+ CRUSH rule 3 x 474 [6,2,5]
+ CRUSH rule 3 x 475 [6,2,5]
+ CRUSH rule 3 x 476 [5,7,2]
+ CRUSH rule 3 x 477 [5,6,2]
+ CRUSH rule 3 x 478 [6,2,5]
+ CRUSH rule 3 x 479 [2,5,6]
+ CRUSH rule 3 x 480 [2,6,5]
+ CRUSH rule 3 x 481 [2,5,7]
+ CRUSH rule 3 x 482 [5,8,2]
+ CRUSH rule 3 x 483 [2,6,5]
+ CRUSH rule 3 x 484 [2,8,5]
+ CRUSH rule 3 x 485 [5,8,2]
+ CRUSH rule 3 x 486 [5,2,8]
+ CRUSH rule 3 x 487 [5,2,8]
+ CRUSH rule 3 x 488 [5,7,2]
+ CRUSH rule 3 x 489 [2,8,5]
+ CRUSH rule 3 x 490 [6,5,2]
+ CRUSH rule 3 x 491 [2,6,5]
+ CRUSH rule 3 x 492 [6,5,2]
+ CRUSH rule 3 x 493 [2,8,5]
+ CRUSH rule 3 x 494 [2,6,5]
+ CRUSH rule 3 x 495 [5,2,6]
+ CRUSH rule 3 x 496 [7,5,2]
+ CRUSH rule 3 x 497 [5,7,2]
+ CRUSH rule 3 x 498 [2,5,6]
+ CRUSH rule 3 x 499 [8,5,2]
+ CRUSH rule 3 x 500 [5,6,2]
+ CRUSH rule 3 x 501 [2,7,5]
+ CRUSH rule 3 x 502 [7,2,5]
+ CRUSH rule 3 x 503 [2,5,7]
+ CRUSH rule 3 x 504 [5,8,2]
+ CRUSH rule 3 x 505 [2,7,5]
+ CRUSH rule 3 x 506 [5,2,7]
+ CRUSH rule 3 x 507 [6,2,5]
+ CRUSH rule 3 x 508 [2,5,7]
+ CRUSH rule 3 x 509 [7,5,2]
+ CRUSH rule 3 x 510 [6,2,5]
+ CRUSH rule 3 x 511 [5,6,2]
+ CRUSH rule 3 x 512 [7,2,5]
+ CRUSH rule 3 x 513 [7,2,5]
+ CRUSH rule 3 x 514 [5,7,2]
+ CRUSH rule 3 x 515 [8,5,2]
+ CRUSH rule 3 x 516 [5,2,7]
+ CRUSH rule 3 x 517 [7,2,5]
+ CRUSH rule 3 x 518 [5,6,2]
+ CRUSH rule 3 x 519 [7,5,2]
+ CRUSH rule 3 x 520 [2,8,5]
+ CRUSH rule 3 x 521 [8,2,5]
+ CRUSH rule 3 x 522 [6,2,5]
+ CRUSH rule 3 x 523 [5,2,7]
+ CRUSH rule 3 x 524 [2,5,7]
+ CRUSH rule 3 x 525 [2,5,8]
+ CRUSH rule 3 x 526 [2,5,8]
+ CRUSH rule 3 x 527 [2,5,6]
+ CRUSH rule 3 x 528 [5,2,7]
+ CRUSH rule 3 x 529 [5,6,2]
+ CRUSH rule 3 x 530 [6,5,2]
+ CRUSH rule 3 x 531 [6,2,5]
+ CRUSH rule 3 x 532 [6,5,2]
+ CRUSH rule 3 x 533 [5,8,2]
+ CRUSH rule 3 x 534 [7,5,2]
+ CRUSH rule 3 x 535 [8,2,5]
+ CRUSH rule 3 x 536 [6,2,5]
+ CRUSH rule 3 x 537 [5,8,2]
+ CRUSH rule 3 x 538 [6,5,2]
+ CRUSH rule 3 x 539 [8,5,2]
+ CRUSH rule 3 x 540 [2,7,5]
+ CRUSH rule 3 x 541 [2,5,7]
+ CRUSH rule 3 x 542 [5,2,8]
+ CRUSH rule 3 x 543 [6,2,5]
+ CRUSH rule 3 x 544 [5,7,2]
+ CRUSH rule 3 x 545 [5,7,2]
+ CRUSH rule 3 x 546 [6,2,5]
+ CRUSH rule 3 x 547 [8,2,5]
+ CRUSH rule 3 x 548 [5,2,8]
+ CRUSH rule 3 x 549 [5,7,2]
+ CRUSH rule 3 x 550 [2,5,6]
+ CRUSH rule 3 x 551 [7,5,2]
+ CRUSH rule 3 x 552 [5,7,2]
+ CRUSH rule 3 x 553 [5,2,8]
+ CRUSH rule 3 x 554 [2,6,5]
+ CRUSH rule 3 x 555 [5,2,8]
+ CRUSH rule 3 x 556 [5,6,2]
+ CRUSH rule 3 x 557 [7,5,2]
+ CRUSH rule 3 x 558 [5,2,6]
+ CRUSH rule 3 x 559 [5,2,8]
+ CRUSH rule 3 x 560 [8,5,2]
+ CRUSH rule 3 x 561 [6,5,2]
+ CRUSH rule 3 x 562 [5,2,8]
+ CRUSH rule 3 x 563 [2,7,5]
+ CRUSH rule 3 x 564 [5,2,7]
+ CRUSH rule 3 x 565 [5,8,2]
+ CRUSH rule 3 x 566 [5,6,2]
+ CRUSH rule 3 x 567 [5,7,2]
+ CRUSH rule 3 x 568 [7,5,2]
+ CRUSH rule 3 x 569 [5,2,6]
+ CRUSH rule 3 x 570 [2,5,7]
+ CRUSH rule 3 x 571 [5,6,2]
+ CRUSH rule 3 x 572 [5,2,7]
+ CRUSH rule 3 x 573 [5,2,7]
+ CRUSH rule 3 x 574 [2,5,6]
+ CRUSH rule 3 x 575 [8,2,5]
+ CRUSH rule 3 x 576 [5,7,2]
+ CRUSH rule 3 x 577 [8,2,5]
+ CRUSH rule 3 x 578 [6,2,5]
+ CRUSH rule 3 x 579 [5,2,6]
+ CRUSH rule 3 x 580 [5,2,8]
+ CRUSH rule 3 x 581 [7,2,5]
+ CRUSH rule 3 x 582 [2,8,5]
+ CRUSH rule 3 x 583 [6,2,5]
+ CRUSH rule 3 x 584 [8,2,5]
+ CRUSH rule 3 x 585 [7,2,5]
+ CRUSH rule 3 x 586 [2,8,5]
+ CRUSH rule 3 x 587 [2,5,6]
+ CRUSH rule 3 x 588 [5,8,2]
+ CRUSH rule 3 x 589 [7,2,5]
+ CRUSH rule 3 x 590 [6,2,5]
+ CRUSH rule 3 x 591 [5,2,7]
+ CRUSH rule 3 x 592 [2,5,6]
+ CRUSH rule 3 x 593 [2,8,5]
+ CRUSH rule 3 x 594 [2,6,5]
+ CRUSH rule 3 x 595 [7,2,5]
+ CRUSH rule 3 x 596 [5,2,6]
+ CRUSH rule 3 x 597 [5,2,7]
+ CRUSH rule 3 x 598 [5,2,7]
+ CRUSH rule 3 x 599 [5,2,6]
+ CRUSH rule 3 x 600 [7,2,5]
+ CRUSH rule 3 x 601 [2,6,5]
+ CRUSH rule 3 x 602 [5,8,2]
+ CRUSH rule 3 x 603 [5,2,8]
+ CRUSH rule 3 x 604 [7,5,2]
+ CRUSH rule 3 x 605 [5,2,6]
+ CRUSH rule 3 x 606 [2,7,5]
+ CRUSH rule 3 x 607 [2,5,8]
+ CRUSH rule 3 x 608 [5,2,8]
+ CRUSH rule 3 x 609 [5,2,6]
+ CRUSH rule 3 x 610 [5,8,2]
+ CRUSH rule 3 x 611 [2,8,5]
+ CRUSH rule 3 x 612 [2,8,5]
+ CRUSH rule 3 x 613 [7,2,5]
+ CRUSH rule 3 x 614 [7,2,5]
+ CRUSH rule 3 x 615 [6,2,5]
+ CRUSH rule 3 x 616 [2,7,5]
+ CRUSH rule 3 x 617 [6,2,5]
+ CRUSH rule 3 x 618 [7,5,2]
+ CRUSH rule 3 x 619 [5,2,6]
+ CRUSH rule 3 x 620 [5,2,8]
+ CRUSH rule 3 x 621 [5,6,2]
+ CRUSH rule 3 x 622 [2,5,8]
+ CRUSH rule 3 x 623 [2,8,5]
+ CRUSH rule 3 x 624 [5,2,8]
+ CRUSH rule 3 x 625 [2,5,8]
+ CRUSH rule 3 x 626 [7,2,5]
+ CRUSH rule 3 x 627 [2,6,5]
+ CRUSH rule 3 x 628 [8,2,5]
+ CRUSH rule 3 x 629 [2,6,5]
+ CRUSH rule 3 x 630 [2,6,5]
+ CRUSH rule 3 x 631 [2,6,5]
+ CRUSH rule 3 x 632 [7,2,5]
+ CRUSH rule 3 x 633 [8,5,2]
+ CRUSH rule 3 x 634 [2,5,6]
+ CRUSH rule 3 x 635 [5,6,2]
+ CRUSH rule 3 x 636 [2,5,7]
+ CRUSH rule 3 x 637 [5,2,8]
+ CRUSH rule 3 x 638 [6,2,5]
+ CRUSH rule 3 x 639 [5,2,6]
+ CRUSH rule 3 x 640 [5,2,7]
+ CRUSH rule 3 x 641 [7,2,5]
+ CRUSH rule 3 x 642 [2,8,5]
+ CRUSH rule 3 x 643 [5,2,8]
+ CRUSH rule 3 x 644 [8,2,5]
+ CRUSH rule 3 x 645 [5,7,2]
+ CRUSH rule 3 x 646 [8,2,5]
+ CRUSH rule 3 x 647 [7,2,5]
+ CRUSH rule 3 x 648 [2,8,5]
+ CRUSH rule 3 x 649 [5,7,2]
+ CRUSH rule 3 x 650 [7,5,2]
+ CRUSH rule 3 x 651 [5,6,2]
+ CRUSH rule 3 x 652 [5,6,2]
+ CRUSH rule 3 x 653 [8,5,2]
+ CRUSH rule 3 x 654 [7,5,2]
+ CRUSH rule 3 x 655 [2,5,6]
+ CRUSH rule 3 x 656 [5,7,2]
+ CRUSH rule 3 x 657 [6,2,5]
+ CRUSH rule 3 x 658 [5,8,2]
+ CRUSH rule 3 x 659 [5,7,2]
+ CRUSH rule 3 x 660 [7,5,2]
+ CRUSH rule 3 x 661 [2,7,5]
+ CRUSH rule 3 x 662 [5,2,8]
+ CRUSH rule 3 x 663 [2,5,7]
+ CRUSH rule 3 x 664 [2,5,6]
+ CRUSH rule 3 x 665 [5,6,2]
+ CRUSH rule 3 x 666 [2,7,5]
+ CRUSH rule 3 x 667 [2,5,8]
+ CRUSH rule 3 x 668 [5,7,2]
+ CRUSH rule 3 x 669 [6,5,2]
+ CRUSH rule 3 x 670 [5,2,6]
+ CRUSH rule 3 x 671 [2,8,5]
+ CRUSH rule 3 x 672 [5,2,8]
+ CRUSH rule 3 x 673 [5,2,7]
+ CRUSH rule 3 x 674 [5,2,7]
+ CRUSH rule 3 x 675 [2,8,5]
+ CRUSH rule 3 x 676 [2,5,7]
+ CRUSH rule 3 x 677 [5,2,6]
+ CRUSH rule 3 x 678 [2,5,8]
+ CRUSH rule 3 x 679 [6,2,5]
+ CRUSH rule 3 x 680 [2,5,8]
+ CRUSH rule 3 x 681 [5,6,2]
+ CRUSH rule 3 x 682 [2,5,8]
+ CRUSH rule 3 x 683 [2,5,8]
+ CRUSH rule 3 x 684 [7,2,5]
+ CRUSH rule 3 x 685 [7,2,5]
+ CRUSH rule 3 x 686 [2,5,8]
+ CRUSH rule 3 x 687 [5,7,2]
+ CRUSH rule 3 x 688 [5,6,2]
+ CRUSH rule 3 x 689 [6,5,2]
+ CRUSH rule 3 x 690 [8,2,5]
+ CRUSH rule 3 x 691 [5,2,6]
+ CRUSH rule 3 x 692 [7,2,5]
+ CRUSH rule 3 x 693 [6,5,2]
+ CRUSH rule 3 x 694 [6,5,2]
+ CRUSH rule 3 x 695 [2,6,5]
+ CRUSH rule 3 x 696 [2,5,8]
+ CRUSH rule 3 x 697 [6,2,5]
+ CRUSH rule 3 x 698 [6,2,5]
+ CRUSH rule 3 x 699 [2,8,5]
+ CRUSH rule 3 x 700 [2,5,6]
+ CRUSH rule 3 x 701 [5,2,6]
+ CRUSH rule 3 x 702 [5,2,8]
+ CRUSH rule 3 x 703 [8,5,2]
+ CRUSH rule 3 x 704 [2,5,6]
+ CRUSH rule 3 x 705 [8,2,5]
+ CRUSH rule 3 x 706 [2,5,8]
+ CRUSH rule 3 x 707 [7,5,2]
+ CRUSH rule 3 x 708 [5,7,2]
+ CRUSH rule 3 x 709 [6,5,2]
+ CRUSH rule 3 x 710 [8,5,2]
+ CRUSH rule 3 x 711 [2,5,7]
+ CRUSH rule 3 x 712 [2,5,6]
+ CRUSH rule 3 x 713 [6,5,2]
+ CRUSH rule 3 x 714 [5,2,6]
+ CRUSH rule 3 x 715 [2,5,6]
+ CRUSH rule 3 x 716 [5,6,2]
+ CRUSH rule 3 x 717 [8,2,5]
+ CRUSH rule 3 x 718 [5,7,2]
+ CRUSH rule 3 x 719 [2,6,5]
+ CRUSH rule 3 x 720 [6,2,5]
+ CRUSH rule 3 x 721 [5,7,2]
+ CRUSH rule 3 x 722 [5,7,2]
+ CRUSH rule 3 x 723 [5,2,7]
+ CRUSH rule 3 x 724 [2,7,5]
+ CRUSH rule 3 x 725 [2,5,6]
+ CRUSH rule 3 x 726 [5,7,2]
+ CRUSH rule 3 x 727 [5,7,2]
+ CRUSH rule 3 x 728 [2,6,5]
+ CRUSH rule 3 x 729 [5,6,2]
+ CRUSH rule 3 x 730 [5,8,2]
+ CRUSH rule 3 x 731 [5,2,6]
+ CRUSH rule 3 x 732 [2,5,8]
+ CRUSH rule 3 x 733 [5,6,2]
+ CRUSH rule 3 x 734 [6,5,2]
+ CRUSH rule 3 x 735 [5,6,2]
+ CRUSH rule 3 x 736 [5,8,2]
+ CRUSH rule 3 x 737 [2,8,5]
+ CRUSH rule 3 x 738 [5,2,6]
+ CRUSH rule 3 x 739 [2,7,5]
+ CRUSH rule 3 x 740 [2,7,5]
+ CRUSH rule 3 x 741 [7,2,5]
+ CRUSH rule 3 x 742 [8,2,5]
+ CRUSH rule 3 x 743 [7,2,5]
+ CRUSH rule 3 x 744 [5,6,2]
+ CRUSH rule 3 x 745 [5,2,8]
+ CRUSH rule 3 x 746 [5,2,7]
+ CRUSH rule 3 x 747 [6,2,5]
+ CRUSH rule 3 x 748 [2,6,5]
+ CRUSH rule 3 x 749 [5,7,2]
+ CRUSH rule 3 x 750 [2,7,5]
+ CRUSH rule 3 x 751 [2,7,5]
+ CRUSH rule 3 x 752 [8,2,5]
+ CRUSH rule 3 x 753 [7,5,2]
+ CRUSH rule 3 x 754 [8,5,2]
+ CRUSH rule 3 x 755 [2,6,5]
+ CRUSH rule 3 x 756 [5,8,2]
+ CRUSH rule 3 x 757 [8,2,5]
+ CRUSH rule 3 x 758 [6,2,5]
+ CRUSH rule 3 x 759 [8,5,2]
+ CRUSH rule 3 x 760 [2,5,7]
+ CRUSH rule 3 x 761 [5,2,6]
+ CRUSH rule 3 x 762 [2,8,5]
+ CRUSH rule 3 x 763 [8,5,2]
+ CRUSH rule 3 x 764 [2,7,5]
+ CRUSH rule 3 x 765 [6,5,2]
+ CRUSH rule 3 x 766 [8,5,2]
+ CRUSH rule 3 x 767 [2,8,5]
+ CRUSH rule 3 x 768 [8,5,2]
+ CRUSH rule 3 x 769 [6,2,5]
+ CRUSH rule 3 x 770 [6,2,5]
+ CRUSH rule 3 x 771 [7,2,5]
+ CRUSH rule 3 x 772 [8,5,2]
+ CRUSH rule 3 x 773 [5,2,6]
+ CRUSH rule 3 x 774 [5,7,2]
+ CRUSH rule 3 x 775 [6,5,2]
+ CRUSH rule 3 x 776 [7,2,5]
+ CRUSH rule 3 x 777 [5,2,6]
+ CRUSH rule 3 x 778 [2,6,5]
+ CRUSH rule 3 x 779 [2,6,5]
+ CRUSH rule 3 x 780 [2,5,8]
+ CRUSH rule 3 x 781 [6,5,2]
+ CRUSH rule 3 x 782 [5,2,8]
+ CRUSH rule 3 x 783 [7,2,5]
+ CRUSH rule 3 x 784 [2,5,8]
+ CRUSH rule 3 x 785 [6,2,5]
+ CRUSH rule 3 x 786 [7,5,2]
+ CRUSH rule 3 x 787 [2,8,5]
+ CRUSH rule 3 x 788 [6,2,5]
+ CRUSH rule 3 x 789 [2,5,8]
+ CRUSH rule 3 x 790 [8,5,2]
+ CRUSH rule 3 x 791 [5,6,2]
+ CRUSH rule 3 x 792 [5,6,2]
+ CRUSH rule 3 x 793 [6,2,5]
+ CRUSH rule 3 x 794 [2,8,5]
+ CRUSH rule 3 x 795 [2,5,6]
+ CRUSH rule 3 x 796 [5,7,2]
+ CRUSH rule 3 x 797 [2,5,6]
+ CRUSH rule 3 x 798 [6,2,5]
+ CRUSH rule 3 x 799 [5,2,7]
+ CRUSH rule 3 x 800 [5,2,8]
+ CRUSH rule 3 x 801 [5,7,2]
+ CRUSH rule 3 x 802 [2,6,5]
+ CRUSH rule 3 x 803 [2,5,7]
+ CRUSH rule 3 x 804 [6,2,5]
+ CRUSH rule 3 x 805 [5,8,2]
+ CRUSH rule 3 x 806 [2,5,6]
+ CRUSH rule 3 x 807 [5,7,2]
+ CRUSH rule 3 x 808 [5,7,2]
+ CRUSH rule 3 x 809 [2,5,6]
+ CRUSH rule 3 x 810 [5,7,2]
+ CRUSH rule 3 x 811 [8,5,2]
+ CRUSH rule 3 x 812 [8,5,2]
+ CRUSH rule 3 x 813 [6,5,2]
+ CRUSH rule 3 x 814 [5,8,2]
+ CRUSH rule 3 x 815 [5,2,6]
+ CRUSH rule 3 x 816 [2,6,5]
+ CRUSH rule 3 x 817 [5,6,2]
+ CRUSH rule 3 x 818 [5,2,7]
+ CRUSH rule 3 x 819 [5,2,7]
+ CRUSH rule 3 x 820 [5,7,2]
+ CRUSH rule 3 x 821 [5,8,2]
+ CRUSH rule 3 x 822 [2,5,7]
+ CRUSH rule 3 x 823 [5,7,2]
+ CRUSH rule 3 x 824 [5,7,2]
+ CRUSH rule 3 x 825 [2,8,5]
+ CRUSH rule 3 x 826 [7,2,5]
+ CRUSH rule 3 x 827 [2,6,5]
+ CRUSH rule 3 x 828 [2,5,8]
+ CRUSH rule 3 x 829 [5,6,2]
+ CRUSH rule 3 x 830 [2,5,7]
+ CRUSH rule 3 x 831 [2,6,5]
+ CRUSH rule 3 x 832 [5,8,2]
+ CRUSH rule 3 x 833 [2,6,5]
+ CRUSH rule 3 x 834 [5,2,8]
+ CRUSH rule 3 x 835 [8,5,2]
+ CRUSH rule 3 x 836 [5,8,2]
+ CRUSH rule 3 x 837 [6,5,2]
+ CRUSH rule 3 x 838 [6,2,5]
+ CRUSH rule 3 x 839 [5,2,8]
+ CRUSH rule 3 x 840 [7,5,2]
+ CRUSH rule 3 x 841 [5,8,2]
+ CRUSH rule 3 x 842 [2,5,8]
+ CRUSH rule 3 x 843 [6,5,2]
+ CRUSH rule 3 x 844 [5,8,2]
+ CRUSH rule 3 x 845 [5,6,2]
+ CRUSH rule 3 x 846 [5,2,6]
+ CRUSH rule 3 x 847 [2,8,5]
+ CRUSH rule 3 x 848 [2,6,5]
+ CRUSH rule 3 x 849 [5,8,2]
+ CRUSH rule 3 x 850 [2,5,7]
+ CRUSH rule 3 x 851 [6,5,2]
+ CRUSH rule 3 x 852 [7,5,2]
+ CRUSH rule 3 x 853 [6,2,5]
+ CRUSH rule 3 x 854 [7,2,5]
+ CRUSH rule 3 x 855 [5,7,2]
+ CRUSH rule 3 x 856 [6,5,2]
+ CRUSH rule 3 x 857 [8,5,2]
+ CRUSH rule 3 x 858 [6,5,2]
+ CRUSH rule 3 x 859 [6,2,5]
+ CRUSH rule 3 x 860 [5,2,8]
+ CRUSH rule 3 x 861 [8,5,2]
+ CRUSH rule 3 x 862 [6,2,5]
+ CRUSH rule 3 x 863 [8,2,5]
+ CRUSH rule 3 x 864 [5,6,2]
+ CRUSH rule 3 x 865 [8,2,5]
+ CRUSH rule 3 x 866 [5,6,2]
+ CRUSH rule 3 x 867 [6,5,2]
+ CRUSH rule 3 x 868 [6,5,2]
+ CRUSH rule 3 x 869 [8,5,2]
+ CRUSH rule 3 x 870 [2,5,7]
+ CRUSH rule 3 x 871 [5,2,6]
+ CRUSH rule 3 x 872 [5,2,6]
+ CRUSH rule 3 x 873 [5,6,2]
+ CRUSH rule 3 x 874 [2,6,5]
+ CRUSH rule 3 x 875 [2,6,5]
+ CRUSH rule 3 x 876 [5,8,2]
+ CRUSH rule 3 x 877 [6,5,2]
+ CRUSH rule 3 x 878 [5,2,7]
+ CRUSH rule 3 x 879 [7,5,2]
+ CRUSH rule 3 x 880 [5,2,7]
+ CRUSH rule 3 x 881 [5,8,2]
+ CRUSH rule 3 x 882 [5,2,6]
+ CRUSH rule 3 x 883 [2,5,6]
+ CRUSH rule 3 x 884 [6,2,5]
+ CRUSH rule 3 x 885 [5,2,6]
+ CRUSH rule 3 x 886 [5,7,2]
+ CRUSH rule 3 x 887 [7,5,2]
+ CRUSH rule 3 x 888 [6,2,5]
+ CRUSH rule 3 x 889 [2,8,5]
+ CRUSH rule 3 x 890 [7,2,5]
+ CRUSH rule 3 x 891 [2,7,5]
+ CRUSH rule 3 x 892 [6,2,5]
+ CRUSH rule 3 x 893 [2,5,6]
+ CRUSH rule 3 x 894 [7,5,2]
+ CRUSH rule 3 x 895 [5,2,6]
+ CRUSH rule 3 x 896 [2,6,5]
+ CRUSH rule 3 x 897 [5,2,8]
+ CRUSH rule 3 x 898 [2,5,8]
+ CRUSH rule 3 x 899 [2,6,5]
+ CRUSH rule 3 x 900 [5,2,7]
+ CRUSH rule 3 x 901 [5,2,6]
+ CRUSH rule 3 x 902 [8,5,2]
+ CRUSH rule 3 x 903 [5,6,2]
+ CRUSH rule 3 x 904 [5,7,2]
+ CRUSH rule 3 x 905 [6,2,5]
+ CRUSH rule 3 x 906 [2,7,5]
+ CRUSH rule 3 x 907 [7,2,5]
+ CRUSH rule 3 x 908 [5,6,2]
+ CRUSH rule 3 x 909 [2,5,8]
+ CRUSH rule 3 x 910 [6,5,2]
+ CRUSH rule 3 x 911 [5,7,2]
+ CRUSH rule 3 x 912 [2,8,5]
+ CRUSH rule 3 x 913 [7,2,5]
+ CRUSH rule 3 x 914 [6,5,2]
+ CRUSH rule 3 x 915 [8,2,5]
+ CRUSH rule 3 x 916 [5,2,6]
+ CRUSH rule 3 x 917 [2,5,6]
+ CRUSH rule 3 x 918 [8,2,5]
+ CRUSH rule 3 x 919 [6,2,5]
+ CRUSH rule 3 x 920 [7,5,2]
+ CRUSH rule 3 x 921 [2,5,8]
+ CRUSH rule 3 x 922 [6,5,2]
+ CRUSH rule 3 x 923 [5,6,2]
+ CRUSH rule 3 x 924 [5,2,8]
+ CRUSH rule 3 x 925 [5,6,2]
+ CRUSH rule 3 x 926 [5,2,7]
+ CRUSH rule 3 x 927 [2,6,5]
+ CRUSH rule 3 x 928 [8,2,5]
+ CRUSH rule 3 x 929 [5,2,6]
+ CRUSH rule 3 x 930 [2,5,8]
+ CRUSH rule 3 x 931 [5,2,8]
+ CRUSH rule 3 x 932 [5,2,7]
+ CRUSH rule 3 x 933 [8,5,2]
+ CRUSH rule 3 x 934 [5,8,2]
+ CRUSH rule 3 x 935 [6,5,2]
+ CRUSH rule 3 x 936 [2,7,5]
+ CRUSH rule 3 x 937 [5,8,2]
+ CRUSH rule 3 x 938 [6,5,2]
+ CRUSH rule 3 x 939 [2,8,5]
+ CRUSH rule 3 x 940 [8,5,2]
+ CRUSH rule 3 x 941 [5,2,6]
+ CRUSH rule 3 x 942 [2,6,5]
+ CRUSH rule 3 x 943 [8,2,5]
+ CRUSH rule 3 x 944 [5,8,2]
+ CRUSH rule 3 x 945 [7,2,5]
+ CRUSH rule 3 x 946 [2,8,5]
+ CRUSH rule 3 x 947 [5,2,6]
+ CRUSH rule 3 x 948 [7,5,2]
+ CRUSH rule 3 x 949 [6,2,5]
+ CRUSH rule 3 x 950 [5,7,2]
+ CRUSH rule 3 x 951 [5,6,2]
+ CRUSH rule 3 x 952 [2,7,5]
+ CRUSH rule 3 x 953 [2,5,6]
+ CRUSH rule 3 x 954 [5,2,8]
+ CRUSH rule 3 x 955 [8,2,5]
+ CRUSH rule 3 x 956 [2,7,5]
+ CRUSH rule 3 x 957 [7,2,5]
+ CRUSH rule 3 x 958 [8,5,2]
+ CRUSH rule 3 x 959 [5,2,6]
+ CRUSH rule 3 x 960 [5,6,2]
+ CRUSH rule 3 x 961 [5,2,6]
+ CRUSH rule 3 x 962 [7,5,2]
+ CRUSH rule 3 x 963 [2,5,8]
+ CRUSH rule 3 x 964 [5,2,7]
+ CRUSH rule 3 x 965 [7,5,2]
+ CRUSH rule 3 x 966 [5,6,2]
+ CRUSH rule 3 x 967 [8,5,2]
+ CRUSH rule 3 x 968 [7,2,5]
+ CRUSH rule 3 x 969 [8,2,5]
+ CRUSH rule 3 x 970 [2,8,5]
+ CRUSH rule 3 x 971 [2,8,5]
+ CRUSH rule 3 x 972 [2,7,5]
+ CRUSH rule 3 x 973 [2,8,5]
+ CRUSH rule 3 x 974 [5,2,7]
+ CRUSH rule 3 x 975 [5,8,2]
+ CRUSH rule 3 x 976 [5,7,2]
+ CRUSH rule 3 x 977 [8,5,2]
+ CRUSH rule 3 x 978 [7,2,5]
+ CRUSH rule 3 x 979 [7,2,5]
+ CRUSH rule 3 x 980 [6,2,5]
+ CRUSH rule 3 x 981 [7,5,2]
+ CRUSH rule 3 x 982 [5,2,8]
+ CRUSH rule 3 x 983 [5,6,2]
+ CRUSH rule 3 x 984 [2,8,5]
+ CRUSH rule 3 x 985 [2,5,8]
+ CRUSH rule 3 x 986 [8,5,2]
+ CRUSH rule 3 x 987 [2,5,8]
+ CRUSH rule 3 x 988 [2,5,6]
+ CRUSH rule 3 x 989 [2,8,5]
+ CRUSH rule 3 x 990 [2,6,5]
+ CRUSH rule 3 x 991 [2,5,8]
+ CRUSH rule 3 x 992 [7,2,5]
+ CRUSH rule 3 x 993 [2,6,5]
+ CRUSH rule 3 x 994 [5,6,2]
+ CRUSH rule 3 x 995 [7,2,5]
+ CRUSH rule 3 x 996 [6,5,2]
+ CRUSH rule 3 x 997 [6,5,2]
+ CRUSH rule 3 x 998 [8,2,5]
+ CRUSH rule 3 x 999 [2,7,5]
+ CRUSH rule 3 x 1000 [8,5,2]
+ CRUSH rule 3 x 1001 [2,5,6]
+ CRUSH rule 3 x 1002 [2,5,8]
+ CRUSH rule 3 x 1003 [2,7,5]
+ CRUSH rule 3 x 1004 [6,2,5]
+ CRUSH rule 3 x 1005 [6,2,5]
+ CRUSH rule 3 x 1006 [2,8,5]
+ CRUSH rule 3 x 1007 [2,5,8]
+ CRUSH rule 3 x 1008 [2,7,5]
+ CRUSH rule 3 x 1009 [6,5,2]
+ CRUSH rule 3 x 1010 [5,2,6]
+ CRUSH rule 3 x 1011 [5,2,8]
+ CRUSH rule 3 x 1012 [5,2,8]
+ CRUSH rule 3 x 1013 [5,2,8]
+ CRUSH rule 3 x 1014 [2,8,5]
+ CRUSH rule 3 x 1015 [6,5,2]
+ CRUSH rule 3 x 1016 [2,5,6]
+ CRUSH rule 3 x 1017 [6,2,5]
+ CRUSH rule 3 x 1018 [5,2,6]
+ CRUSH rule 3 x 1019 [5,8,2]
+ CRUSH rule 3 x 1020 [5,2,7]
+ CRUSH rule 3 x 1021 [5,2,7]
+ CRUSH rule 3 x 1022 [2,7,5]
+ CRUSH rule 3 x 1023 [5,2,7]
+ rule 3 (choose-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+ rule 4 (choose-set-two), x = 0..1023, numrep = 2..3
+ CRUSH rule 4 x 0 [2]
+ CRUSH rule 4 x 1 [2,8]
+ CRUSH rule 4 x 2 [2,5]
+ CRUSH rule 4 x 3 [8,2]
+ CRUSH rule 4 x 4 [5]
+ CRUSH rule 4 x 5 [7,8]
+ CRUSH rule 4 x 6 [2,6]
+ CRUSH rule 4 x 7 [5]
+ CRUSH rule 4 x 8 [5]
+ CRUSH rule 4 x 9 [2,5]
+ CRUSH rule 4 x 10 [2]
+ CRUSH rule 4 x 11 [2,7]
+ CRUSH rule 4 x 12 [2]
+ CRUSH rule 4 x 13 [5,8]
+ CRUSH rule 4 x 14 [7,6]
+ CRUSH rule 4 x 15 [7,2]
+ CRUSH rule 4 x 16 [5,6]
+ CRUSH rule 4 x 17 [5]
+ CRUSH rule 4 x 18 [2,5]
+ CRUSH rule 4 x 19 [7,5]
+ CRUSH rule 4 x 20 [2,5]
+ CRUSH rule 4 x 21 [5,7]
+ CRUSH rule 4 x 22 [8,5]
+ CRUSH rule 4 x 23 [5,6]
+ CRUSH rule 4 x 24 [2]
+ CRUSH rule 4 x 25 [5,7]
+ CRUSH rule 4 x 26 [2,8]
+ CRUSH rule 4 x 27 [5,2]
+ CRUSH rule 4 x 28 [6,2]
+ CRUSH rule 4 x 29 [8,5]
+ CRUSH rule 4 x 30 [5,7]
+ CRUSH rule 4 x 31 [8,7]
+ CRUSH rule 4 x 32 [5,6]
+ CRUSH rule 4 x 33 [2,7]
+ CRUSH rule 4 x 34 [2,5]
+ CRUSH rule 4 x 35 [2,8]
+ CRUSH rule 4 x 36 [5,8]
+ CRUSH rule 4 x 37 [2,5]
+ CRUSH rule 4 x 38 [5,8]
+ CRUSH rule 4 x 39 [5,7]
+ CRUSH rule 4 x 40 [7,8]
+ CRUSH rule 4 x 41 [2]
+ CRUSH rule 4 x 42 [5]
+ CRUSH rule 4 x 43 [2,5]
+ CRUSH rule 4 x 44 [2,6]
+ CRUSH rule 4 x 45 [8,2]
+ CRUSH rule 4 x 46 [2,5]
+ CRUSH rule 4 x 47 [5]
+ CRUSH rule 4 x 48 [5,6]
+ CRUSH rule 4 x 49 [5]
+ CRUSH rule 4 x 50 [5]
+ CRUSH rule 4 x 51 [5,6]
+ CRUSH rule 4 x 52 [8,6]
+ CRUSH rule 4 x 53 [5]
+ CRUSH rule 4 x 54 [7,6]
+ CRUSH rule 4 x 55 [8,7]
+ CRUSH rule 4 x 56 [6,5]
+ CRUSH rule 4 x 57 [5]
+ CRUSH rule 4 x 58 [2]
+ CRUSH rule 4 x 59 [5,2]
+ CRUSH rule 4 x 60 [5]
+ CRUSH rule 4 x 61 [5,6]
+ CRUSH rule 4 x 62 [7,2]
+ CRUSH rule 4 x 63 [5,6]
+ CRUSH rule 4 x 64 [5]
+ CRUSH rule 4 x 65 [7,5]
+ CRUSH rule 4 x 66 [5]
+ CRUSH rule 4 x 67 [5,2]
+ CRUSH rule 4 x 68 [2,5]
+ CRUSH rule 4 x 69 [5,2]
+ CRUSH rule 4 x 70 [7,2]
+ CRUSH rule 4 x 71 [2,8]
+ CRUSH rule 4 x 72 [6,2]
+ CRUSH rule 4 x 73 [2,7]
+ CRUSH rule 4 x 74 [2,7]
+ CRUSH rule 4 x 75 [5,2]
+ CRUSH rule 4 x 76 [5,2]
+ CRUSH rule 4 x 77 [7,2]
+ CRUSH rule 4 x 78 [2,5]
+ CRUSH rule 4 x 79 [5]
+ CRUSH rule 4 x 80 [2,5]
+ CRUSH rule 4 x 81 [2]
+ CRUSH rule 4 x 82 [7,2]
+ CRUSH rule 4 x 83 [2,6]
+ CRUSH rule 4 x 84 [7,2]
+ CRUSH rule 4 x 85 [5]
+ CRUSH rule 4 x 86 [2]
+ CRUSH rule 4 x 87 [2,7]
+ CRUSH rule 4 x 88 [2,6]
+ CRUSH rule 4 x 89 [5,2]
+ CRUSH rule 4 x 90 [6,7]
+ CRUSH rule 4 x 91 [5,8]
+ CRUSH rule 4 x 92 [2,8]
+ CRUSH rule 4 x 93 [7,5]
+ CRUSH rule 4 x 94 [2,5]
+ CRUSH rule 4 x 95 [7,5]
+ CRUSH rule 4 x 96 [5,6]
+ CRUSH rule 4 x 97 [8,7]
+ CRUSH rule 4 x 98 [2]
+ CRUSH rule 4 x 99 [2,7]
+ CRUSH rule 4 x 100 [2,7]
+ CRUSH rule 4 x 101 [5,7]
+ CRUSH rule 4 x 102 [5,2]
+ CRUSH rule 4 x 103 [5,7]
+ CRUSH rule 4 x 104 [7,5]
+ CRUSH rule 4 x 105 [2,5]
+ CRUSH rule 4 x 106 [2,6]
+ CRUSH rule 4 x 107 [5,2]
+ CRUSH rule 4 x 108 [7,2]
+ CRUSH rule 4 x 109 [2]
+ CRUSH rule 4 x 110 [5,2]
+ CRUSH rule 4 x 111 [2]
+ CRUSH rule 4 x 112 [2]
+ CRUSH rule 4 x 113 [6,2]
+ CRUSH rule 4 x 114 [7,6]
+ CRUSH rule 4 x 115 [8,2]
+ CRUSH rule 4 x 116 [2]
+ CRUSH rule 4 x 117 [7,5]
+ CRUSH rule 4 x 118 [2,5]
+ CRUSH rule 4 x 119 [5,6]
+ CRUSH rule 4 x 120 [2]
+ CRUSH rule 4 x 121 [2]
+ CRUSH rule 4 x 122 [8,5]
+ CRUSH rule 4 x 123 [2,5]
+ CRUSH rule 4 x 124 [5]
+ CRUSH rule 4 x 125 [2,7]
+ CRUSH rule 4 x 126 [5,2]
+ CRUSH rule 4 x 127 [5,6]
+ CRUSH rule 4 x 128 [5]
+ CRUSH rule 4 x 129 [2]
+ CRUSH rule 4 x 130 [5,8]
+ CRUSH rule 4 x 131 [2]
+ CRUSH rule 4 x 132 [2]
+ CRUSH rule 4 x 133 [5,6]
+ CRUSH rule 4 x 134 [2,8]
+ CRUSH rule 4 x 135 [5,6]
+ CRUSH rule 4 x 136 [2]
+ CRUSH rule 4 x 137 [7,5]
+ CRUSH rule 4 x 138 [8,7]
+ CRUSH rule 4 x 139 [5,2]
+ CRUSH rule 4 x 140 [2,6]
+ CRUSH rule 4 x 141 [6,8]
+ CRUSH rule 4 x 142 [5]
+ CRUSH rule 4 x 143 [5,8]
+ CRUSH rule 4 x 144 [8,2]
+ CRUSH rule 4 x 145 [8,5]
+ CRUSH rule 4 x 146 [2,6]
+ CRUSH rule 4 x 147 [2,8]
+ CRUSH rule 4 x 148 [5]
+ CRUSH rule 4 x 149 [5,8]
+ CRUSH rule 4 x 150 [2,6]
+ CRUSH rule 4 x 151 [5]
+ CRUSH rule 4 x 152 [8,5]
+ CRUSH rule 4 x 153 [8,6]
+ CRUSH rule 4 x 154 [5,2]
+ CRUSH rule 4 x 155 [5]
+ CRUSH rule 4 x 156 [5]
+ CRUSH rule 4 x 157 [5,2]
+ CRUSH rule 4 x 158 [2,8]
+ CRUSH rule 4 x 159 [7,2]
+ CRUSH rule 4 x 160 [2,8]
+ CRUSH rule 4 x 161 [2,5]
+ CRUSH rule 4 x 162 [2,6]
+ CRUSH rule 4 x 163 [5,6]
+ CRUSH rule 4 x 164 [7,8]
+ CRUSH rule 4 x 165 [7,2]
+ CRUSH rule 4 x 166 [2,5]
+ CRUSH rule 4 x 167 [2]
+ CRUSH rule 4 x 168 [5,2]
+ CRUSH rule 4 x 169 [2,6]
+ CRUSH rule 4 x 170 [2]
+ CRUSH rule 4 x 171 [7,5]
+ CRUSH rule 4 x 172 [2,7]
+ CRUSH rule 4 x 173 [8,5]
+ CRUSH rule 4 x 174 [2,5]
+ CRUSH rule 4 x 175 [6,2]
+ CRUSH rule 4 x 176 [5]
+ CRUSH rule 4 x 177 [5]
+ CRUSH rule 4 x 178 [5,2]
+ CRUSH rule 4 x 179 [5,2]
+ CRUSH rule 4 x 180 [5,8]
+ CRUSH rule 4 x 181 [6,2]
+ CRUSH rule 4 x 182 [8,5]
+ CRUSH rule 4 x 183 [7,8]
+ CRUSH rule 4 x 184 [5,7]
+ CRUSH rule 4 x 185 [6,8]
+ CRUSH rule 4 x 186 [2]
+ CRUSH rule 4 x 187 [2,6]
+ CRUSH rule 4 x 188 [2,8]
+ CRUSH rule 4 x 189 [2,7]
+ CRUSH rule 4 x 190 [5,2]
+ CRUSH rule 4 x 191 [7,6]
+ CRUSH rule 4 x 192 [5,2]
+ CRUSH rule 4 x 193 [5,2]
+ CRUSH rule 4 x 194 [2,5]
+ CRUSH rule 4 x 195 [6,5]
+ CRUSH rule 4 x 196 [6,7]
+ CRUSH rule 4 x 197 [6,5]
+ CRUSH rule 4 x 198 [2,5]
+ CRUSH rule 4 x 199 [2,5]
+ CRUSH rule 4 x 200 [2,5]
+ CRUSH rule 4 x 201 [7,2]
+ CRUSH rule 4 x 202 [6,5]
+ CRUSH rule 4 x 203 [5,8]
+ CRUSH rule 4 x 204 [2]
+ CRUSH rule 4 x 205 [2,7]
+ CRUSH rule 4 x 206 [2]
+ CRUSH rule 4 x 207 [5,2]
+ CRUSH rule 4 x 208 [7,2]
+ CRUSH rule 4 x 209 [2]
+ CRUSH rule 4 x 210 [2]
+ CRUSH rule 4 x 211 [5]
+ CRUSH rule 4 x 212 [7,5]
+ CRUSH rule 4 x 213 [8,5]
+ CRUSH rule 4 x 214 [5]
+ CRUSH rule 4 x 215 [8,2]
+ CRUSH rule 4 x 216 [5,2]
+ CRUSH rule 4 x 217 [2]
+ CRUSH rule 4 x 218 [2,7]
+ CRUSH rule 4 x 219 [5,8]
+ CRUSH rule 4 x 220 [5,7]
+ CRUSH rule 4 x 221 [5,6]
+ CRUSH rule 4 x 222 [6,8]
+ CRUSH rule 4 x 223 [2,5]
+ CRUSH rule 4 x 224 [2,5]
+ CRUSH rule 4 x 225 [8,6]
+ CRUSH rule 4 x 226 [7,2]
+ CRUSH rule 4 x 227 [5]
+ CRUSH rule 4 x 228 [5]
+ CRUSH rule 4 x 229 [5]
+ CRUSH rule 4 x 230 [5,7]
+ CRUSH rule 4 x 231 [5]
+ CRUSH rule 4 x 232 [2,7]
+ CRUSH rule 4 x 233 [5]
+ CRUSH rule 4 x 234 [2]
+ CRUSH rule 4 x 235 [5,8]
+ CRUSH rule 4 x 236 [5,2]
+ CRUSH rule 4 x 237 [5,7]
+ CRUSH rule 4 x 238 [5]
+ CRUSH rule 4 x 239 [8,7]
+ CRUSH rule 4 x 240 [5,7]
+ CRUSH rule 4 x 241 [5,2]
+ CRUSH rule 4 x 242 [5]
+ CRUSH rule 4 x 243 [5,7]
+ CRUSH rule 4 x 244 [5,6]
+ CRUSH rule 4 x 245 [7,6]
+ CRUSH rule 4 x 246 [2,5]
+ CRUSH rule 4 x 247 [6,2]
+ CRUSH rule 4 x 248 [8,2]
+ CRUSH rule 4 x 249 [2]
+ CRUSH rule 4 x 250 [2]
+ CRUSH rule 4 x 251 [2,5]
+ CRUSH rule 4 x 252 [5,7]
+ CRUSH rule 4 x 253 [5,2]
+ CRUSH rule 4 x 254 [5]
+ CRUSH rule 4 x 255 [2,7]
+ CRUSH rule 4 x 256 [5,7]
+ CRUSH rule 4 x 257 [2,8]
+ CRUSH rule 4 x 258 [5]
+ CRUSH rule 4 x 259 [5]
+ CRUSH rule 4 x 260 [5,6]
+ CRUSH rule 4 x 261 [8,7]
+ CRUSH rule 4 x 262 [5]
+ CRUSH rule 4 x 263 [6,8]
+ CRUSH rule 4 x 264 [5,6]
+ CRUSH rule 4 x 265 [8,6]
+ CRUSH rule 4 x 266 [8,2]
+ CRUSH rule 4 x 267 [2,5]
+ CRUSH rule 4 x 268 [2,7]
+ CRUSH rule 4 x 269 [2,8]
+ CRUSH rule 4 x 270 [5,2]
+ CRUSH rule 4 x 271 [7,5]
+ CRUSH rule 4 x 272 [2,8]
+ CRUSH rule 4 x 273 [5]
+ CRUSH rule 4 x 274 [6,8]
+ CRUSH rule 4 x 275 [5]
+ CRUSH rule 4 x 276 [7,2]
+ CRUSH rule 4 x 277 [6,5]
+ CRUSH rule 4 x 278 [6,8]
+ CRUSH rule 4 x 279 [8,5]
+ CRUSH rule 4 x 280 [2,6]
+ CRUSH rule 4 x 281 [8,2]
+ CRUSH rule 4 x 282 [5,2]
+ CRUSH rule 4 x 283 [8,2]
+ CRUSH rule 4 x 284 [6,5]
+ CRUSH rule 4 x 285 [5]
+ CRUSH rule 4 x 286 [2]
+ CRUSH rule 4 x 287 [2,5]
+ CRUSH rule 4 x 288 [8,2]
+ CRUSH rule 4 x 289 [5,6]
+ CRUSH rule 4 x 290 [2,5]
+ CRUSH rule 4 x 291 [2]
+ CRUSH rule 4 x 292 [8,2]
+ CRUSH rule 4 x 293 [6,2]
+ CRUSH rule 4 x 294 [7,5]
+ CRUSH rule 4 x 295 [5,8]
+ CRUSH rule 4 x 296 [5,2]
+ CRUSH rule 4 x 297 [6,2]
+ CRUSH rule 4 x 298 [2]
+ CRUSH rule 4 x 299 [2]
+ CRUSH rule 4 x 300 [8,7]
+ CRUSH rule 4 x 301 [2,8]
+ CRUSH rule 4 x 302 [5,2]
+ CRUSH rule 4 x 303 [7,5]
+ CRUSH rule 4 x 304 [2,7]
+ CRUSH rule 4 x 305 [5,8]
+ CRUSH rule 4 x 306 [2,7]
+ CRUSH rule 4 x 307 [2]
+ CRUSH rule 4 x 308 [2,8]
+ CRUSH rule 4 x 309 [7,5]
+ CRUSH rule 4 x 310 [5]
+ CRUSH rule 4 x 311 [5]
+ CRUSH rule 4 x 312 [2]
+ CRUSH rule 4 x 313 [5]
+ CRUSH rule 4 x 314 [5]
+ CRUSH rule 4 x 315 [2]
+ CRUSH rule 4 x 316 [6,5]
+ CRUSH rule 4 x 317 [2,6]
+ CRUSH rule 4 x 318 [8,6]
+ CRUSH rule 4 x 319 [5,2]
+ CRUSH rule 4 x 320 [5,7]
+ CRUSH rule 4 x 321 [2,5]
+ CRUSH rule 4 x 322 [2,7]
+ CRUSH rule 4 x 323 [5,7]
+ CRUSH rule 4 x 324 [7,2]
+ CRUSH rule 4 x 325 [5,6]
+ CRUSH rule 4 x 326 [5]
+ CRUSH rule 4 x 327 [2,6]
+ CRUSH rule 4 x 328 [7,5]
+ CRUSH rule 4 x 329 [5,6]
+ CRUSH rule 4 x 330 [5,7]
+ CRUSH rule 4 x 331 [2,6]
+ CRUSH rule 4 x 332 [2]
+ CRUSH rule 4 x 333 [6,8]
+ CRUSH rule 4 x 334 [8,5]
+ CRUSH rule 4 x 335 [7,2]
+ CRUSH rule 4 x 336 [5,6]
+ CRUSH rule 4 x 337 [7,2]
+ CRUSH rule 4 x 338 [5,6]
+ CRUSH rule 4 x 339 [7,5]
+ CRUSH rule 4 x 340 [2]
+ CRUSH rule 4 x 341 [5,2]
+ CRUSH rule 4 x 342 [2,7]
+ CRUSH rule 4 x 343 [6,7]
+ CRUSH rule 4 x 344 [6,2]
+ CRUSH rule 4 x 345 [5]
+ CRUSH rule 4 x 346 [8,2]
+ CRUSH rule 4 x 347 [5,2]
+ CRUSH rule 4 x 348 [8,2]
+ CRUSH rule 4 x 349 [2,6]
+ CRUSH rule 4 x 350 [8,5]
+ CRUSH rule 4 x 351 [5,6]
+ CRUSH rule 4 x 352 [2]
+ CRUSH rule 4 x 353 [6,5]
+ CRUSH rule 4 x 354 [2,5]
+ CRUSH rule 4 x 355 [5]
+ CRUSH rule 4 x 356 [5]
+ CRUSH rule 4 x 357 [6,2]
+ CRUSH rule 4 x 358 [2]
+ CRUSH rule 4 x 359 [6,7]
+ CRUSH rule 4 x 360 [5]
+ CRUSH rule 4 x 361 [8,5]
+ CRUSH rule 4 x 362 [5]
+ CRUSH rule 4 x 363 [5,2]
+ CRUSH rule 4 x 364 [2,5]
+ CRUSH rule 4 x 365 [6,7]
+ CRUSH rule 4 x 366 [7,2]
+ CRUSH rule 4 x 367 [5]
+ CRUSH rule 4 x 368 [7,5]
+ CRUSH rule 4 x 369 [5,7]
+ CRUSH rule 4 x 370 [8,7]
+ CRUSH rule 4 x 371 [2,5]
+ CRUSH rule 4 x 372 [5,2]
+ CRUSH rule 4 x 373 [2,6]
+ CRUSH rule 4 x 374 [5,8]
+ CRUSH rule 4 x 375 [6,5]
+ CRUSH rule 4 x 376 [7,2]
+ CRUSH rule 4 x 377 [2]
+ CRUSH rule 4 x 378 [2]
+ CRUSH rule 4 x 379 [8,5]
+ CRUSH rule 4 x 380 [2,5]
+ CRUSH rule 4 x 381 [2,5]
+ CRUSH rule 4 x 382 [2,5]
+ CRUSH rule 4 x 383 [5]
+ CRUSH rule 4 x 384 [7,2]
+ CRUSH rule 4 x 385 [7,5]
+ CRUSH rule 4 x 386 [2,5]
+ CRUSH rule 4 x 387 [2,5]
+ CRUSH rule 4 x 388 [5,2]
+ CRUSH rule 4 x 389 [2,5]
+ CRUSH rule 4 x 390 [5,6]
+ CRUSH rule 4 x 391 [5,6]
+ CRUSH rule 4 x 392 [2,8]
+ CRUSH rule 4 x 393 [5,2]
+ CRUSH rule 4 x 394 [5,7]
+ CRUSH rule 4 x 395 [5,2]
+ CRUSH rule 4 x 396 [5,2]
+ CRUSH rule 4 x 397 [2]
+ CRUSH rule 4 x 398 [2,5]
+ CRUSH rule 4 x 399 [8,7]
+ CRUSH rule 4 x 400 [8,2]
+ CRUSH rule 4 x 401 [2]
+ CRUSH rule 4 x 402 [7,8]
+ CRUSH rule 4 x 403 [2]
+ CRUSH rule 4 x 404 [5]
+ CRUSH rule 4 x 405 [6,5]
+ CRUSH rule 4 x 406 [2]
+ CRUSH rule 4 x 407 [2,8]
+ CRUSH rule 4 x 408 [5,2]
+ CRUSH rule 4 x 409 [7,5]
+ CRUSH rule 4 x 410 [8,6]
+ CRUSH rule 4 x 411 [2]
+ CRUSH rule 4 x 412 [2,5]
+ CRUSH rule 4 x 413 [5,2]
+ CRUSH rule 4 x 414 [5,2]
+ CRUSH rule 4 x 415 [2,6]
+ CRUSH rule 4 x 416 [2]
+ CRUSH rule 4 x 417 [8,7]
+ CRUSH rule 4 x 418 [7,6]
+ CRUSH rule 4 x 419 [8,5]
+ CRUSH rule 4 x 420 [2,5]
+ CRUSH rule 4 x 421 [8,6]
+ CRUSH rule 4 x 422 [6,7]
+ CRUSH rule 4 x 423 [2,5]
+ CRUSH rule 4 x 424 [8,5]
+ CRUSH rule 4 x 425 [2,5]
+ CRUSH rule 4 x 426 [6,7]
+ CRUSH rule 4 x 427 [2,7]
+ CRUSH rule 4 x 428 [5]
+ CRUSH rule 4 x 429 [5,6]
+ CRUSH rule 4 x 430 [5,6]
+ CRUSH rule 4 x 431 [5]
+ CRUSH rule 4 x 432 [7,2]
+ CRUSH rule 4 x 433 [6,5]
+ CRUSH rule 4 x 434 [5,2]
+ CRUSH rule 4 x 435 [2,5]
+ CRUSH rule 4 x 436 [5,2]
+ CRUSH rule 4 x 437 [7,5]
+ CRUSH rule 4 x 438 [2,5]
+ CRUSH rule 4 x 439 [2,5]
+ CRUSH rule 4 x 440 [2,7]
+ CRUSH rule 4 x 441 [5,7]
+ CRUSH rule 4 x 442 [2,5]
+ CRUSH rule 4 x 443 [6,8]
+ CRUSH rule 4 x 444 [7,2]
+ CRUSH rule 4 x 445 [6,5]
+ CRUSH rule 4 x 446 [5]
+ CRUSH rule 4 x 447 [2]
+ CRUSH rule 4 x 448 [7,2]
+ CRUSH rule 4 x 449 [7,8]
+ CRUSH rule 4 x 450 [5]
+ CRUSH rule 4 x 451 [6,8]
+ CRUSH rule 4 x 452 [8,5]
+ CRUSH rule 4 x 453 [6,8]
+ CRUSH rule 4 x 454 [6,7]
+ CRUSH rule 4 x 455 [2,7]
+ CRUSH rule 4 x 456 [6,8]
+ CRUSH rule 4 x 457 [7,2]
+ CRUSH rule 4 x 458 [2,8]
+ CRUSH rule 4 x 459 [2]
+ CRUSH rule 4 x 460 [6,5]
+ CRUSH rule 4 x 461 [6,5]
+ CRUSH rule 4 x 462 [8,2]
+ CRUSH rule 4 x 463 [6,7]
+ CRUSH rule 4 x 464 [7,5]
+ CRUSH rule 4 x 465 [7,6]
+ CRUSH rule 4 x 466 [5,8]
+ CRUSH rule 4 x 467 [6,5]
+ CRUSH rule 4 x 468 [7,8]
+ CRUSH rule 4 x 469 [7,2]
+ CRUSH rule 4 x 470 [5,2]
+ CRUSH rule 4 x 471 [2]
+ CRUSH rule 4 x 472 [5]
+ CRUSH rule 4 x 473 [2]
+ CRUSH rule 4 x 474 [6,2]
+ CRUSH rule 4 x 475 [6,7]
+ CRUSH rule 4 x 476 [5]
+ CRUSH rule 4 x 477 [5,8]
+ CRUSH rule 4 x 478 [6,7]
+ CRUSH rule 4 x 479 [2,5]
+ CRUSH rule 4 x 480 [2,8]
+ CRUSH rule 4 x 481 [2,5]
+ CRUSH rule 4 x 482 [5]
+ CRUSH rule 4 x 483 [2]
+ CRUSH rule 4 x 484 [2]
+ CRUSH rule 4 x 485 [5,7]
+ CRUSH rule 4 x 486 [5,2]
+ CRUSH rule 4 x 487 [5,2]
+ CRUSH rule 4 x 488 [5,7]
+ CRUSH rule 4 x 489 [2,8]
+ CRUSH rule 4 x 490 [6,5]
+ CRUSH rule 4 x 491 [2]
+ CRUSH rule 4 x 492 [6,5]
+ CRUSH rule 4 x 493 [2]
+ CRUSH rule 4 x 494 [2]
+ CRUSH rule 4 x 495 [5]
+ CRUSH rule 4 x 496 [7,5]
+ CRUSH rule 4 x 497 [5,7]
+ CRUSH rule 4 x 498 [2,5]
+ CRUSH rule 4 x 499 [8,5]
+ CRUSH rule 4 x 500 [5,6]
+ CRUSH rule 4 x 501 [2,7]
+ CRUSH rule 4 x 502 [7,2]
+ CRUSH rule 4 x 503 [2,5]
+ CRUSH rule 4 x 504 [5,6]
+ CRUSH rule 4 x 505 [2,7]
+ CRUSH rule 4 x 506 [5]
+ CRUSH rule 4 x 507 [6,2]
+ CRUSH rule 4 x 508 [2]
+ CRUSH rule 4 x 509 [7,5]
+ CRUSH rule 4 x 510 [6,2]
+ CRUSH rule 4 x 511 [5,8]
+ CRUSH rule 4 x 512 [7,6]
+ CRUSH rule 4 x 513 [7,2]
+ CRUSH rule 4 x 514 [5]
+ CRUSH rule 4 x 515 [8,5]
+ CRUSH rule 4 x 516 [5,2]
+ CRUSH rule 4 x 517 [7,8]
+ CRUSH rule 4 x 518 [5,6]
+ CRUSH rule 4 x 519 [7,5]
+ CRUSH rule 4 x 520 [2,6]
+ CRUSH rule 4 x 521 [8,7]
+ CRUSH rule 4 x 522 [6,8]
+ CRUSH rule 4 x 523 [5,2]
+ CRUSH rule 4 x 524 [2,5]
+ CRUSH rule 4 x 525 [2,5]
+ CRUSH rule 4 x 526 [2,5]
+ CRUSH rule 4 x 527 [2]
+ CRUSH rule 4 x 528 [5]
+ CRUSH rule 4 x 529 [5,7]
+ CRUSH rule 4 x 530 [6,7]
+ CRUSH rule 4 x 531 [6,2]
+ CRUSH rule 4 x 532 [6,5]
+ CRUSH rule 4 x 533 [5,6]
+ CRUSH rule 4 x 534 [7,5]
+ CRUSH rule 4 x 535 [8,6]
+ CRUSH rule 4 x 536 [6,7]
+ CRUSH rule 4 x 537 [5,7]
+ CRUSH rule 4 x 538 [6,8]
+ CRUSH rule 4 x 539 [8,5]
+ CRUSH rule 4 x 540 [2,6]
+ CRUSH rule 4 x 541 [2,5]
+ CRUSH rule 4 x 542 [5]
+ CRUSH rule 4 x 543 [6,2]
+ CRUSH rule 4 x 544 [5,7]
+ CRUSH rule 4 x 545 [5,7]
+ CRUSH rule 4 x 546 [6,2]
+ CRUSH rule 4 x 547 [8,2]
+ CRUSH rule 4 x 548 [5,2]
+ CRUSH rule 4 x 549 [5,8]
+ CRUSH rule 4 x 550 [2,5]
+ CRUSH rule 4 x 551 [7,5]
+ CRUSH rule 4 x 552 [5]
+ CRUSH rule 4 x 553 [5,2]
+ CRUSH rule 4 x 554 [2,8]
+ CRUSH rule 4 x 555 [5,2]
+ CRUSH rule 4 x 556 [5]
+ CRUSH rule 4 x 557 [7,5]
+ CRUSH rule 4 x 558 [5,2]
+ CRUSH rule 4 x 559 [5,2]
+ CRUSH rule 4 x 560 [8,5]
+ CRUSH rule 4 x 561 [6,5]
+ CRUSH rule 4 x 562 [5]
+ CRUSH rule 4 x 563 [2,6]
+ CRUSH rule 4 x 564 [5,2]
+ CRUSH rule 4 x 565 [5,6]
+ CRUSH rule 4 x 566 [5,7]
+ CRUSH rule 4 x 567 [5,6]
+ CRUSH rule 4 x 568 [7,5]
+ CRUSH rule 4 x 569 [5]
+ CRUSH rule 4 x 570 [2,5]
+ CRUSH rule 4 x 571 [5,7]
+ CRUSH rule 4 x 572 [5]
+ CRUSH rule 4 x 573 [5,2]
+ CRUSH rule 4 x 574 [2]
+ CRUSH rule 4 x 575 [8,6]
+ CRUSH rule 4 x 576 [5,6]
+ CRUSH rule 4 x 577 [8,2]
+ CRUSH rule 4 x 578 [6,8]
+ CRUSH rule 4 x 579 [5,2]
+ CRUSH rule 4 x 580 [5,2]
+ CRUSH rule 4 x 581 [7,2]
+ CRUSH rule 4 x 582 [2,8]
+ CRUSH rule 4 x 583 [6,2]
+ CRUSH rule 4 x 584 [8,2]
+ CRUSH rule 4 x 585 [7,2]
+ CRUSH rule 4 x 586 [2]
+ CRUSH rule 4 x 587 [2,5]
+ CRUSH rule 4 x 588 [5]
+ CRUSH rule 4 x 589 [7,2]
+ CRUSH rule 4 x 590 [6,2]
+ CRUSH rule 4 x 591 [5,2]
+ CRUSH rule 4 x 592 [2]
+ CRUSH rule 4 x 593 [2,8]
+ CRUSH rule 4 x 594 [2,7]
+ CRUSH rule 4 x 595 [7,2]
+ CRUSH rule 4 x 596 [5]
+ CRUSH rule 4 x 597 [5,2]
+ CRUSH rule 4 x 598 [5,2]
+ CRUSH rule 4 x 599 [5,2]
+ CRUSH rule 4 x 600 [7,2]
+ CRUSH rule 4 x 601 [2,7]
+ CRUSH rule 4 x 602 [5,7]
+ CRUSH rule 4 x 603 [5,2]
+ CRUSH rule 4 x 604 [7,5]
+ CRUSH rule 4 x 605 [5,2]
+ CRUSH rule 4 x 606 [2]
+ CRUSH rule 4 x 607 [2,5]
+ CRUSH rule 4 x 608 [5]
+ CRUSH rule 4 x 609 [5,2]
+ CRUSH rule 4 x 610 [5,7]
+ CRUSH rule 4 x 611 [2]
+ CRUSH rule 4 x 612 [2]
+ CRUSH rule 4 x 613 [7,2]
+ CRUSH rule 4 x 614 [7,8]
+ CRUSH rule 4 x 615 [6,8]
+ CRUSH rule 4 x 616 [2,8]
+ CRUSH rule 4 x 617 [6,2]
+ CRUSH rule 4 x 618 [7,6]
+ CRUSH rule 4 x 619 [5,2]
+ CRUSH rule 4 x 620 [5,2]
+ CRUSH rule 4 x 621 [5,8]
+ CRUSH rule 4 x 622 [2,5]
+ CRUSH rule 4 x 623 [2,6]
+ CRUSH rule 4 x 624 [5]
+ CRUSH rule 4 x 625 [2,5]
+ CRUSH rule 4 x 626 [7,8]
+ CRUSH rule 4 x 627 [2,7]
+ CRUSH rule 4 x 628 [8,2]
+ CRUSH rule 4 x 629 [2,6]
+ CRUSH rule 4 x 630 [2,7]
+ CRUSH rule 4 x 631 [2,7]
+ CRUSH rule 4 x 632 [7,2]
+ CRUSH rule 4 x 633 [8,6]
+ CRUSH rule 4 x 634 [2,5]
+ CRUSH rule 4 x 635 [5,6]
+ CRUSH rule 4 x 636 [2,5]
+ CRUSH rule 4 x 637 [5,2]
+ CRUSH rule 4 x 638 [6,8]
+ CRUSH rule 4 x 639 [5,2]
+ CRUSH rule 4 x 640 [5,2]
+ CRUSH rule 4 x 641 [7,2]
+ CRUSH rule 4 x 642 [2]
+ CRUSH rule 4 x 643 [5]
+ CRUSH rule 4 x 644 [8,2]
+ CRUSH rule 4 x 645 [5]
+ CRUSH rule 4 x 646 [8,2]
+ CRUSH rule 4 x 647 [7,2]
+ CRUSH rule 4 x 648 [2,6]
+ CRUSH rule 4 x 649 [5,7]
+ CRUSH rule 4 x 650 [7,8]
+ CRUSH rule 4 x 651 [5,7]
+ CRUSH rule 4 x 652 [5]
+ CRUSH rule 4 x 653 [8,5]
+ CRUSH rule 4 x 654 [7,5]
+ CRUSH rule 4 x 655 [2,5]
+ CRUSH rule 4 x 656 [5]
+ CRUSH rule 4 x 657 [6,2]
+ CRUSH rule 4 x 658 [5]
+ CRUSH rule 4 x 659 [5,6]
+ CRUSH rule 4 x 660 [7,8]
+ CRUSH rule 4 x 661 [2,8]
+ CRUSH rule 4 x 662 [5]
+ CRUSH rule 4 x 663 [2,5]
+ CRUSH rule 4 x 664 [2,5]
+ CRUSH rule 4 x 665 [5,7]
+ CRUSH rule 4 x 666 [2,8]
+ CRUSH rule 4 x 667 [2,5]
+ CRUSH rule 4 x 668 [5,7]
+ CRUSH rule 4 x 669 [6,5]
+ CRUSH rule 4 x 670 [5,2]
+ CRUSH rule 4 x 671 [2]
+ CRUSH rule 4 x 672 [5]
+ CRUSH rule 4 x 673 [5]
+ CRUSH rule 4 x 674 [5,2]
+ CRUSH rule 4 x 675 [2,8]
+ CRUSH rule 4 x 676 [2]
+ CRUSH rule 4 x 677 [5,2]
+ CRUSH rule 4 x 678 [2,5]
+ CRUSH rule 4 x 679 [6,2]
+ CRUSH rule 4 x 680 [2,5]
+ CRUSH rule 4 x 681 [5,7]
+ CRUSH rule 4 x 682 [2,5]
+ CRUSH rule 4 x 683 [2]
+ CRUSH rule 4 x 684 [7,2]
+ CRUSH rule 4 x 685 [7,2]
+ CRUSH rule 4 x 686 [2,5]
+ CRUSH rule 4 x 687 [5]
+ CRUSH rule 4 x 688 [5,7]
+ CRUSH rule 4 x 689 [6,5]
+ CRUSH rule 4 x 690 [8,2]
+ CRUSH rule 4 x 691 [5,2]
+ CRUSH rule 4 x 692 [7,2]
+ CRUSH rule 4 x 693 [6,7]
+ CRUSH rule 4 x 694 [6,5]
+ CRUSH rule 4 x 695 [2,8]
+ CRUSH rule 4 x 696 [2,5]
+ CRUSH rule 4 x 697 [6,2]
+ CRUSH rule 4 x 698 [6,2]
+ CRUSH rule 4 x 699 [2,6]
+ CRUSH rule 4 x 700 [2,5]
+ CRUSH rule 4 x 701 [5]
+ CRUSH rule 4 x 702 [5]
+ CRUSH rule 4 x 703 [8,5]
+ CRUSH rule 4 x 704 [2,5]
+ CRUSH rule 4 x 705 [8,6]
+ CRUSH rule 4 x 706 [2]
+ CRUSH rule 4 x 707 [7,8]
+ CRUSH rule 4 x 708 [5]
+ CRUSH rule 4 x 709 [6,5]
+ CRUSH rule 4 x 710 [8,5]
+ CRUSH rule 4 x 711 [2,5]
+ CRUSH rule 4 x 712 [2,5]
+ CRUSH rule 4 x 713 [6,7]
+ CRUSH rule 4 x 714 [5,2]
+ CRUSH rule 4 x 715 [2]
+ CRUSH rule 4 x 716 [5,6]
+ CRUSH rule 4 x 717 [8,7]
+ CRUSH rule 4 x 718 [5,7]
+ CRUSH rule 4 x 719 [2,6]
+ CRUSH rule 4 x 720 [6,8]
+ CRUSH rule 4 x 721 [5]
+ CRUSH rule 4 x 722 [5]
+ CRUSH rule 4 x 723 [5,2]
+ CRUSH rule 4 x 724 [2,6]
+ CRUSH rule 4 x 725 [2]
+ CRUSH rule 4 x 726 [5,8]
+ CRUSH rule 4 x 727 [5,6]
+ CRUSH rule 4 x 728 [2]
+ CRUSH rule 4 x 729 [5]
+ CRUSH rule 4 x 730 [5,7]
+ CRUSH rule 4 x 731 [5,2]
+ CRUSH rule 4 x 732 [2,5]
+ CRUSH rule 4 x 733 [5]
+ CRUSH rule 4 x 734 [6,5]
+ CRUSH rule 4 x 735 [5,8]
+ CRUSH rule 4 x 736 [5]
+ CRUSH rule 4 x 737 [2]
+ CRUSH rule 4 x 738 [5,2]
+ CRUSH rule 4 x 739 [2]
+ CRUSH rule 4 x 740 [2]
+ CRUSH rule 4 x 741 [7,8]
+ CRUSH rule 4 x 742 [8,2]
+ CRUSH rule 4 x 743 [7,2]
+ CRUSH rule 4 x 744 [5,7]
+ CRUSH rule 4 x 745 [5]
+ CRUSH rule 4 x 746 [5,2]
+ CRUSH rule 4 x 747 [6,2]
+ CRUSH rule 4 x 748 [2,7]
+ CRUSH rule 4 x 749 [5]
+ CRUSH rule 4 x 750 [2,6]
+ CRUSH rule 4 x 751 [2]
+ CRUSH rule 4 x 752 [8,2]
+ CRUSH rule 4 x 753 [7,8]
+ CRUSH rule 4 x 754 [8,6]
+ CRUSH rule 4 x 755 [2]
+ CRUSH rule 4 x 756 [5,6]
+ CRUSH rule 4 x 757 [8,6]
+ CRUSH rule 4 x 758 [6,2]
+ CRUSH rule 4 x 759 [8,5]
+ CRUSH rule 4 x 760 [2,5]
+ CRUSH rule 4 x 761 [5,2]
+ CRUSH rule 4 x 762 [2,7]
+ CRUSH rule 4 x 763 [8,6]
+ CRUSH rule 4 x 764 [2,7]
+ CRUSH rule 4 x 765 [6,5]
+ CRUSH rule 4 x 766 [8,5]
+ CRUSH rule 4 x 767 [2]
+ CRUSH rule 4 x 768 [8,5]
+ CRUSH rule 4 x 769 [6,2]
+ CRUSH rule 4 x 770 [6,2]
+ CRUSH rule 4 x 771 [7,2]
+ CRUSH rule 4 x 772 [8,5]
+ CRUSH rule 4 x 773 [5,2]
+ CRUSH rule 4 x 774 [5,6]
+ CRUSH rule 4 x 775 [6,8]
+ CRUSH rule 4 x 776 [7,2]
+ CRUSH rule 4 x 777 [5,2]
+ CRUSH rule 4 x 778 [2,8]
+ CRUSH rule 4 x 779 [2,7]
+ CRUSH rule 4 x 780 [2]
+ CRUSH rule 4 x 781 [6,5]
+ CRUSH rule 4 x 782 [5]
+ CRUSH rule 4 x 783 [7,2]
+ CRUSH rule 4 x 784 [2]
+ CRUSH rule 4 x 785 [6,2]
+ CRUSH rule 4 x 786 [7,6]
+ CRUSH rule 4 x 787 [2]
+ CRUSH rule 4 x 788 [6,2]
+ CRUSH rule 4 x 789 [2,5]
+ CRUSH rule 4 x 790 [8,5]
+ CRUSH rule 4 x 791 [5,8]
+ CRUSH rule 4 x 792 [5,8]
+ CRUSH rule 4 x 793 [6,2]
+ CRUSH rule 4 x 794 [2,6]
+ CRUSH rule 4 x 795 [2,5]
+ CRUSH rule 4 x 796 [5]
+ CRUSH rule 4 x 797 [2,5]
+ CRUSH rule 4 x 798 [6,8]
+ CRUSH rule 4 x 799 [5,2]
+ CRUSH rule 4 x 800 [5,2]
+ CRUSH rule 4 x 801 [5,6]
+ CRUSH rule 4 x 802 [2,8]
+ CRUSH rule 4 x 803 [2,5]
+ CRUSH rule 4 x 804 [6,2]
+ CRUSH rule 4 x 805 [5,6]
+ CRUSH rule 4 x 806 [2,5]
+ CRUSH rule 4 x 807 [5]
+ CRUSH rule 4 x 808 [5,6]
+ CRUSH rule 4 x 809 [2,5]
+ CRUSH rule 4 x 810 [5,7]
+ CRUSH rule 4 x 811 [8,5]
+ CRUSH rule 4 x 812 [8,5]
+ CRUSH rule 4 x 813 [6,5]
+ CRUSH rule 4 x 814 [5,6]
+ CRUSH rule 4 x 815 [5,2]
+ CRUSH rule 4 x 816 [2]
+ CRUSH rule 4 x 817 [5]
+ CRUSH rule 4 x 818 [5]
+ CRUSH rule 4 x 819 [5,2]
+ CRUSH rule 4 x 820 [5]
+ CRUSH rule 4 x 821 [5]
+ CRUSH rule 4 x 822 [2]
+ CRUSH rule 4 x 823 [5,8]
+ CRUSH rule 4 x 824 [5,7]
+ CRUSH rule 4 x 825 [2,8]
+ CRUSH rule 4 x 826 [7,2]
+ CRUSH rule 4 x 827 [2,8]
+ CRUSH rule 4 x 828 [2,5]
+ CRUSH rule 4 x 829 [5,6]
+ CRUSH rule 4 x 830 [2,5]
+ CRUSH rule 4 x 831 [2,6]
+ CRUSH rule 4 x 832 [5]
+ CRUSH rule 4 x 833 [2]
+ CRUSH rule 4 x 834 [5]
+ CRUSH rule 4 x 835 [8,5]
+ CRUSH rule 4 x 836 [5]
+ CRUSH rule 4 x 837 [6,5]
+ CRUSH rule 4 x 838 [6,7]
+ CRUSH rule 4 x 839 [5,2]
+ CRUSH rule 4 x 840 [7,8]
+ CRUSH rule 4 x 841 [5,8]
+ CRUSH rule 4 x 842 [2,5]
+ CRUSH rule 4 x 843 [6,5]
+ CRUSH rule 4 x 844 [5,8]
+ CRUSH rule 4 x 845 [5,8]
+ CRUSH rule 4 x 846 [5,2]
+ CRUSH rule 4 x 847 [2]
+ CRUSH rule 4 x 848 [2,6]
+ CRUSH rule 4 x 849 [5]
+ CRUSH rule 4 x 850 [2]
+ CRUSH rule 4 x 851 [6,8]
+ CRUSH rule 4 x 852 [7,5]
+ CRUSH rule 4 x 853 [6,8]
+ CRUSH rule 4 x 854 [7,6]
+ CRUSH rule 4 x 855 [5,7]
+ CRUSH rule 4 x 856 [6,7]
+ CRUSH rule 4 x 857 [8,5]
+ CRUSH rule 4 x 858 [6,5]
+ CRUSH rule 4 x 859 [6,2]
+ CRUSH rule 4 x 860 [5,2]
+ CRUSH rule 4 x 861 [8,7]
+ CRUSH rule 4 x 862 [6,2]
+ CRUSH rule 4 x 863 [8,7]
+ CRUSH rule 4 x 864 [5,6]
+ CRUSH rule 4 x 865 [8,2]
+ CRUSH rule 4 x 866 [5]
+ CRUSH rule 4 x 867 [6,5]
+ CRUSH rule 4 x 868 [6,5]
+ CRUSH rule 4 x 869 [8,7]
+ CRUSH rule 4 x 870 [2,5]
+ CRUSH rule 4 x 871 [5]
+ CRUSH rule 4 x 872 [5,2]
+ CRUSH rule 4 x 873 [5,6]
+ CRUSH rule 4 x 874 [2,6]
+ CRUSH rule 4 x 875 [2,6]
+ CRUSH rule 4 x 876 [5,8]
+ CRUSH rule 4 x 877 [6,5]
+ CRUSH rule 4 x 878 [5]
+ CRUSH rule 4 x 879 [7,5]
+ CRUSH rule 4 x 880 [5]
+ CRUSH rule 4 x 881 [5,6]
+ CRUSH rule 4 x 882 [5,2]
+ CRUSH rule 4 x 883 [2]
+ CRUSH rule 4 x 884 [6,2]
+ CRUSH rule 4 x 885 [5,2]
+ CRUSH rule 4 x 886 [5,6]
+ CRUSH rule 4 x 887 [7,5]
+ CRUSH rule 4 x 888 [6,8]
+ CRUSH rule 4 x 889 [2]
+ CRUSH rule 4 x 890 [7,2]
+ CRUSH rule 4 x 891 [2,8]
+ CRUSH rule 4 x 892 [6,2]
+ CRUSH rule 4 x 893 [2,5]
+ CRUSH rule 4 x 894 [7,5]
+ CRUSH rule 4 x 895 [5]
+ CRUSH rule 4 x 896 [2,8]
+ CRUSH rule 4 x 897 [5,2]
+ CRUSH rule 4 x 898 [2,5]
+ CRUSH rule 4 x 899 [2,7]
+ CRUSH rule 4 x 900 [5,2]
+ CRUSH rule 4 x 901 [5,2]
+ CRUSH rule 4 x 902 [8,5]
+ CRUSH rule 4 x 903 [5,7]
+ CRUSH rule 4 x 904 [5,6]
+ CRUSH rule 4 x 905 [6,2]
+ CRUSH rule 4 x 906 [2]
+ CRUSH rule 4 x 907 [7,2]
+ CRUSH rule 4 x 908 [5,8]
+ CRUSH rule 4 x 909 [2,5]
+ CRUSH rule 4 x 910 [6,5]
+ CRUSH rule 4 x 911 [5,8]
+ CRUSH rule 4 x 912 [2]
+ CRUSH rule 4 x 913 [7,6]
+ CRUSH rule 4 x 914 [6,5]
+ CRUSH rule 4 x 915 [8,2]
+ CRUSH rule 4 x 916 [5,2]
+ CRUSH rule 4 x 917 [2,5]
+ CRUSH rule 4 x 918 [8,2]
+ CRUSH rule 4 x 919 [6,2]
+ CRUSH rule 4 x 920 [7,6]
+ CRUSH rule 4 x 921 [2,5]
+ CRUSH rule 4 x 922 [6,7]
+ CRUSH rule 4 x 923 [5]
+ CRUSH rule 4 x 924 [5]
+ CRUSH rule 4 x 925 [5,7]
+ CRUSH rule 4 x 926 [5]
+ CRUSH rule 4 x 927 [2,6]
+ CRUSH rule 4 x 928 [8,2]
+ CRUSH rule 4 x 929 [5]
+ CRUSH rule 4 x 930 [2,5]
+ CRUSH rule 4 x 931 [5,2]
+ CRUSH rule 4 x 932 [5]
+ CRUSH rule 4 x 933 [8,5]
+ CRUSH rule 4 x 934 [5]
+ CRUSH rule 4 x 935 [6,5]
+ CRUSH rule 4 x 936 [2,6]
+ CRUSH rule 4 x 937 [5]
+ CRUSH rule 4 x 938 [6,5]
+ CRUSH rule 4 x 939 [2,7]
+ CRUSH rule 4 x 940 [8,7]
+ CRUSH rule 4 x 941 [5,2]
+ CRUSH rule 4 x 942 [2]
+ CRUSH rule 4 x 943 [8,2]
+ CRUSH rule 4 x 944 [5]
+ CRUSH rule 4 x 945 [7,2]
+ CRUSH rule 4 x 946 [2]
+ CRUSH rule 4 x 947 [5]
+ CRUSH rule 4 x 948 [7,8]
+ CRUSH rule 4 x 949 [6,2]
+ CRUSH rule 4 x 950 [5]
+ CRUSH rule 4 x 951 [5]
+ CRUSH rule 4 x 952 [2]
+ CRUSH rule 4 x 953 [2,5]
+ CRUSH rule 4 x 954 [5,2]
+ CRUSH rule 4 x 955 [8,6]
+ CRUSH rule 4 x 956 [2]
+ CRUSH rule 4 x 957 [7,6]
+ CRUSH rule 4 x 958 [8,7]
+ CRUSH rule 4 x 959 [5,2]
+ CRUSH rule 4 x 960 [5,6]
+ CRUSH rule 4 x 961 [5,2]
+ CRUSH rule 4 x 962 [7,5]
+ CRUSH rule 4 x 963 [2,5]
+ CRUSH rule 4 x 964 [5,2]
+ CRUSH rule 4 x 965 [7,6]
+ CRUSH rule 4 x 966 [5,8]
+ CRUSH rule 4 x 967 [8,6]
+ CRUSH rule 4 x 968 [7,2]
+ CRUSH rule 4 x 969 [8,2]
+ CRUSH rule 4 x 970 [2,6]
+ CRUSH rule 4 x 971 [2,7]
+ CRUSH rule 4 x 972 [2,8]
+ CRUSH rule 4 x 973 [2]
+ CRUSH rule 4 x 974 [5]
+ CRUSH rule 4 x 975 [5,7]
+ CRUSH rule 4 x 976 [5]
+ CRUSH rule 4 x 977 [8,5]
+ CRUSH rule 4 x 978 [7,2]
+ CRUSH rule 4 x 979 [7,6]
+ CRUSH rule 4 x 980 [6,2]
+ CRUSH rule 4 x 981 [7,5]
+ CRUSH rule 4 x 982 [5,2]
+ CRUSH rule 4 x 983 [5]
+ CRUSH rule 4 x 984 [2]
+ CRUSH rule 4 x 985 [2,5]
+ CRUSH rule 4 x 986 [8,7]
+ CRUSH rule 4 x 987 [2,5]
+ CRUSH rule 4 x 988 [2,5]
+ CRUSH rule 4 x 989 [2,6]
+ CRUSH rule 4 x 990 [2]
+ CRUSH rule 4 x 991 [2,5]
+ CRUSH rule 4 x 992 [7,2]
+ CRUSH rule 4 x 993 [2,6]
+ CRUSH rule 4 x 994 [5]
+ CRUSH rule 4 x 995 [7,6]
+ CRUSH rule 4 x 996 [6,7]
+ CRUSH rule 4 x 997 [6,5]
+ CRUSH rule 4 x 998 [8,2]
+ CRUSH rule 4 x 999 [2,7]
+ CRUSH rule 4 x 1000 [8,5]
+ CRUSH rule 4 x 1001 [2]
+ CRUSH rule 4 x 1002 [2,5]
+ CRUSH rule 4 x 1003 [2,8]
+ CRUSH rule 4 x 1004 [6,2]
+ CRUSH rule 4 x 1005 [6,2]
+ CRUSH rule 4 x 1006 [2]
+ CRUSH rule 4 x 1007 [2]
+ CRUSH rule 4 x 1008 [2,7]
+ CRUSH rule 4 x 1009 [6,8]
+ CRUSH rule 4 x 1010 [5]
+ CRUSH rule 4 x 1011 [5,2]
+ CRUSH rule 4 x 1012 [5,2]
+ CRUSH rule 4 x 1013 [5,2]
+ CRUSH rule 4 x 1014 [2,8]
+ CRUSH rule 4 x 1015 [6,8]
+ CRUSH rule 4 x 1016 [2]
+ CRUSH rule 4 x 1017 [6,2]
+ CRUSH rule 4 x 1018 [5]
+ CRUSH rule 4 x 1019 [5]
+ CRUSH rule 4 x 1020 [5,2]
+ CRUSH rule 4 x 1021 [5,2]
+ CRUSH rule 4 x 1022 [2,6]
+ CRUSH rule 4 x 1023 [5,2]
+ rule 4 (choose-set-two) num_rep 2 result size == 1:\t230/1024 (esc)
+ rule 4 (choose-set-two) num_rep 2 result size == 2:\t794/1024 (esc)
+ CRUSH rule 4 x 0 [2,5]
+ CRUSH rule 4 x 1 [2,8]
+ CRUSH rule 4 x 2 [2,5]
+ CRUSH rule 4 x 3 [8,2,6]
+ CRUSH rule 4 x 4 [5,2]
+ CRUSH rule 4 x 5 [7,8,2]
+ CRUSH rule 4 x 6 [2,6,7]
+ CRUSH rule 4 x 7 [5]
+ CRUSH rule 4 x 8 [5,8]
+ CRUSH rule 4 x 9 [2,5]
+ CRUSH rule 4 x 10 [2,8]
+ CRUSH rule 4 x 11 [2,7]
+ CRUSH rule 4 x 12 [2,5]
+ CRUSH rule 4 x 13 [5,8]
+ CRUSH rule 4 x 14 [7,6,8]
+ CRUSH rule 4 x 15 [7,2,5]
+ CRUSH rule 4 x 16 [5,6,2]
+ CRUSH rule 4 x 17 [5,2]
+ CRUSH rule 4 x 18 [2,5]
+ CRUSH rule 4 x 19 [7,5]
+ CRUSH rule 4 x 20 [2,5,7]
+ CRUSH rule 4 x 21 [5,7,2]
+ CRUSH rule 4 x 22 [8,5,7]
+ CRUSH rule 4 x 23 [5,6,8]
+ CRUSH rule 4 x 24 [2,6]
+ CRUSH rule 4 x 25 [5,7,2]
+ CRUSH rule 4 x 26 [2,8,7]
+ CRUSH rule 4 x 27 [5,2]
+ CRUSH rule 4 x 28 [6,2,5]
+ CRUSH rule 4 x 29 [8,5,7]
+ CRUSH rule 4 x 30 [5,7]
+ CRUSH rule 4 x 31 [8,7,6]
+ CRUSH rule 4 x 32 [5,6,8]
+ CRUSH rule 4 x 33 [2,7,8]
+ CRUSH rule 4 x 34 [2,5]
+ CRUSH rule 4 x 35 [2,8]
+ CRUSH rule 4 x 36 [5,8,2]
+ CRUSH rule 4 x 37 [2,5]
+ CRUSH rule 4 x 38 [5,8,7]
+ CRUSH rule 4 x 39 [5,7,2]
+ CRUSH rule 4 x 40 [7,8,2]
+ CRUSH rule 4 x 41 [2,6]
+ CRUSH rule 4 x 42 [5,6]
+ CRUSH rule 4 x 43 [2,5,7]
+ CRUSH rule 4 x 44 [2,6,7]
+ CRUSH rule 4 x 45 [8,2,5]
+ CRUSH rule 4 x 46 [2,5]
+ CRUSH rule 4 x 47 [5]
+ CRUSH rule 4 x 48 [5,6]
+ CRUSH rule 4 x 49 [5,8]
+ CRUSH rule 4 x 50 [5,2]
+ CRUSH rule 4 x 51 [5,6,2]
+ CRUSH rule 4 x 52 [8,6,2]
+ CRUSH rule 4 x 53 [5]
+ CRUSH rule 4 x 54 [7,6,8]
+ CRUSH rule 4 x 55 [8,7,2]
+ CRUSH rule 4 x 56 [6,5,8]
+ CRUSH rule 4 x 57 [5,6]
+ CRUSH rule 4 x 58 [2]
+ CRUSH rule 4 x 59 [5,2]
+ CRUSH rule 4 x 60 [5,2]
+ CRUSH rule 4 x 61 [5,6,8]
+ CRUSH rule 4 x 62 [7,2]
+ CRUSH rule 4 x 63 [5,6,8]
+ CRUSH rule 4 x 64 [5,2]
+ CRUSH rule 4 x 65 [7,5,6]
+ CRUSH rule 4 x 66 [5]
+ CRUSH rule 4 x 67 [5,2,8]
+ CRUSH rule 4 x 68 [2,5,8]
+ CRUSH rule 4 x 69 [5,2,6]
+ CRUSH rule 4 x 70 [7,2]
+ CRUSH rule 4 x 71 [2,8,6]
+ CRUSH rule 4 x 72 [6,2,5]
+ CRUSH rule 4 x 73 [2,7,5]
+ CRUSH rule 4 x 74 [2,7]
+ CRUSH rule 4 x 75 [5,2]
+ CRUSH rule 4 x 76 [5,2,7]
+ CRUSH rule 4 x 77 [7,2,6]
+ CRUSH rule 4 x 78 [2,5]
+ CRUSH rule 4 x 79 [5,2]
+ CRUSH rule 4 x 80 [2,5]
+ CRUSH rule 4 x 81 [2,5]
+ CRUSH rule 4 x 82 [7,2,6]
+ CRUSH rule 4 x 83 [2,6,8]
+ CRUSH rule 4 x 84 [7,2]
+ CRUSH rule 4 x 85 [5,8]
+ CRUSH rule 4 x 86 [2]
+ CRUSH rule 4 x 87 [2,7,5]
+ CRUSH rule 4 x 88 [2,6]
+ CRUSH rule 4 x 89 [5,2]
+ CRUSH rule 4 x 90 [6,7,5]
+ CRUSH rule 4 x 91 [5,8,7]
+ CRUSH rule 4 x 92 [2,8,5]
+ CRUSH rule 4 x 93 [7,5]
+ CRUSH rule 4 x 94 [2,5]
+ CRUSH rule 4 x 95 [7,5,2]
+ CRUSH rule 4 x 96 [5,6]
+ CRUSH rule 4 x 97 [8,7,5]
+ CRUSH rule 4 x 98 [2,7]
+ CRUSH rule 4 x 99 [2,7]
+ CRUSH rule 4 x 100 [2,7,5]
+ CRUSH rule 4 x 101 [5,7,2]
+ CRUSH rule 4 x 102 [5,2,7]
+ CRUSH rule 4 x 103 [5,7,6]
+ CRUSH rule 4 x 104 [7,5]
+ CRUSH rule 4 x 105 [2,5]
+ CRUSH rule 4 x 106 [2,6]
+ CRUSH rule 4 x 107 [5,2]
+ CRUSH rule 4 x 108 [7,2,8]
+ CRUSH rule 4 x 109 [2,5]
+ CRUSH rule 4 x 110 [5,2,7]
+ CRUSH rule 4 x 111 [2]
+ CRUSH rule 4 x 112 [2,6]
+ CRUSH rule 4 x 113 [6,2]
+ CRUSH rule 4 x 114 [7,6,5]
+ CRUSH rule 4 x 115 [8,2,5]
+ CRUSH rule 4 x 116 [2,6]
+ CRUSH rule 4 x 117 [7,5,6]
+ CRUSH rule 4 x 118 [2,5,8]
+ CRUSH rule 4 x 119 [5,6,2]
+ CRUSH rule 4 x 120 [2,5]
+ CRUSH rule 4 x 121 [2]
+ CRUSH rule 4 x 122 [8,5,2]
+ CRUSH rule 4 x 123 [2,5]
+ CRUSH rule 4 x 124 [5,2]
+ CRUSH rule 4 x 125 [2,7,5]
+ CRUSH rule 4 x 126 [5,2,6]
+ CRUSH rule 4 x 127 [5,6,7]
+ CRUSH rule 4 x 128 [5,8]
+ CRUSH rule 4 x 129 [2]
+ CRUSH rule 4 x 130 [5,8]
+ CRUSH rule 4 x 131 [2]
+ CRUSH rule 4 x 132 [2,5]
+ CRUSH rule 4 x 133 [5,6,7]
+ CRUSH rule 4 x 134 [2,8]
+ CRUSH rule 4 x 135 [5,6]
+ CRUSH rule 4 x 136 [2,5]
+ CRUSH rule 4 x 137 [7,5]
+ CRUSH rule 4 x 138 [8,7,6]
+ CRUSH rule 4 x 139 [5,2]
+ CRUSH rule 4 x 140 [2,6,7]
+ CRUSH rule 4 x 141 [6,8,2]
+ CRUSH rule 4 x 142 [5,2]
+ CRUSH rule 4 x 143 [5,8,7]
+ CRUSH rule 4 x 144 [8,2]
+ CRUSH rule 4 x 145 [8,5,6]
+ CRUSH rule 4 x 146 [2,6,5]
+ CRUSH rule 4 x 147 [2,8,7]
+ CRUSH rule 4 x 148 [5,2]
+ CRUSH rule 4 x 149 [5,8,6]
+ CRUSH rule 4 x 150 [2,6,5]
+ CRUSH rule 4 x 151 [5]
+ CRUSH rule 4 x 152 [8,5,6]
+ CRUSH rule 4 x 153 [8,6,7]
+ CRUSH rule 4 x 154 [5,2]
+ CRUSH rule 4 x 155 [5,8]
+ CRUSH rule 4 x 156 [5]
+ CRUSH rule 4 x 157 [5,2]
+ CRUSH rule 4 x 158 [2,8]
+ CRUSH rule 4 x 159 [7,2,5]
+ CRUSH rule 4 x 160 [2,8]
+ CRUSH rule 4 x 161 [2,5]
+ CRUSH rule 4 x 162 [2,6]
+ CRUSH rule 4 x 163 [5,6,2]
+ CRUSH rule 4 x 164 [7,8,2]
+ CRUSH rule 4 x 165 [7,2,5]
+ CRUSH rule 4 x 166 [2,5]
+ CRUSH rule 4 x 167 [2,6]
+ CRUSH rule 4 x 168 [5,2]
+ CRUSH rule 4 x 169 [2,6]
+ CRUSH rule 4 x 170 [2,5]
+ CRUSH rule 4 x 171 [7,5]
+ CRUSH rule 4 x 172 [2,7,8]
+ CRUSH rule 4 x 173 [8,5]
+ CRUSH rule 4 x 174 [2,5,7]
+ CRUSH rule 4 x 175 [6,2,5]
+ CRUSH rule 4 x 176 [5]
+ CRUSH rule 4 x 177 [5,2]
+ CRUSH rule 4 x 178 [5,2]
+ CRUSH rule 4 x 179 [5,2]
+ CRUSH rule 4 x 180 [5,8,7]
+ CRUSH rule 4 x 181 [6,2]
+ CRUSH rule 4 x 182 [8,5]
+ CRUSH rule 4 x 183 [7,8,6]
+ CRUSH rule 4 x 184 [5,7]
+ CRUSH rule 4 x 185 [6,8,2]
+ CRUSH rule 4 x 186 [2,5]
+ CRUSH rule 4 x 187 [2,6,7]
+ CRUSH rule 4 x 188 [2,8,7]
+ CRUSH rule 4 x 189 [2,7,8]
+ CRUSH rule 4 x 190 [5,2]
+ CRUSH rule 4 x 191 [7,6,8]
+ CRUSH rule 4 x 192 [5,2]
+ CRUSH rule 4 x 193 [5,2,6]
+ CRUSH rule 4 x 194 [2,5]
+ CRUSH rule 4 x 195 [6,5]
+ CRUSH rule 4 x 196 [6,7,2]
+ CRUSH rule 4 x 197 [6,5,2]
+ CRUSH rule 4 x 198 [2,5,6]
+ CRUSH rule 4 x 199 [2,5,7]
+ CRUSH rule 4 x 200 [2,5]
+ CRUSH rule 4 x 201 [7,2]
+ CRUSH rule 4 x 202 [6,5]
+ CRUSH rule 4 x 203 [5,8,7]
+ CRUSH rule 4 x 204 [2,5]
+ CRUSH rule 4 x 205 [2,7,6]
+ CRUSH rule 4 x 206 [2,7]
+ CRUSH rule 4 x 207 [5,2]
+ CRUSH rule 4 x 208 [7,2]
+ CRUSH rule 4 x 209 [2]
+ CRUSH rule 4 x 210 [2,5]
+ CRUSH rule 4 x 211 [5,2]
+ CRUSH rule 4 x 212 [7,5,2]
+ CRUSH rule 4 x 213 [8,5,2]
+ CRUSH rule 4 x 214 [5,7]
+ CRUSH rule 4 x 215 [8,2,7]
+ CRUSH rule 4 x 216 [5,2]
+ CRUSH rule 4 x 217 [2,7]
+ CRUSH rule 4 x 218 [2,7,8]
+ CRUSH rule 4 x 219 [5,8,2]
+ CRUSH rule 4 x 220 [5,7]
+ CRUSH rule 4 x 221 [5,6]
+ CRUSH rule 4 x 222 [6,8,5]
+ CRUSH rule 4 x 223 [2,5,6]
+ CRUSH rule 4 x 224 [2,5]
+ CRUSH rule 4 x 225 [8,6,2]
+ CRUSH rule 4 x 226 [7,2,5]
+ CRUSH rule 4 x 227 [5,2]
+ CRUSH rule 4 x 228 [5,8]
+ CRUSH rule 4 x 229 [5,7]
+ CRUSH rule 4 x 230 [5,7,2]
+ CRUSH rule 4 x 231 [5]
+ CRUSH rule 4 x 232 [2,7,5]
+ CRUSH rule 4 x 233 [5]
+ CRUSH rule 4 x 234 [2]
+ CRUSH rule 4 x 235 [5,8,2]
+ CRUSH rule 4 x 236 [5,2,7]
+ CRUSH rule 4 x 237 [5,7,6]
+ CRUSH rule 4 x 238 [5]
+ CRUSH rule 4 x 239 [8,7,6]
+ CRUSH rule 4 x 240 [5,7]
+ CRUSH rule 4 x 241 [5,2]
+ CRUSH rule 4 x 242 [5]
+ CRUSH rule 4 x 243 [5,7]
+ CRUSH rule 4 x 244 [5,6,2]
+ CRUSH rule 4 x 245 [7,6,2]
+ CRUSH rule 4 x 246 [2,5]
+ CRUSH rule 4 x 247 [6,2]
+ CRUSH rule 4 x 248 [8,2,7]
+ CRUSH rule 4 x 249 [2]
+ CRUSH rule 4 x 250 [2]
+ CRUSH rule 4 x 251 [2,5,6]
+ CRUSH rule 4 x 252 [5,7,8]
+ CRUSH rule 4 x 253 [5,2]
+ CRUSH rule 4 x 254 [5]
+ CRUSH rule 4 x 255 [2,7,8]
+ CRUSH rule 4 x 256 [5,7,2]
+ CRUSH rule 4 x 257 [2,8,5]
+ CRUSH rule 4 x 258 [5,2]
+ CRUSH rule 4 x 259 [5,6]
+ CRUSH rule 4 x 260 [5,6,2]
+ CRUSH rule 4 x 261 [8,7,5]
+ CRUSH rule 4 x 262 [5,8]
+ CRUSH rule 4 x 263 [6,8,2]
+ CRUSH rule 4 x 264 [5,6]
+ CRUSH rule 4 x 265 [8,6,7]
+ CRUSH rule 4 x 266 [8,2]
+ CRUSH rule 4 x 267 [2,5]
+ CRUSH rule 4 x 268 [2,7]
+ CRUSH rule 4 x 269 [2,8]
+ CRUSH rule 4 x 270 [5,2]
+ CRUSH rule 4 x 271 [7,5]
+ CRUSH rule 4 x 272 [2,8,5]
+ CRUSH rule 4 x 273 [5,2]
+ CRUSH rule 4 x 274 [6,8,5]
+ CRUSH rule 4 x 275 [5,6]
+ CRUSH rule 4 x 276 [7,2]
+ CRUSH rule 4 x 277 [6,5,2]
+ CRUSH rule 4 x 278 [6,8,2]
+ CRUSH rule 4 x 279 [8,5,6]
+ CRUSH rule 4 x 280 [2,6]
+ CRUSH rule 4 x 281 [8,2]
+ CRUSH rule 4 x 282 [5,2,6]
+ CRUSH rule 4 x 283 [8,2]
+ CRUSH rule 4 x 284 [6,5,2]
+ CRUSH rule 4 x 285 [5,7]
+ CRUSH rule 4 x 286 [2,6]
+ CRUSH rule 4 x 287 [2,5]
+ CRUSH rule 4 x 288 [8,2,7]
+ CRUSH rule 4 x 289 [5,6,2]
+ CRUSH rule 4 x 290 [2,5,7]
+ CRUSH rule 4 x 291 [2,5]
+ CRUSH rule 4 x 292 [8,2]
+ CRUSH rule 4 x 293 [6,2,8]
+ CRUSH rule 4 x 294 [7,5]
+ CRUSH rule 4 x 295 [5,8,7]
+ CRUSH rule 4 x 296 [5,2]
+ CRUSH rule 4 x 297 [6,2,8]
+ CRUSH rule 4 x 298 [2,5]
+ CRUSH rule 4 x 299 [2,7]
+ CRUSH rule 4 x 300 [8,7,5]
+ CRUSH rule 4 x 301 [2,8]
+ CRUSH rule 4 x 302 [5,2,6]
+ CRUSH rule 4 x 303 [7,5,8]
+ CRUSH rule 4 x 304 [2,7,5]
+ CRUSH rule 4 x 305 [5,8,2]
+ CRUSH rule 4 x 306 [2,7]
+ CRUSH rule 4 x 307 [2,8]
+ CRUSH rule 4 x 308 [2,8,5]
+ CRUSH rule 4 x 309 [7,5]
+ CRUSH rule 4 x 310 [5,2]
+ CRUSH rule 4 x 311 [5]
+ CRUSH rule 4 x 312 [2]
+ CRUSH rule 4 x 313 [5]
+ CRUSH rule 4 x 314 [5,2]
+ CRUSH rule 4 x 315 [2]
+ CRUSH rule 4 x 316 [6,5,8]
+ CRUSH rule 4 x 317 [2,6]
+ CRUSH rule 4 x 318 [8,6,7]
+ CRUSH rule 4 x 319 [5,2,8]
+ CRUSH rule 4 x 320 [5,7,2]
+ CRUSH rule 4 x 321 [2,5]
+ CRUSH rule 4 x 322 [2,7,5]
+ CRUSH rule 4 x 323 [5,7,2]
+ CRUSH rule 4 x 324 [7,2,8]
+ CRUSH rule 4 x 325 [5,6,2]
+ CRUSH rule 4 x 326 [5,2]
+ CRUSH rule 4 x 327 [2,6,7]
+ CRUSH rule 4 x 328 [7,5,8]
+ CRUSH rule 4 x 329 [5,6]
+ CRUSH rule 4 x 330 [5,7]
+ CRUSH rule 4 x 331 [2,6,5]
+ CRUSH rule 4 x 332 [2,5]
+ CRUSH rule 4 x 333 [6,8,5]
+ CRUSH rule 4 x 334 [8,5]
+ CRUSH rule 4 x 335 [7,2]
+ CRUSH rule 4 x 336 [5,6,2]
+ CRUSH rule 4 x 337 [7,2,6]
+ CRUSH rule 4 x 338 [5,6]
+ CRUSH rule 4 x 339 [7,5,2]
+ CRUSH rule 4 x 340 [2]
+ CRUSH rule 4 x 341 [5,2,7]
+ CRUSH rule 4 x 342 [2,7,5]
+ CRUSH rule 4 x 343 [6,7,5]
+ CRUSH rule 4 x 344 [6,2,5]
+ CRUSH rule 4 x 345 [5]
+ CRUSH rule 4 x 346 [8,2,5]
+ CRUSH rule 4 x 347 [5,2]
+ CRUSH rule 4 x 348 [8,2]
+ CRUSH rule 4 x 349 [2,6,7]
+ CRUSH rule 4 x 350 [8,5,7]
+ CRUSH rule 4 x 351 [5,6]
+ CRUSH rule 4 x 352 [2]
+ CRUSH rule 4 x 353 [6,5]
+ CRUSH rule 4 x 354 [2,5]
+ CRUSH rule 4 x 355 [5,6]
+ CRUSH rule 4 x 356 [5]
+ CRUSH rule 4 x 357 [6,2]
+ CRUSH rule 4 x 358 [2,8]
+ CRUSH rule 4 x 359 [6,7,8]
+ CRUSH rule 4 x 360 [5,2]
+ CRUSH rule 4 x 361 [8,5]
+ CRUSH rule 4 x 362 [5]
+ CRUSH rule 4 x 363 [5,2]
+ CRUSH rule 4 x 364 [2,5]
+ CRUSH rule 4 x 365 [6,7,8]
+ CRUSH rule 4 x 366 [7,2,8]
+ CRUSH rule 4 x 367 [5,2]
+ CRUSH rule 4 x 368 [7,5,6]
+ CRUSH rule 4 x 369 [5,7,2]
+ CRUSH rule 4 x 370 [8,7,6]
+ CRUSH rule 4 x 371 [2,5]
+ CRUSH rule 4 x 372 [5,2,8]
+ CRUSH rule 4 x 373 [2,6,8]
+ CRUSH rule 4 x 374 [5,8]
+ CRUSH rule 4 x 375 [6,5]
+ CRUSH rule 4 x 376 [7,2]
+ CRUSH rule 4 x 377 [2,5]
+ CRUSH rule 4 x 378 [2]
+ CRUSH rule 4 x 379 [8,5]
+ CRUSH rule 4 x 380 [2,5]
+ CRUSH rule 4 x 381 [2,5,7]
+ CRUSH rule 4 x 382 [2,5]
+ CRUSH rule 4 x 383 [5]
+ CRUSH rule 4 x 384 [7,2,6]
+ CRUSH rule 4 x 385 [7,5,6]
+ CRUSH rule 4 x 386 [2,5]
+ CRUSH rule 4 x 387 [2,5,6]
+ CRUSH rule 4 x 388 [5,2]
+ CRUSH rule 4 x 389 [2,5]
+ CRUSH rule 4 x 390 [5,6,2]
+ CRUSH rule 4 x 391 [5,6,2]
+ CRUSH rule 4 x 392 [2,8,6]
+ CRUSH rule 4 x 393 [5,2,6]
+ CRUSH rule 4 x 394 [5,7,6]
+ CRUSH rule 4 x 395 [5,2]
+ CRUSH rule 4 x 396 [5,2]
+ CRUSH rule 4 x 397 [2,5]
+ CRUSH rule 4 x 398 [2,5]
+ CRUSH rule 4 x 399 [8,7,5]
+ CRUSH rule 4 x 400 [8,2,5]
+ CRUSH rule 4 x 401 [2]
+ CRUSH rule 4 x 402 [7,8,6]
+ CRUSH rule 4 x 403 [2,5]
+ CRUSH rule 4 x 404 [5,2]
+ CRUSH rule 4 x 405 [6,5,2]
+ CRUSH rule 4 x 406 [2]
+ CRUSH rule 4 x 407 [2,8,7]
+ CRUSH rule 4 x 408 [5,2,6]
+ CRUSH rule 4 x 409 [7,5,6]
+ CRUSH rule 4 x 410 [8,6,5]
+ CRUSH rule 4 x 411 [2,6]
+ CRUSH rule 4 x 412 [2,5,8]
+ CRUSH rule 4 x 413 [5,2,8]
+ CRUSH rule 4 x 414 [5,2]
+ CRUSH rule 4 x 415 [2,6]
+ CRUSH rule 4 x 416 [2,7]
+ CRUSH rule 4 x 417 [8,7,2]
+ CRUSH rule 4 x 418 [7,6,8]
+ CRUSH rule 4 x 419 [8,5,2]
+ CRUSH rule 4 x 420 [2,5]
+ CRUSH rule 4 x 421 [8,6,7]
+ CRUSH rule 4 x 422 [6,7,8]
+ CRUSH rule 4 x 423 [2,5,6]
+ CRUSH rule 4 x 424 [8,5,7]
+ CRUSH rule 4 x 425 [2,5]
+ CRUSH rule 4 x 426 [6,7,2]
+ CRUSH rule 4 x 427 [2,7,5]
+ CRUSH rule 4 x 428 [5]
+ CRUSH rule 4 x 429 [5,6,7]
+ CRUSH rule 4 x 430 [5,6]
+ CRUSH rule 4 x 431 [5,2]
+ CRUSH rule 4 x 432 [7,2]
+ CRUSH rule 4 x 433 [6,5,2]
+ CRUSH rule 4 x 434 [5,2]
+ CRUSH rule 4 x 435 [2,5]
+ CRUSH rule 4 x 436 [5,2]
+ CRUSH rule 4 x 437 [7,5]
+ CRUSH rule 4 x 438 [2,5,8]
+ CRUSH rule 4 x 439 [2,5]
+ CRUSH rule 4 x 440 [2,7]
+ CRUSH rule 4 x 441 [5,7,2]
+ CRUSH rule 4 x 442 [2,5]
+ CRUSH rule 4 x 443 [6,8,2]
+ CRUSH rule 4 x 444 [7,2]
+ CRUSH rule 4 x 445 [6,5]
+ CRUSH rule 4 x 446 [5]
+ CRUSH rule 4 x 447 [2,5]
+ CRUSH rule 4 x 448 [7,2,5]
+ CRUSH rule 4 x 449 [7,8,5]
+ CRUSH rule 4 x 450 [5,2]
+ CRUSH rule 4 x 451 [6,8,5]
+ CRUSH rule 4 x 452 [8,5]
+ CRUSH rule 4 x 453 [6,8,7]
+ CRUSH rule 4 x 454 [6,7,5]
+ CRUSH rule 4 x 455 [2,7,5]
+ CRUSH rule 4 x 456 [6,8,7]
+ CRUSH rule 4 x 457 [7,2,8]
+ CRUSH rule 4 x 458 [2,8]
+ CRUSH rule 4 x 459 [2,6]
+ CRUSH rule 4 x 460 [6,5,7]
+ CRUSH rule 4 x 461 [6,5]
+ CRUSH rule 4 x 462 [8,2,5]
+ CRUSH rule 4 x 463 [6,7,2]
+ CRUSH rule 4 x 464 [7,5,2]
+ CRUSH rule 4 x 465 [7,6,2]
+ CRUSH rule 4 x 466 [5,8]
+ CRUSH rule 4 x 467 [6,5,7]
+ CRUSH rule 4 x 468 [7,8,2]
+ CRUSH rule 4 x 469 [7,2]
+ CRUSH rule 4 x 470 [5,2]
+ CRUSH rule 4 x 471 [2]
+ CRUSH rule 4 x 472 [5,2]
+ CRUSH rule 4 x 473 [2,5]
+ CRUSH rule 4 x 474 [6,2,7]
+ CRUSH rule 4 x 475 [6,7,8]
+ CRUSH rule 4 x 476 [5]
+ CRUSH rule 4 x 477 [5,8,6]
+ CRUSH rule 4 x 478 [6,7,2]
+ CRUSH rule 4 x 479 [2,5,8]
+ CRUSH rule 4 x 480 [2,8,6]
+ CRUSH rule 4 x 481 [2,5]
+ CRUSH rule 4 x 482 [5]
+ CRUSH rule 4 x 483 [2,6]
+ CRUSH rule 4 x 484 [2,7]
+ CRUSH rule 4 x 485 [5,7]
+ CRUSH rule 4 x 486 [5,2,7]
+ CRUSH rule 4 x 487 [5,2]
+ CRUSH rule 4 x 488 [5,7,2]
+ CRUSH rule 4 x 489 [2,8,5]
+ CRUSH rule 4 x 490 [6,5,2]
+ CRUSH rule 4 x 491 [2,6]
+ CRUSH rule 4 x 492 [6,5]
+ CRUSH rule 4 x 493 [2,7]
+ CRUSH rule 4 x 494 [2,8]
+ CRUSH rule 4 x 495 [5]
+ CRUSH rule 4 x 496 [7,5]
+ CRUSH rule 4 x 497 [5,7]
+ CRUSH rule 4 x 498 [2,5,8]
+ CRUSH rule 4 x 499 [8,5]
+ CRUSH rule 4 x 500 [5,6]
+ CRUSH rule 4 x 501 [2,7]
+ CRUSH rule 4 x 502 [7,2,5]
+ CRUSH rule 4 x 503 [2,5]
+ CRUSH rule 4 x 504 [5,6]
+ CRUSH rule 4 x 505 [2,7,6]
+ CRUSH rule 4 x 506 [5,2]
+ CRUSH rule 4 x 507 [6,2,8]
+ CRUSH rule 4 x 508 [2]
+ CRUSH rule 4 x 509 [7,5,8]
+ CRUSH rule 4 x 510 [6,2]
+ CRUSH rule 4 x 511 [5,8,7]
+ CRUSH rule 4 x 512 [7,6,2]
+ CRUSH rule 4 x 513 [7,2]
+ CRUSH rule 4 x 514 [5]
+ CRUSH rule 4 x 515 [8,5,6]
+ CRUSH rule 4 x 516 [5,2]
+ CRUSH rule 4 x 517 [7,8,6]
+ CRUSH rule 4 x 518 [5,6,7]
+ CRUSH rule 4 x 519 [7,5]
+ CRUSH rule 4 x 520 [2,6,5]
+ CRUSH rule 4 x 521 [8,7,6]
+ CRUSH rule 4 x 522 [6,8,2]
+ CRUSH rule 4 x 523 [5,2]
+ CRUSH rule 4 x 524 [2,5]
+ CRUSH rule 4 x 525 [2,5]
+ CRUSH rule 4 x 526 [2,5,8]
+ CRUSH rule 4 x 527 [2,5]
+ CRUSH rule 4 x 528 [5,2]
+ CRUSH rule 4 x 529 [5,7,2]
+ CRUSH rule 4 x 530 [6,7,8]
+ CRUSH rule 4 x 531 [6,2]
+ CRUSH rule 4 x 532 [6,5,7]
+ CRUSH rule 4 x 533 [5,6]
+ CRUSH rule 4 x 534 [7,5]
+ CRUSH rule 4 x 535 [8,6,2]
+ CRUSH rule 4 x 536 [6,7,2]
+ CRUSH rule 4 x 537 [5,7]
+ CRUSH rule 4 x 538 [6,8,5]
+ CRUSH rule 4 x 539 [8,5,7]
+ CRUSH rule 4 x 540 [2,6]
+ CRUSH rule 4 x 541 [2,5]
+ CRUSH rule 4 x 542 [5]
+ CRUSH rule 4 x 543 [6,2,8]
+ CRUSH rule 4 x 544 [5,7,6]
+ CRUSH rule 4 x 545 [5,7,2]
+ CRUSH rule 4 x 546 [6,2,7]
+ CRUSH rule 4 x 547 [8,2]
+ CRUSH rule 4 x 548 [5,2]
+ CRUSH rule 4 x 549 [5,8,2]
+ CRUSH rule 4 x 550 [2,5]
+ CRUSH rule 4 x 551 [7,5]
+ CRUSH rule 4 x 552 [5]
+ CRUSH rule 4 x 553 [5,2]
+ CRUSH rule 4 x 554 [2,8,5]
+ CRUSH rule 4 x 555 [5,2,8]
+ CRUSH rule 4 x 556 [5]
+ CRUSH rule 4 x 557 [7,5,6]
+ CRUSH rule 4 x 558 [5,2]
+ CRUSH rule 4 x 559 [5,2,6]
+ CRUSH rule 4 x 560 [8,5]
+ CRUSH rule 4 x 561 [6,5,7]
+ CRUSH rule 4 x 562 [5,2]
+ CRUSH rule 4 x 563 [2,6]
+ CRUSH rule 4 x 564 [5,2,7]
+ CRUSH rule 4 x 565 [5,6]
+ CRUSH rule 4 x 566 [5,7,2]
+ CRUSH rule 4 x 567 [5,6,2]
+ CRUSH rule 4 x 568 [7,5,2]
+ CRUSH rule 4 x 569 [5,2]
+ CRUSH rule 4 x 570 [2,5]
+ CRUSH rule 4 x 571 [5,7]
+ CRUSH rule 4 x 572 [5,2]
+ CRUSH rule 4 x 573 [5,2,7]
+ CRUSH rule 4 x 574 [2]
+ CRUSH rule 4 x 575 [8,6,2]
+ CRUSH rule 4 x 576 [5,6,8]
+ CRUSH rule 4 x 577 [8,2,6]
+ CRUSH rule 4 x 578 [6,8,7]
+ CRUSH rule 4 x 579 [5,2,6]
+ CRUSH rule 4 x 580 [5,2,7]
+ CRUSH rule 4 x 581 [7,2,6]
+ CRUSH rule 4 x 582 [2,8,7]
+ CRUSH rule 4 x 583 [6,2,8]
+ CRUSH rule 4 x 584 [8,2]
+ CRUSH rule 4 x 585 [7,2,5]
+ CRUSH rule 4 x 586 [2,7]
+ CRUSH rule 4 x 587 [2,5]
+ CRUSH rule 4 x 588 [5,7]
+ CRUSH rule 4 x 589 [7,2,5]
+ CRUSH rule 4 x 590 [6,2,5]
+ CRUSH rule 4 x 591 [5,2]
+ CRUSH rule 4 x 592 [2,5]
+ CRUSH rule 4 x 593 [2,8]
+ CRUSH rule 4 x 594 [2,7,5]
+ CRUSH rule 4 x 595 [7,2,8]
+ CRUSH rule 4 x 596 [5]
+ CRUSH rule 4 x 597 [5,2]
+ CRUSH rule 4 x 598 [5,2]
+ CRUSH rule 4 x 599 [5,2]
+ CRUSH rule 4 x 600 [7,2,8]
+ CRUSH rule 4 x 601 [2,7,8]
+ CRUSH rule 4 x 602 [5,7]
+ CRUSH rule 4 x 603 [5,2,6]
+ CRUSH rule 4 x 604 [7,5]
+ CRUSH rule 4 x 605 [5,2]
+ CRUSH rule 4 x 606 [2]
+ CRUSH rule 4 x 607 [2,5,8]
+ CRUSH rule 4 x 608 [5,2]
+ CRUSH rule 4 x 609 [5,2]
+ CRUSH rule 4 x 610 [5,7,6]
+ CRUSH rule 4 x 611 [2]
+ CRUSH rule 4 x 612 [2,8]
+ CRUSH rule 4 x 613 [7,2]
+ CRUSH rule 4 x 614 [7,8,6]
+ CRUSH rule 4 x 615 [6,8,2]
+ CRUSH rule 4 x 616 [2,8]
+ CRUSH rule 4 x 617 [6,2]
+ CRUSH rule 4 x 618 [7,6,5]
+ CRUSH rule 4 x 619 [5,2,8]
+ CRUSH rule 4 x 620 [5,2]
+ CRUSH rule 4 x 621 [5,8]
+ CRUSH rule 4 x 622 [2,5]
+ CRUSH rule 4 x 623 [2,6]
+ CRUSH rule 4 x 624 [5,2]
+ CRUSH rule 4 x 625 [2,5,7]
+ CRUSH rule 4 x 626 [7,8,2]
+ CRUSH rule 4 x 627 [2,7]
+ CRUSH rule 4 x 628 [8,2,5]
+ CRUSH rule 4 x 629 [2,6,5]
+ CRUSH rule 4 x 630 [2,7]
+ CRUSH rule 4 x 631 [2,7,8]
+ CRUSH rule 4 x 632 [7,2]
+ CRUSH rule 4 x 633 [8,6,5]
+ CRUSH rule 4 x 634 [2,5]
+ CRUSH rule 4 x 635 [5,6]
+ CRUSH rule 4 x 636 [2,5]
+ CRUSH rule 4 x 637 [5,2]
+ CRUSH rule 4 x 638 [6,8,2]
+ CRUSH rule 4 x 639 [5,2]
+ CRUSH rule 4 x 640 [5,2]
+ CRUSH rule 4 x 641 [7,2]
+ CRUSH rule 4 x 642 [2]
+ CRUSH rule 4 x 643 [5,2]
+ CRUSH rule 4 x 644 [8,2,5]
+ CRUSH rule 4 x 645 [5,6]
+ CRUSH rule 4 x 646 [8,2,5]
+ CRUSH rule 4 x 647 [7,2,5]
+ CRUSH rule 4 x 648 [2,6,5]
+ CRUSH rule 4 x 649 [5,7]
+ CRUSH rule 4 x 650 [7,8,6]
+ CRUSH rule 4 x 651 [5,7,6]
+ CRUSH rule 4 x 652 [5,8]
+ CRUSH rule 4 x 653 [8,5,2]
+ CRUSH rule 4 x 654 [7,5,2]
+ CRUSH rule 4 x 655 [2,5]
+ CRUSH rule 4 x 656 [5,8]
+ CRUSH rule 4 x 657 [6,2]
+ CRUSH rule 4 x 658 [5,6]
+ CRUSH rule 4 x 659 [5,6,7]
+ CRUSH rule 4 x 660 [7,8,6]
+ CRUSH rule 4 x 661 [2,8,5]
+ CRUSH rule 4 x 662 [5]
+ CRUSH rule 4 x 663 [2,5,8]
+ CRUSH rule 4 x 664 [2,5]
+ CRUSH rule 4 x 665 [5,7,6]
+ CRUSH rule 4 x 666 [2,8,5]
+ CRUSH rule 4 x 667 [2,5]
+ CRUSH rule 4 x 668 [5,7,6]
+ CRUSH rule 4 x 669 [6,5]
+ CRUSH rule 4 x 670 [5,2]
+ CRUSH rule 4 x 671 [2]
+ CRUSH rule 4 x 672 [5,2]
+ CRUSH rule 4 x 673 [5,2]
+ CRUSH rule 4 x 674 [5,2,8]
+ CRUSH rule 4 x 675 [2,8,6]
+ CRUSH rule 4 x 676 [2,5]
+ CRUSH rule 4 x 677 [5,2]
+ CRUSH rule 4 x 678 [2,5]
+ CRUSH rule 4 x 679 [6,2]
+ CRUSH rule 4 x 680 [2,5,6]
+ CRUSH rule 4 x 681 [5,7]
+ CRUSH rule 4 x 682 [2,5]
+ CRUSH rule 4 x 683 [2,5]
+ CRUSH rule 4 x 684 [7,2,6]
+ CRUSH rule 4 x 685 [7,2,6]
+ CRUSH rule 4 x 686 [2,5]
+ CRUSH rule 4 x 687 [5,7]
+ CRUSH rule 4 x 688 [5,7,2]
+ CRUSH rule 4 x 689 [6,5,2]
+ CRUSH rule 4 x 690 [8,2,7]
+ CRUSH rule 4 x 691 [5,2]
+ CRUSH rule 4 x 692 [7,2,8]
+ CRUSH rule 4 x 693 [6,7,5]
+ CRUSH rule 4 x 694 [6,5,2]
+ CRUSH rule 4 x 695 [2,8,7]
+ CRUSH rule 4 x 696 [2,5]
+ CRUSH rule 4 x 697 [6,2]
+ CRUSH rule 4 x 698 [6,2]
+ CRUSH rule 4 x 699 [2,6,8]
+ CRUSH rule 4 x 700 [2,5]
+ CRUSH rule 4 x 701 [5,2]
+ CRUSH rule 4 x 702 [5,2]
+ CRUSH rule 4 x 703 [8,5]
+ CRUSH rule 4 x 704 [2,5,8]
+ CRUSH rule 4 x 705 [8,6,2]
+ CRUSH rule 4 x 706 [2,5]
+ CRUSH rule 4 x 707 [7,8,6]
+ CRUSH rule 4 x 708 [5,8]
+ CRUSH rule 4 x 709 [6,5,2]
+ CRUSH rule 4 x 710 [8,5]
+ CRUSH rule 4 x 711 [2,5,8]
+ CRUSH rule 4 x 712 [2,5,7]
+ CRUSH rule 4 x 713 [6,7,8]
+ CRUSH rule 4 x 714 [5,2]
+ CRUSH rule 4 x 715 [2,5]
+ CRUSH rule 4 x 716 [5,6,2]
+ CRUSH rule 4 x 717 [8,7,2]
+ CRUSH rule 4 x 718 [5,7,8]
+ CRUSH rule 4 x 719 [2,6,5]
+ CRUSH rule 4 x 720 [6,8,2]
+ CRUSH rule 4 x 721 [5,7]
+ CRUSH rule 4 x 722 [5,6]
+ CRUSH rule 4 x 723 [5,2]
+ CRUSH rule 4 x 724 [2,6]
+ CRUSH rule 4 x 725 [2,5]
+ CRUSH rule 4 x 726 [5,8]
+ CRUSH rule 4 x 727 [5,6,8]
+ CRUSH rule 4 x 728 [2]
+ CRUSH rule 4 x 729 [5,7]
+ CRUSH rule 4 x 730 [5,7]
+ CRUSH rule 4 x 731 [5,2]
+ CRUSH rule 4 x 732 [2,5]
+ CRUSH rule 4 x 733 [5,7]
+ CRUSH rule 4 x 734 [6,5,2]
+ CRUSH rule 4 x 735 [5,8]
+ CRUSH rule 4 x 736 [5,6]
+ CRUSH rule 4 x 737 [2,8]
+ CRUSH rule 4 x 738 [5,2]
+ CRUSH rule 4 x 739 [2]
+ CRUSH rule 4 x 740 [2,7]
+ CRUSH rule 4 x 741 [7,8,2]
+ CRUSH rule 4 x 742 [8,2]
+ CRUSH rule 4 x 743 [7,2]
+ CRUSH rule 4 x 744 [5,7]
+ CRUSH rule 4 x 745 [5,2]
+ CRUSH rule 4 x 746 [5,2]
+ CRUSH rule 4 x 747 [6,2,5]
+ CRUSH rule 4 x 748 [2,7]
+ CRUSH rule 4 x 749 [5,8]
+ CRUSH rule 4 x 750 [2,6,5]
+ CRUSH rule 4 x 751 [2,6]
+ CRUSH rule 4 x 752 [8,2,5]
+ CRUSH rule 4 x 753 [7,8,5]
+ CRUSH rule 4 x 754 [8,6,7]
+ CRUSH rule 4 x 755 [2]
+ CRUSH rule 4 x 756 [5,6,2]
+ CRUSH rule 4 x 757 [8,6,2]
+ CRUSH rule 4 x 758 [6,2,5]
+ CRUSH rule 4 x 759 [8,5]
+ CRUSH rule 4 x 760 [2,5]
+ CRUSH rule 4 x 761 [5,2]
+ CRUSH rule 4 x 762 [2,7,8]
+ CRUSH rule 4 x 763 [8,6,7]
+ CRUSH rule 4 x 764 [2,7]
+ CRUSH rule 4 x 765 [6,5,2]
+ CRUSH rule 4 x 766 [8,5,7]
+ CRUSH rule 4 x 767 [2]
+ CRUSH rule 4 x 768 [8,5,2]
+ CRUSH rule 4 x 769 [6,2,8]
+ CRUSH rule 4 x 770 [6,2,7]
+ CRUSH rule 4 x 771 [7,2,5]
+ CRUSH rule 4 x 772 [8,5,7]
+ CRUSH rule 4 x 773 [5,2]
+ CRUSH rule 4 x 774 [5,6]
+ CRUSH rule 4 x 775 [6,8,5]
+ CRUSH rule 4 x 776 [7,2]
+ CRUSH rule 4 x 777 [5,2,6]
+ CRUSH rule 4 x 778 [2,8]
+ CRUSH rule 4 x 779 [2,7,5]
+ CRUSH rule 4 x 780 [2,5]
+ CRUSH rule 4 x 781 [6,5,7]
+ CRUSH rule 4 x 782 [5,2]
+ CRUSH rule 4 x 783 [7,2,8]
+ CRUSH rule 4 x 784 [2,5]
+ CRUSH rule 4 x 785 [6,2]
+ CRUSH rule 4 x 786 [7,6,5]
+ CRUSH rule 4 x 787 [2,6]
+ CRUSH rule 4 x 788 [6,2,8]
+ CRUSH rule 4 x 789 [2,5]
+ CRUSH rule 4 x 790 [8,5,7]
+ CRUSH rule 4 x 791 [5,8,7]
+ CRUSH rule 4 x 792 [5,8]
+ CRUSH rule 4 x 793 [6,2,5]
+ CRUSH rule 4 x 794 [2,6,8]
+ CRUSH rule 4 x 795 [2,5]
+ CRUSH rule 4 x 796 [5]
+ CRUSH rule 4 x 797 [2,5]
+ CRUSH rule 4 x 798 [6,8,7]
+ CRUSH rule 4 x 799 [5,2]
+ CRUSH rule 4 x 800 [5,2]
+ CRUSH rule 4 x 801 [5,6,8]
+ CRUSH rule 4 x 802 [2,8]
+ CRUSH rule 4 x 803 [2,5,7]
+ CRUSH rule 4 x 804 [6,2,5]
+ CRUSH rule 4 x 805 [5,6,7]
+ CRUSH rule 4 x 806 [2,5]
+ CRUSH rule 4 x 807 [5,7]
+ CRUSH rule 4 x 808 [5,6,2]
+ CRUSH rule 4 x 809 [2,5]
+ CRUSH rule 4 x 810 [5,7]
+ CRUSH rule 4 x 811 [8,5]
+ CRUSH rule 4 x 812 [8,5]
+ CRUSH rule 4 x 813 [6,5,8]
+ CRUSH rule 4 x 814 [5,6,8]
+ CRUSH rule 4 x 815 [5,2]
+ CRUSH rule 4 x 816 [2,8]
+ CRUSH rule 4 x 817 [5,7]
+ CRUSH rule 4 x 818 [5,2]
+ CRUSH rule 4 x 819 [5,2]
+ CRUSH rule 4 x 820 [5]
+ CRUSH rule 4 x 821 [5,8]
+ CRUSH rule 4 x 822 [2,5]
+ CRUSH rule 4 x 823 [5,8,2]
+ CRUSH rule 4 x 824 [5,7]
+ CRUSH rule 4 x 825 [2,8,7]
+ CRUSH rule 4 x 826 [7,2,5]
+ CRUSH rule 4 x 827 [2,8]
+ CRUSH rule 4 x 828 [2,5]
+ CRUSH rule 4 x 829 [5,6,7]
+ CRUSH rule 4 x 830 [2,5]
+ CRUSH rule 4 x 831 [2,6]
+ CRUSH rule 4 x 832 [5]
+ CRUSH rule 4 x 833 [2,7]
+ CRUSH rule 4 x 834 [5,2]
+ CRUSH rule 4 x 835 [8,5]
+ CRUSH rule 4 x 836 [5]
+ CRUSH rule 4 x 837 [6,5,2]
+ CRUSH rule 4 x 838 [6,7,2]
+ CRUSH rule 4 x 839 [5,2,6]
+ CRUSH rule 4 x 840 [7,8,5]
+ CRUSH rule 4 x 841 [5,8]
+ CRUSH rule 4 x 842 [2,5]
+ CRUSH rule 4 x 843 [6,5,7]
+ CRUSH rule 4 x 844 [5,8,2]
+ CRUSH rule 4 x 845 [5,8,6]
+ CRUSH rule 4 x 846 [5,2]
+ CRUSH rule 4 x 847 [2,7]
+ CRUSH rule 4 x 848 [2,6]
+ CRUSH rule 4 x 849 [5]
+ CRUSH rule 4 x 850 [2,5]
+ CRUSH rule 4 x 851 [6,8,7]
+ CRUSH rule 4 x 852 [7,5,8]
+ CRUSH rule 4 x 853 [6,8,2]
+ CRUSH rule 4 x 854 [7,6,2]
+ CRUSH rule 4 x 855 [5,7,2]
+ CRUSH rule 4 x 856 [6,7,5]
+ CRUSH rule 4 x 857 [8,5,2]
+ CRUSH rule 4 x 858 [6,5,2]
+ CRUSH rule 4 x 859 [6,2,7]
+ CRUSH rule 4 x 860 [5,2]
+ CRUSH rule 4 x 861 [8,7,6]
+ CRUSH rule 4 x 862 [6,2,7]
+ CRUSH rule 4 x 863 [8,7,2]
+ CRUSH rule 4 x 864 [5,6,8]
+ CRUSH rule 4 x 865 [8,2]
+ CRUSH rule 4 x 866 [5,8]
+ CRUSH rule 4 x 867 [6,5,2]
+ CRUSH rule 4 x 868 [6,5,2]
+ CRUSH rule 4 x 869 [8,7,5]
+ CRUSH rule 4 x 870 [2,5,8]
+ CRUSH rule 4 x 871 [5]
+ CRUSH rule 4 x 872 [5,2]
+ CRUSH rule 4 x 873 [5,6]
+ CRUSH rule 4 x 874 [2,6]
+ CRUSH rule 4 x 875 [2,6,5]
+ CRUSH rule 4 x 876 [5,8,2]
+ CRUSH rule 4 x 877 [6,5,2]
+ CRUSH rule 4 x 878 [5,2]
+ CRUSH rule 4 x 879 [7,5,8]
+ CRUSH rule 4 x 880 [5,2]
+ CRUSH rule 4 x 881 [5,6,2]
+ CRUSH rule 4 x 882 [5,2]
+ CRUSH rule 4 x 883 [2]
+ CRUSH rule 4 x 884 [6,2,5]
+ CRUSH rule 4 x 885 [5,2]
+ CRUSH rule 4 x 886 [5,6]
+ CRUSH rule 4 x 887 [7,5,2]
+ CRUSH rule 4 x 888 [6,8,2]
+ CRUSH rule 4 x 889 [2,7]
+ CRUSH rule 4 x 890 [7,2]
+ CRUSH rule 4 x 891 [2,8]
+ CRUSH rule 4 x 892 [6,2,5]
+ CRUSH rule 4 x 893 [2,5,7]
+ CRUSH rule 4 x 894 [7,5,2]
+ CRUSH rule 4 x 895 [5,2]
+ CRUSH rule 4 x 896 [2,8]
+ CRUSH rule 4 x 897 [5,2,6]
+ CRUSH rule 4 x 898 [2,5]
+ CRUSH rule 4 x 899 [2,7,6]
+ CRUSH rule 4 x 900 [5,2]
+ CRUSH rule 4 x 901 [5,2]
+ CRUSH rule 4 x 902 [8,5,7]
+ CRUSH rule 4 x 903 [5,7]
+ CRUSH rule 4 x 904 [5,6,8]
+ CRUSH rule 4 x 905 [6,2,5]
+ CRUSH rule 4 x 906 [2]
+ CRUSH rule 4 x 907 [7,2]
+ CRUSH rule 4 x 908 [5,8,2]
+ CRUSH rule 4 x 909 [2,5]
+ CRUSH rule 4 x 910 [6,5,2]
+ CRUSH rule 4 x 911 [5,8]
+ CRUSH rule 4 x 912 [2,7]
+ CRUSH rule 4 x 913 [7,6,8]
+ CRUSH rule 4 x 914 [6,5,7]
+ CRUSH rule 4 x 915 [8,2,6]
+ CRUSH rule 4 x 916 [5,2]
+ CRUSH rule 4 x 917 [2,5]
+ CRUSH rule 4 x 918 [8,2]
+ CRUSH rule 4 x 919 [6,2,8]
+ CRUSH rule 4 x 920 [7,6,5]
+ CRUSH rule 4 x 921 [2,5]
+ CRUSH rule 4 x 922 [6,7,8]
+ CRUSH rule 4 x 923 [5,6]
+ CRUSH rule 4 x 924 [5]
+ CRUSH rule 4 x 925 [5,7]
+ CRUSH rule 4 x 926 [5]
+ CRUSH rule 4 x 927 [2,6,5]
+ CRUSH rule 4 x 928 [8,2]
+ CRUSH rule 4 x 929 [5,2]
+ CRUSH rule 4 x 930 [2,5,6]
+ CRUSH rule 4 x 931 [5,2]
+ CRUSH rule 4 x 932 [5,2]
+ CRUSH rule 4 x 933 [8,5]
+ CRUSH rule 4 x 934 [5,8]
+ CRUSH rule 4 x 935 [6,5]
+ CRUSH rule 4 x 936 [2,6,7]
+ CRUSH rule 4 x 937 [5]
+ CRUSH rule 4 x 938 [6,5,8]
+ CRUSH rule 4 x 939 [2,7]
+ CRUSH rule 4 x 940 [8,7,6]
+ CRUSH rule 4 x 941 [5,2]
+ CRUSH rule 4 x 942 [2]
+ CRUSH rule 4 x 943 [8,2,5]
+ CRUSH rule 4 x 944 [5,7]
+ CRUSH rule 4 x 945 [7,2,5]
+ CRUSH rule 4 x 946 [2,7]
+ CRUSH rule 4 x 947 [5]
+ CRUSH rule 4 x 948 [7,8,6]
+ CRUSH rule 4 x 949 [6,2,7]
+ CRUSH rule 4 x 950 [5,8]
+ CRUSH rule 4 x 951 [5]
+ CRUSH rule 4 x 952 [2,7]
+ CRUSH rule 4 x 953 [2,5]
+ CRUSH rule 4 x 954 [5,2]
+ CRUSH rule 4 x 955 [8,6,2]
+ CRUSH rule 4 x 956 [2,8]
+ CRUSH rule 4 x 957 [7,6,2]
+ CRUSH rule 4 x 958 [8,7,5]
+ CRUSH rule 4 x 959 [5,2,7]
+ CRUSH rule 4 x 960 [5,6]
+ CRUSH rule 4 x 961 [5,2]
+ CRUSH rule 4 x 962 [7,5]
+ CRUSH rule 4 x 963 [2,5]
+ CRUSH rule 4 x 964 [5,2]
+ CRUSH rule 4 x 965 [7,6,5]
+ CRUSH rule 4 x 966 [5,8]
+ CRUSH rule 4 x 967 [8,6,5]
+ CRUSH rule 4 x 968 [7,2,5]
+ CRUSH rule 4 x 969 [8,2,6]
+ CRUSH rule 4 x 970 [2,6,5]
+ CRUSH rule 4 x 971 [2,7,8]
+ CRUSH rule 4 x 972 [2,8,5]
+ CRUSH rule 4 x 973 [2]
+ CRUSH rule 4 x 974 [5,2]
+ CRUSH rule 4 x 975 [5,7]
+ CRUSH rule 4 x 976 [5]
+ CRUSH rule 4 x 977 [8,5,2]
+ CRUSH rule 4 x 978 [7,2,8]
+ CRUSH rule 4 x 979 [7,6,2]
+ CRUSH rule 4 x 980 [6,2,7]
+ CRUSH rule 4 x 981 [7,5,2]
+ CRUSH rule 4 x 982 [5,2]
+ CRUSH rule 4 x 983 [5,6]
+ CRUSH rule 4 x 984 [2]
+ CRUSH rule 4 x 985 [2,5]
+ CRUSH rule 4 x 986 [8,7,5]
+ CRUSH rule 4 x 987 [2,5]
+ CRUSH rule 4 x 988 [2,5]
+ CRUSH rule 4 x 989 [2,6,5]
+ CRUSH rule 4 x 990 [2,8]
+ CRUSH rule 4 x 991 [2,5]
+ CRUSH rule 4 x 992 [7,2,5]
+ CRUSH rule 4 x 993 [2,6]
+ CRUSH rule 4 x 994 [5]
+ CRUSH rule 4 x 995 [7,6,2]
+ CRUSH rule 4 x 996 [6,7,5]
+ CRUSH rule 4 x 997 [6,5,2]
+ CRUSH rule 4 x 998 [8,2]
+ CRUSH rule 4 x 999 [2,7,8]
+ CRUSH rule 4 x 1000 [8,5,2]
+ CRUSH rule 4 x 1001 [2,5]
+ CRUSH rule 4 x 1002 [2,5]
+ CRUSH rule 4 x 1003 [2,8,7]
+ CRUSH rule 4 x 1004 [6,2]
+ CRUSH rule 4 x 1005 [6,2]
+ CRUSH rule 4 x 1006 [2]
+ CRUSH rule 4 x 1007 [2,5]
+ CRUSH rule 4 x 1008 [2,7]
+ CRUSH rule 4 x 1009 [6,8,5]
+ CRUSH rule 4 x 1010 [5,2]
+ CRUSH rule 4 x 1011 [5,2]
+ CRUSH rule 4 x 1012 [5,2,7]
+ CRUSH rule 4 x 1013 [5,2]
+ CRUSH rule 4 x 1014 [2,8,5]
+ CRUSH rule 4 x 1015 [6,8,5]
+ CRUSH rule 4 x 1016 [2]
+ CRUSH rule 4 x 1017 [6,2]
+ CRUSH rule 4 x 1018 [5]
+ CRUSH rule 4 x 1019 [5,8]
+ CRUSH rule 4 x 1020 [5,2]
+ CRUSH rule 4 x 1021 [5,2]
+ CRUSH rule 4 x 1022 [2,6,7]
+ CRUSH rule 4 x 1023 [5,2]
+ rule 4 (choose-set-two) num_rep 3 result size == 1:\t78/1024 (esc)
+ rule 4 (choose-set-two) num_rep 3 result size == 2:\t480/1024 (esc)
+ rule 4 (choose-set-two) num_rep 3 result size == 3:\t466/1024 (esc)
+ rule 5 (chooseleaf-set), x = 0..1023, numrep = 2..3
+ CRUSH rule 5 x 0 [2,5]
+ CRUSH rule 5 x 1 [2,8]
+ CRUSH rule 5 x 2 [2,5]
+ CRUSH rule 5 x 3 [8,2]
+ CRUSH rule 5 x 4 [5,2]
+ CRUSH rule 5 x 5 [7,2]
+ CRUSH rule 5 x 6 [2,6]
+ CRUSH rule 5 x 7 [5,8]
+ CRUSH rule 5 x 8 [5,6]
+ CRUSH rule 5 x 9 [2,5]
+ CRUSH rule 5 x 10 [2,7]
+ CRUSH rule 5 x 11 [2,7]
+ CRUSH rule 5 x 12 [2,5]
+ CRUSH rule 5 x 13 [5,8]
+ CRUSH rule 5 x 14 [7,2]
+ CRUSH rule 5 x 15 [7,2]
+ CRUSH rule 5 x 16 [5,6]
+ CRUSH rule 5 x 17 [5,2]
+ CRUSH rule 5 x 18 [2,5]
+ CRUSH rule 5 x 19 [7,5]
+ CRUSH rule 5 x 20 [2,5]
+ CRUSH rule 5 x 21 [5,7]
+ CRUSH rule 5 x 22 [8,5]
+ CRUSH rule 5 x 23 [5,6]
+ CRUSH rule 5 x 24 [2,7]
+ CRUSH rule 5 x 25 [5,7]
+ CRUSH rule 5 x 26 [2,8]
+ CRUSH rule 5 x 27 [5,2]
+ CRUSH rule 5 x 28 [6,2]
+ CRUSH rule 5 x 29 [8,5]
+ CRUSH rule 5 x 30 [5,7]
+ CRUSH rule 5 x 31 [8,2]
+ CRUSH rule 5 x 32 [5,6]
+ CRUSH rule 5 x 33 [2,7]
+ CRUSH rule 5 x 34 [2,5]
+ CRUSH rule 5 x 35 [2,8]
+ CRUSH rule 5 x 36 [5,8]
+ CRUSH rule 5 x 37 [2,5]
+ CRUSH rule 5 x 38 [5,8]
+ CRUSH rule 5 x 39 [5,7]
+ CRUSH rule 5 x 40 [7,2]
+ CRUSH rule 5 x 41 [2,6]
+ CRUSH rule 5 x 42 [5,6]
+ CRUSH rule 5 x 43 [2,5]
+ CRUSH rule 5 x 44 [2,6]
+ CRUSH rule 5 x 45 [8,2]
+ CRUSH rule 5 x 46 [2,5]
+ CRUSH rule 5 x 47 [5,2]
+ CRUSH rule 5 x 48 [5,6]
+ CRUSH rule 5 x 49 [5,7]
+ CRUSH rule 5 x 50 [5,2]
+ CRUSH rule 5 x 51 [5,6]
+ CRUSH rule 5 x 52 [8,2]
+ CRUSH rule 5 x 53 [5,8]
+ CRUSH rule 5 x 54 [7,5]
+ CRUSH rule 5 x 55 [8,2]
+ CRUSH rule 5 x 56 [6,5]
+ CRUSH rule 5 x 57 [5,8]
+ CRUSH rule 5 x 58 [2,8]
+ CRUSH rule 5 x 59 [5,2]
+ CRUSH rule 5 x 60 [5,2]
+ CRUSH rule 5 x 61 [5,6]
+ CRUSH rule 5 x 62 [7,2]
+ CRUSH rule 5 x 63 [5,6]
+ CRUSH rule 5 x 64 [5,2]
+ CRUSH rule 5 x 65 [7,5]
+ CRUSH rule 5 x 66 [5,6]
+ CRUSH rule 5 x 67 [5,2]
+ CRUSH rule 5 x 68 [2,5]
+ CRUSH rule 5 x 69 [5,2]
+ CRUSH rule 5 x 70 [7,2]
+ CRUSH rule 5 x 71 [2,8]
+ CRUSH rule 5 x 72 [6,2]
+ CRUSH rule 5 x 73 [2,7]
+ CRUSH rule 5 x 74 [2,7]
+ CRUSH rule 5 x 75 [5,2]
+ CRUSH rule 5 x 76 [5,2]
+ CRUSH rule 5 x 77 [7,2]
+ CRUSH rule 5 x 78 [2,5]
+ CRUSH rule 5 x 79 [5,2]
+ CRUSH rule 5 x 80 [2,5]
+ CRUSH rule 5 x 81 [2,5]
+ CRUSH rule 5 x 82 [7,2]
+ CRUSH rule 5 x 83 [2,6]
+ CRUSH rule 5 x 84 [7,2]
+ CRUSH rule 5 x 85 [5,8]
+ CRUSH rule 5 x 86 [2,6]
+ CRUSH rule 5 x 87 [2,7]
+ CRUSH rule 5 x 88 [2,6]
+ CRUSH rule 5 x 89 [5,2]
+ CRUSH rule 5 x 90 [6,5]
+ CRUSH rule 5 x 91 [5,8]
+ CRUSH rule 5 x 92 [2,8]
+ CRUSH rule 5 x 93 [7,5]
+ CRUSH rule 5 x 94 [2,5]
+ CRUSH rule 5 x 95 [7,5]
+ CRUSH rule 5 x 96 [5,6]
+ CRUSH rule 5 x 97 [8,5]
+ CRUSH rule 5 x 98 [2,7]
+ CRUSH rule 5 x 99 [2,7]
+ CRUSH rule 5 x 100 [2,7]
+ CRUSH rule 5 x 101 [5,7]
+ CRUSH rule 5 x 102 [5,2]
+ CRUSH rule 5 x 103 [5,7]
+ CRUSH rule 5 x 104 [7,5]
+ CRUSH rule 5 x 105 [2,5]
+ CRUSH rule 5 x 106 [2,6]
+ CRUSH rule 5 x 107 [5,2]
+ CRUSH rule 5 x 108 [7,2]
+ CRUSH rule 5 x 109 [2,5]
+ CRUSH rule 5 x 110 [5,2]
+ CRUSH rule 5 x 111 [2,5]
+ CRUSH rule 5 x 112 [2,6]
+ CRUSH rule 5 x 113 [6,2]
+ CRUSH rule 5 x 114 [7,5]
+ CRUSH rule 5 x 115 [8,2]
+ CRUSH rule 5 x 116 [2,6]
+ CRUSH rule 5 x 117 [7,5]
+ CRUSH rule 5 x 118 [2,5]
+ CRUSH rule 5 x 119 [5,6]
+ CRUSH rule 5 x 120 [2,5]
+ CRUSH rule 5 x 121 [2,7]
+ CRUSH rule 5 x 122 [8,5]
+ CRUSH rule 5 x 123 [2,5]
+ CRUSH rule 5 x 124 [5,2]
+ CRUSH rule 5 x 125 [2,7]
+ CRUSH rule 5 x 126 [5,2]
+ CRUSH rule 5 x 127 [5,6]
+ CRUSH rule 5 x 128 [5,6]
+ CRUSH rule 5 x 129 [2,5]
+ CRUSH rule 5 x 130 [5,8]
+ CRUSH rule 5 x 131 [2,5]
+ CRUSH rule 5 x 132 [2,5]
+ CRUSH rule 5 x 133 [5,6]
+ CRUSH rule 5 x 134 [2,8]
+ CRUSH rule 5 x 135 [5,6]
+ CRUSH rule 5 x 136 [2,5]
+ CRUSH rule 5 x 137 [7,5]
+ CRUSH rule 5 x 138 [8,5]
+ CRUSH rule 5 x 139 [5,2]
+ CRUSH rule 5 x 140 [2,6]
+ CRUSH rule 5 x 141 [6,2]
+ CRUSH rule 5 x 142 [5,2]
+ CRUSH rule 5 x 143 [5,8]
+ CRUSH rule 5 x 144 [8,2]
+ CRUSH rule 5 x 145 [8,5]
+ CRUSH rule 5 x 146 [2,6]
+ CRUSH rule 5 x 147 [2,8]
+ CRUSH rule 5 x 148 [5,2]
+ CRUSH rule 5 x 149 [5,8]
+ CRUSH rule 5 x 150 [2,6]
+ CRUSH rule 5 x 151 [5,6]
+ CRUSH rule 5 x 152 [8,5]
+ CRUSH rule 5 x 153 [8,5]
+ CRUSH rule 5 x 154 [5,2]
+ CRUSH rule 5 x 155 [5,7]
+ CRUSH rule 5 x 156 [5,2]
+ CRUSH rule 5 x 157 [5,2]
+ CRUSH rule 5 x 158 [2,8]
+ CRUSH rule 5 x 159 [7,2]
+ CRUSH rule 5 x 160 [2,8]
+ CRUSH rule 5 x 161 [2,5]
+ CRUSH rule 5 x 162 [2,6]
+ CRUSH rule 5 x 163 [5,6]
+ CRUSH rule 5 x 164 [7,2]
+ CRUSH rule 5 x 165 [7,2]
+ CRUSH rule 5 x 166 [2,5]
+ CRUSH rule 5 x 167 [2,7]
+ CRUSH rule 5 x 168 [5,2]
+ CRUSH rule 5 x 169 [2,6]
+ CRUSH rule 5 x 170 [2,5]
+ CRUSH rule 5 x 171 [7,5]
+ CRUSH rule 5 x 172 [2,7]
+ CRUSH rule 5 x 173 [8,5]
+ CRUSH rule 5 x 174 [2,5]
+ CRUSH rule 5 x 175 [6,2]
+ CRUSH rule 5 x 176 [5,2]
+ CRUSH rule 5 x 177 [5,2]
+ CRUSH rule 5 x 178 [5,2]
+ CRUSH rule 5 x 179 [5,2]
+ CRUSH rule 5 x 180 [5,8]
+ CRUSH rule 5 x 181 [6,2]
+ CRUSH rule 5 x 182 [8,5]
+ CRUSH rule 5 x 183 [7,5]
+ CRUSH rule 5 x 184 [5,7]
+ CRUSH rule 5 x 185 [6,2]
+ CRUSH rule 5 x 186 [2,5]
+ CRUSH rule 5 x 187 [2,6]
+ CRUSH rule 5 x 188 [2,8]
+ CRUSH rule 5 x 189 [2,7]
+ CRUSH rule 5 x 190 [5,2]
+ CRUSH rule 5 x 191 [7,2]
+ CRUSH rule 5 x 192 [5,2]
+ CRUSH rule 5 x 193 [5,2]
+ CRUSH rule 5 x 194 [2,5]
+ CRUSH rule 5 x 195 [6,5]
+ CRUSH rule 5 x 196 [6,2]
+ CRUSH rule 5 x 197 [6,5]
+ CRUSH rule 5 x 198 [2,5]
+ CRUSH rule 5 x 199 [2,5]
+ CRUSH rule 5 x 200 [2,5]
+ CRUSH rule 5 x 201 [7,2]
+ CRUSH rule 5 x 202 [6,5]
+ CRUSH rule 5 x 203 [5,8]
+ CRUSH rule 5 x 204 [2,5]
+ CRUSH rule 5 x 205 [2,7]
+ CRUSH rule 5 x 206 [2,7]
+ CRUSH rule 5 x 207 [5,2]
+ CRUSH rule 5 x 208 [7,2]
+ CRUSH rule 5 x 209 [2,8]
+ CRUSH rule 5 x 210 [2,5]
+ CRUSH rule 5 x 211 [5,2]
+ CRUSH rule 5 x 212 [7,5]
+ CRUSH rule 5 x 213 [8,5]
+ CRUSH rule 5 x 214 [5,8]
+ CRUSH rule 5 x 215 [8,2]
+ CRUSH rule 5 x 216 [5,2]
+ CRUSH rule 5 x 217 [2,7]
+ CRUSH rule 5 x 218 [2,7]
+ CRUSH rule 5 x 219 [5,8]
+ CRUSH rule 5 x 220 [5,7]
+ CRUSH rule 5 x 221 [5,6]
+ CRUSH rule 5 x 222 [6,5]
+ CRUSH rule 5 x 223 [2,5]
+ CRUSH rule 5 x 224 [2,5]
+ CRUSH rule 5 x 225 [8,2]
+ CRUSH rule 5 x 226 [7,2]
+ CRUSH rule 5 x 227 [5,2]
+ CRUSH rule 5 x 228 [5,6]
+ CRUSH rule 5 x 229 [5,8]
+ CRUSH rule 5 x 230 [5,7]
+ CRUSH rule 5 x 231 [5,7]
+ CRUSH rule 5 x 232 [2,7]
+ CRUSH rule 5 x 233 [5,7]
+ CRUSH rule 5 x 234 [2,5]
+ CRUSH rule 5 x 235 [5,8]
+ CRUSH rule 5 x 236 [5,2]
+ CRUSH rule 5 x 237 [5,7]
+ CRUSH rule 5 x 238 [5,2]
+ CRUSH rule 5 x 239 [8,5]
+ CRUSH rule 5 x 240 [5,7]
+ CRUSH rule 5 x 241 [5,2]
+ CRUSH rule 5 x 242 [5,2]
+ CRUSH rule 5 x 243 [5,7]
+ CRUSH rule 5 x 244 [5,6]
+ CRUSH rule 5 x 245 [7,2]
+ CRUSH rule 5 x 246 [2,5]
+ CRUSH rule 5 x 247 [6,2]
+ CRUSH rule 5 x 248 [8,2]
+ CRUSH rule 5 x 249 [2,5]
+ CRUSH rule 5 x 250 [2,5]
+ CRUSH rule 5 x 251 [2,5]
+ CRUSH rule 5 x 252 [5,7]
+ CRUSH rule 5 x 253 [5,2]
+ CRUSH rule 5 x 254 [5,2]
+ CRUSH rule 5 x 255 [2,7]
+ CRUSH rule 5 x 256 [5,7]
+ CRUSH rule 5 x 257 [2,8]
+ CRUSH rule 5 x 258 [5,2]
+ CRUSH rule 5 x 259 [5,6]
+ CRUSH rule 5 x 260 [5,6]
+ CRUSH rule 5 x 261 [8,5]
+ CRUSH rule 5 x 262 [5,6]
+ CRUSH rule 5 x 263 [6,2]
+ CRUSH rule 5 x 264 [5,6]
+ CRUSH rule 5 x 265 [8,5]
+ CRUSH rule 5 x 266 [8,2]
+ CRUSH rule 5 x 267 [2,5]
+ CRUSH rule 5 x 268 [2,7]
+ CRUSH rule 5 x 269 [2,8]
+ CRUSH rule 5 x 270 [5,2]
+ CRUSH rule 5 x 271 [7,5]
+ CRUSH rule 5 x 272 [2,8]
+ CRUSH rule 5 x 273 [5,2]
+ CRUSH rule 5 x 274 [6,5]
+ CRUSH rule 5 x 275 [5,7]
+ CRUSH rule 5 x 276 [7,2]
+ CRUSH rule 5 x 277 [6,5]
+ CRUSH rule 5 x 278 [6,2]
+ CRUSH rule 5 x 279 [8,5]
+ CRUSH rule 5 x 280 [2,6]
+ CRUSH rule 5 x 281 [8,2]
+ CRUSH rule 5 x 282 [5,2]
+ CRUSH rule 5 x 283 [8,2]
+ CRUSH rule 5 x 284 [6,5]
+ CRUSH rule 5 x 285 [5,7]
+ CRUSH rule 5 x 286 [2,6]
+ CRUSH rule 5 x 287 [2,5]
+ CRUSH rule 5 x 288 [8,2]
+ CRUSH rule 5 x 289 [5,6]
+ CRUSH rule 5 x 290 [2,5]
+ CRUSH rule 5 x 291 [2,5]
+ CRUSH rule 5 x 292 [8,2]
+ CRUSH rule 5 x 293 [6,2]
+ CRUSH rule 5 x 294 [7,5]
+ CRUSH rule 5 x 295 [5,8]
+ CRUSH rule 5 x 296 [5,2]
+ CRUSH rule 5 x 297 [6,2]
+ CRUSH rule 5 x 298 [2,5]
+ CRUSH rule 5 x 299 [2,8]
+ CRUSH rule 5 x 300 [8,5]
+ CRUSH rule 5 x 301 [2,8]
+ CRUSH rule 5 x 302 [5,2]
+ CRUSH rule 5 x 303 [7,5]
+ CRUSH rule 5 x 304 [2,7]
+ CRUSH rule 5 x 305 [5,8]
+ CRUSH rule 5 x 306 [2,7]
+ CRUSH rule 5 x 307 [2,7]
+ CRUSH rule 5 x 308 [2,8]
+ CRUSH rule 5 x 309 [7,5]
+ CRUSH rule 5 x 310 [5,2]
+ CRUSH rule 5 x 311 [5,6]
+ CRUSH rule 5 x 312 [2,6]
+ CRUSH rule 5 x 313 [5,2]
+ CRUSH rule 5 x 314 [5,2]
+ CRUSH rule 5 x 315 [2,5]
+ CRUSH rule 5 x 316 [6,5]
+ CRUSH rule 5 x 317 [2,6]
+ CRUSH rule 5 x 318 [8,2]
+ CRUSH rule 5 x 319 [5,2]
+ CRUSH rule 5 x 320 [5,7]
+ CRUSH rule 5 x 321 [2,5]
+ CRUSH rule 5 x 322 [2,7]
+ CRUSH rule 5 x 323 [5,7]
+ CRUSH rule 5 x 324 [7,2]
+ CRUSH rule 5 x 325 [5,6]
+ CRUSH rule 5 x 326 [5,2]
+ CRUSH rule 5 x 327 [2,6]
+ CRUSH rule 5 x 328 [7,5]
+ CRUSH rule 5 x 329 [5,6]
+ CRUSH rule 5 x 330 [5,7]
+ CRUSH rule 5 x 331 [2,6]
+ CRUSH rule 5 x 332 [2,5]
+ CRUSH rule 5 x 333 [6,5]
+ CRUSH rule 5 x 334 [8,5]
+ CRUSH rule 5 x 335 [7,2]
+ CRUSH rule 5 x 336 [5,6]
+ CRUSH rule 5 x 337 [7,2]
+ CRUSH rule 5 x 338 [5,6]
+ CRUSH rule 5 x 339 [7,5]
+ CRUSH rule 5 x 340 [2,8]
+ CRUSH rule 5 x 341 [5,2]
+ CRUSH rule 5 x 342 [2,7]
+ CRUSH rule 5 x 343 [6,5]
+ CRUSH rule 5 x 344 [6,2]
+ CRUSH rule 5 x 345 [5,7]
+ CRUSH rule 5 x 346 [8,2]
+ CRUSH rule 5 x 347 [5,2]
+ CRUSH rule 5 x 348 [8,2]
+ CRUSH rule 5 x 349 [2,6]
+ CRUSH rule 5 x 350 [8,5]
+ CRUSH rule 5 x 351 [5,6]
+ CRUSH rule 5 x 352 [2,8]
+ CRUSH rule 5 x 353 [6,5]
+ CRUSH rule 5 x 354 [2,5]
+ CRUSH rule 5 x 355 [5,8]
+ CRUSH rule 5 x 356 [5,2]
+ CRUSH rule 5 x 357 [6,2]
+ CRUSH rule 5 x 358 [2,8]
+ CRUSH rule 5 x 359 [6,2]
+ CRUSH rule 5 x 360 [5,2]
+ CRUSH rule 5 x 361 [8,5]
+ CRUSH rule 5 x 362 [5,2]
+ CRUSH rule 5 x 363 [5,2]
+ CRUSH rule 5 x 364 [2,5]
+ CRUSH rule 5 x 365 [6,5]
+ CRUSH rule 5 x 366 [7,2]
+ CRUSH rule 5 x 367 [5,2]
+ CRUSH rule 5 x 368 [7,5]
+ CRUSH rule 5 x 369 [5,7]
+ CRUSH rule 5 x 370 [8,2]
+ CRUSH rule 5 x 371 [2,5]
+ CRUSH rule 5 x 372 [5,2]
+ CRUSH rule 5 x 373 [2,6]
+ CRUSH rule 5 x 374 [5,8]
+ CRUSH rule 5 x 375 [6,5]
+ CRUSH rule 5 x 376 [7,2]
+ CRUSH rule 5 x 377 [2,5]
+ CRUSH rule 5 x 378 [2,8]
+ CRUSH rule 5 x 379 [8,5]
+ CRUSH rule 5 x 380 [2,5]
+ CRUSH rule 5 x 381 [2,5]
+ CRUSH rule 5 x 382 [2,5]
+ CRUSH rule 5 x 383 [5,6]
+ CRUSH rule 5 x 384 [7,2]
+ CRUSH rule 5 x 385 [7,5]
+ CRUSH rule 5 x 386 [2,5]
+ CRUSH rule 5 x 387 [2,5]
+ CRUSH rule 5 x 388 [5,2]
+ CRUSH rule 5 x 389 [2,5]
+ CRUSH rule 5 x 390 [5,6]
+ CRUSH rule 5 x 391 [5,6]
+ CRUSH rule 5 x 392 [2,8]
+ CRUSH rule 5 x 393 [5,2]
+ CRUSH rule 5 x 394 [5,7]
+ CRUSH rule 5 x 395 [5,2]
+ CRUSH rule 5 x 396 [5,2]
+ CRUSH rule 5 x 397 [2,5]
+ CRUSH rule 5 x 398 [2,5]
+ CRUSH rule 5 x 399 [8,5]
+ CRUSH rule 5 x 400 [8,2]
+ CRUSH rule 5 x 401 [2,5]
+ CRUSH rule 5 x 402 [7,5]
+ CRUSH rule 5 x 403 [2,5]
+ CRUSH rule 5 x 404 [5,2]
+ CRUSH rule 5 x 405 [6,5]
+ CRUSH rule 5 x 406 [2,6]
+ CRUSH rule 5 x 407 [2,8]
+ CRUSH rule 5 x 408 [5,2]
+ CRUSH rule 5 x 409 [7,5]
+ CRUSH rule 5 x 410 [8,5]
+ CRUSH rule 5 x 411 [2,8]
+ CRUSH rule 5 x 412 [2,5]
+ CRUSH rule 5 x 413 [5,2]
+ CRUSH rule 5 x 414 [5,2]
+ CRUSH rule 5 x 415 [2,6]
+ CRUSH rule 5 x 416 [2,6]
+ CRUSH rule 5 x 417 [8,2]
+ CRUSH rule 5 x 418 [7,2]
+ CRUSH rule 5 x 419 [8,5]
+ CRUSH rule 5 x 420 [2,5]
+ CRUSH rule 5 x 421 [8,5]
+ CRUSH rule 5 x 422 [6,5]
+ CRUSH rule 5 x 423 [2,5]
+ CRUSH rule 5 x 424 [8,5]
+ CRUSH rule 5 x 425 [2,5]
+ CRUSH rule 5 x 426 [6,2]
+ CRUSH rule 5 x 427 [2,7]
+ CRUSH rule 5 x 428 [5,7]
+ CRUSH rule 5 x 429 [5,6]
+ CRUSH rule 5 x 430 [5,6]
+ CRUSH rule 5 x 431 [5,2]
+ CRUSH rule 5 x 432 [7,2]
+ CRUSH rule 5 x 433 [6,5]
+ CRUSH rule 5 x 434 [5,2]
+ CRUSH rule 5 x 435 [2,5]
+ CRUSH rule 5 x 436 [5,2]
+ CRUSH rule 5 x 437 [7,5]
+ CRUSH rule 5 x 438 [2,5]
+ CRUSH rule 5 x 439 [2,5]
+ CRUSH rule 5 x 440 [2,7]
+ CRUSH rule 5 x 441 [5,7]
+ CRUSH rule 5 x 442 [2,5]
+ CRUSH rule 5 x 443 [6,2]
+ CRUSH rule 5 x 444 [7,2]
+ CRUSH rule 5 x 445 [6,5]
+ CRUSH rule 5 x 446 [5,2]
+ CRUSH rule 5 x 447 [2,5]
+ CRUSH rule 5 x 448 [7,2]
+ CRUSH rule 5 x 449 [7,5]
+ CRUSH rule 5 x 450 [5,2]
+ CRUSH rule 5 x 451 [6,5]
+ CRUSH rule 5 x 452 [8,5]
+ CRUSH rule 5 x 453 [6,5]
+ CRUSH rule 5 x 454 [6,5]
+ CRUSH rule 5 x 455 [2,7]
+ CRUSH rule 5 x 456 [6,2]
+ CRUSH rule 5 x 457 [7,2]
+ CRUSH rule 5 x 458 [2,8]
+ CRUSH rule 5 x 459 [2,7]
+ CRUSH rule 5 x 460 [6,5]
+ CRUSH rule 5 x 461 [6,5]
+ CRUSH rule 5 x 462 [8,2]
+ CRUSH rule 5 x 463 [6,2]
+ CRUSH rule 5 x 464 [7,5]
+ CRUSH rule 5 x 465 [7,2]
+ CRUSH rule 5 x 466 [5,8]
+ CRUSH rule 5 x 467 [6,5]
+ CRUSH rule 5 x 468 [7,2]
+ CRUSH rule 5 x 469 [7,2]
+ CRUSH rule 5 x 470 [5,2]
+ CRUSH rule 5 x 471 [2,7]
+ CRUSH rule 5 x 472 [5,2]
+ CRUSH rule 5 x 473 [2,5]
+ CRUSH rule 5 x 474 [6,2]
+ CRUSH rule 5 x 475 [6,2]
+ CRUSH rule 5 x 476 [5,6]
+ CRUSH rule 5 x 477 [5,8]
+ CRUSH rule 5 x 478 [6,2]
+ CRUSH rule 5 x 479 [2,5]
+ CRUSH rule 5 x 480 [2,8]
+ CRUSH rule 5 x 481 [2,5]
+ CRUSH rule 5 x 482 [5,7]
+ CRUSH rule 5 x 483 [2,6]
+ CRUSH rule 5 x 484 [2,7]
+ CRUSH rule 5 x 485 [5,7]
+ CRUSH rule 5 x 486 [5,2]
+ CRUSH rule 5 x 487 [5,2]
+ CRUSH rule 5 x 488 [5,7]
+ CRUSH rule 5 x 489 [2,8]
+ CRUSH rule 5 x 490 [6,5]
+ CRUSH rule 5 x 491 [2,7]
+ CRUSH rule 5 x 492 [6,5]
+ CRUSH rule 5 x 493 [2,7]
+ CRUSH rule 5 x 494 [2,7]
+ CRUSH rule 5 x 495 [5,2]
+ CRUSH rule 5 x 496 [7,5]
+ CRUSH rule 5 x 497 [5,7]
+ CRUSH rule 5 x 498 [2,5]
+ CRUSH rule 5 x 499 [8,5]
+ CRUSH rule 5 x 500 [5,6]
+ CRUSH rule 5 x 501 [2,7]
+ CRUSH rule 5 x 502 [7,2]
+ CRUSH rule 5 x 503 [2,5]
+ CRUSH rule 5 x 504 [5,6]
+ CRUSH rule 5 x 505 [2,7]
+ CRUSH rule 5 x 506 [5,2]
+ CRUSH rule 5 x 507 [6,2]
+ CRUSH rule 5 x 508 [2,5]
+ CRUSH rule 5 x 509 [7,5]
+ CRUSH rule 5 x 510 [6,2]
+ CRUSH rule 5 x 511 [5,8]
+ CRUSH rule 5 x 512 [7,2]
+ CRUSH rule 5 x 513 [7,2]
+ CRUSH rule 5 x 514 [5,6]
+ CRUSH rule 5 x 515 [8,5]
+ CRUSH rule 5 x 516 [5,2]
+ CRUSH rule 5 x 517 [7,2]
+ CRUSH rule 5 x 518 [5,6]
+ CRUSH rule 5 x 519 [7,5]
+ CRUSH rule 5 x 520 [2,6]
+ CRUSH rule 5 x 521 [8,2]
+ CRUSH rule 5 x 522 [6,2]
+ CRUSH rule 5 x 523 [5,2]
+ CRUSH rule 5 x 524 [2,5]
+ CRUSH rule 5 x 525 [2,5]
+ CRUSH rule 5 x 526 [2,5]
+ CRUSH rule 5 x 527 [2,5]
+ CRUSH rule 5 x 528 [5,2]
+ CRUSH rule 5 x 529 [5,7]
+ CRUSH rule 5 x 530 [6,5]
+ CRUSH rule 5 x 531 [6,2]
+ CRUSH rule 5 x 532 [6,5]
+ CRUSH rule 5 x 533 [5,6]
+ CRUSH rule 5 x 534 [7,5]
+ CRUSH rule 5 x 535 [8,2]
+ CRUSH rule 5 x 536 [6,2]
+ CRUSH rule 5 x 537 [5,7]
+ CRUSH rule 5 x 538 [6,5]
+ CRUSH rule 5 x 539 [8,5]
+ CRUSH rule 5 x 540 [2,6]
+ CRUSH rule 5 x 541 [2,5]
+ CRUSH rule 5 x 542 [5,2]
+ CRUSH rule 5 x 543 [6,2]
+ CRUSH rule 5 x 544 [5,7]
+ CRUSH rule 5 x 545 [5,7]
+ CRUSH rule 5 x 546 [6,2]
+ CRUSH rule 5 x 547 [8,2]
+ CRUSH rule 5 x 548 [5,2]
+ CRUSH rule 5 x 549 [5,8]
+ CRUSH rule 5 x 550 [2,5]
+ CRUSH rule 5 x 551 [7,5]
+ CRUSH rule 5 x 552 [5,8]
+ CRUSH rule 5 x 553 [5,2]
+ CRUSH rule 5 x 554 [2,8]
+ CRUSH rule 5 x 555 [5,2]
+ CRUSH rule 5 x 556 [5,6]
+ CRUSH rule 5 x 557 [7,5]
+ CRUSH rule 5 x 558 [5,2]
+ CRUSH rule 5 x 559 [5,2]
+ CRUSH rule 5 x 560 [8,5]
+ CRUSH rule 5 x 561 [6,5]
+ CRUSH rule 5 x 562 [5,2]
+ CRUSH rule 5 x 563 [2,6]
+ CRUSH rule 5 x 564 [5,2]
+ CRUSH rule 5 x 565 [5,6]
+ CRUSH rule 5 x 566 [5,7]
+ CRUSH rule 5 x 567 [5,6]
+ CRUSH rule 5 x 568 [7,5]
+ CRUSH rule 5 x 569 [5,2]
+ CRUSH rule 5 x 570 [2,5]
+ CRUSH rule 5 x 571 [5,7]
+ CRUSH rule 5 x 572 [5,2]
+ CRUSH rule 5 x 573 [5,2]
+ CRUSH rule 5 x 574 [2,5]
+ CRUSH rule 5 x 575 [8,2]
+ CRUSH rule 5 x 576 [5,6]
+ CRUSH rule 5 x 577 [8,2]
+ CRUSH rule 5 x 578 [6,2]
+ CRUSH rule 5 x 579 [5,2]
+ CRUSH rule 5 x 580 [5,2]
+ CRUSH rule 5 x 581 [7,2]
+ CRUSH rule 5 x 582 [2,8]
+ CRUSH rule 5 x 583 [6,2]
+ CRUSH rule 5 x 584 [8,2]
+ CRUSH rule 5 x 585 [7,2]
+ CRUSH rule 5 x 586 [2,7]
+ CRUSH rule 5 x 587 [2,5]
+ CRUSH rule 5 x 588 [5,7]
+ CRUSH rule 5 x 589 [7,2]
+ CRUSH rule 5 x 590 [6,2]
+ CRUSH rule 5 x 591 [5,2]
+ CRUSH rule 5 x 592 [2,5]
+ CRUSH rule 5 x 593 [2,8]
+ CRUSH rule 5 x 594 [2,7]
+ CRUSH rule 5 x 595 [7,2]
+ CRUSH rule 5 x 596 [5,2]
+ CRUSH rule 5 x 597 [5,2]
+ CRUSH rule 5 x 598 [5,2]
+ CRUSH rule 5 x 599 [5,2]
+ CRUSH rule 5 x 600 [7,2]
+ CRUSH rule 5 x 601 [2,7]
+ CRUSH rule 5 x 602 [5,7]
+ CRUSH rule 5 x 603 [5,2]
+ CRUSH rule 5 x 604 [7,5]
+ CRUSH rule 5 x 605 [5,2]
+ CRUSH rule 5 x 606 [2,7]
+ CRUSH rule 5 x 607 [2,5]
+ CRUSH rule 5 x 608 [5,2]
+ CRUSH rule 5 x 609 [5,2]
+ CRUSH rule 5 x 610 [5,7]
+ CRUSH rule 5 x 611 [2,8]
+ CRUSH rule 5 x 612 [2,6]
+ CRUSH rule 5 x 613 [7,2]
+ CRUSH rule 5 x 614 [7,2]
+ CRUSH rule 5 x 615 [6,2]
+ CRUSH rule 5 x 616 [2,8]
+ CRUSH rule 5 x 617 [6,2]
+ CRUSH rule 5 x 618 [7,5]
+ CRUSH rule 5 x 619 [5,2]
+ CRUSH rule 5 x 620 [5,2]
+ CRUSH rule 5 x 621 [5,8]
+ CRUSH rule 5 x 622 [2,5]
+ CRUSH rule 5 x 623 [2,6]
+ CRUSH rule 5 x 624 [5,2]
+ CRUSH rule 5 x 625 [2,5]
+ CRUSH rule 5 x 626 [7,2]
+ CRUSH rule 5 x 627 [2,7]
+ CRUSH rule 5 x 628 [8,2]
+ CRUSH rule 5 x 629 [2,6]
+ CRUSH rule 5 x 630 [2,7]
+ CRUSH rule 5 x 631 [2,7]
+ CRUSH rule 5 x 632 [7,2]
+ CRUSH rule 5 x 633 [8,5]
+ CRUSH rule 5 x 634 [2,5]
+ CRUSH rule 5 x 635 [5,6]
+ CRUSH rule 5 x 636 [2,5]
+ CRUSH rule 5 x 637 [5,2]
+ CRUSH rule 5 x 638 [6,2]
+ CRUSH rule 5 x 639 [5,2]
+ CRUSH rule 5 x 640 [5,2]
+ CRUSH rule 5 x 641 [7,2]
+ CRUSH rule 5 x 642 [2,7]
+ CRUSH rule 5 x 643 [5,2]
+ CRUSH rule 5 x 644 [8,2]
+ CRUSH rule 5 x 645 [5,7]
+ CRUSH rule 5 x 646 [8,2]
+ CRUSH rule 5 x 647 [7,2]
+ CRUSH rule 5 x 648 [2,6]
+ CRUSH rule 5 x 649 [5,7]
+ CRUSH rule 5 x 650 [7,5]
+ CRUSH rule 5 x 651 [5,7]
+ CRUSH rule 5 x 652 [5,7]
+ CRUSH rule 5 x 653 [8,5]
+ CRUSH rule 5 x 654 [7,5]
+ CRUSH rule 5 x 655 [2,5]
+ CRUSH rule 5 x 656 [5,7]
+ CRUSH rule 5 x 657 [6,2]
+ CRUSH rule 5 x 658 [5,6]
+ CRUSH rule 5 x 659 [5,6]
+ CRUSH rule 5 x 660 [7,5]
+ CRUSH rule 5 x 661 [2,8]
+ CRUSH rule 5 x 662 [5,2]
+ CRUSH rule 5 x 663 [2,5]
+ CRUSH rule 5 x 664 [2,5]
+ CRUSH rule 5 x 665 [5,7]
+ CRUSH rule 5 x 666 [2,8]
+ CRUSH rule 5 x 667 [2,5]
+ CRUSH rule 5 x 668 [5,7]
+ CRUSH rule 5 x 669 [6,5]
+ CRUSH rule 5 x 670 [5,2]
+ CRUSH rule 5 x 671 [2,7]
+ CRUSH rule 5 x 672 [5,2]
+ CRUSH rule 5 x 673 [5,2]
+ CRUSH rule 5 x 674 [5,2]
+ CRUSH rule 5 x 675 [2,8]
+ CRUSH rule 5 x 676 [2,5]
+ CRUSH rule 5 x 677 [5,2]
+ CRUSH rule 5 x 678 [2,5]
+ CRUSH rule 5 x 679 [6,2]
+ CRUSH rule 5 x 680 [2,5]
+ CRUSH rule 5 x 681 [5,7]
+ CRUSH rule 5 x 682 [2,5]
+ CRUSH rule 5 x 683 [2,5]
+ CRUSH rule 5 x 684 [7,2]
+ CRUSH rule 5 x 685 [7,2]
+ CRUSH rule 5 x 686 [2,5]
+ CRUSH rule 5 x 687 [5,6]
+ CRUSH rule 5 x 688 [5,7]
+ CRUSH rule 5 x 689 [6,5]
+ CRUSH rule 5 x 690 [8,2]
+ CRUSH rule 5 x 691 [5,2]
+ CRUSH rule 5 x 692 [7,2]
+ CRUSH rule 5 x 693 [6,5]
+ CRUSH rule 5 x 694 [6,5]
+ CRUSH rule 5 x 695 [2,8]
+ CRUSH rule 5 x 696 [2,5]
+ CRUSH rule 5 x 697 [6,2]
+ CRUSH rule 5 x 698 [6,2]
+ CRUSH rule 5 x 699 [2,6]
+ CRUSH rule 5 x 700 [2,5]
+ CRUSH rule 5 x 701 [5,2]
+ CRUSH rule 5 x 702 [5,2]
+ CRUSH rule 5 x 703 [8,5]
+ CRUSH rule 5 x 704 [2,5]
+ CRUSH rule 5 x 705 [8,2]
+ CRUSH rule 5 x 706 [2,5]
+ CRUSH rule 5 x 707 [7,5]
+ CRUSH rule 5 x 708 [5,7]
+ CRUSH rule 5 x 709 [6,5]
+ CRUSH rule 5 x 710 [8,5]
+ CRUSH rule 5 x 711 [2,5]
+ CRUSH rule 5 x 712 [2,5]
+ CRUSH rule 5 x 713 [6,5]
+ CRUSH rule 5 x 714 [5,2]
+ CRUSH rule 5 x 715 [2,5]
+ CRUSH rule 5 x 716 [5,6]
+ CRUSH rule 5 x 717 [8,2]
+ CRUSH rule 5 x 718 [5,7]
+ CRUSH rule 5 x 719 [2,6]
+ CRUSH rule 5 x 720 [6,2]
+ CRUSH rule 5 x 721 [5,7]
+ CRUSH rule 5 x 722 [5,7]
+ CRUSH rule 5 x 723 [5,2]
+ CRUSH rule 5 x 724 [2,6]
+ CRUSH rule 5 x 725 [2,5]
+ CRUSH rule 5 x 726 [5,8]
+ CRUSH rule 5 x 727 [5,6]
+ CRUSH rule 5 x 728 [2,7]
+ CRUSH rule 5 x 729 [5,6]
+ CRUSH rule 5 x 730 [5,7]
+ CRUSH rule 5 x 731 [5,2]
+ CRUSH rule 5 x 732 [2,5]
+ CRUSH rule 5 x 733 [5,7]
+ CRUSH rule 5 x 734 [6,5]
+ CRUSH rule 5 x 735 [5,8]
+ CRUSH rule 5 x 736 [5,8]
+ CRUSH rule 5 x 737 [2,6]
+ CRUSH rule 5 x 738 [5,2]
+ CRUSH rule 5 x 739 [2,7]
+ CRUSH rule 5 x 740 [2,8]
+ CRUSH rule 5 x 741 [7,2]
+ CRUSH rule 5 x 742 [8,2]
+ CRUSH rule 5 x 743 [7,2]
+ CRUSH rule 5 x 744 [5,7]
+ CRUSH rule 5 x 745 [5,2]
+ CRUSH rule 5 x 746 [5,2]
+ CRUSH rule 5 x 747 [6,2]
+ CRUSH rule 5 x 748 [2,7]
+ CRUSH rule 5 x 749 [5,8]
+ CRUSH rule 5 x 750 [2,6]
+ CRUSH rule 5 x 751 [2,8]
+ CRUSH rule 5 x 752 [8,2]
+ CRUSH rule 5 x 753 [7,5]
+ CRUSH rule 5 x 754 [8,5]
+ CRUSH rule 5 x 755 [2,6]
+ CRUSH rule 5 x 756 [5,6]
+ CRUSH rule 5 x 757 [8,2]
+ CRUSH rule 5 x 758 [6,2]
+ CRUSH rule 5 x 759 [8,5]
+ CRUSH rule 5 x 760 [2,5]
+ CRUSH rule 5 x 761 [5,2]
+ CRUSH rule 5 x 762 [2,7]
+ CRUSH rule 5 x 763 [8,5]
+ CRUSH rule 5 x 764 [2,7]
+ CRUSH rule 5 x 765 [6,5]
+ CRUSH rule 5 x 766 [8,5]
+ CRUSH rule 5 x 767 [2,8]
+ CRUSH rule 5 x 768 [8,5]
+ CRUSH rule 5 x 769 [6,2]
+ CRUSH rule 5 x 770 [6,2]
+ CRUSH rule 5 x 771 [7,2]
+ CRUSH rule 5 x 772 [8,5]
+ CRUSH rule 5 x 773 [5,2]
+ CRUSH rule 5 x 774 [5,6]
+ CRUSH rule 5 x 775 [6,5]
+ CRUSH rule 5 x 776 [7,2]
+ CRUSH rule 5 x 777 [5,2]
+ CRUSH rule 5 x 778 [2,8]
+ CRUSH rule 5 x 779 [2,7]
+ CRUSH rule 5 x 780 [2,5]
+ CRUSH rule 5 x 781 [6,5]
+ CRUSH rule 5 x 782 [5,2]
+ CRUSH rule 5 x 783 [7,2]
+ CRUSH rule 5 x 784 [2,5]
+ CRUSH rule 5 x 785 [6,2]
+ CRUSH rule 5 x 786 [7,5]
+ CRUSH rule 5 x 787 [2,6]
+ CRUSH rule 5 x 788 [6,2]
+ CRUSH rule 5 x 789 [2,5]
+ CRUSH rule 5 x 790 [8,5]
+ CRUSH rule 5 x 791 [5,8]
+ CRUSH rule 5 x 792 [5,8]
+ CRUSH rule 5 x 793 [6,2]
+ CRUSH rule 5 x 794 [2,6]
+ CRUSH rule 5 x 795 [2,5]
+ CRUSH rule 5 x 796 [5,7]
+ CRUSH rule 5 x 797 [2,5]
+ CRUSH rule 5 x 798 [6,2]
+ CRUSH rule 5 x 799 [5,2]
+ CRUSH rule 5 x 800 [5,2]
+ CRUSH rule 5 x 801 [5,6]
+ CRUSH rule 5 x 802 [2,8]
+ CRUSH rule 5 x 803 [2,5]
+ CRUSH rule 5 x 804 [6,2]
+ CRUSH rule 5 x 805 [5,6]
+ CRUSH rule 5 x 806 [2,5]
+ CRUSH rule 5 x 807 [5,7]
+ CRUSH rule 5 x 808 [5,6]
+ CRUSH rule 5 x 809 [2,5]
+ CRUSH rule 5 x 810 [5,7]
+ CRUSH rule 5 x 811 [8,5]
+ CRUSH rule 5 x 812 [8,5]
+ CRUSH rule 5 x 813 [6,5]
+ CRUSH rule 5 x 814 [5,6]
+ CRUSH rule 5 x 815 [5,2]
+ CRUSH rule 5 x 816 [2,7]
+ CRUSH rule 5 x 817 [5,8]
+ CRUSH rule 5 x 818 [5,2]
+ CRUSH rule 5 x 819 [5,2]
+ CRUSH rule 5 x 820 [5,6]
+ CRUSH rule 5 x 821 [5,6]
+ CRUSH rule 5 x 822 [2,5]
+ CRUSH rule 5 x 823 [5,8]
+ CRUSH rule 5 x 824 [5,7]
+ CRUSH rule 5 x 825 [2,8]
+ CRUSH rule 5 x 826 [7,2]
+ CRUSH rule 5 x 827 [2,8]
+ CRUSH rule 5 x 828 [2,5]
+ CRUSH rule 5 x 829 [5,6]
+ CRUSH rule 5 x 830 [2,5]
+ CRUSH rule 5 x 831 [2,6]
+ CRUSH rule 5 x 832 [5,7]
+ CRUSH rule 5 x 833 [2,7]
+ CRUSH rule 5 x 834 [5,2]
+ CRUSH rule 5 x 835 [8,5]
+ CRUSH rule 5 x 836 [5,7]
+ CRUSH rule 5 x 837 [6,5]
+ CRUSH rule 5 x 838 [6,2]
+ CRUSH rule 5 x 839 [5,2]
+ CRUSH rule 5 x 840 [7,5]
+ CRUSH rule 5 x 841 [5,8]
+ CRUSH rule 5 x 842 [2,5]
+ CRUSH rule 5 x 843 [6,5]
+ CRUSH rule 5 x 844 [5,8]
+ CRUSH rule 5 x 845 [5,8]
+ CRUSH rule 5 x 846 [5,2]
+ CRUSH rule 5 x 847 [2,8]
+ CRUSH rule 5 x 848 [2,6]
+ CRUSH rule 5 x 849 [5,6]
+ CRUSH rule 5 x 850 [2,5]
+ CRUSH rule 5 x 851 [6,5]
+ CRUSH rule 5 x 852 [7,5]
+ CRUSH rule 5 x 853 [6,2]
+ CRUSH rule 5 x 854 [7,2]
+ CRUSH rule 5 x 855 [5,7]
+ CRUSH rule 5 x 856 [6,5]
+ CRUSH rule 5 x 857 [8,5]
+ CRUSH rule 5 x 858 [6,5]
+ CRUSH rule 5 x 859 [6,2]
+ CRUSH rule 5 x 860 [5,2]
+ CRUSH rule 5 x 861 [8,5]
+ CRUSH rule 5 x 862 [6,2]
+ CRUSH rule 5 x 863 [8,2]
+ CRUSH rule 5 x 864 [5,6]
+ CRUSH rule 5 x 865 [8,2]
+ CRUSH rule 5 x 866 [5,6]
+ CRUSH rule 5 x 867 [6,5]
+ CRUSH rule 5 x 868 [6,5]
+ CRUSH rule 5 x 869 [8,5]
+ CRUSH rule 5 x 870 [2,5]
+ CRUSH rule 5 x 871 [5,2]
+ CRUSH rule 5 x 872 [5,2]
+ CRUSH rule 5 x 873 [5,6]
+ CRUSH rule 5 x 874 [2,6]
+ CRUSH rule 5 x 875 [2,6]
+ CRUSH rule 5 x 876 [5,8]
+ CRUSH rule 5 x 877 [6,5]
+ CRUSH rule 5 x 878 [5,2]
+ CRUSH rule 5 x 879 [7,5]
+ CRUSH rule 5 x 880 [5,2]
+ CRUSH rule 5 x 881 [5,6]
+ CRUSH rule 5 x 882 [5,2]
+ CRUSH rule 5 x 883 [2,5]
+ CRUSH rule 5 x 884 [6,2]
+ CRUSH rule 5 x 885 [5,2]
+ CRUSH rule 5 x 886 [5,6]
+ CRUSH rule 5 x 887 [7,5]
+ CRUSH rule 5 x 888 [6,2]
+ CRUSH rule 5 x 889 [2,6]
+ CRUSH rule 5 x 890 [7,2]
+ CRUSH rule 5 x 891 [2,8]
+ CRUSH rule 5 x 892 [6,2]
+ CRUSH rule 5 x 893 [2,5]
+ CRUSH rule 5 x 894 [7,5]
+ CRUSH rule 5 x 895 [5,2]
+ CRUSH rule 5 x 896 [2,8]
+ CRUSH rule 5 x 897 [5,2]
+ CRUSH rule 5 x 898 [2,5]
+ CRUSH rule 5 x 899 [2,7]
+ CRUSH rule 5 x 900 [5,2]
+ CRUSH rule 5 x 901 [5,2]
+ CRUSH rule 5 x 902 [8,5]
+ CRUSH rule 5 x 903 [5,7]
+ CRUSH rule 5 x 904 [5,6]
+ CRUSH rule 5 x 905 [6,2]
+ CRUSH rule 5 x 906 [2,6]
+ CRUSH rule 5 x 907 [7,2]
+ CRUSH rule 5 x 908 [5,8]
+ CRUSH rule 5 x 909 [2,5]
+ CRUSH rule 5 x 910 [6,5]
+ CRUSH rule 5 x 911 [5,8]
+ CRUSH rule 5 x 912 [2,7]
+ CRUSH rule 5 x 913 [7,2]
+ CRUSH rule 5 x 914 [6,5]
+ CRUSH rule 5 x 915 [8,2]
+ CRUSH rule 5 x 916 [5,2]
+ CRUSH rule 5 x 917 [2,5]
+ CRUSH rule 5 x 918 [8,2]
+ CRUSH rule 5 x 919 [6,2]
+ CRUSH rule 5 x 920 [7,5]
+ CRUSH rule 5 x 921 [2,5]
+ CRUSH rule 5 x 922 [6,5]
+ CRUSH rule 5 x 923 [5,8]
+ CRUSH rule 5 x 924 [5,2]
+ CRUSH rule 5 x 925 [5,7]
+ CRUSH rule 5 x 926 [5,2]
+ CRUSH rule 5 x 927 [2,6]
+ CRUSH rule 5 x 928 [8,2]
+ CRUSH rule 5 x 929 [5,2]
+ CRUSH rule 5 x 930 [2,5]
+ CRUSH rule 5 x 931 [5,2]
+ CRUSH rule 5 x 932 [5,2]
+ CRUSH rule 5 x 933 [8,5]
+ CRUSH rule 5 x 934 [5,6]
+ CRUSH rule 5 x 935 [6,5]
+ CRUSH rule 5 x 936 [2,6]
+ CRUSH rule 5 x 937 [5,8]
+ CRUSH rule 5 x 938 [6,5]
+ CRUSH rule 5 x 939 [2,7]
+ CRUSH rule 5 x 940 [8,5]
+ CRUSH rule 5 x 941 [5,2]
+ CRUSH rule 5 x 942 [2,8]
+ CRUSH rule 5 x 943 [8,2]
+ CRUSH rule 5 x 944 [5,8]
+ CRUSH rule 5 x 945 [7,2]
+ CRUSH rule 5 x 946 [2,8]
+ CRUSH rule 5 x 947 [5,2]
+ CRUSH rule 5 x 948 [7,5]
+ CRUSH rule 5 x 949 [6,2]
+ CRUSH rule 5 x 950 [5,6]
+ CRUSH rule 5 x 951 [5,8]
+ CRUSH rule 5 x 952 [2,7]
+ CRUSH rule 5 x 953 [2,5]
+ CRUSH rule 5 x 954 [5,2]
+ CRUSH rule 5 x 955 [8,2]
+ CRUSH rule 5 x 956 [2,6]
+ CRUSH rule 5 x 957 [7,2]
+ CRUSH rule 5 x 958 [8,5]
+ CRUSH rule 5 x 959 [5,2]
+ CRUSH rule 5 x 960 [5,6]
+ CRUSH rule 5 x 961 [5,2]
+ CRUSH rule 5 x 962 [7,5]
+ CRUSH rule 5 x 963 [2,5]
+ CRUSH rule 5 x 964 [5,2]
+ CRUSH rule 5 x 965 [7,5]
+ CRUSH rule 5 x 966 [5,8]
+ CRUSH rule 5 x 967 [8,5]
+ CRUSH rule 5 x 968 [7,2]
+ CRUSH rule 5 x 969 [8,2]
+ CRUSH rule 5 x 970 [2,6]
+ CRUSH rule 5 x 971 [2,7]
+ CRUSH rule 5 x 972 [2,8]
+ CRUSH rule 5 x 973 [2,6]
+ CRUSH rule 5 x 974 [5,2]
+ CRUSH rule 5 x 975 [5,7]
+ CRUSH rule 5 x 976 [5,8]
+ CRUSH rule 5 x 977 [8,5]
+ CRUSH rule 5 x 978 [7,2]
+ CRUSH rule 5 x 979 [7,2]
+ CRUSH rule 5 x 980 [6,2]
+ CRUSH rule 5 x 981 [7,5]
+ CRUSH rule 5 x 982 [5,2]
+ CRUSH rule 5 x 983 [5,7]
+ CRUSH rule 5 x 984 [2,7]
+ CRUSH rule 5 x 985 [2,5]
+ CRUSH rule 5 x 986 [8,5]
+ CRUSH rule 5 x 987 [2,5]
+ CRUSH rule 5 x 988 [2,5]
+ CRUSH rule 5 x 989 [2,6]
+ CRUSH rule 5 x 990 [2,6]
+ CRUSH rule 5 x 991 [2,5]
+ CRUSH rule 5 x 992 [7,2]
+ CRUSH rule 5 x 993 [2,6]
+ CRUSH rule 5 x 994 [5,7]
+ CRUSH rule 5 x 995 [7,2]
+ CRUSH rule 5 x 996 [6,5]
+ CRUSH rule 5 x 997 [6,5]
+ CRUSH rule 5 x 998 [8,2]
+ CRUSH rule 5 x 999 [2,7]
+ CRUSH rule 5 x 1000 [8,5]
+ CRUSH rule 5 x 1001 [2,5]
+ CRUSH rule 5 x 1002 [2,5]
+ CRUSH rule 5 x 1003 [2,8]
+ CRUSH rule 5 x 1004 [6,2]
+ CRUSH rule 5 x 1005 [6,2]
+ CRUSH rule 5 x 1006 [2,6]
+ CRUSH rule 5 x 1007 [2,5]
+ CRUSH rule 5 x 1008 [2,7]
+ CRUSH rule 5 x 1009 [6,5]
+ CRUSH rule 5 x 1010 [5,2]
+ CRUSH rule 5 x 1011 [5,2]
+ CRUSH rule 5 x 1012 [5,2]
+ CRUSH rule 5 x 1013 [5,2]
+ CRUSH rule 5 x 1014 [2,8]
+ CRUSH rule 5 x 1015 [6,5]
+ CRUSH rule 5 x 1016 [2,5]
+ CRUSH rule 5 x 1017 [6,2]
+ CRUSH rule 5 x 1018 [5,2]
+ CRUSH rule 5 x 1019 [5,8]
+ CRUSH rule 5 x 1020 [5,2]
+ CRUSH rule 5 x 1021 [5,2]
+ CRUSH rule 5 x 1022 [2,6]
+ CRUSH rule 5 x 1023 [5,2]
+ rule 5 (chooseleaf-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 5 x 0 [2,5,7]
+ CRUSH rule 5 x 1 [2,8,5]
+ CRUSH rule 5 x 2 [2,5,7]
+ CRUSH rule 5 x 3 [8,2,5]
+ CRUSH rule 5 x 4 [5,2,7]
+ CRUSH rule 5 x 5 [7,2,5]
+ CRUSH rule 5 x 6 [2,6,5]
+ CRUSH rule 5 x 7 [5,8,2]
+ CRUSH rule 5 x 8 [5,6,2]
+ CRUSH rule 5 x 9 [2,5,8]
+ CRUSH rule 5 x 10 [2,7,5]
+ CRUSH rule 5 x 11 [2,7,5]
+ CRUSH rule 5 x 12 [2,5,6]
+ CRUSH rule 5 x 13 [5,8,2]
+ CRUSH rule 5 x 14 [7,2,5]
+ CRUSH rule 5 x 15 [7,2,5]
+ CRUSH rule 5 x 16 [5,6,2]
+ CRUSH rule 5 x 17 [5,2,6]
+ CRUSH rule 5 x 18 [2,5,6]
+ CRUSH rule 5 x 19 [7,5,2]
+ CRUSH rule 5 x 20 [2,5,7]
+ CRUSH rule 5 x 21 [5,7,2]
+ CRUSH rule 5 x 22 [8,5,2]
+ CRUSH rule 5 x 23 [5,6,2]
+ CRUSH rule 5 x 24 [2,7,5]
+ CRUSH rule 5 x 25 [5,7,2]
+ CRUSH rule 5 x 26 [2,8,5]
+ CRUSH rule 5 x 27 [5,2,8]
+ CRUSH rule 5 x 28 [6,2,5]
+ CRUSH rule 5 x 29 [8,5,2]
+ CRUSH rule 5 x 30 [5,7,2]
+ CRUSH rule 5 x 31 [8,2,5]
+ CRUSH rule 5 x 32 [5,6,2]
+ CRUSH rule 5 x 33 [2,7,5]
+ CRUSH rule 5 x 34 [2,5,7]
+ CRUSH rule 5 x 35 [2,8,5]
+ CRUSH rule 5 x 36 [5,8,2]
+ CRUSH rule 5 x 37 [2,5,6]
+ CRUSH rule 5 x 38 [5,8,2]
+ CRUSH rule 5 x 39 [5,7,2]
+ CRUSH rule 5 x 40 [7,2,5]
+ CRUSH rule 5 x 41 [2,6,5]
+ CRUSH rule 5 x 42 [5,6,2]
+ CRUSH rule 5 x 43 [2,5,7]
+ CRUSH rule 5 x 44 [2,6,5]
+ CRUSH rule 5 x 45 [8,2,5]
+ CRUSH rule 5 x 46 [2,5,8]
+ CRUSH rule 5 x 47 [5,2,8]
+ CRUSH rule 5 x 48 [5,6,2]
+ CRUSH rule 5 x 49 [5,7,2]
+ CRUSH rule 5 x 50 [5,2,7]
+ CRUSH rule 5 x 51 [5,6,2]
+ CRUSH rule 5 x 52 [8,2,5]
+ CRUSH rule 5 x 53 [5,8,2]
+ CRUSH rule 5 x 54 [7,5,2]
+ CRUSH rule 5 x 55 [8,2,5]
+ CRUSH rule 5 x 56 [6,5,2]
+ CRUSH rule 5 x 57 [5,8,2]
+ CRUSH rule 5 x 58 [2,8,5]
+ CRUSH rule 5 x 59 [5,2,7]
+ CRUSH rule 5 x 60 [5,2,6]
+ CRUSH rule 5 x 61 [5,6,2]
+ CRUSH rule 5 x 62 [7,2,5]
+ CRUSH rule 5 x 63 [5,6,2]
+ CRUSH rule 5 x 64 [5,2,8]
+ CRUSH rule 5 x 65 [7,5,2]
+ CRUSH rule 5 x 66 [5,6,2]
+ CRUSH rule 5 x 67 [5,2,8]
+ CRUSH rule 5 x 68 [2,5,8]
+ CRUSH rule 5 x 69 [5,2,6]
+ CRUSH rule 5 x 70 [7,2,5]
+ CRUSH rule 5 x 71 [2,8,5]
+ CRUSH rule 5 x 72 [6,2,5]
+ CRUSH rule 5 x 73 [2,7,5]
+ CRUSH rule 5 x 74 [2,7,5]
+ CRUSH rule 5 x 75 [5,2,6]
+ CRUSH rule 5 x 76 [5,2,7]
+ CRUSH rule 5 x 77 [7,2,5]
+ CRUSH rule 5 x 78 [2,5,8]
+ CRUSH rule 5 x 79 [5,2,7]
+ CRUSH rule 5 x 80 [2,5,6]
+ CRUSH rule 5 x 81 [2,5,6]
+ CRUSH rule 5 x 82 [7,2,5]
+ CRUSH rule 5 x 83 [2,6,5]
+ CRUSH rule 5 x 84 [7,2,5]
+ CRUSH rule 5 x 85 [5,8,2]
+ CRUSH rule 5 x 86 [2,6,5]
+ CRUSH rule 5 x 87 [2,7,5]
+ CRUSH rule 5 x 88 [2,6,5]
+ CRUSH rule 5 x 89 [5,2,7]
+ CRUSH rule 5 x 90 [6,5,2]
+ CRUSH rule 5 x 91 [5,8,2]
+ CRUSH rule 5 x 92 [2,8,5]
+ CRUSH rule 5 x 93 [7,5,2]
+ CRUSH rule 5 x 94 [2,5,8]
+ CRUSH rule 5 x 95 [7,5,2]
+ CRUSH rule 5 x 96 [5,6,2]
+ CRUSH rule 5 x 97 [8,5,2]
+ CRUSH rule 5 x 98 [2,7,5]
+ CRUSH rule 5 x 99 [2,7,5]
+ CRUSH rule 5 x 100 [2,7,5]
+ CRUSH rule 5 x 101 [5,7,2]
+ CRUSH rule 5 x 102 [5,2,7]
+ CRUSH rule 5 x 103 [5,7,2]
+ CRUSH rule 5 x 104 [7,5,2]
+ CRUSH rule 5 x 105 [2,5,6]
+ CRUSH rule 5 x 106 [2,6,5]
+ CRUSH rule 5 x 107 [5,2,6]
+ CRUSH rule 5 x 108 [7,2,5]
+ CRUSH rule 5 x 109 [2,5,6]
+ CRUSH rule 5 x 110 [5,2,7]
+ CRUSH rule 5 x 111 [2,5,6]
+ CRUSH rule 5 x 112 [2,6,5]
+ CRUSH rule 5 x 113 [6,2,5]
+ CRUSH rule 5 x 114 [7,5,2]
+ CRUSH rule 5 x 115 [8,2,5]
+ CRUSH rule 5 x 116 [2,6,5]
+ CRUSH rule 5 x 117 [7,5,2]
+ CRUSH rule 5 x 118 [2,5,8]
+ CRUSH rule 5 x 119 [5,6,2]
+ CRUSH rule 5 x 120 [2,5,7]
+ CRUSH rule 5 x 121 [2,7,5]
+ CRUSH rule 5 x 122 [8,5,2]
+ CRUSH rule 5 x 123 [2,5,8]
+ CRUSH rule 5 x 124 [5,2,8]
+ CRUSH rule 5 x 125 [2,7,5]
+ CRUSH rule 5 x 126 [5,2,6]
+ CRUSH rule 5 x 127 [5,6,2]
+ CRUSH rule 5 x 128 [5,6,2]
+ CRUSH rule 5 x 129 [2,5,7]
+ CRUSH rule 5 x 130 [5,8,2]
+ CRUSH rule 5 x 131 [2,5,8]
+ CRUSH rule 5 x 132 [2,5,6]
+ CRUSH rule 5 x 133 [5,6,2]
+ CRUSH rule 5 x 134 [2,8,5]
+ CRUSH rule 5 x 135 [5,6,2]
+ CRUSH rule 5 x 136 [2,5,6]
+ CRUSH rule 5 x 137 [7,5,2]
+ CRUSH rule 5 x 138 [8,5,2]
+ CRUSH rule 5 x 139 [5,2,7]
+ CRUSH rule 5 x 140 [2,6,5]
+ CRUSH rule 5 x 141 [6,2,5]
+ CRUSH rule 5 x 142 [5,2,8]
+ CRUSH rule 5 x 143 [5,8,2]
+ CRUSH rule 5 x 144 [8,2,5]
+ CRUSH rule 5 x 145 [8,5,2]
+ CRUSH rule 5 x 146 [2,6,5]
+ CRUSH rule 5 x 147 [2,8,5]
+ CRUSH rule 5 x 148 [5,2,7]
+ CRUSH rule 5 x 149 [5,8,2]
+ CRUSH rule 5 x 150 [2,6,5]
+ CRUSH rule 5 x 151 [5,6,2]
+ CRUSH rule 5 x 152 [8,5,2]
+ CRUSH rule 5 x 153 [8,5,2]
+ CRUSH rule 5 x 154 [5,2,7]
+ CRUSH rule 5 x 155 [5,7,2]
+ CRUSH rule 5 x 156 [5,2,6]
+ CRUSH rule 5 x 157 [5,2,6]
+ CRUSH rule 5 x 158 [2,8,5]
+ CRUSH rule 5 x 159 [7,2,5]
+ CRUSH rule 5 x 160 [2,8,5]
+ CRUSH rule 5 x 161 [2,5,6]
+ CRUSH rule 5 x 162 [2,6,5]
+ CRUSH rule 5 x 163 [5,6,2]
+ CRUSH rule 5 x 164 [7,2,5]
+ CRUSH rule 5 x 165 [7,2,5]
+ CRUSH rule 5 x 166 [2,5,6]
+ CRUSH rule 5 x 167 [2,7,5]
+ CRUSH rule 5 x 168 [5,2,7]
+ CRUSH rule 5 x 169 [2,6,5]
+ CRUSH rule 5 x 170 [2,5,8]
+ CRUSH rule 5 x 171 [7,5,2]
+ CRUSH rule 5 x 172 [2,7,5]
+ CRUSH rule 5 x 173 [8,5,2]
+ CRUSH rule 5 x 174 [2,5,7]
+ CRUSH rule 5 x 175 [6,2,5]
+ CRUSH rule 5 x 176 [5,2,7]
+ CRUSH rule 5 x 177 [5,2,6]
+ CRUSH rule 5 x 178 [5,2,8]
+ CRUSH rule 5 x 179 [5,2,7]
+ CRUSH rule 5 x 180 [5,8,2]
+ CRUSH rule 5 x 181 [6,2,5]
+ CRUSH rule 5 x 182 [8,5,2]
+ CRUSH rule 5 x 183 [7,5,2]
+ CRUSH rule 5 x 184 [5,7,2]
+ CRUSH rule 5 x 185 [6,2,5]
+ CRUSH rule 5 x 186 [2,5,7]
+ CRUSH rule 5 x 187 [2,6,5]
+ CRUSH rule 5 x 188 [2,8,5]
+ CRUSH rule 5 x 189 [2,7,5]
+ CRUSH rule 5 x 190 [5,2,8]
+ CRUSH rule 5 x 191 [7,2,5]
+ CRUSH rule 5 x 192 [5,2,8]
+ CRUSH rule 5 x 193 [5,2,6]
+ CRUSH rule 5 x 194 [2,5,8]
+ CRUSH rule 5 x 195 [6,5,2]
+ CRUSH rule 5 x 196 [6,2,5]
+ CRUSH rule 5 x 197 [6,5,2]
+ CRUSH rule 5 x 198 [2,5,6]
+ CRUSH rule 5 x 199 [2,5,7]
+ CRUSH rule 5 x 200 [2,5,6]
+ CRUSH rule 5 x 201 [7,2,5]
+ CRUSH rule 5 x 202 [6,5,2]
+ CRUSH rule 5 x 203 [5,8,2]
+ CRUSH rule 5 x 204 [2,5,7]
+ CRUSH rule 5 x 205 [2,7,5]
+ CRUSH rule 5 x 206 [2,7,5]
+ CRUSH rule 5 x 207 [5,2,7]
+ CRUSH rule 5 x 208 [7,2,5]
+ CRUSH rule 5 x 209 [2,8,5]
+ CRUSH rule 5 x 210 [2,5,8]
+ CRUSH rule 5 x 211 [5,2,8]
+ CRUSH rule 5 x 212 [7,5,2]
+ CRUSH rule 5 x 213 [8,5,2]
+ CRUSH rule 5 x 214 [5,8,2]
+ CRUSH rule 5 x 215 [8,2,5]
+ CRUSH rule 5 x 216 [5,2,6]
+ CRUSH rule 5 x 217 [2,7,5]
+ CRUSH rule 5 x 218 [2,7,5]
+ CRUSH rule 5 x 219 [5,8,2]
+ CRUSH rule 5 x 220 [5,7,2]
+ CRUSH rule 5 x 221 [5,6,2]
+ CRUSH rule 5 x 222 [6,5,2]
+ CRUSH rule 5 x 223 [2,5,6]
+ CRUSH rule 5 x 224 [2,5,8]
+ CRUSH rule 5 x 225 [8,2,5]
+ CRUSH rule 5 x 226 [7,2,5]
+ CRUSH rule 5 x 227 [5,2,6]
+ CRUSH rule 5 x 228 [5,6,2]
+ CRUSH rule 5 x 229 [5,8,2]
+ CRUSH rule 5 x 230 [5,7,2]
+ CRUSH rule 5 x 231 [5,7,2]
+ CRUSH rule 5 x 232 [2,7,5]
+ CRUSH rule 5 x 233 [5,7,2]
+ CRUSH rule 5 x 234 [2,5,6]
+ CRUSH rule 5 x 235 [5,8,2]
+ CRUSH rule 5 x 236 [5,2,7]
+ CRUSH rule 5 x 237 [5,7,2]
+ CRUSH rule 5 x 238 [5,2,6]
+ CRUSH rule 5 x 239 [8,5,2]
+ CRUSH rule 5 x 240 [5,7,2]
+ CRUSH rule 5 x 241 [5,2,8]
+ CRUSH rule 5 x 242 [5,2,6]
+ CRUSH rule 5 x 243 [5,7,2]
+ CRUSH rule 5 x 244 [5,6,2]
+ CRUSH rule 5 x 245 [7,2,5]
+ CRUSH rule 5 x 246 [2,5,8]
+ CRUSH rule 5 x 247 [6,2,5]
+ CRUSH rule 5 x 248 [8,2,5]
+ CRUSH rule 5 x 249 [2,5,7]
+ CRUSH rule 5 x 250 [2,5,6]
+ CRUSH rule 5 x 251 [2,5,6]
+ CRUSH rule 5 x 252 [5,7,2]
+ CRUSH rule 5 x 253 [5,2,6]
+ CRUSH rule 5 x 254 [5,2,7]
+ CRUSH rule 5 x 255 [2,7,5]
+ CRUSH rule 5 x 256 [5,7,2]
+ CRUSH rule 5 x 257 [2,8,5]
+ CRUSH rule 5 x 258 [5,2,6]
+ CRUSH rule 5 x 259 [5,6,2]
+ CRUSH rule 5 x 260 [5,6,2]
+ CRUSH rule 5 x 261 [8,5,2]
+ CRUSH rule 5 x 262 [5,6,2]
+ CRUSH rule 5 x 263 [6,2,5]
+ CRUSH rule 5 x 264 [5,6,2]
+ CRUSH rule 5 x 265 [8,5,2]
+ CRUSH rule 5 x 266 [8,2,5]
+ CRUSH rule 5 x 267 [2,5,8]
+ CRUSH rule 5 x 268 [2,7,5]
+ CRUSH rule 5 x 269 [2,8,5]
+ CRUSH rule 5 x 270 [5,2,7]
+ CRUSH rule 5 x 271 [7,5,2]
+ CRUSH rule 5 x 272 [2,8,5]
+ CRUSH rule 5 x 273 [5,2,7]
+ CRUSH rule 5 x 274 [6,5,2]
+ CRUSH rule 5 x 275 [5,7,2]
+ CRUSH rule 5 x 276 [7,2,5]
+ CRUSH rule 5 x 277 [6,5,2]
+ CRUSH rule 5 x 278 [6,2,5]
+ CRUSH rule 5 x 279 [8,5,2]
+ CRUSH rule 5 x 280 [2,6,5]
+ CRUSH rule 5 x 281 [8,2,5]
+ CRUSH rule 5 x 282 [5,2,6]
+ CRUSH rule 5 x 283 [8,2,5]
+ CRUSH rule 5 x 284 [6,5,2]
+ CRUSH rule 5 x 285 [5,7,2]
+ CRUSH rule 5 x 286 [2,6,5]
+ CRUSH rule 5 x 287 [2,5,6]
+ CRUSH rule 5 x 288 [8,2,5]
+ CRUSH rule 5 x 289 [5,6,2]
+ CRUSH rule 5 x 290 [2,5,7]
+ CRUSH rule 5 x 291 [2,5,8]
+ CRUSH rule 5 x 292 [8,2,5]
+ CRUSH rule 5 x 293 [6,2,5]
+ CRUSH rule 5 x 294 [7,5,2]
+ CRUSH rule 5 x 295 [5,8,2]
+ CRUSH rule 5 x 296 [5,2,6]
+ CRUSH rule 5 x 297 [6,2,5]
+ CRUSH rule 5 x 298 [2,5,7]
+ CRUSH rule 5 x 299 [2,8,5]
+ CRUSH rule 5 x 300 [8,5,2]
+ CRUSH rule 5 x 301 [2,8,5]
+ CRUSH rule 5 x 302 [5,2,6]
+ CRUSH rule 5 x 303 [7,5,2]
+ CRUSH rule 5 x 304 [2,7,5]
+ CRUSH rule 5 x 305 [5,8,2]
+ CRUSH rule 5 x 306 [2,7,5]
+ CRUSH rule 5 x 307 [2,7,5]
+ CRUSH rule 5 x 308 [2,8,5]
+ CRUSH rule 5 x 309 [7,5,2]
+ CRUSH rule 5 x 310 [5,2,7]
+ CRUSH rule 5 x 311 [5,6,2]
+ CRUSH rule 5 x 312 [2,6,5]
+ CRUSH rule 5 x 313 [5,2,6]
+ CRUSH rule 5 x 314 [5,2,6]
+ CRUSH rule 5 x 315 [2,5,8]
+ CRUSH rule 5 x 316 [6,5,2]
+ CRUSH rule 5 x 317 [2,6,5]
+ CRUSH rule 5 x 318 [8,2,5]
+ CRUSH rule 5 x 319 [5,2,8]
+ CRUSH rule 5 x 320 [5,7,2]
+ CRUSH rule 5 x 321 [2,5,8]
+ CRUSH rule 5 x 322 [2,7,5]
+ CRUSH rule 5 x 323 [5,7,2]
+ CRUSH rule 5 x 324 [7,2,5]
+ CRUSH rule 5 x 325 [5,6,2]
+ CRUSH rule 5 x 326 [5,2,6]
+ CRUSH rule 5 x 327 [2,6,5]
+ CRUSH rule 5 x 328 [7,5,2]
+ CRUSH rule 5 x 329 [5,6,2]
+ CRUSH rule 5 x 330 [5,7,2]
+ CRUSH rule 5 x 331 [2,6,5]
+ CRUSH rule 5 x 332 [2,5,6]
+ CRUSH rule 5 x 333 [6,5,2]
+ CRUSH rule 5 x 334 [8,5,2]
+ CRUSH rule 5 x 335 [7,2,5]
+ CRUSH rule 5 x 336 [5,6,2]
+ CRUSH rule 5 x 337 [7,2,5]
+ CRUSH rule 5 x 338 [5,6,2]
+ CRUSH rule 5 x 339 [7,5,2]
+ CRUSH rule 5 x 340 [2,8,5]
+ CRUSH rule 5 x 341 [5,2,7]
+ CRUSH rule 5 x 342 [2,7,5]
+ CRUSH rule 5 x 343 [6,5,2]
+ CRUSH rule 5 x 344 [6,2,5]
+ CRUSH rule 5 x 345 [5,7,2]
+ CRUSH rule 5 x 346 [8,2,5]
+ CRUSH rule 5 x 347 [5,2,7]
+ CRUSH rule 5 x 348 [8,2,5]
+ CRUSH rule 5 x 349 [2,6,5]
+ CRUSH rule 5 x 350 [8,5,2]
+ CRUSH rule 5 x 351 [5,6,2]
+ CRUSH rule 5 x 352 [2,8,5]
+ CRUSH rule 5 x 353 [6,5,2]
+ CRUSH rule 5 x 354 [2,5,6]
+ CRUSH rule 5 x 355 [5,8,2]
+ CRUSH rule 5 x 356 [5,2,6]
+ CRUSH rule 5 x 357 [6,2,5]
+ CRUSH rule 5 x 358 [2,8,5]
+ CRUSH rule 5 x 359 [6,2,5]
+ CRUSH rule 5 x 360 [5,2,7]
+ CRUSH rule 5 x 361 [8,5,2]
+ CRUSH rule 5 x 362 [5,2,6]
+ CRUSH rule 5 x 363 [5,2,6]
+ CRUSH rule 5 x 364 [2,5,6]
+ CRUSH rule 5 x 365 [6,5,2]
+ CRUSH rule 5 x 366 [7,2,5]
+ CRUSH rule 5 x 367 [5,2,6]
+ CRUSH rule 5 x 368 [7,5,2]
+ CRUSH rule 5 x 369 [5,7,2]
+ CRUSH rule 5 x 370 [8,2,5]
+ CRUSH rule 5 x 371 [2,5,6]
+ CRUSH rule 5 x 372 [5,2,8]
+ CRUSH rule 5 x 373 [2,6,5]
+ CRUSH rule 5 x 374 [5,8,2]
+ CRUSH rule 5 x 375 [6,5,2]
+ CRUSH rule 5 x 376 [7,2,5]
+ CRUSH rule 5 x 377 [2,5,6]
+ CRUSH rule 5 x 378 [2,8,5]
+ CRUSH rule 5 x 379 [8,5,2]
+ CRUSH rule 5 x 380 [2,5,8]
+ CRUSH rule 5 x 381 [2,5,7]
+ CRUSH rule 5 x 382 [2,5,8]
+ CRUSH rule 5 x 383 [5,6,2]
+ CRUSH rule 5 x 384 [7,2,5]
+ CRUSH rule 5 x 385 [7,5,2]
+ CRUSH rule 5 x 386 [2,5,6]
+ CRUSH rule 5 x 387 [2,5,6]
+ CRUSH rule 5 x 388 [5,2,8]
+ CRUSH rule 5 x 389 [2,5,7]
+ CRUSH rule 5 x 390 [5,6,2]
+ CRUSH rule 5 x 391 [5,6,2]
+ CRUSH rule 5 x 392 [2,8,5]
+ CRUSH rule 5 x 393 [5,2,6]
+ CRUSH rule 5 x 394 [5,7,2]
+ CRUSH rule 5 x 395 [5,2,8]
+ CRUSH rule 5 x 396 [5,2,7]
+ CRUSH rule 5 x 397 [2,5,8]
+ CRUSH rule 5 x 398 [2,5,8]
+ CRUSH rule 5 x 399 [8,5,2]
+ CRUSH rule 5 x 400 [8,2,5]
+ CRUSH rule 5 x 401 [2,5,8]
+ CRUSH rule 5 x 402 [7,5,2]
+ CRUSH rule 5 x 403 [2,5,8]
+ CRUSH rule 5 x 404 [5,2,8]
+ CRUSH rule 5 x 405 [6,5,2]
+ CRUSH rule 5 x 406 [2,6,5]
+ CRUSH rule 5 x 407 [2,8,5]
+ CRUSH rule 5 x 408 [5,2,6]
+ CRUSH rule 5 x 409 [7,5,2]
+ CRUSH rule 5 x 410 [8,5,2]
+ CRUSH rule 5 x 411 [2,8,5]
+ CRUSH rule 5 x 412 [2,5,8]
+ CRUSH rule 5 x 413 [5,2,8]
+ CRUSH rule 5 x 414 [5,2,6]
+ CRUSH rule 5 x 415 [2,6,5]
+ CRUSH rule 5 x 416 [2,6,5]
+ CRUSH rule 5 x 417 [8,2,5]
+ CRUSH rule 5 x 418 [7,2,5]
+ CRUSH rule 5 x 419 [8,5,2]
+ CRUSH rule 5 x 420 [2,5,6]
+ CRUSH rule 5 x 421 [8,5,2]
+ CRUSH rule 5 x 422 [6,5,2]
+ CRUSH rule 5 x 423 [2,5,6]
+ CRUSH rule 5 x 424 [8,5,2]
+ CRUSH rule 5 x 425 [2,5,7]
+ CRUSH rule 5 x 426 [6,2,5]
+ CRUSH rule 5 x 427 [2,7,5]
+ CRUSH rule 5 x 428 [5,7,2]
+ CRUSH rule 5 x 429 [5,6,2]
+ CRUSH rule 5 x 430 [5,6,2]
+ CRUSH rule 5 x 431 [5,2,7]
+ CRUSH rule 5 x 432 [7,2,5]
+ CRUSH rule 5 x 433 [6,5,2]
+ CRUSH rule 5 x 434 [5,2,8]
+ CRUSH rule 5 x 435 [2,5,6]
+ CRUSH rule 5 x 436 [5,2,7]
+ CRUSH rule 5 x 437 [7,5,2]
+ CRUSH rule 5 x 438 [2,5,8]
+ CRUSH rule 5 x 439 [2,5,7]
+ CRUSH rule 5 x 440 [2,7,5]
+ CRUSH rule 5 x 441 [5,7,2]
+ CRUSH rule 5 x 442 [2,5,7]
+ CRUSH rule 5 x 443 [6,2,5]
+ CRUSH rule 5 x 444 [7,2,5]
+ CRUSH rule 5 x 445 [6,5,2]
+ CRUSH rule 5 x 446 [5,2,8]
+ CRUSH rule 5 x 447 [2,5,7]
+ CRUSH rule 5 x 448 [7,2,5]
+ CRUSH rule 5 x 449 [7,5,2]
+ CRUSH rule 5 x 450 [5,2,8]
+ CRUSH rule 5 x 451 [6,5,2]
+ CRUSH rule 5 x 452 [8,5,2]
+ CRUSH rule 5 x 453 [6,5,2]
+ CRUSH rule 5 x 454 [6,5,2]
+ CRUSH rule 5 x 455 [2,7,5]
+ CRUSH rule 5 x 456 [6,2,5]
+ CRUSH rule 5 x 457 [7,2,5]
+ CRUSH rule 5 x 458 [2,8,5]
+ CRUSH rule 5 x 459 [2,7,5]
+ CRUSH rule 5 x 460 [6,5,2]
+ CRUSH rule 5 x 461 [6,5,2]
+ CRUSH rule 5 x 462 [8,2,5]
+ CRUSH rule 5 x 463 [6,2,5]
+ CRUSH rule 5 x 464 [7,5,2]
+ CRUSH rule 5 x 465 [7,2,5]
+ CRUSH rule 5 x 466 [5,8,2]
+ CRUSH rule 5 x 467 [6,5,2]
+ CRUSH rule 5 x 468 [7,2,5]
+ CRUSH rule 5 x 469 [7,2,5]
+ CRUSH rule 5 x 470 [5,2,6]
+ CRUSH rule 5 x 471 [2,7,5]
+ CRUSH rule 5 x 472 [5,2,8]
+ CRUSH rule 5 x 473 [2,5,6]
+ CRUSH rule 5 x 474 [6,2,5]
+ CRUSH rule 5 x 475 [6,2,5]
+ CRUSH rule 5 x 476 [5,6,2]
+ CRUSH rule 5 x 477 [5,8,2]
+ CRUSH rule 5 x 478 [6,2,5]
+ CRUSH rule 5 x 479 [2,5,8]
+ CRUSH rule 5 x 480 [2,8,5]
+ CRUSH rule 5 x 481 [2,5,7]
+ CRUSH rule 5 x 482 [5,7,2]
+ CRUSH rule 5 x 483 [2,6,5]
+ CRUSH rule 5 x 484 [2,7,5]
+ CRUSH rule 5 x 485 [5,7,2]
+ CRUSH rule 5 x 486 [5,2,7]
+ CRUSH rule 5 x 487 [5,2,8]
+ CRUSH rule 5 x 488 [5,7,2]
+ CRUSH rule 5 x 489 [2,8,5]
+ CRUSH rule 5 x 490 [6,5,2]
+ CRUSH rule 5 x 491 [2,7,5]
+ CRUSH rule 5 x 492 [6,5,2]
+ CRUSH rule 5 x 493 [2,7,5]
+ CRUSH rule 5 x 494 [2,7,5]
+ CRUSH rule 5 x 495 [5,2,8]
+ CRUSH rule 5 x 496 [7,5,2]
+ CRUSH rule 5 x 497 [5,7,2]
+ CRUSH rule 5 x 498 [2,5,8]
+ CRUSH rule 5 x 499 [8,5,2]
+ CRUSH rule 5 x 500 [5,6,2]
+ CRUSH rule 5 x 501 [2,7,5]
+ CRUSH rule 5 x 502 [7,2,5]
+ CRUSH rule 5 x 503 [2,5,7]
+ CRUSH rule 5 x 504 [5,6,2]
+ CRUSH rule 5 x 505 [2,7,5]
+ CRUSH rule 5 x 506 [5,2,8]
+ CRUSH rule 5 x 507 [6,2,5]
+ CRUSH rule 5 x 508 [2,5,8]
+ CRUSH rule 5 x 509 [7,5,2]
+ CRUSH rule 5 x 510 [6,2,5]
+ CRUSH rule 5 x 511 [5,8,2]
+ CRUSH rule 5 x 512 [7,2,5]
+ CRUSH rule 5 x 513 [7,2,5]
+ CRUSH rule 5 x 514 [5,6,2]
+ CRUSH rule 5 x 515 [8,5,2]
+ CRUSH rule 5 x 516 [5,2,8]
+ CRUSH rule 5 x 517 [7,2,5]
+ CRUSH rule 5 x 518 [5,6,2]
+ CRUSH rule 5 x 519 [7,5,2]
+ CRUSH rule 5 x 520 [2,6,5]
+ CRUSH rule 5 x 521 [8,2,5]
+ CRUSH rule 5 x 522 [6,2,5]
+ CRUSH rule 5 x 523 [5,2,7]
+ CRUSH rule 5 x 524 [2,5,8]
+ CRUSH rule 5 x 525 [2,5,6]
+ CRUSH rule 5 x 526 [2,5,8]
+ CRUSH rule 5 x 527 [2,5,6]
+ CRUSH rule 5 x 528 [5,2,6]
+ CRUSH rule 5 x 529 [5,7,2]
+ CRUSH rule 5 x 530 [6,5,2]
+ CRUSH rule 5 x 531 [6,2,5]
+ CRUSH rule 5 x 532 [6,5,2]
+ CRUSH rule 5 x 533 [5,6,2]
+ CRUSH rule 5 x 534 [7,5,2]
+ CRUSH rule 5 x 535 [8,2,5]
+ CRUSH rule 5 x 536 [6,2,5]
+ CRUSH rule 5 x 537 [5,7,2]
+ CRUSH rule 5 x 538 [6,5,2]
+ CRUSH rule 5 x 539 [8,5,2]
+ CRUSH rule 5 x 540 [2,6,5]
+ CRUSH rule 5 x 541 [2,5,8]
+ CRUSH rule 5 x 542 [5,2,8]
+ CRUSH rule 5 x 543 [6,2,5]
+ CRUSH rule 5 x 544 [5,7,2]
+ CRUSH rule 5 x 545 [5,7,2]
+ CRUSH rule 5 x 546 [6,2,5]
+ CRUSH rule 5 x 547 [8,2,5]
+ CRUSH rule 5 x 548 [5,2,8]
+ CRUSH rule 5 x 549 [5,8,2]
+ CRUSH rule 5 x 550 [2,5,7]
+ CRUSH rule 5 x 551 [7,5,2]
+ CRUSH rule 5 x 552 [5,8,2]
+ CRUSH rule 5 x 553 [5,2,7]
+ CRUSH rule 5 x 554 [2,8,5]
+ CRUSH rule 5 x 555 [5,2,8]
+ CRUSH rule 5 x 556 [5,6,2]
+ CRUSH rule 5 x 557 [7,5,2]
+ CRUSH rule 5 x 558 [5,2,6]
+ CRUSH rule 5 x 559 [5,2,6]
+ CRUSH rule 5 x 560 [8,5,2]
+ CRUSH rule 5 x 561 [6,5,2]
+ CRUSH rule 5 x 562 [5,2,6]
+ CRUSH rule 5 x 563 [2,6,5]
+ CRUSH rule 5 x 564 [5,2,7]
+ CRUSH rule 5 x 565 [5,6,2]
+ CRUSH rule 5 x 566 [5,7,2]
+ CRUSH rule 5 x 567 [5,6,2]
+ CRUSH rule 5 x 568 [7,5,2]
+ CRUSH rule 5 x 569 [5,2,7]
+ CRUSH rule 5 x 570 [2,5,8]
+ CRUSH rule 5 x 571 [5,7,2]
+ CRUSH rule 5 x 572 [5,2,8]
+ CRUSH rule 5 x 573 [5,2,7]
+ CRUSH rule 5 x 574 [2,5,8]
+ CRUSH rule 5 x 575 [8,2,5]
+ CRUSH rule 5 x 576 [5,6,2]
+ CRUSH rule 5 x 577 [8,2,5]
+ CRUSH rule 5 x 578 [6,2,5]
+ CRUSH rule 5 x 579 [5,2,6]
+ CRUSH rule 5 x 580 [5,2,7]
+ CRUSH rule 5 x 581 [7,2,5]
+ CRUSH rule 5 x 582 [2,8,5]
+ CRUSH rule 5 x 583 [6,2,5]
+ CRUSH rule 5 x 584 [8,2,5]
+ CRUSH rule 5 x 585 [7,2,5]
+ CRUSH rule 5 x 586 [2,7,5]
+ CRUSH rule 5 x 587 [2,5,7]
+ CRUSH rule 5 x 588 [5,7,2]
+ CRUSH rule 5 x 589 [7,2,5]
+ CRUSH rule 5 x 590 [6,2,5]
+ CRUSH rule 5 x 591 [5,2,8]
+ CRUSH rule 5 x 592 [2,5,7]
+ CRUSH rule 5 x 593 [2,8,5]
+ CRUSH rule 5 x 594 [2,7,5]
+ CRUSH rule 5 x 595 [7,2,5]
+ CRUSH rule 5 x 596 [5,2,6]
+ CRUSH rule 5 x 597 [5,2,7]
+ CRUSH rule 5 x 598 [5,2,6]
+ CRUSH rule 5 x 599 [5,2,8]
+ CRUSH rule 5 x 600 [7,2,5]
+ CRUSH rule 5 x 601 [2,7,5]
+ CRUSH rule 5 x 602 [5,7,2]
+ CRUSH rule 5 x 603 [5,2,6]
+ CRUSH rule 5 x 604 [7,5,2]
+ CRUSH rule 5 x 605 [5,2,7]
+ CRUSH rule 5 x 606 [2,7,5]
+ CRUSH rule 5 x 607 [2,5,8]
+ CRUSH rule 5 x 608 [5,2,7]
+ CRUSH rule 5 x 609 [5,2,8]
+ CRUSH rule 5 x 610 [5,7,2]
+ CRUSH rule 5 x 611 [2,8,5]
+ CRUSH rule 5 x 612 [2,6,5]
+ CRUSH rule 5 x 613 [7,2,5]
+ CRUSH rule 5 x 614 [7,2,5]
+ CRUSH rule 5 x 615 [6,2,5]
+ CRUSH rule 5 x 616 [2,8,5]
+ CRUSH rule 5 x 617 [6,2,5]
+ CRUSH rule 5 x 618 [7,5,2]
+ CRUSH rule 5 x 619 [5,2,8]
+ CRUSH rule 5 x 620 [5,2,6]
+ CRUSH rule 5 x 621 [5,8,2]
+ CRUSH rule 5 x 622 [2,5,8]
+ CRUSH rule 5 x 623 [2,6,5]
+ CRUSH rule 5 x 624 [5,2,8]
+ CRUSH rule 5 x 625 [2,5,7]
+ CRUSH rule 5 x 626 [7,2,5]
+ CRUSH rule 5 x 627 [2,7,5]
+ CRUSH rule 5 x 628 [8,2,5]
+ CRUSH rule 5 x 629 [2,6,5]
+ CRUSH rule 5 x 630 [2,7,5]
+ CRUSH rule 5 x 631 [2,7,5]
+ CRUSH rule 5 x 632 [7,2,5]
+ CRUSH rule 5 x 633 [8,5,2]
+ CRUSH rule 5 x 634 [2,5,7]
+ CRUSH rule 5 x 635 [5,6,2]
+ CRUSH rule 5 x 636 [2,5,8]
+ CRUSH rule 5 x 637 [5,2,7]
+ CRUSH rule 5 x 638 [6,2,5]
+ CRUSH rule 5 x 639 [5,2,8]
+ CRUSH rule 5 x 640 [5,2,8]
+ CRUSH rule 5 x 641 [7,2,5]
+ CRUSH rule 5 x 642 [2,7,5]
+ CRUSH rule 5 x 643 [5,2,8]
+ CRUSH rule 5 x 644 [8,2,5]
+ CRUSH rule 5 x 645 [5,7,2]
+ CRUSH rule 5 x 646 [8,2,5]
+ CRUSH rule 5 x 647 [7,2,5]
+ CRUSH rule 5 x 648 [2,6,5]
+ CRUSH rule 5 x 649 [5,7,2]
+ CRUSH rule 5 x 650 [7,5,2]
+ CRUSH rule 5 x 651 [5,7,2]
+ CRUSH rule 5 x 652 [5,7,2]
+ CRUSH rule 5 x 653 [8,5,2]
+ CRUSH rule 5 x 654 [7,5,2]
+ CRUSH rule 5 x 655 [2,5,7]
+ CRUSH rule 5 x 656 [5,7,2]
+ CRUSH rule 5 x 657 [6,2,5]
+ CRUSH rule 5 x 658 [5,6,2]
+ CRUSH rule 5 x 659 [5,6,2]
+ CRUSH rule 5 x 660 [7,5,2]
+ CRUSH rule 5 x 661 [2,8,5]
+ CRUSH rule 5 x 662 [5,2,7]
+ CRUSH rule 5 x 663 [2,5,8]
+ CRUSH rule 5 x 664 [2,5,7]
+ CRUSH rule 5 x 665 [5,7,2]
+ CRUSH rule 5 x 666 [2,8,5]
+ CRUSH rule 5 x 667 [2,5,7]
+ CRUSH rule 5 x 668 [5,7,2]
+ CRUSH rule 5 x 669 [6,5,2]
+ CRUSH rule 5 x 670 [5,2,6]
+ CRUSH rule 5 x 671 [2,7,5]
+ CRUSH rule 5 x 672 [5,2,7]
+ CRUSH rule 5 x 673 [5,2,7]
+ CRUSH rule 5 x 674 [5,2,8]
+ CRUSH rule 5 x 675 [2,8,5]
+ CRUSH rule 5 x 676 [2,5,8]
+ CRUSH rule 5 x 677 [5,2,7]
+ CRUSH rule 5 x 678 [2,5,8]
+ CRUSH rule 5 x 679 [6,2,5]
+ CRUSH rule 5 x 680 [2,5,6]
+ CRUSH rule 5 x 681 [5,7,2]
+ CRUSH rule 5 x 682 [2,5,7]
+ CRUSH rule 5 x 683 [2,5,6]
+ CRUSH rule 5 x 684 [7,2,5]
+ CRUSH rule 5 x 685 [7,2,5]
+ CRUSH rule 5 x 686 [2,5,8]
+ CRUSH rule 5 x 687 [5,6,2]
+ CRUSH rule 5 x 688 [5,7,2]
+ CRUSH rule 5 x 689 [6,5,2]
+ CRUSH rule 5 x 690 [8,2,5]
+ CRUSH rule 5 x 691 [5,2,6]
+ CRUSH rule 5 x 692 [7,2,5]
+ CRUSH rule 5 x 693 [6,5,2]
+ CRUSH rule 5 x 694 [6,5,2]
+ CRUSH rule 5 x 695 [2,8,5]
+ CRUSH rule 5 x 696 [2,5,8]
+ CRUSH rule 5 x 697 [6,2,5]
+ CRUSH rule 5 x 698 [6,2,5]
+ CRUSH rule 5 x 699 [2,6,5]
+ CRUSH rule 5 x 700 [2,5,7]
+ CRUSH rule 5 x 701 [5,2,7]
+ CRUSH rule 5 x 702 [5,2,8]
+ CRUSH rule 5 x 703 [8,5,2]
+ CRUSH rule 5 x 704 [2,5,8]
+ CRUSH rule 5 x 705 [8,2,5]
+ CRUSH rule 5 x 706 [2,5,6]
+ CRUSH rule 5 x 707 [7,5,2]
+ CRUSH rule 5 x 708 [5,7,2]
+ CRUSH rule 5 x 709 [6,5,2]
+ CRUSH rule 5 x 710 [8,5,2]
+ CRUSH rule 5 x 711 [2,5,8]
+ CRUSH rule 5 x 712 [2,5,7]
+ CRUSH rule 5 x 713 [6,5,2]
+ CRUSH rule 5 x 714 [5,2,7]
+ CRUSH rule 5 x 715 [2,5,6]
+ CRUSH rule 5 x 716 [5,6,2]
+ CRUSH rule 5 x 717 [8,2,5]
+ CRUSH rule 5 x 718 [5,7,2]
+ CRUSH rule 5 x 719 [2,6,5]
+ CRUSH rule 5 x 720 [6,2,5]
+ CRUSH rule 5 x 721 [5,7,2]
+ CRUSH rule 5 x 722 [5,7,2]
+ CRUSH rule 5 x 723 [5,2,7]
+ CRUSH rule 5 x 724 [2,6,5]
+ CRUSH rule 5 x 725 [2,5,7]
+ CRUSH rule 5 x 726 [5,8,2]
+ CRUSH rule 5 x 727 [5,6,2]
+ CRUSH rule 5 x 728 [2,7,5]
+ CRUSH rule 5 x 729 [5,6,2]
+ CRUSH rule 5 x 730 [5,7,2]
+ CRUSH rule 5 x 731 [5,2,8]
+ CRUSH rule 5 x 732 [2,5,6]
+ CRUSH rule 5 x 733 [5,7,2]
+ CRUSH rule 5 x 734 [6,5,2]
+ CRUSH rule 5 x 735 [5,8,2]
+ CRUSH rule 5 x 736 [5,8,2]
+ CRUSH rule 5 x 737 [2,6,5]
+ CRUSH rule 5 x 738 [5,2,7]
+ CRUSH rule 5 x 739 [2,7,5]
+ CRUSH rule 5 x 740 [2,8,5]
+ CRUSH rule 5 x 741 [7,2,5]
+ CRUSH rule 5 x 742 [8,2,5]
+ CRUSH rule 5 x 743 [7,2,5]
+ CRUSH rule 5 x 744 [5,7,2]
+ CRUSH rule 5 x 745 [5,2,8]
+ CRUSH rule 5 x 746 [5,2,7]
+ CRUSH rule 5 x 747 [6,2,5]
+ CRUSH rule 5 x 748 [2,7,5]
+ CRUSH rule 5 x 749 [5,8,2]
+ CRUSH rule 5 x 750 [2,6,5]
+ CRUSH rule 5 x 751 [2,8,5]
+ CRUSH rule 5 x 752 [8,2,5]
+ CRUSH rule 5 x 753 [7,5,2]
+ CRUSH rule 5 x 754 [8,5,2]
+ CRUSH rule 5 x 755 [2,6,5]
+ CRUSH rule 5 x 756 [5,6,2]
+ CRUSH rule 5 x 757 [8,2,5]
+ CRUSH rule 5 x 758 [6,2,5]
+ CRUSH rule 5 x 759 [8,5,2]
+ CRUSH rule 5 x 760 [2,5,7]
+ CRUSH rule 5 x 761 [5,2,8]
+ CRUSH rule 5 x 762 [2,7,5]
+ CRUSH rule 5 x 763 [8,5,2]
+ CRUSH rule 5 x 764 [2,7,5]
+ CRUSH rule 5 x 765 [6,5,2]
+ CRUSH rule 5 x 766 [8,5,2]
+ CRUSH rule 5 x 767 [2,8,5]
+ CRUSH rule 5 x 768 [8,5,2]
+ CRUSH rule 5 x 769 [6,2,5]
+ CRUSH rule 5 x 770 [6,2,5]
+ CRUSH rule 5 x 771 [7,2,5]
+ CRUSH rule 5 x 772 [8,5,2]
+ CRUSH rule 5 x 773 [5,2,7]
+ CRUSH rule 5 x 774 [5,6,2]
+ CRUSH rule 5 x 775 [6,5,2]
+ CRUSH rule 5 x 776 [7,2,5]
+ CRUSH rule 5 x 777 [5,2,6]
+ CRUSH rule 5 x 778 [2,8,5]
+ CRUSH rule 5 x 779 [2,7,5]
+ CRUSH rule 5 x 780 [2,5,7]
+ CRUSH rule 5 x 781 [6,5,2]
+ CRUSH rule 5 x 782 [5,2,8]
+ CRUSH rule 5 x 783 [7,2,5]
+ CRUSH rule 5 x 784 [2,5,6]
+ CRUSH rule 5 x 785 [6,2,5]
+ CRUSH rule 5 x 786 [7,5,2]
+ CRUSH rule 5 x 787 [2,6,5]
+ CRUSH rule 5 x 788 [6,2,5]
+ CRUSH rule 5 x 789 [2,5,8]
+ CRUSH rule 5 x 790 [8,5,2]
+ CRUSH rule 5 x 791 [5,8,2]
+ CRUSH rule 5 x 792 [5,8,2]
+ CRUSH rule 5 x 793 [6,2,5]
+ CRUSH rule 5 x 794 [2,6,5]
+ CRUSH rule 5 x 795 [2,5,8]
+ CRUSH rule 5 x 796 [5,7,2]
+ CRUSH rule 5 x 797 [2,5,8]
+ CRUSH rule 5 x 798 [6,2,5]
+ CRUSH rule 5 x 799 [5,2,8]
+ CRUSH rule 5 x 800 [5,2,7]
+ CRUSH rule 5 x 801 [5,6,2]
+ CRUSH rule 5 x 802 [2,8,5]
+ CRUSH rule 5 x 803 [2,5,7]
+ CRUSH rule 5 x 804 [6,2,5]
+ CRUSH rule 5 x 805 [5,6,2]
+ CRUSH rule 5 x 806 [2,5,7]
+ CRUSH rule 5 x 807 [5,7,2]
+ CRUSH rule 5 x 808 [5,6,2]
+ CRUSH rule 5 x 809 [2,5,8]
+ CRUSH rule 5 x 810 [5,7,2]
+ CRUSH rule 5 x 811 [8,5,2]
+ CRUSH rule 5 x 812 [8,5,2]
+ CRUSH rule 5 x 813 [6,5,2]
+ CRUSH rule 5 x 814 [5,6,2]
+ CRUSH rule 5 x 815 [5,2,8]
+ CRUSH rule 5 x 816 [2,7,5]
+ CRUSH rule 5 x 817 [5,8,2]
+ CRUSH rule 5 x 818 [5,2,7]
+ CRUSH rule 5 x 819 [5,2,8]
+ CRUSH rule 5 x 820 [5,6,2]
+ CRUSH rule 5 x 821 [5,6,2]
+ CRUSH rule 5 x 822 [2,5,8]
+ CRUSH rule 5 x 823 [5,8,2]
+ CRUSH rule 5 x 824 [5,7,2]
+ CRUSH rule 5 x 825 [2,8,5]
+ CRUSH rule 5 x 826 [7,2,5]
+ CRUSH rule 5 x 827 [2,8,5]
+ CRUSH rule 5 x 828 [2,5,8]
+ CRUSH rule 5 x 829 [5,6,2]
+ CRUSH rule 5 x 830 [2,5,8]
+ CRUSH rule 5 x 831 [2,6,5]
+ CRUSH rule 5 x 832 [5,7,2]
+ CRUSH rule 5 x 833 [2,7,5]
+ CRUSH rule 5 x 834 [5,2,7]
+ CRUSH rule 5 x 835 [8,5,2]
+ CRUSH rule 5 x 836 [5,7,2]
+ CRUSH rule 5 x 837 [6,5,2]
+ CRUSH rule 5 x 838 [6,2,5]
+ CRUSH rule 5 x 839 [5,2,6]
+ CRUSH rule 5 x 840 [7,5,2]
+ CRUSH rule 5 x 841 [5,8,2]
+ CRUSH rule 5 x 842 [2,5,6]
+ CRUSH rule 5 x 843 [6,5,2]
+ CRUSH rule 5 x 844 [5,8,2]
+ CRUSH rule 5 x 845 [5,8,2]
+ CRUSH rule 5 x 846 [5,2,7]
+ CRUSH rule 5 x 847 [2,8,5]
+ CRUSH rule 5 x 848 [2,6,5]
+ CRUSH rule 5 x 849 [5,6,2]
+ CRUSH rule 5 x 850 [2,5,6]
+ CRUSH rule 5 x 851 [6,5,2]
+ CRUSH rule 5 x 852 [7,5,2]
+ CRUSH rule 5 x 853 [6,2,5]
+ CRUSH rule 5 x 854 [7,2,5]
+ CRUSH rule 5 x 855 [5,7,2]
+ CRUSH rule 5 x 856 [6,5,2]
+ CRUSH rule 5 x 857 [8,5,2]
+ CRUSH rule 5 x 858 [6,5,2]
+ CRUSH rule 5 x 859 [6,2,5]
+ CRUSH rule 5 x 860 [5,2,7]
+ CRUSH rule 5 x 861 [8,5,2]
+ CRUSH rule 5 x 862 [6,2,5]
+ CRUSH rule 5 x 863 [8,2,5]
+ CRUSH rule 5 x 864 [5,6,2]
+ CRUSH rule 5 x 865 [8,2,5]
+ CRUSH rule 5 x 866 [5,6,2]
+ CRUSH rule 5 x 867 [6,5,2]
+ CRUSH rule 5 x 868 [6,5,2]
+ CRUSH rule 5 x 869 [8,5,2]
+ CRUSH rule 5 x 870 [2,5,8]
+ CRUSH rule 5 x 871 [5,2,8]
+ CRUSH rule 5 x 872 [5,2,8]
+ CRUSH rule 5 x 873 [5,6,2]
+ CRUSH rule 5 x 874 [2,6,5]
+ CRUSH rule 5 x 875 [2,6,5]
+ CRUSH rule 5 x 876 [5,8,2]
+ CRUSH rule 5 x 877 [6,5,2]
+ CRUSH rule 5 x 878 [5,2,7]
+ CRUSH rule 5 x 879 [7,5,2]
+ CRUSH rule 5 x 880 [5,2,8]
+ CRUSH rule 5 x 881 [5,6,2]
+ CRUSH rule 5 x 882 [5,2,7]
+ CRUSH rule 5 x 883 [2,5,7]
+ CRUSH rule 5 x 884 [6,2,5]
+ CRUSH rule 5 x 885 [5,2,8]
+ CRUSH rule 5 x 886 [5,6,2]
+ CRUSH rule 5 x 887 [7,5,2]
+ CRUSH rule 5 x 888 [6,2,5]
+ CRUSH rule 5 x 889 [2,6,5]
+ CRUSH rule 5 x 890 [7,2,5]
+ CRUSH rule 5 x 891 [2,8,5]
+ CRUSH rule 5 x 892 [6,2,5]
+ CRUSH rule 5 x 893 [2,5,7]
+ CRUSH rule 5 x 894 [7,5,2]
+ CRUSH rule 5 x 895 [5,2,8]
+ CRUSH rule 5 x 896 [2,8,5]
+ CRUSH rule 5 x 897 [5,2,6]
+ CRUSH rule 5 x 898 [2,5,7]
+ CRUSH rule 5 x 899 [2,7,5]
+ CRUSH rule 5 x 900 [5,2,6]
+ CRUSH rule 5 x 901 [5,2,8]
+ CRUSH rule 5 x 902 [8,5,2]
+ CRUSH rule 5 x 903 [5,7,2]
+ CRUSH rule 5 x 904 [5,6,2]
+ CRUSH rule 5 x 905 [6,2,5]
+ CRUSH rule 5 x 906 [2,6,5]
+ CRUSH rule 5 x 907 [7,2,5]
+ CRUSH rule 5 x 908 [5,8,2]
+ CRUSH rule 5 x 909 [2,5,7]
+ CRUSH rule 5 x 910 [6,5,2]
+ CRUSH rule 5 x 911 [5,8,2]
+ CRUSH rule 5 x 912 [2,7,5]
+ CRUSH rule 5 x 913 [7,2,5]
+ CRUSH rule 5 x 914 [6,5,2]
+ CRUSH rule 5 x 915 [8,2,5]
+ CRUSH rule 5 x 916 [5,2,8]
+ CRUSH rule 5 x 917 [2,5,8]
+ CRUSH rule 5 x 918 [8,2,5]
+ CRUSH rule 5 x 919 [6,2,5]
+ CRUSH rule 5 x 920 [7,5,2]
+ CRUSH rule 5 x 921 [2,5,6]
+ CRUSH rule 5 x 922 [6,5,2]
+ CRUSH rule 5 x 923 [5,8,2]
+ CRUSH rule 5 x 924 [5,2,7]
+ CRUSH rule 5 x 925 [5,7,2]
+ CRUSH rule 5 x 926 [5,2,8]
+ CRUSH rule 5 x 927 [2,6,5]
+ CRUSH rule 5 x 928 [8,2,5]
+ CRUSH rule 5 x 929 [5,2,7]
+ CRUSH rule 5 x 930 [2,5,6]
+ CRUSH rule 5 x 931 [5,2,7]
+ CRUSH rule 5 x 932 [5,2,8]
+ CRUSH rule 5 x 933 [8,5,2]
+ CRUSH rule 5 x 934 [5,6,2]
+ CRUSH rule 5 x 935 [6,5,2]
+ CRUSH rule 5 x 936 [2,6,5]
+ CRUSH rule 5 x 937 [5,8,2]
+ CRUSH rule 5 x 938 [6,5,2]
+ CRUSH rule 5 x 939 [2,7,5]
+ CRUSH rule 5 x 940 [8,5,2]
+ CRUSH rule 5 x 941 [5,2,8]
+ CRUSH rule 5 x 942 [2,8,5]
+ CRUSH rule 5 x 943 [8,2,5]
+ CRUSH rule 5 x 944 [5,8,2]
+ CRUSH rule 5 x 945 [7,2,5]
+ CRUSH rule 5 x 946 [2,8,5]
+ CRUSH rule 5 x 947 [5,2,8]
+ CRUSH rule 5 x 948 [7,5,2]
+ CRUSH rule 5 x 949 [6,2,5]
+ CRUSH rule 5 x 950 [5,6,2]
+ CRUSH rule 5 x 951 [5,8,2]
+ CRUSH rule 5 x 952 [2,7,5]
+ CRUSH rule 5 x 953 [2,5,6]
+ CRUSH rule 5 x 954 [5,2,7]
+ CRUSH rule 5 x 955 [8,2,5]
+ CRUSH rule 5 x 956 [2,6,5]
+ CRUSH rule 5 x 957 [7,2,5]
+ CRUSH rule 5 x 958 [8,5,2]
+ CRUSH rule 5 x 959 [5,2,7]
+ CRUSH rule 5 x 960 [5,6,2]
+ CRUSH rule 5 x 961 [5,2,8]
+ CRUSH rule 5 x 962 [7,5,2]
+ CRUSH rule 5 x 963 [2,5,6]
+ CRUSH rule 5 x 964 [5,2,8]
+ CRUSH rule 5 x 965 [7,5,2]
+ CRUSH rule 5 x 966 [5,8,2]
+ CRUSH rule 5 x 967 [8,5,2]
+ CRUSH rule 5 x 968 [7,2,5]
+ CRUSH rule 5 x 969 [8,2,5]
+ CRUSH rule 5 x 970 [2,6,5]
+ CRUSH rule 5 x 971 [2,7,5]
+ CRUSH rule 5 x 972 [2,8,5]
+ CRUSH rule 5 x 973 [2,6,5]
+ CRUSH rule 5 x 974 [5,2,8]
+ CRUSH rule 5 x 975 [5,7,2]
+ CRUSH rule 5 x 976 [5,8,2]
+ CRUSH rule 5 x 977 [8,5,2]
+ CRUSH rule 5 x 978 [7,2,5]
+ CRUSH rule 5 x 979 [7,2,5]
+ CRUSH rule 5 x 980 [6,2,5]
+ CRUSH rule 5 x 981 [7,5,2]
+ CRUSH rule 5 x 982 [5,2,8]
+ CRUSH rule 5 x 983 [5,7,2]
+ CRUSH rule 5 x 984 [2,7,5]
+ CRUSH rule 5 x 985 [2,5,7]
+ CRUSH rule 5 x 986 [8,5,2]
+ CRUSH rule 5 x 987 [2,5,8]
+ CRUSH rule 5 x 988 [2,5,7]
+ CRUSH rule 5 x 989 [2,6,5]
+ CRUSH rule 5 x 990 [2,6,5]
+ CRUSH rule 5 x 991 [2,5,8]
+ CRUSH rule 5 x 992 [7,2,5]
+ CRUSH rule 5 x 993 [2,6,5]
+ CRUSH rule 5 x 994 [5,7,2]
+ CRUSH rule 5 x 995 [7,2,5]
+ CRUSH rule 5 x 996 [6,5,2]
+ CRUSH rule 5 x 997 [6,5,2]
+ CRUSH rule 5 x 998 [8,2,5]
+ CRUSH rule 5 x 999 [2,7,5]
+ CRUSH rule 5 x 1000 [8,5,2]
+ CRUSH rule 5 x 1001 [2,5,6]
+ CRUSH rule 5 x 1002 [2,5,7]
+ CRUSH rule 5 x 1003 [2,8,5]
+ CRUSH rule 5 x 1004 [6,2,5]
+ CRUSH rule 5 x 1005 [6,2,5]
+ CRUSH rule 5 x 1006 [2,6,5]
+ CRUSH rule 5 x 1007 [2,5,7]
+ CRUSH rule 5 x 1008 [2,7,5]
+ CRUSH rule 5 x 1009 [6,5,2]
+ CRUSH rule 5 x 1010 [5,2,7]
+ CRUSH rule 5 x 1011 [5,2,8]
+ CRUSH rule 5 x 1012 [5,2,7]
+ CRUSH rule 5 x 1013 [5,2,7]
+ CRUSH rule 5 x 1014 [2,8,5]
+ CRUSH rule 5 x 1015 [6,5,2]
+ CRUSH rule 5 x 1016 [2,5,7]
+ CRUSH rule 5 x 1017 [6,2,5]
+ CRUSH rule 5 x 1018 [5,2,6]
+ CRUSH rule 5 x 1019 [5,8,2]
+ CRUSH rule 5 x 1020 [5,2,7]
+ CRUSH rule 5 x 1021 [5,2,6]
+ CRUSH rule 5 x 1022 [2,6,5]
+ CRUSH rule 5 x 1023 [5,2,8]
+ rule 5 (chooseleaf-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+ $ crushtool -i set-choose.crushmap --test --show-statistics --weight 0 0 --weight 3 0 --weight 4 .5 --weight 5 0 --weight 6 .1 --weight 7 0
+ rule 0 (choose), x = 0..1023, numrep = 2..3
+ CRUSH rule 0 x 0 [2,4]
+ CRUSH rule 0 x 1 [2,8]
+ CRUSH rule 0 x 2 [1]
+ CRUSH rule 0 x 3 [8,1]
+ CRUSH rule 0 x 4 [4,1]
+ CRUSH rule 0 x 5 [8,1]
+ CRUSH rule 0 x 6 [2,8]
+ CRUSH rule 0 x 7 [4,8]
+ CRUSH rule 0 x 8 [4,8]
+ CRUSH rule 0 x 9 [2,4]
+ CRUSH rule 0 x 10 [2,8]
+ CRUSH rule 0 x 11 [2,8]
+ CRUSH rule 0 x 12 [2]
+ CRUSH rule 0 x 13 [4,8]
+ CRUSH rule 0 x 14 [8,1]
+ CRUSH rule 0 x 15 [8,1]
+ CRUSH rule 0 x 16 [8]
+ CRUSH rule 0 x 17 [4,1]
+ CRUSH rule 0 x 18 [1]
+ CRUSH rule 0 x 19 [8,4]
+ CRUSH rule 0 x 20 [2]
+ CRUSH rule 0 x 21 [8]
+ CRUSH rule 0 x 22 [8]
+ CRUSH rule 0 x 23 [4,8]
+ CRUSH rule 0 x 24 [1,8]
+ CRUSH rule 0 x 25 [4,8]
+ CRUSH rule 0 x 26 [2,8]
+ CRUSH rule 0 x 27 [4,1]
+ CRUSH rule 0 x 28 [8,2]
+ CRUSH rule 0 x 29 [8,4]
+ CRUSH rule 0 x 30 [4,8]
+ CRUSH rule 0 x 31 [8,2]
+ CRUSH rule 0 x 32 [6]
+ CRUSH rule 0 x 33 [2,8]
+ CRUSH rule 0 x 34 [2]
+ CRUSH rule 0 x 35 [1,8]
+ CRUSH rule 0 x 36 [8]
+ CRUSH rule 0 x 37 [1]
+ CRUSH rule 0 x 38 [4,8]
+ CRUSH rule 0 x 39 [8]
+ CRUSH rule 0 x 40 [8,2]
+ CRUSH rule 0 x 41 [2,8]
+ CRUSH rule 0 x 42 [8]
+ CRUSH rule 0 x 43 [1]
+ CRUSH rule 0 x 44 [1,8]
+ CRUSH rule 0 x 45 [8,2]
+ CRUSH rule 0 x 46 [2]
+ CRUSH rule 0 x 47 [4,1]
+ CRUSH rule 0 x 48 [8]
+ CRUSH rule 0 x 49 [8]
+ CRUSH rule 0 x 50 [4,1]
+ CRUSH rule 0 x 51 [8]
+ CRUSH rule 0 x 52 [8,2]
+ CRUSH rule 0 x 53 [4,8]
+ CRUSH rule 0 x 54 [8,4]
+ CRUSH rule 0 x 55 [8,2]
+ CRUSH rule 0 x 56 [8,4]
+ CRUSH rule 0 x 57 [8]
+ CRUSH rule 0 x 58 [1,8]
+ CRUSH rule 0 x 59 [2]
+ CRUSH rule 0 x 60 [4,1]
+ CRUSH rule 0 x 61 [4,8]
+ CRUSH rule 0 x 62 [8,1]
+ CRUSH rule 0 x 63 [8]
+ CRUSH rule 0 x 64 [4,1]
+ CRUSH rule 0 x 65 [8,4]
+ CRUSH rule 0 x 66 [4,8]
+ CRUSH rule 0 x 67 [4,2]
+ CRUSH rule 0 x 68 [1]
+ CRUSH rule 0 x 69 [2]
+ CRUSH rule 0 x 70 [8,2]
+ CRUSH rule 0 x 71 [2,8]
+ CRUSH rule 0 x 72 [8,2]
+ CRUSH rule 0 x 73 [2,8]
+ CRUSH rule 0 x 74 [1,8]
+ CRUSH rule 0 x 75 [4,2]
+ CRUSH rule 0 x 76 [4,2]
+ CRUSH rule 0 x 77 [8,2]
+ CRUSH rule 0 x 78 [1]
+ CRUSH rule 0 x 79 [4,1]
+ CRUSH rule 0 x 80 [2,4]
+ CRUSH rule 0 x 81 [2]
+ CRUSH rule 0 x 82 [6,2]
+ CRUSH rule 0 x 83 [2,8]
+ CRUSH rule 0 x 84 [8,2]
+ CRUSH rule 0 x 85 [4,8]
+ CRUSH rule 0 x 86 [2,8]
+ CRUSH rule 0 x 87 [2,8]
+ CRUSH rule 0 x 88 [1,8]
+ CRUSH rule 0 x 89 [1]
+ CRUSH rule 0 x 90 [8,4]
+ CRUSH rule 0 x 91 [4,8]
+ CRUSH rule 0 x 92 [1,8]
+ CRUSH rule 0 x 93 [8,4]
+ CRUSH rule 0 x 94 [1]
+ CRUSH rule 0 x 95 [8]
+ CRUSH rule 0 x 96 [8]
+ CRUSH rule 0 x 97 [8]
+ CRUSH rule 0 x 98 [2,8]
+ CRUSH rule 0 x 99 [2,8]
+ CRUSH rule 0 x 100 [1,8]
+ CRUSH rule 0 x 101 [8]
+ CRUSH rule 0 x 102 [2]
+ CRUSH rule 0 x 103 [8]
+ CRUSH rule 0 x 104 [8,4]
+ CRUSH rule 0 x 105 [2,4]
+ CRUSH rule 0 x 106 [1,8]
+ CRUSH rule 0 x 107 [1]
+ CRUSH rule 0 x 108 [8,1]
+ CRUSH rule 0 x 109 [1,4]
+ CRUSH rule 0 x 110 [4,2]
+ CRUSH rule 0 x 111 [2,4]
+ CRUSH rule 0 x 112 [2,8]
+ CRUSH rule 0 x 113 [8,1]
+ CRUSH rule 0 x 114 [8,4]
+ CRUSH rule 0 x 115 [8,2]
+ CRUSH rule 0 x 116 [1,8]
+ CRUSH rule 0 x 117 [6]
+ CRUSH rule 0 x 118 [2]
+ CRUSH rule 0 x 119 [8]
+ CRUSH rule 0 x 120 [2,4]
+ CRUSH rule 0 x 121 [2,8]
+ CRUSH rule 0 x 122 [8]
+ CRUSH rule 0 x 123 [2]
+ CRUSH rule 0 x 124 [2]
+ CRUSH rule 0 x 125 [1,8]
+ CRUSH rule 0 x 126 [1]
+ CRUSH rule 0 x 127 [4,8]
+ CRUSH rule 0 x 128 [8]
+ CRUSH rule 0 x 129 [2,4]
+ CRUSH rule 0 x 130 [4,8]
+ CRUSH rule 0 x 131 [1,4]
+ CRUSH rule 0 x 132 [1]
+ CRUSH rule 0 x 133 [8]
+ CRUSH rule 0 x 134 [1,8]
+ CRUSH rule 0 x 135 [4,8]
+ CRUSH rule 0 x 136 [2,4]
+ CRUSH rule 0 x 137 [8,4]
+ CRUSH rule 0 x 138 [8,4]
+ CRUSH rule 0 x 139 [4,2]
+ CRUSH rule 0 x 140 [1,8]
+ CRUSH rule 0 x 141 [8,2]
+ CRUSH rule 0 x 142 [4,2]
+ CRUSH rule 0 x 143 [4,8]
+ CRUSH rule 0 x 144 [8,2]
+ CRUSH rule 0 x 145 [8]
+ CRUSH rule 0 x 146 [2,8]
+ CRUSH rule 0 x 147 [2,8]
+ CRUSH rule 0 x 148 [4,1]
+ CRUSH rule 0 x 149 [4,8]
+ CRUSH rule 0 x 150 [1,8]
+ CRUSH rule 0 x 151 [8]
+ CRUSH rule 0 x 152 [8]
+ CRUSH rule 0 x 153 [8,4]
+ CRUSH rule 0 x 154 [4,2]
+ CRUSH rule 0 x 155 [4,8]
+ CRUSH rule 0 x 156 [4,2]
+ CRUSH rule 0 x 157 [1]
+ CRUSH rule 0 x 158 [2,8]
+ CRUSH rule 0 x 159 [8,2]
+ CRUSH rule 0 x 160 [2,8]
+ CRUSH rule 0 x 161 [1,4]
+ CRUSH rule 0 x 162 [1,8]
+ CRUSH rule 0 x 163 [4,8]
+ CRUSH rule 0 x 164 [8,2]
+ CRUSH rule 0 x 165 [8,2]
+ CRUSH rule 0 x 166 [2]
+ CRUSH rule 0 x 167 [1,8]
+ CRUSH rule 0 x 168 [4,2]
+ CRUSH rule 0 x 169 [2,8]
+ CRUSH rule 0 x 170 [1]
+ CRUSH rule 0 x 171 [8,4]
+ CRUSH rule 0 x 172 [1,8]
+ CRUSH rule 0 x 173 [8,4]
+ CRUSH rule 0 x 174 [1]
+ CRUSH rule 0 x 175 [8,1]
+ CRUSH rule 0 x 176 [2]
+ CRUSH rule 0 x 177 [1]
+ CRUSH rule 0 x 178 [4,2]
+ CRUSH rule 0 x 179 [1]
+ CRUSH rule 0 x 180 [8]
+ CRUSH rule 0 x 181 [8,2]
+ CRUSH rule 0 x 182 [8]
+ CRUSH rule 0 x 183 [8,4]
+ CRUSH rule 0 x 184 [4,8]
+ CRUSH rule 0 x 185 [8,1]
+ CRUSH rule 0 x 186 [2,4]
+ CRUSH rule 0 x 187 [1,8]
+ CRUSH rule 0 x 188 [1,8]
+ CRUSH rule 0 x 189 [1,8]
+ CRUSH rule 0 x 190 [1]
+ CRUSH rule 0 x 191 [8,1]
+ CRUSH rule 0 x 192 [4,2]
+ CRUSH rule 0 x 193 [4,2]
+ CRUSH rule 0 x 194 [1]
+ CRUSH rule 0 x 195 [8,4]
+ CRUSH rule 0 x 196 [8,1]
+ CRUSH rule 0 x 197 [8,4]
+ CRUSH rule 0 x 198 [2]
+ CRUSH rule 0 x 199 [1,4]
+ CRUSH rule 0 x 200 [1]
+ CRUSH rule 0 x 201 [8,1]
+ CRUSH rule 0 x 202 [8]
+ CRUSH rule 0 x 203 [8]
+ CRUSH rule 0 x 204 [2,4]
+ CRUSH rule 0 x 205 [1,8]
+ CRUSH rule 0 x 206 [1,8]
+ CRUSH rule 0 x 207 [2]
+ CRUSH rule 0 x 208 [8,2]
+ CRUSH rule 0 x 209 [1,8]
+ CRUSH rule 0 x 210 [1,4]
+ CRUSH rule 0 x 211 [4,2]
+ CRUSH rule 0 x 212 [8]
+ CRUSH rule 0 x 213 [8,4]
+ CRUSH rule 0 x 214 [8]
+ CRUSH rule 0 x 215 [8,1]
+ CRUSH rule 0 x 216 [2]
+ CRUSH rule 0 x 217 [1,8]
+ CRUSH rule 0 x 218 [2,8]
+ CRUSH rule 0 x 219 [8]
+ CRUSH rule 0 x 220 [4,8]
+ CRUSH rule 0 x 221 [8]
+ CRUSH rule 0 x 222 [8]
+ CRUSH rule 0 x 223 [1]
+ CRUSH rule 0 x 224 [1,4]
+ CRUSH rule 0 x 225 [8,2]
+ CRUSH rule 0 x 226 [8,2]
+ CRUSH rule 0 x 227 [4,1]
+ CRUSH rule 0 x 228 [8]
+ CRUSH rule 0 x 229 [4,8]
+ CRUSH rule 0 x 230 [4,8]
+ CRUSH rule 0 x 231 [4,8]
+ CRUSH rule 0 x 232 [2,8]
+ CRUSH rule 0 x 233 [8]
+ CRUSH rule 0 x 234 [1]
+ CRUSH rule 0 x 235 [4,8]
+ CRUSH rule 0 x 236 [2]
+ CRUSH rule 0 x 237 [4,8]
+ CRUSH rule 0 x 238 [2]
+ CRUSH rule 0 x 239 [8]
+ CRUSH rule 0 x 240 [4,8]
+ CRUSH rule 0 x 241 [1]
+ CRUSH rule 0 x 242 [2]
+ CRUSH rule 0 x 243 [8]
+ CRUSH rule 0 x 244 [4,8]
+ CRUSH rule 0 x 245 [8,2]
+ CRUSH rule 0 x 246 [1]
+ CRUSH rule 0 x 247 [8,1]
+ CRUSH rule 0 x 248 [8,1]
+ CRUSH rule 0 x 249 [2]
+ CRUSH rule 0 x 250 [2,4]
+ CRUSH rule 0 x 251 [2]
+ CRUSH rule 0 x 252 [4,8]
+ CRUSH rule 0 x 253 [2]
+ CRUSH rule 0 x 254 [4,2]
+ CRUSH rule 0 x 255 [1,8]
+ CRUSH rule 0 x 256 [4,8]
+ CRUSH rule 0 x 257 [2,6]
+ CRUSH rule 0 x 258 [4,2]
+ CRUSH rule 0 x 259 [6]
+ CRUSH rule 0 x 260 [8]
+ CRUSH rule 0 x 261 [8]
+ CRUSH rule 0 x 262 [8]
+ CRUSH rule 0 x 263 [8,1]
+ CRUSH rule 0 x 264 [8]
+ CRUSH rule 0 x 265 [8]
+ CRUSH rule 0 x 266 [8,2]
+ CRUSH rule 0 x 267 [2]
+ CRUSH rule 0 x 268 [1,8]
+ CRUSH rule 0 x 269 [1,8]
+ CRUSH rule 0 x 270 [4,1]
+ CRUSH rule 0 x 271 [8,4]
+ CRUSH rule 0 x 272 [2,8]
+ CRUSH rule 0 x 273 [4,2]
+ CRUSH rule 0 x 274 [8,4]
+ CRUSH rule 0 x 275 [4,8]
+ CRUSH rule 0 x 276 [8,1]
+ CRUSH rule 0 x 277 [8]
+ CRUSH rule 0 x 278 [8,2]
+ CRUSH rule 0 x 279 [8,4]
+ CRUSH rule 0 x 280 [2,8]
+ CRUSH rule 0 x 281 [8,1]
+ CRUSH rule 0 x 282 [2]
+ CRUSH rule 0 x 283 [8,1]
+ CRUSH rule 0 x 284 [8]
+ CRUSH rule 0 x 285 [4,8]
+ CRUSH rule 0 x 286 [2,8]
+ CRUSH rule 0 x 287 [1]
+ CRUSH rule 0 x 288 [8,1]
+ CRUSH rule 0 x 289 [4,8]
+ CRUSH rule 0 x 290 [1,4]
+ CRUSH rule 0 x 291 [1,4]
+ CRUSH rule 0 x 292 [8,1]
+ CRUSH rule 0 x 293 [8,2]
+ CRUSH rule 0 x 294 [8,4]
+ CRUSH rule 0 x 295 [4,8]
+ CRUSH rule 0 x 296 [4,1]
+ CRUSH rule 0 x 297 [8,2]
+ CRUSH rule 0 x 298 [1]
+ CRUSH rule 0 x 299 [2,8]
+ CRUSH rule 0 x 300 [8]
+ CRUSH rule 0 x 301 [1,8]
+ CRUSH rule 0 x 302 [1]
+ CRUSH rule 0 x 303 [8,4]
+ CRUSH rule 0 x 304 [2,8]
+ CRUSH rule 0 x 305 [8]
+ CRUSH rule 0 x 306 [1,8]
+ CRUSH rule 0 x 307 [2,8]
+ CRUSH rule 0 x 308 [2,8]
+ CRUSH rule 0 x 309 [8]
+ CRUSH rule 0 x 310 [4,1]
+ CRUSH rule 0 x 311 [4,8]
+ CRUSH rule 0 x 312 [2,8]
+ CRUSH rule 0 x 313 [4,1]
+ CRUSH rule 0 x 314 [2]
+ CRUSH rule 0 x 315 [2]
+ CRUSH rule 0 x 316 [8]
+ CRUSH rule 0 x 317 [2,8]
+ CRUSH rule 0 x 318 [8,1]
+ CRUSH rule 0 x 319 [2]
+ CRUSH rule 0 x 320 [8]
+ CRUSH rule 0 x 321 [1]
+ CRUSH rule 0 x 322 [2,6]
+ CRUSH rule 0 x 323 [4,8]
+ CRUSH rule 0 x 324 [8,2]
+ CRUSH rule 0 x 325 [4,8]
+ CRUSH rule 0 x 326 [1]
+ CRUSH rule 0 x 327 [1,8]
+ CRUSH rule 0 x 328 [8,4]
+ CRUSH rule 0 x 329 [4,8]
+ CRUSH rule 0 x 330 [4,8]
+ CRUSH rule 0 x 331 [2,8]
+ CRUSH rule 0 x 332 [2]
+ CRUSH rule 0 x 333 [8]
+ CRUSH rule 0 x 334 [8]
+ CRUSH rule 0 x 335 [8,2]
+ CRUSH rule 0 x 336 [4,8]
+ CRUSH rule 0 x 337 [8,2]
+ CRUSH rule 0 x 338 [8]
+ CRUSH rule 0 x 339 [8]
+ CRUSH rule 0 x 340 [2,8]
+ CRUSH rule 0 x 341 [4,1]
+ CRUSH rule 0 x 342 [2,8]
+ CRUSH rule 0 x 343 [8]
+ CRUSH rule 0 x 344 [6,1]
+ CRUSH rule 0 x 345 [8]
+ CRUSH rule 0 x 346 [8,2]
+ CRUSH rule 0 x 347 [4,1]
+ CRUSH rule 0 x 348 [8,2]
+ CRUSH rule 0 x 349 [1,8]
+ CRUSH rule 0 x 350 [8]
+ CRUSH rule 0 x 351 [8]
+ CRUSH rule 0 x 352 [1,8]
+ CRUSH rule 0 x 353 [8]
+ CRUSH rule 0 x 354 [1]
+ CRUSH rule 0 x 355 [8]
+ CRUSH rule 0 x 356 [4,1]
+ CRUSH rule 0 x 357 [8,1]
+ CRUSH rule 0 x 358 [2,8]
+ CRUSH rule 0 x 359 [6,1]
+ CRUSH rule 0 x 360 [2]
+ CRUSH rule 0 x 361 [8,4]
+ CRUSH rule 0 x 362 [4,1]
+ CRUSH rule 0 x 363 [4,2]
+ CRUSH rule 0 x 364 [2]
+ CRUSH rule 0 x 365 [8]
+ CRUSH rule 0 x 366 [8,2]
+ CRUSH rule 0 x 367 [4,2]
+ CRUSH rule 0 x 368 [8,4]
+ CRUSH rule 0 x 369 [8]
+ CRUSH rule 0 x 370 [8,2]
+ CRUSH rule 0 x 371 [1,4]
+ CRUSH rule 0 x 372 [1]
+ CRUSH rule 0 x 373 [1,8]
+ CRUSH rule 0 x 374 [8]
+ CRUSH rule 0 x 375 [8,4]
+ CRUSH rule 0 x 376 [8,1]
+ CRUSH rule 0 x 377 [1,4]
+ CRUSH rule 0 x 378 [1,8]
+ CRUSH rule 0 x 379 [8]
+ CRUSH rule 0 x 380 [2]
+ CRUSH rule 0 x 381 [1,4]
+ CRUSH rule 0 x 382 [1,4]
+ CRUSH rule 0 x 383 [4,8]
+ CRUSH rule 0 x 384 [8,2]
+ CRUSH rule 0 x 385 [8]
+ CRUSH rule 0 x 386 [1]
+ CRUSH rule 0 x 387 [1,4]
+ CRUSH rule 0 x 388 [2]
+ CRUSH rule 0 x 389 [1,4]
+ CRUSH rule 0 x 390 [4,8]
+ CRUSH rule 0 x 391 [4,8]
+ CRUSH rule 0 x 392 [1,8]
+ CRUSH rule 0 x 393 [2]
+ CRUSH rule 0 x 394 [8]
+ CRUSH rule 0 x 395 [1]
+ CRUSH rule 0 x 396 [4,2]
+ CRUSH rule 0 x 397 [2,4]
+ CRUSH rule 0 x 398 [2,4]
+ CRUSH rule 0 x 399 [8,4]
+ CRUSH rule 0 x 400 [8,1]
+ CRUSH rule 0 x 401 [1,4]
+ CRUSH rule 0 x 402 [8,4]
+ CRUSH rule 0 x 403 [1,4]
+ CRUSH rule 0 x 404 [4,2]
+ CRUSH rule 0 x 405 [8,4]
+ CRUSH rule 0 x 406 [2,8]
+ CRUSH rule 0 x 407 [2,8]
+ CRUSH rule 0 x 408 [4,1]
+ CRUSH rule 0 x 409 [8,4]
+ CRUSH rule 0 x 410 [8,4]
+ CRUSH rule 0 x 411 [2,8]
+ CRUSH rule 0 x 412 [2]
+ CRUSH rule 0 x 413 [2]
+ CRUSH rule 0 x 414 [4,1]
+ CRUSH rule 0 x 415 [2,8]
+ CRUSH rule 0 x 416 [2,8]
+ CRUSH rule 0 x 417 [8,2]
+ CRUSH rule 0 x 418 [8,1]
+ CRUSH rule 0 x 419 [8,4]
+ CRUSH rule 0 x 420 [1,4]
+ CRUSH rule 0 x 421 [8,4]
+ CRUSH rule 0 x 422 [6]
+ CRUSH rule 0 x 423 [2,4]
+ CRUSH rule 0 x 424 [8]
+ CRUSH rule 0 x 425 [1]
+ CRUSH rule 0 x 426 [8,2]
+ CRUSH rule 0 x 427 [1,8]
+ CRUSH rule 0 x 428 [4,8]
+ CRUSH rule 0 x 429 [4,8]
+ CRUSH rule 0 x 430 [4,8]
+ CRUSH rule 0 x 431 [4,2]
+ CRUSH rule 0 x 432 [8,1]
+ CRUSH rule 0 x 433 [8]
+ CRUSH rule 0 x 434 [2]
+ CRUSH rule 0 x 435 [2]
+ CRUSH rule 0 x 436 [4,1]
+ CRUSH rule 0 x 437 [8]
+ CRUSH rule 0 x 438 [2,4]
+ CRUSH rule 0 x 439 [1]
+ CRUSH rule 0 x 440 [2,8]
+ CRUSH rule 0 x 441 [4,8]
+ CRUSH rule 0 x 442 [2]
+ CRUSH rule 0 x 443 [8,1]
+ CRUSH rule 0 x 444 [8,1]
+ CRUSH rule 0 x 445 [8]
+ CRUSH rule 0 x 446 [1]
+ CRUSH rule 0 x 447 [2,4]
+ CRUSH rule 0 x 448 [8,2]
+ CRUSH rule 0 x 449 [8]
+ CRUSH rule 0 x 450 [1]
+ CRUSH rule 0 x 451 [8,4]
+ CRUSH rule 0 x 452 [8]
+ CRUSH rule 0 x 453 [6]
+ CRUSH rule 0 x 454 [8]
+ CRUSH rule 0 x 455 [2,8]
+ CRUSH rule 0 x 456 [8,2]
+ CRUSH rule 0 x 457 [8,1]
+ CRUSH rule 0 x 458 [2,8]
+ CRUSH rule 0 x 459 [2,8]
+ CRUSH rule 0 x 460 [8]
+ CRUSH rule 0 x 461 [8]
+ CRUSH rule 0 x 462 [8,1]
+ CRUSH rule 0 x 463 [8,2]
+ CRUSH rule 0 x 464 [8,4]
+ CRUSH rule 0 x 465 [6,1]
+ CRUSH rule 0 x 466 [8]
+ CRUSH rule 0 x 467 [8]
+ CRUSH rule 0 x 468 [8,2]
+ CRUSH rule 0 x 469 [8,2]
+ CRUSH rule 0 x 470 [4,1]
+ CRUSH rule 0 x 471 [1,8]
+ CRUSH rule 0 x 472 [1]
+ CRUSH rule 0 x 473 [1,4]
+ CRUSH rule 0 x 474 [8,1]
+ CRUSH rule 0 x 475 [8,2]
+ CRUSH rule 0 x 476 [4,8]
+ CRUSH rule 0 x 477 [4,8]
+ CRUSH rule 0 x 478 [8,1]
+ CRUSH rule 0 x 479 [2]
+ CRUSH rule 0 x 480 [1,8]
+ CRUSH rule 0 x 481 [2,4]
+ CRUSH rule 0 x 482 [8]
+ CRUSH rule 0 x 483 [2,8]
+ CRUSH rule 0 x 484 [1,8]
+ CRUSH rule 0 x 485 [8]
+ CRUSH rule 0 x 486 [4,1]
+ CRUSH rule 0 x 487 [1]
+ CRUSH rule 0 x 488 [8]
+ CRUSH rule 0 x 489 [2,8]
+ CRUSH rule 0 x 490 [6]
+ CRUSH rule 0 x 491 [1,8]
+ CRUSH rule 0 x 492 [8]
+ CRUSH rule 0 x 493 [2,8]
+ CRUSH rule 0 x 494 [1,8]
+ CRUSH rule 0 x 495 [4,1]
+ CRUSH rule 0 x 496 [8,4]
+ CRUSH rule 0 x 497 [4,8]
+ CRUSH rule 0 x 498 [2,4]
+ CRUSH rule 0 x 499 [8,4]
+ CRUSH rule 0 x 500 [4,8]
+ CRUSH rule 0 x 501 [2,8]
+ CRUSH rule 0 x 502 [6,1]
+ CRUSH rule 0 x 503 [2]
+ CRUSH rule 0 x 504 [8]
+ CRUSH rule 0 x 505 [1,8]
+ CRUSH rule 0 x 506 [4,1]
+ CRUSH rule 0 x 507 [8,2]
+ CRUSH rule 0 x 508 [1]
+ CRUSH rule 0 x 509 [8]
+ CRUSH rule 0 x 510 [8,2]
+ CRUSH rule 0 x 511 [4,8]
+ CRUSH rule 0 x 512 [8,2]
+ CRUSH rule 0 x 513 [8,2]
+ CRUSH rule 0 x 514 [8]
+ CRUSH rule 0 x 515 [8,4]
+ CRUSH rule 0 x 516 [4,1]
+ CRUSH rule 0 x 517 [8,2]
+ CRUSH rule 0 x 518 [4,8]
+ CRUSH rule 0 x 519 [8,4]
+ CRUSH rule 0 x 520 [2,8]
+ CRUSH rule 0 x 521 [8,2]
+ CRUSH rule 0 x 522 [8,1]
+ CRUSH rule 0 x 523 [4,1]
+ CRUSH rule 0 x 524 [2]
+ CRUSH rule 0 x 525 [2]
+ CRUSH rule 0 x 526 [1]
+ CRUSH rule 0 x 527 [1,4]
+ CRUSH rule 0 x 528 [2]
+ CRUSH rule 0 x 529 [4,8]
+ CRUSH rule 0 x 530 [8]
+ CRUSH rule 0 x 531 [8,1]
+ CRUSH rule 0 x 532 [6,4]
+ CRUSH rule 0 x 533 [4,8]
+ CRUSH rule 0 x 534 [8]
+ CRUSH rule 0 x 535 [8,1]
+ CRUSH rule 0 x 536 [8,2]
+ CRUSH rule 0 x 537 [4,8]
+ CRUSH rule 0 x 538 [8,4]
+ CRUSH rule 0 x 539 [8]
+ CRUSH rule 0 x 540 [1,8]
+ CRUSH rule 0 x 541 [2,4]
+ CRUSH rule 0 x 542 [2]
+ CRUSH rule 0 x 543 [8,2]
+ CRUSH rule 0 x 544 [4,8]
+ CRUSH rule 0 x 545 [8]
+ CRUSH rule 0 x 546 [8,2]
+ CRUSH rule 0 x 547 [8,1]
+ CRUSH rule 0 x 548 [4,1]
+ CRUSH rule 0 x 549 [8]
+ CRUSH rule 0 x 550 [2,4]
+ CRUSH rule 0 x 551 [8]
+ CRUSH rule 0 x 552 [4,8]
+ CRUSH rule 0 x 553 [2]
+ CRUSH rule 0 x 554 [1,8]
+ CRUSH rule 0 x 555 [4,2]
+ CRUSH rule 0 x 556 [8]
+ CRUSH rule 0 x 557 [8]
+ CRUSH rule 0 x 558 [4,2]
+ CRUSH rule 0 x 559 [1]
+ CRUSH rule 0 x 560 [8]
+ CRUSH rule 0 x 561 [8,4]
+ CRUSH rule 0 x 562 [4,1]
+ CRUSH rule 0 x 563 [2,8]
+ CRUSH rule 0 x 564 [1]
+ CRUSH rule 0 x 565 [4,8]
+ CRUSH rule 0 x 566 [4,8]
+ CRUSH rule 0 x 567 [4,8]
+ CRUSH rule 0 x 568 [8]
+ CRUSH rule 0 x 569 [4,1]
+ CRUSH rule 0 x 570 [1]
+ CRUSH rule 0 x 571 [6]
+ CRUSH rule 0 x 572 [4,2]
+ CRUSH rule 0 x 573 [1]
+ CRUSH rule 0 x 574 [2]
+ CRUSH rule 0 x 575 [8,1]
+ CRUSH rule 0 x 576 [4,8]
+ CRUSH rule 0 x 577 [8,2]
+ CRUSH rule 0 x 578 [8,1]
+ CRUSH rule 0 x 579 [4,2]
+ CRUSH rule 0 x 580 [1]
+ CRUSH rule 0 x 581 [8,1]
+ CRUSH rule 0 x 582 [2,8]
+ CRUSH rule 0 x 583 [8,2]
+ CRUSH rule 0 x 584 [8,1]
+ CRUSH rule 0 x 585 [8,1]
+ CRUSH rule 0 x 586 [1,8]
+ CRUSH rule 0 x 587 [2,4]
+ CRUSH rule 0 x 588 [4,8]
+ CRUSH rule 0 x 589 [8,1]
+ CRUSH rule 0 x 590 [8,2]
+ CRUSH rule 0 x 591 [4,2]
+ CRUSH rule 0 x 592 [2,4]
+ CRUSH rule 0 x 593 [1,8]
+ CRUSH rule 0 x 594 [2,8]
+ CRUSH rule 0 x 595 [8,1]
+ CRUSH rule 0 x 596 [2]
+ CRUSH rule 0 x 597 [1]
+ CRUSH rule 0 x 598 [2]
+ CRUSH rule 0 x 599 [4,1]
+ CRUSH rule 0 x 600 [8,1]
+ CRUSH rule 0 x 601 [1,8]
+ CRUSH rule 0 x 602 [8]
+ CRUSH rule 0 x 603 [1]
+ CRUSH rule 0 x 604 [8]
+ CRUSH rule 0 x 605 [2]
+ CRUSH rule 0 x 606 [2,6]
+ CRUSH rule 0 x 607 [2,4]
+ CRUSH rule 0 x 608 [4,1]
+ CRUSH rule 0 x 609 [4,2]
+ CRUSH rule 0 x 610 [8]
+ CRUSH rule 0 x 611 [1,8]
+ CRUSH rule 0 x 612 [2,8]
+ CRUSH rule 0 x 613 [8,1]
+ CRUSH rule 0 x 614 [8,2]
+ CRUSH rule 0 x 615 [8,2]
+ CRUSH rule 0 x 616 [1,8]
+ CRUSH rule 0 x 617 [8,2]
+ CRUSH rule 0 x 618 [8,4]
+ CRUSH rule 0 x 619 [4,1]
+ CRUSH rule 0 x 620 [1]
+ CRUSH rule 0 x 621 [8]
+ CRUSH rule 0 x 622 [2,4]
+ CRUSH rule 0 x 623 [2,8]
+ CRUSH rule 0 x 624 [4,2]
+ CRUSH rule 0 x 625 [2]
+ CRUSH rule 0 x 626 [8,2]
+ CRUSH rule 0 x 627 [2,8]
+ CRUSH rule 0 x 628 [8,1]
+ CRUSH rule 0 x 629 [2,8]
+ CRUSH rule 0 x 630 [2,8]
+ CRUSH rule 0 x 631 [1,8]
+ CRUSH rule 0 x 632 [8,2]
+ CRUSH rule 0 x 633 [8]
+ CRUSH rule 0 x 634 [1]
+ CRUSH rule 0 x 635 [4,8]
+ CRUSH rule 0 x 636 [1,4]
+ CRUSH rule 0 x 637 [1]
+ CRUSH rule 0 x 638 [8,2]
+ CRUSH rule 0 x 639 [2]
+ CRUSH rule 0 x 640 [2]
+ CRUSH rule 0 x 641 [8,2]
+ CRUSH rule 0 x 642 [2,8]
+ CRUSH rule 0 x 643 [1]
+ CRUSH rule 0 x 644 [8,1]
+ CRUSH rule 0 x 645 [8]
+ CRUSH rule 0 x 646 [8,1]
+ CRUSH rule 0 x 647 [8,1]
+ CRUSH rule 0 x 648 [1,8]
+ CRUSH rule 0 x 649 [4,8]
+ CRUSH rule 0 x 650 [8,4]
+ CRUSH rule 0 x 651 [4,6]
+ CRUSH rule 0 x 652 [4,8]
+ CRUSH rule 0 x 653 [8]
+ CRUSH rule 0 x 654 [6]
+ CRUSH rule 0 x 655 [1,4]
+ CRUSH rule 0 x 656 [8]
+ CRUSH rule 0 x 657 [6,1]
+ CRUSH rule 0 x 658 [8]
+ CRUSH rule 0 x 659 [4,8]
+ CRUSH rule 0 x 660 [8]
+ CRUSH rule 0 x 661 [1,8]
+ CRUSH rule 0 x 662 [2]
+ CRUSH rule 0 x 663 [1,4]
+ CRUSH rule 0 x 664 [1,4]
+ CRUSH rule 0 x 665 [4,6]
+ CRUSH rule 0 x 666 [2,8]
+ CRUSH rule 0 x 667 [1,4]
+ CRUSH rule 0 x 668 [4,8]
+ CRUSH rule 0 x 669 [6,4]
+ CRUSH rule 0 x 670 [4,1]
+ CRUSH rule 0 x 671 [2,8]
+ CRUSH rule 0 x 672 [4,2]
+ CRUSH rule 0 x 673 [4,2]
+ CRUSH rule 0 x 674 [1]
+ CRUSH rule 0 x 675 [1,8]
+ CRUSH rule 0 x 676 [2,4]
+ CRUSH rule 0 x 677 [4,1]
+ CRUSH rule 0 x 678 [2,4]
+ CRUSH rule 0 x 679 [8,2]
+ CRUSH rule 0 x 680 [2]
+ CRUSH rule 0 x 681 [8]
+ CRUSH rule 0 x 682 [1,4]
+ CRUSH rule 0 x 683 [1,4]
+ CRUSH rule 0 x 684 [8,2]
+ CRUSH rule 0 x 685 [8,2]
+ CRUSH rule 0 x 686 [1,4]
+ CRUSH rule 0 x 687 [6]
+ CRUSH rule 0 x 688 [4,8]
+ CRUSH rule 0 x 689 [8,4]
+ CRUSH rule 0 x 690 [8,1]
+ CRUSH rule 0 x 691 [1]
+ CRUSH rule 0 x 692 [8,1]
+ CRUSH rule 0 x 693 [8,4]
+ CRUSH rule 0 x 694 [8,4]
+ CRUSH rule 0 x 695 [2,8]
+ CRUSH rule 0 x 696 [1]
+ CRUSH rule 0 x 697 [8,1]
+ CRUSH rule 0 x 698 [8,2]
+ CRUSH rule 0 x 699 [1,8]
+ CRUSH rule 0 x 700 [1]
+ CRUSH rule 0 x 701 [1]
+ CRUSH rule 0 x 702 [2]
+ CRUSH rule 0 x 703 [8]
+ CRUSH rule 0 x 704 [1,4]
+ CRUSH rule 0 x 705 [8,1]
+ CRUSH rule 0 x 706 [1,4]
+ CRUSH rule 0 x 707 [8,4]
+ CRUSH rule 0 x 708 [4,8]
+ CRUSH rule 0 x 709 [8]
+ CRUSH rule 0 x 710 [8]
+ CRUSH rule 0 x 711 [2,4]
+ CRUSH rule 0 x 712 [2]
+ CRUSH rule 0 x 713 [8,4]
+ CRUSH rule 0 x 714 [2]
+ CRUSH rule 0 x 715 [1]
+ CRUSH rule 0 x 716 [4,8]
+ CRUSH rule 0 x 717 [8,2]
+ CRUSH rule 0 x 718 [8]
+ CRUSH rule 0 x 719 [2,6]
+ CRUSH rule 0 x 720 [8,1]
+ CRUSH rule 0 x 721 [4,6]
+ CRUSH rule 0 x 722 [8]
+ CRUSH rule 0 x 723 [4,2]
+ CRUSH rule 0 x 724 [2,6]
+ CRUSH rule 0 x 725 [1]
+ CRUSH rule 0 x 726 [4,8]
+ CRUSH rule 0 x 727 [4,8]
+ CRUSH rule 0 x 728 [2,8]
+ CRUSH rule 0 x 729 [8]
+ CRUSH rule 0 x 730 [4,8]
+ CRUSH rule 0 x 731 [4,1]
+ CRUSH rule 0 x 732 [1]
+ CRUSH rule 0 x 733 [4,8]
+ CRUSH rule 0 x 734 [8,4]
+ CRUSH rule 0 x 735 [4,8]
+ CRUSH rule 0 x 736 [4,8]
+ CRUSH rule 0 x 737 [1,8]
+ CRUSH rule 0 x 738 [4,1]
+ CRUSH rule 0 x 739 [2,8]
+ CRUSH rule 0 x 740 [1,8]
+ CRUSH rule 0 x 741 [8,2]
+ CRUSH rule 0 x 742 [8,2]
+ CRUSH rule 0 x 743 [8,1]
+ CRUSH rule 0 x 744 [4,8]
+ CRUSH rule 0 x 745 [1]
+ CRUSH rule 0 x 746 [1]
+ CRUSH rule 0 x 747 [8,2]
+ CRUSH rule 0 x 748 [2,8]
+ CRUSH rule 0 x 749 [4,8]
+ CRUSH rule 0 x 750 [1,8]
+ CRUSH rule 0 x 751 [2,8]
+ CRUSH rule 0 x 752 [8,1]
+ CRUSH rule 0 x 753 [8,4]
+ CRUSH rule 0 x 754 [8,4]
+ CRUSH rule 0 x 755 [1,8]
+ CRUSH rule 0 x 756 [8]
+ CRUSH rule 0 x 757 [8,1]
+ CRUSH rule 0 x 758 [8,1]
+ CRUSH rule 0 x 759 [8,4]
+ CRUSH rule 0 x 760 [1,4]
+ CRUSH rule 0 x 761 [2]
+ CRUSH rule 0 x 762 [2,8]
+ CRUSH rule 0 x 763 [8,4]
+ CRUSH rule 0 x 764 [1,8]
+ CRUSH rule 0 x 765 [8]
+ CRUSH rule 0 x 766 [8]
+ CRUSH rule 0 x 767 [1,8]
+ CRUSH rule 0 x 768 [8,4]
+ CRUSH rule 0 x 769 [8,2]
+ CRUSH rule 0 x 770 [8,1]
+ CRUSH rule 0 x 771 [8,2]
+ CRUSH rule 0 x 772 [8,4]
+ CRUSH rule 0 x 773 [4,2]
+ CRUSH rule 0 x 774 [8]
+ CRUSH rule 0 x 775 [8,4]
+ CRUSH rule 0 x 776 [6,2]
+ CRUSH rule 0 x 777 [4,1]
+ CRUSH rule 0 x 778 [1,6]
+ CRUSH rule 0 x 779 [2,8]
+ CRUSH rule 0 x 780 [2,4]
+ CRUSH rule 0 x 781 [8]
+ CRUSH rule 0 x 782 [4,2]
+ CRUSH rule 0 x 783 [8,1]
+ CRUSH rule 0 x 784 [1,4]
+ CRUSH rule 0 x 785 [8,1]
+ CRUSH rule 0 x 786 [8]
+ CRUSH rule 0 x 787 [1,8]
+ CRUSH rule 0 x 788 [8,2]
+ CRUSH rule 0 x 789 [1]
+ CRUSH rule 0 x 790 [8]
+ CRUSH rule 0 x 791 [4,8]
+ CRUSH rule 0 x 792 [4,8]
+ CRUSH rule 0 x 793 [8,2]
+ CRUSH rule 0 x 794 [2,8]
+ CRUSH rule 0 x 795 [1]
+ CRUSH rule 0 x 796 [8]
+ CRUSH rule 0 x 797 [2,4]
+ CRUSH rule 0 x 798 [6,1]
+ CRUSH rule 0 x 799 [4,2]
+ CRUSH rule 0 x 800 [2]
+ CRUSH rule 0 x 801 [4,8]
+ CRUSH rule 0 x 802 [1,8]
+ CRUSH rule 0 x 803 [2]
+ CRUSH rule 0 x 804 [8,2]
+ CRUSH rule 0 x 805 [8]
+ CRUSH rule 0 x 806 [1,4]
+ CRUSH rule 0 x 807 [4,8]
+ CRUSH rule 0 x 808 [8]
+ CRUSH rule 0 x 809 [1]
+ CRUSH rule 0 x 810 [8]
+ CRUSH rule 0 x 811 [8]
+ CRUSH rule 0 x 812 [8,4]
+ CRUSH rule 0 x 813 [8,4]
+ CRUSH rule 0 x 814 [8]
+ CRUSH rule 0 x 815 [4,1]
+ CRUSH rule 0 x 816 [2,8]
+ CRUSH rule 0 x 817 [8]
+ CRUSH rule 0 x 818 [1]
+ CRUSH rule 0 x 819 [1]
+ CRUSH rule 0 x 820 [4,8]
+ CRUSH rule 0 x 821 [4,8]
+ CRUSH rule 0 x 822 [2,4]
+ CRUSH rule 0 x 823 [4,8]
+ CRUSH rule 0 x 824 [8]
+ CRUSH rule 0 x 825 [2,8]
+ CRUSH rule 0 x 826 [8,1]
+ CRUSH rule 0 x 827 [2,6]
+ CRUSH rule 0 x 828 [2]
+ CRUSH rule 0 x 829 [8]
+ CRUSH rule 0 x 830 [2,4]
+ CRUSH rule 0 x 831 [1,8]
+ CRUSH rule 0 x 832 [4,8]
+ CRUSH rule 0 x 833 [2,8]
+ CRUSH rule 0 x 834 [1]
+ CRUSH rule 0 x 835 [8,4]
+ CRUSH rule 0 x 836 [4,8]
+ CRUSH rule 0 x 837 [8,4]
+ CRUSH rule 0 x 838 [6,2]
+ CRUSH rule 0 x 839 [2]
+ CRUSH rule 0 x 840 [8]
+ CRUSH rule 0 x 841 [4,8]
+ CRUSH rule 0 x 842 [2]
+ CRUSH rule 0 x 843 [8,4]
+ CRUSH rule 0 x 844 [8]
+ CRUSH rule 0 x 845 [4,8]
+ CRUSH rule 0 x 846 [4,2]
+ CRUSH rule 0 x 847 [2,8]
+ CRUSH rule 0 x 848 [2,8]
+ CRUSH rule 0 x 849 [4,8]
+ CRUSH rule 0 x 850 [1]
+ CRUSH rule 0 x 851 [6]
+ CRUSH rule 0 x 852 [8,4]
+ CRUSH rule 0 x 853 [6,2]
+ CRUSH rule 0 x 854 [8,2]
+ CRUSH rule 0 x 855 [8]
+ CRUSH rule 0 x 856 [8,4]
+ CRUSH rule 0 x 857 [8]
+ CRUSH rule 0 x 858 [6]
+ CRUSH rule 0 x 859 [8,2]
+ CRUSH rule 0 x 860 [2]
+ CRUSH rule 0 x 861 [8]
+ CRUSH rule 0 x 862 [8,1]
+ CRUSH rule 0 x 863 [8,1]
+ CRUSH rule 0 x 864 [8]
+ CRUSH rule 0 x 865 [8,1]
+ CRUSH rule 0 x 866 [8]
+ CRUSH rule 0 x 867 [8]
+ CRUSH rule 0 x 868 [8]
+ CRUSH rule 0 x 869 [8,4]
+ CRUSH rule 0 x 870 [2]
+ CRUSH rule 0 x 871 [1]
+ CRUSH rule 0 x 872 [1]
+ CRUSH rule 0 x 873 [4,8]
+ CRUSH rule 0 x 874 [2,6]
+ CRUSH rule 0 x 875 [2,8]
+ CRUSH rule 0 x 876 [4,8]
+ CRUSH rule 0 x 877 [8,4]
+ CRUSH rule 0 x 878 [2]
+ CRUSH rule 0 x 879 [8]
+ CRUSH rule 0 x 880 [1]
+ CRUSH rule 0 x 881 [4,8]
+ CRUSH rule 0 x 882 [1]
+ CRUSH rule 0 x 883 [2,4]
+ CRUSH rule 0 x 884 [8,1]
+ CRUSH rule 0 x 885 [4,1]
+ CRUSH rule 0 x 886 [8]
+ CRUSH rule 0 x 887 [8,4]
+ CRUSH rule 0 x 888 [8,1]
+ CRUSH rule 0 x 889 [2,8]
+ CRUSH rule 0 x 890 [8,1]
+ CRUSH rule 0 x 891 [1,8]
+ CRUSH rule 0 x 892 [8,2]
+ CRUSH rule 0 x 893 [2]
+ CRUSH rule 0 x 894 [8,4]
+ CRUSH rule 0 x 895 [4,1]
+ CRUSH rule 0 x 896 [1,8]
+ CRUSH rule 0 x 897 [2]
+ CRUSH rule 0 x 898 [1,4]
+ CRUSH rule 0 x 899 [1,8]
+ CRUSH rule 0 x 900 [4,1]
+ CRUSH rule 0 x 901 [2]
+ CRUSH rule 0 x 902 [8,4]
+ CRUSH rule 0 x 903 [8]
+ CRUSH rule 0 x 904 [8]
+ CRUSH rule 0 x 905 [8,2]
+ CRUSH rule 0 x 906 [1,8]
+ CRUSH rule 0 x 907 [8,2]
+ CRUSH rule 0 x 908 [8]
+ CRUSH rule 0 x 909 [2]
+ CRUSH rule 0 x 910 [8]
+ CRUSH rule 0 x 911 [8]
+ CRUSH rule 0 x 912 [1,8]
+ CRUSH rule 0 x 913 [8,1]
+ CRUSH rule 0 x 914 [6,4]
+ CRUSH rule 0 x 915 [8,2]
+ CRUSH rule 0 x 916 [4,1]
+ CRUSH rule 0 x 917 [1,4]
+ CRUSH rule 0 x 918 [8,2]
+ CRUSH rule 0 x 919 [8,2]
+ CRUSH rule 0 x 920 [8]
+ CRUSH rule 0 x 921 [1]
+ CRUSH rule 0 x 922 [8,4]
+ CRUSH rule 0 x 923 [4,8]
+ CRUSH rule 0 x 924 [1]
+ CRUSH rule 0 x 925 [4,8]
+ CRUSH rule 0 x 926 [2]
+ CRUSH rule 0 x 927 [1,8]
+ CRUSH rule 0 x 928 [8,1]
+ CRUSH rule 0 x 929 [4,2]
+ CRUSH rule 0 x 930 [2]
+ CRUSH rule 0 x 931 [2]
+ CRUSH rule 0 x 932 [4,2]
+ CRUSH rule 0 x 933 [8,4]
+ CRUSH rule 0 x 934 [8]
+ CRUSH rule 0 x 935 [8]
+ CRUSH rule 0 x 936 [1,8]
+ CRUSH rule 0 x 937 [4,8]
+ CRUSH rule 0 x 938 [8,4]
+ CRUSH rule 0 x 939 [2,8]
+ CRUSH rule 0 x 940 [8]
+ CRUSH rule 0 x 941 [2]
+ CRUSH rule 0 x 942 [1,8]
+ CRUSH rule 0 x 943 [8,2]
+ CRUSH rule 0 x 944 [8]
+ CRUSH rule 0 x 945 [8,2]
+ CRUSH rule 0 x 946 [2,8]
+ CRUSH rule 0 x 947 [2]
+ CRUSH rule 0 x 948 [8]
+ CRUSH rule 0 x 949 [6,2]
+ CRUSH rule 0 x 950 [8]
+ CRUSH rule 0 x 951 [8]
+ CRUSH rule 0 x 952 [2,8]
+ CRUSH rule 0 x 953 [1,4]
+ CRUSH rule 0 x 954 [2]
+ CRUSH rule 0 x 955 [8,1]
+ CRUSH rule 0 x 956 [1,8]
+ CRUSH rule 0 x 957 [8,1]
+ CRUSH rule 0 x 958 [8,4]
+ CRUSH rule 0 x 959 [4,1]
+ CRUSH rule 0 x 960 [6]
+ CRUSH rule 0 x 961 [1]
+ CRUSH rule 0 x 962 [8,4]
+ CRUSH rule 0 x 963 [2,4]
+ CRUSH rule 0 x 964 [2]
+ CRUSH rule 0 x 965 [8]
+ CRUSH rule 0 x 966 [4,8]
+ CRUSH rule 0 x 967 [8,4]
+ CRUSH rule 0 x 968 [8,2]
+ CRUSH rule 0 x 969 [8,2]
+ CRUSH rule 0 x 970 [2,8]
+ CRUSH rule 0 x 971 [1,8]
+ CRUSH rule 0 x 972 [1,8]
+ CRUSH rule 0 x 973 [1,8]
+ CRUSH rule 0 x 974 [4,1]
+ CRUSH rule 0 x 975 [4,8]
+ CRUSH rule 0 x 976 [4,8]
+ CRUSH rule 0 x 977 [8,4]
+ CRUSH rule 0 x 978 [8,2]
+ CRUSH rule 0 x 979 [8,2]
+ CRUSH rule 0 x 980 [8,2]
+ CRUSH rule 0 x 981 [8]
+ CRUSH rule 0 x 982 [1]
+ CRUSH rule 0 x 983 [4,8]
+ CRUSH rule 0 x 984 [2,8]
+ CRUSH rule 0 x 985 [2,4]
+ CRUSH rule 0 x 986 [8,4]
+ CRUSH rule 0 x 987 [2]
+ CRUSH rule 0 x 988 [1,4]
+ CRUSH rule 0 x 989 [1,8]
+ CRUSH rule 0 x 990 [1,8]
+ CRUSH rule 0 x 991 [1,4]
+ CRUSH rule 0 x 992 [8,1]
+ CRUSH rule 0 x 993 [2,8]
+ CRUSH rule 0 x 994 [4,8]
+ CRUSH rule 0 x 995 [8,1]
+ CRUSH rule 0 x 996 [8,4]
+ CRUSH rule 0 x 997 [8,4]
+ CRUSH rule 0 x 998 [8,1]
+ CRUSH rule 0 x 999 [1,8]
+ CRUSH rule 0 x 1000 [8,4]
+ CRUSH rule 0 x 1001 [2]
+ CRUSH rule 0 x 1002 [1]
+ CRUSH rule 0 x 1003 [2,8]
+ CRUSH rule 0 x 1004 [8,1]
+ CRUSH rule 0 x 1005 [8,1]
+ CRUSH rule 0 x 1006 [1,8]
+ CRUSH rule 0 x 1007 [1,4]
+ CRUSH rule 0 x 1008 [1,8]
+ CRUSH rule 0 x 1009 [6,4]
+ CRUSH rule 0 x 1010 [1]
+ CRUSH rule 0 x 1011 [4,2]
+ CRUSH rule 0 x 1012 [1]
+ CRUSH rule 0 x 1013 [2]
+ CRUSH rule 0 x 1014 [2,8]
+ CRUSH rule 0 x 1015 [8]
+ CRUSH rule 0 x 1016 [2,4]
+ CRUSH rule 0 x 1017 [6,1]
+ CRUSH rule 0 x 1018 [4,1]
+ CRUSH rule 0 x 1019 [4,8]
+ CRUSH rule 0 x 1020 [1]
+ CRUSH rule 0 x 1021 [2]
+ CRUSH rule 0 x 1022 [1,8]
+ CRUSH rule 0 x 1023 [4,2]
+ rule 0 (choose) num_rep 2 result size == 1:\t325/1024 (esc)
+ rule 0 (choose) num_rep 2 result size == 2:\t699/1024 (esc)
+ CRUSH rule 0 x 0 [2,4,8]
+ CRUSH rule 0 x 1 [2,8,4]
+ CRUSH rule 0 x 2 [1,8]
+ CRUSH rule 0 x 3 [8,1]
+ CRUSH rule 0 x 4 [4,1,8]
+ CRUSH rule 0 x 5 [8,1]
+ CRUSH rule 0 x 6 [2,8,4]
+ CRUSH rule 0 x 7 [4,8,1]
+ CRUSH rule 0 x 8 [4,8,2]
+ CRUSH rule 0 x 9 [2,4,8]
+ CRUSH rule 0 x 10 [2,8]
+ CRUSH rule 0 x 11 [2,8]
+ CRUSH rule 0 x 12 [2,8]
+ CRUSH rule 0 x 13 [4,8,1]
+ CRUSH rule 0 x 14 [8,1]
+ CRUSH rule 0 x 15 [8,1]
+ CRUSH rule 0 x 16 [8,1]
+ CRUSH rule 0 x 17 [4,1,8]
+ CRUSH rule 0 x 18 [1,6]
+ CRUSH rule 0 x 19 [8,4,2]
+ CRUSH rule 0 x 20 [2,8]
+ CRUSH rule 0 x 21 [8,2]
+ CRUSH rule 0 x 22 [8,1]
+ CRUSH rule 0 x 23 [4,8,2]
+ CRUSH rule 0 x 24 [1,8,4]
+ CRUSH rule 0 x 25 [4,8,1]
+ CRUSH rule 0 x 26 [2,8,4]
+ CRUSH rule 0 x 27 [4,1,8]
+ CRUSH rule 0 x 28 [8,2]
+ CRUSH rule 0 x 29 [8,4,2]
+ CRUSH rule 0 x 30 [4,8,1]
+ CRUSH rule 0 x 31 [8,2]
+ CRUSH rule 0 x 32 [6,2]
+ CRUSH rule 0 x 33 [2,8]
+ CRUSH rule 0 x 34 [2,8]
+ CRUSH rule 0 x 35 [1,8,4]
+ CRUSH rule 0 x 36 [8,1]
+ CRUSH rule 0 x 37 [1,8]
+ CRUSH rule 0 x 38 [4,8,1]
+ CRUSH rule 0 x 39 [8,2]
+ CRUSH rule 0 x 40 [8,2,4]
+ CRUSH rule 0 x 41 [2,8,4]
+ CRUSH rule 0 x 42 [8,2]
+ CRUSH rule 0 x 43 [1,8]
+ CRUSH rule 0 x 44 [1,8,4]
+ CRUSH rule 0 x 45 [8,2,4]
+ CRUSH rule 0 x 46 [2,8]
+ CRUSH rule 0 x 47 [4,1,8]
+ CRUSH rule 0 x 48 [8,1]
+ CRUSH rule 0 x 49 [8,2]
+ CRUSH rule 0 x 50 [4,1,8]
+ CRUSH rule 0 x 51 [8,1]
+ CRUSH rule 0 x 52 [8,2,4]
+ CRUSH rule 0 x 53 [4,8,2]
+ CRUSH rule 0 x 54 [8,4,2]
+ CRUSH rule 0 x 55 [8,2]
+ CRUSH rule 0 x 56 [8,4,1]
+ CRUSH rule 0 x 57 [8,1]
+ CRUSH rule 0 x 58 [1,8]
+ CRUSH rule 0 x 59 [2,8]
+ CRUSH rule 0 x 60 [4,1,8]
+ CRUSH rule 0 x 61 [4,8,2]
+ CRUSH rule 0 x 62 [8,1]
+ CRUSH rule 0 x 63 [8,2]
+ CRUSH rule 0 x 64 [4,1,8]
+ CRUSH rule 0 x 65 [8,4,2]
+ CRUSH rule 0 x 66 [4,8,1]
+ CRUSH rule 0 x 67 [4,2,8]
+ CRUSH rule 0 x 68 [1,8]
+ CRUSH rule 0 x 69 [2,8]
+ CRUSH rule 0 x 70 [8,2]
+ CRUSH rule 0 x 71 [2,8,4]
+ CRUSH rule 0 x 72 [8,2,4]
+ CRUSH rule 0 x 73 [2,8]
+ CRUSH rule 0 x 74 [1,8]
+ CRUSH rule 0 x 75 [4,2,8]
+ CRUSH rule 0 x 76 [4,2,6]
+ CRUSH rule 0 x 77 [8,2,4]
+ CRUSH rule 0 x 78 [1,6]
+ CRUSH rule 0 x 79 [4,1,8]
+ CRUSH rule 0 x 80 [2,4,8]
+ CRUSH rule 0 x 81 [2,8]
+ CRUSH rule 0 x 82 [6,2]
+ CRUSH rule 0 x 83 [2,8]
+ CRUSH rule 0 x 84 [8,2]
+ CRUSH rule 0 x 85 [4,8,2]
+ CRUSH rule 0 x 86 [2,8]
+ CRUSH rule 0 x 87 [2,8,4]
+ CRUSH rule 0 x 88 [1,8]
+ CRUSH rule 0 x 89 [1,8]
+ CRUSH rule 0 x 90 [8,4,2]
+ CRUSH rule 0 x 91 [4,8,2]
+ CRUSH rule 0 x 92 [1,8]
+ CRUSH rule 0 x 93 [8,4,1]
+ CRUSH rule 0 x 94 [1,8]
+ CRUSH rule 0 x 95 [8,1]
+ CRUSH rule 0 x 96 [8,2]
+ CRUSH rule 0 x 97 [8,1]
+ CRUSH rule 0 x 98 [2,8]
+ CRUSH rule 0 x 99 [2,8]
+ CRUSH rule 0 x 100 [1,8,4]
+ CRUSH rule 0 x 101 [8,1]
+ CRUSH rule 0 x 102 [2,8]
+ CRUSH rule 0 x 103 [8,1]
+ CRUSH rule 0 x 104 [8,4,1]
+ CRUSH rule 0 x 105 [2,4,8]
+ CRUSH rule 0 x 106 [1,8,4]
+ CRUSH rule 0 x 107 [1,8]
+ CRUSH rule 0 x 108 [8,1]
+ CRUSH rule 0 x 109 [1,4,8]
+ CRUSH rule 0 x 110 [4,2,8]
+ CRUSH rule 0 x 111 [2,4,8]
+ CRUSH rule 0 x 112 [2,8]
+ CRUSH rule 0 x 113 [8,1]
+ CRUSH rule 0 x 114 [8,4,2]
+ CRUSH rule 0 x 115 [8,2,4]
+ CRUSH rule 0 x 116 [1,8]
+ CRUSH rule 0 x 117 [6,1]
+ CRUSH rule 0 x 118 [2,8]
+ CRUSH rule 0 x 119 [8,1]
+ CRUSH rule 0 x 120 [2,4,8]
+ CRUSH rule 0 x 121 [2,8,4]
+ CRUSH rule 0 x 122 [8,2]
+ CRUSH rule 0 x 123 [2,8]
+ CRUSH rule 0 x 124 [2,8]
+ CRUSH rule 0 x 125 [1,8,4]
+ CRUSH rule 0 x 126 [1,8]
+ CRUSH rule 0 x 127 [4,8,1]
+ CRUSH rule 0 x 128 [8,1]
+ CRUSH rule 0 x 129 [2,4,8]
+ CRUSH rule 0 x 130 [4,8,1]
+ CRUSH rule 0 x 131 [1,4,8]
+ CRUSH rule 0 x 132 [1,8]
+ CRUSH rule 0 x 133 [8,1]
+ CRUSH rule 0 x 134 [1,8,4]
+ CRUSH rule 0 x 135 [4,8,2]
+ CRUSH rule 0 x 136 [2,4,8]
+ CRUSH rule 0 x 137 [8,4,1]
+ CRUSH rule 0 x 138 [8,4,2]
+ CRUSH rule 0 x 139 [4,2,8]
+ CRUSH rule 0 x 140 [1,8,4]
+ CRUSH rule 0 x 141 [8,2]
+ CRUSH rule 0 x 142 [4,2,8]
+ CRUSH rule 0 x 143 [4,8,1]
+ CRUSH rule 0 x 144 [8,2]
+ CRUSH rule 0 x 145 [8,2]
+ CRUSH rule 0 x 146 [2,8]
+ CRUSH rule 0 x 147 [2,8,4]
+ CRUSH rule 0 x 148 [4,1,8]
+ CRUSH rule 0 x 149 [4,8,2]
+ CRUSH rule 0 x 150 [1,8]
+ CRUSH rule 0 x 151 [8,1]
+ CRUSH rule 0 x 152 [8,2]
+ CRUSH rule 0 x 153 [8,4,1]
+ CRUSH rule 0 x 154 [4,2,8]
+ CRUSH rule 0 x 155 [4,8,1]
+ CRUSH rule 0 x 156 [4,2,8]
+ CRUSH rule 0 x 157 [1,8]
+ CRUSH rule 0 x 158 [2,8,4]
+ CRUSH rule 0 x 159 [8,2,4]
+ CRUSH rule 0 x 160 [2,8,4]
+ CRUSH rule 0 x 161 [1,4,8]
+ CRUSH rule 0 x 162 [1,8]
+ CRUSH rule 0 x 163 [4,8,2]
+ CRUSH rule 0 x 164 [8,2]
+ CRUSH rule 0 x 165 [8,2,4]
+ CRUSH rule 0 x 166 [2,8]
+ CRUSH rule 0 x 167 [1,8,4]
+ CRUSH rule 0 x 168 [4,2,8]
+ CRUSH rule 0 x 169 [2,8,4]
+ CRUSH rule 0 x 170 [1,8]
+ CRUSH rule 0 x 171 [8,4,2]
+ CRUSH rule 0 x 172 [1,8]
+ CRUSH rule 0 x 173 [8,4,1]
+ CRUSH rule 0 x 174 [1,8]
+ CRUSH rule 0 x 175 [8,1]
+ CRUSH rule 0 x 176 [2,8]
+ CRUSH rule 0 x 177 [1,8]
+ CRUSH rule 0 x 178 [4,2,8]
+ CRUSH rule 0 x 179 [1,8]
+ CRUSH rule 0 x 180 [8,2]
+ CRUSH rule 0 x 181 [8,2,4]
+ CRUSH rule 0 x 182 [8,1]
+ CRUSH rule 0 x 183 [8,4,1]
+ CRUSH rule 0 x 184 [4,8,1]
+ CRUSH rule 0 x 185 [8,1,4]
+ CRUSH rule 0 x 186 [2,4,8]
+ CRUSH rule 0 x 187 [1,8]
+ CRUSH rule 0 x 188 [1,8,4]
+ CRUSH rule 0 x 189 [1,8,4]
+ CRUSH rule 0 x 190 [1,8]
+ CRUSH rule 0 x 191 [8,1,4]
+ CRUSH rule 0 x 192 [4,2,8]
+ CRUSH rule 0 x 193 [4,2,8]
+ CRUSH rule 0 x 194 [1,8]
+ CRUSH rule 0 x 195 [8,4,2]
+ CRUSH rule 0 x 196 [8,1]
+ CRUSH rule 0 x 197 [8,4,1]
+ CRUSH rule 0 x 198 [2,8]
+ CRUSH rule 0 x 199 [1,4,8]
+ CRUSH rule 0 x 200 [1,8]
+ CRUSH rule 0 x 201 [8,1,4]
+ CRUSH rule 0 x 202 [8,1]
+ CRUSH rule 0 x 203 [8,1]
+ CRUSH rule 0 x 204 [2,4,8]
+ CRUSH rule 0 x 205 [1,8]
+ CRUSH rule 0 x 206 [1,8,4]
+ CRUSH rule 0 x 207 [2,8]
+ CRUSH rule 0 x 208 [8,2]
+ CRUSH rule 0 x 209 [1,8]
+ CRUSH rule 0 x 210 [1,4,8]
+ CRUSH rule 0 x 211 [4,2,8]
+ CRUSH rule 0 x 212 [8,1]
+ CRUSH rule 0 x 213 [8,4,1]
+ CRUSH rule 0 x 214 [8,1]
+ CRUSH rule 0 x 215 [8,1]
+ CRUSH rule 0 x 216 [2,8]
+ CRUSH rule 0 x 217 [1,8,4]
+ CRUSH rule 0 x 218 [2,8]
+ CRUSH rule 0 x 219 [8,2]
+ CRUSH rule 0 x 220 [4,8,1]
+ CRUSH rule 0 x 221 [8,1]
+ CRUSH rule 0 x 222 [8,1]
+ CRUSH rule 0 x 223 [1,8]
+ CRUSH rule 0 x 224 [1,4,8]
+ CRUSH rule 0 x 225 [8,2]
+ CRUSH rule 0 x 226 [8,2,4]
+ CRUSH rule 0 x 227 [4,1,8]
+ CRUSH rule 0 x 228 [8,1]
+ CRUSH rule 0 x 229 [4,8,1]
+ CRUSH rule 0 x 230 [4,8,1]
+ CRUSH rule 0 x 231 [4,8,1]
+ CRUSH rule 0 x 232 [2,8,4]
+ CRUSH rule 0 x 233 [8,2]
+ CRUSH rule 0 x 234 [1,8]
+ CRUSH rule 0 x 235 [4,8,1]
+ CRUSH rule 0 x 236 [2,8]
+ CRUSH rule 0 x 237 [4,8,2]
+ CRUSH rule 0 x 238 [2,8]
+ CRUSH rule 0 x 239 [8,2]
+ CRUSH rule 0 x 240 [4,8,2]
+ CRUSH rule 0 x 241 [1,8]
+ CRUSH rule 0 x 242 [2,8]
+ CRUSH rule 0 x 243 [8,2]
+ CRUSH rule 0 x 244 [4,8,2]
+ CRUSH rule 0 x 245 [8,2]
+ CRUSH rule 0 x 246 [1,8]
+ CRUSH rule 0 x 247 [8,1]
+ CRUSH rule 0 x 248 [8,1,4]
+ CRUSH rule 0 x 249 [2,8]
+ CRUSH rule 0 x 250 [2,4,8]
+ CRUSH rule 0 x 251 [2,8]
+ CRUSH rule 0 x 252 [4,8,2]
+ CRUSH rule 0 x 253 [2,8]
+ CRUSH rule 0 x 254 [4,2,8]
+ CRUSH rule 0 x 255 [1,8]
+ CRUSH rule 0 x 256 [4,8,1]
+ CRUSH rule 0 x 257 [2,6,4]
+ CRUSH rule 0 x 258 [4,2,8]
+ CRUSH rule 0 x 259 [6,1]
+ CRUSH rule 0 x 260 [8,1]
+ CRUSH rule 0 x 261 [8,2]
+ CRUSH rule 0 x 262 [8,2]
+ CRUSH rule 0 x 263 [8,1,4]
+ CRUSH rule 0 x 264 [8,1]
+ CRUSH rule 0 x 265 [8,2]
+ CRUSH rule 0 x 266 [8,2,4]
+ CRUSH rule 0 x 267 [2,8]
+ CRUSH rule 0 x 268 [1,8]
+ CRUSH rule 0 x 269 [1,8,4]
+ CRUSH rule 0 x 270 [4,1,8]
+ CRUSH rule 0 x 271 [8,4,2]
+ CRUSH rule 0 x 272 [2,8,4]
+ CRUSH rule 0 x 273 [4,2,8]
+ CRUSH rule 0 x 274 [8,4,2]
+ CRUSH rule 0 x 275 [4,8,1]
+ CRUSH rule 0 x 276 [8,1,4]
+ CRUSH rule 0 x 277 [8,1]
+ CRUSH rule 0 x 278 [8,2,4]
+ CRUSH rule 0 x 279 [8,4,1]
+ CRUSH rule 0 x 280 [2,8,4]
+ CRUSH rule 0 x 281 [8,1]
+ CRUSH rule 0 x 282 [2,8]
+ CRUSH rule 0 x 283 [8,1]
+ CRUSH rule 0 x 284 [8,1]
+ CRUSH rule 0 x 285 [4,8,2]
+ CRUSH rule 0 x 286 [2,8,4]
+ CRUSH rule 0 x 287 [1,8]
+ CRUSH rule 0 x 288 [8,1,4]
+ CRUSH rule 0 x 289 [4,8,2]
+ CRUSH rule 0 x 290 [1,4,8]
+ CRUSH rule 0 x 291 [1,4,8]
+ CRUSH rule 0 x 292 [8,1,4]
+ CRUSH rule 0 x 293 [8,2]
+ CRUSH rule 0 x 294 [8,4,2]
+ CRUSH rule 0 x 295 [4,8,1]
+ CRUSH rule 0 x 296 [4,1,8]
+ CRUSH rule 0 x 297 [8,2,4]
+ CRUSH rule 0 x 298 [1,8]
+ CRUSH rule 0 x 299 [2,8]
+ CRUSH rule 0 x 300 [8,1]
+ CRUSH rule 0 x 301 [1,8]
+ CRUSH rule 0 x 302 [1,8]
+ CRUSH rule 0 x 303 [8,4,1]
+ CRUSH rule 0 x 304 [2,8]
+ CRUSH rule 0 x 305 [8,1]
+ CRUSH rule 0 x 306 [1,8]
+ CRUSH rule 0 x 307 [2,8]
+ CRUSH rule 0 x 308 [2,8,4]
+ CRUSH rule 0 x 309 [8,2]
+ CRUSH rule 0 x 310 [4,1,6]
+ CRUSH rule 0 x 311 [4,8,2]
+ CRUSH rule 0 x 312 [2,8]
+ CRUSH rule 0 x 313 [4,1,8]
+ CRUSH rule 0 x 314 [2,6]
+ CRUSH rule 0 x 315 [2,8]
+ CRUSH rule 0 x 316 [8,1]
+ CRUSH rule 0 x 317 [2,8]
+ CRUSH rule 0 x 318 [8,1]
+ CRUSH rule 0 x 319 [2,8]
+ CRUSH rule 0 x 320 [8,1]
+ CRUSH rule 0 x 321 [1,8]
+ CRUSH rule 0 x 322 [2,6,4]
+ CRUSH rule 0 x 323 [4,8,2]
+ CRUSH rule 0 x 324 [8,2,4]
+ CRUSH rule 0 x 325 [4,8,1]
+ CRUSH rule 0 x 326 [1,6]
+ CRUSH rule 0 x 327 [1,8]
+ CRUSH rule 0 x 328 [8,4,1]
+ CRUSH rule 0 x 329 [4,8,2]
+ CRUSH rule 0 x 330 [4,8,1]
+ CRUSH rule 0 x 331 [2,8]
+ CRUSH rule 0 x 332 [2,8]
+ CRUSH rule 0 x 333 [8,1]
+ CRUSH rule 0 x 334 [8,2]
+ CRUSH rule 0 x 335 [8,2]
+ CRUSH rule 0 x 336 [4,8,2]
+ CRUSH rule 0 x 337 [8,2,4]
+ CRUSH rule 0 x 338 [8,2]
+ CRUSH rule 0 x 339 [8,2]
+ CRUSH rule 0 x 340 [2,8,4]
+ CRUSH rule 0 x 341 [4,1,6]
+ CRUSH rule 0 x 342 [2,8,4]
+ CRUSH rule 0 x 343 [8,1]
+ CRUSH rule 0 x 344 [6,1,4]
+ CRUSH rule 0 x 345 [8,2]
+ CRUSH rule 0 x 346 [8,2,4]
+ CRUSH rule 0 x 347 [4,1,8]
+ CRUSH rule 0 x 348 [8,2,4]
+ CRUSH rule 0 x 349 [1,8]
+ CRUSH rule 0 x 350 [8,1]
+ CRUSH rule 0 x 351 [8,2]
+ CRUSH rule 0 x 352 [1,8,4]
+ CRUSH rule 0 x 353 [8,1]
+ CRUSH rule 0 x 354 [1,8]
+ CRUSH rule 0 x 355 [8,2]
+ CRUSH rule 0 x 356 [4,1,8]
+ CRUSH rule 0 x 357 [8,1,4]
+ CRUSH rule 0 x 358 [2,8,4]
+ CRUSH rule 0 x 359 [6,1,4]
+ CRUSH rule 0 x 360 [2,8]
+ CRUSH rule 0 x 361 [8,4,1]
+ CRUSH rule 0 x 362 [4,1,8]
+ CRUSH rule 0 x 363 [4,2,8]
+ CRUSH rule 0 x 364 [2,8]
+ CRUSH rule 0 x 365 [8,2]
+ CRUSH rule 0 x 366 [8,2]
+ CRUSH rule 0 x 367 [4,2,8]
+ CRUSH rule 0 x 368 [8,4,1]
+ CRUSH rule 0 x 369 [8,2]
+ CRUSH rule 0 x 370 [8,2]
+ CRUSH rule 0 x 371 [1,4,8]
+ CRUSH rule 0 x 372 [1,8]
+ CRUSH rule 0 x 373 [1,8]
+ CRUSH rule 0 x 374 [8,1]
+ CRUSH rule 0 x 375 [8,4,2]
+ CRUSH rule 0 x 376 [8,1,4]
+ CRUSH rule 0 x 377 [1,4,8]
+ CRUSH rule 0 x 378 [1,8]
+ CRUSH rule 0 x 379 [8,2]
+ CRUSH rule 0 x 380 [2,8]
+ CRUSH rule 0 x 381 [1,4,8]
+ CRUSH rule 0 x 382 [1,4,8]
+ CRUSH rule 0 x 383 [4,8,1]
+ CRUSH rule 0 x 384 [8,2,4]
+ CRUSH rule 0 x 385 [8,1]
+ CRUSH rule 0 x 386 [1,8]
+ CRUSH rule 0 x 387 [1,4,8]
+ CRUSH rule 0 x 388 [2,6]
+ CRUSH rule 0 x 389 [1,4,8]
+ CRUSH rule 0 x 390 [4,8,1]
+ CRUSH rule 0 x 391 [4,8,2]
+ CRUSH rule 0 x 392 [1,8,4]
+ CRUSH rule 0 x 393 [2,8]
+ CRUSH rule 0 x 394 [8,2]
+ CRUSH rule 0 x 395 [1,8]
+ CRUSH rule 0 x 396 [4,2,8]
+ CRUSH rule 0 x 397 [2,4,6]
+ CRUSH rule 0 x 398 [2,4,8]
+ CRUSH rule 0 x 399 [8,4,2]
+ CRUSH rule 0 x 400 [8,1,4]
+ CRUSH rule 0 x 401 [1,4,8]
+ CRUSH rule 0 x 402 [8,4,2]
+ CRUSH rule 0 x 403 [1,4,8]
+ CRUSH rule 0 x 404 [4,2,8]
+ CRUSH rule 0 x 405 [8,4,2]
+ CRUSH rule 0 x 406 [2,8]
+ CRUSH rule 0 x 407 [2,8,4]
+ CRUSH rule 0 x 408 [4,1,8]
+ CRUSH rule 0 x 409 [8,4,2]
+ CRUSH rule 0 x 410 [8,4,1]
+ CRUSH rule 0 x 411 [2,8,4]
+ CRUSH rule 0 x 412 [2,6]
+ CRUSH rule 0 x 413 [2,8]
+ CRUSH rule 0 x 414 [4,1,8]
+ CRUSH rule 0 x 415 [2,8]
+ CRUSH rule 0 x 416 [2,8]
+ CRUSH rule 0 x 417 [8,2]
+ CRUSH rule 0 x 418 [8,1,4]
+ CRUSH rule 0 x 419 [8,4,2]
+ CRUSH rule 0 x 420 [1,4,8]
+ CRUSH rule 0 x 421 [8,4,1]
+ CRUSH rule 0 x 422 [6,1]
+ CRUSH rule 0 x 423 [2,4,8]
+ CRUSH rule 0 x 424 [8,1]
+ CRUSH rule 0 x 425 [1,8]
+ CRUSH rule 0 x 426 [8,2]
+ CRUSH rule 0 x 427 [1,8]
+ CRUSH rule 0 x 428 [4,8,1]
+ CRUSH rule 0 x 429 [4,8,1]
+ CRUSH rule 0 x 430 [4,8,2]
+ CRUSH rule 0 x 431 [4,2,8]
+ CRUSH rule 0 x 432 [8,1]
+ CRUSH rule 0 x 433 [8,1]
+ CRUSH rule 0 x 434 [2,8]
+ CRUSH rule 0 x 435 [2,6]
+ CRUSH rule 0 x 436 [4,1,8]
+ CRUSH rule 0 x 437 [8,1]
+ CRUSH rule 0 x 438 [2,4,8]
+ CRUSH rule 0 x 439 [1,8]
+ CRUSH rule 0 x 440 [2,8]
+ CRUSH rule 0 x 441 [4,8,2]
+ CRUSH rule 0 x 442 [2,8]
+ CRUSH rule 0 x 443 [8,1,4]
+ CRUSH rule 0 x 444 [8,1]
+ CRUSH rule 0 x 445 [8,2]
+ CRUSH rule 0 x 446 [1,8]
+ CRUSH rule 0 x 447 [2,4,8]
+ CRUSH rule 0 x 448 [8,2,4]
+ CRUSH rule 0 x 449 [8,2]
+ CRUSH rule 0 x 450 [1,8]
+ CRUSH rule 0 x 451 [8,4,1]
+ CRUSH rule 0 x 452 [8,2]
+ CRUSH rule 0 x 453 [6,2]
+ CRUSH rule 0 x 454 [8,2]
+ CRUSH rule 0 x 455 [2,8,4]
+ CRUSH rule 0 x 456 [8,2]
+ CRUSH rule 0 x 457 [8,1]
+ CRUSH rule 0 x 458 [2,8]
+ CRUSH rule 0 x 459 [2,8,4]
+ CRUSH rule 0 x 460 [8,2]
+ CRUSH rule 0 x 461 [8,2]
+ CRUSH rule 0 x 462 [8,1]
+ CRUSH rule 0 x 463 [8,2]
+ CRUSH rule 0 x 464 [8,4,1]
+ CRUSH rule 0 x 465 [6,1,4]
+ CRUSH rule 0 x 466 [8,2]
+ CRUSH rule 0 x 467 [8,1]
+ CRUSH rule 0 x 468 [8,2,4]
+ CRUSH rule 0 x 469 [8,2]
+ CRUSH rule 0 x 470 [4,1,8]
+ CRUSH rule 0 x 471 [1,8]
+ CRUSH rule 0 x 472 [1,8]
+ CRUSH rule 0 x 473 [1,4,8]
+ CRUSH rule 0 x 474 [8,1]
+ CRUSH rule 0 x 475 [8,2,4]
+ CRUSH rule 0 x 476 [4,8,1]
+ CRUSH rule 0 x 477 [4,8,1]
+ CRUSH rule 0 x 478 [8,1,4]
+ CRUSH rule 0 x 479 [2,8]
+ CRUSH rule 0 x 480 [1,8]
+ CRUSH rule 0 x 481 [2,4,6]
+ CRUSH rule 0 x 482 [8,1]
+ CRUSH rule 0 x 483 [2,8,4]
+ CRUSH rule 0 x 484 [1,8]
+ CRUSH rule 0 x 485 [8,1]
+ CRUSH rule 0 x 486 [4,1,8]
+ CRUSH rule 0 x 487 [1,8]
+ CRUSH rule 0 x 488 [8,1]
+ CRUSH rule 0 x 489 [2,8]
+ CRUSH rule 0 x 490 [6,2]
+ CRUSH rule 0 x 491 [1,8]
+ CRUSH rule 0 x 492 [8,2]
+ CRUSH rule 0 x 493 [2,8]
+ CRUSH rule 0 x 494 [1,8]
+ CRUSH rule 0 x 495 [4,1,6]
+ CRUSH rule 0 x 496 [8,4,1]
+ CRUSH rule 0 x 497 [4,8,2]
+ CRUSH rule 0 x 498 [2,4,8]
+ CRUSH rule 0 x 499 [8,4,2]
+ CRUSH rule 0 x 500 [4,8,1]
+ CRUSH rule 0 x 501 [2,8]
+ CRUSH rule 0 x 502 [6,1]
+ CRUSH rule 0 x 503 [2,8]
+ CRUSH rule 0 x 504 [8,2]
+ CRUSH rule 0 x 505 [1,8]
+ CRUSH rule 0 x 506 [4,1,8]
+ CRUSH rule 0 x 507 [8,2,4]
+ CRUSH rule 0 x 508 [1,8]
+ CRUSH rule 0 x 509 [8,1]
+ CRUSH rule 0 x 510 [8,2]
+ CRUSH rule 0 x 511 [4,8,2]
+ CRUSH rule 0 x 512 [8,2]
+ CRUSH rule 0 x 513 [8,2]
+ CRUSH rule 0 x 514 [8,2]
+ CRUSH rule 0 x 515 [8,4,2]
+ CRUSH rule 0 x 516 [4,1,8]
+ CRUSH rule 0 x 517 [8,2]
+ CRUSH rule 0 x 518 [4,8,1]
+ CRUSH rule 0 x 519 [8,4,1]
+ CRUSH rule 0 x 520 [2,8,4]
+ CRUSH rule 0 x 521 [8,2,4]
+ CRUSH rule 0 x 522 [8,1,4]
+ CRUSH rule 0 x 523 [4,1,8]
+ CRUSH rule 0 x 524 [2,6]
+ CRUSH rule 0 x 525 [2,8]
+ CRUSH rule 0 x 526 [1,8]
+ CRUSH rule 0 x 527 [1,4,6]
+ CRUSH rule 0 x 528 [2,8]
+ CRUSH rule 0 x 529 [4,8,1]
+ CRUSH rule 0 x 530 [8,1]
+ CRUSH rule 0 x 531 [8,1,4]
+ CRUSH rule 0 x 532 [6,4,1]
+ CRUSH rule 0 x 533 [4,8,1]
+ CRUSH rule 0 x 534 [8,2]
+ CRUSH rule 0 x 535 [8,1]
+ CRUSH rule 0 x 536 [8,2]
+ CRUSH rule 0 x 537 [4,8,1]
+ CRUSH rule 0 x 538 [8,4,2]
+ CRUSH rule 0 x 539 [8,2]
+ CRUSH rule 0 x 540 [1,8,4]
+ CRUSH rule 0 x 541 [2,4,8]
+ CRUSH rule 0 x 542 [2,8]
+ CRUSH rule 0 x 543 [8,2]
+ CRUSH rule 0 x 544 [4,8,1]
+ CRUSH rule 0 x 545 [8,1]
+ CRUSH rule 0 x 546 [8,2,4]
+ CRUSH rule 0 x 547 [8,1,4]
+ CRUSH rule 0 x 548 [4,1,8]
+ CRUSH rule 0 x 549 [8,1]
+ CRUSH rule 0 x 550 [2,4,8]
+ CRUSH rule 0 x 551 [8,1]
+ CRUSH rule 0 x 552 [4,8,2]
+ CRUSH rule 0 x 553 [2,8]
+ CRUSH rule 0 x 554 [1,8]
+ CRUSH rule 0 x 555 [4,2,8]
+ CRUSH rule 0 x 556 [8,1]
+ CRUSH rule 0 x 557 [8,2]
+ CRUSH rule 0 x 558 [4,2,8]
+ CRUSH rule 0 x 559 [1,8]
+ CRUSH rule 0 x 560 [8,2]
+ CRUSH rule 0 x 561 [8,4,1]
+ CRUSH rule 0 x 562 [4,1,8]
+ CRUSH rule 0 x 563 [2,8]
+ CRUSH rule 0 x 564 [1,8]
+ CRUSH rule 0 x 565 [4,8,1]
+ CRUSH rule 0 x 566 [4,8,2]
+ CRUSH rule 0 x 567 [4,8,1]
+ CRUSH rule 0 x 568 [8,1]
+ CRUSH rule 0 x 569 [4,1,8]
+ CRUSH rule 0 x 570 [1,8]
+ CRUSH rule 0 x 571 [6,1]
+ CRUSH rule 0 x 572 [4,2,8]
+ CRUSH rule 0 x 573 [1,8]
+ CRUSH rule 0 x 574 [2,8]
+ CRUSH rule 0 x 575 [8,1,4]
+ CRUSH rule 0 x 576 [4,8,1]
+ CRUSH rule 0 x 577 [8,2]
+ CRUSH rule 0 x 578 [8,1]
+ CRUSH rule 0 x 579 [4,2,8]
+ CRUSH rule 0 x 580 [1,8]
+ CRUSH rule 0 x 581 [8,1,4]
+ CRUSH rule 0 x 582 [2,8,4]
+ CRUSH rule 0 x 583 [8,2]
+ CRUSH rule 0 x 584 [8,1,4]
+ CRUSH rule 0 x 585 [8,1,4]
+ CRUSH rule 0 x 586 [1,8,4]
+ CRUSH rule 0 x 587 [2,4,8]
+ CRUSH rule 0 x 588 [4,8,1]
+ CRUSH rule 0 x 589 [8,1]
+ CRUSH rule 0 x 590 [8,2]
+ CRUSH rule 0 x 591 [4,2,8]
+ CRUSH rule 0 x 592 [2,4,8]
+ CRUSH rule 0 x 593 [1,8,4]
+ CRUSH rule 0 x 594 [2,8]
+ CRUSH rule 0 x 595 [8,1]
+ CRUSH rule 0 x 596 [2,8]
+ CRUSH rule 0 x 597 [1,8]
+ CRUSH rule 0 x 598 [2,8]
+ CRUSH rule 0 x 599 [4,1,8]
+ CRUSH rule 0 x 600 [8,1,4]
+ CRUSH rule 0 x 601 [1,8,4]
+ CRUSH rule 0 x 602 [8,2]
+ CRUSH rule 0 x 603 [1,8]
+ CRUSH rule 0 x 604 [8,2]
+ CRUSH rule 0 x 605 [2,8]
+ CRUSH rule 0 x 606 [2,6,4]
+ CRUSH rule 0 x 607 [2,4,8]
+ CRUSH rule 0 x 608 [4,1,8]
+ CRUSH rule 0 x 609 [4,2,8]
+ CRUSH rule 0 x 610 [8,2]
+ CRUSH rule 0 x 611 [1,8]
+ CRUSH rule 0 x 612 [2,8]
+ CRUSH rule 0 x 613 [8,1,4]
+ CRUSH rule 0 x 614 [8,2,4]
+ CRUSH rule 0 x 615 [8,2,4]
+ CRUSH rule 0 x 616 [1,8]
+ CRUSH rule 0 x 617 [8,2,4]
+ CRUSH rule 0 x 618 [8,4,1]
+ CRUSH rule 0 x 619 [4,1,8]
+ CRUSH rule 0 x 620 [1,8]
+ CRUSH rule 0 x 621 [8,1]
+ CRUSH rule 0 x 622 [2,4,8]
+ CRUSH rule 0 x 623 [2,8]
+ CRUSH rule 0 x 624 [4,2,8]
+ CRUSH rule 0 x 625 [2,8]
+ CRUSH rule 0 x 626 [8,2,4]
+ CRUSH rule 0 x 627 [2,8,4]
+ CRUSH rule 0 x 628 [8,1]
+ CRUSH rule 0 x 629 [2,8,4]
+ CRUSH rule 0 x 630 [2,8]
+ CRUSH rule 0 x 631 [1,8,4]
+ CRUSH rule 0 x 632 [8,2]
+ CRUSH rule 0 x 633 [8,2]
+ CRUSH rule 0 x 634 [1,8]
+ CRUSH rule 0 x 635 [4,8,2]
+ CRUSH rule 0 x 636 [1,4,8]
+ CRUSH rule 0 x 637 [1,8]
+ CRUSH rule 0 x 638 [8,2,4]
+ CRUSH rule 0 x 639 [2,8]
+ CRUSH rule 0 x 640 [2,8]
+ CRUSH rule 0 x 641 [8,2]
+ CRUSH rule 0 x 642 [2,8]
+ CRUSH rule 0 x 643 [1,8]
+ CRUSH rule 0 x 644 [8,1]
+ CRUSH rule 0 x 645 [8,2]
+ CRUSH rule 0 x 646 [8,1,4]
+ CRUSH rule 0 x 647 [8,1]
+ CRUSH rule 0 x 648 [1,8]
+ CRUSH rule 0 x 649 [4,8,1]
+ CRUSH rule 0 x 650 [8,4,1]
+ CRUSH rule 0 x 651 [4,6,1]
+ CRUSH rule 0 x 652 [4,8,1]
+ CRUSH rule 0 x 653 [8,2]
+ CRUSH rule 0 x 654 [6,2]
+ CRUSH rule 0 x 655 [1,4,8]
+ CRUSH rule 0 x 656 [8,1]
+ CRUSH rule 0 x 657 [6,1]
+ CRUSH rule 0 x 658 [8,2]
+ CRUSH rule 0 x 659 [4,8,1]
+ CRUSH rule 0 x 660 [8,2]
+ CRUSH rule 0 x 661 [1,8]
+ CRUSH rule 0 x 662 [2,8]
+ CRUSH rule 0 x 663 [1,4,8]
+ CRUSH rule 0 x 664 [1,4,8]
+ CRUSH rule 0 x 665 [4,6,1]
+ CRUSH rule 0 x 666 [2,8]
+ CRUSH rule 0 x 667 [1,4,8]
+ CRUSH rule 0 x 668 [4,8,2]
+ CRUSH rule 0 x 669 [6,4,1]
+ CRUSH rule 0 x 670 [4,1,8]
+ CRUSH rule 0 x 671 [2,8]
+ CRUSH rule 0 x 672 [4,2,8]
+ CRUSH rule 0 x 673 [4,2,8]
+ CRUSH rule 0 x 674 [1,8]
+ CRUSH rule 0 x 675 [1,8,4]
+ CRUSH rule 0 x 676 [2,4,8]
+ CRUSH rule 0 x 677 [4,1,8]
+ CRUSH rule 0 x 678 [2,4,8]
+ CRUSH rule 0 x 679 [8,2]
+ CRUSH rule 0 x 680 [2,8]
+ CRUSH rule 0 x 681 [8,1]
+ CRUSH rule 0 x 682 [1,4,8]
+ CRUSH rule 0 x 683 [1,4,8]
+ CRUSH rule 0 x 684 [8,2,4]
+ CRUSH rule 0 x 685 [8,2,4]
+ CRUSH rule 0 x 686 [1,4,8]
+ CRUSH rule 0 x 687 [6,2]
+ CRUSH rule 0 x 688 [4,8,2]
+ CRUSH rule 0 x 689 [8,4,1]
+ CRUSH rule 0 x 690 [8,1,4]
+ CRUSH rule 0 x 691 [1,8]
+ CRUSH rule 0 x 692 [8,1]
+ CRUSH rule 0 x 693 [8,4,2]
+ CRUSH rule 0 x 694 [8,4,2]
+ CRUSH rule 0 x 695 [2,8,4]
+ CRUSH rule 0 x 696 [1,8]
+ CRUSH rule 0 x 697 [8,1,4]
+ CRUSH rule 0 x 698 [8,2,4]
+ CRUSH rule 0 x 699 [1,8,4]
+ CRUSH rule 0 x 700 [1,6]
+ CRUSH rule 0 x 701 [1,8]
+ CRUSH rule 0 x 702 [2,8]
+ CRUSH rule 0 x 703 [8,1]
+ CRUSH rule 0 x 704 [1,4,8]
+ CRUSH rule 0 x 705 [8,1,4]
+ CRUSH rule 0 x 706 [1,4,8]
+ CRUSH rule 0 x 707 [8,4,1]
+ CRUSH rule 0 x 708 [4,8,1]
+ CRUSH rule 0 x 709 [8,1]
+ CRUSH rule 0 x 710 [8,1]
+ CRUSH rule 0 x 711 [2,4,8]
+ CRUSH rule 0 x 712 [2,8]
+ CRUSH rule 0 x 713 [8,4,1]
+ CRUSH rule 0 x 714 [2,8]
+ CRUSH rule 0 x 715 [1,8]
+ CRUSH rule 0 x 716 [4,8,1]
+ CRUSH rule 0 x 717 [8,2,4]
+ CRUSH rule 0 x 718 [8,1]
+ CRUSH rule 0 x 719 [2,6,4]
+ CRUSH rule 0 x 720 [8,1,4]
+ CRUSH rule 0 x 721 [4,6,1]
+ CRUSH rule 0 x 722 [8,1]
+ CRUSH rule 0 x 723 [4,2,8]
+ CRUSH rule 0 x 724 [2,6]
+ CRUSH rule 0 x 725 [1,8]
+ CRUSH rule 0 x 726 [4,8,2]
+ CRUSH rule 0 x 727 [4,8,2]
+ CRUSH rule 0 x 728 [2,8,4]
+ CRUSH rule 0 x 729 [8,1]
+ CRUSH rule 0 x 730 [4,8,2]
+ CRUSH rule 0 x 731 [4,1,8]
+ CRUSH rule 0 x 732 [1,8]
+ CRUSH rule 0 x 733 [4,8,2]
+ CRUSH rule 0 x 734 [8,4,1]
+ CRUSH rule 0 x 735 [4,8,1]
+ CRUSH rule 0 x 736 [4,8,1]
+ CRUSH rule 0 x 737 [1,8,4]
+ CRUSH rule 0 x 738 [4,1,8]
+ CRUSH rule 0 x 739 [2,8]
+ CRUSH rule 0 x 740 [1,8,4]
+ CRUSH rule 0 x 741 [8,2]
+ CRUSH rule 0 x 742 [8,2]
+ CRUSH rule 0 x 743 [8,1,4]
+ CRUSH rule 0 x 744 [4,8,1]
+ CRUSH rule 0 x 745 [1,8]
+ CRUSH rule 0 x 746 [1,8]
+ CRUSH rule 0 x 747 [8,2]
+ CRUSH rule 0 x 748 [2,8,4]
+ CRUSH rule 0 x 749 [4,8,2]
+ CRUSH rule 0 x 750 [1,8,4]
+ CRUSH rule 0 x 751 [2,8]
+ CRUSH rule 0 x 752 [8,1]
+ CRUSH rule 0 x 753 [8,4,2]
+ CRUSH rule 0 x 754 [8,4,2]
+ CRUSH rule 0 x 755 [1,8,4]
+ CRUSH rule 0 x 756 [8,1]
+ CRUSH rule 0 x 757 [8,1,4]
+ CRUSH rule 0 x 758 [8,1]
+ CRUSH rule 0 x 759 [8,4,2]
+ CRUSH rule 0 x 760 [1,4,8]
+ CRUSH rule 0 x 761 [2,6]
+ CRUSH rule 0 x 762 [2,8]
+ CRUSH rule 0 x 763 [8,4,1]
+ CRUSH rule 0 x 764 [1,8]
+ CRUSH rule 0 x 765 [8,2]
+ CRUSH rule 0 x 766 [8,1]
+ CRUSH rule 0 x 767 [1,8,4]
+ CRUSH rule 0 x 768 [8,4,2]
+ CRUSH rule 0 x 769 [8,2,4]
+ CRUSH rule 0 x 770 [8,1,4]
+ CRUSH rule 0 x 771 [8,2,4]
+ CRUSH rule 0 x 772 [8,4,2]
+ CRUSH rule 0 x 773 [4,2,8]
+ CRUSH rule 0 x 774 [8,1]
+ CRUSH rule 0 x 775 [8,4,2]
+ CRUSH rule 0 x 776 [6,2]
+ CRUSH rule 0 x 777 [4,1,8]
+ CRUSH rule 0 x 778 [1,6,4]
+ CRUSH rule 0 x 779 [2,8]
+ CRUSH rule 0 x 780 [2,4,8]
+ CRUSH rule 0 x 781 [8,2]
+ CRUSH rule 0 x 782 [4,2,8]
+ CRUSH rule 0 x 783 [8,1,4]
+ CRUSH rule 0 x 784 [1,4,8]
+ CRUSH rule 0 x 785 [8,1,4]
+ CRUSH rule 0 x 786 [8,1]
+ CRUSH rule 0 x 787 [1,8,4]
+ CRUSH rule 0 x 788 [8,2,4]
+ CRUSH rule 0 x 789 [1,8]
+ CRUSH rule 0 x 790 [8,1]
+ CRUSH rule 0 x 791 [4,8,2]
+ CRUSH rule 0 x 792 [4,8,1]
+ CRUSH rule 0 x 793 [8,2,4]
+ CRUSH rule 0 x 794 [2,8,4]
+ CRUSH rule 0 x 795 [1,8]
+ CRUSH rule 0 x 796 [8,1]
+ CRUSH rule 0 x 797 [2,4,8]
+ CRUSH rule 0 x 798 [6,1]
+ CRUSH rule 0 x 799 [4,2,8]
+ CRUSH rule 0 x 800 [2,8]
+ CRUSH rule 0 x 801 [4,8,2]
+ CRUSH rule 0 x 802 [1,8,4]
+ CRUSH rule 0 x 803 [2,8]
+ CRUSH rule 0 x 804 [8,2]
+ CRUSH rule 0 x 805 [8,2]
+ CRUSH rule 0 x 806 [1,4,8]
+ CRUSH rule 0 x 807 [4,8,2]
+ CRUSH rule 0 x 808 [8,2]
+ CRUSH rule 0 x 809 [1,8]
+ CRUSH rule 0 x 810 [8,2]
+ CRUSH rule 0 x 811 [8,1]
+ CRUSH rule 0 x 812 [8,4,1]
+ CRUSH rule 0 x 813 [8,4,2]
+ CRUSH rule 0 x 814 [8,2]
+ CRUSH rule 0 x 815 [4,1,8]
+ CRUSH rule 0 x 816 [2,8]
+ CRUSH rule 0 x 817 [8,2]
+ CRUSH rule 0 x 818 [1,8]
+ CRUSH rule 0 x 819 [1,8]
+ CRUSH rule 0 x 820 [4,8,1]
+ CRUSH rule 0 x 821 [4,8,2]
+ CRUSH rule 0 x 822 [2,4,8]
+ CRUSH rule 0 x 823 [4,8,2]
+ CRUSH rule 0 x 824 [8,1]
+ CRUSH rule 0 x 825 [2,8,4]
+ CRUSH rule 0 x 826 [8,1,4]
+ CRUSH rule 0 x 827 [2,6,4]
+ CRUSH rule 0 x 828 [2,8]
+ CRUSH rule 0 x 829 [8,1]
+ CRUSH rule 0 x 830 [2,4,8]
+ CRUSH rule 0 x 831 [1,8]
+ CRUSH rule 0 x 832 [4,8,2]
+ CRUSH rule 0 x 833 [2,8]
+ CRUSH rule 0 x 834 [1,8]
+ CRUSH rule 0 x 835 [8,4,1]
+ CRUSH rule 0 x 836 [4,8,1]
+ CRUSH rule 0 x 837 [8,4,1]
+ CRUSH rule 0 x 838 [6,2,4]
+ CRUSH rule 0 x 839 [2,8]
+ CRUSH rule 0 x 840 [8,2]
+ CRUSH rule 0 x 841 [4,8,1]
+ CRUSH rule 0 x 842 [2,8]
+ CRUSH rule 0 x 843 [8,4,2]
+ CRUSH rule 0 x 844 [8,2]
+ CRUSH rule 0 x 845 [4,8,2]
+ CRUSH rule 0 x 846 [4,2,8]
+ CRUSH rule 0 x 847 [2,8]
+ CRUSH rule 0 x 848 [2,8,4]
+ CRUSH rule 0 x 849 [4,8,1]
+ CRUSH rule 0 x 850 [1,6]
+ CRUSH rule 0 x 851 [6,1]
+ CRUSH rule 0 x 852 [8,4,2]
+ CRUSH rule 0 x 853 [6,2]
+ CRUSH rule 0 x 854 [8,2]
+ CRUSH rule 0 x 855 [8,2]
+ CRUSH rule 0 x 856 [8,4,1]
+ CRUSH rule 0 x 857 [8,1]
+ CRUSH rule 0 x 858 [6,2]
+ CRUSH rule 0 x 859 [8,2,4]
+ CRUSH rule 0 x 860 [2,8]
+ CRUSH rule 0 x 861 [8,1]
+ CRUSH rule 0 x 862 [8,1]
+ CRUSH rule 0 x 863 [8,1]
+ CRUSH rule 0 x 864 [8,1]
+ CRUSH rule 0 x 865 [8,1]
+ CRUSH rule 0 x 866 [8,2]
+ CRUSH rule 0 x 867 [8,1]
+ CRUSH rule 0 x 868 [8,2]
+ CRUSH rule 0 x 869 [8,4,2]
+ CRUSH rule 0 x 870 [2,8]
+ CRUSH rule 0 x 871 [1,8]
+ CRUSH rule 0 x 872 [1,8]
+ CRUSH rule 0 x 873 [4,8,1]
+ CRUSH rule 0 x 874 [2,6]
+ CRUSH rule 0 x 875 [2,8,4]
+ CRUSH rule 0 x 876 [4,8,1]
+ CRUSH rule 0 x 877 [8,4,2]
+ CRUSH rule 0 x 878 [2,8]
+ CRUSH rule 0 x 879 [8,1]
+ CRUSH rule 0 x 880 [1,8]
+ CRUSH rule 0 x 881 [4,8,2]
+ CRUSH rule 0 x 882 [1,8]
+ CRUSH rule 0 x 883 [2,4,8]
+ CRUSH rule 0 x 884 [8,1,4]
+ CRUSH rule 0 x 885 [4,1,8]
+ CRUSH rule 0 x 886 [8,2]
+ CRUSH rule 0 x 887 [8,4,2]
+ CRUSH rule 0 x 888 [8,1]
+ CRUSH rule 0 x 889 [2,8]
+ CRUSH rule 0 x 890 [8,1,4]
+ CRUSH rule 0 x 891 [1,8]
+ CRUSH rule 0 x 892 [8,2,4]
+ CRUSH rule 0 x 893 [2,6]
+ CRUSH rule 0 x 894 [8,4,1]
+ CRUSH rule 0 x 895 [4,1,8]
+ CRUSH rule 0 x 896 [1,8]
+ CRUSH rule 0 x 897 [2,8]
+ CRUSH rule 0 x 898 [1,4,8]
+ CRUSH rule 0 x 899 [1,8]
+ CRUSH rule 0 x 900 [4,1,8]
+ CRUSH rule 0 x 901 [2,8]
+ CRUSH rule 0 x 902 [8,4,2]
+ CRUSH rule 0 x 903 [8,1]
+ CRUSH rule 0 x 904 [8,2]
+ CRUSH rule 0 x 905 [8,2]
+ CRUSH rule 0 x 906 [1,8]
+ CRUSH rule 0 x 907 [8,2]
+ CRUSH rule 0 x 908 [8,1]
+ CRUSH rule 0 x 909 [2,8]
+ CRUSH rule 0 x 910 [8,1]
+ CRUSH rule 0 x 911 [8,1]
+ CRUSH rule 0 x 912 [1,8]
+ CRUSH rule 0 x 913 [8,1,4]
+ CRUSH rule 0 x 914 [6,4,1]
+ CRUSH rule 0 x 915 [8,2]
+ CRUSH rule 0 x 916 [4,1,8]
+ CRUSH rule 0 x 917 [1,4,8]
+ CRUSH rule 0 x 918 [8,2]
+ CRUSH rule 0 x 919 [8,2]
+ CRUSH rule 0 x 920 [8,2]
+ CRUSH rule 0 x 921 [1,8]
+ CRUSH rule 0 x 922 [8,4,1]
+ CRUSH rule 0 x 923 [4,8,2]
+ CRUSH rule 0 x 924 [1,8]
+ CRUSH rule 0 x 925 [4,8,2]
+ CRUSH rule 0 x 926 [2,8]
+ CRUSH rule 0 x 927 [1,8,4]
+ CRUSH rule 0 x 928 [8,1]
+ CRUSH rule 0 x 929 [4,2,8]
+ CRUSH rule 0 x 930 [2,8]
+ CRUSH rule 0 x 931 [2,8]
+ CRUSH rule 0 x 932 [4,2,8]
+ CRUSH rule 0 x 933 [8,4,2]
+ CRUSH rule 0 x 934 [8,2]
+ CRUSH rule 0 x 935 [8,2]
+ CRUSH rule 0 x 936 [1,8]
+ CRUSH rule 0 x 937 [4,8,1]
+ CRUSH rule 0 x 938 [8,4,2]
+ CRUSH rule 0 x 939 [2,8,4]
+ CRUSH rule 0 x 940 [8,2]
+ CRUSH rule 0 x 941 [2,6]
+ CRUSH rule 0 x 942 [1,8]
+ CRUSH rule 0 x 943 [8,2]
+ CRUSH rule 0 x 944 [8,1]
+ CRUSH rule 0 x 945 [8,2,4]
+ CRUSH rule 0 x 946 [2,8,4]
+ CRUSH rule 0 x 947 [2,8]
+ CRUSH rule 0 x 948 [8,1]
+ CRUSH rule 0 x 949 [6,2]
+ CRUSH rule 0 x 950 [8,1]
+ CRUSH rule 0 x 951 [8,2]
+ CRUSH rule 0 x 952 [2,8,4]
+ CRUSH rule 0 x 953 [1,4,8]
+ CRUSH rule 0 x 954 [2,8]
+ CRUSH rule 0 x 955 [8,1,4]
+ CRUSH rule 0 x 956 [1,8,4]
+ CRUSH rule 0 x 957 [8,1,4]
+ CRUSH rule 0 x 958 [8,4,2]
+ CRUSH rule 0 x 959 [4,1,8]
+ CRUSH rule 0 x 960 [6,2]
+ CRUSH rule 0 x 961 [1,8]
+ CRUSH rule 0 x 962 [8,4,2]
+ CRUSH rule 0 x 963 [2,4,8]
+ CRUSH rule 0 x 964 [2,8]
+ CRUSH rule 0 x 965 [8,1]
+ CRUSH rule 0 x 966 [4,8,1]
+ CRUSH rule 0 x 967 [8,4,2]
+ CRUSH rule 0 x 968 [8,2]
+ CRUSH rule 0 x 969 [8,2,4]
+ CRUSH rule 0 x 970 [2,8,4]
+ CRUSH rule 0 x 971 [1,8]
+ CRUSH rule 0 x 972 [1,8]
+ CRUSH rule 0 x 973 [1,8]
+ CRUSH rule 0 x 974 [4,1,8]
+ CRUSH rule 0 x 975 [4,8,1]
+ CRUSH rule 0 x 976 [4,8,2]
+ CRUSH rule 0 x 977 [8,4,2]
+ CRUSH rule 0 x 978 [8,2,4]
+ CRUSH rule 0 x 979 [8,2,4]
+ CRUSH rule 0 x 980 [8,2,4]
+ CRUSH rule 0 x 981 [8,1]
+ CRUSH rule 0 x 982 [1,8]
+ CRUSH rule 0 x 983 [4,8,1]
+ CRUSH rule 0 x 984 [2,8]
+ CRUSH rule 0 x 985 [2,4,8]
+ CRUSH rule 0 x 986 [8,4,2]
+ CRUSH rule 0 x 987 [2,8]
+ CRUSH rule 0 x 988 [1,4,6]
+ CRUSH rule 0 x 989 [1,8]
+ CRUSH rule 0 x 990 [1,8,4]
+ CRUSH rule 0 x 991 [1,4,8]
+ CRUSH rule 0 x 992 [8,1,4]
+ CRUSH rule 0 x 993 [2,8,4]
+ CRUSH rule 0 x 994 [4,8,1]
+ CRUSH rule 0 x 995 [8,1,4]
+ CRUSH rule 0 x 996 [8,4,1]
+ CRUSH rule 0 x 997 [8,4,1]
+ CRUSH rule 0 x 998 [8,1,4]
+ CRUSH rule 0 x 999 [1,8,4]
+ CRUSH rule 0 x 1000 [8,4,1]
+ CRUSH rule 0 x 1001 [2,8]
+ CRUSH rule 0 x 1002 [1,8]
+ CRUSH rule 0 x 1003 [2,8]
+ CRUSH rule 0 x 1004 [8,1,4]
+ CRUSH rule 0 x 1005 [8,1]
+ CRUSH rule 0 x 1006 [1,8,4]
+ CRUSH rule 0 x 1007 [1,4,8]
+ CRUSH rule 0 x 1008 [1,8]
+ CRUSH rule 0 x 1009 [6,4,1]
+ CRUSH rule 0 x 1010 [1,8]
+ CRUSH rule 0 x 1011 [4,2,8]
+ CRUSH rule 0 x 1012 [1,8]
+ CRUSH rule 0 x 1013 [2,8]
+ CRUSH rule 0 x 1014 [2,8,4]
+ CRUSH rule 0 x 1015 [8,2]
+ CRUSH rule 0 x 1016 [2,4,8]
+ CRUSH rule 0 x 1017 [6,1,4]
+ CRUSH rule 0 x 1018 [4,1,8]
+ CRUSH rule 0 x 1019 [4,8,1]
+ CRUSH rule 0 x 1020 [1,8]
+ CRUSH rule 0 x 1021 [2,8]
+ CRUSH rule 0 x 1022 [1,8,4]
+ CRUSH rule 0 x 1023 [4,2,8]
+ rule 0 (choose) num_rep 3 result size == 2:\t501/1024 (esc)
+ rule 0 (choose) num_rep 3 result size == 3:\t523/1024 (esc)
+ rule 1 (choose-two), x = 0..1023, numrep = 2..3
+ CRUSH rule 1 x 0 [2,1]
+ CRUSH rule 1 x 1 [2,8]
+ CRUSH rule 1 x 2 [1,8]
+ CRUSH rule 1 x 3 [8,1]
+ CRUSH rule 1 x 4 [4,8]
+ CRUSH rule 1 x 5 [8,2]
+ CRUSH rule 1 x 6 [2,8]
+ CRUSH rule 1 x 7 [4,8]
+ CRUSH rule 1 x 8 [4,2]
+ CRUSH rule 1 x 9 [2,4]
+ CRUSH rule 1 x 10 [2,1]
+ CRUSH rule 1 x 11 [2,8]
+ CRUSH rule 1 x 12 [2,1]
+ CRUSH rule 1 x 13 [4,8]
+ CRUSH rule 1 x 14 [8,1]
+ CRUSH rule 1 x 15 [8,2]
+ CRUSH rule 1 x 16 [8,1]
+ CRUSH rule 1 x 17 [4,8]
+ CRUSH rule 1 x 18 [1,6]
+ CRUSH rule 1 x 19 [8,4]
+ CRUSH rule 1 x 20 [2,1]
+ CRUSH rule 1 x 21 [8,1]
+ CRUSH rule 1 x 22 [8]
+ CRUSH rule 1 x 23 [4,8]
+ CRUSH rule 1 x 24 [1,2]
+ CRUSH rule 1 x 25 [4,8]
+ CRUSH rule 1 x 26 [2,8]
+ CRUSH rule 1 x 27 [4,1]
+ CRUSH rule 1 x 28 [8,2]
+ CRUSH rule 1 x 29 [8,4]
+ CRUSH rule 1 x 30 [4,8]
+ CRUSH rule 1 x 31 [8,2]
+ CRUSH rule 1 x 32 [2,6]
+ CRUSH rule 1 x 33 [2,8]
+ CRUSH rule 1 x 34 [2,1]
+ CRUSH rule 1 x 35 [1,8]
+ CRUSH rule 1 x 36 [8,1]
+ CRUSH rule 1 x 37 [1,8]
+ CRUSH rule 1 x 38 [4,8]
+ CRUSH rule 1 x 39 [8,1]
+ CRUSH rule 1 x 40 [8,2]
+ CRUSH rule 1 x 41 [2,1]
+ CRUSH rule 1 x 42 [1,2]
+ CRUSH rule 1 x 43 [1,2]
+ CRUSH rule 1 x 44 [1,8]
+ CRUSH rule 1 x 45 [8,2]
+ CRUSH rule 1 x 46 [2,1]
+ CRUSH rule 1 x 47 [4,2]
+ CRUSH rule 1 x 48 [8,1]
+ CRUSH rule 1 x 49 [8,2]
+ CRUSH rule 1 x 50 [4,8]
+ CRUSH rule 1 x 51 [2,8]
+ CRUSH rule 1 x 52 [8,2]
+ CRUSH rule 1 x 53 [4,2]
+ CRUSH rule 1 x 54 [8,4]
+ CRUSH rule 1 x 55 [8,1]
+ CRUSH rule 1 x 56 [8,4]
+ CRUSH rule 1 x 57 [2,1]
+ CRUSH rule 1 x 58 [1,2]
+ CRUSH rule 1 x 59 [8,2]
+ CRUSH rule 1 x 60 [4,8]
+ CRUSH rule 1 x 61 [4,8]
+ CRUSH rule 1 x 62 [8,1]
+ CRUSH rule 1 x 63 [8,1]
+ CRUSH rule 1 x 64 [4,2]
+ CRUSH rule 1 x 65 [8,4]
+ CRUSH rule 1 x 66 [4,2]
+ CRUSH rule 1 x 67 [4,2]
+ CRUSH rule 1 x 68 [1,2]
+ CRUSH rule 1 x 69 [1,2]
+ CRUSH rule 1 x 70 [8,2]
+ CRUSH rule 1 x 71 [2,8]
+ CRUSH rule 1 x 72 [8,1]
+ CRUSH rule 1 x 73 [2,8]
+ CRUSH rule 1 x 74 [1,8]
+ CRUSH rule 1 x 75 [4,2]
+ CRUSH rule 1 x 76 [4,1]
+ CRUSH rule 1 x 77 [8,2]
+ CRUSH rule 1 x 78 [1]
+ CRUSH rule 1 x 79 [4,1]
+ CRUSH rule 1 x 80 [2,4]
+ CRUSH rule 1 x 81 [2,1]
+ CRUSH rule 1 x 82 [6,1]
+ CRUSH rule 1 x 83 [2,8]
+ CRUSH rule 1 x 84 [8,2]
+ CRUSH rule 1 x 85 [4,8]
+ CRUSH rule 1 x 86 [2,1]
+ CRUSH rule 1 x 87 [2,8]
+ CRUSH rule 1 x 88 [1,6]
+ CRUSH rule 1 x 89 [2,1]
+ CRUSH rule 1 x 90 [8,2]
+ CRUSH rule 1 x 91 [4,8]
+ CRUSH rule 1 x 92 [1,8]
+ CRUSH rule 1 x 93 [8,4]
+ CRUSH rule 1 x 94 [1,2]
+ CRUSH rule 1 x 95 [8,1]
+ CRUSH rule 1 x 96 [1,8]
+ CRUSH rule 1 x 97 [8,1]
+ CRUSH rule 1 x 98 [2,1]
+ CRUSH rule 1 x 99 [2,8]
+ CRUSH rule 1 x 100 [1,8]
+ CRUSH rule 1 x 101 [8,2]
+ CRUSH rule 1 x 102 [2,1]
+ CRUSH rule 1 x 103 [2,8]
+ CRUSH rule 1 x 104 [8,4]
+ CRUSH rule 1 x 105 [2,4]
+ CRUSH rule 1 x 106 [1,8]
+ CRUSH rule 1 x 107 [2,1]
+ CRUSH rule 1 x 108 [8,2]
+ CRUSH rule 1 x 109 [1,2]
+ CRUSH rule 1 x 110 [4,2]
+ CRUSH rule 1 x 111 [2,1]
+ CRUSH rule 1 x 112 [2,1]
+ CRUSH rule 1 x 113 [8,2]
+ CRUSH rule 1 x 114 [8,4]
+ CRUSH rule 1 x 115 [8,2]
+ CRUSH rule 1 x 116 [1,2]
+ CRUSH rule 1 x 117 [6,8]
+ CRUSH rule 1 x 118 [2,8]
+ CRUSH rule 1 x 119 [2,8]
+ CRUSH rule 1 x 120 [2,1]
+ CRUSH rule 1 x 121 [2,1]
+ CRUSH rule 1 x 122 [8,1]
+ CRUSH rule 1 x 123 [2,8]
+ CRUSH rule 1 x 124 [8]
+ CRUSH rule 1 x 125 [1,8]
+ CRUSH rule 1 x 126 [8,2]
+ CRUSH rule 1 x 127 [4,8]
+ CRUSH rule 1 x 128 [2,8]
+ CRUSH rule 1 x 129 [2,1]
+ CRUSH rule 1 x 130 [4,8]
+ CRUSH rule 1 x 131 [1,2]
+ CRUSH rule 1 x 132 [1,2]
+ CRUSH rule 1 x 133 [8,2]
+ CRUSH rule 1 x 134 [1,8]
+ CRUSH rule 1 x 135 [4,8]
+ CRUSH rule 1 x 136 [2,1]
+ CRUSH rule 1 x 137 [8,4]
+ CRUSH rule 1 x 138 [8,4]
+ CRUSH rule 1 x 139 [4,2]
+ CRUSH rule 1 x 140 [1,8]
+ CRUSH rule 1 x 141 [8,1]
+ CRUSH rule 1 x 142 [4,6]
+ CRUSH rule 1 x 143 [4,8]
+ CRUSH rule 1 x 144 [8,1]
+ CRUSH rule 1 x 145 [8,2]
+ CRUSH rule 1 x 146 [2,8]
+ CRUSH rule 1 x 147 [2,8]
+ CRUSH rule 1 x 148 [4,1]
+ CRUSH rule 1 x 149 [4,8]
+ CRUSH rule 1 x 150 [1,8]
+ CRUSH rule 1 x 151 [1,2]
+ CRUSH rule 1 x 152 [8,1]
+ CRUSH rule 1 x 153 [8,4]
+ CRUSH rule 1 x 154 [4,2]
+ CRUSH rule 1 x 155 [4,8]
+ CRUSH rule 1 x 156 [4,8]
+ CRUSH rule 1 x 157 [2,1]
+ CRUSH rule 1 x 158 [2,8]
+ CRUSH rule 1 x 159 [8,2]
+ CRUSH rule 1 x 160 [2,8]
+ CRUSH rule 1 x 161 [1,4]
+ CRUSH rule 1 x 162 [1,8]
+ CRUSH rule 1 x 163 [4,8]
+ CRUSH rule 1 x 164 [8,2]
+ CRUSH rule 1 x 165 [8,2]
+ CRUSH rule 1 x 166 [2,1]
+ CRUSH rule 1 x 167 [1,2]
+ CRUSH rule 1 x 168 [4,2]
+ CRUSH rule 1 x 169 [2,8]
+ CRUSH rule 1 x 170 [1,2]
+ CRUSH rule 1 x 171 [8,4]
+ CRUSH rule 1 x 172 [1,8]
+ CRUSH rule 1 x 173 [8,4]
+ CRUSH rule 1 x 174 [1,2]
+ CRUSH rule 1 x 175 [8,1]
+ CRUSH rule 1 x 176 [2,1]
+ CRUSH rule 1 x 177 [8,2]
+ CRUSH rule 1 x 178 [4,2]
+ CRUSH rule 1 x 179 [8,1]
+ CRUSH rule 1 x 180 [1,8]
+ CRUSH rule 1 x 181 [8,2]
+ CRUSH rule 1 x 182 [8,1]
+ CRUSH rule 1 x 183 [8,1]
+ CRUSH rule 1 x 184 [4,8]
+ CRUSH rule 1 x 185 [8,4]
+ CRUSH rule 1 x 186 [2,1]
+ CRUSH rule 1 x 187 [1,8]
+ CRUSH rule 1 x 188 [1,8]
+ CRUSH rule 1 x 189 [1,8]
+ CRUSH rule 1 x 190 [1,2]
+ CRUSH rule 1 x 191 [8,4]
+ CRUSH rule 1 x 192 [4,1]
+ CRUSH rule 1 x 193 [4,2]
+ CRUSH rule 1 x 194 [1,8]
+ CRUSH rule 1 x 195 [8,4]
+ CRUSH rule 1 x 196 [8,1]
+ CRUSH rule 1 x 197 [8,4]
+ CRUSH rule 1 x 198 [2,1]
+ CRUSH rule 1 x 199 [1,4]
+ CRUSH rule 1 x 200 [1,8]
+ CRUSH rule 1 x 201 [8,1]
+ CRUSH rule 1 x 202 [8,2]
+ CRUSH rule 1 x 203 [1,8]
+ CRUSH rule 1 x 204 [2,1]
+ CRUSH rule 1 x 205 [1,8]
+ CRUSH rule 1 x 206 [1,2]
+ CRUSH rule 1 x 207 [2,1]
+ CRUSH rule 1 x 208 [8,1]
+ CRUSH rule 1 x 209 [1,2]
+ CRUSH rule 1 x 210 [1,2]
+ CRUSH rule 1 x 211 [4,8]
+ CRUSH rule 1 x 212 [8,1]
+ CRUSH rule 1 x 213 [8,4]
+ CRUSH rule 1 x 214 [8]
+ CRUSH rule 1 x 215 [8,1]
+ CRUSH rule 1 x 216 [8,1]
+ CRUSH rule 1 x 217 [1,2]
+ CRUSH rule 1 x 218 [2,8]
+ CRUSH rule 1 x 219 [2,8]
+ CRUSH rule 1 x 220 [4,8]
+ CRUSH rule 1 x 221 [1,8]
+ CRUSH rule 1 x 222 [8,1]
+ CRUSH rule 1 x 223 [1,2]
+ CRUSH rule 1 x 224 [1,4]
+ CRUSH rule 1 x 225 [8,2]
+ CRUSH rule 1 x 226 [8,2]
+ CRUSH rule 1 x 227 [4,8]
+ CRUSH rule 1 x 228 [1,8]
+ CRUSH rule 1 x 229 [4,1]
+ CRUSH rule 1 x 230 [4,8]
+ CRUSH rule 1 x 231 [4,8]
+ CRUSH rule 1 x 232 [2,8]
+ CRUSH rule 1 x 233 [8,2]
+ CRUSH rule 1 x 234 [1,2]
+ CRUSH rule 1 x 235 [4,8]
+ CRUSH rule 1 x 236 [1,2]
+ CRUSH rule 1 x 237 [4,8]
+ CRUSH rule 1 x 238 [8]
+ CRUSH rule 1 x 239 [8,6]
+ CRUSH rule 1 x 240 [4,8]
+ CRUSH rule 1 x 241 [2,1]
+ CRUSH rule 1 x 242 [2,8]
+ CRUSH rule 1 x 243 [8,2]
+ CRUSH rule 1 x 244 [4,8]
+ CRUSH rule 1 x 245 [8,2]
+ CRUSH rule 1 x 246 [1,2]
+ CRUSH rule 1 x 247 [8,2]
+ CRUSH rule 1 x 248 [8,2]
+ CRUSH rule 1 x 249 [2,1]
+ CRUSH rule 1 x 250 [2,1]
+ CRUSH rule 1 x 251 [2,8]
+ CRUSH rule 1 x 252 [4,8]
+ CRUSH rule 1 x 253 [1,2]
+ CRUSH rule 1 x 254 [4,8]
+ CRUSH rule 1 x 255 [1,8]
+ CRUSH rule 1 x 256 [4,8]
+ CRUSH rule 1 x 257 [2,8]
+ CRUSH rule 1 x 258 [4]
+ CRUSH rule 1 x 259 [8,6]
+ CRUSH rule 1 x 260 [8,2]
+ CRUSH rule 1 x 261 [8,1]
+ CRUSH rule 1 x 262 [8,2]
+ CRUSH rule 1 x 263 [8,4]
+ CRUSH rule 1 x 264 [1,8]
+ CRUSH rule 1 x 265 [8]
+ CRUSH rule 1 x 266 [8,2]
+ CRUSH rule 1 x 267 [2,8]
+ CRUSH rule 1 x 268 [1,8]
+ CRUSH rule 1 x 269 [1,8]
+ CRUSH rule 1 x 270 [4,1]
+ CRUSH rule 1 x 271 [8,4]
+ CRUSH rule 1 x 272 [2,8]
+ CRUSH rule 1 x 273 [4,1]
+ CRUSH rule 1 x 274 [8,2]
+ CRUSH rule 1 x 275 [4,8]
+ CRUSH rule 1 x 276 [8,1]
+ CRUSH rule 1 x 277 [8,1]
+ CRUSH rule 1 x 278 [8,2]
+ CRUSH rule 1 x 279 [8,4]
+ CRUSH rule 1 x 280 [2,8]
+ CRUSH rule 1 x 281 [8,2]
+ CRUSH rule 1 x 282 [1,2]
+ CRUSH rule 1 x 283 [8,2]
+ CRUSH rule 1 x 284 [8]
+ CRUSH rule 1 x 285 [4,8]
+ CRUSH rule 1 x 286 [2,1]
+ CRUSH rule 1 x 287 [1,2]
+ CRUSH rule 1 x 288 [8,1]
+ CRUSH rule 1 x 289 [4,8]
+ CRUSH rule 1 x 290 [1,4]
+ CRUSH rule 1 x 291 [1,2]
+ CRUSH rule 1 x 292 [8,2]
+ CRUSH rule 1 x 293 [8,1]
+ CRUSH rule 1 x 294 [8,4]
+ CRUSH rule 1 x 295 [4,8]
+ CRUSH rule 1 x 296 [4,1]
+ CRUSH rule 1 x 297 [8,2]
+ CRUSH rule 1 x 298 [1,2]
+ CRUSH rule 1 x 299 [2,1]
+ CRUSH rule 1 x 300 [8,2]
+ CRUSH rule 1 x 301 [1,8]
+ CRUSH rule 1 x 302 [8,1]
+ CRUSH rule 1 x 303 [8,4]
+ CRUSH rule 1 x 304 [2,8]
+ CRUSH rule 1 x 305 [2,8]
+ CRUSH rule 1 x 306 [1,8]
+ CRUSH rule 1 x 307 [2,1]
+ CRUSH rule 1 x 308 [2,8]
+ CRUSH rule 1 x 309 [8,2]
+ CRUSH rule 1 x 310 [4,6]
+ CRUSH rule 1 x 311 [4,2]
+ CRUSH rule 1 x 312 [2,1]
+ CRUSH rule 1 x 313 [4,8]
+ CRUSH rule 1 x 314 [6,1]
+ CRUSH rule 1 x 315 [2,1]
+ CRUSH rule 1 x 316 [8,2]
+ CRUSH rule 1 x 317 [2,8]
+ CRUSH rule 1 x 318 [8,2]
+ CRUSH rule 1 x 319 [8,2]
+ CRUSH rule 1 x 320 [8,2]
+ CRUSH rule 1 x 321 [1,2]
+ CRUSH rule 1 x 322 [2,8]
+ CRUSH rule 1 x 323 [4,8]
+ CRUSH rule 1 x 324 [8,1]
+ CRUSH rule 1 x 325 [4,8]
+ CRUSH rule 1 x 326 [8,6]
+ CRUSH rule 1 x 327 [1,8]
+ CRUSH rule 1 x 328 [8,4]
+ CRUSH rule 1 x 329 [4,8]
+ CRUSH rule 1 x 330 [4,8]
+ CRUSH rule 1 x 331 [2,8]
+ CRUSH rule 1 x 332 [2,1]
+ CRUSH rule 1 x 333 [8,1]
+ CRUSH rule 1 x 334 [8,1]
+ CRUSH rule 1 x 335 [8,1]
+ CRUSH rule 1 x 336 [4,8]
+ CRUSH rule 1 x 337 [8,2]
+ CRUSH rule 1 x 338 [1,8]
+ CRUSH rule 1 x 339 [8]
+ CRUSH rule 1 x 340 [2,1]
+ CRUSH rule 1 x 341 [4,1]
+ CRUSH rule 1 x 342 [2,8]
+ CRUSH rule 1 x 343 [8]
+ CRUSH rule 1 x 344 [6,2]
+ CRUSH rule 1 x 345 [2,1]
+ CRUSH rule 1 x 346 [8,2]
+ CRUSH rule 1 x 347 [4,1]
+ CRUSH rule 1 x 348 [8,2]
+ CRUSH rule 1 x 349 [1,8]
+ CRUSH rule 1 x 350 [8,1]
+ CRUSH rule 1 x 351 [1,8]
+ CRUSH rule 1 x 352 [1,2]
+ CRUSH rule 1 x 353 [8,1]
+ CRUSH rule 1 x 354 [1,8]
+ CRUSH rule 1 x 355 [1,2]
+ CRUSH rule 1 x 356 [4,8]
+ CRUSH rule 1 x 357 [8,1]
+ CRUSH rule 1 x 358 [2,1]
+ CRUSH rule 1 x 359 [6,8]
+ CRUSH rule 1 x 360 [1,2]
+ CRUSH rule 1 x 361 [8,4]
+ CRUSH rule 1 x 362 [4,6]
+ CRUSH rule 1 x 363 [4,1]
+ CRUSH rule 1 x 364 [2,8]
+ CRUSH rule 1 x 365 [8,1]
+ CRUSH rule 1 x 366 [8,2]
+ CRUSH rule 1 x 367 [4,8]
+ CRUSH rule 1 x 368 [8,4]
+ CRUSH rule 1 x 369 [8]
+ CRUSH rule 1 x 370 [8,2]
+ CRUSH rule 1 x 371 [1,4]
+ CRUSH rule 1 x 372 [8,1]
+ CRUSH rule 1 x 373 [1,8]
+ CRUSH rule 1 x 374 [2,8]
+ CRUSH rule 1 x 375 [8,4]
+ CRUSH rule 1 x 376 [8,1]
+ CRUSH rule 1 x 377 [1,2]
+ CRUSH rule 1 x 378 [1,2]
+ CRUSH rule 1 x 379 [8,1]
+ CRUSH rule 1 x 380 [2,1]
+ CRUSH rule 1 x 381 [1,4]
+ CRUSH rule 1 x 382 [1,4]
+ CRUSH rule 1 x 383 [4,1]
+ CRUSH rule 1 x 384 [8,2]
+ CRUSH rule 1 x 385 [8,1]
+ CRUSH rule 1 x 386 [1,8]
+ CRUSH rule 1 x 387 [1,4]
+ CRUSH rule 1 x 388 [8,1]
+ CRUSH rule 1 x 389 [1,4]
+ CRUSH rule 1 x 390 [4,8]
+ CRUSH rule 1 x 391 [4,8]
+ CRUSH rule 1 x 392 [1,8]
+ CRUSH rule 1 x 393 [8,2]
+ CRUSH rule 1 x 394 [8,1]
+ CRUSH rule 1 x 395 [8,1]
+ CRUSH rule 1 x 396 [4,2]
+ CRUSH rule 1 x 397 [2,1]
+ CRUSH rule 1 x 398 [2,4]
+ CRUSH rule 1 x 399 [8,2]
+ CRUSH rule 1 x 400 [8,1]
+ CRUSH rule 1 x 401 [1,2]
+ CRUSH rule 1 x 402 [8,1]
+ CRUSH rule 1 x 403 [1,2]
+ CRUSH rule 1 x 404 [4,2]
+ CRUSH rule 1 x 405 [8,4]
+ CRUSH rule 1 x 406 [2,1]
+ CRUSH rule 1 x 407 [2,8]
+ CRUSH rule 1 x 408 [4,1]
+ CRUSH rule 1 x 409 [8,4]
+ CRUSH rule 1 x 410 [8,1]
+ CRUSH rule 1 x 411 [2,1]
+ CRUSH rule 1 x 412 [2,6]
+ CRUSH rule 1 x 413 [8,2]
+ CRUSH rule 1 x 414 [4,1]
+ CRUSH rule 1 x 415 [2,8]
+ CRUSH rule 1 x 416 [2,1]
+ CRUSH rule 1 x 417 [8,6]
+ CRUSH rule 1 x 418 [8,2]
+ CRUSH rule 1 x 419 [8,4]
+ CRUSH rule 1 x 420 [1,4]
+ CRUSH rule 1 x 421 [8,4]
+ CRUSH rule 1 x 422 [6,8]
+ CRUSH rule 1 x 423 [2,4]
+ CRUSH rule 1 x 424 [8,2]
+ CRUSH rule 1 x 425 [1,2]
+ CRUSH rule 1 x 426 [8]
+ CRUSH rule 1 x 427 [1,8]
+ CRUSH rule 1 x 428 [4,8]
+ CRUSH rule 1 x 429 [4,8]
+ CRUSH rule 1 x 430 [4,8]
+ CRUSH rule 1 x 431 [4,1]
+ CRUSH rule 1 x 432 [8,1]
+ CRUSH rule 1 x 433 [8,2]
+ CRUSH rule 1 x 434 [8,2]
+ CRUSH rule 1 x 435 [2,6]
+ CRUSH rule 1 x 436 [4,1]
+ CRUSH rule 1 x 437 [8,2]
+ CRUSH rule 1 x 438 [2,4]
+ CRUSH rule 1 x 439 [1,6]
+ CRUSH rule 1 x 440 [2,8]
+ CRUSH rule 1 x 441 [4,6]
+ CRUSH rule 1 x 442 [2,1]
+ CRUSH rule 1 x 443 [8,4]
+ CRUSH rule 1 x 444 [8,1]
+ CRUSH rule 1 x 445 [8,1]
+ CRUSH rule 1 x 446 [2,8]
+ CRUSH rule 1 x 447 [2,1]
+ CRUSH rule 1 x 448 [8,2]
+ CRUSH rule 1 x 449 [8,6]
+ CRUSH rule 1 x 450 [1,8]
+ CRUSH rule 1 x 451 [8,4]
+ CRUSH rule 1 x 452 [8]
+ CRUSH rule 1 x 453 [6,8]
+ CRUSH rule 1 x 454 [8,2]
+ CRUSH rule 1 x 455 [2,8]
+ CRUSH rule 1 x 456 [8,2]
+ CRUSH rule 1 x 457 [8,2]
+ CRUSH rule 1 x 458 [2,8]
+ CRUSH rule 1 x 459 [2,1]
+ CRUSH rule 1 x 460 [8,2]
+ CRUSH rule 1 x 461 [8,2]
+ CRUSH rule 1 x 462 [8,1]
+ CRUSH rule 1 x 463 [8,2]
+ CRUSH rule 1 x 464 [8,4]
+ CRUSH rule 1 x 465 [6,8]
+ CRUSH rule 1 x 466 [8,2]
+ CRUSH rule 1 x 467 [8,2]
+ CRUSH rule 1 x 468 [8,4]
+ CRUSH rule 1 x 469 [8,1]
+ CRUSH rule 1 x 470 [4,2]
+ CRUSH rule 1 x 471 [1,2]
+ CRUSH rule 1 x 472 [1,8]
+ CRUSH rule 1 x 473 [1,2]
+ CRUSH rule 1 x 474 [8,1]
+ CRUSH rule 1 x 475 [8,4]
+ CRUSH rule 1 x 476 [4,1]
+ CRUSH rule 1 x 477 [4,8]
+ CRUSH rule 1 x 478 [8,1]
+ CRUSH rule 1 x 479 [2,1]
+ CRUSH rule 1 x 480 [1,8]
+ CRUSH rule 1 x 481 [2,4]
+ CRUSH rule 1 x 482 [2,8]
+ CRUSH rule 1 x 483 [2,1]
+ CRUSH rule 1 x 484 [1,2]
+ CRUSH rule 1 x 485 [1,8]
+ CRUSH rule 1 x 486 [4,1]
+ CRUSH rule 1 x 487 [8,1]
+ CRUSH rule 1 x 488 [2,8]
+ CRUSH rule 1 x 489 [2,8]
+ CRUSH rule 1 x 490 [6,2]
+ CRUSH rule 1 x 491 [1,2]
+ CRUSH rule 1 x 492 [8,2]
+ CRUSH rule 1 x 493 [2,1]
+ CRUSH rule 1 x 494 [1,2]
+ CRUSH rule 1 x 495 [4,6]
+ CRUSH rule 1 x 496 [8,4]
+ CRUSH rule 1 x 497 [4,8]
+ CRUSH rule 1 x 498 [2,4]
+ CRUSH rule 1 x 499 [8,4]
+ CRUSH rule 1 x 500 [4,8]
+ CRUSH rule 1 x 501 [2,8]
+ CRUSH rule 1 x 502 [6,1]
+ CRUSH rule 1 x 503 [2,1]
+ CRUSH rule 1 x 504 [8,2]
+ CRUSH rule 1 x 505 [1,8]
+ CRUSH rule 1 x 506 [4,1]
+ CRUSH rule 1 x 507 [8,1]
+ CRUSH rule 1 x 508 [1,2]
+ CRUSH rule 1 x 509 [8]
+ CRUSH rule 1 x 510 [8,2]
+ CRUSH rule 1 x 511 [4,8]
+ CRUSH rule 1 x 512 [8,2]
+ CRUSH rule 1 x 513 [8,2]
+ CRUSH rule 1 x 514 [1,8]
+ CRUSH rule 1 x 515 [8,4]
+ CRUSH rule 1 x 516 [4,1]
+ CRUSH rule 1 x 517 [8,2]
+ CRUSH rule 1 x 518 [4,8]
+ CRUSH rule 1 x 519 [8,4]
+ CRUSH rule 1 x 520 [2,8]
+ CRUSH rule 1 x 521 [8,2]
+ CRUSH rule 1 x 522 [8,4]
+ CRUSH rule 1 x 523 [4,2]
+ CRUSH rule 1 x 524 [2,1]
+ CRUSH rule 1 x 525 [2,8]
+ CRUSH rule 1 x 526 [1,2]
+ CRUSH rule 1 x 527 [1,2]
+ CRUSH rule 1 x 528 [8,2]
+ CRUSH rule 1 x 529 [4,8]
+ CRUSH rule 1 x 530 [8]
+ CRUSH rule 1 x 531 [8,1]
+ CRUSH rule 1 x 532 [6,4]
+ CRUSH rule 1 x 533 [4,8]
+ CRUSH rule 1 x 534 [8,2]
+ CRUSH rule 1 x 535 [8,6]
+ CRUSH rule 1 x 536 [8,2]
+ CRUSH rule 1 x 537 [4,8]
+ CRUSH rule 1 x 538 [8,4]
+ CRUSH rule 1 x 539 [8,2]
+ CRUSH rule 1 x 540 [1,8]
+ CRUSH rule 1 x 541 [2,4]
+ CRUSH rule 1 x 542 [2,1]
+ CRUSH rule 1 x 543 [8,2]
+ CRUSH rule 1 x 544 [4,8]
+ CRUSH rule 1 x 545 [1,8]
+ CRUSH rule 1 x 546 [8,1]
+ CRUSH rule 1 x 547 [8,2]
+ CRUSH rule 1 x 548 [4,2]
+ CRUSH rule 1 x 549 [1,8]
+ CRUSH rule 1 x 550 [2,4]
+ CRUSH rule 1 x 551 [8,2]
+ CRUSH rule 1 x 552 [4,1]
+ CRUSH rule 1 x 553 [8,2]
+ CRUSH rule 1 x 554 [1,8]
+ CRUSH rule 1 x 555 [4,1]
+ CRUSH rule 1 x 556 [2,8]
+ CRUSH rule 1 x 557 [8,1]
+ CRUSH rule 1 x 558 [4,1]
+ CRUSH rule 1 x 559 [1,2]
+ CRUSH rule 1 x 560 [8,1]
+ CRUSH rule 1 x 561 [8,4]
+ CRUSH rule 1 x 562 [4,8]
+ CRUSH rule 1 x 563 [2,8]
+ CRUSH rule 1 x 564 [8,1]
+ CRUSH rule 1 x 565 [4,8]
+ CRUSH rule 1 x 566 [4,8]
+ CRUSH rule 1 x 567 [4,8]
+ CRUSH rule 1 x 568 [8,2]
+ CRUSH rule 1 x 569 [4,2]
+ CRUSH rule 1 x 570 [1,8]
+ CRUSH rule 1 x 571 [6,8]
+ CRUSH rule 1 x 572 [4]
+ CRUSH rule 1 x 573 [1,2]
+ CRUSH rule 1 x 574 [2,1]
+ CRUSH rule 1 x 575 [8,2]
+ CRUSH rule 1 x 576 [4,8]
+ CRUSH rule 1 x 577 [8,2]
+ CRUSH rule 1 x 578 [8,1]
+ CRUSH rule 1 x 579 [4,1]
+ CRUSH rule 1 x 580 [8,2]
+ CRUSH rule 1 x 581 [8,2]
+ CRUSH rule 1 x 582 [2,8]
+ CRUSH rule 1 x 583 [8,1]
+ CRUSH rule 1 x 584 [8,1]
+ CRUSH rule 1 x 585 [8,1]
+ CRUSH rule 1 x 586 [1,2]
+ CRUSH rule 1 x 587 [2,4]
+ CRUSH rule 1 x 588 [4,1]
+ CRUSH rule 1 x 589 [8,1]
+ CRUSH rule 1 x 590 [8,2]
+ CRUSH rule 1 x 591 [4,2]
+ CRUSH rule 1 x 592 [2,1]
+ CRUSH rule 1 x 593 [1,8]
+ CRUSH rule 1 x 594 [2,8]
+ CRUSH rule 1 x 595 [8,1]
+ CRUSH rule 1 x 596 [2,8]
+ CRUSH rule 1 x 597 [1,2]
+ CRUSH rule 1 x 598 [8,2]
+ CRUSH rule 1 x 599 [4,2]
+ CRUSH rule 1 x 600 [8,1]
+ CRUSH rule 1 x 601 [1,8]
+ CRUSH rule 1 x 602 [2,8]
+ CRUSH rule 1 x 603 [1,2]
+ CRUSH rule 1 x 604 [8,2]
+ CRUSH rule 1 x 605 [8,2]
+ CRUSH rule 1 x 606 [2,1]
+ CRUSH rule 1 x 607 [2,4]
+ CRUSH rule 1 x 608 [4,1]
+ CRUSH rule 1 x 609 [4,2]
+ CRUSH rule 1 x 610 [1,8]
+ CRUSH rule 1 x 611 [1,2]
+ CRUSH rule 1 x 612 [2,1]
+ CRUSH rule 1 x 613 [8,2]
+ CRUSH rule 1 x 614 [8,4]
+ CRUSH rule 1 x 615 [8,1]
+ CRUSH rule 1 x 616 [1,8]
+ CRUSH rule 1 x 617 [8,1]
+ CRUSH rule 1 x 618 [8,1]
+ CRUSH rule 1 x 619 [4,1]
+ CRUSH rule 1 x 620 [8,1]
+ CRUSH rule 1 x 621 [2,8]
+ CRUSH rule 1 x 622 [2,4]
+ CRUSH rule 1 x 623 [2,8]
+ CRUSH rule 1 x 624 [4,8]
+ CRUSH rule 1 x 625 [2,1]
+ CRUSH rule 1 x 626 [8,1]
+ CRUSH rule 1 x 627 [2,8]
+ CRUSH rule 1 x 628 [8,2]
+ CRUSH rule 1 x 629 [2,8]
+ CRUSH rule 1 x 630 [2,8]
+ CRUSH rule 1 x 631 [1,8]
+ CRUSH rule 1 x 632 [8,2]
+ CRUSH rule 1 x 633 [8,2]
+ CRUSH rule 1 x 634 [1,8]
+ CRUSH rule 1 x 635 [4,8]
+ CRUSH rule 1 x 636 [1,4]
+ CRUSH rule 1 x 637 [1,2]
+ CRUSH rule 1 x 638 [8,1]
+ CRUSH rule 1 x 639 [2,1]
+ CRUSH rule 1 x 640 [1,2]
+ CRUSH rule 1 x 641 [8,2]
+ CRUSH rule 1 x 642 [2,1]
+ CRUSH rule 1 x 643 [8,2]
+ CRUSH rule 1 x 644 [8,1]
+ CRUSH rule 1 x 645 [2,1]
+ CRUSH rule 1 x 646 [8,1]
+ CRUSH rule 1 x 647 [8,1]
+ CRUSH rule 1 x 648 [1,8]
+ CRUSH rule 1 x 649 [4,8]
+ CRUSH rule 1 x 650 [8,4]
+ CRUSH rule 1 x 651 [4,6]
+ CRUSH rule 1 x 652 [4,8]
+ CRUSH rule 1 x 653 [8,1]
+ CRUSH rule 1 x 654 [6,1]
+ CRUSH rule 1 x 655 [1,4]
+ CRUSH rule 1 x 656 [8,2]
+ CRUSH rule 1 x 657 [6,1]
+ CRUSH rule 1 x 658 [8,1]
+ CRUSH rule 1 x 659 [4,8]
+ CRUSH rule 1 x 660 [8,1]
+ CRUSH rule 1 x 661 [1,8]
+ CRUSH rule 1 x 662 [8,2]
+ CRUSH rule 1 x 663 [1,4]
+ CRUSH rule 1 x 664 [1,4]
+ CRUSH rule 1 x 665 [4,6]
+ CRUSH rule 1 x 666 [2,8]
+ CRUSH rule 1 x 667 [1,4]
+ CRUSH rule 1 x 668 [4,8]
+ CRUSH rule 1 x 669 [6,4]
+ CRUSH rule 1 x 670 [4,2]
+ CRUSH rule 1 x 671 [2,1]
+ CRUSH rule 1 x 672 [4,8]
+ CRUSH rule 1 x 673 [4,2]
+ CRUSH rule 1 x 674 [8,1]
+ CRUSH rule 1 x 675 [1,8]
+ CRUSH rule 1 x 676 [2,1]
+ CRUSH rule 1 x 677 [4,1]
+ CRUSH rule 1 x 678 [2,4]
+ CRUSH rule 1 x 679 [8,2]
+ CRUSH rule 1 x 680 [2]
+ CRUSH rule 1 x 681 [2,8]
+ CRUSH rule 1 x 682 [1,4]
+ CRUSH rule 1 x 683 [1,2]
+ CRUSH rule 1 x 684 [8,1]
+ CRUSH rule 1 x 685 [8,1]
+ CRUSH rule 1 x 686 [1,4]
+ CRUSH rule 1 x 687 [1,6]
+ CRUSH rule 1 x 688 [4,8]
+ CRUSH rule 1 x 689 [8,4]
+ CRUSH rule 1 x 690 [8,1]
+ CRUSH rule 1 x 691 [8,1]
+ CRUSH rule 1 x 692 [8,2]
+ CRUSH rule 1 x 693 [8,4]
+ CRUSH rule 1 x 694 [8,4]
+ CRUSH rule 1 x 695 [2,8]
+ CRUSH rule 1 x 696 [1,2]
+ CRUSH rule 1 x 697 [8,1]
+ CRUSH rule 1 x 698 [8,2]
+ CRUSH rule 1 x 699 [1,8]
+ CRUSH rule 1 x 700 [1,2]
+ CRUSH rule 1 x 701 [2,1]
+ CRUSH rule 1 x 702 [8]
+ CRUSH rule 1 x 703 [8,1]
+ CRUSH rule 1 x 704 [1,4]
+ CRUSH rule 1 x 705 [8,4]
+ CRUSH rule 1 x 706 [1,2]
+ CRUSH rule 1 x 707 [8,2]
+ CRUSH rule 1 x 708 [4,8]
+ CRUSH rule 1 x 709 [8,2]
+ CRUSH rule 1 x 710 [8,1]
+ CRUSH rule 1 x 711 [2,4]
+ CRUSH rule 1 x 712 [2,8]
+ CRUSH rule 1 x 713 [8,2]
+ CRUSH rule 1 x 714 [1,2]
+ CRUSH rule 1 x 715 [1,2]
+ CRUSH rule 1 x 716 [4,8]
+ CRUSH rule 1 x 717 [8,4]
+ CRUSH rule 1 x 718 [2,8]
+ CRUSH rule 1 x 719 [2,6]
+ CRUSH rule 1 x 720 [8,1]
+ CRUSH rule 1 x 721 [4,6]
+ CRUSH rule 1 x 722 [8,1]
+ CRUSH rule 1 x 723 [4,1]
+ CRUSH rule 1 x 724 [2,6]
+ CRUSH rule 1 x 725 [1,2]
+ CRUSH rule 1 x 726 [4,8]
+ CRUSH rule 1 x 727 [4,8]
+ CRUSH rule 1 x 728 [2,1]
+ CRUSH rule 1 x 729 [2,8]
+ CRUSH rule 1 x 730 [4,8]
+ CRUSH rule 1 x 731 [4,1]
+ CRUSH rule 1 x 732 [1,2]
+ CRUSH rule 1 x 733 [4,1]
+ CRUSH rule 1 x 734 [8,4]
+ CRUSH rule 1 x 735 [4,8]
+ CRUSH rule 1 x 736 [4,8]
+ CRUSH rule 1 x 737 [1,2]
+ CRUSH rule 1 x 738 [4,2]
+ CRUSH rule 1 x 739 [2,1]
+ CRUSH rule 1 x 740 [1,2]
+ CRUSH rule 1 x 741 [8,2]
+ CRUSH rule 1 x 742 [8,2]
+ CRUSH rule 1 x 743 [8,1]
+ CRUSH rule 1 x 744 [4,8]
+ CRUSH rule 1 x 745 [8,2]
+ CRUSH rule 1 x 746 [8,1]
+ CRUSH rule 1 x 747 [8,1]
+ CRUSH rule 1 x 748 [2,8]
+ CRUSH rule 1 x 749 [4,8]
+ CRUSH rule 1 x 750 [1,8]
+ CRUSH rule 1 x 751 [2,1]
+ CRUSH rule 1 x 752 [8,1]
+ CRUSH rule 1 x 753 [8,1]
+ CRUSH rule 1 x 754 [8,4]
+ CRUSH rule 1 x 755 [1,2]
+ CRUSH rule 1 x 756 [8,2]
+ CRUSH rule 1 x 757 [8,4]
+ CRUSH rule 1 x 758 [8,2]
+ CRUSH rule 1 x 759 [8,4]
+ CRUSH rule 1 x 760 [1,4]
+ CRUSH rule 1 x 761 [1,2]
+ CRUSH rule 1 x 762 [2,8]
+ CRUSH rule 1 x 763 [8,2]
+ CRUSH rule 1 x 764 [1,8]
+ CRUSH rule 1 x 765 [8,2]
+ CRUSH rule 1 x 766 [8]
+ CRUSH rule 1 x 767 [1,2]
+ CRUSH rule 1 x 768 [8,4]
+ CRUSH rule 1 x 769 [8,2]
+ CRUSH rule 1 x 770 [8,2]
+ CRUSH rule 1 x 771 [8,1]
+ CRUSH rule 1 x 772 [8,4]
+ CRUSH rule 1 x 773 [4,1]
+ CRUSH rule 1 x 774 [8,1]
+ CRUSH rule 1 x 775 [8,2]
+ CRUSH rule 1 x 776 [6,2]
+ CRUSH rule 1 x 777 [4,1]
+ CRUSH rule 1 x 778 [1,8]
+ CRUSH rule 1 x 779 [2,8]
+ CRUSH rule 1 x 780 [2,1]
+ CRUSH rule 1 x 781 [8,2]
+ CRUSH rule 1 x 782 [4,1]
+ CRUSH rule 1 x 783 [8,1]
+ CRUSH rule 1 x 784 [1,2]
+ CRUSH rule 1 x 785 [8,1]
+ CRUSH rule 1 x 786 [8,2]
+ CRUSH rule 1 x 787 [1,2]
+ CRUSH rule 1 x 788 [8,2]
+ CRUSH rule 1 x 789 [1,6]
+ CRUSH rule 1 x 790 [8,2]
+ CRUSH rule 1 x 791 [4,8]
+ CRUSH rule 1 x 792 [4,8]
+ CRUSH rule 1 x 793 [8,1]
+ CRUSH rule 1 x 794 [2,8]
+ CRUSH rule 1 x 795 [1,8]
+ CRUSH rule 1 x 796 [2,8]
+ CRUSH rule 1 x 797 [2,4]
+ CRUSH rule 1 x 798 [6,8]
+ CRUSH rule 1 x 799 [4,1]
+ CRUSH rule 1 x 800 [8,2]
+ CRUSH rule 1 x 801 [4,8]
+ CRUSH rule 1 x 802 [1,8]
+ CRUSH rule 1 x 803 [2,8]
+ CRUSH rule 1 x 804 [8,2]
+ CRUSH rule 1 x 805 [2,8]
+ CRUSH rule 1 x 806 [1,4]
+ CRUSH rule 1 x 807 [4,8]
+ CRUSH rule 1 x 808 [2,8]
+ CRUSH rule 1 x 809 [1,8]
+ CRUSH rule 1 x 810 [2,8]
+ CRUSH rule 1 x 811 [8,2]
+ CRUSH rule 1 x 812 [8,4]
+ CRUSH rule 1 x 813 [8,4]
+ CRUSH rule 1 x 814 [8,1]
+ CRUSH rule 1 x 815 [4,1]
+ CRUSH rule 1 x 816 [2,1]
+ CRUSH rule 1 x 817 [8,1]
+ CRUSH rule 1 x 818 [8,2]
+ CRUSH rule 1 x 819 [8,1]
+ CRUSH rule 1 x 820 [4,1]
+ CRUSH rule 1 x 821 [4,1]
+ CRUSH rule 1 x 822 [2,1]
+ CRUSH rule 1 x 823 [4,8]
+ CRUSH rule 1 x 824 [8,2]
+ CRUSH rule 1 x 825 [2,8]
+ CRUSH rule 1 x 826 [8,2]
+ CRUSH rule 1 x 827 [2,8]
+ CRUSH rule 1 x 828 [2,1]
+ CRUSH rule 1 x 829 [8,2]
+ CRUSH rule 1 x 830 [2,4]
+ CRUSH rule 1 x 831 [1,8]
+ CRUSH rule 1 x 832 [4,8]
+ CRUSH rule 1 x 833 [2,1]
+ CRUSH rule 1 x 834 [2]
+ CRUSH rule 1 x 835 [8,4]
+ CRUSH rule 1 x 836 [4,1]
+ CRUSH rule 1 x 837 [8,4]
+ CRUSH rule 1 x 838 [6,8]
+ CRUSH rule 1 x 839 [8,2]
+ CRUSH rule 1 x 840 [8,2]
+ CRUSH rule 1 x 841 [4,8]
+ CRUSH rule 1 x 842 [2]
+ CRUSH rule 1 x 843 [8,4]
+ CRUSH rule 1 x 844 [1,8]
+ CRUSH rule 1 x 845 [4,8]
+ CRUSH rule 1 x 846 [4,2]
+ CRUSH rule 1 x 847 [2,1]
+ CRUSH rule 1 x 848 [2,8]
+ CRUSH rule 1 x 849 [4,8]
+ CRUSH rule 1 x 850 [1,2]
+ CRUSH rule 1 x 851 [6,8]
+ CRUSH rule 1 x 852 [8,4]
+ CRUSH rule 1 x 853 [6,8]
+ CRUSH rule 1 x 854 [8,1]
+ CRUSH rule 1 x 855 [8,1]
+ CRUSH rule 1 x 856 [8,4]
+ CRUSH rule 1 x 857 [8,1]
+ CRUSH rule 1 x 858 [6,2]
+ CRUSH rule 1 x 859 [8,2]
+ CRUSH rule 1 x 860 [8,1]
+ CRUSH rule 1 x 861 [8,1]
+ CRUSH rule 1 x 862 [8,1]
+ CRUSH rule 1 x 863 [8,1]
+ CRUSH rule 1 x 864 [8,2]
+ CRUSH rule 1 x 865 [8,1]
+ CRUSH rule 1 x 866 [2,8]
+ CRUSH rule 1 x 867 [8]
+ CRUSH rule 1 x 868 [8,2]
+ CRUSH rule 1 x 869 [8,6]
+ CRUSH rule 1 x 870 [2,1]
+ CRUSH rule 1 x 871 [2,8]
+ CRUSH rule 1 x 872 [8,1]
+ CRUSH rule 1 x 873 [4,8]
+ CRUSH rule 1 x 874 [2,6]
+ CRUSH rule 1 x 875 [2,8]
+ CRUSH rule 1 x 876 [4,8]
+ CRUSH rule 1 x 877 [8,4]
+ CRUSH rule 1 x 878 [8,1]
+ CRUSH rule 1 x 879 [8,2]
+ CRUSH rule 1 x 880 [6,1]
+ CRUSH rule 1 x 881 [4,8]
+ CRUSH rule 1 x 882 [8,2]
+ CRUSH rule 1 x 883 [2,1]
+ CRUSH rule 1 x 884 [8,2]
+ CRUSH rule 1 x 885 [4,1]
+ CRUSH rule 1 x 886 [2,8]
+ CRUSH rule 1 x 887 [8,4]
+ CRUSH rule 1 x 888 [8,2]
+ CRUSH rule 1 x 889 [2,1]
+ CRUSH rule 1 x 890 [8,2]
+ CRUSH rule 1 x 891 [1,8]
+ CRUSH rule 1 x 892 [8,2]
+ CRUSH rule 1 x 893 [2,8]
+ CRUSH rule 1 x 894 [8,4]
+ CRUSH rule 1 x 895 [4,8]
+ CRUSH rule 1 x 896 [1,8]
+ CRUSH rule 1 x 897 [8,2]
+ CRUSH rule 1 x 898 [1,4]
+ CRUSH rule 1 x 899 [1,8]
+ CRUSH rule 1 x 900 [4,1]
+ CRUSH rule 1 x 901 [8,1]
+ CRUSH rule 1 x 902 [8,4]
+ CRUSH rule 1 x 903 [1,8]
+ CRUSH rule 1 x 904 [1,8]
+ CRUSH rule 1 x 905 [8,2]
+ CRUSH rule 1 x 906 [1,2]
+ CRUSH rule 1 x 907 [8,1]
+ CRUSH rule 1 x 908 [8,1]
+ CRUSH rule 1 x 909 [2,1]
+ CRUSH rule 1 x 910 [8,2]
+ CRUSH rule 1 x 911 [8,1]
+ CRUSH rule 1 x 912 [1,2]
+ CRUSH rule 1 x 913 [8,4]
+ CRUSH rule 1 x 914 [6,4]
+ CRUSH rule 1 x 915 [8,2]
+ CRUSH rule 1 x 916 [4,1]
+ CRUSH rule 1 x 917 [1,4]
+ CRUSH rule 1 x 918 [8,2]
+ CRUSH rule 1 x 919 [8,2]
+ CRUSH rule 1 x 920 [8,1]
+ CRUSH rule 1 x 921 [1,2]
+ CRUSH rule 1 x 922 [8,4]
+ CRUSH rule 1 x 923 [4,8]
+ CRUSH rule 1 x 924 [1,8]
+ CRUSH rule 1 x 925 [4,8]
+ CRUSH rule 1 x 926 [1,8]
+ CRUSH rule 1 x 927 [1,8]
+ CRUSH rule 1 x 928 [8,1]
+ CRUSH rule 1 x 929 [4,1]
+ CRUSH rule 1 x 930 [2,1]
+ CRUSH rule 1 x 931 [8,1]
+ CRUSH rule 1 x 932 [4,8]
+ CRUSH rule 1 x 933 [8,4]
+ CRUSH rule 1 x 934 [8,1]
+ CRUSH rule 1 x 935 [8,1]
+ CRUSH rule 1 x 936 [1,8]
+ CRUSH rule 1 x 937 [4,8]
+ CRUSH rule 1 x 938 [8,4]
+ CRUSH rule 1 x 939 [2,8]
+ CRUSH rule 1 x 940 [8,2]
+ CRUSH rule 1 x 941 [8,2]
+ CRUSH rule 1 x 942 [1,2]
+ CRUSH rule 1 x 943 [8,2]
+ CRUSH rule 1 x 944 [2,1]
+ CRUSH rule 1 x 945 [8,2]
+ CRUSH rule 1 x 946 [2,1]
+ CRUSH rule 1 x 947 [8,1]
+ CRUSH rule 1 x 948 [8,1]
+ CRUSH rule 1 x 949 [6,1]
+ CRUSH rule 1 x 950 [8,1]
+ CRUSH rule 1 x 951 [2,8]
+ CRUSH rule 1 x 952 [2,1]
+ CRUSH rule 1 x 953 [1,4]
+ CRUSH rule 1 x 954 [8,2]
+ CRUSH rule 1 x 955 [8,1]
+ CRUSH rule 1 x 956 [1,2]
+ CRUSH rule 1 x 957 [8,1]
+ CRUSH rule 1 x 958 [8,2]
+ CRUSH rule 1 x 959 [4,2]
+ CRUSH rule 1 x 960 [2,6]
+ CRUSH rule 1 x 961 [1,2]
+ CRUSH rule 1 x 962 [8,4]
+ CRUSH rule 1 x 963 [2,4]
+ CRUSH rule 1 x 964 [8,1]
+ CRUSH rule 1 x 965 [8,2]
+ CRUSH rule 1 x 966 [4,8]
+ CRUSH rule 1 x 967 [8,1]
+ CRUSH rule 1 x 968 [8,2]
+ CRUSH rule 1 x 969 [8,2]
+ CRUSH rule 1 x 970 [2,8]
+ CRUSH rule 1 x 971 [1,8]
+ CRUSH rule 1 x 972 [1,8]
+ CRUSH rule 1 x 973 [1,2]
+ CRUSH rule 1 x 974 [4,1]
+ CRUSH rule 1 x 975 [4,8]
+ CRUSH rule 1 x 976 [4,8]
+ CRUSH rule 1 x 977 [8,4]
+ CRUSH rule 1 x 978 [8,2]
+ CRUSH rule 1 x 979 [8,1]
+ CRUSH rule 1 x 980 [8,2]
+ CRUSH rule 1 x 981 [8]
+ CRUSH rule 1 x 982 [1,2]
+ CRUSH rule 1 x 983 [4,8]
+ CRUSH rule 1 x 984 [2,1]
+ CRUSH rule 1 x 985 [2,4]
+ CRUSH rule 1 x 986 [8,4]
+ CRUSH rule 1 x 987 [2,1]
+ CRUSH rule 1 x 988 [1,4]
+ CRUSH rule 1 x 989 [1,8]
+ CRUSH rule 1 x 990 [1,2]
+ CRUSH rule 1 x 991 [1,4]
+ CRUSH rule 1 x 992 [8,1]
+ CRUSH rule 1 x 993 [2,8]
+ CRUSH rule 1 x 994 [4,2]
+ CRUSH rule 1 x 995 [8,1]
+ CRUSH rule 1 x 996 [8,2]
+ CRUSH rule 1 x 997 [8,4]
+ CRUSH rule 1 x 998 [8,1]
+ CRUSH rule 1 x 999 [1,8]
+ CRUSH rule 1 x 1000 [8,4]
+ CRUSH rule 1 x 1001 [2,1]
+ CRUSH rule 1 x 1002 [1,2]
+ CRUSH rule 1 x 1003 [2,8]
+ CRUSH rule 1 x 1004 [8,1]
+ CRUSH rule 1 x 1005 [8,1]
+ CRUSH rule 1 x 1006 [1,2]
+ CRUSH rule 1 x 1007 [1,2]
+ CRUSH rule 1 x 1008 [1,8]
+ CRUSH rule 1 x 1009 [6,8]
+ CRUSH rule 1 x 1010 [2,8]
+ CRUSH rule 1 x 1011 [4,2]
+ CRUSH rule 1 x 1012 [1,2]
+ CRUSH rule 1 x 1013 [1,2]
+ CRUSH rule 1 x 1014 [2,8]
+ CRUSH rule 1 x 1015 [8,1]
+ CRUSH rule 1 x 1016 [2,1]
+ CRUSH rule 1 x 1017 [6,2]
+ CRUSH rule 1 x 1018 [4,2]
+ CRUSH rule 1 x 1019 [4,8]
+ CRUSH rule 1 x 1020 [1,2]
+ CRUSH rule 1 x 1021 [8,2]
+ CRUSH rule 1 x 1022 [1,8]
+ CRUSH rule 1 x 1023 [4,2]
+ rule 1 (choose-two) num_rep 2 result size == 1:\t23/1024 (esc)
+ rule 1 (choose-two) num_rep 2 result size == 2:\t1001/1024 (esc)
+ CRUSH rule 1 x 0 [2,1,4]
+ CRUSH rule 1 x 1 [2,8,1]
+ CRUSH rule 1 x 2 [1,8,2]
+ CRUSH rule 1 x 3 [8,1]
+ CRUSH rule 1 x 4 [4,8,2]
+ CRUSH rule 1 x 5 [8,2,1]
+ CRUSH rule 1 x 6 [2,8,1]
+ CRUSH rule 1 x 7 [4,8,2]
+ CRUSH rule 1 x 8 [4,2,8]
+ CRUSH rule 1 x 9 [2,4,1]
+ CRUSH rule 1 x 10 [2,1,8]
+ CRUSH rule 1 x 11 [2,8,1]
+ CRUSH rule 1 x 12 [2,1,8]
+ CRUSH rule 1 x 13 [4,8,2]
+ CRUSH rule 1 x 14 [8,1,2]
+ CRUSH rule 1 x 15 [8,2,1]
+ CRUSH rule 1 x 16 [8,1,2]
+ CRUSH rule 1 x 17 [4,8,2]
+ CRUSH rule 1 x 18 [1,6,8]
+ CRUSH rule 1 x 19 [8,4]
+ CRUSH rule 1 x 20 [2,1,8]
+ CRUSH rule 1 x 21 [8,1,2]
+ CRUSH rule 1 x 22 [8,2]
+ CRUSH rule 1 x 23 [4,8,1]
+ CRUSH rule 1 x 24 [1,2,8]
+ CRUSH rule 1 x 25 [4,8,1]
+ CRUSH rule 1 x 26 [2,8,1]
+ CRUSH rule 1 x 27 [4,1]
+ CRUSH rule 1 x 28 [8,2,1]
+ CRUSH rule 1 x 29 [8,4,1]
+ CRUSH rule 1 x 30 [4,8,1]
+ CRUSH rule 1 x 31 [8,2,1]
+ CRUSH rule 1 x 32 [2,6,8]
+ CRUSH rule 1 x 33 [2,8,1]
+ CRUSH rule 1 x 34 [2,1,8]
+ CRUSH rule 1 x 35 [1,8,2]
+ CRUSH rule 1 x 36 [8,1,2]
+ CRUSH rule 1 x 37 [1,8,2]
+ CRUSH rule 1 x 38 [4,8,1]
+ CRUSH rule 1 x 39 [8,1,2]
+ CRUSH rule 1 x 40 [8,2,1]
+ CRUSH rule 1 x 41 [2,1,8]
+ CRUSH rule 1 x 42 [1,2,8]
+ CRUSH rule 1 x 43 [1,2,8]
+ CRUSH rule 1 x 44 [1,8,2]
+ CRUSH rule 1 x 45 [8,2,4]
+ CRUSH rule 1 x 46 [2,1,8]
+ CRUSH rule 1 x 47 [4,2,8]
+ CRUSH rule 1 x 48 [8,1]
+ CRUSH rule 1 x 49 [8,2,1]
+ CRUSH rule 1 x 50 [4,8,1]
+ CRUSH rule 1 x 51 [2,8,1]
+ CRUSH rule 1 x 52 [8,2,1]
+ CRUSH rule 1 x 53 [4,2,8]
+ CRUSH rule 1 x 54 [8,4]
+ CRUSH rule 1 x 55 [8,1,2]
+ CRUSH rule 1 x 56 [8,4,2]
+ CRUSH rule 1 x 57 [2,1,8]
+ CRUSH rule 1 x 58 [1,2,8]
+ CRUSH rule 1 x 59 [8,2,1]
+ CRUSH rule 1 x 60 [4,8,1]
+ CRUSH rule 1 x 61 [4,8,2]
+ CRUSH rule 1 x 62 [8,1,2]
+ CRUSH rule 1 x 63 [8,1,2]
+ CRUSH rule 1 x 64 [4,2,1]
+ CRUSH rule 1 x 65 [8,4]
+ CRUSH rule 1 x 66 [4,2,1]
+ CRUSH rule 1 x 67 [4,2,8]
+ CRUSH rule 1 x 68 [1,2,8]
+ CRUSH rule 1 x 69 [1,2,8]
+ CRUSH rule 1 x 70 [8,2,1]
+ CRUSH rule 1 x 71 [2,8,1]
+ CRUSH rule 1 x 72 [8,1,4]
+ CRUSH rule 1 x 73 [2,8,1]
+ CRUSH rule 1 x 74 [1,8,2]
+ CRUSH rule 1 x 75 [4,2,1]
+ CRUSH rule 1 x 76 [4,1,6]
+ CRUSH rule 1 x 77 [8,2,4]
+ CRUSH rule 1 x 78 [1,2]
+ CRUSH rule 1 x 79 [4,1,2]
+ CRUSH rule 1 x 80 [2,4,1]
+ CRUSH rule 1 x 81 [2,1]
+ CRUSH rule 1 x 82 [6,1,8]
+ CRUSH rule 1 x 83 [2,8]
+ CRUSH rule 1 x 84 [8,2,1]
+ CRUSH rule 1 x 85 [4,8,1]
+ CRUSH rule 1 x 86 [2,1,8]
+ CRUSH rule 1 x 87 [2,8,4]
+ CRUSH rule 1 x 88 [1,6,2]
+ CRUSH rule 1 x 89 [2,1,8]
+ CRUSH rule 1 x 90 [8,2,4]
+ CRUSH rule 1 x 91 [4,8,1]
+ CRUSH rule 1 x 92 [1,8,2]
+ CRUSH rule 1 x 93 [8,4,1]
+ CRUSH rule 1 x 94 [1,2,8]
+ CRUSH rule 1 x 95 [8,1,2]
+ CRUSH rule 1 x 96 [1,8,2]
+ CRUSH rule 1 x 97 [8,1,2]
+ CRUSH rule 1 x 98 [2,1,8]
+ CRUSH rule 1 x 99 [2,8,1]
+ CRUSH rule 1 x 100 [1,8,4]
+ CRUSH rule 1 x 101 [8,2,1]
+ CRUSH rule 1 x 102 [2,1,8]
+ CRUSH rule 1 x 103 [2,8,1]
+ CRUSH rule 1 x 104 [8,4,1]
+ CRUSH rule 1 x 105 [2,4,1]
+ CRUSH rule 1 x 106 [1,8,2]
+ CRUSH rule 1 x 107 [2,1,8]
+ CRUSH rule 1 x 108 [8,2]
+ CRUSH rule 1 x 109 [1,2,4]
+ CRUSH rule 1 x 110 [4,2,8]
+ CRUSH rule 1 x 111 [2,1,4]
+ CRUSH rule 1 x 112 [2,1,8]
+ CRUSH rule 1 x 113 [8,2,1]
+ CRUSH rule 1 x 114 [8,4,1]
+ CRUSH rule 1 x 115 [8,2,4]
+ CRUSH rule 1 x 116 [1,2,8]
+ CRUSH rule 1 x 117 [6,8,1]
+ CRUSH rule 1 x 118 [2,8,1]
+ CRUSH rule 1 x 119 [2,8,1]
+ CRUSH rule 1 x 120 [2,1,4]
+ CRUSH rule 1 x 121 [2,1,8]
+ CRUSH rule 1 x 122 [8,1,2]
+ CRUSH rule 1 x 123 [2,8,1]
+ CRUSH rule 1 x 124 [8,1]
+ CRUSH rule 1 x 125 [1,8,4]
+ CRUSH rule 1 x 126 [8,2,1]
+ CRUSH rule 1 x 127 [4,8,2]
+ CRUSH rule 1 x 128 [2,8,6]
+ CRUSH rule 1 x 129 [2,1,8]
+ CRUSH rule 1 x 130 [4,8,1]
+ CRUSH rule 1 x 131 [1,2,8]
+ CRUSH rule 1 x 132 [1,2,8]
+ CRUSH rule 1 x 133 [8,2,1]
+ CRUSH rule 1 x 134 [1,8,2]
+ CRUSH rule 1 x 135 [4,8,2]
+ CRUSH rule 1 x 136 [2,1,4]
+ CRUSH rule 1 x 137 [8,4]
+ CRUSH rule 1 x 138 [8,4,2]
+ CRUSH rule 1 x 139 [4,2,6]
+ CRUSH rule 1 x 140 [1,8,2]
+ CRUSH rule 1 x 141 [8,1,2]
+ CRUSH rule 1 x 142 [4,6,1]
+ CRUSH rule 1 x 143 [4,8,1]
+ CRUSH rule 1 x 144 [8,1,2]
+ CRUSH rule 1 x 145 [8,2,1]
+ CRUSH rule 1 x 146 [2,8,1]
+ CRUSH rule 1 x 147 [2,8,4]
+ CRUSH rule 1 x 148 [4,1,2]
+ CRUSH rule 1 x 149 [4,8,2]
+ CRUSH rule 1 x 150 [1,8,2]
+ CRUSH rule 1 x 151 [1,2]
+ CRUSH rule 1 x 152 [8,1,2]
+ CRUSH rule 1 x 153 [8,4,1]
+ CRUSH rule 1 x 154 [4,2,1]
+ CRUSH rule 1 x 155 [4,8,2]
+ CRUSH rule 1 x 156 [4,8,2]
+ CRUSH rule 1 x 157 [2,1,8]
+ CRUSH rule 1 x 158 [2,8,1]
+ CRUSH rule 1 x 159 [8,2,4]
+ CRUSH rule 1 x 160 [2,8,1]
+ CRUSH rule 1 x 161 [1,4,2]
+ CRUSH rule 1 x 162 [1,8,2]
+ CRUSH rule 1 x 163 [4,8,1]
+ CRUSH rule 1 x 164 [8,2,1]
+ CRUSH rule 1 x 165 [8,2,4]
+ CRUSH rule 1 x 166 [2,1,8]
+ CRUSH rule 1 x 167 [1,2,8]
+ CRUSH rule 1 x 168 [4,2,8]
+ CRUSH rule 1 x 169 [2,8,1]
+ CRUSH rule 1 x 170 [1,2,8]
+ CRUSH rule 1 x 171 [8,4,1]
+ CRUSH rule 1 x 172 [1,8]
+ CRUSH rule 1 x 173 [8,4]
+ CRUSH rule 1 x 174 [1,2,6]
+ CRUSH rule 1 x 175 [8,1,2]
+ CRUSH rule 1 x 176 [2,1,8]
+ CRUSH rule 1 x 177 [8,2,1]
+ CRUSH rule 1 x 178 [4,2,1]
+ CRUSH rule 1 x 179 [8,1,2]
+ CRUSH rule 1 x 180 [1,8,2]
+ CRUSH rule 1 x 181 [8,2,1]
+ CRUSH rule 1 x 182 [8,1,2]
+ CRUSH rule 1 x 183 [8,1,2]
+ CRUSH rule 1 x 184 [4,8]
+ CRUSH rule 1 x 185 [8,4,1]
+ CRUSH rule 1 x 186 [2,1,4]
+ CRUSH rule 1 x 187 [1,8,2]
+ CRUSH rule 1 x 188 [1,8,2]
+ CRUSH rule 1 x 189 [1,8,2]
+ CRUSH rule 1 x 190 [1,2,8]
+ CRUSH rule 1 x 191 [8,4,1]
+ CRUSH rule 1 x 192 [4,1,2]
+ CRUSH rule 1 x 193 [4,2,8]
+ CRUSH rule 1 x 194 [1,8,2]
+ CRUSH rule 1 x 195 [8,4,1]
+ CRUSH rule 1 x 196 [8,1,2]
+ CRUSH rule 1 x 197 [8,4,2]
+ CRUSH rule 1 x 198 [2,1,8]
+ CRUSH rule 1 x 199 [1,4,8]
+ CRUSH rule 1 x 200 [1,8,2]
+ CRUSH rule 1 x 201 [8,1,2]
+ CRUSH rule 1 x 202 [8,2,1]
+ CRUSH rule 1 x 203 [1,8]
+ CRUSH rule 1 x 204 [2,1,4]
+ CRUSH rule 1 x 205 [1,8,2]
+ CRUSH rule 1 x 206 [1,2,8]
+ CRUSH rule 1 x 207 [2,1,8]
+ CRUSH rule 1 x 208 [8,1,2]
+ CRUSH rule 1 x 209 [1,2,8]
+ CRUSH rule 1 x 210 [1,2,4]
+ CRUSH rule 1 x 211 [4,8,2]
+ CRUSH rule 1 x 212 [8,1,2]
+ CRUSH rule 1 x 213 [8,4,2]
+ CRUSH rule 1 x 214 [8,1]
+ CRUSH rule 1 x 215 [8,1,2]
+ CRUSH rule 1 x 216 [8,1,2]
+ CRUSH rule 1 x 217 [1,2,8]
+ CRUSH rule 1 x 218 [2,8,1]
+ CRUSH rule 1 x 219 [2,8,1]
+ CRUSH rule 1 x 220 [4,8,2]
+ CRUSH rule 1 x 221 [1,8,2]
+ CRUSH rule 1 x 222 [8,1,2]
+ CRUSH rule 1 x 223 [1,2,8]
+ CRUSH rule 1 x 224 [1,4,2]
+ CRUSH rule 1 x 225 [8,2,1]
+ CRUSH rule 1 x 226 [8,2,4]
+ CRUSH rule 1 x 227 [4,8,1]
+ CRUSH rule 1 x 228 [1,8,2]
+ CRUSH rule 1 x 229 [4,1,6]
+ CRUSH rule 1 x 230 [4,8,2]
+ CRUSH rule 1 x 231 [4,8,2]
+ CRUSH rule 1 x 232 [2,8,4]
+ CRUSH rule 1 x 233 [8,2,1]
+ CRUSH rule 1 x 234 [1,2,8]
+ CRUSH rule 1 x 235 [4,8,1]
+ CRUSH rule 1 x 236 [1,2,6]
+ CRUSH rule 1 x 237 [4,8,2]
+ CRUSH rule 1 x 238 [8,1]
+ CRUSH rule 1 x 239 [8,6,1]
+ CRUSH rule 1 x 240 [4,8,1]
+ CRUSH rule 1 x 241 [2,1,8]
+ CRUSH rule 1 x 242 [2,8,1]
+ CRUSH rule 1 x 243 [8,2,1]
+ CRUSH rule 1 x 244 [4,8,2]
+ CRUSH rule 1 x 245 [8,2,1]
+ CRUSH rule 1 x 246 [1,2,8]
+ CRUSH rule 1 x 247 [8,2,1]
+ CRUSH rule 1 x 248 [8,2,4]
+ CRUSH rule 1 x 249 [2,1,8]
+ CRUSH rule 1 x 250 [2,1,4]
+ CRUSH rule 1 x 251 [2,8,1]
+ CRUSH rule 1 x 252 [4,8,2]
+ CRUSH rule 1 x 253 [1,2,8]
+ CRUSH rule 1 x 254 [4,8,1]
+ CRUSH rule 1 x 255 [1,8,2]
+ CRUSH rule 1 x 256 [4,8,1]
+ CRUSH rule 1 x 257 [2,8,4]
+ CRUSH rule 1 x 258 [4,1]
+ CRUSH rule 1 x 259 [8,6,2]
+ CRUSH rule 1 x 260 [8,2,1]
+ CRUSH rule 1 x 261 [8,1,2]
+ CRUSH rule 1 x 262 [8,2,1]
+ CRUSH rule 1 x 263 [8,4,2]
+ CRUSH rule 1 x 264 [1,8,2]
+ CRUSH rule 1 x 265 [8,1]
+ CRUSH rule 1 x 266 [8,2,1]
+ CRUSH rule 1 x 267 [2,8,1]
+ CRUSH rule 1 x 268 [1,8,2]
+ CRUSH rule 1 x 269 [1,8,2]
+ CRUSH rule 1 x 270 [4,1,2]
+ CRUSH rule 1 x 271 [8,4,2]
+ CRUSH rule 1 x 272 [2,8,4]
+ CRUSH rule 1 x 273 [4,1,2]
+ CRUSH rule 1 x 274 [8,2,4]
+ CRUSH rule 1 x 275 [4,8,1]
+ CRUSH rule 1 x 276 [8,1,2]
+ CRUSH rule 1 x 277 [8,1,2]
+ CRUSH rule 1 x 278 [8,2,1]
+ CRUSH rule 1 x 279 [8,4,2]
+ CRUSH rule 1 x 280 [2,8,1]
+ CRUSH rule 1 x 281 [8,2,1]
+ CRUSH rule 1 x 282 [1,2,8]
+ CRUSH rule 1 x 283 [8,2,1]
+ CRUSH rule 1 x 284 [8,2]
+ CRUSH rule 1 x 285 [4,8,1]
+ CRUSH rule 1 x 286 [2,1,8]
+ CRUSH rule 1 x 287 [1,2,8]
+ CRUSH rule 1 x 288 [8,1,4]
+ CRUSH rule 1 x 289 [4,8,2]
+ CRUSH rule 1 x 290 [1,4,8]
+ CRUSH rule 1 x 291 [1,2,4]
+ CRUSH rule 1 x 292 [8,2,1]
+ CRUSH rule 1 x 293 [8,1,2]
+ CRUSH rule 1 x 294 [8,4,2]
+ CRUSH rule 1 x 295 [4,8,1]
+ CRUSH rule 1 x 296 [4,1,8]
+ CRUSH rule 1 x 297 [8,2,1]
+ CRUSH rule 1 x 298 [1,2,8]
+ CRUSH rule 1 x 299 [2,1,8]
+ CRUSH rule 1 x 300 [8,2]
+ CRUSH rule 1 x 301 [1,8,2]
+ CRUSH rule 1 x 302 [8,1,2]
+ CRUSH rule 1 x 303 [8,4,1]
+ CRUSH rule 1 x 304 [2,8,1]
+ CRUSH rule 1 x 305 [2,8,1]
+ CRUSH rule 1 x 306 [1,8,2]
+ CRUSH rule 1 x 307 [2,1,8]
+ CRUSH rule 1 x 308 [2,8,4]
+ CRUSH rule 1 x 309 [8,2]
+ CRUSH rule 1 x 310 [4,6,1]
+ CRUSH rule 1 x 311 [4,2,1]
+ CRUSH rule 1 x 312 [2,1,8]
+ CRUSH rule 1 x 313 [4,8]
+ CRUSH rule 1 x 314 [6,1,2]
+ CRUSH rule 1 x 315 [2,1,8]
+ CRUSH rule 1 x 316 [8,2,1]
+ CRUSH rule 1 x 317 [2,8,1]
+ CRUSH rule 1 x 318 [8,2,1]
+ CRUSH rule 1 x 319 [8,2,1]
+ CRUSH rule 1 x 320 [8,2,1]
+ CRUSH rule 1 x 321 [1,2]
+ CRUSH rule 1 x 322 [2,8,4]
+ CRUSH rule 1 x 323 [4,8,1]
+ CRUSH rule 1 x 324 [8,1,4]
+ CRUSH rule 1 x 325 [4,8,2]
+ CRUSH rule 1 x 326 [8,6,1]
+ CRUSH rule 1 x 327 [1,8,2]
+ CRUSH rule 1 x 328 [8,4,1]
+ CRUSH rule 1 x 329 [4,8,2]
+ CRUSH rule 1 x 330 [4,8,1]
+ CRUSH rule 1 x 331 [2,8,1]
+ CRUSH rule 1 x 332 [2,1,8]
+ CRUSH rule 1 x 333 [8,1,2]
+ CRUSH rule 1 x 334 [8,1]
+ CRUSH rule 1 x 335 [8,1,2]
+ CRUSH rule 1 x 336 [4,8,2]
+ CRUSH rule 1 x 337 [8,2,4]
+ CRUSH rule 1 x 338 [1,8]
+ CRUSH rule 1 x 339 [8,2]
+ CRUSH rule 1 x 340 [2,1,8]
+ CRUSH rule 1 x 341 [4,1,8]
+ CRUSH rule 1 x 342 [2,8,4]
+ CRUSH rule 1 x 343 [8,1]
+ CRUSH rule 1 x 344 [6,2,4]
+ CRUSH rule 1 x 345 [2,1,8]
+ CRUSH rule 1 x 346 [8,2,4]
+ CRUSH rule 1 x 347 [4,1,8]
+ CRUSH rule 1 x 348 [8,2,1]
+ CRUSH rule 1 x 349 [1,8,2]
+ CRUSH rule 1 x 350 [8,1,2]
+ CRUSH rule 1 x 351 [1,8,2]
+ CRUSH rule 1 x 352 [1,2,4]
+ CRUSH rule 1 x 353 [8,1,2]
+ CRUSH rule 1 x 354 [1,8,2]
+ CRUSH rule 1 x 355 [1,2,8]
+ CRUSH rule 1 x 356 [4,8,1]
+ CRUSH rule 1 x 357 [8,1,2]
+ CRUSH rule 1 x 358 [2,1,8]
+ CRUSH rule 1 x 359 [6,8,1]
+ CRUSH rule 1 x 360 [1,2,8]
+ CRUSH rule 1 x 361 [8,4,2]
+ CRUSH rule 1 x 362 [4,6,8]
+ CRUSH rule 1 x 363 [4,1,2]
+ CRUSH rule 1 x 364 [2,8,1]
+ CRUSH rule 1 x 365 [8,1,2]
+ CRUSH rule 1 x 366 [8,2,1]
+ CRUSH rule 1 x 367 [4,8,2]
+ CRUSH rule 1 x 368 [8,4,2]
+ CRUSH rule 1 x 369 [8,1]
+ CRUSH rule 1 x 370 [8,2]
+ CRUSH rule 1 x 371 [1,4,2]
+ CRUSH rule 1 x 372 [8,1,2]
+ CRUSH rule 1 x 373 [1,8]
+ CRUSH rule 1 x 374 [2,8]
+ CRUSH rule 1 x 375 [8,4]
+ CRUSH rule 1 x 376 [8,1,2]
+ CRUSH rule 1 x 377 [1,2,4]
+ CRUSH rule 1 x 378 [1,2,8]
+ CRUSH rule 1 x 379 [8,1,2]
+ CRUSH rule 1 x 380 [2,1]
+ CRUSH rule 1 x 381 [1,4,8]
+ CRUSH rule 1 x 382 [1,4,2]
+ CRUSH rule 1 x 383 [4,1,2]
+ CRUSH rule 1 x 384 [8,2,1]
+ CRUSH rule 1 x 385 [8,1,6]
+ CRUSH rule 1 x 386 [1,8,2]
+ CRUSH rule 1 x 387 [1,4,8]
+ CRUSH rule 1 x 388 [8,1,6]
+ CRUSH rule 1 x 389 [1,4,8]
+ CRUSH rule 1 x 390 [4,8,1]
+ CRUSH rule 1 x 391 [4,8,2]
+ CRUSH rule 1 x 392 [1,8,2]
+ CRUSH rule 1 x 393 [8,2]
+ CRUSH rule 1 x 394 [8,1,2]
+ CRUSH rule 1 x 395 [8,1,2]
+ CRUSH rule 1 x 396 [4,2,8]
+ CRUSH rule 1 x 397 [2,1,4]
+ CRUSH rule 1 x 398 [2,4,1]
+ CRUSH rule 1 x 399 [8,2,4]
+ CRUSH rule 1 x 400 [8,1,4]
+ CRUSH rule 1 x 401 [1,2,4]
+ CRUSH rule 1 x 402 [8,1,2]
+ CRUSH rule 1 x 403 [1,2,4]
+ CRUSH rule 1 x 404 [4,2,1]
+ CRUSH rule 1 x 405 [8,4,2]
+ CRUSH rule 1 x 406 [2,1]
+ CRUSH rule 1 x 407 [2,8,4]
+ CRUSH rule 1 x 408 [4,1,8]
+ CRUSH rule 1 x 409 [8,4,2]
+ CRUSH rule 1 x 410 [8,1,4]
+ CRUSH rule 1 x 411 [2,1,8]
+ CRUSH rule 1 x 412 [2,6,8]
+ CRUSH rule 1 x 413 [8,2]
+ CRUSH rule 1 x 414 [4,1,8]
+ CRUSH rule 1 x 415 [2,8,1]
+ CRUSH rule 1 x 416 [2,1,8]
+ CRUSH rule 1 x 417 [8,6,2]
+ CRUSH rule 1 x 418 [8,2,4]
+ CRUSH rule 1 x 419 [8,4,1]
+ CRUSH rule 1 x 420 [1,4,2]
+ CRUSH rule 1 x 421 [8,4]
+ CRUSH rule 1 x 422 [6,8,2]
+ CRUSH rule 1 x 423 [2,4,8]
+ CRUSH rule 1 x 424 [8,2,1]
+ CRUSH rule 1 x 425 [1,2,8]
+ CRUSH rule 1 x 426 [8,2]
+ CRUSH rule 1 x 427 [1,8,2]
+ CRUSH rule 1 x 428 [4,8]
+ CRUSH rule 1 x 429 [4,8,2]
+ CRUSH rule 1 x 430 [4,8,1]
+ CRUSH rule 1 x 431 [4,1,2]
+ CRUSH rule 1 x 432 [8,1,2]
+ CRUSH rule 1 x 433 [8,2,1]
+ CRUSH rule 1 x 434 [8,2,1]
+ CRUSH rule 1 x 435 [2,6,1]
+ CRUSH rule 1 x 436 [4,1,8]
+ CRUSH rule 1 x 437 [8,2,1]
+ CRUSH rule 1 x 438 [2,4,8]
+ CRUSH rule 1 x 439 [1,6,8]
+ CRUSH rule 1 x 440 [2,8,1]
+ CRUSH rule 1 x 441 [4,6,2]
+ CRUSH rule 1 x 442 [2,1,8]
+ CRUSH rule 1 x 443 [8,4,2]
+ CRUSH rule 1 x 444 [8,1,2]
+ CRUSH rule 1 x 445 [8,1,2]
+ CRUSH rule 1 x 446 [2,8,1]
+ CRUSH rule 1 x 447 [2,1,4]
+ CRUSH rule 1 x 448 [8,2,4]
+ CRUSH rule 1 x 449 [8,6,2]
+ CRUSH rule 1 x 450 [1,8,2]
+ CRUSH rule 1 x 451 [8,4]
+ CRUSH rule 1 x 452 [8]
+ CRUSH rule 1 x 453 [6,8,2]
+ CRUSH rule 1 x 454 [8,2,1]
+ CRUSH rule 1 x 455 [2,8,4]
+ CRUSH rule 1 x 456 [8,2]
+ CRUSH rule 1 x 457 [8,2,1]
+ CRUSH rule 1 x 458 [2,8,1]
+ CRUSH rule 1 x 459 [2,1,8]
+ CRUSH rule 1 x 460 [8,2,1]
+ CRUSH rule 1 x 461 [8,2,1]
+ CRUSH rule 1 x 462 [8,1,2]
+ CRUSH rule 1 x 463 [8,2,1]
+ CRUSH rule 1 x 464 [8,4,2]
+ CRUSH rule 1 x 465 [6,8,1]
+ CRUSH rule 1 x 466 [8,2,1]
+ CRUSH rule 1 x 467 [8,2,1]
+ CRUSH rule 1 x 468 [8,4,1]
+ CRUSH rule 1 x 469 [8,1,2]
+ CRUSH rule 1 x 470 [4,2,6]
+ CRUSH rule 1 x 471 [1,2,8]
+ CRUSH rule 1 x 472 [1,8,2]
+ CRUSH rule 1 x 473 [1,2,4]
+ CRUSH rule 1 x 474 [8,1]
+ CRUSH rule 1 x 475 [8,4,1]
+ CRUSH rule 1 x 476 [4,1,2]
+ CRUSH rule 1 x 477 [4,8,1]
+ CRUSH rule 1 x 478 [8,1,2]
+ CRUSH rule 1 x 479 [2,1,8]
+ CRUSH rule 1 x 480 [1,8,2]
+ CRUSH rule 1 x 481 [2,4,1]
+ CRUSH rule 1 x 482 [2,8,1]
+ CRUSH rule 1 x 483 [2,1,8]
+ CRUSH rule 1 x 484 [1,2,8]
+ CRUSH rule 1 x 485 [1,8,2]
+ CRUSH rule 1 x 486 [4,1,8]
+ CRUSH rule 1 x 487 [8,1,2]
+ CRUSH rule 1 x 488 [2,8,1]
+ CRUSH rule 1 x 489 [2,8,1]
+ CRUSH rule 1 x 490 [6,2,1]
+ CRUSH rule 1 x 491 [1,2,8]
+ CRUSH rule 1 x 492 [8,2,1]
+ CRUSH rule 1 x 493 [2,1,8]
+ CRUSH rule 1 x 494 [1,2,8]
+ CRUSH rule 1 x 495 [4,6,1]
+ CRUSH rule 1 x 496 [8,4]
+ CRUSH rule 1 x 497 [4,8,2]
+ CRUSH rule 1 x 498 [2,4,8]
+ CRUSH rule 1 x 499 [8,4,1]
+ CRUSH rule 1 x 500 [4,8,1]
+ CRUSH rule 1 x 501 [2,8,1]
+ CRUSH rule 1 x 502 [6,1,8]
+ CRUSH rule 1 x 503 [2,1,8]
+ CRUSH rule 1 x 504 [8,2,1]
+ CRUSH rule 1 x 505 [1,8,2]
+ CRUSH rule 1 x 506 [4,1,2]
+ CRUSH rule 1 x 507 [8,1,2]
+ CRUSH rule 1 x 508 [1,2,8]
+ CRUSH rule 1 x 509 [8,2]
+ CRUSH rule 1 x 510 [8,2,1]
+ CRUSH rule 1 x 511 [4,8]
+ CRUSH rule 1 x 512 [8,2,1]
+ CRUSH rule 1 x 513 [8,2,1]
+ CRUSH rule 1 x 514 [1,8,2]
+ CRUSH rule 1 x 515 [8,4,2]
+ CRUSH rule 1 x 516 [4,1,2]
+ CRUSH rule 1 x 517 [8,2,1]
+ CRUSH rule 1 x 518 [4,8,1]
+ CRUSH rule 1 x 519 [8,4,1]
+ CRUSH rule 1 x 520 [2,8,4]
+ CRUSH rule 1 x 521 [8,2,1]
+ CRUSH rule 1 x 522 [8,4,1]
+ CRUSH rule 1 x 523 [4,2,1]
+ CRUSH rule 1 x 524 [2,1,8]
+ CRUSH rule 1 x 525 [2,8,1]
+ CRUSH rule 1 x 526 [1,2,8]
+ CRUSH rule 1 x 527 [1,2,4]
+ CRUSH rule 1 x 528 [8,2,1]
+ CRUSH rule 1 x 529 [4,8,2]
+ CRUSH rule 1 x 530 [8,2]
+ CRUSH rule 1 x 531 [8,1,2]
+ CRUSH rule 1 x 532 [6,4,8]
+ CRUSH rule 1 x 533 [4,8,2]
+ CRUSH rule 1 x 534 [8,2,1]
+ CRUSH rule 1 x 535 [8,6,1]
+ CRUSH rule 1 x 536 [8,2,1]
+ CRUSH rule 1 x 537 [4,8]
+ CRUSH rule 1 x 538 [8,4,1]
+ CRUSH rule 1 x 539 [8,2,1]
+ CRUSH rule 1 x 540 [1,8,2]
+ CRUSH rule 1 x 541 [2,4,1]
+ CRUSH rule 1 x 542 [2,1,8]
+ CRUSH rule 1 x 543 [8,2,1]
+ CRUSH rule 1 x 544 [4,8,1]
+ CRUSH rule 1 x 545 [1,8,2]
+ CRUSH rule 1 x 546 [8,1,2]
+ CRUSH rule 1 x 547 [8,2,1]
+ CRUSH rule 1 x 548 [4,2,1]
+ CRUSH rule 1 x 549 [1,8,2]
+ CRUSH rule 1 x 550 [2,4,1]
+ CRUSH rule 1 x 551 [8,2]
+ CRUSH rule 1 x 552 [4,1,2]
+ CRUSH rule 1 x 553 [8,2]
+ CRUSH rule 1 x 554 [1,8,2]
+ CRUSH rule 1 x 555 [4,1,8]
+ CRUSH rule 1 x 556 [2,8,1]
+ CRUSH rule 1 x 557 [8,1]
+ CRUSH rule 1 x 558 [4,1,2]
+ CRUSH rule 1 x 559 [1,2,8]
+ CRUSH rule 1 x 560 [8,1]
+ CRUSH rule 1 x 561 [8,4,1]
+ CRUSH rule 1 x 562 [4,8,1]
+ CRUSH rule 1 x 563 [2,8,1]
+ CRUSH rule 1 x 564 [8,1,2]
+ CRUSH rule 1 x 565 [4,8,2]
+ CRUSH rule 1 x 566 [4,8,2]
+ CRUSH rule 1 x 567 [4,8,1]
+ CRUSH rule 1 x 568 [8,2,1]
+ CRUSH rule 1 x 569 [4,2,1]
+ CRUSH rule 1 x 570 [1,8,2]
+ CRUSH rule 1 x 571 [6,8,1]
+ CRUSH rule 1 x 572 [4,1]
+ CRUSH rule 1 x 573 [1,2,8]
+ CRUSH rule 1 x 574 [2,1,8]
+ CRUSH rule 1 x 575 [8,2,1]
+ CRUSH rule 1 x 576 [4,8,2]
+ CRUSH rule 1 x 577 [8,2,1]
+ CRUSH rule 1 x 578 [8,1,2]
+ CRUSH rule 1 x 579 [4,1,8]
+ CRUSH rule 1 x 580 [8,2,1]
+ CRUSH rule 1 x 581 [8,2,1]
+ CRUSH rule 1 x 582 [2,8,4]
+ CRUSH rule 1 x 583 [8,1,2]
+ CRUSH rule 1 x 584 [8,1,2]
+ CRUSH rule 1 x 585 [8,1,4]
+ CRUSH rule 1 x 586 [1,2,8]
+ CRUSH rule 1 x 587 [2,4,1]
+ CRUSH rule 1 x 588 [4,1,8]
+ CRUSH rule 1 x 589 [8,1]
+ CRUSH rule 1 x 590 [8,2,1]
+ CRUSH rule 1 x 591 [4,2,1]
+ CRUSH rule 1 x 592 [2,1,4]
+ CRUSH rule 1 x 593 [1,8,2]
+ CRUSH rule 1 x 594 [2,8,1]
+ CRUSH rule 1 x 595 [8,1,2]
+ CRUSH rule 1 x 596 [2,8,1]
+ CRUSH rule 1 x 597 [1,2,8]
+ CRUSH rule 1 x 598 [8,2,1]
+ CRUSH rule 1 x 599 [4,2,1]
+ CRUSH rule 1 x 600 [8,1,2]
+ CRUSH rule 1 x 601 [1,8,4]
+ CRUSH rule 1 x 602 [2,8,1]
+ CRUSH rule 1 x 603 [1,2,8]
+ CRUSH rule 1 x 604 [8,2,1]
+ CRUSH rule 1 x 605 [8,2,1]
+ CRUSH rule 1 x 606 [2,1,8]
+ CRUSH rule 1 x 607 [2,4,8]
+ CRUSH rule 1 x 608 [4,1,2]
+ CRUSH rule 1 x 609 [4,2,1]
+ CRUSH rule 1 x 610 [1,8,2]
+ CRUSH rule 1 x 611 [1,2,8]
+ CRUSH rule 1 x 612 [2,1,8]
+ CRUSH rule 1 x 613 [8,2,1]
+ CRUSH rule 1 x 614 [8,4,1]
+ CRUSH rule 1 x 615 [8,1,2]
+ CRUSH rule 1 x 616 [1,8,2]
+ CRUSH rule 1 x 617 [8,1,2]
+ CRUSH rule 1 x 618 [8,1,4]
+ CRUSH rule 1 x 619 [4,1,8]
+ CRUSH rule 1 x 620 [8,1,2]
+ CRUSH rule 1 x 621 [2,8,1]
+ CRUSH rule 1 x 622 [2,4,1]
+ CRUSH rule 1 x 623 [2,8,1]
+ CRUSH rule 1 x 624 [4,8,1]
+ CRUSH rule 1 x 625 [2,1,8]
+ CRUSH rule 1 x 626 [8,1,2]
+ CRUSH rule 1 x 627 [2,8,1]
+ CRUSH rule 1 x 628 [8,2,1]
+ CRUSH rule 1 x 629 [2,8,4]
+ CRUSH rule 1 x 630 [2,8,1]
+ CRUSH rule 1 x 631 [1,8,4]
+ CRUSH rule 1 x 632 [8,2,1]
+ CRUSH rule 1 x 633 [8,2,1]
+ CRUSH rule 1 x 634 [1,8,2]
+ CRUSH rule 1 x 635 [4,8,1]
+ CRUSH rule 1 x 636 [1,4,2]
+ CRUSH rule 1 x 637 [1,2,8]
+ CRUSH rule 1 x 638 [8,1,2]
+ CRUSH rule 1 x 639 [2,1,8]
+ CRUSH rule 1 x 640 [1,2,8]
+ CRUSH rule 1 x 641 [8,2,1]
+ CRUSH rule 1 x 642 [2,1,8]
+ CRUSH rule 1 x 643 [8,2,1]
+ CRUSH rule 1 x 644 [8,1,2]
+ CRUSH rule 1 x 645 [2,1,8]
+ CRUSH rule 1 x 646 [8,1,4]
+ CRUSH rule 1 x 647 [8,1,2]
+ CRUSH rule 1 x 648 [1,8,2]
+ CRUSH rule 1 x 649 [4,8,1]
+ CRUSH rule 1 x 650 [8,4,1]
+ CRUSH rule 1 x 651 [4,6,8]
+ CRUSH rule 1 x 652 [4,8,1]
+ CRUSH rule 1 x 653 [8,1,2]
+ CRUSH rule 1 x 654 [6,1,2]
+ CRUSH rule 1 x 655 [1,4,8]
+ CRUSH rule 1 x 656 [8,2,1]
+ CRUSH rule 1 x 657 [6,1,2]
+ CRUSH rule 1 x 658 [8,1,2]
+ CRUSH rule 1 x 659 [4,8,1]
+ CRUSH rule 1 x 660 [8,1,2]
+ CRUSH rule 1 x 661 [1,8,2]
+ CRUSH rule 1 x 662 [8,2]
+ CRUSH rule 1 x 663 [1,4,8]
+ CRUSH rule 1 x 664 [1,4,2]
+ CRUSH rule 1 x 665 [4,6,8]
+ CRUSH rule 1 x 666 [2,8,1]
+ CRUSH rule 1 x 667 [1,4,2]
+ CRUSH rule 1 x 668 [4,8,2]
+ CRUSH rule 1 x 669 [6,4,2]
+ CRUSH rule 1 x 670 [4,2,1]
+ CRUSH rule 1 x 671 [2,1,8]
+ CRUSH rule 1 x 672 [4,8,2]
+ CRUSH rule 1 x 673 [4,2,1]
+ CRUSH rule 1 x 674 [8,1,2]
+ CRUSH rule 1 x 675 [1,8,6]
+ CRUSH rule 1 x 676 [2,1,4]
+ CRUSH rule 1 x 677 [4,1,8]
+ CRUSH rule 1 x 678 [2,4,1]
+ CRUSH rule 1 x 679 [8,2,1]
+ CRUSH rule 1 x 680 [2,8]
+ CRUSH rule 1 x 681 [2,8,1]
+ CRUSH rule 1 x 682 [1,4,2]
+ CRUSH rule 1 x 683 [1,2,4]
+ CRUSH rule 1 x 684 [8,1,4]
+ CRUSH rule 1 x 685 [8,1,2]
+ CRUSH rule 1 x 686 [1,4,2]
+ CRUSH rule 1 x 687 [1,6,8]
+ CRUSH rule 1 x 688 [4,8,2]
+ CRUSH rule 1 x 689 [8,4,2]
+ CRUSH rule 1 x 690 [8,1,4]
+ CRUSH rule 1 x 691 [8,1,2]
+ CRUSH rule 1 x 692 [8,2,1]
+ CRUSH rule 1 x 693 [8,4,1]
+ CRUSH rule 1 x 694 [8,4,1]
+ CRUSH rule 1 x 695 [2,8,4]
+ CRUSH rule 1 x 696 [1,2,8]
+ CRUSH rule 1 x 697 [8,1,2]
+ CRUSH rule 1 x 698 [8,2,1]
+ CRUSH rule 1 x 699 [1,8,2]
+ CRUSH rule 1 x 700 [1,2,8]
+ CRUSH rule 1 x 701 [2,1,8]
+ CRUSH rule 1 x 702 [8,1]
+ CRUSH rule 1 x 703 [8,1,2]
+ CRUSH rule 1 x 704 [1,4,8]
+ CRUSH rule 1 x 705 [8,4,2]
+ CRUSH rule 1 x 706 [1,2,4]
+ CRUSH rule 1 x 707 [8,2,1]
+ CRUSH rule 1 x 708 [4,8,1]
+ CRUSH rule 1 x 709 [8,2,1]
+ CRUSH rule 1 x 710 [8,1,2]
+ CRUSH rule 1 x 711 [2,4,8]
+ CRUSH rule 1 x 712 [2,8,1]
+ CRUSH rule 1 x 713 [8,2,4]
+ CRUSH rule 1 x 714 [1,2,8]
+ CRUSH rule 1 x 715 [1,2,8]
+ CRUSH rule 1 x 716 [4,8,2]
+ CRUSH rule 1 x 717 [8,4,2]
+ CRUSH rule 1 x 718 [2,8,6]
+ CRUSH rule 1 x 719 [2,6,4]
+ CRUSH rule 1 x 720 [8,1,2]
+ CRUSH rule 1 x 721 [4,6,8]
+ CRUSH rule 1 x 722 [8,1,2]
+ CRUSH rule 1 x 723 [4,1,2]
+ CRUSH rule 1 x 724 [2,6,1]
+ CRUSH rule 1 x 725 [1,2,8]
+ CRUSH rule 1 x 726 [4,8,1]
+ CRUSH rule 1 x 727 [4,8,1]
+ CRUSH rule 1 x 728 [2,1,8]
+ CRUSH rule 1 x 729 [2,8]
+ CRUSH rule 1 x 730 [4,8,2]
+ CRUSH rule 1 x 731 [4,1,8]
+ CRUSH rule 1 x 732 [1,2,8]
+ CRUSH rule 1 x 733 [4,1,8]
+ CRUSH rule 1 x 734 [8,4,2]
+ CRUSH rule 1 x 735 [4,8,2]
+ CRUSH rule 1 x 736 [4,8]
+ CRUSH rule 1 x 737 [1,2,8]
+ CRUSH rule 1 x 738 [4,2,8]
+ CRUSH rule 1 x 739 [2,1,8]
+ CRUSH rule 1 x 740 [1,2,8]
+ CRUSH rule 1 x 741 [8,2,1]
+ CRUSH rule 1 x 742 [8,2,1]
+ CRUSH rule 1 x 743 [8,1,2]
+ CRUSH rule 1 x 744 [4,8,2]
+ CRUSH rule 1 x 745 [8,2,1]
+ CRUSH rule 1 x 746 [8,1,2]
+ CRUSH rule 1 x 747 [8,1,2]
+ CRUSH rule 1 x 748 [2,8,1]
+ CRUSH rule 1 x 749 [4,8,1]
+ CRUSH rule 1 x 750 [1,8,4]
+ CRUSH rule 1 x 751 [2,1,8]
+ CRUSH rule 1 x 752 [8,1]
+ CRUSH rule 1 x 753 [8,1,4]
+ CRUSH rule 1 x 754 [8,4,2]
+ CRUSH rule 1 x 755 [1,2,4]
+ CRUSH rule 1 x 756 [8,2,1]
+ CRUSH rule 1 x 757 [8,4,1]
+ CRUSH rule 1 x 758 [8,2]
+ CRUSH rule 1 x 759 [8,4,1]
+ CRUSH rule 1 x 760 [1,4,2]
+ CRUSH rule 1 x 761 [1,2,6]
+ CRUSH rule 1 x 762 [2,8,1]
+ CRUSH rule 1 x 763 [8,2,4]
+ CRUSH rule 1 x 764 [1,8,2]
+ CRUSH rule 1 x 765 [8,2,1]
+ CRUSH rule 1 x 766 [8]
+ CRUSH rule 1 x 767 [1,2,4]
+ CRUSH rule 1 x 768 [8,4,2]
+ CRUSH rule 1 x 769 [8,2,4]
+ CRUSH rule 1 x 770 [8,2,4]
+ CRUSH rule 1 x 771 [8,1,4]
+ CRUSH rule 1 x 772 [8,4,2]
+ CRUSH rule 1 x 773 [4,1,8]
+ CRUSH rule 1 x 774 [8,1,2]
+ CRUSH rule 1 x 775 [8,2,4]
+ CRUSH rule 1 x 776 [6,2,1]
+ CRUSH rule 1 x 777 [4,1,8]
+ CRUSH rule 1 x 778 [1,8,2]
+ CRUSH rule 1 x 779 [2,8,1]
+ CRUSH rule 1 x 780 [2,1,4]
+ CRUSH rule 1 x 781 [8,2,1]
+ CRUSH rule 1 x 782 [4,1,2]
+ CRUSH rule 1 x 783 [8,1,4]
+ CRUSH rule 1 x 784 [1,2,4]
+ CRUSH rule 1 x 785 [8,1,2]
+ CRUSH rule 1 x 786 [8,2,1]
+ CRUSH rule 1 x 787 [1,2,6]
+ CRUSH rule 1 x 788 [8,2,1]
+ CRUSH rule 1 x 789 [1,6,8]
+ CRUSH rule 1 x 790 [8,2,1]
+ CRUSH rule 1 x 791 [4,8,1]
+ CRUSH rule 1 x 792 [4,8]
+ CRUSH rule 1 x 793 [8,1,4]
+ CRUSH rule 1 x 794 [2,8,4]
+ CRUSH rule 1 x 795 [1,8,2]
+ CRUSH rule 1 x 796 [2,8,1]
+ CRUSH rule 1 x 797 [2,4,8]
+ CRUSH rule 1 x 798 [6,8,1]
+ CRUSH rule 1 x 799 [4,1,8]
+ CRUSH rule 1 x 800 [8,2,1]
+ CRUSH rule 1 x 801 [4,8,2]
+ CRUSH rule 1 x 802 [1,8,2]
+ CRUSH rule 1 x 803 [2,8,1]
+ CRUSH rule 1 x 804 [8,2,1]
+ CRUSH rule 1 x 805 [2,8,1]
+ CRUSH rule 1 x 806 [1,4,2]
+ CRUSH rule 1 x 807 [4,8]
+ CRUSH rule 1 x 808 [2,8,1]
+ CRUSH rule 1 x 809 [1,8,2]
+ CRUSH rule 1 x 810 [2,8,1]
+ CRUSH rule 1 x 811 [8,2,1]
+ CRUSH rule 1 x 812 [8,4,1]
+ CRUSH rule 1 x 813 [8,4,2]
+ CRUSH rule 1 x 814 [8,1]
+ CRUSH rule 1 x 815 [4,1,2]
+ CRUSH rule 1 x 816 [2,1,8]
+ CRUSH rule 1 x 817 [8,1,2]
+ CRUSH rule 1 x 818 [8,2,1]
+ CRUSH rule 1 x 819 [8,1,2]
+ CRUSH rule 1 x 820 [4,1,2]
+ CRUSH rule 1 x 821 [4,1,8]
+ CRUSH rule 1 x 822 [2,1,4]
+ CRUSH rule 1 x 823 [4,8,2]
+ CRUSH rule 1 x 824 [8,2,1]
+ CRUSH rule 1 x 825 [2,8,4]
+ CRUSH rule 1 x 826 [8,2,4]
+ CRUSH rule 1 x 827 [2,8,1]
+ CRUSH rule 1 x 828 [2,1,8]
+ CRUSH rule 1 x 829 [8,2]
+ CRUSH rule 1 x 830 [2,4,1]
+ CRUSH rule 1 x 831 [1,8,2]
+ CRUSH rule 1 x 832 [4,8,1]
+ CRUSH rule 1 x 833 [2,1,8]
+ CRUSH rule 1 x 834 [2,1]
+ CRUSH rule 1 x 835 [8,4,2]
+ CRUSH rule 1 x 836 [4,1,2]
+ CRUSH rule 1 x 837 [8,4,1]
+ CRUSH rule 1 x 838 [6,8,2]
+ CRUSH rule 1 x 839 [8,2,6]
+ CRUSH rule 1 x 840 [8,2,1]
+ CRUSH rule 1 x 841 [4,8,2]
+ CRUSH rule 1 x 842 [2,1]
+ CRUSH rule 1 x 843 [8,4,1]
+ CRUSH rule 1 x 844 [1,8,2]
+ CRUSH rule 1 x 845 [4,8,2]
+ CRUSH rule 1 x 846 [4,2,1]
+ CRUSH rule 1 x 847 [2,1,8]
+ CRUSH rule 1 x 848 [2,8,1]
+ CRUSH rule 1 x 849 [4,8,1]
+ CRUSH rule 1 x 850 [1,2,6]
+ CRUSH rule 1 x 851 [6,8]
+ CRUSH rule 1 x 852 [8,4,2]
+ CRUSH rule 1 x 853 [6,8,1]
+ CRUSH rule 1 x 854 [8,1,2]
+ CRUSH rule 1 x 855 [8,1,2]
+ CRUSH rule 1 x 856 [8,4,1]
+ CRUSH rule 1 x 857 [8,1,2]
+ CRUSH rule 1 x 858 [6,2,1]
+ CRUSH rule 1 x 859 [8,2,1]
+ CRUSH rule 1 x 860 [8,1,2]
+ CRUSH rule 1 x 861 [8,1]
+ CRUSH rule 1 x 862 [8,1]
+ CRUSH rule 1 x 863 [8,1,2]
+ CRUSH rule 1 x 864 [8,2,1]
+ CRUSH rule 1 x 865 [8,1,2]
+ CRUSH rule 1 x 866 [2,8]
+ CRUSH rule 1 x 867 [8,1]
+ CRUSH rule 1 x 868 [8,2,1]
+ CRUSH rule 1 x 869 [8,6,4]
+ CRUSH rule 1 x 870 [2,1,8]
+ CRUSH rule 1 x 871 [2,8,1]
+ CRUSH rule 1 x 872 [8,1,2]
+ CRUSH rule 1 x 873 [4,8,2]
+ CRUSH rule 1 x 874 [2,6,1]
+ CRUSH rule 1 x 875 [2,8,4]
+ CRUSH rule 1 x 876 [4,8,1]
+ CRUSH rule 1 x 877 [8,4,2]
+ CRUSH rule 1 x 878 [8,1,2]
+ CRUSH rule 1 x 879 [8,2,1]
+ CRUSH rule 1 x 880 [6,1,2]
+ CRUSH rule 1 x 881 [4,8,1]
+ CRUSH rule 1 x 882 [8,2,1]
+ CRUSH rule 1 x 883 [2,1,4]
+ CRUSH rule 1 x 884 [8,2,4]
+ CRUSH rule 1 x 885 [4,1,8]
+ CRUSH rule 1 x 886 [2,8,1]
+ CRUSH rule 1 x 887 [8,4,1]
+ CRUSH rule 1 x 888 [8,2,1]
+ CRUSH rule 1 x 889 [2,1,8]
+ CRUSH rule 1 x 890 [8,2,1]
+ CRUSH rule 1 x 891 [1,8,2]
+ CRUSH rule 1 x 892 [8,2,4]
+ CRUSH rule 1 x 893 [2,8,6]
+ CRUSH rule 1 x 894 [8,4,2]
+ CRUSH rule 1 x 895 [4,8,2]
+ CRUSH rule 1 x 896 [1,8,2]
+ CRUSH rule 1 x 897 [8,2,1]
+ CRUSH rule 1 x 898 [1,4,8]
+ CRUSH rule 1 x 899 [1,8,2]
+ CRUSH rule 1 x 900 [4,1,2]
+ CRUSH rule 1 x 901 [8,1,2]
+ CRUSH rule 1 x 902 [8,4,2]
+ CRUSH rule 1 x 903 [1,8,2]
+ CRUSH rule 1 x 904 [1,8,2]
+ CRUSH rule 1 x 905 [8,2,1]
+ CRUSH rule 1 x 906 [1,2,8]
+ CRUSH rule 1 x 907 [8,1,2]
+ CRUSH rule 1 x 908 [8,1,2]
+ CRUSH rule 1 x 909 [2,1,8]
+ CRUSH rule 1 x 910 [8,2,1]
+ CRUSH rule 1 x 911 [8,1,2]
+ CRUSH rule 1 x 912 [1,2,8]
+ CRUSH rule 1 x 913 [8,4,1]
+ CRUSH rule 1 x 914 [6,4,8]
+ CRUSH rule 1 x 915 [8,2]
+ CRUSH rule 1 x 916 [4,1,2]
+ CRUSH rule 1 x 917 [1,4,8]
+ CRUSH rule 1 x 918 [8,2,1]
+ CRUSH rule 1 x 919 [8,2,1]
+ CRUSH rule 1 x 920 [8,1,2]
+ CRUSH rule 1 x 921 [1,2,8]
+ CRUSH rule 1 x 922 [8,4,2]
+ CRUSH rule 1 x 923 [4,8,1]
+ CRUSH rule 1 x 924 [1,8,2]
+ CRUSH rule 1 x 925 [4,8,2]
+ CRUSH rule 1 x 926 [1,8,2]
+ CRUSH rule 1 x 927 [1,8,4]
+ CRUSH rule 1 x 928 [8,1,2]
+ CRUSH rule 1 x 929 [4,1,2]
+ CRUSH rule 1 x 930 [2,1,8]
+ CRUSH rule 1 x 931 [8,1,2]
+ CRUSH rule 1 x 932 [4,8,1]
+ CRUSH rule 1 x 933 [8,4,2]
+ CRUSH rule 1 x 934 [8,1,2]
+ CRUSH rule 1 x 935 [8,1,2]
+ CRUSH rule 1 x 936 [1,8,2]
+ CRUSH rule 1 x 937 [4,8]
+ CRUSH rule 1 x 938 [8,4,2]
+ CRUSH rule 1 x 939 [2,8,1]
+ CRUSH rule 1 x 940 [8,2,1]
+ CRUSH rule 1 x 941 [8,2,1]
+ CRUSH rule 1 x 942 [1,2,8]
+ CRUSH rule 1 x 943 [8,2,1]
+ CRUSH rule 1 x 944 [2,1,8]
+ CRUSH rule 1 x 945 [8,2,4]
+ CRUSH rule 1 x 946 [2,1,8]
+ CRUSH rule 1 x 947 [8,1,2]
+ CRUSH rule 1 x 948 [8,1,2]
+ CRUSH rule 1 x 949 [6,1,8]
+ CRUSH rule 1 x 950 [8,1,2]
+ CRUSH rule 1 x 951 [2,8,1]
+ CRUSH rule 1 x 952 [2,1,8]
+ CRUSH rule 1 x 953 [1,4,2]
+ CRUSH rule 1 x 954 [8,2,1]
+ CRUSH rule 1 x 955 [8,1,2]
+ CRUSH rule 1 x 956 [1,2,8]
+ CRUSH rule 1 x 957 [8,1,2]
+ CRUSH rule 1 x 958 [8,2,4]
+ CRUSH rule 1 x 959 [4,2,8]
+ CRUSH rule 1 x 960 [2,6,8]
+ CRUSH rule 1 x 961 [1,2,8]
+ CRUSH rule 1 x 962 [8,4,1]
+ CRUSH rule 1 x 963 [2,4,1]
+ CRUSH rule 1 x 964 [8,1,2]
+ CRUSH rule 1 x 965 [8,2,1]
+ CRUSH rule 1 x 966 [4,8,1]
+ CRUSH rule 1 x 967 [8,1,4]
+ CRUSH rule 1 x 968 [8,2,1]
+ CRUSH rule 1 x 969 [8,2,4]
+ CRUSH rule 1 x 970 [2,8,4]
+ CRUSH rule 1 x 971 [1,8,2]
+ CRUSH rule 1 x 972 [1,8,2]
+ CRUSH rule 1 x 973 [1,2]
+ CRUSH rule 1 x 974 [4,1,2]
+ CRUSH rule 1 x 975 [4,8]
+ CRUSH rule 1 x 976 [4,8,2]
+ CRUSH rule 1 x 977 [8,4,2]
+ CRUSH rule 1 x 978 [8,2,1]
+ CRUSH rule 1 x 979 [8,1,2]
+ CRUSH rule 1 x 980 [8,2,1]
+ CRUSH rule 1 x 981 [8,2]
+ CRUSH rule 1 x 982 [1,2,8]
+ CRUSH rule 1 x 983 [4,8,1]
+ CRUSH rule 1 x 984 [2,1,8]
+ CRUSH rule 1 x 985 [2,4,8]
+ CRUSH rule 1 x 986 [8,4,1]
+ CRUSH rule 1 x 987 [2,1,8]
+ CRUSH rule 1 x 988 [1,4,6]
+ CRUSH rule 1 x 989 [1,8,2]
+ CRUSH rule 1 x 990 [1,2,8]
+ CRUSH rule 1 x 991 [1,4,2]
+ CRUSH rule 1 x 992 [8,1,4]
+ CRUSH rule 1 x 993 [2,8,1]
+ CRUSH rule 1 x 994 [4,2,1]
+ CRUSH rule 1 x 995 [8,1,2]
+ CRUSH rule 1 x 996 [8,2,4]
+ CRUSH rule 1 x 997 [8,4,1]
+ CRUSH rule 1 x 998 [8,1,2]
+ CRUSH rule 1 x 999 [1,8,4]
+ CRUSH rule 1 x 1000 [8,4,2]
+ CRUSH rule 1 x 1001 [2,1]
+ CRUSH rule 1 x 1002 [1,2,8]
+ CRUSH rule 1 x 1003 [2,8]
+ CRUSH rule 1 x 1004 [8,1,2]
+ CRUSH rule 1 x 1005 [8,1,2]
+ CRUSH rule 1 x 1006 [1,2,4]
+ CRUSH rule 1 x 1007 [1,2,4]
+ CRUSH rule 1 x 1008 [1,8,2]
+ CRUSH rule 1 x 1009 [6,8,4]
+ CRUSH rule 1 x 1010 [2,8,1]
+ CRUSH rule 1 x 1011 [4,2,8]
+ CRUSH rule 1 x 1012 [1,2,8]
+ CRUSH rule 1 x 1013 [1,2,8]
+ CRUSH rule 1 x 1014 [2,8,4]
+ CRUSH rule 1 x 1015 [8,1,2]
+ CRUSH rule 1 x 1016 [2,1,4]
+ CRUSH rule 1 x 1017 [6,2,1]
+ CRUSH rule 1 x 1018 [4,2,1]
+ CRUSH rule 1 x 1019 [4,8,2]
+ CRUSH rule 1 x 1020 [1,2,8]
+ CRUSH rule 1 x 1021 [8,2,1]
+ CRUSH rule 1 x 1022 [1,8,4]
+ CRUSH rule 1 x 1023 [4,2,1]
+ rule 1 (choose-two) num_rep 3 result size == 1:\t2/1024 (esc)
+ rule 1 (choose-two) num_rep 3 result size == 2:\t82/1024 (esc)
+ rule 1 (choose-two) num_rep 3 result size == 3:\t940/1024 (esc)
+ rule 2 (chooseleaf), x = 0..1023, numrep = 2..3
+ CRUSH rule 2 x 0 [2,4]
+ CRUSH rule 2 x 1 [2,8]
+ CRUSH rule 2 x 2 [1,8]
+ CRUSH rule 2 x 3 [8,1]
+ CRUSH rule 2 x 4 [4,2]
+ CRUSH rule 2 x 5 [8,2]
+ CRUSH rule 2 x 6 [2,8]
+ CRUSH rule 2 x 7 [4,8]
+ CRUSH rule 2 x 8 [4,8]
+ CRUSH rule 2 x 9 [2,4]
+ CRUSH rule 2 x 10 [2,8]
+ CRUSH rule 2 x 11 [2,8]
+ CRUSH rule 2 x 12 [2,8]
+ CRUSH rule 2 x 13 [4,8]
+ CRUSH rule 2 x 14 [8,2]
+ CRUSH rule 2 x 15 [8,2]
+ CRUSH rule 2 x 16 [8,2]
+ CRUSH rule 2 x 17 [4,1]
+ CRUSH rule 2 x 18 [1,8]
+ CRUSH rule 2 x 19 [8,4]
+ CRUSH rule 2 x 20 [2,8]
+ CRUSH rule 2 x 21 [8,2]
+ CRUSH rule 2 x 22 [8,1]
+ CRUSH rule 2 x 23 [4,8]
+ CRUSH rule 2 x 24 [1,8]
+ CRUSH rule 2 x 25 [4,8]
+ CRUSH rule 2 x 26 [2,8]
+ CRUSH rule 2 x 27 [4,1]
+ CRUSH rule 2 x 28 [8,2]
+ CRUSH rule 2 x 29 [8,4]
+ CRUSH rule 2 x 30 [4,8]
+ CRUSH rule 2 x 31 [8,1]
+ CRUSH rule 2 x 32 [6,1]
+ CRUSH rule 2 x 33 [2,8]
+ CRUSH rule 2 x 34 [2,8]
+ CRUSH rule 2 x 35 [1,8]
+ CRUSH rule 2 x 36 [8,2]
+ CRUSH rule 2 x 37 [1,8]
+ CRUSH rule 2 x 38 [4,8]
+ CRUSH rule 2 x 39 [8,2]
+ CRUSH rule 2 x 40 [8,2]
+ CRUSH rule 2 x 41 [2,8]
+ CRUSH rule 2 x 42 [8,2]
+ CRUSH rule 2 x 43 [1,8]
+ CRUSH rule 2 x 44 [1,8]
+ CRUSH rule 2 x 45 [8,2]
+ CRUSH rule 2 x 46 [2,8]
+ CRUSH rule 2 x 47 [4,2]
+ CRUSH rule 2 x 48 [8,1]
+ CRUSH rule 2 x 49 [8,2]
+ CRUSH rule 2 x 50 [4,1]
+ CRUSH rule 2 x 51 [8,2]
+ CRUSH rule 2 x 52 [8,1]
+ CRUSH rule 2 x 53 [4,8]
+ CRUSH rule 2 x 54 [8,4]
+ CRUSH rule 2 x 55 [8,2]
+ CRUSH rule 2 x 56 [8,4]
+ CRUSH rule 2 x 57 [8,1]
+ CRUSH rule 2 x 58 [1,8]
+ CRUSH rule 2 x 59 [2,8]
+ CRUSH rule 2 x 60 [4,2]
+ CRUSH rule 2 x 61 [4,8]
+ CRUSH rule 2 x 62 [8,1]
+ CRUSH rule 2 x 63 [8,2]
+ CRUSH rule 2 x 64 [4,2]
+ CRUSH rule 2 x 65 [8,4]
+ CRUSH rule 2 x 66 [4,8]
+ CRUSH rule 2 x 67 [4,2]
+ CRUSH rule 2 x 68 [1,8]
+ CRUSH rule 2 x 69 [2,8]
+ CRUSH rule 2 x 70 [8,2]
+ CRUSH rule 2 x 71 [2,8]
+ CRUSH rule 2 x 72 [8,1]
+ CRUSH rule 2 x 73 [2,8]
+ CRUSH rule 2 x 74 [1,8]
+ CRUSH rule 2 x 75 [4,2]
+ CRUSH rule 2 x 76 [4,1]
+ CRUSH rule 2 x 77 [8,2]
+ CRUSH rule 2 x 78 [1,6]
+ CRUSH rule 2 x 79 [4,1]
+ CRUSH rule 2 x 80 [2,4]
+ CRUSH rule 2 x 81 [2,8]
+ CRUSH rule 2 x 82 [6,1]
+ CRUSH rule 2 x 83 [2,8]
+ CRUSH rule 2 x 84 [8,2]
+ CRUSH rule 2 x 85 [4,8]
+ CRUSH rule 2 x 86 [2,8]
+ CRUSH rule 2 x 87 [2,8]
+ CRUSH rule 2 x 88 [1,6]
+ CRUSH rule 2 x 89 [1,8]
+ CRUSH rule 2 x 90 [8,4]
+ CRUSH rule 2 x 91 [4,8]
+ CRUSH rule 2 x 92 [1,8]
+ CRUSH rule 2 x 93 [8,4]
+ CRUSH rule 2 x 94 [1,8]
+ CRUSH rule 2 x 95 [8,1]
+ CRUSH rule 2 x 96 [8,2]
+ CRUSH rule 2 x 97 [8,1]
+ CRUSH rule 2 x 98 [2,8]
+ CRUSH rule 2 x 99 [2,8]
+ CRUSH rule 2 x 100 [1,8]
+ CRUSH rule 2 x 101 [8,1]
+ CRUSH rule 2 x 102 [2,8]
+ CRUSH rule 2 x 103 [8,2]
+ CRUSH rule 2 x 104 [8,4]
+ CRUSH rule 2 x 105 [2,4]
+ CRUSH rule 2 x 106 [1,8]
+ CRUSH rule 2 x 107 [1,8]
+ CRUSH rule 2 x 108 [8,2]
+ CRUSH rule 2 x 109 [1,4]
+ CRUSH rule 2 x 110 [4,2]
+ CRUSH rule 2 x 111 [2,4]
+ CRUSH rule 2 x 112 [2,8]
+ CRUSH rule 2 x 113 [8,2]
+ CRUSH rule 2 x 114 [8,4]
+ CRUSH rule 2 x 115 [8,2]
+ CRUSH rule 2 x 116 [1,8]
+ CRUSH rule 2 x 117 [6,1]
+ CRUSH rule 2 x 118 [2,8]
+ CRUSH rule 2 x 119 [8,1]
+ CRUSH rule 2 x 120 [2,4]
+ CRUSH rule 2 x 121 [2,8]
+ CRUSH rule 2 x 122 [8,1]
+ CRUSH rule 2 x 123 [2,8]
+ CRUSH rule 2 x 124 [2,8]
+ CRUSH rule 2 x 125 [1,8]
+ CRUSH rule 2 x 126 [1,8]
+ CRUSH rule 2 x 127 [4,8]
+ CRUSH rule 2 x 128 [8,2]
+ CRUSH rule 2 x 129 [2,4]
+ CRUSH rule 2 x 130 [4,8]
+ CRUSH rule 2 x 131 [1,4]
+ CRUSH rule 2 x 132 [1,8]
+ CRUSH rule 2 x 133 [8,1]
+ CRUSH rule 2 x 134 [1,8]
+ CRUSH rule 2 x 135 [4,8]
+ CRUSH rule 2 x 136 [2,4]
+ CRUSH rule 2 x 137 [8,4]
+ CRUSH rule 2 x 138 [8,4]
+ CRUSH rule 2 x 139 [4,2]
+ CRUSH rule 2 x 140 [1,8]
+ CRUSH rule 2 x 141 [8,2]
+ CRUSH rule 2 x 142 [4,1]
+ CRUSH rule 2 x 143 [4,8]
+ CRUSH rule 2 x 144 [8,1]
+ CRUSH rule 2 x 145 [8,1]
+ CRUSH rule 2 x 146 [2,8]
+ CRUSH rule 2 x 147 [2,8]
+ CRUSH rule 2 x 148 [4,1]
+ CRUSH rule 2 x 149 [4,8]
+ CRUSH rule 2 x 150 [1,8]
+ CRUSH rule 2 x 151 [1,8]
+ CRUSH rule 2 x 152 [8,2]
+ CRUSH rule 2 x 153 [8,4]
+ CRUSH rule 2 x 154 [4,2]
+ CRUSH rule 2 x 155 [4,8]
+ CRUSH rule 2 x 156 [4,2]
+ CRUSH rule 2 x 157 [1,8]
+ CRUSH rule 2 x 158 [2,8]
+ CRUSH rule 2 x 159 [8,2]
+ CRUSH rule 2 x 160 [2,8]
+ CRUSH rule 2 x 161 [1,4]
+ CRUSH rule 2 x 162 [1,8]
+ CRUSH rule 2 x 163 [4,8]
+ CRUSH rule 2 x 164 [8,1]
+ CRUSH rule 2 x 165 [8,2]
+ CRUSH rule 2 x 166 [2,8]
+ CRUSH rule 2 x 167 [1,8]
+ CRUSH rule 2 x 168 [4,2]
+ CRUSH rule 2 x 169 [2,8]
+ CRUSH rule 2 x 170 [1,8]
+ CRUSH rule 2 x 171 [8,4]
+ CRUSH rule 2 x 172 [1,8]
+ CRUSH rule 2 x 173 [8,4]
+ CRUSH rule 2 x 174 [1,6]
+ CRUSH rule 2 x 175 [8,1]
+ CRUSH rule 2 x 176 [2,8]
+ CRUSH rule 2 x 177 [1,8]
+ CRUSH rule 2 x 178 [4,2]
+ CRUSH rule 2 x 179 [1,8]
+ CRUSH rule 2 x 180 [8,1]
+ CRUSH rule 2 x 181 [8,2]
+ CRUSH rule 2 x 182 [8,1]
+ CRUSH rule 2 x 183 [8,4]
+ CRUSH rule 2 x 184 [4,8]
+ CRUSH rule 2 x 185 [8,1]
+ CRUSH rule 2 x 186 [2,4]
+ CRUSH rule 2 x 187 [1,8]
+ CRUSH rule 2 x 188 [1,8]
+ CRUSH rule 2 x 189 [1,8]
+ CRUSH rule 2 x 190 [1,8]
+ CRUSH rule 2 x 191 [8,1]
+ CRUSH rule 2 x 192 [4,1]
+ CRUSH rule 2 x 193 [4,2]
+ CRUSH rule 2 x 194 [1,8]
+ CRUSH rule 2 x 195 [8,4]
+ CRUSH rule 2 x 196 [8,2]
+ CRUSH rule 2 x 197 [8,4]
+ CRUSH rule 2 x 198 [2,8]
+ CRUSH rule 2 x 199 [1,4]
+ CRUSH rule 2 x 200 [1,8]
+ CRUSH rule 2 x 201 [8,1]
+ CRUSH rule 2 x 202 [8,1]
+ CRUSH rule 2 x 203 [8,1]
+ CRUSH rule 2 x 204 [2,4]
+ CRUSH rule 2 x 205 [1,8]
+ CRUSH rule 2 x 206 [1,8]
+ CRUSH rule 2 x 207 [2,8]
+ CRUSH rule 2 x 208 [8,1]
+ CRUSH rule 2 x 209 [1,8]
+ CRUSH rule 2 x 210 [1,4]
+ CRUSH rule 2 x 211 [4,2]
+ CRUSH rule 2 x 212 [8,1]
+ CRUSH rule 2 x 213 [8,4]
+ CRUSH rule 2 x 214 [8,2]
+ CRUSH rule 2 x 215 [8,1]
+ CRUSH rule 2 x 216 [2,8]
+ CRUSH rule 2 x 217 [1,8]
+ CRUSH rule 2 x 218 [2,8]
+ CRUSH rule 2 x 219 [8,2]
+ CRUSH rule 2 x 220 [4,8]
+ CRUSH rule 2 x 221 [8,1]
+ CRUSH rule 2 x 222 [8,1]
+ CRUSH rule 2 x 223 [1,8]
+ CRUSH rule 2 x 224 [1,4]
+ CRUSH rule 2 x 225 [8,2]
+ CRUSH rule 2 x 226 [8,2]
+ CRUSH rule 2 x 227 [4,1]
+ CRUSH rule 2 x 228 [8,2]
+ CRUSH rule 2 x 229 [4,8]
+ CRUSH rule 2 x 230 [4,8]
+ CRUSH rule 2 x 231 [4,8]
+ CRUSH rule 2 x 232 [2,8]
+ CRUSH rule 2 x 233 [2,8]
+ CRUSH rule 2 x 234 [1,8]
+ CRUSH rule 2 x 235 [4,8]
+ CRUSH rule 2 x 236 [2,6]
+ CRUSH rule 2 x 237 [4,8]
+ CRUSH rule 2 x 238 [2,8]
+ CRUSH rule 2 x 239 [8,1]
+ CRUSH rule 2 x 240 [4,8]
+ CRUSH rule 2 x 241 [1,8]
+ CRUSH rule 2 x 242 [8,2]
+ CRUSH rule 2 x 243 [8,2]
+ CRUSH rule 2 x 244 [4,8]
+ CRUSH rule 2 x 245 [8,1]
+ CRUSH rule 2 x 246 [1,8]
+ CRUSH rule 2 x 247 [8,2]
+ CRUSH rule 2 x 248 [8,2]
+ CRUSH rule 2 x 249 [2,8]
+ CRUSH rule 2 x 250 [2,4]
+ CRUSH rule 2 x 251 [2,8]
+ CRUSH rule 2 x 252 [4,8]
+ CRUSH rule 2 x 253 [2,8]
+ CRUSH rule 2 x 254 [4,2]
+ CRUSH rule 2 x 255 [1,8]
+ CRUSH rule 2 x 256 [4,8]
+ CRUSH rule 2 x 257 [2,8]
+ CRUSH rule 2 x 258 [4,2]
+ CRUSH rule 2 x 259 [6,2]
+ CRUSH rule 2 x 260 [8,2]
+ CRUSH rule 2 x 261 [8,1]
+ CRUSH rule 2 x 262 [8,1]
+ CRUSH rule 2 x 263 [8,1]
+ CRUSH rule 2 x 264 [8,2]
+ CRUSH rule 2 x 265 [8,2]
+ CRUSH rule 2 x 266 [8,2]
+ CRUSH rule 2 x 267 [2,8]
+ CRUSH rule 2 x 268 [1,8]
+ CRUSH rule 2 x 269 [1,8]
+ CRUSH rule 2 x 270 [4,1]
+ CRUSH rule 2 x 271 [8,4]
+ CRUSH rule 2 x 272 [2,8]
+ CRUSH rule 2 x 273 [4,1]
+ CRUSH rule 2 x 274 [8,4]
+ CRUSH rule 2 x 275 [4,8]
+ CRUSH rule 2 x 276 [8,1]
+ CRUSH rule 2 x 277 [8,1]
+ CRUSH rule 2 x 278 [8,1]
+ CRUSH rule 2 x 279 [8,4]
+ CRUSH rule 2 x 280 [2,8]
+ CRUSH rule 2 x 281 [8,2]
+ CRUSH rule 2 x 282 [2,8]
+ CRUSH rule 2 x 283 [8,2]
+ CRUSH rule 2 x 284 [8,2]
+ CRUSH rule 2 x 285 [4,8]
+ CRUSH rule 2 x 286 [2,8]
+ CRUSH rule 2 x 287 [1,8]
+ CRUSH rule 2 x 288 [8,1]
+ CRUSH rule 2 x 289 [4,8]
+ CRUSH rule 2 x 290 [1,4]
+ CRUSH rule 2 x 291 [1,4]
+ CRUSH rule 2 x 292 [8,2]
+ CRUSH rule 2 x 293 [8,1]
+ CRUSH rule 2 x 294 [8,4]
+ CRUSH rule 2 x 295 [4,8]
+ CRUSH rule 2 x 296 [4,1]
+ CRUSH rule 2 x 297 [8,2]
+ CRUSH rule 2 x 298 [1,8]
+ CRUSH rule 2 x 299 [2,8]
+ CRUSH rule 2 x 300 [8,2]
+ CRUSH rule 2 x 301 [1,8]
+ CRUSH rule 2 x 302 [1,8]
+ CRUSH rule 2 x 303 [8,4]
+ CRUSH rule 2 x 304 [2,8]
+ CRUSH rule 2 x 305 [8,2]
+ CRUSH rule 2 x 306 [1,8]
+ CRUSH rule 2 x 307 [2,8]
+ CRUSH rule 2 x 308 [2,8]
+ CRUSH rule 2 x 309 [8,1]
+ CRUSH rule 2 x 310 [4,1]
+ CRUSH rule 2 x 311 [4,8]
+ CRUSH rule 2 x 312 [2,8]
+ CRUSH rule 2 x 313 [4,1]
+ CRUSH rule 2 x 314 [2,8]
+ CRUSH rule 2 x 315 [2,8]
+ CRUSH rule 2 x 316 [8,1]
+ CRUSH rule 2 x 317 [2,8]
+ CRUSH rule 2 x 318 [8,1]
+ CRUSH rule 2 x 319 [2,8]
+ CRUSH rule 2 x 320 [8,1]
+ CRUSH rule 2 x 321 [1,8]
+ CRUSH rule 2 x 322 [2,8]
+ CRUSH rule 2 x 323 [4,8]
+ CRUSH rule 2 x 324 [8,1]
+ CRUSH rule 2 x 325 [4,8]
+ CRUSH rule 2 x 326 [1,6]
+ CRUSH rule 2 x 327 [1,8]
+ CRUSH rule 2 x 328 [8,4]
+ CRUSH rule 2 x 329 [4,8]
+ CRUSH rule 2 x 330 [4,8]
+ CRUSH rule 2 x 331 [2,8]
+ CRUSH rule 2 x 332 [2,8]
+ CRUSH rule 2 x 333 [8,1]
+ CRUSH rule 2 x 334 [8,2]
+ CRUSH rule 2 x 335 [8,1]
+ CRUSH rule 2 x 336 [4,8]
+ CRUSH rule 2 x 337 [8,2]
+ CRUSH rule 2 x 338 [8,1]
+ CRUSH rule 2 x 339 [8,2]
+ CRUSH rule 2 x 340 [2,8]
+ CRUSH rule 2 x 341 [4,1]
+ CRUSH rule 2 x 342 [2,8]
+ CRUSH rule 2 x 343 [8,1]
+ CRUSH rule 2 x 344 [6,2]
+ CRUSH rule 2 x 345 [2,8]
+ CRUSH rule 2 x 346 [8,2]
+ CRUSH rule 2 x 347 [4,1]
+ CRUSH rule 2 x 348 [8,2]
+ CRUSH rule 2 x 349 [1,8]
+ CRUSH rule 2 x 350 [8,1]
+ CRUSH rule 2 x 351 [8,2]
+ CRUSH rule 2 x 352 [1,8]
+ CRUSH rule 2 x 353 [8,1]
+ CRUSH rule 2 x 354 [1,8]
+ CRUSH rule 2 x 355 [8,2]
+ CRUSH rule 2 x 356 [4,1]
+ CRUSH rule 2 x 357 [8,1]
+ CRUSH rule 2 x 358 [2,8]
+ CRUSH rule 2 x 359 [6,1]
+ CRUSH rule 2 x 360 [2,8]
+ CRUSH rule 2 x 361 [8,4]
+ CRUSH rule 2 x 362 [4,1]
+ CRUSH rule 2 x 363 [4,1]
+ CRUSH rule 2 x 364 [2,8]
+ CRUSH rule 2 x 365 [8,1]
+ CRUSH rule 2 x 366 [8,2]
+ CRUSH rule 2 x 367 [4,2]
+ CRUSH rule 2 x 368 [8,4]
+ CRUSH rule 2 x 369 [8,1]
+ CRUSH rule 2 x 370 [8,2]
+ CRUSH rule 2 x 371 [1,4]
+ CRUSH rule 2 x 372 [1,8]
+ CRUSH rule 2 x 373 [1,8]
+ CRUSH rule 2 x 374 [8,1]
+ CRUSH rule 2 x 375 [8,4]
+ CRUSH rule 2 x 376 [8,1]
+ CRUSH rule 2 x 377 [1,4]
+ CRUSH rule 2 x 378 [1,8]
+ CRUSH rule 2 x 379 [8,2]
+ CRUSH rule 2 x 380 [2,8]
+ CRUSH rule 2 x 381 [1,4]
+ CRUSH rule 2 x 382 [1,4]
+ CRUSH rule 2 x 383 [4,8]
+ CRUSH rule 2 x 384 [8,2]
+ CRUSH rule 2 x 385 [8,1]
+ CRUSH rule 2 x 386 [1,8]
+ CRUSH rule 2 x 387 [1,4]
+ CRUSH rule 2 x 388 [2,6]
+ CRUSH rule 2 x 389 [1,4]
+ CRUSH rule 2 x 390 [4,8]
+ CRUSH rule 2 x 391 [4,8]
+ CRUSH rule 2 x 392 [1,8]
+ CRUSH rule 2 x 393 [2,8]
+ CRUSH rule 2 x 394 [8,2]
+ CRUSH rule 2 x 395 [1,8]
+ CRUSH rule 2 x 396 [4,2]
+ CRUSH rule 2 x 397 [2,4]
+ CRUSH rule 2 x 398 [2,4]
+ CRUSH rule 2 x 399 [8,4]
+ CRUSH rule 2 x 400 [8,1]
+ CRUSH rule 2 x 401 [1,4]
+ CRUSH rule 2 x 402 [8,4]
+ CRUSH rule 2 x 403 [1,4]
+ CRUSH rule 2 x 404 [4,2]
+ CRUSH rule 2 x 405 [8,4]
+ CRUSH rule 2 x 406 [2,8]
+ CRUSH rule 2 x 407 [2,8]
+ CRUSH rule 2 x 408 [4,1]
+ CRUSH rule 2 x 409 [8,4]
+ CRUSH rule 2 x 410 [8,4]
+ CRUSH rule 2 x 411 [2,8]
+ CRUSH rule 2 x 412 [2,6]
+ CRUSH rule 2 x 413 [2,8]
+ CRUSH rule 2 x 414 [4,1]
+ CRUSH rule 2 x 415 [2,8]
+ CRUSH rule 2 x 416 [2,8]
+ CRUSH rule 2 x 417 [8,2]
+ CRUSH rule 2 x 418 [8,1]
+ CRUSH rule 2 x 419 [8,4]
+ CRUSH rule 2 x 420 [1,4]
+ CRUSH rule 2 x 421 [8,4]
+ CRUSH rule 2 x 422 [6,2]
+ CRUSH rule 2 x 423 [2,4]
+ CRUSH rule 2 x 424 [8,1]
+ CRUSH rule 2 x 425 [1,8]
+ CRUSH rule 2 x 426 [8,2]
+ CRUSH rule 2 x 427 [1,8]
+ CRUSH rule 2 x 428 [4,8]
+ CRUSH rule 2 x 429 [4,8]
+ CRUSH rule 2 x 430 [4,8]
+ CRUSH rule 2 x 431 [4,1]
+ CRUSH rule 2 x 432 [8,1]
+ CRUSH rule 2 x 433 [8,1]
+ CRUSH rule 2 x 434 [2,8]
+ CRUSH rule 2 x 435 [2,8]
+ CRUSH rule 2 x 436 [4,1]
+ CRUSH rule 2 x 437 [8,2]
+ CRUSH rule 2 x 438 [2,4]
+ CRUSH rule 2 x 439 [1,6]
+ CRUSH rule 2 x 440 [2,8]
+ CRUSH rule 2 x 441 [4,6]
+ CRUSH rule 2 x 442 [2,8]
+ CRUSH rule 2 x 443 [8,2]
+ CRUSH rule 2 x 444 [8,1]
+ CRUSH rule 2 x 445 [8,2]
+ CRUSH rule 2 x 446 [8,1]
+ CRUSH rule 2 x 447 [2,4]
+ CRUSH rule 2 x 448 [8,2]
+ CRUSH rule 2 x 449 [8,1]
+ CRUSH rule 2 x 450 [1,8]
+ CRUSH rule 2 x 451 [8,4]
+ CRUSH rule 2 x 452 [8,2]
+ CRUSH rule 2 x 453 [6,2]
+ CRUSH rule 2 x 454 [8,2]
+ CRUSH rule 2 x 455 [2,8]
+ CRUSH rule 2 x 456 [8,2]
+ CRUSH rule 2 x 457 [8,2]
+ CRUSH rule 2 x 458 [2,8]
+ CRUSH rule 2 x 459 [2,8]
+ CRUSH rule 2 x 460 [8,2]
+ CRUSH rule 2 x 461 [8,1]
+ CRUSH rule 2 x 462 [8,1]
+ CRUSH rule 2 x 463 [8,2]
+ CRUSH rule 2 x 464 [8,4]
+ CRUSH rule 2 x 465 [6,2]
+ CRUSH rule 2 x 466 [8,1]
+ CRUSH rule 2 x 467 [8,2]
+ CRUSH rule 2 x 468 [8,1]
+ CRUSH rule 2 x 469 [8,1]
+ CRUSH rule 2 x 470 [4,2]
+ CRUSH rule 2 x 471 [1,8]
+ CRUSH rule 2 x 472 [1,8]
+ CRUSH rule 2 x 473 [1,4]
+ CRUSH rule 2 x 474 [8,1]
+ CRUSH rule 2 x 475 [8,2]
+ CRUSH rule 2 x 476 [4,8]
+ CRUSH rule 2 x 477 [4,8]
+ CRUSH rule 2 x 478 [8,2]
+ CRUSH rule 2 x 479 [2,8]
+ CRUSH rule 2 x 480 [1,8]
+ CRUSH rule 2 x 481 [2,4]
+ CRUSH rule 2 x 482 [1,8]
+ CRUSH rule 2 x 483 [2,8]
+ CRUSH rule 2 x 484 [1,8]
+ CRUSH rule 2 x 485 [8,1]
+ CRUSH rule 2 x 486 [4,1]
+ CRUSH rule 2 x 487 [1,8]
+ CRUSH rule 2 x 488 [8,1]
+ CRUSH rule 2 x 489 [2,8]
+ CRUSH rule 2 x 490 [6,2]
+ CRUSH rule 2 x 491 [1,8]
+ CRUSH rule 2 x 492 [8,1]
+ CRUSH rule 2 x 493 [2,8]
+ CRUSH rule 2 x 494 [1,8]
+ CRUSH rule 2 x 495 [4,1]
+ CRUSH rule 2 x 496 [8,4]
+ CRUSH rule 2 x 497 [4,8]
+ CRUSH rule 2 x 498 [2,4]
+ CRUSH rule 2 x 499 [8,4]
+ CRUSH rule 2 x 500 [4,8]
+ CRUSH rule 2 x 501 [2,8]
+ CRUSH rule 2 x 502 [6,1]
+ CRUSH rule 2 x 503 [2,8]
+ CRUSH rule 2 x 504 [8,1]
+ CRUSH rule 2 x 505 [1,8]
+ CRUSH rule 2 x 506 [4,2]
+ CRUSH rule 2 x 507 [8,1]
+ CRUSH rule 2 x 508 [1,8]
+ CRUSH rule 2 x 509 [8,1]
+ CRUSH rule 2 x 510 [8,2]
+ CRUSH rule 2 x 511 [4,8]
+ CRUSH rule 2 x 512 [8,2]
+ CRUSH rule 2 x 513 [8,2]
+ CRUSH rule 2 x 514 [2,8]
+ CRUSH rule 2 x 515 [8,4]
+ CRUSH rule 2 x 516 [4,1]
+ CRUSH rule 2 x 517 [8,2]
+ CRUSH rule 2 x 518 [4,8]
+ CRUSH rule 2 x 519 [8,4]
+ CRUSH rule 2 x 520 [2,8]
+ CRUSH rule 2 x 521 [8,2]
+ CRUSH rule 2 x 522 [8,1]
+ CRUSH rule 2 x 523 [4,2]
+ CRUSH rule 2 x 524 [2,6]
+ CRUSH rule 2 x 525 [2,8]
+ CRUSH rule 2 x 526 [1,8]
+ CRUSH rule 2 x 527 [1,4]
+ CRUSH rule 2 x 528 [2,8]
+ CRUSH rule 2 x 529 [4,8]
+ CRUSH rule 2 x 530 [8,1]
+ CRUSH rule 2 x 531 [8,1]
+ CRUSH rule 2 x 532 [6,4]
+ CRUSH rule 2 x 533 [4,8]
+ CRUSH rule 2 x 534 [8,1]
+ CRUSH rule 2 x 535 [8,1]
+ CRUSH rule 2 x 536 [8,2]
+ CRUSH rule 2 x 537 [4,8]
+ CRUSH rule 2 x 538 [8,4]
+ CRUSH rule 2 x 539 [8,1]
+ CRUSH rule 2 x 540 [1,8]
+ CRUSH rule 2 x 541 [2,4]
+ CRUSH rule 2 x 542 [2,8]
+ CRUSH rule 2 x 543 [8,2]
+ CRUSH rule 2 x 544 [4,8]
+ CRUSH rule 2 x 545 [8,1]
+ CRUSH rule 2 x 546 [8,1]
+ CRUSH rule 2 x 547 [8,2]
+ CRUSH rule 2 x 548 [4,2]
+ CRUSH rule 2 x 549 [8,2]
+ CRUSH rule 2 x 550 [2,4]
+ CRUSH rule 2 x 551 [8,1]
+ CRUSH rule 2 x 552 [4,8]
+ CRUSH rule 2 x 553 [2,8]
+ CRUSH rule 2 x 554 [1,8]
+ CRUSH rule 2 x 555 [4,1]
+ CRUSH rule 2 x 556 [8,1]
+ CRUSH rule 2 x 557 [8,2]
+ CRUSH rule 2 x 558 [4,1]
+ CRUSH rule 2 x 559 [1,8]
+ CRUSH rule 2 x 560 [8,1]
+ CRUSH rule 2 x 561 [8,4]
+ CRUSH rule 2 x 562 [4,1]
+ CRUSH rule 2 x 563 [2,8]
+ CRUSH rule 2 x 564 [1,8]
+ CRUSH rule 2 x 565 [4,8]
+ CRUSH rule 2 x 566 [4,8]
+ CRUSH rule 2 x 567 [4,8]
+ CRUSH rule 2 x 568 [8,1]
+ CRUSH rule 2 x 569 [4,1]
+ CRUSH rule 2 x 570 [1,8]
+ CRUSH rule 2 x 571 [6,1]
+ CRUSH rule 2 x 572 [4,2]
+ CRUSH rule 2 x 573 [1,8]
+ CRUSH rule 2 x 574 [2,8]
+ CRUSH rule 2 x 575 [8,2]
+ CRUSH rule 2 x 576 [4,8]
+ CRUSH rule 2 x 577 [8,2]
+ CRUSH rule 2 x 578 [8,1]
+ CRUSH rule 2 x 579 [4,1]
+ CRUSH rule 2 x 580 [1,8]
+ CRUSH rule 2 x 581 [8,2]
+ CRUSH rule 2 x 582 [2,8]
+ CRUSH rule 2 x 583 [8,1]
+ CRUSH rule 2 x 584 [8,1]
+ CRUSH rule 2 x 585 [8,1]
+ CRUSH rule 2 x 586 [1,8]
+ CRUSH rule 2 x 587 [2,4]
+ CRUSH rule 2 x 588 [4,8]
+ CRUSH rule 2 x 589 [8,1]
+ CRUSH rule 2 x 590 [8,2]
+ CRUSH rule 2 x 591 [4,2]
+ CRUSH rule 2 x 592 [2,4]
+ CRUSH rule 2 x 593 [1,8]
+ CRUSH rule 2 x 594 [2,8]
+ CRUSH rule 2 x 595 [8,1]
+ CRUSH rule 2 x 596 [8,2]
+ CRUSH rule 2 x 597 [1,8]
+ CRUSH rule 2 x 598 [2,8]
+ CRUSH rule 2 x 599 [4,2]
+ CRUSH rule 2 x 600 [8,1]
+ CRUSH rule 2 x 601 [1,8]
+ CRUSH rule 2 x 602 [8,2]
+ CRUSH rule 2 x 603 [1,8]
+ CRUSH rule 2 x 604 [8,2]
+ CRUSH rule 2 x 605 [2,8]
+ CRUSH rule 2 x 606 [2,6]
+ CRUSH rule 2 x 607 [2,4]
+ CRUSH rule 2 x 608 [4,2]
+ CRUSH rule 2 x 609 [4,2]
+ CRUSH rule 2 x 610 [8,1]
+ CRUSH rule 2 x 611 [1,8]
+ CRUSH rule 2 x 612 [2,8]
+ CRUSH rule 2 x 613 [8,2]
+ CRUSH rule 2 x 614 [8,2]
+ CRUSH rule 2 x 615 [8,2]
+ CRUSH rule 2 x 616 [1,8]
+ CRUSH rule 2 x 617 [8,1]
+ CRUSH rule 2 x 618 [8,4]
+ CRUSH rule 2 x 619 [4,1]
+ CRUSH rule 2 x 620 [1,8]
+ CRUSH rule 2 x 621 [8,1]
+ CRUSH rule 2 x 622 [2,4]
+ CRUSH rule 2 x 623 [2,8]
+ CRUSH rule 2 x 624 [4,2]
+ CRUSH rule 2 x 625 [2,8]
+ CRUSH rule 2 x 626 [8,2]
+ CRUSH rule 2 x 627 [2,8]
+ CRUSH rule 2 x 628 [8,2]
+ CRUSH rule 2 x 629 [2,8]
+ CRUSH rule 2 x 630 [2,8]
+ CRUSH rule 2 x 631 [1,8]
+ CRUSH rule 2 x 632 [8,2]
+ CRUSH rule 2 x 633 [8,2]
+ CRUSH rule 2 x 634 [1,8]
+ CRUSH rule 2 x 635 [4,8]
+ CRUSH rule 2 x 636 [1,4]
+ CRUSH rule 2 x 637 [1,8]
+ CRUSH rule 2 x 638 [8,1]
+ CRUSH rule 2 x 639 [2,8]
+ CRUSH rule 2 x 640 [2,8]
+ CRUSH rule 2 x 641 [8,2]
+ CRUSH rule 2 x 642 [2,8]
+ CRUSH rule 2 x 643 [1,8]
+ CRUSH rule 2 x 644 [8,1]
+ CRUSH rule 2 x 645 [8,1]
+ CRUSH rule 2 x 646 [8,1]
+ CRUSH rule 2 x 647 [8,1]
+ CRUSH rule 2 x 648 [1,8]
+ CRUSH rule 2 x 649 [4,8]
+ CRUSH rule 2 x 650 [8,4]
+ CRUSH rule 2 x 651 [4,6]
+ CRUSH rule 2 x 652 [4,8]
+ CRUSH rule 2 x 653 [8,2]
+ CRUSH rule 2 x 654 [6,2]
+ CRUSH rule 2 x 655 [1,4]
+ CRUSH rule 2 x 656 [8,1]
+ CRUSH rule 2 x 657 [6,1]
+ CRUSH rule 2 x 658 [8,2]
+ CRUSH rule 2 x 659 [4,8]
+ CRUSH rule 2 x 660 [8,2]
+ CRUSH rule 2 x 661 [1,8]
+ CRUSH rule 2 x 662 [8,2]
+ CRUSH rule 2 x 663 [1,4]
+ CRUSH rule 2 x 664 [1,4]
+ CRUSH rule 2 x 665 [4,6]
+ CRUSH rule 2 x 666 [2,8]
+ CRUSH rule 2 x 667 [1,4]
+ CRUSH rule 2 x 668 [4,8]
+ CRUSH rule 2 x 669 [6,4]
+ CRUSH rule 2 x 670 [4,2]
+ CRUSH rule 2 x 671 [2,8]
+ CRUSH rule 2 x 672 [4,2]
+ CRUSH rule 2 x 673 [4,2]
+ CRUSH rule 2 x 674 [1,8]
+ CRUSH rule 2 x 675 [1,8]
+ CRUSH rule 2 x 676 [2,4]
+ CRUSH rule 2 x 677 [4,1]
+ CRUSH rule 2 x 678 [2,4]
+ CRUSH rule 2 x 679 [8,2]
+ CRUSH rule 2 x 680 [2,8]
+ CRUSH rule 2 x 681 [8,1]
+ CRUSH rule 2 x 682 [1,4]
+ CRUSH rule 2 x 683 [1,4]
+ CRUSH rule 2 x 684 [8,1]
+ CRUSH rule 2 x 685 [8,1]
+ CRUSH rule 2 x 686 [1,4]
+ CRUSH rule 2 x 687 [6,1]
+ CRUSH rule 2 x 688 [4,8]
+ CRUSH rule 2 x 689 [8,4]
+ CRUSH rule 2 x 690 [8,1]
+ CRUSH rule 2 x 691 [1,8]
+ CRUSH rule 2 x 692 [8,2]
+ CRUSH rule 2 x 693 [8,4]
+ CRUSH rule 2 x 694 [8,4]
+ CRUSH rule 2 x 695 [2,8]
+ CRUSH rule 2 x 696 [1,8]
+ CRUSH rule 2 x 697 [8,1]
+ CRUSH rule 2 x 698 [8,2]
+ CRUSH rule 2 x 699 [1,8]
+ CRUSH rule 2 x 700 [1,8]
+ CRUSH rule 2 x 701 [1,8]
+ CRUSH rule 2 x 702 [2,8]
+ CRUSH rule 2 x 703 [8,1]
+ CRUSH rule 2 x 704 [1,4]
+ CRUSH rule 2 x 705 [8,1]
+ CRUSH rule 2 x 706 [1,4]
+ CRUSH rule 2 x 707 [8,4]
+ CRUSH rule 2 x 708 [4,8]
+ CRUSH rule 2 x 709 [8,2]
+ CRUSH rule 2 x 710 [8,2]
+ CRUSH rule 2 x 711 [2,4]
+ CRUSH rule 2 x 712 [2,8]
+ CRUSH rule 2 x 713 [8,4]
+ CRUSH rule 2 x 714 [2,8]
+ CRUSH rule 2 x 715 [1,8]
+ CRUSH rule 2 x 716 [4,8]
+ CRUSH rule 2 x 717 [8,2]
+ CRUSH rule 2 x 718 [8,1]
+ CRUSH rule 2 x 719 [2,6]
+ CRUSH rule 2 x 720 [8,1]
+ CRUSH rule 2 x 721 [4,6]
+ CRUSH rule 2 x 722 [8,2]
+ CRUSH rule 2 x 723 [4,1]
+ CRUSH rule 2 x 724 [2,6]
+ CRUSH rule 2 x 725 [1,8]
+ CRUSH rule 2 x 726 [4,8]
+ CRUSH rule 2 x 727 [4,8]
+ CRUSH rule 2 x 728 [2,8]
+ CRUSH rule 2 x 729 [8,2]
+ CRUSH rule 2 x 730 [4,8]
+ CRUSH rule 2 x 731 [4,1]
+ CRUSH rule 2 x 732 [1,8]
+ CRUSH rule 2 x 733 [4,8]
+ CRUSH rule 2 x 734 [8,4]
+ CRUSH rule 2 x 735 [4,8]
+ CRUSH rule 2 x 736 [4,8]
+ CRUSH rule 2 x 737 [1,8]
+ CRUSH rule 2 x 738 [4,2]
+ CRUSH rule 2 x 739 [2,8]
+ CRUSH rule 2 x 740 [1,8]
+ CRUSH rule 2 x 741 [8,1]
+ CRUSH rule 2 x 742 [8,2]
+ CRUSH rule 2 x 743 [8,1]
+ CRUSH rule 2 x 744 [4,8]
+ CRUSH rule 2 x 745 [1,8]
+ CRUSH rule 2 x 746 [1,8]
+ CRUSH rule 2 x 747 [8,1]
+ CRUSH rule 2 x 748 [2,8]
+ CRUSH rule 2 x 749 [4,8]
+ CRUSH rule 2 x 750 [1,8]
+ CRUSH rule 2 x 751 [2,8]
+ CRUSH rule 2 x 752 [8,1]
+ CRUSH rule 2 x 753 [8,4]
+ CRUSH rule 2 x 754 [8,4]
+ CRUSH rule 2 x 755 [1,8]
+ CRUSH rule 2 x 756 [8,1]
+ CRUSH rule 2 x 757 [8,1]
+ CRUSH rule 2 x 758 [8,2]
+ CRUSH rule 2 x 759 [8,4]
+ CRUSH rule 2 x 760 [1,4]
+ CRUSH rule 2 x 761 [2,8]
+ CRUSH rule 2 x 762 [2,8]
+ CRUSH rule 2 x 763 [8,4]
+ CRUSH rule 2 x 764 [1,8]
+ CRUSH rule 2 x 765 [8,2]
+ CRUSH rule 2 x 766 [8,1]
+ CRUSH rule 2 x 767 [1,8]
+ CRUSH rule 2 x 768 [8,4]
+ CRUSH rule 2 x 769 [8,2]
+ CRUSH rule 2 x 770 [8,2]
+ CRUSH rule 2 x 771 [8,1]
+ CRUSH rule 2 x 772 [8,4]
+ CRUSH rule 2 x 773 [4,1]
+ CRUSH rule 2 x 774 [8,1]
+ CRUSH rule 2 x 775 [8,4]
+ CRUSH rule 2 x 776 [6,2]
+ CRUSH rule 2 x 777 [4,1]
+ CRUSH rule 2 x 778 [1,8]
+ CRUSH rule 2 x 779 [2,8]
+ CRUSH rule 2 x 780 [2,4]
+ CRUSH rule 2 x 781 [8,2]
+ CRUSH rule 2 x 782 [4,1]
+ CRUSH rule 2 x 783 [8,1]
+ CRUSH rule 2 x 784 [1,4]
+ CRUSH rule 2 x 785 [8,1]
+ CRUSH rule 2 x 786 [8,1]
+ CRUSH rule 2 x 787 [1,6]
+ CRUSH rule 2 x 788 [8,2]
+ CRUSH rule 2 x 789 [1,8]
+ CRUSH rule 2 x 790 [8,1]
+ CRUSH rule 2 x 791 [4,8]
+ CRUSH rule 2 x 792 [4,8]
+ CRUSH rule 2 x 793 [8,1]
+ CRUSH rule 2 x 794 [2,8]
+ CRUSH rule 2 x 795 [1,8]
+ CRUSH rule 2 x 796 [8,2]
+ CRUSH rule 2 x 797 [2,4]
+ CRUSH rule 2 x 798 [6,1]
+ CRUSH rule 2 x 799 [4,1]
+ CRUSH rule 2 x 800 [2,8]
+ CRUSH rule 2 x 801 [4,8]
+ CRUSH rule 2 x 802 [1,8]
+ CRUSH rule 2 x 803 [2,8]
+ CRUSH rule 2 x 804 [8,2]
+ CRUSH rule 2 x 805 [8,2]
+ CRUSH rule 2 x 806 [1,4]
+ CRUSH rule 2 x 807 [4,8]
+ CRUSH rule 2 x 808 [8,2]
+ CRUSH rule 2 x 809 [1,8]
+ CRUSH rule 2 x 810 [8,2]
+ CRUSH rule 2 x 811 [8,1]
+ CRUSH rule 2 x 812 [8,4]
+ CRUSH rule 2 x 813 [8,4]
+ CRUSH rule 2 x 814 [8,2]
+ CRUSH rule 2 x 815 [4,1]
+ CRUSH rule 2 x 816 [2,8]
+ CRUSH rule 2 x 817 [8,1]
+ CRUSH rule 2 x 818 [1,8]
+ CRUSH rule 2 x 819 [1,8]
+ CRUSH rule 2 x 820 [4,8]
+ CRUSH rule 2 x 821 [4,8]
+ CRUSH rule 2 x 822 [2,4]
+ CRUSH rule 2 x 823 [4,8]
+ CRUSH rule 2 x 824 [8,2]
+ CRUSH rule 2 x 825 [2,8]
+ CRUSH rule 2 x 826 [8,2]
+ CRUSH rule 2 x 827 [2,8]
+ CRUSH rule 2 x 828 [2,8]
+ CRUSH rule 2 x 829 [8,1]
+ CRUSH rule 2 x 830 [2,4]
+ CRUSH rule 2 x 831 [1,8]
+ CRUSH rule 2 x 832 [4,8]
+ CRUSH rule 2 x 833 [2,8]
+ CRUSH rule 2 x 834 [1,8]
+ CRUSH rule 2 x 835 [8,4]
+ CRUSH rule 2 x 836 [4,8]
+ CRUSH rule 2 x 837 [8,4]
+ CRUSH rule 2 x 838 [6,2]
+ CRUSH rule 2 x 839 [2,8]
+ CRUSH rule 2 x 840 [8,1]
+ CRUSH rule 2 x 841 [4,8]
+ CRUSH rule 2 x 842 [2,8]
+ CRUSH rule 2 x 843 [8,4]
+ CRUSH rule 2 x 844 [8,2]
+ CRUSH rule 2 x 845 [4,8]
+ CRUSH rule 2 x 846 [4,2]
+ CRUSH rule 2 x 847 [2,8]
+ CRUSH rule 2 x 848 [2,8]
+ CRUSH rule 2 x 849 [4,8]
+ CRUSH rule 2 x 850 [1,6]
+ CRUSH rule 2 x 851 [6,1]
+ CRUSH rule 2 x 852 [8,4]
+ CRUSH rule 2 x 853 [6,1]
+ CRUSH rule 2 x 854 [8,1]
+ CRUSH rule 2 x 855 [8,1]
+ CRUSH rule 2 x 856 [8,4]
+ CRUSH rule 2 x 857 [8,2]
+ CRUSH rule 2 x 858 [6,1]
+ CRUSH rule 2 x 859 [8,2]
+ CRUSH rule 2 x 860 [2,8]
+ CRUSH rule 2 x 861 [8,2]
+ CRUSH rule 2 x 862 [8,1]
+ CRUSH rule 2 x 863 [8,2]
+ CRUSH rule 2 x 864 [8,2]
+ CRUSH rule 2 x 865 [8,1]
+ CRUSH rule 2 x 866 [8,2]
+ CRUSH rule 2 x 867 [8,2]
+ CRUSH rule 2 x 868 [8,1]
+ CRUSH rule 2 x 869 [8,4]
+ CRUSH rule 2 x 870 [2,8]
+ CRUSH rule 2 x 871 [1,8]
+ CRUSH rule 2 x 872 [1,8]
+ CRUSH rule 2 x 873 [4,8]
+ CRUSH rule 2 x 874 [2,6]
+ CRUSH rule 2 x 875 [2,8]
+ CRUSH rule 2 x 876 [4,8]
+ CRUSH rule 2 x 877 [8,4]
+ CRUSH rule 2 x 878 [2,8]
+ CRUSH rule 2 x 879 [8,1]
+ CRUSH rule 2 x 880 [1,8]
+ CRUSH rule 2 x 881 [4,8]
+ CRUSH rule 2 x 882 [1,8]
+ CRUSH rule 2 x 883 [2,4]
+ CRUSH rule 2 x 884 [8,2]
+ CRUSH rule 2 x 885 [4,1]
+ CRUSH rule 2 x 886 [8,2]
+ CRUSH rule 2 x 887 [8,4]
+ CRUSH rule 2 x 888 [8,2]
+ CRUSH rule 2 x 889 [2,6]
+ CRUSH rule 2 x 890 [8,2]
+ CRUSH rule 2 x 891 [1,8]
+ CRUSH rule 2 x 892 [8,2]
+ CRUSH rule 2 x 893 [2,6]
+ CRUSH rule 2 x 894 [8,4]
+ CRUSH rule 2 x 895 [4,1]
+ CRUSH rule 2 x 896 [1,8]
+ CRUSH rule 2 x 897 [2,8]
+ CRUSH rule 2 x 898 [1,4]
+ CRUSH rule 2 x 899 [1,8]
+ CRUSH rule 2 x 900 [4,1]
+ CRUSH rule 2 x 901 [2,8]
+ CRUSH rule 2 x 902 [8,4]
+ CRUSH rule 2 x 903 [8,2]
+ CRUSH rule 2 x 904 [8,2]
+ CRUSH rule 2 x 905 [8,2]
+ CRUSH rule 2 x 906 [1,8]
+ CRUSH rule 2 x 907 [8,1]
+ CRUSH rule 2 x 908 [8,1]
+ CRUSH rule 2 x 909 [2,8]
+ CRUSH rule 2 x 910 [8,2]
+ CRUSH rule 2 x 911 [8,1]
+ CRUSH rule 2 x 912 [1,8]
+ CRUSH rule 2 x 913 [8,2]
+ CRUSH rule 2 x 914 [6,4]
+ CRUSH rule 2 x 915 [8,2]
+ CRUSH rule 2 x 916 [4,1]
+ CRUSH rule 2 x 917 [1,4]
+ CRUSH rule 2 x 918 [8,2]
+ CRUSH rule 2 x 919 [8,2]
+ CRUSH rule 2 x 920 [8,1]
+ CRUSH rule 2 x 921 [1,8]
+ CRUSH rule 2 x 922 [8,4]
+ CRUSH rule 2 x 923 [4,8]
+ CRUSH rule 2 x 924 [8,1]
+ CRUSH rule 2 x 925 [4,8]
+ CRUSH rule 2 x 926 [2,8]
+ CRUSH rule 2 x 927 [1,8]
+ CRUSH rule 2 x 928 [8,1]
+ CRUSH rule 2 x 929 [4,1]
+ CRUSH rule 2 x 930 [2,8]
+ CRUSH rule 2 x 931 [2,8]
+ CRUSH rule 2 x 932 [4,1]
+ CRUSH rule 2 x 933 [8,4]
+ CRUSH rule 2 x 934 [8,2]
+ CRUSH rule 2 x 935 [8,2]
+ CRUSH rule 2 x 936 [1,8]
+ CRUSH rule 2 x 937 [4,8]
+ CRUSH rule 2 x 938 [8,4]
+ CRUSH rule 2 x 939 [2,8]
+ CRUSH rule 2 x 940 [8,1]
+ CRUSH rule 2 x 941 [2,8]
+ CRUSH rule 2 x 942 [1,8]
+ CRUSH rule 2 x 943 [8,2]
+ CRUSH rule 2 x 944 [8,2]
+ CRUSH rule 2 x 945 [8,2]
+ CRUSH rule 2 x 946 [2,8]
+ CRUSH rule 2 x 947 [8,2]
+ CRUSH rule 2 x 948 [8,1]
+ CRUSH rule 2 x 949 [6,1]
+ CRUSH rule 2 x 950 [8,1]
+ CRUSH rule 2 x 951 [8,1]
+ CRUSH rule 2 x 952 [2,8]
+ CRUSH rule 2 x 953 [1,4]
+ CRUSH rule 2 x 954 [2,8]
+ CRUSH rule 2 x 955 [8,1]
+ CRUSH rule 2 x 956 [1,8]
+ CRUSH rule 2 x 957 [8,1]
+ CRUSH rule 2 x 958 [8,4]
+ CRUSH rule 2 x 959 [4,2]
+ CRUSH rule 2 x 960 [6,1]
+ CRUSH rule 2 x 961 [1,8]
+ CRUSH rule 2 x 962 [8,4]
+ CRUSH rule 2 x 963 [2,4]
+ CRUSH rule 2 x 964 [2,8]
+ CRUSH rule 2 x 965 [8,2]
+ CRUSH rule 2 x 966 [4,8]
+ CRUSH rule 2 x 967 [8,4]
+ CRUSH rule 2 x 968 [8,2]
+ CRUSH rule 2 x 969 [8,2]
+ CRUSH rule 2 x 970 [2,8]
+ CRUSH rule 2 x 971 [1,8]
+ CRUSH rule 2 x 972 [1,8]
+ CRUSH rule 2 x 973 [1,8]
+ CRUSH rule 2 x 974 [4,1]
+ CRUSH rule 2 x 975 [4,8]
+ CRUSH rule 2 x 976 [4,8]
+ CRUSH rule 2 x 977 [8,4]
+ CRUSH rule 2 x 978 [8,2]
+ CRUSH rule 2 x 979 [8,1]
+ CRUSH rule 2 x 980 [8,2]
+ CRUSH rule 2 x 981 [8,2]
+ CRUSH rule 2 x 982 [1,8]
+ CRUSH rule 2 x 983 [4,8]
+ CRUSH rule 2 x 984 [2,8]
+ CRUSH rule 2 x 985 [2,4]
+ CRUSH rule 2 x 986 [8,4]
+ CRUSH rule 2 x 987 [2,8]
+ CRUSH rule 2 x 988 [1,4]
+ CRUSH rule 2 x 989 [1,8]
+ CRUSH rule 2 x 990 [1,8]
+ CRUSH rule 2 x 991 [1,4]
+ CRUSH rule 2 x 992 [8,1]
+ CRUSH rule 2 x 993 [2,8]
+ CRUSH rule 2 x 994 [4,8]
+ CRUSH rule 2 x 995 [8,1]
+ CRUSH rule 2 x 996 [8,4]
+ CRUSH rule 2 x 997 [8,4]
+ CRUSH rule 2 x 998 [8,1]
+ CRUSH rule 2 x 999 [1,8]
+ CRUSH rule 2 x 1000 [8,4]
+ CRUSH rule 2 x 1001 [2,8]
+ CRUSH rule 2 x 1002 [1,8]
+ CRUSH rule 2 x 1003 [2,8]
+ CRUSH rule 2 x 1004 [8,1]
+ CRUSH rule 2 x 1005 [8,1]
+ CRUSH rule 2 x 1006 [1,8]
+ CRUSH rule 2 x 1007 [1,4]
+ CRUSH rule 2 x 1008 [1,8]
+ CRUSH rule 2 x 1009 [6,4]
+ CRUSH rule 2 x 1010 [1,8]
+ CRUSH rule 2 x 1011 [4,2]
+ CRUSH rule 2 x 1012 [1,8]
+ CRUSH rule 2 x 1013 [2,8]
+ CRUSH rule 2 x 1014 [2,8]
+ CRUSH rule 2 x 1015 [8,1]
+ CRUSH rule 2 x 1016 [2,4]
+ CRUSH rule 2 x 1017 [6,2]
+ CRUSH rule 2 x 1018 [4,1]
+ CRUSH rule 2 x 1019 [4,8]
+ CRUSH rule 2 x 1020 [1,8]
+ CRUSH rule 2 x 1021 [2,8]
+ CRUSH rule 2 x 1022 [1,8]
+ CRUSH rule 2 x 1023 [4,2]
+ rule 2 (chooseleaf) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 2 x 0 [2,4,8]
+ CRUSH rule 2 x 1 [2,8,4]
+ CRUSH rule 2 x 2 [1,8]
+ CRUSH rule 2 x 3 [8,1]
+ CRUSH rule 2 x 4 [4,2,6]
+ CRUSH rule 2 x 5 [8,2]
+ CRUSH rule 2 x 6 [2,8,4]
+ CRUSH rule 2 x 7 [4,8,2]
+ CRUSH rule 2 x 8 [4,8,1]
+ CRUSH rule 2 x 9 [2,4,8]
+ CRUSH rule 2 x 10 [2,8]
+ CRUSH rule 2 x 11 [2,8]
+ CRUSH rule 2 x 12 [2,8]
+ CRUSH rule 2 x 13 [4,8,1]
+ CRUSH rule 2 x 14 [8,2]
+ CRUSH rule 2 x 15 [8,2]
+ CRUSH rule 2 x 16 [8,2]
+ CRUSH rule 2 x 17 [4,1,8]
+ CRUSH rule 2 x 18 [1,8]
+ CRUSH rule 2 x 19 [8,4,2]
+ CRUSH rule 2 x 20 [2,8]
+ CRUSH rule 2 x 21 [8,2]
+ CRUSH rule 2 x 22 [8,1]
+ CRUSH rule 2 x 23 [4,8,2]
+ CRUSH rule 2 x 24 [1,8,4]
+ CRUSH rule 2 x 25 [4,8,1]
+ CRUSH rule 2 x 26 [2,8,4]
+ CRUSH rule 2 x 27 [4,1,8]
+ CRUSH rule 2 x 28 [8,2]
+ CRUSH rule 2 x 29 [8,4,2]
+ CRUSH rule 2 x 30 [4,8,2]
+ CRUSH rule 2 x 31 [8,1]
+ CRUSH rule 2 x 32 [6,1]
+ CRUSH rule 2 x 33 [2,8]
+ CRUSH rule 2 x 34 [2,8]
+ CRUSH rule 2 x 35 [1,8,4]
+ CRUSH rule 2 x 36 [8,2]
+ CRUSH rule 2 x 37 [1,8]
+ CRUSH rule 2 x 38 [4,8,2]
+ CRUSH rule 2 x 39 [8,2]
+ CRUSH rule 2 x 40 [8,2,4]
+ CRUSH rule 2 x 41 [2,8,4]
+ CRUSH rule 2 x 42 [8,2]
+ CRUSH rule 2 x 43 [1,8]
+ CRUSH rule 2 x 44 [1,8,4]
+ CRUSH rule 2 x 45 [8,2,4]
+ CRUSH rule 2 x 46 [2,8]
+ CRUSH rule 2 x 47 [4,2,8]
+ CRUSH rule 2 x 48 [8,1]
+ CRUSH rule 2 x 49 [8,2]
+ CRUSH rule 2 x 50 [4,1,8]
+ CRUSH rule 2 x 51 [8,2]
+ CRUSH rule 2 x 52 [8,1,4]
+ CRUSH rule 2 x 53 [4,8,2]
+ CRUSH rule 2 x 54 [8,4,1]
+ CRUSH rule 2 x 55 [8,2]
+ CRUSH rule 2 x 56 [8,4,2]
+ CRUSH rule 2 x 57 [8,1]
+ CRUSH rule 2 x 58 [1,8]
+ CRUSH rule 2 x 59 [2,8]
+ CRUSH rule 2 x 60 [4,2,8]
+ CRUSH rule 2 x 61 [4,8,2]
+ CRUSH rule 2 x 62 [8,1]
+ CRUSH rule 2 x 63 [8,2]
+ CRUSH rule 2 x 64 [4,2,8]
+ CRUSH rule 2 x 65 [8,4,2]
+ CRUSH rule 2 x 66 [4,8,2]
+ CRUSH rule 2 x 67 [4,2,8]
+ CRUSH rule 2 x 68 [1,8]
+ CRUSH rule 2 x 69 [2,8]
+ CRUSH rule 2 x 70 [8,2]
+ CRUSH rule 2 x 71 [2,8,4]
+ CRUSH rule 2 x 72 [8,1,4]
+ CRUSH rule 2 x 73 [2,8]
+ CRUSH rule 2 x 74 [1,8]
+ CRUSH rule 2 x 75 [4,2,8]
+ CRUSH rule 2 x 76 [4,1,6]
+ CRUSH rule 2 x 77 [8,2,4]
+ CRUSH rule 2 x 78 [1,6]
+ CRUSH rule 2 x 79 [4,1,8]
+ CRUSH rule 2 x 80 [2,4,8]
+ CRUSH rule 2 x 81 [2,8]
+ CRUSH rule 2 x 82 [6,1]
+ CRUSH rule 2 x 83 [2,8]
+ CRUSH rule 2 x 84 [8,2]
+ CRUSH rule 2 x 85 [4,8,1]
+ CRUSH rule 2 x 86 [2,8]
+ CRUSH rule 2 x 87 [2,8,4]
+ CRUSH rule 2 x 88 [1,6]
+ CRUSH rule 2 x 89 [1,8]
+ CRUSH rule 2 x 90 [8,4,1]
+ CRUSH rule 2 x 91 [4,8,1]
+ CRUSH rule 2 x 92 [1,8]
+ CRUSH rule 2 x 93 [8,4,2]
+ CRUSH rule 2 x 94 [1,8]
+ CRUSH rule 2 x 95 [8,1]
+ CRUSH rule 2 x 96 [8,2]
+ CRUSH rule 2 x 97 [8,1]
+ CRUSH rule 2 x 98 [2,8]
+ CRUSH rule 2 x 99 [2,8]
+ CRUSH rule 2 x 100 [1,8,4]
+ CRUSH rule 2 x 101 [8,1]
+ CRUSH rule 2 x 102 [2,8]
+ CRUSH rule 2 x 103 [8,2]
+ CRUSH rule 2 x 104 [8,4,1]
+ CRUSH rule 2 x 105 [2,4,8]
+ CRUSH rule 2 x 106 [1,8,4]
+ CRUSH rule 2 x 107 [1,8]
+ CRUSH rule 2 x 108 [8,2]
+ CRUSH rule 2 x 109 [1,4,8]
+ CRUSH rule 2 x 110 [4,2,8]
+ CRUSH rule 2 x 111 [2,4,8]
+ CRUSH rule 2 x 112 [2,8]
+ CRUSH rule 2 x 113 [8,2]
+ CRUSH rule 2 x 114 [8,4,1]
+ CRUSH rule 2 x 115 [8,2,4]
+ CRUSH rule 2 x 116 [1,8]
+ CRUSH rule 2 x 117 [6,1]
+ CRUSH rule 2 x 118 [2,8]
+ CRUSH rule 2 x 119 [8,1]
+ CRUSH rule 2 x 120 [2,4,8]
+ CRUSH rule 2 x 121 [2,8,4]
+ CRUSH rule 2 x 122 [8,1]
+ CRUSH rule 2 x 123 [2,8]
+ CRUSH rule 2 x 124 [2,8]
+ CRUSH rule 2 x 125 [1,8,4]
+ CRUSH rule 2 x 126 [1,8]
+ CRUSH rule 2 x 127 [4,8,2]
+ CRUSH rule 2 x 128 [8,2]
+ CRUSH rule 2 x 129 [2,4,8]
+ CRUSH rule 2 x 130 [4,8,2]
+ CRUSH rule 2 x 131 [1,4,8]
+ CRUSH rule 2 x 132 [1,8]
+ CRUSH rule 2 x 133 [8,1]
+ CRUSH rule 2 x 134 [1,8,4]
+ CRUSH rule 2 x 135 [4,8,2]
+ CRUSH rule 2 x 136 [2,4,8]
+ CRUSH rule 2 x 137 [8,4,2]
+ CRUSH rule 2 x 138 [8,4,2]
+ CRUSH rule 2 x 139 [4,2,8]
+ CRUSH rule 2 x 140 [1,8,4]
+ CRUSH rule 2 x 141 [8,2]
+ CRUSH rule 2 x 142 [4,1,8]
+ CRUSH rule 2 x 143 [4,8,1]
+ CRUSH rule 2 x 144 [8,1]
+ CRUSH rule 2 x 145 [8,1]
+ CRUSH rule 2 x 146 [2,8]
+ CRUSH rule 2 x 147 [2,8,4]
+ CRUSH rule 2 x 148 [4,1,8]
+ CRUSH rule 2 x 149 [4,8,1]
+ CRUSH rule 2 x 150 [1,8]
+ CRUSH rule 2 x 151 [1,8]
+ CRUSH rule 2 x 152 [8,2]
+ CRUSH rule 2 x 153 [8,4,2]
+ CRUSH rule 2 x 154 [4,2,8]
+ CRUSH rule 2 x 155 [4,8,2]
+ CRUSH rule 2 x 156 [4,2,8]
+ CRUSH rule 2 x 157 [1,8]
+ CRUSH rule 2 x 158 [2,8,4]
+ CRUSH rule 2 x 159 [8,2,4]
+ CRUSH rule 2 x 160 [2,8,4]
+ CRUSH rule 2 x 161 [1,4,8]
+ CRUSH rule 2 x 162 [1,8]
+ CRUSH rule 2 x 163 [4,8,1]
+ CRUSH rule 2 x 164 [8,1]
+ CRUSH rule 2 x 165 [8,2,4]
+ CRUSH rule 2 x 166 [2,8]
+ CRUSH rule 2 x 167 [1,8,4]
+ CRUSH rule 2 x 168 [4,2,8]
+ CRUSH rule 2 x 169 [2,8,4]
+ CRUSH rule 2 x 170 [1,8]
+ CRUSH rule 2 x 171 [8,4,2]
+ CRUSH rule 2 x 172 [1,8]
+ CRUSH rule 2 x 173 [8,4,1]
+ CRUSH rule 2 x 174 [1,6]
+ CRUSH rule 2 x 175 [8,1]
+ CRUSH rule 2 x 176 [2,8]
+ CRUSH rule 2 x 177 [1,8]
+ CRUSH rule 2 x 178 [4,2,8]
+ CRUSH rule 2 x 179 [1,8]
+ CRUSH rule 2 x 180 [8,1]
+ CRUSH rule 2 x 181 [8,2,4]
+ CRUSH rule 2 x 182 [8,1]
+ CRUSH rule 2 x 183 [8,4,1]
+ CRUSH rule 2 x 184 [4,8,2]
+ CRUSH rule 2 x 185 [8,1,4]
+ CRUSH rule 2 x 186 [2,4,8]
+ CRUSH rule 2 x 187 [1,8]
+ CRUSH rule 2 x 188 [1,8,4]
+ CRUSH rule 2 x 189 [1,8,4]
+ CRUSH rule 2 x 190 [1,8]
+ CRUSH rule 2 x 191 [8,1,4]
+ CRUSH rule 2 x 192 [4,1,8]
+ CRUSH rule 2 x 193 [4,2,8]
+ CRUSH rule 2 x 194 [1,8]
+ CRUSH rule 2 x 195 [8,4,1]
+ CRUSH rule 2 x 196 [8,2]
+ CRUSH rule 2 x 197 [8,4,2]
+ CRUSH rule 2 x 198 [2,8]
+ CRUSH rule 2 x 199 [1,4,8]
+ CRUSH rule 2 x 200 [1,8]
+ CRUSH rule 2 x 201 [8,1,4]
+ CRUSH rule 2 x 202 [8,1]
+ CRUSH rule 2 x 203 [8,1]
+ CRUSH rule 2 x 204 [2,4,8]
+ CRUSH rule 2 x 205 [1,8]
+ CRUSH rule 2 x 206 [1,8,4]
+ CRUSH rule 2 x 207 [2,8]
+ CRUSH rule 2 x 208 [8,1]
+ CRUSH rule 2 x 209 [1,8]
+ CRUSH rule 2 x 210 [1,4,8]
+ CRUSH rule 2 x 211 [4,2,8]
+ CRUSH rule 2 x 212 [8,1]
+ CRUSH rule 2 x 213 [8,4,2]
+ CRUSH rule 2 x 214 [8,2]
+ CRUSH rule 2 x 215 [8,1]
+ CRUSH rule 2 x 216 [2,8]
+ CRUSH rule 2 x 217 [1,8,4]
+ CRUSH rule 2 x 218 [2,8]
+ CRUSH rule 2 x 219 [8,2]
+ CRUSH rule 2 x 220 [4,8,1]
+ CRUSH rule 2 x 221 [8,1]
+ CRUSH rule 2 x 222 [8,1]
+ CRUSH rule 2 x 223 [1,8]
+ CRUSH rule 2 x 224 [1,4,8]
+ CRUSH rule 2 x 225 [8,2]
+ CRUSH rule 2 x 226 [8,2,4]
+ CRUSH rule 2 x 227 [4,1,8]
+ CRUSH rule 2 x 228 [8,2]
+ CRUSH rule 2 x 229 [4,8,2]
+ CRUSH rule 2 x 230 [4,8,2]
+ CRUSH rule 2 x 231 [4,8,2]
+ CRUSH rule 2 x 232 [2,8,4]
+ CRUSH rule 2 x 233 [2,8]
+ CRUSH rule 2 x 234 [1,8]
+ CRUSH rule 2 x 235 [4,8,1]
+ CRUSH rule 2 x 236 [2,6]
+ CRUSH rule 2 x 237 [4,8,1]
+ CRUSH rule 2 x 238 [2,8]
+ CRUSH rule 2 x 239 [8,1]
+ CRUSH rule 2 x 240 [4,8,2]
+ CRUSH rule 2 x 241 [1,8]
+ CRUSH rule 2 x 242 [8,2]
+ CRUSH rule 2 x 243 [8,2]
+ CRUSH rule 2 x 244 [4,8,2]
+ CRUSH rule 2 x 245 [8,1]
+ CRUSH rule 2 x 246 [1,8]
+ CRUSH rule 2 x 247 [8,2]
+ CRUSH rule 2 x 248 [8,2,4]
+ CRUSH rule 2 x 249 [2,8]
+ CRUSH rule 2 x 250 [2,4,6]
+ CRUSH rule 2 x 251 [2,8]
+ CRUSH rule 2 x 252 [4,8,1]
+ CRUSH rule 2 x 253 [2,8]
+ CRUSH rule 2 x 254 [4,2,8]
+ CRUSH rule 2 x 255 [1,8]
+ CRUSH rule 2 x 256 [4,8,1]
+ CRUSH rule 2 x 257 [2,8,4]
+ CRUSH rule 2 x 258 [4,2,8]
+ CRUSH rule 2 x 259 [6,2]
+ CRUSH rule 2 x 260 [8,2]
+ CRUSH rule 2 x 261 [8,1]
+ CRUSH rule 2 x 262 [8,1]
+ CRUSH rule 2 x 263 [8,1,4]
+ CRUSH rule 2 x 264 [8,2]
+ CRUSH rule 2 x 265 [8,2]
+ CRUSH rule 2 x 266 [8,2,4]
+ CRUSH rule 2 x 267 [2,8]
+ CRUSH rule 2 x 268 [1,8]
+ CRUSH rule 2 x 269 [1,8,4]
+ CRUSH rule 2 x 270 [4,1,8]
+ CRUSH rule 2 x 271 [8,4,1]
+ CRUSH rule 2 x 272 [2,8,4]
+ CRUSH rule 2 x 273 [4,1,8]
+ CRUSH rule 2 x 274 [8,4,1]
+ CRUSH rule 2 x 275 [4,8,1]
+ CRUSH rule 2 x 276 [8,1,4]
+ CRUSH rule 2 x 277 [8,1]
+ CRUSH rule 2 x 278 [8,1,4]
+ CRUSH rule 2 x 279 [8,4,2]
+ CRUSH rule 2 x 280 [2,8,4]
+ CRUSH rule 2 x 281 [8,2]
+ CRUSH rule 2 x 282 [2,8]
+ CRUSH rule 2 x 283 [8,2]
+ CRUSH rule 2 x 284 [8,2]
+ CRUSH rule 2 x 285 [4,8,2]
+ CRUSH rule 2 x 286 [2,8,4]
+ CRUSH rule 2 x 287 [1,8]
+ CRUSH rule 2 x 288 [8,1,4]
+ CRUSH rule 2 x 289 [4,8,2]
+ CRUSH rule 2 x 290 [1,4,8]
+ CRUSH rule 2 x 291 [1,4,8]
+ CRUSH rule 2 x 292 [8,2,4]
+ CRUSH rule 2 x 293 [8,1]
+ CRUSH rule 2 x 294 [8,4,1]
+ CRUSH rule 2 x 295 [4,8,2]
+ CRUSH rule 2 x 296 [4,1,8]
+ CRUSH rule 2 x 297 [8,2,4]
+ CRUSH rule 2 x 298 [1,8]
+ CRUSH rule 2 x 299 [2,8]
+ CRUSH rule 2 x 300 [8,2]
+ CRUSH rule 2 x 301 [1,8]
+ CRUSH rule 2 x 302 [1,8]
+ CRUSH rule 2 x 303 [8,4,1]
+ CRUSH rule 2 x 304 [2,8]
+ CRUSH rule 2 x 305 [8,2]
+ CRUSH rule 2 x 306 [1,8]
+ CRUSH rule 2 x 307 [2,8]
+ CRUSH rule 2 x 308 [2,8,4]
+ CRUSH rule 2 x 309 [8,1]
+ CRUSH rule 2 x 310 [4,1,6]
+ CRUSH rule 2 x 311 [4,8,1]
+ CRUSH rule 2 x 312 [2,8]
+ CRUSH rule 2 x 313 [4,1,8]
+ CRUSH rule 2 x 314 [2,8]
+ CRUSH rule 2 x 315 [2,8]
+ CRUSH rule 2 x 316 [8,1]
+ CRUSH rule 2 x 317 [2,8]
+ CRUSH rule 2 x 318 [8,1]
+ CRUSH rule 2 x 319 [2,8]
+ CRUSH rule 2 x 320 [8,1]
+ CRUSH rule 2 x 321 [1,8]
+ CRUSH rule 2 x 322 [2,8,4]
+ CRUSH rule 2 x 323 [4,8,1]
+ CRUSH rule 2 x 324 [8,1,4]
+ CRUSH rule 2 x 325 [4,8,2]
+ CRUSH rule 2 x 326 [1,6]
+ CRUSH rule 2 x 327 [1,8]
+ CRUSH rule 2 x 328 [8,4,1]
+ CRUSH rule 2 x 329 [4,8,2]
+ CRUSH rule 2 x 330 [4,8,2]
+ CRUSH rule 2 x 331 [2,8]
+ CRUSH rule 2 x 332 [2,8]
+ CRUSH rule 2 x 333 [8,1]
+ CRUSH rule 2 x 334 [8,2]
+ CRUSH rule 2 x 335 [8,1]
+ CRUSH rule 2 x 336 [4,8,2]
+ CRUSH rule 2 x 337 [8,2,4]
+ CRUSH rule 2 x 338 [8,1]
+ CRUSH rule 2 x 339 [8,2]
+ CRUSH rule 2 x 340 [2,8,4]
+ CRUSH rule 2 x 341 [4,1,8]
+ CRUSH rule 2 x 342 [2,8,4]
+ CRUSH rule 2 x 343 [8,1]
+ CRUSH rule 2 x 344 [6,2,4]
+ CRUSH rule 2 x 345 [2,8]
+ CRUSH rule 2 x 346 [8,2,4]
+ CRUSH rule 2 x 347 [4,1,8]
+ CRUSH rule 2 x 348 [8,2,4]
+ CRUSH rule 2 x 349 [1,8]
+ CRUSH rule 2 x 350 [8,1]
+ CRUSH rule 2 x 351 [8,2]
+ CRUSH rule 2 x 352 [1,8,4]
+ CRUSH rule 2 x 353 [8,1]
+ CRUSH rule 2 x 354 [1,8]
+ CRUSH rule 2 x 355 [8,2]
+ CRUSH rule 2 x 356 [4,1,8]
+ CRUSH rule 2 x 357 [8,1,4]
+ CRUSH rule 2 x 358 [2,8,4]
+ CRUSH rule 2 x 359 [6,1,4]
+ CRUSH rule 2 x 360 [2,8]
+ CRUSH rule 2 x 361 [8,4,1]
+ CRUSH rule 2 x 362 [4,1,6]
+ CRUSH rule 2 x 363 [4,1,8]
+ CRUSH rule 2 x 364 [2,8]
+ CRUSH rule 2 x 365 [8,1]
+ CRUSH rule 2 x 366 [8,2]
+ CRUSH rule 2 x 367 [4,2,8]
+ CRUSH rule 2 x 368 [8,4,1]
+ CRUSH rule 2 x 369 [8,1]
+ CRUSH rule 2 x 370 [8,2]
+ CRUSH rule 2 x 371 [1,4,8]
+ CRUSH rule 2 x 372 [1,8]
+ CRUSH rule 2 x 373 [1,8]
+ CRUSH rule 2 x 374 [8,1]
+ CRUSH rule 2 x 375 [8,4,1]
+ CRUSH rule 2 x 376 [8,1,4]
+ CRUSH rule 2 x 377 [1,4,8]
+ CRUSH rule 2 x 378 [1,8]
+ CRUSH rule 2 x 379 [8,2]
+ CRUSH rule 2 x 380 [2,8]
+ CRUSH rule 2 x 381 [1,4,8]
+ CRUSH rule 2 x 382 [1,4,8]
+ CRUSH rule 2 x 383 [4,8,2]
+ CRUSH rule 2 x 384 [8,2,4]
+ CRUSH rule 2 x 385 [8,1]
+ CRUSH rule 2 x 386 [1,8]
+ CRUSH rule 2 x 387 [1,4,8]
+ CRUSH rule 2 x 388 [2,6]
+ CRUSH rule 2 x 389 [1,4,8]
+ CRUSH rule 2 x 390 [4,8,1]
+ CRUSH rule 2 x 391 [4,8,2]
+ CRUSH rule 2 x 392 [1,8,4]
+ CRUSH rule 2 x 393 [2,8]
+ CRUSH rule 2 x 394 [8,2]
+ CRUSH rule 2 x 395 [1,8]
+ CRUSH rule 2 x 396 [4,2,8]
+ CRUSH rule 2 x 397 [2,4,8]
+ CRUSH rule 2 x 398 [2,4,8]
+ CRUSH rule 2 x 399 [8,4,2]
+ CRUSH rule 2 x 400 [8,1,4]
+ CRUSH rule 2 x 401 [1,4,8]
+ CRUSH rule 2 x 402 [8,4,2]
+ CRUSH rule 2 x 403 [1,4,8]
+ CRUSH rule 2 x 404 [4,2,8]
+ CRUSH rule 2 x 405 [8,4,2]
+ CRUSH rule 2 x 406 [2,8]
+ CRUSH rule 2 x 407 [2,8,4]
+ CRUSH rule 2 x 408 [4,1,8]
+ CRUSH rule 2 x 409 [8,4,1]
+ CRUSH rule 2 x 410 [8,4,2]
+ CRUSH rule 2 x 411 [2,8,4]
+ CRUSH rule 2 x 412 [2,6]
+ CRUSH rule 2 x 413 [2,8]
+ CRUSH rule 2 x 414 [4,1,6]
+ CRUSH rule 2 x 415 [2,8]
+ CRUSH rule 2 x 416 [2,8]
+ CRUSH rule 2 x 417 [8,2]
+ CRUSH rule 2 x 418 [8,1,4]
+ CRUSH rule 2 x 419 [8,4,1]
+ CRUSH rule 2 x 420 [1,4,8]
+ CRUSH rule 2 x 421 [8,4,1]
+ CRUSH rule 2 x 422 [6,2]
+ CRUSH rule 2 x 423 [2,4,8]
+ CRUSH rule 2 x 424 [8,1]
+ CRUSH rule 2 x 425 [1,8]
+ CRUSH rule 2 x 426 [8,2]
+ CRUSH rule 2 x 427 [1,8]
+ CRUSH rule 2 x 428 [4,8,1]
+ CRUSH rule 2 x 429 [4,8,2]
+ CRUSH rule 2 x 430 [4,8,2]
+ CRUSH rule 2 x 431 [4,1,8]
+ CRUSH rule 2 x 432 [8,1]
+ CRUSH rule 2 x 433 [8,1]
+ CRUSH rule 2 x 434 [2,8]
+ CRUSH rule 2 x 435 [2,8]
+ CRUSH rule 2 x 436 [4,1,8]
+ CRUSH rule 2 x 437 [8,2]
+ CRUSH rule 2 x 438 [2,4,8]
+ CRUSH rule 2 x 439 [1,6]
+ CRUSH rule 2 x 440 [2,8]
+ CRUSH rule 2 x 441 [4,6,2]
+ CRUSH rule 2 x 442 [2,8]
+ CRUSH rule 2 x 443 [8,2,4]
+ CRUSH rule 2 x 444 [8,1]
+ CRUSH rule 2 x 445 [8,2]
+ CRUSH rule 2 x 446 [8,1]
+ CRUSH rule 2 x 447 [2,4,8]
+ CRUSH rule 2 x 448 [8,2,4]
+ CRUSH rule 2 x 449 [8,1]
+ CRUSH rule 2 x 450 [1,8]
+ CRUSH rule 2 x 451 [8,4,2]
+ CRUSH rule 2 x 452 [8,2]
+ CRUSH rule 2 x 453 [6,2]
+ CRUSH rule 2 x 454 [8,2]
+ CRUSH rule 2 x 455 [2,8,4]
+ CRUSH rule 2 x 456 [8,2]
+ CRUSH rule 2 x 457 [8,2]
+ CRUSH rule 2 x 458 [2,8]
+ CRUSH rule 2 x 459 [2,8,4]
+ CRUSH rule 2 x 460 [8,2]
+ CRUSH rule 2 x 461 [8,1]
+ CRUSH rule 2 x 462 [8,1]
+ CRUSH rule 2 x 463 [8,2]
+ CRUSH rule 2 x 464 [8,4,2]
+ CRUSH rule 2 x 465 [6,2,4]
+ CRUSH rule 2 x 466 [8,1]
+ CRUSH rule 2 x 467 [8,2]
+ CRUSH rule 2 x 468 [8,1,4]
+ CRUSH rule 2 x 469 [8,1]
+ CRUSH rule 2 x 470 [4,2,6]
+ CRUSH rule 2 x 471 [1,8]
+ CRUSH rule 2 x 472 [1,8]
+ CRUSH rule 2 x 473 [1,4,8]
+ CRUSH rule 2 x 474 [8,1]
+ CRUSH rule 2 x 475 [8,2,4]
+ CRUSH rule 2 x 476 [4,8,1]
+ CRUSH rule 2 x 477 [4,8,2]
+ CRUSH rule 2 x 478 [8,2,4]
+ CRUSH rule 2 x 479 [2,8]
+ CRUSH rule 2 x 480 [1,8]
+ CRUSH rule 2 x 481 [2,4,6]
+ CRUSH rule 2 x 482 [1,8]
+ CRUSH rule 2 x 483 [2,8,4]
+ CRUSH rule 2 x 484 [1,8]
+ CRUSH rule 2 x 485 [8,1]
+ CRUSH rule 2 x 486 [4,1,8]
+ CRUSH rule 2 x 487 [1,8]
+ CRUSH rule 2 x 488 [8,1]
+ CRUSH rule 2 x 489 [2,8]
+ CRUSH rule 2 x 490 [6,2]
+ CRUSH rule 2 x 491 [1,8]
+ CRUSH rule 2 x 492 [8,1]
+ CRUSH rule 2 x 493 [2,8]
+ CRUSH rule 2 x 494 [1,8]
+ CRUSH rule 2 x 495 [4,1,8]
+ CRUSH rule 2 x 496 [8,4,1]
+ CRUSH rule 2 x 497 [4,8,1]
+ CRUSH rule 2 x 498 [2,4,8]
+ CRUSH rule 2 x 499 [8,4,2]
+ CRUSH rule 2 x 500 [4,8,2]
+ CRUSH rule 2 x 501 [2,8]
+ CRUSH rule 2 x 502 [6,1]
+ CRUSH rule 2 x 503 [2,8]
+ CRUSH rule 2 x 504 [8,1]
+ CRUSH rule 2 x 505 [1,8]
+ CRUSH rule 2 x 506 [4,2,8]
+ CRUSH rule 2 x 507 [8,1,4]
+ CRUSH rule 2 x 508 [1,8]
+ CRUSH rule 2 x 509 [8,1]
+ CRUSH rule 2 x 510 [8,2]
+ CRUSH rule 2 x 511 [4,8,2]
+ CRUSH rule 2 x 512 [8,2]
+ CRUSH rule 2 x 513 [8,2]
+ CRUSH rule 2 x 514 [2,8]
+ CRUSH rule 2 x 515 [8,4,1]
+ CRUSH rule 2 x 516 [4,1,8]
+ CRUSH rule 2 x 517 [8,2]
+ CRUSH rule 2 x 518 [4,8,1]
+ CRUSH rule 2 x 519 [8,4,1]
+ CRUSH rule 2 x 520 [2,8,4]
+ CRUSH rule 2 x 521 [8,2,4]
+ CRUSH rule 2 x 522 [8,1,4]
+ CRUSH rule 2 x 523 [4,2,8]
+ CRUSH rule 2 x 524 [2,6]
+ CRUSH rule 2 x 525 [2,8]
+ CRUSH rule 2 x 526 [1,8]
+ CRUSH rule 2 x 527 [1,4,6]
+ CRUSH rule 2 x 528 [2,8]
+ CRUSH rule 2 x 529 [4,8,2]
+ CRUSH rule 2 x 530 [8,1]
+ CRUSH rule 2 x 531 [8,1,4]
+ CRUSH rule 2 x 532 [6,4,1]
+ CRUSH rule 2 x 533 [4,8,2]
+ CRUSH rule 2 x 534 [8,1]
+ CRUSH rule 2 x 535 [8,1]
+ CRUSH rule 2 x 536 [8,2]
+ CRUSH rule 2 x 537 [4,8,2]
+ CRUSH rule 2 x 538 [8,4,1]
+ CRUSH rule 2 x 539 [8,1]
+ CRUSH rule 2 x 540 [1,8,4]
+ CRUSH rule 2 x 541 [2,4,8]
+ CRUSH rule 2 x 542 [2,8]
+ CRUSH rule 2 x 543 [8,2]
+ CRUSH rule 2 x 544 [4,8,2]
+ CRUSH rule 2 x 545 [8,1]
+ CRUSH rule 2 x 546 [8,1,4]
+ CRUSH rule 2 x 547 [8,2,4]
+ CRUSH rule 2 x 548 [4,2,8]
+ CRUSH rule 2 x 549 [8,2]
+ CRUSH rule 2 x 550 [2,4,8]
+ CRUSH rule 2 x 551 [8,1]
+ CRUSH rule 2 x 552 [4,8,1]
+ CRUSH rule 2 x 553 [2,8]
+ CRUSH rule 2 x 554 [1,8]
+ CRUSH rule 2 x 555 [4,1,8]
+ CRUSH rule 2 x 556 [8,1]
+ CRUSH rule 2 x 557 [8,2]
+ CRUSH rule 2 x 558 [4,1,8]
+ CRUSH rule 2 x 559 [1,8]
+ CRUSH rule 2 x 560 [8,1]
+ CRUSH rule 2 x 561 [8,4,1]
+ CRUSH rule 2 x 562 [4,1,8]
+ CRUSH rule 2 x 563 [2,8]
+ CRUSH rule 2 x 564 [1,8]
+ CRUSH rule 2 x 565 [4,8,2]
+ CRUSH rule 2 x 566 [4,8,2]
+ CRUSH rule 2 x 567 [4,8,1]
+ CRUSH rule 2 x 568 [8,1]
+ CRUSH rule 2 x 569 [4,1,8]
+ CRUSH rule 2 x 570 [1,8]
+ CRUSH rule 2 x 571 [6,1]
+ CRUSH rule 2 x 572 [4,2,8]
+ CRUSH rule 2 x 573 [1,8]
+ CRUSH rule 2 x 574 [2,8]
+ CRUSH rule 2 x 575 [8,2,4]
+ CRUSH rule 2 x 576 [4,8,2]
+ CRUSH rule 2 x 577 [8,2]
+ CRUSH rule 2 x 578 [8,1]
+ CRUSH rule 2 x 579 [4,1,8]
+ CRUSH rule 2 x 580 [1,8]
+ CRUSH rule 2 x 581 [8,2,4]
+ CRUSH rule 2 x 582 [2,8,4]
+ CRUSH rule 2 x 583 [8,1]
+ CRUSH rule 2 x 584 [8,1,4]
+ CRUSH rule 2 x 585 [8,1,4]
+ CRUSH rule 2 x 586 [1,8,4]
+ CRUSH rule 2 x 587 [2,4,8]
+ CRUSH rule 2 x 588 [4,8,1]
+ CRUSH rule 2 x 589 [8,1]
+ CRUSH rule 2 x 590 [8,2]
+ CRUSH rule 2 x 591 [4,2,8]
+ CRUSH rule 2 x 592 [2,4,8]
+ CRUSH rule 2 x 593 [1,8,4]
+ CRUSH rule 2 x 594 [2,8]
+ CRUSH rule 2 x 595 [8,1]
+ CRUSH rule 2 x 596 [8,2]
+ CRUSH rule 2 x 597 [1,8]
+ CRUSH rule 2 x 598 [2,8]
+ CRUSH rule 2 x 599 [4,2,8]
+ CRUSH rule 2 x 600 [8,1,4]
+ CRUSH rule 2 x 601 [1,8,4]
+ CRUSH rule 2 x 602 [8,2]
+ CRUSH rule 2 x 603 [1,8]
+ CRUSH rule 2 x 604 [8,2]
+ CRUSH rule 2 x 605 [2,8]
+ CRUSH rule 2 x 606 [2,6,4]
+ CRUSH rule 2 x 607 [2,4,8]
+ CRUSH rule 2 x 608 [4,2,8]
+ CRUSH rule 2 x 609 [4,2,8]
+ CRUSH rule 2 x 610 [8,1]
+ CRUSH rule 2 x 611 [1,8]
+ CRUSH rule 2 x 612 [2,8]
+ CRUSH rule 2 x 613 [8,2,4]
+ CRUSH rule 2 x 614 [8,2,4]
+ CRUSH rule 2 x 615 [8,2,4]
+ CRUSH rule 2 x 616 [1,8]
+ CRUSH rule 2 x 617 [8,1,4]
+ CRUSH rule 2 x 618 [8,4,2]
+ CRUSH rule 2 x 619 [4,1,8]
+ CRUSH rule 2 x 620 [1,8]
+ CRUSH rule 2 x 621 [8,1]
+ CRUSH rule 2 x 622 [2,4,8]
+ CRUSH rule 2 x 623 [2,8]
+ CRUSH rule 2 x 624 [4,2,8]
+ CRUSH rule 2 x 625 [2,8]
+ CRUSH rule 2 x 626 [8,2,4]
+ CRUSH rule 2 x 627 [2,8,4]
+ CRUSH rule 2 x 628 [8,2]
+ CRUSH rule 2 x 629 [2,8,4]
+ CRUSH rule 2 x 630 [2,8]
+ CRUSH rule 2 x 631 [1,8,4]
+ CRUSH rule 2 x 632 [8,2]
+ CRUSH rule 2 x 633 [8,2]
+ CRUSH rule 2 x 634 [1,8]
+ CRUSH rule 2 x 635 [4,8,2]
+ CRUSH rule 2 x 636 [1,4,8]
+ CRUSH rule 2 x 637 [1,8]
+ CRUSH rule 2 x 638 [8,1,4]
+ CRUSH rule 2 x 639 [2,8]
+ CRUSH rule 2 x 640 [2,8]
+ CRUSH rule 2 x 641 [8,2]
+ CRUSH rule 2 x 642 [2,8]
+ CRUSH rule 2 x 643 [1,8]
+ CRUSH rule 2 x 644 [8,1]
+ CRUSH rule 2 x 645 [8,1]
+ CRUSH rule 2 x 646 [8,1,4]
+ CRUSH rule 2 x 647 [8,1]
+ CRUSH rule 2 x 648 [1,8]
+ CRUSH rule 2 x 649 [4,8,2]
+ CRUSH rule 2 x 650 [8,4,1]
+ CRUSH rule 2 x 651 [4,6,1]
+ CRUSH rule 2 x 652 [4,8,1]
+ CRUSH rule 2 x 653 [8,2]
+ CRUSH rule 2 x 654 [6,2]
+ CRUSH rule 2 x 655 [1,4,8]
+ CRUSH rule 2 x 656 [8,1]
+ CRUSH rule 2 x 657 [6,1]
+ CRUSH rule 2 x 658 [8,2]
+ CRUSH rule 2 x 659 [4,8,2]
+ CRUSH rule 2 x 660 [8,2]
+ CRUSH rule 2 x 661 [1,8]
+ CRUSH rule 2 x 662 [8,2]
+ CRUSH rule 2 x 663 [1,4,8]
+ CRUSH rule 2 x 664 [1,4,8]
+ CRUSH rule 2 x 665 [4,6,1]
+ CRUSH rule 2 x 666 [2,8]
+ CRUSH rule 2 x 667 [1,4,8]
+ CRUSH rule 2 x 668 [4,8,1]
+ CRUSH rule 2 x 669 [6,4,2]
+ CRUSH rule 2 x 670 [4,2,8]
+ CRUSH rule 2 x 671 [2,8]
+ CRUSH rule 2 x 672 [4,2,8]
+ CRUSH rule 2 x 673 [4,2,8]
+ CRUSH rule 2 x 674 [1,8]
+ CRUSH rule 2 x 675 [1,8,4]
+ CRUSH rule 2 x 676 [2,4,8]
+ CRUSH rule 2 x 677 [4,1,8]
+ CRUSH rule 2 x 678 [2,4,8]
+ CRUSH rule 2 x 679 [8,2]
+ CRUSH rule 2 x 680 [2,8]
+ CRUSH rule 2 x 681 [8,1]
+ CRUSH rule 2 x 682 [1,4,8]
+ CRUSH rule 2 x 683 [1,4,8]
+ CRUSH rule 2 x 684 [8,1,4]
+ CRUSH rule 2 x 685 [8,1,4]
+ CRUSH rule 2 x 686 [1,4,8]
+ CRUSH rule 2 x 687 [6,1]
+ CRUSH rule 2 x 688 [4,8,2]
+ CRUSH rule 2 x 689 [8,4,2]
+ CRUSH rule 2 x 690 [8,1,4]
+ CRUSH rule 2 x 691 [1,8]
+ CRUSH rule 2 x 692 [8,2]
+ CRUSH rule 2 x 693 [8,4,1]
+ CRUSH rule 2 x 694 [8,4,1]
+ CRUSH rule 2 x 695 [2,8,4]
+ CRUSH rule 2 x 696 [1,8]
+ CRUSH rule 2 x 697 [8,1,4]
+ CRUSH rule 2 x 698 [8,2,4]
+ CRUSH rule 2 x 699 [1,8,4]
+ CRUSH rule 2 x 700 [1,8]
+ CRUSH rule 2 x 701 [1,8]
+ CRUSH rule 2 x 702 [2,8]
+ CRUSH rule 2 x 703 [8,1]
+ CRUSH rule 2 x 704 [1,4,8]
+ CRUSH rule 2 x 705 [8,1,4]
+ CRUSH rule 2 x 706 [1,4,8]
+ CRUSH rule 2 x 707 [8,4,1]
+ CRUSH rule 2 x 708 [4,8,1]
+ CRUSH rule 2 x 709 [8,2]
+ CRUSH rule 2 x 710 [8,2]
+ CRUSH rule 2 x 711 [2,4,8]
+ CRUSH rule 2 x 712 [2,8]
+ CRUSH rule 2 x 713 [8,4,1]
+ CRUSH rule 2 x 714 [2,8]
+ CRUSH rule 2 x 715 [1,8]
+ CRUSH rule 2 x 716 [4,8,2]
+ CRUSH rule 2 x 717 [8,2,4]
+ CRUSH rule 2 x 718 [8,1]
+ CRUSH rule 2 x 719 [2,6,4]
+ CRUSH rule 2 x 720 [8,1,4]
+ CRUSH rule 2 x 721 [4,6,2]
+ CRUSH rule 2 x 722 [8,2]
+ CRUSH rule 2 x 723 [4,1,8]
+ CRUSH rule 2 x 724 [2,6]
+ CRUSH rule 2 x 725 [1,8]
+ CRUSH rule 2 x 726 [4,8,1]
+ CRUSH rule 2 x 727 [4,8,1]
+ CRUSH rule 2 x 728 [2,8,4]
+ CRUSH rule 2 x 729 [8,2]
+ CRUSH rule 2 x 730 [4,8,2]
+ CRUSH rule 2 x 731 [4,1,8]
+ CRUSH rule 2 x 732 [1,8]
+ CRUSH rule 2 x 733 [4,8,1]
+ CRUSH rule 2 x 734 [8,4,2]
+ CRUSH rule 2 x 735 [4,8,1]
+ CRUSH rule 2 x 736 [4,8,1]
+ CRUSH rule 2 x 737 [1,8,4]
+ CRUSH rule 2 x 738 [4,2,8]
+ CRUSH rule 2 x 739 [2,8]
+ CRUSH rule 2 x 740 [1,8,4]
+ CRUSH rule 2 x 741 [8,1]
+ CRUSH rule 2 x 742 [8,2]
+ CRUSH rule 2 x 743 [8,1,4]
+ CRUSH rule 2 x 744 [4,8,1]
+ CRUSH rule 2 x 745 [1,8]
+ CRUSH rule 2 x 746 [1,8]
+ CRUSH rule 2 x 747 [8,1]
+ CRUSH rule 2 x 748 [2,8,4]
+ CRUSH rule 2 x 749 [4,8,2]
+ CRUSH rule 2 x 750 [1,8,4]
+ CRUSH rule 2 x 751 [2,8]
+ CRUSH rule 2 x 752 [8,1]
+ CRUSH rule 2 x 753 [8,4,1]
+ CRUSH rule 2 x 754 [8,4,2]
+ CRUSH rule 2 x 755 [1,8,4]
+ CRUSH rule 2 x 756 [8,1]
+ CRUSH rule 2 x 757 [8,1,4]
+ CRUSH rule 2 x 758 [8,2]
+ CRUSH rule 2 x 759 [8,4,2]
+ CRUSH rule 2 x 760 [1,4,8]
+ CRUSH rule 2 x 761 [2,8]
+ CRUSH rule 2 x 762 [2,8]
+ CRUSH rule 2 x 763 [8,4,1]
+ CRUSH rule 2 x 764 [1,8]
+ CRUSH rule 2 x 765 [8,2]
+ CRUSH rule 2 x 766 [8,1]
+ CRUSH rule 2 x 767 [1,8,4]
+ CRUSH rule 2 x 768 [8,4,2]
+ CRUSH rule 2 x 769 [8,2,4]
+ CRUSH rule 2 x 770 [8,2,4]
+ CRUSH rule 2 x 771 [8,1,4]
+ CRUSH rule 2 x 772 [8,4,1]
+ CRUSH rule 2 x 773 [4,1,8]
+ CRUSH rule 2 x 774 [8,1]
+ CRUSH rule 2 x 775 [8,4,2]
+ CRUSH rule 2 x 776 [6,2]
+ CRUSH rule 2 x 777 [4,1,8]
+ CRUSH rule 2 x 778 [1,8,4]
+ CRUSH rule 2 x 779 [2,8]
+ CRUSH rule 2 x 780 [2,4,8]
+ CRUSH rule 2 x 781 [8,2]
+ CRUSH rule 2 x 782 [4,1,8]
+ CRUSH rule 2 x 783 [8,1,4]
+ CRUSH rule 2 x 784 [1,4,8]
+ CRUSH rule 2 x 785 [8,1,4]
+ CRUSH rule 2 x 786 [8,1]
+ CRUSH rule 2 x 787 [1,6,4]
+ CRUSH rule 2 x 788 [8,2,4]
+ CRUSH rule 2 x 789 [1,8]
+ CRUSH rule 2 x 790 [8,1]
+ CRUSH rule 2 x 791 [4,8,2]
+ CRUSH rule 2 x 792 [4,8,2]
+ CRUSH rule 2 x 793 [8,1,4]
+ CRUSH rule 2 x 794 [2,8,4]
+ CRUSH rule 2 x 795 [1,8]
+ CRUSH rule 2 x 796 [8,2]
+ CRUSH rule 2 x 797 [2,4,8]
+ CRUSH rule 2 x 798 [6,1]
+ CRUSH rule 2 x 799 [4,1,8]
+ CRUSH rule 2 x 800 [2,8]
+ CRUSH rule 2 x 801 [4,8,1]
+ CRUSH rule 2 x 802 [1,8,4]
+ CRUSH rule 2 x 803 [2,8]
+ CRUSH rule 2 x 804 [8,2]
+ CRUSH rule 2 x 805 [8,2]
+ CRUSH rule 2 x 806 [1,4,8]
+ CRUSH rule 2 x 807 [4,8,2]
+ CRUSH rule 2 x 808 [8,2]
+ CRUSH rule 2 x 809 [1,8]
+ CRUSH rule 2 x 810 [8,2]
+ CRUSH rule 2 x 811 [8,1]
+ CRUSH rule 2 x 812 [8,4,2]
+ CRUSH rule 2 x 813 [8,4,2]
+ CRUSH rule 2 x 814 [8,2]
+ CRUSH rule 2 x 815 [4,1,8]
+ CRUSH rule 2 x 816 [2,8]
+ CRUSH rule 2 x 817 [8,1]
+ CRUSH rule 2 x 818 [1,8]
+ CRUSH rule 2 x 819 [1,8]
+ CRUSH rule 2 x 820 [4,8,2]
+ CRUSH rule 2 x 821 [4,8,2]
+ CRUSH rule 2 x 822 [2,4,8]
+ CRUSH rule 2 x 823 [4,8,2]
+ CRUSH rule 2 x 824 [8,2]
+ CRUSH rule 2 x 825 [2,8,4]
+ CRUSH rule 2 x 826 [8,2,4]
+ CRUSH rule 2 x 827 [2,8,4]
+ CRUSH rule 2 x 828 [2,8]
+ CRUSH rule 2 x 829 [8,1]
+ CRUSH rule 2 x 830 [2,4,8]
+ CRUSH rule 2 x 831 [1,8]
+ CRUSH rule 2 x 832 [4,8,2]
+ CRUSH rule 2 x 833 [2,8]
+ CRUSH rule 2 x 834 [1,8]
+ CRUSH rule 2 x 835 [8,4,1]
+ CRUSH rule 2 x 836 [4,8,1]
+ CRUSH rule 2 x 837 [8,4,1]
+ CRUSH rule 2 x 838 [6,2,4]
+ CRUSH rule 2 x 839 [2,8]
+ CRUSH rule 2 x 840 [8,1]
+ CRUSH rule 2 x 841 [4,8,2]
+ CRUSH rule 2 x 842 [2,8]
+ CRUSH rule 2 x 843 [8,4,1]
+ CRUSH rule 2 x 844 [8,2]
+ CRUSH rule 2 x 845 [4,8,2]
+ CRUSH rule 2 x 846 [4,2,8]
+ CRUSH rule 2 x 847 [2,8]
+ CRUSH rule 2 x 848 [2,8,4]
+ CRUSH rule 2 x 849 [4,8,2]
+ CRUSH rule 2 x 850 [1,6]
+ CRUSH rule 2 x 851 [6,1]
+ CRUSH rule 2 x 852 [8,4,2]
+ CRUSH rule 2 x 853 [6,1]
+ CRUSH rule 2 x 854 [8,1]
+ CRUSH rule 2 x 855 [8,1]
+ CRUSH rule 2 x 856 [8,4,2]
+ CRUSH rule 2 x 857 [8,2]
+ CRUSH rule 2 x 858 [6,1]
+ CRUSH rule 2 x 859 [8,2,4]
+ CRUSH rule 2 x 860 [2,8]
+ CRUSH rule 2 x 861 [8,2]
+ CRUSH rule 2 x 862 [8,1]
+ CRUSH rule 2 x 863 [8,2]
+ CRUSH rule 2 x 864 [8,2]
+ CRUSH rule 2 x 865 [8,1]
+ CRUSH rule 2 x 866 [8,2]
+ CRUSH rule 2 x 867 [8,2]
+ CRUSH rule 2 x 868 [8,1]
+ CRUSH rule 2 x 869 [8,4,2]
+ CRUSH rule 2 x 870 [2,8]
+ CRUSH rule 2 x 871 [1,8]
+ CRUSH rule 2 x 872 [1,8]
+ CRUSH rule 2 x 873 [4,8,2]
+ CRUSH rule 2 x 874 [2,6]
+ CRUSH rule 2 x 875 [2,8,4]
+ CRUSH rule 2 x 876 [4,8,1]
+ CRUSH rule 2 x 877 [8,4,2]
+ CRUSH rule 2 x 878 [2,8]
+ CRUSH rule 2 x 879 [8,1]
+ CRUSH rule 2 x 880 [1,8]
+ CRUSH rule 2 x 881 [4,8,1]
+ CRUSH rule 2 x 882 [1,8]
+ CRUSH rule 2 x 883 [2,4,8]
+ CRUSH rule 2 x 884 [8,2,4]
+ CRUSH rule 2 x 885 [4,1,8]
+ CRUSH rule 2 x 886 [8,2]
+ CRUSH rule 2 x 887 [8,4,1]
+ CRUSH rule 2 x 888 [8,2]
+ CRUSH rule 2 x 889 [2,6]
+ CRUSH rule 2 x 890 [8,2,4]
+ CRUSH rule 2 x 891 [1,8]
+ CRUSH rule 2 x 892 [8,2,4]
+ CRUSH rule 2 x 893 [2,6]
+ CRUSH rule 2 x 894 [8,4,2]
+ CRUSH rule 2 x 895 [4,1,8]
+ CRUSH rule 2 x 896 [1,8]
+ CRUSH rule 2 x 897 [2,8]
+ CRUSH rule 2 x 898 [1,4,8]
+ CRUSH rule 2 x 899 [1,8]
+ CRUSH rule 2 x 900 [4,1,8]
+ CRUSH rule 2 x 901 [2,8]
+ CRUSH rule 2 x 902 [8,4,1]
+ CRUSH rule 2 x 903 [8,2]
+ CRUSH rule 2 x 904 [8,2]
+ CRUSH rule 2 x 905 [8,2]
+ CRUSH rule 2 x 906 [1,8]
+ CRUSH rule 2 x 907 [8,1]
+ CRUSH rule 2 x 908 [8,1]
+ CRUSH rule 2 x 909 [2,8]
+ CRUSH rule 2 x 910 [8,2]
+ CRUSH rule 2 x 911 [8,1]
+ CRUSH rule 2 x 912 [1,8]
+ CRUSH rule 2 x 913 [8,2,4]
+ CRUSH rule 2 x 914 [6,4,2]
+ CRUSH rule 2 x 915 [8,2]
+ CRUSH rule 2 x 916 [4,1,8]
+ CRUSH rule 2 x 917 [1,4,8]
+ CRUSH rule 2 x 918 [8,2]
+ CRUSH rule 2 x 919 [8,2]
+ CRUSH rule 2 x 920 [8,1]
+ CRUSH rule 2 x 921 [1,8]
+ CRUSH rule 2 x 922 [8,4,2]
+ CRUSH rule 2 x 923 [4,8,2]
+ CRUSH rule 2 x 924 [8,1]
+ CRUSH rule 2 x 925 [4,8,2]
+ CRUSH rule 2 x 926 [2,8]
+ CRUSH rule 2 x 927 [1,8,4]
+ CRUSH rule 2 x 928 [8,1]
+ CRUSH rule 2 x 929 [4,1,8]
+ CRUSH rule 2 x 930 [2,8]
+ CRUSH rule 2 x 931 [2,8]
+ CRUSH rule 2 x 932 [4,1,8]
+ CRUSH rule 2 x 933 [8,4,1]
+ CRUSH rule 2 x 934 [8,2]
+ CRUSH rule 2 x 935 [8,2]
+ CRUSH rule 2 x 936 [1,8]
+ CRUSH rule 2 x 937 [4,8,2]
+ CRUSH rule 2 x 938 [8,4,2]
+ CRUSH rule 2 x 939 [2,8,4]
+ CRUSH rule 2 x 940 [8,1]
+ CRUSH rule 2 x 941 [2,8]
+ CRUSH rule 2 x 942 [1,8]
+ CRUSH rule 2 x 943 [8,2]
+ CRUSH rule 2 x 944 [8,2]
+ CRUSH rule 2 x 945 [8,2,4]
+ CRUSH rule 2 x 946 [2,8,4]
+ CRUSH rule 2 x 947 [8,2]
+ CRUSH rule 2 x 948 [8,1]
+ CRUSH rule 2 x 949 [6,1]
+ CRUSH rule 2 x 950 [8,1]
+ CRUSH rule 2 x 951 [8,1]
+ CRUSH rule 2 x 952 [2,8,4]
+ CRUSH rule 2 x 953 [1,4,8]
+ CRUSH rule 2 x 954 [2,8]
+ CRUSH rule 2 x 955 [8,1,4]
+ CRUSH rule 2 x 956 [1,8,4]
+ CRUSH rule 2 x 957 [8,1,4]
+ CRUSH rule 2 x 958 [8,4,1]
+ CRUSH rule 2 x 959 [4,2,8]
+ CRUSH rule 2 x 960 [6,1]
+ CRUSH rule 2 x 961 [1,8]
+ CRUSH rule 2 x 962 [8,4,2]
+ CRUSH rule 2 x 963 [2,4,6]
+ CRUSH rule 2 x 964 [2,8]
+ CRUSH rule 2 x 965 [8,2]
+ CRUSH rule 2 x 966 [4,8,1]
+ CRUSH rule 2 x 967 [8,4,2]
+ CRUSH rule 2 x 968 [8,2]
+ CRUSH rule 2 x 969 [8,2,4]
+ CRUSH rule 2 x 970 [2,8,4]
+ CRUSH rule 2 x 971 [1,8]
+ CRUSH rule 2 x 972 [1,8]
+ CRUSH rule 2 x 973 [1,8]
+ CRUSH rule 2 x 974 [4,1,8]
+ CRUSH rule 2 x 975 [4,8,1]
+ CRUSH rule 2 x 976 [4,8,2]
+ CRUSH rule 2 x 977 [8,4,2]
+ CRUSH rule 2 x 978 [8,2,4]
+ CRUSH rule 2 x 979 [8,1,4]
+ CRUSH rule 2 x 980 [8,2,4]
+ CRUSH rule 2 x 981 [8,2]
+ CRUSH rule 2 x 982 [1,8]
+ CRUSH rule 2 x 983 [4,8,2]
+ CRUSH rule 2 x 984 [2,8]
+ CRUSH rule 2 x 985 [2,4,8]
+ CRUSH rule 2 x 986 [8,4,1]
+ CRUSH rule 2 x 987 [2,8]
+ CRUSH rule 2 x 988 [1,4,6]
+ CRUSH rule 2 x 989 [1,8]
+ CRUSH rule 2 x 990 [1,8,4]
+ CRUSH rule 2 x 991 [1,4,8]
+ CRUSH rule 2 x 992 [8,1,4]
+ CRUSH rule 2 x 993 [2,8,4]
+ CRUSH rule 2 x 994 [4,8,2]
+ CRUSH rule 2 x 995 [8,1,4]
+ CRUSH rule 2 x 996 [8,4,1]
+ CRUSH rule 2 x 997 [8,4,1]
+ CRUSH rule 2 x 998 [8,1,4]
+ CRUSH rule 2 x 999 [1,8,4]
+ CRUSH rule 2 x 1000 [8,4,2]
+ CRUSH rule 2 x 1001 [2,8]
+ CRUSH rule 2 x 1002 [1,8]
+ CRUSH rule 2 x 1003 [2,8]
+ CRUSH rule 2 x 1004 [8,1,4]
+ CRUSH rule 2 x 1005 [8,1]
+ CRUSH rule 2 x 1006 [1,8,4]
+ CRUSH rule 2 x 1007 [1,4,8]
+ CRUSH rule 2 x 1008 [1,8]
+ CRUSH rule 2 x 1009 [6,4,1]
+ CRUSH rule 2 x 1010 [1,8]
+ CRUSH rule 2 x 1011 [4,2,8]
+ CRUSH rule 2 x 1012 [1,8]
+ CRUSH rule 2 x 1013 [2,8]
+ CRUSH rule 2 x 1014 [2,8,4]
+ CRUSH rule 2 x 1015 [8,1]
+ CRUSH rule 2 x 1016 [2,4,8]
+ CRUSH rule 2 x 1017 [6,2,4]
+ CRUSH rule 2 x 1018 [4,1,8]
+ CRUSH rule 2 x 1019 [4,8,2]
+ CRUSH rule 2 x 1020 [1,8]
+ CRUSH rule 2 x 1021 [2,8]
+ CRUSH rule 2 x 1022 [1,8,4]
+ CRUSH rule 2 x 1023 [4,2,8]
+ rule 2 (chooseleaf) num_rep 3 result size == 2:\t501/1024 (esc)
+ rule 2 (chooseleaf) num_rep 3 result size == 3:\t523/1024 (esc)
+ rule 3 (choose-set), x = 0..1023, numrep = 2..3
+ CRUSH rule 3 x 0 [2,4]
+ CRUSH rule 3 x 1 [2,8]
+ CRUSH rule 3 x 2 [1]
+ CRUSH rule 3 x 3 [8,1]
+ CRUSH rule 3 x 4 [4,1]
+ CRUSH rule 3 x 5 [8,1]
+ CRUSH rule 3 x 6 [2,8]
+ CRUSH rule 3 x 7 [4,8]
+ CRUSH rule 3 x 8 [4,8]
+ CRUSH rule 3 x 9 [2,4]
+ CRUSH rule 3 x 10 [2,8]
+ CRUSH rule 3 x 11 [2,8]
+ CRUSH rule 3 x 12 [2]
+ CRUSH rule 3 x 13 [4,8]
+ CRUSH rule 3 x 14 [8,1]
+ CRUSH rule 3 x 15 [8,1]
+ CRUSH rule 3 x 16 [8]
+ CRUSH rule 3 x 17 [4,1]
+ CRUSH rule 3 x 18 [1]
+ CRUSH rule 3 x 19 [8,4]
+ CRUSH rule 3 x 20 [2]
+ CRUSH rule 3 x 21 [8]
+ CRUSH rule 3 x 22 [8]
+ CRUSH rule 3 x 23 [4,8]
+ CRUSH rule 3 x 24 [1,8]
+ CRUSH rule 3 x 25 [4,8]
+ CRUSH rule 3 x 26 [2,8]
+ CRUSH rule 3 x 27 [4,1]
+ CRUSH rule 3 x 28 [8,2]
+ CRUSH rule 3 x 29 [8,4]
+ CRUSH rule 3 x 30 [4,8]
+ CRUSH rule 3 x 31 [8,2]
+ CRUSH rule 3 x 32 [6]
+ CRUSH rule 3 x 33 [2,8]
+ CRUSH rule 3 x 34 [2]
+ CRUSH rule 3 x 35 [1,8]
+ CRUSH rule 3 x 36 [8]
+ CRUSH rule 3 x 37 [1]
+ CRUSH rule 3 x 38 [4,8]
+ CRUSH rule 3 x 39 [8]
+ CRUSH rule 3 x 40 [8,2]
+ CRUSH rule 3 x 41 [2,8]
+ CRUSH rule 3 x 42 [8]
+ CRUSH rule 3 x 43 [1]
+ CRUSH rule 3 x 44 [1,8]
+ CRUSH rule 3 x 45 [8,2]
+ CRUSH rule 3 x 46 [2]
+ CRUSH rule 3 x 47 [4,1]
+ CRUSH rule 3 x 48 [8]
+ CRUSH rule 3 x 49 [8]
+ CRUSH rule 3 x 50 [4,1]
+ CRUSH rule 3 x 51 [8]
+ CRUSH rule 3 x 52 [8,2]
+ CRUSH rule 3 x 53 [4,8]
+ CRUSH rule 3 x 54 [8,4]
+ CRUSH rule 3 x 55 [8,2]
+ CRUSH rule 3 x 56 [8,4]
+ CRUSH rule 3 x 57 [8]
+ CRUSH rule 3 x 58 [1,8]
+ CRUSH rule 3 x 59 [2]
+ CRUSH rule 3 x 60 [4,1]
+ CRUSH rule 3 x 61 [4,8]
+ CRUSH rule 3 x 62 [8,1]
+ CRUSH rule 3 x 63 [8]
+ CRUSH rule 3 x 64 [4,1]
+ CRUSH rule 3 x 65 [8,4]
+ CRUSH rule 3 x 66 [4,8]
+ CRUSH rule 3 x 67 [4,2]
+ CRUSH rule 3 x 68 [1]
+ CRUSH rule 3 x 69 [2]
+ CRUSH rule 3 x 70 [8,2]
+ CRUSH rule 3 x 71 [2,8]
+ CRUSH rule 3 x 72 [8,2]
+ CRUSH rule 3 x 73 [2,8]
+ CRUSH rule 3 x 74 [1,8]
+ CRUSH rule 3 x 75 [4,2]
+ CRUSH rule 3 x 76 [4,2]
+ CRUSH rule 3 x 77 [8,2]
+ CRUSH rule 3 x 78 [1]
+ CRUSH rule 3 x 79 [4,1]
+ CRUSH rule 3 x 80 [2,4]
+ CRUSH rule 3 x 81 [2]
+ CRUSH rule 3 x 82 [6,2]
+ CRUSH rule 3 x 83 [2,8]
+ CRUSH rule 3 x 84 [8,2]
+ CRUSH rule 3 x 85 [4,8]
+ CRUSH rule 3 x 86 [2,8]
+ CRUSH rule 3 x 87 [2,8]
+ CRUSH rule 3 x 88 [1,8]
+ CRUSH rule 3 x 89 [1]
+ CRUSH rule 3 x 90 [8,4]
+ CRUSH rule 3 x 91 [4,8]
+ CRUSH rule 3 x 92 [1,8]
+ CRUSH rule 3 x 93 [8,4]
+ CRUSH rule 3 x 94 [1]
+ CRUSH rule 3 x 95 [8]
+ CRUSH rule 3 x 96 [8]
+ CRUSH rule 3 x 97 [8]
+ CRUSH rule 3 x 98 [2,8]
+ CRUSH rule 3 x 99 [2,8]
+ CRUSH rule 3 x 100 [1,8]
+ CRUSH rule 3 x 101 [8]
+ CRUSH rule 3 x 102 [2]
+ CRUSH rule 3 x 103 [8]
+ CRUSH rule 3 x 104 [8,4]
+ CRUSH rule 3 x 105 [2,4]
+ CRUSH rule 3 x 106 [1,8]
+ CRUSH rule 3 x 107 [1]
+ CRUSH rule 3 x 108 [8,1]
+ CRUSH rule 3 x 109 [1,4]
+ CRUSH rule 3 x 110 [4,2]
+ CRUSH rule 3 x 111 [2,4]
+ CRUSH rule 3 x 112 [2,8]
+ CRUSH rule 3 x 113 [8,1]
+ CRUSH rule 3 x 114 [8,4]
+ CRUSH rule 3 x 115 [8,2]
+ CRUSH rule 3 x 116 [1,8]
+ CRUSH rule 3 x 117 [6]
+ CRUSH rule 3 x 118 [2]
+ CRUSH rule 3 x 119 [8]
+ CRUSH rule 3 x 120 [2,4]
+ CRUSH rule 3 x 121 [2,8]
+ CRUSH rule 3 x 122 [8]
+ CRUSH rule 3 x 123 [2]
+ CRUSH rule 3 x 124 [2]
+ CRUSH rule 3 x 125 [1,8]
+ CRUSH rule 3 x 126 [1]
+ CRUSH rule 3 x 127 [4,8]
+ CRUSH rule 3 x 128 [8]
+ CRUSH rule 3 x 129 [2,4]
+ CRUSH rule 3 x 130 [4,8]
+ CRUSH rule 3 x 131 [1,4]
+ CRUSH rule 3 x 132 [1]
+ CRUSH rule 3 x 133 [8]
+ CRUSH rule 3 x 134 [1,8]
+ CRUSH rule 3 x 135 [4,8]
+ CRUSH rule 3 x 136 [2,4]
+ CRUSH rule 3 x 137 [8,4]
+ CRUSH rule 3 x 138 [8,4]
+ CRUSH rule 3 x 139 [4,2]
+ CRUSH rule 3 x 140 [1,8]
+ CRUSH rule 3 x 141 [8,2]
+ CRUSH rule 3 x 142 [4,2]
+ CRUSH rule 3 x 143 [4,8]
+ CRUSH rule 3 x 144 [8,2]
+ CRUSH rule 3 x 145 [8]
+ CRUSH rule 3 x 146 [2,8]
+ CRUSH rule 3 x 147 [2,8]
+ CRUSH rule 3 x 148 [4,1]
+ CRUSH rule 3 x 149 [4,8]
+ CRUSH rule 3 x 150 [1,8]
+ CRUSH rule 3 x 151 [8]
+ CRUSH rule 3 x 152 [8]
+ CRUSH rule 3 x 153 [8,4]
+ CRUSH rule 3 x 154 [4,2]
+ CRUSH rule 3 x 155 [4,8]
+ CRUSH rule 3 x 156 [4,2]
+ CRUSH rule 3 x 157 [1]
+ CRUSH rule 3 x 158 [2,8]
+ CRUSH rule 3 x 159 [8,2]
+ CRUSH rule 3 x 160 [2,8]
+ CRUSH rule 3 x 161 [1,4]
+ CRUSH rule 3 x 162 [1,8]
+ CRUSH rule 3 x 163 [4,8]
+ CRUSH rule 3 x 164 [8,2]
+ CRUSH rule 3 x 165 [8,2]
+ CRUSH rule 3 x 166 [2]
+ CRUSH rule 3 x 167 [1,8]
+ CRUSH rule 3 x 168 [4,2]
+ CRUSH rule 3 x 169 [2,8]
+ CRUSH rule 3 x 170 [1]
+ CRUSH rule 3 x 171 [8,4]
+ CRUSH rule 3 x 172 [1,8]
+ CRUSH rule 3 x 173 [8,4]
+ CRUSH rule 3 x 174 [1]
+ CRUSH rule 3 x 175 [8,1]
+ CRUSH rule 3 x 176 [2]
+ CRUSH rule 3 x 177 [1]
+ CRUSH rule 3 x 178 [4,2]
+ CRUSH rule 3 x 179 [1]
+ CRUSH rule 3 x 180 [8]
+ CRUSH rule 3 x 181 [8,2]
+ CRUSH rule 3 x 182 [8]
+ CRUSH rule 3 x 183 [8,4]
+ CRUSH rule 3 x 184 [4,8]
+ CRUSH rule 3 x 185 [8,1]
+ CRUSH rule 3 x 186 [2,4]
+ CRUSH rule 3 x 187 [1,8]
+ CRUSH rule 3 x 188 [1,8]
+ CRUSH rule 3 x 189 [1,8]
+ CRUSH rule 3 x 190 [1]
+ CRUSH rule 3 x 191 [8,1]
+ CRUSH rule 3 x 192 [4,2]
+ CRUSH rule 3 x 193 [4,2]
+ CRUSH rule 3 x 194 [1]
+ CRUSH rule 3 x 195 [8,4]
+ CRUSH rule 3 x 196 [8,1]
+ CRUSH rule 3 x 197 [8,4]
+ CRUSH rule 3 x 198 [2]
+ CRUSH rule 3 x 199 [1,4]
+ CRUSH rule 3 x 200 [1]
+ CRUSH rule 3 x 201 [8,1]
+ CRUSH rule 3 x 202 [8]
+ CRUSH rule 3 x 203 [8]
+ CRUSH rule 3 x 204 [2,4]
+ CRUSH rule 3 x 205 [1,8]
+ CRUSH rule 3 x 206 [1,8]
+ CRUSH rule 3 x 207 [2]
+ CRUSH rule 3 x 208 [8,2]
+ CRUSH rule 3 x 209 [1,8]
+ CRUSH rule 3 x 210 [1,4]
+ CRUSH rule 3 x 211 [4,2]
+ CRUSH rule 3 x 212 [8]
+ CRUSH rule 3 x 213 [8,4]
+ CRUSH rule 3 x 214 [8]
+ CRUSH rule 3 x 215 [8,1]
+ CRUSH rule 3 x 216 [2]
+ CRUSH rule 3 x 217 [1,8]
+ CRUSH rule 3 x 218 [2,8]
+ CRUSH rule 3 x 219 [8]
+ CRUSH rule 3 x 220 [4,8]
+ CRUSH rule 3 x 221 [8]
+ CRUSH rule 3 x 222 [8]
+ CRUSH rule 3 x 223 [1]
+ CRUSH rule 3 x 224 [1,4]
+ CRUSH rule 3 x 225 [8,2]
+ CRUSH rule 3 x 226 [8,2]
+ CRUSH rule 3 x 227 [4,1]
+ CRUSH rule 3 x 228 [8]
+ CRUSH rule 3 x 229 [4,8]
+ CRUSH rule 3 x 230 [4,8]
+ CRUSH rule 3 x 231 [4,8]
+ CRUSH rule 3 x 232 [2,8]
+ CRUSH rule 3 x 233 [8]
+ CRUSH rule 3 x 234 [1]
+ CRUSH rule 3 x 235 [4,8]
+ CRUSH rule 3 x 236 [2]
+ CRUSH rule 3 x 237 [4,8]
+ CRUSH rule 3 x 238 [2]
+ CRUSH rule 3 x 239 [8]
+ CRUSH rule 3 x 240 [4,8]
+ CRUSH rule 3 x 241 [1]
+ CRUSH rule 3 x 242 [2]
+ CRUSH rule 3 x 243 [8]
+ CRUSH rule 3 x 244 [4,8]
+ CRUSH rule 3 x 245 [8,2]
+ CRUSH rule 3 x 246 [1]
+ CRUSH rule 3 x 247 [8,1]
+ CRUSH rule 3 x 248 [8,1]
+ CRUSH rule 3 x 249 [2]
+ CRUSH rule 3 x 250 [2,4]
+ CRUSH rule 3 x 251 [2]
+ CRUSH rule 3 x 252 [4,8]
+ CRUSH rule 3 x 253 [2]
+ CRUSH rule 3 x 254 [4,2]
+ CRUSH rule 3 x 255 [1,8]
+ CRUSH rule 3 x 256 [4,8]
+ CRUSH rule 3 x 257 [2,6]
+ CRUSH rule 3 x 258 [4,2]
+ CRUSH rule 3 x 259 [6]
+ CRUSH rule 3 x 260 [8]
+ CRUSH rule 3 x 261 [8]
+ CRUSH rule 3 x 262 [8]
+ CRUSH rule 3 x 263 [8,1]
+ CRUSH rule 3 x 264 [8]
+ CRUSH rule 3 x 265 [8]
+ CRUSH rule 3 x 266 [8,2]
+ CRUSH rule 3 x 267 [2]
+ CRUSH rule 3 x 268 [1,8]
+ CRUSH rule 3 x 269 [1,8]
+ CRUSH rule 3 x 270 [4,1]
+ CRUSH rule 3 x 271 [8,4]
+ CRUSH rule 3 x 272 [2,8]
+ CRUSH rule 3 x 273 [4,2]
+ CRUSH rule 3 x 274 [8,4]
+ CRUSH rule 3 x 275 [4,8]
+ CRUSH rule 3 x 276 [8,1]
+ CRUSH rule 3 x 277 [8]
+ CRUSH rule 3 x 278 [8,2]
+ CRUSH rule 3 x 279 [8,4]
+ CRUSH rule 3 x 280 [2,8]
+ CRUSH rule 3 x 281 [8,1]
+ CRUSH rule 3 x 282 [2]
+ CRUSH rule 3 x 283 [8,1]
+ CRUSH rule 3 x 284 [8]
+ CRUSH rule 3 x 285 [4,8]
+ CRUSH rule 3 x 286 [2,8]
+ CRUSH rule 3 x 287 [1]
+ CRUSH rule 3 x 288 [8,1]
+ CRUSH rule 3 x 289 [4,8]
+ CRUSH rule 3 x 290 [1,4]
+ CRUSH rule 3 x 291 [1,4]
+ CRUSH rule 3 x 292 [8,1]
+ CRUSH rule 3 x 293 [8,2]
+ CRUSH rule 3 x 294 [8,4]
+ CRUSH rule 3 x 295 [4,8]
+ CRUSH rule 3 x 296 [4,1]
+ CRUSH rule 3 x 297 [8,2]
+ CRUSH rule 3 x 298 [1]
+ CRUSH rule 3 x 299 [2,8]
+ CRUSH rule 3 x 300 [8]
+ CRUSH rule 3 x 301 [1,8]
+ CRUSH rule 3 x 302 [1]
+ CRUSH rule 3 x 303 [8,4]
+ CRUSH rule 3 x 304 [2,8]
+ CRUSH rule 3 x 305 [8]
+ CRUSH rule 3 x 306 [1,8]
+ CRUSH rule 3 x 307 [2,8]
+ CRUSH rule 3 x 308 [2,8]
+ CRUSH rule 3 x 309 [8]
+ CRUSH rule 3 x 310 [4,1]
+ CRUSH rule 3 x 311 [4,8]
+ CRUSH rule 3 x 312 [2,8]
+ CRUSH rule 3 x 313 [4,1]
+ CRUSH rule 3 x 314 [2]
+ CRUSH rule 3 x 315 [2]
+ CRUSH rule 3 x 316 [8]
+ CRUSH rule 3 x 317 [2,8]
+ CRUSH rule 3 x 318 [8,1]
+ CRUSH rule 3 x 319 [2]
+ CRUSH rule 3 x 320 [8]
+ CRUSH rule 3 x 321 [1]
+ CRUSH rule 3 x 322 [2,6]
+ CRUSH rule 3 x 323 [4,8]
+ CRUSH rule 3 x 324 [8,2]
+ CRUSH rule 3 x 325 [4,8]
+ CRUSH rule 3 x 326 [1]
+ CRUSH rule 3 x 327 [1,8]
+ CRUSH rule 3 x 328 [8,4]
+ CRUSH rule 3 x 329 [4,8]
+ CRUSH rule 3 x 330 [4,8]
+ CRUSH rule 3 x 331 [2,8]
+ CRUSH rule 3 x 332 [2]
+ CRUSH rule 3 x 333 [8]
+ CRUSH rule 3 x 334 [8]
+ CRUSH rule 3 x 335 [8,2]
+ CRUSH rule 3 x 336 [4,8]
+ CRUSH rule 3 x 337 [8,2]
+ CRUSH rule 3 x 338 [8]
+ CRUSH rule 3 x 339 [8]
+ CRUSH rule 3 x 340 [2,8]
+ CRUSH rule 3 x 341 [4,1]
+ CRUSH rule 3 x 342 [2,8]
+ CRUSH rule 3 x 343 [8]
+ CRUSH rule 3 x 344 [6,1]
+ CRUSH rule 3 x 345 [8]
+ CRUSH rule 3 x 346 [8,2]
+ CRUSH rule 3 x 347 [4,1]
+ CRUSH rule 3 x 348 [8,2]
+ CRUSH rule 3 x 349 [1,8]
+ CRUSH rule 3 x 350 [8]
+ CRUSH rule 3 x 351 [8]
+ CRUSH rule 3 x 352 [1,8]
+ CRUSH rule 3 x 353 [8]
+ CRUSH rule 3 x 354 [1]
+ CRUSH rule 3 x 355 [8]
+ CRUSH rule 3 x 356 [4,1]
+ CRUSH rule 3 x 357 [8,1]
+ CRUSH rule 3 x 358 [2,8]
+ CRUSH rule 3 x 359 [6,1]
+ CRUSH rule 3 x 360 [2]
+ CRUSH rule 3 x 361 [8,4]
+ CRUSH rule 3 x 362 [4,1]
+ CRUSH rule 3 x 363 [4,2]
+ CRUSH rule 3 x 364 [2]
+ CRUSH rule 3 x 365 [8]
+ CRUSH rule 3 x 366 [8,2]
+ CRUSH rule 3 x 367 [4,2]
+ CRUSH rule 3 x 368 [8,4]
+ CRUSH rule 3 x 369 [8]
+ CRUSH rule 3 x 370 [8,2]
+ CRUSH rule 3 x 371 [1,4]
+ CRUSH rule 3 x 372 [1]
+ CRUSH rule 3 x 373 [1,8]
+ CRUSH rule 3 x 374 [8]
+ CRUSH rule 3 x 375 [8,4]
+ CRUSH rule 3 x 376 [8,1]
+ CRUSH rule 3 x 377 [1,4]
+ CRUSH rule 3 x 378 [1,8]
+ CRUSH rule 3 x 379 [8]
+ CRUSH rule 3 x 380 [2]
+ CRUSH rule 3 x 381 [1,4]
+ CRUSH rule 3 x 382 [1,4]
+ CRUSH rule 3 x 383 [4,8]
+ CRUSH rule 3 x 384 [8,2]
+ CRUSH rule 3 x 385 [8]
+ CRUSH rule 3 x 386 [1]
+ CRUSH rule 3 x 387 [1,4]
+ CRUSH rule 3 x 388 [2]
+ CRUSH rule 3 x 389 [1,4]
+ CRUSH rule 3 x 390 [4,8]
+ CRUSH rule 3 x 391 [4,8]
+ CRUSH rule 3 x 392 [1,8]
+ CRUSH rule 3 x 393 [2]
+ CRUSH rule 3 x 394 [8]
+ CRUSH rule 3 x 395 [1]
+ CRUSH rule 3 x 396 [4,2]
+ CRUSH rule 3 x 397 [2,4]
+ CRUSH rule 3 x 398 [2,4]
+ CRUSH rule 3 x 399 [8,4]
+ CRUSH rule 3 x 400 [8,1]
+ CRUSH rule 3 x 401 [1,4]
+ CRUSH rule 3 x 402 [8,4]
+ CRUSH rule 3 x 403 [1,4]
+ CRUSH rule 3 x 404 [4,2]
+ CRUSH rule 3 x 405 [8,4]
+ CRUSH rule 3 x 406 [2,8]
+ CRUSH rule 3 x 407 [2,8]
+ CRUSH rule 3 x 408 [4,1]
+ CRUSH rule 3 x 409 [8,4]
+ CRUSH rule 3 x 410 [8,4]
+ CRUSH rule 3 x 411 [2,8]
+ CRUSH rule 3 x 412 [2]
+ CRUSH rule 3 x 413 [2]
+ CRUSH rule 3 x 414 [4,1]
+ CRUSH rule 3 x 415 [2,8]
+ CRUSH rule 3 x 416 [2,8]
+ CRUSH rule 3 x 417 [8,2]
+ CRUSH rule 3 x 418 [8,1]
+ CRUSH rule 3 x 419 [8,4]
+ CRUSH rule 3 x 420 [1,4]
+ CRUSH rule 3 x 421 [8,4]
+ CRUSH rule 3 x 422 [6]
+ CRUSH rule 3 x 423 [2,4]
+ CRUSH rule 3 x 424 [8]
+ CRUSH rule 3 x 425 [1]
+ CRUSH rule 3 x 426 [8,2]
+ CRUSH rule 3 x 427 [1,8]
+ CRUSH rule 3 x 428 [4,8]
+ CRUSH rule 3 x 429 [4,8]
+ CRUSH rule 3 x 430 [4,8]
+ CRUSH rule 3 x 431 [4,2]
+ CRUSH rule 3 x 432 [8,1]
+ CRUSH rule 3 x 433 [8]
+ CRUSH rule 3 x 434 [2]
+ CRUSH rule 3 x 435 [2]
+ CRUSH rule 3 x 436 [4,1]
+ CRUSH rule 3 x 437 [8]
+ CRUSH rule 3 x 438 [2,4]
+ CRUSH rule 3 x 439 [1]
+ CRUSH rule 3 x 440 [2,8]
+ CRUSH rule 3 x 441 [4,8]
+ CRUSH rule 3 x 442 [2]
+ CRUSH rule 3 x 443 [8,1]
+ CRUSH rule 3 x 444 [8,1]
+ CRUSH rule 3 x 445 [8]
+ CRUSH rule 3 x 446 [1]
+ CRUSH rule 3 x 447 [2,4]
+ CRUSH rule 3 x 448 [8,2]
+ CRUSH rule 3 x 449 [8]
+ CRUSH rule 3 x 450 [1]
+ CRUSH rule 3 x 451 [8,4]
+ CRUSH rule 3 x 452 [8]
+ CRUSH rule 3 x 453 [6]
+ CRUSH rule 3 x 454 [8]
+ CRUSH rule 3 x 455 [2,8]
+ CRUSH rule 3 x 456 [8,2]
+ CRUSH rule 3 x 457 [8,1]
+ CRUSH rule 3 x 458 [2,8]
+ CRUSH rule 3 x 459 [2,8]
+ CRUSH rule 3 x 460 [8]
+ CRUSH rule 3 x 461 [8]
+ CRUSH rule 3 x 462 [8,1]
+ CRUSH rule 3 x 463 [8,2]
+ CRUSH rule 3 x 464 [8,4]
+ CRUSH rule 3 x 465 [6,1]
+ CRUSH rule 3 x 466 [8]
+ CRUSH rule 3 x 467 [8]
+ CRUSH rule 3 x 468 [8,2]
+ CRUSH rule 3 x 469 [8,2]
+ CRUSH rule 3 x 470 [4,1]
+ CRUSH rule 3 x 471 [1,8]
+ CRUSH rule 3 x 472 [1]
+ CRUSH rule 3 x 473 [1,4]
+ CRUSH rule 3 x 474 [8,1]
+ CRUSH rule 3 x 475 [8,2]
+ CRUSH rule 3 x 476 [4,8]
+ CRUSH rule 3 x 477 [4,8]
+ CRUSH rule 3 x 478 [8,1]
+ CRUSH rule 3 x 479 [2]
+ CRUSH rule 3 x 480 [1,8]
+ CRUSH rule 3 x 481 [2,4]
+ CRUSH rule 3 x 482 [8]
+ CRUSH rule 3 x 483 [2,8]
+ CRUSH rule 3 x 484 [1,8]
+ CRUSH rule 3 x 485 [8]
+ CRUSH rule 3 x 486 [4,1]
+ CRUSH rule 3 x 487 [1]
+ CRUSH rule 3 x 488 [8]
+ CRUSH rule 3 x 489 [2,8]
+ CRUSH rule 3 x 490 [6]
+ CRUSH rule 3 x 491 [1,8]
+ CRUSH rule 3 x 492 [8]
+ CRUSH rule 3 x 493 [2,8]
+ CRUSH rule 3 x 494 [1,8]
+ CRUSH rule 3 x 495 [4,1]
+ CRUSH rule 3 x 496 [8,4]
+ CRUSH rule 3 x 497 [4,8]
+ CRUSH rule 3 x 498 [2,4]
+ CRUSH rule 3 x 499 [8,4]
+ CRUSH rule 3 x 500 [4,8]
+ CRUSH rule 3 x 501 [2,8]
+ CRUSH rule 3 x 502 [6,1]
+ CRUSH rule 3 x 503 [2]
+ CRUSH rule 3 x 504 [8]
+ CRUSH rule 3 x 505 [1,8]
+ CRUSH rule 3 x 506 [4,1]
+ CRUSH rule 3 x 507 [8,2]
+ CRUSH rule 3 x 508 [1]
+ CRUSH rule 3 x 509 [8]
+ CRUSH rule 3 x 510 [8,2]
+ CRUSH rule 3 x 511 [4,8]
+ CRUSH rule 3 x 512 [8,2]
+ CRUSH rule 3 x 513 [8,2]
+ CRUSH rule 3 x 514 [8]
+ CRUSH rule 3 x 515 [8,4]
+ CRUSH rule 3 x 516 [4,1]
+ CRUSH rule 3 x 517 [8,2]
+ CRUSH rule 3 x 518 [4,8]
+ CRUSH rule 3 x 519 [8,4]
+ CRUSH rule 3 x 520 [2,8]
+ CRUSH rule 3 x 521 [8,2]
+ CRUSH rule 3 x 522 [8,1]
+ CRUSH rule 3 x 523 [4,1]
+ CRUSH rule 3 x 524 [2]
+ CRUSH rule 3 x 525 [2]
+ CRUSH rule 3 x 526 [1]
+ CRUSH rule 3 x 527 [1,4]
+ CRUSH rule 3 x 528 [2]
+ CRUSH rule 3 x 529 [4,8]
+ CRUSH rule 3 x 530 [8]
+ CRUSH rule 3 x 531 [8,1]
+ CRUSH rule 3 x 532 [6,4]
+ CRUSH rule 3 x 533 [4,8]
+ CRUSH rule 3 x 534 [8]
+ CRUSH rule 3 x 535 [8,1]
+ CRUSH rule 3 x 536 [8,2]
+ CRUSH rule 3 x 537 [4,8]
+ CRUSH rule 3 x 538 [8,4]
+ CRUSH rule 3 x 539 [8]
+ CRUSH rule 3 x 540 [1,8]
+ CRUSH rule 3 x 541 [2,4]
+ CRUSH rule 3 x 542 [2]
+ CRUSH rule 3 x 543 [8,2]
+ CRUSH rule 3 x 544 [4,8]
+ CRUSH rule 3 x 545 [8]
+ CRUSH rule 3 x 546 [8,2]
+ CRUSH rule 3 x 547 [8,1]
+ CRUSH rule 3 x 548 [4,1]
+ CRUSH rule 3 x 549 [8]
+ CRUSH rule 3 x 550 [2,4]
+ CRUSH rule 3 x 551 [8]
+ CRUSH rule 3 x 552 [4,8]
+ CRUSH rule 3 x 553 [2]
+ CRUSH rule 3 x 554 [1,8]
+ CRUSH rule 3 x 555 [4,2]
+ CRUSH rule 3 x 556 [8]
+ CRUSH rule 3 x 557 [8]
+ CRUSH rule 3 x 558 [4,2]
+ CRUSH rule 3 x 559 [1]
+ CRUSH rule 3 x 560 [8]
+ CRUSH rule 3 x 561 [8,4]
+ CRUSH rule 3 x 562 [4,1]
+ CRUSH rule 3 x 563 [2,8]
+ CRUSH rule 3 x 564 [1]
+ CRUSH rule 3 x 565 [4,8]
+ CRUSH rule 3 x 566 [4,8]
+ CRUSH rule 3 x 567 [4,8]
+ CRUSH rule 3 x 568 [8]
+ CRUSH rule 3 x 569 [4,1]
+ CRUSH rule 3 x 570 [1]
+ CRUSH rule 3 x 571 [6]
+ CRUSH rule 3 x 572 [4,2]
+ CRUSH rule 3 x 573 [1]
+ CRUSH rule 3 x 574 [2]
+ CRUSH rule 3 x 575 [8,1]
+ CRUSH rule 3 x 576 [4,8]
+ CRUSH rule 3 x 577 [8,2]
+ CRUSH rule 3 x 578 [8,1]
+ CRUSH rule 3 x 579 [4,2]
+ CRUSH rule 3 x 580 [1]
+ CRUSH rule 3 x 581 [8,1]
+ CRUSH rule 3 x 582 [2,8]
+ CRUSH rule 3 x 583 [8,2]
+ CRUSH rule 3 x 584 [8,1]
+ CRUSH rule 3 x 585 [8,1]
+ CRUSH rule 3 x 586 [1,8]
+ CRUSH rule 3 x 587 [2,4]
+ CRUSH rule 3 x 588 [4,8]
+ CRUSH rule 3 x 589 [8,1]
+ CRUSH rule 3 x 590 [8,2]
+ CRUSH rule 3 x 591 [4,2]
+ CRUSH rule 3 x 592 [2,4]
+ CRUSH rule 3 x 593 [1,8]
+ CRUSH rule 3 x 594 [2,8]
+ CRUSH rule 3 x 595 [8,1]
+ CRUSH rule 3 x 596 [2]
+ CRUSH rule 3 x 597 [1]
+ CRUSH rule 3 x 598 [2]
+ CRUSH rule 3 x 599 [4,1]
+ CRUSH rule 3 x 600 [8,1]
+ CRUSH rule 3 x 601 [1,8]
+ CRUSH rule 3 x 602 [8]
+ CRUSH rule 3 x 603 [1]
+ CRUSH rule 3 x 604 [8]
+ CRUSH rule 3 x 605 [2]
+ CRUSH rule 3 x 606 [2,6]
+ CRUSH rule 3 x 607 [2,4]
+ CRUSH rule 3 x 608 [4,1]
+ CRUSH rule 3 x 609 [4,2]
+ CRUSH rule 3 x 610 [8]
+ CRUSH rule 3 x 611 [1,8]
+ CRUSH rule 3 x 612 [2,8]
+ CRUSH rule 3 x 613 [8,1]
+ CRUSH rule 3 x 614 [8,2]
+ CRUSH rule 3 x 615 [8,2]
+ CRUSH rule 3 x 616 [1,8]
+ CRUSH rule 3 x 617 [8,2]
+ CRUSH rule 3 x 618 [8,4]
+ CRUSH rule 3 x 619 [4,1]
+ CRUSH rule 3 x 620 [1]
+ CRUSH rule 3 x 621 [8]
+ CRUSH rule 3 x 622 [2,4]
+ CRUSH rule 3 x 623 [2,8]
+ CRUSH rule 3 x 624 [4,2]
+ CRUSH rule 3 x 625 [2]
+ CRUSH rule 3 x 626 [8,2]
+ CRUSH rule 3 x 627 [2,8]
+ CRUSH rule 3 x 628 [8,1]
+ CRUSH rule 3 x 629 [2,8]
+ CRUSH rule 3 x 630 [2,8]
+ CRUSH rule 3 x 631 [1,8]
+ CRUSH rule 3 x 632 [8,2]
+ CRUSH rule 3 x 633 [8]
+ CRUSH rule 3 x 634 [1]
+ CRUSH rule 3 x 635 [4,8]
+ CRUSH rule 3 x 636 [1,4]
+ CRUSH rule 3 x 637 [1]
+ CRUSH rule 3 x 638 [8,2]
+ CRUSH rule 3 x 639 [2]
+ CRUSH rule 3 x 640 [2]
+ CRUSH rule 3 x 641 [8,2]
+ CRUSH rule 3 x 642 [2,8]
+ CRUSH rule 3 x 643 [1]
+ CRUSH rule 3 x 644 [8,1]
+ CRUSH rule 3 x 645 [8]
+ CRUSH rule 3 x 646 [8,1]
+ CRUSH rule 3 x 647 [8,1]
+ CRUSH rule 3 x 648 [1,8]
+ CRUSH rule 3 x 649 [4,8]
+ CRUSH rule 3 x 650 [8,4]
+ CRUSH rule 3 x 651 [4,6]
+ CRUSH rule 3 x 652 [4,8]
+ CRUSH rule 3 x 653 [8]
+ CRUSH rule 3 x 654 [6]
+ CRUSH rule 3 x 655 [1,4]
+ CRUSH rule 3 x 656 [8]
+ CRUSH rule 3 x 657 [6,1]
+ CRUSH rule 3 x 658 [8]
+ CRUSH rule 3 x 659 [4,8]
+ CRUSH rule 3 x 660 [8]
+ CRUSH rule 3 x 661 [1,8]
+ CRUSH rule 3 x 662 [2]
+ CRUSH rule 3 x 663 [1,4]
+ CRUSH rule 3 x 664 [1,4]
+ CRUSH rule 3 x 665 [4,6]
+ CRUSH rule 3 x 666 [2,8]
+ CRUSH rule 3 x 667 [1,4]
+ CRUSH rule 3 x 668 [4,8]
+ CRUSH rule 3 x 669 [6,4]
+ CRUSH rule 3 x 670 [4,1]
+ CRUSH rule 3 x 671 [2,8]
+ CRUSH rule 3 x 672 [4,2]
+ CRUSH rule 3 x 673 [4,2]
+ CRUSH rule 3 x 674 [1]
+ CRUSH rule 3 x 675 [1,8]
+ CRUSH rule 3 x 676 [2,4]
+ CRUSH rule 3 x 677 [4,1]
+ CRUSH rule 3 x 678 [2,4]
+ CRUSH rule 3 x 679 [8,2]
+ CRUSH rule 3 x 680 [2]
+ CRUSH rule 3 x 681 [8]
+ CRUSH rule 3 x 682 [1,4]
+ CRUSH rule 3 x 683 [1,4]
+ CRUSH rule 3 x 684 [8,2]
+ CRUSH rule 3 x 685 [8,2]
+ CRUSH rule 3 x 686 [1,4]
+ CRUSH rule 3 x 687 [6]
+ CRUSH rule 3 x 688 [4,8]
+ CRUSH rule 3 x 689 [8,4]
+ CRUSH rule 3 x 690 [8,1]
+ CRUSH rule 3 x 691 [1]
+ CRUSH rule 3 x 692 [8,1]
+ CRUSH rule 3 x 693 [8,4]
+ CRUSH rule 3 x 694 [8,4]
+ CRUSH rule 3 x 695 [2,8]
+ CRUSH rule 3 x 696 [1]
+ CRUSH rule 3 x 697 [8,1]
+ CRUSH rule 3 x 698 [8,2]
+ CRUSH rule 3 x 699 [1,8]
+ CRUSH rule 3 x 700 [1]
+ CRUSH rule 3 x 701 [1]
+ CRUSH rule 3 x 702 [2]
+ CRUSH rule 3 x 703 [8]
+ CRUSH rule 3 x 704 [1,4]
+ CRUSH rule 3 x 705 [8,1]
+ CRUSH rule 3 x 706 [1,4]
+ CRUSH rule 3 x 707 [8,4]
+ CRUSH rule 3 x 708 [4,8]
+ CRUSH rule 3 x 709 [8]
+ CRUSH rule 3 x 710 [8]
+ CRUSH rule 3 x 711 [2,4]
+ CRUSH rule 3 x 712 [2]
+ CRUSH rule 3 x 713 [8,4]
+ CRUSH rule 3 x 714 [2]
+ CRUSH rule 3 x 715 [1]
+ CRUSH rule 3 x 716 [4,8]
+ CRUSH rule 3 x 717 [8,2]
+ CRUSH rule 3 x 718 [8]
+ CRUSH rule 3 x 719 [2,6]
+ CRUSH rule 3 x 720 [8,1]
+ CRUSH rule 3 x 721 [4,6]
+ CRUSH rule 3 x 722 [8]
+ CRUSH rule 3 x 723 [4,2]
+ CRUSH rule 3 x 724 [2,6]
+ CRUSH rule 3 x 725 [1]
+ CRUSH rule 3 x 726 [4,8]
+ CRUSH rule 3 x 727 [4,8]
+ CRUSH rule 3 x 728 [2,8]
+ CRUSH rule 3 x 729 [8]
+ CRUSH rule 3 x 730 [4,8]
+ CRUSH rule 3 x 731 [4,1]
+ CRUSH rule 3 x 732 [1]
+ CRUSH rule 3 x 733 [4,8]
+ CRUSH rule 3 x 734 [8,4]
+ CRUSH rule 3 x 735 [4,8]
+ CRUSH rule 3 x 736 [4,8]
+ CRUSH rule 3 x 737 [1,8]
+ CRUSH rule 3 x 738 [4,1]
+ CRUSH rule 3 x 739 [2,8]
+ CRUSH rule 3 x 740 [1,8]
+ CRUSH rule 3 x 741 [8,2]
+ CRUSH rule 3 x 742 [8,2]
+ CRUSH rule 3 x 743 [8,1]
+ CRUSH rule 3 x 744 [4,8]
+ CRUSH rule 3 x 745 [1]
+ CRUSH rule 3 x 746 [1]
+ CRUSH rule 3 x 747 [8,2]
+ CRUSH rule 3 x 748 [2,8]
+ CRUSH rule 3 x 749 [4,8]
+ CRUSH rule 3 x 750 [1,8]
+ CRUSH rule 3 x 751 [2,8]
+ CRUSH rule 3 x 752 [8,1]
+ CRUSH rule 3 x 753 [8,4]
+ CRUSH rule 3 x 754 [8,4]
+ CRUSH rule 3 x 755 [1,8]
+ CRUSH rule 3 x 756 [8]
+ CRUSH rule 3 x 757 [8,1]
+ CRUSH rule 3 x 758 [8,1]
+ CRUSH rule 3 x 759 [8,4]
+ CRUSH rule 3 x 760 [1,4]
+ CRUSH rule 3 x 761 [2]
+ CRUSH rule 3 x 762 [2,8]
+ CRUSH rule 3 x 763 [8,4]
+ CRUSH rule 3 x 764 [1,8]
+ CRUSH rule 3 x 765 [8]
+ CRUSH rule 3 x 766 [8]
+ CRUSH rule 3 x 767 [1,8]
+ CRUSH rule 3 x 768 [8,4]
+ CRUSH rule 3 x 769 [8,2]
+ CRUSH rule 3 x 770 [8,1]
+ CRUSH rule 3 x 771 [8,2]
+ CRUSH rule 3 x 772 [8,4]
+ CRUSH rule 3 x 773 [4,2]
+ CRUSH rule 3 x 774 [8]
+ CRUSH rule 3 x 775 [8,4]
+ CRUSH rule 3 x 776 [6,2]
+ CRUSH rule 3 x 777 [4,1]
+ CRUSH rule 3 x 778 [1,6]
+ CRUSH rule 3 x 779 [2,8]
+ CRUSH rule 3 x 780 [2,4]
+ CRUSH rule 3 x 781 [8]
+ CRUSH rule 3 x 782 [4,2]
+ CRUSH rule 3 x 783 [8,1]
+ CRUSH rule 3 x 784 [1,4]
+ CRUSH rule 3 x 785 [8,1]
+ CRUSH rule 3 x 786 [8]
+ CRUSH rule 3 x 787 [1,8]
+ CRUSH rule 3 x 788 [8,2]
+ CRUSH rule 3 x 789 [1]
+ CRUSH rule 3 x 790 [8]
+ CRUSH rule 3 x 791 [4,8]
+ CRUSH rule 3 x 792 [4,8]
+ CRUSH rule 3 x 793 [8,2]
+ CRUSH rule 3 x 794 [2,8]
+ CRUSH rule 3 x 795 [1]
+ CRUSH rule 3 x 796 [8]
+ CRUSH rule 3 x 797 [2,4]
+ CRUSH rule 3 x 798 [6,1]
+ CRUSH rule 3 x 799 [4,2]
+ CRUSH rule 3 x 800 [2]
+ CRUSH rule 3 x 801 [4,8]
+ CRUSH rule 3 x 802 [1,8]
+ CRUSH rule 3 x 803 [2]
+ CRUSH rule 3 x 804 [8,2]
+ CRUSH rule 3 x 805 [8]
+ CRUSH rule 3 x 806 [1,4]
+ CRUSH rule 3 x 807 [4,8]
+ CRUSH rule 3 x 808 [8]
+ CRUSH rule 3 x 809 [1]
+ CRUSH rule 3 x 810 [8]
+ CRUSH rule 3 x 811 [8]
+ CRUSH rule 3 x 812 [8,4]
+ CRUSH rule 3 x 813 [8,4]
+ CRUSH rule 3 x 814 [8]
+ CRUSH rule 3 x 815 [4,1]
+ CRUSH rule 3 x 816 [2,8]
+ CRUSH rule 3 x 817 [8]
+ CRUSH rule 3 x 818 [1]
+ CRUSH rule 3 x 819 [1]
+ CRUSH rule 3 x 820 [4,8]
+ CRUSH rule 3 x 821 [4,8]
+ CRUSH rule 3 x 822 [2,4]
+ CRUSH rule 3 x 823 [4,8]
+ CRUSH rule 3 x 824 [8]
+ CRUSH rule 3 x 825 [2,8]
+ CRUSH rule 3 x 826 [8,1]
+ CRUSH rule 3 x 827 [2,6]
+ CRUSH rule 3 x 828 [2]
+ CRUSH rule 3 x 829 [8]
+ CRUSH rule 3 x 830 [2,4]
+ CRUSH rule 3 x 831 [1,8]
+ CRUSH rule 3 x 832 [4,8]
+ CRUSH rule 3 x 833 [2,8]
+ CRUSH rule 3 x 834 [1]
+ CRUSH rule 3 x 835 [8,4]
+ CRUSH rule 3 x 836 [4,8]
+ CRUSH rule 3 x 837 [8,4]
+ CRUSH rule 3 x 838 [6,2]
+ CRUSH rule 3 x 839 [2]
+ CRUSH rule 3 x 840 [8]
+ CRUSH rule 3 x 841 [4,8]
+ CRUSH rule 3 x 842 [2]
+ CRUSH rule 3 x 843 [8,4]
+ CRUSH rule 3 x 844 [8]
+ CRUSH rule 3 x 845 [4,8]
+ CRUSH rule 3 x 846 [4,2]
+ CRUSH rule 3 x 847 [2,8]
+ CRUSH rule 3 x 848 [2,8]
+ CRUSH rule 3 x 849 [4,8]
+ CRUSH rule 3 x 850 [1]
+ CRUSH rule 3 x 851 [6]
+ CRUSH rule 3 x 852 [8,4]
+ CRUSH rule 3 x 853 [6,2]
+ CRUSH rule 3 x 854 [8,2]
+ CRUSH rule 3 x 855 [8]
+ CRUSH rule 3 x 856 [8,4]
+ CRUSH rule 3 x 857 [8]
+ CRUSH rule 3 x 858 [6]
+ CRUSH rule 3 x 859 [8,2]
+ CRUSH rule 3 x 860 [2]
+ CRUSH rule 3 x 861 [8]
+ CRUSH rule 3 x 862 [8,1]
+ CRUSH rule 3 x 863 [8,1]
+ CRUSH rule 3 x 864 [8]
+ CRUSH rule 3 x 865 [8,1]
+ CRUSH rule 3 x 866 [8]
+ CRUSH rule 3 x 867 [8]
+ CRUSH rule 3 x 868 [8]
+ CRUSH rule 3 x 869 [8,4]
+ CRUSH rule 3 x 870 [2]
+ CRUSH rule 3 x 871 [1]
+ CRUSH rule 3 x 872 [1]
+ CRUSH rule 3 x 873 [4,8]
+ CRUSH rule 3 x 874 [2,6]
+ CRUSH rule 3 x 875 [2,8]
+ CRUSH rule 3 x 876 [4,8]
+ CRUSH rule 3 x 877 [8,4]
+ CRUSH rule 3 x 878 [2]
+ CRUSH rule 3 x 879 [8]
+ CRUSH rule 3 x 880 [1]
+ CRUSH rule 3 x 881 [4,8]
+ CRUSH rule 3 x 882 [1]
+ CRUSH rule 3 x 883 [2,4]
+ CRUSH rule 3 x 884 [8,1]
+ CRUSH rule 3 x 885 [4,1]
+ CRUSH rule 3 x 886 [8]
+ CRUSH rule 3 x 887 [8,4]
+ CRUSH rule 3 x 888 [8,1]
+ CRUSH rule 3 x 889 [2,8]
+ CRUSH rule 3 x 890 [8,1]
+ CRUSH rule 3 x 891 [1,8]
+ CRUSH rule 3 x 892 [8,2]
+ CRUSH rule 3 x 893 [2]
+ CRUSH rule 3 x 894 [8,4]
+ CRUSH rule 3 x 895 [4,1]
+ CRUSH rule 3 x 896 [1,8]
+ CRUSH rule 3 x 897 [2]
+ CRUSH rule 3 x 898 [1,4]
+ CRUSH rule 3 x 899 [1,8]
+ CRUSH rule 3 x 900 [4,1]
+ CRUSH rule 3 x 901 [2]
+ CRUSH rule 3 x 902 [8,4]
+ CRUSH rule 3 x 903 [8]
+ CRUSH rule 3 x 904 [8]
+ CRUSH rule 3 x 905 [8,2]
+ CRUSH rule 3 x 906 [1,8]
+ CRUSH rule 3 x 907 [8,2]
+ CRUSH rule 3 x 908 [8]
+ CRUSH rule 3 x 909 [2]
+ CRUSH rule 3 x 910 [8]
+ CRUSH rule 3 x 911 [8]
+ CRUSH rule 3 x 912 [1,8]
+ CRUSH rule 3 x 913 [8,1]
+ CRUSH rule 3 x 914 [6,4]
+ CRUSH rule 3 x 915 [8,2]
+ CRUSH rule 3 x 916 [4,1]
+ CRUSH rule 3 x 917 [1,4]
+ CRUSH rule 3 x 918 [8,2]
+ CRUSH rule 3 x 919 [8,2]
+ CRUSH rule 3 x 920 [8]
+ CRUSH rule 3 x 921 [1]
+ CRUSH rule 3 x 922 [8,4]
+ CRUSH rule 3 x 923 [4,8]
+ CRUSH rule 3 x 924 [1]
+ CRUSH rule 3 x 925 [4,8]
+ CRUSH rule 3 x 926 [2]
+ CRUSH rule 3 x 927 [1,8]
+ CRUSH rule 3 x 928 [8,1]
+ CRUSH rule 3 x 929 [4,2]
+ CRUSH rule 3 x 930 [2]
+ CRUSH rule 3 x 931 [2]
+ CRUSH rule 3 x 932 [4,2]
+ CRUSH rule 3 x 933 [8,4]
+ CRUSH rule 3 x 934 [8]
+ CRUSH rule 3 x 935 [8]
+ CRUSH rule 3 x 936 [1,8]
+ CRUSH rule 3 x 937 [4,8]
+ CRUSH rule 3 x 938 [8,4]
+ CRUSH rule 3 x 939 [2,8]
+ CRUSH rule 3 x 940 [8]
+ CRUSH rule 3 x 941 [2]
+ CRUSH rule 3 x 942 [1,8]
+ CRUSH rule 3 x 943 [8,2]
+ CRUSH rule 3 x 944 [8]
+ CRUSH rule 3 x 945 [8,2]
+ CRUSH rule 3 x 946 [2,8]
+ CRUSH rule 3 x 947 [2]
+ CRUSH rule 3 x 948 [8]
+ CRUSH rule 3 x 949 [6,2]
+ CRUSH rule 3 x 950 [8]
+ CRUSH rule 3 x 951 [8]
+ CRUSH rule 3 x 952 [2,8]
+ CRUSH rule 3 x 953 [1,4]
+ CRUSH rule 3 x 954 [2]
+ CRUSH rule 3 x 955 [8,1]
+ CRUSH rule 3 x 956 [1,8]
+ CRUSH rule 3 x 957 [8,1]
+ CRUSH rule 3 x 958 [8,4]
+ CRUSH rule 3 x 959 [4,1]
+ CRUSH rule 3 x 960 [6]
+ CRUSH rule 3 x 961 [1]
+ CRUSH rule 3 x 962 [8,4]
+ CRUSH rule 3 x 963 [2,4]
+ CRUSH rule 3 x 964 [2]
+ CRUSH rule 3 x 965 [8]
+ CRUSH rule 3 x 966 [4,8]
+ CRUSH rule 3 x 967 [8,4]
+ CRUSH rule 3 x 968 [8,2]
+ CRUSH rule 3 x 969 [8,2]
+ CRUSH rule 3 x 970 [2,8]
+ CRUSH rule 3 x 971 [1,8]
+ CRUSH rule 3 x 972 [1,8]
+ CRUSH rule 3 x 973 [1,8]
+ CRUSH rule 3 x 974 [4,1]
+ CRUSH rule 3 x 975 [4,8]
+ CRUSH rule 3 x 976 [4,8]
+ CRUSH rule 3 x 977 [8,4]
+ CRUSH rule 3 x 978 [8,2]
+ CRUSH rule 3 x 979 [8,2]
+ CRUSH rule 3 x 980 [8,2]
+ CRUSH rule 3 x 981 [8]
+ CRUSH rule 3 x 982 [1]
+ CRUSH rule 3 x 983 [4,8]
+ CRUSH rule 3 x 984 [2,8]
+ CRUSH rule 3 x 985 [2,4]
+ CRUSH rule 3 x 986 [8,4]
+ CRUSH rule 3 x 987 [2]
+ CRUSH rule 3 x 988 [1,4]
+ CRUSH rule 3 x 989 [1,8]
+ CRUSH rule 3 x 990 [1,8]
+ CRUSH rule 3 x 991 [1,4]
+ CRUSH rule 3 x 992 [8,1]
+ CRUSH rule 3 x 993 [2,8]
+ CRUSH rule 3 x 994 [4,8]
+ CRUSH rule 3 x 995 [8,1]
+ CRUSH rule 3 x 996 [8,4]
+ CRUSH rule 3 x 997 [8,4]
+ CRUSH rule 3 x 998 [8,1]
+ CRUSH rule 3 x 999 [1,8]
+ CRUSH rule 3 x 1000 [8,4]
+ CRUSH rule 3 x 1001 [2]
+ CRUSH rule 3 x 1002 [1]
+ CRUSH rule 3 x 1003 [2,8]
+ CRUSH rule 3 x 1004 [8,1]
+ CRUSH rule 3 x 1005 [8,1]
+ CRUSH rule 3 x 1006 [1,8]
+ CRUSH rule 3 x 1007 [1,4]
+ CRUSH rule 3 x 1008 [1,8]
+ CRUSH rule 3 x 1009 [6,4]
+ CRUSH rule 3 x 1010 [1]
+ CRUSH rule 3 x 1011 [4,2]
+ CRUSH rule 3 x 1012 [1]
+ CRUSH rule 3 x 1013 [2]
+ CRUSH rule 3 x 1014 [2,8]
+ CRUSH rule 3 x 1015 [8]
+ CRUSH rule 3 x 1016 [2,4]
+ CRUSH rule 3 x 1017 [6,1]
+ CRUSH rule 3 x 1018 [4,1]
+ CRUSH rule 3 x 1019 [4,8]
+ CRUSH rule 3 x 1020 [1]
+ CRUSH rule 3 x 1021 [2]
+ CRUSH rule 3 x 1022 [1,8]
+ CRUSH rule 3 x 1023 [4,2]
+ rule 3 (choose-set) num_rep 2 result size == 1:\t325/1024 (esc)
+ rule 3 (choose-set) num_rep 2 result size == 2:\t699/1024 (esc)
+ CRUSH rule 3 x 0 [2,4,8]
+ CRUSH rule 3 x 1 [2,8,4]
+ CRUSH rule 3 x 2 [1,8]
+ CRUSH rule 3 x 3 [8,1]
+ CRUSH rule 3 x 4 [4,1,8]
+ CRUSH rule 3 x 5 [8,1]
+ CRUSH rule 3 x 6 [2,8,4]
+ CRUSH rule 3 x 7 [4,8,1]
+ CRUSH rule 3 x 8 [4,8,2]
+ CRUSH rule 3 x 9 [2,4,8]
+ CRUSH rule 3 x 10 [2,8]
+ CRUSH rule 3 x 11 [2,8]
+ CRUSH rule 3 x 12 [2,8]
+ CRUSH rule 3 x 13 [4,8,1]
+ CRUSH rule 3 x 14 [8,1]
+ CRUSH rule 3 x 15 [8,1]
+ CRUSH rule 3 x 16 [8,1]
+ CRUSH rule 3 x 17 [4,1,8]
+ CRUSH rule 3 x 18 [1,6]
+ CRUSH rule 3 x 19 [8,4,2]
+ CRUSH rule 3 x 20 [2,8]
+ CRUSH rule 3 x 21 [8,2]
+ CRUSH rule 3 x 22 [8,1]
+ CRUSH rule 3 x 23 [4,8,2]
+ CRUSH rule 3 x 24 [1,8,4]
+ CRUSH rule 3 x 25 [4,8,1]
+ CRUSH rule 3 x 26 [2,8,4]
+ CRUSH rule 3 x 27 [4,1,8]
+ CRUSH rule 3 x 28 [8,2]
+ CRUSH rule 3 x 29 [8,4,2]
+ CRUSH rule 3 x 30 [4,8,1]
+ CRUSH rule 3 x 31 [8,2]
+ CRUSH rule 3 x 32 [6,2]
+ CRUSH rule 3 x 33 [2,8]
+ CRUSH rule 3 x 34 [2,8]
+ CRUSH rule 3 x 35 [1,8,4]
+ CRUSH rule 3 x 36 [8,1]
+ CRUSH rule 3 x 37 [1,8]
+ CRUSH rule 3 x 38 [4,8,1]
+ CRUSH rule 3 x 39 [8,2]
+ CRUSH rule 3 x 40 [8,2,4]
+ CRUSH rule 3 x 41 [2,8,4]
+ CRUSH rule 3 x 42 [8,2]
+ CRUSH rule 3 x 43 [1,8]
+ CRUSH rule 3 x 44 [1,8,4]
+ CRUSH rule 3 x 45 [8,2,4]
+ CRUSH rule 3 x 46 [2,8]
+ CRUSH rule 3 x 47 [4,1,8]
+ CRUSH rule 3 x 48 [8,1]
+ CRUSH rule 3 x 49 [8,2]
+ CRUSH rule 3 x 50 [4,1,8]
+ CRUSH rule 3 x 51 [8,1]
+ CRUSH rule 3 x 52 [8,2,4]
+ CRUSH rule 3 x 53 [4,8,2]
+ CRUSH rule 3 x 54 [8,4,2]
+ CRUSH rule 3 x 55 [8,2]
+ CRUSH rule 3 x 56 [8,4,1]
+ CRUSH rule 3 x 57 [8,1]
+ CRUSH rule 3 x 58 [1,8]
+ CRUSH rule 3 x 59 [2,8]
+ CRUSH rule 3 x 60 [4,1,8]
+ CRUSH rule 3 x 61 [4,8,2]
+ CRUSH rule 3 x 62 [8,1]
+ CRUSH rule 3 x 63 [8,2]
+ CRUSH rule 3 x 64 [4,1,8]
+ CRUSH rule 3 x 65 [8,4,2]
+ CRUSH rule 3 x 66 [4,8,1]
+ CRUSH rule 3 x 67 [4,2,8]
+ CRUSH rule 3 x 68 [1,8]
+ CRUSH rule 3 x 69 [2,8]
+ CRUSH rule 3 x 70 [8,2]
+ CRUSH rule 3 x 71 [2,8,4]
+ CRUSH rule 3 x 72 [8,2,4]
+ CRUSH rule 3 x 73 [2,8]
+ CRUSH rule 3 x 74 [1,8]
+ CRUSH rule 3 x 75 [4,2,8]
+ CRUSH rule 3 x 76 [4,2,6]
+ CRUSH rule 3 x 77 [8,2,4]
+ CRUSH rule 3 x 78 [1,6]
+ CRUSH rule 3 x 79 [4,1,8]
+ CRUSH rule 3 x 80 [2,4,8]
+ CRUSH rule 3 x 81 [2,8]
+ CRUSH rule 3 x 82 [6,2]
+ CRUSH rule 3 x 83 [2,8]
+ CRUSH rule 3 x 84 [8,2]
+ CRUSH rule 3 x 85 [4,8,2]
+ CRUSH rule 3 x 86 [2,8]
+ CRUSH rule 3 x 87 [2,8,4]
+ CRUSH rule 3 x 88 [1,8]
+ CRUSH rule 3 x 89 [1,8]
+ CRUSH rule 3 x 90 [8,4,2]
+ CRUSH rule 3 x 91 [4,8,2]
+ CRUSH rule 3 x 92 [1,8]
+ CRUSH rule 3 x 93 [8,4,1]
+ CRUSH rule 3 x 94 [1,8]
+ CRUSH rule 3 x 95 [8,1]
+ CRUSH rule 3 x 96 [8,2]
+ CRUSH rule 3 x 97 [8,1]
+ CRUSH rule 3 x 98 [2,8]
+ CRUSH rule 3 x 99 [2,8]
+ CRUSH rule 3 x 100 [1,8,4]
+ CRUSH rule 3 x 101 [8,1]
+ CRUSH rule 3 x 102 [2,8]
+ CRUSH rule 3 x 103 [8,1]
+ CRUSH rule 3 x 104 [8,4,1]
+ CRUSH rule 3 x 105 [2,4,8]
+ CRUSH rule 3 x 106 [1,8,4]
+ CRUSH rule 3 x 107 [1,8]
+ CRUSH rule 3 x 108 [8,1]
+ CRUSH rule 3 x 109 [1,4,8]
+ CRUSH rule 3 x 110 [4,2,8]
+ CRUSH rule 3 x 111 [2,4,8]
+ CRUSH rule 3 x 112 [2,8]
+ CRUSH rule 3 x 113 [8,1]
+ CRUSH rule 3 x 114 [8,4,2]
+ CRUSH rule 3 x 115 [8,2,4]
+ CRUSH rule 3 x 116 [1,8]
+ CRUSH rule 3 x 117 [6,1]
+ CRUSH rule 3 x 118 [2,8]
+ CRUSH rule 3 x 119 [8,1]
+ CRUSH rule 3 x 120 [2,4,8]
+ CRUSH rule 3 x 121 [2,8,4]
+ CRUSH rule 3 x 122 [8,2]
+ CRUSH rule 3 x 123 [2,8]
+ CRUSH rule 3 x 124 [2,8]
+ CRUSH rule 3 x 125 [1,8,4]
+ CRUSH rule 3 x 126 [1,8]
+ CRUSH rule 3 x 127 [4,8,1]
+ CRUSH rule 3 x 128 [8,1]
+ CRUSH rule 3 x 129 [2,4,8]
+ CRUSH rule 3 x 130 [4,8,1]
+ CRUSH rule 3 x 131 [1,4,8]
+ CRUSH rule 3 x 132 [1,8]
+ CRUSH rule 3 x 133 [8,1]
+ CRUSH rule 3 x 134 [1,8,4]
+ CRUSH rule 3 x 135 [4,8,2]
+ CRUSH rule 3 x 136 [2,4,8]
+ CRUSH rule 3 x 137 [8,4,1]
+ CRUSH rule 3 x 138 [8,4,2]
+ CRUSH rule 3 x 139 [4,2,8]
+ CRUSH rule 3 x 140 [1,8,4]
+ CRUSH rule 3 x 141 [8,2]
+ CRUSH rule 3 x 142 [4,2,8]
+ CRUSH rule 3 x 143 [4,8,1]
+ CRUSH rule 3 x 144 [8,2]
+ CRUSH rule 3 x 145 [8,2]
+ CRUSH rule 3 x 146 [2,8]
+ CRUSH rule 3 x 147 [2,8,4]
+ CRUSH rule 3 x 148 [4,1,8]
+ CRUSH rule 3 x 149 [4,8,2]
+ CRUSH rule 3 x 150 [1,8]
+ CRUSH rule 3 x 151 [8,1]
+ CRUSH rule 3 x 152 [8,2]
+ CRUSH rule 3 x 153 [8,4,1]
+ CRUSH rule 3 x 154 [4,2,8]
+ CRUSH rule 3 x 155 [4,8,1]
+ CRUSH rule 3 x 156 [4,2,8]
+ CRUSH rule 3 x 157 [1,8]
+ CRUSH rule 3 x 158 [2,8,4]
+ CRUSH rule 3 x 159 [8,2,4]
+ CRUSH rule 3 x 160 [2,8,4]
+ CRUSH rule 3 x 161 [1,4,8]
+ CRUSH rule 3 x 162 [1,8]
+ CRUSH rule 3 x 163 [4,8,2]
+ CRUSH rule 3 x 164 [8,2]
+ CRUSH rule 3 x 165 [8,2,4]
+ CRUSH rule 3 x 166 [2,8]
+ CRUSH rule 3 x 167 [1,8,4]
+ CRUSH rule 3 x 168 [4,2,8]
+ CRUSH rule 3 x 169 [2,8,4]
+ CRUSH rule 3 x 170 [1,8]
+ CRUSH rule 3 x 171 [8,4,2]
+ CRUSH rule 3 x 172 [1,8]
+ CRUSH rule 3 x 173 [8,4,1]
+ CRUSH rule 3 x 174 [1,8]
+ CRUSH rule 3 x 175 [8,1]
+ CRUSH rule 3 x 176 [2,8]
+ CRUSH rule 3 x 177 [1,8]
+ CRUSH rule 3 x 178 [4,2,8]
+ CRUSH rule 3 x 179 [1,8]
+ CRUSH rule 3 x 180 [8,2]
+ CRUSH rule 3 x 181 [8,2,4]
+ CRUSH rule 3 x 182 [8,1]
+ CRUSH rule 3 x 183 [8,4,1]
+ CRUSH rule 3 x 184 [4,8,1]
+ CRUSH rule 3 x 185 [8,1,4]
+ CRUSH rule 3 x 186 [2,4,8]
+ CRUSH rule 3 x 187 [1,8]
+ CRUSH rule 3 x 188 [1,8,4]
+ CRUSH rule 3 x 189 [1,8,4]
+ CRUSH rule 3 x 190 [1,8]
+ CRUSH rule 3 x 191 [8,1,4]
+ CRUSH rule 3 x 192 [4,2,8]
+ CRUSH rule 3 x 193 [4,2,8]
+ CRUSH rule 3 x 194 [1,8]
+ CRUSH rule 3 x 195 [8,4,2]
+ CRUSH rule 3 x 196 [8,1]
+ CRUSH rule 3 x 197 [8,4,1]
+ CRUSH rule 3 x 198 [2,8]
+ CRUSH rule 3 x 199 [1,4,8]
+ CRUSH rule 3 x 200 [1,8]
+ CRUSH rule 3 x 201 [8,1,4]
+ CRUSH rule 3 x 202 [8,1]
+ CRUSH rule 3 x 203 [8,1]
+ CRUSH rule 3 x 204 [2,4,8]
+ CRUSH rule 3 x 205 [1,8]
+ CRUSH rule 3 x 206 [1,8,4]
+ CRUSH rule 3 x 207 [2,8]
+ CRUSH rule 3 x 208 [8,2]
+ CRUSH rule 3 x 209 [1,8]
+ CRUSH rule 3 x 210 [1,4,8]
+ CRUSH rule 3 x 211 [4,2,8]
+ CRUSH rule 3 x 212 [8,1]
+ CRUSH rule 3 x 213 [8,4,1]
+ CRUSH rule 3 x 214 [8,1]
+ CRUSH rule 3 x 215 [8,1]
+ CRUSH rule 3 x 216 [2,8]
+ CRUSH rule 3 x 217 [1,8,4]
+ CRUSH rule 3 x 218 [2,8]
+ CRUSH rule 3 x 219 [8,2]
+ CRUSH rule 3 x 220 [4,8,1]
+ CRUSH rule 3 x 221 [8,1]
+ CRUSH rule 3 x 222 [8,1]
+ CRUSH rule 3 x 223 [1,8]
+ CRUSH rule 3 x 224 [1,4,8]
+ CRUSH rule 3 x 225 [8,2]
+ CRUSH rule 3 x 226 [8,2,4]
+ CRUSH rule 3 x 227 [4,1,8]
+ CRUSH rule 3 x 228 [8,1]
+ CRUSH rule 3 x 229 [4,8,1]
+ CRUSH rule 3 x 230 [4,8,1]
+ CRUSH rule 3 x 231 [4,8,1]
+ CRUSH rule 3 x 232 [2,8,4]
+ CRUSH rule 3 x 233 [8,2]
+ CRUSH rule 3 x 234 [1,8]
+ CRUSH rule 3 x 235 [4,8,1]
+ CRUSH rule 3 x 236 [2,8]
+ CRUSH rule 3 x 237 [4,8,2]
+ CRUSH rule 3 x 238 [2,8]
+ CRUSH rule 3 x 239 [8,2]
+ CRUSH rule 3 x 240 [4,8,2]
+ CRUSH rule 3 x 241 [1,8]
+ CRUSH rule 3 x 242 [2,8]
+ CRUSH rule 3 x 243 [8,2]
+ CRUSH rule 3 x 244 [4,8,2]
+ CRUSH rule 3 x 245 [8,2]
+ CRUSH rule 3 x 246 [1,8]
+ CRUSH rule 3 x 247 [8,1]
+ CRUSH rule 3 x 248 [8,1,4]
+ CRUSH rule 3 x 249 [2,8]
+ CRUSH rule 3 x 250 [2,4,8]
+ CRUSH rule 3 x 251 [2,8]
+ CRUSH rule 3 x 252 [4,8,2]
+ CRUSH rule 3 x 253 [2,8]
+ CRUSH rule 3 x 254 [4,2,8]
+ CRUSH rule 3 x 255 [1,8]
+ CRUSH rule 3 x 256 [4,8,1]
+ CRUSH rule 3 x 257 [2,6,4]
+ CRUSH rule 3 x 258 [4,2,8]
+ CRUSH rule 3 x 259 [6,1]
+ CRUSH rule 3 x 260 [8,1]
+ CRUSH rule 3 x 261 [8,2]
+ CRUSH rule 3 x 262 [8,2]
+ CRUSH rule 3 x 263 [8,1,4]
+ CRUSH rule 3 x 264 [8,1]
+ CRUSH rule 3 x 265 [8,2]
+ CRUSH rule 3 x 266 [8,2,4]
+ CRUSH rule 3 x 267 [2,8]
+ CRUSH rule 3 x 268 [1,8]
+ CRUSH rule 3 x 269 [1,8,4]
+ CRUSH rule 3 x 270 [4,1,8]
+ CRUSH rule 3 x 271 [8,4,2]
+ CRUSH rule 3 x 272 [2,8,4]
+ CRUSH rule 3 x 273 [4,2,8]
+ CRUSH rule 3 x 274 [8,4,2]
+ CRUSH rule 3 x 275 [4,8,1]
+ CRUSH rule 3 x 276 [8,1,4]
+ CRUSH rule 3 x 277 [8,1]
+ CRUSH rule 3 x 278 [8,2,4]
+ CRUSH rule 3 x 279 [8,4,1]
+ CRUSH rule 3 x 280 [2,8,4]
+ CRUSH rule 3 x 281 [8,1]
+ CRUSH rule 3 x 282 [2,8]
+ CRUSH rule 3 x 283 [8,1]
+ CRUSH rule 3 x 284 [8,1]
+ CRUSH rule 3 x 285 [4,8,2]
+ CRUSH rule 3 x 286 [2,8,4]
+ CRUSH rule 3 x 287 [1,8]
+ CRUSH rule 3 x 288 [8,1,4]
+ CRUSH rule 3 x 289 [4,8,2]
+ CRUSH rule 3 x 290 [1,4,8]
+ CRUSH rule 3 x 291 [1,4,8]
+ CRUSH rule 3 x 292 [8,1,4]
+ CRUSH rule 3 x 293 [8,2]
+ CRUSH rule 3 x 294 [8,4,2]
+ CRUSH rule 3 x 295 [4,8,1]
+ CRUSH rule 3 x 296 [4,1,8]
+ CRUSH rule 3 x 297 [8,2,4]
+ CRUSH rule 3 x 298 [1,8]
+ CRUSH rule 3 x 299 [2,8]
+ CRUSH rule 3 x 300 [8,1]
+ CRUSH rule 3 x 301 [1,8]
+ CRUSH rule 3 x 302 [1,8]
+ CRUSH rule 3 x 303 [8,4,1]
+ CRUSH rule 3 x 304 [2,8]
+ CRUSH rule 3 x 305 [8,1]
+ CRUSH rule 3 x 306 [1,8]
+ CRUSH rule 3 x 307 [2,8]
+ CRUSH rule 3 x 308 [2,8,4]
+ CRUSH rule 3 x 309 [8,2]
+ CRUSH rule 3 x 310 [4,1,6]
+ CRUSH rule 3 x 311 [4,8,2]
+ CRUSH rule 3 x 312 [2,8]
+ CRUSH rule 3 x 313 [4,1,8]
+ CRUSH rule 3 x 314 [2,6]
+ CRUSH rule 3 x 315 [2,8]
+ CRUSH rule 3 x 316 [8,1]
+ CRUSH rule 3 x 317 [2,8]
+ CRUSH rule 3 x 318 [8,1]
+ CRUSH rule 3 x 319 [2,8]
+ CRUSH rule 3 x 320 [8,1]
+ CRUSH rule 3 x 321 [1,8]
+ CRUSH rule 3 x 322 [2,6,4]
+ CRUSH rule 3 x 323 [4,8,2]
+ CRUSH rule 3 x 324 [8,2,4]
+ CRUSH rule 3 x 325 [4,8,1]
+ CRUSH rule 3 x 326 [1,6]
+ CRUSH rule 3 x 327 [1,8]
+ CRUSH rule 3 x 328 [8,4,1]
+ CRUSH rule 3 x 329 [4,8,2]
+ CRUSH rule 3 x 330 [4,8,1]
+ CRUSH rule 3 x 331 [2,8]
+ CRUSH rule 3 x 332 [2,8]
+ CRUSH rule 3 x 333 [8,1]
+ CRUSH rule 3 x 334 [8,2]
+ CRUSH rule 3 x 335 [8,2]
+ CRUSH rule 3 x 336 [4,8,2]
+ CRUSH rule 3 x 337 [8,2,4]
+ CRUSH rule 3 x 338 [8,2]
+ CRUSH rule 3 x 339 [8,2]
+ CRUSH rule 3 x 340 [2,8,4]
+ CRUSH rule 3 x 341 [4,1,6]
+ CRUSH rule 3 x 342 [2,8,4]
+ CRUSH rule 3 x 343 [8,1]
+ CRUSH rule 3 x 344 [6,1,4]
+ CRUSH rule 3 x 345 [8,2]
+ CRUSH rule 3 x 346 [8,2,4]
+ CRUSH rule 3 x 347 [4,1,8]
+ CRUSH rule 3 x 348 [8,2,4]
+ CRUSH rule 3 x 349 [1,8]
+ CRUSH rule 3 x 350 [8,1]
+ CRUSH rule 3 x 351 [8,2]
+ CRUSH rule 3 x 352 [1,8,4]
+ CRUSH rule 3 x 353 [8,1]
+ CRUSH rule 3 x 354 [1,8]
+ CRUSH rule 3 x 355 [8,2]
+ CRUSH rule 3 x 356 [4,1,8]
+ CRUSH rule 3 x 357 [8,1,4]
+ CRUSH rule 3 x 358 [2,8,4]
+ CRUSH rule 3 x 359 [6,1,4]
+ CRUSH rule 3 x 360 [2,8]
+ CRUSH rule 3 x 361 [8,4,1]
+ CRUSH rule 3 x 362 [4,1,8]
+ CRUSH rule 3 x 363 [4,2,8]
+ CRUSH rule 3 x 364 [2,8]
+ CRUSH rule 3 x 365 [8,2]
+ CRUSH rule 3 x 366 [8,2]
+ CRUSH rule 3 x 367 [4,2,8]
+ CRUSH rule 3 x 368 [8,4,1]
+ CRUSH rule 3 x 369 [8,2]
+ CRUSH rule 3 x 370 [8,2]
+ CRUSH rule 3 x 371 [1,4,8]
+ CRUSH rule 3 x 372 [1,8]
+ CRUSH rule 3 x 373 [1,8]
+ CRUSH rule 3 x 374 [8,1]
+ CRUSH rule 3 x 375 [8,4,2]
+ CRUSH rule 3 x 376 [8,1,4]
+ CRUSH rule 3 x 377 [1,4,8]
+ CRUSH rule 3 x 378 [1,8]
+ CRUSH rule 3 x 379 [8,2]
+ CRUSH rule 3 x 380 [2,8]
+ CRUSH rule 3 x 381 [1,4,8]
+ CRUSH rule 3 x 382 [1,4,8]
+ CRUSH rule 3 x 383 [4,8,1]
+ CRUSH rule 3 x 384 [8,2,4]
+ CRUSH rule 3 x 385 [8,1]
+ CRUSH rule 3 x 386 [1,8]
+ CRUSH rule 3 x 387 [1,4,8]
+ CRUSH rule 3 x 388 [2,6]
+ CRUSH rule 3 x 389 [1,4,8]
+ CRUSH rule 3 x 390 [4,8,1]
+ CRUSH rule 3 x 391 [4,8,2]
+ CRUSH rule 3 x 392 [1,8,4]
+ CRUSH rule 3 x 393 [2,8]
+ CRUSH rule 3 x 394 [8,2]
+ CRUSH rule 3 x 395 [1,8]
+ CRUSH rule 3 x 396 [4,2,8]
+ CRUSH rule 3 x 397 [2,4,6]
+ CRUSH rule 3 x 398 [2,4,8]
+ CRUSH rule 3 x 399 [8,4,2]
+ CRUSH rule 3 x 400 [8,1,4]
+ CRUSH rule 3 x 401 [1,4,8]
+ CRUSH rule 3 x 402 [8,4,2]
+ CRUSH rule 3 x 403 [1,4,8]
+ CRUSH rule 3 x 404 [4,2,8]
+ CRUSH rule 3 x 405 [8,4,2]
+ CRUSH rule 3 x 406 [2,8]
+ CRUSH rule 3 x 407 [2,8,4]
+ CRUSH rule 3 x 408 [4,1,8]
+ CRUSH rule 3 x 409 [8,4,2]
+ CRUSH rule 3 x 410 [8,4,1]
+ CRUSH rule 3 x 411 [2,8,4]
+ CRUSH rule 3 x 412 [2,6]
+ CRUSH rule 3 x 413 [2,8]
+ CRUSH rule 3 x 414 [4,1,8]
+ CRUSH rule 3 x 415 [2,8]
+ CRUSH rule 3 x 416 [2,8]
+ CRUSH rule 3 x 417 [8,2]
+ CRUSH rule 3 x 418 [8,1,4]
+ CRUSH rule 3 x 419 [8,4,2]
+ CRUSH rule 3 x 420 [1,4,8]
+ CRUSH rule 3 x 421 [8,4,1]
+ CRUSH rule 3 x 422 [6,1]
+ CRUSH rule 3 x 423 [2,4,8]
+ CRUSH rule 3 x 424 [8,1]
+ CRUSH rule 3 x 425 [1,8]
+ CRUSH rule 3 x 426 [8,2]
+ CRUSH rule 3 x 427 [1,8]
+ CRUSH rule 3 x 428 [4,8,1]
+ CRUSH rule 3 x 429 [4,8,1]
+ CRUSH rule 3 x 430 [4,8,2]
+ CRUSH rule 3 x 431 [4,2,8]
+ CRUSH rule 3 x 432 [8,1]
+ CRUSH rule 3 x 433 [8,1]
+ CRUSH rule 3 x 434 [2,8]
+ CRUSH rule 3 x 435 [2,6]
+ CRUSH rule 3 x 436 [4,1,8]
+ CRUSH rule 3 x 437 [8,1]
+ CRUSH rule 3 x 438 [2,4,8]
+ CRUSH rule 3 x 439 [1,8]
+ CRUSH rule 3 x 440 [2,8]
+ CRUSH rule 3 x 441 [4,8,2]
+ CRUSH rule 3 x 442 [2,8]
+ CRUSH rule 3 x 443 [8,1,4]
+ CRUSH rule 3 x 444 [8,1]
+ CRUSH rule 3 x 445 [8,2]
+ CRUSH rule 3 x 446 [1,8]
+ CRUSH rule 3 x 447 [2,4,8]
+ CRUSH rule 3 x 448 [8,2,4]
+ CRUSH rule 3 x 449 [8,2]
+ CRUSH rule 3 x 450 [1,8]
+ CRUSH rule 3 x 451 [8,4,1]
+ CRUSH rule 3 x 452 [8,2]
+ CRUSH rule 3 x 453 [6,2]
+ CRUSH rule 3 x 454 [8,2]
+ CRUSH rule 3 x 455 [2,8,4]
+ CRUSH rule 3 x 456 [8,2]
+ CRUSH rule 3 x 457 [8,1]
+ CRUSH rule 3 x 458 [2,8]
+ CRUSH rule 3 x 459 [2,8,4]
+ CRUSH rule 3 x 460 [8,2]
+ CRUSH rule 3 x 461 [8,2]
+ CRUSH rule 3 x 462 [8,1]
+ CRUSH rule 3 x 463 [8,2]
+ CRUSH rule 3 x 464 [8,4,1]
+ CRUSH rule 3 x 465 [6,1,4]
+ CRUSH rule 3 x 466 [8,2]
+ CRUSH rule 3 x 467 [8,1]
+ CRUSH rule 3 x 468 [8,2,4]
+ CRUSH rule 3 x 469 [8,2]
+ CRUSH rule 3 x 470 [4,1,8]
+ CRUSH rule 3 x 471 [1,8]
+ CRUSH rule 3 x 472 [1,8]
+ CRUSH rule 3 x 473 [1,4,8]
+ CRUSH rule 3 x 474 [8,1]
+ CRUSH rule 3 x 475 [8,2,4]
+ CRUSH rule 3 x 476 [4,8,1]
+ CRUSH rule 3 x 477 [4,8,1]
+ CRUSH rule 3 x 478 [8,1,4]
+ CRUSH rule 3 x 479 [2,8]
+ CRUSH rule 3 x 480 [1,8]
+ CRUSH rule 3 x 481 [2,4,6]
+ CRUSH rule 3 x 482 [8,1]
+ CRUSH rule 3 x 483 [2,8,4]
+ CRUSH rule 3 x 484 [1,8]
+ CRUSH rule 3 x 485 [8,1]
+ CRUSH rule 3 x 486 [4,1,8]
+ CRUSH rule 3 x 487 [1,8]
+ CRUSH rule 3 x 488 [8,1]
+ CRUSH rule 3 x 489 [2,8]
+ CRUSH rule 3 x 490 [6,2]
+ CRUSH rule 3 x 491 [1,8]
+ CRUSH rule 3 x 492 [8,2]
+ CRUSH rule 3 x 493 [2,8]
+ CRUSH rule 3 x 494 [1,8]
+ CRUSH rule 3 x 495 [4,1,6]
+ CRUSH rule 3 x 496 [8,4,1]
+ CRUSH rule 3 x 497 [4,8,2]
+ CRUSH rule 3 x 498 [2,4,8]
+ CRUSH rule 3 x 499 [8,4,2]
+ CRUSH rule 3 x 500 [4,8,1]
+ CRUSH rule 3 x 501 [2,8]
+ CRUSH rule 3 x 502 [6,1]
+ CRUSH rule 3 x 503 [2,8]
+ CRUSH rule 3 x 504 [8,2]
+ CRUSH rule 3 x 505 [1,8]
+ CRUSH rule 3 x 506 [4,1,8]
+ CRUSH rule 3 x 507 [8,2,4]
+ CRUSH rule 3 x 508 [1,8]
+ CRUSH rule 3 x 509 [8,1]
+ CRUSH rule 3 x 510 [8,2]
+ CRUSH rule 3 x 511 [4,8,2]
+ CRUSH rule 3 x 512 [8,2]
+ CRUSH rule 3 x 513 [8,2]
+ CRUSH rule 3 x 514 [8,2]
+ CRUSH rule 3 x 515 [8,4,2]
+ CRUSH rule 3 x 516 [4,1,8]
+ CRUSH rule 3 x 517 [8,2]
+ CRUSH rule 3 x 518 [4,8,1]
+ CRUSH rule 3 x 519 [8,4,1]
+ CRUSH rule 3 x 520 [2,8,4]
+ CRUSH rule 3 x 521 [8,2,4]
+ CRUSH rule 3 x 522 [8,1,4]
+ CRUSH rule 3 x 523 [4,1,8]
+ CRUSH rule 3 x 524 [2,6]
+ CRUSH rule 3 x 525 [2,8]
+ CRUSH rule 3 x 526 [1,8]
+ CRUSH rule 3 x 527 [1,4,6]
+ CRUSH rule 3 x 528 [2,8]
+ CRUSH rule 3 x 529 [4,8,1]
+ CRUSH rule 3 x 530 [8,1]
+ CRUSH rule 3 x 531 [8,1,4]
+ CRUSH rule 3 x 532 [6,4,1]
+ CRUSH rule 3 x 533 [4,8,1]
+ CRUSH rule 3 x 534 [8,2]
+ CRUSH rule 3 x 535 [8,1]
+ CRUSH rule 3 x 536 [8,2]
+ CRUSH rule 3 x 537 [4,8,1]
+ CRUSH rule 3 x 538 [8,4,2]
+ CRUSH rule 3 x 539 [8,2]
+ CRUSH rule 3 x 540 [1,8,4]
+ CRUSH rule 3 x 541 [2,4,8]
+ CRUSH rule 3 x 542 [2,8]
+ CRUSH rule 3 x 543 [8,2]
+ CRUSH rule 3 x 544 [4,8,1]
+ CRUSH rule 3 x 545 [8,1]
+ CRUSH rule 3 x 546 [8,2,4]
+ CRUSH rule 3 x 547 [8,1,4]
+ CRUSH rule 3 x 548 [4,1,8]
+ CRUSH rule 3 x 549 [8,1]
+ CRUSH rule 3 x 550 [2,4,8]
+ CRUSH rule 3 x 551 [8,1]
+ CRUSH rule 3 x 552 [4,8,2]
+ CRUSH rule 3 x 553 [2,8]
+ CRUSH rule 3 x 554 [1,8]
+ CRUSH rule 3 x 555 [4,2,8]
+ CRUSH rule 3 x 556 [8,1]
+ CRUSH rule 3 x 557 [8,2]
+ CRUSH rule 3 x 558 [4,2,8]
+ CRUSH rule 3 x 559 [1,8]
+ CRUSH rule 3 x 560 [8,2]
+ CRUSH rule 3 x 561 [8,4,1]
+ CRUSH rule 3 x 562 [4,1,8]
+ CRUSH rule 3 x 563 [2,8]
+ CRUSH rule 3 x 564 [1,8]
+ CRUSH rule 3 x 565 [4,8,1]
+ CRUSH rule 3 x 566 [4,8,2]
+ CRUSH rule 3 x 567 [4,8,1]
+ CRUSH rule 3 x 568 [8,1]
+ CRUSH rule 3 x 569 [4,1,8]
+ CRUSH rule 3 x 570 [1,8]
+ CRUSH rule 3 x 571 [6,1]
+ CRUSH rule 3 x 572 [4,2,8]
+ CRUSH rule 3 x 573 [1,8]
+ CRUSH rule 3 x 574 [2,8]
+ CRUSH rule 3 x 575 [8,1,4]
+ CRUSH rule 3 x 576 [4,8,1]
+ CRUSH rule 3 x 577 [8,2]
+ CRUSH rule 3 x 578 [8,1]
+ CRUSH rule 3 x 579 [4,2,8]
+ CRUSH rule 3 x 580 [1,8]
+ CRUSH rule 3 x 581 [8,1,4]
+ CRUSH rule 3 x 582 [2,8,4]
+ CRUSH rule 3 x 583 [8,2]
+ CRUSH rule 3 x 584 [8,1,4]
+ CRUSH rule 3 x 585 [8,1,4]
+ CRUSH rule 3 x 586 [1,8,4]
+ CRUSH rule 3 x 587 [2,4,8]
+ CRUSH rule 3 x 588 [4,8,1]
+ CRUSH rule 3 x 589 [8,1]
+ CRUSH rule 3 x 590 [8,2]
+ CRUSH rule 3 x 591 [4,2,8]
+ CRUSH rule 3 x 592 [2,4,8]
+ CRUSH rule 3 x 593 [1,8,4]
+ CRUSH rule 3 x 594 [2,8]
+ CRUSH rule 3 x 595 [8,1]
+ CRUSH rule 3 x 596 [2,8]
+ CRUSH rule 3 x 597 [1,8]
+ CRUSH rule 3 x 598 [2,8]
+ CRUSH rule 3 x 599 [4,1,8]
+ CRUSH rule 3 x 600 [8,1,4]
+ CRUSH rule 3 x 601 [1,8,4]
+ CRUSH rule 3 x 602 [8,2]
+ CRUSH rule 3 x 603 [1,8]
+ CRUSH rule 3 x 604 [8,2]
+ CRUSH rule 3 x 605 [2,8]
+ CRUSH rule 3 x 606 [2,6,4]
+ CRUSH rule 3 x 607 [2,4,8]
+ CRUSH rule 3 x 608 [4,1,8]
+ CRUSH rule 3 x 609 [4,2,8]
+ CRUSH rule 3 x 610 [8,2]
+ CRUSH rule 3 x 611 [1,8]
+ CRUSH rule 3 x 612 [2,8]
+ CRUSH rule 3 x 613 [8,1,4]
+ CRUSH rule 3 x 614 [8,2,4]
+ CRUSH rule 3 x 615 [8,2,4]
+ CRUSH rule 3 x 616 [1,8]
+ CRUSH rule 3 x 617 [8,2,4]
+ CRUSH rule 3 x 618 [8,4,1]
+ CRUSH rule 3 x 619 [4,1,8]
+ CRUSH rule 3 x 620 [1,8]
+ CRUSH rule 3 x 621 [8,1]
+ CRUSH rule 3 x 622 [2,4,8]
+ CRUSH rule 3 x 623 [2,8]
+ CRUSH rule 3 x 624 [4,2,8]
+ CRUSH rule 3 x 625 [2,8]
+ CRUSH rule 3 x 626 [8,2,4]
+ CRUSH rule 3 x 627 [2,8,4]
+ CRUSH rule 3 x 628 [8,1]
+ CRUSH rule 3 x 629 [2,8,4]
+ CRUSH rule 3 x 630 [2,8]
+ CRUSH rule 3 x 631 [1,8,4]
+ CRUSH rule 3 x 632 [8,2]
+ CRUSH rule 3 x 633 [8,2]
+ CRUSH rule 3 x 634 [1,8]
+ CRUSH rule 3 x 635 [4,8,2]
+ CRUSH rule 3 x 636 [1,4,8]
+ CRUSH rule 3 x 637 [1,8]
+ CRUSH rule 3 x 638 [8,2,4]
+ CRUSH rule 3 x 639 [2,8]
+ CRUSH rule 3 x 640 [2,8]
+ CRUSH rule 3 x 641 [8,2]
+ CRUSH rule 3 x 642 [2,8]
+ CRUSH rule 3 x 643 [1,8]
+ CRUSH rule 3 x 644 [8,1]
+ CRUSH rule 3 x 645 [8,2]
+ CRUSH rule 3 x 646 [8,1,4]
+ CRUSH rule 3 x 647 [8,1]
+ CRUSH rule 3 x 648 [1,8]
+ CRUSH rule 3 x 649 [4,8,1]
+ CRUSH rule 3 x 650 [8,4,1]
+ CRUSH rule 3 x 651 [4,6,1]
+ CRUSH rule 3 x 652 [4,8,1]
+ CRUSH rule 3 x 653 [8,2]
+ CRUSH rule 3 x 654 [6,2]
+ CRUSH rule 3 x 655 [1,4,8]
+ CRUSH rule 3 x 656 [8,1]
+ CRUSH rule 3 x 657 [6,1]
+ CRUSH rule 3 x 658 [8,2]
+ CRUSH rule 3 x 659 [4,8,1]
+ CRUSH rule 3 x 660 [8,2]
+ CRUSH rule 3 x 661 [1,8]
+ CRUSH rule 3 x 662 [2,8]
+ CRUSH rule 3 x 663 [1,4,8]
+ CRUSH rule 3 x 664 [1,4,8]
+ CRUSH rule 3 x 665 [4,6,1]
+ CRUSH rule 3 x 666 [2,8]
+ CRUSH rule 3 x 667 [1,4,8]
+ CRUSH rule 3 x 668 [4,8,2]
+ CRUSH rule 3 x 669 [6,4,1]
+ CRUSH rule 3 x 670 [4,1,8]
+ CRUSH rule 3 x 671 [2,8]
+ CRUSH rule 3 x 672 [4,2,8]
+ CRUSH rule 3 x 673 [4,2,8]
+ CRUSH rule 3 x 674 [1,8]
+ CRUSH rule 3 x 675 [1,8,4]
+ CRUSH rule 3 x 676 [2,4,8]
+ CRUSH rule 3 x 677 [4,1,8]
+ CRUSH rule 3 x 678 [2,4,8]
+ CRUSH rule 3 x 679 [8,2]
+ CRUSH rule 3 x 680 [2,8]
+ CRUSH rule 3 x 681 [8,1]
+ CRUSH rule 3 x 682 [1,4,8]
+ CRUSH rule 3 x 683 [1,4,8]
+ CRUSH rule 3 x 684 [8,2,4]
+ CRUSH rule 3 x 685 [8,2,4]
+ CRUSH rule 3 x 686 [1,4,8]
+ CRUSH rule 3 x 687 [6,2]
+ CRUSH rule 3 x 688 [4,8,2]
+ CRUSH rule 3 x 689 [8,4,1]
+ CRUSH rule 3 x 690 [8,1,4]
+ CRUSH rule 3 x 691 [1,8]
+ CRUSH rule 3 x 692 [8,1]
+ CRUSH rule 3 x 693 [8,4,2]
+ CRUSH rule 3 x 694 [8,4,2]
+ CRUSH rule 3 x 695 [2,8,4]
+ CRUSH rule 3 x 696 [1,8]
+ CRUSH rule 3 x 697 [8,1,4]
+ CRUSH rule 3 x 698 [8,2,4]
+ CRUSH rule 3 x 699 [1,8,4]
+ CRUSH rule 3 x 700 [1,6]
+ CRUSH rule 3 x 701 [1,8]
+ CRUSH rule 3 x 702 [2,8]
+ CRUSH rule 3 x 703 [8,1]
+ CRUSH rule 3 x 704 [1,4,8]
+ CRUSH rule 3 x 705 [8,1,4]
+ CRUSH rule 3 x 706 [1,4,8]
+ CRUSH rule 3 x 707 [8,4,1]
+ CRUSH rule 3 x 708 [4,8,1]
+ CRUSH rule 3 x 709 [8,1]
+ CRUSH rule 3 x 710 [8,1]
+ CRUSH rule 3 x 711 [2,4,8]
+ CRUSH rule 3 x 712 [2,8]
+ CRUSH rule 3 x 713 [8,4,1]
+ CRUSH rule 3 x 714 [2,8]
+ CRUSH rule 3 x 715 [1,8]
+ CRUSH rule 3 x 716 [4,8,1]
+ CRUSH rule 3 x 717 [8,2,4]
+ CRUSH rule 3 x 718 [8,1]
+ CRUSH rule 3 x 719 [2,6,4]
+ CRUSH rule 3 x 720 [8,1,4]
+ CRUSH rule 3 x 721 [4,6,1]
+ CRUSH rule 3 x 722 [8,1]
+ CRUSH rule 3 x 723 [4,2,8]
+ CRUSH rule 3 x 724 [2,6]
+ CRUSH rule 3 x 725 [1,8]
+ CRUSH rule 3 x 726 [4,8,2]
+ CRUSH rule 3 x 727 [4,8,2]
+ CRUSH rule 3 x 728 [2,8,4]
+ CRUSH rule 3 x 729 [8,1]
+ CRUSH rule 3 x 730 [4,8,2]
+ CRUSH rule 3 x 731 [4,1,8]
+ CRUSH rule 3 x 732 [1,8]
+ CRUSH rule 3 x 733 [4,8,2]
+ CRUSH rule 3 x 734 [8,4,1]
+ CRUSH rule 3 x 735 [4,8,1]
+ CRUSH rule 3 x 736 [4,8,1]
+ CRUSH rule 3 x 737 [1,8,4]
+ CRUSH rule 3 x 738 [4,1,8]
+ CRUSH rule 3 x 739 [2,8]
+ CRUSH rule 3 x 740 [1,8,4]
+ CRUSH rule 3 x 741 [8,2]
+ CRUSH rule 3 x 742 [8,2]
+ CRUSH rule 3 x 743 [8,1,4]
+ CRUSH rule 3 x 744 [4,8,1]
+ CRUSH rule 3 x 745 [1,8]
+ CRUSH rule 3 x 746 [1,8]
+ CRUSH rule 3 x 747 [8,2]
+ CRUSH rule 3 x 748 [2,8,4]
+ CRUSH rule 3 x 749 [4,8,2]
+ CRUSH rule 3 x 750 [1,8,4]
+ CRUSH rule 3 x 751 [2,8]
+ CRUSH rule 3 x 752 [8,1]
+ CRUSH rule 3 x 753 [8,4,2]
+ CRUSH rule 3 x 754 [8,4,2]
+ CRUSH rule 3 x 755 [1,8,4]
+ CRUSH rule 3 x 756 [8,1]
+ CRUSH rule 3 x 757 [8,1,4]
+ CRUSH rule 3 x 758 [8,1]
+ CRUSH rule 3 x 759 [8,4,2]
+ CRUSH rule 3 x 760 [1,4,8]
+ CRUSH rule 3 x 761 [2,6]
+ CRUSH rule 3 x 762 [2,8]
+ CRUSH rule 3 x 763 [8,4,1]
+ CRUSH rule 3 x 764 [1,8]
+ CRUSH rule 3 x 765 [8,2]
+ CRUSH rule 3 x 766 [8,1]
+ CRUSH rule 3 x 767 [1,8,4]
+ CRUSH rule 3 x 768 [8,4,2]
+ CRUSH rule 3 x 769 [8,2,4]
+ CRUSH rule 3 x 770 [8,1,4]
+ CRUSH rule 3 x 771 [8,2,4]
+ CRUSH rule 3 x 772 [8,4,2]
+ CRUSH rule 3 x 773 [4,2,8]
+ CRUSH rule 3 x 774 [8,1]
+ CRUSH rule 3 x 775 [8,4,2]
+ CRUSH rule 3 x 776 [6,2]
+ CRUSH rule 3 x 777 [4,1,8]
+ CRUSH rule 3 x 778 [1,6,4]
+ CRUSH rule 3 x 779 [2,8]
+ CRUSH rule 3 x 780 [2,4,8]
+ CRUSH rule 3 x 781 [8,2]
+ CRUSH rule 3 x 782 [4,2,8]
+ CRUSH rule 3 x 783 [8,1,4]
+ CRUSH rule 3 x 784 [1,4,8]
+ CRUSH rule 3 x 785 [8,1,4]
+ CRUSH rule 3 x 786 [8,1]
+ CRUSH rule 3 x 787 [1,8,4]
+ CRUSH rule 3 x 788 [8,2,4]
+ CRUSH rule 3 x 789 [1,8]
+ CRUSH rule 3 x 790 [8,1]
+ CRUSH rule 3 x 791 [4,8,2]
+ CRUSH rule 3 x 792 [4,8,1]
+ CRUSH rule 3 x 793 [8,2,4]
+ CRUSH rule 3 x 794 [2,8,4]
+ CRUSH rule 3 x 795 [1,8]
+ CRUSH rule 3 x 796 [8,1]
+ CRUSH rule 3 x 797 [2,4,8]
+ CRUSH rule 3 x 798 [6,1]
+ CRUSH rule 3 x 799 [4,2,8]
+ CRUSH rule 3 x 800 [2,8]
+ CRUSH rule 3 x 801 [4,8,2]
+ CRUSH rule 3 x 802 [1,8,4]
+ CRUSH rule 3 x 803 [2,8]
+ CRUSH rule 3 x 804 [8,2]
+ CRUSH rule 3 x 805 [8,2]
+ CRUSH rule 3 x 806 [1,4,8]
+ CRUSH rule 3 x 807 [4,8,2]
+ CRUSH rule 3 x 808 [8,2]
+ CRUSH rule 3 x 809 [1,8]
+ CRUSH rule 3 x 810 [8,2]
+ CRUSH rule 3 x 811 [8,1]
+ CRUSH rule 3 x 812 [8,4,1]
+ CRUSH rule 3 x 813 [8,4,2]
+ CRUSH rule 3 x 814 [8,2]
+ CRUSH rule 3 x 815 [4,1,8]
+ CRUSH rule 3 x 816 [2,8]
+ CRUSH rule 3 x 817 [8,2]
+ CRUSH rule 3 x 818 [1,8]
+ CRUSH rule 3 x 819 [1,8]
+ CRUSH rule 3 x 820 [4,8,1]
+ CRUSH rule 3 x 821 [4,8,2]
+ CRUSH rule 3 x 822 [2,4,8]
+ CRUSH rule 3 x 823 [4,8,2]
+ CRUSH rule 3 x 824 [8,1]
+ CRUSH rule 3 x 825 [2,8,4]
+ CRUSH rule 3 x 826 [8,1,4]
+ CRUSH rule 3 x 827 [2,6,4]
+ CRUSH rule 3 x 828 [2,8]
+ CRUSH rule 3 x 829 [8,1]
+ CRUSH rule 3 x 830 [2,4,8]
+ CRUSH rule 3 x 831 [1,8]
+ CRUSH rule 3 x 832 [4,8,2]
+ CRUSH rule 3 x 833 [2,8]
+ CRUSH rule 3 x 834 [1,8]
+ CRUSH rule 3 x 835 [8,4,1]
+ CRUSH rule 3 x 836 [4,8,1]
+ CRUSH rule 3 x 837 [8,4,1]
+ CRUSH rule 3 x 838 [6,2,4]
+ CRUSH rule 3 x 839 [2,8]
+ CRUSH rule 3 x 840 [8,2]
+ CRUSH rule 3 x 841 [4,8,1]
+ CRUSH rule 3 x 842 [2,8]
+ CRUSH rule 3 x 843 [8,4,2]
+ CRUSH rule 3 x 844 [8,2]
+ CRUSH rule 3 x 845 [4,8,2]
+ CRUSH rule 3 x 846 [4,2,8]
+ CRUSH rule 3 x 847 [2,8]
+ CRUSH rule 3 x 848 [2,8,4]
+ CRUSH rule 3 x 849 [4,8,1]
+ CRUSH rule 3 x 850 [1,6]
+ CRUSH rule 3 x 851 [6,1]
+ CRUSH rule 3 x 852 [8,4,2]
+ CRUSH rule 3 x 853 [6,2]
+ CRUSH rule 3 x 854 [8,2]
+ CRUSH rule 3 x 855 [8,2]
+ CRUSH rule 3 x 856 [8,4,1]
+ CRUSH rule 3 x 857 [8,1]
+ CRUSH rule 3 x 858 [6,2]
+ CRUSH rule 3 x 859 [8,2,4]
+ CRUSH rule 3 x 860 [2,8]
+ CRUSH rule 3 x 861 [8,1]
+ CRUSH rule 3 x 862 [8,1]
+ CRUSH rule 3 x 863 [8,1]
+ CRUSH rule 3 x 864 [8,1]
+ CRUSH rule 3 x 865 [8,1]
+ CRUSH rule 3 x 866 [8,2]
+ CRUSH rule 3 x 867 [8,1]
+ CRUSH rule 3 x 868 [8,2]
+ CRUSH rule 3 x 869 [8,4,2]
+ CRUSH rule 3 x 870 [2,8]
+ CRUSH rule 3 x 871 [1,8]
+ CRUSH rule 3 x 872 [1,8]
+ CRUSH rule 3 x 873 [4,8,1]
+ CRUSH rule 3 x 874 [2,6]
+ CRUSH rule 3 x 875 [2,8,4]
+ CRUSH rule 3 x 876 [4,8,1]
+ CRUSH rule 3 x 877 [8,4,2]
+ CRUSH rule 3 x 878 [2,8]
+ CRUSH rule 3 x 879 [8,1]
+ CRUSH rule 3 x 880 [1,8]
+ CRUSH rule 3 x 881 [4,8,2]
+ CRUSH rule 3 x 882 [1,8]
+ CRUSH rule 3 x 883 [2,4,8]
+ CRUSH rule 3 x 884 [8,1,4]
+ CRUSH rule 3 x 885 [4,1,8]
+ CRUSH rule 3 x 886 [8,2]
+ CRUSH rule 3 x 887 [8,4,2]
+ CRUSH rule 3 x 888 [8,1]
+ CRUSH rule 3 x 889 [2,8]
+ CRUSH rule 3 x 890 [8,1,4]
+ CRUSH rule 3 x 891 [1,8]
+ CRUSH rule 3 x 892 [8,2,4]
+ CRUSH rule 3 x 893 [2,6]
+ CRUSH rule 3 x 894 [8,4,1]
+ CRUSH rule 3 x 895 [4,1,8]
+ CRUSH rule 3 x 896 [1,8]
+ CRUSH rule 3 x 897 [2,8]
+ CRUSH rule 3 x 898 [1,4,8]
+ CRUSH rule 3 x 899 [1,8]
+ CRUSH rule 3 x 900 [4,1,8]
+ CRUSH rule 3 x 901 [2,8]
+ CRUSH rule 3 x 902 [8,4,2]
+ CRUSH rule 3 x 903 [8,1]
+ CRUSH rule 3 x 904 [8,2]
+ CRUSH rule 3 x 905 [8,2]
+ CRUSH rule 3 x 906 [1,8]
+ CRUSH rule 3 x 907 [8,2]
+ CRUSH rule 3 x 908 [8,1]
+ CRUSH rule 3 x 909 [2,8]
+ CRUSH rule 3 x 910 [8,1]
+ CRUSH rule 3 x 911 [8,1]
+ CRUSH rule 3 x 912 [1,8]
+ CRUSH rule 3 x 913 [8,1,4]
+ CRUSH rule 3 x 914 [6,4,1]
+ CRUSH rule 3 x 915 [8,2]
+ CRUSH rule 3 x 916 [4,1,8]
+ CRUSH rule 3 x 917 [1,4,8]
+ CRUSH rule 3 x 918 [8,2]
+ CRUSH rule 3 x 919 [8,2]
+ CRUSH rule 3 x 920 [8,2]
+ CRUSH rule 3 x 921 [1,8]
+ CRUSH rule 3 x 922 [8,4,1]
+ CRUSH rule 3 x 923 [4,8,2]
+ CRUSH rule 3 x 924 [1,8]
+ CRUSH rule 3 x 925 [4,8,2]
+ CRUSH rule 3 x 926 [2,8]
+ CRUSH rule 3 x 927 [1,8,4]
+ CRUSH rule 3 x 928 [8,1]
+ CRUSH rule 3 x 929 [4,2,8]
+ CRUSH rule 3 x 930 [2,8]
+ CRUSH rule 3 x 931 [2,8]
+ CRUSH rule 3 x 932 [4,2,8]
+ CRUSH rule 3 x 933 [8,4,2]
+ CRUSH rule 3 x 934 [8,2]
+ CRUSH rule 3 x 935 [8,2]
+ CRUSH rule 3 x 936 [1,8]
+ CRUSH rule 3 x 937 [4,8,1]
+ CRUSH rule 3 x 938 [8,4,2]
+ CRUSH rule 3 x 939 [2,8,4]
+ CRUSH rule 3 x 940 [8,2]
+ CRUSH rule 3 x 941 [2,6]
+ CRUSH rule 3 x 942 [1,8]
+ CRUSH rule 3 x 943 [8,2]
+ CRUSH rule 3 x 944 [8,1]
+ CRUSH rule 3 x 945 [8,2,4]
+ CRUSH rule 3 x 946 [2,8,4]
+ CRUSH rule 3 x 947 [2,8]
+ CRUSH rule 3 x 948 [8,1]
+ CRUSH rule 3 x 949 [6,2]
+ CRUSH rule 3 x 950 [8,1]
+ CRUSH rule 3 x 951 [8,2]
+ CRUSH rule 3 x 952 [2,8,4]
+ CRUSH rule 3 x 953 [1,4,8]
+ CRUSH rule 3 x 954 [2,8]
+ CRUSH rule 3 x 955 [8,1,4]
+ CRUSH rule 3 x 956 [1,8,4]
+ CRUSH rule 3 x 957 [8,1,4]
+ CRUSH rule 3 x 958 [8,4,2]
+ CRUSH rule 3 x 959 [4,1,8]
+ CRUSH rule 3 x 960 [6,2]
+ CRUSH rule 3 x 961 [1,8]
+ CRUSH rule 3 x 962 [8,4,2]
+ CRUSH rule 3 x 963 [2,4,8]
+ CRUSH rule 3 x 964 [2,8]
+ CRUSH rule 3 x 965 [8,1]
+ CRUSH rule 3 x 966 [4,8,1]
+ CRUSH rule 3 x 967 [8,4,2]
+ CRUSH rule 3 x 968 [8,2]
+ CRUSH rule 3 x 969 [8,2,4]
+ CRUSH rule 3 x 970 [2,8,4]
+ CRUSH rule 3 x 971 [1,8]
+ CRUSH rule 3 x 972 [1,8]
+ CRUSH rule 3 x 973 [1,8]
+ CRUSH rule 3 x 974 [4,1,8]
+ CRUSH rule 3 x 975 [4,8,1]
+ CRUSH rule 3 x 976 [4,8,2]
+ CRUSH rule 3 x 977 [8,4,2]
+ CRUSH rule 3 x 978 [8,2,4]
+ CRUSH rule 3 x 979 [8,2,4]
+ CRUSH rule 3 x 980 [8,2,4]
+ CRUSH rule 3 x 981 [8,1]
+ CRUSH rule 3 x 982 [1,8]
+ CRUSH rule 3 x 983 [4,8,1]
+ CRUSH rule 3 x 984 [2,8]
+ CRUSH rule 3 x 985 [2,4,8]
+ CRUSH rule 3 x 986 [8,4,2]
+ CRUSH rule 3 x 987 [2,8]
+ CRUSH rule 3 x 988 [1,4,6]
+ CRUSH rule 3 x 989 [1,8]
+ CRUSH rule 3 x 990 [1,8,4]
+ CRUSH rule 3 x 991 [1,4,8]
+ CRUSH rule 3 x 992 [8,1,4]
+ CRUSH rule 3 x 993 [2,8,4]
+ CRUSH rule 3 x 994 [4,8,1]
+ CRUSH rule 3 x 995 [8,1,4]
+ CRUSH rule 3 x 996 [8,4,1]
+ CRUSH rule 3 x 997 [8,4,1]
+ CRUSH rule 3 x 998 [8,1,4]
+ CRUSH rule 3 x 999 [1,8,4]
+ CRUSH rule 3 x 1000 [8,4,1]
+ CRUSH rule 3 x 1001 [2,8]
+ CRUSH rule 3 x 1002 [1,8]
+ CRUSH rule 3 x 1003 [2,8]
+ CRUSH rule 3 x 1004 [8,1,4]
+ CRUSH rule 3 x 1005 [8,1]
+ CRUSH rule 3 x 1006 [1,8,4]
+ CRUSH rule 3 x 1007 [1,4,8]
+ CRUSH rule 3 x 1008 [1,8]
+ CRUSH rule 3 x 1009 [6,4,1]
+ CRUSH rule 3 x 1010 [1,8]
+ CRUSH rule 3 x 1011 [4,2,8]
+ CRUSH rule 3 x 1012 [1,8]
+ CRUSH rule 3 x 1013 [2,8]
+ CRUSH rule 3 x 1014 [2,8,4]
+ CRUSH rule 3 x 1015 [8,2]
+ CRUSH rule 3 x 1016 [2,4,8]
+ CRUSH rule 3 x 1017 [6,1,4]
+ CRUSH rule 3 x 1018 [4,1,8]
+ CRUSH rule 3 x 1019 [4,8,1]
+ CRUSH rule 3 x 1020 [1,8]
+ CRUSH rule 3 x 1021 [2,8]
+ CRUSH rule 3 x 1022 [1,8,4]
+ CRUSH rule 3 x 1023 [4,2,8]
+ rule 3 (choose-set) num_rep 3 result size == 2:\t501/1024 (esc)
+ rule 3 (choose-set) num_rep 3 result size == 3:\t523/1024 (esc)
+ rule 4 (choose-set-two), x = 0..1023, numrep = 2..3
+ CRUSH rule 4 x 0 [2,1]
+ CRUSH rule 4 x 1 [2,8]
+ CRUSH rule 4 x 2 [1]
+ CRUSH rule 4 x 3 [8,1]
+ CRUSH rule 4 x 4 [4]
+ CRUSH rule 4 x 5 [8]
+ CRUSH rule 4 x 6 [2,8]
+ CRUSH rule 4 x 7 [4]
+ CRUSH rule 4 x 8 [4]
+ CRUSH rule 4 x 9 [2,4]
+ CRUSH rule 4 x 10 [2,1]
+ CRUSH rule 4 x 11 [2,8]
+ CRUSH rule 4 x 12 [2,1]
+ CRUSH rule 4 x 13 [4,8]
+ CRUSH rule 4 x 14 [8]
+ CRUSH rule 4 x 15 [8,2]
+ CRUSH rule 4 x 16 [8]
+ CRUSH rule 4 x 17 [4]
+ CRUSH rule 4 x 18 [1]
+ CRUSH rule 4 x 19 [8,4]
+ CRUSH rule 4 x 20 [2]
+ CRUSH rule 4 x 21 [8]
+ CRUSH rule 4 x 22 [8]
+ CRUSH rule 4 x 23 [4,8]
+ CRUSH rule 4 x 24 [1,2]
+ CRUSH rule 4 x 25 [4,8]
+ CRUSH rule 4 x 26 [2,8]
+ CRUSH rule 4 x 27 [4,1]
+ CRUSH rule 4 x 28 [8,2]
+ CRUSH rule 4 x 29 [8,4]
+ CRUSH rule 4 x 30 [4,8]
+ CRUSH rule 4 x 31 [8]
+ CRUSH rule 4 x 32 [6]
+ CRUSH rule 4 x 33 [2,8]
+ CRUSH rule 4 x 34 [2]
+ CRUSH rule 4 x 35 [1,8]
+ CRUSH rule 4 x 36 [8]
+ CRUSH rule 4 x 37 [1]
+ CRUSH rule 4 x 38 [4,8]
+ CRUSH rule 4 x 39 [8]
+ CRUSH rule 4 x 40 [8]
+ CRUSH rule 4 x 41 [2,1]
+ CRUSH rule 4 x 42 []
+ CRUSH rule 4 x 43 [1]
+ CRUSH rule 4 x 44 [1,8]
+ CRUSH rule 4 x 45 [8,2]
+ CRUSH rule 4 x 46 [2]
+ CRUSH rule 4 x 47 [4]
+ CRUSH rule 4 x 48 [8]
+ CRUSH rule 4 x 49 []
+ CRUSH rule 4 x 50 [4]
+ CRUSH rule 4 x 51 [8]
+ CRUSH rule 4 x 52 [8]
+ CRUSH rule 4 x 53 [4]
+ CRUSH rule 4 x 54 [8]
+ CRUSH rule 4 x 55 [8]
+ CRUSH rule 4 x 56 [8,4]
+ CRUSH rule 4 x 57 []
+ CRUSH rule 4 x 58 [1,2]
+ CRUSH rule 4 x 59 [2]
+ CRUSH rule 4 x 60 [4]
+ CRUSH rule 4 x 61 [4,8]
+ CRUSH rule 4 x 62 [8,1]
+ CRUSH rule 4 x 63 [8]
+ CRUSH rule 4 x 64 [4]
+ CRUSH rule 4 x 65 [8,4]
+ CRUSH rule 4 x 66 [4]
+ CRUSH rule 4 x 67 [4,2]
+ CRUSH rule 4 x 68 [1]
+ CRUSH rule 4 x 69 [1]
+ CRUSH rule 4 x 70 [8,2]
+ CRUSH rule 4 x 71 [2,8]
+ CRUSH rule 4 x 72 [8,1]
+ CRUSH rule 4 x 73 [2,8]
+ CRUSH rule 4 x 74 [1,8]
+ CRUSH rule 4 x 75 [4,2]
+ CRUSH rule 4 x 76 [4,1]
+ CRUSH rule 4 x 77 [8,2]
+ CRUSH rule 4 x 78 [1]
+ CRUSH rule 4 x 79 [4]
+ CRUSH rule 4 x 80 [2,4]
+ CRUSH rule 4 x 81 [2,1]
+ CRUSH rule 4 x 82 [6,1]
+ CRUSH rule 4 x 83 [2,8]
+ CRUSH rule 4 x 84 [8,2]
+ CRUSH rule 4 x 85 [4]
+ CRUSH rule 4 x 86 [2,1]
+ CRUSH rule 4 x 87 [2,8]
+ CRUSH rule 4 x 88 [1,6]
+ CRUSH rule 4 x 89 [2]
+ CRUSH rule 4 x 90 [8]
+ CRUSH rule 4 x 91 [4,8]
+ CRUSH rule 4 x 92 [1,8]
+ CRUSH rule 4 x 93 [8,4]
+ CRUSH rule 4 x 94 [1]
+ CRUSH rule 4 x 95 [8]
+ CRUSH rule 4 x 96 [8]
+ CRUSH rule 4 x 97 [8]
+ CRUSH rule 4 x 98 [2,1]
+ CRUSH rule 4 x 99 [2,8]
+ CRUSH rule 4 x 100 [1,8]
+ CRUSH rule 4 x 101 [8]
+ CRUSH rule 4 x 102 [2]
+ CRUSH rule 4 x 103 [8]
+ CRUSH rule 4 x 104 [8,4]
+ CRUSH rule 4 x 105 [2,4]
+ CRUSH rule 4 x 106 [1,8]
+ CRUSH rule 4 x 107 [2]
+ CRUSH rule 4 x 108 [8,2]
+ CRUSH rule 4 x 109 [1,2]
+ CRUSH rule 4 x 110 [4,2]
+ CRUSH rule 4 x 111 [2,1]
+ CRUSH rule 4 x 112 [2,1]
+ CRUSH rule 4 x 113 [8,2]
+ CRUSH rule 4 x 114 [8]
+ CRUSH rule 4 x 115 [8,2]
+ CRUSH rule 4 x 116 [1,2]
+ CRUSH rule 4 x 117 [6]
+ CRUSH rule 4 x 118 [2]
+ CRUSH rule 4 x 119 [8]
+ CRUSH rule 4 x 120 [2,1]
+ CRUSH rule 4 x 121 [2,1]
+ CRUSH rule 4 x 122 [8]
+ CRUSH rule 4 x 123 [2]
+ CRUSH rule 4 x 124 []
+ CRUSH rule 4 x 125 [1,8]
+ CRUSH rule 4 x 126 [2]
+ CRUSH rule 4 x 127 [4,8]
+ CRUSH rule 4 x 128 []
+ CRUSH rule 4 x 129 [2,1]
+ CRUSH rule 4 x 130 [4,8]
+ CRUSH rule 4 x 131 [1,2]
+ CRUSH rule 4 x 132 [1,2]
+ CRUSH rule 4 x 133 [8]
+ CRUSH rule 4 x 134 [1,8]
+ CRUSH rule 4 x 135 [4,8]
+ CRUSH rule 4 x 136 [2,1]
+ CRUSH rule 4 x 137 [8,4]
+ CRUSH rule 4 x 138 [8]
+ CRUSH rule 4 x 139 [4,2]
+ CRUSH rule 4 x 140 [1,8]
+ CRUSH rule 4 x 141 [8]
+ CRUSH rule 4 x 142 [4]
+ CRUSH rule 4 x 143 [4,8]
+ CRUSH rule 4 x 144 [8,1]
+ CRUSH rule 4 x 145 [8]
+ CRUSH rule 4 x 146 [2,8]
+ CRUSH rule 4 x 147 [2,8]
+ CRUSH rule 4 x 148 [4]
+ CRUSH rule 4 x 149 [4,8]
+ CRUSH rule 4 x 150 [1,8]
+ CRUSH rule 4 x 151 []
+ CRUSH rule 4 x 152 [8]
+ CRUSH rule 4 x 153 [8]
+ CRUSH rule 4 x 154 [4,2]
+ CRUSH rule 4 x 155 [4]
+ CRUSH rule 4 x 156 [4]
+ CRUSH rule 4 x 157 [1]
+ CRUSH rule 4 x 158 [2,8]
+ CRUSH rule 4 x 159 [8,2]
+ CRUSH rule 4 x 160 [2,8]
+ CRUSH rule 4 x 161 [1,4]
+ CRUSH rule 4 x 162 [1,8]
+ CRUSH rule 4 x 163 [4,8]
+ CRUSH rule 4 x 164 [8]
+ CRUSH rule 4 x 165 [8,2]
+ CRUSH rule 4 x 166 [2]
+ CRUSH rule 4 x 167 [1,2]
+ CRUSH rule 4 x 168 [4,2]
+ CRUSH rule 4 x 169 [2,8]
+ CRUSH rule 4 x 170 [1,2]
+ CRUSH rule 4 x 171 [8,4]
+ CRUSH rule 4 x 172 [1,8]
+ CRUSH rule 4 x 173 [8,4]
+ CRUSH rule 4 x 174 [1]
+ CRUSH rule 4 x 175 [8,1]
+ CRUSH rule 4 x 176 []
+ CRUSH rule 4 x 177 []
+ CRUSH rule 4 x 178 [4,2]
+ CRUSH rule 4 x 179 [1]
+ CRUSH rule 4 x 180 [8]
+ CRUSH rule 4 x 181 [8,2]
+ CRUSH rule 4 x 182 [8]
+ CRUSH rule 4 x 183 [8]
+ CRUSH rule 4 x 184 [4,8]
+ CRUSH rule 4 x 185 [8]
+ CRUSH rule 4 x 186 [2,1]
+ CRUSH rule 4 x 187 [1,8]
+ CRUSH rule 4 x 188 [1,8]
+ CRUSH rule 4 x 189 [1,8]
+ CRUSH rule 4 x 190 [1]
+ CRUSH rule 4 x 191 [8]
+ CRUSH rule 4 x 192 [4,1]
+ CRUSH rule 4 x 193 [4,2]
+ CRUSH rule 4 x 194 [1]
+ CRUSH rule 4 x 195 [8,4]
+ CRUSH rule 4 x 196 [8]
+ CRUSH rule 4 x 197 [8,4]
+ CRUSH rule 4 x 198 [2]
+ CRUSH rule 4 x 199 [1,4]
+ CRUSH rule 4 x 200 [1]
+ CRUSH rule 4 x 201 [8,1]
+ CRUSH rule 4 x 202 [8]
+ CRUSH rule 4 x 203 [8]
+ CRUSH rule 4 x 204 [2,1]
+ CRUSH rule 4 x 205 [1,8]
+ CRUSH rule 4 x 206 [1,2]
+ CRUSH rule 4 x 207 [2]
+ CRUSH rule 4 x 208 [8,1]
+ CRUSH rule 4 x 209 [1,2]
+ CRUSH rule 4 x 210 [1,2]
+ CRUSH rule 4 x 211 [4]
+ CRUSH rule 4 x 212 [8]
+ CRUSH rule 4 x 213 [8,4]
+ CRUSH rule 4 x 214 []
+ CRUSH rule 4 x 215 [8,1]
+ CRUSH rule 4 x 216 [1]
+ CRUSH rule 4 x 217 [1,2]
+ CRUSH rule 4 x 218 [2,8]
+ CRUSH rule 4 x 219 [8]
+ CRUSH rule 4 x 220 [4,8]
+ CRUSH rule 4 x 221 [8]
+ CRUSH rule 4 x 222 [8]
+ CRUSH rule 4 x 223 [1]
+ CRUSH rule 4 x 224 [1,4]
+ CRUSH rule 4 x 225 [8]
+ CRUSH rule 4 x 226 [8,2]
+ CRUSH rule 4 x 227 [4]
+ CRUSH rule 4 x 228 []
+ CRUSH rule 4 x 229 [4]
+ CRUSH rule 4 x 230 [4,8]
+ CRUSH rule 4 x 231 [4]
+ CRUSH rule 4 x 232 [2,8]
+ CRUSH rule 4 x 233 []
+ CRUSH rule 4 x 234 [1,2]
+ CRUSH rule 4 x 235 [4,8]
+ CRUSH rule 4 x 236 [2]
+ CRUSH rule 4 x 237 [4,8]
+ CRUSH rule 4 x 238 []
+ CRUSH rule 4 x 239 [8,6]
+ CRUSH rule 4 x 240 [4,8]
+ CRUSH rule 4 x 241 [1]
+ CRUSH rule 4 x 242 []
+ CRUSH rule 4 x 243 [8]
+ CRUSH rule 4 x 244 [4,8]
+ CRUSH rule 4 x 245 [8]
+ CRUSH rule 4 x 246 [1]
+ CRUSH rule 4 x 247 [8,2]
+ CRUSH rule 4 x 248 [8,2]
+ CRUSH rule 4 x 249 [2,1]
+ CRUSH rule 4 x 250 [2,1]
+ CRUSH rule 4 x 251 [2]
+ CRUSH rule 4 x 252 [4,8]
+ CRUSH rule 4 x 253 [2]
+ CRUSH rule 4 x 254 [4]
+ CRUSH rule 4 x 255 [1,8]
+ CRUSH rule 4 x 256 [4,8]
+ CRUSH rule 4 x 257 [2,8]
+ CRUSH rule 4 x 258 [4]
+ CRUSH rule 4 x 259 []
+ CRUSH rule 4 x 260 [8]
+ CRUSH rule 4 x 261 [8]
+ CRUSH rule 4 x 262 []
+ CRUSH rule 4 x 263 [8]
+ CRUSH rule 4 x 264 [8]
+ CRUSH rule 4 x 265 [8]
+ CRUSH rule 4 x 266 [8,2]
+ CRUSH rule 4 x 267 [2]
+ CRUSH rule 4 x 268 [1,8]
+ CRUSH rule 4 x 269 [1,8]
+ CRUSH rule 4 x 270 [4,1]
+ CRUSH rule 4 x 271 [8,4]
+ CRUSH rule 4 x 272 [2,8]
+ CRUSH rule 4 x 273 [4]
+ CRUSH rule 4 x 274 [8]
+ CRUSH rule 4 x 275 [4]
+ CRUSH rule 4 x 276 [8,1]
+ CRUSH rule 4 x 277 [8]
+ CRUSH rule 4 x 278 [8]
+ CRUSH rule 4 x 279 [8,4]
+ CRUSH rule 4 x 280 [2,8]
+ CRUSH rule 4 x 281 [8,2]
+ CRUSH rule 4 x 282 [1]
+ CRUSH rule 4 x 283 [8,2]
+ CRUSH rule 4 x 284 [8]
+ CRUSH rule 4 x 285 [4]
+ CRUSH rule 4 x 286 [2,1]
+ CRUSH rule 4 x 287 [1]
+ CRUSH rule 4 x 288 [8,1]
+ CRUSH rule 4 x 289 [4,8]
+ CRUSH rule 4 x 290 [1,4]
+ CRUSH rule 4 x 291 [1,2]
+ CRUSH rule 4 x 292 [8,2]
+ CRUSH rule 4 x 293 [8,1]
+ CRUSH rule 4 x 294 [8,4]
+ CRUSH rule 4 x 295 [4,8]
+ CRUSH rule 4 x 296 [4,1]
+ CRUSH rule 4 x 297 [8,2]
+ CRUSH rule 4 x 298 [1,2]
+ CRUSH rule 4 x 299 [2,1]
+ CRUSH rule 4 x 300 [8]
+ CRUSH rule 4 x 301 [1,8]
+ CRUSH rule 4 x 302 [1]
+ CRUSH rule 4 x 303 [8,4]
+ CRUSH rule 4 x 304 [2,8]
+ CRUSH rule 4 x 305 [8]
+ CRUSH rule 4 x 306 [1,8]
+ CRUSH rule 4 x 307 [2,1]
+ CRUSH rule 4 x 308 [2,8]
+ CRUSH rule 4 x 309 [8]
+ CRUSH rule 4 x 310 [4]
+ CRUSH rule 4 x 311 [4]
+ CRUSH rule 4 x 312 [2,1]
+ CRUSH rule 4 x 313 [4]
+ CRUSH rule 4 x 314 []
+ CRUSH rule 4 x 315 [2,1]
+ CRUSH rule 4 x 316 [8]
+ CRUSH rule 4 x 317 [2,8]
+ CRUSH rule 4 x 318 [8]
+ CRUSH rule 4 x 319 [2]
+ CRUSH rule 4 x 320 [8]
+ CRUSH rule 4 x 321 [1]
+ CRUSH rule 4 x 322 [2,8]
+ CRUSH rule 4 x 323 [4,8]
+ CRUSH rule 4 x 324 [8,1]
+ CRUSH rule 4 x 325 [4,8]
+ CRUSH rule 4 x 326 []
+ CRUSH rule 4 x 327 [1,8]
+ CRUSH rule 4 x 328 [8,4]
+ CRUSH rule 4 x 329 [4,8]
+ CRUSH rule 4 x 330 [4,8]
+ CRUSH rule 4 x 331 [2,8]
+ CRUSH rule 4 x 332 [2,1]
+ CRUSH rule 4 x 333 [8]
+ CRUSH rule 4 x 334 [8]
+ CRUSH rule 4 x 335 [8,1]
+ CRUSH rule 4 x 336 [4,8]
+ CRUSH rule 4 x 337 [8,2]
+ CRUSH rule 4 x 338 [8]
+ CRUSH rule 4 x 339 [8]
+ CRUSH rule 4 x 340 [2,1]
+ CRUSH rule 4 x 341 [4,1]
+ CRUSH rule 4 x 342 [2,8]
+ CRUSH rule 4 x 343 [8]
+ CRUSH rule 4 x 344 [6,2]
+ CRUSH rule 4 x 345 []
+ CRUSH rule 4 x 346 [8,2]
+ CRUSH rule 4 x 347 [4,1]
+ CRUSH rule 4 x 348 [8,2]
+ CRUSH rule 4 x 349 [1,8]
+ CRUSH rule 4 x 350 [8]
+ CRUSH rule 4 x 351 [8]
+ CRUSH rule 4 x 352 [1,2]
+ CRUSH rule 4 x 353 [8]
+ CRUSH rule 4 x 354 [1]
+ CRUSH rule 4 x 355 []
+ CRUSH rule 4 x 356 [4]
+ CRUSH rule 4 x 357 [8,1]
+ CRUSH rule 4 x 358 [2,1]
+ CRUSH rule 4 x 359 [6,8]
+ CRUSH rule 4 x 360 []
+ CRUSH rule 4 x 361 [8,4]
+ CRUSH rule 4 x 362 [4]
+ CRUSH rule 4 x 363 [4,1]
+ CRUSH rule 4 x 364 [2]
+ CRUSH rule 4 x 365 [8]
+ CRUSH rule 4 x 366 [8,2]
+ CRUSH rule 4 x 367 [4]
+ CRUSH rule 4 x 368 [8,4]
+ CRUSH rule 4 x 369 [8]
+ CRUSH rule 4 x 370 [8]
+ CRUSH rule 4 x 371 [1,4]
+ CRUSH rule 4 x 372 [1]
+ CRUSH rule 4 x 373 [1,8]
+ CRUSH rule 4 x 374 [8]
+ CRUSH rule 4 x 375 [8,4]
+ CRUSH rule 4 x 376 [8,1]
+ CRUSH rule 4 x 377 [1,2]
+ CRUSH rule 4 x 378 [1,2]
+ CRUSH rule 4 x 379 [8]
+ CRUSH rule 4 x 380 [2]
+ CRUSH rule 4 x 381 [1,4]
+ CRUSH rule 4 x 382 [1,4]
+ CRUSH rule 4 x 383 [4]
+ CRUSH rule 4 x 384 [8,2]
+ CRUSH rule 4 x 385 [8]
+ CRUSH rule 4 x 386 [1]
+ CRUSH rule 4 x 387 [1,4]
+ CRUSH rule 4 x 388 [1]
+ CRUSH rule 4 x 389 [1,4]
+ CRUSH rule 4 x 390 [4,8]
+ CRUSH rule 4 x 391 [4,8]
+ CRUSH rule 4 x 392 [1,8]
+ CRUSH rule 4 x 393 [2]
+ CRUSH rule 4 x 394 [8]
+ CRUSH rule 4 x 395 [1]
+ CRUSH rule 4 x 396 [4,2]
+ CRUSH rule 4 x 397 [2,1]
+ CRUSH rule 4 x 398 [2,4]
+ CRUSH rule 4 x 399 [8]
+ CRUSH rule 4 x 400 [8,1]
+ CRUSH rule 4 x 401 [1,2]
+ CRUSH rule 4 x 402 [8]
+ CRUSH rule 4 x 403 [1,2]
+ CRUSH rule 4 x 404 [4]
+ CRUSH rule 4 x 405 [8,4]
+ CRUSH rule 4 x 406 [2,1]
+ CRUSH rule 4 x 407 [2,8]
+ CRUSH rule 4 x 408 [4,1]
+ CRUSH rule 4 x 409 [8,4]
+ CRUSH rule 4 x 410 [8]
+ CRUSH rule 4 x 411 [2,1]
+ CRUSH rule 4 x 412 [2]
+ CRUSH rule 4 x 413 [2]
+ CRUSH rule 4 x 414 [4,1]
+ CRUSH rule 4 x 415 [2,8]
+ CRUSH rule 4 x 416 [2,1]
+ CRUSH rule 4 x 417 [8,6]
+ CRUSH rule 4 x 418 [8]
+ CRUSH rule 4 x 419 [8,4]
+ CRUSH rule 4 x 420 [1,4]
+ CRUSH rule 4 x 421 [8]
+ CRUSH rule 4 x 422 [6,8]
+ CRUSH rule 4 x 423 [2,4]
+ CRUSH rule 4 x 424 [8]
+ CRUSH rule 4 x 425 [1]
+ CRUSH rule 4 x 426 [8]
+ CRUSH rule 4 x 427 [1,8]
+ CRUSH rule 4 x 428 [4]
+ CRUSH rule 4 x 429 [4,8]
+ CRUSH rule 4 x 430 [4,8]
+ CRUSH rule 4 x 431 [4]
+ CRUSH rule 4 x 432 [8,1]
+ CRUSH rule 4 x 433 [8]
+ CRUSH rule 4 x 434 [2]
+ CRUSH rule 4 x 435 [2]
+ CRUSH rule 4 x 436 [4,1]
+ CRUSH rule 4 x 437 [8]
+ CRUSH rule 4 x 438 [2,4]
+ CRUSH rule 4 x 439 [1]
+ CRUSH rule 4 x 440 [2,8]
+ CRUSH rule 4 x 441 [4,6]
+ CRUSH rule 4 x 442 [2]
+ CRUSH rule 4 x 443 [8]
+ CRUSH rule 4 x 444 [8,1]
+ CRUSH rule 4 x 445 [8]
+ CRUSH rule 4 x 446 []
+ CRUSH rule 4 x 447 [2,1]
+ CRUSH rule 4 x 448 [8,2]
+ CRUSH rule 4 x 449 [8,6]
+ CRUSH rule 4 x 450 []
+ CRUSH rule 4 x 451 [8]
+ CRUSH rule 4 x 452 [8]
+ CRUSH rule 4 x 453 [6,8]
+ CRUSH rule 4 x 454 [8]
+ CRUSH rule 4 x 455 [2,8]
+ CRUSH rule 4 x 456 [8]
+ CRUSH rule 4 x 457 [8,2]
+ CRUSH rule 4 x 458 [2,8]
+ CRUSH rule 4 x 459 [2,1]
+ CRUSH rule 4 x 460 [8]
+ CRUSH rule 4 x 461 [8]
+ CRUSH rule 4 x 462 [8,1]
+ CRUSH rule 4 x 463 [8]
+ CRUSH rule 4 x 464 [8,4]
+ CRUSH rule 4 x 465 [6,8]
+ CRUSH rule 4 x 466 [8]
+ CRUSH rule 4 x 467 [8]
+ CRUSH rule 4 x 468 [8]
+ CRUSH rule 4 x 469 [8,1]
+ CRUSH rule 4 x 470 [4,2]
+ CRUSH rule 4 x 471 [1,2]
+ CRUSH rule 4 x 472 []
+ CRUSH rule 4 x 473 [1,2]
+ CRUSH rule 4 x 474 [8,1]
+ CRUSH rule 4 x 475 [8]
+ CRUSH rule 4 x 476 [4]
+ CRUSH rule 4 x 477 [4,8]
+ CRUSH rule 4 x 478 [8]
+ CRUSH rule 4 x 479 [2]
+ CRUSH rule 4 x 480 [1,8]
+ CRUSH rule 4 x 481 [2,4]
+ CRUSH rule 4 x 482 []
+ CRUSH rule 4 x 483 [2,1]
+ CRUSH rule 4 x 484 [1,2]
+ CRUSH rule 4 x 485 [8]
+ CRUSH rule 4 x 486 [4,1]
+ CRUSH rule 4 x 487 [1]
+ CRUSH rule 4 x 488 [8]
+ CRUSH rule 4 x 489 [2,8]
+ CRUSH rule 4 x 490 [6]
+ CRUSH rule 4 x 491 [1,2]
+ CRUSH rule 4 x 492 [8]
+ CRUSH rule 4 x 493 [2,1]
+ CRUSH rule 4 x 494 [1,2]
+ CRUSH rule 4 x 495 [4]
+ CRUSH rule 4 x 496 [8,4]
+ CRUSH rule 4 x 497 [4,8]
+ CRUSH rule 4 x 498 [2,4]
+ CRUSH rule 4 x 499 [8,4]
+ CRUSH rule 4 x 500 [4,8]
+ CRUSH rule 4 x 501 [2,8]
+ CRUSH rule 4 x 502 [6,1]
+ CRUSH rule 4 x 503 [2]
+ CRUSH rule 4 x 504 [8]
+ CRUSH rule 4 x 505 [1,8]
+ CRUSH rule 4 x 506 [4]
+ CRUSH rule 4 x 507 [8,1]
+ CRUSH rule 4 x 508 [1,2]
+ CRUSH rule 4 x 509 [8]
+ CRUSH rule 4 x 510 [8,2]
+ CRUSH rule 4 x 511 [4,8]
+ CRUSH rule 4 x 512 [8]
+ CRUSH rule 4 x 513 [8,2]
+ CRUSH rule 4 x 514 []
+ CRUSH rule 4 x 515 [8,4]
+ CRUSH rule 4 x 516 [4,1]
+ CRUSH rule 4 x 517 [8]
+ CRUSH rule 4 x 518 [4,8]
+ CRUSH rule 4 x 519 [8,4]
+ CRUSH rule 4 x 520 [2,8]
+ CRUSH rule 4 x 521 [8]
+ CRUSH rule 4 x 522 [8]
+ CRUSH rule 4 x 523 [4,2]
+ CRUSH rule 4 x 524 [2]
+ CRUSH rule 4 x 525 [2]
+ CRUSH rule 4 x 526 [1]
+ CRUSH rule 4 x 527 [1,2]
+ CRUSH rule 4 x 528 []
+ CRUSH rule 4 x 529 [4,8]
+ CRUSH rule 4 x 530 [8]
+ CRUSH rule 4 x 531 [8,1]
+ CRUSH rule 4 x 532 [6,4]
+ CRUSH rule 4 x 533 [4,8]
+ CRUSH rule 4 x 534 [8]
+ CRUSH rule 4 x 535 [8,6]
+ CRUSH rule 4 x 536 [8]
+ CRUSH rule 4 x 537 [4,8]
+ CRUSH rule 4 x 538 [8]
+ CRUSH rule 4 x 539 [8]
+ CRUSH rule 4 x 540 [1,8]
+ CRUSH rule 4 x 541 [2,4]
+ CRUSH rule 4 x 542 []
+ CRUSH rule 4 x 543 [8,2]
+ CRUSH rule 4 x 544 [4,8]
+ CRUSH rule 4 x 545 [8]
+ CRUSH rule 4 x 546 [8,1]
+ CRUSH rule 4 x 547 [8,2]
+ CRUSH rule 4 x 548 [4,2]
+ CRUSH rule 4 x 549 [8]
+ CRUSH rule 4 x 550 [2,4]
+ CRUSH rule 4 x 551 [8]
+ CRUSH rule 4 x 552 [4]
+ CRUSH rule 4 x 553 [2]
+ CRUSH rule 4 x 554 [1,8]
+ CRUSH rule 4 x 555 [4,1]
+ CRUSH rule 4 x 556 []
+ CRUSH rule 4 x 557 [8]
+ CRUSH rule 4 x 558 [4,1]
+ CRUSH rule 4 x 559 [2]
+ CRUSH rule 4 x 560 [8]
+ CRUSH rule 4 x 561 [8,4]
+ CRUSH rule 4 x 562 [4]
+ CRUSH rule 4 x 563 [2,8]
+ CRUSH rule 4 x 564 [1]
+ CRUSH rule 4 x 565 [4,8]
+ CRUSH rule 4 x 566 [4,8]
+ CRUSH rule 4 x 567 [4,8]
+ CRUSH rule 4 x 568 [8]
+ CRUSH rule 4 x 569 [4]
+ CRUSH rule 4 x 570 [1]
+ CRUSH rule 4 x 571 [8]
+ CRUSH rule 4 x 572 [4]
+ CRUSH rule 4 x 573 [1]
+ CRUSH rule 4 x 574 [2,1]
+ CRUSH rule 4 x 575 [8]
+ CRUSH rule 4 x 576 [4,8]
+ CRUSH rule 4 x 577 [8,2]
+ CRUSH rule 4 x 578 [8]
+ CRUSH rule 4 x 579 [4,1]
+ CRUSH rule 4 x 580 [2]
+ CRUSH rule 4 x 581 [8,2]
+ CRUSH rule 4 x 582 [2,8]
+ CRUSH rule 4 x 583 [8,1]
+ CRUSH rule 4 x 584 [8,1]
+ CRUSH rule 4 x 585 [8,1]
+ CRUSH rule 4 x 586 [1,2]
+ CRUSH rule 4 x 587 [2,4]
+ CRUSH rule 4 x 588 [4]
+ CRUSH rule 4 x 589 [8,1]
+ CRUSH rule 4 x 590 [8,2]
+ CRUSH rule 4 x 591 [4,2]
+ CRUSH rule 4 x 592 [2,1]
+ CRUSH rule 4 x 593 [1,8]
+ CRUSH rule 4 x 594 [2,8]
+ CRUSH rule 4 x 595 [8,1]
+ CRUSH rule 4 x 596 []
+ CRUSH rule 4 x 597 [1]
+ CRUSH rule 4 x 598 [2]
+ CRUSH rule 4 x 599 [4,2]
+ CRUSH rule 4 x 600 [8,1]
+ CRUSH rule 4 x 601 [1,8]
+ CRUSH rule 4 x 602 [8]
+ CRUSH rule 4 x 603 [1]
+ CRUSH rule 4 x 604 [8]
+ CRUSH rule 4 x 605 [2]
+ CRUSH rule 4 x 606 [2,1]
+ CRUSH rule 4 x 607 [2,4]
+ CRUSH rule 4 x 608 [4]
+ CRUSH rule 4 x 609 [4,2]
+ CRUSH rule 4 x 610 [8]
+ CRUSH rule 4 x 611 [1,2]
+ CRUSH rule 4 x 612 [2,1]
+ CRUSH rule 4 x 613 [8,2]
+ CRUSH rule 4 x 614 [8]
+ CRUSH rule 4 x 615 [8]
+ CRUSH rule 4 x 616 [1,8]
+ CRUSH rule 4 x 617 [8,1]
+ CRUSH rule 4 x 618 [8]
+ CRUSH rule 4 x 619 [4,1]
+ CRUSH rule 4 x 620 [1]
+ CRUSH rule 4 x 621 [8]
+ CRUSH rule 4 x 622 [2,4]
+ CRUSH rule 4 x 623 [2,8]
+ CRUSH rule 4 x 624 [4]
+ CRUSH rule 4 x 625 [2]
+ CRUSH rule 4 x 626 [8]
+ CRUSH rule 4 x 627 [2,8]
+ CRUSH rule 4 x 628 [8,2]
+ CRUSH rule 4 x 629 [2,8]
+ CRUSH rule 4 x 630 [2,8]
+ CRUSH rule 4 x 631 [1,8]
+ CRUSH rule 4 x 632 [8,2]
+ CRUSH rule 4 x 633 [8]
+ CRUSH rule 4 x 634 [1]
+ CRUSH rule 4 x 635 [4,8]
+ CRUSH rule 4 x 636 [1,4]
+ CRUSH rule 4 x 637 [1]
+ CRUSH rule 4 x 638 [8]
+ CRUSH rule 4 x 639 [2]
+ CRUSH rule 4 x 640 [1]
+ CRUSH rule 4 x 641 [8,2]
+ CRUSH rule 4 x 642 [2,1]
+ CRUSH rule 4 x 643 []
+ CRUSH rule 4 x 644 [8,1]
+ CRUSH rule 4 x 645 []
+ CRUSH rule 4 x 646 [8,1]
+ CRUSH rule 4 x 647 [8,1]
+ CRUSH rule 4 x 648 [1,8]
+ CRUSH rule 4 x 649 [4,8]
+ CRUSH rule 4 x 650 [8]
+ CRUSH rule 4 x 651 [4,6]
+ CRUSH rule 4 x 652 [4]
+ CRUSH rule 4 x 653 [8]
+ CRUSH rule 4 x 654 [6]
+ CRUSH rule 4 x 655 [1,4]
+ CRUSH rule 4 x 656 []
+ CRUSH rule 4 x 657 [6,1]
+ CRUSH rule 4 x 658 []
+ CRUSH rule 4 x 659 [4,8]
+ CRUSH rule 4 x 660 [8]
+ CRUSH rule 4 x 661 [1,8]
+ CRUSH rule 4 x 662 []
+ CRUSH rule 4 x 663 [1,4]
+ CRUSH rule 4 x 664 [1,4]
+ CRUSH rule 4 x 665 [4,6]
+ CRUSH rule 4 x 666 [2,8]
+ CRUSH rule 4 x 667 [1,4]
+ CRUSH rule 4 x 668 [4,8]
+ CRUSH rule 4 x 669 [6,4]
+ CRUSH rule 4 x 670 [4,2]
+ CRUSH rule 4 x 671 [2,1]
+ CRUSH rule 4 x 672 [4]
+ CRUSH rule 4 x 673 [4]
+ CRUSH rule 4 x 674 [1]
+ CRUSH rule 4 x 675 [1,8]
+ CRUSH rule 4 x 676 [2,1]
+ CRUSH rule 4 x 677 [4,1]
+ CRUSH rule 4 x 678 [2,4]
+ CRUSH rule 4 x 679 [8,2]
+ CRUSH rule 4 x 680 [2]
+ CRUSH rule 4 x 681 [8]
+ CRUSH rule 4 x 682 [1,4]
+ CRUSH rule 4 x 683 [1,2]
+ CRUSH rule 4 x 684 [8,1]
+ CRUSH rule 4 x 685 [8,1]
+ CRUSH rule 4 x 686 [1,4]
+ CRUSH rule 4 x 687 []
+ CRUSH rule 4 x 688 [4,8]
+ CRUSH rule 4 x 689 [8,4]
+ CRUSH rule 4 x 690 [8,1]
+ CRUSH rule 4 x 691 [1]
+ CRUSH rule 4 x 692 [8,2]
+ CRUSH rule 4 x 693 [8]
+ CRUSH rule 4 x 694 [8,4]
+ CRUSH rule 4 x 695 [2,8]
+ CRUSH rule 4 x 696 [1]
+ CRUSH rule 4 x 697 [8,1]
+ CRUSH rule 4 x 698 [8,2]
+ CRUSH rule 4 x 699 [1,8]
+ CRUSH rule 4 x 700 [1]
+ CRUSH rule 4 x 701 []
+ CRUSH rule 4 x 702 []
+ CRUSH rule 4 x 703 [8]
+ CRUSH rule 4 x 704 [1,4]
+ CRUSH rule 4 x 705 [8]
+ CRUSH rule 4 x 706 [1,2]
+ CRUSH rule 4 x 707 [8]
+ CRUSH rule 4 x 708 [4]
+ CRUSH rule 4 x 709 [8]
+ CRUSH rule 4 x 710 [8]
+ CRUSH rule 4 x 711 [2,4]
+ CRUSH rule 4 x 712 [2]
+ CRUSH rule 4 x 713 [8]
+ CRUSH rule 4 x 714 [2]
+ CRUSH rule 4 x 715 [1,2]
+ CRUSH rule 4 x 716 [4,8]
+ CRUSH rule 4 x 717 [8]
+ CRUSH rule 4 x 718 [8]
+ CRUSH rule 4 x 719 [2,6]
+ CRUSH rule 4 x 720 [8]
+ CRUSH rule 4 x 721 [4]
+ CRUSH rule 4 x 722 []
+ CRUSH rule 4 x 723 [4,1]
+ CRUSH rule 4 x 724 [2,6]
+ CRUSH rule 4 x 725 [1,2]
+ CRUSH rule 4 x 726 [4,8]
+ CRUSH rule 4 x 727 [4,8]
+ CRUSH rule 4 x 728 [2,1]
+ CRUSH rule 4 x 729 []
+ CRUSH rule 4 x 730 [4,8]
+ CRUSH rule 4 x 731 [4,1]
+ CRUSH rule 4 x 732 [1]
+ CRUSH rule 4 x 733 [4]
+ CRUSH rule 4 x 734 [8,4]
+ CRUSH rule 4 x 735 [4,8]
+ CRUSH rule 4 x 736 [4]
+ CRUSH rule 4 x 737 [1,2]
+ CRUSH rule 4 x 738 [4,2]
+ CRUSH rule 4 x 739 [2,1]
+ CRUSH rule 4 x 740 [1,2]
+ CRUSH rule 4 x 741 [8]
+ CRUSH rule 4 x 742 [8,2]
+ CRUSH rule 4 x 743 [8,1]
+ CRUSH rule 4 x 744 [4,8]
+ CRUSH rule 4 x 745 []
+ CRUSH rule 4 x 746 [1]
+ CRUSH rule 4 x 747 [8,1]
+ CRUSH rule 4 x 748 [2,8]
+ CRUSH rule 4 x 749 [4]
+ CRUSH rule 4 x 750 [1,8]
+ CRUSH rule 4 x 751 [2,1]
+ CRUSH rule 4 x 752 [8,1]
+ CRUSH rule 4 x 753 [8]
+ CRUSH rule 4 x 754 [8]
+ CRUSH rule 4 x 755 [1,2]
+ CRUSH rule 4 x 756 [8]
+ CRUSH rule 4 x 757 [8]
+ CRUSH rule 4 x 758 [8,2]
+ CRUSH rule 4 x 759 [8,4]
+ CRUSH rule 4 x 760 [1,4]
+ CRUSH rule 4 x 761 [1]
+ CRUSH rule 4 x 762 [2,8]
+ CRUSH rule 4 x 763 [8]
+ CRUSH rule 4 x 764 [1,8]
+ CRUSH rule 4 x 765 [8]
+ CRUSH rule 4 x 766 [8]
+ CRUSH rule 4 x 767 [1,2]
+ CRUSH rule 4 x 768 [8,4]
+ CRUSH rule 4 x 769 [8,2]
+ CRUSH rule 4 x 770 [8,2]
+ CRUSH rule 4 x 771 [8,1]
+ CRUSH rule 4 x 772 [8,4]
+ CRUSH rule 4 x 773 [4,1]
+ CRUSH rule 4 x 774 [8]
+ CRUSH rule 4 x 775 [8]
+ CRUSH rule 4 x 776 [6,2]
+ CRUSH rule 4 x 777 [4,1]
+ CRUSH rule 4 x 778 [1,8]
+ CRUSH rule 4 x 779 [2,8]
+ CRUSH rule 4 x 780 [2,1]
+ CRUSH rule 4 x 781 [8]
+ CRUSH rule 4 x 782 [4]
+ CRUSH rule 4 x 783 [8,1]
+ CRUSH rule 4 x 784 [1,2]
+ CRUSH rule 4 x 785 [8,1]
+ CRUSH rule 4 x 786 [8]
+ CRUSH rule 4 x 787 [1,2]
+ CRUSH rule 4 x 788 [8,2]
+ CRUSH rule 4 x 789 [1]
+ CRUSH rule 4 x 790 [8]
+ CRUSH rule 4 x 791 [4,8]
+ CRUSH rule 4 x 792 [4,8]
+ CRUSH rule 4 x 793 [8,1]
+ CRUSH rule 4 x 794 [2,8]
+ CRUSH rule 4 x 795 [1]
+ CRUSH rule 4 x 796 []
+ CRUSH rule 4 x 797 [2,4]
+ CRUSH rule 4 x 798 [6,8]
+ CRUSH rule 4 x 799 [4,1]
+ CRUSH rule 4 x 800 [2]
+ CRUSH rule 4 x 801 [4,8]
+ CRUSH rule 4 x 802 [1,8]
+ CRUSH rule 4 x 803 [2]
+ CRUSH rule 4 x 804 [8,2]
+ CRUSH rule 4 x 805 [8]
+ CRUSH rule 4 x 806 [1,4]
+ CRUSH rule 4 x 807 [4]
+ CRUSH rule 4 x 808 [8]
+ CRUSH rule 4 x 809 [1]
+ CRUSH rule 4 x 810 [8]
+ CRUSH rule 4 x 811 [8]
+ CRUSH rule 4 x 812 [8,4]
+ CRUSH rule 4 x 813 [8,4]
+ CRUSH rule 4 x 814 [8]
+ CRUSH rule 4 x 815 [4,1]
+ CRUSH rule 4 x 816 [2,1]
+ CRUSH rule 4 x 817 []
+ CRUSH rule 4 x 818 []
+ CRUSH rule 4 x 819 [1]
+ CRUSH rule 4 x 820 [4]
+ CRUSH rule 4 x 821 [4]
+ CRUSH rule 4 x 822 [2,1]
+ CRUSH rule 4 x 823 [4,8]
+ CRUSH rule 4 x 824 [8]
+ CRUSH rule 4 x 825 [2,8]
+ CRUSH rule 4 x 826 [8,2]
+ CRUSH rule 4 x 827 [2,8]
+ CRUSH rule 4 x 828 [2]
+ CRUSH rule 4 x 829 [8]
+ CRUSH rule 4 x 830 [2,4]
+ CRUSH rule 4 x 831 [1,8]
+ CRUSH rule 4 x 832 [4]
+ CRUSH rule 4 x 833 [2,1]
+ CRUSH rule 4 x 834 []
+ CRUSH rule 4 x 835 [8,4]
+ CRUSH rule 4 x 836 [4]
+ CRUSH rule 4 x 837 [8,4]
+ CRUSH rule 4 x 838 [6,8]
+ CRUSH rule 4 x 839 [2]
+ CRUSH rule 4 x 840 [8]
+ CRUSH rule 4 x 841 [4,8]
+ CRUSH rule 4 x 842 [2]
+ CRUSH rule 4 x 843 [8,4]
+ CRUSH rule 4 x 844 [8]
+ CRUSH rule 4 x 845 [4,8]
+ CRUSH rule 4 x 846 [4,2]
+ CRUSH rule 4 x 847 [2,1]
+ CRUSH rule 4 x 848 [2,8]
+ CRUSH rule 4 x 849 [4]
+ CRUSH rule 4 x 850 [1,2]
+ CRUSH rule 4 x 851 [6,8]
+ CRUSH rule 4 x 852 [8,4]
+ CRUSH rule 4 x 853 [6,8]
+ CRUSH rule 4 x 854 [8]
+ CRUSH rule 4 x 855 [8]
+ CRUSH rule 4 x 856 [8]
+ CRUSH rule 4 x 857 [8]
+ CRUSH rule 4 x 858 [6]
+ CRUSH rule 4 x 859 [8,2]
+ CRUSH rule 4 x 860 [1]
+ CRUSH rule 4 x 861 [8]
+ CRUSH rule 4 x 862 [8,1]
+ CRUSH rule 4 x 863 [8]
+ CRUSH rule 4 x 864 [8]
+ CRUSH rule 4 x 865 [8,1]
+ CRUSH rule 4 x 866 []
+ CRUSH rule 4 x 867 [8]
+ CRUSH rule 4 x 868 [8]
+ CRUSH rule 4 x 869 [8,6]
+ CRUSH rule 4 x 870 [2]
+ CRUSH rule 4 x 871 []
+ CRUSH rule 4 x 872 [1]
+ CRUSH rule 4 x 873 [4,8]
+ CRUSH rule 4 x 874 [2,6]
+ CRUSH rule 4 x 875 [2,8]
+ CRUSH rule 4 x 876 [4,8]
+ CRUSH rule 4 x 877 [8,4]
+ CRUSH rule 4 x 878 []
+ CRUSH rule 4 x 879 [8]
+ CRUSH rule 4 x 880 []
+ CRUSH rule 4 x 881 [4,8]
+ CRUSH rule 4 x 882 [2]
+ CRUSH rule 4 x 883 [2,1]
+ CRUSH rule 4 x 884 [8,2]
+ CRUSH rule 4 x 885 [4,1]
+ CRUSH rule 4 x 886 [8]
+ CRUSH rule 4 x 887 [8,4]
+ CRUSH rule 4 x 888 [8]
+ CRUSH rule 4 x 889 [2,1]
+ CRUSH rule 4 x 890 [8,2]
+ CRUSH rule 4 x 891 [1,8]
+ CRUSH rule 4 x 892 [8,2]
+ CRUSH rule 4 x 893 [2]
+ CRUSH rule 4 x 894 [8,4]
+ CRUSH rule 4 x 895 [4]
+ CRUSH rule 4 x 896 [1,8]
+ CRUSH rule 4 x 897 [2]
+ CRUSH rule 4 x 898 [1,4]
+ CRUSH rule 4 x 899 [1,8]
+ CRUSH rule 4 x 900 [4,1]
+ CRUSH rule 4 x 901 [1]
+ CRUSH rule 4 x 902 [8,4]
+ CRUSH rule 4 x 903 [8]
+ CRUSH rule 4 x 904 [8]
+ CRUSH rule 4 x 905 [8,2]
+ CRUSH rule 4 x 906 [1,2]
+ CRUSH rule 4 x 907 [8,1]
+ CRUSH rule 4 x 908 [8]
+ CRUSH rule 4 x 909 [2]
+ CRUSH rule 4 x 910 [8]
+ CRUSH rule 4 x 911 [8]
+ CRUSH rule 4 x 912 [1,2]
+ CRUSH rule 4 x 913 [8]
+ CRUSH rule 4 x 914 [6,4]
+ CRUSH rule 4 x 915 [8,2]
+ CRUSH rule 4 x 916 [4,1]
+ CRUSH rule 4 x 917 [1,4]
+ CRUSH rule 4 x 918 [8,2]
+ CRUSH rule 4 x 919 [8,2]
+ CRUSH rule 4 x 920 [8]
+ CRUSH rule 4 x 921 [1]
+ CRUSH rule 4 x 922 [8]
+ CRUSH rule 4 x 923 [4]
+ CRUSH rule 4 x 924 []
+ CRUSH rule 4 x 925 [4,8]
+ CRUSH rule 4 x 926 []
+ CRUSH rule 4 x 927 [1,8]
+ CRUSH rule 4 x 928 [8,1]
+ CRUSH rule 4 x 929 [4]
+ CRUSH rule 4 x 930 [2]
+ CRUSH rule 4 x 931 [1]
+ CRUSH rule 4 x 932 [4]
+ CRUSH rule 4 x 933 [8,4]
+ CRUSH rule 4 x 934 []
+ CRUSH rule 4 x 935 [8]
+ CRUSH rule 4 x 936 [1,8]
+ CRUSH rule 4 x 937 [4]
+ CRUSH rule 4 x 938 [8,4]
+ CRUSH rule 4 x 939 [2,8]
+ CRUSH rule 4 x 940 [8]
+ CRUSH rule 4 x 941 [2]
+ CRUSH rule 4 x 942 [1,2]
+ CRUSH rule 4 x 943 [8,2]
+ CRUSH rule 4 x 944 []
+ CRUSH rule 4 x 945 [8,2]
+ CRUSH rule 4 x 946 [2,1]
+ CRUSH rule 4 x 947 []
+ CRUSH rule 4 x 948 [8]
+ CRUSH rule 4 x 949 [6,1]
+ CRUSH rule 4 x 950 []
+ CRUSH rule 4 x 951 []
+ CRUSH rule 4 x 952 [2,1]
+ CRUSH rule 4 x 953 [1,4]
+ CRUSH rule 4 x 954 [2]
+ CRUSH rule 4 x 955 [8]
+ CRUSH rule 4 x 956 [1,2]
+ CRUSH rule 4 x 957 [8]
+ CRUSH rule 4 x 958 [8]
+ CRUSH rule 4 x 959 [4,2]
+ CRUSH rule 4 x 960 [6]
+ CRUSH rule 4 x 961 [1]
+ CRUSH rule 4 x 962 [8,4]
+ CRUSH rule 4 x 963 [2,4]
+ CRUSH rule 4 x 964 [1]
+ CRUSH rule 4 x 965 [8]
+ CRUSH rule 4 x 966 [4,8]
+ CRUSH rule 4 x 967 [8]
+ CRUSH rule 4 x 968 [8,2]
+ CRUSH rule 4 x 969 [8,2]
+ CRUSH rule 4 x 970 [2,8]
+ CRUSH rule 4 x 971 [1,8]
+ CRUSH rule 4 x 972 [1,8]
+ CRUSH rule 4 x 973 [1,2]
+ CRUSH rule 4 x 974 [4]
+ CRUSH rule 4 x 975 [4,8]
+ CRUSH rule 4 x 976 [4]
+ CRUSH rule 4 x 977 [8,4]
+ CRUSH rule 4 x 978 [8,2]
+ CRUSH rule 4 x 979 [8]
+ CRUSH rule 4 x 980 [8,2]
+ CRUSH rule 4 x 981 [8]
+ CRUSH rule 4 x 982 [2]
+ CRUSH rule 4 x 983 [4]
+ CRUSH rule 4 x 984 [2,1]
+ CRUSH rule 4 x 985 [2,4]
+ CRUSH rule 4 x 986 [8]
+ CRUSH rule 4 x 987 [2]
+ CRUSH rule 4 x 988 [1,4]
+ CRUSH rule 4 x 989 [1,8]
+ CRUSH rule 4 x 990 [1,2]
+ CRUSH rule 4 x 991 [1,4]
+ CRUSH rule 4 x 992 [8,1]
+ CRUSH rule 4 x 993 [2,8]
+ CRUSH rule 4 x 994 [4]
+ CRUSH rule 4 x 995 [8]
+ CRUSH rule 4 x 996 [8]
+ CRUSH rule 4 x 997 [8,4]
+ CRUSH rule 4 x 998 [8,1]
+ CRUSH rule 4 x 999 [1,8]
+ CRUSH rule 4 x 1000 [8,4]
+ CRUSH rule 4 x 1001 [2,1]
+ CRUSH rule 4 x 1002 [1]
+ CRUSH rule 4 x 1003 [2,8]
+ CRUSH rule 4 x 1004 [8,1]
+ CRUSH rule 4 x 1005 [8,1]
+ CRUSH rule 4 x 1006 [1,2]
+ CRUSH rule 4 x 1007 [1,2]
+ CRUSH rule 4 x 1008 [1,8]
+ CRUSH rule 4 x 1009 [6,8]
+ CRUSH rule 4 x 1010 []
+ CRUSH rule 4 x 1011 [4,2]
+ CRUSH rule 4 x 1012 [2]
+ CRUSH rule 4 x 1013 [1]
+ CRUSH rule 4 x 1014 [2,8]
+ CRUSH rule 4 x 1015 [8]
+ CRUSH rule 4 x 1016 [2,1]
+ CRUSH rule 4 x 1017 [6,2]
+ CRUSH rule 4 x 1018 [4]
+ CRUSH rule 4 x 1019 [4]
+ CRUSH rule 4 x 1020 [1]
+ CRUSH rule 4 x 1021 [2]
+ CRUSH rule 4 x 1022 [1,8]
+ CRUSH rule 4 x 1023 [4,2]
+ rule 4 (choose-set-two) num_rep 2 result size == 0:\t56/1024 (esc)
+ rule 4 (choose-set-two) num_rep 2 result size == 1:\t397/1024 (esc)
+ rule 4 (choose-set-two) num_rep 2 result size == 2:\t571/1024 (esc)
+ CRUSH rule 4 x 0 [2,1,4]
+ CRUSH rule 4 x 1 [2,8,1]
+ CRUSH rule 4 x 2 [1]
+ CRUSH rule 4 x 3 [8,1]
+ CRUSH rule 4 x 4 [4,2]
+ CRUSH rule 4 x 5 [8,1]
+ CRUSH rule 4 x 6 [2,8]
+ CRUSH rule 4 x 7 [4]
+ CRUSH rule 4 x 8 [4,8]
+ CRUSH rule 4 x 9 [2,4,1]
+ CRUSH rule 4 x 10 [2,1,8]
+ CRUSH rule 4 x 11 [2,8,1]
+ CRUSH rule 4 x 12 [2,1]
+ CRUSH rule 4 x 13 [4,8]
+ CRUSH rule 4 x 14 [8]
+ CRUSH rule 4 x 15 [8,2]
+ CRUSH rule 4 x 16 [8,2]
+ CRUSH rule 4 x 17 [4,2]
+ CRUSH rule 4 x 18 [1]
+ CRUSH rule 4 x 19 [8,4]
+ CRUSH rule 4 x 20 [2,8]
+ CRUSH rule 4 x 21 [8,2]
+ CRUSH rule 4 x 22 [8]
+ CRUSH rule 4 x 23 [4,8]
+ CRUSH rule 4 x 24 [1,2,8]
+ CRUSH rule 4 x 25 [4,8,1]
+ CRUSH rule 4 x 26 [2,8]
+ CRUSH rule 4 x 27 [4,1]
+ CRUSH rule 4 x 28 [8,2]
+ CRUSH rule 4 x 29 [8,4]
+ CRUSH rule 4 x 30 [4,8]
+ CRUSH rule 4 x 31 [8]
+ CRUSH rule 4 x 32 [6,8]
+ CRUSH rule 4 x 33 [2,8]
+ CRUSH rule 4 x 34 [2]
+ CRUSH rule 4 x 35 [1,8,2]
+ CRUSH rule 4 x 36 [8,2]
+ CRUSH rule 4 x 37 [1]
+ CRUSH rule 4 x 38 [4,8]
+ CRUSH rule 4 x 39 [8,1]
+ CRUSH rule 4 x 40 [8,1]
+ CRUSH rule 4 x 41 [2,1,8]
+ CRUSH rule 4 x 42 [8]
+ CRUSH rule 4 x 43 [1,8]
+ CRUSH rule 4 x 44 [1,8]
+ CRUSH rule 4 x 45 [8,2,4]
+ CRUSH rule 4 x 46 [2]
+ CRUSH rule 4 x 47 [4]
+ CRUSH rule 4 x 48 [8]
+ CRUSH rule 4 x 49 [8]
+ CRUSH rule 4 x 50 [4,1]
+ CRUSH rule 4 x 51 [8,2]
+ CRUSH rule 4 x 52 [8,1]
+ CRUSH rule 4 x 53 [4]
+ CRUSH rule 4 x 54 [8]
+ CRUSH rule 4 x 55 [8,2]
+ CRUSH rule 4 x 56 [8,4]
+ CRUSH rule 4 x 57 [8]
+ CRUSH rule 4 x 58 [1,2]
+ CRUSH rule 4 x 59 [2,1]
+ CRUSH rule 4 x 60 [4,1]
+ CRUSH rule 4 x 61 [4,8]
+ CRUSH rule 4 x 62 [8,1,2]
+ CRUSH rule 4 x 63 [8]
+ CRUSH rule 4 x 64 [4,1]
+ CRUSH rule 4 x 65 [8,4]
+ CRUSH rule 4 x 66 [4]
+ CRUSH rule 4 x 67 [4,2,8]
+ CRUSH rule 4 x 68 [1,8]
+ CRUSH rule 4 x 69 [1,8]
+ CRUSH rule 4 x 70 [8,2,1]
+ CRUSH rule 4 x 71 [2,8]
+ CRUSH rule 4 x 72 [8,1,4]
+ CRUSH rule 4 x 73 [2,8]
+ CRUSH rule 4 x 74 [1,8,2]
+ CRUSH rule 4 x 75 [4,2,1]
+ CRUSH rule 4 x 76 [4,1,6]
+ CRUSH rule 4 x 77 [8,2]
+ CRUSH rule 4 x 78 [1,2]
+ CRUSH rule 4 x 79 [4,1]
+ CRUSH rule 4 x 80 [2,4]
+ CRUSH rule 4 x 81 [2,1]
+ CRUSH rule 4 x 82 [6,1,8]
+ CRUSH rule 4 x 83 [2,8]
+ CRUSH rule 4 x 84 [8,2,1]
+ CRUSH rule 4 x 85 [4,8]
+ CRUSH rule 4 x 86 [2,1]
+ CRUSH rule 4 x 87 [2,8,4]
+ CRUSH rule 4 x 88 [1,6,2]
+ CRUSH rule 4 x 89 [2,1]
+ CRUSH rule 4 x 90 [8,4]
+ CRUSH rule 4 x 91 [4,8]
+ CRUSH rule 4 x 92 [1,8]
+ CRUSH rule 4 x 93 [8,4]
+ CRUSH rule 4 x 94 [1]
+ CRUSH rule 4 x 95 [8,1]
+ CRUSH rule 4 x 96 [8]
+ CRUSH rule 4 x 97 [8]
+ CRUSH rule 4 x 98 [2,1,8]
+ CRUSH rule 4 x 99 [2,8,1]
+ CRUSH rule 4 x 100 [1,8,4]
+ CRUSH rule 4 x 101 [8,2]
+ CRUSH rule 4 x 102 [2,8]
+ CRUSH rule 4 x 103 [8]
+ CRUSH rule 4 x 104 [8,4]
+ CRUSH rule 4 x 105 [2,4,1]
+ CRUSH rule 4 x 106 [1,8,2]
+ CRUSH rule 4 x 107 [2,1]
+ CRUSH rule 4 x 108 [8,2]
+ CRUSH rule 4 x 109 [1,2,4]
+ CRUSH rule 4 x 110 [4,2,8]
+ CRUSH rule 4 x 111 [2,1]
+ CRUSH rule 4 x 112 [2,1,8]
+ CRUSH rule 4 x 113 [8,2,1]
+ CRUSH rule 4 x 114 [8,4]
+ CRUSH rule 4 x 115 [8,2,4]
+ CRUSH rule 4 x 116 [1,2,8]
+ CRUSH rule 4 x 117 [6,8]
+ CRUSH rule 4 x 118 [2,8]
+ CRUSH rule 4 x 119 [8,1]
+ CRUSH rule 4 x 120 [2,1,4]
+ CRUSH rule 4 x 121 [2,1]
+ CRUSH rule 4 x 122 [8,2]
+ CRUSH rule 4 x 123 [2,1]
+ CRUSH rule 4 x 124 [1]
+ CRUSH rule 4 x 125 [1,8,4]
+ CRUSH rule 4 x 126 [2,8]
+ CRUSH rule 4 x 127 [4,8]
+ CRUSH rule 4 x 128 [8]
+ CRUSH rule 4 x 129 [2,1]
+ CRUSH rule 4 x 130 [4,8]
+ CRUSH rule 4 x 131 [1,2]
+ CRUSH rule 4 x 132 [1,2]
+ CRUSH rule 4 x 133 [8]
+ CRUSH rule 4 x 134 [1,8,2]
+ CRUSH rule 4 x 135 [4,8]
+ CRUSH rule 4 x 136 [2,1,4]
+ CRUSH rule 4 x 137 [8,4]
+ CRUSH rule 4 x 138 [8]
+ CRUSH rule 4 x 139 [4,2]
+ CRUSH rule 4 x 140 [1,8]
+ CRUSH rule 4 x 141 [8,1]
+ CRUSH rule 4 x 142 [4,1]
+ CRUSH rule 4 x 143 [4,8]
+ CRUSH rule 4 x 144 [8,1,2]
+ CRUSH rule 4 x 145 [8]
+ CRUSH rule 4 x 146 [2,8]
+ CRUSH rule 4 x 147 [2,8]
+ CRUSH rule 4 x 148 [4,1]
+ CRUSH rule 4 x 149 [4,8]
+ CRUSH rule 4 x 150 [1,8]
+ CRUSH rule 4 x 151 []
+ CRUSH rule 4 x 152 [8]
+ CRUSH rule 4 x 153 [8]
+ CRUSH rule 4 x 154 [4,2,1]
+ CRUSH rule 4 x 155 [4,8]
+ CRUSH rule 4 x 156 [4]
+ CRUSH rule 4 x 157 [1,2]
+ CRUSH rule 4 x 158 [2,8,1]
+ CRUSH rule 4 x 159 [8,2,4]
+ CRUSH rule 4 x 160 [2,8,1]
+ CRUSH rule 4 x 161 [1,4,2]
+ CRUSH rule 4 x 162 [1,8,2]
+ CRUSH rule 4 x 163 [4,8,1]
+ CRUSH rule 4 x 164 [8,1]
+ CRUSH rule 4 x 165 [8,2,4]
+ CRUSH rule 4 x 166 [2,1]
+ CRUSH rule 4 x 167 [1,2,8]
+ CRUSH rule 4 x 168 [4,2]
+ CRUSH rule 4 x 169 [2,8,1]
+ CRUSH rule 4 x 170 [1,2]
+ CRUSH rule 4 x 171 [8,4]
+ CRUSH rule 4 x 172 [1,8]
+ CRUSH rule 4 x 173 [8,4]
+ CRUSH rule 4 x 174 [1,6]
+ CRUSH rule 4 x 175 [8,1]
+ CRUSH rule 4 x 176 []
+ CRUSH rule 4 x 177 [2]
+ CRUSH rule 4 x 178 [4,2,1]
+ CRUSH rule 4 x 179 [1,2]
+ CRUSH rule 4 x 180 [8]
+ CRUSH rule 4 x 181 [8,2,1]
+ CRUSH rule 4 x 182 [8]
+ CRUSH rule 4 x 183 [8]
+ CRUSH rule 4 x 184 [4,8]
+ CRUSH rule 4 x 185 [8,1]
+ CRUSH rule 4 x 186 [2,1,4]
+ CRUSH rule 4 x 187 [1,8]
+ CRUSH rule 4 x 188 [1,8]
+ CRUSH rule 4 x 189 [1,8]
+ CRUSH rule 4 x 190 [1,2]
+ CRUSH rule 4 x 191 [8]
+ CRUSH rule 4 x 192 [4,1,2]
+ CRUSH rule 4 x 193 [4,2,8]
+ CRUSH rule 4 x 194 [1]
+ CRUSH rule 4 x 195 [8,4]
+ CRUSH rule 4 x 196 [8,2]
+ CRUSH rule 4 x 197 [8,4,2]
+ CRUSH rule 4 x 198 [2,8]
+ CRUSH rule 4 x 199 [1,4,8]
+ CRUSH rule 4 x 200 [1,2]
+ CRUSH rule 4 x 201 [8,1,2]
+ CRUSH rule 4 x 202 [8]
+ CRUSH rule 4 x 203 [8]
+ CRUSH rule 4 x 204 [2,1,4]
+ CRUSH rule 4 x 205 [1,8]
+ CRUSH rule 4 x 206 [1,2,8]
+ CRUSH rule 4 x 207 [2]
+ CRUSH rule 4 x 208 [8,1,2]
+ CRUSH rule 4 x 209 [1,2]
+ CRUSH rule 4 x 210 [1,2,4]
+ CRUSH rule 4 x 211 [4,2]
+ CRUSH rule 4 x 212 [8,1]
+ CRUSH rule 4 x 213 [8,4,2]
+ CRUSH rule 4 x 214 [8]
+ CRUSH rule 4 x 215 [8,1]
+ CRUSH rule 4 x 216 [1,2]
+ CRUSH rule 4 x 217 [1,2,8]
+ CRUSH rule 4 x 218 [2,8]
+ CRUSH rule 4 x 219 [8,2]
+ CRUSH rule 4 x 220 [4,8]
+ CRUSH rule 4 x 221 [8]
+ CRUSH rule 4 x 222 [8]
+ CRUSH rule 4 x 223 [1,8]
+ CRUSH rule 4 x 224 [1,4,2]
+ CRUSH rule 4 x 225 [8,2]
+ CRUSH rule 4 x 226 [8,2,4]
+ CRUSH rule 4 x 227 [4,1]
+ CRUSH rule 4 x 228 [8]
+ CRUSH rule 4 x 229 [4,6]
+ CRUSH rule 4 x 230 [4,8,2]
+ CRUSH rule 4 x 231 [4]
+ CRUSH rule 4 x 232 [2,8,4]
+ CRUSH rule 4 x 233 []
+ CRUSH rule 4 x 234 [1,2]
+ CRUSH rule 4 x 235 [4,8,1]
+ CRUSH rule 4 x 236 [2,6]
+ CRUSH rule 4 x 237 [4,8]
+ CRUSH rule 4 x 238 []
+ CRUSH rule 4 x 239 [8,6]
+ CRUSH rule 4 x 240 [4,8]
+ CRUSH rule 4 x 241 [1]
+ CRUSH rule 4 x 242 []
+ CRUSH rule 4 x 243 [8]
+ CRUSH rule 4 x 244 [4,8,2]
+ CRUSH rule 4 x 245 [8,1]
+ CRUSH rule 4 x 246 [1,2]
+ CRUSH rule 4 x 247 [8,2,1]
+ CRUSH rule 4 x 248 [8,2]
+ CRUSH rule 4 x 249 [2,1]
+ CRUSH rule 4 x 250 [2,1]
+ CRUSH rule 4 x 251 [2,8]
+ CRUSH rule 4 x 252 [4,8]
+ CRUSH rule 4 x 253 [2]
+ CRUSH rule 4 x 254 [4]
+ CRUSH rule 4 x 255 [1,8]
+ CRUSH rule 4 x 256 [4,8,1]
+ CRUSH rule 4 x 257 [2,8,4]
+ CRUSH rule 4 x 258 [4,1]
+ CRUSH rule 4 x 259 [6]
+ CRUSH rule 4 x 260 [8,2]
+ CRUSH rule 4 x 261 [8]
+ CRUSH rule 4 x 262 [8]
+ CRUSH rule 4 x 263 [8,2]
+ CRUSH rule 4 x 264 [8]
+ CRUSH rule 4 x 265 [8]
+ CRUSH rule 4 x 266 [8,2,1]
+ CRUSH rule 4 x 267 [2,1]
+ CRUSH rule 4 x 268 [1,8,2]
+ CRUSH rule 4 x 269 [1,8,2]
+ CRUSH rule 4 x 270 [4,1,2]
+ CRUSH rule 4 x 271 [8,4]
+ CRUSH rule 4 x 272 [2,8,4]
+ CRUSH rule 4 x 273 [4,1]
+ CRUSH rule 4 x 274 [8,4]
+ CRUSH rule 4 x 275 [4,8]
+ CRUSH rule 4 x 276 [8,1,2]
+ CRUSH rule 4 x 277 [8,1]
+ CRUSH rule 4 x 278 [8,1]
+ CRUSH rule 4 x 279 [8,4]
+ CRUSH rule 4 x 280 [2,8,1]
+ CRUSH rule 4 x 281 [8,2,1]
+ CRUSH rule 4 x 282 [1,8]
+ CRUSH rule 4 x 283 [8,2,1]
+ CRUSH rule 4 x 284 [8,2]
+ CRUSH rule 4 x 285 [4,8]
+ CRUSH rule 4 x 286 [2,1,8]
+ CRUSH rule 4 x 287 [1,2]
+ CRUSH rule 4 x 288 [8,1]
+ CRUSH rule 4 x 289 [4,8,2]
+ CRUSH rule 4 x 290 [1,4,8]
+ CRUSH rule 4 x 291 [1,2,4]
+ CRUSH rule 4 x 292 [8,2,1]
+ CRUSH rule 4 x 293 [8,1]
+ CRUSH rule 4 x 294 [8,4]
+ CRUSH rule 4 x 295 [4,8]
+ CRUSH rule 4 x 296 [4,1]
+ CRUSH rule 4 x 297 [8,2]
+ CRUSH rule 4 x 298 [1,2]
+ CRUSH rule 4 x 299 [2,1,8]
+ CRUSH rule 4 x 300 [8]
+ CRUSH rule 4 x 301 [1,8,2]
+ CRUSH rule 4 x 302 [1,8]
+ CRUSH rule 4 x 303 [8,4]
+ CRUSH rule 4 x 304 [2,8]
+ CRUSH rule 4 x 305 [8,2]
+ CRUSH rule 4 x 306 [1,8,2]
+ CRUSH rule 4 x 307 [2,1,8]
+ CRUSH rule 4 x 308 [2,8,4]
+ CRUSH rule 4 x 309 [8]
+ CRUSH rule 4 x 310 [4,1]
+ CRUSH rule 4 x 311 [4]
+ CRUSH rule 4 x 312 [2,1]
+ CRUSH rule 4 x 313 [4]
+ CRUSH rule 4 x 314 [1]
+ CRUSH rule 4 x 315 [2,1]
+ CRUSH rule 4 x 316 [8]
+ CRUSH rule 4 x 317 [2,8,1]
+ CRUSH rule 4 x 318 [8]
+ CRUSH rule 4 x 319 [2,8]
+ CRUSH rule 4 x 320 [8,1]
+ CRUSH rule 4 x 321 [1,2]
+ CRUSH rule 4 x 322 [2,8,4]
+ CRUSH rule 4 x 323 [4,8,1]
+ CRUSH rule 4 x 324 [8,1]
+ CRUSH rule 4 x 325 [4,8,2]
+ CRUSH rule 4 x 326 [1]
+ CRUSH rule 4 x 327 [1,8]
+ CRUSH rule 4 x 328 [8,4]
+ CRUSH rule 4 x 329 [4,8]
+ CRUSH rule 4 x 330 [4,8]
+ CRUSH rule 4 x 331 [2,8]
+ CRUSH rule 4 x 332 [2,1]
+ CRUSH rule 4 x 333 [8]
+ CRUSH rule 4 x 334 [8]
+ CRUSH rule 4 x 335 [8,1,2]
+ CRUSH rule 4 x 336 [4,8,2]
+ CRUSH rule 4 x 337 [8,2]
+ CRUSH rule 4 x 338 [8]
+ CRUSH rule 4 x 339 [8,2]
+ CRUSH rule 4 x 340 [2,1]
+ CRUSH rule 4 x 341 [4,1,8]
+ CRUSH rule 4 x 342 [2,8,4]
+ CRUSH rule 4 x 343 [8]
+ CRUSH rule 4 x 344 [6,2,4]
+ CRUSH rule 4 x 345 []
+ CRUSH rule 4 x 346 [8,2,4]
+ CRUSH rule 4 x 347 [4,1]
+ CRUSH rule 4 x 348 [8,2,1]
+ CRUSH rule 4 x 349 [1,8]
+ CRUSH rule 4 x 350 [8]
+ CRUSH rule 4 x 351 [8]
+ CRUSH rule 4 x 352 [1,2]
+ CRUSH rule 4 x 353 [8]
+ CRUSH rule 4 x 354 [1,2]
+ CRUSH rule 4 x 355 [8]
+ CRUSH rule 4 x 356 [4]
+ CRUSH rule 4 x 357 [8,1,2]
+ CRUSH rule 4 x 358 [2,1,8]
+ CRUSH rule 4 x 359 [6,8]
+ CRUSH rule 4 x 360 [1]
+ CRUSH rule 4 x 361 [8,4]
+ CRUSH rule 4 x 362 [4]
+ CRUSH rule 4 x 363 [4,1]
+ CRUSH rule 4 x 364 [2,1]
+ CRUSH rule 4 x 365 [8]
+ CRUSH rule 4 x 366 [8,2]
+ CRUSH rule 4 x 367 [4,2]
+ CRUSH rule 4 x 368 [8,4]
+ CRUSH rule 4 x 369 [8,1]
+ CRUSH rule 4 x 370 [8]
+ CRUSH rule 4 x 371 [1,4]
+ CRUSH rule 4 x 372 [1,8]
+ CRUSH rule 4 x 373 [1,8]
+ CRUSH rule 4 x 374 [8]
+ CRUSH rule 4 x 375 [8,4]
+ CRUSH rule 4 x 376 [8,1,2]
+ CRUSH rule 4 x 377 [1,2,4]
+ CRUSH rule 4 x 378 [1,2]
+ CRUSH rule 4 x 379 [8]
+ CRUSH rule 4 x 380 [2,1]
+ CRUSH rule 4 x 381 [1,4,8]
+ CRUSH rule 4 x 382 [1,4,2]
+ CRUSH rule 4 x 383 [4]
+ CRUSH rule 4 x 384 [8,2]
+ CRUSH rule 4 x 385 [8,6]
+ CRUSH rule 4 x 386 [1]
+ CRUSH rule 4 x 387 [1,4,8]
+ CRUSH rule 4 x 388 [1]
+ CRUSH rule 4 x 389 [1,4]
+ CRUSH rule 4 x 390 [4,8,1]
+ CRUSH rule 4 x 391 [4,8,2]
+ CRUSH rule 4 x 392 [1,8]
+ CRUSH rule 4 x 393 [2,8]
+ CRUSH rule 4 x 394 [8]
+ CRUSH rule 4 x 395 [1,2]
+ CRUSH rule 4 x 396 [4,2]
+ CRUSH rule 4 x 397 [2,1,4]
+ CRUSH rule 4 x 398 [2,4]
+ CRUSH rule 4 x 399 [8,4]
+ CRUSH rule 4 x 400 [8,1,4]
+ CRUSH rule 4 x 401 [1,2]
+ CRUSH rule 4 x 402 [8]
+ CRUSH rule 4 x 403 [1,2,4]
+ CRUSH rule 4 x 404 [4,2]
+ CRUSH rule 4 x 405 [8,4,2]
+ CRUSH rule 4 x 406 [2,1]
+ CRUSH rule 4 x 407 [2,8]
+ CRUSH rule 4 x 408 [4,1,8]
+ CRUSH rule 4 x 409 [8,4]
+ CRUSH rule 4 x 410 [8,4]
+ CRUSH rule 4 x 411 [2,1,8]
+ CRUSH rule 4 x 412 [2,8]
+ CRUSH rule 4 x 413 [2,8]
+ CRUSH rule 4 x 414 [4,1]
+ CRUSH rule 4 x 415 [2,8,1]
+ CRUSH rule 4 x 416 [2,1,8]
+ CRUSH rule 4 x 417 [8,6,2]
+ CRUSH rule 4 x 418 [8]
+ CRUSH rule 4 x 419 [8,4,1]
+ CRUSH rule 4 x 420 [1,4,2]
+ CRUSH rule 4 x 421 [8]
+ CRUSH rule 4 x 422 [6,8]
+ CRUSH rule 4 x 423 [2,4,8]
+ CRUSH rule 4 x 424 [8]
+ CRUSH rule 4 x 425 [1]
+ CRUSH rule 4 x 426 [8,2]
+ CRUSH rule 4 x 427 [1,8]
+ CRUSH rule 4 x 428 [4]
+ CRUSH rule 4 x 429 [4,8]
+ CRUSH rule 4 x 430 [4,8]
+ CRUSH rule 4 x 431 [4,1]
+ CRUSH rule 4 x 432 [8,1,2]
+ CRUSH rule 4 x 433 [8,1]
+ CRUSH rule 4 x 434 [2,1]
+ CRUSH rule 4 x 435 [2]
+ CRUSH rule 4 x 436 [4,1]
+ CRUSH rule 4 x 437 [8]
+ CRUSH rule 4 x 438 [2,4,8]
+ CRUSH rule 4 x 439 [1]
+ CRUSH rule 4 x 440 [2,8,1]
+ CRUSH rule 4 x 441 [4,6,2]
+ CRUSH rule 4 x 442 [2]
+ CRUSH rule 4 x 443 [8,2]
+ CRUSH rule 4 x 444 [8,1,2]
+ CRUSH rule 4 x 445 [8]
+ CRUSH rule 4 x 446 []
+ CRUSH rule 4 x 447 [2,1,4]
+ CRUSH rule 4 x 448 [8,2,4]
+ CRUSH rule 4 x 449 [8,6]
+ CRUSH rule 4 x 450 [2]
+ CRUSH rule 4 x 451 [8,4]
+ CRUSH rule 4 x 452 [8]
+ CRUSH rule 4 x 453 [6,8]
+ CRUSH rule 4 x 454 [8]
+ CRUSH rule 4 x 455 [2,8,4]
+ CRUSH rule 4 x 456 [8]
+ CRUSH rule 4 x 457 [8,2]
+ CRUSH rule 4 x 458 [2,8,1]
+ CRUSH rule 4 x 459 [2,1,8]
+ CRUSH rule 4 x 460 [8]
+ CRUSH rule 4 x 461 [8]
+ CRUSH rule 4 x 462 [8,1]
+ CRUSH rule 4 x 463 [8,2]
+ CRUSH rule 4 x 464 [8,4,2]
+ CRUSH rule 4 x 465 [6,8,1]
+ CRUSH rule 4 x 466 [8]
+ CRUSH rule 4 x 467 [8]
+ CRUSH rule 4 x 468 [8,1]
+ CRUSH rule 4 x 469 [8,1,2]
+ CRUSH rule 4 x 470 [4,2]
+ CRUSH rule 4 x 471 [1,2]
+ CRUSH rule 4 x 472 [1]
+ CRUSH rule 4 x 473 [1,2,4]
+ CRUSH rule 4 x 474 [8,1]
+ CRUSH rule 4 x 475 [8]
+ CRUSH rule 4 x 476 [4]
+ CRUSH rule 4 x 477 [4,8]
+ CRUSH rule 4 x 478 [8,2]
+ CRUSH rule 4 x 479 [2,8]
+ CRUSH rule 4 x 480 [1,8]
+ CRUSH rule 4 x 481 [2,4]
+ CRUSH rule 4 x 482 []
+ CRUSH rule 4 x 483 [2,1,8]
+ CRUSH rule 4 x 484 [1,2,8]
+ CRUSH rule 4 x 485 [8]
+ CRUSH rule 4 x 486 [4,1,8]
+ CRUSH rule 4 x 487 [1,2]
+ CRUSH rule 4 x 488 [8,1]
+ CRUSH rule 4 x 489 [2,8]
+ CRUSH rule 4 x 490 [6,1]
+ CRUSH rule 4 x 491 [1,2,8]
+ CRUSH rule 4 x 492 [8]
+ CRUSH rule 4 x 493 [2,1,8]
+ CRUSH rule 4 x 494 [1,2,8]
+ CRUSH rule 4 x 495 [4]
+ CRUSH rule 4 x 496 [8,4]
+ CRUSH rule 4 x 497 [4,8]
+ CRUSH rule 4 x 498 [2,4,8]
+ CRUSH rule 4 x 499 [8,4]
+ CRUSH rule 4 x 500 [4,8]
+ CRUSH rule 4 x 501 [2,8,1]
+ CRUSH rule 4 x 502 [6,1]
+ CRUSH rule 4 x 503 [2]
+ CRUSH rule 4 x 504 [8]
+ CRUSH rule 4 x 505 [1,8]
+ CRUSH rule 4 x 506 [4,2]
+ CRUSH rule 4 x 507 [8,1]
+ CRUSH rule 4 x 508 [1,2]
+ CRUSH rule 4 x 509 [8]
+ CRUSH rule 4 x 510 [8,2,1]
+ CRUSH rule 4 x 511 [4,8]
+ CRUSH rule 4 x 512 [8,2]
+ CRUSH rule 4 x 513 [8,2,1]
+ CRUSH rule 4 x 514 []
+ CRUSH rule 4 x 515 [8,4]
+ CRUSH rule 4 x 516 [4,1,2]
+ CRUSH rule 4 x 517 [8]
+ CRUSH rule 4 x 518 [4,8]
+ CRUSH rule 4 x 519 [8,4]
+ CRUSH rule 4 x 520 [2,8,4]
+ CRUSH rule 4 x 521 [8]
+ CRUSH rule 4 x 522 [8,1]
+ CRUSH rule 4 x 523 [4,2]
+ CRUSH rule 4 x 524 [2]
+ CRUSH rule 4 x 525 [2,1]
+ CRUSH rule 4 x 526 [1,8]
+ CRUSH rule 4 x 527 [1,2,4]
+ CRUSH rule 4 x 528 [1]
+ CRUSH rule 4 x 529 [4,8,2]
+ CRUSH rule 4 x 530 [8]
+ CRUSH rule 4 x 531 [8,1,2]
+ CRUSH rule 4 x 532 [6,4,8]
+ CRUSH rule 4 x 533 [4,8]
+ CRUSH rule 4 x 534 [8]
+ CRUSH rule 4 x 535 [8,6,1]
+ CRUSH rule 4 x 536 [8,2]
+ CRUSH rule 4 x 537 [4,8]
+ CRUSH rule 4 x 538 [8,4]
+ CRUSH rule 4 x 539 [8]
+ CRUSH rule 4 x 540 [1,8,2]
+ CRUSH rule 4 x 541 [2,4,1]
+ CRUSH rule 4 x 542 []
+ CRUSH rule 4 x 543 [8,2]
+ CRUSH rule 4 x 544 [4,8]
+ CRUSH rule 4 x 545 [8,2]
+ CRUSH rule 4 x 546 [8,1]
+ CRUSH rule 4 x 547 [8,2,1]
+ CRUSH rule 4 x 548 [4,2,1]
+ CRUSH rule 4 x 549 [8,2]
+ CRUSH rule 4 x 550 [2,4]
+ CRUSH rule 4 x 551 [8]
+ CRUSH rule 4 x 552 [4]
+ CRUSH rule 4 x 553 [2]
+ CRUSH rule 4 x 554 [1,8]
+ CRUSH rule 4 x 555 [4,1,8]
+ CRUSH rule 4 x 556 []
+ CRUSH rule 4 x 557 [8]
+ CRUSH rule 4 x 558 [4,1,2]
+ CRUSH rule 4 x 559 [2,8]
+ CRUSH rule 4 x 560 [8]
+ CRUSH rule 4 x 561 [8,4]
+ CRUSH rule 4 x 562 [4,1]
+ CRUSH rule 4 x 563 [2,8,1]
+ CRUSH rule 4 x 564 [1,8]
+ CRUSH rule 4 x 565 [4,8]
+ CRUSH rule 4 x 566 [4,8,2]
+ CRUSH rule 4 x 567 [4,8,1]
+ CRUSH rule 4 x 568 [8,1]
+ CRUSH rule 4 x 569 [4,2]
+ CRUSH rule 4 x 570 [1]
+ CRUSH rule 4 x 571 [8]
+ CRUSH rule 4 x 572 [4,1]
+ CRUSH rule 4 x 573 [1,8]
+ CRUSH rule 4 x 574 [2,1]
+ CRUSH rule 4 x 575 [8,1]
+ CRUSH rule 4 x 576 [4,8]
+ CRUSH rule 4 x 577 [8,2]
+ CRUSH rule 4 x 578 [8]
+ CRUSH rule 4 x 579 [4,1,8]
+ CRUSH rule 4 x 580 [2,8]
+ CRUSH rule 4 x 581 [8,2]
+ CRUSH rule 4 x 582 [2,8]
+ CRUSH rule 4 x 583 [8,1]
+ CRUSH rule 4 x 584 [8,1,2]
+ CRUSH rule 4 x 585 [8,1,4]
+ CRUSH rule 4 x 586 [1,2,8]
+ CRUSH rule 4 x 587 [2,4,1]
+ CRUSH rule 4 x 588 [4,8]
+ CRUSH rule 4 x 589 [8,1]
+ CRUSH rule 4 x 590 [8,2]
+ CRUSH rule 4 x 591 [4,2]
+ CRUSH rule 4 x 592 [2,1,4]
+ CRUSH rule 4 x 593 [1,8,2]
+ CRUSH rule 4 x 594 [2,8]
+ CRUSH rule 4 x 595 [8,1]
+ CRUSH rule 4 x 596 []
+ CRUSH rule 4 x 597 [1,2]
+ CRUSH rule 4 x 598 [2]
+ CRUSH rule 4 x 599 [4,2,1]
+ CRUSH rule 4 x 600 [8,1]
+ CRUSH rule 4 x 601 [1,8]
+ CRUSH rule 4 x 602 [8]
+ CRUSH rule 4 x 603 [1,8]
+ CRUSH rule 4 x 604 [8]
+ CRUSH rule 4 x 605 [2]
+ CRUSH rule 4 x 606 [2,1]
+ CRUSH rule 4 x 607 [2,4,8]
+ CRUSH rule 4 x 608 [4,2]
+ CRUSH rule 4 x 609 [4,2]
+ CRUSH rule 4 x 610 [8]
+ CRUSH rule 4 x 611 [1,2]
+ CRUSH rule 4 x 612 [2,1,8]
+ CRUSH rule 4 x 613 [8,2,1]
+ CRUSH rule 4 x 614 [8]
+ CRUSH rule 4 x 615 [8,2]
+ CRUSH rule 4 x 616 [1,8,2]
+ CRUSH rule 4 x 617 [8,1,2]
+ CRUSH rule 4 x 618 [8,4]
+ CRUSH rule 4 x 619 [4,1,8]
+ CRUSH rule 4 x 620 [1,2]
+ CRUSH rule 4 x 621 [8]
+ CRUSH rule 4 x 622 [2,4]
+ CRUSH rule 4 x 623 [2,8,1]
+ CRUSH rule 4 x 624 [4,1]
+ CRUSH rule 4 x 625 [2,8]
+ CRUSH rule 4 x 626 [8,2]
+ CRUSH rule 4 x 627 [2,8,1]
+ CRUSH rule 4 x 628 [8,2]
+ CRUSH rule 4 x 629 [2,8,4]
+ CRUSH rule 4 x 630 [2,8,1]
+ CRUSH rule 4 x 631 [1,8]
+ CRUSH rule 4 x 632 [8,2,1]
+ CRUSH rule 4 x 633 [8]
+ CRUSH rule 4 x 634 [1,2]
+ CRUSH rule 4 x 635 [4,8]
+ CRUSH rule 4 x 636 [1,4,2]
+ CRUSH rule 4 x 637 [1,2]
+ CRUSH rule 4 x 638 [8,1]
+ CRUSH rule 4 x 639 [2,1]
+ CRUSH rule 4 x 640 [1,2]
+ CRUSH rule 4 x 641 [8,2,1]
+ CRUSH rule 4 x 642 [2,1]
+ CRUSH rule 4 x 643 [1]
+ CRUSH rule 4 x 644 [8,1]
+ CRUSH rule 4 x 645 [8]
+ CRUSH rule 4 x 646 [8,1,4]
+ CRUSH rule 4 x 647 [8,1]
+ CRUSH rule 4 x 648 [1,8]
+ CRUSH rule 4 x 649 [4,8]
+ CRUSH rule 4 x 650 [8]
+ CRUSH rule 4 x 651 [4,6,8]
+ CRUSH rule 4 x 652 [4,8]
+ CRUSH rule 4 x 653 [8,2]
+ CRUSH rule 4 x 654 [6,2]
+ CRUSH rule 4 x 655 [1,4]
+ CRUSH rule 4 x 656 [8]
+ CRUSH rule 4 x 657 [6,1,2]
+ CRUSH rule 4 x 658 [8]
+ CRUSH rule 4 x 659 [4,8]
+ CRUSH rule 4 x 660 [8]
+ CRUSH rule 4 x 661 [1,8]
+ CRUSH rule 4 x 662 []
+ CRUSH rule 4 x 663 [1,4,8]
+ CRUSH rule 4 x 664 [1,4,2]
+ CRUSH rule 4 x 665 [4,6,8]
+ CRUSH rule 4 x 666 [2,8]
+ CRUSH rule 4 x 667 [1,4]
+ CRUSH rule 4 x 668 [4,8]
+ CRUSH rule 4 x 669 [6,4]
+ CRUSH rule 4 x 670 [4,2]
+ CRUSH rule 4 x 671 [2,1]
+ CRUSH rule 4 x 672 [4,2]
+ CRUSH rule 4 x 673 [4,2]
+ CRUSH rule 4 x 674 [1,8]
+ CRUSH rule 4 x 675 [1,8,6]
+ CRUSH rule 4 x 676 [2,1,4]
+ CRUSH rule 4 x 677 [4,1]
+ CRUSH rule 4 x 678 [2,4,1]
+ CRUSH rule 4 x 679 [8,2,1]
+ CRUSH rule 4 x 680 [2,8]
+ CRUSH rule 4 x 681 [8]
+ CRUSH rule 4 x 682 [1,4,2]
+ CRUSH rule 4 x 683 [1,2,4]
+ CRUSH rule 4 x 684 [8,1]
+ CRUSH rule 4 x 685 [8,1]
+ CRUSH rule 4 x 686 [1,4]
+ CRUSH rule 4 x 687 [6]
+ CRUSH rule 4 x 688 [4,8,2]
+ CRUSH rule 4 x 689 [8,4,2]
+ CRUSH rule 4 x 690 [8,1]
+ CRUSH rule 4 x 691 [1]
+ CRUSH rule 4 x 692 [8,2]
+ CRUSH rule 4 x 693 [8,4]
+ CRUSH rule 4 x 694 [8,4,1]
+ CRUSH rule 4 x 695 [2,8]
+ CRUSH rule 4 x 696 [1]
+ CRUSH rule 4 x 697 [8,1,2]
+ CRUSH rule 4 x 698 [8,2,1]
+ CRUSH rule 4 x 699 [1,8]
+ CRUSH rule 4 x 700 [1,2]
+ CRUSH rule 4 x 701 [2]
+ CRUSH rule 4 x 702 [1]
+ CRUSH rule 4 x 703 [8]
+ CRUSH rule 4 x 704 [1,4,8]
+ CRUSH rule 4 x 705 [8,2]
+ CRUSH rule 4 x 706 [1,2,4]
+ CRUSH rule 4 x 707 [8]
+ CRUSH rule 4 x 708 [4,8]
+ CRUSH rule 4 x 709 [8,2]
+ CRUSH rule 4 x 710 [8]
+ CRUSH rule 4 x 711 [2,4,8]
+ CRUSH rule 4 x 712 [2,8]
+ CRUSH rule 4 x 713 [8]
+ CRUSH rule 4 x 714 [2,1]
+ CRUSH rule 4 x 715 [1,2]
+ CRUSH rule 4 x 716 [4,8,2]
+ CRUSH rule 4 x 717 [8,2]
+ CRUSH rule 4 x 718 [8,6]
+ CRUSH rule 4 x 719 [2,6,4]
+ CRUSH rule 4 x 720 [8,2]
+ CRUSH rule 4 x 721 [4,6]
+ CRUSH rule 4 x 722 [8]
+ CRUSH rule 4 x 723 [4,1,2]
+ CRUSH rule 4 x 724 [2,6,1]
+ CRUSH rule 4 x 725 [1,2]
+ CRUSH rule 4 x 726 [4,8]
+ CRUSH rule 4 x 727 [4,8]
+ CRUSH rule 4 x 728 [2,1]
+ CRUSH rule 4 x 729 [8]
+ CRUSH rule 4 x 730 [4,8]
+ CRUSH rule 4 x 731 [4,1]
+ CRUSH rule 4 x 732 [1]
+ CRUSH rule 4 x 733 [4,8]
+ CRUSH rule 4 x 734 [8,4,2]
+ CRUSH rule 4 x 735 [4,8]
+ CRUSH rule 4 x 736 [4,8]
+ CRUSH rule 4 x 737 [1,2,8]
+ CRUSH rule 4 x 738 [4,2]
+ CRUSH rule 4 x 739 [2,1]
+ CRUSH rule 4 x 740 [1,2,8]
+ CRUSH rule 4 x 741 [8,1]
+ CRUSH rule 4 x 742 [8,2,1]
+ CRUSH rule 4 x 743 [8,1,2]
+ CRUSH rule 4 x 744 [4,8]
+ CRUSH rule 4 x 745 [1]
+ CRUSH rule 4 x 746 [1]
+ CRUSH rule 4 x 747 [8,1]
+ CRUSH rule 4 x 748 [2,8,1]
+ CRUSH rule 4 x 749 [4,8]
+ CRUSH rule 4 x 750 [1,8,4]
+ CRUSH rule 4 x 751 [2,1,8]
+ CRUSH rule 4 x 752 [8,1]
+ CRUSH rule 4 x 753 [8,4]
+ CRUSH rule 4 x 754 [8]
+ CRUSH rule 4 x 755 [1,2]
+ CRUSH rule 4 x 756 [8,1]
+ CRUSH rule 4 x 757 [8,1]
+ CRUSH rule 4 x 758 [8,2]
+ CRUSH rule 4 x 759 [8,4]
+ CRUSH rule 4 x 760 [1,4]
+ CRUSH rule 4 x 761 [1,2]
+ CRUSH rule 4 x 762 [2,8]
+ CRUSH rule 4 x 763 [8]
+ CRUSH rule 4 x 764 [1,8,2]
+ CRUSH rule 4 x 765 [8,2]
+ CRUSH rule 4 x 766 [8]
+ CRUSH rule 4 x 767 [1,2]
+ CRUSH rule 4 x 768 [8,4,2]
+ CRUSH rule 4 x 769 [8,2]
+ CRUSH rule 4 x 770 [8,2]
+ CRUSH rule 4 x 771 [8,1,4]
+ CRUSH rule 4 x 772 [8,4]
+ CRUSH rule 4 x 773 [4,1]
+ CRUSH rule 4 x 774 [8]
+ CRUSH rule 4 x 775 [8,4]
+ CRUSH rule 4 x 776 [6,2,1]
+ CRUSH rule 4 x 777 [4,1,8]
+ CRUSH rule 4 x 778 [1,8,2]
+ CRUSH rule 4 x 779 [2,8]
+ CRUSH rule 4 x 780 [2,1,4]
+ CRUSH rule 4 x 781 [8]
+ CRUSH rule 4 x 782 [4,1]
+ CRUSH rule 4 x 783 [8,1]
+ CRUSH rule 4 x 784 [1,2,4]
+ CRUSH rule 4 x 785 [8,1,2]
+ CRUSH rule 4 x 786 [8]
+ CRUSH rule 4 x 787 [1,2,6]
+ CRUSH rule 4 x 788 [8,2]
+ CRUSH rule 4 x 789 [1]
+ CRUSH rule 4 x 790 [8]
+ CRUSH rule 4 x 791 [4,8]
+ CRUSH rule 4 x 792 [4,8]
+ CRUSH rule 4 x 793 [8,1,4]
+ CRUSH rule 4 x 794 [2,8]
+ CRUSH rule 4 x 795 [1,2]
+ CRUSH rule 4 x 796 []
+ CRUSH rule 4 x 797 [2,4]
+ CRUSH rule 4 x 798 [6,8]
+ CRUSH rule 4 x 799 [4,1]
+ CRUSH rule 4 x 800 [2]
+ CRUSH rule 4 x 801 [4,8]
+ CRUSH rule 4 x 802 [1,8,2]
+ CRUSH rule 4 x 803 [2,8]
+ CRUSH rule 4 x 804 [8,2]
+ CRUSH rule 4 x 805 [8]
+ CRUSH rule 4 x 806 [1,4]
+ CRUSH rule 4 x 807 [4,8]
+ CRUSH rule 4 x 808 [8,2]
+ CRUSH rule 4 x 809 [1]
+ CRUSH rule 4 x 810 [8]
+ CRUSH rule 4 x 811 [8]
+ CRUSH rule 4 x 812 [8,4]
+ CRUSH rule 4 x 813 [8,4]
+ CRUSH rule 4 x 814 [8]
+ CRUSH rule 4 x 815 [4,1,2]
+ CRUSH rule 4 x 816 [2,1,8]
+ CRUSH rule 4 x 817 [8]
+ CRUSH rule 4 x 818 [1]
+ CRUSH rule 4 x 819 [1]
+ CRUSH rule 4 x 820 [4]
+ CRUSH rule 4 x 821 [4,8]
+ CRUSH rule 4 x 822 [2,1,4]
+ CRUSH rule 4 x 823 [4,8,2]
+ CRUSH rule 4 x 824 [8]
+ CRUSH rule 4 x 825 [2,8]
+ CRUSH rule 4 x 826 [8,2,4]
+ CRUSH rule 4 x 827 [2,8,1]
+ CRUSH rule 4 x 828 [2,1]
+ CRUSH rule 4 x 829 [8]
+ CRUSH rule 4 x 830 [2,4]
+ CRUSH rule 4 x 831 [1,8,2]
+ CRUSH rule 4 x 832 [4]
+ CRUSH rule 4 x 833 [2,1,8]
+ CRUSH rule 4 x 834 [2]
+ CRUSH rule 4 x 835 [8,4]
+ CRUSH rule 4 x 836 [4]
+ CRUSH rule 4 x 837 [8,4,1]
+ CRUSH rule 4 x 838 [6,8,2]
+ CRUSH rule 4 x 839 [2,6]
+ CRUSH rule 4 x 840 [8]
+ CRUSH rule 4 x 841 [4,8]
+ CRUSH rule 4 x 842 [2,1]
+ CRUSH rule 4 x 843 [8,4]
+ CRUSH rule 4 x 844 [8,1]
+ CRUSH rule 4 x 845 [4,8]
+ CRUSH rule 4 x 846 [4,2,1]
+ CRUSH rule 4 x 847 [2,1,8]
+ CRUSH rule 4 x 848 [2,8,1]
+ CRUSH rule 4 x 849 [4]
+ CRUSH rule 4 x 850 [1,2]
+ CRUSH rule 4 x 851 [6,8]
+ CRUSH rule 4 x 852 [8,4]
+ CRUSH rule 4 x 853 [6,8,1]
+ CRUSH rule 4 x 854 [8,1]
+ CRUSH rule 4 x 855 [8,2]
+ CRUSH rule 4 x 856 [8,4]
+ CRUSH rule 4 x 857 [8,2]
+ CRUSH rule 4 x 858 [6,1]
+ CRUSH rule 4 x 859 [8,2]
+ CRUSH rule 4 x 860 [1,2]
+ CRUSH rule 4 x 861 [8]
+ CRUSH rule 4 x 862 [8,1]
+ CRUSH rule 4 x 863 [8,2]
+ CRUSH rule 4 x 864 [8]
+ CRUSH rule 4 x 865 [8,1,2]
+ CRUSH rule 4 x 866 [8]
+ CRUSH rule 4 x 867 [8,1]
+ CRUSH rule 4 x 868 [8,1]
+ CRUSH rule 4 x 869 [8,6,4]
+ CRUSH rule 4 x 870 [2,8]
+ CRUSH rule 4 x 871 []
+ CRUSH rule 4 x 872 [1]
+ CRUSH rule 4 x 873 [4,8]
+ CRUSH rule 4 x 874 [2,6,1]
+ CRUSH rule 4 x 875 [2,8,4]
+ CRUSH rule 4 x 876 [4,8,1]
+ CRUSH rule 4 x 877 [8,4,2]
+ CRUSH rule 4 x 878 [2]
+ CRUSH rule 4 x 879 [8]
+ CRUSH rule 4 x 880 [1]
+ CRUSH rule 4 x 881 [4,8,1]
+ CRUSH rule 4 x 882 [2,1]
+ CRUSH rule 4 x 883 [2,1]
+ CRUSH rule 4 x 884 [8,2,4]
+ CRUSH rule 4 x 885 [4,1]
+ CRUSH rule 4 x 886 [8]
+ CRUSH rule 4 x 887 [8,4,1]
+ CRUSH rule 4 x 888 [8,1]
+ CRUSH rule 4 x 889 [2,1,8]
+ CRUSH rule 4 x 890 [8,2,1]
+ CRUSH rule 4 x 891 [1,8,2]
+ CRUSH rule 4 x 892 [8,2,4]
+ CRUSH rule 4 x 893 [2,6]
+ CRUSH rule 4 x 894 [8,4,2]
+ CRUSH rule 4 x 895 [4,2]
+ CRUSH rule 4 x 896 [1,8,2]
+ CRUSH rule 4 x 897 [2,8]
+ CRUSH rule 4 x 898 [1,4]
+ CRUSH rule 4 x 899 [1,8]
+ CRUSH rule 4 x 900 [4,1,2]
+ CRUSH rule 4 x 901 [1,2]
+ CRUSH rule 4 x 902 [8,4]
+ CRUSH rule 4 x 903 [8]
+ CRUSH rule 4 x 904 [8]
+ CRUSH rule 4 x 905 [8,2]
+ CRUSH rule 4 x 906 [1,2]
+ CRUSH rule 4 x 907 [8,1,2]
+ CRUSH rule 4 x 908 [8,1]
+ CRUSH rule 4 x 909 [2]
+ CRUSH rule 4 x 910 [8,2]
+ CRUSH rule 4 x 911 [8]
+ CRUSH rule 4 x 912 [1,2,8]
+ CRUSH rule 4 x 913 [8]
+ CRUSH rule 4 x 914 [6,4,8]
+ CRUSH rule 4 x 915 [8,2]
+ CRUSH rule 4 x 916 [4,1]
+ CRUSH rule 4 x 917 [1,4]
+ CRUSH rule 4 x 918 [8,2,1]
+ CRUSH rule 4 x 919 [8,2]
+ CRUSH rule 4 x 920 [8]
+ CRUSH rule 4 x 921 [1]
+ CRUSH rule 4 x 922 [8]
+ CRUSH rule 4 x 923 [4,8]
+ CRUSH rule 4 x 924 []
+ CRUSH rule 4 x 925 [4,8]
+ CRUSH rule 4 x 926 []
+ CRUSH rule 4 x 927 [1,8,4]
+ CRUSH rule 4 x 928 [8,1,2]
+ CRUSH rule 4 x 929 [4,1]
+ CRUSH rule 4 x 930 [2,8]
+ CRUSH rule 4 x 931 [1,2]
+ CRUSH rule 4 x 932 [4,1]
+ CRUSH rule 4 x 933 [8,4]
+ CRUSH rule 4 x 934 [8]
+ CRUSH rule 4 x 935 [8]
+ CRUSH rule 4 x 936 [1,8]
+ CRUSH rule 4 x 937 [4]
+ CRUSH rule 4 x 938 [8,4]
+ CRUSH rule 4 x 939 [2,8,1]
+ CRUSH rule 4 x 940 [8]
+ CRUSH rule 4 x 941 [2,1]
+ CRUSH rule 4 x 942 [1,2]
+ CRUSH rule 4 x 943 [8,2]
+ CRUSH rule 4 x 944 [8]
+ CRUSH rule 4 x 945 [8,2,4]
+ CRUSH rule 4 x 946 [2,1,8]
+ CRUSH rule 4 x 947 []
+ CRUSH rule 4 x 948 [8]
+ CRUSH rule 4 x 949 [6,1,8]
+ CRUSH rule 4 x 950 [8]
+ CRUSH rule 4 x 951 []
+ CRUSH rule 4 x 952 [2,1,8]
+ CRUSH rule 4 x 953 [1,4]
+ CRUSH rule 4 x 954 [2]
+ CRUSH rule 4 x 955 [8,1]
+ CRUSH rule 4 x 956 [1,2,8]
+ CRUSH rule 4 x 957 [8,1]
+ CRUSH rule 4 x 958 [8,4]
+ CRUSH rule 4 x 959 [4,2,8]
+ CRUSH rule 4 x 960 [6]
+ CRUSH rule 4 x 961 [1,2]
+ CRUSH rule 4 x 962 [8,4]
+ CRUSH rule 4 x 963 [2,4,1]
+ CRUSH rule 4 x 964 [1]
+ CRUSH rule 4 x 965 [8]
+ CRUSH rule 4 x 966 [4,8]
+ CRUSH rule 4 x 967 [8,4]
+ CRUSH rule 4 x 968 [8,2]
+ CRUSH rule 4 x 969 [8,2]
+ CRUSH rule 4 x 970 [2,8,4]
+ CRUSH rule 4 x 971 [1,8]
+ CRUSH rule 4 x 972 [1,8]
+ CRUSH rule 4 x 973 [1,2]
+ CRUSH rule 4 x 974 [4,2]
+ CRUSH rule 4 x 975 [4,8]
+ CRUSH rule 4 x 976 [4]
+ CRUSH rule 4 x 977 [8,4,2]
+ CRUSH rule 4 x 978 [8,2]
+ CRUSH rule 4 x 979 [8,2]
+ CRUSH rule 4 x 980 [8,2]
+ CRUSH rule 4 x 981 [8,2]
+ CRUSH rule 4 x 982 [2,1]
+ CRUSH rule 4 x 983 [4,8]
+ CRUSH rule 4 x 984 [2,1]
+ CRUSH rule 4 x 985 [2,4]
+ CRUSH rule 4 x 986 [8,4]
+ CRUSH rule 4 x 987 [2,1]
+ CRUSH rule 4 x 988 [1,4]
+ CRUSH rule 4 x 989 [1,8]
+ CRUSH rule 4 x 990 [1,2,8]
+ CRUSH rule 4 x 991 [1,4,2]
+ CRUSH rule 4 x 992 [8,1,4]
+ CRUSH rule 4 x 993 [2,8,1]
+ CRUSH rule 4 x 994 [4]
+ CRUSH rule 4 x 995 [8,2]
+ CRUSH rule 4 x 996 [8,4]
+ CRUSH rule 4 x 997 [8,4,1]
+ CRUSH rule 4 x 998 [8,1,2]
+ CRUSH rule 4 x 999 [1,8]
+ CRUSH rule 4 x 1000 [8,4,2]
+ CRUSH rule 4 x 1001 [2,1]
+ CRUSH rule 4 x 1002 [1,2]
+ CRUSH rule 4 x 1003 [2,8]
+ CRUSH rule 4 x 1004 [8,1,2]
+ CRUSH rule 4 x 1005 [8,1,2]
+ CRUSH rule 4 x 1006 [1,2]
+ CRUSH rule 4 x 1007 [1,2,4]
+ CRUSH rule 4 x 1008 [1,8,2]
+ CRUSH rule 4 x 1009 [6,8,4]
+ CRUSH rule 4 x 1010 [1]
+ CRUSH rule 4 x 1011 [4,2]
+ CRUSH rule 4 x 1012 [2,8]
+ CRUSH rule 4 x 1013 [1,2]
+ CRUSH rule 4 x 1014 [2,8,4]
+ CRUSH rule 4 x 1015 [8]
+ CRUSH rule 4 x 1016 [2,1]
+ CRUSH rule 4 x 1017 [6,2,1]
+ CRUSH rule 4 x 1018 [4]
+ CRUSH rule 4 x 1019 [4,8]
+ CRUSH rule 4 x 1020 [1]
+ CRUSH rule 4 x 1021 [2,1]
+ CRUSH rule 4 x 1022 [1,8]
+ CRUSH rule 4 x 1023 [4,2,1]
+ rule 4 (choose-set-two) num_rep 3 result size == 0:\t19/1024 (esc)
+ rule 4 (choose-set-two) num_rep 3 result size == 1:\t210/1024 (esc)
+ rule 4 (choose-set-two) num_rep 3 result size == 2:\t508/1024 (esc)
+ rule 4 (choose-set-two) num_rep 3 result size == 3:\t287/1024 (esc)
+ rule 5 (chooseleaf-set), x = 0..1023, numrep = 2..3
+ CRUSH rule 5 x 0 [2,4]
+ CRUSH rule 5 x 1 [2,8]
+ CRUSH rule 5 x 2 [1,8]
+ CRUSH rule 5 x 3 [8,1]
+ CRUSH rule 5 x 4 [4,2]
+ CRUSH rule 5 x 5 [8,2]
+ CRUSH rule 5 x 6 [2,8]
+ CRUSH rule 5 x 7 [4,8]
+ CRUSH rule 5 x 8 [4,8]
+ CRUSH rule 5 x 9 [2,4]
+ CRUSH rule 5 x 10 [2,8]
+ CRUSH rule 5 x 11 [2,8]
+ CRUSH rule 5 x 12 [2,8]
+ CRUSH rule 5 x 13 [4,8]
+ CRUSH rule 5 x 14 [8,2]
+ CRUSH rule 5 x 15 [8,2]
+ CRUSH rule 5 x 16 [8,2]
+ CRUSH rule 5 x 17 [4,1]
+ CRUSH rule 5 x 18 [1,8]
+ CRUSH rule 5 x 19 [8,4]
+ CRUSH rule 5 x 20 [2,8]
+ CRUSH rule 5 x 21 [8,2]
+ CRUSH rule 5 x 22 [8,1]
+ CRUSH rule 5 x 23 [4,8]
+ CRUSH rule 5 x 24 [1,8]
+ CRUSH rule 5 x 25 [4,8]
+ CRUSH rule 5 x 26 [2,8]
+ CRUSH rule 5 x 27 [4,1]
+ CRUSH rule 5 x 28 [8,2]
+ CRUSH rule 5 x 29 [8,4]
+ CRUSH rule 5 x 30 [4,8]
+ CRUSH rule 5 x 31 [8,1]
+ CRUSH rule 5 x 32 [6,1]
+ CRUSH rule 5 x 33 [2,8]
+ CRUSH rule 5 x 34 [2,8]
+ CRUSH rule 5 x 35 [1,8]
+ CRUSH rule 5 x 36 [8,2]
+ CRUSH rule 5 x 37 [1,8]
+ CRUSH rule 5 x 38 [4,8]
+ CRUSH rule 5 x 39 [8,2]
+ CRUSH rule 5 x 40 [8,2]
+ CRUSH rule 5 x 41 [2,8]
+ CRUSH rule 5 x 42 [8,2]
+ CRUSH rule 5 x 43 [1,8]
+ CRUSH rule 5 x 44 [1,8]
+ CRUSH rule 5 x 45 [8,2]
+ CRUSH rule 5 x 46 [2,8]
+ CRUSH rule 5 x 47 [4,2]
+ CRUSH rule 5 x 48 [8,1]
+ CRUSH rule 5 x 49 [8,2]
+ CRUSH rule 5 x 50 [4,1]
+ CRUSH rule 5 x 51 [8,2]
+ CRUSH rule 5 x 52 [8,1]
+ CRUSH rule 5 x 53 [4,8]
+ CRUSH rule 5 x 54 [8,4]
+ CRUSH rule 5 x 55 [8,2]
+ CRUSH rule 5 x 56 [8,4]
+ CRUSH rule 5 x 57 [8,1]
+ CRUSH rule 5 x 58 [1,8]
+ CRUSH rule 5 x 59 [2,8]
+ CRUSH rule 5 x 60 [4,2]
+ CRUSH rule 5 x 61 [4,8]
+ CRUSH rule 5 x 62 [8,1]
+ CRUSH rule 5 x 63 [8,2]
+ CRUSH rule 5 x 64 [4,2]
+ CRUSH rule 5 x 65 [8,4]
+ CRUSH rule 5 x 66 [4,8]
+ CRUSH rule 5 x 67 [4,2]
+ CRUSH rule 5 x 68 [1,8]
+ CRUSH rule 5 x 69 [2,8]
+ CRUSH rule 5 x 70 [8,2]
+ CRUSH rule 5 x 71 [2,8]
+ CRUSH rule 5 x 72 [8,1]
+ CRUSH rule 5 x 73 [2,8]
+ CRUSH rule 5 x 74 [1,8]
+ CRUSH rule 5 x 75 [4,2]
+ CRUSH rule 5 x 76 [4,1]
+ CRUSH rule 5 x 77 [8,2]
+ CRUSH rule 5 x 78 [1,6]
+ CRUSH rule 5 x 79 [4,1]
+ CRUSH rule 5 x 80 [2,4]
+ CRUSH rule 5 x 81 [2,8]
+ CRUSH rule 5 x 82 [6,1]
+ CRUSH rule 5 x 83 [2,8]
+ CRUSH rule 5 x 84 [8,2]
+ CRUSH rule 5 x 85 [4,8]
+ CRUSH rule 5 x 86 [2,8]
+ CRUSH rule 5 x 87 [2,8]
+ CRUSH rule 5 x 88 [1,6]
+ CRUSH rule 5 x 89 [1,8]
+ CRUSH rule 5 x 90 [8,4]
+ CRUSH rule 5 x 91 [4,8]
+ CRUSH rule 5 x 92 [1,8]
+ CRUSH rule 5 x 93 [8,4]
+ CRUSH rule 5 x 94 [1,8]
+ CRUSH rule 5 x 95 [8,1]
+ CRUSH rule 5 x 96 [8,2]
+ CRUSH rule 5 x 97 [8,1]
+ CRUSH rule 5 x 98 [2,8]
+ CRUSH rule 5 x 99 [2,8]
+ CRUSH rule 5 x 100 [1,8]
+ CRUSH rule 5 x 101 [8,1]
+ CRUSH rule 5 x 102 [2,8]
+ CRUSH rule 5 x 103 [8,2]
+ CRUSH rule 5 x 104 [8,4]
+ CRUSH rule 5 x 105 [2,4]
+ CRUSH rule 5 x 106 [1,8]
+ CRUSH rule 5 x 107 [1,8]
+ CRUSH rule 5 x 108 [8,2]
+ CRUSH rule 5 x 109 [1,4]
+ CRUSH rule 5 x 110 [4,2]
+ CRUSH rule 5 x 111 [2,4]
+ CRUSH rule 5 x 112 [2,8]
+ CRUSH rule 5 x 113 [8,2]
+ CRUSH rule 5 x 114 [8,4]
+ CRUSH rule 5 x 115 [8,2]
+ CRUSH rule 5 x 116 [1,8]
+ CRUSH rule 5 x 117 [6,1]
+ CRUSH rule 5 x 118 [2,8]
+ CRUSH rule 5 x 119 [8,1]
+ CRUSH rule 5 x 120 [2,4]
+ CRUSH rule 5 x 121 [2,8]
+ CRUSH rule 5 x 122 [8,1]
+ CRUSH rule 5 x 123 [2,8]
+ CRUSH rule 5 x 124 [2,8]
+ CRUSH rule 5 x 125 [1,8]
+ CRUSH rule 5 x 126 [1,8]
+ CRUSH rule 5 x 127 [4,8]
+ CRUSH rule 5 x 128 [8,2]
+ CRUSH rule 5 x 129 [2,4]
+ CRUSH rule 5 x 130 [4,8]
+ CRUSH rule 5 x 131 [1,4]
+ CRUSH rule 5 x 132 [1,8]
+ CRUSH rule 5 x 133 [8,1]
+ CRUSH rule 5 x 134 [1,8]
+ CRUSH rule 5 x 135 [4,8]
+ CRUSH rule 5 x 136 [2,4]
+ CRUSH rule 5 x 137 [8,4]
+ CRUSH rule 5 x 138 [8,4]
+ CRUSH rule 5 x 139 [4,2]
+ CRUSH rule 5 x 140 [1,8]
+ CRUSH rule 5 x 141 [8,2]
+ CRUSH rule 5 x 142 [4,1]
+ CRUSH rule 5 x 143 [4,8]
+ CRUSH rule 5 x 144 [8,1]
+ CRUSH rule 5 x 145 [8,1]
+ CRUSH rule 5 x 146 [2,8]
+ CRUSH rule 5 x 147 [2,8]
+ CRUSH rule 5 x 148 [4,1]
+ CRUSH rule 5 x 149 [4,8]
+ CRUSH rule 5 x 150 [1,8]
+ CRUSH rule 5 x 151 [1,8]
+ CRUSH rule 5 x 152 [8,2]
+ CRUSH rule 5 x 153 [8,4]
+ CRUSH rule 5 x 154 [4,2]
+ CRUSH rule 5 x 155 [4,8]
+ CRUSH rule 5 x 156 [4,2]
+ CRUSH rule 5 x 157 [1,8]
+ CRUSH rule 5 x 158 [2,8]
+ CRUSH rule 5 x 159 [8,2]
+ CRUSH rule 5 x 160 [2,8]
+ CRUSH rule 5 x 161 [1,4]
+ CRUSH rule 5 x 162 [1,8]
+ CRUSH rule 5 x 163 [4,8]
+ CRUSH rule 5 x 164 [8,1]
+ CRUSH rule 5 x 165 [8,2]
+ CRUSH rule 5 x 166 [2,8]
+ CRUSH rule 5 x 167 [1,8]
+ CRUSH rule 5 x 168 [4,2]
+ CRUSH rule 5 x 169 [2,8]
+ CRUSH rule 5 x 170 [1,8]
+ CRUSH rule 5 x 171 [8,4]
+ CRUSH rule 5 x 172 [1,8]
+ CRUSH rule 5 x 173 [8,4]
+ CRUSH rule 5 x 174 [1,6]
+ CRUSH rule 5 x 175 [8,1]
+ CRUSH rule 5 x 176 [2,8]
+ CRUSH rule 5 x 177 [1,8]
+ CRUSH rule 5 x 178 [4,2]
+ CRUSH rule 5 x 179 [1,8]
+ CRUSH rule 5 x 180 [8,1]
+ CRUSH rule 5 x 181 [8,2]
+ CRUSH rule 5 x 182 [8,1]
+ CRUSH rule 5 x 183 [8,4]
+ CRUSH rule 5 x 184 [4,8]
+ CRUSH rule 5 x 185 [8,1]
+ CRUSH rule 5 x 186 [2,4]
+ CRUSH rule 5 x 187 [1,8]
+ CRUSH rule 5 x 188 [1,8]
+ CRUSH rule 5 x 189 [1,8]
+ CRUSH rule 5 x 190 [1,8]
+ CRUSH rule 5 x 191 [8,1]
+ CRUSH rule 5 x 192 [4,1]
+ CRUSH rule 5 x 193 [4,2]
+ CRUSH rule 5 x 194 [1,8]
+ CRUSH rule 5 x 195 [8,4]
+ CRUSH rule 5 x 196 [8,2]
+ CRUSH rule 5 x 197 [8,4]
+ CRUSH rule 5 x 198 [2,8]
+ CRUSH rule 5 x 199 [1,4]
+ CRUSH rule 5 x 200 [1,8]
+ CRUSH rule 5 x 201 [8,1]
+ CRUSH rule 5 x 202 [8,1]
+ CRUSH rule 5 x 203 [8,1]
+ CRUSH rule 5 x 204 [2,4]
+ CRUSH rule 5 x 205 [1,8]
+ CRUSH rule 5 x 206 [1,8]
+ CRUSH rule 5 x 207 [2,8]
+ CRUSH rule 5 x 208 [8,1]
+ CRUSH rule 5 x 209 [1,8]
+ CRUSH rule 5 x 210 [1,4]
+ CRUSH rule 5 x 211 [4,2]
+ CRUSH rule 5 x 212 [8,1]
+ CRUSH rule 5 x 213 [8,4]
+ CRUSH rule 5 x 214 [8,2]
+ CRUSH rule 5 x 215 [8,1]
+ CRUSH rule 5 x 216 [2,8]
+ CRUSH rule 5 x 217 [1,8]
+ CRUSH rule 5 x 218 [2,8]
+ CRUSH rule 5 x 219 [8,2]
+ CRUSH rule 5 x 220 [4,8]
+ CRUSH rule 5 x 221 [8,1]
+ CRUSH rule 5 x 222 [8,1]
+ CRUSH rule 5 x 223 [1,8]
+ CRUSH rule 5 x 224 [1,4]
+ CRUSH rule 5 x 225 [8,2]
+ CRUSH rule 5 x 226 [8,2]
+ CRUSH rule 5 x 227 [4,1]
+ CRUSH rule 5 x 228 [8,2]
+ CRUSH rule 5 x 229 [4,8]
+ CRUSH rule 5 x 230 [4,8]
+ CRUSH rule 5 x 231 [4,8]
+ CRUSH rule 5 x 232 [2,8]
+ CRUSH rule 5 x 233 [2,8]
+ CRUSH rule 5 x 234 [1,8]
+ CRUSH rule 5 x 235 [4,8]
+ CRUSH rule 5 x 236 [2,6]
+ CRUSH rule 5 x 237 [4,8]
+ CRUSH rule 5 x 238 [2,8]
+ CRUSH rule 5 x 239 [8,1]
+ CRUSH rule 5 x 240 [4,8]
+ CRUSH rule 5 x 241 [1,8]
+ CRUSH rule 5 x 242 [8,2]
+ CRUSH rule 5 x 243 [8,2]
+ CRUSH rule 5 x 244 [4,8]
+ CRUSH rule 5 x 245 [8,1]
+ CRUSH rule 5 x 246 [1,8]
+ CRUSH rule 5 x 247 [8,2]
+ CRUSH rule 5 x 248 [8,2]
+ CRUSH rule 5 x 249 [2,8]
+ CRUSH rule 5 x 250 [2,4]
+ CRUSH rule 5 x 251 [2,8]
+ CRUSH rule 5 x 252 [4,8]
+ CRUSH rule 5 x 253 [2,8]
+ CRUSH rule 5 x 254 [4,2]
+ CRUSH rule 5 x 255 [1,8]
+ CRUSH rule 5 x 256 [4,8]
+ CRUSH rule 5 x 257 [2,8]
+ CRUSH rule 5 x 258 [4,2]
+ CRUSH rule 5 x 259 [6,2]
+ CRUSH rule 5 x 260 [8,2]
+ CRUSH rule 5 x 261 [8,1]
+ CRUSH rule 5 x 262 [8,1]
+ CRUSH rule 5 x 263 [8,1]
+ CRUSH rule 5 x 264 [8,2]
+ CRUSH rule 5 x 265 [8,2]
+ CRUSH rule 5 x 266 [8,2]
+ CRUSH rule 5 x 267 [2,8]
+ CRUSH rule 5 x 268 [1,8]
+ CRUSH rule 5 x 269 [1,8]
+ CRUSH rule 5 x 270 [4,1]
+ CRUSH rule 5 x 271 [8,4]
+ CRUSH rule 5 x 272 [2,8]
+ CRUSH rule 5 x 273 [4,1]
+ CRUSH rule 5 x 274 [8,4]
+ CRUSH rule 5 x 275 [4,8]
+ CRUSH rule 5 x 276 [8,1]
+ CRUSH rule 5 x 277 [8,1]
+ CRUSH rule 5 x 278 [8,1]
+ CRUSH rule 5 x 279 [8,4]
+ CRUSH rule 5 x 280 [2,8]
+ CRUSH rule 5 x 281 [8,2]
+ CRUSH rule 5 x 282 [2,8]
+ CRUSH rule 5 x 283 [8,2]
+ CRUSH rule 5 x 284 [8,2]
+ CRUSH rule 5 x 285 [4,8]
+ CRUSH rule 5 x 286 [2,8]
+ CRUSH rule 5 x 287 [1,8]
+ CRUSH rule 5 x 288 [8,1]
+ CRUSH rule 5 x 289 [4,8]
+ CRUSH rule 5 x 290 [1,4]
+ CRUSH rule 5 x 291 [1,4]
+ CRUSH rule 5 x 292 [8,2]
+ CRUSH rule 5 x 293 [8,1]
+ CRUSH rule 5 x 294 [8,4]
+ CRUSH rule 5 x 295 [4,8]
+ CRUSH rule 5 x 296 [4,1]
+ CRUSH rule 5 x 297 [8,2]
+ CRUSH rule 5 x 298 [1,8]
+ CRUSH rule 5 x 299 [2,8]
+ CRUSH rule 5 x 300 [8,2]
+ CRUSH rule 5 x 301 [1,8]
+ CRUSH rule 5 x 302 [1,8]
+ CRUSH rule 5 x 303 [8,4]
+ CRUSH rule 5 x 304 [2,8]
+ CRUSH rule 5 x 305 [8,2]
+ CRUSH rule 5 x 306 [1,8]
+ CRUSH rule 5 x 307 [2,8]
+ CRUSH rule 5 x 308 [2,8]
+ CRUSH rule 5 x 309 [8,1]
+ CRUSH rule 5 x 310 [4,1]
+ CRUSH rule 5 x 311 [4,8]
+ CRUSH rule 5 x 312 [2,8]
+ CRUSH rule 5 x 313 [4,1]
+ CRUSH rule 5 x 314 [2,8]
+ CRUSH rule 5 x 315 [2,8]
+ CRUSH rule 5 x 316 [8,1]
+ CRUSH rule 5 x 317 [2,8]
+ CRUSH rule 5 x 318 [8,1]
+ CRUSH rule 5 x 319 [2,8]
+ CRUSH rule 5 x 320 [8,1]
+ CRUSH rule 5 x 321 [1,8]
+ CRUSH rule 5 x 322 [2,8]
+ CRUSH rule 5 x 323 [4,8]
+ CRUSH rule 5 x 324 [8,1]
+ CRUSH rule 5 x 325 [4,8]
+ CRUSH rule 5 x 326 [1,6]
+ CRUSH rule 5 x 327 [1,8]
+ CRUSH rule 5 x 328 [8,4]
+ CRUSH rule 5 x 329 [4,8]
+ CRUSH rule 5 x 330 [4,8]
+ CRUSH rule 5 x 331 [2,8]
+ CRUSH rule 5 x 332 [2,8]
+ CRUSH rule 5 x 333 [8,1]
+ CRUSH rule 5 x 334 [8,2]
+ CRUSH rule 5 x 335 [8,1]
+ CRUSH rule 5 x 336 [4,8]
+ CRUSH rule 5 x 337 [8,2]
+ CRUSH rule 5 x 338 [8,1]
+ CRUSH rule 5 x 339 [8,2]
+ CRUSH rule 5 x 340 [2,8]
+ CRUSH rule 5 x 341 [4,1]
+ CRUSH rule 5 x 342 [2,8]
+ CRUSH rule 5 x 343 [8,1]
+ CRUSH rule 5 x 344 [6,2]
+ CRUSH rule 5 x 345 [2,8]
+ CRUSH rule 5 x 346 [8,2]
+ CRUSH rule 5 x 347 [4,1]
+ CRUSH rule 5 x 348 [8,2]
+ CRUSH rule 5 x 349 [1,8]
+ CRUSH rule 5 x 350 [8,1]
+ CRUSH rule 5 x 351 [8,2]
+ CRUSH rule 5 x 352 [1,8]
+ CRUSH rule 5 x 353 [8,1]
+ CRUSH rule 5 x 354 [1,8]
+ CRUSH rule 5 x 355 [8,2]
+ CRUSH rule 5 x 356 [4,1]
+ CRUSH rule 5 x 357 [8,1]
+ CRUSH rule 5 x 358 [2,8]
+ CRUSH rule 5 x 359 [6,1]
+ CRUSH rule 5 x 360 [2,8]
+ CRUSH rule 5 x 361 [8,4]
+ CRUSH rule 5 x 362 [4,1]
+ CRUSH rule 5 x 363 [4,1]
+ CRUSH rule 5 x 364 [2,8]
+ CRUSH rule 5 x 365 [8,1]
+ CRUSH rule 5 x 366 [8,2]
+ CRUSH rule 5 x 367 [4,2]
+ CRUSH rule 5 x 368 [8,4]
+ CRUSH rule 5 x 369 [8,1]
+ CRUSH rule 5 x 370 [8,2]
+ CRUSH rule 5 x 371 [1,4]
+ CRUSH rule 5 x 372 [1,8]
+ CRUSH rule 5 x 373 [1,8]
+ CRUSH rule 5 x 374 [8,1]
+ CRUSH rule 5 x 375 [8,4]
+ CRUSH rule 5 x 376 [8,1]
+ CRUSH rule 5 x 377 [1,4]
+ CRUSH rule 5 x 378 [1,8]
+ CRUSH rule 5 x 379 [8,2]
+ CRUSH rule 5 x 380 [2,8]
+ CRUSH rule 5 x 381 [1,4]
+ CRUSH rule 5 x 382 [1,4]
+ CRUSH rule 5 x 383 [4,8]
+ CRUSH rule 5 x 384 [8,2]
+ CRUSH rule 5 x 385 [8,1]
+ CRUSH rule 5 x 386 [1,8]
+ CRUSH rule 5 x 387 [1,4]
+ CRUSH rule 5 x 388 [2,6]
+ CRUSH rule 5 x 389 [1,4]
+ CRUSH rule 5 x 390 [4,8]
+ CRUSH rule 5 x 391 [4,8]
+ CRUSH rule 5 x 392 [1,8]
+ CRUSH rule 5 x 393 [2,8]
+ CRUSH rule 5 x 394 [8,2]
+ CRUSH rule 5 x 395 [1,8]
+ CRUSH rule 5 x 396 [4,2]
+ CRUSH rule 5 x 397 [2,4]
+ CRUSH rule 5 x 398 [2,4]
+ CRUSH rule 5 x 399 [8,4]
+ CRUSH rule 5 x 400 [8,1]
+ CRUSH rule 5 x 401 [1,4]
+ CRUSH rule 5 x 402 [8,4]
+ CRUSH rule 5 x 403 [1,4]
+ CRUSH rule 5 x 404 [4,2]
+ CRUSH rule 5 x 405 [8,4]
+ CRUSH rule 5 x 406 [2,8]
+ CRUSH rule 5 x 407 [2,8]
+ CRUSH rule 5 x 408 [4,1]
+ CRUSH rule 5 x 409 [8,4]
+ CRUSH rule 5 x 410 [8,4]
+ CRUSH rule 5 x 411 [2,8]
+ CRUSH rule 5 x 412 [2,6]
+ CRUSH rule 5 x 413 [2,8]
+ CRUSH rule 5 x 414 [4,1]
+ CRUSH rule 5 x 415 [2,8]
+ CRUSH rule 5 x 416 [2,8]
+ CRUSH rule 5 x 417 [8,2]
+ CRUSH rule 5 x 418 [8,1]
+ CRUSH rule 5 x 419 [8,4]
+ CRUSH rule 5 x 420 [1,4]
+ CRUSH rule 5 x 421 [8,4]
+ CRUSH rule 5 x 422 [6,2]
+ CRUSH rule 5 x 423 [2,4]
+ CRUSH rule 5 x 424 [8,1]
+ CRUSH rule 5 x 425 [1,8]
+ CRUSH rule 5 x 426 [8,2]
+ CRUSH rule 5 x 427 [1,8]
+ CRUSH rule 5 x 428 [4,8]
+ CRUSH rule 5 x 429 [4,8]
+ CRUSH rule 5 x 430 [4,8]
+ CRUSH rule 5 x 431 [4,1]
+ CRUSH rule 5 x 432 [8,1]
+ CRUSH rule 5 x 433 [8,1]
+ CRUSH rule 5 x 434 [2,8]
+ CRUSH rule 5 x 435 [2,8]
+ CRUSH rule 5 x 436 [4,1]
+ CRUSH rule 5 x 437 [8,2]
+ CRUSH rule 5 x 438 [2,4]
+ CRUSH rule 5 x 439 [1,6]
+ CRUSH rule 5 x 440 [2,8]
+ CRUSH rule 5 x 441 [4,6]
+ CRUSH rule 5 x 442 [2,8]
+ CRUSH rule 5 x 443 [8,2]
+ CRUSH rule 5 x 444 [8,1]
+ CRUSH rule 5 x 445 [8,2]
+ CRUSH rule 5 x 446 [8,1]
+ CRUSH rule 5 x 447 [2,4]
+ CRUSH rule 5 x 448 [8,2]
+ CRUSH rule 5 x 449 [8,1]
+ CRUSH rule 5 x 450 [1,8]
+ CRUSH rule 5 x 451 [8,4]
+ CRUSH rule 5 x 452 [8,2]
+ CRUSH rule 5 x 453 [6,2]
+ CRUSH rule 5 x 454 [8,2]
+ CRUSH rule 5 x 455 [2,8]
+ CRUSH rule 5 x 456 [8,2]
+ CRUSH rule 5 x 457 [8,2]
+ CRUSH rule 5 x 458 [2,8]
+ CRUSH rule 5 x 459 [2,8]
+ CRUSH rule 5 x 460 [8,2]
+ CRUSH rule 5 x 461 [8,1]
+ CRUSH rule 5 x 462 [8,1]
+ CRUSH rule 5 x 463 [8,2]
+ CRUSH rule 5 x 464 [8,4]
+ CRUSH rule 5 x 465 [6,2]
+ CRUSH rule 5 x 466 [8,1]
+ CRUSH rule 5 x 467 [8,2]
+ CRUSH rule 5 x 468 [8,1]
+ CRUSH rule 5 x 469 [8,1]
+ CRUSH rule 5 x 470 [4,2]
+ CRUSH rule 5 x 471 [1,8]
+ CRUSH rule 5 x 472 [1,8]
+ CRUSH rule 5 x 473 [1,4]
+ CRUSH rule 5 x 474 [8,1]
+ CRUSH rule 5 x 475 [8,2]
+ CRUSH rule 5 x 476 [4,8]
+ CRUSH rule 5 x 477 [4,8]
+ CRUSH rule 5 x 478 [8,2]
+ CRUSH rule 5 x 479 [2,8]
+ CRUSH rule 5 x 480 [1,8]
+ CRUSH rule 5 x 481 [2,4]
+ CRUSH rule 5 x 482 [1,8]
+ CRUSH rule 5 x 483 [2,8]
+ CRUSH rule 5 x 484 [1,8]
+ CRUSH rule 5 x 485 [8,1]
+ CRUSH rule 5 x 486 [4,1]
+ CRUSH rule 5 x 487 [1,8]
+ CRUSH rule 5 x 488 [8,1]
+ CRUSH rule 5 x 489 [2,8]
+ CRUSH rule 5 x 490 [6,2]
+ CRUSH rule 5 x 491 [1,8]
+ CRUSH rule 5 x 492 [8,1]
+ CRUSH rule 5 x 493 [2,8]
+ CRUSH rule 5 x 494 [1,8]
+ CRUSH rule 5 x 495 [4,1]
+ CRUSH rule 5 x 496 [8,4]
+ CRUSH rule 5 x 497 [4,8]
+ CRUSH rule 5 x 498 [2,4]
+ CRUSH rule 5 x 499 [8,4]
+ CRUSH rule 5 x 500 [4,8]
+ CRUSH rule 5 x 501 [2,8]
+ CRUSH rule 5 x 502 [6,1]
+ CRUSH rule 5 x 503 [2,8]
+ CRUSH rule 5 x 504 [8,1]
+ CRUSH rule 5 x 505 [1,8]
+ CRUSH rule 5 x 506 [4,2]
+ CRUSH rule 5 x 507 [8,1]
+ CRUSH rule 5 x 508 [1,8]
+ CRUSH rule 5 x 509 [8,1]
+ CRUSH rule 5 x 510 [8,2]
+ CRUSH rule 5 x 511 [4,8]
+ CRUSH rule 5 x 512 [8,2]
+ CRUSH rule 5 x 513 [8,2]
+ CRUSH rule 5 x 514 [2,8]
+ CRUSH rule 5 x 515 [8,4]
+ CRUSH rule 5 x 516 [4,1]
+ CRUSH rule 5 x 517 [8,2]
+ CRUSH rule 5 x 518 [4,8]
+ CRUSH rule 5 x 519 [8,4]
+ CRUSH rule 5 x 520 [2,8]
+ CRUSH rule 5 x 521 [8,2]
+ CRUSH rule 5 x 522 [8,1]
+ CRUSH rule 5 x 523 [4,2]
+ CRUSH rule 5 x 524 [2,6]
+ CRUSH rule 5 x 525 [2,8]
+ CRUSH rule 5 x 526 [1,8]
+ CRUSH rule 5 x 527 [1,4]
+ CRUSH rule 5 x 528 [2,8]
+ CRUSH rule 5 x 529 [4,8]
+ CRUSH rule 5 x 530 [8,1]
+ CRUSH rule 5 x 531 [8,1]
+ CRUSH rule 5 x 532 [6,4]
+ CRUSH rule 5 x 533 [4,8]
+ CRUSH rule 5 x 534 [8,1]
+ CRUSH rule 5 x 535 [8,1]
+ CRUSH rule 5 x 536 [8,2]
+ CRUSH rule 5 x 537 [4,8]
+ CRUSH rule 5 x 538 [8,4]
+ CRUSH rule 5 x 539 [8,1]
+ CRUSH rule 5 x 540 [1,8]
+ CRUSH rule 5 x 541 [2,4]
+ CRUSH rule 5 x 542 [2,8]
+ CRUSH rule 5 x 543 [8,2]
+ CRUSH rule 5 x 544 [4,8]
+ CRUSH rule 5 x 545 [8,1]
+ CRUSH rule 5 x 546 [8,1]
+ CRUSH rule 5 x 547 [8,2]
+ CRUSH rule 5 x 548 [4,2]
+ CRUSH rule 5 x 549 [8,2]
+ CRUSH rule 5 x 550 [2,4]
+ CRUSH rule 5 x 551 [8,1]
+ CRUSH rule 5 x 552 [4,8]
+ CRUSH rule 5 x 553 [2,8]
+ CRUSH rule 5 x 554 [1,8]
+ CRUSH rule 5 x 555 [4,1]
+ CRUSH rule 5 x 556 [8,1]
+ CRUSH rule 5 x 557 [8,2]
+ CRUSH rule 5 x 558 [4,1]
+ CRUSH rule 5 x 559 [1,8]
+ CRUSH rule 5 x 560 [8,1]
+ CRUSH rule 5 x 561 [8,4]
+ CRUSH rule 5 x 562 [4,1]
+ CRUSH rule 5 x 563 [2,8]
+ CRUSH rule 5 x 564 [1,8]
+ CRUSH rule 5 x 565 [4,8]
+ CRUSH rule 5 x 566 [4,8]
+ CRUSH rule 5 x 567 [4,8]
+ CRUSH rule 5 x 568 [8,1]
+ CRUSH rule 5 x 569 [4,1]
+ CRUSH rule 5 x 570 [1,8]
+ CRUSH rule 5 x 571 [6,1]
+ CRUSH rule 5 x 572 [4,2]
+ CRUSH rule 5 x 573 [1,8]
+ CRUSH rule 5 x 574 [2,8]
+ CRUSH rule 5 x 575 [8,2]
+ CRUSH rule 5 x 576 [4,8]
+ CRUSH rule 5 x 577 [8,2]
+ CRUSH rule 5 x 578 [8,1]
+ CRUSH rule 5 x 579 [4,1]
+ CRUSH rule 5 x 580 [1,8]
+ CRUSH rule 5 x 581 [8,2]
+ CRUSH rule 5 x 582 [2,8]
+ CRUSH rule 5 x 583 [8,1]
+ CRUSH rule 5 x 584 [8,1]
+ CRUSH rule 5 x 585 [8,1]
+ CRUSH rule 5 x 586 [1,8]
+ CRUSH rule 5 x 587 [2,4]
+ CRUSH rule 5 x 588 [4,8]
+ CRUSH rule 5 x 589 [8,1]
+ CRUSH rule 5 x 590 [8,2]
+ CRUSH rule 5 x 591 [4,2]
+ CRUSH rule 5 x 592 [2,4]
+ CRUSH rule 5 x 593 [1,8]
+ CRUSH rule 5 x 594 [2,8]
+ CRUSH rule 5 x 595 [8,1]
+ CRUSH rule 5 x 596 [8,2]
+ CRUSH rule 5 x 597 [1,8]
+ CRUSH rule 5 x 598 [2,8]
+ CRUSH rule 5 x 599 [4,2]
+ CRUSH rule 5 x 600 [8,1]
+ CRUSH rule 5 x 601 [1,8]
+ CRUSH rule 5 x 602 [8,2]
+ CRUSH rule 5 x 603 [1,8]
+ CRUSH rule 5 x 604 [8,2]
+ CRUSH rule 5 x 605 [2,8]
+ CRUSH rule 5 x 606 [2,6]
+ CRUSH rule 5 x 607 [2,4]
+ CRUSH rule 5 x 608 [4,2]
+ CRUSH rule 5 x 609 [4,2]
+ CRUSH rule 5 x 610 [8,1]
+ CRUSH rule 5 x 611 [1,8]
+ CRUSH rule 5 x 612 [2,8]
+ CRUSH rule 5 x 613 [8,2]
+ CRUSH rule 5 x 614 [8,2]
+ CRUSH rule 5 x 615 [8,2]
+ CRUSH rule 5 x 616 [1,8]
+ CRUSH rule 5 x 617 [8,1]
+ CRUSH rule 5 x 618 [8,4]
+ CRUSH rule 5 x 619 [4,1]
+ CRUSH rule 5 x 620 [1,8]
+ CRUSH rule 5 x 621 [8,1]
+ CRUSH rule 5 x 622 [2,4]
+ CRUSH rule 5 x 623 [2,8]
+ CRUSH rule 5 x 624 [4,2]
+ CRUSH rule 5 x 625 [2,8]
+ CRUSH rule 5 x 626 [8,2]
+ CRUSH rule 5 x 627 [2,8]
+ CRUSH rule 5 x 628 [8,2]
+ CRUSH rule 5 x 629 [2,8]
+ CRUSH rule 5 x 630 [2,8]
+ CRUSH rule 5 x 631 [1,8]
+ CRUSH rule 5 x 632 [8,2]
+ CRUSH rule 5 x 633 [8,2]
+ CRUSH rule 5 x 634 [1,8]
+ CRUSH rule 5 x 635 [4,8]
+ CRUSH rule 5 x 636 [1,4]
+ CRUSH rule 5 x 637 [1,8]
+ CRUSH rule 5 x 638 [8,1]
+ CRUSH rule 5 x 639 [2,8]
+ CRUSH rule 5 x 640 [2,8]
+ CRUSH rule 5 x 641 [8,2]
+ CRUSH rule 5 x 642 [2,8]
+ CRUSH rule 5 x 643 [1,8]
+ CRUSH rule 5 x 644 [8,1]
+ CRUSH rule 5 x 645 [8,1]
+ CRUSH rule 5 x 646 [8,1]
+ CRUSH rule 5 x 647 [8,1]
+ CRUSH rule 5 x 648 [1,8]
+ CRUSH rule 5 x 649 [4,8]
+ CRUSH rule 5 x 650 [8,4]
+ CRUSH rule 5 x 651 [4,6]
+ CRUSH rule 5 x 652 [4,8]
+ CRUSH rule 5 x 653 [8,2]
+ CRUSH rule 5 x 654 [6,2]
+ CRUSH rule 5 x 655 [1,4]
+ CRUSH rule 5 x 656 [8,1]
+ CRUSH rule 5 x 657 [6,1]
+ CRUSH rule 5 x 658 [8,2]
+ CRUSH rule 5 x 659 [4,8]
+ CRUSH rule 5 x 660 [8,2]
+ CRUSH rule 5 x 661 [1,8]
+ CRUSH rule 5 x 662 [8,2]
+ CRUSH rule 5 x 663 [1,4]
+ CRUSH rule 5 x 664 [1,4]
+ CRUSH rule 5 x 665 [4,6]
+ CRUSH rule 5 x 666 [2,8]
+ CRUSH rule 5 x 667 [1,4]
+ CRUSH rule 5 x 668 [4,8]
+ CRUSH rule 5 x 669 [6,4]
+ CRUSH rule 5 x 670 [4,2]
+ CRUSH rule 5 x 671 [2,8]
+ CRUSH rule 5 x 672 [4,2]
+ CRUSH rule 5 x 673 [4,2]
+ CRUSH rule 5 x 674 [1,8]
+ CRUSH rule 5 x 675 [1,8]
+ CRUSH rule 5 x 676 [2,4]
+ CRUSH rule 5 x 677 [4,1]
+ CRUSH rule 5 x 678 [2,4]
+ CRUSH rule 5 x 679 [8,2]
+ CRUSH rule 5 x 680 [2,8]
+ CRUSH rule 5 x 681 [8,1]
+ CRUSH rule 5 x 682 [1,4]
+ CRUSH rule 5 x 683 [1,4]
+ CRUSH rule 5 x 684 [8,1]
+ CRUSH rule 5 x 685 [8,1]
+ CRUSH rule 5 x 686 [1,4]
+ CRUSH rule 5 x 687 [6,1]
+ CRUSH rule 5 x 688 [4,8]
+ CRUSH rule 5 x 689 [8,4]
+ CRUSH rule 5 x 690 [8,1]
+ CRUSH rule 5 x 691 [1,8]
+ CRUSH rule 5 x 692 [8,2]
+ CRUSH rule 5 x 693 [8,4]
+ CRUSH rule 5 x 694 [8,4]
+ CRUSH rule 5 x 695 [2,8]
+ CRUSH rule 5 x 696 [1,8]
+ CRUSH rule 5 x 697 [8,1]
+ CRUSH rule 5 x 698 [8,2]
+ CRUSH rule 5 x 699 [1,8]
+ CRUSH rule 5 x 700 [1,8]
+ CRUSH rule 5 x 701 [1,8]
+ CRUSH rule 5 x 702 [2,8]
+ CRUSH rule 5 x 703 [8,1]
+ CRUSH rule 5 x 704 [1,4]
+ CRUSH rule 5 x 705 [8,1]
+ CRUSH rule 5 x 706 [1,4]
+ CRUSH rule 5 x 707 [8,4]
+ CRUSH rule 5 x 708 [4,8]
+ CRUSH rule 5 x 709 [8,2]
+ CRUSH rule 5 x 710 [8,2]
+ CRUSH rule 5 x 711 [2,4]
+ CRUSH rule 5 x 712 [2,8]
+ CRUSH rule 5 x 713 [8,4]
+ CRUSH rule 5 x 714 [2,8]
+ CRUSH rule 5 x 715 [1,8]
+ CRUSH rule 5 x 716 [4,8]
+ CRUSH rule 5 x 717 [8,2]
+ CRUSH rule 5 x 718 [8,1]
+ CRUSH rule 5 x 719 [2,6]
+ CRUSH rule 5 x 720 [8,1]
+ CRUSH rule 5 x 721 [4,6]
+ CRUSH rule 5 x 722 [8,2]
+ CRUSH rule 5 x 723 [4,1]
+ CRUSH rule 5 x 724 [2,6]
+ CRUSH rule 5 x 725 [1,8]
+ CRUSH rule 5 x 726 [4,8]
+ CRUSH rule 5 x 727 [4,8]
+ CRUSH rule 5 x 728 [2,8]
+ CRUSH rule 5 x 729 [8,2]
+ CRUSH rule 5 x 730 [4,8]
+ CRUSH rule 5 x 731 [4,1]
+ CRUSH rule 5 x 732 [1,8]
+ CRUSH rule 5 x 733 [4,8]
+ CRUSH rule 5 x 734 [8,4]
+ CRUSH rule 5 x 735 [4,8]
+ CRUSH rule 5 x 736 [4,8]
+ CRUSH rule 5 x 737 [1,8]
+ CRUSH rule 5 x 738 [4,2]
+ CRUSH rule 5 x 739 [2,8]
+ CRUSH rule 5 x 740 [1,8]
+ CRUSH rule 5 x 741 [8,1]
+ CRUSH rule 5 x 742 [8,2]
+ CRUSH rule 5 x 743 [8,1]
+ CRUSH rule 5 x 744 [4,8]
+ CRUSH rule 5 x 745 [1,8]
+ CRUSH rule 5 x 746 [1,8]
+ CRUSH rule 5 x 747 [8,1]
+ CRUSH rule 5 x 748 [2,8]
+ CRUSH rule 5 x 749 [4,8]
+ CRUSH rule 5 x 750 [1,8]
+ CRUSH rule 5 x 751 [2,8]
+ CRUSH rule 5 x 752 [8,1]
+ CRUSH rule 5 x 753 [8,4]
+ CRUSH rule 5 x 754 [8,4]
+ CRUSH rule 5 x 755 [1,8]
+ CRUSH rule 5 x 756 [8,1]
+ CRUSH rule 5 x 757 [8,1]
+ CRUSH rule 5 x 758 [8,2]
+ CRUSH rule 5 x 759 [8,4]
+ CRUSH rule 5 x 760 [1,4]
+ CRUSH rule 5 x 761 [2,8]
+ CRUSH rule 5 x 762 [2,8]
+ CRUSH rule 5 x 763 [8,4]
+ CRUSH rule 5 x 764 [1,8]
+ CRUSH rule 5 x 765 [8,2]
+ CRUSH rule 5 x 766 [8,1]
+ CRUSH rule 5 x 767 [1,8]
+ CRUSH rule 5 x 768 [8,4]
+ CRUSH rule 5 x 769 [8,2]
+ CRUSH rule 5 x 770 [8,2]
+ CRUSH rule 5 x 771 [8,1]
+ CRUSH rule 5 x 772 [8,4]
+ CRUSH rule 5 x 773 [4,1]
+ CRUSH rule 5 x 774 [8,1]
+ CRUSH rule 5 x 775 [8,4]
+ CRUSH rule 5 x 776 [6,2]
+ CRUSH rule 5 x 777 [4,1]
+ CRUSH rule 5 x 778 [1,8]
+ CRUSH rule 5 x 779 [2,8]
+ CRUSH rule 5 x 780 [2,4]
+ CRUSH rule 5 x 781 [8,2]
+ CRUSH rule 5 x 782 [4,1]
+ CRUSH rule 5 x 783 [8,1]
+ CRUSH rule 5 x 784 [1,4]
+ CRUSH rule 5 x 785 [8,1]
+ CRUSH rule 5 x 786 [8,1]
+ CRUSH rule 5 x 787 [1,6]
+ CRUSH rule 5 x 788 [8,2]
+ CRUSH rule 5 x 789 [1,8]
+ CRUSH rule 5 x 790 [8,1]
+ CRUSH rule 5 x 791 [4,8]
+ CRUSH rule 5 x 792 [4,8]
+ CRUSH rule 5 x 793 [8,1]
+ CRUSH rule 5 x 794 [2,8]
+ CRUSH rule 5 x 795 [1,8]
+ CRUSH rule 5 x 796 [8,2]
+ CRUSH rule 5 x 797 [2,4]
+ CRUSH rule 5 x 798 [6,1]
+ CRUSH rule 5 x 799 [4,1]
+ CRUSH rule 5 x 800 [2,8]
+ CRUSH rule 5 x 801 [4,8]
+ CRUSH rule 5 x 802 [1,8]
+ CRUSH rule 5 x 803 [2,8]
+ CRUSH rule 5 x 804 [8,2]
+ CRUSH rule 5 x 805 [8,2]
+ CRUSH rule 5 x 806 [1,4]
+ CRUSH rule 5 x 807 [4,8]
+ CRUSH rule 5 x 808 [8,2]
+ CRUSH rule 5 x 809 [1,8]
+ CRUSH rule 5 x 810 [8,2]
+ CRUSH rule 5 x 811 [8,1]
+ CRUSH rule 5 x 812 [8,4]
+ CRUSH rule 5 x 813 [8,4]
+ CRUSH rule 5 x 814 [8,2]
+ CRUSH rule 5 x 815 [4,1]
+ CRUSH rule 5 x 816 [2,8]
+ CRUSH rule 5 x 817 [8,1]
+ CRUSH rule 5 x 818 [1,8]
+ CRUSH rule 5 x 819 [1,8]
+ CRUSH rule 5 x 820 [4,8]
+ CRUSH rule 5 x 821 [4,8]
+ CRUSH rule 5 x 822 [2,4]
+ CRUSH rule 5 x 823 [4,8]
+ CRUSH rule 5 x 824 [8,2]
+ CRUSH rule 5 x 825 [2,8]
+ CRUSH rule 5 x 826 [8,2]
+ CRUSH rule 5 x 827 [2,8]
+ CRUSH rule 5 x 828 [2,8]
+ CRUSH rule 5 x 829 [8,1]
+ CRUSH rule 5 x 830 [2,4]
+ CRUSH rule 5 x 831 [1,8]
+ CRUSH rule 5 x 832 [4,8]
+ CRUSH rule 5 x 833 [2,8]
+ CRUSH rule 5 x 834 [1,8]
+ CRUSH rule 5 x 835 [8,4]
+ CRUSH rule 5 x 836 [4,8]
+ CRUSH rule 5 x 837 [8,4]
+ CRUSH rule 5 x 838 [6,2]
+ CRUSH rule 5 x 839 [2,8]
+ CRUSH rule 5 x 840 [8,1]
+ CRUSH rule 5 x 841 [4,8]
+ CRUSH rule 5 x 842 [2,8]
+ CRUSH rule 5 x 843 [8,4]
+ CRUSH rule 5 x 844 [8,2]
+ CRUSH rule 5 x 845 [4,8]
+ CRUSH rule 5 x 846 [4,2]
+ CRUSH rule 5 x 847 [2,8]
+ CRUSH rule 5 x 848 [2,8]
+ CRUSH rule 5 x 849 [4,8]
+ CRUSH rule 5 x 850 [1,6]
+ CRUSH rule 5 x 851 [6,1]
+ CRUSH rule 5 x 852 [8,4]
+ CRUSH rule 5 x 853 [6,1]
+ CRUSH rule 5 x 854 [8,1]
+ CRUSH rule 5 x 855 [8,1]
+ CRUSH rule 5 x 856 [8,4]
+ CRUSH rule 5 x 857 [8,2]
+ CRUSH rule 5 x 858 [6,1]
+ CRUSH rule 5 x 859 [8,2]
+ CRUSH rule 5 x 860 [2,8]
+ CRUSH rule 5 x 861 [8,2]
+ CRUSH rule 5 x 862 [8,1]
+ CRUSH rule 5 x 863 [8,2]
+ CRUSH rule 5 x 864 [8,2]
+ CRUSH rule 5 x 865 [8,1]
+ CRUSH rule 5 x 866 [8,2]
+ CRUSH rule 5 x 867 [8,2]
+ CRUSH rule 5 x 868 [8,1]
+ CRUSH rule 5 x 869 [8,4]
+ CRUSH rule 5 x 870 [2,8]
+ CRUSH rule 5 x 871 [1,8]
+ CRUSH rule 5 x 872 [1,8]
+ CRUSH rule 5 x 873 [4,8]
+ CRUSH rule 5 x 874 [2,6]
+ CRUSH rule 5 x 875 [2,8]
+ CRUSH rule 5 x 876 [4,8]
+ CRUSH rule 5 x 877 [8,4]
+ CRUSH rule 5 x 878 [2,8]
+ CRUSH rule 5 x 879 [8,1]
+ CRUSH rule 5 x 880 [1,8]
+ CRUSH rule 5 x 881 [4,8]
+ CRUSH rule 5 x 882 [1,8]
+ CRUSH rule 5 x 883 [2,4]
+ CRUSH rule 5 x 884 [8,2]
+ CRUSH rule 5 x 885 [4,1]
+ CRUSH rule 5 x 886 [8,2]
+ CRUSH rule 5 x 887 [8,4]
+ CRUSH rule 5 x 888 [8,2]
+ CRUSH rule 5 x 889 [2,6]
+ CRUSH rule 5 x 890 [8,2]
+ CRUSH rule 5 x 891 [1,8]
+ CRUSH rule 5 x 892 [8,2]
+ CRUSH rule 5 x 893 [2,6]
+ CRUSH rule 5 x 894 [8,4]
+ CRUSH rule 5 x 895 [4,1]
+ CRUSH rule 5 x 896 [1,8]
+ CRUSH rule 5 x 897 [2,8]
+ CRUSH rule 5 x 898 [1,4]
+ CRUSH rule 5 x 899 [1,8]
+ CRUSH rule 5 x 900 [4,1]
+ CRUSH rule 5 x 901 [2,8]
+ CRUSH rule 5 x 902 [8,4]
+ CRUSH rule 5 x 903 [8,2]
+ CRUSH rule 5 x 904 [8,2]
+ CRUSH rule 5 x 905 [8,2]
+ CRUSH rule 5 x 906 [1,8]
+ CRUSH rule 5 x 907 [8,1]
+ CRUSH rule 5 x 908 [8,1]
+ CRUSH rule 5 x 909 [2,8]
+ CRUSH rule 5 x 910 [8,2]
+ CRUSH rule 5 x 911 [8,1]
+ CRUSH rule 5 x 912 [1,8]
+ CRUSH rule 5 x 913 [8,2]
+ CRUSH rule 5 x 914 [6,4]
+ CRUSH rule 5 x 915 [8,2]
+ CRUSH rule 5 x 916 [4,1]
+ CRUSH rule 5 x 917 [1,4]
+ CRUSH rule 5 x 918 [8,2]
+ CRUSH rule 5 x 919 [8,2]
+ CRUSH rule 5 x 920 [8,1]
+ CRUSH rule 5 x 921 [1,8]
+ CRUSH rule 5 x 922 [8,4]
+ CRUSH rule 5 x 923 [4,8]
+ CRUSH rule 5 x 924 [8,1]
+ CRUSH rule 5 x 925 [4,8]
+ CRUSH rule 5 x 926 [2,8]
+ CRUSH rule 5 x 927 [1,8]
+ CRUSH rule 5 x 928 [8,1]
+ CRUSH rule 5 x 929 [4,1]
+ CRUSH rule 5 x 930 [2,8]
+ CRUSH rule 5 x 931 [2,8]
+ CRUSH rule 5 x 932 [4,1]
+ CRUSH rule 5 x 933 [8,4]
+ CRUSH rule 5 x 934 [8,2]
+ CRUSH rule 5 x 935 [8,2]
+ CRUSH rule 5 x 936 [1,8]
+ CRUSH rule 5 x 937 [4,8]
+ CRUSH rule 5 x 938 [8,4]
+ CRUSH rule 5 x 939 [2,8]
+ CRUSH rule 5 x 940 [8,1]
+ CRUSH rule 5 x 941 [2,8]
+ CRUSH rule 5 x 942 [1,8]
+ CRUSH rule 5 x 943 [8,2]
+ CRUSH rule 5 x 944 [8,2]
+ CRUSH rule 5 x 945 [8,2]
+ CRUSH rule 5 x 946 [2,8]
+ CRUSH rule 5 x 947 [8,2]
+ CRUSH rule 5 x 948 [8,1]
+ CRUSH rule 5 x 949 [6,1]
+ CRUSH rule 5 x 950 [8,1]
+ CRUSH rule 5 x 951 [8,1]
+ CRUSH rule 5 x 952 [2,8]
+ CRUSH rule 5 x 953 [1,4]
+ CRUSH rule 5 x 954 [2,8]
+ CRUSH rule 5 x 955 [8,1]
+ CRUSH rule 5 x 956 [1,8]
+ CRUSH rule 5 x 957 [8,1]
+ CRUSH rule 5 x 958 [8,4]
+ CRUSH rule 5 x 959 [4,2]
+ CRUSH rule 5 x 960 [6,1]
+ CRUSH rule 5 x 961 [1,8]
+ CRUSH rule 5 x 962 [8,4]
+ CRUSH rule 5 x 963 [2,4]
+ CRUSH rule 5 x 964 [2,8]
+ CRUSH rule 5 x 965 [8,2]
+ CRUSH rule 5 x 966 [4,8]
+ CRUSH rule 5 x 967 [8,4]
+ CRUSH rule 5 x 968 [8,2]
+ CRUSH rule 5 x 969 [8,2]
+ CRUSH rule 5 x 970 [2,8]
+ CRUSH rule 5 x 971 [1,8]
+ CRUSH rule 5 x 972 [1,8]
+ CRUSH rule 5 x 973 [1,8]
+ CRUSH rule 5 x 974 [4,1]
+ CRUSH rule 5 x 975 [4,8]
+ CRUSH rule 5 x 976 [4,8]
+ CRUSH rule 5 x 977 [8,4]
+ CRUSH rule 5 x 978 [8,2]
+ CRUSH rule 5 x 979 [8,1]
+ CRUSH rule 5 x 980 [8,2]
+ CRUSH rule 5 x 981 [8,2]
+ CRUSH rule 5 x 982 [1,8]
+ CRUSH rule 5 x 983 [4,8]
+ CRUSH rule 5 x 984 [2,8]
+ CRUSH rule 5 x 985 [2,4]
+ CRUSH rule 5 x 986 [8,4]
+ CRUSH rule 5 x 987 [2,8]
+ CRUSH rule 5 x 988 [1,4]
+ CRUSH rule 5 x 989 [1,8]
+ CRUSH rule 5 x 990 [1,8]
+ CRUSH rule 5 x 991 [1,4]
+ CRUSH rule 5 x 992 [8,1]
+ CRUSH rule 5 x 993 [2,8]
+ CRUSH rule 5 x 994 [4,8]
+ CRUSH rule 5 x 995 [8,1]
+ CRUSH rule 5 x 996 [8,4]
+ CRUSH rule 5 x 997 [8,4]
+ CRUSH rule 5 x 998 [8,1]
+ CRUSH rule 5 x 999 [1,8]
+ CRUSH rule 5 x 1000 [8,4]
+ CRUSH rule 5 x 1001 [2,8]
+ CRUSH rule 5 x 1002 [1,8]
+ CRUSH rule 5 x 1003 [2,8]
+ CRUSH rule 5 x 1004 [8,1]
+ CRUSH rule 5 x 1005 [8,1]
+ CRUSH rule 5 x 1006 [1,8]
+ CRUSH rule 5 x 1007 [1,4]
+ CRUSH rule 5 x 1008 [1,8]
+ CRUSH rule 5 x 1009 [6,4]
+ CRUSH rule 5 x 1010 [1,8]
+ CRUSH rule 5 x 1011 [4,2]
+ CRUSH rule 5 x 1012 [1,8]
+ CRUSH rule 5 x 1013 [2,8]
+ CRUSH rule 5 x 1014 [2,8]
+ CRUSH rule 5 x 1015 [8,1]
+ CRUSH rule 5 x 1016 [2,4]
+ CRUSH rule 5 x 1017 [6,2]
+ CRUSH rule 5 x 1018 [4,1]
+ CRUSH rule 5 x 1019 [4,8]
+ CRUSH rule 5 x 1020 [1,8]
+ CRUSH rule 5 x 1021 [2,8]
+ CRUSH rule 5 x 1022 [1,8]
+ CRUSH rule 5 x 1023 [4,2]
+ rule 5 (chooseleaf-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 5 x 0 [2,4,8]
+ CRUSH rule 5 x 1 [2,8,4]
+ CRUSH rule 5 x 2 [1,8]
+ CRUSH rule 5 x 3 [8,1]
+ CRUSH rule 5 x 4 [4,2,6]
+ CRUSH rule 5 x 5 [8,2]
+ CRUSH rule 5 x 6 [2,8,4]
+ CRUSH rule 5 x 7 [4,8,2]
+ CRUSH rule 5 x 8 [4,8,1]
+ CRUSH rule 5 x 9 [2,4,8]
+ CRUSH rule 5 x 10 [2,8]
+ CRUSH rule 5 x 11 [2,8]
+ CRUSH rule 5 x 12 [2,8]
+ CRUSH rule 5 x 13 [4,8,1]
+ CRUSH rule 5 x 14 [8,2]
+ CRUSH rule 5 x 15 [8,2]
+ CRUSH rule 5 x 16 [8,2]
+ CRUSH rule 5 x 17 [4,1,8]
+ CRUSH rule 5 x 18 [1,8]
+ CRUSH rule 5 x 19 [8,4,2]
+ CRUSH rule 5 x 20 [2,8]
+ CRUSH rule 5 x 21 [8,2]
+ CRUSH rule 5 x 22 [8,1]
+ CRUSH rule 5 x 23 [4,8,2]
+ CRUSH rule 5 x 24 [1,8,4]
+ CRUSH rule 5 x 25 [4,8,1]
+ CRUSH rule 5 x 26 [2,8,4]
+ CRUSH rule 5 x 27 [4,1,8]
+ CRUSH rule 5 x 28 [8,2]
+ CRUSH rule 5 x 29 [8,4,2]
+ CRUSH rule 5 x 30 [4,8,2]
+ CRUSH rule 5 x 31 [8,1]
+ CRUSH rule 5 x 32 [6,1]
+ CRUSH rule 5 x 33 [2,8]
+ CRUSH rule 5 x 34 [2,8]
+ CRUSH rule 5 x 35 [1,8,4]
+ CRUSH rule 5 x 36 [8,2]
+ CRUSH rule 5 x 37 [1,8]
+ CRUSH rule 5 x 38 [4,8,2]
+ CRUSH rule 5 x 39 [8,2]
+ CRUSH rule 5 x 40 [8,2,4]
+ CRUSH rule 5 x 41 [2,8,4]
+ CRUSH rule 5 x 42 [8,2]
+ CRUSH rule 5 x 43 [1,8]
+ CRUSH rule 5 x 44 [1,8,4]
+ CRUSH rule 5 x 45 [8,2,4]
+ CRUSH rule 5 x 46 [2,8]
+ CRUSH rule 5 x 47 [4,2,8]
+ CRUSH rule 5 x 48 [8,1]
+ CRUSH rule 5 x 49 [8,2]
+ CRUSH rule 5 x 50 [4,1,8]
+ CRUSH rule 5 x 51 [8,2]
+ CRUSH rule 5 x 52 [8,1,4]
+ CRUSH rule 5 x 53 [4,8,2]
+ CRUSH rule 5 x 54 [8,4,1]
+ CRUSH rule 5 x 55 [8,2]
+ CRUSH rule 5 x 56 [8,4,2]
+ CRUSH rule 5 x 57 [8,1]
+ CRUSH rule 5 x 58 [1,8]
+ CRUSH rule 5 x 59 [2,8]
+ CRUSH rule 5 x 60 [4,2,8]
+ CRUSH rule 5 x 61 [4,8,2]
+ CRUSH rule 5 x 62 [8,1]
+ CRUSH rule 5 x 63 [8,2]
+ CRUSH rule 5 x 64 [4,2,8]
+ CRUSH rule 5 x 65 [8,4,2]
+ CRUSH rule 5 x 66 [4,8,2]
+ CRUSH rule 5 x 67 [4,2,8]
+ CRUSH rule 5 x 68 [1,8]
+ CRUSH rule 5 x 69 [2,8]
+ CRUSH rule 5 x 70 [8,2]
+ CRUSH rule 5 x 71 [2,8,4]
+ CRUSH rule 5 x 72 [8,1,4]
+ CRUSH rule 5 x 73 [2,8]
+ CRUSH rule 5 x 74 [1,8]
+ CRUSH rule 5 x 75 [4,2,8]
+ CRUSH rule 5 x 76 [4,1,6]
+ CRUSH rule 5 x 77 [8,2,4]
+ CRUSH rule 5 x 78 [1,6]
+ CRUSH rule 5 x 79 [4,1,8]
+ CRUSH rule 5 x 80 [2,4,8]
+ CRUSH rule 5 x 81 [2,8]
+ CRUSH rule 5 x 82 [6,1]
+ CRUSH rule 5 x 83 [2,8]
+ CRUSH rule 5 x 84 [8,2]
+ CRUSH rule 5 x 85 [4,8,1]
+ CRUSH rule 5 x 86 [2,8]
+ CRUSH rule 5 x 87 [2,8,4]
+ CRUSH rule 5 x 88 [1,6]
+ CRUSH rule 5 x 89 [1,8]
+ CRUSH rule 5 x 90 [8,4,1]
+ CRUSH rule 5 x 91 [4,8,1]
+ CRUSH rule 5 x 92 [1,8]
+ CRUSH rule 5 x 93 [8,4,2]
+ CRUSH rule 5 x 94 [1,8]
+ CRUSH rule 5 x 95 [8,1]
+ CRUSH rule 5 x 96 [8,2]
+ CRUSH rule 5 x 97 [8,1]
+ CRUSH rule 5 x 98 [2,8]
+ CRUSH rule 5 x 99 [2,8]
+ CRUSH rule 5 x 100 [1,8,4]
+ CRUSH rule 5 x 101 [8,1]
+ CRUSH rule 5 x 102 [2,8]
+ CRUSH rule 5 x 103 [8,2]
+ CRUSH rule 5 x 104 [8,4,1]
+ CRUSH rule 5 x 105 [2,4,8]
+ CRUSH rule 5 x 106 [1,8,4]
+ CRUSH rule 5 x 107 [1,8]
+ CRUSH rule 5 x 108 [8,2]
+ CRUSH rule 5 x 109 [1,4,8]
+ CRUSH rule 5 x 110 [4,2,8]
+ CRUSH rule 5 x 111 [2,4,8]
+ CRUSH rule 5 x 112 [2,8]
+ CRUSH rule 5 x 113 [8,2]
+ CRUSH rule 5 x 114 [8,4,1]
+ CRUSH rule 5 x 115 [8,2,4]
+ CRUSH rule 5 x 116 [1,8]
+ CRUSH rule 5 x 117 [6,1]
+ CRUSH rule 5 x 118 [2,8]
+ CRUSH rule 5 x 119 [8,1]
+ CRUSH rule 5 x 120 [2,4,8]
+ CRUSH rule 5 x 121 [2,8,4]
+ CRUSH rule 5 x 122 [8,1]
+ CRUSH rule 5 x 123 [2,8]
+ CRUSH rule 5 x 124 [2,8]
+ CRUSH rule 5 x 125 [1,8,4]
+ CRUSH rule 5 x 126 [1,8]
+ CRUSH rule 5 x 127 [4,8,2]
+ CRUSH rule 5 x 128 [8,2]
+ CRUSH rule 5 x 129 [2,4,8]
+ CRUSH rule 5 x 130 [4,8,2]
+ CRUSH rule 5 x 131 [1,4,8]
+ CRUSH rule 5 x 132 [1,8]
+ CRUSH rule 5 x 133 [8,1]
+ CRUSH rule 5 x 134 [1,8,4]
+ CRUSH rule 5 x 135 [4,8,2]
+ CRUSH rule 5 x 136 [2,4,8]
+ CRUSH rule 5 x 137 [8,4,2]
+ CRUSH rule 5 x 138 [8,4,2]
+ CRUSH rule 5 x 139 [4,2,8]
+ CRUSH rule 5 x 140 [1,8,4]
+ CRUSH rule 5 x 141 [8,2]
+ CRUSH rule 5 x 142 [4,1,8]
+ CRUSH rule 5 x 143 [4,8,1]
+ CRUSH rule 5 x 144 [8,1]
+ CRUSH rule 5 x 145 [8,1]
+ CRUSH rule 5 x 146 [2,8]
+ CRUSH rule 5 x 147 [2,8,4]
+ CRUSH rule 5 x 148 [4,1,8]
+ CRUSH rule 5 x 149 [4,8,1]
+ CRUSH rule 5 x 150 [1,8]
+ CRUSH rule 5 x 151 [1,8]
+ CRUSH rule 5 x 152 [8,2]
+ CRUSH rule 5 x 153 [8,4,2]
+ CRUSH rule 5 x 154 [4,2,8]
+ CRUSH rule 5 x 155 [4,8,2]
+ CRUSH rule 5 x 156 [4,2,8]
+ CRUSH rule 5 x 157 [1,8]
+ CRUSH rule 5 x 158 [2,8,4]
+ CRUSH rule 5 x 159 [8,2,4]
+ CRUSH rule 5 x 160 [2,8,4]
+ CRUSH rule 5 x 161 [1,4,8]
+ CRUSH rule 5 x 162 [1,8]
+ CRUSH rule 5 x 163 [4,8,1]
+ CRUSH rule 5 x 164 [8,1]
+ CRUSH rule 5 x 165 [8,2,4]
+ CRUSH rule 5 x 166 [2,8]
+ CRUSH rule 5 x 167 [1,8,4]
+ CRUSH rule 5 x 168 [4,2,8]
+ CRUSH rule 5 x 169 [2,8,4]
+ CRUSH rule 5 x 170 [1,8]
+ CRUSH rule 5 x 171 [8,4,2]
+ CRUSH rule 5 x 172 [1,8]
+ CRUSH rule 5 x 173 [8,4,1]
+ CRUSH rule 5 x 174 [1,6]
+ CRUSH rule 5 x 175 [8,1]
+ CRUSH rule 5 x 176 [2,8]
+ CRUSH rule 5 x 177 [1,8]
+ CRUSH rule 5 x 178 [4,2,8]
+ CRUSH rule 5 x 179 [1,8]
+ CRUSH rule 5 x 180 [8,1]
+ CRUSH rule 5 x 181 [8,2,4]
+ CRUSH rule 5 x 182 [8,1]
+ CRUSH rule 5 x 183 [8,4,1]
+ CRUSH rule 5 x 184 [4,8,2]
+ CRUSH rule 5 x 185 [8,1,4]
+ CRUSH rule 5 x 186 [2,4,8]
+ CRUSH rule 5 x 187 [1,8]
+ CRUSH rule 5 x 188 [1,8,4]
+ CRUSH rule 5 x 189 [1,8,4]
+ CRUSH rule 5 x 190 [1,8]
+ CRUSH rule 5 x 191 [8,1,4]
+ CRUSH rule 5 x 192 [4,1,8]
+ CRUSH rule 5 x 193 [4,2,8]
+ CRUSH rule 5 x 194 [1,8]
+ CRUSH rule 5 x 195 [8,4,1]
+ CRUSH rule 5 x 196 [8,2]
+ CRUSH rule 5 x 197 [8,4,2]
+ CRUSH rule 5 x 198 [2,8]
+ CRUSH rule 5 x 199 [1,4,8]
+ CRUSH rule 5 x 200 [1,8]
+ CRUSH rule 5 x 201 [8,1,4]
+ CRUSH rule 5 x 202 [8,1]
+ CRUSH rule 5 x 203 [8,1]
+ CRUSH rule 5 x 204 [2,4,8]
+ CRUSH rule 5 x 205 [1,8]
+ CRUSH rule 5 x 206 [1,8,4]
+ CRUSH rule 5 x 207 [2,8]
+ CRUSH rule 5 x 208 [8,1]
+ CRUSH rule 5 x 209 [1,8]
+ CRUSH rule 5 x 210 [1,4,8]
+ CRUSH rule 5 x 211 [4,2,8]
+ CRUSH rule 5 x 212 [8,1]
+ CRUSH rule 5 x 213 [8,4,2]
+ CRUSH rule 5 x 214 [8,2]
+ CRUSH rule 5 x 215 [8,1]
+ CRUSH rule 5 x 216 [2,8]
+ CRUSH rule 5 x 217 [1,8,4]
+ CRUSH rule 5 x 218 [2,8]
+ CRUSH rule 5 x 219 [8,2]
+ CRUSH rule 5 x 220 [4,8,1]
+ CRUSH rule 5 x 221 [8,1]
+ CRUSH rule 5 x 222 [8,1]
+ CRUSH rule 5 x 223 [1,8]
+ CRUSH rule 5 x 224 [1,4,8]
+ CRUSH rule 5 x 225 [8,2]
+ CRUSH rule 5 x 226 [8,2,4]
+ CRUSH rule 5 x 227 [4,1,8]
+ CRUSH rule 5 x 228 [8,2]
+ CRUSH rule 5 x 229 [4,8,2]
+ CRUSH rule 5 x 230 [4,8,2]
+ CRUSH rule 5 x 231 [4,8,2]
+ CRUSH rule 5 x 232 [2,8,4]
+ CRUSH rule 5 x 233 [2,8]
+ CRUSH rule 5 x 234 [1,8]
+ CRUSH rule 5 x 235 [4,8,1]
+ CRUSH rule 5 x 236 [2,6]
+ CRUSH rule 5 x 237 [4,8,1]
+ CRUSH rule 5 x 238 [2,8]
+ CRUSH rule 5 x 239 [8,1]
+ CRUSH rule 5 x 240 [4,8,2]
+ CRUSH rule 5 x 241 [1,8]
+ CRUSH rule 5 x 242 [8,2]
+ CRUSH rule 5 x 243 [8,2]
+ CRUSH rule 5 x 244 [4,8,2]
+ CRUSH rule 5 x 245 [8,1]
+ CRUSH rule 5 x 246 [1,8]
+ CRUSH rule 5 x 247 [8,2]
+ CRUSH rule 5 x 248 [8,2,4]
+ CRUSH rule 5 x 249 [2,8]
+ CRUSH rule 5 x 250 [2,4,6]
+ CRUSH rule 5 x 251 [2,8]
+ CRUSH rule 5 x 252 [4,8,1]
+ CRUSH rule 5 x 253 [2,8]
+ CRUSH rule 5 x 254 [4,2,8]
+ CRUSH rule 5 x 255 [1,8]
+ CRUSH rule 5 x 256 [4,8,1]
+ CRUSH rule 5 x 257 [2,8,4]
+ CRUSH rule 5 x 258 [4,2,8]
+ CRUSH rule 5 x 259 [6,2]
+ CRUSH rule 5 x 260 [8,2]
+ CRUSH rule 5 x 261 [8,1]
+ CRUSH rule 5 x 262 [8,1]
+ CRUSH rule 5 x 263 [8,1,4]
+ CRUSH rule 5 x 264 [8,2]
+ CRUSH rule 5 x 265 [8,2]
+ CRUSH rule 5 x 266 [8,2,4]
+ CRUSH rule 5 x 267 [2,8]
+ CRUSH rule 5 x 268 [1,8]
+ CRUSH rule 5 x 269 [1,8,4]
+ CRUSH rule 5 x 270 [4,1,8]
+ CRUSH rule 5 x 271 [8,4,1]
+ CRUSH rule 5 x 272 [2,8,4]
+ CRUSH rule 5 x 273 [4,1,8]
+ CRUSH rule 5 x 274 [8,4,1]
+ CRUSH rule 5 x 275 [4,8,1]
+ CRUSH rule 5 x 276 [8,1,4]
+ CRUSH rule 5 x 277 [8,1]
+ CRUSH rule 5 x 278 [8,1,4]
+ CRUSH rule 5 x 279 [8,4,2]
+ CRUSH rule 5 x 280 [2,8,4]
+ CRUSH rule 5 x 281 [8,2]
+ CRUSH rule 5 x 282 [2,8]
+ CRUSH rule 5 x 283 [8,2]
+ CRUSH rule 5 x 284 [8,2]
+ CRUSH rule 5 x 285 [4,8,2]
+ CRUSH rule 5 x 286 [2,8,4]
+ CRUSH rule 5 x 287 [1,8]
+ CRUSH rule 5 x 288 [8,1,4]
+ CRUSH rule 5 x 289 [4,8,2]
+ CRUSH rule 5 x 290 [1,4,8]
+ CRUSH rule 5 x 291 [1,4,8]
+ CRUSH rule 5 x 292 [8,2,4]
+ CRUSH rule 5 x 293 [8,1]
+ CRUSH rule 5 x 294 [8,4,1]
+ CRUSH rule 5 x 295 [4,8,2]
+ CRUSH rule 5 x 296 [4,1,8]
+ CRUSH rule 5 x 297 [8,2,4]
+ CRUSH rule 5 x 298 [1,8]
+ CRUSH rule 5 x 299 [2,8]
+ CRUSH rule 5 x 300 [8,2]
+ CRUSH rule 5 x 301 [1,8]
+ CRUSH rule 5 x 302 [1,8]
+ CRUSH rule 5 x 303 [8,4,1]
+ CRUSH rule 5 x 304 [2,8]
+ CRUSH rule 5 x 305 [8,2]
+ CRUSH rule 5 x 306 [1,8]
+ CRUSH rule 5 x 307 [2,8]
+ CRUSH rule 5 x 308 [2,8,4]
+ CRUSH rule 5 x 309 [8,1]
+ CRUSH rule 5 x 310 [4,1,6]
+ CRUSH rule 5 x 311 [4,8,1]
+ CRUSH rule 5 x 312 [2,8]
+ CRUSH rule 5 x 313 [4,1,8]
+ CRUSH rule 5 x 314 [2,8]
+ CRUSH rule 5 x 315 [2,8]
+ CRUSH rule 5 x 316 [8,1]
+ CRUSH rule 5 x 317 [2,8]
+ CRUSH rule 5 x 318 [8,1]
+ CRUSH rule 5 x 319 [2,8]
+ CRUSH rule 5 x 320 [8,1]
+ CRUSH rule 5 x 321 [1,8]
+ CRUSH rule 5 x 322 [2,8,4]
+ CRUSH rule 5 x 323 [4,8,1]
+ CRUSH rule 5 x 324 [8,1,4]
+ CRUSH rule 5 x 325 [4,8,2]
+ CRUSH rule 5 x 326 [1,6]
+ CRUSH rule 5 x 327 [1,8]
+ CRUSH rule 5 x 328 [8,4,1]
+ CRUSH rule 5 x 329 [4,8,2]
+ CRUSH rule 5 x 330 [4,8,2]
+ CRUSH rule 5 x 331 [2,8]
+ CRUSH rule 5 x 332 [2,8]
+ CRUSH rule 5 x 333 [8,1]
+ CRUSH rule 5 x 334 [8,2]
+ CRUSH rule 5 x 335 [8,1]
+ CRUSH rule 5 x 336 [4,8,2]
+ CRUSH rule 5 x 337 [8,2,4]
+ CRUSH rule 5 x 338 [8,1]
+ CRUSH rule 5 x 339 [8,2]
+ CRUSH rule 5 x 340 [2,8,4]
+ CRUSH rule 5 x 341 [4,1,8]
+ CRUSH rule 5 x 342 [2,8,4]
+ CRUSH rule 5 x 343 [8,1]
+ CRUSH rule 5 x 344 [6,2,4]
+ CRUSH rule 5 x 345 [2,8]
+ CRUSH rule 5 x 346 [8,2,4]
+ CRUSH rule 5 x 347 [4,1,8]
+ CRUSH rule 5 x 348 [8,2,4]
+ CRUSH rule 5 x 349 [1,8]
+ CRUSH rule 5 x 350 [8,1]
+ CRUSH rule 5 x 351 [8,2]
+ CRUSH rule 5 x 352 [1,8,4]
+ CRUSH rule 5 x 353 [8,1]
+ CRUSH rule 5 x 354 [1,8]
+ CRUSH rule 5 x 355 [8,2]
+ CRUSH rule 5 x 356 [4,1,8]
+ CRUSH rule 5 x 357 [8,1,4]
+ CRUSH rule 5 x 358 [2,8,4]
+ CRUSH rule 5 x 359 [6,1,4]
+ CRUSH rule 5 x 360 [2,8]
+ CRUSH rule 5 x 361 [8,4,1]
+ CRUSH rule 5 x 362 [4,1,6]
+ CRUSH rule 5 x 363 [4,1,8]
+ CRUSH rule 5 x 364 [2,8]
+ CRUSH rule 5 x 365 [8,1]
+ CRUSH rule 5 x 366 [8,2]
+ CRUSH rule 5 x 367 [4,2,8]
+ CRUSH rule 5 x 368 [8,4,1]
+ CRUSH rule 5 x 369 [8,1]
+ CRUSH rule 5 x 370 [8,2]
+ CRUSH rule 5 x 371 [1,4,8]
+ CRUSH rule 5 x 372 [1,8]
+ CRUSH rule 5 x 373 [1,8]
+ CRUSH rule 5 x 374 [8,1]
+ CRUSH rule 5 x 375 [8,4,1]
+ CRUSH rule 5 x 376 [8,1,4]
+ CRUSH rule 5 x 377 [1,4,8]
+ CRUSH rule 5 x 378 [1,8]
+ CRUSH rule 5 x 379 [8,2]
+ CRUSH rule 5 x 380 [2,8]
+ CRUSH rule 5 x 381 [1,4,8]
+ CRUSH rule 5 x 382 [1,4,8]
+ CRUSH rule 5 x 383 [4,8,2]
+ CRUSH rule 5 x 384 [8,2,4]
+ CRUSH rule 5 x 385 [8,1]
+ CRUSH rule 5 x 386 [1,8]
+ CRUSH rule 5 x 387 [1,4,8]
+ CRUSH rule 5 x 388 [2,6]
+ CRUSH rule 5 x 389 [1,4,8]
+ CRUSH rule 5 x 390 [4,8,1]
+ CRUSH rule 5 x 391 [4,8,2]
+ CRUSH rule 5 x 392 [1,8,4]
+ CRUSH rule 5 x 393 [2,8]
+ CRUSH rule 5 x 394 [8,2]
+ CRUSH rule 5 x 395 [1,8]
+ CRUSH rule 5 x 396 [4,2,8]
+ CRUSH rule 5 x 397 [2,4,8]
+ CRUSH rule 5 x 398 [2,4,8]
+ CRUSH rule 5 x 399 [8,4,2]
+ CRUSH rule 5 x 400 [8,1,4]
+ CRUSH rule 5 x 401 [1,4,8]
+ CRUSH rule 5 x 402 [8,4,2]
+ CRUSH rule 5 x 403 [1,4,8]
+ CRUSH rule 5 x 404 [4,2,8]
+ CRUSH rule 5 x 405 [8,4,2]
+ CRUSH rule 5 x 406 [2,8]
+ CRUSH rule 5 x 407 [2,8,4]
+ CRUSH rule 5 x 408 [4,1,8]
+ CRUSH rule 5 x 409 [8,4,1]
+ CRUSH rule 5 x 410 [8,4,2]
+ CRUSH rule 5 x 411 [2,8,4]
+ CRUSH rule 5 x 412 [2,6]
+ CRUSH rule 5 x 413 [2,8]
+ CRUSH rule 5 x 414 [4,1,6]
+ CRUSH rule 5 x 415 [2,8]
+ CRUSH rule 5 x 416 [2,8]
+ CRUSH rule 5 x 417 [8,2]
+ CRUSH rule 5 x 418 [8,1,4]
+ CRUSH rule 5 x 419 [8,4,1]
+ CRUSH rule 5 x 420 [1,4,8]
+ CRUSH rule 5 x 421 [8,4,1]
+ CRUSH rule 5 x 422 [6,2]
+ CRUSH rule 5 x 423 [2,4,8]
+ CRUSH rule 5 x 424 [8,1]
+ CRUSH rule 5 x 425 [1,8]
+ CRUSH rule 5 x 426 [8,2]
+ CRUSH rule 5 x 427 [1,8]
+ CRUSH rule 5 x 428 [4,8,1]
+ CRUSH rule 5 x 429 [4,8,2]
+ CRUSH rule 5 x 430 [4,8,2]
+ CRUSH rule 5 x 431 [4,1,8]
+ CRUSH rule 5 x 432 [8,1]
+ CRUSH rule 5 x 433 [8,1]
+ CRUSH rule 5 x 434 [2,8]
+ CRUSH rule 5 x 435 [2,8]
+ CRUSH rule 5 x 436 [4,1,8]
+ CRUSH rule 5 x 437 [8,2]
+ CRUSH rule 5 x 438 [2,4,8]
+ CRUSH rule 5 x 439 [1,6]
+ CRUSH rule 5 x 440 [2,8]
+ CRUSH rule 5 x 441 [4,6,2]
+ CRUSH rule 5 x 442 [2,8]
+ CRUSH rule 5 x 443 [8,2,4]
+ CRUSH rule 5 x 444 [8,1]
+ CRUSH rule 5 x 445 [8,2]
+ CRUSH rule 5 x 446 [8,1]
+ CRUSH rule 5 x 447 [2,4,8]
+ CRUSH rule 5 x 448 [8,2,4]
+ CRUSH rule 5 x 449 [8,1]
+ CRUSH rule 5 x 450 [1,8]
+ CRUSH rule 5 x 451 [8,4,2]
+ CRUSH rule 5 x 452 [8,2]
+ CRUSH rule 5 x 453 [6,2]
+ CRUSH rule 5 x 454 [8,2]
+ CRUSH rule 5 x 455 [2,8,4]
+ CRUSH rule 5 x 456 [8,2]
+ CRUSH rule 5 x 457 [8,2]
+ CRUSH rule 5 x 458 [2,8]
+ CRUSH rule 5 x 459 [2,8,4]
+ CRUSH rule 5 x 460 [8,2]
+ CRUSH rule 5 x 461 [8,1]
+ CRUSH rule 5 x 462 [8,1]
+ CRUSH rule 5 x 463 [8,2]
+ CRUSH rule 5 x 464 [8,4,2]
+ CRUSH rule 5 x 465 [6,2,4]
+ CRUSH rule 5 x 466 [8,1]
+ CRUSH rule 5 x 467 [8,2]
+ CRUSH rule 5 x 468 [8,1,4]
+ CRUSH rule 5 x 469 [8,1]
+ CRUSH rule 5 x 470 [4,2,6]
+ CRUSH rule 5 x 471 [1,8]
+ CRUSH rule 5 x 472 [1,8]
+ CRUSH rule 5 x 473 [1,4,8]
+ CRUSH rule 5 x 474 [8,1]
+ CRUSH rule 5 x 475 [8,2,4]
+ CRUSH rule 5 x 476 [4,8,1]
+ CRUSH rule 5 x 477 [4,8,2]
+ CRUSH rule 5 x 478 [8,2,4]
+ CRUSH rule 5 x 479 [2,8]
+ CRUSH rule 5 x 480 [1,8]
+ CRUSH rule 5 x 481 [2,4,6]
+ CRUSH rule 5 x 482 [1,8]
+ CRUSH rule 5 x 483 [2,8,4]
+ CRUSH rule 5 x 484 [1,8]
+ CRUSH rule 5 x 485 [8,1]
+ CRUSH rule 5 x 486 [4,1,8]
+ CRUSH rule 5 x 487 [1,8]
+ CRUSH rule 5 x 488 [8,1]
+ CRUSH rule 5 x 489 [2,8]
+ CRUSH rule 5 x 490 [6,2]
+ CRUSH rule 5 x 491 [1,8]
+ CRUSH rule 5 x 492 [8,1]
+ CRUSH rule 5 x 493 [2,8]
+ CRUSH rule 5 x 494 [1,8]
+ CRUSH rule 5 x 495 [4,1,8]
+ CRUSH rule 5 x 496 [8,4,1]
+ CRUSH rule 5 x 497 [4,8,1]
+ CRUSH rule 5 x 498 [2,4,8]
+ CRUSH rule 5 x 499 [8,4,2]
+ CRUSH rule 5 x 500 [4,8,2]
+ CRUSH rule 5 x 501 [2,8]
+ CRUSH rule 5 x 502 [6,1]
+ CRUSH rule 5 x 503 [2,8]
+ CRUSH rule 5 x 504 [8,1]
+ CRUSH rule 5 x 505 [1,8]
+ CRUSH rule 5 x 506 [4,2,8]
+ CRUSH rule 5 x 507 [8,1,4]
+ CRUSH rule 5 x 508 [1,8]
+ CRUSH rule 5 x 509 [8,1]
+ CRUSH rule 5 x 510 [8,2]
+ CRUSH rule 5 x 511 [4,8,2]
+ CRUSH rule 5 x 512 [8,2]
+ CRUSH rule 5 x 513 [8,2]
+ CRUSH rule 5 x 514 [2,8]
+ CRUSH rule 5 x 515 [8,4,1]
+ CRUSH rule 5 x 516 [4,1,8]
+ CRUSH rule 5 x 517 [8,2]
+ CRUSH rule 5 x 518 [4,8,1]
+ CRUSH rule 5 x 519 [8,4,1]
+ CRUSH rule 5 x 520 [2,8,4]
+ CRUSH rule 5 x 521 [8,2,4]
+ CRUSH rule 5 x 522 [8,1,4]
+ CRUSH rule 5 x 523 [4,2,8]
+ CRUSH rule 5 x 524 [2,6]
+ CRUSH rule 5 x 525 [2,8]
+ CRUSH rule 5 x 526 [1,8]
+ CRUSH rule 5 x 527 [1,4,6]
+ CRUSH rule 5 x 528 [2,8]
+ CRUSH rule 5 x 529 [4,8,2]
+ CRUSH rule 5 x 530 [8,1]
+ CRUSH rule 5 x 531 [8,1,4]
+ CRUSH rule 5 x 532 [6,4,1]
+ CRUSH rule 5 x 533 [4,8,2]
+ CRUSH rule 5 x 534 [8,1]
+ CRUSH rule 5 x 535 [8,1]
+ CRUSH rule 5 x 536 [8,2]
+ CRUSH rule 5 x 537 [4,8,2]
+ CRUSH rule 5 x 538 [8,4,1]
+ CRUSH rule 5 x 539 [8,1]
+ CRUSH rule 5 x 540 [1,8,4]
+ CRUSH rule 5 x 541 [2,4,8]
+ CRUSH rule 5 x 542 [2,8]
+ CRUSH rule 5 x 543 [8,2]
+ CRUSH rule 5 x 544 [4,8,2]
+ CRUSH rule 5 x 545 [8,1]
+ CRUSH rule 5 x 546 [8,1,4]
+ CRUSH rule 5 x 547 [8,2,4]
+ CRUSH rule 5 x 548 [4,2,8]
+ CRUSH rule 5 x 549 [8,2]
+ CRUSH rule 5 x 550 [2,4,8]
+ CRUSH rule 5 x 551 [8,1]
+ CRUSH rule 5 x 552 [4,8,1]
+ CRUSH rule 5 x 553 [2,8]
+ CRUSH rule 5 x 554 [1,8]
+ CRUSH rule 5 x 555 [4,1,8]
+ CRUSH rule 5 x 556 [8,1]
+ CRUSH rule 5 x 557 [8,2]
+ CRUSH rule 5 x 558 [4,1,8]
+ CRUSH rule 5 x 559 [1,8]
+ CRUSH rule 5 x 560 [8,1]
+ CRUSH rule 5 x 561 [8,4,1]
+ CRUSH rule 5 x 562 [4,1,8]
+ CRUSH rule 5 x 563 [2,8]
+ CRUSH rule 5 x 564 [1,8]
+ CRUSH rule 5 x 565 [4,8,2]
+ CRUSH rule 5 x 566 [4,8,2]
+ CRUSH rule 5 x 567 [4,8,1]
+ CRUSH rule 5 x 568 [8,1]
+ CRUSH rule 5 x 569 [4,1,8]
+ CRUSH rule 5 x 570 [1,8]
+ CRUSH rule 5 x 571 [6,1]
+ CRUSH rule 5 x 572 [4,2,8]
+ CRUSH rule 5 x 573 [1,8]
+ CRUSH rule 5 x 574 [2,8]
+ CRUSH rule 5 x 575 [8,2,4]
+ CRUSH rule 5 x 576 [4,8,2]
+ CRUSH rule 5 x 577 [8,2]
+ CRUSH rule 5 x 578 [8,1]
+ CRUSH rule 5 x 579 [4,1,8]
+ CRUSH rule 5 x 580 [1,8]
+ CRUSH rule 5 x 581 [8,2,4]
+ CRUSH rule 5 x 582 [2,8,4]
+ CRUSH rule 5 x 583 [8,1]
+ CRUSH rule 5 x 584 [8,1,4]
+ CRUSH rule 5 x 585 [8,1,4]
+ CRUSH rule 5 x 586 [1,8,4]
+ CRUSH rule 5 x 587 [2,4,8]
+ CRUSH rule 5 x 588 [4,8,1]
+ CRUSH rule 5 x 589 [8,1]
+ CRUSH rule 5 x 590 [8,2]
+ CRUSH rule 5 x 591 [4,2,8]
+ CRUSH rule 5 x 592 [2,4,8]
+ CRUSH rule 5 x 593 [1,8,4]
+ CRUSH rule 5 x 594 [2,8]
+ CRUSH rule 5 x 595 [8,1]
+ CRUSH rule 5 x 596 [8,2]
+ CRUSH rule 5 x 597 [1,8]
+ CRUSH rule 5 x 598 [2,8]
+ CRUSH rule 5 x 599 [4,2,8]
+ CRUSH rule 5 x 600 [8,1,4]
+ CRUSH rule 5 x 601 [1,8,4]
+ CRUSH rule 5 x 602 [8,2]
+ CRUSH rule 5 x 603 [1,8]
+ CRUSH rule 5 x 604 [8,2]
+ CRUSH rule 5 x 605 [2,8]
+ CRUSH rule 5 x 606 [2,6,4]
+ CRUSH rule 5 x 607 [2,4,8]
+ CRUSH rule 5 x 608 [4,2,8]
+ CRUSH rule 5 x 609 [4,2,8]
+ CRUSH rule 5 x 610 [8,1]
+ CRUSH rule 5 x 611 [1,8]
+ CRUSH rule 5 x 612 [2,8]
+ CRUSH rule 5 x 613 [8,2,4]
+ CRUSH rule 5 x 614 [8,2,4]
+ CRUSH rule 5 x 615 [8,2,4]
+ CRUSH rule 5 x 616 [1,8]
+ CRUSH rule 5 x 617 [8,1,4]
+ CRUSH rule 5 x 618 [8,4,2]
+ CRUSH rule 5 x 619 [4,1,8]
+ CRUSH rule 5 x 620 [1,8]
+ CRUSH rule 5 x 621 [8,1]
+ CRUSH rule 5 x 622 [2,4,8]
+ CRUSH rule 5 x 623 [2,8]
+ CRUSH rule 5 x 624 [4,2,8]
+ CRUSH rule 5 x 625 [2,8]
+ CRUSH rule 5 x 626 [8,2,4]
+ CRUSH rule 5 x 627 [2,8,4]
+ CRUSH rule 5 x 628 [8,2]
+ CRUSH rule 5 x 629 [2,8,4]
+ CRUSH rule 5 x 630 [2,8]
+ CRUSH rule 5 x 631 [1,8,4]
+ CRUSH rule 5 x 632 [8,2]
+ CRUSH rule 5 x 633 [8,2]
+ CRUSH rule 5 x 634 [1,8]
+ CRUSH rule 5 x 635 [4,8,2]
+ CRUSH rule 5 x 636 [1,4,8]
+ CRUSH rule 5 x 637 [1,8]
+ CRUSH rule 5 x 638 [8,1,4]
+ CRUSH rule 5 x 639 [2,8]
+ CRUSH rule 5 x 640 [2,8]
+ CRUSH rule 5 x 641 [8,2]
+ CRUSH rule 5 x 642 [2,8]
+ CRUSH rule 5 x 643 [1,8]
+ CRUSH rule 5 x 644 [8,1]
+ CRUSH rule 5 x 645 [8,1]
+ CRUSH rule 5 x 646 [8,1,4]
+ CRUSH rule 5 x 647 [8,1]
+ CRUSH rule 5 x 648 [1,8]
+ CRUSH rule 5 x 649 [4,8,2]
+ CRUSH rule 5 x 650 [8,4,1]
+ CRUSH rule 5 x 651 [4,6,1]
+ CRUSH rule 5 x 652 [4,8,1]
+ CRUSH rule 5 x 653 [8,2]
+ CRUSH rule 5 x 654 [6,2]
+ CRUSH rule 5 x 655 [1,4,8]
+ CRUSH rule 5 x 656 [8,1]
+ CRUSH rule 5 x 657 [6,1]
+ CRUSH rule 5 x 658 [8,2]
+ CRUSH rule 5 x 659 [4,8,2]
+ CRUSH rule 5 x 660 [8,2]
+ CRUSH rule 5 x 661 [1,8]
+ CRUSH rule 5 x 662 [8,2]
+ CRUSH rule 5 x 663 [1,4,8]
+ CRUSH rule 5 x 664 [1,4,8]
+ CRUSH rule 5 x 665 [4,6,1]
+ CRUSH rule 5 x 666 [2,8]
+ CRUSH rule 5 x 667 [1,4,8]
+ CRUSH rule 5 x 668 [4,8,1]
+ CRUSH rule 5 x 669 [6,4,2]
+ CRUSH rule 5 x 670 [4,2,8]
+ CRUSH rule 5 x 671 [2,8]
+ CRUSH rule 5 x 672 [4,2,8]
+ CRUSH rule 5 x 673 [4,2,8]
+ CRUSH rule 5 x 674 [1,8]
+ CRUSH rule 5 x 675 [1,8,4]
+ CRUSH rule 5 x 676 [2,4,8]
+ CRUSH rule 5 x 677 [4,1,8]
+ CRUSH rule 5 x 678 [2,4,8]
+ CRUSH rule 5 x 679 [8,2]
+ CRUSH rule 5 x 680 [2,8]
+ CRUSH rule 5 x 681 [8,1]
+ CRUSH rule 5 x 682 [1,4,8]
+ CRUSH rule 5 x 683 [1,4,8]
+ CRUSH rule 5 x 684 [8,1,4]
+ CRUSH rule 5 x 685 [8,1,4]
+ CRUSH rule 5 x 686 [1,4,8]
+ CRUSH rule 5 x 687 [6,1]
+ CRUSH rule 5 x 688 [4,8,2]
+ CRUSH rule 5 x 689 [8,4,2]
+ CRUSH rule 5 x 690 [8,1,4]
+ CRUSH rule 5 x 691 [1,8]
+ CRUSH rule 5 x 692 [8,2]
+ CRUSH rule 5 x 693 [8,4,1]
+ CRUSH rule 5 x 694 [8,4,1]
+ CRUSH rule 5 x 695 [2,8,4]
+ CRUSH rule 5 x 696 [1,8]
+ CRUSH rule 5 x 697 [8,1,4]
+ CRUSH rule 5 x 698 [8,2,4]
+ CRUSH rule 5 x 699 [1,8,4]
+ CRUSH rule 5 x 700 [1,8]
+ CRUSH rule 5 x 701 [1,8]
+ CRUSH rule 5 x 702 [2,8]
+ CRUSH rule 5 x 703 [8,1]
+ CRUSH rule 5 x 704 [1,4,8]
+ CRUSH rule 5 x 705 [8,1,4]
+ CRUSH rule 5 x 706 [1,4,8]
+ CRUSH rule 5 x 707 [8,4,1]
+ CRUSH rule 5 x 708 [4,8,1]
+ CRUSH rule 5 x 709 [8,2]
+ CRUSH rule 5 x 710 [8,2]
+ CRUSH rule 5 x 711 [2,4,8]
+ CRUSH rule 5 x 712 [2,8]
+ CRUSH rule 5 x 713 [8,4,1]
+ CRUSH rule 5 x 714 [2,8]
+ CRUSH rule 5 x 715 [1,8]
+ CRUSH rule 5 x 716 [4,8,2]
+ CRUSH rule 5 x 717 [8,2,4]
+ CRUSH rule 5 x 718 [8,1]
+ CRUSH rule 5 x 719 [2,6,4]
+ CRUSH rule 5 x 720 [8,1,4]
+ CRUSH rule 5 x 721 [4,6,2]
+ CRUSH rule 5 x 722 [8,2]
+ CRUSH rule 5 x 723 [4,1,8]
+ CRUSH rule 5 x 724 [2,6]
+ CRUSH rule 5 x 725 [1,8]
+ CRUSH rule 5 x 726 [4,8,1]
+ CRUSH rule 5 x 727 [4,8,1]
+ CRUSH rule 5 x 728 [2,8,4]
+ CRUSH rule 5 x 729 [8,2]
+ CRUSH rule 5 x 730 [4,8,2]
+ CRUSH rule 5 x 731 [4,1,8]
+ CRUSH rule 5 x 732 [1,8]
+ CRUSH rule 5 x 733 [4,8,1]
+ CRUSH rule 5 x 734 [8,4,2]
+ CRUSH rule 5 x 735 [4,8,1]
+ CRUSH rule 5 x 736 [4,8,1]
+ CRUSH rule 5 x 737 [1,8,4]
+ CRUSH rule 5 x 738 [4,2,8]
+ CRUSH rule 5 x 739 [2,8]
+ CRUSH rule 5 x 740 [1,8,4]
+ CRUSH rule 5 x 741 [8,1]
+ CRUSH rule 5 x 742 [8,2]
+ CRUSH rule 5 x 743 [8,1,4]
+ CRUSH rule 5 x 744 [4,8,1]
+ CRUSH rule 5 x 745 [1,8]
+ CRUSH rule 5 x 746 [1,8]
+ CRUSH rule 5 x 747 [8,1]
+ CRUSH rule 5 x 748 [2,8,4]
+ CRUSH rule 5 x 749 [4,8,2]
+ CRUSH rule 5 x 750 [1,8,4]
+ CRUSH rule 5 x 751 [2,8]
+ CRUSH rule 5 x 752 [8,1]
+ CRUSH rule 5 x 753 [8,4,1]
+ CRUSH rule 5 x 754 [8,4,2]
+ CRUSH rule 5 x 755 [1,8,4]
+ CRUSH rule 5 x 756 [8,1]
+ CRUSH rule 5 x 757 [8,1,4]
+ CRUSH rule 5 x 758 [8,2]
+ CRUSH rule 5 x 759 [8,4,2]
+ CRUSH rule 5 x 760 [1,4,8]
+ CRUSH rule 5 x 761 [2,8]
+ CRUSH rule 5 x 762 [2,8]
+ CRUSH rule 5 x 763 [8,4,1]
+ CRUSH rule 5 x 764 [1,8]
+ CRUSH rule 5 x 765 [8,2]
+ CRUSH rule 5 x 766 [8,1]
+ CRUSH rule 5 x 767 [1,8,4]
+ CRUSH rule 5 x 768 [8,4,2]
+ CRUSH rule 5 x 769 [8,2,4]
+ CRUSH rule 5 x 770 [8,2,4]
+ CRUSH rule 5 x 771 [8,1,4]
+ CRUSH rule 5 x 772 [8,4,1]
+ CRUSH rule 5 x 773 [4,1,8]
+ CRUSH rule 5 x 774 [8,1]
+ CRUSH rule 5 x 775 [8,4,2]
+ CRUSH rule 5 x 776 [6,2]
+ CRUSH rule 5 x 777 [4,1,8]
+ CRUSH rule 5 x 778 [1,8,4]
+ CRUSH rule 5 x 779 [2,8]
+ CRUSH rule 5 x 780 [2,4,8]
+ CRUSH rule 5 x 781 [8,2]
+ CRUSH rule 5 x 782 [4,1,8]
+ CRUSH rule 5 x 783 [8,1,4]
+ CRUSH rule 5 x 784 [1,4,8]
+ CRUSH rule 5 x 785 [8,1,4]
+ CRUSH rule 5 x 786 [8,1]
+ CRUSH rule 5 x 787 [1,6,4]
+ CRUSH rule 5 x 788 [8,2,4]
+ CRUSH rule 5 x 789 [1,8]
+ CRUSH rule 5 x 790 [8,1]
+ CRUSH rule 5 x 791 [4,8,2]
+ CRUSH rule 5 x 792 [4,8,2]
+ CRUSH rule 5 x 793 [8,1,4]
+ CRUSH rule 5 x 794 [2,8,4]
+ CRUSH rule 5 x 795 [1,8]
+ CRUSH rule 5 x 796 [8,2]
+ CRUSH rule 5 x 797 [2,4,8]
+ CRUSH rule 5 x 798 [6,1]
+ CRUSH rule 5 x 799 [4,1,8]
+ CRUSH rule 5 x 800 [2,8]
+ CRUSH rule 5 x 801 [4,8,1]
+ CRUSH rule 5 x 802 [1,8,4]
+ CRUSH rule 5 x 803 [2,8]
+ CRUSH rule 5 x 804 [8,2]
+ CRUSH rule 5 x 805 [8,2]
+ CRUSH rule 5 x 806 [1,4,8]
+ CRUSH rule 5 x 807 [4,8,2]
+ CRUSH rule 5 x 808 [8,2]
+ CRUSH rule 5 x 809 [1,8]
+ CRUSH rule 5 x 810 [8,2]
+ CRUSH rule 5 x 811 [8,1]
+ CRUSH rule 5 x 812 [8,4,2]
+ CRUSH rule 5 x 813 [8,4,2]
+ CRUSH rule 5 x 814 [8,2]
+ CRUSH rule 5 x 815 [4,1,8]
+ CRUSH rule 5 x 816 [2,8]
+ CRUSH rule 5 x 817 [8,1]
+ CRUSH rule 5 x 818 [1,8]
+ CRUSH rule 5 x 819 [1,8]
+ CRUSH rule 5 x 820 [4,8,2]
+ CRUSH rule 5 x 821 [4,8,2]
+ CRUSH rule 5 x 822 [2,4,8]
+ CRUSH rule 5 x 823 [4,8,2]
+ CRUSH rule 5 x 824 [8,2]
+ CRUSH rule 5 x 825 [2,8,4]
+ CRUSH rule 5 x 826 [8,2,4]
+ CRUSH rule 5 x 827 [2,8,4]
+ CRUSH rule 5 x 828 [2,8]
+ CRUSH rule 5 x 829 [8,1]
+ CRUSH rule 5 x 830 [2,4,8]
+ CRUSH rule 5 x 831 [1,8]
+ CRUSH rule 5 x 832 [4,8,2]
+ CRUSH rule 5 x 833 [2,8]
+ CRUSH rule 5 x 834 [1,8]
+ CRUSH rule 5 x 835 [8,4,1]
+ CRUSH rule 5 x 836 [4,8,1]
+ CRUSH rule 5 x 837 [8,4,1]
+ CRUSH rule 5 x 838 [6,2,4]
+ CRUSH rule 5 x 839 [2,8]
+ CRUSH rule 5 x 840 [8,1]
+ CRUSH rule 5 x 841 [4,8,2]
+ CRUSH rule 5 x 842 [2,8]
+ CRUSH rule 5 x 843 [8,4,1]
+ CRUSH rule 5 x 844 [8,2]
+ CRUSH rule 5 x 845 [4,8,2]
+ CRUSH rule 5 x 846 [4,2,8]
+ CRUSH rule 5 x 847 [2,8]
+ CRUSH rule 5 x 848 [2,8,4]
+ CRUSH rule 5 x 849 [4,8,2]
+ CRUSH rule 5 x 850 [1,6]
+ CRUSH rule 5 x 851 [6,1]
+ CRUSH rule 5 x 852 [8,4,2]
+ CRUSH rule 5 x 853 [6,1]
+ CRUSH rule 5 x 854 [8,1]
+ CRUSH rule 5 x 855 [8,1]
+ CRUSH rule 5 x 856 [8,4,2]
+ CRUSH rule 5 x 857 [8,2]
+ CRUSH rule 5 x 858 [6,1]
+ CRUSH rule 5 x 859 [8,2,4]
+ CRUSH rule 5 x 860 [2,8]
+ CRUSH rule 5 x 861 [8,2]
+ CRUSH rule 5 x 862 [8,1]
+ CRUSH rule 5 x 863 [8,2]
+ CRUSH rule 5 x 864 [8,2]
+ CRUSH rule 5 x 865 [8,1]
+ CRUSH rule 5 x 866 [8,2]
+ CRUSH rule 5 x 867 [8,2]
+ CRUSH rule 5 x 868 [8,1]
+ CRUSH rule 5 x 869 [8,4,2]
+ CRUSH rule 5 x 870 [2,8]
+ CRUSH rule 5 x 871 [1,8]
+ CRUSH rule 5 x 872 [1,8]
+ CRUSH rule 5 x 873 [4,8,2]
+ CRUSH rule 5 x 874 [2,6]
+ CRUSH rule 5 x 875 [2,8,4]
+ CRUSH rule 5 x 876 [4,8,1]
+ CRUSH rule 5 x 877 [8,4,2]
+ CRUSH rule 5 x 878 [2,8]
+ CRUSH rule 5 x 879 [8,1]
+ CRUSH rule 5 x 880 [1,8]
+ CRUSH rule 5 x 881 [4,8,1]
+ CRUSH rule 5 x 882 [1,8]
+ CRUSH rule 5 x 883 [2,4,8]
+ CRUSH rule 5 x 884 [8,2,4]
+ CRUSH rule 5 x 885 [4,1,8]
+ CRUSH rule 5 x 886 [8,2]
+ CRUSH rule 5 x 887 [8,4,1]
+ CRUSH rule 5 x 888 [8,2]
+ CRUSH rule 5 x 889 [2,6]
+ CRUSH rule 5 x 890 [8,2,4]
+ CRUSH rule 5 x 891 [1,8]
+ CRUSH rule 5 x 892 [8,2,4]
+ CRUSH rule 5 x 893 [2,6]
+ CRUSH rule 5 x 894 [8,4,2]
+ CRUSH rule 5 x 895 [4,1,8]
+ CRUSH rule 5 x 896 [1,8]
+ CRUSH rule 5 x 897 [2,8]
+ CRUSH rule 5 x 898 [1,4,8]
+ CRUSH rule 5 x 899 [1,8]
+ CRUSH rule 5 x 900 [4,1,8]
+ CRUSH rule 5 x 901 [2,8]
+ CRUSH rule 5 x 902 [8,4,1]
+ CRUSH rule 5 x 903 [8,2]
+ CRUSH rule 5 x 904 [8,2]
+ CRUSH rule 5 x 905 [8,2]
+ CRUSH rule 5 x 906 [1,8]
+ CRUSH rule 5 x 907 [8,1]
+ CRUSH rule 5 x 908 [8,1]
+ CRUSH rule 5 x 909 [2,8]
+ CRUSH rule 5 x 910 [8,2]
+ CRUSH rule 5 x 911 [8,1]
+ CRUSH rule 5 x 912 [1,8]
+ CRUSH rule 5 x 913 [8,2,4]
+ CRUSH rule 5 x 914 [6,4,2]
+ CRUSH rule 5 x 915 [8,2]
+ CRUSH rule 5 x 916 [4,1,8]
+ CRUSH rule 5 x 917 [1,4,8]
+ CRUSH rule 5 x 918 [8,2]
+ CRUSH rule 5 x 919 [8,2]
+ CRUSH rule 5 x 920 [8,1]
+ CRUSH rule 5 x 921 [1,8]
+ CRUSH rule 5 x 922 [8,4,2]
+ CRUSH rule 5 x 923 [4,8,2]
+ CRUSH rule 5 x 924 [8,1]
+ CRUSH rule 5 x 925 [4,8,2]
+ CRUSH rule 5 x 926 [2,8]
+ CRUSH rule 5 x 927 [1,8,4]
+ CRUSH rule 5 x 928 [8,1]
+ CRUSH rule 5 x 929 [4,1,8]
+ CRUSH rule 5 x 930 [2,8]
+ CRUSH rule 5 x 931 [2,8]
+ CRUSH rule 5 x 932 [4,1,8]
+ CRUSH rule 5 x 933 [8,4,1]
+ CRUSH rule 5 x 934 [8,2]
+ CRUSH rule 5 x 935 [8,2]
+ CRUSH rule 5 x 936 [1,8]
+ CRUSH rule 5 x 937 [4,8,2]
+ CRUSH rule 5 x 938 [8,4,2]
+ CRUSH rule 5 x 939 [2,8,4]
+ CRUSH rule 5 x 940 [8,1]
+ CRUSH rule 5 x 941 [2,8]
+ CRUSH rule 5 x 942 [1,8]
+ CRUSH rule 5 x 943 [8,2]
+ CRUSH rule 5 x 944 [8,2]
+ CRUSH rule 5 x 945 [8,2,4]
+ CRUSH rule 5 x 946 [2,8,4]
+ CRUSH rule 5 x 947 [8,2]
+ CRUSH rule 5 x 948 [8,1]
+ CRUSH rule 5 x 949 [6,1]
+ CRUSH rule 5 x 950 [8,1]
+ CRUSH rule 5 x 951 [8,1]
+ CRUSH rule 5 x 952 [2,8,4]
+ CRUSH rule 5 x 953 [1,4,8]
+ CRUSH rule 5 x 954 [2,8]
+ CRUSH rule 5 x 955 [8,1,4]
+ CRUSH rule 5 x 956 [1,8,4]
+ CRUSH rule 5 x 957 [8,1,4]
+ CRUSH rule 5 x 958 [8,4,1]
+ CRUSH rule 5 x 959 [4,2,8]
+ CRUSH rule 5 x 960 [6,1]
+ CRUSH rule 5 x 961 [1,8]
+ CRUSH rule 5 x 962 [8,4,2]
+ CRUSH rule 5 x 963 [2,4,6]
+ CRUSH rule 5 x 964 [2,8]
+ CRUSH rule 5 x 965 [8,2]
+ CRUSH rule 5 x 966 [4,8,1]
+ CRUSH rule 5 x 967 [8,4,2]
+ CRUSH rule 5 x 968 [8,2]
+ CRUSH rule 5 x 969 [8,2,4]
+ CRUSH rule 5 x 970 [2,8,4]
+ CRUSH rule 5 x 971 [1,8]
+ CRUSH rule 5 x 972 [1,8]
+ CRUSH rule 5 x 973 [1,8]
+ CRUSH rule 5 x 974 [4,1,8]
+ CRUSH rule 5 x 975 [4,8,1]
+ CRUSH rule 5 x 976 [4,8,2]
+ CRUSH rule 5 x 977 [8,4,2]
+ CRUSH rule 5 x 978 [8,2,4]
+ CRUSH rule 5 x 979 [8,1,4]
+ CRUSH rule 5 x 980 [8,2,4]
+ CRUSH rule 5 x 981 [8,2]
+ CRUSH rule 5 x 982 [1,8]
+ CRUSH rule 5 x 983 [4,8,2]
+ CRUSH rule 5 x 984 [2,8]
+ CRUSH rule 5 x 985 [2,4,8]
+ CRUSH rule 5 x 986 [8,4,1]
+ CRUSH rule 5 x 987 [2,8]
+ CRUSH rule 5 x 988 [1,4,6]
+ CRUSH rule 5 x 989 [1,8]
+ CRUSH rule 5 x 990 [1,8,4]
+ CRUSH rule 5 x 991 [1,4,8]
+ CRUSH rule 5 x 992 [8,1,4]
+ CRUSH rule 5 x 993 [2,8,4]
+ CRUSH rule 5 x 994 [4,8,2]
+ CRUSH rule 5 x 995 [8,1,4]
+ CRUSH rule 5 x 996 [8,4,1]
+ CRUSH rule 5 x 997 [8,4,1]
+ CRUSH rule 5 x 998 [8,1,4]
+ CRUSH rule 5 x 999 [1,8,4]
+ CRUSH rule 5 x 1000 [8,4,2]
+ CRUSH rule 5 x 1001 [2,8]
+ CRUSH rule 5 x 1002 [1,8]
+ CRUSH rule 5 x 1003 [2,8]
+ CRUSH rule 5 x 1004 [8,1,4]
+ CRUSH rule 5 x 1005 [8,1]
+ CRUSH rule 5 x 1006 [1,8,4]
+ CRUSH rule 5 x 1007 [1,4,8]
+ CRUSH rule 5 x 1008 [1,8]
+ CRUSH rule 5 x 1009 [6,4,1]
+ CRUSH rule 5 x 1010 [1,8]
+ CRUSH rule 5 x 1011 [4,2,8]
+ CRUSH rule 5 x 1012 [1,8]
+ CRUSH rule 5 x 1013 [2,8]
+ CRUSH rule 5 x 1014 [2,8,4]
+ CRUSH rule 5 x 1015 [8,1]
+ CRUSH rule 5 x 1016 [2,4,8]
+ CRUSH rule 5 x 1017 [6,2,4]
+ CRUSH rule 5 x 1018 [4,1,8]
+ CRUSH rule 5 x 1019 [4,8,2]
+ CRUSH rule 5 x 1020 [1,8]
+ CRUSH rule 5 x 1021 [2,8]
+ CRUSH rule 5 x 1022 [1,8,4]
+ CRUSH rule 5 x 1023 [4,2,8]
+ rule 5 (chooseleaf-set) num_rep 3 result size == 2:\t501/1024 (esc)
+ rule 5 (chooseleaf-set) num_rep 3 result size == 3:\t523/1024 (esc)
diff --git a/src/test/cli/crushtool/one-hundered-devices.crushmap b/src/test/cli/crushtool/test-map-a.crushmap
similarity index 99%
rename from src/test/cli/crushtool/one-hundered-devices.crushmap
rename to src/test/cli/crushtool/test-map-a.crushmap
index d7637d6..00f7579 100644
Binary files a/src/test/cli/crushtool/one-hundered-devices.crushmap and b/src/test/cli/crushtool/test-map-a.crushmap differ
diff --git a/src/test/cli/crushtool/new-tunable-values.t b/src/test/cli/crushtool/test-map-bobtail-tunables.t
similarity index 99%
rename from src/test/cli/crushtool/new-tunable-values.t
rename to src/test/cli/crushtool/test-map-bobtail-tunables.t
index 07d7921..7c38260 100644
--- a/src/test/cli/crushtool/new-tunable-values.t
+++ b/src/test/cli/crushtool/test-map-bobtail-tunables.t
@@ -1,4 +1,5 @@
- $ crushtool -i "$TESTDIR/one-hundered-devices-new-tunables.crushmap" --test --show-statistics --rule 0
+ $ crushtool -i "$TESTDIR/test-map-a.crushmap" --test --show-statistics --rule 0 --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 --set-chooseleaf-descend-once 1
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
rule 0 (data), x = 0..1023, numrep = 1..10
CRUSH rule 0 x 0 [36]
CRUSH rule 0 x 1 [876]
diff --git a/src/test/cli/crushtool/one-hundered-devices-new-tunables.crushmap b/src/test/cli/crushtool/test-map-indep.crushmap
similarity index 99%
rename from src/test/cli/crushtool/one-hundered-devices-new-tunables.crushmap
rename to src/test/cli/crushtool/test-map-indep.crushmap
index 8b9fe46..b3fd3c2 100644
Binary files a/src/test/cli/crushtool/one-hundered-devices-new-tunables.crushmap and b/src/test/cli/crushtool/test-map-indep.crushmap differ
diff --git a/src/test/cli/crushtool/test-map-indep.t b/src/test/cli/crushtool/test-map-indep.t
new file mode 100644
index 0000000..5f6dbb3
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-indep.t
@@ -0,0 +1,10253 @@
+ $ crushtool -i "$TESTDIR/test-map-indep.crushmap" --test --show-statistics --rule 1 --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 --set-chooseleaf-descend-once 2
+ crushtool successfully built or modified map. Use '-o <file>' to write it out.
+ rule 1 (metadata), x = 0..1023, numrep = 1..10
+ CRUSH rule 1 x 0 [36]
+ CRUSH rule 1 x 1 [876]
+ CRUSH rule 1 x 2 [292]
+ CRUSH rule 1 x 3 [623]
+ CRUSH rule 1 x 4 [61]
+ CRUSH rule 1 x 5 [946]
+ CRUSH rule 1 x 6 [576]
+ CRUSH rule 1 x 7 [645]
+ CRUSH rule 1 x 8 [243]
+ CRUSH rule 1 x 9 [22]
+ CRUSH rule 1 x 10 [758]
+ CRUSH rule 1 x 11 [769]
+ CRUSH rule 1 x 12 [780]
+ CRUSH rule 1 x 13 [557]
+ CRUSH rule 1 x 14 [59]
+ CRUSH rule 1 x 15 [718]
+ CRUSH rule 1 x 16 [673]
+ CRUSH rule 1 x 17 [648]
+ CRUSH rule 1 x 18 [654]
+ CRUSH rule 1 x 19 [850]
+ CRUSH rule 1 x 20 [717]
+ CRUSH rule 1 x 21 [420]
+ CRUSH rule 1 x 22 [503]
+ CRUSH rule 1 x 23 [411]
+ CRUSH rule 1 x 24 [266]
+ CRUSH rule 1 x 25 [760]
+ CRUSH rule 1 x 26 [903]
+ CRUSH rule 1 x 27 [946]
+ CRUSH rule 1 x 28 [69]
+ CRUSH rule 1 x 29 [844]
+ CRUSH rule 1 x 30 [621]
+ CRUSH rule 1 x 31 [784]
+ CRUSH rule 1 x 32 [173]
+ CRUSH rule 1 x 33 [698]
+ CRUSH rule 1 x 34 [168]
+ CRUSH rule 1 x 35 [274]
+ CRUSH rule 1 x 36 [318]
+ CRUSH rule 1 x 37 [173]
+ CRUSH rule 1 x 38 [708]
+ CRUSH rule 1 x 39 [662]
+ CRUSH rule 1 x 40 [620]
+ CRUSH rule 1 x 41 [811]
+ CRUSH rule 1 x 42 [863]
+ CRUSH rule 1 x 43 [686]
+ CRUSH rule 1 x 44 [396]
+ CRUSH rule 1 x 45 [991]
+ CRUSH rule 1 x 46 [420]
+ CRUSH rule 1 x 47 [467]
+ CRUSH rule 1 x 48 [955]
+ CRUSH rule 1 x 49 [974]
+ CRUSH rule 1 x 50 [870]
+ CRUSH rule 1 x 51 [182]
+ CRUSH rule 1 x 52 [704]
+ CRUSH rule 1 x 53 [185]
+ CRUSH rule 1 x 54 [270]
+ CRUSH rule 1 x 55 [895]
+ CRUSH rule 1 x 56 [564]
+ CRUSH rule 1 x 57 [738]
+ CRUSH rule 1 x 58 [524]
+ CRUSH rule 1 x 59 [408]
+ CRUSH rule 1 x 60 [228]
+ CRUSH rule 1 x 61 [154]
+ CRUSH rule 1 x 62 [594]
+ CRUSH rule 1 x 63 [646]
+ CRUSH rule 1 x 64 [175]
+ CRUSH rule 1 x 65 [745]
+ CRUSH rule 1 x 66 [275]
+ CRUSH rule 1 x 67 [246]
+ CRUSH rule 1 x 68 [711]
+ CRUSH rule 1 x 69 [493]
+ CRUSH rule 1 x 70 [30]
+ CRUSH rule 1 x 71 [984]
+ CRUSH rule 1 x 72 [71]
+ CRUSH rule 1 x 73 [922]
+ CRUSH rule 1 x 74 [629]
+ CRUSH rule 1 x 75 [222]
+ CRUSH rule 1 x 76 [262]
+ CRUSH rule 1 x 77 [638]
+ CRUSH rule 1 x 78 [324]
+ CRUSH rule 1 x 79 [577]
+ CRUSH rule 1 x 80 [501]
+ CRUSH rule 1 x 81 [506]
+ CRUSH rule 1 x 82 [222]
+ CRUSH rule 1 x 83 [71]
+ CRUSH rule 1 x 84 [49]
+ CRUSH rule 1 x 85 [985]
+ CRUSH rule 1 x 86 [537]
+ CRUSH rule 1 x 87 [997]
+ CRUSH rule 1 x 88 [957]
+ CRUSH rule 1 x 89 [399]
+ CRUSH rule 1 x 90 [943]
+ CRUSH rule 1 x 91 [22]
+ CRUSH rule 1 x 92 [532]
+ CRUSH rule 1 x 93 [218]
+ CRUSH rule 1 x 94 [181]
+ CRUSH rule 1 x 95 [343]
+ CRUSH rule 1 x 96 [861]
+ CRUSH rule 1 x 97 [459]
+ CRUSH rule 1 x 98 [327]
+ CRUSH rule 1 x 99 [974]
+ CRUSH rule 1 x 100 [32]
+ CRUSH rule 1 x 101 [142]
+ CRUSH rule 1 x 102 [172]
+ CRUSH rule 1 x 103 [630]
+ CRUSH rule 1 x 104 [758]
+ CRUSH rule 1 x 105 [843]
+ CRUSH rule 1 x 106 [28]
+ CRUSH rule 1 x 107 [74]
+ CRUSH rule 1 x 108 [875]
+ CRUSH rule 1 x 109 [411]
+ CRUSH rule 1 x 110 [440]
+ CRUSH rule 1 x 111 [405]
+ CRUSH rule 1 x 112 [143]
+ CRUSH rule 1 x 113 [153]
+ CRUSH rule 1 x 114 [804]
+ CRUSH rule 1 x 115 [588]
+ CRUSH rule 1 x 116 [327]
+ CRUSH rule 1 x 117 [95]
+ CRUSH rule 1 x 118 [80]
+ CRUSH rule 1 x 119 [386]
+ CRUSH rule 1 x 120 [366]
+ CRUSH rule 1 x 121 [129]
+ CRUSH rule 1 x 122 [873]
+ CRUSH rule 1 x 123 [533]
+ CRUSH rule 1 x 124 [461]
+ CRUSH rule 1 x 125 [342]
+ CRUSH rule 1 x 126 [819]
+ CRUSH rule 1 x 127 [437]
+ CRUSH rule 1 x 128 [679]
+ CRUSH rule 1 x 129 [380]
+ CRUSH rule 1 x 130 [992]
+ CRUSH rule 1 x 131 [469]
+ CRUSH rule 1 x 132 [571]
+ CRUSH rule 1 x 133 [964]
+ CRUSH rule 1 x 134 [999]
+ CRUSH rule 1 x 135 [634]
+ CRUSH rule 1 x 136 [114]
+ CRUSH rule 1 x 137 [839]
+ CRUSH rule 1 x 138 [967]
+ CRUSH rule 1 x 139 [308]
+ CRUSH rule 1 x 140 [764]
+ CRUSH rule 1 x 141 [423]
+ CRUSH rule 1 x 142 [252]
+ CRUSH rule 1 x 143 [33]
+ CRUSH rule 1 x 144 [472]
+ CRUSH rule 1 x 145 [242]
+ CRUSH rule 1 x 146 [290]
+ CRUSH rule 1 x 147 [447]
+ CRUSH rule 1 x 148 [212]
+ CRUSH rule 1 x 149 [9]
+ CRUSH rule 1 x 150 [166]
+ CRUSH rule 1 x 151 [811]
+ CRUSH rule 1 x 152 [449]
+ CRUSH rule 1 x 153 [523]
+ CRUSH rule 1 x 154 [208]
+ CRUSH rule 1 x 155 [569]
+ CRUSH rule 1 x 156 [488]
+ CRUSH rule 1 x 157 [140]
+ CRUSH rule 1 x 158 [786]
+ CRUSH rule 1 x 159 [134]
+ CRUSH rule 1 x 160 [690]
+ CRUSH rule 1 x 161 [324]
+ CRUSH rule 1 x 162 [748]
+ CRUSH rule 1 x 163 [575]
+ CRUSH rule 1 x 164 [314]
+ CRUSH rule 1 x 165 [116]
+ CRUSH rule 1 x 166 [352]
+ CRUSH rule 1 x 167 [27]
+ CRUSH rule 1 x 168 [953]
+ CRUSH rule 1 x 169 [912]
+ CRUSH rule 1 x 170 [421]
+ CRUSH rule 1 x 171 [488]
+ CRUSH rule 1 x 172 [366]
+ CRUSH rule 1 x 173 [863]
+ CRUSH rule 1 x 174 [263]
+ CRUSH rule 1 x 175 [875]
+ CRUSH rule 1 x 176 [745]
+ CRUSH rule 1 x 177 [128]
+ CRUSH rule 1 x 178 [155]
+ CRUSH rule 1 x 179 [593]
+ CRUSH rule 1 x 180 [154]
+ CRUSH rule 1 x 181 [289]
+ CRUSH rule 1 x 182 [730]
+ CRUSH rule 1 x 183 [639]
+ CRUSH rule 1 x 184 [704]
+ CRUSH rule 1 x 185 [97]
+ CRUSH rule 1 x 186 [26]
+ CRUSH rule 1 x 187 [649]
+ CRUSH rule 1 x 188 [682]
+ CRUSH rule 1 x 189 [325]
+ CRUSH rule 1 x 190 [399]
+ CRUSH rule 1 x 191 [629]
+ CRUSH rule 1 x 192 [503]
+ CRUSH rule 1 x 193 [546]
+ CRUSH rule 1 x 194 [242]
+ CRUSH rule 1 x 195 [625]
+ CRUSH rule 1 x 196 [357]
+ CRUSH rule 1 x 197 [306]
+ CRUSH rule 1 x 198 [863]
+ CRUSH rule 1 x 199 [935]
+ CRUSH rule 1 x 200 [373]
+ CRUSH rule 1 x 201 [659]
+ CRUSH rule 1 x 202 [260]
+ CRUSH rule 1 x 203 [36]
+ CRUSH rule 1 x 204 [92]
+ CRUSH rule 1 x 205 [68]
+ CRUSH rule 1 x 206 [570]
+ CRUSH rule 1 x 207 [834]
+ CRUSH rule 1 x 208 [927]
+ CRUSH rule 1 x 209 [878]
+ CRUSH rule 1 x 210 [572]
+ CRUSH rule 1 x 211 [107]
+ CRUSH rule 1 x 212 [389]
+ CRUSH rule 1 x 213 [497]
+ CRUSH rule 1 x 214 [798]
+ CRUSH rule 1 x 215 [233]
+ CRUSH rule 1 x 216 [494]
+ CRUSH rule 1 x 217 [352]
+ CRUSH rule 1 x 218 [895]
+ CRUSH rule 1 x 219 [222]
+ CRUSH rule 1 x 220 [281]
+ CRUSH rule 1 x 221 [64]
+ CRUSH rule 1 x 222 [40]
+ CRUSH rule 1 x 223 [645]
+ CRUSH rule 1 x 224 [647]
+ CRUSH rule 1 x 225 [219]
+ CRUSH rule 1 x 226 [372]
+ CRUSH rule 1 x 227 [925]
+ CRUSH rule 1 x 228 [682]
+ CRUSH rule 1 x 229 [880]
+ CRUSH rule 1 x 230 [328]
+ CRUSH rule 1 x 231 [320]
+ CRUSH rule 1 x 232 [924]
+ CRUSH rule 1 x 233 [948]
+ CRUSH rule 1 x 234 [484]
+ CRUSH rule 1 x 235 [750]
+ CRUSH rule 1 x 236 [551]
+ CRUSH rule 1 x 237 [390]
+ CRUSH rule 1 x 238 [570]
+ CRUSH rule 1 x 239 [729]
+ CRUSH rule 1 x 240 [981]
+ CRUSH rule 1 x 241 [310]
+ CRUSH rule 1 x 242 [161]
+ CRUSH rule 1 x 243 [180]
+ CRUSH rule 1 x 244 [52]
+ CRUSH rule 1 x 245 [523]
+ CRUSH rule 1 x 246 [362]
+ CRUSH rule 1 x 247 [382]
+ CRUSH rule 1 x 248 [129]
+ CRUSH rule 1 x 249 [159]
+ CRUSH rule 1 x 250 [404]
+ CRUSH rule 1 x 251 [661]
+ CRUSH rule 1 x 252 [961]
+ CRUSH rule 1 x 253 [651]
+ CRUSH rule 1 x 254 [123]
+ CRUSH rule 1 x 255 [314]
+ CRUSH rule 1 x 256 [315]
+ CRUSH rule 1 x 257 [825]
+ CRUSH rule 1 x 258 [624]
+ CRUSH rule 1 x 259 [602]
+ CRUSH rule 1 x 260 [717]
+ CRUSH rule 1 x 261 [145]
+ CRUSH rule 1 x 262 [223]
+ CRUSH rule 1 x 263 [462]
+ CRUSH rule 1 x 264 [654]
+ CRUSH rule 1 x 265 [302]
+ CRUSH rule 1 x 266 [202]
+ CRUSH rule 1 x 267 [282]
+ CRUSH rule 1 x 268 [338]
+ CRUSH rule 1 x 269 [738]
+ CRUSH rule 1 x 270 [707]
+ CRUSH rule 1 x 271 [705]
+ CRUSH rule 1 x 272 [756]
+ CRUSH rule 1 x 273 [197]
+ CRUSH rule 1 x 274 [992]
+ CRUSH rule 1 x 275 [544]
+ CRUSH rule 1 x 276 [658]
+ CRUSH rule 1 x 277 [143]
+ CRUSH rule 1 x 278 [492]
+ CRUSH rule 1 x 279 [517]
+ CRUSH rule 1 x 280 [825]
+ CRUSH rule 1 x 281 [224]
+ CRUSH rule 1 x 282 [298]
+ CRUSH rule 1 x 283 [311]
+ CRUSH rule 1 x 284 [771]
+ CRUSH rule 1 x 285 [693]
+ CRUSH rule 1 x 286 [364]
+ CRUSH rule 1 x 287 [591]
+ CRUSH rule 1 x 288 [965]
+ CRUSH rule 1 x 289 [225]
+ CRUSH rule 1 x 290 [577]
+ CRUSH rule 1 x 291 [160]
+ CRUSH rule 1 x 292 [873]
+ CRUSH rule 1 x 293 [100]
+ CRUSH rule 1 x 294 [285]
+ CRUSH rule 1 x 295 [938]
+ CRUSH rule 1 x 296 [850]
+ CRUSH rule 1 x 297 [951]
+ CRUSH rule 1 x 298 [173]
+ CRUSH rule 1 x 299 [598]
+ CRUSH rule 1 x 300 [531]
+ CRUSH rule 1 x 301 [823]
+ CRUSH rule 1 x 302 [184]
+ CRUSH rule 1 x 303 [521]
+ CRUSH rule 1 x 304 [980]
+ CRUSH rule 1 x 305 [153]
+ CRUSH rule 1 x 306 [423]
+ CRUSH rule 1 x 307 [997]
+ CRUSH rule 1 x 308 [991]
+ CRUSH rule 1 x 309 [860]
+ CRUSH rule 1 x 310 [589]
+ CRUSH rule 1 x 311 [477]
+ CRUSH rule 1 x 312 [887]
+ CRUSH rule 1 x 313 [802]
+ CRUSH rule 1 x 314 [654]
+ CRUSH rule 1 x 315 [767]
+ CRUSH rule 1 x 316 [778]
+ CRUSH rule 1 x 317 [184]
+ CRUSH rule 1 x 318 [525]
+ CRUSH rule 1 x 319 [476]
+ CRUSH rule 1 x 320 [149]
+ CRUSH rule 1 x 321 [710]
+ CRUSH rule 1 x 322 [175]
+ CRUSH rule 1 x 323 [819]
+ CRUSH rule 1 x 324 [16]
+ CRUSH rule 1 x 325 [486]
+ CRUSH rule 1 x 326 [613]
+ CRUSH rule 1 x 327 [125]
+ CRUSH rule 1 x 328 [807]
+ CRUSH rule 1 x 329 [588]
+ CRUSH rule 1 x 330 [932]
+ CRUSH rule 1 x 331 [341]
+ CRUSH rule 1 x 332 [153]
+ CRUSH rule 1 x 333 [745]
+ CRUSH rule 1 x 334 [614]
+ CRUSH rule 1 x 335 [518]
+ CRUSH rule 1 x 336 [389]
+ CRUSH rule 1 x 337 [753]
+ CRUSH rule 1 x 338 [128]
+ CRUSH rule 1 x 339 [430]
+ CRUSH rule 1 x 340 [541]
+ CRUSH rule 1 x 341 [402]
+ CRUSH rule 1 x 342 [982]
+ CRUSH rule 1 x 343 [833]
+ CRUSH rule 1 x 344 [784]
+ CRUSH rule 1 x 345 [546]
+ CRUSH rule 1 x 346 [302]
+ CRUSH rule 1 x 347 [488]
+ CRUSH rule 1 x 348 [903]
+ CRUSH rule 1 x 349 [471]
+ CRUSH rule 1 x 350 [348]
+ CRUSH rule 1 x 351 [961]
+ CRUSH rule 1 x 352 [728]
+ CRUSH rule 1 x 353 [904]
+ CRUSH rule 1 x 354 [345]
+ CRUSH rule 1 x 355 [50]
+ CRUSH rule 1 x 356 [87]
+ CRUSH rule 1 x 357 [762]
+ CRUSH rule 1 x 358 [908]
+ CRUSH rule 1 x 359 [484]
+ CRUSH rule 1 x 360 [173]
+ CRUSH rule 1 x 361 [404]
+ CRUSH rule 1 x 362 [403]
+ CRUSH rule 1 x 363 [639]
+ CRUSH rule 1 x 364 [752]
+ CRUSH rule 1 x 365 [956]
+ CRUSH rule 1 x 366 [860]
+ CRUSH rule 1 x 367 [205]
+ CRUSH rule 1 x 368 [301]
+ CRUSH rule 1 x 369 [452]
+ CRUSH rule 1 x 370 [11]
+ CRUSH rule 1 x 371 [124]
+ CRUSH rule 1 x 372 [253]
+ CRUSH rule 1 x 373 [715]
+ CRUSH rule 1 x 374 [191]
+ CRUSH rule 1 x 375 [711]
+ CRUSH rule 1 x 376 [597]
+ CRUSH rule 1 x 377 [294]
+ CRUSH rule 1 x 378 [34]
+ CRUSH rule 1 x 379 [869]
+ CRUSH rule 1 x 380 [294]
+ CRUSH rule 1 x 381 [119]
+ CRUSH rule 1 x 382 [69]
+ CRUSH rule 1 x 383 [922]
+ CRUSH rule 1 x 384 [221]
+ CRUSH rule 1 x 385 [561]
+ CRUSH rule 1 x 386 [335]
+ CRUSH rule 1 x 387 [514]
+ CRUSH rule 1 x 388 [587]
+ CRUSH rule 1 x 389 [109]
+ CRUSH rule 1 x 390 [925]
+ CRUSH rule 1 x 391 [267]
+ CRUSH rule 1 x 392 [382]
+ CRUSH rule 1 x 393 [425]
+ CRUSH rule 1 x 394 [898]
+ CRUSH rule 1 x 395 [806]
+ CRUSH rule 1 x 396 [790]
+ CRUSH rule 1 x 397 [136]
+ CRUSH rule 1 x 398 [914]
+ CRUSH rule 1 x 399 [261]
+ CRUSH rule 1 x 400 [661]
+ CRUSH rule 1 x 401 [953]
+ CRUSH rule 1 x 402 [738]
+ CRUSH rule 1 x 403 [573]
+ CRUSH rule 1 x 404 [526]
+ CRUSH rule 1 x 405 [582]
+ CRUSH rule 1 x 406 [768]
+ CRUSH rule 1 x 407 [260]
+ CRUSH rule 1 x 408 [657]
+ CRUSH rule 1 x 409 [498]
+ CRUSH rule 1 x 410 [28]
+ CRUSH rule 1 x 411 [684]
+ CRUSH rule 1 x 412 [261]
+ CRUSH rule 1 x 413 [891]
+ CRUSH rule 1 x 414 [127]
+ CRUSH rule 1 x 415 [272]
+ CRUSH rule 1 x 416 [739]
+ CRUSH rule 1 x 417 [106]
+ CRUSH rule 1 x 418 [525]
+ CRUSH rule 1 x 419 [603]
+ CRUSH rule 1 x 420 [988]
+ CRUSH rule 1 x 421 [761]
+ CRUSH rule 1 x 422 [317]
+ CRUSH rule 1 x 423 [137]
+ CRUSH rule 1 x 424 [920]
+ CRUSH rule 1 x 425 [277]
+ CRUSH rule 1 x 426 [485]
+ CRUSH rule 1 x 427 [242]
+ CRUSH rule 1 x 428 [632]
+ CRUSH rule 1 x 429 [641]
+ CRUSH rule 1 x 430 [626]
+ CRUSH rule 1 x 431 [697]
+ CRUSH rule 1 x 432 [590]
+ CRUSH rule 1 x 433 [284]
+ CRUSH rule 1 x 434 [538]
+ CRUSH rule 1 x 435 [30]
+ CRUSH rule 1 x 436 [164]
+ CRUSH rule 1 x 437 [322]
+ CRUSH rule 1 x 438 [142]
+ CRUSH rule 1 x 439 [119]
+ CRUSH rule 1 x 440 [333]
+ CRUSH rule 1 x 441 [477]
+ CRUSH rule 1 x 442 [274]
+ CRUSH rule 1 x 443 [983]
+ CRUSH rule 1 x 444 [536]
+ CRUSH rule 1 x 445 [485]
+ CRUSH rule 1 x 446 [345]
+ CRUSH rule 1 x 447 [61]
+ CRUSH rule 1 x 448 [333]
+ CRUSH rule 1 x 449 [680]
+ CRUSH rule 1 x 450 [235]
+ CRUSH rule 1 x 451 [961]
+ CRUSH rule 1 x 452 [525]
+ CRUSH rule 1 x 453 [138]
+ CRUSH rule 1 x 454 [137]
+ CRUSH rule 1 x 455 [173]
+ CRUSH rule 1 x 456 [235]
+ CRUSH rule 1 x 457 [450]
+ CRUSH rule 1 x 458 [195]
+ CRUSH rule 1 x 459 [381]
+ CRUSH rule 1 x 460 [972]
+ CRUSH rule 1 x 461 [506]
+ CRUSH rule 1 x 462 [692]
+ CRUSH rule 1 x 463 [788]
+ CRUSH rule 1 x 464 [133]
+ CRUSH rule 1 x 465 [971]
+ CRUSH rule 1 x 466 [394]
+ CRUSH rule 1 x 467 [517]
+ CRUSH rule 1 x 468 [829]
+ CRUSH rule 1 x 469 [987]
+ CRUSH rule 1 x 470 [107]
+ CRUSH rule 1 x 471 [181]
+ CRUSH rule 1 x 472 [547]
+ CRUSH rule 1 x 473 [760]
+ CRUSH rule 1 x 474 [787]
+ CRUSH rule 1 x 475 [662]
+ CRUSH rule 1 x 476 [110]
+ CRUSH rule 1 x 477 [393]
+ CRUSH rule 1 x 478 [246]
+ CRUSH rule 1 x 479 [70]
+ CRUSH rule 1 x 480 [753]
+ CRUSH rule 1 x 481 [470]
+ CRUSH rule 1 x 482 [451]
+ CRUSH rule 1 x 483 [816]
+ CRUSH rule 1 x 484 [540]
+ CRUSH rule 1 x 485 [74]
+ CRUSH rule 1 x 486 [958]
+ CRUSH rule 1 x 487 [228]
+ CRUSH rule 1 x 488 [180]
+ CRUSH rule 1 x 489 [47]
+ CRUSH rule 1 x 490 [905]
+ CRUSH rule 1 x 491 [892]
+ CRUSH rule 1 x 492 [588]
+ CRUSH rule 1 x 493 [353]
+ CRUSH rule 1 x 494 [378]
+ CRUSH rule 1 x 495 [845]
+ CRUSH rule 1 x 496 [13]
+ CRUSH rule 1 x 497 [796]
+ CRUSH rule 1 x 498 [412]
+ CRUSH rule 1 x 499 [330]
+ CRUSH rule 1 x 500 [820]
+ CRUSH rule 1 x 501 [110]
+ CRUSH rule 1 x 502 [336]
+ CRUSH rule 1 x 503 [922]
+ CRUSH rule 1 x 504 [483]
+ CRUSH rule 1 x 505 [482]
+ CRUSH rule 1 x 506 [493]
+ CRUSH rule 1 x 507 [12]
+ CRUSH rule 1 x 508 [227]
+ CRUSH rule 1 x 509 [807]
+ CRUSH rule 1 x 510 [134]
+ CRUSH rule 1 x 511 [212]
+ CRUSH rule 1 x 512 [236]
+ CRUSH rule 1 x 513 [994]
+ CRUSH rule 1 x 514 [45]
+ CRUSH rule 1 x 515 [504]
+ CRUSH rule 1 x 516 [285]
+ CRUSH rule 1 x 517 [300]
+ CRUSH rule 1 x 518 [397]
+ CRUSH rule 1 x 519 [86]
+ CRUSH rule 1 x 520 [900]
+ CRUSH rule 1 x 521 [31]
+ CRUSH rule 1 x 522 [390]
+ CRUSH rule 1 x 523 [618]
+ CRUSH rule 1 x 524 [635]
+ CRUSH rule 1 x 525 [311]
+ CRUSH rule 1 x 526 [48]
+ CRUSH rule 1 x 527 [202]
+ CRUSH rule 1 x 528 [565]
+ CRUSH rule 1 x 529 [934]
+ CRUSH rule 1 x 530 [502]
+ CRUSH rule 1 x 531 [681]
+ CRUSH rule 1 x 532 [422]
+ CRUSH rule 1 x 533 [863]
+ CRUSH rule 1 x 534 [962]
+ CRUSH rule 1 x 535 [89]
+ CRUSH rule 1 x 536 [499]
+ CRUSH rule 1 x 537 [676]
+ CRUSH rule 1 x 538 [58]
+ CRUSH rule 1 x 539 [837]
+ CRUSH rule 1 x 540 [831]
+ CRUSH rule 1 x 541 [582]
+ CRUSH rule 1 x 542 [472]
+ CRUSH rule 1 x 543 [382]
+ CRUSH rule 1 x 544 [947]
+ CRUSH rule 1 x 545 [425]
+ CRUSH rule 1 x 546 [18]
+ CRUSH rule 1 x 547 [445]
+ CRUSH rule 1 x 548 [367]
+ CRUSH rule 1 x 549 [125]
+ CRUSH rule 1 x 550 [425]
+ CRUSH rule 1 x 551 [44]
+ CRUSH rule 1 x 552 [246]
+ CRUSH rule 1 x 553 [71]
+ CRUSH rule 1 x 554 [207]
+ CRUSH rule 1 x 555 [570]
+ CRUSH rule 1 x 556 [674]
+ CRUSH rule 1 x 557 [347]
+ CRUSH rule 1 x 558 [627]
+ CRUSH rule 1 x 559 [940]
+ CRUSH rule 1 x 560 [295]
+ CRUSH rule 1 x 561 [506]
+ CRUSH rule 1 x 562 [718]
+ CRUSH rule 1 x 563 [552]
+ CRUSH rule 1 x 564 [835]
+ CRUSH rule 1 x 565 [8]
+ CRUSH rule 1 x 566 [600]
+ CRUSH rule 1 x 567 [999]
+ CRUSH rule 1 x 568 [252]
+ CRUSH rule 1 x 569 [643]
+ CRUSH rule 1 x 570 [617]
+ CRUSH rule 1 x 571 [757]
+ CRUSH rule 1 x 572 [299]
+ CRUSH rule 1 x 573 [25]
+ CRUSH rule 1 x 574 [215]
+ CRUSH rule 1 x 575 [225]
+ CRUSH rule 1 x 576 [627]
+ CRUSH rule 1 x 577 [237]
+ CRUSH rule 1 x 578 [885]
+ CRUSH rule 1 x 579 [924]
+ CRUSH rule 1 x 580 [718]
+ CRUSH rule 1 x 581 [219]
+ CRUSH rule 1 x 582 [893]
+ CRUSH rule 1 x 583 [246]
+ CRUSH rule 1 x 584 [336]
+ CRUSH rule 1 x 585 [324]
+ CRUSH rule 1 x 586 [558]
+ CRUSH rule 1 x 587 [985]
+ CRUSH rule 1 x 588 [211]
+ CRUSH rule 1 x 589 [129]
+ CRUSH rule 1 x 590 [467]
+ CRUSH rule 1 x 591 [758]
+ CRUSH rule 1 x 592 [525]
+ CRUSH rule 1 x 593 [601]
+ CRUSH rule 1 x 594 [227]
+ CRUSH rule 1 x 595 [720]
+ CRUSH rule 1 x 596 [751]
+ CRUSH rule 1 x 597 [129]
+ CRUSH rule 1 x 598 [679]
+ CRUSH rule 1 x 599 [668]
+ CRUSH rule 1 x 600 [143]
+ CRUSH rule 1 x 601 [326]
+ CRUSH rule 1 x 602 [860]
+ CRUSH rule 1 x 603 [709]
+ CRUSH rule 1 x 604 [571]
+ CRUSH rule 1 x 605 [252]
+ CRUSH rule 1 x 606 [339]
+ CRUSH rule 1 x 607 [590]
+ CRUSH rule 1 x 608 [145]
+ CRUSH rule 1 x 609 [973]
+ CRUSH rule 1 x 610 [435]
+ CRUSH rule 1 x 611 [559]
+ CRUSH rule 1 x 612 [273]
+ CRUSH rule 1 x 613 [828]
+ CRUSH rule 1 x 614 [478]
+ CRUSH rule 1 x 615 [392]
+ CRUSH rule 1 x 616 [778]
+ CRUSH rule 1 x 617 [622]
+ CRUSH rule 1 x 618 [149]
+ CRUSH rule 1 x 619 [604]
+ CRUSH rule 1 x 620 [181]
+ CRUSH rule 1 x 621 [735]
+ CRUSH rule 1 x 622 [661]
+ CRUSH rule 1 x 623 [142]
+ CRUSH rule 1 x 624 [360]
+ CRUSH rule 1 x 625 [541]
+ CRUSH rule 1 x 626 [364]
+ CRUSH rule 1 x 627 [458]
+ CRUSH rule 1 x 628 [250]
+ CRUSH rule 1 x 629 [928]
+ CRUSH rule 1 x 630 [243]
+ CRUSH rule 1 x 631 [438]
+ CRUSH rule 1 x 632 [797]
+ CRUSH rule 1 x 633 [993]
+ CRUSH rule 1 x 634 [239]
+ CRUSH rule 1 x 635 [640]
+ CRUSH rule 1 x 636 [173]
+ CRUSH rule 1 x 637 [0]
+ CRUSH rule 1 x 638 [702]
+ CRUSH rule 1 x 639 [475]
+ CRUSH rule 1 x 640 [31]
+ CRUSH rule 1 x 641 [296]
+ CRUSH rule 1 x 642 [894]
+ CRUSH rule 1 x 643 [117]
+ CRUSH rule 1 x 644 [438]
+ CRUSH rule 1 x 645 [982]
+ CRUSH rule 1 x 646 [334]
+ CRUSH rule 1 x 647 [933]
+ CRUSH rule 1 x 648 [22]
+ CRUSH rule 1 x 649 [503]
+ CRUSH rule 1 x 650 [328]
+ CRUSH rule 1 x 651 [3]
+ CRUSH rule 1 x 652 [495]
+ CRUSH rule 1 x 653 [185]
+ CRUSH rule 1 x 654 [130]
+ CRUSH rule 1 x 655 [560]
+ CRUSH rule 1 x 656 [219]
+ CRUSH rule 1 x 657 [233]
+ CRUSH rule 1 x 658 [778]
+ CRUSH rule 1 x 659 [240]
+ CRUSH rule 1 x 660 [244]
+ CRUSH rule 1 x 661 [184]
+ CRUSH rule 1 x 662 [65]
+ CRUSH rule 1 x 663 [323]
+ CRUSH rule 1 x 664 [865]
+ CRUSH rule 1 x 665 [420]
+ CRUSH rule 1 x 666 [319]
+ CRUSH rule 1 x 667 [875]
+ CRUSH rule 1 x 668 [331]
+ CRUSH rule 1 x 669 [915]
+ CRUSH rule 1 x 670 [845]
+ CRUSH rule 1 x 671 [108]
+ CRUSH rule 1 x 672 [578]
+ CRUSH rule 1 x 673 [442]
+ CRUSH rule 1 x 674 [588]
+ CRUSH rule 1 x 675 [489]
+ CRUSH rule 1 x 676 [928]
+ CRUSH rule 1 x 677 [399]
+ CRUSH rule 1 x 678 [546]
+ CRUSH rule 1 x 679 [988]
+ CRUSH rule 1 x 680 [335]
+ CRUSH rule 1 x 681 [690]
+ CRUSH rule 1 x 682 [196]
+ CRUSH rule 1 x 683 [627]
+ CRUSH rule 1 x 684 [38]
+ CRUSH rule 1 x 685 [841]
+ CRUSH rule 1 x 686 [336]
+ CRUSH rule 1 x 687 [20]
+ CRUSH rule 1 x 688 [463]
+ CRUSH rule 1 x 689 [569]
+ CRUSH rule 1 x 690 [551]
+ CRUSH rule 1 x 691 [766]
+ CRUSH rule 1 x 692 [739]
+ CRUSH rule 1 x 693 [339]
+ CRUSH rule 1 x 694 [405]
+ CRUSH rule 1 x 695 [622]
+ CRUSH rule 1 x 696 [558]
+ CRUSH rule 1 x 697 [818]
+ CRUSH rule 1 x 698 [178]
+ CRUSH rule 1 x 699 [450]
+ CRUSH rule 1 x 700 [502]
+ CRUSH rule 1 x 701 [4]
+ CRUSH rule 1 x 702 [177]
+ CRUSH rule 1 x 703 [354]
+ CRUSH rule 1 x 704 [646]
+ CRUSH rule 1 x 705 [921]
+ CRUSH rule 1 x 706 [652]
+ CRUSH rule 1 x 707 [345]
+ CRUSH rule 1 x 708 [333]
+ CRUSH rule 1 x 709 [45]
+ CRUSH rule 1 x 710 [94]
+ CRUSH rule 1 x 711 [227]
+ CRUSH rule 1 x 712 [398]
+ CRUSH rule 1 x 713 [116]
+ CRUSH rule 1 x 714 [111]
+ CRUSH rule 1 x 715 [531]
+ CRUSH rule 1 x 716 [169]
+ CRUSH rule 1 x 717 [417]
+ CRUSH rule 1 x 718 [992]
+ CRUSH rule 1 x 719 [936]
+ CRUSH rule 1 x 720 [370]
+ CRUSH rule 1 x 721 [320]
+ CRUSH rule 1 x 722 [7]
+ CRUSH rule 1 x 723 [270]
+ CRUSH rule 1 x 724 [666]
+ CRUSH rule 1 x 725 [794]
+ CRUSH rule 1 x 726 [420]
+ CRUSH rule 1 x 727 [561]
+ CRUSH rule 1 x 728 [951]
+ CRUSH rule 1 x 729 [656]
+ CRUSH rule 1 x 730 [3]
+ CRUSH rule 1 x 731 [852]
+ CRUSH rule 1 x 732 [983]
+ CRUSH rule 1 x 733 [285]
+ CRUSH rule 1 x 734 [125]
+ CRUSH rule 1 x 735 [417]
+ CRUSH rule 1 x 736 [749]
+ CRUSH rule 1 x 737 [644]
+ CRUSH rule 1 x 738 [449]
+ CRUSH rule 1 x 739 [341]
+ CRUSH rule 1 x 740 [874]
+ CRUSH rule 1 x 741 [189]
+ CRUSH rule 1 x 742 [912]
+ CRUSH rule 1 x 743 [654]
+ CRUSH rule 1 x 744 [725]
+ CRUSH rule 1 x 745 [787]
+ CRUSH rule 1 x 746 [757]
+ CRUSH rule 1 x 747 [700]
+ CRUSH rule 1 x 748 [557]
+ CRUSH rule 1 x 749 [772]
+ CRUSH rule 1 x 750 [946]
+ CRUSH rule 1 x 751 [996]
+ CRUSH rule 1 x 752 [746]
+ CRUSH rule 1 x 753 [741]
+ CRUSH rule 1 x 754 [648]
+ CRUSH rule 1 x 755 [157]
+ CRUSH rule 1 x 756 [416]
+ CRUSH rule 1 x 757 [599]
+ CRUSH rule 1 x 758 [994]
+ CRUSH rule 1 x 759 [959]
+ CRUSH rule 1 x 760 [518]
+ CRUSH rule 1 x 761 [285]
+ CRUSH rule 1 x 762 [591]
+ CRUSH rule 1 x 763 [908]
+ CRUSH rule 1 x 764 [787]
+ CRUSH rule 1 x 765 [327]
+ CRUSH rule 1 x 766 [84]
+ CRUSH rule 1 x 767 [370]
+ CRUSH rule 1 x 768 [826]
+ CRUSH rule 1 x 769 [67]
+ CRUSH rule 1 x 770 [593]
+ CRUSH rule 1 x 771 [309]
+ CRUSH rule 1 x 772 [12]
+ CRUSH rule 1 x 773 [253]
+ CRUSH rule 1 x 774 [164]
+ CRUSH rule 1 x 775 [703]
+ CRUSH rule 1 x 776 [728]
+ CRUSH rule 1 x 777 [981]
+ CRUSH rule 1 x 778 [411]
+ CRUSH rule 1 x 779 [346]
+ CRUSH rule 1 x 780 [476]
+ CRUSH rule 1 x 781 [10]
+ CRUSH rule 1 x 782 [462]
+ CRUSH rule 1 x 783 [580]
+ CRUSH rule 1 x 784 [413]
+ CRUSH rule 1 x 785 [341]
+ CRUSH rule 1 x 786 [411]
+ CRUSH rule 1 x 787 [605]
+ CRUSH rule 1 x 788 [226]
+ CRUSH rule 1 x 789 [545]
+ CRUSH rule 1 x 790 [414]
+ CRUSH rule 1 x 791 [660]
+ CRUSH rule 1 x 792 [287]
+ CRUSH rule 1 x 793 [631]
+ CRUSH rule 1 x 794 [931]
+ CRUSH rule 1 x 795 [551]
+ CRUSH rule 1 x 796 [814]
+ CRUSH rule 1 x 797 [64]
+ CRUSH rule 1 x 798 [422]
+ CRUSH rule 1 x 799 [824]
+ CRUSH rule 1 x 800 [862]
+ CRUSH rule 1 x 801 [145]
+ CRUSH rule 1 x 802 [570]
+ CRUSH rule 1 x 803 [151]
+ CRUSH rule 1 x 804 [467]
+ CRUSH rule 1 x 805 [621]
+ CRUSH rule 1 x 806 [898]
+ CRUSH rule 1 x 807 [354]
+ CRUSH rule 1 x 808 [7]
+ CRUSH rule 1 x 809 [70]
+ CRUSH rule 1 x 810 [701]
+ CRUSH rule 1 x 811 [248]
+ CRUSH rule 1 x 812 [230]
+ CRUSH rule 1 x 813 [805]
+ CRUSH rule 1 x 814 [54]
+ CRUSH rule 1 x 815 [679]
+ CRUSH rule 1 x 816 [919]
+ CRUSH rule 1 x 817 [765]
+ CRUSH rule 1 x 818 [415]
+ CRUSH rule 1 x 819 [721]
+ CRUSH rule 1 x 820 [218]
+ CRUSH rule 1 x 821 [185]
+ CRUSH rule 1 x 822 [356]
+ CRUSH rule 1 x 823 [220]
+ CRUSH rule 1 x 824 [292]
+ CRUSH rule 1 x 825 [949]
+ CRUSH rule 1 x 826 [767]
+ CRUSH rule 1 x 827 [631]
+ CRUSH rule 1 x 828 [288]
+ CRUSH rule 1 x 829 [990]
+ CRUSH rule 1 x 830 [152]
+ CRUSH rule 1 x 831 [814]
+ CRUSH rule 1 x 832 [235]
+ CRUSH rule 1 x 833 [657]
+ CRUSH rule 1 x 834 [907]
+ CRUSH rule 1 x 835 [784]
+ CRUSH rule 1 x 836 [951]
+ CRUSH rule 1 x 837 [556]
+ CRUSH rule 1 x 838 [329]
+ CRUSH rule 1 x 839 [568]
+ CRUSH rule 1 x 840 [45]
+ CRUSH rule 1 x 841 [652]
+ CRUSH rule 1 x 842 [629]
+ CRUSH rule 1 x 843 [799]
+ CRUSH rule 1 x 844 [694]
+ CRUSH rule 1 x 845 [332]
+ CRUSH rule 1 x 846 [452]
+ CRUSH rule 1 x 847 [399]
+ CRUSH rule 1 x 848 [303]
+ CRUSH rule 1 x 849 [666]
+ CRUSH rule 1 x 850 [644]
+ CRUSH rule 1 x 851 [527]
+ CRUSH rule 1 x 852 [31]
+ CRUSH rule 1 x 853 [483]
+ CRUSH rule 1 x 854 [697]
+ CRUSH rule 1 x 855 [837]
+ CRUSH rule 1 x 856 [712]
+ CRUSH rule 1 x 857 [77]
+ CRUSH rule 1 x 858 [412]
+ CRUSH rule 1 x 859 [173]
+ CRUSH rule 1 x 860 [776]
+ CRUSH rule 1 x 861 [705]
+ CRUSH rule 1 x 862 [809]
+ CRUSH rule 1 x 863 [349]
+ CRUSH rule 1 x 864 [717]
+ CRUSH rule 1 x 865 [857]
+ CRUSH rule 1 x 866 [394]
+ CRUSH rule 1 x 867 [640]
+ CRUSH rule 1 x 868 [613]
+ CRUSH rule 1 x 869 [973]
+ CRUSH rule 1 x 870 [505]
+ CRUSH rule 1 x 871 [239]
+ CRUSH rule 1 x 872 [21]
+ CRUSH rule 1 x 873 [954]
+ CRUSH rule 1 x 874 [54]
+ CRUSH rule 1 x 875 [809]
+ CRUSH rule 1 x 876 [483]
+ CRUSH rule 1 x 877 [542]
+ CRUSH rule 1 x 878 [217]
+ CRUSH rule 1 x 879 [999]
+ CRUSH rule 1 x 880 [678]
+ CRUSH rule 1 x 881 [394]
+ CRUSH rule 1 x 882 [467]
+ CRUSH rule 1 x 883 [802]
+ CRUSH rule 1 x 884 [653]
+ CRUSH rule 1 x 885 [898]
+ CRUSH rule 1 x 886 [434]
+ CRUSH rule 1 x 887 [297]
+ CRUSH rule 1 x 888 [863]
+ CRUSH rule 1 x 889 [105]
+ CRUSH rule 1 x 890 [550]
+ CRUSH rule 1 x 891 [575]
+ CRUSH rule 1 x 892 [259]
+ CRUSH rule 1 x 893 [902]
+ CRUSH rule 1 x 894 [180]
+ CRUSH rule 1 x 895 [725]
+ CRUSH rule 1 x 896 [951]
+ CRUSH rule 1 x 897 [810]
+ CRUSH rule 1 x 898 [979]
+ CRUSH rule 1 x 899 [685]
+ CRUSH rule 1 x 900 [530]
+ CRUSH rule 1 x 901 [740]
+ CRUSH rule 1 x 902 [800]
+ CRUSH rule 1 x 903 [230]
+ CRUSH rule 1 x 904 [346]
+ CRUSH rule 1 x 905 [530]
+ CRUSH rule 1 x 906 [80]
+ CRUSH rule 1 x 907 [365]
+ CRUSH rule 1 x 908 [204]
+ CRUSH rule 1 x 909 [883]
+ CRUSH rule 1 x 910 [549]
+ CRUSH rule 1 x 911 [325]
+ CRUSH rule 1 x 912 [874]
+ CRUSH rule 1 x 913 [331]
+ CRUSH rule 1 x 914 [836]
+ CRUSH rule 1 x 915 [245]
+ CRUSH rule 1 x 916 [77]
+ CRUSH rule 1 x 917 [239]
+ CRUSH rule 1 x 918 [988]
+ CRUSH rule 1 x 919 [783]
+ CRUSH rule 1 x 920 [623]
+ CRUSH rule 1 x 921 [105]
+ CRUSH rule 1 x 922 [887]
+ CRUSH rule 1 x 923 [223]
+ CRUSH rule 1 x 924 [25]
+ CRUSH rule 1 x 925 [912]
+ CRUSH rule 1 x 926 [968]
+ CRUSH rule 1 x 927 [277]
+ CRUSH rule 1 x 928 [554]
+ CRUSH rule 1 x 929 [761]
+ CRUSH rule 1 x 930 [814]
+ CRUSH rule 1 x 931 [29]
+ CRUSH rule 1 x 932 [446]
+ CRUSH rule 1 x 933 [352]
+ CRUSH rule 1 x 934 [730]
+ CRUSH rule 1 x 935 [731]
+ CRUSH rule 1 x 936 [322]
+ CRUSH rule 1 x 937 [822]
+ CRUSH rule 1 x 938 [557]
+ CRUSH rule 1 x 939 [150]
+ CRUSH rule 1 x 940 [638]
+ CRUSH rule 1 x 941 [730]
+ CRUSH rule 1 x 942 [62]
+ CRUSH rule 1 x 943 [165]
+ CRUSH rule 1 x 944 [199]
+ CRUSH rule 1 x 945 [946]
+ CRUSH rule 1 x 946 [595]
+ CRUSH rule 1 x 947 [800]
+ CRUSH rule 1 x 948 [132]
+ CRUSH rule 1 x 949 [792]
+ CRUSH rule 1 x 950 [111]
+ CRUSH rule 1 x 951 [414]
+ CRUSH rule 1 x 952 [775]
+ CRUSH rule 1 x 953 [349]
+ CRUSH rule 1 x 954 [570]
+ CRUSH rule 1 x 955 [729]
+ CRUSH rule 1 x 956 [519]
+ CRUSH rule 1 x 957 [242]
+ CRUSH rule 1 x 958 [84]
+ CRUSH rule 1 x 959 [270]
+ CRUSH rule 1 x 960 [458]
+ CRUSH rule 1 x 961 [981]
+ CRUSH rule 1 x 962 [623]
+ CRUSH rule 1 x 963 [291]
+ CRUSH rule 1 x 964 [28]
+ CRUSH rule 1 x 965 [675]
+ CRUSH rule 1 x 966 [836]
+ CRUSH rule 1 x 967 [966]
+ CRUSH rule 1 x 968 [864]
+ CRUSH rule 1 x 969 [729]
+ CRUSH rule 1 x 970 [800]
+ CRUSH rule 1 x 971 [737]
+ CRUSH rule 1 x 972 [952]
+ CRUSH rule 1 x 973 [356]
+ CRUSH rule 1 x 974 [545]
+ CRUSH rule 1 x 975 [336]
+ CRUSH rule 1 x 976 [446]
+ CRUSH rule 1 x 977 [202]
+ CRUSH rule 1 x 978 [612]
+ CRUSH rule 1 x 979 [843]
+ CRUSH rule 1 x 980 [60]
+ CRUSH rule 1 x 981 [702]
+ CRUSH rule 1 x 982 [298]
+ CRUSH rule 1 x 983 [723]
+ CRUSH rule 1 x 984 [723]
+ CRUSH rule 1 x 985 [945]
+ CRUSH rule 1 x 986 [772]
+ CRUSH rule 1 x 987 [88]
+ CRUSH rule 1 x 988 [522]
+ CRUSH rule 1 x 989 [578]
+ CRUSH rule 1 x 990 [638]
+ CRUSH rule 1 x 991 [530]
+ CRUSH rule 1 x 992 [925]
+ CRUSH rule 1 x 993 [991]
+ CRUSH rule 1 x 994 [276]
+ CRUSH rule 1 x 995 [288]
+ CRUSH rule 1 x 996 [887]
+ CRUSH rule 1 x 997 [110]
+ CRUSH rule 1 x 998 [435]
+ CRUSH rule 1 x 999 [876]
+ CRUSH rule 1 x 1000 [178]
+ CRUSH rule 1 x 1001 [99]
+ CRUSH rule 1 x 1002 [515]
+ CRUSH rule 1 x 1003 [104]
+ CRUSH rule 1 x 1004 [269]
+ CRUSH rule 1 x 1005 [369]
+ CRUSH rule 1 x 1006 [40]
+ CRUSH rule 1 x 1007 [978]
+ CRUSH rule 1 x 1008 [965]
+ CRUSH rule 1 x 1009 [598]
+ CRUSH rule 1 x 1010 [767]
+ CRUSH rule 1 x 1011 [289]
+ CRUSH rule 1 x 1012 [128]
+ CRUSH rule 1 x 1013 [979]
+ CRUSH rule 1 x 1014 [979]
+ CRUSH rule 1 x 1015 [277]
+ CRUSH rule 1 x 1016 [262]
+ CRUSH rule 1 x 1017 [150]
+ CRUSH rule 1 x 1018 [555]
+ CRUSH rule 1 x 1019 [513]
+ CRUSH rule 1 x 1020 [158]
+ CRUSH rule 1 x 1021 [915]
+ CRUSH rule 1 x 1022 [967]
+ CRUSH rule 1 x 1023 [488]
+ rule 1 (metadata) num_rep 1 result size == 1:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705]
+ CRUSH rule 1 x 1 [876,250]
+ CRUSH rule 1 x 2 [292,832]
+ CRUSH rule 1 x 3 [623,387]
+ CRUSH rule 1 x 4 [61,334]
+ CRUSH rule 1 x 5 [946,557]
+ CRUSH rule 1 x 6 [576,668]
+ CRUSH rule 1 x 7 [645,753]
+ CRUSH rule 1 x 8 [243,6]
+ CRUSH rule 1 x 9 [22,578]
+ CRUSH rule 1 x 10 [758,828]
+ CRUSH rule 1 x 11 [769,120]
+ CRUSH rule 1 x 12 [780,364]
+ CRUSH rule 1 x 13 [557,18]
+ CRUSH rule 1 x 14 [59,561]
+ CRUSH rule 1 x 15 [718,928]
+ CRUSH rule 1 x 16 [673,632]
+ CRUSH rule 1 x 17 [648,43]
+ CRUSH rule 1 x 18 [654,219]
+ CRUSH rule 1 x 19 [850,545]
+ CRUSH rule 1 x 20 [717,785]
+ CRUSH rule 1 x 21 [420,57]
+ CRUSH rule 1 x 22 [503,998]
+ CRUSH rule 1 x 23 [411,663]
+ CRUSH rule 1 x 24 [266,861]
+ CRUSH rule 1 x 25 [760,483]
+ CRUSH rule 1 x 26 [903,24]
+ CRUSH rule 1 x 27 [946,188]
+ CRUSH rule 1 x 28 [69,312]
+ CRUSH rule 1 x 29 [844,883]
+ CRUSH rule 1 x 30 [621,18]
+ CRUSH rule 1 x 31 [784,943]
+ CRUSH rule 1 x 32 [173,374]
+ CRUSH rule 1 x 33 [698,336]
+ CRUSH rule 1 x 34 [168,836]
+ CRUSH rule 1 x 35 [274,509]
+ CRUSH rule 1 x 36 [318,215]
+ CRUSH rule 1 x 37 [173,604]
+ CRUSH rule 1 x 38 [708,444]
+ CRUSH rule 1 x 39 [662,198]
+ CRUSH rule 1 x 40 [620,801]
+ CRUSH rule 1 x 41 [811,264]
+ CRUSH rule 1 x 42 [863,179]
+ CRUSH rule 1 x 43 [686,822]
+ CRUSH rule 1 x 44 [396,222]
+ CRUSH rule 1 x 45 [991,694]
+ CRUSH rule 1 x 46 [420,909]
+ CRUSH rule 1 x 47 [467,211]
+ CRUSH rule 1 x 48 [955,329]
+ CRUSH rule 1 x 49 [974,891]
+ CRUSH rule 1 x 50 [870,441]
+ CRUSH rule 1 x 51 [182,930]
+ CRUSH rule 1 x 52 [704,812]
+ CRUSH rule 1 x 53 [185,713]
+ CRUSH rule 1 x 54 [270,441]
+ CRUSH rule 1 x 55 [895,734]
+ CRUSH rule 1 x 56 [564,963]
+ CRUSH rule 1 x 57 [738,130]
+ CRUSH rule 1 x 58 [524,113]
+ CRUSH rule 1 x 59 [408,337]
+ CRUSH rule 1 x 60 [228,790]
+ CRUSH rule 1 x 61 [154,843]
+ CRUSH rule 1 x 62 [594,811]
+ CRUSH rule 1 x 63 [646,67]
+ CRUSH rule 1 x 64 [175,542]
+ CRUSH rule 1 x 65 [745,619]
+ CRUSH rule 1 x 66 [275,468]
+ CRUSH rule 1 x 67 [246,958]
+ CRUSH rule 1 x 68 [711,473]
+ CRUSH rule 1 x 69 [493,924]
+ CRUSH rule 1 x 70 [30,499]
+ CRUSH rule 1 x 71 [984,883]
+ CRUSH rule 1 x 72 [71,286]
+ CRUSH rule 1 x 73 [922,618]
+ CRUSH rule 1 x 74 [629,414]
+ CRUSH rule 1 x 75 [222,20]
+ CRUSH rule 1 x 76 [262,366]
+ CRUSH rule 1 x 77 [638,469]
+ CRUSH rule 1 x 78 [324,511]
+ CRUSH rule 1 x 79 [577,990]
+ CRUSH rule 1 x 80 [501,95]
+ CRUSH rule 1 x 81 [506,812]
+ CRUSH rule 1 x 82 [222,145]
+ CRUSH rule 1 x 83 [71,634]
+ CRUSH rule 1 x 84 [49,761]
+ CRUSH rule 1 x 85 [985,896]
+ CRUSH rule 1 x 86 [537,745]
+ CRUSH rule 1 x 87 [997,317]
+ CRUSH rule 1 x 88 [957,350]
+ CRUSH rule 1 x 89 [399,730]
+ CRUSH rule 1 x 90 [943,706]
+ CRUSH rule 1 x 91 [22,368]
+ CRUSH rule 1 x 92 [532,424]
+ CRUSH rule 1 x 93 [218,489]
+ CRUSH rule 1 x 94 [181,96]
+ CRUSH rule 1 x 95 [343,957]
+ CRUSH rule 1 x 96 [861,270]
+ CRUSH rule 1 x 97 [459,706]
+ CRUSH rule 1 x 98 [327,867]
+ CRUSH rule 1 x 99 [974,133]
+ CRUSH rule 1 x 100 [32,445]
+ CRUSH rule 1 x 101 [142,90]
+ CRUSH rule 1 x 102 [172,129]
+ CRUSH rule 1 x 103 [630,47]
+ CRUSH rule 1 x 104 [758,133]
+ CRUSH rule 1 x 105 [843,604]
+ CRUSH rule 1 x 106 [28,681]
+ CRUSH rule 1 x 107 [74,320]
+ CRUSH rule 1 x 108 [875,593]
+ CRUSH rule 1 x 109 [411,985]
+ CRUSH rule 1 x 110 [440,774]
+ CRUSH rule 1 x 111 [405,742]
+ CRUSH rule 1 x 112 [143,181]
+ CRUSH rule 1 x 113 [153,846]
+ CRUSH rule 1 x 114 [804,892]
+ CRUSH rule 1 x 115 [588,508]
+ CRUSH rule 1 x 116 [327,148]
+ CRUSH rule 1 x 117 [95,594]
+ CRUSH rule 1 x 118 [80,957]
+ CRUSH rule 1 x 119 [386,932]
+ CRUSH rule 1 x 120 [366,312]
+ CRUSH rule 1 x 121 [129,154]
+ CRUSH rule 1 x 122 [873,1]
+ CRUSH rule 1 x 123 [533,415]
+ CRUSH rule 1 x 124 [461,691]
+ CRUSH rule 1 x 125 [342,599]
+ CRUSH rule 1 x 126 [819,781]
+ CRUSH rule 1 x 127 [437,893]
+ CRUSH rule 1 x 128 [679,994]
+ CRUSH rule 1 x 129 [380,685]
+ CRUSH rule 1 x 130 [992,52]
+ CRUSH rule 1 x 131 [469,90]
+ CRUSH rule 1 x 132 [571,250]
+ CRUSH rule 1 x 133 [964,728]
+ CRUSH rule 1 x 134 [999,19]
+ CRUSH rule 1 x 135 [634,101]
+ CRUSH rule 1 x 136 [114,889]
+ CRUSH rule 1 x 137 [839,8]
+ CRUSH rule 1 x 138 [967,949]
+ CRUSH rule 1 x 139 [308,711]
+ CRUSH rule 1 x 140 [764,936]
+ CRUSH rule 1 x 141 [423,302]
+ CRUSH rule 1 x 142 [252,821]
+ CRUSH rule 1 x 143 [33,808]
+ CRUSH rule 1 x 144 [472,88]
+ CRUSH rule 1 x 145 [242,208]
+ CRUSH rule 1 x 146 [290,70]
+ CRUSH rule 1 x 147 [447,352]
+ CRUSH rule 1 x 148 [212,644]
+ CRUSH rule 1 x 149 [9,775]
+ CRUSH rule 1 x 150 [166,456]
+ CRUSH rule 1 x 151 [811,875]
+ CRUSH rule 1 x 152 [449,617]
+ CRUSH rule 1 x 153 [523,537]
+ CRUSH rule 1 x 154 [208,559]
+ CRUSH rule 1 x 155 [569,325]
+ CRUSH rule 1 x 156 [488,121]
+ CRUSH rule 1 x 157 [140,723]
+ CRUSH rule 1 x 158 [786,451]
+ CRUSH rule 1 x 159 [134,664]
+ CRUSH rule 1 x 160 [690,112]
+ CRUSH rule 1 x 161 [324,912]
+ CRUSH rule 1 x 162 [748,567]
+ CRUSH rule 1 x 163 [575,499]
+ CRUSH rule 1 x 164 [314,489]
+ CRUSH rule 1 x 165 [116,209]
+ CRUSH rule 1 x 166 [352,706]
+ CRUSH rule 1 x 167 [27,743]
+ CRUSH rule 1 x 168 [953,898]
+ CRUSH rule 1 x 169 [912,147]
+ CRUSH rule 1 x 170 [421,515]
+ CRUSH rule 1 x 171 [488,584]
+ CRUSH rule 1 x 172 [366,443]
+ CRUSH rule 1 x 173 [863,291]
+ CRUSH rule 1 x 174 [263,555]
+ CRUSH rule 1 x 175 [875,961]
+ CRUSH rule 1 x 176 [745,83]
+ CRUSH rule 1 x 177 [128,244]
+ CRUSH rule 1 x 178 [155,41]
+ CRUSH rule 1 x 179 [593,833]
+ CRUSH rule 1 x 180 [154,734]
+ CRUSH rule 1 x 181 [289,675]
+ CRUSH rule 1 x 182 [730,931]
+ CRUSH rule 1 x 183 [639,237]
+ CRUSH rule 1 x 184 [704,312]
+ CRUSH rule 1 x 185 [97,100]
+ CRUSH rule 1 x 186 [26,665]
+ CRUSH rule 1 x 187 [649,14]
+ CRUSH rule 1 x 188 [682,695]
+ CRUSH rule 1 x 189 [325,693]
+ CRUSH rule 1 x 190 [399,933]
+ CRUSH rule 1 x 191 [629,533]
+ CRUSH rule 1 x 192 [503,578]
+ CRUSH rule 1 x 193 [546,333]
+ CRUSH rule 1 x 194 [242,473]
+ CRUSH rule 1 x 195 [625,719]
+ CRUSH rule 1 x 196 [357,114]
+ CRUSH rule 1 x 197 [306,954]
+ CRUSH rule 1 x 198 [863,791]
+ CRUSH rule 1 x 199 [935,906]
+ CRUSH rule 1 x 200 [373,774]
+ CRUSH rule 1 x 201 [659,320]
+ CRUSH rule 1 x 202 [260,433]
+ CRUSH rule 1 x 203 [36,239]
+ CRUSH rule 1 x 204 [92,516]
+ CRUSH rule 1 x 205 [68,395]
+ CRUSH rule 1 x 206 [570,530]
+ CRUSH rule 1 x 207 [834,457]
+ CRUSH rule 1 x 208 [927,484]
+ CRUSH rule 1 x 209 [878,66]
+ CRUSH rule 1 x 210 [572,981]
+ CRUSH rule 1 x 211 [107,597]
+ CRUSH rule 1 x 212 [389,107]
+ CRUSH rule 1 x 213 [497,717]
+ CRUSH rule 1 x 214 [798,65]
+ CRUSH rule 1 x 215 [233,419]
+ CRUSH rule 1 x 216 [494,464]
+ CRUSH rule 1 x 217 [352,396]
+ CRUSH rule 1 x 218 [895,864]
+ CRUSH rule 1 x 219 [222,534]
+ CRUSH rule 1 x 220 [281,19]
+ CRUSH rule 1 x 221 [64,928]
+ CRUSH rule 1 x 222 [40,544]
+ CRUSH rule 1 x 223 [645,556]
+ CRUSH rule 1 x 224 [647,165]
+ CRUSH rule 1 x 225 [219,714]
+ CRUSH rule 1 x 226 [372,511]
+ CRUSH rule 1 x 227 [925,156]
+ CRUSH rule 1 x 228 [682,404]
+ CRUSH rule 1 x 229 [880,838]
+ CRUSH rule 1 x 230 [328,659]
+ CRUSH rule 1 x 231 [320,383]
+ CRUSH rule 1 x 232 [924,846]
+ CRUSH rule 1 x 233 [948,652]
+ CRUSH rule 1 x 234 [484,943]
+ CRUSH rule 1 x 235 [750,65]
+ CRUSH rule 1 x 236 [551,787]
+ CRUSH rule 1 x 237 [390,157]
+ CRUSH rule 1 x 238 [570,6]
+ CRUSH rule 1 x 239 [729,959]
+ CRUSH rule 1 x 240 [981,241]
+ CRUSH rule 1 x 241 [310,816]
+ CRUSH rule 1 x 242 [161,63]
+ CRUSH rule 1 x 243 [180,394]
+ CRUSH rule 1 x 244 [52,174]
+ CRUSH rule 1 x 245 [523,121]
+ CRUSH rule 1 x 246 [362,893]
+ CRUSH rule 1 x 247 [382,184]
+ CRUSH rule 1 x 248 [129,114]
+ CRUSH rule 1 x 249 [159,683]
+ CRUSH rule 1 x 250 [404,945]
+ CRUSH rule 1 x 251 [661,225]
+ CRUSH rule 1 x 252 [961,226]
+ CRUSH rule 1 x 253 [651,97]
+ CRUSH rule 1 x 254 [123,33]
+ CRUSH rule 1 x 255 [314,649]
+ CRUSH rule 1 x 256 [315,215]
+ CRUSH rule 1 x 257 [825,264]
+ CRUSH rule 1 x 258 [624,789]
+ CRUSH rule 1 x 259 [602,542]
+ CRUSH rule 1 x 260 [717,878]
+ CRUSH rule 1 x 261 [145,517]
+ CRUSH rule 1 x 262 [223,1]
+ CRUSH rule 1 x 263 [462,211]
+ CRUSH rule 1 x 264 [654,471]
+ CRUSH rule 1 x 265 [302,794]
+ CRUSH rule 1 x 266 [202,132]
+ CRUSH rule 1 x 267 [282,938]
+ CRUSH rule 1 x 268 [338,309]
+ CRUSH rule 1 x 269 [738,122]
+ CRUSH rule 1 x 270 [707,982]
+ CRUSH rule 1 x 271 [705,432]
+ CRUSH rule 1 x 272 [756,545]
+ CRUSH rule 1 x 273 [197,502]
+ CRUSH rule 1 x 274 [992,44]
+ CRUSH rule 1 x 275 [544,789]
+ CRUSH rule 1 x 276 [658,467]
+ CRUSH rule 1 x 277 [143,490]
+ CRUSH rule 1 x 278 [492,647]
+ CRUSH rule 1 x 279 [517,792]
+ CRUSH rule 1 x 280 [825,740]
+ CRUSH rule 1 x 281 [224,629]
+ CRUSH rule 1 x 282 [298,661]
+ CRUSH rule 1 x 283 [311,606]
+ CRUSH rule 1 x 284 [771,466]
+ CRUSH rule 1 x 285 [693,362]
+ CRUSH rule 1 x 286 [364,477]
+ CRUSH rule 1 x 287 [591,611]
+ CRUSH rule 1 x 288 [965,541]
+ CRUSH rule 1 x 289 [225,551]
+ CRUSH rule 1 x 290 [577,762]
+ CRUSH rule 1 x 291 [160,903]
+ CRUSH rule 1 x 292 [873,598]
+ CRUSH rule 1 x 293 [100,234]
+ CRUSH rule 1 x 294 [285,943]
+ CRUSH rule 1 x 295 [938,262]
+ CRUSH rule 1 x 296 [850,327]
+ CRUSH rule 1 x 297 [951,53]
+ CRUSH rule 1 x 298 [173,336]
+ CRUSH rule 1 x 299 [598,591]
+ CRUSH rule 1 x 300 [531,957]
+ CRUSH rule 1 x 301 [823,628]
+ CRUSH rule 1 x 302 [184,80]
+ CRUSH rule 1 x 303 [521,766]
+ CRUSH rule 1 x 304 [980,127]
+ CRUSH rule 1 x 305 [153,816]
+ CRUSH rule 1 x 306 [423,739]
+ CRUSH rule 1 x 307 [997,557]
+ CRUSH rule 1 x 308 [991,874]
+ CRUSH rule 1 x 309 [860,394]
+ CRUSH rule 1 x 310 [589,818]
+ CRUSH rule 1 x 311 [477,774]
+ CRUSH rule 1 x 312 [887,853]
+ CRUSH rule 1 x 313 [802,646]
+ CRUSH rule 1 x 314 [654,974]
+ CRUSH rule 1 x 315 [767,227]
+ CRUSH rule 1 x 316 [778,83]
+ CRUSH rule 1 x 317 [184,418]
+ CRUSH rule 1 x 318 [525,410]
+ CRUSH rule 1 x 319 [476,724]
+ CRUSH rule 1 x 320 [149,610]
+ CRUSH rule 1 x 321 [710,79]
+ CRUSH rule 1 x 322 [175,275]
+ CRUSH rule 1 x 323 [819,604]
+ CRUSH rule 1 x 324 [16,745]
+ CRUSH rule 1 x 325 [486,400]
+ CRUSH rule 1 x 326 [613,765]
+ CRUSH rule 1 x 327 [125,289]
+ CRUSH rule 1 x 328 [807,383]
+ CRUSH rule 1 x 329 [588,938]
+ CRUSH rule 1 x 330 [932,644]
+ CRUSH rule 1 x 331 [341,953]
+ CRUSH rule 1 x 332 [153,726]
+ CRUSH rule 1 x 333 [745,845]
+ CRUSH rule 1 x 334 [614,751]
+ CRUSH rule 1 x 335 [518,721]
+ CRUSH rule 1 x 336 [389,424]
+ CRUSH rule 1 x 337 [753,508]
+ CRUSH rule 1 x 338 [128,810]
+ CRUSH rule 1 x 339 [430,308]
+ CRUSH rule 1 x 340 [541,44]
+ CRUSH rule 1 x 341 [402,26]
+ CRUSH rule 1 x 342 [982,57]
+ CRUSH rule 1 x 343 [833,412]
+ CRUSH rule 1 x 344 [784,533]
+ CRUSH rule 1 x 345 [546,300]
+ CRUSH rule 1 x 346 [302,420]
+ CRUSH rule 1 x 347 [488,778]
+ CRUSH rule 1 x 348 [903,744]
+ CRUSH rule 1 x 349 [471,547]
+ CRUSH rule 1 x 350 [348,221]
+ CRUSH rule 1 x 351 [961,582]
+ CRUSH rule 1 x 352 [728,137]
+ CRUSH rule 1 x 353 [904,202]
+ CRUSH rule 1 x 354 [345,226]
+ CRUSH rule 1 x 355 [50,430]
+ CRUSH rule 1 x 356 [87,185]
+ CRUSH rule 1 x 357 [762,459]
+ CRUSH rule 1 x 358 [908,25]
+ CRUSH rule 1 x 359 [484,15]
+ CRUSH rule 1 x 360 [173,378]
+ CRUSH rule 1 x 361 [404,577]
+ CRUSH rule 1 x 362 [403,1]
+ CRUSH rule 1 x 363 [639,911]
+ CRUSH rule 1 x 364 [752,689]
+ CRUSH rule 1 x 365 [956,999]
+ CRUSH rule 1 x 366 [860,925]
+ CRUSH rule 1 x 367 [205,609]
+ CRUSH rule 1 x 368 [301,284]
+ CRUSH rule 1 x 369 [452,658]
+ CRUSH rule 1 x 370 [11,467]
+ CRUSH rule 1 x 371 [124,487]
+ CRUSH rule 1 x 372 [253,48]
+ CRUSH rule 1 x 373 [715,605]
+ CRUSH rule 1 x 374 [191,887]
+ CRUSH rule 1 x 375 [711,385]
+ CRUSH rule 1 x 376 [597,818]
+ CRUSH rule 1 x 377 [294,256]
+ CRUSH rule 1 x 378 [34,151]
+ CRUSH rule 1 x 379 [869,136]
+ CRUSH rule 1 x 380 [294,97]
+ CRUSH rule 1 x 381 [119,710]
+ CRUSH rule 1 x 382 [69,631]
+ CRUSH rule 1 x 383 [922,588]
+ CRUSH rule 1 x 384 [221,945]
+ CRUSH rule 1 x 385 [561,737]
+ CRUSH rule 1 x 386 [335,442]
+ CRUSH rule 1 x 387 [514,43]
+ CRUSH rule 1 x 388 [587,89]
+ CRUSH rule 1 x 389 [109,641]
+ CRUSH rule 1 x 390 [925,149]
+ CRUSH rule 1 x 391 [267,87]
+ CRUSH rule 1 x 392 [382,485]
+ CRUSH rule 1 x 393 [425,721]
+ CRUSH rule 1 x 394 [898,18]
+ CRUSH rule 1 x 395 [806,876]
+ CRUSH rule 1 x 396 [790,970]
+ CRUSH rule 1 x 397 [136,363]
+ CRUSH rule 1 x 398 [914,116]
+ CRUSH rule 1 x 399 [261,94]
+ CRUSH rule 1 x 400 [661,197]
+ CRUSH rule 1 x 401 [953,979]
+ CRUSH rule 1 x 402 [738,819]
+ CRUSH rule 1 x 403 [573,238]
+ CRUSH rule 1 x 404 [526,848]
+ CRUSH rule 1 x 405 [582,505]
+ CRUSH rule 1 x 406 [768,324]
+ CRUSH rule 1 x 407 [260,951]
+ CRUSH rule 1 x 408 [657,81]
+ CRUSH rule 1 x 409 [498,89]
+ CRUSH rule 1 x 410 [28,793]
+ CRUSH rule 1 x 411 [684,992]
+ CRUSH rule 1 x 412 [261,958]
+ CRUSH rule 1 x 413 [891,835]
+ CRUSH rule 1 x 414 [127,459]
+ CRUSH rule 1 x 415 [272,540]
+ CRUSH rule 1 x 416 [739,617]
+ CRUSH rule 1 x 417 [106,209]
+ CRUSH rule 1 x 418 [525,441]
+ CRUSH rule 1 x 419 [603,673]
+ CRUSH rule 1 x 420 [988,213]
+ CRUSH rule 1 x 421 [761,521]
+ CRUSH rule 1 x 422 [317,160]
+ CRUSH rule 1 x 423 [137,807]
+ CRUSH rule 1 x 424 [920,37]
+ CRUSH rule 1 x 425 [277,693]
+ CRUSH rule 1 x 426 [485,936]
+ CRUSH rule 1 x 427 [242,515]
+ CRUSH rule 1 x 428 [632,635]
+ CRUSH rule 1 x 429 [641,73]
+ CRUSH rule 1 x 430 [626,585]
+ CRUSH rule 1 x 431 [697,76]
+ CRUSH rule 1 x 432 [590,526]
+ CRUSH rule 1 x 433 [284,387]
+ CRUSH rule 1 x 434 [538,985]
+ CRUSH rule 1 x 435 [30,318]
+ CRUSH rule 1 x 436 [164,919]
+ CRUSH rule 1 x 437 [322,212]
+ CRUSH rule 1 x 438 [142,392]
+ CRUSH rule 1 x 439 [119,370]
+ CRUSH rule 1 x 440 [333,403]
+ CRUSH rule 1 x 441 [477,727]
+ CRUSH rule 1 x 442 [274,590]
+ CRUSH rule 1 x 443 [983,748]
+ CRUSH rule 1 x 444 [536,509]
+ CRUSH rule 1 x 445 [485,209]
+ CRUSH rule 1 x 446 [345,634]
+ CRUSH rule 1 x 447 [61,845]
+ CRUSH rule 1 x 448 [333,232]
+ CRUSH rule 1 x 449 [680,16]
+ CRUSH rule 1 x 450 [235,214]
+ CRUSH rule 1 x 451 [961,468]
+ CRUSH rule 1 x 452 [525,479]
+ CRUSH rule 1 x 453 [138,466]
+ CRUSH rule 1 x 454 [137,625]
+ CRUSH rule 1 x 455 [173,150]
+ CRUSH rule 1 x 456 [235,226]
+ CRUSH rule 1 x 457 [450,577]
+ CRUSH rule 1 x 458 [195,537]
+ CRUSH rule 1 x 459 [381,555]
+ CRUSH rule 1 x 460 [972,730]
+ CRUSH rule 1 x 461 [506,279]
+ CRUSH rule 1 x 462 [692,959]
+ CRUSH rule 1 x 463 [788,667]
+ CRUSH rule 1 x 464 [133,122]
+ CRUSH rule 1 x 465 [971,190]
+ CRUSH rule 1 x 466 [394,576]
+ CRUSH rule 1 x 467 [517,28]
+ CRUSH rule 1 x 468 [829,143]
+ CRUSH rule 1 x 469 [987,936]
+ CRUSH rule 1 x 470 [107,982]
+ CRUSH rule 1 x 471 [181,897]
+ CRUSH rule 1 x 472 [547,512]
+ CRUSH rule 1 x 473 [760,997]
+ CRUSH rule 1 x 474 [787,418]
+ CRUSH rule 1 x 475 [662,312]
+ CRUSH rule 1 x 476 [110,495]
+ CRUSH rule 1 x 477 [393,954]
+ CRUSH rule 1 x 478 [246,483]
+ CRUSH rule 1 x 479 [70,929]
+ CRUSH rule 1 x 480 [753,119]
+ CRUSH rule 1 x 481 [470,429]
+ CRUSH rule 1 x 482 [451,566]
+ CRUSH rule 1 x 483 [816,72]
+ CRUSH rule 1 x 484 [540,454]
+ CRUSH rule 1 x 485 [74,582]
+ CRUSH rule 1 x 486 [958,595]
+ CRUSH rule 1 x 487 [228,302]
+ CRUSH rule 1 x 488 [180,529]
+ CRUSH rule 1 x 489 [47,617]
+ CRUSH rule 1 x 490 [905,822]
+ CRUSH rule 1 x 491 [892,370]
+ CRUSH rule 1 x 492 [588,959]
+ CRUSH rule 1 x 493 [353,461]
+ CRUSH rule 1 x 494 [378,848]
+ CRUSH rule 1 x 495 [845,653]
+ CRUSH rule 1 x 496 [13,988]
+ CRUSH rule 1 x 497 [796,877]
+ CRUSH rule 1 x 498 [412,337]
+ CRUSH rule 1 x 499 [330,695]
+ CRUSH rule 1 x 500 [820,272]
+ CRUSH rule 1 x 501 [110,44]
+ CRUSH rule 1 x 502 [336,595]
+ CRUSH rule 1 x 503 [922,211]
+ CRUSH rule 1 x 504 [483,52]
+ CRUSH rule 1 x 505 [482,598]
+ CRUSH rule 1 x 506 [493,123]
+ CRUSH rule 1 x 507 [12,598]
+ CRUSH rule 1 x 508 [227,157]
+ CRUSH rule 1 x 509 [807,242]
+ CRUSH rule 1 x 510 [134,437]
+ CRUSH rule 1 x 511 [212,54]
+ CRUSH rule 1 x 512 [236,630]
+ CRUSH rule 1 x 513 [994,693]
+ CRUSH rule 1 x 514 [45,508]
+ CRUSH rule 1 x 515 [504,138]
+ CRUSH rule 1 x 516 [285,409]
+ CRUSH rule 1 x 517 [300,232]
+ CRUSH rule 1 x 518 [397,674]
+ CRUSH rule 1 x 519 [86,750]
+ CRUSH rule 1 x 520 [900,833]
+ CRUSH rule 1 x 521 [31,47]
+ CRUSH rule 1 x 522 [390,16]
+ CRUSH rule 1 x 523 [618,308]
+ CRUSH rule 1 x 524 [635,189]
+ CRUSH rule 1 x 525 [311,916]
+ CRUSH rule 1 x 526 [48,738]
+ CRUSH rule 1 x 527 [202,851]
+ CRUSH rule 1 x 528 [565,827]
+ CRUSH rule 1 x 529 [934,864]
+ CRUSH rule 1 x 530 [502,934]
+ CRUSH rule 1 x 531 [681,627]
+ CRUSH rule 1 x 532 [422,6]
+ CRUSH rule 1 x 533 [863,68]
+ CRUSH rule 1 x 534 [962,931]
+ CRUSH rule 1 x 535 [89,565]
+ CRUSH rule 1 x 536 [499,351]
+ CRUSH rule 1 x 537 [676,547]
+ CRUSH rule 1 x 538 [58,644]
+ CRUSH rule 1 x 539 [837,953]
+ CRUSH rule 1 x 540 [831,50]
+ CRUSH rule 1 x 541 [582,757]
+ CRUSH rule 1 x 542 [472,132]
+ CRUSH rule 1 x 543 [382,272]
+ CRUSH rule 1 x 544 [947,930]
+ CRUSH rule 1 x 545 [425,570]
+ CRUSH rule 1 x 546 [18,65]
+ CRUSH rule 1 x 547 [445,715]
+ CRUSH rule 1 x 548 [367,569]
+ CRUSH rule 1 x 549 [125,715]
+ CRUSH rule 1 x 550 [425,599]
+ CRUSH rule 1 x 551 [44,1]
+ CRUSH rule 1 x 552 [246,104]
+ CRUSH rule 1 x 553 [71,703]
+ CRUSH rule 1 x 554 [207,124]
+ CRUSH rule 1 x 555 [570,28]
+ CRUSH rule 1 x 556 [674,152]
+ CRUSH rule 1 x 557 [347,817]
+ CRUSH rule 1 x 558 [627,426]
+ CRUSH rule 1 x 559 [940,630]
+ CRUSH rule 1 x 560 [295,903]
+ CRUSH rule 1 x 561 [506,682]
+ CRUSH rule 1 x 562 [718,529]
+ CRUSH rule 1 x 563 [552,332]
+ CRUSH rule 1 x 564 [835,769]
+ CRUSH rule 1 x 565 [8,167]
+ CRUSH rule 1 x 566 [600,481]
+ CRUSH rule 1 x 567 [999,994]
+ CRUSH rule 1 x 568 [252,431]
+ CRUSH rule 1 x 569 [643,218]
+ CRUSH rule 1 x 570 [617,635]
+ CRUSH rule 1 x 571 [757,80]
+ CRUSH rule 1 x 572 [299,348]
+ CRUSH rule 1 x 573 [25,505]
+ CRUSH rule 1 x 574 [215,431]
+ CRUSH rule 1 x 575 [225,252]
+ CRUSH rule 1 x 576 [627,94]
+ CRUSH rule 1 x 577 [237,809]
+ CRUSH rule 1 x 578 [885,313]
+ CRUSH rule 1 x 579 [924,575]
+ CRUSH rule 1 x 580 [718,51]
+ CRUSH rule 1 x 581 [219,807]
+ CRUSH rule 1 x 582 [893,701]
+ CRUSH rule 1 x 583 [246,930]
+ CRUSH rule 1 x 584 [336,432]
+ CRUSH rule 1 x 585 [324,999]
+ CRUSH rule 1 x 586 [558,230]
+ CRUSH rule 1 x 587 [985,830]
+ CRUSH rule 1 x 588 [211,544]
+ CRUSH rule 1 x 589 [129,21]
+ CRUSH rule 1 x 590 [467,969]
+ CRUSH rule 1 x 591 [758,514]
+ CRUSH rule 1 x 592 [525,253]
+ CRUSH rule 1 x 593 [601,885]
+ CRUSH rule 1 x 594 [227,60]
+ CRUSH rule 1 x 595 [720,854]
+ CRUSH rule 1 x 596 [751,195]
+ CRUSH rule 1 x 597 [129,574]
+ CRUSH rule 1 x 598 [679,207]
+ CRUSH rule 1 x 599 [668,315]
+ CRUSH rule 1 x 600 [143,396]
+ CRUSH rule 1 x 601 [326,573]
+ CRUSH rule 1 x 602 [860,281]
+ CRUSH rule 1 x 603 [709,328]
+ CRUSH rule 1 x 604 [571,62]
+ CRUSH rule 1 x 605 [252,739]
+ CRUSH rule 1 x 606 [339,236]
+ CRUSH rule 1 x 607 [590,248]
+ CRUSH rule 1 x 608 [145,635]
+ CRUSH rule 1 x 609 [973,547]
+ CRUSH rule 1 x 610 [435,816]
+ CRUSH rule 1 x 611 [559,283]
+ CRUSH rule 1 x 612 [273,149]
+ CRUSH rule 1 x 613 [828,614]
+ CRUSH rule 1 x 614 [478,748]
+ CRUSH rule 1 x 615 [392,155]
+ CRUSH rule 1 x 616 [778,637]
+ CRUSH rule 1 x 617 [622,713]
+ CRUSH rule 1 x 618 [149,877]
+ CRUSH rule 1 x 619 [604,163]
+ CRUSH rule 1 x 620 [181,23]
+ CRUSH rule 1 x 621 [735,902]
+ CRUSH rule 1 x 622 [661,824]
+ CRUSH rule 1 x 623 [142,121]
+ CRUSH rule 1 x 624 [360,716]
+ CRUSH rule 1 x 625 [541,167]
+ CRUSH rule 1 x 626 [364,431]
+ CRUSH rule 1 x 627 [458,137]
+ CRUSH rule 1 x 628 [250,350]
+ CRUSH rule 1 x 629 [928,160]
+ CRUSH rule 1 x 630 [243,19]
+ CRUSH rule 1 x 631 [438,221]
+ CRUSH rule 1 x 632 [797,368]
+ CRUSH rule 1 x 633 [993,749]
+ CRUSH rule 1 x 634 [239,351]
+ CRUSH rule 1 x 635 [640,965]
+ CRUSH rule 1 x 636 [173,290]
+ CRUSH rule 1 x 637 [0,918]
+ CRUSH rule 1 x 638 [702,235]
+ CRUSH rule 1 x 639 [475,687]
+ CRUSH rule 1 x 640 [31,664]
+ CRUSH rule 1 x 641 [296,473]
+ CRUSH rule 1 x 642 [894,273]
+ CRUSH rule 1 x 643 [117,111]
+ CRUSH rule 1 x 644 [438,336]
+ CRUSH rule 1 x 645 [982,702]
+ CRUSH rule 1 x 646 [334,804]
+ CRUSH rule 1 x 647 [933,787]
+ CRUSH rule 1 x 648 [22,444]
+ CRUSH rule 1 x 649 [503,229]
+ CRUSH rule 1 x 650 [328,659]
+ CRUSH rule 1 x 651 [3,880]
+ CRUSH rule 1 x 652 [495,977]
+ CRUSH rule 1 x 653 [185,718]
+ CRUSH rule 1 x 654 [130,528]
+ CRUSH rule 1 x 655 [560,872]
+ CRUSH rule 1 x 656 [219,885]
+ CRUSH rule 1 x 657 [233,684]
+ CRUSH rule 1 x 658 [778,6]
+ CRUSH rule 1 x 659 [240,663]
+ CRUSH rule 1 x 660 [244,855]
+ CRUSH rule 1 x 661 [184,270]
+ CRUSH rule 1 x 662 [65,883]
+ CRUSH rule 1 x 663 [323,721]
+ CRUSH rule 1 x 664 [865,113]
+ CRUSH rule 1 x 665 [420,850]
+ CRUSH rule 1 x 666 [319,767]
+ CRUSH rule 1 x 667 [875,39]
+ CRUSH rule 1 x 668 [331,122]
+ CRUSH rule 1 x 669 [915,521]
+ CRUSH rule 1 x 670 [845,659]
+ CRUSH rule 1 x 671 [108,634]
+ CRUSH rule 1 x 672 [578,216]
+ CRUSH rule 1 x 673 [442,74]
+ CRUSH rule 1 x 674 [588,364]
+ CRUSH rule 1 x 675 [489,698]
+ CRUSH rule 1 x 676 [928,911]
+ CRUSH rule 1 x 677 [399,269]
+ CRUSH rule 1 x 678 [546,752]
+ CRUSH rule 1 x 679 [988,25]
+ CRUSH rule 1 x 680 [335,963]
+ CRUSH rule 1 x 681 [690,462]
+ CRUSH rule 1 x 682 [196,588]
+ CRUSH rule 1 x 683 [627,25]
+ CRUSH rule 1 x 684 [38,804]
+ CRUSH rule 1 x 685 [841,368]
+ CRUSH rule 1 x 686 [336,287]
+ CRUSH rule 1 x 687 [20,682]
+ CRUSH rule 1 x 688 [463,371]
+ CRUSH rule 1 x 689 [569,250]
+ CRUSH rule 1 x 690 [551,144]
+ CRUSH rule 1 x 691 [766,464]
+ CRUSH rule 1 x 692 [739,634]
+ CRUSH rule 1 x 693 [339,297]
+ CRUSH rule 1 x 694 [405,26]
+ CRUSH rule 1 x 695 [622,576]
+ CRUSH rule 1 x 696 [558,902]
+ CRUSH rule 1 x 697 [818,222]
+ CRUSH rule 1 x 698 [178,48]
+ CRUSH rule 1 x 699 [450,244]
+ CRUSH rule 1 x 700 [502,771]
+ CRUSH rule 1 x 701 [4,612]
+ CRUSH rule 1 x 702 [177,630]
+ CRUSH rule 1 x 703 [354,178]
+ CRUSH rule 1 x 704 [646,601]
+ CRUSH rule 1 x 705 [921,401]
+ CRUSH rule 1 x 706 [652,877]
+ CRUSH rule 1 x 707 [345,745]
+ CRUSH rule 1 x 708 [333,607]
+ CRUSH rule 1 x 709 [45,187]
+ CRUSH rule 1 x 710 [94,855]
+ CRUSH rule 1 x 711 [227,653]
+ CRUSH rule 1 x 712 [398,953]
+ CRUSH rule 1 x 713 [116,800]
+ CRUSH rule 1 x 714 [111,629]
+ CRUSH rule 1 x 715 [531,291]
+ CRUSH rule 1 x 716 [169,541]
+ CRUSH rule 1 x 717 [417,446]
+ CRUSH rule 1 x 718 [992,383]
+ CRUSH rule 1 x 719 [936,674]
+ CRUSH rule 1 x 720 [370,188]
+ CRUSH rule 1 x 721 [320,859]
+ CRUSH rule 1 x 722 [7,2]
+ CRUSH rule 1 x 723 [270,553]
+ CRUSH rule 1 x 724 [666,822]
+ CRUSH rule 1 x 725 [794,406]
+ CRUSH rule 1 x 726 [420,556]
+ CRUSH rule 1 x 727 [561,461]
+ CRUSH rule 1 x 728 [951,330]
+ CRUSH rule 1 x 729 [656,644]
+ CRUSH rule 1 x 730 [3,558]
+ CRUSH rule 1 x 731 [852,89]
+ CRUSH rule 1 x 732 [983,840]
+ CRUSH rule 1 x 733 [285,396]
+ CRUSH rule 1 x 734 [125,510]
+ CRUSH rule 1 x 735 [417,773]
+ CRUSH rule 1 x 736 [749,396]
+ CRUSH rule 1 x 737 [644,991]
+ CRUSH rule 1 x 738 [449,683]
+ CRUSH rule 1 x 739 [341,220]
+ CRUSH rule 1 x 740 [874,524]
+ CRUSH rule 1 x 741 [189,472]
+ CRUSH rule 1 x 742 [912,581]
+ CRUSH rule 1 x 743 [654,914]
+ CRUSH rule 1 x 744 [725,295]
+ CRUSH rule 1 x 745 [787,858]
+ CRUSH rule 1 x 746 [757,848]
+ CRUSH rule 1 x 747 [700,81]
+ CRUSH rule 1 x 748 [557,436]
+ CRUSH rule 1 x 749 [772,622]
+ CRUSH rule 1 x 750 [946,97]
+ CRUSH rule 1 x 751 [996,618]
+ CRUSH rule 1 x 752 [746,887]
+ CRUSH rule 1 x 753 [741,14]
+ CRUSH rule 1 x 754 [648,349]
+ CRUSH rule 1 x 755 [157,460]
+ CRUSH rule 1 x 756 [416,97]
+ CRUSH rule 1 x 757 [599,839]
+ CRUSH rule 1 x 758 [994,218]
+ CRUSH rule 1 x 759 [959,682]
+ CRUSH rule 1 x 760 [518,943]
+ CRUSH rule 1 x 761 [285,849]
+ CRUSH rule 1 x 762 [591,313]
+ CRUSH rule 1 x 763 [908,411]
+ CRUSH rule 1 x 764 [787,234]
+ CRUSH rule 1 x 765 [327,921]
+ CRUSH rule 1 x 766 [84,161]
+ CRUSH rule 1 x 767 [370,895]
+ CRUSH rule 1 x 768 [826,760]
+ CRUSH rule 1 x 769 [67,768]
+ CRUSH rule 1 x 770 [593,909]
+ CRUSH rule 1 x 771 [309,935]
+ CRUSH rule 1 x 772 [12,125]
+ CRUSH rule 1 x 773 [253,466]
+ CRUSH rule 1 x 774 [164,390]
+ CRUSH rule 1 x 775 [703,47]
+ CRUSH rule 1 x 776 [728,231]
+ CRUSH rule 1 x 777 [981,621]
+ CRUSH rule 1 x 778 [411,456]
+ CRUSH rule 1 x 779 [346,121]
+ CRUSH rule 1 x 780 [476,39]
+ CRUSH rule 1 x 781 [10,130]
+ CRUSH rule 1 x 782 [462,246]
+ CRUSH rule 1 x 783 [580,373]
+ CRUSH rule 1 x 784 [413,113]
+ CRUSH rule 1 x 785 [341,856]
+ CRUSH rule 1 x 786 [411,140]
+ CRUSH rule 1 x 787 [605,522]
+ CRUSH rule 1 x 788 [226,545]
+ CRUSH rule 1 x 789 [545,320]
+ CRUSH rule 1 x 790 [414,748]
+ CRUSH rule 1 x 791 [660,906]
+ CRUSH rule 1 x 792 [287,392]
+ CRUSH rule 1 x 793 [631,133]
+ CRUSH rule 1 x 794 [931,517]
+ CRUSH rule 1 x 795 [551,962]
+ CRUSH rule 1 x 796 [814,4]
+ CRUSH rule 1 x 797 [64,201]
+ CRUSH rule 1 x 798 [422,530]
+ CRUSH rule 1 x 799 [824,32]
+ CRUSH rule 1 x 800 [862,623]
+ CRUSH rule 1 x 801 [145,550]
+ CRUSH rule 1 x 802 [570,19]
+ CRUSH rule 1 x 803 [151,812]
+ CRUSH rule 1 x 804 [467,93]
+ CRUSH rule 1 x 805 [621,223]
+ CRUSH rule 1 x 806 [898,957]
+ CRUSH rule 1 x 807 [354,531]
+ CRUSH rule 1 x 808 [7,96]
+ CRUSH rule 1 x 809 [70,734]
+ CRUSH rule 1 x 810 [701,18]
+ CRUSH rule 1 x 811 [248,547]
+ CRUSH rule 1 x 812 [230,576]
+ CRUSH rule 1 x 813 [805,114]
+ CRUSH rule 1 x 814 [54,619]
+ CRUSH rule 1 x 815 [679,412]
+ CRUSH rule 1 x 816 [919,448]
+ CRUSH rule 1 x 817 [765,830]
+ CRUSH rule 1 x 818 [415,566]
+ CRUSH rule 1 x 819 [721,319]
+ CRUSH rule 1 x 820 [218,301]
+ CRUSH rule 1 x 821 [185,795]
+ CRUSH rule 1 x 822 [356,261]
+ CRUSH rule 1 x 823 [220,281]
+ CRUSH rule 1 x 824 [292,809]
+ CRUSH rule 1 x 825 [949,778]
+ CRUSH rule 1 x 826 [767,818]
+ CRUSH rule 1 x 827 [631,83]
+ CRUSH rule 1 x 828 [288,986]
+ CRUSH rule 1 x 829 [990,667]
+ CRUSH rule 1 x 830 [152,571]
+ CRUSH rule 1 x 831 [814,563]
+ CRUSH rule 1 x 832 [235,641]
+ CRUSH rule 1 x 833 [657,565]
+ CRUSH rule 1 x 834 [907,231]
+ CRUSH rule 1 x 835 [784,262]
+ CRUSH rule 1 x 836 [951,158]
+ CRUSH rule 1 x 837 [556,498]
+ CRUSH rule 1 x 838 [329,274]
+ CRUSH rule 1 x 839 [568,209]
+ CRUSH rule 1 x 840 [45,579]
+ CRUSH rule 1 x 841 [652,702]
+ CRUSH rule 1 x 842 [629,984]
+ CRUSH rule 1 x 843 [799,690]
+ CRUSH rule 1 x 844 [694,600]
+ CRUSH rule 1 x 845 [332,30]
+ CRUSH rule 1 x 846 [452,251]
+ CRUSH rule 1 x 847 [399,681]
+ CRUSH rule 1 x 848 [303,138]
+ CRUSH rule 1 x 849 [666,346]
+ CRUSH rule 1 x 850 [644,511]
+ CRUSH rule 1 x 851 [527,546]
+ CRUSH rule 1 x 852 [31,809]
+ CRUSH rule 1 x 853 [483,330]
+ CRUSH rule 1 x 854 [697,953]
+ CRUSH rule 1 x 855 [837,996]
+ CRUSH rule 1 x 856 [712,40]
+ CRUSH rule 1 x 857 [77,984]
+ CRUSH rule 1 x 858 [412,384]
+ CRUSH rule 1 x 859 [173,760]
+ CRUSH rule 1 x 860 [776,429]
+ CRUSH rule 1 x 861 [705,405]
+ CRUSH rule 1 x 862 [809,44]
+ CRUSH rule 1 x 863 [349,496]
+ CRUSH rule 1 x 864 [717,858]
+ CRUSH rule 1 x 865 [857,603]
+ CRUSH rule 1 x 866 [394,304]
+ CRUSH rule 1 x 867 [640,773]
+ CRUSH rule 1 x 868 [613,950]
+ CRUSH rule 1 x 869 [973,889]
+ CRUSH rule 1 x 870 [505,35]
+ CRUSH rule 1 x 871 [239,264]
+ CRUSH rule 1 x 872 [21,767]
+ CRUSH rule 1 x 873 [954,666]
+ CRUSH rule 1 x 874 [54,510]
+ CRUSH rule 1 x 875 [809,418]
+ CRUSH rule 1 x 876 [483,457]
+ CRUSH rule 1 x 877 [542,531]
+ CRUSH rule 1 x 878 [217,674]
+ CRUSH rule 1 x 879 [999,475]
+ CRUSH rule 1 x 880 [678,573]
+ CRUSH rule 1 x 881 [394,835]
+ CRUSH rule 1 x 882 [467,382]
+ CRUSH rule 1 x 883 [802,744]
+ CRUSH rule 1 x 884 [653,660]
+ CRUSH rule 1 x 885 [898,704]
+ CRUSH rule 1 x 886 [434,357]
+ CRUSH rule 1 x 887 [297,226]
+ CRUSH rule 1 x 888 [863,324]
+ CRUSH rule 1 x 889 [105,102]
+ CRUSH rule 1 x 890 [550,248]
+ CRUSH rule 1 x 891 [575,928]
+ CRUSH rule 1 x 892 [259,862]
+ CRUSH rule 1 x 893 [902,880]
+ CRUSH rule 1 x 894 [180,169]
+ CRUSH rule 1 x 895 [725,849]
+ CRUSH rule 1 x 896 [951,34]
+ CRUSH rule 1 x 897 [810,352]
+ CRUSH rule 1 x 898 [979,433]
+ CRUSH rule 1 x 899 [685,668]
+ CRUSH rule 1 x 900 [530,978]
+ CRUSH rule 1 x 901 [740,107]
+ CRUSH rule 1 x 902 [800,743]
+ CRUSH rule 1 x 903 [230,267]
+ CRUSH rule 1 x 904 [346,949]
+ CRUSH rule 1 x 905 [530,397]
+ CRUSH rule 1 x 906 [80,426]
+ CRUSH rule 1 x 907 [365,968]
+ CRUSH rule 1 x 908 [204,832]
+ CRUSH rule 1 x 909 [883,989]
+ CRUSH rule 1 x 910 [549,593]
+ CRUSH rule 1 x 911 [325,847]
+ CRUSH rule 1 x 912 [874,888]
+ CRUSH rule 1 x 913 [331,463]
+ CRUSH rule 1 x 914 [836,468]
+ CRUSH rule 1 x 915 [245,228]
+ CRUSH rule 1 x 916 [77,967]
+ CRUSH rule 1 x 917 [239,60]
+ CRUSH rule 1 x 918 [988,115]
+ CRUSH rule 1 x 919 [783,139]
+ CRUSH rule 1 x 920 [623,408]
+ CRUSH rule 1 x 921 [105,799]
+ CRUSH rule 1 x 922 [887,505]
+ CRUSH rule 1 x 923 [223,318]
+ CRUSH rule 1 x 924 [25,778]
+ CRUSH rule 1 x 925 [912,601]
+ CRUSH rule 1 x 926 [968,133]
+ CRUSH rule 1 x 927 [277,724]
+ CRUSH rule 1 x 928 [554,203]
+ CRUSH rule 1 x 929 [761,802]
+ CRUSH rule 1 x 930 [814,61]
+ CRUSH rule 1 x 931 [29,193]
+ CRUSH rule 1 x 932 [446,198]
+ CRUSH rule 1 x 933 [352,742]
+ CRUSH rule 1 x 934 [730,2]
+ CRUSH rule 1 x 935 [731,23]
+ CRUSH rule 1 x 936 [322,975]
+ CRUSH rule 1 x 937 [822,221]
+ CRUSH rule 1 x 938 [557,850]
+ CRUSH rule 1 x 939 [150,11]
+ CRUSH rule 1 x 940 [638,398]
+ CRUSH rule 1 x 941 [730,342]
+ CRUSH rule 1 x 942 [62,292]
+ CRUSH rule 1 x 943 [165,314]
+ CRUSH rule 1 x 944 [199,625]
+ CRUSH rule 1 x 945 [946,999]
+ CRUSH rule 1 x 946 [595,93]
+ CRUSH rule 1 x 947 [800,582]
+ CRUSH rule 1 x 948 [132,551]
+ CRUSH rule 1 x 949 [792,920]
+ CRUSH rule 1 x 950 [111,345]
+ CRUSH rule 1 x 951 [414,619]
+ CRUSH rule 1 x 952 [775,469]
+ CRUSH rule 1 x 953 [349,1]
+ CRUSH rule 1 x 954 [570,940]
+ CRUSH rule 1 x 955 [729,774]
+ CRUSH rule 1 x 956 [519,141]
+ CRUSH rule 1 x 957 [242,709]
+ CRUSH rule 1 x 958 [84,217]
+ CRUSH rule 1 x 959 [270,413]
+ CRUSH rule 1 x 960 [458,192]
+ CRUSH rule 1 x 961 [981,388]
+ CRUSH rule 1 x 962 [623,834]
+ CRUSH rule 1 x 963 [291,167]
+ CRUSH rule 1 x 964 [28,156]
+ CRUSH rule 1 x 965 [675,557]
+ CRUSH rule 1 x 966 [836,306]
+ CRUSH rule 1 x 967 [966,386]
+ CRUSH rule 1 x 968 [864,756]
+ CRUSH rule 1 x 969 [729,625]
+ CRUSH rule 1 x 970 [800,362]
+ CRUSH rule 1 x 971 [737,381]
+ CRUSH rule 1 x 972 [952,245]
+ CRUSH rule 1 x 973 [356,455]
+ CRUSH rule 1 x 974 [545,758]
+ CRUSH rule 1 x 975 [336,191]
+ CRUSH rule 1 x 976 [446,208]
+ CRUSH rule 1 x 977 [202,896]
+ CRUSH rule 1 x 978 [612,324]
+ CRUSH rule 1 x 979 [843,457]
+ CRUSH rule 1 x 980 [60,914]
+ CRUSH rule 1 x 981 [702,749]
+ CRUSH rule 1 x 982 [298,928]
+ CRUSH rule 1 x 983 [723,572]
+ CRUSH rule 1 x 984 [723,864]
+ CRUSH rule 1 x 985 [945,459]
+ CRUSH rule 1 x 986 [772,664]
+ CRUSH rule 1 x 987 [88,324]
+ CRUSH rule 1 x 988 [522,927]
+ CRUSH rule 1 x 989 [578,332]
+ CRUSH rule 1 x 990 [638,228]
+ CRUSH rule 1 x 991 [530,221]
+ CRUSH rule 1 x 992 [925,705]
+ CRUSH rule 1 x 993 [991,301]
+ CRUSH rule 1 x 994 [276,51]
+ CRUSH rule 1 x 995 [288,836]
+ CRUSH rule 1 x 996 [887,983]
+ CRUSH rule 1 x 997 [110,924]
+ CRUSH rule 1 x 998 [435,830]
+ CRUSH rule 1 x 999 [876,738]
+ CRUSH rule 1 x 1000 [178,963]
+ CRUSH rule 1 x 1001 [99,519]
+ CRUSH rule 1 x 1002 [515,534]
+ CRUSH rule 1 x 1003 [104,611]
+ CRUSH rule 1 x 1004 [269,638]
+ CRUSH rule 1 x 1005 [369,223]
+ CRUSH rule 1 x 1006 [40,107]
+ CRUSH rule 1 x 1007 [978,111]
+ CRUSH rule 1 x 1008 [965,956]
+ CRUSH rule 1 x 1009 [598,476]
+ CRUSH rule 1 x 1010 [767,523]
+ CRUSH rule 1 x 1011 [289,871]
+ CRUSH rule 1 x 1012 [128,28]
+ CRUSH rule 1 x 1013 [979,765]
+ CRUSH rule 1 x 1014 [979,948]
+ CRUSH rule 1 x 1015 [277,790]
+ CRUSH rule 1 x 1016 [262,73]
+ CRUSH rule 1 x 1017 [150,269]
+ CRUSH rule 1 x 1018 [555,829]
+ CRUSH rule 1 x 1019 [513,356]
+ CRUSH rule 1 x 1020 [158,161]
+ CRUSH rule 1 x 1021 [915,998]
+ CRUSH rule 1 x 1022 [967,829]
+ CRUSH rule 1 x 1023 [488,257]
+ rule 1 (metadata) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536]
+ CRUSH rule 1 x 1 [876,250,334]
+ CRUSH rule 1 x 2 [292,832,53]
+ CRUSH rule 1 x 3 [623,387,124]
+ CRUSH rule 1 x 4 [61,334,710]
+ CRUSH rule 1 x 5 [946,557,713]
+ CRUSH rule 1 x 6 [576,668,212]
+ CRUSH rule 1 x 7 [645,753,906]
+ CRUSH rule 1 x 8 [243,6,863]
+ CRUSH rule 1 x 9 [22,578,251]
+ CRUSH rule 1 x 10 [758,828,360]
+ CRUSH rule 1 x 11 [769,120,124]
+ CRUSH rule 1 x 12 [780,364,689]
+ CRUSH rule 1 x 13 [557,18,351]
+ CRUSH rule 1 x 14 [59,561,249]
+ CRUSH rule 1 x 15 [718,928,993]
+ CRUSH rule 1 x 16 [673,632,841]
+ CRUSH rule 1 x 17 [648,43,560]
+ CRUSH rule 1 x 18 [654,219,181]
+ CRUSH rule 1 x 19 [850,545,377]
+ CRUSH rule 1 x 20 [717,785,974]
+ CRUSH rule 1 x 21 [420,57,519]
+ CRUSH rule 1 x 22 [503,998,193]
+ CRUSH rule 1 x 23 [411,663,168]
+ CRUSH rule 1 x 24 [266,861,353]
+ CRUSH rule 1 x 25 [760,483,818]
+ CRUSH rule 1 x 26 [903,24,573]
+ CRUSH rule 1 x 27 [946,188,289]
+ CRUSH rule 1 x 28 [69,312,73]
+ CRUSH rule 1 x 29 [844,883,337]
+ CRUSH rule 1 x 30 [621,18,613]
+ CRUSH rule 1 x 31 [784,943,814]
+ CRUSH rule 1 x 32 [173,374,369]
+ CRUSH rule 1 x 33 [698,336,357]
+ CRUSH rule 1 x 34 [168,836,210]
+ CRUSH rule 1 x 35 [274,509,534]
+ CRUSH rule 1 x 36 [318,215,153]
+ CRUSH rule 1 x 37 [173,604,109]
+ CRUSH rule 1 x 38 [708,444,683]
+ CRUSH rule 1 x 39 [662,198,417]
+ CRUSH rule 1 x 40 [620,801,414]
+ CRUSH rule 1 x 41 [811,264,177]
+ CRUSH rule 1 x 42 [863,179,527]
+ CRUSH rule 1 x 43 [686,822,988]
+ CRUSH rule 1 x 44 [396,222,46]
+ CRUSH rule 1 x 45 [991,694,253]
+ CRUSH rule 1 x 46 [420,909,184]
+ CRUSH rule 1 x 47 [467,211,605]
+ CRUSH rule 1 x 48 [955,329,368]
+ CRUSH rule 1 x 49 [974,891,931]
+ CRUSH rule 1 x 50 [870,441,691]
+ CRUSH rule 1 x 51 [182,930,25]
+ CRUSH rule 1 x 52 [704,812,894]
+ CRUSH rule 1 x 53 [185,713,631]
+ CRUSH rule 1 x 54 [270,441,100]
+ CRUSH rule 1 x 55 [895,734,958]
+ CRUSH rule 1 x 56 [564,963,683]
+ CRUSH rule 1 x 57 [738,130,208]
+ CRUSH rule 1 x 58 [524,113,806]
+ CRUSH rule 1 x 59 [408,337,668]
+ CRUSH rule 1 x 60 [228,790,857]
+ CRUSH rule 1 x 61 [154,843,717]
+ CRUSH rule 1 x 62 [594,811,549]
+ CRUSH rule 1 x 63 [646,67,884]
+ CRUSH rule 1 x 64 [175,542,155]
+ CRUSH rule 1 x 65 [745,619,131]
+ CRUSH rule 1 x 66 [275,468,23]
+ CRUSH rule 1 x 67 [246,958,524]
+ CRUSH rule 1 x 68 [711,473,403]
+ CRUSH rule 1 x 69 [493,924,850]
+ CRUSH rule 1 x 70 [30,499,644]
+ CRUSH rule 1 x 71 [984,883,574]
+ CRUSH rule 1 x 72 [71,286,942]
+ CRUSH rule 1 x 73 [922,618,3]
+ CRUSH rule 1 x 74 [629,414,185]
+ CRUSH rule 1 x 75 [222,20,174]
+ CRUSH rule 1 x 76 [262,366,339]
+ CRUSH rule 1 x 77 [638,469,992]
+ CRUSH rule 1 x 78 [324,511,788]
+ CRUSH rule 1 x 79 [577,990,64]
+ CRUSH rule 1 x 80 [501,95,278]
+ CRUSH rule 1 x 81 [506,812,9]
+ CRUSH rule 1 x 82 [222,145,80]
+ CRUSH rule 1 x 83 [71,634,61]
+ CRUSH rule 1 x 84 [49,761,773]
+ CRUSH rule 1 x 85 [985,896,708]
+ CRUSH rule 1 x 86 [537,745,93]
+ CRUSH rule 1 x 87 [997,317,463]
+ CRUSH rule 1 x 88 [957,350,890]
+ CRUSH rule 1 x 89 [399,730,148]
+ CRUSH rule 1 x 90 [943,706,683]
+ CRUSH rule 1 x 91 [22,368,149]
+ CRUSH rule 1 x 92 [532,424,426]
+ CRUSH rule 1 x 93 [218,489,405]
+ CRUSH rule 1 x 94 [181,96,102]
+ CRUSH rule 1 x 95 [343,957,820]
+ CRUSH rule 1 x 96 [861,270,87]
+ CRUSH rule 1 x 97 [459,706,45]
+ CRUSH rule 1 x 98 [327,867,353]
+ CRUSH rule 1 x 99 [974,133,468]
+ CRUSH rule 1 x 100 [32,445,547]
+ CRUSH rule 1 x 101 [142,90,337]
+ CRUSH rule 1 x 102 [172,129,139]
+ CRUSH rule 1 x 103 [630,47,161]
+ CRUSH rule 1 x 104 [758,133,278]
+ CRUSH rule 1 x 105 [843,604,47]
+ CRUSH rule 1 x 106 [28,681,193]
+ CRUSH rule 1 x 107 [74,320,85]
+ CRUSH rule 1 x 108 [875,593,575]
+ CRUSH rule 1 x 109 [411,985,811]
+ CRUSH rule 1 x 110 [440,774,799]
+ CRUSH rule 1 x 111 [405,742,276]
+ CRUSH rule 1 x 112 [143,181,922]
+ CRUSH rule 1 x 113 [153,846,160]
+ CRUSH rule 1 x 114 [804,892,939]
+ CRUSH rule 1 x 115 [588,508,958]
+ CRUSH rule 1 x 116 [327,148,637]
+ CRUSH rule 1 x 117 [95,594,989]
+ CRUSH rule 1 x 118 [80,957,897]
+ CRUSH rule 1 x 119 [386,932,951]
+ CRUSH rule 1 x 120 [366,312,653]
+ CRUSH rule 1 x 121 [129,154,847]
+ CRUSH rule 1 x 122 [873,1,110]
+ CRUSH rule 1 x 123 [533,415,789]
+ CRUSH rule 1 x 124 [461,691,898]
+ CRUSH rule 1 x 125 [342,599,830]
+ CRUSH rule 1 x 126 [819,781,822]
+ CRUSH rule 1 x 127 [437,893,585]
+ CRUSH rule 1 x 128 [679,994,982]
+ CRUSH rule 1 x 129 [380,685,947]
+ CRUSH rule 1 x 130 [992,52,466]
+ CRUSH rule 1 x 131 [469,90,208]
+ CRUSH rule 1 x 132 [571,250,316]
+ CRUSH rule 1 x 133 [964,728,329]
+ CRUSH rule 1 x 134 [999,19,716]
+ CRUSH rule 1 x 135 [634,101,52]
+ CRUSH rule 1 x 136 [114,889,692]
+ CRUSH rule 1 x 137 [839,8,959]
+ CRUSH rule 1 x 138 [967,949,138]
+ CRUSH rule 1 x 139 [308,711,736]
+ CRUSH rule 1 x 140 [764,936,926]
+ CRUSH rule 1 x 141 [423,302,112]
+ CRUSH rule 1 x 142 [252,821,715]
+ CRUSH rule 1 x 143 [33,808,518]
+ CRUSH rule 1 x 144 [472,88,969]
+ CRUSH rule 1 x 145 [242,208,252]
+ CRUSH rule 1 x 146 [290,70,570]
+ CRUSH rule 1 x 147 [447,352,657]
+ CRUSH rule 1 x 148 [212,644,432]
+ CRUSH rule 1 x 149 [9,775,87]
+ CRUSH rule 1 x 150 [166,456,582]
+ CRUSH rule 1 x 151 [811,875,307]
+ CRUSH rule 1 x 152 [449,617,223]
+ CRUSH rule 1 x 153 [523,537,695]
+ CRUSH rule 1 x 154 [208,559,874]
+ CRUSH rule 1 x 155 [569,325,192]
+ CRUSH rule 1 x 156 [488,121,521]
+ CRUSH rule 1 x 157 [140,723,633]
+ CRUSH rule 1 x 158 [786,451,320]
+ CRUSH rule 1 x 159 [134,664,517]
+ CRUSH rule 1 x 160 [690,112,414]
+ CRUSH rule 1 x 161 [324,912,397]
+ CRUSH rule 1 x 162 [748,567,284]
+ CRUSH rule 1 x 163 [575,499,31]
+ CRUSH rule 1 x 164 [314,489,308]
+ CRUSH rule 1 x 165 [116,209,750]
+ CRUSH rule 1 x 166 [352,706,701]
+ CRUSH rule 1 x 167 [27,743,174]
+ CRUSH rule 1 x 168 [953,898,880]
+ CRUSH rule 1 x 169 [912,147,266]
+ CRUSH rule 1 x 170 [421,515,828]
+ CRUSH rule 1 x 171 [488,584,880]
+ CRUSH rule 1 x 172 [366,443,957]
+ CRUSH rule 1 x 173 [863,291,625]
+ CRUSH rule 1 x 174 [263,555,650]
+ CRUSH rule 1 x 175 [875,961,361]
+ CRUSH rule 1 x 176 [745,83,701]
+ CRUSH rule 1 x 177 [128,244,41]
+ CRUSH rule 1 x 178 [155,41,264]
+ CRUSH rule 1 x 179 [593,833,202]
+ CRUSH rule 1 x 180 [154,734,17]
+ CRUSH rule 1 x 181 [289,675,723]
+ CRUSH rule 1 x 182 [730,931,560]
+ CRUSH rule 1 x 183 [639,237,794]
+ CRUSH rule 1 x 184 [704,312,685]
+ CRUSH rule 1 x 185 [97,100,762]
+ CRUSH rule 1 x 186 [26,665,554]
+ CRUSH rule 1 x 187 [649,14,740]
+ CRUSH rule 1 x 188 [682,695,590]
+ CRUSH rule 1 x 189 [325,693,726]
+ CRUSH rule 1 x 190 [399,933,136]
+ CRUSH rule 1 x 191 [629,533,17]
+ CRUSH rule 1 x 192 [503,578,38]
+ CRUSH rule 1 x 193 [546,333,651]
+ CRUSH rule 1 x 194 [242,473,58]
+ CRUSH rule 1 x 195 [625,719,135]
+ CRUSH rule 1 x 196 [357,114,125]
+ CRUSH rule 1 x 197 [306,954,453]
+ CRUSH rule 1 x 198 [863,791,311]
+ CRUSH rule 1 x 199 [935,906,929]
+ CRUSH rule 1 x 200 [373,774,229]
+ CRUSH rule 1 x 201 [659,320,477]
+ CRUSH rule 1 x 202 [260,433,524]
+ CRUSH rule 1 x 203 [36,239,675]
+ CRUSH rule 1 x 204 [92,516,993]
+ CRUSH rule 1 x 205 [68,395,473]
+ CRUSH rule 1 x 206 [570,530,642]
+ CRUSH rule 1 x 207 [834,457,850]
+ CRUSH rule 1 x 208 [927,484,640]
+ CRUSH rule 1 x 209 [878,66,58]
+ CRUSH rule 1 x 210 [572,981,484]
+ CRUSH rule 1 x 211 [107,597,780]
+ CRUSH rule 1 x 212 [389,107,838]
+ CRUSH rule 1 x 213 [497,717,567]
+ CRUSH rule 1 x 214 [798,65,254]
+ CRUSH rule 1 x 215 [233,419,283]
+ CRUSH rule 1 x 216 [494,464,742]
+ CRUSH rule 1 x 217 [352,396,309]
+ CRUSH rule 1 x 218 [895,864,988]
+ CRUSH rule 1 x 219 [222,534,277]
+ CRUSH rule 1 x 220 [281,19,584]
+ CRUSH rule 1 x 221 [64,928,963]
+ CRUSH rule 1 x 222 [40,544,161]
+ CRUSH rule 1 x 223 [645,556,159]
+ CRUSH rule 1 x 224 [647,165,957]
+ CRUSH rule 1 x 225 [219,714,858]
+ CRUSH rule 1 x 226 [372,511,181]
+ CRUSH rule 1 x 227 [925,156,714]
+ CRUSH rule 1 x 228 [682,404,839]
+ CRUSH rule 1 x 229 [880,838,770]
+ CRUSH rule 1 x 230 [328,659,916]
+ CRUSH rule 1 x 231 [320,383,669]
+ CRUSH rule 1 x 232 [924,846,394]
+ CRUSH rule 1 x 233 [948,652,575]
+ CRUSH rule 1 x 234 [484,943,42]
+ CRUSH rule 1 x 235 [750,65,590]
+ CRUSH rule 1 x 236 [551,787,490]
+ CRUSH rule 1 x 237 [390,157,166]
+ CRUSH rule 1 x 238 [570,6,989]
+ CRUSH rule 1 x 239 [729,959,376]
+ CRUSH rule 1 x 240 [981,241,156]
+ CRUSH rule 1 x 241 [310,816,641]
+ CRUSH rule 1 x 242 [161,63,642]
+ CRUSH rule 1 x 243 [180,394,33]
+ CRUSH rule 1 x 244 [52,174,685]
+ CRUSH rule 1 x 245 [523,121,915]
+ CRUSH rule 1 x 246 [362,893,390]
+ CRUSH rule 1 x 247 [382,184,116]
+ CRUSH rule 1 x 248 [129,114,852]
+ CRUSH rule 1 x 249 [159,683,91]
+ CRUSH rule 1 x 250 [404,945,569]
+ CRUSH rule 1 x 251 [661,225,738]
+ CRUSH rule 1 x 252 [961,226,542]
+ CRUSH rule 1 x 253 [651,97,225]
+ CRUSH rule 1 x 254 [123,33,741]
+ CRUSH rule 1 x 255 [314,649,891]
+ CRUSH rule 1 x 256 [315,215,651]
+ CRUSH rule 1 x 257 [825,264,867]
+ CRUSH rule 1 x 258 [624,789,370]
+ CRUSH rule 1 x 259 [602,542,70]
+ CRUSH rule 1 x 260 [717,878,43]
+ CRUSH rule 1 x 261 [145,517,20]
+ CRUSH rule 1 x 262 [223,1,561]
+ CRUSH rule 1 x 263 [462,211,405]
+ CRUSH rule 1 x 264 [654,471,266]
+ CRUSH rule 1 x 265 [302,794,704]
+ CRUSH rule 1 x 266 [202,132,884]
+ CRUSH rule 1 x 267 [282,938,657]
+ CRUSH rule 1 x 268 [338,309,356]
+ CRUSH rule 1 x 269 [738,122,266]
+ CRUSH rule 1 x 270 [707,982,946]
+ CRUSH rule 1 x 271 [705,432,364]
+ CRUSH rule 1 x 272 [756,545,942]
+ CRUSH rule 1 x 273 [197,502,527]
+ CRUSH rule 1 x 274 [992,44,653]
+ CRUSH rule 1 x 275 [544,789,170]
+ CRUSH rule 1 x 276 [658,467,577]
+ CRUSH rule 1 x 277 [143,490,880]
+ CRUSH rule 1 x 278 [492,647,355]
+ CRUSH rule 1 x 279 [517,792,604]
+ CRUSH rule 1 x 280 [825,740,27]
+ CRUSH rule 1 x 281 [224,629,120]
+ CRUSH rule 1 x 282 [298,661,380]
+ CRUSH rule 1 x 283 [311,606,208]
+ CRUSH rule 1 x 284 [771,466,371]
+ CRUSH rule 1 x 285 [693,362,404]
+ CRUSH rule 1 x 286 [364,477,285]
+ CRUSH rule 1 x 287 [591,611,828]
+ CRUSH rule 1 x 288 [965,541,848]
+ CRUSH rule 1 x 289 [225,551,948]
+ CRUSH rule 1 x 290 [577,762,777]
+ CRUSH rule 1 x 291 [160,903,477]
+ CRUSH rule 1 x 292 [873,598,216]
+ CRUSH rule 1 x 293 [100,234,874]
+ CRUSH rule 1 x 294 [285,943,379]
+ CRUSH rule 1 x 295 [938,262,880]
+ CRUSH rule 1 x 296 [850,327,86]
+ CRUSH rule 1 x 297 [951,53,99]
+ CRUSH rule 1 x 298 [173,336,85]
+ CRUSH rule 1 x 299 [598,591,315]
+ CRUSH rule 1 x 300 [531,957,62]
+ CRUSH rule 1 x 301 [823,628,23]
+ CRUSH rule 1 x 302 [184,80,780]
+ CRUSH rule 1 x 303 [521,766,222]
+ CRUSH rule 1 x 304 [980,127,807]
+ CRUSH rule 1 x 305 [153,816,22]
+ CRUSH rule 1 x 306 [423,739,664]
+ CRUSH rule 1 x 307 [997,557,682]
+ CRUSH rule 1 x 308 [991,874,534]
+ CRUSH rule 1 x 309 [860,394,724]
+ CRUSH rule 1 x 310 [589,818,546]
+ CRUSH rule 1 x 311 [477,774,225]
+ CRUSH rule 1 x 312 [887,853,950]
+ CRUSH rule 1 x 313 [802,646,447]
+ CRUSH rule 1 x 314 [654,974,229]
+ CRUSH rule 1 x 315 [767,227,28]
+ CRUSH rule 1 x 316 [778,83,733]
+ CRUSH rule 1 x 317 [184,418,642]
+ CRUSH rule 1 x 318 [525,410,500]
+ CRUSH rule 1 x 319 [476,724,569]
+ CRUSH rule 1 x 320 [149,610,697]
+ CRUSH rule 1 x 321 [710,79,667]
+ CRUSH rule 1 x 322 [175,275,323]
+ CRUSH rule 1 x 323 [819,604,638]
+ CRUSH rule 1 x 324 [16,745,511]
+ CRUSH rule 1 x 325 [486,400,872]
+ CRUSH rule 1 x 326 [613,765,207]
+ CRUSH rule 1 x 327 [125,289,738]
+ CRUSH rule 1 x 328 [807,383,476]
+ CRUSH rule 1 x 329 [588,938,599]
+ CRUSH rule 1 x 330 [932,644,41]
+ CRUSH rule 1 x 331 [341,953,950]
+ CRUSH rule 1 x 332 [153,726,459]
+ CRUSH rule 1 x 333 [745,845,853]
+ CRUSH rule 1 x 334 [614,751,807]
+ CRUSH rule 1 x 335 [518,721,221]
+ CRUSH rule 1 x 336 [389,424,77]
+ CRUSH rule 1 x 337 [753,508,765]
+ CRUSH rule 1 x 338 [128,810,490]
+ CRUSH rule 1 x 339 [430,308,58]
+ CRUSH rule 1 x 340 [541,44,630]
+ CRUSH rule 1 x 341 [402,26,631]
+ CRUSH rule 1 x 342 [982,57,992]
+ CRUSH rule 1 x 343 [833,412,572]
+ CRUSH rule 1 x 344 [784,533,792]
+ CRUSH rule 1 x 345 [546,300,304]
+ CRUSH rule 1 x 346 [302,420,428]
+ CRUSH rule 1 x 347 [488,778,101]
+ CRUSH rule 1 x 348 [903,744,937]
+ CRUSH rule 1 x 349 [471,547,582]
+ CRUSH rule 1 x 350 [348,221,823]
+ CRUSH rule 1 x 351 [961,582,705]
+ CRUSH rule 1 x 352 [728,137,461]
+ CRUSH rule 1 x 353 [904,202,184]
+ CRUSH rule 1 x 354 [345,226,319]
+ CRUSH rule 1 x 355 [50,430,175]
+ CRUSH rule 1 x 356 [87,185,55]
+ CRUSH rule 1 x 357 [762,459,921]
+ CRUSH rule 1 x 358 [908,25,280]
+ CRUSH rule 1 x 359 [484,15,132]
+ CRUSH rule 1 x 360 [173,378,337]
+ CRUSH rule 1 x 361 [404,577,115]
+ CRUSH rule 1 x 362 [403,1,422]
+ CRUSH rule 1 x 363 [639,911,510]
+ CRUSH rule 1 x 364 [752,689,610]
+ CRUSH rule 1 x 365 [956,999,212]
+ CRUSH rule 1 x 366 [860,925,924]
+ CRUSH rule 1 x 367 [205,609,647]
+ CRUSH rule 1 x 368 [301,284,810]
+ CRUSH rule 1 x 369 [452,658,339]
+ CRUSH rule 1 x 370 [11,467,695]
+ CRUSH rule 1 x 371 [124,487,55]
+ CRUSH rule 1 x 372 [253,48,979]
+ CRUSH rule 1 x 373 [715,605,775]
+ CRUSH rule 1 x 374 [191,887,920]
+ CRUSH rule 1 x 375 [711,385,651]
+ CRUSH rule 1 x 376 [597,818,49]
+ CRUSH rule 1 x 377 [294,256,933]
+ CRUSH rule 1 x 378 [34,151,681]
+ CRUSH rule 1 x 379 [869,136,315]
+ CRUSH rule 1 x 380 [294,97,575]
+ CRUSH rule 1 x 381 [119,710,219]
+ CRUSH rule 1 x 382 [69,631,508]
+ CRUSH rule 1 x 383 [922,588,589]
+ CRUSH rule 1 x 384 [221,945,671]
+ CRUSH rule 1 x 385 [561,737,953]
+ CRUSH rule 1 x 386 [335,442,788]
+ CRUSH rule 1 x 387 [514,43,353]
+ CRUSH rule 1 x 388 [587,89,157]
+ CRUSH rule 1 x 389 [109,641,255]
+ CRUSH rule 1 x 390 [925,149,421]
+ CRUSH rule 1 x 391 [267,87,387]
+ CRUSH rule 1 x 392 [382,485,370]
+ CRUSH rule 1 x 393 [425,721,221]
+ CRUSH rule 1 x 394 [898,18,38]
+ CRUSH rule 1 x 395 [806,876,269]
+ CRUSH rule 1 x 396 [790,970,437]
+ CRUSH rule 1 x 397 [136,363,507]
+ CRUSH rule 1 x 398 [914,116,558]
+ CRUSH rule 1 x 399 [261,94,299]
+ CRUSH rule 1 x 400 [661,197,338]
+ CRUSH rule 1 x 401 [953,979,287]
+ CRUSH rule 1 x 402 [738,819,618]
+ CRUSH rule 1 x 403 [573,238,425]
+ CRUSH rule 1 x 404 [526,848,790]
+ CRUSH rule 1 x 405 [582,505,330]
+ CRUSH rule 1 x 406 [768,324,493]
+ CRUSH rule 1 x 407 [260,951,437]
+ CRUSH rule 1 x 408 [657,81,770]
+ CRUSH rule 1 x 409 [498,89,182]
+ CRUSH rule 1 x 410 [28,793,737]
+ CRUSH rule 1 x 411 [684,992,60]
+ CRUSH rule 1 x 412 [261,958,699]
+ CRUSH rule 1 x 413 [891,835,297]
+ CRUSH rule 1 x 414 [127,459,119]
+ CRUSH rule 1 x 415 [272,540,631]
+ CRUSH rule 1 x 416 [739,617,115]
+ CRUSH rule 1 x 417 [106,209,157]
+ CRUSH rule 1 x 418 [525,441,147]
+ CRUSH rule 1 x 419 [603,673,615]
+ CRUSH rule 1 x 420 [988,213,251]
+ CRUSH rule 1 x 421 [761,521,748]
+ CRUSH rule 1 x 422 [317,160,924]
+ CRUSH rule 1 x 423 [137,807,168]
+ CRUSH rule 1 x 424 [920,37,146]
+ CRUSH rule 1 x 425 [277,693,285]
+ CRUSH rule 1 x 426 [485,936,407]
+ CRUSH rule 1 x 427 [242,515,9]
+ CRUSH rule 1 x 428 [632,635,26]
+ CRUSH rule 1 x 429 [641,73,465]
+ CRUSH rule 1 x 430 [626,585,6]
+ CRUSH rule 1 x 431 [697,76,753]
+ CRUSH rule 1 x 432 [590,526,306]
+ CRUSH rule 1 x 433 [284,387,149]
+ CRUSH rule 1 x 434 [538,985,79]
+ CRUSH rule 1 x 435 [30,318,593]
+ CRUSH rule 1 x 436 [164,919,851]
+ CRUSH rule 1 x 437 [322,212,163]
+ CRUSH rule 1 x 438 [142,392,85]
+ CRUSH rule 1 x 439 [119,370,68]
+ CRUSH rule 1 x 440 [333,403,187]
+ CRUSH rule 1 x 441 [477,727,906]
+ CRUSH rule 1 x 442 [274,590,933]
+ CRUSH rule 1 x 443 [983,748,574]
+ CRUSH rule 1 x 444 [536,509,431]
+ CRUSH rule 1 x 445 [485,964,528]
+ CRUSH rule 1 x 446 [345,634,42]
+ CRUSH rule 1 x 447 [61,845,767]
+ CRUSH rule 1 x 448 [333,232,292]
+ CRUSH rule 1 x 449 [680,16,484]
+ CRUSH rule 1 x 450 [235,214,79]
+ CRUSH rule 1 x 451 [961,468,333]
+ CRUSH rule 1 x 452 [525,479,153]
+ CRUSH rule 1 x 453 [138,466,302]
+ CRUSH rule 1 x 454 [137,625,215]
+ CRUSH rule 1 x 455 [173,150,997]
+ CRUSH rule 1 x 456 [235,226,238]
+ CRUSH rule 1 x 457 [450,577,253]
+ CRUSH rule 1 x 458 [195,537,91]
+ CRUSH rule 1 x 459 [381,555,312]
+ CRUSH rule 1 x 460 [972,730,534]
+ CRUSH rule 1 x 461 [506,279,142]
+ CRUSH rule 1 x 462 [692,959,578]
+ CRUSH rule 1 x 463 [788,667,949]
+ CRUSH rule 1 x 464 [133,122,588]
+ CRUSH rule 1 x 465 [971,190,230]
+ CRUSH rule 1 x 466 [394,576,148]
+ CRUSH rule 1 x 467 [517,28,366]
+ CRUSH rule 1 x 468 [829,143,874]
+ CRUSH rule 1 x 469 [987,936,106]
+ CRUSH rule 1 x 470 [107,982,56]
+ CRUSH rule 1 x 471 [181,897,629]
+ CRUSH rule 1 x 472 [547,512,172]
+ CRUSH rule 1 x 473 [760,997,824]
+ CRUSH rule 1 x 474 [787,418,743]
+ CRUSH rule 1 x 475 [662,312,253]
+ CRUSH rule 1 x 476 [110,495,185]
+ CRUSH rule 1 x 477 [393,954,834]
+ CRUSH rule 1 x 478 [246,483,480]
+ CRUSH rule 1 x 479 [70,929,697]
+ CRUSH rule 1 x 480 [753,119,961]
+ CRUSH rule 1 x 481 [470,429,677]
+ CRUSH rule 1 x 482 [451,566,961]
+ CRUSH rule 1 x 483 [816,72,371]
+ CRUSH rule 1 x 484 [540,454,389]
+ CRUSH rule 1 x 485 [74,582,624]
+ CRUSH rule 1 x 486 [958,595,199]
+ CRUSH rule 1 x 487 [228,302,804]
+ CRUSH rule 1 x 488 [180,529,722]
+ CRUSH rule 1 x 489 [47,617,812]
+ CRUSH rule 1 x 490 [905,822,479]
+ CRUSH rule 1 x 491 [892,370,609]
+ CRUSH rule 1 x 492 [588,959,127]
+ CRUSH rule 1 x 493 [353,461,593]
+ CRUSH rule 1 x 494 [378,848,443]
+ CRUSH rule 1 x 495 [845,653,768]
+ CRUSH rule 1 x 496 [13,988,0]
+ CRUSH rule 1 x 497 [796,877,788]
+ CRUSH rule 1 x 498 [412,337,270]
+ CRUSH rule 1 x 499 [330,695,8]
+ CRUSH rule 1 x 500 [820,272,547]
+ CRUSH rule 1 x 501 [110,44,132]
+ CRUSH rule 1 x 502 [336,595,650]
+ CRUSH rule 1 x 503 [922,211,157]
+ CRUSH rule 1 x 504 [483,52,122]
+ CRUSH rule 1 x 505 [482,598,224]
+ CRUSH rule 1 x 506 [493,123,43]
+ CRUSH rule 1 x 507 [12,598,264]
+ CRUSH rule 1 x 508 [227,157,611]
+ CRUSH rule 1 x 509 [807,242,363]
+ CRUSH rule 1 x 510 [134,437,227]
+ CRUSH rule 1 x 511 [212,54,83]
+ CRUSH rule 1 x 512 [236,630,758]
+ CRUSH rule 1 x 513 [994,693,644]
+ CRUSH rule 1 x 514 [45,508,831]
+ CRUSH rule 1 x 515 [504,138,480]
+ CRUSH rule 1 x 516 [285,409,136]
+ CRUSH rule 1 x 517 [300,232,23]
+ CRUSH rule 1 x 518 [397,674,98]
+ CRUSH rule 1 x 519 [86,750,772]
+ CRUSH rule 1 x 520 [900,833,614]
+ CRUSH rule 1 x 521 [31,47,236]
+ CRUSH rule 1 x 522 [390,16,280]
+ CRUSH rule 1 x 523 [618,308,424]
+ CRUSH rule 1 x 524 [635,189,687]
+ CRUSH rule 1 x 525 [311,916,699]
+ CRUSH rule 1 x 526 [48,738,227]
+ CRUSH rule 1 x 527 [202,851,889]
+ CRUSH rule 1 x 528 [565,827,590]
+ CRUSH rule 1 x 529 [934,864,241]
+ CRUSH rule 1 x 530 [502,934,298]
+ CRUSH rule 1 x 531 [681,627,942]
+ CRUSH rule 1 x 532 [422,6,147]
+ CRUSH rule 1 x 533 [863,68,364]
+ CRUSH rule 1 x 534 [962,931,775]
+ CRUSH rule 1 x 535 [89,565,397]
+ CRUSH rule 1 x 536 [499,351,760]
+ CRUSH rule 1 x 537 [676,547,787]
+ CRUSH rule 1 x 538 [58,644,571]
+ CRUSH rule 1 x 539 [837,953,457]
+ CRUSH rule 1 x 540 [831,50,132]
+ CRUSH rule 1 x 541 [582,757,121]
+ CRUSH rule 1 x 542 [472,132,790]
+ CRUSH rule 1 x 543 [382,272,797]
+ CRUSH rule 1 x 544 [947,930,496]
+ CRUSH rule 1 x 545 [425,570,305]
+ CRUSH rule 1 x 546 [18,65,529]
+ CRUSH rule 1 x 547 [445,715,600]
+ CRUSH rule 1 x 548 [367,569,980]
+ CRUSH rule 1 x 549 [125,715,671]
+ CRUSH rule 1 x 550 [425,599,744]
+ CRUSH rule 1 x 551 [44,1,528]
+ CRUSH rule 1 x 552 [246,104,68]
+ CRUSH rule 1 x 553 [71,703,615]
+ CRUSH rule 1 x 554 [207,124,217]
+ CRUSH rule 1 x 555 [570,28,317]
+ CRUSH rule 1 x 556 [674,152,421]
+ CRUSH rule 1 x 557 [347,817,191]
+ CRUSH rule 1 x 558 [627,426,369]
+ CRUSH rule 1 x 559 [940,630,924]
+ CRUSH rule 1 x 560 [295,903,541]
+ CRUSH rule 1 x 561 [506,682,384]
+ CRUSH rule 1 x 562 [718,529,87]
+ CRUSH rule 1 x 563 [552,332,747]
+ CRUSH rule 1 x 564 [835,769,736]
+ CRUSH rule 1 x 565 [8,167,539]
+ CRUSH rule 1 x 566 [600,481,301]
+ CRUSH rule 1 x 567 [999,994,509]
+ CRUSH rule 1 x 568 [252,431,157]
+ CRUSH rule 1 x 569 [643,218,943]
+ CRUSH rule 1 x 570 [617,635,765]
+ CRUSH rule 1 x 571 [757,80,59]
+ CRUSH rule 1 x 572 [299,348,575]
+ CRUSH rule 1 x 573 [25,505,270]
+ CRUSH rule 1 x 574 [215,431,624]
+ CRUSH rule 1 x 575 [225,252,611]
+ CRUSH rule 1 x 576 [627,94,159]
+ CRUSH rule 1 x 577 [237,809,778]
+ CRUSH rule 1 x 578 [885,313,120]
+ CRUSH rule 1 x 579 [924,575,787]
+ CRUSH rule 1 x 580 [718,51,766]
+ CRUSH rule 1 x 581 [219,807,129]
+ CRUSH rule 1 x 582 [893,701,598]
+ CRUSH rule 1 x 583 [246,930,964]
+ CRUSH rule 1 x 584 [336,432,680]
+ CRUSH rule 1 x 585 [324,999,397]
+ CRUSH rule 1 x 586 [558,230,976]
+ CRUSH rule 1 x 587 [985,830,597]
+ CRUSH rule 1 x 588 [211,544,57]
+ CRUSH rule 1 x 589 [129,21,112]
+ CRUSH rule 1 x 590 [467,969,652]
+ CRUSH rule 1 x 591 [758,514,316]
+ CRUSH rule 1 x 592 [525,253,190]
+ CRUSH rule 1 x 593 [601,885,339]
+ CRUSH rule 1 x 594 [227,60,450]
+ CRUSH rule 1 x 595 [720,854,496]
+ CRUSH rule 1 x 596 [751,195,997]
+ CRUSH rule 1 x 597 [129,574,714]
+ CRUSH rule 1 x 598 [679,207,604]
+ CRUSH rule 1 x 599 [668,315,683]
+ CRUSH rule 1 x 600 [143,396,464]
+ CRUSH rule 1 x 601 [326,573,873]
+ CRUSH rule 1 x 602 [860,281,875]
+ CRUSH rule 1 x 603 [709,328,445]
+ CRUSH rule 1 x 604 [571,62,814]
+ CRUSH rule 1 x 605 [252,739,860]
+ CRUSH rule 1 x 606 [339,236,759]
+ CRUSH rule 1 x 607 [590,248,759]
+ CRUSH rule 1 x 608 [145,635,309]
+ CRUSH rule 1 x 609 [973,547,223]
+ CRUSH rule 1 x 610 [435,816,961]
+ CRUSH rule 1 x 611 [559,283,422]
+ CRUSH rule 1 x 612 [273,149,123]
+ CRUSH rule 1 x 613 [828,614,642]
+ CRUSH rule 1 x 614 [478,748,393]
+ CRUSH rule 1 x 615 [392,155,144]
+ CRUSH rule 1 x 616 [778,637,452]
+ CRUSH rule 1 x 617 [622,713,996]
+ CRUSH rule 1 x 618 [149,877,270]
+ CRUSH rule 1 x 619 [604,163,656]
+ CRUSH rule 1 x 620 [181,23,409]
+ CRUSH rule 1 x 621 [735,902,386]
+ CRUSH rule 1 x 622 [661,824,717]
+ CRUSH rule 1 x 623 [142,121,643]
+ CRUSH rule 1 x 624 [360,716,420]
+ CRUSH rule 1 x 625 [541,167,385]
+ CRUSH rule 1 x 626 [364,431,610]
+ CRUSH rule 1 x 627 [458,137,557]
+ CRUSH rule 1 x 628 [250,350,556]
+ CRUSH rule 1 x 629 [928,160,710]
+ CRUSH rule 1 x 630 [243,19,918]
+ CRUSH rule 1 x 631 [438,221,574]
+ CRUSH rule 1 x 632 [797,368,247]
+ CRUSH rule 1 x 633 [993,749,525]
+ CRUSH rule 1 x 634 [239,351,633]
+ CRUSH rule 1 x 635 [640,965,25]
+ CRUSH rule 1 x 636 [173,290,297]
+ CRUSH rule 1 x 637 [0,918,98]
+ CRUSH rule 1 x 638 [702,235,424]
+ CRUSH rule 1 x 639 [475,687,31]
+ CRUSH rule 1 x 640 [31,664,399]
+ CRUSH rule 1 x 641 [296,473,108]
+ CRUSH rule 1 x 642 [894,273,427]
+ CRUSH rule 1 x 643 [117,111,732]
+ CRUSH rule 1 x 644 [438,336,327]
+ CRUSH rule 1 x 645 [982,702,351]
+ CRUSH rule 1 x 646 [334,804,146]
+ CRUSH rule 1 x 647 [933,787,185]
+ CRUSH rule 1 x 648 [22,444,400]
+ CRUSH rule 1 x 649 [503,229,213]
+ CRUSH rule 1 x 650 [328,659,420]
+ CRUSH rule 1 x 651 [3,880,823]
+ CRUSH rule 1 x 652 [495,977,563]
+ CRUSH rule 1 x 653 [185,718,804]
+ CRUSH rule 1 x 654 [130,528,380]
+ CRUSH rule 1 x 655 [560,872,454]
+ CRUSH rule 1 x 656 [219,885,178]
+ CRUSH rule 1 x 657 [233,684,813]
+ CRUSH rule 1 x 658 [778,6,756]
+ CRUSH rule 1 x 659 [240,663,306]
+ CRUSH rule 1 x 660 [244,855,196]
+ CRUSH rule 1 x 661 [184,270,128]
+ CRUSH rule 1 x 662 [65,883,921]
+ CRUSH rule 1 x 663 [323,721,594]
+ CRUSH rule 1 x 664 [865,113,512]
+ CRUSH rule 1 x 665 [420,850,591]
+ CRUSH rule 1 x 666 [319,767,246]
+ CRUSH rule 1 x 667 [875,39,343]
+ CRUSH rule 1 x 668 [331,122,263]
+ CRUSH rule 1 x 669 [915,521,402]
+ CRUSH rule 1 x 670 [845,659,943]
+ CRUSH rule 1 x 671 [108,634,527]
+ CRUSH rule 1 x 672 [578,216,110]
+ CRUSH rule 1 x 673 [442,74,579]
+ CRUSH rule 1 x 674 [588,364,281]
+ CRUSH rule 1 x 675 [489,698,744]
+ CRUSH rule 1 x 676 [928,911,40]
+ CRUSH rule 1 x 677 [399,269,692]
+ CRUSH rule 1 x 678 [546,752,544]
+ CRUSH rule 1 x 679 [988,25,275]
+ CRUSH rule 1 x 680 [335,963,382]
+ CRUSH rule 1 x 681 [690,462,623]
+ CRUSH rule 1 x 682 [196,588,154]
+ CRUSH rule 1 x 683 [627,25,421]
+ CRUSH rule 1 x 684 [38,804,592]
+ CRUSH rule 1 x 685 [841,368,548]
+ CRUSH rule 1 x 686 [336,287,525]
+ CRUSH rule 1 x 687 [20,682,924]
+ CRUSH rule 1 x 688 [463,371,780]
+ CRUSH rule 1 x 689 [569,250,78]
+ CRUSH rule 1 x 690 [551,144,587]
+ CRUSH rule 1 x 691 [766,464,446]
+ CRUSH rule 1 x 692 [739,634,18]
+ CRUSH rule 1 x 693 [339,297,118]
+ CRUSH rule 1 x 694 [405,26,830]
+ CRUSH rule 1 x 695 [622,576,597]
+ CRUSH rule 1 x 696 [558,902,689]
+ CRUSH rule 1 x 697 [818,222,406]
+ CRUSH rule 1 x 698 [178,48,402]
+ CRUSH rule 1 x 699 [450,244,180]
+ CRUSH rule 1 x 700 [502,771,987]
+ CRUSH rule 1 x 701 [4,612,782]
+ CRUSH rule 1 x 702 [177,630,232]
+ CRUSH rule 1 x 703 [354,178,389]
+ CRUSH rule 1 x 704 [646,601,156]
+ CRUSH rule 1 x 705 [921,401,890]
+ CRUSH rule 1 x 706 [652,877,562]
+ CRUSH rule 1 x 707 [345,745,67]
+ CRUSH rule 1 x 708 [333,607,180]
+ CRUSH rule 1 x 709 [45,187,302]
+ CRUSH rule 1 x 710 [94,855,43]
+ CRUSH rule 1 x 711 [227,653,731]
+ CRUSH rule 1 x 712 [398,953,136]
+ CRUSH rule 1 x 713 [116,800,503]
+ CRUSH rule 1 x 714 [111,629,866]
+ CRUSH rule 1 x 715 [531,291,486]
+ CRUSH rule 1 x 716 [169,541,291]
+ CRUSH rule 1 x 717 [417,446,994]
+ CRUSH rule 1 x 718 [992,383,298]
+ CRUSH rule 1 x 719 [936,674,324]
+ CRUSH rule 1 x 720 [370,188,174]
+ CRUSH rule 1 x 721 [320,859,278]
+ CRUSH rule 1 x 722 [7,2,673]
+ CRUSH rule 1 x 723 [270,553,831]
+ CRUSH rule 1 x 724 [666,822,708]
+ CRUSH rule 1 x 725 [794,406,875]
+ CRUSH rule 1 x 726 [420,556,341]
+ CRUSH rule 1 x 727 [561,461,129]
+ CRUSH rule 1 x 728 [951,330,196]
+ CRUSH rule 1 x 729 [656,644,436]
+ CRUSH rule 1 x 730 [3,558,629]
+ CRUSH rule 1 x 731 [852,89,75]
+ CRUSH rule 1 x 732 [983,840,869]
+ CRUSH rule 1 x 733 [285,396,388]
+ CRUSH rule 1 x 734 [125,510,402]
+ CRUSH rule 1 x 735 [417,773,686]
+ CRUSH rule 1 x 736 [749,396,632]
+ CRUSH rule 1 x 737 [644,991,946]
+ CRUSH rule 1 x 738 [449,683,290]
+ CRUSH rule 1 x 739 [341,220,641]
+ CRUSH rule 1 x 740 [874,524,674]
+ CRUSH rule 1 x 741 [189,472,712]
+ CRUSH rule 1 x 742 [912,581,114]
+ CRUSH rule 1 x 743 [654,914,425]
+ CRUSH rule 1 x 744 [725,295,579]
+ CRUSH rule 1 x 745 [787,858,850]
+ CRUSH rule 1 x 746 [757,848,704]
+ CRUSH rule 1 x 747 [700,81,867]
+ CRUSH rule 1 x 748 [557,436,238]
+ CRUSH rule 1 x 749 [772,622,337]
+ CRUSH rule 1 x 750 [946,97,376]
+ CRUSH rule 1 x 751 [996,618,343]
+ CRUSH rule 1 x 752 [746,887,695]
+ CRUSH rule 1 x 753 [741,14,463]
+ CRUSH rule 1 x 754 [648,349,333]
+ CRUSH rule 1 x 755 [157,460,466]
+ CRUSH rule 1 x 756 [416,97,197]
+ CRUSH rule 1 x 757 [599,839,776]
+ CRUSH rule 1 x 758 [994,218,620]
+ CRUSH rule 1 x 759 [959,682,514]
+ CRUSH rule 1 x 760 [518,943,215]
+ CRUSH rule 1 x 761 [285,849,420]
+ CRUSH rule 1 x 762 [591,313,41]
+ CRUSH rule 1 x 763 [908,411,200]
+ CRUSH rule 1 x 764 [787,234,894]
+ CRUSH rule 1 x 765 [327,921,882]
+ CRUSH rule 1 x 766 [84,161,878]
+ CRUSH rule 1 x 767 [370,895,702]
+ CRUSH rule 1 x 768 [826,760,879]
+ CRUSH rule 1 x 769 [67,768,663]
+ CRUSH rule 1 x 770 [593,909,482]
+ CRUSH rule 1 x 771 [309,935,121]
+ CRUSH rule 1 x 772 [12,125,797]
+ CRUSH rule 1 x 773 [253,466,820]
+ CRUSH rule 1 x 774 [164,390,705]
+ CRUSH rule 1 x 775 [703,47,43]
+ CRUSH rule 1 x 776 [728,231,80]
+ CRUSH rule 1 x 777 [981,621,568]
+ CRUSH rule 1 x 778 [411,456,544]
+ CRUSH rule 1 x 779 [346,121,519]
+ CRUSH rule 1 x 780 [476,39,288]
+ CRUSH rule 1 x 781 [10,130,585]
+ CRUSH rule 1 x 782 [462,246,581]
+ CRUSH rule 1 x 783 [580,373,153]
+ CRUSH rule 1 x 784 [413,113,978]
+ CRUSH rule 1 x 785 [341,856,332]
+ CRUSH rule 1 x 786 [411,140,313]
+ CRUSH rule 1 x 787 [605,522,211]
+ CRUSH rule 1 x 788 [226,545,35]
+ CRUSH rule 1 x 789 [545,320,414]
+ CRUSH rule 1 x 790 [414,748,816]
+ CRUSH rule 1 x 791 [660,906,406]
+ CRUSH rule 1 x 792 [287,392,514]
+ CRUSH rule 1 x 793 [631,133,850]
+ CRUSH rule 1 x 794 [931,517,543]
+ CRUSH rule 1 x 795 [551,962,477]
+ CRUSH rule 1 x 796 [814,4,95]
+ CRUSH rule 1 x 797 [64,201,299]
+ CRUSH rule 1 x 798 [422,530,114]
+ CRUSH rule 1 x 799 [824,32,679]
+ CRUSH rule 1 x 800 [862,623,489]
+ CRUSH rule 1 x 801 [145,550,329]
+ CRUSH rule 1 x 802 [570,19,847]
+ CRUSH rule 1 x 803 [151,812,662]
+ CRUSH rule 1 x 804 [467,93,264]
+ CRUSH rule 1 x 805 [621,223,938]
+ CRUSH rule 1 x 806 [898,957,805]
+ CRUSH rule 1 x 807 [354,531,422]
+ CRUSH rule 1 x 808 [7,96,76]
+ CRUSH rule 1 x 809 [70,734,719]
+ CRUSH rule 1 x 810 [701,18,972]
+ CRUSH rule 1 x 811 [248,547,103]
+ CRUSH rule 1 x 812 [230,576,821]
+ CRUSH rule 1 x 813 [805,114,683]
+ CRUSH rule 1 x 814 [54,619,973]
+ CRUSH rule 1 x 815 [679,412,613]
+ CRUSH rule 1 x 816 [919,448,826]
+ CRUSH rule 1 x 817 [765,830,436]
+ CRUSH rule 1 x 818 [415,566,644]
+ CRUSH rule 1 x 819 [721,319,865]
+ CRUSH rule 1 x 820 [218,301,333]
+ CRUSH rule 1 x 821 [185,795,680]
+ CRUSH rule 1 x 822 [356,261,54]
+ CRUSH rule 1 x 823 [220,281,549]
+ CRUSH rule 1 x 824 [292,809,887]
+ CRUSH rule 1 x 825 [949,778,101]
+ CRUSH rule 1 x 826 [767,818,833]
+ CRUSH rule 1 x 827 [631,83,406]
+ CRUSH rule 1 x 828 [288,986,445]
+ CRUSH rule 1 x 829 [990,667,915]
+ CRUSH rule 1 x 830 [152,571,778]
+ CRUSH rule 1 x 831 [814,563,630]
+ CRUSH rule 1 x 832 [235,641,616]
+ CRUSH rule 1 x 833 [657,565,922]
+ CRUSH rule 1 x 834 [907,231,644]
+ CRUSH rule 1 x 835 [784,262,771]
+ CRUSH rule 1 x 836 [951,158,366]
+ CRUSH rule 1 x 837 [556,498,334]
+ CRUSH rule 1 x 838 [329,274,964]
+ CRUSH rule 1 x 839 [568,209,939]
+ CRUSH rule 1 x 840 [45,579,842]
+ CRUSH rule 1 x 841 [652,702,24]
+ CRUSH rule 1 x 842 [629,984,314]
+ CRUSH rule 1 x 843 [799,690,688]
+ CRUSH rule 1 x 844 [694,600,534]
+ CRUSH rule 1 x 845 [332,30,179]
+ CRUSH rule 1 x 846 [452,251,712]
+ CRUSH rule 1 x 847 [399,681,847]
+ CRUSH rule 1 x 848 [303,138,440]
+ CRUSH rule 1 x 849 [666,346,708]
+ CRUSH rule 1 x 850 [644,511,345]
+ CRUSH rule 1 x 851 [527,546,737]
+ CRUSH rule 1 x 852 [31,809,94]
+ CRUSH rule 1 x 853 [483,330,869]
+ CRUSH rule 1 x 854 [697,953,968]
+ CRUSH rule 1 x 855 [837,996,239]
+ CRUSH rule 1 x 856 [712,40,547]
+ CRUSH rule 1 x 857 [77,984,576]
+ CRUSH rule 1 x 858 [412,384,841]
+ CRUSH rule 1 x 859 [173,760,26]
+ CRUSH rule 1 x 860 [776,429,328]
+ CRUSH rule 1 x 861 [705,405,477]
+ CRUSH rule 1 x 862 [809,44,788]
+ CRUSH rule 1 x 863 [349,496,963]
+ CRUSH rule 1 x 864 [717,858,101]
+ CRUSH rule 1 x 865 [857,603,586]
+ CRUSH rule 1 x 866 [394,304,71]
+ CRUSH rule 1 x 867 [640,773,663]
+ CRUSH rule 1 x 868 [613,950,712]
+ CRUSH rule 1 x 869 [973,889,524]
+ CRUSH rule 1 x 870 [505,35,386]
+ CRUSH rule 1 x 871 [239,264,262]
+ CRUSH rule 1 x 872 [21,767,456]
+ CRUSH rule 1 x 873 [954,666,980]
+ CRUSH rule 1 x 874 [54,510,947]
+ CRUSH rule 1 x 875 [809,418,452]
+ CRUSH rule 1 x 876 [483,457,61]
+ CRUSH rule 1 x 877 [542,531,952]
+ CRUSH rule 1 x 878 [217,674,857]
+ CRUSH rule 1 x 879 [999,475,134]
+ CRUSH rule 1 x 880 [678,573,935]
+ CRUSH rule 1 x 881 [394,835,789]
+ CRUSH rule 1 x 882 [467,382,353]
+ CRUSH rule 1 x 883 [802,744,237]
+ CRUSH rule 1 x 884 [653,660,638]
+ CRUSH rule 1 x 885 [898,704,307]
+ CRUSH rule 1 x 886 [434,357,938]
+ CRUSH rule 1 x 887 [297,226,711]
+ CRUSH rule 1 x 888 [863,324,443]
+ CRUSH rule 1 x 889 [105,102,308]
+ CRUSH rule 1 x 890 [550,248,606]
+ CRUSH rule 1 x 891 [575,928,880]
+ CRUSH rule 1 x 892 [259,862,133]
+ CRUSH rule 1 x 893 [902,880,543]
+ CRUSH rule 1 x 894 [180,169,916]
+ CRUSH rule 1 x 895 [725,849,182]
+ CRUSH rule 1 x 896 [951,34,874]
+ CRUSH rule 1 x 897 [810,352,73]
+ CRUSH rule 1 x 898 [979,433,719]
+ CRUSH rule 1 x 899 [685,668,534]
+ CRUSH rule 1 x 900 [530,978,41]
+ CRUSH rule 1 x 901 [740,107,336]
+ CRUSH rule 1 x 902 [800,743,693]
+ CRUSH rule 1 x 903 [230,267,842]
+ CRUSH rule 1 x 904 [346,949,460]
+ CRUSH rule 1 x 905 [530,397,619]
+ CRUSH rule 1 x 906 [80,426,138]
+ CRUSH rule 1 x 907 [365,968,475]
+ CRUSH rule 1 x 908 [204,832,742]
+ CRUSH rule 1 x 909 [883,989,146]
+ CRUSH rule 1 x 910 [549,593,249]
+ CRUSH rule 1 x 911 [325,847,352]
+ CRUSH rule 1 x 912 [874,888,582]
+ CRUSH rule 1 x 913 [331,463,342]
+ CRUSH rule 1 x 914 [836,468,601]
+ CRUSH rule 1 x 915 [245,228,100]
+ CRUSH rule 1 x 916 [77,967,364]
+ CRUSH rule 1 x 917 [239,60,866]
+ CRUSH rule 1 x 918 [988,115,922]
+ CRUSH rule 1 x 919 [783,139,696]
+ CRUSH rule 1 x 920 [623,408,685]
+ CRUSH rule 1 x 921 [105,799,144]
+ CRUSH rule 1 x 922 [887,505,652]
+ CRUSH rule 1 x 923 [223,318,552]
+ CRUSH rule 1 x 924 [25,778,366]
+ CRUSH rule 1 x 925 [912,601,297]
+ CRUSH rule 1 x 926 [968,133,155]
+ CRUSH rule 1 x 927 [277,724,214]
+ CRUSH rule 1 x 928 [554,203,658]
+ CRUSH rule 1 x 929 [761,802,367]
+ CRUSH rule 1 x 930 [814,61,788]
+ CRUSH rule 1 x 931 [29,193,61]
+ CRUSH rule 1 x 932 [446,198,862]
+ CRUSH rule 1 x 933 [352,742,216]
+ CRUSH rule 1 x 934 [730,2,332]
+ CRUSH rule 1 x 935 [731,23,736]
+ CRUSH rule 1 x 936 [322,975,20]
+ CRUSH rule 1 x 937 [822,221,841]
+ CRUSH rule 1 x 938 [557,850,66]
+ CRUSH rule 1 x 939 [150,11,971]
+ CRUSH rule 1 x 940 [638,398,169]
+ CRUSH rule 1 x 941 [730,342,929]
+ CRUSH rule 1 x 942 [62,292,166]
+ CRUSH rule 1 x 943 [165,314,519]
+ CRUSH rule 1 x 944 [199,625,766]
+ CRUSH rule 1 x 945 [946,999,699]
+ CRUSH rule 1 x 946 [595,93,852]
+ CRUSH rule 1 x 947 [800,582,356]
+ CRUSH rule 1 x 948 [132,551,139]
+ CRUSH rule 1 x 949 [792,920,466]
+ CRUSH rule 1 x 950 [111,345,176]
+ CRUSH rule 1 x 951 [414,619,648]
+ CRUSH rule 1 x 952 [775,469,500]
+ CRUSH rule 1 x 953 [349,1,5]
+ CRUSH rule 1 x 954 [570,940,410]
+ CRUSH rule 1 x 955 [729,774,823]
+ CRUSH rule 1 x 956 [519,141,575]
+ CRUSH rule 1 x 957 [242,709,611]
+ CRUSH rule 1 x 958 [84,217,227]
+ CRUSH rule 1 x 959 [270,413,918]
+ CRUSH rule 1 x 960 [458,192,307]
+ CRUSH rule 1 x 961 [981,388,777]
+ CRUSH rule 1 x 962 [623,834,277]
+ CRUSH rule 1 x 963 [291,167,714]
+ CRUSH rule 1 x 964 [28,156,788]
+ CRUSH rule 1 x 965 [675,557,290]
+ CRUSH rule 1 x 966 [836,306,946]
+ CRUSH rule 1 x 967 [966,386,735]
+ CRUSH rule 1 x 968 [864,756,690]
+ CRUSH rule 1 x 969 [729,625,480]
+ CRUSH rule 1 x 970 [800,362,646]
+ CRUSH rule 1 x 971 [737,381,153]
+ CRUSH rule 1 x 972 [952,245,720]
+ CRUSH rule 1 x 973 [356,455,579]
+ CRUSH rule 1 x 974 [545,758,586]
+ CRUSH rule 1 x 975 [336,191,202]
+ CRUSH rule 1 x 976 [446,208,757]
+ CRUSH rule 1 x 977 [202,896,196]
+ CRUSH rule 1 x 978 [612,324,996]
+ CRUSH rule 1 x 979 [843,457,675]
+ CRUSH rule 1 x 980 [60,914,881]
+ CRUSH rule 1 x 981 [702,749,937]
+ CRUSH rule 1 x 982 [298,928,738]
+ CRUSH rule 1 x 983 [723,572,395]
+ CRUSH rule 1 x 984 [723,864,804]
+ CRUSH rule 1 x 985 [945,459,868]
+ CRUSH rule 1 x 986 [772,664,535]
+ CRUSH rule 1 x 987 [88,324,312]
+ CRUSH rule 1 x 988 [522,927,131]
+ CRUSH rule 1 x 989 [578,332,208]
+ CRUSH rule 1 x 990 [638,228,414]
+ CRUSH rule 1 x 991 [530,221,451]
+ CRUSH rule 1 x 992 [925,705,275]
+ CRUSH rule 1 x 993 [991,301,43]
+ CRUSH rule 1 x 994 [276,51,868]
+ CRUSH rule 1 x 995 [288,836,753]
+ CRUSH rule 1 x 996 [887,983,252]
+ CRUSH rule 1 x 997 [110,924,386]
+ CRUSH rule 1 x 998 [435,830,485]
+ CRUSH rule 1 x 999 [876,738,357]
+ CRUSH rule 1 x 1000 [178,963,638]
+ CRUSH rule 1 x 1001 [99,519,66]
+ CRUSH rule 1 x 1002 [515,534,468]
+ CRUSH rule 1 x 1003 [104,611,937]
+ CRUSH rule 1 x 1004 [269,638,724]
+ CRUSH rule 1 x 1005 [369,223,309]
+ CRUSH rule 1 x 1006 [40,107,69]
+ CRUSH rule 1 x 1007 [978,111,416]
+ CRUSH rule 1 x 1008 [965,956,624]
+ CRUSH rule 1 x 1009 [598,476,356]
+ CRUSH rule 1 x 1010 [767,523,239]
+ CRUSH rule 1 x 1011 [289,871,207]
+ CRUSH rule 1 x 1012 [128,28,370]
+ CRUSH rule 1 x 1013 [979,765,660]
+ CRUSH rule 1 x 1014 [979,948,513]
+ CRUSH rule 1 x 1015 [277,790,396]
+ CRUSH rule 1 x 1016 [262,73,128]
+ CRUSH rule 1 x 1017 [150,269,61]
+ CRUSH rule 1 x 1018 [555,829,554]
+ CRUSH rule 1 x 1019 [513,356,265]
+ CRUSH rule 1 x 1020 [158,161,877]
+ CRUSH rule 1 x 1021 [915,998,957]
+ CRUSH rule 1 x 1022 [967,829,973]
+ CRUSH rule 1 x 1023 [488,257,614]
+ rule 1 (metadata) num_rep 3 result size == 3:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450]
+ CRUSH rule 1 x 1 [876,250,334,633]
+ CRUSH rule 1 x 2 [292,832,53,392]
+ CRUSH rule 1 x 3 [623,387,124,998]
+ CRUSH rule 1 x 4 [61,334,710,4]
+ CRUSH rule 1 x 5 [946,557,713,664]
+ CRUSH rule 1 x 6 [576,668,212,163]
+ CRUSH rule 1 x 7 [645,753,906,393]
+ CRUSH rule 1 x 8 [243,6,863,781]
+ CRUSH rule 1 x 9 [22,578,251,410]
+ CRUSH rule 1 x 10 [758,828,360,477]
+ CRUSH rule 1 x 11 [769,120,124,527]
+ CRUSH rule 1 x 12 [780,364,689,755]
+ CRUSH rule 1 x 13 [557,18,351,719]
+ CRUSH rule 1 x 14 [59,561,249,461]
+ CRUSH rule 1 x 15 [718,928,993,21]
+ CRUSH rule 1 x 16 [673,632,841,954]
+ CRUSH rule 1 x 17 [648,43,560,514]
+ CRUSH rule 1 x 18 [654,219,181,568]
+ CRUSH rule 1 x 19 [850,545,377,848]
+ CRUSH rule 1 x 20 [717,785,974,5]
+ CRUSH rule 1 x 21 [420,57,519,306]
+ CRUSH rule 1 x 22 [503,998,193,821]
+ CRUSH rule 1 x 23 [411,663,168,110]
+ CRUSH rule 1 x 24 [266,861,353,1]
+ CRUSH rule 1 x 25 [760,483,818,600]
+ CRUSH rule 1 x 26 [903,24,573,718]
+ CRUSH rule 1 x 27 [946,188,289,510]
+ CRUSH rule 1 x 28 [69,312,73,198]
+ CRUSH rule 1 x 29 [844,883,337,628]
+ CRUSH rule 1 x 30 [621,18,613,794]
+ CRUSH rule 1 x 31 [784,943,814,539]
+ CRUSH rule 1 x 32 [173,374,369,972]
+ CRUSH rule 1 x 33 [698,336,357,966]
+ CRUSH rule 1 x 34 [168,836,210,798]
+ CRUSH rule 1 x 35 [274,509,534,818]
+ CRUSH rule 1 x 36 [318,215,153,628]
+ CRUSH rule 1 x 37 [173,604,109,935]
+ CRUSH rule 1 x 38 [708,444,683,604]
+ CRUSH rule 1 x 39 [662,198,417,680]
+ CRUSH rule 1 x 40 [620,801,414,78]
+ CRUSH rule 1 x 41 [811,264,177,127]
+ CRUSH rule 1 x 42 [863,179,527,660]
+ CRUSH rule 1 x 43 [686,822,988,228]
+ CRUSH rule 1 x 44 [396,222,46,841]
+ CRUSH rule 1 x 45 [991,694,253,142]
+ CRUSH rule 1 x 46 [420,909,184,285]
+ CRUSH rule 1 x 47 [467,211,605,207]
+ CRUSH rule 1 x 48 [955,329,368,168]
+ CRUSH rule 1 x 49 [974,891,931,29]
+ CRUSH rule 1 x 50 [870,441,691,823]
+ CRUSH rule 1 x 51 [182,930,25,936]
+ CRUSH rule 1 x 52 [704,812,894,794]
+ CRUSH rule 1 x 53 [185,713,631,280]
+ CRUSH rule 1 x 54 [270,441,100,82]
+ CRUSH rule 1 x 55 [895,734,958,793]
+ CRUSH rule 1 x 56 [564,963,683,324]
+ CRUSH rule 1 x 57 [738,130,208,973]
+ CRUSH rule 1 x 58 [524,113,806,903]
+ CRUSH rule 1 x 59 [408,337,668,529]
+ CRUSH rule 1 x 60 [228,790,857,309]
+ CRUSH rule 1 x 61 [154,843,717,467]
+ CRUSH rule 1 x 62 [594,811,549,276]
+ CRUSH rule 1 x 63 [646,67,884,925]
+ CRUSH rule 1 x 64 [175,542,155,837]
+ CRUSH rule 1 x 65 [745,619,131,867]
+ CRUSH rule 1 x 66 [275,468,23,35]
+ CRUSH rule 1 x 67 [246,958,524,493]
+ CRUSH rule 1 x 68 [711,473,403,228]
+ CRUSH rule 1 x 69 [493,924,850,939]
+ CRUSH rule 1 x 70 [30,499,644,33]
+ CRUSH rule 1 x 71 [984,883,574,716]
+ CRUSH rule 1 x 72 [71,286,942,363]
+ CRUSH rule 1 x 73 [922,618,3,371]
+ CRUSH rule 1 x 74 [629,414,185,573]
+ CRUSH rule 1 x 75 [222,20,174,820]
+ CRUSH rule 1 x 76 [262,366,339,290]
+ CRUSH rule 1 x 77 [638,469,992,280]
+ CRUSH rule 1 x 78 [324,511,788,7]
+ CRUSH rule 1 x 79 [577,990,64,94]
+ CRUSH rule 1 x 80 [501,95,278,903]
+ CRUSH rule 1 x 81 [506,812,9,698]
+ CRUSH rule 1 x 82 [222,145,80,785]
+ CRUSH rule 1 x 83 [71,634,61,91]
+ CRUSH rule 1 x 84 [49,761,773,368]
+ CRUSH rule 1 x 85 [985,896,708,861]
+ CRUSH rule 1 x 86 [537,745,93,524]
+ CRUSH rule 1 x 87 [997,317,463,626]
+ CRUSH rule 1 x 88 [957,350,890,857]
+ CRUSH rule 1 x 89 [399,730,148,314]
+ CRUSH rule 1 x 90 [943,706,683,267]
+ CRUSH rule 1 x 91 [22,368,149,928]
+ CRUSH rule 1 x 92 [532,424,426,773]
+ CRUSH rule 1 x 93 [218,489,405,681]
+ CRUSH rule 1 x 94 [181,96,102,515]
+ CRUSH rule 1 x 95 [343,957,820,139]
+ CRUSH rule 1 x 96 [861,270,87,797]
+ CRUSH rule 1 x 97 [459,706,45,328]
+ CRUSH rule 1 x 98 [327,867,353,948]
+ CRUSH rule 1 x 99 [974,133,468,906]
+ CRUSH rule 1 x 100 [32,445,547,371]
+ CRUSH rule 1 x 101 [142,90,337,950]
+ CRUSH rule 1 x 102 [172,129,139,22]
+ CRUSH rule 1 x 103 [630,47,161,356]
+ CRUSH rule 1 x 104 [758,133,278,11]
+ CRUSH rule 1 x 105 [843,604,47,33]
+ CRUSH rule 1 x 106 [28,681,193,679]
+ CRUSH rule 1 x 107 [74,320,85,819]
+ CRUSH rule 1 x 108 [875,593,575,517]
+ CRUSH rule 1 x 109 [411,985,811,720]
+ CRUSH rule 1 x 110 [440,774,799,660]
+ CRUSH rule 1 x 111 [405,742,276,359]
+ CRUSH rule 1 x 112 [143,181,922,545]
+ CRUSH rule 1 x 113 [153,846,160,903]
+ CRUSH rule 1 x 114 [804,892,939,20]
+ CRUSH rule 1 x 115 [588,508,958,580]
+ CRUSH rule 1 x 116 [327,148,637,486]
+ CRUSH rule 1 x 117 [95,594,989,131]
+ CRUSH rule 1 x 118 [80,957,897,239]
+ CRUSH rule 1 x 119 [386,932,951,768]
+ CRUSH rule 1 x 120 [366,312,653,936]
+ CRUSH rule 1 x 121 [129,154,847,16]
+ CRUSH rule 1 x 122 [873,1,110,939]
+ CRUSH rule 1 x 123 [533,415,789,600]
+ CRUSH rule 1 x 124 [461,691,898,723]
+ CRUSH rule 1 x 125 [342,599,830,402]
+ CRUSH rule 1 x 126 [819,781,822,548]
+ CRUSH rule 1 x 127 [437,893,585,707]
+ CRUSH rule 1 x 128 [679,994,982,550]
+ CRUSH rule 1 x 129 [380,685,947,302]
+ CRUSH rule 1 x 130 [992,52,466,867]
+ CRUSH rule 1 x 131 [469,90,208,599]
+ CRUSH rule 1 x 132 [571,250,316,535]
+ CRUSH rule 1 x 133 [964,728,329,902]
+ CRUSH rule 1 x 134 [999,19,716,963]
+ CRUSH rule 1 x 135 [634,101,52,938]
+ CRUSH rule 1 x 136 [114,889,692,768]
+ CRUSH rule 1 x 137 [839,8,959,280]
+ CRUSH rule 1 x 138 [967,949,138,451]
+ CRUSH rule 1 x 139 [308,711,736,247]
+ CRUSH rule 1 x 140 [764,936,926,55]
+ CRUSH rule 1 x 141 [423,302,112,216]
+ CRUSH rule 1 x 142 [252,821,715,340]
+ CRUSH rule 1 x 143 [33,808,518,477]
+ CRUSH rule 1 x 144 [472,88,969,162]
+ CRUSH rule 1 x 145 [242,208,252,604]
+ CRUSH rule 1 x 146 [290,70,570,384]
+ CRUSH rule 1 x 147 [447,352,657,493]
+ CRUSH rule 1 x 148 [212,644,432,658]
+ CRUSH rule 1 x 149 [9,775,87,35]
+ CRUSH rule 1 x 150 [166,456,582,144]
+ CRUSH rule 1 x 151 [811,875,307,20]
+ CRUSH rule 1 x 152 [449,617,223,9]
+ CRUSH rule 1 x 153 [523,537,695,627]
+ CRUSH rule 1 x 154 [208,559,874,597]
+ CRUSH rule 1 x 155 [569,325,192,296]
+ CRUSH rule 1 x 156 [488,121,521,213]
+ CRUSH rule 1 x 157 [140,723,633,260]
+ CRUSH rule 1 x 158 [786,451,320,239]
+ CRUSH rule 1 x 159 [134,664,517,821]
+ CRUSH rule 1 x 160 [690,112,414,990]
+ CRUSH rule 1 x 161 [324,912,397,423]
+ CRUSH rule 1 x 162 [748,567,284,183]
+ CRUSH rule 1 x 163 [575,499,31,816]
+ CRUSH rule 1 x 164 [314,489,308,326]
+ CRUSH rule 1 x 165 [116,209,750,53]
+ CRUSH rule 1 x 166 [352,706,701,810]
+ CRUSH rule 1 x 167 [27,743,174,142]
+ CRUSH rule 1 x 168 [953,898,880,660]
+ CRUSH rule 1 x 169 [912,147,266,547]
+ CRUSH rule 1 x 170 [421,515,828,844]
+ CRUSH rule 1 x 171 [488,584,880,964]
+ CRUSH rule 1 x 172 [366,443,957,66]
+ CRUSH rule 1 x 173 [863,291,625,287]
+ CRUSH rule 1 x 174 [263,555,650,410]
+ CRUSH rule 1 x 175 [875,961,361,575]
+ CRUSH rule 1 x 176 [745,83,701,680]
+ CRUSH rule 1 x 177 [128,244,41,123]
+ CRUSH rule 1 x 178 [155,41,264,777]
+ CRUSH rule 1 x 179 [593,833,202,183]
+ CRUSH rule 1 x 180 [154,734,17,831]
+ CRUSH rule 1 x 181 [289,675,723,800]
+ CRUSH rule 1 x 182 [730,931,560,209]
+ CRUSH rule 1 x 183 [639,237,794,815]
+ CRUSH rule 1 x 184 [704,312,685,645]
+ CRUSH rule 1 x 185 [97,100,762,82]
+ CRUSH rule 1 x 186 [26,665,554,215]
+ CRUSH rule 1 x 187 [649,14,740,494]
+ CRUSH rule 1 x 188 [682,695,590,743]
+ CRUSH rule 1 x 189 [325,693,726,51]
+ CRUSH rule 1 x 190 [399,933,136,955]
+ CRUSH rule 1 x 191 [629,533,17,126]
+ CRUSH rule 1 x 192 [503,578,38,492]
+ CRUSH rule 1 x 193 [546,333,651,678]
+ CRUSH rule 1 x 194 [242,473,58,655]
+ CRUSH rule 1 x 195 [625,719,135,81]
+ CRUSH rule 1 x 196 [357,114,125,867]
+ CRUSH rule 1 x 197 [306,954,453,873]
+ CRUSH rule 1 x 198 [863,791,311,911]
+ CRUSH rule 1 x 199 [935,906,929,252]
+ CRUSH rule 1 x 200 [373,774,229,454]
+ CRUSH rule 1 x 201 [659,320,477,313]
+ CRUSH rule 1 x 202 [260,433,524,880]
+ CRUSH rule 1 x 203 [36,239,675,971]
+ CRUSH rule 1 x 204 [92,516,993,728]
+ CRUSH rule 1 x 205 [68,395,473,45]
+ CRUSH rule 1 x 206 [570,530,642,380]
+ CRUSH rule 1 x 207 [834,457,850,917]
+ CRUSH rule 1 x 208 [927,484,640,976]
+ CRUSH rule 1 x 209 [878,66,58,940]
+ CRUSH rule 1 x 210 [572,981,484,29]
+ CRUSH rule 1 x 211 [107,597,780,857]
+ CRUSH rule 1 x 212 [389,107,838,624]
+ CRUSH rule 1 x 213 [497,717,567,728]
+ CRUSH rule 1 x 214 [798,65,254,572]
+ CRUSH rule 1 x 215 [233,419,283,638]
+ CRUSH rule 1 x 216 [494,464,742,523]
+ CRUSH rule 1 x 217 [352,396,309,938]
+ CRUSH rule 1 x 218 [895,864,988,650]
+ CRUSH rule 1 x 219 [222,534,277,242]
+ CRUSH rule 1 x 220 [281,19,584,563]
+ CRUSH rule 1 x 221 [64,928,963,130]
+ CRUSH rule 1 x 222 [40,544,161,199]
+ CRUSH rule 1 x 223 [645,556,159,417]
+ CRUSH rule 1 x 224 [647,165,957,263]
+ CRUSH rule 1 x 225 [219,714,858,747]
+ CRUSH rule 1 x 226 [372,511,181,277]
+ CRUSH rule 1 x 227 [925,156,714,863]
+ CRUSH rule 1 x 228 [682,404,839,263]
+ CRUSH rule 1 x 229 [880,838,770,891]
+ CRUSH rule 1 x 230 [328,659,916,468]
+ CRUSH rule 1 x 231 [320,383,669,109]
+ CRUSH rule 1 x 232 [924,846,394,319]
+ CRUSH rule 1 x 233 [948,652,575,838]
+ CRUSH rule 1 x 234 [484,943,42,575]
+ CRUSH rule 1 x 235 [750,65,590,168]
+ CRUSH rule 1 x 236 [551,787,490,136]
+ CRUSH rule 1 x 237 [390,157,166,251]
+ CRUSH rule 1 x 238 [570,6,989,707]
+ CRUSH rule 1 x 239 [729,959,376,975]
+ CRUSH rule 1 x 240 [981,241,156,767]
+ CRUSH rule 1 x 241 [310,816,641,177]
+ CRUSH rule 1 x 242 [161,63,642,837]
+ CRUSH rule 1 x 243 [180,394,33,683]
+ CRUSH rule 1 x 244 [52,174,685,189]
+ CRUSH rule 1 x 245 [523,121,915,84]
+ CRUSH rule 1 x 246 [362,893,390,487]
+ CRUSH rule 1 x 247 [382,184,116,34]
+ CRUSH rule 1 x 248 [129,114,852,469]
+ CRUSH rule 1 x 249 [159,683,91,856]
+ CRUSH rule 1 x 250 [404,945,569,955]
+ CRUSH rule 1 x 251 [661,225,738,757]
+ CRUSH rule 1 x 252 [961,226,542,103]
+ CRUSH rule 1 x 253 [651,97,225,364]
+ CRUSH rule 1 x 254 [123,33,741,692]
+ CRUSH rule 1 x 255 [314,649,891,855]
+ CRUSH rule 1 x 256 [315,215,651,126]
+ CRUSH rule 1 x 257 [825,264,867,732]
+ CRUSH rule 1 x 258 [624,789,370,723]
+ CRUSH rule 1 x 259 [602,542,70,563]
+ CRUSH rule 1 x 260 [717,878,43,56]
+ CRUSH rule 1 x 261 [145,517,20,903]
+ CRUSH rule 1 x 262 [223,1,561,420]
+ CRUSH rule 1 x 263 [462,211,405,508]
+ CRUSH rule 1 x 264 [654,471,266,662]
+ CRUSH rule 1 x 265 [302,794,704,798]
+ CRUSH rule 1 x 266 [202,132,884,209]
+ CRUSH rule 1 x 267 [282,938,657,113]
+ CRUSH rule 1 x 268 [338,309,356,278]
+ CRUSH rule 1 x 269 [738,122,266,200]
+ CRUSH rule 1 x 270 [707,982,946,196]
+ CRUSH rule 1 x 271 [705,432,364,735]
+ CRUSH rule 1 x 272 [756,545,942,56]
+ CRUSH rule 1 x 273 [197,502,527,721]
+ CRUSH rule 1 x 274 [992,44,653,573]
+ CRUSH rule 1 x 275 [544,789,170,434]
+ CRUSH rule 1 x 276 [658,467,577,268]
+ CRUSH rule 1 x 277 [143,490,880,483]
+ CRUSH rule 1 x 278 [492,647,355,282]
+ CRUSH rule 1 x 279 [517,792,604,987]
+ CRUSH rule 1 x 280 [825,740,27,848]
+ CRUSH rule 1 x 281 [224,629,120,562]
+ CRUSH rule 1 x 282 [298,661,380,416]
+ CRUSH rule 1 x 283 [311,606,208,50]
+ CRUSH rule 1 x 284 [771,466,371,743]
+ CRUSH rule 1 x 285 [693,362,404,676]
+ CRUSH rule 1 x 286 [364,477,285,167]
+ CRUSH rule 1 x 287 [591,611,828,995]
+ CRUSH rule 1 x 288 [965,541,848,796]
+ CRUSH rule 1 x 289 [225,551,948,877]
+ CRUSH rule 1 x 290 [577,762,777,751]
+ CRUSH rule 1 x 291 [160,903,477,381]
+ CRUSH rule 1 x 292 [873,598,216,666]
+ CRUSH rule 1 x 293 [100,234,874,47]
+ CRUSH rule 1 x 294 [285,943,379,520]
+ CRUSH rule 1 x 295 [938,262,880,327]
+ CRUSH rule 1 x 296 [850,327,86,472]
+ CRUSH rule 1 x 297 [951,53,99,558]
+ CRUSH rule 1 x 298 [173,336,85,766]
+ CRUSH rule 1 x 299 [598,591,315,386]
+ CRUSH rule 1 x 300 [531,957,62,459]
+ CRUSH rule 1 x 301 [823,628,23,858]
+ CRUSH rule 1 x 302 [184,80,780,871]
+ CRUSH rule 1 x 303 [521,766,222,830]
+ CRUSH rule 1 x 304 [980,127,807,507]
+ CRUSH rule 1 x 305 [153,816,22,927]
+ CRUSH rule 1 x 306 [423,739,664,753]
+ CRUSH rule 1 x 307 [997,557,682,456]
+ CRUSH rule 1 x 308 [991,874,534,465]
+ CRUSH rule 1 x 309 [860,394,724,858]
+ CRUSH rule 1 x 310 [589,818,546,201]
+ CRUSH rule 1 x 311 [477,774,225,590]
+ CRUSH rule 1 x 312 [887,853,950,354]
+ CRUSH rule 1 x 313 [802,646,447,416]
+ CRUSH rule 1 x 314 [654,974,229,511]
+ CRUSH rule 1 x 315 [767,227,28,740]
+ CRUSH rule 1 x 316 [778,83,733,359]
+ CRUSH rule 1 x 317 [184,418,642,986]
+ CRUSH rule 1 x 318 [525,410,500,543]
+ CRUSH rule 1 x 319 [476,724,569,382]
+ CRUSH rule 1 x 320 [149,610,697,296]
+ CRUSH rule 1 x 321 [710,79,667,671]
+ CRUSH rule 1 x 322 [175,275,323,333]
+ CRUSH rule 1 x 323 [819,604,638,792]
+ CRUSH rule 1 x 324 [16,745,511,439]
+ CRUSH rule 1 x 325 [486,400,872,873]
+ CRUSH rule 1 x 326 [613,765,207,19]
+ CRUSH rule 1 x 327 [125,289,738,408]
+ CRUSH rule 1 x 328 [807,383,476,583]
+ CRUSH rule 1 x 329 [588,938,599,432]
+ CRUSH rule 1 x 330 [932,644,41,611]
+ CRUSH rule 1 x 331 [341,953,950,537]
+ CRUSH rule 1 x 332 [153,726,459,950]
+ CRUSH rule 1 x 333 [745,845,853,860]
+ CRUSH rule 1 x 334 [614,751,807,58]
+ CRUSH rule 1 x 335 [518,721,221,283]
+ CRUSH rule 1 x 336 [389,424,77,309]
+ CRUSH rule 1 x 337 [753,508,765,720]
+ CRUSH rule 1 x 338 [128,810,490,753]
+ CRUSH rule 1 x 339 [430,308,58,751]
+ CRUSH rule 1 x 340 [541,44,630,231]
+ CRUSH rule 1 x 341 [402,26,631,439]
+ CRUSH rule 1 x 342 [982,57,992,461]
+ CRUSH rule 1 x 343 [833,412,572,732]
+ CRUSH rule 1 x 344 [784,533,792,41]
+ CRUSH rule 1 x 345 [546,300,304,691]
+ CRUSH rule 1 x 346 [302,420,428,891]
+ CRUSH rule 1 x 347 [488,778,101,217]
+ CRUSH rule 1 x 348 [903,744,937,718]
+ CRUSH rule 1 x 349 [471,547,582,306]
+ CRUSH rule 1 x 350 [348,221,823,335]
+ CRUSH rule 1 x 351 [961,582,705,346]
+ CRUSH rule 1 x 352 [728,137,461,298]
+ CRUSH rule 1 x 353 [904,202,184,447]
+ CRUSH rule 1 x 354 [345,226,319,256]
+ CRUSH rule 1 x 355 [50,430,175,43]
+ CRUSH rule 1 x 356 [87,185,55,423]
+ CRUSH rule 1 x 357 [762,459,921,473]
+ CRUSH rule 1 x 358 [908,25,280,6]
+ CRUSH rule 1 x 359 [484,15,132,121]
+ CRUSH rule 1 x 360 [173,378,337,702]
+ CRUSH rule 1 x 361 [404,577,115,25]
+ CRUSH rule 1 x 362 [403,1,422,945]
+ CRUSH rule 1 x 363 [639,911,510,162]
+ CRUSH rule 1 x 364 [752,689,610,990]
+ CRUSH rule 1 x 365 [956,999,212,230]
+ CRUSH rule 1 x 366 [860,925,924,763]
+ CRUSH rule 1 x 367 [205,609,647,665]
+ CRUSH rule 1 x 368 [301,284,810,169]
+ CRUSH rule 1 x 369 [452,658,339,217]
+ CRUSH rule 1 x 370 [11,467,695,989]
+ CRUSH rule 1 x 371 [124,487,55,514]
+ CRUSH rule 1 x 372 [253,48,979,846]
+ CRUSH rule 1 x 373 [715,605,775,748]
+ CRUSH rule 1 x 374 [191,887,920,760]
+ CRUSH rule 1 x 375 [711,385,651,665]
+ CRUSH rule 1 x 376 [597,818,49,458]
+ CRUSH rule 1 x 377 [294,256,933,771]
+ CRUSH rule 1 x 378 [34,151,681,707]
+ CRUSH rule 1 x 379 [869,136,315,378]
+ CRUSH rule 1 x 380 [294,97,575,791]
+ CRUSH rule 1 x 381 [119,710,219,827]
+ CRUSH rule 1 x 382 [69,631,508,706]
+ CRUSH rule 1 x 383 [922,588,589,925]
+ CRUSH rule 1 x 384 [221,945,671,117]
+ CRUSH rule 1 x 385 [561,737,953,723]
+ CRUSH rule 1 x 386 [335,442,788,696]
+ CRUSH rule 1 x 387 [514,43,353,88]
+ CRUSH rule 1 x 388 [587,89,157,996]
+ CRUSH rule 1 x 389 [109,641,255,466]
+ CRUSH rule 1 x 390 [925,149,421,489]
+ CRUSH rule 1 x 391 [267,87,387,527]
+ CRUSH rule 1 x 392 [382,485,370,849]
+ CRUSH rule 1 x 393 [425,721,221,753]
+ CRUSH rule 1 x 394 [898,18,38,793]
+ CRUSH rule 1 x 395 [806,876,269,679]
+ CRUSH rule 1 x 396 [790,970,437,449]
+ CRUSH rule 1 x 397 [136,363,507,613]
+ CRUSH rule 1 x 398 [914,116,558,258]
+ CRUSH rule 1 x 399 [261,94,299,202]
+ CRUSH rule 1 x 400 [661,197,338,461]
+ CRUSH rule 1 x 401 [953,979,287,803]
+ CRUSH rule 1 x 402 [738,819,618,522]
+ CRUSH rule 1 x 403 [573,238,425,546]
+ CRUSH rule 1 x 404 [526,848,790,253]
+ CRUSH rule 1 x 405 [582,505,330,334]
+ CRUSH rule 1 x 406 [768,324,493,60]
+ CRUSH rule 1 x 407 [260,951,437,587]
+ CRUSH rule 1 x 408 [657,81,770,734]
+ CRUSH rule 1 x 409 [498,89,182,423]
+ CRUSH rule 1 x 410 [28,793,737,352]
+ CRUSH rule 1 x 411 [684,992,60,659]
+ CRUSH rule 1 x 412 [261,958,699,950]
+ CRUSH rule 1 x 413 [891,835,297,441]
+ CRUSH rule 1 x 414 [127,459,119,965]
+ CRUSH rule 1 x 415 [272,540,631,328]
+ CRUSH rule 1 x 416 [739,617,115,530]
+ CRUSH rule 1 x 417 [106,209,157,878]
+ CRUSH rule 1 x 418 [525,441,147,390]
+ CRUSH rule 1 x 419 [603,673,615,465]
+ CRUSH rule 1 x 420 [988,213,251,226]
+ CRUSH rule 1 x 421 [761,521,748,368]
+ CRUSH rule 1 x 422 [317,160,924,548]
+ CRUSH rule 1 x 423 [137,807,168,472]
+ CRUSH rule 1 x 424 [920,37,146,263]
+ CRUSH rule 1 x 425 [277,693,285,221]
+ CRUSH rule 1 x 426 [485,936,407,854]
+ CRUSH rule 1 x 427 [242,515,9,564]
+ CRUSH rule 1 x 428 [632,635,26,473]
+ CRUSH rule 1 x 429 [641,73,465,127]
+ CRUSH rule 1 x 430 [626,585,6,387]
+ CRUSH rule 1 x 431 [697,76,753,570]
+ CRUSH rule 1 x 432 [590,526,306,283]
+ CRUSH rule 1 x 433 [284,387,149,817]
+ CRUSH rule 1 x 434 [538,985,79,953]
+ CRUSH rule 1 x 435 [30,318,593,635]
+ CRUSH rule 1 x 436 [164,919,851,693]
+ CRUSH rule 1 x 437 [322,212,163,606]
+ CRUSH rule 1 x 438 [142,392,85,594]
+ CRUSH rule 1 x 439 [119,370,68,443]
+ CRUSH rule 1 x 440 [333,403,187,863]
+ CRUSH rule 1 x 441 [477,727,906,145]
+ CRUSH rule 1 x 442 [274,590,933,244]
+ CRUSH rule 1 x 443 [983,748,574,718]
+ CRUSH rule 1 x 444 [536,509,431,146]
+ CRUSH rule 1 x 445 [485,753,528,209]
+ CRUSH rule 1 x 446 [345,634,42,294]
+ CRUSH rule 1 x 447 [61,845,767,600]
+ CRUSH rule 1 x 448 [333,232,292,846]
+ CRUSH rule 1 x 449 [680,16,484,670]
+ CRUSH rule 1 x 450 [235,214,79,423]
+ CRUSH rule 1 x 451 [961,468,333,640]
+ CRUSH rule 1 x 452 [525,479,153,528]
+ CRUSH rule 1 x 453 [138,466,302,86]
+ CRUSH rule 1 x 454 [137,625,215,402]
+ CRUSH rule 1 x 455 [173,150,997,16]
+ CRUSH rule 1 x 456 [235,226,238,258]
+ CRUSH rule 1 x 457 [450,577,253,413]
+ CRUSH rule 1 x 458 [195,537,91,814]
+ CRUSH rule 1 x 459 [381,555,312,573]
+ CRUSH rule 1 x 460 [972,730,534,678]
+ CRUSH rule 1 x 461 [506,279,142,830]
+ CRUSH rule 1 x 462 [692,959,578,57]
+ CRUSH rule 1 x 463 [788,667,949,550]
+ CRUSH rule 1 x 464 [133,122,588,999]
+ CRUSH rule 1 x 465 [971,190,230,777]
+ CRUSH rule 1 x 466 [394,576,148,157]
+ CRUSH rule 1 x 467 [517,28,366,362]
+ CRUSH rule 1 x 468 [829,143,874,225]
+ CRUSH rule 1 x 469 [987,936,106,725]
+ CRUSH rule 1 x 470 [107,982,56,889]
+ CRUSH rule 1 x 471 [181,897,629,860]
+ CRUSH rule 1 x 472 [547,512,172,24]
+ CRUSH rule 1 x 473 [760,997,824,905]
+ CRUSH rule 1 x 474 [787,418,743,628]
+ CRUSH rule 1 x 475 [662,312,253,617]
+ CRUSH rule 1 x 476 [110,495,185,508]
+ CRUSH rule 1 x 477 [393,954,834,132]
+ CRUSH rule 1 x 478 [246,483,480,644]
+ CRUSH rule 1 x 479 [70,929,697,931]
+ CRUSH rule 1 x 480 [753,119,961,607]
+ CRUSH rule 1 x 481 [470,429,677,242]
+ CRUSH rule 1 x 482 [451,566,961,675]
+ CRUSH rule 1 x 483 [816,72,371,278]
+ CRUSH rule 1 x 484 [540,454,389,31]
+ CRUSH rule 1 x 485 [74,582,624,684]
+ CRUSH rule 1 x 486 [958,595,199,763]
+ CRUSH rule 1 x 487 [228,302,804,833]
+ CRUSH rule 1 x 488 [180,529,722,956]
+ CRUSH rule 1 x 489 [47,617,812,187]
+ CRUSH rule 1 x 490 [905,822,479,124]
+ CRUSH rule 1 x 491 [892,370,609,998]
+ CRUSH rule 1 x 492 [588,959,127,948]
+ CRUSH rule 1 x 493 [353,461,593,291]
+ CRUSH rule 1 x 494 [378,848,443,368]
+ CRUSH rule 1 x 495 [845,653,768,234]
+ CRUSH rule 1 x 496 [13,988,0,691]
+ CRUSH rule 1 x 497 [796,877,788,394]
+ CRUSH rule 1 x 498 [412,337,270,705]
+ CRUSH rule 1 x 499 [330,695,8,74]
+ CRUSH rule 1 x 500 [820,272,547,765]
+ CRUSH rule 1 x 501 [110,44,132,442]
+ CRUSH rule 1 x 502 [336,595,650,274]
+ CRUSH rule 1 x 503 [922,211,157,722]
+ CRUSH rule 1 x 504 [483,52,122,432]
+ CRUSH rule 1 x 505 [482,598,224,279]
+ CRUSH rule 1 x 506 [493,123,43,856]
+ CRUSH rule 1 x 507 [12,598,264,422]
+ CRUSH rule 1 x 508 [227,157,611,301]
+ CRUSH rule 1 x 509 [807,242,363,122]
+ CRUSH rule 1 x 510 [134,437,227,75]
+ CRUSH rule 1 x 511 [212,54,83,799]
+ CRUSH rule 1 x 512 [236,630,758,752]
+ CRUSH rule 1 x 513 [994,693,644,938]
+ CRUSH rule 1 x 514 [45,508,831,19]
+ CRUSH rule 1 x 515 [504,138,480,272]
+ CRUSH rule 1 x 516 [285,409,136,570]
+ CRUSH rule 1 x 517 [300,232,23,906]
+ CRUSH rule 1 x 518 [397,674,98,898]
+ CRUSH rule 1 x 519 [86,750,772,913]
+ CRUSH rule 1 x 520 [900,833,614,130]
+ CRUSH rule 1 x 521 [31,47,236,751]
+ CRUSH rule 1 x 522 [390,16,280,144]
+ CRUSH rule 1 x 523 [618,308,424,590]
+ CRUSH rule 1 x 524 [635,189,687,963]
+ CRUSH rule 1 x 525 [311,916,699,262]
+ CRUSH rule 1 x 526 [48,738,227,718]
+ CRUSH rule 1 x 527 [202,851,889,216]
+ CRUSH rule 1 x 528 [565,827,590,273]
+ CRUSH rule 1 x 529 [934,864,241,43]
+ CRUSH rule 1 x 530 [502,934,298,670]
+ CRUSH rule 1 x 531 [681,627,942,487]
+ CRUSH rule 1 x 532 [422,6,147,205]
+ CRUSH rule 1 x 533 [863,68,364,983]
+ CRUSH rule 1 x 534 [962,931,775,172]
+ CRUSH rule 1 x 535 [89,565,397,693]
+ CRUSH rule 1 x 536 [499,351,760,458]
+ CRUSH rule 1 x 537 [676,547,787,311]
+ CRUSH rule 1 x 538 [58,644,571,649]
+ CRUSH rule 1 x 539 [837,953,457,711]
+ CRUSH rule 1 x 540 [831,50,132,213]
+ CRUSH rule 1 x 541 [582,757,121,525]
+ CRUSH rule 1 x 542 [472,132,790,997]
+ CRUSH rule 1 x 543 [382,272,797,330]
+ CRUSH rule 1 x 544 [947,930,496,883]
+ CRUSH rule 1 x 545 [425,570,305,77]
+ CRUSH rule 1 x 546 [18,65,529,437]
+ CRUSH rule 1 x 547 [445,715,600,472]
+ CRUSH rule 1 x 548 [367,569,980,167]
+ CRUSH rule 1 x 549 [125,715,671,817]
+ CRUSH rule 1 x 550 [425,599,744,199]
+ CRUSH rule 1 x 551 [44,1,528,922]
+ CRUSH rule 1 x 552 [246,104,68,239]
+ CRUSH rule 1 x 553 [71,703,615,28]
+ CRUSH rule 1 x 554 [207,124,217,166]
+ CRUSH rule 1 x 555 [570,28,317,420]
+ CRUSH rule 1 x 556 [674,152,421,79]
+ CRUSH rule 1 x 557 [347,817,191,391]
+ CRUSH rule 1 x 558 [627,426,369,692]
+ CRUSH rule 1 x 559 [940,630,924,242]
+ CRUSH rule 1 x 560 [295,903,541,29]
+ CRUSH rule 1 x 561 [506,682,384,637]
+ CRUSH rule 1 x 562 [718,529,87,729]
+ CRUSH rule 1 x 563 [552,332,747,206]
+ CRUSH rule 1 x 564 [835,769,736,486]
+ CRUSH rule 1 x 565 [8,167,539,182]
+ CRUSH rule 1 x 566 [600,481,301,263]
+ CRUSH rule 1 x 567 [999,994,509,899]
+ CRUSH rule 1 x 568 [252,431,157,62]
+ CRUSH rule 1 x 569 [643,218,943,455]
+ CRUSH rule 1 x 570 [617,635,765,422]
+ CRUSH rule 1 x 571 [757,80,59,98]
+ CRUSH rule 1 x 572 [299,348,575,889]
+ CRUSH rule 1 x 573 [25,505,270,167]
+ CRUSH rule 1 x 574 [215,431,624,177]
+ CRUSH rule 1 x 575 [225,252,611,546]
+ CRUSH rule 1 x 576 [627,94,159,857]
+ CRUSH rule 1 x 577 [237,809,778,636]
+ CRUSH rule 1 x 578 [885,313,120,344]
+ CRUSH rule 1 x 579 [924,575,787,831]
+ CRUSH rule 1 x 580 [718,51,766,121]
+ CRUSH rule 1 x 581 [219,807,129,571]
+ CRUSH rule 1 x 582 [893,701,598,863]
+ CRUSH rule 1 x 583 [246,930,964,170]
+ CRUSH rule 1 x 584 [336,432,680,175]
+ CRUSH rule 1 x 585 [324,999,397,485]
+ CRUSH rule 1 x 586 [558,230,976,541]
+ CRUSH rule 1 x 587 [985,830,597,21]
+ CRUSH rule 1 x 588 [211,544,57,134]
+ CRUSH rule 1 x 589 [129,21,112,190]
+ CRUSH rule 1 x 590 [467,969,652,593]
+ CRUSH rule 1 x 591 [758,514,316,164]
+ CRUSH rule 1 x 592 [525,253,190,443]
+ CRUSH rule 1 x 593 [601,885,339,152]
+ CRUSH rule 1 x 594 [227,60,450,30]
+ CRUSH rule 1 x 595 [720,854,496,912]
+ CRUSH rule 1 x 596 [751,195,997,77]
+ CRUSH rule 1 x 597 [129,574,714,8]
+ CRUSH rule 1 x 598 [679,207,604,396]
+ CRUSH rule 1 x 599 [668,315,683,349]
+ CRUSH rule 1 x 600 [143,396,464,444]
+ CRUSH rule 1 x 601 [326,573,873,902]
+ CRUSH rule 1 x 602 [860,281,875,535]
+ CRUSH rule 1 x 603 [709,328,445,349]
+ CRUSH rule 1 x 604 [571,62,814,95]
+ CRUSH rule 1 x 605 [252,739,860,27]
+ CRUSH rule 1 x 606 [339,236,759,842]
+ CRUSH rule 1 x 607 [590,248,759,868]
+ CRUSH rule 1 x 608 [145,635,309,467]
+ CRUSH rule 1 x 609 [973,547,223,79]
+ CRUSH rule 1 x 610 [435,816,961,983]
+ CRUSH rule 1 x 611 [559,283,422,584]
+ CRUSH rule 1 x 612 [273,149,123,576]
+ CRUSH rule 1 x 613 [828,614,642,674]
+ CRUSH rule 1 x 614 [478,748,393,34]
+ CRUSH rule 1 x 615 [392,155,144,326]
+ CRUSH rule 1 x 616 [778,637,452,248]
+ CRUSH rule 1 x 617 [622,713,996,833]
+ CRUSH rule 1 x 618 [149,877,270,329]
+ CRUSH rule 1 x 619 [604,163,656,409]
+ CRUSH rule 1 x 620 [181,23,409,198]
+ CRUSH rule 1 x 621 [735,902,386,237]
+ CRUSH rule 1 x 622 [661,824,717,568]
+ CRUSH rule 1 x 623 [142,121,643,61]
+ CRUSH rule 1 x 624 [360,716,420,398]
+ CRUSH rule 1 x 625 [541,167,385,1]
+ CRUSH rule 1 x 626 [364,431,610,363]
+ CRUSH rule 1 x 627 [458,137,557,410]
+ CRUSH rule 1 x 628 [250,350,556,497]
+ CRUSH rule 1 x 629 [928,160,710,572]
+ CRUSH rule 1 x 630 [243,19,918,556]
+ CRUSH rule 1 x 631 [438,221,574,676]
+ CRUSH rule 1 x 632 [797,368,247,5]
+ CRUSH rule 1 x 633 [993,749,525,485]
+ CRUSH rule 1 x 634 [239,351,633,299]
+ CRUSH rule 1 x 635 [640,965,25,961]
+ CRUSH rule 1 x 636 [173,290,297,991]
+ CRUSH rule 1 x 637 [0,918,98,108]
+ CRUSH rule 1 x 638 [702,235,424,900]
+ CRUSH rule 1 x 639 [475,687,31,785]
+ CRUSH rule 1 x 640 [31,664,399,677]
+ CRUSH rule 1 x 641 [296,473,108,963]
+ CRUSH rule 1 x 642 [894,273,427,606]
+ CRUSH rule 1 x 643 [117,111,732,191]
+ CRUSH rule 1 x 644 [438,336,327,512]
+ CRUSH rule 1 x 645 [982,702,351,573]
+ CRUSH rule 1 x 646 [334,804,146,842]
+ CRUSH rule 1 x 647 [933,787,185,334]
+ CRUSH rule 1 x 648 [22,444,400,862]
+ CRUSH rule 1 x 649 [503,229,213,460]
+ CRUSH rule 1 x 650 [328,659,420,443]
+ CRUSH rule 1 x 651 [3,880,823,123]
+ CRUSH rule 1 x 652 [495,977,563,733]
+ CRUSH rule 1 x 653 [185,718,804,280]
+ CRUSH rule 1 x 654 [130,528,380,81]
+ CRUSH rule 1 x 655 [560,872,454,504]
+ CRUSH rule 1 x 656 [219,885,178,981]
+ CRUSH rule 1 x 657 [233,684,813,490]
+ CRUSH rule 1 x 658 [778,6,756,380]
+ CRUSH rule 1 x 659 [240,663,306,540]
+ CRUSH rule 1 x 660 [244,855,196,147]
+ CRUSH rule 1 x 661 [184,270,128,398]
+ CRUSH rule 1 x 662 [65,883,921,438]
+ CRUSH rule 1 x 663 [323,721,594,812]
+ CRUSH rule 1 x 664 [865,113,512,51]
+ CRUSH rule 1 x 665 [420,850,591,475]
+ CRUSH rule 1 x 666 [319,767,246,3]
+ CRUSH rule 1 x 667 [875,39,343,100]
+ CRUSH rule 1 x 668 [331,122,263,599]
+ CRUSH rule 1 x 669 [915,521,402,747]
+ CRUSH rule 1 x 670 [845,659,943,447]
+ CRUSH rule 1 x 671 [108,634,527,363]
+ CRUSH rule 1 x 672 [578,216,110,589]
+ CRUSH rule 1 x 673 [442,74,579,797]
+ CRUSH rule 1 x 674 [588,364,281,308]
+ CRUSH rule 1 x 675 [489,698,744,671]
+ CRUSH rule 1 x 676 [928,911,40,180]
+ CRUSH rule 1 x 677 [399,269,692,131]
+ CRUSH rule 1 x 678 [546,752,544,155]
+ CRUSH rule 1 x 679 [988,25,275,433]
+ CRUSH rule 1 x 680 [335,963,382,486]
+ CRUSH rule 1 x 681 [690,462,623,466]
+ CRUSH rule 1 x 682 [196,588,154,257]
+ CRUSH rule 1 x 683 [627,25,421,160]
+ CRUSH rule 1 x 684 [38,804,592,158]
+ CRUSH rule 1 x 685 [841,368,548,362]
+ CRUSH rule 1 x 686 [336,287,525,440]
+ CRUSH rule 1 x 687 [20,682,924,653]
+ CRUSH rule 1 x 688 [463,371,780,556]
+ CRUSH rule 1 x 689 [569,250,78,816]
+ CRUSH rule 1 x 690 [551,144,587,263]
+ CRUSH rule 1 x 691 [766,464,446,533]
+ CRUSH rule 1 x 692 [739,634,18,245]
+ CRUSH rule 1 x 693 [339,297,118,330]
+ CRUSH rule 1 x 694 [405,26,830,181]
+ CRUSH rule 1 x 695 [622,576,597,535]
+ CRUSH rule 1 x 696 [558,902,689,13]
+ CRUSH rule 1 x 697 [818,222,406,691]
+ CRUSH rule 1 x 698 [178,48,402,233]
+ CRUSH rule 1 x 699 [450,244,180,919]
+ CRUSH rule 1 x 700 [502,771,987,706]
+ CRUSH rule 1 x 701 [4,612,782,216]
+ CRUSH rule 1 x 702 [177,630,232,923]
+ CRUSH rule 1 x 703 [354,178,389,393]
+ CRUSH rule 1 x 704 [646,601,156,171]
+ CRUSH rule 1 x 705 [921,401,890,265]
+ CRUSH rule 1 x 706 [652,877,562,452]
+ CRUSH rule 1 x 707 [345,745,67,716]
+ CRUSH rule 1 x 708 [333,607,180,469]
+ CRUSH rule 1 x 709 [45,187,302,115]
+ CRUSH rule 1 x 710 [94,855,43,199]
+ CRUSH rule 1 x 711 [227,653,731,150]
+ CRUSH rule 1 x 712 [398,953,136,870]
+ CRUSH rule 1 x 713 [116,800,503,662]
+ CRUSH rule 1 x 714 [111,629,866,709]
+ CRUSH rule 1 x 715 [531,291,486,382]
+ CRUSH rule 1 x 716 [169,541,291,42]
+ CRUSH rule 1 x 717 [417,446,994,894]
+ CRUSH rule 1 x 718 [992,383,298,844]
+ CRUSH rule 1 x 719 [936,674,324,759]
+ CRUSH rule 1 x 720 [370,188,174,464]
+ CRUSH rule 1 x 721 [320,859,278,259]
+ CRUSH rule 1 x 722 [7,2,673,129]
+ CRUSH rule 1 x 723 [270,553,831,662]
+ CRUSH rule 1 x 724 [666,822,708,895]
+ CRUSH rule 1 x 725 [794,406,875,459]
+ CRUSH rule 1 x 726 [420,556,341,292]
+ CRUSH rule 1 x 727 [561,461,129,635]
+ CRUSH rule 1 x 728 [951,330,196,756]
+ CRUSH rule 1 x 729 [656,644,436,591]
+ CRUSH rule 1 x 730 [3,558,629,184]
+ CRUSH rule 1 x 731 [852,89,75,735]
+ CRUSH rule 1 x 732 [983,840,869,976]
+ CRUSH rule 1 x 733 [285,396,388,122]
+ CRUSH rule 1 x 734 [125,510,402,640]
+ CRUSH rule 1 x 735 [417,773,686,504]
+ CRUSH rule 1 x 736 [749,396,632,550]
+ CRUSH rule 1 x 737 [644,991,946,135]
+ CRUSH rule 1 x 738 [449,683,290,220]
+ CRUSH rule 1 x 739 [341,220,641,454]
+ CRUSH rule 1 x 740 [874,524,674,650]
+ CRUSH rule 1 x 741 [189,472,712,798]
+ CRUSH rule 1 x 742 [912,581,114,81]
+ CRUSH rule 1 x 743 [654,914,425,441]
+ CRUSH rule 1 x 744 [725,295,579,377]
+ CRUSH rule 1 x 745 [787,858,850,506]
+ CRUSH rule 1 x 746 [757,848,704,30]
+ CRUSH rule 1 x 747 [700,81,867,681]
+ CRUSH rule 1 x 748 [557,436,238,664]
+ CRUSH rule 1 x 749 [772,622,337,42]
+ CRUSH rule 1 x 750 [946,97,376,677]
+ CRUSH rule 1 x 751 [996,618,343,911]
+ CRUSH rule 1 x 752 [746,887,695,868]
+ CRUSH rule 1 x 753 [741,14,463,479]
+ CRUSH rule 1 x 754 [648,349,333,355]
+ CRUSH rule 1 x 755 [157,460,466,187]
+ CRUSH rule 1 x 756 [416,97,197,497]
+ CRUSH rule 1 x 757 [599,839,776,410]
+ CRUSH rule 1 x 758 [994,218,620,256]
+ CRUSH rule 1 x 759 [959,682,514,745]
+ CRUSH rule 1 x 760 [518,943,215,83]
+ CRUSH rule 1 x 761 [285,849,420,324]
+ CRUSH rule 1 x 762 [591,313,41,335]
+ CRUSH rule 1 x 763 [908,411,200,740]
+ CRUSH rule 1 x 764 [787,234,894,485]
+ CRUSH rule 1 x 765 [327,921,882,393]
+ CRUSH rule 1 x 766 [84,161,878,704]
+ CRUSH rule 1 x 767 [370,895,702,701]
+ CRUSH rule 1 x 768 [826,760,879,864]
+ CRUSH rule 1 x 769 [67,768,663,735]
+ CRUSH rule 1 x 770 [593,909,482,259]
+ CRUSH rule 1 x 771 [309,935,121,578]
+ CRUSH rule 1 x 772 [12,125,797,301]
+ CRUSH rule 1 x 773 [253,466,820,549]
+ CRUSH rule 1 x 774 [164,390,705,109]
+ CRUSH rule 1 x 775 [703,47,43,973]
+ CRUSH rule 1 x 776 [728,231,80,916]
+ CRUSH rule 1 x 777 [981,621,568,729]
+ CRUSH rule 1 x 778 [411,456,544,597]
+ CRUSH rule 1 x 779 [346,121,519,921]
+ CRUSH rule 1 x 780 [476,39,288,381]
+ CRUSH rule 1 x 781 [10,130,585,844]
+ CRUSH rule 1 x 782 [462,246,581,902]
+ CRUSH rule 1 x 783 [580,373,153,775]
+ CRUSH rule 1 x 784 [413,113,978,990]
+ CRUSH rule 1 x 785 [341,856,332,354]
+ CRUSH rule 1 x 786 [411,140,313,393]
+ CRUSH rule 1 x 787 [605,522,211,813]
+ CRUSH rule 1 x 788 [226,545,35,142]
+ CRUSH rule 1 x 789 [545,320,414,702]
+ CRUSH rule 1 x 790 [414,748,816,327]
+ CRUSH rule 1 x 791 [660,906,406,697]
+ CRUSH rule 1 x 792 [287,392,514,204]
+ CRUSH rule 1 x 793 [631,133,850,713]
+ CRUSH rule 1 x 794 [931,517,543,210]
+ CRUSH rule 1 x 795 [551,962,477,948]
+ CRUSH rule 1 x 796 [814,4,95,27]
+ CRUSH rule 1 x 797 [64,201,299,734]
+ CRUSH rule 1 x 798 [422,530,114,431]
+ CRUSH rule 1 x 799 [824,32,679,562]
+ CRUSH rule 1 x 800 [862,623,489,637]
+ CRUSH rule 1 x 801 [145,550,329,324]
+ CRUSH rule 1 x 802 [570,19,847,308]
+ CRUSH rule 1 x 803 [151,812,662,358]
+ CRUSH rule 1 x 804 [467,93,264,863]
+ CRUSH rule 1 x 805 [621,223,938,809]
+ CRUSH rule 1 x 806 [898,957,805,430]
+ CRUSH rule 1 x 807 [354,531,422,159]
+ CRUSH rule 1 x 808 [7,96,76,897]
+ CRUSH rule 1 x 809 [70,734,719,56]
+ CRUSH rule 1 x 810 [701,18,972,327]
+ CRUSH rule 1 x 811 [248,547,103,728]
+ CRUSH rule 1 x 812 [230,576,821,566]
+ CRUSH rule 1 x 813 [805,114,683,629]
+ CRUSH rule 1 x 814 [54,619,973,741]
+ CRUSH rule 1 x 815 [679,412,613,132]
+ CRUSH rule 1 x 816 [919,448,826,414]
+ CRUSH rule 1 x 817 [765,830,436,521]
+ CRUSH rule 1 x 818 [415,566,644,687]
+ CRUSH rule 1 x 819 [721,319,865,750]
+ CRUSH rule 1 x 820 [218,301,333,190]
+ CRUSH rule 1 x 821 [185,795,680,953]
+ CRUSH rule 1 x 822 [356,261,54,522]
+ CRUSH rule 1 x 823 [220,281,549,456]
+ CRUSH rule 1 x 824 [292,809,887,74]
+ CRUSH rule 1 x 825 [949,778,101,311]
+ CRUSH rule 1 x 826 [767,818,833,927]
+ CRUSH rule 1 x 827 [631,83,406,635]
+ CRUSH rule 1 x 828 [288,986,445,26]
+ CRUSH rule 1 x 829 [990,667,915,694]
+ CRUSH rule 1 x 830 [152,571,778,505]
+ CRUSH rule 1 x 831 [814,563,630,97]
+ CRUSH rule 1 x 832 [235,641,616,110]
+ CRUSH rule 1 x 833 [657,565,922,140]
+ CRUSH rule 1 x 834 [907,231,644,13]
+ CRUSH rule 1 x 835 [784,262,771,264]
+ CRUSH rule 1 x 836 [951,158,366,710]
+ CRUSH rule 1 x 837 [556,498,334,633]
+ CRUSH rule 1 x 838 [329,274,964,547]
+ CRUSH rule 1 x 839 [568,209,939,364]
+ CRUSH rule 1 x 840 [45,579,842,70]
+ CRUSH rule 1 x 841 [652,702,24,605]
+ CRUSH rule 1 x 842 [629,984,314,895]
+ CRUSH rule 1 x 843 [799,690,688,648]
+ CRUSH rule 1 x 844 [694,600,534,700]
+ CRUSH rule 1 x 845 [332,30,179,93]
+ CRUSH rule 1 x 846 [452,251,712,719]
+ CRUSH rule 1 x 847 [399,681,847,739]
+ CRUSH rule 1 x 848 [303,138,440,346]
+ CRUSH rule 1 x 849 [666,346,708,873]
+ CRUSH rule 1 x 850 [644,511,345,844]
+ CRUSH rule 1 x 851 [527,546,737,425]
+ CRUSH rule 1 x 852 [31,809,94,618]
+ CRUSH rule 1 x 853 [483,330,869,184]
+ CRUSH rule 1 x 854 [697,953,968,143]
+ CRUSH rule 1 x 855 [837,996,239,621]
+ CRUSH rule 1 x 856 [712,40,547,430]
+ CRUSH rule 1 x 857 [77,984,576,551]
+ CRUSH rule 1 x 858 [412,384,841,465]
+ CRUSH rule 1 x 859 [173,760,26,300]
+ CRUSH rule 1 x 860 [776,429,328,917]
+ CRUSH rule 1 x 861 [705,405,477,50]
+ CRUSH rule 1 x 862 [809,44,788,938]
+ CRUSH rule 1 x 863 [349,496,963,178]
+ CRUSH rule 1 x 864 [717,858,101,239]
+ CRUSH rule 1 x 865 [857,603,586,262]
+ CRUSH rule 1 x 866 [394,304,71,96]
+ CRUSH rule 1 x 867 [640,773,663,974]
+ CRUSH rule 1 x 868 [613,950,712,663]
+ CRUSH rule 1 x 869 [973,889,524,22]
+ CRUSH rule 1 x 870 [505,35,386,498]
+ CRUSH rule 1 x 871 [239,264,262,773]
+ CRUSH rule 1 x 872 [21,767,456,748]
+ CRUSH rule 1 x 873 [954,666,980,264]
+ CRUSH rule 1 x 874 [54,510,947,1]
+ CRUSH rule 1 x 875 [809,418,452,462]
+ CRUSH rule 1 x 876 [483,457,61,248]
+ CRUSH rule 1 x 877 [542,531,952,939]
+ CRUSH rule 1 x 878 [217,674,857,644]
+ CRUSH rule 1 x 879 [999,475,134,250]
+ CRUSH rule 1 x 880 [678,573,935,385]
+ CRUSH rule 1 x 881 [394,835,789,802]
+ CRUSH rule 1 x 882 [467,382,353,56]
+ CRUSH rule 1 x 883 [802,744,237,337]
+ CRUSH rule 1 x 884 [653,660,638,700]
+ CRUSH rule 1 x 885 [898,704,307,445]
+ CRUSH rule 1 x 886 [434,357,938,641]
+ CRUSH rule 1 x 887 [297,226,711,428]
+ CRUSH rule 1 x 888 [863,324,443,213]
+ CRUSH rule 1 x 889 [105,102,308,163]
+ CRUSH rule 1 x 890 [550,248,606,704]
+ CRUSH rule 1 x 891 [575,928,880,891]
+ CRUSH rule 1 x 892 [259,862,133,271]
+ CRUSH rule 1 x 893 [902,880,543,542]
+ CRUSH rule 1 x 894 [180,169,916,43]
+ CRUSH rule 1 x 895 [725,849,182,129]
+ CRUSH rule 1 x 896 [951,34,874,537]
+ CRUSH rule 1 x 897 [810,352,73,939]
+ CRUSH rule 1 x 898 [979,433,719,411]
+ CRUSH rule 1 x 899 [685,668,534,932]
+ CRUSH rule 1 x 900 [530,978,41,894]
+ CRUSH rule 1 x 901 [740,107,336,175]
+ CRUSH rule 1 x 902 [800,743,693,310]
+ CRUSH rule 1 x 903 [230,267,842,266]
+ CRUSH rule 1 x 904 [346,949,460,973]
+ CRUSH rule 1 x 905 [530,397,619,958]
+ CRUSH rule 1 x 906 [80,426,138,672]
+ CRUSH rule 1 x 907 [365,968,475,297]
+ CRUSH rule 1 x 908 [204,832,742,809]
+ CRUSH rule 1 x 909 [883,989,146,959]
+ CRUSH rule 1 x 910 [549,593,249,853]
+ CRUSH rule 1 x 911 [325,847,352,214]
+ CRUSH rule 1 x 912 [874,888,582,796]
+ CRUSH rule 1 x 913 [331,463,342,574]
+ CRUSH rule 1 x 914 [836,468,601,732]
+ CRUSH rule 1 x 915 [245,228,100,661]
+ CRUSH rule 1 x 916 [77,967,364,435]
+ CRUSH rule 1 x 917 [239,60,866,221]
+ CRUSH rule 1 x 918 [988,115,922,80]
+ CRUSH rule 1 x 919 [783,139,696,1]
+ CRUSH rule 1 x 920 [623,408,685,953]
+ CRUSH rule 1 x 921 [105,799,144,90]
+ CRUSH rule 1 x 922 [887,505,652,348]
+ CRUSH rule 1 x 923 [223,318,552,458]
+ CRUSH rule 1 x 924 [25,778,366,333]
+ CRUSH rule 1 x 925 [912,601,297,682]
+ CRUSH rule 1 x 926 [968,133,709,144]
+ CRUSH rule 1 x 927 [277,724,214,988]
+ CRUSH rule 1 x 928 [554,203,658,789]
+ CRUSH rule 1 x 929 [761,802,367,528]
+ CRUSH rule 1 x 930 [814,61,788,736]
+ CRUSH rule 1 x 931 [29,193,61,41]
+ CRUSH rule 1 x 932 [446,198,862,534]
+ CRUSH rule 1 x 933 [352,742,216,321]
+ CRUSH rule 1 x 934 [730,2,332,631]
+ CRUSH rule 1 x 935 [731,23,736,79]
+ CRUSH rule 1 x 936 [322,975,20,904]
+ CRUSH rule 1 x 937 [822,221,841,161]
+ CRUSH rule 1 x 938 [557,850,66,630]
+ CRUSH rule 1 x 939 [150,11,971,371]
+ CRUSH rule 1 x 940 [638,398,169,616]
+ CRUSH rule 1 x 941 [730,342,929,577]
+ CRUSH rule 1 x 942 [62,292,166,814]
+ CRUSH rule 1 x 943 [165,314,519,548]
+ CRUSH rule 1 x 944 [199,625,766,176]
+ CRUSH rule 1 x 945 [946,999,699,303]
+ CRUSH rule 1 x 946 [595,93,852,142]
+ CRUSH rule 1 x 947 [800,582,356,93]
+ CRUSH rule 1 x 948 [132,551,139,920]
+ CRUSH rule 1 x 949 [792,920,466,380]
+ CRUSH rule 1 x 950 [111,345,176,543]
+ CRUSH rule 1 x 951 [414,619,648,655]
+ CRUSH rule 1 x 952 [775,469,500,356]
+ CRUSH rule 1 x 953 [349,1,5,251]
+ CRUSH rule 1 x 954 [570,940,410,249]
+ CRUSH rule 1 x 955 [729,774,823,800]
+ CRUSH rule 1 x 956 [519,141,575,625]
+ CRUSH rule 1 x 957 [242,709,611,97]
+ CRUSH rule 1 x 958 [84,217,227,253]
+ CRUSH rule 1 x 959 [270,413,918,789]
+ CRUSH rule 1 x 960 [458,192,307,279]
+ CRUSH rule 1 x 961 [981,388,777,546]
+ CRUSH rule 1 x 962 [623,834,277,134]
+ CRUSH rule 1 x 963 [291,167,714,468]
+ CRUSH rule 1 x 964 [28,156,788,127]
+ CRUSH rule 1 x 965 [675,557,290,517]
+ CRUSH rule 1 x 966 [836,306,946,283]
+ CRUSH rule 1 x 967 [966,386,735,837]
+ CRUSH rule 1 x 968 [864,756,690,121]
+ CRUSH rule 1 x 969 [729,625,480,769]
+ CRUSH rule 1 x 970 [800,362,646,582]
+ CRUSH rule 1 x 971 [737,381,153,684]
+ CRUSH rule 1 x 972 [952,245,720,884]
+ CRUSH rule 1 x 973 [356,455,579,857]
+ CRUSH rule 1 x 974 [545,758,586,596]
+ CRUSH rule 1 x 975 [336,191,202,146]
+ CRUSH rule 1 x 976 [446,208,757,620]
+ CRUSH rule 1 x 977 [202,896,196,956]
+ CRUSH rule 1 x 978 [612,324,996,225]
+ CRUSH rule 1 x 979 [843,457,675,650]
+ CRUSH rule 1 x 980 [60,914,881,626]
+ CRUSH rule 1 x 981 [702,749,937,153]
+ CRUSH rule 1 x 982 [298,928,738,167]
+ CRUSH rule 1 x 983 [723,572,395,358]
+ CRUSH rule 1 x 984 [723,864,804,935]
+ CRUSH rule 1 x 985 [945,459,868,211]
+ CRUSH rule 1 x 986 [772,664,535,169]
+ CRUSH rule 1 x 987 [88,324,312,843]
+ CRUSH rule 1 x 988 [522,927,131,996]
+ CRUSH rule 1 x 989 [578,332,208,605]
+ CRUSH rule 1 x 990 [638,228,414,311]
+ CRUSH rule 1 x 991 [530,221,451,422]
+ CRUSH rule 1 x 992 [925,705,275,81]
+ CRUSH rule 1 x 993 [991,301,43,469]
+ CRUSH rule 1 x 994 [276,51,868,683]
+ CRUSH rule 1 x 995 [288,836,753,790]
+ CRUSH rule 1 x 996 [887,983,252,686]
+ CRUSH rule 1 x 997 [110,924,386,79]
+ CRUSH rule 1 x 998 [435,830,485,853]
+ CRUSH rule 1 x 999 [876,738,357,913]
+ CRUSH rule 1 x 1000 [178,963,638,430]
+ CRUSH rule 1 x 1001 [99,519,66,759]
+ CRUSH rule 1 x 1002 [515,534,468,866]
+ CRUSH rule 1 x 1003 [104,611,937,698]
+ CRUSH rule 1 x 1004 [269,638,724,375]
+ CRUSH rule 1 x 1005 [369,223,309,409]
+ CRUSH rule 1 x 1006 [40,107,69,275]
+ CRUSH rule 1 x 1007 [978,111,416,758]
+ CRUSH rule 1 x 1008 [965,956,624,832]
+ CRUSH rule 1 x 1009 [598,476,356,695]
+ CRUSH rule 1 x 1010 [767,523,239,517]
+ CRUSH rule 1 x 1011 [289,871,207,576]
+ CRUSH rule 1 x 1012 [128,28,370,31]
+ CRUSH rule 1 x 1013 [979,765,660,812]
+ CRUSH rule 1 x 1014 [979,948,513,88]
+ CRUSH rule 1 x 1015 [277,790,396,672]
+ CRUSH rule 1 x 1016 [262,73,128,886]
+ CRUSH rule 1 x 1017 [150,269,61,499]
+ CRUSH rule 1 x 1018 [555,829,554,944]
+ CRUSH rule 1 x 1019 [513,356,265,446]
+ CRUSH rule 1 x 1020 [158,161,877,704]
+ CRUSH rule 1 x 1021 [915,998,957,285]
+ CRUSH rule 1 x 1022 [967,829,973,640]
+ CRUSH rule 1 x 1023 [488,257,614,859]
+ rule 1 (metadata) num_rep 4 result size == 4:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450,604]
+ CRUSH rule 1 x 1 [876,250,334,633,744]
+ CRUSH rule 1 x 2 [292,832,53,392,386]
+ CRUSH rule 1 x 3 [623,387,124,998,749]
+ CRUSH rule 1 x 4 [61,334,710,4,994]
+ CRUSH rule 1 x 5 [946,557,713,664,141]
+ CRUSH rule 1 x 6 [576,668,212,163,732]
+ CRUSH rule 1 x 7 [645,753,906,393,341]
+ CRUSH rule 1 x 8 [243,6,863,781,211]
+ CRUSH rule 1 x 9 [22,578,251,410,297]
+ CRUSH rule 1 x 10 [758,828,360,477,821]
+ CRUSH rule 1 x 11 [769,120,124,527,119]
+ CRUSH rule 1 x 12 [780,364,689,755,675]
+ CRUSH rule 1 x 13 [557,18,351,719,742]
+ CRUSH rule 1 x 14 [59,561,249,461,971]
+ CRUSH rule 1 x 15 [718,928,993,21,76]
+ CRUSH rule 1 x 16 [673,632,841,954,788]
+ CRUSH rule 1 x 17 [648,43,560,514,142]
+ CRUSH rule 1 x 18 [654,219,181,568,381]
+ CRUSH rule 1 x 19 [850,545,377,848,863]
+ CRUSH rule 1 x 20 [717,785,974,5,225]
+ CRUSH rule 1 x 21 [420,57,519,306,312]
+ CRUSH rule 1 x 22 [503,998,193,821,634]
+ CRUSH rule 1 x 23 [411,663,168,110,899]
+ CRUSH rule 1 x 24 [266,861,353,1,456]
+ CRUSH rule 1 x 25 [760,483,818,600,509]
+ CRUSH rule 1 x 26 [903,24,573,718,112]
+ CRUSH rule 1 x 27 [946,188,289,510,687]
+ CRUSH rule 1 x 28 [69,312,73,198,256]
+ CRUSH rule 1 x 29 [844,883,337,628,496]
+ CRUSH rule 1 x 30 [621,18,613,794,910]
+ CRUSH rule 1 x 31 [784,943,814,539,962]
+ CRUSH rule 1 x 32 [173,374,369,972,315]
+ CRUSH rule 1 x 33 [698,336,357,966,582]
+ CRUSH rule 1 x 34 [168,836,210,798,904]
+ CRUSH rule 1 x 35 [274,509,534,818,912]
+ CRUSH rule 1 x 36 [318,215,153,628,87]
+ CRUSH rule 1 x 37 [173,604,109,935,203]
+ CRUSH rule 1 x 38 [708,444,683,604,722]
+ CRUSH rule 1 x 39 [662,198,417,680,226]
+ CRUSH rule 1 x 40 [620,801,414,78,560]
+ CRUSH rule 1 x 41 [811,264,177,127,148]
+ CRUSH rule 1 x 42 [863,179,527,660,133]
+ CRUSH rule 1 x 43 [686,822,988,228,791]
+ CRUSH rule 1 x 44 [396,222,46,841,536]
+ CRUSH rule 1 x 45 [991,694,253,142,54]
+ CRUSH rule 1 x 46 [420,909,184,285,508]
+ CRUSH rule 1 x 47 [467,211,605,207,241]
+ CRUSH rule 1 x 48 [955,329,368,168,698]
+ CRUSH rule 1 x 49 [974,891,931,29,813]
+ CRUSH rule 1 x 50 [870,441,691,823,761]
+ CRUSH rule 1 x 51 [182,930,25,936,97]
+ CRUSH rule 1 x 52 [704,812,894,794,481]
+ CRUSH rule 1 x 53 [185,713,631,280,345]
+ CRUSH rule 1 x 54 [270,441,100,82,983]
+ CRUSH rule 1 x 55 [895,734,958,793,651]
+ CRUSH rule 1 x 56 [564,963,683,324,40]
+ CRUSH rule 1 x 57 [738,130,208,973,498]
+ CRUSH rule 1 x 58 [524,113,806,903,531]
+ CRUSH rule 1 x 59 [408,337,668,529,34]
+ CRUSH rule 1 x 60 [228,790,857,309,616]
+ CRUSH rule 1 x 61 [154,843,717,467,883]
+ CRUSH rule 1 x 62 [594,811,549,276,693]
+ CRUSH rule 1 x 63 [646,67,884,925,941]
+ CRUSH rule 1 x 64 [175,542,155,837,594]
+ CRUSH rule 1 x 65 [745,619,131,867,269]
+ CRUSH rule 1 x 66 [275,468,23,35,328]
+ CRUSH rule 1 x 67 [246,958,524,493,636]
+ CRUSH rule 1 x 68 [711,473,403,228,835]
+ CRUSH rule 1 x 69 [493,924,850,939,950]
+ CRUSH rule 1 x 70 [30,499,644,33,804]
+ CRUSH rule 1 x 71 [984,883,574,716,575]
+ CRUSH rule 1 x 72 [71,286,942,363,628]
+ CRUSH rule 1 x 73 [922,618,3,371,464]
+ CRUSH rule 1 x 74 [629,414,185,573,678]
+ CRUSH rule 1 x 75 [222,20,174,820,312]
+ CRUSH rule 1 x 76 [262,366,339,290,718]
+ CRUSH rule 1 x 77 [638,469,992,280,773]
+ CRUSH rule 1 x 78 [324,511,788,7,308]
+ CRUSH rule 1 x 79 [577,990,64,94,447]
+ CRUSH rule 1 x 80 [501,95,278,903,631]
+ CRUSH rule 1 x 81 [506,812,9,698,173]
+ CRUSH rule 1 x 82 [222,145,80,785,835]
+ CRUSH rule 1 x 83 [71,634,61,91,856]
+ CRUSH rule 1 x 84 [49,761,773,368,318]
+ CRUSH rule 1 x 85 [985,896,708,861,325]
+ CRUSH rule 1 x 86 [537,745,93,524,466]
+ CRUSH rule 1 x 87 [997,317,463,626,685]
+ CRUSH rule 1 x 88 [957,350,890,857,375]
+ CRUSH rule 1 x 89 [399,730,148,314,159]
+ CRUSH rule 1 x 90 [943,706,683,267,579]
+ CRUSH rule 1 x 91 [22,368,149,928,140]
+ CRUSH rule 1 x 92 [532,424,426,773,623]
+ CRUSH rule 1 x 93 [218,489,405,681,549]
+ CRUSH rule 1 x 94 [181,96,102,515,776]
+ CRUSH rule 1 x 95 [343,957,820,139,334]
+ CRUSH rule 1 x 96 [861,270,87,797,0]
+ CRUSH rule 1 x 97 [459,706,45,328,274]
+ CRUSH rule 1 x 98 [327,867,353,948,728]
+ CRUSH rule 1 x 99 [974,133,468,906,235]
+ CRUSH rule 1 x 100 [32,445,547,371,960]
+ CRUSH rule 1 x 101 [142,90,337,950,970]
+ CRUSH rule 1 x 102 [172,129,139,22,403]
+ CRUSH rule 1 x 103 [630,47,161,356,911]
+ CRUSH rule 1 x 104 [758,133,278,11,947]
+ CRUSH rule 1 x 105 [843,604,47,33,401]
+ CRUSH rule 1 x 106 [28,681,193,679,990]
+ CRUSH rule 1 x 107 [74,320,85,819,315]
+ CRUSH rule 1 x 108 [875,593,575,517,107]
+ CRUSH rule 1 x 109 [411,985,811,720,198]
+ CRUSH rule 1 x 110 [440,774,799,660,715]
+ CRUSH rule 1 x 111 [405,742,276,359,936]
+ CRUSH rule 1 x 112 [143,181,922,545,185]
+ CRUSH rule 1 x 113 [153,846,160,903,789]
+ CRUSH rule 1 x 114 [804,892,939,20,312]
+ CRUSH rule 1 x 115 [588,508,958,580,232]
+ CRUSH rule 1 x 116 [327,148,637,486,712]
+ CRUSH rule 1 x 117 [95,594,989,131,714]
+ CRUSH rule 1 x 118 [80,957,897,239,359]
+ CRUSH rule 1 x 119 [386,932,951,768,679]
+ CRUSH rule 1 x 120 [366,312,653,936,71]
+ CRUSH rule 1 x 121 [129,154,847,16,471]
+ CRUSH rule 1 x 122 [873,1,110,939,90]
+ CRUSH rule 1 x 123 [533,415,789,600,713]
+ CRUSH rule 1 x 124 [461,691,898,723,957]
+ CRUSH rule 1 x 125 [342,599,830,402,615]
+ CRUSH rule 1 x 126 [819,781,822,548,279]
+ CRUSH rule 1 x 127 [437,893,585,707,353]
+ CRUSH rule 1 x 128 [679,994,982,550,991]
+ CRUSH rule 1 x 129 [380,685,947,302,698]
+ CRUSH rule 1 x 130 [992,52,466,867,998]
+ CRUSH rule 1 x 131 [469,90,208,599,829]
+ CRUSH rule 1 x 132 [571,250,316,535,54]
+ CRUSH rule 1 x 133 [964,728,329,902,108]
+ CRUSH rule 1 x 134 [999,19,716,963,323]
+ CRUSH rule 1 x 135 [634,101,52,938,413]
+ CRUSH rule 1 x 136 [114,889,692,768,694]
+ CRUSH rule 1 x 137 [839,8,959,280,922]
+ CRUSH rule 1 x 138 [967,949,138,451,292]
+ CRUSH rule 1 x 139 [308,711,736,247,632]
+ CRUSH rule 1 x 140 [764,936,926,55,331]
+ CRUSH rule 1 x 141 [423,302,112,216,603]
+ CRUSH rule 1 x 142 [252,821,715,340,635]
+ CRUSH rule 1 x 143 [33,808,518,477,325]
+ CRUSH rule 1 x 144 [472,88,969,162,401]
+ CRUSH rule 1 x 145 [242,208,252,604,266]
+ CRUSH rule 1 x 146 [290,70,570,384,934]
+ CRUSH rule 1 x 147 [447,352,657,493,467]
+ CRUSH rule 1 x 148 [212,644,432,658,109]
+ CRUSH rule 1 x 149 [9,775,87,35,260]
+ CRUSH rule 1 x 150 [166,456,582,144,324]
+ CRUSH rule 1 x 151 [811,875,307,20,782]
+ CRUSH rule 1 x 152 [449,617,223,9,182]
+ CRUSH rule 1 x 153 [523,537,695,627,959]
+ CRUSH rule 1 x 154 [208,559,874,597,243]
+ CRUSH rule 1 x 155 [569,325,192,296,367]
+ CRUSH rule 1 x 156 [488,121,521,213,595]
+ CRUSH rule 1 x 157 [140,723,633,260,487]
+ CRUSH rule 1 x 158 [786,451,320,239,667]
+ CRUSH rule 1 x 159 [134,664,517,821,667]
+ CRUSH rule 1 x 160 [690,112,414,990,183]
+ CRUSH rule 1 x 161 [324,912,397,423,991]
+ CRUSH rule 1 x 162 [748,567,284,183,463]
+ CRUSH rule 1 x 163 [575,499,31,816,749]
+ CRUSH rule 1 x 164 [314,489,308,326,51]
+ CRUSH rule 1 x 165 [116,209,750,53,813]
+ CRUSH rule 1 x 166 [352,706,701,810,718]
+ CRUSH rule 1 x 167 [27,743,174,142,551]
+ CRUSH rule 1 x 168 [953,898,880,660,500]
+ CRUSH rule 1 x 169 [912,147,266,547,331]
+ CRUSH rule 1 x 170 [421,515,828,844,151]
+ CRUSH rule 1 x 171 [488,584,880,964,936]
+ CRUSH rule 1 x 172 [366,443,957,66,162]
+ CRUSH rule 1 x 173 [863,291,625,287,158]
+ CRUSH rule 1 x 174 [263,555,650,410,339]
+ CRUSH rule 1 x 175 [875,961,361,575,33]
+ CRUSH rule 1 x 176 [745,83,701,680,250]
+ CRUSH rule 1 x 177 [128,244,41,123,422]
+ CRUSH rule 1 x 178 [155,41,264,777,314]
+ CRUSH rule 1 x 179 [593,833,202,183,971]
+ CRUSH rule 1 x 180 [154,734,17,831,824]
+ CRUSH rule 1 x 181 [289,675,723,800,166]
+ CRUSH rule 1 x 182 [730,931,560,209,943]
+ CRUSH rule 1 x 183 [639,237,794,815,827]
+ CRUSH rule 1 x 184 [704,312,685,645,691]
+ CRUSH rule 1 x 185 [97,100,762,82,999]
+ CRUSH rule 1 x 186 [26,665,554,215,280]
+ CRUSH rule 1 x 187 [649,14,740,494,402]
+ CRUSH rule 1 x 188 [682,695,590,743,927]
+ CRUSH rule 1 x 189 [325,693,726,51,448]
+ CRUSH rule 1 x 190 [399,933,136,955,57]
+ CRUSH rule 1 x 191 [629,533,17,126,60]
+ CRUSH rule 1 x 192 [503,578,38,492,222]
+ CRUSH rule 1 x 193 [546,333,651,678,823]
+ CRUSH rule 1 x 194 [242,473,58,655,449]
+ CRUSH rule 1 x 195 [625,719,135,81,636]
+ CRUSH rule 1 x 196 [357,114,125,867,250]
+ CRUSH rule 1 x 197 [306,954,453,873,211]
+ CRUSH rule 1 x 198 [863,791,311,911,206]
+ CRUSH rule 1 x 199 [935,906,929,252,893]
+ CRUSH rule 1 x 200 [373,774,229,454,909]
+ CRUSH rule 1 x 201 [659,320,477,313,779]
+ CRUSH rule 1 x 202 [260,433,524,880,223]
+ CRUSH rule 1 x 203 [36,239,675,971,703]
+ CRUSH rule 1 x 204 [92,516,993,728,279]
+ CRUSH rule 1 x 205 [68,395,473,45,683]
+ CRUSH rule 1 x 206 [570,530,642,380,311]
+ CRUSH rule 1 x 207 [834,457,850,917,456]
+ CRUSH rule 1 x 208 [927,484,640,976,803]
+ CRUSH rule 1 x 209 [878,66,58,940,48]
+ CRUSH rule 1 x 210 [572,981,484,29,0]
+ CRUSH rule 1 x 211 [107,597,780,857,895]
+ CRUSH rule 1 x 212 [389,107,838,624,698]
+ CRUSH rule 1 x 213 [497,717,567,728,905]
+ CRUSH rule 1 x 214 [798,65,254,572,32]
+ CRUSH rule 1 x 215 [233,419,283,638,520]
+ CRUSH rule 1 x 216 [494,464,742,523,459]
+ CRUSH rule 1 x 217 [352,396,309,938,66]
+ CRUSH rule 1 x 218 [895,864,988,650,593]
+ CRUSH rule 1 x 219 [222,534,277,242,658]
+ CRUSH rule 1 x 220 [281,19,584,563,858]
+ CRUSH rule 1 x 221 [64,928,963,130,312]
+ CRUSH rule 1 x 222 [40,544,161,199,861]
+ CRUSH rule 1 x 223 [645,556,159,417,46]
+ CRUSH rule 1 x 224 [647,165,957,263,961]
+ CRUSH rule 1 x 225 [219,714,858,747,461]
+ CRUSH rule 1 x 226 [372,511,181,277,695]
+ CRUSH rule 1 x 227 [925,156,714,863,257]
+ CRUSH rule 1 x 228 [682,404,839,263,521]
+ CRUSH rule 1 x 229 [880,838,770,891,236]
+ CRUSH rule 1 x 230 [328,659,916,468,646]
+ CRUSH rule 1 x 231 [320,383,669,109,627]
+ CRUSH rule 1 x 232 [924,846,394,319,43]
+ CRUSH rule 1 x 233 [948,652,575,838,498]
+ CRUSH rule 1 x 234 [484,943,42,575,936]
+ CRUSH rule 1 x 235 [750,65,590,168,870]
+ CRUSH rule 1 x 236 [551,787,490,136,370]
+ CRUSH rule 1 x 237 [390,157,166,251,752]
+ CRUSH rule 1 x 238 [570,6,989,707,514]
+ CRUSH rule 1 x 239 [729,959,376,975,496]
+ CRUSH rule 1 x 240 [981,241,156,767,631]
+ CRUSH rule 1 x 241 [310,816,641,177,996]
+ CRUSH rule 1 x 242 [161,63,642,837,763]
+ CRUSH rule 1 x 243 [180,394,33,683,189]
+ CRUSH rule 1 x 244 [52,174,685,189,78]
+ CRUSH rule 1 x 245 [523,121,915,84,386]
+ CRUSH rule 1 x 246 [362,893,390,487,817]
+ CRUSH rule 1 x 247 [382,184,116,34,143]
+ CRUSH rule 1 x 248 [129,114,852,469,359]
+ CRUSH rule 1 x 249 [159,683,91,856,475]
+ CRUSH rule 1 x 250 [404,945,569,955,228]
+ CRUSH rule 1 x 251 [661,225,738,757,37]
+ CRUSH rule 1 x 252 [961,226,542,103,945]
+ CRUSH rule 1 x 253 [651,97,225,364,189]
+ CRUSH rule 1 x 254 [123,33,741,692,599]
+ CRUSH rule 1 x 255 [314,649,891,855,517]
+ CRUSH rule 1 x 256 [315,215,651,126,470]
+ CRUSH rule 1 x 257 [825,264,867,224,529]
+ CRUSH rule 1 x 258 [624,789,370,723,131]
+ CRUSH rule 1 x 259 [602,542,70,563,947]
+ CRUSH rule 1 x 260 [717,878,43,56,377]
+ CRUSH rule 1 x 261 [145,517,20,903,786]
+ CRUSH rule 1 x 262 [223,1,561,420,498]
+ CRUSH rule 1 x 263 [462,211,405,508,787]
+ CRUSH rule 1 x 264 [654,471,266,662,135]
+ CRUSH rule 1 x 265 [302,794,704,798,659]
+ CRUSH rule 1 x 266 [202,132,884,209,551]
+ CRUSH rule 1 x 267 [282,938,657,113,672]
+ CRUSH rule 1 x 268 [338,309,356,278,928]
+ CRUSH rule 1 x 269 [738,122,266,200,894]
+ CRUSH rule 1 x 270 [707,982,946,196,407]
+ CRUSH rule 1 x 271 [705,432,364,735,512]
+ CRUSH rule 1 x 272 [756,545,942,56,542]
+ CRUSH rule 1 x 273 [197,502,527,721,239]
+ CRUSH rule 1 x 274 [992,44,653,573,527]
+ CRUSH rule 1 x 275 [544,789,170,434,23]
+ CRUSH rule 1 x 276 [658,467,577,268,336]
+ CRUSH rule 1 x 277 [143,490,880,483,928]
+ CRUSH rule 1 x 278 [492,647,355,282,834]
+ CRUSH rule 1 x 279 [517,792,604,987,527]
+ CRUSH rule 1 x 280 [825,740,27,848,514]
+ CRUSH rule 1 x 281 [224,629,120,562,616]
+ CRUSH rule 1 x 282 [298,661,380,416,35]
+ CRUSH rule 1 x 283 [311,606,208,50,913]
+ CRUSH rule 1 x 284 [771,466,371,743,672]
+ CRUSH rule 1 x 285 [693,362,404,676,797]
+ CRUSH rule 1 x 286 [364,477,285,167,270]
+ CRUSH rule 1 x 287 [591,611,828,995,170]
+ CRUSH rule 1 x 288 [965,541,848,796,251]
+ CRUSH rule 1 x 289 [225,551,948,877,219]
+ CRUSH rule 1 x 290 [577,762,777,751,291]
+ CRUSH rule 1 x 291 [160,903,477,381,490]
+ CRUSH rule 1 x 292 [873,598,216,666,222]
+ CRUSH rule 1 x 293 [100,234,874,47,28]
+ CRUSH rule 1 x 294 [285,943,379,520,725]
+ CRUSH rule 1 x 295 [938,262,880,327,687]
+ CRUSH rule 1 x 296 [850,327,86,472,1]
+ CRUSH rule 1 x 297 [951,53,99,558,753]
+ CRUSH rule 1 x 298 [173,336,85,766,910]
+ CRUSH rule 1 x 299 [598,591,315,386,895]
+ CRUSH rule 1 x 300 [531,957,62,459,156]
+ CRUSH rule 1 x 301 [823,628,23,858,629]
+ CRUSH rule 1 x 302 [184,80,780,871,531]
+ CRUSH rule 1 x 303 [521,766,222,830,988]
+ CRUSH rule 1 x 304 [980,127,807,507,555]
+ CRUSH rule 1 x 305 [153,816,22,927,696]
+ CRUSH rule 1 x 306 [423,739,664,753,178]
+ CRUSH rule 1 x 307 [997,557,682,456,479]
+ CRUSH rule 1 x 308 [991,874,534,465,330]
+ CRUSH rule 1 x 309 [860,394,724,858,246]
+ CRUSH rule 1 x 310 [589,818,546,201,94]
+ CRUSH rule 1 x 311 [477,774,225,590,830]
+ CRUSH rule 1 x 312 [887,853,950,354,58]
+ CRUSH rule 1 x 313 [802,646,447,416,557]
+ CRUSH rule 1 x 314 [654,974,229,511,562]
+ CRUSH rule 1 x 315 [767,227,28,740,828]
+ CRUSH rule 1 x 316 [778,83,733,359,858]
+ CRUSH rule 1 x 317 [184,418,642,986,939]
+ CRUSH rule 1 x 318 [525,410,500,543,212]
+ CRUSH rule 1 x 319 [476,724,569,382,409]
+ CRUSH rule 1 x 320 [149,610,697,296,818]
+ CRUSH rule 1 x 321 [710,79,667,671,234]
+ CRUSH rule 1 x 322 [175,275,323,333,744]
+ CRUSH rule 1 x 323 [819,604,638,792,316]
+ CRUSH rule 1 x 324 [16,745,511,439,272]
+ CRUSH rule 1 x 325 [486,400,872,873,251]
+ CRUSH rule 1 x 326 [613,765,207,19,359]
+ CRUSH rule 1 x 327 [125,289,738,408,456]
+ CRUSH rule 1 x 328 [807,383,476,583,645]
+ CRUSH rule 1 x 329 [588,938,599,432,446]
+ CRUSH rule 1 x 330 [932,644,41,611,209]
+ CRUSH rule 1 x 331 [341,953,950,537,578]
+ CRUSH rule 1 x 332 [153,726,459,950,466]
+ CRUSH rule 1 x 333 [745,845,853,860,52]
+ CRUSH rule 1 x 334 [614,751,807,58,396]
+ CRUSH rule 1 x 335 [518,721,221,283,454]
+ CRUSH rule 1 x 336 [389,424,77,309,5]
+ CRUSH rule 1 x 337 [753,508,765,720,221]
+ CRUSH rule 1 x 338 [128,810,490,753,406]
+ CRUSH rule 1 x 339 [430,308,58,751,856]
+ CRUSH rule 1 x 340 [541,44,630,231,289]
+ CRUSH rule 1 x 341 [402,26,631,439,165]
+ CRUSH rule 1 x 342 [982,57,992,461,131]
+ CRUSH rule 1 x 343 [833,412,572,732,107]
+ CRUSH rule 1 x 344 [784,533,792,41,642]
+ CRUSH rule 1 x 345 [546,300,304,691,763]
+ CRUSH rule 1 x 346 [302,420,428,891,357]
+ CRUSH rule 1 x 347 [488,778,101,217,366]
+ CRUSH rule 1 x 348 [903,744,937,718,85]
+ CRUSH rule 1 x 349 [471,547,582,306,600]
+ CRUSH rule 1 x 350 [348,221,823,335,383]
+ CRUSH rule 1 x 351 [961,582,705,346,361]
+ CRUSH rule 1 x 352 [728,137,461,298,36]
+ CRUSH rule 1 x 353 [904,202,184,447,58]
+ CRUSH rule 1 x 354 [345,226,319,256,544]
+ CRUSH rule 1 x 355 [50,430,175,43,187]
+ CRUSH rule 1 x 356 [87,185,55,423,829]
+ CRUSH rule 1 x 357 [762,459,921,473,182]
+ CRUSH rule 1 x 358 [908,25,280,6,808]
+ CRUSH rule 1 x 359 [484,15,132,121,394]
+ CRUSH rule 1 x 360 [173,378,337,702,145]
+ CRUSH rule 1 x 361 [404,577,115,25,56]
+ CRUSH rule 1 x 362 [403,1,422,945,132]
+ CRUSH rule 1 x 363 [639,911,510,162,418]
+ CRUSH rule 1 x 364 [752,689,610,990,665]
+ CRUSH rule 1 x 365 [956,999,212,230,624]
+ CRUSH rule 1 x 366 [860,925,924,763,687]
+ CRUSH rule 1 x 367 [205,609,647,665,969]
+ CRUSH rule 1 x 368 [301,284,810,169,78]
+ CRUSH rule 1 x 369 [452,658,339,217,674]
+ CRUSH rule 1 x 370 [11,467,695,989,394]
+ CRUSH rule 1 x 371 [124,487,55,514,313]
+ CRUSH rule 1 x 372 [253,48,979,846,207]
+ CRUSH rule 1 x 373 [715,605,775,748,227]
+ CRUSH rule 1 x 374 [191,887,920,571,223]
+ CRUSH rule 1 x 375 [711,385,651,665,15]
+ CRUSH rule 1 x 376 [597,818,49,458,415]
+ CRUSH rule 1 x 377 [294,256,933,771,184]
+ CRUSH rule 1 x 378 [34,151,681,707,552]
+ CRUSH rule 1 x 379 [869,136,315,378,813]
+ CRUSH rule 1 x 380 [294,97,575,791,690]
+ CRUSH rule 1 x 381 [119,710,219,827,328]
+ CRUSH rule 1 x 382 [69,631,508,706,697]
+ CRUSH rule 1 x 383 [922,588,589,925,471]
+ CRUSH rule 1 x 384 [221,945,671,117,857]
+ CRUSH rule 1 x 385 [561,737,953,723,658]
+ CRUSH rule 1 x 386 [335,442,788,696,507]
+ CRUSH rule 1 x 387 [514,43,353,88,100]
+ CRUSH rule 1 x 388 [587,89,157,996,915]
+ CRUSH rule 1 x 389 [109,641,255,466,372]
+ CRUSH rule 1 x 390 [925,149,421,489,599]
+ CRUSH rule 1 x 391 [267,87,387,527,768]
+ CRUSH rule 1 x 392 [382,485,370,849,936]
+ CRUSH rule 1 x 393 [425,721,221,753,268]
+ CRUSH rule 1 x 394 [898,18,38,793,173]
+ CRUSH rule 1 x 395 [806,876,269,679,32]
+ CRUSH rule 1 x 396 [790,970,437,449,875]
+ CRUSH rule 1 x 397 [136,363,507,613,11]
+ CRUSH rule 1 x 398 [914,116,558,258,722]
+ CRUSH rule 1 x 399 [261,94,299,202,174]
+ CRUSH rule 1 x 400 [661,197,338,461,977]
+ CRUSH rule 1 x 401 [953,979,287,803,41]
+ CRUSH rule 1 x 402 [738,819,618,522,667]
+ CRUSH rule 1 x 403 [573,238,425,546,130]
+ CRUSH rule 1 x 404 [526,848,790,253,922]
+ CRUSH rule 1 x 405 [582,505,330,334,201]
+ CRUSH rule 1 x 406 [768,324,493,60,186]
+ CRUSH rule 1 x 407 [260,951,437,587,692]
+ CRUSH rule 1 x 408 [657,81,770,734,830]
+ CRUSH rule 1 x 409 [498,89,182,423,672]
+ CRUSH rule 1 x 410 [28,793,737,352,166]
+ CRUSH rule 1 x 411 [684,992,60,659,769]
+ CRUSH rule 1 x 412 [261,958,699,950,165]
+ CRUSH rule 1 x 413 [891,835,297,441,384]
+ CRUSH rule 1 x 414 [127,459,119,965,662]
+ CRUSH rule 1 x 415 [272,540,631,328,609]
+ CRUSH rule 1 x 416 [739,617,115,530,339]
+ CRUSH rule 1 x 417 [106,209,157,878,117]
+ CRUSH rule 1 x 418 [525,441,147,390,320]
+ CRUSH rule 1 x 419 [603,673,615,465,266]
+ CRUSH rule 1 x 420 [988,213,251,226,209]
+ CRUSH rule 1 x 421 [761,521,748,368,923]
+ CRUSH rule 1 x 422 [317,160,924,548,198]
+ CRUSH rule 1 x 423 [137,807,168,472,619]
+ CRUSH rule 1 x 424 [920,37,146,263,598]
+ CRUSH rule 1 x 425 [277,693,285,221,478]
+ CRUSH rule 1 x 426 [485,936,407,854,726]
+ CRUSH rule 1 x 427 [242,515,9,564,174]
+ CRUSH rule 1 x 428 [632,635,26,473,494]
+ CRUSH rule 1 x 429 [641,73,465,127,171]
+ CRUSH rule 1 x 430 [626,585,6,387,881]
+ CRUSH rule 1 x 431 [697,76,753,570,964]
+ CRUSH rule 1 x 432 [590,526,306,283,656]
+ CRUSH rule 1 x 433 [284,387,149,817,886]
+ CRUSH rule 1 x 434 [538,985,79,953,770]
+ CRUSH rule 1 x 435 [30,318,593,635,975]
+ CRUSH rule 1 x 436 [164,919,851,693,0]
+ CRUSH rule 1 x 437 [322,212,163,606,302]
+ CRUSH rule 1 x 438 [142,392,85,594,376]
+ CRUSH rule 1 x 439 [119,370,68,443,997]
+ CRUSH rule 1 x 440 [333,403,187,863,475]
+ CRUSH rule 1 x 441 [477,727,906,145,429]
+ CRUSH rule 1 x 442 [274,590,933,244,434]
+ CRUSH rule 1 x 443 [983,748,574,718,700]
+ CRUSH rule 1 x 444 [536,509,431,146,170]
+ CRUSH rule 1 x 445 [485,554,528,209,964]
+ CRUSH rule 1 x 446 [345,634,42,294,711]
+ CRUSH rule 1 x 447 [61,845,767,600,321]
+ CRUSH rule 1 x 448 [333,232,292,846,364]
+ CRUSH rule 1 x 449 [680,16,484,670,851]
+ CRUSH rule 1 x 450 [235,214,79,423,96]
+ CRUSH rule 1 x 451 [961,468,333,640,823]
+ CRUSH rule 1 x 452 [525,479,153,528,570]
+ CRUSH rule 1 x 453 [138,466,302,86,249]
+ CRUSH rule 1 x 454 [137,625,215,402,389]
+ CRUSH rule 1 x 455 [173,150,997,16,846]
+ CRUSH rule 1 x 456 [235,226,238,258,347]
+ CRUSH rule 1 x 457 [450,577,253,413,717]
+ CRUSH rule 1 x 458 [195,537,91,814,351]
+ CRUSH rule 1 x 459 [381,555,312,573,915]
+ CRUSH rule 1 x 460 [972,730,534,678,756]
+ CRUSH rule 1 x 461 [506,279,142,830,784]
+ CRUSH rule 1 x 462 [692,959,578,57,983]
+ CRUSH rule 1 x 463 [788,667,949,550,685]
+ CRUSH rule 1 x 464 [133,122,588,999,270]
+ CRUSH rule 1 x 465 [971,190,230,777,452]
+ CRUSH rule 1 x 466 [394,576,148,157,103]
+ CRUSH rule 1 x 467 [517,28,366,362,984]
+ CRUSH rule 1 x 468 [829,143,874,225,162]
+ CRUSH rule 1 x 469 [987,936,106,725,633]
+ CRUSH rule 1 x 470 [107,982,56,889,67]
+ CRUSH rule 1 x 471 [181,897,629,860,307]
+ CRUSH rule 1 x 472 [547,512,172,24,705]
+ CRUSH rule 1 x 473 [760,997,824,905,888]
+ CRUSH rule 1 x 474 [787,418,743,628,272]
+ CRUSH rule 1 x 475 [662,312,253,617,105]
+ CRUSH rule 1 x 476 [110,495,185,508,961]
+ CRUSH rule 1 x 477 [393,954,834,132,841]
+ CRUSH rule 1 x 478 [246,483,480,644,985]
+ CRUSH rule 1 x 479 [70,929,697,931,744]
+ CRUSH rule 1 x 480 [753,119,961,607,317]
+ CRUSH rule 1 x 481 [470,429,677,242,574]
+ CRUSH rule 1 x 482 [451,566,961,675,354]
+ CRUSH rule 1 x 483 [816,72,371,278,635]
+ CRUSH rule 1 x 484 [540,454,389,31,654]
+ CRUSH rule 1 x 485 [74,582,624,684,566]
+ CRUSH rule 1 x 486 [958,595,199,763,715]
+ CRUSH rule 1 x 487 [228,302,804,833,876]
+ CRUSH rule 1 x 488 [180,529,722,956,353]
+ CRUSH rule 1 x 489 [47,617,812,187,291]
+ CRUSH rule 1 x 490 [905,822,479,124,750]
+ CRUSH rule 1 x 491 [892,370,609,998,433]
+ CRUSH rule 1 x 492 [588,959,127,948,505]
+ CRUSH rule 1 x 493 [353,461,593,291,301]
+ CRUSH rule 1 x 494 [378,848,443,368,507]
+ CRUSH rule 1 x 495 [845,653,768,234,405]
+ CRUSH rule 1 x 496 [13,988,0,691,389]
+ CRUSH rule 1 x 497 [796,877,788,394,648]
+ CRUSH rule 1 x 498 [412,337,270,705,511]
+ CRUSH rule 1 x 499 [330,695,8,74,618]
+ CRUSH rule 1 x 500 [820,272,547,765,755]
+ CRUSH rule 1 x 501 [110,44,132,442,294]
+ CRUSH rule 1 x 502 [336,595,650,274,993]
+ CRUSH rule 1 x 503 [922,211,157,722,502]
+ CRUSH rule 1 x 504 [483,52,122,432,778]
+ CRUSH rule 1 x 505 [482,598,224,279,480]
+ CRUSH rule 1 x 506 [493,123,43,856,936]
+ CRUSH rule 1 x 507 [12,598,264,422,416]
+ CRUSH rule 1 x 508 [227,157,611,301,223]
+ CRUSH rule 1 x 509 [807,242,363,122,582]
+ CRUSH rule 1 x 510 [134,437,227,75,313]
+ CRUSH rule 1 x 511 [212,54,83,799,457]
+ CRUSH rule 1 x 512 [236,630,758,752,361]
+ CRUSH rule 1 x 513 [994,693,644,938,846]
+ CRUSH rule 1 x 514 [45,508,831,19,817]
+ CRUSH rule 1 x 515 [504,138,480,272,530]
+ CRUSH rule 1 x 516 [285,409,136,570,841]
+ CRUSH rule 1 x 517 [300,232,23,906,438]
+ CRUSH rule 1 x 518 [397,674,98,898,967]
+ CRUSH rule 1 x 519 [86,750,772,913,101]
+ CRUSH rule 1 x 520 [900,833,614,130,261]
+ CRUSH rule 1 x 521 [31,47,236,751,911]
+ CRUSH rule 1 x 522 [390,16,280,144,291]
+ CRUSH rule 1 x 523 [618,308,424,590,300]
+ CRUSH rule 1 x 524 [635,189,687,963,601]
+ CRUSH rule 1 x 525 [311,916,699,262,775]
+ CRUSH rule 1 x 526 [48,738,227,718,244]
+ CRUSH rule 1 x 527 [202,851,889,216,763]
+ CRUSH rule 1 x 528 [565,827,590,273,918]
+ CRUSH rule 1 x 529 [934,864,241,43,466]
+ CRUSH rule 1 x 530 [502,934,298,670,986]
+ CRUSH rule 1 x 531 [681,627,942,487,288]
+ CRUSH rule 1 x 532 [422,6,147,205,861]
+ CRUSH rule 1 x 533 [863,68,364,983,247]
+ CRUSH rule 1 x 534 [962,931,775,172,663]
+ CRUSH rule 1 x 535 [89,565,397,693,839]
+ CRUSH rule 1 x 536 [499,351,760,458,918]
+ CRUSH rule 1 x 537 [676,547,787,311,867]
+ CRUSH rule 1 x 538 [58,644,571,649,941]
+ CRUSH rule 1 x 539 [837,953,457,711,458]
+ CRUSH rule 1 x 540 [831,50,132,213,197]
+ CRUSH rule 1 x 541 [582,757,121,525,532]
+ CRUSH rule 1 x 542 [472,132,790,997,948]
+ CRUSH rule 1 x 543 [382,272,797,330,315]
+ CRUSH rule 1 x 544 [947,930,496,883,509]
+ CRUSH rule 1 x 545 [425,570,305,77,821]
+ CRUSH rule 1 x 546 [18,65,529,437,343]
+ CRUSH rule 1 x 547 [445,715,600,472,213]
+ CRUSH rule 1 x 548 [367,569,980,167,627]
+ CRUSH rule 1 x 549 [125,715,671,817,285]
+ CRUSH rule 1 x 550 [425,599,744,199,923]
+ CRUSH rule 1 x 551 [44,1,528,922,944]
+ CRUSH rule 1 x 552 [246,104,68,239,123]
+ CRUSH rule 1 x 553 [71,703,615,28,593]
+ CRUSH rule 1 x 554 [207,124,217,166,525]
+ CRUSH rule 1 x 555 [570,28,317,420,931]
+ CRUSH rule 1 x 556 [674,152,421,79,215]
+ CRUSH rule 1 x 557 [347,817,191,391,741]
+ CRUSH rule 1 x 558 [627,426,369,692,815]
+ CRUSH rule 1 x 559 [940,630,924,242,224]
+ CRUSH rule 1 x 560 [295,903,541,29,245]
+ CRUSH rule 1 x 561 [506,682,384,637,878]
+ CRUSH rule 1 x 562 [718,529,87,729,842]
+ CRUSH rule 1 x 563 [552,332,747,206,274]
+ CRUSH rule 1 x 564 [835,769,736,486,630]
+ CRUSH rule 1 x 565 [8,167,539,182,607]
+ CRUSH rule 1 x 566 [600,481,301,263,90]
+ CRUSH rule 1 x 567 [999,994,509,899,947]
+ CRUSH rule 1 x 568 [252,431,157,62,601]
+ CRUSH rule 1 x 569 [643,218,943,455,83]
+ CRUSH rule 1 x 570 [617,635,765,422,250]
+ CRUSH rule 1 x 571 [757,80,59,98,328]
+ CRUSH rule 1 x 572 [299,348,575,889,943]
+ CRUSH rule 1 x 573 [25,505,270,167,58]
+ CRUSH rule 1 x 574 [215,431,624,177,628]
+ CRUSH rule 1 x 575 [225,252,611,546,32]
+ CRUSH rule 1 x 576 [627,94,159,857,430]
+ CRUSH rule 1 x 577 [237,809,778,636,61]
+ CRUSH rule 1 x 578 [885,313,120,344,771]
+ CRUSH rule 1 x 579 [924,575,787,831,47]
+ CRUSH rule 1 x 580 [718,51,766,121,118]
+ CRUSH rule 1 x 581 [219,807,129,571,856]
+ CRUSH rule 1 x 582 [893,701,598,863,285]
+ CRUSH rule 1 x 583 [246,930,964,170,993]
+ CRUSH rule 1 x 584 [336,432,680,175,495]
+ CRUSH rule 1 x 585 [324,999,397,485,457]
+ CRUSH rule 1 x 586 [558,230,976,541,816]
+ CRUSH rule 1 x 587 [985,830,597,21,308]
+ CRUSH rule 1 x 588 [211,544,57,134,162]
+ CRUSH rule 1 x 589 [129,21,112,190,885]
+ CRUSH rule 1 x 590 [467,969,652,593,287]
+ CRUSH rule 1 x 591 [758,514,316,164,35]
+ CRUSH rule 1 x 592 [525,253,190,443,315]
+ CRUSH rule 1 x 593 [601,885,339,152,297]
+ CRUSH rule 1 x 594 [227,60,450,30,717]
+ CRUSH rule 1 x 595 [720,854,496,912,80]
+ CRUSH rule 1 x 596 [751,195,997,77,261]
+ CRUSH rule 1 x 597 [129,574,714,8,789]
+ CRUSH rule 1 x 598 [679,207,604,396,841]
+ CRUSH rule 1 x 599 [668,315,683,349,681]
+ CRUSH rule 1 x 600 [143,396,464,444,59]
+ CRUSH rule 1 x 601 [326,573,873,902,136]
+ CRUSH rule 1 x 602 [860,281,875,535,672]
+ CRUSH rule 1 x 603 [709,328,445,349,190]
+ CRUSH rule 1 x 604 [571,62,814,95,866]
+ CRUSH rule 1 x 605 [252,739,860,27,313]
+ CRUSH rule 1 x 606 [339,236,759,842,67]
+ CRUSH rule 1 x 607 [590,248,759,868,433]
+ CRUSH rule 1 x 608 [145,635,309,467,875]
+ CRUSH rule 1 x 609 [973,547,223,79,762]
+ CRUSH rule 1 x 610 [435,816,961,983,255]
+ CRUSH rule 1 x 611 [559,283,422,584,176]
+ CRUSH rule 1 x 612 [273,149,123,576,911]
+ CRUSH rule 1 x 613 [828,614,642,674,33]
+ CRUSH rule 1 x 614 [478,748,393,34,171]
+ CRUSH rule 1 x 615 [392,155,144,326,626]
+ CRUSH rule 1 x 616 [778,637,452,248,15]
+ CRUSH rule 1 x 617 [622,713,996,833,611]
+ CRUSH rule 1 x 618 [149,877,270,329,180]
+ CRUSH rule 1 x 619 [604,163,656,409,322]
+ CRUSH rule 1 x 620 [181,23,409,198,64]
+ CRUSH rule 1 x 621 [735,902,386,237,939]
+ CRUSH rule 1 x 622 [661,824,717,568,858]
+ CRUSH rule 1 x 623 [142,121,643,61,695]
+ CRUSH rule 1 x 624 [360,716,420,398,49]
+ CRUSH rule 1 x 625 [541,167,385,1,601]
+ CRUSH rule 1 x 626 [364,431,610,363,535]
+ CRUSH rule 1 x 627 [458,137,557,410,287]
+ CRUSH rule 1 x 628 [250,350,556,497,821]
+ CRUSH rule 1 x 629 [928,160,710,572,365]
+ CRUSH rule 1 x 630 [243,19,918,556,601]
+ CRUSH rule 1 x 631 [438,221,574,676,797]
+ CRUSH rule 1 x 632 [797,368,247,5,32]
+ CRUSH rule 1 x 633 [993,749,525,485,27]
+ CRUSH rule 1 x 634 [239,351,633,299,651]
+ CRUSH rule 1 x 635 [640,965,25,961,306]
+ CRUSH rule 1 x 636 [173,290,297,991,937]
+ CRUSH rule 1 x 637 [0,918,98,108,111]
+ CRUSH rule 1 x 638 [702,235,424,900,983]
+ CRUSH rule 1 x 639 [475,687,31,785,918]
+ CRUSH rule 1 x 640 [31,664,399,677,123]
+ CRUSH rule 1 x 641 [296,473,108,963,341]
+ CRUSH rule 1 x 642 [894,273,427,606,677]
+ CRUSH rule 1 x 643 [117,111,732,191,114]
+ CRUSH rule 1 x 644 [438,336,327,512,599]
+ CRUSH rule 1 x 645 [982,702,351,573,907]
+ CRUSH rule 1 x 646 [334,804,146,842,697]
+ CRUSH rule 1 x 647 [933,787,185,334,752]
+ CRUSH rule 1 x 648 [22,444,400,862,207]
+ CRUSH rule 1 x 649 [503,229,213,460,639]
+ CRUSH rule 1 x 650 [328,659,420,443,739]
+ CRUSH rule 1 x 651 [3,880,823,123,378]
+ CRUSH rule 1 x 652 [495,977,563,733,92]
+ CRUSH rule 1 x 653 [185,718,804,280,975]
+ CRUSH rule 1 x 654 [130,528,380,81,906]
+ CRUSH rule 1 x 655 [560,872,454,504,319]
+ CRUSH rule 1 x 656 [219,885,178,981,863]
+ CRUSH rule 1 x 657 [233,684,813,490,208]
+ CRUSH rule 1 x 658 [778,6,756,380,750]
+ CRUSH rule 1 x 659 [240,663,306,540,789]
+ CRUSH rule 1 x 660 [244,855,196,147,678]
+ CRUSH rule 1 x 661 [184,270,128,398,910]
+ CRUSH rule 1 x 662 [65,883,921,438,79]
+ CRUSH rule 1 x 663 [323,721,594,812,43]
+ CRUSH rule 1 x 664 [865,113,512,51,427]
+ CRUSH rule 1 x 665 [420,850,591,475,202]
+ CRUSH rule 1 x 666 [319,767,246,3,369]
+ CRUSH rule 1 x 667 [875,39,343,100,829]
+ CRUSH rule 1 x 668 [331,122,263,599,355]
+ CRUSH rule 1 x 669 [915,521,402,747,673]
+ CRUSH rule 1 x 670 [845,659,943,447,401]
+ CRUSH rule 1 x 671 [108,634,527,363,856]
+ CRUSH rule 1 x 672 [578,216,110,589,302]
+ CRUSH rule 1 x 673 [442,74,579,797,622]
+ CRUSH rule 1 x 674 [588,364,281,308,645]
+ CRUSH rule 1 x 675 [489,698,744,671,870]
+ CRUSH rule 1 x 676 [928,911,40,180,722]
+ CRUSH rule 1 x 677 [399,269,692,131,615]
+ CRUSH rule 1 x 678 [546,752,544,155,5]
+ CRUSH rule 1 x 679 [988,25,275,433,628]
+ CRUSH rule 1 x 680 [335,963,382,486,749]
+ CRUSH rule 1 x 681 [690,462,623,466,49]
+ CRUSH rule 1 x 682 [196,588,154,257,807]
+ CRUSH rule 1 x 683 [627,25,421,160,873]
+ CRUSH rule 1 x 684 [38,804,592,158,991]
+ CRUSH rule 1 x 685 [841,368,548,362,166]
+ CRUSH rule 1 x 686 [336,287,525,440,166]
+ CRUSH rule 1 x 687 [20,682,924,653,356]
+ CRUSH rule 1 x 688 [463,371,780,556,385]
+ CRUSH rule 1 x 689 [569,250,78,816,847]
+ CRUSH rule 1 x 690 [551,144,587,263,378]
+ CRUSH rule 1 x 691 [766,464,446,533,449]
+ CRUSH rule 1 x 692 [739,634,18,245,624]
+ CRUSH rule 1 x 693 [339,297,118,330,817]
+ CRUSH rule 1 x 694 [405,26,830,181,533]
+ CRUSH rule 1 x 695 [622,576,597,535,600]
+ CRUSH rule 1 x 696 [558,902,689,13,715]
+ CRUSH rule 1 x 697 [818,222,406,691,427]
+ CRUSH rule 1 x 698 [178,48,402,233,841]
+ CRUSH rule 1 x 699 [450,244,180,919,100]
+ CRUSH rule 1 x 700 [502,771,987,706,416]
+ CRUSH rule 1 x 701 [4,612,782,216,853]
+ CRUSH rule 1 x 702 [177,630,232,923,281]
+ CRUSH rule 1 x 703 [354,178,389,393,778]
+ CRUSH rule 1 x 704 [646,601,156,171,603]
+ CRUSH rule 1 x 705 [921,401,890,265,244]
+ CRUSH rule 1 x 706 [652,877,562,452,26]
+ CRUSH rule 1 x 707 [345,745,67,716,789]
+ CRUSH rule 1 x 708 [333,607,180,469,170]
+ CRUSH rule 1 x 709 [45,187,302,115,896]
+ CRUSH rule 1 x 710 [94,855,43,199,18]
+ CRUSH rule 1 x 711 [227,653,731,150,452]
+ CRUSH rule 1 x 712 [398,953,136,870,181]
+ CRUSH rule 1 x 713 [116,800,503,662,635]
+ CRUSH rule 1 x 714 [111,629,866,709,902]
+ CRUSH rule 1 x 715 [531,291,486,382,192]
+ CRUSH rule 1 x 716 [169,541,291,42,343]
+ CRUSH rule 1 x 717 [417,446,994,894,239]
+ CRUSH rule 1 x 718 [992,383,298,844,377]
+ CRUSH rule 1 x 719 [936,674,324,759,194]
+ CRUSH rule 1 x 720 [370,188,174,464,644]
+ CRUSH rule 1 x 721 [320,859,278,259,170]
+ CRUSH rule 1 x 722 [7,2,673,129,96]
+ CRUSH rule 1 x 723 [270,553,831,662,38]
+ CRUSH rule 1 x 724 [666,822,708,895,633]
+ CRUSH rule 1 x 725 [794,406,875,459,981]
+ CRUSH rule 1 x 726 [420,556,341,292,240]
+ CRUSH rule 1 x 727 [561,461,129,635,965]
+ CRUSH rule 1 x 728 [951,330,196,756,589]
+ CRUSH rule 1 x 729 [656,644,436,591,27]
+ CRUSH rule 1 x 730 [3,558,629,184,50]
+ CRUSH rule 1 x 731 [852,89,75,735,713]
+ CRUSH rule 1 x 732 [983,840,869,976,697]
+ CRUSH rule 1 x 733 [285,396,388,122,387]
+ CRUSH rule 1 x 734 [125,510,402,640,676]
+ CRUSH rule 1 x 735 [417,773,686,504,459]
+ CRUSH rule 1 x 736 [749,396,632,550,779]
+ CRUSH rule 1 x 737 [644,991,946,135,448]
+ CRUSH rule 1 x 738 [449,683,290,220,245]
+ CRUSH rule 1 x 739 [341,220,641,454,740]
+ CRUSH rule 1 x 740 [874,524,674,650,472]
+ CRUSH rule 1 x 741 [189,472,712,798,715]
+ CRUSH rule 1 x 742 [912,581,114,145,730]
+ CRUSH rule 1 x 743 [654,914,425,441,763]
+ CRUSH rule 1 x 744 [725,295,579,377,162]
+ CRUSH rule 1 x 745 [787,858,850,506,612]
+ CRUSH rule 1 x 746 [757,848,704,30,47]
+ CRUSH rule 1 x 747 [700,81,867,681,801]
+ CRUSH rule 1 x 748 [557,436,238,664,293]
+ CRUSH rule 1 x 749 [772,622,337,42,156]
+ CRUSH rule 1 x 750 [946,97,376,677,316]
+ CRUSH rule 1 x 751 [996,618,343,911,83]
+ CRUSH rule 1 x 752 [746,887,695,868,610]
+ CRUSH rule 1 x 753 [741,14,463,479,172]
+ CRUSH rule 1 x 754 [648,349,333,355,65]
+ CRUSH rule 1 x 755 [157,460,466,187,959]
+ CRUSH rule 1 x 756 [416,97,197,497,227]
+ CRUSH rule 1 x 757 [599,839,776,410,256]
+ CRUSH rule 1 x 758 [994,218,620,256,361]
+ CRUSH rule 1 x 759 [959,682,514,745,100]
+ CRUSH rule 1 x 760 [518,943,215,83,706]
+ CRUSH rule 1 x 761 [285,849,420,324,987]
+ CRUSH rule 1 x 762 [591,313,41,335,110]
+ CRUSH rule 1 x 763 [908,411,200,740,292]
+ CRUSH rule 1 x 764 [787,234,894,485,883]
+ CRUSH rule 1 x 765 [327,921,882,393,444]
+ CRUSH rule 1 x 766 [84,161,878,704,416]
+ CRUSH rule 1 x 767 [370,895,702,701,890]
+ CRUSH rule 1 x 768 [826,760,879,864,460]
+ CRUSH rule 1 x 769 [67,768,663,735,814]
+ CRUSH rule 1 x 770 [593,909,482,259,5]
+ CRUSH rule 1 x 771 [309,935,121,578,937]
+ CRUSH rule 1 x 772 [12,125,797,301,348]
+ CRUSH rule 1 x 773 [253,466,820,549,591]
+ CRUSH rule 1 x 774 [164,390,705,109,881]
+ CRUSH rule 1 x 775 [703,47,43,973,643]
+ CRUSH rule 1 x 776 [728,231,80,916,2]
+ CRUSH rule 1 x 777 [981,621,568,729,869]
+ CRUSH rule 1 x 778 [411,456,544,597,789]
+ CRUSH rule 1 x 779 [346,121,519,921,587]
+ CRUSH rule 1 x 780 [476,39,288,381,303]
+ CRUSH rule 1 x 781 [10,130,585,844,729]
+ CRUSH rule 1 x 782 [462,246,581,902,623]
+ CRUSH rule 1 x 783 [580,373,153,775,668]
+ CRUSH rule 1 x 784 [413,113,978,990,994]
+ CRUSH rule 1 x 785 [341,856,332,354,59]
+ CRUSH rule 1 x 786 [411,140,313,393,215]
+ CRUSH rule 1 x 787 [605,522,211,813,636]
+ CRUSH rule 1 x 788 [226,545,35,142,726]
+ CRUSH rule 1 x 789 [545,320,414,702,731]
+ CRUSH rule 1 x 790 [414,748,816,327,130]
+ CRUSH rule 1 x 791 [660,906,406,697,916]
+ CRUSH rule 1 x 792 [287,392,514,204,75]
+ CRUSH rule 1 x 793 [631,133,850,713,720]
+ CRUSH rule 1 x 794 [931,517,543,210,963]
+ CRUSH rule 1 x 795 [551,962,477,948,425]
+ CRUSH rule 1 x 796 [814,4,95,27,368]
+ CRUSH rule 1 x 797 [64,201,299,734,605]
+ CRUSH rule 1 x 798 [422,530,114,431,565]
+ CRUSH rule 1 x 799 [824,32,679,562,266]
+ CRUSH rule 1 x 800 [862,623,489,637,861]
+ CRUSH rule 1 x 801 [145,550,329,324,734]
+ CRUSH rule 1 x 802 [570,19,847,308,387]
+ CRUSH rule 1 x 803 [151,812,662,358,880]
+ CRUSH rule 1 x 804 [467,93,264,863,176]
+ CRUSH rule 1 x 805 [621,223,938,809,591]
+ CRUSH rule 1 x 806 [898,957,805,430,499]
+ CRUSH rule 1 x 807 [354,531,422,159,921]
+ CRUSH rule 1 x 808 [7,96,76,897,446]
+ CRUSH rule 1 x 809 [70,734,719,56,687]
+ CRUSH rule 1 x 810 [701,18,972,327,771]
+ CRUSH rule 1 x 811 [248,547,103,728,901]
+ CRUSH rule 1 x 812 [230,576,821,566,993]
+ CRUSH rule 1 x 813 [805,114,683,629,742]
+ CRUSH rule 1 x 814 [54,619,973,741,497]
+ CRUSH rule 1 x 815 [679,412,613,132,969]
+ CRUSH rule 1 x 816 [919,448,826,414,36]
+ CRUSH rule 1 x 817 [765,830,436,521,332]
+ CRUSH rule 1 x 818 [415,566,644,687,692]
+ CRUSH rule 1 x 819 [721,319,865,750,546]
+ CRUSH rule 1 x 820 [218,301,333,190,686]
+ CRUSH rule 1 x 821 [185,795,680,953,329]
+ CRUSH rule 1 x 822 [356,261,54,522,900]
+ CRUSH rule 1 x 823 [220,281,549,456,64]
+ CRUSH rule 1 x 824 [292,809,887,74,776]
+ CRUSH rule 1 x 825 [949,778,101,311,110]
+ CRUSH rule 1 x 826 [767,818,833,927,356]
+ CRUSH rule 1 x 827 [631,83,406,635,657]
+ CRUSH rule 1 x 828 [288,986,445,26,414]
+ CRUSH rule 1 x 829 [990,667,915,694,974]
+ CRUSH rule 1 x 830 [152,571,778,505,685]
+ CRUSH rule 1 x 831 [814,563,630,97,582]
+ CRUSH rule 1 x 832 [235,641,616,110,979]
+ CRUSH rule 1 x 833 [657,565,922,140,825]
+ CRUSH rule 1 x 834 [907,231,644,13,617]
+ CRUSH rule 1 x 835 [784,262,771,264,612]
+ CRUSH rule 1 x 836 [951,158,366,710,43]
+ CRUSH rule 1 x 837 [556,498,334,633,895]
+ CRUSH rule 1 x 838 [329,274,964,547,119]
+ CRUSH rule 1 x 839 [568,209,939,364,658]
+ CRUSH rule 1 x 840 [45,579,842,70,655]
+ CRUSH rule 1 x 841 [652,702,24,605,152]
+ CRUSH rule 1 x 842 [629,984,314,895,408]
+ CRUSH rule 1 x 843 [799,690,688,648,151]
+ CRUSH rule 1 x 844 [694,600,534,700,569]
+ CRUSH rule 1 x 845 [332,30,179,93,951]
+ CRUSH rule 1 x 846 [452,251,712,719,404]
+ CRUSH rule 1 x 847 [399,681,847,739,13]
+ CRUSH rule 1 x 848 [303,138,440,346,547]
+ CRUSH rule 1 x 849 [666,346,708,873,64]
+ CRUSH rule 1 x 850 [644,511,345,844,545]
+ CRUSH rule 1 x 851 [527,546,737,425,100]
+ CRUSH rule 1 x 852 [31,809,94,618,156]
+ CRUSH rule 1 x 853 [483,330,869,184,46]
+ CRUSH rule 1 x 854 [697,953,968,143,502]
+ CRUSH rule 1 x 855 [837,996,239,621,32]
+ CRUSH rule 1 x 856 [712,40,547,430,195]
+ CRUSH rule 1 x 857 [77,984,576,551,568]
+ CRUSH rule 1 x 858 [412,384,841,465,572]
+ CRUSH rule 1 x 859 [173,760,26,300,87]
+ CRUSH rule 1 x 860 [776,429,328,917,658]
+ CRUSH rule 1 x 861 [705,405,477,50,73]
+ CRUSH rule 1 x 862 [809,44,788,938,964]
+ CRUSH rule 1 x 863 [349,496,963,178,675]
+ CRUSH rule 1 x 864 [717,858,101,239,992]
+ CRUSH rule 1 x 865 [857,603,586,262,550]
+ CRUSH rule 1 x 866 [394,304,71,96,642]
+ CRUSH rule 1 x 867 [640,773,663,974,261]
+ CRUSH rule 1 x 868 [613,950,712,663,16]
+ CRUSH rule 1 x 869 [973,889,524,22,671]
+ CRUSH rule 1 x 870 [505,35,386,498,348]
+ CRUSH rule 1 x 871 [239,264,262,773,781]
+ CRUSH rule 1 x 872 [21,767,456,748,783]
+ CRUSH rule 1 x 873 [954,666,980,264,435]
+ CRUSH rule 1 x 874 [54,510,947,1,500]
+ CRUSH rule 1 x 875 [809,418,452,462,88]
+ CRUSH rule 1 x 876 [483,457,61,248,523]
+ CRUSH rule 1 x 877 [542,531,952,939,710]
+ CRUSH rule 1 x 878 [217,674,857,644,678]
+ CRUSH rule 1 x 879 [999,475,134,250,319]
+ CRUSH rule 1 x 880 [678,573,935,385,570]
+ CRUSH rule 1 x 881 [394,835,789,802,587]
+ CRUSH rule 1 x 882 [467,382,353,56,979]
+ CRUSH rule 1 x 883 [802,744,237,337,50]
+ CRUSH rule 1 x 884 [653,660,638,700,31]
+ CRUSH rule 1 x 885 [898,704,307,445,879]
+ CRUSH rule 1 x 886 [434,357,938,641,737]
+ CRUSH rule 1 x 887 [297,226,711,428,370]
+ CRUSH rule 1 x 888 [863,324,443,213,902]
+ CRUSH rule 1 x 889 [105,102,308,163,947]
+ CRUSH rule 1 x 890 [550,248,606,704,615]
+ CRUSH rule 1 x 891 [575,928,880,891,826]
+ CRUSH rule 1 x 892 [259,862,133,271,292]
+ CRUSH rule 1 x 893 [902,880,543,542,37]
+ CRUSH rule 1 x 894 [180,169,916,43,945]
+ CRUSH rule 1 x 895 [725,849,182,129,177]
+ CRUSH rule 1 x 896 [951,34,874,537,969]
+ CRUSH rule 1 x 897 [810,352,73,939,943]
+ CRUSH rule 1 x 898 [979,433,719,411,787]
+ CRUSH rule 1 x 899 [685,668,534,932,399]
+ CRUSH rule 1 x 900 [530,978,41,894,941]
+ CRUSH rule 1 x 901 [740,107,336,175,574]
+ CRUSH rule 1 x 902 [800,743,693,310,67]
+ CRUSH rule 1 x 903 [230,267,842,266,550]
+ CRUSH rule 1 x 904 [346,949,460,973,696]
+ CRUSH rule 1 x 905 [530,397,619,958,576]
+ CRUSH rule 1 x 906 [80,426,138,672,73]
+ CRUSH rule 1 x 907 [365,968,475,297,296]
+ CRUSH rule 1 x 908 [204,832,742,809,862]
+ CRUSH rule 1 x 909 [883,989,146,959,366]
+ CRUSH rule 1 x 910 [549,593,249,853,792]
+ CRUSH rule 1 x 911 [325,847,352,214,851]
+ CRUSH rule 1 x 912 [874,888,582,796,557]
+ CRUSH rule 1 x 913 [331,463,342,574,989]
+ CRUSH rule 1 x 914 [836,468,601,732,607]
+ CRUSH rule 1 x 915 [245,228,100,661,799]
+ CRUSH rule 1 x 916 [77,967,364,435,27]
+ CRUSH rule 1 x 917 [239,60,866,221,772]
+ CRUSH rule 1 x 918 [988,115,922,80,201]
+ CRUSH rule 1 x 919 [783,139,696,1,848]
+ CRUSH rule 1 x 920 [623,408,685,953,974]
+ CRUSH rule 1 x 921 [105,799,144,90,399]
+ CRUSH rule 1 x 922 [887,505,652,348,514]
+ CRUSH rule 1 x 923 [223,318,552,458,743]
+ CRUSH rule 1 x 924 [25,778,366,333,163]
+ CRUSH rule 1 x 925 [912,601,297,682,770]
+ CRUSH rule 1 x 926 [968,133,158,144,814]
+ CRUSH rule 1 x 927 [277,724,214,988,690]
+ CRUSH rule 1 x 928 [554,203,658,789,298]
+ CRUSH rule 1 x 929 [761,802,367,528,758]
+ CRUSH rule 1 x 930 [814,61,788,736,660]
+ CRUSH rule 1 x 931 [29,193,61,41,343]
+ CRUSH rule 1 x 932 [446,198,862,534,168]
+ CRUSH rule 1 x 933 [352,742,216,321,525]
+ CRUSH rule 1 x 934 [730,2,332,631,613]
+ CRUSH rule 1 x 935 [731,23,736,79,361]
+ CRUSH rule 1 x 936 [322,975,20,904,827]
+ CRUSH rule 1 x 937 [822,221,841,161,723]
+ CRUSH rule 1 x 938 [557,850,66,630,499]
+ CRUSH rule 1 x 939 [150,11,971,371,124]
+ CRUSH rule 1 x 940 [638,398,169,616,333]
+ CRUSH rule 1 x 941 [730,342,929,577,451]
+ CRUSH rule 1 x 942 [62,292,166,814,587]
+ CRUSH rule 1 x 943 [165,314,519,548,41]
+ CRUSH rule 1 x 944 [199,625,766,176,194]
+ CRUSH rule 1 x 945 [946,999,699,303,38]
+ CRUSH rule 1 x 946 [595,93,852,142,503]
+ CRUSH rule 1 x 947 [800,582,356,93,716]
+ CRUSH rule 1 x 948 [132,551,139,920,87]
+ CRUSH rule 1 x 949 [792,920,466,380,97]
+ CRUSH rule 1 x 950 [111,345,176,543,879]
+ CRUSH rule 1 x 951 [414,619,648,655,364]
+ CRUSH rule 1 x 952 [775,469,500,356,287]
+ CRUSH rule 1 x 953 [349,1,5,251,168]
+ CRUSH rule 1 x 954 [570,940,410,249,929]
+ CRUSH rule 1 x 955 [729,774,823,800,7]
+ CRUSH rule 1 x 956 [519,141,575,625,738]
+ CRUSH rule 1 x 957 [242,709,611,97,760]
+ CRUSH rule 1 x 958 [84,217,227,253,246]
+ CRUSH rule 1 x 959 [270,413,918,789,703]
+ CRUSH rule 1 x 960 [458,192,307,279,920]
+ CRUSH rule 1 x 961 [981,388,777,546,359]
+ CRUSH rule 1 x 962 [623,834,277,134,729]
+ CRUSH rule 1 x 963 [291,167,714,468,109]
+ CRUSH rule 1 x 964 [28,156,788,127,598]
+ CRUSH rule 1 x 965 [675,557,290,517,840]
+ CRUSH rule 1 x 966 [836,306,946,283,642]
+ CRUSH rule 1 x 967 [966,386,735,837,392]
+ CRUSH rule 1 x 968 [864,756,690,121,328]
+ CRUSH rule 1 x 969 [729,625,480,769,512]
+ CRUSH rule 1 x 970 [800,362,646,582,309]
+ CRUSH rule 1 x 971 [737,381,153,684,298]
+ CRUSH rule 1 x 972 [952,245,720,884,334]
+ CRUSH rule 1 x 973 [356,455,579,857,832]
+ CRUSH rule 1 x 974 [545,758,586,596,405]
+ CRUSH rule 1 x 975 [336,191,202,146,720]
+ CRUSH rule 1 x 976 [446,208,757,620,252]
+ CRUSH rule 1 x 977 [202,896,196,956,763]
+ CRUSH rule 1 x 978 [612,324,996,225,418]
+ CRUSH rule 1 x 979 [843,457,675,650,958]
+ CRUSH rule 1 x 980 [60,914,881,626,850]
+ CRUSH rule 1 x 981 [702,749,937,153,724]
+ CRUSH rule 1 x 982 [298,928,738,167,99]
+ CRUSH rule 1 x 983 [723,572,395,358,900]
+ CRUSH rule 1 x 984 [723,864,804,935,846]
+ CRUSH rule 1 x 985 [945,459,868,211,524]
+ CRUSH rule 1 x 986 [772,664,535,169,297]
+ CRUSH rule 1 x 987 [88,324,312,843,661]
+ CRUSH rule 1 x 988 [522,927,131,996,351]
+ CRUSH rule 1 x 989 [578,332,208,605,975]
+ CRUSH rule 1 x 990 [638,228,414,311,738]
+ CRUSH rule 1 x 991 [530,221,451,422,879]
+ CRUSH rule 1 x 992 [925,705,275,81,234]
+ CRUSH rule 1 x 993 [991,301,43,469,830]
+ CRUSH rule 1 x 994 [276,51,868,683,843]
+ CRUSH rule 1 x 995 [288,836,753,790,758]
+ CRUSH rule 1 x 996 [887,983,252,686,470]
+ CRUSH rule 1 x 997 [110,924,386,79,705]
+ CRUSH rule 1 x 998 [435,830,485,853,926]
+ CRUSH rule 1 x 999 [876,738,357,913,723]
+ CRUSH rule 1 x 1000 [178,963,638,430,845]
+ CRUSH rule 1 x 1001 [99,519,66,759,583]
+ CRUSH rule 1 x 1002 [515,534,468,866,878]
+ CRUSH rule 1 x 1003 [104,611,937,698,94]
+ CRUSH rule 1 x 1004 [269,638,724,375,491]
+ CRUSH rule 1 x 1005 [369,223,309,409,822]
+ CRUSH rule 1 x 1006 [40,107,69,275,79]
+ CRUSH rule 1 x 1007 [978,111,416,758,454]
+ CRUSH rule 1 x 1008 [965,956,624,832,421]
+ CRUSH rule 1 x 1009 [598,476,356,695,919]
+ CRUSH rule 1 x 1010 [767,523,239,517,29]
+ CRUSH rule 1 x 1011 [289,871,207,576,347]
+ CRUSH rule 1 x 1012 [128,28,370,31,341]
+ CRUSH rule 1 x 1013 [979,765,660,812,666]
+ CRUSH rule 1 x 1014 [979,948,513,88,47]
+ CRUSH rule 1 x 1015 [277,790,396,672,542]
+ CRUSH rule 1 x 1016 [262,73,128,886,839]
+ CRUSH rule 1 x 1017 [150,269,61,499,832]
+ CRUSH rule 1 x 1018 [555,829,554,944,406]
+ CRUSH rule 1 x 1019 [513,356,265,446,65]
+ CRUSH rule 1 x 1020 [158,161,877,704,948]
+ CRUSH rule 1 x 1021 [915,998,957,285,546]
+ CRUSH rule 1 x 1022 [967,829,973,640,703]
+ CRUSH rule 1 x 1023 [488,257,614,859,325]
+ rule 1 (metadata) num_rep 5 result size == 5:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450,604,380]
+ CRUSH rule 1 x 1 [876,250,334,633,744,843]
+ CRUSH rule 1 x 2 [292,832,53,392,386,787]
+ CRUSH rule 1 x 3 [623,387,124,998,749,211]
+ CRUSH rule 1 x 4 [61,334,710,4,994,982]
+ CRUSH rule 1 x 5 [946,557,713,664,141,817]
+ CRUSH rule 1 x 6 [576,668,212,163,732,381]
+ CRUSH rule 1 x 7 [645,753,906,393,341,44]
+ CRUSH rule 1 x 8 [243,6,863,781,211,100]
+ CRUSH rule 1 x 9 [22,578,251,410,297,430]
+ CRUSH rule 1 x 10 [758,828,360,477,821,801]
+ CRUSH rule 1 x 11 [769,120,124,527,119,504]
+ CRUSH rule 1 x 12 [780,364,689,755,675,199]
+ CRUSH rule 1 x 13 [557,18,351,719,742,780]
+ CRUSH rule 1 x 14 [59,561,249,461,971,835]
+ CRUSH rule 1 x 15 [718,928,993,21,76,313]
+ CRUSH rule 1 x 16 [673,632,841,954,788,90]
+ CRUSH rule 1 x 17 [648,43,560,514,142,289]
+ CRUSH rule 1 x 18 [654,219,181,568,381,253]
+ CRUSH rule 1 x 19 [850,545,377,848,863,543]
+ CRUSH rule 1 x 20 [717,785,974,5,225,552]
+ CRUSH rule 1 x 21 [420,57,519,306,312,983]
+ CRUSH rule 1 x 22 [503,998,193,821,634,684]
+ CRUSH rule 1 x 23 [411,663,168,110,899,488]
+ CRUSH rule 1 x 24 [266,861,353,1,456,128]
+ CRUSH rule 1 x 25 [760,483,818,600,509,951]
+ CRUSH rule 1 x 26 [903,24,573,718,112,694]
+ CRUSH rule 1 x 27 [946,188,289,510,687,827]
+ CRUSH rule 1 x 28 [69,312,73,198,256,629]
+ CRUSH rule 1 x 29 [844,883,337,628,496,405]
+ CRUSH rule 1 x 30 [621,18,613,794,910,936]
+ CRUSH rule 1 x 31 [784,943,814,539,962,392]
+ CRUSH rule 1 x 32 [173,374,369,972,315,83]
+ CRUSH rule 1 x 33 [698,336,357,966,582,407]
+ CRUSH rule 1 x 34 [168,836,210,798,904,190]
+ CRUSH rule 1 x 35 [274,509,534,818,912,671]
+ CRUSH rule 1 x 36 [318,215,153,628,87,407]
+ CRUSH rule 1 x 37 [173,604,109,935,203,401]
+ CRUSH rule 1 x 38 [708,444,683,604,722,900]
+ CRUSH rule 1 x 39 [662,198,417,680,226,342]
+ CRUSH rule 1 x 40 [620,801,414,78,560,766]
+ CRUSH rule 1 x 41 [811,264,177,127,148,791]
+ CRUSH rule 1 x 42 [863,179,527,660,133,529]
+ CRUSH rule 1 x 43 [686,822,988,228,791,549]
+ CRUSH rule 1 x 44 [396,222,46,841,536,140]
+ CRUSH rule 1 x 45 [991,694,253,142,54,422]
+ CRUSH rule 1 x 46 [420,909,184,285,508,458]
+ CRUSH rule 1 x 47 [467,211,605,207,241,881]
+ CRUSH rule 1 x 48 [955,329,368,168,698,787]
+ CRUSH rule 1 x 49 [974,891,931,29,813,506]
+ CRUSH rule 1 x 50 [870,441,691,823,761,6]
+ CRUSH rule 1 x 51 [182,930,25,936,97,260]
+ CRUSH rule 1 x 52 [704,812,894,794,481,37]
+ CRUSH rule 1 x 53 [185,713,631,280,345,558]
+ CRUSH rule 1 x 54 [270,441,100,82,983,930]
+ CRUSH rule 1 x 55 [895,734,958,793,651,572]
+ CRUSH rule 1 x 56 [564,963,683,324,40,189]
+ CRUSH rule 1 x 57 [738,130,208,973,498,861]
+ CRUSH rule 1 x 58 [524,113,806,903,531,334]
+ CRUSH rule 1 x 59 [408,337,668,529,34,384]
+ CRUSH rule 1 x 60 [228,790,857,309,616,895]
+ CRUSH rule 1 x 61 [154,843,717,467,883,536]
+ CRUSH rule 1 x 62 [594,811,549,276,693,917]
+ CRUSH rule 1 x 63 [646,67,884,925,941,434]
+ CRUSH rule 1 x 64 [175,542,155,837,594,197]
+ CRUSH rule 1 x 65 [745,619,131,867,269,62]
+ CRUSH rule 1 x 66 [275,468,23,35,328,432]
+ CRUSH rule 1 x 67 [246,958,524,493,636,227]
+ CRUSH rule 1 x 68 [711,473,403,228,835,126]
+ CRUSH rule 1 x 69 [493,924,850,939,950,105]
+ CRUSH rule 1 x 70 [30,499,644,33,804,654]
+ CRUSH rule 1 x 71 [984,883,574,716,575,391]
+ CRUSH rule 1 x 72 [71,286,942,363,628,632]
+ CRUSH rule 1 x 73 [922,618,3,371,464,442]
+ CRUSH rule 1 x 74 [629,414,185,573,678,338]
+ CRUSH rule 1 x 75 [222,20,174,820,312,361]
+ CRUSH rule 1 x 76 [262,366,339,290,718,143]
+ CRUSH rule 1 x 77 [638,469,992,280,773,892]
+ CRUSH rule 1 x 78 [324,511,788,7,308,228]
+ CRUSH rule 1 x 79 [577,990,64,94,447,924]
+ CRUSH rule 1 x 80 [501,95,278,903,631,842]
+ CRUSH rule 1 x 81 [506,812,9,698,173,664]
+ CRUSH rule 1 x 82 [222,145,80,785,835,745]
+ CRUSH rule 1 x 83 [71,634,61,91,856,529]
+ CRUSH rule 1 x 84 [49,761,773,368,318,708]
+ CRUSH rule 1 x 85 [985,896,708,861,325,307]
+ CRUSH rule 1 x 86 [537,745,93,524,466,356]
+ CRUSH rule 1 x 87 [997,317,463,626,685,429]
+ CRUSH rule 1 x 88 [957,350,890,857,375,176]
+ CRUSH rule 1 x 89 [399,730,148,314,159,982]
+ CRUSH rule 1 x 90 [943,706,683,267,579,141]
+ CRUSH rule 1 x 91 [22,368,149,928,140,529]
+ CRUSH rule 1 x 92 [532,424,426,773,623,197]
+ CRUSH rule 1 x 93 [218,489,405,681,549,201]
+ CRUSH rule 1 x 94 [181,96,102,515,776,365]
+ CRUSH rule 1 x 95 [343,957,820,139,334,37]
+ CRUSH rule 1 x 96 [861,270,87,797,0,245]
+ CRUSH rule 1 x 97 [459,706,45,328,274,605]
+ CRUSH rule 1 x 98 [327,867,353,948,728,280]
+ CRUSH rule 1 x 99 [974,133,468,906,235,988]
+ CRUSH rule 1 x 100 [32,445,547,371,960,885]
+ CRUSH rule 1 x 101 [142,90,337,950,970,570]
+ CRUSH rule 1 x 102 [172,129,139,22,403,867]
+ CRUSH rule 1 x 103 [630,47,161,356,911,421]
+ CRUSH rule 1 x 104 [758,133,278,11,947,799]
+ CRUSH rule 1 x 105 [843,604,47,33,401,632]
+ CRUSH rule 1 x 106 [28,681,193,679,990,343]
+ CRUSH rule 1 x 107 [74,320,85,819,315,253]
+ CRUSH rule 1 x 108 [875,593,575,517,107,153]
+ CRUSH rule 1 x 109 [411,985,811,720,198,666]
+ CRUSH rule 1 x 110 [440,774,799,660,715,167]
+ CRUSH rule 1 x 111 [405,742,276,359,936,360]
+ CRUSH rule 1 x 112 [143,181,922,545,185,303]
+ CRUSH rule 1 x 113 [153,846,160,903,789,897]
+ CRUSH rule 1 x 114 [804,892,939,20,312,692]
+ CRUSH rule 1 x 115 [588,508,958,580,232,722]
+ CRUSH rule 1 x 116 [327,148,637,486,712,464]
+ CRUSH rule 1 x 117 [95,594,989,131,714,275]
+ CRUSH rule 1 x 118 [80,957,897,239,359,432]
+ CRUSH rule 1 x 119 [386,932,951,768,679,300]
+ CRUSH rule 1 x 120 [366,312,653,936,71,241]
+ CRUSH rule 1 x 121 [129,154,847,16,471,481]
+ CRUSH rule 1 x 122 [873,1,110,939,90,412]
+ CRUSH rule 1 x 123 [533,415,789,600,713,800]
+ CRUSH rule 1 x 124 [461,691,898,723,957,759]
+ CRUSH rule 1 x 125 [342,599,830,402,615,994]
+ CRUSH rule 1 x 126 [819,781,822,548,279,255]
+ CRUSH rule 1 x 127 [437,893,585,707,353,189]
+ CRUSH rule 1 x 128 [679,994,982,550,991,324]
+ CRUSH rule 1 x 129 [380,685,947,302,698,144]
+ CRUSH rule 1 x 130 [992,52,466,867,998,777]
+ CRUSH rule 1 x 131 [469,90,208,599,829,656]
+ CRUSH rule 1 x 132 [571,250,316,535,54,418]
+ CRUSH rule 1 x 133 [964,728,329,902,108,118]
+ CRUSH rule 1 x 134 [999,19,716,963,323,559]
+ CRUSH rule 1 x 135 [634,101,52,938,413,573]
+ CRUSH rule 1 x 136 [114,889,692,768,694,279]
+ CRUSH rule 1 x 137 [839,8,959,280,922,870]
+ CRUSH rule 1 x 138 [967,949,138,451,292,548]
+ CRUSH rule 1 x 139 [308,711,736,247,632,126]
+ CRUSH rule 1 x 140 [764,936,926,55,331,115]
+ CRUSH rule 1 x 141 [423,302,112,216,603,873]
+ CRUSH rule 1 x 142 [252,821,715,340,635,668]
+ CRUSH rule 1 x 143 [33,808,518,477,325,316]
+ CRUSH rule 1 x 144 [472,88,969,162,401,771]
+ CRUSH rule 1 x 145 [242,208,252,604,266,743]
+ CRUSH rule 1 x 146 [290,70,570,384,934,856]
+ CRUSH rule 1 x 147 [447,352,657,493,467,918]
+ CRUSH rule 1 x 148 [212,644,432,658,109,275]
+ CRUSH rule 1 x 149 [9,775,87,35,260,646]
+ CRUSH rule 1 x 150 [166,456,582,144,324,340]
+ CRUSH rule 1 x 151 [811,875,307,20,782,229]
+ CRUSH rule 1 x 152 [449,617,223,9,182,407]
+ CRUSH rule 1 x 153 [523,537,695,627,959,613]
+ CRUSH rule 1 x 154 [208,559,874,597,243,706]
+ CRUSH rule 1 x 155 [569,325,192,296,367,848]
+ CRUSH rule 1 x 156 [488,121,521,213,595,837]
+ CRUSH rule 1 x 157 [140,723,633,260,487,856]
+ CRUSH rule 1 x 158 [786,451,320,239,667,632]
+ CRUSH rule 1 x 159 [134,664,517,821,667,944]
+ CRUSH rule 1 x 160 [690,112,414,990,183,590]
+ CRUSH rule 1 x 161 [324,912,397,423,991,284]
+ CRUSH rule 1 x 162 [748,567,284,183,463,336]
+ CRUSH rule 1 x 163 [575,499,31,816,749,737]
+ CRUSH rule 1 x 164 [314,489,308,326,51,568]
+ CRUSH rule 1 x 165 [116,209,750,53,813,640]
+ CRUSH rule 1 x 166 [352,706,701,810,718,527]
+ CRUSH rule 1 x 167 [27,743,174,142,551,1]
+ CRUSH rule 1 x 168 [953,898,880,660,500,799]
+ CRUSH rule 1 x 169 [912,147,266,547,331,770]
+ CRUSH rule 1 x 170 [421,515,828,844,151,981]
+ CRUSH rule 1 x 171 [488,584,880,964,936,196]
+ CRUSH rule 1 x 172 [366,443,957,66,162,693]
+ CRUSH rule 1 x 173 [863,291,625,287,158,496]
+ CRUSH rule 1 x 174 [263,555,650,410,339,616]
+ CRUSH rule 1 x 175 [875,961,361,575,33,109]
+ CRUSH rule 1 x 176 [745,83,701,680,250,420]
+ CRUSH rule 1 x 177 [128,244,41,123,422,902]
+ CRUSH rule 1 x 178 [155,41,264,777,314,564]
+ CRUSH rule 1 x 179 [593,833,202,183,971,38]
+ CRUSH rule 1 x 180 [154,734,17,831,824,522]
+ CRUSH rule 1 x 181 [289,675,723,800,166,712]
+ CRUSH rule 1 x 182 [730,931,560,209,943,261]
+ CRUSH rule 1 x 183 [639,237,794,815,827,400]
+ CRUSH rule 1 x 184 [704,312,685,645,691,778]
+ CRUSH rule 1 x 185 [97,100,762,82,999,542]
+ CRUSH rule 1 x 186 [26,665,554,215,280,421]
+ CRUSH rule 1 x 187 [649,14,740,494,402,684]
+ CRUSH rule 1 x 188 [682,695,590,743,927,945]
+ CRUSH rule 1 x 189 [325,693,726,51,448,169]
+ CRUSH rule 1 x 190 [399,933,136,955,57,504]
+ CRUSH rule 1 x 191 [629,533,17,126,60,146]
+ CRUSH rule 1 x 192 [503,578,38,492,222,251]
+ CRUSH rule 1 x 193 [546,333,651,678,823,652]
+ CRUSH rule 1 x 194 [242,473,58,655,911,277]
+ CRUSH rule 1 x 195 [625,719,135,81,636,513]
+ CRUSH rule 1 x 196 [357,114,125,867,250,522]
+ CRUSH rule 1 x 197 [306,954,453,873,211,334]
+ CRUSH rule 1 x 198 [863,791,311,911,206,61]
+ CRUSH rule 1 x 199 [935,906,929,252,893,75]
+ CRUSH rule 1 x 200 [373,774,229,454,909,611]
+ CRUSH rule 1 x 201 [659,320,477,313,779,16]
+ CRUSH rule 1 x 202 [260,433,524,880,223,818]
+ CRUSH rule 1 x 203 [36,239,675,971,703,209]
+ CRUSH rule 1 x 204 [92,516,993,728,279,478]
+ CRUSH rule 1 x 205 [68,395,473,45,683,662]
+ CRUSH rule 1 x 206 [570,530,642,380,311,398]
+ CRUSH rule 1 x 207 [834,457,850,917,456,296]
+ CRUSH rule 1 x 208 [927,484,640,976,803,626]
+ CRUSH rule 1 x 209 [878,66,58,940,48,233]
+ CRUSH rule 1 x 210 [572,981,484,29,0,426]
+ CRUSH rule 1 x 211 [107,597,780,857,895,57]
+ CRUSH rule 1 x 212 [389,107,838,624,698,562]
+ CRUSH rule 1 x 213 [497,717,567,728,905,134]
+ CRUSH rule 1 x 214 [798,65,254,572,32,393]
+ CRUSH rule 1 x 215 [233,419,283,638,520,891]
+ CRUSH rule 1 x 216 [494,464,742,523,459,174]
+ CRUSH rule 1 x 217 [352,396,309,938,66,41]
+ CRUSH rule 1 x 218 [895,864,988,650,593,740]
+ CRUSH rule 1 x 219 [222,534,277,242,658,482]
+ CRUSH rule 1 x 220 [281,19,584,563,858,965]
+ CRUSH rule 1 x 221 [64,928,963,130,312,394]
+ CRUSH rule 1 x 222 [40,544,161,199,861,644]
+ CRUSH rule 1 x 223 [645,556,159,417,46,135]
+ CRUSH rule 1 x 224 [647,165,957,263,961,576]
+ CRUSH rule 1 x 225 [219,714,858,747,461,175]
+ CRUSH rule 1 x 226 [372,511,181,277,695,404]
+ CRUSH rule 1 x 227 [925,156,714,863,257,74]
+ CRUSH rule 1 x 228 [682,404,839,263,521,195]
+ CRUSH rule 1 x 229 [880,838,770,891,236,542]
+ CRUSH rule 1 x 230 [328,659,916,468,646,572]
+ CRUSH rule 1 x 231 [320,383,669,109,627,621]
+ CRUSH rule 1 x 232 [924,846,394,319,43,519]
+ CRUSH rule 1 x 233 [948,652,575,838,498,395]
+ CRUSH rule 1 x 234 [484,943,42,575,936,180]
+ CRUSH rule 1 x 235 [750,65,590,168,870,308]
+ CRUSH rule 1 x 236 [551,787,490,136,370,833]
+ CRUSH rule 1 x 237 [390,157,166,251,752,75]
+ CRUSH rule 1 x 238 [570,6,989,707,514,905]
+ CRUSH rule 1 x 239 [729,959,376,975,496,49]
+ CRUSH rule 1 x 240 [981,241,156,767,631,576]
+ CRUSH rule 1 x 241 [310,816,641,177,996,454]
+ CRUSH rule 1 x 242 [161,63,642,837,763,458]
+ CRUSH rule 1 x 243 [180,394,33,683,189,419]
+ CRUSH rule 1 x 244 [52,174,685,189,78,310]
+ CRUSH rule 1 x 245 [523,121,915,84,386,409]
+ CRUSH rule 1 x 246 [362,893,390,487,817,88]
+ CRUSH rule 1 x 247 [382,184,116,34,143,15]
+ CRUSH rule 1 x 248 [129,114,852,469,359,291]
+ CRUSH rule 1 x 249 [159,683,91,856,475,369]
+ CRUSH rule 1 x 250 [404,945,569,955,228,910]
+ CRUSH rule 1 x 251 [661,225,738,757,37,642]
+ CRUSH rule 1 x 252 [961,226,542,103,945,885]
+ CRUSH rule 1 x 253 [651,97,225,364,189,248]
+ CRUSH rule 1 x 254 [123,33,741,692,599,11]
+ CRUSH rule 1 x 255 [314,649,891,855,517,344]
+ CRUSH rule 1 x 256 [315,215,651,126,470,849]
+ CRUSH rule 1 x 257 [825,264,867,841,529,409]
+ CRUSH rule 1 x 258 [624,789,370,723,131,982]
+ CRUSH rule 1 x 259 [602,542,70,563,947,723]
+ CRUSH rule 1 x 260 [717,878,43,56,377,481]
+ CRUSH rule 1 x 261 [145,517,20,903,786,939]
+ CRUSH rule 1 x 262 [223,1,561,420,357,16]
+ CRUSH rule 1 x 263 [462,211,405,508,787,669]
+ CRUSH rule 1 x 264 [654,471,266,662,135,564]
+ CRUSH rule 1 x 265 [302,794,704,798,659,487]
+ CRUSH rule 1 x 266 [202,132,884,209,551,984]
+ CRUSH rule 1 x 267 [282,938,657,113,672,993]
+ CRUSH rule 1 x 268 [338,309,356,278,928,797]
+ CRUSH rule 1 x 269 [738,122,266,200,894,118]
+ CRUSH rule 1 x 270 [707,982,946,196,407,804]
+ CRUSH rule 1 x 271 [705,432,364,735,512,595]
+ CRUSH rule 1 x 272 [756,545,942,56,542,449]
+ CRUSH rule 1 x 273 [197,502,527,721,239,648]
+ CRUSH rule 1 x 274 [992,44,653,573,527,702]
+ CRUSH rule 1 x 275 [544,789,170,434,23,926]
+ CRUSH rule 1 x 276 [658,467,577,268,336,5]
+ CRUSH rule 1 x 277 [143,490,880,483,928,272]
+ CRUSH rule 1 x 278 [492,647,355,282,834,64]
+ CRUSH rule 1 x 279 [517,792,604,987,527,894]
+ CRUSH rule 1 x 280 [825,740,27,848,514,750]
+ CRUSH rule 1 x 281 [224,629,120,562,616,200]
+ CRUSH rule 1 x 282 [298,661,380,416,35,585]
+ CRUSH rule 1 x 283 [311,606,208,50,913,678]
+ CRUSH rule 1 x 284 [771,466,371,743,672,119]
+ CRUSH rule 1 x 285 [693,362,404,676,797,531]
+ CRUSH rule 1 x 286 [364,477,285,167,270,617]
+ CRUSH rule 1 x 287 [591,611,828,995,170,987]
+ CRUSH rule 1 x 288 [965,541,848,796,251,668]
+ CRUSH rule 1 x 289 [225,551,948,877,219,167]
+ CRUSH rule 1 x 290 [577,762,777,751,291,349]
+ CRUSH rule 1 x 291 [160,903,477,381,490,559]
+ CRUSH rule 1 x 292 [873,598,216,666,222,228]
+ CRUSH rule 1 x 293 [100,234,874,47,28,452]
+ CRUSH rule 1 x 294 [285,943,379,520,725,547]
+ CRUSH rule 1 x 295 [938,262,880,327,687,3]
+ CRUSH rule 1 x 296 [850,327,86,472,1,776]
+ CRUSH rule 1 x 297 [951,53,99,558,753,228]
+ CRUSH rule 1 x 298 [173,336,85,766,910,657]
+ CRUSH rule 1 x 299 [598,591,315,386,895,296]
+ CRUSH rule 1 x 300 [531,957,62,459,156,538]
+ CRUSH rule 1 x 301 [823,628,23,858,629,808]
+ CRUSH rule 1 x 302 [184,80,780,871,531,211]
+ CRUSH rule 1 x 303 [521,766,222,830,988,275]
+ CRUSH rule 1 x 304 [980,127,807,507,555,245]
+ CRUSH rule 1 x 305 [153,816,22,927,696,911]
+ CRUSH rule 1 x 306 [423,739,664,753,178,431]
+ CRUSH rule 1 x 307 [997,557,682,456,479,631]
+ CRUSH rule 1 x 308 [991,874,534,465,330,284]
+ CRUSH rule 1 x 309 [860,394,724,858,246,866]
+ CRUSH rule 1 x 310 [589,818,546,201,94,653]
+ CRUSH rule 1 x 311 [477,774,225,590,830,559]
+ CRUSH rule 1 x 312 [887,853,950,354,58,23]
+ CRUSH rule 1 x 313 [802,646,447,416,557,118]
+ CRUSH rule 1 x 314 [654,974,229,511,562,916]
+ CRUSH rule 1 x 315 [767,227,28,740,828,156]
+ CRUSH rule 1 x 316 [778,83,733,359,858,319]
+ CRUSH rule 1 x 317 [184,418,642,986,939,675]
+ CRUSH rule 1 x 318 [525,410,500,543,212,95]
+ CRUSH rule 1 x 319 [476,724,569,382,409,521]
+ CRUSH rule 1 x 320 [149,610,697,296,818,955]
+ CRUSH rule 1 x 321 [710,79,667,671,234,4]
+ CRUSH rule 1 x 322 [175,275,323,333,744,718]
+ CRUSH rule 1 x 323 [819,604,638,792,316,544]
+ CRUSH rule 1 x 324 [16,745,511,439,272,205]
+ CRUSH rule 1 x 325 [486,400,872,873,251,68]
+ CRUSH rule 1 x 326 [613,765,207,19,359,370]
+ CRUSH rule 1 x 327 [125,289,738,408,456,784]
+ CRUSH rule 1 x 328 [807,383,476,583,645,141]
+ CRUSH rule 1 x 329 [588,938,599,432,446,840]
+ CRUSH rule 1 x 330 [932,644,41,611,209,406]
+ CRUSH rule 1 x 331 [341,953,950,537,578,862]
+ CRUSH rule 1 x 332 [153,726,459,950,466,804]
+ CRUSH rule 1 x 333 [745,845,853,860,52,615]
+ CRUSH rule 1 x 334 [614,751,807,58,396,159]
+ CRUSH rule 1 x 335 [518,721,221,283,454,187]
+ CRUSH rule 1 x 336 [389,424,77,309,5,898]
+ CRUSH rule 1 x 337 [753,508,765,720,221,807]
+ CRUSH rule 1 x 338 [128,810,490,753,406,760]
+ CRUSH rule 1 x 339 [430,308,58,751,856,823]
+ CRUSH rule 1 x 340 [541,44,630,231,289,966]
+ CRUSH rule 1 x 341 [402,26,631,439,165,928]
+ CRUSH rule 1 x 342 [982,57,992,461,131,32]
+ CRUSH rule 1 x 343 [833,412,572,732,107,805]
+ CRUSH rule 1 x 344 [784,533,792,41,642,869]
+ CRUSH rule 1 x 345 [546,300,304,691,763,556]
+ CRUSH rule 1 x 346 [302,420,428,891,357,124]
+ CRUSH rule 1 x 347 [488,778,101,217,366,442]
+ CRUSH rule 1 x 348 [903,744,937,718,85,314]
+ CRUSH rule 1 x 349 [471,547,582,306,600,486]
+ CRUSH rule 1 x 350 [348,221,823,335,383,708]
+ CRUSH rule 1 x 351 [961,582,705,346,361,32]
+ CRUSH rule 1 x 352 [728,137,461,298,36,903]
+ CRUSH rule 1 x 353 [904,202,184,447,58,294]
+ CRUSH rule 1 x 354 [345,226,319,256,544,311]
+ CRUSH rule 1 x 355 [50,430,175,43,187,458]
+ CRUSH rule 1 x 356 [87,185,55,423,829,1]
+ CRUSH rule 1 x 357 [762,459,921,473,182,231]
+ CRUSH rule 1 x 358 [908,25,280,6,808,676]
+ CRUSH rule 1 x 359 [484,15,132,121,394,423]
+ CRUSH rule 1 x 360 [173,378,337,702,145,499]
+ CRUSH rule 1 x 361 [404,577,115,25,56,914]
+ CRUSH rule 1 x 362 [403,1,422,945,132,685]
+ CRUSH rule 1 x 363 [639,911,510,162,418,294]
+ CRUSH rule 1 x 364 [752,689,610,990,665,222]
+ CRUSH rule 1 x 365 [956,999,212,230,624,84]
+ CRUSH rule 1 x 366 [860,925,924,763,687,851]
+ CRUSH rule 1 x 367 [205,609,647,665,969,720]
+ CRUSH rule 1 x 368 [301,284,810,169,78,340]
+ CRUSH rule 1 x 369 [452,658,339,217,674,210]
+ CRUSH rule 1 x 370 [11,467,695,989,394,576]
+ CRUSH rule 1 x 371 [124,487,55,514,313,411]
+ CRUSH rule 1 x 372 [253,48,979,846,207,631]
+ CRUSH rule 1 x 373 [715,605,775,748,227,493]
+ CRUSH rule 1 x 374 [191,887,920,549,223,714]
+ CRUSH rule 1 x 375 [711,385,651,665,15,71]
+ CRUSH rule 1 x 376 [597,818,49,458,415,755]
+ CRUSH rule 1 x 377 [294,256,933,771,184,861]
+ CRUSH rule 1 x 378 [34,151,681,707,552,127]
+ CRUSH rule 1 x 379 [869,136,315,378,813,153]
+ CRUSH rule 1 x 380 [294,97,575,791,690,482]
+ CRUSH rule 1 x 381 [119,710,219,827,328,886]
+ CRUSH rule 1 x 382 [69,631,508,706,697,168]
+ CRUSH rule 1 x 383 [922,588,589,925,471,601]
+ CRUSH rule 1 x 384 [221,945,671,117,857,655]
+ CRUSH rule 1 x 385 [561,737,953,723,658,368]
+ CRUSH rule 1 x 386 [335,442,788,696,507,716]
+ CRUSH rule 1 x 387 [514,43,353,88,100,842]
+ CRUSH rule 1 x 388 [587,89,157,996,915,927]
+ CRUSH rule 1 x 389 [109,641,255,466,372,563]
+ CRUSH rule 1 x 390 [925,149,421,489,599,810]
+ CRUSH rule 1 x 391 [267,87,387,527,768,873]
+ CRUSH rule 1 x 392 [382,485,370,849,936,636]
+ CRUSH rule 1 x 393 [425,721,221,753,268,463]
+ CRUSH rule 1 x 394 [898,18,38,793,173,738]
+ CRUSH rule 1 x 395 [806,876,269,679,32,744]
+ CRUSH rule 1 x 396 [790,970,437,449,875,395]
+ CRUSH rule 1 x 397 [136,363,507,613,11,30]
+ CRUSH rule 1 x 398 [914,116,558,258,722,904]
+ CRUSH rule 1 x 399 [261,94,299,202,174,622]
+ CRUSH rule 1 x 400 [661,197,338,461,977,848]
+ CRUSH rule 1 x 401 [953,979,287,803,41,349]
+ CRUSH rule 1 x 402 [738,819,618,522,667,334]
+ CRUSH rule 1 x 403 [573,238,425,546,130,68]
+ CRUSH rule 1 x 404 [526,848,790,253,922,820]
+ CRUSH rule 1 x 405 [582,505,330,334,201,110]
+ CRUSH rule 1 x 406 [768,324,493,60,186,165]
+ CRUSH rule 1 x 407 [260,951,437,587,692,648]
+ CRUSH rule 1 x 408 [657,81,770,734,830,821]
+ CRUSH rule 1 x 409 [498,89,182,423,672,152]
+ CRUSH rule 1 x 410 [28,793,737,352,166,645]
+ CRUSH rule 1 x 411 [684,992,60,659,769,267]
+ CRUSH rule 1 x 412 [261,958,699,950,165,14]
+ CRUSH rule 1 x 413 [891,835,297,441,384,979]
+ CRUSH rule 1 x 414 [127,459,119,965,662,594]
+ CRUSH rule 1 x 415 [272,540,631,328,609,568]
+ CRUSH rule 1 x 416 [739,617,115,530,339,371]
+ CRUSH rule 1 x 417 [106,209,157,878,117,128]
+ CRUSH rule 1 x 418 [525,441,147,390,320,300]
+ CRUSH rule 1 x 419 [603,673,615,465,266,855]
+ CRUSH rule 1 x 420 [988,213,251,226,209,245]
+ CRUSH rule 1 x 421 [761,521,748,368,923,992]
+ CRUSH rule 1 x 422 [317,160,924,548,198,709]
+ CRUSH rule 1 x 423 [137,807,168,472,619,443]
+ CRUSH rule 1 x 424 [920,37,146,263,598,748]
+ CRUSH rule 1 x 425 [277,693,285,221,478,165]
+ CRUSH rule 1 x 426 [485,936,407,854,726,524]
+ CRUSH rule 1 x 427 [242,515,9,564,174,453]
+ CRUSH rule 1 x 428 [632,635,26,473,494,478]
+ CRUSH rule 1 x 429 [641,73,465,127,171,397]
+ CRUSH rule 1 x 430 [626,585,6,387,881,583]
+ CRUSH rule 1 x 431 [697,76,753,570,964,339]
+ CRUSH rule 1 x 432 [590,526,306,283,656,728]
+ CRUSH rule 1 x 433 [284,387,149,817,886,714]
+ CRUSH rule 1 x 434 [538,985,79,953,770,468]
+ CRUSH rule 1 x 435 [30,318,593,635,975,833]
+ CRUSH rule 1 x 436 [164,919,851,693,0,874]
+ CRUSH rule 1 x 437 [322,212,163,606,302,282]
+ CRUSH rule 1 x 438 [142,392,85,594,376,419]
+ CRUSH rule 1 x 439 [119,370,68,443,997,837]
+ CRUSH rule 1 x 440 [333,403,187,863,475,844]
+ CRUSH rule 1 x 441 [477,727,906,145,429,91]
+ CRUSH rule 1 x 442 [274,590,933,244,434,49]
+ CRUSH rule 1 x 443 [983,748,574,718,700,442]
+ CRUSH rule 1 x 444 [536,509,431,146,170,149]
+ CRUSH rule 1 x 445 [485,931,528,209,964,753]
+ CRUSH rule 1 x 446 [345,634,42,294,711,376]
+ CRUSH rule 1 x 447 [61,845,767,600,321,716]
+ CRUSH rule 1 x 448 [333,232,292,846,364,951]
+ CRUSH rule 1 x 449 [680,16,484,670,851,500]
+ CRUSH rule 1 x 450 [235,214,79,423,96,822]
+ CRUSH rule 1 x 451 [961,468,333,640,823,151]
+ CRUSH rule 1 x 452 [525,479,153,528,570,806]
+ CRUSH rule 1 x 453 [138,466,302,86,249,154]
+ CRUSH rule 1 x 454 [137,625,215,402,389,914]
+ CRUSH rule 1 x 455 [173,150,997,16,846,888]
+ CRUSH rule 1 x 456 [235,226,238,258,347,784]
+ CRUSH rule 1 x 457 [450,577,253,413,717,609]
+ CRUSH rule 1 x 458 [195,537,91,814,351,90]
+ CRUSH rule 1 x 459 [381,555,312,573,915,623]
+ CRUSH rule 1 x 460 [972,730,534,678,756,692]
+ CRUSH rule 1 x 461 [506,279,142,830,784,124]
+ CRUSH rule 1 x 462 [692,959,578,57,983,299]
+ CRUSH rule 1 x 463 [788,667,949,550,685,702]
+ CRUSH rule 1 x 464 [133,122,588,999,270,880]
+ CRUSH rule 1 x 465 [971,190,230,777,452,914]
+ CRUSH rule 1 x 466 [394,576,148,157,103,822]
+ CRUSH rule 1 x 467 [517,28,366,362,984,521]
+ CRUSH rule 1 x 468 [829,143,874,225,162,413]
+ CRUSH rule 1 x 469 [987,936,106,725,633,238]
+ CRUSH rule 1 x 470 [107,982,56,889,67,65]
+ CRUSH rule 1 x 471 [181,897,629,860,307,116]
+ CRUSH rule 1 x 472 [547,512,172,24,705,837]
+ CRUSH rule 1 x 473 [760,997,824,905,888,755]
+ CRUSH rule 1 x 474 [787,418,743,628,272,341]
+ CRUSH rule 1 x 475 [662,312,253,617,105,58]
+ CRUSH rule 1 x 476 [110,495,185,508,961,837]
+ CRUSH rule 1 x 477 [393,954,834,132,841,367]
+ CRUSH rule 1 x 478 [246,483,480,644,985,420]
+ CRUSH rule 1 x 479 [70,929,697,931,744,487]
+ CRUSH rule 1 x 480 [753,119,961,607,317,717]
+ CRUSH rule 1 x 481 [470,429,677,242,574,757]
+ CRUSH rule 1 x 482 [451,566,961,675,354,746]
+ CRUSH rule 1 x 483 [816,72,371,278,635,30]
+ CRUSH rule 1 x 484 [540,454,389,31,654,494]
+ CRUSH rule 1 x 485 [74,582,624,684,566,677]
+ CRUSH rule 1 x 486 [958,595,199,763,715,973]
+ CRUSH rule 1 x 487 [228,302,804,833,876,647]
+ CRUSH rule 1 x 488 [180,529,722,956,353,890]
+ CRUSH rule 1 x 489 [47,617,812,187,291,828]
+ CRUSH rule 1 x 490 [905,822,479,124,750,843]
+ CRUSH rule 1 x 491 [892,370,609,998,433,957]
+ CRUSH rule 1 x 492 [588,959,127,948,505,936]
+ CRUSH rule 1 x 493 [353,461,593,291,301,830]
+ CRUSH rule 1 x 494 [378,848,443,368,507,423]
+ CRUSH rule 1 x 495 [845,653,768,234,405,367]
+ CRUSH rule 1 x 496 [13,988,0,691,389,757]
+ CRUSH rule 1 x 497 [796,877,788,394,648,829]
+ CRUSH rule 1 x 498 [412,337,270,705,511,227]
+ CRUSH rule 1 x 499 [330,695,8,74,618,101]
+ CRUSH rule 1 x 500 [820,272,547,765,755,96]
+ CRUSH rule 1 x 501 [110,44,132,442,294,423]
+ CRUSH rule 1 x 502 [336,595,650,274,993,312]
+ CRUSH rule 1 x 503 [922,211,157,722,502,971]
+ CRUSH rule 1 x 504 [483,52,122,432,778,461]
+ CRUSH rule 1 x 505 [482,598,224,279,480,310]
+ CRUSH rule 1 x 506 [493,123,43,856,936,622]
+ CRUSH rule 1 x 507 [12,598,264,422,416,947]
+ CRUSH rule 1 x 508 [227,157,611,301,223,746]
+ CRUSH rule 1 x 509 [807,242,363,122,582,530]
+ CRUSH rule 1 x 510 [134,437,227,75,313,351]
+ CRUSH rule 1 x 511 [212,54,83,799,457,218]
+ CRUSH rule 1 x 512 [236,630,758,752,361,249]
+ CRUSH rule 1 x 513 [994,693,644,938,846,685]
+ CRUSH rule 1 x 514 [45,508,831,19,817,52]
+ CRUSH rule 1 x 515 [504,138,480,272,530,377]
+ CRUSH rule 1 x 516 [285,409,136,570,841,610]
+ CRUSH rule 1 x 517 [300,232,23,906,438,236]
+ CRUSH rule 1 x 518 [397,674,98,898,967,113]
+ CRUSH rule 1 x 519 [86,750,772,913,101,864]
+ CRUSH rule 1 x 520 [900,833,614,130,261,885]
+ CRUSH rule 1 x 521 [31,47,236,751,911,599]
+ CRUSH rule 1 x 522 [390,16,280,144,291,175]
+ CRUSH rule 1 x 523 [618,308,424,590,300,206]
+ CRUSH rule 1 x 524 [635,189,687,963,601,518]
+ CRUSH rule 1 x 525 [311,916,699,262,775,32]
+ CRUSH rule 1 x 526 [48,738,227,718,244,942]
+ CRUSH rule 1 x 527 [202,851,889,216,763,351]
+ CRUSH rule 1 x 528 [565,827,590,273,918,106]
+ CRUSH rule 1 x 529 [934,864,241,43,466,924]
+ CRUSH rule 1 x 530 [502,934,298,670,986,360]
+ CRUSH rule 1 x 531 [681,627,942,487,288,561]
+ CRUSH rule 1 x 532 [422,6,147,205,861,141]
+ CRUSH rule 1 x 533 [863,68,364,983,247,199]
+ CRUSH rule 1 x 534 [962,931,775,172,663,119]
+ CRUSH rule 1 x 535 [89,565,397,693,839,632]
+ CRUSH rule 1 x 536 [499,351,760,458,918,86]
+ CRUSH rule 1 x 537 [676,547,787,311,867,748]
+ CRUSH rule 1 x 538 [58,644,571,649,941,7]
+ CRUSH rule 1 x 539 [837,953,457,711,458,621]
+ CRUSH rule 1 x 540 [831,50,132,213,197,709]
+ CRUSH rule 1 x 541 [582,757,121,525,532,963]
+ CRUSH rule 1 x 542 [472,132,790,997,948,269]
+ CRUSH rule 1 x 543 [382,272,797,330,315,748]
+ CRUSH rule 1 x 544 [947,930,496,883,509,219]
+ CRUSH rule 1 x 545 [425,570,305,77,821,422]
+ CRUSH rule 1 x 546 [18,65,529,437,343,547]
+ CRUSH rule 1 x 547 [445,715,600,472,213,851]
+ CRUSH rule 1 x 548 [367,569,980,167,627,442]
+ CRUSH rule 1 x 549 [125,715,671,817,285,420]
+ CRUSH rule 1 x 550 [425,599,744,199,923,222]
+ CRUSH rule 1 x 551 [44,1,528,922,944,115]
+ CRUSH rule 1 x 552 [246,104,68,239,123,427]
+ CRUSH rule 1 x 553 [71,703,615,28,593,724]
+ CRUSH rule 1 x 554 [207,124,217,166,525,226]
+ CRUSH rule 1 x 555 [570,28,317,420,931,413]
+ CRUSH rule 1 x 556 [674,152,421,79,215,347]
+ CRUSH rule 1 x 557 [347,817,191,391,741,571]
+ CRUSH rule 1 x 558 [627,426,369,692,815,371]
+ CRUSH rule 1 x 559 [940,630,924,242,224,912]
+ CRUSH rule 1 x 560 [295,903,541,29,245,753]
+ CRUSH rule 1 x 561 [506,682,384,637,878,991]
+ CRUSH rule 1 x 562 [718,529,87,729,842,341]
+ CRUSH rule 1 x 563 [552,332,747,206,274,871]
+ CRUSH rule 1 x 564 [835,769,736,486,630,209]
+ CRUSH rule 1 x 565 [8,167,539,182,607,62]
+ CRUSH rule 1 x 566 [600,481,301,263,90,450]
+ CRUSH rule 1 x 567 [999,994,509,899,947,24]
+ CRUSH rule 1 x 568 [252,431,157,62,601,863]
+ CRUSH rule 1 x 569 [643,218,943,455,83,969]
+ CRUSH rule 1 x 570 [617,635,765,422,250,156]
+ CRUSH rule 1 x 571 [757,80,59,98,328,700]
+ CRUSH rule 1 x 572 [299,348,575,889,943,675]
+ CRUSH rule 1 x 573 [25,505,270,167,58,901]
+ CRUSH rule 1 x 574 [215,431,624,177,628,814]
+ CRUSH rule 1 x 575 [225,252,611,546,32,815]
+ CRUSH rule 1 x 576 [627,94,159,857,430,691]
+ CRUSH rule 1 x 577 [237,809,778,636,61,167]
+ CRUSH rule 1 x 578 [885,313,120,344,771,614]
+ CRUSH rule 1 x 579 [924,575,787,831,47,996]
+ CRUSH rule 1 x 580 [718,51,766,121,118,471]
+ CRUSH rule 1 x 581 [219,807,129,571,856,179]
+ CRUSH rule 1 x 582 [893,701,598,863,285,829]
+ CRUSH rule 1 x 583 [246,930,964,170,993,409]
+ CRUSH rule 1 x 584 [336,432,680,175,495,839]
+ CRUSH rule 1 x 585 [324,999,397,485,457,527]
+ CRUSH rule 1 x 586 [558,230,976,541,816,72]
+ CRUSH rule 1 x 587 [985,830,597,21,308,890]
+ CRUSH rule 1 x 588 [211,544,57,134,162,496]
+ CRUSH rule 1 x 589 [129,21,112,190,885,844]
+ CRUSH rule 1 x 590 [467,969,652,593,287,76]
+ CRUSH rule 1 x 591 [758,514,316,164,35,110]
+ CRUSH rule 1 x 592 [525,253,190,443,315,603]
+ CRUSH rule 1 x 593 [601,885,339,152,297,223]
+ CRUSH rule 1 x 594 [227,60,450,30,717,840]
+ CRUSH rule 1 x 595 [720,854,496,912,80,655]
+ CRUSH rule 1 x 596 [751,195,997,77,261,490]
+ CRUSH rule 1 x 597 [129,574,714,8,789,847]
+ CRUSH rule 1 x 598 [679,207,604,396,841,284]
+ CRUSH rule 1 x 599 [668,315,683,349,681,253]
+ CRUSH rule 1 x 600 [143,396,464,444,59,57]
+ CRUSH rule 1 x 601 [326,573,873,902,136,921]
+ CRUSH rule 1 x 602 [860,281,875,535,672,474]
+ CRUSH rule 1 x 603 [709,328,445,349,190,455]
+ CRUSH rule 1 x 604 [571,62,814,95,866,978]
+ CRUSH rule 1 x 605 [252,739,860,27,313,362]
+ CRUSH rule 1 x 606 [339,236,759,842,67,644]
+ CRUSH rule 1 x 607 [590,248,759,868,433,398]
+ CRUSH rule 1 x 608 [145,635,309,467,875,115]
+ CRUSH rule 1 x 609 [973,547,223,79,762,863]
+ CRUSH rule 1 x 610 [435,816,961,983,255,886]
+ CRUSH rule 1 x 611 [559,283,422,584,176,429]
+ CRUSH rule 1 x 612 [273,149,123,576,911,270]
+ CRUSH rule 1 x 613 [828,614,642,674,33,361]
+ CRUSH rule 1 x 614 [478,748,393,34,171,80]
+ CRUSH rule 1 x 615 [392,155,144,326,626,134]
+ CRUSH rule 1 x 616 [778,637,452,248,15,888]
+ CRUSH rule 1 x 617 [622,713,996,833,611,407]
+ CRUSH rule 1 x 618 [149,877,270,329,180,327]
+ CRUSH rule 1 x 619 [604,163,656,409,322,848]
+ CRUSH rule 1 x 620 [181,23,409,198,64,898]
+ CRUSH rule 1 x 621 [735,902,386,237,939,475]
+ CRUSH rule 1 x 622 [661,824,717,568,858,583]
+ CRUSH rule 1 x 623 [142,121,643,61,695,852]
+ CRUSH rule 1 x 624 [360,716,420,398,49,717]
+ CRUSH rule 1 x 625 [541,167,385,1,601,481]
+ CRUSH rule 1 x 626 [364,431,610,363,535,747]
+ CRUSH rule 1 x 627 [458,137,557,410,287,749]
+ CRUSH rule 1 x 628 [250,350,556,497,821,65]
+ CRUSH rule 1 x 629 [928,160,710,572,365,772]
+ CRUSH rule 1 x 630 [243,19,918,556,601,16]
+ CRUSH rule 1 x 631 [438,221,574,676,797,580]
+ CRUSH rule 1 x 632 [797,368,247,5,32,102]
+ CRUSH rule 1 x 633 [993,749,525,485,27,330]
+ CRUSH rule 1 x 634 [239,351,633,299,651,678]
+ CRUSH rule 1 x 635 [640,965,25,961,306,172]
+ CRUSH rule 1 x 636 [173,290,297,991,937,823]
+ CRUSH rule 1 x 637 [0,918,98,108,111,495]
+ CRUSH rule 1 x 638 [702,235,424,900,983,754]
+ CRUSH rule 1 x 639 [475,687,31,785,918,611]
+ CRUSH rule 1 x 640 [31,664,399,677,123,609]
+ CRUSH rule 1 x 641 [296,473,108,963,341,876]
+ CRUSH rule 1 x 642 [894,273,427,606,677,670]
+ CRUSH rule 1 x 643 [117,111,732,191,114,153]
+ CRUSH rule 1 x 644 [438,336,327,512,599,862]
+ CRUSH rule 1 x 645 [982,702,351,573,907,915]
+ CRUSH rule 1 x 646 [334,804,146,842,697,638]
+ CRUSH rule 1 x 647 [933,787,185,334,752,285]
+ CRUSH rule 1 x 648 [22,444,400,862,207,842]
+ CRUSH rule 1 x 649 [503,229,213,460,639,760]
+ CRUSH rule 1 x 650 [328,659,420,443,739,950]
+ CRUSH rule 1 x 651 [3,880,823,123,378,585]
+ CRUSH rule 1 x 652 [495,977,563,733,92,997]
+ CRUSH rule 1 x 653 [185,718,804,280,975,912]
+ CRUSH rule 1 x 654 [130,528,380,81,906,511]
+ CRUSH rule 1 x 655 [560,872,454,504,319,284]
+ CRUSH rule 1 x 656 [219,885,178,981,863,508]
+ CRUSH rule 1 x 657 [233,684,813,490,208,941]
+ CRUSH rule 1 x 658 [778,6,756,380,750,836]
+ CRUSH rule 1 x 659 [240,663,306,540,789,902]
+ CRUSH rule 1 x 660 [244,855,196,147,678,323]
+ CRUSH rule 1 x 661 [184,270,128,398,910,230]
+ CRUSH rule 1 x 662 [65,883,921,438,79,957]
+ CRUSH rule 1 x 663 [323,721,594,812,43,992]
+ CRUSH rule 1 x 664 [865,113,512,51,427,123]
+ CRUSH rule 1 x 665 [420,850,591,475,202,733]
+ CRUSH rule 1 x 666 [319,767,246,3,369,493]
+ CRUSH rule 1 x 667 [875,39,343,100,829,2]
+ CRUSH rule 1 x 668 [331,122,263,599,355,484]
+ CRUSH rule 1 x 669 [915,521,402,747,673,445]
+ CRUSH rule 1 x 670 [845,659,943,447,401,322]
+ CRUSH rule 1 x 671 [108,634,527,363,856,238]
+ CRUSH rule 1 x 672 [578,216,110,589,302,137]
+ CRUSH rule 1 x 673 [442,74,579,797,622,950]
+ CRUSH rule 1 x 674 [588,364,281,308,645,631]
+ CRUSH rule 1 x 675 [489,698,744,671,870,174]
+ CRUSH rule 1 x 676 [928,911,40,180,722,729]
+ CRUSH rule 1 x 677 [399,269,692,131,615,136]
+ CRUSH rule 1 x 678 [546,752,544,155,5,463]
+ CRUSH rule 1 x 679 [988,25,275,433,628,57]
+ CRUSH rule 1 x 680 [335,963,382,486,749,257]
+ CRUSH rule 1 x 681 [690,462,623,466,49,471]
+ CRUSH rule 1 x 682 [196,588,154,257,807,776]
+ CRUSH rule 1 x 683 [627,25,421,160,873,102]
+ CRUSH rule 1 x 684 [38,804,592,158,991,264]
+ CRUSH rule 1 x 685 [841,368,548,362,166,211]
+ CRUSH rule 1 x 686 [336,287,525,440,166,993]
+ CRUSH rule 1 x 687 [20,682,924,653,356,16]
+ CRUSH rule 1 x 688 [463,371,780,556,385,883]
+ CRUSH rule 1 x 689 [569,250,78,816,847,775]
+ CRUSH rule 1 x 690 [551,144,587,263,378,394]
+ CRUSH rule 1 x 691 [766,464,446,533,449,541]
+ CRUSH rule 1 x 692 [739,634,18,245,624,35]
+ CRUSH rule 1 x 693 [339,297,118,330,817,91]
+ CRUSH rule 1 x 694 [405,26,830,181,533,166]
+ CRUSH rule 1 x 695 [622,576,597,535,600,593]
+ CRUSH rule 1 x 696 [558,902,689,13,715,28]
+ CRUSH rule 1 x 697 [818,222,406,691,427,863]
+ CRUSH rule 1 x 698 [178,48,402,233,841,604]
+ CRUSH rule 1 x 699 [450,244,180,919,401,332]
+ CRUSH rule 1 x 700 [502,771,987,706,416,240]
+ CRUSH rule 1 x 701 [4,612,782,216,853,303]
+ CRUSH rule 1 x 702 [177,630,232,923,281,708]
+ CRUSH rule 1 x 703 [354,178,389,393,778,803]
+ CRUSH rule 1 x 704 [646,601,156,171,603,116]
+ CRUSH rule 1 x 705 [921,401,890,265,244,690]
+ CRUSH rule 1 x 706 [652,877,562,452,26,323]
+ CRUSH rule 1 x 707 [345,745,67,716,789,576]
+ CRUSH rule 1 x 708 [333,607,180,469,170,555]
+ CRUSH rule 1 x 709 [45,187,302,115,896,579]
+ CRUSH rule 1 x 710 [94,855,43,199,18,948]
+ CRUSH rule 1 x 711 [227,653,731,150,502,842]
+ CRUSH rule 1 x 712 [398,953,136,870,181,408]
+ CRUSH rule 1 x 713 [116,800,503,662,635,579]
+ CRUSH rule 1 x 714 [111,629,866,709,902,557]
+ CRUSH rule 1 x 715 [531,291,486,382,192,807]
+ CRUSH rule 1 x 716 [169,541,291,42,343,724]
+ CRUSH rule 1 x 717 [417,446,994,894,239,494]
+ CRUSH rule 1 x 718 [992,383,298,844,377,463]
+ CRUSH rule 1 x 719 [936,674,324,759,194,409]
+ CRUSH rule 1 x 720 [370,188,174,464,644,218]
+ CRUSH rule 1 x 721 [320,859,278,259,170,957]
+ CRUSH rule 1 x 722 [7,2,673,129,96,445]
+ CRUSH rule 1 x 723 [270,553,831,662,38,101]
+ CRUSH rule 1 x 724 [666,822,708,895,633,800]
+ CRUSH rule 1 x 725 [794,406,875,459,981,751]
+ CRUSH rule 1 x 726 [420,556,341,292,240,68]
+ CRUSH rule 1 x 727 [561,461,129,635,965,610]
+ CRUSH rule 1 x 728 [951,330,196,756,589,849]
+ CRUSH rule 1 x 729 [656,644,436,591,27,119]
+ CRUSH rule 1 x 730 [3,558,629,184,50,765]
+ CRUSH rule 1 x 731 [852,89,75,735,713,113]
+ CRUSH rule 1 x 732 [983,840,869,976,697,307]
+ CRUSH rule 1 x 733 [285,396,388,122,387,364]
+ CRUSH rule 1 x 734 [125,510,402,640,676,501]
+ CRUSH rule 1 x 735 [417,773,686,504,459,912]
+ CRUSH rule 1 x 736 [749,396,632,550,779,109]
+ CRUSH rule 1 x 737 [644,991,946,135,448,903]
+ CRUSH rule 1 x 738 [449,683,290,220,245,525]
+ CRUSH rule 1 x 739 [341,220,641,454,740,661]
+ CRUSH rule 1 x 740 [874,524,674,650,472,282]
+ CRUSH rule 1 x 741 [189,472,712,798,715,757]
+ CRUSH rule 1 x 742 [912,581,114,695,730,21]
+ CRUSH rule 1 x 743 [654,914,425,441,763,39]
+ CRUSH rule 1 x 744 [725,295,579,377,162,447]
+ CRUSH rule 1 x 745 [787,858,850,506,612,735]
+ CRUSH rule 1 x 746 [757,848,704,30,47,940]
+ CRUSH rule 1 x 747 [700,81,867,681,801,64]
+ CRUSH rule 1 x 748 [557,436,238,664,293,865]
+ CRUSH rule 1 x 749 [772,622,337,42,156,302]
+ CRUSH rule 1 x 750 [946,97,376,677,316,670]
+ CRUSH rule 1 x 751 [996,618,343,911,83,22]
+ CRUSH rule 1 x 752 [746,887,695,868,610,950]
+ CRUSH rule 1 x 753 [741,14,463,479,172,192]
+ CRUSH rule 1 x 754 [648,349,333,355,65,63]
+ CRUSH rule 1 x 755 [157,460,466,187,959,674]
+ CRUSH rule 1 x 756 [416,97,197,497,227,3]
+ CRUSH rule 1 x 757 [599,839,776,410,256,823]
+ CRUSH rule 1 x 758 [994,218,620,256,361,749]
+ CRUSH rule 1 x 759 [959,682,514,745,100,519]
+ CRUSH rule 1 x 760 [518,943,215,83,706,137]
+ CRUSH rule 1 x 761 [285,849,420,324,987,338]
+ CRUSH rule 1 x 762 [591,313,41,335,110,696]
+ CRUSH rule 1 x 763 [908,411,200,740,292,295]
+ CRUSH rule 1 x 764 [787,234,894,485,883,711]
+ CRUSH rule 1 x 765 [327,921,882,393,444,792]
+ CRUSH rule 1 x 766 [84,161,878,704,416,144]
+ CRUSH rule 1 x 767 [370,895,702,701,890,2]
+ CRUSH rule 1 x 768 [826,760,879,864,460,474]
+ CRUSH rule 1 x 769 [67,768,663,735,814,66]
+ CRUSH rule 1 x 770 [593,909,482,259,5,550]
+ CRUSH rule 1 x 771 [309,935,121,578,937,685]
+ CRUSH rule 1 x 772 [12,125,797,301,348,419]
+ CRUSH rule 1 x 773 [253,466,820,549,591,193]
+ CRUSH rule 1 x 774 [164,390,705,109,881,505]
+ CRUSH rule 1 x 775 [703,47,43,973,643,406]
+ CRUSH rule 1 x 776 [728,231,80,916,2,850]
+ CRUSH rule 1 x 777 [981,621,568,729,869,952]
+ CRUSH rule 1 x 778 [411,456,544,597,789,784]
+ CRUSH rule 1 x 779 [346,121,519,921,587,48]
+ CRUSH rule 1 x 780 [476,39,288,381,303,29]
+ CRUSH rule 1 x 781 [10,130,585,844,729,705]
+ CRUSH rule 1 x 782 [462,246,581,902,623,877]
+ CRUSH rule 1 x 783 [580,373,153,775,668,661]
+ CRUSH rule 1 x 784 [413,113,978,990,994,56]
+ CRUSH rule 1 x 785 [341,856,332,354,59,581]
+ CRUSH rule 1 x 786 [411,140,313,393,215,618]
+ CRUSH rule 1 x 787 [605,522,211,813,636,224]
+ CRUSH rule 1 x 788 [226,545,35,142,726,851]
+ CRUSH rule 1 x 789 [545,320,414,702,731,277]
+ CRUSH rule 1 x 790 [414,748,816,327,130,115]
+ CRUSH rule 1 x 791 [660,906,406,697,916,322]
+ CRUSH rule 1 x 792 [287,392,514,204,75,789]
+ CRUSH rule 1 x 793 [631,133,850,713,720,487]
+ CRUSH rule 1 x 794 [931,517,543,210,963,898]
+ CRUSH rule 1 x 795 [551,962,477,948,425,434]
+ CRUSH rule 1 x 796 [814,4,95,27,368,300]
+ CRUSH rule 1 x 797 [64,201,299,734,605,864]
+ CRUSH rule 1 x 798 [422,530,114,431,565,716]
+ CRUSH rule 1 x 799 [824,32,679,562,266,549]
+ CRUSH rule 1 x 800 [862,623,489,637,861,196]
+ CRUSH rule 1 x 801 [145,550,329,324,734,160]
+ CRUSH rule 1 x 802 [570,19,847,308,387,518]
+ CRUSH rule 1 x 803 [151,812,662,358,880,349]
+ CRUSH rule 1 x 804 [467,93,264,863,176,842]
+ CRUSH rule 1 x 805 [621,223,938,809,591,686]
+ CRUSH rule 1 x 806 [898,957,805,430,499,584]
+ CRUSH rule 1 x 807 [354,531,422,159,921,431]
+ CRUSH rule 1 x 808 [7,96,76,897,446,2]
+ CRUSH rule 1 x 809 [70,734,719,56,687,21]
+ CRUSH rule 1 x 810 [701,18,972,327,771,649]
+ CRUSH rule 1 x 811 [248,547,103,728,901,264]
+ CRUSH rule 1 x 812 [230,576,821,566,993,762]
+ CRUSH rule 1 x 813 [805,114,683,629,605,462]
+ CRUSH rule 1 x 814 [54,619,973,741,497,894]
+ CRUSH rule 1 x 815 [679,412,613,132,969,411]
+ CRUSH rule 1 x 816 [919,448,826,414,36,289]
+ CRUSH rule 1 x 817 [765,830,436,521,332,458]
+ CRUSH rule 1 x 818 [415,566,644,687,692,414]
+ CRUSH rule 1 x 819 [721,319,865,750,546,859]
+ CRUSH rule 1 x 820 [218,301,333,190,686,179]
+ CRUSH rule 1 x 821 [185,795,680,953,329,750]
+ CRUSH rule 1 x 822 [356,261,54,522,900,103]
+ CRUSH rule 1 x 823 [220,281,549,456,64,306]
+ CRUSH rule 1 x 824 [292,809,887,74,776,788]
+ CRUSH rule 1 x 825 [949,778,101,311,110,480]
+ CRUSH rule 1 x 826 [767,818,833,927,356,954]
+ CRUSH rule 1 x 827 [631,83,406,635,657,713]
+ CRUSH rule 1 x 828 [288,986,445,26,414,607]
+ CRUSH rule 1 x 829 [990,667,915,694,974,453]
+ CRUSH rule 1 x 830 [152,571,778,505,685,209]
+ CRUSH rule 1 x 831 [814,563,630,97,582,107]
+ CRUSH rule 1 x 832 [235,641,616,110,979,844]
+ CRUSH rule 1 x 833 [657,565,922,140,825,457]
+ CRUSH rule 1 x 834 [907,231,644,13,617,130]
+ CRUSH rule 1 x 835 [784,262,771,264,612,238]
+ CRUSH rule 1 x 836 [951,158,366,710,43,427]
+ CRUSH rule 1 x 837 [556,498,334,633,895,627]
+ CRUSH rule 1 x 838 [329,274,964,547,119,342]
+ CRUSH rule 1 x 839 [568,209,939,364,658,747]
+ CRUSH rule 1 x 840 [45,579,842,70,655,862]
+ CRUSH rule 1 x 841 [652,702,24,605,152,93]
+ CRUSH rule 1 x 842 [629,984,314,895,408,897]
+ CRUSH rule 1 x 843 [799,690,688,648,151,812]
+ CRUSH rule 1 x 844 [694,600,534,700,569,11]
+ CRUSH rule 1 x 845 [332,30,179,93,951,324]
+ CRUSH rule 1 x 846 [452,251,712,719,404,739]
+ CRUSH rule 1 x 847 [399,681,847,739,13,555]
+ CRUSH rule 1 x 848 [303,138,440,346,547,216]
+ CRUSH rule 1 x 849 [666,346,708,873,64,694]
+ CRUSH rule 1 x 850 [644,511,345,844,545,337]
+ CRUSH rule 1 x 851 [527,546,737,425,100,331]
+ CRUSH rule 1 x 852 [31,809,94,618,156,853]
+ CRUSH rule 1 x 853 [483,330,869,184,46,942]
+ CRUSH rule 1 x 854 [697,953,968,143,502,955]
+ CRUSH rule 1 x 855 [837,996,239,621,32,191]
+ CRUSH rule 1 x 856 [712,40,547,430,195,857]
+ CRUSH rule 1 x 857 [77,984,576,551,568,96]
+ CRUSH rule 1 x 858 [412,384,841,465,572,576]
+ CRUSH rule 1 x 859 [173,760,26,300,87,567]
+ CRUSH rule 1 x 860 [776,429,328,917,658,783]
+ CRUSH rule 1 x 861 [705,405,477,50,73,714]
+ CRUSH rule 1 x 862 [809,44,788,938,964,177]
+ CRUSH rule 1 x 863 [349,496,963,178,675,853]
+ CRUSH rule 1 x 864 [717,858,101,239,992,244]
+ CRUSH rule 1 x 865 [857,603,586,262,550,289]
+ CRUSH rule 1 x 866 [394,304,71,96,642,155]
+ CRUSH rule 1 x 867 [640,773,663,974,261,296]
+ CRUSH rule 1 x 868 [613,950,712,663,649,460]
+ CRUSH rule 1 x 869 [973,889,524,22,671,477]
+ CRUSH rule 1 x 870 [505,35,386,498,348,503]
+ CRUSH rule 1 x 871 [239,264,262,773,781,734]
+ CRUSH rule 1 x 872 [21,767,456,748,783,797]
+ CRUSH rule 1 x 873 [954,666,980,264,435,233]
+ CRUSH rule 1 x 874 [54,510,947,1,500,119]
+ CRUSH rule 1 x 875 [809,418,452,462,88,673]
+ CRUSH rule 1 x 876 [483,457,61,248,523,277]
+ CRUSH rule 1 x 877 [542,531,952,939,710,179]
+ CRUSH rule 1 x 878 [217,674,857,644,678,809]
+ CRUSH rule 1 x 879 [999,475,134,250,319,357]
+ CRUSH rule 1 x 880 [678,573,935,385,570,651]
+ CRUSH rule 1 x 881 [394,835,789,802,587,155]
+ CRUSH rule 1 x 882 [467,382,353,56,979,674]
+ CRUSH rule 1 x 883 [802,744,237,337,50,96]
+ CRUSH rule 1 x 884 [653,660,638,700,31,558]
+ CRUSH rule 1 x 885 [898,704,307,445,879,872]
+ CRUSH rule 1 x 886 [434,357,938,641,737,8]
+ CRUSH rule 1 x 887 [297,226,711,428,370,318]
+ CRUSH rule 1 x 888 [863,324,443,213,902,25]
+ CRUSH rule 1 x 889 [105,102,308,163,947,548]
+ CRUSH rule 1 x 890 [550,248,606,704,615,708]
+ CRUSH rule 1 x 891 [575,928,880,891,826,763]
+ CRUSH rule 1 x 892 [259,862,133,271,292,162]
+ CRUSH rule 1 x 893 [902,880,543,542,37,942]
+ CRUSH rule 1 x 894 [180,169,916,43,945,713]
+ CRUSH rule 1 x 895 [725,849,182,129,177,272]
+ CRUSH rule 1 x 896 [951,34,874,537,969,123]
+ CRUSH rule 1 x 897 [810,352,73,939,943,895]
+ CRUSH rule 1 x 898 [979,433,719,411,787,359]
+ CRUSH rule 1 x 899 [685,668,534,932,399,156]
+ CRUSH rule 1 x 900 [530,978,41,894,941,681]
+ CRUSH rule 1 x 901 [740,107,336,175,574,706]
+ CRUSH rule 1 x 902 [800,743,693,310,67,111]
+ CRUSH rule 1 x 903 [230,267,842,266,550,769]
+ CRUSH rule 1 x 904 [346,949,460,973,696,91]
+ CRUSH rule 1 x 905 [530,397,619,958,576,973]
+ CRUSH rule 1 x 906 [80,426,138,672,73,776]
+ CRUSH rule 1 x 907 [365,968,475,297,296,724]
+ CRUSH rule 1 x 908 [204,832,742,809,862,745]
+ CRUSH rule 1 x 909 [883,989,146,959,366,59]
+ CRUSH rule 1 x 910 [549,593,249,853,792,769]
+ CRUSH rule 1 x 911 [325,847,352,214,851,732]
+ CRUSH rule 1 x 912 [874,888,582,796,557,601]
+ CRUSH rule 1 x 913 [331,463,342,574,989,362]
+ CRUSH rule 1 x 914 [836,468,601,732,607,275]
+ CRUSH rule 1 x 915 [245,228,100,661,799,13]
+ CRUSH rule 1 x 916 [77,967,364,435,27,474]
+ CRUSH rule 1 x 917 [239,60,866,221,772,967]
+ CRUSH rule 1 x 918 [988,115,922,80,201,544]
+ CRUSH rule 1 x 919 [783,139,696,1,848,169]
+ CRUSH rule 1 x 920 [623,408,685,953,974,696]
+ CRUSH rule 1 x 921 [105,799,144,90,399,373]
+ CRUSH rule 1 x 922 [887,505,652,348,514,806]
+ CRUSH rule 1 x 923 [223,318,552,458,743,871]
+ CRUSH rule 1 x 924 [25,778,366,333,163,801]
+ CRUSH rule 1 x 925 [912,601,297,682,770,173]
+ CRUSH rule 1 x 926 [968,133,96,144,814,155]
+ CRUSH rule 1 x 927 [277,724,214,988,690,342]
+ CRUSH rule 1 x 928 [554,203,658,789,298,299]
+ CRUSH rule 1 x 929 [761,802,367,528,758,522]
+ CRUSH rule 1 x 930 [814,61,788,736,660,491]
+ CRUSH rule 1 x 931 [29,193,61,41,343,664]
+ CRUSH rule 1 x 932 [446,198,862,534,168,35]
+ CRUSH rule 1 x 933 [352,742,216,321,525,44]
+ CRUSH rule 1 x 934 [730,2,332,631,613,249]
+ CRUSH rule 1 x 935 [731,23,736,79,361,992]
+ CRUSH rule 1 x 936 [322,975,20,904,827,603]
+ CRUSH rule 1 x 937 [822,221,841,161,723,137]
+ CRUSH rule 1 x 938 [557,850,66,630,499,404]
+ CRUSH rule 1 x 939 [150,11,971,371,124,785]
+ CRUSH rule 1 x 940 [638,398,169,616,333,751]
+ CRUSH rule 1 x 941 [730,342,929,577,451,838]
+ CRUSH rule 1 x 942 [62,292,166,814,587,172]
+ CRUSH rule 1 x 943 [165,314,519,548,41,726]
+ CRUSH rule 1 x 944 [199,625,766,176,194,297]
+ CRUSH rule 1 x 945 [946,999,699,303,38,81]
+ CRUSH rule 1 x 946 [595,93,852,142,503,647]
+ CRUSH rule 1 x 947 [800,582,356,93,716,117]
+ CRUSH rule 1 x 948 [132,551,139,920,87,46]
+ CRUSH rule 1 x 949 [792,920,466,380,97,568]
+ CRUSH rule 1 x 950 [111,345,176,543,879,954]
+ CRUSH rule 1 x 951 [414,619,648,655,364,971]
+ CRUSH rule 1 x 952 [775,469,500,356,287,4]
+ CRUSH rule 1 x 953 [349,1,5,251,168,680]
+ CRUSH rule 1 x 954 [570,940,410,249,929,394]
+ CRUSH rule 1 x 955 [729,774,823,800,7,127]
+ CRUSH rule 1 x 956 [519,141,575,625,738,475]
+ CRUSH rule 1 x 957 [242,709,611,97,760,309]
+ CRUSH rule 1 x 958 [84,217,227,253,246,604]
+ CRUSH rule 1 x 959 [270,413,918,789,703,608]
+ CRUSH rule 1 x 960 [458,192,307,279,920,139]
+ CRUSH rule 1 x 961 [981,388,777,546,359,660]
+ CRUSH rule 1 x 962 [623,834,277,134,729,246]
+ CRUSH rule 1 x 963 [291,167,714,468,109,373]
+ CRUSH rule 1 x 964 [28,156,788,127,598,215]
+ CRUSH rule 1 x 965 [675,557,290,517,840,510]
+ CRUSH rule 1 x 966 [836,306,946,283,642,606]
+ CRUSH rule 1 x 967 [966,386,735,837,392,116]
+ CRUSH rule 1 x 968 [864,756,690,121,328,122]
+ CRUSH rule 1 x 969 [729,625,480,769,512,882]
+ CRUSH rule 1 x 970 [800,362,646,582,309,102]
+ CRUSH rule 1 x 971 [737,381,153,684,298,166]
+ CRUSH rule 1 x 972 [952,245,720,884,334,311]
+ CRUSH rule 1 x 973 [356,455,579,857,832,596]
+ CRUSH rule 1 x 974 [545,758,586,596,869,790]
+ CRUSH rule 1 x 975 [336,191,202,146,720,897]
+ CRUSH rule 1 x 976 [446,208,757,620,252,846]
+ CRUSH rule 1 x 977 [202,896,196,956,763,126]
+ CRUSH rule 1 x 978 [612,324,996,225,418,583]
+ CRUSH rule 1 x 979 [843,457,675,650,958,657]
+ CRUSH rule 1 x 980 [60,914,881,626,850,759]
+ CRUSH rule 1 x 981 [702,749,937,153,724,514]
+ CRUSH rule 1 x 982 [298,928,738,167,99,668]
+ CRUSH rule 1 x 983 [723,572,395,358,900,37]
+ CRUSH rule 1 x 984 [723,864,804,935,846,993]
+ CRUSH rule 1 x 985 [945,459,868,211,524,954]
+ CRUSH rule 1 x 986 [772,664,535,169,297,996]
+ CRUSH rule 1 x 987 [88,324,312,843,661,580]
+ CRUSH rule 1 x 988 [522,927,131,996,351,685]
+ CRUSH rule 1 x 989 [578,332,208,605,975,207]
+ CRUSH rule 1 x 990 [638,228,414,311,738,698]
+ CRUSH rule 1 x 991 [530,221,451,422,879,916]
+ CRUSH rule 1 x 992 [925,705,275,81,234,310]
+ CRUSH rule 1 x 993 [991,301,43,469,830,242]
+ CRUSH rule 1 x 994 [276,51,868,683,843,815]
+ CRUSH rule 1 x 995 [288,836,753,790,758,120]
+ CRUSH rule 1 x 996 [887,983,252,686,470,345]
+ CRUSH rule 1 x 997 [110,924,386,79,705,697]
+ CRUSH rule 1 x 998 [435,830,485,853,926,730]
+ CRUSH rule 1 x 999 [876,738,357,913,723,51]
+ CRUSH rule 1 x 1000 [178,963,638,430,845,586]
+ CRUSH rule 1 x 1001 [99,519,66,759,583,944]
+ CRUSH rule 1 x 1002 [515,534,468,866,878,717]
+ CRUSH rule 1 x 1003 [104,611,937,698,94,67]
+ CRUSH rule 1 x 1004 [269,638,724,375,491,121]
+ CRUSH rule 1 x 1005 [369,223,309,409,822,39]
+ CRUSH rule 1 x 1006 [40,107,69,275,79,429]
+ CRUSH rule 1 x 1007 [978,111,416,758,454,640]
+ CRUSH rule 1 x 1008 [965,956,624,832,421,96]
+ CRUSH rule 1 x 1009 [598,476,356,695,919,566]
+ CRUSH rule 1 x 1010 [767,523,239,517,29,77]
+ CRUSH rule 1 x 1011 [289,871,207,576,347,698]
+ CRUSH rule 1 x 1012 [128,28,370,31,341,755]
+ CRUSH rule 1 x 1013 [979,765,660,812,666,187]
+ CRUSH rule 1 x 1014 [979,948,513,88,47,825]
+ CRUSH rule 1 x 1015 [277,790,396,672,542,647]
+ CRUSH rule 1 x 1016 [262,73,128,886,839,685]
+ CRUSH rule 1 x 1017 [150,269,61,499,832,591]
+ CRUSH rule 1 x 1018 [555,829,554,944,406,576]
+ CRUSH rule 1 x 1019 [513,356,265,446,65,288]
+ CRUSH rule 1 x 1020 [158,161,877,704,948,570]
+ CRUSH rule 1 x 1021 [915,998,957,285,546,202]
+ CRUSH rule 1 x 1022 [967,829,973,640,703,470]
+ CRUSH rule 1 x 1023 [488,257,614,859,325,419]
+ rule 1 (metadata) num_rep 6 result size == 6:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450,604,380,966]
+ CRUSH rule 1 x 1 [876,250,334,633,744,843,672]
+ CRUSH rule 1 x 2 [292,832,53,392,386,787,527]
+ CRUSH rule 1 x 3 [623,387,124,998,749,211,481]
+ CRUSH rule 1 x 4 [61,334,710,4,994,982,847]
+ CRUSH rule 1 x 5 [946,557,713,664,141,817,964]
+ CRUSH rule 1 x 6 [576,668,212,163,732,381,884]
+ CRUSH rule 1 x 7 [645,753,906,393,341,44,578]
+ CRUSH rule 1 x 8 [243,6,863,781,211,100,462]
+ CRUSH rule 1 x 9 [22,578,251,410,297,430,3]
+ CRUSH rule 1 x 10 [758,828,360,477,821,801,811]
+ CRUSH rule 1 x 11 [769,120,124,527,119,504,380]
+ CRUSH rule 1 x 12 [780,364,689,755,675,199,117]
+ CRUSH rule 1 x 13 [557,18,351,719,742,780,78]
+ CRUSH rule 1 x 14 [59,561,249,461,971,835,855]
+ CRUSH rule 1 x 15 [718,928,993,21,76,313,437]
+ CRUSH rule 1 x 16 [673,632,841,954,788,90,786]
+ CRUSH rule 1 x 17 [648,43,560,514,142,289,935]
+ CRUSH rule 1 x 18 [654,219,181,568,381,253,883]
+ CRUSH rule 1 x 19 [850,545,377,848,863,543,51]
+ CRUSH rule 1 x 20 [717,785,974,5,225,552,975]
+ CRUSH rule 1 x 21 [420,57,519,306,312,983,263]
+ CRUSH rule 1 x 22 [503,998,193,821,634,684,557]
+ CRUSH rule 1 x 23 [411,663,168,110,899,488,477]
+ CRUSH rule 1 x 24 [266,861,353,1,456,128,800]
+ CRUSH rule 1 x 25 [760,483,818,600,509,951,248]
+ CRUSH rule 1 x 26 [903,24,573,718,112,694,501]
+ CRUSH rule 1 x 27 [946,188,289,510,687,827,676]
+ CRUSH rule 1 x 28 [69,312,73,198,256,629,770]
+ CRUSH rule 1 x 29 [844,883,337,628,496,405,719]
+ CRUSH rule 1 x 30 [621,18,613,794,910,936,426]
+ CRUSH rule 1 x 31 [784,943,814,539,962,392,813]
+ CRUSH rule 1 x 32 [173,374,369,972,315,83,428]
+ CRUSH rule 1 x 33 [698,336,357,966,582,407,618]
+ CRUSH rule 1 x 34 [168,836,210,798,904,190,663]
+ CRUSH rule 1 x 35 [274,509,534,818,912,671,75]
+ CRUSH rule 1 x 36 [318,215,153,628,87,407,676]
+ CRUSH rule 1 x 37 [173,604,109,935,203,401,311]
+ CRUSH rule 1 x 38 [708,444,683,604,722,900,929]
+ CRUSH rule 1 x 39 [662,198,417,680,226,342,856]
+ CRUSH rule 1 x 40 [620,801,414,78,560,766,980]
+ CRUSH rule 1 x 41 [811,264,177,127,148,791,930]
+ CRUSH rule 1 x 42 [863,179,527,660,133,529,456]
+ CRUSH rule 1 x 43 [686,822,988,228,791,549,514]
+ CRUSH rule 1 x 44 [396,222,46,841,536,140,160]
+ CRUSH rule 1 x 45 [991,694,253,142,54,422,658]
+ CRUSH rule 1 x 46 [420,909,184,285,508,458,45]
+ CRUSH rule 1 x 47 [467,211,605,207,241,881,959]
+ CRUSH rule 1 x 48 [955,329,368,168,698,787,738]
+ CRUSH rule 1 x 49 [974,891,931,29,813,506,822]
+ CRUSH rule 1 x 50 [870,441,691,823,761,6,83]
+ CRUSH rule 1 x 51 [182,930,25,936,97,260,406]
+ CRUSH rule 1 x 52 [704,812,894,794,481,37,304]
+ CRUSH rule 1 x 53 [185,713,631,280,345,558,882]
+ CRUSH rule 1 x 54 [270,441,100,82,983,930,339]
+ CRUSH rule 1 x 55 [895,734,958,793,651,572,508]
+ CRUSH rule 1 x 56 [564,963,683,324,40,189,77]
+ CRUSH rule 1 x 57 [738,130,208,973,498,861,670]
+ CRUSH rule 1 x 58 [524,113,806,903,531,334,8]
+ CRUSH rule 1 x 59 [408,337,668,529,34,384,643]
+ CRUSH rule 1 x 60 [228,790,857,309,616,895,194]
+ CRUSH rule 1 x 61 [154,843,717,467,883,536,812]
+ CRUSH rule 1 x 62 [594,811,549,276,693,917,45]
+ CRUSH rule 1 x 63 [646,67,884,925,941,434,705]
+ CRUSH rule 1 x 64 [175,542,155,837,594,197,451]
+ CRUSH rule 1 x 65 [745,619,131,867,269,62,862]
+ CRUSH rule 1 x 66 [275,468,23,35,328,432,334]
+ CRUSH rule 1 x 67 [246,958,524,493,636,227,783]
+ CRUSH rule 1 x 68 [711,473,403,228,835,126,705]
+ CRUSH rule 1 x 69 [493,924,850,939,950,105,871]
+ CRUSH rule 1 x 70 [30,499,644,33,804,654,684]
+ CRUSH rule 1 x 71 [984,883,574,716,575,391,587]
+ CRUSH rule 1 x 72 [71,286,942,363,628,632,642]
+ CRUSH rule 1 x 73 [922,618,3,371,464,442,835]
+ CRUSH rule 1 x 74 [629,414,185,573,678,338,633]
+ CRUSH rule 1 x 75 [222,20,174,820,312,361,366]
+ CRUSH rule 1 x 76 [262,366,339,290,718,143,735]
+ CRUSH rule 1 x 77 [638,469,992,280,773,892,197]
+ CRUSH rule 1 x 78 [324,511,788,7,308,228,183]
+ CRUSH rule 1 x 79 [577,990,64,94,447,924,339]
+ CRUSH rule 1 x 80 [501,95,278,903,631,842,51]
+ CRUSH rule 1 x 81 [506,812,9,698,173,664,247]
+ CRUSH rule 1 x 82 [222,145,80,785,835,745,580]
+ CRUSH rule 1 x 83 [71,634,61,91,856,529,66]
+ CRUSH rule 1 x 84 [49,761,773,368,318,708,681]
+ CRUSH rule 1 x 85 [985,896,708,861,325,307,567]
+ CRUSH rule 1 x 86 [537,745,93,524,466,356,38]
+ CRUSH rule 1 x 87 [997,317,463,626,685,594,909]
+ CRUSH rule 1 x 88 [957,350,890,857,375,176,99]
+ CRUSH rule 1 x 89 [399,730,148,314,159,982,320]
+ CRUSH rule 1 x 90 [943,706,683,267,579,141,412]
+ CRUSH rule 1 x 91 [22,368,149,928,140,529,495]
+ CRUSH rule 1 x 92 [532,424,426,773,623,197,167]
+ CRUSH rule 1 x 93 [218,489,405,681,549,201,343]
+ CRUSH rule 1 x 94 [181,96,102,515,776,365,82]
+ CRUSH rule 1 x 95 [343,957,820,139,334,37,648]
+ CRUSH rule 1 x 96 [861,270,87,797,0,245,204]
+ CRUSH rule 1 x 97 [459,706,45,328,274,605,83]
+ CRUSH rule 1 x 98 [327,867,353,948,728,280,270]
+ CRUSH rule 1 x 99 [974,133,468,906,235,988,37]
+ CRUSH rule 1 x 100 [32,445,547,371,960,885,9]
+ CRUSH rule 1 x 101 [142,90,337,950,970,570,12]
+ CRUSH rule 1 x 102 [172,129,139,22,403,867,923]
+ CRUSH rule 1 x 103 [630,47,161,356,911,421,933]
+ CRUSH rule 1 x 104 [758,133,278,11,947,799,401]
+ CRUSH rule 1 x 105 [843,604,47,33,401,632,434]
+ CRUSH rule 1 x 106 [28,681,193,679,990,343,878]
+ CRUSH rule 1 x 107 [74,320,85,819,315,253,589]
+ CRUSH rule 1 x 108 [875,593,575,517,107,153,631]
+ CRUSH rule 1 x 109 [411,985,811,720,198,666,856]
+ CRUSH rule 1 x 110 [440,774,799,660,715,167,510]
+ CRUSH rule 1 x 111 [405,742,276,359,936,360,18]
+ CRUSH rule 1 x 112 [143,181,922,545,185,303,725]
+ CRUSH rule 1 x 113 [153,846,160,903,789,897,738]
+ CRUSH rule 1 x 114 [804,892,939,20,312,692,598]
+ CRUSH rule 1 x 115 [588,508,958,580,232,722,421]
+ CRUSH rule 1 x 116 [327,148,637,486,712,464,9]
+ CRUSH rule 1 x 117 [95,594,989,131,714,275,725]
+ CRUSH rule 1 x 118 [80,957,897,239,359,432,766]
+ CRUSH rule 1 x 119 [386,932,951,768,679,300,570]
+ CRUSH rule 1 x 120 [366,312,653,936,71,241,49]
+ CRUSH rule 1 x 121 [129,154,847,16,471,481,424]
+ CRUSH rule 1 x 122 [873,1,110,939,90,412,551]
+ CRUSH rule 1 x 123 [533,415,789,600,713,800,877]
+ CRUSH rule 1 x 124 [461,691,898,723,957,759,482]
+ CRUSH rule 1 x 125 [342,599,830,402,615,994,736]
+ CRUSH rule 1 x 126 [819,781,822,548,279,255,689]
+ CRUSH rule 1 x 127 [437,893,585,707,353,189,909]
+ CRUSH rule 1 x 128 [679,994,982,550,991,324,666]
+ CRUSH rule 1 x 129 [380,685,947,302,698,144,149]
+ CRUSH rule 1 x 130 [992,52,466,867,998,777,270]
+ CRUSH rule 1 x 131 [469,90,208,599,829,656,203]
+ CRUSH rule 1 x 132 [571,250,316,535,54,418,922]
+ CRUSH rule 1 x 133 [964,728,329,902,108,118,14]
+ CRUSH rule 1 x 134 [999,19,716,963,323,559,893]
+ CRUSH rule 1 x 135 [634,101,52,938,413,573,712]
+ CRUSH rule 1 x 136 [114,889,692,768,694,279,846]
+ CRUSH rule 1 x 137 [839,8,959,280,922,870,363]
+ CRUSH rule 1 x 138 [967,949,138,451,292,548,400]
+ CRUSH rule 1 x 139 [308,711,736,247,632,126,384]
+ CRUSH rule 1 x 140 [764,936,926,55,331,115,178]
+ CRUSH rule 1 x 141 [423,302,112,216,603,873,193]
+ CRUSH rule 1 x 142 [252,821,715,340,635,668,424]
+ CRUSH rule 1 x 143 [33,808,518,477,325,316,266]
+ CRUSH rule 1 x 144 [472,88,969,162,401,771,697]
+ CRUSH rule 1 x 145 [242,208,252,604,266,743,577]
+ CRUSH rule 1 x 146 [290,70,570,384,934,856,929]
+ CRUSH rule 1 x 147 [447,352,657,493,467,918,514]
+ CRUSH rule 1 x 148 [212,644,432,658,109,275,352]
+ CRUSH rule 1 x 149 [9,775,87,35,260,646,406]
+ CRUSH rule 1 x 150 [166,456,582,144,324,340,484]
+ CRUSH rule 1 x 151 [811,875,307,20,782,229,671]
+ CRUSH rule 1 x 152 [449,617,223,9,182,407,807]
+ CRUSH rule 1 x 153 [523,537,695,627,959,613,942]
+ CRUSH rule 1 x 154 [208,559,874,597,243,706,443]
+ CRUSH rule 1 x 155 [569,325,192,296,367,848,58]
+ CRUSH rule 1 x 156 [488,121,521,213,595,837,271]
+ CRUSH rule 1 x 157 [140,723,633,260,487,856,384]
+ CRUSH rule 1 x 158 [786,451,320,239,667,632,899]
+ CRUSH rule 1 x 159 [134,664,517,821,667,944,209]
+ CRUSH rule 1 x 160 [690,112,414,990,183,590,242]
+ CRUSH rule 1 x 161 [324,912,397,423,991,284,909]
+ CRUSH rule 1 x 162 [748,567,284,183,463,336,148]
+ CRUSH rule 1 x 163 [575,499,31,816,749,737,587]
+ CRUSH rule 1 x 164 [314,489,308,326,51,568,110]
+ CRUSH rule 1 x 165 [116,209,750,53,813,640,524]
+ CRUSH rule 1 x 166 [352,706,701,810,718,527,548]
+ CRUSH rule 1 x 167 [27,743,174,142,551,1,935]
+ CRUSH rule 1 x 168 [953,898,880,660,500,799,667]
+ CRUSH rule 1 x 169 [912,147,266,547,331,770,601]
+ CRUSH rule 1 x 170 [421,515,828,844,151,981,835]
+ CRUSH rule 1 x 171 [488,584,880,964,936,196,100]
+ CRUSH rule 1 x 172 [366,443,957,66,162,693,36]
+ CRUSH rule 1 x 173 [863,291,625,287,158,496,471]
+ CRUSH rule 1 x 174 [263,555,650,410,339,616,780]
+ CRUSH rule 1 x 175 [875,961,361,575,33,109,51]
+ CRUSH rule 1 x 176 [745,83,701,680,250,420,240]
+ CRUSH rule 1 x 177 [128,244,41,123,422,902,756]
+ CRUSH rule 1 x 178 [155,41,264,777,314,564,856]
+ CRUSH rule 1 x 179 [593,833,202,183,971,38,724]
+ CRUSH rule 1 x 180 [154,734,17,831,824,522,736]
+ CRUSH rule 1 x 181 [289,675,723,800,166,712,168]
+ CRUSH rule 1 x 182 [730,931,560,209,943,261,485]
+ CRUSH rule 1 x 183 [639,237,794,815,827,400,109]
+ CRUSH rule 1 x 184 [704,312,685,645,691,778,74]
+ CRUSH rule 1 x 185 [97,100,762,82,999,542,485]
+ CRUSH rule 1 x 186 [26,665,554,215,280,421,369]
+ CRUSH rule 1 x 187 [649,14,740,494,402,684,566]
+ CRUSH rule 1 x 188 [682,695,590,743,927,945,833]
+ CRUSH rule 1 x 189 [325,693,726,51,448,169,37]
+ CRUSH rule 1 x 190 [399,933,136,955,57,504,527]
+ CRUSH rule 1 x 191 [629,533,17,126,60,146,999]
+ CRUSH rule 1 x 192 [503,578,38,492,222,251,123]
+ CRUSH rule 1 x 193 [546,333,651,678,823,652,359]
+ CRUSH rule 1 x 194 [242,473,58,655,888,277,792]
+ CRUSH rule 1 x 195 [625,719,135,81,636,513,755]
+ CRUSH rule 1 x 196 [357,114,125,867,250,522,413]
+ CRUSH rule 1 x 197 [306,954,453,873,211,334,666]
+ CRUSH rule 1 x 198 [863,791,311,911,206,61,355]
+ CRUSH rule 1 x 199 [935,906,929,252,893,75,960]
+ CRUSH rule 1 x 200 [373,774,229,454,909,611,132]
+ CRUSH rule 1 x 201 [659,320,477,313,779,16,495]
+ CRUSH rule 1 x 202 [260,433,524,880,223,818,153]
+ CRUSH rule 1 x 203 [36,239,675,971,703,209,669]
+ CRUSH rule 1 x 204 [92,516,993,728,279,478,697]
+ CRUSH rule 1 x 205 [68,395,473,45,683,662,776]
+ CRUSH rule 1 x 206 [570,530,642,380,311,398,230]
+ CRUSH rule 1 x 207 [834,457,850,917,456,296,76]
+ CRUSH rule 1 x 208 [927,484,640,976,803,626,96]
+ CRUSH rule 1 x 209 [878,66,58,940,48,233,522]
+ CRUSH rule 1 x 210 [572,981,484,29,0,426,14]
+ CRUSH rule 1 x 211 [107,597,780,857,895,57,922]
+ CRUSH rule 1 x 212 [389,107,838,624,698,562,857]
+ CRUSH rule 1 x 213 [497,717,567,728,905,134,687]
+ CRUSH rule 1 x 214 [798,65,254,572,32,393,579]
+ CRUSH rule 1 x 215 [233,419,283,638,520,891,982]
+ CRUSH rule 1 x 216 [494,464,742,523,459,174,973]
+ CRUSH rule 1 x 217 [352,396,309,938,66,41,264]
+ CRUSH rule 1 x 218 [895,864,988,650,593,740,34]
+ CRUSH rule 1 x 219 [222,534,277,242,658,482,697]
+ CRUSH rule 1 x 220 [281,19,584,563,858,965,686]
+ CRUSH rule 1 x 221 [64,928,963,130,312,394,61]
+ CRUSH rule 1 x 222 [40,544,161,199,861,644,597]
+ CRUSH rule 1 x 223 [645,556,159,417,46,135,465]
+ CRUSH rule 1 x 224 [647,165,957,263,961,576,329]
+ CRUSH rule 1 x 225 [219,714,858,747,461,175,606]
+ CRUSH rule 1 x 226 [372,511,181,277,695,404,876]
+ CRUSH rule 1 x 227 [925,156,714,863,257,74,966]
+ CRUSH rule 1 x 228 [682,404,839,263,521,195,261]
+ CRUSH rule 1 x 229 [880,838,770,891,236,542,262]
+ CRUSH rule 1 x 230 [328,659,916,468,646,572,93]
+ CRUSH rule 1 x 231 [320,383,669,109,627,621,50]
+ CRUSH rule 1 x 232 [924,846,394,319,43,519,106]
+ CRUSH rule 1 x 233 [948,652,575,838,498,395,796]
+ CRUSH rule 1 x 234 [484,943,42,575,936,180,103]
+ CRUSH rule 1 x 235 [750,65,590,168,870,308,471]
+ CRUSH rule 1 x 236 [551,787,490,136,370,833,573]
+ CRUSH rule 1 x 237 [390,157,166,251,752,75,327]
+ CRUSH rule 1 x 238 [570,6,989,707,514,905,894]
+ CRUSH rule 1 x 239 [729,959,376,975,496,49,426]
+ CRUSH rule 1 x 240 [981,241,156,767,631,576,450]
+ CRUSH rule 1 x 241 [310,816,641,177,996,454,413]
+ CRUSH rule 1 x 242 [161,63,642,837,763,458,234]
+ CRUSH rule 1 x 243 [180,394,33,683,189,419,799]
+ CRUSH rule 1 x 244 [52,174,685,189,78,310,785]
+ CRUSH rule 1 x 245 [523,121,915,84,386,409,605]
+ CRUSH rule 1 x 246 [362,893,390,487,817,88,989]
+ CRUSH rule 1 x 247 [382,184,116,34,143,15,590]
+ CRUSH rule 1 x 248 [129,114,852,469,359,291,713]
+ CRUSH rule 1 x 249 [159,683,91,856,475,369,886]
+ CRUSH rule 1 x 250 [404,945,569,955,228,910,270]
+ CRUSH rule 1 x 251 [661,225,738,757,37,642,58]
+ CRUSH rule 1 x 252 [961,226,542,103,945,885,838]
+ CRUSH rule 1 x 253 [651,97,225,364,189,248,797]
+ CRUSH rule 1 x 254 [123,33,741,692,599,11,605]
+ CRUSH rule 1 x 255 [314,649,891,855,517,344,607]
+ CRUSH rule 1 x 256 [315,215,651,126,470,849,189]
+ CRUSH rule 1 x 257 [825,264,867,681,529,409,291]
+ CRUSH rule 1 x 258 [624,789,370,723,131,982,863]
+ CRUSH rule 1 x 259 [602,542,70,563,947,723,77]
+ CRUSH rule 1 x 260 [717,878,43,56,377,481,533]
+ CRUSH rule 1 x 261 [145,517,20,903,786,939,516]
+ CRUSH rule 1 x 262 [223,1,561,420,209,16,88]
+ CRUSH rule 1 x 263 [462,211,405,508,787,669,773]
+ CRUSH rule 1 x 264 [654,471,266,662,135,564,715]
+ CRUSH rule 1 x 265 [302,794,704,798,659,487,833]
+ CRUSH rule 1 x 266 [202,132,884,209,551,984,7]
+ CRUSH rule 1 x 267 [282,938,657,113,672,993,972]
+ CRUSH rule 1 x 268 [338,309,356,278,928,797,715]
+ CRUSH rule 1 x 269 [738,122,266,200,894,118,146]
+ CRUSH rule 1 x 270 [707,982,946,196,407,804,476]
+ CRUSH rule 1 x 271 [705,432,364,735,512,595,263]
+ CRUSH rule 1 x 272 [756,545,942,56,542,449,710]
+ CRUSH rule 1 x 273 [197,502,527,721,239,648,982]
+ CRUSH rule 1 x 274 [992,44,653,573,527,702,370]
+ CRUSH rule 1 x 275 [544,789,170,434,23,926,992]
+ CRUSH rule 1 x 276 [658,467,577,268,336,5,634]
+ CRUSH rule 1 x 277 [143,490,880,483,928,272,783]
+ CRUSH rule 1 x 278 [492,647,355,282,834,64,350]
+ CRUSH rule 1 x 279 [517,792,604,987,527,894,952]
+ CRUSH rule 1 x 280 [825,740,27,848,514,750,895]
+ CRUSH rule 1 x 281 [224,629,120,562,616,200,443]
+ CRUSH rule 1 x 282 [298,661,380,416,35,585,939]
+ CRUSH rule 1 x 283 [311,606,208,50,913,678,369]
+ CRUSH rule 1 x 284 [771,466,371,743,672,119,60]
+ CRUSH rule 1 x 285 [693,362,404,676,797,531,582]
+ CRUSH rule 1 x 286 [364,477,285,167,270,617,699]
+ CRUSH rule 1 x 287 [591,611,828,995,170,987,137]
+ CRUSH rule 1 x 288 [965,541,848,796,251,668,195]
+ CRUSH rule 1 x 289 [225,551,948,877,219,167,795]
+ CRUSH rule 1 x 290 [577,762,777,751,291,349,473]
+ CRUSH rule 1 x 291 [160,903,477,381,490,559,557]
+ CRUSH rule 1 x 292 [873,598,216,666,222,228,806]
+ CRUSH rule 1 x 293 [100,234,874,47,28,452,775]
+ CRUSH rule 1 x 294 [285,943,379,520,725,547,459]
+ CRUSH rule 1 x 295 [938,262,880,327,687,3,440]
+ CRUSH rule 1 x 296 [850,327,86,472,1,776,266]
+ CRUSH rule 1 x 297 [951,53,99,558,753,228,232]
+ CRUSH rule 1 x 298 [173,336,85,766,910,657,213]
+ CRUSH rule 1 x 299 [598,591,315,386,895,296,924]
+ CRUSH rule 1 x 300 [531,957,62,459,156,538,904]
+ CRUSH rule 1 x 301 [823,628,23,858,629,808,220]
+ CRUSH rule 1 x 302 [184,80,780,871,531,211,400]
+ CRUSH rule 1 x 303 [521,766,222,830,988,275,561]
+ CRUSH rule 1 x 304 [980,127,807,507,555,245,214]
+ CRUSH rule 1 x 305 [153,816,22,927,696,911,685]
+ CRUSH rule 1 x 306 [423,739,664,753,178,431,761]
+ CRUSH rule 1 x 307 [997,557,682,456,479,631,459]
+ CRUSH rule 1 x 308 [991,874,534,465,330,284,976]
+ CRUSH rule 1 x 309 [860,394,724,858,246,866,857]
+ CRUSH rule 1 x 310 [589,818,546,201,94,653,90]
+ CRUSH rule 1 x 311 [477,774,225,590,830,559,256]
+ CRUSH rule 1 x 312 [887,853,950,354,58,23,497]
+ CRUSH rule 1 x 313 [802,646,447,416,557,118,24]
+ CRUSH rule 1 x 314 [654,974,229,511,562,916,952]
+ CRUSH rule 1 x 315 [767,227,28,740,828,156,749]
+ CRUSH rule 1 x 316 [778,83,733,359,858,319,761]
+ CRUSH rule 1 x 317 [184,418,642,986,939,675,892]
+ CRUSH rule 1 x 318 [525,410,500,543,212,95,290]
+ CRUSH rule 1 x 319 [476,724,569,382,409,521,800]
+ CRUSH rule 1 x 320 [149,610,697,296,818,955,523]
+ CRUSH rule 1 x 321 [710,79,667,671,234,4,868]
+ CRUSH rule 1 x 322 [175,275,323,333,744,718,187]
+ CRUSH rule 1 x 323 [819,604,638,792,316,544,236]
+ CRUSH rule 1 x 324 [16,745,511,439,272,294,668]
+ CRUSH rule 1 x 325 [486,400,872,873,251,68,462]
+ CRUSH rule 1 x 326 [613,765,207,19,359,370,461]
+ CRUSH rule 1 x 327 [125,289,738,408,456,784,750]
+ CRUSH rule 1 x 328 [807,383,476,583,645,141,33]
+ CRUSH rule 1 x 329 [588,938,599,432,446,840,516]
+ CRUSH rule 1 x 330 [932,644,41,611,209,406,420]
+ CRUSH rule 1 x 331 [341,953,950,537,578,862,624]
+ CRUSH rule 1 x 332 [153,726,459,950,466,804,644]
+ CRUSH rule 1 x 333 [745,845,853,860,52,615,243]
+ CRUSH rule 1 x 334 [614,751,807,58,396,159,408]
+ CRUSH rule 1 x 335 [518,721,221,283,454,187,635]
+ CRUSH rule 1 x 336 [389,424,77,309,5,898,698]
+ CRUSH rule 1 x 337 [753,508,765,720,221,807,956]
+ CRUSH rule 1 x 338 [128,810,490,753,406,760,69]
+ CRUSH rule 1 x 339 [430,308,58,751,856,823,607]
+ CRUSH rule 1 x 340 [541,44,630,231,289,966,707]
+ CRUSH rule 1 x 341 [402,26,631,439,165,928,720]
+ CRUSH rule 1 x 342 [982,57,992,461,131,32,516]
+ CRUSH rule 1 x 343 [833,412,572,732,107,805,660]
+ CRUSH rule 1 x 344 [784,533,792,41,642,869,142]
+ CRUSH rule 1 x 345 [546,300,304,691,763,556,127]
+ CRUSH rule 1 x 346 [302,420,428,891,357,124,419]
+ CRUSH rule 1 x 347 [488,778,101,217,366,442,783]
+ CRUSH rule 1 x 348 [903,744,937,718,85,314,862]
+ CRUSH rule 1 x 349 [471,547,582,306,600,486,795]
+ CRUSH rule 1 x 350 [348,221,823,335,383,708,841]
+ CRUSH rule 1 x 351 [961,582,705,346,361,32,766]
+ CRUSH rule 1 x 352 [728,137,461,298,36,903,899]
+ CRUSH rule 1 x 353 [904,202,184,447,58,294,279]
+ CRUSH rule 1 x 354 [345,226,319,256,544,311,612]
+ CRUSH rule 1 x 355 [50,430,175,43,187,458,985]
+ CRUSH rule 1 x 356 [87,185,55,423,829,1,629]
+ CRUSH rule 1 x 357 [762,459,921,473,182,231,891]
+ CRUSH rule 1 x 358 [908,25,280,6,808,676,874]
+ CRUSH rule 1 x 359 [484,15,132,121,394,423,397]
+ CRUSH rule 1 x 360 [173,378,337,702,145,499,29]
+ CRUSH rule 1 x 361 [404,577,115,25,56,914,643]
+ CRUSH rule 1 x 362 [403,1,422,945,132,685,265]
+ CRUSH rule 1 x 363 [639,911,510,162,418,294,444]
+ CRUSH rule 1 x 364 [752,689,610,990,665,222,203]
+ CRUSH rule 1 x 365 [956,999,212,230,624,84,113]
+ CRUSH rule 1 x 366 [860,925,924,763,687,851,59]
+ CRUSH rule 1 x 367 [205,609,647,665,969,720,685]
+ CRUSH rule 1 x 368 [301,284,810,169,78,340,616]
+ CRUSH rule 1 x 369 [452,658,339,217,674,210,284]
+ CRUSH rule 1 x 370 [11,467,695,989,394,576,850]
+ CRUSH rule 1 x 371 [124,487,55,514,313,411,797]
+ CRUSH rule 1 x 372 [253,48,979,846,207,631,212]
+ CRUSH rule 1 x 373 [715,605,775,748,227,493,128]
+ CRUSH rule 1 x 374 [191,887,920,723,223,714,961]
+ CRUSH rule 1 x 375 [711,385,651,665,15,71,934]
+ CRUSH rule 1 x 376 [597,818,49,458,415,755,446]
+ CRUSH rule 1 x 377 [294,256,933,771,184,861,654]
+ CRUSH rule 1 x 378 [34,151,681,707,552,127,728]
+ CRUSH rule 1 x 379 [869,136,315,378,813,153,115]
+ CRUSH rule 1 x 380 [294,97,575,791,690,482,255]
+ CRUSH rule 1 x 381 [119,710,219,827,328,886,773]
+ CRUSH rule 1 x 382 [69,631,508,706,697,168,276]
+ CRUSH rule 1 x 383 [922,588,589,925,471,601,29]
+ CRUSH rule 1 x 384 [221,945,671,117,857,655,488]
+ CRUSH rule 1 x 385 [561,737,953,723,658,368,910]
+ CRUSH rule 1 x 386 [335,442,788,696,507,716,232]
+ CRUSH rule 1 x 387 [514,43,353,88,100,842,164]
+ CRUSH rule 1 x 388 [587,89,157,996,915,927,474]
+ CRUSH rule 1 x 389 [109,641,255,466,372,563,340]
+ CRUSH rule 1 x 390 [925,149,421,489,599,810,852]
+ CRUSH rule 1 x 391 [267,87,387,527,768,873,735]
+ CRUSH rule 1 x 392 [382,485,370,849,936,636,901]
+ CRUSH rule 1 x 393 [425,721,221,753,268,463,652]
+ CRUSH rule 1 x 394 [898,18,38,793,173,738,15]
+ CRUSH rule 1 x 395 [806,876,269,679,32,744,126]
+ CRUSH rule 1 x 396 [790,970,437,449,875,395,726]
+ CRUSH rule 1 x 397 [136,363,507,613,11,30,996]
+ CRUSH rule 1 x 398 [914,116,558,258,722,904,349]
+ CRUSH rule 1 x 399 [261,94,299,202,174,622,749]
+ CRUSH rule 1 x 400 [661,197,338,461,977,848,536]
+ CRUSH rule 1 x 401 [953,979,287,803,41,349,79]
+ CRUSH rule 1 x 402 [738,819,618,522,667,334,658]
+ CRUSH rule 1 x 403 [573,238,425,546,130,68,202]
+ CRUSH rule 1 x 404 [526,848,790,253,922,820,299]
+ CRUSH rule 1 x 405 [582,505,330,334,201,110,776]
+ CRUSH rule 1 x 406 [768,324,493,60,186,165,718]
+ CRUSH rule 1 x 407 [260,951,437,587,692,648,72]
+ CRUSH rule 1 x 408 [657,81,770,734,830,821,246]
+ CRUSH rule 1 x 409 [498,89,182,423,672,152,213]
+ CRUSH rule 1 x 410 [28,793,737,352,166,645,949]
+ CRUSH rule 1 x 411 [684,992,60,659,769,267,313]
+ CRUSH rule 1 x 412 [261,958,699,950,165,14,560]
+ CRUSH rule 1 x 413 [891,835,297,441,384,979,618]
+ CRUSH rule 1 x 414 [127,459,119,965,662,594,97]
+ CRUSH rule 1 x 415 [272,540,631,328,609,568,694]
+ CRUSH rule 1 x 416 [739,617,115,530,339,371,889]
+ CRUSH rule 1 x 417 [106,209,157,878,117,128,138]
+ CRUSH rule 1 x 418 [525,441,147,390,320,300,848]
+ CRUSH rule 1 x 419 [603,673,615,465,266,855,823]
+ CRUSH rule 1 x 420 [988,213,251,226,209,245,506]
+ CRUSH rule 1 x 421 [761,521,748,368,923,992,764]
+ CRUSH rule 1 x 422 [317,160,924,548,198,709,839]
+ CRUSH rule 1 x 423 [137,807,168,472,619,443,905]
+ CRUSH rule 1 x 424 [920,37,146,263,598,748,785]
+ CRUSH rule 1 x 425 [277,693,285,221,478,165,80]
+ CRUSH rule 1 x 426 [485,936,407,854,726,524,791]
+ CRUSH rule 1 x 427 [242,515,9,564,174,453,334]
+ CRUSH rule 1 x 428 [632,635,26,473,494,478,225]
+ CRUSH rule 1 x 429 [641,73,465,127,171,397,857]
+ CRUSH rule 1 x 430 [626,585,6,387,881,583,859]
+ CRUSH rule 1 x 431 [697,76,753,570,964,339,194]
+ CRUSH rule 1 x 432 [590,526,306,283,656,728,513]
+ CRUSH rule 1 x 433 [284,387,149,817,886,714,52]
+ CRUSH rule 1 x 434 [538,985,79,953,770,468,644]
+ CRUSH rule 1 x 435 [30,318,593,635,975,833,371]
+ CRUSH rule 1 x 436 [164,919,851,693,0,874,10]
+ CRUSH rule 1 x 437 [322,212,163,606,302,282,443]
+ CRUSH rule 1 x 438 [142,392,85,594,376,419,755]
+ CRUSH rule 1 x 439 [119,370,68,443,997,837,414]
+ CRUSH rule 1 x 440 [333,403,187,863,475,844,800]
+ CRUSH rule 1 x 441 [477,727,906,145,429,91,205]
+ CRUSH rule 1 x 442 [274,590,933,244,434,49,864]
+ CRUSH rule 1 x 443 [983,748,574,718,700,442,774]
+ CRUSH rule 1 x 444 [536,509,431,146,170,149,182]
+ CRUSH rule 1 x 445 [485,638,528,209,964,753,554]
+ CRUSH rule 1 x 446 [345,634,42,294,711,376,314]
+ CRUSH rule 1 x 447 [61,845,767,600,321,716,58]
+ CRUSH rule 1 x 448 [333,232,292,846,364,951,807]
+ CRUSH rule 1 x 449 [680,16,484,670,851,500,258]
+ CRUSH rule 1 x 450 [235,214,79,423,96,822,721]
+ CRUSH rule 1 x 451 [961,468,333,640,823,151,878]
+ CRUSH rule 1 x 452 [525,479,153,528,570,806,604]
+ CRUSH rule 1 x 453 [138,466,302,86,249,154,514]
+ CRUSH rule 1 x 454 [137,625,215,402,389,914,106]
+ CRUSH rule 1 x 455 [173,150,997,16,846,888,295]
+ CRUSH rule 1 x 456 [235,226,238,258,347,784,504]
+ CRUSH rule 1 x 457 [450,577,253,413,717,609,762]
+ CRUSH rule 1 x 458 [195,537,91,814,351,90,399]
+ CRUSH rule 1 x 459 [381,555,312,573,915,623,147]
+ CRUSH rule 1 x 460 [972,730,534,678,756,692,841]
+ CRUSH rule 1 x 461 [506,279,142,830,784,124,385]
+ CRUSH rule 1 x 462 [692,959,578,57,983,299,240]
+ CRUSH rule 1 x 463 [788,667,949,550,685,702,538]
+ CRUSH rule 1 x 464 [133,122,588,999,270,880,789]
+ CRUSH rule 1 x 465 [971,190,230,777,452,914,137]
+ CRUSH rule 1 x 466 [394,576,148,157,103,822,659]
+ CRUSH rule 1 x 467 [517,28,366,362,984,521,187]
+ CRUSH rule 1 x 468 [829,143,874,225,162,413,201]
+ CRUSH rule 1 x 469 [987,936,106,725,633,238,681]
+ CRUSH rule 1 x 470 [107,982,56,889,67,65,558]
+ CRUSH rule 1 x 471 [181,897,629,860,307,116,256]
+ CRUSH rule 1 x 472 [547,512,172,24,705,837,809]
+ CRUSH rule 1 x 473 [760,997,824,905,888,755,756]
+ CRUSH rule 1 x 474 [787,418,743,628,272,341,446]
+ CRUSH rule 1 x 475 [662,312,253,617,105,58,237]
+ CRUSH rule 1 x 476 [110,495,185,508,961,837,984]
+ CRUSH rule 1 x 477 [393,954,834,132,841,367,753]
+ CRUSH rule 1 x 478 [246,483,480,644,985,420,941]
+ CRUSH rule 1 x 479 [70,929,697,931,744,487,158]
+ CRUSH rule 1 x 480 [753,119,961,607,317,717,371]
+ CRUSH rule 1 x 481 [470,429,677,242,574,757,135]
+ CRUSH rule 1 x 482 [451,566,961,675,354,746,731]
+ CRUSH rule 1 x 483 [816,72,371,278,635,30,448]
+ CRUSH rule 1 x 484 [540,454,389,31,654,494,283]
+ CRUSH rule 1 x 485 [74,582,624,684,566,677,866]
+ CRUSH rule 1 x 486 [958,595,199,763,715,973,621]
+ CRUSH rule 1 x 487 [228,302,804,833,876,647,857]
+ CRUSH rule 1 x 488 [180,529,722,956,353,890,924]
+ CRUSH rule 1 x 489 [47,617,812,187,291,828,154]
+ CRUSH rule 1 x 490 [905,822,479,124,750,843,566]
+ CRUSH rule 1 x 491 [892,370,609,998,433,957,188]
+ CRUSH rule 1 x 492 [588,959,127,948,505,936,591]
+ CRUSH rule 1 x 493 [353,461,593,291,301,830,231]
+ CRUSH rule 1 x 494 [378,848,443,368,507,423,389]
+ CRUSH rule 1 x 495 [845,653,768,234,405,367,823]
+ CRUSH rule 1 x 496 [13,988,0,691,389,757,129]
+ CRUSH rule 1 x 497 [796,877,788,394,648,829,542]
+ CRUSH rule 1 x 498 [412,337,270,705,511,227,949]
+ CRUSH rule 1 x 499 [330,695,8,74,618,101,440]
+ CRUSH rule 1 x 500 [820,272,547,765,755,96,930]
+ CRUSH rule 1 x 501 [110,44,132,442,294,423,880]
+ CRUSH rule 1 x 502 [336,595,650,274,993,312,490]
+ CRUSH rule 1 x 503 [922,211,157,722,502,971,262]
+ CRUSH rule 1 x 504 [483,52,122,432,778,461,758]
+ CRUSH rule 1 x 505 [482,598,224,279,480,310,764]
+ CRUSH rule 1 x 506 [493,123,43,856,936,622,898]
+ CRUSH rule 1 x 507 [12,598,264,422,416,947,591]
+ CRUSH rule 1 x 508 [227,157,611,301,223,746,313]
+ CRUSH rule 1 x 509 [807,242,363,122,582,530,798]
+ CRUSH rule 1 x 510 [134,437,227,75,313,351,786]
+ CRUSH rule 1 x 511 [212,54,83,799,457,218,600]
+ CRUSH rule 1 x 512 [236,630,758,752,361,249,899]
+ CRUSH rule 1 x 513 [994,693,644,938,846,685,52]
+ CRUSH rule 1 x 514 [45,508,831,19,817,52,374]
+ CRUSH rule 1 x 515 [504,138,480,272,530,377,481]
+ CRUSH rule 1 x 516 [285,409,136,570,841,610,453]
+ CRUSH rule 1 x 517 [300,232,23,906,438,236,519]
+ CRUSH rule 1 x 518 [397,674,98,898,967,113,625]
+ CRUSH rule 1 x 519 [86,750,772,913,101,864,375]
+ CRUSH rule 1 x 520 [900,833,614,130,261,885,558]
+ CRUSH rule 1 x 521 [31,47,236,751,911,599,495]
+ CRUSH rule 1 x 522 [390,16,280,144,291,175,753]
+ CRUSH rule 1 x 523 [618,308,424,590,300,206,834]
+ CRUSH rule 1 x 524 [635,189,687,963,601,518,8]
+ CRUSH rule 1 x 525 [311,916,699,262,775,32,45]
+ CRUSH rule 1 x 526 [48,738,227,718,244,942,853]
+ CRUSH rule 1 x 527 [202,851,889,216,763,351,270]
+ CRUSH rule 1 x 528 [565,827,590,273,918,106,651]
+ CRUSH rule 1 x 529 [934,864,241,43,466,924,278]
+ CRUSH rule 1 x 530 [502,934,298,670,986,360,577]
+ CRUSH rule 1 x 531 [681,627,942,487,288,561,925]
+ CRUSH rule 1 x 532 [422,6,147,205,861,141,949]
+ CRUSH rule 1 x 533 [863,68,364,983,247,199,54]
+ CRUSH rule 1 x 534 [962,931,775,172,663,119,206]
+ CRUSH rule 1 x 535 [89,565,397,693,839,632,859]
+ CRUSH rule 1 x 536 [499,351,760,458,918,86,148]
+ CRUSH rule 1 x 537 [676,547,787,311,867,748,152]
+ CRUSH rule 1 x 538 [58,644,571,649,941,7,37]
+ CRUSH rule 1 x 539 [837,953,457,711,458,621,528]
+ CRUSH rule 1 x 540 [831,50,132,213,197,709,95]
+ CRUSH rule 1 x 541 [582,757,121,525,532,963,738]
+ CRUSH rule 1 x 542 [472,132,790,997,948,269,137]
+ CRUSH rule 1 x 543 [382,272,797,330,315,748,324]
+ CRUSH rule 1 x 544 [947,930,496,883,509,219,250]
+ CRUSH rule 1 x 545 [425,570,305,77,821,422,117]
+ CRUSH rule 1 x 546 [18,65,529,437,343,547,699]
+ CRUSH rule 1 x 547 [445,715,600,472,213,851,428]
+ CRUSH rule 1 x 548 [367,569,980,167,627,442,517]
+ CRUSH rule 1 x 549 [125,715,671,817,285,420,37]
+ CRUSH rule 1 x 550 [425,599,744,199,923,222,915]
+ CRUSH rule 1 x 551 [44,1,528,922,944,115,161]
+ CRUSH rule 1 x 552 [246,104,68,239,123,427,57]
+ CRUSH rule 1 x 553 [71,703,615,28,593,724,218]
+ CRUSH rule 1 x 554 [207,124,217,166,525,226,693]
+ CRUSH rule 1 x 555 [570,28,317,420,931,413,623]
+ CRUSH rule 1 x 556 [674,152,421,79,215,347,830]
+ CRUSH rule 1 x 557 [347,817,191,391,741,571,593]
+ CRUSH rule 1 x 558 [627,426,369,692,815,371,124]
+ CRUSH rule 1 x 559 [940,630,924,242,224,912,185]
+ CRUSH rule 1 x 560 [295,903,541,29,245,753,887]
+ CRUSH rule 1 x 561 [506,682,384,637,878,991,700]
+ CRUSH rule 1 x 562 [718,529,87,729,842,341,62]
+ CRUSH rule 1 x 563 [552,332,747,206,274,871,903]
+ CRUSH rule 1 x 564 [835,769,736,486,630,209,641]
+ CRUSH rule 1 x 565 [8,167,539,182,607,62,738]
+ CRUSH rule 1 x 566 [600,481,301,263,90,450,184]
+ CRUSH rule 1 x 567 [999,994,509,899,947,24,267]
+ CRUSH rule 1 x 568 [252,431,157,62,601,863,398]
+ CRUSH rule 1 x 569 [643,218,943,455,83,969,494]
+ CRUSH rule 1 x 570 [617,635,765,422,250,156,533]
+ CRUSH rule 1 x 571 [757,80,59,98,328,700,329]
+ CRUSH rule 1 x 572 [299,348,575,889,943,675,33]
+ CRUSH rule 1 x 573 [25,505,270,167,58,901,878]
+ CRUSH rule 1 x 574 [215,431,624,177,628,814,333]
+ CRUSH rule 1 x 575 [225,252,611,546,32,815,389]
+ CRUSH rule 1 x 576 [627,94,159,857,430,691,177]
+ CRUSH rule 1 x 577 [237,809,778,636,61,167,700]
+ CRUSH rule 1 x 578 [885,313,120,344,771,614,487]
+ CRUSH rule 1 x 579 [924,575,787,831,47,996,557]
+ CRUSH rule 1 x 580 [718,51,766,121,118,471,608]
+ CRUSH rule 1 x 581 [219,807,129,571,856,179,874]
+ CRUSH rule 1 x 582 [893,701,598,863,285,829,984]
+ CRUSH rule 1 x 583 [246,930,964,170,993,409,469]
+ CRUSH rule 1 x 584 [336,432,680,175,495,839,642]
+ CRUSH rule 1 x 585 [324,999,397,485,457,527,73]
+ CRUSH rule 1 x 586 [558,230,976,541,816,72,794]
+ CRUSH rule 1 x 587 [985,830,597,21,308,890,952]
+ CRUSH rule 1 x 588 [211,544,57,134,162,496,195]
+ CRUSH rule 1 x 589 [129,21,112,190,885,844,753]
+ CRUSH rule 1 x 590 [467,969,652,593,287,76,811]
+ CRUSH rule 1 x 591 [758,514,316,164,35,110,54]
+ CRUSH rule 1 x 592 [525,253,190,443,315,603,667]
+ CRUSH rule 1 x 593 [601,885,339,152,297,223,269]
+ CRUSH rule 1 x 594 [227,60,450,30,717,840,994]
+ CRUSH rule 1 x 595 [720,854,496,912,80,655,917]
+ CRUSH rule 1 x 596 [751,195,997,77,261,490,180]
+ CRUSH rule 1 x 597 [129,574,714,8,789,847,725]
+ CRUSH rule 1 x 598 [679,207,604,396,841,284,286]
+ CRUSH rule 1 x 599 [668,315,683,349,681,253,599]
+ CRUSH rule 1 x 600 [143,396,464,444,59,57,243]
+ CRUSH rule 1 x 601 [326,573,873,902,136,921,633]
+ CRUSH rule 1 x 602 [860,281,875,535,672,474,697]
+ CRUSH rule 1 x 603 [709,328,445,349,190,455,924]
+ CRUSH rule 1 x 604 [571,62,814,95,866,978,983]
+ CRUSH rule 1 x 605 [252,739,860,27,313,362,857]
+ CRUSH rule 1 x 606 [339,236,759,842,67,644,954]
+ CRUSH rule 1 x 607 [590,248,759,868,433,398,578]
+ CRUSH rule 1 x 608 [145,635,309,467,875,115,148]
+ CRUSH rule 1 x 609 [973,547,223,79,762,863,249]
+ CRUSH rule 1 x 610 [435,816,961,983,255,886,160]
+ CRUSH rule 1 x 611 [559,283,422,584,176,429,570]
+ CRUSH rule 1 x 612 [273,149,123,576,911,270,296]
+ CRUSH rule 1 x 613 [828,614,642,674,33,361,958]
+ CRUSH rule 1 x 614 [478,748,393,34,171,80,92]
+ CRUSH rule 1 x 615 [392,155,144,326,626,134,149]
+ CRUSH rule 1 x 616 [778,637,452,248,15,888,74]
+ CRUSH rule 1 x 617 [622,713,996,833,611,407,364]
+ CRUSH rule 1 x 618 [149,877,270,329,180,327,222]
+ CRUSH rule 1 x 619 [604,163,656,409,322,848,519]
+ CRUSH rule 1 x 620 [181,23,409,198,64,898,35]
+ CRUSH rule 1 x 621 [735,902,386,237,939,475,725]
+ CRUSH rule 1 x 622 [661,824,717,568,858,583,446]
+ CRUSH rule 1 x 623 [142,121,643,61,695,852,485]
+ CRUSH rule 1 x 624 [360,716,420,398,49,717,137]
+ CRUSH rule 1 x 625 [541,167,385,1,601,481,308]
+ CRUSH rule 1 x 626 [364,431,610,363,535,747,225]
+ CRUSH rule 1 x 627 [458,137,557,410,287,749,467]
+ CRUSH rule 1 x 628 [250,350,556,497,821,65,205]
+ CRUSH rule 1 x 629 [928,160,710,572,365,772,538]
+ CRUSH rule 1 x 630 [243,19,918,556,601,16,920]
+ CRUSH rule 1 x 631 [438,221,574,676,797,580,219]
+ CRUSH rule 1 x 632 [797,368,247,5,32,102,416]
+ CRUSH rule 1 x 633 [993,749,525,485,27,330,275]
+ CRUSH rule 1 x 634 [239,351,633,299,651,678,296]
+ CRUSH rule 1 x 635 [640,965,25,961,306,172,849]
+ CRUSH rule 1 x 636 [173,290,297,991,937,823,236]
+ CRUSH rule 1 x 637 [0,918,98,108,111,495,887]
+ CRUSH rule 1 x 638 [702,235,424,900,983,754,701]
+ CRUSH rule 1 x 639 [475,687,31,785,918,611,27]
+ CRUSH rule 1 x 640 [31,664,399,677,123,609,858]
+ CRUSH rule 1 x 641 [296,473,108,963,341,876,897]
+ CRUSH rule 1 x 642 [894,273,427,606,677,670,610]
+ CRUSH rule 1 x 643 [117,111,732,191,114,153,500]
+ CRUSH rule 1 x 644 [438,336,327,512,599,862,660]
+ CRUSH rule 1 x 645 [982,702,351,573,907,915,279]
+ CRUSH rule 1 x 646 [334,804,146,842,697,638,720]
+ CRUSH rule 1 x 647 [933,787,185,334,752,285,372]
+ CRUSH rule 1 x 648 [22,444,400,862,207,842,453]
+ CRUSH rule 1 x 649 [503,229,213,460,639,760,722]
+ CRUSH rule 1 x 650 [328,659,420,443,739,950,869]
+ CRUSH rule 1 x 651 [3,880,823,123,378,585,715]
+ CRUSH rule 1 x 652 [495,977,563,733,92,997,119]
+ CRUSH rule 1 x 653 [185,718,804,280,975,912,198]
+ CRUSH rule 1 x 654 [130,528,380,81,906,511,904]
+ CRUSH rule 1 x 655 [560,872,454,504,319,284,605]
+ CRUSH rule 1 x 656 [219,885,178,981,863,508,708]
+ CRUSH rule 1 x 657 [233,684,813,490,208,941,858]
+ CRUSH rule 1 x 658 [778,6,756,380,750,836,547]
+ CRUSH rule 1 x 659 [240,663,306,540,789,902,170]
+ CRUSH rule 1 x 660 [244,855,196,147,678,323,63]
+ CRUSH rule 1 x 661 [184,270,128,398,910,230,402]
+ CRUSH rule 1 x 662 [65,883,921,438,79,957,464]
+ CRUSH rule 1 x 663 [323,721,594,812,43,992,170]
+ CRUSH rule 1 x 664 [865,113,512,51,427,123,585]
+ CRUSH rule 1 x 665 [420,850,591,475,202,733,798]
+ CRUSH rule 1 x 666 [319,767,246,3,369,493,796]
+ CRUSH rule 1 x 667 [875,39,343,100,829,2,795]
+ CRUSH rule 1 x 668 [331,122,263,599,355,484,943]
+ CRUSH rule 1 x 669 [915,521,402,747,673,445,938]
+ CRUSH rule 1 x 670 [845,659,943,447,401,322,168]
+ CRUSH rule 1 x 671 [108,634,527,363,856,238,755]
+ CRUSH rule 1 x 672 [578,216,110,589,302,137,954]
+ CRUSH rule 1 x 673 [442,74,579,797,622,950,371]
+ CRUSH rule 1 x 674 [588,364,281,308,645,631,229]
+ CRUSH rule 1 x 675 [489,698,744,671,870,174,528]
+ CRUSH rule 1 x 676 [928,911,40,180,722,729,673]
+ CRUSH rule 1 x 677 [399,269,692,131,615,136,103]
+ CRUSH rule 1 x 678 [546,752,544,155,5,463,666]
+ CRUSH rule 1 x 679 [988,25,275,433,628,57,247]
+ CRUSH rule 1 x 680 [335,963,382,486,749,257,795]
+ CRUSH rule 1 x 681 [690,462,623,466,49,471,774]
+ CRUSH rule 1 x 682 [196,588,154,257,807,776,367]
+ CRUSH rule 1 x 683 [627,25,421,160,873,102,345]
+ CRUSH rule 1 x 684 [38,804,592,158,991,264,652]
+ CRUSH rule 1 x 685 [841,368,548,362,166,211,154]
+ CRUSH rule 1 x 686 [336,287,525,440,166,993,911]
+ CRUSH rule 1 x 687 [20,682,924,653,356,16,917]
+ CRUSH rule 1 x 688 [463,371,780,556,385,883,115]
+ CRUSH rule 1 x 689 [569,250,78,816,847,775,333]
+ CRUSH rule 1 x 690 [551,144,587,263,378,394,970]
+ CRUSH rule 1 x 691 [766,464,446,533,449,541,451]
+ CRUSH rule 1 x 692 [739,634,18,245,624,35,268]
+ CRUSH rule 1 x 693 [339,297,118,330,817,91,828]
+ CRUSH rule 1 x 694 [405,26,830,181,533,166,488]
+ CRUSH rule 1 x 695 [622,576,597,535,600,593,300]
+ CRUSH rule 1 x 696 [558,902,689,13,715,28,664]
+ CRUSH rule 1 x 697 [818,222,406,691,427,863,153]
+ CRUSH rule 1 x 698 [178,48,402,233,841,604,468]
+ CRUSH rule 1 x 699 [450,244,180,919,335,332,747]
+ CRUSH rule 1 x 700 [502,771,987,706,416,240,68]
+ CRUSH rule 1 x 701 [4,612,782,216,853,303,585]
+ CRUSH rule 1 x 702 [177,630,232,923,281,708,466]
+ CRUSH rule 1 x 703 [354,178,389,393,778,803,796]
+ CRUSH rule 1 x 704 [646,601,156,171,603,116,655]
+ CRUSH rule 1 x 705 [921,401,890,265,244,690,372]
+ CRUSH rule 1 x 706 [652,877,562,452,26,323,923]
+ CRUSH rule 1 x 707 [345,745,67,716,789,576,2]
+ CRUSH rule 1 x 708 [333,607,180,469,170,555,939]
+ CRUSH rule 1 x 709 [45,187,302,115,896,579,733]
+ CRUSH rule 1 x 710 [94,855,43,199,18,948,449]
+ CRUSH rule 1 x 711 [227,653,731,150,356,842,534]
+ CRUSH rule 1 x 712 [398,953,136,870,181,408,895]
+ CRUSH rule 1 x 713 [116,800,503,662,635,579,53]
+ CRUSH rule 1 x 714 [111,629,866,709,902,557,875]
+ CRUSH rule 1 x 715 [531,291,486,382,192,807,322]
+ CRUSH rule 1 x 716 [169,541,291,42,343,724,138]
+ CRUSH rule 1 x 717 [417,446,994,894,239,494,237]
+ CRUSH rule 1 x 718 [992,383,298,844,377,463,544]
+ CRUSH rule 1 x 719 [936,674,324,759,194,409,828]
+ CRUSH rule 1 x 720 [370,188,174,464,644,218,214]
+ CRUSH rule 1 x 721 [320,859,278,259,170,957,177]
+ CRUSH rule 1 x 722 [7,2,673,129,96,445,823]
+ CRUSH rule 1 x 723 [270,553,831,662,38,101,985]
+ CRUSH rule 1 x 724 [666,822,708,895,633,800,616]
+ CRUSH rule 1 x 725 [794,406,875,459,981,751,359]
+ CRUSH rule 1 x 726 [420,556,341,292,240,68,966]
+ CRUSH rule 1 x 727 [561,461,129,635,965,610,105]
+ CRUSH rule 1 x 728 [951,330,196,756,589,849,753]
+ CRUSH rule 1 x 729 [656,644,436,591,27,119,572]
+ CRUSH rule 1 x 730 [3,558,629,184,50,765,760]
+ CRUSH rule 1 x 731 [852,89,75,735,713,113,528]
+ CRUSH rule 1 x 732 [983,840,869,976,697,307,368]
+ CRUSH rule 1 x 733 [285,396,388,122,387,364,880]
+ CRUSH rule 1 x 734 [125,510,402,640,676,501,535]
+ CRUSH rule 1 x 735 [417,773,686,504,459,912,690]
+ CRUSH rule 1 x 736 [749,396,632,550,779,109,845]
+ CRUSH rule 1 x 737 [644,991,946,135,448,903,482]
+ CRUSH rule 1 x 738 [449,683,290,220,245,525,429]
+ CRUSH rule 1 x 739 [341,220,641,454,740,661,146]
+ CRUSH rule 1 x 740 [874,524,674,650,472,282,214]
+ CRUSH rule 1 x 741 [189,472,712,798,715,757,863]
+ CRUSH rule 1 x 742 [912,581,114,245,730,21,687]
+ CRUSH rule 1 x 743 [654,914,425,441,763,39,451]
+ CRUSH rule 1 x 744 [725,295,579,377,162,447,843]
+ CRUSH rule 1 x 745 [787,858,850,506,612,735,926]
+ CRUSH rule 1 x 746 [757,848,704,30,47,940,450]
+ CRUSH rule 1 x 747 [700,81,867,681,801,64,879]
+ CRUSH rule 1 x 748 [557,436,238,664,293,865,304]
+ CRUSH rule 1 x 749 [772,622,337,42,156,302,383]
+ CRUSH rule 1 x 750 [946,97,376,677,316,670,169]
+ CRUSH rule 1 x 751 [996,618,343,911,83,22,388]
+ CRUSH rule 1 x 752 [746,887,695,868,610,950,88]
+ CRUSH rule 1 x 753 [741,14,463,479,172,192,481]
+ CRUSH rule 1 x 754 [648,349,333,355,65,63,336]
+ CRUSH rule 1 x 755 [157,460,466,187,959,674,192]
+ CRUSH rule 1 x 756 [416,97,197,497,227,3,850]
+ CRUSH rule 1 x 757 [599,839,776,410,256,823,121]
+ CRUSH rule 1 x 758 [994,218,620,256,361,749,165]
+ CRUSH rule 1 x 759 [959,682,514,745,100,519,15]
+ CRUSH rule 1 x 760 [518,943,215,83,706,137,345]
+ CRUSH rule 1 x 761 [285,849,420,324,987,338,373]
+ CRUSH rule 1 x 762 [591,313,41,335,110,696,664]
+ CRUSH rule 1 x 763 [908,411,200,740,292,295,387]
+ CRUSH rule 1 x 764 [787,234,894,485,883,711,70]
+ CRUSH rule 1 x 765 [327,921,882,393,444,792,402]
+ CRUSH rule 1 x 766 [84,161,878,704,416,144,357]
+ CRUSH rule 1 x 767 [370,895,702,701,890,2,251]
+ CRUSH rule 1 x 768 [826,760,879,864,460,474,645]
+ CRUSH rule 1 x 769 [67,768,663,735,814,66,213]
+ CRUSH rule 1 x 770 [593,909,482,259,5,550,961]
+ CRUSH rule 1 x 771 [309,935,121,578,937,685,933]
+ CRUSH rule 1 x 772 [12,125,797,301,348,419,891]
+ CRUSH rule 1 x 773 [253,466,820,549,591,193,783]
+ CRUSH rule 1 x 774 [164,390,705,109,881,505,890]
+ CRUSH rule 1 x 775 [703,47,43,973,643,406,885]
+ CRUSH rule 1 x 776 [728,231,80,916,2,850,396]
+ CRUSH rule 1 x 777 [981,621,568,729,869,952,563]
+ CRUSH rule 1 x 778 [411,456,544,597,789,784,65]
+ CRUSH rule 1 x 779 [346,121,519,921,587,48,772]
+ CRUSH rule 1 x 780 [476,39,288,381,303,29,17]
+ CRUSH rule 1 x 781 [10,130,585,844,729,705,714]
+ CRUSH rule 1 x 782 [462,246,581,902,623,877,812]
+ CRUSH rule 1 x 783 [580,373,153,775,668,661,626]
+ CRUSH rule 1 x 784 [413,113,978,990,994,56,481]
+ CRUSH rule 1 x 785 [341,856,332,354,59,581,632]
+ CRUSH rule 1 x 786 [411,140,313,393,215,618,490]
+ CRUSH rule 1 x 787 [605,522,211,813,636,224,600]
+ CRUSH rule 1 x 788 [226,545,35,142,726,851,194]
+ CRUSH rule 1 x 789 [545,320,414,702,731,277,237]
+ CRUSH rule 1 x 790 [414,748,816,327,130,115,788]
+ CRUSH rule 1 x 791 [660,906,406,697,916,322,124]
+ CRUSH rule 1 x 792 [287,392,514,204,75,789,406]
+ CRUSH rule 1 x 793 [631,133,850,713,720,487,376]
+ CRUSH rule 1 x 794 [931,517,543,210,963,898,811]
+ CRUSH rule 1 x 795 [551,962,477,948,425,434,268]
+ CRUSH rule 1 x 796 [814,4,95,27,368,300,646]
+ CRUSH rule 1 x 797 [64,201,299,734,605,864,596]
+ CRUSH rule 1 x 798 [422,530,114,431,565,716,473]
+ CRUSH rule 1 x 799 [824,32,679,562,266,549,859]
+ CRUSH rule 1 x 800 [862,623,489,637,861,196,941]
+ CRUSH rule 1 x 801 [145,550,329,324,734,160,219]
+ CRUSH rule 1 x 802 [570,19,847,308,387,518,846]
+ CRUSH rule 1 x 803 [151,812,662,358,880,349,834]
+ CRUSH rule 1 x 804 [467,93,264,863,176,842,663]
+ CRUSH rule 1 x 805 [621,223,938,809,591,686,121]
+ CRUSH rule 1 x 806 [898,957,805,430,499,584,640]
+ CRUSH rule 1 x 807 [354,531,422,159,921,431,802]
+ CRUSH rule 1 x 808 [7,96,76,897,446,2,166]
+ CRUSH rule 1 x 809 [70,734,719,56,687,21,23]
+ CRUSH rule 1 x 810 [701,18,972,327,771,649,620]
+ CRUSH rule 1 x 811 [248,547,103,728,901,264,948]
+ CRUSH rule 1 x 812 [230,576,821,566,993,762,675]
+ CRUSH rule 1 x 813 [805,114,683,629,845,462,285]
+ CRUSH rule 1 x 814 [54,619,973,741,497,894,401]
+ CRUSH rule 1 x 815 [679,412,613,132,969,411,314]
+ CRUSH rule 1 x 816 [919,448,826,414,36,289,44]
+ CRUSH rule 1 x 817 [765,830,436,521,332,458,260]
+ CRUSH rule 1 x 818 [415,566,644,687,692,414,769]
+ CRUSH rule 1 x 819 [721,319,865,750,546,859,523]
+ CRUSH rule 1 x 820 [218,301,333,190,686,179,535]
+ CRUSH rule 1 x 821 [185,795,680,953,329,750,621]
+ CRUSH rule 1 x 822 [356,261,54,522,900,103,883]
+ CRUSH rule 1 x 823 [220,281,549,456,64,306,282]
+ CRUSH rule 1 x 824 [292,809,887,74,776,788,559]
+ CRUSH rule 1 x 825 [949,778,101,311,110,480,161]
+ CRUSH rule 1 x 826 [767,818,833,927,356,954,910]
+ CRUSH rule 1 x 827 [631,83,406,635,657,713,212]
+ CRUSH rule 1 x 828 [288,986,445,26,414,607,937]
+ CRUSH rule 1 x 829 [990,667,915,694,974,453,669]
+ CRUSH rule 1 x 830 [152,571,778,505,685,209,448]
+ CRUSH rule 1 x 831 [814,563,630,97,582,107,142]
+ CRUSH rule 1 x 832 [235,641,616,110,979,844,656]
+ CRUSH rule 1 x 833 [657,565,922,140,825,457,764]
+ CRUSH rule 1 x 834 [907,231,644,13,617,130,83]
+ CRUSH rule 1 x 835 [784,262,771,264,612,238,537]
+ CRUSH rule 1 x 836 [951,158,366,710,43,427,351]
+ CRUSH rule 1 x 837 [556,498,334,633,895,627,903]
+ CRUSH rule 1 x 838 [329,274,964,547,119,342,983]
+ CRUSH rule 1 x 839 [568,209,939,364,658,747,47]
+ CRUSH rule 1 x 840 [45,579,842,70,655,862,815]
+ CRUSH rule 1 x 841 [652,702,24,605,152,93,226]
+ CRUSH rule 1 x 842 [629,984,314,895,408,897,575]
+ CRUSH rule 1 x 843 [799,690,688,648,151,812,486]
+ CRUSH rule 1 x 844 [694,600,534,700,569,11,899]
+ CRUSH rule 1 x 845 [332,30,179,93,951,324,611]
+ CRUSH rule 1 x 846 [452,251,712,719,404,739,606]
+ CRUSH rule 1 x 847 [399,681,847,739,13,555,363]
+ CRUSH rule 1 x 848 [303,138,440,346,547,216,700]
+ CRUSH rule 1 x 849 [666,346,708,873,64,694,847]
+ CRUSH rule 1 x 850 [644,511,345,844,545,337,358]
+ CRUSH rule 1 x 851 [527,546,737,425,100,331,95]
+ CRUSH rule 1 x 852 [31,809,94,618,156,853,469]
+ CRUSH rule 1 x 853 [483,330,869,184,46,942,774]
+ CRUSH rule 1 x 854 [697,953,968,143,502,955,441]
+ CRUSH rule 1 x 855 [837,996,239,621,32,191,686]
+ CRUSH rule 1 x 856 [712,40,547,430,195,857,224]
+ CRUSH rule 1 x 857 [77,984,576,551,568,96,12]
+ CRUSH rule 1 x 858 [412,384,841,465,572,576,688]
+ CRUSH rule 1 x 859 [173,760,26,300,87,567,463]
+ CRUSH rule 1 x 860 [776,429,328,917,658,783,699]
+ CRUSH rule 1 x 861 [705,405,477,50,73,714,901]
+ CRUSH rule 1 x 862 [809,44,788,938,964,177,490]
+ CRUSH rule 1 x 863 [349,496,963,178,675,853,172]
+ CRUSH rule 1 x 864 [717,858,101,239,992,244,43]
+ CRUSH rule 1 x 865 [857,603,586,262,550,289,850]
+ CRUSH rule 1 x 866 [394,304,71,96,642,155,255]
+ CRUSH rule 1 x 867 [640,773,663,974,261,296,988]
+ CRUSH rule 1 x 868 [613,950,712,663,528,460,643]
+ CRUSH rule 1 x 869 [973,889,524,22,671,477,718]
+ CRUSH rule 1 x 870 [505,35,386,498,348,503,54]
+ CRUSH rule 1 x 871 [239,264,262,773,781,734,387]
+ CRUSH rule 1 x 872 [21,767,456,748,783,797,180]
+ CRUSH rule 1 x 873 [954,666,980,264,435,233,199]
+ CRUSH rule 1 x 874 [54,510,947,1,500,119,93]
+ CRUSH rule 1 x 875 [809,418,452,462,88,673,634]
+ CRUSH rule 1 x 876 [483,457,61,248,523,277,322]
+ CRUSH rule 1 x 877 [542,531,952,939,710,179,181]
+ CRUSH rule 1 x 878 [217,674,857,644,678,809,329]
+ CRUSH rule 1 x 879 [999,475,134,250,319,357,145]
+ CRUSH rule 1 x 880 [678,573,935,385,570,651,319]
+ CRUSH rule 1 x 881 [394,835,789,802,587,155,570]
+ CRUSH rule 1 x 882 [467,382,353,56,979,674,974]
+ CRUSH rule 1 x 883 [802,744,237,337,50,96,202]
+ CRUSH rule 1 x 884 [653,660,638,700,31,558,389]
+ CRUSH rule 1 x 885 [898,704,307,445,879,872,174]
+ CRUSH rule 1 x 886 [434,357,938,641,737,8,56]
+ CRUSH rule 1 x 887 [297,226,711,428,370,318,472]
+ CRUSH rule 1 x 888 [863,324,443,213,902,25,806]
+ CRUSH rule 1 x 889 [105,102,308,163,947,548,399]
+ CRUSH rule 1 x 890 [550,248,606,704,615,708,996]
+ CRUSH rule 1 x 891 [575,928,880,891,826,763,706]
+ CRUSH rule 1 x 892 [259,862,133,271,292,162,53]
+ CRUSH rule 1 x 893 [902,880,543,542,37,942,672]
+ CRUSH rule 1 x 894 [180,169,916,43,945,713,648]
+ CRUSH rule 1 x 895 [725,849,182,129,177,272,599]
+ CRUSH rule 1 x 896 [951,34,874,537,969,123,210]
+ CRUSH rule 1 x 897 [810,352,73,939,943,895,12]
+ CRUSH rule 1 x 898 [979,433,719,411,787,359,342]
+ CRUSH rule 1 x 899 [685,668,534,932,399,156,124]
+ CRUSH rule 1 x 900 [530,978,41,894,941,681,380]
+ CRUSH rule 1 x 901 [740,107,336,175,574,706,157]
+ CRUSH rule 1 x 902 [800,743,693,310,67,111,178]
+ CRUSH rule 1 x 903 [230,267,842,266,550,769,66]
+ CRUSH rule 1 x 904 [346,949,460,973,696,91,957]
+ CRUSH rule 1 x 905 [530,397,619,958,576,973,685]
+ CRUSH rule 1 x 906 [80,426,138,672,73,776,30]
+ CRUSH rule 1 x 907 [365,968,475,297,296,724,664]
+ CRUSH rule 1 x 908 [204,832,742,809,862,745,484]
+ CRUSH rule 1 x 909 [883,989,146,959,366,59,686]
+ CRUSH rule 1 x 910 [549,593,249,853,792,769,824]
+ CRUSH rule 1 x 911 [325,847,352,214,851,732,789]
+ CRUSH rule 1 x 912 [874,888,582,796,557,601,226]
+ CRUSH rule 1 x 913 [331,463,342,574,989,362,925]
+ CRUSH rule 1 x 914 [836,468,601,732,607,275,70]
+ CRUSH rule 1 x 915 [245,228,100,661,799,13,126]
+ CRUSH rule 1 x 916 [77,967,364,435,27,474,255]
+ CRUSH rule 1 x 917 [239,60,866,221,772,967,725]
+ CRUSH rule 1 x 918 [988,115,922,80,201,544,583]
+ CRUSH rule 1 x 919 [783,139,696,1,848,169,888]
+ CRUSH rule 1 x 920 [623,408,685,953,974,696,532]
+ CRUSH rule 1 x 921 [105,799,144,90,399,373,633]
+ CRUSH rule 1 x 922 [887,505,652,348,514,806,952]
+ CRUSH rule 1 x 923 [223,318,552,458,743,871,964]
+ CRUSH rule 1 x 924 [25,778,366,333,163,801,584]
+ CRUSH rule 1 x 925 [912,601,297,682,770,173,969]
+ CRUSH rule 1 x 926 [968,133,739,144,814,155,709]
+ CRUSH rule 1 x 927 [277,724,214,988,690,342,465]
+ CRUSH rule 1 x 928 [554,203,658,789,298,299,847]
+ CRUSH rule 1 x 929 [761,802,367,528,758,522,744]
+ CRUSH rule 1 x 930 [814,61,788,736,660,491,832]
+ CRUSH rule 1 x 931 [29,193,61,41,343,664,487]
+ CRUSH rule 1 x 932 [446,198,862,534,168,35,530]
+ CRUSH rule 1 x 933 [352,742,216,321,525,44,568]
+ CRUSH rule 1 x 934 [730,2,332,631,613,249,533]
+ CRUSH rule 1 x 935 [731,23,736,79,361,992,772]
+ CRUSH rule 1 x 936 [322,975,20,904,827,603,138]
+ CRUSH rule 1 x 937 [822,221,841,161,723,137,630]
+ CRUSH rule 1 x 938 [557,850,66,630,499,404,286]
+ CRUSH rule 1 x 939 [150,11,971,371,124,785,408]
+ CRUSH rule 1 x 940 [638,398,169,616,333,751,25]
+ CRUSH rule 1 x 941 [730,342,929,577,451,838,964]
+ CRUSH rule 1 x 942 [62,292,166,814,587,172,237]
+ CRUSH rule 1 x 943 [165,314,519,548,41,726,759]
+ CRUSH rule 1 x 944 [199,625,766,176,194,297,678]
+ CRUSH rule 1 x 945 [946,999,699,303,38,81,952]
+ CRUSH rule 1 x 946 [595,93,852,142,503,647,933]
+ CRUSH rule 1 x 947 [800,582,356,93,716,117,922]
+ CRUSH rule 1 x 948 [132,551,139,920,87,46,81]
+ CRUSH rule 1 x 949 [792,920,466,380,97,568,799]
+ CRUSH rule 1 x 950 [111,345,176,543,879,954,355]
+ CRUSH rule 1 x 951 [414,619,648,655,364,971,829]
+ CRUSH rule 1 x 952 [775,469,500,356,287,4,16]
+ CRUSH rule 1 x 953 [349,1,5,251,168,680,141]
+ CRUSH rule 1 x 954 [570,940,410,249,929,394,129]
+ CRUSH rule 1 x 955 [729,774,823,800,7,127,536]
+ CRUSH rule 1 x 956 [519,141,575,625,738,475,169]
+ CRUSH rule 1 x 957 [242,709,611,97,760,309,393]
+ CRUSH rule 1 x 958 [84,217,227,253,246,604,346]
+ CRUSH rule 1 x 959 [270,413,918,789,703,608,543]
+ CRUSH rule 1 x 960 [458,192,307,279,920,139,855]
+ CRUSH rule 1 x 961 [981,388,777,546,359,660,455]
+ CRUSH rule 1 x 962 [623,834,277,134,729,246,856]
+ CRUSH rule 1 x 963 [291,167,714,468,109,373,485]
+ CRUSH rule 1 x 964 [28,156,788,127,598,215,361]
+ CRUSH rule 1 x 965 [675,557,290,517,840,510,59]
+ CRUSH rule 1 x 966 [836,306,946,283,642,606,929]
+ CRUSH rule 1 x 967 [966,386,735,837,392,116,19]
+ CRUSH rule 1 x 968 [864,756,690,121,328,122,433]
+ CRUSH rule 1 x 969 [729,625,480,769,512,882,518]
+ CRUSH rule 1 x 970 [800,362,646,582,309,102,576]
+ CRUSH rule 1 x 971 [737,381,153,684,298,166,344]
+ CRUSH rule 1 x 972 [952,245,720,884,334,311,754]
+ CRUSH rule 1 x 973 [356,455,579,857,832,596,549]
+ CRUSH rule 1 x 974 [545,758,586,596,269,790,116]
+ CRUSH rule 1 x 975 [336,191,202,146,720,897,330]
+ CRUSH rule 1 x 976 [446,208,757,620,252,846,397]
+ CRUSH rule 1 x 977 [202,896,196,956,763,126,783]
+ CRUSH rule 1 x 978 [612,324,996,225,418,583,514]
+ CRUSH rule 1 x 979 [843,457,675,650,958,657,677]
+ CRUSH rule 1 x 980 [60,914,881,626,850,759,398]
+ CRUSH rule 1 x 981 [702,749,937,153,724,514,536]
+ CRUSH rule 1 x 982 [298,928,738,167,99,668,395]
+ CRUSH rule 1 x 983 [723,572,395,358,900,37,927]
+ CRUSH rule 1 x 984 [723,864,804,935,846,993,950]
+ CRUSH rule 1 x 985 [945,459,868,211,524,954,911]
+ CRUSH rule 1 x 986 [772,664,535,169,297,996,864]
+ CRUSH rule 1 x 987 [88,324,312,843,661,580,76]
+ CRUSH rule 1 x 988 [522,927,131,996,351,685,865]
+ CRUSH rule 1 x 989 [578,332,208,605,975,207,155]
+ CRUSH rule 1 x 990 [638,228,414,311,738,698,340]
+ CRUSH rule 1 x 991 [530,221,451,422,879,916,754]
+ CRUSH rule 1 x 992 [925,705,275,81,234,310,117]
+ CRUSH rule 1 x 993 [991,301,43,469,830,242,382]
+ CRUSH rule 1 x 994 [276,51,868,683,843,815,557]
+ CRUSH rule 1 x 995 [288,836,753,790,758,120,158]
+ CRUSH rule 1 x 996 [887,983,252,686,470,345,459]
+ CRUSH rule 1 x 997 [110,924,386,79,705,697,210]
+ CRUSH rule 1 x 998 [435,830,485,853,926,730,786]
+ CRUSH rule 1 x 999 [876,738,357,913,723,51,15]
+ CRUSH rule 1 x 1000 [178,963,638,430,845,586,317]
+ CRUSH rule 1 x 1001 [99,519,66,759,583,944,739]
+ CRUSH rule 1 x 1002 [515,534,468,866,878,717,729]
+ CRUSH rule 1 x 1003 [104,611,937,698,94,67,614]
+ CRUSH rule 1 x 1004 [269,638,724,375,491,121,891]
+ CRUSH rule 1 x 1005 [369,223,309,409,822,39,597]
+ CRUSH rule 1 x 1006 [40,107,69,275,79,429,234]
+ CRUSH rule 1 x 1007 [978,111,416,758,454,640,5]
+ CRUSH rule 1 x 1008 [965,956,624,832,421,96,975]
+ CRUSH rule 1 x 1009 [598,476,356,695,919,566,234]
+ CRUSH rule 1 x 1010 [767,523,239,517,29,77,23]
+ CRUSH rule 1 x 1011 [289,871,207,576,347,698,48]
+ CRUSH rule 1 x 1012 [128,28,370,31,341,755,268]
+ CRUSH rule 1 x 1013 [979,765,660,812,666,187,808]
+ CRUSH rule 1 x 1014 [979,948,513,88,47,825,969]
+ CRUSH rule 1 x 1015 [277,790,396,672,542,647,145]
+ CRUSH rule 1 x 1016 [262,73,128,886,839,685,456]
+ CRUSH rule 1 x 1017 [150,269,61,499,832,591,637]
+ CRUSH rule 1 x 1018 [555,829,554,944,406,576,463]
+ CRUSH rule 1 x 1019 [513,356,265,446,65,288,768]
+ CRUSH rule 1 x 1020 [158,161,877,704,948,570,495]
+ CRUSH rule 1 x 1021 [915,998,957,285,546,202,676]
+ CRUSH rule 1 x 1022 [967,829,973,640,703,470,871]
+ CRUSH rule 1 x 1023 [488,257,614,859,325,419,50]
+ rule 1 (metadata) num_rep 7 result size == 7:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450,604,380,966,750]
+ CRUSH rule 1 x 1 [876,250,334,633,744,843,672,820]
+ CRUSH rule 1 x 2 [292,832,53,392,386,787,527,901]
+ CRUSH rule 1 x 3 [623,387,124,998,749,211,481,169]
+ CRUSH rule 1 x 4 [61,334,710,4,994,982,847,220]
+ CRUSH rule 1 x 5 [946,557,713,664,141,817,964,872]
+ CRUSH rule 1 x 6 [576,668,212,163,732,381,884,726]
+ CRUSH rule 1 x 7 [645,753,906,393,341,44,578,14]
+ CRUSH rule 1 x 8 [243,6,863,781,211,100,462,207]
+ CRUSH rule 1 x 9 [22,578,251,410,297,430,3,569]
+ CRUSH rule 1 x 10 [758,828,360,477,821,801,811,484]
+ CRUSH rule 1 x 11 [769,120,124,527,119,504,380,821]
+ CRUSH rule 1 x 12 [780,364,689,755,675,199,117,393]
+ CRUSH rule 1 x 13 [557,18,351,719,742,780,78,170]
+ CRUSH rule 1 x 14 [59,561,249,461,971,835,855,76]
+ CRUSH rule 1 x 15 [718,928,993,21,76,313,437,664]
+ CRUSH rule 1 x 16 [673,632,841,954,788,90,786,969]
+ CRUSH rule 1 x 17 [648,43,560,514,142,289,935,605]
+ CRUSH rule 1 x 18 [654,219,181,568,381,253,883,394]
+ CRUSH rule 1 x 19 [850,545,377,848,863,543,51,834]
+ CRUSH rule 1 x 20 [717,785,974,5,225,552,975,636]
+ CRUSH rule 1 x 21 [420,57,519,306,312,983,263,267]
+ CRUSH rule 1 x 22 [503,998,193,821,634,684,557,633]
+ CRUSH rule 1 x 23 [411,663,168,110,899,488,477,468]
+ CRUSH rule 1 x 24 [266,861,353,1,456,128,800,309]
+ CRUSH rule 1 x 25 [760,483,818,600,509,951,248,908]
+ CRUSH rule 1 x 26 [903,24,573,718,112,694,501,909]
+ CRUSH rule 1 x 27 [946,188,289,510,687,827,676,560]
+ CRUSH rule 1 x 28 [69,312,73,198,256,629,770,569]
+ CRUSH rule 1 x 29 [844,883,337,628,496,405,719,581]
+ CRUSH rule 1 x 30 [621,18,613,794,910,936,426,522]
+ CRUSH rule 1 x 31 [784,943,814,539,962,392,813,217]
+ CRUSH rule 1 x 32 [173,374,369,972,315,83,428,63]
+ CRUSH rule 1 x 33 [698,336,357,966,582,407,618,288]
+ CRUSH rule 1 x 34 [168,836,210,798,904,190,663,877]
+ CRUSH rule 1 x 35 [274,509,534,818,912,671,75,580]
+ CRUSH rule 1 x 36 [318,215,153,628,87,407,676,524]
+ CRUSH rule 1 x 37 [173,604,109,935,203,401,311,758]
+ CRUSH rule 1 x 38 [708,444,683,604,722,900,929,910]
+ CRUSH rule 1 x 39 [662,198,417,680,226,342,856,248]
+ CRUSH rule 1 x 40 [620,801,414,78,560,766,980,503]
+ CRUSH rule 1 x 41 [811,264,177,127,148,791,930,74]
+ CRUSH rule 1 x 42 [863,179,527,660,133,529,456,713]
+ CRUSH rule 1 x 43 [686,822,988,228,791,549,514,40]
+ CRUSH rule 1 x 44 [396,222,46,841,536,140,160,527]
+ CRUSH rule 1 x 45 [991,694,253,142,54,422,658,876]
+ CRUSH rule 1 x 46 [420,909,184,285,508,458,45,390]
+ CRUSH rule 1 x 47 [467,211,605,207,241,881,959,800]
+ CRUSH rule 1 x 48 [955,329,368,168,698,787,738,47]
+ CRUSH rule 1 x 49 [974,891,931,29,813,506,822,628]
+ CRUSH rule 1 x 50 [870,441,691,823,761,6,83,344]
+ CRUSH rule 1 x 51 [182,930,25,936,97,260,406,281]
+ CRUSH rule 1 x 52 [704,812,894,794,481,37,304,899]
+ CRUSH rule 1 x 53 [185,713,631,280,345,558,882,503]
+ CRUSH rule 1 x 54 [270,441,100,82,983,930,339,902]
+ CRUSH rule 1 x 55 [895,734,958,793,651,572,508,763]
+ CRUSH rule 1 x 56 [564,963,683,324,40,189,77,500]
+ CRUSH rule 1 x 57 [738,130,208,973,498,861,670,67]
+ CRUSH rule 1 x 58 [524,113,806,903,531,334,8,762]
+ CRUSH rule 1 x 59 [408,337,668,529,34,384,643,511]
+ CRUSH rule 1 x 60 [228,790,857,309,616,895,194,277]
+ CRUSH rule 1 x 61 [154,843,717,467,883,536,812,14]
+ CRUSH rule 1 x 62 [594,811,549,276,693,917,45,723]
+ CRUSH rule 1 x 63 [646,67,884,925,941,434,705,268]
+ CRUSH rule 1 x 64 [175,542,155,837,594,197,451,891]
+ CRUSH rule 1 x 65 [745,619,131,867,269,62,862,221]
+ CRUSH rule 1 x 66 [275,468,23,35,328,432,334,656]
+ CRUSH rule 1 x 67 [246,958,524,493,636,227,783,593]
+ CRUSH rule 1 x 68 [711,473,403,228,835,126,705,114]
+ CRUSH rule 1 x 69 [493,924,850,939,950,105,871,361]
+ CRUSH rule 1 x 70 [30,499,644,33,804,654,684,411]
+ CRUSH rule 1 x 71 [984,883,574,716,575,391,587,264]
+ CRUSH rule 1 x 72 [71,286,942,363,628,632,642,529]
+ CRUSH rule 1 x 73 [922,618,3,371,464,442,835,705]
+ CRUSH rule 1 x 74 [629,414,185,573,678,338,633,560]
+ CRUSH rule 1 x 75 [222,20,174,820,312,361,366,258]
+ CRUSH rule 1 x 76 [262,366,339,290,718,143,735,953]
+ CRUSH rule 1 x 77 [638,469,992,280,773,892,197,690]
+ CRUSH rule 1 x 78 [324,511,788,7,308,228,183,917]
+ CRUSH rule 1 x 79 [577,990,64,94,447,924,339,24]
+ CRUSH rule 1 x 80 [501,95,278,903,631,842,51,766]
+ CRUSH rule 1 x 81 [506,812,9,698,173,664,247,963]
+ CRUSH rule 1 x 82 [222,145,80,785,835,745,580,51]
+ CRUSH rule 1 x 83 [71,634,61,91,856,529,66,197]
+ CRUSH rule 1 x 84 [49,761,773,368,318,708,681,618]
+ CRUSH rule 1 x 85 [985,896,708,861,325,307,567,908]
+ CRUSH rule 1 x 86 [537,745,93,524,466,356,38,326]
+ CRUSH rule 1 x 87 [997,317,463,626,685,458,909,49]
+ CRUSH rule 1 x 88 [957,350,890,857,375,176,99,737]
+ CRUSH rule 1 x 89 [399,730,148,314,159,982,320,921]
+ CRUSH rule 1 x 90 [943,706,683,267,579,141,412,184]
+ CRUSH rule 1 x 91 [22,368,149,928,140,529,495,299]
+ CRUSH rule 1 x 92 [532,424,426,773,623,197,167,634]
+ CRUSH rule 1 x 93 [218,489,405,681,549,201,343,949]
+ CRUSH rule 1 x 94 [181,96,102,515,776,365,82,422]
+ CRUSH rule 1 x 95 [343,957,820,139,334,37,648,661]
+ CRUSH rule 1 x 96 [861,270,87,797,0,245,204,750]
+ CRUSH rule 1 x 97 [459,706,45,328,274,605,83,542]
+ CRUSH rule 1 x 98 [327,867,353,948,728,280,270,511]
+ CRUSH rule 1 x 99 [974,133,468,906,235,988,37,138]
+ CRUSH rule 1 x 100 [32,445,547,371,960,885,9,168]
+ CRUSH rule 1 x 101 [142,90,337,950,970,570,12,369]
+ CRUSH rule 1 x 102 [172,129,139,22,403,867,923,106]
+ CRUSH rule 1 x 103 [630,47,161,356,911,421,933,231]
+ CRUSH rule 1 x 104 [758,133,278,11,947,799,401,85]
+ CRUSH rule 1 x 105 [843,604,47,33,401,632,434,121]
+ CRUSH rule 1 x 106 [28,681,193,679,990,343,878,493]
+ CRUSH rule 1 x 107 [74,320,85,819,315,253,589,614]
+ CRUSH rule 1 x 108 [875,593,575,517,107,153,631,996]
+ CRUSH rule 1 x 109 [411,985,811,720,198,666,856,296]
+ CRUSH rule 1 x 110 [440,774,799,660,715,167,510,472]
+ CRUSH rule 1 x 111 [405,742,276,359,936,360,18,949]
+ CRUSH rule 1 x 112 [143,181,922,545,185,303,725,413]
+ CRUSH rule 1 x 113 [153,846,160,903,789,897,738,253]
+ CRUSH rule 1 x 114 [804,892,939,20,312,692,598,418]
+ CRUSH rule 1 x 115 [588,508,958,580,232,722,421,39]
+ CRUSH rule 1 x 116 [327,148,637,486,712,464,9,448]
+ CRUSH rule 1 x 117 [95,594,989,131,714,275,725,142]
+ CRUSH rule 1 x 118 [80,957,897,239,359,432,766,210]
+ CRUSH rule 1 x 119 [386,932,951,768,679,300,570,278]
+ CRUSH rule 1 x 120 [366,312,653,936,71,241,49,126]
+ CRUSH rule 1 x 121 [129,154,847,16,471,481,424,868]
+ CRUSH rule 1 x 122 [873,1,110,939,90,412,551,43]
+ CRUSH rule 1 x 123 [533,415,789,600,713,800,877,248]
+ CRUSH rule 1 x 124 [461,691,898,723,957,759,482,254]
+ CRUSH rule 1 x 125 [342,599,830,402,615,994,736,737]
+ CRUSH rule 1 x 126 [819,781,822,548,279,255,689,209]
+ CRUSH rule 1 x 127 [437,893,585,707,353,189,909,809]
+ CRUSH rule 1 x 128 [679,994,982,550,991,324,666,691]
+ CRUSH rule 1 x 129 [380,685,947,302,698,144,149,51]
+ CRUSH rule 1 x 130 [992,52,466,867,998,777,270,425]
+ CRUSH rule 1 x 131 [469,90,208,599,829,656,203,667]
+ CRUSH rule 1 x 132 [571,250,316,535,54,418,922,597]
+ CRUSH rule 1 x 133 [964,728,329,902,108,118,14,444]
+ CRUSH rule 1 x 134 [999,19,716,963,323,559,893,281]
+ CRUSH rule 1 x 135 [634,101,52,938,413,573,712,649]
+ CRUSH rule 1 x 136 [114,889,692,768,694,279,846,890]
+ CRUSH rule 1 x 137 [839,8,959,280,922,870,363,323]
+ CRUSH rule 1 x 138 [967,949,138,451,292,548,400,885]
+ CRUSH rule 1 x 139 [308,711,736,247,632,126,384,58]
+ CRUSH rule 1 x 140 [764,936,926,55,331,115,178,532]
+ CRUSH rule 1 x 141 [423,302,112,216,603,873,193,258]
+ CRUSH rule 1 x 142 [252,821,715,340,635,668,424,87]
+ CRUSH rule 1 x 143 [33,808,518,477,325,316,266,70]
+ CRUSH rule 1 x 144 [472,88,969,162,401,771,697,610]
+ CRUSH rule 1 x 145 [242,208,252,604,266,743,577,348]
+ CRUSH rule 1 x 146 [290,70,570,384,934,856,929,196]
+ CRUSH rule 1 x 147 [447,352,657,493,467,918,514,546]
+ CRUSH rule 1 x 148 [212,644,432,658,109,275,352,820]
+ CRUSH rule 1 x 149 [9,775,87,35,260,646,406,556]
+ CRUSH rule 1 x 150 [166,456,582,144,324,340,484,553]
+ CRUSH rule 1 x 151 [811,875,307,20,782,229,671,883]
+ CRUSH rule 1 x 152 [449,617,223,9,182,407,807,50]
+ CRUSH rule 1 x 153 [523,537,695,627,959,613,942,864]
+ CRUSH rule 1 x 154 [208,559,874,597,243,706,443,98]
+ CRUSH rule 1 x 155 [569,325,192,296,367,848,58,641]
+ CRUSH rule 1 x 156 [488,121,521,213,595,837,271,229]
+ CRUSH rule 1 x 157 [140,723,633,260,487,856,384,446]
+ CRUSH rule 1 x 158 [786,451,320,239,667,632,899,902]
+ CRUSH rule 1 x 159 [134,664,517,821,667,944,209,641]
+ CRUSH rule 1 x 160 [690,112,414,990,183,590,242,999]
+ CRUSH rule 1 x 161 [324,912,397,423,991,284,909,642]
+ CRUSH rule 1 x 162 [748,567,284,183,463,336,148,88]
+ CRUSH rule 1 x 163 [575,499,31,816,749,737,587,854]
+ CRUSH rule 1 x 164 [314,489,308,326,51,568,110,329]
+ CRUSH rule 1 x 165 [116,209,750,53,813,640,524,389]
+ CRUSH rule 1 x 166 [352,706,701,810,718,527,548,676]
+ CRUSH rule 1 x 167 [27,743,174,142,551,1,935,266]
+ CRUSH rule 1 x 168 [953,898,880,660,500,799,667,463]
+ CRUSH rule 1 x 169 [912,147,266,547,331,770,601,909]
+ CRUSH rule 1 x 170 [421,515,828,844,151,981,835,840]
+ CRUSH rule 1 x 171 [488,584,880,964,936,196,100,910]
+ CRUSH rule 1 x 172 [366,443,957,66,162,693,36,356]
+ CRUSH rule 1 x 173 [863,291,625,287,158,496,471,529]
+ CRUSH rule 1 x 174 [263,555,650,410,339,616,780,932]
+ CRUSH rule 1 x 175 [875,961,361,575,33,109,51,211]
+ CRUSH rule 1 x 176 [745,83,701,680,250,420,240,316]
+ CRUSH rule 1 x 177 [128,244,41,123,422,902,756,647]
+ CRUSH rule 1 x 178 [155,41,264,777,314,564,856,992]
+ CRUSH rule 1 x 179 [593,833,202,183,971,38,724,923]
+ CRUSH rule 1 x 180 [154,734,17,831,824,522,736,846]
+ CRUSH rule 1 x 181 [289,675,723,800,166,712,168,224]
+ CRUSH rule 1 x 182 [730,931,560,209,943,261,485,571]
+ CRUSH rule 1 x 183 [639,237,794,815,827,400,109,903]
+ CRUSH rule 1 x 184 [704,312,685,645,691,778,74,45]
+ CRUSH rule 1 x 185 [97,100,762,82,999,542,485,511]
+ CRUSH rule 1 x 186 [26,665,554,215,280,421,369,270]
+ CRUSH rule 1 x 187 [649,14,740,494,402,684,566,378]
+ CRUSH rule 1 x 188 [682,695,590,743,927,945,833,650]
+ CRUSH rule 1 x 189 [325,693,726,51,448,169,37,1]
+ CRUSH rule 1 x 190 [399,933,136,955,57,504,527,237]
+ CRUSH rule 1 x 191 [629,533,17,126,60,146,999,754]
+ CRUSH rule 1 x 192 [503,578,38,492,222,251,123,759]
+ CRUSH rule 1 x 193 [546,333,651,678,823,652,359,721]
+ CRUSH rule 1 x 194 [242,473,58,655,311,277,792,887]
+ CRUSH rule 1 x 195 [625,719,135,81,636,513,755,471]
+ CRUSH rule 1 x 196 [357,114,125,867,250,522,413,834]
+ CRUSH rule 1 x 197 [306,954,453,873,211,334,666,316]
+ CRUSH rule 1 x 198 [863,791,311,911,206,61,355,574]
+ CRUSH rule 1 x 199 [935,906,929,252,893,75,960,369]
+ CRUSH rule 1 x 200 [373,774,229,454,909,611,132,271]
+ CRUSH rule 1 x 201 [659,320,477,313,779,16,495,76]
+ CRUSH rule 1 x 202 [260,433,524,880,223,818,153,272]
+ CRUSH rule 1 x 203 [36,239,675,971,703,209,669,676]
+ CRUSH rule 1 x 204 [92,516,993,728,279,478,697,881]
+ CRUSH rule 1 x 205 [68,395,473,45,683,662,776,463]
+ CRUSH rule 1 x 206 [570,530,642,380,311,398,230,367]
+ CRUSH rule 1 x 207 [834,457,850,917,456,296,76,708]
+ CRUSH rule 1 x 208 [927,484,640,976,803,626,96,841]
+ CRUSH rule 1 x 209 [878,66,58,940,48,233,522,185]
+ CRUSH rule 1 x 210 [572,981,484,29,0,426,14,921]
+ CRUSH rule 1 x 211 [107,597,780,857,895,57,922,372]
+ CRUSH rule 1 x 212 [389,107,838,624,698,562,857,894]
+ CRUSH rule 1 x 213 [497,717,567,728,905,134,687,903]
+ CRUSH rule 1 x 214 [798,65,254,572,32,393,579,79]
+ CRUSH rule 1 x 215 [233,419,283,638,520,891,982,826]
+ CRUSH rule 1 x 216 [494,464,742,523,459,174,973,898]
+ CRUSH rule 1 x 217 [352,396,309,938,66,41,264,6]
+ CRUSH rule 1 x 218 [895,864,988,650,593,740,34,497]
+ CRUSH rule 1 x 219 [222,534,277,242,658,482,697,805]
+ CRUSH rule 1 x 220 [281,19,584,563,858,965,686,982]
+ CRUSH rule 1 x 221 [64,928,963,130,312,394,61,559]
+ CRUSH rule 1 x 222 [40,544,161,199,861,644,597,904]
+ CRUSH rule 1 x 223 [645,556,159,417,46,135,465,429]
+ CRUSH rule 1 x 224 [647,165,957,263,961,576,329,320]
+ CRUSH rule 1 x 225 [219,714,858,747,461,175,606,465]
+ CRUSH rule 1 x 226 [372,511,181,277,695,404,876,984]
+ CRUSH rule 1 x 227 [925,156,714,863,257,74,966,217]
+ CRUSH rule 1 x 228 [682,404,839,263,521,195,261,389]
+ CRUSH rule 1 x 229 [880,838,770,891,236,542,262,884]
+ CRUSH rule 1 x 230 [328,659,916,468,646,572,93,880]
+ CRUSH rule 1 x 231 [320,383,669,109,627,621,50,182]
+ CRUSH rule 1 x 232 [924,846,394,319,43,519,106,877]
+ CRUSH rule 1 x 233 [948,652,575,838,498,395,796,835]
+ CRUSH rule 1 x 234 [484,943,42,575,936,180,103,95]
+ CRUSH rule 1 x 235 [750,65,590,168,870,308,471,753]
+ CRUSH rule 1 x 236 [551,787,490,136,370,833,573,128]
+ CRUSH rule 1 x 237 [390,157,166,251,752,75,327,509]
+ CRUSH rule 1 x 238 [570,6,989,707,514,905,894,884]
+ CRUSH rule 1 x 239 [729,959,376,975,496,49,426,427]
+ CRUSH rule 1 x 240 [981,241,156,767,631,576,450,677]
+ CRUSH rule 1 x 241 [310,816,641,177,996,454,413,136]
+ CRUSH rule 1 x 242 [161,63,642,837,763,458,234,756]
+ CRUSH rule 1 x 243 [180,394,33,683,189,419,799,21]
+ CRUSH rule 1 x 244 [52,174,685,189,78,310,785,107]
+ CRUSH rule 1 x 245 [523,121,915,84,386,409,605,837]
+ CRUSH rule 1 x 246 [362,893,390,487,817,88,989,999]
+ CRUSH rule 1 x 247 [382,184,116,34,143,15,590,840]
+ CRUSH rule 1 x 248 [129,114,852,469,359,291,713,237]
+ CRUSH rule 1 x 249 [159,683,91,856,475,369,886,650]
+ CRUSH rule 1 x 250 [404,945,569,955,228,910,270,619]
+ CRUSH rule 1 x 251 [661,225,738,757,37,642,58,354]
+ CRUSH rule 1 x 252 [961,226,542,103,945,885,838,131]
+ CRUSH rule 1 x 253 [651,97,225,364,189,248,797,675]
+ CRUSH rule 1 x 254 [123,33,741,692,599,11,605,453]
+ CRUSH rule 1 x 255 [314,649,891,855,517,344,607,95]
+ CRUSH rule 1 x 256 [315,215,651,126,470,849,189,627]
+ CRUSH rule 1 x 257 [825,264,867,141,529,409,291,732]
+ CRUSH rule 1 x 258 [624,789,370,723,131,982,863,427]
+ CRUSH rule 1 x 259 [602,542,70,563,947,723,77,191]
+ CRUSH rule 1 x 260 [717,878,43,56,377,481,533,646]
+ CRUSH rule 1 x 261 [145,517,20,903,786,939,516,136]
+ CRUSH rule 1 x 262 [223,1,561,420,680,16,88,534]
+ CRUSH rule 1 x 263 [462,211,405,508,787,669,773,979]
+ CRUSH rule 1 x 264 [654,471,266,662,135,564,715,916]
+ CRUSH rule 1 x 265 [302,794,704,798,659,487,833,987]
+ CRUSH rule 1 x 266 [202,132,884,209,551,984,7,557]
+ CRUSH rule 1 x 267 [282,938,657,113,672,993,972,645]
+ CRUSH rule 1 x 268 [338,309,356,278,928,797,715,536]
+ CRUSH rule 1 x 269 [738,122,266,200,894,118,146,14]
+ CRUSH rule 1 x 270 [707,982,946,196,407,804,476,571]
+ CRUSH rule 1 x 271 [705,432,364,735,512,595,263,138]
+ CRUSH rule 1 x 272 [756,545,942,56,542,449,710,779]
+ CRUSH rule 1 x 273 [197,502,527,721,239,648,982,735]
+ CRUSH rule 1 x 274 [992,44,653,573,527,702,370,990]
+ CRUSH rule 1 x 275 [544,789,170,434,23,926,992,823]
+ CRUSH rule 1 x 276 [658,467,577,268,336,5,634,98]
+ CRUSH rule 1 x 277 [143,490,880,483,928,272,783,648]
+ CRUSH rule 1 x 278 [492,647,355,282,834,64,350,600]
+ CRUSH rule 1 x 279 [517,792,604,987,527,894,952,250]
+ CRUSH rule 1 x 280 [825,740,27,848,514,750,895,914]
+ CRUSH rule 1 x 281 [224,629,120,562,616,200,443,604]
+ CRUSH rule 1 x 282 [298,661,380,416,35,585,939,879]
+ CRUSH rule 1 x 283 [311,606,208,50,913,678,369,544]
+ CRUSH rule 1 x 284 [771,466,371,743,672,119,60,546]
+ CRUSH rule 1 x 285 [693,362,404,676,797,531,582,975]
+ CRUSH rule 1 x 286 [364,477,285,167,270,617,699,627]
+ CRUSH rule 1 x 287 [591,611,828,995,170,987,137,890]
+ CRUSH rule 1 x 288 [965,541,848,796,251,668,195,538]
+ CRUSH rule 1 x 289 [225,551,948,877,219,167,795,377]
+ CRUSH rule 1 x 290 [577,762,777,751,291,349,473,209]
+ CRUSH rule 1 x 291 [160,903,477,381,490,559,557,86]
+ CRUSH rule 1 x 292 [873,598,216,666,222,228,806,911]
+ CRUSH rule 1 x 293 [100,234,874,47,28,452,775,636]
+ CRUSH rule 1 x 294 [285,943,379,520,725,547,459,833]
+ CRUSH rule 1 x 295 [938,262,880,327,687,3,440,73]
+ CRUSH rule 1 x 296 [850,327,86,472,1,776,266,82]
+ CRUSH rule 1 x 297 [951,53,99,558,753,228,232,343]
+ CRUSH rule 1 x 298 [173,336,85,766,910,657,213,286]
+ CRUSH rule 1 x 299 [598,591,315,386,895,296,924,106]
+ CRUSH rule 1 x 300 [531,957,62,459,156,538,904,838]
+ CRUSH rule 1 x 301 [823,628,23,858,629,808,220,432]
+ CRUSH rule 1 x 302 [184,80,780,871,531,211,400,365]
+ CRUSH rule 1 x 303 [521,766,222,830,988,275,561,905]
+ CRUSH rule 1 x 304 [980,127,807,507,555,245,214,944]
+ CRUSH rule 1 x 305 [153,816,22,927,696,911,685,838]
+ CRUSH rule 1 x 306 [423,739,664,753,178,431,761,648]
+ CRUSH rule 1 x 307 [997,557,682,456,479,631,459,250]
+ CRUSH rule 1 x 308 [991,874,534,465,330,284,976,551]
+ CRUSH rule 1 x 309 [860,394,724,858,246,866,857,153]
+ CRUSH rule 1 x 310 [589,818,546,201,94,653,90,855]
+ CRUSH rule 1 x 311 [477,774,225,590,830,559,256,798]
+ CRUSH rule 1 x 312 [887,853,950,354,58,23,497,929]
+ CRUSH rule 1 x 313 [802,646,447,416,557,118,24,81]
+ CRUSH rule 1 x 314 [654,974,229,511,562,916,952,599]
+ CRUSH rule 1 x 315 [767,227,28,740,828,156,749,841]
+ CRUSH rule 1 x 316 [778,83,733,359,858,319,761,725]
+ CRUSH rule 1 x 317 [184,418,642,986,939,675,892,86]
+ CRUSH rule 1 x 318 [525,410,500,543,212,95,290,97]
+ CRUSH rule 1 x 319 [476,724,569,382,409,521,800,868]
+ CRUSH rule 1 x 320 [149,610,697,296,818,955,523,366]
+ CRUSH rule 1 x 321 [710,79,667,671,234,4,868,841]
+ CRUSH rule 1 x 322 [175,275,323,333,744,718,187,380]
+ CRUSH rule 1 x 323 [819,604,638,792,316,544,236,404]
+ CRUSH rule 1 x 324 [16,745,511,439,272,95,668,959]
+ CRUSH rule 1 x 325 [486,400,872,873,251,68,462,268]
+ CRUSH rule 1 x 326 [613,765,207,19,359,370,461,509]
+ CRUSH rule 1 x 327 [125,289,738,408,456,784,750,669]
+ CRUSH rule 1 x 328 [807,383,476,583,645,141,33,806]
+ CRUSH rule 1 x 329 [588,938,599,432,446,840,516,713]
+ CRUSH rule 1 x 330 [932,644,41,611,209,406,420,520]
+ CRUSH rule 1 x 331 [341,953,950,537,578,862,624,649]
+ CRUSH rule 1 x 332 [153,726,459,950,466,804,644,821]
+ CRUSH rule 1 x 333 [745,845,853,860,52,615,243,633]
+ CRUSH rule 1 x 334 [614,751,807,58,396,159,408,175]
+ CRUSH rule 1 x 335 [518,721,221,283,454,187,635,367]
+ CRUSH rule 1 x 336 [389,424,77,309,5,898,698,533]
+ CRUSH rule 1 x 337 [753,508,765,720,221,807,956,907]
+ CRUSH rule 1 x 338 [128,810,490,753,406,760,69,11]
+ CRUSH rule 1 x 339 [430,308,58,751,856,823,607,953]
+ CRUSH rule 1 x 340 [541,44,630,231,289,966,707,328]
+ CRUSH rule 1 x 341 [402,26,631,439,165,928,720,503]
+ CRUSH rule 1 x 342 [982,57,992,461,131,32,516,661]
+ CRUSH rule 1 x 343 [833,412,572,732,107,805,660,655]
+ CRUSH rule 1 x 344 [784,533,792,41,642,869,142,114]
+ CRUSH rule 1 x 345 [546,300,304,691,763,556,127,732]
+ CRUSH rule 1 x 346 [302,420,428,891,357,124,419,962]
+ CRUSH rule 1 x 347 [488,778,101,217,366,442,783,661]
+ CRUSH rule 1 x 348 [903,744,937,718,85,314,862,513]
+ CRUSH rule 1 x 349 [471,547,582,306,600,486,795,143]
+ CRUSH rule 1 x 350 [348,221,823,335,383,708,841,164]
+ CRUSH rule 1 x 351 [961,582,705,346,361,32,766,775]
+ CRUSH rule 1 x 352 [728,137,461,298,36,903,899,665]
+ CRUSH rule 1 x 353 [904,202,184,447,58,294,279,616]
+ CRUSH rule 1 x 354 [345,226,319,256,544,311,612,33]
+ CRUSH rule 1 x 355 [50,430,175,43,187,458,985,412]
+ CRUSH rule 1 x 356 [87,185,55,423,829,1,629,228]
+ CRUSH rule 1 x 357 [762,459,921,473,182,231,891,656]
+ CRUSH rule 1 x 358 [908,25,280,6,808,676,874,643]
+ CRUSH rule 1 x 359 [484,15,132,121,394,423,397,52]
+ CRUSH rule 1 x 360 [173,378,337,702,145,499,29,529]
+ CRUSH rule 1 x 361 [404,577,115,25,56,914,643,286]
+ CRUSH rule 1 x 362 [403,1,422,945,132,685,265,35]
+ CRUSH rule 1 x 363 [639,911,510,162,418,294,444,613]
+ CRUSH rule 1 x 364 [752,689,610,990,665,222,203,17]
+ CRUSH rule 1 x 365 [956,999,212,230,624,84,113,373]
+ CRUSH rule 1 x 366 [860,925,924,763,687,851,59,914]
+ CRUSH rule 1 x 367 [205,609,647,665,969,720,685,641]
+ CRUSH rule 1 x 368 [301,284,810,169,78,340,616,93]
+ CRUSH rule 1 x 369 [452,658,339,217,674,210,284,184]
+ CRUSH rule 1 x 370 [11,467,695,989,394,576,850,419]
+ CRUSH rule 1 x 371 [124,487,55,514,313,411,797,547]
+ CRUSH rule 1 x 372 [253,48,979,846,207,631,212,241]
+ CRUSH rule 1 x 373 [715,605,775,748,227,493,128,207]
+ CRUSH rule 1 x 374 [191,887,920,764,223,714,961,760]
+ CRUSH rule 1 x 375 [711,385,651,665,15,71,934,619]
+ CRUSH rule 1 x 376 [597,818,49,458,415,755,446,897]
+ CRUSH rule 1 x 377 [294,256,933,771,184,861,654,487]
+ CRUSH rule 1 x 378 [34,151,681,707,552,127,728,860]
+ CRUSH rule 1 x 379 [869,136,315,378,813,153,115,557]
+ CRUSH rule 1 x 380 [294,97,575,791,690,482,255,806]
+ CRUSH rule 1 x 381 [119,710,219,827,328,886,773,496]
+ CRUSH rule 1 x 382 [69,631,508,706,697,168,276,56]
+ CRUSH rule 1 x 383 [922,588,589,925,471,601,29,197]
+ CRUSH rule 1 x 384 [221,945,671,117,857,655,488,435]
+ CRUSH rule 1 x 385 [561,737,953,723,658,368,910,329]
+ CRUSH rule 1 x 386 [335,442,788,696,507,716,232,692]
+ CRUSH rule 1 x 387 [514,43,353,88,100,842,164,934]
+ CRUSH rule 1 x 388 [587,89,157,996,915,927,474,267]
+ CRUSH rule 1 x 389 [109,641,255,466,372,563,340,222]
+ CRUSH rule 1 x 390 [925,149,421,489,599,810,852,196]
+ CRUSH rule 1 x 391 [267,87,387,527,768,873,481,136]
+ CRUSH rule 1 x 392 [382,485,370,849,936,636,901,82]
+ CRUSH rule 1 x 393 [425,721,221,753,268,463,652,543]
+ CRUSH rule 1 x 394 [898,18,38,793,173,738,15,591]
+ CRUSH rule 1 x 395 [806,876,269,679,32,744,126,179]
+ CRUSH rule 1 x 396 [790,970,437,449,875,395,726,935]
+ CRUSH rule 1 x 397 [136,363,507,613,11,30,996,558]
+ CRUSH rule 1 x 398 [914,116,558,258,722,904,349,672]
+ CRUSH rule 1 x 399 [261,94,299,202,174,622,749,410]
+ CRUSH rule 1 x 400 [661,197,338,461,977,848,536,592]
+ CRUSH rule 1 x 401 [953,979,287,803,41,349,79,32]
+ CRUSH rule 1 x 402 [738,819,618,522,667,334,658,449]
+ CRUSH rule 1 x 403 [573,238,425,546,130,68,202,650]
+ CRUSH rule 1 x 404 [526,848,790,253,922,820,299,577]
+ CRUSH rule 1 x 405 [582,505,330,334,201,110,776,296]
+ CRUSH rule 1 x 406 [768,324,493,60,186,165,718,578]
+ CRUSH rule 1 x 407 [260,951,437,587,692,648,72,345]
+ CRUSH rule 1 x 408 [657,81,770,734,830,821,246,695]
+ CRUSH rule 1 x 409 [498,89,182,423,672,152,213,806]
+ CRUSH rule 1 x 410 [28,793,737,352,166,645,949,507]
+ CRUSH rule 1 x 411 [684,992,60,659,769,267,313,351]
+ CRUSH rule 1 x 412 [261,958,699,950,165,14,560,155]
+ CRUSH rule 1 x 413 [891,835,297,441,384,979,618,907]
+ CRUSH rule 1 x 414 [127,459,119,965,662,594,97,124]
+ CRUSH rule 1 x 415 [272,540,631,328,609,568,694,332]
+ CRUSH rule 1 x 416 [739,617,115,530,339,371,889,344]
+ CRUSH rule 1 x 417 [106,209,157,878,117,128,138,374]
+ CRUSH rule 1 x 418 [525,441,147,390,320,300,848,972]
+ CRUSH rule 1 x 419 [603,673,615,465,266,855,823,884]
+ CRUSH rule 1 x 420 [988,213,251,226,209,245,506,670]
+ CRUSH rule 1 x 421 [761,521,748,368,923,992,764,274]
+ CRUSH rule 1 x 422 [317,160,924,548,198,709,839,547]
+ CRUSH rule 1 x 423 [137,807,168,472,619,443,905,588]
+ CRUSH rule 1 x 424 [920,37,146,263,598,748,785,395]
+ CRUSH rule 1 x 425 [277,693,285,221,478,165,80,236]
+ CRUSH rule 1 x 426 [485,936,407,854,726,524,791,565]
+ CRUSH rule 1 x 427 [242,515,9,564,174,453,334,588]
+ CRUSH rule 1 x 428 [632,635,26,473,494,478,225,94]
+ CRUSH rule 1 x 429 [641,73,465,127,171,397,857,562]
+ CRUSH rule 1 x 430 [626,585,6,387,881,583,859,699]
+ CRUSH rule 1 x 431 [697,76,753,570,964,339,194,366]
+ CRUSH rule 1 x 432 [590,526,306,283,656,728,513,591]
+ CRUSH rule 1 x 433 [284,387,149,817,886,714,52,897]
+ CRUSH rule 1 x 434 [538,985,79,953,770,468,644,646]
+ CRUSH rule 1 x 435 [30,318,593,635,975,833,371,731]
+ CRUSH rule 1 x 436 [164,919,851,693,0,874,10,976]
+ CRUSH rule 1 x 437 [322,212,163,606,302,282,443,23]
+ CRUSH rule 1 x 438 [142,392,85,594,376,419,755,841]
+ CRUSH rule 1 x 439 [119,370,68,443,997,837,414,152]
+ CRUSH rule 1 x 440 [333,403,187,863,475,844,800,174]
+ CRUSH rule 1 x 441 [477,727,906,145,429,91,205,236]
+ CRUSH rule 1 x 442 [274,590,933,244,434,49,864,799]
+ CRUSH rule 1 x 443 [983,748,574,718,700,442,774,350]
+ CRUSH rule 1 x 444 [536,509,431,146,170,149,182,145]
+ CRUSH rule 1 x 445 [485,892,528,209,964,753,554,931]
+ CRUSH rule 1 x 446 [345,634,42,294,711,376,314,714]
+ CRUSH rule 1 x 447 [61,845,767,600,321,716,58,531]
+ CRUSH rule 1 x 448 [333,232,292,846,364,951,807,688]
+ CRUSH rule 1 x 449 [680,16,484,670,851,500,258,548]
+ CRUSH rule 1 x 450 [235,214,79,423,96,822,721,31]
+ CRUSH rule 1 x 451 [961,468,333,640,823,151,878,33]
+ CRUSH rule 1 x 452 [525,479,153,528,570,806,604,49]
+ CRUSH rule 1 x 453 [138,466,302,86,249,154,514,5]
+ CRUSH rule 1 x 454 [137,625,215,402,389,914,106,103]
+ CRUSH rule 1 x 455 [173,150,997,16,846,888,295,967]
+ CRUSH rule 1 x 456 [235,226,238,258,347,784,504,96]
+ CRUSH rule 1 x 457 [450,577,253,413,717,609,762,975]
+ CRUSH rule 1 x 458 [195,537,91,814,351,90,399,558]
+ CRUSH rule 1 x 459 [381,555,312,573,915,623,147,483]
+ CRUSH rule 1 x 460 [972,730,534,678,756,692,841,512]
+ CRUSH rule 1 x 461 [506,279,142,830,784,124,385,797]
+ CRUSH rule 1 x 462 [692,959,578,57,983,299,240,911]
+ CRUSH rule 1 x 463 [788,667,949,550,685,702,538,111]
+ CRUSH rule 1 x 464 [133,122,588,999,270,880,789,0]
+ CRUSH rule 1 x 465 [971,190,230,777,452,914,137,466]
+ CRUSH rule 1 x 466 [394,576,148,157,103,822,659,35]
+ CRUSH rule 1 x 467 [517,28,366,362,984,521,187,640]
+ CRUSH rule 1 x 468 [829,143,874,225,162,413,201,249]
+ CRUSH rule 1 x 469 [987,936,106,725,633,238,681,159]
+ CRUSH rule 1 x 470 [107,982,56,889,67,65,558,71]
+ CRUSH rule 1 x 471 [181,897,629,860,307,116,256,978]
+ CRUSH rule 1 x 472 [547,512,172,24,705,837,809,56]
+ CRUSH rule 1 x 473 [760,997,824,905,888,755,756,663]
+ CRUSH rule 1 x 474 [787,418,743,628,272,341,446,333]
+ CRUSH rule 1 x 475 [662,312,253,617,105,58,237,764]
+ CRUSH rule 1 x 476 [110,495,185,508,961,837,984,226]
+ CRUSH rule 1 x 477 [393,954,834,132,841,367,753,794]
+ CRUSH rule 1 x 478 [246,483,480,644,985,420,941,843]
+ CRUSH rule 1 x 479 [70,929,697,931,744,487,158,489]
+ CRUSH rule 1 x 480 [753,119,961,607,317,717,371,807]
+ CRUSH rule 1 x 481 [470,429,677,242,574,757,135,375]
+ CRUSH rule 1 x 482 [451,566,961,675,354,746,731,233]
+ CRUSH rule 1 x 483 [816,72,371,278,635,30,448,437]
+ CRUSH rule 1 x 484 [540,454,389,31,654,494,283,170]
+ CRUSH rule 1 x 485 [74,582,624,684,566,677,866,661]
+ CRUSH rule 1 x 486 [958,595,199,763,715,973,621,955]
+ CRUSH rule 1 x 487 [228,302,804,833,876,647,857,782]
+ CRUSH rule 1 x 488 [180,529,722,956,353,890,924,965]
+ CRUSH rule 1 x 489 [47,617,812,187,291,828,154,478]
+ CRUSH rule 1 x 490 [905,822,479,124,750,843,566,779]
+ CRUSH rule 1 x 491 [892,370,609,998,433,957,188,563]
+ CRUSH rule 1 x 492 [588,959,127,948,505,936,591,423]
+ CRUSH rule 1 x 493 [353,461,593,291,301,830,231,580]
+ CRUSH rule 1 x 494 [378,848,443,368,507,423,389,819]
+ CRUSH rule 1 x 495 [845,653,768,234,405,367,823,789]
+ CRUSH rule 1 x 496 [13,988,0,691,389,757,129,763]
+ CRUSH rule 1 x 497 [796,877,788,394,648,829,542,745]
+ CRUSH rule 1 x 498 [412,337,270,705,511,227,949,173]
+ CRUSH rule 1 x 499 [330,695,8,74,618,101,440,509]
+ CRUSH rule 1 x 500 [820,272,547,765,755,96,930,573]
+ CRUSH rule 1 x 501 [110,44,132,442,294,423,880,279]
+ CRUSH rule 1 x 502 [336,595,650,274,993,312,490,852]
+ CRUSH rule 1 x 503 [922,211,157,722,502,971,262,926]
+ CRUSH rule 1 x 504 [483,52,122,432,778,461,758,104]
+ CRUSH rule 1 x 505 [482,598,224,279,480,310,764,558]
+ CRUSH rule 1 x 506 [493,123,43,856,936,622,898,161]
+ CRUSH rule 1 x 507 [12,598,264,422,416,947,591,702]
+ CRUSH rule 1 x 508 [227,157,611,301,223,746,313,282]
+ CRUSH rule 1 x 509 [807,242,363,122,582,530,798,808]
+ CRUSH rule 1 x 510 [134,437,227,75,313,351,786,152]
+ CRUSH rule 1 x 511 [212,54,83,799,457,218,600,968]
+ CRUSH rule 1 x 512 [236,630,758,752,361,249,899,451]
+ CRUSH rule 1 x 513 [994,693,644,938,846,685,52,185]
+ CRUSH rule 1 x 514 [45,508,831,19,817,52,374,985]
+ CRUSH rule 1 x 515 [504,138,480,272,530,377,481,820]
+ CRUSH rule 1 x 516 [285,409,136,570,841,610,453,660]
+ CRUSH rule 1 x 517 [300,232,23,906,438,236,519,737]
+ CRUSH rule 1 x 518 [397,674,98,898,967,113,625,434]
+ CRUSH rule 1 x 519 [86,750,772,913,101,864,375,328]
+ CRUSH rule 1 x 520 [900,833,614,130,261,885,558,956]
+ CRUSH rule 1 x 521 [31,47,236,751,911,599,495,354]
+ CRUSH rule 1 x 522 [390,16,280,144,291,175,753,624]
+ CRUSH rule 1 x 523 [618,308,424,590,300,206,834,212]
+ CRUSH rule 1 x 524 [635,189,687,963,601,518,8,550]
+ CRUSH rule 1 x 525 [311,916,699,262,775,32,45,478]
+ CRUSH rule 1 x 526 [48,738,227,718,244,942,853,643]
+ CRUSH rule 1 x 527 [202,851,889,216,763,351,270,35]
+ CRUSH rule 1 x 528 [565,827,590,273,918,106,651,368]
+ CRUSH rule 1 x 529 [934,864,241,43,466,924,278,926]
+ CRUSH rule 1 x 530 [502,934,298,670,986,360,577,509]
+ CRUSH rule 1 x 531 [681,627,942,487,288,561,925,474]
+ CRUSH rule 1 x 532 [422,6,147,205,861,141,949,374]
+ CRUSH rule 1 x 533 [863,68,364,983,247,199,54,931]
+ CRUSH rule 1 x 534 [962,931,775,172,663,119,206,682]
+ CRUSH rule 1 x 535 [89,565,397,693,839,632,859,30]
+ CRUSH rule 1 x 536 [499,351,760,458,918,86,148,668]
+ CRUSH rule 1 x 537 [676,547,787,311,867,748,152,797]
+ CRUSH rule 1 x 538 [58,644,571,649,941,7,37,485]
+ CRUSH rule 1 x 539 [837,953,457,711,458,621,528,722]
+ CRUSH rule 1 x 540 [831,50,132,213,197,709,95,789]
+ CRUSH rule 1 x 541 [582,757,121,525,532,963,738,277]
+ CRUSH rule 1 x 542 [472,132,790,997,948,269,137,934]
+ CRUSH rule 1 x 543 [382,272,797,330,315,748,324,134]
+ CRUSH rule 1 x 544 [947,930,496,883,509,219,250,362]
+ CRUSH rule 1 x 545 [425,570,305,77,821,422,117,172]
+ CRUSH rule 1 x 546 [18,65,529,437,343,547,699,610]
+ CRUSH rule 1 x 547 [445,715,600,472,213,851,428,267]
+ CRUSH rule 1 x 548 [367,569,980,167,627,442,517,684]
+ CRUSH rule 1 x 549 [125,715,671,817,285,420,37,639]
+ CRUSH rule 1 x 550 [425,599,744,199,923,222,915,570]
+ CRUSH rule 1 x 551 [44,1,528,922,944,115,161,901]
+ CRUSH rule 1 x 552 [246,104,68,239,123,427,57,217]
+ CRUSH rule 1 x 553 [71,703,615,28,593,724,218,916]
+ CRUSH rule 1 x 554 [207,124,217,166,525,226,693,953]
+ CRUSH rule 1 x 555 [570,28,317,420,931,413,623,659]
+ CRUSH rule 1 x 556 [674,152,421,79,215,347,830,762]
+ CRUSH rule 1 x 557 [347,817,191,391,741,571,593,267]
+ CRUSH rule 1 x 558 [627,426,369,692,815,371,124,107]
+ CRUSH rule 1 x 559 [940,630,924,242,224,912,185,356]
+ CRUSH rule 1 x 560 [295,903,541,29,245,753,887,376]
+ CRUSH rule 1 x 561 [506,682,384,637,878,991,700,339]
+ CRUSH rule 1 x 562 [718,529,87,729,842,341,62,817]
+ CRUSH rule 1 x 563 [552,332,747,206,274,871,903,900]
+ CRUSH rule 1 x 564 [835,769,736,486,630,209,641,751]
+ CRUSH rule 1 x 565 [8,167,539,182,607,62,738,873]
+ CRUSH rule 1 x 566 [600,481,301,263,90,450,184,127]
+ CRUSH rule 1 x 567 [999,994,509,899,947,24,267,639]
+ CRUSH rule 1 x 568 [252,431,157,62,601,863,398,521]
+ CRUSH rule 1 x 569 [643,218,943,455,83,969,494,624]
+ CRUSH rule 1 x 570 [617,635,765,422,250,156,533,674]
+ CRUSH rule 1 x 571 [757,80,59,98,328,700,329,848]
+ CRUSH rule 1 x 572 [299,348,575,889,943,675,33,312]
+ CRUSH rule 1 x 573 [25,505,270,167,58,901,878,978]
+ CRUSH rule 1 x 574 [215,431,624,177,628,814,333,841]
+ CRUSH rule 1 x 575 [225,252,611,546,32,815,389,486]
+ CRUSH rule 1 x 576 [627,94,159,857,430,691,177,545]
+ CRUSH rule 1 x 577 [237,809,778,636,61,167,700,521]
+ CRUSH rule 1 x 578 [885,313,120,344,771,614,487,976]
+ CRUSH rule 1 x 579 [924,575,787,831,47,996,557,630]
+ CRUSH rule 1 x 580 [718,51,766,121,118,471,608,755]
+ CRUSH rule 1 x 581 [219,807,129,571,856,179,874,902]
+ CRUSH rule 1 x 582 [893,701,598,863,285,829,984,622]
+ CRUSH rule 1 x 583 [246,930,964,170,993,409,469,193]
+ CRUSH rule 1 x 584 [336,432,680,175,495,839,642,226]
+ CRUSH rule 1 x 585 [324,999,397,485,457,527,73,628]
+ CRUSH rule 1 x 586 [558,230,976,541,816,72,794,682]
+ CRUSH rule 1 x 587 [985,830,597,21,308,890,952,421]
+ CRUSH rule 1 x 588 [211,544,57,134,162,496,195,581]
+ CRUSH rule 1 x 589 [129,21,112,190,885,844,753,180]
+ CRUSH rule 1 x 590 [467,969,652,593,287,76,811,413]
+ CRUSH rule 1 x 591 [758,514,316,164,35,110,54,796]
+ CRUSH rule 1 x 592 [525,253,190,443,315,603,667,318]
+ CRUSH rule 1 x 593 [601,885,339,152,297,223,269,455]
+ CRUSH rule 1 x 594 [227,60,450,30,717,840,994,16]
+ CRUSH rule 1 x 595 [720,854,496,912,80,655,917,525]
+ CRUSH rule 1 x 596 [751,195,997,77,261,490,180,482]
+ CRUSH rule 1 x 597 [129,574,714,8,789,847,725,991]
+ CRUSH rule 1 x 598 [679,207,604,396,841,284,286,280]
+ CRUSH rule 1 x 599 [668,315,683,349,681,253,599,364]
+ CRUSH rule 1 x 600 [143,396,464,444,59,57,243,264]
+ CRUSH rule 1 x 601 [326,573,873,902,136,921,633,596]
+ CRUSH rule 1 x 602 [860,281,875,535,672,474,697,763]
+ CRUSH rule 1 x 603 [709,328,445,349,190,455,924,667]
+ CRUSH rule 1 x 604 [571,62,814,95,866,978,983,281]
+ CRUSH rule 1 x 605 [252,739,860,27,313,362,857,899]
+ CRUSH rule 1 x 606 [339,236,759,842,67,644,954,94]
+ CRUSH rule 1 x 607 [590,248,759,868,433,398,578,386]
+ CRUSH rule 1 x 608 [145,635,309,467,875,115,148,33]
+ CRUSH rule 1 x 609 [973,547,223,79,762,863,249,41]
+ CRUSH rule 1 x 610 [435,816,961,983,255,886,160,888]
+ CRUSH rule 1 x 611 [559,283,422,584,176,429,570,43]
+ CRUSH rule 1 x 612 [273,149,123,576,911,270,296,735]
+ CRUSH rule 1 x 613 [828,614,642,674,33,361,958,580]
+ CRUSH rule 1 x 614 [478,748,393,34,171,80,92,12]
+ CRUSH rule 1 x 615 [392,155,144,326,626,134,149,401]
+ CRUSH rule 1 x 616 [778,637,452,248,15,888,74,307]
+ CRUSH rule 1 x 617 [622,713,996,833,611,407,364,8]
+ CRUSH rule 1 x 618 [149,877,270,329,180,327,222,749]
+ CRUSH rule 1 x 619 [604,163,656,409,322,848,519,967]
+ CRUSH rule 1 x 620 [181,23,409,198,64,898,35,620]
+ CRUSH rule 1 x 621 [735,902,386,237,939,475,725,118]
+ CRUSH rule 1 x 622 [661,824,717,568,858,583,446,798]
+ CRUSH rule 1 x 623 [142,121,643,61,695,852,485,478]
+ CRUSH rule 1 x 624 [360,716,420,398,49,717,137,140]
+ CRUSH rule 1 x 625 [541,167,385,1,601,481,308,111]
+ CRUSH rule 1 x 626 [364,431,610,363,535,747,225,841]
+ CRUSH rule 1 x 627 [458,137,557,410,287,749,467,432]
+ CRUSH rule 1 x 628 [250,350,556,497,821,65,205,580]
+ CRUSH rule 1 x 629 [928,160,710,572,365,772,538,46]
+ CRUSH rule 1 x 630 [243,19,918,556,601,16,920,830]
+ CRUSH rule 1 x 631 [438,221,574,676,797,580,219,211]
+ CRUSH rule 1 x 632 [797,368,247,5,32,102,416,45]
+ CRUSH rule 1 x 633 [993,749,525,485,27,330,275,599]
+ CRUSH rule 1 x 634 [239,351,633,299,651,678,296,337]
+ CRUSH rule 1 x 635 [640,965,25,961,306,172,849,357]
+ CRUSH rule 1 x 636 [173,290,297,991,937,823,236,318]
+ CRUSH rule 1 x 637 [0,918,98,108,111,495,887,57]
+ CRUSH rule 1 x 638 [702,235,424,900,983,754,701,887]
+ CRUSH rule 1 x 639 [475,687,31,785,918,611,27,214]
+ CRUSH rule 1 x 640 [31,664,399,677,123,609,858,138]
+ CRUSH rule 1 x 641 [296,473,108,963,341,876,897,449]
+ CRUSH rule 1 x 642 [894,273,427,606,677,670,610,665]
+ CRUSH rule 1 x 643 [117,111,732,191,114,153,500,631]
+ CRUSH rule 1 x 644 [438,336,327,512,599,862,660,857]
+ CRUSH rule 1 x 645 [982,702,351,573,907,915,279,317]
+ CRUSH rule 1 x 646 [334,804,146,842,697,638,720,135]
+ CRUSH rule 1 x 647 [933,787,185,334,752,285,372,890]
+ CRUSH rule 1 x 648 [22,444,400,862,207,842,453,732]
+ CRUSH rule 1 x 649 [503,229,213,460,639,760,722,748]
+ CRUSH rule 1 x 650 [328,659,420,443,739,950,869,150]
+ CRUSH rule 1 x 651 [3,880,823,123,378,585,715,221]
+ CRUSH rule 1 x 652 [495,977,563,733,92,997,119,818]
+ CRUSH rule 1 x 653 [185,718,804,280,975,912,198,291]
+ CRUSH rule 1 x 654 [130,528,380,81,906,511,750,506]
+ CRUSH rule 1 x 655 [560,872,454,504,319,284,605,214]
+ CRUSH rule 1 x 656 [219,885,178,981,863,508,708,6]
+ CRUSH rule 1 x 657 [233,684,813,490,208,941,858,16]
+ CRUSH rule 1 x 658 [778,6,756,380,750,836,547,850]
+ CRUSH rule 1 x 659 [240,663,306,540,789,902,170,954]
+ CRUSH rule 1 x 660 [244,855,196,147,678,323,63,859]
+ CRUSH rule 1 x 661 [184,270,128,398,910,230,402,205]
+ CRUSH rule 1 x 662 [65,883,921,438,79,957,464,902]
+ CRUSH rule 1 x 663 [323,721,594,812,43,992,170,65]
+ CRUSH rule 1 x 664 [865,113,512,51,427,123,585,260]
+ CRUSH rule 1 x 665 [420,850,591,475,202,733,798,658]
+ CRUSH rule 1 x 666 [319,767,246,3,369,493,796,56]
+ CRUSH rule 1 x 667 [875,39,343,100,829,2,795,783]
+ CRUSH rule 1 x 668 [331,122,263,599,355,484,943,554]
+ CRUSH rule 1 x 669 [915,521,402,747,673,445,938,600]
+ CRUSH rule 1 x 670 [845,659,943,447,401,322,168,302]
+ CRUSH rule 1 x 671 [108,634,527,363,856,238,755,330]
+ CRUSH rule 1 x 672 [578,216,110,589,302,137,954,315]
+ CRUSH rule 1 x 673 [442,74,579,797,622,950,371,402]
+ CRUSH rule 1 x 674 [588,364,281,308,645,631,229,506]
+ CRUSH rule 1 x 675 [489,698,744,671,870,174,528,875]
+ CRUSH rule 1 x 676 [928,911,40,180,722,729,673,569]
+ CRUSH rule 1 x 677 [399,269,692,131,615,136,103,763]
+ CRUSH rule 1 x 678 [546,752,544,155,5,463,666,352]
+ CRUSH rule 1 x 679 [988,25,275,433,628,57,247,620]
+ CRUSH rule 1 x 680 [335,963,382,486,749,257,795,347]
+ CRUSH rule 1 x 681 [690,462,623,466,49,471,774,192]
+ CRUSH rule 1 x 682 [196,588,154,257,807,776,367,718]
+ CRUSH rule 1 x 683 [627,25,421,160,873,102,345,599]
+ CRUSH rule 1 x 684 [38,804,592,158,991,264,652,821]
+ CRUSH rule 1 x 685 [841,368,548,362,166,211,154,121]
+ CRUSH rule 1 x 686 [336,287,525,440,166,993,911,638]
+ CRUSH rule 1 x 687 [20,682,924,653,356,16,917,622]
+ CRUSH rule 1 x 688 [463,371,780,556,385,883,115,248]
+ CRUSH rule 1 x 689 [569,250,78,816,847,775,333,161]
+ CRUSH rule 1 x 690 [551,144,587,263,378,394,970,639]
+ CRUSH rule 1 x 691 [766,464,446,533,449,541,451,290]
+ CRUSH rule 1 x 692 [739,634,18,245,624,35,268,525]
+ CRUSH rule 1 x 693 [339,297,118,330,817,91,828,276]
+ CRUSH rule 1 x 694 [405,26,830,181,533,166,488,804]
+ CRUSH rule 1 x 695 [622,576,597,535,600,593,300,989]
+ CRUSH rule 1 x 696 [558,902,689,13,715,28,664,489]
+ CRUSH rule 1 x 697 [818,222,406,691,427,863,153,922]
+ CRUSH rule 1 x 698 [178,48,402,233,841,604,468,180]
+ CRUSH rule 1 x 699 [450,244,180,919,426,332,747,453]
+ CRUSH rule 1 x 700 [502,771,987,706,416,240,68,641]
+ CRUSH rule 1 x 701 [4,612,782,216,853,303,585,513]
+ CRUSH rule 1 x 702 [177,630,232,923,281,708,466,687]
+ CRUSH rule 1 x 703 [354,178,389,393,778,803,796,607]
+ CRUSH rule 1 x 704 [646,601,156,171,603,116,655,595]
+ CRUSH rule 1 x 705 [921,401,890,265,244,690,372,253]
+ CRUSH rule 1 x 706 [652,877,562,452,26,323,923,770]
+ CRUSH rule 1 x 707 [345,745,67,716,789,576,2,133]
+ CRUSH rule 1 x 708 [333,607,180,469,170,555,939,331]
+ CRUSH rule 1 x 709 [45,187,302,115,896,579,733,607]
+ CRUSH rule 1 x 710 [94,855,43,199,18,948,449,28]
+ CRUSH rule 1 x 711 [227,653,731,150,278,842,534,110]
+ CRUSH rule 1 x 712 [398,953,136,870,181,408,895,459]
+ CRUSH rule 1 x 713 [116,800,503,662,635,579,53,839]
+ CRUSH rule 1 x 714 [111,629,866,709,902,557,875,649]
+ CRUSH rule 1 x 715 [531,291,486,382,192,807,322,417]
+ CRUSH rule 1 x 716 [169,541,291,42,343,724,138,197]
+ CRUSH rule 1 x 717 [417,446,994,894,239,494,237,62]
+ CRUSH rule 1 x 718 [992,383,298,844,377,463,544,891]
+ CRUSH rule 1 x 719 [936,674,324,759,194,409,828,975]
+ CRUSH rule 1 x 720 [370,188,174,464,644,218,214,76]
+ CRUSH rule 1 x 721 [320,859,278,259,170,957,177,264]
+ CRUSH rule 1 x 722 [7,2,673,129,96,445,823,833]
+ CRUSH rule 1 x 723 [270,553,831,662,38,101,985,846]
+ CRUSH rule 1 x 724 [666,822,708,895,633,800,616,879]
+ CRUSH rule 1 x 725 [794,406,875,459,981,751,359,926]
+ CRUSH rule 1 x 726 [420,556,341,292,240,68,966,535]
+ CRUSH rule 1 x 727 [561,461,129,635,965,610,105,31]
+ CRUSH rule 1 x 728 [951,330,196,756,589,849,753,760]
+ CRUSH rule 1 x 729 [656,644,436,591,27,119,572,933]
+ CRUSH rule 1 x 730 [3,558,629,184,50,765,760,800]
+ CRUSH rule 1 x 731 [852,89,75,735,713,113,528,890]
+ CRUSH rule 1 x 732 [983,840,869,976,697,307,368,271]
+ CRUSH rule 1 x 733 [285,396,388,122,387,364,880,343]
+ CRUSH rule 1 x 734 [125,510,402,640,676,501,535,627]
+ CRUSH rule 1 x 735 [417,773,686,504,459,912,690,59]
+ CRUSH rule 1 x 736 [749,396,632,550,779,109,845,278]
+ CRUSH rule 1 x 737 [644,991,946,135,448,903,482,564]
+ CRUSH rule 1 x 738 [449,683,290,220,245,525,429,397]
+ CRUSH rule 1 x 739 [341,220,641,454,740,661,146,17]
+ CRUSH rule 1 x 740 [874,524,674,650,472,282,214,494]
+ CRUSH rule 1 x 741 [189,472,712,798,715,757,863,571]
+ CRUSH rule 1 x 742 [912,581,114,377,730,21,687,81]
+ CRUSH rule 1 x 743 [654,914,425,441,763,39,451,631]
+ CRUSH rule 1 x 744 [725,295,579,377,162,447,843,699]
+ CRUSH rule 1 x 745 [787,858,850,506,612,735,926,314]
+ CRUSH rule 1 x 746 [757,848,704,30,47,940,450,651]
+ CRUSH rule 1 x 747 [700,81,867,681,801,64,879,857]
+ CRUSH rule 1 x 748 [557,436,238,664,293,865,304,999]
+ CRUSH rule 1 x 749 [772,622,337,42,156,302,383,506]
+ CRUSH rule 1 x 750 [946,97,376,677,316,670,169,171]
+ CRUSH rule 1 x 751 [996,618,343,911,83,22,388,17]
+ CRUSH rule 1 x 752 [746,887,695,868,610,950,88,315]
+ CRUSH rule 1 x 753 [741,14,463,479,172,192,481,702]
+ CRUSH rule 1 x 754 [648,349,333,355,65,63,336,724]
+ CRUSH rule 1 x 755 [157,460,466,187,959,674,192,279]
+ CRUSH rule 1 x 756 [416,97,197,497,227,3,850,191]
+ CRUSH rule 1 x 757 [599,839,776,410,256,823,121,690]
+ CRUSH rule 1 x 758 [994,218,620,256,361,749,165,686]
+ CRUSH rule 1 x 759 [959,682,514,745,100,519,15,347]
+ CRUSH rule 1 x 760 [518,943,215,83,706,137,345,69]
+ CRUSH rule 1 x 761 [285,849,420,324,987,338,373,361]
+ CRUSH rule 1 x 762 [591,313,41,335,110,696,664,350]
+ CRUSH rule 1 x 763 [908,411,200,740,292,295,387,775]
+ CRUSH rule 1 x 764 [787,234,894,485,883,711,70,202]
+ CRUSH rule 1 x 765 [327,921,882,393,444,792,402,123]
+ CRUSH rule 1 x 766 [84,161,878,704,416,144,357,310]
+ CRUSH rule 1 x 767 [370,895,702,701,890,2,251,951]
+ CRUSH rule 1 x 768 [826,760,879,864,460,474,645,975]
+ CRUSH rule 1 x 769 [67,768,663,735,814,66,213,527]
+ CRUSH rule 1 x 770 [593,909,482,259,5,550,961,324]
+ CRUSH rule 1 x 771 [309,935,121,578,937,685,933,571]
+ CRUSH rule 1 x 772 [12,125,797,301,348,419,891,959]
+ CRUSH rule 1 x 773 [253,466,820,549,591,193,783,951]
+ CRUSH rule 1 x 774 [164,390,705,109,881,505,890,425]
+ CRUSH rule 1 x 775 [703,47,43,973,643,406,885,976]
+ CRUSH rule 1 x 776 [728,231,80,916,2,850,396,76]
+ CRUSH rule 1 x 777 [981,621,568,729,869,952,563,860]
+ CRUSH rule 1 x 778 [411,456,544,597,789,784,65,954]
+ CRUSH rule 1 x 779 [346,121,519,921,587,48,772,645]
+ CRUSH rule 1 x 780 [476,39,288,381,303,29,17,336]
+ CRUSH rule 1 x 781 [10,130,585,844,729,705,714,954]
+ CRUSH rule 1 x 782 [462,246,581,902,623,877,812,516]
+ CRUSH rule 1 x 783 [580,373,153,775,668,661,626,961]
+ CRUSH rule 1 x 784 [413,113,978,990,994,56,481,198]
+ CRUSH rule 1 x 785 [341,856,332,354,59,581,632,151]
+ CRUSH rule 1 x 786 [411,140,313,393,215,618,490,481]
+ CRUSH rule 1 x 787 [605,522,211,813,636,224,600,528]
+ CRUSH rule 1 x 788 [226,545,35,142,726,851,194,216]
+ CRUSH rule 1 x 789 [545,320,414,702,731,277,237,916]
+ CRUSH rule 1 x 790 [414,748,816,327,130,115,788,164]
+ CRUSH rule 1 x 791 [660,906,406,697,916,322,124,711]
+ CRUSH rule 1 x 792 [287,392,514,204,75,789,406,858]
+ CRUSH rule 1 x 793 [631,133,850,713,720,487,376,812]
+ CRUSH rule 1 x 794 [931,517,543,210,963,898,811,459]
+ CRUSH rule 1 x 795 [551,962,477,948,425,434,268,94]
+ CRUSH rule 1 x 796 [814,4,95,27,368,300,646,451]
+ CRUSH rule 1 x 797 [64,201,299,734,605,864,596,196]
+ CRUSH rule 1 x 798 [422,530,114,431,565,716,473,250]
+ CRUSH rule 1 x 799 [824,32,679,562,266,549,859,994]
+ CRUSH rule 1 x 800 [862,623,489,637,861,196,941,643]
+ CRUSH rule 1 x 801 [145,550,329,324,734,160,219,662]
+ CRUSH rule 1 x 802 [570,19,847,308,387,518,846,53]
+ CRUSH rule 1 x 803 [151,812,662,358,880,349,834,881]
+ CRUSH rule 1 x 804 [467,93,264,863,176,842,663,949]
+ CRUSH rule 1 x 805 [621,223,938,809,591,686,121,157]
+ CRUSH rule 1 x 806 [898,957,805,430,499,584,640,607]
+ CRUSH rule 1 x 807 [354,531,422,159,921,431,802,136]
+ CRUSH rule 1 x 808 [7,96,76,897,446,2,166,929]
+ CRUSH rule 1 x 809 [70,734,719,56,687,21,23,145]
+ CRUSH rule 1 x 810 [701,18,972,327,771,649,620,648]
+ CRUSH rule 1 x 811 [248,547,103,728,901,264,948,202]
+ CRUSH rule 1 x 812 [230,576,821,566,993,762,675,28]
+ CRUSH rule 1 x 813 [805,114,683,629,396,462,285,450]
+ CRUSH rule 1 x 814 [54,619,973,741,497,894,401,266]
+ CRUSH rule 1 x 815 [679,412,613,132,969,411,314,670]
+ CRUSH rule 1 x 816 [919,448,826,414,36,289,44,822]
+ CRUSH rule 1 x 817 [765,830,436,521,332,458,260,172]
+ CRUSH rule 1 x 818 [415,566,644,687,692,414,769,826]
+ CRUSH rule 1 x 819 [721,319,865,750,546,859,523,770]
+ CRUSH rule 1 x 820 [218,301,333,190,686,179,535,787]
+ CRUSH rule 1 x 821 [185,795,680,953,329,750,621,815]
+ CRUSH rule 1 x 822 [356,261,54,522,900,103,883,112]
+ CRUSH rule 1 x 823 [220,281,549,456,64,306,282,641]
+ CRUSH rule 1 x 824 [292,809,887,74,776,788,559,886]
+ CRUSH rule 1 x 825 [949,778,101,311,110,480,161,998]
+ CRUSH rule 1 x 826 [767,818,833,927,356,954,910,63]
+ CRUSH rule 1 x 827 [631,83,406,635,657,713,212,916]
+ CRUSH rule 1 x 828 [288,986,445,26,414,607,937,595]
+ CRUSH rule 1 x 829 [990,667,915,694,974,453,669,330]
+ CRUSH rule 1 x 830 [152,571,778,505,685,209,448,55]
+ CRUSH rule 1 x 831 [814,563,630,97,582,107,142,157]
+ CRUSH rule 1 x 832 [235,641,616,110,979,844,656,135]
+ CRUSH rule 1 x 833 [657,565,922,140,825,457,764,766]
+ CRUSH rule 1 x 834 [907,231,644,13,617,130,83,483]
+ CRUSH rule 1 x 835 [784,262,771,264,612,238,537,937]
+ CRUSH rule 1 x 836 [951,158,366,710,43,427,351,961]
+ CRUSH rule 1 x 837 [556,498,334,633,895,627,903,29]
+ CRUSH rule 1 x 838 [329,274,964,547,119,342,983,998]
+ CRUSH rule 1 x 839 [568,209,939,364,658,747,47,859]
+ CRUSH rule 1 x 840 [45,579,842,70,655,862,815,109]
+ CRUSH rule 1 x 841 [652,702,24,605,152,93,226,46]
+ CRUSH rule 1 x 842 [629,984,314,895,408,897,575,1]
+ CRUSH rule 1 x 843 [799,690,688,648,151,812,486,199]
+ CRUSH rule 1 x 844 [694,600,534,700,569,11,899,382]
+ CRUSH rule 1 x 845 [332,30,179,93,951,324,611,512]
+ CRUSH rule 1 x 846 [452,251,712,719,404,739,606,237]
+ CRUSH rule 1 x 847 [399,681,847,739,13,555,363,893]
+ CRUSH rule 1 x 848 [303,138,440,346,547,216,700,249]
+ CRUSH rule 1 x 849 [666,346,708,873,64,694,847,463]
+ CRUSH rule 1 x 850 [644,511,345,844,545,337,358,35]
+ CRUSH rule 1 x 851 [527,546,737,425,100,331,95,337]
+ CRUSH rule 1 x 852 [31,809,94,618,156,853,469,511]
+ CRUSH rule 1 x 853 [483,330,869,184,46,942,774,679]
+ CRUSH rule 1 x 854 [697,953,968,143,502,955,441,302]
+ CRUSH rule 1 x 855 [837,996,239,621,32,191,686,702]
+ CRUSH rule 1 x 856 [712,40,547,430,195,857,224,810]
+ CRUSH rule 1 x 857 [77,984,576,551,568,96,12,763]
+ CRUSH rule 1 x 858 [412,384,841,465,572,576,688,61]
+ CRUSH rule 1 x 859 [173,760,26,300,87,567,463,903]
+ CRUSH rule 1 x 860 [776,429,328,917,658,783,699,907]
+ CRUSH rule 1 x 861 [705,405,477,50,73,714,901,487]
+ CRUSH rule 1 x 862 [809,44,788,938,964,177,490,409]
+ CRUSH rule 1 x 863 [349,496,963,178,675,853,172,980]
+ CRUSH rule 1 x 864 [717,858,101,239,992,244,43,15]
+ CRUSH rule 1 x 865 [857,603,586,262,550,289,850,40]
+ CRUSH rule 1 x 866 [394,304,71,96,642,155,255,481]
+ CRUSH rule 1 x 867 [640,773,663,974,261,296,988,730]
+ CRUSH rule 1 x 868 [613,950,712,663,101,460,643,547]
+ CRUSH rule 1 x 869 [973,889,524,22,671,477,718,431]
+ CRUSH rule 1 x 870 [505,35,386,498,348,503,54,992]
+ CRUSH rule 1 x 871 [239,264,262,773,781,734,387,515]
+ CRUSH rule 1 x 872 [21,767,456,748,783,797,180,800]
+ CRUSH rule 1 x 873 [954,666,980,264,435,233,199,358]
+ CRUSH rule 1 x 874 [54,510,947,1,500,119,93,915]
+ CRUSH rule 1 x 875 [809,418,452,462,88,673,634,435]
+ CRUSH rule 1 x 876 [483,457,61,248,523,277,322,141]
+ CRUSH rule 1 x 877 [542,531,952,939,710,179,181,460]
+ CRUSH rule 1 x 878 [217,674,857,644,678,809,329,591]
+ CRUSH rule 1 x 879 [999,475,134,250,319,357,145,750]
+ CRUSH rule 1 x 880 [678,573,935,385,570,651,319,630]
+ CRUSH rule 1 x 881 [394,835,789,802,587,155,570,109]
+ CRUSH rule 1 x 882 [467,382,353,56,979,674,974,483]
+ CRUSH rule 1 x 883 [802,744,237,337,50,96,202,148]
+ CRUSH rule 1 x 884 [653,660,638,700,31,558,389,381]
+ CRUSH rule 1 x 885 [898,704,307,445,879,872,174,972]
+ CRUSH rule 1 x 886 [434,357,938,641,737,8,56,582]
+ CRUSH rule 1 x 887 [297,226,711,428,370,318,472,947]
+ CRUSH rule 1 x 888 [863,324,443,213,902,25,806,53]
+ CRUSH rule 1 x 889 [105,102,308,163,947,548,399,382]
+ CRUSH rule 1 x 890 [550,248,606,704,615,708,996,561]
+ CRUSH rule 1 x 891 [575,928,880,891,826,763,706,701]
+ CRUSH rule 1 x 892 [259,862,133,271,292,162,53,333]
+ CRUSH rule 1 x 893 [902,880,543,542,37,942,672,320]
+ CRUSH rule 1 x 894 [180,169,916,43,945,713,648,685]
+ CRUSH rule 1 x 895 [725,849,182,129,177,272,599,829]
+ CRUSH rule 1 x 896 [951,34,874,537,969,123,210,529]
+ CRUSH rule 1 x 897 [810,352,73,939,943,895,12,481]
+ CRUSH rule 1 x 898 [979,433,719,411,787,359,342,37]
+ CRUSH rule 1 x 899 [685,668,534,932,399,156,124,653]
+ CRUSH rule 1 x 900 [530,978,41,894,941,681,380,419]
+ CRUSH rule 1 x 901 [740,107,336,175,574,706,157,292]
+ CRUSH rule 1 x 902 [800,743,693,310,67,111,178,624]
+ CRUSH rule 1 x 903 [230,267,842,266,550,769,66,738]
+ CRUSH rule 1 x 904 [346,949,460,973,696,91,957,801]
+ CRUSH rule 1 x 905 [530,397,619,958,576,973,685,6]
+ CRUSH rule 1 x 906 [80,426,138,672,73,776,30,169]
+ CRUSH rule 1 x 907 [365,968,475,297,296,724,664,331]
+ CRUSH rule 1 x 908 [204,832,742,809,862,745,484,391]
+ CRUSH rule 1 x 909 [883,989,146,959,366,59,686,965]
+ CRUSH rule 1 x 910 [549,593,249,853,792,769,824,552]
+ CRUSH rule 1 x 911 [325,847,352,214,851,732,789,255]
+ CRUSH rule 1 x 912 [874,888,582,796,557,601,226,889]
+ CRUSH rule 1 x 913 [331,463,342,574,989,362,925,746]
+ CRUSH rule 1 x 914 [836,468,601,732,607,275,70,280]
+ CRUSH rule 1 x 915 [245,228,100,661,799,13,126,79]
+ CRUSH rule 1 x 916 [77,967,364,435,27,474,255,133]
+ CRUSH rule 1 x 917 [239,60,866,221,772,967,725,707]
+ CRUSH rule 1 x 918 [988,115,922,80,201,544,583,923]
+ CRUSH rule 1 x 919 [783,139,696,1,848,169,888,980]
+ CRUSH rule 1 x 920 [623,408,685,953,974,696,532,124]
+ CRUSH rule 1 x 921 [105,799,144,90,399,373,633,290]
+ CRUSH rule 1 x 922 [887,505,652,348,514,806,952,474]
+ CRUSH rule 1 x 923 [223,318,552,458,743,871,964,384]
+ CRUSH rule 1 x 924 [25,778,366,333,163,801,584,31]
+ CRUSH rule 1 x 925 [912,601,297,682,770,173,969,168]
+ CRUSH rule 1 x 926 [968,133,175,144,814,155,709,158]
+ CRUSH rule 1 x 927 [277,724,214,988,690,342,465,775]
+ CRUSH rule 1 x 928 [554,203,658,789,298,299,847,752]
+ CRUSH rule 1 x 929 [761,802,367,528,758,522,744,171]
+ CRUSH rule 1 x 930 [814,61,788,736,660,491,832,654]
+ CRUSH rule 1 x 931 [29,193,61,41,343,664,487,839]
+ CRUSH rule 1 x 932 [446,198,862,534,168,35,530,462]
+ CRUSH rule 1 x 933 [352,742,216,321,525,44,568,61]
+ CRUSH rule 1 x 934 [730,2,332,631,613,249,533,116]
+ CRUSH rule 1 x 935 [731,23,736,79,361,992,772,49]
+ CRUSH rule 1 x 936 [322,975,20,904,827,603,138,802]
+ CRUSH rule 1 x 937 [822,221,841,161,723,137,630,308]
+ CRUSH rule 1 x 938 [557,850,66,630,499,404,286,395]
+ CRUSH rule 1 x 939 [150,11,971,371,124,785,408,49]
+ CRUSH rule 1 x 940 [638,398,169,616,333,751,25,883]
+ CRUSH rule 1 x 941 [730,342,929,577,451,838,964,28]
+ CRUSH rule 1 x 942 [62,292,166,814,587,172,724,16]
+ CRUSH rule 1 x 943 [165,314,519,548,41,726,759,851]
+ CRUSH rule 1 x 944 [199,625,766,176,194,297,678,915]
+ CRUSH rule 1 x 945 [946,999,699,303,38,81,952,885]
+ CRUSH rule 1 x 946 [595,93,852,142,503,647,933,267]
+ CRUSH rule 1 x 947 [800,582,356,93,716,117,922,868]
+ CRUSH rule 1 x 948 [132,551,139,920,87,46,81,220]
+ CRUSH rule 1 x 949 [792,920,466,380,97,568,799,961]
+ CRUSH rule 1 x 950 [111,345,176,543,879,954,355,220]
+ CRUSH rule 1 x 951 [414,619,648,655,364,971,829,408]
+ CRUSH rule 1 x 952 [775,469,500,356,287,4,16,746]
+ CRUSH rule 1 x 953 [349,1,5,251,168,680,141,619]
+ CRUSH rule 1 x 954 [570,940,410,249,929,394,129,696]
+ CRUSH rule 1 x 955 [729,774,823,800,7,127,536,766]
+ CRUSH rule 1 x 956 [519,141,575,625,738,475,169,751]
+ CRUSH rule 1 x 957 [242,709,611,97,760,309,393,281]
+ CRUSH rule 1 x 958 [84,217,227,253,246,604,346,377]
+ CRUSH rule 1 x 959 [270,413,918,789,703,608,543,519]
+ CRUSH rule 1 x 960 [458,192,307,279,920,139,855,49]
+ CRUSH rule 1 x 961 [981,388,777,546,359,660,455,708]
+ CRUSH rule 1 x 962 [623,834,277,134,729,246,856,477]
+ CRUSH rule 1 x 963 [291,167,714,468,109,373,485,701]
+ CRUSH rule 1 x 964 [28,156,788,127,598,215,361,255]
+ CRUSH rule 1 x 965 [675,557,290,517,840,510,59,229]
+ CRUSH rule 1 x 966 [836,306,946,283,642,606,929,773]
+ CRUSH rule 1 x 967 [966,386,735,837,392,116,19,674]
+ CRUSH rule 1 x 968 [864,756,690,121,328,122,433,520]
+ CRUSH rule 1 x 969 [729,625,480,769,512,882,518,956]
+ CRUSH rule 1 x 970 [800,362,646,582,309,102,576,411]
+ CRUSH rule 1 x 971 [737,381,153,684,298,166,344,520]
+ CRUSH rule 1 x 972 [952,245,720,884,334,311,754,540]
+ CRUSH rule 1 x 973 [356,455,579,857,832,596,549,524]
+ CRUSH rule 1 x 974 [545,758,586,596,46,790,116,993]
+ CRUSH rule 1 x 975 [336,191,202,146,720,897,330,308]
+ CRUSH rule 1 x 976 [446,208,757,620,252,846,397,58]
+ CRUSH rule 1 x 977 [202,896,196,956,763,126,783,828]
+ CRUSH rule 1 x 978 [612,324,996,225,418,583,514,169]
+ CRUSH rule 1 x 979 [843,457,675,650,958,657,677,173]
+ CRUSH rule 1 x 980 [60,914,881,626,850,759,398,943]
+ CRUSH rule 1 x 981 [702,749,937,153,724,514,536,212]
+ CRUSH rule 1 x 982 [298,928,738,167,99,668,395,198]
+ CRUSH rule 1 x 983 [723,572,395,358,900,37,927,597]
+ CRUSH rule 1 x 984 [723,864,804,935,846,993,950,840]
+ CRUSH rule 1 x 985 [945,459,868,211,524,954,911,208]
+ CRUSH rule 1 x 986 [772,664,535,169,297,996,864,555]
+ CRUSH rule 1 x 987 [88,324,312,843,661,580,76,894]
+ CRUSH rule 1 x 988 [522,927,131,996,351,685,865,47]
+ CRUSH rule 1 x 989 [578,332,208,605,975,207,155,380]
+ CRUSH rule 1 x 990 [638,228,414,311,738,698,340,526]
+ CRUSH rule 1 x 991 [530,221,451,422,879,916,754,928]
+ CRUSH rule 1 x 992 [925,705,275,81,234,310,117,546]
+ CRUSH rule 1 x 993 [991,301,43,469,830,242,382,428]
+ CRUSH rule 1 x 994 [276,51,868,683,843,815,557,378]
+ CRUSH rule 1 x 995 [288,836,753,790,758,120,158,265]
+ CRUSH rule 1 x 996 [887,983,252,686,470,345,459,764]
+ CRUSH rule 1 x 997 [110,924,386,79,705,697,210,698]
+ CRUSH rule 1 x 998 [435,830,485,853,926,730,786,762]
+ CRUSH rule 1 x 999 [876,738,357,913,723,51,15,585]
+ CRUSH rule 1 x 1000 [178,963,638,430,845,586,317,102]
+ CRUSH rule 1 x 1001 [99,519,66,759,583,944,739,922]
+ CRUSH rule 1 x 1002 [515,534,468,866,878,717,729,370]
+ CRUSH rule 1 x 1003 [104,611,937,698,94,67,614,783]
+ CRUSH rule 1 x 1004 [269,638,724,375,491,121,891,113]
+ CRUSH rule 1 x 1005 [369,223,309,409,822,39,597,969]
+ CRUSH rule 1 x 1006 [40,107,69,275,79,429,234,945]
+ CRUSH rule 1 x 1007 [978,111,416,758,454,640,5,444]
+ CRUSH rule 1 x 1008 [965,956,624,832,421,96,975,723]
+ CRUSH rule 1 x 1009 [598,476,356,695,919,566,234,383]
+ CRUSH rule 1 x 1010 [767,523,239,517,29,77,23,241]
+ CRUSH rule 1 x 1011 [289,871,207,576,347,698,48,570]
+ CRUSH rule 1 x 1012 [128,28,370,31,341,755,268,647]
+ CRUSH rule 1 x 1013 [979,765,660,812,666,187,808,351]
+ CRUSH rule 1 x 1014 [979,948,513,88,47,825,969,81]
+ CRUSH rule 1 x 1015 [277,790,396,672,542,647,145,11]
+ CRUSH rule 1 x 1016 [262,73,128,886,839,685,456,560]
+ CRUSH rule 1 x 1017 [150,269,61,499,832,591,637,731]
+ CRUSH rule 1 x 1018 [555,829,554,944,406,576,463,926]
+ CRUSH rule 1 x 1019 [513,356,265,446,65,288,768,245]
+ CRUSH rule 1 x 1020 [158,161,877,704,948,570,495,865]
+ CRUSH rule 1 x 1021 [915,998,957,285,546,202,676,322]
+ CRUSH rule 1 x 1022 [967,829,973,640,703,470,871,828]
+ CRUSH rule 1 x 1023 [488,257,614,859,325,419,50,560]
+ rule 1 (metadata) num_rep 8 result size == 8:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450,604,380,966,750,695]
+ CRUSH rule 1 x 1 [876,250,334,633,744,843,672,820,782]
+ CRUSH rule 1 x 2 [292,832,53,392,386,787,527,901,106]
+ CRUSH rule 1 x 3 [623,387,124,998,749,211,481,169,816]
+ CRUSH rule 1 x 4 [61,334,710,4,994,982,847,220,87]
+ CRUSH rule 1 x 5 [946,557,713,664,141,817,964,872,66]
+ CRUSH rule 1 x 6 [576,668,212,163,732,381,884,726,456]
+ CRUSH rule 1 x 7 [645,753,906,393,341,44,578,14,543]
+ CRUSH rule 1 x 8 [243,6,863,781,211,100,462,207,759]
+ CRUSH rule 1 x 9 [22,578,251,410,297,430,3,569,603]
+ CRUSH rule 1 x 10 [758,828,360,477,821,801,811,484,296]
+ CRUSH rule 1 x 11 [769,120,124,527,119,504,380,821,470]
+ CRUSH rule 1 x 12 [780,364,689,755,675,199,117,393,435]
+ CRUSH rule 1 x 13 [557,18,351,719,742,780,78,170,333]
+ CRUSH rule 1 x 14 [59,561,249,461,971,835,855,76,269]
+ CRUSH rule 1 x 15 [718,928,993,21,76,313,437,462,680]
+ CRUSH rule 1 x 16 [673,632,841,954,788,90,786,969,378]
+ CRUSH rule 1 x 17 [648,43,560,514,142,289,935,605,228]
+ CRUSH rule 1 x 18 [654,219,181,568,381,253,883,394,188]
+ CRUSH rule 1 x 19 [850,545,377,848,863,543,51,834,690]
+ CRUSH rule 1 x 20 [717,785,974,5,225,552,975,636,387]
+ CRUSH rule 1 x 21 [420,57,519,306,312,983,263,267,128]
+ CRUSH rule 1 x 22 [503,998,193,821,634,684,557,633,812]
+ CRUSH rule 1 x 23 [411,663,168,110,899,488,477,468,303]
+ CRUSH rule 1 x 24 [266,861,353,1,456,128,800,309,622]
+ CRUSH rule 1 x 25 [760,483,818,600,509,951,248,908,624]
+ CRUSH rule 1 x 26 [903,24,573,718,112,694,501,909,219]
+ CRUSH rule 1 x 27 [946,188,289,510,687,827,676,560,753]
+ CRUSH rule 1 x 28 [69,312,73,198,256,629,770,569,359]
+ CRUSH rule 1 x 29 [844,883,337,628,496,405,719,581,816]
+ CRUSH rule 1 x 30 [621,18,613,794,910,936,426,522,208]
+ CRUSH rule 1 x 31 [784,943,814,539,962,392,813,217,750]
+ CRUSH rule 1 x 32 [173,374,369,972,315,83,428,63,801]
+ CRUSH rule 1 x 33 [698,336,357,966,582,407,618,288,846]
+ CRUSH rule 1 x 34 [168,836,210,798,904,190,663,877,177]
+ CRUSH rule 1 x 35 [274,509,534,818,912,671,75,580,568]
+ CRUSH rule 1 x 36 [318,215,153,628,87,407,676,524,510]
+ CRUSH rule 1 x 37 [173,604,109,935,203,401,311,758,201]
+ CRUSH rule 1 x 38 [708,444,683,604,722,900,929,910,698]
+ CRUSH rule 1 x 39 [662,198,417,680,226,342,856,248,279]
+ CRUSH rule 1 x 40 [620,801,414,78,560,766,980,503,287]
+ CRUSH rule 1 x 41 [811,264,177,127,148,791,930,74,844]
+ CRUSH rule 1 x 42 [863,179,527,660,133,529,456,713,348]
+ CRUSH rule 1 x 43 [686,822,988,228,791,549,514,40,261]
+ CRUSH rule 1 x 44 [396,222,46,841,536,140,160,527,250]
+ CRUSH rule 1 x 45 [991,694,253,142,54,422,658,876,201]
+ CRUSH rule 1 x 46 [420,909,184,285,508,458,45,390,546]
+ CRUSH rule 1 x 47 [467,211,605,207,241,881,959,800,743]
+ CRUSH rule 1 x 48 [955,329,368,168,698,787,738,47,812]
+ CRUSH rule 1 x 49 [974,891,931,29,813,506,822,628,696]
+ CRUSH rule 1 x 50 [870,441,691,823,761,6,83,344,713]
+ CRUSH rule 1 x 51 [182,930,25,936,97,260,406,281,991]
+ CRUSH rule 1 x 52 [704,812,894,794,481,37,304,899,629]
+ CRUSH rule 1 x 53 [185,713,631,280,345,558,882,503,327]
+ CRUSH rule 1 x 54 [270,441,100,82,983,930,339,902,81]
+ CRUSH rule 1 x 55 [895,734,958,793,651,572,508,763,108]
+ CRUSH rule 1 x 56 [564,963,683,324,40,189,77,500,553]
+ CRUSH rule 1 x 57 [738,130,208,973,498,861,670,67,114]
+ CRUSH rule 1 x 58 [524,113,806,903,531,334,8,762,842]
+ CRUSH rule 1 x 59 [408,337,668,529,34,384,643,511,370]
+ CRUSH rule 1 x 60 [228,790,857,309,616,895,194,277,985]
+ CRUSH rule 1 x 61 [154,843,717,467,883,536,812,14,55]
+ CRUSH rule 1 x 62 [594,811,549,276,693,917,45,723,926]
+ CRUSH rule 1 x 63 [646,67,884,925,941,434,705,268,140]
+ CRUSH rule 1 x 64 [175,542,155,837,594,197,451,891,654]
+ CRUSH rule 1 x 65 [745,619,131,867,269,62,862,221,66]
+ CRUSH rule 1 x 66 [275,468,23,35,328,432,334,656,719]
+ CRUSH rule 1 x 67 [246,958,524,493,636,227,783,593,814]
+ CRUSH rule 1 x 68 [711,473,403,228,835,126,705,114,981]
+ CRUSH rule 1 x 69 [493,924,850,939,950,105,871,361,533]
+ CRUSH rule 1 x 70 [30,499,644,33,804,654,684,411,114]
+ CRUSH rule 1 x 71 [984,883,574,716,575,391,587,264,446]
+ CRUSH rule 1 x 72 [71,286,942,363,628,632,642,529,966]
+ CRUSH rule 1 x 73 [922,618,3,371,464,442,835,705,260]
+ CRUSH rule 1 x 74 [629,414,185,573,678,338,633,560,565]
+ CRUSH rule 1 x 75 [222,20,174,820,312,361,366,258,711]
+ CRUSH rule 1 x 76 [262,366,339,290,718,143,735,953,188]
+ CRUSH rule 1 x 77 [638,469,992,280,773,892,197,690,426]
+ CRUSH rule 1 x 78 [324,511,788,7,308,228,183,917,464]
+ CRUSH rule 1 x 79 [577,990,64,94,447,924,339,24,581]
+ CRUSH rule 1 x 80 [501,95,278,903,631,842,51,766,822]
+ CRUSH rule 1 x 81 [506,812,9,698,173,664,247,963,0]
+ CRUSH rule 1 x 82 [222,145,80,785,835,745,580,51,939]
+ CRUSH rule 1 x 83 [71,634,61,91,856,529,66,197,698]
+ CRUSH rule 1 x 84 [49,761,773,368,318,708,681,618,723]
+ CRUSH rule 1 x 85 [985,896,708,861,325,307,567,908,514]
+ CRUSH rule 1 x 86 [537,745,93,524,466,356,38,326,385]
+ CRUSH rule 1 x 87 [997,317,463,626,685,162,909,49,28]
+ CRUSH rule 1 x 88 [957,350,890,857,375,176,99,737,942]
+ CRUSH rule 1 x 89 [399,730,148,314,159,982,320,921,812]
+ CRUSH rule 1 x 90 [943,706,683,267,579,141,412,184,529]
+ CRUSH rule 1 x 91 [22,368,149,928,140,529,495,299,812]
+ CRUSH rule 1 x 92 [532,424,426,773,623,197,167,634,781]
+ CRUSH rule 1 x 93 [218,489,405,681,549,201,343,949,51]
+ CRUSH rule 1 x 94 [181,96,102,515,776,365,82,422,738]
+ CRUSH rule 1 x 95 [343,957,820,139,334,37,648,661,46]
+ CRUSH rule 1 x 96 [861,270,87,797,0,245,204,750,322]
+ CRUSH rule 1 x 97 [459,706,45,328,274,605,83,542,131]
+ CRUSH rule 1 x 98 [327,867,353,948,728,280,270,511,586]
+ CRUSH rule 1 x 99 [974,133,468,906,235,988,37,138,326]
+ CRUSH rule 1 x 100 [32,445,547,371,960,885,9,168,590]
+ CRUSH rule 1 x 101 [142,90,337,950,970,570,12,369,9]
+ CRUSH rule 1 x 102 [172,129,139,22,403,867,923,106,653]
+ CRUSH rule 1 x 103 [630,47,161,356,911,421,933,231,520]
+ CRUSH rule 1 x 104 [758,133,278,11,947,799,401,85,139]
+ CRUSH rule 1 x 105 [843,604,47,33,401,632,434,121,488]
+ CRUSH rule 1 x 106 [28,681,193,679,990,343,878,493,550]
+ CRUSH rule 1 x 107 [74,320,85,819,315,253,589,614,814]
+ CRUSH rule 1 x 108 [875,593,575,517,107,153,631,996,630]
+ CRUSH rule 1 x 109 [411,985,811,720,198,666,856,296,122]
+ CRUSH rule 1 x 110 [440,774,799,660,715,167,510,472,270]
+ CRUSH rule 1 x 111 [405,742,276,359,936,360,18,949,341]
+ CRUSH rule 1 x 112 [143,181,922,545,185,303,725,413,187]
+ CRUSH rule 1 x 113 [153,846,160,903,789,897,738,253,213]
+ CRUSH rule 1 x 114 [804,892,939,20,312,692,598,418,641]
+ CRUSH rule 1 x 115 [588,508,958,580,232,722,421,39,241]
+ CRUSH rule 1 x 116 [327,148,637,486,712,464,9,448,816]
+ CRUSH rule 1 x 117 [95,594,989,131,714,275,725,142,304]
+ CRUSH rule 1 x 118 [80,957,897,239,359,432,766,210,528]
+ CRUSH rule 1 x 119 [386,932,951,768,679,300,570,278,867]
+ CRUSH rule 1 x 120 [366,312,653,936,71,241,49,126,410]
+ CRUSH rule 1 x 121 [129,154,847,16,471,481,424,868,469]
+ CRUSH rule 1 x 122 [873,1,110,939,90,412,551,43,590]
+ CRUSH rule 1 x 123 [533,415,789,600,713,800,877,248,753]
+ CRUSH rule 1 x 124 [461,691,898,723,957,759,482,254,158]
+ CRUSH rule 1 x 125 [342,599,830,402,615,994,736,737,508]
+ CRUSH rule 1 x 126 [819,781,822,548,279,255,689,209,99]
+ CRUSH rule 1 x 127 [437,893,585,707,353,189,909,809,553]
+ CRUSH rule 1 x 128 [679,994,982,550,991,324,666,691,899]
+ CRUSH rule 1 x 129 [380,685,947,302,698,144,149,718,904]
+ CRUSH rule 1 x 130 [992,52,466,867,998,777,270,425,864]
+ CRUSH rule 1 x 131 [469,90,208,599,829,656,203,667,528]
+ CRUSH rule 1 x 132 [571,250,316,535,54,418,922,597,680]
+ CRUSH rule 1 x 133 [964,728,329,902,108,118,14,444,709]
+ CRUSH rule 1 x 134 [999,19,716,963,323,559,893,281,226]
+ CRUSH rule 1 x 135 [634,101,52,938,413,573,712,649,27]
+ CRUSH rule 1 x 136 [114,889,692,768,694,279,846,890,151]
+ CRUSH rule 1 x 137 [839,8,959,280,922,870,363,323,153]
+ CRUSH rule 1 x 138 [967,949,138,451,292,548,400,885,907]
+ CRUSH rule 1 x 139 [308,711,736,247,632,126,384,58,373]
+ CRUSH rule 1 x 140 [764,936,926,55,331,115,178,532,883]
+ CRUSH rule 1 x 141 [423,302,112,216,603,873,193,258,445]
+ CRUSH rule 1 x 142 [252,821,715,340,635,668,424,881,751]
+ CRUSH rule 1 x 143 [33,808,518,477,325,316,266,70,210]
+ CRUSH rule 1 x 144 [472,88,969,162,401,771,697,610,203]
+ CRUSH rule 1 x 145 [242,208,252,604,266,743,577,348,1]
+ CRUSH rule 1 x 146 [290,70,570,384,934,856,929,196,880]
+ CRUSH rule 1 x 147 [447,352,657,493,467,918,514,546,861]
+ CRUSH rule 1 x 148 [212,644,432,658,109,275,352,820,857]
+ CRUSH rule 1 x 149 [9,775,87,35,260,646,406,556,532]
+ CRUSH rule 1 x 150 [166,456,582,144,324,340,484,553,315]
+ CRUSH rule 1 x 151 [811,875,307,20,782,229,671,883,204]
+ CRUSH rule 1 x 152 [449,617,223,9,182,407,807,50,206]
+ CRUSH rule 1 x 153 [523,537,695,627,959,613,942,864,388]
+ CRUSH rule 1 x 154 [208,559,874,597,243,706,443,98,27]
+ CRUSH rule 1 x 155 [569,325,192,296,367,848,58,641,186]
+ CRUSH rule 1 x 156 [488,121,521,213,595,837,271,229,961]
+ CRUSH rule 1 x 157 [140,723,633,260,487,856,384,446,836]
+ CRUSH rule 1 x 158 [786,451,320,239,667,632,899,902,956]
+ CRUSH rule 1 x 159 [134,664,517,821,667,944,209,641,228]
+ CRUSH rule 1 x 160 [690,112,414,990,183,590,242,999,974]
+ CRUSH rule 1 x 161 [324,912,397,423,991,284,909,642,188]
+ CRUSH rule 1 x 162 [748,567,284,183,463,336,148,88,764]
+ CRUSH rule 1 x 163 [575,499,31,816,749,737,587,854,482]
+ CRUSH rule 1 x 164 [314,489,308,326,51,568,110,329,361]
+ CRUSH rule 1 x 165 [116,209,750,53,813,640,524,389,185]
+ CRUSH rule 1 x 166 [352,706,701,810,718,527,548,676,448]
+ CRUSH rule 1 x 167 [27,743,174,142,551,1,935,266,883]
+ CRUSH rule 1 x 168 [953,898,880,660,500,799,667,463,818]
+ CRUSH rule 1 x 169 [912,147,266,547,331,770,601,909,60]
+ CRUSH rule 1 x 170 [421,515,828,844,151,981,835,840,548]
+ CRUSH rule 1 x 171 [488,584,880,964,936,196,100,910,446]
+ CRUSH rule 1 x 172 [366,443,957,66,162,693,36,356,274]
+ CRUSH rule 1 x 173 [863,291,625,287,158,496,471,529,359]
+ CRUSH rule 1 x 174 [263,555,650,410,339,616,780,932,573]
+ CRUSH rule 1 x 175 [875,961,361,575,33,109,51,211,409]
+ CRUSH rule 1 x 176 [745,83,701,680,250,420,240,316,337]
+ CRUSH rule 1 x 177 [128,244,41,123,422,902,756,647,45]
+ CRUSH rule 1 x 178 [155,41,264,777,314,564,856,992,696]
+ CRUSH rule 1 x 179 [593,833,202,183,971,38,724,923,450]
+ CRUSH rule 1 x 180 [154,734,17,831,824,522,736,846,926]
+ CRUSH rule 1 x 181 [289,675,723,800,166,712,168,224,705]
+ CRUSH rule 1 x 182 [730,931,560,209,943,261,485,571,796]
+ CRUSH rule 1 x 183 [639,237,794,815,827,400,109,903,96]
+ CRUSH rule 1 x 184 [704,312,685,645,691,778,74,45,438]
+ CRUSH rule 1 x 185 [97,100,762,82,999,542,485,511,14]
+ CRUSH rule 1 x 186 [26,665,554,215,280,421,369,270,16]
+ CRUSH rule 1 x 187 [649,14,740,494,402,684,566,378,816]
+ CRUSH rule 1 x 188 [682,695,590,743,927,945,833,650,761]
+ CRUSH rule 1 x 189 [325,693,726,51,448,169,37,1,939]
+ CRUSH rule 1 x 190 [399,933,136,955,57,504,527,237,295]
+ CRUSH rule 1 x 191 [629,533,17,126,60,146,999,754,339]
+ CRUSH rule 1 x 192 [503,578,38,492,222,251,123,759,147]
+ CRUSH rule 1 x 193 [546,333,651,678,823,652,359,721,996]
+ CRUSH rule 1 x 194 [242,473,58,655,851,277,792,887,561]
+ CRUSH rule 1 x 195 [625,719,135,81,636,513,755,471,629]
+ CRUSH rule 1 x 196 [357,114,125,867,250,522,413,834,832]
+ CRUSH rule 1 x 197 [306,954,453,873,211,334,666,316,243]
+ CRUSH rule 1 x 198 [863,791,311,911,206,61,355,574,781]
+ CRUSH rule 1 x 199 [935,906,929,252,893,75,960,369,584]
+ CRUSH rule 1 x 200 [373,774,229,454,909,611,132,271,128]
+ CRUSH rule 1 x 201 [659,320,477,313,779,16,495,76,598]
+ CRUSH rule 1 x 202 [260,433,524,880,223,818,153,272,944]
+ CRUSH rule 1 x 203 [36,239,675,971,703,209,669,676,762]
+ CRUSH rule 1 x 204 [92,516,993,728,279,478,697,881,64]
+ CRUSH rule 1 x 205 [68,395,473,45,683,662,776,463,327]
+ CRUSH rule 1 x 206 [570,530,642,380,311,398,230,367,890]
+ CRUSH rule 1 x 207 [834,457,850,917,456,296,76,708,101]
+ CRUSH rule 1 x 208 [927,484,640,976,803,626,96,841,811]
+ CRUSH rule 1 x 209 [878,66,58,940,48,233,522,185,949]
+ CRUSH rule 1 x 210 [572,981,484,29,0,426,14,921,544]
+ CRUSH rule 1 x 211 [107,597,780,857,895,57,922,372,581]
+ CRUSH rule 1 x 212 [389,107,838,624,698,562,857,894,60]
+ CRUSH rule 1 x 213 [497,717,567,728,905,134,687,903,620]
+ CRUSH rule 1 x 214 [798,65,254,572,32,393,579,79,258]
+ CRUSH rule 1 x 215 [233,419,283,638,520,891,982,826,488]
+ CRUSH rule 1 x 216 [494,464,742,523,459,174,973,898,556]
+ CRUSH rule 1 x 217 [352,396,309,938,66,41,264,6,603]
+ CRUSH rule 1 x 218 [895,864,988,650,593,740,34,497,108]
+ CRUSH rule 1 x 219 [222,534,277,242,658,482,697,805,976]
+ CRUSH rule 1 x 220 [281,19,584,563,858,965,686,982,0]
+ CRUSH rule 1 x 221 [64,928,963,130,312,394,61,559,846]
+ CRUSH rule 1 x 222 [40,544,161,199,861,644,597,904,897]
+ CRUSH rule 1 x 223 [645,556,159,417,46,135,465,429,614]
+ CRUSH rule 1 x 224 [647,165,957,263,961,576,329,320,645]
+ CRUSH rule 1 x 225 [219,714,858,747,461,175,606,465,354]
+ CRUSH rule 1 x 226 [372,511,181,277,695,404,876,984,491]
+ CRUSH rule 1 x 227 [925,156,714,863,257,74,966,217,501]
+ CRUSH rule 1 x 228 [682,404,839,263,521,195,261,389,281]
+ CRUSH rule 1 x 229 [880,838,770,891,236,542,262,884,215]
+ CRUSH rule 1 x 230 [328,659,916,468,646,572,93,880,959]
+ CRUSH rule 1 x 231 [320,383,669,109,627,621,50,182,541]
+ CRUSH rule 1 x 232 [924,846,394,319,43,519,106,877,130]
+ CRUSH rule 1 x 233 [948,652,575,838,498,395,796,835,714]
+ CRUSH rule 1 x 234 [484,943,42,575,936,180,103,95,634]
+ CRUSH rule 1 x 235 [750,65,590,168,870,308,471,753,350]
+ CRUSH rule 1 x 236 [551,787,490,136,370,833,573,128,154]
+ CRUSH rule 1 x 237 [390,157,166,251,752,75,327,509,325]
+ CRUSH rule 1 x 238 [570,6,989,707,514,905,894,884,824]
+ CRUSH rule 1 x 239 [729,959,376,975,496,49,426,427,736]
+ CRUSH rule 1 x 240 [981,241,156,767,631,576,450,677,659]
+ CRUSH rule 1 x 241 [310,816,641,177,996,454,413,136,411]
+ CRUSH rule 1 x 242 [161,63,642,837,763,458,234,756,496]
+ CRUSH rule 1 x 243 [180,394,33,683,189,419,799,21,13]
+ CRUSH rule 1 x 244 [52,174,685,189,78,310,785,107,816]
+ CRUSH rule 1 x 245 [523,121,915,84,386,409,605,837,1]
+ CRUSH rule 1 x 246 [362,893,390,487,817,88,989,999,138]
+ CRUSH rule 1 x 247 [382,184,116,34,143,15,590,840,586]
+ CRUSH rule 1 x 248 [129,114,852,469,359,291,713,237,468]
+ CRUSH rule 1 x 249 [159,683,91,856,475,369,886,650,827]
+ CRUSH rule 1 x 250 [404,945,569,955,228,910,270,619,450]
+ CRUSH rule 1 x 251 [661,225,738,757,37,642,58,354,16]
+ CRUSH rule 1 x 252 [961,226,542,103,945,885,838,131,387]
+ CRUSH rule 1 x 253 [651,97,225,364,189,248,797,675,452]
+ CRUSH rule 1 x 254 [123,33,741,692,599,11,605,453,987]
+ CRUSH rule 1 x 255 [314,649,891,855,517,344,607,95,121]
+ CRUSH rule 1 x 256 [315,215,651,126,470,849,189,627,592]
+ CRUSH rule 1 x 257 [825,264,867,17,529,409,291,732,224]
+ CRUSH rule 1 x 258 [624,789,370,723,131,982,863,427,873]
+ CRUSH rule 1 x 259 [602,542,70,563,947,723,77,191,669]
+ CRUSH rule 1 x 260 [717,878,43,56,377,481,533,646,475]
+ CRUSH rule 1 x 261 [145,517,20,903,786,939,516,136,87]
+ CRUSH rule 1 x 262 [223,1,561,420,256,16,88,534,289]
+ CRUSH rule 1 x 263 [462,211,405,508,787,669,773,979,719]
+ CRUSH rule 1 x 264 [654,471,266,662,135,564,715,916,633]
+ CRUSH rule 1 x 265 [302,794,704,798,659,487,833,987,445]
+ CRUSH rule 1 x 266 [202,132,884,209,551,984,7,557,76]
+ CRUSH rule 1 x 267 [282,938,657,113,672,993,972,645,882]
+ CRUSH rule 1 x 268 [338,309,356,278,928,797,715,536,983]
+ CRUSH rule 1 x 269 [738,122,266,200,894,118,146,14,414]
+ CRUSH rule 1 x 270 [707,982,946,196,407,804,476,571,314]
+ CRUSH rule 1 x 271 [705,432,364,735,512,595,263,138,526]
+ CRUSH rule 1 x 272 [756,545,942,56,542,449,710,779,161]
+ CRUSH rule 1 x 273 [197,502,527,721,239,648,982,735,58]
+ CRUSH rule 1 x 274 [992,44,653,573,527,702,370,990,320]
+ CRUSH rule 1 x 275 [544,789,170,434,23,926,992,823,321]
+ CRUSH rule 1 x 276 [658,467,577,268,336,5,634,98,457]
+ CRUSH rule 1 x 277 [143,490,880,483,928,272,783,648,927]
+ CRUSH rule 1 x 278 [492,647,355,282,834,64,350,600,283]
+ CRUSH rule 1 x 279 [517,792,604,987,527,894,952,250,206]
+ CRUSH rule 1 x 280 [825,740,27,848,514,750,895,914,892]
+ CRUSH rule 1 x 281 [224,629,120,562,616,200,443,604,859]
+ CRUSH rule 1 x 282 [298,661,380,416,35,585,939,879,338]
+ CRUSH rule 1 x 283 [311,606,208,50,913,678,369,544,721]
+ CRUSH rule 1 x 284 [771,466,371,743,672,119,60,546,39]
+ CRUSH rule 1 x 285 [693,362,404,676,797,531,582,975,810]
+ CRUSH rule 1 x 286 [364,477,285,167,270,617,699,627,725]
+ CRUSH rule 1 x 287 [591,611,828,995,170,987,137,890,487]
+ CRUSH rule 1 x 288 [965,541,848,796,251,668,195,538,356]
+ CRUSH rule 1 x 289 [225,551,948,877,219,167,795,377,825]
+ CRUSH rule 1 x 290 [577,762,777,751,291,349,473,209,59]
+ CRUSH rule 1 x 291 [160,903,477,381,490,559,557,86,89]
+ CRUSH rule 1 x 292 [873,598,216,666,222,228,806,911,738]
+ CRUSH rule 1 x 293 [100,234,874,47,28,452,775,636,232]
+ CRUSH rule 1 x 294 [285,943,379,520,725,547,459,833,503]
+ CRUSH rule 1 x 295 [938,262,880,327,687,3,440,73,29]
+ CRUSH rule 1 x 296 [850,327,86,472,1,776,266,82,671]
+ CRUSH rule 1 x 297 [951,53,99,558,753,228,232,343,831]
+ CRUSH rule 1 x 298 [173,336,85,766,910,657,213,286,61]
+ CRUSH rule 1 x 299 [598,591,315,386,895,296,924,106,63]
+ CRUSH rule 1 x 300 [531,957,62,459,156,538,904,838,458]
+ CRUSH rule 1 x 301 [823,628,23,858,629,808,220,432,393]
+ CRUSH rule 1 x 302 [184,80,780,871,531,211,400,365,697]
+ CRUSH rule 1 x 303 [521,766,222,830,988,275,561,905,522]
+ CRUSH rule 1 x 304 [980,127,807,507,555,245,214,944,845]
+ CRUSH rule 1 x 305 [153,816,22,927,696,911,685,838,3]
+ CRUSH rule 1 x 306 [423,739,664,753,178,431,761,648,867]
+ CRUSH rule 1 x 307 [997,557,682,456,479,631,459,250,415]
+ CRUSH rule 1 x 308 [991,874,534,465,330,284,976,551,126]
+ CRUSH rule 1 x 309 [860,394,724,858,246,866,857,153,970]
+ CRUSH rule 1 x 310 [589,818,546,201,94,653,90,855,441]
+ CRUSH rule 1 x 311 [477,774,225,590,830,559,256,798,743]
+ CRUSH rule 1 x 312 [887,853,950,354,58,23,497,929,92]
+ CRUSH rule 1 x 313 [802,646,447,416,557,118,24,81,215]
+ CRUSH rule 1 x 314 [654,974,229,511,562,916,952,599,398]
+ CRUSH rule 1 x 315 [767,227,28,740,828,156,749,841,969]
+ CRUSH rule 1 x 316 [778,83,733,359,858,319,761,725,923]
+ CRUSH rule 1 x 317 [184,418,642,986,939,675,892,86,214]
+ CRUSH rule 1 x 318 [525,410,500,543,212,95,290,97,529]
+ CRUSH rule 1 x 319 [476,724,569,382,409,521,800,868,364]
+ CRUSH rule 1 x 320 [149,610,697,296,818,955,523,366,891]
+ CRUSH rule 1 x 321 [710,79,667,671,234,4,868,841,563]
+ CRUSH rule 1 x 322 [175,275,323,333,744,718,187,380,947]
+ CRUSH rule 1 x 323 [819,604,638,792,316,544,236,597,969]
+ CRUSH rule 1 x 324 [16,745,511,439,272,417,668,959,845]
+ CRUSH rule 1 x 325 [486,400,872,873,251,68,462,268,124]
+ CRUSH rule 1 x 326 [613,765,207,19,359,370,461,509,75]
+ CRUSH rule 1 x 327 [125,289,738,408,456,784,750,669,296]
+ CRUSH rule 1 x 328 [807,383,476,583,645,141,33,806,181]
+ CRUSH rule 1 x 329 [588,938,599,432,446,840,516,713,223]
+ CRUSH rule 1 x 330 [932,644,41,611,209,406,420,520,395]
+ CRUSH rule 1 x 331 [341,953,950,537,578,862,624,649,626]
+ CRUSH rule 1 x 332 [153,726,459,950,466,804,644,821,238]
+ CRUSH rule 1 x 333 [745,845,853,860,52,615,243,633,309]
+ CRUSH rule 1 x 334 [614,751,807,58,396,159,408,175,189]
+ CRUSH rule 1 x 335 [518,721,221,283,454,187,635,367,997]
+ CRUSH rule 1 x 336 [389,424,77,309,5,898,698,533,683]
+ CRUSH rule 1 x 337 [753,508,765,720,221,807,956,907,464]
+ CRUSH rule 1 x 338 [128,810,490,753,406,760,69,11,624]
+ CRUSH rule 1 x 339 [430,308,58,751,856,823,607,953,125]
+ CRUSH rule 1 x 340 [541,44,630,231,289,966,707,328,325]
+ CRUSH rule 1 x 341 [402,26,631,439,165,928,720,503,209]
+ CRUSH rule 1 x 342 [982,57,992,461,131,32,516,661,985]
+ CRUSH rule 1 x 343 [833,412,572,732,107,805,660,655,149]
+ CRUSH rule 1 x 344 [784,533,792,41,642,869,142,114,108]
+ CRUSH rule 1 x 345 [546,300,304,691,763,556,127,732,290]
+ CRUSH rule 1 x 346 [302,420,428,891,357,124,419,962,304]
+ CRUSH rule 1 x 347 [488,778,101,217,366,442,783,661,622]
+ CRUSH rule 1 x 348 [903,744,937,718,85,314,862,513,112]
+ CRUSH rule 1 x 349 [471,547,582,306,600,486,795,143,529]
+ CRUSH rule 1 x 350 [348,221,823,335,383,708,841,164,765]
+ CRUSH rule 1 x 351 [961,582,705,346,361,32,766,775,518]
+ CRUSH rule 1 x 352 [728,137,461,298,36,903,899,665,802]
+ CRUSH rule 1 x 353 [904,202,184,447,58,294,279,616,892]
+ CRUSH rule 1 x 354 [345,226,319,256,544,311,612,33,122]
+ CRUSH rule 1 x 355 [50,430,175,43,187,458,985,412,599]
+ CRUSH rule 1 x 356 [87,185,55,423,829,1,629,228,150]
+ CRUSH rule 1 x 357 [762,459,921,473,182,231,891,656,196]
+ CRUSH rule 1 x 358 [908,25,280,6,808,676,874,643,550]
+ CRUSH rule 1 x 359 [484,15,132,121,394,423,397,52,702]
+ CRUSH rule 1 x 360 [173,378,337,702,145,499,29,529,156]
+ CRUSH rule 1 x 361 [404,577,115,25,56,914,643,286,552]
+ CRUSH rule 1 x 362 [403,1,422,945,132,685,265,35,662]
+ CRUSH rule 1 x 363 [639,911,510,162,418,294,444,613,813]
+ CRUSH rule 1 x 364 [752,689,610,990,665,222,203,17,743]
+ CRUSH rule 1 x 365 [956,999,212,230,624,84,113,373,426]
+ CRUSH rule 1 x 366 [860,925,924,763,687,851,59,914,521]
+ CRUSH rule 1 x 367 [205,609,647,665,969,720,685,641,894]
+ CRUSH rule 1 x 368 [301,284,810,169,78,340,616,93,283]
+ CRUSH rule 1 x 369 [452,658,339,217,674,210,284,184,718]
+ CRUSH rule 1 x 370 [11,467,695,989,394,576,850,419,307]
+ CRUSH rule 1 x 371 [124,487,55,514,313,411,797,547,778]
+ CRUSH rule 1 x 372 [253,48,979,846,207,631,212,241,346]
+ CRUSH rule 1 x 373 [715,605,775,748,227,493,128,207,88]
+ CRUSH rule 1 x 374 [191,887,920,291,223,714,961,760,571]
+ CRUSH rule 1 x 375 [711,385,651,665,15,71,934,619,527]
+ CRUSH rule 1 x 376 [597,818,49,458,415,755,446,897,460]
+ CRUSH rule 1 x 377 [294,256,933,771,184,861,654,487,891]
+ CRUSH rule 1 x 378 [34,151,681,707,552,127,728,860,968]
+ CRUSH rule 1 x 379 [869,136,315,378,813,153,115,557,165]
+ CRUSH rule 1 x 380 [294,97,575,791,690,482,255,806,429]
+ CRUSH rule 1 x 381 [119,710,219,827,328,886,773,496,433]
+ CRUSH rule 1 x 382 [69,631,508,706,697,168,276,56,278]
+ CRUSH rule 1 x 383 [922,588,589,925,471,601,29,197,822]
+ CRUSH rule 1 x 384 [221,945,671,117,857,655,488,435,223]
+ CRUSH rule 1 x 385 [561,737,953,723,658,368,910,329,396]
+ CRUSH rule 1 x 386 [335,442,788,696,507,716,232,692,742]
+ CRUSH rule 1 x 387 [514,43,353,88,100,842,164,934,297]
+ CRUSH rule 1 x 388 [587,89,157,996,915,927,474,267,640]
+ CRUSH rule 1 x 389 [109,641,255,466,372,563,340,222,74]
+ CRUSH rule 1 x 390 [925,149,421,489,599,810,852,196,469]
+ CRUSH rule 1 x 391 [267,87,387,527,768,873,41,136,818]
+ CRUSH rule 1 x 392 [382,485,370,849,936,636,901,82,695]
+ CRUSH rule 1 x 393 [425,721,221,753,268,463,652,543,10]
+ CRUSH rule 1 x 394 [898,18,38,793,173,738,15,591,420]
+ CRUSH rule 1 x 395 [806,876,269,679,32,744,126,179,607]
+ CRUSH rule 1 x 396 [790,970,437,449,875,395,726,935,278]
+ CRUSH rule 1 x 397 [136,363,507,613,11,30,996,558,602]
+ CRUSH rule 1 x 398 [914,116,558,258,722,904,349,672,826]
+ CRUSH rule 1 x 399 [261,94,299,202,174,622,749,410,815]
+ CRUSH rule 1 x 400 [661,197,338,461,977,848,536,592,886]
+ CRUSH rule 1 x 401 [953,979,287,803,41,349,79,32,343]
+ CRUSH rule 1 x 402 [738,819,618,522,667,334,658,449,886]
+ CRUSH rule 1 x 403 [573,238,425,546,130,68,202,650,501]
+ CRUSH rule 1 x 404 [526,848,790,253,922,820,299,577,563]
+ CRUSH rule 1 x 405 [582,505,330,334,201,110,776,296,19]
+ CRUSH rule 1 x 406 [768,324,493,60,186,165,718,578,580]
+ CRUSH rule 1 x 407 [260,951,437,587,692,648,72,345,709]
+ CRUSH rule 1 x 408 [657,81,770,734,830,821,246,695,76]
+ CRUSH rule 1 x 409 [498,89,182,423,672,152,213,806,168]
+ CRUSH rule 1 x 410 [28,793,737,352,166,645,949,507,361]
+ CRUSH rule 1 x 411 [684,992,60,659,769,267,313,351,497]
+ CRUSH rule 1 x 412 [261,958,699,950,165,14,560,155,661]
+ CRUSH rule 1 x 413 [891,835,297,441,384,979,618,907,9]
+ CRUSH rule 1 x 414 [127,459,119,965,662,594,97,124,229]
+ CRUSH rule 1 x 415 [272,540,631,328,609,568,694,332,572]
+ CRUSH rule 1 x 416 [739,617,115,530,339,371,889,344,838]
+ CRUSH rule 1 x 417 [106,209,157,878,117,128,138,374,470]
+ CRUSH rule 1 x 418 [525,441,147,390,320,300,848,972,781]
+ CRUSH rule 1 x 419 [603,673,615,465,266,855,823,884,832]
+ CRUSH rule 1 x 420 [988,213,251,226,209,245,506,670,285]
+ CRUSH rule 1 x 421 [761,521,748,368,923,992,764,274,623]
+ CRUSH rule 1 x 422 [317,160,924,548,198,709,839,547,599]
+ CRUSH rule 1 x 423 [137,807,168,472,619,443,905,588,312]
+ CRUSH rule 1 x 424 [920,37,146,263,598,748,785,395,884]
+ CRUSH rule 1 x 425 [277,693,285,221,478,165,80,236,988]
+ CRUSH rule 1 x 426 [485,936,407,854,726,524,791,565,352]
+ CRUSH rule 1 x 427 [242,515,9,564,174,453,334,588,571]
+ CRUSH rule 1 x 428 [632,635,26,473,494,478,225,94,303]
+ CRUSH rule 1 x 429 [641,73,465,127,171,397,857,562,976]
+ CRUSH rule 1 x 430 [626,585,6,387,881,583,859,699,91]
+ CRUSH rule 1 x 431 [697,76,753,570,964,339,194,366,279]
+ CRUSH rule 1 x 432 [590,526,306,283,656,728,513,591,599]
+ CRUSH rule 1 x 433 [284,387,149,817,886,714,52,897,705]
+ CRUSH rule 1 x 434 [538,985,79,953,770,468,644,646,747]
+ CRUSH rule 1 x 435 [30,318,593,635,975,833,371,731,906]
+ CRUSH rule 1 x 436 [164,919,851,693,0,874,10,976,284]
+ CRUSH rule 1 x 437 [322,212,163,606,302,282,443,23,696]
+ CRUSH rule 1 x 438 [142,392,85,594,376,419,755,841,94]
+ CRUSH rule 1 x 439 [119,370,68,443,997,837,414,152,331]
+ CRUSH rule 1 x 440 [333,403,187,863,475,844,800,174,117]
+ CRUSH rule 1 x 441 [477,727,906,145,429,91,205,236,86]
+ CRUSH rule 1 x 442 [274,590,933,244,434,49,864,799,762]
+ CRUSH rule 1 x 443 [983,748,574,718,700,442,774,350,37]
+ CRUSH rule 1 x 444 [536,509,431,146,170,149,182,145,347]
+ CRUSH rule 1 x 445 [485,46,528,209,964,753,554,931,638]
+ CRUSH rule 1 x 446 [345,634,42,294,711,376,314,714,212]
+ CRUSH rule 1 x 447 [61,845,767,600,321,716,58,531,827]
+ CRUSH rule 1 x 448 [333,232,292,846,364,951,807,688,21]
+ CRUSH rule 1 x 449 [680,16,484,670,851,500,258,548,905]
+ CRUSH rule 1 x 450 [235,214,79,423,96,822,721,31,312]
+ CRUSH rule 1 x 451 [961,468,333,640,823,151,878,33,3]
+ CRUSH rule 1 x 452 [525,479,153,528,570,806,604,49,922]
+ CRUSH rule 1 x 453 [138,466,302,86,249,154,514,5,494]
+ CRUSH rule 1 x 454 [137,625,215,402,389,914,106,103,511]
+ CRUSH rule 1 x 455 [173,150,997,16,846,888,295,967,132]
+ CRUSH rule 1 x 456 [235,226,238,258,347,784,504,96,890]
+ CRUSH rule 1 x 457 [450,577,253,413,717,609,762,975,485]
+ CRUSH rule 1 x 458 [195,537,91,814,351,90,399,558,15]
+ CRUSH rule 1 x 459 [381,555,312,573,915,623,147,483,517]
+ CRUSH rule 1 x 460 [972,730,534,678,756,692,841,512,70]
+ CRUSH rule 1 x 461 [506,279,142,830,784,124,385,797,917]
+ CRUSH rule 1 x 462 [692,959,578,57,983,299,240,911,375]
+ CRUSH rule 1 x 463 [788,667,949,550,685,702,538,111,232]
+ CRUSH rule 1 x 464 [133,122,588,999,270,880,789,0,653]
+ CRUSH rule 1 x 465 [971,190,230,777,452,914,137,466,531]
+ CRUSH rule 1 x 466 [394,576,148,157,103,822,659,35,797]
+ CRUSH rule 1 x 467 [517,28,366,362,984,521,187,640,601]
+ CRUSH rule 1 x 468 [829,143,874,225,162,413,201,249,555]
+ CRUSH rule 1 x 469 [987,936,106,725,633,238,681,463,551]
+ CRUSH rule 1 x 470 [107,982,56,889,67,65,558,71,676]
+ CRUSH rule 1 x 471 [181,897,629,860,307,116,256,978,409]
+ CRUSH rule 1 x 472 [547,512,172,24,705,837,809,56,476]
+ CRUSH rule 1 x 473 [760,997,824,905,888,755,756,663,167]
+ CRUSH rule 1 x 474 [787,418,743,628,272,341,446,333,245]
+ CRUSH rule 1 x 475 [662,312,253,617,105,58,237,764,682]
+ CRUSH rule 1 x 476 [110,495,185,508,961,837,984,226,333]
+ CRUSH rule 1 x 477 [393,954,834,132,841,367,753,794,237]
+ CRUSH rule 1 x 478 [246,483,480,644,985,420,941,843,751]
+ CRUSH rule 1 x 479 [70,929,697,931,744,487,158,489,515]
+ CRUSH rule 1 x 480 [753,119,961,607,317,717,371,807,687]
+ CRUSH rule 1 x 481 [470,429,677,242,574,757,135,375,613]
+ CRUSH rule 1 x 482 [451,566,961,675,354,746,731,233,640]
+ CRUSH rule 1 x 483 [816,72,371,278,635,30,448,437,219]
+ CRUSH rule 1 x 484 [540,454,389,31,654,494,283,170,278]
+ CRUSH rule 1 x 485 [74,582,624,684,566,677,866,661,581]
+ CRUSH rule 1 x 486 [958,595,199,763,715,973,621,955,400]
+ CRUSH rule 1 x 487 [228,302,804,833,876,647,857,782,24]
+ CRUSH rule 1 x 488 [180,529,722,956,353,890,924,965,25]
+ CRUSH rule 1 x 489 [47,617,812,187,291,828,154,478,512]
+ CRUSH rule 1 x 490 [905,822,479,124,750,843,566,779,963]
+ CRUSH rule 1 x 491 [892,370,609,998,433,957,188,563,490]
+ CRUSH rule 1 x 492 [588,959,127,948,505,936,591,423,668]
+ CRUSH rule 1 x 493 [353,461,593,291,301,830,231,442,474]
+ CRUSH rule 1 x 494 [378,848,443,368,507,423,389,819,956]
+ CRUSH rule 1 x 495 [845,653,768,234,405,367,823,789,217]
+ CRUSH rule 1 x 496 [13,988,0,691,389,757,129,763,39]
+ CRUSH rule 1 x 497 [796,877,788,394,648,829,542,745,131]
+ CRUSH rule 1 x 498 [412,337,270,705,511,227,949,173,398]
+ CRUSH rule 1 x 499 [330,695,8,74,618,101,440,509,295]
+ CRUSH rule 1 x 500 [820,272,547,765,755,96,930,573,357]
+ CRUSH rule 1 x 501 [110,44,132,442,294,423,880,279,616]
+ CRUSH rule 1 x 502 [336,595,650,274,993,312,490,852,962]
+ CRUSH rule 1 x 503 [922,211,157,722,502,971,262,926,316]
+ CRUSH rule 1 x 504 [483,52,122,432,778,461,758,104,831]
+ CRUSH rule 1 x 505 [482,598,224,279,480,310,764,558,891]
+ CRUSH rule 1 x 506 [493,123,43,856,936,622,898,161,78]
+ CRUSH rule 1 x 507 [12,598,264,422,416,947,591,702,346]
+ CRUSH rule 1 x 508 [227,157,611,301,223,746,313,282,207]
+ CRUSH rule 1 x 509 [807,242,363,122,582,530,798,808,139]
+ CRUSH rule 1 x 510 [134,437,227,75,313,351,786,152,921]
+ CRUSH rule 1 x 511 [212,54,83,799,457,218,600,968,355]
+ CRUSH rule 1 x 512 [236,630,758,752,361,249,899,451,415]
+ CRUSH rule 1 x 513 [994,693,644,938,846,685,52,185,197]
+ CRUSH rule 1 x 514 [45,508,831,19,817,52,374,985,944]
+ CRUSH rule 1 x 515 [504,138,480,272,530,377,481,820,517]
+ CRUSH rule 1 x 516 [285,409,136,570,841,610,453,660,93]
+ CRUSH rule 1 x 517 [300,232,23,906,438,236,519,737,20]
+ CRUSH rule 1 x 518 [397,674,98,898,967,113,625,434,527]
+ CRUSH rule 1 x 519 [86,750,772,913,101,864,375,328,3]
+ CRUSH rule 1 x 520 [900,833,614,130,261,885,558,956,664]
+ CRUSH rule 1 x 521 [31,47,236,751,911,599,495,354,665]
+ CRUSH rule 1 x 522 [390,16,280,144,291,175,753,624,769]
+ CRUSH rule 1 x 523 [618,308,424,590,300,206,834,212,906]
+ CRUSH rule 1 x 524 [635,189,687,963,601,518,8,550,769]
+ CRUSH rule 1 x 525 [311,916,699,262,775,32,45,478,911]
+ CRUSH rule 1 x 526 [48,738,227,718,244,942,853,643,625]
+ CRUSH rule 1 x 527 [202,851,889,216,763,351,270,35,809]
+ CRUSH rule 1 x 528 [565,827,590,273,918,106,651,368,118]
+ CRUSH rule 1 x 529 [934,864,241,43,466,924,278,926,280]
+ CRUSH rule 1 x 530 [502,934,298,670,986,360,577,509,195]
+ CRUSH rule 1 x 531 [681,627,942,487,288,561,925,474,669]
+ CRUSH rule 1 x 532 [422,6,147,205,861,141,949,374,988]
+ CRUSH rule 1 x 533 [863,68,364,983,247,199,54,931,4]
+ CRUSH rule 1 x 534 [962,931,775,172,663,119,206,682,627]
+ CRUSH rule 1 x 535 [89,565,397,693,839,632,859,30,61]
+ CRUSH rule 1 x 536 [499,351,760,458,918,86,148,668,436]
+ CRUSH rule 1 x 537 [676,547,787,311,867,748,152,797,492]
+ CRUSH rule 1 x 538 [58,644,571,649,941,7,37,485,88]
+ CRUSH rule 1 x 539 [837,953,457,711,458,621,528,722,59]
+ CRUSH rule 1 x 540 [831,50,132,213,197,709,95,789,348]
+ CRUSH rule 1 x 541 [582,757,121,525,532,963,738,277,225]
+ CRUSH rule 1 x 542 [472,132,790,997,948,269,137,934,547]
+ CRUSH rule 1 x 543 [382,272,797,330,315,748,324,134,839]
+ CRUSH rule 1 x 544 [947,930,496,883,509,219,250,362,614]
+ CRUSH rule 1 x 545 [425,570,305,77,821,422,117,172,764]
+ CRUSH rule 1 x 546 [18,65,529,437,343,547,699,610,785]
+ CRUSH rule 1 x 547 [445,715,600,472,213,851,428,267,229]
+ CRUSH rule 1 x 548 [367,569,980,167,627,442,517,684,154]
+ CRUSH rule 1 x 549 [125,715,671,817,285,420,37,639,934]
+ CRUSH rule 1 x 550 [425,599,744,199,923,222,915,570,546]
+ CRUSH rule 1 x 551 [44,1,528,922,944,115,161,901,342]
+ CRUSH rule 1 x 552 [246,104,68,239,123,427,57,217,21]
+ CRUSH rule 1 x 553 [71,703,615,28,593,724,218,916,561]
+ CRUSH rule 1 x 554 [207,124,217,166,525,226,693,953,606]
+ CRUSH rule 1 x 555 [570,28,317,420,931,413,623,659,403]
+ CRUSH rule 1 x 556 [674,152,421,79,215,347,830,762,691]
+ CRUSH rule 1 x 557 [347,817,191,391,741,571,593,267,17]
+ CRUSH rule 1 x 558 [627,426,369,692,815,371,124,107,766]
+ CRUSH rule 1 x 559 [940,630,924,242,224,912,185,356,87]
+ CRUSH rule 1 x 560 [295,903,541,29,245,753,887,376,658]
+ CRUSH rule 1 x 561 [506,682,384,637,878,991,700,339,687]
+ CRUSH rule 1 x 562 [718,529,87,729,842,341,62,817,766]
+ CRUSH rule 1 x 563 [552,332,747,206,274,871,903,900,812]
+ CRUSH rule 1 x 564 [835,769,736,486,630,209,641,751,930]
+ CRUSH rule 1 x 565 [8,167,539,182,607,62,738,873,47]
+ CRUSH rule 1 x 566 [600,481,301,263,90,450,184,127,448]
+ CRUSH rule 1 x 567 [999,994,509,899,947,24,267,639,646]
+ CRUSH rule 1 x 568 [252,431,157,62,601,863,398,521,59]
+ CRUSH rule 1 x 569 [643,218,943,455,83,969,494,624,352]
+ CRUSH rule 1 x 570 [617,635,765,422,250,156,533,674,23]
+ CRUSH rule 1 x 571 [757,80,59,98,328,700,329,848,235]
+ CRUSH rule 1 x 572 [299,348,575,889,943,675,33,312,202]
+ CRUSH rule 1 x 573 [25,505,270,167,58,901,878,978,1]
+ CRUSH rule 1 x 574 [215,431,624,177,628,814,333,841,193]
+ CRUSH rule 1 x 575 [225,252,611,546,32,815,389,486,10]
+ CRUSH rule 1 x 576 [627,94,159,857,430,691,177,545,839]
+ CRUSH rule 1 x 577 [237,809,778,636,61,167,700,521,825]
+ CRUSH rule 1 x 578 [885,313,120,344,771,614,487,976,977]
+ CRUSH rule 1 x 579 [924,575,787,831,47,996,557,630,468]
+ CRUSH rule 1 x 580 [718,51,766,121,118,471,608,755,326]
+ CRUSH rule 1 x 581 [219,807,129,571,856,179,874,902,958]
+ CRUSH rule 1 x 582 [893,701,598,863,285,829,984,622,175]
+ CRUSH rule 1 x 583 [246,930,964,170,993,409,469,193,737]
+ CRUSH rule 1 x 584 [336,432,680,175,495,839,642,226,122]
+ CRUSH rule 1 x 585 [324,999,397,485,457,527,73,628,884]
+ CRUSH rule 1 x 586 [558,230,976,541,816,72,794,682,127]
+ CRUSH rule 1 x 587 [985,830,597,21,308,890,952,421,875]
+ CRUSH rule 1 x 588 [211,544,57,134,162,496,195,581,649]
+ CRUSH rule 1 x 589 [129,21,112,190,885,844,753,180,160]
+ CRUSH rule 1 x 590 [467,969,652,593,287,76,811,413,436]
+ CRUSH rule 1 x 591 [758,514,316,164,35,110,54,796,369]
+ CRUSH rule 1 x 592 [525,253,190,443,315,603,667,318,496]
+ CRUSH rule 1 x 593 [601,885,339,152,297,223,269,455,168]
+ CRUSH rule 1 x 594 [227,60,450,30,717,840,994,16,777]
+ CRUSH rule 1 x 595 [720,854,496,912,80,655,917,525,945]
+ CRUSH rule 1 x 596 [751,195,997,77,261,490,180,482,449]
+ CRUSH rule 1 x 597 [129,574,714,8,789,847,725,991,955]
+ CRUSH rule 1 x 598 [679,207,604,396,841,284,286,280,507]
+ CRUSH rule 1 x 599 [668,315,683,349,681,253,599,364,546]
+ CRUSH rule 1 x 600 [143,396,464,444,59,57,243,264,31]
+ CRUSH rule 1 x 601 [326,573,873,902,136,921,633,596,988]
+ CRUSH rule 1 x 602 [860,281,875,535,672,474,697,763,442]
+ CRUSH rule 1 x 603 [709,328,445,349,190,455,924,667,356]
+ CRUSH rule 1 x 604 [571,62,814,95,866,978,983,281,292]
+ CRUSH rule 1 x 605 [252,739,860,27,313,362,857,899,349]
+ CRUSH rule 1 x 606 [339,236,759,842,67,644,954,94,88]
+ CRUSH rule 1 x 607 [590,248,759,868,433,398,578,386,226]
+ CRUSH rule 1 x 608 [145,635,309,467,875,115,148,33,420]
+ CRUSH rule 1 x 609 [973,547,223,79,762,863,249,41,778]
+ CRUSH rule 1 x 610 [435,816,961,983,255,886,160,888,685]
+ CRUSH rule 1 x 611 [559,283,422,584,176,429,570,43,362]
+ CRUSH rule 1 x 612 [273,149,123,576,911,270,296,735,245]
+ CRUSH rule 1 x 613 [828,614,642,674,33,361,958,580,197]
+ CRUSH rule 1 x 614 [478,748,393,34,171,80,92,12,62]
+ CRUSH rule 1 x 615 [392,155,144,326,626,134,149,401,14]
+ CRUSH rule 1 x 616 [778,637,452,248,15,888,74,307,976]
+ CRUSH rule 1 x 617 [622,713,996,833,611,407,364,8,342]
+ CRUSH rule 1 x 618 [149,877,270,329,180,327,222,749,697]
+ CRUSH rule 1 x 619 [604,163,656,409,322,848,519,967,737]
+ CRUSH rule 1 x 620 [181,23,409,198,64,898,35,620,268]
+ CRUSH rule 1 x 621 [735,902,386,237,939,475,725,118,875]
+ CRUSH rule 1 x 622 [661,824,717,568,858,583,446,798,869]
+ CRUSH rule 1 x 623 [142,121,643,61,695,852,485,478,185]
+ CRUSH rule 1 x 624 [360,716,420,398,49,717,137,140,488]
+ CRUSH rule 1 x 625 [541,167,385,1,601,481,308,111,207]
+ CRUSH rule 1 x 626 [364,431,610,363,535,747,225,841,868]
+ CRUSH rule 1 x 627 [458,137,557,410,287,749,467,432,944]
+ CRUSH rule 1 x 628 [250,350,556,497,821,65,205,580,972]
+ CRUSH rule 1 x 629 [928,160,710,572,365,772,538,46,300]
+ CRUSH rule 1 x 630 [243,19,918,556,601,16,920,830,171]
+ CRUSH rule 1 x 631 [438,221,574,676,797,580,219,211,157]
+ CRUSH rule 1 x 632 [797,368,247,5,32,102,416,45,624]
+ CRUSH rule 1 x 633 [993,749,525,485,27,330,275,599,219]
+ CRUSH rule 1 x 634 [239,351,633,299,651,678,296,337,676]
+ CRUSH rule 1 x 635 [640,965,25,961,306,172,849,357,317]
+ CRUSH rule 1 x 636 [173,290,297,991,937,823,236,318,228]
+ CRUSH rule 1 x 637 [0,918,98,108,111,495,887,57,16]
+ CRUSH rule 1 x 638 [702,235,424,900,983,754,701,887,355]
+ CRUSH rule 1 x 639 [475,687,31,785,918,611,27,214,226]
+ CRUSH rule 1 x 640 [31,664,399,677,123,609,858,138,726]
+ CRUSH rule 1 x 641 [296,473,108,963,341,876,897,449,42]
+ CRUSH rule 1 x 642 [894,273,427,606,677,670,610,665,299]
+ CRUSH rule 1 x 643 [117,111,732,191,114,153,500,631,833]
+ CRUSH rule 1 x 644 [438,336,327,512,599,862,660,857,123]
+ CRUSH rule 1 x 645 [982,702,351,573,907,915,279,317,414]
+ CRUSH rule 1 x 646 [334,804,146,842,697,638,720,135,369]
+ CRUSH rule 1 x 647 [933,787,185,334,752,285,372,890,30]
+ CRUSH rule 1 x 648 [22,444,400,862,207,842,453,732,262]
+ CRUSH rule 1 x 649 [503,229,213,460,639,760,722,748,599]
+ CRUSH rule 1 x 650 [328,659,420,443,739,950,869,150,743]
+ CRUSH rule 1 x 651 [3,880,823,123,378,585,715,221,31]
+ CRUSH rule 1 x 652 [495,977,563,733,92,997,119,818,459]
+ CRUSH rule 1 x 653 [185,718,804,280,975,912,198,291,71]
+ CRUSH rule 1 x 654 [130,528,380,81,906,511,431,506,546]
+ CRUSH rule 1 x 655 [560,872,454,504,319,284,605,214,833]
+ CRUSH rule 1 x 656 [219,885,178,981,863,508,708,6,746]
+ CRUSH rule 1 x 657 [233,684,813,490,208,941,858,16,128]
+ CRUSH rule 1 x 658 [778,6,756,380,750,836,547,850,499]
+ CRUSH rule 1 x 659 [240,663,306,540,789,902,170,954,22]
+ CRUSH rule 1 x 660 [244,855,196,147,678,323,63,859,215]
+ CRUSH rule 1 x 661 [184,270,128,398,910,230,402,205,609]
+ CRUSH rule 1 x 662 [65,883,921,438,79,957,464,902,276]
+ CRUSH rule 1 x 663 [323,721,594,812,43,992,170,65,906]
+ CRUSH rule 1 x 664 [865,113,512,51,427,123,585,260,254]
+ CRUSH rule 1 x 665 [420,850,591,475,202,733,798,658,28]
+ CRUSH rule 1 x 666 [319,767,246,3,369,493,796,56,736]
+ CRUSH rule 1 x 667 [875,39,343,100,829,2,795,783,386]
+ CRUSH rule 1 x 668 [331,122,263,599,355,484,943,554,395]
+ CRUSH rule 1 x 669 [915,521,402,747,673,445,938,600,517]
+ CRUSH rule 1 x 670 [845,659,943,447,401,322,168,302,681]
+ CRUSH rule 1 x 671 [108,634,527,363,856,238,755,330,584]
+ CRUSH rule 1 x 672 [578,216,110,589,302,137,954,315,735]
+ CRUSH rule 1 x 673 [442,74,579,797,622,950,371,402,725]
+ CRUSH rule 1 x 674 [588,364,281,308,645,631,229,506,565]
+ CRUSH rule 1 x 675 [489,698,744,671,870,174,528,875,982]
+ CRUSH rule 1 x 676 [928,911,40,180,722,729,673,569,701]
+ CRUSH rule 1 x 677 [399,269,692,131,615,136,103,763,527]
+ CRUSH rule 1 x 678 [546,752,544,155,5,463,666,352,576]
+ CRUSH rule 1 x 679 [988,25,275,433,628,57,247,620,437]
+ CRUSH rule 1 x 680 [335,963,382,486,749,257,795,347,831]
+ CRUSH rule 1 x 681 [690,462,623,466,49,471,774,192,454]
+ CRUSH rule 1 x 682 [196,588,154,257,807,776,367,718,345]
+ CRUSH rule 1 x 683 [627,25,421,160,873,102,345,599,30]
+ CRUSH rule 1 x 684 [38,804,592,158,991,264,652,821,641]
+ CRUSH rule 1 x 685 [841,368,548,362,166,211,154,121,51]
+ CRUSH rule 1 x 686 [336,287,525,440,166,993,911,638,690]
+ CRUSH rule 1 x 687 [20,682,924,653,356,16,917,622,156]
+ CRUSH rule 1 x 688 [463,371,780,556,385,883,115,248,566]
+ CRUSH rule 1 x 689 [569,250,78,816,847,775,333,161,74]
+ CRUSH rule 1 x 690 [551,144,587,263,378,394,970,639,835]
+ CRUSH rule 1 x 691 [766,464,446,533,449,541,451,290,789]
+ CRUSH rule 1 x 692 [739,634,18,245,624,35,268,525,425]
+ CRUSH rule 1 x 693 [339,297,118,330,817,91,828,276,264]
+ CRUSH rule 1 x 694 [405,26,830,181,533,166,488,804,501]
+ CRUSH rule 1 x 695 [622,576,597,535,600,593,300,989,804]
+ CRUSH rule 1 x 696 [558,902,689,13,715,28,664,489,598]
+ CRUSH rule 1 x 697 [818,222,406,691,427,863,153,922,986]
+ CRUSH rule 1 x 698 [178,48,402,233,841,604,468,180,783]
+ CRUSH rule 1 x 699 [450,244,180,919,711,332,747,453,519]
+ CRUSH rule 1 x 700 [502,771,987,706,416,240,68,641,109]
+ CRUSH rule 1 x 701 [4,612,782,216,853,303,585,513,907]
+ CRUSH rule 1 x 702 [177,630,232,923,281,708,466,687,742]
+ CRUSH rule 1 x 703 [354,178,389,393,778,803,796,607,894]
+ CRUSH rule 1 x 704 [646,601,156,171,603,116,655,595,888]
+ CRUSH rule 1 x 705 [921,401,890,265,244,690,372,253,807]
+ CRUSH rule 1 x 706 [652,877,562,452,26,323,923,770,516]
+ CRUSH rule 1 x 707 [345,745,67,716,789,576,2,133,256]
+ CRUSH rule 1 x 708 [333,607,180,469,170,555,939,331,41]
+ CRUSH rule 1 x 709 [45,187,302,115,896,579,733,607,763]
+ CRUSH rule 1 x 710 [94,855,43,199,18,948,449,28,731]
+ CRUSH rule 1 x 711 [227,653,731,150,693,842,534,110,639]
+ CRUSH rule 1 x 712 [398,953,136,870,181,408,895,459,341]
+ CRUSH rule 1 x 713 [116,800,503,662,635,579,53,839,56]
+ CRUSH rule 1 x 714 [111,629,866,709,902,557,875,649,23]
+ CRUSH rule 1 x 715 [531,291,486,382,192,807,322,417,973]
+ CRUSH rule 1 x 716 [169,541,291,42,343,724,138,197,32]
+ CRUSH rule 1 x 717 [417,446,994,894,239,494,237,62,327]
+ CRUSH rule 1 x 718 [992,383,298,844,377,463,544,891,210]
+ CRUSH rule 1 x 719 [936,674,324,759,194,409,828,975,119]
+ CRUSH rule 1 x 720 [370,188,174,464,644,218,214,76,870]
+ CRUSH rule 1 x 721 [320,859,278,259,170,957,177,264,867]
+ CRUSH rule 1 x 722 [7,2,673,129,96,445,823,833,1]
+ CRUSH rule 1 x 723 [270,553,831,662,38,101,985,846,77]
+ CRUSH rule 1 x 724 [666,822,708,895,633,800,616,879,480]
+ CRUSH rule 1 x 725 [794,406,875,459,981,751,359,34,720]
+ CRUSH rule 1 x 726 [420,556,341,292,240,68,966,535,669]
+ CRUSH rule 1 x 727 [561,461,129,635,965,610,105,31,506]
+ CRUSH rule 1 x 728 [951,330,196,756,589,849,753,760,254]
+ CRUSH rule 1 x 729 [656,644,436,591,27,119,572,933,434]
+ CRUSH rule 1 x 730 [3,558,629,184,50,765,760,800,945]
+ CRUSH rule 1 x 731 [852,89,75,735,713,113,528,890,625]
+ CRUSH rule 1 x 732 [983,840,869,976,697,307,368,271,778]
+ CRUSH rule 1 x 733 [285,396,388,122,387,364,880,343,590]
+ CRUSH rule 1 x 734 [125,510,402,640,676,501,535,627,224]
+ CRUSH rule 1 x 735 [417,773,686,504,459,912,690,59,294]
+ CRUSH rule 1 x 736 [749,396,632,550,779,109,845,278,559]
+ CRUSH rule 1 x 737 [644,991,946,135,448,903,482,564,259]
+ CRUSH rule 1 x 738 [449,683,290,220,245,525,429,397,872]
+ CRUSH rule 1 x 739 [341,220,641,454,740,661,146,17,314]
+ CRUSH rule 1 x 740 [874,524,674,650,472,282,214,494,593]
+ CRUSH rule 1 x 741 [189,472,712,798,715,757,863,571,876]
+ CRUSH rule 1 x 742 [912,581,114,759,730,21,687,81,145]
+ CRUSH rule 1 x 743 [654,914,425,441,763,39,451,631,911]
+ CRUSH rule 1 x 744 [725,295,579,377,162,447,843,699,24]
+ CRUSH rule 1 x 745 [787,858,850,506,612,735,926,314,771]
+ CRUSH rule 1 x 746 [757,848,704,30,47,940,450,651,105]
+ CRUSH rule 1 x 747 [700,81,867,681,801,64,879,857,727]
+ CRUSH rule 1 x 748 [557,436,238,664,293,865,304,999,685]
+ CRUSH rule 1 x 749 [772,622,337,42,156,302,383,506,570]
+ CRUSH rule 1 x 750 [946,97,376,677,316,670,169,171,9]
+ CRUSH rule 1 x 751 [996,618,343,911,83,22,388,17,892]
+ CRUSH rule 1 x 752 [746,887,695,868,610,950,88,315,728]
+ CRUSH rule 1 x 753 [741,14,463,479,172,192,481,702,431]
+ CRUSH rule 1 x 754 [648,349,333,355,65,63,336,724,262]
+ CRUSH rule 1 x 755 [157,460,466,187,959,674,192,279,371]
+ CRUSH rule 1 x 756 [416,97,197,497,227,3,850,191,991]
+ CRUSH rule 1 x 757 [599,839,776,410,256,823,121,690,544]
+ CRUSH rule 1 x 758 [994,218,620,256,361,749,165,686,449]
+ CRUSH rule 1 x 759 [959,682,514,745,100,519,15,347,311]
+ CRUSH rule 1 x 760 [518,943,215,83,706,137,345,69,39]
+ CRUSH rule 1 x 761 [285,849,420,324,987,338,373,361,684]
+ CRUSH rule 1 x 762 [591,313,41,335,110,696,664,350,339]
+ CRUSH rule 1 x 763 [908,411,200,740,292,295,387,775,797]
+ CRUSH rule 1 x 764 [787,234,894,485,883,711,70,202,557]
+ CRUSH rule 1 x 765 [327,921,882,393,444,792,402,123,902]
+ CRUSH rule 1 x 766 [84,161,878,704,416,144,357,310,890]
+ CRUSH rule 1 x 767 [370,895,702,701,890,2,251,951,675]
+ CRUSH rule 1 x 768 [826,760,879,864,460,474,645,975,947]
+ CRUSH rule 1 x 769 [67,768,663,735,814,66,213,527,546]
+ CRUSH rule 1 x 770 [593,909,482,259,5,550,961,324,309]
+ CRUSH rule 1 x 771 [309,935,121,578,937,685,933,571,822]
+ CRUSH rule 1 x 772 [12,125,797,301,348,419,891,959,487]
+ CRUSH rule 1 x 773 [253,466,820,549,591,193,783,951,982]
+ CRUSH rule 1 x 774 [164,390,705,109,881,505,890,425,599]
+ CRUSH rule 1 x 775 [703,47,43,973,643,406,885,976,936]
+ CRUSH rule 1 x 776 [728,231,80,916,2,850,396,76,680]
+ CRUSH rule 1 x 777 [981,621,568,729,869,952,563,860,388]
+ CRUSH rule 1 x 778 [411,456,544,597,789,784,65,954,125]
+ CRUSH rule 1 x 779 [346,121,519,921,587,48,772,645,254]
+ CRUSH rule 1 x 780 [476,39,288,381,303,29,17,336,147]
+ CRUSH rule 1 x 781 [10,130,585,844,729,705,714,954,271]
+ CRUSH rule 1 x 782 [462,246,581,902,623,877,812,516,774]
+ CRUSH rule 1 x 783 [580,373,153,775,668,661,626,961,576]
+ CRUSH rule 1 x 784 [413,113,978,990,994,56,481,198,171]
+ CRUSH rule 1 x 785 [341,856,332,354,59,581,632,151,586]
+ CRUSH rule 1 x 786 [411,140,313,393,215,618,490,481,627]
+ CRUSH rule 1 x 787 [605,522,211,813,636,224,600,528,966]
+ CRUSH rule 1 x 788 [226,545,35,142,726,851,194,216,486]
+ CRUSH rule 1 x 789 [545,320,414,702,731,277,237,916,374]
+ CRUSH rule 1 x 790 [414,748,816,327,130,115,788,164,691]
+ CRUSH rule 1 x 791 [660,906,406,697,916,322,124,267,742]
+ CRUSH rule 1 x 792 [287,392,514,204,75,789,406,858,694]
+ CRUSH rule 1 x 793 [631,133,850,713,720,487,376,812,886]
+ CRUSH rule 1 x 794 [931,517,543,210,963,898,811,459,344]
+ CRUSH rule 1 x 795 [551,962,477,948,425,434,268,94,648]
+ CRUSH rule 1 x 796 [814,4,95,27,368,300,646,451,67]
+ CRUSH rule 1 x 797 [64,201,299,734,605,864,596,196,93]
+ CRUSH rule 1 x 798 [422,530,114,431,565,716,473,250,839]
+ CRUSH rule 1 x 799 [824,32,679,562,266,549,859,994,831]
+ CRUSH rule 1 x 800 [862,623,489,637,861,196,941,643,398]
+ CRUSH rule 1 x 801 [145,550,329,324,734,160,219,662,142]
+ CRUSH rule 1 x 802 [570,19,847,308,387,518,846,53,783]
+ CRUSH rule 1 x 803 [151,812,662,358,880,349,834,881,23]
+ CRUSH rule 1 x 804 [467,93,264,863,176,842,663,949,380]
+ CRUSH rule 1 x 805 [621,223,938,809,591,686,121,157,934]
+ CRUSH rule 1 x 806 [898,957,805,430,499,584,640,607,790]
+ CRUSH rule 1 x 807 [354,531,422,159,921,431,802,136,305]
+ CRUSH rule 1 x 808 [7,96,76,897,446,2,166,929,234]
+ CRUSH rule 1 x 809 [70,734,719,56,687,21,23,145,184]
+ CRUSH rule 1 x 810 [701,18,972,327,771,649,620,648,433]
+ CRUSH rule 1 x 811 [248,547,103,728,901,264,948,202,521]
+ CRUSH rule 1 x 812 [230,576,821,566,993,762,675,28,263]
+ CRUSH rule 1 x 813 [805,114,683,629,288,462,285,450,948]
+ CRUSH rule 1 x 814 [54,619,973,741,497,894,401,266,905]
+ CRUSH rule 1 x 815 [679,412,613,132,969,411,314,670,928]
+ CRUSH rule 1 x 816 [919,448,826,414,36,289,44,822,332]
+ CRUSH rule 1 x 817 [765,830,436,521,332,458,260,172,193]
+ CRUSH rule 1 x 818 [415,566,644,687,692,414,769,826,519]
+ CRUSH rule 1 x 819 [721,319,865,750,546,859,523,770,56]
+ CRUSH rule 1 x 820 [218,301,333,190,686,179,535,787,267]
+ CRUSH rule 1 x 821 [185,795,680,953,329,750,621,815,313]
+ CRUSH rule 1 x 822 [356,261,54,522,900,103,883,112,601]
+ CRUSH rule 1 x 823 [220,281,549,456,64,306,282,641,216]
+ CRUSH rule 1 x 824 [292,809,887,74,776,788,559,886,753]
+ CRUSH rule 1 x 825 [949,778,101,311,110,480,161,998,370]
+ CRUSH rule 1 x 826 [767,818,833,927,356,954,910,63,288]
+ CRUSH rule 1 x 827 [631,83,406,635,657,713,212,916,692]
+ CRUSH rule 1 x 828 [288,986,445,26,414,607,937,595,935]
+ CRUSH rule 1 x 829 [990,667,915,694,974,453,669,330,822]
+ CRUSH rule 1 x 830 [152,571,778,505,685,209,448,55,965]
+ CRUSH rule 1 x 831 [814,563,630,97,582,107,142,157,957]
+ CRUSH rule 1 x 832 [235,641,616,110,979,844,656,135,341]
+ CRUSH rule 1 x 833 [657,565,922,140,825,457,764,766,853]
+ CRUSH rule 1 x 834 [907,231,644,13,617,130,83,483,811]
+ CRUSH rule 1 x 835 [784,262,771,264,612,238,537,937,101]
+ CRUSH rule 1 x 836 [951,158,366,710,43,427,351,961,52]
+ CRUSH rule 1 x 837 [556,498,334,633,895,627,903,29,454]
+ CRUSH rule 1 x 838 [329,274,964,547,119,342,983,998,320]
+ CRUSH rule 1 x 839 [568,209,939,364,658,747,47,859,402]
+ CRUSH rule 1 x 840 [45,579,842,70,655,862,815,109,762]
+ CRUSH rule 1 x 841 [652,702,24,605,152,93,226,46,918]
+ CRUSH rule 1 x 842 [629,984,314,895,408,897,575,1,312]
+ CRUSH rule 1 x 843 [799,690,688,648,151,812,486,199,966]
+ CRUSH rule 1 x 844 [694,600,534,700,569,11,899,382,851]
+ CRUSH rule 1 x 845 [332,30,179,93,951,324,611,512,855]
+ CRUSH rule 1 x 846 [452,251,712,719,404,739,606,237,414]
+ CRUSH rule 1 x 847 [399,681,847,739,13,555,363,893,592]
+ CRUSH rule 1 x 848 [303,138,440,346,547,216,700,249,214]
+ CRUSH rule 1 x 849 [666,346,708,873,64,694,847,463,995]
+ CRUSH rule 1 x 850 [644,511,345,844,545,337,358,35,913]
+ CRUSH rule 1 x 851 [527,546,737,425,100,331,95,337,677]
+ CRUSH rule 1 x 852 [31,809,94,618,156,853,469,511,999]
+ CRUSH rule 1 x 853 [483,330,869,184,46,942,774,679,616]
+ CRUSH rule 1 x 854 [697,953,968,143,502,955,441,302,437]
+ CRUSH rule 1 x 855 [837,996,239,621,32,191,686,702,919]
+ CRUSH rule 1 x 856 [712,40,547,430,195,857,224,810,404]
+ CRUSH rule 1 x 857 [77,984,576,551,568,96,12,763,594]
+ CRUSH rule 1 x 858 [412,384,841,465,572,576,688,61,545]
+ CRUSH rule 1 x 859 [173,760,26,300,87,567,463,903,272]
+ CRUSH rule 1 x 860 [776,429,328,917,658,783,699,907,532]
+ CRUSH rule 1 x 861 [705,405,477,50,73,714,901,487,725]
+ CRUSH rule 1 x 862 [809,44,788,938,964,177,490,409,15]
+ CRUSH rule 1 x 863 [349,496,963,178,675,853,172,980,772]
+ CRUSH rule 1 x 864 [717,858,101,239,992,244,43,15,29]
+ CRUSH rule 1 x 865 [857,603,586,262,550,289,850,40,170]
+ CRUSH rule 1 x 866 [394,304,71,96,642,155,255,481,435]
+ CRUSH rule 1 x 867 [640,773,663,974,261,296,988,730,753]
+ CRUSH rule 1 x 868 [613,950,712,663,761,460,643,547,734]
+ CRUSH rule 1 x 869 [973,889,524,22,671,477,718,431,968]
+ CRUSH rule 1 x 870 [505,35,386,498,348,503,54,992,726]
+ CRUSH rule 1 x 871 [239,264,262,773,781,734,387,515,98]
+ CRUSH rule 1 x 872 [21,767,456,748,783,797,180,800,521]
+ CRUSH rule 1 x 873 [954,666,980,264,435,233,199,358,805]
+ CRUSH rule 1 x 874 [54,510,947,1,500,119,93,915,801]
+ CRUSH rule 1 x 875 [809,418,452,462,88,673,634,435,778]
+ CRUSH rule 1 x 876 [483,457,61,248,523,277,322,141,82]
+ CRUSH rule 1 x 877 [542,531,952,939,710,179,181,460,459]
+ CRUSH rule 1 x 878 [217,674,857,644,678,809,329,591,59]
+ CRUSH rule 1 x 879 [999,475,134,250,319,357,145,750,54]
+ CRUSH rule 1 x 880 [678,573,935,385,570,651,319,630,888]
+ CRUSH rule 1 x 881 [394,835,789,802,587,155,570,109,896]
+ CRUSH rule 1 x 882 [467,382,353,56,979,674,974,483,412]
+ CRUSH rule 1 x 883 [802,744,237,337,50,96,202,148,129]
+ CRUSH rule 1 x 884 [653,660,638,700,31,558,389,381,347]
+ CRUSH rule 1 x 885 [898,704,307,445,879,872,174,972,544]
+ CRUSH rule 1 x 886 [434,357,938,641,737,8,56,582,915]
+ CRUSH rule 1 x 887 [297,226,711,428,370,318,472,947,35]
+ CRUSH rule 1 x 888 [863,324,443,213,902,25,806,53,385]
+ CRUSH rule 1 x 889 [105,102,308,163,947,548,399,382,761]
+ CRUSH rule 1 x 890 [550,248,606,704,615,708,996,561,485]
+ CRUSH rule 1 x 891 [575,928,880,891,826,763,706,701,501]
+ CRUSH rule 1 x 892 [259,862,133,271,292,162,53,333,458]
+ CRUSH rule 1 x 893 [902,880,543,542,37,942,672,320,394]
+ CRUSH rule 1 x 894 [180,169,916,43,945,713,648,685,895]
+ CRUSH rule 1 x 895 [725,849,182,129,177,272,599,829,809]
+ CRUSH rule 1 x 896 [951,34,874,537,969,123,210,529,491]
+ CRUSH rule 1 x 897 [810,352,73,939,943,895,12,481,539]
+ CRUSH rule 1 x 898 [979,433,719,411,787,359,342,37,303]
+ CRUSH rule 1 x 899 [685,668,534,932,399,156,124,653,574]
+ CRUSH rule 1 x 900 [530,978,41,894,941,681,380,419,667]
+ CRUSH rule 1 x 901 [740,107,336,175,574,706,157,292,724]
+ CRUSH rule 1 x 902 [800,743,693,310,67,111,178,624,733]
+ CRUSH rule 1 x 903 [230,267,842,266,550,769,66,738,419]
+ CRUSH rule 1 x 904 [346,949,460,973,696,91,957,801,74]
+ CRUSH rule 1 x 905 [530,397,619,958,576,973,685,6,689]
+ CRUSH rule 1 x 906 [80,426,138,672,73,776,30,169,506]
+ CRUSH rule 1 x 907 [365,968,475,297,296,724,664,331,184]
+ CRUSH rule 1 x 908 [204,832,742,809,862,745,484,391,841]
+ CRUSH rule 1 x 909 [883,989,146,959,366,59,686,965,515]
+ CRUSH rule 1 x 910 [549,593,249,853,792,769,824,552,717]
+ CRUSH rule 1 x 911 [325,847,352,214,851,732,789,255,896]
+ CRUSH rule 1 x 912 [874,888,582,796,557,601,226,889,69]
+ CRUSH rule 1 x 913 [331,463,342,574,989,362,925,746,664]
+ CRUSH rule 1 x 914 [836,468,601,732,607,275,70,280,837]
+ CRUSH rule 1 x 915 [245,228,100,661,799,13,126,79,652]
+ CRUSH rule 1 x 916 [77,967,364,435,27,474,255,133,892]
+ CRUSH rule 1 x 917 [239,60,866,221,772,967,725,707,47]
+ CRUSH rule 1 x 918 [988,115,922,80,201,544,583,923,863]
+ CRUSH rule 1 x 919 [783,139,696,1,848,169,888,980,33]
+ CRUSH rule 1 x 920 [623,408,685,953,974,696,532,124,911]
+ CRUSH rule 1 x 921 [105,799,144,90,399,373,633,290,155]
+ CRUSH rule 1 x 922 [887,505,652,348,514,806,952,474,67]
+ CRUSH rule 1 x 923 [223,318,552,458,743,871,964,384,454]
+ CRUSH rule 1 x 924 [25,778,366,333,163,801,584,31,151]
+ CRUSH rule 1 x 925 [912,601,297,682,770,173,969,168,500]
+ CRUSH rule 1 x 926 [968,133,135,144,814,155,709,158,96]
+ CRUSH rule 1 x 927 [277,724,214,988,690,342,465,775,725]
+ CRUSH rule 1 x 928 [554,203,658,789,298,299,847,752,780]
+ CRUSH rule 1 x 929 [761,802,367,528,758,522,744,171,144]
+ CRUSH rule 1 x 930 [814,61,788,736,660,491,832,654,567]
+ CRUSH rule 1 x 931 [29,193,61,41,343,664,487,839,776]
+ CRUSH rule 1 x 932 [446,198,862,534,168,35,530,462,202]
+ CRUSH rule 1 x 933 [352,742,216,321,525,44,568,61,945]
+ CRUSH rule 1 x 934 [730,2,332,631,613,249,533,116,254]
+ CRUSH rule 1 x 935 [731,23,736,79,361,992,772,49,567]
+ CRUSH rule 1 x 936 [322,975,20,904,827,603,138,802,885]
+ CRUSH rule 1 x 937 [822,221,841,161,723,137,630,308,973]
+ CRUSH rule 1 x 938 [557,850,66,630,499,404,286,395,927]
+ CRUSH rule 1 x 939 [150,11,971,371,124,785,408,49,977]
+ CRUSH rule 1 x 940 [638,398,169,616,333,751,25,883,867]
+ CRUSH rule 1 x 941 [730,342,929,577,451,838,964,28,633]
+ CRUSH rule 1 x 942 [62,292,166,814,587,172,854,16,440]
+ CRUSH rule 1 x 943 [165,314,519,548,41,726,759,851,617]
+ CRUSH rule 1 x 944 [199,625,766,176,194,297,678,915,619]
+ CRUSH rule 1 x 945 [946,999,699,303,38,81,952,885,987]
+ CRUSH rule 1 x 946 [595,93,852,142,503,647,933,267,846]
+ CRUSH rule 1 x 947 [800,582,356,93,716,117,922,868,413]
+ CRUSH rule 1 x 948 [132,551,139,920,87,46,81,220,725]
+ CRUSH rule 1 x 949 [792,920,466,380,97,568,799,961,564]
+ CRUSH rule 1 x 950 [111,345,176,543,879,954,355,220,528]
+ CRUSH rule 1 x 951 [414,619,648,655,364,971,829,408,568]
+ CRUSH rule 1 x 952 [775,469,500,356,287,4,16,746,835]
+ CRUSH rule 1 x 953 [349,1,5,251,168,680,141,619,234]
+ CRUSH rule 1 x 954 [570,940,410,249,929,394,129,696,115]
+ CRUSH rule 1 x 955 [729,774,823,800,7,127,536,766,579]
+ CRUSH rule 1 x 956 [519,141,575,625,738,475,169,751,667]
+ CRUSH rule 1 x 957 [242,709,611,97,760,309,393,281,227]
+ CRUSH rule 1 x 958 [84,217,227,253,246,604,346,377,425]
+ CRUSH rule 1 x 959 [270,413,918,789,703,608,543,519,496]
+ CRUSH rule 1 x 960 [458,192,307,279,920,139,855,49,548]
+ CRUSH rule 1 x 961 [981,388,777,546,359,660,455,708,649]
+ CRUSH rule 1 x 962 [623,834,277,134,729,246,856,477,895]
+ CRUSH rule 1 x 963 [291,167,714,468,109,373,485,701,76]
+ CRUSH rule 1 x 964 [28,156,788,127,598,215,361,255,507]
+ CRUSH rule 1 x 965 [675,557,290,517,840,510,59,229,819]
+ CRUSH rule 1 x 966 [836,306,946,283,642,606,929,773,928]
+ CRUSH rule 1 x 967 [966,386,735,837,392,116,19,674,395]
+ CRUSH rule 1 x 968 [864,756,690,121,328,122,433,520,916]
+ CRUSH rule 1 x 969 [729,625,480,769,512,882,518,956,398]
+ CRUSH rule 1 x 970 [800,362,646,582,309,102,576,411,416]
+ CRUSH rule 1 x 971 [737,381,153,684,298,166,344,520,546]
+ CRUSH rule 1 x 972 [952,245,720,884,334,311,754,540,79]
+ CRUSH rule 1 x 973 [356,455,579,857,832,596,549,524,109]
+ CRUSH rule 1 x 974 [545,758,586,596,539,790,116,993,644]
+ CRUSH rule 1 x 975 [336,191,202,146,720,897,330,308,744]
+ CRUSH rule 1 x 976 [446,208,757,620,252,846,397,58,57]
+ CRUSH rule 1 x 977 [202,896,196,956,763,126,783,828,409]
+ CRUSH rule 1 x 978 [612,324,996,225,418,583,514,169,99]
+ CRUSH rule 1 x 979 [843,457,675,650,958,657,677,173,903]
+ CRUSH rule 1 x 980 [60,914,881,626,850,759,398,943,764]
+ CRUSH rule 1 x 981 [702,749,937,153,724,514,536,212,247]
+ CRUSH rule 1 x 982 [298,928,738,167,99,668,395,198,100]
+ CRUSH rule 1 x 983 [723,572,395,358,900,37,927,597,103]
+ CRUSH rule 1 x 984 [723,864,804,935,846,993,950,840,427]
+ CRUSH rule 1 x 985 [945,459,868,211,524,954,911,208,91]
+ CRUSH rule 1 x 986 [772,664,535,169,297,996,864,555,687]
+ CRUSH rule 1 x 987 [88,324,312,843,661,580,76,894,480]
+ CRUSH rule 1 x 988 [522,927,131,996,351,685,865,47,116]
+ CRUSH rule 1 x 989 [578,332,208,605,975,207,155,380,797]
+ CRUSH rule 1 x 990 [638,228,414,311,738,698,340,526,728]
+ CRUSH rule 1 x 991 [530,221,451,422,879,916,754,928,288]
+ CRUSH rule 1 x 992 [925,705,275,81,234,310,117,546,798]
+ CRUSH rule 1 x 993 [991,301,43,469,830,242,382,428,451]
+ CRUSH rule 1 x 994 [276,51,868,683,843,815,557,378,936]
+ CRUSH rule 1 x 995 [288,836,753,790,758,120,158,265,110]
+ CRUSH rule 1 x 996 [887,983,252,686,470,345,459,764,859]
+ CRUSH rule 1 x 997 [110,924,386,79,705,697,210,698,273]
+ CRUSH rule 1 x 998 [435,830,485,853,926,730,786,762,444]
+ CRUSH rule 1 x 999 [876,738,357,913,723,51,15,585,898]
+ CRUSH rule 1 x 1000 [178,963,638,430,845,586,317,102,200]
+ CRUSH rule 1 x 1001 [99,519,66,759,583,944,739,922,343]
+ CRUSH rule 1 x 1002 [515,534,468,866,878,717,729,370,326]
+ CRUSH rule 1 x 1003 [104,611,937,698,94,67,614,783,865]
+ CRUSH rule 1 x 1004 [269,638,724,375,491,121,891,113,424]
+ CRUSH rule 1 x 1005 [369,223,309,409,822,39,597,969,911]
+ CRUSH rule 1 x 1006 [40,107,69,275,79,429,234,945,598]
+ CRUSH rule 1 x 1007 [978,111,416,758,454,640,5,444,795]
+ CRUSH rule 1 x 1008 [965,956,624,832,421,96,975,723,909]
+ CRUSH rule 1 x 1009 [598,476,356,695,919,566,234,383,604]
+ CRUSH rule 1 x 1010 [767,523,239,517,29,77,23,241,838]
+ CRUSH rule 1 x 1011 [289,871,207,576,347,698,48,570,639]
+ CRUSH rule 1 x 1012 [128,28,370,31,341,755,268,647,669]
+ CRUSH rule 1 x 1013 [979,765,660,812,666,187,808,351,572]
+ CRUSH rule 1 x 1014 [979,948,513,88,47,825,969,81,586]
+ CRUSH rule 1 x 1015 [277,790,396,672,542,647,145,11,965]
+ CRUSH rule 1 x 1016 [262,73,128,886,839,685,456,560,935]
+ CRUSH rule 1 x 1017 [150,269,61,499,832,591,637,731,738]
+ CRUSH rule 1 x 1018 [555,829,554,944,406,576,463,926,475]
+ CRUSH rule 1 x 1019 [513,356,265,446,65,288,768,245,337]
+ CRUSH rule 1 x 1020 [158,161,877,704,948,570,495,865,698]
+ CRUSH rule 1 x 1021 [915,998,957,285,546,202,676,322,671]
+ CRUSH rule 1 x 1022 [967,829,973,640,703,470,871,828,440]
+ CRUSH rule 1 x 1023 [488,257,614,859,325,419,50,560,595]
+ rule 1 (metadata) num_rep 9 result size == 9:\t1024/1024 (esc)
+ CRUSH rule 1 x 0 [36,705,536,450,604,380,966,750,695,503]
+ CRUSH rule 1 x 1 [876,250,334,633,744,843,672,820,782,802]
+ CRUSH rule 1 x 2 [292,832,53,392,386,787,527,901,106,273]
+ CRUSH rule 1 x 3 [623,387,124,998,749,211,481,169,816,732]
+ CRUSH rule 1 x 4 [61,334,710,4,994,982,847,220,87,254]
+ CRUSH rule 1 x 5 [946,557,713,664,141,817,964,872,66,161]
+ CRUSH rule 1 x 6 [576,668,212,163,732,381,884,726,456,796]
+ CRUSH rule 1 x 7 [645,753,906,393,341,44,578,14,543,287]
+ CRUSH rule 1 x 8 [243,6,863,781,211,100,462,207,759,701]
+ CRUSH rule 1 x 9 [22,578,251,410,297,430,3,569,603,47]
+ CRUSH rule 1 x 10 [758,828,360,477,821,801,811,484,296,320]
+ CRUSH rule 1 x 11 [769,120,124,527,119,504,380,821,470,230]
+ CRUSH rule 1 x 12 [780,364,689,755,675,199,117,393,435,514]
+ CRUSH rule 1 x 13 [557,18,351,719,742,780,78,170,333,295]
+ CRUSH rule 1 x 14 [59,561,249,461,971,835,855,76,269,673]
+ CRUSH rule 1 x 15 [718,928,993,21,76,313,437,797,680,761]
+ CRUSH rule 1 x 16 [673,632,841,954,788,90,786,969,378,246]
+ CRUSH rule 1 x 17 [648,43,560,514,142,289,935,605,228,737]
+ CRUSH rule 1 x 18 [654,219,181,568,381,253,883,394,188,459]
+ CRUSH rule 1 x 19 [850,545,377,848,863,543,51,834,690,375]
+ CRUSH rule 1 x 20 [717,785,974,5,225,552,975,636,387,600]
+ CRUSH rule 1 x 21 [420,57,519,306,312,983,263,267,128,828]
+ CRUSH rule 1 x 22 [503,998,193,821,634,684,557,633,812,521]
+ CRUSH rule 1 x 23 [411,663,168,110,899,488,477,468,303,367]
+ CRUSH rule 1 x 24 [266,861,353,1,456,128,800,309,622,673]
+ CRUSH rule 1 x 25 [760,483,818,600,509,951,248,908,624,643]
+ CRUSH rule 1 x 26 [903,24,573,718,112,694,501,909,219,686]
+ CRUSH rule 1 x 27 [946,188,289,510,687,827,676,560,753,77]
+ CRUSH rule 1 x 28 [69,312,73,198,256,629,770,569,359,733]
+ CRUSH rule 1 x 29 [844,883,337,628,496,405,719,581,816,349]
+ CRUSH rule 1 x 30 [621,18,613,794,910,936,426,522,208,699]
+ CRUSH rule 1 x 31 [784,943,814,539,962,392,813,217,750,155]
+ CRUSH rule 1 x 32 [173,374,369,972,315,83,428,63,801,735]
+ CRUSH rule 1 x 33 [698,336,357,966,582,407,618,288,846,659]
+ CRUSH rule 1 x 34 [168,836,210,798,904,190,663,877,177,567]
+ CRUSH rule 1 x 35 [274,509,534,818,912,671,75,580,568,523]
+ CRUSH rule 1 x 36 [318,215,153,628,87,407,676,524,510,480]
+ CRUSH rule 1 x 37 [173,604,109,935,203,401,311,758,201,999]
+ CRUSH rule 1 x 38 [708,444,683,604,722,900,929,910,698,386]
+ CRUSH rule 1 x 39 [662,198,417,680,226,342,856,248,279,416]
+ CRUSH rule 1 x 40 [620,801,414,78,560,766,980,503,287,564]
+ CRUSH rule 1 x 41 [811,264,177,127,148,791,930,74,844,943]
+ CRUSH rule 1 x 42 [863,179,527,660,133,529,456,713,348,311]
+ CRUSH rule 1 x 43 [686,822,988,228,791,549,514,40,261,223]
+ CRUSH rule 1 x 44 [396,222,46,841,536,140,160,527,250,247]
+ CRUSH rule 1 x 45 [991,694,253,142,54,422,658,876,201,45]
+ CRUSH rule 1 x 46 [420,909,184,285,508,458,45,390,546,908]
+ CRUSH rule 1 x 47 [467,211,605,207,241,881,959,800,743,161]
+ CRUSH rule 1 x 48 [955,329,368,168,698,787,738,47,812,166]
+ CRUSH rule 1 x 49 [974,891,931,29,813,506,822,628,696,407]
+ CRUSH rule 1 x 50 [870,441,691,823,761,6,83,344,713,857]
+ CRUSH rule 1 x 51 [182,930,25,936,97,260,406,281,991,336]
+ CRUSH rule 1 x 52 [704,812,894,794,481,37,304,899,629,701]
+ CRUSH rule 1 x 53 [185,713,631,280,345,558,882,503,327,402]
+ CRUSH rule 1 x 54 [270,441,100,82,983,930,339,902,81,239]
+ CRUSH rule 1 x 55 [895,734,958,793,651,572,508,763,108,185]
+ CRUSH rule 1 x 56 [564,963,683,324,40,189,77,500,553,417]
+ CRUSH rule 1 x 57 [738,130,208,973,498,861,670,67,114,685]
+ CRUSH rule 1 x 58 [524,113,806,903,531,334,8,762,842,884]
+ CRUSH rule 1 x 59 [408,337,668,529,34,384,643,511,370,336]
+ CRUSH rule 1 x 60 [228,790,857,309,616,895,194,277,985,554]
+ CRUSH rule 1 x 61 [154,843,717,467,883,536,812,14,55,752]
+ CRUSH rule 1 x 62 [594,811,549,276,693,917,45,723,926,180]
+ CRUSH rule 1 x 63 [646,67,884,925,941,434,705,268,140,942]
+ CRUSH rule 1 x 64 [175,542,155,837,594,197,451,891,654,294]
+ CRUSH rule 1 x 65 [745,619,131,867,269,62,862,221,66,354]
+ CRUSH rule 1 x 66 [275,468,23,35,328,432,334,656,719,810]
+ CRUSH rule 1 x 67 [246,958,524,493,636,227,783,593,814,970]
+ CRUSH rule 1 x 68 [711,473,403,228,835,126,705,114,981,267]
+ CRUSH rule 1 x 69 [493,924,850,939,950,105,871,361,533,433]
+ CRUSH rule 1 x 70 [30,499,644,33,804,654,684,411,114,42]
+ CRUSH rule 1 x 71 [984,883,574,716,575,391,587,264,446,572]
+ CRUSH rule 1 x 72 [71,286,942,363,628,632,642,529,966,919]
+ CRUSH rule 1 x 73 [922,618,3,371,464,442,835,705,61,745]
+ CRUSH rule 1 x 74 [629,414,185,573,678,338,633,560,565,410]
+ CRUSH rule 1 x 75 [222,20,174,820,312,361,366,258,711,657]
+ CRUSH rule 1 x 76 [262,366,339,290,718,143,735,953,188,516]
+ CRUSH rule 1 x 77 [638,469,992,280,773,892,197,690,426,681]
+ CRUSH rule 1 x 78 [324,511,788,7,308,228,183,917,464,518]
+ CRUSH rule 1 x 79 [577,990,64,94,447,924,339,24,581,969]
+ CRUSH rule 1 x 80 [501,95,278,903,631,842,51,766,822,687]
+ CRUSH rule 1 x 81 [506,812,9,698,173,664,247,963,0,732]
+ CRUSH rule 1 x 82 [222,145,80,785,835,745,580,51,939,278]
+ CRUSH rule 1 x 83 [71,634,61,91,856,529,66,197,698,318]
+ CRUSH rule 1 x 84 [49,761,773,368,318,708,681,618,723,516]
+ CRUSH rule 1 x 85 [985,896,708,861,325,307,567,908,514,355]
+ CRUSH rule 1 x 86 [537,745,93,524,466,356,38,326,385,899]
+ CRUSH rule 1 x 87 [997,317,463,626,685,845,909,49,28,698]
+ CRUSH rule 1 x 88 [957,350,890,857,375,176,99,737,942,647]
+ CRUSH rule 1 x 89 [399,730,148,314,159,982,320,921,812,908]
+ CRUSH rule 1 x 90 [943,706,683,267,579,141,412,184,529,127]
+ CRUSH rule 1 x 91 [22,368,149,928,140,529,495,299,812,743]
+ CRUSH rule 1 x 92 [532,424,426,773,623,197,167,634,781,242]
+ CRUSH rule 1 x 93 [218,489,405,681,549,201,343,949,51,732]
+ CRUSH rule 1 x 94 [181,96,102,515,776,365,82,422,738,933]
+ CRUSH rule 1 x 95 [343,957,820,139,334,37,648,661,46,112]
+ CRUSH rule 1 x 96 [861,270,87,797,0,245,204,750,322,75]
+ CRUSH rule 1 x 97 [459,706,45,328,274,605,83,542,131,240]
+ CRUSH rule 1 x 98 [327,867,353,948,728,280,270,511,586,230]
+ CRUSH rule 1 x 99 [974,133,468,906,235,988,37,138,326,603]
+ CRUSH rule 1 x 100 [32,445,547,371,960,885,9,168,590,873]
+ CRUSH rule 1 x 101 [142,90,337,950,970,570,12,369,9,872]
+ CRUSH rule 1 x 102 [172,129,139,22,403,867,923,106,653,999]
+ CRUSH rule 1 x 103 [630,47,161,356,911,421,933,231,520,303]
+ CRUSH rule 1 x 104 [758,133,278,11,947,799,401,85,139,855]
+ CRUSH rule 1 x 105 [843,604,47,33,401,632,434,121,488,644]
+ CRUSH rule 1 x 106 [28,681,193,679,990,343,878,493,550,484]
+ CRUSH rule 1 x 107 [74,320,85,819,315,253,589,614,814,970]
+ CRUSH rule 1 x 108 [875,593,575,517,107,153,631,996,630,597]
+ CRUSH rule 1 x 109 [411,985,811,720,198,666,856,296,122,477]
+ CRUSH rule 1 x 110 [440,774,799,660,715,167,510,472,270,753]
+ CRUSH rule 1 x 111 [405,742,276,359,936,360,18,949,341,837]
+ CRUSH rule 1 x 112 [143,181,922,545,185,303,725,413,187,840]
+ CRUSH rule 1 x 113 [153,846,160,903,789,897,738,253,213,541]
+ CRUSH rule 1 x 114 [804,892,939,20,312,692,598,418,641,891]
+ CRUSH rule 1 x 115 [588,508,958,580,232,722,421,39,241,881]
+ CRUSH rule 1 x 116 [327,148,637,486,712,464,9,448,816,609]
+ CRUSH rule 1 x 117 [95,594,989,131,714,275,725,142,304,591]
+ CRUSH rule 1 x 118 [80,957,897,239,359,432,766,210,528,252]
+ CRUSH rule 1 x 119 [386,932,951,768,679,300,570,278,867,489]
+ CRUSH rule 1 x 120 [366,312,653,936,71,241,49,126,410,33]
+ CRUSH rule 1 x 121 [129,154,847,16,471,481,424,868,469,183]
+ CRUSH rule 1 x 122 [873,1,110,939,90,412,551,43,590,51]
+ CRUSH rule 1 x 123 [533,415,789,600,713,800,877,248,753,395]
+ CRUSH rule 1 x 124 [461,691,898,723,957,759,482,254,158,641]
+ CRUSH rule 1 x 125 [342,599,830,402,615,994,736,737,508,150]
+ CRUSH rule 1 x 126 [819,781,822,548,279,255,689,209,99,479]
+ CRUSH rule 1 x 127 [437,893,585,707,353,189,909,809,553,785]
+ CRUSH rule 1 x 128 [679,994,982,550,991,324,666,691,899,665]
+ CRUSH rule 1 x 129 [380,685,947,302,698,144,149,983,904,70]
+ CRUSH rule 1 x 130 [992,52,466,867,998,777,270,425,864,38]
+ CRUSH rule 1 x 131 [469,90,208,599,829,656,203,667,528,387]
+ CRUSH rule 1 x 132 [571,250,316,535,54,418,922,597,680,25]
+ CRUSH rule 1 x 133 [964,728,329,902,108,118,14,444,709,592]
+ CRUSH rule 1 x 134 [999,19,716,963,323,559,893,281,226,739]
+ CRUSH rule 1 x 135 [634,101,52,938,413,573,712,649,27,274]
+ CRUSH rule 1 x 136 [114,889,692,768,694,279,846,890,151,872]
+ CRUSH rule 1 x 137 [839,8,959,280,922,870,363,323,153,238]
+ CRUSH rule 1 x 138 [967,949,138,451,292,548,400,885,907,214]
+ CRUSH rule 1 x 139 [308,711,736,247,632,126,384,58,373,121]
+ CRUSH rule 1 x 140 [764,936,926,55,331,115,178,532,883,380]
+ CRUSH rule 1 x 141 [423,302,112,216,603,873,193,258,445,451]
+ CRUSH rule 1 x 142 [252,821,715,340,635,668,424,515,751,746]
+ CRUSH rule 1 x 143 [33,808,518,477,325,316,266,70,210,61]
+ CRUSH rule 1 x 144 [472,88,969,162,401,771,697,610,203,382]
+ CRUSH rule 1 x 145 [242,208,252,604,266,743,577,348,1,323]
+ CRUSH rule 1 x 146 [290,70,570,384,934,856,929,196,880,458]
+ CRUSH rule 1 x 147 [447,352,657,493,467,918,514,546,861,796]
+ CRUSH rule 1 x 148 [212,644,432,658,109,275,352,820,857,282]
+ CRUSH rule 1 x 149 [9,775,87,35,260,646,406,556,532,750]
+ CRUSH rule 1 x 150 [166,456,582,144,324,340,484,553,315,504]
+ CRUSH rule 1 x 151 [811,875,307,20,782,229,671,883,204,12]
+ CRUSH rule 1 x 152 [449,617,223,9,182,407,807,50,206,368]
+ CRUSH rule 1 x 153 [523,537,695,627,959,613,942,864,388,639]
+ CRUSH rule 1 x 154 [208,559,874,597,243,706,443,98,27,120]
+ CRUSH rule 1 x 155 [569,325,192,296,367,848,58,641,186,553]
+ CRUSH rule 1 x 156 [488,121,521,213,595,837,271,229,961,163]
+ CRUSH rule 1 x 157 [140,723,633,260,487,856,384,446,836,917]
+ CRUSH rule 1 x 158 [786,451,320,239,667,632,899,902,956,424]
+ CRUSH rule 1 x 159 [134,664,517,821,667,944,209,641,228,213]
+ CRUSH rule 1 x 160 [690,112,414,990,183,590,242,999,974,652]
+ CRUSH rule 1 x 161 [324,912,397,423,991,284,909,642,188,143]
+ CRUSH rule 1 x 162 [748,567,284,183,463,336,148,88,764,40]
+ CRUSH rule 1 x 163 [575,499,31,816,749,737,587,854,482,522]
+ CRUSH rule 1 x 164 [314,489,308,326,51,568,110,329,361,743]
+ CRUSH rule 1 x 165 [116,209,750,53,813,640,524,389,185,893]
+ CRUSH rule 1 x 166 [352,706,701,810,718,527,548,676,448,991]
+ CRUSH rule 1 x 167 [27,743,174,142,551,1,935,266,883,77]
+ CRUSH rule 1 x 168 [953,898,880,660,500,799,667,463,818,819]
+ CRUSH rule 1 x 169 [912,147,266,547,331,770,601,909,60,255]
+ CRUSH rule 1 x 170 [421,515,828,844,151,981,835,840,548,588]
+ CRUSH rule 1 x 171 [488,584,880,964,936,196,100,910,446,541]
+ CRUSH rule 1 x 172 [366,443,957,66,162,693,36,356,274,976]
+ CRUSH rule 1 x 173 [863,291,625,287,158,496,471,529,359,571]
+ CRUSH rule 1 x 174 [263,555,650,410,339,616,780,932,573,814]
+ CRUSH rule 1 x 175 [875,961,361,575,33,109,51,211,409,865]
+ CRUSH rule 1 x 176 [745,83,701,680,250,420,240,316,337,361]
+ CRUSH rule 1 x 177 [128,244,41,123,422,902,756,647,45,752]
+ CRUSH rule 1 x 178 [155,41,264,777,314,564,856,992,696,784]
+ CRUSH rule 1 x 179 [593,833,202,183,971,38,724,923,450,340]
+ CRUSH rule 1 x 180 [154,734,17,831,824,522,736,846,926,129]
+ CRUSH rule 1 x 181 [289,675,723,800,166,712,168,224,705,185]
+ CRUSH rule 1 x 182 [730,931,560,209,943,261,485,571,796,587]
+ CRUSH rule 1 x 183 [639,237,794,815,827,400,109,903,96,526]
+ CRUSH rule 1 x 184 [704,312,685,645,691,778,74,45,438,26]
+ CRUSH rule 1 x 185 [97,100,762,82,999,542,485,511,14,329]
+ CRUSH rule 1 x 186 [26,665,554,215,280,421,369,270,16,920]
+ CRUSH rule 1 x 187 [649,14,740,494,402,684,566,378,816,553]
+ CRUSH rule 1 x 188 [682,695,590,743,927,945,833,650,761,468]
+ CRUSH rule 1 x 189 [325,693,726,51,448,169,37,1,939,463]
+ CRUSH rule 1 x 190 [399,933,136,955,57,504,527,237,295,716]
+ CRUSH rule 1 x 191 [629,533,17,126,60,146,999,754,339,271]
+ CRUSH rule 1 x 192 [503,578,38,492,222,251,123,759,147,99]
+ CRUSH rule 1 x 193 [546,333,651,678,823,652,359,721,996,318]
+ CRUSH rule 1 x 194 [242,473,58,655,907,277,792,887,561,449]
+ CRUSH rule 1 x 195 [625,719,135,81,636,513,755,471,658,744]
+ CRUSH rule 1 x 196 [357,114,125,867,250,522,413,834,832,368]
+ CRUSH rule 1 x 197 [306,954,453,873,211,334,666,316,243,320]
+ CRUSH rule 1 x 198 [863,791,311,911,206,61,355,574,781,550]
+ CRUSH rule 1 x 199 [935,906,929,252,893,75,960,369,584,612]
+ CRUSH rule 1 x 200 [373,774,229,454,909,611,132,271,128,632]
+ CRUSH rule 1 x 201 [659,320,477,313,779,16,495,76,598,301]
+ CRUSH rule 1 x 202 [260,433,524,880,223,818,153,272,944,741]
+ CRUSH rule 1 x 203 [36,239,675,971,703,209,669,676,762,200]
+ CRUSH rule 1 x 204 [92,516,993,728,279,478,697,881,64,107]
+ CRUSH rule 1 x 205 [68,395,473,45,683,662,776,463,327,721]
+ CRUSH rule 1 x 206 [570,530,642,380,311,398,230,367,890,953]
+ CRUSH rule 1 x 207 [834,457,850,917,456,296,76,708,101,928]
+ CRUSH rule 1 x 208 [927,484,640,976,803,626,96,841,811,979]
+ CRUSH rule 1 x 209 [878,66,58,940,48,233,522,185,949,590]
+ CRUSH rule 1 x 210 [572,981,484,29,0,426,14,921,544,334]
+ CRUSH rule 1 x 211 [107,597,780,857,895,57,922,372,581,629]
+ CRUSH rule 1 x 212 [389,107,838,624,698,562,857,894,60,426]
+ CRUSH rule 1 x 213 [497,717,567,728,905,134,687,903,620,572]
+ CRUSH rule 1 x 214 [798,65,254,572,32,393,579,79,258,42]
+ CRUSH rule 1 x 215 [233,419,283,638,520,891,982,826,488,314]
+ CRUSH rule 1 x 216 [494,464,742,523,459,174,973,898,556,293]
+ CRUSH rule 1 x 217 [352,396,309,938,66,41,264,6,603,317]
+ CRUSH rule 1 x 218 [895,864,988,650,593,740,34,497,108,180]
+ CRUSH rule 1 x 219 [222,534,277,242,658,482,697,805,976,758]
+ CRUSH rule 1 x 220 [281,19,584,563,858,965,686,982,0,32]
+ CRUSH rule 1 x 221 [64,928,963,130,312,394,61,559,846,994]
+ CRUSH rule 1 x 222 [40,544,161,199,861,644,597,904,897,376]
+ CRUSH rule 1 x 223 [645,556,159,417,46,135,465,429,614,711]
+ CRUSH rule 1 x 224 [647,165,957,263,961,576,329,320,645,829]
+ CRUSH rule 1 x 225 [219,714,858,747,461,175,606,465,354,404]
+ CRUSH rule 1 x 226 [372,511,181,277,695,404,876,984,491,784]
+ CRUSH rule 1 x 227 [925,156,714,863,257,74,966,217,501,536]
+ CRUSH rule 1 x 228 [682,404,839,263,521,195,261,389,281,467]
+ CRUSH rule 1 x 229 [880,838,770,891,236,542,262,884,215,687]
+ CRUSH rule 1 x 230 [328,659,916,468,646,572,93,880,959,111]
+ CRUSH rule 1 x 231 [320,383,669,109,627,621,50,182,541,483]
+ CRUSH rule 1 x 232 [924,846,394,319,43,519,106,877,130,387]
+ CRUSH rule 1 x 233 [948,652,575,838,498,395,796,835,714,751]
+ CRUSH rule 1 x 234 [484,943,42,575,936,180,103,95,634,844]
+ CRUSH rule 1 x 235 [750,65,590,168,870,308,471,753,350,224]
+ CRUSH rule 1 x 236 [551,787,490,136,370,833,573,128,154,326]
+ CRUSH rule 1 x 237 [390,157,166,251,752,75,327,509,325,245]
+ CRUSH rule 1 x 238 [570,6,989,707,514,905,894,884,824,343]
+ CRUSH rule 1 x 239 [729,959,376,975,496,49,426,427,736,836]
+ CRUSH rule 1 x 240 [981,241,156,767,631,576,450,677,659,183]
+ CRUSH rule 1 x 241 [310,816,641,177,996,454,413,136,411,549]
+ CRUSH rule 1 x 242 [161,63,642,837,763,458,234,756,496,779]
+ CRUSH rule 1 x 243 [180,394,33,683,189,419,799,21,13,874]
+ CRUSH rule 1 x 244 [52,174,685,189,78,310,785,107,816,89]
+ CRUSH rule 1 x 245 [523,121,915,84,386,409,605,837,1,141]
+ CRUSH rule 1 x 246 [362,893,390,487,817,88,989,999,138,674]
+ CRUSH rule 1 x 247 [382,184,116,34,143,15,590,840,586,594]
+ CRUSH rule 1 x 248 [129,114,852,469,359,291,713,237,468,340]
+ CRUSH rule 1 x 249 [159,683,91,856,475,369,886,650,827,663]
+ CRUSH rule 1 x 250 [404,945,569,955,228,910,270,619,450,707]
+ CRUSH rule 1 x 251 [661,225,738,757,37,642,58,354,16,905]
+ CRUSH rule 1 x 252 [961,226,542,103,945,885,838,131,387,664]
+ CRUSH rule 1 x 253 [651,97,225,364,189,248,797,675,452,129]
+ CRUSH rule 1 x 254 [123,33,741,692,599,11,605,453,987,316]
+ CRUSH rule 1 x 255 [314,649,891,855,517,344,607,95,121,784]
+ CRUSH rule 1 x 256 [315,215,651,126,470,849,189,627,592,241]
+ CRUSH rule 1 x 257 [825,264,867,301,529,409,291,732,224,841]
+ CRUSH rule 1 x 258 [624,789,370,723,131,982,863,427,873,223]
+ CRUSH rule 1 x 259 [602,542,70,563,947,723,77,191,669,61]
+ CRUSH rule 1 x 260 [717,878,43,56,377,481,533,646,475,686]
+ CRUSH rule 1 x 261 [145,517,20,903,786,939,516,136,87,410]
+ CRUSH rule 1 x 262 [223,1,561,420,805,16,88,534,289,498]
+ CRUSH rule 1 x 263 [462,211,405,508,787,669,773,979,719,421]
+ CRUSH rule 1 x 264 [654,471,266,662,135,564,715,916,633,121]
+ CRUSH rule 1 x 265 [302,794,704,798,659,487,833,987,445,23]
+ CRUSH rule 1 x 266 [202,132,884,209,551,984,7,557,76,987]
+ CRUSH rule 1 x 267 [282,938,657,113,672,993,972,645,882,451]
+ CRUSH rule 1 x 268 [338,309,356,278,928,797,715,536,983,688]
+ CRUSH rule 1 x 269 [738,122,266,200,894,118,146,14,414,236]
+ CRUSH rule 1 x 270 [707,982,946,196,407,804,476,571,314,538]
+ CRUSH rule 1 x 271 [705,432,364,735,512,595,263,138,526,607]
+ CRUSH rule 1 x 272 [756,545,942,56,542,449,710,779,161,222]
+ CRUSH rule 1 x 273 [197,502,527,721,239,648,982,735,58,1]
+ CRUSH rule 1 x 274 [992,44,653,573,527,702,370,990,320,52]
+ CRUSH rule 1 x 275 [544,789,170,434,23,926,992,823,321,784]
+ CRUSH rule 1 x 276 [658,467,577,268,336,5,634,98,457,487]
+ CRUSH rule 1 x 277 [143,490,880,483,928,272,783,648,927,285]
+ CRUSH rule 1 x 278 [492,647,355,282,834,64,350,600,283,422]
+ CRUSH rule 1 x 279 [517,792,604,987,527,894,952,250,206,714]
+ CRUSH rule 1 x 280 [825,740,27,848,514,750,895,914,892,149]
+ CRUSH rule 1 x 281 [224,629,120,562,616,200,443,604,52,638]
+ CRUSH rule 1 x 282 [298,661,380,416,35,585,939,879,338,786]
+ CRUSH rule 1 x 283 [311,606,208,50,913,678,369,544,721,267]
+ CRUSH rule 1 x 284 [771,466,371,743,672,119,60,546,39,71]
+ CRUSH rule 1 x 285 [693,362,404,676,797,531,582,975,810,703]
+ CRUSH rule 1 x 286 [364,477,285,167,270,617,699,627,725,389]
+ CRUSH rule 1 x 287 [591,611,828,995,170,987,137,890,487,621]
+ CRUSH rule 1 x 288 [965,541,848,796,251,668,195,538,356,523]
+ CRUSH rule 1 x 289 [225,551,948,877,219,167,795,377,825,874]
+ CRUSH rule 1 x 290 [577,762,777,751,291,349,473,209,59,346]
+ CRUSH rule 1 x 291 [160,903,477,381,490,559,557,86,89,417]
+ CRUSH rule 1 x 292 [873,598,216,666,222,228,806,911,738,969]
+ CRUSH rule 1 x 293 [100,234,874,47,28,452,775,636,232,518]
+ CRUSH rule 1 x 294 [285,943,379,520,725,547,459,833,503,207]
+ CRUSH rule 1 x 295 [938,262,880,327,687,3,440,73,29,38]
+ CRUSH rule 1 x 296 [850,327,86,472,1,776,266,82,671,320]
+ CRUSH rule 1 x 297 [951,53,99,558,753,228,232,343,831,540]
+ CRUSH rule 1 x 298 [173,336,85,766,910,657,213,286,61,961]
+ CRUSH rule 1 x 299 [598,591,315,386,895,296,924,106,63,457]
+ CRUSH rule 1 x 300 [531,957,62,459,156,538,904,838,458,828]
+ CRUSH rule 1 x 301 [823,628,23,858,629,808,220,432,393,433]
+ CRUSH rule 1 x 302 [184,80,780,871,531,211,400,365,697,497]
+ CRUSH rule 1 x 303 [521,766,222,830,988,275,561,905,522,342]
+ CRUSH rule 1 x 304 [980,127,807,507,555,245,214,944,845,895]
+ CRUSH rule 1 x 305 [153,816,22,927,696,911,685,838,3,983]
+ CRUSH rule 1 x 306 [423,739,664,753,178,431,761,648,867,488]
+ CRUSH rule 1 x 307 [997,557,682,456,479,631,459,250,415,194]
+ CRUSH rule 1 x 308 [991,874,534,465,330,284,976,551,126,307]
+ CRUSH rule 1 x 309 [860,394,724,858,246,866,857,153,970,99]
+ CRUSH rule 1 x 310 [589,818,546,201,94,653,90,855,441,736]
+ CRUSH rule 1 x 311 [477,774,225,590,830,559,256,798,743,645]
+ CRUSH rule 1 x 312 [887,853,950,354,58,23,497,929,92,639]
+ CRUSH rule 1 x 313 [802,646,447,416,557,118,24,81,215,850]
+ CRUSH rule 1 x 314 [654,974,229,511,562,916,952,599,201,763]
+ CRUSH rule 1 x 315 [767,227,28,740,828,156,749,841,969,314]
+ CRUSH rule 1 x 316 [778,83,733,359,858,319,761,725,923,461]
+ CRUSH rule 1 x 317 [184,418,642,986,939,675,892,86,214,189]
+ CRUSH rule 1 x 318 [525,410,500,543,212,95,290,97,529,220]
+ CRUSH rule 1 x 319 [476,724,569,382,409,521,800,868,364,427]
+ CRUSH rule 1 x 320 [149,610,697,296,818,955,523,366,891,998]
+ CRUSH rule 1 x 321 [710,79,667,671,234,4,868,841,563,961]
+ CRUSH rule 1 x 322 [175,275,323,333,744,718,187,380,947,952]
+ CRUSH rule 1 x 323 [819,604,638,792,316,544,236,307,969,232]
+ CRUSH rule 1 x 324 [16,745,511,439,272,932,668,959,845,759]
+ CRUSH rule 1 x 325 [486,400,872,873,251,68,462,268,124,431]
+ CRUSH rule 1 x 326 [613,765,207,19,359,370,461,509,75,767]
+ CRUSH rule 1 x 327 [125,289,738,408,456,784,750,669,296,314]
+ CRUSH rule 1 x 328 [807,383,476,583,645,141,33,806,181,597]
+ CRUSH rule 1 x 329 [588,938,599,432,446,840,516,713,223,395]
+ CRUSH rule 1 x 330 [932,644,41,611,209,406,420,520,395,665]
+ CRUSH rule 1 x 331 [341,953,950,537,578,862,624,649,626,928]
+ CRUSH rule 1 x 332 [153,726,459,950,466,804,644,821,238,85]
+ CRUSH rule 1 x 333 [745,845,853,860,52,615,243,633,309,616]
+ CRUSH rule 1 x 334 [614,751,807,58,396,159,408,175,189,500]
+ CRUSH rule 1 x 335 [518,721,221,283,454,187,635,367,997,819]
+ CRUSH rule 1 x 336 [389,424,77,309,5,898,698,533,683,851]
+ CRUSH rule 1 x 337 [753,508,765,720,221,807,956,907,464,39]
+ CRUSH rule 1 x 338 [128,810,490,753,406,760,69,11,624,272]
+ CRUSH rule 1 x 339 [430,308,58,751,856,823,607,953,125,899]
+ CRUSH rule 1 x 340 [541,44,630,231,289,966,707,328,325,81]
+ CRUSH rule 1 x 341 [402,26,631,439,165,928,720,503,209,748]
+ CRUSH rule 1 x 342 [982,57,992,461,131,32,516,661,985,860]
+ CRUSH rule 1 x 343 [833,412,572,732,107,805,660,655,149,994]
+ CRUSH rule 1 x 344 [784,533,792,41,642,869,142,114,108,961]
+ CRUSH rule 1 x 345 [546,300,304,691,763,556,127,732,290,494]
+ CRUSH rule 1 x 346 [302,420,428,891,357,124,419,962,304,12]
+ CRUSH rule 1 x 347 [488,778,101,217,366,442,783,661,622,426]
+ CRUSH rule 1 x 348 [903,744,937,718,85,314,862,513,112,334]
+ CRUSH rule 1 x 349 [471,547,582,306,600,486,795,143,529,765]
+ CRUSH rule 1 x 350 [348,221,823,335,383,708,841,164,765,563]
+ CRUSH rule 1 x 351 [961,582,705,346,361,32,766,775,518,155]
+ CRUSH rule 1 x 352 [728,137,461,298,36,903,899,665,802,620]
+ CRUSH rule 1 x 353 [904,202,184,447,58,294,279,616,892,262]
+ CRUSH rule 1 x 354 [345,226,319,256,544,311,612,33,122,190]
+ CRUSH rule 1 x 355 [50,430,175,43,187,458,985,412,599,375]
+ CRUSH rule 1 x 356 [87,185,55,423,829,1,629,228,150,889]
+ CRUSH rule 1 x 357 [762,459,921,473,182,231,891,656,196,232]
+ CRUSH rule 1 x 358 [908,25,280,6,808,676,874,643,550,633]
+ CRUSH rule 1 x 359 [484,15,132,121,394,423,397,52,702,981]
+ CRUSH rule 1 x 360 [173,378,337,702,145,499,29,529,156,595]
+ CRUSH rule 1 x 361 [404,577,115,25,56,914,643,286,552,985]
+ CRUSH rule 1 x 362 [403,1,422,945,132,685,265,35,662,708]
+ CRUSH rule 1 x 363 [639,911,510,162,418,294,444,613,466,499]
+ CRUSH rule 1 x 364 [752,689,610,990,665,222,203,17,743,570]
+ CRUSH rule 1 x 365 [956,999,212,230,624,84,113,373,426,941]
+ CRUSH rule 1 x 366 [860,925,924,763,687,851,59,914,521,629]
+ CRUSH rule 1 x 367 [205,609,647,665,969,720,685,641,894,813]
+ CRUSH rule 1 x 368 [301,284,810,169,78,340,616,93,283,353]
+ CRUSH rule 1 x 369 [452,658,339,217,674,210,284,184,718,684]
+ CRUSH rule 1 x 370 [11,467,695,989,394,576,850,419,307,965]
+ CRUSH rule 1 x 371 [124,487,55,514,313,411,797,547,778,958]
+ CRUSH rule 1 x 372 [253,48,979,846,207,631,212,241,346,153]
+ CRUSH rule 1 x 373 [715,605,775,748,227,493,128,207,88,641]
+ CRUSH rule 1 x 374 [191,887,920,340,223,714,961,760,571,549]
+ CRUSH rule 1 x 375 [711,385,651,665,15,71,934,619,527,735]
+ CRUSH rule 1 x 376 [597,818,49,458,415,755,446,897,460,869]
+ CRUSH rule 1 x 377 [294,256,933,771,184,861,654,487,891,733]
+ CRUSH rule 1 x 378 [34,151,681,707,552,127,728,860,968,475]
+ CRUSH rule 1 x 379 [869,136,315,378,813,153,115,557,165,292]
+ CRUSH rule 1 x 380 [294,97,575,791,690,482,255,806,429,306]
+ CRUSH rule 1 x 381 [119,710,219,827,328,886,773,496,433,750]
+ CRUSH rule 1 x 382 [69,631,508,706,697,168,276,56,278,772]
+ CRUSH rule 1 x 383 [922,588,589,925,471,601,29,197,822,218]
+ CRUSH rule 1 x 384 [221,945,671,117,857,655,488,435,223,783]
+ CRUSH rule 1 x 385 [561,737,953,723,658,368,910,329,396,482]
+ CRUSH rule 1 x 386 [335,442,788,696,507,716,232,692,742,939]
+ CRUSH rule 1 x 387 [514,43,353,88,100,842,164,934,297,902]
+ CRUSH rule 1 x 388 [587,89,157,996,915,927,474,267,640,53]
+ CRUSH rule 1 x 389 [109,641,255,466,372,563,340,222,74,503]
+ CRUSH rule 1 x 390 [925,149,421,489,599,810,852,196,469,672]
+ CRUSH rule 1 x 391 [267,87,387,527,768,873,886,136,818,516]
+ CRUSH rule 1 x 392 [382,485,370,849,936,636,901,82,695,640]
+ CRUSH rule 1 x 393 [425,721,221,753,268,463,652,543,10,287]
+ CRUSH rule 1 x 394 [898,18,38,793,173,738,15,591,420,525]
+ CRUSH rule 1 x 395 [806,876,269,679,32,744,126,179,607,623]
+ CRUSH rule 1 x 396 [790,970,437,449,875,395,726,935,278,138]
+ CRUSH rule 1 x 397 [136,363,507,613,11,30,996,558,602,528]
+ CRUSH rule 1 x 398 [914,116,558,258,722,904,349,672,826,569]
+ CRUSH rule 1 x 399 [261,94,299,202,174,622,749,410,815,214]
+ CRUSH rule 1 x 400 [661,197,338,461,977,848,536,592,886,981]
+ CRUSH rule 1 x 401 [953,979,287,803,41,349,79,32,343,468]
+ CRUSH rule 1 x 402 [738,819,618,522,667,334,658,449,886,260]
+ CRUSH rule 1 x 403 [573,238,425,546,130,68,202,650,501,628]
+ CRUSH rule 1 x 404 [526,848,790,253,922,820,299,577,563,37]
+ CRUSH rule 1 x 405 [582,505,330,334,201,110,776,296,19,972]
+ CRUSH rule 1 x 406 [768,324,493,60,186,165,718,578,580,249]
+ CRUSH rule 1 x 407 [260,951,437,587,692,648,72,345,709,89]
+ CRUSH rule 1 x 408 [657,81,770,734,830,821,246,695,76,647]
+ CRUSH rule 1 x 409 [498,89,182,423,672,152,213,806,168,907]
+ CRUSH rule 1 x 410 [28,793,737,352,166,645,949,507,361,615]
+ CRUSH rule 1 x 411 [684,992,60,659,769,267,313,351,497,571]
+ CRUSH rule 1 x 412 [261,958,699,950,165,14,560,155,661,678]
+ CRUSH rule 1 x 413 [891,835,297,441,384,979,618,907,9,291]
+ CRUSH rule 1 x 414 [127,459,119,965,662,594,97,124,229,641]
+ CRUSH rule 1 x 415 [272,540,631,328,609,568,694,332,572,681]
+ CRUSH rule 1 x 416 [739,617,115,530,339,371,889,344,838,541]
+ CRUSH rule 1 x 417 [106,209,157,878,117,128,138,374,470,59]
+ CRUSH rule 1 x 418 [525,441,147,390,320,300,848,972,781,361]
+ CRUSH rule 1 x 419 [603,673,615,465,266,855,823,884,832,361]
+ CRUSH rule 1 x 420 [988,213,251,226,209,245,506,670,285,2]
+ CRUSH rule 1 x 421 [761,521,748,368,923,992,764,274,623,613]
+ CRUSH rule 1 x 422 [317,160,924,548,198,709,839,547,599,779]
+ CRUSH rule 1 x 423 [137,807,168,472,619,443,905,588,312,114]
+ CRUSH rule 1 x 424 [920,37,146,263,598,748,785,395,884,360]
+ CRUSH rule 1 x 425 [277,693,285,221,478,165,80,236,988,682]
+ CRUSH rule 1 x 426 [485,936,407,854,726,524,791,565,352,949]
+ CRUSH rule 1 x 427 [242,515,9,564,174,453,334,588,571,428]
+ CRUSH rule 1 x 428 [632,635,26,473,494,478,225,94,303,757]
+ CRUSH rule 1 x 429 [641,73,465,127,171,397,857,562,976,977]
+ CRUSH rule 1 x 430 [626,585,6,387,881,583,859,699,91,148]
+ CRUSH rule 1 x 431 [697,76,753,570,964,339,194,366,279,30]
+ CRUSH rule 1 x 432 [590,526,306,283,656,728,513,591,599,474]
+ CRUSH rule 1 x 433 [284,387,149,817,886,714,52,897,705,256]
+ CRUSH rule 1 x 434 [538,985,79,953,770,468,644,646,747,123]
+ CRUSH rule 1 x 435 [30,318,593,635,975,833,371,731,906,721]
+ CRUSH rule 1 x 436 [164,919,851,693,0,874,10,976,284,126]
+ CRUSH rule 1 x 437 [322,212,163,606,302,282,443,23,696,245]
+ CRUSH rule 1 x 438 [142,392,85,594,376,419,755,841,94,52]
+ CRUSH rule 1 x 439 [119,370,68,443,997,837,414,152,331,985]
+ CRUSH rule 1 x 440 [333,403,187,863,475,844,800,174,117,518]
+ CRUSH rule 1 x 441 [477,727,906,145,429,91,205,236,86,929]
+ CRUSH rule 1 x 442 [274,590,933,244,434,49,864,799,762,611]
+ CRUSH rule 1 x 443 [983,748,574,718,700,442,774,350,37,929]
+ CRUSH rule 1 x 444 [536,509,431,146,170,149,182,145,347,172]
+ CRUSH rule 1 x 445 [485,311,528,209,964,753,554,931,638,892]
+ CRUSH rule 1 x 446 [345,634,42,294,711,376,314,714,212,646]
+ CRUSH rule 1 x 447 [61,845,767,600,321,716,58,531,827,968]
+ CRUSH rule 1 x 448 [333,232,292,846,364,951,807,688,21,841]
+ CRUSH rule 1 x 449 [680,16,484,670,851,500,258,548,905,988]
+ CRUSH rule 1 x 450 [235,214,79,423,96,822,721,31,312,491]
+ CRUSH rule 1 x 451 [961,468,333,640,823,151,878,33,3,917]
+ CRUSH rule 1 x 452 [525,479,153,528,570,806,604,49,922,414]
+ CRUSH rule 1 x 453 [138,466,302,86,249,154,514,5,494,960]
+ CRUSH rule 1 x 454 [137,625,215,402,389,914,106,103,511,624]
+ CRUSH rule 1 x 455 [173,150,997,16,846,888,295,967,132,319]
+ CRUSH rule 1 x 456 [235,226,238,258,347,784,504,96,890,230]
+ CRUSH rule 1 x 457 [450,577,253,413,717,609,762,975,485,228]
+ CRUSH rule 1 x 458 [195,537,91,814,351,90,399,558,15,441]
+ CRUSH rule 1 x 459 [381,555,312,573,915,623,147,483,517,733]
+ CRUSH rule 1 x 460 [972,730,534,678,756,692,841,512,70,914]
+ CRUSH rule 1 x 461 [506,279,142,830,784,124,385,797,917,561]
+ CRUSH rule 1 x 462 [692,959,578,57,983,299,240,911,375,412]
+ CRUSH rule 1 x 463 [788,667,949,550,685,702,538,111,232,539]
+ CRUSH rule 1 x 464 [133,122,588,999,270,880,789,0,653,566]
+ CRUSH rule 1 x 465 [971,190,230,777,452,914,137,466,531,493]
+ CRUSH rule 1 x 466 [394,576,148,157,103,822,659,35,797,235]
+ CRUSH rule 1 x 467 [517,28,366,362,984,521,187,640,601,622]
+ CRUSH rule 1 x 468 [829,143,874,225,162,413,201,249,555,646]
+ CRUSH rule 1 x 469 [987,936,106,725,633,238,681,851,551,768]
+ CRUSH rule 1 x 470 [107,982,56,889,67,65,558,71,676,655]
+ CRUSH rule 1 x 471 [181,897,629,860,307,116,256,978,409,691]
+ CRUSH rule 1 x 472 [547,512,172,24,705,837,809,56,476,137]
+ CRUSH rule 1 x 473 [760,997,824,905,888,755,756,663,167,196]
+ CRUSH rule 1 x 474 [787,418,743,628,272,341,446,333,245,689]
+ CRUSH rule 1 x 475 [662,312,253,617,105,58,237,764,682,318]
+ CRUSH rule 1 x 476 [110,495,185,508,961,837,984,226,333,916]
+ CRUSH rule 1 x 477 [393,954,834,132,841,367,753,794,237,996]
+ CRUSH rule 1 x 478 [246,483,480,644,985,420,941,843,751,451]
+ CRUSH rule 1 x 479 [70,929,697,931,744,487,158,489,515,496]
+ CRUSH rule 1 x 480 [753,119,961,607,317,717,371,807,687,932]
+ CRUSH rule 1 x 481 [470,429,677,242,574,757,135,375,613,657]
+ CRUSH rule 1 x 482 [451,566,961,675,354,746,731,233,640,492]
+ CRUSH rule 1 x 483 [816,72,371,278,635,30,448,437,219,982]
+ CRUSH rule 1 x 484 [540,454,389,31,654,494,283,170,278,77]
+ CRUSH rule 1 x 485 [74,582,624,684,566,677,866,661,581,943]
+ CRUSH rule 1 x 486 [958,595,199,763,715,973,621,955,400,261]
+ CRUSH rule 1 x 487 [228,302,804,833,876,647,857,782,24,970]
+ CRUSH rule 1 x 488 [180,529,722,956,353,890,924,965,25,925]
+ CRUSH rule 1 x 489 [47,617,812,187,291,828,154,478,512,528]
+ CRUSH rule 1 x 490 [905,822,479,124,750,843,566,779,936,507]
+ CRUSH rule 1 x 491 [892,370,609,998,433,957,188,563,490,369]
+ CRUSH rule 1 x 492 [588,959,127,948,505,936,591,423,668,365]
+ CRUSH rule 1 x 493 [353,461,593,291,301,830,231,893,474,946]
+ CRUSH rule 1 x 494 [378,848,443,368,507,423,389,819,956,597]
+ CRUSH rule 1 x 495 [845,653,768,234,405,367,823,789,217,720]
+ CRUSH rule 1 x 496 [13,988,0,691,389,757,129,763,39,651]
+ CRUSH rule 1 x 497 [796,877,788,394,648,829,542,745,131,753]
+ CRUSH rule 1 x 498 [412,337,270,705,511,227,949,173,398,586]
+ CRUSH rule 1 x 499 [330,695,8,74,618,101,440,509,295,921]
+ CRUSH rule 1 x 500 [820,272,547,765,755,96,930,573,357,491]
+ CRUSH rule 1 x 501 [110,44,132,442,294,423,880,279,616,919]
+ CRUSH rule 1 x 502 [336,595,650,274,993,312,490,852,962,387]
+ CRUSH rule 1 x 503 [922,211,157,722,502,971,262,926,316,527]
+ CRUSH rule 1 x 504 [483,52,122,432,778,461,758,104,831,710]
+ CRUSH rule 1 x 505 [482,598,224,279,480,310,764,558,891,406]
+ CRUSH rule 1 x 506 [493,123,43,856,936,622,898,161,78,414]
+ CRUSH rule 1 x 507 [12,598,264,422,416,947,591,702,346,619]
+ CRUSH rule 1 x 508 [227,157,611,301,223,746,313,282,207,626]
+ CRUSH rule 1 x 509 [807,242,363,122,582,530,798,808,139,377]
+ CRUSH rule 1 x 510 [134,437,227,75,313,351,786,152,921,884]
+ CRUSH rule 1 x 511 [212,54,83,799,457,218,600,968,355,109]
+ CRUSH rule 1 x 512 [236,630,758,752,361,249,899,451,415,920]
+ CRUSH rule 1 x 513 [994,693,644,938,846,685,52,185,197,986]
+ CRUSH rule 1 x 514 [45,508,831,19,817,52,374,985,944,101]
+ CRUSH rule 1 x 515 [504,138,480,272,530,377,481,820,517,850]
+ CRUSH rule 1 x 516 [285,409,136,570,841,610,453,660,93,134]
+ CRUSH rule 1 x 517 [300,232,23,906,438,236,519,737,20,892]
+ CRUSH rule 1 x 518 [397,674,98,898,967,113,625,434,527,630]
+ CRUSH rule 1 x 519 [86,750,772,913,101,864,375,328,3,688]
+ CRUSH rule 1 x 520 [900,833,614,130,261,885,558,956,664,468]
+ CRUSH rule 1 x 521 [31,47,236,751,911,599,495,354,665,945]
+ CRUSH rule 1 x 522 [390,16,280,144,291,175,753,624,769,853]
+ CRUSH rule 1 x 523 [618,308,424,590,300,206,834,212,906,305]
+ CRUSH rule 1 x 524 [635,189,687,963,601,518,8,550,769,975]
+ CRUSH rule 1 x 525 [311,916,699,262,775,32,45,478,911,233]
+ CRUSH rule 1 x 526 [48,738,227,718,244,942,853,643,625,43]
+ CRUSH rule 1 x 527 [202,851,889,216,763,351,270,35,809,509]
+ CRUSH rule 1 x 528 [565,827,590,273,918,106,651,368,118,1]
+ CRUSH rule 1 x 529 [934,864,241,43,466,924,278,926,280,321]
+ CRUSH rule 1 x 530 [502,934,298,670,986,360,577,509,195,722]
+ CRUSH rule 1 x 531 [681,627,942,487,288,561,925,474,669,212]
+ CRUSH rule 1 x 532 [422,6,147,205,861,141,949,374,988,367]
+ CRUSH rule 1 x 533 [863,68,364,983,247,199,54,931,4,279]
+ CRUSH rule 1 x 534 [962,931,775,172,663,119,206,682,627,827]
+ CRUSH rule 1 x 535 [89,565,397,693,839,632,859,30,61,75]
+ CRUSH rule 1 x 536 [499,351,760,458,918,86,148,668,436,192]
+ CRUSH rule 1 x 537 [676,547,787,311,867,748,152,797,492,926]
+ CRUSH rule 1 x 538 [58,644,571,649,941,7,37,485,88,273]
+ CRUSH rule 1 x 539 [837,953,457,711,458,621,528,722,59,237]
+ CRUSH rule 1 x 540 [831,50,132,213,197,709,95,789,348,342]
+ CRUSH rule 1 x 541 [582,757,121,525,532,963,738,277,225,142]
+ CRUSH rule 1 x 542 [472,132,790,997,948,269,137,934,547,351]
+ CRUSH rule 1 x 543 [382,272,797,330,315,748,324,134,839,685]
+ CRUSH rule 1 x 544 [947,930,496,883,509,219,250,362,614,123]
+ CRUSH rule 1 x 545 [425,570,305,77,821,422,117,172,764,372]
+ CRUSH rule 1 x 546 [18,65,529,437,343,547,699,610,785,811]
+ CRUSH rule 1 x 547 [445,715,600,472,213,851,428,267,229,379]
+ CRUSH rule 1 x 548 [367,569,980,167,627,442,517,684,154,108]
+ CRUSH rule 1 x 549 [125,715,671,817,285,420,37,639,934,330]
+ CRUSH rule 1 x 550 [425,599,744,199,923,222,915,570,546,724]
+ CRUSH rule 1 x 551 [44,1,528,922,944,115,161,901,342,941]
+ CRUSH rule 1 x 552 [246,104,68,239,123,427,57,217,21,70]
+ CRUSH rule 1 x 553 [71,703,615,28,593,724,218,916,561,416]
+ CRUSH rule 1 x 554 [207,124,217,166,525,226,693,953,606,894]
+ CRUSH rule 1 x 555 [570,28,317,420,931,413,623,659,403,573]
+ CRUSH rule 1 x 556 [674,152,421,79,215,347,830,762,691,951]
+ CRUSH rule 1 x 557 [347,817,191,391,741,571,593,267,17,386]
+ CRUSH rule 1 x 558 [627,426,369,692,815,371,124,107,766,260]
+ CRUSH rule 1 x 559 [940,630,924,242,224,912,185,356,87,113]
+ CRUSH rule 1 x 560 [295,903,541,29,245,753,887,376,658,366]
+ CRUSH rule 1 x 561 [506,682,384,637,878,991,700,339,687,507]
+ CRUSH rule 1 x 562 [718,529,87,729,842,341,62,817,766,376]
+ CRUSH rule 1 x 563 [552,332,747,206,274,871,903,900,812,290]
+ CRUSH rule 1 x 564 [835,769,736,486,630,209,641,751,930,856]
+ CRUSH rule 1 x 565 [8,167,539,182,607,62,738,873,47,84]
+ CRUSH rule 1 x 566 [600,481,301,263,90,450,184,127,448,327]
+ CRUSH rule 1 x 567 [999,994,509,899,947,24,267,639,646,85]
+ CRUSH rule 1 x 568 [252,431,157,62,601,863,398,521,59,250]
+ CRUSH rule 1 x 569 [643,218,943,455,83,969,494,624,352,562]
+ CRUSH rule 1 x 570 [617,635,765,422,250,156,533,674,23,683]
+ CRUSH rule 1 x 571 [757,80,59,98,328,700,329,848,235,502]
+ CRUSH rule 1 x 572 [299,348,575,889,943,675,33,312,202,355]
+ CRUSH rule 1 x 573 [25,505,270,167,58,901,878,978,1,291]
+ CRUSH rule 1 x 574 [215,431,624,177,628,814,333,841,193,146]
+ CRUSH rule 1 x 575 [225,252,611,546,32,815,389,486,10,402]
+ CRUSH rule 1 x 576 [627,94,159,857,430,691,177,545,839,722]
+ CRUSH rule 1 x 577 [237,809,778,636,61,167,700,521,825,444]
+ CRUSH rule 1 x 578 [885,313,120,344,771,614,487,976,977,58]
+ CRUSH rule 1 x 579 [924,575,787,831,47,996,557,630,468,348]
+ CRUSH rule 1 x 580 [718,51,766,121,118,471,608,755,326,604]
+ CRUSH rule 1 x 581 [219,807,129,571,856,179,874,902,958,415]
+ CRUSH rule 1 x 582 [893,701,598,863,285,829,984,622,175,804]
+ CRUSH rule 1 x 583 [246,930,964,170,993,409,469,193,737,681]
+ CRUSH rule 1 x 584 [336,432,680,175,495,839,642,226,122,703]
+ CRUSH rule 1 x 585 [324,999,397,485,457,527,73,628,884,255]
+ CRUSH rule 1 x 586 [558,230,976,541,816,72,794,682,127,372]
+ CRUSH rule 1 x 587 [985,830,597,21,308,890,952,421,875,65]
+ CRUSH rule 1 x 588 [211,544,57,134,162,496,195,581,649,488]
+ CRUSH rule 1 x 589 [129,21,112,190,885,844,753,180,160,465]
+ CRUSH rule 1 x 590 [467,969,652,593,287,76,811,413,436,162]
+ CRUSH rule 1 x 591 [758,514,316,164,35,110,54,796,369,958]
+ CRUSH rule 1 x 592 [525,253,190,443,315,603,667,318,496,74]
+ CRUSH rule 1 x 593 [601,885,339,152,297,223,269,455,168,635]
+ CRUSH rule 1 x 594 [227,60,450,30,717,840,994,16,777,901]
+ CRUSH rule 1 x 595 [720,854,496,912,80,655,917,525,945,715]
+ CRUSH rule 1 x 596 [751,195,997,77,261,490,180,482,449,647]
+ CRUSH rule 1 x 597 [129,574,714,8,789,847,725,991,955,316]
+ CRUSH rule 1 x 598 [679,207,604,396,841,284,286,280,507,912]
+ CRUSH rule 1 x 599 [668,315,683,349,681,253,599,364,546,849]
+ CRUSH rule 1 x 600 [143,396,464,444,59,57,243,264,31,897]
+ CRUSH rule 1 x 601 [326,573,873,902,136,921,633,596,988,727]
+ CRUSH rule 1 x 602 [860,281,875,535,672,474,697,763,442,542]
+ CRUSH rule 1 x 603 [709,328,445,349,190,455,924,667,356,316]
+ CRUSH rule 1 x 604 [571,62,814,95,866,978,983,281,292,953]
+ CRUSH rule 1 x 605 [252,739,860,27,313,362,857,899,349,926]
+ CRUSH rule 1 x 606 [339,236,759,842,67,644,954,94,88,617]
+ CRUSH rule 1 x 607 [590,248,759,868,433,398,578,386,226,269]
+ CRUSH rule 1 x 608 [145,635,309,467,875,115,148,33,420,669]
+ CRUSH rule 1 x 609 [973,547,223,79,762,863,249,41,778,929]
+ CRUSH rule 1 x 610 [435,816,961,983,255,886,160,888,597,767]
+ CRUSH rule 1 x 611 [559,283,422,584,176,429,570,43,362,401]
+ CRUSH rule 1 x 612 [273,149,123,576,911,270,296,735,245,714]
+ CRUSH rule 1 x 613 [828,614,642,674,33,361,958,580,197,897]
+ CRUSH rule 1 x 614 [478,748,393,34,171,80,92,12,62,719]
+ CRUSH rule 1 x 615 [392,155,144,326,626,134,149,401,14,59]
+ CRUSH rule 1 x 616 [778,637,452,248,15,888,74,307,976,613]
+ CRUSH rule 1 x 617 [622,713,996,833,611,407,364,8,342,512]
+ CRUSH rule 1 x 618 [149,877,270,329,180,327,222,749,697,853]
+ CRUSH rule 1 x 619 [604,163,656,409,322,848,519,967,737,892]
+ CRUSH rule 1 x 620 [181,23,409,198,64,898,35,620,268,902]
+ CRUSH rule 1 x 621 [735,902,386,237,939,475,725,118,875,359]
+ CRUSH rule 1 x 622 [661,824,717,568,858,583,446,798,869,586]
+ CRUSH rule 1 x 623 [142,121,643,61,695,852,485,478,185,854]
+ CRUSH rule 1 x 624 [360,716,420,398,49,717,137,140,488,725]
+ CRUSH rule 1 x 625 [541,167,385,1,601,481,308,111,207,48]
+ CRUSH rule 1 x 626 [364,431,610,363,535,747,225,841,868,249]
+ CRUSH rule 1 x 627 [458,137,557,410,287,749,467,432,944,781]
+ CRUSH rule 1 x 628 [250,350,556,497,821,65,205,580,972,427]
+ CRUSH rule 1 x 629 [928,160,710,572,365,772,538,46,300,112]
+ CRUSH rule 1 x 630 [243,19,918,556,601,16,920,830,171,759]
+ CRUSH rule 1 x 631 [438,221,574,676,797,580,219,211,157,614]
+ CRUSH rule 1 x 632 [797,368,247,5,32,102,416,45,624,253]
+ CRUSH rule 1 x 633 [993,749,525,485,27,330,275,599,219,357]
+ CRUSH rule 1 x 634 [239,351,633,299,651,678,296,337,676,416]
+ CRUSH rule 1 x 635 [640,965,25,961,306,172,849,357,317,599]
+ CRUSH rule 1 x 636 [173,290,297,991,937,823,236,318,228,575]
+ CRUSH rule 1 x 637 [0,918,98,108,111,495,887,57,16,319]
+ CRUSH rule 1 x 638 [702,235,424,900,983,754,701,887,355,632]
+ CRUSH rule 1 x 639 [475,687,31,785,918,611,27,214,226,515]
+ CRUSH rule 1 x 640 [31,664,399,677,123,609,858,138,726,1]
+ CRUSH rule 1 x 641 [296,473,108,963,341,876,897,449,42,193]
+ CRUSH rule 1 x 642 [894,273,427,606,677,670,610,665,299,852]
+ CRUSH rule 1 x 643 [117,111,732,191,114,153,500,631,833,439]
+ CRUSH rule 1 x 644 [438,336,327,512,599,862,660,857,123,910]
+ CRUSH rule 1 x 645 [982,702,351,573,907,915,279,317,414,917]
+ CRUSH rule 1 x 646 [334,804,146,842,697,638,720,135,369,711]
+ CRUSH rule 1 x 647 [933,787,185,334,752,285,372,890,30,747]
+ CRUSH rule 1 x 648 [22,444,400,862,207,842,453,732,262,803]
+ CRUSH rule 1 x 649 [503,229,213,460,639,760,722,748,599,556]
+ CRUSH rule 1 x 650 [328,659,420,443,739,950,869,150,743,438]
+ CRUSH rule 1 x 651 [3,880,823,123,378,585,715,221,31,92]
+ CRUSH rule 1 x 652 [495,977,563,733,92,997,119,818,459,782]
+ CRUSH rule 1 x 653 [185,718,804,280,975,912,198,291,71,792]
+ CRUSH rule 1 x 654 [130,528,380,81,906,511,773,506,546,266]
+ CRUSH rule 1 x 655 [560,872,454,504,319,284,605,214,833,862]
+ CRUSH rule 1 x 656 [219,885,178,981,863,508,708,6,746,734]
+ CRUSH rule 1 x 657 [233,684,813,490,208,941,858,16,128,144]
+ CRUSH rule 1 x 658 [778,6,756,380,750,836,547,850,499,125]
+ CRUSH rule 1 x 659 [240,663,306,540,789,902,170,954,22,394]
+ CRUSH rule 1 x 660 [244,855,196,147,678,323,63,859,215,171]
+ CRUSH rule 1 x 661 [184,270,128,398,910,230,402,205,609,831]
+ CRUSH rule 1 x 662 [65,883,921,438,79,957,464,902,276,289]
+ CRUSH rule 1 x 663 [323,721,594,812,43,992,170,65,906,943]
+ CRUSH rule 1 x 664 [865,113,512,51,427,123,585,260,254,209]
+ CRUSH rule 1 x 665 [420,850,591,475,202,733,798,658,28,334]
+ CRUSH rule 1 x 666 [319,767,246,3,369,493,796,56,736,0]
+ CRUSH rule 1 x 667 [875,39,343,100,829,2,795,783,386,956]
+ CRUSH rule 1 x 668 [331,122,263,599,355,484,943,554,395,713]
+ CRUSH rule 1 x 669 [915,521,402,747,673,445,938,600,517,49]
+ CRUSH rule 1 x 670 [845,659,943,447,401,322,168,302,681,978]
+ CRUSH rule 1 x 671 [108,634,527,363,856,238,755,330,584,525]
+ CRUSH rule 1 x 672 [578,216,110,589,302,137,954,315,735,751]
+ CRUSH rule 1 x 673 [442,74,579,797,622,950,371,402,725,870]
+ CRUSH rule 1 x 674 [588,364,281,308,645,631,229,506,565,362]
+ CRUSH rule 1 x 675 [489,698,744,671,870,174,528,875,982,782]
+ CRUSH rule 1 x 676 [928,911,40,180,722,729,673,569,701,403]
+ CRUSH rule 1 x 677 [399,269,692,131,615,136,103,763,527,83]
+ CRUSH rule 1 x 678 [546,752,544,155,5,463,666,352,576,959]
+ CRUSH rule 1 x 679 [988,25,275,433,628,57,247,620,437,29]
+ CRUSH rule 1 x 680 [335,963,382,486,749,257,795,347,831,761]
+ CRUSH rule 1 x 681 [690,462,623,466,49,471,774,192,454,380]
+ CRUSH rule 1 x 682 [196,588,154,257,807,776,367,718,345,677]
+ CRUSH rule 1 x 683 [627,25,421,160,873,102,345,599,30,892]
+ CRUSH rule 1 x 684 [38,804,592,158,991,264,652,821,641,757]
+ CRUSH rule 1 x 685 [841,368,548,362,166,211,154,121,937,804]
+ CRUSH rule 1 x 686 [336,287,525,440,166,993,911,638,690,393]
+ CRUSH rule 1 x 687 [20,682,924,653,356,16,917,622,156,826]
+ CRUSH rule 1 x 688 [463,371,780,556,385,883,115,248,566,11]
+ CRUSH rule 1 x 689 [569,250,78,816,847,775,333,161,74,907]
+ CRUSH rule 1 x 690 [551,144,587,263,378,394,970,639,835,238]
+ CRUSH rule 1 x 691 [766,464,446,533,449,541,451,290,789,853]
+ CRUSH rule 1 x 692 [739,634,18,245,624,35,268,525,425,499]
+ CRUSH rule 1 x 693 [339,297,118,330,817,91,828,276,264,237]
+ CRUSH rule 1 x 694 [405,26,830,181,533,166,488,804,501,885]
+ CRUSH rule 1 x 695 [622,576,597,535,600,593,300,989,804,72]
+ CRUSH rule 1 x 696 [558,902,689,13,715,28,664,489,598,261]
+ CRUSH rule 1 x 697 [818,222,406,691,427,863,153,922,986,480]
+ CRUSH rule 1 x 698 [178,48,402,233,841,604,468,180,783,915]
+ CRUSH rule 1 x 699 [450,244,180,919,276,332,747,453,519,100]
+ CRUSH rule 1 x 700 [502,771,987,706,416,240,68,641,109,182]
+ CRUSH rule 1 x 701 [4,612,782,216,853,303,585,513,907,414]
+ CRUSH rule 1 x 702 [177,630,232,923,281,708,466,687,742,170]
+ CRUSH rule 1 x 703 [354,178,389,393,778,803,796,607,894,1]
+ CRUSH rule 1 x 704 [646,601,156,171,603,116,655,595,888,354]
+ CRUSH rule 1 x 705 [921,401,890,265,244,690,372,253,807,28]
+ CRUSH rule 1 x 706 [652,877,562,452,26,323,923,770,516,982]
+ CRUSH rule 1 x 707 [345,745,67,716,789,576,2,133,256,374]
+ CRUSH rule 1 x 708 [333,607,180,469,170,555,939,331,41,175]
+ CRUSH rule 1 x 709 [45,187,302,115,896,579,733,607,763,845]
+ CRUSH rule 1 x 710 [94,855,43,199,18,948,449,28,731,573]
+ CRUSH rule 1 x 711 [227,653,731,150,380,842,534,110,639,452]
+ CRUSH rule 1 x 712 [398,953,136,870,181,408,895,459,341,833]
+ CRUSH rule 1 x 713 [116,800,503,662,635,579,53,839,56,829]
+ CRUSH rule 1 x 714 [111,629,866,709,902,557,875,649,23,79]
+ CRUSH rule 1 x 715 [531,291,486,382,192,807,322,417,973,582]
+ CRUSH rule 1 x 716 [169,541,291,42,343,724,138,197,32,415]
+ CRUSH rule 1 x 717 [417,446,994,894,239,494,237,62,327,958]
+ CRUSH rule 1 x 718 [992,383,298,844,377,463,544,891,210,370]
+ CRUSH rule 1 x 719 [936,674,324,759,194,409,828,975,119,87]
+ CRUSH rule 1 x 720 [370,188,174,464,644,218,214,76,870,779]
+ CRUSH rule 1 x 721 [320,859,278,259,170,957,177,264,867,327]
+ CRUSH rule 1 x 722 [7,2,673,129,96,445,823,833,1,774]
+ CRUSH rule 1 x 723 [270,553,831,662,38,101,985,846,77,467]
+ CRUSH rule 1 x 724 [666,822,708,895,633,800,616,879,480,309]
+ CRUSH rule 1 x 725 [794,406,875,459,981,751,359,687,720,128]
+ CRUSH rule 1 x 726 [420,556,341,292,240,68,966,535,669,74]
+ CRUSH rule 1 x 727 [561,461,129,635,965,610,105,31,506,430]
+ CRUSH rule 1 x 728 [951,330,196,756,589,849,753,760,254,379]
+ CRUSH rule 1 x 729 [656,644,436,591,27,119,572,933,434,816]
+ CRUSH rule 1 x 730 [3,558,629,184,50,765,760,800,945,743]
+ CRUSH rule 1 x 731 [852,89,75,735,713,113,528,890,625,535]
+ CRUSH rule 1 x 732 [983,840,869,976,697,307,368,271,778,172]
+ CRUSH rule 1 x 733 [285,396,388,122,387,364,880,343,590,539]
+ CRUSH rule 1 x 734 [125,510,402,640,676,501,535,627,224,790]
+ CRUSH rule 1 x 735 [417,773,686,504,459,912,690,59,294,569]
+ CRUSH rule 1 x 736 [749,396,632,550,779,109,845,278,559,613]
+ CRUSH rule 1 x 737 [644,991,946,135,448,903,482,564,259,896]
+ CRUSH rule 1 x 738 [449,683,290,220,245,525,429,397,872,716]
+ CRUSH rule 1 x 739 [341,220,641,454,740,661,146,17,314,156]
+ CRUSH rule 1 x 740 [874,524,674,650,472,282,214,494,593,155]
+ CRUSH rule 1 x 741 [189,472,712,798,715,757,863,571,876,528]
+ CRUSH rule 1 x 742 [912,581,114,447,730,21,687,81,145,695]
+ CRUSH rule 1 x 743 [654,914,425,441,763,39,451,631,911,829]
+ CRUSH rule 1 x 744 [725,295,579,377,162,447,843,699,24,714]
+ CRUSH rule 1 x 745 [787,858,850,506,612,735,926,314,771,910]
+ CRUSH rule 1 x 746 [757,848,704,30,47,940,450,651,105,921]
+ CRUSH rule 1 x 747 [700,81,867,681,801,64,879,857,727,565]
+ CRUSH rule 1 x 748 [557,436,238,664,293,865,304,999,685,843]
+ CRUSH rule 1 x 749 [772,622,337,42,156,302,383,506,570,828]
+ CRUSH rule 1 x 750 [946,97,376,677,316,670,169,171,9,58]
+ CRUSH rule 1 x 751 [996,618,343,911,83,22,388,17,892,537]
+ CRUSH rule 1 x 752 [746,887,695,868,610,950,88,315,728,669]
+ CRUSH rule 1 x 753 [741,14,463,479,172,192,481,702,431,675]
+ CRUSH rule 1 x 754 [648,349,333,355,65,63,336,724,262,61]
+ CRUSH rule 1 x 755 [157,460,466,187,959,674,192,279,371,970]
+ CRUSH rule 1 x 756 [416,97,197,497,227,3,850,191,991,63]
+ CRUSH rule 1 x 757 [599,839,776,410,256,823,121,690,544,28]
+ CRUSH rule 1 x 758 [994,218,620,256,361,749,165,686,449,831]
+ CRUSH rule 1 x 759 [959,682,514,745,100,519,15,347,311,752]
+ CRUSH rule 1 x 760 [518,943,215,83,706,137,345,69,39,199]
+ CRUSH rule 1 x 761 [285,849,420,324,987,338,373,361,684,654]
+ CRUSH rule 1 x 762 [591,313,41,335,110,696,664,350,339,980]
+ CRUSH rule 1 x 763 [908,411,200,740,292,295,387,775,797,990]
+ CRUSH rule 1 x 764 [787,234,894,485,883,711,70,202,557,471]
+ CRUSH rule 1 x 765 [327,921,882,393,444,792,402,123,902,592]
+ CRUSH rule 1 x 766 [84,161,878,704,416,144,357,310,890,850]
+ CRUSH rule 1 x 767 [370,895,702,701,890,2,251,951,675,322]
+ CRUSH rule 1 x 768 [826,760,879,864,460,474,645,975,947,199]
+ CRUSH rule 1 x 769 [67,768,663,735,814,66,213,527,546,42]
+ CRUSH rule 1 x 770 [593,909,482,259,5,550,961,324,309,772]
+ CRUSH rule 1 x 771 [309,935,121,578,937,685,933,571,822,256]
+ CRUSH rule 1 x 772 [12,125,797,301,348,419,891,959,487,355]
+ CRUSH rule 1 x 773 [253,466,820,549,591,193,783,951,982,160]
+ CRUSH rule 1 x 774 [164,390,705,109,881,505,890,425,599,485]
+ CRUSH rule 1 x 775 [703,47,43,973,643,406,885,976,936,221]
+ CRUSH rule 1 x 776 [728,231,80,916,2,850,396,76,680,108]
+ CRUSH rule 1 x 777 [981,621,568,729,869,952,563,860,388,456]
+ CRUSH rule 1 x 778 [411,456,544,597,789,784,65,954,125,358]
+ CRUSH rule 1 x 779 [346,121,519,921,587,48,772,645,254,759]
+ CRUSH rule 1 x 780 [476,39,288,381,303,29,17,336,147,829]
+ CRUSH rule 1 x 781 [10,130,585,844,729,705,714,954,271,58]
+ CRUSH rule 1 x 782 [462,246,581,902,623,877,812,516,774,985]
+ CRUSH rule 1 x 783 [580,373,153,775,668,661,626,961,576,119]
+ CRUSH rule 1 x 784 [413,113,978,990,994,56,481,198,171,944]
+ CRUSH rule 1 x 785 [341,856,332,354,59,581,632,151,586,360]
+ CRUSH rule 1 x 786 [411,140,313,393,215,618,490,481,627,740]
+ CRUSH rule 1 x 787 [605,522,211,813,636,224,600,528,966,556]
+ CRUSH rule 1 x 788 [226,545,35,142,726,851,194,216,486,782]
+ CRUSH rule 1 x 789 [545,320,414,702,731,277,237,916,374,670]
+ CRUSH rule 1 x 790 [414,748,816,327,130,115,788,164,691,329]
+ CRUSH rule 1 x 791 [660,906,406,697,916,322,124,401,742,990]
+ CRUSH rule 1 x 792 [287,392,514,204,75,789,406,858,694,351]
+ CRUSH rule 1 x 793 [631,133,850,713,720,487,376,812,886,264]
+ CRUSH rule 1 x 794 [931,517,543,210,963,898,811,459,344,719]
+ CRUSH rule 1 x 795 [551,962,477,948,425,434,268,94,648,402]
+ CRUSH rule 1 x 796 [814,4,95,27,368,300,646,451,67,738]
+ CRUSH rule 1 x 797 [64,201,299,734,605,864,596,196,93,636]
+ CRUSH rule 1 x 798 [422,530,114,431,565,716,473,250,839,895]
+ CRUSH rule 1 x 799 [824,32,679,562,266,549,859,994,831,60]
+ CRUSH rule 1 x 800 [862,623,489,637,861,196,941,643,398,325]
+ CRUSH rule 1 x 801 [145,550,329,324,734,160,219,662,142,28]
+ CRUSH rule 1 x 802 [570,19,847,308,387,518,846,53,783,511]
+ CRUSH rule 1 x 803 [151,812,662,358,880,349,834,881,23,229]
+ CRUSH rule 1 x 804 [467,93,264,863,176,842,663,949,380,39]
+ CRUSH rule 1 x 805 [621,223,938,809,591,686,121,157,934,660]
+ CRUSH rule 1 x 806 [898,957,805,430,499,584,640,607,790,832]
+ CRUSH rule 1 x 807 [354,531,422,159,921,431,802,136,305,983]
+ CRUSH rule 1 x 808 [7,96,76,897,446,2,166,929,234,460]
+ CRUSH rule 1 x 809 [70,734,719,56,687,21,23,145,184,465]
+ CRUSH rule 1 x 810 [701,18,972,327,771,649,620,648,433,997]
+ CRUSH rule 1 x 811 [248,547,103,728,901,264,948,202,521,278]
+ CRUSH rule 1 x 812 [230,576,821,566,993,762,675,28,263,410]
+ CRUSH rule 1 x 813 [805,114,683,629,99,462,285,450,948,742]
+ CRUSH rule 1 x 814 [54,619,973,741,497,894,401,266,905,320]
+ CRUSH rule 1 x 815 [679,412,613,132,969,411,314,670,928,727]
+ CRUSH rule 1 x 816 [919,448,826,414,36,289,44,822,332,959]
+ CRUSH rule 1 x 817 [765,830,436,521,332,458,260,172,193,516]
+ CRUSH rule 1 x 818 [415,566,644,687,692,414,769,826,519,277]
+ CRUSH rule 1 x 819 [721,319,865,750,546,859,523,770,56,437]
+ CRUSH rule 1 x 820 [218,301,333,190,686,179,535,787,267,46]
+ CRUSH rule 1 x 821 [185,795,680,953,329,750,621,815,313,916]
+ CRUSH rule 1 x 822 [356,261,54,522,900,103,883,112,601,15]
+ CRUSH rule 1 x 823 [220,281,549,456,64,306,282,641,216,929]
+ CRUSH rule 1 x 824 [292,809,887,74,776,788,559,886,753,749]
+ CRUSH rule 1 x 825 [949,778,101,311,110,480,161,998,370,10]
+ CRUSH rule 1 x 826 [767,818,833,927,356,954,910,63,288,836]
+ CRUSH rule 1 x 827 [631,83,406,635,657,713,212,916,692,653]
+ CRUSH rule 1 x 828 [288,986,445,26,414,607,937,595,935,672]
+ CRUSH rule 1 x 829 [990,667,915,694,974,453,669,330,822,36]
+ CRUSH rule 1 x 830 [152,571,778,505,685,209,448,55,965,851]
+ CRUSH rule 1 x 831 [814,563,630,97,582,107,142,157,957,330]
+ CRUSH rule 1 x 832 [235,641,616,110,979,844,656,135,341,922]
+ CRUSH rule 1 x 833 [657,565,922,140,825,457,764,766,853,890]
+ CRUSH rule 1 x 834 [907,231,644,13,617,130,83,483,811,98]
+ CRUSH rule 1 x 835 [784,262,771,264,612,238,537,937,101,507]
+ CRUSH rule 1 x 836 [951,158,366,710,43,427,351,961,52,44]
+ CRUSH rule 1 x 837 [556,498,334,633,895,627,903,29,454,647]
+ CRUSH rule 1 x 838 [329,274,964,547,119,342,983,998,320,935]
+ CRUSH rule 1 x 839 [568,209,939,364,658,747,47,859,402,947]
+ CRUSH rule 1 x 840 [45,579,842,70,655,862,815,109,762,642]
+ CRUSH rule 1 x 841 [652,702,24,605,152,93,226,46,918,220]
+ CRUSH rule 1 x 842 [629,984,314,895,408,897,575,1,312,542]
+ CRUSH rule 1 x 843 [799,690,688,648,151,812,486,199,966,501]
+ CRUSH rule 1 x 844 [694,600,534,700,569,11,899,382,851,472]
+ CRUSH rule 1 x 845 [332,30,179,93,951,324,611,512,855,760]
+ CRUSH rule 1 x 846 [452,251,712,719,404,739,606,237,414,844]
+ CRUSH rule 1 x 847 [399,681,847,739,13,555,363,893,592,634]
+ CRUSH rule 1 x 848 [303,138,440,346,547,216,700,249,214,100]
+ CRUSH rule 1 x 849 [666,346,708,873,64,694,847,463,995,314]
+ CRUSH rule 1 x 850 [644,511,345,844,545,337,358,35,913,310]
+ CRUSH rule 1 x 851 [527,546,737,425,100,331,95,337,677,275]
+ CRUSH rule 1 x 852 [31,809,94,618,156,853,469,511,999,340]
+ CRUSH rule 1 x 853 [483,330,869,184,46,942,774,679,616,492]
+ CRUSH rule 1 x 854 [697,953,968,143,502,955,441,302,437,53]
+ CRUSH rule 1 x 855 [837,996,239,621,32,191,686,702,919,971]
+ CRUSH rule 1 x 856 [712,40,547,430,195,857,224,810,404,126]
+ CRUSH rule 1 x 857 [77,984,576,551,568,96,12,763,594,668]
+ CRUSH rule 1 x 858 [412,384,841,465,572,576,688,61,545,491]
+ CRUSH rule 1 x 859 [173,760,26,300,87,567,463,903,272,8]
+ CRUSH rule 1 x 860 [776,429,328,917,658,783,699,907,532,627]
+ CRUSH rule 1 x 861 [705,405,477,50,73,714,901,487,725,204]
+ CRUSH rule 1 x 862 [809,44,788,938,964,177,490,409,15,842]
+ CRUSH rule 1 x 863 [349,496,963,178,675,853,172,980,772,115]
+ CRUSH rule 1 x 864 [717,858,101,239,992,244,43,15,29,974]
+ CRUSH rule 1 x 865 [857,603,586,262,550,289,850,40,170,31]
+ CRUSH rule 1 x 866 [394,304,71,96,642,155,255,481,435,119]
+ CRUSH rule 1 x 867 [640,773,663,974,261,296,988,730,753,888]
+ CRUSH rule 1 x 868 [613,950,712,663,448,460,643,547,734,16]
+ CRUSH rule 1 x 869 [973,889,524,22,671,477,718,431,968,472]
+ CRUSH rule 1 x 870 [505,35,386,498,348,503,54,992,726,783]
+ CRUSH rule 1 x 871 [239,264,262,773,781,734,387,515,98,232]
+ CRUSH rule 1 x 872 [21,767,456,748,783,797,180,800,521,270]
+ CRUSH rule 1 x 873 [954,666,980,264,435,233,199,358,805,255]
+ CRUSH rule 1 x 874 [54,510,947,1,500,119,93,915,801,43]
+ CRUSH rule 1 x 875 [809,418,452,462,88,673,634,435,778,884]
+ CRUSH rule 1 x 876 [483,457,61,248,523,277,322,141,82,412]
+ CRUSH rule 1 x 877 [542,531,952,939,710,179,181,460,459,527]
+ CRUSH rule 1 x 878 [217,674,857,644,678,809,329,591,59,4]
+ CRUSH rule 1 x 879 [999,475,134,250,319,357,145,750,54,997]
+ CRUSH rule 1 x 880 [678,573,935,385,570,651,319,630,888,970]
+ CRUSH rule 1 x 881 [394,835,789,802,587,155,570,109,896,826]
+ CRUSH rule 1 x 882 [467,382,353,56,979,674,974,483,412,547]
+ CRUSH rule 1 x 883 [802,744,237,337,50,96,202,148,129,72]
+ CRUSH rule 1 x 884 [653,660,638,700,31,558,389,381,347,314]
+ CRUSH rule 1 x 885 [898,704,307,445,879,872,174,972,544,894]
+ CRUSH rule 1 x 886 [434,357,938,641,737,8,56,582,915,541]
+ CRUSH rule 1 x 887 [297,226,711,428,370,318,472,947,35,528]
+ CRUSH rule 1 x 888 [863,324,443,213,902,25,806,53,385,387]
+ CRUSH rule 1 x 889 [105,102,308,163,947,548,399,382,761,907]
+ CRUSH rule 1 x 890 [550,248,606,704,615,708,996,561,485,482]
+ CRUSH rule 1 x 891 [575,928,880,891,826,763,706,701,501,680]
+ CRUSH rule 1 x 892 [259,862,133,271,292,162,53,333,458,77]
+ CRUSH rule 1 x 893 [902,880,543,542,37,942,672,320,394,373]
+ CRUSH rule 1 x 894 [180,169,916,43,945,713,648,685,895,735]
+ CRUSH rule 1 x 895 [725,849,182,129,177,272,599,829,809,713]
+ CRUSH rule 1 x 896 [951,34,874,537,969,123,210,529,491,289]
+ CRUSH rule 1 x 897 [810,352,73,939,943,895,12,481,539,562]
+ CRUSH rule 1 x 898 [979,433,719,411,787,359,342,37,303,70]
+ CRUSH rule 1 x 899 [685,668,534,932,399,156,124,653,574,384]
+ CRUSH rule 1 x 900 [530,978,41,894,941,681,380,419,667,56]
+ CRUSH rule 1 x 901 [740,107,336,175,574,706,157,292,724,805]
+ CRUSH rule 1 x 902 [800,743,693,310,67,111,178,624,733,498]
+ CRUSH rule 1 x 903 [230,267,842,266,550,769,66,738,419,199]
+ CRUSH rule 1 x 904 [346,949,460,973,696,91,957,801,74,934]
+ CRUSH rule 1 x 905 [530,397,619,958,576,973,685,6,689,387]
+ CRUSH rule 1 x 906 [80,426,138,672,73,776,30,169,506,497]
+ CRUSH rule 1 x 907 [365,968,475,297,296,724,664,331,184,461]
+ CRUSH rule 1 x 908 [204,832,742,809,862,745,484,391,841,967]
+ CRUSH rule 1 x 909 [883,989,146,959,366,59,686,965,515,421]
+ CRUSH rule 1 x 910 [549,593,249,853,792,769,824,552,717,159]
+ CRUSH rule 1 x 911 [325,847,352,214,851,732,789,255,896,868]
+ CRUSH rule 1 x 912 [874,888,582,796,557,601,226,889,69,237]
+ CRUSH rule 1 x 913 [331,463,342,574,989,362,925,746,664,533]
+ CRUSH rule 1 x 914 [836,468,601,732,607,275,70,280,837,367]
+ CRUSH rule 1 x 915 [245,228,100,661,799,13,126,79,652,793]
+ CRUSH rule 1 x 916 [77,967,364,435,27,474,255,133,892,524]
+ CRUSH rule 1 x 917 [239,60,866,221,772,967,725,707,47,216]
+ CRUSH rule 1 x 918 [988,115,922,80,201,544,583,923,863,232]
+ CRUSH rule 1 x 919 [783,139,696,1,848,169,888,980,33,261]
+ CRUSH rule 1 x 920 [623,408,685,953,974,696,532,124,911,206]
+ CRUSH rule 1 x 921 [105,799,144,90,399,373,633,290,155,137]
+ CRUSH rule 1 x 922 [887,505,652,348,514,806,952,474,67,938]
+ CRUSH rule 1 x 923 [223,318,552,458,743,871,964,384,454,448]
+ CRUSH rule 1 x 924 [25,778,366,333,163,801,584,31,151,178]
+ CRUSH rule 1 x 925 [912,601,297,682,770,173,969,168,500,68]
+ CRUSH rule 1 x 926 [968,133,690,144,814,155,709,158,96,739]
+ CRUSH rule 1 x 927 [277,724,214,988,690,342,465,775,725,414]
+ CRUSH rule 1 x 928 [554,203,658,789,298,299,847,752,780,738]
+ CRUSH rule 1 x 929 [761,802,367,528,758,522,744,171,144,704]
+ CRUSH rule 1 x 930 [814,61,788,736,660,491,832,654,567,160]
+ CRUSH rule 1 x 931 [29,193,61,41,343,664,487,839,776,117]
+ CRUSH rule 1 x 932 [446,198,862,534,168,35,530,462,202,11]
+ CRUSH rule 1 x 933 [352,742,216,321,525,44,568,61,945,154]
+ CRUSH rule 1 x 934 [730,2,332,631,613,249,533,116,254,569]
+ CRUSH rule 1 x 935 [731,23,736,79,361,992,772,49,567,47]
+ CRUSH rule 1 x 936 [322,975,20,904,827,603,138,802,885,447]
+ CRUSH rule 1 x 937 [822,221,841,161,723,137,630,308,973,934]
+ CRUSH rule 1 x 938 [557,850,66,630,499,404,286,395,927,611]
+ CRUSH rule 1 x 939 [150,11,971,371,124,785,408,49,977,243]
+ CRUSH rule 1 x 940 [638,398,169,616,333,751,25,883,867,270]
+ CRUSH rule 1 x 941 [730,342,929,577,451,838,964,28,633,960]
+ CRUSH rule 1 x 942 [62,292,166,814,587,172,553,16,440,31]
+ CRUSH rule 1 x 943 [165,314,519,548,41,726,759,851,617,420]
+ CRUSH rule 1 x 944 [199,625,766,176,194,297,678,915,619,69]
+ CRUSH rule 1 x 945 [946,999,699,303,38,81,952,885,987,775]
+ CRUSH rule 1 x 946 [595,93,852,142,503,647,933,267,846,866]
+ CRUSH rule 1 x 947 [800,582,356,93,716,117,922,868,413,545]
+ CRUSH rule 1 x 948 [132,551,139,920,87,46,81,220,725,211]
+ CRUSH rule 1 x 949 [792,920,466,380,97,568,799,961,564,71]
+ CRUSH rule 1 x 950 [111,345,176,543,879,954,355,220,528,747]
+ CRUSH rule 1 x 951 [414,619,648,655,364,971,829,408,568,734]
+ CRUSH rule 1 x 952 [775,469,500,356,287,4,16,746,835,529]
+ CRUSH rule 1 x 953 [349,1,5,251,168,680,141,619,234,517]
+ CRUSH rule 1 x 954 [570,940,410,249,929,394,129,696,115,984]
+ CRUSH rule 1 x 955 [729,774,823,800,7,127,536,766,579,398]
+ CRUSH rule 1 x 956 [519,141,575,625,738,475,169,751,667,381]
+ CRUSH rule 1 x 957 [242,709,611,97,760,309,393,281,227,412]
+ CRUSH rule 1 x 958 [84,217,227,253,246,604,346,377,425,533]
+ CRUSH rule 1 x 959 [270,413,918,789,703,608,543,519,496,956]
+ CRUSH rule 1 x 960 [458,192,307,279,920,139,855,49,548,367]
+ CRUSH rule 1 x 961 [981,388,777,546,359,660,455,708,649,93]
+ CRUSH rule 1 x 962 [623,834,277,134,729,246,856,477,895,89]
+ CRUSH rule 1 x 963 [291,167,714,468,109,373,485,701,76,55]
+ CRUSH rule 1 x 964 [28,156,788,127,598,215,361,255,507,540]
+ CRUSH rule 1 x 965 [675,557,290,517,840,510,59,229,819,610]
+ CRUSH rule 1 x 966 [836,306,946,283,642,606,929,773,928,579]
+ CRUSH rule 1 x 967 [966,386,735,837,392,116,19,674,395,483]
+ CRUSH rule 1 x 968 [864,756,690,121,328,122,433,520,916,41]
+ CRUSH rule 1 x 969 [729,625,480,769,512,882,518,956,398,269]
+ CRUSH rule 1 x 970 [800,362,646,582,309,102,576,411,416,523]
+ CRUSH rule 1 x 971 [737,381,153,684,298,166,344,520,546,612]
+ CRUSH rule 1 x 972 [952,245,720,884,334,311,754,540,79,174]
+ CRUSH rule 1 x 973 [356,455,579,857,832,596,549,524,109,364]
+ CRUSH rule 1 x 974 [545,758,586,596,300,790,116,993,644,405]
+ CRUSH rule 1 x 975 [336,191,202,146,720,897,330,308,744,843]
+ CRUSH rule 1 x 976 [446,208,757,620,252,846,397,58,57,603]
+ CRUSH rule 1 x 977 [202,896,196,956,763,126,783,828,409,529]
+ CRUSH rule 1 x 978 [612,324,996,225,418,583,514,169,99,878]
+ CRUSH rule 1 x 979 [843,457,675,650,958,657,677,173,903,781]
+ CRUSH rule 1 x 980 [60,914,881,626,850,759,398,943,764,867]
+ CRUSH rule 1 x 981 [702,749,937,153,724,514,536,212,247,523]
+ CRUSH rule 1 x 982 [298,928,738,167,99,668,395,198,100,580]
+ CRUSH rule 1 x 983 [723,572,395,358,900,37,927,597,103,461]
+ CRUSH rule 1 x 984 [723,864,804,935,846,993,950,840,427,469]
+ CRUSH rule 1 x 985 [945,459,868,211,524,954,911,208,91,999]
+ CRUSH rule 1 x 986 [772,664,535,169,297,996,864,555,687,212]
+ CRUSH rule 1 x 987 [88,324,312,843,661,580,76,894,480,323]
+ CRUSH rule 1 x 988 [522,927,131,996,351,685,865,47,116,230]
+ CRUSH rule 1 x 989 [578,332,208,605,975,207,155,380,797,177]
+ CRUSH rule 1 x 990 [638,228,414,311,738,698,340,526,728,595]
+ CRUSH rule 1 x 991 [530,221,451,422,879,916,754,928,288,668]
+ CRUSH rule 1 x 992 [925,705,275,81,234,310,117,546,798,777]
+ CRUSH rule 1 x 993 [991,301,43,469,830,242,382,428,451,216]
+ CRUSH rule 1 x 994 [276,51,868,683,843,815,557,378,936,192]
+ CRUSH rule 1 x 995 [288,836,753,790,758,120,158,265,110,171]
+ CRUSH rule 1 x 996 [887,983,252,686,470,345,459,764,859,776]
+ CRUSH rule 1 x 997 [110,924,386,79,705,697,210,698,273,955]
+ CRUSH rule 1 x 998 [435,830,485,853,926,730,786,762,444,561]
+ CRUSH rule 1 x 999 [876,738,357,913,723,51,15,585,898,902]
+ CRUSH rule 1 x 1000 [178,963,638,430,845,586,317,102,200,662]
+ CRUSH rule 1 x 1001 [99,519,66,759,583,944,739,922,343,574]
+ CRUSH rule 1 x 1002 [515,534,468,866,878,717,729,370,326,640]
+ CRUSH rule 1 x 1003 [104,611,937,698,94,67,614,783,865,245]
+ CRUSH rule 1 x 1004 [269,638,724,375,491,121,891,113,424,320]
+ CRUSH rule 1 x 1005 [369,223,309,409,822,39,597,969,911,241]
+ CRUSH rule 1 x 1006 [40,107,69,275,79,429,234,945,598,498]
+ CRUSH rule 1 x 1007 [978,111,416,758,454,640,5,444,795,150]
+ CRUSH rule 1 x 1008 [965,956,624,832,421,96,975,723,909,93]
+ CRUSH rule 1 x 1009 [598,476,356,695,919,566,234,383,604,903]
+ CRUSH rule 1 x 1010 [767,523,239,517,29,77,23,241,838,865]
+ CRUSH rule 1 x 1011 [289,871,207,576,347,698,48,570,639,230]
+ CRUSH rule 1 x 1012 [128,28,370,31,341,755,268,647,669,90]
+ CRUSH rule 1 x 1013 [979,765,660,812,666,187,808,351,572,403]
+ CRUSH rule 1 x 1014 [979,948,513,88,47,825,969,81,586,62]
+ CRUSH rule 1 x 1015 [277,790,396,672,542,647,145,11,965,669]
+ CRUSH rule 1 x 1016 [262,73,128,886,839,685,456,560,935,733]
+ CRUSH rule 1 x 1017 [150,269,61,499,832,591,637,731,738,154]
+ CRUSH rule 1 x 1018 [555,829,554,944,406,576,463,926,475,316]
+ CRUSH rule 1 x 1019 [513,356,265,446,65,288,768,245,337,197]
+ CRUSH rule 1 x 1020 [158,161,877,704,948,570,495,865,698,835]
+ CRUSH rule 1 x 1021 [915,998,957,285,546,202,676,322,671,622]
+ CRUSH rule 1 x 1022 [967,829,973,640,703,470,871,828,440,449]
+ CRUSH rule 1 x 1023 [488,257,614,859,325,419,50,560,595,554]
+ rule 1 (metadata) num_rep 10 result size == 10:\t1024/1024 (esc)
diff --git a/src/test/cli/crushtool/default-tunable-values.t b/src/test/cli/crushtool/test-map-legacy-tunables.t
similarity index 99%
rename from src/test/cli/crushtool/default-tunable-values.t
rename to src/test/cli/crushtool/test-map-legacy-tunables.t
index d15ce9e..12bf604 100644
--- a/src/test/cli/crushtool/default-tunable-values.t
+++ b/src/test/cli/crushtool/test-map-legacy-tunables.t
@@ -1,4 +1,4 @@
- $ crushtool -i "$TESTDIR/one-hundered-devices.crushmap" --test --show-statistics --rule 0
+ $ crushtool -i "$TESTDIR/test-map-a.crushmap" --test --show-statistics --rule 0
rule 0 (data), x = 0..1023, numrep = 1..10
CRUSH rule 0 x 0 [36]
CRUSH rule 0 x 1 [876]
diff --git a/src/test/cli/crushtool/test-map-tries-vs-retries.crushmap b/src/test/cli/crushtool/test-map-tries-vs-retries.crushmap
new file mode 100644
index 0000000..e36c0eb
Binary files /dev/null and b/src/test/cli/crushtool/test-map-tries-vs-retries.crushmap differ
diff --git a/src/test/cli/crushtool/test-map-tries-vs-retries.t b/src/test/cli/crushtool/test-map-tries-vs-retries.t
new file mode 100644
index 0000000..8eac255
--- /dev/null
+++ b/src/test/cli/crushtool/test-map-tries-vs-retries.t
@@ -0,0 +1,10259 @@
+ $ crushtool -i "$TESTDIR/test-map-tries-vs-retries.crushmap" --test --show-statistics --weight 0 0 --weight 8 0
+ rule 0 (replicated_ruleset), x = 0..1023, numrep = 1..10
+ CRUSH rule 0 x 0 [7]
+ CRUSH rule 0 x 1 [10]
+ CRUSH rule 0 x 2 [1]
+ CRUSH rule 0 x 3 [15]
+ CRUSH rule 0 x 4 [14]
+ CRUSH rule 0 x 5 [7]
+ CRUSH rule 0 x 6 [12]
+ CRUSH rule 0 x 7 [9]
+ CRUSH rule 0 x 8 [10]
+ CRUSH rule 0 x 9 [7]
+ CRUSH rule 0 x 10 [10]
+ CRUSH rule 0 x 11 [13]
+ CRUSH rule 0 x 12 [7]
+ CRUSH rule 0 x 13 [3]
+ CRUSH rule 0 x 14 [13]
+ CRUSH rule 0 x 15 [15]
+ CRUSH rule 0 x 16 [7]
+ CRUSH rule 0 x 17 [10]
+ CRUSH rule 0 x 18 [1]
+ CRUSH rule 0 x 19 [7]
+ CRUSH rule 0 x 20 [14]
+ CRUSH rule 0 x 21 [3]
+ CRUSH rule 0 x 22 [6]
+ CRUSH rule 0 x 23 [10]
+ CRUSH rule 0 x 24 [12]
+ CRUSH rule 0 x 25 [7]
+ CRUSH rule 0 x 26 [1]
+ CRUSH rule 0 x 27 [3]
+ CRUSH rule 0 x 28 [14]
+ CRUSH rule 0 x 29 [5]
+ CRUSH rule 0 x 30 [2]
+ CRUSH rule 0 x 31 [5]
+ CRUSH rule 0 x 32 [9]
+ CRUSH rule 0 x 33 [13]
+ CRUSH rule 0 x 34 [13]
+ CRUSH rule 0 x 35 [4]
+ CRUSH rule 0 x 36 [3]
+ CRUSH rule 0 x 37 [9]
+ CRUSH rule 0 x 38 [3]
+ CRUSH rule 0 x 39 [12]
+ CRUSH rule 0 x 40 [10]
+ CRUSH rule 0 x 41 [4]
+ CRUSH rule 0 x 42 [3]
+ CRUSH rule 0 x 43 [10]
+ CRUSH rule 0 x 44 [11]
+ CRUSH rule 0 x 45 [11]
+ CRUSH rule 0 x 46 [6]
+ CRUSH rule 0 x 47 [3]
+ CRUSH rule 0 x 48 [4]
+ CRUSH rule 0 x 49 [9]
+ CRUSH rule 0 x 50 [14]
+ CRUSH rule 0 x 51 [10]
+ CRUSH rule 0 x 52 [12]
+ CRUSH rule 0 x 53 [3]
+ CRUSH rule 0 x 54 [4]
+ CRUSH rule 0 x 55 [4]
+ CRUSH rule 0 x 56 [5]
+ CRUSH rule 0 x 57 [6]
+ CRUSH rule 0 x 58 [7]
+ CRUSH rule 0 x 59 [2]
+ CRUSH rule 0 x 60 [3]
+ CRUSH rule 0 x 61 [3]
+ CRUSH rule 0 x 62 [15]
+ CRUSH rule 0 x 63 [10]
+ CRUSH rule 0 x 64 [3]
+ CRUSH rule 0 x 65 [4]
+ CRUSH rule 0 x 66 [15]
+ CRUSH rule 0 x 67 [2]
+ CRUSH rule 0 x 68 [15]
+ CRUSH rule 0 x 69 [2]
+ CRUSH rule 0 x 70 [9]
+ CRUSH rule 0 x 71 [15]
+ CRUSH rule 0 x 72 [9]
+ CRUSH rule 0 x 73 [5]
+ CRUSH rule 0 x 74 [11]
+ CRUSH rule 0 x 75 [9]
+ CRUSH rule 0 x 76 [6]
+ CRUSH rule 0 x 77 [7]
+ CRUSH rule 0 x 78 [9]
+ CRUSH rule 0 x 79 [13]
+ CRUSH rule 0 x 80 [15]
+ CRUSH rule 0 x 81 [15]
+ CRUSH rule 0 x 82 [14]
+ CRUSH rule 0 x 83 [4]
+ CRUSH rule 0 x 84 [10]
+ CRUSH rule 0 x 85 [3]
+ CRUSH rule 0 x 86 [10]
+ CRUSH rule 0 x 87 [15]
+ CRUSH rule 0 x 88 [4]
+ CRUSH rule 0 x 89 [3]
+ CRUSH rule 0 x 90 [4]
+ CRUSH rule 0 x 91 [6]
+ CRUSH rule 0 x 92 [1]
+ CRUSH rule 0 x 93 [9]
+ CRUSH rule 0 x 94 [9]
+ CRUSH rule 0 x 95 [7]
+ CRUSH rule 0 x 96 [2]
+ CRUSH rule 0 x 97 [4]
+ CRUSH rule 0 x 98 [11]
+ CRUSH rule 0 x 99 [12]
+ CRUSH rule 0 x 100 [9]
+ CRUSH rule 0 x 101 [15]
+ CRUSH rule 0 x 102 [3]
+ CRUSH rule 0 x 103 [13]
+ CRUSH rule 0 x 104 [14]
+ CRUSH rule 0 x 105 [14]
+ CRUSH rule 0 x 106 [6]
+ CRUSH rule 0 x 107 [3]
+ CRUSH rule 0 x 108 [5]
+ CRUSH rule 0 x 109 [9]
+ CRUSH rule 0 x 110 [5]
+ CRUSH rule 0 x 111 [10]
+ CRUSH rule 0 x 112 [1]
+ CRUSH rule 0 x 113 [6]
+ CRUSH rule 0 x 114 [5]
+ CRUSH rule 0 x 115 [10]
+ CRUSH rule 0 x 116 [1]
+ CRUSH rule 0 x 117 [5]
+ CRUSH rule 0 x 118 [10]
+ CRUSH rule 0 x 119 [14]
+ CRUSH rule 0 x 120 [11]
+ CRUSH rule 0 x 121 [9]
+ CRUSH rule 0 x 122 [4]
+ CRUSH rule 0 x 123 [3]
+ CRUSH rule 0 x 124 [12]
+ CRUSH rule 0 x 125 [9]
+ CRUSH rule 0 x 126 [7]
+ CRUSH rule 0 x 127 [4]
+ CRUSH rule 0 x 128 [3]
+ CRUSH rule 0 x 129 [11]
+ CRUSH rule 0 x 130 [3]
+ CRUSH rule 0 x 131 [12]
+ CRUSH rule 0 x 132 [11]
+ CRUSH rule 0 x 133 [3]
+ CRUSH rule 0 x 134 [12]
+ CRUSH rule 0 x 135 [3]
+ CRUSH rule 0 x 136 [15]
+ CRUSH rule 0 x 137 [14]
+ CRUSH rule 0 x 138 [13]
+ CRUSH rule 0 x 139 [11]
+ CRUSH rule 0 x 140 [11]
+ CRUSH rule 0 x 141 [6]
+ CRUSH rule 0 x 142 [3]
+ CRUSH rule 0 x 143 [9]
+ CRUSH rule 0 x 144 [13]
+ CRUSH rule 0 x 145 [12]
+ CRUSH rule 0 x 146 [1]
+ CRUSH rule 0 x 147 [1]
+ CRUSH rule 0 x 148 [12]
+ CRUSH rule 0 x 149 [2]
+ CRUSH rule 0 x 150 [1]
+ CRUSH rule 0 x 151 [2]
+ CRUSH rule 0 x 152 [5]
+ CRUSH rule 0 x 153 [6]
+ CRUSH rule 0 x 154 [3]
+ CRUSH rule 0 x 155 [14]
+ CRUSH rule 0 x 156 [7]
+ CRUSH rule 0 x 157 [15]
+ CRUSH rule 0 x 158 [15]
+ CRUSH rule 0 x 159 [4]
+ CRUSH rule 0 x 160 [5]
+ CRUSH rule 0 x 161 [1]
+ CRUSH rule 0 x 162 [10]
+ CRUSH rule 0 x 163 [15]
+ CRUSH rule 0 x 164 [9]
+ CRUSH rule 0 x 165 [11]
+ CRUSH rule 0 x 166 [1]
+ CRUSH rule 0 x 167 [9]
+ CRUSH rule 0 x 168 [13]
+ CRUSH rule 0 x 169 [1]
+ CRUSH rule 0 x 170 [1]
+ CRUSH rule 0 x 171 [9]
+ CRUSH rule 0 x 172 [14]
+ CRUSH rule 0 x 173 [5]
+ CRUSH rule 0 x 174 [15]
+ CRUSH rule 0 x 175 [5]
+ CRUSH rule 0 x 176 [9]
+ CRUSH rule 0 x 177 [2]
+ CRUSH rule 0 x 178 [12]
+ CRUSH rule 0 x 179 [2]
+ CRUSH rule 0 x 180 [3]
+ CRUSH rule 0 x 181 [9]
+ CRUSH rule 0 x 182 [5]
+ CRUSH rule 0 x 183 [5]
+ CRUSH rule 0 x 184 [2]
+ CRUSH rule 0 x 185 [13]
+ CRUSH rule 0 x 186 [6]
+ CRUSH rule 0 x 187 [1]
+ CRUSH rule 0 x 188 [9]
+ CRUSH rule 0 x 189 [6]
+ CRUSH rule 0 x 190 [9]
+ CRUSH rule 0 x 191 [7]
+ CRUSH rule 0 x 192 [2]
+ CRUSH rule 0 x 193 [3]
+ CRUSH rule 0 x 194 [3]
+ CRUSH rule 0 x 195 [5]
+ CRUSH rule 0 x 196 [4]
+ CRUSH rule 0 x 197 [14]
+ CRUSH rule 0 x 198 [2]
+ CRUSH rule 0 x 199 [2]
+ CRUSH rule 0 x 200 [7]
+ CRUSH rule 0 x 201 [9]
+ CRUSH rule 0 x 202 [14]
+ CRUSH rule 0 x 203 [12]
+ CRUSH rule 0 x 204 [6]
+ CRUSH rule 0 x 205 [15]
+ CRUSH rule 0 x 206 [13]
+ CRUSH rule 0 x 207 [2]
+ CRUSH rule 0 x 208 [13]
+ CRUSH rule 0 x 209 [6]
+ CRUSH rule 0 x 210 [13]
+ CRUSH rule 0 x 211 [2]
+ CRUSH rule 0 x 212 [10]
+ CRUSH rule 0 x 213 [3]
+ CRUSH rule 0 x 214 [7]
+ CRUSH rule 0 x 215 [6]
+ CRUSH rule 0 x 216 [12]
+ CRUSH rule 0 x 217 [12]
+ CRUSH rule 0 x 218 [12]
+ CRUSH rule 0 x 219 [3]
+ CRUSH rule 0 x 220 [14]
+ CRUSH rule 0 x 221 [15]
+ CRUSH rule 0 x 222 [10]
+ CRUSH rule 0 x 223 [9]
+ CRUSH rule 0 x 224 [1]
+ CRUSH rule 0 x 225 [10]
+ CRUSH rule 0 x 226 [4]
+ CRUSH rule 0 x 227 [7]
+ CRUSH rule 0 x 228 [2]
+ CRUSH rule 0 x 229 [9]
+ CRUSH rule 0 x 230 [10]
+ CRUSH rule 0 x 231 [2]
+ CRUSH rule 0 x 232 [10]
+ CRUSH rule 0 x 233 [6]
+ CRUSH rule 0 x 234 [10]
+ CRUSH rule 0 x 235 [13]
+ CRUSH rule 0 x 236 [2]
+ CRUSH rule 0 x 237 [3]
+ CRUSH rule 0 x 238 [2]
+ CRUSH rule 0 x 239 [4]
+ CRUSH rule 0 x 240 [15]
+ CRUSH rule 0 x 241 [7]
+ CRUSH rule 0 x 242 [14]
+ CRUSH rule 0 x 243 [2]
+ CRUSH rule 0 x 244 [13]
+ CRUSH rule 0 x 245 [12]
+ CRUSH rule 0 x 246 [15]
+ CRUSH rule 0 x 247 [6]
+ CRUSH rule 0 x 248 [5]
+ CRUSH rule 0 x 249 [10]
+ CRUSH rule 0 x 250 [12]
+ CRUSH rule 0 x 251 [13]
+ CRUSH rule 0 x 252 [7]
+ CRUSH rule 0 x 253 [3]
+ CRUSH rule 0 x 254 [2]
+ CRUSH rule 0 x 255 [1]
+ CRUSH rule 0 x 256 [6]
+ CRUSH rule 0 x 257 [15]
+ CRUSH rule 0 x 258 [12]
+ CRUSH rule 0 x 259 [9]
+ CRUSH rule 0 x 260 [10]
+ CRUSH rule 0 x 261 [13]
+ CRUSH rule 0 x 262 [15]
+ CRUSH rule 0 x 263 [12]
+ CRUSH rule 0 x 264 [13]
+ CRUSH rule 0 x 265 [12]
+ CRUSH rule 0 x 266 [14]
+ CRUSH rule 0 x 267 [12]
+ CRUSH rule 0 x 268 [4]
+ CRUSH rule 0 x 269 [11]
+ CRUSH rule 0 x 270 [7]
+ CRUSH rule 0 x 271 [4]
+ CRUSH rule 0 x 272 [15]
+ CRUSH rule 0 x 273 [2]
+ CRUSH rule 0 x 274 [10]
+ CRUSH rule 0 x 275 [10]
+ CRUSH rule 0 x 276 [5]
+ CRUSH rule 0 x 277 [14]
+ CRUSH rule 0 x 278 [5]
+ CRUSH rule 0 x 279 [6]
+ CRUSH rule 0 x 280 [7]
+ CRUSH rule 0 x 281 [5]
+ CRUSH rule 0 x 282 [2]
+ CRUSH rule 0 x 283 [4]
+ CRUSH rule 0 x 284 [5]
+ CRUSH rule 0 x 285 [15]
+ CRUSH rule 0 x 286 [10]
+ CRUSH rule 0 x 287 [12]
+ CRUSH rule 0 x 288 [4]
+ CRUSH rule 0 x 289 [2]
+ CRUSH rule 0 x 290 [12]
+ CRUSH rule 0 x 291 [7]
+ CRUSH rule 0 x 292 [4]
+ CRUSH rule 0 x 293 [6]
+ CRUSH rule 0 x 294 [9]
+ CRUSH rule 0 x 295 [6]
+ CRUSH rule 0 x 296 [3]
+ CRUSH rule 0 x 297 [6]
+ CRUSH rule 0 x 298 [14]
+ CRUSH rule 0 x 299 [14]
+ CRUSH rule 0 x 300 [15]
+ CRUSH rule 0 x 301 [9]
+ CRUSH rule 0 x 302 [9]
+ CRUSH rule 0 x 303 [4]
+ CRUSH rule 0 x 304 [6]
+ CRUSH rule 0 x 305 [13]
+ CRUSH rule 0 x 306 [10]
+ CRUSH rule 0 x 307 [11]
+ CRUSH rule 0 x 308 [12]
+ CRUSH rule 0 x 309 [9]
+ CRUSH rule 0 x 310 [3]
+ CRUSH rule 0 x 311 [3]
+ CRUSH rule 0 x 312 [15]
+ CRUSH rule 0 x 313 [9]
+ CRUSH rule 0 x 314 [2]
+ CRUSH rule 0 x 315 [15]
+ CRUSH rule 0 x 316 [4]
+ CRUSH rule 0 x 317 [1]
+ CRUSH rule 0 x 318 [4]
+ CRUSH rule 0 x 319 [2]
+ CRUSH rule 0 x 320 [5]
+ CRUSH rule 0 x 321 [1]
+ CRUSH rule 0 x 322 [13]
+ CRUSH rule 0 x 323 [7]
+ CRUSH rule 0 x 324 [5]
+ CRUSH rule 0 x 325 [9]
+ CRUSH rule 0 x 326 [11]
+ CRUSH rule 0 x 327 [12]
+ CRUSH rule 0 x 328 [5]
+ CRUSH rule 0 x 329 [2]
+ CRUSH rule 0 x 330 [3]
+ CRUSH rule 0 x 331 [12]
+ CRUSH rule 0 x 332 [10]
+ CRUSH rule 0 x 333 [6]
+ CRUSH rule 0 x 334 [4]
+ CRUSH rule 0 x 335 [11]
+ CRUSH rule 0 x 336 [6]
+ CRUSH rule 0 x 337 [15]
+ CRUSH rule 0 x 338 [10]
+ CRUSH rule 0 x 339 [11]
+ CRUSH rule 0 x 340 [11]
+ CRUSH rule 0 x 341 [7]
+ CRUSH rule 0 x 342 [12]
+ CRUSH rule 0 x 343 [12]
+ CRUSH rule 0 x 344 [9]
+ CRUSH rule 0 x 345 [14]
+ CRUSH rule 0 x 346 [5]
+ CRUSH rule 0 x 347 [10]
+ CRUSH rule 0 x 348 [7]
+ CRUSH rule 0 x 349 [9]
+ CRUSH rule 0 x 350 [13]
+ CRUSH rule 0 x 351 [13]
+ CRUSH rule 0 x 352 [1]
+ CRUSH rule 0 x 353 [10]
+ CRUSH rule 0 x 354 [6]
+ CRUSH rule 0 x 355 [13]
+ CRUSH rule 0 x 356 [15]
+ CRUSH rule 0 x 357 [4]
+ CRUSH rule 0 x 358 [12]
+ CRUSH rule 0 x 359 [5]
+ CRUSH rule 0 x 360 [13]
+ CRUSH rule 0 x 361 [5]
+ CRUSH rule 0 x 362 [2]
+ CRUSH rule 0 x 363 [7]
+ CRUSH rule 0 x 364 [2]
+ CRUSH rule 0 x 365 [13]
+ CRUSH rule 0 x 366 [12]
+ CRUSH rule 0 x 367 [7]
+ CRUSH rule 0 x 368 [7]
+ CRUSH rule 0 x 369 [7]
+ CRUSH rule 0 x 370 [4]
+ CRUSH rule 0 x 371 [1]
+ CRUSH rule 0 x 372 [10]
+ CRUSH rule 0 x 373 [15]
+ CRUSH rule 0 x 374 [3]
+ CRUSH rule 0 x 375 [5]
+ CRUSH rule 0 x 376 [5]
+ CRUSH rule 0 x 377 [1]
+ CRUSH rule 0 x 378 [9]
+ CRUSH rule 0 x 379 [11]
+ CRUSH rule 0 x 380 [6]
+ CRUSH rule 0 x 381 [15]
+ CRUSH rule 0 x 382 [14]
+ CRUSH rule 0 x 383 [3]
+ CRUSH rule 0 x 384 [4]
+ CRUSH rule 0 x 385 [4]
+ CRUSH rule 0 x 386 [14]
+ CRUSH rule 0 x 387 [1]
+ CRUSH rule 0 x 388 [2]
+ CRUSH rule 0 x 389 [12]
+ CRUSH rule 0 x 390 [2]
+ CRUSH rule 0 x 391 [3]
+ CRUSH rule 0 x 392 [11]
+ CRUSH rule 0 x 393 [2]
+ CRUSH rule 0 x 394 [4]
+ CRUSH rule 0 x 395 [10]
+ CRUSH rule 0 x 396 [2]
+ CRUSH rule 0 x 397 [1]
+ CRUSH rule 0 x 398 [9]
+ CRUSH rule 0 x 399 [5]
+ CRUSH rule 0 x 400 [10]
+ CRUSH rule 0 x 401 [6]
+ CRUSH rule 0 x 402 [4]
+ CRUSH rule 0 x 403 [7]
+ CRUSH rule 0 x 404 [14]
+ CRUSH rule 0 x 405 [9]
+ CRUSH rule 0 x 406 [12]
+ CRUSH rule 0 x 407 [9]
+ CRUSH rule 0 x 408 [7]
+ CRUSH rule 0 x 409 [11]
+ CRUSH rule 0 x 410 [6]
+ CRUSH rule 0 x 411 [13]
+ CRUSH rule 0 x 412 [5]
+ CRUSH rule 0 x 413 [13]
+ CRUSH rule 0 x 414 [3]
+ CRUSH rule 0 x 415 [6]
+ CRUSH rule 0 x 416 [13]
+ CRUSH rule 0 x 417 [4]
+ CRUSH rule 0 x 418 [14]
+ CRUSH rule 0 x 419 [5]
+ CRUSH rule 0 x 420 [2]
+ CRUSH rule 0 x 421 [15]
+ CRUSH rule 0 x 422 [4]
+ CRUSH rule 0 x 423 [3]
+ CRUSH rule 0 x 424 [6]
+ CRUSH rule 0 x 425 [11]
+ CRUSH rule 0 x 426 [12]
+ CRUSH rule 0 x 427 [14]
+ CRUSH rule 0 x 428 [12]
+ CRUSH rule 0 x 429 [3]
+ CRUSH rule 0 x 430 [3]
+ CRUSH rule 0 x 431 [9]
+ CRUSH rule 0 x 432 [4]
+ CRUSH rule 0 x 433 [4]
+ CRUSH rule 0 x 434 [2]
+ CRUSH rule 0 x 435 [13]
+ CRUSH rule 0 x 436 [9]
+ CRUSH rule 0 x 437 [9]
+ CRUSH rule 0 x 438 [7]
+ CRUSH rule 0 x 439 [7]
+ CRUSH rule 0 x 440 [14]
+ CRUSH rule 0 x 441 [2]
+ CRUSH rule 0 x 442 [10]
+ CRUSH rule 0 x 443 [12]
+ CRUSH rule 0 x 444 [4]
+ CRUSH rule 0 x 445 [4]
+ CRUSH rule 0 x 446 [12]
+ CRUSH rule 0 x 447 [15]
+ CRUSH rule 0 x 448 [5]
+ CRUSH rule 0 x 449 [14]
+ CRUSH rule 0 x 450 [2]
+ CRUSH rule 0 x 451 [6]
+ CRUSH rule 0 x 452 [14]
+ CRUSH rule 0 x 453 [5]
+ CRUSH rule 0 x 454 [10]
+ CRUSH rule 0 x 455 [6]
+ CRUSH rule 0 x 456 [5]
+ CRUSH rule 0 x 457 [9]
+ CRUSH rule 0 x 458 [9]
+ CRUSH rule 0 x 459 [13]
+ CRUSH rule 0 x 460 [5]
+ CRUSH rule 0 x 461 [4]
+ CRUSH rule 0 x 462 [4]
+ CRUSH rule 0 x 463 [4]
+ CRUSH rule 0 x 464 [4]
+ CRUSH rule 0 x 465 [5]
+ CRUSH rule 0 x 466 [13]
+ CRUSH rule 0 x 467 [13]
+ CRUSH rule 0 x 468 [10]
+ CRUSH rule 0 x 469 [4]
+ CRUSH rule 0 x 470 [3]
+ CRUSH rule 0 x 471 [6]
+ CRUSH rule 0 x 472 [2]
+ CRUSH rule 0 x 473 [15]
+ CRUSH rule 0 x 474 [15]
+ CRUSH rule 0 x 475 [10]
+ CRUSH rule 0 x 476 [3]
+ CRUSH rule 0 x 477 [6]
+ CRUSH rule 0 x 478 [4]
+ CRUSH rule 0 x 479 [13]
+ CRUSH rule 0 x 480 [1]
+ CRUSH rule 0 x 481 [15]
+ CRUSH rule 0 x 482 [2]
+ CRUSH rule 0 x 483 [10]
+ CRUSH rule 0 x 484 [1]
+ CRUSH rule 0 x 485 [9]
+ CRUSH rule 0 x 486 [3]
+ CRUSH rule 0 x 487 [12]
+ CRUSH rule 0 x 488 [14]
+ CRUSH rule 0 x 489 [11]
+ CRUSH rule 0 x 490 [4]
+ CRUSH rule 0 x 491 [1]
+ CRUSH rule 0 x 492 [5]
+ CRUSH rule 0 x 493 [12]
+ CRUSH rule 0 x 494 [1]
+ CRUSH rule 0 x 495 [3]
+ CRUSH rule 0 x 496 [5]
+ CRUSH rule 0 x 497 [13]
+ CRUSH rule 0 x 498 [10]
+ CRUSH rule 0 x 499 [14]
+ CRUSH rule 0 x 500 [15]
+ CRUSH rule 0 x 501 [10]
+ CRUSH rule 0 x 502 [5]
+ CRUSH rule 0 x 503 [15]
+ CRUSH rule 0 x 504 [13]
+ CRUSH rule 0 x 505 [12]
+ CRUSH rule 0 x 506 [11]
+ CRUSH rule 0 x 507 [4]
+ CRUSH rule 0 x 508 [12]
+ CRUSH rule 0 x 509 [4]
+ CRUSH rule 0 x 510 [5]
+ CRUSH rule 0 x 511 [2]
+ CRUSH rule 0 x 512 [15]
+ CRUSH rule 0 x 513 [4]
+ CRUSH rule 0 x 514 [11]
+ CRUSH rule 0 x 515 [12]
+ CRUSH rule 0 x 516 [14]
+ CRUSH rule 0 x 517 [11]
+ CRUSH rule 0 x 518 [3]
+ CRUSH rule 0 x 519 [12]
+ CRUSH rule 0 x 520 [12]
+ CRUSH rule 0 x 521 [11]
+ CRUSH rule 0 x 522 [4]
+ CRUSH rule 0 x 523 [3]
+ CRUSH rule 0 x 524 [15]
+ CRUSH rule 0 x 525 [3]
+ CRUSH rule 0 x 526 [10]
+ CRUSH rule 0 x 527 [3]
+ CRUSH rule 0 x 528 [12]
+ CRUSH rule 0 x 529 [6]
+ CRUSH rule 0 x 530 [11]
+ CRUSH rule 0 x 531 [9]
+ CRUSH rule 0 x 532 [5]
+ CRUSH rule 0 x 533 [12]
+ CRUSH rule 0 x 534 [11]
+ CRUSH rule 0 x 535 [11]
+ CRUSH rule 0 x 536 [9]
+ CRUSH rule 0 x 537 [15]
+ CRUSH rule 0 x 538 [13]
+ CRUSH rule 0 x 539 [10]
+ CRUSH rule 0 x 540 [12]
+ CRUSH rule 0 x 541 [2]
+ CRUSH rule 0 x 542 [3]
+ CRUSH rule 0 x 543 [4]
+ CRUSH rule 0 x 544 [3]
+ CRUSH rule 0 x 545 [14]
+ CRUSH rule 0 x 546 [5]
+ CRUSH rule 0 x 547 [5]
+ CRUSH rule 0 x 548 [11]
+ CRUSH rule 0 x 549 [14]
+ CRUSH rule 0 x 550 [9]
+ CRUSH rule 0 x 551 [11]
+ CRUSH rule 0 x 552 [2]
+ CRUSH rule 0 x 553 [11]
+ CRUSH rule 0 x 554 [11]
+ CRUSH rule 0 x 555 [6]
+ CRUSH rule 0 x 556 [15]
+ CRUSH rule 0 x 557 [12]
+ CRUSH rule 0 x 558 [12]
+ CRUSH rule 0 x 559 [2]
+ CRUSH rule 0 x 560 [4]
+ CRUSH rule 0 x 561 [12]
+ CRUSH rule 0 x 562 [7]
+ CRUSH rule 0 x 563 [15]
+ CRUSH rule 0 x 564 [2]
+ CRUSH rule 0 x 565 [3]
+ CRUSH rule 0 x 566 [6]
+ CRUSH rule 0 x 567 [15]
+ CRUSH rule 0 x 568 [4]
+ CRUSH rule 0 x 569 [11]
+ CRUSH rule 0 x 570 [1]
+ CRUSH rule 0 x 571 [10]
+ CRUSH rule 0 x 572 [12]
+ CRUSH rule 0 x 573 [7]
+ CRUSH rule 0 x 574 [11]
+ CRUSH rule 0 x 575 [5]
+ CRUSH rule 0 x 576 [3]
+ CRUSH rule 0 x 577 [13]
+ CRUSH rule 0 x 578 [4]
+ CRUSH rule 0 x 579 [13]
+ CRUSH rule 0 x 580 [3]
+ CRUSH rule 0 x 581 [7]
+ CRUSH rule 0 x 582 [10]
+ CRUSH rule 0 x 583 [4]
+ CRUSH rule 0 x 584 [10]
+ CRUSH rule 0 x 585 [5]
+ CRUSH rule 0 x 586 [7]
+ CRUSH rule 0 x 587 [11]
+ CRUSH rule 0 x 588 [3]
+ CRUSH rule 0 x 589 [9]
+ CRUSH rule 0 x 590 [12]
+ CRUSH rule 0 x 591 [2]
+ CRUSH rule 0 x 592 [15]
+ CRUSH rule 0 x 593 [13]
+ CRUSH rule 0 x 594 [12]
+ CRUSH rule 0 x 595 [12]
+ CRUSH rule 0 x 596 [2]
+ CRUSH rule 0 x 597 [15]
+ CRUSH rule 0 x 598 [11]
+ CRUSH rule 0 x 599 [13]
+ CRUSH rule 0 x 600 [4]
+ CRUSH rule 0 x 601 [13]
+ CRUSH rule 0 x 602 [3]
+ CRUSH rule 0 x 603 [3]
+ CRUSH rule 0 x 604 [14]
+ CRUSH rule 0 x 605 [2]
+ CRUSH rule 0 x 606 [12]
+ CRUSH rule 0 x 607 [3]
+ CRUSH rule 0 x 608 [13]
+ CRUSH rule 0 x 609 [14]
+ CRUSH rule 0 x 610 [7]
+ CRUSH rule 0 x 611 [13]
+ CRUSH rule 0 x 612 [7]
+ CRUSH rule 0 x 613 [10]
+ CRUSH rule 0 x 614 [9]
+ CRUSH rule 0 x 615 [9]
+ CRUSH rule 0 x 616 [10]
+ CRUSH rule 0 x 617 [15]
+ CRUSH rule 0 x 618 [4]
+ CRUSH rule 0 x 619 [15]
+ CRUSH rule 0 x 620 [3]
+ CRUSH rule 0 x 621 [3]
+ CRUSH rule 0 x 622 [10]
+ CRUSH rule 0 x 623 [4]
+ CRUSH rule 0 x 624 [3]
+ CRUSH rule 0 x 625 [11]
+ CRUSH rule 0 x 626 [10]
+ CRUSH rule 0 x 627 [1]
+ CRUSH rule 0 x 628 [15]
+ CRUSH rule 0 x 629 [5]
+ CRUSH rule 0 x 630 [1]
+ CRUSH rule 0 x 631 [5]
+ CRUSH rule 0 x 632 [12]
+ CRUSH rule 0 x 633 [14]
+ CRUSH rule 0 x 634 [6]
+ CRUSH rule 0 x 635 [6]
+ CRUSH rule 0 x 636 [13]
+ CRUSH rule 0 x 637 [3]
+ CRUSH rule 0 x 638 [10]
+ CRUSH rule 0 x 639 [6]
+ CRUSH rule 0 x 640 [9]
+ CRUSH rule 0 x 641 [10]
+ CRUSH rule 0 x 642 [1]
+ CRUSH rule 0 x 643 [3]
+ CRUSH rule 0 x 644 [15]
+ CRUSH rule 0 x 645 [14]
+ CRUSH rule 0 x 646 [5]
+ CRUSH rule 0 x 647 [10]
+ CRUSH rule 0 x 648 [6]
+ CRUSH rule 0 x 649 [3]
+ CRUSH rule 0 x 650 [10]
+ CRUSH rule 0 x 651 [3]
+ CRUSH rule 0 x 652 [15]
+ CRUSH rule 0 x 653 [11]
+ CRUSH rule 0 x 654 [13]
+ CRUSH rule 0 x 655 [6]
+ CRUSH rule 0 x 656 [3]
+ CRUSH rule 0 x 657 [11]
+ CRUSH rule 0 x 658 [7]
+ CRUSH rule 0 x 659 [2]
+ CRUSH rule 0 x 660 [13]
+ CRUSH rule 0 x 661 [7]
+ CRUSH rule 0 x 662 [15]
+ CRUSH rule 0 x 663 [14]
+ CRUSH rule 0 x 664 [6]
+ CRUSH rule 0 x 665 [2]
+ CRUSH rule 0 x 666 [12]
+ CRUSH rule 0 x 667 [1]
+ CRUSH rule 0 x 668 [9]
+ CRUSH rule 0 x 669 [9]
+ CRUSH rule 0 x 670 [6]
+ CRUSH rule 0 x 671 [6]
+ CRUSH rule 0 x 672 [2]
+ CRUSH rule 0 x 673 [7]
+ CRUSH rule 0 x 674 [7]
+ CRUSH rule 0 x 675 [9]
+ CRUSH rule 0 x 676 [10]
+ CRUSH rule 0 x 677 [2]
+ CRUSH rule 0 x 678 [1]
+ CRUSH rule 0 x 679 [5]
+ CRUSH rule 0 x 680 [7]
+ CRUSH rule 0 x 681 [6]
+ CRUSH rule 0 x 682 [6]
+ CRUSH rule 0 x 683 [6]
+ CRUSH rule 0 x 684 [9]
+ CRUSH rule 0 x 685 [5]
+ CRUSH rule 0 x 686 [1]
+ CRUSH rule 0 x 687 [7]
+ CRUSH rule 0 x 688 [11]
+ CRUSH rule 0 x 689 [5]
+ CRUSH rule 0 x 690 [9]
+ CRUSH rule 0 x 691 [11]
+ CRUSH rule 0 x 692 [15]
+ CRUSH rule 0 x 693 [5]
+ CRUSH rule 0 x 694 [4]
+ CRUSH rule 0 x 695 [6]
+ CRUSH rule 0 x 696 [1]
+ CRUSH rule 0 x 697 [13]
+ CRUSH rule 0 x 698 [11]
+ CRUSH rule 0 x 699 [7]
+ CRUSH rule 0 x 700 [12]
+ CRUSH rule 0 x 701 [3]
+ CRUSH rule 0 x 702 [3]
+ CRUSH rule 0 x 703 [15]
+ CRUSH rule 0 x 704 [6]
+ CRUSH rule 0 x 705 [14]
+ CRUSH rule 0 x 706 [1]
+ CRUSH rule 0 x 707 [4]
+ CRUSH rule 0 x 708 [3]
+ CRUSH rule 0 x 709 [11]
+ CRUSH rule 0 x 710 [14]
+ CRUSH rule 0 x 711 [14]
+ CRUSH rule 0 x 712 [12]
+ CRUSH rule 0 x 713 [11]
+ CRUSH rule 0 x 714 [12]
+ CRUSH rule 0 x 715 [6]
+ CRUSH rule 0 x 716 [11]
+ CRUSH rule 0 x 717 [12]
+ CRUSH rule 0 x 718 [7]
+ CRUSH rule 0 x 719 [5]
+ CRUSH rule 0 x 720 [4]
+ CRUSH rule 0 x 721 [11]
+ CRUSH rule 0 x 722 [2]
+ CRUSH rule 0 x 723 [2]
+ CRUSH rule 0 x 724 [7]
+ CRUSH rule 0 x 725 [11]
+ CRUSH rule 0 x 726 [7]
+ CRUSH rule 0 x 727 [2]
+ CRUSH rule 0 x 728 [13]
+ CRUSH rule 0 x 729 [15]
+ CRUSH rule 0 x 730 [3]
+ CRUSH rule 0 x 731 [9]
+ CRUSH rule 0 x 732 [1]
+ CRUSH rule 0 x 733 [11]
+ CRUSH rule 0 x 734 [14]
+ CRUSH rule 0 x 735 [6]
+ CRUSH rule 0 x 736 [3]
+ CRUSH rule 0 x 737 [1]
+ CRUSH rule 0 x 738 [11]
+ CRUSH rule 0 x 739 [11]
+ CRUSH rule 0 x 740 [7]
+ CRUSH rule 0 x 741 [12]
+ CRUSH rule 0 x 742 [9]
+ CRUSH rule 0 x 743 [5]
+ CRUSH rule 0 x 744 [6]
+ CRUSH rule 0 x 745 [3]
+ CRUSH rule 0 x 746 [3]
+ CRUSH rule 0 x 747 [15]
+ CRUSH rule 0 x 748 [6]
+ CRUSH rule 0 x 749 [14]
+ CRUSH rule 0 x 750 [1]
+ CRUSH rule 0 x 751 [15]
+ CRUSH rule 0 x 752 [13]
+ CRUSH rule 0 x 753 [4]
+ CRUSH rule 0 x 754 [14]
+ CRUSH rule 0 x 755 [13]
+ CRUSH rule 0 x 756 [3]
+ CRUSH rule 0 x 757 [10]
+ CRUSH rule 0 x 758 [6]
+ CRUSH rule 0 x 759 [5]
+ CRUSH rule 0 x 760 [1]
+ CRUSH rule 0 x 761 [2]
+ CRUSH rule 0 x 762 [1]
+ CRUSH rule 0 x 763 [4]
+ CRUSH rule 0 x 764 [1]
+ CRUSH rule 0 x 765 [9]
+ CRUSH rule 0 x 766 [11]
+ CRUSH rule 0 x 767 [6]
+ CRUSH rule 0 x 768 [2]
+ CRUSH rule 0 x 769 [15]
+ CRUSH rule 0 x 770 [15]
+ CRUSH rule 0 x 771 [9]
+ CRUSH rule 0 x 772 [4]
+ CRUSH rule 0 x 773 [3]
+ CRUSH rule 0 x 774 [12]
+ CRUSH rule 0 x 775 [5]
+ CRUSH rule 0 x 776 [10]
+ CRUSH rule 0 x 777 [11]
+ CRUSH rule 0 x 778 [13]
+ CRUSH rule 0 x 779 [5]
+ CRUSH rule 0 x 780 [13]
+ CRUSH rule 0 x 781 [5]
+ CRUSH rule 0 x 782 [2]
+ CRUSH rule 0 x 783 [12]
+ CRUSH rule 0 x 784 [14]
+ CRUSH rule 0 x 785 [6]
+ CRUSH rule 0 x 786 [10]
+ CRUSH rule 0 x 787 [1]
+ CRUSH rule 0 x 788 [4]
+ CRUSH rule 0 x 789 [9]
+ CRUSH rule 0 x 790 [15]
+ CRUSH rule 0 x 791 [9]
+ CRUSH rule 0 x 792 [6]
+ CRUSH rule 0 x 793 [15]
+ CRUSH rule 0 x 794 [5]
+ CRUSH rule 0 x 795 [6]
+ CRUSH rule 0 x 796 [11]
+ CRUSH rule 0 x 797 [14]
+ CRUSH rule 0 x 798 [5]
+ CRUSH rule 0 x 799 [2]
+ CRUSH rule 0 x 800 [6]
+ CRUSH rule 0 x 801 [2]
+ CRUSH rule 0 x 802 [1]
+ CRUSH rule 0 x 803 [7]
+ CRUSH rule 0 x 804 [5]
+ CRUSH rule 0 x 805 [13]
+ CRUSH rule 0 x 806 [6]
+ CRUSH rule 0 x 807 [14]
+ CRUSH rule 0 x 808 [2]
+ CRUSH rule 0 x 809 [1]
+ CRUSH rule 0 x 810 [2]
+ CRUSH rule 0 x 811 [15]
+ CRUSH rule 0 x 812 [7]
+ CRUSH rule 0 x 813 [4]
+ CRUSH rule 0 x 814 [13]
+ CRUSH rule 0 x 815 [15]
+ CRUSH rule 0 x 816 [14]
+ CRUSH rule 0 x 817 [10]
+ CRUSH rule 0 x 818 [15]
+ CRUSH rule 0 x 819 [5]
+ CRUSH rule 0 x 820 [3]
+ CRUSH rule 0 x 821 [15]
+ CRUSH rule 0 x 822 [10]
+ CRUSH rule 0 x 823 [2]
+ CRUSH rule 0 x 824 [3]
+ CRUSH rule 0 x 825 [10]
+ CRUSH rule 0 x 826 [5]
+ CRUSH rule 0 x 827 [13]
+ CRUSH rule 0 x 828 [12]
+ CRUSH rule 0 x 829 [13]
+ CRUSH rule 0 x 830 [15]
+ CRUSH rule 0 x 831 [1]
+ CRUSH rule 0 x 832 [14]
+ CRUSH rule 0 x 833 [9]
+ CRUSH rule 0 x 834 [9]
+ CRUSH rule 0 x 835 [14]
+ CRUSH rule 0 x 836 [3]
+ CRUSH rule 0 x 837 [15]
+ CRUSH rule 0 x 838 [12]
+ CRUSH rule 0 x 839 [3]
+ CRUSH rule 0 x 840 [10]
+ CRUSH rule 0 x 841 [3]
+ CRUSH rule 0 x 842 [9]
+ CRUSH rule 0 x 843 [14]
+ CRUSH rule 0 x 844 [7]
+ CRUSH rule 0 x 845 [13]
+ CRUSH rule 0 x 846 [3]
+ CRUSH rule 0 x 847 [12]
+ CRUSH rule 0 x 848 [11]
+ CRUSH rule 0 x 849 [3]
+ CRUSH rule 0 x 850 [1]
+ CRUSH rule 0 x 851 [14]
+ CRUSH rule 0 x 852 [9]
+ CRUSH rule 0 x 853 [13]
+ CRUSH rule 0 x 854 [7]
+ CRUSH rule 0 x 855 [14]
+ CRUSH rule 0 x 856 [5]
+ CRUSH rule 0 x 857 [4]
+ CRUSH rule 0 x 858 [5]
+ CRUSH rule 0 x 859 [5]
+ CRUSH rule 0 x 860 [11]
+ CRUSH rule 0 x 861 [13]
+ CRUSH rule 0 x 862 [5]
+ CRUSH rule 0 x 863 [11]
+ CRUSH rule 0 x 864 [6]
+ CRUSH rule 0 x 865 [4]
+ CRUSH rule 0 x 866 [2]
+ CRUSH rule 0 x 867 [12]
+ CRUSH rule 0 x 868 [14]
+ CRUSH rule 0 x 869 [10]
+ CRUSH rule 0 x 870 [14]
+ CRUSH rule 0 x 871 [6]
+ CRUSH rule 0 x 872 [6]
+ CRUSH rule 0 x 873 [2]
+ CRUSH rule 0 x 874 [12]
+ CRUSH rule 0 x 875 [10]
+ CRUSH rule 0 x 876 [14]
+ CRUSH rule 0 x 877 [15]
+ CRUSH rule 0 x 878 [7]
+ CRUSH rule 0 x 879 [12]
+ CRUSH rule 0 x 880 [2]
+ CRUSH rule 0 x 881 [6]
+ CRUSH rule 0 x 882 [11]
+ CRUSH rule 0 x 883 [13]
+ CRUSH rule 0 x 884 [6]
+ CRUSH rule 0 x 885 [14]
+ CRUSH rule 0 x 886 [13]
+ CRUSH rule 0 x 887 [14]
+ CRUSH rule 0 x 888 [10]
+ CRUSH rule 0 x 889 [15]
+ CRUSH rule 0 x 890 [10]
+ CRUSH rule 0 x 891 [9]
+ CRUSH rule 0 x 892 [12]
+ CRUSH rule 0 x 893 [1]
+ CRUSH rule 0 x 894 [7]
+ CRUSH rule 0 x 895 [2]
+ CRUSH rule 0 x 896 [9]
+ CRUSH rule 0 x 897 [7]
+ CRUSH rule 0 x 898 [10]
+ CRUSH rule 0 x 899 [1]
+ CRUSH rule 0 x 900 [2]
+ CRUSH rule 0 x 901 [9]
+ CRUSH rule 0 x 902 [4]
+ CRUSH rule 0 x 903 [14]
+ CRUSH rule 0 x 904 [15]
+ CRUSH rule 0 x 905 [12]
+ CRUSH rule 0 x 906 [14]
+ CRUSH rule 0 x 907 [7]
+ CRUSH rule 0 x 908 [2]
+ CRUSH rule 0 x 909 [10]
+ CRUSH rule 0 x 910 [12]
+ CRUSH rule 0 x 911 [11]
+ CRUSH rule 0 x 912 [6]
+ CRUSH rule 0 x 913 [4]
+ CRUSH rule 0 x 914 [4]
+ CRUSH rule 0 x 915 [12]
+ CRUSH rule 0 x 916 [3]
+ CRUSH rule 0 x 917 [1]
+ CRUSH rule 0 x 918 [7]
+ CRUSH rule 0 x 919 [10]
+ CRUSH rule 0 x 920 [4]
+ CRUSH rule 0 x 921 [1]
+ CRUSH rule 0 x 922 [6]
+ CRUSH rule 0 x 923 [12]
+ CRUSH rule 0 x 924 [6]
+ CRUSH rule 0 x 925 [12]
+ CRUSH rule 0 x 926 [3]
+ CRUSH rule 0 x 927 [6]
+ CRUSH rule 0 x 928 [13]
+ CRUSH rule 0 x 929 [10]
+ CRUSH rule 0 x 930 [7]
+ CRUSH rule 0 x 931 [6]
+ CRUSH rule 0 x 932 [13]
+ CRUSH rule 0 x 933 [12]
+ CRUSH rule 0 x 934 [12]
+ CRUSH rule 0 x 935 [6]
+ CRUSH rule 0 x 936 [9]
+ CRUSH rule 0 x 937 [14]
+ CRUSH rule 0 x 938 [14]
+ CRUSH rule 0 x 939 [6]
+ CRUSH rule 0 x 940 [13]
+ CRUSH rule 0 x 941 [3]
+ CRUSH rule 0 x 942 [15]
+ CRUSH rule 0 x 943 [10]
+ CRUSH rule 0 x 944 [2]
+ CRUSH rule 0 x 945 [10]
+ CRUSH rule 0 x 946 [11]
+ CRUSH rule 0 x 947 [11]
+ CRUSH rule 0 x 948 [7]
+ CRUSH rule 0 x 949 [9]
+ CRUSH rule 0 x 950 [9]
+ CRUSH rule 0 x 951 [2]
+ CRUSH rule 0 x 952 [9]
+ CRUSH rule 0 x 953 [1]
+ CRUSH rule 0 x 954 [10]
+ CRUSH rule 0 x 955 [7]
+ CRUSH rule 0 x 956 [1]
+ CRUSH rule 0 x 957 [14]
+ CRUSH rule 0 x 958 [15]
+ CRUSH rule 0 x 959 [2]
+ CRUSH rule 0 x 960 [2]
+ CRUSH rule 0 x 961 [3]
+ CRUSH rule 0 x 962 [5]
+ CRUSH rule 0 x 963 [13]
+ CRUSH rule 0 x 964 [7]
+ CRUSH rule 0 x 965 [12]
+ CRUSH rule 0 x 966 [12]
+ CRUSH rule 0 x 967 [7]
+ CRUSH rule 0 x 968 [12]
+ CRUSH rule 0 x 969 [11]
+ CRUSH rule 0 x 970 [5]
+ CRUSH rule 0 x 971 [1]
+ CRUSH rule 0 x 972 [12]
+ CRUSH rule 0 x 973 [1]
+ CRUSH rule 0 x 974 [7]
+ CRUSH rule 0 x 975 [7]
+ CRUSH rule 0 x 976 [7]
+ CRUSH rule 0 x 977 [14]
+ CRUSH rule 0 x 978 [12]
+ CRUSH rule 0 x 979 [5]
+ CRUSH rule 0 x 980 [15]
+ CRUSH rule 0 x 981 [5]
+ CRUSH rule 0 x 982 [2]
+ CRUSH rule 0 x 983 [3]
+ CRUSH rule 0 x 984 [15]
+ CRUSH rule 0 x 985 [11]
+ CRUSH rule 0 x 986 [6]
+ CRUSH rule 0 x 987 [13]
+ CRUSH rule 0 x 988 [12]
+ CRUSH rule 0 x 989 [7]
+ CRUSH rule 0 x 990 [1]
+ CRUSH rule 0 x 991 [7]
+ CRUSH rule 0 x 992 [9]
+ CRUSH rule 0 x 993 [6]
+ CRUSH rule 0 x 994 [3]
+ CRUSH rule 0 x 995 [15]
+ CRUSH rule 0 x 996 [15]
+ CRUSH rule 0 x 997 [15]
+ CRUSH rule 0 x 998 [6]
+ CRUSH rule 0 x 999 [9]
+ CRUSH rule 0 x 1000 [14]
+ CRUSH rule 0 x 1001 [11]
+ CRUSH rule 0 x 1002 [1]
+ CRUSH rule 0 x 1003 [10]
+ CRUSH rule 0 x 1004 [15]
+ CRUSH rule 0 x 1005 [6]
+ CRUSH rule 0 x 1006 [10]
+ CRUSH rule 0 x 1007 [1]
+ CRUSH rule 0 x 1008 [7]
+ CRUSH rule 0 x 1009 [5]
+ CRUSH rule 0 x 1010 [10]
+ CRUSH rule 0 x 1011 [6]
+ CRUSH rule 0 x 1012 [12]
+ CRUSH rule 0 x 1013 [2]
+ CRUSH rule 0 x 1014 [1]
+ CRUSH rule 0 x 1015 [12]
+ CRUSH rule 0 x 1016 [10]
+ CRUSH rule 0 x 1017 [5]
+ CRUSH rule 0 x 1018 [13]
+ CRUSH rule 0 x 1019 [10]
+ CRUSH rule 0 x 1020 [3]
+ CRUSH rule 0 x 1021 [2]
+ CRUSH rule 0 x 1022 [15]
+ CRUSH rule 0 x 1023 [15]
+ rule 0 (replicated_ruleset) num_rep 1 result size == 1:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [7,10]
+ CRUSH rule 0 x 1 [10,15]
+ CRUSH rule 0 x 2 [1,12]
+ CRUSH rule 0 x 3 [15,4]
+ CRUSH rule 0 x 4 [14,2]
+ CRUSH rule 0 x 5 [7,4]
+ CRUSH rule 0 x 6 [12,6]
+ CRUSH rule 0 x 7 [9,2]
+ CRUSH rule 0 x 8 [10,2]
+ CRUSH rule 0 x 9 [7,1]
+ CRUSH rule 0 x 10 [10,14]
+ CRUSH rule 0 x 11 [13,9]
+ CRUSH rule 0 x 12 [7,1]
+ CRUSH rule 0 x 13 [3,5]
+ CRUSH rule 0 x 14 [13,5]
+ CRUSH rule 0 x 15 [15,1]
+ CRUSH rule 0 x 16 [7,11]
+ CRUSH rule 0 x 17 [10,1]
+ CRUSH rule 0 x 18 [1,7]
+ CRUSH rule 0 x 19 [7,12]
+ CRUSH rule 0 x 20 [14,12]
+ CRUSH rule 0 x 21 [3,12]
+ CRUSH rule 0 x 22 [6,3]
+ CRUSH rule 0 x 23 [10,5]
+ CRUSH rule 0 x 24 [12,11]
+ CRUSH rule 0 x 25 [7,12]
+ CRUSH rule 0 x 26 [1,7]
+ CRUSH rule 0 x 27 [3,6]
+ CRUSH rule 0 x 28 [14,4]
+ CRUSH rule 0 x 29 [5,14]
+ CRUSH rule 0 x 30 [2,5]
+ CRUSH rule 0 x 31 [5,15]
+ CRUSH rule 0 x 32 [9,10]
+ CRUSH rule 0 x 33 [13,4]
+ CRUSH rule 0 x 34 [13,15]
+ CRUSH rule 0 x 35 [4,14]
+ CRUSH rule 0 x 36 [3,12]
+ CRUSH rule 0 x 37 [9,2]
+ CRUSH rule 0 x 38 [3,4]
+ CRUSH rule 0 x 39 [12,7]
+ CRUSH rule 0 x 40 [10,1]
+ CRUSH rule 0 x 41 [4,9]
+ CRUSH rule 0 x 42 [3,6]
+ CRUSH rule 0 x 43 [10,5]
+ CRUSH rule 0 x 44 [11,4]
+ CRUSH rule 0 x 45 [11,12]
+ CRUSH rule 0 x 46 [6,9]
+ CRUSH rule 0 x 47 [3,9]
+ CRUSH rule 0 x 48 [4,6]
+ CRUSH rule 0 x 49 [9,15]
+ CRUSH rule 0 x 50 [14,12]
+ CRUSH rule 0 x 51 [10,6]
+ CRUSH rule 0 x 52 [12,1]
+ CRUSH rule 0 x 53 [3,6]
+ CRUSH rule 0 x 54 [4,13]
+ CRUSH rule 0 x 55 [4,11]
+ CRUSH rule 0 x 56 [5,9]
+ CRUSH rule 0 x 57 [6,2]
+ CRUSH rule 0 x 58 [7,1]
+ CRUSH rule 0 x 59 [2,13]
+ CRUSH rule 0 x 60 [3,6]
+ CRUSH rule 0 x 61 [3,15]
+ CRUSH rule 0 x 62 [15,11]
+ CRUSH rule 0 x 63 [10,14]
+ CRUSH rule 0 x 64 [3,9]
+ CRUSH rule 0 x 65 [4,12]
+ CRUSH rule 0 x 66 [15,11]
+ CRUSH rule 0 x 67 [2,6]
+ CRUSH rule 0 x 68 [15,7]
+ CRUSH rule 0 x 69 [2,1]
+ CRUSH rule 0 x 70 [9,6]
+ CRUSH rule 0 x 71 [15,5]
+ CRUSH rule 0 x 72 [9,10]
+ CRUSH rule 0 x 73 [5,3]
+ CRUSH rule 0 x 74 [11,7]
+ CRUSH rule 0 x 75 [9,7]
+ CRUSH rule 0 x 76 [6,1]
+ CRUSH rule 0 x 77 [7,4]
+ CRUSH rule 0 x 78 [9,3]
+ CRUSH rule 0 x 79 [13,2]
+ CRUSH rule 0 x 80 [15,2]
+ CRUSH rule 0 x 81 [15,2]
+ CRUSH rule 0 x 82 [14,13]
+ CRUSH rule 0 x 83 [4,15]
+ CRUSH rule 0 x 84 [10,7]
+ CRUSH rule 0 x 85 [3,15]
+ CRUSH rule 0 x 86 [10,9]
+ CRUSH rule 0 x 87 [15,10]
+ CRUSH rule 0 x 88 [4,13]
+ CRUSH rule 0 x 89 [3,9]
+ CRUSH rule 0 x 90 [4,9]
+ CRUSH rule 0 x 91 [6,11]
+ CRUSH rule 0 x 92 [1,5]
+ CRUSH rule 0 x 93 [9,3]
+ CRUSH rule 0 x 94 [9,2]
+ CRUSH rule 0 x 95 [7,15]
+ CRUSH rule 0 x 96 [2,15]
+ CRUSH rule 0 x 97 [4,11]
+ CRUSH rule 0 x 98 [11,13]
+ CRUSH rule 0 x 99 [12,4]
+ CRUSH rule 0 x 100 [9,4]
+ CRUSH rule 0 x 101 [15,7]
+ CRUSH rule 0 x 102 [3,11]
+ CRUSH rule 0 x 103 [13,11]
+ CRUSH rule 0 x 104 [14,6]
+ CRUSH rule 0 x 105 [14,10]
+ CRUSH rule 0 x 106 [6,5]
+ CRUSH rule 0 x 107 [3,1]
+ CRUSH rule 0 x 108 [5,10]
+ CRUSH rule 0 x 109 [9,1]
+ CRUSH rule 0 x 110 [5,1]
+ CRUSH rule 0 x 111 [10,1]
+ CRUSH rule 0 x 112 [1,10]
+ CRUSH rule 0 x 113 [6,10]
+ CRUSH rule 0 x 114 [5,13]
+ CRUSH rule 0 x 115 [10,13]
+ CRUSH rule 0 x 116 [1,14]
+ CRUSH rule 0 x 117 [5,6]
+ CRUSH rule 0 x 118 [10,4]
+ CRUSH rule 0 x 119 [14,12]
+ CRUSH rule 0 x 120 [11,3]
+ CRUSH rule 0 x 121 [9,5]
+ CRUSH rule 0 x 122 [4,3]
+ CRUSH rule 0 x 123 [3,10]
+ CRUSH rule 0 x 124 [12,2]
+ CRUSH rule 0 x 125 [9,12]
+ CRUSH rule 0 x 126 [7,15]
+ CRUSH rule 0 x 127 [4,14]
+ CRUSH rule 0 x 128 [3,12]
+ CRUSH rule 0 x 129 [11,13]
+ CRUSH rule 0 x 130 [3,13]
+ CRUSH rule 0 x 131 [12,1]
+ CRUSH rule 0 x 132 [11,15]
+ CRUSH rule 0 x 133 [3,6]
+ CRUSH rule 0 x 134 [12,5]
+ CRUSH rule 0 x 135 [3,14]
+ CRUSH rule 0 x 136 [15,6]
+ CRUSH rule 0 x 137 [14,3]
+ CRUSH rule 0 x 138 [13,15]
+ CRUSH rule 0 x 139 [11,2]
+ CRUSH rule 0 x 140 [11,4]
+ CRUSH rule 0 x 141 [6,12]
+ CRUSH rule 0 x 142 [3,14]
+ CRUSH rule 0 x 143 [9,6]
+ CRUSH rule 0 x 144 [13,7]
+ CRUSH rule 0 x 145 [12,2]
+ CRUSH rule 0 x 146 [1,5]
+ CRUSH rule 0 x 147 [1,4]
+ CRUSH rule 0 x 148 [12,7]
+ CRUSH rule 0 x 149 [2,5]
+ CRUSH rule 0 x 150 [1,15]
+ CRUSH rule 0 x 151 [2,9]
+ CRUSH rule 0 x 152 [5,9]
+ CRUSH rule 0 x 153 [6,9]
+ CRUSH rule 0 x 154 [3,11]
+ CRUSH rule 0 x 155 [14,12]
+ CRUSH rule 0 x 156 [7,13]
+ CRUSH rule 0 x 157 [15,1]
+ CRUSH rule 0 x 158 [15,1]
+ CRUSH rule 0 x 159 [4,14]
+ CRUSH rule 0 x 160 [5,7]
+ CRUSH rule 0 x 161 [1,2]
+ CRUSH rule 0 x 162 [10,6]
+ CRUSH rule 0 x 163 [15,1]
+ CRUSH rule 0 x 164 [9,14]
+ CRUSH rule 0 x 165 [11,7]
+ CRUSH rule 0 x 166 [1,2]
+ CRUSH rule 0 x 167 [9,7]
+ CRUSH rule 0 x 168 [13,2]
+ CRUSH rule 0 x 169 [1,4]
+ CRUSH rule 0 x 170 [1,15]
+ CRUSH rule 0 x 171 [9,2]
+ CRUSH rule 0 x 172 [14,4]
+ CRUSH rule 0 x 173 [5,10]
+ CRUSH rule 0 x 174 [15,6]
+ CRUSH rule 0 x 175 [5,7]
+ CRUSH rule 0 x 176 [9,6]
+ CRUSH rule 0 x 177 [2,9]
+ CRUSH rule 0 x 178 [12,11]
+ CRUSH rule 0 x 179 [2,10]
+ CRUSH rule 0 x 180 [3,11]
+ CRUSH rule 0 x 181 [9,12]
+ CRUSH rule 0 x 182 [5,13]
+ CRUSH rule 0 x 183 [5,7]
+ CRUSH rule 0 x 184 [2,5]
+ CRUSH rule 0 x 185 [13,5]
+ CRUSH rule 0 x 186 [6,14]
+ CRUSH rule 0 x 187 [1,4]
+ CRUSH rule 0 x 188 [9,13]
+ CRUSH rule 0 x 189 [6,12]
+ CRUSH rule 0 x 190 [9,13]
+ CRUSH rule 0 x 191 [7,11]
+ CRUSH rule 0 x 192 [2,11]
+ CRUSH rule 0 x 193 [3,13]
+ CRUSH rule 0 x 194 [3,13]
+ CRUSH rule 0 x 195 [5,7]
+ CRUSH rule 0 x 196 [4,15]
+ CRUSH rule 0 x 197 [14,10]
+ CRUSH rule 0 x 198 [2,5]
+ CRUSH rule 0 x 199 [2,10]
+ CRUSH rule 0 x 200 [7,14]
+ CRUSH rule 0 x 201 [9,14]
+ CRUSH rule 0 x 202 [14,11]
+ CRUSH rule 0 x 203 [12,5]
+ CRUSH rule 0 x 204 [6,11]
+ CRUSH rule 0 x 205 [15,4]
+ CRUSH rule 0 x 206 [13,11]
+ CRUSH rule 0 x 207 [2,11]
+ CRUSH rule 0 x 208 [13,1]
+ CRUSH rule 0 x 209 [6,15]
+ CRUSH rule 0 x 210 [13,11]
+ CRUSH rule 0 x 211 [2,14]
+ CRUSH rule 0 x 212 [10,1]
+ CRUSH rule 0 x 213 [3,9]
+ CRUSH rule 0 x 214 [7,15]
+ CRUSH rule 0 x 215 [6,1]
+ CRUSH rule 0 x 216 [12,9]
+ CRUSH rule 0 x 217 [12,11]
+ CRUSH rule 0 x 218 [12,10]
+ CRUSH rule 0 x 219 [3,11]
+ CRUSH rule 0 x 220 [14,4]
+ CRUSH rule 0 x 221 [15,5]
+ CRUSH rule 0 x 222 [10,4]
+ CRUSH rule 0 x 223 [9,7]
+ CRUSH rule 0 x 224 [1,7]
+ CRUSH rule 0 x 225 [10,5]
+ CRUSH rule 0 x 226 [4,1]
+ CRUSH rule 0 x 227 [7,2]
+ CRUSH rule 0 x 228 [2,15]
+ CRUSH rule 0 x 229 [9,3]
+ CRUSH rule 0 x 230 [10,5]
+ CRUSH rule 0 x 231 [2,7]
+ CRUSH rule 0 x 232 [10,5]
+ CRUSH rule 0 x 233 [6,12]
+ CRUSH rule 0 x 234 [10,1]
+ CRUSH rule 0 x 235 [13,14]
+ CRUSH rule 0 x 236 [2,15]
+ CRUSH rule 0 x 237 [3,12]
+ CRUSH rule 0 x 238 [2,10]
+ CRUSH rule 0 x 239 [4,15]
+ CRUSH rule 0 x 240 [15,5]
+ CRUSH rule 0 x 241 [7,9]
+ CRUSH rule 0 x 242 [14,2]
+ CRUSH rule 0 x 243 [2,11]
+ CRUSH rule 0 x 244 [13,9]
+ CRUSH rule 0 x 245 [12,9]
+ CRUSH rule 0 x 246 [15,3]
+ CRUSH rule 0 x 247 [6,4]
+ CRUSH rule 0 x 248 [5,13]
+ CRUSH rule 0 x 249 [10,14]
+ CRUSH rule 0 x 250 [12,15]
+ CRUSH rule 0 x 251 [13,2]
+ CRUSH rule 0 x 252 [7,5]
+ CRUSH rule 0 x 253 [3,13]
+ CRUSH rule 0 x 254 [2,9]
+ CRUSH rule 0 x 255 [1,9]
+ CRUSH rule 0 x 256 [6,9]
+ CRUSH rule 0 x 257 [15,12]
+ CRUSH rule 0 x 258 [12,5]
+ CRUSH rule 0 x 259 [9,10]
+ CRUSH rule 0 x 260 [10,12]
+ CRUSH rule 0 x 261 [13,7]
+ CRUSH rule 0 x 262 [15,3]
+ CRUSH rule 0 x 263 [12,6]
+ CRUSH rule 0 x 264 [13,14]
+ CRUSH rule 0 x 265 [12,10]
+ CRUSH rule 0 x 266 [14,7]
+ CRUSH rule 0 x 267 [12,11]
+ CRUSH rule 0 x 268 [4,1]
+ CRUSH rule 0 x 269 [11,1]
+ CRUSH rule 0 x 270 [7,11]
+ CRUSH rule 0 x 271 [4,7]
+ CRUSH rule 0 x 272 [15,5]
+ CRUSH rule 0 x 273 [2,10]
+ CRUSH rule 0 x 274 [10,2]
+ CRUSH rule 0 x 275 [10,3]
+ CRUSH rule 0 x 276 [5,12]
+ CRUSH rule 0 x 277 [14,3]
+ CRUSH rule 0 x 278 [5,6]
+ CRUSH rule 0 x 279 [6,10]
+ CRUSH rule 0 x 280 [7,3]
+ CRUSH rule 0 x 281 [5,11]
+ CRUSH rule 0 x 282 [2,1]
+ CRUSH rule 0 x 283 [4,1]
+ CRUSH rule 0 x 284 [5,11]
+ CRUSH rule 0 x 285 [15,5]
+ CRUSH rule 0 x 286 [10,4]
+ CRUSH rule 0 x 287 [12,4]
+ CRUSH rule 0 x 288 [4,12]
+ CRUSH rule 0 x 289 [2,5]
+ CRUSH rule 0 x 290 [12,2]
+ CRUSH rule 0 x 291 [7,11]
+ CRUSH rule 0 x 292 [4,10]
+ CRUSH rule 0 x 293 [6,5]
+ CRUSH rule 0 x 294 [9,12]
+ CRUSH rule 0 x 295 [6,10]
+ CRUSH rule 0 x 296 [3,1]
+ CRUSH rule 0 x 297 [6,13]
+ CRUSH rule 0 x 298 [14,9]
+ CRUSH rule 0 x 299 [14,12]
+ CRUSH rule 0 x 300 [15,7]
+ CRUSH rule 0 x 301 [9,11]
+ CRUSH rule 0 x 302 [9,7]
+ CRUSH rule 0 x 303 [4,13]
+ CRUSH rule 0 x 304 [6,9]
+ CRUSH rule 0 x 305 [13,7]
+ CRUSH rule 0 x 306 [10,12]
+ CRUSH rule 0 x 307 [11,12]
+ CRUSH rule 0 x 308 [12,14]
+ CRUSH rule 0 x 309 [9,3]
+ CRUSH rule 0 x 310 [3,1]
+ CRUSH rule 0 x 311 [3,9]
+ CRUSH rule 0 x 312 [15,13]
+ CRUSH rule 0 x 313 [9,15]
+ CRUSH rule 0 x 314 [2,15]
+ CRUSH rule 0 x 315 [15,2]
+ CRUSH rule 0 x 316 [4,9]
+ CRUSH rule 0 x 317 [1,5]
+ CRUSH rule 0 x 318 [4,1]
+ CRUSH rule 0 x 319 [2,15]
+ CRUSH rule 0 x 320 [5,7]
+ CRUSH rule 0 x 321 [1,6]
+ CRUSH rule 0 x 322 [13,7]
+ CRUSH rule 0 x 323 [7,4]
+ CRUSH rule 0 x 324 [5,6]
+ CRUSH rule 0 x 325 [9,10]
+ CRUSH rule 0 x 326 [11,7]
+ CRUSH rule 0 x 327 [12,5]
+ CRUSH rule 0 x 328 [5,2]
+ CRUSH rule 0 x 329 [2,6]
+ CRUSH rule 0 x 330 [3,9]
+ CRUSH rule 0 x 331 [12,14]
+ CRUSH rule 0 x 332 [10,12]
+ CRUSH rule 0 x 333 [6,5]
+ CRUSH rule 0 x 334 [4,9]
+ CRUSH rule 0 x 335 [11,7]
+ CRUSH rule 0 x 336 [6,14]
+ CRUSH rule 0 x 337 [15,11]
+ CRUSH rule 0 x 338 [10,5]
+ CRUSH rule 0 x 339 [11,14]
+ CRUSH rule 0 x 340 [11,6]
+ CRUSH rule 0 x 341 [7,5]
+ CRUSH rule 0 x 342 [12,14]
+ CRUSH rule 0 x 343 [12,14]
+ CRUSH rule 0 x 344 [9,11]
+ CRUSH rule 0 x 345 [14,2]
+ CRUSH rule 0 x 346 [5,3]
+ CRUSH rule 0 x 347 [10,2]
+ CRUSH rule 0 x 348 [7,9]
+ CRUSH rule 0 x 349 [9,6]
+ CRUSH rule 0 x 350 [13,9]
+ CRUSH rule 0 x 351 [13,5]
+ CRUSH rule 0 x 352 [1,12]
+ CRUSH rule 0 x 353 [10,14]
+ CRUSH rule 0 x 354 [6,3]
+ CRUSH rule 0 x 355 [13,14]
+ CRUSH rule 0 x 356 [15,13]
+ CRUSH rule 0 x 357 [4,11]
+ CRUSH rule 0 x 358 [12,7]
+ CRUSH rule 0 x 359 [5,15]
+ CRUSH rule 0 x 360 [13,10]
+ CRUSH rule 0 x 361 [5,3]
+ CRUSH rule 0 x 362 [2,9]
+ CRUSH rule 0 x 363 [7,12]
+ CRUSH rule 0 x 364 [2,12]
+ CRUSH rule 0 x 365 [13,5]
+ CRUSH rule 0 x 366 [12,7]
+ CRUSH rule 0 x 367 [7,13]
+ CRUSH rule 0 x 368 [7,9]
+ CRUSH rule 0 x 369 [7,5]
+ CRUSH rule 0 x 370 [4,7]
+ CRUSH rule 0 x 371 [1,7]
+ CRUSH rule 0 x 372 [10,4]
+ CRUSH rule 0 x 373 [15,5]
+ CRUSH rule 0 x 374 [3,15]
+ CRUSH rule 0 x 375 [5,2]
+ CRUSH rule 0 x 376 [5,14]
+ CRUSH rule 0 x 377 [1,15]
+ CRUSH rule 0 x 378 [9,12]
+ CRUSH rule 0 x 379 [11,2]
+ CRUSH rule 0 x 380 [6,1]
+ CRUSH rule 0 x 381 [15,13]
+ CRUSH rule 0 x 382 [14,3]
+ CRUSH rule 0 x 383 [3,6]
+ CRUSH rule 0 x 384 [4,13]
+ CRUSH rule 0 x 385 [4,6]
+ CRUSH rule 0 x 386 [14,3]
+ CRUSH rule 0 x 387 [1,11]
+ CRUSH rule 0 x 388 [2,6]
+ CRUSH rule 0 x 389 [12,7]
+ CRUSH rule 0 x 390 [2,11]
+ CRUSH rule 0 x 391 [3,4]
+ CRUSH rule 0 x 392 [11,5]
+ CRUSH rule 0 x 393 [2,14]
+ CRUSH rule 0 x 394 [4,9]
+ CRUSH rule 0 x 395 [10,13]
+ CRUSH rule 0 x 396 [2,12]
+ CRUSH rule 0 x 397 [1,14]
+ CRUSH rule 0 x 398 [9,2]
+ CRUSH rule 0 x 399 [5,9]
+ CRUSH rule 0 x 400 [10,6]
+ CRUSH rule 0 x 401 [6,9]
+ CRUSH rule 0 x 402 [4,7]
+ CRUSH rule 0 x 403 [7,15]
+ CRUSH rule 0 x 404 [14,12]
+ CRUSH rule 0 x 405 [9,15]
+ CRUSH rule 0 x 406 [12,14]
+ CRUSH rule 0 x 407 [9,5]
+ CRUSH rule 0 x 408 [7,1]
+ CRUSH rule 0 x 409 [11,2]
+ CRUSH rule 0 x 410 [6,4]
+ CRUSH rule 0 x 411 [13,11]
+ CRUSH rule 0 x 412 [5,9]
+ CRUSH rule 0 x 413 [13,5]
+ CRUSH rule 0 x 414 [3,11]
+ CRUSH rule 0 x 415 [6,10]
+ CRUSH rule 0 x 416 [13,1]
+ CRUSH rule 0 x 417 [4,12]
+ CRUSH rule 0 x 418 [14,5]
+ CRUSH rule 0 x 419 [5,14]
+ CRUSH rule 0 x 420 [2,4]
+ CRUSH rule 0 x 421 [15,4]
+ CRUSH rule 0 x 422 [4,11]
+ CRUSH rule 0 x 423 [3,15]
+ CRUSH rule 0 x 424 [6,10]
+ CRUSH rule 0 x 425 [11,15]
+ CRUSH rule 0 x 426 [12,4]
+ CRUSH rule 0 x 427 [14,10]
+ CRUSH rule 0 x 428 [12,7]
+ CRUSH rule 0 x 429 [3,4]
+ CRUSH rule 0 x 430 [3,5]
+ CRUSH rule 0 x 431 [9,3]
+ CRUSH rule 0 x 432 [4,1]
+ CRUSH rule 0 x 433 [4,11]
+ CRUSH rule 0 x 434 [2,14]
+ CRUSH rule 0 x 435 [13,11]
+ CRUSH rule 0 x 436 [9,15]
+ CRUSH rule 0 x 437 [9,6]
+ CRUSH rule 0 x 438 [7,2]
+ CRUSH rule 0 x 439 [7,14]
+ CRUSH rule 0 x 440 [14,11]
+ CRUSH rule 0 x 441 [2,4]
+ CRUSH rule 0 x 442 [10,13]
+ CRUSH rule 0 x 443 [12,15]
+ CRUSH rule 0 x 444 [4,13]
+ CRUSH rule 0 x 445 [4,2]
+ CRUSH rule 0 x 446 [12,10]
+ CRUSH rule 0 x 447 [15,7]
+ CRUSH rule 0 x 448 [5,2]
+ CRUSH rule 0 x 449 [14,5]
+ CRUSH rule 0 x 450 [2,4]
+ CRUSH rule 0 x 451 [6,14]
+ CRUSH rule 0 x 452 [14,9]
+ CRUSH rule 0 x 453 [5,15]
+ CRUSH rule 0 x 454 [10,4]
+ CRUSH rule 0 x 455 [6,13]
+ CRUSH rule 0 x 456 [5,7]
+ CRUSH rule 0 x 457 [9,1]
+ CRUSH rule 0 x 458 [9,11]
+ CRUSH rule 0 x 459 [13,15]
+ CRUSH rule 0 x 460 [5,12]
+ CRUSH rule 0 x 461 [4,3]
+ CRUSH rule 0 x 462 [4,7]
+ CRUSH rule 0 x 463 [4,12]
+ CRUSH rule 0 x 464 [4,2]
+ CRUSH rule 0 x 465 [5,10]
+ CRUSH rule 0 x 466 [13,5]
+ CRUSH rule 0 x 467 [13,6]
+ CRUSH rule 0 x 468 [10,7]
+ CRUSH rule 0 x 469 [4,9]
+ CRUSH rule 0 x 470 [3,9]
+ CRUSH rule 0 x 471 [6,1]
+ CRUSH rule 0 x 472 [2,14]
+ CRUSH rule 0 x 473 [15,10]
+ CRUSH rule 0 x 474 [15,10]
+ CRUSH rule 0 x 475 [10,5]
+ CRUSH rule 0 x 476 [3,6]
+ CRUSH rule 0 x 477 [6,13]
+ CRUSH rule 0 x 478 [4,15]
+ CRUSH rule 0 x 479 [13,11]
+ CRUSH rule 0 x 480 [1,13]
+ CRUSH rule 0 x 481 [15,12]
+ CRUSH rule 0 x 482 [2,12]
+ CRUSH rule 0 x 483 [10,1]
+ CRUSH rule 0 x 484 [1,4]
+ CRUSH rule 0 x 485 [9,4]
+ CRUSH rule 0 x 486 [3,10]
+ CRUSH rule 0 x 487 [12,11]
+ CRUSH rule 0 x 488 [14,4]
+ CRUSH rule 0 x 489 [11,4]
+ CRUSH rule 0 x 490 [4,9]
+ CRUSH rule 0 x 491 [1,12]
+ CRUSH rule 0 x 492 [5,7]
+ CRUSH rule 0 x 493 [12,1]
+ CRUSH rule 0 x 494 [1,7]
+ CRUSH rule 0 x 495 [3,15]
+ CRUSH rule 0 x 496 [5,3]
+ CRUSH rule 0 x 497 [13,10]
+ CRUSH rule 0 x 498 [10,6]
+ CRUSH rule 0 x 499 [14,3]
+ CRUSH rule 0 x 500 [15,9]
+ CRUSH rule 0 x 501 [10,13]
+ CRUSH rule 0 x 502 [5,1]
+ CRUSH rule 0 x 503 [15,10]
+ CRUSH rule 0 x 504 [13,2]
+ CRUSH rule 0 x 505 [12,7]
+ CRUSH rule 0 x 506 [11,7]
+ CRUSH rule 0 x 507 [4,14]
+ CRUSH rule 0 x 508 [12,1]
+ CRUSH rule 0 x 509 [4,2]
+ CRUSH rule 0 x 510 [5,3]
+ CRUSH rule 0 x 511 [2,12]
+ CRUSH rule 0 x 512 [15,11]
+ CRUSH rule 0 x 513 [4,9]
+ CRUSH rule 0 x 514 [11,9]
+ CRUSH rule 0 x 515 [12,14]
+ CRUSH rule 0 x 516 [14,11]
+ CRUSH rule 0 x 517 [11,5]
+ CRUSH rule 0 x 518 [3,5]
+ CRUSH rule 0 x 519 [12,14]
+ CRUSH rule 0 x 520 [12,4]
+ CRUSH rule 0 x 521 [11,5]
+ CRUSH rule 0 x 522 [4,12]
+ CRUSH rule 0 x 523 [3,1]
+ CRUSH rule 0 x 524 [15,9]
+ CRUSH rule 0 x 525 [3,15]
+ CRUSH rule 0 x 526 [10,2]
+ CRUSH rule 0 x 527 [3,13]
+ CRUSH rule 0 x 528 [12,7]
+ CRUSH rule 0 x 529 [6,4]
+ CRUSH rule 0 x 530 [11,9]
+ CRUSH rule 0 x 531 [9,15]
+ CRUSH rule 0 x 532 [5,3]
+ CRUSH rule 0 x 533 [12,15]
+ CRUSH rule 0 x 534 [11,9]
+ CRUSH rule 0 x 535 [11,1]
+ CRUSH rule 0 x 536 [9,1]
+ CRUSH rule 0 x 537 [15,5]
+ CRUSH rule 0 x 538 [13,5]
+ CRUSH rule 0 x 539 [10,12]
+ CRUSH rule 0 x 540 [12,15]
+ CRUSH rule 0 x 541 [2,1]
+ CRUSH rule 0 x 542 [3,9]
+ CRUSH rule 0 x 543 [4,10]
+ CRUSH rule 0 x 544 [3,15]
+ CRUSH rule 0 x 545 [14,10]
+ CRUSH rule 0 x 546 [5,15]
+ CRUSH rule 0 x 547 [5,13]
+ CRUSH rule 0 x 548 [11,7]
+ CRUSH rule 0 x 549 [14,1]
+ CRUSH rule 0 x 550 [9,15]
+ CRUSH rule 0 x 551 [11,2]
+ CRUSH rule 0 x 552 [2,11]
+ CRUSH rule 0 x 553 [11,9]
+ CRUSH rule 0 x 554 [11,14]
+ CRUSH rule 0 x 555 [6,5]
+ CRUSH rule 0 x 556 [15,6]
+ CRUSH rule 0 x 557 [12,2]
+ CRUSH rule 0 x 558 [12,1]
+ CRUSH rule 0 x 559 [2,13]
+ CRUSH rule 0 x 560 [4,9]
+ CRUSH rule 0 x 561 [12,7]
+ CRUSH rule 0 x 562 [7,13]
+ CRUSH rule 0 x 563 [15,4]
+ CRUSH rule 0 x 564 [2,13]
+ CRUSH rule 0 x 565 [3,12]
+ CRUSH rule 0 x 566 [6,14]
+ CRUSH rule 0 x 567 [15,4]
+ CRUSH rule 0 x 568 [4,14]
+ CRUSH rule 0 x 569 [11,3]
+ CRUSH rule 0 x 570 [1,10]
+ CRUSH rule 0 x 571 [10,12]
+ CRUSH rule 0 x 572 [12,14]
+ CRUSH rule 0 x 573 [7,15]
+ CRUSH rule 0 x 574 [11,14]
+ CRUSH rule 0 x 575 [5,13]
+ CRUSH rule 0 x 576 [3,15]
+ CRUSH rule 0 x 577 [13,9]
+ CRUSH rule 0 x 578 [4,10]
+ CRUSH rule 0 x 579 [13,1]
+ CRUSH rule 0 x 580 [3,12]
+ CRUSH rule 0 x 581 [7,14]
+ CRUSH rule 0 x 582 [10,5]
+ CRUSH rule 0 x 583 [4,15]
+ CRUSH rule 0 x 584 [10,1]
+ CRUSH rule 0 x 585 [5,3]
+ CRUSH rule 0 x 586 [7,10]
+ CRUSH rule 0 x 587 [11,6]
+ CRUSH rule 0 x 588 [3,12]
+ CRUSH rule 0 x 589 [9,7]
+ CRUSH rule 0 x 590 [12,1]
+ CRUSH rule 0 x 591 [2,6]
+ CRUSH rule 0 x 592 [15,12]
+ CRUSH rule 0 x 593 [13,14]
+ CRUSH rule 0 x 594 [12,14]
+ CRUSH rule 0 x 595 [12,7]
+ CRUSH rule 0 x 596 [2,7]
+ CRUSH rule 0 x 597 [15,1]
+ CRUSH rule 0 x 598 [11,5]
+ CRUSH rule 0 x 599 [13,11]
+ CRUSH rule 0 x 600 [4,12]
+ CRUSH rule 0 x 601 [13,5]
+ CRUSH rule 0 x 602 [3,11]
+ CRUSH rule 0 x 603 [3,1]
+ CRUSH rule 0 x 604 [14,2]
+ CRUSH rule 0 x 605 [2,7]
+ CRUSH rule 0 x 606 [12,15]
+ CRUSH rule 0 x 607 [3,9]
+ CRUSH rule 0 x 608 [13,10]
+ CRUSH rule 0 x 609 [14,3]
+ CRUSH rule 0 x 610 [7,10]
+ CRUSH rule 0 x 611 [13,1]
+ CRUSH rule 0 x 612 [7,1]
+ CRUSH rule 0 x 613 [10,7]
+ CRUSH rule 0 x 614 [9,4]
+ CRUSH rule 0 x 615 [9,4]
+ CRUSH rule 0 x 616 [10,14]
+ CRUSH rule 0 x 617 [15,7]
+ CRUSH rule 0 x 618 [4,2]
+ CRUSH rule 0 x 619 [15,4]
+ CRUSH rule 0 x 620 [3,7]
+ CRUSH rule 0 x 621 [3,6]
+ CRUSH rule 0 x 622 [10,2]
+ CRUSH rule 0 x 623 [4,9]
+ CRUSH rule 0 x 624 [3,9]
+ CRUSH rule 0 x 625 [11,7]
+ CRUSH rule 0 x 626 [10,12]
+ CRUSH rule 0 x 627 [1,12]
+ CRUSH rule 0 x 628 [15,13]
+ CRUSH rule 0 x 629 [5,6]
+ CRUSH rule 0 x 630 [1,4]
+ CRUSH rule 0 x 631 [5,7]
+ CRUSH rule 0 x 632 [12,3]
+ CRUSH rule 0 x 633 [14,4]
+ CRUSH rule 0 x 634 [6,9]
+ CRUSH rule 0 x 635 [6,5]
+ CRUSH rule 0 x 636 [13,6]
+ CRUSH rule 0 x 637 [3,1]
+ CRUSH rule 0 x 638 [10,15]
+ CRUSH rule 0 x 639 [6,9]
+ CRUSH rule 0 x 640 [9,6]
+ CRUSH rule 0 x 641 [10,6]
+ CRUSH rule 0 x 642 [1,15]
+ CRUSH rule 0 x 643 [3,7]
+ CRUSH rule 0 x 644 [15,13]
+ CRUSH rule 0 x 645 [14,2]
+ CRUSH rule 0 x 646 [5,13]
+ CRUSH rule 0 x 647 [10,1]
+ CRUSH rule 0 x 648 [6,5]
+ CRUSH rule 0 x 649 [3,9]
+ CRUSH rule 0 x 650 [10,9]
+ CRUSH rule 0 x 651 [3,9]
+ CRUSH rule 0 x 652 [15,9]
+ CRUSH rule 0 x 653 [11,14]
+ CRUSH rule 0 x 654 [13,6]
+ CRUSH rule 0 x 655 [6,3]
+ CRUSH rule 0 x 656 [3,15]
+ CRUSH rule 0 x 657 [11,15]
+ CRUSH rule 0 x 658 [7,2]
+ CRUSH rule 0 x 659 [2,5]
+ CRUSH rule 0 x 660 [13,14]
+ CRUSH rule 0 x 661 [7,15]
+ CRUSH rule 0 x 662 [15,2]
+ CRUSH rule 0 x 663 [14,9]
+ CRUSH rule 0 x 664 [6,10]
+ CRUSH rule 0 x 665 [2,9]
+ CRUSH rule 0 x 666 [12,3]
+ CRUSH rule 0 x 667 [1,9]
+ CRUSH rule 0 x 668 [9,5]
+ CRUSH rule 0 x 669 [9,7]
+ CRUSH rule 0 x 670 [6,10]
+ CRUSH rule 0 x 671 [6,15]
+ CRUSH rule 0 x 672 [2,9]
+ CRUSH rule 0 x 673 [7,10]
+ CRUSH rule 0 x 674 [7,12]
+ CRUSH rule 0 x 675 [9,5]
+ CRUSH rule 0 x 676 [10,12]
+ CRUSH rule 0 x 677 [2,12]
+ CRUSH rule 0 x 678 [1,2]
+ CRUSH rule 0 x 679 [5,6]
+ CRUSH rule 0 x 680 [7,11]
+ CRUSH rule 0 x 681 [6,4]
+ CRUSH rule 0 x 682 [6,1]
+ CRUSH rule 0 x 683 [6,13]
+ CRUSH rule 0 x 684 [9,11]
+ CRUSH rule 0 x 685 [5,1]
+ CRUSH rule 0 x 686 [1,9]
+ CRUSH rule 0 x 687 [7,13]
+ CRUSH rule 0 x 688 [11,9]
+ CRUSH rule 0 x 689 [5,2]
+ CRUSH rule 0 x 690 [9,7]
+ CRUSH rule 0 x 691 [11,15]
+ CRUSH rule 0 x 692 [15,5]
+ CRUSH rule 0 x 693 [5,6]
+ CRUSH rule 0 x 694 [4,7]
+ CRUSH rule 0 x 695 [6,13]
+ CRUSH rule 0 x 696 [1,2]
+ CRUSH rule 0 x 697 [13,11]
+ CRUSH rule 0 x 698 [11,13]
+ CRUSH rule 0 x 699 [7,14]
+ CRUSH rule 0 x 700 [12,14]
+ CRUSH rule 0 x 701 [3,13]
+ CRUSH rule 0 x 702 [3,12]
+ CRUSH rule 0 x 703 [15,11]
+ CRUSH rule 0 x 704 [6,4]
+ CRUSH rule 0 x 705 [14,6]
+ CRUSH rule 0 x 706 [1,12]
+ CRUSH rule 0 x 707 [4,7]
+ CRUSH rule 0 x 708 [3,10]
+ CRUSH rule 0 x 709 [11,12]
+ CRUSH rule 0 x 710 [14,2]
+ CRUSH rule 0 x 711 [14,3]
+ CRUSH rule 0 x 712 [12,3]
+ CRUSH rule 0 x 713 [11,9]
+ CRUSH rule 0 x 714 [12,1]
+ CRUSH rule 0 x 715 [6,1]
+ CRUSH rule 0 x 716 [11,13]
+ CRUSH rule 0 x 717 [12,4]
+ CRUSH rule 0 x 718 [7,15]
+ CRUSH rule 0 x 719 [5,15]
+ CRUSH rule 0 x 720 [4,13]
+ CRUSH rule 0 x 721 [11,3]
+ CRUSH rule 0 x 722 [2,4]
+ CRUSH rule 0 x 723 [2,1]
+ CRUSH rule 0 x 724 [7,1]
+ CRUSH rule 0 x 725 [11,12]
+ CRUSH rule 0 x 726 [7,14]
+ CRUSH rule 0 x 727 [2,5]
+ CRUSH rule 0 x 728 [13,11]
+ CRUSH rule 0 x 729 [15,11]
+ CRUSH rule 0 x 730 [3,7]
+ CRUSH rule 0 x 731 [9,1]
+ CRUSH rule 0 x 732 [1,2]
+ CRUSH rule 0 x 733 [11,3]
+ CRUSH rule 0 x 734 [14,3]
+ CRUSH rule 0 x 735 [6,9]
+ CRUSH rule 0 x 736 [3,9]
+ CRUSH rule 0 x 737 [1,4]
+ CRUSH rule 0 x 738 [11,15]
+ CRUSH rule 0 x 739 [11,12]
+ CRUSH rule 0 x 740 [7,9]
+ CRUSH rule 0 x 741 [12,11]
+ CRUSH rule 0 x 742 [9,7]
+ CRUSH rule 0 x 743 [5,13]
+ CRUSH rule 0 x 744 [6,2]
+ CRUSH rule 0 x 745 [3,6]
+ CRUSH rule 0 x 746 [3,7]
+ CRUSH rule 0 x 747 [15,11]
+ CRUSH rule 0 x 748 [6,10]
+ CRUSH rule 0 x 749 [14,9]
+ CRUSH rule 0 x 750 [1,14]
+ CRUSH rule 0 x 751 [15,1]
+ CRUSH rule 0 x 752 [13,1]
+ CRUSH rule 0 x 753 [4,11]
+ CRUSH rule 0 x 754 [14,12]
+ CRUSH rule 0 x 755 [13,6]
+ CRUSH rule 0 x 756 [3,4]
+ CRUSH rule 0 x 757 [10,6]
+ CRUSH rule 0 x 758 [6,3]
+ CRUSH rule 0 x 759 [5,7]
+ CRUSH rule 0 x 760 [1,15]
+ CRUSH rule 0 x 761 [2,12]
+ CRUSH rule 0 x 762 [1,4]
+ CRUSH rule 0 x 763 [4,13]
+ CRUSH rule 0 x 764 [1,14]
+ CRUSH rule 0 x 765 [9,15]
+ CRUSH rule 0 x 766 [11,2]
+ CRUSH rule 0 x 767 [6,11]
+ CRUSH rule 0 x 768 [2,12]
+ CRUSH rule 0 x 769 [15,1]
+ CRUSH rule 0 x 770 [15,13]
+ CRUSH rule 0 x 771 [9,2]
+ CRUSH rule 0 x 772 [4,3]
+ CRUSH rule 0 x 773 [3,7]
+ CRUSH rule 0 x 774 [12,6]
+ CRUSH rule 0 x 775 [5,10]
+ CRUSH rule 0 x 776 [10,15]
+ CRUSH rule 0 x 777 [11,13]
+ CRUSH rule 0 x 778 [13,1]
+ CRUSH rule 0 x 779 [5,11]
+ CRUSH rule 0 x 780 [13,9]
+ CRUSH rule 0 x 781 [5,7]
+ CRUSH rule 0 x 782 [2,15]
+ CRUSH rule 0 x 783 [12,7]
+ CRUSH rule 0 x 784 [14,1]
+ CRUSH rule 0 x 785 [6,12]
+ CRUSH rule 0 x 786 [10,5]
+ CRUSH rule 0 x 787 [1,12]
+ CRUSH rule 0 x 788 [4,2]
+ CRUSH rule 0 x 789 [9,2]
+ CRUSH rule 0 x 790 [15,2]
+ CRUSH rule 0 x 791 [9,4]
+ CRUSH rule 0 x 792 [6,4]
+ CRUSH rule 0 x 793 [15,9]
+ CRUSH rule 0 x 794 [5,12]
+ CRUSH rule 0 x 795 [6,14]
+ CRUSH rule 0 x 796 [11,2]
+ CRUSH rule 0 x 797 [14,3]
+ CRUSH rule 0 x 798 [5,11]
+ CRUSH rule 0 x 799 [2,9]
+ CRUSH rule 0 x 800 [6,3]
+ CRUSH rule 0 x 801 [2,5]
+ CRUSH rule 0 x 802 [1,4]
+ CRUSH rule 0 x 803 [7,2]
+ CRUSH rule 0 x 804 [5,14]
+ CRUSH rule 0 x 805 [13,4]
+ CRUSH rule 0 x 806 [6,2]
+ CRUSH rule 0 x 807 [14,2]
+ CRUSH rule 0 x 808 [2,15]
+ CRUSH rule 0 x 809 [1,11]
+ CRUSH rule 0 x 810 [2,5]
+ CRUSH rule 0 x 811 [15,6]
+ CRUSH rule 0 x 812 [7,11]
+ CRUSH rule 0 x 813 [4,10]
+ CRUSH rule 0 x 814 [13,4]
+ CRUSH rule 0 x 815 [15,12]
+ CRUSH rule 0 x 816 [14,10]
+ CRUSH rule 0 x 817 [10,7]
+ CRUSH rule 0 x 818 [15,2]
+ CRUSH rule 0 x 819 [5,12]
+ CRUSH rule 0 x 820 [3,6]
+ CRUSH rule 0 x 821 [15,10]
+ CRUSH rule 0 x 822 [10,13]
+ CRUSH rule 0 x 823 [2,6]
+ CRUSH rule 0 x 824 [3,7]
+ CRUSH rule 0 x 825 [10,5]
+ CRUSH rule 0 x 826 [5,2]
+ CRUSH rule 0 x 827 [13,5]
+ CRUSH rule 0 x 828 [12,6]
+ CRUSH rule 0 x 829 [13,6]
+ CRUSH rule 0 x 830 [15,13]
+ CRUSH rule 0 x 831 [1,4]
+ CRUSH rule 0 x 832 [14,11]
+ CRUSH rule 0 x 833 [9,13]
+ CRUSH rule 0 x 834 [9,7]
+ CRUSH rule 0 x 835 [14,3]
+ CRUSH rule 0 x 836 [3,9]
+ CRUSH rule 0 x 837 [15,12]
+ CRUSH rule 0 x 838 [12,14]
+ CRUSH rule 0 x 839 [3,4]
+ CRUSH rule 0 x 840 [10,15]
+ CRUSH rule 0 x 841 [3,5]
+ CRUSH rule 0 x 842 [9,13]
+ CRUSH rule 0 x 843 [14,7]
+ CRUSH rule 0 x 844 [7,1]
+ CRUSH rule 0 x 845 [13,6]
+ CRUSH rule 0 x 846 [3,7]
+ CRUSH rule 0 x 847 [12,15]
+ CRUSH rule 0 x 848 [11,13]
+ CRUSH rule 0 x 849 [3,15]
+ CRUSH rule 0 x 850 [1,3]
+ CRUSH rule 0 x 851 [14,4]
+ CRUSH rule 0 x 852 [9,12]
+ CRUSH rule 0 x 853 [13,14]
+ CRUSH rule 0 x 854 [7,11]
+ CRUSH rule 0 x 855 [14,4]
+ CRUSH rule 0 x 856 [5,10]
+ CRUSH rule 0 x 857 [4,3]
+ CRUSH rule 0 x 858 [5,15]
+ CRUSH rule 0 x 859 [5,15]
+ CRUSH rule 0 x 860 [11,14]
+ CRUSH rule 0 x 861 [13,7]
+ CRUSH rule 0 x 862 [5,10]
+ CRUSH rule 0 x 863 [11,6]
+ CRUSH rule 0 x 864 [6,13]
+ CRUSH rule 0 x 865 [4,1]
+ CRUSH rule 0 x 866 [2,13]
+ CRUSH rule 0 x 867 [12,2]
+ CRUSH rule 0 x 868 [14,11]
+ CRUSH rule 0 x 869 [10,13]
+ CRUSH rule 0 x 870 [14,9]
+ CRUSH rule 0 x 871 [6,2]
+ CRUSH rule 0 x 872 [6,1]
+ CRUSH rule 0 x 873 [2,5]
+ CRUSH rule 0 x 874 [12,4]
+ CRUSH rule 0 x 875 [10,6]
+ CRUSH rule 0 x 876 [14,7]
+ CRUSH rule 0 x 877 [15,11]
+ CRUSH rule 0 x 878 [7,14]
+ CRUSH rule 0 x 879 [12,2]
+ CRUSH rule 0 x 880 [2,12]
+ CRUSH rule 0 x 881 [6,3]
+ CRUSH rule 0 x 882 [11,13]
+ CRUSH rule 0 x 883 [13,1]
+ CRUSH rule 0 x 884 [6,15]
+ CRUSH rule 0 x 885 [14,7]
+ CRUSH rule 0 x 886 [13,11]
+ CRUSH rule 0 x 887 [14,4]
+ CRUSH rule 0 x 888 [10,12]
+ CRUSH rule 0 x 889 [15,13]
+ CRUSH rule 0 x 890 [10,12]
+ CRUSH rule 0 x 891 [9,5]
+ CRUSH rule 0 x 892 [12,15]
+ CRUSH rule 0 x 893 [1,3]
+ CRUSH rule 0 x 894 [7,2]
+ CRUSH rule 0 x 895 [2,1]
+ CRUSH rule 0 x 896 [9,1]
+ CRUSH rule 0 x 897 [7,5]
+ CRUSH rule 0 x 898 [10,6]
+ CRUSH rule 0 x 899 [1,11]
+ CRUSH rule 0 x 900 [2,9]
+ CRUSH rule 0 x 901 [9,12]
+ CRUSH rule 0 x 902 [4,2]
+ CRUSH rule 0 x 903 [14,10]
+ CRUSH rule 0 x 904 [15,12]
+ CRUSH rule 0 x 905 [12,6]
+ CRUSH rule 0 x 906 [14,11]
+ CRUSH rule 0 x 907 [7,12]
+ CRUSH rule 0 x 908 [2,15]
+ CRUSH rule 0 x 909 [10,14]
+ CRUSH rule 0 x 910 [12,7]
+ CRUSH rule 0 x 911 [11,15]
+ CRUSH rule 0 x 912 [6,4]
+ CRUSH rule 0 x 913 [4,6]
+ CRUSH rule 0 x 914 [4,15]
+ CRUSH rule 0 x 915 [12,14]
+ CRUSH rule 0 x 916 [3,1]
+ CRUSH rule 0 x 917 [1,15]
+ CRUSH rule 0 x 918 [7,14]
+ CRUSH rule 0 x 919 [10,7]
+ CRUSH rule 0 x 920 [4,2]
+ CRUSH rule 0 x 921 [1,11]
+ CRUSH rule 0 x 922 [6,4]
+ CRUSH rule 0 x 923 [12,2]
+ CRUSH rule 0 x 924 [6,2]
+ CRUSH rule 0 x 925 [12,15]
+ CRUSH rule 0 x 926 [3,13]
+ CRUSH rule 0 x 927 [6,5]
+ CRUSH rule 0 x 928 [13,1]
+ CRUSH rule 0 x 929 [10,7]
+ CRUSH rule 0 x 930 [7,15]
+ CRUSH rule 0 x 931 [6,15]
+ CRUSH rule 0 x 932 [13,2]
+ CRUSH rule 0 x 933 [12,7]
+ CRUSH rule 0 x 934 [12,2]
+ CRUSH rule 0 x 935 [6,11]
+ CRUSH rule 0 x 936 [9,12]
+ CRUSH rule 0 x 937 [14,2]
+ CRUSH rule 0 x 938 [14,3]
+ CRUSH rule 0 x 939 [6,4]
+ CRUSH rule 0 x 940 [13,11]
+ CRUSH rule 0 x 941 [3,12]
+ CRUSH rule 0 x 942 [15,12]
+ CRUSH rule 0 x 943 [10,2]
+ CRUSH rule 0 x 944 [2,9]
+ CRUSH rule 0 x 945 [10,15]
+ CRUSH rule 0 x 946 [11,15]
+ CRUSH rule 0 x 947 [11,3]
+ CRUSH rule 0 x 948 [7,13]
+ CRUSH rule 0 x 949 [9,1]
+ CRUSH rule 0 x 950 [9,15]
+ CRUSH rule 0 x 951 [2,6]
+ CRUSH rule 0 x 952 [9,7]
+ CRUSH rule 0 x 953 [1,3]
+ CRUSH rule 0 x 954 [10,2]
+ CRUSH rule 0 x 955 [7,14]
+ CRUSH rule 0 x 956 [1,6]
+ CRUSH rule 0 x 957 [14,11]
+ CRUSH rule 0 x 958 [15,4]
+ CRUSH rule 0 x 959 [2,1]
+ CRUSH rule 0 x 960 [2,6]
+ CRUSH rule 0 x 961 [3,13]
+ CRUSH rule 0 x 962 [5,11]
+ CRUSH rule 0 x 963 [13,10]
+ CRUSH rule 0 x 964 [7,11]
+ CRUSH rule 0 x 965 [12,2]
+ CRUSH rule 0 x 966 [12,14]
+ CRUSH rule 0 x 967 [7,5]
+ CRUSH rule 0 x 968 [12,15]
+ CRUSH rule 0 x 969 [11,4]
+ CRUSH rule 0 x 970 [5,12]
+ CRUSH rule 0 x 971 [1,9]
+ CRUSH rule 0 x 972 [12,3]
+ CRUSH rule 0 x 973 [1,10]
+ CRUSH rule 0 x 974 [7,11]
+ CRUSH rule 0 x 975 [7,9]
+ CRUSH rule 0 x 976 [7,3]
+ CRUSH rule 0 x 977 [14,3]
+ CRUSH rule 0 x 978 [12,5]
+ CRUSH rule 0 x 979 [5,1]
+ CRUSH rule 0 x 980 [15,11]
+ CRUSH rule 0 x 981 [5,11]
+ CRUSH rule 0 x 982 [2,6]
+ CRUSH rule 0 x 983 [3,12]
+ CRUSH rule 0 x 984 [15,13]
+ CRUSH rule 0 x 985 [11,2]
+ CRUSH rule 0 x 986 [6,13]
+ CRUSH rule 0 x 987 [13,14]
+ CRUSH rule 0 x 988 [12,9]
+ CRUSH rule 0 x 989 [7,4]
+ CRUSH rule 0 x 990 [1,10]
+ CRUSH rule 0 x 991 [7,11]
+ CRUSH rule 0 x 992 [9,10]
+ CRUSH rule 0 x 993 [6,10]
+ CRUSH rule 0 x 994 [3,13]
+ CRUSH rule 0 x 995 [15,6]
+ CRUSH rule 0 x 996 [15,10]
+ CRUSH rule 0 x 997 [15,2]
+ CRUSH rule 0 x 998 [6,1]
+ CRUSH rule 0 x 999 [9,10]
+ CRUSH rule 0 x 1000 [14,2]
+ CRUSH rule 0 x 1001 [11,14]
+ CRUSH rule 0 x 1002 [1,10]
+ CRUSH rule 0 x 1003 [10,7]
+ CRUSH rule 0 x 1004 [15,1]
+ CRUSH rule 0 x 1005 [6,12]
+ CRUSH rule 0 x 1006 [10,12]
+ CRUSH rule 0 x 1007 [1,7]
+ CRUSH rule 0 x 1008 [7,4]
+ CRUSH rule 0 x 1009 [5,2]
+ CRUSH rule 0 x 1010 [10,2]
+ CRUSH rule 0 x 1011 [6,3]
+ CRUSH rule 0 x 1012 [12,6]
+ CRUSH rule 0 x 1013 [2,14]
+ CRUSH rule 0 x 1014 [1,13]
+ CRUSH rule 0 x 1015 [12,6]
+ CRUSH rule 0 x 1016 [10,13]
+ CRUSH rule 0 x 1017 [5,11]
+ CRUSH rule 0 x 1018 [13,11]
+ CRUSH rule 0 x 1019 [10,13]
+ CRUSH rule 0 x 1020 [3,1]
+ CRUSH rule 0 x 1021 [2,11]
+ CRUSH rule 0 x 1022 [15,5]
+ CRUSH rule 0 x 1023 [15,2]
+ rule 0 (replicated_ruleset) num_rep 2 result size == 2:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3]
+ CRUSH rule 0 x 1 [10,15,1]
+ CRUSH rule 0 x 2 [1,12,2]
+ CRUSH rule 0 x 3 [15,4,10]
+ CRUSH rule 0 x 4 [14,2,10]
+ CRUSH rule 0 x 5 [7,4,11]
+ CRUSH rule 0 x 6 [12,6,10]
+ CRUSH rule 0 x 7 [9,2,6]
+ CRUSH rule 0 x 8 [10,2,15]
+ CRUSH rule 0 x 9 [7,1,14]
+ CRUSH rule 0 x 10 [10,14,4]
+ CRUSH rule 0 x 11 [13,9,14]
+ CRUSH rule 0 x 12 [7,1,2]
+ CRUSH rule 0 x 13 [3,5,12]
+ CRUSH rule 0 x 14 [13,5,2]
+ CRUSH rule 0 x 15 [15,1,9]
+ CRUSH rule 0 x 16 [7,11,14]
+ CRUSH rule 0 x 17 [10,1,13]
+ CRUSH rule 0 x 18 [1,7,3]
+ CRUSH rule 0 x 19 [7,12,2]
+ CRUSH rule 0 x 20 [14,12,3]
+ CRUSH rule 0 x 21 [3,12,1]
+ CRUSH rule 0 x 22 [6,3,13]
+ CRUSH rule 0 x 23 [10,5,13]
+ CRUSH rule 0 x 24 [12,11,3]
+ CRUSH rule 0 x 25 [7,12,15]
+ CRUSH rule 0 x 26 [1,7,13]
+ CRUSH rule 0 x 27 [3,6,15]
+ CRUSH rule 0 x 28 [14,4,3]
+ CRUSH rule 0 x 29 [5,14,12]
+ CRUSH rule 0 x 30 [2,5,6]
+ CRUSH rule 0 x 31 [5,15,10]
+ CRUSH rule 0 x 32 [9,10,2]
+ CRUSH rule 0 x 33 [13,4,9]
+ CRUSH rule 0 x 34 [13,15,2]
+ CRUSH rule 0 x 35 [4,14,3]
+ CRUSH rule 0 x 36 [3,12,9]
+ CRUSH rule 0 x 37 [9,2,6]
+ CRUSH rule 0 x 38 [3,4,13]
+ CRUSH rule 0 x 39 [12,7,14]
+ CRUSH rule 0 x 40 [10,1,9]
+ CRUSH rule 0 x 41 [4,9,11]
+ CRUSH rule 0 x 42 [3,6,14]
+ CRUSH rule 0 x 43 [10,5,15]
+ CRUSH rule 0 x 44 [11,4,13]
+ CRUSH rule 0 x 45 [11,12,15]
+ CRUSH rule 0 x 46 [6,9,2]
+ CRUSH rule 0 x 47 [3,9,6]
+ CRUSH rule 0 x 48 [4,6,2]
+ CRUSH rule 0 x 49 [9,15,10]
+ CRUSH rule 0 x 50 [14,12,1]
+ CRUSH rule 0 x 51 [10,6,5]
+ CRUSH rule 0 x 52 [12,1,9]
+ CRUSH rule 0 x 53 [3,6,13]
+ CRUSH rule 0 x 54 [4,13,9]
+ CRUSH rule 0 x 55 [4,11,2]
+ CRUSH rule 0 x 56 [5,9,10]
+ CRUSH rule 0 x 57 [6,2,1]
+ CRUSH rule 0 x 58 [7,1,11]
+ CRUSH rule 0 x 59 [2,13,1]
+ CRUSH rule 0 x 60 [3,6,11]
+ CRUSH rule 0 x 61 [3,15,13]
+ CRUSH rule 0 x 62 [15,11,7]
+ CRUSH rule 0 x 63 [10,14,12]
+ CRUSH rule 0 x 64 [3,9,1]
+ CRUSH rule 0 x 65 [4,12,11]
+ CRUSH rule 0 x 66 [15,11,6]
+ CRUSH rule 0 x 67 [2,6,4]
+ CRUSH rule 0 x 68 [15,7,4]
+ CRUSH rule 0 x 69 [2,1,15]
+ CRUSH rule 0 x 70 [9,6,1]
+ CRUSH rule 0 x 71 [15,5,1]
+ CRUSH rule 0 x 72 [9,10,3]
+ CRUSH rule 0 x 73 [5,3,11]
+ CRUSH rule 0 x 74 [11,7,9]
+ CRUSH rule 0 x 75 [9,7,11]
+ CRUSH rule 0 x 76 [6,1,3]
+ CRUSH rule 0 x 77 [7,4,2]
+ CRUSH rule 0 x 78 [9,3,1]
+ CRUSH rule 0 x 79 [13,2,15]
+ CRUSH rule 0 x 80 [15,2,6]
+ CRUSH rule 0 x 81 [15,2,1]
+ CRUSH rule 0 x 82 [14,13,5]
+ CRUSH rule 0 x 83 [4,15,3]
+ CRUSH rule 0 x 84 [10,7,9]
+ CRUSH rule 0 x 85 [3,15,9]
+ CRUSH rule 0 x 86 [10,9,14]
+ CRUSH rule 0 x 87 [15,10,7]
+ CRUSH rule 0 x 88 [4,13,3]
+ CRUSH rule 0 x 89 [3,9,7]
+ CRUSH rule 0 x 90 [4,9,7]
+ CRUSH rule 0 x 91 [6,11,9]
+ CRUSH rule 0 x 92 [1,5,10]
+ CRUSH rule 0 x 93 [9,3,15]
+ CRUSH rule 0 x 94 [9,2,12]
+ CRUSH rule 0 x 95 [7,15,4]
+ CRUSH rule 0 x 96 [2,15,11]
+ CRUSH rule 0 x 97 [4,11,2]
+ CRUSH rule 0 x 98 [11,13,9]
+ CRUSH rule 0 x 99 [12,4,11]
+ CRUSH rule 0 x 100 [9,4,10]
+ CRUSH rule 0 x 101 [15,7,1]
+ CRUSH rule 0 x 102 [3,11,14]
+ CRUSH rule 0 x 103 [13,11,6]
+ CRUSH rule 0 x 104 [14,6,3]
+ CRUSH rule 0 x 105 [14,10,1]
+ CRUSH rule 0 x 106 [6,5,13]
+ CRUSH rule 0 x 107 [3,1,10]
+ CRUSH rule 0 x 108 [5,10,7]
+ CRUSH rule 0 x 109 [9,1,13]
+ CRUSH rule 0 x 110 [5,1,11]
+ CRUSH rule 0 x 111 [10,1,9]
+ CRUSH rule 0 x 112 [1,10,4]
+ CRUSH rule 0 x 113 [6,10,13]
+ CRUSH rule 0 x 114 [5,13,6]
+ CRUSH rule 0 x 115 [10,13,14]
+ CRUSH rule 0 x 116 [1,14,13]
+ CRUSH rule 0 x 117 [5,6,1]
+ CRUSH rule 0 x 118 [10,4,13]
+ CRUSH rule 0 x 119 [14,12,11]
+ CRUSH rule 0 x 120 [11,3,14]
+ CRUSH rule 0 x 121 [9,5,1]
+ CRUSH rule 0 x 122 [4,3,14]
+ CRUSH rule 0 x 123 [3,10,5]
+ CRUSH rule 0 x 124 [12,2,1]
+ CRUSH rule 0 x 125 [9,12,15]
+ CRUSH rule 0 x 126 [7,15,10]
+ CRUSH rule 0 x 127 [4,14,9]
+ CRUSH rule 0 x 128 [3,12,1]
+ CRUSH rule 0 x 129 [11,13,14]
+ CRUSH rule 0 x 130 [3,13,5]
+ CRUSH rule 0 x 131 [12,1,6]
+ CRUSH rule 0 x 132 [11,15,13]
+ CRUSH rule 0 x 133 [3,6,9]
+ CRUSH rule 0 x 134 [12,5,6]
+ CRUSH rule 0 x 135 [3,14,12]
+ CRUSH rule 0 x 136 [15,6,9]
+ CRUSH rule 0 x 137 [14,3,6]
+ CRUSH rule 0 x 138 [13,15,4]
+ CRUSH rule 0 x 139 [11,2,13]
+ CRUSH rule 0 x 140 [11,4,12]
+ CRUSH rule 0 x 141 [6,12,15]
+ CRUSH rule 0 x 142 [3,14,7]
+ CRUSH rule 0 x 143 [9,6,4]
+ CRUSH rule 0 x 144 [13,7,11]
+ CRUSH rule 0 x 145 [12,2,6]
+ CRUSH rule 0 x 146 [1,5,9]
+ CRUSH rule 0 x 147 [1,4,9]
+ CRUSH rule 0 x 148 [12,7,9]
+ CRUSH rule 0 x 149 [2,5,9]
+ CRUSH rule 0 x 150 [1,15,2]
+ CRUSH rule 0 x 151 [2,9,14]
+ CRUSH rule 0 x 152 [5,9,2]
+ CRUSH rule 0 x 153 [6,9,4]
+ CRUSH rule 0 x 154 [3,11,7]
+ CRUSH rule 0 x 155 [14,12,7]
+ CRUSH rule 0 x 156 [7,13,3]
+ CRUSH rule 0 x 157 [15,1,6]
+ CRUSH rule 0 x 158 [15,1,10]
+ CRUSH rule 0 x 159 [4,14,3]
+ CRUSH rule 0 x 160 [5,7,3]
+ CRUSH rule 0 x 161 [1,2,11]
+ CRUSH rule 0 x 162 [10,6,1]
+ CRUSH rule 0 x 163 [15,1,10]
+ CRUSH rule 0 x 164 [9,14,10]
+ CRUSH rule 0 x 165 [11,7,2]
+ CRUSH rule 0 x 166 [1,2,12]
+ CRUSH rule 0 x 167 [9,7,3]
+ CRUSH rule 0 x 168 [13,2,4]
+ CRUSH rule 0 x 169 [1,4,9]
+ CRUSH rule 0 x 170 [1,15,7]
+ CRUSH rule 0 x 171 [9,2,10]
+ CRUSH rule 0 x 172 [14,4,10]
+ CRUSH rule 0 x 173 [5,10,12]
+ CRUSH rule 0 x 174 [15,6,4]
+ CRUSH rule 0 x 175 [5,7,9]
+ CRUSH rule 0 x 176 [9,6,3]
+ CRUSH rule 0 x 177 [2,9,10]
+ CRUSH rule 0 x 178 [12,11,7]
+ CRUSH rule 0 x 179 [2,10,13]
+ CRUSH rule 0 x 180 [3,11,5]
+ CRUSH rule 0 x 181 [9,12,6]
+ CRUSH rule 0 x 182 [5,13,11]
+ CRUSH rule 0 x 183 [5,7,10]
+ CRUSH rule 0 x 184 [2,5,11]
+ CRUSH rule 0 x 185 [13,5,7]
+ CRUSH rule 0 x 186 [6,14,13]
+ CRUSH rule 0 x 187 [1,4,11]
+ CRUSH rule 0 x 188 [9,13,5]
+ CRUSH rule 0 x 189 [6,12,4]
+ CRUSH rule 0 x 190 [9,13,15]
+ CRUSH rule 0 x 191 [7,11,4]
+ CRUSH rule 0 x 192 [2,11,5]
+ CRUSH rule 0 x 193 [3,13,6]
+ CRUSH rule 0 x 194 [3,13,4]
+ CRUSH rule 0 x 195 [5,7,10]
+ CRUSH rule 0 x 196 [4,15,1]
+ CRUSH rule 0 x 197 [14,10,13]
+ CRUSH rule 0 x 198 [2,5,6]
+ CRUSH rule 0 x 199 [2,10,4]
+ CRUSH rule 0 x 200 [7,14,11]
+ CRUSH rule 0 x 201 [9,14,1]
+ CRUSH rule 0 x 202 [14,11,7]
+ CRUSH rule 0 x 203 [12,5,7]
+ CRUSH rule 0 x 204 [6,11,3]
+ CRUSH rule 0 x 205 [15,4,6]
+ CRUSH rule 0 x 206 [13,11,2]
+ CRUSH rule 0 x 207 [2,11,7]
+ CRUSH rule 0 x 208 [13,1,6]
+ CRUSH rule 0 x 209 [6,15,13]
+ CRUSH rule 0 x 210 [13,11,2]
+ CRUSH rule 0 x 211 [2,14,1]
+ CRUSH rule 0 x 212 [10,1,12]
+ CRUSH rule 0 x 213 [3,9,6]
+ CRUSH rule 0 x 214 [7,15,4]
+ CRUSH rule 0 x 215 [6,1,4]
+ CRUSH rule 0 x 216 [12,9,6]
+ CRUSH rule 0 x 217 [12,11,1]
+ CRUSH rule 0 x 218 [12,10,15]
+ CRUSH rule 0 x 219 [3,11,14]
+ CRUSH rule 0 x 220 [14,4,3]
+ CRUSH rule 0 x 221 [15,5,2]
+ CRUSH rule 0 x 222 [10,4,3]
+ CRUSH rule 0 x 223 [9,7,11]
+ CRUSH rule 0 x 224 [1,7,10]
+ CRUSH rule 0 x 225 [10,5,2]
+ CRUSH rule 0 x 226 [4,1,9]
+ CRUSH rule 0 x 227 [7,2,12]
+ CRUSH rule 0 x 228 [2,15,11]
+ CRUSH rule 0 x 229 [9,3,7]
+ CRUSH rule 0 x 230 [10,5,7]
+ CRUSH rule 0 x 231 [2,7,5]
+ CRUSH rule 0 x 232 [10,5,13]
+ CRUSH rule 0 x 233 [6,12,11]
+ CRUSH rule 0 x 234 [10,1,2]
+ CRUSH rule 0 x 235 [13,14,7]
+ CRUSH rule 0 x 236 [2,15,9]
+ CRUSH rule 0 x 237 [3,12,9]
+ CRUSH rule 0 x 238 [2,10,4]
+ CRUSH rule 0 x 239 [4,15,10]
+ CRUSH rule 0 x 240 [15,5,13]
+ CRUSH rule 0 x 241 [7,9,15]
+ CRUSH rule 0 x 242 [14,2,6]
+ CRUSH rule 0 x 243 [2,11,5]
+ CRUSH rule 0 x 244 [13,9,15]
+ CRUSH rule 0 x 245 [12,9,15]
+ CRUSH rule 0 x 246 [15,3,5]
+ CRUSH rule 0 x 247 [6,4,9]
+ CRUSH rule 0 x 248 [5,13,7]
+ CRUSH rule 0 x 249 [10,14,7]
+ CRUSH rule 0 x 250 [12,15,1]
+ CRUSH rule 0 x 251 [13,2,15]
+ CRUSH rule 0 x 252 [7,5,13]
+ CRUSH rule 0 x 253 [3,13,15]
+ CRUSH rule 0 x 254 [2,9,13]
+ CRUSH rule 0 x 255 [1,9,13]
+ CRUSH rule 0 x 256 [6,9,13]
+ CRUSH rule 0 x 257 [15,12,3]
+ CRUSH rule 0 x 258 [12,5,6]
+ CRUSH rule 0 x 259 [9,10,4]
+ CRUSH rule 0 x 260 [10,12,6]
+ CRUSH rule 0 x 261 [13,7,2]
+ CRUSH rule 0 x 262 [15,3,12]
+ CRUSH rule 0 x 263 [12,6,10]
+ CRUSH rule 0 x 264 [13,14,11]
+ CRUSH rule 0 x 265 [12,10,14]
+ CRUSH rule 0 x 266 [14,7,11]
+ CRUSH rule 0 x 267 [12,11,6]
+ CRUSH rule 0 x 268 [4,1,15]
+ CRUSH rule 0 x 269 [11,1,15]
+ CRUSH rule 0 x 270 [7,11,12]
+ CRUSH rule 0 x 271 [4,7,3]
+ CRUSH rule 0 x 272 [15,5,13]
+ CRUSH rule 0 x 273 [2,10,7]
+ CRUSH rule 0 x 274 [10,2,5]
+ CRUSH rule 0 x 275 [10,3,4]
+ CRUSH rule 0 x 276 [5,12,9]
+ CRUSH rule 0 x 277 [14,3,13]
+ CRUSH rule 0 x 278 [5,6,14]
+ CRUSH rule 0 x 279 [6,10,13]
+ CRUSH rule 0 x 280 [7,3,14]
+ CRUSH rule 0 x 281 [5,11,14]
+ CRUSH rule 0 x 282 [2,1,13]
+ CRUSH rule 0 x 283 [4,1,12]
+ CRUSH rule 0 x 284 [5,11,7]
+ CRUSH rule 0 x 285 [15,5,3]
+ CRUSH rule 0 x 286 [10,4,3]
+ CRUSH rule 0 x 287 [12,4,9]
+ CRUSH rule 0 x 288 [4,12,10]
+ CRUSH rule 0 x 289 [2,5,14]
+ CRUSH rule 0 x 290 [12,2,5]
+ CRUSH rule 0 x 291 [7,11,1]
+ CRUSH rule 0 x 292 [4,10,6]
+ CRUSH rule 0 x 293 [6,5,11]
+ CRUSH rule 0 x 294 [9,12,3]
+ CRUSH rule 0 x 295 [6,10,3]
+ CRUSH rule 0 x 296 [3,1,13]
+ CRUSH rule 0 x 297 [6,13,4]
+ CRUSH rule 0 x 298 [14,9,13]
+ CRUSH rule 0 x 299 [14,12,11]
+ CRUSH rule 0 x 300 [15,7,10]
+ CRUSH rule 0 x 301 [9,11,7]
+ CRUSH rule 0 x 302 [9,7,1]
+ CRUSH rule 0 x 303 [4,13,3]
+ CRUSH rule 0 x 304 [6,9,2]
+ CRUSH rule 0 x 305 [13,7,5]
+ CRUSH rule 0 x 306 [10,12,4]
+ CRUSH rule 0 x 307 [11,12,15]
+ CRUSH rule 0 x 308 [12,14,10]
+ CRUSH rule 0 x 309 [9,3,12]
+ CRUSH rule 0 x 310 [3,1,5]
+ CRUSH rule 0 x 311 [3,9,7]
+ CRUSH rule 0 x 312 [15,13,9]
+ CRUSH rule 0 x 313 [9,15,3]
+ CRUSH rule 0 x 314 [2,15,9]
+ CRUSH rule 0 x 315 [15,2,13]
+ CRUSH rule 0 x 316 [4,9,11]
+ CRUSH rule 0 x 317 [1,5,3]
+ CRUSH rule 0 x 318 [4,1,15]
+ CRUSH rule 0 x 319 [2,15,4]
+ CRUSH rule 0 x 320 [5,7,13]
+ CRUSH rule 0 x 321 [1,6,11]
+ CRUSH rule 0 x 322 [13,7,5]
+ CRUSH rule 0 x 323 [7,4,10]
+ CRUSH rule 0 x 324 [5,6,10]
+ CRUSH rule 0 x 325 [9,10,14]
+ CRUSH rule 0 x 326 [11,7,13]
+ CRUSH rule 0 x 327 [12,5,10]
+ CRUSH rule 0 x 328 [5,2,6]
+ CRUSH rule 0 x 329 [2,6,15]
+ CRUSH rule 0 x 330 [3,9,11]
+ CRUSH rule 0 x 331 [12,14,6]
+ CRUSH rule 0 x 332 [10,12,6]
+ CRUSH rule 0 x 333 [6,5,3]
+ CRUSH rule 0 x 334 [4,9,2]
+ CRUSH rule 0 x 335 [11,7,1]
+ CRUSH rule 0 x 336 [6,14,13]
+ CRUSH rule 0 x 337 [15,11,3]
+ CRUSH rule 0 x 338 [10,5,3]
+ CRUSH rule 0 x 339 [11,14,13]
+ CRUSH rule 0 x 340 [11,6,12]
+ CRUSH rule 0 x 341 [7,5,2]
+ CRUSH rule 0 x 342 [12,14,1]
+ CRUSH rule 0 x 343 [12,14,9]
+ CRUSH rule 0 x 344 [9,11,5]
+ CRUSH rule 0 x 345 [14,2,11]
+ CRUSH rule 0 x 346 [5,3,14]
+ CRUSH rule 0 x 347 [10,2,12]
+ CRUSH rule 0 x 348 [7,9,10]
+ CRUSH rule 0 x 349 [9,6,10]
+ CRUSH rule 0 x 350 [13,9,15]
+ CRUSH rule 0 x 351 [13,5,15]
+ CRUSH rule 0 x 352 [1,12,11]
+ CRUSH rule 0 x 353 [10,14,12]
+ CRUSH rule 0 x 354 [6,3,15]
+ CRUSH rule 0 x 355 [13,14,6]
+ CRUSH rule 0 x 356 [15,13,2]
+ CRUSH rule 0 x 357 [4,11,1]
+ CRUSH rule 0 x 358 [12,7,2]
+ CRUSH rule 0 x 359 [5,15,7]
+ CRUSH rule 0 x 360 [13,10,1]
+ CRUSH rule 0 x 361 [5,3,13]
+ CRUSH rule 0 x 362 [2,9,11]
+ CRUSH rule 0 x 363 [7,12,3]
+ CRUSH rule 0 x 364 [2,12,6]
+ CRUSH rule 0 x 365 [13,5,11]
+ CRUSH rule 0 x 366 [12,7,3]
+ CRUSH rule 0 x 367 [7,13,3]
+ CRUSH rule 0 x 368 [7,9,10]
+ CRUSH rule 0 x 369 [7,5,3]
+ CRUSH rule 0 x 370 [4,7,14]
+ CRUSH rule 0 x 371 [1,7,12]
+ CRUSH rule 0 x 372 [10,4,3]
+ CRUSH rule 0 x 373 [15,5,2]
+ CRUSH rule 0 x 374 [3,15,12]
+ CRUSH rule 0 x 375 [5,2,14]
+ CRUSH rule 0 x 376 [5,14,10]
+ CRUSH rule 0 x 377 [1,15,2]
+ CRUSH rule 0 x 378 [9,12,2]
+ CRUSH rule 0 x 379 [11,2,15]
+ CRUSH rule 0 x 380 [6,1,12]
+ CRUSH rule 0 x 381 [15,13,7]
+ CRUSH rule 0 x 382 [14,3,1]
+ CRUSH rule 0 x 383 [3,6,11]
+ CRUSH rule 0 x 384 [4,13,6]
+ CRUSH rule 0 x 385 [4,6,15]
+ CRUSH rule 0 x 386 [14,3,11]
+ CRUSH rule 0 x 387 [1,11,5]
+ CRUSH rule 0 x 388 [2,6,11]
+ CRUSH rule 0 x 389 [12,7,2]
+ CRUSH rule 0 x 390 [2,11,13]
+ CRUSH rule 0 x 391 [3,4,9]
+ CRUSH rule 0 x 392 [11,5,14]
+ CRUSH rule 0 x 393 [2,14,5]
+ CRUSH rule 0 x 394 [4,9,3]
+ CRUSH rule 0 x 395 [10,13,5]
+ CRUSH rule 0 x 396 [2,12,15]
+ CRUSH rule 0 x 397 [1,14,9]
+ CRUSH rule 0 x 398 [9,2,1]
+ CRUSH rule 0 x 399 [5,9,14]
+ CRUSH rule 0 x 400 [10,6,2]
+ CRUSH rule 0 x 401 [6,9,11]
+ CRUSH rule 0 x 402 [4,7,9]
+ CRUSH rule 0 x 403 [7,15,13]
+ CRUSH rule 0 x 404 [14,12,7]
+ CRUSH rule 0 x 405 [9,15,11]
+ CRUSH rule 0 x 406 [12,14,9]
+ CRUSH rule 0 x 407 [9,5,12]
+ CRUSH rule 0 x 408 [7,1,5]
+ CRUSH rule 0 x 409 [11,2,4]
+ CRUSH rule 0 x 410 [6,4,14]
+ CRUSH rule 0 x 411 [13,11,15]
+ CRUSH rule 0 x 412 [5,9,6]
+ CRUSH rule 0 x 413 [13,5,3]
+ CRUSH rule 0 x 414 [3,11,9]
+ CRUSH rule 0 x 415 [6,10,14]
+ CRUSH rule 0 x 416 [13,1,4]
+ CRUSH rule 0 x 417 [4,12,1]
+ CRUSH rule 0 x 418 [14,5,10]
+ CRUSH rule 0 x 419 [5,14,10]
+ CRUSH rule 0 x 420 [2,4,9]
+ CRUSH rule 0 x 421 [15,4,10]
+ CRUSH rule 0 x 422 [4,11,2]
+ CRUSH rule 0 x 423 [3,15,12]
+ CRUSH rule 0 x 424 [6,10,12]
+ CRUSH rule 0 x 425 [11,15,2]
+ CRUSH rule 0 x 426 [12,4,7]
+ CRUSH rule 0 x 427 [14,10,3]
+ CRUSH rule 0 x 428 [12,7,9]
+ CRUSH rule 0 x 429 [3,4,9]
+ CRUSH rule 0 x 430 [3,5,10]
+ CRUSH rule 0 x 431 [9,3,7]
+ CRUSH rule 0 x 432 [4,1,12]
+ CRUSH rule 0 x 433 [4,11,12]
+ CRUSH rule 0 x 434 [2,14,9]
+ CRUSH rule 0 x 435 [13,11,5]
+ CRUSH rule 0 x 436 [9,15,10]
+ CRUSH rule 0 x 437 [9,6,3]
+ CRUSH rule 0 x 438 [7,2,13]
+ CRUSH rule 0 x 439 [7,14,4]
+ CRUSH rule 0 x 440 [14,11,9]
+ CRUSH rule 0 x 441 [2,4,11]
+ CRUSH rule 0 x 442 [10,13,9]
+ CRUSH rule 0 x 443 [12,15,10]
+ CRUSH rule 0 x 444 [4,13,7]
+ CRUSH rule 0 x 445 [4,2,15]
+ CRUSH rule 0 x 446 [12,10,6]
+ CRUSH rule 0 x 447 [15,7,13]
+ CRUSH rule 0 x 448 [5,2,13]
+ CRUSH rule 0 x 449 [14,5,3]
+ CRUSH rule 0 x 450 [2,4,6]
+ CRUSH rule 0 x 451 [6,14,11]
+ CRUSH rule 0 x 452 [14,9,10]
+ CRUSH rule 0 x 453 [5,15,13]
+ CRUSH rule 0 x 454 [10,4,2]
+ CRUSH rule 0 x 455 [6,13,2]
+ CRUSH rule 0 x 456 [5,7,13]
+ CRUSH rule 0 x 457 [9,1,5]
+ CRUSH rule 0 x 458 [9,11,15]
+ CRUSH rule 0 x 459 [13,15,11]
+ CRUSH rule 0 x 460 [5,12,10]
+ CRUSH rule 0 x 461 [4,3,9]
+ CRUSH rule 0 x 462 [4,7,12]
+ CRUSH rule 0 x 463 [4,12,14]
+ CRUSH rule 0 x 464 [4,2,15]
+ CRUSH rule 0 x 465 [5,10,9]
+ CRUSH rule 0 x 466 [13,5,2]
+ CRUSH rule 0 x 467 [13,6,14]
+ CRUSH rule 0 x 468 [10,7,12]
+ CRUSH rule 0 x 469 [4,9,6]
+ CRUSH rule 0 x 470 [3,9,12]
+ CRUSH rule 0 x 471 [6,1,5]
+ CRUSH rule 0 x 472 [2,14,7]
+ CRUSH rule 0 x 473 [15,10,6]
+ CRUSH rule 0 x 474 [15,10,4]
+ CRUSH rule 0 x 475 [10,5,12]
+ CRUSH rule 0 x 476 [3,6,10]
+ CRUSH rule 0 x 477 [6,13,5]
+ CRUSH rule 0 x 478 [4,15,1]
+ CRUSH rule 0 x 479 [13,11,1]
+ CRUSH rule 0 x 480 [1,13,6]
+ CRUSH rule 0 x 481 [15,12,7]
+ CRUSH rule 0 x 482 [2,12,9]
+ CRUSH rule 0 x 483 [10,1,4]
+ CRUSH rule 0 x 484 [1,4,10]
+ CRUSH rule 0 x 485 [9,4,3]
+ CRUSH rule 0 x 486 [3,10,15]
+ CRUSH rule 0 x 487 [12,11,4]
+ CRUSH rule 0 x 488 [14,4,1]
+ CRUSH rule 0 x 489 [11,4,2]
+ CRUSH rule 0 x 490 [4,9,1]
+ CRUSH rule 0 x 491 [1,12,5]
+ CRUSH rule 0 x 492 [5,7,11]
+ CRUSH rule 0 x 493 [12,1,4]
+ CRUSH rule 0 x 494 [1,7,13]
+ CRUSH rule 0 x 495 [3,15,7]
+ CRUSH rule 0 x 496 [5,3,7]
+ CRUSH rule 0 x 497 [13,10,3]
+ CRUSH rule 0 x 498 [10,6,1]
+ CRUSH rule 0 x 499 [14,3,12]
+ CRUSH rule 0 x 500 [15,9,6]
+ CRUSH rule 0 x 501 [10,13,1]
+ CRUSH rule 0 x 502 [5,1,14]
+ CRUSH rule 0 x 503 [15,10,7]
+ CRUSH rule 0 x 504 [13,2,7]
+ CRUSH rule 0 x 505 [12,7,5]
+ CRUSH rule 0 x 506 [11,7,9]
+ CRUSH rule 0 x 507 [4,14,13]
+ CRUSH rule 0 x 508 [12,1,4]
+ CRUSH rule 0 x 509 [4,2,6]
+ CRUSH rule 0 x 510 [5,3,1]
+ CRUSH rule 0 x 511 [2,12,10]
+ CRUSH rule 0 x 512 [15,11,3]
+ CRUSH rule 0 x 513 [4,9,11]
+ CRUSH rule 0 x 514 [11,9,3]
+ CRUSH rule 0 x 515 [12,14,6]
+ CRUSH rule 0 x 516 [14,11,1]
+ CRUSH rule 0 x 517 [11,5,6]
+ CRUSH rule 0 x 518 [3,5,7]
+ CRUSH rule 0 x 519 [12,14,2]
+ CRUSH rule 0 x 520 [12,4,2]
+ CRUSH rule 0 x 521 [11,5,9]
+ CRUSH rule 0 x 522 [4,12,11]
+ CRUSH rule 0 x 523 [3,1,5]
+ CRUSH rule 0 x 524 [15,9,3]
+ CRUSH rule 0 x 525 [3,15,11]
+ CRUSH rule 0 x 526 [10,2,5]
+ CRUSH rule 0 x 527 [3,13,4]
+ CRUSH rule 0 x 528 [12,7,15]
+ CRUSH rule 0 x 529 [6,4,10]
+ CRUSH rule 0 x 530 [11,9,12]
+ CRUSH rule 0 x 531 [9,15,4]
+ CRUSH rule 0 x 532 [5,3,13]
+ CRUSH rule 0 x 533 [12,15,1]
+ CRUSH rule 0 x 534 [11,9,3]
+ CRUSH rule 0 x 535 [11,1,3]
+ CRUSH rule 0 x 536 [9,1,14]
+ CRUSH rule 0 x 537 [15,5,13]
+ CRUSH rule 0 x 538 [13,5,11]
+ CRUSH rule 0 x 539 [10,12,6]
+ CRUSH rule 0 x 540 [12,15,7]
+ CRUSH rule 0 x 541 [2,1,6]
+ CRUSH rule 0 x 542 [3,9,15]
+ CRUSH rule 0 x 543 [4,10,9]
+ CRUSH rule 0 x 544 [3,15,9]
+ CRUSH rule 0 x 545 [14,10,7]
+ CRUSH rule 0 x 546 [5,15,13]
+ CRUSH rule 0 x 547 [5,13,7]
+ CRUSH rule 0 x 548 [11,7,12]
+ CRUSH rule 0 x 549 [14,1,4]
+ CRUSH rule 0 x 550 [9,15,3]
+ CRUSH rule 0 x 551 [11,2,15]
+ CRUSH rule 0 x 552 [2,11,14]
+ CRUSH rule 0 x 553 [11,9,14]
+ CRUSH rule 0 x 554 [11,14,6]
+ CRUSH rule 0 x 555 [6,5,10]
+ CRUSH rule 0 x 556 [15,6,3]
+ CRUSH rule 0 x 557 [12,2,5]
+ CRUSH rule 0 x 558 [12,1,6]
+ CRUSH rule 0 x 559 [2,13,5]
+ CRUSH rule 0 x 560 [4,9,12]
+ CRUSH rule 0 x 561 [12,7,1]
+ CRUSH rule 0 x 562 [7,13,9]
+ CRUSH rule 0 x 563 [15,4,3]
+ CRUSH rule 0 x 564 [2,13,7]
+ CRUSH rule 0 x 565 [3,12,4]
+ CRUSH rule 0 x 566 [6,14,4]
+ CRUSH rule 0 x 567 [15,4,11]
+ CRUSH rule 0 x 568 [4,14,1]
+ CRUSH rule 0 x 569 [11,3,15]
+ CRUSH rule 0 x 570 [1,10,13]
+ CRUSH rule 0 x 571 [10,12,14]
+ CRUSH rule 0 x 572 [12,14,3]
+ CRUSH rule 0 x 573 [7,15,11]
+ CRUSH rule 0 x 574 [11,14,13]
+ CRUSH rule 0 x 575 [5,13,15]
+ CRUSH rule 0 x 576 [3,15,11]
+ CRUSH rule 0 x 577 [13,9,6]
+ CRUSH rule 0 x 578 [4,10,1]
+ CRUSH rule 0 x 579 [13,1,15]
+ CRUSH rule 0 x 580 [3,12,4]
+ CRUSH rule 0 x 581 [7,14,12]
+ CRUSH rule 0 x 582 [10,5,13]
+ CRUSH rule 0 x 583 [4,15,1]
+ CRUSH rule 0 x 584 [10,1,5]
+ CRUSH rule 0 x 585 [5,3,6]
+ CRUSH rule 0 x 586 [7,10,14]
+ CRUSH rule 0 x 587 [11,6,9]
+ CRUSH rule 0 x 588 [3,12,7]
+ CRUSH rule 0 x 589 [9,7,12]
+ CRUSH rule 0 x 590 [12,1,3]
+ CRUSH rule 0 x 591 [2,6,14]
+ CRUSH rule 0 x 592 [15,12,9]
+ CRUSH rule 0 x 593 [13,14,5]
+ CRUSH rule 0 x 594 [12,14,2]
+ CRUSH rule 0 x 595 [12,7,10]
+ CRUSH rule 0 x 596 [2,7,12]
+ CRUSH rule 0 x 597 [15,1,2]
+ CRUSH rule 0 x 598 [11,5,9]
+ CRUSH rule 0 x 599 [13,11,1]
+ CRUSH rule 0 x 600 [4,12,3]
+ CRUSH rule 0 x 601 [13,5,15]
+ CRUSH rule 0 x 602 [3,11,7]
+ CRUSH rule 0 x 603 [3,1,4]
+ CRUSH rule 0 x 604 [14,2,6]
+ CRUSH rule 0 x 605 [2,7,12]
+ CRUSH rule 0 x 606 [12,15,1]
+ CRUSH rule 0 x 607 [3,9,10]
+ CRUSH rule 0 x 608 [13,10,1]
+ CRUSH rule 0 x 609 [14,3,7]
+ CRUSH rule 0 x 610 [7,10,5]
+ CRUSH rule 0 x 611 [13,1,5]
+ CRUSH rule 0 x 612 [7,1,2]
+ CRUSH rule 0 x 613 [10,7,14]
+ CRUSH rule 0 x 614 [9,4,15]
+ CRUSH rule 0 x 615 [9,4,11]
+ CRUSH rule 0 x 616 [10,14,1]
+ CRUSH rule 0 x 617 [15,7,2]
+ CRUSH rule 0 x 618 [4,2,10]
+ CRUSH rule 0 x 619 [15,4,3]
+ CRUSH rule 0 x 620 [3,7,11]
+ CRUSH rule 0 x 621 [3,6,4]
+ CRUSH rule 0 x 622 [10,2,13]
+ CRUSH rule 0 x 623 [4,9,14]
+ CRUSH rule 0 x 624 [3,9,15]
+ CRUSH rule 0 x 625 [11,7,3]
+ CRUSH rule 0 x 626 [10,12,2]
+ CRUSH rule 0 x 627 [1,12,10]
+ CRUSH rule 0 x 628 [15,13,11]
+ CRUSH rule 0 x 629 [5,6,15]
+ CRUSH rule 0 x 630 [1,4,12]
+ CRUSH rule 0 x 631 [5,7,1]
+ CRUSH rule 0 x 632 [12,3,11]
+ CRUSH rule 0 x 633 [14,4,3]
+ CRUSH rule 0 x 634 [6,9,5]
+ CRUSH rule 0 x 635 [6,5,2]
+ CRUSH rule 0 x 636 [13,6,11]
+ CRUSH rule 0 x 637 [3,1,10]
+ CRUSH rule 0 x 638 [10,15,3]
+ CRUSH rule 0 x 639 [6,9,14]
+ CRUSH rule 0 x 640 [9,6,1]
+ CRUSH rule 0 x 641 [10,6,5]
+ CRUSH rule 0 x 642 [1,15,4]
+ CRUSH rule 0 x 643 [3,7,5]
+ CRUSH rule 0 x 644 [15,13,6]
+ CRUSH rule 0 x 645 [14,2,4]
+ CRUSH rule 0 x 646 [5,13,14]
+ CRUSH rule 0 x 647 [10,1,9]
+ CRUSH rule 0 x 648 [6,5,2]
+ CRUSH rule 0 x 649 [3,9,13]
+ CRUSH rule 0 x 650 [10,9,4]
+ CRUSH rule 0 x 651 [3,9,5]
+ CRUSH rule 0 x 652 [15,9,4]
+ CRUSH rule 0 x 653 [11,14,1]
+ CRUSH rule 0 x 654 [13,6,2]
+ CRUSH rule 0 x 655 [6,3,4]
+ CRUSH rule 0 x 656 [3,15,1]
+ CRUSH rule 0 x 657 [11,15,3]
+ CRUSH rule 0 x 658 [7,2,10]
+ CRUSH rule 0 x 659 [2,5,14]
+ CRUSH rule 0 x 660 [13,14,10]
+ CRUSH rule 0 x 661 [7,15,3]
+ CRUSH rule 0 x 662 [15,2,12]
+ CRUSH rule 0 x 663 [14,9,13]
+ CRUSH rule 0 x 664 [6,10,12]
+ CRUSH rule 0 x 665 [2,9,12]
+ CRUSH rule 0 x 666 [12,3,6]
+ CRUSH rule 0 x 667 [1,9,12]
+ CRUSH rule 0 x 668 [9,5,1]
+ CRUSH rule 0 x 669 [9,7,14]
+ CRUSH rule 0 x 670 [6,10,9]
+ CRUSH rule 0 x 671 [6,15,5]
+ CRUSH rule 0 x 672 [2,9,13]
+ CRUSH rule 0 x 673 [7,10,5]
+ CRUSH rule 0 x 674 [7,12,10]
+ CRUSH rule 0 x 675 [9,5,1]
+ CRUSH rule 0 x 676 [10,12,2]
+ CRUSH rule 0 x 677 [2,12,1]
+ CRUSH rule 0 x 678 [1,2,4]
+ CRUSH rule 0 x 679 [5,6,12]
+ CRUSH rule 0 x 680 [7,11,3]
+ CRUSH rule 0 x 681 [6,4,3]
+ CRUSH rule 0 x 682 [6,1,11]
+ CRUSH rule 0 x 683 [6,13,2]
+ CRUSH rule 0 x 684 [9,11,3]
+ CRUSH rule 0 x 685 [5,1,15]
+ CRUSH rule 0 x 686 [1,9,11]
+ CRUSH rule 0 x 687 [7,13,3]
+ CRUSH rule 0 x 688 [11,9,1]
+ CRUSH rule 0 x 689 [5,2,9]
+ CRUSH rule 0 x 690 [9,7,10]
+ CRUSH rule 0 x 691 [11,15,9]
+ CRUSH rule 0 x 692 [15,5,1]
+ CRUSH rule 0 x 693 [5,6,12]
+ CRUSH rule 0 x 694 [4,7,1]
+ CRUSH rule 0 x 695 [6,13,14]
+ CRUSH rule 0 x 696 [1,2,4]
+ CRUSH rule 0 x 697 [13,11,3]
+ CRUSH rule 0 x 698 [11,13,4]
+ CRUSH rule 0 x 699 [7,14,12]
+ CRUSH rule 0 x 700 [12,14,11]
+ CRUSH rule 0 x 701 [3,13,1]
+ CRUSH rule 0 x 702 [3,12,15]
+ CRUSH rule 0 x 703 [15,11,13]
+ CRUSH rule 0 x 704 [6,4,2]
+ CRUSH rule 0 x 705 [14,6,11]
+ CRUSH rule 0 x 706 [1,12,3]
+ CRUSH rule 0 x 707 [4,7,14]
+ CRUSH rule 0 x 708 [3,10,5]
+ CRUSH rule 0 x 709 [11,12,3]
+ CRUSH rule 0 x 710 [14,2,11]
+ CRUSH rule 0 x 711 [14,3,9]
+ CRUSH rule 0 x 712 [12,3,11]
+ CRUSH rule 0 x 713 [11,9,3]
+ CRUSH rule 0 x 714 [12,1,9]
+ CRUSH rule 0 x 715 [6,1,14]
+ CRUSH rule 0 x 716 [11,13,9]
+ CRUSH rule 0 x 717 [12,4,10]
+ CRUSH rule 0 x 718 [7,15,5]
+ CRUSH rule 0 x 719 [5,15,13]
+ CRUSH rule 0 x 720 [4,13,10]
+ CRUSH rule 0 x 721 [11,3,14]
+ CRUSH rule 0 x 722 [2,4,6]
+ CRUSH rule 0 x 723 [2,1,12]
+ CRUSH rule 0 x 724 [7,1,9]
+ CRUSH rule 0 x 725 [11,12,7]
+ CRUSH rule 0 x 726 [7,14,4]
+ CRUSH rule 0 x 727 [2,5,1]
+ CRUSH rule 0 x 728 [13,11,4]
+ CRUSH rule 0 x 729 [15,11,4]
+ CRUSH rule 0 x 730 [3,7,1]
+ CRUSH rule 0 x 731 [9,1,6]
+ CRUSH rule 0 x 732 [1,2,10]
+ CRUSH rule 0 x 733 [11,3,5]
+ CRUSH rule 0 x 734 [14,3,11]
+ CRUSH rule 0 x 735 [6,9,2]
+ CRUSH rule 0 x 736 [3,9,1]
+ CRUSH rule 0 x 737 [1,4,2]
+ CRUSH rule 0 x 738 [11,15,7]
+ CRUSH rule 0 x 739 [11,12,6]
+ CRUSH rule 0 x 740 [7,9,10]
+ CRUSH rule 0 x 741 [12,11,7]
+ CRUSH rule 0 x 742 [9,7,4]
+ CRUSH rule 0 x 743 [5,13,9]
+ CRUSH rule 0 x 744 [6,2,13]
+ CRUSH rule 0 x 745 [3,6,1]
+ CRUSH rule 0 x 746 [3,7,9]
+ CRUSH rule 0 x 747 [15,11,5]
+ CRUSH rule 0 x 748 [6,10,13]
+ CRUSH rule 0 x 749 [14,9,10]
+ CRUSH rule 0 x 750 [1,14,6]
+ CRUSH rule 0 x 751 [15,1,6]
+ CRUSH rule 0 x 752 [13,1,7]
+ CRUSH rule 0 x 753 [4,11,1]
+ CRUSH rule 0 x 754 [14,12,11]
+ CRUSH rule 0 x 755 [13,6,1]
+ CRUSH rule 0 x 756 [3,4,14]
+ CRUSH rule 0 x 757 [10,6,1]
+ CRUSH rule 0 x 758 [6,3,4]
+ CRUSH rule 0 x 759 [5,7,3]
+ CRUSH rule 0 x 760 [1,15,10]
+ CRUSH rule 0 x 761 [2,12,1]
+ CRUSH rule 0 x 762 [1,4,10]
+ CRUSH rule 0 x 763 [4,13,1]
+ CRUSH rule 0 x 764 [1,14,6]
+ CRUSH rule 0 x 765 [9,15,2]
+ CRUSH rule 0 x 766 [11,2,7]
+ CRUSH rule 0 x 767 [6,11,4]
+ CRUSH rule 0 x 768 [2,12,15]
+ CRUSH rule 0 x 769 [15,1,9]
+ CRUSH rule 0 x 770 [15,13,4]
+ CRUSH rule 0 x 771 [9,2,12]
+ CRUSH rule 0 x 772 [4,3,13]
+ CRUSH rule 0 x 773 [3,7,4]
+ CRUSH rule 0 x 774 [12,6,3]
+ CRUSH rule 0 x 775 [5,10,14]
+ CRUSH rule 0 x 776 [10,15,3]
+ CRUSH rule 0 x 777 [11,13,4]
+ CRUSH rule 0 x 778 [13,1,9]
+ CRUSH rule 0 x 779 [5,11,1]
+ CRUSH rule 0 x 780 [13,9,3]
+ CRUSH rule 0 x 781 [5,7,14]
+ CRUSH rule 0 x 782 [2,15,9]
+ CRUSH rule 0 x 783 [12,7,5]
+ CRUSH rule 0 x 784 [14,1,10]
+ CRUSH rule 0 x 785 [6,12,1]
+ CRUSH rule 0 x 786 [10,5,2]
+ CRUSH rule 0 x 787 [1,12,10]
+ CRUSH rule 0 x 788 [4,2,9]
+ CRUSH rule 0 x 789 [9,2,14]
+ CRUSH rule 0 x 790 [15,2,7]
+ CRUSH rule 0 x 791 [9,4,7]
+ CRUSH rule 0 x 792 [6,4,15]
+ CRUSH rule 0 x 793 [15,9,6]
+ CRUSH rule 0 x 794 [5,12,2]
+ CRUSH rule 0 x 795 [6,14,12]
+ CRUSH rule 0 x 796 [11,2,12]
+ CRUSH rule 0 x 797 [14,3,7]
+ CRUSH rule 0 x 798 [5,11,6]
+ CRUSH rule 0 x 799 [2,9,14]
+ CRUSH rule 0 x 800 [6,3,4]
+ CRUSH rule 0 x 801 [2,5,6]
+ CRUSH rule 0 x 802 [1,4,12]
+ CRUSH rule 0 x 803 [7,2,4]
+ CRUSH rule 0 x 804 [5,14,9]
+ CRUSH rule 0 x 805 [13,4,3]
+ CRUSH rule 0 x 806 [6,2,13]
+ CRUSH rule 0 x 807 [14,2,7]
+ CRUSH rule 0 x 808 [2,15,12]
+ CRUSH rule 0 x 809 [1,11,7]
+ CRUSH rule 0 x 810 [2,5,9]
+ CRUSH rule 0 x 811 [15,6,3]
+ CRUSH rule 0 x 812 [7,11,2]
+ CRUSH rule 0 x 813 [4,10,13]
+ CRUSH rule 0 x 814 [13,4,9]
+ CRUSH rule 0 x 815 [15,12,9]
+ CRUSH rule 0 x 816 [14,10,13]
+ CRUSH rule 0 x 817 [10,7,2]
+ CRUSH rule 0 x 818 [15,2,11]
+ CRUSH rule 0 x 819 [5,12,10]
+ CRUSH rule 0 x 820 [3,6,9]
+ CRUSH rule 0 x 821 [15,10,9]
+ CRUSH rule 0 x 822 [10,13,2]
+ CRUSH rule 0 x 823 [2,6,12]
+ CRUSH rule 0 x 824 [3,7,9]
+ CRUSH rule 0 x 825 [10,5,14]
+ CRUSH rule 0 x 826 [5,2,11]
+ CRUSH rule 0 x 827 [13,5,1]
+ CRUSH rule 0 x 828 [12,6,10]
+ CRUSH rule 0 x 829 [13,6,15]
+ CRUSH rule 0 x 830 [15,13,2]
+ CRUSH rule 0 x 831 [1,4,11]
+ CRUSH rule 0 x 832 [14,11,13]
+ CRUSH rule 0 x 833 [9,13,3]
+ CRUSH rule 0 x 834 [9,7,5]
+ CRUSH rule 0 x 835 [14,3,13]
+ CRUSH rule 0 x 836 [3,9,10]
+ CRUSH rule 0 x 837 [15,12,11]
+ CRUSH rule 0 x 838 [12,14,9]
+ CRUSH rule 0 x 839 [3,4,6]
+ CRUSH rule 0 x 840 [10,15,12]
+ CRUSH rule 0 x 841 [3,5,7]
+ CRUSH rule 0 x 842 [9,13,2]
+ CRUSH rule 0 x 843 [14,7,4]
+ CRUSH rule 0 x 844 [7,1,4]
+ CRUSH rule 0 x 845 [13,6,1]
+ CRUSH rule 0 x 846 [3,7,15]
+ CRUSH rule 0 x 847 [12,15,11]
+ CRUSH rule 0 x 848 [11,13,1]
+ CRUSH rule 0 x 849 [3,15,11]
+ CRUSH rule 0 x 850 [1,3,10]
+ CRUSH rule 0 x 851 [14,4,3]
+ CRUSH rule 0 x 852 [9,12,4]
+ CRUSH rule 0 x 853 [13,14,6]
+ CRUSH rule 0 x 854 [7,11,12]
+ CRUSH rule 0 x 855 [14,4,12]
+ CRUSH rule 0 x 856 [5,10,7]
+ CRUSH rule 0 x 857 [4,3,13]
+ CRUSH rule 0 x 858 [5,15,6]
+ CRUSH rule 0 x 859 [5,15,6]
+ CRUSH rule 0 x 860 [11,14,1]
+ CRUSH rule 0 x 861 [13,7,4]
+ CRUSH rule 0 x 862 [5,10,9]
+ CRUSH rule 0 x 863 [11,6,3]
+ CRUSH rule 0 x 864 [6,13,4]
+ CRUSH rule 0 x 865 [4,1,14]
+ CRUSH rule 0 x 866 [2,13,4]
+ CRUSH rule 0 x 867 [12,2,9]
+ CRUSH rule 0 x 868 [14,11,7]
+ CRUSH rule 0 x 869 [10,13,7]
+ CRUSH rule 0 x 870 [14,9,11]
+ CRUSH rule 0 x 871 [6,2,1]
+ CRUSH rule 0 x 872 [6,1,15]
+ CRUSH rule 0 x 873 [2,5,12]
+ CRUSH rule 0 x 874 [12,4,7]
+ CRUSH rule 0 x 875 [10,6,14]
+ CRUSH rule 0 x 876 [14,7,13]
+ CRUSH rule 0 x 877 [15,11,13]
+ CRUSH rule 0 x 878 [7,14,3]
+ CRUSH rule 0 x 879 [12,2,7]
+ CRUSH rule 0 x 880 [2,12,10]
+ CRUSH rule 0 x 881 [6,3,1]
+ CRUSH rule 0 x 882 [11,13,7]
+ CRUSH rule 0 x 883 [13,1,3]
+ CRUSH rule 0 x 884 [6,15,4]
+ CRUSH rule 0 x 885 [14,7,9]
+ CRUSH rule 0 x 886 [13,11,4]
+ CRUSH rule 0 x 887 [14,4,12]
+ CRUSH rule 0 x 888 [10,12,7]
+ CRUSH rule 0 x 889 [15,13,4]
+ CRUSH rule 0 x 890 [10,12,14]
+ CRUSH rule 0 x 891 [9,5,11]
+ CRUSH rule 0 x 892 [12,15,2]
+ CRUSH rule 0 x 893 [1,3,5]
+ CRUSH rule 0 x 894 [7,2,11]
+ CRUSH rule 0 x 895 [2,1,11]
+ CRUSH rule 0 x 896 [9,1,14]
+ CRUSH rule 0 x 897 [7,5,14]
+ CRUSH rule 0 x 898 [10,6,12]
+ CRUSH rule 0 x 899 [1,11,5]
+ CRUSH rule 0 x 900 [2,9,10]
+ CRUSH rule 0 x 901 [9,12,11]
+ CRUSH rule 0 x 902 [4,2,6]
+ CRUSH rule 0 x 903 [14,10,3]
+ CRUSH rule 0 x 904 [15,12,4]
+ CRUSH rule 0 x 905 [12,6,11]
+ CRUSH rule 0 x 906 [14,11,12]
+ CRUSH rule 0 x 907 [7,12,3]
+ CRUSH rule 0 x 908 [2,15,9]
+ CRUSH rule 0 x 909 [10,14,1]
+ CRUSH rule 0 x 910 [12,7,4]
+ CRUSH rule 0 x 911 [11,15,2]
+ CRUSH rule 0 x 912 [6,4,14]
+ CRUSH rule 0 x 913 [4,6,10]
+ CRUSH rule 0 x 914 [4,15,2]
+ CRUSH rule 0 x 915 [12,14,1]
+ CRUSH rule 0 x 916 [3,1,11]
+ CRUSH rule 0 x 917 [1,15,6]
+ CRUSH rule 0 x 918 [7,14,11]
+ CRUSH rule 0 x 919 [10,7,3]
+ CRUSH rule 0 x 920 [4,2,10]
+ CRUSH rule 0 x 921 [1,11,6]
+ CRUSH rule 0 x 922 [6,4,14]
+ CRUSH rule 0 x 923 [12,2,5]
+ CRUSH rule 0 x 924 [6,2,14]
+ CRUSH rule 0 x 925 [12,15,2]
+ CRUSH rule 0 x 926 [3,13,10]
+ CRUSH rule 0 x 927 [6,5,1]
+ CRUSH rule 0 x 928 [13,1,3]
+ CRUSH rule 0 x 929 [10,7,1]
+ CRUSH rule 0 x 930 [7,15,10]
+ CRUSH rule 0 x 931 [6,15,11]
+ CRUSH rule 0 x 932 [13,2,5]
+ CRUSH rule 0 x 933 [12,7,14]
+ CRUSH rule 0 x 934 [12,2,5]
+ CRUSH rule 0 x 935 [6,11,1]
+ CRUSH rule 0 x 936 [9,12,7]
+ CRUSH rule 0 x 937 [14,2,11]
+ CRUSH rule 0 x 938 [14,3,5]
+ CRUSH rule 0 x 939 [6,4,14]
+ CRUSH rule 0 x 940 [13,11,4]
+ CRUSH rule 0 x 941 [3,12,4]
+ CRUSH rule 0 x 942 [15,12,10]
+ CRUSH rule 0 x 943 [10,2,4]
+ CRUSH rule 0 x 944 [2,9,4]
+ CRUSH rule 0 x 945 [10,15,2]
+ CRUSH rule 0 x 946 [11,15,7]
+ CRUSH rule 0 x 947 [11,3,14]
+ CRUSH rule 0 x 948 [7,13,11]
+ CRUSH rule 0 x 949 [9,1,12]
+ CRUSH rule 0 x 950 [9,15,13]
+ CRUSH rule 0 x 951 [2,6,12]
+ CRUSH rule 0 x 952 [9,7,15]
+ CRUSH rule 0 x 953 [1,3,6]
+ CRUSH rule 0 x 954 [10,2,14]
+ CRUSH rule 0 x 955 [7,14,3]
+ CRUSH rule 0 x 956 [1,6,11]
+ CRUSH rule 0 x 957 [14,11,1]
+ CRUSH rule 0 x 958 [15,4,3]
+ CRUSH rule 0 x 959 [2,1,12]
+ CRUSH rule 0 x 960 [2,6,11]
+ CRUSH rule 0 x 961 [3,13,11]
+ CRUSH rule 0 x 962 [5,11,3]
+ CRUSH rule 0 x 963 [13,10,15]
+ CRUSH rule 0 x 964 [7,11,4]
+ CRUSH rule 0 x 965 [12,2,9]
+ CRUSH rule 0 x 966 [12,14,9]
+ CRUSH rule 0 x 967 [7,5,3]
+ CRUSH rule 0 x 968 [12,15,4]
+ CRUSH rule 0 x 969 [11,4,7]
+ CRUSH rule 0 x 970 [5,12,10]
+ CRUSH rule 0 x 971 [1,9,4]
+ CRUSH rule 0 x 972 [12,3,14]
+ CRUSH rule 0 x 973 [1,10,4]
+ CRUSH rule 0 x 974 [7,11,1]
+ CRUSH rule 0 x 975 [7,9,15]
+ CRUSH rule 0 x 976 [7,3,15]
+ CRUSH rule 0 x 977 [14,3,6]
+ CRUSH rule 0 x 978 [12,5,11]
+ CRUSH rule 0 x 979 [5,1,13]
+ CRUSH rule 0 x 980 [15,11,5]
+ CRUSH rule 0 x 981 [5,11,15]
+ CRUSH rule 0 x 982 [2,6,14]
+ CRUSH rule 0 x 983 [3,12,10]
+ CRUSH rule 0 x 984 [15,13,1]
+ CRUSH rule 0 x 985 [11,2,15]
+ CRUSH rule 0 x 986 [6,13,9]
+ CRUSH rule 0 x 987 [13,14,5]
+ CRUSH rule 0 x 988 [12,9,10]
+ CRUSH rule 0 x 989 [7,4,3]
+ CRUSH rule 0 x 990 [1,10,9]
+ CRUSH rule 0 x 991 [7,11,1]
+ CRUSH rule 0 x 992 [9,10,2]
+ CRUSH rule 0 x 993 [6,10,14]
+ CRUSH rule 0 x 994 [3,13,15]
+ CRUSH rule 0 x 995 [15,6,12]
+ CRUSH rule 0 x 996 [15,10,5]
+ CRUSH rule 0 x 997 [15,2,1]
+ CRUSH rule 0 x 998 [6,1,9]
+ CRUSH rule 0 x 999 [9,10,15]
+ CRUSH rule 0 x 1000 [14,2,9]
+ CRUSH rule 0 x 1001 [11,14,4]
+ CRUSH rule 0 x 1002 [1,10,14]
+ CRUSH rule 0 x 1003 [10,7,5]
+ CRUSH rule 0 x 1004 [15,1,4]
+ CRUSH rule 0 x 1005 [6,12,2]
+ CRUSH rule 0 x 1006 [10,12,15]
+ CRUSH rule 0 x 1007 [1,7,13]
+ CRUSH rule 0 x 1008 [7,4,9]
+ CRUSH rule 0 x 1009 [5,2,11]
+ CRUSH rule 0 x 1010 [10,2,15]
+ CRUSH rule 0 x 1011 [6,3,12]
+ CRUSH rule 0 x 1012 [12,6,9]
+ CRUSH rule 0 x 1013 [2,14,12]
+ CRUSH rule 0 x 1014 [1,13,7]
+ CRUSH rule 0 x 1015 [12,6,10]
+ CRUSH rule 0 x 1016 [10,13,14]
+ CRUSH rule 0 x 1017 [5,11,14]
+ CRUSH rule 0 x 1018 [13,11,14]
+ CRUSH rule 0 x 1019 [10,13,14]
+ CRUSH rule 0 x 1020 [3,1,13]
+ CRUSH rule 0 x 1021 [2,11,14]
+ CRUSH rule 0 x 1022 [15,5,7]
+ CRUSH rule 0 x 1023 [15,2,9]
+ rule 0 (replicated_ruleset) num_rep 3 result size == 3:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15]
+ CRUSH rule 0 x 1 [10,15,1,2]
+ CRUSH rule 0 x 2 [1,12,2,6]
+ CRUSH rule 0 x 3 [15,4,10,2]
+ CRUSH rule 0 x 4 [14,2,10,1]
+ CRUSH rule 0 x 5 [7,4,11,2]
+ CRUSH rule 0 x 6 [12,6,10,9]
+ CRUSH rule 0 x 7 [9,2,6,12]
+ CRUSH rule 0 x 8 [10,2,15,1]
+ CRUSH rule 0 x 9 [7,1,14,2]
+ CRUSH rule 0 x 10 [10,14,4,1]
+ CRUSH rule 0 x 11 [13,9,14,7]
+ CRUSH rule 0 x 12 [7,1,2,5]
+ CRUSH rule 0 x 13 [3,5,12,7]
+ CRUSH rule 0 x 14 [13,5,2,7]
+ CRUSH rule 0 x 15 [15,1,9,6]
+ CRUSH rule 0 x 16 [7,11,14,2]
+ CRUSH rule 0 x 17 [10,1,13,2]
+ CRUSH rule 0 x 18 [1,7,3,10]
+ CRUSH rule 0 x 19 [7,12,2,4]
+ CRUSH rule 0 x 20 [14,12,3,10]
+ CRUSH rule 0 x 21 [3,12,1,10]
+ CRUSH rule 0 x 22 [6,3,13,11]
+ CRUSH rule 0 x 23 [10,5,13,9]
+ CRUSH rule 0 x 24 [12,11,3,1]
+ CRUSH rule 0 x 25 [7,12,15,1]
+ CRUSH rule 0 x 26 [1,7,13,2]
+ CRUSH rule 0 x 27 [3,6,15,4]
+ CRUSH rule 0 x 28 [14,4,3,9]
+ CRUSH rule 0 x 29 [5,14,12,11]
+ CRUSH rule 0 x 30 [2,5,6,9]
+ CRUSH rule 0 x 31 [5,15,10,1]
+ CRUSH rule 0 x 32 [9,10,2,1]
+ CRUSH rule 0 x 33 [13,4,9,2]
+ CRUSH rule 0 x 34 [13,15,2,4]
+ CRUSH rule 0 x 35 [4,14,3,13]
+ CRUSH rule 0 x 36 [3,12,9,7]
+ CRUSH rule 0 x 37 [9,2,6,14]
+ CRUSH rule 0 x 38 [3,4,13,10]
+ CRUSH rule 0 x 39 [12,7,14,11]
+ CRUSH rule 0 x 40 [10,1,9,5]
+ CRUSH rule 0 x 41 [4,9,11,1]
+ CRUSH rule 0 x 42 [3,6,14,10]
+ CRUSH rule 0 x 43 [10,5,15,7]
+ CRUSH rule 0 x 44 [11,4,13,3]
+ CRUSH rule 0 x 45 [11,12,15,9]
+ CRUSH rule 0 x 46 [6,9,2,14]
+ CRUSH rule 0 x 47 [3,9,6,4]
+ CRUSH rule 0 x 48 [4,6,2,1]
+ CRUSH rule 0 x 49 [9,15,10,7]
+ CRUSH rule 0 x 50 [14,12,1,4]
+ CRUSH rule 0 x 51 [10,6,5,12]
+ CRUSH rule 0 x 52 [12,1,9,11]
+ CRUSH rule 0 x 53 [3,6,13,9]
+ CRUSH rule 0 x 54 [4,13,9,2]
+ CRUSH rule 0 x 55 [4,11,2,7]
+ CRUSH rule 0 x 56 [5,9,10,1]
+ CRUSH rule 0 x 57 [6,2,1,15]
+ CRUSH rule 0 x 58 [7,1,11,4]
+ CRUSH rule 0 x 59 [2,13,1,10]
+ CRUSH rule 0 x 60 [3,6,11,1]
+ CRUSH rule 0 x 61 [3,15,13,7]
+ CRUSH rule 0 x 62 [15,11,7,12]
+ CRUSH rule 0 x 63 [10,14,12,1]
+ CRUSH rule 0 x 64 [3,9,1,4]
+ CRUSH rule 0 x 65 [4,12,11,7]
+ CRUSH rule 0 x 66 [15,11,6,9]
+ CRUSH rule 0 x 67 [2,6,4,14]
+ CRUSH rule 0 x 68 [15,7,4,2]
+ CRUSH rule 0 x 69 [2,1,15,10]
+ CRUSH rule 0 x 70 [9,6,1,3]
+ CRUSH rule 0 x 71 [15,5,1,3]
+ CRUSH rule 0 x 72 [9,10,3,5]
+ CRUSH rule 0 x 73 [5,3,11,1]
+ CRUSH rule 0 x 74 [11,7,9,5]
+ CRUSH rule 0 x 75 [9,7,11,14]
+ CRUSH rule 0 x 76 [6,1,3,5]
+ CRUSH rule 0 x 77 [7,4,2,13]
+ CRUSH rule 0 x 78 [9,3,1,5]
+ CRUSH rule 0 x 79 [13,2,15,5]
+ CRUSH rule 0 x 80 [15,2,6,4]
+ CRUSH rule 0 x 81 [15,2,1,11]
+ CRUSH rule 0 x 82 [14,13,5,11]
+ CRUSH rule 0 x 83 [4,15,3,9]
+ CRUSH rule 0 x 84 [10,7,9,15]
+ CRUSH rule 0 x 85 [3,15,9,7]
+ CRUSH rule 0 x 86 [10,9,14,1]
+ CRUSH rule 0 x 87 [15,10,7,12]
+ CRUSH rule 0 x 88 [4,13,3,1]
+ CRUSH rule 0 x 89 [3,9,7,4]
+ CRUSH rule 0 x 90 [4,9,7,12]
+ CRUSH rule 0 x 91 [6,11,9,1]
+ CRUSH rule 0 x 92 [1,5,10,9]
+ CRUSH rule 0 x 93 [9,3,15,13]
+ CRUSH rule 0 x 94 [9,2,12,5]
+ CRUSH rule 0 x 95 [7,15,4,10]
+ CRUSH rule 0 x 96 [2,15,11,7]
+ CRUSH rule 0 x 97 [4,11,2,13]
+ CRUSH rule 0 x 98 [11,13,9,3]
+ CRUSH rule 0 x 99 [12,4,11,7]
+ CRUSH rule 0 x 100 [9,4,10,15]
+ CRUSH rule 0 x 101 [15,7,1,9]
+ CRUSH rule 0 x 102 [3,11,14,6]
+ CRUSH rule 0 x 103 [13,11,6,14]
+ CRUSH rule 0 x 104 [14,6,3,5]
+ CRUSH rule 0 x 105 [14,10,1,9]
+ CRUSH rule 0 x 106 [6,5,13,2]
+ CRUSH rule 0 x 107 [3,1,10,14]
+ CRUSH rule 0 x 108 [5,10,7,2]
+ CRUSH rule 0 x 109 [9,1,13,7]
+ CRUSH rule 0 x 110 [5,1,11,3]
+ CRUSH rule 0 x 111 [10,1,9,7]
+ CRUSH rule 0 x 112 [1,10,4,14]
+ CRUSH rule 0 x 113 [6,10,13,9]
+ CRUSH rule 0 x 114 [5,13,6,2]
+ CRUSH rule 0 x 115 [10,13,14,3]
+ CRUSH rule 0 x 116 [1,14,13,2]
+ CRUSH rule 0 x 117 [5,6,1,12]
+ CRUSH rule 0 x 118 [10,4,13,15]
+ CRUSH rule 0 x 119 [14,12,11,4]
+ CRUSH rule 0 x 120 [11,3,14,13]
+ CRUSH rule 0 x 121 [9,5,1,11]
+ CRUSH rule 0 x 122 [4,3,14,1]
+ CRUSH rule 0 x 123 [3,10,5,6]
+ CRUSH rule 0 x 124 [12,2,1,5]
+ CRUSH rule 0 x 125 [9,12,15,1]
+ CRUSH rule 0 x 126 [7,15,10,9]
+ CRUSH rule 0 x 127 [4,14,9,13]
+ CRUSH rule 0 x 128 [3,12,1,10]
+ CRUSH rule 0 x 129 [11,13,14,2]
+ CRUSH rule 0 x 130 [3,13,5,14]
+ CRUSH rule 0 x 131 [12,1,6,15]
+ CRUSH rule 0 x 132 [11,15,13,9]
+ CRUSH rule 0 x 133 [3,6,9,11]
+ CRUSH rule 0 x 134 [12,5,6,15]
+ CRUSH rule 0 x 135 [3,14,12,4]
+ CRUSH rule 0 x 136 [15,6,9,4]
+ CRUSH rule 0 x 137 [14,3,6,11]
+ CRUSH rule 0 x 138 [13,15,4,10]
+ CRUSH rule 0 x 139 [11,2,13,9]
+ CRUSH rule 0 x 140 [11,4,12,15]
+ CRUSH rule 0 x 141 [6,12,15,11]
+ CRUSH rule 0 x 142 [3,14,7,9]
+ CRUSH rule 0 x 143 [9,6,4,2]
+ CRUSH rule 0 x 144 [13,7,11,2]
+ CRUSH rule 0 x 145 [12,2,6,10]
+ CRUSH rule 0 x 146 [1,5,9,2]
+ CRUSH rule 0 x 147 [1,4,9,11]
+ CRUSH rule 0 x 148 [12,7,9,2]
+ CRUSH rule 0 x 149 [2,5,9,12]
+ CRUSH rule 0 x 150 [1,15,2,10]
+ CRUSH rule 0 x 151 [2,9,14,7]
+ CRUSH rule 0 x 152 [5,9,2,6]
+ CRUSH rule 0 x 153 [6,9,4,15]
+ CRUSH rule 0 x 154 [3,11,7,1]
+ CRUSH rule 0 x 155 [14,12,7,3]
+ CRUSH rule 0 x 156 [7,13,3,10]
+ CRUSH rule 0 x 157 [15,1,6,4]
+ CRUSH rule 0 x 158 [15,1,10,6]
+ CRUSH rule 0 x 159 [4,14,3,12]
+ CRUSH rule 0 x 160 [5,7,3,14]
+ CRUSH rule 0 x 161 [1,2,11,4]
+ CRUSH rule 0 x 162 [10,6,1,12]
+ CRUSH rule 0 x 163 [15,1,10,2]
+ CRUSH rule 0 x 164 [9,14,10,7]
+ CRUSH rule 0 x 165 [11,7,2,13]
+ CRUSH rule 0 x 166 [1,2,12,14]
+ CRUSH rule 0 x 167 [9,7,3,4]
+ CRUSH rule 0 x 168 [13,2,4,1]
+ CRUSH rule 0 x 169 [1,4,9,14]
+ CRUSH rule 0 x 170 [1,15,7,9]
+ CRUSH rule 0 x 171 [9,2,10,7]
+ CRUSH rule 0 x 172 [14,4,10,12]
+ CRUSH rule 0 x 173 [5,10,12,15]
+ CRUSH rule 0 x 174 [15,6,4,12]
+ CRUSH rule 0 x 175 [5,7,9,3]
+ CRUSH rule 0 x 176 [9,6,3,14]
+ CRUSH rule 0 x 177 [2,9,10,13]
+ CRUSH rule 0 x 178 [12,11,7,14]
+ CRUSH rule 0 x 179 [2,10,13,9]
+ CRUSH rule 0 x 180 [3,11,5,15]
+ CRUSH rule 0 x 181 [9,12,6,5]
+ CRUSH rule 0 x 182 [5,13,11,2]
+ CRUSH rule 0 x 183 [5,7,10,13]
+ CRUSH rule 0 x 184 [2,5,11,12]
+ CRUSH rule 0 x 185 [13,5,7,11]
+ CRUSH rule 0 x 186 [6,14,13,5]
+ CRUSH rule 0 x 187 [1,4,11,13]
+ CRUSH rule 0 x 188 [9,13,5,14]
+ CRUSH rule 0 x 189 [6,12,4,9]
+ CRUSH rule 0 x 190 [9,13,15,10]
+ CRUSH rule 0 x 191 [7,11,4,1]
+ CRUSH rule 0 x 192 [2,11,5,15]
+ CRUSH rule 0 x 193 [3,13,6,10]
+ CRUSH rule 0 x 194 [3,13,4,14]
+ CRUSH rule 0 x 195 [5,7,10,12]
+ CRUSH rule 0 x 196 [4,15,1,10]
+ CRUSH rule 0 x 197 [14,10,13,4]
+ CRUSH rule 0 x 198 [2,5,6,15]
+ CRUSH rule 0 x 199 [2,10,4,15]
+ CRUSH rule 0 x 200 [7,14,11,4]
+ CRUSH rule 0 x 201 [9,14,1,7]
+ CRUSH rule 0 x 202 [14,11,7,3]
+ CRUSH rule 0 x 203 [12,5,7,15]
+ CRUSH rule 0 x 204 [6,11,3,12]
+ CRUSH rule 0 x 205 [15,4,6,10]
+ CRUSH rule 0 x 206 [13,11,2,15]
+ CRUSH rule 0 x 207 [2,11,7,4]
+ CRUSH rule 0 x 208 [13,1,6,14]
+ CRUSH rule 0 x 209 [6,15,13,1]
+ CRUSH rule 0 x 210 [13,11,2,7]
+ CRUSH rule 0 x 211 [2,14,1,13]
+ CRUSH rule 0 x 212 [10,1,12,15]
+ CRUSH rule 0 x 213 [3,9,6,5]
+ CRUSH rule 0 x 214 [7,15,4,1]
+ CRUSH rule 0 x 215 [6,1,4,13]
+ CRUSH rule 0 x 216 [12,9,6,2]
+ CRUSH rule 0 x 217 [12,11,1,14]
+ CRUSH rule 0 x 218 [12,10,15,6]
+ CRUSH rule 0 x 219 [3,11,14,6]
+ CRUSH rule 0 x 220 [14,4,3,12]
+ CRUSH rule 0 x 221 [15,5,2,6]
+ CRUSH rule 0 x 222 [10,4,3,15]
+ CRUSH rule 0 x 223 [9,7,11,1]
+ CRUSH rule 0 x 224 [1,7,10,2]
+ CRUSH rule 0 x 225 [10,5,2,6]
+ CRUSH rule 0 x 226 [4,1,9,3]
+ CRUSH rule 0 x 227 [7,2,12,15]
+ CRUSH rule 0 x 228 [2,15,11,1]
+ CRUSH rule 0 x 229 [9,3,7,14]
+ CRUSH rule 0 x 230 [10,5,7,2]
+ CRUSH rule 0 x 231 [2,7,5,13]
+ CRUSH rule 0 x 232 [10,5,13,1]
+ CRUSH rule 0 x 233 [6,12,11,4]
+ CRUSH rule 0 x 234 [10,1,2,12]
+ CRUSH rule 0 x 235 [13,14,7,10]
+ CRUSH rule 0 x 236 [2,15,9,12]
+ CRUSH rule 0 x 237 [3,12,9,10]
+ CRUSH rule 0 x 238 [2,10,4,15]
+ CRUSH rule 0 x 239 [4,15,10,7]
+ CRUSH rule 0 x 240 [15,5,13,7]
+ CRUSH rule 0 x 241 [7,9,15,12]
+ CRUSH rule 0 x 242 [14,2,6,9]
+ CRUSH rule 0 x 243 [2,11,5,1]
+ CRUSH rule 0 x 244 [13,9,15,3]
+ CRUSH rule 0 x 245 [12,9,15,3]
+ CRUSH rule 0 x 246 [15,3,5,11]
+ CRUSH rule 0 x 247 [6,4,9,12]
+ CRUSH rule 0 x 248 [5,13,7,11]
+ CRUSH rule 0 x 249 [10,14,7,3]
+ CRUSH rule 0 x 250 [12,15,1,10]
+ CRUSH rule 0 x 251 [13,2,15,5]
+ CRUSH rule 0 x 252 [7,5,13,9]
+ CRUSH rule 0 x 253 [3,13,15,10]
+ CRUSH rule 0 x 254 [2,9,13,14]
+ CRUSH rule 0 x 255 [1,9,13,2]
+ CRUSH rule 0 x 256 [6,9,13,1]
+ CRUSH rule 0 x 257 [15,12,3,9]
+ CRUSH rule 0 x 258 [12,5,6,10]
+ CRUSH rule 0 x 259 [9,10,4,3]
+ CRUSH rule 0 x 260 [10,12,6,9]
+ CRUSH rule 0 x 261 [13,7,2,1]
+ CRUSH rule 0 x 262 [15,3,12,7]
+ CRUSH rule 0 x 263 [12,6,10,9]
+ CRUSH rule 0 x 264 [13,14,11,3]
+ CRUSH rule 0 x 265 [12,10,14,5]
+ CRUSH rule 0 x 266 [14,7,11,1]
+ CRUSH rule 0 x 267 [12,11,6,5]
+ CRUSH rule 0 x 268 [4,1,15,12]
+ CRUSH rule 0 x 269 [11,1,15,5]
+ CRUSH rule 0 x 270 [7,11,12,3]
+ CRUSH rule 0 x 271 [4,7,3,13]
+ CRUSH rule 0 x 272 [15,5,13,10]
+ CRUSH rule 0 x 273 [2,10,7,12]
+ CRUSH rule 0 x 274 [10,2,5,6]
+ CRUSH rule 0 x 275 [10,3,4,7]
+ CRUSH rule 0 x 276 [5,12,9,2]
+ CRUSH rule 0 x 277 [14,3,13,4]
+ CRUSH rule 0 x 278 [5,6,14,3]
+ CRUSH rule 0 x 279 [6,10,13,3]
+ CRUSH rule 0 x 280 [7,3,14,9]
+ CRUSH rule 0 x 281 [5,11,14,7]
+ CRUSH rule 0 x 282 [2,1,13,14]
+ CRUSH rule 0 x 283 [4,1,12,3]
+ CRUSH rule 0 x 284 [5,11,7,15]
+ CRUSH rule 0 x 285 [15,5,3,1]
+ CRUSH rule 0 x 286 [10,4,3,6]
+ CRUSH rule 0 x 287 [12,4,9,1]
+ CRUSH rule 0 x 288 [4,12,10,7]
+ CRUSH rule 0 x 289 [2,5,14,9]
+ CRUSH rule 0 x 290 [12,2,5,6]
+ CRUSH rule 0 x 291 [7,11,1,14]
+ CRUSH rule 0 x 292 [4,10,6,3]
+ CRUSH rule 0 x 293 [6,5,11,1]
+ CRUSH rule 0 x 294 [9,12,3,14]
+ CRUSH rule 0 x 295 [6,10,3,14]
+ CRUSH rule 0 x 296 [3,1,13,7]
+ CRUSH rule 0 x 297 [6,13,4,14]
+ CRUSH rule 0 x 298 [14,9,13,1]
+ CRUSH rule 0 x 299 [14,12,11,6]
+ CRUSH rule 0 x 300 [15,7,10,5]
+ CRUSH rule 0 x 301 [9,11,7,1]
+ CRUSH rule 0 x 302 [9,7,1,13]
+ CRUSH rule 0 x 303 [4,13,3,7]
+ CRUSH rule 0 x 304 [6,9,2,11]
+ CRUSH rule 0 x 305 [13,7,5,11]
+ CRUSH rule 0 x 306 [10,12,4,6]
+ CRUSH rule 0 x 307 [11,12,15,5]
+ CRUSH rule 0 x 308 [12,14,10,9]
+ CRUSH rule 0 x 309 [9,3,12,5]
+ CRUSH rule 0 x 310 [3,1,5,10]
+ CRUSH rule 0 x 311 [3,9,7,1]
+ CRUSH rule 0 x 312 [15,13,9,7]
+ CRUSH rule 0 x 313 [9,15,3,7]
+ CRUSH rule 0 x 314 [2,15,9,5]
+ CRUSH rule 0 x 315 [15,2,13,1]
+ CRUSH rule 0 x 316 [4,9,11,2]
+ CRUSH rule 0 x 317 [1,5,3,13]
+ CRUSH rule 0 x 318 [4,1,15,11]
+ CRUSH rule 0 x 319 [2,15,4,1]
+ CRUSH rule 0 x 320 [5,7,13,9]
+ CRUSH rule 0 x 321 [1,6,11,15]
+ CRUSH rule 0 x 322 [13,7,5,3]
+ CRUSH rule 0 x 323 [7,4,10,1]
+ CRUSH rule 0 x 324 [5,6,10,15]
+ CRUSH rule 0 x 325 [9,10,14,5]
+ CRUSH rule 0 x 326 [11,7,13,4]
+ CRUSH rule 0 x 327 [12,5,10,14]
+ CRUSH rule 0 x 328 [5,2,6,14]
+ CRUSH rule 0 x 329 [2,6,15,5]
+ CRUSH rule 0 x 330 [3,9,11,13]
+ CRUSH rule 0 x 331 [12,14,6,3]
+ CRUSH rule 0 x 332 [10,12,6,15]
+ CRUSH rule 0 x 333 [6,5,3,12]
+ CRUSH rule 0 x 334 [4,9,2,12]
+ CRUSH rule 0 x 335 [11,7,1,5]
+ CRUSH rule 0 x 336 [6,14,13,2]
+ CRUSH rule 0 x 337 [15,11,3,7]
+ CRUSH rule 0 x 338 [10,5,3,6]
+ CRUSH rule 0 x 339 [11,14,13,5]
+ CRUSH rule 0 x 340 [11,6,12,4]
+ CRUSH rule 0 x 341 [7,5,2,10]
+ CRUSH rule 0 x 342 [12,14,1,9]
+ CRUSH rule 0 x 343 [12,14,9,6]
+ CRUSH rule 0 x 344 [9,11,5,2]
+ CRUSH rule 0 x 345 [14,2,11,9]
+ CRUSH rule 0 x 346 [5,3,14,10]
+ CRUSH rule 0 x 347 [10,2,12,6]
+ CRUSH rule 0 x 348 [7,9,10,1]
+ CRUSH rule 0 x 349 [9,6,10,12]
+ CRUSH rule 0 x 350 [13,9,15,4]
+ CRUSH rule 0 x 351 [13,5,15,3]
+ CRUSH rule 0 x 352 [1,12,11,9]
+ CRUSH rule 0 x 353 [10,14,12,2]
+ CRUSH rule 0 x 354 [6,3,15,10]
+ CRUSH rule 0 x 355 [13,14,6,10]
+ CRUSH rule 0 x 356 [15,13,2,9]
+ CRUSH rule 0 x 357 [4,11,1,13]
+ CRUSH rule 0 x 358 [12,7,2,9]
+ CRUSH rule 0 x 359 [5,15,7,11]
+ CRUSH rule 0 x 360 [13,10,1,2]
+ CRUSH rule 0 x 361 [5,3,13,6]
+ CRUSH rule 0 x 362 [2,9,11,13]
+ CRUSH rule 0 x 363 [7,12,3,9]
+ CRUSH rule 0 x 364 [2,12,6,9]
+ CRUSH rule 0 x 365 [13,5,11,15]
+ CRUSH rule 0 x 366 [12,7,3,14]
+ CRUSH rule 0 x 367 [7,13,3,1]
+ CRUSH rule 0 x 368 [7,9,10,15]
+ CRUSH rule 0 x 369 [7,5,3,13]
+ CRUSH rule 0 x 370 [4,7,14,1]
+ CRUSH rule 0 x 371 [1,7,12,3]
+ CRUSH rule 0 x 372 [10,4,3,14]
+ CRUSH rule 0 x 373 [15,5,2,6]
+ CRUSH rule 0 x 374 [3,15,12,5]
+ CRUSH rule 0 x 375 [5,2,14,1]
+ CRUSH rule 0 x 376 [5,14,10,13]
+ CRUSH rule 0 x 377 [1,15,2,4]
+ CRUSH rule 0 x 378 [9,12,2,15]
+ CRUSH rule 0 x 379 [11,2,15,5]
+ CRUSH rule 0 x 380 [6,1,12,11]
+ CRUSH rule 0 x 381 [15,13,7,5]
+ CRUSH rule 0 x 382 [14,3,1,4]
+ CRUSH rule 0 x 383 [3,6,11,4]
+ CRUSH rule 0 x 384 [4,13,6,3]
+ CRUSH rule 0 x 385 [4,6,15,3]
+ CRUSH rule 0 x 386 [14,3,11,13]
+ CRUSH rule 0 x 387 [1,11,5,7]
+ CRUSH rule 0 x 388 [2,6,11,9]
+ CRUSH rule 0 x 389 [12,7,2,4]
+ CRUSH rule 0 x 390 [2,11,13,7]
+ CRUSH rule 0 x 391 [3,4,9,13]
+ CRUSH rule 0 x 392 [11,5,14,7]
+ CRUSH rule 0 x 393 [2,14,5,9]
+ CRUSH rule 0 x 394 [4,9,3,15]
+ CRUSH rule 0 x 395 [10,13,5,15]
+ CRUSH rule 0 x 396 [2,12,15,9]
+ CRUSH rule 0 x 397 [1,14,9,4]
+ CRUSH rule 0 x 398 [9,2,1,5]
+ CRUSH rule 0 x 399 [5,9,14,3]
+ CRUSH rule 0 x 400 [10,6,2,4]
+ CRUSH rule 0 x 401 [6,9,11,12]
+ CRUSH rule 0 x 402 [4,7,9,2]
+ CRUSH rule 0 x 403 [7,15,13,3]
+ CRUSH rule 0 x 404 [14,12,7,9]
+ CRUSH rule 0 x 405 [9,15,11,2]
+ CRUSH rule 0 x 406 [12,14,9,2]
+ CRUSH rule 0 x 407 [9,5,12,10]
+ CRUSH rule 0 x 408 [7,1,5,2]
+ CRUSH rule 0 x 409 [11,2,4,13]
+ CRUSH rule 0 x 410 [6,4,14,2]
+ CRUSH rule 0 x 411 [13,11,15,6]
+ CRUSH rule 0 x 412 [5,9,6,11]
+ CRUSH rule 0 x 413 [13,5,3,11]
+ CRUSH rule 0 x 414 [3,11,9,13]
+ CRUSH rule 0 x 415 [6,10,14,5]
+ CRUSH rule 0 x 416 [13,1,4,7]
+ CRUSH rule 0 x 417 [4,12,1,15]
+ CRUSH rule 0 x 418 [14,5,10,2]
+ CRUSH rule 0 x 419 [5,14,10,9]
+ CRUSH rule 0 x 420 [2,4,9,11]
+ CRUSH rule 0 x 421 [15,4,10,3]
+ CRUSH rule 0 x 422 [4,11,2,7]
+ CRUSH rule 0 x 423 [3,15,12,6]
+ CRUSH rule 0 x 424 [6,10,12,2]
+ CRUSH rule 0 x 425 [11,15,2,13]
+ CRUSH rule 0 x 426 [12,4,7,1]
+ CRUSH rule 0 x 427 [14,10,3,1]
+ CRUSH rule 0 x 428 [12,7,9,4]
+ CRUSH rule 0 x 429 [3,4,9,7]
+ CRUSH rule 0 x 430 [3,5,10,13]
+ CRUSH rule 0 x 431 [9,3,7,1]
+ CRUSH rule 0 x 432 [4,1,12,7]
+ CRUSH rule 0 x 433 [4,11,12,15]
+ CRUSH rule 0 x 434 [2,14,9,1]
+ CRUSH rule 0 x 435 [13,11,5,6]
+ CRUSH rule 0 x 436 [9,15,10,2]
+ CRUSH rule 0 x 437 [9,6,3,14]
+ CRUSH rule 0 x 438 [7,2,13,4]
+ CRUSH rule 0 x 439 [7,14,4,3]
+ CRUSH rule 0 x 440 [14,11,9,2]
+ CRUSH rule 0 x 441 [2,4,11,9]
+ CRUSH rule 0 x 442 [10,13,9,7]
+ CRUSH rule 0 x 443 [12,15,10,9]
+ CRUSH rule 0 x 444 [4,13,7,14]
+ CRUSH rule 0 x 445 [4,2,15,7]
+ CRUSH rule 0 x 446 [12,10,6,9]
+ CRUSH rule 0 x 447 [15,7,13,1]
+ CRUSH rule 0 x 448 [5,2,13,7]
+ CRUSH rule 0 x 449 [14,5,3,12]
+ CRUSH rule 0 x 450 [2,4,6,9]
+ CRUSH rule 0 x 451 [6,14,11,3]
+ CRUSH rule 0 x 452 [14,9,10,4]
+ CRUSH rule 0 x 453 [5,15,13,2]
+ CRUSH rule 0 x 454 [10,4,2,6]
+ CRUSH rule 0 x 455 [6,13,2,4]
+ CRUSH rule 0 x 456 [5,7,13,1]
+ CRUSH rule 0 x 457 [9,1,5,7]
+ CRUSH rule 0 x 458 [9,11,15,4]
+ CRUSH rule 0 x 459 [13,15,11,1]
+ CRUSH rule 0 x 460 [5,12,10,15]
+ CRUSH rule 0 x 461 [4,3,9,13]
+ CRUSH rule 0 x 462 [4,7,12,14]
+ CRUSH rule 0 x 463 [4,12,14,11]
+ CRUSH rule 0 x 464 [4,2,15,10]
+ CRUSH rule 0 x 465 [5,10,9,7]
+ CRUSH rule 0 x 466 [13,5,2,15]
+ CRUSH rule 0 x 467 [13,6,14,3]
+ CRUSH rule 0 x 468 [10,7,12,14]
+ CRUSH rule 0 x 469 [4,9,6,14]
+ CRUSH rule 0 x 470 [3,9,12,15]
+ CRUSH rule 0 x 471 [6,1,5,14]
+ CRUSH rule 0 x 472 [2,14,7,5]
+ CRUSH rule 0 x 473 [15,10,6,9]
+ CRUSH rule 0 x 474 [15,10,4,12]
+ CRUSH rule 0 x 475 [10,5,12,9]
+ CRUSH rule 0 x 476 [3,6,10,12]
+ CRUSH rule 0 x 477 [6,13,5,15]
+ CRUSH rule 0 x 478 [4,15,1,3]
+ CRUSH rule 0 x 479 [13,11,1,6]
+ CRUSH rule 0 x 480 [1,13,6,4]
+ CRUSH rule 0 x 481 [15,12,7,9]
+ CRUSH rule 0 x 482 [2,12,9,1]
+ CRUSH rule 0 x 483 [10,1,4,15]
+ CRUSH rule 0 x 484 [1,4,10,13]
+ CRUSH rule 0 x 485 [9,4,3,1]
+ CRUSH rule 0 x 486 [3,10,15,9]
+ CRUSH rule 0 x 487 [12,11,4,14]
+ CRUSH rule 0 x 488 [14,4,1,9]
+ CRUSH rule 0 x 489 [11,4,2,13]
+ CRUSH rule 0 x 490 [4,9,1,3]
+ CRUSH rule 0 x 491 [1,12,5,2]
+ CRUSH rule 0 x 492 [5,7,11,3]
+ CRUSH rule 0 x 493 [12,1,4,15]
+ CRUSH rule 0 x 494 [1,7,13,4]
+ CRUSH rule 0 x 495 [3,15,7,1]
+ CRUSH rule 0 x 496 [5,3,7,13]
+ CRUSH rule 0 x 497 [13,10,3,6]
+ CRUSH rule 0 x 498 [10,6,1,5]
+ CRUSH rule 0 x 499 [14,3,12,5]
+ CRUSH rule 0 x 500 [15,9,6,12]
+ CRUSH rule 0 x 501 [10,13,1,9]
+ CRUSH rule 0 x 502 [5,1,14,11]
+ CRUSH rule 0 x 503 [15,10,7,9]
+ CRUSH rule 0 x 504 [13,2,7,1]
+ CRUSH rule 0 x 505 [12,7,5,2]
+ CRUSH rule 0 x 506 [11,7,9,14]
+ CRUSH rule 0 x 507 [4,14,13,3]
+ CRUSH rule 0 x 508 [12,1,4,9]
+ CRUSH rule 0 x 509 [4,2,6,9]
+ CRUSH rule 0 x 510 [5,3,1,12]
+ CRUSH rule 0 x 511 [2,12,10,6]
+ CRUSH rule 0 x 512 [15,11,3,5]
+ CRUSH rule 0 x 513 [4,9,11,3]
+ CRUSH rule 0 x 514 [11,9,3,4]
+ CRUSH rule 0 x 515 [12,14,6,5]
+ CRUSH rule 0 x 516 [14,11,1,12]
+ CRUSH rule 0 x 517 [11,5,6,13]
+ CRUSH rule 0 x 518 [3,5,7,12]
+ CRUSH rule 0 x 519 [12,14,2,1]
+ CRUSH rule 0 x 520 [12,4,2,10]
+ CRUSH rule 0 x 521 [11,5,9,6]
+ CRUSH rule 0 x 522 [4,12,11,1]
+ CRUSH rule 0 x 523 [3,1,5,9]
+ CRUSH rule 0 x 524 [15,9,3,11]
+ CRUSH rule 0 x 525 [3,15,11,6]
+ CRUSH rule 0 x 526 [10,2,5,13]
+ CRUSH rule 0 x 527 [3,13,4,1]
+ CRUSH rule 0 x 528 [12,7,15,10]
+ CRUSH rule 0 x 529 [6,4,10,12]
+ CRUSH rule 0 x 530 [11,9,12,7]
+ CRUSH rule 0 x 531 [9,15,4,7]
+ CRUSH rule 0 x 532 [5,3,13,7]
+ CRUSH rule 0 x 533 [12,15,1,2]
+ CRUSH rule 0 x 534 [11,9,3,7]
+ CRUSH rule 0 x 535 [11,1,3,5]
+ CRUSH rule 0 x 536 [9,1,14,13]
+ CRUSH rule 0 x 537 [15,5,13,2]
+ CRUSH rule 0 x 538 [13,5,11,2]
+ CRUSH rule 0 x 539 [10,12,6,14]
+ CRUSH rule 0 x 540 [12,15,7,3]
+ CRUSH rule 0 x 541 [2,1,6,11]
+ CRUSH rule 0 x 542 [3,9,15,5]
+ CRUSH rule 0 x 543 [4,10,9,3]
+ CRUSH rule 0 x 544 [3,15,9,11]
+ CRUSH rule 0 x 545 [14,10,7,12]
+ CRUSH rule 0 x 546 [5,15,13,7]
+ CRUSH rule 0 x 547 [5,13,7,9]
+ CRUSH rule 0 x 548 [11,7,12,15]
+ CRUSH rule 0 x 549 [14,1,4,9]
+ CRUSH rule 0 x 550 [9,15,3,13]
+ CRUSH rule 0 x 551 [11,2,15,6]
+ CRUSH rule 0 x 552 [2,11,14,1]
+ CRUSH rule 0 x 553 [11,9,14,6]
+ CRUSH rule 0 x 554 [11,14,6,4]
+ CRUSH rule 0 x 555 [6,5,10,9]
+ CRUSH rule 0 x 556 [15,6,3,13]
+ CRUSH rule 0 x 557 [12,2,5,14]
+ CRUSH rule 0 x 558 [12,1,6,15]
+ CRUSH rule 0 x 559 [2,13,5,10]
+ CRUSH rule 0 x 560 [4,9,12,6]
+ CRUSH rule 0 x 561 [12,7,1,2]
+ CRUSH rule 0 x 562 [7,13,9,14]
+ CRUSH rule 0 x 563 [15,4,3,10]
+ CRUSH rule 0 x 564 [2,13,7,1]
+ CRUSH rule 0 x 565 [3,12,4,1]
+ CRUSH rule 0 x 566 [6,14,4,2]
+ CRUSH rule 0 x 567 [15,4,11,6]
+ CRUSH rule 0 x 568 [4,14,1,6]
+ CRUSH rule 0 x 569 [11,3,15,13]
+ CRUSH rule 0 x 570 [1,10,13,4]
+ CRUSH rule 0 x 571 [10,12,14,9]
+ CRUSH rule 0 x 572 [12,14,3,10]
+ CRUSH rule 0 x 573 [7,15,11,2]
+ CRUSH rule 0 x 574 [11,14,13,1]
+ CRUSH rule 0 x 575 [5,13,15,9]
+ CRUSH rule 0 x 576 [3,15,11,9]
+ CRUSH rule 0 x 577 [13,9,6,15]
+ CRUSH rule 0 x 578 [4,10,1,2]
+ CRUSH rule 0 x 579 [13,1,15,2]
+ CRUSH rule 0 x 580 [3,12,4,1]
+ CRUSH rule 0 x 581 [7,14,12,10]
+ CRUSH rule 0 x 582 [10,5,13,14]
+ CRUSH rule 0 x 583 [4,15,1,9]
+ CRUSH rule 0 x 584 [10,1,5,13]
+ CRUSH rule 0 x 585 [5,3,6,1]
+ CRUSH rule 0 x 586 [7,10,14,12]
+ CRUSH rule 0 x 587 [11,6,9,4]
+ CRUSH rule 0 x 588 [3,12,7,15]
+ CRUSH rule 0 x 589 [9,7,12,1]
+ CRUSH rule 0 x 590 [12,1,3,9]
+ CRUSH rule 0 x 591 [2,6,14,13]
+ CRUSH rule 0 x 592 [15,12,9,7]
+ CRUSH rule 0 x 593 [13,14,5,11]
+ CRUSH rule 0 x 594 [12,14,2,9]
+ CRUSH rule 0 x 595 [12,7,10,3]
+ CRUSH rule 0 x 596 [2,7,12,11]
+ CRUSH rule 0 x 597 [15,1,2,10]
+ CRUSH rule 0 x 598 [11,5,9,14]
+ CRUSH rule 0 x 599 [13,11,1,5]
+ CRUSH rule 0 x 600 [4,12,3,10]
+ CRUSH rule 0 x 601 [13,5,15,2]
+ CRUSH rule 0 x 602 [3,11,7,1]
+ CRUSH rule 0 x 603 [3,1,4,14]
+ CRUSH rule 0 x 604 [14,2,6,1]
+ CRUSH rule 0 x 605 [2,7,12,5]
+ CRUSH rule 0 x 606 [12,15,1,5]
+ CRUSH rule 0 x 607 [3,9,10,14]
+ CRUSH rule 0 x 608 [13,10,1,7]
+ CRUSH rule 0 x 609 [14,3,7,9]
+ CRUSH rule 0 x 610 [7,10,5,1]
+ CRUSH rule 0 x 611 [13,1,5,3]
+ CRUSH rule 0 x 612 [7,1,2,13]
+ CRUSH rule 0 x 613 [10,7,14,9]
+ CRUSH rule 0 x 614 [9,4,15,3]
+ CRUSH rule 0 x 615 [9,4,11,2]
+ CRUSH rule 0 x 616 [10,14,1,5]
+ CRUSH rule 0 x 617 [15,7,2,11]
+ CRUSH rule 0 x 618 [4,2,10,6]
+ CRUSH rule 0 x 619 [15,4,3,9]
+ CRUSH rule 0 x 620 [3,7,11,14]
+ CRUSH rule 0 x 621 [3,6,4,14]
+ CRUSH rule 0 x 622 [10,2,13,5]
+ CRUSH rule 0 x 623 [4,9,14,7]
+ CRUSH rule 0 x 624 [3,9,15,6]
+ CRUSH rule 0 x 625 [11,7,3,5]
+ CRUSH rule 0 x 626 [10,12,2,1]
+ CRUSH rule 0 x 627 [1,12,10,14]
+ CRUSH rule 0 x 628 [15,13,11,4]
+ CRUSH rule 0 x 629 [5,6,15,12]
+ CRUSH rule 0 x 630 [1,4,12,9]
+ CRUSH rule 0 x 631 [5,7,1,15]
+ CRUSH rule 0 x 632 [12,3,11,9]
+ CRUSH rule 0 x 633 [14,4,3,7]
+ CRUSH rule 0 x 634 [6,9,5,3]
+ CRUSH rule 0 x 635 [6,5,2,15]
+ CRUSH rule 0 x 636 [13,6,11,3]
+ CRUSH rule 0 x 637 [3,1,10,6]
+ CRUSH rule 0 x 638 [10,15,3,5]
+ CRUSH rule 0 x 639 [6,9,14,4]
+ CRUSH rule 0 x 640 [9,6,1,11]
+ CRUSH rule 0 x 641 [10,6,5,14]
+ CRUSH rule 0 x 642 [1,15,4,6]
+ CRUSH rule 0 x 643 [3,7,5,1]
+ CRUSH rule 0 x 644 [15,13,6,9]
+ CRUSH rule 0 x 645 [14,2,4,9]
+ CRUSH rule 0 x 646 [5,13,14,1]
+ CRUSH rule 0 x 647 [10,1,9,13]
+ CRUSH rule 0 x 648 [6,5,2,14]
+ CRUSH rule 0 x 649 [3,9,13,11]
+ CRUSH rule 0 x 650 [10,9,4,15]
+ CRUSH rule 0 x 651 [3,9,5,7]
+ CRUSH rule 0 x 652 [15,9,4,6]
+ CRUSH rule 0 x 653 [11,14,1,3]
+ CRUSH rule 0 x 654 [13,6,2,10]
+ CRUSH rule 0 x 655 [6,3,4,15]
+ CRUSH rule 0 x 656 [3,15,1,4]
+ CRUSH rule 0 x 657 [11,15,3,5]
+ CRUSH rule 0 x 658 [7,2,10,12]
+ CRUSH rule 0 x 659 [2,5,14,6]
+ CRUSH rule 0 x 660 [13,14,10,6]
+ CRUSH rule 0 x 661 [7,15,3,12]
+ CRUSH rule 0 x 662 [15,2,12,5]
+ CRUSH rule 0 x 663 [14,9,13,10]
+ CRUSH rule 0 x 664 [6,10,12,4]
+ CRUSH rule 0 x 665 [2,9,12,1]
+ CRUSH rule 0 x 666 [12,3,6,1]
+ CRUSH rule 0 x 667 [1,9,12,10]
+ CRUSH rule 0 x 668 [9,5,1,2]
+ CRUSH rule 0 x 669 [9,7,14,5]
+ CRUSH rule 0 x 670 [6,10,9,13]
+ CRUSH rule 0 x 671 [6,15,5,10]
+ CRUSH rule 0 x 672 [2,9,13,1]
+ CRUSH rule 0 x 673 [7,10,5,9]
+ CRUSH rule 0 x 674 [7,12,10,1]
+ CRUSH rule 0 x 675 [9,5,1,10]
+ CRUSH rule 0 x 676 [10,12,2,1]
+ CRUSH rule 0 x 677 [2,12,1,4]
+ CRUSH rule 0 x 678 [1,2,4,10]
+ CRUSH rule 0 x 679 [5,6,12,15]
+ CRUSH rule 0 x 680 [7,11,3,1]
+ CRUSH rule 0 x 681 [6,4,3,11]
+ CRUSH rule 0 x 682 [6,1,11,15]
+ CRUSH rule 0 x 683 [6,13,2,4]
+ CRUSH rule 0 x 684 [9,11,3,7]
+ CRUSH rule 0 x 685 [5,1,15,7]
+ CRUSH rule 0 x 686 [1,9,11,14]
+ CRUSH rule 0 x 687 [7,13,3,5]
+ CRUSH rule 0 x 688 [11,9,1,14]
+ CRUSH rule 0 x 689 [5,2,9,12]
+ CRUSH rule 0 x 690 [9,7,10,3]
+ CRUSH rule 0 x 691 [11,15,9,5]
+ CRUSH rule 0 x 692 [15,5,1,2]
+ CRUSH rule 0 x 693 [5,6,12,15]
+ CRUSH rule 0 x 694 [4,7,1,10]
+ CRUSH rule 0 x 695 [6,13,14,10]
+ CRUSH rule 0 x 696 [1,2,4,14]
+ CRUSH rule 0 x 697 [13,11,3,6]
+ CRUSH rule 0 x 698 [11,13,4,2]
+ CRUSH rule 0 x 699 [7,14,12,4]
+ CRUSH rule 0 x 700 [12,14,11,9]
+ CRUSH rule 0 x 701 [3,13,1,14]
+ CRUSH rule 0 x 702 [3,12,15,6]
+ CRUSH rule 0 x 703 [15,11,13,3]
+ CRUSH rule 0 x 704 [6,4,2,15]
+ CRUSH rule 0 x 705 [14,6,11,5]
+ CRUSH rule 0 x 706 [1,12,3,6]
+ CRUSH rule 0 x 707 [4,7,14,3]
+ CRUSH rule 0 x 708 [3,10,5,1]
+ CRUSH rule 0 x 709 [11,12,3,7]
+ CRUSH rule 0 x 710 [14,2,11,9]
+ CRUSH rule 0 x 711 [14,3,9,10]
+ CRUSH rule 0 x 712 [12,3,11,15]
+ CRUSH rule 0 x 713 [11,9,3,15]
+ CRUSH rule 0 x 714 [12,1,9,7]
+ CRUSH rule 0 x 715 [6,1,14,4]
+ CRUSH rule 0 x 716 [11,13,9,14]
+ CRUSH rule 0 x 717 [12,4,10,9]
+ CRUSH rule 0 x 718 [7,15,5,2]
+ CRUSH rule 0 x 719 [5,15,13,3]
+ CRUSH rule 0 x 720 [4,13,10,2]
+ CRUSH rule 0 x 721 [11,3,14,9]
+ CRUSH rule 0 x 722 [2,4,6,1]
+ CRUSH rule 0 x 723 [2,1,12,15]
+ CRUSH rule 0 x 724 [7,1,9,10]
+ CRUSH rule 0 x 725 [11,12,7,15]
+ CRUSH rule 0 x 726 [7,14,4,3]
+ CRUSH rule 0 x 727 [2,5,1,11]
+ CRUSH rule 0 x 728 [13,11,4,6]
+ CRUSH rule 0 x 729 [15,11,4,6]
+ CRUSH rule 0 x 730 [3,7,1,13]
+ CRUSH rule 0 x 731 [9,1,6,5]
+ CRUSH rule 0 x 732 [1,2,10,13]
+ CRUSH rule 0 x 733 [11,3,5,6]
+ CRUSH rule 0 x 734 [14,3,11,7]
+ CRUSH rule 0 x 735 [6,9,2,10]
+ CRUSH rule 0 x 736 [3,9,1,11]
+ CRUSH rule 0 x 737 [1,4,2,12]
+ CRUSH rule 0 x 738 [11,15,7,4]
+ CRUSH rule 0 x 739 [11,12,6,2]
+ CRUSH rule 0 x 740 [7,9,10,13]
+ CRUSH rule 0 x 741 [12,11,7,15]
+ CRUSH rule 0 x 742 [9,7,4,11]
+ CRUSH rule 0 x 743 [5,13,9,15]
+ CRUSH rule 0 x 744 [6,2,13,1]
+ CRUSH rule 0 x 745 [3,6,1,4]
+ CRUSH rule 0 x 746 [3,7,9,10]
+ CRUSH rule 0 x 747 [15,11,5,2]
+ CRUSH rule 0 x 748 [6,10,13,2]
+ CRUSH rule 0 x 749 [14,9,10,7]
+ CRUSH rule 0 x 750 [1,14,6,5]
+ CRUSH rule 0 x 751 [15,1,6,9]
+ CRUSH rule 0 x 752 [13,1,7,3]
+ CRUSH rule 0 x 753 [4,11,1,3]
+ CRUSH rule 0 x 754 [14,12,11,4]
+ CRUSH rule 0 x 755 [13,6,1,10]
+ CRUSH rule 0 x 756 [3,4,14,6]
+ CRUSH rule 0 x 757 [10,6,1,4]
+ CRUSH rule 0 x 758 [6,3,4,10]
+ CRUSH rule 0 x 759 [5,7,3,14]
+ CRUSH rule 0 x 760 [1,15,10,12]
+ CRUSH rule 0 x 761 [2,12,1,14]
+ CRUSH rule 0 x 762 [1,4,10,9]
+ CRUSH rule 0 x 763 [4,13,1,14]
+ CRUSH rule 0 x 764 [1,14,6,13]
+ CRUSH rule 0 x 765 [9,15,2,13]
+ CRUSH rule 0 x 766 [11,2,7,15]
+ CRUSH rule 0 x 767 [6,11,4,3]
+ CRUSH rule 0 x 768 [2,12,15,7]
+ CRUSH rule 0 x 769 [15,1,9,2]
+ CRUSH rule 0 x 770 [15,13,4,6]
+ CRUSH rule 0 x 771 [9,2,12,11]
+ CRUSH rule 0 x 772 [4,3,13,11]
+ CRUSH rule 0 x 773 [3,7,4,15]
+ CRUSH rule 0 x 774 [12,6,3,15]
+ CRUSH rule 0 x 775 [5,10,14,2]
+ CRUSH rule 0 x 776 [10,15,3,9]
+ CRUSH rule 0 x 777 [11,13,4,7]
+ CRUSH rule 0 x 778 [13,1,9,11]
+ CRUSH rule 0 x 779 [5,11,1,14]
+ CRUSH rule 0 x 780 [13,9,3,6]
+ CRUSH rule 0 x 781 [5,7,14,3]
+ CRUSH rule 0 x 782 [2,15,9,7]
+ CRUSH rule 0 x 783 [12,7,5,14]
+ CRUSH rule 0 x 784 [14,1,10,13]
+ CRUSH rule 0 x 785 [6,12,1,2]
+ CRUSH rule 0 x 786 [10,5,2,15]
+ CRUSH rule 0 x 787 [1,12,10,2]
+ CRUSH rule 0 x 788 [4,2,9,13]
+ CRUSH rule 0 x 789 [9,2,14,7]
+ CRUSH rule 0 x 790 [15,2,7,4]
+ CRUSH rule 0 x 791 [9,4,7,13]
+ CRUSH rule 0 x 792 [6,4,15,10]
+ CRUSH rule 0 x 793 [15,9,6,2]
+ CRUSH rule 0 x 794 [5,12,2,14]
+ CRUSH rule 0 x 795 [6,14,12,4]
+ CRUSH rule 0 x 796 [11,2,12,6]
+ CRUSH rule 0 x 797 [14,3,7,1]
+ CRUSH rule 0 x 798 [5,11,6,13]
+ CRUSH rule 0 x 799 [2,9,14,4]
+ CRUSH rule 0 x 800 [6,3,4,11]
+ CRUSH rule 0 x 801 [2,5,6,13]
+ CRUSH rule 0 x 802 [1,4,12,7]
+ CRUSH rule 0 x 803 [7,2,4,1]
+ CRUSH rule 0 x 804 [5,14,9,7]
+ CRUSH rule 0 x 805 [13,4,3,1]
+ CRUSH rule 0 x 806 [6,2,13,4]
+ CRUSH rule 0 x 807 [14,2,7,4]
+ CRUSH rule 0 x 808 [2,15,12,7]
+ CRUSH rule 0 x 809 [1,11,7,12]
+ CRUSH rule 0 x 810 [2,5,9,12]
+ CRUSH rule 0 x 811 [15,6,3,10]
+ CRUSH rule 0 x 812 [7,11,2,14]
+ CRUSH rule 0 x 813 [4,10,13,14]
+ CRUSH rule 0 x 814 [13,4,9,3]
+ CRUSH rule 0 x 815 [15,12,9,4]
+ CRUSH rule 0 x 816 [14,10,13,7]
+ CRUSH rule 0 x 817 [10,7,2,15]
+ CRUSH rule 0 x 818 [15,2,11,4]
+ CRUSH rule 0 x 819 [5,12,10,6]
+ CRUSH rule 0 x 820 [3,6,9,12]
+ CRUSH rule 0 x 821 [15,10,9,13]
+ CRUSH rule 0 x 822 [10,13,2,9]
+ CRUSH rule 0 x 823 [2,6,12,10]
+ CRUSH rule 0 x 824 [3,7,9,13]
+ CRUSH rule 0 x 825 [10,5,14,6]
+ CRUSH rule 0 x 826 [5,2,11,15]
+ CRUSH rule 0 x 827 [13,5,1,3]
+ CRUSH rule 0 x 828 [12,6,10,5]
+ CRUSH rule 0 x 829 [13,6,15,10]
+ CRUSH rule 0 x 830 [15,13,2,9]
+ CRUSH rule 0 x 831 [1,4,11,12]
+ CRUSH rule 0 x 832 [14,11,13,2]
+ CRUSH rule 0 x 833 [9,13,3,11]
+ CRUSH rule 0 x 834 [9,7,5,1]
+ CRUSH rule 0 x 835 [14,3,13,6]
+ CRUSH rule 0 x 836 [3,9,10,13]
+ CRUSH rule 0 x 837 [15,12,11,2]
+ CRUSH rule 0 x 838 [12,14,9,2]
+ CRUSH rule 0 x 839 [3,4,6,10]
+ CRUSH rule 0 x 840 [10,15,12,4]
+ CRUSH rule 0 x 841 [3,5,7,12]
+ CRUSH rule 0 x 842 [9,13,2,6]
+ CRUSH rule 0 x 843 [14,7,4,9]
+ CRUSH rule 0 x 844 [7,1,4,15]
+ CRUSH rule 0 x 845 [13,6,1,15]
+ CRUSH rule 0 x 846 [3,7,15,13]
+ CRUSH rule 0 x 847 [12,15,11,5]
+ CRUSH rule 0 x 848 [11,13,1,14]
+ CRUSH rule 0 x 849 [3,15,11,9]
+ CRUSH rule 0 x 850 [1,3,10,6]
+ CRUSH rule 0 x 851 [14,4,3,6]
+ CRUSH rule 0 x 852 [9,12,4,7]
+ CRUSH rule 0 x 853 [13,14,6,11]
+ CRUSH rule 0 x 854 [7,11,12,1]
+ CRUSH rule 0 x 855 [14,4,12,6]
+ CRUSH rule 0 x 856 [5,10,7,3]
+ CRUSH rule 0 x 857 [4,3,13,11]
+ CRUSH rule 0 x 858 [5,15,6,3]
+ CRUSH rule 0 x 859 [5,15,6,2]
+ CRUSH rule 0 x 860 [11,14,1,12]
+ CRUSH rule 0 x 861 [13,7,4,10]
+ CRUSH rule 0 x 862 [5,10,9,7]
+ CRUSH rule 0 x 863 [11,6,3,9]
+ CRUSH rule 0 x 864 [6,13,4,2]
+ CRUSH rule 0 x 865 [4,1,14,11]
+ CRUSH rule 0 x 866 [2,13,4,15]
+ CRUSH rule 0 x 867 [12,2,9,10]
+ CRUSH rule 0 x 868 [14,11,7,2]
+ CRUSH rule 0 x 869 [10,13,7,14]
+ CRUSH rule 0 x 870 [14,9,11,4]
+ CRUSH rule 0 x 871 [6,2,1,4]
+ CRUSH rule 0 x 872 [6,1,15,3]
+ CRUSH rule 0 x 873 [2,5,12,10]
+ CRUSH rule 0 x 874 [12,4,7,2]
+ CRUSH rule 0 x 875 [10,6,14,1]
+ CRUSH rule 0 x 876 [14,7,13,3]
+ CRUSH rule 0 x 877 [15,11,13,9]
+ CRUSH rule 0 x 878 [7,14,3,13]
+ CRUSH rule 0 x 879 [12,2,7,4]
+ CRUSH rule 0 x 880 [2,12,10,7]
+ CRUSH rule 0 x 881 [6,3,1,11]
+ CRUSH rule 0 x 882 [11,13,7,1]
+ CRUSH rule 0 x 883 [13,1,3,10]
+ CRUSH rule 0 x 884 [6,15,4,9]
+ CRUSH rule 0 x 885 [14,7,9,4]
+ CRUSH rule 0 x 886 [13,11,4,2]
+ CRUSH rule 0 x 887 [14,4,12,11]
+ CRUSH rule 0 x 888 [10,12,7,15]
+ CRUSH rule 0 x 889 [15,13,4,1]
+ CRUSH rule 0 x 890 [10,12,14,2]
+ CRUSH rule 0 x 891 [9,5,11,6]
+ CRUSH rule 0 x 892 [12,15,2,4]
+ CRUSH rule 0 x 893 [1,3,5,9]
+ CRUSH rule 0 x 894 [7,2,11,13]
+ CRUSH rule 0 x 895 [2,1,11,5]
+ CRUSH rule 0 x 896 [9,1,14,10]
+ CRUSH rule 0 x 897 [7,5,14,3]
+ CRUSH rule 0 x 898 [10,6,12,9]
+ CRUSH rule 0 x 899 [1,11,5,3]
+ CRUSH rule 0 x 900 [2,9,10,7]
+ CRUSH rule 0 x 901 [9,12,11,3]
+ CRUSH rule 0 x 902 [4,2,6,15]
+ CRUSH rule 0 x 903 [14,10,3,1]
+ CRUSH rule 0 x 904 [15,12,4,9]
+ CRUSH rule 0 x 905 [12,6,11,3]
+ CRUSH rule 0 x 906 [14,11,12,2]
+ CRUSH rule 0 x 907 [7,12,3,9]
+ CRUSH rule 0 x 908 [2,15,9,6]
+ CRUSH rule 0 x 909 [10,14,1,13]
+ CRUSH rule 0 x 910 [12,7,4,15]
+ CRUSH rule 0 x 911 [11,15,2,4]
+ CRUSH rule 0 x 912 [6,4,14,13]
+ CRUSH rule 0 x 913 [4,6,10,1]
+ CRUSH rule 0 x 914 [4,15,2,10]
+ CRUSH rule 0 x 915 [12,14,1,9]
+ CRUSH rule 0 x 916 [3,1,11,5]
+ CRUSH rule 0 x 917 [1,15,6,5]
+ CRUSH rule 0 x 918 [7,14,11,4]
+ CRUSH rule 0 x 919 [10,7,3,13]
+ CRUSH rule 0 x 920 [4,2,10,15]
+ CRUSH rule 0 x 921 [1,11,6,13]
+ CRUSH rule 0 x 922 [6,4,14,13]
+ CRUSH rule 0 x 923 [12,2,5,14]
+ CRUSH rule 0 x 924 [6,2,14,13]
+ CRUSH rule 0 x 925 [12,15,2,10]
+ CRUSH rule 0 x 926 [3,13,10,1]
+ CRUSH rule 0 x 927 [6,5,1,11]
+ CRUSH rule 0 x 928 [13,1,3,9]
+ CRUSH rule 0 x 929 [10,7,1,5]
+ CRUSH rule 0 x 930 [7,15,10,5]
+ CRUSH rule 0 x 931 [6,15,11,9]
+ CRUSH rule 0 x 932 [13,2,5,11]
+ CRUSH rule 0 x 933 [12,7,14,10]
+ CRUSH rule 0 x 934 [12,2,5,7]
+ CRUSH rule 0 x 935 [6,11,1,14]
+ CRUSH rule 0 x 936 [9,12,7,5]
+ CRUSH rule 0 x 937 [14,2,11,1]
+ CRUSH rule 0 x 938 [14,3,5,11]
+ CRUSH rule 0 x 939 [6,4,14,9]
+ CRUSH rule 0 x 940 [13,11,4,2]
+ CRUSH rule 0 x 941 [3,12,4,7]
+ CRUSH rule 0 x 942 [15,12,10,4]
+ CRUSH rule 0 x 943 [10,2,4,9]
+ CRUSH rule 0 x 944 [2,9,4,7]
+ CRUSH rule 0 x 945 [10,15,2,9]
+ CRUSH rule 0 x 946 [11,15,7,12]
+ CRUSH rule 0 x 947 [11,3,14,1]
+ CRUSH rule 0 x 948 [7,13,11,5]
+ CRUSH rule 0 x 949 [9,1,12,5]
+ CRUSH rule 0 x 950 [9,15,13,6]
+ CRUSH rule 0 x 951 [2,6,12,9]
+ CRUSH rule 0 x 952 [9,7,15,3]
+ CRUSH rule 0 x 953 [1,3,6,10]
+ CRUSH rule 0 x 954 [10,2,14,9]
+ CRUSH rule 0 x 955 [7,14,3,1]
+ CRUSH rule 0 x 956 [1,6,11,5]
+ CRUSH rule 0 x 957 [14,11,1,12]
+ CRUSH rule 0 x 958 [15,4,3,11]
+ CRUSH rule 0 x 959 [2,1,12,15]
+ CRUSH rule 0 x 960 [2,6,11,13]
+ CRUSH rule 0 x 961 [3,13,11,9]
+ CRUSH rule 0 x 962 [5,11,3,14]
+ CRUSH rule 0 x 963 [13,10,15,4]
+ CRUSH rule 0 x 964 [7,11,4,9]
+ CRUSH rule 0 x 965 [12,2,9,7]
+ CRUSH rule 0 x 966 [12,14,9,4]
+ CRUSH rule 0 x 967 [7,5,3,10]
+ CRUSH rule 0 x 968 [12,15,4,9]
+ CRUSH rule 0 x 969 [11,4,7,1]
+ CRUSH rule 0 x 970 [5,12,10,1]
+ CRUSH rule 0 x 971 [1,9,4,12]
+ CRUSH rule 0 x 972 [12,3,14,5]
+ CRUSH rule 0 x 973 [1,10,4,12]
+ CRUSH rule 0 x 974 [7,11,1,2]
+ CRUSH rule 0 x 975 [7,9,15,12]
+ CRUSH rule 0 x 976 [7,3,15,5]
+ CRUSH rule 0 x 977 [14,3,6,10]
+ CRUSH rule 0 x 978 [12,5,11,1]
+ CRUSH rule 0 x 979 [5,1,13,6]
+ CRUSH rule 0 x 980 [15,11,5,6]
+ CRUSH rule 0 x 981 [5,11,15,12]
+ CRUSH rule 0 x 982 [2,6,14,11]
+ CRUSH rule 0 x 983 [3,12,10,9]
+ CRUSH rule 0 x 984 [15,13,1,10]
+ CRUSH rule 0 x 985 [11,2,15,1]
+ CRUSH rule 0 x 986 [6,13,9,1]
+ CRUSH rule 0 x 987 [13,14,5,10]
+ CRUSH rule 0 x 988 [12,9,10,14]
+ CRUSH rule 0 x 989 [7,4,3,15]
+ CRUSH rule 0 x 990 [1,10,9,13]
+ CRUSH rule 0 x 991 [7,11,1,14]
+ CRUSH rule 0 x 992 [9,10,2,13]
+ CRUSH rule 0 x 993 [6,10,14,12]
+ CRUSH rule 0 x 994 [3,13,15,4]
+ CRUSH rule 0 x 995 [15,6,12,2]
+ CRUSH rule 0 x 996 [15,10,5,3]
+ CRUSH rule 0 x 997 [15,2,1,12]
+ CRUSH rule 0 x 998 [6,1,9,5]
+ CRUSH rule 0 x 999 [9,10,15,5]
+ CRUSH rule 0 x 1000 [14,2,9,4]
+ CRUSH rule 0 x 1001 [11,14,4,2]
+ CRUSH rule 0 x 1002 [1,10,14,2]
+ CRUSH rule 0 x 1003 [10,7,5,14]
+ CRUSH rule 0 x 1004 [15,1,4,6]
+ CRUSH rule 0 x 1005 [6,12,2,10]
+ CRUSH rule 0 x 1006 [10,12,15,1]
+ CRUSH rule 0 x 1007 [1,7,13,14]
+ CRUSH rule 0 x 1008 [7,4,9,11]
+ CRUSH rule 0 x 1009 [5,2,11,7]
+ CRUSH rule 0 x 1010 [10,2,15,6]
+ CRUSH rule 0 x 1011 [6,3,12,1]
+ CRUSH rule 0 x 1012 [12,6,9,15]
+ CRUSH rule 0 x 1013 [2,14,12,4]
+ CRUSH rule 0 x 1014 [1,13,7,2]
+ CRUSH rule 0 x 1015 [12,6,10,1]
+ CRUSH rule 0 x 1016 [10,13,14,3]
+ CRUSH rule 0 x 1017 [5,11,14,7]
+ CRUSH rule 0 x 1018 [13,11,14,1]
+ CRUSH rule 0 x 1019 [10,13,14,7]
+ CRUSH rule 0 x 1020 [3,1,13,4]
+ CRUSH rule 0 x 1021 [2,11,14,9]
+ CRUSH rule 0 x 1022 [15,5,7,2]
+ CRUSH rule 0 x 1023 [15,2,9,12]
+ rule 0 (replicated_ruleset) num_rep 4 result size == 4:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15,12]
+ CRUSH rule 0 x 1 [10,15,1,2,13]
+ CRUSH rule 0 x 2 [1,12,2,6,5]
+ CRUSH rule 0 x 3 [15,4,10,2,9]
+ CRUSH rule 0 x 4 [14,2,10,1,9]
+ CRUSH rule 0 x 5 [7,4,11,2,13]
+ CRUSH rule 0 x 6 [12,6,10,9,3]
+ CRUSH rule 0 x 7 [9,2,6,12,11]
+ CRUSH rule 0 x 8 [10,2,15,1,4]
+ CRUSH rule 0 x 9 [7,1,14,2,11]
+ CRUSH rule 0 x 10 [10,14,4,1,2]
+ CRUSH rule 0 x 11 [13,9,14,7,5]
+ CRUSH rule 0 x 12 [7,1,2,5,13]
+ CRUSH rule 0 x 13 [3,5,12,7,9]
+ CRUSH rule 0 x 14 [13,5,2,7,10]
+ CRUSH rule 0 x 15 [15,1,9,6,13]
+ CRUSH rule 0 x 16 [7,11,14,2,13]
+ CRUSH rule 0 x 17 [10,1,13,2,4]
+ CRUSH rule 0 x 18 [1,7,3,10,5]
+ CRUSH rule 0 x 19 [7,12,2,4,15]
+ CRUSH rule 0 x 20 [14,12,3,10,9]
+ CRUSH rule 0 x 21 [3,12,1,10,4]
+ CRUSH rule 0 x 22 [6,3,13,11,4]
+ CRUSH rule 0 x 23 [10,5,13,9,3]
+ CRUSH rule 0 x 24 [12,11,3,1,9]
+ CRUSH rule 0 x 25 [7,12,15,1,3]
+ CRUSH rule 0 x 26 [1,7,13,2,14]
+ CRUSH rule 0 x 27 [3,6,15,4,13]
+ CRUSH rule 0 x 28 [14,4,3,9,6]
+ CRUSH rule 0 x 29 [5,14,12,11,6]
+ CRUSH rule 0 x 30 [2,5,6,9,1]
+ CRUSH rule 0 x 31 [5,15,10,1,9]
+ CRUSH rule 0 x 32 [9,10,2,1,13]
+ CRUSH rule 0 x 33 [13,4,9,2,7]
+ CRUSH rule 0 x 34 [13,15,2,4,1]
+ CRUSH rule 0 x 35 [4,14,3,13,10]
+ CRUSH rule 0 x 36 [3,12,9,7,5]
+ CRUSH rule 0 x 37 [9,2,6,14,11]
+ CRUSH rule 0 x 38 [3,4,13,10,9]
+ CRUSH rule 0 x 39 [12,7,14,11,1]
+ CRUSH rule 0 x 40 [10,1,9,5,15]
+ CRUSH rule 0 x 41 [4,9,11,1,14]
+ CRUSH rule 0 x 42 [3,6,14,10,12]
+ CRUSH rule 0 x 43 [10,5,15,7,2]
+ CRUSH rule 0 x 44 [11,4,13,3,7]
+ CRUSH rule 0 x 45 [11,12,15,9,1]
+ CRUSH rule 0 x 46 [6,9,2,14,11]
+ CRUSH rule 0 x 47 [3,9,6,4,13]
+ CRUSH rule 0 x 48 [4,6,2,1,10]
+ CRUSH rule 0 x 49 [9,15,10,7,4]
+ CRUSH rule 0 x 50 [14,12,1,4,2]
+ CRUSH rule 0 x 51 [10,6,5,12,15]
+ CRUSH rule 0 x 52 [12,1,9,11,7]
+ CRUSH rule 0 x 53 [3,6,13,9,5]
+ CRUSH rule 0 x 54 [4,13,9,2,14]
+ CRUSH rule 0 x 55 [4,11,2,7,1]
+ CRUSH rule 0 x 56 [5,9,10,1,3]
+ CRUSH rule 0 x 57 [6,2,1,15,10]
+ CRUSH rule 0 x 58 [7,1,11,4,3]
+ CRUSH rule 0 x 59 [2,13,1,10,9]
+ CRUSH rule 0 x 60 [3,6,11,1,4]
+ CRUSH rule 0 x 61 [3,15,13,7,4]
+ CRUSH rule 0 x 62 [15,11,7,12,5]
+ CRUSH rule 0 x 63 [10,14,12,1,7]
+ CRUSH rule 0 x 64 [3,9,1,4,7]
+ CRUSH rule 0 x 65 [4,12,11,7,14]
+ CRUSH rule 0 x 66 [15,11,6,9,4]
+ CRUSH rule 0 x 67 [2,6,4,14,1]
+ CRUSH rule 0 x 68 [15,7,4,2,9]
+ CRUSH rule 0 x 69 [2,1,15,10,4]
+ CRUSH rule 0 x 70 [9,6,1,3,13]
+ CRUSH rule 0 x 71 [15,5,1,3,13]
+ CRUSH rule 0 x 72 [9,10,3,5,7]
+ CRUSH rule 0 x 73 [5,3,11,1,7]
+ CRUSH rule 0 x 74 [11,7,9,5,1]
+ CRUSH rule 0 x 75 [9,7,11,14,12]
+ CRUSH rule 0 x 76 [6,1,3,5,14]
+ CRUSH rule 0 x 77 [7,4,2,13,9]
+ CRUSH rule 0 x 78 [9,3,1,5,6]
+ CRUSH rule 0 x 79 [13,2,15,5,7]
+ CRUSH rule 0 x 80 [15,2,6,4,13]
+ CRUSH rule 0 x 81 [15,2,1,11,4]
+ CRUSH rule 0 x 82 [14,13,5,11,6]
+ CRUSH rule 0 x 83 [4,15,3,9,10]
+ CRUSH rule 0 x 84 [10,7,9,15,3]
+ CRUSH rule 0 x 85 [3,15,9,7,4]
+ CRUSH rule 0 x 86 [10,9,14,1,13]
+ CRUSH rule 0 x 87 [15,10,7,12,5]
+ CRUSH rule 0 x 88 [4,13,3,1,9]
+ CRUSH rule 0 x 89 [3,9,7,4,1]
+ CRUSH rule 0 x 90 [4,9,7,12,11]
+ CRUSH rule 0 x 91 [6,11,9,1,2]
+ CRUSH rule 0 x 92 [1,5,10,9,13]
+ CRUSH rule 0 x 93 [9,3,15,13,7]
+ CRUSH rule 0 x 94 [9,2,12,5,6]
+ CRUSH rule 0 x 95 [7,15,4,10,9]
+ CRUSH rule 0 x 96 [2,15,11,7,5]
+ CRUSH rule 0 x 97 [4,11,2,13,1]
+ CRUSH rule 0 x 98 [11,13,9,3,15]
+ CRUSH rule 0 x 99 [12,4,11,7,3]
+ CRUSH rule 0 x 100 [9,4,10,15,7]
+ CRUSH rule 0 x 101 [15,7,1,9,10]
+ CRUSH rule 0 x 102 [3,11,14,6,13]
+ CRUSH rule 0 x 103 [13,11,6,14,4]
+ CRUSH rule 0 x 104 [14,6,3,5,9]
+ CRUSH rule 0 x 105 [14,10,1,9,3]
+ CRUSH rule 0 x 106 [6,5,13,2,14]
+ CRUSH rule 0 x 107 [3,1,10,14,13]
+ CRUSH rule 0 x 108 [5,10,7,2,15]
+ CRUSH rule 0 x 109 [9,1,13,7,15]
+ CRUSH rule 0 x 110 [5,1,11,3,7]
+ CRUSH rule 0 x 111 [10,1,9,7,5]
+ CRUSH rule 0 x 112 [1,10,4,14,2]
+ CRUSH rule 0 x 113 [6,10,13,9,1]
+ CRUSH rule 0 x 114 [5,13,6,2,1]
+ CRUSH rule 0 x 115 [10,13,14,3,9]
+ CRUSH rule 0 x 116 [1,14,13,2,11]
+ CRUSH rule 0 x 117 [5,6,1,12,15]
+ CRUSH rule 0 x 118 [10,4,13,15,9]
+ CRUSH rule 0 x 119 [14,12,11,4,6]
+ CRUSH rule 0 x 120 [11,3,14,13,4]
+ CRUSH rule 0 x 121 [9,5,1,11,7]
+ CRUSH rule 0 x 122 [4,3,14,1,11]
+ CRUSH rule 0 x 123 [3,10,5,6,9]
+ CRUSH rule 0 x 124 [12,2,1,5,14]
+ CRUSH rule 0 x 125 [9,12,15,1,6]
+ CRUSH rule 0 x 126 [7,15,10,9,2]
+ CRUSH rule 0 x 127 [4,14,9,13,1]
+ CRUSH rule 0 x 128 [3,12,1,10,4]
+ CRUSH rule 0 x 129 [11,13,14,2,9]
+ CRUSH rule 0 x 130 [3,13,5,14,10]
+ CRUSH rule 0 x 131 [12,1,6,15,4]
+ CRUSH rule 0 x 132 [11,15,13,9,2]
+ CRUSH rule 0 x 133 [3,6,9,11,15]
+ CRUSH rule 0 x 134 [12,5,6,15,3]
+ CRUSH rule 0 x 135 [3,14,12,4,6]
+ CRUSH rule 0 x 136 [15,6,9,4,10]
+ CRUSH rule 0 x 137 [14,3,6,11,1]
+ CRUSH rule 0 x 138 [13,15,4,10,2]
+ CRUSH rule 0 x 139 [11,2,13,9,1]
+ CRUSH rule 0 x 140 [11,4,12,15,2]
+ CRUSH rule 0 x 141 [6,12,15,11,3]
+ CRUSH rule 0 x 142 [3,14,7,9,11]
+ CRUSH rule 0 x 143 [9,6,4,2,14]
+ CRUSH rule 0 x 144 [13,7,11,2,14]
+ CRUSH rule 0 x 145 [12,2,6,10,9]
+ CRUSH rule 0 x 146 [1,5,9,2,6]
+ CRUSH rule 0 x 147 [1,4,9,11,2]
+ CRUSH rule 0 x 148 [12,7,9,2,14]
+ CRUSH rule 0 x 149 [2,5,9,12,11]
+ CRUSH rule 0 x 150 [1,15,2,10,7]
+ CRUSH rule 0 x 151 [2,9,14,7,1]
+ CRUSH rule 0 x 152 [5,9,2,6,10]
+ CRUSH rule 0 x 153 [6,9,4,15,2]
+ CRUSH rule 0 x 154 [3,11,7,1,4]
+ CRUSH rule 0 x 155 [14,12,7,3,5]
+ CRUSH rule 0 x 156 [7,13,3,10,15]
+ CRUSH rule 0 x 157 [15,1,6,4,3]
+ CRUSH rule 0 x 158 [15,1,10,6,12]
+ CRUSH rule 0 x 159 [4,14,3,12,10]
+ CRUSH rule 0 x 160 [5,7,3,14,11]
+ CRUSH rule 0 x 161 [1,2,11,4,6]
+ CRUSH rule 0 x 162 [10,6,1,12,2]
+ CRUSH rule 0 x 163 [15,1,10,2,6]
+ CRUSH rule 0 x 164 [9,14,10,7,12]
+ CRUSH rule 0 x 165 [11,7,2,13,9]
+ CRUSH rule 0 x 166 [1,2,12,14,4]
+ CRUSH rule 0 x 167 [9,7,3,4,11]
+ CRUSH rule 0 x 168 [13,2,4,1,6]
+ CRUSH rule 0 x 169 [1,4,9,14,13]
+ CRUSH rule 0 x 170 [1,15,7,9,12]
+ CRUSH rule 0 x 171 [9,2,10,7,1]
+ CRUSH rule 0 x 172 [14,4,10,12,9]
+ CRUSH rule 0 x 173 [5,10,12,15,6]
+ CRUSH rule 0 x 174 [15,6,4,12,1]
+ CRUSH rule 0 x 175 [5,7,9,3,10]
+ CRUSH rule 0 x 176 [9,6,3,14,13]
+ CRUSH rule 0 x 177 [2,9,10,13,4]
+ CRUSH rule 0 x 178 [12,11,7,14,3]
+ CRUSH rule 0 x 179 [2,10,13,9,5]
+ CRUSH rule 0 x 180 [3,11,5,15,7]
+ CRUSH rule 0 x 181 [9,12,6,5,1]
+ CRUSH rule 0 x 182 [5,13,11,2,1]
+ CRUSH rule 0 x 183 [5,7,10,13,3]
+ CRUSH rule 0 x 184 [2,5,11,12,7]
+ CRUSH rule 0 x 185 [13,5,7,11,2]
+ CRUSH rule 0 x 186 [6,14,13,5,10]
+ CRUSH rule 0 x 187 [1,4,11,13,6]
+ CRUSH rule 0 x 188 [9,13,5,14,10]
+ CRUSH rule 0 x 189 [6,12,4,9,2]
+ CRUSH rule 0 x 190 [9,13,15,10,3]
+ CRUSH rule 0 x 191 [7,11,4,1,15]
+ CRUSH rule 0 x 192 [2,11,5,15,6]
+ CRUSH rule 0 x 193 [3,13,6,10,4]
+ CRUSH rule 0 x 194 [3,13,4,14,6]
+ CRUSH rule 0 x 195 [5,7,10,12,1]
+ CRUSH rule 0 x 196 [4,15,1,10,9]
+ CRUSH rule 0 x 197 [14,10,13,4,6]
+ CRUSH rule 0 x 198 [2,5,6,15,9]
+ CRUSH rule 0 x 199 [2,10,4,15,1]
+ CRUSH rule 0 x 200 [7,14,11,4,1]
+ CRUSH rule 0 x 201 [9,14,1,7,4]
+ CRUSH rule 0 x 202 [14,11,7,3,5]
+ CRUSH rule 0 x 203 [12,5,7,15,1]
+ CRUSH rule 0 x 204 [6,11,3,12,14]
+ CRUSH rule 0 x 205 [15,4,6,10,13]
+ CRUSH rule 0 x 206 [13,11,2,15,7]
+ CRUSH rule 0 x 207 [2,11,7,4,14]
+ CRUSH rule 0 x 208 [13,1,6,14,9]
+ CRUSH rule 0 x 209 [6,15,13,1,11]
+ CRUSH rule 0 x 210 [13,11,2,7,5]
+ CRUSH rule 0 x 211 [2,14,1,13,11]
+ CRUSH rule 0 x 212 [10,1,12,15,5]
+ CRUSH rule 0 x 213 [3,9,6,5,15]
+ CRUSH rule 0 x 214 [7,15,4,1,10]
+ CRUSH rule 0 x 215 [6,1,4,13,3]
+ CRUSH rule 0 x 216 [12,9,6,2,1]
+ CRUSH rule 0 x 217 [12,11,1,14,2]
+ CRUSH rule 0 x 218 [12,10,15,6,1]
+ CRUSH rule 0 x 219 [3,11,14,6,4]
+ CRUSH rule 0 x 220 [14,4,3,12,10]
+ CRUSH rule 0 x 221 [15,5,2,6,12]
+ CRUSH rule 0 x 222 [10,4,3,15,7]
+ CRUSH rule 0 x 223 [9,7,11,1,4]
+ CRUSH rule 0 x 224 [1,7,10,2,12]
+ CRUSH rule 0 x 225 [10,5,2,6,1]
+ CRUSH rule 0 x 226 [4,1,9,3,13]
+ CRUSH rule 0 x 227 [7,2,12,15,5]
+ CRUSH rule 0 x 228 [2,15,11,1,6]
+ CRUSH rule 0 x 229 [9,3,7,14,1]
+ CRUSH rule 0 x 230 [10,5,7,2,15]
+ CRUSH rule 0 x 231 [2,7,5,13,9]
+ CRUSH rule 0 x 232 [10,5,13,1,9]
+ CRUSH rule 0 x 233 [6,12,11,4,9]
+ CRUSH rule 0 x 234 [10,1,2,12,5]
+ CRUSH rule 0 x 235 [13,14,7,10,1]
+ CRUSH rule 0 x 236 [2,15,9,12,1]
+ CRUSH rule 0 x 237 [3,12,9,10,4]
+ CRUSH rule 0 x 238 [2,10,4,15,6]
+ CRUSH rule 0 x 239 [4,15,10,7,9]
+ CRUSH rule 0 x 240 [15,5,13,7,2]
+ CRUSH rule 0 x 241 [7,9,15,12,1]
+ CRUSH rule 0 x 242 [14,2,6,9,10]
+ CRUSH rule 0 x 243 [2,11,5,1,15]
+ CRUSH rule 0 x 244 [13,9,15,3,11]
+ CRUSH rule 0 x 245 [12,9,15,3,1]
+ CRUSH rule 0 x 246 [15,3,5,11,7]
+ CRUSH rule 0 x 247 [6,4,9,12,1]
+ CRUSH rule 0 x 248 [5,13,7,11,9]
+ CRUSH rule 0 x 249 [10,14,7,3,9]
+ CRUSH rule 0 x 250 [12,15,1,10,5]
+ CRUSH rule 0 x 251 [13,2,15,5,6]
+ CRUSH rule 0 x 252 [7,5,13,9,3]
+ CRUSH rule 0 x 253 [3,13,15,10,7]
+ CRUSH rule 0 x 254 [2,9,13,14,4]
+ CRUSH rule 0 x 255 [1,9,13,2,6]
+ CRUSH rule 0 x 256 [6,9,13,1,3]
+ CRUSH rule 0 x 257 [15,12,3,9,6]
+ CRUSH rule 0 x 258 [12,5,6,10,2]
+ CRUSH rule 0 x 259 [9,10,4,3,14]
+ CRUSH rule 0 x 260 [10,12,6,9,3]
+ CRUSH rule 0 x 261 [13,7,2,1,15]
+ CRUSH rule 0 x 262 [15,3,12,7,4]
+ CRUSH rule 0 x 263 [12,6,10,9,5]
+ CRUSH rule 0 x 264 [13,14,11,3,1]
+ CRUSH rule 0 x 265 [12,10,14,5,7]
+ CRUSH rule 0 x 266 [14,7,11,1,2]
+ CRUSH rule 0 x 267 [12,11,6,5,1]
+ CRUSH rule 0 x 268 [4,1,15,12,6]
+ CRUSH rule 0 x 269 [11,1,15,5,13]
+ CRUSH rule 0 x 270 [7,11,12,3,1]
+ CRUSH rule 0 x 271 [4,7,3,13,15]
+ CRUSH rule 0 x 272 [15,5,13,10,6]
+ CRUSH rule 0 x 273 [2,10,7,12,1]
+ CRUSH rule 0 x 274 [10,2,5,6,13]
+ CRUSH rule 0 x 275 [10,3,4,7,14]
+ CRUSH rule 0 x 276 [5,12,9,2,11]
+ CRUSH rule 0 x 277 [14,3,13,4,1]
+ CRUSH rule 0 x 278 [5,6,14,3,1]
+ CRUSH rule 0 x 279 [6,10,13,3,9]
+ CRUSH rule 0 x 280 [7,3,14,9,1]
+ CRUSH rule 0 x 281 [5,11,14,7,9]
+ CRUSH rule 0 x 282 [2,1,13,14,9]
+ CRUSH rule 0 x 283 [4,1,12,3,10]
+ CRUSH rule 0 x 284 [5,11,7,15,3]
+ CRUSH rule 0 x 285 [15,5,3,1,6]
+ CRUSH rule 0 x 286 [10,4,3,6,12]
+ CRUSH rule 0 x 287 [12,4,9,1,3]
+ CRUSH rule 0 x 288 [4,12,10,7,1]
+ CRUSH rule 0 x 289 [2,5,14,9,13]
+ CRUSH rule 0 x 290 [12,2,5,6,15]
+ CRUSH rule 0 x 291 [7,11,1,14,5]
+ CRUSH rule 0 x 292 [4,10,6,3,14]
+ CRUSH rule 0 x 293 [6,5,11,1,2]
+ CRUSH rule 0 x 294 [9,12,3,14,6]
+ CRUSH rule 0 x 295 [6,10,3,14,9]
+ CRUSH rule 0 x 296 [3,1,13,7,14]
+ CRUSH rule 0 x 297 [6,13,4,14,10]
+ CRUSH rule 0 x 298 [14,9,13,1,4]
+ CRUSH rule 0 x 299 [14,12,11,6,4]
+ CRUSH rule 0 x 300 [15,7,10,5,1]
+ CRUSH rule 0 x 301 [9,11,7,1,13]
+ CRUSH rule 0 x 302 [9,7,1,13,5]
+ CRUSH rule 0 x 303 [4,13,3,7,10]
+ CRUSH rule 0 x 304 [6,9,2,11,15]
+ CRUSH rule 0 x 305 [13,7,5,11,2]
+ CRUSH rule 0 x 306 [10,12,4,6,9]
+ CRUSH rule 0 x 307 [11,12,15,5,6]
+ CRUSH rule 0 x 308 [12,14,10,9,1]
+ CRUSH rule 0 x 309 [9,3,12,5,11]
+ CRUSH rule 0 x 310 [3,1,5,10,14]
+ CRUSH rule 0 x 311 [3,9,7,1,14]
+ CRUSH rule 0 x 312 [15,13,9,7,5]
+ CRUSH rule 0 x 313 [9,15,3,7,5]
+ CRUSH rule 0 x 314 [2,15,9,5,6]
+ CRUSH rule 0 x 315 [15,2,13,1,11]
+ CRUSH rule 0 x 316 [4,9,11,2,12]
+ CRUSH rule 0 x 317 [1,5,3,13,15]
+ CRUSH rule 0 x 318 [4,1,15,11,9]
+ CRUSH rule 0 x 319 [2,15,4,1,11]
+ CRUSH rule 0 x 320 [5,7,13,9,11]
+ CRUSH rule 0 x 321 [1,6,11,15,5]
+ CRUSH rule 0 x 322 [13,7,5,3,14]
+ CRUSH rule 0 x 323 [7,4,10,1,2]
+ CRUSH rule 0 x 324 [5,6,10,15,2]
+ CRUSH rule 0 x 325 [9,10,14,5,1]
+ CRUSH rule 0 x 326 [11,7,13,4,2]
+ CRUSH rule 0 x 327 [12,5,10,14,3]
+ CRUSH rule 0 x 328 [5,2,6,14,1]
+ CRUSH rule 0 x 329 [2,6,15,5,9]
+ CRUSH rule 0 x 330 [3,9,11,13,1]
+ CRUSH rule 0 x 331 [12,14,6,3,1]
+ CRUSH rule 0 x 332 [10,12,6,15,9]
+ CRUSH rule 0 x 333 [6,5,3,12,14]
+ CRUSH rule 0 x 334 [4,9,2,12,7]
+ CRUSH rule 0 x 335 [11,7,1,5,13]
+ CRUSH rule 0 x 336 [6,14,13,2,5]
+ CRUSH rule 0 x 337 [15,11,3,7,12]
+ CRUSH rule 0 x 338 [10,5,3,6,15]
+ CRUSH rule 0 x 339 [11,14,13,5,3]
+ CRUSH rule 0 x 340 [11,6,12,4,9]
+ CRUSH rule 0 x 341 [7,5,2,10,14]
+ CRUSH rule 0 x 342 [12,14,1,9,2]
+ CRUSH rule 0 x 343 [12,14,9,6,10]
+ CRUSH rule 0 x 344 [9,11,5,2,14]
+ CRUSH rule 0 x 345 [14,2,11,9,6]
+ CRUSH rule 0 x 346 [5,3,14,10,7]
+ CRUSH rule 0 x 347 [10,2,12,6,9]
+ CRUSH rule 0 x 348 [7,9,10,1,14]
+ CRUSH rule 0 x 349 [9,6,10,12,1]
+ CRUSH rule 0 x 350 [13,9,15,4,10]
+ CRUSH rule 0 x 351 [13,5,15,3,1]
+ CRUSH rule 0 x 352 [1,12,11,9,4]
+ CRUSH rule 0 x 353 [10,14,12,2,9]
+ CRUSH rule 0 x 354 [6,3,15,10,9]
+ CRUSH rule 0 x 355 [13,14,6,10,2]
+ CRUSH rule 0 x 356 [15,13,2,9,6]
+ CRUSH rule 0 x 357 [4,11,1,13,3]
+ CRUSH rule 0 x 358 [12,7,2,9,1]
+ CRUSH rule 0 x 359 [5,15,7,11,3]
+ CRUSH rule 0 x 360 [13,10,1,2,6]
+ CRUSH rule 0 x 361 [5,3,13,6,1]
+ CRUSH rule 0 x 362 [2,9,11,13,1]
+ CRUSH rule 0 x 363 [7,12,3,9,15]
+ CRUSH rule 0 x 364 [2,12,6,9,5]
+ CRUSH rule 0 x 365 [13,5,11,15,6]
+ CRUSH rule 0 x 366 [12,7,3,14,5]
+ CRUSH rule 0 x 367 [7,13,3,1,5]
+ CRUSH rule 0 x 368 [7,9,10,15,3]
+ CRUSH rule 0 x 369 [7,5,3,13,14]
+ CRUSH rule 0 x 370 [4,7,14,1,2]
+ CRUSH rule 0 x 371 [1,7,12,3,4]
+ CRUSH rule 0 x 372 [10,4,3,14,6]
+ CRUSH rule 0 x 373 [15,5,2,6,13]
+ CRUSH rule 0 x 374 [3,15,12,5,1]
+ CRUSH rule 0 x 375 [5,2,14,1,6]
+ CRUSH rule 0 x 376 [5,14,10,13,3]
+ CRUSH rule 0 x 377 [1,15,2,4,9]
+ CRUSH rule 0 x 378 [9,12,2,15,1]
+ CRUSH rule 0 x 379 [11,2,15,5,7]
+ CRUSH rule 0 x 380 [6,1,12,11,2]
+ CRUSH rule 0 x 381 [15,13,7,5,10]
+ CRUSH rule 0 x 382 [14,3,1,4,13]
+ CRUSH rule 0 x 383 [3,6,11,4,13]
+ CRUSH rule 0 x 384 [4,13,6,3,15]
+ CRUSH rule 0 x 385 [4,6,15,3,10]
+ CRUSH rule 0 x 386 [14,3,11,13,5]
+ CRUSH rule 0 x 387 [1,11,5,7,9]
+ CRUSH rule 0 x 388 [2,6,11,9,15]
+ CRUSH rule 0 x 389 [12,7,2,4,15]
+ CRUSH rule 0 x 390 [2,11,13,7,5]
+ CRUSH rule 0 x 391 [3,4,9,13,7]
+ CRUSH rule 0 x 392 [11,5,14,7,1]
+ CRUSH rule 0 x 393 [2,14,5,9,7]
+ CRUSH rule 0 x 394 [4,9,3,15,13]
+ CRUSH rule 0 x 395 [10,13,5,15,6]
+ CRUSH rule 0 x 396 [2,12,15,9,4]
+ CRUSH rule 0 x 397 [1,14,9,4,12]
+ CRUSH rule 0 x 398 [9,2,1,5,12]
+ CRUSH rule 0 x 399 [5,9,14,3,1]
+ CRUSH rule 0 x 400 [10,6,2,4,15]
+ CRUSH rule 0 x 401 [6,9,11,12,4]
+ CRUSH rule 0 x 402 [4,7,9,2,13]
+ CRUSH rule 0 x 403 [7,15,13,3,5]
+ CRUSH rule 0 x 404 [14,12,7,9,2]
+ CRUSH rule 0 x 405 [9,15,11,2,4]
+ CRUSH rule 0 x 406 [12,14,9,2,7]
+ CRUSH rule 0 x 407 [9,5,12,10,15]
+ CRUSH rule 0 x 408 [7,1,5,2,10]
+ CRUSH rule 0 x 409 [11,2,4,13,1]
+ CRUSH rule 0 x 410 [6,4,14,2,12]
+ CRUSH rule 0 x 411 [13,11,15,6,4]
+ CRUSH rule 0 x 412 [5,9,6,11,14]
+ CRUSH rule 0 x 413 [13,5,3,11,6]
+ CRUSH rule 0 x 414 [3,11,9,13,4]
+ CRUSH rule 0 x 415 [6,10,14,5,1]
+ CRUSH rule 0 x 416 [13,1,4,7,2]
+ CRUSH rule 0 x 417 [4,12,1,15,2]
+ CRUSH rule 0 x 418 [14,5,10,2,6]
+ CRUSH rule 0 x 419 [5,14,10,9,2]
+ CRUSH rule 0 x 420 [2,4,9,11,6]
+ CRUSH rule 0 x 421 [15,4,10,3,9]
+ CRUSH rule 0 x 422 [4,11,2,7,13]
+ CRUSH rule 0 x 423 [3,15,12,6,5]
+ CRUSH rule 0 x 424 [6,10,12,2,5]
+ CRUSH rule 0 x 425 [11,15,2,13,5]
+ CRUSH rule 0 x 426 [12,4,7,1,9]
+ CRUSH rule 0 x 427 [14,10,3,1,9]
+ CRUSH rule 0 x 428 [12,7,9,4,2]
+ CRUSH rule 0 x 429 [3,4,9,7,11]
+ CRUSH rule 0 x 430 [3,5,10,13,1]
+ CRUSH rule 0 x 431 [9,3,7,1,12]
+ CRUSH rule 0 x 432 [4,1,12,7,15]
+ CRUSH rule 0 x 433 [4,11,12,15,7]
+ CRUSH rule 0 x 434 [2,14,9,1,5]
+ CRUSH rule 0 x 435 [13,11,5,6,9]
+ CRUSH rule 0 x 436 [9,15,10,2,4]
+ CRUSH rule 0 x 437 [9,6,3,14,10]
+ CRUSH rule 0 x 438 [7,2,13,4,11]
+ CRUSH rule 0 x 439 [7,14,4,3,12]
+ CRUSH rule 0 x 440 [14,11,9,2,7]
+ CRUSH rule 0 x 441 [2,4,11,9,13]
+ CRUSH rule 0 x 442 [10,13,9,7,15]
+ CRUSH rule 0 x 443 [12,15,10,9,2]
+ CRUSH rule 0 x 444 [4,13,7,14,3]
+ CRUSH rule 0 x 445 [4,2,15,7,1]
+ CRUSH rule 0 x 446 [12,10,6,9,4]
+ CRUSH rule 0 x 447 [15,7,13,1,4]
+ CRUSH rule 0 x 448 [5,2,13,7,15]
+ CRUSH rule 0 x 449 [14,5,3,12,10]
+ CRUSH rule 0 x 450 [2,4,6,9,15]
+ CRUSH rule 0 x 451 [6,14,11,3,9]
+ CRUSH rule 0 x 452 [14,9,10,4,2]
+ CRUSH rule 0 x 453 [5,15,13,2,6]
+ CRUSH rule 0 x 454 [10,4,2,6,15]
+ CRUSH rule 0 x 455 [6,13,2,4,10]
+ CRUSH rule 0 x 456 [5,7,13,1,11]
+ CRUSH rule 0 x 457 [9,1,5,7,11]
+ CRUSH rule 0 x 458 [9,11,15,4,7]
+ CRUSH rule 0 x 459 [13,15,11,1,5]
+ CRUSH rule 0 x 460 [5,12,10,15,7]
+ CRUSH rule 0 x 461 [4,3,9,13,15]
+ CRUSH rule 0 x 462 [4,7,12,14,11]
+ CRUSH rule 0 x 463 [4,12,14,11,2]
+ CRUSH rule 0 x 464 [4,2,15,10,1]
+ CRUSH rule 0 x 465 [5,10,9,7,13]
+ CRUSH rule 0 x 466 [13,5,2,15,9]
+ CRUSH rule 0 x 467 [13,6,14,3,9]
+ CRUSH rule 0 x 468 [10,7,12,14,4]
+ CRUSH rule 0 x 469 [4,9,6,14,12]
+ CRUSH rule 0 x 470 [3,9,12,15,5]
+ CRUSH rule 0 x 471 [6,1,5,14,13]
+ CRUSH rule 0 x 472 [2,14,7,5,13]
+ CRUSH rule 0 x 473 [15,10,6,9,4]
+ CRUSH rule 0 x 474 [15,10,4,12,6]
+ CRUSH rule 0 x 475 [10,5,12,9,14]
+ CRUSH rule 0 x 476 [3,6,10,12,1]
+ CRUSH rule 0 x 477 [6,13,5,15,11]
+ CRUSH rule 0 x 478 [4,15,1,3,7]
+ CRUSH rule 0 x 479 [13,11,1,6,14]
+ CRUSH rule 0 x 480 [1,13,6,4,9]
+ CRUSH rule 0 x 481 [15,12,7,9,1]
+ CRUSH rule 0 x 482 [2,12,9,1,7]
+ CRUSH rule 0 x 483 [10,1,4,15,9]
+ CRUSH rule 0 x 484 [1,4,10,13,7]
+ CRUSH rule 0 x 485 [9,4,3,1,14]
+ CRUSH rule 0 x 486 [3,10,15,9,7]
+ CRUSH rule 0 x 487 [12,11,4,14,7]
+ CRUSH rule 0 x 488 [14,4,1,9,2]
+ CRUSH rule 0 x 489 [11,4,2,13,15]
+ CRUSH rule 0 x 490 [4,9,1,3,13]
+ CRUSH rule 0 x 491 [1,12,5,2,14]
+ CRUSH rule 0 x 492 [5,7,11,3,14]
+ CRUSH rule 0 x 493 [12,1,4,15,3]
+ CRUSH rule 0 x 494 [1,7,13,4,15]
+ CRUSH rule 0 x 495 [3,15,7,1,9]
+ CRUSH rule 0 x 496 [5,3,7,13,9]
+ CRUSH rule 0 x 497 [13,10,3,6,5]
+ CRUSH rule 0 x 498 [10,6,1,5,9]
+ CRUSH rule 0 x 499 [14,3,12,5,1]
+ CRUSH rule 0 x 500 [15,9,6,12,11]
+ CRUSH rule 0 x 501 [10,13,1,9,3]
+ CRUSH rule 0 x 502 [5,1,14,11,7]
+ CRUSH rule 0 x 503 [15,10,7,9,1]
+ CRUSH rule 0 x 504 [13,2,7,1,14]
+ CRUSH rule 0 x 505 [12,7,5,2,14]
+ CRUSH rule 0 x 506 [11,7,9,14,12]
+ CRUSH rule 0 x 507 [4,14,13,3,9]
+ CRUSH rule 0 x 508 [12,1,4,9,2]
+ CRUSH rule 0 x 509 [4,2,6,9,14]
+ CRUSH rule 0 x 510 [5,3,1,12,11]
+ CRUSH rule 0 x 511 [2,12,10,6,14]
+ CRUSH rule 0 x 512 [15,11,3,5,7]
+ CRUSH rule 0 x 513 [4,9,11,3,13]
+ CRUSH rule 0 x 514 [11,9,3,4,12]
+ CRUSH rule 0 x 515 [12,14,6,5,3]
+ CRUSH rule 0 x 516 [14,11,1,12,3]
+ CRUSH rule 0 x 517 [11,5,6,13,9]
+ CRUSH rule 0 x 518 [3,5,7,12,15]
+ CRUSH rule 0 x 519 [12,14,2,1,4]
+ CRUSH rule 0 x 520 [12,4,2,10,6]
+ CRUSH rule 0 x 521 [11,5,9,6,15]
+ CRUSH rule 0 x 522 [4,12,11,1,15]
+ CRUSH rule 0 x 523 [3,1,5,9,15]
+ CRUSH rule 0 x 524 [15,9,3,11,13]
+ CRUSH rule 0 x 525 [3,15,11,6,9]
+ CRUSH rule 0 x 526 [10,2,5,13,6]
+ CRUSH rule 0 x 527 [3,13,4,1,9]
+ CRUSH rule 0 x 528 [12,7,15,10,2]
+ CRUSH rule 0 x 529 [6,4,10,12,2]
+ CRUSH rule 0 x 530 [11,9,12,7,5]
+ CRUSH rule 0 x 531 [9,15,4,7,2]
+ CRUSH rule 0 x 532 [5,3,13,7,9]
+ CRUSH rule 0 x 533 [12,15,1,2,7]
+ CRUSH rule 0 x 534 [11,9,3,7,15]
+ CRUSH rule 0 x 535 [11,1,3,5,14]
+ CRUSH rule 0 x 536 [9,1,14,13,4]
+ CRUSH rule 0 x 537 [15,5,13,2,7]
+ CRUSH rule 0 x 538 [13,5,11,2,6]
+ CRUSH rule 0 x 539 [10,12,6,14,1]
+ CRUSH rule 0 x 540 [12,15,7,3,9]
+ CRUSH rule 0 x 541 [2,1,6,11,14]
+ CRUSH rule 0 x 542 [3,9,15,5,11]
+ CRUSH rule 0 x 543 [4,10,9,3,6]
+ CRUSH rule 0 x 544 [3,15,9,11,7]
+ CRUSH rule 0 x 545 [14,10,7,12,4]
+ CRUSH rule 0 x 546 [5,15,13,7,1]
+ CRUSH rule 0 x 547 [5,13,7,9,3]
+ CRUSH rule 0 x 548 [11,7,12,15,4]
+ CRUSH rule 0 x 549 [14,1,4,9,13]
+ CRUSH rule 0 x 550 [9,15,3,13,1]
+ CRUSH rule 0 x 551 [11,2,15,6,13]
+ CRUSH rule 0 x 552 [2,11,14,1,9]
+ CRUSH rule 0 x 553 [11,9,14,6,4]
+ CRUSH rule 0 x 554 [11,14,6,4,13]
+ CRUSH rule 0 x 555 [6,5,10,9,14]
+ CRUSH rule 0 x 556 [15,6,3,13,11]
+ CRUSH rule 0 x 557 [12,2,5,14,10]
+ CRUSH rule 0 x 558 [12,1,6,15,5]
+ CRUSH rule 0 x 559 [2,13,5,10,14]
+ CRUSH rule 0 x 560 [4,9,12,6,3]
+ CRUSH rule 0 x 561 [12,7,1,2,5]
+ CRUSH rule 0 x 562 [7,13,9,14,2]
+ CRUSH rule 0 x 563 [15,4,3,10,13]
+ CRUSH rule 0 x 564 [2,13,7,1,15]
+ CRUSH rule 0 x 565 [3,12,4,1,14]
+ CRUSH rule 0 x 566 [6,14,4,2,13]
+ CRUSH rule 0 x 567 [15,4,11,6,3]
+ CRUSH rule 0 x 568 [4,14,1,6,10]
+ CRUSH rule 0 x 569 [11,3,15,13,5]
+ CRUSH rule 0 x 570 [1,10,13,4,7]
+ CRUSH rule 0 x 571 [10,12,14,9,4]
+ CRUSH rule 0 x 572 [12,14,3,10,6]
+ CRUSH rule 0 x 573 [7,15,11,2,12]
+ CRUSH rule 0 x 574 [11,14,13,1,3]
+ CRUSH rule 0 x 575 [5,13,15,9,6]
+ CRUSH rule 0 x 576 [3,15,11,9,1]
+ CRUSH rule 0 x 577 [13,9,6,15,3]
+ CRUSH rule 0 x 578 [4,10,1,2,7]
+ CRUSH rule 0 x 579 [13,1,15,2,10]
+ CRUSH rule 0 x 580 [3,12,4,1,10]
+ CRUSH rule 0 x 581 [7,14,12,10,1]
+ CRUSH rule 0 x 582 [10,5,13,14,1]
+ CRUSH rule 0 x 583 [4,15,1,9,10]
+ CRUSH rule 0 x 584 [10,1,5,13,6]
+ CRUSH rule 0 x 585 [5,3,6,1,11]
+ CRUSH rule 0 x 586 [7,10,14,12,9]
+ CRUSH rule 0 x 587 [11,6,9,4,1]
+ CRUSH rule 0 x 588 [3,12,7,15,4]
+ CRUSH rule 0 x 589 [9,7,12,1,10]
+ CRUSH rule 0 x 590 [12,1,3,9,10]
+ CRUSH rule 0 x 591 [2,6,14,13,9]
+ CRUSH rule 0 x 592 [15,12,9,7,5]
+ CRUSH rule 0 x 593 [13,14,5,11,9]
+ CRUSH rule 0 x 594 [12,14,2,9,7]
+ CRUSH rule 0 x 595 [12,7,10,3,1]
+ CRUSH rule 0 x 596 [2,7,12,11,1]
+ CRUSH rule 0 x 597 [15,1,2,10,7]
+ CRUSH rule 0 x 598 [11,5,9,14,12]
+ CRUSH rule 0 x 599 [13,11,1,5,6]
+ CRUSH rule 0 x 600 [4,12,3,10,9]
+ CRUSH rule 0 x 601 [13,5,15,2,1]
+ CRUSH rule 0 x 602 [3,11,7,1,13]
+ CRUSH rule 0 x 603 [3,1,4,14,10]
+ CRUSH rule 0 x 604 [14,2,6,1,11]
+ CRUSH rule 0 x 605 [2,7,12,5,14]
+ CRUSH rule 0 x 606 [12,15,1,5,7]
+ CRUSH rule 0 x 607 [3,9,10,14,7]
+ CRUSH rule 0 x 608 [13,10,1,7,9]
+ CRUSH rule 0 x 609 [14,3,7,9,11]
+ CRUSH rule 0 x 610 [7,10,5,1,12]
+ CRUSH rule 0 x 611 [13,1,5,3,10]
+ CRUSH rule 0 x 612 [7,1,2,13,9]
+ CRUSH rule 0 x 613 [10,7,14,9,5]
+ CRUSH rule 0 x 614 [9,4,15,3,1]
+ CRUSH rule 0 x 615 [9,4,11,2,1]
+ CRUSH rule 0 x 616 [10,14,1,5,3]
+ CRUSH rule 0 x 617 [15,7,2,11,12]
+ CRUSH rule 0 x 618 [4,2,10,6,14]
+ CRUSH rule 0 x 619 [15,4,3,9,6]
+ CRUSH rule 0 x 620 [3,7,11,14,13]
+ CRUSH rule 0 x 621 [3,6,4,14,1]
+ CRUSH rule 0 x 622 [10,2,13,5,15]
+ CRUSH rule 0 x 623 [4,9,14,7,3]
+ CRUSH rule 0 x 624 [3,9,15,6,10]
+ CRUSH rule 0 x 625 [11,7,3,5,13]
+ CRUSH rule 0 x 626 [10,12,2,1,9]
+ CRUSH rule 0 x 627 [1,12,10,14,3]
+ CRUSH rule 0 x 628 [15,13,11,4,2]
+ CRUSH rule 0 x 629 [5,6,15,12,1]
+ CRUSH rule 0 x 630 [1,4,12,9,3]
+ CRUSH rule 0 x 631 [5,7,1,15,12]
+ CRUSH rule 0 x 632 [12,3,11,9,6]
+ CRUSH rule 0 x 633 [14,4,3,7,10]
+ CRUSH rule 0 x 634 [6,9,5,3,13]
+ CRUSH rule 0 x 635 [6,5,2,15,9]
+ CRUSH rule 0 x 636 [13,6,11,3,15]
+ CRUSH rule 0 x 637 [3,1,10,6,9]
+ CRUSH rule 0 x 638 [10,15,3,5,13]
+ CRUSH rule 0 x 639 [6,9,14,4,3]
+ CRUSH rule 0 x 640 [9,6,1,11,14]
+ CRUSH rule 0 x 641 [10,6,5,14,1]
+ CRUSH rule 0 x 642 [1,15,4,6,2]
+ CRUSH rule 0 x 643 [3,7,5,1,10]
+ CRUSH rule 0 x 644 [15,13,6,9,3]
+ CRUSH rule 0 x 645 [14,2,4,9,10]
+ CRUSH rule 0 x 646 [5,13,14,1,6]
+ CRUSH rule 0 x 647 [10,1,9,13,6]
+ CRUSH rule 0 x 648 [6,5,2,14,11]
+ CRUSH rule 0 x 649 [3,9,13,11,4]
+ CRUSH rule 0 x 650 [10,9,4,15,12]
+ CRUSH rule 0 x 651 [3,9,5,7,14]
+ CRUSH rule 0 x 652 [15,9,4,6,13]
+ CRUSH rule 0 x 653 [11,14,1,3,6]
+ CRUSH rule 0 x 654 [13,6,2,10,15]
+ CRUSH rule 0 x 655 [6,3,4,15,12]
+ CRUSH rule 0 x 656 [3,15,1,4,6]
+ CRUSH rule 0 x 657 [11,15,3,5,7]
+ CRUSH rule 0 x 658 [7,2,10,12,1]
+ CRUSH rule 0 x 659 [2,5,14,6,10]
+ CRUSH rule 0 x 660 [13,14,10,6,4]
+ CRUSH rule 0 x 661 [7,15,3,12,11]
+ CRUSH rule 0 x 662 [15,2,12,5,1]
+ CRUSH rule 0 x 663 [14,9,13,10,5]
+ CRUSH rule 0 x 664 [6,10,12,4,9]
+ CRUSH rule 0 x 665 [2,9,12,1,7]
+ CRUSH rule 0 x 666 [12,3,6,1,15]
+ CRUSH rule 0 x 667 [1,9,12,10,2]
+ CRUSH rule 0 x 668 [9,5,1,2,6]
+ CRUSH rule 0 x 669 [9,7,14,5,11]
+ CRUSH rule 0 x 670 [6,10,9,13,1]
+ CRUSH rule 0 x 671 [6,15,5,10,13]
+ CRUSH rule 0 x 672 [2,9,13,1,4]
+ CRUSH rule 0 x 673 [7,10,5,9,15]
+ CRUSH rule 0 x 674 [7,12,10,1,14]
+ CRUSH rule 0 x 675 [9,5,1,10,6]
+ CRUSH rule 0 x 676 [10,12,2,1,4]
+ CRUSH rule 0 x 677 [2,12,1,4,10]
+ CRUSH rule 0 x 678 [1,2,4,10,12]
+ CRUSH rule 0 x 679 [5,6,12,15,9]
+ CRUSH rule 0 x 680 [7,11,3,1,15]
+ CRUSH rule 0 x 681 [6,4,3,11,14]
+ CRUSH rule 0 x 682 [6,1,11,15,12]
+ CRUSH rule 0 x 683 [6,13,2,4,9]
+ CRUSH rule 0 x 684 [9,11,3,7,15]
+ CRUSH rule 0 x 685 [5,1,15,7,9]
+ CRUSH rule 0 x 686 [1,9,11,14,6]
+ CRUSH rule 0 x 687 [7,13,3,5,11]
+ CRUSH rule 0 x 688 [11,9,1,14,3]
+ CRUSH rule 0 x 689 [5,2,9,12,1]
+ CRUSH rule 0 x 690 [9,7,10,3,13]
+ CRUSH rule 0 x 691 [11,15,9,5,7]
+ CRUSH rule 0 x 692 [15,5,1,2,9]
+ CRUSH rule 0 x 693 [5,6,12,15,2]
+ CRUSH rule 0 x 694 [4,7,1,10,12]
+ CRUSH rule 0 x 695 [6,13,14,10,9]
+ CRUSH rule 0 x 696 [1,2,4,14,7]
+ CRUSH rule 0 x 697 [13,11,3,6,4]
+ CRUSH rule 0 x 698 [11,13,4,2,6]
+ CRUSH rule 0 x 699 [7,14,12,4,2]
+ CRUSH rule 0 x 700 [12,14,11,9,4]
+ CRUSH rule 0 x 701 [3,13,1,14,4]
+ CRUSH rule 0 x 702 [3,12,15,6,5]
+ CRUSH rule 0 x 703 [15,11,13,3,4]
+ CRUSH rule 0 x 704 [6,4,2,15,11]
+ CRUSH rule 0 x 705 [14,6,11,5,1]
+ CRUSH rule 0 x 706 [1,12,3,6,4]
+ CRUSH rule 0 x 707 [4,7,14,3,10]
+ CRUSH rule 0 x 708 [3,10,5,1,15]
+ CRUSH rule 0 x 709 [11,12,3,7,5]
+ CRUSH rule 0 x 710 [14,2,11,9,5]
+ CRUSH rule 0 x 711 [14,3,9,10,12]
+ CRUSH rule 0 x 712 [12,3,11,15,9]
+ CRUSH rule 0 x 713 [11,9,3,15,13]
+ CRUSH rule 0 x 714 [12,1,9,7,2]
+ CRUSH rule 0 x 715 [6,1,14,4,11]
+ CRUSH rule 0 x 716 [11,13,9,14,5]
+ CRUSH rule 0 x 717 [12,4,10,9,15]
+ CRUSH rule 0 x 718 [7,15,5,2,11]
+ CRUSH rule 0 x 719 [5,15,13,3,1]
+ CRUSH rule 0 x 720 [4,13,10,2,7]
+ CRUSH rule 0 x 721 [11,3,14,9,1]
+ CRUSH rule 0 x 722 [2,4,6,1,9]
+ CRUSH rule 0 x 723 [2,1,12,15,11]
+ CRUSH rule 0 x 724 [7,1,9,10,5]
+ CRUSH rule 0 x 725 [11,12,7,15,4]
+ CRUSH rule 0 x 726 [7,14,4,3,11]
+ CRUSH rule 0 x 727 [2,5,1,11,15]
+ CRUSH rule 0 x 728 [13,11,4,6,15]
+ CRUSH rule 0 x 729 [15,11,4,6,2]
+ CRUSH rule 0 x 730 [3,7,1,13,11]
+ CRUSH rule 0 x 731 [9,1,6,5,2]
+ CRUSH rule 0 x 732 [1,2,10,13,9]
+ CRUSH rule 0 x 733 [11,3,5,6,1]
+ CRUSH rule 0 x 734 [14,3,11,7,12]
+ CRUSH rule 0 x 735 [6,9,2,10,13]
+ CRUSH rule 0 x 736 [3,9,1,11,7]
+ CRUSH rule 0 x 737 [1,4,2,12,9]
+ CRUSH rule 0 x 738 [11,15,7,4,9]
+ CRUSH rule 0 x 739 [11,12,6,2,4]
+ CRUSH rule 0 x 740 [7,9,10,13,1]
+ CRUSH rule 0 x 741 [12,11,7,15,2]
+ CRUSH rule 0 x 742 [9,7,4,11,12]
+ CRUSH rule 0 x 743 [5,13,9,15,10]
+ CRUSH rule 0 x 744 [6,2,13,1,14]
+ CRUSH rule 0 x 745 [3,6,1,4,11]
+ CRUSH rule 0 x 746 [3,7,9,10,14]
+ CRUSH rule 0 x 747 [15,11,5,2,13]
+ CRUSH rule 0 x 748 [6,10,13,2,14]
+ CRUSH rule 0 x 749 [14,9,10,7,5]
+ CRUSH rule 0 x 750 [1,14,6,5,11]
+ CRUSH rule 0 x 751 [15,1,6,9,5]
+ CRUSH rule 0 x 752 [13,1,7,3,11]
+ CRUSH rule 0 x 753 [4,11,1,3,15]
+ CRUSH rule 0 x 754 [14,12,11,4,2]
+ CRUSH rule 0 x 755 [13,6,1,10,4]
+ CRUSH rule 0 x 756 [3,4,14,6,1]
+ CRUSH rule 0 x 757 [10,6,1,4,13]
+ CRUSH rule 0 x 758 [6,3,4,10,15]
+ CRUSH rule 0 x 759 [5,7,3,14,11]
+ CRUSH rule 0 x 760 [1,15,10,12,4]
+ CRUSH rule 0 x 761 [2,12,1,14,5]
+ CRUSH rule 0 x 762 [1,4,10,9,3]
+ CRUSH rule 0 x 763 [4,13,1,14,7]
+ CRUSH rule 0 x 764 [1,14,6,13,9]
+ CRUSH rule 0 x 765 [9,15,2,13,4]
+ CRUSH rule 0 x 766 [11,2,7,15,9]
+ CRUSH rule 0 x 767 [6,11,4,3,12]
+ CRUSH rule 0 x 768 [2,12,15,7,1]
+ CRUSH rule 0 x 769 [15,1,9,2,11]
+ CRUSH rule 0 x 770 [15,13,4,6,3]
+ CRUSH rule 0 x 771 [9,2,12,11,6]
+ CRUSH rule 0 x 772 [4,3,13,11,14]
+ CRUSH rule 0 x 773 [3,7,4,15,1]
+ CRUSH rule 0 x 774 [12,6,3,15,5]
+ CRUSH rule 0 x 775 [5,10,14,2,6]
+ CRUSH rule 0 x 776 [10,15,3,9,6]
+ CRUSH rule 0 x 777 [11,13,4,7,1]
+ CRUSH rule 0 x 778 [13,1,9,11,15]
+ CRUSH rule 0 x 779 [5,11,1,14,2]
+ CRUSH rule 0 x 780 [13,9,3,6,4]
+ CRUSH rule 0 x 781 [5,7,14,3,1]
+ CRUSH rule 0 x 782 [2,15,9,7,11]
+ CRUSH rule 0 x 783 [12,7,5,14,9]
+ CRUSH rule 0 x 784 [14,1,10,13,3]
+ CRUSH rule 0 x 785 [6,12,1,2,4]
+ CRUSH rule 0 x 786 [10,5,2,15,1]
+ CRUSH rule 0 x 787 [1,12,10,2,9]
+ CRUSH rule 0 x 788 [4,2,9,13,6]
+ CRUSH rule 0 x 789 [9,2,14,7,4]
+ CRUSH rule 0 x 790 [15,2,7,4,1]
+ CRUSH rule 0 x 791 [9,4,7,13,14]
+ CRUSH rule 0 x 792 [6,4,15,10,12]
+ CRUSH rule 0 x 793 [15,9,6,2,13]
+ CRUSH rule 0 x 794 [5,12,2,14,9]
+ CRUSH rule 0 x 795 [6,14,12,4,10]
+ CRUSH rule 0 x 796 [11,2,12,6,15]
+ CRUSH rule 0 x 797 [14,3,7,1,5]
+ CRUSH rule 0 x 798 [5,11,6,13,1]
+ CRUSH rule 0 x 799 [2,9,14,4,13]
+ CRUSH rule 0 x 800 [6,3,4,11,15]
+ CRUSH rule 0 x 801 [2,5,6,13,9]
+ CRUSH rule 0 x 802 [1,4,12,7,3]
+ CRUSH rule 0 x 803 [7,2,4,1,11]
+ CRUSH rule 0 x 804 [5,14,9,7,3]
+ CRUSH rule 0 x 805 [13,4,3,1,10]
+ CRUSH rule 0 x 806 [6,2,13,4,15]
+ CRUSH rule 0 x 807 [14,2,7,4,9]
+ CRUSH rule 0 x 808 [2,15,12,7,9]
+ CRUSH rule 0 x 809 [1,11,7,12,4]
+ CRUSH rule 0 x 810 [2,5,9,12,15]
+ CRUSH rule 0 x 811 [15,6,3,10,1]
+ CRUSH rule 0 x 812 [7,11,2,14,9]
+ CRUSH rule 0 x 813 [4,10,13,14,2]
+ CRUSH rule 0 x 814 [13,4,9,3,10]
+ CRUSH rule 0 x 815 [15,12,9,4,10]
+ CRUSH rule 0 x 816 [14,10,13,7,3]
+ CRUSH rule 0 x 817 [10,7,2,15,13]
+ CRUSH rule 0 x 818 [15,2,11,4,1]
+ CRUSH rule 0 x 819 [5,12,10,6,1]
+ CRUSH rule 0 x 820 [3,6,9,12,11]
+ CRUSH rule 0 x 821 [15,10,9,13,3]
+ CRUSH rule 0 x 822 [10,13,2,9,7]
+ CRUSH rule 0 x 823 [2,6,12,10,15]
+ CRUSH rule 0 x 824 [3,7,9,13,15]
+ CRUSH rule 0 x 825 [10,5,14,6,12]
+ CRUSH rule 0 x 826 [5,2,11,15,1]
+ CRUSH rule 0 x 827 [13,5,1,3,7]
+ CRUSH rule 0 x 828 [12,6,10,5,1]
+ CRUSH rule 0 x 829 [13,6,15,10,5]
+ CRUSH rule 0 x 830 [15,13,2,9,7]
+ CRUSH rule 0 x 831 [1,4,11,12,6]
+ CRUSH rule 0 x 832 [14,11,13,2,9]
+ CRUSH rule 0 x 833 [9,13,3,11,7]
+ CRUSH rule 0 x 834 [9,7,5,1,11]
+ CRUSH rule 0 x 835 [14,3,13,6,4]
+ CRUSH rule 0 x 836 [3,9,10,13,1]
+ CRUSH rule 0 x 837 [15,12,11,2,7]
+ CRUSH rule 0 x 838 [12,14,9,2,5]
+ CRUSH rule 0 x 839 [3,4,6,10,15]
+ CRUSH rule 0 x 840 [10,15,12,4,7]
+ CRUSH rule 0 x 841 [3,5,7,12,11]
+ CRUSH rule 0 x 842 [9,13,2,6,5]
+ CRUSH rule 0 x 843 [14,7,4,9,3]
+ CRUSH rule 0 x 844 [7,1,4,15,9]
+ CRUSH rule 0 x 845 [13,6,1,15,4]
+ CRUSH rule 0 x 846 [3,7,15,13,1]
+ CRUSH rule 0 x 847 [12,15,11,5,2]
+ CRUSH rule 0 x 848 [11,13,1,14,5]
+ CRUSH rule 0 x 849 [3,15,11,9,6]
+ CRUSH rule 0 x 850 [1,3,10,6,14]
+ CRUSH rule 0 x 851 [14,4,3,6,11]
+ CRUSH rule 0 x 852 [9,12,4,7,15]
+ CRUSH rule 0 x 853 [13,14,6,11,2]
+ CRUSH rule 0 x 854 [7,11,12,1,4]
+ CRUSH rule 0 x 855 [14,4,12,6,3]
+ CRUSH rule 0 x 856 [5,10,7,3,15]
+ CRUSH rule 0 x 857 [4,3,13,11,9]
+ CRUSH rule 0 x 858 [5,15,6,3,9]
+ CRUSH rule 0 x 859 [5,15,6,2,1]
+ CRUSH rule 0 x 860 [11,14,1,12,6]
+ CRUSH rule 0 x 861 [13,7,4,10,1]
+ CRUSH rule 0 x 862 [5,10,9,7,3]
+ CRUSH rule 0 x 863 [11,6,3,9,4]
+ CRUSH rule 0 x 864 [6,13,4,2,10]
+ CRUSH rule 0 x 865 [4,1,14,11,6]
+ CRUSH rule 0 x 866 [2,13,4,15,9]
+ CRUSH rule 0 x 867 [12,2,9,10,4]
+ CRUSH rule 0 x 868 [14,11,7,2,1]
+ CRUSH rule 0 x 869 [10,13,7,14,3]
+ CRUSH rule 0 x 870 [14,9,11,4,3]
+ CRUSH rule 0 x 871 [6,2,1,4,15]
+ CRUSH rule 0 x 872 [6,1,15,3,10]
+ CRUSH rule 0 x 873 [2,5,12,10,1]
+ CRUSH rule 0 x 874 [12,4,7,2,15]
+ CRUSH rule 0 x 875 [10,6,14,1,12]
+ CRUSH rule 0 x 876 [14,7,13,3,9]
+ CRUSH rule 0 x 877 [15,11,13,9,5]
+ CRUSH rule 0 x 878 [7,14,3,13,9]
+ CRUSH rule 0 x 879 [12,2,7,4,10]
+ CRUSH rule 0 x 880 [2,12,10,7,1]
+ CRUSH rule 0 x 881 [6,3,1,11,4]
+ CRUSH rule 0 x 882 [11,13,7,1,2]
+ CRUSH rule 0 x 883 [13,1,3,10,6]
+ CRUSH rule 0 x 884 [6,15,4,9,3]
+ CRUSH rule 0 x 885 [14,7,9,4,2]
+ CRUSH rule 0 x 886 [13,11,4,2,1]
+ CRUSH rule 0 x 887 [14,4,12,11,2]
+ CRUSH rule 0 x 888 [10,12,7,15,9]
+ CRUSH rule 0 x 889 [15,13,4,1,6]
+ CRUSH rule 0 x 890 [10,12,14,2,9]
+ CRUSH rule 0 x 891 [9,5,11,6,3]
+ CRUSH rule 0 x 892 [12,15,2,4,7]
+ CRUSH rule 0 x 893 [1,3,5,9,6]
+ CRUSH rule 0 x 894 [7,2,11,13,4]
+ CRUSH rule 0 x 895 [2,1,11,5,7]
+ CRUSH rule 0 x 896 [9,1,14,10,4]
+ CRUSH rule 0 x 897 [7,5,14,3,1]
+ CRUSH rule 0 x 898 [10,6,12,9,15]
+ CRUSH rule 0 x 899 [1,11,5,3,13]
+ CRUSH rule 0 x 900 [2,9,10,7,13]
+ CRUSH rule 0 x 901 [9,12,11,3,14]
+ CRUSH rule 0 x 902 [4,2,6,15,12]
+ CRUSH rule 0 x 903 [14,10,3,1,12]
+ CRUSH rule 0 x 904 [15,12,4,9,6]
+ CRUSH rule 0 x 905 [12,6,11,3,9]
+ CRUSH rule 0 x 906 [14,11,12,2,4]
+ CRUSH rule 0 x 907 [7,12,3,9,10]
+ CRUSH rule 0 x 908 [2,15,9,6,10]
+ CRUSH rule 0 x 909 [10,14,1,13,2]
+ CRUSH rule 0 x 910 [12,7,4,15,10]
+ CRUSH rule 0 x 911 [11,15,2,4,9]
+ CRUSH rule 0 x 912 [6,4,14,13,3]
+ CRUSH rule 0 x 913 [4,6,10,1,12]
+ CRUSH rule 0 x 914 [4,15,2,10,1]
+ CRUSH rule 0 x 915 [12,14,1,9,4]
+ CRUSH rule 0 x 916 [3,1,11,5,6]
+ CRUSH rule 0 x 917 [1,15,6,5,10]
+ CRUSH rule 0 x 918 [7,14,11,4,9]
+ CRUSH rule 0 x 919 [10,7,3,13,15]
+ CRUSH rule 0 x 920 [4,2,10,15,1]
+ CRUSH rule 0 x 921 [1,11,6,13,4]
+ CRUSH rule 0 x 922 [6,4,14,13,3]
+ CRUSH rule 0 x 923 [12,2,5,14,10]
+ CRUSH rule 0 x 924 [6,2,14,13,9]
+ CRUSH rule 0 x 925 [12,15,2,10,1]
+ CRUSH rule 0 x 926 [3,13,10,1,14]
+ CRUSH rule 0 x 927 [6,5,1,11,14]
+ CRUSH rule 0 x 928 [13,1,3,9,6]
+ CRUSH rule 0 x 929 [10,7,1,5,2]
+ CRUSH rule 0 x 930 [7,15,10,5,1]
+ CRUSH rule 0 x 931 [6,15,11,9,5]
+ CRUSH rule 0 x 932 [13,2,5,11,9]
+ CRUSH rule 0 x 933 [12,7,14,10,4]
+ CRUSH rule 0 x 934 [12,2,5,7,9]
+ CRUSH rule 0 x 935 [6,11,1,14,5]
+ CRUSH rule 0 x 936 [9,12,7,5,1]
+ CRUSH rule 0 x 937 [14,2,11,1,13]
+ CRUSH rule 0 x 938 [14,3,5,11,7]
+ CRUSH rule 0 x 939 [6,4,14,9,12]
+ CRUSH rule 0 x 940 [13,11,4,2,1]
+ CRUSH rule 0 x 941 [3,12,4,7,14]
+ CRUSH rule 0 x 942 [15,12,10,4,1]
+ CRUSH rule 0 x 943 [10,2,4,9,6]
+ CRUSH rule 0 x 944 [2,9,4,7,1]
+ CRUSH rule 0 x 945 [10,15,2,9,5]
+ CRUSH rule 0 x 946 [11,15,7,12,5]
+ CRUSH rule 0 x 947 [11,3,14,1,12]
+ CRUSH rule 0 x 948 [7,13,11,5,14]
+ CRUSH rule 0 x 949 [9,1,12,5,15]
+ CRUSH rule 0 x 950 [9,15,13,6,4]
+ CRUSH rule 0 x 951 [2,6,12,9,10]
+ CRUSH rule 0 x 952 [9,7,15,3,5]
+ CRUSH rule 0 x 953 [1,3,6,10,12]
+ CRUSH rule 0 x 954 [10,2,14,9,4]
+ CRUSH rule 0 x 955 [7,14,3,1,10]
+ CRUSH rule 0 x 956 [1,6,11,5,14]
+ CRUSH rule 0 x 957 [14,11,1,12,6]
+ CRUSH rule 0 x 958 [15,4,3,11,1]
+ CRUSH rule 0 x 959 [2,1,12,15,10]
+ CRUSH rule 0 x 960 [2,6,11,13,15]
+ CRUSH rule 0 x 961 [3,13,11,9,6]
+ CRUSH rule 0 x 962 [5,11,3,14,1]
+ CRUSH rule 0 x 963 [13,10,15,4,6]
+ CRUSH rule 0 x 964 [7,11,4,9,2]
+ CRUSH rule 0 x 965 [12,2,9,7,4]
+ CRUSH rule 0 x 966 [12,14,9,4,1]
+ CRUSH rule 0 x 967 [7,5,3,10,12]
+ CRUSH rule 0 x 968 [12,15,4,9,11]
+ CRUSH rule 0 x 969 [11,4,7,1,9]
+ CRUSH rule 0 x 970 [5,12,10,1,3]
+ CRUSH rule 0 x 971 [1,9,4,12,7]
+ CRUSH rule 0 x 972 [12,3,14,5,1]
+ CRUSH rule 0 x 973 [1,10,4,12,2]
+ CRUSH rule 0 x 974 [7,11,1,2,15]
+ CRUSH rule 0 x 975 [7,9,15,12,2]
+ CRUSH rule 0 x 976 [7,3,15,5,12]
+ CRUSH rule 0 x 977 [14,3,6,10,4]
+ CRUSH rule 0 x 978 [12,5,11,1,15]
+ CRUSH rule 0 x 979 [5,1,13,6,15]
+ CRUSH rule 0 x 980 [15,11,5,6,1]
+ CRUSH rule 0 x 981 [5,11,15,12,7]
+ CRUSH rule 0 x 982 [2,6,14,11,12]
+ CRUSH rule 0 x 983 [3,12,10,9,14]
+ CRUSH rule 0 x 984 [15,13,1,10,2]
+ CRUSH rule 0 x 985 [11,2,15,1,4]
+ CRUSH rule 0 x 986 [6,13,9,1,15]
+ CRUSH rule 0 x 987 [13,14,5,10,6]
+ CRUSH rule 0 x 988 [12,9,10,14,3]
+ CRUSH rule 0 x 989 [7,4,3,15,9]
+ CRUSH rule 0 x 990 [1,10,9,13,3]
+ CRUSH rule 0 x 991 [7,11,1,14,2]
+ CRUSH rule 0 x 992 [9,10,2,13,7]
+ CRUSH rule 0 x 993 [6,10,14,12,4]
+ CRUSH rule 0 x 994 [3,13,15,4,11]
+ CRUSH rule 0 x 995 [15,6,12,2,5]
+ CRUSH rule 0 x 996 [15,10,5,3,13]
+ CRUSH rule 0 x 997 [15,2,1,12,7]
+ CRUSH rule 0 x 998 [6,1,9,5,12]
+ CRUSH rule 0 x 999 [9,10,15,5,13]
+ CRUSH rule 0 x 1000 [14,2,9,4,12]
+ CRUSH rule 0 x 1001 [11,14,4,2,6]
+ CRUSH rule 0 x 1002 [1,10,14,2,9]
+ CRUSH rule 0 x 1003 [10,7,5,14,2]
+ CRUSH rule 0 x 1004 [15,1,4,6,10]
+ CRUSH rule 0 x 1005 [6,12,2,10,9]
+ CRUSH rule 0 x 1006 [10,12,15,1,2]
+ CRUSH rule 0 x 1007 [1,7,13,14,3]
+ CRUSH rule 0 x 1008 [7,4,9,11,3]
+ CRUSH rule 0 x 1009 [5,2,11,7,15]
+ CRUSH rule 0 x 1010 [10,2,15,6,9]
+ CRUSH rule 0 x 1011 [6,3,12,1,10]
+ CRUSH rule 0 x 1012 [12,6,9,15,3]
+ CRUSH rule 0 x 1013 [2,14,12,4,9]
+ CRUSH rule 0 x 1014 [1,13,7,2,10]
+ CRUSH rule 0 x 1015 [12,6,10,1,4]
+ CRUSH rule 0 x 1016 [10,13,14,3,5]
+ CRUSH rule 0 x 1017 [5,11,14,7,13]
+ CRUSH rule 0 x 1018 [13,11,14,1,9]
+ CRUSH rule 0 x 1019 [10,13,14,7,5]
+ CRUSH rule 0 x 1020 [3,1,13,4,10]
+ CRUSH rule 0 x 1021 [2,11,14,9,4]
+ CRUSH rule 0 x 1022 [15,5,7,2,12]
+ CRUSH rule 0 x 1023 [15,2,9,12,1]
+ rule 0 (replicated_ruleset) num_rep 5 result size == 5:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15,12,1]
+ CRUSH rule 0 x 1 [10,15,1,2,13,4]
+ CRUSH rule 0 x 2 [1,12,2,6,5,10]
+ CRUSH rule 0 x 3 [15,4,10,2,9,6]
+ CRUSH rule 0 x 4 [14,2,10,1,9,4]
+ CRUSH rule 0 x 5 [7,4,11,2,13,15]
+ CRUSH rule 0 x 6 [12,6,10,9,3,4]
+ CRUSH rule 0 x 7 [9,2,6,12,11,4]
+ CRUSH rule 0 x 8 [10,2,15,1,4,13]
+ CRUSH rule 0 x 9 [7,1,14,2,11,9]
+ CRUSH rule 0 x 10 [10,14,4,1,2,7]
+ CRUSH rule 0 x 11 [13,9,14,7,5,11]
+ CRUSH rule 0 x 12 [7,1,2,5,13,15]
+ CRUSH rule 0 x 13 [3,5,12,7,9,1]
+ CRUSH rule 0 x 14 [13,5,2,7,10,15]
+ CRUSH rule 0 x 15 [15,1,9,6,13,3]
+ CRUSH rule 0 x 16 [7,11,14,2,13,1]
+ CRUSH rule 0 x 17 [10,1,13,2,4,6]
+ CRUSH rule 0 x 18 [1,7,3,10,5,12]
+ CRUSH rule 0 x 19 [7,12,2,4,15,10]
+ CRUSH rule 0 x 20 [14,12,3,10,9,4]
+ CRUSH rule 0 x 21 [3,12,1,10,4,15]
+ CRUSH rule 0 x 22 [6,3,13,11,4,1]
+ CRUSH rule 0 x 23 [10,5,13,9,3,15]
+ CRUSH rule 0 x 24 [12,11,3,1,9,4]
+ CRUSH rule 0 x 25 [7,12,15,1,3,10]
+ CRUSH rule 0 x 26 [1,7,13,2,14,5]
+ CRUSH rule 0 x 27 [3,6,15,4,13,9]
+ CRUSH rule 0 x 28 [14,4,3,9,6,11]
+ CRUSH rule 0 x 29 [5,14,12,11,6,3]
+ CRUSH rule 0 x 30 [2,5,6,9,1,11]
+ CRUSH rule 0 x 31 [5,15,10,1,9,13]
+ CRUSH rule 0 x 32 [9,10,2,1,13,14]
+ CRUSH rule 0 x 33 [13,4,9,2,7,1]
+ CRUSH rule 0 x 34 [13,15,2,4,1,10]
+ CRUSH rule 0 x 35 [4,14,3,13,10,9]
+ CRUSH rule 0 x 36 [3,12,9,7,5,10]
+ CRUSH rule 0 x 37 [9,2,6,14,11,1]
+ CRUSH rule 0 x 38 [3,4,13,10,9,1]
+ CRUSH rule 0 x 39 [12,7,14,11,1,9]
+ CRUSH rule 0 x 40 [10,1,9,5,15,2]
+ CRUSH rule 0 x 41 [4,9,11,1,14,13]
+ CRUSH rule 0 x 42 [3,6,14,10,12,5]
+ CRUSH rule 0 x 43 [10,5,15,7,2,9]
+ CRUSH rule 0 x 44 [11,4,13,3,7,14]
+ CRUSH rule 0 x 45 [11,12,15,9,1,5]
+ CRUSH rule 0 x 46 [6,9,2,14,11,13]
+ CRUSH rule 0 x 47 [3,9,6,4,13,1]
+ CRUSH rule 0 x 48 [4,6,2,1,10,14]
+ CRUSH rule 0 x 49 [9,15,10,7,4,3]
+ CRUSH rule 0 x 50 [14,12,1,4,2,11]
+ CRUSH rule 0 x 51 [10,6,5,12,15,2]
+ CRUSH rule 0 x 52 [12,1,9,11,7,3]
+ CRUSH rule 0 x 53 [3,6,13,9,5,1]
+ CRUSH rule 0 x 54 [4,13,9,2,14,10]
+ CRUSH rule 0 x 55 [4,11,2,7,1,13]
+ CRUSH rule 0 x 56 [5,9,10,1,3,13]
+ CRUSH rule 0 x 57 [6,2,1,15,10,12]
+ CRUSH rule 0 x 58 [7,1,11,4,3,14]
+ CRUSH rule 0 x 59 [2,13,1,10,9,5]
+ CRUSH rule 0 x 60 [3,6,11,1,4,9]
+ CRUSH rule 0 x 61 [3,15,13,7,4,1]
+ CRUSH rule 0 x 62 [15,11,7,12,5,9]
+ CRUSH rule 0 x 63 [10,14,12,1,7,3]
+ CRUSH rule 0 x 64 [3,9,1,4,7,12]
+ CRUSH rule 0 x 65 [4,12,11,7,14,3]
+ CRUSH rule 0 x 66 [15,11,6,9,4,1]
+ CRUSH rule 0 x 67 [2,6,4,14,1,11]
+ CRUSH rule 0 x 68 [15,7,4,2,9,12]
+ CRUSH rule 0 x 69 [2,1,15,10,4,9]
+ CRUSH rule 0 x 70 [9,6,1,3,13,15]
+ CRUSH rule 0 x 71 [15,5,1,3,13,10]
+ CRUSH rule 0 x 72 [9,10,3,5,7,12]
+ CRUSH rule 0 x 73 [5,3,11,1,7,12]
+ CRUSH rule 0 x 74 [11,7,9,5,1,15]
+ CRUSH rule 0 x 75 [9,7,11,14,12,1]
+ CRUSH rule 0 x 76 [6,1,3,5,14,10]
+ CRUSH rule 0 x 77 [7,4,2,13,9,1]
+ CRUSH rule 0 x 78 [9,3,1,5,6,13]
+ CRUSH rule 0 x 79 [13,2,15,5,7,9]
+ CRUSH rule 0 x 80 [15,2,6,4,13,10]
+ CRUSH rule 0 x 81 [15,2,1,11,4,6]
+ CRUSH rule 0 x 82 [14,13,5,11,6,2]
+ CRUSH rule 0 x 83 [4,15,3,9,10,13]
+ CRUSH rule 0 x 84 [10,7,9,15,3,4]
+ CRUSH rule 0 x 85 [3,15,9,7,4,11]
+ CRUSH rule 0 x 86 [10,9,14,1,13,4]
+ CRUSH rule 0 x 87 [15,10,7,12,5,3]
+ CRUSH rule 0 x 88 [4,13,3,1,9,15]
+ CRUSH rule 0 x 89 [3,9,7,4,1,14]
+ CRUSH rule 0 x 90 [4,9,7,12,11,14]
+ CRUSH rule 0 x 91 [6,11,9,1,2,4]
+ CRUSH rule 0 x 92 [1,5,10,9,13,15]
+ CRUSH rule 0 x 93 [9,3,15,13,7,5]
+ CRUSH rule 0 x 94 [9,2,12,5,6,11]
+ CRUSH rule 0 x 95 [7,15,4,10,9,13]
+ CRUSH rule 0 x 96 [2,15,11,7,5,1]
+ CRUSH rule 0 x 97 [4,11,2,13,1,7]
+ CRUSH rule 0 x 98 [11,13,9,3,15,1]
+ CRUSH rule 0 x 99 [12,4,11,7,3,14]
+ CRUSH rule 0 x 100 [9,4,10,15,7,3]
+ CRUSH rule 0 x 101 [15,7,1,9,10,5]
+ CRUSH rule 0 x 102 [3,11,14,6,13,4]
+ CRUSH rule 0 x 103 [13,11,6,14,4,3]
+ CRUSH rule 0 x 104 [14,6,3,5,9,1]
+ CRUSH rule 0 x 105 [14,10,1,9,3,5]
+ CRUSH rule 0 x 106 [6,5,13,2,14,11]
+ CRUSH rule 0 x 107 [3,1,10,14,13,5]
+ CRUSH rule 0 x 108 [5,10,7,2,15,9]
+ CRUSH rule 0 x 109 [9,1,13,7,15,5]
+ CRUSH rule 0 x 110 [5,1,11,3,7,14]
+ CRUSH rule 0 x 111 [10,1,9,7,5,2]
+ CRUSH rule 0 x 112 [1,10,4,14,2,12]
+ CRUSH rule 0 x 113 [6,10,13,9,1,5]
+ CRUSH rule 0 x 114 [5,13,6,2,1,14]
+ CRUSH rule 0 x 115 [10,13,14,3,9,1]
+ CRUSH rule 0 x 116 [1,14,13,2,11,5]
+ CRUSH rule 0 x 117 [5,6,1,12,15,9]
+ CRUSH rule 0 x 118 [10,4,13,15,9,3]
+ CRUSH rule 0 x 119 [14,12,11,4,6,9]
+ CRUSH rule 0 x 120 [11,3,14,13,4,7]
+ CRUSH rule 0 x 121 [9,5,1,11,7,3]
+ CRUSH rule 0 x 122 [4,3,14,1,11,13]
+ CRUSH rule 0 x 123 [3,10,5,6,9,1]
+ CRUSH rule 0 x 124 [12,2,1,5,14,7]
+ CRUSH rule 0 x 125 [9,12,15,1,6,5]
+ CRUSH rule 0 x 126 [7,15,10,9,2,12]
+ CRUSH rule 0 x 127 [4,14,9,13,1,3]
+ CRUSH rule 0 x 128 [3,12,1,10,4,9]
+ CRUSH rule 0 x 129 [11,13,14,2,9,4]
+ CRUSH rule 0 x 130 [3,13,5,14,10,1]
+ CRUSH rule 0 x 131 [12,1,6,15,4,2]
+ CRUSH rule 0 x 132 [11,15,13,9,2,5]
+ CRUSH rule 0 x 133 [3,6,9,11,15,12]
+ CRUSH rule 0 x 134 [12,5,6,15,3,9]
+ CRUSH rule 0 x 135 [3,14,12,4,6,11]
+ CRUSH rule 0 x 136 [15,6,9,4,10,3]
+ CRUSH rule 0 x 137 [14,3,6,11,1,9]
+ CRUSH rule 0 x 138 [13,15,4,10,2,7]
+ CRUSH rule 0 x 139 [11,2,13,9,1,15]
+ CRUSH rule 0 x 140 [11,4,12,15,2,6]
+ CRUSH rule 0 x 141 [6,12,15,11,3,5]
+ CRUSH rule 0 x 142 [3,14,7,9,11,1]
+ CRUSH rule 0 x 143 [9,6,4,2,14,10]
+ CRUSH rule 0 x 144 [13,7,11,2,14,4]
+ CRUSH rule 0 x 145 [12,2,6,10,9,4]
+ CRUSH rule 0 x 146 [1,5,9,2,6,13]
+ CRUSH rule 0 x 147 [1,4,9,11,2,7]
+ CRUSH rule 0 x 148 [12,7,9,2,14,11]
+ CRUSH rule 0 x 149 [2,5,9,12,11,1]
+ CRUSH rule 0 x 150 [1,15,2,10,7,9]
+ CRUSH rule 0 x 151 [2,9,14,7,1,10]
+ CRUSH rule 0 x 152 [5,9,2,6,10,13]
+ CRUSH rule 0 x 153 [6,9,4,15,2,1]
+ CRUSH rule 0 x 154 [3,11,7,1,4,12]
+ CRUSH rule 0 x 155 [14,12,7,3,5,1]
+ CRUSH rule 0 x 156 [7,13,3,10,15,5]
+ CRUSH rule 0 x 157 [15,1,6,4,3,10]
+ CRUSH rule 0 x 158 [15,1,10,6,12,2]
+ CRUSH rule 0 x 159 [4,14,3,12,10,6]
+ CRUSH rule 0 x 160 [5,7,3,14,11,1]
+ CRUSH rule 0 x 161 [1,2,11,4,6,13]
+ CRUSH rule 0 x 162 [10,6,1,12,2,4]
+ CRUSH rule 0 x 163 [15,1,10,2,6,4]
+ CRUSH rule 0 x 164 [9,14,10,7,12,2]
+ CRUSH rule 0 x 165 [11,7,2,13,9,15]
+ CRUSH rule 0 x 166 [1,2,12,14,4,11]
+ CRUSH rule 0 x 167 [9,7,3,4,11,13]
+ CRUSH rule 0 x 168 [13,2,4,1,6,15]
+ CRUSH rule 0 x 169 [1,4,9,14,13,10]
+ CRUSH rule 0 x 170 [1,15,7,9,12,10]
+ CRUSH rule 0 x 171 [9,2,10,7,1,5]
+ CRUSH rule 0 x 172 [14,4,10,12,9,3]
+ CRUSH rule 0 x 173 [5,10,12,15,6,1]
+ CRUSH rule 0 x 174 [15,6,4,12,1,11]
+ CRUSH rule 0 x 175 [5,7,9,3,10,1]
+ CRUSH rule 0 x 176 [9,6,3,14,13,10]
+ CRUSH rule 0 x 177 [2,9,10,13,4,1]
+ CRUSH rule 0 x 178 [12,11,7,14,3,4]
+ CRUSH rule 0 x 179 [2,10,13,9,5,1]
+ CRUSH rule 0 x 180 [3,11,5,15,7,12]
+ CRUSH rule 0 x 181 [9,12,6,5,1,10]
+ CRUSH rule 0 x 182 [5,13,11,2,1,6]
+ CRUSH rule 0 x 183 [5,7,10,13,3,9]
+ CRUSH rule 0 x 184 [2,5,11,12,7,1]
+ CRUSH rule 0 x 185 [13,5,7,11,2,14]
+ CRUSH rule 0 x 186 [6,14,13,5,10,1]
+ CRUSH rule 0 x 187 [1,4,11,13,6,14]
+ CRUSH rule 0 x 188 [9,13,5,14,10,6]
+ CRUSH rule 0 x 189 [6,12,4,9,2,1]
+ CRUSH rule 0 x 190 [9,13,15,10,3,1]
+ CRUSH rule 0 x 191 [7,11,4,1,15,12]
+ CRUSH rule 0 x 192 [2,11,5,15,6,1]
+ CRUSH rule 0 x 193 [3,13,6,10,4,1]
+ CRUSH rule 0 x 194 [3,13,4,14,6,9]
+ CRUSH rule 0 x 195 [5,7,10,12,1,3]
+ CRUSH rule 0 x 196 [4,15,1,10,9,2]
+ CRUSH rule 0 x 197 [14,10,13,4,6,3]
+ CRUSH rule 0 x 198 [2,5,6,15,9,13]
+ CRUSH rule 0 x 199 [2,10,4,15,1,9]
+ CRUSH rule 0 x 200 [7,14,11,4,1,3]
+ CRUSH rule 0 x 201 [9,14,1,7,4,3]
+ CRUSH rule 0 x 202 [14,11,7,3,5,1]
+ CRUSH rule 0 x 203 [12,5,7,15,1,2]
+ CRUSH rule 0 x 204 [6,11,3,12,14,1]
+ CRUSH rule 0 x 205 [15,4,6,10,13,9]
+ CRUSH rule 0 x 206 [13,11,2,15,7,1]
+ CRUSH rule 0 x 207 [2,11,7,4,14,1]
+ CRUSH rule 0 x 208 [13,1,6,14,9,11]
+ CRUSH rule 0 x 209 [6,15,13,1,11,4]
+ CRUSH rule 0 x 210 [13,11,2,7,5,14]
+ CRUSH rule 0 x 211 [2,14,1,13,11,7]
+ CRUSH rule 0 x 212 [10,1,12,15,5,6]
+ CRUSH rule 0 x 213 [3,9,6,5,15,13]
+ CRUSH rule 0 x 214 [7,15,4,1,10,2]
+ CRUSH rule 0 x 215 [6,1,4,13,3,11]
+ CRUSH rule 0 x 216 [12,9,6,2,1,11]
+ CRUSH rule 0 x 217 [12,11,1,14,2,4]
+ CRUSH rule 0 x 218 [12,10,15,6,1,4]
+ CRUSH rule 0 x 219 [3,11,14,6,4,1]
+ CRUSH rule 0 x 220 [14,4,3,12,10,9]
+ CRUSH rule 0 x 221 [15,5,2,6,12,11]
+ CRUSH rule 0 x 222 [10,4,3,15,7,12]
+ CRUSH rule 0 x 223 [9,7,11,1,4,14]
+ CRUSH rule 0 x 224 [1,7,10,2,12,9]
+ CRUSH rule 0 x 225 [10,5,2,6,1,13]
+ CRUSH rule 0 x 226 [4,1,9,3,13,10]
+ CRUSH rule 0 x 227 [7,2,12,15,5,11]
+ CRUSH rule 0 x 228 [2,15,11,1,6,13]
+ CRUSH rule 0 x 229 [9,3,7,14,1,12]
+ CRUSH rule 0 x 230 [10,5,7,2,15,1]
+ CRUSH rule 0 x 231 [2,7,5,13,9,15]
+ CRUSH rule 0 x 232 [10,5,13,1,9,2]
+ CRUSH rule 0 x 233 [6,12,11,4,9,14]
+ CRUSH rule 0 x 234 [10,1,2,12,5,9]
+ CRUSH rule 0 x 235 [13,14,7,10,1,9]
+ CRUSH rule 0 x 236 [2,15,9,12,1,7]
+ CRUSH rule 0 x 237 [3,12,9,10,4,7]
+ CRUSH rule 0 x 238 [2,10,4,15,6,12]
+ CRUSH rule 0 x 239 [4,15,10,7,9,13]
+ CRUSH rule 0 x 240 [15,5,13,7,2,9]
+ CRUSH rule 0 x 241 [7,9,15,12,1,5]
+ CRUSH rule 0 x 242 [14,2,6,9,10,12]
+ CRUSH rule 0 x 243 [2,11,5,1,15,6]
+ CRUSH rule 0 x 244 [13,9,15,3,11,7]
+ CRUSH rule 0 x 245 [12,9,15,3,1,5]
+ CRUSH rule 0 x 246 [15,3,5,11,7,1]
+ CRUSH rule 0 x 247 [6,4,9,12,1,2]
+ CRUSH rule 0 x 248 [5,13,7,11,9,15]
+ CRUSH rule 0 x 249 [10,14,7,3,9,13]
+ CRUSH rule 0 x 250 [12,15,1,10,5,6]
+ CRUSH rule 0 x 251 [13,2,15,5,6,1]
+ CRUSH rule 0 x 252 [7,5,13,9,3,10]
+ CRUSH rule 0 x 253 [3,13,15,10,7,4]
+ CRUSH rule 0 x 254 [2,9,13,14,4,6]
+ CRUSH rule 0 x 255 [1,9,13,2,6,10]
+ CRUSH rule 0 x 256 [6,9,13,1,3,14]
+ CRUSH rule 0 x 257 [15,12,3,9,6,4]
+ CRUSH rule 0 x 258 [12,5,6,10,2,1]
+ CRUSH rule 0 x 259 [9,10,4,3,14,13]
+ CRUSH rule 0 x 260 [10,12,6,9,3,15]
+ CRUSH rule 0 x 261 [13,7,2,1,15,5]
+ CRUSH rule 0 x 262 [15,3,12,7,4,9]
+ CRUSH rule 0 x 263 [12,6,10,9,5,15]
+ CRUSH rule 0 x 264 [13,14,11,3,1,4]
+ CRUSH rule 0 x 265 [12,10,14,5,7,1]
+ CRUSH rule 0 x 266 [14,7,11,1,2,9]
+ CRUSH rule 0 x 267 [12,11,6,5,1,2]
+ CRUSH rule 0 x 268 [4,1,15,12,6,11]
+ CRUSH rule 0 x 269 [11,1,15,5,13,9]
+ CRUSH rule 0 x 270 [7,11,12,3,1,14]
+ CRUSH rule 0 x 271 [4,7,3,13,15,10]
+ CRUSH rule 0 x 272 [15,5,13,10,6,2]
+ CRUSH rule 0 x 273 [2,10,7,12,1,15]
+ CRUSH rule 0 x 274 [10,2,5,6,13,9]
+ CRUSH rule 0 x 275 [10,3,4,7,14,13]
+ CRUSH rule 0 x 276 [5,12,9,2,11,7]
+ CRUSH rule 0 x 277 [14,3,13,4,1,9]
+ CRUSH rule 0 x 278 [5,6,14,3,1,11]
+ CRUSH rule 0 x 279 [6,10,13,3,9,4]
+ CRUSH rule 0 x 280 [7,3,14,9,1,11]
+ CRUSH rule 0 x 281 [5,11,14,7,9,13]
+ CRUSH rule 0 x 282 [2,1,13,14,9,7]
+ CRUSH rule 0 x 283 [4,1,12,3,10,7]
+ CRUSH rule 0 x 284 [5,11,7,15,3,13]
+ CRUSH rule 0 x 285 [15,5,3,1,6,13]
+ CRUSH rule 0 x 286 [10,4,3,6,12,15]
+ CRUSH rule 0 x 287 [12,4,9,1,3,11]
+ CRUSH rule 0 x 288 [4,12,10,7,1,3]
+ CRUSH rule 0 x 289 [2,5,14,9,13,6]
+ CRUSH rule 0 x 290 [12,2,5,6,15,9]
+ CRUSH rule 0 x 291 [7,11,1,14,5,9]
+ CRUSH rule 0 x 292 [4,10,6,3,14,9]
+ CRUSH rule 0 x 293 [6,5,11,1,2,14]
+ CRUSH rule 0 x 294 [9,12,3,14,6,11]
+ CRUSH rule 0 x 295 [6,10,3,14,9,4]
+ CRUSH rule 0 x 296 [3,1,13,7,14,9]
+ CRUSH rule 0 x 297 [6,13,4,14,10,1]
+ CRUSH rule 0 x 298 [14,9,13,1,4,2]
+ CRUSH rule 0 x 299 [14,12,11,6,4,2]
+ CRUSH rule 0 x 300 [15,7,10,5,1,3]
+ CRUSH rule 0 x 301 [9,11,7,1,13,14]
+ CRUSH rule 0 x 302 [9,7,1,13,5,10]
+ CRUSH rule 0 x 303 [4,13,3,7,10,15]
+ CRUSH rule 0 x 304 [6,9,2,11,15,13]
+ CRUSH rule 0 x 305 [13,7,5,11,2,15]
+ CRUSH rule 0 x 306 [10,12,4,6,9,2]
+ CRUSH rule 0 x 307 [11,12,15,5,6,2]
+ CRUSH rule 0 x 308 [12,14,10,9,1,2]
+ CRUSH rule 0 x 309 [9,3,12,5,11,15]
+ CRUSH rule 0 x 310 [3,1,5,10,14,9]
+ CRUSH rule 0 x 311 [3,9,7,1,14,13]
+ CRUSH rule 0 x 312 [15,13,9,7,5,10]
+ CRUSH rule 0 x 313 [9,15,3,7,5,13]
+ CRUSH rule 0 x 314 [2,15,9,5,6,12]
+ CRUSH rule 0 x 315 [15,2,13,1,11,9]
+ CRUSH rule 0 x 316 [4,9,11,2,12,14]
+ CRUSH rule 0 x 317 [1,5,3,13,15,7]
+ CRUSH rule 0 x 318 [4,1,15,11,9,13]
+ CRUSH rule 0 x 319 [2,15,4,1,11,9]
+ CRUSH rule 0 x 320 [5,7,13,9,11,2]
+ CRUSH rule 0 x 321 [1,6,11,15,5,3]
+ CRUSH rule 0 x 322 [13,7,5,3,14,11]
+ CRUSH rule 0 x 323 [7,4,10,1,2,13]
+ CRUSH rule 0 x 324 [5,6,10,15,2,13]
+ CRUSH rule 0 x 325 [9,10,14,5,1,6]
+ CRUSH rule 0 x 326 [11,7,13,4,2,15]
+ CRUSH rule 0 x 327 [12,5,10,14,3,7]
+ CRUSH rule 0 x 328 [5,2,6,14,1,11]
+ CRUSH rule 0 x 329 [2,6,15,5,9,10]
+ CRUSH rule 0 x 330 [3,9,11,13,1,6]
+ CRUSH rule 0 x 331 [12,14,6,3,1,4]
+ CRUSH rule 0 x 332 [10,12,6,15,9,2]
+ CRUSH rule 0 x 333 [6,5,3,12,14,10]
+ CRUSH rule 0 x 334 [4,9,2,12,7,11]
+ CRUSH rule 0 x 335 [11,7,1,5,13,2]
+ CRUSH rule 0 x 336 [6,14,13,2,5,9]
+ CRUSH rule 0 x 337 [15,11,3,7,12,5]
+ CRUSH rule 0 x 338 [10,5,3,6,15,1]
+ CRUSH rule 0 x 339 [11,14,13,5,3,7]
+ CRUSH rule 0 x 340 [11,6,12,4,9,3]
+ CRUSH rule 0 x 341 [7,5,2,10,14,9]
+ CRUSH rule 0 x 342 [12,14,1,9,2,11]
+ CRUSH rule 0 x 343 [12,14,9,6,10,2]
+ CRUSH rule 0 x 344 [9,11,5,2,14,13]
+ CRUSH rule 0 x 345 [14,2,11,9,6,12]
+ CRUSH rule 0 x 346 [5,3,14,10,7,1]
+ CRUSH rule 0 x 347 [10,2,12,6,9,1]
+ CRUSH rule 0 x 348 [7,9,10,1,14,13]
+ CRUSH rule 0 x 349 [9,6,10,12,1,5]
+ CRUSH rule 0 x 350 [13,9,15,4,10,7]
+ CRUSH rule 0 x 351 [13,5,15,3,1,6]
+ CRUSH rule 0 x 352 [1,12,11,9,4,7]
+ CRUSH rule 0 x 353 [10,14,12,2,9,1]
+ CRUSH rule 0 x 354 [6,3,15,10,9,4]
+ CRUSH rule 0 x 355 [13,14,6,10,2,5]
+ CRUSH rule 0 x 356 [15,13,2,9,6,5]
+ CRUSH rule 0 x 357 [4,11,1,13,3,14]
+ CRUSH rule 0 x 358 [12,7,2,9,1,14]
+ CRUSH rule 0 x 359 [5,15,7,11,3,13]
+ CRUSH rule 0 x 360 [13,10,1,2,6,14]
+ CRUSH rule 0 x 361 [5,3,13,6,1,14]
+ CRUSH rule 0 x 362 [2,9,11,13,1,6]
+ CRUSH rule 0 x 363 [7,12,3,9,15,4]
+ CRUSH rule 0 x 364 [2,12,6,9,5,10]
+ CRUSH rule 0 x 365 [13,5,11,15,6,2]
+ CRUSH rule 0 x 366 [12,7,3,14,5,10]
+ CRUSH rule 0 x 367 [7,13,3,1,5,11]
+ CRUSH rule 0 x 368 [7,9,10,15,3,4]
+ CRUSH rule 0 x 369 [7,5,3,13,14,9]
+ CRUSH rule 0 x 370 [4,7,14,1,2,9]
+ CRUSH rule 0 x 371 [1,7,12,3,4,15]
+ CRUSH rule 0 x 372 [10,4,3,14,6,1]
+ CRUSH rule 0 x 373 [15,5,2,6,13,1]
+ CRUSH rule 0 x 374 [3,15,12,5,1,6]
+ CRUSH rule 0 x 375 [5,2,14,1,6,13]
+ CRUSH rule 0 x 376 [5,14,10,13,3,6]
+ CRUSH rule 0 x 377 [1,15,2,4,9,11]
+ CRUSH rule 0 x 378 [9,12,2,15,1,5]
+ CRUSH rule 0 x 379 [11,2,15,5,7,9]
+ CRUSH rule 0 x 380 [6,1,12,11,2,9]
+ CRUSH rule 0 x 381 [15,13,7,5,10,2]
+ CRUSH rule 0 x 382 [14,3,1,4,13,7]
+ CRUSH rule 0 x 383 [3,6,11,4,13,15]
+ CRUSH rule 0 x 384 [4,13,6,3,15,11]
+ CRUSH rule 0 x 385 [4,6,15,3,10,9]
+ CRUSH rule 0 x 386 [14,3,11,13,5,6]
+ CRUSH rule 0 x 387 [1,11,5,7,9,2]
+ CRUSH rule 0 x 388 [2,6,11,9,15,4]
+ CRUSH rule 0 x 389 [12,7,2,4,15,10]
+ CRUSH rule 0 x 390 [2,11,13,7,5,9]
+ CRUSH rule 0 x 391 [3,4,9,13,7,10]
+ CRUSH rule 0 x 392 [11,5,14,7,1,9]
+ CRUSH rule 0 x 393 [2,14,5,9,7,13]
+ CRUSH rule 0 x 394 [4,9,3,15,13,6]
+ CRUSH rule 0 x 395 [10,13,5,15,6,9]
+ CRUSH rule 0 x 396 [2,12,15,9,4,6]
+ CRUSH rule 0 x 397 [1,14,9,4,12,10]
+ CRUSH rule 0 x 398 [9,2,1,5,12,6]
+ CRUSH rule 0 x 399 [5,9,14,3,1,10]
+ CRUSH rule 0 x 400 [10,6,2,4,15,12]
+ CRUSH rule 0 x 401 [6,9,11,12,4,3]
+ CRUSH rule 0 x 402 [4,7,9,2,13,1]
+ CRUSH rule 0 x 403 [7,15,13,3,5,9]
+ CRUSH rule 0 x 404 [14,12,7,9,2,1]
+ CRUSH rule 0 x 405 [9,15,11,2,4,7]
+ CRUSH rule 0 x 406 [12,14,9,2,7,10]
+ CRUSH rule 0 x 407 [9,5,12,10,15,6]
+ CRUSH rule 0 x 408 [7,1,5,2,10,15]
+ CRUSH rule 0 x 409 [11,2,4,13,1,15]
+ CRUSH rule 0 x 410 [6,4,14,2,12,9]
+ CRUSH rule 0 x 411 [13,11,15,6,4,1]
+ CRUSH rule 0 x 412 [5,9,6,11,14,2]
+ CRUSH rule 0 x 413 [13,5,3,11,6,9]
+ CRUSH rule 0 x 414 [3,11,9,13,4,1]
+ CRUSH rule 0 x 415 [6,10,14,5,1,13]
+ CRUSH rule 0 x 416 [13,1,4,7,2,9]
+ CRUSH rule 0 x 417 [4,12,1,15,2,11]
+ CRUSH rule 0 x 418 [14,5,10,2,6,9]
+ CRUSH rule 0 x 419 [5,14,10,9,2,12]
+ CRUSH rule 0 x 420 [2,4,9,11,6,14]
+ CRUSH rule 0 x 421 [15,4,10,3,9,12]
+ CRUSH rule 0 x 422 [4,11,2,7,13,9]
+ CRUSH rule 0 x 423 [3,15,12,6,5,1]
+ CRUSH rule 0 x 424 [6,10,12,2,5,1]
+ CRUSH rule 0 x 425 [11,15,2,13,5,7]
+ CRUSH rule 0 x 426 [12,4,7,1,9,10]
+ CRUSH rule 0 x 427 [14,10,3,1,9,7]
+ CRUSH rule 0 x 428 [12,7,9,4,2,1]
+ CRUSH rule 0 x 429 [3,4,9,7,11,12]
+ CRUSH rule 0 x 430 [3,5,10,13,1,15]
+ CRUSH rule 0 x 431 [9,3,7,1,12,5]
+ CRUSH rule 0 x 432 [4,1,12,7,15,2]
+ CRUSH rule 0 x 433 [4,11,12,15,7,3]
+ CRUSH rule 0 x 434 [2,14,9,1,5,11]
+ CRUSH rule 0 x 435 [13,11,5,6,9,2]
+ CRUSH rule 0 x 436 [9,15,10,2,4,1]
+ CRUSH rule 0 x 437 [9,6,3,14,10,12]
+ CRUSH rule 0 x 438 [7,2,13,4,11,1]
+ CRUSH rule 0 x 439 [7,14,4,3,12,10]
+ CRUSH rule 0 x 440 [14,11,9,2,7,12]
+ CRUSH rule 0 x 441 [2,4,11,9,13,6]
+ CRUSH rule 0 x 442 [10,13,9,7,15,1]
+ CRUSH rule 0 x 443 [12,15,10,9,2,1]
+ CRUSH rule 0 x 444 [4,13,7,14,3,1]
+ CRUSH rule 0 x 445 [4,2,15,7,1,9]
+ CRUSH rule 0 x 446 [12,10,6,9,4,1]
+ CRUSH rule 0 x 447 [15,7,13,1,4,9]
+ CRUSH rule 0 x 448 [5,2,13,7,15,10]
+ CRUSH rule 0 x 449 [14,5,3,12,10,9]
+ CRUSH rule 0 x 450 [2,4,6,9,15,1]
+ CRUSH rule 0 x 451 [6,14,11,3,9,1]
+ CRUSH rule 0 x 452 [14,9,10,4,2,13]
+ CRUSH rule 0 x 453 [5,15,13,2,6,9]
+ CRUSH rule 0 x 454 [10,4,2,6,15,12]
+ CRUSH rule 0 x 455 [6,13,2,4,10,1]
+ CRUSH rule 0 x 456 [5,7,13,1,11,3]
+ CRUSH rule 0 x 457 [9,1,5,7,11,13]
+ CRUSH rule 0 x 458 [9,11,15,4,7,2]
+ CRUSH rule 0 x 459 [13,15,11,1,5,2]
+ CRUSH rule 0 x 460 [5,12,10,15,7,3]
+ CRUSH rule 0 x 461 [4,3,9,13,15,6]
+ CRUSH rule 0 x 462 [4,7,12,14,11,1]
+ CRUSH rule 0 x 463 [4,12,14,11,2,7]
+ CRUSH rule 0 x 464 [4,2,15,10,1,9]
+ CRUSH rule 0 x 465 [5,10,9,7,13,1]
+ CRUSH rule 0 x 466 [13,5,2,15,9,11]
+ CRUSH rule 0 x 467 [13,6,14,3,9,1]
+ CRUSH rule 0 x 468 [10,7,12,14,4,1]
+ CRUSH rule 0 x 469 [4,9,6,14,12,11]
+ CRUSH rule 0 x 470 [3,9,12,15,5,6]
+ CRUSH rule 0 x 471 [6,1,5,14,13,10]
+ CRUSH rule 0 x 472 [2,14,7,5,13,1]
+ CRUSH rule 0 x 473 [15,10,6,9,4,12]
+ CRUSH rule 0 x 474 [15,10,4,12,6,9]
+ CRUSH rule 0 x 475 [10,5,12,9,14,3]
+ CRUSH rule 0 x 476 [3,6,10,12,1,15]
+ CRUSH rule 0 x 477 [6,13,5,15,11,9]
+ CRUSH rule 0 x 478 [4,15,1,3,7,12]
+ CRUSH rule 0 x 479 [13,11,1,6,14,5]
+ CRUSH rule 0 x 480 [1,13,6,4,9,14]
+ CRUSH rule 0 x 481 [15,12,7,9,1,3]
+ CRUSH rule 0 x 482 [2,12,9,1,7,11]
+ CRUSH rule 0 x 483 [10,1,4,15,9,7]
+ CRUSH rule 0 x 484 [1,4,10,13,7,14]
+ CRUSH rule 0 x 485 [9,4,3,1,14,12]
+ CRUSH rule 0 x 486 [3,10,15,9,7,13]
+ CRUSH rule 0 x 487 [12,11,4,14,7,2]
+ CRUSH rule 0 x 488 [14,4,1,9,2,6]
+ CRUSH rule 0 x 489 [11,4,2,13,15,7]
+ CRUSH rule 0 x 490 [4,9,1,3,13,15]
+ CRUSH rule 0 x 491 [1,12,5,2,14,11]
+ CRUSH rule 0 x 492 [5,7,11,3,14,9]
+ CRUSH rule 0 x 493 [12,1,4,15,3,11]
+ CRUSH rule 0 x 494 [1,7,13,4,15,9]
+ CRUSH rule 0 x 495 [3,15,7,1,9,5]
+ CRUSH rule 0 x 496 [5,3,7,13,9,14]
+ CRUSH rule 0 x 497 [13,10,3,6,5,14]
+ CRUSH rule 0 x 498 [10,6,1,5,9,12]
+ CRUSH rule 0 x 499 [14,3,12,5,1,11]
+ CRUSH rule 0 x 500 [15,9,6,12,11,2]
+ CRUSH rule 0 x 501 [10,13,1,9,3,14]
+ CRUSH rule 0 x 502 [5,1,14,11,7,12]
+ CRUSH rule 0 x 503 [15,10,7,9,1,12]
+ CRUSH rule 0 x 504 [13,2,7,1,14,11]
+ CRUSH rule 0 x 505 [12,7,5,2,14,10]
+ CRUSH rule 0 x 506 [11,7,9,14,12,1]
+ CRUSH rule 0 x 507 [4,14,13,3,9,7]
+ CRUSH rule 0 x 508 [12,1,4,9,2,11]
+ CRUSH rule 0 x 509 [4,2,6,9,14,1]
+ CRUSH rule 0 x 510 [5,3,1,12,11,14]
+ CRUSH rule 0 x 511 [2,12,10,6,14,5]
+ CRUSH rule 0 x 512 [15,11,3,5,7,1]
+ CRUSH rule 0 x 513 [4,9,11,3,13,7]
+ CRUSH rule 0 x 514 [11,9,3,4,12,15]
+ CRUSH rule 0 x 515 [12,14,6,5,3,9]
+ CRUSH rule 0 x 516 [14,11,1,12,3,7]
+ CRUSH rule 0 x 517 [11,5,6,13,9,3]
+ CRUSH rule 0 x 518 [3,5,7,12,15,11]
+ CRUSH rule 0 x 519 [12,14,2,1,4,6]
+ CRUSH rule 0 x 520 [12,4,2,10,6,15]
+ CRUSH rule 0 x 521 [11,5,9,6,15,3]
+ CRUSH rule 0 x 522 [4,12,11,1,15,3]
+ CRUSH rule 0 x 523 [3,1,5,9,15,10]
+ CRUSH rule 0 x 524 [15,9,3,11,13,7]
+ CRUSH rule 0 x 525 [3,15,11,6,9,12]
+ CRUSH rule 0 x 526 [10,2,5,13,6,15]
+ CRUSH rule 0 x 527 [3,13,4,1,9,10]
+ CRUSH rule 0 x 528 [12,7,15,10,2,5]
+ CRUSH rule 0 x 529 [6,4,10,12,2,9]
+ CRUSH rule 0 x 530 [11,9,12,7,5,1]
+ CRUSH rule 0 x 531 [9,15,4,7,2,13]
+ CRUSH rule 0 x 532 [5,3,13,7,9,14]
+ CRUSH rule 0 x 533 [12,15,1,2,7,5]
+ CRUSH rule 0 x 534 [11,9,3,7,15,4]
+ CRUSH rule 0 x 535 [11,1,3,5,14,9]
+ CRUSH rule 0 x 536 [9,1,14,13,4,6]
+ CRUSH rule 0 x 537 [15,5,13,2,7,11]
+ CRUSH rule 0 x 538 [13,5,11,2,6,15]
+ CRUSH rule 0 x 539 [10,12,6,14,1,2]
+ CRUSH rule 0 x 540 [12,15,7,3,9,11]
+ CRUSH rule 0 x 541 [2,1,6,11,14,13]
+ CRUSH rule 0 x 542 [3,9,15,5,11,12]
+ CRUSH rule 0 x 543 [4,10,9,3,6,13]
+ CRUSH rule 0 x 544 [3,15,9,11,7,4]
+ CRUSH rule 0 x 545 [14,10,7,12,4,9]
+ CRUSH rule 0 x 546 [5,15,13,7,1,10]
+ CRUSH rule 0 x 547 [5,13,7,9,3,14]
+ CRUSH rule 0 x 548 [11,7,12,15,4,2]
+ CRUSH rule 0 x 549 [14,1,4,9,13,6]
+ CRUSH rule 0 x 550 [9,15,3,13,1,6]
+ CRUSH rule 0 x 551 [11,2,15,6,13,5]
+ CRUSH rule 0 x 552 [2,11,14,1,9,6]
+ CRUSH rule 0 x 553 [11,9,14,6,4,13]
+ CRUSH rule 0 x 554 [11,14,6,4,13,9]
+ CRUSH rule 0 x 555 [6,5,10,9,14,2]
+ CRUSH rule 0 x 556 [15,6,3,13,11,4]
+ CRUSH rule 0 x 557 [12,2,5,14,10,9]
+ CRUSH rule 0 x 558 [12,1,6,15,5,10]
+ CRUSH rule 0 x 559 [2,13,5,10,14,7]
+ CRUSH rule 0 x 560 [4,9,12,6,3,10]
+ CRUSH rule 0 x 561 [12,7,1,2,5,15]
+ CRUSH rule 0 x 562 [7,13,9,14,2,1]
+ CRUSH rule 0 x 563 [15,4,3,10,13,9]
+ CRUSH rule 0 x 564 [2,13,7,1,15,10]
+ CRUSH rule 0 x 565 [3,12,4,1,14,7]
+ CRUSH rule 0 x 566 [6,14,4,2,13,11]
+ CRUSH rule 0 x 567 [15,4,11,6,3,12]
+ CRUSH rule 0 x 568 [4,14,1,6,10,13]
+ CRUSH rule 0 x 569 [11,3,15,13,5,1]
+ CRUSH rule 0 x 570 [1,10,13,4,7,2]
+ CRUSH rule 0 x 571 [10,12,14,9,4,2]
+ CRUSH rule 0 x 572 [12,14,3,10,6,1]
+ CRUSH rule 0 x 573 [7,15,11,2,12,9]
+ CRUSH rule 0 x 574 [11,14,13,1,3,7]
+ CRUSH rule 0 x 575 [5,13,15,9,6,10]
+ CRUSH rule 0 x 576 [3,15,11,9,1,6]
+ CRUSH rule 0 x 577 [13,9,6,15,3,11]
+ CRUSH rule 0 x 578 [4,10,1,2,7,13]
+ CRUSH rule 0 x 579 [13,1,15,2,10,7]
+ CRUSH rule 0 x 580 [3,12,4,1,10,15]
+ CRUSH rule 0 x 581 [7,14,12,10,1,2]
+ CRUSH rule 0 x 582 [10,5,13,14,1,2]
+ CRUSH rule 0 x 583 [4,15,1,9,10,12]
+ CRUSH rule 0 x 584 [10,1,5,13,6,9]
+ CRUSH rule 0 x 585 [5,3,6,1,11,14]
+ CRUSH rule 0 x 586 [7,10,14,12,9,3]
+ CRUSH rule 0 x 587 [11,6,9,4,1,14]
+ CRUSH rule 0 x 588 [3,12,7,15,4,9]
+ CRUSH rule 0 x 589 [9,7,12,1,10,3]
+ CRUSH rule 0 x 590 [12,1,3,9,10,6]
+ CRUSH rule 0 x 591 [2,6,14,13,9,11]
+ CRUSH rule 0 x 592 [15,12,9,7,5,2]
+ CRUSH rule 0 x 593 [13,14,5,11,9,6]
+ CRUSH rule 0 x 594 [12,14,2,9,7,4]
+ CRUSH rule 0 x 595 [12,7,10,3,1,14]
+ CRUSH rule 0 x 596 [2,7,12,11,1,5]
+ CRUSH rule 0 x 597 [15,1,2,10,7,13]
+ CRUSH rule 0 x 598 [11,5,9,14,12,7]
+ CRUSH rule 0 x 599 [13,11,1,5,6,2]
+ CRUSH rule 0 x 600 [4,12,3,10,9,7]
+ CRUSH rule 0 x 601 [13,5,15,2,1,7]
+ CRUSH rule 0 x 602 [3,11,7,1,13,15]
+ CRUSH rule 0 x 603 [3,1,4,14,10,9]
+ CRUSH rule 0 x 604 [14,2,6,1,11,13]
+ CRUSH rule 0 x 605 [2,7,12,5,14,10]
+ CRUSH rule 0 x 606 [12,15,1,5,7,9]
+ CRUSH rule 0 x 607 [3,9,10,14,7,1]
+ CRUSH rule 0 x 608 [13,10,1,7,9,15]
+ CRUSH rule 0 x 609 [14,3,7,9,11,12]
+ CRUSH rule 0 x 610 [7,10,5,1,12,2]
+ CRUSH rule 0 x 611 [13,1,5,3,10,7]
+ CRUSH rule 0 x 612 [7,1,2,13,9,15]
+ CRUSH rule 0 x 613 [10,7,14,9,5,2]
+ CRUSH rule 0 x 614 [9,4,15,3,1,11]
+ CRUSH rule 0 x 615 [9,4,11,2,1,12]
+ CRUSH rule 0 x 616 [10,14,1,5,3,6]
+ CRUSH rule 0 x 617 [15,7,2,11,12,1]
+ CRUSH rule 0 x 618 [4,2,10,6,14,9]
+ CRUSH rule 0 x 619 [15,4,3,9,6,1]
+ CRUSH rule 0 x 620 [3,7,11,14,13,1]
+ CRUSH rule 0 x 621 [3,6,4,14,1,11]
+ CRUSH rule 0 x 622 [10,2,13,5,15,9]
+ CRUSH rule 0 x 623 [4,9,14,7,3,13]
+ CRUSH rule 0 x 624 [3,9,15,6,10,1]
+ CRUSH rule 0 x 625 [11,7,3,5,13,15]
+ CRUSH rule 0 x 626 [10,12,2,1,9,7]
+ CRUSH rule 0 x 627 [1,12,10,14,3,5]
+ CRUSH rule 0 x 628 [15,13,11,4,2,1]
+ CRUSH rule 0 x 629 [5,6,15,12,1,10]
+ CRUSH rule 0 x 630 [1,4,12,9,3,7]
+ CRUSH rule 0 x 631 [5,7,1,15,12,11]
+ CRUSH rule 0 x 632 [12,3,11,9,6,1]
+ CRUSH rule 0 x 633 [14,4,3,7,10,12]
+ CRUSH rule 0 x 634 [6,9,5,3,13,11]
+ CRUSH rule 0 x 635 [6,5,2,15,9,12]
+ CRUSH rule 0 x 636 [13,6,11,3,15,9]
+ CRUSH rule 0 x 637 [3,1,10,6,9,12]
+ CRUSH rule 0 x 638 [10,15,3,5,13,1]
+ CRUSH rule 0 x 639 [6,9,14,4,3,1]
+ CRUSH rule 0 x 640 [9,6,1,11,14,2]
+ CRUSH rule 0 x 641 [10,6,5,14,1,9]
+ CRUSH rule 0 x 642 [1,15,4,6,2,10]
+ CRUSH rule 0 x 643 [3,7,5,1,10,15]
+ CRUSH rule 0 x 644 [15,13,6,9,3,11]
+ CRUSH rule 0 x 645 [14,2,4,9,10,1]
+ CRUSH rule 0 x 646 [5,13,14,1,6,9]
+ CRUSH rule 0 x 647 [10,1,9,13,6,2]
+ CRUSH rule 0 x 648 [6,5,2,14,11,1]
+ CRUSH rule 0 x 649 [3,9,13,11,4,14]
+ CRUSH rule 0 x 650 [10,9,4,15,12,7]
+ CRUSH rule 0 x 651 [3,9,5,7,14,1]
+ CRUSH rule 0 x 652 [15,9,4,6,13,1]
+ CRUSH rule 0 x 653 [11,14,1,3,6,9]
+ CRUSH rule 0 x 654 [13,6,2,10,15,4]
+ CRUSH rule 0 x 655 [6,3,4,15,12,11]
+ CRUSH rule 0 x 656 [3,15,1,4,6,12]
+ CRUSH rule 0 x 657 [11,15,3,5,7,13]
+ CRUSH rule 0 x 658 [7,2,10,12,1,4]
+ CRUSH rule 0 x 659 [2,5,14,6,10,12]
+ CRUSH rule 0 x 660 [13,14,10,6,4,9]
+ CRUSH rule 0 x 661 [7,15,3,12,11,4]
+ CRUSH rule 0 x 662 [15,2,12,5,1,10]
+ CRUSH rule 0 x 663 [14,9,13,10,5,3]
+ CRUSH rule 0 x 664 [6,10,12,4,9,2]
+ CRUSH rule 0 x 665 [2,9,12,1,7,10]
+ CRUSH rule 0 x 666 [12,3,6,1,15,9]
+ CRUSH rule 0 x 667 [1,9,12,10,2,14]
+ CRUSH rule 0 x 668 [9,5,1,2,6,11]
+ CRUSH rule 0 x 669 [9,7,14,5,11,13]
+ CRUSH rule 0 x 670 [6,10,9,13,1,2]
+ CRUSH rule 0 x 671 [6,15,5,10,13,3]
+ CRUSH rule 0 x 672 [2,9,13,1,4,14]
+ CRUSH rule 0 x 673 [7,10,5,9,15,13]
+ CRUSH rule 0 x 674 [7,12,10,1,14,9]
+ CRUSH rule 0 x 675 [9,5,1,10,6,14]
+ CRUSH rule 0 x 676 [10,12,2,1,4,15]
+ CRUSH rule 0 x 677 [2,12,1,4,10,6]
+ CRUSH rule 0 x 678 [1,2,4,10,12,14]
+ CRUSH rule 0 x 679 [5,6,12,15,9,11]
+ CRUSH rule 0 x 680 [7,11,3,1,15,4]
+ CRUSH rule 0 x 681 [6,4,3,11,14,13]
+ CRUSH rule 0 x 682 [6,1,11,15,12,2]
+ CRUSH rule 0 x 683 [6,13,2,4,9,14]
+ CRUSH rule 0 x 684 [9,11,3,7,15,4]
+ CRUSH rule 0 x 685 [5,1,15,7,9,2]
+ CRUSH rule 0 x 686 [1,9,11,14,6,13]
+ CRUSH rule 0 x 687 [7,13,3,5,11,9]
+ CRUSH rule 0 x 688 [11,9,1,14,3,5]
+ CRUSH rule 0 x 689 [5,2,9,12,1,14]
+ CRUSH rule 0 x 690 [9,7,10,3,13,15]
+ CRUSH rule 0 x 691 [11,15,9,5,7,13]
+ CRUSH rule 0 x 692 [15,5,1,2,9,11]
+ CRUSH rule 0 x 693 [5,6,12,15,2,10]
+ CRUSH rule 0 x 694 [4,7,1,10,12,3]
+ CRUSH rule 0 x 695 [6,13,14,10,9,5]
+ CRUSH rule 0 x 696 [1,2,4,14,7,11]
+ CRUSH rule 0 x 697 [13,11,3,6,4,14]
+ CRUSH rule 0 x 698 [11,13,4,2,6,1]
+ CRUSH rule 0 x 699 [7,14,12,4,2,11]
+ CRUSH rule 0 x 700 [12,14,11,9,4,6]
+ CRUSH rule 0 x 701 [3,13,1,14,4,7]
+ CRUSH rule 0 x 702 [3,12,15,6,5,11]
+ CRUSH rule 0 x 703 [15,11,13,3,4,7]
+ CRUSH rule 0 x 704 [6,4,2,15,11,1]
+ CRUSH rule 0 x 705 [14,6,11,5,1,13]
+ CRUSH rule 0 x 706 [1,12,3,6,4,10]
+ CRUSH rule 0 x 707 [4,7,14,3,10,9]
+ CRUSH rule 0 x 708 [3,10,5,1,15,9]
+ CRUSH rule 0 x 709 [11,12,3,7,5,14]
+ CRUSH rule 0 x 710 [14,2,11,9,5,7]
+ CRUSH rule 0 x 711 [14,3,9,10,12,5]
+ CRUSH rule 0 x 712 [12,3,11,15,9,1]
+ CRUSH rule 0 x 713 [11,9,3,15,13,6]
+ CRUSH rule 0 x 714 [12,1,9,7,2,15]
+ CRUSH rule 0 x 715 [6,1,14,4,11,12]
+ CRUSH rule 0 x 716 [11,13,9,14,5,2]
+ CRUSH rule 0 x 717 [12,4,10,9,15,1]
+ CRUSH rule 0 x 718 [7,15,5,2,11,13]
+ CRUSH rule 0 x 719 [5,15,13,3,1,7]
+ CRUSH rule 0 x 720 [4,13,10,2,7,9]
+ CRUSH rule 0 x 721 [11,3,14,9,1,12]
+ CRUSH rule 0 x 722 [2,4,6,1,9,15]
+ CRUSH rule 0 x 723 [2,1,12,15,11,7]
+ CRUSH rule 0 x 724 [7,1,9,10,5,15]
+ CRUSH rule 0 x 725 [11,12,7,15,4,1]
+ CRUSH rule 0 x 726 [7,14,4,3,11,13]
+ CRUSH rule 0 x 727 [2,5,1,11,15,7]
+ CRUSH rule 0 x 728 [13,11,4,6,15,2]
+ CRUSH rule 0 x 729 [15,11,4,6,2,9]
+ CRUSH rule 0 x 730 [3,7,1,13,11,15]
+ CRUSH rule 0 x 731 [9,1,6,5,2,11]
+ CRUSH rule 0 x 732 [1,2,10,13,9,4]
+ CRUSH rule 0 x 733 [11,3,5,6,1,9]
+ CRUSH rule 0 x 734 [14,3,11,7,12,9]
+ CRUSH rule 0 x 735 [6,9,2,10,13,14]
+ CRUSH rule 0 x 736 [3,9,1,11,7,5]
+ CRUSH rule 0 x 737 [1,4,2,12,9,10]
+ CRUSH rule 0 x 738 [11,15,7,4,9,2]
+ CRUSH rule 0 x 739 [11,12,6,2,4,1]
+ CRUSH rule 0 x 740 [7,9,10,13,1,15]
+ CRUSH rule 0 x 741 [12,11,7,15,2,5]
+ CRUSH rule 0 x 742 [9,7,4,11,12,1]
+ CRUSH rule 0 x 743 [5,13,9,15,10,7]
+ CRUSH rule 0 x 744 [6,2,13,1,14,11]
+ CRUSH rule 0 x 745 [3,6,1,4,11,12]
+ CRUSH rule 0 x 746 [3,7,9,10,14,5]
+ CRUSH rule 0 x 747 [15,11,5,2,13,9]
+ CRUSH rule 0 x 748 [6,10,13,2,14,5]
+ CRUSH rule 0 x 749 [14,9,10,7,5,1]
+ CRUSH rule 0 x 750 [1,14,6,5,11,2]
+ CRUSH rule 0 x 751 [15,1,6,9,5,11]
+ CRUSH rule 0 x 752 [13,1,7,3,11,15]
+ CRUSH rule 0 x 753 [4,11,1,3,15,7]
+ CRUSH rule 0 x 754 [14,12,11,4,2,1]
+ CRUSH rule 0 x 755 [13,6,1,10,4,2]
+ CRUSH rule 0 x 756 [3,4,14,6,1,10]
+ CRUSH rule 0 x 757 [10,6,1,4,13,15]
+ CRUSH rule 0 x 758 [6,3,4,10,15,13]
+ CRUSH rule 0 x 759 [5,7,3,14,11,1]
+ CRUSH rule 0 x 760 [1,15,10,12,4,3]
+ CRUSH rule 0 x 761 [2,12,1,14,5,7]
+ CRUSH rule 0 x 762 [1,4,10,9,3,7]
+ CRUSH rule 0 x 763 [4,13,1,14,7,10]
+ CRUSH rule 0 x 764 [1,14,6,13,9,5]
+ CRUSH rule 0 x 765 [9,15,2,13,4,1]
+ CRUSH rule 0 x 766 [11,2,7,15,9,12]
+ CRUSH rule 0 x 767 [6,11,4,3,12,14]
+ CRUSH rule 0 x 768 [2,12,15,7,1,11]
+ CRUSH rule 0 x 769 [15,1,9,2,11,12]
+ CRUSH rule 0 x 770 [15,13,4,6,3,10]
+ CRUSH rule 0 x 771 [9,2,12,11,6,14]
+ CRUSH rule 0 x 772 [4,3,13,11,14,1]
+ CRUSH rule 0 x 773 [3,7,4,15,1,12]
+ CRUSH rule 0 x 774 [12,6,3,15,5,9]
+ CRUSH rule 0 x 775 [5,10,14,2,6,1]
+ CRUSH rule 0 x 776 [10,15,3,9,6,13]
+ CRUSH rule 0 x 777 [11,13,4,7,1,14]
+ CRUSH rule 0 x 778 [13,1,9,11,15,6]
+ CRUSH rule 0 x 779 [5,11,1,14,2,9]
+ CRUSH rule 0 x 780 [13,9,3,6,4,1]
+ CRUSH rule 0 x 781 [5,7,14,3,1,12]
+ CRUSH rule 0 x 782 [2,15,9,7,11,13]
+ CRUSH rule 0 x 783 [12,7,5,14,9,1]
+ CRUSH rule 0 x 784 [14,1,10,13,3,4]
+ CRUSH rule 0 x 785 [6,12,1,2,4,9]
+ CRUSH rule 0 x 786 [10,5,2,15,1,7]
+ CRUSH rule 0 x 787 [1,12,10,2,9,4]
+ CRUSH rule 0 x 788 [4,2,9,13,6,15]
+ CRUSH rule 0 x 789 [9,2,14,7,4,12]
+ CRUSH rule 0 x 790 [15,2,7,4,1,10]
+ CRUSH rule 0 x 791 [9,4,7,13,14,11]
+ CRUSH rule 0 x 792 [6,4,15,10,12,3]
+ CRUSH rule 0 x 793 [15,9,6,2,13,11]
+ CRUSH rule 0 x 794 [5,12,2,14,9,10]
+ CRUSH rule 0 x 795 [6,14,12,4,10,1]
+ CRUSH rule 0 x 796 [11,2,12,6,15,4]
+ CRUSH rule 0 x 797 [14,3,7,1,5,13]
+ CRUSH rule 0 x 798 [5,11,6,13,1,3]
+ CRUSH rule 0 x 799 [2,9,14,4,13,6]
+ CRUSH rule 0 x 800 [6,3,4,11,15,13]
+ CRUSH rule 0 x 801 [2,5,6,13,9,1]
+ CRUSH rule 0 x 802 [1,4,12,7,3,9]
+ CRUSH rule 0 x 803 [7,2,4,1,11,13]
+ CRUSH rule 0 x 804 [5,14,9,7,3,1]
+ CRUSH rule 0 x 805 [13,4,3,1,10,15]
+ CRUSH rule 0 x 806 [6,2,13,4,15,1]
+ CRUSH rule 0 x 807 [14,2,7,4,9,12]
+ CRUSH rule 0 x 808 [2,15,12,7,9,1]
+ CRUSH rule 0 x 809 [1,11,7,12,4,2]
+ CRUSH rule 0 x 810 [2,5,9,12,15,1]
+ CRUSH rule 0 x 811 [15,6,3,10,1,5]
+ CRUSH rule 0 x 812 [7,11,2,14,9,5]
+ CRUSH rule 0 x 813 [4,10,13,14,2,6]
+ CRUSH rule 0 x 814 [13,4,9,3,10,6]
+ CRUSH rule 0 x 815 [15,12,9,4,10,6]
+ CRUSH rule 0 x 816 [14,10,13,7,3,9]
+ CRUSH rule 0 x 817 [10,7,2,15,13,9]
+ CRUSH rule 0 x 818 [15,2,11,4,1,12]
+ CRUSH rule 0 x 819 [5,12,10,6,1,14]
+ CRUSH rule 0 x 820 [3,6,9,12,11,15]
+ CRUSH rule 0 x 821 [15,10,9,13,3,4]
+ CRUSH rule 0 x 822 [10,13,2,9,7,4]
+ CRUSH rule 0 x 823 [2,6,12,10,15,4]
+ CRUSH rule 0 x 824 [3,7,9,13,15,5]
+ CRUSH rule 0 x 825 [10,5,14,6,12,9]
+ CRUSH rule 0 x 826 [5,2,11,15,1,12]
+ CRUSH rule 0 x 827 [13,5,1,3,7,9]
+ CRUSH rule 0 x 828 [12,6,10,5,1,9]
+ CRUSH rule 0 x 829 [13,6,15,10,5,3]
+ CRUSH rule 0 x 830 [15,13,2,9,7,11]
+ CRUSH rule 0 x 831 [1,4,11,12,6,3]
+ CRUSH rule 0 x 832 [14,11,13,2,9,4]
+ CRUSH rule 0 x 833 [9,13,3,11,7,5]
+ CRUSH rule 0 x 834 [9,7,5,1,11,2]
+ CRUSH rule 0 x 835 [14,3,13,6,4,9]
+ CRUSH rule 0 x 836 [3,9,10,13,1,5]
+ CRUSH rule 0 x 837 [15,12,11,2,7,9]
+ CRUSH rule 0 x 838 [12,14,9,2,5,7]
+ CRUSH rule 0 x 839 [3,4,6,10,15,1]
+ CRUSH rule 0 x 840 [10,15,12,4,7,1]
+ CRUSH rule 0 x 841 [3,5,7,12,11,15]
+ CRUSH rule 0 x 842 [9,13,2,6,5,14]
+ CRUSH rule 0 x 843 [14,7,4,9,3,12]
+ CRUSH rule 0 x 844 [7,1,4,15,9,2]
+ CRUSH rule 0 x 845 [13,6,1,15,4,2]
+ CRUSH rule 0 x 846 [3,7,15,13,1,9]
+ CRUSH rule 0 x 847 [12,15,11,5,2,7]
+ CRUSH rule 0 x 848 [11,13,1,14,5,9]
+ CRUSH rule 0 x 849 [3,15,11,9,6,1]
+ CRUSH rule 0 x 850 [1,3,10,6,14,4]
+ CRUSH rule 0 x 851 [14,4,3,6,11,1]
+ CRUSH rule 0 x 852 [9,12,4,7,15,2]
+ CRUSH rule 0 x 853 [13,14,6,11,2,4]
+ CRUSH rule 0 x 854 [7,11,12,1,4,15]
+ CRUSH rule 0 x 855 [14,4,12,6,3,1]
+ CRUSH rule 0 x 856 [5,10,7,3,15,9]
+ CRUSH rule 0 x 857 [4,3,13,11,9,1]
+ CRUSH rule 0 x 858 [5,15,6,3,9,12]
+ CRUSH rule 0 x 859 [5,15,6,2,1,11]
+ CRUSH rule 0 x 860 [11,14,1,12,6,9]
+ CRUSH rule 0 x 861 [13,7,4,10,1,14]
+ CRUSH rule 0 x 862 [5,10,9,7,3,12]
+ CRUSH rule 0 x 863 [11,6,3,9,4,12]
+ CRUSH rule 0 x 864 [6,13,4,2,10,15]
+ CRUSH rule 0 x 865 [4,1,14,11,6,9]
+ CRUSH rule 0 x 866 [2,13,4,15,9,6]
+ CRUSH rule 0 x 867 [12,2,9,10,4,14]
+ CRUSH rule 0 x 868 [14,11,7,2,1,4]
+ CRUSH rule 0 x 869 [10,13,7,14,3,5]
+ CRUSH rule 0 x 870 [14,9,11,4,3,12]
+ CRUSH rule 0 x 871 [6,2,1,4,15,13]
+ CRUSH rule 0 x 872 [6,1,15,3,10,12]
+ CRUSH rule 0 x 873 [2,5,12,10,1,9]
+ CRUSH rule 0 x 874 [12,4,7,2,15,10]
+ CRUSH rule 0 x 875 [10,6,14,1,12,5]
+ CRUSH rule 0 x 876 [14,7,13,3,9,1]
+ CRUSH rule 0 x 877 [15,11,13,9,5,1]
+ CRUSH rule 0 x 878 [7,14,3,13,9,1]
+ CRUSH rule 0 x 879 [12,2,7,4,10,15]
+ CRUSH rule 0 x 880 [2,12,10,7,1,4]
+ CRUSH rule 0 x 881 [6,3,1,11,4,15]
+ CRUSH rule 0 x 882 [11,13,7,1,2,15]
+ CRUSH rule 0 x 883 [13,1,3,10,6,5]
+ CRUSH rule 0 x 884 [6,15,4,9,3,11]
+ CRUSH rule 0 x 885 [14,7,9,4,2,13]
+ CRUSH rule 0 x 886 [13,11,4,2,1,14]
+ CRUSH rule 0 x 887 [14,4,12,11,2,6]
+ CRUSH rule 0 x 888 [10,12,7,15,9,2]
+ CRUSH rule 0 x 889 [15,13,4,1,6,2]
+ CRUSH rule 0 x 890 [10,12,14,2,9,5]
+ CRUSH rule 0 x 891 [9,5,11,6,3,15]
+ CRUSH rule 0 x 892 [12,15,2,4,7,9]
+ CRUSH rule 0 x 893 [1,3,5,9,6,10]
+ CRUSH rule 0 x 894 [7,2,11,13,4,1]
+ CRUSH rule 0 x 895 [2,1,11,5,7,15]
+ CRUSH rule 0 x 896 [9,1,14,10,4,12]
+ CRUSH rule 0 x 897 [7,5,14,3,1,9]
+ CRUSH rule 0 x 898 [10,6,12,9,15,5]
+ CRUSH rule 0 x 899 [1,11,5,3,13,14]
+ CRUSH rule 0 x 900 [2,9,10,7,13,14]
+ CRUSH rule 0 x 901 [9,12,11,3,14,4]
+ CRUSH rule 0 x 902 [4,2,6,15,12,10]
+ CRUSH rule 0 x 903 [14,10,3,1,12,6]
+ CRUSH rule 0 x 904 [15,12,4,9,6,3]
+ CRUSH rule 0 x 905 [12,6,11,3,9,4]
+ CRUSH rule 0 x 906 [14,11,12,2,4,9]
+ CRUSH rule 0 x 907 [7,12,3,9,10,5]
+ CRUSH rule 0 x 908 [2,15,9,6,10,13]
+ CRUSH rule 0 x 909 [10,14,1,13,2,9]
+ CRUSH rule 0 x 910 [12,7,4,15,10,3]
+ CRUSH rule 0 x 911 [11,15,2,4,9,13]
+ CRUSH rule 0 x 912 [6,4,14,13,3,1]
+ CRUSH rule 0 x 913 [4,6,10,1,12,3]
+ CRUSH rule 0 x 914 [4,15,2,10,1,13]
+ CRUSH rule 0 x 915 [12,14,1,9,4,3]
+ CRUSH rule 0 x 916 [3,1,11,5,6,13]
+ CRUSH rule 0 x 917 [1,15,6,5,10,3]
+ CRUSH rule 0 x 918 [7,14,11,4,9,2]
+ CRUSH rule 0 x 919 [10,7,3,13,15,1]
+ CRUSH rule 0 x 920 [4,2,10,15,1,13]
+ CRUSH rule 0 x 921 [1,11,6,13,4,2]
+ CRUSH rule 0 x 922 [6,4,14,13,3,1]
+ CRUSH rule 0 x 923 [12,2,5,14,10,1]
+ CRUSH rule 0 x 924 [6,2,14,13,9,1]
+ CRUSH rule 0 x 925 [12,15,2,10,1,5]
+ CRUSH rule 0 x 926 [3,13,10,1,14,9]
+ CRUSH rule 0 x 927 [6,5,1,11,14,2]
+ CRUSH rule 0 x 928 [13,1,3,9,6,11]
+ CRUSH rule 0 x 929 [10,7,1,5,2,12]
+ CRUSH rule 0 x 930 [7,15,10,5,1,13]
+ CRUSH rule 0 x 931 [6,15,11,9,5,3]
+ CRUSH rule 0 x 932 [13,2,5,11,9,1]
+ CRUSH rule 0 x 933 [12,7,14,10,4,1]
+ CRUSH rule 0 x 934 [12,2,5,7,9,1]
+ CRUSH rule 0 x 935 [6,11,1,14,5,13]
+ CRUSH rule 0 x 936 [9,12,7,5,1,2]
+ CRUSH rule 0 x 937 [14,2,11,1,13,4]
+ CRUSH rule 0 x 938 [14,3,5,11,7,9]
+ CRUSH rule 0 x 939 [6,4,14,9,12,1]
+ CRUSH rule 0 x 940 [13,11,4,2,1,6]
+ CRUSH rule 0 x 941 [3,12,4,7,14,10]
+ CRUSH rule 0 x 942 [15,12,10,4,1,9]
+ CRUSH rule 0 x 943 [10,2,4,9,6,15]
+ CRUSH rule 0 x 944 [2,9,4,7,1,14]
+ CRUSH rule 0 x 945 [10,15,2,9,5,12]
+ CRUSH rule 0 x 946 [11,15,7,12,5,9]
+ CRUSH rule 0 x 947 [11,3,14,1,12,5]
+ CRUSH rule 0 x 948 [7,13,11,5,14,2]
+ CRUSH rule 0 x 949 [9,1,12,5,15,10]
+ CRUSH rule 0 x 950 [9,15,13,6,4,2]
+ CRUSH rule 0 x 951 [2,6,12,9,10,4]
+ CRUSH rule 0 x 952 [9,7,15,3,5,13]
+ CRUSH rule 0 x 953 [1,3,6,10,12,14]
+ CRUSH rule 0 x 954 [10,2,14,9,4,6]
+ CRUSH rule 0 x 955 [7,14,3,1,10,4]
+ CRUSH rule 0 x 956 [1,6,11,5,14,3]
+ CRUSH rule 0 x 957 [14,11,1,12,6,9]
+ CRUSH rule 0 x 958 [15,4,3,11,1,6]
+ CRUSH rule 0 x 959 [2,1,12,15,10,9]
+ CRUSH rule 0 x 960 [2,6,11,13,15,4]
+ CRUSH rule 0 x 961 [3,13,11,9,6,1]
+ CRUSH rule 0 x 962 [5,11,3,14,1,6]
+ CRUSH rule 0 x 963 [13,10,15,4,6,9]
+ CRUSH rule 0 x 964 [7,11,4,9,2,12]
+ CRUSH rule 0 x 965 [12,2,9,7,4,15]
+ CRUSH rule 0 x 966 [12,14,9,4,1,2]
+ CRUSH rule 0 x 967 [7,5,3,10,12,14]
+ CRUSH rule 0 x 968 [12,15,4,9,11,6]
+ CRUSH rule 0 x 969 [11,4,7,1,9,14]
+ CRUSH rule 0 x 970 [5,12,10,1,3,14]
+ CRUSH rule 0 x 971 [1,9,4,12,7,2]
+ CRUSH rule 0 x 972 [12,3,14,5,1,9]
+ CRUSH rule 0 x 973 [1,10,4,12,2,7]
+ CRUSH rule 0 x 974 [7,11,1,2,15,4]
+ CRUSH rule 0 x 975 [7,9,15,12,2,11]
+ CRUSH rule 0 x 976 [7,3,15,5,12,11]
+ CRUSH rule 0 x 977 [14,3,6,10,4,1]
+ CRUSH rule 0 x 978 [12,5,11,1,15,3]
+ CRUSH rule 0 x 979 [5,1,13,6,15,10]
+ CRUSH rule 0 x 980 [15,11,5,6,1,3]
+ CRUSH rule 0 x 981 [5,11,15,12,7,1]
+ CRUSH rule 0 x 982 [2,6,14,11,12,9]
+ CRUSH rule 0 x 983 [3,12,10,9,14,5]
+ CRUSH rule 0 x 984 [15,13,1,10,2,5]
+ CRUSH rule 0 x 985 [11,2,15,1,4,13]
+ CRUSH rule 0 x 986 [6,13,9,1,15,10]
+ CRUSH rule 0 x 987 [13,14,5,10,6,1]
+ CRUSH rule 0 x 988 [12,9,10,14,3,1]
+ CRUSH rule 0 x 989 [7,4,3,15,9,13]
+ CRUSH rule 0 x 990 [1,10,9,13,3,4]
+ CRUSH rule 0 x 991 [7,11,1,14,2,5]
+ CRUSH rule 0 x 992 [9,10,2,13,7,4]
+ CRUSH rule 0 x 993 [6,10,14,12,4,1]
+ CRUSH rule 0 x 994 [3,13,15,4,11,7]
+ CRUSH rule 0 x 995 [15,6,12,2,5,11]
+ CRUSH rule 0 x 996 [15,10,5,3,13,1]
+ CRUSH rule 0 x 997 [15,2,1,12,7,9]
+ CRUSH rule 0 x 998 [6,1,9,5,12,11]
+ CRUSH rule 0 x 999 [9,10,15,5,13,3]
+ CRUSH rule 0 x 1000 [14,2,9,4,12,1]
+ CRUSH rule 0 x 1001 [11,14,4,2,6,9]
+ CRUSH rule 0 x 1002 [1,10,14,2,9,5]
+ CRUSH rule 0 x 1003 [10,7,5,14,2,1]
+ CRUSH rule 0 x 1004 [15,1,4,6,10,12]
+ CRUSH rule 0 x 1005 [6,12,2,10,9,15]
+ CRUSH rule 0 x 1006 [10,12,15,1,2,6]
+ CRUSH rule 0 x 1007 [1,7,13,14,3,4]
+ CRUSH rule 0 x 1008 [7,4,9,11,3,15]
+ CRUSH rule 0 x 1009 [5,2,11,7,15,9]
+ CRUSH rule 0 x 1010 [10,2,15,6,9,13]
+ CRUSH rule 0 x 1011 [6,3,12,1,10,4]
+ CRUSH rule 0 x 1012 [12,6,9,15,3,1]
+ CRUSH rule 0 x 1013 [2,14,12,4,9,1]
+ CRUSH rule 0 x 1014 [1,13,7,2,10,14]
+ CRUSH rule 0 x 1015 [12,6,10,1,4,15]
+ CRUSH rule 0 x 1016 [10,13,14,3,5,6]
+ CRUSH rule 0 x 1017 [5,11,14,7,13,9]
+ CRUSH rule 0 x 1018 [13,11,14,1,9,3]
+ CRUSH rule 0 x 1019 [10,13,14,7,5,1]
+ CRUSH rule 0 x 1020 [3,1,13,4,10,9]
+ CRUSH rule 0 x 1021 [2,11,14,9,4,6]
+ CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+ CRUSH rule 0 x 1023 [15,2,9,12,1,7]
+ rule 0 (replicated_ruleset) num_rep 6 result size == 6:\t1024/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15,12,1,4]
+ CRUSH rule 0 x 1 [10,15,1,2,13,4,7]
+ CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+ CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+ CRUSH rule 0 x 4 [14,2,10,1,9,4,7]
+ CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+ CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+ CRUSH rule 0 x 7 [9,2,6,12,11,4,1]
+ CRUSH rule 0 x 8 [10,2,15,1,4,13,6]
+ CRUSH rule 0 x 9 [7,1,14,2,11,9,12]
+ CRUSH rule 0 x 10 [10,14,4,1,2,7,13]
+ CRUSH rule 0 x 11 [13,9,14,7,5,11,2]
+ CRUSH rule 0 x 12 [7,1,2,5,13,15,11]
+ CRUSH rule 0 x 13 [3,5,12,7,9,1,14]
+ CRUSH rule 0 x 14 [13,5,2,7,10,15,1]
+ CRUSH rule 0 x 15 [15,1,9,6,13,3,5]
+ CRUSH rule 0 x 16 [7,11,14,2,13,1,9]
+ CRUSH rule 0 x 17 [10,1,13,2,4,6,14]
+ CRUSH rule 0 x 18 [1,7,3,10,5,12,9]
+ CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+ CRUSH rule 0 x 20 [14,12,3,10,9,4,7]
+ CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+ CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+ CRUSH rule 0 x 23 [10,5,13,9,3,15,1]
+ CRUSH rule 0 x 24 [12,11,3,1,9,4,7]
+ CRUSH rule 0 x 25 [7,12,15,1,3,10,4]
+ CRUSH rule 0 x 26 [1,7,13,2,14,5,9]
+ CRUSH rule 0 x 27 [3,6,15,4,13,9,11]
+ CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+ CRUSH rule 0 x 29 [5,14,12,11,6,3,1]
+ CRUSH rule 0 x 30 [2,5,6,9,1,11,13]
+ CRUSH rule 0 x 31 [5,15,10,1,9,13,6]
+ CRUSH rule 0 x 32 [9,10,2,1,13,14,6]
+ CRUSH rule 0 x 33 [13,4,9,2,7,1,10]
+ CRUSH rule 0 x 34 [13,15,2,4,1,10,9]
+ CRUSH rule 0 x 35 [4,14,3,13,10,9,1]
+ CRUSH rule 0 x 36 [3,12,9,7,5,10,14]
+ CRUSH rule 0 x 37 [9,2,6,14,11,1,4]
+ CRUSH rule 0 x 38 [3,4,13,10,9,1,14]
+ CRUSH rule 0 x 39 [12,7,14,11,1,9,5]
+ CRUSH rule 0 x 40 [10,1,9,5,15,2,6]
+ CRUSH rule 0 x 41 [4,9,11,1,14,13,6]
+ CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+ CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+ CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+ CRUSH rule 0 x 45 [11,12,15,9,1,5,6]
+ CRUSH rule 0 x 46 [6,9,2,14,11,13,1]
+ CRUSH rule 0 x 47 [3,9,6,4,13,1,11]
+ CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+ CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+ CRUSH rule 0 x 50 [14,12,1,4,2,11,6]
+ CRUSH rule 0 x 51 [10,6,5,12,15,2,1]
+ CRUSH rule 0 x 52 [12,1,9,11,7,3,14]
+ CRUSH rule 0 x 53 [3,6,13,9,5,1,11]
+ CRUSH rule 0 x 54 [4,13,9,2,14,10,6]
+ CRUSH rule 0 x 55 [4,11,2,7,1,13,9]
+ CRUSH rule 0 x 56 [5,9,10,1,3,13,14]
+ CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+ CRUSH rule 0 x 58 [7,1,11,4,3,14,12]
+ CRUSH rule 0 x 59 [2,13,1,10,9,5,14]
+ CRUSH rule 0 x 60 [3,6,11,1,4,9,12]
+ CRUSH rule 0 x 61 [3,15,13,7,4,1,10]
+ CRUSH rule 0 x 62 [15,11,7,12,5,9,2]
+ CRUSH rule 0 x 63 [10,14,12,1,7,3]
+ CRUSH rule 0 x 64 [3,9,1,4,7,12,11]
+ CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+ CRUSH rule 0 x 66 [15,11,6,9,4,1,3]
+ CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+ CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+ CRUSH rule 0 x 69 [2,1,15,10,4,9,13]
+ CRUSH rule 0 x 70 [9,6,1,3,13,15,11]
+ CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+ CRUSH rule 0 x 72 [9,10,3,5,7,12,15]
+ CRUSH rule 0 x 73 [5,3,11,1,7,12,15]
+ CRUSH rule 0 x 74 [11,7,9,5,1,15,3]
+ CRUSH rule 0 x 75 [9,7,11,14,12,1,2]
+ CRUSH rule 0 x 76 [6,1,3,5,14,10,12]
+ CRUSH rule 0 x 77 [7,4,2,13,9,1,11]
+ CRUSH rule 0 x 78 [9,3,1,5,6,13,14]
+ CRUSH rule 0 x 79 [13,2,15,5,7,9,11]
+ CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+ CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+ CRUSH rule 0 x 82 [14,13,5,11,6,2,1]
+ CRUSH rule 0 x 83 [4,15,3,9,10,13,6]
+ CRUSH rule 0 x 84 [10,7,9,15,3,4,1]
+ CRUSH rule 0 x 85 [3,15,9,7,4,11,1]
+ CRUSH rule 0 x 86 [10,9,14,1,13,4,2]
+ CRUSH rule 0 x 87 [15,10,7,12,5,3,9]
+ CRUSH rule 0 x 88 [4,13,3,1,9,15,11]
+ CRUSH rule 0 x 89 [3,9,7,4,1,14,10]
+ CRUSH rule 0 x 90 [4,9,7,12,11,14,2]
+ CRUSH rule 0 x 91 [6,11,9,1,2,4,14]
+ CRUSH rule 0 x 92 [1,5,10,9,13,15,6]
+ CRUSH rule 0 x 93 [9,3,15,13,7,5,1]
+ CRUSH rule 0 x 94 [9,2,12,5,6,11,1]
+ CRUSH rule 0 x 95 [7,15,4,10,9,13,2]
+ CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+ CRUSH rule 0 x 97 [4,11,2,13,1,7,9]
+ CRUSH rule 0 x 98 [11,13,9,3,15,1,5]
+ CRUSH rule 0 x 99 [12,4,11,7,3,14,9]
+ CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+ CRUSH rule 0 x 101 [15,7,1,9,10,5,2]
+ CRUSH rule 0 x 102 [3,11,14,6,13,4,9]
+ CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+ CRUSH rule 0 x 104 [14,6,3,5,9,1,10]
+ CRUSH rule 0 x 105 [14,10,1,9,3,5,6]
+ CRUSH rule 0 x 106 [6,5,13,2,14,11,1]
+ CRUSH rule 0 x 107 [3,1,10,14,13,5,9]
+ CRUSH rule 0 x 108 [5,10,7,2,15,9,12]
+ CRUSH rule 0 x 109 [9,1,13,7,15,5,11]
+ CRUSH rule 0 x 110 [5,1,11,3,7,14,13]
+ CRUSH rule 0 x 111 [10,1,9,7,5,2,13]
+ CRUSH rule 0 x 112 [1,10,4,14,2,12,6]
+ CRUSH rule 0 x 113 [6,10,13,9,1,5,2]
+ CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+ CRUSH rule 0 x 115 [10,13,14,3,9,1,6]
+ CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+ CRUSH rule 0 x 117 [5,6,1,12,15,9,11]
+ CRUSH rule 0 x 118 [10,4,13,15,9,3,1]
+ CRUSH rule 0 x 119 [14,12,11,4,6,9,3]
+ CRUSH rule 0 x 120 [11,3,14,13,4,7]
+ CRUSH rule 0 x 121 [9,5,1,11,7,3,15]
+ CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+ CRUSH rule 0 x 123 [3,10,5,6,9,1,12]
+ CRUSH rule 0 x 124 [12,2,1,5,14,7,11]
+ CRUSH rule 0 x 125 [9,12,15,1,6,5,3]
+ CRUSH rule 0 x 126 [7,15,10,9,2,12,5]
+ CRUSH rule 0 x 127 [4,14,9,13,1,3,7]
+ CRUSH rule 0 x 128 [3,12,1,10,4,9,7]
+ CRUSH rule 0 x 129 [11,13,14,2,9,4,6]
+ CRUSH rule 0 x 130 [3,13,5,14,10,1,9]
+ CRUSH rule 0 x 131 [12,1,6,15,4,2,11]
+ CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+ CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+ CRUSH rule 0 x 134 [12,5,6,15,3,9,10]
+ CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+ CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+ CRUSH rule 0 x 137 [14,3,6,11,1,9,4]
+ CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+ CRUSH rule 0 x 139 [11,2,13,9,1,15,7]
+ CRUSH rule 0 x 140 [11,4,12,15,2,6,9]
+ CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+ CRUSH rule 0 x 142 [3,14,7,9,11,1,4]
+ CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+ CRUSH rule 0 x 144 [13,7,11,2,14,4,1]
+ CRUSH rule 0 x 145 [12,2,6,10,9,4,14]
+ CRUSH rule 0 x 146 [1,5,9,2,6,13,14]
+ CRUSH rule 0 x 147 [1,4,9,11,2,7,15]
+ CRUSH rule 0 x 148 [12,7,9,2,14,11,1]
+ CRUSH rule 0 x 149 [2,5,9,12,11,1,7]
+ CRUSH rule 0 x 150 [1,15,2,10,7,9,5]
+ CRUSH rule 0 x 151 [2,9,14,7,1,10,5]
+ CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+ CRUSH rule 0 x 153 [6,9,4,15,2,1,10]
+ CRUSH rule 0 x 154 [3,11,7,1,4,12,15]
+ CRUSH rule 0 x 155 [14,12,7,3,5,1,9]
+ CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+ CRUSH rule 0 x 157 [15,1,6,4,3,10,9]
+ CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+ CRUSH rule 0 x 159 [4,14,3,12,10,6,1]
+ CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+ CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+ CRUSH rule 0 x 162 [10,6,1,12,2,4,14]
+ CRUSH rule 0 x 163 [15,1,10,2,6,4,13]
+ CRUSH rule 0 x 164 [9,14,10,7,12,2,5]
+ CRUSH rule 0 x 165 [11,7,2,13,9,15,1]
+ CRUSH rule 0 x 166 [1,2,12,14,4,11,7]
+ CRUSH rule 0 x 167 [9,7,3,4,11,13,15]
+ CRUSH rule 0 x 168 [13,2,4,1,6,15,10]
+ CRUSH rule 0 x 169 [1,4,9,14,13,10,2]
+ CRUSH rule 0 x 170 [1,15,7,9,12,10,3]
+ CRUSH rule 0 x 171 [9,2,10,7,1,5,15]
+ CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+ CRUSH rule 0 x 173 [5,10,12,15,6,1,2]
+ CRUSH rule 0 x 174 [15,6,4,12,1,11,9]
+ CRUSH rule 0 x 175 [5,7,9,3,10,1,14]
+ CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+ CRUSH rule 0 x 177 [2,9,10,13,4,1,14]
+ CRUSH rule 0 x 178 [12,11,7,14,3,4]
+ CRUSH rule 0 x 179 [2,10,13,9,5,1,7]
+ CRUSH rule 0 x 180 [3,11,5,15,7,12]
+ CRUSH rule 0 x 181 [9,12,6,5,1,10,14]
+ CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+ CRUSH rule 0 x 183 [5,7,10,13,3,9,14]
+ CRUSH rule 0 x 184 [2,5,11,12,7,1,9]
+ CRUSH rule 0 x 185 [13,5,7,11,2,14]
+ CRUSH rule 0 x 186 [6,14,13,5,10,1,3]
+ CRUSH rule 0 x 187 [1,4,11,13,6,14,9]
+ CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+ CRUSH rule 0 x 189 [6,12,4,9,2,1,11]
+ CRUSH rule 0 x 190 [9,13,15,10,3,1,5]
+ CRUSH rule 0 x 191 [7,11,4,1,15,12,9]
+ CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+ CRUSH rule 0 x 193 [3,13,6,10,4,1,9]
+ CRUSH rule 0 x 194 [3,13,4,14,6,9,1]
+ CRUSH rule 0 x 195 [5,7,10,12,1,3,15]
+ CRUSH rule 0 x 196 [4,15,1,10,9,2,13]
+ CRUSH rule 0 x 197 [14,10,13,4,6,3,1]
+ CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+ CRUSH rule 0 x 199 [2,10,4,15,1,9,6]
+ CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+ CRUSH rule 0 x 201 [9,14,1,7,4,3,10]
+ CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+ CRUSH rule 0 x 203 [12,5,7,15,1,2,10]
+ CRUSH rule 0 x 204 [6,11,3,12,14,1,9]
+ CRUSH rule 0 x 205 [15,4,6,10,13,9,2]
+ CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+ CRUSH rule 0 x 207 [2,11,7,4,14,1,12]
+ CRUSH rule 0 x 208 [13,1,6,14,9,11,3]
+ CRUSH rule 0 x 209 [6,15,13,1,11,4,9]
+ CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+ CRUSH rule 0 x 211 [2,14,1,13,11,7,9]
+ CRUSH rule 0 x 212 [10,1,12,15,5,6,2]
+ CRUSH rule 0 x 213 [3,9,6,5,15,13,1]
+ CRUSH rule 0 x 214 [7,15,4,1,10,2,13]
+ CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+ CRUSH rule 0 x 216 [12,9,6,2,1,11,5]
+ CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+ CRUSH rule 0 x 218 [12,10,15,6,1,4,9]
+ CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+ CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+ CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+ CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+ CRUSH rule 0 x 223 [9,7,11,1,4,14,13]
+ CRUSH rule 0 x 224 [1,7,10,2,12,9,14]
+ CRUSH rule 0 x 225 [10,5,2,6,1,13,9]
+ CRUSH rule 0 x 226 [4,1,9,3,13,10,15]
+ CRUSH rule 0 x 227 [7,2,12,15,5,11]
+ CRUSH rule 0 x 228 [2,15,11,1,6,13,9]
+ CRUSH rule 0 x 229 [9,3,7,14,1,12,4]
+ CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+ CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+ CRUSH rule 0 x 232 [10,5,13,1,9,2,7]
+ CRUSH rule 0 x 233 [6,12,11,4,9,14,1]
+ CRUSH rule 0 x 234 [10,1,2,12,5,9,15]
+ CRUSH rule 0 x 235 [13,14,7,10,1,9,5]
+ CRUSH rule 0 x 236 [2,15,9,12,1,7,4]
+ CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+ CRUSH rule 0 x 238 [2,10,4,15,6,12,9]
+ CRUSH rule 0 x 239 [4,15,10,7,9,13,3]
+ CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+ CRUSH rule 0 x 241 [7,9,15,12,1,5,2]
+ CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+ CRUSH rule 0 x 243 [2,11,5,1,15,6,9]
+ CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+ CRUSH rule 0 x 245 [12,9,15,3,1,5,10]
+ CRUSH rule 0 x 246 [15,3,5,11,7,1,12]
+ CRUSH rule 0 x 247 [6,4,9,12,1,2,10]
+ CRUSH rule 0 x 248 [5,13,7,11,9,15,3]
+ CRUSH rule 0 x 249 [10,14,7,3,9,13,1]
+ CRUSH rule 0 x 250 [12,15,1,10,5,6,3]
+ CRUSH rule 0 x 251 [13,2,15,5,6,1,9]
+ CRUSH rule 0 x 252 [7,5,13,9,3,10,14]
+ CRUSH rule 0 x 253 [3,13,15,10,7,4]
+ CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+ CRUSH rule 0 x 255 [1,9,13,2,6,10,4]
+ CRUSH rule 0 x 256 [6,9,13,1,3,14,5]
+ CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+ CRUSH rule 0 x 258 [12,5,6,10,2,1,14]
+ CRUSH rule 0 x 259 [9,10,4,3,14,13,1]
+ CRUSH rule 0 x 260 [10,12,6,9,3,15,1]
+ CRUSH rule 0 x 261 [13,7,2,1,15,5,11]
+ CRUSH rule 0 x 262 [15,3,12,7,4,9,1]
+ CRUSH rule 0 x 263 [12,6,10,9,5,15,3]
+ CRUSH rule 0 x 264 [13,14,11,3,1,4,7]
+ CRUSH rule 0 x 265 [12,10,14,5,7,1,9]
+ CRUSH rule 0 x 266 [14,7,11,1,2,9,4]
+ CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+ CRUSH rule 0 x 268 [4,1,15,12,6,11,3]
+ CRUSH rule 0 x 269 [11,1,15,5,13,9,7]
+ CRUSH rule 0 x 270 [7,11,12,3,1,14,9]
+ CRUSH rule 0 x 271 [4,7,3,13,15,10,9]
+ CRUSH rule 0 x 272 [15,5,13,10,6,2]
+ CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+ CRUSH rule 0 x 274 [10,2,5,6,13,9,15]
+ CRUSH rule 0 x 275 [10,3,4,7,14,13]
+ CRUSH rule 0 x 276 [5,12,9,2,11,7,15]
+ CRUSH rule 0 x 277 [14,3,13,4,1,9,11]
+ CRUSH rule 0 x 278 [5,6,14,3,1,11,13]
+ CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+ CRUSH rule 0 x 280 [7,3,14,9,1,11,4]
+ CRUSH rule 0 x 281 [5,11,14,7,9,13,2]
+ CRUSH rule 0 x 282 [2,1,13,14,9,7,5]
+ CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+ CRUSH rule 0 x 284 [5,11,7,15,3,13,1]
+ CRUSH rule 0 x 285 [15,5,3,1,6,13,11]
+ CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+ CRUSH rule 0 x 287 [12,4,9,1,3,11,15]
+ CRUSH rule 0 x 288 [4,12,10,7,1,3,14]
+ CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+ CRUSH rule 0 x 290 [12,2,5,6,15,9,1]
+ CRUSH rule 0 x 291 [7,11,1,14,5,9,2]
+ CRUSH rule 0 x 292 [4,10,6,3,14,9,12]
+ CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+ CRUSH rule 0 x 294 [9,12,3,14,6,11,5]
+ CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+ CRUSH rule 0 x 296 [3,1,13,7,14,9,10]
+ CRUSH rule 0 x 297 [6,13,4,14,10,1,2]
+ CRUSH rule 0 x 298 [14,9,13,1,4,2,7]
+ CRUSH rule 0 x 299 [14,12,11,6,4,2,1]
+ CRUSH rule 0 x 300 [15,7,10,5,1,3,13]
+ CRUSH rule 0 x 301 [9,11,7,1,13,14,4]
+ CRUSH rule 0 x 302 [9,7,1,13,5,10,3]
+ CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+ CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+ CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+ CRUSH rule 0 x 306 [10,12,4,6,9,2,15]
+ CRUSH rule 0 x 307 [11,12,15,5,6,2,1]
+ CRUSH rule 0 x 308 [12,14,10,9,1,2,5]
+ CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+ CRUSH rule 0 x 310 [3,1,5,10,14,9,7]
+ CRUSH rule 0 x 311 [3,9,7,1,14,13,10]
+ CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+ CRUSH rule 0 x 313 [9,15,3,7,5,13,1]
+ CRUSH rule 0 x 314 [2,15,9,5,6,12,1]
+ CRUSH rule 0 x 315 [15,2,13,1,11,9,6]
+ CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+ CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+ CRUSH rule 0 x 318 [4,1,15,11,9,13,6]
+ CRUSH rule 0 x 319 [2,15,4,1,11,9,7]
+ CRUSH rule 0 x 320 [5,7,13,9,11,2,1]
+ CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+ CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+ CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+ CRUSH rule 0 x 324 [5,6,10,15,2,13]
+ CRUSH rule 0 x 325 [9,10,14,5,1,6,2]
+ CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+ CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+ CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+ CRUSH rule 0 x 329 [2,6,15,5,9,10,13]
+ CRUSH rule 0 x 330 [3,9,11,13,1,6,5]
+ CRUSH rule 0 x 331 [12,14,6,3,1,4,10]
+ CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+ CRUSH rule 0 x 333 [6,5,3,12,14,10,9]
+ CRUSH rule 0 x 334 [4,9,2,12,7,11,15]
+ CRUSH rule 0 x 335 [11,7,1,5,13,2,9]
+ CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+ CRUSH rule 0 x 337 [15,11,3,7,12,5]
+ CRUSH rule 0 x 338 [10,5,3,6,15,1,9]
+ CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+ CRUSH rule 0 x 340 [11,6,12,4,9,3,14]
+ CRUSH rule 0 x 341 [7,5,2,10,14,9,1]
+ CRUSH rule 0 x 342 [12,14,1,9,2,11,4]
+ CRUSH rule 0 x 343 [12,14,9,6,10,2,4]
+ CRUSH rule 0 x 344 [9,11,5,2,14,13,1]
+ CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+ CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+ CRUSH rule 0 x 347 [10,2,12,6,9,1,14]
+ CRUSH rule 0 x 348 [7,9,10,1,14,13,3]
+ CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+ CRUSH rule 0 x 350 [13,9,15,4,10,7,2]
+ CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+ CRUSH rule 0 x 352 [1,12,11,9,4,7,3]
+ CRUSH rule 0 x 353 [10,14,12,2,9,1,4]
+ CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+ CRUSH rule 0 x 355 [13,14,6,10,2,5,1]
+ CRUSH rule 0 x 356 [15,13,2,9,6,5,1]
+ CRUSH rule 0 x 357 [4,11,1,13,3,14,6]
+ CRUSH rule 0 x 358 [12,7,2,9,1,14,10]
+ CRUSH rule 0 x 359 [5,15,7,11,3,13]
+ CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+ CRUSH rule 0 x 361 [5,3,13,6,1,14,11]
+ CRUSH rule 0 x 362 [2,9,11,13,1,6,5]
+ CRUSH rule 0 x 363 [7,12,3,9,15,4,1]
+ CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+ CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+ CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+ CRUSH rule 0 x 367 [7,13,3,1,5,11,15]
+ CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+ CRUSH rule 0 x 369 [7,5,3,13,14,9,11]
+ CRUSH rule 0 x 370 [4,7,14,1,2,9,11]
+ CRUSH rule 0 x 371 [1,7,12,3,4,15,10]
+ CRUSH rule 0 x 372 [10,4,3,14,6,1,12]
+ CRUSH rule 0 x 373 [15,5,2,6,13,1,9]
+ CRUSH rule 0 x 374 [3,15,12,5,1,6,10]
+ CRUSH rule 0 x 375 [5,2,14,1,6,13,11]
+ CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+ CRUSH rule 0 x 377 [1,15,2,4,9,11,12]
+ CRUSH rule 0 x 378 [9,12,2,15,1,5,11]
+ CRUSH rule 0 x 379 [11,2,15,5,7,9,13]
+ CRUSH rule 0 x 380 [6,1,12,11,2,9,5]
+ CRUSH rule 0 x 381 [15,13,7,5,10,2,1]
+ CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+ CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+ CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+ CRUSH rule 0 x 385 [4,6,15,3,10,9,1]
+ CRUSH rule 0 x 386 [14,3,11,13,5,6,9]
+ CRUSH rule 0 x 387 [1,11,5,7,9,2,14]
+ CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+ CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+ CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+ CRUSH rule 0 x 391 [3,4,9,13,7,10,1]
+ CRUSH rule 0 x 392 [11,5,14,7,1,9,2]
+ CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+ CRUSH rule 0 x 394 [4,9,3,15,13,6,1]
+ CRUSH rule 0 x 395 [10,13,5,15,6,9,3]
+ CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+ CRUSH rule 0 x 397 [1,14,9,4,12,10,3]
+ CRUSH rule 0 x 398 [9,2,1,5,12,6,11]
+ CRUSH rule 0 x 399 [5,9,14,3,1,10,13]
+ CRUSH rule 0 x 400 [10,6,2,4,15,12,1]
+ CRUSH rule 0 x 401 [6,9,11,12,4,3,15]
+ CRUSH rule 0 x 402 [4,7,9,2,13,1,15]
+ CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+ CRUSH rule 0 x 404 [14,12,7,9,2,1,5]
+ CRUSH rule 0 x 405 [9,15,11,2,4,7,13]
+ CRUSH rule 0 x 406 [12,14,9,2,7,10,4]
+ CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+ CRUSH rule 0 x 408 [7,1,5,2,10,15,13]
+ CRUSH rule 0 x 409 [11,2,4,13,1,15,7]
+ CRUSH rule 0 x 410 [6,4,14,2,12,9,10]
+ CRUSH rule 0 x 411 [13,11,15,6,4,1,9]
+ CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+ CRUSH rule 0 x 413 [13,5,3,11,6,9,1]
+ CRUSH rule 0 x 414 [3,11,9,13,4,1,6]
+ CRUSH rule 0 x 415 [6,10,14,5,1,13,3]
+ CRUSH rule 0 x 416 [13,1,4,7,2,9,14]
+ CRUSH rule 0 x 417 [4,12,1,15,2,11,9]
+ CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+ CRUSH rule 0 x 419 [5,14,10,9,2,12,6]
+ CRUSH rule 0 x 420 [2,4,9,11,6,14,13]
+ CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+ CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+ CRUSH rule 0 x 423 [3,15,12,6,5,1,9]
+ CRUSH rule 0 x 424 [6,10,12,2,5,1,14]
+ CRUSH rule 0 x 425 [11,15,2,13,5,7,9]
+ CRUSH rule 0 x 426 [12,4,7,1,9,10,14]
+ CRUSH rule 0 x 427 [14,10,3,1,9,7,5]
+ CRUSH rule 0 x 428 [12,7,9,4,2,1,14]
+ CRUSH rule 0 x 429 [3,4,9,7,11,12,1]
+ CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+ CRUSH rule 0 x 431 [9,3,7,1,12,5,14]
+ CRUSH rule 0 x 432 [4,1,12,7,15,2,10]
+ CRUSH rule 0 x 433 [4,11,12,15,7,3]
+ CRUSH rule 0 x 434 [2,14,9,1,5,11,7]
+ CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+ CRUSH rule 0 x 436 [9,15,10,2,4,1,12]
+ CRUSH rule 0 x 437 [9,6,3,14,10,12,5]
+ CRUSH rule 0 x 438 [7,2,13,4,11,1,9]
+ CRUSH rule 0 x 439 [7,14,4,3,12,10]
+ CRUSH rule 0 x 440 [14,11,9,2,7,12,1]
+ CRUSH rule 0 x 441 [2,4,11,9,13,6,1]
+ CRUSH rule 0 x 442 [10,13,9,7,15,1,4]
+ CRUSH rule 0 x 443 [12,15,10,9,2,1,6]
+ CRUSH rule 0 x 444 [4,13,7,14,3,1,9]
+ CRUSH rule 0 x 445 [4,2,15,7,1,9,11]
+ CRUSH rule 0 x 446 [12,10,6,9,4,1,15]
+ CRUSH rule 0 x 447 [15,7,13,1,4,9,3]
+ CRUSH rule 0 x 448 [5,2,13,7,15,10]
+ CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+ CRUSH rule 0 x 450 [2,4,6,9,15,1,13]
+ CRUSH rule 0 x 451 [6,14,11,3,9,1,12]
+ CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+ CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+ CRUSH rule 0 x 454 [10,4,2,6,15,12,9]
+ CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+ CRUSH rule 0 x 456 [5,7,13,1,11,3,9]
+ CRUSH rule 0 x 457 [9,1,5,7,11,13,15]
+ CRUSH rule 0 x 458 [9,11,15,4,7,2,12]
+ CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+ CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+ CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+ CRUSH rule 0 x 462 [4,7,12,14,11,1,3]
+ CRUSH rule 0 x 463 [4,12,14,11,2,7,1]
+ CRUSH rule 0 x 464 [4,2,15,10,1,9,13]
+ CRUSH rule 0 x 465 [5,10,9,7,13,1,3]
+ CRUSH rule 0 x 466 [13,5,2,15,9,11,6]
+ CRUSH rule 0 x 467 [13,6,14,3,9,1,11]
+ CRUSH rule 0 x 468 [10,7,12,14,4,1,9]
+ CRUSH rule 0 x 469 [4,9,6,14,12,11,3]
+ CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+ CRUSH rule 0 x 471 [6,1,5,14,13,10,9]
+ CRUSH rule 0 x 472 [2,14,7,5,13,1,11]
+ CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+ CRUSH rule 0 x 474 [15,10,4,12,6,9,2]
+ CRUSH rule 0 x 475 [10,5,12,9,14,3,6]
+ CRUSH rule 0 x 476 [3,6,10,12,1,15,9]
+ CRUSH rule 0 x 477 [6,13,5,15,11,9,2]
+ CRUSH rule 0 x 478 [4,15,1,3,7,12,10]
+ CRUSH rule 0 x 479 [13,11,1,6,14,5,9]
+ CRUSH rule 0 x 480 [1,13,6,4,9,14,11]
+ CRUSH rule 0 x 481 [15,12,7,9,1,3,10]
+ CRUSH rule 0 x 482 [2,12,9,1,7,11,14]
+ CRUSH rule 0 x 483 [10,1,4,15,9,7,13]
+ CRUSH rule 0 x 484 [1,4,10,13,7,14,2]
+ CRUSH rule 0 x 485 [9,4,3,1,14,12,7]
+ CRUSH rule 0 x 486 [3,10,15,9,7,13,4]
+ CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+ CRUSH rule 0 x 488 [14,4,1,9,2,6,10]
+ CRUSH rule 0 x 489 [11,4,2,13,15,7]
+ CRUSH rule 0 x 490 [4,9,1,3,13,15,6]
+ CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+ CRUSH rule 0 x 492 [5,7,11,3,14,9,1]
+ CRUSH rule 0 x 493 [12,1,4,15,3,11,9]
+ CRUSH rule 0 x 494 [1,7,13,4,15,9,10]
+ CRUSH rule 0 x 495 [3,15,7,1,9,5,12]
+ CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+ CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+ CRUSH rule 0 x 498 [10,6,1,5,9,12,3]
+ CRUSH rule 0 x 499 [14,3,12,5,1,11,9]
+ CRUSH rule 0 x 500 [15,9,6,12,11,2,1]
+ CRUSH rule 0 x 501 [10,13,1,9,3,14,5]
+ CRUSH rule 0 x 502 [5,1,14,11,7,12,9]
+ CRUSH rule 0 x 503 [15,10,7,9,1,12,4]
+ CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+ CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+ CRUSH rule 0 x 506 [11,7,9,14,12,1,2]
+ CRUSH rule 0 x 507 [4,14,13,3,9,7,1]
+ CRUSH rule 0 x 508 [12,1,4,9,2,11,15]
+ CRUSH rule 0 x 509 [4,2,6,9,14,1,10]
+ CRUSH rule 0 x 510 [5,3,1,12,11,14,9]
+ CRUSH rule 0 x 511 [2,12,10,6,14,5]
+ CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+ CRUSH rule 0 x 513 [4,9,11,3,13,7,1]
+ CRUSH rule 0 x 514 [11,9,3,4,12,15,6]
+ CRUSH rule 0 x 515 [12,14,6,5,3,9,1]
+ CRUSH rule 0 x 516 [14,11,1,12,3,7,4]
+ CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+ CRUSH rule 0 x 518 [3,5,7,12,15,11,9]
+ CRUSH rule 0 x 519 [12,14,2,1,4,6,9]
+ CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+ CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+ CRUSH rule 0 x 522 [4,12,11,1,15,3,9]
+ CRUSH rule 0 x 523 [3,1,5,9,15,10,13]
+ CRUSH rule 0 x 524 [15,9,3,11,13,7,4]
+ CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+ CRUSH rule 0 x 526 [10,2,5,13,6,15,1]
+ CRUSH rule 0 x 527 [3,13,4,1,9,10,14]
+ CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+ CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+ CRUSH rule 0 x 530 [11,9,12,7,5,1,3]
+ CRUSH rule 0 x 531 [9,15,4,7,2,13,1]
+ CRUSH rule 0 x 532 [5,3,13,7,9,14,1]
+ CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+ CRUSH rule 0 x 534 [11,9,3,7,15,4,1]
+ CRUSH rule 0 x 535 [11,1,3,5,14,9,12]
+ CRUSH rule 0 x 536 [9,1,14,13,4,6,2]
+ CRUSH rule 0 x 537 [15,5,13,2,7,11]
+ CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+ CRUSH rule 0 x 539 [10,12,6,14,1,2,9]
+ CRUSH rule 0 x 540 [12,15,7,3,9,11,1]
+ CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+ CRUSH rule 0 x 542 [3,9,15,5,11,12,7]
+ CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+ CRUSH rule 0 x 544 [3,15,9,11,7,4,12]
+ CRUSH rule 0 x 545 [14,10,7,12,4,9,1]
+ CRUSH rule 0 x 546 [5,15,13,7,1,10,9]
+ CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+ CRUSH rule 0 x 548 [11,7,12,15,4,2]
+ CRUSH rule 0 x 549 [14,1,4,9,13,6,3]
+ CRUSH rule 0 x 550 [9,15,3,13,1,6,4]
+ CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+ CRUSH rule 0 x 552 [2,11,14,1,9,6,5]
+ CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+ CRUSH rule 0 x 554 [11,14,6,4,13,9,3]
+ CRUSH rule 0 x 555 [6,5,10,9,14,2,13]
+ CRUSH rule 0 x 556 [15,6,3,13,11,4,1]
+ CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+ CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+ CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+ CRUSH rule 0 x 560 [4,9,12,6,3,10,1]
+ CRUSH rule 0 x 561 [12,7,1,2,5,15,11]
+ CRUSH rule 0 x 562 [7,13,9,14,2,1,11]
+ CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+ CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+ CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+ CRUSH rule 0 x 566 [6,14,4,2,13,11]
+ CRUSH rule 0 x 567 [15,4,11,6,3,12]
+ CRUSH rule 0 x 568 [4,14,1,6,10,13,3]
+ CRUSH rule 0 x 569 [11,3,15,13,5,1,9]
+ CRUSH rule 0 x 570 [1,10,13,4,7,2,9]
+ CRUSH rule 0 x 571 [10,12,14,9,4,2,1]
+ CRUSH rule 0 x 572 [12,14,3,10,6,1,4]
+ CRUSH rule 0 x 573 [7,15,11,2,12,9,4]
+ CRUSH rule 0 x 574 [11,14,13,1,3,7,4]
+ CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+ CRUSH rule 0 x 576 [3,15,11,9,1,6,5]
+ CRUSH rule 0 x 577 [13,9,6,15,3,11,4]
+ CRUSH rule 0 x 578 [4,10,1,2,7,13,14]
+ CRUSH rule 0 x 579 [13,1,15,2,10,7,5]
+ CRUSH rule 0 x 580 [3,12,4,1,10,15,7]
+ CRUSH rule 0 x 581 [7,14,12,10,1,2,9]
+ CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+ CRUSH rule 0 x 583 [4,15,1,9,10,12,2]
+ CRUSH rule 0 x 584 [10,1,5,13,6,9,2]
+ CRUSH rule 0 x 585 [5,3,6,1,11,14,13]
+ CRUSH rule 0 x 586 [7,10,14,12,9,3,5]
+ CRUSH rule 0 x 587 [11,6,9,4,1,14,13]
+ CRUSH rule 0 x 588 [3,12,7,15,4,9,1]
+ CRUSH rule 0 x 589 [9,7,12,1,10,3,4]
+ CRUSH rule 0 x 590 [12,1,3,9,10,6,4]
+ CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+ CRUSH rule 0 x 592 [15,12,9,7,5,2,11]
+ CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+ CRUSH rule 0 x 594 [12,14,2,9,7,4,1]
+ CRUSH rule 0 x 595 [12,7,10,3,1,14,9]
+ CRUSH rule 0 x 596 [2,7,12,11,1,5,15]
+ CRUSH rule 0 x 597 [15,1,2,10,7,13,5]
+ CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+ CRUSH rule 0 x 599 [13,11,1,5,6,2,15]
+ CRUSH rule 0 x 600 [4,12,3,10,9,7,1]
+ CRUSH rule 0 x 601 [13,5,15,2,1,7,9]
+ CRUSH rule 0 x 602 [3,11,7,1,13,15,5]
+ CRUSH rule 0 x 603 [3,1,4,14,10,9,6]
+ CRUSH rule 0 x 604 [14,2,6,1,11,13,9]
+ CRUSH rule 0 x 605 [2,7,12,5,14,10,1]
+ CRUSH rule 0 x 606 [12,15,1,5,7,9,3]
+ CRUSH rule 0 x 607 [3,9,10,14,7,1,4]
+ CRUSH rule 0 x 608 [13,10,1,7,9,15,5]
+ CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+ CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+ CRUSH rule 0 x 611 [13,1,5,3,10,7,15]
+ CRUSH rule 0 x 612 [7,1,2,13,9,15,4]
+ CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+ CRUSH rule 0 x 614 [9,4,15,3,1,11,6]
+ CRUSH rule 0 x 615 [9,4,11,2,1,12,6]
+ CRUSH rule 0 x 616 [10,14,1,5,3,6,12]
+ CRUSH rule 0 x 617 [15,7,2,11,12,1,9]
+ CRUSH rule 0 x 618 [4,2,10,6,14,9,1]
+ CRUSH rule 0 x 619 [15,4,3,9,6,1,13]
+ CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+ CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+ CRUSH rule 0 x 622 [10,2,13,5,15,9,1]
+ CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+ CRUSH rule 0 x 624 [3,9,15,6,10,1,5]
+ CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+ CRUSH rule 0 x 626 [10,12,2,1,9,7,5]
+ CRUSH rule 0 x 627 [1,12,10,14,3,5,9]
+ CRUSH rule 0 x 628 [15,13,11,4,2,1,7]
+ CRUSH rule 0 x 629 [5,6,15,12,1,10,3]
+ CRUSH rule 0 x 630 [1,4,12,9,3,7,15]
+ CRUSH rule 0 x 631 [5,7,1,15,12,11,3]
+ CRUSH rule 0 x 632 [12,3,11,9,6,1,15]
+ CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+ CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+ CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+ CRUSH rule 0 x 636 [13,6,11,3,15,9,1]
+ CRUSH rule 0 x 637 [3,1,10,6,9,12,4]
+ CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+ CRUSH rule 0 x 639 [6,9,14,4,3,1,10]
+ CRUSH rule 0 x 640 [9,6,1,11,14,2,4]
+ CRUSH rule 0 x 641 [10,6,5,14,1,9,12]
+ CRUSH rule 0 x 642 [1,15,4,6,2,10,9]
+ CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+ CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+ CRUSH rule 0 x 645 [14,2,4,9,10,1,7]
+ CRUSH rule 0 x 646 [5,13,14,1,6,9,2]
+ CRUSH rule 0 x 647 [10,1,9,13,6,2,14]
+ CRUSH rule 0 x 648 [6,5,2,14,11,1,12]
+ CRUSH rule 0 x 649 [3,9,13,11,4,14,1]
+ CRUSH rule 0 x 650 [10,9,4,15,12,7,1]
+ CRUSH rule 0 x 651 [3,9,5,7,14,1,13]
+ CRUSH rule 0 x 652 [15,9,4,6,13,1,2]
+ CRUSH rule 0 x 653 [11,14,1,3,6,9,12]
+ CRUSH rule 0 x 654 [13,6,2,10,15,4,1]
+ CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+ CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+ CRUSH rule 0 x 657 [11,15,3,5,7,13,1]
+ CRUSH rule 0 x 658 [7,2,10,12,1,4,9]
+ CRUSH rule 0 x 659 [2,5,14,6,10,12]
+ CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+ CRUSH rule 0 x 661 [7,15,3,12,11,4,9]
+ CRUSH rule 0 x 662 [15,2,12,5,1,10,9]
+ CRUSH rule 0 x 663 [14,9,13,10,5,3,1]
+ CRUSH rule 0 x 664 [6,10,12,4,9,2,1]
+ CRUSH rule 0 x 665 [2,9,12,1,7,10,4]
+ CRUSH rule 0 x 666 [12,3,6,1,15,9,10]
+ CRUSH rule 0 x 667 [1,9,12,10,2,14,7]
+ CRUSH rule 0 x 668 [9,5,1,2,6,11,13]
+ CRUSH rule 0 x 669 [9,7,14,5,11,13,1]
+ CRUSH rule 0 x 670 [6,10,9,13,1,2,15]
+ CRUSH rule 0 x 671 [6,15,5,10,13,3]
+ CRUSH rule 0 x 672 [2,9,13,1,4,14,6]
+ CRUSH rule 0 x 673 [7,10,5,9,15,13,2]
+ CRUSH rule 0 x 674 [7,12,10,1,14,9,3]
+ CRUSH rule 0 x 675 [9,5,1,10,6,14,12]
+ CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+ CRUSH rule 0 x 677 [2,12,1,4,10,6,15]
+ CRUSH rule 0 x 678 [1,2,4,10,12,14,9]
+ CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+ CRUSH rule 0 x 680 [7,11,3,1,15,4,9]
+ CRUSH rule 0 x 681 [6,4,3,11,14,13,1]
+ CRUSH rule 0 x 682 [6,1,11,15,12,2,5]
+ CRUSH rule 0 x 683 [6,13,2,4,9,14,10]
+ CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+ CRUSH rule 0 x 685 [5,1,15,7,9,2,10]
+ CRUSH rule 0 x 686 [1,9,11,14,6,13,4]
+ CRUSH rule 0 x 687 [7,13,3,5,11,9,15]
+ CRUSH rule 0 x 688 [11,9,1,14,3,5,7]
+ CRUSH rule 0 x 689 [5,2,9,12,1,14,11]
+ CRUSH rule 0 x 690 [9,7,10,3,13,15,5]
+ CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+ CRUSH rule 0 x 692 [15,5,1,2,9,11,12]
+ CRUSH rule 0 x 693 [5,6,12,15,2,10,9]
+ CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+ CRUSH rule 0 x 695 [6,13,14,10,9,5,1]
+ CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+ CRUSH rule 0 x 697 [13,11,3,6,4,14,9]
+ CRUSH rule 0 x 698 [11,13,4,2,6,1,9]
+ CRUSH rule 0 x 699 [7,14,12,4,2,11]
+ CRUSH rule 0 x 700 [12,14,11,9,4,6,3]
+ CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+ CRUSH rule 0 x 702 [3,12,15,6,5,11,1]
+ CRUSH rule 0 x 703 [15,11,13,3,4,7,1]
+ CRUSH rule 0 x 704 [6,4,2,15,11,1,13]
+ CRUSH rule 0 x 705 [14,6,11,5,1,13,9]
+ CRUSH rule 0 x 706 [1,12,3,6,4,10,15]
+ CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+ CRUSH rule 0 x 708 [3,10,5,1,15,9,7]
+ CRUSH rule 0 x 709 [11,12,3,7,5,14,1]
+ CRUSH rule 0 x 710 [14,2,11,9,5,7,12]
+ CRUSH rule 0 x 711 [14,3,9,10,12,5,6]
+ CRUSH rule 0 x 712 [12,3,11,15,9,1,6]
+ CRUSH rule 0 x 713 [11,9,3,15,13,6,4]
+ CRUSH rule 0 x 714 [12,1,9,7,2,15,10]
+ CRUSH rule 0 x 715 [6,1,14,4,11,12,3]
+ CRUSH rule 0 x 716 [11,13,9,14,5,2,1]
+ CRUSH rule 0 x 717 [12,4,10,9,15,1,2]
+ CRUSH rule 0 x 718 [7,15,5,2,11,13]
+ CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+ CRUSH rule 0 x 720 [4,13,10,2,7,9,1]
+ CRUSH rule 0 x 721 [11,3,14,9,1,12,4]
+ CRUSH rule 0 x 722 [2,4,6,1,9,15,13]
+ CRUSH rule 0 x 723 [2,1,12,15,11,7,5]
+ CRUSH rule 0 x 724 [7,1,9,10,5,15,13]
+ CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+ CRUSH rule 0 x 726 [7,14,4,3,11,13,9]
+ CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+ CRUSH rule 0 x 728 [13,11,4,6,15,2]
+ CRUSH rule 0 x 729 [15,11,4,6,2,9,1]
+ CRUSH rule 0 x 730 [3,7,1,13,11,15,9]
+ CRUSH rule 0 x 731 [9,1,6,5,2,11,13]
+ CRUSH rule 0 x 732 [1,2,10,13,9,4,7]
+ CRUSH rule 0 x 733 [11,3,5,6,1,9,12]
+ CRUSH rule 0 x 734 [14,3,11,7,12,9,4]
+ CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+ CRUSH rule 0 x 736 [3,9,1,11,7,5,13]
+ CRUSH rule 0 x 737 [1,4,2,12,9,10,6]
+ CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+ CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+ CRUSH rule 0 x 740 [7,9,10,13,1,15,2]
+ CRUSH rule 0 x 741 [12,11,7,15,2,5]
+ CRUSH rule 0 x 742 [9,7,4,11,12,1,14]
+ CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+ CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+ CRUSH rule 0 x 745 [3,6,1,4,11,12,14]
+ CRUSH rule 0 x 746 [3,7,9,10,14,5,1]
+ CRUSH rule 0 x 747 [15,11,5,2,13,9,1]
+ CRUSH rule 0 x 748 [6,10,13,2,14,5,9]
+ CRUSH rule 0 x 749 [14,9,10,7,5,1,2]
+ CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+ CRUSH rule 0 x 751 [15,1,6,9,5,11,12]
+ CRUSH rule 0 x 752 [13,1,7,3,11,15,9]
+ CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+ CRUSH rule 0 x 754 [14,12,11,4,2,1,9]
+ CRUSH rule 0 x 755 [13,6,1,10,4,2,14]
+ CRUSH rule 0 x 756 [3,4,14,6,1,10,13]
+ CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+ CRUSH rule 0 x 758 [6,3,4,10,15,13,9]
+ CRUSH rule 0 x 759 [5,7,3,14,11,1,9]
+ CRUSH rule 0 x 760 [1,15,10,12,4,3,9]
+ CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+ CRUSH rule 0 x 762 [1,4,10,9,3,7,14]
+ CRUSH rule 0 x 763 [4,13,1,14,7,10,2]
+ CRUSH rule 0 x 764 [1,14,6,13,9,5,2]
+ CRUSH rule 0 x 765 [9,15,2,13,4,1,11]
+ CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+ CRUSH rule 0 x 767 [6,11,4,3,12,14]
+ CRUSH rule 0 x 768 [2,12,15,7,1,11,9]
+ CRUSH rule 0 x 769 [15,1,9,2,11,12,7]
+ CRUSH rule 0 x 770 [15,13,4,6,3,10,1]
+ CRUSH rule 0 x 771 [9,2,12,11,6,14,5]
+ CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+ CRUSH rule 0 x 773 [3,7,4,15,1,12,11]
+ CRUSH rule 0 x 774 [12,6,3,15,5,9,10]
+ CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+ CRUSH rule 0 x 776 [10,15,3,9,6,13,1]
+ CRUSH rule 0 x 777 [11,13,4,7,1,14,9]
+ CRUSH rule 0 x 778 [13,1,9,11,15,6,3]
+ CRUSH rule 0 x 779 [5,11,1,14,2,9,13]
+ CRUSH rule 0 x 780 [13,9,3,6,4,1,14]
+ CRUSH rule 0 x 781 [5,7,14,3,1,12,11]
+ CRUSH rule 0 x 782 [2,15,9,7,11,13,4]
+ CRUSH rule 0 x 783 [12,7,5,14,9,1,2]
+ CRUSH rule 0 x 784 [14,1,10,13,3,4,7]
+ CRUSH rule 0 x 785 [6,12,1,2,4,9,15]
+ CRUSH rule 0 x 786 [10,5,2,15,1,7,12]
+ CRUSH rule 0 x 787 [1,12,10,2,9,4,14]
+ CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+ CRUSH rule 0 x 789 [9,2,14,7,4,12,1]
+ CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+ CRUSH rule 0 x 791 [9,4,7,13,14,11,1]
+ CRUSH rule 0 x 792 [6,4,15,10,12,3]
+ CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+ CRUSH rule 0 x 794 [5,12,2,14,9,10,1]
+ CRUSH rule 0 x 795 [6,14,12,4,10,1,2]
+ CRUSH rule 0 x 796 [11,2,12,6,15,4]
+ CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+ CRUSH rule 0 x 798 [5,11,6,13,1,3,15]
+ CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+ CRUSH rule 0 x 800 [6,3,4,11,15,13]
+ CRUSH rule 0 x 801 [2,5,6,13,9,1,10]
+ CRUSH rule 0 x 802 [1,4,12,7,3,9,10]
+ CRUSH rule 0 x 803 [7,2,4,1,11,13,9]
+ CRUSH rule 0 x 804 [5,14,9,7,3,1,12]
+ CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+ CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+ CRUSH rule 0 x 807 [14,2,7,4,9,12,1]
+ CRUSH rule 0 x 808 [2,15,12,7,9,1,5]
+ CRUSH rule 0 x 809 [1,11,7,12,4,2,15]
+ CRUSH rule 0 x 810 [2,5,9,12,15,1,7]
+ CRUSH rule 0 x 811 [15,6,3,10,1,5,9]
+ CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+ CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+ CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+ CRUSH rule 0 x 815 [15,12,9,4,10,6,1]
+ CRUSH rule 0 x 816 [14,10,13,7,3,9,4]
+ CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+ CRUSH rule 0 x 818 [15,2,11,4,1,12,6]
+ CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+ CRUSH rule 0 x 820 [3,6,9,12,11,15,4]
+ CRUSH rule 0 x 821 [15,10,9,13,3,4,7]
+ CRUSH rule 0 x 822 [10,13,2,9,7,4,14]
+ CRUSH rule 0 x 823 [2,6,12,10,15,4,1]
+ CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+ CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+ CRUSH rule 0 x 826 [5,2,11,15,1,12,9]
+ CRUSH rule 0 x 827 [13,5,1,3,7,9,11]
+ CRUSH rule 0 x 828 [12,6,10,5,1,9,2]
+ CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+ CRUSH rule 0 x 830 [15,13,2,9,7,11,1]
+ CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+ CRUSH rule 0 x 832 [14,11,13,2,9,4,6]
+ CRUSH rule 0 x 833 [9,13,3,11,7,5,15]
+ CRUSH rule 0 x 834 [9,7,5,1,11,2,13]
+ CRUSH rule 0 x 835 [14,3,13,6,4,9,1]
+ CRUSH rule 0 x 836 [3,9,10,13,1,5,14]
+ CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+ CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+ CRUSH rule 0 x 839 [3,4,6,10,15,1,13]
+ CRUSH rule 0 x 840 [10,15,12,4,7,1,2]
+ CRUSH rule 0 x 841 [3,5,7,12,11,15,1]
+ CRUSH rule 0 x 842 [9,13,2,6,5,14,10]
+ CRUSH rule 0 x 843 [14,7,4,9,3,12,1]
+ CRUSH rule 0 x 844 [7,1,4,15,9,2,11]
+ CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+ CRUSH rule 0 x 846 [3,7,15,13,1,9,10]
+ CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+ CRUSH rule 0 x 848 [11,13,1,14,5,9,2]
+ CRUSH rule 0 x 849 [3,15,11,9,6,1,13]
+ CRUSH rule 0 x 850 [1,3,10,6,14,4,9]
+ CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+ CRUSH rule 0 x 852 [9,12,4,7,15,2,11]
+ CRUSH rule 0 x 853 [13,14,6,11,2,4,9]
+ CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+ CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+ CRUSH rule 0 x 856 [5,10,7,3,15,9,12]
+ CRUSH rule 0 x 857 [4,3,13,11,9,1,7]
+ CRUSH rule 0 x 858 [5,15,6,3,9,12,1]
+ CRUSH rule 0 x 859 [5,15,6,2,1,11,12]
+ CRUSH rule 0 x 860 [11,14,1,12,6,9,2]
+ CRUSH rule 0 x 861 [13,7,4,10,1,14,3]
+ CRUSH rule 0 x 862 [5,10,9,7,3,12,1]
+ CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+ CRUSH rule 0 x 864 [6,13,4,2,10,15,1]
+ CRUSH rule 0 x 865 [4,1,14,11,6,9,3]
+ CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+ CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+ CRUSH rule 0 x 868 [14,11,7,2,1,4,9]
+ CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+ CRUSH rule 0 x 870 [14,9,11,4,3,12,6]
+ CRUSH rule 0 x 871 [6,2,1,4,15,13,11]
+ CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+ CRUSH rule 0 x 873 [2,5,12,10,1,9,15]
+ CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+ CRUSH rule 0 x 875 [10,6,14,1,12,5,9]
+ CRUSH rule 0 x 876 [14,7,13,3,9,1,11]
+ CRUSH rule 0 x 877 [15,11,13,9,5,1,6]
+ CRUSH rule 0 x 878 [7,14,3,13,9,1,11]
+ CRUSH rule 0 x 879 [12,2,7,4,10,15]
+ CRUSH rule 0 x 880 [2,12,10,7,1,4,9]
+ CRUSH rule 0 x 881 [6,3,1,11,4,15,9]
+ CRUSH rule 0 x 882 [11,13,7,1,2,15,4]
+ CRUSH rule 0 x 883 [13,1,3,10,6,5,9]
+ CRUSH rule 0 x 884 [6,15,4,9,3,11,12]
+ CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+ CRUSH rule 0 x 886 [13,11,4,2,1,14,9]
+ CRUSH rule 0 x 887 [14,4,12,11,2,6,9]
+ CRUSH rule 0 x 888 [10,12,7,15,9,2,1]
+ CRUSH rule 0 x 889 [15,13,4,1,6,2,10]
+ CRUSH rule 0 x 890 [10,12,14,2,9,5,6]
+ CRUSH rule 0 x 891 [9,5,11,6,3,15,12]
+ CRUSH rule 0 x 892 [12,15,2,4,7,9,11]
+ CRUSH rule 0 x 893 [1,3,5,9,6,10,14]
+ CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+ CRUSH rule 0 x 895 [2,1,11,5,7,15,13]
+ CRUSH rule 0 x 896 [9,1,14,10,4,12,2]
+ CRUSH rule 0 x 897 [7,5,14,3,1,9,11]
+ CRUSH rule 0 x 898 [10,6,12,9,15,5,2]
+ CRUSH rule 0 x 899 [1,11,5,3,13,14,9]
+ CRUSH rule 0 x 900 [2,9,10,7,13,14,5]
+ CRUSH rule 0 x 901 [9,12,11,3,14,4,1]
+ CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+ CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+ CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+ CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+ CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+ CRUSH rule 0 x 907 [7,12,3,9,10,5,14]
+ CRUSH rule 0 x 908 [2,15,9,6,10,13,5]
+ CRUSH rule 0 x 909 [10,14,1,13,2,9,7]
+ CRUSH rule 0 x 910 [12,7,4,15,10,3,1]
+ CRUSH rule 0 x 911 [11,15,2,4,9,13,6]
+ CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+ CRUSH rule 0 x 913 [4,6,10,1,12,3,9]
+ CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+ CRUSH rule 0 x 915 [12,14,1,9,4,3,11]
+ CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+ CRUSH rule 0 x 917 [1,15,6,5,10,3,13]
+ CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+ CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+ CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+ CRUSH rule 0 x 921 [1,11,6,13,4,2,9]
+ CRUSH rule 0 x 922 [6,4,14,13,3,1,10]
+ CRUSH rule 0 x 923 [12,2,5,14,10,1,9]
+ CRUSH rule 0 x 924 [6,2,14,13,9,1,11]
+ CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+ CRUSH rule 0 x 926 [3,13,10,1,14,9,6]
+ CRUSH rule 0 x 927 [6,5,1,11,14,2,13]
+ CRUSH rule 0 x 928 [13,1,3,9,6,11,15]
+ CRUSH rule 0 x 929 [10,7,1,5,2,12,9]
+ CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+ CRUSH rule 0 x 931 [6,15,11,9,5,3,1]
+ CRUSH rule 0 x 932 [13,2,5,11,9,1,6]
+ CRUSH rule 0 x 933 [12,7,14,10,4,1,2]
+ CRUSH rule 0 x 934 [12,2,5,7,9,1,15]
+ CRUSH rule 0 x 935 [6,11,1,14,5,13,3]
+ CRUSH rule 0 x 936 [9,12,7,5,1,2,14]
+ CRUSH rule 0 x 937 [14,2,11,1,13,4,9]
+ CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+ CRUSH rule 0 x 939 [6,4,14,9,12,1,11]
+ CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+ CRUSH rule 0 x 941 [3,12,4,7,14,10]
+ CRUSH rule 0 x 942 [15,12,10,4,1,9,3]
+ CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+ CRUSH rule 0 x 944 [2,9,4,7,1,14,12]
+ CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+ CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+ CRUSH rule 0 x 947 [11,3,14,1,12,5,6]
+ CRUSH rule 0 x 948 [7,13,11,5,14,2,1]
+ CRUSH rule 0 x 949 [9,1,12,5,15,10,2]
+ CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+ CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+ CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+ CRUSH rule 0 x 953 [1,3,6,10,12,14,4]
+ CRUSH rule 0 x 954 [10,2,14,9,4,6,12]
+ CRUSH rule 0 x 955 [7,14,3,1,10,4,9]
+ CRUSH rule 0 x 956 [1,6,11,5,14,3,9]
+ CRUSH rule 0 x 957 [14,11,1,12,6,9,4]
+ CRUSH rule 0 x 958 [15,4,3,11,1,6,12]
+ CRUSH rule 0 x 959 [2,1,12,15,10,9,4]
+ CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+ CRUSH rule 0 x 961 [3,13,11,9,6,1,4]
+ CRUSH rule 0 x 962 [5,11,3,14,1,6,13]
+ CRUSH rule 0 x 963 [13,10,15,4,6,9,1]
+ CRUSH rule 0 x 964 [7,11,4,9,2,12,1]
+ CRUSH rule 0 x 965 [12,2,9,7,4,15,11]
+ CRUSH rule 0 x 966 [12,14,9,4,1,2,11]
+ CRUSH rule 0 x 967 [7,5,3,10,12,14]
+ CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+ CRUSH rule 0 x 969 [11,4,7,1,9,14,13]
+ CRUSH rule 0 x 970 [5,12,10,1,3,14,9]
+ CRUSH rule 0 x 971 [1,9,4,12,7,2,10]
+ CRUSH rule 0 x 972 [12,3,14,5,1,9,7]
+ CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+ CRUSH rule 0 x 974 [7,11,1,2,15,4,12]
+ CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+ CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+ CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+ CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+ CRUSH rule 0 x 979 [5,1,13,6,15,10,3]
+ CRUSH rule 0 x 980 [15,11,5,6,1,3,13]
+ CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+ CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+ CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+ CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+ CRUSH rule 0 x 985 [11,2,15,1,4,13,6]
+ CRUSH rule 0 x 986 [6,13,9,1,15,10,5]
+ CRUSH rule 0 x 987 [13,14,5,10,6,1,3]
+ CRUSH rule 0 x 988 [12,9,10,14,3,1,4]
+ CRUSH rule 0 x 989 [7,4,3,15,9,13,10]
+ CRUSH rule 0 x 990 [1,10,9,13,3,4,6]
+ CRUSH rule 0 x 991 [7,11,1,14,2,5,9]
+ CRUSH rule 0 x 992 [9,10,2,13,7,4,1]
+ CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+ CRUSH rule 0 x 994 [3,13,15,4,11,7,1]
+ CRUSH rule 0 x 995 [15,6,12,2,5,11]
+ CRUSH rule 0 x 996 [15,10,5,3,13,1,9]
+ CRUSH rule 0 x 997 [15,2,1,12,7,9,4]
+ CRUSH rule 0 x 998 [6,1,9,5,12,11,15]
+ CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+ CRUSH rule 0 x 1000 [14,2,9,4,12,1,6]
+ CRUSH rule 0 x 1001 [11,14,4,2,6,9,1]
+ CRUSH rule 0 x 1002 [1,10,14,2,9,5,13]
+ CRUSH rule 0 x 1003 [10,7,5,14,2,1,9]
+ CRUSH rule 0 x 1004 [15,1,4,6,10,12,9]
+ CRUSH rule 0 x 1005 [6,12,2,10,9,15,5]
+ CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+ CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+ CRUSH rule 0 x 1008 [7,4,9,11,3,15,1]
+ CRUSH rule 0 x 1009 [5,2,11,7,15,9,1]
+ CRUSH rule 0 x 1010 [10,2,15,6,9,13,4]
+ CRUSH rule 0 x 1011 [6,3,12,1,10,4,9]
+ CRUSH rule 0 x 1012 [12,6,9,15,3,1,5]
+ CRUSH rule 0 x 1013 [2,14,12,4,9,1,6]
+ CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+ CRUSH rule 0 x 1015 [12,6,10,1,4,15,9]
+ CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+ CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+ CRUSH rule 0 x 1018 [13,11,14,1,9,3,5]
+ CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+ CRUSH rule 0 x 1020 [3,1,13,4,10,9,14]
+ CRUSH rule 0 x 1021 [2,11,14,9,4,6,1]
+ CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+ CRUSH rule 0 x 1023 [15,2,9,12,1,7,4]
+ rule 0 (replicated_ruleset) num_rep 7 result size == 6:\t36/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 7 result size == 7:\t988/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15,12,1,4,9]
+ CRUSH rule 0 x 1 [10,15,1,2,13,4,7,9]
+ CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+ CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+ CRUSH rule 0 x 4 [14,2,10,1,9,4,7,13]
+ CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+ CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+ CRUSH rule 0 x 7 [9,2,6,12,11,4,1,14]
+ CRUSH rule 0 x 8 [10,2,15,1,4,13,6,9]
+ CRUSH rule 0 x 9 [7,1,14,2,11,9,12,4]
+ CRUSH rule 0 x 10 [10,14,4,1,2,7,13,9]
+ CRUSH rule 0 x 11 [13,9,14,7,5,11,2,1]
+ CRUSH rule 0 x 12 [7,1,2,5,13,15,11,9]
+ CRUSH rule 0 x 13 [3,5,12,7,9,1,14,11]
+ CRUSH rule 0 x 14 [13,5,2,7,10,15,1,9]
+ CRUSH rule 0 x 15 [15,1,9,6,13,3,5,11]
+ CRUSH rule 0 x 16 [7,11,14,2,13,1,9,4]
+ CRUSH rule 0 x 17 [10,1,13,2,4,6,14,9]
+ CRUSH rule 0 x 18 [1,7,3,10,5,12,9,14]
+ CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+ CRUSH rule 0 x 20 [14,12,3,10,9,4,7,1]
+ CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+ CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+ CRUSH rule 0 x 23 [10,5,13,9,3,15,1,6]
+ CRUSH rule 0 x 24 [12,11,3,1,9,4,7,15]
+ CRUSH rule 0 x 25 [7,12,15,1,3,10,4,9]
+ CRUSH rule 0 x 26 [1,7,13,2,14,5,9,11]
+ CRUSH rule 0 x 27 [3,6,15,4,13,9,11,1]
+ CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+ CRUSH rule 0 x 29 [5,14,12,11,6,3,1,9]
+ CRUSH rule 0 x 30 [2,5,6,9,1,11,13,14]
+ CRUSH rule 0 x 31 [5,15,10,1,9,13,6,3]
+ CRUSH rule 0 x 32 [9,10,2,1,13,14,6,4]
+ CRUSH rule 0 x 33 [13,4,9,2,7,1,10,14]
+ CRUSH rule 0 x 34 [13,15,2,4,1,10,9,6]
+ CRUSH rule 0 x 35 [4,14,3,13,10,9,1,6]
+ CRUSH rule 0 x 36 [3,12,9,7,5,10,14,1]
+ CRUSH rule 0 x 37 [9,2,6,14,11,1,4,13]
+ CRUSH rule 0 x 38 [3,4,13,10,9,1,14,6]
+ CRUSH rule 0 x 39 [12,7,14,11,1,9,5,2]
+ CRUSH rule 0 x 40 [10,1,9,5,15,2,6,13]
+ CRUSH rule 0 x 41 [4,9,11,1,14,13,6,3]
+ CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+ CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+ CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+ CRUSH rule 0 x 45 [11,12,15,9,1,5,6,3]
+ CRUSH rule 0 x 46 [6,9,2,14,11,13,1,5]
+ CRUSH rule 0 x 47 [3,9,6,4,13,1,11,15]
+ CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+ CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+ CRUSH rule 0 x 50 [14,12,1,4,2,11,6,9]
+ CRUSH rule 0 x 51 [10,6,5,12,15,2,1,9]
+ CRUSH rule 0 x 52 [12,1,9,11,7,3,14,4]
+ CRUSH rule 0 x 53 [3,6,13,9,5,1,11,15]
+ CRUSH rule 0 x 54 [4,13,9,2,14,10,6,1]
+ CRUSH rule 0 x 55 [4,11,2,7,1,13,9,15]
+ CRUSH rule 0 x 56 [5,9,10,1,3,13,14,7]
+ CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+ CRUSH rule 0 x 58 [7,1,11,4,3,14,12,9]
+ CRUSH rule 0 x 59 [2,13,1,10,9,5,14,6]
+ CRUSH rule 0 x 60 [3,6,11,1,4,9,12,15]
+ CRUSH rule 0 x 61 [3,15,13,7,4,1,10,9]
+ CRUSH rule 0 x 62 [15,11,7,12,5,9,2,1]
+ CRUSH rule 0 x 63 [10,14,12,1,7,3]
+ CRUSH rule 0 x 64 [3,9,1,4,7,12,11,15]
+ CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+ CRUSH rule 0 x 66 [15,11,6,9,4,1,3,12]
+ CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+ CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+ CRUSH rule 0 x 69 [2,1,15,10,4,9,13,7]
+ CRUSH rule 0 x 70 [9,6,1,3,13,15,11,5]
+ CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+ CRUSH rule 0 x 72 [9,10,3,5,7,12,15,1]
+ CRUSH rule 0 x 73 [5,3,11,1,7,12,15,9]
+ CRUSH rule 0 x 74 [11,7,9,5,1,15,3,12]
+ CRUSH rule 0 x 75 [9,7,11,14,12,1,2,5]
+ CRUSH rule 0 x 76 [6,1,3,5,14,10,12,9]
+ CRUSH rule 0 x 77 [7,4,2,13,9,1,11,15]
+ CRUSH rule 0 x 78 [9,3,1,5,6,13,14,11]
+ CRUSH rule 0 x 79 [13,2,15,5,7,9,11,1]
+ CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+ CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+ CRUSH rule 0 x 82 [14,13,5,11,6,2,1,9]
+ CRUSH rule 0 x 83 [4,15,3,9,10,13,6,1]
+ CRUSH rule 0 x 84 [10,7,9,15,3,4,1,13]
+ CRUSH rule 0 x 85 [3,15,9,7,4,11,1,12]
+ CRUSH rule 0 x 86 [10,9,14,1,13,4,2,7]
+ CRUSH rule 0 x 87 [15,10,7,12,5,3,9,1]
+ CRUSH rule 0 x 88 [4,13,3,1,9,15,11,7]
+ CRUSH rule 0 x 89 [3,9,7,4,1,14,10,12]
+ CRUSH rule 0 x 90 [4,9,7,12,11,14,2,1]
+ CRUSH rule 0 x 91 [6,11,9,1,2,4,14,13]
+ CRUSH rule 0 x 92 [1,5,10,9,13,15,6,2]
+ CRUSH rule 0 x 93 [9,3,15,13,7,5,1,10]
+ CRUSH rule 0 x 94 [9,2,12,5,6,11,1,14]
+ CRUSH rule 0 x 95 [7,15,4,10,9,13,2,1]
+ CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+ CRUSH rule 0 x 97 [4,11,2,13,1,7,9,14]
+ CRUSH rule 0 x 98 [11,13,9,3,15,1,5,6]
+ CRUSH rule 0 x 99 [12,4,11,7,3,14,9,1]
+ CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+ CRUSH rule 0 x 101 [15,7,1,9,10,5,2,12]
+ CRUSH rule 0 x 102 [3,11,14,6,13,4,9,1]
+ CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+ CRUSH rule 0 x 104 [14,6,3,5,9,1,10,13]
+ CRUSH rule 0 x 105 [14,10,1,9,3,5,6,13]
+ CRUSH rule 0 x 106 [6,5,13,2,14,11,1,9]
+ CRUSH rule 0 x 107 [3,1,10,14,13,5,9,6]
+ CRUSH rule 0 x 108 [5,10,7,2,15,9,12,1]
+ CRUSH rule 0 x 109 [9,1,13,7,15,5,11,3]
+ CRUSH rule 0 x 110 [5,1,11,3,7,14,13,9]
+ CRUSH rule 0 x 111 [10,1,9,7,5,2,13,14]
+ CRUSH rule 0 x 112 [1,10,4,14,2,12,6,9]
+ CRUSH rule 0 x 113 [6,10,13,9,1,5,2,14]
+ CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+ CRUSH rule 0 x 115 [10,13,14,3,9,1,6,5]
+ CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+ CRUSH rule 0 x 117 [5,6,1,12,15,9,11,3]
+ CRUSH rule 0 x 118 [10,4,13,15,9,3,1,6]
+ CRUSH rule 0 x 119 [14,12,11,4,6,9,3,1]
+ CRUSH rule 0 x 120 [11,3,14,13,4,7]
+ CRUSH rule 0 x 121 [9,5,1,11,7,3,15,12]
+ CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+ CRUSH rule 0 x 123 [3,10,5,6,9,1,12,15]
+ CRUSH rule 0 x 124 [12,2,1,5,14,7,11,9]
+ CRUSH rule 0 x 125 [9,12,15,1,6,5,3,10]
+ CRUSH rule 0 x 126 [7,15,10,9,2,12,5,1]
+ CRUSH rule 0 x 127 [4,14,9,13,1,3,7,10]
+ CRUSH rule 0 x 128 [3,12,1,10,4,9,7,14]
+ CRUSH rule 0 x 129 [11,13,14,2,9,4,6,1]
+ CRUSH rule 0 x 130 [3,13,5,14,10,1,9,6]
+ CRUSH rule 0 x 131 [12,1,6,15,4,2,11,9]
+ CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+ CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+ CRUSH rule 0 x 134 [12,5,6,15,3,9,10,1]
+ CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+ CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+ CRUSH rule 0 x 137 [14,3,6,11,1,9,4,13]
+ CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+ CRUSH rule 0 x 139 [11,2,13,9,1,15,7,5]
+ CRUSH rule 0 x 140 [11,4,12,15,2,6,9,1]
+ CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+ CRUSH rule 0 x 142 [3,14,7,9,11,1,4,13]
+ CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+ CRUSH rule 0 x 144 [13,7,11,2,14,4,1,9]
+ CRUSH rule 0 x 145 [12,2,6,10,9,4,14,1]
+ CRUSH rule 0 x 146 [1,5,9,2,6,13,14,10]
+ CRUSH rule 0 x 147 [1,4,9,11,2,7,15,12]
+ CRUSH rule 0 x 148 [12,7,9,2,14,11,1,4]
+ CRUSH rule 0 x 149 [2,5,9,12,11,1,7,14]
+ CRUSH rule 0 x 150 [1,15,2,10,7,9,5,12]
+ CRUSH rule 0 x 151 [2,9,14,7,1,10,5,13]
+ CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+ CRUSH rule 0 x 153 [6,9,4,15,2,1,10,12]
+ CRUSH rule 0 x 154 [3,11,7,1,4,12,15,9]
+ CRUSH rule 0 x 155 [14,12,7,3,5,1,9,11]
+ CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+ CRUSH rule 0 x 157 [15,1,6,4,3,10,9,13]
+ CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+ CRUSH rule 0 x 159 [4,14,3,12,10,6,1,9]
+ CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+ CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+ CRUSH rule 0 x 162 [10,6,1,12,2,4,14,9]
+ CRUSH rule 0 x 163 [15,1,10,2,6,4,13,9]
+ CRUSH rule 0 x 164 [9,14,10,7,12,2,5,1]
+ CRUSH rule 0 x 165 [11,7,2,13,9,15,1,5]
+ CRUSH rule 0 x 166 [1,2,12,14,4,11,7,9]
+ CRUSH rule 0 x 167 [9,7,3,4,11,13,15,1]
+ CRUSH rule 0 x 168 [13,2,4,1,6,15,10,9]
+ CRUSH rule 0 x 169 [1,4,9,14,13,10,2,7]
+ CRUSH rule 0 x 170 [1,15,7,9,12,10,3,5]
+ CRUSH rule 0 x 171 [9,2,10,7,1,5,15,12]
+ CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+ CRUSH rule 0 x 173 [5,10,12,15,6,1,2,9]
+ CRUSH rule 0 x 174 [15,6,4,12,1,11,9,2]
+ CRUSH rule 0 x 175 [5,7,9,3,10,1,14,13]
+ CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+ CRUSH rule 0 x 177 [2,9,10,13,4,1,14,7]
+ CRUSH rule 0 x 178 [12,11,7,14,3,4]
+ CRUSH rule 0 x 179 [2,10,13,9,5,1,7,14]
+ CRUSH rule 0 x 180 [3,11,5,15,7,12]
+ CRUSH rule 0 x 181 [9,12,6,5,1,10,14,2]
+ CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+ CRUSH rule 0 x 183 [5,7,10,13,3,9,14,1]
+ CRUSH rule 0 x 184 [2,5,11,12,7,1,9,15]
+ CRUSH rule 0 x 185 [13,5,7,11,2,14]
+ CRUSH rule 0 x 186 [6,14,13,5,10,1,3,9]
+ CRUSH rule 0 x 187 [1,4,11,13,6,14,9,3]
+ CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+ CRUSH rule 0 x 189 [6,12,4,9,2,1,11,15]
+ CRUSH rule 0 x 190 [9,13,15,10,3,1,5,6]
+ CRUSH rule 0 x 191 [7,11,4,1,15,12,9,2]
+ CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+ CRUSH rule 0 x 193 [3,13,6,10,4,1,9,14]
+ CRUSH rule 0 x 194 [3,13,4,14,6,9,1,11]
+ CRUSH rule 0 x 195 [5,7,10,12,1,3,15,9]
+ CRUSH rule 0 x 196 [4,15,1,10,9,2,13,7]
+ CRUSH rule 0 x 197 [14,10,13,4,6,3,1,9]
+ CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+ CRUSH rule 0 x 199 [2,10,4,15,1,9,6,12]
+ CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+ CRUSH rule 0 x 201 [9,14,1,7,4,3,10,13]
+ CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+ CRUSH rule 0 x 203 [12,5,7,15,1,2,10,9]
+ CRUSH rule 0 x 204 [6,11,3,12,14,1,9,4]
+ CRUSH rule 0 x 205 [15,4,6,10,13,9,2,1]
+ CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+ CRUSH rule 0 x 207 [2,11,7,4,14,1,12,9]
+ CRUSH rule 0 x 208 [13,1,6,14,9,11,3,5]
+ CRUSH rule 0 x 209 [6,15,13,1,11,4,9,2]
+ CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+ CRUSH rule 0 x 211 [2,14,1,13,11,7,9,5]
+ CRUSH rule 0 x 212 [10,1,12,15,5,6,2,9]
+ CRUSH rule 0 x 213 [3,9,6,5,15,13,1,11]
+ CRUSH rule 0 x 214 [7,15,4,1,10,2,13,9]
+ CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+ CRUSH rule 0 x 216 [12,9,6,2,1,11,5,15]
+ CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+ CRUSH rule 0 x 218 [12,10,15,6,1,4,9,2]
+ CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+ CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+ CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+ CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+ CRUSH rule 0 x 223 [9,7,11,1,4,14,13,3]
+ CRUSH rule 0 x 224 [1,7,10,2,12,9,14,5]
+ CRUSH rule 0 x 225 [10,5,2,6,1,13,9,15]
+ CRUSH rule 0 x 226 [4,1,9,3,13,10,15,7]
+ CRUSH rule 0 x 227 [7,2,12,15,5,11]
+ CRUSH rule 0 x 228 [2,15,11,1,6,13,9,4]
+ CRUSH rule 0 x 229 [9,3,7,14,1,12,4,10]
+ CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+ CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+ CRUSH rule 0 x 232 [10,5,13,1,9,2,7,14]
+ CRUSH rule 0 x 233 [6,12,11,4,9,14,1,3]
+ CRUSH rule 0 x 234 [10,1,2,12,5,9,15,7]
+ CRUSH rule 0 x 235 [13,14,7,10,1,9,5,3]
+ CRUSH rule 0 x 236 [2,15,9,12,1,7,4,10]
+ CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+ CRUSH rule 0 x 238 [2,10,4,15,6,12,9,1]
+ CRUSH rule 0 x 239 [4,15,10,7,9,13,3,1]
+ CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+ CRUSH rule 0 x 241 [7,9,15,12,1,5,2,10]
+ CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+ CRUSH rule 0 x 243 [2,11,5,1,15,6,9,13]
+ CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+ CRUSH rule 0 x 245 [12,9,15,3,1,5,10,7]
+ CRUSH rule 0 x 246 [15,3,5,11,7,1,12,9]
+ CRUSH rule 0 x 247 [6,4,9,12,1,2,10,14]
+ CRUSH rule 0 x 248 [5,13,7,11,9,15,3,1]
+ CRUSH rule 0 x 249 [10,14,7,3,9,13,1,4]
+ CRUSH rule 0 x 250 [12,15,1,10,5,6,3,9]
+ CRUSH rule 0 x 251 [13,2,15,5,6,1,9,10]
+ CRUSH rule 0 x 252 [7,5,13,9,3,10,14,1]
+ CRUSH rule 0 x 253 [3,13,15,10,7,4]
+ CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+ CRUSH rule 0 x 255 [1,9,13,2,6,10,4,15]
+ CRUSH rule 0 x 256 [6,9,13,1,3,14,5,10]
+ CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+ CRUSH rule 0 x 258 [12,5,6,10,2,1,14,9]
+ CRUSH rule 0 x 259 [9,10,4,3,14,13,1,7]
+ CRUSH rule 0 x 260 [10,12,6,9,3,15,1,4]
+ CRUSH rule 0 x 261 [13,7,2,1,15,5,11,9]
+ CRUSH rule 0 x 262 [15,3,12,7,4,9,1,11]
+ CRUSH rule 0 x 263 [12,6,10,9,5,15,3,1]
+ CRUSH rule 0 x 264 [13,14,11,3,1,4,7,9]
+ CRUSH rule 0 x 265 [12,10,14,5,7,1,9,3]
+ CRUSH rule 0 x 266 [14,7,11,1,2,9,4,12]
+ CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+ CRUSH rule 0 x 268 [4,1,15,12,6,11,3,9]
+ CRUSH rule 0 x 269 [11,1,15,5,13,9,7,2]
+ CRUSH rule 0 x 270 [7,11,12,3,1,14,9,4]
+ CRUSH rule 0 x 271 [4,7,3,13,15,10,9,1]
+ CRUSH rule 0 x 272 [15,5,13,10,6,2]
+ CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+ CRUSH rule 0 x 274 [10,2,5,6,13,9,15,1]
+ CRUSH rule 0 x 275 [10,3,4,7,14,13]
+ CRUSH rule 0 x 276 [5,12,9,2,11,7,15,1]
+ CRUSH rule 0 x 277 [14,3,13,4,1,9,11,7]
+ CRUSH rule 0 x 278 [5,6,14,3,1,11,13,9]
+ CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+ CRUSH rule 0 x 280 [7,3,14,9,1,11,4,13]
+ CRUSH rule 0 x 281 [5,11,14,7,9,13,2,1]
+ CRUSH rule 0 x 282 [2,1,13,14,9,7,5,10]
+ CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+ CRUSH rule 0 x 284 [5,11,7,15,3,13,1,9]
+ CRUSH rule 0 x 285 [15,5,3,1,6,13,11,9]
+ CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+ CRUSH rule 0 x 287 [12,4,9,1,3,11,15,7]
+ CRUSH rule 0 x 288 [4,12,10,7,1,3,14,9]
+ CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+ CRUSH rule 0 x 290 [12,2,5,6,15,9,1,10]
+ CRUSH rule 0 x 291 [7,11,1,14,5,9,2,12]
+ CRUSH rule 0 x 292 [4,10,6,3,14,9,12,1]
+ CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+ CRUSH rule 0 x 294 [9,12,3,14,6,11,5,1]
+ CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+ CRUSH rule 0 x 296 [3,1,13,7,14,9,10,4]
+ CRUSH rule 0 x 297 [6,13,4,14,10,1,2,9]
+ CRUSH rule 0 x 298 [14,9,13,1,4,2,7,10]
+ CRUSH rule 0 x 299 [14,12,11,6,4,2,1,9]
+ CRUSH rule 0 x 300 [15,7,10,5,1,3,13,9]
+ CRUSH rule 0 x 301 [9,11,7,1,13,14,4,2]
+ CRUSH rule 0 x 302 [9,7,1,13,5,10,3,15]
+ CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+ CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+ CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+ CRUSH rule 0 x 306 [10,12,4,6,9,2,15,1]
+ CRUSH rule 0 x 307 [11,12,15,5,6,2,1,9]
+ CRUSH rule 0 x 308 [12,14,10,9,1,2,5,7]
+ CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+ CRUSH rule 0 x 310 [3,1,5,10,14,9,7,12]
+ CRUSH rule 0 x 311 [3,9,7,1,14,13,10,5]
+ CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+ CRUSH rule 0 x 313 [9,15,3,7,5,13,1,11]
+ CRUSH rule 0 x 314 [2,15,9,5,6,12,1,11]
+ CRUSH rule 0 x 315 [15,2,13,1,11,9,6,4]
+ CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+ CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+ CRUSH rule 0 x 318 [4,1,15,11,9,13,6,2]
+ CRUSH rule 0 x 319 [2,15,4,1,11,9,7,12]
+ CRUSH rule 0 x 320 [5,7,13,9,11,2,1,15]
+ CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+ CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+ CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+ CRUSH rule 0 x 324 [5,6,10,15,2,13]
+ CRUSH rule 0 x 325 [9,10,14,5,1,6,2,13]
+ CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+ CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+ CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+ CRUSH rule 0 x 329 [2,6,15,5,9,10,13,1]
+ CRUSH rule 0 x 330 [3,9,11,13,1,6,5,14]
+ CRUSH rule 0 x 331 [12,14,6,3,1,4,10,9]
+ CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+ CRUSH rule 0 x 333 [6,5,3,12,14,10,9,1]
+ CRUSH rule 0 x 334 [4,9,2,12,7,11,15,1]
+ CRUSH rule 0 x 335 [11,7,1,5,13,2,9,15]
+ CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+ CRUSH rule 0 x 337 [15,11,3,7,12,5]
+ CRUSH rule 0 x 338 [10,5,3,6,15,1,9,13]
+ CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+ CRUSH rule 0 x 340 [11,6,12,4,9,3,14,1]
+ CRUSH rule 0 x 341 [7,5,2,10,14,9,1,12]
+ CRUSH rule 0 x 342 [12,14,1,9,2,11,4,7]
+ CRUSH rule 0 x 343 [12,14,9,6,10,2,4,1]
+ CRUSH rule 0 x 344 [9,11,5,2,14,13,1,7]
+ CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+ CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+ CRUSH rule 0 x 347 [10,2,12,6,9,1,14,5]
+ CRUSH rule 0 x 348 [7,9,10,1,14,13,3,4]
+ CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+ CRUSH rule 0 x 350 [13,9,15,4,10,7,2,1]
+ CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+ CRUSH rule 0 x 352 [1,12,11,9,4,7,3,15]
+ CRUSH rule 0 x 353 [10,14,12,2,9,1,4,6]
+ CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+ CRUSH rule 0 x 355 [13,14,6,10,2,5,1,9]
+ CRUSH rule 0 x 356 [15,13,2,9,6,5,1,11]
+ CRUSH rule 0 x 357 [4,11,1,13,3,14,6,9]
+ CRUSH rule 0 x 358 [12,7,2,9,1,14,10,4]
+ CRUSH rule 0 x 359 [5,15,7,11,3,13]
+ CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+ CRUSH rule 0 x 361 [5,3,13,6,1,14,11,9]
+ CRUSH rule 0 x 362 [2,9,11,13,1,6,5,15]
+ CRUSH rule 0 x 363 [7,12,3,9,15,4,1,10]
+ CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+ CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+ CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+ CRUSH rule 0 x 367 [7,13,3,1,5,11,15,9]
+ CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+ CRUSH rule 0 x 369 [7,5,3,13,14,9,11,1]
+ CRUSH rule 0 x 370 [4,7,14,1,2,9,11,12]
+ CRUSH rule 0 x 371 [1,7,12,3,4,15,10,9]
+ CRUSH rule 0 x 372 [10,4,3,14,6,1,12,9]
+ CRUSH rule 0 x 373 [15,5,2,6,13,1,9,10]
+ CRUSH rule 0 x 374 [3,15,12,5,1,6,10,9]
+ CRUSH rule 0 x 375 [5,2,14,1,6,13,11,9]
+ CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+ CRUSH rule 0 x 377 [1,15,2,4,9,11,12,6]
+ CRUSH rule 0 x 378 [9,12,2,15,1,5,11,6]
+ CRUSH rule 0 x 379 [11,2,15,5,7,9,13,1]
+ CRUSH rule 0 x 380 [6,1,12,11,2,9,5,14]
+ CRUSH rule 0 x 381 [15,13,7,5,10,2,1,9]
+ CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+ CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+ CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+ CRUSH rule 0 x 385 [4,6,15,3,10,9,1,13]
+ CRUSH rule 0 x 386 [14,3,11,13,5,6,9,1]
+ CRUSH rule 0 x 387 [1,11,5,7,9,2,14,12]
+ CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+ CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+ CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+ CRUSH rule 0 x 391 [3,4,9,13,7,10,1,14]
+ CRUSH rule 0 x 392 [11,5,14,7,1,9,2,12]
+ CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+ CRUSH rule 0 x 394 [4,9,3,15,13,6,1,11]
+ CRUSH rule 0 x 395 [10,13,5,15,6,9,3,1]
+ CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+ CRUSH rule 0 x 397 [1,14,9,4,12,10,3,7]
+ CRUSH rule 0 x 398 [9,2,1,5,12,6,11,15]
+ CRUSH rule 0 x 399 [5,9,14,3,1,10,13,7]
+ CRUSH rule 0 x 400 [10,6,2,4,15,12,1,9]
+ CRUSH rule 0 x 401 [6,9,11,12,4,3,15,1]
+ CRUSH rule 0 x 402 [4,7,9,2,13,1,15,11]
+ CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+ CRUSH rule 0 x 404 [14,12,7,9,2,1,5,11]
+ CRUSH rule 0 x 405 [9,15,11,2,4,7,13,1]
+ CRUSH rule 0 x 406 [12,14,9,2,7,10,4,1]
+ CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+ CRUSH rule 0 x 408 [7,1,5,2,10,15,13,9]
+ CRUSH rule 0 x 409 [11,2,4,13,1,15,7,9]
+ CRUSH rule 0 x 410 [6,4,14,2,12,9,10,1]
+ CRUSH rule 0 x 411 [13,11,15,6,4,1,9,2]
+ CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+ CRUSH rule 0 x 413 [13,5,3,11,6,9,1,14]
+ CRUSH rule 0 x 414 [3,11,9,13,4,1,6,15]
+ CRUSH rule 0 x 415 [6,10,14,5,1,13,3,9]
+ CRUSH rule 0 x 416 [13,1,4,7,2,9,14,11]
+ CRUSH rule 0 x 417 [4,12,1,15,2,11,9,6]
+ CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+ CRUSH rule 0 x 419 [5,14,10,9,2,12,6,1]
+ CRUSH rule 0 x 420 [2,4,9,11,6,14,13,1]
+ CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+ CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+ CRUSH rule 0 x 423 [3,15,12,6,5,1,9,10]
+ CRUSH rule 0 x 424 [6,10,12,2,5,1,14,9]
+ CRUSH rule 0 x 425 [11,15,2,13,5,7,9,1]
+ CRUSH rule 0 x 426 [12,4,7,1,9,10,14,2]
+ CRUSH rule 0 x 427 [14,10,3,1,9,7,5,13]
+ CRUSH rule 0 x 428 [12,7,9,4,2,1,14,10]
+ CRUSH rule 0 x 429 [3,4,9,7,11,12,1,14]
+ CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+ CRUSH rule 0 x 431 [9,3,7,1,12,5,14,11]
+ CRUSH rule 0 x 432 [4,1,12,7,15,2,10,9]
+ CRUSH rule 0 x 433 [4,11,12,15,7,3]
+ CRUSH rule 0 x 434 [2,14,9,1,5,11,7,13]
+ CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+ CRUSH rule 0 x 436 [9,15,10,2,4,1,12,7]
+ CRUSH rule 0 x 437 [9,6,3,14,10,12,5,1]
+ CRUSH rule 0 x 438 [7,2,13,4,11,1,9,14]
+ CRUSH rule 0 x 439 [7,14,4,3,12,10]
+ CRUSH rule 0 x 440 [14,11,9,2,7,12,1,5]
+ CRUSH rule 0 x 441 [2,4,11,9,13,6,1,14]
+ CRUSH rule 0 x 442 [10,13,9,7,15,1,4,2]
+ CRUSH rule 0 x 443 [12,15,10,9,2,1,6,4]
+ CRUSH rule 0 x 444 [4,13,7,14,3,1,9,11]
+ CRUSH rule 0 x 445 [4,2,15,7,1,9,11,12]
+ CRUSH rule 0 x 446 [12,10,6,9,4,1,15,3]
+ CRUSH rule 0 x 447 [15,7,13,1,4,9,3,11]
+ CRUSH rule 0 x 448 [5,2,13,7,15,10]
+ CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+ CRUSH rule 0 x 450 [2,4,6,9,15,1,13,10]
+ CRUSH rule 0 x 451 [6,14,11,3,9,1,12,5]
+ CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+ CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+ CRUSH rule 0 x 454 [10,4,2,6,15,12,9,1]
+ CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+ CRUSH rule 0 x 456 [5,7,13,1,11,3,9,15]
+ CRUSH rule 0 x 457 [9,1,5,7,11,13,15]
+ CRUSH rule 0 x 458 [9,11,15,4,7,2,12,1]
+ CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+ CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+ CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+ CRUSH rule 0 x 462 [4,7,12,14,11,1,3,9]
+ CRUSH rule 0 x 463 [4,12,14,11,2,7,1,9]
+ CRUSH rule 0 x 464 [4,2,15,10,1,9,13,7]
+ CRUSH rule 0 x 465 [5,10,9,7,13,1,3,14]
+ CRUSH rule 0 x 466 [13,5,2,15,9,11,6,1]
+ CRUSH rule 0 x 467 [13,6,14,3,9,1,11,5]
+ CRUSH rule 0 x 468 [10,7,12,14,4,1,9,2]
+ CRUSH rule 0 x 469 [4,9,6,14,12,11,3,1]
+ CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+ CRUSH rule 0 x 471 [6,1,5,14,13,10,9,3]
+ CRUSH rule 0 x 472 [2,14,7,5,13,1,11,9]
+ CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+ CRUSH rule 0 x 474 [15,10,4,12,6,9,2,1]
+ CRUSH rule 0 x 475 [10,5,12,9,14,3,6,1]
+ CRUSH rule 0 x 476 [3,6,10,12,1,15,9,4]
+ CRUSH rule 0 x 477 [6,13,5,15,11,9,2,1]
+ CRUSH rule 0 x 478 [4,15,1,3,7,12,10,9]
+ CRUSH rule 0 x 479 [13,11,1,6,14,5,9,3]
+ CRUSH rule 0 x 480 [1,13,6,4,9,14,11,3]
+ CRUSH rule 0 x 481 [15,12,7,9,1,3,10,4]
+ CRUSH rule 0 x 482 [2,12,9,1,7,11,14,4]
+ CRUSH rule 0 x 483 [10,1,4,15,9,7,13,2]
+ CRUSH rule 0 x 484 [1,4,10,13,7,14,2,9]
+ CRUSH rule 0 x 485 [9,4,3,1,14,12,7,10]
+ CRUSH rule 0 x 486 [3,10,15,9,7,13,4,1]
+ CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+ CRUSH rule 0 x 488 [14,4,1,9,2,6,10,12]
+ CRUSH rule 0 x 489 [11,4,2,13,15,7]
+ CRUSH rule 0 x 490 [4,9,1,3,13,15,6,11]
+ CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+ CRUSH rule 0 x 492 [5,7,11,3,14,9,1,13]
+ CRUSH rule 0 x 493 [12,1,4,15,3,11,9,6]
+ CRUSH rule 0 x 494 [1,7,13,4,15,9,10,3]
+ CRUSH rule 0 x 495 [3,15,7,1,9,5,12,11]
+ CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+ CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+ CRUSH rule 0 x 498 [10,6,1,5,9,12,3,15]
+ CRUSH rule 0 x 499 [14,3,12,5,1,11,9,7]
+ CRUSH rule 0 x 500 [15,9,6,12,11,2,1,5]
+ CRUSH rule 0 x 501 [10,13,1,9,3,14,5,7]
+ CRUSH rule 0 x 502 [5,1,14,11,7,12,9,2]
+ CRUSH rule 0 x 503 [15,10,7,9,1,12,4,2]
+ CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+ CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+ CRUSH rule 0 x 506 [11,7,9,14,12,1,2,5]
+ CRUSH rule 0 x 507 [4,14,13,3,9,7,1,10]
+ CRUSH rule 0 x 508 [12,1,4,9,2,11,15,7]
+ CRUSH rule 0 x 509 [4,2,6,9,14,1,10,13]
+ CRUSH rule 0 x 510 [5,3,1,12,11,14,9,7]
+ CRUSH rule 0 x 511 [2,12,10,6,14,5]
+ CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+ CRUSH rule 0 x 513 [4,9,11,3,13,7,1,14]
+ CRUSH rule 0 x 514 [11,9,3,4,12,15,6,1]
+ CRUSH rule 0 x 515 [12,14,6,5,3,9,1,10]
+ CRUSH rule 0 x 516 [14,11,1,12,3,7,4,9]
+ CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+ CRUSH rule 0 x 518 [3,5,7,12,15,11,9,1]
+ CRUSH rule 0 x 519 [12,14,2,1,4,6,9,10]
+ CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+ CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+ CRUSH rule 0 x 522 [4,12,11,1,15,3,9,6]
+ CRUSH rule 0 x 523 [3,1,5,9,15,10,13,7]
+ CRUSH rule 0 x 524 [15,9,3,11,13,7,4,1]
+ CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+ CRUSH rule 0 x 526 [10,2,5,13,6,15,1,9]
+ CRUSH rule 0 x 527 [3,13,4,1,9,10,14,7]
+ CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+ CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+ CRUSH rule 0 x 530 [11,9,12,7,5,1,3,15]
+ CRUSH rule 0 x 531 [9,15,4,7,2,13,1,11]
+ CRUSH rule 0 x 532 [5,3,13,7,9,14,1,10]
+ CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+ CRUSH rule 0 x 534 [11,9,3,7,15,4,1,12]
+ CRUSH rule 0 x 535 [11,1,3,5,14,9,12,7]
+ CRUSH rule 0 x 536 [9,1,14,13,4,6,2,11]
+ CRUSH rule 0 x 537 [15,5,13,2,7,11]
+ CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+ CRUSH rule 0 x 539 [10,12,6,14,1,2,9,5]
+ CRUSH rule 0 x 540 [12,15,7,3,9,11,1,4]
+ CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+ CRUSH rule 0 x 542 [3,9,15,5,11,12,7,1]
+ CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+ CRUSH rule 0 x 544 [3,15,9,11,7,4,12,1]
+ CRUSH rule 0 x 545 [14,10,7,12,4,9,1,3]
+ CRUSH rule 0 x 546 [5,15,13,7,1,10,9,2]
+ CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+ CRUSH rule 0 x 548 [11,7,12,15,4,2]
+ CRUSH rule 0 x 549 [14,1,4,9,13,6,3,10]
+ CRUSH rule 0 x 550 [9,15,3,13,1,6,4,11]
+ CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+ CRUSH rule 0 x 552 [2,11,14,1,9,6,5,12]
+ CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+ CRUSH rule 0 x 554 [11,14,6,4,13,9,3,1]
+ CRUSH rule 0 x 555 [6,5,10,9,14,2,13,1]
+ CRUSH rule 0 x 556 [15,6,3,13,11,4,1,9]
+ CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+ CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+ CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+ CRUSH rule 0 x 560 [4,9,12,6,3,10,1,15]
+ CRUSH rule 0 x 561 [12,7,1,2,5,15,11,9]
+ CRUSH rule 0 x 562 [7,13,9,14,2,1,11,4]
+ CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+ CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+ CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+ CRUSH rule 0 x 566 [6,14,4,2,13,11]
+ CRUSH rule 0 x 567 [15,4,11,6,3,12]
+ CRUSH rule 0 x 568 [4,14,1,6,10,13,3,9]
+ CRUSH rule 0 x 569 [11,3,15,13,5,1,9,7]
+ CRUSH rule 0 x 570 [1,10,13,4,7,2,9,14]
+ CRUSH rule 0 x 571 [10,12,14,9,4,2,1,6]
+ CRUSH rule 0 x 572 [12,14,3,10,6,1,4,9]
+ CRUSH rule 0 x 573 [7,15,11,2,12,9,4,1]
+ CRUSH rule 0 x 574 [11,14,13,1,3,7,4,9]
+ CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+ CRUSH rule 0 x 576 [3,15,11,9,1,6,5,13]
+ CRUSH rule 0 x 577 [13,9,6,15,3,11,4,1]
+ CRUSH rule 0 x 578 [4,10,1,2,7,13,14,9]
+ CRUSH rule 0 x 579 [13,1,15,2,10,7,5,9]
+ CRUSH rule 0 x 580 [3,12,4,1,10,15,7,9]
+ CRUSH rule 0 x 581 [7,14,12,10,1,2,9,5]
+ CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+ CRUSH rule 0 x 583 [4,15,1,9,10,12,2,6]
+ CRUSH rule 0 x 584 [10,1,5,13,6,9,2,15]
+ CRUSH rule 0 x 585 [5,3,6,1,11,14,13,9]
+ CRUSH rule 0 x 586 [7,10,14,12,9,3,5,1]
+ CRUSH rule 0 x 587 [11,6,9,4,1,14,13,2]
+ CRUSH rule 0 x 588 [3,12,7,15,4,9,1,10]
+ CRUSH rule 0 x 589 [9,7,12,1,10,3,4,15]
+ CRUSH rule 0 x 590 [12,1,3,9,10,6,4,14]
+ CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+ CRUSH rule 0 x 592 [15,12,9,7,5,2,11,1]
+ CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+ CRUSH rule 0 x 594 [12,14,2,9,7,4,1,11]
+ CRUSH rule 0 x 595 [12,7,10,3,1,14,9,4]
+ CRUSH rule 0 x 596 [2,7,12,11,1,5,15,9]
+ CRUSH rule 0 x 597 [15,1,2,10,7,13,5,9]
+ CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+ CRUSH rule 0 x 599 [13,11,1,5,6,2,15,9]
+ CRUSH rule 0 x 600 [4,12,3,10,9,7,1,14]
+ CRUSH rule 0 x 601 [13,5,15,2,1,7,9,10]
+ CRUSH rule 0 x 602 [3,11,7,1,13,15,5,9]
+ CRUSH rule 0 x 603 [3,1,4,14,10,9,6,12]
+ CRUSH rule 0 x 604 [14,2,6,1,11,13,9,4]
+ CRUSH rule 0 x 605 [2,7,12,5,14,10,1,9]
+ CRUSH rule 0 x 606 [12,15,1,5,7,9,3,11]
+ CRUSH rule 0 x 607 [3,9,10,14,7,1,4,12]
+ CRUSH rule 0 x 608 [13,10,1,7,9,15,5,2]
+ CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+ CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+ CRUSH rule 0 x 611 [13,1,5,3,10,7,15,9]
+ CRUSH rule 0 x 612 [7,1,2,13,9,15,4,11]
+ CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+ CRUSH rule 0 x 614 [9,4,15,3,1,11,6,12]
+ CRUSH rule 0 x 615 [9,4,11,2,1,12,6,15]
+ CRUSH rule 0 x 616 [10,14,1,5,3,6,12,9]
+ CRUSH rule 0 x 617 [15,7,2,11,12,1,9,4]
+ CRUSH rule 0 x 618 [4,2,10,6,14,9,1,12]
+ CRUSH rule 0 x 619 [15,4,3,9,6,1,13,11]
+ CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+ CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+ CRUSH rule 0 x 622 [10,2,13,5,15,9,1,7]
+ CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+ CRUSH rule 0 x 624 [3,9,15,6,10,1,5,12]
+ CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+ CRUSH rule 0 x 626 [10,12,2,1,9,7,5,14]
+ CRUSH rule 0 x 627 [1,12,10,14,3,5,9,7]
+ CRUSH rule 0 x 628 [15,13,11,4,2,1,7,9]
+ CRUSH rule 0 x 629 [5,6,15,12,1,10,3,9]
+ CRUSH rule 0 x 630 [1,4,12,9,3,7,15,11]
+ CRUSH rule 0 x 631 [5,7,1,15,12,11,3,9]
+ CRUSH rule 0 x 632 [12,3,11,9,6,1,15,5]
+ CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+ CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+ CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+ CRUSH rule 0 x 636 [13,6,11,3,15,9,1,4]
+ CRUSH rule 0 x 637 [3,1,10,6,9,12,4,14]
+ CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+ CRUSH rule 0 x 639 [6,9,14,4,3,1,10,13]
+ CRUSH rule 0 x 640 [9,6,1,11,14,2,4,13]
+ CRUSH rule 0 x 641 [10,6,5,14,1,9,12,2]
+ CRUSH rule 0 x 642 [1,15,4,6,2,10,9,12]
+ CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+ CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+ CRUSH rule 0 x 645 [14,2,4,9,10,1,7,13]
+ CRUSH rule 0 x 646 [5,13,14,1,6,9,2,11]
+ CRUSH rule 0 x 647 [10,1,9,13,6,2,14,5]
+ CRUSH rule 0 x 648 [6,5,2,14,11,1,12,9]
+ CRUSH rule 0 x 649 [3,9,13,11,4,14,1,7]
+ CRUSH rule 0 x 650 [10,9,4,15,12,7,1,2]
+ CRUSH rule 0 x 651 [3,9,5,7,14,1,13,11]
+ CRUSH rule 0 x 652 [15,9,4,6,13,1,2,11]
+ CRUSH rule 0 x 653 [11,14,1,3,6,9,12,5]
+ CRUSH rule 0 x 654 [13,6,2,10,15,4,1,9]
+ CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+ CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+ CRUSH rule 0 x 657 [11,15,3,5,7,13,1,9]
+ CRUSH rule 0 x 658 [7,2,10,12,1,4,9,14]
+ CRUSH rule 0 x 659 [2,5,14,6,10,12]
+ CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+ CRUSH rule 0 x 661 [7,15,3,12,11,4,9,1]
+ CRUSH rule 0 x 662 [15,2,12,5,1,10,9,7]
+ CRUSH rule 0 x 663 [14,9,13,10,5,3,1,6]
+ CRUSH rule 0 x 664 [6,10,12,4,9,2,1,15]
+ CRUSH rule 0 x 665 [2,9,12,1,7,10,4,15]
+ CRUSH rule 0 x 666 [12,3,6,1,15,9,10,4]
+ CRUSH rule 0 x 667 [1,9,12,10,2,14,7,4]
+ CRUSH rule 0 x 668 [9,5,1,2,6,11,13,15]
+ CRUSH rule 0 x 669 [9,7,14,5,11,13,1,2]
+ CRUSH rule 0 x 670 [6,10,9,13,1,2,15,4]
+ CRUSH rule 0 x 671 [6,15,5,10,13,3]
+ CRUSH rule 0 x 672 [2,9,13,1,4,14,6,10]
+ CRUSH rule 0 x 673 [7,10,5,9,15,13,2,1]
+ CRUSH rule 0 x 674 [7,12,10,1,14,9,3,4]
+ CRUSH rule 0 x 675 [9,5,1,10,6,14,12,2]
+ CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+ CRUSH rule 0 x 677 [2,12,1,4,10,6,15,9]
+ CRUSH rule 0 x 678 [1,2,4,10,12,14,9,6]
+ CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+ CRUSH rule 0 x 680 [7,11,3,1,15,4,9,12]
+ CRUSH rule 0 x 681 [6,4,3,11,14,13,1,9]
+ CRUSH rule 0 x 682 [6,1,11,15,12,2,5,9]
+ CRUSH rule 0 x 683 [6,13,2,4,9,14,10,1]
+ CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+ CRUSH rule 0 x 685 [5,1,15,7,9,2,10,13]
+ CRUSH rule 0 x 686 [1,9,11,14,6,13,4,3]
+ CRUSH rule 0 x 687 [7,13,3,5,11,9,15,1]
+ CRUSH rule 0 x 688 [11,9,1,14,3,5,7,12]
+ CRUSH rule 0 x 689 [5,2,9,12,1,14,11,7]
+ CRUSH rule 0 x 690 [9,7,10,3,13,15,5,1]
+ CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+ CRUSH rule 0 x 692 [15,5,1,2,9,11,12,7]
+ CRUSH rule 0 x 693 [5,6,12,15,2,10,9,1]
+ CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+ CRUSH rule 0 x 695 [6,13,14,10,9,5,1,3]
+ CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+ CRUSH rule 0 x 697 [13,11,3,6,4,14,9,1]
+ CRUSH rule 0 x 698 [11,13,4,2,6,1,9,15]
+ CRUSH rule 0 x 699 [7,14,12,4,2,11]
+ CRUSH rule 0 x 700 [12,14,11,9,4,6,3,1]
+ CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+ CRUSH rule 0 x 702 [3,12,15,6,5,11,1,9]
+ CRUSH rule 0 x 703 [15,11,13,3,4,7,1,9]
+ CRUSH rule 0 x 704 [6,4,2,15,11,1,13,9]
+ CRUSH rule 0 x 705 [14,6,11,5,1,13,9,3]
+ CRUSH rule 0 x 706 [1,12,3,6,4,10,15,9]
+ CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+ CRUSH rule 0 x 708 [3,10,5,1,15,9,7,13]
+ CRUSH rule 0 x 709 [11,12,3,7,5,14,1,9]
+ CRUSH rule 0 x 710 [14,2,11,9,5,7,12,1]
+ CRUSH rule 0 x 711 [14,3,9,10,12,5,6,1]
+ CRUSH rule 0 x 712 [12,3,11,15,9,1,6,4]
+ CRUSH rule 0 x 713 [11,9,3,15,13,6,4,1]
+ CRUSH rule 0 x 714 [12,1,9,7,2,15,10,5]
+ CRUSH rule 0 x 715 [6,1,14,4,11,12,3,9]
+ CRUSH rule 0 x 716 [11,13,9,14,5,2,1,7]
+ CRUSH rule 0 x 717 [12,4,10,9,15,1,2,7]
+ CRUSH rule 0 x 718 [7,15,5,2,11,13]
+ CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+ CRUSH rule 0 x 720 [4,13,10,2,7,9,1,14]
+ CRUSH rule 0 x 721 [11,3,14,9,1,12,4,6]
+ CRUSH rule 0 x 722 [2,4,6,1,9,15,13,10]
+ CRUSH rule 0 x 723 [2,1,12,15,11,7,5,9]
+ CRUSH rule 0 x 724 [7,1,9,10,5,15,13,2]
+ CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+ CRUSH rule 0 x 726 [7,14,4,3,11,13,9,1]
+ CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+ CRUSH rule 0 x 728 [13,11,4,6,15,2]
+ CRUSH rule 0 x 729 [15,11,4,6,2,9,1,13]
+ CRUSH rule 0 x 730 [3,7,1,13,11,15,9,5]
+ CRUSH rule 0 x 731 [9,1,6,5,2,11,13,15]
+ CRUSH rule 0 x 732 [1,2,10,13,9,4,7,15]
+ CRUSH rule 0 x 733 [11,3,5,6,1,9,12,15]
+ CRUSH rule 0 x 734 [14,3,11,7,12,9,4,1]
+ CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+ CRUSH rule 0 x 736 [3,9,1,11,7,5,13,14]
+ CRUSH rule 0 x 737 [1,4,2,12,9,10,6,15]
+ CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+ CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+ CRUSH rule 0 x 740 [7,9,10,13,1,15,2,5]
+ CRUSH rule 0 x 741 [12,11,7,15,2,5]
+ CRUSH rule 0 x 742 [9,7,4,11,12,1,14,3]
+ CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+ CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+ CRUSH rule 0 x 745 [3,6,1,4,11,12,14,9]
+ CRUSH rule 0 x 746 [3,7,9,10,14,5,1,13]
+ CRUSH rule 0 x 747 [15,11,5,2,13,9,1,7]
+ CRUSH rule 0 x 748 [6,10,13,2,14,5,9,1]
+ CRUSH rule 0 x 749 [14,9,10,7,5,1,2,12]
+ CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+ CRUSH rule 0 x 751 [15,1,6,9,5,11,12,3]
+ CRUSH rule 0 x 752 [13,1,7,3,11,15,9,4]
+ CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+ CRUSH rule 0 x 754 [14,12,11,4,2,1,9,6]
+ CRUSH rule 0 x 755 [13,6,1,10,4,2,14,9]
+ CRUSH rule 0 x 756 [3,4,14,6,1,10,13,9]
+ CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+ CRUSH rule 0 x 758 [6,3,4,10,15,13,9,1]
+ CRUSH rule 0 x 759 [5,7,3,14,11,1,9,13]
+ CRUSH rule 0 x 760 [1,15,10,12,4,3,9,7]
+ CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+ CRUSH rule 0 x 762 [1,4,10,9,3,7,14,12]
+ CRUSH rule 0 x 763 [4,13,1,14,7,10,2,9]
+ CRUSH rule 0 x 764 [1,14,6,13,9,5,2,10]
+ CRUSH rule 0 x 765 [9,15,2,13,4,1,11,7]
+ CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+ CRUSH rule 0 x 767 [6,11,4,3,12,14]
+ CRUSH rule 0 x 768 [2,12,15,7,1,11,9,4]
+ CRUSH rule 0 x 769 [15,1,9,2,11,12,7,4]
+ CRUSH rule 0 x 770 [15,13,4,6,3,10,1,9]
+ CRUSH rule 0 x 771 [9,2,12,11,6,14,5,1]
+ CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+ CRUSH rule 0 x 773 [3,7,4,15,1,12,11,9]
+ CRUSH rule 0 x 774 [12,6,3,15,5,9,10,1]
+ CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+ CRUSH rule 0 x 776 [10,15,3,9,6,13,1,5]
+ CRUSH rule 0 x 777 [11,13,4,7,1,14,9,2]
+ CRUSH rule 0 x 778 [13,1,9,11,15,6,3,5]
+ CRUSH rule 0 x 779 [5,11,1,14,2,9,13,6]
+ CRUSH rule 0 x 780 [13,9,3,6,4,1,14,10]
+ CRUSH rule 0 x 781 [5,7,14,3,1,12,11,9]
+ CRUSH rule 0 x 782 [2,15,9,7,11,13,4,1]
+ CRUSH rule 0 x 783 [12,7,5,14,9,1,2,10]
+ CRUSH rule 0 x 784 [14,1,10,13,3,4,7,9]
+ CRUSH rule 0 x 785 [6,12,1,2,4,9,15,10]
+ CRUSH rule 0 x 786 [10,5,2,15,1,7,12,9]
+ CRUSH rule 0 x 787 [1,12,10,2,9,4,14,6]
+ CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+ CRUSH rule 0 x 789 [9,2,14,7,4,12,1,10]
+ CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+ CRUSH rule 0 x 791 [9,4,7,13,14,11,1,3]
+ CRUSH rule 0 x 792 [6,4,15,10,12,3]
+ CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+ CRUSH rule 0 x 794 [5,12,2,14,9,10,1,6]
+ CRUSH rule 0 x 795 [6,14,12,4,10,1,2,9]
+ CRUSH rule 0 x 796 [11,2,12,6,15,4]
+ CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+ CRUSH rule 0 x 798 [5,11,6,13,1,3,15,9]
+ CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+ CRUSH rule 0 x 800 [6,3,4,11,15,13]
+ CRUSH rule 0 x 801 [2,5,6,13,9,1,10,15]
+ CRUSH rule 0 x 802 [1,4,12,7,3,9,10,14]
+ CRUSH rule 0 x 803 [7,2,4,1,11,13,9,14]
+ CRUSH rule 0 x 804 [5,14,9,7,3,1,12,10]
+ CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+ CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+ CRUSH rule 0 x 807 [14,2,7,4,9,12,1,10]
+ CRUSH rule 0 x 808 [2,15,12,7,9,1,5,10]
+ CRUSH rule 0 x 809 [1,11,7,12,4,2,15,9]
+ CRUSH rule 0 x 810 [2,5,9,12,15,1,7,11]
+ CRUSH rule 0 x 811 [15,6,3,10,1,5,9,12]
+ CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+ CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+ CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+ CRUSH rule 0 x 815 [15,12,9,4,10,6,1,2]
+ CRUSH rule 0 x 816 [14,10,13,7,3,9,4,1]
+ CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+ CRUSH rule 0 x 818 [15,2,11,4,1,12,6,9]
+ CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+ CRUSH rule 0 x 820 [3,6,9,12,11,15,4,1]
+ CRUSH rule 0 x 821 [15,10,9,13,3,4,7,1]
+ CRUSH rule 0 x 822 [10,13,2,9,7,4,14,1]
+ CRUSH rule 0 x 823 [2,6,12,10,15,4,1,9]
+ CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+ CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+ CRUSH rule 0 x 826 [5,2,11,15,1,12,9,7]
+ CRUSH rule 0 x 827 [13,5,1,3,7,9,11,14]
+ CRUSH rule 0 x 828 [12,6,10,5,1,9,2,15]
+ CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+ CRUSH rule 0 x 830 [15,13,2,9,7,11,1,5]
+ CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+ CRUSH rule 0 x 832 [14,11,13,2,9,4,6,1]
+ CRUSH rule 0 x 833 [9,13,3,11,7,5,15,1]
+ CRUSH rule 0 x 834 [9,7,5,1,11,2,13,14]
+ CRUSH rule 0 x 835 [14,3,13,6,4,9,1,10]
+ CRUSH rule 0 x 836 [3,9,10,13,1,5,14,7]
+ CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+ CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+ CRUSH rule 0 x 839 [3,4,6,10,15,1,13,9]
+ CRUSH rule 0 x 840 [10,15,12,4,7,1,2,9]
+ CRUSH rule 0 x 841 [3,5,7,12,11,15,1,9]
+ CRUSH rule 0 x 842 [9,13,2,6,5,14,10,1]
+ CRUSH rule 0 x 843 [14,7,4,9,3,12,1,10]
+ CRUSH rule 0 x 844 [7,1,4,15,9,2,11,12]
+ CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+ CRUSH rule 0 x 846 [3,7,15,13,1,9,10,4]
+ CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+ CRUSH rule 0 x 848 [11,13,1,14,5,9,2,7]
+ CRUSH rule 0 x 849 [3,15,11,9,6,1,13,5]
+ CRUSH rule 0 x 850 [1,3,10,6,14,4,9,12]
+ CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+ CRUSH rule 0 x 852 [9,12,4,7,15,2,11,1]
+ CRUSH rule 0 x 853 [13,14,6,11,2,4,9,1]
+ CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+ CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+ CRUSH rule 0 x 856 [5,10,7,3,15,9,12,1]
+ CRUSH rule 0 x 857 [4,3,13,11,9,1,7,14]
+ CRUSH rule 0 x 858 [5,15,6,3,9,12,1,10]
+ CRUSH rule 0 x 859 [5,15,6,2,1,11,12,9]
+ CRUSH rule 0 x 860 [11,14,1,12,6,9,2,4]
+ CRUSH rule 0 x 861 [13,7,4,10,1,14,3,9]
+ CRUSH rule 0 x 862 [5,10,9,7,3,12,1,15]
+ CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+ CRUSH rule 0 x 864 [6,13,4,2,10,15,1,9]
+ CRUSH rule 0 x 865 [4,1,14,11,6,9,3,13]
+ CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+ CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+ CRUSH rule 0 x 868 [14,11,7,2,1,4,9,12]
+ CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+ CRUSH rule 0 x 870 [14,9,11,4,3,12,6,1]
+ CRUSH rule 0 x 871 [6,2,1,4,15,13,11,9]
+ CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+ CRUSH rule 0 x 873 [2,5,12,10,1,9,15,7]
+ CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+ CRUSH rule 0 x 875 [10,6,14,1,12,5,9,3]
+ CRUSH rule 0 x 876 [14,7,13,3,9,1,11,4]
+ CRUSH rule 0 x 877 [15,11,13,9,5,1,6,3]
+ CRUSH rule 0 x 878 [7,14,3,13,9,1,11,4]
+ CRUSH rule 0 x 879 [12,2,7,4,10,15]
+ CRUSH rule 0 x 880 [2,12,10,7,1,4,9,14]
+ CRUSH rule 0 x 881 [6,3,1,11,4,15,9,13]
+ CRUSH rule 0 x 882 [11,13,7,1,2,15,4,9]
+ CRUSH rule 0 x 883 [13,1,3,10,6,5,9,15]
+ CRUSH rule 0 x 884 [6,15,4,9,3,11,12,1]
+ CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+ CRUSH rule 0 x 886 [13,11,4,2,1,14,9,6]
+ CRUSH rule 0 x 887 [14,4,12,11,2,6,9,1]
+ CRUSH rule 0 x 888 [10,12,7,15,9,2,1,5]
+ CRUSH rule 0 x 889 [15,13,4,1,6,2,10,9]
+ CRUSH rule 0 x 890 [10,12,14,2,9,5,6,1]
+ CRUSH rule 0 x 891 [9,5,11,6,3,15,12,1]
+ CRUSH rule 0 x 892 [12,15,2,4,7,9,11,1]
+ CRUSH rule 0 x 893 [1,3,5,9,6,10,14,12]
+ CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+ CRUSH rule 0 x 895 [2,1,11,5,7,15,13,9]
+ CRUSH rule 0 x 896 [9,1,14,10,4,12,2,7]
+ CRUSH rule 0 x 897 [7,5,14,3,1,9,11,12]
+ CRUSH rule 0 x 898 [10,6,12,9,15,5,2,1]
+ CRUSH rule 0 x 899 [1,11,5,3,13,14,9,6]
+ CRUSH rule 0 x 900 [2,9,10,7,13,14,5,1]
+ CRUSH rule 0 x 901 [9,12,11,3,14,4,1,6]
+ CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+ CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+ CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+ CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+ CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+ CRUSH rule 0 x 907 [7,12,3,9,10,5,14,1]
+ CRUSH rule 0 x 908 [2,15,9,6,10,13,5,1]
+ CRUSH rule 0 x 909 [10,14,1,13,2,9,7,4]
+ CRUSH rule 0 x 910 [12,7,4,15,10,3,1,9]
+ CRUSH rule 0 x 911 [11,15,2,4,9,13,6,1]
+ CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+ CRUSH rule 0 x 913 [4,6,10,1,12,3,9,14]
+ CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+ CRUSH rule 0 x 915 [12,14,1,9,4,3,11,6]
+ CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+ CRUSH rule 0 x 917 [1,15,6,5,10,3,13,9]
+ CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+ CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+ CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+ CRUSH rule 0 x 921 [1,11,6,13,4,2,9,14]
+ CRUSH rule 0 x 922 [6,4,14,13,3,1,10,9]
+ CRUSH rule 0 x 923 [12,2,5,14,10,1,9,6]
+ CRUSH rule 0 x 924 [6,2,14,13,9,1,11,5]
+ CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+ CRUSH rule 0 x 926 [3,13,10,1,14,9,6,5]
+ CRUSH rule 0 x 927 [6,5,1,11,14,2,13,9]
+ CRUSH rule 0 x 928 [13,1,3,9,6,11,15,5]
+ CRUSH rule 0 x 929 [10,7,1,5,2,12,9,14]
+ CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+ CRUSH rule 0 x 931 [6,15,11,9,5,3,1,13]
+ CRUSH rule 0 x 932 [13,2,5,11,9,1,6,15]
+ CRUSH rule 0 x 933 [12,7,14,10,4,1,2,9]
+ CRUSH rule 0 x 934 [12,2,5,7,9,1,15,11]
+ CRUSH rule 0 x 935 [6,11,1,14,5,13,3,9]
+ CRUSH rule 0 x 936 [9,12,7,5,1,2,14,11]
+ CRUSH rule 0 x 937 [14,2,11,1,13,4,9,6]
+ CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+ CRUSH rule 0 x 939 [6,4,14,9,12,1,11,2]
+ CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+ CRUSH rule 0 x 941 [3,12,4,7,14,10]
+ CRUSH rule 0 x 942 [15,12,10,4,1,9,3,7]
+ CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+ CRUSH rule 0 x 944 [2,9,4,7,1,14,12,11]
+ CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+ CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+ CRUSH rule 0 x 947 [11,3,14,1,12,5,6,9]
+ CRUSH rule 0 x 948 [7,13,11,5,14,2,1,9]
+ CRUSH rule 0 x 949 [9,1,12,5,15,10,2,6]
+ CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+ CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+ CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+ CRUSH rule 0 x 953 [1,3,6,10,12,14,4,9]
+ CRUSH rule 0 x 954 [10,2,14,9,4,6,12,1]
+ CRUSH rule 0 x 955 [7,14,3,1,10,4,9,12]
+ CRUSH rule 0 x 956 [1,6,11,5,14,3,9,13]
+ CRUSH rule 0 x 957 [14,11,1,12,6,9,4,3]
+ CRUSH rule 0 x 958 [15,4,3,11,1,6,12,9]
+ CRUSH rule 0 x 959 [2,1,12,15,10,9,4,6]
+ CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+ CRUSH rule 0 x 961 [3,13,11,9,6,1,4,15]
+ CRUSH rule 0 x 962 [5,11,3,14,1,6,13,9]
+ CRUSH rule 0 x 963 [13,10,15,4,6,9,1,3]
+ CRUSH rule 0 x 964 [7,11,4,9,2,12,1,15]
+ CRUSH rule 0 x 965 [12,2,9,7,4,15,11,1]
+ CRUSH rule 0 x 966 [12,14,9,4,1,2,11,7]
+ CRUSH rule 0 x 967 [7,5,3,10,12,14]
+ CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+ CRUSH rule 0 x 969 [11,4,7,1,9,14,13,2]
+ CRUSH rule 0 x 970 [5,12,10,1,3,14,9,6]
+ CRUSH rule 0 x 971 [1,9,4,12,7,2,10,15]
+ CRUSH rule 0 x 972 [12,3,14,5,1,9,7,11]
+ CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+ CRUSH rule 0 x 974 [7,11,1,2,15,4,12,9]
+ CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+ CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+ CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+ CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+ CRUSH rule 0 x 979 [5,1,13,6,15,10,3,9]
+ CRUSH rule 0 x 980 [15,11,5,6,1,3,13,9]
+ CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+ CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+ CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+ CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+ CRUSH rule 0 x 985 [11,2,15,1,4,13,6,9]
+ CRUSH rule 0 x 986 [6,13,9,1,15,10,5,2]
+ CRUSH rule 0 x 987 [13,14,5,10,6,1,3,9]
+ CRUSH rule 0 x 988 [12,9,10,14,3,1,4,7]
+ CRUSH rule 0 x 989 [7,4,3,15,9,13,10,1]
+ CRUSH rule 0 x 990 [1,10,9,13,3,4,6,15]
+ CRUSH rule 0 x 991 [7,11,1,14,2,5,9,12]
+ CRUSH rule 0 x 992 [9,10,2,13,7,4,1,15]
+ CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+ CRUSH rule 0 x 994 [3,13,15,4,11,7,1,9]
+ CRUSH rule 0 x 995 [15,6,12,2,5,11]
+ CRUSH rule 0 x 996 [15,10,5,3,13,1,9,7]
+ CRUSH rule 0 x 997 [15,2,1,12,7,9,4,10]
+ CRUSH rule 0 x 998 [6,1,9,5,12,11,15,2]
+ CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+ CRUSH rule 0 x 1000 [14,2,9,4,12,1,6,11]
+ CRUSH rule 0 x 1001 [11,14,4,2,6,9,1,13]
+ CRUSH rule 0 x 1002 [1,10,14,2,9,5,13,7]
+ CRUSH rule 0 x 1003 [10,7,5,14,2,1,9,12]
+ CRUSH rule 0 x 1004 [15,1,4,6,10,12,9,3]
+ CRUSH rule 0 x 1005 [6,12,2,10,9,15,5,1]
+ CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+ CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+ CRUSH rule 0 x 1008 [7,4,9,11,3,15,1,13]
+ CRUSH rule 0 x 1009 [5,2,11,7,15,9,1,12]
+ CRUSH rule 0 x 1010 [10,2,15,6,9,13,4,1]
+ CRUSH rule 0 x 1011 [6,3,12,1,10,4,9,14]
+ CRUSH rule 0 x 1012 [12,6,9,15,3,1,5,11]
+ CRUSH rule 0 x 1013 [2,14,12,4,9,1,6,10]
+ CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+ CRUSH rule 0 x 1015 [12,6,10,1,4,15,9,2]
+ CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+ CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+ CRUSH rule 0 x 1018 [13,11,14,1,9,3,5,7]
+ CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+ CRUSH rule 0 x 1020 [3,1,13,4,10,9,14,6]
+ CRUSH rule 0 x 1021 [2,11,14,9,4,6,1,13]
+ CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+ CRUSH rule 0 x 1023 [15,2,9,12,1,7,4,11]
+ rule 0 (replicated_ruleset) num_rep 8 result size == 6:\t36/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 8 result size == 7:\t264/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 8 result size == 8:\t724/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15,12,1,4,9]
+ CRUSH rule 0 x 1 [10,15,1,2,13,4,7,9]
+ CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+ CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+ CRUSH rule 0 x 4 [14,2,10,1,9,4,7,13]
+ CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+ CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+ CRUSH rule 0 x 7 [9,2,6,12,11,4,1,14]
+ CRUSH rule 0 x 8 [10,2,15,1,4,13,6,9]
+ CRUSH rule 0 x 9 [7,1,14,2,11,9,12,4]
+ CRUSH rule 0 x 10 [10,14,4,1,2,7,13,9]
+ CRUSH rule 0 x 11 [13,9,14,7,5,11,2,1]
+ CRUSH rule 0 x 12 [7,1,2,5,13,15,11,9]
+ CRUSH rule 0 x 13 [3,5,12,7,9,1,14,11]
+ CRUSH rule 0 x 14 [13,5,2,7,10,15,1,9]
+ CRUSH rule 0 x 15 [15,1,9,6,13,3,5,11]
+ CRUSH rule 0 x 16 [7,11,14,2,13,1,9,4]
+ CRUSH rule 0 x 17 [10,1,13,2,4,6,14,9]
+ CRUSH rule 0 x 18 [1,7,3,10,5,12,9,14]
+ CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+ CRUSH rule 0 x 20 [14,12,3,10,9,4,7,1]
+ CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+ CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+ CRUSH rule 0 x 23 [10,5,13,9,3,15,1,6]
+ CRUSH rule 0 x 24 [12,11,3,1,9,4,7,15]
+ CRUSH rule 0 x 25 [7,12,15,1,3,10,4,9]
+ CRUSH rule 0 x 26 [1,7,13,2,14,5,9,11]
+ CRUSH rule 0 x 27 [3,6,15,4,13,9,11,1]
+ CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+ CRUSH rule 0 x 29 [5,14,12,11,6,3,1,9]
+ CRUSH rule 0 x 30 [2,5,6,9,1,11,13,14]
+ CRUSH rule 0 x 31 [5,15,10,1,9,13,6,3]
+ CRUSH rule 0 x 32 [9,10,2,1,13,14,6,4]
+ CRUSH rule 0 x 33 [13,4,9,2,7,1,10,14]
+ CRUSH rule 0 x 34 [13,15,2,4,1,10,9,6]
+ CRUSH rule 0 x 35 [4,14,3,13,10,9,1,6]
+ CRUSH rule 0 x 36 [3,12,9,7,5,10,14,1]
+ CRUSH rule 0 x 37 [9,2,6,14,11,1,4,13]
+ CRUSH rule 0 x 38 [3,4,13,10,9,1,14,6]
+ CRUSH rule 0 x 39 [12,7,14,11,1,9,5,2]
+ CRUSH rule 0 x 40 [10,1,9,5,15,2,6,13]
+ CRUSH rule 0 x 41 [4,9,11,1,14,13,6,3]
+ CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+ CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+ CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+ CRUSH rule 0 x 45 [11,12,15,9,1,5,6,3]
+ CRUSH rule 0 x 46 [6,9,2,14,11,13,1,5]
+ CRUSH rule 0 x 47 [3,9,6,4,13,1,11,15]
+ CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+ CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+ CRUSH rule 0 x 50 [14,12,1,4,2,11,6,9]
+ CRUSH rule 0 x 51 [10,6,5,12,15,2,1,9]
+ CRUSH rule 0 x 52 [12,1,9,11,7,3,14,4]
+ CRUSH rule 0 x 53 [3,6,13,9,5,1,11,15]
+ CRUSH rule 0 x 54 [4,13,9,2,14,10,6,1]
+ CRUSH rule 0 x 55 [4,11,2,7,1,13,9,15]
+ CRUSH rule 0 x 56 [5,9,10,1,3,13,14,7]
+ CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+ CRUSH rule 0 x 58 [7,1,11,4,3,14,12,9]
+ CRUSH rule 0 x 59 [2,13,1,10,9,5,14,6]
+ CRUSH rule 0 x 60 [3,6,11,1,4,9,12,15]
+ CRUSH rule 0 x 61 [3,15,13,7,4,1,10,9]
+ CRUSH rule 0 x 62 [15,11,7,12,5,9,2,1]
+ CRUSH rule 0 x 63 [10,14,12,1,7,3]
+ CRUSH rule 0 x 64 [3,9,1,4,7,12,11,15]
+ CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+ CRUSH rule 0 x 66 [15,11,6,9,4,1,3,12]
+ CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+ CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+ CRUSH rule 0 x 69 [2,1,15,10,4,9,13,7]
+ CRUSH rule 0 x 70 [9,6,1,3,13,15,11,5]
+ CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+ CRUSH rule 0 x 72 [9,10,3,5,7,12,15,1]
+ CRUSH rule 0 x 73 [5,3,11,1,7,12,15,9]
+ CRUSH rule 0 x 74 [11,7,9,5,1,15,3,12]
+ CRUSH rule 0 x 75 [9,7,11,14,12,1,2,5]
+ CRUSH rule 0 x 76 [6,1,3,5,14,10,12,9]
+ CRUSH rule 0 x 77 [7,4,2,13,9,1,11,15]
+ CRUSH rule 0 x 78 [9,3,1,5,6,13,14,11]
+ CRUSH rule 0 x 79 [13,2,15,5,7,9,11,1]
+ CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+ CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+ CRUSH rule 0 x 82 [14,13,5,11,6,2,1,9]
+ CRUSH rule 0 x 83 [4,15,3,9,10,13,6,1]
+ CRUSH rule 0 x 84 [10,7,9,15,3,4,1,13]
+ CRUSH rule 0 x 85 [3,15,9,7,4,11,1,12]
+ CRUSH rule 0 x 86 [10,9,14,1,13,4,2,7]
+ CRUSH rule 0 x 87 [15,10,7,12,5,3,9,1]
+ CRUSH rule 0 x 88 [4,13,3,1,9,15,11,7]
+ CRUSH rule 0 x 89 [3,9,7,4,1,14,10,12]
+ CRUSH rule 0 x 90 [4,9,7,12,11,14,2,1]
+ CRUSH rule 0 x 91 [6,11,9,1,2,4,14,13]
+ CRUSH rule 0 x 92 [1,5,10,9,13,15,6,2]
+ CRUSH rule 0 x 93 [9,3,15,13,7,5,1,10]
+ CRUSH rule 0 x 94 [9,2,12,5,6,11,1,14]
+ CRUSH rule 0 x 95 [7,15,4,10,9,13,2,1]
+ CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+ CRUSH rule 0 x 97 [4,11,2,13,1,7,9,14]
+ CRUSH rule 0 x 98 [11,13,9,3,15,1,5,6]
+ CRUSH rule 0 x 99 [12,4,11,7,3,14,9,1]
+ CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+ CRUSH rule 0 x 101 [15,7,1,9,10,5,2,12]
+ CRUSH rule 0 x 102 [3,11,14,6,13,4,9,1]
+ CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+ CRUSH rule 0 x 104 [14,6,3,5,9,1,10,13]
+ CRUSH rule 0 x 105 [14,10,1,9,3,5,6,13]
+ CRUSH rule 0 x 106 [6,5,13,2,14,11,1,9]
+ CRUSH rule 0 x 107 [3,1,10,14,13,5,9,6]
+ CRUSH rule 0 x 108 [5,10,7,2,15,9,12,1]
+ CRUSH rule 0 x 109 [9,1,13,7,15,5,11,3]
+ CRUSH rule 0 x 110 [5,1,11,3,7,14,13,9]
+ CRUSH rule 0 x 111 [10,1,9,7,5,2,13,14]
+ CRUSH rule 0 x 112 [1,10,4,14,2,12,6,9]
+ CRUSH rule 0 x 113 [6,10,13,9,1,5,2,14]
+ CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+ CRUSH rule 0 x 115 [10,13,14,3,9,1,6,5]
+ CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+ CRUSH rule 0 x 117 [5,6,1,12,15,9,11,3]
+ CRUSH rule 0 x 118 [10,4,13,15,9,3,1,6]
+ CRUSH rule 0 x 119 [14,12,11,4,6,9,3,1]
+ CRUSH rule 0 x 120 [11,3,14,13,4,7]
+ CRUSH rule 0 x 121 [9,5,1,11,7,3,15,12]
+ CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+ CRUSH rule 0 x 123 [3,10,5,6,9,1,12,15]
+ CRUSH rule 0 x 124 [12,2,1,5,14,7,11,9]
+ CRUSH rule 0 x 125 [9,12,15,1,6,5,3,10]
+ CRUSH rule 0 x 126 [7,15,10,9,2,12,5,1]
+ CRUSH rule 0 x 127 [4,14,9,13,1,3,7,10]
+ CRUSH rule 0 x 128 [3,12,1,10,4,9,7,14]
+ CRUSH rule 0 x 129 [11,13,14,2,9,4,6,1]
+ CRUSH rule 0 x 130 [3,13,5,14,10,1,9,6]
+ CRUSH rule 0 x 131 [12,1,6,15,4,2,11,9]
+ CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+ CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+ CRUSH rule 0 x 134 [12,5,6,15,3,9,10,1]
+ CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+ CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+ CRUSH rule 0 x 137 [14,3,6,11,1,9,4,13]
+ CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+ CRUSH rule 0 x 139 [11,2,13,9,1,15,7,5]
+ CRUSH rule 0 x 140 [11,4,12,15,2,6,9,1]
+ CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+ CRUSH rule 0 x 142 [3,14,7,9,11,1,4,13]
+ CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+ CRUSH rule 0 x 144 [13,7,11,2,14,4,1,9]
+ CRUSH rule 0 x 145 [12,2,6,10,9,4,14,1]
+ CRUSH rule 0 x 146 [1,5,9,2,6,13,14,10]
+ CRUSH rule 0 x 147 [1,4,9,11,2,7,15,12]
+ CRUSH rule 0 x 148 [12,7,9,2,14,11,1,4]
+ CRUSH rule 0 x 149 [2,5,9,12,11,1,7,14]
+ CRUSH rule 0 x 150 [1,15,2,10,7,9,5,12]
+ CRUSH rule 0 x 151 [2,9,14,7,1,10,5,13]
+ CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+ CRUSH rule 0 x 153 [6,9,4,15,2,1,10,12]
+ CRUSH rule 0 x 154 [3,11,7,1,4,12,15,9]
+ CRUSH rule 0 x 155 [14,12,7,3,5,1,9,11]
+ CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+ CRUSH rule 0 x 157 [15,1,6,4,3,10,9,13]
+ CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+ CRUSH rule 0 x 159 [4,14,3,12,10,6,1,9]
+ CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+ CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+ CRUSH rule 0 x 162 [10,6,1,12,2,4,14,9]
+ CRUSH rule 0 x 163 [15,1,10,2,6,4,13,9]
+ CRUSH rule 0 x 164 [9,14,10,7,12,2,5,1]
+ CRUSH rule 0 x 165 [11,7,2,13,9,15,1,5]
+ CRUSH rule 0 x 166 [1,2,12,14,4,11,7,9]
+ CRUSH rule 0 x 167 [9,7,3,4,11,13,15,1]
+ CRUSH rule 0 x 168 [13,2,4,1,6,15,10,9]
+ CRUSH rule 0 x 169 [1,4,9,14,13,10,2,7]
+ CRUSH rule 0 x 170 [1,15,7,9,12,10,3,5]
+ CRUSH rule 0 x 171 [9,2,10,7,1,5,15,12]
+ CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+ CRUSH rule 0 x 173 [5,10,12,15,6,1,2,9]
+ CRUSH rule 0 x 174 [15,6,4,12,1,11,9,2]
+ CRUSH rule 0 x 175 [5,7,9,3,10,1,14,13]
+ CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+ CRUSH rule 0 x 177 [2,9,10,13,4,1,14,7]
+ CRUSH rule 0 x 178 [12,11,7,14,3,4]
+ CRUSH rule 0 x 179 [2,10,13,9,5,1,7,14]
+ CRUSH rule 0 x 180 [3,11,5,15,7,12]
+ CRUSH rule 0 x 181 [9,12,6,5,1,10,14,2]
+ CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+ CRUSH rule 0 x 183 [5,7,10,13,3,9,14,1]
+ CRUSH rule 0 x 184 [2,5,11,12,7,1,9,15]
+ CRUSH rule 0 x 185 [13,5,7,11,2,14]
+ CRUSH rule 0 x 186 [6,14,13,5,10,1,3,9]
+ CRUSH rule 0 x 187 [1,4,11,13,6,14,9,3]
+ CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+ CRUSH rule 0 x 189 [6,12,4,9,2,1,11,15]
+ CRUSH rule 0 x 190 [9,13,15,10,3,1,5,6]
+ CRUSH rule 0 x 191 [7,11,4,1,15,12,9,2]
+ CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+ CRUSH rule 0 x 193 [3,13,6,10,4,1,9,14]
+ CRUSH rule 0 x 194 [3,13,4,14,6,9,1,11]
+ CRUSH rule 0 x 195 [5,7,10,12,1,3,15,9]
+ CRUSH rule 0 x 196 [4,15,1,10,9,2,13,7]
+ CRUSH rule 0 x 197 [14,10,13,4,6,3,1,9]
+ CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+ CRUSH rule 0 x 199 [2,10,4,15,1,9,6,12]
+ CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+ CRUSH rule 0 x 201 [9,14,1,7,4,3,10,13]
+ CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+ CRUSH rule 0 x 203 [12,5,7,15,1,2,10,9]
+ CRUSH rule 0 x 204 [6,11,3,12,14,1,9,4]
+ CRUSH rule 0 x 205 [15,4,6,10,13,9,2,1]
+ CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+ CRUSH rule 0 x 207 [2,11,7,4,14,1,12,9]
+ CRUSH rule 0 x 208 [13,1,6,14,9,11,3,5]
+ CRUSH rule 0 x 209 [6,15,13,1,11,4,9,2]
+ CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+ CRUSH rule 0 x 211 [2,14,1,13,11,7,9,5]
+ CRUSH rule 0 x 212 [10,1,12,15,5,6,2,9]
+ CRUSH rule 0 x 213 [3,9,6,5,15,13,1,11]
+ CRUSH rule 0 x 214 [7,15,4,1,10,2,13,9]
+ CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+ CRUSH rule 0 x 216 [12,9,6,2,1,11,5,15]
+ CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+ CRUSH rule 0 x 218 [12,10,15,6,1,4,9,2]
+ CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+ CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+ CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+ CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+ CRUSH rule 0 x 223 [9,7,11,1,4,14,13,3]
+ CRUSH rule 0 x 224 [1,7,10,2,12,9,14,5]
+ CRUSH rule 0 x 225 [10,5,2,6,1,13,9,15]
+ CRUSH rule 0 x 226 [4,1,9,3,13,10,15,7]
+ CRUSH rule 0 x 227 [7,2,12,15,5,11]
+ CRUSH rule 0 x 228 [2,15,11,1,6,13,9,4]
+ CRUSH rule 0 x 229 [9,3,7,14,1,12,4,10]
+ CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+ CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+ CRUSH rule 0 x 232 [10,5,13,1,9,2,7,14]
+ CRUSH rule 0 x 233 [6,12,11,4,9,14,1,3]
+ CRUSH rule 0 x 234 [10,1,2,12,5,9,15,7]
+ CRUSH rule 0 x 235 [13,14,7,10,1,9,5,3]
+ CRUSH rule 0 x 236 [2,15,9,12,1,7,4,10]
+ CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+ CRUSH rule 0 x 238 [2,10,4,15,6,12,9,1]
+ CRUSH rule 0 x 239 [4,15,10,7,9,13,3,1]
+ CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+ CRUSH rule 0 x 241 [7,9,15,12,1,5,2,10]
+ CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+ CRUSH rule 0 x 243 [2,11,5,1,15,6,9,13]
+ CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+ CRUSH rule 0 x 245 [12,9,15,3,1,5,10,7]
+ CRUSH rule 0 x 246 [15,3,5,11,7,1,12,9]
+ CRUSH rule 0 x 247 [6,4,9,12,1,2,10,14]
+ CRUSH rule 0 x 248 [5,13,7,11,9,15,3,1]
+ CRUSH rule 0 x 249 [10,14,7,3,9,13,1,4]
+ CRUSH rule 0 x 250 [12,15,1,10,5,6,3,9]
+ CRUSH rule 0 x 251 [13,2,15,5,6,1,9,10]
+ CRUSH rule 0 x 252 [7,5,13,9,3,10,14,1]
+ CRUSH rule 0 x 253 [3,13,15,10,7,4]
+ CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+ CRUSH rule 0 x 255 [1,9,13,2,6,10,4,15]
+ CRUSH rule 0 x 256 [6,9,13,1,3,14,5,10]
+ CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+ CRUSH rule 0 x 258 [12,5,6,10,2,1,14,9]
+ CRUSH rule 0 x 259 [9,10,4,3,14,13,1,7]
+ CRUSH rule 0 x 260 [10,12,6,9,3,15,1,4]
+ CRUSH rule 0 x 261 [13,7,2,1,15,5,11,9]
+ CRUSH rule 0 x 262 [15,3,12,7,4,9,1,11]
+ CRUSH rule 0 x 263 [12,6,10,9,5,15,3,1]
+ CRUSH rule 0 x 264 [13,14,11,3,1,4,7,9]
+ CRUSH rule 0 x 265 [12,10,14,5,7,1,9,3]
+ CRUSH rule 0 x 266 [14,7,11,1,2,9,4,12]
+ CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+ CRUSH rule 0 x 268 [4,1,15,12,6,11,3,9]
+ CRUSH rule 0 x 269 [11,1,15,5,13,9,7,2]
+ CRUSH rule 0 x 270 [7,11,12,3,1,14,9,4]
+ CRUSH rule 0 x 271 [4,7,3,13,15,10,9,1]
+ CRUSH rule 0 x 272 [15,5,13,10,6,2]
+ CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+ CRUSH rule 0 x 274 [10,2,5,6,13,9,15,1]
+ CRUSH rule 0 x 275 [10,3,4,7,14,13]
+ CRUSH rule 0 x 276 [5,12,9,2,11,7,15,1]
+ CRUSH rule 0 x 277 [14,3,13,4,1,9,11,7]
+ CRUSH rule 0 x 278 [5,6,14,3,1,11,13,9]
+ CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+ CRUSH rule 0 x 280 [7,3,14,9,1,11,4,13]
+ CRUSH rule 0 x 281 [5,11,14,7,9,13,2,1]
+ CRUSH rule 0 x 282 [2,1,13,14,9,7,5,10]
+ CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+ CRUSH rule 0 x 284 [5,11,7,15,3,13,1,9]
+ CRUSH rule 0 x 285 [15,5,3,1,6,13,11,9]
+ CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+ CRUSH rule 0 x 287 [12,4,9,1,3,11,15,7]
+ CRUSH rule 0 x 288 [4,12,10,7,1,3,14,9]
+ CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+ CRUSH rule 0 x 290 [12,2,5,6,15,9,1,10]
+ CRUSH rule 0 x 291 [7,11,1,14,5,9,2,12]
+ CRUSH rule 0 x 292 [4,10,6,3,14,9,12,1]
+ CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+ CRUSH rule 0 x 294 [9,12,3,14,6,11,5,1]
+ CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+ CRUSH rule 0 x 296 [3,1,13,7,14,9,10,4]
+ CRUSH rule 0 x 297 [6,13,4,14,10,1,2,9]
+ CRUSH rule 0 x 298 [14,9,13,1,4,2,7,10]
+ CRUSH rule 0 x 299 [14,12,11,6,4,2,1,9]
+ CRUSH rule 0 x 300 [15,7,10,5,1,3,13,9]
+ CRUSH rule 0 x 301 [9,11,7,1,13,14,4,2]
+ CRUSH rule 0 x 302 [9,7,1,13,5,10,3,15]
+ CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+ CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+ CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+ CRUSH rule 0 x 306 [10,12,4,6,9,2,15,1]
+ CRUSH rule 0 x 307 [11,12,15,5,6,2,1,9]
+ CRUSH rule 0 x 308 [12,14,10,9,1,2,5,7]
+ CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+ CRUSH rule 0 x 310 [3,1,5,10,14,9,7,12]
+ CRUSH rule 0 x 311 [3,9,7,1,14,13,10,5]
+ CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+ CRUSH rule 0 x 313 [9,15,3,7,5,13,1,11]
+ CRUSH rule 0 x 314 [2,15,9,5,6,12,1,11]
+ CRUSH rule 0 x 315 [15,2,13,1,11,9,6,4]
+ CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+ CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+ CRUSH rule 0 x 318 [4,1,15,11,9,13,6,2]
+ CRUSH rule 0 x 319 [2,15,4,1,11,9,7,12]
+ CRUSH rule 0 x 320 [5,7,13,9,11,2,1,15]
+ CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+ CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+ CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+ CRUSH rule 0 x 324 [5,6,10,15,2,13]
+ CRUSH rule 0 x 325 [9,10,14,5,1,6,2,13]
+ CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+ CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+ CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+ CRUSH rule 0 x 329 [2,6,15,5,9,10,13,1]
+ CRUSH rule 0 x 330 [3,9,11,13,1,6,5,14]
+ CRUSH rule 0 x 331 [12,14,6,3,1,4,10,9]
+ CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+ CRUSH rule 0 x 333 [6,5,3,12,14,10,9,1]
+ CRUSH rule 0 x 334 [4,9,2,12,7,11,15,1]
+ CRUSH rule 0 x 335 [11,7,1,5,13,2,9,15]
+ CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+ CRUSH rule 0 x 337 [15,11,3,7,12,5]
+ CRUSH rule 0 x 338 [10,5,3,6,15,1,9,13]
+ CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+ CRUSH rule 0 x 340 [11,6,12,4,9,3,14,1]
+ CRUSH rule 0 x 341 [7,5,2,10,14,9,1,12]
+ CRUSH rule 0 x 342 [12,14,1,9,2,11,4,7]
+ CRUSH rule 0 x 343 [12,14,9,6,10,2,4,1]
+ CRUSH rule 0 x 344 [9,11,5,2,14,13,1,7]
+ CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+ CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+ CRUSH rule 0 x 347 [10,2,12,6,9,1,14,5]
+ CRUSH rule 0 x 348 [7,9,10,1,14,13,3,4]
+ CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+ CRUSH rule 0 x 350 [13,9,15,4,10,7,2,1]
+ CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+ CRUSH rule 0 x 352 [1,12,11,9,4,7,3,15]
+ CRUSH rule 0 x 353 [10,14,12,2,9,1,4,6]
+ CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+ CRUSH rule 0 x 355 [13,14,6,10,2,5,1,9]
+ CRUSH rule 0 x 356 [15,13,2,9,6,5,1,11]
+ CRUSH rule 0 x 357 [4,11,1,13,3,14,6,9]
+ CRUSH rule 0 x 358 [12,7,2,9,1,14,10,4]
+ CRUSH rule 0 x 359 [5,15,7,11,3,13]
+ CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+ CRUSH rule 0 x 361 [5,3,13,6,1,14,11,9]
+ CRUSH rule 0 x 362 [2,9,11,13,1,6,5,15]
+ CRUSH rule 0 x 363 [7,12,3,9,15,4,1,10]
+ CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+ CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+ CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+ CRUSH rule 0 x 367 [7,13,3,1,5,11,15,9]
+ CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+ CRUSH rule 0 x 369 [7,5,3,13,14,9,11,1]
+ CRUSH rule 0 x 370 [4,7,14,1,2,9,11,12]
+ CRUSH rule 0 x 371 [1,7,12,3,4,15,10,9]
+ CRUSH rule 0 x 372 [10,4,3,14,6,1,12,9]
+ CRUSH rule 0 x 373 [15,5,2,6,13,1,9,10]
+ CRUSH rule 0 x 374 [3,15,12,5,1,6,10,9]
+ CRUSH rule 0 x 375 [5,2,14,1,6,13,11,9]
+ CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+ CRUSH rule 0 x 377 [1,15,2,4,9,11,12,6]
+ CRUSH rule 0 x 378 [9,12,2,15,1,5,11,6]
+ CRUSH rule 0 x 379 [11,2,15,5,7,9,13,1]
+ CRUSH rule 0 x 380 [6,1,12,11,2,9,5,14]
+ CRUSH rule 0 x 381 [15,13,7,5,10,2,1,9]
+ CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+ CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+ CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+ CRUSH rule 0 x 385 [4,6,15,3,10,9,1,13]
+ CRUSH rule 0 x 386 [14,3,11,13,5,6,9,1]
+ CRUSH rule 0 x 387 [1,11,5,7,9,2,14,12]
+ CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+ CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+ CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+ CRUSH rule 0 x 391 [3,4,9,13,7,10,1,14]
+ CRUSH rule 0 x 392 [11,5,14,7,1,9,2,12]
+ CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+ CRUSH rule 0 x 394 [4,9,3,15,13,6,1,11]
+ CRUSH rule 0 x 395 [10,13,5,15,6,9,3,1]
+ CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+ CRUSH rule 0 x 397 [1,14,9,4,12,10,3,7]
+ CRUSH rule 0 x 398 [9,2,1,5,12,6,11,15]
+ CRUSH rule 0 x 399 [5,9,14,3,1,10,13,7]
+ CRUSH rule 0 x 400 [10,6,2,4,15,12,1,9]
+ CRUSH rule 0 x 401 [6,9,11,12,4,3,15,1]
+ CRUSH rule 0 x 402 [4,7,9,2,13,1,15,11]
+ CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+ CRUSH rule 0 x 404 [14,12,7,9,2,1,5,11]
+ CRUSH rule 0 x 405 [9,15,11,2,4,7,13,1]
+ CRUSH rule 0 x 406 [12,14,9,2,7,10,4,1]
+ CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+ CRUSH rule 0 x 408 [7,1,5,2,10,15,13,9]
+ CRUSH rule 0 x 409 [11,2,4,13,1,15,7,9]
+ CRUSH rule 0 x 410 [6,4,14,2,12,9,10,1]
+ CRUSH rule 0 x 411 [13,11,15,6,4,1,9,2]
+ CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+ CRUSH rule 0 x 413 [13,5,3,11,6,9,1,14]
+ CRUSH rule 0 x 414 [3,11,9,13,4,1,6,15]
+ CRUSH rule 0 x 415 [6,10,14,5,1,13,3,9]
+ CRUSH rule 0 x 416 [13,1,4,7,2,9,14,11]
+ CRUSH rule 0 x 417 [4,12,1,15,2,11,9,6]
+ CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+ CRUSH rule 0 x 419 [5,14,10,9,2,12,6,1]
+ CRUSH rule 0 x 420 [2,4,9,11,6,14,13,1]
+ CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+ CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+ CRUSH rule 0 x 423 [3,15,12,6,5,1,9,10]
+ CRUSH rule 0 x 424 [6,10,12,2,5,1,14,9]
+ CRUSH rule 0 x 425 [11,15,2,13,5,7,9,1]
+ CRUSH rule 0 x 426 [12,4,7,1,9,10,14,2]
+ CRUSH rule 0 x 427 [14,10,3,1,9,7,5,13]
+ CRUSH rule 0 x 428 [12,7,9,4,2,1,14,10]
+ CRUSH rule 0 x 429 [3,4,9,7,11,12,1,14]
+ CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+ CRUSH rule 0 x 431 [9,3,7,1,12,5,14,11]
+ CRUSH rule 0 x 432 [4,1,12,7,15,2,10,9]
+ CRUSH rule 0 x 433 [4,11,12,15,7,3]
+ CRUSH rule 0 x 434 [2,14,9,1,5,11,7,13]
+ CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+ CRUSH rule 0 x 436 [9,15,10,2,4,1,12,7]
+ CRUSH rule 0 x 437 [9,6,3,14,10,12,5,1]
+ CRUSH rule 0 x 438 [7,2,13,4,11,1,9,14]
+ CRUSH rule 0 x 439 [7,14,4,3,12,10]
+ CRUSH rule 0 x 440 [14,11,9,2,7,12,1,5]
+ CRUSH rule 0 x 441 [2,4,11,9,13,6,1,14]
+ CRUSH rule 0 x 442 [10,13,9,7,15,1,4,2]
+ CRUSH rule 0 x 443 [12,15,10,9,2,1,6,4]
+ CRUSH rule 0 x 444 [4,13,7,14,3,1,9,11]
+ CRUSH rule 0 x 445 [4,2,15,7,1,9,11,12]
+ CRUSH rule 0 x 446 [12,10,6,9,4,1,15,3]
+ CRUSH rule 0 x 447 [15,7,13,1,4,9,3,11]
+ CRUSH rule 0 x 448 [5,2,13,7,15,10]
+ CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+ CRUSH rule 0 x 450 [2,4,6,9,15,1,13,10]
+ CRUSH rule 0 x 451 [6,14,11,3,9,1,12,5]
+ CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+ CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+ CRUSH rule 0 x 454 [10,4,2,6,15,12,9,1]
+ CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+ CRUSH rule 0 x 456 [5,7,13,1,11,3,9,15]
+ CRUSH rule 0 x 457 [9,1,5,7,11,13,15,2]
+ CRUSH rule 0 x 458 [9,11,15,4,7,2,12,1]
+ CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+ CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+ CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+ CRUSH rule 0 x 462 [4,7,12,14,11,1,3,9]
+ CRUSH rule 0 x 463 [4,12,14,11,2,7,1,9]
+ CRUSH rule 0 x 464 [4,2,15,10,1,9,13,7]
+ CRUSH rule 0 x 465 [5,10,9,7,13,1,3,14]
+ CRUSH rule 0 x 466 [13,5,2,15,9,11,6,1]
+ CRUSH rule 0 x 467 [13,6,14,3,9,1,11,5]
+ CRUSH rule 0 x 468 [10,7,12,14,4,1,9,2]
+ CRUSH rule 0 x 469 [4,9,6,14,12,11,3,1]
+ CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+ CRUSH rule 0 x 471 [6,1,5,14,13,10,9,3]
+ CRUSH rule 0 x 472 [2,14,7,5,13,1,11,9]
+ CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+ CRUSH rule 0 x 474 [15,10,4,12,6,9,2,1]
+ CRUSH rule 0 x 475 [10,5,12,9,14,3,6,1]
+ CRUSH rule 0 x 476 [3,6,10,12,1,15,9,4]
+ CRUSH rule 0 x 477 [6,13,5,15,11,9,2,1]
+ CRUSH rule 0 x 478 [4,15,1,3,7,12,10,9]
+ CRUSH rule 0 x 479 [13,11,1,6,14,5,9,3]
+ CRUSH rule 0 x 480 [1,13,6,4,9,14,11,3]
+ CRUSH rule 0 x 481 [15,12,7,9,1,3,10,4]
+ CRUSH rule 0 x 482 [2,12,9,1,7,11,14,4]
+ CRUSH rule 0 x 483 [10,1,4,15,9,7,13,2]
+ CRUSH rule 0 x 484 [1,4,10,13,7,14,2,9]
+ CRUSH rule 0 x 485 [9,4,3,1,14,12,7,10]
+ CRUSH rule 0 x 486 [3,10,15,9,7,13,4,1]
+ CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+ CRUSH rule 0 x 488 [14,4,1,9,2,6,10,12]
+ CRUSH rule 0 x 489 [11,4,2,13,15,7]
+ CRUSH rule 0 x 490 [4,9,1,3,13,15,6,11]
+ CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+ CRUSH rule 0 x 492 [5,7,11,3,14,9,1,13]
+ CRUSH rule 0 x 493 [12,1,4,15,3,11,9,6]
+ CRUSH rule 0 x 494 [1,7,13,4,15,9,10,3]
+ CRUSH rule 0 x 495 [3,15,7,1,9,5,12,11]
+ CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+ CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+ CRUSH rule 0 x 498 [10,6,1,5,9,12,3,15]
+ CRUSH rule 0 x 499 [14,3,12,5,1,11,9,7]
+ CRUSH rule 0 x 500 [15,9,6,12,11,2,1,5]
+ CRUSH rule 0 x 501 [10,13,1,9,3,14,5,7]
+ CRUSH rule 0 x 502 [5,1,14,11,7,12,9,2]
+ CRUSH rule 0 x 503 [15,10,7,9,1,12,4,2]
+ CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+ CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+ CRUSH rule 0 x 506 [11,7,9,14,12,1,2,5]
+ CRUSH rule 0 x 507 [4,14,13,3,9,7,1,10]
+ CRUSH rule 0 x 508 [12,1,4,9,2,11,15,7]
+ CRUSH rule 0 x 509 [4,2,6,9,14,1,10,13]
+ CRUSH rule 0 x 510 [5,3,1,12,11,14,9,7]
+ CRUSH rule 0 x 511 [2,12,10,6,14,5]
+ CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+ CRUSH rule 0 x 513 [4,9,11,3,13,7,1,14]
+ CRUSH rule 0 x 514 [11,9,3,4,12,15,6,1]
+ CRUSH rule 0 x 515 [12,14,6,5,3,9,1,10]
+ CRUSH rule 0 x 516 [14,11,1,12,3,7,4,9]
+ CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+ CRUSH rule 0 x 518 [3,5,7,12,15,11,9,1]
+ CRUSH rule 0 x 519 [12,14,2,1,4,6,9,10]
+ CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+ CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+ CRUSH rule 0 x 522 [4,12,11,1,15,3,9,6]
+ CRUSH rule 0 x 523 [3,1,5,9,15,10,13,7]
+ CRUSH rule 0 x 524 [15,9,3,11,13,7,4,1]
+ CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+ CRUSH rule 0 x 526 [10,2,5,13,6,15,1,9]
+ CRUSH rule 0 x 527 [3,13,4,1,9,10,14,7]
+ CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+ CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+ CRUSH rule 0 x 530 [11,9,12,7,5,1,3,15]
+ CRUSH rule 0 x 531 [9,15,4,7,2,13,1,11]
+ CRUSH rule 0 x 532 [5,3,13,7,9,14,1,10]
+ CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+ CRUSH rule 0 x 534 [11,9,3,7,15,4,1,12]
+ CRUSH rule 0 x 535 [11,1,3,5,14,9,12,7]
+ CRUSH rule 0 x 536 [9,1,14,13,4,6,2,11]
+ CRUSH rule 0 x 537 [15,5,13,2,7,11]
+ CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+ CRUSH rule 0 x 539 [10,12,6,14,1,2,9,5]
+ CRUSH rule 0 x 540 [12,15,7,3,9,11,1,4]
+ CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+ CRUSH rule 0 x 542 [3,9,15,5,11,12,7,1]
+ CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+ CRUSH rule 0 x 544 [3,15,9,11,7,4,12,1]
+ CRUSH rule 0 x 545 [14,10,7,12,4,9,1,3]
+ CRUSH rule 0 x 546 [5,15,13,7,1,10,9,2]
+ CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+ CRUSH rule 0 x 548 [11,7,12,15,4,2]
+ CRUSH rule 0 x 549 [14,1,4,9,13,6,3,10]
+ CRUSH rule 0 x 550 [9,15,3,13,1,6,4,11]
+ CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+ CRUSH rule 0 x 552 [2,11,14,1,9,6,5,12]
+ CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+ CRUSH rule 0 x 554 [11,14,6,4,13,9,3,1]
+ CRUSH rule 0 x 555 [6,5,10,9,14,2,13,1]
+ CRUSH rule 0 x 556 [15,6,3,13,11,4,1,9]
+ CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+ CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+ CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+ CRUSH rule 0 x 560 [4,9,12,6,3,10,1,15]
+ CRUSH rule 0 x 561 [12,7,1,2,5,15,11,9]
+ CRUSH rule 0 x 562 [7,13,9,14,2,1,11,4]
+ CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+ CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+ CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+ CRUSH rule 0 x 566 [6,14,4,2,13,11]
+ CRUSH rule 0 x 567 [15,4,11,6,3,12]
+ CRUSH rule 0 x 568 [4,14,1,6,10,13,3,9]
+ CRUSH rule 0 x 569 [11,3,15,13,5,1,9,7]
+ CRUSH rule 0 x 570 [1,10,13,4,7,2,9,14]
+ CRUSH rule 0 x 571 [10,12,14,9,4,2,1,6]
+ CRUSH rule 0 x 572 [12,14,3,10,6,1,4,9]
+ CRUSH rule 0 x 573 [7,15,11,2,12,9,4,1]
+ CRUSH rule 0 x 574 [11,14,13,1,3,7,4,9]
+ CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+ CRUSH rule 0 x 576 [3,15,11,9,1,6,5,13]
+ CRUSH rule 0 x 577 [13,9,6,15,3,11,4,1]
+ CRUSH rule 0 x 578 [4,10,1,2,7,13,14,9]
+ CRUSH rule 0 x 579 [13,1,15,2,10,7,5,9]
+ CRUSH rule 0 x 580 [3,12,4,1,10,15,7,9]
+ CRUSH rule 0 x 581 [7,14,12,10,1,2,9,5]
+ CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+ CRUSH rule 0 x 583 [4,15,1,9,10,12,2,6]
+ CRUSH rule 0 x 584 [10,1,5,13,6,9,2,15]
+ CRUSH rule 0 x 585 [5,3,6,1,11,14,13,9]
+ CRUSH rule 0 x 586 [7,10,14,12,9,3,5,1]
+ CRUSH rule 0 x 587 [11,6,9,4,1,14,13,2]
+ CRUSH rule 0 x 588 [3,12,7,15,4,9,1,10]
+ CRUSH rule 0 x 589 [9,7,12,1,10,3,4,15]
+ CRUSH rule 0 x 590 [12,1,3,9,10,6,4,14]
+ CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+ CRUSH rule 0 x 592 [15,12,9,7,5,2,11,1]
+ CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+ CRUSH rule 0 x 594 [12,14,2,9,7,4,1,11]
+ CRUSH rule 0 x 595 [12,7,10,3,1,14,9,4]
+ CRUSH rule 0 x 596 [2,7,12,11,1,5,15,9]
+ CRUSH rule 0 x 597 [15,1,2,10,7,13,5,9]
+ CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+ CRUSH rule 0 x 599 [13,11,1,5,6,2,15,9]
+ CRUSH rule 0 x 600 [4,12,3,10,9,7,1,14]
+ CRUSH rule 0 x 601 [13,5,15,2,1,7,9,10]
+ CRUSH rule 0 x 602 [3,11,7,1,13,15,5,9]
+ CRUSH rule 0 x 603 [3,1,4,14,10,9,6,12]
+ CRUSH rule 0 x 604 [14,2,6,1,11,13,9,4]
+ CRUSH rule 0 x 605 [2,7,12,5,14,10,1,9]
+ CRUSH rule 0 x 606 [12,15,1,5,7,9,3,11]
+ CRUSH rule 0 x 607 [3,9,10,14,7,1,4,12]
+ CRUSH rule 0 x 608 [13,10,1,7,9,15,5,2]
+ CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+ CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+ CRUSH rule 0 x 611 [13,1,5,3,10,7,15,9]
+ CRUSH rule 0 x 612 [7,1,2,13,9,15,4,11]
+ CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+ CRUSH rule 0 x 614 [9,4,15,3,1,11,6,12]
+ CRUSH rule 0 x 615 [9,4,11,2,1,12,6,15]
+ CRUSH rule 0 x 616 [10,14,1,5,3,6,12,9]
+ CRUSH rule 0 x 617 [15,7,2,11,12,1,9,4]
+ CRUSH rule 0 x 618 [4,2,10,6,14,9,1,12]
+ CRUSH rule 0 x 619 [15,4,3,9,6,1,13,11]
+ CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+ CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+ CRUSH rule 0 x 622 [10,2,13,5,15,9,1,7]
+ CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+ CRUSH rule 0 x 624 [3,9,15,6,10,1,5,12]
+ CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+ CRUSH rule 0 x 626 [10,12,2,1,9,7,5,14]
+ CRUSH rule 0 x 627 [1,12,10,14,3,5,9,7]
+ CRUSH rule 0 x 628 [15,13,11,4,2,1,7,9]
+ CRUSH rule 0 x 629 [5,6,15,12,1,10,3,9]
+ CRUSH rule 0 x 630 [1,4,12,9,3,7,15,11]
+ CRUSH rule 0 x 631 [5,7,1,15,12,11,3,9]
+ CRUSH rule 0 x 632 [12,3,11,9,6,1,15,5]
+ CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+ CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+ CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+ CRUSH rule 0 x 636 [13,6,11,3,15,9,1,4]
+ CRUSH rule 0 x 637 [3,1,10,6,9,12,4,14]
+ CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+ CRUSH rule 0 x 639 [6,9,14,4,3,1,10,13]
+ CRUSH rule 0 x 640 [9,6,1,11,14,2,4,13]
+ CRUSH rule 0 x 641 [10,6,5,14,1,9,12,2]
+ CRUSH rule 0 x 642 [1,15,4,6,2,10,9,12]
+ CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+ CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+ CRUSH rule 0 x 645 [14,2,4,9,10,1,7,13]
+ CRUSH rule 0 x 646 [5,13,14,1,6,9,2,11]
+ CRUSH rule 0 x 647 [10,1,9,13,6,2,14,5]
+ CRUSH rule 0 x 648 [6,5,2,14,11,1,12,9]
+ CRUSH rule 0 x 649 [3,9,13,11,4,14,1,7]
+ CRUSH rule 0 x 650 [10,9,4,15,12,7,1,2]
+ CRUSH rule 0 x 651 [3,9,5,7,14,1,13,11]
+ CRUSH rule 0 x 652 [15,9,4,6,13,1,2,11]
+ CRUSH rule 0 x 653 [11,14,1,3,6,9,12,5]
+ CRUSH rule 0 x 654 [13,6,2,10,15,4,1,9]
+ CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+ CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+ CRUSH rule 0 x 657 [11,15,3,5,7,13,1,9]
+ CRUSH rule 0 x 658 [7,2,10,12,1,4,9,14]
+ CRUSH rule 0 x 659 [2,5,14,6,10,12]
+ CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+ CRUSH rule 0 x 661 [7,15,3,12,11,4,9,1]
+ CRUSH rule 0 x 662 [15,2,12,5,1,10,9,7]
+ CRUSH rule 0 x 663 [14,9,13,10,5,3,1,6]
+ CRUSH rule 0 x 664 [6,10,12,4,9,2,1,15]
+ CRUSH rule 0 x 665 [2,9,12,1,7,10,4,15]
+ CRUSH rule 0 x 666 [12,3,6,1,15,9,10,4]
+ CRUSH rule 0 x 667 [1,9,12,10,2,14,7,4]
+ CRUSH rule 0 x 668 [9,5,1,2,6,11,13,15]
+ CRUSH rule 0 x 669 [9,7,14,5,11,13,1,2]
+ CRUSH rule 0 x 670 [6,10,9,13,1,2,15,4]
+ CRUSH rule 0 x 671 [6,15,5,10,13,3]
+ CRUSH rule 0 x 672 [2,9,13,1,4,14,6,10]
+ CRUSH rule 0 x 673 [7,10,5,9,15,13,2,1]
+ CRUSH rule 0 x 674 [7,12,10,1,14,9,3,4]
+ CRUSH rule 0 x 675 [9,5,1,10,6,14,12,2]
+ CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+ CRUSH rule 0 x 677 [2,12,1,4,10,6,15,9]
+ CRUSH rule 0 x 678 [1,2,4,10,12,14,9,6]
+ CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+ CRUSH rule 0 x 680 [7,11,3,1,15,4,9,12]
+ CRUSH rule 0 x 681 [6,4,3,11,14,13,1,9]
+ CRUSH rule 0 x 682 [6,1,11,15,12,2,5,9]
+ CRUSH rule 0 x 683 [6,13,2,4,9,14,10,1]
+ CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+ CRUSH rule 0 x 685 [5,1,15,7,9,2,10,13]
+ CRUSH rule 0 x 686 [1,9,11,14,6,13,4,3]
+ CRUSH rule 0 x 687 [7,13,3,5,11,9,15,1]
+ CRUSH rule 0 x 688 [11,9,1,14,3,5,7,12]
+ CRUSH rule 0 x 689 [5,2,9,12,1,14,11,7]
+ CRUSH rule 0 x 690 [9,7,10,3,13,15,5,1]
+ CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+ CRUSH rule 0 x 692 [15,5,1,2,9,11,12,7]
+ CRUSH rule 0 x 693 [5,6,12,15,2,10,9,1]
+ CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+ CRUSH rule 0 x 695 [6,13,14,10,9,5,1,3]
+ CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+ CRUSH rule 0 x 697 [13,11,3,6,4,14,9,1]
+ CRUSH rule 0 x 698 [11,13,4,2,6,1,9,15]
+ CRUSH rule 0 x 699 [7,14,12,4,2,11]
+ CRUSH rule 0 x 700 [12,14,11,9,4,6,3,1]
+ CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+ CRUSH rule 0 x 702 [3,12,15,6,5,11,1,9]
+ CRUSH rule 0 x 703 [15,11,13,3,4,7,1,9]
+ CRUSH rule 0 x 704 [6,4,2,15,11,1,13,9]
+ CRUSH rule 0 x 705 [14,6,11,5,1,13,9,3]
+ CRUSH rule 0 x 706 [1,12,3,6,4,10,15,9]
+ CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+ CRUSH rule 0 x 708 [3,10,5,1,15,9,7,13]
+ CRUSH rule 0 x 709 [11,12,3,7,5,14,1,9]
+ CRUSH rule 0 x 710 [14,2,11,9,5,7,12,1]
+ CRUSH rule 0 x 711 [14,3,9,10,12,5,6,1]
+ CRUSH rule 0 x 712 [12,3,11,15,9,1,6,4]
+ CRUSH rule 0 x 713 [11,9,3,15,13,6,4,1]
+ CRUSH rule 0 x 714 [12,1,9,7,2,15,10,5]
+ CRUSH rule 0 x 715 [6,1,14,4,11,12,3,9]
+ CRUSH rule 0 x 716 [11,13,9,14,5,2,1,7]
+ CRUSH rule 0 x 717 [12,4,10,9,15,1,2,7]
+ CRUSH rule 0 x 718 [7,15,5,2,11,13]
+ CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+ CRUSH rule 0 x 720 [4,13,10,2,7,9,1,14]
+ CRUSH rule 0 x 721 [11,3,14,9,1,12,4,6]
+ CRUSH rule 0 x 722 [2,4,6,1,9,15,13,10]
+ CRUSH rule 0 x 723 [2,1,12,15,11,7,5,9]
+ CRUSH rule 0 x 724 [7,1,9,10,5,15,13,2]
+ CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+ CRUSH rule 0 x 726 [7,14,4,3,11,13,9,1]
+ CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+ CRUSH rule 0 x 728 [13,11,4,6,15,2]
+ CRUSH rule 0 x 729 [15,11,4,6,2,9,1,13]
+ CRUSH rule 0 x 730 [3,7,1,13,11,15,9,5]
+ CRUSH rule 0 x 731 [9,1,6,5,2,11,13,15]
+ CRUSH rule 0 x 732 [1,2,10,13,9,4,7,15]
+ CRUSH rule 0 x 733 [11,3,5,6,1,9,12,15]
+ CRUSH rule 0 x 734 [14,3,11,7,12,9,4,1]
+ CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+ CRUSH rule 0 x 736 [3,9,1,11,7,5,13,14]
+ CRUSH rule 0 x 737 [1,4,2,12,9,10,6,15]
+ CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+ CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+ CRUSH rule 0 x 740 [7,9,10,13,1,15,2,5]
+ CRUSH rule 0 x 741 [12,11,7,15,2,5]
+ CRUSH rule 0 x 742 [9,7,4,11,12,1,14,3]
+ CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+ CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+ CRUSH rule 0 x 745 [3,6,1,4,11,12,14,9]
+ CRUSH rule 0 x 746 [3,7,9,10,14,5,1,13]
+ CRUSH rule 0 x 747 [15,11,5,2,13,9,1,7]
+ CRUSH rule 0 x 748 [6,10,13,2,14,5,9,1]
+ CRUSH rule 0 x 749 [14,9,10,7,5,1,2,12]
+ CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+ CRUSH rule 0 x 751 [15,1,6,9,5,11,12,3]
+ CRUSH rule 0 x 752 [13,1,7,3,11,15,9,4]
+ CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+ CRUSH rule 0 x 754 [14,12,11,4,2,1,9,6]
+ CRUSH rule 0 x 755 [13,6,1,10,4,2,14,9]
+ CRUSH rule 0 x 756 [3,4,14,6,1,10,13,9]
+ CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+ CRUSH rule 0 x 758 [6,3,4,10,15,13,9,1]
+ CRUSH rule 0 x 759 [5,7,3,14,11,1,9,13]
+ CRUSH rule 0 x 760 [1,15,10,12,4,3,9,7]
+ CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+ CRUSH rule 0 x 762 [1,4,10,9,3,7,14,12]
+ CRUSH rule 0 x 763 [4,13,1,14,7,10,2,9]
+ CRUSH rule 0 x 764 [1,14,6,13,9,5,2,10]
+ CRUSH rule 0 x 765 [9,15,2,13,4,1,11,7]
+ CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+ CRUSH rule 0 x 767 [6,11,4,3,12,14]
+ CRUSH rule 0 x 768 [2,12,15,7,1,11,9,4]
+ CRUSH rule 0 x 769 [15,1,9,2,11,12,7,4]
+ CRUSH rule 0 x 770 [15,13,4,6,3,10,1,9]
+ CRUSH rule 0 x 771 [9,2,12,11,6,14,5,1]
+ CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+ CRUSH rule 0 x 773 [3,7,4,15,1,12,11,9]
+ CRUSH rule 0 x 774 [12,6,3,15,5,9,10,1]
+ CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+ CRUSH rule 0 x 776 [10,15,3,9,6,13,1,5]
+ CRUSH rule 0 x 777 [11,13,4,7,1,14,9,2]
+ CRUSH rule 0 x 778 [13,1,9,11,15,6,3,5]
+ CRUSH rule 0 x 779 [5,11,1,14,2,9,13,6]
+ CRUSH rule 0 x 780 [13,9,3,6,4,1,14,10]
+ CRUSH rule 0 x 781 [5,7,14,3,1,12,11,9]
+ CRUSH rule 0 x 782 [2,15,9,7,11,13,4,1]
+ CRUSH rule 0 x 783 [12,7,5,14,9,1,2,10]
+ CRUSH rule 0 x 784 [14,1,10,13,3,4,7,9]
+ CRUSH rule 0 x 785 [6,12,1,2,4,9,15,10]
+ CRUSH rule 0 x 786 [10,5,2,15,1,7,12,9]
+ CRUSH rule 0 x 787 [1,12,10,2,9,4,14,6]
+ CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+ CRUSH rule 0 x 789 [9,2,14,7,4,12,1,10]
+ CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+ CRUSH rule 0 x 791 [9,4,7,13,14,11,1,3]
+ CRUSH rule 0 x 792 [6,4,15,10,12,3]
+ CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+ CRUSH rule 0 x 794 [5,12,2,14,9,10,1,6]
+ CRUSH rule 0 x 795 [6,14,12,4,10,1,2,9]
+ CRUSH rule 0 x 796 [11,2,12,6,15,4]
+ CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+ CRUSH rule 0 x 798 [5,11,6,13,1,3,15,9]
+ CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+ CRUSH rule 0 x 800 [6,3,4,11,15,13]
+ CRUSH rule 0 x 801 [2,5,6,13,9,1,10,15]
+ CRUSH rule 0 x 802 [1,4,12,7,3,9,10,14]
+ CRUSH rule 0 x 803 [7,2,4,1,11,13,9,14]
+ CRUSH rule 0 x 804 [5,14,9,7,3,1,12,10]
+ CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+ CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+ CRUSH rule 0 x 807 [14,2,7,4,9,12,1,10]
+ CRUSH rule 0 x 808 [2,15,12,7,9,1,5,10]
+ CRUSH rule 0 x 809 [1,11,7,12,4,2,15,9]
+ CRUSH rule 0 x 810 [2,5,9,12,15,1,7,11]
+ CRUSH rule 0 x 811 [15,6,3,10,1,5,9,12]
+ CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+ CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+ CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+ CRUSH rule 0 x 815 [15,12,9,4,10,6,1,2]
+ CRUSH rule 0 x 816 [14,10,13,7,3,9,4,1]
+ CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+ CRUSH rule 0 x 818 [15,2,11,4,1,12,6,9]
+ CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+ CRUSH rule 0 x 820 [3,6,9,12,11,15,4,1]
+ CRUSH rule 0 x 821 [15,10,9,13,3,4,7,1]
+ CRUSH rule 0 x 822 [10,13,2,9,7,4,14,1]
+ CRUSH rule 0 x 823 [2,6,12,10,15,4,1,9]
+ CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+ CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+ CRUSH rule 0 x 826 [5,2,11,15,1,12,9,7]
+ CRUSH rule 0 x 827 [13,5,1,3,7,9,11,14]
+ CRUSH rule 0 x 828 [12,6,10,5,1,9,2,15]
+ CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+ CRUSH rule 0 x 830 [15,13,2,9,7,11,1,5]
+ CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+ CRUSH rule 0 x 832 [14,11,13,2,9,4,6,1]
+ CRUSH rule 0 x 833 [9,13,3,11,7,5,15,1]
+ CRUSH rule 0 x 834 [9,7,5,1,11,2,13,14]
+ CRUSH rule 0 x 835 [14,3,13,6,4,9,1,10]
+ CRUSH rule 0 x 836 [3,9,10,13,1,5,14,7]
+ CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+ CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+ CRUSH rule 0 x 839 [3,4,6,10,15,1,13,9]
+ CRUSH rule 0 x 840 [10,15,12,4,7,1,2,9]
+ CRUSH rule 0 x 841 [3,5,7,12,11,15,1,9]
+ CRUSH rule 0 x 842 [9,13,2,6,5,14,10,1]
+ CRUSH rule 0 x 843 [14,7,4,9,3,12,1,10]
+ CRUSH rule 0 x 844 [7,1,4,15,9,2,11,12]
+ CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+ CRUSH rule 0 x 846 [3,7,15,13,1,9,10,4]
+ CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+ CRUSH rule 0 x 848 [11,13,1,14,5,9,2,7]
+ CRUSH rule 0 x 849 [3,15,11,9,6,1,13,5]
+ CRUSH rule 0 x 850 [1,3,10,6,14,4,9,12]
+ CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+ CRUSH rule 0 x 852 [9,12,4,7,15,2,11,1]
+ CRUSH rule 0 x 853 [13,14,6,11,2,4,9,1]
+ CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+ CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+ CRUSH rule 0 x 856 [5,10,7,3,15,9,12,1]
+ CRUSH rule 0 x 857 [4,3,13,11,9,1,7,14]
+ CRUSH rule 0 x 858 [5,15,6,3,9,12,1,10]
+ CRUSH rule 0 x 859 [5,15,6,2,1,11,12,9]
+ CRUSH rule 0 x 860 [11,14,1,12,6,9,2,4]
+ CRUSH rule 0 x 861 [13,7,4,10,1,14,3,9]
+ CRUSH rule 0 x 862 [5,10,9,7,3,12,1,15]
+ CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+ CRUSH rule 0 x 864 [6,13,4,2,10,15,1,9]
+ CRUSH rule 0 x 865 [4,1,14,11,6,9,3,13]
+ CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+ CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+ CRUSH rule 0 x 868 [14,11,7,2,1,4,9,12]
+ CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+ CRUSH rule 0 x 870 [14,9,11,4,3,12,6,1]
+ CRUSH rule 0 x 871 [6,2,1,4,15,13,11,9]
+ CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+ CRUSH rule 0 x 873 [2,5,12,10,1,9,15,7]
+ CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+ CRUSH rule 0 x 875 [10,6,14,1,12,5,9,3]
+ CRUSH rule 0 x 876 [14,7,13,3,9,1,11,4]
+ CRUSH rule 0 x 877 [15,11,13,9,5,1,6,3]
+ CRUSH rule 0 x 878 [7,14,3,13,9,1,11,4]
+ CRUSH rule 0 x 879 [12,2,7,4,10,15]
+ CRUSH rule 0 x 880 [2,12,10,7,1,4,9,14]
+ CRUSH rule 0 x 881 [6,3,1,11,4,15,9,13]
+ CRUSH rule 0 x 882 [11,13,7,1,2,15,4,9]
+ CRUSH rule 0 x 883 [13,1,3,10,6,5,9,15]
+ CRUSH rule 0 x 884 [6,15,4,9,3,11,12,1]
+ CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+ CRUSH rule 0 x 886 [13,11,4,2,1,14,9,6]
+ CRUSH rule 0 x 887 [14,4,12,11,2,6,9,1]
+ CRUSH rule 0 x 888 [10,12,7,15,9,2,1,5]
+ CRUSH rule 0 x 889 [15,13,4,1,6,2,10,9]
+ CRUSH rule 0 x 890 [10,12,14,2,9,5,6,1]
+ CRUSH rule 0 x 891 [9,5,11,6,3,15,12,1]
+ CRUSH rule 0 x 892 [12,15,2,4,7,9,11,1]
+ CRUSH rule 0 x 893 [1,3,5,9,6,10,14,12]
+ CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+ CRUSH rule 0 x 895 [2,1,11,5,7,15,13,9]
+ CRUSH rule 0 x 896 [9,1,14,10,4,12,2,7]
+ CRUSH rule 0 x 897 [7,5,14,3,1,9,11,12]
+ CRUSH rule 0 x 898 [10,6,12,9,15,5,2,1]
+ CRUSH rule 0 x 899 [1,11,5,3,13,14,9,6]
+ CRUSH rule 0 x 900 [2,9,10,7,13,14,5,1]
+ CRUSH rule 0 x 901 [9,12,11,3,14,4,1,6]
+ CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+ CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+ CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+ CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+ CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+ CRUSH rule 0 x 907 [7,12,3,9,10,5,14,1]
+ CRUSH rule 0 x 908 [2,15,9,6,10,13,5,1]
+ CRUSH rule 0 x 909 [10,14,1,13,2,9,7,4]
+ CRUSH rule 0 x 910 [12,7,4,15,10,3,1,9]
+ CRUSH rule 0 x 911 [11,15,2,4,9,13,6,1]
+ CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+ CRUSH rule 0 x 913 [4,6,10,1,12,3,9,14]
+ CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+ CRUSH rule 0 x 915 [12,14,1,9,4,3,11,6]
+ CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+ CRUSH rule 0 x 917 [1,15,6,5,10,3,13,9]
+ CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+ CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+ CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+ CRUSH rule 0 x 921 [1,11,6,13,4,2,9,14]
+ CRUSH rule 0 x 922 [6,4,14,13,3,1,10,9]
+ CRUSH rule 0 x 923 [12,2,5,14,10,1,9,6]
+ CRUSH rule 0 x 924 [6,2,14,13,9,1,11,5]
+ CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+ CRUSH rule 0 x 926 [3,13,10,1,14,9,6,5]
+ CRUSH rule 0 x 927 [6,5,1,11,14,2,13,9]
+ CRUSH rule 0 x 928 [13,1,3,9,6,11,15,5]
+ CRUSH rule 0 x 929 [10,7,1,5,2,12,9,14]
+ CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+ CRUSH rule 0 x 931 [6,15,11,9,5,3,1,13]
+ CRUSH rule 0 x 932 [13,2,5,11,9,1,6,15]
+ CRUSH rule 0 x 933 [12,7,14,10,4,1,2,9]
+ CRUSH rule 0 x 934 [12,2,5,7,9,1,15,11]
+ CRUSH rule 0 x 935 [6,11,1,14,5,13,3,9]
+ CRUSH rule 0 x 936 [9,12,7,5,1,2,14,11]
+ CRUSH rule 0 x 937 [14,2,11,1,13,4,9,6]
+ CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+ CRUSH rule 0 x 939 [6,4,14,9,12,1,11,2]
+ CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+ CRUSH rule 0 x 941 [3,12,4,7,14,10]
+ CRUSH rule 0 x 942 [15,12,10,4,1,9,3,7]
+ CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+ CRUSH rule 0 x 944 [2,9,4,7,1,14,12,11]
+ CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+ CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+ CRUSH rule 0 x 947 [11,3,14,1,12,5,6,9]
+ CRUSH rule 0 x 948 [7,13,11,5,14,2,1,9]
+ CRUSH rule 0 x 949 [9,1,12,5,15,10,2,6]
+ CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+ CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+ CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+ CRUSH rule 0 x 953 [1,3,6,10,12,14,4,9]
+ CRUSH rule 0 x 954 [10,2,14,9,4,6,12,1]
+ CRUSH rule 0 x 955 [7,14,3,1,10,4,9,12]
+ CRUSH rule 0 x 956 [1,6,11,5,14,3,9,13]
+ CRUSH rule 0 x 957 [14,11,1,12,6,9,4,3]
+ CRUSH rule 0 x 958 [15,4,3,11,1,6,12,9]
+ CRUSH rule 0 x 959 [2,1,12,15,10,9,4,6]
+ CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+ CRUSH rule 0 x 961 [3,13,11,9,6,1,4,15]
+ CRUSH rule 0 x 962 [5,11,3,14,1,6,13,9]
+ CRUSH rule 0 x 963 [13,10,15,4,6,9,1,3]
+ CRUSH rule 0 x 964 [7,11,4,9,2,12,1,15]
+ CRUSH rule 0 x 965 [12,2,9,7,4,15,11,1]
+ CRUSH rule 0 x 966 [12,14,9,4,1,2,11,7]
+ CRUSH rule 0 x 967 [7,5,3,10,12,14]
+ CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+ CRUSH rule 0 x 969 [11,4,7,1,9,14,13,2]
+ CRUSH rule 0 x 970 [5,12,10,1,3,14,9,6]
+ CRUSH rule 0 x 971 [1,9,4,12,7,2,10,15]
+ CRUSH rule 0 x 972 [12,3,14,5,1,9,7,11]
+ CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+ CRUSH rule 0 x 974 [7,11,1,2,15,4,12,9]
+ CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+ CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+ CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+ CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+ CRUSH rule 0 x 979 [5,1,13,6,15,10,3,9]
+ CRUSH rule 0 x 980 [15,11,5,6,1,3,13,9]
+ CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+ CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+ CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+ CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+ CRUSH rule 0 x 985 [11,2,15,1,4,13,6,9]
+ CRUSH rule 0 x 986 [6,13,9,1,15,10,5,2]
+ CRUSH rule 0 x 987 [13,14,5,10,6,1,3,9]
+ CRUSH rule 0 x 988 [12,9,10,14,3,1,4,7]
+ CRUSH rule 0 x 989 [7,4,3,15,9,13,10,1]
+ CRUSH rule 0 x 990 [1,10,9,13,3,4,6,15]
+ CRUSH rule 0 x 991 [7,11,1,14,2,5,9,12]
+ CRUSH rule 0 x 992 [9,10,2,13,7,4,1,15]
+ CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+ CRUSH rule 0 x 994 [3,13,15,4,11,7,1,9]
+ CRUSH rule 0 x 995 [15,6,12,2,5,11]
+ CRUSH rule 0 x 996 [15,10,5,3,13,1,9,7]
+ CRUSH rule 0 x 997 [15,2,1,12,7,9,4,10]
+ CRUSH rule 0 x 998 [6,1,9,5,12,11,15,2]
+ CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+ CRUSH rule 0 x 1000 [14,2,9,4,12,1,6,11]
+ CRUSH rule 0 x 1001 [11,14,4,2,6,9,1,13]
+ CRUSH rule 0 x 1002 [1,10,14,2,9,5,13,7]
+ CRUSH rule 0 x 1003 [10,7,5,14,2,1,9,12]
+ CRUSH rule 0 x 1004 [15,1,4,6,10,12,9,3]
+ CRUSH rule 0 x 1005 [6,12,2,10,9,15,5,1]
+ CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+ CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+ CRUSH rule 0 x 1008 [7,4,9,11,3,15,1,13]
+ CRUSH rule 0 x 1009 [5,2,11,7,15,9,1,12]
+ CRUSH rule 0 x 1010 [10,2,15,6,9,13,4,1]
+ CRUSH rule 0 x 1011 [6,3,12,1,10,4,9,14]
+ CRUSH rule 0 x 1012 [12,6,9,15,3,1,5,11]
+ CRUSH rule 0 x 1013 [2,14,12,4,9,1,6,10]
+ CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+ CRUSH rule 0 x 1015 [12,6,10,1,4,15,9,2]
+ CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+ CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+ CRUSH rule 0 x 1018 [13,11,14,1,9,3,5,7]
+ CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+ CRUSH rule 0 x 1020 [3,1,13,4,10,9,14,6]
+ CRUSH rule 0 x 1021 [2,11,14,9,4,6,1,13]
+ CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+ CRUSH rule 0 x 1023 [15,2,9,12,1,7,4,11]
+ rule 0 (replicated_ruleset) num_rep 9 result size == 6:\t36/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 9 result size == 7:\t263/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 9 result size == 8:\t725/1024 (esc)
+ CRUSH rule 0 x 0 [7,10,3,15,12,1,4,9]
+ CRUSH rule 0 x 1 [10,15,1,2,13,4,7,9]
+ CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+ CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+ CRUSH rule 0 x 4 [14,2,10,1,9,4,7,13]
+ CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+ CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+ CRUSH rule 0 x 7 [9,2,6,12,11,4,1,14]
+ CRUSH rule 0 x 8 [10,2,15,1,4,13,6,9]
+ CRUSH rule 0 x 9 [7,1,14,2,11,9,12,4]
+ CRUSH rule 0 x 10 [10,14,4,1,2,7,13,9]
+ CRUSH rule 0 x 11 [13,9,14,7,5,11,2,1]
+ CRUSH rule 0 x 12 [7,1,2,5,13,15,11,9]
+ CRUSH rule 0 x 13 [3,5,12,7,9,1,14,11]
+ CRUSH rule 0 x 14 [13,5,2,7,10,15,1,9]
+ CRUSH rule 0 x 15 [15,1,9,6,13,3,5,11]
+ CRUSH rule 0 x 16 [7,11,14,2,13,1,9,4]
+ CRUSH rule 0 x 17 [10,1,13,2,4,6,14,9]
+ CRUSH rule 0 x 18 [1,7,3,10,5,12,9,14]
+ CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+ CRUSH rule 0 x 20 [14,12,3,10,9,4,7,1]
+ CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+ CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+ CRUSH rule 0 x 23 [10,5,13,9,3,15,1,6]
+ CRUSH rule 0 x 24 [12,11,3,1,9,4,7,15]
+ CRUSH rule 0 x 25 [7,12,15,1,3,10,4,9]
+ CRUSH rule 0 x 26 [1,7,13,2,14,5,9,11]
+ CRUSH rule 0 x 27 [3,6,15,4,13,9,11,1]
+ CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+ CRUSH rule 0 x 29 [5,14,12,11,6,3,1,9]
+ CRUSH rule 0 x 30 [2,5,6,9,1,11,13,14]
+ CRUSH rule 0 x 31 [5,15,10,1,9,13,6,3]
+ CRUSH rule 0 x 32 [9,10,2,1,13,14,6,4]
+ CRUSH rule 0 x 33 [13,4,9,2,7,1,10,14]
+ CRUSH rule 0 x 34 [13,15,2,4,1,10,9,6]
+ CRUSH rule 0 x 35 [4,14,3,13,10,9,1,6]
+ CRUSH rule 0 x 36 [3,12,9,7,5,10,14,1]
+ CRUSH rule 0 x 37 [9,2,6,14,11,1,4,13]
+ CRUSH rule 0 x 38 [3,4,13,10,9,1,14,6]
+ CRUSH rule 0 x 39 [12,7,14,11,1,9,5,2]
+ CRUSH rule 0 x 40 [10,1,9,5,15,2,6,13]
+ CRUSH rule 0 x 41 [4,9,11,1,14,13,6,3]
+ CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+ CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+ CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+ CRUSH rule 0 x 45 [11,12,15,9,1,5,6,3]
+ CRUSH rule 0 x 46 [6,9,2,14,11,13,1,5]
+ CRUSH rule 0 x 47 [3,9,6,4,13,1,11,15]
+ CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+ CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+ CRUSH rule 0 x 50 [14,12,1,4,2,11,6,9]
+ CRUSH rule 0 x 51 [10,6,5,12,15,2,1,9]
+ CRUSH rule 0 x 52 [12,1,9,11,7,3,14,4]
+ CRUSH rule 0 x 53 [3,6,13,9,5,1,11,15]
+ CRUSH rule 0 x 54 [4,13,9,2,14,10,6,1]
+ CRUSH rule 0 x 55 [4,11,2,7,1,13,9,15]
+ CRUSH rule 0 x 56 [5,9,10,1,3,13,14,7]
+ CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+ CRUSH rule 0 x 58 [7,1,11,4,3,14,12,9]
+ CRUSH rule 0 x 59 [2,13,1,10,9,5,14,6]
+ CRUSH rule 0 x 60 [3,6,11,1,4,9,12,15]
+ CRUSH rule 0 x 61 [3,15,13,7,4,1,10,9]
+ CRUSH rule 0 x 62 [15,11,7,12,5,9,2,1]
+ CRUSH rule 0 x 63 [10,14,12,1,7,3]
+ CRUSH rule 0 x 64 [3,9,1,4,7,12,11,15]
+ CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+ CRUSH rule 0 x 66 [15,11,6,9,4,1,3,12]
+ CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+ CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+ CRUSH rule 0 x 69 [2,1,15,10,4,9,13,7]
+ CRUSH rule 0 x 70 [9,6,1,3,13,15,11,5]
+ CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+ CRUSH rule 0 x 72 [9,10,3,5,7,12,15,1]
+ CRUSH rule 0 x 73 [5,3,11,1,7,12,15,9]
+ CRUSH rule 0 x 74 [11,7,9,5,1,15,3,12]
+ CRUSH rule 0 x 75 [9,7,11,14,12,1,2,5]
+ CRUSH rule 0 x 76 [6,1,3,5,14,10,12,9]
+ CRUSH rule 0 x 77 [7,4,2,13,9,1,11,15]
+ CRUSH rule 0 x 78 [9,3,1,5,6,13,14,11]
+ CRUSH rule 0 x 79 [13,2,15,5,7,9,11,1]
+ CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+ CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+ CRUSH rule 0 x 82 [14,13,5,11,6,2,1,9]
+ CRUSH rule 0 x 83 [4,15,3,9,10,13,6,1]
+ CRUSH rule 0 x 84 [10,7,9,15,3,4,1,13]
+ CRUSH rule 0 x 85 [3,15,9,7,4,11,1,12]
+ CRUSH rule 0 x 86 [10,9,14,1,13,4,2,7]
+ CRUSH rule 0 x 87 [15,10,7,12,5,3,9,1]
+ CRUSH rule 0 x 88 [4,13,3,1,9,15,11,7]
+ CRUSH rule 0 x 89 [3,9,7,4,1,14,10,12]
+ CRUSH rule 0 x 90 [4,9,7,12,11,14,2,1]
+ CRUSH rule 0 x 91 [6,11,9,1,2,4,14,13]
+ CRUSH rule 0 x 92 [1,5,10,9,13,15,6,2]
+ CRUSH rule 0 x 93 [9,3,15,13,7,5,1,10]
+ CRUSH rule 0 x 94 [9,2,12,5,6,11,1,14]
+ CRUSH rule 0 x 95 [7,15,4,10,9,13,2,1]
+ CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+ CRUSH rule 0 x 97 [4,11,2,13,1,7,9,14]
+ CRUSH rule 0 x 98 [11,13,9,3,15,1,5,6]
+ CRUSH rule 0 x 99 [12,4,11,7,3,14,9,1]
+ CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+ CRUSH rule 0 x 101 [15,7,1,9,10,5,2,12]
+ CRUSH rule 0 x 102 [3,11,14,6,13,4,9,1]
+ CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+ CRUSH rule 0 x 104 [14,6,3,5,9,1,10,13]
+ CRUSH rule 0 x 105 [14,10,1,9,3,5,6,13]
+ CRUSH rule 0 x 106 [6,5,13,2,14,11,1,9]
+ CRUSH rule 0 x 107 [3,1,10,14,13,5,9,6]
+ CRUSH rule 0 x 108 [5,10,7,2,15,9,12,1]
+ CRUSH rule 0 x 109 [9,1,13,7,15,5,11,3]
+ CRUSH rule 0 x 110 [5,1,11,3,7,14,13,9]
+ CRUSH rule 0 x 111 [10,1,9,7,5,2,13,14]
+ CRUSH rule 0 x 112 [1,10,4,14,2,12,6,9]
+ CRUSH rule 0 x 113 [6,10,13,9,1,5,2,14]
+ CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+ CRUSH rule 0 x 115 [10,13,14,3,9,1,6,5]
+ CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+ CRUSH rule 0 x 117 [5,6,1,12,15,9,11,3]
+ CRUSH rule 0 x 118 [10,4,13,15,9,3,1,6]
+ CRUSH rule 0 x 119 [14,12,11,4,6,9,3,1]
+ CRUSH rule 0 x 120 [11,3,14,13,4,7]
+ CRUSH rule 0 x 121 [9,5,1,11,7,3,15,12]
+ CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+ CRUSH rule 0 x 123 [3,10,5,6,9,1,12,15]
+ CRUSH rule 0 x 124 [12,2,1,5,14,7,11,9]
+ CRUSH rule 0 x 125 [9,12,15,1,6,5,3,10]
+ CRUSH rule 0 x 126 [7,15,10,9,2,12,5,1]
+ CRUSH rule 0 x 127 [4,14,9,13,1,3,7,10]
+ CRUSH rule 0 x 128 [3,12,1,10,4,9,7,14]
+ CRUSH rule 0 x 129 [11,13,14,2,9,4,6,1]
+ CRUSH rule 0 x 130 [3,13,5,14,10,1,9,6]
+ CRUSH rule 0 x 131 [12,1,6,15,4,2,11,9]
+ CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+ CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+ CRUSH rule 0 x 134 [12,5,6,15,3,9,10,1]
+ CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+ CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+ CRUSH rule 0 x 137 [14,3,6,11,1,9,4,13]
+ CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+ CRUSH rule 0 x 139 [11,2,13,9,1,15,7,5]
+ CRUSH rule 0 x 140 [11,4,12,15,2,6,9,1]
+ CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+ CRUSH rule 0 x 142 [3,14,7,9,11,1,4,13]
+ CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+ CRUSH rule 0 x 144 [13,7,11,2,14,4,1,9]
+ CRUSH rule 0 x 145 [12,2,6,10,9,4,14,1]
+ CRUSH rule 0 x 146 [1,5,9,2,6,13,14,10]
+ CRUSH rule 0 x 147 [1,4,9,11,2,7,15,12]
+ CRUSH rule 0 x 148 [12,7,9,2,14,11,1,4]
+ CRUSH rule 0 x 149 [2,5,9,12,11,1,7,14]
+ CRUSH rule 0 x 150 [1,15,2,10,7,9,5,12]
+ CRUSH rule 0 x 151 [2,9,14,7,1,10,5,13]
+ CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+ CRUSH rule 0 x 153 [6,9,4,15,2,1,10,12]
+ CRUSH rule 0 x 154 [3,11,7,1,4,12,15,9]
+ CRUSH rule 0 x 155 [14,12,7,3,5,1,9,11]
+ CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+ CRUSH rule 0 x 157 [15,1,6,4,3,10,9,13]
+ CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+ CRUSH rule 0 x 159 [4,14,3,12,10,6,1,9]
+ CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+ CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+ CRUSH rule 0 x 162 [10,6,1,12,2,4,14,9]
+ CRUSH rule 0 x 163 [15,1,10,2,6,4,13,9]
+ CRUSH rule 0 x 164 [9,14,10,7,12,2,5,1]
+ CRUSH rule 0 x 165 [11,7,2,13,9,15,1,5]
+ CRUSH rule 0 x 166 [1,2,12,14,4,11,7,9]
+ CRUSH rule 0 x 167 [9,7,3,4,11,13,15,1]
+ CRUSH rule 0 x 168 [13,2,4,1,6,15,10,9]
+ CRUSH rule 0 x 169 [1,4,9,14,13,10,2,7]
+ CRUSH rule 0 x 170 [1,15,7,9,12,10,3,5]
+ CRUSH rule 0 x 171 [9,2,10,7,1,5,15,12]
+ CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+ CRUSH rule 0 x 173 [5,10,12,15,6,1,2,9]
+ CRUSH rule 0 x 174 [15,6,4,12,1,11,9,2]
+ CRUSH rule 0 x 175 [5,7,9,3,10,1,14,13]
+ CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+ CRUSH rule 0 x 177 [2,9,10,13,4,1,14,7]
+ CRUSH rule 0 x 178 [12,11,7,14,3,4]
+ CRUSH rule 0 x 179 [2,10,13,9,5,1,7,14]
+ CRUSH rule 0 x 180 [3,11,5,15,7,12]
+ CRUSH rule 0 x 181 [9,12,6,5,1,10,14,2]
+ CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+ CRUSH rule 0 x 183 [5,7,10,13,3,9,14,1]
+ CRUSH rule 0 x 184 [2,5,11,12,7,1,9,15]
+ CRUSH rule 0 x 185 [13,5,7,11,2,14]
+ CRUSH rule 0 x 186 [6,14,13,5,10,1,3,9]
+ CRUSH rule 0 x 187 [1,4,11,13,6,14,9,3]
+ CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+ CRUSH rule 0 x 189 [6,12,4,9,2,1,11,15]
+ CRUSH rule 0 x 190 [9,13,15,10,3,1,5,6]
+ CRUSH rule 0 x 191 [7,11,4,1,15,12,9,2]
+ CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+ CRUSH rule 0 x 193 [3,13,6,10,4,1,9,14]
+ CRUSH rule 0 x 194 [3,13,4,14,6,9,1,11]
+ CRUSH rule 0 x 195 [5,7,10,12,1,3,15,9]
+ CRUSH rule 0 x 196 [4,15,1,10,9,2,13,7]
+ CRUSH rule 0 x 197 [14,10,13,4,6,3,1,9]
+ CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+ CRUSH rule 0 x 199 [2,10,4,15,1,9,6,12]
+ CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+ CRUSH rule 0 x 201 [9,14,1,7,4,3,10,13]
+ CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+ CRUSH rule 0 x 203 [12,5,7,15,1,2,10,9]
+ CRUSH rule 0 x 204 [6,11,3,12,14,1,9,4]
+ CRUSH rule 0 x 205 [15,4,6,10,13,9,2,1]
+ CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+ CRUSH rule 0 x 207 [2,11,7,4,14,1,12,9]
+ CRUSH rule 0 x 208 [13,1,6,14,9,11,3,5]
+ CRUSH rule 0 x 209 [6,15,13,1,11,4,9,2]
+ CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+ CRUSH rule 0 x 211 [2,14,1,13,11,7,9,5]
+ CRUSH rule 0 x 212 [10,1,12,15,5,6,2,9]
+ CRUSH rule 0 x 213 [3,9,6,5,15,13,1,11]
+ CRUSH rule 0 x 214 [7,15,4,1,10,2,13,9]
+ CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+ CRUSH rule 0 x 216 [12,9,6,2,1,11,5,15]
+ CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+ CRUSH rule 0 x 218 [12,10,15,6,1,4,9,2]
+ CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+ CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+ CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+ CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+ CRUSH rule 0 x 223 [9,7,11,1,4,14,13,3]
+ CRUSH rule 0 x 224 [1,7,10,2,12,9,14,5]
+ CRUSH rule 0 x 225 [10,5,2,6,1,13,9,15]
+ CRUSH rule 0 x 226 [4,1,9,3,13,10,15,7]
+ CRUSH rule 0 x 227 [7,2,12,15,5,11]
+ CRUSH rule 0 x 228 [2,15,11,1,6,13,9,4]
+ CRUSH rule 0 x 229 [9,3,7,14,1,12,4,10]
+ CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+ CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+ CRUSH rule 0 x 232 [10,5,13,1,9,2,7,14]
+ CRUSH rule 0 x 233 [6,12,11,4,9,14,1,3]
+ CRUSH rule 0 x 234 [10,1,2,12,5,9,15,7]
+ CRUSH rule 0 x 235 [13,14,7,10,1,9,5,3]
+ CRUSH rule 0 x 236 [2,15,9,12,1,7,4,10]
+ CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+ CRUSH rule 0 x 238 [2,10,4,15,6,12,9,1]
+ CRUSH rule 0 x 239 [4,15,10,7,9,13,3,1]
+ CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+ CRUSH rule 0 x 241 [7,9,15,12,1,5,2,10]
+ CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+ CRUSH rule 0 x 243 [2,11,5,1,15,6,9,13]
+ CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+ CRUSH rule 0 x 245 [12,9,15,3,1,5,10,7]
+ CRUSH rule 0 x 246 [15,3,5,11,7,1,12,9]
+ CRUSH rule 0 x 247 [6,4,9,12,1,2,10,14]
+ CRUSH rule 0 x 248 [5,13,7,11,9,15,3,1]
+ CRUSH rule 0 x 249 [10,14,7,3,9,13,1,4]
+ CRUSH rule 0 x 250 [12,15,1,10,5,6,3,9]
+ CRUSH rule 0 x 251 [13,2,15,5,6,1,9,10]
+ CRUSH rule 0 x 252 [7,5,13,9,3,10,14,1]
+ CRUSH rule 0 x 253 [3,13,15,10,7,4]
+ CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+ CRUSH rule 0 x 255 [1,9,13,2,6,10,4,15]
+ CRUSH rule 0 x 256 [6,9,13,1,3,14,5,10]
+ CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+ CRUSH rule 0 x 258 [12,5,6,10,2,1,14,9]
+ CRUSH rule 0 x 259 [9,10,4,3,14,13,1,7]
+ CRUSH rule 0 x 260 [10,12,6,9,3,15,1,4]
+ CRUSH rule 0 x 261 [13,7,2,1,15,5,11,9]
+ CRUSH rule 0 x 262 [15,3,12,7,4,9,1,11]
+ CRUSH rule 0 x 263 [12,6,10,9,5,15,3,1]
+ CRUSH rule 0 x 264 [13,14,11,3,1,4,7,9]
+ CRUSH rule 0 x 265 [12,10,14,5,7,1,9,3]
+ CRUSH rule 0 x 266 [14,7,11,1,2,9,4,12]
+ CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+ CRUSH rule 0 x 268 [4,1,15,12,6,11,3,9]
+ CRUSH rule 0 x 269 [11,1,15,5,13,9,7,2]
+ CRUSH rule 0 x 270 [7,11,12,3,1,14,9,4]
+ CRUSH rule 0 x 271 [4,7,3,13,15,10,9,1]
+ CRUSH rule 0 x 272 [15,5,13,10,6,2]
+ CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+ CRUSH rule 0 x 274 [10,2,5,6,13,9,15,1]
+ CRUSH rule 0 x 275 [10,3,4,7,14,13]
+ CRUSH rule 0 x 276 [5,12,9,2,11,7,15,1]
+ CRUSH rule 0 x 277 [14,3,13,4,1,9,11,7]
+ CRUSH rule 0 x 278 [5,6,14,3,1,11,13,9]
+ CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+ CRUSH rule 0 x 280 [7,3,14,9,1,11,4,13]
+ CRUSH rule 0 x 281 [5,11,14,7,9,13,2,1]
+ CRUSH rule 0 x 282 [2,1,13,14,9,7,5,10]
+ CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+ CRUSH rule 0 x 284 [5,11,7,15,3,13,1,9]
+ CRUSH rule 0 x 285 [15,5,3,1,6,13,11,9]
+ CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+ CRUSH rule 0 x 287 [12,4,9,1,3,11,15,7]
+ CRUSH rule 0 x 288 [4,12,10,7,1,3,14,9]
+ CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+ CRUSH rule 0 x 290 [12,2,5,6,15,9,1,10]
+ CRUSH rule 0 x 291 [7,11,1,14,5,9,2,12]
+ CRUSH rule 0 x 292 [4,10,6,3,14,9,12,1]
+ CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+ CRUSH rule 0 x 294 [9,12,3,14,6,11,5,1]
+ CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+ CRUSH rule 0 x 296 [3,1,13,7,14,9,10,4]
+ CRUSH rule 0 x 297 [6,13,4,14,10,1,2,9]
+ CRUSH rule 0 x 298 [14,9,13,1,4,2,7,10]
+ CRUSH rule 0 x 299 [14,12,11,6,4,2,1,9]
+ CRUSH rule 0 x 300 [15,7,10,5,1,3,13,9]
+ CRUSH rule 0 x 301 [9,11,7,1,13,14,4,2]
+ CRUSH rule 0 x 302 [9,7,1,13,5,10,3,15]
+ CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+ CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+ CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+ CRUSH rule 0 x 306 [10,12,4,6,9,2,15,1]
+ CRUSH rule 0 x 307 [11,12,15,5,6,2,1,9]
+ CRUSH rule 0 x 308 [12,14,10,9,1,2,5,7]
+ CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+ CRUSH rule 0 x 310 [3,1,5,10,14,9,7,12]
+ CRUSH rule 0 x 311 [3,9,7,1,14,13,10,5]
+ CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+ CRUSH rule 0 x 313 [9,15,3,7,5,13,1,11]
+ CRUSH rule 0 x 314 [2,15,9,5,6,12,1,11]
+ CRUSH rule 0 x 315 [15,2,13,1,11,9,6,4]
+ CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+ CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+ CRUSH rule 0 x 318 [4,1,15,11,9,13,6,2]
+ CRUSH rule 0 x 319 [2,15,4,1,11,9,7,12]
+ CRUSH rule 0 x 320 [5,7,13,9,11,2,1,15]
+ CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+ CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+ CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+ CRUSH rule 0 x 324 [5,6,10,15,2,13]
+ CRUSH rule 0 x 325 [9,10,14,5,1,6,2,13]
+ CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+ CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+ CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+ CRUSH rule 0 x 329 [2,6,15,5,9,10,13,1]
+ CRUSH rule 0 x 330 [3,9,11,13,1,6,5,14]
+ CRUSH rule 0 x 331 [12,14,6,3,1,4,10,9]
+ CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+ CRUSH rule 0 x 333 [6,5,3,12,14,10,9,1]
+ CRUSH rule 0 x 334 [4,9,2,12,7,11,15,1]
+ CRUSH rule 0 x 335 [11,7,1,5,13,2,9,15]
+ CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+ CRUSH rule 0 x 337 [15,11,3,7,12,5]
+ CRUSH rule 0 x 338 [10,5,3,6,15,1,9,13]
+ CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+ CRUSH rule 0 x 340 [11,6,12,4,9,3,14,1]
+ CRUSH rule 0 x 341 [7,5,2,10,14,9,1,12]
+ CRUSH rule 0 x 342 [12,14,1,9,2,11,4,7]
+ CRUSH rule 0 x 343 [12,14,9,6,10,2,4,1]
+ CRUSH rule 0 x 344 [9,11,5,2,14,13,1,7]
+ CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+ CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+ CRUSH rule 0 x 347 [10,2,12,6,9,1,14,5]
+ CRUSH rule 0 x 348 [7,9,10,1,14,13,3,4]
+ CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+ CRUSH rule 0 x 350 [13,9,15,4,10,7,2,1]
+ CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+ CRUSH rule 0 x 352 [1,12,11,9,4,7,3,15]
+ CRUSH rule 0 x 353 [10,14,12,2,9,1,4,6]
+ CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+ CRUSH rule 0 x 355 [13,14,6,10,2,5,1,9]
+ CRUSH rule 0 x 356 [15,13,2,9,6,5,1,11]
+ CRUSH rule 0 x 357 [4,11,1,13,3,14,6,9]
+ CRUSH rule 0 x 358 [12,7,2,9,1,14,10,4]
+ CRUSH rule 0 x 359 [5,15,7,11,3,13]
+ CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+ CRUSH rule 0 x 361 [5,3,13,6,1,14,11,9]
+ CRUSH rule 0 x 362 [2,9,11,13,1,6,5,15]
+ CRUSH rule 0 x 363 [7,12,3,9,15,4,1,10]
+ CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+ CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+ CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+ CRUSH rule 0 x 367 [7,13,3,1,5,11,15,9]
+ CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+ CRUSH rule 0 x 369 [7,5,3,13,14,9,11,1]
+ CRUSH rule 0 x 370 [4,7,14,1,2,9,11,12]
+ CRUSH rule 0 x 371 [1,7,12,3,4,15,10,9]
+ CRUSH rule 0 x 372 [10,4,3,14,6,1,12,9]
+ CRUSH rule 0 x 373 [15,5,2,6,13,1,9,10]
+ CRUSH rule 0 x 374 [3,15,12,5,1,6,10,9]
+ CRUSH rule 0 x 375 [5,2,14,1,6,13,11,9]
+ CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+ CRUSH rule 0 x 377 [1,15,2,4,9,11,12,6]
+ CRUSH rule 0 x 378 [9,12,2,15,1,5,11,6]
+ CRUSH rule 0 x 379 [11,2,15,5,7,9,13,1]
+ CRUSH rule 0 x 380 [6,1,12,11,2,9,5,14]
+ CRUSH rule 0 x 381 [15,13,7,5,10,2,1,9]
+ CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+ CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+ CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+ CRUSH rule 0 x 385 [4,6,15,3,10,9,1,13]
+ CRUSH rule 0 x 386 [14,3,11,13,5,6,9,1]
+ CRUSH rule 0 x 387 [1,11,5,7,9,2,14,12]
+ CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+ CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+ CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+ CRUSH rule 0 x 391 [3,4,9,13,7,10,1,14]
+ CRUSH rule 0 x 392 [11,5,14,7,1,9,2,12]
+ CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+ CRUSH rule 0 x 394 [4,9,3,15,13,6,1,11]
+ CRUSH rule 0 x 395 [10,13,5,15,6,9,3,1]
+ CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+ CRUSH rule 0 x 397 [1,14,9,4,12,10,3,7]
+ CRUSH rule 0 x 398 [9,2,1,5,12,6,11,15]
+ CRUSH rule 0 x 399 [5,9,14,3,1,10,13,7]
+ CRUSH rule 0 x 400 [10,6,2,4,15,12,1,9]
+ CRUSH rule 0 x 401 [6,9,11,12,4,3,15,1]
+ CRUSH rule 0 x 402 [4,7,9,2,13,1,15,11]
+ CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+ CRUSH rule 0 x 404 [14,12,7,9,2,1,5,11]
+ CRUSH rule 0 x 405 [9,15,11,2,4,7,13,1]
+ CRUSH rule 0 x 406 [12,14,9,2,7,10,4,1]
+ CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+ CRUSH rule 0 x 408 [7,1,5,2,10,15,13,9]
+ CRUSH rule 0 x 409 [11,2,4,13,1,15,7,9]
+ CRUSH rule 0 x 410 [6,4,14,2,12,9,10,1]
+ CRUSH rule 0 x 411 [13,11,15,6,4,1,9,2]
+ CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+ CRUSH rule 0 x 413 [13,5,3,11,6,9,1,14]
+ CRUSH rule 0 x 414 [3,11,9,13,4,1,6,15]
+ CRUSH rule 0 x 415 [6,10,14,5,1,13,3,9]
+ CRUSH rule 0 x 416 [13,1,4,7,2,9,14,11]
+ CRUSH rule 0 x 417 [4,12,1,15,2,11,9,6]
+ CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+ CRUSH rule 0 x 419 [5,14,10,9,2,12,6,1]
+ CRUSH rule 0 x 420 [2,4,9,11,6,14,13,1]
+ CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+ CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+ CRUSH rule 0 x 423 [3,15,12,6,5,1,9,10]
+ CRUSH rule 0 x 424 [6,10,12,2,5,1,14,9]
+ CRUSH rule 0 x 425 [11,15,2,13,5,7,9,1]
+ CRUSH rule 0 x 426 [12,4,7,1,9,10,14,2]
+ CRUSH rule 0 x 427 [14,10,3,1,9,7,5,13]
+ CRUSH rule 0 x 428 [12,7,9,4,2,1,14,10]
+ CRUSH rule 0 x 429 [3,4,9,7,11,12,1,14]
+ CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+ CRUSH rule 0 x 431 [9,3,7,1,12,5,14,11]
+ CRUSH rule 0 x 432 [4,1,12,7,15,2,10,9]
+ CRUSH rule 0 x 433 [4,11,12,15,7,3]
+ CRUSH rule 0 x 434 [2,14,9,1,5,11,7,13]
+ CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+ CRUSH rule 0 x 436 [9,15,10,2,4,1,12,7]
+ CRUSH rule 0 x 437 [9,6,3,14,10,12,5,1]
+ CRUSH rule 0 x 438 [7,2,13,4,11,1,9,14]
+ CRUSH rule 0 x 439 [7,14,4,3,12,10]
+ CRUSH rule 0 x 440 [14,11,9,2,7,12,1,5]
+ CRUSH rule 0 x 441 [2,4,11,9,13,6,1,14]
+ CRUSH rule 0 x 442 [10,13,9,7,15,1,4,2]
+ CRUSH rule 0 x 443 [12,15,10,9,2,1,6,4]
+ CRUSH rule 0 x 444 [4,13,7,14,3,1,9,11]
+ CRUSH rule 0 x 445 [4,2,15,7,1,9,11,12]
+ CRUSH rule 0 x 446 [12,10,6,9,4,1,15,3]
+ CRUSH rule 0 x 447 [15,7,13,1,4,9,3,11]
+ CRUSH rule 0 x 448 [5,2,13,7,15,10]
+ CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+ CRUSH rule 0 x 450 [2,4,6,9,15,1,13,10]
+ CRUSH rule 0 x 451 [6,14,11,3,9,1,12,5]
+ CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+ CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+ CRUSH rule 0 x 454 [10,4,2,6,15,12,9,1]
+ CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+ CRUSH rule 0 x 456 [5,7,13,1,11,3,9,15]
+ CRUSH rule 0 x 457 [9,1,5,7,11,13,15,2]
+ CRUSH rule 0 x 458 [9,11,15,4,7,2,12,1]
+ CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+ CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+ CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+ CRUSH rule 0 x 462 [4,7,12,14,11,1,3,9]
+ CRUSH rule 0 x 463 [4,12,14,11,2,7,1,9]
+ CRUSH rule 0 x 464 [4,2,15,10,1,9,13,7]
+ CRUSH rule 0 x 465 [5,10,9,7,13,1,3,14]
+ CRUSH rule 0 x 466 [13,5,2,15,9,11,6,1]
+ CRUSH rule 0 x 467 [13,6,14,3,9,1,11,5]
+ CRUSH rule 0 x 468 [10,7,12,14,4,1,9,2]
+ CRUSH rule 0 x 469 [4,9,6,14,12,11,3,1]
+ CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+ CRUSH rule 0 x 471 [6,1,5,14,13,10,9,3]
+ CRUSH rule 0 x 472 [2,14,7,5,13,1,11,9]
+ CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+ CRUSH rule 0 x 474 [15,10,4,12,6,9,2,1]
+ CRUSH rule 0 x 475 [10,5,12,9,14,3,6,1]
+ CRUSH rule 0 x 476 [3,6,10,12,1,15,9,4]
+ CRUSH rule 0 x 477 [6,13,5,15,11,9,2,1]
+ CRUSH rule 0 x 478 [4,15,1,3,7,12,10,9]
+ CRUSH rule 0 x 479 [13,11,1,6,14,5,9,3]
+ CRUSH rule 0 x 480 [1,13,6,4,9,14,11,3]
+ CRUSH rule 0 x 481 [15,12,7,9,1,3,10,4]
+ CRUSH rule 0 x 482 [2,12,9,1,7,11,14,4]
+ CRUSH rule 0 x 483 [10,1,4,15,9,7,13,2]
+ CRUSH rule 0 x 484 [1,4,10,13,7,14,2,9]
+ CRUSH rule 0 x 485 [9,4,3,1,14,12,7,10]
+ CRUSH rule 0 x 486 [3,10,15,9,7,13,4,1]
+ CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+ CRUSH rule 0 x 488 [14,4,1,9,2,6,10,12]
+ CRUSH rule 0 x 489 [11,4,2,13,15,7]
+ CRUSH rule 0 x 490 [4,9,1,3,13,15,6,11]
+ CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+ CRUSH rule 0 x 492 [5,7,11,3,14,9,1,13]
+ CRUSH rule 0 x 493 [12,1,4,15,3,11,9,6]
+ CRUSH rule 0 x 494 [1,7,13,4,15,9,10,3]
+ CRUSH rule 0 x 495 [3,15,7,1,9,5,12,11]
+ CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+ CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+ CRUSH rule 0 x 498 [10,6,1,5,9,12,3,15]
+ CRUSH rule 0 x 499 [14,3,12,5,1,11,9,7]
+ CRUSH rule 0 x 500 [15,9,6,12,11,2,1,5]
+ CRUSH rule 0 x 501 [10,13,1,9,3,14,5,7]
+ CRUSH rule 0 x 502 [5,1,14,11,7,12,9,2]
+ CRUSH rule 0 x 503 [15,10,7,9,1,12,4,2]
+ CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+ CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+ CRUSH rule 0 x 506 [11,7,9,14,12,1,2,5]
+ CRUSH rule 0 x 507 [4,14,13,3,9,7,1,10]
+ CRUSH rule 0 x 508 [12,1,4,9,2,11,15,7]
+ CRUSH rule 0 x 509 [4,2,6,9,14,1,10,13]
+ CRUSH rule 0 x 510 [5,3,1,12,11,14,9,7]
+ CRUSH rule 0 x 511 [2,12,10,6,14,5]
+ CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+ CRUSH rule 0 x 513 [4,9,11,3,13,7,1,14]
+ CRUSH rule 0 x 514 [11,9,3,4,12,15,6,1]
+ CRUSH rule 0 x 515 [12,14,6,5,3,9,1,10]
+ CRUSH rule 0 x 516 [14,11,1,12,3,7,4,9]
+ CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+ CRUSH rule 0 x 518 [3,5,7,12,15,11,9,1]
+ CRUSH rule 0 x 519 [12,14,2,1,4,6,9,10]
+ CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+ CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+ CRUSH rule 0 x 522 [4,12,11,1,15,3,9,6]
+ CRUSH rule 0 x 523 [3,1,5,9,15,10,13,7]
+ CRUSH rule 0 x 524 [15,9,3,11,13,7,4,1]
+ CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+ CRUSH rule 0 x 526 [10,2,5,13,6,15,1,9]
+ CRUSH rule 0 x 527 [3,13,4,1,9,10,14,7]
+ CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+ CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+ CRUSH rule 0 x 530 [11,9,12,7,5,1,3,15]
+ CRUSH rule 0 x 531 [9,15,4,7,2,13,1,11]
+ CRUSH rule 0 x 532 [5,3,13,7,9,14,1,10]
+ CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+ CRUSH rule 0 x 534 [11,9,3,7,15,4,1,12]
+ CRUSH rule 0 x 535 [11,1,3,5,14,9,12,7]
+ CRUSH rule 0 x 536 [9,1,14,13,4,6,2,11]
+ CRUSH rule 0 x 537 [15,5,13,2,7,11]
+ CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+ CRUSH rule 0 x 539 [10,12,6,14,1,2,9,5]
+ CRUSH rule 0 x 540 [12,15,7,3,9,11,1,4]
+ CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+ CRUSH rule 0 x 542 [3,9,15,5,11,12,7,1]
+ CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+ CRUSH rule 0 x 544 [3,15,9,11,7,4,12,1]
+ CRUSH rule 0 x 545 [14,10,7,12,4,9,1,3]
+ CRUSH rule 0 x 546 [5,15,13,7,1,10,9,2]
+ CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+ CRUSH rule 0 x 548 [11,7,12,15,4,2]
+ CRUSH rule 0 x 549 [14,1,4,9,13,6,3,10]
+ CRUSH rule 0 x 550 [9,15,3,13,1,6,4,11]
+ CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+ CRUSH rule 0 x 552 [2,11,14,1,9,6,5,12]
+ CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+ CRUSH rule 0 x 554 [11,14,6,4,13,9,3,1]
+ CRUSH rule 0 x 555 [6,5,10,9,14,2,13,1]
+ CRUSH rule 0 x 556 [15,6,3,13,11,4,1,9]
+ CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+ CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+ CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+ CRUSH rule 0 x 560 [4,9,12,6,3,10,1,15]
+ CRUSH rule 0 x 561 [12,7,1,2,5,15,11,9]
+ CRUSH rule 0 x 562 [7,13,9,14,2,1,11,4]
+ CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+ CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+ CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+ CRUSH rule 0 x 566 [6,14,4,2,13,11]
+ CRUSH rule 0 x 567 [15,4,11,6,3,12]
+ CRUSH rule 0 x 568 [4,14,1,6,10,13,3,9]
+ CRUSH rule 0 x 569 [11,3,15,13,5,1,9,7]
+ CRUSH rule 0 x 570 [1,10,13,4,7,2,9,14]
+ CRUSH rule 0 x 571 [10,12,14,9,4,2,1,6]
+ CRUSH rule 0 x 572 [12,14,3,10,6,1,4,9]
+ CRUSH rule 0 x 573 [7,15,11,2,12,9,4,1]
+ CRUSH rule 0 x 574 [11,14,13,1,3,7,4,9]
+ CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+ CRUSH rule 0 x 576 [3,15,11,9,1,6,5,13]
+ CRUSH rule 0 x 577 [13,9,6,15,3,11,4,1]
+ CRUSH rule 0 x 578 [4,10,1,2,7,13,14,9]
+ CRUSH rule 0 x 579 [13,1,15,2,10,7,5,9]
+ CRUSH rule 0 x 580 [3,12,4,1,10,15,7,9]
+ CRUSH rule 0 x 581 [7,14,12,10,1,2,9,5]
+ CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+ CRUSH rule 0 x 583 [4,15,1,9,10,12,2,6]
+ CRUSH rule 0 x 584 [10,1,5,13,6,9,2,15]
+ CRUSH rule 0 x 585 [5,3,6,1,11,14,13,9]
+ CRUSH rule 0 x 586 [7,10,14,12,9,3,5,1]
+ CRUSH rule 0 x 587 [11,6,9,4,1,14,13,2]
+ CRUSH rule 0 x 588 [3,12,7,15,4,9,1,10]
+ CRUSH rule 0 x 589 [9,7,12,1,10,3,4,15]
+ CRUSH rule 0 x 590 [12,1,3,9,10,6,4,14]
+ CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+ CRUSH rule 0 x 592 [15,12,9,7,5,2,11,1]
+ CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+ CRUSH rule 0 x 594 [12,14,2,9,7,4,1,11]
+ CRUSH rule 0 x 595 [12,7,10,3,1,14,9,4]
+ CRUSH rule 0 x 596 [2,7,12,11,1,5,15,9]
+ CRUSH rule 0 x 597 [15,1,2,10,7,13,5,9]
+ CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+ CRUSH rule 0 x 599 [13,11,1,5,6,2,15,9]
+ CRUSH rule 0 x 600 [4,12,3,10,9,7,1,14]
+ CRUSH rule 0 x 601 [13,5,15,2,1,7,9,10]
+ CRUSH rule 0 x 602 [3,11,7,1,13,15,5,9]
+ CRUSH rule 0 x 603 [3,1,4,14,10,9,6,12]
+ CRUSH rule 0 x 604 [14,2,6,1,11,13,9,4]
+ CRUSH rule 0 x 605 [2,7,12,5,14,10,1,9]
+ CRUSH rule 0 x 606 [12,15,1,5,7,9,3,11]
+ CRUSH rule 0 x 607 [3,9,10,14,7,1,4,12]
+ CRUSH rule 0 x 608 [13,10,1,7,9,15,5,2]
+ CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+ CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+ CRUSH rule 0 x 611 [13,1,5,3,10,7,15,9]
+ CRUSH rule 0 x 612 [7,1,2,13,9,15,4,11]
+ CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+ CRUSH rule 0 x 614 [9,4,15,3,1,11,6,12]
+ CRUSH rule 0 x 615 [9,4,11,2,1,12,6,15]
+ CRUSH rule 0 x 616 [10,14,1,5,3,6,12,9]
+ CRUSH rule 0 x 617 [15,7,2,11,12,1,9,4]
+ CRUSH rule 0 x 618 [4,2,10,6,14,9,1,12]
+ CRUSH rule 0 x 619 [15,4,3,9,6,1,13,11]
+ CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+ CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+ CRUSH rule 0 x 622 [10,2,13,5,15,9,1,7]
+ CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+ CRUSH rule 0 x 624 [3,9,15,6,10,1,5,12]
+ CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+ CRUSH rule 0 x 626 [10,12,2,1,9,7,5,14]
+ CRUSH rule 0 x 627 [1,12,10,14,3,5,9,7]
+ CRUSH rule 0 x 628 [15,13,11,4,2,1,7,9]
+ CRUSH rule 0 x 629 [5,6,15,12,1,10,3,9]
+ CRUSH rule 0 x 630 [1,4,12,9,3,7,15,11]
+ CRUSH rule 0 x 631 [5,7,1,15,12,11,3,9]
+ CRUSH rule 0 x 632 [12,3,11,9,6,1,15,5]
+ CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+ CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+ CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+ CRUSH rule 0 x 636 [13,6,11,3,15,9,1,4]
+ CRUSH rule 0 x 637 [3,1,10,6,9,12,4,14]
+ CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+ CRUSH rule 0 x 639 [6,9,14,4,3,1,10,13]
+ CRUSH rule 0 x 640 [9,6,1,11,14,2,4,13]
+ CRUSH rule 0 x 641 [10,6,5,14,1,9,12,2]
+ CRUSH rule 0 x 642 [1,15,4,6,2,10,9,12]
+ CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+ CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+ CRUSH rule 0 x 645 [14,2,4,9,10,1,7,13]
+ CRUSH rule 0 x 646 [5,13,14,1,6,9,2,11]
+ CRUSH rule 0 x 647 [10,1,9,13,6,2,14,5]
+ CRUSH rule 0 x 648 [6,5,2,14,11,1,12,9]
+ CRUSH rule 0 x 649 [3,9,13,11,4,14,1,7]
+ CRUSH rule 0 x 650 [10,9,4,15,12,7,1,2]
+ CRUSH rule 0 x 651 [3,9,5,7,14,1,13,11]
+ CRUSH rule 0 x 652 [15,9,4,6,13,1,2,11]
+ CRUSH rule 0 x 653 [11,14,1,3,6,9,12,5]
+ CRUSH rule 0 x 654 [13,6,2,10,15,4,1,9]
+ CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+ CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+ CRUSH rule 0 x 657 [11,15,3,5,7,13,1,9]
+ CRUSH rule 0 x 658 [7,2,10,12,1,4,9,14]
+ CRUSH rule 0 x 659 [2,5,14,6,10,12]
+ CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+ CRUSH rule 0 x 661 [7,15,3,12,11,4,9,1]
+ CRUSH rule 0 x 662 [15,2,12,5,1,10,9,7]
+ CRUSH rule 0 x 663 [14,9,13,10,5,3,1,6]
+ CRUSH rule 0 x 664 [6,10,12,4,9,2,1,15]
+ CRUSH rule 0 x 665 [2,9,12,1,7,10,4,15]
+ CRUSH rule 0 x 666 [12,3,6,1,15,9,10,4]
+ CRUSH rule 0 x 667 [1,9,12,10,2,14,7,4]
+ CRUSH rule 0 x 668 [9,5,1,2,6,11,13,15]
+ CRUSH rule 0 x 669 [9,7,14,5,11,13,1,2]
+ CRUSH rule 0 x 670 [6,10,9,13,1,2,15,4]
+ CRUSH rule 0 x 671 [6,15,5,10,13,3]
+ CRUSH rule 0 x 672 [2,9,13,1,4,14,6,10]
+ CRUSH rule 0 x 673 [7,10,5,9,15,13,2,1]
+ CRUSH rule 0 x 674 [7,12,10,1,14,9,3,4]
+ CRUSH rule 0 x 675 [9,5,1,10,6,14,12,2]
+ CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+ CRUSH rule 0 x 677 [2,12,1,4,10,6,15,9]
+ CRUSH rule 0 x 678 [1,2,4,10,12,14,9,6]
+ CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+ CRUSH rule 0 x 680 [7,11,3,1,15,4,9,12]
+ CRUSH rule 0 x 681 [6,4,3,11,14,13,1,9]
+ CRUSH rule 0 x 682 [6,1,11,15,12,2,5,9]
+ CRUSH rule 0 x 683 [6,13,2,4,9,14,10,1]
+ CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+ CRUSH rule 0 x 685 [5,1,15,7,9,2,10,13]
+ CRUSH rule 0 x 686 [1,9,11,14,6,13,4,3]
+ CRUSH rule 0 x 687 [7,13,3,5,11,9,15,1]
+ CRUSH rule 0 x 688 [11,9,1,14,3,5,7,12]
+ CRUSH rule 0 x 689 [5,2,9,12,1,14,11,7]
+ CRUSH rule 0 x 690 [9,7,10,3,13,15,5,1]
+ CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+ CRUSH rule 0 x 692 [15,5,1,2,9,11,12,7]
+ CRUSH rule 0 x 693 [5,6,12,15,2,10,9,1]
+ CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+ CRUSH rule 0 x 695 [6,13,14,10,9,5,1,3]
+ CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+ CRUSH rule 0 x 697 [13,11,3,6,4,14,9,1]
+ CRUSH rule 0 x 698 [11,13,4,2,6,1,9,15]
+ CRUSH rule 0 x 699 [7,14,12,4,2,11]
+ CRUSH rule 0 x 700 [12,14,11,9,4,6,3,1]
+ CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+ CRUSH rule 0 x 702 [3,12,15,6,5,11,1,9]
+ CRUSH rule 0 x 703 [15,11,13,3,4,7,1,9]
+ CRUSH rule 0 x 704 [6,4,2,15,11,1,13,9]
+ CRUSH rule 0 x 705 [14,6,11,5,1,13,9,3]
+ CRUSH rule 0 x 706 [1,12,3,6,4,10,15,9]
+ CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+ CRUSH rule 0 x 708 [3,10,5,1,15,9,7,13]
+ CRUSH rule 0 x 709 [11,12,3,7,5,14,1,9]
+ CRUSH rule 0 x 710 [14,2,11,9,5,7,12,1]
+ CRUSH rule 0 x 711 [14,3,9,10,12,5,6,1]
+ CRUSH rule 0 x 712 [12,3,11,15,9,1,6,4]
+ CRUSH rule 0 x 713 [11,9,3,15,13,6,4,1]
+ CRUSH rule 0 x 714 [12,1,9,7,2,15,10,5]
+ CRUSH rule 0 x 715 [6,1,14,4,11,12,3,9]
+ CRUSH rule 0 x 716 [11,13,9,14,5,2,1,7]
+ CRUSH rule 0 x 717 [12,4,10,9,15,1,2,7]
+ CRUSH rule 0 x 718 [7,15,5,2,11,13]
+ CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+ CRUSH rule 0 x 720 [4,13,10,2,7,9,1,14]
+ CRUSH rule 0 x 721 [11,3,14,9,1,12,4,6]
+ CRUSH rule 0 x 722 [2,4,6,1,9,15,13,10]
+ CRUSH rule 0 x 723 [2,1,12,15,11,7,5,9]
+ CRUSH rule 0 x 724 [7,1,9,10,5,15,13,2]
+ CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+ CRUSH rule 0 x 726 [7,14,4,3,11,13,9,1]
+ CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+ CRUSH rule 0 x 728 [13,11,4,6,15,2]
+ CRUSH rule 0 x 729 [15,11,4,6,2,9,1,13]
+ CRUSH rule 0 x 730 [3,7,1,13,11,15,9,5]
+ CRUSH rule 0 x 731 [9,1,6,5,2,11,13,15]
+ CRUSH rule 0 x 732 [1,2,10,13,9,4,7,15]
+ CRUSH rule 0 x 733 [11,3,5,6,1,9,12,15]
+ CRUSH rule 0 x 734 [14,3,11,7,12,9,4,1]
+ CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+ CRUSH rule 0 x 736 [3,9,1,11,7,5,13,14]
+ CRUSH rule 0 x 737 [1,4,2,12,9,10,6,15]
+ CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+ CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+ CRUSH rule 0 x 740 [7,9,10,13,1,15,2,5]
+ CRUSH rule 0 x 741 [12,11,7,15,2,5]
+ CRUSH rule 0 x 742 [9,7,4,11,12,1,14,3]
+ CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+ CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+ CRUSH rule 0 x 745 [3,6,1,4,11,12,14,9]
+ CRUSH rule 0 x 746 [3,7,9,10,14,5,1,13]
+ CRUSH rule 0 x 747 [15,11,5,2,13,9,1,7]
+ CRUSH rule 0 x 748 [6,10,13,2,14,5,9,1]
+ CRUSH rule 0 x 749 [14,9,10,7,5,1,2,12]
+ CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+ CRUSH rule 0 x 751 [15,1,6,9,5,11,12,3]
+ CRUSH rule 0 x 752 [13,1,7,3,11,15,9,4]
+ CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+ CRUSH rule 0 x 754 [14,12,11,4,2,1,9,6]
+ CRUSH rule 0 x 755 [13,6,1,10,4,2,14,9]
+ CRUSH rule 0 x 756 [3,4,14,6,1,10,13,9]
+ CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+ CRUSH rule 0 x 758 [6,3,4,10,15,13,9,1]
+ CRUSH rule 0 x 759 [5,7,3,14,11,1,9,13]
+ CRUSH rule 0 x 760 [1,15,10,12,4,3,9,7]
+ CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+ CRUSH rule 0 x 762 [1,4,10,9,3,7,14,12]
+ CRUSH rule 0 x 763 [4,13,1,14,7,10,2,9]
+ CRUSH rule 0 x 764 [1,14,6,13,9,5,2,10]
+ CRUSH rule 0 x 765 [9,15,2,13,4,1,11,7]
+ CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+ CRUSH rule 0 x 767 [6,11,4,3,12,14]
+ CRUSH rule 0 x 768 [2,12,15,7,1,11,9,4]
+ CRUSH rule 0 x 769 [15,1,9,2,11,12,7,4]
+ CRUSH rule 0 x 770 [15,13,4,6,3,10,1,9]
+ CRUSH rule 0 x 771 [9,2,12,11,6,14,5,1]
+ CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+ CRUSH rule 0 x 773 [3,7,4,15,1,12,11,9]
+ CRUSH rule 0 x 774 [12,6,3,15,5,9,10,1]
+ CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+ CRUSH rule 0 x 776 [10,15,3,9,6,13,1,5]
+ CRUSH rule 0 x 777 [11,13,4,7,1,14,9,2]
+ CRUSH rule 0 x 778 [13,1,9,11,15,6,3,5]
+ CRUSH rule 0 x 779 [5,11,1,14,2,9,13,6]
+ CRUSH rule 0 x 780 [13,9,3,6,4,1,14,10]
+ CRUSH rule 0 x 781 [5,7,14,3,1,12,11,9]
+ CRUSH rule 0 x 782 [2,15,9,7,11,13,4,1]
+ CRUSH rule 0 x 783 [12,7,5,14,9,1,2,10]
+ CRUSH rule 0 x 784 [14,1,10,13,3,4,7,9]
+ CRUSH rule 0 x 785 [6,12,1,2,4,9,15,10]
+ CRUSH rule 0 x 786 [10,5,2,15,1,7,12,9]
+ CRUSH rule 0 x 787 [1,12,10,2,9,4,14,6]
+ CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+ CRUSH rule 0 x 789 [9,2,14,7,4,12,1,10]
+ CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+ CRUSH rule 0 x 791 [9,4,7,13,14,11,1,3]
+ CRUSH rule 0 x 792 [6,4,15,10,12,3]
+ CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+ CRUSH rule 0 x 794 [5,12,2,14,9,10,1,6]
+ CRUSH rule 0 x 795 [6,14,12,4,10,1,2,9]
+ CRUSH rule 0 x 796 [11,2,12,6,15,4]
+ CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+ CRUSH rule 0 x 798 [5,11,6,13,1,3,15,9]
+ CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+ CRUSH rule 0 x 800 [6,3,4,11,15,13]
+ CRUSH rule 0 x 801 [2,5,6,13,9,1,10,15]
+ CRUSH rule 0 x 802 [1,4,12,7,3,9,10,14]
+ CRUSH rule 0 x 803 [7,2,4,1,11,13,9,14]
+ CRUSH rule 0 x 804 [5,14,9,7,3,1,12,10]
+ CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+ CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+ CRUSH rule 0 x 807 [14,2,7,4,9,12,1,10]
+ CRUSH rule 0 x 808 [2,15,12,7,9,1,5,10]
+ CRUSH rule 0 x 809 [1,11,7,12,4,2,15,9]
+ CRUSH rule 0 x 810 [2,5,9,12,15,1,7,11]
+ CRUSH rule 0 x 811 [15,6,3,10,1,5,9,12]
+ CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+ CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+ CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+ CRUSH rule 0 x 815 [15,12,9,4,10,6,1,2]
+ CRUSH rule 0 x 816 [14,10,13,7,3,9,4,1]
+ CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+ CRUSH rule 0 x 818 [15,2,11,4,1,12,6,9]
+ CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+ CRUSH rule 0 x 820 [3,6,9,12,11,15,4,1]
+ CRUSH rule 0 x 821 [15,10,9,13,3,4,7,1]
+ CRUSH rule 0 x 822 [10,13,2,9,7,4,14,1]
+ CRUSH rule 0 x 823 [2,6,12,10,15,4,1,9]
+ CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+ CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+ CRUSH rule 0 x 826 [5,2,11,15,1,12,9,7]
+ CRUSH rule 0 x 827 [13,5,1,3,7,9,11,14]
+ CRUSH rule 0 x 828 [12,6,10,5,1,9,2,15]
+ CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+ CRUSH rule 0 x 830 [15,13,2,9,7,11,1,5]
+ CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+ CRUSH rule 0 x 832 [14,11,13,2,9,4,6,1]
+ CRUSH rule 0 x 833 [9,13,3,11,7,5,15,1]
+ CRUSH rule 0 x 834 [9,7,5,1,11,2,13,14]
+ CRUSH rule 0 x 835 [14,3,13,6,4,9,1,10]
+ CRUSH rule 0 x 836 [3,9,10,13,1,5,14,7]
+ CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+ CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+ CRUSH rule 0 x 839 [3,4,6,10,15,1,13,9]
+ CRUSH rule 0 x 840 [10,15,12,4,7,1,2,9]
+ CRUSH rule 0 x 841 [3,5,7,12,11,15,1,9]
+ CRUSH rule 0 x 842 [9,13,2,6,5,14,10,1]
+ CRUSH rule 0 x 843 [14,7,4,9,3,12,1,10]
+ CRUSH rule 0 x 844 [7,1,4,15,9,2,11,12]
+ CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+ CRUSH rule 0 x 846 [3,7,15,13,1,9,10,4]
+ CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+ CRUSH rule 0 x 848 [11,13,1,14,5,9,2,7]
+ CRUSH rule 0 x 849 [3,15,11,9,6,1,13,5]
+ CRUSH rule 0 x 850 [1,3,10,6,14,4,9,12]
+ CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+ CRUSH rule 0 x 852 [9,12,4,7,15,2,11,1]
+ CRUSH rule 0 x 853 [13,14,6,11,2,4,9,1]
+ CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+ CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+ CRUSH rule 0 x 856 [5,10,7,3,15,9,12,1]
+ CRUSH rule 0 x 857 [4,3,13,11,9,1,7,14]
+ CRUSH rule 0 x 858 [5,15,6,3,9,12,1,10]
+ CRUSH rule 0 x 859 [5,15,6,2,1,11,12,9]
+ CRUSH rule 0 x 860 [11,14,1,12,6,9,2,4]
+ CRUSH rule 0 x 861 [13,7,4,10,1,14,3,9]
+ CRUSH rule 0 x 862 [5,10,9,7,3,12,1,15]
+ CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+ CRUSH rule 0 x 864 [6,13,4,2,10,15,1,9]
+ CRUSH rule 0 x 865 [4,1,14,11,6,9,3,13]
+ CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+ CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+ CRUSH rule 0 x 868 [14,11,7,2,1,4,9,12]
+ CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+ CRUSH rule 0 x 870 [14,9,11,4,3,12,6,1]
+ CRUSH rule 0 x 871 [6,2,1,4,15,13,11,9]
+ CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+ CRUSH rule 0 x 873 [2,5,12,10,1,9,15,7]
+ CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+ CRUSH rule 0 x 875 [10,6,14,1,12,5,9,3]
+ CRUSH rule 0 x 876 [14,7,13,3,9,1,11,4]
+ CRUSH rule 0 x 877 [15,11,13,9,5,1,6,3]
+ CRUSH rule 0 x 878 [7,14,3,13,9,1,11,4]
+ CRUSH rule 0 x 879 [12,2,7,4,10,15]
+ CRUSH rule 0 x 880 [2,12,10,7,1,4,9,14]
+ CRUSH rule 0 x 881 [6,3,1,11,4,15,9,13]
+ CRUSH rule 0 x 882 [11,13,7,1,2,15,4,9]
+ CRUSH rule 0 x 883 [13,1,3,10,6,5,9,15]
+ CRUSH rule 0 x 884 [6,15,4,9,3,11,12,1]
+ CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+ CRUSH rule 0 x 886 [13,11,4,2,1,14,9,6]
+ CRUSH rule 0 x 887 [14,4,12,11,2,6,9,1]
+ CRUSH rule 0 x 888 [10,12,7,15,9,2,1,5]
+ CRUSH rule 0 x 889 [15,13,4,1,6,2,10,9]
+ CRUSH rule 0 x 890 [10,12,14,2,9,5,6,1]
+ CRUSH rule 0 x 891 [9,5,11,6,3,15,12,1]
+ CRUSH rule 0 x 892 [12,15,2,4,7,9,11,1]
+ CRUSH rule 0 x 893 [1,3,5,9,6,10,14,12]
+ CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+ CRUSH rule 0 x 895 [2,1,11,5,7,15,13,9]
+ CRUSH rule 0 x 896 [9,1,14,10,4,12,2,7]
+ CRUSH rule 0 x 897 [7,5,14,3,1,9,11,12]
+ CRUSH rule 0 x 898 [10,6,12,9,15,5,2,1]
+ CRUSH rule 0 x 899 [1,11,5,3,13,14,9,6]
+ CRUSH rule 0 x 900 [2,9,10,7,13,14,5,1]
+ CRUSH rule 0 x 901 [9,12,11,3,14,4,1,6]
+ CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+ CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+ CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+ CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+ CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+ CRUSH rule 0 x 907 [7,12,3,9,10,5,14,1]
+ CRUSH rule 0 x 908 [2,15,9,6,10,13,5,1]
+ CRUSH rule 0 x 909 [10,14,1,13,2,9,7,4]
+ CRUSH rule 0 x 910 [12,7,4,15,10,3,1,9]
+ CRUSH rule 0 x 911 [11,15,2,4,9,13,6,1]
+ CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+ CRUSH rule 0 x 913 [4,6,10,1,12,3,9,14]
+ CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+ CRUSH rule 0 x 915 [12,14,1,9,4,3,11,6]
+ CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+ CRUSH rule 0 x 917 [1,15,6,5,10,3,13,9]
+ CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+ CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+ CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+ CRUSH rule 0 x 921 [1,11,6,13,4,2,9,14]
+ CRUSH rule 0 x 922 [6,4,14,13,3,1,10,9]
+ CRUSH rule 0 x 923 [12,2,5,14,10,1,9,6]
+ CRUSH rule 0 x 924 [6,2,14,13,9,1,11,5]
+ CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+ CRUSH rule 0 x 926 [3,13,10,1,14,9,6,5]
+ CRUSH rule 0 x 927 [6,5,1,11,14,2,13,9]
+ CRUSH rule 0 x 928 [13,1,3,9,6,11,15,5]
+ CRUSH rule 0 x 929 [10,7,1,5,2,12,9,14]
+ CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+ CRUSH rule 0 x 931 [6,15,11,9,5,3,1,13]
+ CRUSH rule 0 x 932 [13,2,5,11,9,1,6,15]
+ CRUSH rule 0 x 933 [12,7,14,10,4,1,2,9]
+ CRUSH rule 0 x 934 [12,2,5,7,9,1,15,11]
+ CRUSH rule 0 x 935 [6,11,1,14,5,13,3,9]
+ CRUSH rule 0 x 936 [9,12,7,5,1,2,14,11]
+ CRUSH rule 0 x 937 [14,2,11,1,13,4,9,6]
+ CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+ CRUSH rule 0 x 939 [6,4,14,9,12,1,11,2]
+ CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+ CRUSH rule 0 x 941 [3,12,4,7,14,10]
+ CRUSH rule 0 x 942 [15,12,10,4,1,9,3,7]
+ CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+ CRUSH rule 0 x 944 [2,9,4,7,1,14,12,11]
+ CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+ CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+ CRUSH rule 0 x 947 [11,3,14,1,12,5,6,9]
+ CRUSH rule 0 x 948 [7,13,11,5,14,2,1,9]
+ CRUSH rule 0 x 949 [9,1,12,5,15,10,2,6]
+ CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+ CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+ CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+ CRUSH rule 0 x 953 [1,3,6,10,12,14,4,9]
+ CRUSH rule 0 x 954 [10,2,14,9,4,6,12,1]
+ CRUSH rule 0 x 955 [7,14,3,1,10,4,9,12]
+ CRUSH rule 0 x 956 [1,6,11,5,14,3,9,13]
+ CRUSH rule 0 x 957 [14,11,1,12,6,9,4,3]
+ CRUSH rule 0 x 958 [15,4,3,11,1,6,12,9]
+ CRUSH rule 0 x 959 [2,1,12,15,10,9,4,6]
+ CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+ CRUSH rule 0 x 961 [3,13,11,9,6,1,4,15]
+ CRUSH rule 0 x 962 [5,11,3,14,1,6,13,9]
+ CRUSH rule 0 x 963 [13,10,15,4,6,9,1,3]
+ CRUSH rule 0 x 964 [7,11,4,9,2,12,1,15]
+ CRUSH rule 0 x 965 [12,2,9,7,4,15,11,1]
+ CRUSH rule 0 x 966 [12,14,9,4,1,2,11,7]
+ CRUSH rule 0 x 967 [7,5,3,10,12,14]
+ CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+ CRUSH rule 0 x 969 [11,4,7,1,9,14,13,2]
+ CRUSH rule 0 x 970 [5,12,10,1,3,14,9,6]
+ CRUSH rule 0 x 971 [1,9,4,12,7,2,10,15]
+ CRUSH rule 0 x 972 [12,3,14,5,1,9,7,11]
+ CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+ CRUSH rule 0 x 974 [7,11,1,2,15,4,12,9]
+ CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+ CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+ CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+ CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+ CRUSH rule 0 x 979 [5,1,13,6,15,10,3,9]
+ CRUSH rule 0 x 980 [15,11,5,6,1,3,13,9]
+ CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+ CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+ CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+ CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+ CRUSH rule 0 x 985 [11,2,15,1,4,13,6,9]
+ CRUSH rule 0 x 986 [6,13,9,1,15,10,5,2]
+ CRUSH rule 0 x 987 [13,14,5,10,6,1,3,9]
+ CRUSH rule 0 x 988 [12,9,10,14,3,1,4,7]
+ CRUSH rule 0 x 989 [7,4,3,15,9,13,10,1]
+ CRUSH rule 0 x 990 [1,10,9,13,3,4,6,15]
+ CRUSH rule 0 x 991 [7,11,1,14,2,5,9,12]
+ CRUSH rule 0 x 992 [9,10,2,13,7,4,1,15]
+ CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+ CRUSH rule 0 x 994 [3,13,15,4,11,7,1,9]
+ CRUSH rule 0 x 995 [15,6,12,2,5,11]
+ CRUSH rule 0 x 996 [15,10,5,3,13,1,9,7]
+ CRUSH rule 0 x 997 [15,2,1,12,7,9,4,10]
+ CRUSH rule 0 x 998 [6,1,9,5,12,11,15,2]
+ CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+ CRUSH rule 0 x 1000 [14,2,9,4,12,1,6,11]
+ CRUSH rule 0 x 1001 [11,14,4,2,6,9,1,13]
+ CRUSH rule 0 x 1002 [1,10,14,2,9,5,13,7]
+ CRUSH rule 0 x 1003 [10,7,5,14,2,1,9,12]
+ CRUSH rule 0 x 1004 [15,1,4,6,10,12,9,3]
+ CRUSH rule 0 x 1005 [6,12,2,10,9,15,5,1]
+ CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+ CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+ CRUSH rule 0 x 1008 [7,4,9,11,3,15,1,13]
+ CRUSH rule 0 x 1009 [5,2,11,7,15,9,1,12]
+ CRUSH rule 0 x 1010 [10,2,15,6,9,13,4,1]
+ CRUSH rule 0 x 1011 [6,3,12,1,10,4,9,14]
+ CRUSH rule 0 x 1012 [12,6,9,15,3,1,5,11]
+ CRUSH rule 0 x 1013 [2,14,12,4,9,1,6,10]
+ CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+ CRUSH rule 0 x 1015 [12,6,10,1,4,15,9,2]
+ CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+ CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+ CRUSH rule 0 x 1018 [13,11,14,1,9,3,5,7]
+ CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+ CRUSH rule 0 x 1020 [3,1,13,4,10,9,14,6]
+ CRUSH rule 0 x 1021 [2,11,14,9,4,6,1,13]
+ CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+ CRUSH rule 0 x 1023 [15,2,9,12,1,7,4,11]
+ rule 0 (replicated_ruleset) num_rep 10 result size == 6:\t36/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 10 result size == 7:\t263/1024 (esc)
+ rule 0 (replicated_ruleset) num_rep 10 result size == 8:\t725/1024 (esc)
diff --git a/src/test/cli/osdmaptool/clobber.t b/src/test/cli/osdmaptool/clobber.t
index bbec5f0..b9986d3 100644
--- a/src/test/cli/osdmaptool/clobber.t
+++ b/src/test/cli/osdmaptool/clobber.t
@@ -20,9 +20,9 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' rep size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 crash_replay_interval 45
- pool 1 'metadata' rep size 2 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0
- pool 2 'rbd' rep size 2 min_size 1 crush_ruleset 2 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
max_osd 3
@@ -43,9 +43,9 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' rep size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 crash_replay_interval 45
- pool 1 'metadata' rep size 2 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0
- pool 2 'rbd' rep size 2 min_size 1 crush_ruleset 2 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 owner 0 flags hashpspool
max_osd 1
diff --git a/src/test/cli/osdmaptool/create-print.t b/src/test/cli/osdmaptool/create-print.t
index 81b9194..c8e405e 100644
--- a/src/test/cli/osdmaptool/create-print.t
+++ b/src/test/cli/osdmaptool/create-print.t
@@ -2,6 +2,71 @@
osdmaptool: osdmap file 'myosdmap'
osdmaptool: writing epoch 1 to myosdmap
+ $ osdmaptool --export-crush oc myosdmap
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: exported crush map to oc
+ $ crushtool --decompile oc
+ # begin crush map
+ tunable choose_local_tries 0
+ tunable choose_local_fallback_tries 0
+ tunable choose_total_tries 50
+ tunable chooseleaf_descend_once 1
+
+ # devices
+ device 0 osd.0
+ device 1 osd.1
+ device 2 osd.2
+
+ # types
+ type 0 osd
+ type 1 host
+ type 2 chassis
+ type 3 rack
+ type 4 row
+ type 5 pdu
+ type 6 pod
+ type 7 room
+ type 8 datacenter
+ type 9 region
+ type 10 root
+
+ # buckets
+ host localhost {
+ \tid -2\t\t# do not change unnecessarily (esc)
+ \t# weight 3.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.0 weight 1.000 (esc)
+ \titem osd.1 weight 1.000 (esc)
+ \titem osd.2 weight 1.000 (esc)
+ }
+ rack localrack {
+ \tid -3\t\t# do not change unnecessarily (esc)
+ \t# weight 3.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem localhost weight 3.000 (esc)
+ }
+ root default {
+ \tid -1\t\t# do not change unnecessarily (esc)
+ \t# weight 3.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem localrack weight 3.000 (esc)
+ }
+
+ # rules
+ rule replicated_ruleset {
+ \truleset 0 (esc)
+ \ttype replicated (esc)
+ \tmin_size 1 (esc)
+ \tmax_size 10 (esc)
+ \tstep take default (esc)
+ \tstep chooseleaf firstn 0 type host (esc)
+ \tstep emit (esc)
+ }
+
+ # end crush map
$ osdmaptool --print myosdmap
osdmaptool: osdmap file 'myosdmap'
epoch 1
@@ -10,9 +75,38 @@
modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
flags
- pool 0 'data' rep size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 crash_replay_interval 45
- pool 1 'metadata' rep size 2 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0
- pool 2 'rbd' rep size 2 min_size 1 crush_ruleset 2 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool
max_osd 3
+ $ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_replicated_ruleset 66 myosdmap
+ osdmaptool: osdmap file 'myosdmap'
+ osdmaptool: writing epoch 1 to myosdmap
+ $ osdmaptool --print myosdmap | grep 'pool 0'
+ osdmaptool: osdmap file 'myosdmap'
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 66 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ $ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_rule 55 myosdmap 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+ osdmaptool: osdmap file 'myosdmap'
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+ $ osdmaptool --print myosdmap | grep 'pool 0'
+ osdmaptool: osdmap file 'myosdmap'
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ $ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_replicated_ruleset 66 --osd_pool_default_crush_rule 55 myosdmap 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+ osdmaptool: osdmap file 'myosdmap'
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+ $ osdmaptool --print myosdmap | grep 'pool 0'
+ osdmaptool: osdmap file 'myosdmap'
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ $ rm -f myosdmap
diff --git a/src/test/cli/osdmaptool/create-racks.t b/src/test/cli/osdmaptool/create-racks.t
index f686ef4..4d9b65a 100644
--- a/src/test/cli/osdmaptool/create-racks.t
+++ b/src/test/cli/osdmaptool/create-racks.t
@@ -1,6 +1,826 @@
- $ osdmaptool --create-from-conf om -c $TESTDIR/ceph.conf.withracks > /dev/null
+ $ osdmaptool --create-from-conf om -c $TESTDIR/ceph.conf.withracks
osdmaptool: osdmap file 'om'
+ osdmaptool: writing epoch 1 to om
+ $ osdmaptool --export-crush oc om
+ osdmaptool: osdmap file 'om'
+ osdmaptool: exported crush map to oc
+ $ crushtool --decompile oc
+ # begin crush map
+ tunable choose_local_tries 0
+ tunable choose_local_fallback_tries 0
+ tunable choose_total_tries 50
+ tunable chooseleaf_descend_once 1
+
+ # devices
+ device 0 device0
+ device 1 osd.1
+ device 2 osd.2
+ device 3 osd.3
+ device 4 osd.4
+ device 5 osd.5
+ device 6 osd.6
+ device 7 osd.7
+ device 8 osd.8
+ device 9 osd.9
+ device 10 osd.10
+ device 11 osd.11
+ device 12 osd.12
+ device 13 osd.13
+ device 14 osd.14
+ device 15 osd.15
+ device 16 osd.16
+ device 17 osd.17
+ device 18 osd.18
+ device 19 osd.19
+ device 20 osd.20
+ device 21 osd.21
+ device 22 osd.22
+ device 23 osd.23
+ device 24 osd.24
+ device 25 osd.25
+ device 26 osd.26
+ device 27 osd.27
+ device 28 osd.28
+ device 29 osd.29
+ device 30 osd.30
+ device 31 osd.31
+ device 32 osd.32
+ device 33 osd.33
+ device 34 osd.34
+ device 35 osd.35
+ device 36 osd.36
+ device 37 osd.37
+ device 38 osd.38
+ device 39 osd.39
+ device 40 osd.40
+ device 41 osd.41
+ device 42 osd.42
+ device 43 osd.43
+ device 44 osd.44
+ device 45 osd.45
+ device 46 osd.46
+ device 47 osd.47
+ device 48 osd.48
+ device 49 osd.49
+ device 50 osd.50
+ device 51 osd.51
+ device 52 osd.52
+ device 53 osd.53
+ device 54 osd.54
+ device 55 osd.55
+ device 56 osd.56
+ device 57 osd.57
+ device 58 osd.58
+ device 59 osd.59
+ device 60 osd.60
+ device 61 osd.61
+ device 62 osd.62
+ device 63 osd.63
+ device 64 osd.64
+ device 65 osd.65
+ device 66 osd.66
+ device 67 osd.67
+ device 68 osd.68
+ device 69 osd.69
+ device 70 osd.70
+ device 71 osd.71
+ device 72 osd.72
+ device 73 osd.73
+ device 74 osd.74
+ device 75 osd.75
+ device 76 osd.76
+ device 77 osd.77
+ device 78 osd.78
+ device 79 osd.79
+ device 80 osd.80
+ device 81 osd.81
+ device 82 osd.82
+ device 83 osd.83
+ device 84 osd.84
+ device 85 osd.85
+ device 86 osd.86
+ device 87 osd.87
+ device 88 osd.88
+ device 89 osd.89
+ device 90 osd.90
+ device 91 osd.91
+ device 92 osd.92
+ device 93 osd.93
+ device 94 osd.94
+ device 95 osd.95
+ device 96 osd.96
+ device 97 osd.97
+ device 98 osd.98
+ device 99 osd.99
+ device 100 osd.100
+ device 101 osd.101
+ device 102 osd.102
+ device 103 osd.103
+ device 104 osd.104
+ device 105 osd.105
+ device 106 osd.106
+ device 107 osd.107
+ device 108 osd.108
+ device 109 osd.109
+ device 110 osd.110
+ device 111 osd.111
+ device 112 osd.112
+ device 113 osd.113
+ device 114 osd.114
+ device 115 osd.115
+ device 116 osd.116
+ device 117 osd.117
+ device 118 osd.118
+ device 119 osd.119
+ device 120 osd.120
+ device 121 osd.121
+ device 122 osd.122
+ device 123 osd.123
+ device 124 osd.124
+ device 125 osd.125
+ device 126 osd.126
+ device 127 osd.127
+ device 128 osd.128
+ device 129 osd.129
+ device 130 osd.130
+ device 131 osd.131
+ device 132 osd.132
+ device 133 osd.133
+ device 134 osd.134
+ device 135 osd.135
+ device 136 osd.136
+ device 137 osd.137
+ device 138 osd.138
+ device 139 osd.139
+ device 140 osd.140
+ device 141 osd.141
+ device 142 osd.142
+ device 143 osd.143
+ device 144 osd.144
+ device 145 osd.145
+ device 146 osd.146
+ device 147 osd.147
+ device 148 osd.148
+ device 149 osd.149
+ device 150 osd.150
+ device 151 osd.151
+ device 152 osd.152
+ device 153 osd.153
+ device 154 osd.154
+ device 155 osd.155
+ device 156 osd.156
+ device 157 osd.157
+ device 158 osd.158
+ device 159 osd.159
+ device 160 osd.160
+ device 161 osd.161
+ device 162 osd.162
+ device 163 osd.163
+ device 164 osd.164
+ device 165 osd.165
+ device 166 osd.166
+ device 167 osd.167
+ device 168 osd.168
+ device 169 osd.169
+ device 170 osd.170
+ device 171 osd.171
+ device 172 osd.172
+ device 173 osd.173
+ device 174 osd.174
+ device 175 osd.175
+ device 176 osd.176
+ device 177 osd.177
+ device 178 osd.178
+ device 179 osd.179
+ device 180 osd.180
+ device 181 osd.181
+ device 182 osd.182
+ device 183 osd.183
+ device 184 osd.184
+ device 185 osd.185
+ device 186 osd.186
+ device 187 osd.187
+ device 188 osd.188
+ device 189 osd.189
+ device 190 osd.190
+ device 191 osd.191
+ device 192 osd.192
+ device 193 osd.193
+ device 194 osd.194
+ device 195 osd.195
+ device 196 osd.196
+ device 197 osd.197
+ device 198 osd.198
+ device 199 osd.199
+ device 200 osd.200
+ device 201 osd.201
+ device 202 osd.202
+ device 203 osd.203
+ device 204 osd.204
+ device 205 osd.205
+ device 206 osd.206
+ device 207 osd.207
+ device 208 osd.208
+ device 209 osd.209
+ device 210 osd.210
+ device 211 osd.211
+ device 212 osd.212
+ device 213 osd.213
+ device 214 osd.214
+ device 215 osd.215
+ device 216 osd.216
+ device 217 osd.217
+ device 218 osd.218
+ device 219 osd.219
+ device 220 osd.220
+ device 221 osd.221
+ device 222 osd.222
+ device 223 osd.223
+ device 224 osd.224
+ device 225 osd.225
+ device 226 osd.226
+ device 227 osd.227
+ device 228 osd.228
+ device 229 osd.229
+ device 230 osd.230
+ device 231 osd.231
+ device 232 osd.232
+ device 233 osd.233
+ device 234 osd.234
+ device 235 osd.235
+ device 236 osd.236
+ device 237 osd.237
+ device 238 osd.238
+
+ # types
+ type 0 osd
+ type 1 host
+ type 2 chassis
+ type 3 rack
+ type 4 row
+ type 5 pdu
+ type 6 pod
+ type 7 room
+ type 8 datacenter
+ type 9 region
+ type 10 root
+
+ # buckets
+ host cephstore5522 {
+ \tid -2\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.1 weight 1.000 (esc)
+ \titem osd.2 weight 1.000 (esc)
+ \titem osd.3 weight 1.000 (esc)
+ \titem osd.4 weight 1.000 (esc)
+ \titem osd.5 weight 1.000 (esc)
+ \titem osd.6 weight 1.000 (esc)
+ \titem osd.7 weight 1.000 (esc)
+ }
+ host cephstore5523 {
+ \tid -4\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.10 weight 1.000 (esc)
+ \titem osd.11 weight 1.000 (esc)
+ \titem osd.12 weight 1.000 (esc)
+ \titem osd.13 weight 1.000 (esc)
+ \titem osd.14 weight 1.000 (esc)
+ \titem osd.8 weight 1.000 (esc)
+ \titem osd.9 weight 1.000 (esc)
+ }
+ host cephstore6238 {
+ \tid -8\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.113 weight 1.000 (esc)
+ \titem osd.114 weight 1.000 (esc)
+ \titem osd.115 weight 1.000 (esc)
+ \titem osd.116 weight 1.000 (esc)
+ \titem osd.117 weight 1.000 (esc)
+ \titem osd.118 weight 1.000 (esc)
+ \titem osd.119 weight 1.000 (esc)
+ }
+ host cephstore6240 {
+ \tid -10\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.127 weight 1.000 (esc)
+ \titem osd.128 weight 1.000 (esc)
+ \titem osd.129 weight 1.000 (esc)
+ \titem osd.130 weight 1.000 (esc)
+ \titem osd.131 weight 1.000 (esc)
+ \titem osd.132 weight 1.000 (esc)
+ \titem osd.133 weight 1.000 (esc)
+ }
+ host cephstore6242 {
+ \tid -12\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.141 weight 1.000 (esc)
+ \titem osd.142 weight 1.000 (esc)
+ \titem osd.143 weight 1.000 (esc)
+ \titem osd.144 weight 1.000 (esc)
+ \titem osd.145 weight 1.000 (esc)
+ \titem osd.146 weight 1.000 (esc)
+ \titem osd.147 weight 1.000 (esc)
+ }
+ host cephstore5524 {
+ \tid -14\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.15 weight 1.000 (esc)
+ \titem osd.16 weight 1.000 (esc)
+ \titem osd.17 weight 1.000 (esc)
+ \titem osd.18 weight 1.000 (esc)
+ \titem osd.19 weight 1.000 (esc)
+ \titem osd.20 weight 1.000 (esc)
+ \titem osd.21 weight 1.000 (esc)
+ }
+ host cephstore6244 {
+ \tid -15\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.155 weight 1.000 (esc)
+ \titem osd.156 weight 1.000 (esc)
+ \titem osd.157 weight 1.000 (esc)
+ \titem osd.158 weight 1.000 (esc)
+ \titem osd.159 weight 1.000 (esc)
+ \titem osd.160 weight 1.000 (esc)
+ \titem osd.161 weight 1.000 (esc)
+ }
+ host cephstore6246 {
+ \tid -17\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.169 weight 1.000 (esc)
+ \titem osd.170 weight 1.000 (esc)
+ \titem osd.171 weight 1.000 (esc)
+ \titem osd.172 weight 1.000 (esc)
+ \titem osd.173 weight 1.000 (esc)
+ \titem osd.174 weight 1.000 (esc)
+ \titem osd.175 weight 1.000 (esc)
+ }
+ host cephstore6337 {
+ \tid -19\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.183 weight 1.000 (esc)
+ \titem osd.184 weight 1.000 (esc)
+ \titem osd.185 weight 1.000 (esc)
+ \titem osd.186 weight 1.000 (esc)
+ \titem osd.187 weight 1.000 (esc)
+ \titem osd.188 weight 1.000 (esc)
+ \titem osd.189 weight 1.000 (esc)
+ }
+ host cephstore6341 {
+ \tid -23\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.211 weight 1.000 (esc)
+ \titem osd.212 weight 1.000 (esc)
+ \titem osd.213 weight 1.000 (esc)
+ \titem osd.214 weight 1.000 (esc)
+ \titem osd.215 weight 1.000 (esc)
+ \titem osd.216 weight 1.000 (esc)
+ \titem osd.217 weight 1.000 (esc)
+ }
+ host cephstore6342 {
+ \tid -24\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.218 weight 1.000 (esc)
+ \titem osd.219 weight 1.000 (esc)
+ \titem osd.220 weight 1.000 (esc)
+ \titem osd.221 weight 1.000 (esc)
+ \titem osd.222 weight 1.000 (esc)
+ \titem osd.223 weight 1.000 (esc)
+ \titem osd.224 weight 1.000 (esc)
+ }
+ host cephstore5525 {
+ \tid -25\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.22 weight 1.000 (esc)
+ \titem osd.23 weight 1.000 (esc)
+ \titem osd.24 weight 1.000 (esc)
+ \titem osd.25 weight 1.000 (esc)
+ \titem osd.26 weight 1.000 (esc)
+ \titem osd.27 weight 1.000 (esc)
+ \titem osd.28 weight 1.000 (esc)
+ }
+ host cephstore6345 {
+ \tid -27\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.232 weight 1.000 (esc)
+ \titem osd.233 weight 1.000 (esc)
+ \titem osd.234 weight 1.000 (esc)
+ \titem osd.235 weight 1.000 (esc)
+ \titem osd.236 weight 1.000 (esc)
+ \titem osd.237 weight 1.000 (esc)
+ \titem osd.238 weight 1.000 (esc)
+ }
+ host cephstore5526 {
+ \tid -28\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.29 weight 1.000 (esc)
+ \titem osd.30 weight 1.000 (esc)
+ \titem osd.31 weight 1.000 (esc)
+ \titem osd.32 weight 1.000 (esc)
+ \titem osd.33 weight 1.000 (esc)
+ \titem osd.34 weight 1.000 (esc)
+ \titem osd.35 weight 1.000 (esc)
+ }
+ host cephstore5527 {
+ \tid -29\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.36 weight 1.000 (esc)
+ \titem osd.37 weight 1.000 (esc)
+ \titem osd.38 weight 1.000 (esc)
+ \titem osd.39 weight 1.000 (esc)
+ \titem osd.40 weight 1.000 (esc)
+ \titem osd.41 weight 1.000 (esc)
+ \titem osd.42 weight 1.000 (esc)
+ }
+ host cephstore5529 {
+ \tid -30\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.43 weight 1.000 (esc)
+ \titem osd.44 weight 1.000 (esc)
+ \titem osd.45 weight 1.000 (esc)
+ \titem osd.46 weight 1.000 (esc)
+ \titem osd.47 weight 1.000 (esc)
+ \titem osd.48 weight 1.000 (esc)
+ \titem osd.49 weight 1.000 (esc)
+ }
+ host cephstore5530 {
+ \tid -31\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.50 weight 1.000 (esc)
+ \titem osd.51 weight 1.000 (esc)
+ \titem osd.52 weight 1.000 (esc)
+ \titem osd.53 weight 1.000 (esc)
+ \titem osd.54 weight 1.000 (esc)
+ \titem osd.55 weight 1.000 (esc)
+ \titem osd.56 weight 1.000 (esc)
+ }
+ rack irv-n2 {
+ \tid -3\t\t# do not change unnecessarily (esc)
+ \t# weight 119.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem cephstore5522 weight 7.000 (esc)
+ \titem cephstore5523 weight 7.000 (esc)
+ \titem cephstore6238 weight 7.000 (esc)
+ \titem cephstore6240 weight 7.000 (esc)
+ \titem cephstore6242 weight 7.000 (esc)
+ \titem cephstore5524 weight 7.000 (esc)
+ \titem cephstore6244 weight 7.000 (esc)
+ \titem cephstore6246 weight 7.000 (esc)
+ \titem cephstore6337 weight 7.000 (esc)
+ \titem cephstore6341 weight 7.000 (esc)
+ \titem cephstore6342 weight 7.000 (esc)
+ \titem cephstore5525 weight 7.000 (esc)
+ \titem cephstore6345 weight 7.000 (esc)
+ \titem cephstore5526 weight 7.000 (esc)
+ \titem cephstore5527 weight 7.000 (esc)
+ \titem cephstore5529 weight 7.000 (esc)
+ \titem cephstore5530 weight 7.000 (esc)
+ }
+ host cephstore6236 {
+ \tid -5\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.100 weight 1.000 (esc)
+ \titem osd.101 weight 1.000 (esc)
+ \titem osd.102 weight 1.000 (esc)
+ \titem osd.103 weight 1.000 (esc)
+ \titem osd.104 weight 1.000 (esc)
+ \titem osd.105 weight 1.000 (esc)
+ \titem osd.99 weight 1.000 (esc)
+ }
+ host cephstore6237 {
+ \tid -7\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.106 weight 1.000 (esc)
+ \titem osd.107 weight 1.000 (esc)
+ \titem osd.108 weight 1.000 (esc)
+ \titem osd.109 weight 1.000 (esc)
+ \titem osd.110 weight 1.000 (esc)
+ \titem osd.111 weight 1.000 (esc)
+ \titem osd.112 weight 1.000 (esc)
+ }
+ host cephstore6239 {
+ \tid -9\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.120 weight 1.000 (esc)
+ \titem osd.121 weight 1.000 (esc)
+ \titem osd.122 weight 1.000 (esc)
+ \titem osd.123 weight 1.000 (esc)
+ \titem osd.124 weight 1.000 (esc)
+ \titem osd.125 weight 1.000 (esc)
+ \titem osd.126 weight 1.000 (esc)
+ }
+ host cephstore6241 {
+ \tid -11\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.134 weight 1.000 (esc)
+ \titem osd.135 weight 1.000 (esc)
+ \titem osd.136 weight 1.000 (esc)
+ \titem osd.137 weight 1.000 (esc)
+ \titem osd.138 weight 1.000 (esc)
+ \titem osd.139 weight 1.000 (esc)
+ \titem osd.140 weight 1.000 (esc)
+ }
+ host cephstore6243 {
+ \tid -13\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.148 weight 1.000 (esc)
+ \titem osd.149 weight 1.000 (esc)
+ \titem osd.150 weight 1.000 (esc)
+ \titem osd.151 weight 1.000 (esc)
+ \titem osd.152 weight 1.000 (esc)
+ \titem osd.153 weight 1.000 (esc)
+ \titem osd.154 weight 1.000 (esc)
+ }
+ host cephstore6245 {
+ \tid -16\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.162 weight 1.000 (esc)
+ \titem osd.163 weight 1.000 (esc)
+ \titem osd.164 weight 1.000 (esc)
+ \titem osd.165 weight 1.000 (esc)
+ \titem osd.166 weight 1.000 (esc)
+ \titem osd.167 weight 1.000 (esc)
+ \titem osd.168 weight 1.000 (esc)
+ }
+ host cephstore6336 {
+ \tid -18\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.176 weight 1.000 (esc)
+ \titem osd.177 weight 1.000 (esc)
+ \titem osd.178 weight 1.000 (esc)
+ \titem osd.179 weight 1.000 (esc)
+ \titem osd.180 weight 1.000 (esc)
+ \titem osd.181 weight 1.000 (esc)
+ \titem osd.182 weight 1.000 (esc)
+ }
+ host cephstore6338 {
+ \tid -20\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.190 weight 1.000 (esc)
+ \titem osd.191 weight 1.000 (esc)
+ \titem osd.192 weight 1.000 (esc)
+ \titem osd.193 weight 1.000 (esc)
+ \titem osd.194 weight 1.000 (esc)
+ \titem osd.195 weight 1.000 (esc)
+ \titem osd.196 weight 1.000 (esc)
+ }
+ host cephstore6339 {
+ \tid -21\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.197 weight 1.000 (esc)
+ \titem osd.198 weight 1.000 (esc)
+ \titem osd.199 weight 1.000 (esc)
+ \titem osd.200 weight 1.000 (esc)
+ \titem osd.201 weight 1.000 (esc)
+ \titem osd.202 weight 1.000 (esc)
+ \titem osd.203 weight 1.000 (esc)
+ }
+ host cephstore6340 {
+ \tid -22\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.204 weight 1.000 (esc)
+ \titem osd.205 weight 1.000 (esc)
+ \titem osd.206 weight 1.000 (esc)
+ \titem osd.207 weight 1.000 (esc)
+ \titem osd.208 weight 1.000 (esc)
+ \titem osd.209 weight 1.000 (esc)
+ \titem osd.210 weight 1.000 (esc)
+ }
+ host cephstore6343 {
+ \tid -26\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.225 weight 1.000 (esc)
+ \titem osd.226 weight 1.000 (esc)
+ \titem osd.227 weight 1.000 (esc)
+ \titem osd.228 weight 1.000 (esc)
+ \titem osd.229 weight 1.000 (esc)
+ \titem osd.230 weight 1.000 (esc)
+ \titem osd.231 weight 1.000 (esc)
+ }
+ host cephstore6230 {
+ \tid -32\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.57 weight 1.000 (esc)
+ \titem osd.58 weight 1.000 (esc)
+ \titem osd.59 weight 1.000 (esc)
+ \titem osd.60 weight 1.000 (esc)
+ \titem osd.61 weight 1.000 (esc)
+ \titem osd.62 weight 1.000 (esc)
+ \titem osd.63 weight 1.000 (esc)
+ }
+ host cephstore6231 {
+ \tid -33\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.64 weight 1.000 (esc)
+ \titem osd.65 weight 1.000 (esc)
+ \titem osd.66 weight 1.000 (esc)
+ \titem osd.67 weight 1.000 (esc)
+ \titem osd.68 weight 1.000 (esc)
+ \titem osd.69 weight 1.000 (esc)
+ \titem osd.70 weight 1.000 (esc)
+ }
+ host cephstore6232 {
+ \tid -34\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.71 weight 1.000 (esc)
+ \titem osd.72 weight 1.000 (esc)
+ \titem osd.73 weight 1.000 (esc)
+ \titem osd.74 weight 1.000 (esc)
+ \titem osd.75 weight 1.000 (esc)
+ \titem osd.76 weight 1.000 (esc)
+ \titem osd.77 weight 1.000 (esc)
+ }
+ host cephstore6233 {
+ \tid -35\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.78 weight 1.000 (esc)
+ \titem osd.79 weight 1.000 (esc)
+ \titem osd.80 weight 1.000 (esc)
+ \titem osd.81 weight 1.000 (esc)
+ \titem osd.82 weight 1.000 (esc)
+ \titem osd.83 weight 1.000 (esc)
+ \titem osd.84 weight 1.000 (esc)
+ }
+ host cephstore6234 {
+ \tid -36\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.85 weight 1.000 (esc)
+ \titem osd.86 weight 1.000 (esc)
+ \titem osd.87 weight 1.000 (esc)
+ \titem osd.88 weight 1.000 (esc)
+ \titem osd.89 weight 1.000 (esc)
+ \titem osd.90 weight 1.000 (esc)
+ \titem osd.91 weight 1.000 (esc)
+ }
+ host cephstore6235 {
+ \tid -37\t\t# do not change unnecessarily (esc)
+ \t# weight 7.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem osd.92 weight 1.000 (esc)
+ \titem osd.93 weight 1.000 (esc)
+ \titem osd.94 weight 1.000 (esc)
+ \titem osd.95 weight 1.000 (esc)
+ \titem osd.96 weight 1.000 (esc)
+ \titem osd.97 weight 1.000 (esc)
+ \titem osd.98 weight 1.000 (esc)
+ }
+ rack irv-n1 {
+ \tid -6\t\t# do not change unnecessarily (esc)
+ \t# weight 119.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem cephstore6236 weight 7.000 (esc)
+ \titem cephstore6237 weight 7.000 (esc)
+ \titem cephstore6239 weight 7.000 (esc)
+ \titem cephstore6241 weight 7.000 (esc)
+ \titem cephstore6243 weight 7.000 (esc)
+ \titem cephstore6245 weight 7.000 (esc)
+ \titem cephstore6336 weight 7.000 (esc)
+ \titem cephstore6338 weight 7.000 (esc)
+ \titem cephstore6339 weight 7.000 (esc)
+ \titem cephstore6340 weight 7.000 (esc)
+ \titem cephstore6343 weight 7.000 (esc)
+ \titem cephstore6230 weight 7.000 (esc)
+ \titem cephstore6231 weight 7.000 (esc)
+ \titem cephstore6232 weight 7.000 (esc)
+ \titem cephstore6233 weight 7.000 (esc)
+ \titem cephstore6234 weight 7.000 (esc)
+ \titem cephstore6235 weight 7.000 (esc)
+ }
+ root default {
+ \tid -1\t\t# do not change unnecessarily (esc)
+ \t# weight 238.000 (esc)
+ \talg straw (esc)
+ \thash 0\t# rjenkins1 (esc)
+ \titem irv-n2 weight 119.000 (esc)
+ \titem irv-n1 weight 119.000 (esc)
+ }
+
+ # rules
+ rule replicated_ruleset {
+ \truleset 0 (esc)
+ \ttype replicated (esc)
+ \tmin_size 1 (esc)
+ \tmax_size 10 (esc)
+ \tstep take default (esc)
+ \tstep chooseleaf firstn 0 type host (esc)
+ \tstep emit (esc)
+ }
+
+ # end crush map
+ $ rm oc
$ osdmaptool --test-map-pg 0.0 om
osdmaptool: osdmap file 'om'
parsed '0.0' -> 0.0
- 0.0 raw [] up [] acting []
+ 0.0 raw ([], p-1) up ([], p-1) acting ([], p-1)
+ $ osdmaptool --print om
+ osdmaptool: osdmap file 'om'
+ epoch 1
+ fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+ created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+ modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+ flags
+
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool
+ pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool
+
+ max_osd 239
+
+
+ $ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_replicated_ruleset 55 om -c $TESTDIR/ceph.conf.withracks
+ osdmaptool: osdmap file 'om'
+ osdmaptool: writing epoch 1 to om
+ $ osdmaptool --print om | grep 'pool 0'
+ osdmaptool: osdmap file 'om'
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ $ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_rule 55 om -c $TESTDIR/ceph.conf.withracks 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+ osdmaptool: osdmap file 'om'
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+ $ osdmaptool --print om | grep 'pool 0'
+ osdmaptool: osdmap file 'om'
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ $ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_replicated_ruleset 66 --osd_pool_default_crush_rule 55 om -c $TESTDIR/ceph.conf.withracks 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+ osdmaptool: osdmap file 'om'
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+ default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+ default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+ $ osdmaptool --print om | grep 'pool 0'
+ osdmaptool: osdmap file 'om'
+ pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 owner 0 flags hashpspool crash_replay_interval 45
+ $ rm -f om
diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 0994e42..a7921e2 100644
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -8,6 +8,7 @@
user suspend suspend a user
user enable reenable user after suspension
user check check user info
+ user stats show user stats as accounted by quota subsystem
caps add add user capabilities
caps rm remove user capabilities
subuser create create a new subuser
@@ -75,6 +76,7 @@
--gen-access-key generate random access key (for S3)
--gen-secret generate random secret key
--key-type=<type> key type, options are: swift, s3
+ --temp-url-key[-2]=<key> temp url key
--access=<access> Set access permissions for sub-user, should be one
of read, write, readwrite, full
--display-name=<name>
@@ -105,6 +107,8 @@
subuser keys
--purge-objects remove a bucket's objects before deleting it
(NOTE: required to delete a non-empty bucket)
+ --sync-stats option to 'user stats', update user stats with current
+ stats reported by user's buckets indexes
--show-log-entries=<flag> enable/disable dump of log entries on log show
--show-log-sum=<flag> enable/disable dump of log summation on log show
--skip-zero-entries log show only dumps entries that don't have zero value
diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
index 754e11f..5723bef 100644
--- a/src/test/cli/rbd/help.t
+++ b/src/test/cli/rbd/help.t
@@ -76,5 +76,6 @@
--pretty-format make json or xml output more readable
--no-settle do not wait for udevadm to settle on map/unmap
--no-progress do not show progress for long-running commands
+ -o, --options <map-options> options to use when mapping an image
--read-only set device readonly when mapping image
--allow-shrink allow shrinking of an image when resizing
diff --git a/src/test/cls_rbd/test_cls_rbd.cc b/src/test/cls_rbd/test_cls_rbd.cc
index c7ea79d..efa44fc 100644
--- a/src/test/cls_rbd/test_cls_rbd.cc
+++ b/src/test/cls_rbd/test_cls_rbd.cc
@@ -309,7 +309,7 @@ TEST(cls_rbd, create)
ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
string oid = "testobj";
- uint64_t size = 20 << 30;
+ uint64_t size = 20ULL << 30;
uint64_t features = 0;
uint8_t order = 22;
string object_prefix = "foo";
@@ -476,37 +476,37 @@ TEST(cls_rbd, protection_status)
ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 10, "snap1"));
ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
10, &status));
- ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, status);
+ ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTED, status);
ASSERT_EQ(0, set_protection_status(&ioctx, "foo",
10, RBD_PROTECTION_STATUS_PROTECTED));
ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
10, &status));
- ASSERT_EQ(RBD_PROTECTION_STATUS_PROTECTED, status);
+ ASSERT_EQ(+RBD_PROTECTION_STATUS_PROTECTED, status);
ASSERT_EQ(-EBUSY, snapshot_remove(&ioctx, "foo", 10));
ASSERT_EQ(0, set_protection_status(&ioctx, "foo",
10, RBD_PROTECTION_STATUS_UNPROTECTING));
ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
10, &status));
- ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTING, status);
+ ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTING, status);
ASSERT_EQ(-EBUSY, snapshot_remove(&ioctx, "foo", 10));
ASSERT_EQ(-EINVAL, set_protection_status(&ioctx, "foo",
10, RBD_PROTECTION_STATUS_LAST));
ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
10, &status));
- ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTING, status);
+ ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTING, status);
ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 20, "snap2"));
ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
20, &status));
- ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, status);
+ ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTED, status);
ASSERT_EQ(0, set_protection_status(&ioctx, "foo",
10, RBD_PROTECTION_STATUS_UNPROTECTED));
ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
10, &status));
- ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, status);
+ ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTED, status);
ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", 10));
ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", 20));
diff --git a/src/test/cls_statelog/test_cls_statelog.cc b/src/test/cls_statelog/test_cls_statelog.cc
index a1b4cc3..accab95 100644
--- a/src/test/cls_statelog/test_cls_statelog.cc
+++ b/src/test/cls_statelog/test_cls_statelog.cc
@@ -113,7 +113,7 @@ TEST(cls_rgw, test_statelog_basic)
int id = 0;
string client_id[] = { "client-1", "client-2" };
- int num_ops = 10;
+ const int num_ops = 10;
string op_ids[num_ops];
librados::ObjectWriteOperation *op = new_op();
diff --git a/src/test/common/Throttle.cc b/src/test/common/Throttle.cc
index 60d7dae..0964cbe 100644
--- a/src/test/common/Throttle.cc
+++ b/src/test/common/Throttle.cc
@@ -188,10 +188,8 @@ TEST_F(ThrottleTest, wait) {
//
throttle.wait(throttle_max * 100);
usleep(delay);
- ASSERT_EQ(throttle.get_current(), throttle_max / 2);
-
-
t.join();
+ ASSERT_EQ(throttle.get_current(), throttle_max / 2);
if (!(waited = t.waited)) {
delay *= 2;
@@ -251,6 +249,13 @@ int main(int argc, char **argv) {
return RUN_ALL_TESTS();
}
-// Local Variables:
-// compile-command: "cd ../.. ; make unittest_throttle ; ./unittest_throttle # --gtest_filter=ThrottleTest.destructor --log-to-stderr=true --debug-filestore=20"
-// End:
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make unittest_throttle ;
+ * ./unittest_throttle # --gtest_filter=ThrottleTest.destructor \
+ * --log-to-stderr=true --debug-filestore=20
+ * "
+ * End:
+ */
+
diff --git a/src/test/common/get_command_descriptions.cc b/src/test/common/get_command_descriptions.cc
index aff5575..c35e47b 100644
--- a/src/test/common/get_command_descriptions.cc
+++ b/src/test/common/get_command_descriptions.cc
@@ -24,9 +24,6 @@
#include "mon/Monitor.h"
#include "common/ceph_argparse.h"
#include "global/global_init.h"
-#include "common/debug.h"
-
-#define dout_subsys ceph_subsys_mon
static void usage(ostream &out)
{
@@ -48,10 +45,10 @@ static void json_print(const MonCommand *mon_commands, int size)
{
bufferlist rdata;
Formatter *f = new_formatter("json");
- get_command_descriptions(mon_commands, size, f, &rdata);
+ Monitor::format_command_descriptions(mon_commands, size, f, &rdata);
delete f;
string data(rdata.c_str(), rdata.length());
- dout(0) << data << dendl;
+ cout << data << std::endl;
}
static void all()
diff --git a/src/test/common/test_config.cc b/src/test/common/test_config.cc
new file mode 100644
index 0000000..cffdc76
--- /dev/null
+++ b/src/test/common/test_config.cc
@@ -0,0 +1,187 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Library Public License for more details.
+ *
+ *
+ */
+#include "common/config.h"
+#include "common/errno.h"
+#include "gtest/gtest.h"
+
+#define _STR(x) #x
+#define STRINGIFY(x) _STR(x)
+
+static struct config_option config_optionsp[] = {
+#define OPTION(name, type, def_val) \
+ { STRINGIFY(name), type, offsetof(struct md_config_t, name) },
+#define SUBSYS(name, log, gather)
+#define DEFAULT_SUBSYS(log, gather)
+#include "common/config_opts.h"
+#undef OPTION
+#undef SUBSYS
+#undef DEFAULT_SUBSYS
+};
+
+static const int NUM_CONFIG_OPTIONS = sizeof(config_optionsp) / sizeof(config_option);
+
+class test_md_config_t : public md_config_t, public ::testing::Test {
+public:
+ void test_expand_meta() {
+ Mutex::Locker l(lock);
+ // successfull meta expansion $run_dir and ${run_dir}
+ {
+ ostringstream oss;
+ std::string before = " BEFORE ";
+ std::string after = " AFTER ";
+ std::string val(before + "$run_dir${run_dir}" + after);
+ EXPECT_TRUE(expand_meta(val, &oss));
+ EXPECT_EQ(before + "/var/run/ceph/var/run/ceph" + after, val);
+ EXPECT_EQ("", oss.str());
+ }
+ // no meta expansion if variables are unknown
+ {
+ ostringstream oss;
+ std::string expected = "expect $foo and ${bar} to not expand";
+ std::string val = expected;
+ EXPECT_FALSE(expand_meta(val, &oss));
+ EXPECT_EQ(expected, val);
+ EXPECT_EQ("", oss.str());
+ }
+ // recursive variable expansion
+ {
+ std::string mon_host = "$cluster_network";
+ EXPECT_EQ(0, set_val("mon_host", mon_host.c_str(), false));
+
+ std::string lockdep = "true";
+ EXPECT_EQ(0, set_val("lockdep", lockdep.c_str(), false));
+
+ std::string cluster_network = "$public_network $public_network $lockdep $host";
+ EXPECT_EQ(0, set_val("cluster_network", cluster_network.c_str(), false));
+
+ std::string public_network = "NETWORK";
+ EXPECT_EQ(0, set_val("public_network", public_network.c_str(), false));
+
+ ostringstream oss;
+ std::string val = "$mon_host";
+ EXPECT_TRUE(expand_meta(val, &oss));
+ EXPECT_EQ(public_network + " " +
+ public_network + " " +
+ lockdep + " " +
+ "localhost", val);
+ EXPECT_EQ("", oss.str());
+ }
+ // variable expansion loops are non fatal
+ {
+ std::string mon_host = "$cluster_network";
+ EXPECT_EQ(0, set_val("mon_host", mon_host.c_str(), false));
+
+ std::string cluster_network = "$public_network";
+ EXPECT_EQ(0, set_val("cluster_network", cluster_network.c_str(), false));
+
+ std::string public_network = "$mon_host";
+ EXPECT_EQ(0, set_val("public_network", public_network.c_str(), false));
+
+ ostringstream oss;
+ std::string val = "$mon_host";
+ EXPECT_TRUE(expand_meta(val, &oss));
+ EXPECT_EQ("$cluster_network", val);
+ const char *expected_oss =
+ "variable expansion loop at mon_host=$cluster_network\n"
+ "expansion stack: \n"
+ "public_network=$mon_host\n"
+ "cluster_network=$public_network\n"
+ "mon_host=$cluster_network\n";
+ EXPECT_EQ(expected_oss, oss.str());
+ }
+ }
+
+ void test_expand_all_meta() {
+ Mutex::Locker l(lock);
+ int before_count = 0;
+ for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
+ config_option *opt = config_optionsp + i;
+ if (opt->type == OPT_STR) {
+ std::string *str = (std::string *)opt->conf_ptr(this);
+ if (str->find("$") != string::npos)
+ before_count++;
+ }
+ }
+ // if there are no meta variables in the default configuration,
+ // something must be done to check the expected side effect
+ // of expand_all_meta
+ ASSERT_LT(0, before_count);
+ expand_all_meta();
+ int after_count = 0;
+ for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
+ config_option *opt = config_optionsp + i;
+ if (opt->type == OPT_STR) {
+ std::string *str = (std::string *)opt->conf_ptr(this);
+ if (str->find("$") != string::npos)
+ after_count++;
+ }
+ }
+ ASSERT_EQ(0, after_count);
+ }
+};
+
+TEST_F(test_md_config_t, expand_meta)
+{
+ test_expand_meta();
+}
+
+TEST_F(test_md_config_t, expand_all_meta)
+{
+ test_expand_all_meta();
+}
+
+TEST(md_config_t, set_val)
+{
+ int buf_size = 1024;
+ md_config_t conf;
+ // disable meta variable expansion
+ {
+ char *buf = (char*)malloc(buf_size);
+ std::string expected = "$host";
+ EXPECT_EQ(0, conf.set_val("mon_host", expected.c_str(), false));
+ EXPECT_EQ(0, conf.get_val("mon_host", &buf, buf_size));
+ EXPECT_EQ(expected, buf);
+ free(buf);
+ }
+ // meta variable expansion is enabled by default
+ {
+ char *run_dir = (char*)malloc(buf_size);
+ EXPECT_EQ(0, conf.get_val("run_dir", &run_dir, buf_size));
+ EXPECT_EQ(0, conf.set_val("admin_socket", "$run_dir"));
+ char *admin_socket = (char*)malloc(buf_size);
+ EXPECT_EQ(0, conf.get_val("admin_socket", &admin_socket, buf_size));
+ EXPECT_EQ(std::string(run_dir), std::string(admin_socket));
+ free(run_dir);
+ free(admin_socket);
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make unittest_config &&
+ * valgrind \
+ * --max-stackframe=20000000 --tool=memcheck \
+ * ./unittest_config # --gtest_filter=md_config_t.set_val
+ * "
+ * End:
+ */
diff --git a/src/test/common/test_str_map.cc b/src/test/common/test_str_map.cc
new file mode 100644
index 0000000..f60f0d8
--- /dev/null
+++ b/src/test/common/test_str_map.cc
@@ -0,0 +1,70 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <gtest/gtest.h>
+
+#include "include/str_map.h"
+
+using namespace std;
+
+TEST(str_map, json) {
+ map<string,string> str_map;
+ stringstream ss;
+ // well formatted
+ ASSERT_EQ(0, get_str_map("{\"key\": \"value\"}", ss, &str_map));
+ ASSERT_EQ("value", str_map["key"]);
+ // well formatted but not a JSON object
+ ASSERT_EQ(-EINVAL, get_str_map("\"key\"", ss, &str_map));
+ ASSERT_NE(string::npos, ss.str().find("must be a JSON object"));
+}
+
+TEST(str_map, plaintext) {
+ stringstream ss;
+ {
+ map<string,string> str_map;
+ ASSERT_EQ(0, get_str_map(" foo=bar\t\nfrob=nitz yeah right= \n\t",
+ ss, &str_map));
+ ASSERT_EQ(4u, str_map.size());
+ ASSERT_EQ("bar", str_map["foo"]);
+ ASSERT_EQ("nitz", str_map["frob"]);
+ ASSERT_EQ("", str_map["yeah"]);
+ ASSERT_EQ("", str_map["right"]);
+ }
+ {
+ map<string,string> str_map;
+ ASSERT_EQ(0, get_str_map("that", ss, &str_map));
+ ASSERT_EQ(1u, str_map.size());
+ ASSERT_EQ("", str_map["that"]);
+ }
+ {
+ map<string,string> str_map;
+ ASSERT_EQ(0, get_str_map(" \t \n ", ss, &str_map));
+ ASSERT_EQ(0u, str_map.size());
+ ASSERT_EQ(0, get_str_map("", ss, &str_map));
+ ASSERT_EQ(0u, str_map.size());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_str_map &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./unittest_str_map
+ * "
+ * End:
+ */
diff --git a/src/test/confutils.cc b/src/test/confutils.cc
index 196cd58..61b92d4 100644
--- a/src/test/confutils.cc
+++ b/src/test/confutils.cc
@@ -24,7 +24,7 @@
#include <stdint.h>
#include <sys/stat.h>
#include <sys/types.h>
-#include <tr1/memory>
+#include "include/memory.h"
using ceph::bufferlist;
using std::cerr;
@@ -78,7 +78,7 @@ static int create_tempfile(const std::string &fname, const char *text)
<< get_temp_dir() << "'. " << cpp_strerror(err) << std::endl;
return err;
}
- std::tr1::shared_ptr<FILE> fpp(fp, fclose);
+ ceph::shared_ptr<FILE> fpp(fp, fclose);
if (unlink_idx >= MAX_FILES_TO_DELETE)
return -ENOBUFS;
if (unlink_idx == 0) {
@@ -263,7 +263,7 @@ const char dup_key_config_1[] = "\
log_file = 3\n\
";
-TEST(Whitespace, ConfUtils) {
+TEST(ConfUtils, Whitespace) {
std::string test0("");
ConfFile::trim_whitespace(test0, false);
ASSERT_EQ(test0, "");
@@ -301,7 +301,7 @@ TEST(Whitespace, ConfUtils) {
ASSERT_EQ(test5, "abcd");
}
-TEST(ParseFiles0, ConfUtils) {
+TEST(ConfUtils, ParseFiles0) {
std::deque<std::string> err;
std::string val;
std::ostringstream warn;
@@ -332,7 +332,7 @@ TEST(ParseFiles0, ConfUtils) {
ASSERT_EQ(val, "barbaz");
}
-TEST(ParseFiles1, ConfUtils) {
+TEST(ConfUtils, ParseFiles1) {
std::deque<std::string> err;
std::ostringstream warn;
std::string simple_conf_1_f(next_tempfile(simple_conf_1));
@@ -358,7 +358,7 @@ TEST(ParseFiles1, ConfUtils) {
ASSERT_EQ(err.size(), 0U);
}
-TEST(ReadFiles1, ConfUtils) {
+TEST(ConfUtils, ReadFiles1) {
std::deque<std::string> err;
std::ostringstream warn;
std::string simple_conf_1_f(next_tempfile(simple_conf_1));
@@ -391,7 +391,7 @@ TEST(ReadFiles1, ConfUtils) {
ASSERT_EQ(cf2.read("nonesuch", "keyring", val), -ENOENT);
}
-TEST(ReadFiles2, ConfUtils) {
+TEST(ConfUtils, ReadFiles2) {
std::deque<std::string> err;
std::ostringstream warn;
std::string conf3_f(next_tempfile(conf3));
@@ -412,7 +412,7 @@ TEST(ReadFiles2, ConfUtils) {
ASSERT_EQ(val, "\x66\xd1\x86\xd1\x9d\xd3\xad\xd3\xae");
}
-TEST(IllegalFiles, ConfUtils) {
+TEST(ConfUtils, IllegalFiles) {
std::deque<std::string> err;
std::ostringstream warn;
std::string illegal_conf1_f(next_tempfile(illegal_conf1));
@@ -443,7 +443,7 @@ TEST(IllegalFiles, ConfUtils) {
ASSERT_EQ(err.size(), 1U);
}
-TEST(EscapingFiles, ConfUtils) {
+TEST(ConfUtils, EscapingFiles) {
std::deque<std::string> err;
std::ostringstream warn;
std::string escaping_conf_1_f(next_tempfile(escaping_conf_1));
@@ -470,7 +470,7 @@ TEST(EscapingFiles, ConfUtils) {
ASSERT_EQ(val, "backslash\\");
}
-TEST(Overrides, ConfUtils) {
+TEST(ConfUtils, Overrides) {
md_config_t conf;
std::deque<std::string> err;
std::ostringstream warn;
@@ -492,7 +492,7 @@ TEST(Overrides, ConfUtils) {
ASSERT_EQ(conf.log_file, "osd0_log");
}
-TEST(DupKey, ConfUtils) {
+TEST(ConfUtils, DupKey) {
md_config_t conf;
std::deque<std::string> err;
std::ostringstream warn;
diff --git a/src/test/crush/TestCrushWrapper.cc b/src/test/crush/TestCrushWrapper.cc
new file mode 100644
index 0000000..d936a6c
--- /dev/null
+++ b/src/test/crush/TestCrushWrapper.cc
@@ -0,0 +1,485 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include <iostream>
+#include <gtest/gtest.h>
+
+#include "include/stringify.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+#include "include/Context.h"
+
+#include "crush/CrushWrapper.h"
+
+TEST(CrushWrapper, get_immediate_parent) {
+ CrushWrapper *c = new CrushWrapper;
+
+ const int ROOT_TYPE = 1;
+ c->set_type_name(ROOT_TYPE, "root");
+ const int OSD_TYPE = 0;
+ c->set_type_name(OSD_TYPE, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ int item = 0;
+
+ pair <string,string> loc;
+ int ret;
+ loc = c->get_immediate_parent(item, &ret);
+ EXPECT_EQ(-ENOENT, ret);
+
+ {
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+ "osd.0", loc));
+ }
+
+ loc = c->get_immediate_parent(item, &ret);
+ EXPECT_EQ(0, ret);
+ EXPECT_EQ("root", loc.first);
+ EXPECT_EQ("default", loc.second);
+}
+
+TEST(CrushWrapper, move_bucket) {
+ CrushWrapper *c = new CrushWrapper;
+
+ const int ROOT_TYPE = 2;
+ c->set_type_name(ROOT_TYPE, "root");
+ const int HOST_TYPE = 1;
+ c->set_type_name(HOST_TYPE, "host");
+ const int OSD_TYPE = 0;
+ c->set_type_name(OSD_TYPE, "osd");
+
+ int root0;
+ EXPECT_EQ(0, c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &root0));
+ EXPECT_EQ(0, c->set_item_name(root0, "root0"));
+
+ int item = 0;
+ {
+ map<string,string> loc;
+ loc["root"] = "root0";
+ loc["host"] = "host0";
+
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+ "osd.0", loc));
+ }
+ int host0 = c->get_item_id("host0");
+
+ int root1;
+ EXPECT_EQ(0, c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &root1));
+ EXPECT_EQ(0, c->set_item_name(root1, "root1"));
+
+ map<string,string> loc;
+ loc["root"] = "root1";
+
+ // 0 is not a valid bucket number, must be negative
+ EXPECT_EQ(-EINVAL, c->move_bucket(g_ceph_context, 0, loc));
+ // -100 is not an existing bucket
+ EXPECT_EQ(-ENOENT, c->move_bucket(g_ceph_context, -100, loc));
+ // move host0 from root0 to root1
+ {
+ pair <string,string> loc;
+ int ret;
+ loc = c->get_immediate_parent(host0, &ret);
+ EXPECT_EQ(0, ret);
+ EXPECT_EQ("root", loc.first);
+ EXPECT_EQ("root0", loc.second);
+ }
+ EXPECT_EQ(0, c->move_bucket(g_ceph_context, host0, loc));
+ {
+ pair <string,string> loc;
+ int ret;
+ loc = c->get_immediate_parent(host0, &ret);
+ EXPECT_EQ(0, ret);
+ EXPECT_EQ("root", loc.first);
+ EXPECT_EQ("root1", loc.second);
+ }
+}
+
+TEST(CrushWrapper, check_item_loc) {
+ CrushWrapper *c = new CrushWrapper;
+ int item = 0;
+ float expected_weight = 1.0;
+
+ // fail if loc is empty
+ {
+ float weight;
+ map<string,string> loc;
+ EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ }
+
+ const int ROOT_TYPE = 2;
+ c->set_type_name(ROOT_TYPE, "root");
+ const int HOST_TYPE = 1;
+ c->set_type_name(HOST_TYPE, "host");
+ const int OSD_TYPE = 0;
+ c->set_type_name(OSD_TYPE, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ // fail because the item is not found at the specified location
+ {
+ float weight;
+ map<string,string> loc;
+ loc["root"] = "default";
+ EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ }
+ // fail because the bucket name does not match an existing bucket
+ {
+ float weight;
+ map<string,string> loc;
+ loc["root"] = "default";
+ const string HOST("host0");
+ loc["host"] = HOST;
+ EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ }
+ const string OSD("osd.0");
+ {
+ map<string,string> loc;
+ loc["root"] = "default";
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, expected_weight,
+ OSD, loc));
+ }
+ // fail because osd.0 is not a bucket and must not be in loc, in
+ // addition to being of the wrong type
+ {
+ float weight;
+ map<string,string> loc;
+ loc["root"] = "osd.0";
+ EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ }
+ // succeed and retrieves the expected weight
+ {
+ float weight;
+ map<string,string> loc;
+ loc["root"] = "default";
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ EXPECT_EQ(expected_weight, weight);
+ }
+}
+
+TEST(CrushWrapper, update_item) {
+ CrushWrapper *c = new CrushWrapper;
+
+ const int ROOT_TYPE = 2;
+ c->set_type_name(ROOT_TYPE, "root");
+ const int HOST_TYPE = 1;
+ c->set_type_name(HOST_TYPE, "host");
+ const int OSD_TYPE = 0;
+ c->set_type_name(OSD_TYPE, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ const string HOST0("host0");
+ int host0;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ HOST_TYPE, 0, NULL, NULL, &host0);
+ c->set_item_name(host0, HOST0);
+
+ const string HOST1("host1");
+ int host1;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ HOST_TYPE, 0, NULL, NULL, &host1);
+ c->set_item_name(host1, HOST1);
+
+ int item = 0;
+
+ // fail if invalid names anywhere in loc
+ {
+ map<string,string> loc;
+ loc["rack"] = "\001";
+ EXPECT_EQ(-EINVAL, c->update_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ // fail if invalid item name
+ {
+ map<string,string> loc;
+ EXPECT_EQ(-EINVAL, c->update_item(g_ceph_context, item, 1.0,
+ "\005", loc));
+ }
+ const string OSD0("osd.0");
+ const string OSD1("osd.1");
+ float original_weight = 1.0;
+ float modified_weight = 2.0;
+ float weight;
+
+ map<string,string> loc;
+ loc["root"] = "default";
+ loc["host"] = HOST0;
+ EXPECT_GE(0.0, c->get_item_weightf(host0));
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, original_weight,
+ OSD0, loc));
+
+ // updating nothing changes nothing
+ EXPECT_EQ(OSD0, c->get_item_name(item));
+ EXPECT_EQ(original_weight, c->get_item_weightf(item));
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ EXPECT_EQ(0, c->update_item(g_ceph_context, item, original_weight,
+ OSD0, loc));
+ EXPECT_EQ(OSD0, c->get_item_name(item));
+ EXPECT_EQ(original_weight, c->get_item_weightf(item));
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+
+ // update the name and weight of the item but not the location
+ EXPECT_EQ(OSD0, c->get_item_name(item));
+ EXPECT_EQ(original_weight, c->get_item_weightf(item));
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ EXPECT_EQ(1, c->update_item(g_ceph_context, item, modified_weight,
+ OSD1, loc));
+ EXPECT_EQ(OSD1, c->get_item_name(item));
+ EXPECT_EQ(modified_weight, c->get_item_weightf(item));
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ c->set_item_name(item, OSD0);
+ c->adjust_item_weightf(g_ceph_context, item, original_weight);
+
+ // update the name and weight of the item and change its location
+ map<string,string> other_loc;
+ other_loc["root"] = "default";
+ other_loc["host"] = HOST1;
+
+ EXPECT_EQ(OSD0, c->get_item_name(item));
+ EXPECT_EQ(original_weight, c->get_item_weightf(item));
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, other_loc, &weight));
+ EXPECT_EQ(1, c->update_item(g_ceph_context, item, modified_weight,
+ OSD1, other_loc));
+ EXPECT_EQ(OSD1, c->get_item_name(item));
+ EXPECT_EQ(modified_weight, c->get_item_weightf(item));
+ EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+ EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, other_loc, &weight));
+}
+
+TEST(CrushWrapper, insert_item) {
+ CrushWrapper *c = new CrushWrapper;
+
+ const int ROOT_TYPE = 2;
+ c->set_type_name(ROOT_TYPE, "root");
+ const int HOST_TYPE = 1;
+ c->set_type_name(HOST_TYPE, "host");
+ const int OSD_TYPE = 0;
+ c->set_type_name(OSD_TYPE, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ ROOT_TYPE, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ int item = 0;
+
+ // invalid names anywhere in loc trigger an error
+ {
+ map<string,string> loc;
+ loc["host"] = "\001";
+ EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+
+ // insert an item in an existing bucket
+ {
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ item++;
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ int another_item = item + 1;
+ EXPECT_EQ(-EEXIST, c->insert_item(g_ceph_context, another_item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ // implicit creation of a bucket
+ {
+ string name = "NAME";
+ map<string,string> loc;
+ loc["root"] = "default";
+ loc["host"] = name;
+
+ item++;
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ // adding to an existing item name that is not associated with a bucket
+ {
+ string name = "ITEM_WITHOUT_BUCKET";
+ map<string,string> loc;
+ loc["root"] = "default";
+ loc["host"] = name;
+ item++;
+ c->set_item_name(item, name);
+
+ item++;
+ EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ //
+ // When there is:
+ //
+ // default --> host0 --> item
+ //
+ // Trying to insert the same item higher in the hirarchy will fail
+ // because it would create a loop.
+ //
+ // default --> host0 --> item
+ // |
+ // +-> item
+ //
+ {
+ item++;
+ {
+ map<string,string> loc;
+ loc["root"] = "default";
+ loc["host"] = "host0";
+
+ EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ {
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ }
+ //
+ // When there is:
+ //
+ // default --> host0
+ //
+ // Trying to insert default under host0 must fail
+ // because it would create a loop.
+ //
+ // default --> host0 --> default
+ //
+ {
+ map<string,string> loc;
+ loc["host"] = "host0";
+
+ EXPECT_EQ(-ELOOP, c->insert_item(g_ceph_context, rootno, 1.0,
+ "default", loc));
+ }
+ // fail when mapping a bucket to the wrong type
+ {
+ // create an OSD bucket
+ int osdno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ OSD_TYPE, 0, NULL, NULL, &osdno);
+ c->set_item_name(osdno, "myosd");
+ map<string,string> loc;
+ loc["root"] = "default";
+ // wrongfully pretend the osd is of type host
+ loc["host"] = "myosd";
+
+ item++;
+ EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+ // fail when no location
+ {
+ map<string,string> loc;
+ item++;
+ EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+ "osd." + stringify(item), loc));
+ }
+
+ delete c;
+}
+
+TEST(CrushWrapper, item_bucket_names) {
+ CrushWrapper *c = new CrushWrapper;
+ int index = 123;
+ string name = "NAME";
+ EXPECT_EQ(-EINVAL, c->set_item_name(index, "\001"));
+ EXPECT_EQ(0, c->set_item_name(index, name));
+ EXPECT_TRUE(c->name_exists(name));
+ EXPECT_TRUE(c->item_exists(index));
+ EXPECT_EQ(index, c->get_item_id(name));
+ EXPECT_EQ(name, c->get_item_name(index));
+ delete c;
+}
+
+TEST(CrushWrapper, bucket_types) {
+ CrushWrapper *c = new CrushWrapper;
+ int index = 123;
+ string name = "NAME";
+ c->set_type_name(index, name);
+ EXPECT_EQ(1, c->get_num_type_names());
+ EXPECT_EQ(index, c->get_type_id(name));
+ EXPECT_EQ(name, c->get_type_name(index));
+ delete c;
+}
+
+TEST(CrushWrapper, is_valid_crush_name) {
+ EXPECT_TRUE(CrushWrapper::is_valid_crush_name("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012456789-_"));
+ EXPECT_FALSE(CrushWrapper::is_valid_crush_name(""));
+ EXPECT_FALSE(CrushWrapper::is_valid_crush_name("\001"));
+}
+
+TEST(CrushWrapper, is_valid_crush_loc) {
+ map<string,string> loc;
+ EXPECT_TRUE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+ loc["good"] = "better";
+ EXPECT_TRUE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+ {
+ map<string,string> loc;
+ loc["\005"] = "default";
+ EXPECT_FALSE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+ }
+ {
+ map<string,string> loc;
+ loc["host"] = "\003";
+ EXPECT_FALSE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+ }
+}
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ vector<const char*> def_args;
+ def_args.push_back("--debug-crush=0");
+ global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make unittest_crush_wrapper &&
+ * valgrind \
+ * --max-stackframe=20000000 --tool=memcheck \
+ * ./unittest_crush_wrapper --log-to-stderr=true --debug-crush=20 # --gtest_filter=CrushWrapper.insert_item"
+ * End:
+ */
diff --git a/src/test/crush/indep.cc b/src/test/crush/indep.cc
new file mode 100644
index 0000000..896e58f
--- /dev/null
+++ b/src/test/crush/indep.cc
@@ -0,0 +1,262 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank <info at inktank.com>
+ *
+ * LGPL2.1 (see COPYING-LGPL2.1) or later
+ */
+
+#include <iostream>
+#include <gtest/gtest.h>
+
+#include "include/stringify.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+#include "crush/CrushWrapper.h"
+
+#include <set>
+
+CrushWrapper *build_indep_map(CephContext *cct, int num_rack, int num_host,
+ int num_osd)
+{
+ CrushWrapper *c = new CrushWrapper;
+ c->create();
+
+ c->set_type_name(5, "root");
+ c->set_type_name(4, "row");
+ c->set_type_name(3, "rack");
+ c->set_type_name(2, "chasis");
+ c->set_type_name(1, "host");
+ c->set_type_name(0, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ 5, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int osd = 0;
+ for (int r=0; r<num_rack; ++r) {
+ loc["rack"] = string("rack-") + stringify(r);
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+ }
+
+ crush_rule *rule = crush_make_rule(4, 0, 123, 1, 20);
+ assert(rule);
+ crush_rule_set_step(rule, 0, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
+ crush_rule_set_step(rule, 1, CRUSH_RULE_TAKE, rootno, 0);
+ crush_rule_set_step(rule, 2,
+ CRUSH_RULE_CHOOSELEAF_INDEP,
+ CRUSH_CHOOSE_N,
+ 1);
+ crush_rule_set_step(rule, 3, CRUSH_RULE_EMIT, 0, 0);
+ int rno = crush_add_rule(c->crush, rule, -1);
+ c->set_rule_name(rno, "data");
+
+ if (false) {
+ Formatter *f = new_formatter("json-pretty");
+ f->open_object_section("crush_map");
+ c->dump(f);
+ f->close_section();
+ f->flush(cout);
+ delete f;
+ }
+
+ return c;
+}
+
+int get_num_dups(const vector<int>& v)
+{
+ std::set<int> s;
+ int dups = 0;
+ for (unsigned i=0; i<v.size(); ++i) {
+ if (s.count(v[i]))
+ ++dups;
+ else if (v[i] != CRUSH_ITEM_NONE)
+ s.insert(v[i]);
+ }
+ return dups;
+}
+
+TEST(CRUSH, indep_toosmall) {
+ CrushWrapper *c = build_indep_map(g_ceph_context, 1, 3, 1);
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ c->dump_tree(weight, &cout, NULL);
+
+ for (int x = 0; x < 100; ++x) {
+ vector<int> out;
+ c->do_rule(0, x, out, 5, weight);
+ cout << x << " -> " << out << std::endl;
+ int num_none = 0;
+ for (unsigned i=0; i<out.size(); ++i) {
+ if (out[i] == CRUSH_ITEM_NONE)
+ num_none++;
+ }
+ ASSERT_EQ(2, num_none);
+ ASSERT_EQ(0, get_num_dups(out));
+ }
+ delete c;
+}
+
+TEST(CRUSH, indep_basic) {
+ CrushWrapper *c = build_indep_map(g_ceph_context, 3, 3, 3);
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ c->dump_tree(weight, &cout, NULL);
+
+ for (int x = 0; x < 100; ++x) {
+ vector<int> out;
+ c->do_rule(0, x, out, 5, weight);
+ cout << x << " -> " << out << std::endl;
+ int num_none = 0;
+ for (unsigned i=0; i<out.size(); ++i) {
+ if (out[i] == CRUSH_ITEM_NONE)
+ num_none++;
+ }
+ ASSERT_EQ(0, num_none);
+ ASSERT_EQ(0, get_num_dups(out));
+ }
+ delete c;
+}
+
+TEST(CRUSH, indep_out_alt) {
+ CrushWrapper *c = build_indep_map(g_ceph_context, 3, 3, 3);
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+ // mark a bunch of osds out
+ int num = 3*3*3;
+ for (int i=0; i<num / 2; ++i)
+ weight[i*2] = 0;
+ c->dump_tree(weight, &cout, NULL);
+
+ // need more retries to get 9/9 hosts for x in 0..99
+ c->crush->choose_total_tries = 100;
+ for (int x = 0; x < 100; ++x) {
+ vector<int> out;
+ c->do_rule(0, x, out, 9, weight);
+ cout << x << " -> " << out << std::endl;
+ int num_none = 0;
+ for (unsigned i=0; i<out.size(); ++i) {
+ if (out[i] == CRUSH_ITEM_NONE)
+ num_none++;
+ }
+ ASSERT_EQ(0, num_none);
+ ASSERT_EQ(0, get_num_dups(out));
+ }
+ delete c;
+}
+
+TEST(CRUSH, indep_out_contig) {
+ CrushWrapper *c = build_indep_map(g_ceph_context, 3, 3, 3);
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+ // mark a bunch of osds out
+ int num = 3*3*3;
+ for (int i=0; i<num / 3; ++i)
+ weight[i] = 0;
+ c->dump_tree(weight, &cout, NULL);
+
+ c->crush->choose_total_tries = 100;
+ for (int x = 0; x < 100; ++x) {
+ vector<int> out;
+ c->do_rule(0, x, out, 7, weight);
+ cout << x << " -> " << out << std::endl;
+ int num_none = 0;
+ for (unsigned i=0; i<out.size(); ++i) {
+ if (out[i] == CRUSH_ITEM_NONE)
+ num_none++;
+ }
+ ASSERT_EQ(1, num_none);
+ ASSERT_EQ(0, get_num_dups(out));
+ }
+ delete c;
+}
+
+
+TEST(CRUSH, indep_out_progressive) {
+ CrushWrapper *c = build_indep_map(g_ceph_context, 3, 3, 3);
+ c->crush->choose_total_tries = 100;
+ vector<__u32> tweight(c->get_max_devices(), 0x10000);
+ c->dump_tree(tweight, &cout, NULL);
+
+ int tchanged = 0;
+ for (int x = 1; x < 5; ++x) {
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+ std::map<int,unsigned> pos;
+ vector<int> prev;
+ for (unsigned i=0; i<weight.size(); ++i) {
+ vector<int> out;
+ c->do_rule(0, x, out, 7, weight);
+ cout << "(" << i << "/" << weight.size() << " out) "
+ << x << " -> " << out << std::endl;
+ int num_none = 0;
+ for (unsigned k=0; k<out.size(); ++k) {
+ if (out[k] == CRUSH_ITEM_NONE)
+ num_none++;
+ }
+ ASSERT_EQ(0, get_num_dups(out));
+
+ // make sure nothing moved
+ int moved = 0;
+ int changed = 0;
+ for (unsigned j=0; j<out.size(); ++j) {
+ if (i && out[j] != prev[j]) {
+ ++changed;
+ ++tchanged;
+ }
+ if (out[j] == CRUSH_ITEM_NONE) {
+ continue;
+ }
+ if (i && pos.count(out[j])) {
+ // result shouldn't have moved position
+ if (j != pos[out[j]]) {
+ cout << " " << out[j] << " moved from " << pos[out[j]] << " to " << j << std::endl;
+ ++moved;
+ }
+ //ASSERT_EQ(j, pos[out[j]]);
+ }
+ }
+ if (moved || changed)
+ cout << " " << moved << " moved, " << changed << " changed" << std::endl;
+ ASSERT_LE(moved, 1);
+ ASSERT_LE(changed, 3);
+
+ // mark another osd out
+ weight[i] = 0;
+ prev = out;
+ pos.clear();
+ for (unsigned j=0; j<out.size(); ++j) {
+ if (out[j] != CRUSH_ITEM_NONE)
+ pos[out[j]] = j;
+ }
+ }
+ }
+ cout << tchanged << " total changed" << std::endl;
+
+ delete c;
+}
+
+
+
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/src/test/daemon_config.cc b/src/test/daemon_config.cc
index d643a6f..3e6dc49 100644
--- a/src/test/daemon_config.cc
+++ b/src/test/daemon_config.cc
@@ -101,24 +101,21 @@ TEST(DaemonConfig, SubstitutionBracesTrailing) {
ASSERT_EQ(string("barfoo"), string(buf));
}
-TEST(DaemonConfig, SubstitutionLoop) {
+// config: variable substitution happen only once http://tracker.ceph.com/issues/7103
+TEST(DaemonConfig, SubstitutionMultiple) {
int ret;
- ret = g_ceph_context->_conf->set_val("internal_safe_to_start_threads", "false");
- ret = g_ceph_context->_conf->set_val("host", "foo$public_network", false);
+ ret = g_ceph_context->_conf->set_val("mon_host", "localhost", false);
ASSERT_EQ(ret, 0);
- ret = g_ceph_context->_conf->set_val("public_network", "bar$host", false);
+ ret = g_ceph_context->_conf->set_val("keyring", "$mon_host/$cluster.keyring,$mon_host/$cluster.mon.keyring", false);
ASSERT_EQ(ret, 0);
g_ceph_context->_conf->apply_changes(NULL);
- char buf[128], buf2[128];
+ char buf[512];
memset(buf, 0, sizeof(buf));
- memset(buf2, 0, sizeof(buf2));
char *tmp = buf;
- char *tmp2 = buf;
- ret = g_ceph_context->_conf->get_val("host", &tmp, sizeof(buf));
+ ret = g_ceph_context->_conf->get_val("keyring", &tmp, sizeof(buf));
ASSERT_EQ(ret, 0);
- ret = g_ceph_context->_conf->get_val("public_network", &tmp2, sizeof(buf));
- ASSERT_EQ(ret, 0);
- ASSERT_TRUE(strchr(buf, '$') || strchr(buf2, '$'));
+ ASSERT_EQ(string("localhost/ceph.keyring,localhost/ceph.mon.keyring"), tmp);
+ ASSERT_TRUE(strchr(buf, '$') == NULL);
}
TEST(DaemonConfig, ArgV) {
@@ -332,3 +329,8 @@ TEST(DaemonConfig, ThreadSafety1) {
"false"));
ASSERT_EQ(ret, 0);
}
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make unittest_daemon_config && ./unittest_daemon_config"
+ * End:
+ */
diff --git a/src/test/encoding/ceph_dencoder.cc b/src/test/encoding/ceph_dencoder.cc
index dbed6f5..bb69fdd 100644
--- a/src/test/encoding/ceph_dencoder.cc
+++ b/src/test/encoding/ceph_dencoder.cc
@@ -12,11 +12,13 @@
#define TYPE(t)
#define TYPEWITHSTRAYDATA(t)
#define TYPE_FEATUREFUL(t)
+#define TYPE_NOCOPY(t)
#define MESSAGE(t)
#include "types.h"
#undef TYPE
#undef TYPEWITHSTRAYDATA
#undef TYPE_FEATUREFUL
+#undef TYPE_NOCOPY
#undef MESSAGE
void usage(ostream &out)
@@ -37,6 +39,9 @@ void usage(ostream &out)
out << " encode encode in-memory object\n";
out << " dump_json dump in-memory object as json (to stdout)\n";
out << "\n";
+ out << " copy copy object (via operator=)\n";
+ out << " copy_ctor copy object (via copy ctor)\n";
+ out << "\n";
out << " count_tests print number of generated test objects (to stdout)\n";
out << " select_test <n> select generated test object as in-memory object\n";
}
@@ -45,6 +50,12 @@ struct Dencoder {
virtual string decode(bufferlist bl) = 0;
virtual void encode(bufferlist& out, uint64_t features) = 0;
virtual void dump(ceph::Formatter *f) = 0;
+ virtual void copy() {
+ cerr << "copy operator= not supported" << std::endl;
+ }
+ virtual void copy_ctor() {
+ cerr << "copy ctor not supported" << std::endl;
+ }
virtual void generate() = 0;
virtual int num_generated() = 0;
virtual string select_generated(unsigned n) = 0;
@@ -72,8 +83,11 @@ public:
catch (buffer::error& e) {
return e.what();
}
- if (!stray_okay && !p.end())
- return "stray data at end of buffer";
+ if (!stray_okay && !p.end()) {
+ ostringstream ss;
+ ss << "stray data at end of buffer, offset " << p.get_off();
+ return ss.str();
+ }
return string();
}
@@ -82,7 +96,6 @@ public:
void dump(ceph::Formatter *f) {
m_object->dump(f);
}
-
void generate() {
T::generate_test_instances(m_list);
}
@@ -103,9 +116,10 @@ public:
};
template<class T>
-class DencoderImplNoFeature : public DencoderBase<T> {
+class DencoderImplNoFeatureNoCopy : public DencoderBase<T> {
public:
- DencoderImplNoFeature(bool stray_ok) : DencoderBase<T>(stray_ok) {}
+ DencoderImplNoFeatureNoCopy(bool stray_ok)
+ : DencoderBase<T>(stray_ok) {}
virtual void encode(bufferlist& out, uint64_t features) {
out.clear();
this->m_object->encode(out);
@@ -113,6 +127,24 @@ public:
};
template<class T>
+class DencoderImplNoFeature : public DencoderImplNoFeatureNoCopy<T> {
+public:
+ DencoderImplNoFeature(bool stray_ok)
+ : DencoderImplNoFeatureNoCopy<T>(stray_ok) {}
+ void copy() {
+ T *n = new T;
+ *n = *this->m_object;
+ delete this->m_object;
+ this->m_object = n;
+ }
+ void copy_ctor() {
+ T *n = new T(*this->m_object);
+ delete this->m_object;
+ this->m_object = n;
+ }
+};
+
+template<class T>
class DencoderImplFeatureful : public DencoderBase<T> {
public:
DencoderImplFeatureful(bool stray_ok) : DencoderBase<T>(stray_ok) {}
@@ -153,8 +185,11 @@ public:
catch (buffer::error& e) {
return e.what();
}
- if (!p.end())
- return "stray data at end of buffer";
+ if (!p.end()) {
+ ostringstream ss;
+ ss << "stray data at end of buffer, offset " << p.get_off();
+ return ss.str();
+ }
return string();
}
@@ -166,7 +201,6 @@ public:
void dump(ceph::Formatter *f) {
m_object->dump(f);
}
-
void generate() {
//T::generate_test_instances(m_list);
}
@@ -203,13 +237,14 @@ int main(int argc, const char **argv)
#define TYPE(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature<t>(false);
#define TYPEWITHSTRAYDATA(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature<t>(true);
#define TYPE_FEATUREFUL(t) dencoders[T_STRINGIFY(t)] = new DencoderImplFeatureful<t>(false);
+#define TYPE_NOCOPY(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeatureNoCopy<t>(false);
#define MESSAGE(t) dencoders[T_STRINGIFY(t)] = new MessageDencoderImpl<t>;
#include "types.h"
#undef TYPE
#undef TYPEWITHSTRAYDATA
#undef TYPE_FEATUREFUL
#undef T_STR
-#undef T_STRRINGIFY
+#undef T_STRINGIFY
vector<const char*> args;
argv_to_vec(argc, argv, args);
@@ -275,6 +310,20 @@ int main(int argc, const char **argv)
exit(1);
}
err = den->decode(encbl);
+ } else if (*i == string("copy_ctor")) {
+ if (!den) {
+ cerr << "must first select type with 'type <name>'" << std::endl;
+ usage(cerr);
+ exit(1);
+ }
+ den->copy_ctor();
+ } else if (*i == string("copy")) {
+ if (!den) {
+ cerr << "must first select type with 'type <name>'" << std::endl;
+ usage(cerr);
+ exit(1);
+ }
+ den->copy();
} else if (*i == string("dump_json")) {
if (!den) {
cerr << "must first select type with 'type <name>'" << std::endl;
@@ -294,7 +343,11 @@ int main(int argc, const char **argv)
usage(cerr);
exit(1);
}
- encbl.read_file(*i, &err);
+ int r = encbl.read_file(*i, &err);
+ if (r < 0) {
+ cerr << "error reading " << *i << ": " << err << std::endl;
+ exit(1);
+ }
} else if (*i == string("export")) {
++i;
if (i == args.end()) {
diff --git a/src/test/encoding/check-generated.sh b/src/test/encoding/check-generated.sh
index 486e2f9..a429c4b 100755
--- a/src/test/encoding/check-generated.sh
+++ b/src/test/encoding/check-generated.sh
@@ -6,6 +6,8 @@ set -e
tmp1=`mktemp /tmp/typ-XXXXXXXXX`
tmp2=`mktemp /tmp/typ-XXXXXXXXX`
+tmp3=`mktemp /tmp/typ-XXXXXXXXX`
+tmp4=`mktemp /tmp/typ-XXXXXXXXX`
failed=0
numtests=0
@@ -14,7 +16,7 @@ echo "numgen type"
for type in `./ceph-dencoder list_types`; do
num=`./ceph-dencoder type $type count_tests`
echo "$num $type"
- for n in `seq 1 $num`; do
+ for n in `seq 1 1 $num 2>/dev/null`; do
if ! ./ceph-dencoder type $type select_test $n encode decode; then
echo "**** $type test $n encode+decode check failed ****"
echo " ceph-dencoder type $type select_test $n encode decode"
@@ -24,6 +26,9 @@ for type in `./ceph-dencoder list_types`; do
./ceph-dencoder type $type select_test $n dump_json > $tmp1
./ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2
+ ./ceph-dencoder type $type select_test $n copy dump_json > $tmp3
+ ./ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp4
+
if ! cmp $tmp1 $tmp2; then
echo "**** $type test $n dump_json check failed ****"
echo " ceph-dencoder type $type select_test $n dump_json > $tmp1"
@@ -32,6 +37,22 @@ for type in `./ceph-dencoder list_types`; do
failed=$(($failed + 1))
fi
+ if ! cmp $tmp1 $tmp3; then
+ echo "**** $type test $n copy dump_json check failed ****"
+ echo " ceph-dencoder type $type select_test $n dump_json > $tmp1"
+ echo " ceph-dencoder type $type select_test $n copy dump_json > $tmp2"
+ echo " diff $tmp1 $tmp2"
+ failed=$(($failed + 1))
+ fi
+
+ if ! cmp $tmp1 $tmp4; then
+ echo "**** $type test $n copy_ctor dump_json check failed ****"
+ echo " ceph-dencoder type $type select_test $n dump_json > $tmp1"
+ echo " ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2"
+ echo " diff $tmp1 $tmp2"
+ failed=$(($failed + 1))
+ fi
+
./ceph-dencoder type $type select_test $n encode export $tmp1
./ceph-dencoder type $type select_test $n encode decode encode export $tmp2
if ! cmp $tmp1 $tmp2; then
@@ -41,11 +62,13 @@ for type in `./ceph-dencoder list_types`; do
echo " cmp $tmp1 $tmp2"
failed=$(($failed + 1))
fi
+
+
numtests=$(($numtests + 3))
done
done
-rm -f $tmp1 $tmp2
+rm -f $tmp1 $tmp2 $tmp3 $tmp4
if [ $failed -gt 0 ]; then
echo "FAILED $failed / $numtests tests."
diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h
index e446e7d..ce0af02 100644
--- a/src/test/encoding/types.h
+++ b/src/test/encoding/types.h
@@ -34,7 +34,7 @@ TYPEWITHSTRAYDATA(OSDMap)
TYPEWITHSTRAYDATA(OSDMap::Incremental)
#include "crush/CrushWrapper.h"
-TYPE(CrushWrapper)
+TYPE_NOCOPY(CrushWrapper)
#include "include/histogram.h"
TYPE(pow2_hist_t)
@@ -45,7 +45,7 @@ TYPE(object_locator_t)
TYPE(request_redirect_t)
TYPE(pg_t)
TYPE(coll_t)
-TYPE(filestore_perf_stat_t)
+TYPE(objectstore_perf_stat_t)
TYPE(osd_stat_t)
TYPE(OSDSuperblock)
TYPE_FEATUREFUL(pool_snap_info_t)
@@ -73,6 +73,8 @@ TYPE(ObjectRecoveryInfo)
TYPE(ObjectRecoveryProgress)
TYPE(ScrubMap::object)
TYPE(ScrubMap)
+TYPE(pg_hit_set_info_t)
+TYPE(pg_hit_set_history_t)
TYPE(osd_peer_stat_t)
TYPE(clone_info)
TYPE(obj_list_snap_response_t)
@@ -80,6 +82,13 @@ TYPE(PullOp)
TYPE(PushOp)
TYPE(PushReplyOp)
+#include "osd/HitSet.h"
+TYPE(ExplicitHashHitSet)
+TYPE(ExplicitObjectHitSet)
+TYPE(BloomHitSet)
+TYPE(HitSet)
+TYPE(HitSet::Params)
+
#include "os/ObjectStore.h"
TYPE(ObjectStore::Transaction)
@@ -143,7 +152,7 @@ TYPE_FEATUREFUL(MDSMap)
TYPE_FEATUREFUL(MDSMap::mds_info_t)
#include "mds/Capability.h"
-TYPE(Capability)
+TYPE_NOCOPY(Capability)
#include "mds/AnchorServer.h"
TYPE(AnchorServer)
@@ -162,7 +171,7 @@ TYPE(EImportFinish)
#include "mds/events/EImportStart.h"
TYPE(EImportStart)
#include "mds/events/EMetaBlob.h"
-TYPE(EMetaBlob::fullbit)
+TYPE_NOCOPY(EMetaBlob::fullbit)
TYPE(EMetaBlob::remotebit)
TYPE(EMetaBlob::nullbit)
TYPE(EMetaBlob::dirlump)
@@ -239,6 +248,21 @@ TYPE(cls_rgw_bi_log_list_ret)
#include "cls/rgw/cls_rgw_client.h"
TYPE(rgw_bi_log_entry)
+#include "cls/user/cls_user_types.h"
+TYPE(cls_user_bucket)
+TYPE(cls_user_bucket_entry)
+TYPE(cls_user_stats)
+TYPE(cls_user_header)
+
+#include "cls/user/cls_user_ops.h"
+TYPE(cls_user_set_buckets_op)
+TYPE(cls_user_remove_bucket_op)
+TYPE(cls_user_list_buckets_op)
+TYPE(cls_user_list_buckets_ret)
+TYPE(cls_user_get_header_op)
+TYPE(cls_user_get_header_ret)
+TYPE(cls_user_complete_stats_sync_op)
+
#include "rgw/rgw_common.h"
TYPE(RGWAccessKey);
TYPE(RGWSubUser);
diff --git a/src/test/filestore/store_test.cc b/src/test/filestore/store_test.cc
index 50450f4..8e88ab3 100644
--- a/src/test/filestore/store_test.cc
+++ b/src/test/filestore/store_test.cc
@@ -18,32 +18,44 @@
#include <time.h>
#include <sys/mount.h>
#include "os/FileStore.h"
+#include "os/KeyValueStore.h"
#include "include/Context.h"
#include "common/ceph_argparse.h"
#include "global/global_init.h"
#include "common/Mutex.h"
#include "common/Cond.h"
+#include "common/errno.h"
#include <boost/scoped_ptr.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <boost/random/binomial_distribution.hpp>
#include <gtest/gtest.h>
-#include <ext/hash_map>
-using __gnu_cxx::hash_map;
+#include "include/unordered_map.h"
typedef boost::mt11213b gen_type;
-class StoreTest : public ::testing::Test {
+#if GTEST_HAS_PARAM_TEST
+
+class StoreTest : public ::testing::TestWithParam<const char*> {
public:
boost::scoped_ptr<ObjectStore> store;
StoreTest() : store(0) {}
virtual void SetUp() {
- ::mkdir("store_test_temp_dir", 0777);
- ObjectStore *store_ = new FileStore(string("store_test_temp_dir"), string("store_test_temp_journal"));
+ int r = ::mkdir("store_test_temp_dir", 0777);
+ if (r < 0 && errno != EEXIST) {
+ r = -errno;
+ cerr << __func__ << ": unable to create store_test_temp_dir" << ": " << cpp_strerror(r) << std::endl;
+ return;
+ }
+
+ ObjectStore *store_ = ObjectStore::create(g_ceph_context,
+ string(GetParam()),
+ string("store_test_temp_dir"),
+ string("store_test_temp_journal"));
store.reset(store_);
- store->mkfs();
- store->mount();
+ EXPECT_EQ(store->mkfs(), 0);
+ EXPECT_EQ(store->mount(), 0);
}
virtual void TearDown() {
@@ -62,7 +74,7 @@ bool sorted(const vector<ghobject_t> &in) {
return true;
}
-TEST_F(StoreTest, SimpleColTest) {
+TEST_P(StoreTest, SimpleColTest) {
coll_t cid = coll_t("initial");
int r = 0;
{
@@ -95,7 +107,7 @@ TEST_F(StoreTest, SimpleColTest) {
}
}
-TEST_F(StoreTest, SimpleObjectTest) {
+TEST_P(StoreTest, SimpleObjectTest) {
int r;
coll_t cid = coll_t("coll");
{
@@ -123,7 +135,7 @@ TEST_F(StoreTest, SimpleObjectTest) {
}
}
-TEST_F(StoreTest, SimpleObjectLongnameTest) {
+TEST_P(StoreTest, SimpleObjectLongnameTest) {
int r;
coll_t cid = coll_t("coll");
{
@@ -151,7 +163,7 @@ TEST_F(StoreTest, SimpleObjectLongnameTest) {
}
}
-TEST_F(StoreTest, ManyObjectTest) {
+TEST_P(StoreTest, ManyObjectTest) {
int NUM_OBJS = 2000;
int r = 0;
coll_t cid("blah");
@@ -462,7 +474,7 @@ public:
}
};
-TEST_F(StoreTest, Synthetic) {
+TEST_P(StoreTest, Synthetic) {
ObjectStore::Sequencer osr("test");
MixedGenerator gen;
gen_type rng(time(NULL));
@@ -494,7 +506,7 @@ TEST_F(StoreTest, Synthetic) {
test_obj.wait_for_done();
}
-TEST_F(StoreTest, HashCollisionTest) {
+TEST_P(StoreTest, HashCollisionTest) {
coll_t cid("blah");
int r;
{
@@ -574,7 +586,7 @@ TEST_F(StoreTest, HashCollisionTest) {
store->apply_transaction(t);
}
-TEST_F(StoreTest, OMapTest) {
+TEST_P(StoreTest, OMapTest) {
coll_t cid("blah");
ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
int r;
@@ -670,7 +682,7 @@ TEST_F(StoreTest, OMapTest) {
store->apply_transaction(t);
}
-TEST_F(StoreTest, XattrTest) {
+TEST_P(StoreTest, XattrTest) {
coll_t cid("blah");
ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
bufferlist big;
@@ -744,7 +756,7 @@ TEST_F(StoreTest, XattrTest) {
ASSERT_EQ(r, -ENODATA);
r = store->getattr(cid, hoid, "attr3", bp);
- ASSERT_EQ(r, 0);
+ ASSERT_GE(r, 0);
bufferlist bl2;
bl2.push_back(bp);
ASSERT_TRUE(bl2 == attrs["attr3"]);
@@ -816,15 +828,15 @@ void colsplittest(
ASSERT_EQ(r, 0);
}
-TEST_F(StoreTest, ColSplitTest1) {
+TEST_P(StoreTest, ColSplitTest1) {
colsplittest(store.get(), 10000, 11);
}
-TEST_F(StoreTest, ColSplitTest2) {
+TEST_P(StoreTest, ColSplitTest2) {
colsplittest(store.get(), 100, 7);
}
#if 0
-TEST_F(StoreTest, ColSplitTest3) {
+TEST_P(StoreTest, ColSplitTest3) {
colsplittest(store.get(), 100000, 25);
}
#endif
@@ -836,7 +848,7 @@ TEST_F(StoreTest, ColSplitTest3) {
* in order to verify that the merging correctly
* stops at the common prefix subdir. See bug
* #5273 */
-TEST_F(StoreTest, TwoHash) {
+TEST_P(StoreTest, TwoHash) {
coll_t cid("asdf");
int r;
{
@@ -898,7 +910,7 @@ TEST_F(StoreTest, TwoHash) {
ASSERT_EQ(r, 0);
}
-TEST_F(StoreTest, MoveRename) {
+TEST_P(StoreTest, MoveRename) {
coll_t temp_cid("mytemp");
hobject_t temp_oid("tmp_oid", "", CEPH_NOSNAP, 0, 0, "");
coll_t cid("dest");
@@ -957,6 +969,24 @@ TEST_F(StoreTest, MoveRename) {
}
}
+INSTANTIATE_TEST_CASE_P(
+ ObjectStore,
+ StoreTest,
+ ::testing::Values("filestore", "keyvaluestore-dev"));
+
+#else
+
+// Google Test may not support value-parameterized tests with some
+// compilers. If we use conditional compilation to compile out all
+// code referring to the gtest_main library, MSVC linker will not link
+// that library at all and consequently complain about missing entry
+// point defined in that library (fatal error LNK1561: entry point
+// must be defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
+
+#endif
+
+
//
// support tests for qa/workunits/filestore/filestore.sh
//
@@ -1013,6 +1043,7 @@ TEST(EXT4StoreTest, _detect_fs) {
}
}
+
int main(int argc, char **argv) {
vector<const char*> args;
argv_to_vec(argc, (const char **)argv, args);
diff --git a/src/test/filestore/workload_generator.cc b/src/test/filestore/workload_generator.cc
index 704d930..aadb475 100644
--- a/src/test/filestore/workload_generator.cc
+++ b/src/test/filestore/workload_generator.cc
@@ -65,7 +65,8 @@ WorkloadGenerator::WorkloadGenerator(vector<const char*> args)
dout(0) << "journal = " << g_conf->osd_journal << dendl;
dout(0) << "journal size = " << g_conf->osd_journal_size << dendl;
- ::mkdir(g_conf->osd_data.c_str(), 0755);
+ err = ::mkdir(g_conf->osd_data.c_str(), 0755);
+ ceph_assert(err == 0 || (err < 0 && errno == EEXIST));
ObjectStore *store_ptr = new FileStore(g_conf->osd_data, g_conf->osd_journal);
m_store.reset(store_ptr);
err = m_store->mkfs();
diff --git a/src/test/libcephfs/caps.cc b/src/test/libcephfs/caps.cc
index 9fa92cf..e49c63c 100644
--- a/src/test/libcephfs/caps.cc
+++ b/src/test/libcephfs/caps.cc
@@ -15,7 +15,6 @@
#include "gtest/gtest.h"
#include "include/cephfs/libcephfs.h"
-#include <linux/types.h>
#include "include/ceph_fs.h"
#include <errno.h>
#include <sys/fcntl.h>
diff --git a/src/test/libcephfs/readdir_r_cb.cc b/src/test/libcephfs/readdir_r_cb.cc
index 788260b..946a405 100644
--- a/src/test/libcephfs/readdir_r_cb.cc
+++ b/src/test/libcephfs/readdir_r_cb.cc
@@ -38,6 +38,7 @@ TEST(LibCephFS, ReaddirRCB) {
sprintf(c_file, "/readdir_r_cb_tests_%d/foo", getpid());
int fd = ceph_open(cmount, c_file, O_CREAT, 0777);
ASSERT_LT(0, fd);
+ ASSERT_EQ(0, ceph_close(cmount, fd));
// check correctness with one entry
ASSERT_LE(0, ceph_closedir(cmount, dirp));
@@ -54,4 +55,9 @@ TEST(LibCephFS, ReaddirRCB) {
ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
ASSERT_EQ(5, ceph_getdnames(cmount, dirp, buf, 6));
ASSERT_EQ(4, ceph_getdnames(cmount, dirp, buf, 6));
+
+ // free cmount after finishing testing
+ ASSERT_LE(0, ceph_closedir(cmount, dirp));
+ ASSERT_EQ(0, ceph_unmount(cmount));
+ ASSERT_EQ(0, ceph_release(cmount));
}
diff --git a/src/test/libcephfs/test.cc b/src/test/libcephfs/test.cc
index 597d049..825c10c 100644
--- a/src/test/libcephfs/test.cc
+++ b/src/test/libcephfs/test.cc
@@ -1021,24 +1021,24 @@ TEST(LibCephFS, GetExtentOsds) {
int ret = ceph_get_file_extent_osds(cmount, fd, 0, NULL, NULL, 0);
EXPECT_GT(ret, 0);
- loff_t len;
+ int64_t len;
int osds[ret];
/* full stripe extent */
EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 0, &len, osds, ret));
- EXPECT_EQ(len, (loff_t)stripe_unit);
+ EXPECT_EQ(len, (int64_t)stripe_unit);
/* half stripe extent */
EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, stripe_unit/2, &len, osds, ret));
- EXPECT_EQ(len, (loff_t)stripe_unit/2);
+ EXPECT_EQ(len, (int64_t)stripe_unit/2);
/* 1.5 stripe unit offset -1 byte */
EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 3*stripe_unit/2-1, &len, osds, ret));
- EXPECT_EQ(len, (loff_t)stripe_unit/2+1);
+ EXPECT_EQ(len, (int64_t)stripe_unit/2+1);
/* 1.5 stripe unit offset +1 byte */
EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 3*stripe_unit/2+1, &len, osds, ret));
- EXPECT_EQ(len, (loff_t)stripe_unit/2-1);
+ EXPECT_EQ(len, (int64_t)stripe_unit/2-1);
/* only when more than 1 osd */
if (ret > 1)
@@ -1063,7 +1063,7 @@ TEST(LibCephFS, GetOsdCrushLocation) {
char path[256];
ASSERT_EQ(ceph_get_osd_crush_location(cmount, 9999999, path, 0), -ENOENT);
- ASSERT_EQ(ceph_get_osd_crush_location(cmount, -1, path, 0), -ENOENT);
+ ASSERT_EQ(ceph_get_osd_crush_location(cmount, -1, path, 0), -EINVAL);
char test_file[256];
sprintf(test_file, "test_osds_loc_%d", getpid());
diff --git a/src/test/librados/c_write_operations.cc b/src/test/librados/c_write_operations.cc
new file mode 100644
index 0000000..906a386
--- /dev/null
+++ b/src/test/librados/c_write_operations.cc
@@ -0,0 +1,117 @@
+// Tests for the C API coverage of atomic write operations
+
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+TEST(LibradosCWriteOps, NewDelete) {
+ rados_write_op_t op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_release_write_op(op);
+}
+
+TEST(LibRadosCWriteOps, assertExists) {
+ rados_t cluster;
+ rados_ioctx_t ioctx;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+ rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+ rados_write_op_t op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_assert_exists(op);
+
+ // -2, ENOENT
+ ASSERT_EQ(-2, rados_write_op_operate(op, ioctx, "test", NULL));
+ rados_release_write_op(op);
+
+ rados_write_op_t op2 = rados_create_write_op();
+ ASSERT_TRUE(op2);
+ rados_write_op_assert_exists(op2);
+
+ rados_completion_t completion;
+ ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
+ ASSERT_EQ(0, rados_aio_write_op_operate(op2, ioctx, completion, "test", NULL));
+ rados_aio_wait_for_complete(completion);
+ ASSERT_EQ(-2, rados_aio_get_return_value(completion));
+
+ rados_ioctx_destroy(ioctx);
+ rados_release_write_op(op2);
+ ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Xattrs) {
+ rados_t cluster;
+ rados_ioctx_t ioctx;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+ rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+ // Create an object with an xattr
+ rados_write_op_t op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+ rados_write_op_setxattr(op, "key", "value", 5);
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ rados_release_write_op(op);
+
+ // Check that xattr exists, if it does, delete it.
+ op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_create(op, LIBRADOS_CREATE_IDEMPOTENT, NULL);
+ rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
+ rados_write_op_rmxattr(op, "key");
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ rados_release_write_op(op);
+
+ // Check the xattr exits, if it does, add it again (will fail) with -125
+ // (ECANCELED)
+ op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
+ rados_write_op_setxattr(op, "key", "value", 5);
+ ASSERT_EQ(-125, rados_write_op_operate(op, ioctx, "test", NULL));
+
+ rados_release_write_op(op);
+ ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Write) {
+ rados_t cluster;
+ rados_ioctx_t ioctx;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+ rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+ // Create an object, write and write full to it
+ rados_write_op_t op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+ rados_write_op_write(op, "four", 4, 0);
+ rados_write_op_write_full(op, "hi", 2);
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ char hi[4];
+ ASSERT_EQ(2, rados_read(ioctx, "test", hi, 4, 0));
+ rados_release_write_op(op);
+
+ // Truncate and append
+ op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_truncate(op, 1);
+ rados_write_op_append(op, "hi", 2);
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ ASSERT_EQ(3, rados_read(ioctx, "test", hi, 4, 0));
+ rados_release_write_op(op);
+
+ // zero and remove
+ op = rados_create_write_op();
+ ASSERT_TRUE(op);
+ rados_write_op_zero(op, 0, 3);
+ rados_write_op_remove(op);
+ ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL));
+ // ENOENT
+ ASSERT_EQ(-2, rados_read(ioctx, "test", hi, 4, 0));
+ rados_release_write_op(op);
+
+ ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
diff --git a/src/test/librados/cmd.cc b/src/test/librados/cmd.cc
index f47cc9f..79dc658 100644
--- a/src/test/librados/cmd.cc
+++ b/src/test/librados/cmd.cc
@@ -57,6 +57,21 @@ TEST(LibRadosCmd, MonDescribe) {
ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
+TEST(LibRadosCmd, MonDescribePP) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+
+ bufferlist inbl, outbl;
+ string outs;
+ ASSERT_EQ(0, cluster.mon_command("{\"prefix\": \"get_command_descriptions\"}",
+ inbl, &outbl, &outs));
+ ASSERT_LT(0u, outbl.length());
+ ASSERT_LE(0u, outs.length());
+
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
TEST(LibRadosCmd, OSDCmd) {
rados_t cluster;
std::string pool_name = get_temp_pool_name();
diff --git a/src/test/librados/list.cc b/src/test/librados/list.cc
index 5adbe07..30e18b7 100644
--- a/src/test/librados/list.cc
+++ b/src/test/librados/list.cc
@@ -1,8 +1,10 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
+#include "include/stringify.h"
#include "test/librados/test.h"
+#include "include/types.h"
#include "gtest/gtest.h"
#include <errno.h>
#include <string>
@@ -191,3 +193,118 @@ TEST(LibRadosList, ListObjectsPPNS) {
ioctx.close();
ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
+
+TEST(LibRadosList, ListObjectsManyPP) {
+ std::string pool_name = get_temp_pool_name();
+ Rados cluster;
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl;
+ bl.append(buf, sizeof(buf));
+
+ for (int i=0; i<256; ++i) {
+ ASSERT_EQ((int)sizeof(buf), ioctx.write(stringify(i), bl, bl.length(), 0));
+ }
+
+ librados::ObjectIterator it = ioctx.objects_begin();
+ std::set<std::string> saw_obj;
+ std::set<int> saw_pg;
+ for (; it != ioctx.objects_end(); ++it) {
+ std::cout << it->first
+ << " " << it.get_pg_hash_position() << std::endl;
+ saw_obj.insert(it->first);
+ saw_pg.insert(it.get_pg_hash_position());
+ }
+ std::cout << "saw " << saw_pg.size() << " pgs " << std::endl;
+
+ // make sure they are 0..n
+ for (unsigned i = 0; i < saw_pg.size(); ++i)
+ ASSERT_TRUE(saw_pg.count(i));
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+
+
+TEST(LibRadosList, ListObjectsStart) {
+ rados_t cluster;
+ rados_ioctx_t ioctx;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+ rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+
+ for (int i=0; i<16; ++i) {
+ string n = stringify(i);
+ ASSERT_EQ((int)sizeof(buf), rados_write(ioctx, n.c_str(), buf, sizeof(buf), 0));
+ }
+
+ rados_list_ctx_t ctx;
+ ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+ std::map<int, std::set<std::string> > pg_to_obj;
+ const char *entry;
+ while (rados_objects_list_next(ctx, &entry, NULL) == 0) {
+ uint32_t pos = rados_objects_list_get_pg_hash_position(ctx);
+ std::cout << entry << " " << pos << std::endl;
+ pg_to_obj[pos].insert(entry);
+ }
+ rados_objects_list_close(ctx);
+
+ std::map<int, std::set<std::string> >::reverse_iterator p =
+ pg_to_obj.rbegin();
+ ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+ while (p != pg_to_obj.rend()) {
+ ASSERT_EQ((uint32_t)p->first, rados_objects_list_seek(ctx, p->first));
+ ASSERT_EQ(0, rados_objects_list_next(ctx, &entry, NULL));
+ std::cout << "have " << entry << " expect one of " << p->second << std::endl;
+ ASSERT_TRUE(p->second.count(entry));
+ ++p;
+ }
+ rados_objects_list_close(ctx);
+ rados_ioctx_destroy(ioctx);
+ ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosList, ListObjectsStartPP) {
+ std::string pool_name = get_temp_pool_name();
+ Rados cluster;
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+ char buf[128];
+ memset(buf, 0xcc, sizeof(buf));
+ bufferlist bl;
+ bl.append(buf, sizeof(buf));
+
+ for (int i=0; i<16; ++i) {
+ ASSERT_EQ((int)sizeof(buf), ioctx.write(stringify(i), bl, bl.length(), 0));
+ }
+
+ librados::ObjectIterator it = ioctx.objects_begin();
+ std::map<int, std::set<std::string> > pg_to_obj;
+ for (; it != ioctx.objects_end(); ++it) {
+ std::cout << it->first << " " << it.get_pg_hash_position() << std::endl;
+ pg_to_obj[it.get_pg_hash_position()].insert(it->first);
+ }
+
+ std::map<int, std::set<std::string> >::reverse_iterator p =
+ pg_to_obj.rbegin();
+ it = ioctx.objects_begin(p->first);
+ while (p != pg_to_obj.rend()) {
+ ASSERT_EQ((uint32_t)p->first, it.seek(p->first));
+ std::cout << "have " << it->first << " expect one of " << p->second << std::endl;
+ ASSERT_TRUE(p->second.count(it->first));
+ ++p;
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
diff --git a/src/test/librados/misc.cc b/src/test/librados/misc.cc
index 9abac9c..0b2ba6c 100644
--- a/src/test/librados/misc.cc
+++ b/src/test/librados/misc.cc
@@ -1,3 +1,5 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
#include "gtest/gtest.h"
#include "mds/mdstypes.h"
@@ -46,6 +48,16 @@ TEST(LibRadosMisc, ClusterFSID) {
ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
}
+TEST(LibRadosMisc, WaitOSDMapPP) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+
+ ASSERT_EQ(0, cluster.wait_for_latest_osdmap());
+
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
static std::string read_key_from_tmap(IoCtx& ioctx, const std::string &obj,
const std::string &key)
{
@@ -253,6 +265,71 @@ TEST(LibRadosMisc, TmapUpdateMisorderedPutPP) {
ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
+TEST(LibRadosMisc, Tmap2OmapPP) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+ // create tmap
+ bufferlist hdr;
+ hdr.append("header");
+ map<string, bufferlist> omap;
+ omap["1"].append("a");
+ omap["2"].append("b");
+ omap["3"].append("c");
+ {
+ bufferlist bl;
+ ::encode(hdr, bl);
+ ::encode(omap, bl);
+ ASSERT_EQ(0, ioctx.tmap_put("foo", bl));
+ }
+
+ // convert tmap to omap
+ ASSERT_EQ(0, ioctx.tmap_to_omap("foo", false));
+
+ // if tmap was truncated ?
+ {
+ uint64_t size;
+ time_t mtime;
+ ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+ ASSERT_EQ(0U, size);
+ }
+
+ // if 'nullok' works
+ ASSERT_EQ(0, ioctx.tmap_to_omap("foo", true));
+ ASSERT_LE(ioctx.tmap_to_omap("foo", false), 0);
+
+ {
+ // read omap
+ bufferlist got;
+ map<string, bufferlist> m;
+ ObjectReadOperation o;
+ o.omap_get_header(&got, NULL);
+ o.omap_get_vals("", 1024, &m, NULL);
+ ASSERT_EQ(0, ioctx.operate("foo", &o, NULL));
+
+ // compare header
+ ASSERT_TRUE(hdr.contents_equal(got));
+
+ // compare values
+ ASSERT_EQ(omap.size(), m.size());
+ bool same = true;
+ for (map<string, bufferlist>::iterator p = omap.begin(); p != omap.end(); ++p) {
+ map<string, bufferlist>::iterator q = m.find(p->first);
+ if (q == m.end() || !p->second.contents_equal(q->second)) {
+ same = false;
+ break;
+ }
+ }
+ ASSERT_TRUE(same);
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
TEST(LibRadosMisc, Exec) {
char buf[128];
rados_t cluster;
@@ -312,6 +389,7 @@ TEST(LibRadosMisc, Operate1PP) {
bufferlist bl;
bl.append(val1.c_str(), val1.size() + 1);
o.setxattr("key1", bl);
+ o.omap_clear(); // shouldn't affect attrs!
}
ASSERT_EQ(0, ioctx.operate("foo", &o));
@@ -585,9 +663,22 @@ TEST(LibRadosMisc, CopyPP) {
ASSERT_EQ(0, ioctx.write_full("foo", blc));
ASSERT_EQ(0, ioctx.setxattr("foo", "myattr", xc));
+ version_t uv = ioctx.get_last_version();
{
+ // pass future version
ObjectWriteOperation op;
- op.copy_from("foo", ioctx, ioctx.get_last_version());
+ op.copy_from("foo", ioctx, uv + 1);
+ ASSERT_EQ(-EOVERFLOW, ioctx.operate("foo.copy", &op));
+ }
+ {
+ // pass old version
+ ObjectWriteOperation op;
+ op.copy_from("foo", ioctx, uv - 1);
+ ASSERT_EQ(-ERANGE, ioctx.operate("foo.copy", &op));
+ }
+ {
+ ObjectWriteOperation op;
+ op.copy_from("foo", ioctx, uv);
ASSERT_EQ(0, ioctx.operate("foo.copy", &op));
bufferlist bl2, x2;
@@ -647,60 +738,6 @@ TEST(LibRadosMisc, CopyPP) {
ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
}
-TEST(LibRadosMisc, Dirty) {
- Rados cluster;
- std::string pool_name = get_temp_pool_name();
- ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
- IoCtx ioctx;
- ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
-
- {
- ObjectWriteOperation op;
- op.create(true);
- ASSERT_EQ(0, ioctx.operate("foo", &op));
- }
- {
- bool dirty = false;
- int r = -1;
- ObjectReadOperation op;
- op.is_dirty(&dirty, &r);
- ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
- ASSERT_TRUE(dirty);
- ASSERT_EQ(0, r);
- }
- {
- ObjectWriteOperation op;
- op.undirty();
- ASSERT_EQ(0, ioctx.operate("foo", &op));
- }
- {
- bool dirty = false;
- int r = -1;
- ObjectReadOperation op;
- op.is_dirty(&dirty, &r);
- ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
- ASSERT_FALSE(dirty);
- ASSERT_EQ(0, r);
- }
- {
- ObjectWriteOperation op;
- op.truncate(0); // still a write even tho it is a no-op
- ASSERT_EQ(0, ioctx.operate("foo", &op));
- }
- {
- bool dirty = false;
- int r = -1;
- ObjectReadOperation op;
- op.is_dirty(&dirty, &r);
- ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
- ASSERT_TRUE(dirty);
- ASSERT_EQ(0, r);
- }
-
- ioctx.close();
- ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
-}
-
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);
diff --git a/src/test/librados/snapshots.cc b/src/test/librados/snapshots.cc
index fc83cf1..d2bbe74 100644
--- a/src/test/librados/snapshots.cc
+++ b/src/test/librados/snapshots.cc
@@ -212,8 +212,11 @@ TEST(LibRadosSnapshots, SelfManagedSnapTest) {
char buf2[sizeof(buf)];
memset(buf2, 0xdd, sizeof(buf2));
ASSERT_EQ((int)sizeof(buf2), rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
- rados_ioctx_snap_set_read(ioctx, my_snaps[1]);
+ rados_ioctx_snap_set_read(ioctx, my_snaps[1]-1);
char buf3[sizeof(buf)];
+ ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+
+ rados_ioctx_snap_set_read(ioctx, my_snaps[1]);
ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
diff --git a/src/test/librados/tier.cc b/src/test/librados/tier.cc
new file mode 100644
index 0000000..065ad34
--- /dev/null
+++ b/src/test/librados/tier.cc
@@ -0,0 +1,2216 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "mds/mdstypes.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "include/types.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/common_init.h"
+#include "common/Cond.h"
+#include "test/librados/test.h"
+#include "json_spirit/json_spirit.h"
+
+#include "osd/HitSet.h"
+
+#include <errno.h>
+#include <map>
+#include <sstream>
+#include <string>
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+TEST(LibRadosTier, Dirty) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+ {
+ ObjectWriteOperation op;
+ op.undirty();
+ ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
+ }
+ {
+ ObjectWriteOperation op;
+ op.create(true);
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
+ }
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_TRUE(dirty);
+ ASSERT_EQ(0, r);
+ }
+ {
+ ObjectWriteOperation op;
+ op.undirty();
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
+ }
+ {
+ ObjectWriteOperation op;
+ op.undirty();
+ ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
+ }
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_FALSE(dirty);
+ ASSERT_EQ(0, r);
+ }
+ {
+ ObjectWriteOperation op;
+ op.truncate(0); // still a write even tho it is a no-op
+ ASSERT_EQ(0, ioctx.operate("foo", &op));
+ }
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+ ASSERT_TRUE(dirty);
+ ASSERT_EQ(0, r);
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(LibRadosTier, Overlay) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create objects
+ {
+ bufferlist bl;
+ bl.append("base");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("cache");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // by default, the overlay sends us to cache pool
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+
+ // unless we say otherwise
+ {
+ bufferlist bl;
+ ObjectReadOperation op;
+ op.read(0, 1, &bl, NULL);
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ ASSERT_EQ('b', bl[0]);
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, Promote) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // read, trigger a promote
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ }
+
+ // read, trigger a whiteout
+ {
+ bufferlist bl;
+ ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
+ }
+
+ // verify the object is present in the cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it != cache_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+ ++it;
+ ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+ ++it;
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, PromoteSnap) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ }
+
+ // create a snapshot, clone
+ vector<uint64_t> my_snaps(1);
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ my_snaps));
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ }
+ {
+ ObjectWriteOperation op;
+ op.remove();
+ ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // read, trigger a promote on the head
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("bam", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+
+ base_ioctx.snap_set_read(my_snaps[0]);
+
+ // read foo snap
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('h', bl[0]);
+ }
+
+ // read bar snap
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ('h', bl[0]);
+ }
+
+ // read baz snap
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("baz", bl, 1, 0));
+ ASSERT_EQ('h', bl[0]);
+ }
+
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+
+ // read foo
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+
+ // read bar
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+
+ // read baz
+ {
+ bufferlist bl;
+ ASSERT_EQ(-ENOENT, base_ioctx.read("baz", bl, 1, 0));
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, PromoteSnapTrimRace) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // create a snapshot, clone
+ vector<uint64_t> my_snaps(1);
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ my_snaps));
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // delete the snap
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_remove(my_snaps[0]));
+
+ base_ioctx.snap_set_read(my_snaps[0]);
+
+ // read foo snap
+ {
+ bufferlist bl;
+ ASSERT_EQ(-ENOENT, base_ioctx.read("foo", bl, 1, 0));
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ // cluster.pool_delete(cache_pool_name.c_str());
+ //ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, Whiteout) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // create some whiteouts, verify they behave
+ ASSERT_EQ(0, base_ioctx.remove("foo"));
+
+ ASSERT_EQ(-ENOENT, base_ioctx.remove("bar"));
+ ASSERT_EQ(-ENOENT, base_ioctx.remove("bar"));
+
+ // verify the whiteouts are there in the cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it != cache_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+ ++it;
+ ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+ ++it;
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ ASSERT_EQ(-ENOENT, base_ioctx.remove("foo"));
+
+ // recreate an object and verify we can read it
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('h', bl[0]);
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, Evict) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // read, trigger a promote
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ }
+
+ // read, trigger a whiteout, and a dirty object
+ {
+ bufferlist bl;
+ ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(-ENOENT, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ(0, base_ioctx.write("bar", bl, bl.length(), 0));
+ }
+
+ // verify the object is present in the cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it != cache_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+ ++it;
+ ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+ ++it;
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // evict
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE,
+ NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "fooberdoodle", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-ENOENT, completion->get_return_value());
+ completion->release();
+ }
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "bar", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-EBUSY, completion->get_return_value());
+ completion->release();
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, EvictSnap) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ }
+
+ // create a snapshot, clone
+ vector<uint64_t> my_snaps(1);
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ my_snaps));
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bar", &op));
+ }
+ {
+ ObjectWriteOperation op;
+ op.remove();
+ ASSERT_EQ(0, base_ioctx.operate("baz", &op));
+ }
+ {
+ bufferlist bl;
+ bl.append("ciao!");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("bam", &op));
+ }
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // read, trigger a promote on the head
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("bam", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+
+ // evict bam
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "bam", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ {
+ bufferlist bl;
+ ObjectReadOperation op;
+ op.read(1, 0, &bl, NULL);
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "bam", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-ENOENT, completion->get_return_value());
+ completion->release();
+ }
+
+ // read foo snap
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('h', bl[0]);
+ }
+
+ // evict foo snap
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ // snap is gone...
+ {
+ bufferlist bl;
+ ObjectReadOperation op;
+ op.read(1, 0, &bl, NULL);
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-ENOENT, completion->get_return_value());
+ completion->release();
+ }
+ // head is still there...
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ bufferlist bl;
+ ObjectReadOperation op;
+ op.read(1, 0, &bl, NULL);
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // promote head + snap of bar
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("bar", bl, 1, 0));
+ ASSERT_EQ('h', bl[0]);
+ }
+
+ // evict bar head (fail)
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "bar", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-EBUSY, completion->get_return_value());
+ completion->release();
+ }
+
+ // evict bar snap
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "bar", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ // ...and then head
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ bufferlist bl;
+ ObjectReadOperation op;
+ op.read(1, 0, &bl, NULL);
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "bar", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "bar", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, TryFlush) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // verify the object is present in the cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it != cache_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo"));
+ ++it;
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // verify the object is NOT present in the base tier
+ {
+ ObjectIterator it = base_ioctx.objects_begin();
+ ASSERT_TRUE(it == base_ioctx.objects_end());
+ }
+
+ // verify dirty
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+ ASSERT_TRUE(dirty);
+ ASSERT_EQ(0, r);
+ }
+
+ // flush
+ {
+ ObjectReadOperation op;
+ op.cache_try_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // verify clean
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+ ASSERT_FALSE(dirty);
+ ASSERT_EQ(0, r);
+ }
+
+ // verify in base tier
+ {
+ ObjectIterator it = base_ioctx.objects_begin();
+ ASSERT_TRUE(it != base_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo"));
+ ++it;
+ ASSERT_TRUE(it == base_ioctx.objects_end());
+ }
+
+ // evict it
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // verify no longer in cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, Flush) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ uint64_t user_version = 0;
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // verify the object is present in the cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it != cache_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo"));
+ ++it;
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // verify the object is NOT present in the base tier
+ {
+ ObjectIterator it = base_ioctx.objects_begin();
+ ASSERT_TRUE(it == base_ioctx.objects_end());
+ }
+
+ // verify dirty
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+ ASSERT_TRUE(dirty);
+ ASSERT_EQ(0, r);
+ user_version = cache_ioctx.get_last_version();
+ }
+
+ // flush
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // verify clean
+ {
+ bool dirty = false;
+ int r = -1;
+ ObjectReadOperation op;
+ op.is_dirty(&dirty, &r);
+ ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+ ASSERT_FALSE(dirty);
+ ASSERT_EQ(0, r);
+ }
+
+ // verify in base tier
+ {
+ ObjectIterator it = base_ioctx.objects_begin();
+ ASSERT_TRUE(it != base_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo"));
+ ++it;
+ ASSERT_TRUE(it == base_ioctx.objects_end());
+ }
+
+ // evict it
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // verify no longer in cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // read it again and verify the version is consistent
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ(user_version, cache_ioctx.get_last_version());
+ }
+
+ // erase it
+ {
+ ObjectWriteOperation op;
+ op.remove();
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // flush whiteout
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // evict
+ {
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // verify no longer in cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+ // or base tier
+ {
+ ObjectIterator it = base_ioctx.objects_begin();
+ ASSERT_TRUE(it == base_ioctx.objects_end());
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, FlushSnap) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // create object
+ {
+ bufferlist bl;
+ bl.append("a");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // create a snapshot, clone
+ vector<uint64_t> my_snaps(1);
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ my_snaps));
+ {
+ bufferlist bl;
+ bl.append("b");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // and another
+ my_snaps.resize(2);
+ my_snaps[1] = my_snaps[0];
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+ ASSERT_EQ(0, base_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+ my_snaps));
+ {
+ bufferlist bl;
+ bl.append("c");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // verify the object is present in the cache tier
+ {
+ ObjectIterator it = cache_ioctx.objects_begin();
+ ASSERT_TRUE(it != cache_ioctx.objects_end());
+ ASSERT_TRUE(it->first == string("foo"));
+ ++it;
+ ASSERT_TRUE(it == cache_ioctx.objects_end());
+ }
+
+ // verify the object is NOT present in the base tier
+ {
+ ObjectIterator it = base_ioctx.objects_begin();
+ ASSERT_TRUE(it == base_ioctx.objects_end());
+ }
+
+ // flush on head (should fail)
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-EBUSY, completion->get_return_value());
+ completion->release();
+ }
+ // flush on recent snap (should fail)
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(-EBUSY, completion->get_return_value());
+ completion->release();
+ }
+ // flush on oldest snap
+ base_ioctx.snap_set_read(my_snaps[1]);
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ // flush on next oldest snap
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+ // flush on head
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_CACHE, NULL));
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+ }
+
+ // verify i can read the snaps from the cache pool
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('b', bl[0]);
+ }
+ base_ioctx.snap_set_read(my_snaps[1]);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('a', bl[0]);
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // verify i can read the snaps from the base pool
+ base_ioctx.snap_set_read(librados::SNAP_HEAD);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('c', bl[0]);
+ }
+ base_ioctx.snap_set_read(my_snaps[0]);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('b', bl[0]);
+ }
+ base_ioctx.snap_set_read(my_snaps[1]);
+ {
+ bufferlist bl;
+ ASSERT_EQ(1, base_ioctx.read("foo", bl, 1, 0));
+ ASSERT_EQ('a', bl[0]);
+ }
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, FlushWriteRaces) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // create/dirty object
+ bufferlist bl;
+ bl.append("hi there");
+ {
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // flush + write
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+ ObjectWriteOperation op2;
+ op2.write_full(bl);
+ librados::AioCompletion *completion2 = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate(
+ "foo", completion2, &op2, 0));
+
+ completion->wait_for_safe();
+ completion2->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ ASSERT_EQ(0, completion2->get_return_value());
+ completion->release();
+ completion2->release();
+ }
+
+ int tries = 1000;
+ do {
+ // create/dirty object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // try-flush + write
+ {
+ ObjectReadOperation op;
+ op.cache_try_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+
+ ObjectWriteOperation op2;
+ op2.write_full(bl);
+ librados::AioCompletion *completion2 = cluster.aio_create_completion();
+ ASSERT_EQ(0, base_ioctx.aio_operate("foo", completion2, &op2, 0));
+
+ completion->wait_for_safe();
+ completion2->wait_for_safe();
+ int r = completion->get_return_value();
+ ASSERT_TRUE(r == -EBUSY || r == 0);
+ ASSERT_EQ(0, completion2->get_return_value());
+ completion->release();
+ completion2->release();
+ if (r == -EBUSY)
+ break;
+ cout << "didn't get EBUSY, trying again" << std::endl;
+ }
+ ASSERT_TRUE(--tries);
+ } while (true);
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, FlushTryFlushRaces) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // create/dirty object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // flush + flush
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+ ObjectReadOperation op2;
+ op2.cache_flush();
+ librados::AioCompletion *completion2 = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion2, &op2,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+ completion->wait_for_safe();
+ completion2->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ ASSERT_EQ(0, completion2->get_return_value());
+ completion->release();
+ completion2->release();
+ }
+
+ // create/dirty object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // flush + try-flush
+ {
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+ ObjectReadOperation op2;
+ op2.cache_try_flush();
+ librados::AioCompletion *completion2 = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion2, &op2,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+
+ completion->wait_for_safe();
+ completion2->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ ASSERT_EQ(0, completion2->get_return_value());
+ completion->release();
+ completion2->release();
+ }
+
+ // create/dirty object
+ int tries = 1000;
+ do {
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // try-flush + flush
+ // (flush will not piggyback on try-flush)
+ {
+ ObjectReadOperation op;
+ op.cache_try_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+
+ ObjectReadOperation op2;
+ op2.cache_flush();
+ librados::AioCompletion *completion2 = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion2, &op2,
+ librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+ completion->wait_for_safe();
+ completion2->wait_for_safe();
+ int r = completion->get_return_value();
+ ASSERT_TRUE(r == -EBUSY || r == 0);
+ ASSERT_EQ(0, completion2->get_return_value());
+ completion->release();
+ completion2->release();
+ if (r == -EBUSY)
+ break;
+ cout << "didn't get EBUSY, trying again" << std::endl;
+ }
+ ASSERT_TRUE(--tries);
+ } while (true);
+
+ // create/dirty object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // try-flush + try-flush
+ {
+ ObjectReadOperation op;
+ op.cache_try_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+
+ ObjectReadOperation op2;
+ op2.cache_try_flush();
+ librados::AioCompletion *completion2 = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion2, &op2,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+
+ completion->wait_for_safe();
+ completion2->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ ASSERT_EQ(0, completion2->get_return_value());
+ completion->release();
+ completion2->release();
+ }
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+
+IoCtx *read_ioctx = 0;
+Mutex test_lock("FlushReadRaces::lock");
+Cond cond;
+int max_reads = 100;
+int num_reads = 0; // in progress
+
+void flush_read_race_cb(completion_t cb, void *arg);
+
+void start_flush_read()
+{
+ //cout << " starting read" << std::endl;
+ ObjectReadOperation op;
+ op.stat(NULL, NULL, NULL);
+ librados::AioCompletion *completion =
+ librados::Rados::aio_create_completion();
+ completion->set_complete_callback(0, flush_read_race_cb);
+ read_ioctx->aio_operate("foo", completion, &op, NULL);
+}
+
+void flush_read_race_cb(completion_t cb, void *arg)
+{
+ //cout << " finished read" << std::endl;
+ test_lock.Lock();
+ if (num_reads > max_reads) {
+ num_reads--;
+ cond.Signal();
+ } else {
+ start_flush_read();
+ }
+ // fixme: i'm leaking cb...
+ test_lock.Unlock();
+}
+
+TEST(LibRadosTier, TryFlushReadRace) {
+ Rados cluster;
+ std::string base_pool_name = get_temp_pool_name();
+ std::string cache_pool_name = base_pool_name + "-cache";
+ ASSERT_EQ("", create_one_pool_pp(base_pool_name, cluster));
+ ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+ IoCtx cache_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+ IoCtx base_ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
+
+ // configure cache
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool_name +
+ "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+ "\", \"mode\": \"writeback\"}",
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // create/dirty object
+ {
+ bufferlist bl;
+ bl.append("hi there");
+ bufferptr bp(4000000); // make it big!
+ bp.zero();
+ bl.append(bp);
+ ObjectWriteOperation op;
+ op.write_full(bl);
+ ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+ }
+
+ // start a continuous stream of reads
+ read_ioctx = &base_ioctx;
+ test_lock.Lock();
+ for (int i = 0; i < max_reads; ++i) {
+ start_flush_read();
+ num_reads++;
+ }
+ test_lock.Unlock();
+
+ // try-flush
+ ObjectReadOperation op;
+ op.cache_try_flush();
+ librados::AioCompletion *completion = cluster.aio_create_completion();
+ ASSERT_EQ(0, cache_ioctx.aio_operate(
+ "foo", completion, &op,
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS, NULL));
+
+ completion->wait_for_safe();
+ ASSERT_EQ(0, completion->get_return_value());
+ completion->release();
+
+ // stop reads
+ test_lock.Lock();
+ max_reads = 0;
+ while (num_reads > 0)
+ cond.Wait(test_lock);
+ test_lock.Unlock();
+
+ // tear down tiers
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +
+ "\"}",
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(
+ "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool_name +
+ "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+ inbl, NULL, NULL));
+
+ base_ioctx.close();
+ cache_ioctx.close();
+
+ cluster.pool_delete(cache_pool_name.c_str());
+ ASSERT_EQ(0, destroy_one_pool_pp(base_pool_name, cluster));
+}
+
+TEST(LibRadosTier, HitSetNone) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+ {
+ list< pair<time_t,time_t> > ls;
+ AioCompletion *c = librados::Rados::aio_create_completion();
+ ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
+ c->wait_for_complete();
+ ASSERT_EQ(0, c->get_return_value());
+ ASSERT_TRUE(ls.empty());
+ c->release();
+ }
+ {
+ bufferlist bl;
+ AioCompletion *c = librados::Rados::aio_create_completion();
+ ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
+ c->wait_for_complete();
+ ASSERT_EQ(-ENOENT, c->get_return_value());
+ c->release();
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+string set_pool_str(string pool, string var, string val)
+{
+ return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
+ + string("\",\"var\": \"") + var + string("\",\"val\": \"")
+ + val + string("\"}");
+}
+
+string set_pool_str(string pool, string var, int val)
+{
+ return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
+ + string("\",\"var\": \"") + var + string("\",\"val\": ")
+ + stringify(val) + string("}");
+}
+
+TEST(LibRadosTier, HitSetRead) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+ // enable hitset tracking for this pool
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 2),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
+ "explicit_object"),
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // keep reading until we see our object appear in the HitSet
+ utime_t start = ceph_clock_now(NULL);
+ utime_t hard_stop = start + utime_t(600, 0);
+
+ while (true) {
+ utime_t now = ceph_clock_now(NULL);
+ ASSERT_TRUE(now < hard_stop);
+
+ string name = "foo";
+ uint32_t hash = ioctx.get_object_hash_position(name);
+ hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
+ cluster.pool_lookup(pool_name.c_str()), "");
+
+ bufferlist bl;
+ ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
+
+ bufferlist hbl;
+ AioCompletion *c = librados::Rados::aio_create_completion();
+ ASSERT_EQ(0, ioctx.hit_set_get(hash, c, now.sec(), &hbl));
+ c->wait_for_complete();
+ c->release();
+
+ if (hbl.length()) {
+ bufferlist::iterator p = hbl.begin();
+ HitSet hs;
+ ::decode(hs, p);
+ if (hs.contains(oid)) {
+ cout << "ok, hit_set contains " << oid << std::endl;
+ break;
+ }
+ cout << "hmm, not in HitSet yet" << std::endl;
+ } else {
+ cout << "hmm, no HitSet yet" << std::endl;
+ }
+
+ sleep(1);
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+static int _get_pg_num(Rados& cluster, string pool_name)
+{
+ bufferlist inbl;
+ string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
+ + pool_name
+ + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
+ bufferlist outbl;
+ int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
+ assert(r >= 0);
+ string outstr(outbl.c_str(), outbl.length());
+ json_spirit::Value v;
+ if (!json_spirit::read(outstr, v)) {
+ cerr <<" unable to parse json " << outstr << std::endl;
+ return -1;
+ }
+
+ json_spirit::Object& o = v.get_obj();
+ for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
+ json_spirit::Pair& p = o[i];
+ if (p.name_ == "pg_num") {
+ cout << "pg_num = " << p.value_.get_int() << std::endl;
+ return p.value_.get_int();
+ }
+ }
+ cerr << "didn't find pg_num in " << outstr << std::endl;
+ return -1;
+}
+
+
+TEST(LibRadosTier, HitSetWrite) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+ int num_pg = _get_pg_num(cluster, pool_name);
+ assert(num_pg > 0);
+
+ // enable hitset tracking for this pool
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
+ "explicit_hash"),
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // do a bunch of writes
+ for (int i=0; i<1000; ++i) {
+ bufferlist bl;
+ bl.append("a");
+ ASSERT_EQ(1, ioctx.write(stringify(i), bl, 1, 0));
+ }
+
+ // get HitSets
+ std::map<int,HitSet> hitsets;
+ for (int i=0; i<num_pg; ++i) {
+ list< pair<time_t,time_t> > ls;
+ AioCompletion *c = librados::Rados::aio_create_completion();
+ ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
+ c->wait_for_complete();
+ c->release();
+ std::cout << "pg " << i << " ls " << ls << std::endl;
+ ASSERT_FALSE(ls.empty());
+
+ // get the latest
+ c = librados::Rados::aio_create_completion();
+ bufferlist bl;
+ ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
+ c->wait_for_complete();
+ c->release();
+
+ //std::cout << "bl len is " << bl.length() << "\n";
+ //bl.hexdump(std::cout);
+ //std::cout << std::endl;
+
+ bufferlist::iterator p = bl.begin();
+ ::decode(hitsets[i], p);
+
+ // cope with racing splits by refreshing pg_num
+ if (i == num_pg - 1)
+ num_pg = _get_pg_num(cluster, pool_name);
+ }
+
+ for (int i=0; i<1000; ++i) {
+ string n = stringify(i);
+ uint32_t hash = ioctx.get_object_hash_position(n);
+ hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
+ cluster.pool_lookup(pool_name.c_str()), "");
+ std::cout << "checking for " << oid << std::endl;
+ bool found = false;
+ for (int p=0; p<num_pg; ++p) {
+ if (hitsets[p].contains(oid)) {
+ found = true;
+ break;
+ }
+ }
+ ASSERT_TRUE(found);
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(LibRadosTier, HitSetTrim) {
+ Rados cluster;
+ std::string pool_name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+ IoCtx ioctx;
+ ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+ unsigned count = 3;
+ unsigned period = 3;
+
+ // enable hitset tracking for this pool
+ bufferlist inbl;
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", count),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", period),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type", "bloom"),
+ inbl, NULL, NULL));
+ ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_fpp", ".01"),
+ inbl, NULL, NULL));
+
+ // wait for maps to settle
+ cluster.wait_for_latest_osdmap();
+
+ // do a bunch of writes and make sure the hitsets rotate
+ utime_t start = ceph_clock_now(NULL);
+ utime_t hard_stop = start + utime_t(count * period * 12, 0);
+
+ time_t first = 0;
+ while (true) {
+ string name = "foo";
+ uint32_t hash = ioctx.get_object_hash_position(name);
+ hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
+
+ bufferlist bl;
+ bl.append("f");
+ ASSERT_EQ(1, ioctx.write("foo", bl, 1, 0));
+
+ list<pair<time_t, time_t> > ls;
+ AioCompletion *c = librados::Rados::aio_create_completion();
+ ASSERT_EQ(0, ioctx.hit_set_list(hash, c, &ls));
+ c->wait_for_complete();
+ c->release();
+
+ ASSERT_TRUE(ls.size() <= count + 1);
+ cout << " got ls " << ls << std::endl;
+ if (!ls.empty()) {
+ if (!first) {
+ first = ls.front().first;
+ cout << "first is " << first << std::endl;
+ } else {
+ if (ls.front().first != first) {
+ cout << "first now " << ls.front().first << ", trimmed" << std::endl;
+ break;
+ }
+ }
+ }
+
+ utime_t now = ceph_clock_now(NULL);
+ ASSERT_TRUE(now < hard_stop);
+
+ sleep(1);
+ }
+
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+
+int main(int argc, char **argv)
+{
+ ::testing::InitGoogleTest(&argc, argv);
+
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ return RUN_ALL_TESTS();
+}
diff --git a/src/test/mon/moncap.cc b/src/test/mon/moncap.cc
index 238442b..d381319 100644
--- a/src/test/mon/moncap.cc
+++ b/src/test/mon/moncap.cc
@@ -29,7 +29,8 @@ const char *parse_good[] = {
" allow rwx",
"allow rwx ",
" allow rwx ",
- " allow rwx ",
+ " allow\t rwx ",
+ "\tallow\nrwx\t",
"allow service=foo x",
"allow service=\"froo\" x",
"allow profile osd",
diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc
index e5bde68..138bdc0 100644
--- a/src/test/mon/test_mon_workloadgen.cc
+++ b/src/test/mon/test_mon_workloadgen.cc
@@ -11,6 +11,16 @@
* Foundation. See file COPYING.
*
*/
+#include "acconfig.h"
+
+#ifdef HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
#include <iostream>
#include <string>
#include <map>
@@ -521,7 +531,13 @@ class OSDStub : public TestStub
void update_osd_stat() {
struct statfs stbuf;
- statfs(".", &stbuf);
+ int ret = statfs(".", &stbuf);
+ if (ret < 0) {
+ ret = -errno;
+ dout(0) << __func__
+ << " cannot statfs ." << cpp_strerror(ret) << dendl;
+ return;
+ }
osd_stat.kb = stbuf.f_blocks * stbuf.f_bsize / 1024;
osd_stat.kb_used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize / 1024;
diff --git a/src/test/on_exit.cc b/src/test/on_exit.cc
new file mode 100644
index 0000000..26f357e
--- /dev/null
+++ b/src/test/on_exit.cc
@@ -0,0 +1,113 @@
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include "include/on_exit.h"
+
+#ifndef MAP_ANONYMOUS
+# ifdef MAP_ANON
+# define MAP_ANONYMOUS MAP_ANON
+# else
+# error "Don't know how to create anonymous mmap"
+# endif
+#endif
+
+static int func_scope_val;
+
+static void add(void *incp)
+{
+ func_scope_val += *((int*)incp);
+}
+
+static void func_scope(void)
+{
+ OnExitManager mgr;
+
+ int *inc_1 = (int*)malloc(sizeof(*inc_1));
+ *inc_1 = 5;
+ mgr.add_callback(add, inc_1);
+
+ int *inc_2 = (int*)malloc(sizeof(*inc_2));
+ *inc_2 = 3;
+ mgr.add_callback(add, inc_2);
+}
+
+// shared between processes
+static int *shared_val;
+
+#define MAIN_SCOPE_VAL 0x1111111
+static OnExitManager main_scope_mgr;
+static void main_scope_cb(void *val)
+{
+ *shared_val = *((int*)val);
+}
+
+#define EXIT_FUNC_VAL 0x22222222
+static OnExitManager exit_func_mgr;
+static void exit_func_cb(void *val)
+{
+ *shared_val = *((int*)val);
+}
+
+static void call_exit()
+{
+ exit(3);
+}
+
+int main(int argc, char **argv)
+{
+ // test basic function scope behavior
+ assert(func_scope_val == 0);
+ func_scope();
+ assert(func_scope_val == 8);
+
+ // shared mem for exit tests
+ shared_val = (int*)mmap(NULL, sizeof(int),
+ PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, 0, 0);
+ assert(shared_val != MAP_FAILED);
+
+ // test normal exit returning from main
+ *shared_val = 0;
+ int pid = fork();
+ assert(pid >= 0);
+ if (pid) {
+ int status;
+ int ret = waitpid(pid, &status, 0);
+ assert(ret == pid); // should be our child
+ assert(status == 0);
+ assert(*shared_val == MAIN_SCOPE_VAL);
+ } else {
+ // child adds a callback to the static scope callback manager and then
+ // exits by returning from main. The parent checks the value after the
+ // child exits via the memory map.
+ assert(*shared_val == 0);
+ int *new_val = (int*)malloc(sizeof(*new_val));
+ *new_val = MAIN_SCOPE_VAL;
+ main_scope_mgr.add_callback(main_scope_cb, new_val);
+ return 0;
+ }
+
+ // test exit via exit()
+ *shared_val = 0;
+ pid = fork();
+ assert(pid >= 0);
+ if (pid) {
+ int status;
+ int ret = waitpid(pid, &status, 0);
+ assert(ret == pid); // should be our child
+ assert(WEXITSTATUS(status) == 3);
+ assert(*shared_val == EXIT_FUNC_VAL);
+ } else {
+ // child adds a callback to the static scope callback manager and then
+ // exits via exit().
+ assert(*shared_val == 0);
+ int *new_val = (int*)malloc(sizeof(*new_val));
+ *new_val = EXIT_FUNC_VAL;
+ exit_func_mgr.add_callback(exit_func_cb, new_val);
+ call_exit();
+ assert(0);
+ }
+
+ return 0;
+}
diff --git a/src/test/os/TestFlatIndex.cc b/src/test/os/TestFlatIndex.cc
index 53d2bbe..7bb2d42 100644
--- a/src/test/os/TestFlatIndex.cc
+++ b/src/test/os/TestFlatIndex.cc
@@ -41,6 +41,7 @@ TEST(FlatIndex, FlatIndex) {
EXPECT_EQ(0, index.cleanup());
}
+#ifdef GTEST_HAS_DEATH_TEST
TEST(FlatIndex, collection) {
coll_t collection("ABC");
const std::string base_path("PATH");
@@ -53,13 +54,14 @@ TEST(FlatIndex, collection) {
vector<ghobject_t> ls;
ASSERT_DEATH(index.collection_list_partial(hoid, 0, 0, 0, &ls, &hoid), "0");
}
+#endif //GTEST_HAS_DEATH_TEST
TEST(FlatIndex, created_unlink) {
coll_t collection("ABC");
const std::string base_path("PATH");
EXPECT_EQ(0, ::system("rm -fr PATH"));
EXPECT_EQ(0, ::mkdir("PATH", 0700));
- std::tr1::shared_ptr<CollectionIndex> index(new FlatIndex(collection, base_path));
+ ceph::shared_ptr<CollectionIndex> index(new FlatIndex(collection, base_path));
const std::string key("KEY");
uint64_t hash = 111;
uint64_t pool = 222;
@@ -109,7 +111,7 @@ TEST(FlatIndex, collection_list) {
const std::string object_name("ABC");
const std::string filename("PATH/" + object_name + "_head");
EXPECT_EQ(0, ::close(::creat(filename.c_str(), 0600)));
- std::tr1::shared_ptr<CollectionIndex> index(new FlatIndex(collection, base_path));
+ ceph::shared_ptr<CollectionIndex> index(new FlatIndex(collection, base_path));
vector<ghobject_t> ls;
index->collection_list(&ls);
EXPECT_EQ((unsigned)1, ls.size());
diff --git a/src/test/os/TestLFNIndex.cc b/src/test/os/TestLFNIndex.cc
index 02578eb..74fdf66 100644
--- a/src/test/os/TestLFNIndex.cc
+++ b/src/test/os/TestLFNIndex.cc
@@ -42,7 +42,7 @@ public:
virtual int _split(
uint32_t match,
uint32_t bits,
- std::tr1::shared_ptr<CollectionIndex> dest
+ ceph::shared_ptr<CollectionIndex> dest
) { return 0; }
void test_generate_and_parse(const ghobject_t &hoid, const std::string &mangled_expected) {
diff --git a/src/test/osd/ErasureCodeExample.h b/src/test/osd/ErasureCodeExample.h
index 0fd5518..ab2bcf0 100644
--- a/src/test/osd/ErasureCodeExample.h
+++ b/src/test/osd/ErasureCodeExample.h
@@ -76,11 +76,23 @@ public:
set <int> available_chunks;
for (map<int, int>::const_iterator i = c2c.begin();
i != c2c.end();
- i++)
+ ++i)
available_chunks.insert(i->first);
return minimum_to_decode(want_to_read, available_chunks, minimum);
}
+ virtual unsigned int get_chunk_count() const {
+ return DATA_CHUNKS + CODING_CHUNKS;
+ }
+
+ virtual unsigned int get_data_chunk_count() const {
+ return DATA_CHUNKS;
+ }
+
+ virtual unsigned int get_chunk_size(unsigned int object_size) const {
+ return ( object_size / DATA_CHUNKS ) + 1;
+ }
+
virtual int encode(const set<int> &want_to_encode,
const bufferlist &in,
map<int, bufferlist> *encoded) {
@@ -88,11 +100,11 @@ public:
// make sure all data chunks have the same length, allocating
// padding if necessary.
//
- unsigned chunk_length = ( in.length() / DATA_CHUNKS ) + 1;
- unsigned length = chunk_length * ( DATA_CHUNKS + CODING_CHUNKS );
+ unsigned int chunk_length = get_chunk_size(in.length());
bufferlist out(in);
- bufferptr pad(length - in.length());
- pad.zero(0, DATA_CHUNKS);
+ unsigned int width = get_chunk_count() * get_chunk_size(in.length());
+ bufferptr pad(width - in.length());
+ pad.zero(0, get_data_chunk_count());
out.push_back(pad);
//
// compute the coding chunk with first chunk ^ second chunk
@@ -109,7 +121,7 @@ public:
const bufferptr ptr = out.buffers().front();
for (set<int>::iterator j = want_to_encode.begin();
j != want_to_encode.end();
- j++) {
+ ++j) {
bufferptr chunk(ptr, (*j) * chunk_length, chunk_length);
(*encoded)[*j].push_front(chunk);
}
@@ -125,7 +137,7 @@ public:
unsigned chunk_length = (*chunks.begin()).second.length();
for (set<int>::iterator i = want_to_read.begin();
i != want_to_read.end();
- i++) {
+ ++i) {
if (chunks.find(*i) != chunks.end()) {
//
// If the chunk is available, just copy the bufferptr pointer
@@ -143,13 +155,14 @@ public:
// No matter what the missing chunk is, XOR of the other
// two recovers it.
//
- bufferptr chunk(chunk_length);
map<int, bufferlist>::const_iterator k = chunks.begin();
const char *a = k->second.buffers().front().c_str();
- k++;
+ ++k;
const char *b = k->second.buffers().front().c_str();
+ bufferptr chunk(chunk_length);
+ char *c = chunk.c_str();
for (unsigned j = 0; j < chunk_length; j++) {
- chunk[j] = a[j] ^ b[j];
+ c[j] = a[j] ^ b[j];
}
(*decoded)[*i].push_front(chunk);
}
diff --git a/src/test/osd/ErasureCodePluginMissingEntryPoint.cc b/src/test/osd/ErasureCodePluginMissingEntryPoint.cc
index fc60f86..4d8f1ba 100644
--- a/src/test/osd/ErasureCodePluginMissingEntryPoint.cc
+++ b/src/test/osd/ErasureCodePluginMissingEntryPoint.cc
@@ -1 +1,4 @@
// missing int __erasure_code_init(char *plugin_name) {}
+
+// avoid warnings about library containing no symbols
+int __this_is_an_used_variable_to_avoid_warnings;
diff --git a/src/test/osd/Object.cc b/src/test/osd/Object.cc
index d4be4df..279ae23 100644
--- a/src/test/osd/Object.cc
+++ b/src/test/osd/Object.cc
@@ -7,6 +7,28 @@
#include "Object.h"
+void ContDesc::encode(bufferlist &bl) const
+{
+ ENCODE_START(1, 1, bl);
+ ::encode(objnum, bl);
+ ::encode(cursnap, bl);
+ ::encode(seqnum, bl);
+ ::encode(prefix, bl);
+ ::encode(oid, bl);
+ ENCODE_FINISH(bl);
+}
+
+void ContDesc::decode(bufferlist::iterator &bl)
+{
+ DECODE_START(1, bl);
+ ::decode(objnum, bl);
+ ::decode(cursnap, bl);
+ ::decode(seqnum, bl);
+ ::decode(prefix, bl);
+ ::decode(oid, bl);
+ DECODE_FINISH(bl);
+}
+
ostream &operator<<(ostream &out, const ContDesc &rhs)
{
return out << "(ObjNum " << rhs.objnum
@@ -16,67 +38,50 @@ ostream &operator<<(ostream &out, const ContDesc &rhs)
<< ")";
}
-void VarLenGenerator::get_ranges(const ContDesc &cont, interval_set<uint64_t> &out) {
+void AppendGenerator::get_ranges_map(
+ const ContDesc &cont, map<uint64_t, uint64_t> &out) {
+ RandWrap rand(cont.seqnum);
+ uint64_t pos = off;
+ uint64_t limit = off + get_append_size(cont);
+ while (pos < limit) {
+ uint64_t segment_length = (
+ rand() % (max_append_size - min_append_size)) + min_append_size;
+ assert(segment_length < max_append_size);
+ assert(segment_length >= min_append_size);
+ if (segment_length + pos > limit) {
+ segment_length = limit - pos;
+ }
+ out.insert(make_pair(pos, segment_length));
+ pos += segment_length;
+ }
+}
+
+void VarLenGenerator::get_ranges_map(
+ const ContDesc &cont, map<uint64_t, uint64_t> &out) {
RandWrap rand(cont.seqnum);
- uint64_t pos = get_header_length(cont);
+ uint64_t pos = 0;
uint64_t limit = get_length(cont);
- out.insert(0, pos);
bool include = false;
while (pos < limit) {
uint64_t segment_length = (rand() % (max_stride_size - min_stride_size)) + min_stride_size;
assert(segment_length < max_stride_size);
assert(segment_length >= min_stride_size);
- if (segment_length + pos >= limit) {
+ if (segment_length + pos > limit) {
segment_length = limit - pos;
}
if (include) {
- out.insert(pos, segment_length);
+ out.insert(make_pair(pos, segment_length));
include = false;
} else {
include = true;
}
pos += segment_length;
}
-}
-
-void VarLenGenerator::write_header(const ContDesc &in, bufferlist &output) {
- int data[6];
- data[0] = 0xDEADBEEF;
- data[1] = in.objnum;
- data[2] = in.cursnap;
- data[3] = (int)in.seqnum;
- data[4] = in.prefix.size();
- data[5] = 0xDEADBEEF;
- output.append((char *)data, sizeof(data));
- output.append(in.prefix.c_str(), in.prefix.size());
- output.append((char *)data, sizeof(data[0]));
-}
-
-bool VarLenGenerator::read_header(bufferlist::iterator &p, ContDesc &out) {
- try {
- int data[6];
- p.copy(sizeof(data), (char *)data);
- if ((unsigned)data[0] != 0xDEADBEEF || (unsigned)data[5] != 0xDEADBEEF) return false;
- out.objnum = data[1];
- out.cursnap = data[2];
- out.seqnum = (unsigned) data[3];
- int prefix_size = data[4];
- if (prefix_size >= 1000 || prefix_size <= 0) {
- std::cerr << "prefix size is " << prefix_size << std::endl;
- return false;
- }
- char buffer[1000];
- p.copy(prefix_size, buffer);
- buffer[prefix_size] = 0;
- out.prefix = buffer;
- unsigned test;
- p.copy(sizeof(test), (char *)&test);
- if (test != 0xDEADBEEF) return false;
- } catch (ceph::buffer::end_of_buffer &e) {
- std::cerr << "end_of_buffer" << endl;
- return false;
- }
- return true;
+ // make sure we write up to the limit
+ if (limit > 0 && (
+ out.empty() ||
+ (out.rbegin()->first + out.rbegin()->second < limit)))
+ out[limit-1] = 1;
}
ObjectDesc::iterator &ObjectDesc::iterator::advance(bool init) {
@@ -89,9 +94,9 @@ ObjectDesc::iterator &ObjectDesc::iterator::advance(bool init) {
return *this;
}
while (pos == limit) {
- limit = *stack.begin();
+ cur_cont = stack.begin()->first;
+ limit = stack.begin()->second;
stack.pop_front();
- --cur_cont;
}
if (cur_cont == obj.layers.end()) {
@@ -99,65 +104,55 @@ ObjectDesc::iterator &ObjectDesc::iterator::advance(bool init) {
}
interval_set<uint64_t> ranges;
- cont_gen->get_ranges(*cur_cont, ranges);
+ cur_cont->first->get_ranges(cur_cont->second, ranges);
while (!ranges.contains(pos)) {
- stack.push_front(limit);
+ stack.push_front(make_pair(cur_cont, limit));
+ uint64_t length = cur_cont->first->get_length(cur_cont->second);
uint64_t next;
- if (pos >= ranges.range_end()) {
+ if (pos >= length) {
next = limit;
+ cur_cont = obj.layers.end();
+ } else if (ranges.empty() || pos >= ranges.range_end()) {
+ next = length;
+ ++cur_cont;
} else {
next = ranges.start_after(pos);
+ ++cur_cont;
}
if (next < limit) {
limit = next;
}
- ++cur_cont;
if (cur_cont == obj.layers.end()) {
break;
}
ranges.clear();
- cont_gen->get_ranges(*cur_cont, ranges);
+ cur_cont->first->get_ranges(cur_cont->second, ranges);
}
if (cur_cont == obj.layers.end()) {
return *this;
}
- if (!cont_iters.count(*cur_cont)) {
- cont_iters.insert(pair<ContDesc,ContentsGenerator::iterator>(*cur_cont,
- cont_gen->get_iterator(*cur_cont)));
+ if (!cont_iters.count(cur_cont->second)) {
+ cont_iters.insert(pair<ContDesc,ContentsGenerator::iterator>(
+ cur_cont->second,
+ cur_cont->first->get_iterator(cur_cont->second)));
}
- map<ContDesc,ContentsGenerator::iterator>::iterator j = cont_iters.find(*cur_cont);
+ map<ContDesc,ContentsGenerator::iterator>::iterator j = cont_iters.find(
+ cur_cont->second);
assert(j != cont_iters.end());
j->second.seek(pos);
return *this;
}
const ContDesc &ObjectDesc::most_recent() {
- return *layers.begin();
+ return layers.begin()->second;
}
-void ObjectDesc::update(const ContDesc &next) {
- layers.push_front(next);
+void ObjectDesc::update(ContentsGenerator *gen, const ContDesc &next) {
+ layers.push_front(make_pair(gen, next));
return;
- /*
- interval_set<uint64_t> fall_through;
- fall_through.insert(0, cont_gen->get_length(next));
- for (list<ContDesc>::iterator i = layers.begin();
- i != layers.end();
- ) {
- interval_set<uint64_t> valid;
- cont_gen->get_ranges(*i, valid);
- valid.intersection_of(fall_through);
- if (valid.empty()) {
- layers.erase(i++);
- continue;
- }
- fall_through.subtract(valid);
- ++i;
- }
- */
}
bool ObjectDesc::check(bufferlist &to_check) {
@@ -175,5 +170,11 @@ bool ObjectDesc::check(bufferlist &to_check) {
return false;
}
}
+ uint64_t size = layers.empty() ? 0 :
+ most_recent_gen()->get_length(most_recent());
+ if (pos != size) {
+ std::cout << "only read " << pos << " out of size " << size << std::endl;
+ return false;
+ }
return true;
}
diff --git a/src/test/osd/Object.h b/src/test/osd/Object.h
index 09f0a5f..1f4defd 100644
--- a/src/test/osd/Object.h
+++ b/src/test/osd/Object.h
@@ -1,6 +1,7 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
#include "include/interval_set.h"
#include "include/buffer.h"
+#include "include/encoding.h"
#include <list>
#include <map>
#include <set>
@@ -42,7 +43,10 @@ public:
bool operator!=(const ContDesc &rhs) {
return !((*this) == rhs);
}
+ void encode(bufferlist &bl) const;
+ void decode(bufferlist::iterator &bp);
};
+WRITE_CLASS_ENCODER(ContDesc)
ostream &operator<<(ostream &out, const ContDesc &rhs);
@@ -90,16 +94,20 @@ public:
parent(parent), impl(impl) {}
};
- virtual bool read_header(bufferlist::iterator& input,
- ContDesc &output) = 0;
-
virtual uint64_t get_length(const ContDesc &in) = 0;
- virtual uint64_t get_attr_length(const ContDesc &in) = 0;
-
- virtual bufferlist gen_attribute(const ContDesc &in) = 0;
+ virtual void get_ranges_map(
+ const ContDesc &cont, map<uint64_t, uint64_t> &out) = 0;
+ void get_ranges(const ContDesc &cont, interval_set<uint64_t> &out) {
+ map<uint64_t, uint64_t> ranges;
+ get_ranges_map(cont, ranges);
+ for (map<uint64_t, uint64_t>::iterator i = ranges.begin();
+ i != ranges.end();
+ ++i) {
+ out.insert(i->first, i->second);
+ }
+ }
- virtual void get_ranges(const ContDesc &in, interval_set<uint64_t> &ranges) = 0;
virtual iterator_impl *get_iterator_impl(const ContDesc &in) = 0;
@@ -114,7 +122,7 @@ public:
}
};
-class VarLenGenerator : public ContentsGenerator {
+class RandGenerator : public ContentsGenerator {
public:
class RandWrap {
public:
@@ -135,29 +143,19 @@ public:
uint64_t pos;
ContDesc cont;
RandWrap rand;
- bufferlist header;
- bufferlist::iterator header_pos;
- VarLenGenerator *cont_gen;
+ RandGenerator *cont_gen;
char current;
- iterator_impl(const ContDesc &cont, VarLenGenerator *cont_gen) :
+ iterator_impl(const ContDesc &cont, RandGenerator *cont_gen) :
pos(0), cont(cont), rand(cont.seqnum), cont_gen(cont_gen) {
- cont_gen->write_header(cont, header);
- header_pos = header.begin();
- current = *header_pos;
- ++header_pos;
+ current = rand();
}
- virtual ContDesc get_cont() const { return cont; }
- virtual uint64_t get_pos() const { return pos; }
+ ContDesc get_cont() const { return cont; }
+ uint64_t get_pos() const { return pos; }
iterator_impl &operator++() {
pos++;
- if (header_pos.end()) {
- current = rand();
- } else {
- current = *header_pos;
- ++header_pos;
- }
+ current = rand();
return *this;
}
@@ -181,10 +179,8 @@ public:
}
};
- void get_ranges(const ContDesc &cont, interval_set<uint64_t> &out);
-
ContentsGenerator::iterator_impl *get_iterator_impl(const ContDesc &in) {
- VarLenGenerator::iterator_impl *i = new iterator_impl(in, this);
+ RandGenerator::iterator_impl *i = new iterator_impl(in, this);
return i;
}
@@ -192,73 +188,105 @@ public:
delete in;
}
- ContentsGenerator::iterator_impl *dup_iterator_impl(const ContentsGenerator::iterator_impl *in) {
+ ContentsGenerator::iterator_impl *dup_iterator_impl(
+ const ContentsGenerator::iterator_impl *in) {
ContentsGenerator::iterator_impl *retval = get_iterator_impl(in->get_cont());
retval->seek(in->get_pos());
return retval;
}
+};
- int get_header_length(const ContDesc &in) {
- return 7*sizeof(int) + in.prefix.size();
+class VarLenGenerator : public RandGenerator {
+ uint64_t max_length;
+ uint64_t min_stride_size;
+ uint64_t max_stride_size;
+public:
+ VarLenGenerator(
+ uint64_t length, uint64_t min_stride_size, uint64_t max_stride_size) :
+ max_length(length),
+ min_stride_size(min_stride_size),
+ max_stride_size(max_stride_size) {}
+ void get_ranges_map(
+ const ContDesc &cont, map<uint64_t, uint64_t> &out);
+ uint64_t get_length(const ContDesc &in) {
+ RandWrap rand(in.seqnum);
+ return (rand() % max_length);
}
+};
+class AttrGenerator : public RandGenerator {
+ uint64_t max_len;
+public:
+ AttrGenerator(uint64_t max_len) : max_len(max_len) {}
+ void get_ranges_map(
+ const ContDesc &cont, map<uint64_t, uint64_t> &out) {
+ out.insert(make_pair(0, get_length(cont)));
+ }
uint64_t get_length(const ContDesc &in) {
RandWrap rand(in.seqnum);
- return (rand() % length) + get_header_length(in);
+ return (rand() % max_len);
}
-
- bufferlist gen_attribute(const ContDesc &in) {
- bufferlist header;
- write_header(in, header);
- ContentsGenerator::iterator iter = get_iterator(in);
- for (uint64_t to_write = get_attr_length(in); to_write > 0;
- --to_write) {
- header.append(*iter);
- ++iter;
+ bufferlist gen_bl(const ContDesc &in) {
+ bufferlist bl;
+ for (iterator i = get_iterator(in); !i.end(); ++i) {
+ bl.append(*i);
}
- return header;
+ assert(bl.length() < max_len);
+ return bl;
}
+};
-
- uint64_t get_attr_length(const ContDesc &in) {
+class AppendGenerator : public RandGenerator {
+ uint64_t off;
+ uint64_t min_append_size;
+ uint64_t max_append_size;
+ uint64_t max_append_total;
+public:
+ AppendGenerator(
+ uint64_t off,
+ uint64_t min_append_size,
+ uint64_t max_append_size,
+ uint64_t max_append_total) :
+ off(off), min_append_size(min_append_size),
+ max_append_size(max_append_size),
+ max_append_total(max_append_total) {}
+ uint64_t get_append_size(const ContDesc &in) {
RandWrap rand(in.seqnum);
- return (rand() % attr_length) + get_header_length(in);
+ return rand() % max_append_total;
}
-
- void write_header(const ContDesc &in, bufferlist &output);
-
- bool read_header(bufferlist::iterator &p, ContDesc &out);
- uint64_t length;
- uint64_t attr_length;
- uint64_t min_stride_size;
- uint64_t max_stride_size;
- VarLenGenerator(uint64_t length, uint64_t min_stride_size, uint64_t max_stride_size, uint64_t attr_length = 2000) :
- length(length), attr_length(attr_length),
- min_stride_size(min_stride_size), max_stride_size(max_stride_size) {}
+ uint64_t get_length(const ContDesc &in) {
+ return off + get_append_size(in);
+ }
+ void get_ranges_map(
+ const ContDesc &cont, map<uint64_t, uint64_t> &out);
};
class ObjectDesc {
public:
- ObjectDesc(ContentsGenerator *cont_gen) :
- exists(false), tmap(false), version(0), layers(), cont_gen(cont_gen) {};
- ObjectDesc(const ContDesc &init, ContentsGenerator *cont_gen) :
- exists(false), tmap(false), version(0), layers(), cont_gen(cont_gen) {
- layers.push_front(init);
- };
+ ObjectDesc()
+ : exists(false), dirty(false),
+ version(0) {}
+ ObjectDesc(const ContDesc &init, ContentsGenerator *cont_gen)
+ : exists(false), dirty(false),
+ version(0) {
+ layers.push_front(make_pair(cont_gen, init));
+ }
class iterator {
public:
uint64_t pos;
ObjectDesc &obj;
- ContentsGenerator *cont_gen;
- list<uint64_t> stack;
+ list<pair<list<pair<ceph::shared_ptr<ContentsGenerator>,
+ ContDesc> >::iterator,
+ uint64_t> > stack;
map<ContDesc,ContentsGenerator::iterator> cont_iters;
uint64_t limit;
- list<ContDesc>::iterator cur_cont;
+ list<pair<ceph::shared_ptr<ContentsGenerator>,
+ ContDesc> >::iterator cur_cont;
- iterator(ObjectDesc &obj, ContentsGenerator *cont_gen) :
- pos(0), obj(obj), cont_gen(cont_gen) {
- limit = cont_gen->get_length(*obj.layers.begin());
+ iterator(ObjectDesc &obj) :
+ pos(0), obj(obj) {
+ limit = obj.layers.begin()->first->get_length(obj.layers.begin()->second);
cur_cont = obj.layers.begin();
advance(true);
}
@@ -269,19 +297,19 @@ public:
}
char operator*() {
- if (cur_cont == obj.layers.end() && pos < obj.tmap_contents.length()) {
- return obj.tmap_contents[pos];
- } else if (cur_cont == obj.layers.end()) {
+ if (cur_cont == obj.layers.end()) {
return '\0';
} else {
- map<ContDesc,ContentsGenerator::iterator>::iterator j = cont_iters.find(*cur_cont);
+ map<ContDesc,ContentsGenerator::iterator>::iterator j = cont_iters.find(
+ cur_cont->second);
assert(j != cont_iters.end());
return *(j->second);
}
}
bool end() {
- return pos >= cont_gen->get_length(*obj.layers.begin());
+ return pos >= obj.layers.begin()->first->get_length(
+ obj.layers.begin()->second);
}
void seek(uint64_t _pos) {
@@ -293,9 +321,9 @@ public:
}
}
};
-
+
iterator begin() {
- return iterator(*this, this->cont_gen);
+ return iterator(*this);
}
bool deleted() {
@@ -306,19 +334,21 @@ public:
return layers.size();
}
- void update(const ContDesc &next);
+ // takes ownership of gen
+ void update(ContentsGenerator *gen, const ContDesc &next);
bool check(bufferlist &to_check);
const ContDesc &most_recent();
+ ContentsGenerator *most_recent_gen() {
+ return layers.begin()->first.get();
+ }
map<string, ContDesc> attrs; // Both omap and xattrs
bufferlist header;
bool exists;
- bool tmap;
- bufferlist tmap_contents;
+ bool dirty;
+
uint64_t version;
private:
- list<ContDesc> layers;
- ContentsGenerator *cont_gen;
- ObjectDesc();
+ list<pair<ceph::shared_ptr<ContentsGenerator>, ContDesc> > layers;
};
#endif
diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h
index 80bcf00..4a5b861 100644
--- a/src/test/osd/RadosModel.h
+++ b/src/test/osd/RadosModel.h
@@ -18,7 +18,10 @@
#include "Object.h"
#include "TestOpStat.h"
#include "test/librados/test.h"
+#include "include/memory.h"
#include "common/sharedptr_registry.hpp"
+#include "common/errno.h"
+#include "osd/HitSet.h"
#ifndef RADOSMODEL_H
#define RADOSMODEL_H
@@ -49,9 +52,15 @@ enum TestOpType {
TEST_OP_ROLLBACK,
TEST_OP_SETATTR,
TEST_OP_RMATTR,
- TEST_OP_TMAPPUT,
TEST_OP_WATCH,
- TEST_OP_COPY_FROM
+ TEST_OP_COPY_FROM,
+ TEST_OP_HIT_SET_LIST,
+ TEST_OP_UNDIRTY,
+ TEST_OP_IS_DIRTY,
+ TEST_OP_CACHE_FLUSH,
+ TEST_OP_CACHE_TRY_FLUSH,
+ TEST_OP_CACHE_EVICT,
+ TEST_OP_APPEND
};
class TestWatchContext : public librados::WatchCtx {
@@ -144,6 +153,8 @@ public:
map<int, map<string,ObjectDesc> > pool_obj_cont;
set<string> oid_in_use;
set<string> oid_not_in_use;
+ set<string> oid_flushing;
+ set<string> oid_not_flushing;
SharedPtrRegistry<int, int> snaps_in_use;
int current_snap;
string pool_name;
@@ -153,18 +164,24 @@ public:
string prefix;
int errors;
int max_in_flight;
- ContentsGenerator &cont_gen;
int seq_num;
map<int,uint64_t> snaps;
uint64_t seq;
const char *rados_id;
bool initialized;
map<string, TestWatchContext*> watches;
-
+ const uint64_t max_size;
+ const uint64_t min_stride_size;
+ const uint64_t max_stride_size;
+ AttrGenerator attr_gen;
+ const bool ec_pool;
RadosTestContext(const string &pool_name,
int max_in_flight,
- ContentsGenerator &cont_gen,
+ uint64_t max_size,
+ uint64_t min_stride_size,
+ uint64_t max_stride_size,
+ bool ec_pool,
const char *id = 0) :
state_lock("Context Lock"),
pool_obj_cont(),
@@ -173,8 +190,12 @@ public:
next_oid(0),
errors(0),
max_in_flight(max_in_flight),
- cont_gen(cont_gen), seq_num(0), seq(0),
- rados_id(id), initialized(false)
+ seq_num(0), seq(0),
+ rados_id(id), initialized(false),
+ max_size(max_size),
+ min_stride_size(min_stride_size), max_stride_size(max_stride_size),
+ attr_gen(2000),
+ ec_pool(ec_pool)
{
}
@@ -279,9 +300,8 @@ public:
watches.erase(oid);
}
- void rm_object_attrs(const string &oid, const set<string> &attrs)
- {
- ObjectDesc new_obj(&cont_gen);
+ ObjectDesc get_most_recent(const string &oid) {
+ ObjectDesc new_obj;
for (map<int, map<string,ObjectDesc> >::reverse_iterator i =
pool_obj_cont.rbegin();
i != pool_obj_cont.rend();
@@ -292,30 +312,27 @@ public:
break;
}
}
+ return new_obj;
+ }
+
+ void rm_object_attrs(const string &oid, const set<string> &attrs)
+ {
+ ObjectDesc new_obj = get_most_recent(oid);
for (set<string>::const_iterator i = attrs.begin();
i != attrs.end();
++i) {
new_obj.attrs.erase(*i);
}
- new_obj.tmap = false;
+ new_obj.dirty = true;
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
}
void remove_object_header(const string &oid)
{
- ObjectDesc new_obj(&cont_gen);
- for (map<int, map<string,ObjectDesc> >::reverse_iterator i =
- pool_obj_cont.rbegin();
- i != pool_obj_cont.rend();
- ++i) {
- map<string,ObjectDesc>::iterator j = i->second.find(oid);
- if (j != i->second.end()) {
- new_obj = j->second;
- break;
- }
- }
+ ObjectDesc new_obj = get_most_recent(oid);
new_obj.header = bufferlist();
+ new_obj.dirty = true;
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
}
@@ -323,82 +340,36 @@ public:
void update_object_header(const string &oid, const bufferlist &bl)
{
- ObjectDesc new_obj(&cont_gen);
- for (map<int, map<string,ObjectDesc> >::reverse_iterator i =
- pool_obj_cont.rbegin();
- i != pool_obj_cont.rend();
- ++i) {
- map<string,ObjectDesc>::iterator j = i->second.find(oid);
- if (j != i->second.end()) {
- new_obj = j->second;
- break;
- }
- }
+ ObjectDesc new_obj = get_most_recent(oid);
new_obj.header = bl;
- new_obj.tmap = false;
new_obj.exists = true;
+ new_obj.dirty = true;
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
}
- void set_object_tmap(const string &oid, const map<string, ContDesc> &attrs,
- bufferlist header,
- bufferlist tmap_contents)
- {
- ObjectDesc new_obj(&cont_gen);
- pool_obj_cont[current_snap].erase(oid);
- new_obj.attrs = attrs;
- new_obj.tmap_contents = tmap_contents;
- new_obj.header = header;
- new_obj.tmap = true;
- new_obj.exists = true;
- pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
- }
-
void update_object_attrs(const string &oid, const map<string, ContDesc> &attrs)
{
- ObjectDesc new_obj(&cont_gen);
- for (map<int, map<string,ObjectDesc> >::reverse_iterator i =
- pool_obj_cont.rbegin();
- i != pool_obj_cont.rend();
- ++i) {
- map<string,ObjectDesc>::iterator j = i->second.find(oid);
- if (j != i->second.end()) {
- new_obj = j->second;
- break;
- }
- }
+ ObjectDesc new_obj = get_most_recent(oid);
for (map<string, ContDesc>::const_iterator i = attrs.begin();
i != attrs.end();
++i) {
new_obj.attrs[i->first] = i->second;
}
- new_obj.tmap = false;
new_obj.exists = true;
+ new_obj.dirty = true;
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
}
- void update_object(const string &oid, const ContDesc &contents)
+ void update_object(ContentsGenerator *cont_gen,
+ const string &oid, const ContDesc &contents)
{
- ObjectDesc new_obj(&cont_gen);
- for (map<int, map<string,ObjectDesc> >::reverse_iterator i =
- pool_obj_cont.rbegin();
- i != pool_obj_cont.rend();
- ++i) {
- map<string,ObjectDesc>::iterator j = i->second.find(oid);
- if (j != i->second.end()) {
- new_obj = j->second;
- break;
- }
- }
+ ObjectDesc new_obj = get_most_recent(oid);
new_obj.exists = true;
- if (new_obj.tmap) {
- new_obj.tmap = false;
- new_obj.attrs.clear();
- new_obj.header = bufferlist();
- }
- new_obj.update(contents);
+ new_obj.dirty = true;
+ new_obj.update(cont_gen,
+ contents);
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
}
@@ -409,16 +380,32 @@ public:
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, contents));
}
- void update_object_version(const string &oid, uint64_t version)
+ void update_object_undirty(const string &oid)
+ {
+ ObjectDesc new_obj = get_most_recent(oid);
+ new_obj.dirty = false;
+ pool_obj_cont[current_snap].erase(oid);
+ pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
+ }
+
+ void update_object_version(const string &oid, uint64_t version,
+ int snap = -1)
{
for (map<int, map<string,ObjectDesc> >::reverse_iterator i =
pool_obj_cont.rbegin();
i != pool_obj_cont.rend();
++i) {
+ if (snap != -1 && snap < i->first)
+ continue;
map<string,ObjectDesc>::iterator j = i->second.find(oid);
if (j != i->second.end()) {
- j->second.version = version;
- cout << __func__ << " oid " << oid << " v " << version << " " << j->second.most_recent() << std::endl;
+ if (version)
+ j->second.version = version;
+ cout << __func__ << " oid " << oid
+ << " v " << version << " " << j->second.most_recent()
+ << " " << (j->second.dirty ? "dirty" : "clean")
+ << " " << (j->second.exists ? "exists" : "dne")
+ << std::endl;
break;
}
}
@@ -427,7 +414,7 @@ public:
void remove_object(const string &oid)
{
assert(!get_watch_context(oid));
- ObjectDesc new_obj(&cont_gen);
+ ObjectDesc new_obj;
pool_obj_cont[current_snap].erase(oid);
pool_obj_cont[current_snap].insert(pair<string,ObjectDesc>(oid, new_obj));
}
@@ -477,8 +464,9 @@ public:
void roll_back(const string &oid, int snap)
{
assert(!get_watch_context(oid));
- ObjectDesc contents(&cont_gen);
+ ObjectDesc contents;
find_object(oid, &contents, snap);
+ contents.dirty = true;
pool_obj_cont.rbegin()->second.erase(oid);
pool_obj_cont.rbegin()->second.insert(pair<string,ObjectDesc>(oid, contents));
}
@@ -505,7 +493,7 @@ public:
set<string> to_remove;
{
Mutex::Locker l(context->state_lock);
- ObjectDesc obj(&context->cont_gen);
+ ObjectDesc obj;
if (!context->find_object(oid, &obj)) {
context->kick();
done = true;
@@ -517,7 +505,7 @@ public:
context->oid_not_in_use.erase(oid);
if (rand() % 30) {
- ContentsGenerator::iterator iter = context->cont_gen.get_iterator(cont);
+ ContentsGenerator::iterator iter = context->attr_gen.get_iterator(cont);
for (map<string, ContDesc>::iterator i = obj.attrs.begin();
i != obj.attrs.end();
++i, ++iter) {
@@ -534,9 +522,13 @@ public:
done = true;
return;
}
- op.omap_rm_keys(to_remove);
+ if (!context->ec_pool) {
+ op.omap_rm_keys(to_remove);
+ }
} else {
- op.omap_clear();
+ if (!context->ec_pool) {
+ op.omap_clear();
+ }
for (map<string, ContDesc>::iterator i = obj.attrs.begin();
i != obj.attrs.end();
++i) {
@@ -560,99 +552,6 @@ public:
{
Mutex::Locker l(context->state_lock);
done = true;
- context->oid_in_use.erase(oid);
- context->oid_not_in_use.insert(oid);
- context->kick();
- }
-
- bool finished()
- {
- return done;
- }
-
- string getType()
- {
- return "RemoveAttrsOp";
- }
-};
-
-class TmapPutOp : public TestOp {
-public:
- string oid;
- librados::ObjectWriteOperation op;
- librados::AioCompletion *comp;
- bool done;
- TmapPutOp(int n,
- RadosTestContext *context,
- const string &oid,
- TestOpStat *stat)
- : TestOp(n, context, stat), oid(oid), comp(NULL), done(false)
- {}
-
- void _begin()
- {
- ContDesc cont;
- {
- Mutex::Locker l(context->state_lock);
- cont = ContDesc(context->seq_num, context->current_snap,
- context->seq_num, "");
- context->oid_in_use.insert(oid);
- context->oid_not_in_use.erase(oid);
- }
-
- op.remove();
- map<string, bufferlist> omap_contents;
- map<string, ContDesc> omap;
- bufferlist header;
- ContentsGenerator::iterator keygen = context->cont_gen.get_iterator(cont);
- while (!*keygen) ++keygen;
- while (*keygen) {
- header.append(*keygen);
- ++keygen;
- }
- for (int i = 0; i < 20; ++i) {
- string key;
- while (!*keygen) ++keygen;
- while (*keygen && key.size() < 40) {
- key.push_back((*keygen % 20) + 'a');
- ++keygen;
- }
- ContDesc val(cont);
- val.seqnum += context->cont_gen.get_length(cont);
- val.prefix = ("oid: " + oid);
- omap[key] = val;
- bufferlist val_buffer = context->cont_gen.gen_attribute(val);
- omap_contents[key] = val_buffer;
- op.setxattr(key.c_str(), val_buffer);
- }
-
- bufferlist tmap_contents;
- ::encode(header, tmap_contents);
- ::encode(omap_contents, tmap_contents);
- op.tmap_put(tmap_contents);
- {
- Mutex::Locker l(context->state_lock);
- context->set_object_tmap(oid, omap, header,
- tmap_contents);
- }
-
- pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
- new pair<TestOp*, TestOp::CallbackInfo*>(this,
- new TestOp::CallbackInfo(0));
- comp = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
- NULL);
- context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
- }
-
- void _finish(CallbackInfo *info)
- {
- Mutex::Locker l(context->state_lock);
- int r;
- if ((r = comp->get_return_value())) {
- cerr << "err " << r << std::endl;
- assert(0);
- }
- done = true;
context->update_object_version(oid, comp->get_version64());
context->oid_in_use.erase(oid);
context->oid_not_in_use.insert(oid);
@@ -666,7 +565,7 @@ public:
string getType()
{
- return "TmapPutOp";
+ return "RemoveAttrsOp";
}
};
@@ -698,11 +597,12 @@ public:
map<string, bufferlist> omap_contents;
map<string, ContDesc> omap;
bufferlist header;
- ContentsGenerator::iterator keygen = context->cont_gen.get_iterator(cont);
+ ContentsGenerator::iterator keygen = context->attr_gen.get_iterator(cont);
op.create(false);
while (!*keygen) ++keygen;
while (*keygen) {
- header.append(*keygen);
+ if (*keygen != '_')
+ header.append(*keygen);
++keygen;
}
for (int i = 0; i < 20; ++i) {
@@ -713,15 +613,17 @@ public:
++keygen;
}
ContDesc val(cont);
- val.seqnum += context->cont_gen.get_length(cont);
+ val.seqnum += (unsigned)(*keygen);
val.prefix = ("oid: " + oid);
omap[key] = val;
- bufferlist val_buffer = context->cont_gen.gen_attribute(val);
+ bufferlist val_buffer = context->attr_gen.gen_bl(val);
omap_contents[key] = val_buffer;
op.setxattr(key.c_str(), val_buffer);
}
- op.omap_set_header(header);
- op.omap_set(omap_contents);
+ if (!context->ec_pool) {
+ op.omap_set_header(header);
+ op.omap_set(omap_contents);
+ }
{
Mutex::Locker l(context->state_lock);
@@ -773,14 +675,18 @@ public:
uint64_t last_acked_tid;
librados::ObjectReadOperation read_op;
+ librados::ObjectWriteOperation write_op;
bufferlist rbuffer;
+ bool do_append;
+
WriteOp(int n,
RadosTestContext *context,
const string &oid,
+ bool do_append,
TestOpStat *stat = 0)
: TestOp(n, context, stat),
- oid(oid), waiting_on(0), last_acked_tid(0)
+ oid(oid), waiting_on(0), last_acked_tid(0), do_append(do_append)
{}
void _begin()
@@ -793,47 +699,89 @@ public:
cont = ContDesc(context->seq_num, context->current_snap, context->seq_num, prefix);
- context->update_object(oid, cont);
+ ContentsGenerator *cont_gen;
+ if (do_append) {
+ ObjectDesc old_value;
+ bool found = context->find_object(oid, &old_value);
+ uint64_t prev_length = found && old_value.has_contents() ?
+ old_value.most_recent_gen()->get_length(old_value.most_recent()) :
+ 0;
+ cont_gen = new AppendGenerator(
+ prev_length,
+ context->min_stride_size,
+ context->max_stride_size,
+ 3 * context->max_stride_size);
+ } else {
+ cont_gen = new VarLenGenerator(
+ context->max_size, context->min_stride_size, context->max_stride_size);
+ }
+ context->update_object(cont_gen, oid, cont);
context->oid_in_use.insert(oid);
context->oid_not_in_use.erase(oid);
- interval_set<uint64_t> ranges;
- context->cont_gen.get_ranges(cont, ranges);
+ map<uint64_t, uint64_t> ranges;
+
+ cont_gen->get_ranges_map(cont, ranges);
std::cout << num << ": seq_num " << context->seq_num << " ranges " << ranges << std::endl;
context->seq_num++;
context->state_lock.Unlock();
- waiting_on = ranges.num_intervals();
+ waiting_on = ranges.size();
//cout << " waiting_on = " << waiting_on << std::endl;
- ContentsGenerator::iterator gen_pos = context->cont_gen.get_iterator(cont);
+ ContentsGenerator::iterator gen_pos = cont_gen->get_iterator(cont);
uint64_t tid = 1;
- for (interval_set<uint64_t>::iterator i = ranges.begin();
+ for (map<uint64_t, uint64_t>::iterator i = ranges.begin();
i != ranges.end();
++i, ++tid) {
bufferlist to_write;
- gen_pos.seek(i.get_start());
- for (uint64_t k = 0; k != i.get_len(); ++k, ++gen_pos) {
+ gen_pos.seek(i->first);
+ for (uint64_t k = 0; k != i->second; ++k, ++gen_pos) {
to_write.append(*gen_pos);
}
- assert(to_write.length() == i.get_len());
+ assert(to_write.length() == i->second);
assert(to_write.length() > 0);
- std::cout << num << ": writing " << context->prefix+oid << " from " << i.get_start()
- << " to " << i.get_len() + i.get_start() << " tid " << tid << std::endl;
+ std::cout << num << ": writing " << context->prefix+oid
+ << " from " << i->first
+ << " to " << i->first + i->second << " tid " << tid << std::endl;
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(this,
new TestOp::CallbackInfo(tid));
librados::AioCompletion *completion =
context->rados.aio_create_completion((void*) cb_arg, &write_callback, NULL);
waiting.insert(completion);
- context->io_ctx.aio_write(context->prefix+oid, completion,
- to_write, i.get_len(), i.get_start());
+ librados::ObjectWriteOperation op;
+ if (do_append) {
+ op.append(to_write);
+ } else {
+ op.write(i->first, to_write);
+ }
+ context->io_ctx.aio_operate(
+ context->prefix+oid, completion,
+ &op);
}
+ bufferlist contbl;
+ ::encode(cont, contbl);
pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
new pair<TestOp*, TestOp::CallbackInfo*>(
this,
- new TestOp::CallbackInfo(tid));
+ new TestOp::CallbackInfo(++tid));
+ librados::AioCompletion *completion = context->rados.aio_create_completion(
+ (void*) cb_arg, &write_callback, NULL);
+ waiting.insert(completion);
+ waiting_on++;
+ write_op.setxattr("_header", contbl);
+ if (!do_append) {
+ write_op.truncate(cont_gen->get_length(cont));
+ }
+ context->io_ctx.aio_operate(
+ context->prefix+oid, completion, &write_op);
+
+ cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(
+ this,
+ new TestOp::CallbackInfo(++tid));
rcompletion = context->rados.aio_create_completion(
(void*) cb_arg, &write_callback, NULL);
waiting_on++;
@@ -841,7 +789,6 @@ public:
context->io_ctx.aio_operate(
context->prefix+oid, rcompletion,
&read_op,
- librados::SNAP_HEAD,
librados::OPERATION_ORDER_READS_WRITES, // order wrt previous write/update
0);
}
@@ -926,7 +873,7 @@ public:
return;
}
- ObjectDesc contents(&context->cont_gen);
+ ObjectDesc contents;
context->find_object(oid, &contents);
bool present = !contents.deleted();
@@ -966,6 +913,8 @@ public:
ObjectDesc old_value;
int snap;
+ ceph::shared_ptr<int> in_use;
+
bufferlist result;
int retval;
@@ -986,7 +935,6 @@ public:
: TestOp(n, context, stat),
completion(NULL),
oid(oid),
- old_value(&context->cont_gen),
snap(0),
retval(0),
attrretval(0)
@@ -995,11 +943,13 @@ public:
void _begin()
{
context->state_lock.Lock();
- if (0 && !(rand() % 4) && !context->snaps.empty()) {
+ if (!(rand() % 4) && !context->snaps.empty()) {
snap = rand_choose(context->snaps)->first;
+ in_use = context->snaps_in_use.lookup_or_create(snap, snap);
} else {
snap = -1;
}
+ std::cout << num << ": read oid " << oid << " snap " << snap << std::endl;
done = 0;
completion = context->rados.aio_create_completion((void *) this, &read_callback, 0);
@@ -1029,13 +979,11 @@ public:
context->io_ctx.snap_set_read(context->snaps[snap]);
}
- if (!old_value.tmap) {
- op.read(0,
- !old_value.has_contents() ? 0 :
- context->cont_gen.get_length(old_value.most_recent()),
- &result,
- &retval);
- }
+ op.read(0,
+ !old_value.has_contents() ? 0 :
+ old_value.most_recent_gen()->get_length(old_value.most_recent()),
+ &result,
+ &retval);
for (map<string, ContDesc>::iterator i = old_value.attrs.begin();
i != old_value.attrs.end();
@@ -1047,12 +995,14 @@ public:
omap_requested_keys.insert(key);
}
}
- op.omap_get_vals_by_keys(omap_requested_keys, &omap_returned_values, 0);
+ if (!context->ec_pool) {
+ op.omap_get_vals_by_keys(omap_requested_keys, &omap_returned_values, 0);
- op.omap_get_keys("", -1, &omap_keys, 0);
- op.omap_get_vals("", -1, &omap, 0);
+ op.omap_get_keys("", -1, &omap_keys, 0);
+ op.omap_get_vals("", -1, &omap, 0);
+ op.omap_get_header(&header, 0);
+ }
op.getxattrs(&xattrs, 0);
- op.omap_get_header(&header, 0);
assert(!context->io_ctx.aio_operate(context->prefix+oid, completion, &op, 0));
if (snap >= 0) {
context->io_ctx.snap_set_read(0);
@@ -1071,18 +1021,26 @@ public:
if (!(err == -ENOENT && old_value.deleted())) {
cerr << num << ": Error: oid " << oid << " read returned error code "
<< err << std::endl;
- context->errors++;
+ assert(0);
}
} else {
+ map<string, bufferlist>::iterator iter = xattrs.find("_header");
+ bufferlist headerbl;
+ if (iter == xattrs.end()) {
+ cerr << num << ": Error: did not find header attr, has_contents: "
+ << old_value.has_contents()
+ << std::endl;
+ assert(!old_value.has_contents());
+ } else {
+ headerbl = iter->second;
+ xattrs.erase(iter);
+ }
cout << num << ": expect " << old_value.most_recent() << std::endl;
assert(!old_value.deleted());
if (old_value.has_contents()) {
ContDesc to_check;
- bufferlist::iterator p = result.begin();
- if (!context->cont_gen.read_header(p, to_check)) {
- cerr << num << ": Unable to decode oid " << oid << " at snap " << context->current_snap << std::endl;
- context->errors++;
- }
+ bufferlist::iterator p = headerbl.begin();
+ ::decode(to_check, p);
if (to_check != old_value.most_recent()) {
cerr << num << ": oid " << oid << " found incorrect object contents " << to_check
<< ", expected " << old_value.most_recent() << std::endl;
@@ -1096,21 +1054,23 @@ public:
}
// Attributes
- if (!(old_value.header == header)) {
- cerr << num << ": oid " << oid << " header does not match, old size: "
- << old_value.header.length() << " new size " << header.length()
- << std::endl;
- assert(old_value.header == header);
- }
- if (omap.size() != old_value.attrs.size()) {
- cerr << num << ": oid " << oid << " tmap.size() is " << omap.size()
- << " and old is " << old_value.attrs.size() << std::endl;
- assert(omap.size() == old_value.attrs.size());
- }
- if (omap_keys.size() != old_value.attrs.size()) {
- cerr << num << ": oid " << oid << " tmap.size() is " << omap_keys.size()
- << " and old is " << old_value.attrs.size() << std::endl;
- assert(omap_keys.size() == old_value.attrs.size());
+ if (!context->ec_pool) {
+ if (!(old_value.header == header)) {
+ cerr << num << ": oid " << oid << " header does not match, old size: "
+ << old_value.header.length() << " new size " << header.length()
+ << std::endl;
+ assert(old_value.header == header);
+ }
+ if (omap.size() != old_value.attrs.size()) {
+ cerr << num << ": oid " << oid << " omap.size() is " << omap.size()
+ << " and old is " << old_value.attrs.size() << std::endl;
+ assert(omap.size() == old_value.attrs.size());
+ }
+ if (omap_keys.size() != old_value.attrs.size()) {
+ cerr << num << ": oid " << oid << " omap.size() is " << omap_keys.size()
+ << " and old is " << old_value.attrs.size() << std::endl;
+ assert(omap_keys.size() == old_value.attrs.size());
+ }
}
if (xattrs.size() != old_value.attrs.size()) {
cerr << num << ": oid " << oid << " xattrs.size() is " << xattrs.size()
@@ -1122,39 +1082,49 @@ public:
<< " and expected " << old_value.version << std::endl;
assert(version == old_value.version);
}
- for (map<string, bufferlist>::iterator omap_iter = omap.begin();
- omap_iter != omap.end();
- ++omap_iter) {
- assert(old_value.attrs.count(omap_iter->first));
- assert(xattrs.count(omap_iter->first));
- bufferlist bl = context->cont_gen.gen_attribute(
- old_value.attrs[omap_iter->first]);
- assert(bl.length() == omap_iter->second.length());
- assert(bl.length() == xattrs[omap_iter->first].length());
+ for (map<string, ContDesc>::iterator iter = old_value.attrs.begin();
+ iter != old_value.attrs.end();
+ ++iter) {
+ bufferlist bl = context->attr_gen.gen_bl(
+ iter->second);
+ if (!context->ec_pool) {
+ map<string, bufferlist>::iterator omap_iter = omap.find(iter->first);
+ assert(omap_iter != omap.end());
+ assert(bl.length() == omap_iter->second.length());
+ bufferlist::iterator k = bl.begin();
+ for(bufferlist::iterator l = omap_iter->second.begin();
+ !k.end() && !l.end();
+ ++k, ++l) {
+ assert(*l == *k);
+ }
+ }
+ map<string, bufferlist>::iterator xattr_iter = xattrs.find(iter->first);
+ assert(xattr_iter != xattrs.end());
+ assert(bl.length() == xattr_iter->second.length());
bufferlist::iterator k = bl.begin();
- bufferlist::iterator j = xattrs[omap_iter->first].begin();
- for(bufferlist::iterator l = omap_iter->second.begin();
- !k.end() && !l.end() && !j.end();
- ++k, ++l, ++j) {
- assert(*l == *k);
+ for (bufferlist::iterator j = xattr_iter->second.begin();
+ !k.end() && !j.end();
+ ++j, ++k) {
assert(*j == *k);
}
}
- for (set<string>::iterator i = omap_requested_keys.begin();
- i != omap_requested_keys.end();
- ++i) {
- if (!omap_returned_values.count(*i))
- assert(!old_value.attrs.count(*i));
- if (!old_value.attrs.count(*i))
- assert(!omap_returned_values.count(*i));
- }
- for (map<string, bufferlist>::iterator i = omap_returned_values.begin();
- i != omap_returned_values.end();
- ++i) {
- assert(omap_requested_keys.count(i->first));
- assert(omap.count(i->first));
- assert(old_value.attrs.count(i->first));
- assert(i->second == omap[i->first]);
+ if (!context->ec_pool) {
+ for (set<string>::iterator i = omap_requested_keys.begin();
+ i != omap_requested_keys.end();
+ ++i) {
+ if (!omap_returned_values.count(*i))
+ assert(!old_value.attrs.count(*i));
+ if (!old_value.attrs.count(*i))
+ assert(!omap_returned_values.count(*i));
+ }
+ for (map<string, bufferlist>::iterator i = omap_returned_values.begin();
+ i != omap_returned_values.end();
+ ++i) {
+ assert(omap_requested_keys.count(i->first));
+ assert(omap.count(i->first));
+ assert(old_value.attrs.count(i->first));
+ assert(i->second == omap[i->first]);
+ }
}
}
context->kick();
@@ -1265,7 +1235,7 @@ public:
void _begin()
{
context->state_lock.Lock();
- ObjectDesc contents(&context->cont_gen);
+ ObjectDesc contents;
context->find_object(oid, &contents);
if (contents.deleted()) {
context->kick();
@@ -1322,7 +1292,7 @@ public:
bool done;
librados::ObjectWriteOperation op;
librados::AioCompletion *comp;
- std::tr1::shared_ptr<int> in_use;
+ ceph::shared_ptr<int> in_use;
RollbackOp(int n,
RadosTestContext *context,
@@ -1357,6 +1327,7 @@ public:
roll_back_to,
roll_back_to);
+
cout << "rollback oid " << oid << " to " << roll_back_to << std::endl;
context->roll_back(oid, roll_back_to);
@@ -1386,7 +1357,7 @@ public:
context->update_object_version(oid, comp->get_version64());
context->oid_in_use.erase(oid);
context->oid_not_in_use.insert(oid);
- in_use = std::tr1::shared_ptr<int>();
+ in_use = ceph::shared_ptr<int>();
context->kick();
}
@@ -1409,6 +1380,7 @@ public:
librados::ObjectReadOperation rd_op;
librados::AioCompletion *comp;
librados::AioCompletion *comp_racing_read;
+ ceph::shared_ptr<int> in_use;
int snap;
int done;
uint64_t version;
@@ -1420,7 +1392,6 @@ public:
TestOpStat *stat)
: TestOp(n, context, stat),
oid(oid), oid_src(oid_src),
- src_value(&context->cont_gen),
comp(NULL), done(0), version(0), r(0)
{}
@@ -1435,17 +1406,18 @@ public:
context->oid_not_in_use.erase(oid);
context->oid_in_use.insert(oid_src);
context->oid_not_in_use.erase(oid_src);
- }
- // choose source snap
- if (0 && !(rand() % 4) && !context->snaps.empty()) {
- snap = rand_choose(context->snaps)->first;
- } else {
- snap = -1;
+ // choose source snap
+ if (0 && !(rand() % 4) && !context->snaps.empty()) {
+ snap = rand_choose(context->snaps)->first;
+ in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+ } else {
+ snap = -1;
+ }
+ context->find_object(oid_src, &src_value, snap);
+ if (!src_value.deleted())
+ context->update_object_full(oid, src_value);
}
- context->find_object(oid_src, &src_value, snap);
- if (!src_value.deleted())
- context->update_object_full(oid, src_value);
string src = context->prefix+oid_src;
op.copy_from(src.c_str(), context->io_ctx, src_value.version);
@@ -1465,7 +1437,6 @@ public:
NULL);
rd_op.stat(NULL, NULL, NULL);
context->io_ctx.aio_operate(context->prefix+oid, comp_racing_read, &rd_op,
- librados::SNAP_HEAD,
librados::OPERATION_ORDER_READS_WRITES, // order wrt previous write/update
NULL);
@@ -1527,7 +1498,439 @@ public:
string getType()
{
- return "TmapPutOp";
+ return "CopyFromOp";
+ }
+};
+
+class HitSetListOp : public TestOp {
+ bool done;
+ librados::AioCompletion *comp1, *comp2;
+ uint32_t hash;
+ std::list< std::pair<time_t, time_t> > ls;
+ bufferlist bl;
+
+public:
+ HitSetListOp(int n,
+ RadosTestContext *context,
+ uint32_t hash,
+ TestOpStat *stat = 0)
+ : TestOp(n, context, stat),
+ done(false), comp1(NULL), comp2(NULL),
+ hash(hash)
+ {}
+
+ void _begin()
+ {
+ pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(this,
+ new TestOp::CallbackInfo(0));
+ comp1 = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
+ NULL);
+ int r = context->io_ctx.hit_set_list(hash, comp1, &ls);
+ assert(r == 0);
+ }
+
+ void _finish(CallbackInfo *info) {
+ Mutex::Locker l(context->state_lock);
+ if (!comp2) {
+ if (ls.empty()) {
+ cerr << num << ": no hitsets" << std::endl;
+ done = true;
+ } else {
+ cerr << num << ": hitsets are " << ls << std::endl;
+ int r = rand() % ls.size();
+ std::list<pair<time_t,time_t> >::iterator p = ls.begin();
+ while (r--)
+ ++p;
+ pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(this,
+ new TestOp::CallbackInfo(0));
+ comp2 = context->rados.aio_create_completion((void*) cb_arg, &write_callback,
+ NULL);
+ r = context->io_ctx.hit_set_get(hash, comp2, p->second, &bl);
+ assert(r == 0);
+ }
+ } else {
+ int r = comp2->get_return_value();
+ if (r == 0) {
+ HitSet hitset;
+ bufferlist::iterator p = bl.begin();
+ ::decode(hitset, p);
+ cout << num << ": got hitset of type " << hitset.get_type_name()
+ << " size " << bl.length()
+ << std::endl;
+ } else {
+ // FIXME: we could verify that we did in fact race with a trim...
+ assert(r == -ENOENT);
+ }
+ done = true;
+ }
+
+ context->kick();
+ }
+
+ bool finished() {
+ return done;
+ }
+
+ string getType() {
+ return "HitSetListOp";
+ }
+};
+
+class UndirtyOp : public TestOp {
+public:
+ librados::AioCompletion *completion;
+ librados::ObjectWriteOperation op;
+ string oid;
+
+ UndirtyOp(int n,
+ RadosTestContext *context,
+ const string &oid,
+ TestOpStat *stat = 0)
+ : TestOp(n, context, stat),
+ completion(NULL),
+ oid(oid)
+ {}
+
+ void _begin()
+ {
+ context->state_lock.Lock();
+ pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(this,
+ new TestOp::CallbackInfo(0));
+ completion = context->rados.aio_create_completion((void *) cb_arg,
+ &write_callback, 0);
+
+ context->oid_in_use.insert(oid);
+ context->oid_not_in_use.erase(oid);
+ context->update_object_undirty(oid);
+ context->state_lock.Unlock();
+
+ op.undirty();
+ int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+ &op, 0);
+ assert(!r);
+ }
+
+ void _finish(CallbackInfo *info)
+ {
+ context->state_lock.Lock();
+ assert(!done);
+ assert(completion->is_complete());
+ context->oid_in_use.erase(oid);
+ context->oid_not_in_use.insert(oid);
+ context->update_object_version(oid, completion->get_version64());
+ context->kick();
+ done = true;
+ context->state_lock.Unlock();
+ }
+
+ bool finished()
+ {
+ return done;
+ }
+
+ string getType()
+ {
+ return "UndirtyOp";
+ }
+};
+
+class IsDirtyOp : public TestOp {
+public:
+ librados::AioCompletion *completion;
+ librados::ObjectReadOperation op;
+ string oid;
+ bool dirty;
+ ObjectDesc old_value;
+ int snap;
+ ceph::shared_ptr<int> in_use;
+
+ IsDirtyOp(int n,
+ RadosTestContext *context,
+ const string &oid,
+ TestOpStat *stat = 0)
+ : TestOp(n, context, stat),
+ completion(NULL),
+ oid(oid),
+ dirty(false)
+ {}
+
+ void _begin()
+ {
+ context->state_lock.Lock();
+
+ if (!(rand() % 4) && !context->snaps.empty()) {
+ snap = rand_choose(context->snaps)->first;
+ in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+ } else {
+ snap = -1;
+ }
+ std::cout << num << ": is_dirty oid " << oid << " snap " << snap
+ << std::endl;
+
+ pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(this,
+ new TestOp::CallbackInfo(0));
+ completion = context->rados.aio_create_completion((void *) cb_arg,
+ &write_callback, 0);
+
+ context->oid_in_use.insert(oid);
+ context->oid_not_in_use.erase(oid);
+ context->state_lock.Unlock();
+
+ if (snap >= 0) {
+ context->io_ctx.snap_set_read(context->snaps[snap]);
+ }
+
+ op.is_dirty(&dirty, NULL);
+ int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+ &op, 0);
+ assert(!r);
+
+ if (snap >= 0) {
+ context->io_ctx.snap_set_read(0);
+ }
+ }
+
+ void _finish(CallbackInfo *info)
+ {
+ context->state_lock.Lock();
+ assert(!done);
+ assert(completion->is_complete());
+ context->oid_in_use.erase(oid);
+ context->oid_not_in_use.insert(oid);
+
+ assert(context->find_object(oid, &old_value, snap));
+
+ int r = completion->get_return_value();
+ if (r == 0) {
+ cout << num << ": " << (dirty ? "dirty" : "clean") << std::endl;
+ assert(!old_value.deleted());
+ assert(dirty == old_value.dirty);
+ } else {
+ cout << num << ": got " << r << std::endl;
+ assert(r == -ENOENT);
+ assert(old_value.deleted());
+ }
+ context->kick();
+ done = true;
+ context->state_lock.Unlock();
+ }
+
+ bool finished()
+ {
+ return done;
+ }
+
+ string getType()
+ {
+ return "IsDirtyOp";
+ }
+};
+
+
+
+class CacheFlushOp : public TestOp {
+public:
+ librados::AioCompletion *completion;
+ librados::ObjectReadOperation op;
+ string oid;
+ bool blocking;
+ int snap;
+ bool can_fail;
+ ceph::shared_ptr<int> in_use;
+
+ CacheFlushOp(int n,
+ RadosTestContext *context,
+ const string &oid,
+ TestOpStat *stat,
+ bool b)
+ : TestOp(n, context, stat),
+ completion(NULL),
+ oid(oid),
+ blocking(b),
+ snap(0),
+ can_fail(false)
+ {}
+
+ void _begin()
+ {
+ context->state_lock.Lock();
+
+ if (!(rand() % 4) && !context->snaps.empty()) {
+ snap = rand_choose(context->snaps)->first;
+ in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+ } else {
+ snap = -1;
+ }
+ // not being particularly specific here about knowing which
+ // flushes are on the oldest clean snap and which ones are not.
+ can_fail = !blocking || !context->snaps.empty();
+ // FIXME: we can could fail if we've ever removed a snap due to
+ // the async snap trimming.
+ can_fail = true;
+ cout << num << ": " << (blocking ? "cache_flush" : "cache_try_flush")
+ << " oid " << oid << " snap " << snap << std::endl;
+
+ if (snap >= 0) {
+ context->io_ctx.snap_set_read(context->snaps[snap]);
+ }
+
+ pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(this,
+ new TestOp::CallbackInfo(0));
+ completion = context->rados.aio_create_completion((void *) cb_arg,
+ &write_callback, 0);
+ // leave object in unused list so that we race with other operations
+ //context->oid_in_use.insert(oid);
+ //context->oid_not_in_use.erase(oid);
+ context->oid_flushing.insert(oid);
+ context->oid_not_flushing.erase(oid);
+ context->state_lock.Unlock();
+
+ unsigned flags = librados::OPERATION_IGNORE_CACHE;
+ if (blocking) {
+ op.cache_flush();
+ } else {
+ op.cache_try_flush();
+ flags = librados::OPERATION_SKIPRWLOCKS;
+ }
+ int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+ &op, flags, NULL);
+ assert(!r);
+
+ if (snap >= 0) {
+ context->io_ctx.snap_set_read(0);
+ }
+ }
+
+ void _finish(CallbackInfo *info)
+ {
+ context->state_lock.Lock();
+ assert(!done);
+ assert(completion->is_complete());
+ //context->oid_in_use.erase(oid);
+ //context->oid_not_in_use.insert(oid);
+ context->oid_flushing.erase(oid);
+ context->oid_not_flushing.insert(oid);
+ int r = completion->get_return_value();
+ cout << num << ": got " << cpp_strerror(r) << std::endl;
+ if (r == 0) {
+ context->update_object_version(oid, 0, snap);
+ } else if (r == -EBUSY) {
+ assert(can_fail);
+ } else if (r == -EINVAL) {
+ // caching not enabled?
+ } else if (r == -ENOENT) {
+ // may have raced with a remove?
+ } else {
+ assert(0 == "shouldn't happen");
+ }
+ context->kick();
+ done = true;
+ context->state_lock.Unlock();
+ }
+
+ bool finished()
+ {
+ return done;
+ }
+
+ string getType()
+ {
+ return "CacheFlushOp";
+ }
+};
+
+class CacheEvictOp : public TestOp {
+public:
+ librados::AioCompletion *completion;
+ librados::ObjectReadOperation op;
+ string oid;
+ ceph::shared_ptr<int> in_use;
+
+ CacheEvictOp(int n,
+ RadosTestContext *context,
+ const string &oid,
+ TestOpStat *stat)
+ : TestOp(n, context, stat),
+ completion(NULL),
+ oid(oid)
+ {}
+
+ void _begin()
+ {
+ context->state_lock.Lock();
+
+ int snap;
+ if (!(rand() % 4) && !context->snaps.empty()) {
+ snap = rand_choose(context->snaps)->first;
+ in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+ } else {
+ snap = -1;
+ }
+ cout << num << ": cache_evict oid " << oid << " snap " << snap << std::endl;
+
+ if (snap >= 0) {
+ context->io_ctx.snap_set_read(context->snaps[snap]);
+ }
+
+ pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
+ new pair<TestOp*, TestOp::CallbackInfo*>(this,
+ new TestOp::CallbackInfo(0));
+ completion = context->rados.aio_create_completion((void *) cb_arg,
+ &write_callback, 0);
+ // leave object in unused list so that we race with other operations
+ //context->oid_in_use.insert(oid);
+ //context->oid_not_in_use.erase(oid);
+ context->state_lock.Unlock();
+
+ op.cache_evict();
+ int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+ &op, librados::OPERATION_IGNORE_CACHE,
+ NULL);
+ assert(!r);
+
+ if (snap >= 0) {
+ context->io_ctx.snap_set_read(0);
+ }
+ }
+
+ void _finish(CallbackInfo *info)
+ {
+ context->state_lock.Lock();
+ assert(!done);
+ assert(completion->is_complete());
+ //context->oid_in_use.erase(oid);
+ //context->oid_not_in_use.insert(oid);
+ int r = completion->get_return_value();
+ cout << num << ": got " << cpp_strerror(r) << std::endl;
+ if (r == 0) {
+ // yay!
+ } else if (r == -EBUSY) {
+ // raced with something that dirtied the object
+ } else if (r == -EINVAL) {
+ // caching not enabled?
+ } else if (r == -ENOENT) {
+ // may have raced with a remove?
+ } else {
+ assert(0 == "shouldn't happen");
+ }
+ context->kick();
+ done = true;
+ context->state_lock.Unlock();
+ }
+
+ bool finished()
+ {
+ return done;
+ }
+
+ string getType()
+ {
+ return "CacheEvictOp";
}
};
diff --git a/src/test/osd/TestErasureCodeExample.cc b/src/test/osd/TestErasureCodeExample.cc
index f12e80c..faa863e 100644
--- a/src/test/osd/TestErasureCodeExample.cc
+++ b/src/test/osd/TestErasureCodeExample.cc
@@ -20,6 +20,13 @@
#include "global/global_context.h"
#include "gtest/gtest.h"
+TEST(ErasureCodeExample, chunk_size)
+{
+ ErasureCodeExample example;
+ EXPECT_EQ(3u, example.get_chunk_count());
+ EXPECT_EQ(11u, example.get_chunk_size(20));
+}
+
TEST(ErasureCodeExample, minimum_to_decode)
{
ErasureCodeExample example;
@@ -105,13 +112,13 @@ TEST(ErasureCodeExample, encode_decode)
bufferlist in;
in.append("ABCDE");
- int want_to_encode[] = { 0, 1, 2 };
+ set<int> want_to_encode;
+ for(unsigned int i = 0; i < example.get_chunk_count(); i++)
+ want_to_encode.insert(i);
map<int, bufferlist> encoded;
- EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
- in,
- &encoded));
- EXPECT_EQ(3u, encoded.size());
- EXPECT_EQ(3u, encoded[0].length());
+ EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(example.get_chunk_count(), encoded.size());
+ EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
EXPECT_EQ('A', encoded[0][0]);
EXPECT_EQ('B', encoded[0][1]);
EXPECT_EQ('C', encoded[0][2]);
@@ -157,6 +164,43 @@ TEST(ErasureCodeExample, encode_decode)
}
}
+TEST(ErasureCodeExample, decode)
+{
+ ErasureCodeExample example;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_front(in_ptr);
+ int want_to_encode[] = { 0, 1, 2 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
+ in,
+ &encoded));
+ EXPECT_EQ(3u, encoded.size());
+
+ // successfull decode
+ bufferlist out;
+ EXPECT_EQ(0, example.decode_concat(encoded, &out));
+ bufferlist usable;
+ usable.substr_of(out, 0, in.length());
+ EXPECT_TRUE(usable == in);
+
+ // cannot recover
+ map<int, bufferlist> degraded;
+ degraded[0] = encoded[0];
+ EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
+}
+
int main(int argc, char **argv) {
vector<const char*> args;
argv_to_vec(argc, (const char **)argv, args);
@@ -168,6 +212,15 @@ int main(int argc, char **argv) {
return RUN_ALL_TESTS();
}
-// Local Variables:
-// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_example && valgrind --leak-check=full --tool=memcheck ./unittest_erasure_code_example --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
-// End:
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 &&
+ * make unittest_erasure_code_example &&
+ * valgrind --leak-check=full --tool=memcheck \
+ * ./unittest_erasure_code_example --gtest_filter=*.* \
+ * --log-to-stderr=true --debug-osd=20
+ * "
+ * End:
+ */
+
diff --git a/src/test/osd/TestErasureCodeJerasure.cc b/src/test/osd/TestErasureCodeJerasure.cc
index a51cb85..0e2aaa1 100644
--- a/src/test/osd/TestErasureCodeJerasure.cc
+++ b/src/test/osd/TestErasureCodeJerasure.cc
@@ -48,7 +48,7 @@ TYPED_TEST(ErasureCodeTest, encode_decode)
jerasure.init(parameters);
#define LARGE_ENOUGH 2048
- bufferptr in_ptr(LARGE_ENOUGH);
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
in_ptr.zero();
in_ptr.set_length(0);
const char *payload =
@@ -217,31 +217,8 @@ TEST(ErasureCodeTest, encode)
unsigned alignment = jerasure.get_alignment();
{
//
- // When the input bufferlist is perfectly aligned, it is
- // pointed to unmodified by the returned encoded chunks.
- //
- bufferlist in;
- map<int,bufferlist> encoded;
- int want_to_encode[] = { 0, 1, 2, 3 };
- in.append(string(alignment * 2, 'X'));
- EXPECT_EQ(alignment * 2, in.length());
- EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
- in,
- &encoded));
- EXPECT_EQ(4u, encoded.size());
- for(int i = 0; i < 4; i++)
- EXPECT_EQ(alignment, encoded[i].length());
- EXPECT_EQ(in.c_str(), encoded[0].c_str());
- EXPECT_EQ(in.c_str() + alignment, encoded[1].c_str());
- }
-
- {
- //
// When the input bufferlist needs to be padded because
// it is not properly aligned, it is padded with zeros.
- // The beginning of the input bufferlist is pointed to
- // unmodified by the returned encoded chunk, only the
- // trailing chunk is allocated and copied.
//
bufferlist in;
map<int,bufferlist> encoded;
@@ -254,8 +231,6 @@ TEST(ErasureCodeTest, encode)
EXPECT_EQ(4u, encoded.size());
for(int i = 0; i < 4; i++)
EXPECT_EQ(alignment, encoded[i].length());
- EXPECT_EQ(in.c_str(), encoded[0].c_str());
- EXPECT_NE(in.c_str() + alignment, encoded[1].c_str());
char *last_chunk = encoded[1].c_str();
EXPECT_EQ('X', last_chunk[0]);
EXPECT_EQ('\0', last_chunk[trail_length]);
@@ -268,7 +243,7 @@ TEST(ErasureCodeTest, encode)
// internally allocated a buffer because of padding requirements
// and also computes the coding chunks, they are released before
// the return of the method, as shown when running the tests thru
- // valgrind that shows there is no leak.
+ // valgrind (there is no leak).
//
bufferlist in;
map<int,bufferlist> encoded;
@@ -279,7 +254,6 @@ TEST(ErasureCodeTest, encode)
EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded));
EXPECT_EQ(1u, encoded.size());
EXPECT_EQ(alignment, encoded[0].length());
- EXPECT_EQ(in.c_str(), encoded[0].c_str());
}
}
diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc
new file mode 100644
index 0000000..ee5e9e3
--- /dev/null
+++ b/src/test/osd/TestOSDMap.cc
@@ -0,0 +1,239 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "gtest/gtest.h"
+#include "osd/OSDMap.h"
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/common_init.h"
+
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char **argv) {
+ std::vector<const char *> preargs;
+ std::vector<const char*> args(argv, argv+argc);
+ global_init(&preargs, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ // make sure we have 3 copies, or some tests won't work
+ g_ceph_context->_conf->set_val("osd_pool_default_size", "3", false);
+ // our map is flat, so just try and split across OSDs, not hosts or whatever
+ g_ceph_context->_conf->set_val("osd_crush_chooseleaf_type", "0", false);
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+class OSDMapTest : public testing::Test {
+ const static int num_osds = 6;
+public:
+ OSDMap osdmap;
+ OSDMapTest() {}
+
+ void set_up_map() {
+ uuid_d fsid;
+ osdmap.build_simple(g_ceph_context, 0, fsid, num_osds, 6, 6);
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.fsid = osdmap.get_fsid();
+ entity_addr_t sample_addr;
+ uuid_d sample_uuid;
+ for (int i = 0; i < num_osds; ++i) {
+ sample_uuid.uuid[i] = i;
+ sample_addr.nonce = i;
+ pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW;
+ pending_inc.new_up_client[i] = sample_addr;
+ pending_inc.new_up_cluster[i] = sample_addr;
+ pending_inc.new_hb_back_up[i] = sample_addr;
+ pending_inc.new_hb_front_up[i] = sample_addr;
+ pending_inc.new_weight[i] = CEPH_OSD_IN;
+ pending_inc.new_uuid[i] = sample_uuid;
+ }
+ osdmap.apply_incremental(pending_inc);
+ }
+ unsigned int get_num_osds() { return num_osds; }
+};
+
+TEST_F(OSDMapTest, Create) {
+ set_up_map();
+ ASSERT_EQ(get_num_osds(), (unsigned)osdmap.get_max_osd());
+ ASSERT_EQ(get_num_osds(), osdmap.get_num_in_osds());
+}
+
+TEST_F(OSDMapTest, MapPG) {
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+
+ vector<int> old_up_osds, old_acting_osds;
+ osdmap.pg_to_up_acting_osds(pgid, old_up_osds, old_acting_osds);
+ ASSERT_EQ(old_up_osds, up_osds);
+ ASSERT_EQ(old_acting_osds, acting_osds);
+
+ ASSERT_EQ(osdmap.get_pg_pool(0)->get_size(), up_osds.size());
+}
+
+TEST_F(OSDMapTest, MapFunctionsMatch) {
+ // TODO: make sure pg_to_up_acting_osds and pg_to_acting_osds match
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+
+ vector<int> up_osds_two, acting_osds_two;
+
+ osdmap.pg_to_up_acting_osds(pgid, up_osds_two, acting_osds_two);
+
+ ASSERT_EQ(up_osds, up_osds_two);
+ ASSERT_EQ(acting_osds, acting_osds_two);
+
+ int acting_primary_two;
+ osdmap.pg_to_acting_osds(pgid, &acting_osds_two, &acting_primary_two);
+ EXPECT_EQ(acting_osds, acting_osds_two);
+ EXPECT_EQ(acting_primary, acting_primary_two);
+ osdmap.pg_to_acting_osds(pgid, acting_osds_two);
+ EXPECT_EQ(acting_osds, acting_osds_two);
+}
+
+/** This test must be removed or modified appropriately when we allow
+ * other ways to specify a primary. */
+TEST_F(OSDMapTest, PrimaryIsFirst) {
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+ EXPECT_EQ(up_osds[0], up_primary);
+ EXPECT_EQ(acting_osds[0], acting_primary);
+}
+
+TEST_F(OSDMapTest, PGTempRespected) {
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+
+ // copy and swap first and last element in acting_osds
+ vector<int> new_acting_osds(acting_osds);
+ int first = new_acting_osds[0];
+ new_acting_osds[0] = *new_acting_osds.rbegin();
+ *new_acting_osds.rbegin() = first;
+
+ // apply pg_temp to osdmap
+ OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+ pgtemp_map.new_pg_temp[pgid] = new_acting_osds;
+ osdmap.apply_incremental(pgtemp_map);
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+ EXPECT_EQ(new_acting_osds, acting_osds);
+}
+
+TEST_F(OSDMapTest, PrimaryTempRespected) {
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+
+ // make second OSD primary via incremental
+ OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+ pgtemp_map.new_primary_temp[pgid] = acting_osds[1];
+ osdmap.apply_incremental(pgtemp_map);
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+ EXPECT_EQ(acting_primary, acting_osds[1]);
+}
+
+TEST_F(OSDMapTest, RemovesRedundantTemps) {
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+
+ // stick calculated values in to temps
+ OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+ pgtemp_map.new_pg_temp[pgid] = up_osds;
+ pgtemp_map.new_primary_temp[pgid] = up_primary;
+ osdmap.apply_incremental(pgtemp_map);
+
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ OSDMap::remove_redundant_temporaries(g_ceph_context, osdmap, &pending_inc);
+
+ EXPECT_TRUE(pending_inc.new_pg_temp.count(pgid) &&
+ pending_inc.new_pg_temp[pgid].size() == 0);
+ EXPECT_EQ(-1, pending_inc.new_primary_temp[pgid]);
+}
+
+TEST_F(OSDMapTest, KeepsNecessaryTemps) {
+ set_up_map();
+
+ pg_t rawpg(0, 0, -1);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up_osds, acting_osds;
+ int up_primary, acting_primary;
+
+ osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+ &acting_osds, &acting_primary);
+
+ // find unused OSD and stick it in there
+ OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+ // find an unused osd and put it in place of the first one
+ int i = 0;
+ for(; i != (int)get_num_osds(); ++i) {
+ bool in_use = false;
+ for (vector<int>::iterator osd_it = up_osds.begin();
+ osd_it != up_osds.end();
+ ++osd_it) {
+ if (i == *osd_it) {
+ in_use = true;
+ break;
+ }
+ }
+ if (!in_use) {
+ up_osds[1] = i;
+ break;
+ }
+ }
+ if (i == (int)get_num_osds())
+ ASSERT_EQ(0, "did not find unused OSD for temp mapping");
+
+ pgtemp_map.new_pg_temp[pgid] = up_osds;
+ pgtemp_map.new_primary_temp[pgid] = up_osds[1];
+ osdmap.apply_incremental(pgtemp_map);
+
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+
+ OSDMap::remove_redundant_temporaries(g_ceph_context, osdmap, &pending_inc);
+ EXPECT_FALSE(pending_inc.new_pg_temp.count(pgid));
+ EXPECT_FALSE(pending_inc.new_primary_temp.count(pgid));
+}
diff --git a/src/test/osd/TestOpStat.h b/src/test/osd/TestOpStat.h
index e08fab9..3d70ece 100644
--- a/src/test/osd/TestOpStat.h
+++ b/src/test/osd/TestOpStat.h
@@ -6,7 +6,7 @@
#ifndef TESTOPSTAT_H
#define TESTOPSTAT_H
-struct TestOp;
+class TestOp;
class TestOpStat {
public:
diff --git a/src/test/osd/TestPGLog.cc b/src/test/osd/TestPGLog.cc
index e0863f7..39af882 100644
--- a/src/test/osd/TestPGLog.cc
+++ b/src/test/osd/TestPGLog.cc
@@ -35,6 +35,21 @@ public:
}
};
+struct TestHandler : public PGLog::LogEntryHandler {
+ list<hobject_t> &removed;
+ TestHandler(list<hobject_t> &removed) : removed(removed) {}
+
+ void rollback(
+ const pg_log_entry_t &entry) {}
+ void remove(
+ const hobject_t &hoid) {
+ removed.push_back(hoid);
+ }
+ void cant_rollback(const pg_log_entry_t &entry) {}
+ void trim(
+ const pg_log_entry_t &entry) {}
+};
+
TEST_F(PGLogTest, rewind_divergent_log) {
// newhead > log.tail : throw an assert
{
@@ -47,7 +62,8 @@ TEST_F(PGLogTest, rewind_divergent_log) {
bool dirty_big_info = false;
log.tail = eversion_t(2, 1);
- EXPECT_THROW(rewind_divergent_log(t, eversion_t(1, 1), info, remove_snap,
+ TestHandler h(remove_snap);
+ EXPECT_THROW(rewind_divergent_log(t, eversion_t(1, 1), info, &h,
dirty_info, dirty_big_info),
FailedAssertion);
}
@@ -88,6 +104,7 @@ TEST_F(PGLogTest, rewind_divergent_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
@@ -119,7 +136,8 @@ TEST_F(PGLogTest, rewind_divergent_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- rewind_divergent_log(t, newhead, info, remove_snap,
+ TestHandler h(remove_snap);
+ rewind_divergent_log(t, newhead, info, &h,
dirty_info, dirty_big_info);
EXPECT_TRUE(log.objects.count(divergent));
@@ -165,6 +183,7 @@ TEST_F(PGLogTest, rewind_divergent_log) {
eversion_t newhead;
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
info.log_tail = log.tail = eversion_t(1, 1);
newhead = eversion_t(1, 3);
@@ -185,7 +204,8 @@ TEST_F(PGLogTest, rewind_divergent_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- rewind_divergent_log(t, newhead, info, remove_snap,
+ TestHandler h(remove_snap);
+ rewind_divergent_log(t, newhead, info, &h,
dirty_info, dirty_big_info);
EXPECT_TRUE(missing.is_missing(divergent_object));
@@ -206,6 +226,7 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
@@ -219,7 +240,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_FALSE(missing.have_missing());
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -234,13 +256,15 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
oe.op = pg_log_entry_t::CLONE;
oe.soid.snap = CEPH_NOSNAP;
- EXPECT_THROW(merge_old_entry(t, oe, info, remove_snap), FailedAssertion);
+ TestHandler h(remove_snap);
+ EXPECT_THROW(merge_old_entry(t, oe, info, &h), FailedAssertion);
oe.soid.snap = 1U;
missing.add(oe.soid, eversion_t(), eversion_t());
@@ -251,7 +275,7 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(missing.is_missing(oe.soid));
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_EQ(oe.soid, remove_snap.front());
@@ -267,6 +291,7 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
@@ -279,7 +304,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_FALSE(missing.have_missing());
EXPECT_EQ(1U, log.log.size());
- EXPECT_TRUE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_TRUE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -299,6 +325,7 @@ TEST_F(PGLogTest, merge_old_entry) {
list<hobject_t> remove_snap;
pg_log_entry_t ne;
+ ne.mod_desc.mark_unrollbackable();
ne.version = eversion_t(2,1);
log.add(ne);
@@ -313,9 +340,11 @@ TEST_F(PGLogTest, merge_old_entry) {
{
log.log.front().op = pg_log_entry_t::DELETE;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
oe.version = eversion_t(1,1);
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
}
// if the newer entry is not DELETE, the object must be in missing
@@ -324,23 +353,15 @@ TEST_F(PGLogTest, merge_old_entry) {
ne.op = pg_log_entry_t::MODIFY;
missing.add_next_event(ne);
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
oe.version = eversion_t(1,1);
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
missing.rm(ne.soid, ne.version);
}
- // throw if the newer entry is not DELETE and not in missing
- {
- pg_log_entry_t &ne = log.log.front();
- ne.op = pg_log_entry_t::MODIFY;
- pg_log_entry_t oe;
- oe.version = eversion_t(1,1);
-
- EXPECT_THROW(merge_old_entry(t, oe, info, remove_snap), FailedAssertion);
- }
-
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
EXPECT_TRUE(t.empty());
@@ -358,10 +379,12 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
pg_log_entry_t ne;
+ ne.mod_desc.mark_unrollbackable();
ne.version = eversion_t(1,1);
ne.op = pg_log_entry_t::DELETE;
log.add(ne);
@@ -375,7 +398,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_FALSE(missing.have_missing());
EXPECT_EQ(1U, log.log.size());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -399,10 +423,12 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
pg_log_entry_t ne;
+ ne.mod_desc.mark_unrollbackable();
ne.version = eversion_t(1,1);
ne.op = pg_log_entry_t::MODIFY;
log.add(ne);
@@ -419,7 +445,8 @@ TEST_F(PGLogTest, merge_old_entry) {
eversion_t old_version(0, 0);
// if the object is not already in missing, add it
{
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_TRUE(missing.is_missing(ne.soid, ne.version));
EXPECT_FALSE(missing.is_missing(ne.soid, old_version));
@@ -429,7 +456,8 @@ TEST_F(PGLogTest, merge_old_entry) {
missing.revise_need(ne.soid, old_version);
EXPECT_TRUE(missing.is_missing(ne.soid, old_version));
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_TRUE(missing.is_missing(ne.soid, ne.version));
EXPECT_FALSE(missing.is_missing(ne.soid, old_version));
@@ -452,10 +480,12 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
pg_log_entry_t ne;
+ ne.mod_desc.mark_unrollbackable();
ne.version = eversion_t(1,1);
ne.op = pg_log_entry_t::DELETE;
log.add(ne);
@@ -470,10 +500,11 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(missing.is_missing(oe.soid));
EXPECT_EQ(1U, log.log.size());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
- EXPECT_TRUE(remove_snap.empty());
+ EXPECT_TRUE(remove_snap.size() > 0);
EXPECT_TRUE(t.empty());
EXPECT_FALSE(missing.have_missing());
EXPECT_EQ(1U, log.log.size());
@@ -488,25 +519,26 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
info.log_tail = eversion_t(1,1);
oe.op = pg_log_entry_t::MODIFY;
oe.prior_version = eversion_t(2,1);
+ missing_add(oe.soid, oe.prior_version, eversion_t());
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
EXPECT_TRUE(t.empty());
- EXPECT_FALSE(missing.have_missing());
EXPECT_TRUE(log.empty());
}
@@ -523,6 +555,7 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
@@ -537,7 +570,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_FALSE(missing.have_missing());
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_TRUE(is_dirty());
EXPECT_EQ(oe.soid, remove_snap.front());
@@ -559,6 +593,7 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
@@ -573,7 +608,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_FALSE(missing.have_missing());
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_TRUE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -594,6 +630,7 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
@@ -610,7 +647,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(missing.is_missing(oe.soid));
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_TRUE(remove_snap.empty());
@@ -631,6 +669,7 @@ TEST_F(PGLogTest, merge_old_entry) {
ObjectStore::Transaction t;
pg_log_entry_t oe;
+ oe.mod_desc.mark_unrollbackable();
pg_info_t info;
list<hobject_t> remove_snap;
@@ -647,7 +686,8 @@ TEST_F(PGLogTest, merge_old_entry) {
EXPECT_TRUE(missing.is_missing(oe.soid));
EXPECT_TRUE(log.empty());
- EXPECT_FALSE(merge_old_entry(t, oe, info, remove_snap));
+ TestHandler h(remove_snap);
+ EXPECT_FALSE(merge_old_entry(t, oe, info, &h));
EXPECT_FALSE(is_dirty());
EXPECT_EQ(oe.soid, remove_snap.front());
@@ -691,7 +731,8 @@ TEST_F(PGLogTest, merge_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info);
EXPECT_FALSE(missing.have_missing());
@@ -742,7 +783,8 @@ TEST_F(PGLogTest, merge_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info);
EXPECT_FALSE(missing.have_missing());
@@ -808,6 +850,7 @@ TEST_F(PGLogTest, merge_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 4);
e.soid.hash = 0x5;
@@ -847,7 +890,8 @@ TEST_F(PGLogTest, merge_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info);
EXPECT_FALSE(missing.have_missing());
@@ -903,6 +947,7 @@ TEST_F(PGLogTest, merge_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
@@ -956,7 +1001,8 @@ TEST_F(PGLogTest, merge_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info);
/* When the divergent entry is a DELETE and the authoritative
@@ -1014,6 +1060,7 @@ TEST_F(PGLogTest, merge_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
@@ -1056,7 +1103,8 @@ TEST_F(PGLogTest, merge_log) {
EXPECT_FALSE(dirty_info);
EXPECT_FALSE(dirty_big_info);
- merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info);
EXPECT_FALSE(missing.have_missing());
@@ -1086,7 +1134,8 @@ TEST_F(PGLogTest, merge_log) {
// olog has been trimmed
olog.tail = eversion_t(1, 1);
- ASSERT_THROW(merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ ASSERT_THROW(merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info), FailedAssertion);
}
@@ -1095,16 +1144,16 @@ TEST_F(PGLogTest, merge_log) {
+--------+-------+---------+
| |object | |
|version | hash | version |
- | | | |
- tail > (1,1) | x5 | |
+ tail > (0,0) | | |
+ | (1,1) | x5 | |
| | | |
| | | |
head > (1,2) | x3 | |
| | | |
- | | x9 | (2,3) < tail
- | | | |
+ | | | (2,3) < tail
+ | | x9 | (2,4) |
| | | |
- | | x5 | (2,4) < head
+ | | x5 | (2,5) < head
| | | |
+--------+-------+---------+
@@ -1125,10 +1174,11 @@ TEST_F(PGLogTest, merge_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
+ log.tail = eversion_t();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
- log.tail = e.version;
log.log.push_back(e);
e.version = eversion_t(1, 2);
e.soid.hash = 0x3;
@@ -1138,16 +1188,18 @@ TEST_F(PGLogTest, merge_log) {
info.last_update = log.head;
- e.version = eversion_t(2, 3);
+ olog.tail = eversion_t(2, 3);
+ e.version = eversion_t(2, 4);
e.soid.hash = 0x9;
olog.log.push_back(e);
- e.version = eversion_t(2, 4);
+ e.version = eversion_t(2, 5);
e.soid.hash = 0x5;
olog.log.push_back(e);
olog.head = e.version;
}
- EXPECT_THROW(merge_log(t, oinfo, olog, fromosd, info, remove_snap,
+ TestHandler h(remove_snap);
+ EXPECT_THROW(merge_log(t, oinfo, olog, fromosd, info, &h,
dirty_info, dirty_big_info), FailedAssertion);
}
}
@@ -1217,6 +1269,7 @@ TEST_F(PGLogTest, proc_replica_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 2);
e.soid.hash = 0x5;
@@ -1256,40 +1309,7 @@ TEST_F(PGLogTest, proc_replica_log) {
EXPECT_EQ(olog.head, oinfo.last_complete);
}
- /* +--------------------------+
- | log olog |
- +--------+-------+---------+
- | |object | |
- |version | hash | version |
- | | | |
- tail > (1,1) | x5 | (1,1) < tail
- | | | |
- | | | |
- | (1,2) | x3 | (1,2) |
- | | | |
- | | | |
- | (1,3) | x9 | |
- | MODIFY | | |
- | | | |
- | | x7 | (1,4) |
- | | | |
- | (1,6) | x8 | (1,5) |
- | | | |
- | | x9 | (2,3) < head < last_backfill
- | | | DELETE |
- | | | |
- head > (1,7) | xa | (2,5) |
- | | | |
- +--------+-------+---------+
-
- The log entry (1,3) modifies the object x9 but the olog entry
- (2,3) deletes it : log is authoritative and the object is added
- to missing. x7 is divergent and ignored. x8 has a more recent
- version in the log and the olog entry is ignored. xa is past
- last_backfill and ignored.
-
- */
- {
+ {
clear();
ObjectStore::Transaction t;
@@ -1298,56 +1318,83 @@ TEST_F(PGLogTest, proc_replica_log) {
pg_missing_t omissing;
int from = -1;
- eversion_t last_update(1, 2);
hobject_t divergent_object;
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
- e.version = eversion_t(1, 1);
- e.soid.hash = 0x5;
- log.tail = e.version;
- log.log.push_back(e);
- e.version = last_update;
- e.soid.hash = 0x3;
- log.log.push_back(e);
- e.version = eversion_t(1,3);
- e.soid.hash = 0x9;
- divergent_object = e.soid;
- e.op = pg_log_entry_t::MODIFY;
- log.log.push_back(e);
- e.version = eversion_t(1, 6);
- e.soid.hash = 0x8;
- log.log.push_back(e);
- e.version = eversion_t(1, 7);
- e.soid.hash = 0xa;
- log.log.push_back(e);
- log.head = e.version;
+ {
+ e.soid = divergent_object;
+ e.soid.hash = 0x1;
+ e.version = eversion_t(1, 1);
+ log.tail = e.version;
+ log.log.push_back(e);
+
+ e.soid = divergent_object;
+ e.version = eversion_t(1, 2);
+ log.tail = e.version;
+ log.log.push_back(e);
+
+ e.soid.hash = 0x3;
+ e.version = eversion_t(1, 4);
+ log.log.push_back(e);
+
+ e.soid.hash = 0x7;
+ e.version = eversion_t(1, 5);
+ log.log.push_back(e);
+
+ e.soid.hash = 0x8;
+ e.version = eversion_t(1, 6);
+ log.log.push_back(e);
+
+ e.soid.hash = 0x9;
+ e.op = pg_log_entry_t::DELETE;
+ e.version = eversion_t(2, 7);
+ log.log.push_back(e);
+
+ e.soid.hash = 0xa;
+ e.version = eversion_t(2, 8);
+ log.head = e.version;
+ log.log.push_back(e);
+ }
log.index();
- e.version = eversion_t(1, 1);
- e.soid.hash = 0x5;
- olog.tail = e.version;
- olog.log.push_back(e);
- e.version = last_update;
- e.soid.hash = 0x3;
- olog.log.push_back(e);
- e.version = eversion_t(1, 4); // divergent entry : will be ignored
- e.soid.hash = 0x7;
- olog.log.push_back(e);
- e.version = eversion_t(1, 5); // log has newer entry : it will be ignored
- e.soid.hash = 0x8;
- olog.log.push_back(e);
- e.version = eversion_t(2, 3);
- e.soid.hash = 0x9;
- oinfo.last_backfill = e.soid;
- e.op = pg_log_entry_t::DELETE;
- olog.log.push_back(e);
- e.version = eversion_t(2, 4);
- e.soid.hash = 0xa; // > last_backfill are ignored
- olog.log.push_back(e);
- olog.head = e.version;
-
+ {
+ e.soid = divergent_object;
+ e.soid.hash = 0x1;
+ e.version = eversion_t(1, 1);
+ olog.tail = e.version;
+ olog.log.push_back(e);
+
+ e.soid = divergent_object;
+ e.version = eversion_t(1, 2);
+ olog.log.push_back(e);
+
+ e.soid.hash = 0x3;
+ e.version = eversion_t(1, 4);
+ olog.log.push_back(e);
+
+ e.soid.hash = 0x7;
+ e.version = eversion_t(1, 5);
+ olog.log.push_back(e);
+
+ e.soid.hash = 0x8;
+ e.version = eversion_t(1, 6);
+ olog.log.push_back(e);
+
+ e.soid.hash = 0x9; // should be ignored, matches above
+ e.op = pg_log_entry_t::DELETE;
+ e.version = eversion_t(1, 7);
+ olog.log.push_back(e);
+
+ e.soid = divergent_object; // should be marked missing at version 1'2
+ e.op = pg_log_entry_t::MODIFY;
+ e.version = eversion_t(1, 8);
+ e.prior_version = eversion_t(1, 2);
+ olog.log.push_back(e);
+ olog.head = e.version;
+ }
oinfo.last_update = olog.head;
oinfo.last_complete = olog.head;
}
@@ -1362,9 +1409,9 @@ TEST_F(PGLogTest, proc_replica_log) {
EXPECT_TRUE(t.empty());
EXPECT_TRUE(omissing.have_missing());
EXPECT_TRUE(omissing.is_missing(divergent_object));
- EXPECT_EQ(eversion_t(1, 3), omissing.missing[divergent_object].need);
- EXPECT_EQ(last_update, oinfo.last_update);
- EXPECT_EQ(last_update, oinfo.last_complete);
+ EXPECT_EQ(eversion_t(1, 2), omissing.missing[divergent_object].need);
+ EXPECT_EQ(eversion_t(1, 6), oinfo.last_update);
+ EXPECT_EQ(eversion_t(1, 1), oinfo.last_complete);
}
/* +--------------------------+
@@ -1404,6 +1451,7 @@ TEST_F(PGLogTest, proc_replica_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
@@ -1487,6 +1535,7 @@ TEST_F(PGLogTest, proc_replica_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
@@ -1576,6 +1625,7 @@ TEST_F(PGLogTest, proc_replica_log) {
{
pg_log_entry_t e;
+ e.mod_desc.mark_unrollbackable();
e.version = eversion_t(1, 1);
e.soid.hash = 0x5;
diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc
index 20a4f82..79c367c 100644
--- a/src/test/osd/TestRados.cc
+++ b/src/test/osd/TestRados.cc
@@ -26,9 +26,12 @@ public:
int objects,
map<TestOpType, unsigned int> op_weights,
TestOpStat *stats,
- int max_seconds) :
- m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds), m_objects(objects), m_stats(stats),
- m_total_weight(0)
+ int max_seconds,
+ bool ec_pool) :
+ m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds),
+ m_objects(objects), m_stats(stats),
+ m_total_weight(0),
+ m_ec_pool(ec_pool)
{
m_start = time(0);
for (map<TestOpType, unsigned int>::const_iterator it = op_weights.begin();
@@ -49,7 +52,12 @@ public:
stringstream oid;
oid << m_op;
cout << m_op << ": write initial oid " << oid.str() << std::endl;
- return new WriteOp(m_op, &context, oid.str());
+ context.oid_not_flushing.insert(oid.str());
+ if (m_ec_pool) {
+ return new WriteOp(m_op, &context, oid.str(), true);
+ } else {
+ return new WriteOp(m_op, &context, oid.str(), false);
+ }
} else if (m_op >= m_ops) {
return NULL;
}
@@ -87,27 +95,25 @@ private:
//cout << "oids not in use " << context.oid_not_in_use.size() << std::endl;
assert(context.oid_not_in_use.size());
- cout << m_op << ": ";
switch (type) {
case TEST_OP_READ:
oid = *(rand_choose(context.oid_not_in_use));
- cout << "read oid " << oid << std::endl;
return new ReadOp(m_op, &context, oid, m_stats);
case TEST_OP_WRITE:
oid = *(rand_choose(context.oid_not_in_use));
- cout << "write oid " << oid << " current snap is "
+ cout << m_op << ": " << "write oid " << oid << " current snap is "
<< context.current_snap << std::endl;
- return new WriteOp(m_op, &context, oid, m_stats);
+ return new WriteOp(m_op, &context, oid, false, m_stats);
case TEST_OP_DELETE:
oid = *(rand_choose(context.oid_not_in_use));
- cout << "delete oid " << oid << " current snap is "
+ cout << m_op << ": " << "delete oid " << oid << " current snap is "
<< context.current_snap << std::endl;
return new DeleteOp(m_op, &context, oid, m_stats);
case TEST_OP_SNAP_CREATE:
- cout << "snap_create" << std::endl;
+ cout << m_op << ": " << "snap_create" << std::endl;
return new SnapCreateOp(m_op, &context, m_stats);
case TEST_OP_SNAP_REMOVE:
@@ -118,38 +124,33 @@ private:
int snap = rand_choose(context.snaps)->first;
if (context.snaps_in_use.lookup(snap))
continue; // in use; try again!
- cout << "snap_remove snap " << snap << std::endl;
+ cout << m_op << ": " << "snap_remove snap " << snap << std::endl;
return new SnapRemoveOp(m_op, &context, snap, m_stats);
}
case TEST_OP_ROLLBACK:
{
string oid = *(rand_choose(context.oid_not_in_use));
- cout << "rollback oid " << oid << std::endl;
+ cout << m_op << ": " << "rollback oid " << oid << " current snap is "
+ << context.current_snap << std::endl;
return new RollbackOp(m_op, &context, oid);
}
case TEST_OP_SETATTR:
oid = *(rand_choose(context.oid_not_in_use));
- cout << "setattr oid " << oid
+ cout << m_op << ": " << "setattr oid " << oid
<< " current snap is " << context.current_snap << std::endl;
return new SetAttrsOp(m_op, &context, oid, m_stats);
case TEST_OP_RMATTR:
oid = *(rand_choose(context.oid_not_in_use));
- cout << "rmattr oid " << oid
+ cout << m_op << ": " << "rmattr oid " << oid
<< " current snap is " << context.current_snap << std::endl;
return new RemoveAttrsOp(m_op, &context, oid, m_stats);
- case TEST_OP_TMAPPUT:
- oid = *(rand_choose(context.oid_not_in_use));
- cout << "tmapput oid " << oid
- << " current snap is " << context.current_snap << std::endl;
- return new TmapPutOp(m_op, &context, oid, m_stats);
-
case TEST_OP_WATCH:
oid = *(rand_choose(context.oid_not_in_use));
- cout << "watch oid " << oid
+ cout << m_op << ": " << "watch oid " << oid
<< " current snap is " << context.current_snap << std::endl;
return new WatchOp(m_op, &context, oid, m_stats);
@@ -158,12 +159,56 @@ private:
do {
oid2 = *(rand_choose(context.oid_not_in_use));
} while (oid == oid2);
- cout << "copy_from oid " << oid << " from oid " << oid2
+ cout << m_op << ": " << "copy_from oid " << oid << " from oid " << oid2
<< " current snap is " << context.current_snap << std::endl;
return new CopyFromOp(m_op, &context, oid, oid2, m_stats);
+ case TEST_OP_HIT_SET_LIST:
+ {
+ uint32_t hash = rjhash32(rand());
+ cout << m_op << ": " << "hit_set_list " << hash << std::endl;
+ return new HitSetListOp(m_op, &context, hash, m_stats);
+ }
+
+ case TEST_OP_UNDIRTY:
+ {
+ oid = *(rand_choose(context.oid_not_in_use));
+ cout << m_op << ": " << "undirty oid " << oid << std::endl;
+ return new UndirtyOp(m_op, &context, oid, m_stats);
+ }
+
+ case TEST_OP_IS_DIRTY:
+ {
+ oid = *(rand_choose(context.oid_not_flushing));
+ return new IsDirtyOp(m_op, &context, oid, m_stats);
+ }
+
+ case TEST_OP_CACHE_FLUSH:
+ {
+ oid = *(rand_choose(context.oid_not_in_use));
+ return new CacheFlushOp(m_op, &context, oid, m_stats, true);
+ }
+
+ case TEST_OP_CACHE_TRY_FLUSH:
+ {
+ oid = *(rand_choose(context.oid_not_in_use));
+ return new CacheFlushOp(m_op, &context, oid, m_stats, false);
+ }
+
+ case TEST_OP_CACHE_EVICT:
+ {
+ oid = *(rand_choose(context.oid_not_in_use));
+ return new CacheEvictOp(m_op, &context, oid, m_stats);
+ }
+
+ case TEST_OP_APPEND:
+ oid = *(rand_choose(context.oid_not_in_use));
+ cout << "append oid " << oid << " current snap is "
+ << context.current_snap << std::endl;
+ return new WriteOp(m_op, &context, oid, true, m_stats);
+
default:
- cerr << "Invalid op type " << type << std::endl;
+ cerr << m_op << ": Invalid op type " << type << std::endl;
assert(0);
}
}
@@ -177,6 +222,7 @@ private:
TestOpStat *m_stats;
map<TestOpType, unsigned int> m_weight_sums;
unsigned int m_total_weight;
+ bool m_ec_pool;
};
int main(int argc, char **argv)
@@ -191,23 +237,31 @@ int main(int argc, char **argv)
struct {
TestOpType op;
const char *name;
+ bool ec_pool_valid;
} op_types[] = {
- { TEST_OP_READ, "read" },
- { TEST_OP_WRITE, "write" },
- { TEST_OP_DELETE, "delete" },
- { TEST_OP_SNAP_CREATE, "snap_create" },
- { TEST_OP_SNAP_REMOVE, "snap_remove" },
- { TEST_OP_ROLLBACK, "rollback" },
- { TEST_OP_SETATTR, "setattr" },
- { TEST_OP_RMATTR, "rmattr" },
- { TEST_OP_TMAPPUT, "tmapput" },
- { TEST_OP_WATCH, "watch" },
- { TEST_OP_COPY_FROM, "copy_from" },
+ { TEST_OP_READ, "read", true },
+ { TEST_OP_WRITE, "write", false },
+ { TEST_OP_DELETE, "delete", true },
+ { TEST_OP_SNAP_CREATE, "snap_create", true },
+ { TEST_OP_SNAP_REMOVE, "snap_remove", true },
+ { TEST_OP_ROLLBACK, "rollback", true },
+ { TEST_OP_SETATTR, "setattr", true },
+ { TEST_OP_RMATTR, "rmattr", true },
+ { TEST_OP_WATCH, "watch", true },
+ { TEST_OP_COPY_FROM, "copy_from", true },
+ { TEST_OP_HIT_SET_LIST, "hit_set_list", true },
+ { TEST_OP_IS_DIRTY, "is_dirty", true },
+ { TEST_OP_UNDIRTY, "undirty", true },
+ { TEST_OP_CACHE_FLUSH, "cache_flush", true },
+ { TEST_OP_CACHE_TRY_FLUSH, "cache_try_flush", true },
+ { TEST_OP_CACHE_EVICT, "cache_evict", true },
+ { TEST_OP_APPEND, "append", true },
{ TEST_OP_READ /* grr */, NULL },
};
map<TestOpType, unsigned int> op_weights;
string pool_name = "data";
+ bool ec_pool = false;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "--max-ops") == 0)
@@ -226,7 +280,13 @@ int main(int argc, char **argv)
min_stride_size = atoi(argv[++i]);
else if (strcmp(argv[i], "--max-stride-size") == 0)
max_stride_size = atoi(argv[++i]);
- else if (strcmp(argv[i], "--op") == 0) {
+ else if (strcmp(argv[i], "--ec-pool") == 0) {
+ if (op_weights.size()) {
+ cerr << "--ec-pool must be specified prior to any ops" << std::endl;
+ exit(1);
+ }
+ ec_pool = true;
+ } else if (strcmp(argv[i], "--op") == 0) {
i++;
int j;
for (j = 0; op_types[j].name; ++j) {
@@ -242,9 +302,15 @@ int main(int argc, char **argv)
if (weight < 0) {
cerr << "Weights must be nonnegative." << std::endl;
return 1;
+ } else if (weight > 0) {
+ if (ec_pool && !op_types[j].ec_pool_valid) {
+ cerr << "Error: cannot use op type " << op_types[j].name
+ << " with --ec-pool" << std::endl;
+ exit(1);
+ }
+ cout << "adding op weight " << op_types[j].name << " -> " << weight << std::endl;
+ op_weights.insert(pair<TestOpType, unsigned int>(op_types[j].op, weight));
}
- cout << "adding op weight " << op_types[j].name << " -> " << weight << std::endl;
- op_weights.insert(pair<TestOpType, unsigned int>(op_types[j].op, weight));
} else {
cerr << "unknown arg " << argv[i] << std::endl;
//usage();
@@ -291,12 +357,20 @@ int main(int argc, char **argv)
}
char *id = getenv("CEPH_CLIENT_ID");
- VarLenGenerator cont_gen(size, min_stride_size, max_stride_size);
- RadosTestContext context(pool_name, max_in_flight, cont_gen, id);
+ RadosTestContext context(
+ pool_name,
+ max_in_flight,
+ size,
+ min_stride_size,
+ max_stride_size,
+ ec_pool,
+ id);
TestOpStat stats;
- WeightedTestGenerator gen = WeightedTestGenerator(ops, objects,
- op_weights, &stats, max_seconds);
+ WeightedTestGenerator gen = WeightedTestGenerator(
+ ops, objects,
+ op_weights, &stats, max_seconds,
+ ec_pool);
int r = context.init();
if (r < 0) {
cerr << "Error initializing rados test context: "
diff --git a/src/test/osd/ceph_erasure_code_benchmark.cc b/src/test/osd/ceph_erasure_code_benchmark.cc
new file mode 100644
index 0000000..f4fdaf3
--- /dev/null
+++ b/src/test/osd/ceph_erasure_code_benchmark.cc
@@ -0,0 +1,218 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "test/osd/ceph_erasure_code_benchmark.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "common/Clock.h"
+#include "include/utime.h"
+#include "osd/ErasureCodePlugin.h"
+
+namespace po = boost::program_options;
+
+int ErasureCodeBench::setup(int argc, char** argv) {
+
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help,h", "produce help message")
+ ("size,s", po::value<int>()->default_value(1024 * 1024),
+ "size of the buffer to be encoded")
+ ("iterations,i", po::value<int>()->default_value(1),
+ "number of encode/decode runs")
+ ("plugin,p", po::value<string>()->default_value("jerasure"),
+ "erasure code plugin name")
+ ("workload,w", po::value<string>()->default_value("encode"),
+ "run either encode or decode")
+ ("erasures,e", po::value<int>()->default_value(1),
+ "number of erasures when decoding")
+ ("parameter,P", po::value<vector<string> >(),
+ "parameters")
+ ;
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+ po::store(
+ parsed,
+ vm);
+ po::notify(vm);
+
+ vector<const char *> ceph_options, def_args;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ global_init(
+ &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ if (vm.count("help")) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ if (vm.count("parameter")) {
+ const vector<string> &p = vm["parameter"].as< vector<string> >();
+ for (vector<string>::const_iterator i = p.begin();
+ i != p.end();
+ i++) {
+ std::vector<std::string> strs;
+ boost::split(strs, *i, boost::is_any_of("="));
+ if (strs.size() != 2) {
+ cerr << "--parameter " << *i << " ignored because it does not contain exactly one =" << endl;
+ } else {
+ parameters[strs[0]] = strs[1];
+ }
+ }
+ }
+
+ if (parameters.count("erasure-code-directory") == 0)
+ parameters["erasure-code-directory"] = ".libs";
+
+ in_size = vm["size"].as<int>();
+ max_iterations = vm["iterations"].as<int>();
+ plugin = vm["plugin"].as<string>();
+ workload = vm["workload"].as<string>();
+ erasures = vm["erasures"].as<int>();
+
+ return 0;
+}
+
+int ErasureCodeBench::run() {
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ instance.disable_dlclose = true;
+
+ if (workload == "encode")
+ return encode();
+ else
+ return decode();
+}
+
+int ErasureCodeBench::encode()
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ int code = instance.factory(plugin, parameters, &erasure_code);
+ if (code)
+ return code;
+ int k = atoi(parameters["erasure-code-k"].c_str());
+ int m = atoi(parameters["erasure-code-m"].c_str());
+
+ bufferlist in;
+ in.append(string(in_size, 'X'));
+ set<int> want_to_encode;
+ for (int i = 0; i < k + m; i++) {
+ want_to_encode.insert(i);
+ }
+ utime_t begin_time = ceph_clock_now(g_ceph_context);
+ for (int i = 0; i < max_iterations; i++) {
+ map<int,bufferlist> encoded;
+ code = erasure_code->encode(want_to_encode, in, &encoded);
+ if (code)
+ return code;
+ }
+ utime_t end_time = ceph_clock_now(g_ceph_context);
+ cout << (end_time - begin_time) << "\t" << (max_iterations * (in_size / 1024)) << endl;
+ return 0;
+}
+
+int ErasureCodeBench::decode()
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ int code = instance.factory(plugin, parameters, &erasure_code);
+ if (code)
+ return code;
+ int k = atoi(parameters["erasure-code-k"].c_str());
+ int m = atoi(parameters["erasure-code-m"].c_str());
+
+ bufferlist in;
+ in.append(string(in_size, 'X'));
+
+ set<int> want_to_encode;
+ for (int i = 0; i < k + m; i++) {
+ want_to_encode.insert(i);
+ }
+
+ map<int,bufferlist> encoded;
+ code = erasure_code->encode(want_to_encode, in, &encoded);
+ if (code)
+ return code;
+
+ set<int> want_to_read = want_to_encode;
+
+ utime_t begin_time = ceph_clock_now(g_ceph_context);
+ for (int i = 0; i < max_iterations; i++) {
+ map<int,bufferlist> chunks = encoded;
+ for (int j = 0; j < erasures; j++) {
+ int erasure;
+ do {
+ erasure = rand() % ( k + m );
+ } while(chunks.count(erasure) == 0);
+ chunks.erase(erasure);
+ }
+ map<int,bufferlist> decoded;
+ code = erasure_code->decode(want_to_read, chunks, &decoded);
+ if (code)
+ return code;
+ }
+ utime_t end_time = ceph_clock_now(g_ceph_context);
+ cout << (end_time - begin_time) << "\t" << (max_iterations * (in_size / 1024)) << endl;
+ return 0;
+}
+
+int main(int argc, char** argv) {
+ ErasureCodeBench ecbench;
+ int err = ecbench.setup(argc, argv);
+ if (err)
+ return err;
+ return ecbench.run();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make ceph_erasure_code_benchmark &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./ceph_erasure_code_benchmark \
+ * --plugin jerasure \
+ * --parameter erasure-code-directory=.libs \
+ * --parameter erasure-code-technique=reed_sol_van \
+ * --parameter erasure-code-k=2 \
+ * --parameter erasure-code-m=2 \
+ * --iterations 1
+ * "
+ * End:
+ */
diff --git a/src/test/osd/ceph_erasure_code_benchmark.h b/src/test/osd/ceph_erasure_code_benchmark.h
new file mode 100644
index 0000000..8ea60f9
--- /dev/null
+++ b/src/test/osd/ceph_erasure_code_benchmark.h
@@ -0,0 +1,38 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_ERASURE_CODE_BENCHMARK_H
+#define CEPH_ERASURE_CODE_BENCHMARK_H
+
+#include <string>
+
+using namespace std;
+
+class ErasureCodeBench {
+ int in_size;
+ int max_iterations;
+ string plugin;
+ int erasures;
+ string workload;
+ map<string,string> parameters;
+public:
+ int setup(int argc, char** argv);
+ int run();
+ int decode();
+ int encode();
+};
+
+#endif
diff --git a/src/test/osd/hitset.cc b/src/test/osd/hitset.cc
new file mode 100644
index 0000000..9e8ee5e
--- /dev/null
+++ b/src/test/osd/hitset.cc
@@ -0,0 +1,197 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ * Copyright 2013 Inktank
+ */
+
+#include "gtest/gtest.h"
+#include "osd/HitSet.h"
+#include <iostream>
+
+class HitSetTestStrap {
+public:
+ HitSet *hitset;
+
+ HitSetTestStrap(HitSet *h) : hitset(h) {}
+
+ void fill(unsigned count) {
+ char buf[50];
+ for (unsigned i = 0; i < count; ++i) {
+ sprintf(buf, "hitsettest_%d", i);
+ hobject_t obj(object_t(buf), "", 0, i, 0, "");
+ hitset->insert(obj);
+ }
+ EXPECT_EQ(count, hitset->insert_count());
+ }
+ void verify_fill(unsigned count) {
+ char buf[50];
+ for (unsigned i = 0; i < count; ++i) {
+ sprintf(buf, "hitsettest_%d", i);
+ hobject_t obj(object_t(buf), "", 0, i, 0, "");
+ EXPECT_TRUE(hitset->contains(obj));
+ }
+ }
+
+};
+
+class BloomHitSetTest : public testing::Test, public HitSetTestStrap {
+public:
+
+ BloomHitSetTest() : HitSetTestStrap(new HitSet(new BloomHitSet)) {}
+
+ void rebuild(double fp, uint64_t target, uint64_t seed) {
+ BloomHitSet::Params *bparams = new BloomHitSet::Params(fp, target, seed);
+ HitSet::Params param(bparams);
+ HitSet new_set(param);
+ *hitset = new_set;
+ }
+
+ BloomHitSet *get_hitset() { return static_cast<BloomHitSet*>(hitset->impl.get()); }
+};
+
+TEST_F(BloomHitSetTest, Params) {
+ BloomHitSet::Params params(0.01, 100, 5);
+ EXPECT_EQ(.01, params.get_fpp());
+ EXPECT_EQ((unsigned)100, params.target_size);
+ EXPECT_EQ((unsigned)5, params.seed);
+ params.set_fpp(0.1);
+ EXPECT_EQ(0.1, params.get_fpp());
+
+ bufferlist bl;
+ params.encode(bl);
+ BloomHitSet::Params p2;
+ bufferlist::iterator iter = bl.begin();
+ p2.decode(iter);
+ EXPECT_EQ(0.1, p2.get_fpp());
+ EXPECT_EQ((unsigned)100, p2.target_size);
+ EXPECT_EQ((unsigned)5, p2.seed);
+}
+
+TEST_F(BloomHitSetTest, Construct) {
+ ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_BLOOM);
+ // success!
+}
+
+TEST_F(BloomHitSetTest, Rebuild) {
+ rebuild(0.1, 100, 1);
+ ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_BLOOM);
+}
+
+TEST_F(BloomHitSetTest, InsertsMatch) {
+ rebuild(0.1, 100, 1);
+ fill(50);
+ /*
+ * the approx unique count is atrocious on bloom filters. Empirical
+ * evidence suggests the current test will produce a value of 62
+ * regardless of hitset size
+ */
+ EXPECT_TRUE(hitset->approx_unique_insert_count() >= 50 &&
+ hitset->approx_unique_insert_count() <= 62);
+ verify_fill(50);
+ EXPECT_FALSE(hitset->is_full());
+}
+
+TEST_F(BloomHitSetTest, FillsUp) {
+ rebuild(0.1, 20, 1);
+ fill(20);
+ verify_fill(20);
+ EXPECT_TRUE(hitset->is_full());
+}
+
+TEST_F(BloomHitSetTest, RejectsNoMatch) {
+ rebuild(0.001, 100, 1);
+ fill(100);
+ verify_fill(100);
+ EXPECT_TRUE(hitset->is_full());
+
+ char buf[50];
+ int matches = 0;
+ for (int i = 100; i < 200; ++i) {
+ sprintf(buf, "hitsettest_%d", i);
+ hobject_t obj(object_t(buf), "", 0, i, 0, "");
+ if (hitset->contains(obj))
+ ++matches;
+ }
+ // we set a 1 in 1000 false positive; allow one in our 100
+ EXPECT_LT(matches, 2);
+}
+
+class ExplicitHashHitSetTest : public testing::Test, public HitSetTestStrap {
+public:
+
+ ExplicitHashHitSetTest() : HitSetTestStrap(new HitSet(new ExplicitHashHitSet)) {}
+
+ ExplicitHashHitSet *get_hitset() { return static_cast<ExplicitHashHitSet*>(hitset->impl.get()); }
+};
+
+TEST_F(ExplicitHashHitSetTest, Construct) {
+ ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_EXPLICIT_HASH);
+ // success!
+}
+
+TEST_F(ExplicitHashHitSetTest, InsertsMatch) {
+ fill(50);
+ verify_fill(50);
+ EXPECT_EQ((unsigned)50, hitset->approx_unique_insert_count());
+ EXPECT_FALSE(hitset->is_full());
+}
+
+TEST_F(ExplicitHashHitSetTest, RejectsNoMatch) {
+ fill(100);
+ verify_fill(100);
+ EXPECT_FALSE(hitset->is_full());
+
+ char buf[50];
+ int matches = 0;
+ for (int i = 100; i < 200; ++i) {
+ sprintf(buf, "hitsettest_%d", i);
+ hobject_t obj(object_t(buf), "", 0, i, 0, "");
+ if (hitset->contains(obj)) {
+ ++matches;
+ }
+ }
+ EXPECT_EQ(matches, 0);
+}
+
+class ExplicitObjectHitSetTest : public testing::Test, public HitSetTestStrap {
+public:
+
+ ExplicitObjectHitSetTest() : HitSetTestStrap(new HitSet(new ExplicitObjectHitSet)) {}
+
+ ExplicitObjectHitSet *get_hitset() { return static_cast<ExplicitObjectHitSet*>(hitset->impl.get()); }
+};
+
+TEST_F(ExplicitObjectHitSetTest, Construct) {
+ ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_EXPLICIT_OBJECT);
+ // success!
+}
+
+TEST_F(ExplicitObjectHitSetTest, InsertsMatch) {
+ fill(50);
+ verify_fill(50);
+ EXPECT_EQ((unsigned)50, hitset->approx_unique_insert_count());
+ EXPECT_FALSE(hitset->is_full());
+}
+
+TEST_F(ExplicitObjectHitSetTest, RejectsNoMatch) {
+ fill(100);
+ verify_fill(100);
+ EXPECT_FALSE(hitset->is_full());
+
+ char buf[50];
+ int matches = 0;
+ for (int i = 100; i < 200; ++i) {
+ sprintf(buf, "hitsettest_%d", i);
+ hobject_t obj(object_t(buf), "", 0, i, 0, "");
+ if (hitset->contains(obj)) {
+ ++matches;
+ }
+ }
+ EXPECT_EQ(matches, 0);
+}
diff --git a/src/test/osd/osdcap.cc b/src/test/osd/osdcap.cc
index 8fc3ddd..a9aefd7 100644
--- a/src/test/osd/osdcap.cc
+++ b/src/test/osd/osdcap.cc
@@ -60,6 +60,7 @@ const char *parse_good[] = {
" allow pool foo rwx; allow pool bar r ",
" allow pool foo rwx; allow pool bar r ",
" allow wx pool taco",
+ "\tallow\nwx\tpool \n taco\t",
"allow r pool foo object_prefix blah ; allow w auid 5",
"allow class-read object_prefix rbd_children, allow pool libvirt-pool-test rwx",
"allow class-read object_prefix rbd-children, allow pool libvirt_pool_test rwx",
diff --git a/src/test/osdc/object_cacher_stress.cc b/src/test/osdc/object_cacher_stress.cc
index ee71b0e..4f6fffe 100644
--- a/src/test/osdc/object_cacher_stress.cc
+++ b/src/test/osdc/object_cacher_stress.cc
@@ -67,7 +67,7 @@ int stress_test(uint64_t num_ops, uint64_t num_objs,
obc.start();
atomic_t outstanding_reads;
- vector<std::tr1::shared_ptr<op_data> > ops;
+ vector<ceph::shared_ptr<op_data> > ops;
ObjectCacher::ObjectSet object_set(NULL, 0, 0);
SnapContext snapc;
ceph::buffer::ptr bp(max_op_len);
@@ -91,7 +91,7 @@ int stress_test(uint64_t num_ops, uint64_t num_objs,
uint64_t length = random() % (MAX(max_len - 1, 1)) + 1;
std::string oid = "test" + stringify(random() % num_objs);
bool is_read = random() < percent_reads * RAND_MAX;
- std::tr1::shared_ptr<op_data> op(new op_data(oid, offset, length, is_read));
+ ceph::shared_ptr<op_data> op(new op_data(oid, offset, length, is_read));
ops.push_back(op);
std::cout << "op " << i << " " << (is_read ? "read" : "write")
<< " " << op->extent << "\n";
diff --git a/src/test/run-cli-tests b/src/test/run-cli-tests
index 321c1de..fec60de 100755
--- a/src/test/run-cli-tests
+++ b/src/test/run-cli-tests
@@ -42,7 +42,7 @@ FAILED=0
for tool in "$SRCDIR"/cli/*; do
toolname="$(basename "$tool")"
install -d -m0755 -- "$BUILDDIR/cli/$toolname"
- if ! env --ignore-environment \
+ if ! env -i \
PATH="$BUILDDIR_ABS/..:$SRCDIR_ABS/..:$PATH" \
CEPH_CONF=/dev/null \
CCACHE_DIR="$CCACHE_DIR" \
diff --git a/src/test/streamtest.cc b/src/test/streamtest.cc
index 21693ac..d4ccadf 100644
--- a/src/test/streamtest.cc
+++ b/src/test/streamtest.cc
@@ -29,15 +29,15 @@ struct io {
};
map<off_t,io> writes;
Cond cond;
-Mutex lock("streamtest.cc lock");
+Mutex test_lock("streamtest.cc lock");
unsigned concurrent = 1;
void throttle()
{
- Mutex::Locker l(lock);
+ Mutex::Locker l(test_lock);
while (writes.size() >= concurrent) {
//generic_dout(0) << "waiting" << dendl;
- cond.Wait(lock);
+ cond.Wait(test_lock);
}
}
@@ -60,13 +60,13 @@ void pr(off_t off)
void set_start(off_t off, utime_t t)
{
- Mutex::Locker l(lock);
+ Mutex::Locker l(test_lock);
writes[off].start = t;
}
void set_ack(off_t off, utime_t t)
{
- Mutex::Locker l(lock);
+ Mutex::Locker l(test_lock);
//generic_dout(0) << "ack " << off << dendl;
writes[off].ack = t;
if (writes[off].done())
@@ -75,7 +75,7 @@ void set_ack(off_t off, utime_t t)
void set_commit(off_t off, utime_t t)
{
- Mutex::Locker l(lock);
+ Mutex::Locker l(test_lock);
//generic_dout(0) << "commit " << off << dendl;
writes[off].commit = t;
if (writes[off].done())
diff --git a/src/test/test_arch.c b/src/test/test_arch.cc
similarity index 83%
rename from src/test/test_arch.c
rename to src/test/test_arch.cc
index 549221e..7b7ffe4 100644
--- a/src/test/test_arch.c
+++ b/src/test/test_arch.cc
@@ -13,6 +13,7 @@ int main(int argc, char **argv)
assert(ceph_arch_probed);
printf("ceph_arch_intel_sse42 = %d\n", ceph_arch_intel_sse42);
+ printf("ceph_arch_intel_sse2 = %d\n", ceph_arch_intel_sse2);
printf("ceph_arch_neon = %d\n", ceph_arch_neon);
return 0;
diff --git a/src/test/test_c_headers.c b/src/test/test_c_headers.c
new file mode 100644
index 0000000..0f41966
--- /dev/null
+++ b/src/test/test_c_headers.c
@@ -0,0 +1,22 @@
+#include "include/cephfs/libcephfs.h"
+#include "include/rados/librados.h"
+
+#ifdef __cplusplus
+#error "test invalid: only use C mode"
+#endif
+
+int main(int argc, char **argv)
+{
+ int ret;
+ (void)ret; // squash unused warning
+
+ /* librados.h */
+ rados_t cluster;
+ ret = rados_create(&cluster, NULL);
+
+ /* libcephfs.h */
+ struct ceph_mount_info *cmount;
+ ret = ceph_create(&cmount, NULL);
+
+ return 0;
+}
diff --git a/src/test/test_crushwrapper.cc b/src/test/test_crushwrapper.cc
new file mode 100644
index 0000000..eeec8f7
--- /dev/null
+++ b/src/test/test_crushwrapper.cc
@@ -0,0 +1,107 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank <info at inktank.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include "crush/CrushWrapper.h"
+#include "gtest/gtest.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+#include "common/ceph_argparse.h"
+#include "common/Formatter.h"
+
+TEST(CrushWrapper, distance) {
+ CrushWrapper c;
+ c.create();
+ c.set_type_name(1, "host");
+ c.set_type_name(2, "rack");
+ c.set_type_name(3, "root");
+ int bno;
+ int r = c.add_bucket(0, CRUSH_BUCKET_STRAW,
+ CRUSH_HASH_DEFAULT, 3, 0, NULL,
+ NULL, &bno);
+ ASSERT_EQ(0, r);
+ ASSERT_EQ(-1, bno);
+ c.set_item_name(bno, "default");
+
+ c.set_max_devices(10);
+
+ //JSONFormatter jf(true);
+
+ map<string,string> loc;
+ loc["host"] = "a1";
+ loc["rack"] = "a";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 0, 1, "osd.0", loc);
+
+ loc.clear();
+ loc["host"] = "a2";
+ loc["rack"] = "a";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 1, 1, "osd.1", loc);
+
+ loc.clear();
+ loc["host"] = "b1";
+ loc["rack"] = "b";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 2, 1, "osd.2", loc);
+
+ loc.clear();
+ loc["host"] = "b2";
+ loc["rack"] = "b";
+ loc["root"] = "default";
+ c.insert_item(g_ceph_context, 3, 1, "osd.3", loc);
+
+ vector<pair<string,string> > ol;
+ c.get_full_location_ordered(3, ol);
+ ASSERT_EQ(3u, ol.size());
+ ASSERT_EQ(make_pair(string("host"),string("b2")), ol[0]);
+ ASSERT_EQ(make_pair(string("rack"),string("b")), ol[1]);
+ ASSERT_EQ(make_pair(string("root"),string("default")), ol[2]);
+
+ //c.dump(&jf);
+ //jf.flush(cout);
+
+ multimap<string,string> p;
+ p.insert(make_pair("host","b2"));
+ p.insert(make_pair("rack","b"));
+ p.insert(make_pair("root","default"));
+ ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 0, p));
+ ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 1, p));
+ ASSERT_EQ(2, c.get_common_ancestor_distance(g_ceph_context, 2, p));
+ ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
+ ASSERT_EQ(-ENOENT, c.get_common_ancestor_distance(g_ceph_context, 123, p));
+
+ // make sure a "multipath" location will reflect a minimal
+ // distance for both paths
+ p.insert(make_pair("host","b1"));
+ ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 2, p));
+ ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
+}
+
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+ env_to_vec(args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/src/test/test_filejournal.cc b/src/test/test_filejournal.cc
index 7365e97..3691fa6 100644
--- a/src/test/test_filejournal.cc
+++ b/src/test/test_filejournal.cc
@@ -22,15 +22,15 @@ bool aio = false;
// ----
Cond cond;
-Mutex lock("lock");
+Mutex wait_lock("lock");
bool done;
void wait()
{
- lock.Lock();
+ wait_lock.Lock();
while (!done)
- cond.Wait(lock);
- lock.Unlock();
+ cond.Wait(wait_lock);
+ wait_lock.Unlock();
}
// ----
@@ -120,7 +120,7 @@ TEST(TestFileJournal, WriteSmall) {
bufferlist bl;
bl.append("small");
- j.submit_entry(1, bl, 0, new C_SafeCond(&lock, &cond, &done));
+ j.submit_entry(1, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
wait();
j.close();
@@ -138,7 +138,7 @@ TEST(TestFileJournal, WriteBig) {
memset(foo, 1, sizeof(foo));
bl.append(foo, sizeof(foo));
}
- j.submit_entry(1, bl, 0, new C_SafeCond(&lock, &cond, &done));
+ j.submit_entry(1, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
wait();
j.close();
@@ -150,7 +150,7 @@ TEST(TestFileJournal, WriteMany) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
bufferlist bl;
bl.append("small");
@@ -173,7 +173,7 @@ TEST(TestFileJournal, WriteManyVecs) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
bufferlist first;
first.append("small");
@@ -210,7 +210,7 @@ TEST(TestFileJournal, ReplaySmall) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
bufferlist bl;
bl.append("small");
@@ -255,7 +255,7 @@ TEST(TestFileJournal, ReplayCorrupt) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
const char *needle = "i am a needle";
const char *newneedle = "in a haystack";
@@ -403,7 +403,7 @@ TEST(TestFileJournal, ReplayDetectCorruptFooterMagic) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
const char *needle = "i am a needle";
for (unsigned i = 1; i <= 4; ++i) {
@@ -416,7 +416,7 @@ TEST(TestFileJournal, ReplayDetectCorruptFooterMagic) {
bufferlist bl;
bl.append("needle");
- j.submit_entry(5, bl, 0, new C_SafeCond(&lock, &cond, &done));
+ j.submit_entry(5, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
wait();
j.close();
@@ -453,7 +453,7 @@ TEST(TestFileJournal, ReplayDetectCorruptPayload) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
const char *needle = "i am a needle";
for (unsigned i = 1; i <= 4; ++i) {
@@ -466,7 +466,7 @@ TEST(TestFileJournal, ReplayDetectCorruptPayload) {
bufferlist bl;
bl.append("needle");
- j.submit_entry(5, bl, 0, new C_SafeCond(&lock, &cond, &done));
+ j.submit_entry(5, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
wait();
j.close();
@@ -503,7 +503,7 @@ TEST(TestFileJournal, ReplayDetectCorruptHeader) {
ASSERT_EQ(0, j.create());
j.make_writeable();
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&lock, &cond, &done));
+ C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
const char *needle = "i am a needle";
for (unsigned i = 1; i <= 4; ++i) {
@@ -516,7 +516,7 @@ TEST(TestFileJournal, ReplayDetectCorruptHeader) {
bufferlist bl;
bl.append("needle");
- j.submit_entry(5, bl, 0, new C_SafeCond(&lock, &cond, &done));
+ j.submit_entry(5, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
wait();
j.close();
diff --git a/src/test/test_get_blkdev_size.cc b/src/test/test_get_blkdev_size.cc
new file mode 100644
index 0000000..ba28f1c
--- /dev/null
+++ b/src/test/test_get_blkdev_size.cc
@@ -0,0 +1,35 @@
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "common/blkdev.h"
+
+int main(int argc, char **argv)
+{
+ int fd, ret;
+ int64_t size;
+
+ if (argc != 2) {
+ fprintf(stderr, "usage: %s <blkdev>\n", argv[0]);
+ return -1;
+ }
+
+ fd = open(argv[1], O_RDONLY);
+ if (fd < 0) {
+ perror("open");
+ return -1;
+ }
+
+ ret = get_block_device_size(fd, &size);
+ if (ret < 0) {
+ fprintf(stderr, "get_block_device_size: %s\n", strerror(-ret));
+ return -1;
+ }
+
+ fprintf(stdout, "%" PRId64, size);
+
+ return 0;
+}
diff --git a/src/test/test_ipaddr.cc b/src/test/test_ipaddr.cc
index f59f0af..8200821 100644
--- a/src/test/test_ipaddr.cc
+++ b/src/test/test_ipaddr.cc
@@ -85,6 +85,28 @@ TEST(CommonIPAddr, TestV4_Prefix25)
ASSERT_EQ((struct sockaddr*)&a_two, result);
}
+TEST(CommonIPAddr, TestV4_Prefix16)
+{
+ struct ifaddrs one, two;
+ struct sockaddr_in a_one;
+ struct sockaddr_in a_two;
+ struct sockaddr_in net;
+ const struct sockaddr *result;
+
+ one.ifa_next = &two;
+ one.ifa_addr = (struct sockaddr*)&a_one;
+
+ two.ifa_next = NULL;
+ two.ifa_addr = (struct sockaddr*)&a_two;
+
+ ipv4(&a_one, "10.1.1.2");
+ ipv4(&a_two, "10.2.1.123");
+ ipv4(&net, "10.2.0.0");
+
+ result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 16);
+ ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
TEST(CommonIPAddr, TestV4_PrefixTooLong)
{
struct ifaddrs one;
diff --git a/src/test/test_osd_types.cc b/src/test/test_osd_types.cc
index 3467435..bcc71c3 100644
--- a/src/test/test_osd_types.cc
+++ b/src/test/test_osd_types.cc
@@ -126,11 +126,11 @@ TEST(pg_interval_t, check_new_interval)
//
int osd_id = 1;
epoch_t epoch = 40;
- std::tr1::shared_ptr<OSDMap> osdmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> osdmap(new OSDMap());
osdmap->set_max_osd(10);
osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
osdmap->set_epoch(epoch);
- std::tr1::shared_ptr<OSDMap> lastmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> lastmap(new OSDMap());
lastmap->set_max_osd(10);
lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
lastmap->set_epoch(epoch);
@@ -184,7 +184,7 @@ TEST(pg_interval_t, check_new_interval)
// pool did not exist in the old osdmap
//
{
- std::tr1::shared_ptr<OSDMap> lastmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> lastmap(new OSDMap());
lastmap->set_max_osd(10);
lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
lastmap->set_epoch(epoch);
@@ -272,7 +272,7 @@ TEST(pg_interval_t, check_new_interval)
// PG is splitting
//
{
- std::tr1::shared_ptr<OSDMap> osdmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> osdmap(new OSDMap());
osdmap->set_max_osd(10);
osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
osdmap->set_epoch(epoch);
@@ -307,7 +307,7 @@ TEST(pg_interval_t, check_new_interval)
// PG size has changed
//
{
- std::tr1::shared_ptr<OSDMap> osdmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> osdmap(new OSDMap());
osdmap->set_max_osd(10);
osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
osdmap->set_epoch(epoch);
@@ -383,7 +383,7 @@ TEST(pg_interval_t, check_new_interval)
// The new osdmap is created so that it triggers the
// bug.
//
- std::tr1::shared_ptr<OSDMap> osdmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> osdmap(new OSDMap());
osdmap->set_max_osd(10);
osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
osdmap->set_epoch(epoch);
@@ -455,7 +455,7 @@ TEST(pg_interval_t, check_new_interval)
new_acting.push_back(osd_id + 4);
new_acting.push_back(osd_id + 5);
- std::tr1::shared_ptr<OSDMap> lastmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> lastmap(new OSDMap());
lastmap->set_max_osd(10);
lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
lastmap->set_epoch(epoch);
@@ -500,7 +500,7 @@ TEST(pg_interval_t, check_new_interval)
epoch_t last_epoch_clean = epoch - 10;
- std::tr1::shared_ptr<OSDMap> lastmap(new OSDMap());
+ ceph::shared_ptr<OSDMap> lastmap(new OSDMap());
lastmap->set_max_osd(10);
lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
lastmap->set_epoch(epoch);
@@ -1160,6 +1160,12 @@ TEST_F(ObjectContextTest, read_write_lock)
}
-// Local Variables:
-// compile-command: "cd .. ; make unittest_osd_types ; ./unittest_osd_types # --gtest_filter=pg_missing_t.constructor "
-// End:
+/*
+ * Local Variables:
+ * compile-command: "cd .. ;
+ * make unittest_osd_types ;
+ * ./unittest_osd_types # --gtest_filter=pg_missing_t.constructor
+ * "
+ * End:
+ */
+
diff --git a/src/test/test_rgw_admin_opstate.cc b/src/test/test_rgw_admin_opstate.cc
index 3f173d1..9c8835c 100644
--- a/src/test/test_rgw_admin_opstate.cc
+++ b/src/test/test_rgw_admin_opstate.cc
@@ -809,7 +809,7 @@ int main(int argc, char *argv[]){
global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
common_init_finish(g_ceph_context);
- store = RGWStoreManager::get_storage(g_ceph_context, false);
+ store = RGWStoreManager::get_storage(g_ceph_context, false, false);
g_test = new admin_log::test_helper();
finisher = new Finisher(g_ceph_context);
#ifdef GTEST
diff --git a/src/test/test_snap_mapper.cc b/src/test/test_snap_mapper.cc
index 41ba044..eaf308c 100644
--- a/src/test/test_snap_mapper.cc
+++ b/src/test/test_snap_mapper.cc
@@ -1,5 +1,5 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#include <tr1/memory>
+#include "include/memory.h"
#include <map>
#include <set>
#include <boost/scoped_ptr.hpp>
@@ -42,7 +42,7 @@ class PausyAsyncMap : public MapCacher::StoreDriver<string, bufferlist> {
virtual void operate(map<string, bufferlist> *store) = 0;
virtual ~_Op() {}
};
- typedef std::tr1::shared_ptr<_Op> Op;
+ typedef ceph::shared_ptr<_Op> Op;
struct Remove : public _Op {
set<string> to_remove;
Remove(const set<string> &to_remove) : to_remove(to_remove) {}
@@ -582,7 +582,7 @@ public:
class SnapMapperTest : public ::testing::Test {
protected:
boost::scoped_ptr< PausyAsyncMap > driver;
- map<pg_t, std::tr1::shared_ptr<MapperVerifier> > mappers;
+ map<pg_t, ceph::shared_ptr<MapperVerifier> > mappers;
uint32_t pgnum;
virtual void SetUp() {
diff --git a/src/test/test_striper.cc b/src/test/test_striper.cc
index 262fba3..9946142 100644
--- a/src/test/test_striper.cc
+++ b/src/test/test_striper.cc
@@ -26,6 +26,37 @@ TEST(Striper, Stripe1)
ASSERT_EQ(94208u, ex[2].truncate_size);
}
+TEST(Striper, EmptyPartialResult)
+{
+ ceph_file_layout l;
+ memset(&l, 0, sizeof(l));
+
+ l.fl_object_size = 4194304;
+ l.fl_stripe_unit = 4194304;
+ l.fl_stripe_count = 1;
+
+ vector<ObjectExtent> ex;
+ Striper::file_to_extents(g_ceph_context, 1, &l, 725549056, 131072, 72554905600, ex);
+ cout << "ex " << ex << std::endl;
+ ASSERT_EQ(2u, ex.size());
+
+ Striper::StripedReadResult r;
+
+ bufferlist bl;
+ r.add_partial_result(g_ceph_context, bl, ex[1].buffer_extents);
+
+ bufferptr bp(65536);
+ bp.zero();
+ bl.append(bp);
+
+ r.add_partial_result(g_ceph_context, bl, ex[0].buffer_extents);
+
+ bufferlist outbl;
+ r.assemble_result(g_ceph_context, outbl, false);
+
+ ASSERT_EQ(65536u, outbl.length());
+}
+
int main(int argc, char **argv)
diff --git a/src/test/testmsgr.cc b/src/test/testmsgr.cc
index 4de779b..bcde69f 100644
--- a/src/test/testmsgr.cc
+++ b/src/test/testmsgr.cc
@@ -39,7 +39,7 @@ using namespace std;
Messenger *messenger = 0;
-Mutex lock("mylock");
+Mutex test_lock("mylock");
Cond cond;
uint64_t received = 0;
@@ -55,10 +55,10 @@ private:
//cerr << "got ping from " << m->get_source() << std::endl;
dout(0) << "got ping from " << m->get_source() << dendl;
- lock.Lock();
+ test_lock.Lock();
++received;
cond.Signal();
- lock.Unlock();
+ test_lock.Unlock();
m->put();
return true;
@@ -112,13 +112,13 @@ int main(int argc, const char **argv, const char *envp[]) {
if (whoami == 0)
isend = 100;
- lock.Lock();
+ test_lock.Lock();
uint64_t sent = 0;
while (1) {
while (received + isend <= sent) {
//cerr << "wait r " << received << " s " << sent << " is " << isend << std::endl;
dout(0) << "wait r " << received << " s " << sent << " is " << isend << dendl;
- cond.Wait(lock);
+ cond.Wait(test_lock);
}
int t = rand() % mc.get_num_mon();
@@ -135,7 +135,7 @@ int main(int argc, const char **argv, const char *envp[]) {
messenger->send_message(new MPing, mc.get_mon_inst(t));
cerr << isend << "\t" << ++sent << "\t" << received << "\r";
}
- lock.Unlock();
+ test_lock.Unlock();
// wait for messenger to finish
rank->wait();
diff --git a/src/test/xattr_bench.cc b/src/test/xattr_bench.cc
index 2baaee8..2b8d09f 100644
--- a/src/test/xattr_bench.cc
+++ b/src/test/xattr_bench.cc
@@ -29,7 +29,7 @@
#include <boost/random/binomial_distribution.hpp>
#include <gtest/gtest.h>
-#include <ext/hash_map>
+#include "include/unordered_map.h"
void usage(const string &name) {
std::cerr << "Usage: " << name << " [xattr|omap] store_path store_journal"
diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am
index db86067..0a386cc 100644
--- a/src/tools/Makefile.am
+++ b/src/tools/Makefile.am
@@ -1,9 +1,9 @@
ceph_osdomap_tool_SOURCES = tools/ceph-osdomap-tool.cc
-ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
bin_DEBUGPROGRAMS += ceph-osdomap-tool
ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc
-ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
bin_DEBUGPROGRAMS += ceph-monstore-tool
ceph_kvstore_tool_SOURCES = tools/ceph-kvstore-tool.cc
@@ -20,14 +20,14 @@ endif # LINUX
bin_PROGRAMS += ceph_filestore_tool
ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
-ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
if LINUX
ceph_filestore_dump_LDADD += -ldl
endif # LINUX
bin_PROGRAMS += ceph_filestore_dump
monmaptool_SOURCES = tools/monmaptool.cc
-monmaptool_LDADD = $(CEPH_GLOBAL)
+monmaptool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
bin_PROGRAMS += monmaptool
crushtool_SOURCES = tools/crushtool.cc
@@ -83,11 +83,11 @@ endif # WITH_SYSTEM_LIBS3
endif # WITH_REST_BENCH
ceph_conf_SOURCES = tools/ceph_conf.cc
-ceph_conf_LDADD = $(CEPH_GLOBAL)
+ceph_conf_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
bin_PROGRAMS += ceph-conf
ceph_authtool_SOURCES = tools/ceph_authtool.cc
-ceph_authtool_LDADD = $(CEPH_GLOBAL)
+ceph_authtool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
bin_PROGRAMS += ceph-authtool
ceph_mon_store_converter_SOURCES = tools/mon_store_converter.cc
diff --git a/src/tools/ceph-kvstore-tool.cc b/src/tools/ceph-kvstore-tool.cc
index e07391d..5a68d98 100644
--- a/src/tools/ceph-kvstore-tool.cc
+++ b/src/tools/ceph-kvstore-tool.cc
@@ -214,9 +214,9 @@ void usage(const char *pname)
<< " list [prefix]\n"
<< " list-crc [prefix]\n"
<< " exists <prefix> [key]\n"
- << " get <prefix> <key>\n"
+ << " get <prefix> <key> [out <file>]\n"
<< " crc <prefix> <key>\n"
- << " get-size\n"
+ << " get-size [<prefix> <key>]\n"
<< " set <prefix> <key> [ver <N>|in <file>]\n"
<< " store-copy <path> [num-keys-per-tx]\n"
<< " store-crc <path>\n"
@@ -286,9 +286,33 @@ int main(int argc, const char *argv[])
return 1;
}
std::cout << std::endl;
- ostringstream os;
- bl.hexdump(os);
- std::cout << os.str() << std::endl;
+
+ if (argc >= 6) {
+ string subcmd(argv[5]);
+ string out(argv[6]);
+
+ if (subcmd != "out") {
+ std::cerr << "unrecognized subcmd '" << subcmd << "'"
+ << std::endl;
+ return 1;
+ }
+
+ if (out.empty()) {
+ std::cerr << "unspecified out file" << std::endl;
+ return 1;
+ }
+
+ int err = bl.write_file(argv[6], 0644);
+ if (err < 0) {
+ std::cerr << "error writing value to '" << out << "': "
+ << cpp_strerror(err) << std::endl;
+ return 1;
+ }
+ } else {
+ ostringstream os;
+ bl.hexdump(os);
+ std::cout << os.str() << std::endl;
+ }
} else if (cmd == "crc") {
if (argc < 5) {
@@ -310,6 +334,26 @@ int main(int argc, const char *argv[])
} else if (cmd == "get-size") {
std::cout << "estimated store size: " << st.get_size() << std::endl;
+ if (argc < 4)
+ return 0;
+
+ if (argc < 5) {
+ usage(argv[0]);
+ return 1;
+ }
+ string prefix(argv[3]);
+ string key(argv[4]);
+
+ bool exists = false;
+ bufferlist bl = st.get(prefix, key, exists);
+ if (!exists) {
+ std::cerr << "(" << prefix << "," << key
+ << ") does not exist" << std::endl;
+ return 1;
+ }
+ std::cout << "(" << prefix << "," << key
+ << ") size " << si_t(bl.length()) << std::endl;
+
} else if (cmd == "set") {
if (argc < 7) {
usage(argv[0]);
diff --git a/src/tools/ceph_conf.cc b/src/tools/ceph_conf.cc
index b2286f4..14045f9 100644
--- a/src/tools/ceph_conf.cc
+++ b/src/tools/ceph_conf.cc
@@ -156,6 +156,7 @@ int main(int argc, const char **argv)
argv_to_vec(argc, argv, args);
env_to_vec(args);
+ vector<const char*> orig_args = args;
global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_DAEMON,
CINIT_FLAG_NO_DAEMON_ACTIONS);
@@ -202,6 +203,11 @@ int main(int argc, const char **argv)
lookup_key = *i++;
} else {
cerr << "unable to parse option: '" << *i << "'" << std::endl;
+ cerr << "args:";
+ for (std::vector<const char *>::iterator ci = orig_args.begin(); ci != orig_args.end(); ++ci) {
+ cerr << " '" << *ci << "'";
+ }
+ cerr << std::endl;
usage();
exit(1);
}
diff --git a/src/tools/crushtool.cc b/src/tools/crushtool.cc
index 03c83f2..7709a2a 100644
--- a/src/tools/crushtool.cc
+++ b/src/tools/crushtool.cc
@@ -4,6 +4,9 @@
* Ceph - scalable distributed file system
*
* Copyright (C) 2004-2006 Sage Weil <sage at newdream.net>
+ * Copyright (C) 2014 Cloudwatt <libre.licensing at cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic at dachary.org>
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -26,6 +29,7 @@
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "global/global_init.h"
+#include "osd/OSDMap.h"
#include "crush/CrushWrapper.h"
#include "crush/CrushCompiler.h"
#include "crush/CrushTester.h"
@@ -186,10 +190,13 @@ int main(int argc, const char **argv)
CrushWrapper crush;
- CrushTester tester(crush, cerr, 1);
+ CrushTester tester(crush, cerr);
- vector<const char *> empty_args; // we use -c, don't confuse the generic arg parsing
- global_init(NULL, empty_args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+ // we use -c, don't confuse the generic arg parsing
+ // only parse arguments from CEPH_ARGS, if in the environment
+ vector<const char *> env_args;
+ env_to_vec(env_args);
+ global_init(NULL, env_args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
common_init_finish(g_ceph_context);
@@ -429,6 +436,7 @@ int main(int argc, const char **argv)
}
else {
if ((args.size() % 3) != 0U) {
+ cerr << "remaining args: " << args << std::endl;
cerr << "layers must be specified with 3-tuples of (name, buckettype, size)"
<< std::endl;
exit(EXIT_FAILURE);
@@ -514,13 +522,13 @@ int main(int argc, const char **argv)
lower_items.push_back(i);
lower_weights.push_back(0x10000);
}
+ crush.set_max_devices(num_osds);
int type = 1;
- int rootid = 0;
for (vector<layer_t>::iterator p = layers.begin(); p != layers.end(); ++p, type++) {
layer_t &l = *p;
- dout(0) << "layer " << type
+ dout(2) << "layer " << type
<< " " << l.name
<< " bucket type " << l.buckettype
<< " " << l.size
@@ -544,8 +552,8 @@ int main(int argc, const char **argv)
vector<int> cur_weights;
unsigned lower_pos = 0; // lower pos
- dout(0) << "lower_items " << lower_items << dendl;
- dout(0) << "lower_weights " << lower_weights << dendl;
+ dout(2) << "lower_items " << lower_items << dendl;
+ dout(2) << "lower_weights " << lower_weights << dendl;
int i = 0;
while (1) {
@@ -564,7 +572,7 @@ int main(int argc, const char **argv)
weights[j] = lower_weights[lower_pos];
weight += weights[j];
lower_pos++;
- dout(0) << " item " << items[j] << " weight " << weights[j] << dendl;
+ dout(2) << " item " << items[j] << " weight " << weights[j] << dendl;
}
crush_bucket *b = crush_make_bucket(buckettype, CRUSH_HASH_DEFAULT, type, j, items, weights);
@@ -572,9 +580,8 @@ int main(int argc, const char **argv)
int id;
int r = crush_add_bucket(crush.crush, 0, b, &id);
if (r < 0) {
- dout(0) << "Couldn't add root bucket: " << strerror(-r) << dendl;
+ dout(2) << "Couldn't add bucket: " << strerror(-r) << dendl;
}
- rootid = id;
char format[20];
format[sizeof(format)-1] = '\0';
@@ -586,7 +593,7 @@ int main(int argc, const char **argv)
snprintf(name, sizeof(name), format, i);
crush.set_item_name(id, name);
- dout(0) << " in bucket " << id << " '" << name << "' size " << j << " weight " << weight << dendl;
+ dout(2) << " in bucket " << id << " '" << name << "' size " << j << " weight " << weight << dendl;
cur_items.push_back(id);
cur_weights.push_back(weight);
@@ -596,16 +603,31 @@ int main(int argc, const char **argv)
lower_items.swap(cur_items);
lower_weights.swap(cur_weights);
}
+
+ {
+ ostringstream oss;
+ vector<__u32> weights(crush.get_max_devices(), 0x10000);
+ crush.dump_tree(weights, &oss, NULL);
+ dout(1) << "\n" << oss.str() << dendl;
+ }
+
+ string root = layers.back().size == 0 ? layers.back().name :
+ string(layers.back().name) + "0";
+
+ {
+ set<int> roots;
+ crush.find_roots(roots);
+ if (roots.size() > 1)
+ dout(1) << "The crush rulesets will use the root " << root << "\n"
+ << "and ignore the others.\n"
+ << "There are " << roots.size() << " roots, they can be\n"
+ << "grouped into a single root by appending something like:\n"
+ << " root straw 0\n"
+ << dendl;
+ }
- // make a generic rules
- int ruleset=1;
- crush_rule *rule = crush_make_rule(3, ruleset, CEPH_PG_TYPE_REP, 2, 2);
- assert(rule);
- crush_rule_set_step(rule, 0, CRUSH_RULE_TAKE, rootid, 0);
- crush_rule_set_step(rule, 1, CRUSH_RULE_CHOOSE_LEAF_FIRSTN, CRUSH_CHOOSE_N, 1);
- crush_rule_set_step(rule, 2, CRUSH_RULE_EMIT, 0, 0);
- int rno = crush_add_rule(crush.crush, rule, -1);
- crush.set_rule_name(rno, "data");
+ if (OSDMap::build_simple_crush_rulesets(g_ceph_context, crush, root, &cerr))
+ exit(EXIT_FAILURE);
modified = true;
}
@@ -664,38 +686,19 @@ int main(int argc, const char **argv)
modified = true;
}
- const char *scary_tunables_message =
- "** tunables are DANGEROUS and NOT YET RECOMMENDED. DO NOT USE without\n"
- "** confirming with developers that your use-case is safe and correct.";
if (choose_local_tries >= 0) {
- if (!unsafe_tunables) {
- cerr << scary_tunables_message << std::endl;
- return -1;
- }
crush.set_choose_local_tries(choose_local_tries);
modified = true;
}
if (choose_local_fallback_tries >= 0) {
- if (!unsafe_tunables) {
- cerr << scary_tunables_message << std::endl;
- return -1;
- }
crush.set_choose_local_fallback_tries(choose_local_fallback_tries);
modified = true;
}
if (choose_total_tries >= 0) {
- if (!unsafe_tunables) {
- cerr << scary_tunables_message << std::endl;
- return -1;
- }
crush.set_choose_total_tries(choose_total_tries);
modified = true;
}
if (chooseleaf_descend_once >= 0) {
- if (!unsafe_tunables) {
- cerr << scary_tunables_message << std::endl;
- return -1;
- }
crush.set_chooseleaf_descend_once(chooseleaf_descend_once);
modified = true;
}
@@ -719,6 +722,10 @@ int main(int argc, const char **argv)
}
if (test) {
+ if (tester.get_output_utilization_all() ||
+ tester.get_output_utilization())
+ tester.set_output_statistics(true);
+
int r = tester.test();
if (r < 0)
exit(1);
@@ -726,3 +733,8 @@ int main(int argc, const char **argv)
return 0;
}
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make crushtool && test/run-cli-tests"
+ * End:
+ */
diff --git a/src/tools/dupstore.cc b/src/tools/dupstore.cc
index c8b8ece..592b4b7 100644
--- a/src/tools/dupstore.cc
+++ b/src/tools/dupstore.cc
@@ -17,8 +17,7 @@
#include "common/ceph_argparse.h"
#include "global/global_init.h"
-#include <ext/hash_map>
-using __gnu_cxx::hash_map;
+#include "include/unordered_map.h"
int dupstore(ObjectStore* src, ObjectStore* dst)
{
@@ -27,7 +26,7 @@ int dupstore(ObjectStore* src, ObjectStore* dst)
if (dst->mount() < 0) return 1;
// objects
- hash_map<ghobject_t, coll_t> did_object;
+ ceph::unordered_map<ghobject_t, coll_t> did_object;
// collections
vector<coll_t> collections;
diff --git a/src/tools/osdmaptool.cc b/src/tools/osdmaptool.cc
index edd3128..7148572 100644
--- a/src/tools/osdmaptool.cc
+++ b/src/tools/osdmaptool.cc
@@ -182,24 +182,20 @@ int main(int argc, const char **argv)
return -1;
}
- if (createsimple) {
- if (num_osd < 1) {
- cerr << me << ": osd count must be > 0" << std::endl;
- exit(1);
+ if (createsimple || create_from_conf) {
+ if (createsimple) {
+ if (num_osd < 1) {
+ cerr << me << ": osd count must be > 0" << std::endl;
+ exit(1);
+ }
+ } else {
+ num_osd = -1;
}
uuid_d fsid;
memset(&fsid, 0, sizeof(uuid_d));
osdmap.build_simple(g_ceph_context, 0, fsid, num_osd, pg_bits, pgp_bits);
modified = true;
}
- if (create_from_conf) {
- uuid_d fsid;
- memset(&fsid, 0, sizeof(uuid_d));
- int r = osdmap.build_simple_from_conf(g_ceph_context, 0, fsid, pg_bits, pgp_bits);
- if (r < 0)
- return -1;
- modified = true;
- }
if (!import_crush.empty()) {
bufferlist cbl;
@@ -270,16 +266,21 @@ int main(int argc, const char **argv)
cout << " parsed '" << test_map_pg << "' -> " << pgid << std::endl;
vector<int> raw, up, acting;
- osdmap.pg_to_osds(pgid, raw);
- osdmap.pg_to_up_acting_osds(pgid, up, acting);
- cout << pgid << " raw " << raw << " up " << up << " acting " << acting << std::endl;
+ int calced_primary, up_primary, acting_primary;
+ osdmap.pg_to_osds(pgid, &raw, &calced_primary);
+ osdmap.pg_to_up_acting_osds(pgid, &up, &up_primary,
+ &acting, &acting_primary);
+ cout << pgid << " raw (" << raw << ", p" << calced_primary
+ << ") up (" << up << ", p" << up_primary
+ << ") acting (" << acting << ", p" << acting_primary << ")"
+ << std::endl;
}
if (test_crush) {
int pass = 0;
while (1) {
cout << "pass " << ++pass << std::endl;
- hash_map<pg_t,vector<int> > m;
+ ceph::unordered_map<pg_t,vector<int> > m;
for (map<int64_t,pg_pool_t>::const_iterator p = osdmap.get_pools().begin();
p != osdmap.get_pools().end();
++p) {
diff --git a/src/tools/psim.cc b/src/tools/psim.cc
index c1adc75..ecc176d 100644
--- a/src/tools/psim.cc
+++ b/src/tools/psim.cc
@@ -47,7 +47,8 @@ int main(int argc, char **argv)
vector<int> osds;
pg_t pgid = pg_t(l.ol_pgid);
//pgid.u.ps = f * 4 + b;
- osdmap.pg_to_osds(pgid, osds);
+ int primary;
+ osdmap.pg_to_osds(pgid, &osds, &primary);
size[osds.size()]++;
#if 0
if (0) {
diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index ad8eaa3..e5ee834 100644
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -42,6 +42,7 @@ using namespace librados;
#include <locale>
#include "cls/lock/cls_lock_client.h"
+#include "include/compat.h"
int rados_tool_sync(const std::map < std::string, std::string > &opts,
std::vector<const char*> &args);
@@ -131,6 +132,13 @@ void usage(ostream& out)
" --lock-duration Lock duration (in seconds)\n"
" --lock-type Lock type (shared, exclusive)\n"
"\n"
+"CACHE POOLS: (for testing/development only)\n"
+" cache-flush <obj-name> flush cache pool object (blocking)\n"
+" cache-try-flush <obj-name> flush cache pool object (non-blocking)\n"
+" cache-evict <obj-name> evict cache pool object\n"
+" cache-flush-evict-all flush+evict all objects\n"
+" cache-try-flush-evict-all try-flush+evict all objects\n"
+"\n"
"GLOBAL OPTIONS:\n"
" --object_locator object_locator\n"
" set object_locator for operation\n"
@@ -169,8 +177,8 @@ void usage(ostream& out)
" --max-backlog max backlog (in MB)\n"
" --percent percent of operations that are read\n"
" --target-throughput target throughput (in MB)\n"
-" --run-length total time (in seconds)\n";
-
+" --run-length total time (in seconds)\n"
+ ;
}
static void usage_exit()
@@ -1041,6 +1049,96 @@ static int do_lock_cmd(std::vector<const char*> &nargs,
return 0;
}
+static int do_cache_flush(IoCtx& io_ctx, string oid)
+{
+ ObjectReadOperation op;
+ op.cache_flush();
+ librados::AioCompletion *completion =
+ librados::Rados::aio_create_completion();
+ io_ctx.aio_operate(oid.c_str(), completion, &op,
+ librados::OPERATION_IGNORE_CACHE |
+ librados::OPERATION_IGNORE_OVERLAY,
+ NULL);
+ completion->wait_for_safe();
+ int r = completion->get_return_value();
+ completion->release();
+ return r;
+}
+
+static int do_cache_try_flush(IoCtx& io_ctx, string oid)
+{
+ ObjectReadOperation op;
+ op.cache_try_flush();
+ librados::AioCompletion *completion =
+ librados::Rados::aio_create_completion();
+ io_ctx.aio_operate(oid.c_str(), completion, &op,
+ librados::OPERATION_IGNORE_CACHE |
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS,
+ NULL);
+ completion->wait_for_safe();
+ int r = completion->get_return_value();
+ completion->release();
+ return r;
+}
+
+static int do_cache_evict(IoCtx& io_ctx, string oid)
+{
+ ObjectReadOperation op;
+ op.cache_evict();
+ librados::AioCompletion *completion =
+ librados::Rados::aio_create_completion();
+ io_ctx.aio_operate(oid.c_str(), completion, &op,
+ librados::OPERATION_IGNORE_CACHE |
+ librados::OPERATION_IGNORE_OVERLAY |
+ librados::OPERATION_SKIPRWLOCKS,
+ NULL);
+ completion->wait_for_safe();
+ int r = completion->get_return_value();
+ completion->release();
+ return r;
+}
+
+static int do_cache_flush_evict_all(IoCtx& io_ctx, bool blocking)
+{
+ int r;
+ int errors = 0;
+ try {
+ librados::ObjectIterator i = io_ctx.objects_begin();
+ librados::ObjectIterator i_end = io_ctx.objects_end();
+ for (; i != i_end; ++i) {
+ cout << i->first << "\t" << i->second << std::endl;
+ if (i->second.size()) {
+ io_ctx.locator_set_key(i->second);
+ } else {
+ io_ctx.locator_set_key(string());
+ }
+ if (blocking)
+ r = do_cache_flush(io_ctx, i->first);
+ else
+ r = do_cache_try_flush(io_ctx, i->first);
+ if (r < 0) {
+ cerr << "failed to flush " << i->first << ": "
+ << cpp_strerror(r) << std::endl;
+ ++errors;
+ continue;
+ }
+ r = do_cache_evict(io_ctx, i->first);
+ if (r < 0) {
+ cerr << "failed to evict " << i->first << ": "
+ << cpp_strerror(r) << std::endl;
+ ++errors;
+ continue;
+ }
+ }
+ }
+ catch (const std::runtime_error& e) {
+ cerr << e.what() << std::endl;
+ return -1;
+ }
+ return errors ? -1 : 0;
+}
+
/**********************************************
**********************************************/
@@ -1665,8 +1763,10 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
<< cpp_strerror(ret) << std::endl;
return 1;
}
+ ret = values.size();
for (map<string, bufferlist>::const_iterator it = values.begin();
it != values.end(); ++it) {
+ last_read = it->first;
// dump key in hex if it contains nonprintable characters
if (std::count_if(it->first.begin(), it->first.end(),
(int (*)(int))isprint) < (int)it->first.length()) {
@@ -2286,6 +2386,54 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
cout << std::endl;
}
+ } else if (strcmp(nargs[0], "cache-flush") == 0) {
+ if (!pool_name || nargs.size() < 2)
+ usage_exit();
+ string oid(nargs[1]);
+ ret = do_cache_flush(io_ctx, oid);
+ if (ret < 0) {
+ cerr << "error from cache-flush " << oid << ": "
+ << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
+ } else if (strcmp(nargs[0], "cache-try-flush") == 0) {
+ if (!pool_name || nargs.size() < 2)
+ usage_exit();
+ string oid(nargs[1]);
+ ret = do_cache_try_flush(io_ctx, oid);
+ if (ret < 0) {
+ cerr << "error from cache-try-flush " << oid << ": "
+ << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
+ } else if (strcmp(nargs[0], "cache-evict") == 0) {
+ if (!pool_name || nargs.size() < 2)
+ usage_exit();
+ string oid(nargs[1]);
+ ret = do_cache_evict(io_ctx, oid);
+ if (ret < 0) {
+ cerr << "error from cache-evict " << oid << ": "
+ << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
+ } else if (strcmp(nargs[0], "cache-flush-evict-all") == 0) {
+ if (!pool_name)
+ usage_exit();
+ ret = do_cache_flush_evict_all(io_ctx, true);
+ if (ret < 0) {
+ cerr << "error from cache-flush-evict-all: "
+ << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
+ } else if (strcmp(nargs[0], "cache-try-flush-evict-all") == 0) {
+ if (!pool_name)
+ usage_exit();
+ ret = do_cache_flush_evict_all(io_ctx, false);
+ if (ret < 0) {
+ cerr << "error from cache-try-flush-evict-all: "
+ << cpp_strerror(ret) << std::endl;
+ goto out;
+ }
} else {
cerr << "unrecognized command " << nargs[0] << "; -h or --help for usage" << std::endl;
ret = -EINVAL;
diff --git a/src/tools/rados/rados_sync.cc b/src/tools/rados/rados_sync.cc
index d2d6ab5..d5ca0e7 100644
--- a/src/tools/rados/rados_sync.cc
+++ b/src/tools/rados/rados_sync.cc
@@ -98,7 +98,7 @@ ExportDir* ExportDir::create_for_writing(const std::string &path, int version,
<< cpp_strerror(err) << std::endl;
return NULL;
}
- return new ExportDir(version, path);
+ return new ExportDir(path);
}
ExportDir* ExportDir::from_file_system(const std::string &path)
@@ -138,7 +138,7 @@ ExportDir* ExportDir::from_file_system(const std::string &path)
<< "handle the data in the new format." << std::endl;
return NULL;
}
- return new ExportDir(ret, path);
+ return new ExportDir(path);
}
std::string ExportDir::get_fs_path(const std::string &rados_name) const
@@ -206,9 +206,8 @@ std::string ExportDir::get_fs_path(const std::string &rados_name) const
return oss.str();
}
-ExportDir::ExportDir(int version_, const std::string &path_)
- : version(version_),
- path(path_)
+ExportDir::ExportDir(const std::string &path_)
+ : path(path_)
{
}
diff --git a/src/tools/rados/rados_sync.h b/src/tools/rados/rados_sync.h
index 0f7226e..3d4cf19 100644
--- a/src/tools/rados/rados_sync.h
+++ b/src/tools/rados/rados_sync.h
@@ -81,9 +81,8 @@ public:
std::string get_fs_path(const std::string &rados_name) const;
private:
- ExportDir(int version_, const std::string &path_);
+ explicit ExportDir(const std::string &path_);
- int version;
std::string path;
};
diff --git a/src/unittest_bufferlist.sh b/src/unittest_bufferlist.sh
index 0f05afe..8ddf24f 100755
--- a/src/unittest_bufferlist.sh
+++ b/src/unittest_bufferlist.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
#
# Ceph - scalable distributed file system
#
diff --git a/src/upstart/ceph-osd.conf b/src/upstart/ceph-osd.conf
index ab6af58..7175c2d 100644
--- a/src/upstart/ceph-osd.conf
+++ b/src/upstart/ceph-osd.conf
@@ -16,12 +16,15 @@ pre-start script
install -d -m0755 /var/run/ceph
update="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_update_on_start || :)"
- if [ "${update:-1}" = "1" -o "{$update:-1}" = "true" ]; then
- # update location in crush; put in some suitable defaults on the
- # command line, ceph.conf can override what it wants
- location="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_location || :)"
+ if [ "${update:-1}" = "1" -o "${update:-1}" = "true" ]; then
+ # update location in crush
+ hook="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_location_hook || :)"
+ if [ -z "$hook" ]; then
+ hook="/usr/bin/ceph-crush-location"
+ fi
+ location="$($hook --cluster ${cluster:-ceph} --id $id --type osd)"
weight="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_initial_weight || :)"
- defaultweight=`df /var/lib/ceph/osd/${cluster:-ceph}-$id/ | tail -1 | awk '{ d= $2/1073741824 ; r = sprintf("%.2f", d); print r }'`
+ defaultweight=`df -P -k /var/lib/ceph/osd/${cluster:-ceph}-$id/ | tail -1 | awk '{ d= $2/1073741824 ; r = sprintf("%.2f", d); print r }'`
ceph \
--cluster="${cluster:-ceph}" \
--name="osd.$id" \
@@ -30,8 +33,6 @@ pre-start script
-- \
"$id" \
"${weight:-${defaultweight:-1}}" \
- root=default \
- host="$(hostname -s)" \
$location
fi
diff --git a/src/upstart/rbdmap.conf b/src/upstart/rbdmap.conf
new file mode 100644
index 0000000..a581c2a
--- /dev/null
+++ b/src/upstart/rbdmap.conf
@@ -0,0 +1,48 @@
+# rbdmap - Ceph RBD Mapping
+#
+# This script does not manage mount and unmount fs which depends on rbd device.
+# You should use _netdev option in fstab to mount and umount in the correct order.
+
+description "ceph rbd mapping"
+
+start on (started networking
+ and remote-filesystems)
+stop on unmounted-remote-filesystems
+
+env RBDMAPFILE="/etc/ceph/rbdmap"
+
+pre-start script
+ if [ ! -f "$RBDMAPFILE" ]; then
+ exit 0
+ fi
+
+ while read DEV PARAMS; do
+ case "$DEV" in
+ ""|\#*)
+ continue
+ ;;
+ */*)
+ ;;
+ *)
+ DEV=rbd/$DEV
+ ;;
+ esac
+ for PARAM in $(echo $PARAM | tr ',' '\n'); do
+ CMDPARAMS="$CMDPARAMS --$(echo $PARAM | tr '=' ' ')"
+ done
+ if [ ! -b /dev/rbd/$DEV ]; then
+ echo "rbd map $DEV"
+ rbd map $DEV $CMDPARAMS
+ fi
+ done < $RBDMAPFILE
+end script
+
+post-stop script
+ if ls /dev/rbd[0-9]* >/dev/null 2>&1; then
+ for DEV in /dev/rbd[0-9]*; do
+ echo "rbd unmap $DEV"
+ rbd unmap $DEV
+ done
+ fi
+end script
+
diff --git a/src/vstart.sh b/src/vstart.sh
index 4839cc1..9c76c53 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -2,6 +2,7 @@
export PYTHONPATH=./pybind
export LD_LIBRARY_PATH=.libs
+export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
# abort on failure
@@ -34,8 +35,11 @@ start_rgw=0
ip=""
nodaemon=0
smallmds=0
+hitset=""
overwrite_conf=1
cephx=1 #turn cephx on by default
+cache=""
+memstore=0
MON_ADDR=""
@@ -60,6 +64,7 @@ usage=$usage"\t-m ip:port\t\tspecify monitor address\n"
usage=$usage"\t-k keep old configuration files\n"
usage=$usage"\t-x enable cephx (on by default)\n"
usage=$usage"\t-X disable cephx\n"
+usage=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n"
usage=$usage"\t-o config\t\t add extra config parameters to mds section\n"
usage_exit() {
@@ -141,11 +146,27 @@ case $1 in
-k )
overwrite_conf=0
;;
+ --memstore )
+ memstore=1
+ ;;
+ --hitset )
+ hitset="$hitset $2 $3"
+ shift
+ shift
+ ;;
-o )
extra_conf="$extra_conf $2
"
shift
;;
+ --cache )
+ if [ -z "$cache" ]; then
+ cache="$2"
+ else
+ cache="$cache $2"
+ fi
+ shift
+ ;;
* )
usage_exit
esac
@@ -220,6 +241,10 @@ if [ -n "$MON_ADDR" ]; then
CMDS_ARGS=" -m "$MON_ADDR
fi
+if [ "$memstore" -eq 1 ]; then
+ COSDMEMSTORE='
+ osd objectstore = memstore'
+fi
# lockdep everywhere?
# export CEPH_ARGS="--lockdep 1"
@@ -297,6 +322,7 @@ if [ "$start_mon" -eq 1 ]; then
cat <<EOF > $conf
; generated by vstart.sh on `date`
[global]
+ fsid = $(uuidgen)
osd pg bits = 3
osd pgp bits = 5 ; (invalid, but ceph should cope!)
osd crush chooseleaf type = 0
@@ -338,6 +364,7 @@ $DAEMONOPTS
osd scrub load threshold = 5.0
osd debug op order = true
$COSDDEBUG
+$COSDMEMSTORE
$extra_conf
[mon]
mon pg warn min per osd = 10
@@ -353,7 +380,7 @@ EOF
if [ `echo $IP | grep '^127\\.'` ]
then
echo
- echo "WARNING: hostname resolves to loopback; remote hosts will not be able to"
+ echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
echo " connect. either adjust /etc/hosts, or edit this script to use your"
echo " machine's real IP."
echo
@@ -451,7 +478,7 @@ if [ "$smallmds" -eq 1 ]; then
EOF
fi
-if [ "$start_mds" -eq 1 ]; then
+if [ "$start_mds" -eq 1 -a "$CEPH_NUM_MDS" -gt 0 ]; then
mds=0
for name in a b c d e f g h i j k l m n o p
do
@@ -493,9 +520,9 @@ EOF
#valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20
#$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
-#$CEPH_ADM mds set_max_mds 2
+#$CEPH_ADM mds set max_mds 2
done
- cmd="$CEPH_ADM mds set_max_mds $CEPH_NUM_MDS"
+ cmd="$CEPH_ADM mds set max_mds $CEPH_NUM_MDS"
echo $cmd
$cmd
fi
@@ -519,10 +546,21 @@ $DAEMONOPTS
EOF
mkdir -p $CEPH_OUT_DIR/htdocs
mkdir -p $CEPH_OUT_DIR/fastcgi_sock
+ APACHE2_MODULE_PATH="/usr/lib/apache2/modules"
+ APACHE2_EXTRA_MODULES_NAME="mpm_prefork authz_core"
+ for module in $APACHE2_EXTRA_MODULES_NAME
+ do
+ if [ -f "${APACHE2_MODULE_PATH}/mod_${module}.so" ]; then
+ APACHE2_EXTRA_MODULES="${APACHE2_EXTRA_MODULES}LoadModule ${module}_module ${APACHE2_MODULE_PATH}/mod_${module}.so
+"
+ fi
+ done
+ echo $APACHE2_EXTRA_MODULES
cat <<EOF > $CEPH_OUT_DIR/apache.conf
LoadModule env_module /usr/lib/apache2/modules/mod_env.so
LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
LoadModule fastcgi_module /usr/lib/apache2/modules/mod_fastcgi.so
+$APACHE2_EXTRA_MODULES
Listen $rgwport
ServerName rgwtest.example.com
@@ -573,6 +611,39 @@ fi
echo "started. stop.sh to stop. see out/* (e.g. 'tail -f out/????') for debug output."
+do_cache() {
+ while [ -n "$*" ]; do
+ p="$1"
+ shift
+ echo "creating cache for pool $p ..."
+ $SUDO $CEPH_ADM <<EOF
+osd pool create ${p}-cache 8
+osd tier add $p ${p}-cache
+osd tier cache-mode ${p}-cache writeback
+osd tier set-overlay $p ${p}-cache
+quit
+EOF
+ done
+}
+do_cache $cache
+
+do_hitsets() {
+ while [ -n "$*" ]; do
+ pool="$1"
+ type="$2"
+ shift
+ shift
+ echo "setting hit_set on pool $pool type $type ..."
+ $CEPH_ADM <<EOF
+osd pool set $pool hit_set_type $type
+osd pool set $pool hit_set_count 8
+osd pool set $pool hit_set_period 30
+quit
+EOF
+ done
+}
+do_hitsets $hitset
+
echo ""
echo "export PYTHONPATH=./pybind"
echo "export LD_LIBRARY_PATH=.libs"
--
Alioth's hooks/post-receive on /srv/git.debian.org/git/pkg-ceph/ceph.git
More information about the Pkg-ceph-commits
mailing list