[Pkg-owncloud-commits] [owncloud] 25/273: move to stream based IObjectStore interface, rearrange & reformat code
David Prévot
taffit at moszumanska.debian.org
Fri Jul 4 03:12:54 UTC 2014
This is an automated email from the git hooks/post-receive script.
taffit pushed a commit to branch master
in repository owncloud.
commit c112a1d323e363eaf12574bf094a90ce2f89f028
Author: Jörn Friedrich Dreyer <jfd at butonic.de>
Date: Fri Jun 20 12:27:47 2014 +0200
move to stream based IObjectStore interface, rearrange & reformat code
---
.../files/objectstore/objectstorestorage.php | 225 ++++++++++-----------
lib/private/files/objectstore/swift.php | 47 +++--
lib/public/files/objectstore/iobjectstore.php | 12 +-
3 files changed, 140 insertions(+), 144 deletions(-)
diff --git a/lib/private/files/objectstore/objectstorestorage.php b/lib/private/files/objectstore/objectstorestorage.php
index b925f2e..0054ee1 100644
--- a/lib/private/files/objectstore/objectstorestorage.php
+++ b/lib/private/files/objectstore/objectstorestorage.php
@@ -25,20 +25,18 @@ use OCP\Files\ObjectStore\IObjectStore;
class ObjectStoreStorage extends \OC\Files\Storage\Common {
/**
+ * @var array
+ */
+ private static $tmpFiles = array();
+ /**
* @var \OCP\Files\ObjectStore\IObjectStore $objectStore
*/
protected $objectStore;
-
/**
* @var \OC\User\User $user
*/
protected $user;
- /**
- * @var array
- */
- private static $tmpFiles = array();
-
public function __construct($params) {
if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) {
$this->objectStore = $params['objectstore'];
@@ -46,57 +44,20 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
throw new \Exception('missing IObjectStore instance');
}
if (isset($params['storageid'])) {
- $this->id = 'object::store:'.$params['storageid'];
+ $this->id = 'object::store:' . $params['storageid'];
} else {
- $this->id = 'object::store:'.$this->objectStore->getStorageId();
+ $this->id = 'object::store:' . $this->objectStore->getStorageId();
}
//initialize cache with root directory in cache
- if ( ! $this->is_dir('/') ) {
+ if (!$this->is_dir('/')) {
$this->mkdir('/');
}
}
- /**
- * Object Stores use a NoopScanner because metadata is directly stored in
- * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
- * @param string $path
- * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
- * @return \OC\Files\ObjectStore\NoopScanner
- */
- public function getScanner($path = '', $storage = null) {
- if (!$storage) {
- $storage = $this;
- }
- if (!isset($this->scanner)) {
- $this->scanner = new NoopScanner($storage);
- }
- return $this->scanner;
- }
-
- /**
- * @param string $path
- * @return string
- */
- private function normalizePath($path) {
- $path = trim($path, '/');
- //FIXME why do we sometimes get a path like 'files//username'?
- $path = str_replace('//', '/', $path);
-
- if (!$path) {
- $path = '.';
- }
-
- return $path;
- }
-
- public function getId () {
- return $this->id;
- }
-
public function mkdir($path) {
$path = $this->normalizePath($path);
- if($this->is_dir($path)) {
+ if ($this->is_dir($path)) {
return false;
}
@@ -113,7 +74,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
'permissions' => \OCP\PERMISSION_ALL,
);
- if ($dirName === '.' && ! $parentExists ) {
+ if ($dirName === '.' && !$parentExists) {
//create root on the fly
$data['etag'] = $this->getETag($dirName);
$this->getCache()->put('', $data);
@@ -128,9 +89,56 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
return false;
}
- public function file_exists($path) {
+ /**
+ * @param string $path
+ * @return string
+ */
+ private function normalizePath($path) {
+ $path = trim($path, '/');
+ //FIXME why do we sometimes get a path like 'files//username'?
+ $path = str_replace('//', '/', $path);
+
+ if (!$path) {
+ $path = '.';
+ }
+
+ return $path;
+ }
+
+ /**
+ * Object Stores use a NoopScanner because metadata is directly stored in
+ * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
+ *
+ * @param string $path
+ * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
+ * @return \OC\Files\ObjectStore\NoopScanner
+ */
+ public function getScanner($path = '', $storage = null) {
+ if (!$storage) {
+ $storage = $this;
+ }
+ if (!isset($this->scanner)) {
+ $this->scanner = new NoopScanner($storage);
+ }
+ return $this->scanner;
+ }
+
+ public function getId() {
+ return $this->id;
+ }
+
+ public function rmdir($path) {
$path = $this->normalizePath($path);
- return (bool)$this->stat($path);
+
+ if (!$this->is_dir($path)) {
+ return false;
+ }
+
+ $this->rmObjects($path);
+
+ $this->getCache()->remove($path);
+
+ return true;
}
private function rmObjects($path) {
@@ -144,23 +152,52 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
}
}
- public function rmdir($path) {
+ public function unlink($path) {
$path = $this->normalizePath($path);
+ $stat = $this->stat($path);
- if (!$this->is_dir($path)) {
- return false;
+ if ($stat && isset($stat['fileid'])) {
+ if ($stat['mimetype'] === 'httpd/unix-directory') {
+ return $this->rmdir($path);
+ }
+ try {
+ $this->objectStore->deleteObject($this->getURN($stat['fileid']));
+ } catch (\Exception $ex) {
+ if ($ex->getCode() !== 404) {
+ \OCP\Util::writeLog('objectstore', 'Could not delete object: ' . $ex->getMessage(), \OCP\Util::ERROR);
+ return false;
+ } else {
+ //removing from cache is ok as it does not exist in the objectstore anyway
+ }
+ }
+ $this->getCache()->remove($path);
+ return true;
}
+ return false;
+ }
- $this->rmObjects($path);
-
- $this->getCache()->remove($path);
+ public function stat($path) {
+ return $this->getCache()->get($path);
+ }
- return true;
+ /**
+ * Override this method if you need a different unique resource identifier for your object storage implementation.
+ * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
+ * You may need a mapping table to store your URN if it cannot be generated from the fileid.
+ *
+ * @param int $fileId the fileid
+ * @return null|string the unified resource name used to identify the object
+ */
+ protected function getURN($fileId) {
+ if (is_numeric($fileId)) {
+ return 'urn:oid:' . $fileId;
+ }
+ return null;
}
public function opendir($path) {
$path = $this->normalizePath($path);
-
+
if ($path === '.') {
$path = '';
} else if ($path) {
@@ -173,7 +210,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
foreach ($folderContents as $file) {
$files[] = $file['name'];
}
-
+
\OC\Files\Stream\Dir::register('objstore' . $path, $files);
return opendir('fakedir://objstore' . $path);
@@ -183,10 +220,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
}
}
- public function stat($path) {
- return $this->getCache()->get($path);
- }
-
public function filetype($path) {
$path = $this->normalizePath($path);
$stat = $this->stat($path);
@@ -200,31 +233,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
}
}
-
- public function unlink($path) {
- $path = $this->normalizePath($path);
- $stat = $this->stat($path);
-
- if ($stat && isset($stat['fileid'])) {
- if ($stat['mimetype'] === 'httpd/unix-directory') {
- return $this->rmdir($path);
- }
- try {
- $this->objectStore->deleteObject($this->getURN($stat['fileid']));
- } catch (\Exception $ex) {
- if ($ex->getCode() !== 404) {
- \OCP\Util::writeLog('objectstore', 'Could not delete object: '.$ex->getMessage(), \OCP\Util::ERROR);
- return false;
- } else {
- //removing from cache is ok as it does not exist in the objectstore anyway
- }
- }
- $this->getCache()->remove($path);
- return true;
- }
- return false;
- }
-
public function fopen($path, $mode) {
$path = $this->normalizePath($path);
@@ -233,15 +241,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
case 'rb':
$stat = $this->stat($path);
if (is_array($stat)) {
- $tmpFile = \OC_Helper::tmpFile();
- self::$tmpFiles[$tmpFile] = $path;
try {
- $this->objectStore->getObject($this->getURN($stat['fileid']), $tmpFile);
+ return $this->objectStore->readObject($this->getURN($stat['fileid']));
} catch (\Exception $ex) {
- \OCP\Util::writeLog('objectstore', 'Could not get object: '.$ex->getMessage(), \OCP\Util::ERROR);
+ \OCP\Util::writeLog('objectstore', 'Could not get object: ' . $ex->getMessage(), \OCP\Util::ERROR);
return false;
}
- return fopen($tmpFile, 'r');
} else {
return false;
}
@@ -275,6 +280,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
return false;
}
+ public function file_exists($path) {
+ $path = $this->normalizePath($path);
+ return (bool)$this->stat($path);
+ }
+
public function rename($path1, $path2) {
$path1 = $this->normalizePath($path1);
$path2 = $this->normalizePath($path2);
@@ -297,12 +307,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
} else {
return false;
}
-
+
} else {
return false;
}
}
-
+
public function getMimeType($path) {
$path = $this->normalizePath($path);
$stat = $this->stat($path);
@@ -312,7 +322,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
return false;
}
}
-
+
public function touch($path, $mtime = null) {
if (is_null($mtime)) {
$mtime = time();
@@ -343,30 +353,17 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
);
$fileId = $this->getCache()->put($path, $stat);
try {
- $this->objectStore->writeObject($this->getURN($fileId));
+ //read an empty file from memory
+ $this->objectStore->writeObject($this->getURN($fileId), fopen('php://memory', 'r'));
} catch (\Exception $ex) {
$this->getCache()->remove($path);
- \OCP\Util::writeLog('objectstore', 'Could not create object: '.$ex->getMessage(), \OCP\Util::ERROR);
+ \OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR);
return false;
}
}
return true;
}
- /**
- * Override this method if you need a different unique resource identifier for your object storage implementation.
- * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
- * You may need a mapping table to store your URN if it cannot be generated from the fileid.
- * @param int $fileId the fileid
- * @return null|string the unified resource name used to identify the object
- */
- protected function getURN($fileId) {
- if (is_numeric($fileId)) {
- return 'urn:oid:'.$fileId;
- }
- return null;
- }
-
public function writeBack($tmpFile) {
if (!isset(self::$tmpFiles[$tmpFile])) {
return false;
@@ -391,10 +388,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
$fileId = $this->getCache()->put($path, $stat);
try {
//upload to object storage
- $this->objectStore->writeObject($this->getURN($fileId), $tmpFile);
+ $this->objectStore->writeObject($this->getURN($fileId), fopen($tmpFile, 'r'));
} catch (\Exception $ex) {
$this->getCache()->remove($path);
- \OCP\Util::writeLog('objectstore', 'Could not create object: '.$ex->getMessage(), \OCP\Util::ERROR);
+ \OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR);
return false;
}
return true;
diff --git a/lib/private/files/objectstore/swift.php b/lib/private/files/objectstore/swift.php
index 14892d2..505b5be 100644
--- a/lib/private/files/objectstore/swift.php
+++ b/lib/private/files/objectstore/swift.php
@@ -21,23 +21,24 @@
namespace OC\Files\ObjectStore;
use Guzzle\Http\Exception\ClientErrorResponseException;
+use OCP\Files\ObjectStore\IObjectStore;
use OpenCloud\OpenStack;
-class Swift implements \OCP\Files\ObjectStore\IObjectStore {
+class Swift implements IObjectStore {
/**
* @var \OpenCloud\ObjectStore\Service
*/
private $objectStoreService;
-
+
/**
* @var \OpenCloud\ObjectStore\Resource\Container
*/
private $container;
public function __construct($params) {
- if (!isset($params['username']) || !isset($params['password']) ) {
- throw new \Exception("Access Key and Secret have to be configured.");
+ if (!isset($params['username']) || !isset($params['password'])) {
+ throw new \Exception('Access Key and Secret have to be configured.');
}
if (!isset($params['container'])) {
$params['container'] = 'owncloud';
@@ -80,40 +81,42 @@ class Swift implements \OCP\Files\ObjectStore\IObjectStore {
}
}
+ /**
+ * @return string the container name where objects are stored
+ */
public function getStorageId() {
return $this->container->name;
}
/**
- * @param string $urn Unified Resource Name
- * @param string $tmpFile
- * @return void
+ * @param string $urn the unified resource name used to identify the object
+ * @param resource $stream stream with the data to write
* @throws Exception from openstack lib when something goes wrong
*/
- public function writeObject($urn, $tmpFile = null) {
- $fileData = '';
- if ($tmpFile) {
- $fileData = fopen($tmpFile, 'r');
- }
-
- $this->container->uploadObject($urn, $fileData);
+ public function writeObject($urn, $stream) {
+ $this->container->uploadObject($urn, $stream);
}
/**
- * @param string $urn Unified Resource Name
- * @param string $tmpFile
- * @return void
+ * @param string $urn the unified resource name used to identify the object
+ * @return resource stream with the read data
* @throws Exception from openstack lib when something goes wrong
*/
- public function getObject($urn, $tmpFile) {
+ public function readObject($urn) {
$object = $this->container->getObject($urn);
- /** @var $objectContent \Guzzle\Http\EntityBody **/
+ // we need to keep a reference to objectContent or
+ // the stream will be closed before we can do anything with it
+ /** @var $objectContent \Guzzle\Http\EntityBody * */
$objectContent = $object->getContent();
-
$objectContent->rewind();
- $stream = $objectContent->getStream();
- file_put_contents($tmpFile, $stream);
+
+ // directly returning the object stream does not work because the GC seems to collect it, so we need a copy
+ $tmpStream = fopen('php://temp', 'r+');
+ stream_copy_to_stream($objectContent->getStream(), $tmpStream);
+ rewind($tmpStream);
+
+ return $tmpStream;
}
/**
diff --git a/lib/public/files/objectstore/iobjectstore.php b/lib/public/files/objectstore/iobjectstore.php
index 3b6bd98..b2c5a9d 100644
--- a/lib/public/files/objectstore/iobjectstore.php
+++ b/lib/public/files/objectstore/iobjectstore.php
@@ -11,21 +11,17 @@ interface IObjectStore {
/**
* @param string $urn the unified resource name used to identify the object
- * @param string $tmpFile path to the local temporary file that should be
- * used to store the object
- * @return void
+ * @return resource stream with the read data
* @throws Exception when something goes wrong, message will be logged
*/
- function getObject($urn, $tmpFile);
+ function readObject($urn);
/**
* @param string $urn the unified resource name used to identify the object
- * @param string $tmpFile path to the local temporary file that the object
- * should be loaded from
- * @return void
+ * @param resource $stream stream with the data to write
* @throws Exception when something goes wrong, message will be logged
*/
- function writeObject($urn, $tmpFile = null);
+ function writeObject($urn, $stream);
/**
* @param string $urn the unified resource name used to identify the object
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud.git
More information about the Pkg-owncloud-commits
mailing list