[Pkg-owncloud-commits] [owncloud] 38/46: Lazy initialize external storages
David Prévot
taffit at moszumanska.debian.org
Fri Oct 24 15:11:44 UTC 2014
This is an automated email from the git hooks/post-receive script.
taffit pushed a commit to branch master
in repository owncloud.
commit 09d80ac9fc723793ee6e81defe5bd667b4e009a4
Author: Vincent Petry <pvince81 at owncloud.com>
Date: Wed Oct 22 14:50:22 2014 +0200
Lazy initialize external storages
Backport of 075e8d8e8658913e1c5b8869f3e457fa6db2d847 from master
---
apps/files_external/lib/amazons3.php | 100 ++++++++++++++++-------------
apps/files_external/lib/dropbox.php | 1 +
apps/files_external/lib/ftp.php | 2 +-
apps/files_external/lib/google.php | 1 +
apps/files_external/lib/sftp.php | 39 ++++++++----
apps/files_external/lib/swift.php | 119 ++++++++++++++++++++++-------------
6 files changed, 159 insertions(+), 103 deletions(-)
diff --git a/apps/files_external/lib/amazons3.php b/apps/files_external/lib/amazons3.php
index 06910e0..c73860d 100644
--- a/apps/files_external/lib/amazons3.php
+++ b/apps/files_external/lib/amazons3.php
@@ -111,34 +111,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$params['port'] = ($params['use_ssl'] === 'false') ? 80 : 443;
}
$base_url = $scheme . '://' . $params['hostname'] . ':' . $params['port'] . '/';
-
- $this->connection = S3Client::factory(array(
- 'key' => $params['key'],
- 'secret' => $params['secret'],
- 'base_url' => $base_url,
- 'region' => $params['region']
- ));
-
- if (!$this->connection->isValidBucketName($this->bucket)) {
- throw new \Exception("The configured bucket name is invalid.");
- }
-
- if (!$this->connection->doesBucketExist($this->bucket)) {
- try {
- $this->connection->createBucket(array(
- 'Bucket' => $this->bucket
- ));
- $this->connection->waitUntilBucketExists(array(
- 'Bucket' => $this->bucket,
- 'waiter.interval' => 1,
- 'waiter.max_attempts' => 15
- ));
- $this->testTimeout();
- } catch (S3Exception $e) {
- \OCP\Util::logException('files_external', $e);
- throw new \Exception('Creation of bucket failed. '.$e->getMessage());
- }
- }
}
/**
@@ -181,7 +153,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
- $this->connection->putObject(array(
+ $this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $path . '/',
'ContentType' => 'httpd/unix-directory'
@@ -216,7 +188,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
protected function clearBucket() {
try {
- $this->connection->clearBucket($this->bucket);
+ $this->getConnection()->clearBucket($this->bucket);
return true;
// clearBucket() is not working with Ceph, so if it fails we try the slower approach
} catch (\Exception $e) {
@@ -237,9 +209,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
// to delete all objects prefixed with the path.
do {
// instead of the iterator, manually loop over the list ...
- $objects = $this->connection->listObjects($params);
+ $objects = $this->getConnection()->listObjects($params);
// ... so we can delete the files in batches
- $this->connection->deleteObjects(array(
+ $this->getConnection()->deleteObjects(array(
'Bucket' => $this->bucket,
'Objects' => $objects['Contents']
));
@@ -264,7 +236,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
try {
$files = array();
- $result = $this->connection->getIterator('ListObjects', array(
+ $result = $this->getConnection()->getIterator('ListObjects', array(
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path
@@ -299,7 +271,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$stat['size'] = -1; //unknown
$stat['mtime'] = time() - $this->rescanDelay * 1000;
} else {
- $result = $this->connection->headObject(array(
+ $result = $this->getConnection()->headObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
@@ -328,10 +300,10 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
- if ($this->connection->doesObjectExist($this->bucket, $path)) {
+ if ($this->getConnection()->doesObjectExist($this->bucket, $path)) {
return 'file';
}
- if ($this->connection->doesObjectExist($this->bucket, $path.'/')) {
+ if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) {
return 'dir';
}
} catch (S3Exception $e) {
@@ -350,7 +322,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
- $this->connection->deleteObject(array(
+ $this->getConnection()->deleteObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
@@ -373,7 +345,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
self::$tmpFiles[$tmpFile] = $path;
try {
- $this->connection->getObject(array(
+ $this->getConnection()->getObject(array(
'Bucket' => $this->bucket,
'Key' => $path,
'SaveAs' => $tmpFile
@@ -421,7 +393,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return 'httpd/unix-directory';
} else if ($this->file_exists($path)) {
try {
- $result = $this->connection->headObject(array(
+ $result = $this->getConnection()->headObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
@@ -449,7 +421,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
if ($fileType === 'dir' && ! $this->isRoot($path)) {
$path .= '/';
}
- $this->connection->copyObject(array(
+ $this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path),
'Metadata' => $metadata,
@@ -458,7 +430,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$this->testTimeout();
} else {
$mimeType = \OC_Helper::getMimetypeDetector()->detectPath($path);
- $this->connection->putObject(array(
+ $this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path),
'Metadata' => $metadata,
@@ -480,7 +452,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
if ($this->is_file($path1)) {
try {
- $this->connection->copyObject(array(
+ $this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path2),
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
@@ -494,7 +466,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$this->remove($path2);
try {
- $this->connection->copyObject(array(
+ $this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $path2 . '/',
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
@@ -552,7 +524,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
public function test() {
- $test = $this->connection->getBucketAcl(array(
+ $test = $this->getConnection()->getBucketAcl(array(
'Bucket' => $this->bucket,
));
if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
@@ -565,7 +537,45 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $this->id;
}
+ /**
+ * Returns the connection
+ *
+ * @return S3Client connected client
+ * @throws \Exception if connection could not be made
+ */
public function getConnection() {
+ if (!is_null($this->connection)) {
+ return $this->connection;
+ }
+
+ $this->connection = S3Client::factory(array(
+ 'key' => $params['key'],
+ 'secret' => $params['secret'],
+ 'base_url' => $base_url,
+ 'region' => $params['region']
+ ));
+
+ if (!$this->connection->isValidBucketName($this->bucket)) {
+ throw new \Exception("The configured bucket name is invalid.");
+ }
+
+ if (!$this->connection->doesBucketExist($this->bucket)) {
+ try {
+ $this->connection->createBucket(array(
+ 'Bucket' => $this->bucket
+ ));
+ $this->connection->waitUntilBucketExists(array(
+ 'Bucket' => $this->bucket,
+ 'waiter.interval' => 1,
+ 'waiter.max_attempts' => 15
+ ));
+ $this->testTimeout();
+ } catch (S3Exception $e) {
+ \OCP\Util::logException('files_external', $e);
+ throw new \Exception('Creation of bucket failed. '.$e->getMessage());
+ }
+ }
+
return $this->connection;
}
@@ -575,7 +585,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
- $this->connection->putObject(array(
+ $this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]),
'SourceFile' => $tmpFile,
diff --git a/apps/files_external/lib/dropbox.php b/apps/files_external/lib/dropbox.php
index 9f297d2..cc1e628 100755
--- a/apps/files_external/lib/dropbox.php
+++ b/apps/files_external/lib/dropbox.php
@@ -44,6 +44,7 @@ class Dropbox extends \OC\Files\Storage\Common {
$this->id = 'dropbox::'.$params['app_key'] . $params['token']. '/' . $this->root;
$oauth = new \Dropbox_OAuth_Curl($params['app_key'], $params['app_secret']);
$oauth->setToken($params['token'], $params['token_secret']);
+ // note: Dropbox_API connection is lazy
$this->dropbox = new \Dropbox_API($oauth, 'auto');
} else {
throw new \Exception('Creating \OC\Files\Storage\Dropbox storage failed');
diff --git a/apps/files_external/lib/ftp.php b/apps/files_external/lib/ftp.php
index 2650a94..4a995d2 100644
--- a/apps/files_external/lib/ftp.php
+++ b/apps/files_external/lib/ftp.php
@@ -39,7 +39,7 @@ class FTP extends \OC\Files\Storage\StreamWrapper{
$this->root .= '/';
}
} else {
- throw new \Exception();
+ throw new \Exception('Creating \OC\Files\Storage\FTP storage failed');
}
}
diff --git a/apps/files_external/lib/google.php b/apps/files_external/lib/google.php
index 56c0d45..e71c7ab 100644
--- a/apps/files_external/lib/google.php
+++ b/apps/files_external/lib/google.php
@@ -52,6 +52,7 @@ class Google extends \OC\Files\Storage\Common {
$client->setScopes(array('https://www.googleapis.com/auth/drive'));
$client->setUseObjects(true);
$client->setAccessToken($params['token']);
+ // note: API connection is lazy
$this->service = new \Google_DriveService($client);
$token = json_decode($params['token'], true);
$this->id = 'google::'.substr($params['client_id'], 0, 30).$token['created'];
diff --git a/apps/files_external/lib/sftp.php b/apps/files_external/lib/sftp.php
index e0655cc..e868129 100644
--- a/apps/files_external/lib/sftp.php
+++ b/apps/files_external/lib/sftp.php
@@ -53,6 +53,18 @@ class SFTP extends \OC\Files\Storage\Common {
if (substr($this->root, -1, 1) != '/') {
$this->root .= '/';
}
+ }
+
+ /**
+ * Returns the connection.
+ *
+ * @return \Net_SFTP connected client instance
+ * @throws \Exception when the connection failed
+ */
+ public function getConnection() {
+ if (!is_null($this->client)) {
+ return $this->client;
+ }
$hostKeys = $this->readHostKeys();
$this->client = new \Net_SFTP($this->host);
@@ -71,6 +83,7 @@ class SFTP extends \OC\Files\Storage\Common {
if (!$this->client->login($this->user, $this->password)) {
throw new \Exception('Login failed');
}
+ return $this->client;
}
public function test() {
@@ -81,7 +94,7 @@ class SFTP extends \OC\Files\Storage\Common {
) {
return false;
}
- return $this->client->nlist() !== false;
+ return $this->getConnection()->nlist() !== false;
}
public function getId(){
@@ -149,7 +162,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function mkdir($path) {
try {
- return $this->client->mkdir($this->absPath($path));
+ return $this->getConnection()->mkdir($this->absPath($path));
} catch (\Exception $e) {
return false;
}
@@ -157,7 +170,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function rmdir($path) {
try {
- return $this->client->delete($this->absPath($path), true);
+ return $this->getConnection()->delete($this->absPath($path), true);
} catch (\Exception $e) {
return false;
}
@@ -165,7 +178,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function opendir($path) {
try {
- $list = $this->client->nlist($this->absPath($path));
+ $list = $this->getConnection()->nlist($this->absPath($path));
$id = md5('sftp:' . $path);
$dirStream = array();
@@ -183,7 +196,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function filetype($path) {
try {
- $stat = $this->client->stat($this->absPath($path));
+ $stat = $this->getConnection()->stat($this->absPath($path));
if ($stat['type'] == NET_SFTP_TYPE_REGULAR) {
return 'file';
}
@@ -199,7 +212,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function file_exists($path) {
try {
- return $this->client->stat($this->absPath($path)) !== false;
+ return $this->getConnection()->stat($this->absPath($path)) !== false;
} catch (\Exception $e) {
return false;
}
@@ -207,7 +220,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function unlink($path) {
try {
- return $this->client->delete($this->absPath($path), true);
+ return $this->getConnection()->delete($this->absPath($path), true);
} catch (\Exception $e) {
return false;
}
@@ -234,7 +247,7 @@ class SFTP extends \OC\Files\Storage\Common {
case 'x+':
case 'c':
case 'c+':
- $context = stream_context_create(array('sftp' => array('session' => $this->client)));
+ $context = stream_context_create(array('sftp' => array('session' => $this->getConnection())));
return fopen($this->constructUrl($path), $mode, false, $context);
}
} catch (\Exception $e) {
@@ -248,7 +261,7 @@ class SFTP extends \OC\Files\Storage\Common {
return false;
}
if (!$this->file_exists($path)) {
- $this->client->put($this->absPath($path), '');
+ $this->getConnection()->put($this->absPath($path), '');
} else {
return false;
}
@@ -259,11 +272,11 @@ class SFTP extends \OC\Files\Storage\Common {
}
public function getFile($path, $target) {
- $this->client->get($path, $target);
+ $this->getConnection()->get($path, $target);
}
public function uploadFile($path, $target) {
- $this->client->put($target, $path, NET_SFTP_LOCAL_FILE);
+ $this->getConnection()->put($target, $path, NET_SFTP_LOCAL_FILE);
}
public function rename($source, $target) {
@@ -271,7 +284,7 @@ class SFTP extends \OC\Files\Storage\Common {
if (!$this->is_dir($target) && $this->file_exists($target)) {
$this->unlink($target);
}
- return $this->client->rename(
+ return $this->getConnection()->rename(
$this->absPath($source),
$this->absPath($target)
);
@@ -282,7 +295,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function stat($path) {
try {
- $stat = $this->client->stat($this->absPath($path));
+ $stat = $this->getConnection()->stat($this->absPath($path));
$mtime = $stat ? $stat['mtime'] : -1;
$size = $stat ? $stat['size'] : 0;
diff --git a/apps/files_external/lib/swift.php b/apps/files_external/lib/swift.php
index 3b298df..35f75d4 100644
--- a/apps/files_external/lib/swift.php
+++ b/apps/files_external/lib/swift.php
@@ -48,6 +48,12 @@ class Swift extends \OC\Files\Storage\Common {
*/
private $bucket;
/**
+ * Connection parameters
+ *
+ * @var array
+ */
+ private $params;
+ /**
* @var array
*/
private static $tmpFiles = array();
@@ -84,7 +90,7 @@ class Swift extends \OC\Files\Storage\Common {
*/
private function doesObjectExist($path) {
try {
- $this->container->getPartialObject($path);
+ $this->getContainer()->getPartialObject($path);
return true;
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@@ -110,37 +116,7 @@ class Swift extends \OC\Files\Storage\Common {
$params['service_name'] = 'cloudFiles';
}
- $settings = array(
- 'username' => $params['user'],
- );
-
- if (isset($params['password'])) {
- $settings['password'] = $params['password'];
- } else if (isset($params['key'])) {
- $settings['apiKey'] = $params['key'];
- }
-
- if (isset($params['tenant'])) {
- $settings['tenantName'] = $params['tenant'];
- }
-
- if (isset($params['timeout'])) {
- $settings['timeout'] = $params['timeout'];
- }
-
- $this->anchor = new OpenStack($params['url'], $settings);
-
- $this->connection = $this->anchor->objectStoreService($params['service_name'], $params['region']);
-
- try {
- $this->container = $this->connection->getContainer($this->bucket);
- } catch (ClientErrorResponseException $e) {
- $this->container = $this->connection->createContainer($this->bucket);
- }
-
- if (!$this->file_exists('.')) {
- $this->mkdir('.');
- }
+ $this->params = $params;
}
public function mkdir($path) {
@@ -158,7 +134,7 @@ class Swift extends \OC\Files\Storage\Common {
$customHeaders = array('content-type' => 'httpd/unix-directory');
$metadataHeaders = DataObject::stockHeaders(array());
$allHeaders = $customHeaders + $metadataHeaders;
- $this->container->uploadObject($path, '', $allHeaders);
+ $this->getContainer()->uploadObject($path, '', $allHeaders);
} catch (Exceptions\CreateUpdateError $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@@ -198,7 +174,7 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
- $this->container->dataObject()->setName($path . '/')->delete();
+ $this->getContainer()->dataObject()->setName($path . '/')->delete();
} catch (Exceptions\DeleteError $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@@ -221,7 +197,7 @@ class Swift extends \OC\Files\Storage\Common {
try {
$files = array();
/** @var OpenCloud\Common\Collection $objects */
- $objects = $this->container->objectList(array(
+ $objects = $this->getContainer()->objectList(array(
'prefix' => $path,
'delimiter' => '/'
));
@@ -251,7 +227,7 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
- $object = $this->container->getPartialObject($path);
+ $object = $this->getContainer()->getPartialObject($path);
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@@ -304,7 +280,7 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
- $this->container->dataObject()->setName($path)->delete();
+ $this->getContainer()->dataObject()->setName($path)->delete();
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@@ -322,7 +298,7 @@ class Swift extends \OC\Files\Storage\Common {
$tmpFile = \OC_Helper::tmpFile();
self::$tmpFiles[$tmpFile] = $path;
try {
- $object = $this->container->getObject($path);
+ $object = $this->getContainer()->getObject($path);
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@@ -375,7 +351,7 @@ class Swift extends \OC\Files\Storage\Common {
if ($this->is_dir($path)) {
return 'httpd/unix-directory';
} else if ($this->file_exists($path)) {
- $object = $this->container->getPartialObject($path);
+ $object = $this->getContainer()->getPartialObject($path);
return $object->getContentType();
}
return false;
@@ -392,7 +368,7 @@ class Swift extends \OC\Files\Storage\Common {
$path .= '/';
}
- $object = $this->container->getPartialObject($path);
+ $object = $this->getContainer()->getPartialObject($path);
$object->saveMetadata($metadata);
return true;
} else {
@@ -400,7 +376,7 @@ class Swift extends \OC\Files\Storage\Common {
$customHeaders = array('content-type' => $mimeType);
$metadataHeaders = DataObject::stockHeaders($metadata);
$allHeaders = $customHeaders + $metadataHeaders;
- $this->container->uploadObject($path, '', $allHeaders);
+ $this->getContainer()->uploadObject($path, '', $allHeaders);
return true;
}
}
@@ -416,7 +392,7 @@ class Swift extends \OC\Files\Storage\Common {
$this->unlink($path2);
try {
- $source = $this->container->getPartialObject($path1);
+ $source = $this->getContainer()->getPartialObject($path1);
$source->copy($this->bucket.'/'.$path2);
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@@ -429,7 +405,7 @@ class Swift extends \OC\Files\Storage\Common {
$this->unlink($path2);
try {
- $source = $this->container->getPartialObject($path1 . '/');
+ $source = $this->getContainer()->getPartialObject($path1 . '/');
$source->copy($this->bucket.'/'.$path2 . '/');
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@@ -487,16 +463,71 @@ class Swift extends \OC\Files\Storage\Common {
return $this->id;
}
+ /**
+ * Returns the connection
+ *
+ * @return OpenCloud\ObjectStore\Service connected client
+ * @throws \Exception if connection could not be made
+ */
public function getConnection() {
+ if (!is_null($this->connection)) {
+ return $this->connection;
+ }
+
+ $settings = array(
+ 'username' => $this->params['user'],
+ );
+
+ if (!empty($this->params['password'])) {
+ $settings['password'] = $this->params['password'];
+ } else if (!empty($this->params['key'])) {
+ $settings['apiKey'] = $this->params['key'];
+ }
+
+ if (!empty($this->params['tenant'])) {
+ $settings['tenantName'] = $this->params['tenant'];
+ }
+
+ if (!empty($this->params['timeout'])) {
+ $settings['timeout'] = $this->params['timeout'];
+ }
+
+ $this->anchor = new OpenStack($this->params['url'], $settings);
+
+ $this->connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);
+
return $this->connection;
}
+ /**
+ * Returns the initialized object store container.
+ *
+ * @return OpenCloud\ObjectStore\Resource\Container
+ */
+ public function getContainer() {
+ if (!is_null($this->container)) {
+ return $this->container;
+ }
+
+ try {
+ $this->container = $this->getConnection()->getContainer($this->bucket);
+ } catch (ClientErrorResponseException $e) {
+ $this->container = $this->getConnection()->createContainer($this->bucket);
+ }
+
+ if (!$this->file_exists('.')) {
+ $this->mkdir('.');
+ }
+
+ return $this->container;
+ }
+
public function writeBack($tmpFile) {
if (!isset(self::$tmpFiles[$tmpFile])) {
return false;
}
$fileData = fopen($tmpFile, 'r');
- $this->container->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
+ $this->getContainer()->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
unlink($tmpFile);
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud.git
More information about the Pkg-owncloud-commits
mailing list