Added 'bufferedadd' plugin including an example and unittests

examples/7.5-plugin-bufferedadd.php

+<?php
+require('init.php');
+
+// this very simple plugin is used to show some events
+class simpleDebug extends Solarium_Plugin_Abstract
+{
+    protected $_output = array();
+
+    public function display()
+    {
+        echo implode('<br/>', $this->_output);
+    }
+
+    public function eventBufferedAddFlushStart($buffer) {
+        $this->_output[] = 'Flushing buffer (' . count($buffer) . 'docs)';
+    }
+}
+
+htmlHeader();
+
+// create a client instance and autoload the buffered add plugin
+$client = new Solarium_Client($config);
+$buffer = $client->getPlugin('bufferedadd');
+$buffer->setBufferSize(10); // this is quite low, in most cases you can use a much higher value
+
+// also register a plugin for outputting events
+$debug = new simpleDebug();
+$client->registerPlugin('debugger', $debug);
+
+// let's insert 25 docs
+for ($i=1; $i<=25; $i++) {
+
+    // create a new document with dummy data and add it to the buffer
+    $data = array(
+        'id' => 'test_'.$i,
+        'name' => 'test for buffered add',
+        'price' => $i,
+    );
+    $buffer->createDocument($data);
+
+    // alternatively you could create document instances yourself and use the addDocument(s) method
+}
+
+// At this point two flushes will already have been done by the buffer automatically (at the 10th and 20th doc), now
+// manually flush the remainder. Alternatively you can use the commit method if you want to include a commit command.
+$buffer->flush();
+
+// In total 3 flushes (requests) have been sent to Solr. This should be visible in this output:
+$debug->display();
+
+htmlFooter();
\ No newline at end of file

examples/index.html

@@ -124,6 +124,7 @@
                 <li><a href="7.2-plugin-postbigrequest.php">7.2 Post Big Requests</a></li>
                 <li><a href="7.3-plugin-customizerequest.php">7.3 Customize Requests</a></li>
                 <li><a href="7.4-plugin-parallelexecution.php">7.4 Parallel Execution</a></li>
+                <li><a href="7.5-plugin-bufferedadd.php">7.5 Buffered Add for documents</a></li>
             </ul>
 
         </ul>

library/Solarium/Client.php

@@ -164,6 +164,7 @@ class Solarium_Client extends Solarium_Configurable
         'postbigrequest' => 'Solarium_Plugin_PostBigRequest',
         'customizerequest' => 'Solarium_Plugin_CustomizeRequest',
         'parallelexecution' => 'Solarium_Plugin_ParallelExecution',
+        'bufferedadd' => 'Solarium_Plugin_BufferedAdd',
     );
 
     /**

library/Solarium/Plugin/Abstract.php

@@ -70,7 +70,7 @@ abstract class Solarium_Plugin_Abstract extends Solarium_Configurable
     /**
      * Plugin init function
      *
-     * This is an extension point for plugin implemenations.
+     * This is an extension point for plugin implementations.
      * Will be called as soon as $this->_client and options have been set.
      *
      * @return void
@@ -157,7 +157,7 @@ abstract class Solarium_Plugin_Abstract extends Solarium_Configurable
 
     /**
      * postExecute hook
-     * 
+     *
      * @param Solarium_Query $query
      * @param Solarium_Result $result
      * @return void
@@ -165,7 +165,7 @@ abstract class Solarium_Plugin_Abstract extends Solarium_Configurable
     public function postExecute($query, $result)
     {
     }
-    
+
     /**
      * preCreateQuery hook
      *
@@ -179,7 +179,7 @@ abstract class Solarium_Plugin_Abstract extends Solarium_Configurable
 
     /**
      * postCreateQuery hook
-     * 
+     *
      * @param string $type
      * @param mixed $options
      * @param Solarium_Query

library/Solarium/Plugin/BufferedAdd.php

+<?php
+/**
+ * Copyright 2011 Bas de Nooijer. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this listof conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the copyright holder.
+ *
+ * @copyright Copyright 2011 Bas de Nooijer <solarium@raspberry.nl>
+ * @license http://github.com/basdenooijer/solarium/raw/master/COPYING
+ * @link http://www.solarium-project.org/
+ *
+ * @package Solarium
+ */
+
+/**
+ * Buffered add plugin
+ *
+ * If you need to add (or update) a big number of documents to Solr it's much more efficient to do so in batches.
+ * This plugin makes this as easy as possible.
+ *
+ * @package Solarium
+ * @subpackage Plugin
+ */
+class Solarium_Plugin_BufferedAdd extends Solarium_Plugin_Abstract
+{
+
+    /**
+     * Default options
+     *
+     * @var array
+     */
+    protected $_options = array(
+        'buffersize' => 100,
+    );
+
+    /**
+     * Update query instance
+     *
+     * @var Solarium_Query_Update
+     */
+    protected $_updateQuery;
+
+    /**
+     * Buffered documents
+     *
+     * @var array
+     */
+    protected $_buffer = array();
+
+    /**
+     * Plugin init function
+     *
+     * This is an extension point for plugin implementations.
+     * Will be called as soon as $this->_client and options have been set.
+     *
+     * @return void
+     */
+    protected function _initPlugin()
+    {
+        $this->_updateQuery = $this->_client->createUpdate();
+    }
+
+    /**
+     * Set buffer size option
+     *
+     * @param int $size
+     * @return Solarium_Configurable
+     */
+    public function setBufferSize($size)
+    {
+        return $this->_setOption('buffersize', $size);
+    }
+
+    /**
+     * Get buffer size option value
+     *
+     * @return int
+     */
+    public function getBufferSize()
+    {
+        return $this->getOption('buffersize');
+    }
+
+    /**
+     * Create a document object instance and add it to the buffer
+     *
+     * @param array $fields
+     * @param array $boosts
+     * @return self Provides fluent interface
+     */
+    public function createDocument($fields, $boosts = array())
+    {
+        $doc = $this->_updateQuery->createDocument($fields, $boosts);
+        $this->addDocument($doc);
+
+        return $this;
+    }
+
+    /**
+     * Add a document
+     *
+     * @param Solarium_Document_ReadOnly $document
+     * @return self Provides fluent interface
+     */
+    public function addDocument($document)
+    {
+        $this->_buffer[] = $document;
+        if (count($this->_buffer) == $this->_options['buffersize']) {
+            $this->flush();
+        }
+
+        return $this;
+    }
+
+    /**
+     * Add multiple documents
+     *
+     * @param array
+     * @return self Provides fluent interface
+     */
+    public function addDocuments($documents)
+    {
+        foreach ($documents as $document) {
+            $this->addDocument($document);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Get all documents currently in the buffer
+     *
+     * Any previously flushed documents will not be included!
+     *
+     * @return array
+     */
+    public function getDocuments()
+    {
+        return $this->_buffer;
+    }
+
+    /**
+     * Clear any buffered documents
+     *
+     * @return self Provides fluent interface
+     */
+    public function clear()
+    {
+        $this->_updateQuery = $this->_client->createUpdate();
+        $this->_buffer = array();
+        return $this;
+    }
+
+    /**
+     * Flush any buffered documents to Solr
+     *
+     * @param boolean $overwrite
+     * @param int $commitWithin
+     * @return boolean|Solarium_Result_Update
+     */
+    public function flush($overwrite = null, $commitWithin = null)
+    {
+        if(count($this->_buffer) == 0) {
+            // nothing to do
+            return false;
+        }
+
+        $this->_client->triggerEvent('BufferedAddFlushStart', array($this->_buffer));
+
+        $this->_updateQuery->addDocuments($this->_buffer, $overwrite, $commitWithin);
+        $result = $this->_client->update($this->_updateQuery);
+        $this->clear();
+
+        $this->_client->triggerEvent('BufferedAddFlushEnd', array($result));
+
+        return $result;
+    }
+
+    /**
+     * Commit changes
+     *
+     * Any remaining documents in the buffer will also be flushed
+     *
+     * @param boolean $overwrite
+     * @param boolean $waitFlush
+     * @param boolean $waitSearcher
+     * @param boolean $expungeDeletes
+     * @return Solarium_Result_Update
+     */
+    public function commit($overwrite = null, $waitFlush = null, $waitSearcher = null, $expungeDeletes = null)
+    {
+        $this->_client->triggerEvent('BufferedAddCommitStart', array($this->_buffer));
+
+        $this->_updateQuery->addDocuments($this->_buffer, $overwrite);
+        $this->_updateQuery->addCommit($waitFlush, $waitSearcher, $expungeDeletes);
+        $result = $this->_client->update($this->_updateQuery);
+        $this->clear();
+
+        $this->_client->triggerEvent('BufferedAddCommitEnd', array($result));
+
+        return $result;
+    }
+
+}
\ No newline at end of file

tests/Solarium/Plugin/BufferedAddTest.php

+<?php
+/**
+ * Copyright 2011 Bas de Nooijer. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this listof conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the copyright holder.
+ */
+
+class Solarium_Plugin_BufferedAddTest extends PHPUnit_Framework_TestCase
+{
+    /**
+     * @var Solarium_Plugin_BufferedAdd
+     */
+    protected $_plugin;
+
+    public function setUp()
+    {
+        $this->_plugin = new Solarium_Plugin_BufferedAdd();
+        $this->_plugin->init(new Solarium_Client(), array());
+    }
+
+    public function testSetAndGetBufferSize()
+    {
+        $this->_plugin->setBufferSize(500);
+        $this->assertEquals(500, $this->_plugin->getBufferSize());
+    }
+
+    public function testAddDocument()
+    {
+        $doc = new Solarium_Document_ReadWrite();
+        $doc->id = '123';
+        $doc->name = 'test';
+
+        $this->_plugin->addDocument($doc);
+
+        $this->assertEquals(array($doc), $this->_plugin->getDocuments());
+    }
+
+    public function testCreateDocument()
+    {
+        $data = array('id' => '123', 'name' => 'test');
+        $doc = new Solarium_Document_ReadWrite($data);
+
+        $this->_plugin->createDocument($data);
+
+        $this->assertEquals(array($doc), $this->_plugin->getDocuments());
+    }
+
+    public function testAddDocuments()
+    {
+        $doc1 = new Solarium_Document_ReadWrite();
+        $doc1->id = '123';
+        $doc1->name = 'test';
+
+        $doc2 = new Solarium_Document_ReadWrite();
+        $doc2->id = '234';
+        $doc2->name = 'test2';
+
+        $docs = array($doc1, $doc2);
+
+        $this->_plugin->addDocuments($docs);
+
+        $this->assertEquals($docs, $this->_plugin->getDocuments());
+    }
+
+    public function testAddDocumentAutoFlush()
+    {
+        $observer = $this->getMock('Solarium_Plugin_BufferedAdd', array('flush'));
+        $observer->expects($this->once())->method('flush');
+        $observer->setBufferSize(1);
+
+        $doc1 = new Solarium_Document_ReadWrite();
+        $doc1->id = '123';
+        $doc1->name = 'test';
+
+        $doc2 = new Solarium_Document_ReadWrite();
+        $doc2->id = '234';
+        $doc2->name = 'test2';
+
+        $docs = array($doc1, $doc2);
+
+        $observer->addDocuments($docs);
+    }
+
+    public function testClear()
+    {
+        $doc = new Solarium_Document_ReadWrite();
+        $doc->id = '123';
+        $doc->name = 'test';
+
+        $this->_plugin->addDocument($doc);
+        $this->_plugin->clear();
+
+        $this->assertEquals(array(), $this->_plugin->getDocuments());
+    }
+
+    public function testFlushEmptyBuffer()
+    {
+        $this->assertEquals(false, $this->_plugin->flush());
+    }
+
+    public function testFlush()
+    {
+        $data = array('id' => '123', 'name' => 'test');
+        $doc = new Solarium_Document_ReadWrite($data);
+
+        $mockUpdate = $this->getMock('Solarium_Query_Update', array('addDocuments'));
+        $mockUpdate->expects($this->once())->method('addDocuments')->with($this->equalTo(array($doc)),$this->equalTo(true),$this->equalTo(12));
+
+        $mockClient = $this->getMock('Solarium_Client', array('createUpdate', 'update', 'triggerEvent'));
+        $mockClient->expects($this->exactly(2))->method('createUpdate')->will($this->returnValue($mockUpdate));
+        $mockClient->expects($this->once())->method('update')->will($this->returnValue('dummyResult'));
+        $mockClient->expects($this->exactly(2))->method('triggerEvent');
+
+        $plugin = new Solarium_Plugin_BufferedAdd();
+        $plugin->init($mockClient, array());
+        $plugin->addDocument($doc);
+
+        $this->assertEquals('dummyResult', $plugin->flush(true,12));
+    }
+
+    public function testCommit()
+    {
+        $data = array('id' => '123', 'name' => 'test');
+        $doc = new Solarium_Document_ReadWrite($data);
+
+        $mockUpdate = $this->getMock('Solarium_Query_Update', array('addDocuments', 'addCommit'));
+        $mockUpdate->expects($this->once())->method('addDocuments')->with($this->equalTo(array($doc)),$this->equalTo(true));
+        $mockUpdate->expects($this->once())->method('addCommit')->with($this->equalTo(false),$this->equalTo(true),$this->equalTo(false));
+
+        $mockClient = $this->getMock('Solarium_Client', array('createUpdate', 'update', 'triggerEvent'));
+        $mockClient->expects($this->exactly(2))->method('createUpdate')->will($this->returnValue($mockUpdate));
+        $mockClient->expects($this->once())->method('update')->will($this->returnValue('dummyResult'));
+        $mockClient->expects($this->exactly(2))->method('triggerEvent');
+
+        $plugin = new Solarium_Plugin_BufferedAdd();
+        $plugin->init($mockClient, array());
+        $plugin->addDocument($doc);
+
+        $this->assertEquals('dummyResult', $plugin->commit(true, false, true, false));
+    }
+
+}
\ No newline at end of file