Add rich text document extraction

New query type to upload rich text documents using the
ExtractingRequestHandler in Solr.

library/Solarium/Client.php

@@ -67,6 +67,11 @@ class Solarium_Client extends Solarium_Configurable
      */
     const QUERYTYPE_UPDATE = 'update';
 
+    /**
+     * Querytype extract
+     */
+    const QUERYTYPE_EXTRACT = 'extract';
+
     /**
      * Querytype ping
      */
@@ -122,6 +127,11 @@ class Solarium_Client extends Solarium_Configurable
             'requestbuilder' => 'Solarium_Client_RequestBuilder_Update',
             'responseparser' => 'Solarium_Client_ResponseParser_Update'
         ),
+        self::QUERYTYPE_EXTRACT => array(
+            'query'          => 'Solarium_Query_Extract',
+            'requestbuilder' => 'Solarium_Client_RequestBuilder_Extract',
+            'responseparser' => 'Solarium_Client_ResponseParser_Update'
+        ),
         self::QUERYTYPE_PING => array(
             'query'          => 'Solarium_Query_Ping',
             'requestbuilder' => 'Solarium_Client_RequestBuilder_Ping',
@@ -776,6 +786,17 @@ class Solarium_Client extends Solarium_Configurable
         return $this->createQuery(self::QUERYTYPE_UPDATE, $options);
     }
 
+    /**
+     * Create an extract query instance
+     *
+     * @param mixed $options
+     * @return Solarium_Query_Extract
+     */
+    public function createExtract($options = null)
+    {
+        return $this->createQuery(self::QUERYTYPE_EXTRACT, $options);
+    }
+
     /**
      * Create a ping query instance
      *

library/Solarium/Client/Adapter/ZendHttp.php

 <?php
 /**
  * Copyright 2011 Bas de Nooijer. All rights reserved.
+ * Copyright 2012 Alexander Brausewetter. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -29,6 +30,7 @@
  * policies, either expressed or implied, of the copyright holder.
  *
  * @copyright Copyright 2011 Bas de Nooijer <solarium@raspberry.nl>
+ * @copyright Copyright 2012 Alexander Brausewetter <alex@helpdeskhq.com>
  * @license http://github.com/basdenooijer/solarium/raw/master/COPYING
  * @link http://www.solarium-project.org/
  *
@@ -151,15 +153,54 @@ class Solarium_Client_Adapter_ZendHttp extends Solarium_Client_Adapter
     public function execute($request)
     {
         $client = $this->getZendHttp();
+        $client->resetParameters();
+
+        switch ($request->getMethod()) {
+            case Solarium_Client_Request::METHOD_GET:
+                $client->setMethod(Zend_Http_Client::GET);
+                $client->setParameterGet($request->getQueryAsArray());
+                break;
+            case Solarium_Client_Request::METHOD_POST:
+                $client->setMethod(Zend_Http_Client::POST);
+
+                if ($request->getFileUpload()) {
+                    $this->_prepareFileUpload($client, $request);
+                } else {
+                    $client->setParameterGet($request->getQueryAsArray());
+                    $client->setRawData($request->getRawData());
+                    $request->addHeader('Content-Type: text/xml; charset=UTF-8');
+                }
+                break;
+            case Solarium_Client_Request::METHOD_HEAD:
+                $client->setMethod(Zend_Http_Client::HEAD);
+                $client->setParameterGet($request->getQueryAsArray());
+                break;
+            default:
+                throw new Solarium_Exception('Unsupported method: ' . $request->getMethod());
+                break;
+        }
 
-        $client->setMethod($request->getMethod());
         $client->setUri($this->getBaseUri() . $request->getUri());
         $client->setHeaders($request->getHeaders());
-        $client->setRawData($request->getRawData());
 
         $response = $client->request();
 
-        // throw an exception in case of a HTTP error
+        return $this->_prepareResponse(
+            $request,
+            $response
+        );
+    }
+
+    /**
+     * Prepare a solarium response from the given request and client 
+     * response
+     * 
+     * @param Solarium_Client_Request $request
+     * @param Zend_Http_Response $response
+     * @return Solarium_Client_Response
+     */
+    protected function _prepareResponse($request, $response)
+    {
         if ($response->isError()) {
             throw new Solarium_Client_HttpException(
                 $response->getMessage(),
@@ -179,4 +220,26 @@ class Solarium_Client_Adapter_ZendHttp extends Solarium_Client_Adapter
         return new Solarium_Client_Response($data, $headers);
     }
 
-}
\ No newline at end of file
+    /**
+     * Prepare the client to send the file and params in request
+     * 
+     * @param Zend_Http_Client $client 
+     * @param Solarium_Client_Request $request 
+     * @return void
+     */
+    protected function _prepareFileUpload($client, $request)
+    {
+        $filename = $request->getFileUpload();
+
+        if (($content = @file_get_contents($filename)) === false) {
+            throw new Solarium_Exception("Unable to read file '{$filename}' for upload");
+        }
+
+        $client->setFileUpload('content', 'content', $content, 'application/octet-stream; charset=binary');
+
+        // set query params as "multipart/form-data" fields
+        foreach ($request->getQueryAsArray() as $name => $value) {
+            $client->setFileUpload(null, $name, $value, 'text/plain; charset=utf-8');
+        }
+    }
+}

library/Solarium/Client/Request.php

@@ -101,6 +101,9 @@ class Solarium_Client_Request extends Solarium_Configurable
                 case 'rawdata':
                     $this->setRawData($value);
                     break;
+                case 'file':
+                    $this->setFileUpload($value);
+                    break;
                 case 'param':
                     $this->setParams($value);
                     break;
@@ -294,6 +297,32 @@ class Solarium_Client_Request extends Solarium_Configurable
         return $this;
     }
 
+    /**
+     * Get the file to upload via "multipart/form-data" POST request
+     *
+     * @return string|null
+     */
+    public function getFileUpload()
+    {
+        return $this->getOption('file');
+    }
+
+    /**
+     * Set the file to upload via "multipart/form-data" POST request
+     *
+     * @param string $filename Name of file to upload
+     * @return Solarium_Client_Request
+     */
+    public function setFileUpload($filename)
+    {
+        if (!is_file($filename) || !is_readable($filename)) {
+            throw new Solarium_Exception("Unable to read file '{$filename}' for upload");
+        }
+
+        $this->_setOption('file', $filename);
+        return $this;
+    }
+
     /**
      * Get all request headers
      *
@@ -357,23 +386,24 @@ class Solarium_Client_Request extends Solarium_Configurable
     }
 
     /**
-     * Get an URI for this request
+     * Get the request URI
      *
      * @return string
      */
     public function getUri()
     {
-        return $this->getHandler() . '?' . $this->getQueryString();
+        return $this->getHandler();
     }
 
     /**
-     * Get the query string for this request
+     * Get the request URI query
      *
      * @return string
      */
-    public function getQueryString()
+    public function getQuery()
     {
         $queryString = '';
+
         if (count($this->_params) > 0) {
             $queryString = http_build_query($this->_params, null, '&');
             $queryString = preg_replace(
@@ -386,6 +416,16 @@ class Solarium_Client_Request extends Solarium_Configurable
         return $queryString;
     }
 
+    /**
+     * Get the request URI query as an associative array of key => value pairs
+     *
+     * @return array
+     */
+    public function getQueryAsArray()
+    {
+        return $this->getParams();
+    }
+
     /**
      * Magic method enables a object to be transformed to a string
      *
@@ -401,9 +441,10 @@ class Solarium_Client_Request extends Solarium_Configurable
                 . 'header: ' . print_r($this->getHeaders(), 1) //don't add newline when using print_r
                 . 'resource: ' . $this->getUri() . "\n"
                 . 'resource urldecoded: ' . urldecode($this->getUri()) . "\n"
-                . 'raw data: ' . $this->getRawData() . "\n";
+                . 'raw data: ' . $this->getRawData() . "\n"
+                . 'file upload: ' . $this->getFileUpload() . "\n";
 
         return $output;
     }
 
-}
\ No newline at end of file
+}

library/Solarium/Client/RequestBuilder/Extract.php

+<?php
+/**
+ * Copyright 2011 Bas de Nooijer. All rights reserved.
+ * Copyright 2012 Alexander Brausewetter. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this listof conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the copyright holder.
+ *
+ * @copyright Copyright 2011 Bas de Nooijer <solarium@raspberry.nl>
+ * @copyright Copyright 2012 Alexander Brausewetter <alex@helpdeskhq.com>
+ * @license http://github.com/basdenooijer/solarium/raw/master/COPYING
+ * @link http://www.solarium-project.org/
+ *
+ * @package Solarium
+ * @subpackage Client
+ */
+
+/**
+ * Build an extract request
+ *
+ * @package Solarium
+ * @subpackage Client
+ */
+class Solarium_Client_RequestBuilder_Extract extends Solarium_Client_RequestBuilder
+{
+    /**
+     * Build the request
+     *
+     * @param Solarium_Query_Extract $query
+     * @return Solarium_Client_Request
+     */
+    public function build($query)
+    {
+        $request = parent::build($query);
+        $request->setMethod(Solarium_Client_Request::METHOD_POST);
+
+        // common options
+
+        $request->addParam('commit',       $query->getCommit());
+        $request->addParam('commitWithin', $query->getCommitWithin());
+
+        $request->addParam('uprefix',      $query->getUprefix());
+        $request->addParam('lowernames',   $query->getLowernames());
+        $request->addParam('defaultField', $query->getDefaultField());
+
+        foreach ($query->getFieldMappings() AS $fromField => $toField) {
+            $request->addParam('fmap.' . $fromField, $toField);
+        }
+
+        // document
+
+        if (($doc = $query->getDocument()) != null) {
+            if ($doc->getBoost() !== null) {
+                throw new Solarium_Exception('Extract does not support document-level boosts, use field boosts instead.');
+            }
+
+            // literal.*
+            foreach ($doc->getFields() AS $name => $value) {
+                $value = (array) $value;
+                foreach ($value AS $multival) {
+                    $request->addParam('literal.' . $name, $multival);
+                }
+            }
+
+            // boost.*
+            foreach ($doc->getFieldBoosts() AS $name => $value) {
+                $request->addParam('boost.' . $name, $value);
+            }
+        }
+
+        // file
+
+        $request->setFileUpload($query->getFile());
+        $request->addParam('resource.name', basename($query->getFile()));
+
+        return $request;
+    }
+}

library/Solarium/Document/ReadWrite.php

@@ -182,6 +182,16 @@ class Solarium_Document_ReadWrite extends Solarium_Document_ReadOnly
         return $this;
     }
 
+    /**
+     * Get boost values for all fields
+     *
+     * @return array
+     */
+    public function getFieldBoosts()
+    {
+        return $this->_fieldBoosts;
+    }
+
     /**
      * Set the document boost value
      *
@@ -249,4 +259,4 @@ class Solarium_Document_ReadWrite extends Solarium_Document_ReadOnly
         $this->removeField($name);
     }
 
-}
\ No newline at end of file
+}

library/Solarium/Query/Extract.php

+<?php
+/**
+ * Copyright 2011 Bas de Nooijer. All rights reserved.
+ * Copyright 2012 Alexander Brausewetter. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this listof conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the copyright holder.
+ *
+ * @copyright Copyright 2011 Bas de Nooijer <solarium@raspberry.nl>
+ * @copyright Copyright 2012 Alexander Brausewetter <alex@helpdeskhq.com>
+ * @license http://github.com/basdenooijer/solarium/raw/master/COPYING
+ * @link http://www.solarium-project.org/
+ *
+ * @package Solarium
+ * @subpackage Query
+ */
+
+/**
+ * Extract query
+ *
+ * Sends a document extract request to Solr, i.e. upload rich document content 
+ * such as PDF, Word or HTML, parse the file contents and add it to the index. 
+ *
+ * The Solr server must have the {@link http://wiki.apache.org/solr/ExtractingRequestHandler 
+ * ExtractingRequestHandler} enabled.
+ *
+ * @package Solarium
+ * @subpackage Query
+ */
+class Solarium_Query_Extract extends Solarium_Query
+{
+    /**
+     * Default options
+     *
+     * @var array
+     */
+    protected $_options = array(
+        'handler'     => 'update/extract',
+        'resultclass' => 'Solarium_Result_Update',
+    );
+
+    /**
+     * Field name mappings
+     *
+     * @var array
+     */
+    protected $_fieldMappings = array();
+
+    /**
+     * Get type for this query
+     *
+     * @return string
+     */
+    public function getType()
+    {
+        return Solarium_Client::QUERYTYPE_EXTRACT;
+    }
+    
+    /**
+     * Initialize options
+     *
+     * Several options need some extra checks or setup work, for these options
+     * the setters are called.
+     *
+     * @return void
+     */
+    protected function _init()
+    {
+        if (isset($this->_options['fmap'])) {
+            $this->setFieldMappings($this->_options['fmap']);
+        }
+    }
+
+    // {{{ Options
+
+    /**
+     * Set the document with literal fields and boost settings
+     *
+     * The fields in the document are indexed together with the generated 
+     * fields that Solr extracts from the file.
+     * 
+     * @param Solarium_Document_ReadWrite $document
+     * @return Solarium_Query_Extract
+     */
+    public function setDocument($document)
+    {
+        return $this->_setOption('document', $document);
+    }
+
+    /**
+     * Get the document with literal fields and boost settings
+     * 
+     * @return Solarium_Document_ReadWrite|null
+     */
+    public function getDocument()
+    {
+        return $this->getOption('document');
+    }
+
+    /**
+     * Set the file to upload and index
+     * 
+     * @param string $filename
+     * @return Solarium_Query_Extract
+     */
+    public function setFile($filename)
+    {
+        return $this->_setOption('file', $filename);
+    }
+
+    /**
+     * Get the file to upload and index
+     * 
+     * @return string|null
+     */
+    public function getFile()
+    {
+        return $this->getOption('file');
+    }
+
+    /**
+     * Set the prefix for fields that are not defined in the schema
+     *
+     * @param string $uprefix
+     * @return Solarium_Query_Extract
+     */
+    public function setUprefix($uprefix)
+    {
+        return $this->_setOption('uprefix', $uprefix);
+    }
+
+    /**
+     * Get the prefix for fields that are not defined in the schema
+     * 
+     * @return string|null
+     */
+    public function getUprefix()
+    {
+        return $this->getOption('uprefix');
+    }
+
+    /**
+     * Set the field to use if uprefix is not specified and a field cannot be 
+     * determined
+     *
+     * @param string $defaultField
+     * @return Solarium_Query_Extract
+     */
+    public function setDefaultField($defaultField)
+    {
+        return $this->_setOption('defaultField', $defaultField);
+    }
+
+    /**
+     * Get the field to use if uprefix is not specified and a field cannot be 
+     * determined
+     * 
+     * @return string|null
+     */
+    public function getDefaultField()
+    {
+        return $this->getOption('defaultField');
+    }
+
+    /**
+     * Set if all field names should be mapped to lowercase with underscores. 
+     * For example, Content-Type would be mapped to content_type.
+     *
+     * @param bool $lowerNames
+     * @return Solarium_Query_Extract
+     */
+    public function setLowernames($lowerNames)
+    {
+        return $this->_setOption('lowernames', (bool) $lowerNames);
+    }
+
+    /**
+     * Get if all field names should be mapped to lowercase with underscores
+     * 
+     * @return bool
+     */
+    public function getLowernames()
+    {
+        return $this->getOption('lowernames');
+    }
+
+    /**
+     * Set if the extract should be committed immediately
+     *
+     * @param bool $commit
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function setCommit($commit)
+    {
+        return $this->_setOption('commit', (bool) $commit);
+    }
+
+    /**
+     * Get if the extract should be committed immediately
+     * 
+     * @return bool
+     */
+    public function getCommit()
+    {
+        return $this->getOption('commit');
+    }
+
+    /**
+     * Set milliseconds until extract update is committed. Since Solr 3.4
+     *
+     * @param int $commitWithin
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function setCommitWithin($commitWithin)
+    {
+        return $this->_setOption('commitWithin', $commitWithin);
+    }
+
+    /**
+     * Get milliseconds until extract update is committed. Since Solr 3.4
+     * 
+     * @return int
+     */
+    public function getCommitWithin()
+    {
+        return $this->getOption('commitWithin');
+    }
+
+    // }}}
+
+    // {{{ Field Mappings
+
+    /**
+     * Add a name mapping from one field to another
+     *
+     * Example: fmap.content=text will cause the content field normally 
+     * generated by Tika to be moved to the "text" field.
+     *
+     * @param string      $fromField Original field name
+     * @param mixed|array $toField   New field name
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function addFieldMapping($fromField, $toField)
+    {
+        $this->_fieldMappings[$fromField] = $toField;
+
+        return $this;
+    }
+
+    /**
+     * Add multiple field name mappings
+     *
+     * @param array $mappings Name mapping in the form [$fromField => $toField, ...]
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function addFieldMappings($mappings)
+    {
+        foreach ($mappings AS $fromField => $toField) {
+            $this->addFieldMapping($fromField, $toField);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Remove a field name mapping
+     *
+     * @param string $fromField
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function removeFieldMapping($fromField)
+    {
+        if (isset($this->_fieldMappings[$fromField])) {
+            unset($this->_fieldMappings[$fromField]);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Remove all field name mappings
+     *
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function clearFieldMappings()
+    {
+        $this->_fieldMappings = array();
+        return $this;
+    }
+
+    /**
+     * Get all field name mappings
+     *
+     * @return array
+     */
+    public function getFieldMappings()
+    {
+        return $this->_fieldMappings;
+    }
+
+    /**
+     * Set many field name mappings. This overwrites any existing fields.
+     *
+     * @param array $mappings Name mapping in the form [$fromField => $toField, ...]
+     * @return Solarium_Query_Extract Provides fluent interface
+     */
+    public function setFieldMappings($mappings)
+    {
+        $this->clearFieldMappings();
+        $this->addFieldMappings($mappings);
+
+        return $this;
+    }
+
+    // }}}
+}