Changeset 11881

Timestamp:

08/30/07 16:07:23 (1 year ago)

Author:

rambo

Message:

refs #102, closes #64

use moving window to handle offset/limit for DBA QB

TODO: clean up

Files:

• trunk/midcom/midcom.core/midcom/core/querybuilder.php (modified) (13 diffs)

trunk/midcom/midcom.core/midcom/core/querybuilder.php

@@ -182 +182 @@
-     * While on-site, this is enabled by default, in AIS it is disabled by default.
-     */
-
+
+
-    /**
-     * The class this qb is working on.
@@ -188 +189 @@
-     */
-    var $classname = null;
+
+    /**
+     * Keep track of GUIDs seen to avoid workaround ML bug
+     */
+    var $_seen_guids = array();
+    
+    var $_qb_error_result = 'UNDEFINED';
+    
+    /**
+     * When determining window sizes for offset/limit queries use this as minimum size
+     */
+    var $min_window_size = 10;
+
+    /**
+     * When determining window sizes for offset/limit queries use this as maximum size
+     */
+    var $max_window_size = 500;
-
-    /**
@@ -196 +214 @@
-     *
-     * @param string $classname The classname which should be queried.
+     * @todo remove baseclass resolution, Midgard core can handle extended classnames correctly nowadays
-     */
-    function midcom_core_querybuilder($classname)
@@ -205 +224 @@
-        {
-            $baseclass = $_class_mapping_cache[$classname];
-    
+
-        else
-        {
@@ -234 +253 @@
-        }
-
+        $this->_real_class = $classname;
-        $this->_qb = new midgard_query_builder($baseclass);
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
-
-    /**
@@ -258 +271 @@
-    }
-
+    function _execute_and_check_privileges($false_on_empty_mgd_resultset = false)
+    {
+        debug_push_class(__CLASS__, __FUNCTION__);
+        $result = $this->_qb->execute();
+        if (!is_array($result))
+        {
+            $this->_qb_error_result = $result;
+            debug_print_r('Result was:', $result);
+            debug_add('The querybuilder failed to execute, aborting.', MIDCOM_LOG_ERROR);
+            debug_add('Last Midgard error was: ' . mgd_errstr(), MIDCOM_LOG_ERROR);
+            if (isset($php_errormsg))
+            {
+                debug_add("Error message was: {$php_errormsg}", MIDCOM_LOG_ERROR);
+            }
+
+            debug_pop();
+            return $result;
+        }
+        debug_add('Got ' . count($result) . ' initial results');
+        if (   empty($result)
+            && $false_on_empty_mgd_resultset)
+        {
+            debug_pop();
+            return false;
+        }
+
+        // Workaround until the QB returns the correct type, refetch everything
+        $newresult = Array();
+        $classname = $this->_real_class;
+        $skipped_objects = 0;
+        $this->denied = 0;
+        foreach ($result as $key => $value)
+        {
+            // Workaround to ML bug where we get multiple results in non-strict mode
+            if (isset($this->_seen_guids[$value->guid]))
+            {
+                debug_add("The {$classname} object {$value->guid} has already been seen, probably MultiLang bug", MIDCOM_LOG_WARN);
+                //debug_add('var_export($seen_guids): ' . var_export($this->_seen_guids, true));
+                continue;
+            }
+            $this->_seen_guids[$value->guid] = true;
+
+            // Create a new object instance (checks read privilege implicitly) using the copy-constuctor.
+            $object = new $classname($value);
+
+            if (mgd_errno() == MGD_ERR_ACCESS_DENIED)
+            {
+                // This is logged by the callers
+                $this->denied++;
+                $skipped_objects++;
+                continue;
+            }
+
+            if (   ! $object
+                || ! is_object($object))
+            {
+                debug_add("Could not create a MidCOM DBA instance of the {$classname} ID {$value->id}. See debug level log for details.",
+                    MIDCOM_LOG_INFO);
+                $skipped_objects++;
+                continue;
+            }
+
+            // Check visibility
+            if ($this->hide_invisible)
+            {
+                $metadata =& midcom_helper_metadata::retrieve($object);
+                if (! $metadata)
+                {
+                    debug_add("Could not create a MidCOM metadata instance for {$classname} ID {$value->id}, assuming an invisible object.",
+                        MIDCOM_LOG_INFO);
+                    $skipped_objects++;
+                    continue;
+                }
+
+                if (! $metadata->is_object_visible_onsite())
+                {
+                    debug_add("The {$classname} ID {$value->id} is hidden by metadata.", MIDCOM_LOG_INFO);
+                    $skipped_objects++;
+                    continue;
+                }
+            }
+
+            $newresult[] = $object;
+        }
+        debug_add('Returning ' . count($newresult) . ' items');
+        debug_pop();
+        return $newresult;
+    }
-
-    /**
@@ -281 +382 @@
-     * @todo Implement proper count / Limit support.
-     */
-
-
-
-
+
+
+
+
+
+
-        if (! call_user_func_array(array($this->_real_class, '_on_prepare_exec_query_builder'), array(&$this)))
-        {
+            debug_push_class(__CLASS__, __FUNCTION__);
-            debug_add('The _on_prepare_exec_query_builder callback returned false, so we abort now.');
-            debug_pop();
@@ -294 +398 @@
-        if ($this->_constraint_count == 0)
-        {
+            debug_push_class(__CLASS__, __FUNCTION__);
+            debug_add('This Query Builder instance has no constraints (set loglevel to debug to see stack trace)', MIDCOM_LOG_WARN);
+            debug_print_function_stack('We were called from here:');
+            debug_pop();
+        }
+
+        if (   empty($this->_limit)
+            && empty($this->_offset))
+        {
+            // No point to do windowing
+            $newresult = $this->_execute_and_check_privileges();
+            if (!is_array($newresult))
+            {
+                return $newresult;
+            }
+        }
+        else
+        {
+            //debug_push_class(__CLASS__, __FUNCTION__);
+            $newresult = array();
+            // Must be copies
+            $limit = $this->_limit;
+            $offset = $this->_offset;
+            $i = 0;
+            $this->_set_limit_offset_window($i);
+            
+            while (($resultset = $this->_execute_and_check_privileges(true)) !== false)
+            {
+                //debug_add("Iteration loop #{$i}");
+                if ($this->_qb_error_result !== 'UNDEFINED')
+                {
+                    // QB failed in above method TODO: better catch
+                    /*
+                    debug_add('_execute_and_check_privileges caught QB error, returning that now', MIDCOM_LOG_WARN);
+                    debug_pop();
+                    */
+                    return $this->_qb_error_result;
+                }
+
+                foreach($resultset as $object)
+                {
+                    // We still have offset left to skip
+                    if ($offset)
+                    {
+                        //debug_add("Offset of {$this->_offset} not yet reached, continuing loop");
+                        $offset--;
+                        continue;
+                    }
+                    // We have hit our limit
+                    if (   $this->_limit > 0
+                        && $limit == 0)
+                    {
+                        //debug_add("Limit of {$this->_limit} hit, breaking out of loops");
+                        break 2;
+                    }
+
+                    $newresult[] = $object;
+
+                    if ($this->_limit > 0)
+                    {
+                        $limit--;
+                    }
+                }
+                ++$i;
+                $this->_set_limit_offset_window($i);
+            }
+        }
+
+        call_user_func_array(array($this->_real_class, '_on_process_query_result'), array(&$newresult));
+
+        /*
+        // correct record count by the number of limit-skipped objects.
+        $this->count = count($newresult) + $skipped_objects;
+        */
+        $this->count = count($newresult);
+
+        //debug_pop();
+        return $newresult;
+    }
+
+    function _set_limit_offset_window($iteration)
+    {
+        /*
+        debug_push_class(__CLASS__, __FUNCTION__);
+        debug_add("Called for iteration #{$iteration}");
+        */
+        static $window_size = 0;
+        if (!$window_size)
+        {
+            // Try to be smart about the window size
+            switch (true)
+            {
+                case (   empty($this->_offset)
+                      && $this->_limit):
+                    // Get limited number from start (I supposed generally less than 50% will be unreadable)
+                    $window_size = round($this->_limit * 1.5);
+                    break;
+                case (   empty($this->_limit)
+                      && $this->_offset):
+                    // Get rest from offset
+                    /* TODO: Somehow factor in that if we have huge number of objects and relatively small offset we want to increase window size
+                    $full_object_count = $this->_qb->count();
+                    */
+                    $window_size = round($this->_offset * 2);
+                case (   $this->_offset > $this->_limit):
+                    // Offset is greater than limit, basically this is almost the same problem as above
+                    $window_size = round($this->_offset * 2);
+                    break;
+                case (   $this->_limit > $this->_offset):
+                    // Limit is greater than offset, this is probably similar to getting limited number from beginning
+                    $window_size = round($this->_limit * 2);
+                    break;
+                case ($this->_limit == $this->_offset):
+                    $window_size = round($this->_offset * 2);
+                    break;
+            }
+
+            if ($window_size > $this->max_window_size)
+            {
+                $window_size = $this->max_window_size;
+            }
+            if ($window_size < $this->min_window_size)
+            {
+                $window_size = $this->min_window_size;
+            }
+        }
+        //debug_add("Got window size {$window_size}");
+        $offset = $iteration*$window_size;
+        if ($offset)
+        {
+            //debug_add("Setting offset to {$offset}");
+            $this->_qb->set_offset($offset);
+        }
+        //debug_add("Setting limit to {$window_size}");
+        $this->_qb->set_limit($window_size);
+        //debug_pop();
+    }
+
+    function execute()
+    {
+        return $this->execute_windowed();
+    }
+
+    /**
+     * This function will execute the Querybuilder and call the appropriate callbacks from the
+     * class it is associated to. This way, class authors have full control over what is actually
+     * returned to the application.
+     *
+     * The calling sequence of all event handlers of the associated class is like this:
+     *
+     * 1. bool _on_prepare_exec_query_builder(&$this) is called before the actual query execution. Return false to
+     *    abort the operation.
+     * 2. The query is executed.
+     * 3. void _on_process_query_result(&$result) is called after the successful execution of the query. You
+     *    may remove any unwanted entries from the resultset at this point.
+     *
+     * If the execution of the query fails for some reason all available error information is logged
+     * and a MIDCOM_ERRCRIT level error is triggered, halting execution.
+     *
+     * @param midgard_query_builder $qb An instance of the Query builder obtained by the new_query_builder
+     *     function of this class.
+     * @return Array The result of the query builder or null on any error. Note, that empty resultsets
+     *     will return an empty array.
+     * @todo Implement proper count / Limit support.
+     */
+    function execute_notwindowed()
+    {
+        debug_push_class(__CLASS__, __FUNCTION__);
+
+        if (! call_user_func_array(array($this->_real_class, '_on_prepare_exec_query_builder'), array(&$this)))
+        {
+            debug_add('The _on_prepare_exec_query_builder callback returned false, so we abort now.');
+            debug_pop();
+            return null;
+        }
+
+        if ($this->_constraint_count == 0)
+        {
-            debug_add('This Query Builder instance has no constraints.', MIDCOM_LOG_WARN);
-            debug_print_function_stack('We were called from here:');
-        }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        }
-
@@ -454 +732 @@
-        }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        }
-
@@ -560 +833 @@
-            return false;
-        }
+
-        $this->_constraint_count++;
-
@@ -612 +886 @@
-        {
-            debug_push_class(__CLASS__, __FUNCTION__);
-: Unknown or invalid column '{$field}'."
+ for column '{$field}', midgard error: " . mgd_errstr()
-            debug_pop();
-        }
@@ -652 +926 @@
-     * @param int $count The maximum number of records in the resultset.
-     */
-coun
-
-coun
+limi
+
+limi
-    }
-
@@ -731 +1005 @@
-    function count_unchecked()
-    {
+        // TODO: Handle limit and offset
-        return $this->_qb->count();
-    }