<?php /* Question2Answer by Gideon Greenspan and contributors http://www.question2answer.org/ File: qa-include/qa-app-recalc.php Description: Managing database recalculations (clean-up operations) and status messages This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. More about this license: http://www.question2answer.org/license.php */ /* A full list of redundant (non-normal) information in the database that can be recalculated: Recalculated in doreindexcontent: ================================ ^titlewords (all): index of words in titles of posts ^contentwords (all): index of words in content of posts ^tagwords (all): index of words in tags of posts (a tag can contain multiple words) ^posttags (all): index tags of posts ^words (all): list of words used for indexes ^options (title=cache_*): cached values for various things (e.g. counting questions) Recalculated in dorecountposts: ============================== ^posts (upvotes, downvotes, netvotes, hotness, acount, amaxvotes, flagcount): number of votes, hotness, answers, answer votes, flags Recalculated in dorecalcpoints: =============================== ^userpoints (all except bonus): points calculation for all users ^options (title=cache_userpointscount): Recalculated in dorecalccategories: =================================== ^posts (categoryid): assign to answers and comments based on their antecedent question ^posts (catidpath1, catidpath2, catidpath3): hierarchical path to category ids (requires QA_CATEGORY_DEPTH=4) ^categories (qcount): number of (visible) questions in each category ^categories (backpath): full (backwards) path of slugs to that category Recalculated in dorebuildupdates: ================================= ^sharedevents (all): per-entity event streams (see big comment in qa-db-favorites.php) ^userevents (all): per-subscriber event streams [but these are not entirely redundant since they can contain historical information no longer in ^posts] */ if (!defined('QA_VERSION')) { // don't allow this page to be requested directly from browser header('Location: ../'); exit; } require_once QA_INCLUDE_DIR.'db/recalc.php'; require_once QA_INCLUDE_DIR.'db/post-create.php'; require_once QA_INCLUDE_DIR.'db/points.php'; require_once QA_INCLUDE_DIR.'db/selects.php'; require_once QA_INCLUDE_DIR.'db/admin.php'; require_once QA_INCLUDE_DIR.'db/users.php'; require_once QA_INCLUDE_DIR.'app/options.php'; require_once QA_INCLUDE_DIR.'app/post-create.php'; require_once QA_INCLUDE_DIR.'app/post-update.php'; function qa_recalc_perform_step(&$state) /* Advance the recalculation operation represented by $state by a single step. $state can also be the name of a recalculation operation on its own. */ { $continue=false; @list($operation, $length, $next, $done)=explode("\t", $state); switch ($operation) { case 'doreindexcontent': qa_recalc_transition($state, 'doreindexcontent_pagereindex'); break; case 'doreindexcontent_pagereindex': $pages=qa_db_pages_get_for_reindexing($next, 10); if (count($pages)) { require_once QA_INCLUDE_DIR.'app/format.php'; $lastpageid=max(array_keys($pages)); foreach ($pages as $pageid => $page) if (!($page['flags'] & QA_PAGE_FLAGS_EXTERNAL)) { $searchmodules=qa_load_modules_with('search', 'unindex_page'); foreach ($searchmodules as $searchmodule) $searchmodule->unindex_page($pageid); $searchmodules=qa_load_modules_with('search', 'index_page'); if (count($searchmodules)) { $indextext=qa_viewer_text($page['content'], 'html'); foreach ($searchmodules as $searchmodule) $searchmodule->index_page($pageid, $page['tags'], $page['heading'], $page['content'], 'html', $indextext); } } $next=1+$lastpageid; $done+=count($pages); $continue=true; } else qa_recalc_transition($state, 'doreindexcontent_postcount'); break; case 'doreindexcontent_postcount': qa_db_qcount_update(); qa_db_acount_update(); qa_db_ccount_update(); qa_recalc_transition($state, 'doreindexcontent_postreindex'); break; case 'doreindexcontent_postreindex': $posts=qa_db_posts_get_for_reindexing($next, 10); if (count($posts)) { require_once QA_INCLUDE_DIR.'app/format.php'; $lastpostid=max(array_keys($posts)); qa_db_prepare_for_reindexing($next, $lastpostid); qa_suspend_update_counts(); foreach ($posts as $postid => $post) { qa_post_unindex($postid); qa_post_index($postid, $post['type'], $post['questionid'], $post['parentid'], $post['title'], $post['content'], $post['format'], qa_viewer_text($post['content'], $post['format']), $post['tags'], $post['categoryid']); } $next=1+$lastpostid; $done+=count($posts); $continue=true; } else { qa_db_truncate_indexes($next); qa_recalc_transition($state, 'doreindexposts_wordcount'); } break; case 'doreindexposts_wordcount': $wordids=qa_db_words_prepare_for_recounting($next, 1000); if (count($wordids)) { $lastwordid=max($wordids); qa_db_words_recount($next, $lastwordid); $next=1+$lastwordid; $done+=count($wordids); $continue=true; } else { qa_db_tagcount_update(); // this is quick so just do it here qa_recalc_transition($state, 'doreindexposts_complete'); } break; case 'dorecountposts': qa_recalc_transition($state, 'dorecountposts_postcount'); break; case 'dorecountposts_postcount': qa_db_qcount_update(); qa_db_acount_update(); qa_db_ccount_update(); qa_db_unaqcount_update(); qa_db_unselqcount_update(); qa_recalc_transition($state, 'dorecountposts_votecount'); break; case 'dorecountposts_votecount': $postids=qa_db_posts_get_for_recounting($next, 1000); if (count($postids)) { $lastpostid=max($postids); qa_db_posts_votes_recount($next, $lastpostid); $next=1+$lastpostid; $done+=count($postids); $continue=true; } else qa_recalc_transition($state, 'dorecountposts_acount'); break; case 'dorecountposts_acount': $postids=qa_db_posts_get_for_recounting($next, 1000); if (count($postids)) { $lastpostid=max($postids); qa_db_posts_answers_recount($next, $lastpostid); $next=1+$lastpostid; $done+=count($postids); $continue=true; } else { qa_db_unupaqcount_update(); qa_recalc_transition($state, 'dorecountposts_complete'); } break; case 'dorecalcpoints': qa_recalc_transition($state, 'dorecalcpoints_usercount'); break; case 'dorecalcpoints_usercount': qa_db_userpointscount_update(); // for progress update - not necessarily accurate qa_db_uapprovecount_update(); // needs to be somewhere and this is the most appropriate place qa_recalc_transition($state, 'dorecalcpoints_recalc'); break; case 'dorecalcpoints_recalc': $recalccount=10; $userids=qa_db_users_get_for_recalc_points($next, $recalccount+1); // get one extra so we know where to start from next $gotcount=count($userids); $recalccount=min($recalccount, $gotcount); // can't recalc more than we got if ($recalccount>0) { $lastuserid=$userids[$recalccount-1]; qa_db_users_recalc_points($next, $lastuserid); $done+=$recalccount; } else $lastuserid=$next; // for truncation if ($gotcount>$recalccount) { // more left to do $next=$userids[$recalccount]; // start next round at first one not recalculated $continue=true; } else { qa_db_truncate_userpoints($lastuserid); qa_db_userpointscount_update(); // quick so just do it here qa_recalc_transition($state, 'dorecalcpoints_complete'); } break; case 'dorefillevents': qa_recalc_transition($state, 'dorefillevents_qcount'); break; case 'dorefillevents_qcount': qa_db_qcount_update(); qa_recalc_transition($state, 'dorefillevents_refill'); break; case 'dorefillevents_refill': $questionids=qa_db_qs_get_for_event_refilling($next, 1); if (count($questionids)) { require_once QA_INCLUDE_DIR.'app/events.php'; require_once QA_INCLUDE_DIR.'app/updates.php'; require_once QA_INCLUDE_DIR.'util/sort.php'; $lastquestionid=max($questionids); foreach ($questionids as $questionid) { // Retrieve all posts relating to this question list($question, $childposts, $achildposts)=qa_db_select_with_pending( qa_db_full_post_selectspec(null, $questionid), qa_db_full_child_posts_selectspec(null, $questionid), qa_db_full_a_child_posts_selectspec(null, $questionid) ); // Merge all posts while preserving keys as postids $posts=array($questionid => $question); foreach ($childposts as $postid => $post) $posts[$postid]=$post; foreach ($achildposts as $postid => $post) $posts[$postid]=$post; // Creation and editing of each post foreach ($posts as $postid => $post) { $followonq=($post['basetype']=='Q') && ($postid!=$questionid); if ($followonq) $updatetype=QA_UPDATE_FOLLOWS; elseif ( ($post['basetype']=='C') && (@$posts[$post['parentid']]['basetype']=='Q') ) $updatetype=QA_UPDATE_C_FOR_Q; elseif ( ($post['basetype']=='C') && (@$posts[$post['parentid']]['basetype']=='A') ) $updatetype=QA_UPDATE_C_FOR_A; else $updatetype=null; qa_create_event_for_q_user($questionid, $postid, $updatetype, $post['userid'], @$posts[$post['parentid']]['userid'], $post['created']); if (isset($post['updated']) && !$followonq) qa_create_event_for_q_user($questionid, $postid, $post['updatetype'], $post['lastuserid'], $post['userid'], $post['updated']); } // Tags and categories of question qa_create_event_for_tags($question['tags'], $questionid, null, $question['userid'], $question['created']); qa_create_event_for_category($question['categoryid'], $questionid, null, $question['userid'], $question['created']); // Collect comment threads $parentidcomments=array(); foreach ($posts as $postid => $post) if ($post['basetype']=='C') $parentidcomments[$post['parentid']][$postid]=$post; // For each comment thread, notify all previous comment authors of each comment in the thread (could get slow) foreach ($parentidcomments as $parentid => $comments) { $keyuserids=array(); qa_sort_by($comments, 'created'); foreach ($comments as $comment) { foreach ($keyuserids as $keyuserid => $dummy) if ( ($keyuserid != $comment['userid']) && ($keyuserid != @$posts[$parentid]['userid']) ) qa_db_event_create_not_entity($keyuserid, $questionid, $comment['postid'], QA_UPDATE_FOLLOWS, $comment['userid'], $comment['created']); if (isset($comment['userid'])) $keyuserids[$comment['userid']]=true; } } } $next=1+$lastquestionid; $done+=count($questionids); $continue=true; } else qa_recalc_transition($state, 'dorefillevents_complete'); break; case 'dorecalccategories': qa_recalc_transition($state, 'dorecalccategories_postcount'); break; case 'dorecalccategories_postcount': qa_db_acount_update(); qa_db_ccount_update(); qa_recalc_transition($state, 'dorecalccategories_postupdate'); break; case 'dorecalccategories_postupdate': $postids=qa_db_posts_get_for_recategorizing($next, 100); if (count($postids)) { $lastpostid=max($postids); qa_db_posts_recalc_categoryid($next, $lastpostid); qa_db_posts_calc_category_path($next, $lastpostid); $next=1+$lastpostid; $done+=count($postids); $continue=true; } else { qa_recalc_transition($state, 'dorecalccategories_recount'); } break; case 'dorecalccategories_recount': $categoryids=qa_db_categories_get_for_recalcs($next, 10); if (count($categoryids)) { $lastcategoryid=max($categoryids); foreach ($categoryids as $categoryid) qa_db_ifcategory_qcount_update($categoryid); $next=1+$lastcategoryid; $done+=count($categoryids); $continue=true; } else { qa_recalc_transition($state, 'dorecalccategories_backpaths'); } break; case 'dorecalccategories_backpaths': $categoryids=qa_db_categories_get_for_recalcs($next, 10); if (count($categoryids)) { $lastcategoryid=max($categoryids); qa_db_categories_recalc_backpaths($next, $lastcategoryid); $next=1+$lastcategoryid; $done+=count($categoryids); $continue=true; } else { qa_recalc_transition($state, 'dorecalccategories_complete'); } break; case 'dodeletehidden': qa_recalc_transition($state, 'dodeletehidden_comments'); break; case 'dodeletehidden_comments': $posts=qa_db_posts_get_for_deleting('C', $next, 1); if (count($posts)) { require_once QA_INCLUDE_DIR.'app/posts.php'; $postid=$posts[0]; qa_post_delete($postid); $next=1+$postid; $done++; $continue=true; } else qa_recalc_transition($state, 'dodeletehidden_answers'); break; case 'dodeletehidden_answers': $posts=qa_db_posts_get_for_deleting('A', $next, 1); if (count($posts)) { require_once QA_INCLUDE_DIR.'app/posts.php'; $postid=$posts[0]; qa_post_delete($postid); $next=1+$postid; $done++; $continue=true; } else qa_recalc_transition($state, 'dodeletehidden_questions'); break; case 'dodeletehidden_questions': $posts=qa_db_posts_get_for_deleting('Q', $next, 1); if (count($posts)) { require_once QA_INCLUDE_DIR.'app/posts.php'; $postid=$posts[0]; qa_post_delete($postid); $next=1+$postid; $done++; $continue=true; } else qa_recalc_transition($state, 'dodeletehidden_complete'); break; case 'doblobstodisk': qa_recalc_transition($state, 'doblobstodisk_move'); break; case 'doblobstodisk_move': $blob=qa_db_get_next_blob_in_db($next); if (isset($blob)) { require_once QA_INCLUDE_DIR.'app/blobs.php'; require_once QA_INCLUDE_DIR.'db/blobs.php'; if (qa_write_blob_file($blob['blobid'], $blob['content'], $blob['format'])) qa_db_blob_set_content($blob['blobid'], null); $next=1+$blob['blobid']; $done++; $continue=true; } else qa_recalc_transition($state, 'doblobstodisk_complete'); break; case 'doblobstodb': qa_recalc_transition($state, 'doblobstodb_move'); break; case 'doblobstodb_move': $blob=qa_db_get_next_blob_on_disk($next); if (isset($blob)) { require_once QA_INCLUDE_DIR.'app/blobs.php'; require_once QA_INCLUDE_DIR.'db/blobs.php'; $content=qa_read_blob_file($blob['blobid'], $blob['format']); qa_db_blob_set_content($blob['blobid'], $content); qa_delete_blob_file($blob['blobid'], $blob['format']); $next=1+$blob['blobid']; $done++; $continue=true; } else qa_recalc_transition($state, 'doblobstodb_complete'); break; default: $state=''; break; } if ($continue) $state=$operation."\t".$length."\t".$next."\t".$done; return $continue && ($done<$length); } function qa_recalc_transition(&$state, $operation) /* Change the $state to represent the beginning of a new $operation */ { $length=qa_recalc_stage_length($operation); $next=(QA_FINAL_EXTERNAL_USERS && ($operation=='dorecalcpoints_recalc')) ? '' : 0; $done=0; $state=$operation."\t".$length."\t".$next."\t".$done; } function qa_recalc_stage_length($operation) /* Return how many steps there will be in recalculation $operation */ { switch ($operation) { case 'doreindexcontent_pagereindex': $length=qa_db_count_pages(); break; case 'doreindexcontent_postreindex': $length=qa_opt('cache_qcount')+qa_opt('cache_acount')+qa_opt('cache_ccount'); break; case 'doreindexposts_wordcount': $length=qa_db_count_words(); break; case 'dorecalcpoints_recalc': $length=qa_opt('cache_userpointscount'); break; case 'dorecountposts_votecount': case 'dorecountposts_acount': case 'dorecalccategories_postupdate': $length=qa_db_count_posts(); break; case 'dorefillevents_refill': $length=qa_opt('cache_qcount')+qa_db_count_posts('Q_HIDDEN'); break; case 'dorecalccategories_recount': case 'dorecalccategories_backpaths': $length=qa_db_count_categories(); break; case 'dodeletehidden_comments': $length=count(qa_db_posts_get_for_deleting('C')); break; case 'dodeletehidden_answers': $length=count(qa_db_posts_get_for_deleting('A')); break; case 'dodeletehidden_questions': $length=count(qa_db_posts_get_for_deleting('Q')); break; case 'doblobstodisk_move': $length=qa_db_count_blobs_in_db(); break; case 'doblobstodb_move': $length=qa_db_count_blobs_on_disk(); break; default: $length=0; break; } return $length; } /** * Return the translated language ID string replacing the progress and total in it. * @access private * @param string $langId Language string ID that contains 2 placeholders (^1 and ^2) * @param int $progress Amount of processed elements * @param int $total Total amount of elements * * @return string Returns the language string ID with their placeholders replaced with * the formatted progress and total numbers */ function qa_recalc_progress_lang($langId, $progress, $total) { return strtr(qa_lang($langId), array( '^1' => qa_format_number($progress), '^2' => qa_format_number($total) )); } function qa_recalc_get_message($state) /* Return a string which gives a user-viewable version of $state */ { require_once QA_INCLUDE_DIR . 'app/format.php'; @list($operation, $length, $next, $done) = explode("\t", $state); $done = (int) $done; $length = (int) $length; switch ($operation) { case 'doreindexcontent_postcount': case 'dorecountposts_postcount': case 'dorecalccategories_postcount': case 'dorefillevents_qcount': $message = qa_lang('admin/recalc_posts_count'); break; case 'doreindexcontent_pagereindex': $message = qa_recalc_progress_lang('admin/reindex_pages_reindexed', $done, $length); break; case 'doreindexcontent_postreindex': $message = qa_recalc_progress_lang('admin/reindex_posts_reindexed', $done, $length); break; case 'doreindexposts_complete': $message = qa_lang('admin/reindex_posts_complete'); break; case 'doreindexposts_wordcount': $message = qa_recalc_progress_lang('admin/reindex_posts_wordcounted', $done, $length); break; case 'dorecountposts_votecount': $message = qa_recalc_progress_lang('admin/recount_posts_votes_recounted', $done, $length); break; case 'dorecountposts_acount': $message = qa_recalc_progress_lang('admin/recount_posts_as_recounted', $done, $length); break; case 'dorecountposts_complete': $message = qa_lang('admin/recount_posts_complete'); break; case 'dorecalcpoints_usercount': $message = qa_lang('admin/recalc_points_usercount'); break; case 'dorecalcpoints_recalc': $message = qa_recalc_progress_lang('admin/recalc_points_recalced', $done, $length); break; case 'dorecalcpoints_complete': $message = qa_lang('admin/recalc_points_complete'); break; case 'dorefillevents_refill': $message = qa_recalc_progress_lang('admin/refill_events_refilled', $done, $length); break; case 'dorefillevents_complete': $message = qa_lang('admin/refill_events_complete'); break; case 'dorecalccategories_postupdate': $message = qa_recalc_progress_lang('admin/recalc_categories_updated', $done, $length); break; case 'dorecalccategories_recount': $message = qa_recalc_progress_lang('admin/recalc_categories_recounting', $done, $length); break; case 'dorecalccategories_backpaths': $message = qa_recalc_progress_lang('admin/recalc_categories_backpaths', $done, $length); break; case 'dorecalccategories_complete': $message = qa_lang('admin/recalc_categories_complete'); break; case 'dodeletehidden_comments': $message = qa_recalc_progress_lang('admin/hidden_comments_deleted', $done, $length); break; case 'dodeletehidden_answers': $message = qa_recalc_progress_lang('admin/hidden_answers_deleted', $done, $length); break; case 'dodeletehidden_questions': $message = qa_recalc_progress_lang('admin/hidden_questions_deleted', $done, $length); break; case 'dodeletehidden_complete': $message = qa_lang('admin/delete_hidden_complete'); break; case 'doblobstodisk_move': case 'doblobstodb_move': $message = qa_recalc_progress_lang('admin/blobs_move_moved', $done, $length); break; case 'doblobstodisk_complete': case 'doblobstodb_complete': $message = qa_lang('admin/blobs_move_complete'); break; default: $message = ''; break; } return $message; } /* Omit PHP closing tag to help avoid accidental output */