From d3ff509f609b1055e7db8bfd4930f15172ce1d98 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 14 Nov 2025 11:39:53 +0530 Subject: [PATCH 01/12] MDEV-28730 Remove internal parser usage from InnoDB fts Introduce QueryExecutor to perform direct InnoDB record scans with a callback interface and consistent-read handling. Also handles basic DML operation on clustered index of the table Newly Added file row0query.h & row0query.cc QueryExecutor class the following apis read(): iterate clustered index with RecordCallback read_by_index(): scan secondary index and fetch clustered row lookup_clustered_record(): resolve PK from secondary rec process_record_with_mvcc(): build version via read view and skip deletes insert_record(): Insert tuple into table's clustered index select_for_update(): Lock the record which matches with search_tuple update_record(): Update the currently selected and X-locked clustered record. delete_record(): Delete the clustered record identified by tuple delete_all(): Delete all clustered records in the table replace_record(): Tries update via select_for_update() + update_record(); if not found, runs insert_record. --- storage/innobase/CMakeLists.txt | 2 + storage/innobase/include/btr0cur.h | 2 +- storage/innobase/include/row0query.h | 258 +++++++++++ storage/innobase/row/row0query.cc | 632 +++++++++++++++++++++++++++ 4 files changed, 893 insertions(+), 1 deletion(-) create mode 100644 storage/innobase/include/row0query.h create mode 100644 storage/innobase/row/row0query.cc diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 3204b66721b17..3cc93b3fe5a74 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -311,6 +311,7 @@ SET(INNOBASE_SOURCES include/row0mysql.h include/row0purge.h include/row0quiesce.h + include/row0query.h include/row0row.h include/row0row.inl include/row0sel.h @@ -399,6 +400,7 @@ SET(INNOBASE_SOURCES row/row0undo.cc row/row0upd.cc row/row0quiesce.cc + row/row0query.cc row/row0vers.cc srv/srv0mon.cc srv/srv0srv.cc diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 53f88cc8ca1f5..9387adb234f24 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -346,7 +346,7 @@ btr_cur_del_mark_set_clust_rec( que_thr_t* thr, /*!< in: query thread */ const dtuple_t* entry, /*!< in: dtuple for the deleting record */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((nonnull(1,2,3,4,5,7), warn_unused_result)); /*************************************************************//** Tries to compress a page of the tree if it seems useful. It is assumed that mtr holds an x-latch on the tree and on the cursor page. To avoid diff --git a/storage/innobase/include/row0query.h b/storage/innobase/include/row0query.h new file mode 100644 index 0000000000000..bb35a1921b764 --- /dev/null +++ b/storage/innobase/include/row0query.h @@ -0,0 +1,258 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/row0query.h +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#ifndef INNOBASE_ROW0QUERY_H +#define INNOBASE_ROW0QUERY_H + +#include "btr0pcur.h" +#include +#include "dict0types.h" +#include "data0types.h" +#include "db0err.h" +#include "lock0types.h" +#include "rem0rec.h" + +/** Comparator action for deciding how to treat a record */ +enum class RecordCompareAction +{ + /** Do not process this record, continue traversal */ + SKIP, + /** Process this record via process_record */ + PROCESS, + /** Stop traversal immediately */ + STOP +}; + +using RecordProcessor= std::function; + +using RecordComparator= std::function< + RecordCompareAction(const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*)>; + +/** Record processing callback interface using std::function. +Can be used by FTS, stats infrastructure, and other components +that need to process database records with custom logic. */ +class RecordCallback +{ +public: + /** Constructor with processor function and optional comparator + @param[in] processor Function to process each record + @param[in] comparator Optional function to filter records (default: accept all) */ + RecordCallback( + RecordProcessor processor, + RecordComparator comparator= [](const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*) + { return RecordCompareAction::PROCESS; }) + : process_record(std::move(processor)), + compare_record(std::move(comparator)) {} + + virtual ~RecordCallback() = default; + + /** Called for each matching record */ + RecordProcessor process_record; + + /** Comparison function for custom filtering */ + RecordComparator compare_record; +}; + +/** General-purpose MVCC-aware record traversal and basic +DML executor. Provides a thin abstraction over B-tree cursors for +reading and mutating records with consistent-read (MVCC) handling, +and callback API. +- Open and iterate clustered/secondary indexes with page cursors. +- Build consistent-read versions when needed via transaction +read views. +- Filter and process records using RecordCallback: + - compare_record: decide SKIP/PROCESS/STOP for each record + - process_record: handle visible records; return whether to continue +- Basic DML helpers (insert/delete/replace) and table locking. */ +class QueryExecutor +{ +private: + trx_t *m_trx; + que_thr_t *m_thr; + btr_pcur_t m_pcur; + mtr_t m_mtr; + mem_heap_t *m_heap; + bool m_mtr_active; + + /** Lookup clustered index record from secondary index record + @param table table containing the indexes + @param sec_index secondary index + @param clust_index clustered index + @param sec_rec secondary index record + @param callback callback to process the clustered record + @param match_count counter for processed records + @return true if should continue processing, false to break */ + bool lookup_clustered_record(dict_table_t *table, + dict_index_t *sec_index, + dict_index_t *clust_index, + const rec_t *sec_rec, + RecordCallback& callback, + ulint& match_count) noexcept; + + /** Process a record with MVCC visibility checking and + version building + @param table table containing the record + @param index index of the record + @param rec record to process + @param offsets record offsets + @param callback callback to process the record + @param mtr mini-transaction for version building + @param match_count counter for processed records + @return true if should continue processing, false to break */ + bool process_record_with_mvcc(dict_table_t *table, + dict_index_t *index, + const rec_t *rec, + rec_offs *offsets, + RecordCallback& callback, + mtr_t *mtr, + ulint& match_count) noexcept; +public: + QueryExecutor(trx_t *trx); + ~QueryExecutor(); + + /** Insert a record in clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t insert_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete a record from the clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete all records from the clustered index of the table + @param table table be be deleted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_all(dict_table_t *table) noexcept; + + /** Acquire and lock a single clustered record for update + Performs a keyed lookup on the clustered index, validates MVCC visibility, + and acquires an X lock on the matching record. + @param[in] table Table containing the record + @param[in] search_tuple Exact key for clustered index lookup + @param[in] callback Optional record callback + @return DB_SUCCESS on successful lock + DB_RECORD_NOT_FOUND if no visible matching record + DB_LOCK_WAIT if waiting was required + error code on failure */ + dberr_t select_for_update(dict_table_t *table, dtuple_t *search_tuple, + RecordCallback *callback= nullptr) noexcept; + + /** Update the currently selected clustered record within an active mtr. + Attempts in-place update; falls back to optimistic/pessimistic update if needed, + including external field storage when required. + select_for_update() has positioned and locked m_pcur on the target row. + @param[in] table target table + @param[in] update update descriptor (fields, new values) + @return DB_SUCCESS on success + DB_OVERFLOW/DB_UNDERFLOW during size-changing paths + error_code on failures */ + dberr_t update_record(dict_table_t *table, const upd_t *update) noexcept; + + /** Commit the current modification mtr. Commits any pending page + changes from an update flow and clears the active mtr state. + @returns DB_SUCCESS if an mtr was active and is now committed, + DB_ERROR if no active mtr */ + dberr_t commit_update() noexcept; + + /** Roll back the current modification mtr. + Rollback is implemented as an mtr commit without persisting + an update, because page-level undo is managed by the + higher-level InnoDB machinery (undo logs). + @return DB_SUCCESS if an mtr was active + DB_ERROR if no active mtr */ + dberr_t rollback_update() noexcept; + + /** Try to update a record by key or insert if not found. + Performs a SELECT ... FOR UPDATE using search_tuple; + if found, updates the row; otherwise inserts a new record. + Note: + On update path, commits or rolls back the active mtr as needed. + On insert path, no active mtr remains upon return + @param[in] table target table + @param[in] search_tuple key identifying the target row + @param[in] update update descriptor (applied when found) + @param[in] insert_tuple tuple to insert when not found + @return DB_SUCCESS on successful update or insert + @retval DB_LOCK_WAIT to be retried, + @return error code on failure */ + dberr_t replace_record(dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept; + + /** Iterate clustered index records and process via callback. + Handles full table scan and index scan for range/select queries + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each matching record. On PROCESS, invokes + callback.process_record() on an MVCC-visible version. + @param table table to read + @param tuple optional search key (range/point). nullptr => full scan + @param mode B-tree search mode (e.g., PAGE_CUR_GE) + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read(dict_table_t *table, dtuple_t *tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Read records via a secondary index and process corresponding + clustered rows. Performs a range or point scan on the given secondary index, + filters secondary records with callback.compare_record(), then looks up + the matching clustered record and invokes callback.process_record() + on a MVCC-visible version. + + @param table Table to read + @param sec_index Secondary index used for traversal + @param search_tuple search key or nullptr for full scan + @param mode Cursor search mode + @param callback RecordCallback with comparator+processor + @return DB_SUCCESS on success + DB_RECORD_NOT_FOUND if no matching record was processed + error code on failure */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, + dtuple_t *search_tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Acquire a table lock in the given mode for transaction. + @param table table to lock + @param mode lock mode + @return DB_SUCCESS, DB_LOCK_WAIT or error code */ + dberr_t lock_table(dict_table_t *table, lock_mode mode) noexcept; + + /** Handle a lock wait for the current transaction and thread context. + @param err the lock-related error to handle (e.g., DB_LOCK_WAIT) + @param table_lock true if the wait originated from table lock, else row lock + @return DB_SUCCESS if the wait completed successfully and lock was granted + @retval DB_LOCK_WAIT_TIMEOUT if timed out */ + dberr_t handle_wait(dberr_t err, bool table_lock) noexcept; + mem_heap_t *get_heap() const { return m_heap; } + trx_t *get_trx() const { return m_trx; } +}; + +#endif /* INNOBASE_ROW0QUERY_H */ diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc new file mode 100644 index 0000000000000..41ddbcdeb6b87 --- /dev/null +++ b/storage/innobase/row/row0query.cc @@ -0,0 +1,632 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file row/row0query.cc +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#include "row0query.h" +#include "pars0pars.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0vers.h" +#include "mem0mem.h" +#include "que0que.h" +#include "lock0lock.h" +#include "rem0rec.h" +#include "btr0pcur.h" +#include "btr0cur.h" + +QueryExecutor::QueryExecutor(trx_t *trx) + : m_trx(trx), m_mtr(trx), m_mtr_active(false) +{ + m_heap= mem_heap_create(256); + m_thr= pars_complete_graph_for_exec(nullptr, m_trx, m_heap, nullptr); + btr_pcur_init(&m_pcur); +} + +QueryExecutor::~QueryExecutor() +{ + btr_pcur_close(&m_pcur); + if (m_heap) mem_heap_free(m_heap); +} + +dberr_t QueryExecutor::insert_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t* index= dict_table_get_first_index(table); + return row_ins_clust_index_entry(index, tuple, m_thr, 0); +} + +dberr_t QueryExecutor::lock_table(dict_table_t *table, lock_mode mode) noexcept +{ + trx_start_if_not_started(m_trx, true); + return ::lock_table(table, nullptr, mode, m_thr); +} + +dberr_t QueryExecutor::handle_wait(dberr_t err, bool table_lock) noexcept +{ + m_trx->error_state= err; + if (table_lock) m_thr->lock_state= QUE_THR_LOCK_TABLE; + else m_thr->lock_state= QUE_THR_LOCK_ROW; + if (m_trx->lock.wait_thr) + { + dberr_t wait_err= lock_wait(m_thr); + if (wait_err == DB_LOCK_WAIT_TIMEOUT) err= wait_err; + if (wait_err == DB_SUCCESS) + { + m_thr->lock_state= QUE_THR_LOCK_NOLOCK; + return DB_SUCCESS; + } + } + return err; +} + +dberr_t QueryExecutor::delete_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t *index= dict_table_get_first_index(table); + btr_pcur_t pcur; + mtr_t mtr(m_trx); + ulint deleted_count= 0; + + mtr.start(); + mtr.set_named_space(table->space); + + pcur.btr_cur.page_cur.index= index; + dberr_t err= btr_pcur_open(tuple, PAGE_CUR_GE, BTR_MODIFY_LEAF, + &pcur, &mtr); + if (err != DB_SUCCESS) + { + mtr.commit(); + return err; + } + while (!btr_pcur_is_after_last_on_page(&pcur) && + !btr_pcur_is_after_last_in_tree(&pcur)) + { + rec_t* rec= btr_pcur_get_rec(&pcur); + if (!rec) break; + + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + { + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + continue; + } + + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + uint16_t matched_fields= 0; + int cmp= cmp_dtuple_rec_with_match(tuple, rec, index, + offsets, &matched_fields); + if (cmp != 0) break; + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&pcur), rec, index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + if (err == DB_LOCK_WAIT) + { + mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) return err; + mtr.start(); + continue; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + mtr.commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&pcur), + rec, index, offsets, m_thr, + nullptr, &mtr); + if (err != DB_SUCCESS) break; + deleted_count++; + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + } + mtr.commit(); + return (deleted_count > 0) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +dberr_t QueryExecutor::delete_all(dict_table_t *table) noexcept +{ + dict_index_t *index= dict_table_get_first_index(table); + btr_pcur_t pcur; + mtr_t mtr(m_trx); + mtr.start(); + mtr.set_named_space(table->space); + + dberr_t err= pcur.open_leaf(true, index, BTR_MODIFY_LEAF, &mtr); + if (err != DB_SUCCESS) + { + mtr.commit(); + return err; + } + + while (!btr_pcur_is_after_last_on_page(&pcur) && + !btr_pcur_is_after_last_in_tree(&pcur)) + { + rec_t* rec= btr_pcur_get_rec(&pcur); + if (!rec) break; + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + { + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + continue; + } + + if (rec_get_info_bits( + rec, dict_table_is_comp(table)) & REC_INFO_MIN_REC_FLAG) + { + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + continue; + } + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&pcur), rec, index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) return err; + mtr.start(); + continue; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + mtr.commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&pcur), + const_cast(rec), index, + offsets, m_thr, nullptr, &mtr); + if (err || !btr_pcur_move_to_next(&pcur, &mtr)) break; + } + + mtr.commit(); + return err; +} + +dberr_t QueryExecutor::select_for_update(dict_table_t *table, + dtuple_t *search_tuple, + RecordCallback *callback) noexcept +{ + dict_index_t *index= dict_table_get_first_index(table); + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + } + + m_mtr.start(); + m_mtr.set_named_space(table->space); + m_mtr_active= true; + + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + m_pcur.btr_cur.page_cur.index= index; + dberr_t err= btr_pcur_open(search_tuple, PAGE_CUR_GE, BTR_MODIFY_LEAF, + &m_pcur, &m_mtr); + if (err != DB_SUCCESS) + { + m_mtr.commit(); + m_mtr_active= false; + return err; + } + + if (btr_pcur_is_after_last_on_page(&m_pcur) || + btr_pcur_is_after_last_in_tree(&m_pcur)) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + rec_t* rec= btr_pcur_get_rec(&m_pcur); + if (!rec) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + if (m_trx && m_trx->read_view.is_open()) + { + trx_id_t rec_trx_id= row_get_rec_trx_id(rec, index, offsets); + if (rec_trx_id && !m_trx->read_view.changes_visible(rec_trx_id)) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + } + uint16_t matched_fields= 0; + int cmp= cmp_dtuple_rec_with_match(search_tuple, rec, index, + offsets, &matched_fields); + if (cmp != 0) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + m_mtr.commit(); + m_mtr_active= false; + err= handle_wait(err, false); + if (err != DB_SUCCESS) return err; + return DB_LOCK_WAIT; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + m_mtr.commit(); + m_mtr_active= false; + return err; + } + + if (callback) + { + RecordCompareAction action= + callback->compare_record(search_tuple, rec, index, offsets); + if (action == RecordCompareAction::PROCESS) + callback->process_record(rec, index, offsets); + m_mtr.commit(); + m_mtr_active= false; + if (action == RecordCompareAction::SKIP) + return DB_RECORD_NOT_FOUND; + } + return DB_SUCCESS; +} + +dberr_t QueryExecutor::update_record(dict_table_t *table, + const upd_t *update) noexcept +{ + if (!m_mtr_active) return DB_ERROR; + dict_index_t *index= dict_table_get_first_index(table); + rec_t *rec= btr_pcur_get_rec(&m_pcur); + if (!rec) return DB_RECORD_NOT_FOUND; + mtr_x_lock_index(index, &m_mtr); + rec_offs *offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + dberr_t err= DB_SUCCESS; + ulint cmpl_info= UPD_NODE_NO_ORD_CHANGE | UPD_NODE_NO_SIZE_CHANGE; + for (ulint i = 0; i < update->n_fields; i++) + { + const upd_field_t *upd_field= &update->fields[i]; + ulint field_no= upd_field->field_no; + if (field_no < rec_offs_n_fields(offsets)) + { + ulint old_len= rec_offs_nth_size(offsets, field_no); + ulint new_len= upd_field->new_val.len; + if (new_len != UNIV_SQL_NULL && new_len != old_len) + { + cmpl_info &= ~UPD_NODE_NO_SIZE_CHANGE; + err= DB_OVERFLOW; + break; + } + } + } + + if (cmpl_info & UPD_NODE_NO_SIZE_CHANGE) + err= btr_cur_update_in_place(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + offsets, const_cast(update), 0, + m_thr, m_trx->id, &m_mtr); + if (err == DB_OVERFLOW) + { + big_rec_t *big_rec= nullptr; + err= btr_cur_optimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &m_heap, + const_cast(update), + cmpl_info, m_thr, m_trx->id, &m_mtr); + + if (err == DB_OVERFLOW || err == DB_UNDERFLOW) + { + mem_heap_t* offsets_heap= nullptr; + err= btr_cur_pessimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &offsets_heap, m_heap, + &big_rec, const_cast(update), + cmpl_info, m_thr, m_trx->id, &m_mtr); + + if (err == DB_SUCCESS && big_rec) + { + err= btr_store_big_rec_extern_fields(&m_pcur, offsets, big_rec, &m_mtr, + BTR_STORE_UPDATE); + dtuple_big_rec_free(big_rec); + } + } + } + return err; +} + +dberr_t QueryExecutor::commit_update() noexcept +{ + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_SUCCESS; + } + return DB_ERROR; +} + +dberr_t QueryExecutor::rollback_update() noexcept +{ + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_SUCCESS; + } + return DB_ERROR; +} + +dberr_t QueryExecutor::replace_record( + dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept +{ +retry_again: + dberr_t err= select_for_update(table, search_tuple); + if (err == DB_SUCCESS) + { + err= update_record(table, update); + if (err == DB_SUCCESS) err= commit_update(); + else rollback_update(); + return err; + } + else if (err == DB_RECORD_NOT_FOUND) + { + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + } + err= insert_record(table, insert_tuple); + return err; + } + else if (err == DB_LOCK_WAIT) + goto retry_again; + return err; +} + +dberr_t QueryExecutor::read(dict_table_t *table, dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + dict_index_t *index= dict_table_get_first_index(table); + if (!index) return DB_ERROR; + + m_mtr.start(); + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + m_pcur.btr_cur.page_cur.index= index; + dberr_t err= DB_SUCCESS; + if (tuple) err= btr_pcur_open(tuple, mode, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + else + { + err= m_pcur.open_leaf(true, index, BTR_SEARCH_LEAF, &m_mtr); + if (err == DB_SUCCESS) btr_pcur_move_to_next(&m_pcur, &m_mtr); + } + if (err != DB_SUCCESS) + { + m_mtr.commit(); + return err; + } + ulint match_count= 0; + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + const rec_t *rec= btr_pcur_get_rec(&m_pcur); + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + RecordCompareAction action= callback.compare_record( + tuple, rec, index, offsets); + if (action == RecordCompareAction::PROCESS) + { + bool continue_processing= process_record_with_mvcc(table, index, rec, + offsets, callback, + &m_mtr, match_count); + if (!continue_processing) break; + } + else if (action == RecordCompareAction::STOP) + break; + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) break; + } + m_mtr.commit(); + return (match_count > 0 || !tuple) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +dberr_t QueryExecutor::read_by_index(dict_table_t *table, + dict_index_t *sec_index, + dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + ut_ad(sec_index); + ut_ad(sec_index->table == table); + ut_ad(!dict_index_is_clust(sec_index)); + + dict_index_t *clust_index= dict_table_get_first_index(table); + if (!clust_index) return DB_ERROR; + + m_mtr.start(); + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + m_pcur.btr_cur.page_cur.index= sec_index; + + dberr_t err= DB_SUCCESS; + if (search_tuple) + err= btr_pcur_open(search_tuple, mode, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + else + { + err= m_pcur.open_leaf(true, sec_index, BTR_SEARCH_LEAF, &m_mtr); + if (err == DB_SUCCESS) btr_pcur_move_to_next(&m_pcur, &m_mtr); + } + + if (err != DB_SUCCESS) + { + m_mtr.commit(); + return err; + } + + ulint match_count= 0; + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + const rec_t *sec_rec= btr_pcur_get_rec(&m_pcur); + rec_offs* sec_offsets= rec_get_offsets(sec_rec, sec_index, nullptr, + sec_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + /* Check if secondary record matches our search criteria */ + RecordCompareAction action= callback.compare_record(search_tuple, sec_rec, + sec_index, sec_offsets); + if (action == RecordCompareAction::PROCESS) + { + /* Lookup clustered record and process it */ + if (!lookup_clustered_record(table, sec_index, clust_index, + sec_rec, callback, match_count)) + break; + } + else if (action == RecordCompareAction::STOP) + break; + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) break; + } + m_mtr.commit(); + return (match_count > 0 || !search_tuple) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +bool QueryExecutor::lookup_clustered_record(dict_table_t *table, + dict_index_t *sec_index, + dict_index_t *clust_index, + const rec_t *sec_rec, + RecordCallback& callback, + ulint& match_count) noexcept +{ + /* Extract primary key from secondary index record */ + dtuple_t *clust_tuple= row_build_row_ref(ROW_COPY_DATA, sec_index, + sec_rec, m_heap); + if (!clust_tuple) return false; + /* Now lookup the complete row using clustered index */ + btr_pcur_t clust_pcur; + clust_pcur.btr_cur.page_cur.index= clust_index; + + mtr_t clust_mtr{m_trx}; + clust_mtr.start(); + bool continue_processing= true; + bool record_processed= false; + dberr_t clust_err= btr_pcur_open(clust_tuple, PAGE_CUR_LE, + BTR_SEARCH_LEAF, &clust_pcur, + &clust_mtr); + if (clust_err == DB_SUCCESS && btr_pcur_is_on_user_rec(&clust_pcur)) + { + const rec_t *clust_rec= btr_pcur_get_rec(&clust_pcur); + rec_offs* clust_offsets= rec_get_offsets(clust_rec, clust_index, + nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + /* Verify this is the exact record we want */ + if (!cmp_dtuple_rec(clust_tuple, clust_rec, clust_index, clust_offsets)) + { + ulint prev_match_count= match_count; + continue_processing= process_record_with_mvcc( + table, clust_index, clust_rec, clust_offsets, callback, &clust_mtr, + match_count); + record_processed= (match_count > prev_match_count); + } + } + clust_mtr.commit(); + return record_processed ? continue_processing : true; +} + +bool QueryExecutor::process_record_with_mvcc(dict_table_t *table, + dict_index_t *index, + const rec_t *rec, + rec_offs *offsets, + RecordCallback& callback, + mtr_t *mtr, + ulint& match_count) noexcept +{ + bool is_deleted= rec_get_deleted_flag(rec, dict_table_is_comp(table)); + rec_t* version_rec= const_cast(rec); + rec_offs* version_offsets= offsets; + mem_heap_t* version_heap= nullptr; + bool should_process_record= false; + bool result= true; + if (m_trx && m_trx->read_view.is_open()) + { + trx_id_t rec_trx_id= row_get_rec_trx_id(rec, index, offsets); + if (rec_trx_id && !m_trx->read_view.changes_visible(rec_trx_id)) + { + version_heap= mem_heap_create(1024); + dberr_t vers_err= row_vers_build_for_consistent_read( + rec, mtr, index, &offsets, &m_trx->read_view, &version_heap, + version_heap, &version_rec, nullptr); + if (vers_err == DB_SUCCESS && version_rec) + { + version_offsets= rec_get_offsets(version_rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &version_heap); + is_deleted= rec_get_deleted_flag(version_rec, dict_table_is_comp(table)); + should_process_record= !is_deleted; + } + } + else should_process_record= !is_deleted; + } + else should_process_record= !is_deleted && version_rec; + + if (should_process_record) + { + result= callback.process_record(version_rec, index, version_offsets); + match_count++; + } + + if (version_heap) mem_heap_free(version_heap); + return result; +} From 94d7fcd669d5edb5045be23ef217a2327e1b71ee Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 14 Nov 2025 16:34:21 +0530 Subject: [PATCH 02/12] MDEV-28730 Remove internal parser usage from InnoDB fts Add FTSQueryExecutor class as a thin abstraction over QueryExecutor. This class takes care of open, lock, read, insert, delete for all auxiliary tables INDEX_[1..6], common FTS tables (DELETED, DELETED_CACHE, BEING_DELETED, CONFIG..) FTSQueryExecutor Class which has the following function: Auxiliary table functions : insert_aux_record(), delete_aux_record(), read_aux(), read_aux_all() FTS common table functions : insert_common_record(), delete_common_record(), delete_all_common_records(), read_all_common() FTS CONFIG table functions : insert_config_record(), update_config_record(), delete_config_record(), read_config(), read_all_config(), read_config_with_lock() --- storage/innobase/CMakeLists.txt | 2 + storage/innobase/fts/fts0exec.cc | 568 ++++++++++++++++++++++++++++ storage/innobase/include/fts0exec.h | 206 ++++++++++ storage/innobase/include/ut0new.h | 1 + 4 files changed, 777 insertions(+) create mode 100644 storage/innobase/fts/fts0exec.cc create mode 100644 storage/innobase/include/fts0exec.h diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 3cc93b3fe5a74..a9852207ab359 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -171,6 +171,7 @@ SET(INNOBASE_SOURCES fts/fts0ast.cc fts/fts0blex.cc fts/fts0config.cc + fts/fts0exec.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc @@ -239,6 +240,7 @@ SET(INNOBASE_SOURCES include/fsp0types.h include/fts0ast.h include/fts0blex.h + include/fts0exec.h include/fts0fts.h include/fts0opt.h include/fts0pars.h diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc new file mode 100644 index 0000000000000..5ca6bb43cdac8 --- /dev/null +++ b/storage/innobase/fts/fts0exec.cc @@ -0,0 +1,568 @@ +/***************************************************************************** +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0exec.cc + +Created 2025/11/05 +*******************************************************/ + +#include "fts0exec.h" +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0vlc.h" +#include "fts0priv.h" +#include "btr0btr.h" +#include "btr0cur.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0sel.h" +#include "pars0pars.h" +#include "eval0eval.h" +#include "que0que.h" +#include "trx0trx.h" +#include "lock0lock.h" +#include "rem0cmp.h" +#include "ha_prototypes.h" + +/** Defined in fts0fts.cc */ +extern const char* fts_common_tables[]; + +/** Find common table index by name */ +uint8_t find_common_table(const char* tbl_name) +{ + for (uint8_t i= 0; fts_common_tables[i]; i++) + if (!strcmp(tbl_name, fts_common_tables[i])) return i; + return UINT8_MAX; +} + +FTSQueryExecutor::FTSQueryExecutor( + trx_t *trx, const dict_index_t *fts_index, const dict_table_t *fts_table, + bool dict_locked) : m_executor(new QueryExecutor(trx)), + m_dict_locked(dict_locked), m_fts_index(fts_index), + m_fts_table(fts_table) +{ + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX; i++) + m_aux_tables[i] = nullptr; + + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX - 1; i++) + m_common_tables[i] = nullptr; +} + +FTSQueryExecutor::~FTSQueryExecutor() +{ + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX; i++) + if (m_aux_tables[i]) m_aux_tables[i]->release(); + + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX - 1; i++) + if (m_common_tables[i]) m_common_tables[i]->release(); + delete m_executor; +} + +dberr_t FTSQueryExecutor::open_aux_table(uint8_t aux_index) noexcept +{ + if (m_aux_tables[aux_index]) return DB_SUCCESS; + fts_table_t fts_table; + FTS_INIT_INDEX_TABLE(&fts_table, nullptr, FTS_INDEX_TABLE, m_fts_index); + fts_table.suffix= fts_get_suffix(aux_index); + + char table_name[MAX_FULL_NAME_LEN]; + fts_get_table_name(&fts_table, table_name, m_dict_locked); + + m_aux_tables[aux_index]= dict_table_open_on_name( + table_name, m_dict_locked, DICT_ERR_IGNORE_TABLESPACE); + return m_aux_tables[aux_index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_common_table(const char *tbl_name) noexcept +{ + uint8_t index= find_common_table(tbl_name); + if (index == UINT8_MAX) return DB_ERROR; + if (m_common_tables[index]) return DB_SUCCESS; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, nullptr, FTS_COMMON_TABLE, m_fts_table); + fts_table.suffix= tbl_name; + char table_name[MAX_FULL_NAME_LEN]; + fts_get_table_name(&fts_table, table_name, m_dict_locked); + + m_common_tables[index]= dict_table_open_on_name( + table_name, m_dict_locked, DICT_ERR_IGNORE_TABLESPACE); + return m_common_tables[index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::lock_aux_tables(uint8_t aux_index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_aux_tables[aux_index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor->lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor->handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::lock_common_tables(uint8_t index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_common_tables[index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err = m_executor->lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor->handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::insert_aux_record( + uint8_t aux_index, const fts_aux_data_t* aux_data) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 7 || index->n_uniq != 2) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[7]; + doc_id_t first_doc_id, last_doc_id; + + dtuple_t tuple{0, 7, 2, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 7); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + /* Field 1: first_doc_id (INT) */ + field= dtuple_get_nth_field(&tuple, 1); + fts_write_doc_id(&first_doc_id, aux_data->first_doc_id); + dfield_set_data(field, &first_doc_id, sizeof(doc_id_t)); + + /* Field 2: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 3: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 4: last_doc_id (UNSIGNED INT) */ + field= dtuple_get_nth_field(&tuple, 4); + fts_write_doc_id(&last_doc_id, aux_data->last_doc_id); + dfield_set_data(field, &last_doc_id, sizeof(doc_id_t)); + + /* Field 5: doc_count (UINT32_T) */ + byte doc_count[4]; + mach_write_to_4(doc_count, aux_data->doc_count); + field= dtuple_get_nth_field(&tuple, 5); + dfield_set_data(field, doc_count, sizeof(doc_count)); + + /* Field 6: ilist (VARBINARY) */ + field= dtuple_get_nth_field(&tuple, 6); + dfield_set_data(field, aux_data->ilist, aux_data->ilist_len); + + return m_executor->insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_common_record( + const char *tbl_name, doc_id_t doc_id) noexcept +{ + dberr_t err= open_common_table(tbl_name); + if (err != DB_SUCCESS) return err; + uint8_t index_no= find_common_table(tbl_name); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 3 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[3]; + + dtuple_t tuple{0, 3, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 3); + /* Field 0: doc_id (INT) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + return m_executor->insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[4]; + + dtuple_t tuple{0, 4, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 4); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, value, strlen(value)); + + return m_executor->insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::update_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t search_fields[1]; + dfield_t insert_fields[4]; + + dtuple_t search_tuple{0, 1, 1, 0, search_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t *field= dtuple_get_nth_field(&search_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + dtuple_t insert_tuple{0, 4, 1, 0, insert_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&insert_tuple, index, 4); + + /* Field 0: key (CHAR(50)) */ + field= dtuple_get_nth_field(&insert_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&insert_tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&insert_tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&insert_tuple, 3); + dfield_set_data(field, value, strlen(value)); + + upd_field_t upd_field; + upd_field.field_no = 3; + upd_field.orig_len = 0; + upd_field.exp = nullptr; + dfield_set_data(&upd_field.new_val, value, strlen(value)); + dict_col_copy_type(dict_index_get_nth_col(index, 3), + dfield_get_type(&upd_field.new_val)); + + upd_t update; + update.heap = nullptr; + update.info_bits = 0; + update.old_vrow = nullptr; + update.n_fields = 1; + update.fields = &upd_field; + + return m_executor->replace_record(table, &search_tuple, &update, + &insert_tuple); +} + +dberr_t FTSQueryExecutor::delete_aux_record( + uint8_t aux_index, const fts_aux_data_t* aux_data) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (dict_table_get_next_index(index) != nullptr) + return DB_ERROR; + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + return m_executor->delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_common_record( + const char *table_name, doc_id_t doc_id) noexcept +{ + dberr_t err= open_common_table(table_name); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= find_common_table(table_name); + err= lock_common_tables(cached_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: doc_id */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + return m_executor->delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_all_common_records( + const char *table_name) noexcept +{ + dberr_t err= open_common_table(table_name); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= find_common_table(table_name); + err= lock_common_tables(cached_index, LOCK_X); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + return m_executor->delete_all(table); +} + +dberr_t FTSQueryExecutor::delete_config_record( + const char *key) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor->delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::read_all_config(RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); +} + +dberr_t FTSQueryExecutor::read_config(const char *key, + RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor->read(table, &tuple, PAGE_CUR_GE, callback); +} + +dberr_t FTSQueryExecutor::read_config_with_lock(const char *key, + RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor->select_for_update(table, &tuple, &callback); +} + +dberr_t FTSQueryExecutor::read_aux(uint8_t aux_index, + const char *word, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + + err= lock_aux_tables(aux_index, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, word, strlen(word)); + + return m_executor->read(table, &tuple, mode, callback); +} + +dberr_t FTSQueryExecutor::read_aux_all(uint8_t aux_index, RecordCallback& callback) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + + err= lock_aux_tables(aux_index, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); +} + +dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, + RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table(tbl_name); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table(tbl_name); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); +} diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h new file mode 100644 index 0000000000000..e373bc1512998 --- /dev/null +++ b/storage/innobase/include/fts0exec.h @@ -0,0 +1,206 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fts0exec.h +FTS Query Builder - Abstraction layer for FTS operations + +Created 2025/10/30 +*******************************************************/ + +#ifndef INNOBASE_FTS0QUERY_H +#define INNOBASE_FTS0QUERY_H + +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0opt.h" +#include "fts0ast.h" +#include + +/** Structure to represent FTS auxiliary table data for insertion */ +struct fts_aux_data_t +{ + /** + CREATE TABLE $FTS_PREFIX_INDEX_[1-6]( + word VARCHAR(FTS_MAX_WORD_LEN), + first_doc_id INT NOT NULL, + last_doc_id UNSIGNED NOT NULL, + doc_count UNSIGNED INT NOT NULL, + ilist VARBINARY NOT NULL, + UNIQUE CLUSTERED INDEX ON (word, first_doc_id)); + */ + const char *word; + ulint word_len; + doc_id_t first_doc_id; + doc_id_t last_doc_id; + uint32_t doc_count; + const byte *ilist; + ulint ilist_len; + + fts_aux_data_t(const char *w, ulint w_len) + : word(w), word_len(w_len) + { + first_doc_id= last_doc_id= doc_count= 0; + ilist= nullptr; + ilist_len= 0; + } + + fts_aux_data_t(const char* w, ulint w_len, doc_id_t first_id, + doc_id_t last_id, uint32_t d_count, const byte* il, + ulint il_len) + : word(w), word_len(w_len), first_doc_id(first_id), + last_doc_id(last_id), doc_count(d_count), ilist(il), + ilist_len(il_len) {} +}; + +/** Abstraction over QueryExecutor for FTS auxiliary/common tables. +Handles table open/lock and provides typed helpers to insert, +delete and read records in FTS INDEX_1..INDEX_6 +and common tables (DELETED, CONFIG, ...) */ +class FTSQueryExecutor +{ +private: + QueryExecutor *m_executor; + bool m_dict_locked; + const dict_index_t *m_fts_index; + const dict_table_t *m_fts_table; + dict_table_t *m_aux_tables[FTS_NUM_AUX_INDEX]; + dict_table_t *m_common_tables[FTS_NUM_AUX_INDEX - 1]; + + /** Table preparation methods */ + + /** Open FTS INDEX_[1..6] table for the given auxiliary index. + @return DB_SUCCESS or error code */ + dberr_t open_aux_table(uint8_t aux_index) noexcept; + + /** Open a common FTS table (DELETED, CONFIG, ...). + @return DB_SUCCESS or error code */ + dberr_t open_common_table(const char *tbl_name) noexcept; + + /** Table lock operation */ + + /** Acquire a lock on an opened INDEX_[1..6] table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_aux_tables(uint8_t aux_index, lock_mode mode) noexcept; + + /** Acquire a lock on an opened common FTS table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_common_tables(uint8_t index, lock_mode mode) noexcept; + +public: + /** Create executor bound to trx and FTS table/index. + @param trx transaction (not owned) + @param fts_index FTS index (for INDEX_[1..6] tables) or nullptr + @param fts_table FTS table (for common tables) or nullptr + @param dict_locked true if dict is already locked */ + FTSQueryExecutor(trx_t *trx, const dict_index_t *fts_index, + const dict_table_t *fts_table, bool dict_locked= false); + + /** Release any opened table handles and executor resources. */ + ~FTSQueryExecutor(); + + /** High level DML operation on FTS TABLE */ + + /** Insert a row into auxiliary INDEX_[1..6] table. + Expects (word, first_doc_id, trx_id, roll_ptr, last_doc_id, + doc_count, ilist). + @param aux_index auxiliary table index + @param aux_data data to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Insert a single doc_id into a common table (e.g. DELETED, ...) + @param tbl_name common table name + @param doc_id document id to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Insert a key/value into CONFIG table. + @param key key for the config table + @param value value for the key + @return DB_SUCCESS or error code */ + dberr_t insert_config_record(const char *key, const char *value) noexcept; + + /** Delete one word row from INDEX_[1..6] by (word). + @param aux_index auxiliary table index + @param aux_data auxiliary table record + @return DB_SUCCESS or error code */ + dberr_t delete_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Delete a single doc_id row from a common table by (doc_id). + @param tbl_name common table name + @param doc_id document id to be deleted + @return DB_SUCCESS or error code */ + dberr_t delete_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Delete all rows from a common table. + @return DB_SUCCESS or error code */ + dberr_t delete_all_common_records(const char *tbl_name) noexcept; + + /** Delete a key from CONFIG table by (key). + @return DB_SUCCESS or error code */ + dberr_t delete_config_record(const char *key) noexcept; + + /** Upsert a key/value in CONFIG table. + Replaces 'value' if key exists, inserts otherwise. + @return DB_SUCCESS or error code */ + dberr_t update_config_record(const char *key, const char *value) noexcept; + + /** Full scan of CONFIG table with callback. + @return DB_SUCCESS or error code */ + dberr_t read_all_config(RecordCallback& callback) noexcept; + + /** Read the key from the config table by key + and process the matching record with callback + @param key config key + @param callback callback to process the record + @return DB_SUCCESS or error code */ + dberr_t read_config(const char *key, RecordCallback& callback) noexcept; + + /** Select-for-update CONFIG row by 'key' + @return DB_SUCCESS or error code */ + dberr_t read_config_with_lock( + const char *key, RecordCallback& callback) noexcept; + + /** Read Auxiliary INDEX_[1..6] table rows at (or) after + 'word' with given cursor mode. Callback is invoked for each + row for comparing it with word and process it if there is a match + @return DB_SUCCESS or error code */ + dberr_t read_aux(uint8_t aux_index, const char *word, + page_cur_mode_t mode, RecordCallback& callback) noexcept; + + /** Read all INDEX_[1..6] rows + Callback is invoked for each row for comparing it with word + and process it if it is matching + @return DB_SUCCESS or error code */ + dberr_t read_aux_all(uint8_t aux_index, RecordCallback& callback) noexcept; + + /** Read all rows from given COMMON table + Callback is invoked for processing the record */ + dberr_t read_all_common(const char *tbl_name, + RecordCallback& callback) noexcept; + mem_heap_t* get_heap() const noexcept + { return m_executor->get_heap(); } +}; + +#endif /* INNOBASE_FTS0QUERY_H */ diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index 398dd0dcc9ecd..ae2a6ca871ae0 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -852,6 +852,7 @@ constexpr const char* const auto_event_names[] = "fts0ast", "fts0blex", "fts0config", + "fts0exec", "fts0file", "fts0fts", "fts0opt", From d4eb80d3838715f1ae8e28b88f41eb947e7bdd3d Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 14 Nov 2025 17:24:29 +0530 Subject: [PATCH 03/12] MDEV-28730 Remove internal parser usage from InnoDB fts Introduce CommonTableReader callback to collect doc_id_t from fulltext common tables (DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE). These table share the same schema strucutre. Simplified all function which uses DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE table. These functions uses executor.insert_common_record(), delete_common_record(), delete_all_common_records() instead of SQL or query graph. fts_table_fetch_doc_ids(): Changed the signature of the function to pass the table name instead of fts_table_t. --- storage/innobase/fts/fts0exec.cc | 17 ++ storage/innobase/fts/fts0fts.cc | 87 ++---- storage/innobase/fts/fts0opt.cc | 400 ++++++++-------------------- storage/innobase/fts/fts0que.cc | 4 +- storage/innobase/handler/i_s.cc | 4 +- storage/innobase/include/fts0exec.h | 14 + storage/innobase/include/fts0fts.h | 21 +- 7 files changed, 169 insertions(+), 378 deletions(-) diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc index 5ca6bb43cdac8..59d89c815a94e 100644 --- a/storage/innobase/fts/fts0exec.cc +++ b/storage/innobase/fts/fts0exec.cc @@ -566,3 +566,20 @@ dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, dict_table_t* table= m_common_tables[index_no]; return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); } + +CommonTableReader::CommonTableReader() : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, const rec_offs* offsets) -> bool { + ulint len; + const byte* id_data= rec_get_nth_field(rec, offsets, 0, &len); + if (id_data && len != UNIV_SQL_NULL && len == 8) + { + doc_id_t doc_id= mach_read_from_8(id_data); + doc_ids.push_back(doc_id); + } + return true; + }, + [](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) -> RecordCompareAction { + return RecordCompareAction::PROCESS; /* Process all records */ + } +) {} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index ad7df21504fdf..2fa3fa8c1c37f 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -30,6 +30,7 @@ Full Text Search interface #include "dict0stats_bg.h" #include "row0sel.h" #include "fts0fts.h" +#include "fts0exec.h" #include "fts0priv.h" #include "fts0types.h" #include "fts0types.inl" @@ -2837,13 +2838,10 @@ fts_delete( fts_trx_table_t*ftt, /*!< in: FTS trx table */ fts_trx_row_t* row) /*!< in: row */ { - que_t* graph; fts_table_t fts_table; - doc_id_t write_doc_id; dict_table_t* table = ftt->table; doc_id_t doc_id = row->doc_id; trx_t* trx = ftt->fts_trx->trx; - pars_info_t* info = pars_info_create(); fts_cache_t* cache = table->fts->cache; /* we do not index Documents whose Doc ID value is 0 */ @@ -2856,10 +2854,6 @@ fts_delete( FTS_INIT_FTS_TABLE(&fts_table, "DELETED", FTS_COMMON_TABLE, table); - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - /* It is possible we update a record that has not yet been sync-ed into cache from last crash (delete Doc will not initialize the sync). Avoid any added counter accounting until the FTS cache @@ -2884,20 +2878,9 @@ fts_delete( } /* Note the deleted document for OPTIMIZE to purge. */ - char table_name[MAX_FULL_NAME_LEN]; - trx->op_info = "adding doc id to FTS DELETED"; - - fts_table.suffix = "DELETED"; - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "deleted", table_name); - - graph = fts_parse_sql(&fts_table, info, - "BEGIN INSERT INTO $deleted VALUES (:doc_id);"); - - dberr_t error = fts_eval_sql(trx, graph); - que_graph_free(graph); + FTSQueryExecutor executor(trx, nullptr, table); + dberr_t error= executor.insert_common_record("DELETED", doc_id); /* Increment the total deleted count, this is used to calculate the number of documents indexed. */ @@ -3888,60 +3871,24 @@ void fts_doc_ids_sort(ib_vector_t *doc_ids) std::sort(data, data + doc_ids->used); } -/*********************************************************************//** -Add rows to the DELETED_CACHE table. +/** Add rows to the DELETED_CACHE table. @return DB_SUCCESS if all went well else error code*/ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_add_deleted_cache( -/*=======================*/ - fts_sync_t* sync, /*!< in: sync state */ - ib_vector_t* doc_ids) /*!< in: doc ids to add */ +fts_sync_add_deleted_cache(fts_sync_t *sync, ib_vector_t *doc_ids) { - ulint i; - pars_info_t* info; - que_t* graph; - fts_table_t fts_table; - char table_name[MAX_FULL_NAME_LEN]; - doc_id_t dummy = 0; - dberr_t error = DB_SUCCESS; - ulint n_elems = ib_vector_size(doc_ids); - - ut_a(ib_vector_size(doc_ids) > 0); - - fts_doc_ids_sort(doc_ids); - - info = pars_info_create(); - - fts_bind_doc_id(info, "doc_id", &dummy); - - FTS_INIT_FTS_TABLE( - &fts_table, "DELETED_CACHE", FTS_COMMON_TABLE, sync->table); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, - info, - "BEGIN INSERT INTO $table_name VALUES (:doc_id);"); - - for (i = 0; i < n_elems && error == DB_SUCCESS; ++i) { - doc_id_t* update; - doc_id_t write_doc_id; - - update = static_cast(ib_vector_get(doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - - error = fts_eval_sql(sync->trx, graph); - } - - que_graph_free(graph); - - return(error); + ulint n_elems= ib_vector_size(doc_ids); + ut_a(n_elems > 0); + fts_doc_ids_sort(doc_ids); + FTSQueryExecutor executor(sync->trx, nullptr, sync->table); + dberr_t error= DB_SUCCESS; + for (uint32_t i= 0; i < n_elems && error == DB_SUCCESS; ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(doc_ids, i)); + error= executor.insert_common_record("DELETED_CACHE", *update); + } + return error; } /** Write the words and ilist to disk. diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index af4b9eed2a491..554721171caeb 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -27,6 +27,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang ***********************************************************************/ #include "fts0fts.h" +#include "fts0exec.h" #include "row0sel.h" #include "que0types.h" #include "fts0priv.h" @@ -238,28 +239,6 @@ static ulint fts_optimize_time_limit; /** It's defined in fts0fts.cc */ extern const char* fts_common_tables[]; -/** SQL Statement for changing state of rows to be deleted from FTS Index. */ -static const char* fts_init_delete_sql = - "BEGIN\n" - "\n" - "INSERT INTO $BEING_DELETED\n" - "SELECT doc_id FROM $DELETED;\n" - "\n" - "INSERT INTO $BEING_DELETED_CACHE\n" - "SELECT doc_id FROM $DELETED_CACHE;\n"; - -static const char* fts_delete_doc_ids_sql = - "BEGIN\n" - "\n" - "DELETE FROM $DELETED WHERE doc_id = :doc_id1;\n" - "DELETE FROM $DELETED_CACHE WHERE doc_id = :doc_id2;\n"; - -static const char* fts_end_delete_sql = - "BEGIN\n" - "\n" - "DELETE FROM $BEING_DELETED;\n" - "DELETE FROM $BEING_DELETED_CACHE;\n"; - /**********************************************************************//** Initialize fts_zip_t. */ static @@ -921,109 +900,35 @@ fts_index_fetch_words( return(error); } -/**********************************************************************//** -Callback function to fetch the doc id from the record. -@return always returns TRUE */ -static -ibool -fts_fetch_doc_ids( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ +dberr_t fts_table_fetch_doc_ids(trx_t *trx, dict_table_t *table, + const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept { - que_node_t* exp; - int i = 0; - sel_node_t* sel_node = static_cast(row); - fts_doc_ids_t* fts_doc_ids = static_cast(user_arg); - doc_id_t* update = static_cast( - ib_vector_push(fts_doc_ids->doc_ids, NULL)); - - for (exp = sel_node->select_list; - exp; - exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - switch (i) { - case 0: /* DOC_ID */ - *update = fts_read_doc_id( - static_cast(data)); - break; - - default: - ut_error; - } - } - - return(TRUE); -} - -/**********************************************************************//** -Read the rows from a FTS common auxiliary table. -@return DB_SUCCESS or error code */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table */ - fts_doc_ids_t* doc_ids) /*!< in: For collecting doc ids */ -{ - dberr_t error; - que_t* graph; - pars_info_t* info = pars_info_create(); - ibool alloc_bk_trx = FALSE; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(fts_table->suffix != NULL); - ut_a(fts_table->type == FTS_COMMON_TABLE); - - if (!trx) { - trx = trx_create(); - alloc_bk_trx = TRUE; - } - - trx->op_info = "fetching FTS doc ids"; - - pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_id FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - que_graph_free(graph); - - if (error == DB_SUCCESS) { - fts_doc_ids_sort(doc_ids->doc_ids); - } - - if (alloc_bk_trx) { - trx->free(); - } + bool bk_trx= false; + if (!trx) + { + trx= trx_create(); + bk_trx= true; + } + trx->op_info = "fetching FTS doc ids"; + FTSQueryExecutor executor(trx, nullptr, table); + CommonTableReader reader; + dberr_t err= executor.read_all_common(tbl_name, reader); - return(error); + if (err == DB_SUCCESS) + { + const auto& doc_id_vector= reader.get_doc_ids(); + for (doc_id_t doc_id : doc_id_vector) + ib_vector_push(doc_ids->doc_ids, &doc_id); + fts_doc_ids_sort(doc_ids->doc_ids); + } + if (bk_trx) + { + if (err == DB_SUCCESS) fts_sql_commit(trx); + else fts_sql_rollback(trx); + trx->free(); + } + return err; } /**********************************************************************//** @@ -2022,190 +1927,97 @@ fts_optimize_index( return(error); } -/**********************************************************************//** -Delete the document ids in the delete, and delete cache tables. +/** Delete the document ids in the delete, and delete cache tables. +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_ids( -/*===============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_ids(fts_optimize_t *optim) { - ulint i; - pars_info_t* info; - que_t* graph; - doc_id_t* update; - doc_id_t write_doc_id; - dberr_t error = DB_SUCCESS; - char deleted[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); - - update = static_cast( - ib_vector_get(optim->to_delete->doc_ids, 0)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - /* This is required for the SQL parser to work. It must be able - to find the following variables. So we do it twice. */ - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - /* Make sure the following two names are consistent with the name - used in the fts_delete_doc_ids_sql */ - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - graph = fts_parse_sql(NULL, info, fts_delete_doc_ids_sql); - - /* Delete the doc ids that were copied at the start. */ - for (i = 0; i < ib_vector_size(optim->to_delete->doc_ids); ++i) { - - update = static_cast(ib_vector_get( - optim->to_delete->doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - error = fts_eval_sql(optim->trx, graph); - - // FIXME: Check whether delete actually succeeded! - if (error != DB_SUCCESS) { - - fts_sql_rollback(optim->trx); - break; - } - } - - que_graph_free(graph); + dberr_t error= DB_SUCCESS; + ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + for (ulint i= 0; + i < ib_vector_size(optim->to_delete->doc_ids) && error != DB_SUCCESS; + ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(optim->to_delete->doc_ids, i)); + error= executor.delete_common_record("DELETED", *update); + if (error == DB_SUCCESS) + error= executor.delete_common_record("DELTED_CACHED", *update); + } - return(error); + if (error != DB_SUCCESS) + fts_sql_rollback(optim->trx); + return error; } -/**********************************************************************//** -Delete the document ids in the pending delete, and delete tables. +/** Delete the document ids in the pending delete, and delete tables. +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_id_snapshot( -/*=======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_id_snapshot(fts_optimize_t *optim) { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following two names are consistent with the name - used in the fts_end_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - /* Delete the doc ids that were copied to delete pending state at - the start of optimize. */ - graph = fts_parse_sql(NULL, info, fts_end_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - que_graph_free(graph); - - return(error); + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + dberr_t error= executor.delete_all_common_records("BEING_DELETED"); + if (error == DB_SUCCESS) + error= executor.delete_all_common_records("BEING_DELETED_CACHE"); + return error; } -/**********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. +/** Copy the deleted doc ids that will be purged during this +optimize run to the being deleted FTS auxiliary tables. +The transaction is committed upon successfull copy and rolled +back on DB_DUPLICATE_KEY error. +@param optim optimize instance +@param n_rows number of rows exist in being_deleted table @return DB_SUCCESS if all OK */ static -ulint -fts_optimize_being_deleted_count( -/*=============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_being_deleted_count(fts_optimize_t *optim, + ulint *n_rows) { - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "BEING_DELETED", FTS_COMMON_TABLE, - optim->table); - - return(fts_get_rows_count(&fts_table)); + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + CommonTableReader reader; + dberr_t err= executor.read_all_common("BEING_DELETED", reader); + if (err == DB_SUCCESS) *n_rows= reader.get_doc_ids().size(); + return err; } -/*********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. -@return DB_SUCCESS if all OK */ +/** Create a snapshot of deleted document IDs by moving them from +DELETED to BEING_DELETED and from DELETED_CACHE to +BEING_DELETED_CACHE. +@param optim optimize fts instance +@return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_create_deleted_doc_id_snapshot( -/*========================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_create_deleted_doc_id_snapshot(fts_optimize_t *optim) { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following four names are consistent with the name - used in the fts_init_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - /* Move doc_ids that are to be deleted to state being deleted. */ - graph = fts_parse_sql(NULL, info, fts_init_delete_sql); + dberr_t err= DB_SUCCESS; + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + CommonTableReader reader; - error = fts_eval_sql(optim->trx, graph); + err= executor.read_all_common("DELETED", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - que_graph_free(graph); + const auto& deleted_doc_ids = reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_doc_ids) + { + err= executor.insert_common_record("BEING_DELETED", doc_id); + if (err != DB_SUCCESS) return err; + } - if (error != DB_SUCCESS) { - fts_sql_rollback(optim->trx); - } else { - fts_sql_commit(optim->trx); - } + reader.clear(); + err= executor.read_all_common("DELETED_CACHE", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->del_list_regenerated = TRUE; + const auto& deleted_cache_doc_ids= reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_cache_doc_ids) + { + err= executor.insert_common_record("BEING_DELETED_CACHE", doc_id); + if (err != DB_SUCCESS) return err; + } - return(error); + optim->del_list_regenerated= TRUE; + return err; } /*********************************************************************//** @@ -2218,29 +2030,24 @@ fts_optimize_read_deleted_doc_id_snapshot( /*======================================*/ fts_optimize_t* optim) /*!< in: optimize instance */ { - dberr_t error; - - optim->fts_common_table.suffix = "BEING_DELETED"; - /* Read the doc_ids to delete. */ - error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + dberr_t error = fts_table_fetch_doc_ids( + optim->trx, optim->table, "BEING_DELETED", + optim->to_delete); if (error == DB_SUCCESS) { optim->fts_common_table.suffix = "BEING_DELETED_CACHE"; - /* Read additional doc_ids to delete. */ error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + optim->trx, optim->table, "BEING_DELETED_CACHE", + optim->to_delete); } if (error != DB_SUCCESS) { - fts_doc_ids_free(optim->to_delete); optim->to_delete = NULL; } - return(error); } @@ -2452,7 +2259,10 @@ fts_optimize_table( // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot. /* Check whether there are still records in BEING_DELETED table */ - if (fts_optimize_being_deleted_count(optim) == 0) { + ulint n_rows = 0; + error= fts_optimize_being_deleted_count(optim, &n_rows); + + if (error == DB_SUCCESS && n_rows == 0) { /* Take a snapshot of the deleted document ids, they are copied to the BEING_ tables. */ error = fts_optimize_create_deleted_doc_id_snapshot(optim); diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 191678db78188..2dd21e99e24c6 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -4005,7 +4005,7 @@ fts_query( /* Read the deleted doc_ids, we need these for filtering. */ error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + nullptr, index->table, "DELETED", query.deleted); if (error != DB_SUCCESS) { goto func_exit; @@ -4014,7 +4014,7 @@ fts_query( query.fts_common_table.suffix = "DELETED_CACHE"; error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + nullptr, index->table, "DELETED_CACHE", query.deleted); if (error != DB_SUCCESS) { goto func_exit; diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 5d260c7f0b072..cb3c8fc4bdd3b 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -2239,7 +2239,9 @@ i_s_fts_deleted_generic_fill( (being_deleted) ? "BEING_DELETED" : "DELETED", FTS_COMMON_TABLE, user_table); - fts_table_fetch_doc_ids(trx, &fts_table, deleted); + fts_table_fetch_doc_ids( + nullptr, user_table, + being_deleted ? "BEING_DELETED" : "DELETED", deleted); dict_table_close(user_table, thd, mdl_ticket); diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h index e373bc1512998..ac4ff7a6f85c9 100644 --- a/storage/innobase/include/fts0exec.h +++ b/storage/innobase/include/fts0exec.h @@ -203,4 +203,18 @@ class FTSQueryExecutor { return m_executor->get_heap(); } }; +/** Callback class for reading common table records +(DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE) */ +class CommonTableReader : public RecordCallback +{ +private: + std::vector doc_ids; + +public: + CommonTableReader(); + + const std::vector& get_doc_ids() const { return doc_ids; } + void clear() { doc_ids.clear(); } +}; + #endif /* INNOBASE_FTS0QUERY_H */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 80b0ab1fd1abd..1145c5b55166a 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -824,16 +824,17 @@ fts_load_stopword( bool reload); /*!< in: Whether it is during reload of FTS table */ -/****************************************************************//** -Read the rows from the FTS index -@return DB_SUCCESS if OK */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_doc_ids_t* doc_ids); /*!< in: For collecting - doc ids */ + +/** Read the rows from the fulltext index +@param trx transaction +@param table Fulltext table +@param tbl_name table name +@param doc_ids collecting doc ids +@return DB_SUCCESS or error code */ +dberr_t fts_table_fetch_doc_ids(trx_t *trx, dict_table_t *table, + const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept; + /****************************************************************//** This function brings FTS index in sync when FTS index is first used. There are documents that have not yet sync-ed to auxiliary From 09f70f73122719014ade8d894a6c4fa72bc4e569 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Sun, 16 Nov 2025 19:17:29 +0530 Subject: [PATCH 04/12] MDEV-28730 Remove internal parser usage from InnoDB fts Introduce ConfigReader callback to extract key, value from fulltext config common table (CONFIG). This table has schema. Simplifield all function which uses CONFIG tale. These functions uses executor.insert_config_record(), update_config_record() instead of SQL or query graph. --- .../suite/innodb_fts/r/index_table.result | 2 +- .../suite/innodb_fts/t/index_table.test | 2 +- storage/innobase/fts/fts0config.cc | 319 ++++------------ storage/innobase/fts/fts0exec.cc | 39 +- storage/innobase/fts/fts0fts.cc | 348 ++++++------------ storage/innobase/fts/fts0opt.cc | 27 +- storage/innobase/handler/ha_innodb.cc | 1 + storage/innobase/handler/i_s.cc | 5 +- storage/innobase/include/fts0exec.h | 7 + storage/innobase/include/fts0priv.h | 30 +- storage/innobase/row/row0query.cc | 1 + 11 files changed, 262 insertions(+), 519 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/index_table.result b/mysql-test/suite/innodb_fts/r/index_table.result index 909a889db4230..78832669cdbca 100644 --- a/mysql-test/suite/innodb_fts/r/index_table.result +++ b/mysql-test/suite/innodb_fts/r/index_table.result @@ -5,7 +5,7 @@ id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, title VARCHAR(200), content TEXT ) ENGINE= InnoDB; -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); ERROR HY000: Got error 11 "Resource temporarily unavailable" from storage engine InnoDB CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/t/index_table.test b/mysql-test/suite/innodb_fts/t/index_table.test index 89c0905323083..cfe27b4226848 100644 --- a/mysql-test/suite/innodb_fts/t/index_table.test +++ b/mysql-test/suite/innodb_fts/t/index_table.test @@ -18,7 +18,7 @@ CREATE TABLE articles ( ) ENGINE= InnoDB; --error ER_GET_ERRNO -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 524f648676eb7..bdb9307a592c6 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -27,100 +27,37 @@ Created 2007/5/9 Sunny Bains #include "trx0roll.h" #include "row0sel.h" +#include "fts0exec.h" #include "fts0priv.h" -/******************************************************************//** -Callback function for fetching the config value. -@return always returns TRUE */ -static -ibool -fts_config_fetch_value( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to - ib_vector_t */ -{ - sel_node_t* node = static_cast(row); - fts_string_t* value = static_cast(user_arg); - - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - if (len != UNIV_SQL_NULL) { - ulint max_len = ut_min(value->f_len - 1, len); - - memcpy(value->f_str, data, max_len); - value->f_len = max_len; - value->f_str[value->f_len] = '\0'; - } - - return(TRUE); -} - -/******************************************************************//** -Get value from the config table. The caller must ensure that enough +/** Get value from the config table. The caller must ensure that enough space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ +dberr_t fts_config_get_value(trx_t *trx, const dict_table_t *table, + const char *name, fts_string_t *value) { - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - *value->f_str = '\0'; - ut_a(value->f_len > 0); - - pars_info_bind_function(info, "my_func", fts_config_fetch_value, - value); - - /* The len field of value must be set to the max bytes that - it can hold. On a successful read, the len field will be set - to the actual number of bytes copied to value. */ - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $table_name" - " WHERE key = :name;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - trx->op_info = "getting FTS config value"; - - error = fts_eval_sql(trx, graph); - que_graph_free(graph); - return(error); + trx->op_info = "getting FTS config value"; + FTSQueryExecutor executor(trx, nullptr, table); + ConfigReader reader; + dberr_t err= executor.read_config(name, reader); + if (err == DB_SUCCESS) + { + if (reader.config_pairs.size() != 1) err= DB_ERROR; + else + { + const std::string& config_value= reader.config_pairs[0].second; + ulint max_len= ut_min(value->f_len - 1, config_value.length()); + memcpy(value->f_str, config_value.c_str(), max_len); + value->f_len= max_len; + value->f_str[value->f_len]= '\0'; + } + } + else value->f_str[0]= '\0'; + return err; } /*********************************************************************//** @@ -164,98 +101,32 @@ fts_config_get_index_value( fts_string_t* value) /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_get_value(trx, &fts_table, name, value); + dberr_t error = fts_config_get_value(trx, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Set the value in the config table for name. +/** Set the value in the config table for name. +@param trx transaction +@param fts_table indexed fulltext table +@param name key for the config +@param value value of the key @return DB_SUCCESS or error code */ dberr_t -fts_config_set_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - const fts_string_t* - value) /*!< in: value to update */ +fts_config_set_value(trx_t *trx, const dict_table_t *table, + const char *name, const fts_string_t *value) { - pars_info_t* info; - que_t* graph; - dberr_t error; - undo_no_t undo_no; - undo_no_t n_rows_updated; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - pars_info_bind_varchar_literal(info, "value", - value->f_str, value->f_len); - - const bool dict_locked = fts_table->table->fts->dict_locked; - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN UPDATE $table_name SET value = :value" - " WHERE key = :name;"); - - trx->op_info = "setting FTS config value"; - - undo_no = trx->undo_no; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - n_rows_updated = trx->undo_no - undo_no; - - /* Check if we need to do an insert. */ - if (error == DB_SUCCESS && n_rows_updated == 0) { - info = pars_info_create(); - - pars_info_bind_varchar_literal( - info, "name", (byte*) name, name_len); - - pars_info_bind_varchar_literal( - info, "value", value->f_str, value->f_len); - - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN\n" - "INSERT INTO $table_name VALUES(:name, :value);"); - - trx->op_info = "inserting FTS config value"; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - } - - return(error); + trx->op_info= "setting FTS config value"; + FTSQueryExecutor executor(trx, nullptr, table, table->fts->dict_locked); + char value_str[FTS_MAX_CONFIG_VALUE_LEN + 1]; + memcpy(value_str, value->f_str, value->f_len); + value_str[value->f_len]= '\0'; + return executor.update_config_record(name, value_str); } /******************************************************************//** @@ -271,17 +142,10 @@ fts_config_set_index_value( fts_string_t* value) /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_set_value(trx, &fts_table, name, value); + dberr_t error = fts_config_set_value(trx, index->table, name, value); ut_free(name); @@ -358,71 +222,50 @@ fts_config_set_index_ulint( } #endif /* FTS_OPTIMIZE_DEBUG */ -/******************************************************************//** -Get an ulint value from the config table. +/** Get an ulint value from the config table. +@param trx transaction +@param table user table +@param name key value +@param int_value value of the key @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ +fts_config_get_ulint(trx_t *trx, const dict_table_t *table, + const char *name, ulint *int_value) { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - error = fts_config_get_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") reading `" << name << "'"; - } else { - *int_value = strtoul((char*) value.f_str, NULL, 10); - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + dberr_t error= fts_config_get_value(trx, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + ib::error() << "(" << error << ") reading `" << name << "'"; + else *int_value = strtoul((char*) value.f_str, NULL, 10); + ut_free(value.f_str); + return error; } -/******************************************************************//** -Set an ulint value in the config table. +/** Set an ulint value in the config table. +@param trx transaction +@param table user table +@param name name of the key +@param int_value value of the key to be set @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_set_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ +fts_config_set_ulint(trx_t *trx, const dict_table_t *table, + const char *name, ulint int_value) { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); - - value.f_len = (ulint) snprintf( - (char*) value.f_str, FTS_MAX_INT_LEN, ULINTPF, int_value); - - error = fts_config_set_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") writing `" << name << "'"; - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); + value.f_len= (ulint) snprintf((char*) value.f_str, FTS_MAX_INT_LEN, + ULINTPF, int_value); + dberr_t error= fts_config_set_value(trx, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + ib::error() << "(" << error << ") writing `" << name << "'"; + ut_free(value.f_str); + return error; } diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc index 59d89c815a94e..a51b672669ef6 100644 --- a/storage/innobase/fts/fts0exec.cc +++ b/storage/innobase/fts/fts0exec.cc @@ -568,7 +568,9 @@ dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, } CommonTableReader::CommonTableReader() : RecordCallback( - [this](const rec_t* rec, const dict_index_t* index, const rec_offs* offsets) -> bool { + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { ulint len; const byte* id_data= rec_get_nth_field(rec, offsets, 0, &len); if (id_data && len != UNIV_SQL_NULL && len == 8) @@ -579,7 +581,34 @@ CommonTableReader::CommonTableReader() : RecordCallback( return true; }, [](const dtuple_t* search_tuple, const rec_t* rec, - const dict_index_t* index, const rec_offs* offsets) -> RecordCompareAction { - return RecordCompareAction::PROCESS; /* Process all records */ - } -) {} + const dict_index_t* index, const rec_offs* offsets) -> RecordCompareAction + { return RecordCompareAction::PROCESS; }) {} + + +ConfigReader::ConfigReader() : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint key_len, value_len; + const byte* key_data= rec_get_nth_field(rec, offsets, 0, &key_len); + const byte* value_data= rec_get_nth_field(rec, offsets, 3, &value_len); + + if (key_data && value_data && key_len != UNIV_SQL_NULL && + value_len != UNIV_SQL_NULL) + { + std::string key(reinterpret_cast(key_data), key_len); + std::string value(reinterpret_cast(value_data), value_len); + config_pairs.emplace_back(key, value); + } + return true; + }, + [](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) -> RecordCompareAction + { + if (!search_tuple) return RecordCompareAction::PROCESS; + uint16_t matched_fields= 0; + int cmp_result= cmp_dtuple_rec_with_match(search_tuple, rec, index, + offsets, &matched_fields); + return (cmp_result == 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + }) {} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 2fa3fa8c1c37f..9309f124abb4d 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -157,27 +157,6 @@ const fts_index_selector_t fts_index_selector[] = { { 0 , NULL } }; -/** Default config values for FTS indexes on a table. */ -static const char* fts_config_table_insert_values_sql = - "PROCEDURE P() IS\n" - "BEGIN\n" - "\n" - "INSERT INTO $config_table VALUES('" - FTS_MAX_CACHE_SIZE_IN_MB "', '256');\n" - "" - "INSERT INTO $config_table VALUES('" - FTS_OPTIMIZE_LIMIT_IN_SECS "', '180');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_SYNCED_DOC_ID "', '0');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_TOTAL_DELETED_COUNT "', '0');\n" - "" /* Note: 0 == FTS_TABLE_STATE_RUNNING */ - "INSERT INTO $config_table VALUES ('" - FTS_TABLE_STATE "', '0');\n" - "END;\n"; - /** FTS tokenize parameter for plugin parser */ struct fts_tokenize_param_t { fts_doc_t* result_doc; /*!< Result doc for tokens */ @@ -1893,88 +1872,74 @@ CREATE TABLE $FTS_PREFIX_CONFIG @param[in] skip_doc_id_index Skip index on doc id @return DB_SUCCESS if succeed */ dberr_t -fts_create_common_tables( - trx_t* trx, - dict_table_t* table, - bool skip_doc_id_index) +fts_create_common_tables(trx_t *trx, dict_table_t *table, + bool skip_doc_id_index) { - dberr_t error; - que_t* graph; - fts_table_t fts_table; - mem_heap_t* heap = mem_heap_create(1024); - pars_info_t* info; - char fts_name[MAX_FULL_NAME_LEN]; - char full_name[sizeof(fts_common_tables) / sizeof(char*)] - [MAX_FULL_NAME_LEN]; - - dict_index_t* index = NULL; - - FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); - - error = fts_drop_common_tables(trx, &fts_table, true); - - if (error != DB_SUCCESS) { - - goto func_exit; - } - - /* Create the FTS tables that are common to an FTS index. */ - for (ulint i = 0; fts_common_tables[i] != NULL; ++i) { - - fts_table.suffix = fts_common_tables[i]; - fts_get_table_name(&fts_table, full_name[i], true); - dict_table_t* common_table = fts_create_one_common_table( - trx, table, full_name[i], fts_table.suffix, heap); - - if (!common_table) { - trx->error_state = DB_SUCCESS; - error = DB_ERROR; - goto func_exit; - } - - mem_heap_empty(heap); - } - - /* Write the default settings to the config table. */ - info = pars_info_create(); - - fts_table.suffix = "CONFIG"; - fts_get_table_name(&fts_table, fts_name, true); - pars_info_bind_id(info, "config_table", fts_name); + dberr_t error= DB_SUCCESS; + char full_name[sizeof(fts_common_tables) / sizeof(char*)][MAX_FULL_NAME_LEN]; + dict_index_t *index= nullptr; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); + error = fts_drop_common_tables(trx, &fts_table, true); + if (error != DB_SUCCESS) return error; + mem_heap_t *heap= mem_heap_create(1024); + /* Create the FTS tables that are common to an FTS index. */ + for (ulint i = 0; fts_common_tables[i] != NULL; ++i) + { + fts_table.suffix = fts_common_tables[i]; + fts_get_table_name(&fts_table, full_name[i], true); + dict_table_t *common_table= fts_create_one_common_table( + trx, table, full_name[i], fts_table.suffix, heap); + if (!common_table) + { + trx->error_state = DB_SUCCESS; + error = DB_ERROR; + mem_heap_free(heap); + return error; + } + mem_heap_empty(heap); + } - graph = pars_sql( - info, fts_config_table_insert_values_sql); + FTSQueryExecutor executor(trx, nullptr, table, true); - error = fts_eval_sql(trx, graph); + /** Does the following insert operation: + INSERT INTO $config_table VALUES('"FTS_MAX_CACHE_SIZE_IN_MB"', '256');" + INSERT INTO $config_table VALUES('"FTS_OPTIMIZE_LIMIT_IN_SECS"', '180');" + INSERT INTO $config_table VALUES ('"FTS_SYNCED_DOC_ID "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TOTAL_DELETED_COUNT "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TABLE_STATE "', '0');" */ + error= executor.insert_config_record(FTS_MAX_CACHE_SIZE_IN_MB, "256"); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_OPTIMIZE_LIMIT_IN_SECS, "180"); - que_graph_free(graph); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_SYNCED_DOC_ID, "0"); - if (error != DB_SUCCESS || skip_doc_id_index) { + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TOTAL_DELETED_COUNT, "0"); - goto func_exit; - } + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TABLE_STATE, "0"); - if (table->versioned()) { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 2); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); - } else { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 1); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - } + if (error != DB_SUCCESS || skip_doc_id_index) goto func_exit; - error = row_create_index_for_mysql(index, trx, NULL, - FIL_ENCRYPTION_DEFAULT, - FIL_DEFAULT_ENCRYPTION_KEY); + if (table->versioned()) + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 2); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); + } + else + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 1); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + } + error= row_create_index_for_mysql(index, trx, NULL, FIL_ENCRYPTION_DEFAULT, + FIL_DEFAULT_ENCRYPTION_KEY); func_exit: - mem_heap_free(heap); - - return(error); + mem_heap_free(heap); + return error; } /** Create one FTS auxiliary index table for an FTS index. @@ -2439,39 +2404,6 @@ fts_trx_add_op( fts_trx_table_add_op(stmt_ftt, doc_id, state, fts_indexes); } -/******************************************************************//** -Fetch callback that converts a textual document id to a binary value and -stores it in the given place. -@return always returns NULL */ -static -ibool -fts_fetch_store_doc_id( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: doc_id_t* to store - doc_id in */ -{ - int n_parsed; - sel_node_t* node = static_cast(row); - doc_id_t* doc_id = static_cast(user_arg); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - char buf[32]; - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - ut_a(len > 0 && len < sizeof(buf)); - - memcpy(buf, dfield_get_data(dfield), len); - buf[len] = '\0'; - - n_parsed = sscanf(buf, FTS_DOC_ID_FORMAT, doc_id); - ut_a(n_parsed == 1); - - return(FALSE); -} - #ifdef FTS_CACHE_SIZE_DEBUG /******************************************************************//** Get the max cache size in bytes. If there is an error reading the @@ -2483,7 +2415,7 @@ ulint fts_get_max_cache_size( /*===================*/ trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: table instance */ + const dict_table_t *table) /*!< in: table */ { dberr_t error; fts_string_t value; @@ -2499,7 +2431,7 @@ fts_get_max_cache_size( value.f_str = ut_malloc_nokey(value.f_len + 1); error = fts_config_get_value( - trx, fts_table, FTS_MAX_CACHE_SIZE_IN_MB, &value); + trx, table, FTS_MAX_CACHE_SIZE_IN_MB, &value); if (UNIV_LIKELY(error == DB_SUCCESS)) { value.f_str[value.f_len] = 0; @@ -2583,43 +2515,22 @@ dberr_t fts_read_synced_doc_id(const dict_table_t *table, doc_id_t *doc_id, trx_t *trx) { - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - fts_table_t fts_table; - fts_table.suffix= "CONFIG"; - fts_table.table_id= table->id; - fts_table.type= FTS_COMMON_TABLE; - fts_table.table= table; ut_a(table->fts->doc_col != ULINT_UNDEFINED); - - trx->op_info = "update the next FTS document id"; - pars_info_t *info= pars_info_create(); - pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id, - doc_id); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "config_table", table_name); - - que_t *graph= fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $config_table" - " WHERE key = 'synced_doc_id' FOR UPDATE;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - + trx->op_info= "reading synced FTS document id"; + FTSQueryExecutor executor(trx, nullptr, table); + ConfigReader reader; *doc_id= 0; - error = fts_eval_sql(trx, graph); - que_graph_free(graph); + dberr_t error= executor.read_config_with_lock("synced_doc_id", reader); + if (error == DB_SUCCESS) + { + const auto& config_pairs= reader.config_pairs; + if (config_pairs.size() == 1) + { + const std::string& value_str= config_pairs[0].second; + int n_parsed= sscanf(value_str.c_str(), FTS_DOC_ID_FORMAT, doc_id); + if (n_parsed != 1) error= DB_ERROR; + } + } return error; } @@ -2712,74 +2623,42 @@ transaction to update the last document id. @param trx update trx or null @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id( - const dict_table_t* table, - doc_id_t doc_id, - trx_t* trx) +fts_update_sync_doc_id(const dict_table_t *table, doc_id_t doc_id, trx_t *trx) { - byte id[FTS_MAX_ID_LEN]; - pars_info_t* info; - fts_table_t fts_table; - ulint id_len; - que_t* graph = NULL; - dberr_t error; - ibool local_trx = FALSE; - fts_cache_t* cache = table->fts->cache; - char fts_name[MAX_FULL_NAME_LEN]; - - if (srv_read_only_mode) { - return DB_READ_ONLY; - } - - fts_table.suffix = "CONFIG"; - fts_table.table_id = table->id; - fts_table.type = FTS_COMMON_TABLE; - fts_table.table = table; - - if (!trx) { - trx = trx_create(); - trx_start_internal(trx); - - trx->op_info = "setting last FTS document id"; - local_trx = TRUE; - } - - info = pars_info_create(); - - id_len = (ulint) snprintf( - (char*) id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - - pars_info_bind_varchar_literal(info, "doc_id", id, id_len); - - fts_get_table_name(&fts_table, fts_name, - table->fts->dict_locked); - pars_info_bind_id(info, "table_name", fts_name); - - graph = fts_parse_sql( - &fts_table, info, - "BEGIN" - " UPDATE $table_name SET value = :doc_id" - " WHERE key = 'synced_doc_id';"); - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - if (local_trx) { - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - cache->synced_doc_id = doc_id; - } else { - ib::error() << "(" << error << ") while" - " updating last doc id for table" - << table->name; + if (srv_read_only_mode) return DB_READ_ONLY; + bool local_trx= false; + if (!trx) + { + trx= trx_create(); + trx_start_internal(trx); + trx->op_info= "setting last FTS document id"; + local_trx= true; + } - fts_sql_rollback(trx); - } - trx->free(); - } + char id[FTS_MAX_ID_LEN]; + snprintf(id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - return(error); + FTSQueryExecutor executor(trx, nullptr, + const_cast(table), + table->fts->dict_locked); + fts_cache_t *cache= table->fts->cache; + dberr_t error= executor.update_config_record("synced_doc_id", id); + if (local_trx) + { + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + fts_sql_commit(trx); + cache->synced_doc_id = doc_id; + } + else + { + sql_print_error("InnoDB: ( %s ) while updating last doc id for table %s", + ut_strerr(error), table->name.m_name); + fts_sql_rollback(trx); + } + trx->free(); + } + return error; } /*********************************************************************//** @@ -4933,8 +4812,6 @@ fts_update_max_cache_size( trx = trx_create(); - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, sync->table); - /* The size returned is in bytes. */ sync->max_cache_size = fts_get_max_cache_size(trx, &fts_table); @@ -5779,7 +5656,6 @@ fts_load_stopword( bool reload) /*!< in: Whether it is for reloading FTS table */ { - fts_table_t fts_table; fts_string_t str; dberr_t error = DB_SUCCESS; ulint use_stopword; @@ -5788,8 +5664,6 @@ fts_load_stopword( ibool new_trx = FALSE; byte str_buffer[MAX_FULL_NAME_LEN + 1]; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, table); - cache = table->fts->cache; if (!reload && !(cache->stopword_info.status & STOPWORD_NOT_INIT)) { @@ -5810,12 +5684,12 @@ fts_load_stopword( /* First check whether stopword filtering is turned off */ if (reload) { error = fts_config_get_ulint( - trx, &fts_table, FTS_USE_STOPWORD, &use_stopword); + trx, table, FTS_USE_STOPWORD, &use_stopword); } else { use_stopword = (ulint) stopword_is_on; error = fts_config_set_ulint( - trx, &fts_table, FTS_USE_STOPWORD, use_stopword); + trx, table, FTS_USE_STOPWORD, use_stopword); } if (error != DB_SUCCESS) { @@ -5837,7 +5711,7 @@ fts_load_stopword( str.f_len = sizeof(str_buffer) - 1; error = fts_config_get_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + trx, table, FTS_STOPWORD_TABLE_NAME, &str); if (error != DB_SUCCESS) { goto cleanup; @@ -5861,7 +5735,7 @@ fts_load_stopword( str.f_len = strlen(stopword_to_use); error = fts_config_set_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + trx, table, FTS_STOPWORD_TABLE_NAME, &str); } } else { /* Load system default stopword list */ diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 554721171caeb..3efc711d68401 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1653,25 +1653,18 @@ fts_optimize_free( mem_heap_free(heap); } -/**********************************************************************//** -Get the max time optimize should run in millisecs. +/** Get the max time optimize should run in millisecs. +@param trx transaction +@param table user table to be optimized @return max optimize time limit in millisecs. */ static -ulint -fts_optimize_get_time_limit( -/*========================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: aux table */ +ulint fts_optimize_get_time_limit(trx_t *trx, const dict_table_t *table) { - ulint time_limit = 0; - - fts_config_get_ulint( - trx, fts_table, - FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); - - /* FIXME: This is returning milliseconds, while the variable - is being stored and interpreted as seconds! */ - return(time_limit * 1000); + ulint time_limit= 0; + fts_config_get_ulint(trx, table, FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); + /* FIXME: This is returning milliseconds, while the variable + is being stored and interpreted as seconds! */ + return(time_limit * 1000); } /**********************************************************************//** @@ -1693,7 +1686,7 @@ fts_optimize_words( /* Get the time limit from the config table. */ fts_optimize_time_limit = fts_optimize_get_time_limit( - optim->trx, &optim->fts_common_table); + optim->trx, optim->table); const time_t start_time = time(NULL); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index eacde4912b89e..b1f38c19079c9 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -11449,6 +11449,7 @@ innobase_fts_load_stopword( bool success= fts_load_stopword(table, trx, stopword_table, THDVAR(thd, ft_enable_stopword), false); table->fts->dict_locked= false; + DBUG_EXECUTE_IF("fts_load_stopword_fail", success= false;); return success; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index cb3c8fc4bdd3b..2aadd421db4f9 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -3118,7 +3118,6 @@ i_s_fts_config_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; dict_table_t* user_table; ulint i = 0; dict_index_t* index = NULL; @@ -3152,8 +3151,6 @@ i_s_fts_config_fill( trx = trx_create(); trx->op_info = "Select for FTS CONFIG TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, user_table); - if (!ib_vector_is_empty(user_table->fts->indexes)) { index = (dict_index_t*) ib_vector_getp_const( user_table->fts->indexes, 0); @@ -3180,7 +3177,7 @@ i_s_fts_config_fill( key_name = (char*) fts_config_key[i]; } - fts_config_get_value(trx, &fts_table, key_name, &value); + fts_config_get_value(trx, user_table, key_name, &value); if (allocated) { ut_free(key_name); diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h index ac4ff7a6f85c9..111451343c6e4 100644 --- a/storage/innobase/include/fts0exec.h +++ b/storage/innobase/include/fts0exec.h @@ -217,4 +217,11 @@ class CommonTableReader : public RecordCallback void clear() { doc_ids.clear(); } }; +/** Callback class for reading FTS config table records */ +class ConfigReader : public RecordCallback +{ +public: + std::vector> config_pairs; + ConfigReader(); +}; #endif /* INNOBASE_FTS0QUERY_H */ diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 1bb1b2a27e519..baae089c031af 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -273,20 +273,18 @@ fts_index_fetch_nodes( MY_ATTRIBUTE((nonnull)); #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() -/******************************************************************//** -Get value from config table. The caller must ensure that enough -space is allocated for value to hold the column contents + +/** Get value from the config table. The caller must ensure that enough +space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /* transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ - MY_ATTRIBUTE((nonnull)); +dberr_t fts_config_get_value(trx_t *trx, const dict_table_t *table, + const char *name, fts_string_t *value) + MY_ATTRIBUTE((nonnull)); + /******************************************************************//** Get value specific to an FTS index from the config table. The caller must ensure that enough space is allocated for value to hold the @@ -309,7 +307,7 @@ dberr_t fts_config_set_value( /*=================*/ trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: get config value for this parameter name */ const fts_string_t* @@ -322,7 +320,7 @@ dberr_t fts_config_set_ulint( /*=================*/ trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ ulint int_value) /*!< in: value */ MY_ATTRIBUTE((nonnull, warn_unused_result)); @@ -372,7 +370,7 @@ dberr_t fts_config_get_ulint( /*=================*/ trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ ulint* int_value) /*!< out: value */ MY_ATTRIBUTE((nonnull)); diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc index 41ddbcdeb6b87..84f8005178198 100644 --- a/storage/innobase/row/row0query.cc +++ b/storage/innobase/row/row0query.cc @@ -374,6 +374,7 @@ dberr_t QueryExecutor::update_record(dict_table_t *table, BTR_STORE_UPDATE); dtuple_big_rec_free(big_rec); } + if (offsets_heap) mem_heap_free(offsets_heap); } } return err; From c518738f522e84515ca38b22901dba6d2ca600d3 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 17 Nov 2025 11:06:55 +0530 Subject: [PATCH 05/12] MDEV-28730 Remove internal parser usage from InnoDB fts Introduce AuxCompareMode and AuxRecordReader to scan FTS auxiliary indexes with compare+process callbacks. Replace legacy SQL-graph APIs with typed executor-based ones: -Add fts_index_fetch_nodes(trx, index, word, user_arg, FTSRecordProcessor,compare_mode). -Redefine fts_write_node() to use FTSQueryExecutor and fts_aux_data_t. Implement write path via delete_aux_record (or) insert_aux_record. Keep lock-wait retry handling and memory limit checks. Change fts_select_index{,_by_range,_by_hash} return type from ulint to uint8_t and simplify return flow. Include fts0exec.h in fts0priv.h and update declarations accordingly. --- storage/innobase/fts/fts0exec.cc | 188 ++++++ storage/innobase/fts/fts0fts.cc | 230 ++----- storage/innobase/fts/fts0opt.cc | 861 ++++++++----------------- storage/innobase/fts/fts0que.cc | 191 ++++-- storage/innobase/handler/i_s.cc | 142 ++-- storage/innobase/include/fts0exec.h | 79 +++ storage/innobase/include/fts0priv.h | 51 +- storage/innobase/include/fts0types.h | 2 +- storage/innobase/include/fts0types.inl | 16 +- 9 files changed, 845 insertions(+), 915 deletions(-) diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc index a51b672669ef6..09a82d7953068 100644 --- a/storage/innobase/fts/fts0exec.cc +++ b/storage/innobase/fts/fts0exec.cc @@ -612,3 +612,191 @@ ConfigReader::ConfigReader() : RecordCallback( return (cmp_result == 0) ? RecordCompareAction::PROCESS : RecordCompareAction::STOP; }) {} + +/** Initial size of nodes in fts_word_t. */ +static const ulint FTS_WORD_NODES_INIT_SIZE = 64; + +/** Initialize fts_word_t structure */ +static void init_fts_word(fts_word_t* word, const byte* utf8, ulint len) +{ + mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); + memset(word, 0, sizeof(*word)); + word->text.f_len = len; + word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); + memcpy(word->text.f_str, utf8, len); + word->text.f_str[len] = 0; + word->heap_alloc = ib_heap_allocator_create(heap); + word->nodes = ib_vector_create(word->heap_alloc, sizeof(fts_node_t), + FTS_WORD_NODES_INIT_SIZE); +} + +/** AuxRecordReader default word processor implementation */ +bool AuxRecordReader::default_word_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + ib_vector_t *words= static_cast(user_arg); + ulint word_len; + const byte *word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + fts_word_t *word; + bool is_word_init = false; + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return true; + + ut_ad(word_len <= FTS_MAX_WORD_LEN); + + if (ib_vector_size(words) == 0) + { + /* First word - push and initialize */ + word = static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init = true; + } + else + { + /* Check if this word is different from the last word */ + word = static_cast(ib_vector_last(words)); + if (word_len != word->text.f_len || + memcmp(word->text.f_str, word_data, word_len)) + { + /* Different word - push new word and initialize */ + word = static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init = true; + } + } + fts_node_t *node= static_cast( + ib_vector_push(word->nodes, nullptr)); + + ulint doc_id_len; + const byte *doc_id_data= rec_get_nth_field(rec, offsets, 1, &doc_id_len); + if (doc_id_data && doc_id_len == 8) + node->first_doc_id= fts_read_doc_id(doc_id_data); + else node->first_doc_id= 0; + + /* Read last_doc_id (field 4) */ + doc_id_data= rec_get_nth_field(rec, offsets, 4, &doc_id_len); + if (doc_id_data && doc_id_len == 8) + node->last_doc_id= fts_read_doc_id(doc_id_data); + else node->last_doc_id= 0; + + /* Read doc_count (field 5) */ + ulint doc_count_len; + const byte *doc_count_data= rec_get_nth_field(rec, offsets, 5, + &doc_count_len); + if (doc_count_data && doc_count_len == 4) + node->doc_count= mach_read_from_4(doc_count_data); + else node->doc_count= 0; + + /* Read ilist (field 6) with external BLOB support */ + ulint ilist_len= 0; + const byte *ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + byte *external_data= nullptr; + mem_heap_t *temp_heap= nullptr; + + node->ilist_size_alloc= node->ilist_size= 0; + node->ilist= nullptr; + + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + if (rec_offs_nth_extern(offsets, 6)) + { + temp_heap= mem_heap_create(ilist_len); + ulint external_len; + external_data= btr_copy_externally_stored_field( + &external_len, ilist_data, index->table->space->zip_size(), + ilist_len, temp_heap); + if (external_data) + { + ilist_data= external_data; + ilist_len= external_len; + } + } + node->ilist_size_alloc= node->ilist_size= ilist_len; + if (ilist_len) + { + node->ilist= static_cast(ut_malloc_nokey(ilist_len)); + memcpy(node->ilist, ilist_data, ilist_len); + } + if (temp_heap) mem_heap_free(temp_heap); + if (ilist_len == 0) return false; + } + + if (this->total_memory) + { + if (is_word_init) + { + *this->total_memory+= + sizeof(fts_word_t) + sizeof(ib_alloc_t) + + sizeof(ib_vector_t) + word_len + + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; + } + *this->total_memory += node->ilist_size; + if (*this->total_memory >= fts_result_cache_limit) + return false; + } + return true; +} + +/** AuxRecordReader comparison logic implementation */ +RecordCompareAction AuxRecordReader::compare_record( + const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) noexcept +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const void* search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + + ulint rec_len; + const byte* rec_data= rec_get_nth_field(rec, offsets, 0, &rec_len); + + if (!rec_data || rec_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + int cmp_result; + switch (compare_mode) + { + case AuxCompareMode::GREATER_EQUAL: + case AuxCompareMode::GREATER: + { + uint16_t matched_fields= 0; + cmp_result= cmp_dtuple_rec_with_match(search_tuple, rec, index, + offsets, &matched_fields); + if (compare_mode == AuxCompareMode::GREATER_EQUAL) + return (cmp_result <= 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + else + return (cmp_result < 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + case AuxCompareMode::LIKE: + case AuxCompareMode::EQUAL: + { + /* Use charset-aware comparison for LIKE and EQUAL modes */ + const dtype_t* type= dfield_get_type(search_field); + cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_data, rec_len); + + if (compare_mode == AuxCompareMode::EQUAL) + return cmp_result == 0 + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + else /* AuxCompareMode::LIKE */ + { + /* For LIKE mode, compare only the prefix (search_len bytes) */ + int prefix_cmp = cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_data, + search_len <= rec_len ? search_len : rec_len); + + if (prefix_cmp != 0) return RecordCompareAction::STOP; + return (search_len <= rec_len) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + } + } + return RecordCompareAction::PROCESS; +} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 9309f124abb4d..c99123cd8c6ec 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -3671,76 +3671,19 @@ fts_doc_fetch_by_doc_id( return(error); } -/*********************************************************************//** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ -{ - pars_info_t* info; - dberr_t error; - ib_uint32_t doc_count; - time_t start_time; - doc_id_t last_doc_id; - doc_id_t first_doc_id; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(node->ilist != NULL); - - if (*graph) { - info = (*graph)->info; - } else { - info = pars_info_create(); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - pars_info_bind_varchar_literal(info, "token", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &first_doc_id, node->first_doc_id); - fts_bind_doc_id(info, "first_doc_id", &first_doc_id); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &last_doc_id, node->last_doc_id); - fts_bind_doc_id(info, "last_doc_id", &last_doc_id); - - ut_a(node->last_doc_id >= node->first_doc_id); - - /* Convert to "storage" byte order. */ - mach_write_to_4((byte*) &doc_count, node->doc_count); - pars_info_bind_int4_literal( - info, "doc_count", (const ib_uint32_t*) &doc_count); - - /* Set copy_name to FALSE since it's a static. */ - pars_info_bind_literal( - info, "ilist", node->ilist, node->ilist_size, - DATA_BLOB, DATA_BINARY_TYPE); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "BEGIN\n" - "INSERT INTO $index_table_name VALUES" - " (:token, :first_doc_id," - " :last_doc_id, :doc_count, :ilist);"); - } - - start_time = time(NULL); - error = fts_eval_sql(trx, *graph); - elapsed_time += time(NULL) - start_time; - ++n_nodes; - - return(error); +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) +{ + time_t start_time = time(NULL); + dberr_t error= executor->insert_aux_record(selected, aux_data); + elapsed_time += time(NULL) - start_time; + ++n_nodes; + return error; } /** Sort an array of doc_id */ @@ -3776,104 +3719,63 @@ fts_sync_add_deleted_cache(fts_sync_t *sync, ib_vector_t *doc_ids) @param[in] unlock_cache whether unlock cache when write node @return DB_SUCCESS if all went well else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_sync_write_words( - trx_t* trx, - fts_index_cache_t* index_cache, - bool unlock_cache) +dberr_t fts_sync_write_words(trx_t *trx, fts_index_cache_t *index_cache, + bool unlock_cache) { - fts_table_t fts_table; - ulint n_nodes = 0; - ulint n_words = 0; - const ib_rbt_node_t* rbt_node; - dberr_t error = DB_SUCCESS; - ibool print_error = FALSE; - dict_table_t* table = index_cache->index->table; - - FTS_INIT_INDEX_TABLE( - &fts_table, NULL, FTS_INDEX_TABLE, index_cache->index); - - n_words = rbt_size(index_cache->words); - - /* We iterate over the entire tree, even if there is an error, - since we want to free the memory used during caching. */ - for (rbt_node = rbt_first(index_cache->words); - rbt_node; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - ulint i; - ulint selected; - fts_tokenizer_word_t* word; - - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - DBUG_EXECUTE_IF( - "fts_instrument_write_words_before_select_index", - std::this_thread::sleep_for( - std::chrono::milliseconds(300));); - - selected = fts_select_index( - index_cache->charset, word->text.f_str, - word->text.f_len); - - fts_table.suffix = fts_get_suffix(selected); - - /* We iterate over all the nodes even if there was an error */ - for (i = 0; i < ib_vector_size(word->nodes); ++i) { - - fts_node_t* fts_node = static_cast( - ib_vector_get(word->nodes, i)); - - if (fts_node->synced) { - continue; - } else { - fts_node->synced = true; - } - - /*FIXME: we need to handle the error properly. */ - if (error == DB_SUCCESS) { - if (unlock_cache) { - mysql_mutex_unlock( - &table->fts->cache->lock); - } - - error = fts_write_node( - trx, - &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); - - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", - DBUG_SUICIDE();); - - DBUG_EXECUTE_IF( - "fts_instrument_sync_sleep", - std::this_thread::sleep_for( - std::chrono::seconds(1));); - - if (unlock_cache) { - mysql_mutex_lock( - &table->fts->cache->lock); - } - } - } - - n_nodes += ib_vector_size(word->nodes); + dict_table_t *table= index_cache->index->table; + FTSQueryExecutor executor(trx, index_cache->index, table); + ulint n_words= rbt_size(index_cache->words); + bool print_error= false; + dberr_t error= DB_SUCCESS; + for (const ib_rbt_node_t *rbt_node= rbt_first(index_cache->words); + rbt_node; rbt_node = rbt_next(index_cache->words, rbt_node)) + { + fts_tokenizer_word_t *word= rbt_value(fts_tokenizer_word_t, rbt_node); + DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index", + std::this_thread::sleep_for( + std::chrono::milliseconds(300));); + uint8_t selected= fts_select_index( + index_cache->charset, word->text.f_str, word->text.f_len); + + for (ulint i = 0; i < ib_vector_size(word->nodes); ++i) + { + fts_node_t* fts_node= + static_cast(ib_vector_get(word->nodes, i)); + if (fts_node->synced) continue; + else fts_node->synced= true; + /* FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) + { + if (unlock_cache) mysql_mutex_unlock(&table->fts->cache->lock); + fts_aux_data_t aux_data((const char*)word->text.f_str, word->text.f_len, + fts_node->first_doc_id, fts_node->last_doc_id, + static_cast(fts_node->doc_count), fts_node->ilist, + fts_node->ilist_size); + error= fts_write_node(&executor, selected, &aux_data); + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + std::this_thread::sleep_for(std::chrono::seconds(1));); + + if (unlock_cache) mysql_mutex_lock(&table->fts->cache->lock); + } - if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { - ib::error() << "(" << error << ") writing" - " word node to FTS auxiliary index table " - << table->name; - print_error = TRUE; - } - } + n_nodes += ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(fts_enable_diag_print)) { - printf("Avg number of nodes: %lf\n", - (double) n_nodes / (double) (n_words > 1 ? n_words : 1)); - } + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) + { + sql_print_error("InnoDB: ( %s ) writing word node to FTS auxiliary " + "index table %s", ut_strerr(error), + table->name.m_name); + print_error= true; + } + } + } - return(error); + if (UNIV_UNLIKELY(fts_enable_diag_print)) + printf("Avg number of nodes: %lf\n", + (double) n_nodes / (double) (n_words > 1 ? n_words : 1)); + return error; } /*********************************************************************//** diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 3efc711d68401..35884b669bf5c 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -71,9 +71,6 @@ static bool fts_opt_start_shutdown = false; Protected by fts_optimize_wq->mutex. */ static pthread_cond_t fts_opt_shutdown_cond; -/** Initial size of nodes in fts_word_t. */ -static const ulint FTS_WORD_NODES_INIT_SIZE = 64; - /** Last time we did check whether system need a sync */ static time_t last_check_sync_time; @@ -309,238 +306,50 @@ fts_zip_init( *zip->word.f_str = '\0'; } -/**********************************************************************//** -Create a fts_optimizer_word_t instance. -@return new instance */ -static -fts_word_t* -fts_word_init( -/*==========*/ - fts_word_t* word, /*!< in: word to initialize */ - byte* utf8, /*!< in: UTF-8 string */ - ulint len) /*!< in: length of string in bytes */ +dberr_t fts_index_fetch_nodes(trx_t *trx, dict_index_t *index, + const fts_string_t *word, void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) { - mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); - - memset(word, 0, sizeof(*word)); - - word->text.f_len = len; - word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); - - /* Need to copy the NUL character too. */ - memcpy(word->text.f_str, utf8, word->text.f_len); - word->text.f_str[word->text.f_len] = 0; - - word->heap_alloc = ib_heap_allocator_create(heap); - - word->nodes = ib_vector_create( - word->heap_alloc, sizeof(fts_node_t), FTS_WORD_NODES_INIT_SIZE); - - return(word); -} - -/**********************************************************************//** -Read the FTS INDEX row. -@return fts_node_t instance */ -static -fts_node_t* -fts_optimize_read_node( -/*===================*/ - fts_word_t* word, /*!< in: */ - que_node_t* exp) /*!< in: */ -{ - int i; - fts_node_t* node = static_cast( - ib_vector_push(word->nodes, NULL)); - - /* Start from 1 since the first node has been read by the caller */ - for (i = 1; exp; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT */ - switch (i) { - case 1: /* DOC_COUNT */ - node->doc_count = mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node->first_doc_id = fts_read_doc_id(data); - break; - - case 3: /* LAST_DOC_ID */ - node->last_doc_id = fts_read_doc_id(data); - break; - - case 4: /* ILIST */ - node->ilist_size_alloc = node->ilist_size = len; - node->ilist = static_cast(ut_malloc_nokey(len)); - memcpy(node->ilist, data, len); - break; - - default: - ut_error; - } - } - - /* Make sure all columns were read. */ - ut_a(i == 5); - - return(node); -} - -/**********************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns non-NULL */ -ibool -fts_optimize_index_fetch_node( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - fts_word_t* word; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - ib_vector_t* words = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - fts_node_t* node; - bool is_word_init = false; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - if (ib_vector_size(words) == 0) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - word = static_cast(ib_vector_last(words)); - - if (dfield_len != word->text.f_len - || memcmp(word->text.f_str, data, dfield_len)) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - node = fts_optimize_read_node(word, que_node_get_next(exp)); - - fetch->total_memory += node->ilist_size; - if (is_word_init) { - fetch->total_memory += sizeof(fts_word_t) - + sizeof(ib_alloc_t) + sizeof(ib_vector_t) + dfield_len - + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; - } else if (ib_vector_size(words) > FTS_WORD_NODES_INIT_SIZE) { - fetch->total_memory += sizeof(fts_node_t); - } - - if (fetch->total_memory >= fts_result_cache_limit) { - return(FALSE); - } - - return(TRUE); -} - -/**********************************************************************//** -Read the rows from the FTS inde. -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: table of the FTS INDEX */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ -{ - pars_info_t* info; - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - ut_a(fts_table->type == FTS_INDEX_TABLE); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - - fts_get_table_name(fts_table, table_name); - - pars_info_bind_id(info, "table_name", table_name); - } - - pars_info_bind_function(info, "my_func", fetch->read_record, fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name\n" - " WHERE word LIKE :word\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - return(error); + dberr_t error= DB_SUCCESS; + trx->op_info= "fetching FTS index nodes"; + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + ulint total_memory= 0; + for (;;) + { + FTSQueryExecutor executor(trx, index, index->table); + AuxRecordReader reader= processor + ? AuxRecordReader(user_arg, processor, compare_mode) + : AuxRecordReader(user_arg, &total_memory, compare_mode); + if (word->f_len == 0) + error= executor.read_aux_all(selected, reader); + else + error= executor.read_aux( + selected, reinterpret_cast(word->f_str), + PAGE_CUR_GE, reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) + { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + fts_sql_commit(trx); + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading FTS index. Retrying!"; + trx->error_state= DB_SUCCESS; + } + else + { + ib::error() << "(" << error << ") while reading FTS index."; + break; + } + } + } + return error; } /**********************************************************************//** @@ -644,88 +453,6 @@ fts_zip_read_word( return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } -/**********************************************************************//** -Callback function to fetch and compress the word in an FTS -INDEX record. -@return FALSE on EOF */ -static -ibool -fts_fetch_index_words( -/*==================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - sel_node_t* sel_node = static_cast(row); - fts_zip_t* zip = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - - ut_a(dfield_get_len(dfield) <= FTS_MAX_WORD_LEN); - - uint16 len = uint16(dfield_get_len(dfield)); - void* data = dfield_get_data(dfield); - - /* Skip the duplicate words. */ - if (zip->word.f_len == len && !memcmp(zip->word.f_str, data, len)) { - return(TRUE); - } - - memcpy(zip->word.f_str, data, len); - zip->word.f_len = len; - - ut_a(zip->zp->avail_in == 0); - ut_a(zip->zp->next_in == NULL); - - /* The string is prefixed by len. */ - /* FIXME: This is not byte order agnostic (InnoDB data files - with FULLTEXT INDEX are not portable between little-endian and - big-endian systems!) */ - zip->zp->next_in = reinterpret_cast(&len); - zip->zp->avail_in = sizeof(len); - - /* Compress the word, create output blocks as necessary. */ - while (zip->zp->avail_in > 0) { - - /* No space left in output buffer, create a new one. */ - if (zip->zp->avail_out == 0) { - byte* block; - - block = static_cast( - ut_malloc_nokey(zip->block_sz)); - - ib_vector_push(zip->blocks, &block); - - zip->zp->next_out = block; - zip->zp->avail_out = static_cast(zip->block_sz); - } - - switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) { - case Z_OK: - if (zip->zp->avail_in == 0) { - zip->zp->next_in = static_cast(data); - zip->zp->avail_in = uInt(len); - ut_a(len <= FTS_MAX_WORD_LEN); - len = 0; - } - continue; - - case Z_STREAM_END: - case Z_BUF_ERROR: - case Z_STREAM_ERROR: - default: - ut_error; - } - } - - /* All data should have been compressed. */ - ut_a(zip->zp->avail_in == 0); - zip->zp->next_in = NULL; - - ++zip->n_words; - - return(zip->n_words >= zip->max_words ? FALSE : TRUE); -} - /**********************************************************************//** Finish Zip deflate. */ static @@ -768,136 +495,163 @@ fts_zip_deflate_end( memset(zip->zp, 0, sizeof(*zip->zp)); } -/**********************************************************************//** -Read the words from the FTS INDEX. +/** Read the words from the FTS INDEX. +@param optim optimize scratch pad +@param word get words gerater than this +@param n_words max words to read @return DB_SUCCESS if all OK, DB_TABLE_NOT_FOUND if no more indexes to search else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_index_fetch_words( -/*==================*/ - fts_optimize_t* optim, /*!< in: optimize scratch pad */ - const fts_string_t* word, /*!< in: get words greater than this - word */ - ulint n_words)/*!< in: max words to read */ +dberr_t fts_index_fetch_words(fts_optimize_t *optim, + const fts_string_t *word, + ulint n_words) { - pars_info_t* info; - que_t* graph; - ulint selected; - fts_zip_t* zip = NULL; - dberr_t error = DB_SUCCESS; - mem_heap_t* heap = static_cast(optim->self_heap->arg); - ibool inited = FALSE; + dberr_t error= DB_SUCCESS; + mem_heap_t *heap= static_cast(optim->self_heap->arg); + optim->trx->op_info= "fetching FTS index words"; - optim->trx->op_info = "fetching FTS index words"; + if (optim->zip == NULL) + optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); + else fts_zip_initialize(optim->zip); - if (optim->zip == NULL) { - optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); - } else { - fts_zip_initialize(optim->zip); - } + CHARSET_INFO *cs= fts_index_get_charset(optim->index); - for (selected = fts_select_index( - optim->fts_index_table.charset, word->f_str, word->f_len); - selected < FTS_NUM_AUX_INDEX; - selected++) { - - char table_name[MAX_FULL_NAME_LEN]; - - optim->fts_index_table.suffix = fts_get_suffix(selected); - - info = pars_info_create(); - - pars_info_bind_function( - info, "my_func", fts_fetch_index_words, optim->zip); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - fts_get_table_name(&optim->fts_index_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &optim->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word\n" - " FROM $table_name\n" - " WHERE word > :word\n" - " ORDER BY word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - zip = optim->zip; - - for (;;) { - int err; - - if (!inited && ((err = deflateInit(zip->zp, 9)) - != Z_OK)) { - ib::error() << "ZLib deflateInit() failed: " - << err; - - error = DB_ERROR; - break; - } else { - inited = TRUE; - error = fts_eval_sql(optim->trx, graph); - } + /* Create compression processor with state */ + bool compress_inited = false; + auto compress_processor= [&compress_inited]( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) + { + fts_zip_t* zip= static_cast(user_arg); - if (UNIV_LIKELY(error == DB_SUCCESS)) { - //FIXME fts_sql_commit(optim->trx); - break; - } else { - //FIXME fts_sql_rollback(optim->trx); + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL || + word_len > FTS_MAX_WORD_LEN) + return false; - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout" - " reading document. Retrying!"; + /* Skip duplicate words */ + if (zip->word.f_len == word_len && + !memcmp(zip->word.f_str, word_data, word_len)) + return true; - /* We need to reset the ZLib state. */ - inited = FALSE; - deflateEnd(zip->zp); - fts_zip_init(zip); + /* Initialize deflate if not done yet */ + if (!compress_inited) + { + int err = deflateInit(zip->zp, 9); + if (err != Z_OK) + { + ib::error() << "ZLib deflateInit() failed: " << err; + return false; + } + compress_inited = true; + } - optim->trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading document."; + /* Update current word */ + memcpy(zip->word.f_str, word_data, word_len); + zip->word.f_len = word_len; + ut_a(zip->zp->avail_in == 0); + ut_a(zip->zp->next_in == NULL); - break; /* Exit the loop. */ - } - } - } + /* Compress the word with length prefix */ + uint16_t len = static_cast(word_len); + zip->zp->next_in = reinterpret_cast(&len); + zip->zp->avail_in = sizeof(len); - que_graph_free(graph); + /* Compress the word, create output blocks as necessary */ + while (zip->zp->avail_in > 0) + { + /* No space left in output buffer, create a new one */ + if (zip->zp->avail_out == 0) + { + byte* block= static_cast(ut_malloc_nokey(zip->block_sz)); + ib_vector_push(zip->blocks, &block); + zip->zp->next_out= block; + zip->zp->avail_out= static_cast(zip->block_sz); + } + + switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) + { + case Z_OK: + if (zip->zp->avail_in == 0) + { + zip->zp->next_in= static_cast(const_cast(word_data)); + zip->zp->avail_in = static_cast(len); + ut_a(len <= FTS_MAX_WORD_LEN); + len = 0; + } + continue; + case Z_STREAM_END: + case Z_BUF_ERROR: + case Z_STREAM_ERROR: + default: + ut_error; + } + } - /* Check if max word to fetch is exceeded */ - if (optim->zip->n_words >= n_words) { - break; - } - } + /* All data should have been compressed */ + ut_a(zip->zp->avail_in == 0); + zip->zp->next_in = NULL; - if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + ++zip->n_words; - /* All data should have been read. */ - ut_a(zip->zp->avail_in == 0); + /* Continue until we reach max words */ + return zip->n_words < zip->max_words; + }; - fts_zip_deflate_end(zip); - } else { - deflateEnd(zip->zp); - } + for (uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + selected < FTS_NUM_AUX_INDEX; selected++) + { + for (;;) + { + FTSQueryExecutor executor(optim->trx, optim->index, optim->table); + AuxRecordReader aux_reader(optim->zip, compress_processor, + AuxCompareMode::GREATER); + + if (word->f_len == 0) + error= executor.read_aux_all(selected, aux_reader); + else error= executor.read_aux( + selected, + reinterpret_cast(word->f_str), + PAGE_CUR_G, aux_reader); + + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading words. Retrying!"; + if (compress_inited) + { + deflateEnd(optim->zip->zp); + fts_zip_init(optim->zip); + compress_inited= false; + } + optim->trx->error_state = DB_SUCCESS; + } + else + { + ib::error() << "(" << ut_strerr(error) << ") while reading words."; + break; + } + } + } - return(error); + if (optim->zip->n_words >= n_words) break; + } + + fts_zip_t *zip = optim->zip; + if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been read */ + ut_a(zip->zp->avail_in == 0); + fts_zip_deflate_end(zip); + } + else deflateEnd(zip->zp); + + return error; } dberr_t fts_table_fetch_doc_ids(trx_t *trx, dict_table_t *table, @@ -1328,87 +1082,55 @@ fts_optimize_word( return(nodes); } -/**********************************************************************//** -Update the FTS index table. This is a delete followed by an insert. +/** Update the FTS index table. This is a delete followed +by an insert operation +@param trx transaction +@param index index to update +@param word word to update +@param nodes nodes to update @return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_write_word( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table of FTS index */ - fts_string_t* word, /*!< in: word data to write */ - ib_vector_t* nodes) /*!< in: the nodes to write */ +dberr_t fts_optimize_write_word(trx_t *trx, dict_index_t *index, + fts_string_t *word, ib_vector_t *nodes) { - ulint i; - pars_info_t* info; - que_t* graph; - ulint selected; - dberr_t error = DB_SUCCESS; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_ad(fts_table->charset); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "BEGIN DELETE FROM $table_name WHERE word = :word;"); - - error = fts_eval_sql(trx, graph); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") during optimize," - " when deleting a word from the FTS index."; - } - - que_graph_free(graph); - graph = NULL; - - /* Even if the operation needs to be rolled back and redone, - we iterate over the nodes in order to free the ilist. */ - for (i = 0; i < ib_vector_size(nodes); ++i) { - - fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); - - if (error == DB_SUCCESS) { - /* Skip empty node. */ - if (node->ilist == NULL) { - ut_ad(node->ilist_size == 0); - continue; - } - - error = fts_write_node( - trx, &graph, fts_table, word, node); + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + FTSQueryExecutor executor(trx, index, index->table); + fts_aux_data_t aux_data((const char*)word->f_str, word->f_len); + dberr_t err= executor.delete_aux_record(selected, &aux_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "deleting a word from the FTS index.", + ut_strerr(err)); + return err; + } - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ")" - " during optimize, while adding a" - " word to the FTS index."; - } - } + for (ulint i = 0; i < ib_vector_size(nodes); ++i) + { + fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); + if (!node->ilist || node->ilist_size == 0) continue; - ut_free(node->ilist); - node->ilist = NULL; - node->ilist_size = node->ilist_size_alloc = 0; - } + fts_aux_data_t insert_data( + (const char*)word->f_str, word->f_len, + node->first_doc_id, node->last_doc_id, + static_cast(node->doc_count), node->ilist, + node->ilist_size); - if (graph != NULL) { - que_graph_free(graph); - } + err = executor.insert_aux_record(selected, &insert_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "inserting a word to the FTS index.", + ut_strerr(err)); + return err; + } + ut_free(node->ilist); + node->ilist= nullptr; + node->ilist_size= node->ilist_size_alloc= 0; + } - return(error); + return DB_SUCCESS; } /**********************************************************************//** @@ -1456,7 +1178,7 @@ fts_optimize_compact( /* Update the data on disk. */ error = fts_optimize_write_word( - trx, &optim->fts_index_table, &word->text, nodes); + trx, index, &word->text, nodes); if (error == DB_SUCCESS) { /* Write the last word optimized to the config table, @@ -1667,96 +1389,56 @@ ulint fts_optimize_get_time_limit(trx_t *trx, const dict_table_t *table) return(time_limit * 1000); } -/**********************************************************************//** -Run OPTIMIZE on the given table. Note: this can take a very long time -(hours). */ +/** Run OPTIMIZE on the given table. Note: this can take a very +long time (hours). +@param optim optimize instance +@param index current fts being optimized +@param word starting word to optimize */ static -void -fts_optimize_words( -/*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: current FTS being optimized */ - fts_string_t* word) /*!< in: the starting word to optimize */ +void fts_optimize_words(fts_optimize_t *optim, dict_index_t *index, + fts_string_t *word) { - fts_fetch_t fetch; - que_t* graph = NULL; - CHARSET_INFO* charset = optim->fts_index_table.charset; - - ut_a(!optim->done); - - /* Get the time limit from the config table. */ - fts_optimize_time_limit = fts_optimize_get_time_limit( - optim->trx, optim->table); - - const time_t start_time = time(NULL); - - /* Setup the callback to use for fetching the word ilist etc. */ - fetch.read_arg = optim->words; - fetch.read_record = fts_optimize_index_fetch_node; - - while (!optim->done) { - dberr_t error; - trx_t* trx = optim->trx; - ulint selected; - - ut_a(ib_vector_size(optim->words) == 0); - - selected = fts_select_index(charset, word->f_str, word->f_len); - - /* Read the index records to optimize. */ - fetch.total_memory = 0; - error = fts_index_fetch_nodes( - trx, &graph, &optim->fts_index_table, word, - &fetch); - ut_ad(fetch.total_memory < fts_result_cache_limit); + ut_a(!optim->done); + /* Get the time limit from the config table. */ + fts_optimize_time_limit= + fts_optimize_get_time_limit(optim->trx, index->table); + const time_t start_time= time(NULL); - if (error == DB_SUCCESS) { - /* There must be some nodes to read. */ - ut_a(ib_vector_size(optim->words) > 0); - - /* Optimize the nodes that were read and write - back to DB. */ - error = fts_optimize_compact(optim, index, start_time); - - if (error == DB_SUCCESS) { - fts_sql_commit(optim->trx); - } else { - fts_sql_rollback(optim->trx); - } - } - - ib_vector_reset(optim->words); - - if (error == DB_SUCCESS) { - if (!optim->done) { - if (!fts_zip_read_word(optim->zip, word)) { - optim->done = TRUE; - } else if (selected - != fts_select_index( - charset, word->f_str, - word->f_len) - && graph) { - que_graph_free(graph); - graph = NULL; - } - } - } else if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout during optimize." - " Retrying!"; - - trx->error_state = DB_SUCCESS; - } else if (error == DB_DEADLOCK) { - ib::warn() << "Deadlock during optimize. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - optim->done = TRUE; /* Exit the loop. */ - } - } + while (!optim->done) + { + trx_t *trx= optim->trx; + ut_a(ib_vector_size(optim->words) == 0); + /* Read the index records to optimize. */ + dberr_t error= fts_index_fetch_nodes( + trx, index, word, optim->words, nullptr, AuxCompareMode::LIKE); + if (error == DB_SUCCESS) + { + /* There must be some nodes to read. */ + ut_a(ib_vector_size(optim->words) > 0); + /* Optimize the nodes that were read and write back to DB. */ + error = fts_optimize_compact(optim, index, start_time); + if (error == DB_SUCCESS) fts_sql_commit(optim->trx); + else fts_sql_rollback(optim->trx); + } + ib_vector_reset(optim->words); - if (graph != NULL) { - que_graph_free(graph); - } + if (error == DB_SUCCESS) + { + if (!optim->done && !fts_zip_read_word(optim->zip, word)) + optim->done= TRUE; + } + else if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout during optimize. Retrying!"; + trx->error_state= DB_SUCCESS; + } + else if (error == DB_DEADLOCK) + { + ib::warn() << "Deadlock during optimize. Retrying!"; + trx->error_state = DB_SUCCESS; + } + else optim->done = TRUE; + } } /**********************************************************************//** @@ -1828,6 +1510,7 @@ fts_optimize_index_read_words( error = DB_SUCCESS; } + optim->index = index; while (error == DB_SUCCESS) { error = fts_index_fetch_words( diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 2dd21e99e24c6..729b1538a38b9 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -310,6 +310,130 @@ fts_ast_visit_sub_exp( fts_ast_callback visitor, void* arg); +/** Process query records for FTS queries. +@param rec record +@param index index +@param offsets record offsets +@param user_arg user argument +@return true to continue processing, false to stop */ +static bool node_query_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + fts_query_t* query= static_cast(user_arg); + fts_string_t key; + ulint word_len; + const byte* word_data = rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL || + word_len > FTS_MAX_WORD_LEN) + return true; + key.f_str= const_cast(word_data); + key.f_len= word_len; + ut_a(query->cur_node->type == FTS_AST_TERM + || query->cur_node->type == FTS_AST_TEXT + || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); + + fts_node_t node; + memset(&node, 0, sizeof(node)); + + fts_string_t term; + byte buf[FTS_MAX_WORD_LEN + 1]; + term.f_str= buf; + + /* Need to consider the wildcard search case, the word frequency + is created on the search string not the actual word. So we need + to assign the frequency on search string behalf. */ + if (query->cur_node->type == FTS_AST_TERM && query->cur_node->term.wildcard) + { + term.f_len = query->cur_node->term.ptr->len; + ut_ad(FTS_MAX_WORD_LEN >= term.f_len); + memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); + } + else + { + term.f_len = word_len; + ut_ad(FTS_MAX_WORD_LEN >= word_len); + memcpy(term.f_str, word_data, word_len); + } + + /* Lookup the word in our rb tree, it must exist. */ + ib_rbt_bound_t parent; + int ret= rbt_search(query->word_freqs, &parent, &term); + + ut_a(ret == 0); + fts_word_freq_t* word_freq= rbt_value(fts_word_freq_t, parent.last); + bool skip = false; + + ulint doc_id_len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, 1, &doc_id_len); + + if (doc_id_data && doc_id_len == 8) + { + node.first_doc_id = fts_read_doc_id(doc_id_data); + skip= (query->oper == FTS_EXIST && query->upper_doc_id > 0 && + node.first_doc_id > query->upper_doc_id); + } + + doc_id_data= rec_get_nth_field(rec, offsets, 4, &doc_id_len); + if (doc_id_data && doc_id_len == 8) + { + node.last_doc_id = fts_read_doc_id(doc_id_data); + skip= (query->oper == FTS_EXIST && query->lower_doc_id > 0 && + node.last_doc_id < query->lower_doc_id); + } + + ulint doc_count_len; + const byte* doc_count_data= rec_get_nth_field(rec, offsets, 5, &doc_count_len); + if (doc_count_data && doc_count_len == 4) + word_freq->doc_count += mach_read_from_4(doc_count_data); + + if (!skip) + { + ulint ilist_len; + const byte* ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + byte* external_data = nullptr; + mem_heap_t* temp_heap = nullptr; + + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + /* Check if ilist is stored externally */ + if (rec_offs_nth_extern(offsets, 6)) + { + /* Create temporary heap for external data */ + temp_heap = mem_heap_create(ilist_len + 1000); + /* Fetch the externally stored BLOB data */ + ulint external_len; + external_data = btr_copy_externally_stored_field( + &external_len, ilist_data, + query->index->table->space->zip_size(), + ilist_len, temp_heap); + + if (external_data) + { + ilist_data = external_data; + ilist_len = external_len; + } + else + { + /* Failed to fetch external data, skip this node */ + if (temp_heap) mem_heap_free(temp_heap); + return true; + } + } + + /* Process the ilist data (either inline or external) */ + query->error= fts_query_filter_doc_ids( + query, &word_freq->word, word_freq, &node, + const_cast(ilist_data), ilist_len, FALSE); + + /* Clean up temporary heap if used */ + if (temp_heap) mem_heap_free(temp_heap); + + if (query->error != DB_SUCCESS) return false; + } + } + return true; +} #if 0 /*****************************************************************//*** Find a doc_id in a word's ilist. @@ -1121,10 +1245,8 @@ fts_query_difference( /* There is nothing we can substract from an empty set. */ if (query->doc_ids && !rbt_empty(query->doc_ids)) { ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1162,21 +1284,21 @@ fts_query_difference( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + trx, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); if (error != DB_SUCCESS) { query->error = error; } - - que_graph_free(graph); } /* The size can't increase. */ @@ -1222,10 +1344,8 @@ fts_query_intersect( if (!(rbt_empty(query->doc_ids) && query->multi_exist)) { ulint n_doc_ids = 0; ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1296,13 +1416,15 @@ fts_query_intersect( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + trx, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1310,8 +1432,6 @@ fts_query_intersect( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* Make the intesection (rb tree) the current doc id set and free the old set. */ @@ -1389,10 +1509,8 @@ fts_query_union( fts_query_t* query, /*!< in: query instance */ fts_string_t* token) /*!< in: token to search */ { - fts_fetch_t fetch; ulint n_doc_ids = 0; trx_t* trx = query->trx; - que_t* graph = NULL; dberr_t error; ut_a(query->oper == FTS_NONE || query->oper == FTS_DECR_RATING || @@ -1417,14 +1535,16 @@ fts_query_union( fts_query_cache(query, token); - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } /* Read the nodes from disk. */ error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + trx, query->index, token, query, node_query_processor, + compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1432,8 +1552,6 @@ fts_query_union( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* The size can't decrease. */ @@ -2751,7 +2869,6 @@ fts_query_phrase_search( fts_fetch_t fetch; trx_t* trx = query->trx; fts_ast_oper_t oper = query->oper; - que_t* graph = NULL; ulint i; dberr_t error; @@ -2801,9 +2918,15 @@ fts_query_phrase_search( query->matched = query->match_array[i]; } + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } + error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, - token, &fetch); + trx, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -2811,9 +2934,6 @@ fts_query_phrase_search( query->error = error; } - que_graph_free(graph); - graph = NULL; - fts_query_cache(query, token); if (!(query->flags & FTS_PHRASE) @@ -2954,12 +3074,11 @@ fts_query_get_token( if (node->term.wildcard) { - token->f_str = static_cast(ut_malloc_nokey(str_len + 2)); - token->f_len = str_len + 1; + token->f_str = static_cast(ut_malloc_nokey(str_len + 1)); + token->f_len = str_len; memcpy(token->f_str, node->term.ptr->str, str_len); - token->f_str[str_len] = '%'; token->f_str[token->f_len] = 0; new_ptr = token->f_str; diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 2aadd421db4f9..a6273161145f9 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -2662,100 +2662,62 @@ struct st_maria_plugin i_s_innodb_ft_index_cache = MariaDB_PLUGIN_MATURITY_STABLE }; -/*******************************************************************//** -Go through a FTS index auxiliary table, fetch its rows and fill +/** Go through a FTS index auxiliary table, fetch its rows and fill FTS word cache structure. +@param index FTS index +@param words vector to hold fetched words +@param selected auxiliary index +@param word word to select @return DB_SUCCESS on success, otherwise error code */ static -dberr_t -i_s_fts_index_table_fill_selected( -/*==============================*/ - dict_index_t* index, /*!< in: FTS index */ - ib_vector_t* words, /*!< in/out: vector to hold - fetched words */ - ulint selected, /*!< in: selected FTS index */ - fts_string_t* word) /*!< in: word to select */ -{ - pars_info_t* info; - fts_table_t fts_table; - trx_t* trx; - que_t* graph; - dberr_t error; - fts_fetch_t fetch; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - fetch.read_arg = words; - fetch.read_record = fts_optimize_index_fetch_node; - fetch.total_memory = 0; - - DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", - fts_result_cache_limit = 8192; - ); - - trx = trx_create(); - - trx->op_info = "fetching FTS index nodes"; - - pars_info_bind_function(info, "my_func", fetch.read_record, &fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), - FTS_INDEX_TABLE, index); - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name WHERE word >= :word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error occurred while reading" - " FTS index: " << error; - break; - } - } - } - - que_graph_free(graph); - - trx->free(); - - if (fetch.total_memory >= fts_result_cache_limit) { - error = DB_FTS_EXCEED_RESULT_CACHE_LIMIT; - } +dberr_t i_s_fts_index_table_fill_selected( + dict_index_t *index, ib_vector_t *words, ulint selected, + fts_string_t *word) +{ + dberr_t error= DB_SUCCESS; + ulint total_memory= 0; + DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", + fts_result_cache_limit = 8192;); + trx_t* trx= trx_create(); + trx->op_info= "fetching FTS index nodes"; + for (;;) + { + FTSQueryExecutor executor(trx, index, index->table); + AuxRecordReader reader(words, &total_memory); + if (word->f_str == nullptr) + error= executor.read_aux_all((uint8_t) selected, reader); + else + error= executor.read_aux( + (uint8_t) selected, + reinterpret_cast(word->f_str), + PAGE_CUR_GE, reader); - return(error); + if (UNIV_LIKELY(error == DB_SUCCESS || + error == DB_RECORD_NOT_FOUND)) + { + fts_sql_commit(trx); + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading FTS index. Retrying!"; + trx->error_state = DB_SUCCESS; + } + else + { + ib::error() << "Error occurred while reading FTS index: " << error; + break; + } + } + } + trx->free(); + if (total_memory >= fts_result_cache_limit) + error= DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + return error; } /*******************************************************************//** diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h index 111451343c6e4..b420fe74972bc 100644 --- a/storage/innobase/include/fts0exec.h +++ b/storage/innobase/include/fts0exec.h @@ -224,4 +224,83 @@ class ConfigReader : public RecordCallback std::vector> config_pairs; ConfigReader(); }; + +/** Type alias for FTS record processor function */ +using FTSRecordProcessor= std::function< + bool(const rec_t*, const dict_index_t*, const rec_offs*, void*)>; + +/** Comparison modes for AuxRecordReader */ +enum class AuxCompareMode +{ + /* >= comparison (range scan from word) */ + GREATER_EQUAL, + /* > comparison (exclude exact match) */ + GREATER, + /* LIKE pattern matching (prefix match) */ + LIKE, + /* = comparison (exact match) */ + EQUAL +}; + +/** Callback class for reading FTS auxiliary index table records */ +class AuxRecordReader : public RecordCallback +{ +private: + void* user_arg; + ulint* total_memory; + AuxCompareMode compare_mode; + +private: + /** FTS-specific record comparison logic */ + RecordCompareAction compare_record( + const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) noexcept; + +public: + /** Default word processor for FTS auxiliary table records */ + bool default_word_processor(const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg); + + /* Constructor with custom processor */ + template + AuxRecordReader(void* user_data, + ProcessorFunc proc_func, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this, proc_func](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + return proc_func(rec, index, offsets, this->user_arg); + }, + [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index, offsets); + } + ), + user_arg(user_data), total_memory(nullptr), + compare_mode(mode) {} + + /* Different constructor with default word processing */ + AuxRecordReader(void* user_data, ulint* memory_counter, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + return this->default_word_processor(rec, index, offsets, + this->user_arg); + }, + [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index, offsets); + } + ), + user_arg(user_data), total_memory(memory_counter), + compare_mode(mode) {} +}; + #endif /* INNOBASE_FTS0QUERY_H */ diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index baae089c031af..fa2aab5f06bea 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -32,6 +32,7 @@ Created 2011/09/02 Sunny Bains #include "que0que.h" #include "que0types.h" #include "fts0types.h" +#include "fts0exec.h" /* The various states of the FTS sub system pertaining to a table with FTS indexes defined on it. */ @@ -198,18 +199,15 @@ fts_query_expansion_fetch_doc( void* row, /*!< in: sel_node_t* */ void* user_arg) /*!< in: fts_doc_t* */ MY_ATTRIBUTE((nonnull)); -/******************************************************************** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: the FTS aux index */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Check if a fts token is a stopword or less than fts_min_token_size or greater than fts_max_token_size. @@ -258,19 +256,22 @@ fts_word_free( /*==========*/ fts_word_t* word) /*!< in: instance to free.*/ MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Read the rows from the FTS inde -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: FTS aux table */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ - MY_ATTRIBUTE((nonnull)); + +/** Read the rows from FTS index +@param trx transaction +@param index fulltext index +@param word word to fetch +@param user_arg user argument +@param processor custom processor to filter the word from record +@param compare_mode comparison mode for record matching +@return error code or DB_SUCCESS */ +dberr_t fts_index_fetch_nodes(trx_t *trx, dict_index_t *index, + const fts_string_t *word, + void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) + MY_ATTRIBUTE((nonnull)); + #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 8db88bfc1765f..53eec73629f61 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -315,7 +315,7 @@ fts_get_suffix( @param[in] len string length in bytes @return the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index c4e7bfed7ffde..421331fcdaf05 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -102,13 +102,13 @@ inline bool fts_is_charset_cjk(const CHARSET_INFO* cs) @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_range( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected = 0; + uint8_t selected = 0; ulint value = innobase_strnxfrm(cs, str, len); while (fts_index_selector[selected].value != 0) { @@ -136,7 +136,7 @@ fts_select_index_by_range( @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_hash( const CHARSET_INFO* cs, const byte* str, @@ -172,21 +172,17 @@ fts_select_index_by_hash( @param[in] len string length in bytes @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected; - if (fts_is_charset_cjk(cs)) { - selected = fts_select_index_by_hash(cs, str, len); - } else { - selected = fts_select_index_by_range(cs, str, len); + return fts_select_index_by_hash(cs, str, len); } - return(selected); + return fts_select_index_by_range(cs, str, len); } /******************************************************************//** From 23883ca438c643ad16483174bd08fbfc37fcd0d1 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 17 Nov 2025 12:23:21 +0530 Subject: [PATCH 06/12] MDEV-28730 Remove internal parser usage from InnoDB fts Refactor fetch, optimize to QueryExecutor and standardize processor API. Replaced legacy SQL-graph paths with QueryExecutor-based reads/writes: fts_query code now uses QueryExecutor::read(), read_by_index() with RecordCallback (updating fts_query_match_document(), fts_query_is_in_proximity_range(), and fts_expand_query() to call fts_query_fetch_document() instead of fts_doc_fetch_by_doc_id(), which was removed along with FTS_FETCH_DOC_BY_DOC_ID_* macros); Rewrote fts_optimize_write_word() to delete (or) insert via FTSQueryExecutor::delete_aux_record()/insert_aux_record() using fts_aux_data_t; --- storage/innobase/fts/fts0fts.cc | 908 +++++++++++----------------- storage/innobase/fts/fts0que.cc | 330 ++++++---- storage/innobase/include/fts0fts.h | 33 +- storage/innobase/include/fts0priv.h | 26 - 4 files changed, 595 insertions(+), 702 deletions(-) diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index c99123cd8c6ec..f99354ea7c7d7 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -45,6 +45,37 @@ static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ #define FTS_MAX_CACHE_SIZE_IN_MB "cache_size_in_mb" +/** Compare function to check if record's doc_id > search tuple's doc_id +@param[in] search_tuple Search tuple containing target doc_id +@param[in] rec Record to check +@param[in] index Index containing the record +@param[in] offsets Record offsets +@return true if record's doc_id > search tuple's doc_id */ +static +RecordCompareAction doc_id_comparator( + const dtuple_t* search_tuple, + const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) +{ + /* Get target doc_id from search tuple */ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + /* Get doc_id from record */ + ulint len; + const byte* rec_data= rec_get_nth_field(rec, offsets, 0, &len); + + if (len != sizeof(doc_id_t)) + return RecordCompareAction::SKIP; + + doc_id_t rec_doc_id= fts_read_doc_id(rec_data); + return (rec_doc_id > target_doc_id) + ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + /** Verify if a aux table name is a obsolete table by looking up the key word in the obsolete table names */ #define FTS_IS_OBSOLETE_AUX_TABLE(table_name) \ @@ -205,30 +236,6 @@ fts_add_doc_by_id( fts_trx_table_t*ftt, /*!< in: FTS trx table */ doc_id_t doc_id); /*!< in: doc id */ -/** Tokenize a document. -@param[in,out] doc document to tokenize -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document( - fts_doc_t* doc, - fts_doc_t* result, - st_mysql_ftparser* parser); - -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document_next( - fts_doc_t* doc, - ulint add_pos, - fts_doc_t* result, - st_mysql_ftparser* parser); - /** Create the vector of fts_get_doc_t instances. @param[in,out] cache fts cache @return vector of fts_get_doc_t instances */ @@ -263,7 +270,6 @@ fts_cache_destroy(fts_cache_t* cache) /** Get a character set based on precise type. @param prtype precise type @return the corresponding character set */ -UNIV_INLINE CHARSET_INFO* fts_get_charset(ulint prtype) { @@ -340,187 +346,154 @@ fts_load_default_stopword( stopword_info->status = STOPWORD_FROM_DEFAULT; } -/****************************************************************//** -Callback function to read a single stopword value. -@return Always return TRUE */ -static -ibool -fts_read_stopword( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - ib_alloc_t* allocator; - fts_stopword_t* stopword_info; - sel_node_t* sel_node; - que_node_t* exp; - ib_rbt_t* stop_words; - dfield_t* dfield; - fts_string_t str; - mem_heap_t* heap; - ib_rbt_bound_t parent; - dict_table_t* table; - - sel_node = static_cast(row); - table = sel_node->table_list->table; - stopword_info = static_cast(user_arg); - - stop_words = stopword_info->cached_stopword; - allocator = static_cast(stopword_info->heap); - heap = static_cast(allocator->arg); - - exp = sel_node->select_list; - - /* We only need to read the first column */ - dfield = que_node_get_val(exp); - - str.f_n_char = 0; - str.f_str = static_cast(dfield_get_data(dfield)); - str.f_len = dfield_get_len(dfield); - exp = que_node_get_next(exp); - ut_ad(exp); - - if (table->versioned()) { - dfield = que_node_get_val(exp); - ut_ad(dfield_get_type(dfield)->vers_sys_end()); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - } - ut_ad(!que_node_get_next(exp)); - - /* Only create new node if it is a value not already existed */ - if (str.f_len != UNIV_SQL_NULL - && rbt_search(stop_words, &parent, &str) != 0) { - - fts_tokenizer_word_t new_word; - - new_word.nodes = ib_vector_create( - allocator, sizeof(fts_node_t), 4); - - new_word.text.f_str = static_cast( - mem_heap_alloc(heap, str.f_len + 1)); - - memcpy(new_word.text.f_str, str.f_str, str.f_len); - - new_word.text.f_n_char = 0; - new_word.text.f_len = str.f_len; - new_word.text.f_str[str.f_len] = 0; - - rbt_insert(stop_words, &new_word, &new_word); - } - - return(TRUE); -} - -/******************************************************************//** -Load user defined stopword from designated user table +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information @return whether the operation is successful */ static -bool -fts_load_user_stopword( -/*===================*/ - fts_t* fts, /*!< in: FTS struct */ - const char* stopword_table_name, /*!< in: Stopword table - name */ - fts_stopword_t* stopword_info) /*!< in: Stopword info */ -{ - if (!fts->dict_locked) { - dict_sys.lock(SRW_LOCK_CALL); - } - - /* Validate the user table existence in the right format */ - bool ret= false; - const char* row_end; - stopword_info->charset = fts_valid_stopword_table(stopword_table_name, - &row_end); - if (!stopword_info->charset) { +bool fts_load_user_stopword(fts_t *fts, const char *stopword_table, + fts_stopword_t *stopword_info) +{ + if (!fts->dict_locked) dict_sys.lock(SRW_LOCK_CALL); + /* Validate the user table existence in the right format */ + bool ret= false; + const char* row_end; + stopword_info->charset= fts_valid_stopword_table( + stopword_table, &row_end); + if (!stopword_info->charset) + { cleanup: - if (!fts->dict_locked) { - dict_sys.unlock(); - } - - return ret; - } + if (!fts->dict_locked) dict_sys.unlock(); + return ret; + } - trx_t* trx = trx_create(); - trx->op_info = "Load user stopword table into FTS cache"; + if (!stopword_info->cached_stopword) + { + /* Create the stopword RB tree with the stopword column + charset. All comparison will use this charset */ + stopword_info->cached_stopword= rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + (void*)stopword_info->charset); + } - if (!stopword_info->cached_stopword) { - /* Create the stopword RB tree with the stopword column - charset. All comparison will use this charset */ - stopword_info->cached_stopword = rbt_create_arg_cmp( - sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, - (void*)stopword_info->charset); + /* Load the stopword table */ + dict_table_t* table= + dict_sys.load_table({stopword_table, strlen(stopword_table)}); + if (!table) + goto cleanup; - } + trx_t *trx= trx_create(); + trx->op_info= "Load user stopword table into FTS cache"; - pars_info_t* info = pars_info_create(); + QueryExecutor executor(trx); + ib_rbt_t* stop_words= stopword_info->cached_stopword; + ib_alloc_t* allocator= static_cast(stopword_info->heap); + mem_heap_t* heap= static_cast(allocator->arg); - pars_info_bind_id(info, "table_stopword", stopword_table_name); - pars_info_bind_id(info, "row_end", row_end); + /* Find the field number for 'value' column */ + dict_index_t* clust_index= dict_table_get_first_index(table); + ulint value_field_no= ULINT_UNDEFINED; + for (ulint i= 0; i < dict_index_get_n_fields(clust_index); i++) + { + const dict_field_t* field= dict_index_get_nth_field(clust_index, i); + if (strcmp(field->name, "value") == 0) + { + value_field_no= i; + break; + } + } + if (value_field_no == ULINT_UNDEFINED) + { + ib::error() << "Could not find 'value' column in stopword table " + << stopword_table; + goto cleanup; + } - pars_info_bind_function(info, "my_func", fts_read_stopword, - stopword_info); + auto process_stopword= [&](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, value_field_no, + &field_len); - que_t* graph = pars_sql( - info, - "PROCEDURE P() IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT value, $row_end" - " FROM $table_stopword;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;" - "END;\n"); + if (field_len == UNIV_SQL_NULL) return true; - for (;;) { - dberr_t error = fts_eval_sql(trx, graph); + fts_string_t str; + str.f_n_char= 0; + str.f_str= const_cast(field_data); + str.f_len= field_len; - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - stopword_info->status = STOPWORD_USER_TABLE; - break; - } else { - fts_sql_rollback(trx); + /* Handle system versioning - check row_end column if versioned */ + if (table->versioned()) + { + ulint end_len; + const byte* end_data= rec_get_nth_field( + rec, offsets, table->vers_end + index->n_uniq + 2, &end_len); - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading user" - " stopword table. Retrying!"; + if (table->versioned_by_id()) + { + ut_ad(end_len == sizeof trx_id_max_bytes); + if (0 != memcmp(end_data, trx_id_max_bytes, end_len)) + return true; /* Skip this version, continue processing */ + } + else + { + ut_ad(end_len == sizeof timestamp_max_bytes); + if (!IS_MAX_TIMESTAMP(end_data)) + return true; /* Skip this version, continue processing */ + } + } - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error '" << error - << "' while reading user stopword" - " table."; - ret = FALSE; - break; - } - } - } + ib_rbt_bound_t parent; + if (str.f_len != UNIV_SQL_NULL && + rbt_search(stop_words, &parent, &str) != 0) + { + fts_tokenizer_word_t new_word; + new_word.nodes= ib_vector_create(allocator, sizeof(fts_node_t), 4); + new_word.text.f_str= static_cast( + mem_heap_alloc(heap, str.f_len + 1)); + memcpy(new_word.text.f_str, str.f_str, str.f_len); + new_word.text.f_n_char= 0; + new_word.text.f_len= str.f_len; + new_word.text.f_str[str.f_len]= 0; + rbt_insert(stop_words, &new_word, &new_word); + } + return true; /* Continue processing */ + }; - que_graph_free(graph); - trx->free(); - ret = true; - goto cleanup; + RecordCallback callback(process_stopword); + /* Read all records from the stopword table */ + for (;;) + { + dberr_t error= executor.read(table, nullptr, PAGE_CUR_G, callback); + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + fts_sql_commit(trx); + stopword_info->status= STOPWORD_USER_TABLE; + ret= true; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading user" + " stopword table. Retrying!"; + trx->error_state= DB_SUCCESS; + } + else + { + ib::error() << "Error '" << error + << "' while reading user stopword table."; + ret= false; + break; + } + } + } + trx->free(); + goto cleanup; } /******************************************************************//** @@ -3449,28 +3422,6 @@ fts_add_doc_by_id( mem_heap_free(heap); } - -/*********************************************************************//** -Callback function to read a single ulint column. -return always returns TRUE */ -static -ibool -fts_read_ulint( -/*===========*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ulint */ -{ - sel_node_t* sel_node = static_cast(row); - ulint* value = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - - *value = mach_read_from_4(static_cast(data)); - - return(TRUE); -} - /*********************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -3547,130 +3498,6 @@ fts_get_max_doc_id( return(doc_id); } -/*********************************************************************//** -Fetch document with the given document id. -@return DB_SUCCESS if OK else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to - fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read */ - void* arg) /*!< in: callback arg */ -{ - pars_info_t* info; - dberr_t error; - const char* select_str; - doc_id_t write_doc_id; - dict_index_t* index; - trx_t* trx = trx_create(); - que_t* graph; - - trx->op_info = "fetching indexed FTS document"; - - /* The FTS index can be supplied by caller directly with - "index_to_use", otherwise, get it from "get_doc" */ - index = (index_to_use) ? index_to_use : get_doc->index_cache->index; - - if (get_doc && get_doc->get_document_graph) { - info = get_doc->get_document_graph->info; - } else { - info = pars_info_create(); - } - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - pars_info_bind_function(info, "my_func", callback, arg); - - select_str = fts_get_select_columns_str(index, info, info->heap); - pars_info_bind_id(info, "table_name", index->table->name.m_name); - - if (!get_doc || !get_doc->get_document_graph) { - if (option == FTS_FETCH_DOC_BY_ID_EQUAL) { - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s FROM $table_name" - " WHERE %s = :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - select_str, - FTS_DOC_ID.str)); - } else { - ut_ad(option == FTS_FETCH_DOC_BY_ID_LARGE); - - /* This is used for crash recovery of table with - hidden DOC ID or FTS indexes. We will scan the table - to re-processing user table rows whose DOC ID or - FTS indexed documents have not been sync-ed to disc - during recent crash. - In the case that all fulltext indexes are dropped - for a table, we will keep the "hidden" FTS_DOC_ID - column, and this scan is to retreive the largest - DOC ID being used in the table to determine the - appropriate next DOC ID. - In the case of there exists fulltext index(es), this - operation will re-tokenize any docs that have not - been sync-ed to the disk, and re-prime the FTS - cached */ - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s, %s FROM $table_name" - " WHERE %s > :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - FTS_DOC_ID.str, - select_str, - FTS_DOC_ID.str)); - } - if (get_doc) { - get_doc->get_document_graph = graph; - } - } else { - graph = get_doc->get_document_graph; - } - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - trx->free(); - - if (!get_doc) { - que_graph_free(graph); - } - - return(error); -} - /** Write out a single word's data as new entry/entries in the INDEX table. @param executor FTS Query Executor @param selected auxiliary index number @@ -4423,7 +4250,6 @@ fts_tokenize_by_parser( @param[in,out] doc document to tokenize @param[out] result tokenization result @param[in] parser pluggable parser */ -static void fts_tokenize_document( fts_doc_t* doc, @@ -4453,12 +4279,6 @@ fts_tokenize_document( } } -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static void fts_tokenize_document_next( fts_doc_t* doc, @@ -4625,81 +4445,6 @@ fts_is_index_updated( } #endif -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table) /*!< in: fts table to read */ -{ - trx_t* trx; - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint count = 0; - char table_name[MAX_FULL_NAME_LEN]; - - trx = trx_create(); - trx->op_info = "fetching FT table rows count"; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_read_ulint, &count); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT COUNT(*)" - " FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS table " - << table_name; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - trx->free(); - - return(count); -} - #ifdef FTS_CACHE_SIZE_DEBUG /*********************************************************************//** Read the max cache size parameter from the config table. */ @@ -5664,168 +5409,203 @@ fts_load_stopword( return error == DB_SUCCESS; } -/**********************************************************************//** -Callback function when we initialize the FTS at the start up -time. It recovers the maximum Doc IDs presented in the current table. -Tested by innodb_fts.crash_recovery -@return: always returns TRUE */ -static -ibool -fts_init_get_doc_id( -/*================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: table with fts */ -{ - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - dict_table_t* table = static_cast(user_arg); - fts_cache_t* cache = table->fts->cache; - - ut_ad(ib_vector_is_empty(cache->get_docs)); - - /* Copy each indexed column content into doc->text.f_str */ - if (exp) { - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); - - exp = que_node_get_next(que_node_get_next(exp)); - if (exp) { - ut_ad(table->versioned()); - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_ad(type->vers_sys_end()); - data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - ut_ad(!(exp = que_node_get_next(exp))); - } - ut_ad(!exp); - - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } - } - - return(TRUE); -} - -/**********************************************************************//** -Callback function when we initialize the FTS at the start up +/** Callback function when we initialize the FTS at the start up time. It recovers Doc IDs that have not sync-ed to the auxiliary table, and require to bring them back into FTS index. +@param get_doc Document +@param doc_id document id to be fetched @return: always returns TRUE */ -static -ibool -fts_init_recover_doc( -/*=================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts cache */ -{ - - fts_doc_t doc; - ulint doc_len = 0; - ulint field_no = 0; - fts_get_doc_t* get_doc = static_cast(user_arg); - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - fts_cache_t* cache = get_doc->cache; - st_mysql_ftparser* parser = get_doc->index_cache->index->parser; - - fts_doc_init(&doc); - doc.found = TRUE; - - ut_ad(cache); - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - ulint len = dfield_get_len(dfield); - - if (field_no == 0) { - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); +static void fts_init_recover_all_docs(fts_get_doc_t *get_doc, + doc_id_t doc_id) +{ + trx_t *trx= trx_create(); + trx->op_info= "fetching indexed FTS document"; + dict_index_t *fts_index= get_doc->index_cache->index; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + fts_cache_t *cache= get_doc->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + QueryExecutor executor(trx); + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } - doc_id = static_cast(mach_read_from_8( - static_cast(data))); + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_recovery= [get_doc, cache, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> bool + { + fts_doc_t doc; + ulint doc_len= 0; + doc_id_t doc_id= FTS_NULL_DOC_ID; + st_mysql_ftparser* parser= fts_index->parser; - field_no++; - exp = que_node_get_next(exp); - continue; - } + fts_doc_init(&doc); + doc.found= TRUE; - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - ut_ad(get_doc); + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - if (!get_doc->index_cache->charset) { - get_doc->index_cache->charset = fts_get_charset( - dfield->type.prtype); - } + if (len == sizeof(doc_id_t)) + { + doc_id= fts_read_doc_id(doc_id_data); - doc.charset = get_doc->index_cache->charset; + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, + &field_len); + if (field_len == UNIV_SQL_NULL) + continue; + if (!get_doc->index_cache->charset) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + get_doc->index_cache->charset= fts_get_charset(fts_field->col->prtype); + } + doc.charset= get_doc->index_cache->charset; + + /* Handle externally stored fields */ + if (rec_offs_nth_extern(offsets, col_pos)) + doc.text.f_str= btr_copy_externally_stored_field( + &doc.text.f_len, const_cast(field_data), + user_table->space->zip_size(), field_len, + static_cast(doc.self_heap->arg)); + else + { + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + } - if (dfield_is_ext(dfield)) { - dict_table_t* table = cache->sync->table; + if (i == 0) fts_tokenize_document(&doc, NULL, parser); + else fts_tokenize_document_next(&doc, doc_len, NULL, parser); - doc.text.f_str = btr_copy_externally_stored_field( - &doc.text.f_len, - static_cast(dfield_get_data(dfield)), - table->space->zip_size(), len, - static_cast(doc.self_heap->arg)); - } else { - doc.text.f_str = static_cast( - dfield_get_data(dfield)); + doc_len+= + (i < get_doc->index_cache->index->n_user_defined_cols - 1) + ? field_len + 1 + : field_len; + } - doc.text.f_len = len; - } + fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + fts_doc_free(&doc); + cache->added++; - if (field_no == 1) { - fts_tokenize_document(&doc, NULL, parser); - } else { - fts_tokenize_document_next(&doc, doc_len, NULL, parser); - } + if (doc_id >= cache->next_doc_id) + cache->next_doc_id= doc_id + 1; + } - exp = que_node_get_next(exp); + return true; + }; + RecordCallback reader(process_doc_recovery, doc_id_comparator); + if (fts_doc_id_index == clust_index) + executor.read(user_table, &search_tuple, PAGE_CUR_G, reader); + else + executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_G, reader); + trx_commit_for_mysql(trx); + trx->free(); +} - doc_len += (exp) ? len + 1 : len; +/** Get the next large document id and update it in fulltext cache +@param doc_id document id to be updated +@param index fulltext index */ +static void fts_init_get_doc_id(doc_id_t doc_id, dict_index_t *index) +{ + trx_t* trx= trx_create(); + trx->op_info= "fetching indexed FTS document"; + dict_table_t* user_table= index->table; + fts_cache_t* cache= user_table->fts->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); - field_no++; - } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_id= [cache, user_table](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - fts_doc_free(&doc); + if (len == sizeof(doc_id_t)) + { + doc_id_t found_doc_id= mach_read_from_8(doc_id_data); + if (user_table->versioned()) + { + ulint vers_end_pos= dict_col_get_index_pos( + &user_table->cols[user_table->vers_end], index); + ulint vers_len; + const byte* vers_data= rec_get_nth_field(rec, offsets, + vers_end_pos, &vers_len); - cache->added++; + if (user_table->versioned_by_id()) + { + if (vers_len == sizeof(trx_id_max_bytes) && + memcmp(vers_data, trx_id_max_bytes, vers_len) != 0) + return true; + } + else + { + if (vers_len == sizeof(timestamp_max_bytes) && + !IS_MAX_TIMESTAMP(vers_data)) + return true; + } + } - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } + /* Update cache->next_doc_id if this doc_id is larger */ + if (found_doc_id >= cache->next_doc_id) + cache->next_doc_id= found_doc_id + 1; + } + return true; + }; - return(TRUE); + RecordCallback reader(process_doc_id, doc_id_comparator); + QueryExecutor executor(trx); + if (dict_index_is_clust(index)) + executor.read(user_table, &search_tuple, PAGE_CUR_G, reader); + else + executor.read_by_index(user_table, index, &search_tuple, + PAGE_CUR_G, reader); + trx_commit_for_mysql(trx); + trx->free(); } /**********************************************************************//** @@ -5888,9 +5668,7 @@ fts_init_index( ut_a(index); - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_get_doc_id, table); + fts_init_get_doc_id(start_doc, index); } else { if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { @@ -5903,9 +5681,7 @@ fts_init_index( index = get_doc->index_cache->index; - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_recover_doc, get_doc); + fts_init_recover_all_docs(get_doc, start_doc); } } diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 729b1538a38b9..da1c124a1d002 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -434,6 +434,27 @@ static bool node_query_processor( } return true; } + +/* Comparator that signals how to treat the current record */ +RecordCompareAction doc_id_exact_match_comparator( + const dtuple_t* search_tuple, const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) +{ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= + static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + ulint len; + const byte* rec_data= rec_get_nth_field(rec, offsets, 0, &len); + if (len != sizeof(doc_id_t)) + return RecordCompareAction::STOP; + doc_id_t rec_doc_id= fts_read_doc_id(rec_data); + return rec_doc_id == target_doc_id + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; +} + #if 0 /*****************************************************************//*** Find a doc_id in a word's ilist. @@ -2068,116 +2089,231 @@ fts_query_match_phrase( return(phrase->found); } -/*****************************************************************//** -Callback function to fetch and search the document. +/** Callback function to fetch and search the document. +@param fts_index fulltext index +@param doc_id document id +@param arg user argument +@param expansion Expansion document @return whether the phrase is found */ static -ibool -fts_query_fetch_document( -/*=====================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ +dberr_t fts_query_fetch_document(dict_index_t *fts_index, + doc_id_t doc_id, + void *arg, bool expansion= false) { + trx_t *trx= trx_create(); + trx->op_info= "fetching FTS document for query"; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + + QueryExecutor executor(trx); + + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_expansion_doc= [arg, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t *index, + const rec_offs *offsets)-> bool + { + fts_doc_t *result_doc= static_cast(arg); + fts_doc_t doc; + CHARSET_INFO *doc_charset= result_doc->charset; + fts_doc_init(&doc); + doc.found= TRUE; - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_phrase_t* phrase = static_cast(user_arg); - ulint prev_len = 0; - ulint total_len = 0; - byte* document_text = NULL; + ulint doc_len= 0; + ulint field_no= 0; - exp = node->select_list; + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + + /* NULL column */ + if (field_len == UNIV_SQL_NULL) { + continue; + } - phrase->found = FALSE; + /* Determine document charset from column if not provided */ + if (!doc_charset) + { + const dict_field_t* ifield= dict_index_get_nth_field(fts_index, i); + doc_charset= fts_get_charset(ifield->col->prtype); + } - /* For proximity search, we will need to get the whole document - from all fields, so first count the total length of the document - from all the fields */ - if (phrase->proximity_pos) { - while (exp) { - ulint field_len; - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); + doc.charset= doc_charset; + /* Skip columns stored externally, as in fts_query_expansion_fetch_doc */ + if (rec_offs_nth_extern(offsets, col_pos)) { + continue; + } - if (dfield_is_ext(dfield)) { - ulint local_len = dfield_get_len(dfield); + /* Use inline field data */ + doc.text.f_n_char= 0; + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; - local_len -= BTR_EXTERN_FIELD_REF_SIZE; + if (field_no == 0) + fts_tokenize_document(&doc, result_doc, result_doc->parser); + else + fts_tokenize_document_next(&doc, doc_len, result_doc, + result_doc->parser); - field_len = mach_read_from_4( - data + local_len + BTR_EXTERN_LEN + 4); - } else { - field_len = dfield_get_len(dfield); - } + /* Next field offset: add 1 for separator if more fields follow */ + doc_len+= ((i + 1) < fts_index->n_user_defined_cols) + ? field_len + 1 + : field_len; - if (field_len != UNIV_SQL_NULL) { - total_len += field_len + 1; - } + field_no++; + } - exp = que_node_get_next(exp); - } + ut_ad(doc_charset); + if (!result_doc->charset) { + result_doc->charset= doc_charset; + } - document_text = static_cast(mem_heap_zalloc( - phrase->heap, total_len)); + fts_doc_free(&doc); - if (!document_text) { - return(FALSE); - } - } + return true; /* continue */ + }; - exp = node->select_list; + auto process_doc_query= [arg, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint prev_len= 0; + ulint total_len= 0; + byte *document_text= nullptr; - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint cur_len; + fts_phrase_t *phrase= static_cast(arg); + phrase->found= FALSE; - if (dfield_is_ext(dfield)) { - data = btr_copy_externally_stored_field( - &cur_len, data, phrase->zip_size, - dfield_get_len(dfield), phrase->heap); - } else { - cur_len = dfield_get_len(dfield); - } + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - if (cur_len != UNIV_SQL_NULL && cur_len != 0) { - if (phrase->proximity_pos) { - ut_ad(prev_len + cur_len <= total_len); - memcpy(document_text + prev_len, data, cur_len); - } else { - /* For phrase search */ - phrase->found = - fts_query_match_phrase( - phrase, - static_cast(data), - cur_len, prev_len, - phrase->heap); - } + ulint len; + rec_get_nth_field(rec, offsets, doc_col_pos, &len); + if (len != sizeof(doc_id_t)) + return true; - /* Document positions are calculated from the beginning - of the first field, need to save the length for each - searched field to adjust the doc position when search - phrases. */ - prev_len += cur_len + 1; - } + /* For proximity search, first count total document length */ + if (phrase->proximity_pos) + { + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + if (rec_offs_nth_extern(offsets, col_pos)) + { + ulint local_len= field_len; + local_len-= BTR_EXTERN_FIELD_REF_SIZE; + field_len= mach_read_from_4( + field_data + local_len + BTR_EXTERN_LEN + 4); + } + if (field_len != UNIV_SQL_NULL) + total_len+= field_len + 1; + } - if (phrase->found) { - break; - } + document_text= + static_cast(mem_heap_zalloc(phrase->heap, total_len)); + if (!document_text) + return false; + } - exp = que_node_get_next(exp); - } + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + byte* data= const_cast(field_data); + ulint cur_len; + + if (rec_offs_nth_extern(offsets, col_pos)) + { + data= btr_copy_externally_stored_field( + &cur_len, const_cast(field_data), phrase->zip_size, + field_len, phrase->heap); + } + else cur_len= field_len; + if (cur_len != UNIV_SQL_NULL && cur_len != 0) + { + if (phrase->proximity_pos) + { + ut_ad(prev_len + cur_len <= total_len); + memcpy(document_text + prev_len, data, cur_len); + } + else + { + /* For phrase search */ + phrase->found= fts_query_match_phrase( + phrase, data, cur_len, prev_len, phrase->heap); + } - if (phrase->proximity_pos) { - ut_ad(prev_len <= total_len); + /* Document positions are calculated from the beginning + of the first field, need to save the length for each + searched field to adjust the doc position when search + phrases. */ + prev_len+= cur_len + 1; + } - phrase->found = fts_proximity_is_word_in_range( - phrase, document_text, total_len); - } + if (phrase->found) + break; + } - return(phrase->found); + if (phrase->proximity_pos) + { + ut_ad(prev_len <= total_len); + phrase->found= fts_proximity_is_word_in_range( + phrase, document_text, total_len); + } + + return !phrase->found; /* Continue only if not found */ + }; + + RecordProcessor proc= expansion + ? RecordProcessor(process_expansion_doc) + : RecordProcessor(process_doc_query); + RecordCallback reader(proc, doc_id_exact_match_comparator); + dberr_t err= DB_SUCCESS; + if (fts_doc_id_index == clust_index) + err= executor.read(user_table, &search_tuple, PAGE_CUR_GE, reader); + else + err= executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_GE, reader); + trx_commit_for_mysql(trx); + trx->free(); + if (err == DB_RECORD_NOT_FOUND) err= DB_SUCCESS; + return err; } #if 0 @@ -2566,9 +2702,8 @@ fts_query_match_document( *found = phrase.found = FALSE; - error = fts_doc_fetch_by_doc_id( - get_doc, match->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + error = fts_query_fetch_document( + get_doc->index_cache->index, match->doc_id, &phrase); if (UNIV_UNLIKELY(error != DB_SUCCESS)) { ib::error() << "(" << error << ") matching document."; @@ -2613,21 +2748,14 @@ fts_query_is_in_proximity_range( phrase.proximity_pos = qualified_pos; phrase.found = FALSE; - err = fts_doc_fetch_by_doc_id( - &get_doc, match[0]->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + err = fts_query_fetch_document( + get_doc.index_cache->index, match[0]->doc_id, &phrase); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { ib::error() << "(" << err << ") in verification" " phase of proximity search"; } - /* Free the prepared statement. */ - if (get_doc.get_document_graph) { - que_graph_free(get_doc.get_document_graph); - get_doc.get_document_graph = NULL; - } - mem_heap_free(phrase.heap); return(err == DB_SUCCESS && phrase.found); @@ -4414,10 +4542,8 @@ fts_expand_query( fetch the original document and parse them. Future optimization could be done here if we support some forms of document-to-word mapping */ - fts_doc_fetch_by_doc_id(NULL, ranking->doc_id, index, - FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_expansion_fetch_doc, - &result_doc); + fts_query_fetch_document(index, ranking->doc_id, + &result_doc, true); /* Estimate memory used, see fts_process_token and fts_token_t. We ignore token size here. */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 1145c5b55166a..4c1a7ffd4d466 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -784,14 +784,6 @@ fts_tokenize_document_internal( const char* doc, /*!< in: document to tokenize */ int len); /*!< in: document length */ -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table); /*!< in: fts table to read */ - /*************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -936,3 +928,28 @@ fts_update_sync_doc_id(const dict_table_t *table, /** Sync the table during commit phase @param[in] table table to be synced */ void fts_sync_during_ddl(dict_table_t* table); + +/** Tokenize a document. +@param[in,out] doc document to tokenize +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document( + fts_doc_t* doc, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Continue to tokenize a document. +@param[in,out] doc document to tokenize +@param[in] add_pos add this position to all tokens from this tokenization +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document_next( + fts_doc_t* doc, + ulint add_pos, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Get a character set based on precise type. +@param prtype precise type +@return the corresponding character set */ +CHARSET_INFO* fts_get_charset(ulint prtype); diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index fa2aab5f06bea..7263536bddc46 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -163,32 +163,6 @@ fts_get_select_columns_str( mem_heap_t* heap) /*!< in: memory heap */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -/** define for fts_doc_fetch_by_doc_id() "option" value, defines whether -we want to get Doc whose ID is equal to or greater or smaller than supplied -ID */ -#define FTS_FETCH_DOC_BY_ID_EQUAL 1 -#define FTS_FETCH_DOC_BY_ID_LARGE 2 -#define FTS_FETCH_DOC_BY_ID_SMALL 3 - -/*************************************************************//** -Fetch document (= a single row's indexed text) with the given -document id. -@return: DB_SUCCESS if fetch is successful, else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read - records */ - void* arg) /*!< in: callback arg */ - MY_ATTRIBUTE((nonnull(6))); - /*******************************************************************//** Callback function for fetch that stores the text of an FTS document, converting each column to UTF-16. From edb947ef0916f6218c9872f6b19492268c76fbee Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 17 Nov 2025 12:57:36 +0530 Subject: [PATCH 07/12] MDEV-28730 Remove internal parser usage from InnoDB fts - Removed fts0sql.cc file. - Removed commented fts funtions - Removed fts_table_t from fts_query_t and fts_optimize_t --- storage/innobase/CMakeLists.txt | 1 - storage/innobase/fts/fts0fts.cc | 133 ++----- storage/innobase/fts/fts0opt.cc | 20 +- storage/innobase/fts/fts0que.cc | 569 +--------------------------- storage/innobase/fts/fts0sql.cc | 208 ---------- storage/innobase/handler/i_s.cc | 5 - storage/innobase/include/fts0priv.h | 53 --- 7 files changed, 48 insertions(+), 941 deletions(-) delete mode 100644 storage/innobase/fts/fts0sql.cc diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index a9852207ab359..63273ce293dbf 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -175,7 +175,6 @@ SET(INNOBASE_SOURCES fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc - fts/fts0sql.cc fts/fts0tlex.cc gis/gis0geo.cc gis/gis0rtree.cc diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index f99354ea7c7d7..5c248f399bb41 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -557,6 +557,46 @@ fts_cache_init( } } +/** Construct the name of an internal FTS table for the given table. +@param[in] fts_table metadata on fulltext-indexed table +@param[out] table_name a name up to MAX_FULL_NAME_LEN +@param[in] dict_locked whether dict_sys.latch is being held */ +void fts_get_table_name(const fts_table_t* fts_table, char* table_name, + bool dict_locked) +{ + if (!dict_locked) dict_sys.freeze(SRW_LOCK_CALL); + ut_ad(dict_sys.frozen()); + /* Include the separator as well. */ + const size_t dbname_len= fts_table->table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, fts_table->table->name.m_name, dbname_len); + if (!dict_locked) dict_sys.unfreeze(); + + memcpy(table_name += dbname_len, "FTS_", 4); + table_name += 4; + int len; + switch (fts_table->type) + { + case FTS_COMMON_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + break; + + case FTS_INDEX_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(fts_table->index_id, table_name + len); + break; + + default: ut_error; + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, fts_table->suffix); +} + /****************************************************************//** Create a FTS cache. */ fts_cache_t* @@ -2690,7 +2730,6 @@ fts_delete( fts_trx_table_t*ftt, /*!< in: FTS trx table */ fts_trx_row_t* row) /*!< in: row */ { - fts_table_t fts_table; dict_table_t* table = ftt->table; doc_id_t doc_id = row->doc_id; trx_t* trx = ftt->fts_trx->trx; @@ -2704,8 +2743,6 @@ fts_delete( ut_a(row->state == FTS_DELETE || row->state == FTS_MODIFY); - FTS_INIT_FTS_TABLE(&fts_table, "DELETED", FTS_COMMON_TABLE, table); - /* It is possible we update a record that has not yet been sync-ed into cache from last crash (delete Doc will not initialize the sync). Avoid any added counter accounting until the FTS cache @@ -2899,93 +2936,6 @@ fts_doc_free( mem_heap_free(heap); } -/*********************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_doc_t* result_doc = static_cast(user_arg); - dfield_t* dfield; - ulint len; - ulint doc_len; - fts_doc_t doc; - CHARSET_INFO* doc_charset = NULL; - ulint field_no = 0; - - len = 0; - - fts_doc_init(&doc); - doc.found = TRUE; - - exp = node->select_list; - doc_len = 0; - - doc_charset = result_doc->charset; - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield = que_node_get_val(exp); - len = dfield_get_len(dfield); - - /* NULL column */ - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } - - if (!doc_charset) { - doc_charset = fts_get_charset(dfield->type.prtype); - } - - doc.charset = doc_charset; - - if (dfield_is_ext(dfield)) { - /* We ignore columns that are stored externally, this - could result in too many words to search */ - exp = que_node_get_next(exp); - continue; - } else { - doc.text.f_n_char = 0; - - doc.text.f_str = static_cast( - dfield_get_data(dfield)); - - doc.text.f_len = len; - } - - if (field_no == 0) { - fts_tokenize_document(&doc, result_doc, - result_doc->parser); - } else { - fts_tokenize_document_next(&doc, doc_len, result_doc, - result_doc->parser); - } - - exp = que_node_get_next(exp); - - doc_len += (exp) ? len + 1 : len; - - field_no++; - } - - ut_ad(doc_charset); - - if (!result_doc->charset) { - result_doc->charset = doc_charset; - } - - fts_doc_free(&doc); - - return(FALSE); -} - /*********************************************************************//** fetch and tokenize the document. */ static @@ -4455,12 +4405,11 @@ fts_update_max_cache_size( fts_sync_t* sync) /*!< in: sync state */ { trx_t* trx; - fts_table_t fts_table; trx = trx_create(); /* The size returned is in bytes. */ - sync->max_cache_size = fts_get_max_cache_size(trx, &fts_table); + sync->max_cache_size = fts_get_max_cache_size(trx, sync->table); fts_sql_commit(trx); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 35884b669bf5c..05844cdd6f361 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -138,11 +138,6 @@ struct fts_optimize_t { char* name_prefix; /*!< FTS table name prefix */ - fts_table_t fts_index_table;/*!< Common table definition */ - - /*!< Common table definition */ - fts_table_t fts_common_table; - dict_table_t* table; /*!< Table that has to be queried */ dict_index_t* index; /*!< The FTS index to be optimized */ @@ -1230,18 +1225,10 @@ fts_optimize_create( optim->trx = trx_create(); trx_start_internal(optim->trx); - optim->fts_common_table.table_id = table->id; - optim->fts_common_table.type = FTS_COMMON_TABLE; - optim->fts_common_table.table = table; - - optim->fts_index_table.table_id = table->id; - optim->fts_index_table.type = FTS_INDEX_TABLE; - optim->fts_index_table.table = table; - /* The common prefix for all this parent table's aux tables. */ char table_id[FTS_AUX_MIN_TABLE_ID_LENGTH]; const size_t table_id_len = 1 - + size_t(fts_get_table_id(&optim->fts_common_table, table_id)); + + size_t(fts_write_object_id(table->id, table_id)); dict_sys.freeze(SRW_LOCK_CALL); /* Include the separator as well. */ const size_t dbname_len = table->name.dblen() + 1; @@ -1546,10 +1533,6 @@ fts_optimize_index( dberr_t error; byte str[FTS_MAX_WORD_LEN + 1]; - /* Set the current index that we have to optimize. */ - optim->fts_index_table.index_id = index->id; - optim->fts_index_table.charset = fts_index_get_charset(index); - optim->done = FALSE; /* Optimize until !done */ /* We need to read the last word optimized so that we start from @@ -1713,7 +1696,6 @@ fts_optimize_read_deleted_doc_id_snapshot( if (error == DB_SUCCESS) { - optim->fts_common_table.suffix = "BEING_DELETED_CACHE"; /* Read additional doc_ids to delete. */ error = fts_table_fetch_doc_ids( optim->trx, optim->table, "BEING_DELETED_CACHE", diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index da1c124a1d002..2d2ebc8fa074a 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -67,9 +67,6 @@ struct fts_query_t { trx_t* trx; /*!< The query transaction */ dict_index_t* index; /*!< The FTS index to search */ - /*!< FTS auxiliary common table def */ - - fts_table_t fts_common_table; fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ @@ -455,20 +452,6 @@ RecordCompareAction doc_id_exact_match_comparator( : RecordCompareAction::STOP; } -#if 0 -/*****************************************************************//*** -Find a doc_id in a word's ilist. -@return TRUE if found. */ -static -ibool -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: search the doc id selected, - update the frequency if found. */ - void* data, /*!< in: doc id ilist */ - ulint len); /*!< in: doc id ilist size */ -#endif - /*************************************************************//** This function implements a simple "blind" query expansion search: words in documents found in the first search pass will be used as @@ -516,107 +499,6 @@ fts_proximity_get_positions( fts_proximity_t* qualified_pos); /*!< out: the position info records ranges containing all matching words. */ -#if 0 -/******************************************************************** -Get the total number of words in a documents. */ -static -ulint -fts_query_terms_in_document( -/*========================*/ - /*!< out: DB_SUCCESS if all go well - else error code */ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total); /*!< out: total words in document */ -#endif - -#if 0 -/*******************************************************************//** -Print the table used for calculating LCS. */ -static -void -fts_print_lcs_table( -/*================*/ - const ulint* table, /*!< in: array to print */ - ulint n_rows, /*!< in: total no. of rows */ - ulint n_cols) /*!< in: total no. of cols */ -{ - ulint i; - - for (i = 0; i < n_rows; ++i) { - ulint j; - - printf("\n"); - - for (j = 0; j < n_cols; ++j) { - - printf("%2lu ", FTS_ELEM(table, n_cols, i, j)); - } - } -} - -/******************************************************************** -Find the longest common subsequence between the query string and -the document. */ -static -ulint -fts_query_lcs( -/*==========*/ - /*!< out: LCS (length) between - two ilists */ - const ulint* p1, /*!< in: word positions of query */ - ulint len_p1, /*!< in: no. of elements in p1 */ - const ulint* p2, /*!< in: word positions within document */ - ulint len_p2) /*!< in: no. of elements in p2 */ -{ - int i; - ulint len = 0; - ulint r = len_p1; - ulint c = len_p2; - ulint size = (r + 1) * (c + 1) * sizeof(ulint); - ulint* table = (ulint*) ut_malloc_nokey(size); - - /* Traverse the table backwards, from the last row to the first and - also from the last column to the first. We compute the smaller - common subsequences first, then use the calculated values to determine - the longest common subsequence. The result will be in TABLE[0][0]. */ - for (i = r; i >= 0; --i) { - int j; - - for (j = c; j >= 0; --j) { - - if (p1[i] == (ulint) -1 || p2[j] == (ulint) -1) { - - FTS_ELEM(table, c, i, j) = 0; - - } else if (p1[i] == p2[j]) { - - FTS_ELEM(table, c, i, j) = FTS_ELEM( - table, c, i + 1, j + 1) + 1; - - } else { - - ulint value; - - value = ut_max( - FTS_ELEM(table, c, i + 1, j), - FTS_ELEM(table, c, i, j + 1)); - - FTS_ELEM(table, c, i, j) = value; - } - } - } - - len = FTS_ELEM(table, c, 0, 0); - - fts_print_lcs_table(table, r, c); - printf("\nLen=" ULINTPF "\n", len); - - ut_free(table); - - return(len); -} -#endif /*******************************************************************//** Compare two fts_ranking_t instance on their rank value and doc ids in @@ -2316,365 +2198,6 @@ dberr_t fts_query_fetch_document(dict_index_t *fts_index, return err; } -#if 0 -/******************************************************************** -Callback function to check whether a record was found or not. */ -static -ibool -fts_query_select( -/*=============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - int i; - que_node_t* exp; - sel_node_t* node = row; - fts_select_t* select = user_arg; - - ut_a(select->word_freq); - ut_a(select->word_freq->doc_freqs); - - exp = node->select_list; - - for (i = 0; exp && !select->found; ++i) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - switch (i) { - case 0: /* DOC_COUNT */ - if (len != UNIV_SQL_NULL && len != 0) { - - select->word_freq->doc_count += - mach_read_from_4(data); - } - break; - - case 1: /* ILIST */ - if (len != UNIV_SQL_NULL && len != 0) { - - fts_query_find_doc_id(select, data, len); - } - break; - - default: - ut_error; - } - - exp = que_node_get_next(exp); - } - - return(FALSE); -} - -/******************************************************************** -Read the rows from the FTS index, that match word and where the -doc id is between first and last doc id. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_find_term( -/*================*/ - fts_query_t* query, /*!< in: FTS query state */ - que_t** graph, /*!< in: prepared statement */ - const fts_string_t* word, /*!< in: the word to fetch */ - doc_id_t doc_id, /*!< in: doc id to match */ - ulint* min_pos,/*!< in/out: pos found must be - greater than this minimum value. */ - ibool* found) /*!< out: TRUE if found else FALSE */ -{ - pars_info_t* info; - dberr_t error; - fts_select_t select; - doc_id_t match_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index matching nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - selected = fts_select_index(*word->f_str); - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - select.found = FALSE; - select.doc_id = doc_id; - select.min_pos = *min_pos; - select.word_freq = fts_query_add_word_freq(query, word->f_str); - - pars_info_bind_function(info, "my_func", fts_query_select, &select); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &match_doc_id, doc_id); - - fts_bind_doc_id(info, "min_doc_id", &match_doc_id); - - fts_bind_doc_id(info, "max_doc_id", &match_doc_id); - - if (!*graph) { - - *graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count, ilist\n" - " FROM $index_table_name\n" - " WHERE word LIKE :word AND" - " first_doc_id <= :min_doc_id AND" - " last_doc_id >= :max_doc_id\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - /* Value to return */ - *found = select.found; - - if (*found) { - *min_pos = select.min_pos; - } - - return(error); -} - -/******************************************************************** -Callback aggregator for int columns. */ -static -ibool -fts_query_sum( -/*==========*/ - /*!< out: always returns TRUE */ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: ulint* */ -{ - - que_node_t* exp; - sel_node_t* node = row; - ulint* total = user_arg; - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - if (len != UNIV_SQL_NULL && len != 0) { - *total += mach_read_from_4(data); - } - - exp = que_node_get_next(exp); - } - - return(TRUE); -} - -/******************************************************************** -Calculate the total documents that contain a particular word (term). -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_total_docs_containing_term( -/*=================================*/ - fts_query_t* query, /*!< in: FTS query state */ - const fts_string_t* word, /*!< in: the word to check */ - ulint* total) /*!< out: documents containing word */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - ulint selected; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN] - - trx->op_info = "fetching FTS index document count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - selected = fts_select_index(*word->f_str); - - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count\n" - " FROM $index_table_name\n" - " WHERE word = :word" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} - -/******************************************************************** -Get the total number of words in a documents. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_terms_in_document( -/*========================*/ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total) /*!< out: total words in document */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - doc_id_t read_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS document term count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &read_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &read_doc_id); - - query->fts_index_table.suffix = "DOC_ID"; - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT count\n" - " FROM $index_table_name\n" - " WHERE doc_id = :doc_id" - " BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " doc id table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error << " while reading FTS" - " doc id table."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} -#endif - /*****************************************************************//** Retrieve the document and match the phrase tokens. @return DB_SUCCESS or error code */ @@ -2868,6 +2391,7 @@ fts_query_phrase_split( ulint len = 0; ulint cur_pos = 0; fts_ast_node_t* term_node = NULL; + CHARSET_INFO* cs = fts_index_get_charset(query->index); if (node->type == FTS_AST_TEXT) { phrase.f_str = node->text.ptr->str; @@ -2891,7 +2415,7 @@ fts_query_phrase_split( } cur_len = innobase_mysql_fts_get_token( - query->fts_index_table.charset, + cs, reinterpret_cast(phrase.f_str) + cur_pos, reinterpret_cast(phrase.f_str) @@ -2914,7 +2438,7 @@ fts_query_phrase_split( result_str.f_str = term_node->term.ptr->str; result_str.f_len = term_node->term.ptr->len; result_str.f_n_char = fts_get_token_size( - query->fts_index_table.charset, + cs, reinterpret_cast(result_str.f_str), result_str.f_len); @@ -2931,8 +2455,7 @@ fts_query_phrase_split( if (fts_check_token( &result_str, - cache->stopword_info.cached_stopword, - query->fts_index_table.charset)) { + cache->stopword_info.cached_stopword, cs)) { /* Add the word to the RB tree so that we can calculate its frequency within a document. */ fts_query_add_word_freq(query, token); @@ -3387,78 +2910,6 @@ fts_ast_visit_sub_exp( DBUG_RETURN(error); } -#if 0 -/*****************************************************************//*** -Check if the doc id exists in the ilist. -@return TRUE if doc id found */ -static -ulint -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: contains the doc id to - find, we update the word freq if - document found */ - void* data, /*!< in: doc id ilist */ - ulint len) /*!< in: doc id ilist size */ -{ - byte* ptr = data; - doc_id_t doc_id = 0; - ulint decoded = 0; - - /* Decode the ilist and search for selected doc_id. We also - calculate the frequency of the word in the document if found. */ - while (decoded < len && !select->found) { - ulint freq = 0; - ulint min_pos = 0; - ulint last_pos = 0; - ulint pos = fts_decode_vlc(&ptr); - - /* Add the delta. */ - doc_id += pos; - - while (*ptr) { - ++freq; - last_pos += fts_decode_vlc(&ptr); - - /* Only if min_pos is not set and the current - term exists in a position greater than the - min_pos of the previous term. */ - if (min_pos == 0 && last_pos > select->min_pos) { - min_pos = last_pos; - } - } - - /* Skip the end of word position marker. */ - ++ptr; - - /* Bytes decoded so far. */ - decoded = ptr - (byte*) data; - - /* A word may exist in the document but we only consider a - match if it exists in a position that is greater than the - position of the previous term. */ - if (doc_id == select->doc_id && min_pos > 0) { - fts_doc_freq_t* doc_freq; - - /* Add the doc id to the doc freq rb tree, if - the doc id doesn't exist it will be created. */ - doc_freq = fts_query_add_doc_freq( - select->word_freq->doc_freqs, doc_id); - - /* Avoid duplicating the frequency tally */ - if (doc_freq->freq == 0) { - doc_freq->freq = freq; - } - - select->found = TRUE; - select->min_pos = min_pos; - } - } - - return(select->found); -} -#endif - /*****************************************************************//** Read and filter nodes. @return DB_SUCCESS if all go well, @@ -4115,7 +3566,7 @@ fts_query_parse( memset(&state, 0x0, sizeof(state)); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); DBUG_EXECUTE_IF("fts_instrument_query_disable_parser", query->parser = NULL;); @@ -4128,7 +3579,7 @@ fts_query_parse( } else { /* Setup the scanner to use, this depends on the mode flag. */ state.lexer = fts_lexer_create(mode, query_str, query_len); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); error = fts_parse(&state); fts_lexer_free(state.lexer); state.lexer = NULL; @@ -4216,10 +3667,6 @@ fts_query( query.deleted = fts_doc_ids_create(); query.cur_node = NULL; - query.fts_common_table.type = FTS_COMMON_TABLE; - query.fts_common_table.table_id = index->table->id; - query.fts_common_table.table = index->table; - charset = fts_index_get_charset(index); query.fts_index_table.type = FTS_INDEX_TABLE; @@ -4248,8 +3695,6 @@ fts_query( query.total_docs = dict_table_get_n_rows(index->table); - query.fts_common_table.suffix = "DELETED"; - /* Read the deleted doc_ids, we need these for filtering. */ error = fts_table_fetch_doc_ids( nullptr, index->table, "DELETED", query.deleted); @@ -4258,8 +3703,6 @@ fts_query( goto func_exit; } - query.fts_common_table.suffix = "DELETED_CACHE"; - error = fts_table_fetch_doc_ids( nullptr, index->table, "DELETED_CACHE", query.deleted); diff --git a/storage/innobase/fts/fts0sql.cc b/storage/innobase/fts/fts0sql.cc deleted file mode 100644 index 781d15f2befb0..0000000000000 --- a/storage/innobase/fts/fts0sql.cc +++ /dev/null @@ -1,208 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file fts/fts0sql.cc -Full Text Search functionality. - -Created 2007-03-27 Sunny Bains -*******************************************************/ - -#include "que0que.h" -#include "trx0roll.h" -#include "pars0pars.h" -#include "dict0dict.h" -#include "fts0types.h" -#include "fts0priv.h" - -/** SQL statements for creating the ancillary FTS tables. */ - -/** Preamble to all SQL statements. */ -static const char* fts_sql_begin= - "PROCEDURE P() IS\n"; - -/** Postamble to non-committing SQL statements. */ -static const char* fts_sql_end= - "\n" - "END;\n"; - -/******************************************************************//** -Get the table id. -@return number of bytes written */ -int -fts_get_table_id( -/*=============*/ - const fts_table_t* - fts_table, /*!< in: FTS Auxiliary table */ - char* table_id) /*!< out: table id, must be at least - FTS_AUX_MIN_TABLE_ID_LENGTH bytes - long */ -{ - int len; - - ut_a(fts_table->table != NULL); - - switch (fts_table->type) { - case FTS_COMMON_TABLE: - len = fts_write_object_id(fts_table->table_id, table_id); - break; - - case FTS_INDEX_TABLE: - - len = fts_write_object_id(fts_table->table_id, table_id); - - table_id[len] = '_'; - ++len; - table_id += len; - - len += fts_write_object_id(fts_table->index_id, table_id); - break; - - default: - ut_error; - } - - ut_a(len >= 16); - ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); - - return(len); -} - -/** Construct the name of an internal FTS table for the given table. -@param[in] fts_table metadata on fulltext-indexed table -@param[out] table_name a name up to MAX_FULL_NAME_LEN -@param[in] dict_locked whether dict_sys.latch is being held */ -void fts_get_table_name(const fts_table_t* fts_table, char* table_name, - bool dict_locked) -{ - if (!dict_locked) { - dict_sys.freeze(SRW_LOCK_CALL); - } - ut_ad(dict_sys.frozen()); - /* Include the separator as well. */ - const size_t dbname_len = fts_table->table->name.dblen() + 1; - ut_ad(dbname_len > 1); - memcpy(table_name, fts_table->table->name.m_name, dbname_len); - if (!dict_locked) { - dict_sys.unfreeze(); - } - memcpy(table_name += dbname_len, "FTS_", 4); - table_name += 4; - table_name += fts_get_table_id(fts_table, table_name); - *table_name++ = '_'; - strcpy(table_name, fts_table->suffix); -} - -/******************************************************************//** -Parse an SQL string. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS auxiliary table info */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ -{ - char* str; - que_t* graph; - ibool dict_locked; - - str = ut_str3cat(fts_sql_begin, sql, fts_sql_end); - - dict_locked = (fts_table && fts_table->table->fts - && fts_table->table->fts->dict_locked); - - if (!dict_locked) { - /* The InnoDB SQL parser is not re-entrant. */ - dict_sys.lock(SRW_LOCK_CALL); - } - - graph = pars_sql(info, str); - ut_a(graph); - - if (!dict_locked) { - dict_sys.unlock(); - } - - ut_free(str); - - return(graph); -} - -/******************************************************************//** -Evaluate an SQL query graph. -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Query graph to evaluate */ -{ - que_thr_t* thr; - - graph->trx = trx; - - ut_a(thr = que_fork_start_command(graph)); - - que_run_threads(thr); - - return(trx->error_state); -} - -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ulint i; - const char* str = ""; - - for (i = 0; i < index->n_user_defined_cols; i++) { - char* sel_str; - - dict_field_t* field = dict_index_get_nth_field(index, i); - - sel_str = mem_heap_printf(heap, "sel%lu", (ulong) i); - - /* Set copy_name to TRUE since it's dynamic. */ - pars_info_bind_id(info, sel_str, field->name); - - str = mem_heap_printf( - heap, "%s%s$%s", str, (*str) ? ", " : "", sel_str); - } - - return(str); -} diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index a6273161145f9..286447a6a9050 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -2204,7 +2204,6 @@ i_s_fts_deleted_generic_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; fts_doc_ids_t* deleted; dict_table_t* user_table; @@ -2235,10 +2234,6 @@ i_s_fts_deleted_generic_fill( trx = trx_create(); trx->op_info = "Select for FTS DELETE TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, - (being_deleted) ? "BEING_DELETED" : "DELETED", - FTS_COMMON_TABLE, user_table); - fts_table_fetch_doc_ids( nullptr, user_table, being_deleted ? "BEING_DELETED" : "DELETED", deleted); diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 7263536bddc46..9711c58ca80e5 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -113,26 +113,6 @@ component. /** Maximum length of an integer stored in the config table value column. */ #define FTS_MAX_INT_LEN 32 -/******************************************************************//** -Parse an SQL string. %s is replaced with the table's id. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS aux table */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ - MY_ATTRIBUTE((nonnull(3), malloc, warn_unused_result)); -/******************************************************************//** -Evaluate a parsed SQL statement -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Parsed statement */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Construct the name of an internal FTS table for the given table. @param[in] fts_table metadata on fulltext-indexed table @param[out] table_name a name up to MAX_FULL_NAME_LEN @@ -140,39 +120,6 @@ fts_eval_sql( void fts_get_table_name(const fts_table_t* fts_table, char* table_name, bool dict_locked = false) MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: FTS index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/*******************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ - MY_ATTRIBUTE((nonnull)); /** Write out a single word's data as new entry/entries in the INDEX table. @param executor FTS Query Executor From ff6a64d018f868f545378c5d85af2ef756f2f0cb Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 17 Nov 2025 17:56:13 +0530 Subject: [PATCH 08/12] MDEV-28730 Remove internal parser usage from InnoDB fts - Fix compilaton issue - delete_all() moves the cursor to user record once you open the left leaf page --- storage/innobase/fts/fts0exec.cc | 1 - storage/innobase/fts/fts0fts.cc | 4 +- storage/innobase/fts/fts0que.cc | 171 +------------------------ storage/innobase/include/fts0priv.h | 1 - storage/innobase/include/fts0types.inl | 2 +- storage/innobase/row/row0query.cc | 1 + 6 files changed, 5 insertions(+), 175 deletions(-) diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc index 09a82d7953068..8f2e1a57fceff 100644 --- a/storage/innobase/fts/fts0exec.cc +++ b/storage/innobase/fts/fts0exec.cc @@ -32,7 +32,6 @@ Created 2025/11/05 #include "row0ins.h" #include "row0upd.h" #include "row0sel.h" -#include "pars0pars.h" #include "eval0eval.h" #include "que0que.h" #include "trx0trx.h" diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 5c248f399bb41..d794ea0daf34f 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -5425,7 +5425,7 @@ static void fts_init_recover_all_docs(fts_get_doc_t *get_doc, doc_id= fts_read_doc_id(doc_id_data); /* Process each indexed column content */ - for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + for (unsigned i= 0; i < fts_index->n_user_defined_cols; i++) { ulint col_pos= clust_field_nos[i]; ulint field_len; @@ -5456,7 +5456,7 @@ static void fts_init_recover_all_docs(fts_get_doc_t *get_doc, else fts_tokenize_document_next(&doc, doc_len, NULL, parser); doc_len+= - (i < get_doc->index_cache->index->n_user_defined_cols - 1) + (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) ? field_len + 1 : field_len; } diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 2d2ebc8fa074a..66329246aad8c 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -267,16 +267,6 @@ struct fts_word_freq_t { double idf; /*!< Inverse document frequency */ }; -/******************************************************************** -Callback function to fetch the rows in an FTS INDEX record. -@return always TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg); /*!< in: pointer to ib_vector_t */ - /******************************************************************** Read and filter nodes. @return fts_node_t instance */ @@ -318,14 +308,11 @@ static bool node_query_processor( const rec_offs* offsets, void* user_arg) { fts_query_t* query= static_cast(user_arg); - fts_string_t key; ulint word_len; const byte* word_data = rec_get_nth_field(rec, offsets, 0, &word_len); if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) return true; - key.f_str= const_cast(word_data); - key.f_len= word_len; ut_a(query->cur_node->type == FTS_AST_TERM || query->cur_node->type == FTS_AST_TEXT || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); @@ -2100,7 +2087,7 @@ dberr_t fts_query_fetch_document(dict_index_t *fts_index, &user_table->cols[user_table->fts->doc_col], index); ulint len; - rec_get_nth_field(rec, offsets, doc_col_pos, &len); + rec_get_nth_field_offs(offsets, doc_col_pos, &len); if (len != sizeof(doc_id_t)) return true; @@ -2517,7 +2504,6 @@ fts_query_phrase_search( /* Ignore empty strings. */ if (num_token > 0) { fts_string_t* token = NULL; - fts_fetch_t fetch; trx_t* trx = query->trx; fts_ast_oper_t oper = query->oper; ulint i; @@ -2554,11 +2540,6 @@ fts_query_phrase_search( } } - /* Setup the callback args for filtering and consolidating - the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; - for (i = 0; i < num_token; i++) { /* Search for the first word from the phrase. */ token = static_cast( @@ -3030,156 +3011,6 @@ fts_query_filter_doc_ids( } } -/*****************************************************************//** -Read the FTS INDEX row. -@return DB_SUCCESS if all go well. */ -static -dberr_t -fts_query_read_node( -/*================*/ - fts_query_t* query, /*!< in: query instance */ - const fts_string_t* word, /*!< in: current word */ - que_node_t* exp) /*!< in: query graph node */ -{ - int i; - int ret; - fts_node_t node; - ib_rbt_bound_t parent; - fts_word_freq_t* word_freq; - ibool skip = FALSE; - fts_string_t term; - byte buf[FTS_MAX_WORD_LEN + 1]; - dberr_t error = DB_SUCCESS; - - ut_a(query->cur_node->type == FTS_AST_TERM - || query->cur_node->type == FTS_AST_TEXT - || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); - - memset(&node, 0, sizeof(node)); - term.f_str = buf; - - /* Need to consider the wildcard search case, the word frequency - is created on the search string not the actual word. So we need - to assign the frequency on search string behalf. */ - if (query->cur_node->type == FTS_AST_TERM - && query->cur_node->term.wildcard) { - - term.f_len = query->cur_node->term.ptr->len; - ut_ad(FTS_MAX_WORD_LEN >= term.f_len); - memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); - } else { - term.f_len = word->f_len; - ut_ad(FTS_MAX_WORD_LEN >= word->f_len); - memcpy(term.f_str, word->f_str, word->f_len); - } - - /* Lookup the word in our rb tree, it must exist. */ - ret = rbt_search(query->word_freqs, &parent, &term); - - ut_a(ret == 0); - - word_freq = rbt_value(fts_word_freq_t, parent.last); - - /* Start from 1 since the first column has been read by the caller. - Also, we rely on the order of the columns projected, to filter - out ilists that are out of range and we always want to read - the doc_count irrespective of the suitability of the row. */ - - for (i = 1; exp && !skip; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - - switch (i) { - case 1: /* DOC_COUNT */ - word_freq->doc_count += mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node.first_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->upper_doc_id > 0 - && node.first_doc_id > query->upper_doc_id) { - skip = TRUE; - } - break; - - case 3: /* LAST_DOC_ID */ - node.last_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->lower_doc_id > 0 - && node.last_doc_id < query->lower_doc_id) { - skip = TRUE; - } - break; - - case 4: /* ILIST */ - - error = fts_query_filter_doc_ids( - query, &word_freq->word, word_freq, - &node, data, len, FALSE); - - break; - - default: - ut_error; - } - } - - if (!skip) { - /* Make sure all columns were read. */ - - ut_a(i == 5); - } - - return error; -} - -/*****************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to fts_fetch_t */ -{ - fts_string_t key; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - fts_query_t* query = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - - key.f_str = static_cast(data); - key.f_len = dfield_len; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - /* Note: we pass error out by 'query->error' */ - query->error = fts_query_read_node(query, &key, que_node_get_next(exp)); - - if (query->error != DB_SUCCESS) { - ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT); - return(FALSE); - } else { - return(TRUE); - } -} - /*****************************************************************//** Calculate the inverse document frequency (IDF) for all the terms. */ static diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 9711c58ca80e5..66fed5c57dabc 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -28,7 +28,6 @@ Created 2011/09/02 Sunny Bains #define INNOBASE_FTS0PRIV_H #include "dict0dict.h" -#include "pars0pars.h" #include "que0que.h" #include "que0types.h" #include "fts0types.h" diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index 421331fcdaf05..d1081bfd0a44d 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -163,7 +163,7 @@ fts_select_index_by_hash( /* Get collation hash code */ my_ci_hash_sort(cs, str, char_len, &nr1, &nr2); - return(nr1 % FTS_NUM_AUX_INDEX); + return static_cast(nr1 % FTS_NUM_AUX_INDEX); } /** Select the FTS auxiliary index for the given character. diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc index 84f8005178198..826c9325ac6b7 100644 --- a/storage/innobase/row/row0query.cc +++ b/storage/innobase/row/row0query.cc @@ -158,6 +158,7 @@ dberr_t QueryExecutor::delete_all(dict_table_t *table) noexcept mtr.set_named_space(table->space); dberr_t err= pcur.open_leaf(true, index, BTR_MODIFY_LEAF, &mtr); + if (err == DB_SUCCESS) btr_pcur_move_to_next(&pcur, &mtr); if (err != DB_SUCCESS) { mtr.commit(); From 1ee1317ddd1103587eb9625729b0fb425da1bb50 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 21 Nov 2025 14:00:42 +0530 Subject: [PATCH 09/12] - Allow the document to tokenize next only if we already process the word --- storage/innobase/fts/fts0fts.cc | 64 +++++++++++++++++---------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index d794ea0daf34f..2a84b5d6dab98 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -5417,6 +5417,7 @@ static void fts_init_recover_all_docs(fts_get_doc_t *get_doc, ulint doc_col_pos= dict_col_get_index_pos( &user_table->cols[user_table->fts->doc_col], index); + ulint processed_field= 0; ulint len; const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); @@ -5426,40 +5427,41 @@ static void fts_init_recover_all_docs(fts_get_doc_t *get_doc, /* Process each indexed column content */ for (unsigned i= 0; i < fts_index->n_user_defined_cols; i++) - { - ulint col_pos= clust_field_nos[i]; - ulint field_len; - const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, - &field_len); - if (field_len == UNIV_SQL_NULL) - continue; - if (!get_doc->index_cache->charset) - { - dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); - get_doc->index_cache->charset= fts_get_charset(fts_field->col->prtype); - } - doc.charset= get_doc->index_cache->charset; - - /* Handle externally stored fields */ - if (rec_offs_nth_extern(offsets, col_pos)) - doc.text.f_str= btr_copy_externally_stored_field( - &doc.text.f_len, const_cast(field_data), - user_table->space->zip_size(), field_len, - static_cast(doc.self_heap->arg)); - else { - doc.text.f_str= const_cast(field_data); - doc.text.f_len= field_len; - } + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, + &field_len); + if (field_len == UNIV_SQL_NULL) + continue; + if (!get_doc->index_cache->charset) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + get_doc->index_cache->charset= fts_get_charset(fts_field->col->prtype); + } + doc.charset= get_doc->index_cache->charset; + + /* Handle externally stored fields */ + if (rec_offs_nth_extern(offsets, col_pos)) + doc.text.f_str= btr_copy_externally_stored_field( + &doc.text.f_len, const_cast(field_data), + user_table->space->zip_size(), field_len, + static_cast(doc.self_heap->arg)); + else + { + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + } - if (i == 0) fts_tokenize_document(&doc, NULL, parser); - else fts_tokenize_document_next(&doc, doc_len, NULL, parser); + if (processed_field == 0) fts_tokenize_document(&doc, NULL, parser); + else fts_tokenize_document_next(&doc, doc_len, NULL, parser); - doc_len+= - (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) - ? field_len + 1 - : field_len; - } + processed_field++; + doc_len+= + (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) + ? field_len + 1 + : field_len; + } fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); fts_doc_free(&doc); From ce70ec4cb162cf36012e06070d5b6df5389bfeb0 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 25 Nov 2025 16:29:01 +0530 Subject: [PATCH 10/12] -fts_optimize_words() should compare the record with exact match of the string. --- storage/innobase/fts/fts0exec.cc | 8 ++++---- storage/innobase/fts/fts0opt.cc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc index 8f2e1a57fceff..0a446a6017fbb 100644 --- a/storage/innobase/fts/fts0exec.cc +++ b/storage/innobase/fts/fts0exec.cc @@ -786,10 +786,10 @@ RecordCompareAction AuxRecordReader::compare_record( else /* AuxCompareMode::LIKE */ { /* For LIKE mode, compare only the prefix (search_len bytes) */ - int prefix_cmp = cmp_data(type->mtype, type->prtype, false, - static_cast(search_data), - search_len, rec_data, - search_len <= rec_len ? search_len : rec_len); + int prefix_cmp= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_data, + search_len <= rec_len ? search_len : rec_len); if (prefix_cmp != 0) return RecordCompareAction::STOP; return (search_len <= rec_len) ? RecordCompareAction::PROCESS diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 05844cdd6f361..45f299417ebea 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1397,7 +1397,7 @@ void fts_optimize_words(fts_optimize_t *optim, dict_index_t *index, ut_a(ib_vector_size(optim->words) == 0); /* Read the index records to optimize. */ dberr_t error= fts_index_fetch_nodes( - trx, index, word, optim->words, nullptr, AuxCompareMode::LIKE); + trx, index, word, optim->words, nullptr, AuxCompareMode::EQUAL); if (error == DB_SUCCESS) { /* There must be some nodes to read. */ From 241874e9dfdf2eaf56d893de2f4d8678fde0fa7d Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 27 Nov 2025 19:43:32 +0530 Subject: [PATCH 11/12] MDEV-28730 Remove internal parser usage from InnoDB fts - Use btr_pcur_open_on_user_rec() instead of btr_pcur_open() in QueryExecutor::read() and QueryExecutor::read_by_index() --- storage/innobase/include/row0query.h | 6 ++++-- storage/innobase/row/row0query.cc | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/storage/innobase/include/row0query.h b/storage/innobase/include/row0query.h index bb35a1921b764..9c8e1d550ff3a 100644 --- a/storage/innobase/include/row0query.h +++ b/storage/innobase/include/row0query.h @@ -218,7 +218,8 @@ class QueryExecutor @return DB_SUCCESS if at least one record was processed @retval DB_RECORD_NOT_FOUND if no record matched @return error code on failure */ - dberr_t read(dict_table_t *table, dtuple_t *tuple, page_cur_mode_t mode, + dberr_t read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, RecordCallback& callback) noexcept; /** Read records via a secondary index and process corresponding @@ -236,7 +237,8 @@ class QueryExecutor DB_RECORD_NOT_FOUND if no matching record was processed error code on failure */ dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, - dtuple_t *search_tuple, page_cur_mode_t mode, + const dtuple_t *search_tuple, + page_cur_mode_t mode, RecordCallback& callback) noexcept; /** Acquire a table lock in the given mode for transaction. diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc index 826c9325ac6b7..ce663d84ef8ad 100644 --- a/storage/innobase/row/row0query.cc +++ b/storage/innobase/row/row0query.cc @@ -431,7 +431,7 @@ dberr_t QueryExecutor::replace_record( return err; } -dberr_t QueryExecutor::read(dict_table_t *table, dtuple_t *tuple, +dberr_t QueryExecutor::read(dict_table_t *table, const dtuple_t *tuple, page_cur_mode_t mode, RecordCallback& callback) noexcept { @@ -447,7 +447,8 @@ dberr_t QueryExecutor::read(dict_table_t *table, dtuple_t *tuple, } m_pcur.btr_cur.page_cur.index= index; dberr_t err= DB_SUCCESS; - if (tuple) err= btr_pcur_open(tuple, mode, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + if (tuple) + err= btr_pcur_open_on_user_rec(tuple, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); else { err= m_pcur.open_leaf(true, index, BTR_SEARCH_LEAF, &m_mtr); @@ -484,7 +485,7 @@ dberr_t QueryExecutor::read(dict_table_t *table, dtuple_t *tuple, dberr_t QueryExecutor::read_by_index(dict_table_t *table, dict_index_t *sec_index, - dtuple_t *search_tuple, + const dtuple_t *search_tuple, page_cur_mode_t mode, RecordCallback& callback) noexcept { @@ -506,7 +507,8 @@ dberr_t QueryExecutor::read_by_index(dict_table_t *table, dberr_t err= DB_SUCCESS; if (search_tuple) - err= btr_pcur_open(search_tuple, mode, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + err= btr_pcur_open_on_user_rec(search_tuple, BTR_SEARCH_LEAF, + &m_pcur, &m_mtr); else { err= m_pcur.open_leaf(true, sec_index, BTR_SEARCH_LEAF, &m_mtr); From b672350ca3eaf3cebbd3217b10249939d4cd46c9 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 28 Nov 2025 16:26:11 +0530 Subject: [PATCH 12/12] MDEV-28730 Remove internal parser usage from InnoDB fts fts_optimize_table() : Assigns thd to transaction even it is called via user_thread or fulltext optimize thread. Acquires MDL_SHARED_NO_WRITE for the table to avoid any DDL/DML while doing fulltext optimization. Tweaked dict_acquire_mdl_shared to acquire MDL_SHARED_NO_WRITE based on the input paramter. --- storage/innobase/dict/dict0dict.cc | 22 ++++++++++++++------- storage/innobase/fts/fts0opt.cc | 28 +++++++++++++++++++-------- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/include/dict0dict.h | 4 ++-- storage/innobase/include/fts0fts.h | 11 +++++------ 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 2edc7068f17e3..7883d19c8f902 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -620,7 +620,7 @@ dict_table_t *dict_sys_t::find_table(const span &name) @param[in] table_op operation to perform when opening @return table object after locking MDL shared @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template __attribute__((nonnull, warn_unused_result)) dict_table_t* dict_acquire_mdl_shared(dict_table_t *table, @@ -655,7 +655,8 @@ dict_acquire_mdl_shared(dict_table_t *table, { MDL_request request; - MDL_REQUEST_INIT(&request,MDL_key::TABLE, db_buf, tbl_buf, MDL_SHARED, + MDL_REQUEST_INIT(&request,MDL_key::TABLE, db_buf, tbl_buf, + shared_no_write ? MDL_SHARED_NO_WRITE : MDL_SHARED, MDL_EXPLICIT); if (trylock ? mdl_context->try_acquire_lock(&request) @@ -755,7 +756,9 @@ dict_acquire_mdl_shared(dict_table_t *table, goto retry; } -template dict_table_t* dict_acquire_mdl_shared +template dict_table_t* dict_acquire_mdl_shared +(dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl_shared (dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); /** Acquire MDL shared for the table name. @@ -766,7 +769,7 @@ template dict_table_t* dict_acquire_mdl_shared @param[in] table_op operation to perform when opening @return table object after locking MDL shared @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template dict_table_t* dict_acquire_mdl_shared(dict_table_t *table, THD *thd, @@ -793,12 +796,17 @@ dict_acquire_mdl_shared(dict_table_t *table, if (db_len == 0) return table; /* InnoDB system tables are not covered by MDL */ - return dict_acquire_mdl_shared(table, &thd->mdl_context, mdl, table_op); + return dict_acquire_mdl_shared( + table, &thd->mdl_context, mdl, table_op); } -template dict_table_t* dict_acquire_mdl_shared +template dict_table_t* dict_acquire_mdl_shared +(dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl_shared +(dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl_shared (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); -template dict_table_t* dict_acquire_mdl_shared +template dict_table_t* dict_acquire_mdl_shared (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); /** Look up a table by numeric identifier. diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 45f299417ebea..a66b07f8679b0 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -39,6 +39,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0opt.h" #include "fts0vlc.h" #include "wsrep.h" +#include "lock0lock.h" #ifdef WITH_WSREP extern Atomic_relaxed wsrep_sst_disable_writes; @@ -1770,6 +1771,7 @@ fts_optimize_indexes( index = static_cast( ib_vector_getp(fts->indexes, i)); error = fts_optimize_index(optim, index); + if (error) break; } if (error == DB_SUCCESS) { @@ -1875,7 +1877,7 @@ fts_optimize_table_bk( if (table->is_accessible() && table->fts && table->fts->cache && table->fts->cache->deleted >= FTS_OPTIMIZE_THRESHOLD) { - error = fts_optimize_table(table); + error = fts_optimize_table(table, fts_opt_thd); slot->last_run = time(NULL); @@ -1891,13 +1893,13 @@ fts_optimize_table_bk( return(error); } -/*********************************************************************//** -Run OPTIMIZE on the given table. -@return DB_SUCCESS if all OK */ + +/** run optimize on the given table. +@param table table to be optimized +@param user_thd user thread +@return db_success if all ok */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table) /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd) { if (srv_read_only_mode) { return DB_READ_ONLY; @@ -1911,7 +1913,15 @@ fts_optimize_table( ib::info() << "FTS start optimize " << table->name; } - optim = fts_optimize_create(table); + MDL_ticket *mdl_ticket = nullptr; + dict_table_t *optim_table = + dict_acquire_mdl_shared(table, thd, &mdl_ticket); + + if (!optim_table) return DB_ERROR; + + optim = fts_optimize_create(optim_table); + + optim->trx->mysql_thd = thd; // FIXME: Call this only at the start of optimize, currently we // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot. @@ -1989,6 +1999,8 @@ fts_optimize_table( ib::info() << "FTS end optimize " << table->name; } + if (mdl_ticket) + dict_table_close(optim_table, thd, mdl_ticket); return(error); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b1f38c19079c9..24fe1d930b052 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -15165,7 +15165,7 @@ ha_innobase::optimize( if (m_prebuilt->table->fts && m_prebuilt->table->fts->cache && m_prebuilt->table->space) { fts_sync_table(m_prebuilt->table); - fts_optimize_table(m_prebuilt->table); + fts_optimize_table(m_prebuilt->table, thd); } try_alter = false; } diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 5bb79e286f0da..85f7146fb5421 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -101,7 +101,7 @@ enum dict_table_op_t { @param[in] table_op operation to perform when opening @return table object after locking MDL shared @retval NULL if the table is not readable, or if trylock && MDL blocked */ -template +template dict_table_t* dict_acquire_mdl_shared(dict_table_t *table, THD *thd, @@ -116,7 +116,7 @@ dict_acquire_mdl_shared(dict_table_t *table, @param[in] table_op operation to perform when opening @return table object after locking MDL shared @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template __attribute__((nonnull, warn_unused_result)) dict_table_t* dict_acquire_mdl_shared(dict_table_t *table, diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 4c1a7ffd4d466..63517029aa434 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -615,13 +615,12 @@ fts_create( dict_table_t* table); /*!< out: table with FTS indexes */ -/*********************************************************************//** -Run OPTIMIZE on the given table. -@return DB_SUCCESS if all OK */ +/** run optimize on the given table. +@param table table to be optimized +@param user_thd user thread +@return db_success if all ok */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table); /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd); /**********************************************************************//** Startup the optimize thread and create the work queue. */