Skip to content

T
tsearch2-driver
Commit c46d5c97 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

Initial Import

parents
Pipeline #660 failed with stages
      Expand all Hide whitespace changes
      Inline Side-by-side
      Showing with 662 additions and 0 deletions
      sql/postgresql/tsearch2-driver-create.sql 0 → 100644
      View file @ c46d5c97
      --
      -- tsearch2 based FTSEngineDriver for Search package
      --
      -- @author Dave Bauer (dave@thedesignexperience.org)
      -- @creation-date 2004-06-05
      -- @arch-tag: 25662a8a-e960-4888-bd9f-3e7a8fff5637
      -- @cvs-id $Id$
      --
      -- FIXME need to load tsearch2.sql from postgresql/share/contrib
      -- (on debian /usr/share/postgresql/contrib)
      create table txt (
      object_id integer
      constraint txt_object_id_fk
      references acs_objects on delete cascade,
      fti tsvector
      );
      create index fti_idx on txt using gist(fti);
      create index object_id_idx on txt (object_id);
      sql/postgresql/tsearch2-driver-drop.sql 0 → 100644
      View file @ c46d5c97
      --
      --
      --
      -- @author Dave Bauer (dave@thedesignexperience.org)
      -- @creation-date 2004-06-05
      -- @arch-tag: da4f668f-eba3-40a1-8efb-23c347e312c9
      -- @cvs-id $Id$
      --
      drop table txt;
      sql/postgresql/tsearch2-driver-package-create.sql 0 → 100644
      View file @ c46d5c97
      create or replace function ts2_to_tsvector ( varchar, varchar ) returns varchar as '
      declare
      ts2_cfg alias for $1;
      ts2_txt alias for $2;
      ts2_result varchar;
      begin
      perform set_curcfg(ts2_cfg);
      select to_tsvector(ts2_cfg,ts2_txt) into ts2_result;
      return ts2_result;
      end;' language 'plpgsql';
      create or replace function ts2_to_tsquery ( varchar, varchar ) returns tsquery as '
      declare
      ts2_cfg alias for $1;
      ts2_txt alias for $2;
      ts2_result tsquery;
      begin
      perform set_curcfg(ts2_cfg);
      select 1 into ts2_result;
      select to_tsquery(ts2_cfg,ts2_txt) into ts2_result;
      return ts2_result;
      end;' language 'plpgsql';
      sql/postgresql/upgrade/upgrade-0.4d1-0.4d2.sql 0 → 100644
      View file @ c46d5c97
      --
      --
      --
      -- @author Dave Bauer (dave@thedesignexperience.org)
      -- @creation-date 2004-12-18
      -- @arch-tag: 46a0ecc0-b123-44dd-9e91-4a565f742071
      -- @cvs-id $Id$
      --
      alter table txt drop constraint txt_object_id_fk;
      alter table txt add constraint txt_object_id_fk foreign key (object_id) references acs_objects (object_id) on delete cascade;
      \ No newline at end of file
      tcl/test/tsearch2-driver-procs.tcl 0 → 100644
      View file @ c46d5c97
      #
      ad_library {
      Tests for tsearch2-driver
      @author Dave Bauer (dave@thedesignexperience.org)
      @creation-date 2005-03-07
      @arch-tag: 17a26a17-9fd6-4ae9-90ae-ce445c020ab7
      @cvs-id $Id$
      }
      aa_register_case build_query {
      build_query test
      } {
      aa_run_with_teardown \
      -test_code {
      # some tests to see if we can turn the english query into
      # something tsearch2 to_tsquery can handle
      set q "openacs test automated"
      aa_true "Multiple terms automatic AND '[tsearch2::build_query -query $q]'" \
      [string equal "openacs & test & automated" \
      [tsearch2::build_query -query $q]]
      set q "openacs test not automated"
      aa_true "Multiple terms automatic AND, explicit NOT '[tsearch2::build_query -query $q]'" \
      [string equal "openacs & test & ! automated" \
      [tsearch2::build_query -query $q]]
      set q "openacs test or automated"
      aa_true "Multiple terms automatic AND, explicit OR '[tsearch2::build_query -query $q]'" \
      [string equal "openacs & test | automated" \
      [tsearch2::build_query -query $q]]
      set q "(openacs test) or automated"
      aa_true "Multiple terms grouped '[tsearch2::build_query -query $q]'" \
      [string equal "(openacs & test) | automated" \
      [tsearch2::build_query -query $q]]
      set q "(openacs or test) automated"
      aa_true "Multiple terms grouped automatic AND '[tsearch2::build_query -query $q]'" \
      [string equal "(openacs | test) & automated" \
      [tsearch2::build_query -query $q]]
      set q "one a two"
      aa_true "Single letter elements" \
      [string equal "one & a & two" \
      [tsearch2::build_query -query $q]]
      set q "or else"
      aa_true "Or at beginning by itself" \
      [string equal "else" \
      [tsearch2::build_query -query $q]]
      set q "not"
      aa_true "Not all alone" \
      [string equal "" \
      [tsearch2::build_query -query $q]]
      }
      }
      tcl/tsearch2-driver-install-procs.tcl 0 → 100644
      View file @ c46d5c97
      #
      ad_library {
      tsearch2 search engine driver installation procedures
      @author Dave Bauer (dave@thedesignexperience.org)
      @creation-date 2004-06-05
      @arch-tag: 2006927e-7f1d-41c9-a111-ac552a4da185
      @cvs-id $Id$
      }
      namespace eval tsearch2_driver::install {}
      ad_proc -public tsearch2_driver::install::preinstall_checks {
      } {
      Make sure postgresql_contrib and tsearch are installed
      before allowing the installation of tsearch2_package
      @author Hamilton Chua (hchua@8tons.dyndns.biz)
      @creation-date 2004-07-02
      } {
      ns_log Notice " ********** STARTING BEFORE-INSTALL CALLBACK ****************"
      # check if tsearch2 is installed
      # in psql we do this by checking the presense of a data type tsvector
      # select typname from pg_type where typename='tsvector';
      if { [db_0or1row "tsearch_compile_check" "select distinct(typname) from pg_type where typname='tsvector'"] } {
      # if tsearch is installed
      ns_log Notice "******* Tsearch2 is compiled and installed. ***********"
      # continue with installation
      } else {
      # tsearch not installed
      ns_log Notice "******* Tsearch2 is not installed. ***********"
      # RPM, Debian, source default locations
      set locs [list "/usr/share/pgsql/contrib/tsearch2.sql" \
      "/usr/local/pgsql/contrib/tsearch2.sql" \
      "/usr/local/pgsql/share/contrib/tsearch2.sql" \
      "/usr/share/postgresql/contrib/tsearch2.sql"]
      foreach loc $locs {
      if { [file exists $loc] } {
      set sql_file_loc $loc
      break
      }
      }
      # Check if we've found it, run the sql file
      if { [exists_and_not_null sql_file_loc] } {
      # we found tsearch2.sql let's run it
      db_source_sql_file $sql_file_loc
      } else {
      # we could not find tserach2.sql, abort the install
      ns_log Notice "************************************************"
      ns_log Notice "********* Can't locate tsearch2.sql. ***********"
      ns_log Notice "********* Install tsearch2.sql manually ********"
      ns_log Notice "************************************************"
      ad_script_abort
      }
      }
      ns_log Notice " ********** ENDING BEFORE-INSTALL CALLBACK ****************"
      }
      ad_proc -public tsearch2_driver::install::package_install {
      } {
      Installation callback for tsearch2 search engine driver
      @author Dave Bauer (dave@thedesignexperience.org)
      @creation-date 2004-06-05
      @error
      } {
      tsearch2_driver::install::register_fts_impl
      }
      ad_proc -private tsearch2_driver::install::register_fts_impl {
      } {
      Register FtsEngineDriver service contract implementation
      @author Dave Bauer (dave@thedesignexperience.org)
      @creation-date 2004-06-05
      @return
      @error
      } {
      set spec {
      name "tsearch2-driver"
      aliases {
      search tsearch2::search
      index tsearch2::index
      unindex tsearch2::unindex
      update_index tsearch2::update_index
      summary tsearch2::summary
      info tsearch2::driver_info
      }
      contract_name "FtsEngineDriver"
      owner "tsearch2-driver"
      }
      acs_sc::impl::new_from_spec -spec $spec
      }
      tcl/tsearch2-driver-procs-postgresql.xql 0 → 100644
      View file @ c46d5c97
      <queryset>
      <fullquery name="tsearch2::index.index">
      <rdbms><type>postgresql</type><version>8.3</version></rdbms>
      <querytext>
      insert into txt (object_id,fti)
      values (:object_id,
      setweight(to_tsvector(coalesce(:title,'')),'A')
      ||setweight(to_tsvector(coalesce(:keywords,'')),'B')
      ||to_tsvector(coalesce(:txt,'')))
      </querytext>
      </fullquery>
      <fullquery name="callback::search::search::impl::tsearch2-driver.base_query">
      <rdbms><type>postgresql</type><version>8.3</version></rdbms>
      <querytext>
      where fti @@ to_tsquery(:query)
      and exists (select 1
      from acs_object_party_privilege_map m
      where m.object_id = txt.object_id
      and m.party_id = :user_id
      and m.privilege = 'read')
      </querytext>
      </fullquery>
      <fullquery name="callback::search::search::impl::tsearch2-driver.search">
      <rdbms><type>postgresql</type><version>8.3</version></rdbms>
      <querytext>
      select txt.object_id
      from
      [join $from_clauses ","]
      $base_query
      [expr {[llength $where_clauses] > 0 ? " and " : ""}]
      [join $where_clauses " and "]
      order by ts_rank(fti,to_tsquery(:query)) desc
      $limit_clause $offset_clause
      </querytext>
      </fullquery>
      <fullquery name="tsearch2::summary.summary">
      <rdbms><type>postgresql</type><version>8.3</version></rdbms>
      <querytext>
      select ts_headline(:txt,to_tsquery(:query))
      </querytext>
      </fullquery>
      <fullquery name="tsearch2::update_index.update_index">
      <rdbms><type>postgresql</type><version>8.3</version></rdbms>
      <querytext>
      update txt set fti =
      setweight(to_tsvector(coalesce(:title,'')),'A')
      ||setweight(to_tsvector(coalesce(:keywords,'')),'B')
      ||to_tsvector(coalesce(:txt,''))
      where object_id=:object_id
      </querytext>
      </fullquery>
      <fullquery name="tsearch2::index.index">
      <rdbms><type>postgresql</type><version>8.0</version></rdbms>
      <querytext>
      insert into txt (object_id,fti)
      values (:object_id,
      setweight(to_tsvector('default',coalesce(:title,'')),'A')
      ||setweight(to_tsvector('default',coalesce(:keywords,'')),'B')
      ||to_tsvector('default',coalesce(:txt,'')))
      </querytext>
      </fullquery>
      <fullquery name="callback::search::search::impl::tsearch2-driver.base_query">
      <rdbms><type>postgresql</type><version>8.0</version></rdbms>
      <querytext>
      where fti @@ to_tsquery('default',:query)
      and exists (select 1
      from acs_object_party_privilege_map m
      where m.object_id = txt.object_id
      and m.party_id = :user_id
      and m.privilege = 'read')
      </querytext>
      </fullquery>
      <fullquery name="callback::search::search::impl::tsearch2-driver.search">
      <rdbms><type>postgresql</type><version>8.0</version></rdbms>
      <querytext>
      select txt.object_id
      from
      [join $from_clauses ","]
      $base_query
      [expr {[llength $where_clauses] > 0 ? " and " : ""}]
      [join $where_clauses " and "]
      order by rank(fti,to_tsquery('default',:query)) desc
      $limit_clause $offset_clause
      </querytext>
      </fullquery>
      <fullquery name="tsearch2::summary.summary">
      <rdbms><type>postgresql</type><version>8.0</version></rdbms>
      <querytext>
      select headline('default',:txt,to_tsquery('default',:query))
      </querytext>
      </fullquery>
      <fullquery name="tsearch2::update_index.update_index">
      <rdbms><type>postgresql</type><version>8.0</version></rdbms>
      <querytext>
      update txt set fti =
      setweight(to_tsvector('default',coalesce(:title,'')),'A')
      ||setweight(to_tsvector('default',coalesce(:keywords,'')),'B')
      ||to_tsvector('default',coalesce(:txt,''))
      where object_id=:object_id
      </querytext>
      </fullquery>
      </queryset>
      tcl/tsearch2-driver-procs.tcl 0 → 100644
      View file @ c46d5c97
      This diff is collapsed.
      tsearch2-driver.info 0 → 100644
      View file @ c46d5c97
      <?xml version="1.0"?>
      <!-- Generated by the OpenACS Package Manager -->
      <package key="tsearch2-driver" url="http://openacs.org/repository/apm/packages/tsearch2-driver" type="apm_service">
      <package-name>Tsearch2 Driver</package-name>
      <pretty-plural>Tsearch2 Drivers</pretty-plural>
      <initial-install-p>f</initial-install-p>
      <singleton-p>t</singleton-p>
      <version name="5.6.0" url="http://openacs.org/repository/download/apm/tsearch2-driver-5.6.0.apm">
      <owner url="mailto:dave@thedesignexperience.org">Dave Bauer</owner>
      <summary>Full text search support for Search via Tsearch2.</summary>
      <release-date>2010-06-17</release-date>
      <description format="text/html">Provides an implementation for the search packages service contract for searching on postgresql.</description>
      <maturity>3</maturity>
      <provides url="tsearch2-driver" version="5.6.0"/>
      <requires url="search" version="5.6.0"/>
      <callbacks>
      <callback type="before-install" proc="tsearch2_driver::install::preinstall_checks"/>
      <callback type="after-install" proc="tsearch2_driver::install::package_install"/>
      </callbacks>
      <parameters>
      <parameter datatype="number" min_n_values="1" max_n_values="1" name="permission_check_enabled_p" default="1" description="Check whether the user is allowed to read the object. This is used to filter the search results w.r.t the privileges of the user performing the query."/>
      <parameter datatype="number" min_n_values="1" max_n_values="1" name="max_size_to_index" default="0" description="Maximum size of doc to index, in bytes. 0 to index the full document. If the doc is greater than this parameter, the indexing is truncated."/>
      </parameters>
      </version>
      </package>
      www/doc/index.html 0 → 100644
      View file @ c46d5c97
      <html> <head> <title>Tsearch2 Full-text Search Engine Driver for OpenACS
      5.x</title> </head> <h1>Tsearch2 Full-text Search Engine Driver for OpenACS 5.x</h1>
      <p>Tsearch2 Driver provides full-text searching of a PostgreSQL database by
      using PostgreSQL's tsearch2 FtsEngineDriver</p>
      <h2>Requirements for this search implementation</h2>
      <ul><li>OpenACS 5.x
      </li><li>PostgreSQL 7.3 or newer
      </li><li>PostgreSQL's <a href="http://openacs.org/xowiki/pages/en/postgresql-tsearch2">tsearch2 module installed</a>. (Pg versions 7.3 and 7.4 require a patch and tsearch2.sql to be
      loaded into the database)
      </li><li>This package installed
      </li><li>search package to be mounted somewhere.
      </li><li>FtsEngineDriver parameter of search package set to
      "tsearch2-driver".
      </li><li>indexing some data
      </li></ul>
      <h2><a name="install-fts-engine"></a>Install OpenACS' Tsearch2 Full-Text Search Package</h2>
      <ol type="1">
      <li>
      <p>
      If you have not yet, install <a href="http://openacs.org/xowiki/pages/en/postgresql-tsearch2">PostgreSQL's tsearch2 module</a>.
      </p>
      </li>
      <li>
      <p>
      Click <code class="computeroutput"><span class="guilabel"><span
      class="guilabel">Admin</span></span></code> on the top of the
      default home page. If prompted, log in with the account and
      password you entered during install.
      </p>
      </li>
      <li>
      <p>
      Click on the <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Install
      software</span></span></code> link.
      </p>
      </li>
      <li>
      <p>
      Click on the <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Install new
      service</span></span></code> link.
      </p>
      </li>
      <li>
      <p>
      Click on the <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">Install</span></span></code> link next to
      Tsearch2 Driver. If you have installed tsearch2 into your
      PostgreSQL database, the installer will automatically enable
      tsearch in your OpenACS database instance.
      </p>
      </li>
      <li>
      <p>
      Restart AOLserver. Wait a minute, then browse back to the home page.
      </p>
      </li>
      <li>
      <p>
      Click on <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">Admin</span></span></code> on the top of the
      screen.
      </p>
      </li>
      <li>
      <p>
      Click on <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Main Site
      Administration</span></span></code> in the "Subsite
      Administration" section.
      </p>
      </li>
      <li>
      <p>
      Click on <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Site
      Map</span></span></code> in the "Advanced Features" section.
      </p>
      </li>
      <li>
      <p>
      Mount the Search interface in the site-map.
      </p>
      <ol type="a">
      <li>
      <p>
      Click the <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">new sub
      folder</span></span></code> link on the Main Site line.
      </p>
      </li>
      <li>
      <p>
      Type <strong class="userinput"><code>search</code></strong>
      and click <code class="computeroutput"><span
      class="guibutton"><span
      class="guibutton">New</span></span></code>.
      </p>
      </li>
      <li>
      <p>
      Click the <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">new
      application</span></span></code> link on the <code
      class="computeroutput"><span class="guilabel"><span
      class="guilabel">search</span></span></code> line.
      </p>
      </li>
      <li>
      <p>
      Type <strong class="userinput"><code>search</code></strong>
      where it says <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">untitled</span></span></code>, choose <code
      class="computeroutput"><span class="guilabel"><span
      class="guilabel">search</span></span></code> from the
      drop-down list, and click <code class="computeroutput"><span
      class="guibutton"><span
      class="guibutton">New</span></span></code>.
      </p>
      </li>
      <li>
      <p>
      Click the <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">Parameters</span></span></code> link next
      to the Search package instance.
      </p>
      </li>
      <li>
      <p>
      Type <strong
      class="userinput"><code>tsearch2-driver</code></strong>
      where it says <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">openfts-driver</span></span></code> in the
      <code class="computeroutput"><span class="guilabel"><span
      class="guilabel">FtsEngineDriver</span></span></code>
      parameter.
      </p>
      </li>
      </ol>
      </li>
      <li>
      <p>
      Restart AOLserver.
      Wait a minute, then click on <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Main
      Site</span></span></code> at the top of the page.
      </p>
      </li>
      </ol>
      <h3 class="title"><a name="install-fts-content-provider"></a>Enable
      Full Text Search in packages</h3>
      <p>
      Weblogger (lars-blogger), ETP (edit-this-page), and a few other packages have code to generate indexed
      content. We are using lars-blogger to illustrate how to enable Full Text Search in packages.
      </p>
      <ol type="1">
      <li>
      <p>
      Install the lars-blogger package, if it is not yet installed.
      </p>
      <ol type="a">
      <li>
      <p>
      Click <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">Admin</span></span></code> on the top of
      the default home page. If prompted, log in with the account
      and password you entered during install.
      </p>
      </li>
      <li>
      <p>
      Click on the <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Install
      software</span></span></code> link.
      </p>
      </li>
      <li>
      <p>
      Click on the <code class="computeroutput"><span
      class="guilabel"><span class="guilabel">Install new
      application</span></span></code> link.
      </p>
      </li>
      <li>
      <p>
      Click on the <code class="computeroutput"><span
      class="guilabel"><span
      class="guilabel">Install</span></span></code> link next to
      Weblogger.
      </p>
      </li>
      <li>
      <p>
      Install all required packages as well (always say okay until
      you shall restart the server)
      </p>
      </li>
      </ol>
      <p>
      </p>
      </li>
      <li>
      <p>
      Loading the service contracts datamodel and enabling the service
      contract usually happens when the package is installed.
      However, Lars-blogger may require manually loading lars-blogger-sc-create.sql to get
      it to register the service contract implementation that indexes
      the content:
      </p>
      <pre class="screen">
      [$OPENACS_SERVICE_NAME $OPENACS_SERVICE_NAME]$ <strong class="userinput"><code>cd packages/lars-blogger/sql/postgresql</code></strong>
      [$OPENACS_SERVICE_NAME postgresql]$ <strong class="userinput"><code>psql $OPENACS_SERVICE_NAME -f lars-blogger-sc-create.sql</code></strong>
      </pre>
      </li>
      <li>
      <p>
      Restart AOLserver.
      </p>
      </li>
      </ol>
      <p>
      Full Text Search should be enabled now, if not consult <a
      href="http://openacs.org/forums/message-view?message_id=154759">http://openacs.org/forums/message-view?message_id=154759</a>.
      This link also contains some hints on how to make sure it is
      enabled.
      </p>
      <h2>Indexing data</h2>
      <p>
      Once tsearch2-driver is installed, add some content to be indexed.
      </p>
      <h3>Adding search indexing to packages</h3>
      <p>Standard coding practice is to put indexing code in
      package-key/sql/postgresql/package-key-sc-create.sql. View these
      examples for how to implement:</p>
      <ul><li><a href="http://cvs.openacs.org/cvs/openacs-4/packages/edit-this-page/sql/postgresql/edit-this-page-sc-create.sql?view=markup">packages/edit-this-page/sql/postgresql/edit-this-page-sc-create.sql</a>
      </li><li><a href="http://cvs.openacs.org/cvs/openacs-4/packages/lars-blogger/sql/postgresql/lars-blogger-sc-create.sql?view=markup">packages/lars-blogger/sql/postgresql/lars-blogger-sc-create.sql</a>
      </li></ul>
      </p>
      <h2>Indexing pre-existing content that has been indexed before</h2>
      <p>If your pre-existing content has been indexed before (e.g. because the
      search package was mounted before as part of a previous search
      service), you have to tell the search package to reindex:</p>
      <pre>
      insert into search_observer_queue (
      select <i>my_id</i>, now(),'INSERT' from <i>my_table</i>
      );
      </pre><p>
      For forums and ETP this looks like:
      </p>
      <pre>
      insert into search_observer_queue (
      select message_id, now(), 'INSERT' from forums_messages
      );
      insert into search_observer_queue (
      select live_revision, now(), 'INSERT' from (
      select live_revision from cr_items where content_type = 'etp_page_revision'
      )
      etp );
      </pre>
      <h2>Implementation notes</h2>
      <p>
      This version includes only the most basic features.
      Many options are possible by adding admin configurable parameters.
      The current service contract definitions are not
      flexible enough to work well with every possible search driver, so
      some features may require making some improvements to the search package also.
      </p>
      <p>Dave Bauer dave@thedesignexperience.org 2004-06-05</p>
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment