Skip to content

T
tsearch2-driver
Commit f187bbb6 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

- Update to OpenACS 5.9.1

parent bf949b34
Hide whitespace changes
Inline Side-by-side
Showing with 257 additions and 95 deletions
sql/postgresql/tsearch2-driver-package-create.sql
View file @ f187bbb6
create or replace function ts2_to_tsvector ( varchar, varchar ) returns varchar as '
declare
ts2_cfg alias for $1;
ts2_txt alias for $2;
--
-- procedure ts2_to_tsvector/2
--
CREATE OR REPLACE FUNCTION ts2_to_tsvector (
ts2_cfg varchar,
ts2_txt varchar
) RETURNS varchar AS $$
DECLARE
ts2_result varchar;
begin
BEGIN
perform set_curcfg(ts2_cfg);
select to_tsvector(ts2_cfg,ts2_txt) into ts2_result;
return ts2_result;
END;
$$ language plpgsql;
perform set_curcfg(ts2_cfg);
select to_tsvector(ts2_cfg,ts2_txt) into ts2_result;
return ts2_result;
end;' language 'plpgsql';
create or replace function ts2_to_tsquery ( varchar, varchar ) returns tsquery as '
declare
ts2_cfg alias for $1;
ts2_txt alias for $2;
--
-- procedure ts2_to_tsquery/2
--
CREATE OR REPLACE FUNCTION ts2_to_tsquery (
ts2_cfg varchar,
ts2_txt varchar
) RETURNS tsquery AS $$
DECLARE
ts2_result tsquery;
begin
perform set_curcfg(ts2_cfg);
select 1 into ts2_result;
select to_tsquery(ts2_cfg,ts2_txt) into ts2_result;
return ts2_result;
end;' language 'plpgsql';
BEGIN
perform set_curcfg(ts2_cfg);
select 1 into ts2_result;
select to_tsquery(ts2_cfg,ts2_txt) into ts2_result;
return ts2_result;
END;
$$ LANGUAGE plpgsql;
tcl/tsearch2-driver-install-procs.tcl
View file @ f187bbb6
......@@ -28,15 +28,15 @@ ad_proc -public tsearch2_driver::install::preinstall_checks {
ns_log Notice " ********** STARTING BEFORE-INSTALL CALLBACK ****************"
# check if tsearch2 is installed
# in psql we do this by checking the presense of a data type tsvector
# in psql we do this by checking the presence of a data type tsvector
# select typname from pg_type where typename='tsvector';
if { [db_0or1row "tsearch_compile_check" "select distinct(typname) from pg_type where typname='tsvector'"] } {
# if tsearch is installed
ns_log Notice "******* Tsearch2 is compiled and installed. ***********"
# continue with installation
if { [db_0or1row tsearch_compile_check {
select distinct(typname) from pg_type where typname='tsvector'
}] } {
# if tsearch is installed
ns_log Notice "******* Tsearch2 is compiled and installed. ***********"
# continue with installation
} else {
# tsearch not installed
......@@ -54,7 +54,7 @@ ad_proc -public tsearch2_driver::install::preinstall_checks {
}
}
# Check if we've found it, run the sql file
if { [exists_and_not_null sql_file_loc] } {
if { ([info exists sql_file_loc] && $sql_file_loc ne "") } {
# we found tsearch2.sql let's run it
db_source_sql_file $sql_file_loc
} else {
......
tcl/tsearch2-driver-procs-postgresql.xql
View file @ f187bbb6
......@@ -11,15 +11,10 @@
</querytext>
</fullquery>
<fullquery name="callback::search::search::impl::tsearch2-driver.base_query">
<rdbms><type>postgresql</type><version>8.3</version></rdbms>
<fullquery name="callback::search::search::impl::tsearch2-driver.base_query">
<rdbms><type>postgresql</type><version>8.4</version></rdbms>
<querytext>
where fti @@ to_tsquery(:query)
and exists (select 1
from acs_object_party_privilege_map m
where m.object_id = txt.object_id
and m.party_id = :user_id
and m.privilege = 'read')
</querytext>
</fullquery>
......@@ -27,8 +22,7 @@
<rdbms><type>postgresql</type><version>8.3</version></rdbms>
<querytext>
select txt.object_id
from
[join $from_clauses ","]
from [join $from_clauses ","]
$base_query
[expr {[llength $where_clauses] > 0 ? " and " : ""}]
[join $where_clauses " and "]
......@@ -37,15 +31,40 @@
</querytext>
</fullquery>
<fullquery name="callback::search::search::impl::tsearch2-driver.search">
<rdbms><type>postgresql</type><version>8.4</version></rdbms>
<querytext>
select distinct(orig_object_id) from acs_permission.permission_p_recursive_array(array(
select txt.object_id
from [join $from_clauses ","]
$base_query
[expr {[llength $where_clauses] > 0 ? " and [join $where_clauses { and }]" : ""}]
order by ts_rank(fti,to_tsquery(:query)) desc
), :user_id, 'read')
$limit_clause $offset_clause
</querytext>
</fullquery>
<fullquery name="callback::search::search::impl::tsearch2-driver.count">
<rdbms><type>postgresql</type><version>8.3</version></rdbms>
<querytext>
select count(*)
from
[join $from_clauses ","]
from [join $from_clauses ","]
$base_query
[expr {[llength $where_clauses] > 0 ? " and [join $where_clauses { and }]" : ""}]
</querytext>
</fullquery>
<fullquery name="dbqd.tsearch2-driver.tcl.tsearch2-driver-procs.callback::search::search::impl::tsearch2-driver.search_result_count">
<rdbms><type>postgresql</type><version>8.4</version></rdbms>
<querytext>
select count(distinct(orig_object_id)) from acs_permission__permission_p_recursive_array(array(
select txt.object_id
from [join $from_clauses ","]
$base_query
[expr {[llength $where_clauses] > 0 ? " and " : ""}]
[join $where_clauses " and "]
), :user_id, 'read')
</querytext>
</fullquery>
......@@ -78,18 +97,6 @@
</querytext>
</fullquery>
<fullquery name="callback::search::search::impl::tsearch2-driver.base_query">
<rdbms><type>postgresql</type><version>8.0</version></rdbms>
<querytext>
where fti @@ to_tsquery('default',:query)
and exists (select 1
from acs_object_party_privilege_map m
where m.object_id = txt.object_id
and m.party_id = :user_id
and m.privilege = 'read')
</querytext>
</fullquery>
<fullquery name="callback::search::search::impl::tsearch2-driver.search">
<rdbms><type>postgresql</type><version>8.0</version></rdbms>
<querytext>
......
tcl/tsearch2-driver-procs.tcl
View file @ f187bbb6
......@@ -84,13 +84,21 @@ ad_proc -public tsearch2::update_index {
}
}
#ad_proc -callback search::search -impl tsearch2-driver {
# {-extra_args {}}
# query
# offset
# limit
# user_id
# df
#}
ad_proc -callback search::search -impl tsearch2-driver {
{-extra_args {}}
query
offset
limit
user_id
df
-query
-offset
-limit
-user_id
-df
} {
ftsenginedriver search operation implementation for tsearch2
......@@ -98,19 +106,13 @@ ad_proc -callback search::search -impl tsearch2-driver {
@creation-date 2004-06-05
@param query
@param offset
@param limit
@param user_id
@param df
@param packages list of packages to search for content in.
@return
@error
} {
set packages {}
......@@ -125,6 +127,10 @@ ad_proc -callback search::search -impl tsearch2-driver {
# clean up query for tsearch2
set query [tsearch2::build_query -query $query]
set where_clauses ""
set from_clauses ""
if {![info exists user_id]} {set user_id 0}
# don't use bind vars since pg7.3 yacks for '1' (which is what comes out of bind vars)
set limit_clause ""
set offset_clause ""
......@@ -139,11 +145,11 @@ ad_proc -callback search::search -impl tsearch2-driver {
set base_query [db_map base_query]
if {$df ne ""} {
set need_acs_objects 1
lappend where_clauses " and o.creation_date > :df"
lappend where_clauses "o.creation_date > :df"
}
if {$dt ne ""} {
set need_acs_objects 1
lappend where_clauses " and o.creation_date < :dt"
lappend where_clauses "o.creation_date < :dt"
}
foreach {arg value} $extra_args {
......@@ -152,7 +158,7 @@ ad_proc -callback search::search -impl tsearch2-driver {
lappend from_clauses $arg_clauses(from_clause)
}
if {[info exists arg_clauses(where_clause)] && $arg_clauses(where_clause) ne ""} {
lappend where_clauses "$arg_clauses(where_clause)"
lappend where_clauses $arg_clauses(where_clause)
}
}
if {[llength $extra_args]} {
......@@ -198,11 +204,9 @@ ad_proc -public tsearch2::summary {
@creation-date 2004-06-05
@param query
@param txt
@return summary containing search query terms
@error
} {
set query [tsearch2::build_query -query $query]
......@@ -244,21 +248,31 @@ ad_proc tsearch2::build_query { -query } {
regsub -all {[^-/@.\d\w\s\(\)]+} $query { } query
# match parens, if they don't match just throw them away
set p 0
for {set i 0} {$i < [string length $query]} {incr i} {
if {[string index $query $i] eq "("} {
incr p
}
if {[string index $query $i] eq ")"} {
incr p -1
}
}
if {$p != 0} {
regsub -all {\(|\)} $query {} query
}
# remove or at beginning of query
# set p 0
# for {set i 0} {$i < [string length $query]} {incr i} {
# if {[string index $query $i] eq "("} {
# incr p
# }
# if {[string index $query $i] eq ")"} {
# incr p -1
# }
# }
# if {$p != 0} {
# regsub -all {\(|\)} $query {} query
# }
# remove all parens
regsub -all {\(|\)} $query {} query
# remove empty ()
regsub -all {\(\s*\)} $query {} query
# remove "or" at beginning of query
regsub -nocase "^or " $query {} query
# remove "not" at end of query
regsub -nocase " not$" $query {} query
# replace boolean words with boolean operators
regsub -nocase "^not " $query {!} query
set query [string map {" and " " & " " or " " | " " not " " ! "} " $query "]
......@@ -272,7 +286,7 @@ ad_proc tsearch2::build_query { -query } {
regsub {([-/@.\d\w\(\)])\s+?([-/@.\d\w\(\)])} $query {\1 \& \2} query
}
# if a ! is by itself then prepend &
regsub {(\w+?)\s*(!)} $query {\1 \& !} query
regsub -all {(\w+?)\s*(!)} $query {\1 \& !} query
# if there is )( then insert an & between them
# or if there is )\w or \w( insert an & between them
regsub {(\))([\(\w])} $query {\1 \& \2} query
......@@ -283,10 +297,10 @@ ad_proc tsearch2::build_query { -query } {
return $query
}
ad_proc -public tsearch2::seperate_query_and_operators {
ad_proc -public tsearch2::separate_query_and_operators {
-query
} {
Seperates special operators from full text query
Separates special operators from full text query
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2004-07-10
......@@ -303,7 +317,7 @@ ad_proc -public tsearch2::seperate_query_and_operators {
# match quotes
set quote_count [regexp -all {\"} $query]
# if quotes don't match, just remove all of them
if {[expr {$quote_count % 2}] == 1} {
if {$quote_count % 2 == 1} {
regsub -all {\"} $query {} query
}
......@@ -314,8 +328,9 @@ ad_proc -public tsearch2::seperate_query_and_operators {
set end_q 0
set valid_operators [tsearch2_driver::valid_operators]
foreach e [split $query] {
if {[regexp {(^\w*):} $e discard operator] \
&& [lsearch -exact $valid_operators $operator] != -1} {
if {[regexp {(^\w*):} $e discard operator]
&& $operator in $valid_operators
} {
# query element contains an operator, split operator from
# query fragment
set e [split $e ":"]
......@@ -355,7 +370,10 @@ ad_proc -public tsearch2::seperate_query_and_operators {
ns_log debug "operator(e)='${e}' start_q=$start_q end_q=$end_q"
if {$last_operator ne ""} {
# FIXME need introspection for operator phrase support
if {($last_operator eq "title:" || $last_operator eq "description:") && ($start_q || $end_q)} {
if {
($last_operator eq "title:" || $last_operator eq "description:")
&& ($start_q || $end_q)
} {
lappend ${last_operator}_phrase [regsub -all {\"} $e {}]
} else {
lappend $last_operator [regsub -all {\"} ${e} {}]
......@@ -370,7 +388,7 @@ ad_proc -public tsearch2::seperate_query_and_operators {
}
foreach op $valid_operators {
if {[exists_and_not_null $op]} {
if {([info exists $op] && [set $op] ne "")} {
lappend operators $op $title
}
}
......
tsearch2-driver.info
View file @ f187bbb6
......@@ -7,15 +7,15 @@
<initial-install-p>f</initial-install-p>
<singleton-p>t</singleton-p>
<version name="5.7.0" url="http://openacs.org/repository/download/apm/tsearch2-driver-5.7.0.apm">
<version name="5.9.1" url="http://openacs.org/repository/download/apm/tsearch2-driver-5.9.1.apm">
<owner url="mailto:dave@thedesignexperience.org">Dave Bauer</owner>
<summary>Full text search support for Search via Tsearch2.</summary>
<release-date>2011-06-12</release-date>
<release-date>2017-08-06</release-date>
<description format="text/html">Provides an implementation for the search packages service contract for searching on postgresql.</description>
<maturity>3</maturity>
<provides url="tsearch2-driver" version="5.7.0"/>
<requires url="search" version="5.7.0"/>
<provides url="tsearch2-driver" version="5.9.1"/>
<requires url="search" version="5.9.1"/>
<callbacks>
<callback type="before-install" proc="tsearch2_driver::install::preinstall_checks"/>
......
www/doc/index.adp 0 → 100644
View file @ f187bbb6
<property name="context">{/doc/tsearch2-driver {Tsearch2 Driver}} {Tsearch2 Full-text Search Engine Driver for OpenACS
5.x}</property>
<property name="doc(title)">Tsearch2 Full-text Search Engine Driver for OpenACS
5.x</property>
<master>
<h1>Tsearch2 Full-text Search Engine Driver for OpenACS 5.x</h1>
<p>Tsearch2 Driver provides full-text searching of a PostgreSQL
database by using PostgreSQL&#39;s tsearch2 FtsEngineDriver</p>
<h2>Requirements for this search implementation</h2>
<ul>
<li>OpenACS 5.x</li><li>PostgreSQL 7.3 or newer</li><li>PostgreSQL&#39;s <a href="http://openacs.org/xowiki/pages/en/postgresql-tsearch2">tsearch2
module installed</a>. (Pg versions 7.3 and 7.4 require a patch and
tsearch2.sql to be loaded into the database)</li><li>This package installed</li><li>search package to be mounted somewhere.</li><li>FtsEngineDriver parameter of search package set to
"tsearch2-driver".</li><li>indexing some data</li>
</ul>
<h2>
<a name="install-fts-engine" id="install-fts-engine"></a>Install OpenACS' Tsearch2 Full-Text
Search Package</h2>
<ol type="1">
<li><p>If you have not yet, install <a href="http://openacs.org/xowiki/pages/en/postgresql-tsearch2">PostgreSQL&#39;s
tsearch2 module</a>.</p></li><li><p>Click <code class="computeroutput"><span class="guilabel"><span class="guilabel">Admin</span></span></code> on the
top of the default home page. If prompted, log in with the account
and password you entered during install.</p></li><li><p>Click on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Install
software</span></span></code> link.</p></li><li><p>Click on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Install new
service</span></span></code> link.</p></li><li><p>Click on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Install</span></span></code> link
next to Tsearch2 Driver. If you have installed tsearch2 into your
PostgreSQL database, the installer will automatically enable
tsearch in your OpenACS database instance.</p></li><li><p>Restart AOLserver. Wait a minute, then browse back to the home
page.</p></li><li><p>Click on <code class="computeroutput"><span class="guilabel"><span class="guilabel">Admin</span></span></code> on the
top of the screen.</p></li><li><p>Click on <code class="computeroutput"><span class="guilabel"><span class="guilabel">Main Site
Administration</span></span></code> in the "Subsite
Administration" section.</p></li><li><p>Click on <code class="computeroutput"><span class="guilabel"><span class="guilabel">Site Map</span></span></code> in
the "Advanced Features" section.</p></li><li>
<p>Mount the Search interface in the site-map.</p><ol type="a">
<li><p>Click the <code class="computeroutput"><span class="guilabel"><span class="guilabel">new sub
folder</span></span></code> link on the Main Site line.</p></li><li><p>Type <strong class="userinput"><code>search</code></strong> and
click <code class="computeroutput"><span class="guibutton"><span class="guibutton">New</span></span></code>.</p></li><li><p>Click the <code class="computeroutput"><span class="guilabel"><span class="guilabel">new
application</span></span></code> link on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">search</span></span></code> line.</p></li><li><p>Type <strong class="userinput"><code>search</code></strong>
where it says <code class="computeroutput"><span class="guilabel"><span class="guilabel">untitled</span></span></code>,
choose <code class="computeroutput"><span class="guilabel"><span class="guilabel">search</span></span></code> from
the drop-down list, and click <code class="computeroutput"><span class="guibutton"><span class="guibutton">New</span></span></code>.</p></li><li><p>Click the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Parameters</span></span></code>
link next to the Search package instance.</p></li><li><p>Type <strong class="userinput"><code>tsearch2-driver</code></strong> where it says
<code class="computeroutput"><span class="guilabel"><span class="guilabel">openfts-driver</span></span></code> in the <code class="computeroutput"><span class="guilabel"><span class="guilabel">FtsEngineDriver</span></span></code> parameter.</p></li>
</ol>
</li><li><p>Restart AOLserver. Wait a minute, then click on <code class="computeroutput"><span class="guilabel"><span class="guilabel">Main
Site</span></span></code> at the top of the page.</p></li>
</ol>
<h3 class="title">
<a name="install-fts-content-provider" id="install-fts-content-provider"></a>Enable Full Text Search in
packages</h3>
<p>Weblogger (lars-blogger), ETP (edit-this-page), and a few other
packages have code to generate indexed content. We are using
lars-blogger to illustrate how to enable Full Text Search in
packages.</p>
<ol type="1">
<li>
<p>Install the lars-blogger package, if it is not yet
installed.</p><ol type="a">
<li><p>Click <code class="computeroutput"><span class="guilabel"><span class="guilabel">Admin</span></span></code> on the
top of the default home page. If prompted, log in with the account
and password you entered during install.</p></li><li><p>Click on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Install
software</span></span></code> link.</p></li><li><p>Click on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Install new
application</span></span></code> link.</p></li><li><p>Click on the <code class="computeroutput"><span class="guilabel"><span class="guilabel">Install</span></span></code> link
next to Weblogger.</p></li><li><p>Install all required packages as well (always say okay until you
shall restart the server)</p></li>
</ol>
</li><li>
<p>Loading the service contracts datamodel and enabling the service
contract usually happens when the package is installed. However,
Lars-blogger may require manually loading
lars-blogger-sc-create.sql to get it to register the service
contract implementation that indexes the content:</p><pre class="screen">
[$OPENACS_SERVICE_NAME $OPENACS_SERVICE_NAME]$ <strong class="userinput"><code>cd packages/lars-blogger/sql/postgresql</code></strong>
[$OPENACS_SERVICE_NAME postgresql]$ <strong class="userinput"><code>psql $OPENACS_SERVICE_NAME -f lars-blogger-sc-create.sql</code></strong>
</pre>
</li><li><p>Restart AOLserver.</p></li>
</ol>
<p>Full Text Search should be enabled now, if not consult <a href="http://openacs.org/forums/message-view?message_id=154759">http://openacs.org/forums/message-view?message_id=154759</a>.
This link also contains some hints on how to make sure it is
enabled.</p>
<h2>Indexing data</h2>
<p>Once tsearch2-driver is installed, add some content to be
indexed.</p>
<h3>Adding search indexing to packages</h3>
<p>Standard coding practice is to put indexing code in
package-key/sql/postgresql/package-key-sc-create.sql. View these
examples for how to implement:</p>
<ul>
<li><a href="https://github.com/openacs/edit-this-page/blob/master/sql/postgresql/edit-this-page-create.sql">
packages/edit-this-page/sql/postgresql/edit-this-page-sc-create.sql</a></li><li><a href="https://github.com/openacs/lars-blogger/blob/master/sql/postgresql/lars-blogger-sc-create.sql">
packages/lars-blogger/sql/postgresql/lars-blogger-sc-create.sql</a></li>
</ul>
<h2>Indexing pre-existing content that has been indexed before</h2>
<p>If your pre-existing content has been indexed before (e.g.
because the search package was mounted before as part of a previous
search service), you have to tell the search package to
reindex:</p>
<pre>
insert into search_observer_queue (
select <em>my_id</em>, now(),'INSERT' from <em>my_table</em>
);
</pre>
<p>For forums and ETP this looks like:</p>
<pre>
insert into search_observer_queue (
select message_id, now(), 'INSERT' from forums_messages
);
insert into search_observer_queue (
select live_revision, now(), 'INSERT' from (
select live_revision from cr_items where content_type = 'etp_page_revision'
)
etp );
</pre>
<h2>Implementation notes</h2>
<p>This version includes only the most basic features. Many options
are possible by adding admin configurable parameters. The current
service contract definitions are not flexible enough to work well
with every possible search driver, so some features may require
making some improvements to the search package also.</p>
<h2>Release Notes</h2>
<p>Please file bugs in the <a href="http://openacs.org/bugtracker/openacs/">Bug Tracker</a>.</p>
<p>Dave Bauer dave\@thedesignexperience.org 2004-06-05</p>
www/doc/index.html
View file @ f187bbb6
......@@ -243,8 +243,8 @@ Full Text Search in packages</h3>
<p>Standard coding practice is to put indexing code in
package-key/sql/postgresql/package-key-sc-create.sql. View these
examples for how to implement:</p>
<ul><li><a href="http://cvs.openacs.org/cvs/openacs-4/packages/edit-this-page/sql/postgresql/edit-this-page-sc-create.sql?view=markup">packages/edit-this-page/sql/postgresql/edit-this-page-sc-create.sql</a>
</li><li><a href="http://cvs.openacs.org/cvs/openacs-4/packages/lars-blogger/sql/postgresql/lars-blogger-sc-create.sql?view=markup">packages/lars-blogger/sql/postgresql/lars-blogger-sc-create.sql</a>
<ul><li><a href="https://github.com/openacs/edit-this-page/blob/master/sql/postgresql/edit-this-page-create.sql">packages/edit-this-page/sql/postgresql/edit-this-page-sc-create.sql</a>
</li><li><a href="https://github.com/openacs/lars-blogger/blob/master/sql/postgresql/lars-blogger-sc-create.sql">packages/lars-blogger/sql/postgresql/lars-blogger-sc-create.sql</a>
</li></ul>
</p>
......@@ -278,4 +278,8 @@ For forums and ETP this looks like:
flexible enough to work well with every possible search driver, so
some features may require making some improvements to the search package also.
</p>
<h2>Release Notes</h2>
<p>Please file bugs in the <a href="http://openacs.org/bugtracker/openacs/">Bug Tracker</a>.</p>
<p>Dave Bauer dave@thedesignexperience.org 2004-06-05</p>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment