Commit 73227396 authored by Frank Bergmann's avatar Frank Bergmann

Initial Import

parents
Pipeline #706 failed with stages
<?xml version="1.0"?>
<!-- Generated by the OpenACS Package Manager -->
<package key="intermedia-driver" url="http://openacs.org/repository/apm/packages/intermedia-driver" type="apm_service">
<package-name>Intermedia Driver</package-name>
<pretty-plural>Intermedia Driver</pretty-plural>
<initial-install-p>f</initial-install-p>
<singleton-p>t</singleton-p>
<version name="5.6.0" url="http://openacs.org/repository/download/apm/intermedia-driver-5.6.0.apm">
<owner url="mailto:dave@thedesignexperience.org">David Bauer</owner>
<owner url="mailto:dirk@dirkgomez.de">Dirk Gomez</owner>
<summary>Search package callback implementation for Oracle Intermedia.</summary>
<release-date>2010-06-17</release-date>
<description format="text/html">Implemented the search package callback interface for search engine driver using Oracle Intermedia.</description>
<maturity>3</maturity>
<provides url="intermedia-driver" version="5.6.0"/>
<requires url="search" version="5.6.0"/>
<callbacks>
</callbacks>
<parameters>
<!-- No version parameters -->
</parameters>
</version>
</package>
@@ intermedia-driver-tables-create.sql
@@ intermedia-driver-index-create.sql
@@ intermedia-driver-packages-create.sql
--
-- Copyright (C) 2005 MIT
--
-- This file is part of dotLRN.
--
-- dotLRN is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 2 of the License, or (at your option) any later
-- version.
--
-- dotLRN is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-- FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
-- details.
--
--
-- Create ctxsys schema objects for .LRN site-wide search
--
-- @author <a href="mailto:openacs@dirkgomez.de">openacs@dirkgomez.de</a>
-- @version $Id$
-- @creation-date 13-May-2005
--
-- Partly ported from ACES.
CREATE OR replace procedure sws_user_proc_&1 ( rid IN ROWID, tlob IN OUT nocopy clob )
AS
BEGIN
&1..sws_user_datastore_proc(rid, tlob);
END;
/
show errors;
grant execute on sws_user_proc_&1 to &1;
grant ctxapp to &1;
-- stuff to make interMedia faster
exec ctx_adm.set_parameter('max_index_memory', '1G');
exit;
@@ intermedia-driver-packages-drop.sql
@@ intermedia-driver-index-drop.sql
@@ intermedia-driver-tables-drop.sql
--
-- Copyright (C) 2005 MIT
--
-- This file is part of dotLRN.
--
-- dotLRN is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 2 of the License, or (at your option) any later
-- version.
--
-- dotLRN is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-- FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
-- details.
--
--
-- Create the intermedia index for .LRN site-wide search
--
-- @author <a href="mailto:openacs@dirkgomez.de">openacs@dirkgomez.de</a>
-- @version $Id$
-- @creation-date 13-May-2005
--
-- Partly ported from ACES.
-- create section groups for within clauses
begin
ctx_ddl.create_section_group('swsgroup', 'basic_section_group');
ctx_ddl.add_field_section('swsgroup', 'oneline', 'oneline', TRUE);
end;
/
create index swi_index on site_wide_index (indexed_content)
indextype is ctxsys.context parameters ('datastore ctxsys.default_datastore memory 250M');
-- create intermedia index for site wide index
begin
ctx_ddl.create_preference('sws_user_datastore', 'user_datastore');
ctx_ddl.set_attribute('sws_user_datastore', 'procedure', 'sws_user_proc_&1');
end;
/
exit;
\ No newline at end of file
--
-- Copyright (C) 2005 MIT
--
-- This file is part of dotLRN.
--
-- dotLRN is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 2 of the License, or (at your option) any later
-- version.
--
-- dotLRN is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-- FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
-- details.
--
--
-- Drop the intermedia index for .LRN site-wide search
--
-- @author <a href="mailto:openacs@dirkgomez.de">Dirk Gomez</a>
-- @version $Id$
-- @creation-date 13-May-2005
begin
ctx_ddl.drop_section_group('swsgroup');
ctx_ddl.drop_preference('sws_user_datastore');
end;
/
drop index swi_index;
exit;
--
-- Copyright (C) 2005 MIT
--
-- This file is part of dotLRN.
--
-- dotLRN is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 2 of the License, or (at your option) any later
-- version.
--
-- dotLRN is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-- FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
-- details.
--
--
-- Create database packages for .LRN site-wide search
--
-- @author <a href="mailto:openacs@dirkgomez.de">Dirk Gomez</a>
-- @version $Id$
-- @creation-date 13-May-2005
-- Partly ported from ACES.
-- The site_wide_search packages holds generally useful
-- PL/SQL procedures and functions.
create or replace package site_wide_search
as
procedure register_event (p_object_id search_observer_queue.object_id%TYPE,
p_event search_observer_queue.event%TYPE);
procedure logger (p_logmessage varchar);
function im_convert(
query in varchar2 default null
) return varchar2;
end site_wide_search;
/
show errors
create or replace package body site_wide_search
as
procedure register_event (p_object_id search_observer_queue.object_id%TYPE,
p_event search_observer_queue.event%TYPE) is
begin
insert into search_observer_queue
(object_id, event)
values
(p_object_id, p_event);
end register_event;
procedure logger (p_logmessage varchar) is
begin
insert into sws_log_messages (logmessage) values (p_logmessage);
end logger;
-- Query to take free text user entered query and frob it into something
-- that will make interMedia happy. Provided by Oracle.
function im_convert(
query in varchar2 default null
) return varchar2
is
i number :=0;
len number :=0;
char varchar2(1);
minusString varchar2(256);
plusString varchar2(256);
mainString varchar2(256);
mainAboutString varchar2(500);
finalString varchar2(500);
hasMain number :=0;
hasPlus number :=0;
hasMinus number :=0;
token varchar2(256);
tokenStart number :=1;
tokenFinish number :=0;
inPhrase number :=0;
inPlus number :=0;
inWord number :=0;
inMinus number :=0;
completePhrase number :=0;
completeWord number :=0;
code number :=0;
begin
len := length(query);
-- we iterate over the string to find special web operators
for i in 1..len loop
char := substr(query,i,1);
if(char = '"') then
if(inPhrase = 0) then
inPhrase := 1;
tokenStart := i;
else
inPhrase := 0;
completePhrase := 1;
tokenFinish := i-1;
end if;
elsif(char = ' ') then
if(inPhrase = 0) then
completeWord := 1;
tokenFinish := i-1;
end if;
elsif(char = '+') then
inPlus := 1;
tokenStart := i+1;
elsif((char = '-') and (i = tokenStart)) then
inMinus :=1;
tokenStart := i+1;
end if;
if(completeWord=1) then
token := '{ '||substr(query,tokenStart,tokenFinish-tokenStart+1)||' }';
if(inPlus=1) then
plusString := plusString||','||token||'*10';
hasPlus :=1;
elsif(inMinus=1) then
minusString := minusString||'OR '||token||' ';
hasMinus :=1;
else
mainString := mainString||' NEAR '||token;
mainAboutString := mainAboutString||' '||token;
hasMain :=1;
end if;
tokenStart :=i+1;
tokenFinish :=0;
inPlus := 0;
inMinus :=0;
end if;
completePhrase := 0;
completeWord :=0;
end loop;
-- find the last token
token := '{ '||substr(query,tokenStart,len-tokenStart+1)||' }';
if(inPlus=1) then
plusString := plusString||','||token||'*10';
hasPlus :=1;
elsif(inMinus=1) then
minusString := minusString||'OR '||token||' ';
hasMinus :=1;
else
mainString := mainString||' NEAR '||token;
mainAboutString := mainAboutString||' '||token;
hasMain :=1;
end if;
mainString := substr(mainString,6,length(mainString)-5);
mainAboutString := replace(mainAboutString,'{',' ');
mainAboutString := replace(mainAboutString,'}',' ');
mainAboutString := replace(mainAboutString,')',' ');
mainAboutString := replace(mainAboutString,'(',' ');
plusString := substr(plusString,2,length(plusString)-1);
minusString := substr(minusString,4,length(minusString)-4);
-- we find the components present and then process them based on the specific combinations
code := hasMain*4+hasPlus*2+hasMinus;
if(code = 7) then
finalString := '('||plusString||','||mainString||'*2.0,about('||mainAboutString||')*0.5) NOT ('||minusString||')';
elsif (code = 6) then
finalString := plusString||','||mainString||'*2.0'||',about('||mainAboutString||')*0.5';
elsif (code = 5) then
finalString := '('||mainString||',about('||mainAboutString||')) NOT ('||minusString||')';
elsif (code = 4) then
finalString := mainString;
finalString := replace(finalString,'*1,',NULL);
finalString := '('||finalString||')*2.0,about('||mainAboutString||')';
elsif (code = 3) then
finalString := '('||plusString||') NOT ('||minusString||')';
elsif (code = 2) then
finalString := plusString;
elsif (code = 1) then
-- not is a binary operator for intermedia text
finalString := 'totallyImpossibleString'||' NOT ('||minusString||')';
elsif (code = 0) then
finalString := '';
end if;
return finalString;
end;
end site_wide_search;
/
show errors
--------------------------------------------------------
-- The user_datastore proc which is called on every change of the datastore.
create or replace procedure sws_user_datastore_proc ( p_rid in rowid, p_tlob in out nocopy clob )
is
v_object_id site_wide_index.object_id%type;
begin
site_wide_search.logger ('entered sws_user_datastore_proc');
select indexed_content
into p_tlob
from site_wide_index swi, acs_objects ao
where swi.object_id = ao.object_id
and p_rid = swi.rowid;
site_wide_search.logger ('in sws_user_datastore_proc with type ' || v_object_id);
end;
/
show errors;
exit;
drop package body site_wide_search;
drop package site_wide_search;
--
-- Copyright (C) 2005 MIT
--
-- This file is part of dotLRN.
--
-- dotLRN is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 2 of the License, or (at your option) any later
-- version.
--
-- dotLRN is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-- FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
-- details.
--
--
-- Create database tables for .LRN site-wide search
--
-- @author <a href="mailto:openacs@dirkgomez.de">openacs@dirkgomez.de</a>
-- @version $Id$
-- @creation-date 13-May-2005
--
-- Partly ported from ACES.
-- Central table for site-wide search.
declare
v_aux integer;
begin
select count(table_name) into v_aux from user_tables where table_name = 'SITE_WIDE_INDEX';
IF v_aux = 0 THEN
execute immediate 'create table site_wide_index (
object_id integer
constraint sws_index_pk primary key
constraint sws_index_fk references acs_objects(object_id) on delete cascade,
object_name varchar(4000),
indexed_content clob,
-- Dirk Gomez: no not null constraint because we also want to
-- be able to index objects which are not tied to an object,
-- in particular people.
package_id integer
constraint swi_package_id_fk
references apm_packages
on delete cascade,
-- Dirk Gomez: That''s the place to put an object''s relevant
-- date which is part of the ranking function. In calendar
-- this is the item date, in forum it could be the last reply
-- date to a thread etc.
relevant_date date
)';
END IF;
select count(table_name) into v_aux from user_tables where table_name = 'SWS_LOG_MESSAGES';
IF v_aux = 0 THEN
-- Intermedia sometimes is painful to debug, so I added a logging
-- mechanism which relies on Oracle's autonomous transactions: DML
-- statements are committed immediately so you can access this data
-- from a different session right away.
execute immediate 'create table sws_log_messages (
logmessage varchar2(4000),
logtime date default sysdate
)';
END IF;
END;
/
exit;
drop table site_wide_index;
drop table sws_log_messages;
#
ad_library {
Scheduled proc init for intermedia driver
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2008-04-15
@cvs-id $Id$
}
ad_schedule_proc -thread t 14400 db_exec_plsql optimize_intermedia_index {begin Ctx_Ddl.Optimize_Index ('swi_index','FAST'); end;}
ad_library {
procedures to support intermedia search engine for Oracle
}
ad_proc -public -callback search::index -impl intermedia-driver {} {
Search Index Callback for Oracle Intermedia
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2005-06-12
} {
# we want the datasource array reference in case we want to do something clever
if {![string equal "" $datasource]} {
upvar $datasource _datasource
}
set content "${title} ${content}"
# if storage type is file, store the text in the site_wide_index table
if {![db_string index_exists "select 1 from site_wide_index where object_id=:object_id" -default 0]} {
db_dml index "insert into site_wide_index
(object_id, object_name, package_id, relevant_date, community_id, indexed_content)
values
(:object_id, :title, :package_id, :relevant_date, :community_id, empty_clob() )
returning indexed_content into :1" -clobs [list $content]
} else {
# call the update index proc since this object is already indexed
callback -impl intermedia-driver search::update_index \
-object_id $object_id \
-content $content \
-title $title \
-keywords $keywords \
-community_id $community_id \
-relevant_date $relevant_date \
-description $description \
-datasource $datasource \
-package_id $package_id
}
}
ad_proc -public -callback search::update_index -impl intermedia-driver {} {
Update item in the index
@author Dave Bauer (dave@thedesignexperience.org
@creation-date 2005-08-01
} {
if {![string equal "" $datasource]} {
upvar $datasource _datasource
}
if {![db_string index_exists "select 1 from site_wide_index where object_id=:object_id" -default 0]} {
callback -impl intermedia-driver search::index \
-object_id $object_id \
-content $content \
-title $title \
-keywords $keywords \
-community_id $community_id \
-relevant_date $relevant_date \
-description $description \
-datasource $datasource \
-package_id $package_id
return
} else {
db_dml index "update site_wide_index
set object_name=:title,
package_id=:package_id,
community_id=:community_id,
relevant_date=:relevant_date,
indexed_content=empty_clob()
where object_id=:object_id
returning indexed_content into :1" -clobs [list $content]
}
}
ad_proc -public -callback search::unindex -impl intermedia-driver {} {
Remove item from search index
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2005-06-12
} {
db_dml unindex "delete from site_wide_index where object_id=:object_id"
}
ad_proc -public -callback search::search -impl intermedia-driver {} {
Search full text index
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2005-05-29
@param query
@param offset
@param limit
@param user_id
@param df
@param dt
@param package_ids
@param object_type
} {
if {[info exists package_ids] && [llength $package_ids]} {
set package_ids_clause " and swi.package_id in ([template::util::tcl_to_sql_list $package_ids]) "
} else {
set package_ids_clause ""
}
if {[info exists object_type] && [string equal $object_type "forums"]} {
set object_type_clause " and o.object_type in ('forums_forum', 'forums_message') "
} elseif {[info exists object_type] && ![string equal $object_type "all"]} {
set object_type_clause " and o.object_type = :object_type "
} else {
set object_type_clause ""
}
set weighted_score "score(10) - case when object_type='faq'
then nvl(months_between(sysdate,relevant_date)/4,20)
when object_type='forums'
then nvl(months_between(sysdate,relevant_date)*1.5,20)
when object_type='phb_person'
then 0
when object_type='news'
then nvl(months_between(sysdate,relevant_date)*2,20)
when object_type='cal_item'
then nvl(months_between(sysdate,relevant_date)*2,20)
when object_type='file_storage_object'
then nvl(months_between(sysdate,relevant_date)*1.5,20)
when object_type='survey'
then nvl(months_between(sysdate,relevant_date)*1.5,20)
when object_type='static_portal_content'
then nvl(months_between(sysdate,relevant_date)*1.5,20)
end"
set people_search_clause { o.object_type = 'phb_person' or }
if [apm_package_installed_p "dotlrn"] {
set is_guest_p [search::is_guest_p]
if {$is_guest_p} {
set people_search_clause { and }; # doesn't look like legal SQL
}
set is_member {
exists ( select 1
from dotlrn_member_rels_approved
where community_id = swi.community_id
and user_id = :user_id)}
set community_id_clause " and (swi.community_id is null or $is_member) "
set member_clause " and $is_member "
} else {
set community_id_clause {}
set member_clause {}
}
set results_ids [db_list search "select s.object_id from
(select rownum as r,o.object_id
from site_wide_index swi, acs_objects o
where swi.object_id= o.object_id
$object_type_clause
and contains (swi.indexed_content,:query, 10)> 0
and (
$people_search_clause
(exists (select 1
from acs_object_party_privilege_map m
where m.object_id = o.object_id
and m.party_id = :user_id
and m.privilege = 'read')
$community_id_clause))
$package_ids_clause
order by $weighted_score desc) s where r > $offset and r <= $offset + $limit"]
# TODO implement stopwords reporting for user query
set count [db_string count "select count(swi.object_id) from site_wide_index swi, acs_objects o where o.object_id=swi.object_id $object_type_clause and contains (swi.indexed_content,:query)> 0
and (
$people_search_clause
(exists (select 1
from acs_object_party_privilege_map m
where m.object_id = o.object_id
and m.party_id = :user_id
and m.privilege = 'read')
$member_clause))
$package_ids_clause "]
set stop_words ""
ns_log notice "
-----------------------------------------------------------------------------
DAVEB99 intermedia::search
query = '{$query}'
package_ids = '${package_ids}'
return = '[list ids $results_ids stopwords $stop_words count $count]'
-----------------------------------------------------------------------------
"
return [list ids $results_ids stopwords $stop_words count $count]
}
ad_proc -public -callback search::summary -impl intermedia-driver {
} {
Get summary for an object
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2005-05-29
@param object_id
} {
# TODO implement intermedia::summary
return [string range $text 0 100]
}
ad_proc -public -callback search::driver_info -impl intermedia-driver {
} {
Info for the service contract implementation
for intermedia
@author Dave Bauer (dave@thedesignexperience.org)
@creation-date 2005-05-29
} {
return [list package_key intermedia-driver version 1 automatic_and_queries_p 1 stopwords_p 1]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment