Skip to content

I
intranet-csv-import
Commit 58789ba8 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

- CSV Import: 

  Now fixed project import
parent a74da935
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 89 deletions
tcl/intranet-csv-import-procs.tcl
View file @ 58789ba8
......@@ -33,27 +33,51 @@ ad_proc -public im_id_from_user_name { name } {
return $user_id
}
ad_proc -public im_csv_import_parser_date_european { arg } {
ad_proc -public im_csv_import_parser_date_european {
{-parser_args "" }
arg
} {
Parses a European date format like '08.06.2011' as the 8th of June, 2011
} {
if {[regexp {^(.+)\.(.+)\.(....)$} $arg match dom month year]} {
if {1 == [string length $dom]} { set dom "0$dom" }
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing European date format '$arg': expected 'dd.mm.yyyy'"]
if {[regexp {^(.+)\.(.+)\.(....)$} $arg match dom month year]} {
if {1 == [string length $dom]} { set dom "0$dom" }
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing European date format '$arg': expected 'dd.mm.yyyy'"]
}
ad_proc -public im_csv_import_parser_date_american { arg } {
ad_proc -public im_csv_import_parser_date_american {
{-parser_args "" }
arg
} {
Parses a American date format like '12/31/2011' as the 31st of December, 2011
} {
if {[regexp {^(.+)\/(.+)\/(....)$} $arg match month dom year]} {
if {1 == [string length $dom]} { set dom "0$dom" }
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing American date format '$arg': expected 'dd.mm.yyyy'"]
if {[regexp {^(.+)\/(.+)\/(....)$} $arg match month dom year]} {
if {1 == [string length $dom]} { set dom "0$dom" }
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing American date format '$arg': expected 'dd.mm.yyyy'"]
}
ad_proc -public im_csv_import_parser_category {
{-parser_args "" }
arg
} {
Parses a category into a category_id
} {
# Empty input - empty output
if {"" == $arg} { return [list "" ""] }
# Parse the category
set result [im_id_from_category $arg $parser_args]
if {"" == $result} {
return [list "" "Category parser: We did not find a value='$arg' in category type '$parser_args'."]
} else {
return [list $result ""]
}
}
# ----------------------------------------------------------------------
......@@ -164,6 +188,7 @@ ad_proc -public im_csv_import_parsers {
no_change "No Change"
date_european "European Date Parser (DD.MM.YYYY)"
date_american "American Date Parser (MM/DD/YYYY)"
category "Category Parser"
}
}
default {
......@@ -185,12 +210,17 @@ ad_proc -public im_csv_import_guess_parser {
-object_type:required
-field_name:required
} {
Returns the best guess for a parser for the given field
Returns the best guess for a parser for the given field as
a list with 1. the parser name and 2. the parser args
} {
# --------------------------------------------------------
# Abort if there are not enough values
if {[llength $sample_values] < 2} { return "" }
if {[llength $sample_values] < 2} { return [list "" ""] }
# Available parsers
# --------------------------------------------------------
# Date parsers
#
set date_european_p 1
set date_american_p 1
set number_plain_p 1
......@@ -199,18 +229,41 @@ ad_proc -public im_csv_import_guess_parser {
# set the parserst to 0 if one of the values doesn't fit
foreach val $sample_values {
if {![regexp {^(.+)\.(.+)\.(....)$} $val match]} { set date_european_p 0 }
if {![regexp {^(.+)\/(.+)\/(....)$} $val match]} { set date_american_p 0 }
if {![regexp {^[0-9]+$} $val match]} { set number_plain 0 }
}
if {$date_european_p} { return "date_european" }
if {$date_american_p} { return "date_american" }
if {$number_plain_p} { return "number_plain" }
if {$date_european_p} { return [list "date_european" ""] }
if {$date_american_p} { return [list "date_american" ""]}
# --------------------------------------------------------
# Parsing for DynFields
if {[db_0or1row dynfield_info {
select dw.widget as tcl_widget,
dw.parameters as tcl_widget_parameters,
substring(dw.parameters from 'category_type "(.*)"') as category_type
from acs_attributes aa,
im_dynfield_attributes da,
im_dynfield_widgets dw
where aa.object_type = :object_type and
aa.attribute_name = :field_name and
aa.attribute_id = da.acs_attribute_id and
da.widget_name = dw.widget_name
}]} {
switch $tcl_widget {
"im_category_tree" {
return [list "category" $category_type]
}
default {
# Default: No specific parser
return [list "" ""]
}
}
}
return ""
return [list "" ""]
}
......
www/import-2.adp
View file @ 58789ba8
......@@ -11,8 +11,11 @@
<td class=rowtitle>Field Name</td>
<td class=rowtitle>Row 1</td>
<td class=rowtitle>Row 2</td>
<td class=rowtitle>Mapping</td>
<td class=rowtitle>Row 3</td>
<td class=rowtitle>Row 4</td>
<td class=rowtitle>Map to Field</td>
<td class=rowtitle>Transformation</td>
<td class=rowtitle>Parameters</td>
</tr>
<multiple name=mapping>
......@@ -23,20 +26,25 @@
<td>@mapping.field_name@ @mapping.column;noquote@</td>
<td>@mapping.row_1@</td>
<td>@mapping.row_2@</td>
<td>@mapping.row_3@</td>
<td>@mapping.row_4@</td>
<td>@mapping.map;noquote@</td>
<td>@mapping.parser;noquote@</td>
<td>@mapping.parser_args;noquote@</td>
</tr>
</multiple>
</table>
<table>
<!--
<tr>
<td>Save Mapping as:</td>
<td><input type=text name=mapping_name></td>
</tr>
-->
<tr>
<td></td>
<td><input type=submit value="#intranet-csv-import.Import_CVS#"></td>
<td><input type=submit value="<%= [lang::message::lookup "" intranet-csv-import.Import_CSV "Import CSV"] %>"></td>
</tr>
</table>
......
www/import-2.tcl
View file @ 58789ba8
......@@ -77,7 +77,6 @@ if {[catch {
# Extract the header line from the file
set lines [split $lines_content "\n"]
if {"" == $lines} { ad_return_complaint 1 "<b>You didn't select a file in the screen before.</b>" }
set separator [im_csv_guess_separator $lines]
# ad_return_complaint 1 $separator
set lines_len [llength $lines]
......@@ -103,13 +102,13 @@ set parser_pairs [im_csv_import_parsers -object_type $object_type]
# ---------------------------------------------------------------------
# Setup the multirow with some sample rows
multirow create mapping field_name column map parser
multirow create mapping field_name column map parser parser_args
for {set i 1} {$i < $max_row} {incr i} {
multirow extend mapping "row_$i"
}
set object_type_pairs [list]
foreach field $object_fields { lappend object_type_pairs $field $field }
set object_type_pairs [list "" ""]
foreach field $object_fields { lappend object_type_pairs [string tolower $field] [string tolower $field] }
set cnt 0
foreach header_name $headers {
......@@ -118,7 +117,7 @@ foreach header_name $headers {
set column "<input type=hidden name=column.$cnt value=\"$header_name\">"
# Mapping - Map to which object field?
set default_map $header_name
set default_map [string tolower $header_name]
set map [im_select map.$cnt $object_type_pairs $default_map]
# Parser - convert the value from CSV values to ]po[ values
......@@ -128,12 +127,14 @@ foreach header_name $headers {
set val [lindex [set $row_name] $cnt]
if {"" != $val} { lappend parser_sample_values $val }
}
set default_parser [im_csv_import_guess_parser -object_type $object_type -field_name $header_name -sample_values $parser_sample_values]
set defs [im_csv_import_guess_parser -object_type $object_type -field_name $header_name -sample_values $parser_sample_values]
set default_parser [lindex $defs 0]
set default_parser_args [lindex $defs 1]
set parser [im_select parser.$cnt $parser_pairs $default_parser]
set args "<input type=text name=parser_args.$cnt value=\"$default_parser_args\">\n"
set first_val [lindex $row_1 $cnt]
set second_val [lindex $row_2 $cnt]
multirow append mapping $header_name $column $map $parser $first_val $second_val
multirow append mapping $header_name $column $map $parser $args [lindex $row_1 $cnt] [lindex $row_2 $cnt] [lindex $row_3 $cnt] [lindex $row_4 $cnt] [lindex $row_5 $cnt]
incr cnt
}
......
www/import-im_project.tcl
View file @ 58789ba8
......@@ -18,6 +18,7 @@ ad_page_contract {
column:array
map:array
parser:array
parser_args:array
}
# ---------------------------------------------------------------------
......@@ -70,9 +71,10 @@ set values_list_of_lists [im_csv_get_values $lines_content $separator]
# Get DynFields
set dynfield_sql {
select aa.*,
a.*,
w.*,
select distinct
aa.attribute_name,
aa.object_type,
w.parameters,
w.widget as tcl_widget,
substring(w.parameters from 'category_type "(.*)"') as category_type
from im_dynfield_widgets w,
......@@ -84,6 +86,12 @@ set dynfield_sql {
(also_hard_coded_p is null OR also_hard_coded_p = 'f')
}
set attribute_names [db_list attribute_names "
select distinct
attribute_name
from ($dynfield_sql) t
order by attribute_name
"]
# ------------------------------------------------------------
# Render Result Header
......@@ -147,14 +155,13 @@ foreach csv_line_fields $values_list_of_lists {
set priority ""
set sort_order ""
db_foreach initiate_dynfields $dynfield_sql {
foreach attribute_name $attribute_names {
set $attribute_name ""
}
# -------------------------------------------------------
# Extract variables from the CSV file
#
set var_name_list [list]
for {set j 0} {$j < $header_len} {incr j} {
......@@ -182,43 +189,12 @@ foreach csv_line_fields $values_list_of_lists {
set result [eval $cmd]
}
# -------------------------------------------------------
# Transform DynFields
db_foreach initiate_dynfields $dynfield_sql {
set value [set $attribute_name]
if {"" != $value} {
switch $tcl_widget {
"im_category_tree" {
set result [im_id_from_category $value $category_type]
if {"" != $result} {
# ns_write "<li>DynField im_category_tree: parse '$value' with category type '$category_type': '$result'\n"
} else {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning:
Failed to parse category '$value' for category type '$category_type'</font>\n"
}
}
set $attribute_name $result
}
default {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning:
DynField parser for widget '$tcl_widget' not implemented yet - using literal value</font>\n"
}
}
}
}
}
# -------------------------------------------------------
# Transform the variables
set i 0
foreach varname $var_name_list {
set p $parser($i)
set p_args $parser_args($i)
switch $p {
no_change { }
default {
......@@ -226,16 +202,20 @@ foreach csv_line_fields $values_list_of_lists {
if {[catch {
set val [set $varname]
if {"" != $val} {
set result [$proc_name $val]
set result [$proc_name -parser_args $p_args $val]
set res [lindex $result 0]
set err [lindex $result 1]
if {"" != $err} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Error parsing field='[set $varname]' using parser '$p':<pre>$err</pre></font>\n" }
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='[set $varname]' using parser '$p':<pre>$err</pre></font>\n"
}
}
set $varname $res
}
} err_msg]} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Error parsing field='[set $varname]' using parser '$p':<pre>$err_msg</pre></font>" }
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='[set $varname]' using parser '$p':<pre>$err_msg</pre></font>"
}
}
}
}
......@@ -300,8 +280,12 @@ foreach csv_line_fields $values_list_of_lists {
set project_priority_id [im_id_from_category [list $project_priority] "Intranet Department Planner Project Priority"]
# customer_id
if {"" == $customer_id } { set customer_id [db_string cust "select company_id from im_companies where lower(company_name) = trim(lower(:customer_name))" -default ""] }
if {"" == $customer_id } { set customer_id [db_string cust "select company_id from im_companies where lower(company_path) = trim(lower(:customer_name))" -default ""] }
if {"" == $customer_id } {
set customer_id [db_string cust "select company_id from im_companies where lower(company_name) = trim(lower(:customer_name))" -default ""]
}
if {"" == $customer_id } {
set customer_id [db_string cust "select company_id from im_companies where lower(company_path) = trim(lower(:customer_name))" -default ""]
}
if {"" == $customer_id } {
if {$ns_write_p} {
ns_write "<li><font color=red>Error: Didn't find customer for '$customer_name'.<br>
......@@ -360,13 +344,13 @@ foreach csv_line_fields $values_list_of_lists {
if {$ns_write_p} { ns_write "<li>Going to create project: name='$project_name', nr='$project_nr'\n" }
if {[catch {
set project_id [project::new \
-project_name $project_name \
-project_nr $project_nr \
-project_path $project_path \
-company_id $customer_id \
-parent_id $parent_id \
-project_type_id $project_type_id \
-project_status_id $project_status_id \
-project_name $project_name \
-project_nr $project_nr \
-project_path $project_path \
-company_id $customer_id \
-parent_id $parent_id \
-project_type_id $project_type_id \
-project_status_id $project_status_id \
]
} err_msg]} {
if {$ns_write_p} { ns_write "<li><font color=red>Error: Creating new project:<br><pre>$err_msg</pre></font>\n" }
......@@ -378,7 +362,6 @@ foreach csv_line_fields $values_list_of_lists {
}
if {$ns_write_p} { ns_write "<li>Going to update the project.\n" }
if {$ns_write_p} { ns_write "<li>parent_id='$parent_id'\n" }
if {[catch {
db_dml update_project "
update im_projects set
......@@ -409,7 +392,6 @@ foreach csv_line_fields $values_list_of_lists {
if {$ns_write_p} { ns_write "<li><font color=red>Error: Error updating project:<br><pre>$err_msg</pre></font>" }
continue
}
if {$ns_write_p} { ns_write "<li>Project successfully updated.\n" }
# -------------------------------------------------------
......@@ -506,8 +488,7 @@ foreach csv_line_fields $values_list_of_lists {
}
}
# ad_return_complaint 1 "<pre>project_dynfield_updates:\n[join $project_dynfield_updates "\n"]\n\n\ntask_dynfield_updates:\n$task_dynfield_updates</pre>"
if {$ns_write_p} { ns_write "<li>Going to update im_project DynFields.\n" }
if {"" != $project_dynfield_updates} {
set project_update_sql "
update im_projects set
......@@ -517,10 +498,11 @@ foreach csv_line_fields $values_list_of_lists {
if {[catch {
db_dml project_dynfield_update $project_update_sql
} err_msg]} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Error updating dynfields:<br><pre>$err_msg</pre>" }
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Error updating im_project dynfields:<br><pre>$err_msg</pre></font>" }
}
}
if {$ns_write_p} { ns_write "<li>Going to update im_timesheet_task DynFields.\n" }
if {"" != $task_dynfield_updates} {
set task_update_sql "
update im_timesheet_tasks set
......@@ -530,7 +512,7 @@ foreach csv_line_fields $values_list_of_lists {
if {[catch {
db_dml task_dynfield_update $task_update_sql
} err_msg]} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Error updating dynfields:<br><pre>$err_msg</pre>" }
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Error updating im_timesheet_task dynfields:<br><pre>$err_msg</pre></font>" }
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment