Skip to content

I
intranet-csv-import
Commit 3b626d49 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

- Working on Infrax

parent 811cfbbe
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 90 deletions
tcl/intranet-csv-import-parser-procs.tcl
View file @ 3b626d49
...@@ -102,7 +102,7 @@ ad_proc -public im_csv_import_parser_office_name { ...@@ -102,7 +102,7 @@ ad_proc -public im_csv_import_parser_office_name {
Returns a office_id from im_companies Returns a office_id from im_companies
} { } {
if {[regexp {'} $arg match]} { if {[regexp {'} $arg match]} {
set err "Found a office name ($arg) with single quote, consider removing single quotes from companies" set err "Found a office name ($arg) with single quote, please consider removing single quotes from companies"
im_security_alert -location "im_csv_import_parser_office_name" -message $err -value $arg im_security_alert -location "im_csv_import_parser_office_name" -message $err -value $arg
return [list $arg $err] return [list $arg $err]
} }
...@@ -125,7 +125,7 @@ ad_proc -public im_csv_import_parser_project_nr { ...@@ -125,7 +125,7 @@ ad_proc -public im_csv_import_parser_project_nr {
Returns a project_id from project_nr Returns a project_id from project_nr
} { } {
if {[regexp {'} $arg match]} { if {[regexp {'} $arg match]} {
set err "Found a Project Nr with single quote" set err "Found a Project Nr ($arg) with single quote"
im_security_alert -location "im_csv_import_parser_project_nr" -message $err -value $arg im_security_alert -location "im_csv_import_parser_project_nr" -message $err -value $arg
return [list $arg $err] return [list $arg $err]
} }
...@@ -151,7 +151,7 @@ ad_proc -public im_csv_import_parser_project_name { ...@@ -151,7 +151,7 @@ ad_proc -public im_csv_import_parser_project_name {
Returns a project_id from project_name Returns a project_id from project_name
} { } {
if {[regexp {'} $arg match]} { if {[regexp {'} $arg match]} {
set err "Found a Project Name with single quote" set err "Found a Project Name ($arg) with single quote"
im_security_alert -location "im_csv_import_parser_project_name" -message $err -value $arg im_security_alert -location "im_csv_import_parser_project_name" -message $err -value $arg
return [list $arg $err] return [list $arg $err]
} }
...@@ -206,7 +206,7 @@ ad_proc -public im_csv_import_parser_boolean { ...@@ -206,7 +206,7 @@ ad_proc -public im_csv_import_parser_boolean {
"t" - "1" { return [list "t" ""] } "t" - "1" { return [list "t" ""] }
"f" - "0" { return [list "f" ""] } "f" - "0" { return [list "f" ""] }
} }
return [list "" "Could not determine boolean value of arg='$arg'"] return [list "" "Could not determine boolean value of '$arg'"]
} }
...@@ -241,7 +241,7 @@ ad_proc -public im_csv_import_parser_date_european_slashes { ...@@ -241,7 +241,7 @@ ad_proc -public im_csv_import_parser_date_european_slashes {
if {1 == [string length $month]} { set dom "0$month" } if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""] return [list "$year-$month-$dom" ""]
} }
return [list "" "Error parsing European date format '$arg': expected 'dd/mm/yyyy'"] return [list "" "Error parsing European date '$arg': expected 'dd/mm/yyyy'"]
} }
ad_proc -public im_csv_import_parser_date_iso { ad_proc -public im_csv_import_parser_date_iso {
...@@ -255,7 +255,7 @@ ad_proc -public im_csv_import_parser_date_iso { ...@@ -255,7 +255,7 @@ ad_proc -public im_csv_import_parser_date_iso {
if {1 == [string length $month]} { set dom "0$month" } if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""] return [list "$year-$month-$dom" ""]
} }
return [list "" "Error parsing ISO '$arg': expected 'yyyy-mm-dd'"] return [list "" "Error parsing ISO date '$arg': expected 'yyyy-mm-dd'"]
} }
...@@ -281,9 +281,8 @@ ad_proc -public im_csv_import_parser_percentage { ...@@ -281,9 +281,8 @@ ad_proc -public im_csv_import_parser_percentage {
Parses a percentage number. Parses a percentage number.
Basically just ignores a trailing % char after a number. Basically just ignores a trailing % char after a number.
} { } {
if {[regexp {^(.+)%$} $arg match number]} { # Fraber 170520: number parser already deals with percentages...
set arg $number # if {[regexp {^(.+)%$} $arg match number]} { set arg $number }
}
return [im_csv_import_parser_number -parser_args $parser_args $arg] return [im_csv_import_parser_number -parser_args $parser_args $arg]
} }
...@@ -327,7 +326,7 @@ ad_proc -public im_csv_import_parser_number { ...@@ -327,7 +326,7 @@ ad_proc -public im_csv_import_parser_number {
return [list [expr "$main.$fraction" / $divisor] ""] return [list [expr "$main.$fraction" / $divisor] ""]
} }
return [list 0 "Could not decide if this is a European or a US number - please use a specific parser"] return [list 0 "Could not decide if '$arg' is a European or a US number - please use a specific parser"]
} }
ad_proc -public im_csv_import_parser_number_european { ad_proc -public im_csv_import_parser_number_european {
...@@ -370,7 +369,7 @@ ad_proc -public im_csv_import_parser_category { ...@@ -370,7 +369,7 @@ ad_proc -public im_csv_import_parser_category {
set result [im_id_from_category $arg $parser_args] set result [im_id_from_category $arg $parser_args]
if {"" == $result} { if {"" == $result} {
return [list "" "Category parser: We did not find a value='$arg' in category type '$parser_args'."] return [list "" "Category parser: We did not find a category '$arg' in category type '$parser_args'."]
} else { } else {
return [list $result ""] return [list $result ""]
} }
...@@ -396,7 +395,7 @@ ad_proc -public im_csv_import_parser_cost_center { ...@@ -396,7 +395,7 @@ ad_proc -public im_csv_import_parser_cost_center {
"] "]
set result [lindex $ccids 0] set result [lindex $ccids 0]
if {"" == $result} { if {"" == $result} {
return [list "" "Cost Center parser: We did not find any cost center with label, code or name matching the value='$arg'."] return [list "" "Cost Center parser: We did not find any cost center with label, code or name matching '$arg'."]
} else { } else {
return [list $result ""] return [list $result ""]
} }
...@@ -423,7 +422,7 @@ ad_proc -public im_csv_import_parser_material { ...@@ -423,7 +422,7 @@ ad_proc -public im_csv_import_parser_material {
"] "]
set result [lindex $ccids 0] set result [lindex $ccids 0]
if {"" == $result} { if {"" == $result} {
return [list "" "Material parser: We did not find any material nr or name matching the value='$arg'."] return [list "" "Material parser: We did not find any material nr or name matching '$arg'."]
} else { } else {
return [list $result ""] return [list $result ""]
} }
......
www/import-im_project.tcl
View file @ 3b626d49
...@@ -70,16 +70,38 @@ set header_len [llength $header_fields] ...@@ -70,16 +70,38 @@ set header_len [llength $header_fields]
set values_list_of_lists [im_csv_get_values $lines_content $separator] set values_list_of_lists [im_csv_get_values $lines_content $separator]
# ------------------------------------------------------------ # ---------------------------------------------------------------------
# Get DynFields # Determine header_var_name
# ---------------------------------------------------------------------
# Determine the list of actually available fields.
set mapped_vars [list "''"] set mapped_vars [list "''"]
foreach k [array names map] { set debug_rows "<tr><td colspan=2>header_len</td><td colspan=5>$header_len</td></tr>"
lappend mapped_vars "'$map($k)'" foreach i [array names column] {
set header [string trim $column($i)]
if {$header ne $column($i)} { set column($i) $header }
set m $map($i)
set p $parser($i)
set args $parser_args($i)
# Calculate var name from header
set var [string tolower $header]
set var [im_mangle_unicode_accents $var]
set var [string map -nocase {" " "_" "\"" "" "'" "" "/" "_" "-" "_" "\[" "(" "\{" "(" "\}" ")" "\]" ")"} $var]
set header_var($i) $var
# Determine the list of actually available fields for SQL statement
lappend mapped_vars "'$map($i)'"
append debug_rows "<tr><td>$i</td><td>$header</td><td>$var</td><td>$m</td><td>$p</td><td>$args</td><td></td></tr>\n"
} }
set debug "<table border=1>$debug_rows</table>"
# ---------------------------------------------------------------------
# Get DynFields
# ---------------------------------------------------------------------
set dynfield_sql " set dynfield_sql "
select distinct select distinct
aa.attribute_name, aa.attribute_name,
...@@ -126,7 +148,7 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -126,7 +148,7 @@ foreach csv_line_fields $values_list_of_lists {
if {$ns_write_p} { ns_write "</ul><hr><ul>\n" } if {$ns_write_p} { ns_write "</ul><hr><ul>\n" }
if {$ns_write_p} { ns_write "<li>Starting to parse line $cnt\n" } if {$ns_write_p} { ns_write "<li>Starting to parse line $cnt\n" }
# Preset values, defined by CSV sheet: # Reset values used in the rest of the loop
set project_name "" set project_name ""
set project_nr "" set project_nr ""
set project_path "" set project_path ""
...@@ -173,89 +195,61 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -173,89 +195,61 @@ foreach csv_line_fields $values_list_of_lists {
set billable_units "" set billable_units ""
set priority "" set priority ""
foreach attribute_name $attribute_names { # Reset DynFields
set $attribute_name "" foreach attribute_name $attribute_names { set $attribute_name "" }
}
# ------------------------------------------------------- # -------------------------------------------------------
# Extract variables from the CSV file # Set default values without parser from header names
# foreach j [array names parser] {
set var_name_list [list] set v $header_var($j)
for {set j 0} {$j < $header_len} {incr j} { if {"" eq $v} { continue }
set var_name [string trim [lindex $header_fields $j]]
if {"" == $var_name} {
# No variable name - probably an empty column
continue
}
set var_name [string tolower $var_name]
set var_name [string map -nocase {" " "_" "\"" "" "'" "" "/" "_" "-" "_" "\[" "(" "\{" "(" "\}" ")" "\]" ")"} $var_name]
lappend var_name_list $var_name
ns_log notice "upload-companies-2: varname([lindex $header_fields $j]) = $var_name"
set var_value [string trim [lindex $csv_line_fields $j]] set var_value [string trim [lindex $csv_line_fields $j]]
set var_value [string map -nocase {"\"" "'" "\[" "(" "\{" "(" "\}" ")" "\]" ")" "\$" "" "\\" ""} $var_value] set var_value [string map -nocase {"\"" "'" "\[" "(" "\{" "(" "\}" ")" "\]" ")" "\$" "" "\\" ""} $var_value]
if {"NULL" eq $var_value} { set var_value ""} if {"NULL" eq $var_value} { set var_value ""}
set cmd "set $v \"$var_value\""
# replace unicode characters by non-accented characters
# Watch out! Does not work with Latin-1 characters
set var_name [im_mangle_unicode_accents $var_name]
set cmd "set $var_name \"$var_value\""
ns_log Notice "upload-companies-2: cmd=$cmd" ns_log Notice "upload-companies-2: cmd=$cmd"
set result [eval $cmd] set result [eval $cmd]
} }
# ------------------------------------------------------- # -------------------------------------------------------
# Transform the variables # Transform the variables
set i 0 foreach i [array names parser] {
set varname $header_var($i)
foreach varname $var_name_list {
set p $parser($i) set p $parser($i)
set p_args $parser_args($i) set p_args $parser_args($i)
set target_varname $map($i) set target_varname $map($i)
ns_log Notice "import-im_project: Parser: $varname -> $target_varname" ns_log Notice "import-im_project: i=$i: var=$varname -> target=$target_varname using parser=$p ($p_args)"
ns_log Notice "import-im_project: Parser: p_args: $p_args"
ns_log Notice "import-im_project: Parser: target_varname: $target_varname" set proc_name "im_csv_import_parser_$p"
if {[catch {
switch $p { set val [set $varname]
no_change { set $target_varname ""
set $target_varname [set $varname] if {"" != $val} {
ns_log Notice "import-im_project: Parser: no change: varname: $varname, val: [set $varname]" set result [$proc_name -parser_args $p_args $val]
} set res [lindex $result 0]
default { set err [lindex $result 1]
set proc_name "im_csv_import_parser_$p" ns_log Notice "import-im_project: var=$varname -> target=$target_varname using parser=$p ($p_args): '$val' -> '$res', err=$err"
if {[catch { if {"" ne $err} {
set val [set $varname]
ns_log Notice "import-im_project: Parser: val: $val"
if {"" != $val} {
set result [$proc_name -parser_args $p_args $val]
set res [lindex $result 0]
set err [lindex $result 1]
ns_log Notice "import-im_project: Parser: '$p -args $p_args $val' -> $target_varname=$res, err=$err"
if {"" != $err} {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err</pre></font>\n"
}
}
set $target_varname $res
}
} err_msg]} {
if {$ns_write_p} { if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err_msg</pre></font>" ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err</pre></font>\n"
} }
} }
set $target_varname $res
}
} err_msg]} {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err_msg</pre></font>"
} }
} }
incr i ns_log Notice "import-im_project: "
ns_log Notice "import-im_project: -------------------------------------------" ns_log Notice "import-im_project: -------------------------------------------"
ns_log Notice "import-im_project: "
} }
# ------------------------------------------------------- # -------------------------------------------------------
# Specific field transformations # Specific field transformations
# -------------------------------------------------------
# project_name needs to be there # project_name needs to be there
if {"" == $project_name} { if {"" == $project_name} {
...@@ -266,15 +260,6 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -266,15 +260,6 @@ foreach csv_line_fields $values_list_of_lists {
continue continue
} }
# project_nr needs to be there
if {"" == $project_nr} {
if {$ns_write_p} {
ns_write "<li><font color=red>Error: We have found an empty 'Project Nr' in line $cnt.<br>
Please correct the CSV file. Every project needs to have a unique Project Nr.</font>\n"
}
continue
}
# Parent0 - Parent1 - Parent2 - Parent3 # Parent0 - Parent1 - Parent2 - Parent3
if {"" ne $parent0} { if {"" ne $parent0} {
if {$ns_write_p} { ns_write "<li>Parent0: Looking for a main project with name=$parent0</font>\n" } if {$ns_write_p} { ns_write "<li>Parent0: Looking for a main project with name=$parent0</font>\n" }
...@@ -373,7 +358,7 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -373,7 +358,7 @@ foreach csv_line_fields $values_list_of_lists {
continue continue
} }
} }
# Status is a required field # Status is a required field
set project_status_id [im_id_from_category $project_status "Intranet Project Status"] set project_status_id [im_id_from_category $project_status "Intranet Project Status"]
if {"" == $project_status_id} { if {"" == $project_status_id} {
...@@ -381,11 +366,12 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -381,11 +366,12 @@ foreach csv_line_fields $values_list_of_lists {
set project_status_id [im_project_status_open] set project_status_id [im_project_status_open]
} }
# Type is a required field # Project type is optional: Main projects are assumed to be "Gantt Projects", while sub-project are assumed to be "Task"
set project_type_id [im_id_from_category [list $project_type] "Intranet Project Type"] set project_type_id [im_id_from_category [list $project_type] "Intranet Project Type"]
if {"" == $project_type_id} { if {"" == $project_type_id} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find project type '$project_type', using default type 'Other'</font>\n" }
set project_type_id [im_project_type_gantt] set project_type_id [im_project_type_gantt]
if {"" ne $parent_id} { set project_type_id [im_project_type_task] }
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find project type '$project_type', using default type '[im_category_from_id $project_type_id]'</font>\n" }
} }
# start_date and end_date are required fields for projects, not tasks # start_date and end_date are required fields for projects, not tasks
...@@ -432,6 +418,18 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -432,6 +418,18 @@ foreach csv_line_fields $values_list_of_lists {
} }
} }
# project_nr is based on the parent
if {"" == $project_nr} {
set project_nr [im_next_project_nr -customer_id $company_id -parent_id $parent_id]
#if {$ns_write_p} {
# ns_write "<li><font color=red>Error: We have found an empty 'Project Nr' in line $cnt.<br>
# Please correct the CSV file. Every project needs to have a unique Project Nr.</font>\n"
#}
#continue
}
# Project Lead / Project Manager # Project Lead / Project Manager
if { "" eq $project_lead_id && 100 != $project_type_id } { if { "" eq $project_lead_id && 100 != $project_type_id } {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: No project manager found. Will try to create project w/o PM. </font>\n" } if {$ns_write_p} { ns_write "<li><font color=brown>Warning: No project manager found. Will try to create project w/o PM. </font>\n" }
...@@ -591,7 +589,7 @@ foreach csv_line_fields $values_list_of_lists { ...@@ -591,7 +589,7 @@ foreach csv_line_fields $values_list_of_lists {
} }
# Task UoM # Task UoM
if {"" eq $uom} { set uom "Hour" } if {"" eq $uom} { set uom "Day" }
set uom_id [im_id_from_category [list $uom] "Intranet UoM"] set uom_id [im_id_from_category [list $uom] "Intranet UoM"]
if {"" eq $uom_id} { if {"" eq $uom_id} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find UoM '$uom', using default 'Hour'</font>\n" } if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find UoM '$uom', using default 'Hour'</font>\n" }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment