Skip to content

I
intranet-csv-import
Commit 3b626d49 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

- Working on Infrax

parent 811cfbbe
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 90 deletions
tcl/intranet-csv-import-parser-procs.tcl
View file @ 3b626d49
......@@ -102,7 +102,7 @@ ad_proc -public im_csv_import_parser_office_name {
Returns a office_id from im_companies
} {
if {[regexp {'} $arg match]} {
set err "Found a office name ($arg) with single quote, consider removing single quotes from companies"
set err "Found a office name ($arg) with single quote, please consider removing single quotes from companies"
im_security_alert -location "im_csv_import_parser_office_name" -message $err -value $arg
return [list $arg $err]
}
......@@ -125,7 +125,7 @@ ad_proc -public im_csv_import_parser_project_nr {
Returns a project_id from project_nr
} {
if {[regexp {'} $arg match]} {
set err "Found a Project Nr with single quote"
set err "Found a Project Nr ($arg) with single quote"
im_security_alert -location "im_csv_import_parser_project_nr" -message $err -value $arg
return [list $arg $err]
}
......@@ -151,7 +151,7 @@ ad_proc -public im_csv_import_parser_project_name {
Returns a project_id from project_name
} {
if {[regexp {'} $arg match]} {
set err "Found a Project Name with single quote"
set err "Found a Project Name ($arg) with single quote"
im_security_alert -location "im_csv_import_parser_project_name" -message $err -value $arg
return [list $arg $err]
}
......@@ -206,7 +206,7 @@ ad_proc -public im_csv_import_parser_boolean {
"t" - "1" { return [list "t" ""] }
"f" - "0" { return [list "f" ""] }
}
return [list "" "Could not determine boolean value of arg='$arg'"]
return [list "" "Could not determine boolean value of '$arg'"]
}
......@@ -241,7 +241,7 @@ ad_proc -public im_csv_import_parser_date_european_slashes {
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing European date format '$arg': expected 'dd/mm/yyyy'"]
return [list "" "Error parsing European date '$arg': expected 'dd/mm/yyyy'"]
}
ad_proc -public im_csv_import_parser_date_iso {
......@@ -255,7 +255,7 @@ ad_proc -public im_csv_import_parser_date_iso {
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing ISO '$arg': expected 'yyyy-mm-dd'"]
return [list "" "Error parsing ISO date '$arg': expected 'yyyy-mm-dd'"]
}
......@@ -281,9 +281,8 @@ ad_proc -public im_csv_import_parser_percentage {
Parses a percentage number.
Basically just ignores a trailing % char after a number.
} {
if {[regexp {^(.+)%$} $arg match number]} {
set arg $number
}
# Fraber 170520: number parser already deals with percentages...
# if {[regexp {^(.+)%$} $arg match number]} { set arg $number }
return [im_csv_import_parser_number -parser_args $parser_args $arg]
}
......@@ -327,7 +326,7 @@ ad_proc -public im_csv_import_parser_number {
return [list [expr "$main.$fraction" / $divisor] ""]
}
return [list 0 "Could not decide if this is a European or a US number - please use a specific parser"]
return [list 0 "Could not decide if '$arg' is a European or a US number - please use a specific parser"]
}
ad_proc -public im_csv_import_parser_number_european {
......@@ -370,7 +369,7 @@ ad_proc -public im_csv_import_parser_category {
set result [im_id_from_category $arg $parser_args]
if {"" == $result} {
return [list "" "Category parser: We did not find a value='$arg' in category type '$parser_args'."]
return [list "" "Category parser: We did not find a category '$arg' in category type '$parser_args'."]
} else {
return [list $result ""]
}
......@@ -396,7 +395,7 @@ ad_proc -public im_csv_import_parser_cost_center {
"]
set result [lindex $ccids 0]
if {"" == $result} {
return [list "" "Cost Center parser: We did not find any cost center with label, code or name matching the value='$arg'."]
return [list "" "Cost Center parser: We did not find any cost center with label, code or name matching '$arg'."]
} else {
return [list $result ""]
}
......@@ -423,7 +422,7 @@ ad_proc -public im_csv_import_parser_material {
"]
set result [lindex $ccids 0]
if {"" == $result} {
return [list "" "Material parser: We did not find any material nr or name matching the value='$arg'."]
return [list "" "Material parser: We did not find any material nr or name matching '$arg'."]
} else {
return [list $result ""]
}
......
www/import-im_project.tcl
View file @ 3b626d49
......@@ -70,16 +70,38 @@ set header_len [llength $header_fields]
set values_list_of_lists [im_csv_get_values $lines_content $separator]
# ------------------------------------------------------------
# Get DynFields
# ---------------------------------------------------------------------
# Determine header_var_name
# ---------------------------------------------------------------------
# Determine the list of actually available fields.
set mapped_vars [list "''"]
foreach k [array names map] {
lappend mapped_vars "'$map($k)'"
set debug_rows "<tr><td colspan=2>header_len</td><td colspan=5>$header_len</td></tr>"
foreach i [array names column] {
set header [string trim $column($i)]
if {$header ne $column($i)} { set column($i) $header }
set m $map($i)
set p $parser($i)
set args $parser_args($i)
# Calculate var name from header
set var [string tolower $header]
set var [im_mangle_unicode_accents $var]
set var [string map -nocase {" " "_" "\"" "" "'" "" "/" "_" "-" "_" "\[" "(" "\{" "(" "\}" ")" "\]" ")"} $var]
set header_var($i) $var
# Determine the list of actually available fields for SQL statement
lappend mapped_vars "'$map($i)'"
append debug_rows "<tr><td>$i</td><td>$header</td><td>$var</td><td>$m</td><td>$p</td><td>$args</td><td></td></tr>\n"
}
set debug "<table border=1>$debug_rows</table>"
# ---------------------------------------------------------------------
# Get DynFields
# ---------------------------------------------------------------------
set dynfield_sql "
select distinct
aa.attribute_name,
......@@ -126,7 +148,7 @@ foreach csv_line_fields $values_list_of_lists {
if {$ns_write_p} { ns_write "</ul><hr><ul>\n" }
if {$ns_write_p} { ns_write "<li>Starting to parse line $cnt\n" }
# Preset values, defined by CSV sheet:
# Reset values used in the rest of the loop
set project_name ""
set project_nr ""
set project_path ""
......@@ -173,89 +195,61 @@ foreach csv_line_fields $values_list_of_lists {
set billable_units ""
set priority ""
foreach attribute_name $attribute_names {
set $attribute_name ""
}
# Reset DynFields
foreach attribute_name $attribute_names { set $attribute_name "" }
# -------------------------------------------------------
# Extract variables from the CSV file
#
set var_name_list [list]
for {set j 0} {$j < $header_len} {incr j} {
set var_name [string trim [lindex $header_fields $j]]
if {"" == $var_name} {
# No variable name - probably an empty column
continue
}
set var_name [string tolower $var_name]
set var_name [string map -nocase {" " "_" "\"" "" "'" "" "/" "_" "-" "_" "\[" "(" "\{" "(" "\}" ")" "\]" ")"} $var_name]
lappend var_name_list $var_name
ns_log notice "upload-companies-2: varname([lindex $header_fields $j]) = $var_name"
# Set default values without parser from header names
foreach j [array names parser] {
set v $header_var($j)
if {"" eq $v} { continue }
set var_value [string trim [lindex $csv_line_fields $j]]
set var_value [string map -nocase {"\"" "'" "\[" "(" "\{" "(" "\}" ")" "\]" ")" "\$" "" "\\" ""} $var_value]
if {"NULL" eq $var_value} { set var_value ""}
# replace unicode characters by non-accented characters
# Watch out! Does not work with Latin-1 characters
set var_name [im_mangle_unicode_accents $var_name]
set cmd "set $var_name \"$var_value\""
set cmd "set $v \"$var_value\""
ns_log Notice "upload-companies-2: cmd=$cmd"
set result [eval $cmd]
}
# -------------------------------------------------------
# Transform the variables
set i 0
foreach varname $var_name_list {
foreach i [array names parser] {
set varname $header_var($i)
set p $parser($i)
set p_args $parser_args($i)
set target_varname $map($i)
ns_log Notice "import-im_project: Parser: $varname -> $target_varname"
ns_log Notice "import-im_project: Parser: p_args: $p_args"
ns_log Notice "import-im_project: Parser: target_varname: $target_varname"
switch $p {
no_change {
set $target_varname [set $varname]
ns_log Notice "import-im_project: Parser: no change: varname: $varname, val: [set $varname]"
}
default {
set proc_name "im_csv_import_parser_$p"
if {[catch {
set val [set $varname]
ns_log Notice "import-im_project: Parser: val: $val"
if {"" != $val} {
set result [$proc_name -parser_args $p_args $val]
set res [lindex $result 0]
set err [lindex $result 1]
ns_log Notice "import-im_project: Parser: '$p -args $p_args $val' -> $target_varname=$res, err=$err"
if {"" != $err} {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err</pre></font>\n"
}
}
set $target_varname $res
}
} err_msg]} {
ns_log Notice "import-im_project: i=$i: var=$varname -> target=$target_varname using parser=$p ($p_args)"
set proc_name "im_csv_import_parser_$p"
if {[catch {
set val [set $varname]
set $target_varname ""
if {"" != $val} {
set result [$proc_name -parser_args $p_args $val]
set res [lindex $result 0]
set err [lindex $result 1]
ns_log Notice "import-im_project: var=$varname -> target=$target_varname using parser=$p ($p_args): '$val' -> '$res', err=$err"
if {"" ne $err} {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err_msg</pre></font>"
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err</pre></font>\n"
}
}
set $target_varname $res
}
} err_msg]} {
if {$ns_write_p} {
ns_write "<li><font color=brown>Warning: Error parsing field='$target_varname' using parser '$p':<pre>$err_msg</pre></font>"
}
}
incr i
ns_log Notice "import-im_project: "
ns_log Notice "import-im_project: -------------------------------------------"
ns_log Notice "import-im_project: "
}
# -------------------------------------------------------
# Specific field transformations
# -------------------------------------------------------
# project_name needs to be there
if {"" == $project_name} {
......@@ -266,15 +260,6 @@ foreach csv_line_fields $values_list_of_lists {
continue
}
# project_nr needs to be there
if {"" == $project_nr} {
if {$ns_write_p} {
ns_write "<li><font color=red>Error: We have found an empty 'Project Nr' in line $cnt.<br>
Please correct the CSV file. Every project needs to have a unique Project Nr.</font>\n"
}
continue
}
# Parent0 - Parent1 - Parent2 - Parent3
if {"" ne $parent0} {
if {$ns_write_p} { ns_write "<li>Parent0: Looking for a main project with name=$parent0</font>\n" }
......@@ -373,7 +358,7 @@ foreach csv_line_fields $values_list_of_lists {
continue
}
}
# Status is a required field
set project_status_id [im_id_from_category $project_status "Intranet Project Status"]
if {"" == $project_status_id} {
......@@ -381,11 +366,12 @@ foreach csv_line_fields $values_list_of_lists {
set project_status_id [im_project_status_open]
}
# Type is a required field
# Project type is optional: Main projects are assumed to be "Gantt Projects", while sub-project are assumed to be "Task"
set project_type_id [im_id_from_category [list $project_type] "Intranet Project Type"]
if {"" == $project_type_id} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find project type '$project_type', using default type 'Other'</font>\n" }
set project_type_id [im_project_type_gantt]
if {"" ne $parent_id} { set project_type_id [im_project_type_task] }
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find project type '$project_type', using default type '[im_category_from_id $project_type_id]'</font>\n" }
}
# start_date and end_date are required fields for projects, not tasks
......@@ -432,6 +418,18 @@ foreach csv_line_fields $values_list_of_lists {
}
}
# project_nr is based on the parent
if {"" == $project_nr} {
set project_nr [im_next_project_nr -customer_id $company_id -parent_id $parent_id]
#if {$ns_write_p} {
# ns_write "<li><font color=red>Error: We have found an empty 'Project Nr' in line $cnt.<br>
# Please correct the CSV file. Every project needs to have a unique Project Nr.</font>\n"
#}
#continue
}
# Project Lead / Project Manager
if { "" eq $project_lead_id && 100 != $project_type_id } {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: No project manager found. Will try to create project w/o PM. </font>\n" }
......@@ -591,7 +589,7 @@ foreach csv_line_fields $values_list_of_lists {
}
# Task UoM
if {"" eq $uom} { set uom "Hour" }
if {"" eq $uom} { set uom "Day" }
set uom_id [im_id_from_category [list $uom] "Intranet UoM"]
if {"" eq $uom_id} {
if {$ns_write_p} { ns_write "<li><font color=brown>Warning: Didn't find UoM '$uom', using default 'Hour'</font>\n" }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment