Skip to content

I
intranet-csv-import
Commit 23916393 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

- Fixed import for im_hours 

- Testing import with sample files
parent c240f6b6
Hide whitespace changes
Inline Side-by-side
Showing with 139 additions and 87 deletions
tcl/intranet-csv-import-parser-procs.tcl
View file @ 23916393
......@@ -125,16 +125,19 @@ ad_proc -public im_csv_import_parser_date {
} {
Generic date parser - front-end for all available date formats
} {
set result [im_csv_import_parser_date_iso -parser_args $parser_args $arg]
if {"" eq [lindex $result 1]} { return $result }
set result [im_csv_import_parser_date_european -parser_args $parser_args $arg]
if {"" eq [lindex $result 1]} { return $result }
set result [im_csv_import_parser_date_european_dashes -parser_args $parser_args $arg]
set result [im_csv_import_parser_date_european_slashes -parser_args $parser_args $arg]
if {"" eq [lindex $result 1]} { return $result }
set result [im_csv_import_parser_date_american -parser_args $parser_args $arg]
if {"" eq [lindex $result 1]} { return $result }
return [list "" "Could not figure out the format of this data field"]
return [list "" "Could not figure out the format of this data field '$arg'"]
}
ad_proc -public im_csv_import_parser_boolean {
......@@ -168,7 +171,7 @@ ad_proc -public im_csv_import_parser_date_european {
}
ad_proc -public im_csv_import_parser_date_european_dashes {
ad_proc -public im_csv_import_parser_date_european_slashes {
{-parser_args "" }
arg
} {
......@@ -179,7 +182,21 @@ ad_proc -public im_csv_import_parser_date_european_dashes {
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing European date format '$arg': expected 'dd/mm/yyyy' ()"]
return [list "" "Error parsing European date format '$arg': expected 'dd/mm/yyyy'"]
}
ad_proc -public im_csv_import_parser_date_iso {
{-parser_args "" }
arg
} {
Parses ISO date format like '2011-06-08' as the 8th of June, 2011
} {
if {[regexp {^(....)\-(.+)\-(.+)$} $arg match year month dom]} {
if {1 == [string length $dom]} { set dom "0$dom" }
if {1 == [string length $month]} { set dom "0$month" }
return [list "$year-$month-$dom" ""]
}
return [list "" "Error parsing ISO '$arg': expected 'yyyy-mm-dd'"]
}
......
tcl/intranet-csv-import-procs.tcl
View file @ 23916393
......@@ -44,15 +44,34 @@ ad_proc -public im_csv_import_guess_im_project { } {} {
return $mapping
}
ad_proc -public im_csv_import_guess_im_hour { } {} {
set mapping {
{project_id "Parent Nrs" project_parent_nrs ""}
{project_id "Project Parent Nrs" project_parent_nrs ""}
{user_id "User Email" user_name ""}
{user_id "User Name" user_name ""}
{user_id "User" user_name ""}
{day "Day" date ""}
{hours "Hours" number ""}
{note "Note" no_change ""}
}
return $mapping
}
ad_proc -public im_csv_import_guess_im_risk { } {} {
set mapping {
{risk_project_id "Risk Project Parent Nrs" project_parent_nrs ""}
{risk_name "Risk Name" no_change ""}
{risk_project_id "Project" project_nr ""}
{risk_status_id "Status" category "Intranet Risk Status"}
{risk_type_id "Type" category "Intranet Risk Type"}
{risk_status_id "Risk Status" category "Intranet Risk Status"}
{risk_type_id "Risk Type" category "Intranet Risk Type"}
{risk_description "Description" no_change ""}
{risk_description "Risk Description" no_change ""}
{risk_impact "Impact" number ""}
{risk_probability_percent "Probability" number ""}
{risk_probability_percent "Risk Probability" number ""}
}
return $mapping
}
......@@ -119,7 +138,7 @@ ad_proc -public im_csv_import_object_fields {
# Special case: im_hour is not an object
if { "im_hour" == $object_type } {
return "project_id project_nr project_nr_path user_id day hours"
return "project_id project_nr project_nr_path user_id day hours note"
}
# Special case: membership is not an object
......@@ -225,7 +244,6 @@ ad_proc -public im_csv_import_guess_map {
ns_log Notice "im_csv_import_guess_map: trying to guess attribute_name for field_name=$field_name_lower"
im_security_alert_check_alphanum -location "im_csv_import_guess_map: object_type" -value $object_type
# Check for manual override static mapping
set static_mapping_lol {}
catch { set static_mapping_lol [im_csv_import_guess_$object_type] }
......
www/import-im_hour.tcl
View file @ 23916393
......@@ -2,10 +2,10 @@
#
ad_page_contract {
@author frank.bergmann@project-open.com
@author klaus.hofeditz@project-open.com
@param mapping_name: Should we store the current mapping in the DB for future use?
@param column: Name of the CSV column
@param map: Name of the ]po[ object attribute
......@@ -34,9 +34,9 @@ ad_proc -private im_write_log {
} {
Writes log message to either screen or textfile
} {
if { !$write_log_p } { return }
set path_tmp [parameter::get -package_id [apm_package_id_from_key intranet-filestorage] -parameter "TmpPathUnix" -default 60]
if { "screen" == $output_device_log } {
ns_write $msg
......@@ -59,8 +59,6 @@ ad_proc -private im_write_log {
if { "" ne $ns_write_p } { set write_log_p $ns_write_p }
# ad_return_complaint xx "ns_write_p: $ns_write_p, write_log_p: $write_log_p, output_device_log: $output_device_log"
# ---------------------------------------------------------------------
# Default & Security
# ---------------------------------------------------------------------
......@@ -89,7 +87,7 @@ if {[catch {
if {![file readable $import_filename]} {
ad_return_complaint 1 "Unable to read the file '$import_filename'. <br>
Please check the file permissions or contact your system administrator.\n"
Please check the file permissions or contact your system administrator.\n"
ad_script_abort
}
......@@ -119,7 +117,7 @@ set values_list_of_lists [im_csv_get_values $lines_content $separator]
# ------------------------------------------------------------
# Render Result Header
if { "screen" == $output_device_log && $write_log_p } {
if {"screen" == $output_device_log && $write_log_p } {
ad_return_top_of_page "
[im_header]
[im_navbar]
......@@ -138,33 +136,33 @@ foreach csv_line_fields $values_list_of_lists {
im_write_log $output_device_log $write_log_p "</ul><hr>\n"
im_write_log $output_device_log $write_log_p "<ul><li>Starting to parse line $cnt</li>\n"
if {[llength $csv_line_fields] < 4} {
im_write_log $output_device_log $write_log_p"<li><font color=red>Error: We found a row with only [llength $csv_line_fields] columns.<br>
This is probabily because of a multi-line field in the row before.<br>Please correct the CSV file.</font>\n"
im_write_log $output_device_log $write_log_p"<li><font color=red>Error: We found a row with only [llength $csv_line_fields] columns.<br>
This is probabily because of a multi-line field in the row before.<br>Please correct the CSV file.</font>\n"
continue
}
# Preset values, defined by CSV sheet:
set user_id ""
set project_id ""
set user_id ""
set project_id ""
set target_project_id ""
set project_nr ""
set project_nr ""
set project_nr_path ""
set project_name ""
set target_project_name ""
set day ""
set hours ""
set billing_rate ""
set billing_currency ""
set note ""
set cost_id ""
set invoice_id ""
set internal_note ""
set material_id ""
set days ""
set hour_id ""
set conf_object_id ""
set project_name ""
set target_project_name ""
set day ""
set hours ""
set billing_rate ""
set billing_currency ""
set note ""
set cost_id ""
set invoice_id ""
set internal_note ""
set material_id ""
set days ""
set hour_id ""
set conf_object_id ""
foreach j [array names column] {
......@@ -208,37 +206,37 @@ foreach csv_line_fields $values_list_of_lists {
# day is mandatory
if {"" == $day} {
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: We have found an empty 'day' in line $cnt.<br>
Please correct the CSV file. 'day' is mandatory. Skipping line!</font></li>\n"
continue
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: We have found an empty 'day' in line $cnt.<br>
Please correct the CSV file. 'day' is mandatory. Skipping line!</font></li>\n"
continue
}
# user_id is mandatory
if {"" == $user_id} {
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: We have found an empty 'user_id' in line $cnt.<br>
Please correct the CSV file. 'user_id' is mandatory. Skipping line!</font></li>\n"
continue
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: We have found an empty 'user_id' in line $cnt.<br>
Please correct the CSV file. 'user_id' is mandatory. Skipping line!</font></li>\n"
continue
}
# either project_id, project_nr or project_path need to be provided
if {""eq $project_id && "" eq $project_nr_path && "" eq $project_nr } {
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: $cnt.<br>
Please correct the CSV file. Import requires at least one of the following fields: 'project_id', 'project_nr' or 'project_nr_path'. Skipping line!</font></li>\n"
continue
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: $cnt.<br>
Please correct the CSV file. Import requires a valid project_id from one of the fields: 'project_id', 'project_nr' or 'project_nr_path'. Skipping line</font></li>\n"
continue
}
# value for hours is mandatory
if {"" == $hours } {
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: We have found an empty 'hours' in line $cnt.<br>
Please correct the CSV file. 'hours' is mandatory. Skupping line!</font></li>\n"
continue
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: We have found an empty 'hours' in line $cnt.<br>
Please correct the CSV file. 'hours' is mandatory. Skupping line!</font></li>\n"
continue
}
# Check permissions
im_project_permissions $current_user_id $project_id view_p read_p write_p admin_p
if {!$write_p} {
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: You don't have write permissions for project #$project_id. Skipping line!</font></li>\n"
continue
continue
}
# -------------------------------------------------------
......@@ -295,7 +293,7 @@ foreach csv_line_fields $values_list_of_lists {
if { "" eq $project_name } { set project_name * }
if { "-1" eq [lsearch -exact $missing_project_list "project_nr_path / $project_nr / $project_name" ] } {
lappend missing_project_list "$project_nr_path / $project_nr / $project_name"
lappend missing_project_list "$project_nr_path / $project_nr / $project_name"
}
continue
}
......@@ -304,17 +302,16 @@ foreach csv_line_fields $values_list_of_lists {
# Check if there is a conf object
#
set conf_object_id_exist_p [db_string get_conf_object_id "
select conf_object_id
from im_hours h
where
h.day = :day and
h.user_id = :user_id and
h.project_id = :target_project_id
" -default 0]
if { $conf_object_id_exist_p } {
im_write_log $output_device_log $write_log_p "<li>Merging hours: <font color=red>Conf Object exists, skipping project_id: $target_project_id, user_id: $user_id, day: $day</font></li>\n"
continue
select count(conf_object_id)
from im_hours h
where h.day = :day and
h.user_id = :user_id and
h.project_id = :target_project_id
" -default 0]
if {$conf_object_id_exist_p} {
im_write_log $output_device_log $write_log_p "<li>Merging hours: <font color=red>Timesheet hours already approved, skipping project_id: $target_project_id, user_id: $user_id, day: $day</font></li>\n"
continue
}
# -------------------------------------------------------
......@@ -323,18 +320,17 @@ foreach csv_line_fields $values_list_of_lists {
set hours_before [db_string get_hours "
select coalesce(h.hours,0) as hours
from im_hours h
where
h.day = :day and
where h.day = :day and
h.user_id = :user_id and
h.project_id = :target_project_id
" -default ""]
" -default 0]
if { $hours_before > 0 && $merge_p } {
# Update
im_write_log $output_device_log $write_log_p "<li>Merging hours: project_id: $target_project_id, user_id: $user_id, day: $day</li>\n"
if { !$test_run_p } {
im_write_log $output_device_log $write_log_p "<li>Updating ...</li>\n"
db_dml sql "update im_hours h set (hours, note) values (h.hours + :hours_before, h.note || ', ' || :note) where h.project_id = :target_project_id and h.user_id = :user_id and h.day = :day"
db_dml sql "update im_hours h set (hours, note) = (h.hours + :hours_before, h.note || ', ' || :note) where h.project_id = :target_project_id and h.user_id = :user_id and h.day = :day"
}
} elseif { $hours_before > 0 && !$merge_p } {
# Overwrite
......@@ -356,7 +352,7 @@ foreach csv_line_fields $values_list_of_lists {
}
}
} else {
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: not writing anything for: $target_project_id, user_id: $user_id, day: $day</font></li>\n"
im_write_log $output_device_log $write_log_p "<li><font color=red>Error: not writing anything for: $target_project_id, user_id: $user_id, day: $day, hours_before: '$hours_before'</font></li>\n"
}
......@@ -387,9 +383,9 @@ if { "screen" == $output_device_log && $write_log_p } {
im_write_log $output_device_log $write_log_p [im_footer]
} elseif {"screen" != $output_device_log && $write_log_p } {
ad_return_top_of_page "
[im_header]
[im_navbar]
"
[im_header]
[im_navbar]
"
ns_write "Files imported. Logfile is located at: $path_tmp <br/>"
set fp [open $path_tmp r]
......@@ -401,9 +397,9 @@ if { "screen" == $output_device_log && $write_log_p } {
} else {
ad_return_top_of_page "
[im_header]
[im_navbar]
"
[im_header]
[im_navbar]
"
ns_write "Import finished."
}
www/import-im_membership.tcl
View file @ 23916393
......@@ -85,6 +85,7 @@ foreach csv_line_fields $values_list_of_lists {
if {$ns_write_p} { ns_write "<li>Starting to parse line $cnt\n" }
# Preset values, defined by CSV sheet:
set user_id ""
set project_name ""
set project_nr ""
set project_path ""
......
www/import-im_risk.tcl
View file @ 23916393
......@@ -62,9 +62,6 @@ set header_fields [im_csv_split $header $separator]
set header_len [llength $header_fields]
set values_list_of_lists [im_csv_get_values $lines_content $separator]
# ad_return_complaint 1 "<pre>[array get column]<br>[array get map]<br>[array get parser]<br>[array get parser_args]<br>$header_fields</pre>"
# ------------------------------------------------------------
# Get DynFields
......
www/index.adp
View file @ 23916393
......@@ -46,24 +46,23 @@
<h1>Working CSV Examples</h1>
<table cellspacing="2" cellpadding="5">
<tr>
<td colspan=3>
<td colspan=4>
<p>Please find below sample CSV files for each of the object types above.</p>
<p>Each file contains a list of column names in the first row. Column names are case insensitive and spaces may be replaced by "_" (underscore characters).</p>
</td>
</tr>
<tr>
<th><nobr>Object Type</nobr></th>
<th colspan=2><nobr>Object Type</nobr></th>
<th><nobr>Example CSV</nobr></th>
<th>Available Column Headers</th>
</tr>
<tr>
<td><%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_company'"]] %></td>
<td>Company</td>
<td align=center>
<a href="examples/im_company.csv">
<%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_company'"]] %>
</a>
<a href="examples/im_company.csv"><%= [im_gif zip-download] %></a>
</td>
<td>
Company Name, Company Path, Company Status, Company Type, Primary Contact, Accounting Contact, Note, VAT Number, Phone, Fax, Address Line1, Address Line2, Address City, Address State, Address Postal Code, Address Country Code, Default Payment Days, Default VAT.<br>
......@@ -73,11 +72,10 @@
</tr>
<tr>
<td><%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_conf_item'"]] %></td>
<td>Conf Item</td>
<td align=center>
<a href="examples/im_conf_item.csv">
<%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_conf_item'"]] %>
</a>
<a href="examples/im_conf_item.csv"><%= [im_gif zip-download] %></a>
</td>
<td>
Conf Item Parent Nrs, Conf Item Name, Conf Item Code, Conf Item Nr, Conf Item Status, Conf Item Type, IP Address, Conf Item Owner, Conf Item Cost Center, Description, Note<br>
......@@ -87,11 +85,23 @@
</tr>
<tr>
<td><%= [im_gif hourglass] %></td>
<td>Timesheet Hour</td>
<td align=center>
<a href="examples/im_hour.csv"><%= [im_gif zip-download] %></a>
</td>
<td>
Hour Project Parent Nrs, Hour Name, Hour Probability, Hour Impact, Hour Status, Hour Type, Hour Description, Hour Mitigation Plan<br>
Hour <a href="/intranet-dynfield/object-type?object_type=im_hour">DynFields</a>
are also recognized by their "pretty name" or "column name".</p>
</td>
</tr>
<tr>
<td><%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_project'"]] %></td>
<td>Project</td>
<td align=center>
<a href="examples/im_project.csv">
<%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_project'"]] %>
</a>
<a href="examples/im_project.csv"><%= [im_gif zip-download] %></a>
</td>
<td>
Parent Nrs, Project Nr, Project Name, Customer Name, Project Status, Project Type, Start Date, End Date, Percent Completed, On Track Status, Budget, Budget Hours, Priority, Note, Project Manager, Description.<br>
......@@ -101,7 +111,20 @@
</tr>
<tr>
<td colspan=3>
<td><%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_risk'"]] %></td>
<td>Risk</td>
<td align=center>
<a href="examples/im_risk.csv"><%= [im_gif zip-download] %></a>
</td>
<td>
Risk Project Parent Nrs, Risk Name, Risk Probability, Risk Impact, Risk Status, Risk Type, Risk Description, Risk Mitigation Plan<br>
Risk <a href="/intranet-dynfield/object-type?object_type=im_risk">DynFields</a>
are also recognized by their "pretty name" or "column name".</p>
</td>
</tr>
<tr>
<td colspan=4>
<p>The ]project-open[ team offers professional services to help you import data from
systems including SAP, Navision, Jira, MS-Project and many more.
Please <a href="http://www.project-open.com/en/contact">contact</a> us for more information.</p>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment