Skip to content

I
intranet-csv-import
Commit b3ebdbf3 authored by Frank Bergmann's avatar Frank Bergmann
Browse files
Options

- CSV Import 

  - Created reasonable sample data
  - Added examples for Conf Item, Company and Projects
  - Added documentations on index.adp page
  - Fixed a number of issues for each of the object types
parent 75d8be5c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 596 additions and 59 deletions
tcl/intranet-csv-import-parser-procs.tcl
View file @ b3ebdbf3
......@@ -54,7 +54,7 @@ ad_proc -public im_csv_import_parser_user_name {
set user_id [db_string user_id_from_username "
select min(user_id)
from users
where lower(trim(username)) = lower(trim(:name))
where lower(trim(username)) = lower(trim(:arg))
" -default ""]
}
......@@ -63,7 +63,7 @@ ad_proc -public im_csv_import_parser_user_name {
set user_id [db_string user_id_from_first_last_name "
select min(person_id)
from persons
where lower(trim(im_name_from_user_id(person_id))) = lower(trim(:name))
where lower(trim(im_name_from_user_id(person_id))) = lower(trim(:arg))
" -default ""]
}
......
tcl/intranet-csv-import-procs.tcl
View file @ b3ebdbf3
......@@ -19,6 +19,18 @@ ad_library {
# - Parser arguments (important for im_category type)
# ---------------------------------------------------------------------
ad_proc -public im_csv_import_guess_im_company { } {} {
set mapping {
{company_name "Company Name" no_change ""}
{company_path "Company Path" no_change ""}
{company_status_id "Company Status" category "Intranet Company Status"}
{company_type_id "Company Type" category "Intranet Company Type"}
{primary_contact_id "Primary Contact" user_name ""}
{accounting_contact_id "Accounting Contact" user_name ""}
}
return $mapping
}
ad_proc -public im_csv_import_guess_im_project { } {} {
set mapping {
{parent_id "Parent Nrs" project_parent_nrs ""}
......@@ -125,9 +137,14 @@ ad_proc -public im_csv_import_object_fields {
}
set super_types $s
# special logic for some object types
switch $object_type {
im_company { lappend super_types "im_office" }
}
# ---------------------------------------------------------------
# Get the list of tables associated with the object type and its super types
#
set tables_sql "
select *
from (
......@@ -179,6 +196,18 @@ ad_proc -public im_csv_import_object_fields {
# Guess the most probable object field (DynField) for a column
# ---------------------------------------------------------------------
ad_proc -public csv_norm {
field_name
} {
Performs normalization including trim, tolower,
replace non-ascii with "_".
} {
set field_name [string tolower [string trim $field_name]]
regsub -all {[^a-zA-Z0-9]} $field_name "_" field_name
return $field_name
}
ad_proc -public im_csv_import_guess_map {
-object_type:required
-field_name:required
......@@ -192,7 +221,7 @@ ad_proc -public im_csv_import_guess_map {
<li>Pretty Name: English pretty name of column ("Project Name")
<li>
} {
set field_name_lower [string tolower $field_name]
set field_name_lower [csv_norm $field_name]
ns_log Notice "im_csv_import_guess_map: trying to guess attribute_name for field_name=$field_name_lower"
im_security_alert_check_alphanum -location "im_csv_import_guess_map: object_type" -value $object_type
......@@ -206,12 +235,15 @@ ad_proc -public im_csv_import_guess_map {
set pretty_name [lindex $tuple 1]
set parser [lindex $tuple 2]
set parser_args [lindex $tuple 3]
if {$field_name_lower == [string tolower $pretty_name]} {
if {$field_name_lower eq [csv_norm $pretty_name]} {
ns_log Notice "im_csv_import_guess_map: found statically encoded match with field_name=$field_name"
return $attribute_name
}
}
switch $object_type {
im_company { lappend object_type "im_office" }
}
set dynfield_sql "
select lower(aa.attribute_name) as attribute_name,
......@@ -223,25 +255,28 @@ ad_proc -public im_csv_import_guess_map {
acs_attributes aa
where a.widget_name = w.widget_name and
a.acs_attribute_id = aa.attribute_id and
aa.object_type = '$object_type'
aa.object_type in ('[join $object_type "', '"]')
order by aa.sort_order, aa.attribute_id
"
# Check if the header name is the attribute_name of a DynField
set dynfield_attribute_names [util_memoize [list db_list otype_dynfields "select attribute_name from ($dynfield_sql) t"]]
ns_log Notice "im_csv_import_guess_map: attribute_names=$dynfield_attribute_names"
if {[lsearch $dynfield_attribute_names $field_name_lower] >= 0} {
ns_log Notice "im_csv_import_guess_map: found attribute_name match with field_name=$field_name"
return $field_name_lower
foreach field $dynfield_attribute_names {
if {$field_name_lower eq [csv_norm $field]} {
ns_log Notice "im_csv_import_guess_map: found attribute_name match with field_name=$field_name"
return $field_name_lower
}
}
# Check for a pretty_name of a DynField
set dynfield_pretty_names [util_memoize [list db_list otype_dynfields "select pretty_name from ($dynfield_sql) t"]]
ns_log Notice "im_csv_import_guess_map: pretty_names=$dynfield_pretty_names"
set idx [lsearch $dynfield_pretty_names $field_name_lower]
if {$idx >= 0} {
ns_log Notice "im_csv_import_guess_map: found pretty_name match with field_name=$field_name"
return [lindex $dynfield_attribute_names $idx]
foreach field $dynfield_pretty_names {
if {$field_name_lower eq [csv_norm $field]} {
ns_log Notice "im_csv_import_guess_map: found pretty_name match with field_name=$field_name"
return $field_name_lower
}
}
ns_log Notice "im_csv_import_guess_map: Did not find any match with a DynField for field_name=$field_name"
......@@ -411,4 +446,24 @@ ad_proc -public im_csv_import_check_list_of_lists {
}
return $result
}
\ No newline at end of file
}
# ----------------------------------------------------------------------
#
# ----------------------------------------------------------------------
ad_proc -public im_csv_import_label_from_object_type {
-object_type:required
} {
Returns the main navbar lable for the object_type
} {
switch $object_type {
im_company { return "companies" }
im_project { return "projects" }
person { return "users" }
default { return "" }
}
}
www/examples/im_company.csv 0 → 100644
View file @ b3ebdbf3
Company Name;Company Path;Company Status;Company Type;Primary Contact;Accounting Contact;Phone;Fax;Address Line1;Address Line2;Address City;Address State;Address Postal Code;Address Country code;Note
]project-open[;projop;Active;Provider;Frank Bergmann;Frank Bergmann;+34 933 250 914;+34 932 890 729;Ronda de Sant Anton, 51, 1o 2a;;Barcelona;Catalonia;8011;es;"Test note with single quote (') and double quote ("")"
ABC Consulting;abc_consulting;Active;Customer;Sheila Carter;Antonio Salas;+41 51 376 60 60;+41 51 376 60 60;Tunastrae 12;Block 2;Basel;;5123;ch;
adromIT;adromit;Active;Customer;David Rolland;;+49 89 33036742;+49 89 33036743;Felix-Fechenbach-Bogen 111;;Mnchen;;80797;de;
Angelique Picard;freelance_8811;Active;Freelance Provider;Angelique Picard;Angelique Picard;+33 594 87 49 92;;198 Promenade des Americaine;;Nice;;6010;fr;
Axios;axios;Active;Customer;;;+41 51 376 60 60;;;;bogota;CU;;co;
www/examples/im_conf_item.csv 0 → 100644
View file @ b3ebdbf3
Conf Item Parent Nrs;Conf Item Name;Conf Item Code;Conf Item Version;Conf Item Nr;Conf Item Status;Conf Item Type;IP Address;Conf Item Owner;Conf Item Cost Center;Description;Note
;]project-open[;;5.0.1;project-open;Active;Software Application;10.0.12.67;Ben Bigboss;CoIt;ITSM and project management;Test note with single quote (')
;Alfresco;;3.4.d2;alfresco;Active;Software Application;10.0.12.124;System Administrator;CoMa;Document management system;
;AlienVault;;4.0;alienvault;Active;Software Application;10.0.12.28;System Administrator;CoIt;Thread detection;"Test note with double quote ("")"
;Bacula ;;7.4.0;bacula;Active;Software Application;10.0.14.52;System Administrator;CoIt;Backup;
;eXo platform;;;exo;Active;Software Application;10.0.14.45;System Administrator;CoIt;Enterprise social platform;
;ICinga 2;;;icinga2;Active;Software Application;10.0.12.5;System Administrator;CoIt;Systems monitoring;
;Jasper Reports;;;jasper-reports;Active;Software Application;10.0.12.20;Ben Bigboss;CoIt;Reporting;
;Mandriva Qnap;;;mandriva-qnap;Active;Software Application;10.0.134.74;System Administrator;CoIt;Windows asset management;
;Openbravo;;;openbravo;Active;Software Application;10.0.12.80;Ben Bigboss;CoAd;ERP;
;PacketFence;;;packetfence;Active;Software Application;10.0.12.54;System Administrator;CoIt;Network access control;
;Pentaho;;;pentaho;Active;Software Application;10.0.12.25;System Administrator;CoIt;Business intelligence;
pentaho;Pentaho Kettle;;;kettle;Active;Software Application;10.0.12.40;System Administrator;CoIt;Pentaho extract, transform and loading;
pentaho;Pentaho Mondrian;;;mondrian;Active;Software Application;10.0.11.6;System Administrator;CoIt;Pentaho data-warehouse;
;PostgreSQL;;9.5.2;postgresql;Active;Software Application;10.0.15.54;System Administrator;CoIt;PostgreSQL database server;
;Samba 4;;4.0.3;samba4;Active;Software Application;10.0.11.89;System Administrator;CoIt;File server and identity management;
;SugarCRM;;4.9;sugarcrm;Active;Software Application;10.0.12.42;Samuel Salesmanager;CoMa;Customer relationship management
;Zimbra;;8.6.0;zimbra;Active;Software Application;10.0.3.6;System Administrator;CoIt;Collaboration
www/examples/im_conf_items-applications.csv deleted 100644 → 0
View file @ 75d8be5c
conf_item_name;conf_item_parent_nrs;conf_item_status;conf_item_type;description
]project-open[;;Active;Software Application;ITSM and project management
AlienVault;;Active;Software Application;Thread detection
Bacula ;;Active;Software Application;Backup
Alfresco;;Active;Software Application;Document management system
eXo platform;;Active;Software Application;Enterprise social platform
ICinga 2;;Active;Software Application;Systems monitoring
Jasper Reports;;Active;Software Application;Reporting
Mandriva Qnap;;Active;Software Application;Windows asset management
Openbravo;;Active;Software Application;ERP
PacketFence;;Active;Software Application;Network access control
Pentaho;;Active;Software Application;Business intelligence
Pentaho Kettle;Pentaho;Active;Software Application;Pentaho extract, transform and loading
Pentaho Mondrian;Pentaho;Active;Software Application;Pentaho data-warehouse
Samba 4;;Active;Software Application;File server and identity management
SugarCRM;;Active;Software Application;Customer relationship management
Zimbra;;Active;Software Application;Collaboration
www/examples/im_project.csv 0 → 100644
View file @ b3ebdbf3
Parent Nrs;Project Nr;Project Name;Customer Name;Project Status;Project Type;Start Date;End Date;Percent Completed;Budget;Budget Hours;Release Item;Milestone?;Program;Presales Probability;Presales Value;Project Sponsor;Priority;Note;Project Manager;On Track;Description
;2016_0001;Infrastructure Program;Tigerpond;Open;Program;1.1.2016;1.7.2017;0.00;200.000;;f;f;;;;Ben Bigboss;;;Ben Bigboss;Green;
;2015_0003;Fargo Samba4 Rollout;Tigerpond;Open;Gantt Project;30/06/2015;04/10/2015;0.00;10.000;200;f;f;Infrastructure Program;95;20000;Ben Bigboss;High;"Test note with single tick (') and double quotes ("")";System Administrator;Green;
2015_0003;task_0002;Write Specs;Tigerpond;Open;Task;30/06/2015;02/07/2015;100;;;f;f;;.00;;;;;;Green;"Test description with single tick (') and double quotes ("")"
2015_0003;task_0003;Negotiate Specs;Tigerpond;Open;Task;02/07/2015;06/08/2015;50;;;f;t;;0;;;;;;Green;
2015_0003;task_0004;Design;Tigerpond;Open;Gantt Project;30/07/2015;06/08/2015;20;;;f;f;;;;;;;;Green;
2015_0003;task_0005;Implementation;Tigerpond;Open;Task;06/08/2015;20/09/2015;0.00;;;f;t;;;;;;;;Green;
2015_0003;task_0006;Testing;Tigerpond;Open;Task;16/09/2015;28/09/2015;5.00;;;f;t;;;;;;;;Green;
2015_0003;task_0007;Sign-Off;Tigerpond;Open;Task;30/09/2015;04/10/2015;.00;;;f;t;;;;;;;;Green;
;2014_0007;Special Machine (2014-05-12);Boom Consuling;Open;Gantt Project;2015-03-01;2016-05-07;0.00;25000;400;f;f;Infrastructure Program;100;30000;bbigboss@tigerpond.com;High;;Ben Bigboss;Yellow;
2014_0007;task_0086;Milestones;Boom Consuling;Open;Gantt Project;2015-11-05;2016-05-07;12.34;;;f;f;;;;;;;;Green;
2014_0007;task_0003;Phase Montage;Boom Consuling;Open;Gantt Project;2016-01-18;2016-03-20;20.00;;;f;f;;;;;;;;Green;
2014_0007;task_0004;Phase Inbetriebnahme;Boom Consuling;Open;Gantt Project;2016-03-26;2016-04-14;30;;;f;f;;;;;;;;Green;
2014_0007;task_0055;Installation Kunde;Boom Consuling;Open;Gantt Project;2016-04-22;2016-05-07;.00;;;f;f;;;;;;;;Green;
;;;;;;
www/import-2.tcl
View file @ b3ebdbf3
......@@ -81,9 +81,11 @@ set headers [im_csv_split $header $separator]
set header_len [llength $headers]
set values_lol [im_csv_get_values $lines_content $separator]
# Check if there are lines with less then 4 elements
set error [im_csv_import_check_list_of_lists $values_lol]
if {"" != $error} { ad_return_complaint 1 $error }
# fraber 160315: Exporting from Excel, the last columns may be shorter
# set error [im_csv_import_check_list_of_lists $values_lol]
# if {"" != $error} { ad_return_complaint 1 $error }
# Take a sample of max_row rows from the file and show
set max_row 10
......
www/import-im_company.tcl
View file @ b3ebdbf3
......@@ -130,42 +130,42 @@ foreach csv_line_fields $values_list_of_lists {
# im_company
set company_id ""
set company_name ""
set company_path ""
set company_status_id ""
set company_type_id ""
set note ""
set company_id ""
set company_name ""
set company_path ""
set main_office_id ""
set deleted_p ""
set crm_status_id ""
set primary_contact_id ""
set accounting_contact_id ""
set note ""
set referral_source ""
set annual_revenue_id ""
set status_modification_date ""
set old_company_status_id ""
set billable_p ""
set site_concept ""
set manager_id ""
set contract_value ""
set crm_status_id ""
set deleted_p ""
set main_office_id ""
set manager_id ""
set note ""
set note ""
set old_company_status_id ""
set primary_contact_id ""
set referral_source ""
set site_concept ""
set start_date ""
set status_modification_date ""
set vat_number ""
set default_vat ""
set default_bill_template_id ""
set default_delnote_template_id ""
set default_discount_perc ""
set default_invoice_template_id ""
set default_payment_method_id ""
set default_payment_days ""
set default_bill_template_id ""
set default_payment_method_id ""
set default_pm_fee_perc ""
set default_po_template_id ""
set default_delnote_template_id ""
set default_quote_template_id ""
set default_tax ""
set default_pm_fee_perc ""
set default_surcharge_perc ""
set default_discount_perc ""
set default_tax ""
set default_vat ""
# im_office
set phone ""
......
www/index.adp
View file @ b3ebdbf3
......@@ -42,13 +42,70 @@
</table>
</form>
<br>&nbsp;<br>
<h1>Working CSV Examples</h1>
<table cellspacing="2" cellpadding="5">
<tr>
<td colspan=3>
<p>Please find below sample CSV files for each of the object types above.</p>
<p>Each file contains a list of column names in the first row. Column names are case insensitive and spaces may be replaced by "_" (underscore characters).</p>
</td>
</tr>
<tr>
<th><nobr>Object Type</nobr></th>
<th><nobr>Example CSV</nobr></th>
<th>Available Column Headers</th>
</tr>
<tr>
<td>Company</td>
<td align=center>
<a href="examples/im_company.csv">
<%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_company'"]] %>
</a>
</td>
<td>
Company Name, Company Path, Company Status, Company Type, Primary Contact, Accounting Contact, Note, VAT Number, Phone, Fax, Address Line1, Address Line2, Address City, Address State, Address Postal Code, Address Country Code, Default Payment Days, Default VAT.<br>
Company <a href="/intranet-dynfield/object-type?object_type=im_company">DynFields</a>
are also recognized by their "pretty name" or "column name".</p>
</td>
</tr>
<tr>
<td>Conf Item</td>
<td align=center>
<a href="examples/im_conf_item.csv">
<%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_conf_item'"]] %>
</a>
</td>
<td>
Conf Item Parent Nrs, Conf Item Name, Conf Item Code, Conf Item Nr, Conf Item Status, Conf Item Type, IP Address, Conf Item Owner, Conf Item Cost Center, Description, Note<br>
Conf Item <a href="/intranet-dynfield/object-type?object_type=im_conf_item">DynFields</a>
are also recognized by their "pretty name" or "column name".</p>
</td>
</tr>
<tr>
<td>Project</td>
<td align=center>
<a href="examples/im_project.csv">
<%= [im_gif [db_string gif "select object_type_gif from acs_object_types where object_type = 'im_project'"]] %>
</a>
</td>
<td>
Parent Nrs, Project Nr, Project Name, Customer Name, Project Status, Project Type, Start Date, End Date, Percent Completed, On Track Status, Budget, Budget Hours, Priority, Note, Project Manager, Description.<br>
Project <a href="/intranet-dynfield/object-type?object_type=im_project">DynFields</a>
are also recognized by their "pretty name" or "column name".</p>
</td>
</tr>
<table>
<tr>
<td>
We expect that the import file contains the column names in the first row.<br/>
</td>
<td colspan=3>
<p>The ]project-open[ team offers professional services to help you import data from
systems including SAP, Navision, Jira, MS-Project and many more.
Please <a href="http://www.project-open.com/en/contact">contact</a> us for more information.</p>
</td>
</tr>
</table>
www/index.tcl
View file @ b3ebdbf3
# /package/intranet-core/companies/upload-companies.tcl
# /package/intranet-csv-import/www/index.tcl
#
# Copyright (C) 2004 ]project-open[
#
......
www/reports/export-projects.tcl 0 → 100644
View file @ b3ebdbf3
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment