Initial Import

intranet-csv-import.info

+<?xml version="1.0"?>
+<!-- Generated by the OpenACS Package Manager -->
+
+<package key="intranet-csv-import" url="http://openacs.org/repository/apm/packages/intranet-csv-import" type="apm_application">
+    <package-name>]project-open[ CSV Import</package-name>
+    <pretty-plural>]project-open[ CSV Import</pretty-plural>
+    <initial-install-p>f</initial-install-p>
+    <singleton-p>t</singleton-p>
+    <implements-subsite-p>f</implements-subsite-p>
+    <inherit-templates-p>f</inherit-templates-p>
+    <auto-mount>intranet-csv-import</auto-mount>
+
+    <version name="0.1d" url="http://openacs.org/repository/download/apm/intranet-csv-import-0.1d.apm">
+        <owner url="mailto:frank.bergmann@project-open.com">Frank Bergmann</owner>
+        <summary>Import objects from CSV files</summary>
+        <vendor url="http://www.project-open.com/">]project-open[</vendor>
+        <description format="text/plain">Allows to map CSV column names to ]po[ fields and imports various type of objects</description>
+        <maturity>0</maturity>
+
+        <provides url="intranet-csv-import" version="0.1d"/>
+
+        <callbacks>
+        </callbacks>
+        <parameters>
+        <!-- No version parameters -->
+        </parameters>
+
+    </version>
+</package>

tcl/intranet-csv-import-procs.tcl

+# /packages/intranet-cvs-import/tcl/intranet-cvs-import-procs.tcl
+#
+# Copyright (C) 2011 ]project-open[
+#
+# All rights reserved. Please check
+# http://www.project-open.com/license/ for details.
+
+ad_library {
+    @author frank.bergmann@project-open.com
+}
+
+
+# ----------------------------------------------------------------------
+# 
+# ----------------------------------------------------------------------
+
+ad_proc -public im_csv_import_label_from_object_typet {
+    -object_type:required
+} {
+    Returns the main navbar lable for the object_type
+} {
+    switch $object_type {
+	im_company { return "companies" }
+	im_project { return "projects" }
+	person { return "users" }
+	default { return "" }
+    }
+}
+
+
+
+
+# ---------------------------------------------------------------------
+# Available Fields per Object Type
+# ---------------------------------------------------------------------
+
+ad_proc -public im_csv_import_object_fields {
+    -object_type:required
+} {
+    Returns the main navbar lable for the object_type
+} {
+    switch $object_type {
+	im_project {
+	    set object_fields {
+		customer_name
+		parent_nrs
+		project_nr
+		project_name
+		project_status
+		project_type
+		start_date
+		end_date
+		customer_contact
+		on_track_status
+		percent_completed
+		project_manager
+		project_priority
+		program
+		milestone_p
+		description
+		note
+		material
+		uom
+		planned_units
+		billable_units
+		cost_center_code
+		timesheet_task_priority
+		sort_order
+		project_budget
+		project_budget_currency
+		project_budget_hours
+		presales_probability
+		presales_value
+		project_path
+		confirm_date
+		source_language
+		subject_area
+		final_company
+		expected_quality
+		customer_project_nr
+	    }
+	}
+	default {
+	    ad_return_complaint 1 "Unknown object type '$object_type'"
+	    ad_script_abort
+	}
+    }
+
+    set dynfield_sql "
+	select	aa.*,
+		a.*,
+		w.*
+	from	im_dynfield_widgets w,
+		im_dynfield_attributes a,
+		acs_attributes aa
+	where	a.widget_name = w.widget_name and
+		a.acs_attribute_id = aa.attribute_id and
+		aa.object_type in ('im_project', 'im_timesheet_task') and
+		(also_hard_coded_p is null OR also_hard_coded_p = 'f')
+    "
+    db_foreach dynfields $dynfield_sql {
+	lappend object_fields $attribute_name
+    }    
+
+    return $object_fields
+}
+
+
+# ---------------------------------------------------------------------
+# Available Parsers
+# ---------------------------------------------------------------------
+
+ad_proc -public im_csv_import_parsers {
+    -object_type:required
+} {
+    Returns the list of available parsers
+} {
+    switch $object_type {
+	im_project {
+	    set parsers {
+		no_change	"No Change"
+		date_european	"European Data Parser (DD.MM.YYYY)"
+	    }
+	}
+	default {
+	    ad_return_complaint 1 "Unknown object type '$object_type'"
+	    ad_script_abort
+	}
+    }
+    return $parsers
+}
+
+
+
+# ---------------------------------------------------------------------
+# Guess the most appropriate parser for a column
+# ---------------------------------------------------------------------
+
+ad_proc -public im_csv_import_guess_parser {
+    {-sample_values {}}
+    -object_type:required
+    -field_name:required
+} {
+    Returns the best guess for a parser for the given field
+} {
+    # Abort if there are not enough values
+    if {[llength $sample_values] < 2} { return "" }
+
+    # Available parsers
+    set date_european_p 1
+    set date_american_p 1
+    set number_plain_p 1
+    set number_european_p 1
+    set number_american_p 1
+
+    # set the parserst to 0 if one of the values doesn't fit
+    foreach val $sample_values { 
+
+	if {![regexp {^(.+)\.(.+)\.(....)$} $val match]} { set date_european_p 0 } 
+	if {![regexp {^[0-9]+$} $val match]} { set number_plain 0 } 
+
+    }
+
+    if {$date_european_p} { return "date_european" }
+    if {$number_plain_p} { return "number_plain" }
+
+    return ""
+}
+
+
+# ---------------------------------------------------------------------
+# Convert the list of parent_nrs into the parent_id
+# ---------------------------------------------------------------------
+
+ad_proc -public im_csv_import_convert_project_parent_nrs { 
+    {-parent_id ""}
+    parent_nrs 
+} {
+    Returns {parent_id err}
+} {
+    ns_log Notice "im_csv_import_convert_project_parent_nrs -parent_id $parent_id $parent_nrs"
+
+    # Recursion end - just return the parent.
+    if {"" == $parent_nrs} { return [list $parent_id ""] }
+    
+    # Lookup the first parent_nr below the current parent_id
+    set parent_nr [lindex $parent_nrs 0]
+    set parent_nrs [lrange $parent_nrs 1 end]
+
+    set parent_sql "= $parent_id"
+    if {"" == $parent_id} { set parent_sql "is null" }
+
+    set parent_id [db_string pid "
+	select	project_id
+	from	im_projects
+	where	parent_id $parent_sql and
+		lower(project_nr) = lower(:parent_nr)
+    "]
+
+    if {"" == $parent_id} {
+	return [list "" "Didn't find project with project_nr='$project_nr' and parent_id='$parent_id'"]
+    }
+
+    return [im_csv_import_convert_project_parent_nrs -parent_id $parent_id $parent_nrs]
+}
\ No newline at end of file

www/import-2.adp

+<master>
+<property name="title">@page_title@</property>
+<property name="context">@context_bar@</property>
+<property name="main_navbar_label">@main_navbar_label@</property>
+
+<form enctype="multipart/form-data" method=POST action="import-@object_type@.tcl">
+<%= [export_form_vars object_type return_url import_filename] %>
+
+     <table>
+     <tr clas=rowtitle>
+     <td class=rowtitle>Field Name</td>
+     <td class=rowtitle>Row 1</td>
+     <td class=rowtitle>Row 2</td>
+     <td class=rowtitle>Mapping</td>
+     <td class=rowtitle>Transformation</td>
+     </tr>
+
+     <multiple name=mapping>
+
+     <if @mapping.rownum@ odd><tr class="list-odd"></if>
+     <else><tr class="list-even"></else>
+
+     <td>@mapping.field_name@ @mapping.column;noquote@</td>
+     <td>@mapping.row_1@</td>
+     <td>@mapping.row_2@</td>
+     <td>@mapping.map;noquote@</td>
+     <td>@mapping.parser;noquote@</td>
+     </tr>
+     </multiple>
+     </table>
+
+     <table>
+     <tr>
+     <td>Save Mapping as:</td>
+     <td><input type=text name=mapping_name></td>
+     </tr>
+     <tr>
+     <td></td>
+     <td><input type=submit value="#intranet-csv-import.Import_CVS#"></td>
+     </tr>
+     </table>
+
+</form>
+

www/import-2.tcl

+# /packages/intranet-csv-import/www/import-2.tcl
+#
+ad_page_contract {
+    Starts the analysis process for the file imported
+    @author frank.bergmann@project-open.com
+} {
+    { return_url "" }
+    { main_navbar_label "" }
+    object_type
+    upload_file
+    object_type
+}
+
+# ---------------------------------------------------------------------
+# Default & Security
+# ---------------------------------------------------------------------
+
+set current_user_id [ad_maybe_redirect_for_registration]
+set page_title [lang::message::lookup "" intranet-cvs-import.Upload_Objects "Upload Objects"]
+set context_bar [im_context_bar "" $page_title]
+set admin_p [im_is_user_site_wide_or_intranet_admin $current_user_id]
+if {!$admin_p} {
+    ad_return_complaint 1 "Only administrators have the right to import objects"
+    ad_script_abort
+}
+
+# Get the file from the user.
+# number_of_bytes is the upper-limit
+set max_n_bytes [ad_parameter -package_id [im_package_filestorage_id] MaxNumberOfBytes "" 0]
+set tmp_filename [ns_queryget upload_file.tmpfile]
+im_security_alert_check_tmpnam -location "import-2.tcl" -value $tmp_filename
+if { $max_n_bytes && ([file size $tmp_filename] > $max_n_bytes) } {
+    ad_return_complaint 1 "Your file is larger than the maximum permissible upload size:  
+    [util_commify_number $max_n_bytes] bytes"
+    ad_script_abort
+}
+
+# strip off the C:\directories... crud and just get the file name
+if ![regexp {([^//\\]+)$} $upload_file match filename] {
+    # couldn't find a match
+    set filename $upload_file
+}
+
+if {[regexp {\.\.} $filename]} {
+    set error "Filename contains forbidden characters"
+    ad_returnredirect "/error.tcl?[export_url_vars error]"
+}
+
+if {![file readable $tmp_filename]} {
+    ad_return_complaint 1 "Unable to read the file '$tmp_filename'. <br>
+    Please check the file permissions or contact your system administrator.\n"
+    ad_script_abort
+}
+
+set import_filename "${tmp_filename}_copy"
+
+catch {
+    exec cp $tmp_filename $import_filename
+}
+
+# ---------------------------------------------------------------------
+# Open and parse the file
+# ---------------------------------------------------------------------
+
+set encoding "utf-8"
+if {[catch {
+    set fl [open $tmp_filename]
+    fconfigure $fl -encoding $encoding
+    set lines_content [read $fl]
+    close $fl
+} err]} {
+    ad_return_complaint 1 "Unable to open file $tmp_filename:<br><pre>\n$err</pre>"
+    ad_script_abort
+}
+
+
+# Extract the header line from the file
+set lines [split $lines_content "\n"]
+set separator [im_csv_guess_separator $lines]
+# ad_return_complaint 1 $separator
+set lines_len [llength $lines]
+set header [lindex $lines 0]
+set headers [im_csv_split $header $separator]
+set header_len [llength $headers]
+set values_lol [im_csv_get_values $lines_content $separator]
+
+# Take a sample of max_row rows from the file and show
+set max_row 10
+for {set i 1} {$i <= $max_row} {incr i} {
+   set row_$i [im_csv_split [lindex $lines $i] $separator]
+}
+
+# Get the list of all available fields for the object_type
+set object_fields [im_csv_import_object_fields -object_type $object_type]
+
+# Determine the list of parsers for the object_type
+set parser_pairs [im_csv_import_parsers -object_type $object_type]
+
+# ---------------------------------------------------------------------
+# Create and fill the multirow
+# ---------------------------------------------------------------------
+
+# Setup the multirow with some sample rows
+multirow create mapping field_name column map parser
+for {set i 1} {$i < $max_row} {incr i} {
+    multirow extend mapping "row_$i"
+}
+
+set object_type_pairs [list]
+foreach field $object_fields { lappend object_type_pairs $field $field }
+
+set cnt 0
+foreach header_name $headers {
+
+    # Column - Name of the CSV colum
+    set column "<input type=hidden name=column.$cnt value=\"$header_name\">"
+
+    # Mapping - Map to which object field?
+    set default_map $header_name
+    set map [im_select map.$cnt $object_type_pairs $default_map]
+
+    # Parser - convert the value from CSV values to ]po[ values
+    set parser_sample_values [list]
+    for {set i 1} {$i <= $max_row} {incr i} {
+	set row_name "row_$i"
+	set val [lindex [set $row_name] $cnt]
+	if {"" != $val} { lappend parser_sample_values $val }
+    }
+    set default_parser [im_csv_import_guess_parser -object_type $object_type -field_name $header_name -sample_values $parser_sample_values]
+    set parser [im_select parser.$cnt $parser_pairs $default_parser]
+
+    set first_val [lindex $row_1 $cnt]
+    set second_val [lindex $row_2 $cnt]
+    multirow append mapping $header_name $column $map $parser $first_val $second_val
+
+    incr cnt
+}
+

www/index.adp

+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<master>
+<property name="title">@page_title@</property>
+<property name="context">@context_bar@</property>
+<property name="main_navbar_label">@main_navbar_label@</property>
+
+<form enctype="multipart/form-data" method=POST action="import-2.tcl">
+<%= [export_form_vars return_url main_navbar_label] %>
+
+     <table border=0>
+     <tr> 
+	<td>#intranet-csv-import.Object_Type#</td>
+	<td> 
+	  <%= [im_select object_type [list im_project "Project" im_company "Company" person "User"] $object_type] %>
+	</td>
+     </tr>
+
+     <tr> 
+	<td>Filename</td>
+	<td> 
+	  <input type=file name=upload_file size=30>
+	<%= [im_gif help "Use the &quot;Browse...&quot; button to locate your file, then click &quot;Open&quot;."] %>
+	</td>
+     </tr>
+
+     <tr> 
+	<td></td>
+	<td> 
+	  <input type=submit>
+	</td>
+    </tr>
+
+    </table>
+</form>
+
+
+
+<table>
+<tr>
+<td>
+We espect that the import file contains the column names in the first row.
+</td>
+</tr>
+</table>
+

www/index.tcl

+# /package/intranet-core/companies/upload-companies.tcl
+#
+# Copyright (C) 2004 ]project-open[
+#
+# This program is free software. You can redistribute it
+# and/or modify it under the terms of the GNU General
+# Public License as published by the Free Software Foundation;
+# either version 2 of the License, or (at your option)
+# any later version. This program is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY;
+# without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+
+ad_page_contract {
+    Serve the user a form to upload a new file or URL
+
+    @author Frank Bergmann (frank.bergmann@project-open.com)
+    @creation-date July 2003
+} {
+    { return_url "" }
+    { object_type "" }
+}
+
+set user_id [ad_maybe_redirect_for_registration]
+set page_title [lang::message::lookup "" intranet-csv-import.Upload_file "Upload File"]
+set context_bar [im_context_bar "" $page_title]
+set main_navbar_label [im_csv_import_label_from_object_typet -object_type $object_type]
+