Commit 2c691ee8 authored by jplang's avatar jplang

Import issues from CSV file (#950).

git-svn-id: https://svn.redmine.org/redmine/trunk@14493 e93f8b46-1217-0410-a6f0-8f06a7374b81
parent 21929e60
# Redmine - project management software
# Copyright (C) 2006-2015 Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
require 'csv'
class ImportsController < ApplicationController
before_filter :find_import, :only => [:show, :settings, :mapping, :run]
before_filter :authorize_global
helper :issues
def new
end
def create
@import = IssueImport.new
@import.user = User.current
@import.file = params[:file]
@import.set_default_settings
if @import.save
redirect_to import_settings_path(@import)
else
render :action => 'new'
end
end
def show
end
def settings
if request.post? && @import.parse_file
redirect_to import_mapping_path(@import)
end
rescue CSV::MalformedCSVError => e
flash.now[:error] = l(:error_invalid_csv_file_or_settings)
rescue ArgumentError, Encoding::InvalidByteSequenceError => e
flash.now[:error] = l(:error_invalid_file_encoding, :encoding => ERB::Util.h(@import.settings['encoding']))
rescue SystemCallError => e
flash.now[:error] = l(:error_can_not_read_import_file)
end
def mapping
issue = Issue.new
issue.project = @import.project
issue.tracker = @import.tracker
@attributes = issue.safe_attribute_names
@custom_fields = issue.editable_custom_field_values.map(&:custom_field)
if request.post?
respond_to do |format|
format.html {
if params[:previous]
redirect_to import_settings_path(@import)
else
redirect_to import_run_path(@import)
end
}
format.js # updates mapping form on project or tracker change
end
end
end
def run
if request.post?
@current = @import.run(
:max_items => max_items_per_request,
:max_time => 10.seconds
)
respond_to do |format|
format.html {
if @import.finished?
redirect_to import_path(@import)
else
redirect_to import_run_path(@import)
end
}
format.js
end
end
end
private
def find_import
@import = Import.where(:user_id => User.current.id, :filename => params[:id]).first
if @import.nil?
render_404
return
elsif @import.finished? && action_name != 'show'
redirect_to import_path(@import)
return
end
update_from_params if request.post?
end
def update_from_params
if params[:import_settings].is_a?(Hash)
@import.settings ||= {}
@import.settings.merge!(params[:import_settings])
@import.save!
end
end
def max_items_per_request
5
end
end
# encoding: utf-8
#
# Redmine - project management software
# Copyright (C) 2006-2015 Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
module ImportsHelper
def options_for_mapping_select(import, field, options={})
tags = "".html_safe
blank_text = options[:required] ? "-- #{l(:actionview_instancetag_blank_option)} --" : "&nbsp;".html_safe
tags << content_tag('option', blank_text, :value => '')
tags << options_for_select(import.columns_options, import.mapping[field])
tags
end
def mapping_select_tag(import, field, options={})
name = "import_settings[mapping][#{field}]"
select_tag name, options_for_mapping_select(import, field, options)
end
end
# Redmine - project management software
# Copyright (C) 2006-2015 Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
require 'csv'
class Import < ActiveRecord::Base
has_many :items, :class_name => 'ImportItem', :dependent => :delete_all
belongs_to :user
serialize :settings
before_destroy :remove_file
validates_presence_of :filename, :user_id
validates_length_of :filename, :maximum => 255
def initialize(*args)
super
self.settings ||= {}
end
def file=(arg)
return unless arg.present? && arg.size > 0
self.filename = generate_filename
Redmine::Utils.save_upload(arg, filepath)
end
def set_default_settings
separator = lu(user, :general_csv_separator)
if file_exists?
begin
content = File.read(filepath, 256, "rb")
separator = [',', ';'].sort_by {|sep| content.count(sep) }.last
rescue Exception => e
end
end
wrapper = '"'
encoding = lu(user, :general_csv_encoding)
self.settings.merge!(
'separator' => separator,
'wrapper' => wrapper,
'encoding' => encoding
)
end
def to_param
filename
end
# Returns the full path of the file to import
# It is stored in tmp/imports with a random hex as filename
def filepath
if filename.present? && filename =~ /\A[0-9a-f]+\z/
File.join(Rails.root, "tmp", "imports", filename)
else
nil
end
end
# Returns true if the file to import exists
def file_exists?
filepath.present? && File.exists?(filepath)
end
# Returns the headers as an array that
# can be used for select options
def columns_options(default=nil)
i = -1
headers.map {|h| [h, i+=1]}
end
# Parses the file to import and updates the total number of items
def parse_file
count = 0
read_items {|row, i| count=i}
update_attribute :total_items, count
count
end
# Reads the items to import and yields the given block for each item
def read_items
i = 0
headers = true
read_rows do |row|
if i == 0 && headers
headers = false
next
end
i+= 1
yield row, i if block_given?
end
end
# Returns the count first rows of the file (including headers)
def first_rows(count=4)
rows = []
read_rows do |row|
rows << row
break if rows.size >= count
end
rows
end
# Returns an array of headers
def headers
first_rows(1).first || []
end
# Returns the mapping options
def mapping
settings['mapping'] || {}
end
# Imports items and returns the position of the last processed item
def run(options={})
max_items = options[:max_items]
max_time = options[:max_time]
current = 0
imported = 0
resume_after = items.maximum(:position) || 0
interrupted = false
started_on = Time.now
read_items do |row, position|
if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time)
interrupted = true
break
end
if position > resume_after
item = items.build
item.position = position
if object = build_object(row)
if object.save
item.obj_id = object.id
else
item.message = object.errors.full_messages.join("\n")
end
end
item.save!
imported += 1
end
current = position
end
if imported == 0 || interrupted == false
if total_items.nil?
update_attribute :total_items, current
end
update_attribute :finished, true
remove_file
end
current
end
def unsaved_items
items.where(:obj_id => nil)
end
def saved_items
items.where("obj_id IS NOT NULL")
end
private
def read_rows
return unless file_exists?
csv_options = {:headers => false}
csv_options[:encoding] = settings['encoding'].to_s.presence || 'UTF-8'
separator = settings['separator'].to_s
csv_options[:col_sep] = separator if separator.size == 1
wrapper = settings['wrapper'].to_s
csv_options[:quote_char] = wrapper if wrapper.size == 1
CSV.foreach(filepath, csv_options) do |row|
yield row if block_given?
end
end
def row_value(row, key)
if index = mapping[key].presence
row[index.to_i].presence
end
end
# Builds a record for the given row and returns it
# To be implemented by subclasses
def build_object(row)
end
# Generates a filename used to store the import file
def generate_filename
Redmine::Utils.random_hex(16)
end
# Deletes the import file
def remove_file
if file_exists?
begin
File.delete filepath
rescue Exception => e
logger.error "Unable to delete file #{filepath}: #{e.message}" if logger
end
end
end
# Returns true if value is a string that represents a true value
def yes?(value)
value == lu(user, :general_text_yes) || value == '1'
end
end
# Redmine - project management software
# Copyright (C) 2006-2015 Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
class ImportItem < ActiveRecord::Base
belongs_to :import
validates_presence_of :import_id, :position
end
......@@ -914,6 +914,14 @@ class Issue < ActiveRecord::Base
end
end
def notify?
@notify != false
end
def notify=(arg)
@notify = arg
end
# Returns the number of hours spent on this issue
def spent_hours
@spent_hours ||= time_entries.sum(:hours) || 0
......@@ -1625,7 +1633,7 @@ class Issue < ActiveRecord::Base
end
def send_notification
if Setting.notified_events.include?('issue_added')
if notify? && Setting.notified_events.include?('issue_added')
Mailer.deliver_issue_add(self)
end
end
......
# Redmine - project management software
# Copyright (C) 2006-2015 Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
class IssueImport < Import
# Returns the objects that were imported
def saved_objects
object_ids = saved_items.pluck(:obj_id)
objects = Issue.where(:id => object_ids).order(:id).preload(:tracker, :priority, :status)
end
# Returns a scope of projects that user is allowed to
# import issue to
def allowed_target_projects
Project.allowed_to(user, :import_issues)
end
def project
project_id = mapping['project_id'].to_i
allowed_target_projects.find_by_id(project_id) || allowed_target_projects.first
end
# Returns a scope of trackers that user is allowed to
# import issue to
def allowed_target_trackers
project.trackers
end
def tracker
tracker_id = mapping['tracker_id'].to_i
allowed_target_trackers.find_by_id(tracker_id) || allowed_target_trackers.first
end
# Returns true if missing categories should be created during the import
def create_categories?
user.allowed_to?(:manage_categories, project) &&
mapping['create_categories'] == '1'
end
# Returns true if missing versions should be created during the import
def create_versions?
user.allowed_to?(:manage_versions, project) &&
mapping['create_versions'] == '1'
end
private
def build_object(row)
issue = Issue.new
issue.author = user
issue.notify = false
attributes = {
'project_id' => mapping['project_id'],
'tracker_id' => mapping['tracker_id'],
'subject' => row_value(row, 'subject'),
'description' => row_value(row, 'description')
}
issue.send :safe_attributes=, attributes, user
attributes = {}
if priority_name = row_value(row, 'priority')
if priority_id = IssuePriority.active.named(priority_name).first.try(:id)
attributes['priority_id'] = priority_id
end
end
if issue.project && category_name = row_value(row, 'category')
if category = issue.project.issue_categories.named(category_name).first
attributes['category_id'] = category.id
elsif create_categories?
category = issue.project.issue_categories.build
category.name = category_name
if category.save
attributes['category_id'] = category.id
end
end
end
if assignee_name = row_value(row, 'assigned_to')
if assignee = issue.assignable_users.detect {|u| u.name.downcase == assignee_name.downcase}
attributes['assigned_to_id'] = assignee.id
end
end
if issue.project && version_name = row_value(row, 'fixed_version')
if version = issue.project.versions.detect {|v| v.name.downcase == version_name.downcase}
attributes['fixed_version_id'] = version.id
elsif create_versions?
version = issue.project.versions.build
version.name = version_name
if version.save
attributes['fixed_version_id'] = version.id
end
end
end
if is_private = row_value(row, 'is_private')
if yes?(is_private)
attributes['is_private'] = '1'
end
end
if parent_issue_id = row_value(row, 'parent_issue_id')
if parent_issue_id =~ /\A(#)?(\d+)\z/
parent_issue_id = $2
if $1
attributes['parent_issue_id'] = parent_issue_id
elsif issue_id = items.where(:position => parent_issue_id).first.try(:obj_id)
attributes['parent_issue_id'] = issue_id
end
else
attributes['parent_issue_id'] = parent_issue_id
end
end
if start_date = row_value(row, 'start_date')
attributes['start_date'] = start_date
end
if due_date = row_value(row, 'due_date')
attributes['due_date'] = due_date
end
if done_ratio = row_value(row, 'done_ratio')
attributes['done_ratio'] = done_ratio
end
attributes['custom_field_values'] = issue.custom_field_values.inject({}) do |h, v|
if value = row_value(row, "cf_#{v.custom_field.id}")
h[v.custom_field.id.to_s] = v.custom_field.value_from_keyword(value, issue)
end
h
end
issue.send :safe_attributes=, attributes, user
issue
end
end
<div class="splitcontent">
<div class="splitcontentleft">
<p>
<label><%= l(:label_project) %></label>
<%= select_tag 'import_settings[mapping][project_id]',
options_for_select(project_tree_options_for_select(@import.allowed_target_projects, :selected => @import.project)),
:id => 'issue_project_id' %>
</p>
<p>
<label><%= l(:label_tracker) %></label>
<%= select_tag 'import_settings[mapping][tracker_id]',
options_for_select(@import.allowed_target_trackers.sorted.map {|t| [t.name, t.id]}, @import.tracker.try(:id)),
:id => 'issue_tracker_id' %>
</p>
<p>
<label><%= l(:field_subject) %></label>
<%= mapping_select_tag @import, 'subject', :required => true %>
</p>
<p>
<label><%= l(:field_description) %></label>
<%= mapping_select_tag @import, 'description' %>
</p>
<p>
<label><%= l(:field_priority) %></label>
<%= mapping_select_tag @import, 'priority' %>
</p>
<p>
<label><%= l(:field_category) %></label>
<%= mapping_select_tag @import, 'category' %>
<% if User.current.allowed_to?(:manage_categories, @import.project) %>
<label class="block">
<%= check_box_tag 'import_settings[mapping][create_categories]', '1', @import.create_categories? %>
<%= l(:label_create_missing_values) %>
</label>
<% end %>
</p>
<p>
<label><%= l(:field_assigned_to) %></label>
<%= mapping_select_tag @import, 'assigned_to' %>
</p>
<p>
<label><%= l(:field_fixed_version) %></label>
<%= mapping_select_tag @import, 'fixed_version' %>
<% if User.current.allowed_to?(:manage_versions, @import.project) %>
<label class="block">
<%= check_box_tag 'import_settings[mapping][create_versions]', '1', @import.create_versions? %>
<%= l(:label_create_missing_values) %>
</label>
<% end %>
</p>
<% @custom_fields.each do |field| %>
<p>
<label><%= field.name %></label>
<%= mapping_select_tag @import, "cf_#{field.id}" %>
</p>
<% end %>
</div>
<div class="splitcontentright">
<p>
<label><%= l(:field_is_private) %></label>
<%= mapping_select_tag @import, 'is_private' %>
</p>
<p>
<label><%= l(:field_parent_issue) %></label>
<%= mapping_select_tag @import, 'parent_issue_id' %>
</p>
<p>
<label><%= l(:field_start_date) %></label>
<%= mapping_select_tag @import, 'start_date' %>
</p>
<p>
<label><%= l(:field_due_date) %></label>
<%= mapping_select_tag @import, 'due_date' %>
</p>
<p>
<label><%= l(:field_done_ratio) %></label>
<%= mapping_select_tag @import, 'done_ratio' %>
</p>
</div>
</div>
<h2><%= l(:label_import_issues) %></h2>
<%= form_tag(import_mapping_path(@import), :id => "import-form") do %>
<fieldset class="box tabular">
<legend><%= l(:label_fields_mapping) %></legend>
<div id="fields-mapping">
<%= render :partial => 'fields_mapping' %>
</div>
</fieldset>
<div class="autoscroll">
<fieldset class="box">
<legend><%= l(:label_file_content_preview) %></legend>
<table class="sample-data">
<% @import.first_rows.each do |row| %>
<tr>
<%= row.map {|c| content_tag 'td', truncate(c.to_s, :length => 50) }.join("").html_safe %>
</tr>
<% end %>
</table>
</fieldset>
</div>
<p>
<%= button_tag("\xc2\xab " + l(:label_previous), :name => 'previous') %>
<%= submit_tag l(:button_import) %>
</p>
<% end %>
<% content_for :sidebar do %>
<%= render :partial => 'issues/sidebar' %>
<% end %>
<%= javascript_tag do %>
$(document).ready(function() {
$('#fields-mapping').on('change', '#issue_project_id, #issue_tracker_id', function(){
$.ajax({
url: '<%= import_mapping_path(@import, :format => 'js') %>',
type: 'post',
data: $('#import-form').serialize()
});
});
$('#import-form').submit(function(){
$('#import-details').show().addClass('ajax-loading');
$('#import-progress').progressbar({value: 0, max: <%= @import.total_items || 0 %>});
});
});
<% end %>
$('#fields-mapping').html('<%= escape_javascript(render :partial => 'fields_mapping') %>');
<h2><%= l(:label_import_issues) %></h2>
<%= form_tag(imports_path, :multipart => true) do %>
<fieldset class="box">
<legend><%= l(:label_select_file_to_import) %> (CSV)</legend>
<p>