清华大佬耗费三个月吐血整理的几百G的资源，免费分享！....>>>

#!/bin/bash
#Author: 		Castle Liu
#Date:			2014/05/22
#Desc:			load data to infobright
#Usage:			loadDataToInfobright.sh -s or loadDataToInfobright.sh -d {datadir} or loadDataToInfobright.sh -f {datafile}

#ERROR CODE INDEX
#001	setting_not_correct_error		shell passwd arg not correct,you should try -h option.
#002	file_not_exists_error		the file you pass in not found,you should check.
#003	dir_not_exists_error		the dir you pass in not found,you should check.

USER=
PASSWD=
DATABASE=
_DEFAULT_D='/data/files'
_DEFAULT_BACKUP_D='/data/backup'
TODAY=`date +"%Y%m%d"`

####functions###
#help
#function used to print help info.
help() {
	cat <<EOF
	-d 	optional,data directory,file names must be start with table name and end with {.[csv|txt]},and in absolute path.
	-f 	optional,data file to be loaded,file name start with table name and end with {.[csv|txt]}.
	-s 	optional,use default setting.
	-b 	optional,backup and delete data files.
	-h 	print help info.
EOF
}

#error_exit
#exit function for the script
error_exit(){
	func_name=$1
	exit_code=$2
	error_code=$3
	error_detail=$4
	END_TIME=`date +%Y%m%d%H%M%S`
	echo "${func_name}::error_code::${error_code}"
	echo "${func_name}::error_detail::${error_detail}" 
	echo "script::Done in ${END_TIME}"
	exit $2
} 

#parse_args
#parse paremeters.
#do pass in the args from shell,for example:parse_args $*
parse_args(){
	while getopts "hd:f:sb" arg
	do 
		case ${arg} in
			h)
				help
				exit 0
				;;
			d)
				export _DATA_D=${OPTARG}
				;;
			f)
				export _DATA_F=${OPTARG}
				;;
			s)
				export _DAFAULT_S=true
				;;
			b)
				export _BACKUP=true
				;;
			?)
				error_exit 1 parse_args 001 'setting_not_correct_error' 
				;;
		esac
	done
}
#parse_args $*

#format_data
#format date file,excute befor load_data,turn the format in to something valid.
format_data() {
	_data_f=$1
	sed -i "s/'//g" ${_data_f} #replace "'"" with null,purpose:conflick with insert sql
}

#load_data
#load data file into a table,file name start with table name and end with {.[csv|txt]}, and in absolute path.
load_data() {
	_data_f=$1
	table_name=`basename ${_data_f} |awk -F [\.] '{print $1}'`
	type_file=`basename ${_data_f} |awk -F [\.] '{print $2}'`
	if [[ ! -z ${_data_f} ]] && [[ -f ${_data_f} ]];then
		if [[ ${type_file} = "csv" ]] || [[ ${type_file} = "txt" ]];then
	    	#generate and excute sql
	    	echo "Loading data from file ${_data_f}."
	    	sql="LOAD DATA INFILE '${_data_f}' INTO TABLE ${DATABASE}.${table_name} FIELDS TERMINATED BY ',';"
	    	echo "sql:  ${sql}"
	    	/usr/bin/mysql-ib -u${USER} -p${PASSWD} -e "${sql}" 
	    	#echo ${sql}
	    fi
	else
		error_exit 1 load_data 002 'file_not_exists_error'
	fi
}

#load_data_d
#load data dir into a list of table,in absolute path.
load_data_d() {
	_data_d=$1
	if [[ ! -z ${_data_d} ]] && [[ -d ${_data_d} ]];then
    ls ${_data_d}|while read lines
    do
        if [[ ${lines} != EOF ]];then
            file_path=${_data_d}/${lines}
            format_data ${file_path}
            load_data ${file_path}
        fi
    done
	else
		error_exit 1 load_data_d 003 'dir_not_exists_error'
	fi
}

#backup_data
#backup data files to default directory
backup_data(){
	backup_file=$1
	if [[ -d ${backup_file} ]];then
		echo "Backuping up directory ${backup_file}."
		tar -czvf ${_DEFAULT_BACKUP_D}/${START_TIME}.tar.gz  ${backup_file}
	elif [[ -f ${backup_file} ]];then
		echo "Backuping up file ${backup_file}."
		file_name=`basename ${backup_file}`
		tar -czvf ${_DEFAULT_BACKUP_D}/${START_TIME}_${file_name}.tar.gz  ${backup_file}
	else
		error_exit 1 backup_data 002 'file_not_exists_error'
	fi 
	rm -rf ${backup_file}
}

###main###
echo ''
START_TIME=`date +%Y%m%d%H%M%S`
echo "Start in ${START_TIME}."
parse_args $*
if [[ ${_DAFAULT_S} ]];then
	echo "Loading data from default subdirectory, ${_DEFAULT_D}... "
	_today_tmp_d=${_DEFAULT_D}/${TODAY}
	load_data_d ${_today_tmp_d}
	if [[ ${_BACKUP} ]];then
		backup_data ${_today_tmp_d}
	fi
elif [[ ! -z ${_DATA_D} ]] && [[ -d ${_DATA_D} ]];then
	echo "Loading data from ${_DATA_D}..."
	load_data_d ${_DATA_D}
	if [[ ${_BACKUP} ]];then
		backup_data ${_DATA_D}
	fi
elif [[ ! -z ${_DATA_F} ]] && [[ -f ${_DATA_F} ]];then
	load_data ${_DATA_F}
	if [[ ${_BACKUP} ]];then
		backup_data ${_DATA_F}
	fi
else
	error_exit 1 main 001 'setting_not_correct_error'
fi

END_TIME=`date +%Y%m%d%H%M%S`
echo "Done in ${END_TIME}"
echo ''