20 #define IMPORT_VERSION "4.0"
22 #include "client/client_priv.h"
27 #include <boost/program_options.hpp>
31 #include <drizzled/internal/my_sys.h>
33 #include <drizzled/gettext.h>
34 #include <drizzled/configmake.h>
36 #include "user_detect.h"
38 namespace po= boost::program_options;
42 extern "C" void * worker_thread(
void *arg);
46 const char *program_name=
"drizzleimport";
50 pthread_mutex_t counter_mutex;
51 pthread_cond_t count_threshhold;
53 static void db_error(drizzle_con_st *con, drizzle_result_st *result,
54 drizzle_return_t ret,
char *table);
55 static char *field_escape(
char *to,
const char *from,uint32_t length);
56 static char *add_load_option(
char *ptr,
const char *
object,
57 const char *statement);
59 static bool verbose=
false, ignore_errors=
false,
60 opt_delete=
false, opt_replace=
false, silent=
false,
61 ignore_unique=
false, opt_low_priority=
false,
62 use_drizzle_protocol=
false, opt_local_file;
64 static uint32_t opt_use_threads;
65 static uint32_t opt_drizzle_port= 0;
66 static int64_t opt_ignore_lines= -1;
68 std::string opt_columns,
82 static int get_options(
void)
85 if (! enclosed.empty() && ! opt_enclosed.empty())
87 fprintf(stderr,
"You can't use ..enclosed.. and ..optionally-enclosed.. at the same time.\n");
90 if (opt_replace && ignore_unique)
92 fprintf(stderr,
"You can't use --ignore_unique (-i) and --replace (-r) at the same time.\n");
97 opt_password=client_get_tty_password(NULL);
103 static int write_to_table(
char *filename, drizzle_con_st *con)
105 char tablename[FN_REFLEN], hard_path[FN_REFLEN],
106 sql_statement[FN_REFLEN*16+256], *end;
107 drizzle_result_st result;
108 drizzle_return_t ret;
110 internal::fn_format(tablename, filename,
"",
"", 1 | 2);
111 if (not opt_local_file)
112 strcpy(hard_path,filename);
114 internal::my_load_path(hard_path, filename, NULL);
119 fprintf(stdout,
"Deleting the old data from table %s\n", tablename);
121 snprintf(sql_statement,
sizeof(sql_statement),
"DELETE FROM %s", tablename);
123 snprintf(sql_statement,
sizeof(sql_statement),
"DELETE FROM %s", tablename);
125 if (drizzle_query_str(con, &result, sql_statement, &ret) == NULL ||
126 ret != DRIZZLE_RETURN_OK)
128 db_error(con, &result, ret, tablename);
131 drizzle_result_free(&result);
136 fprintf(stdout,
"Loading data from LOCAL file: %s into %s\n",
137 hard_path, tablename);
139 fprintf(stdout,
"Loading data from SERVER file: %s into %s\n",
140 hard_path, tablename);
142 snprintf(sql_statement,
sizeof(sql_statement),
"LOAD DATA %s %s INFILE '%s'",
143 opt_low_priority ?
"LOW_PRIORITY" :
"",
144 opt_local_file ?
"LOCAL" :
"", hard_path);
145 end= strchr(sql_statement,
'\0');
147 end= strcpy(end,
" REPLACE")+8;
149 end= strcpy(end,
" IGNORE")+7;
151 end+= sprintf(end,
" INTO TABLE %s", tablename);
153 if (! fields_terminated.empty() || ! enclosed.empty() || ! opt_enclosed.empty() || ! escaped.empty())
154 end= strcpy(end,
" FIELDS")+7;
155 end= add_load_option(end, (
char *)fields_terminated.c_str(),
" TERMINATED BY");
156 end= add_load_option(end, (
char *)enclosed.c_str(),
" ENCLOSED BY");
157 end= add_load_option(end, (
char *)opt_enclosed.c_str(),
158 " OPTIONALLY ENCLOSED BY");
159 end= add_load_option(end, (
char *)escaped.c_str(),
" ESCAPED BY");
160 end= add_load_option(end, (
char *)lines_terminated.c_str(),
" LINES TERMINATED BY");
161 if (opt_ignore_lines >= 0)
163 end= strcpy(end,
" IGNORE ")+8;
164 ostringstream buffer;
165 buffer << opt_ignore_lines;
166 end= strcpy(end, buffer.str().c_str())+ buffer.str().size();
167 end= strcpy(end,
" LINES")+6;
169 if (! opt_columns.empty())
171 end= strcpy(end,
" (")+2;
172 end= strcpy(end, (
char *)opt_columns.c_str()+opt_columns.length());
173 end= strcpy(end,
")")+1;
177 if (drizzle_query_str(con, &result, sql_statement, &ret) == NULL ||
178 ret != DRIZZLE_RETURN_OK)
180 db_error(con, &result, ret, tablename);
185 if (strcmp(drizzle_result_info(&result),
""))
187 fprintf(stdout,
"%s.%s: %s\n", current_db.c_str(), tablename,
188 drizzle_result_info(&result));
191 drizzle_result_free(&result);
196 static drizzle_con_st *db_connect(
const string host,
const string database,
197 const string user,
const string passwd)
201 drizzle_return_t ret;
205 fprintf(stdout,
"Connecting to %s, using protocol %s...\n", ! host.empty() ? host.c_str() :
"localhost", opt_protocol.c_str());
208 if ((drizzle= drizzle_create()) == NULL)
213 if (!(con= drizzle_con_add_tcp(drizzle,
214 host.c_str(), opt_drizzle_port,
215 user.c_str(), passwd.c_str(),
217 use_drizzle_protocol ? DRIZZLE_CON_EXPERIMENTAL : DRIZZLE_CON_MYSQL)))
222 if ((ret= drizzle_con_connect(con)) != DRIZZLE_RETURN_OK)
225 db_error(con, NULL, ret, NULL);
229 fprintf(stdout,
"Selecting database %s\n", database.c_str());
236 static void db_disconnect(
const string host, drizzle_con_st *con)
239 fprintf(stdout,
"Disconnecting from %s\n", ! host.empty() ? host.c_str() :
"localhost");
240 drizzle_free(drizzle_con_drizzle(con));
245 static void safe_exit(
int error, drizzle_con_st *con)
250 drizzle_free(drizzle_con_drizzle(con));
256 static void db_error(drizzle_con_st *con, drizzle_result_st *result,
257 drizzle_return_t ret,
char *table)
259 if (ret == DRIZZLE_RETURN_ERROR_CODE)
261 fprintf(stdout,
"Error: %d, %s%s%s",
262 drizzle_result_error_code(result),
263 drizzle_result_error(result),
264 table ?
", when using table: " :
"", table ? table :
"");
265 drizzle_result_free(result);
269 fprintf(stdout,
"Error: %d, %s%s%s", ret, drizzle_con_error(con),
270 table ?
", when using table: " :
"", table ? table :
"");
277 static char *add_load_option(
char *ptr,
const char *
object,
278 const char *statement)
283 if (
object[0] ==
'0' && (
object[1] ==
'x' ||
object[1] ==
'X'))
284 ptr+= sprintf(ptr,
" %s %s", statement,
object);
288 ptr+= sprintf(ptr,
" %s '", statement);
289 ptr= field_escape(ptr,
object,(uint32_t) strlen(
object));
303 static char *field_escape(
char *to,
const char *from,uint32_t length)
306 uint32_t end_backslashes=0;
308 for (end= from+length; from != end; from++)
315 if (*from ==
'\'' && !end_backslashes)
326 void * worker_thread(
void *arg)
329 char *raw_table_name= (
char *)arg;
332 if (!(con= db_connect(current_host,current_db,current_user,opt_password)))
340 if ((error= write_to_table(raw_table_name, con)))
350 db_disconnect(current_host, con);
353 pthread_mutex_lock(&counter_mutex);
355 pthread_cond_signal(&count_threshhold);
356 pthread_mutex_unlock(&counter_mutex);
362 int main(
int argc,
char **argv)
366 po::options_description commandline_options(
"Options used only in command line");
367 commandline_options.add_options()
369 (
"debug,#", po::value<string>(),
370 "Output debug log. Often this is 'd:t:o,filename'.")
371 (
"delete,d", po::value<bool>(&opt_delete)->default_value(
false)->zero_tokens(),
372 "First delete all rows from table.")
373 (
"help,?",
"Displays this help and exits.")
374 (
"ignore,i", po::value<bool>(&ignore_unique)->default_value(
false)->zero_tokens(),
375 "If duplicate unique key was found, keep old row.")
376 (
"low-priority", po::value<bool>(&opt_low_priority)->default_value(
false)->zero_tokens(),
377 "Use LOW_PRIORITY when updating the table.")
378 (
"replace,r", po::value<bool>(&opt_replace)->default_value(
false)->zero_tokens(),
379 "If duplicate unique key was found, replace old row.")
380 (
"verbose,v", po::value<bool>(&verbose)->default_value(
false)->zero_tokens(),
381 "Print info about the various stages.")
382 (
"version,V",
"Output version information and exit.")
385 po::options_description import_options(
"Options specific to the drizzleimport");
386 import_options.add_options()
387 (
"columns,C", po::value<string>(&opt_columns)->default_value(
""),
388 "Use only these columns to import the data to. Give the column names in a comma separated list. This is same as giving columns to LOAD DATA INFILE.")
389 (
"fields-terminated-by", po::value<string>(&fields_terminated)->default_value(
""),
390 "Fields in the textfile are terminated by ...")
391 (
"fields-enclosed-by", po::value<string>(&enclosed)->default_value(
""),
392 "Fields in the importfile are enclosed by ...")
393 (
"fields-optionally-enclosed-by", po::value<string>(&opt_enclosed)->default_value(
""),
394 "Fields in the i.file are opt. enclosed by ...")
395 (
"fields-escaped-by", po::value<string>(&escaped)->default_value(
""),
396 "Fields in the i.file are escaped by ...")
397 (
"force,f", po::value<bool>(&ignore_errors)->default_value(
false)->zero_tokens(),
398 "Continue even if we get an sql-error.")
399 (
"ignore-lines", po::value<int64_t>(&opt_ignore_lines)->default_value(0),
400 "Ignore first n lines of data infile.")
401 (
"lines-terminated-by", po::value<string>(&lines_terminated)->default_value(
""),
402 "Lines in the i.file are terminated by ...")
403 (
"local,L", po::value<bool>(&opt_local_file)->default_value(
false)->zero_tokens(),
404 "Read all files through the client.")
405 (
"silent,s", po::value<bool>(&silent)->default_value(
false)->zero_tokens(),
407 (
"use-threads", po::value<uint32_t>(&opt_use_threads)->default_value(4),
408 "Load files in parallel. The argument is the number of threads to use for loading data (default is 4.")
411 po::options_description client_options(
"Options specific to the client");
412 client_options.add_options()
413 (
"host,h", po::value<string>(¤t_host)->default_value(
"localhost"),
415 (
"password,P", po::value<string>(&password),
416 "Password to use when connecting to server. If password is not given it's asked from the tty." )
417 (
"port,p", po::value<uint32_t>(&opt_drizzle_port)->default_value(0),
418 "Port number to use for connection")
419 (
"protocol", po::value<string>(&opt_protocol)->default_value(
"mysql"),
420 "The protocol of connection (mysql or drizzle).")
421 (
"user,u", po::value<string>(¤t_user)->default_value(
UserDetect().getUser()),
422 "User for login if not current user.")
425 po::options_description long_options(
"Allowed Options");
426 long_options.add(commandline_options).add(import_options).add(client_options);
428 std::string system_config_dir_import(SYSCONFDIR);
429 system_config_dir_import.append(
"/drizzle/drizzleimport.cnf");
431 std::string system_config_dir_client(SYSCONFDIR);
432 system_config_dir_client.append(
"/drizzle/client.cnf");
434 std::string user_config_dir((getenv(
"XDG_CONFIG_HOME")? getenv(
"XDG_CONFIG_HOME"):
"~/.config"));
436 if (user_config_dir.compare(0, 2,
"~/") == 0)
439 homedir= getenv(
"HOME");
441 user_config_dir.replace(0, 1, homedir);
444 po::variables_map vm;
447 int style = po::command_line_style::default_style & ~po::command_line_style::allow_guessing;
449 po::store(po::command_line_parser(argc, argv).options(long_options).
450 style(style).extra_parser(parse_password_arg).run(), vm);
452 std::string user_config_dir_import(user_config_dir);
453 user_config_dir_import.append(
"/drizzle/drizzleimport.cnf");
455 std::string user_config_dir_client(user_config_dir);
456 user_config_dir_client.append(
"/drizzle/client.cnf");
458 ifstream user_import_ifs(user_config_dir_import.c_str());
459 po::store(parse_config_file(user_import_ifs, import_options), vm);
461 ifstream user_client_ifs(user_config_dir_client.c_str());
462 po::store(parse_config_file(user_client_ifs, client_options), vm);
464 ifstream system_import_ifs(system_config_dir_import.c_str());
465 store(parse_config_file(system_import_ifs, import_options), vm);
467 ifstream system_client_ifs(system_config_dir_client.c_str());
468 po::store(parse_config_file(system_client_ifs, client_options), vm);
471 if (vm.count(
"protocol"))
473 boost::to_lower(opt_protocol);
474 if (not opt_protocol.compare(
"mysql"))
475 use_drizzle_protocol=
false;
476 else if (not opt_protocol.compare(
"drizzle"))
477 use_drizzle_protocol=
true;
480 cout << _(
"Error: Unknown protocol") <<
" '" << opt_protocol <<
"'" << endl;
485 if (vm.count(
"port"))
491 if (opt_drizzle_port > 65535)
493 fprintf(stderr, _(
"Value supplied for port is not valid.\n"));
494 exit(EXIT_ARGUMENT_INVALID);
498 if( vm.count(
"password") )
500 if (!opt_password.empty())
501 opt_password.erase();
502 if (password == PASSWORD_SENTINEL)
508 opt_password= password;
518 if (vm.count(
"version"))
520 printf(
"%s Ver %s Distrib %s, for %s-%s (%s)\n", program_name,
521 IMPORT_VERSION, drizzle_version(),HOST_VENDOR,HOST_OS,HOST_CPU);
524 if (vm.count(
"help") || argc < 2)
526 printf(
"%s Ver %s Distrib %s, for %s-%s (%s)\n", program_name,
527 IMPORT_VERSION, drizzle_version(),HOST_VENDOR,HOST_OS,HOST_CPU);
528 puts(
"This software comes with ABSOLUTELY NO WARRANTY. This is free software,\nand you are welcome to modify and redistribute it under the GPL license\n");
530 Loads tables from text files in various formats. The base name of the\n\
531 text file must be the name of the table that should be used.\n\
532 If one uses sockets to connect to the Drizzle server, the server will open and\n\
533 read the text file directly. In other cases the client will open the text\n\
534 file. The SQL command 'LOAD DATA INFILE' is used to import the rows.\n");
536 printf(
"\nUsage: %s [OPTIONS] database textfile...", program_name);
547 current_db= (*argv)++;
552 pthread_t mainthread;
554 pthread_attr_init(&attr);
555 pthread_attr_setdetachstate(&attr,
556 PTHREAD_CREATE_DETACHED);
558 pthread_mutex_init(&counter_mutex, NULL);
559 pthread_cond_init(&count_threshhold, NULL);
561 for (counter= 0; *argv != NULL; argv++)
563 pthread_mutex_lock(&counter_mutex);
564 while (counter == opt_use_threads)
566 struct timespec abstime;
568 set_timespec(abstime, 3);
569 pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime);
573 pthread_mutex_unlock(&counter_mutex);
575 if (pthread_create(&mainthread, &attr, worker_thread,
578 pthread_mutex_lock(&counter_mutex);
580 pthread_mutex_unlock(&counter_mutex);
581 fprintf(stderr,
"%s: Could not create thread\n", program_name);
588 pthread_mutex_lock(&counter_mutex);
591 struct timespec abstime;
593 set_timespec(abstime, 3);
594 pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime);
596 pthread_mutex_unlock(&counter_mutex);
597 pthread_mutex_destroy(&counter_mutex);
598 pthread_cond_destroy(&count_threshhold);
599 pthread_attr_destroy(&attr);
605 if (!(con= db_connect(current_host,current_db,current_user,opt_password)))
610 for (; *argv != NULL; argv++)
611 if (
int error= write_to_table(*argv, con))
614 db_disconnect(current_host, con);
616 opt_password.empty();
618 catch(exception &err)
620 cerr<<err.what()<<endl;
TODO: Rename this file - func.h is stupid.