43 #include "myisam_priv.h"
44 #include <drizzled/internal/m_string.h>
46 #ifdef HAVE_SYS_VADVISE_H
47 #include <sys/vadvise.h>
50 #include <sys/types.h>
52 #ifdef HAVE_SYS_MMAN_H
55 #include <drizzled/util/test.h>
56 #include <drizzled/error.h>
65 #define my_off_t2double(A) ((double) (my_off_t) (A))
71 my_off_t page,
unsigned char *buff, ha_rows *keys,
72 ha_checksum *key_checksum, uint32_t level);
74 static ha_checksum calc_checksum(ha_rows count);
77 my_off_t pagepos,
int new_file);
80 int sort_key_cmp(
MI_SORT_PARAM *sort_param,
const void *a,
const void *b);
86 unsigned char *key, my_off_t prev_block);
91 uint32_t buffer_length);
92 static ha_checksum mi_byte_checksum(
const unsigned char *buf, uint32_t length);
97 memset(param, 0,
sizeof(*param));
98 param->opt_follow_links=1;
99 param->keys_in_use= ~(uint64_t) 0;
100 param->search_after_block=HA_OFFSET_ERROR;
101 param->auto_increment_value= 0;
102 param->use_buffers=USE_BUFFER_INIT;
103 param->read_buffer_length=READ_BUFFER_INIT;
104 param->write_buffer_length=READ_BUFFER_INIT;
105 param->sort_buffer_length=SORT_BUFFER_INIT;
106 param->sort_key_blocks=BUFFERS_WHEN_SORTING;
107 param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
108 param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
109 param->start_check_pos=0;
110 param->max_record_length= INT64_MAX;
111 param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
112 param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
121 if (mi_is_crashed_on_repair(info))
122 mi_check_print_warning(param,
123 "Table is marked as crashed and last repair failed");
124 else if (mi_is_crashed(info))
125 mi_check_print_warning(param,
126 "Table is marked as crashed");
127 if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
130 uint32_t save=param->warning_printed;
131 mi_check_print_warning(param,
132 share->state.open_count==1 ?
133 "%d client is using or hasn't closed the table properly" :
134 "%d clients are using or haven't closed the table properly",
135 share->state.open_count);
137 if (param->testflag & T_UPDATE_STATE)
138 param->warning_printed=save;
145 int chk_del(
MI_CHECK *param,
register MI_INFO *info, uint32_t test_flag)
147 uint32_t delete_link_length;
148 my_off_t empty, next_link, old_link= 0;
149 char buff[22],buff2[22];
151 param->record_checksum=0;
152 delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 : info->s->rec_reflength+1);
154 if (!(test_flag & T_SILENT))
155 puts(
"- check record delete-chain");
157 next_link=info->s->state.dellink;
158 if (info->state->del == 0)
160 if (test_flag & T_VERBOSE)
162 puts(
"No recordlinks");
167 if (test_flag & T_VERBOSE)
168 printf(
"Recordlinks: ");
170 ha_rows i= info->state->del;
171 for (; i > 0 && next_link != HA_OFFSET_ERROR; i--)
173 if (*killed_ptr(param))
175 if (test_flag & T_VERBOSE)
176 printf(
" %9s",llstr(next_link,buff));
177 if (next_link >= info->state->data_file_length)
179 if (my_pread(info->dfile, (
unsigned char*) buff,delete_link_length, next_link,MYF(MY_NABP)))
181 if (test_flag & T_VERBOSE) puts(
"");
182 mi_check_print_error(param,
"Can't read delete-link at filepos: %s",
183 llstr(next_link,buff));
188 if (test_flag & T_VERBOSE) puts(
"");
189 mi_check_print_error(param,
"Record at pos: %s is not remove-marked",
190 llstr(next_link,buff));
193 if (info->s->options & HA_OPTION_PACK_RECORD)
195 my_off_t prev_link=mi_sizekorr(buff+12);
196 if (empty && prev_link != old_link)
198 if (test_flag & T_VERBOSE) puts(
"");
199 mi_check_print_error(param,
"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
203 next_link=mi_sizekorr(buff+4);
204 empty+=mi_uint3korr(buff+1);
208 param->record_checksum+=(ha_checksum) next_link;
209 next_link=_mi_rec_pos(info->s,(
unsigned char*) buff+1);
210 empty+=info->s->base.pack_reclength;
213 if (test_flag & T_VERBOSE)
215 if (empty != info->state->empty)
217 mi_check_print_warning(param,
218 "Found %s deleted space in delete link chain. Should be %s",
220 llstr(info->state->empty,buff));
222 if (next_link != HA_OFFSET_ERROR)
224 mi_check_print_error(param,
225 "Found more than the expected %s deleted rows in delete link chain",
226 llstr(info->state->del, buff));
231 mi_check_print_error(param,
"Found %s deleted rows in delete link chain. Should be %s", llstr(info->state->del - i, buff2), llstr(info->state->del, buff));
238 param->testflag|=T_RETRY_WITHOUT_QUICK;
239 if (test_flag & T_VERBOSE) puts(
"");
240 mi_check_print_error(param,
"record delete-link-chain corrupted");
247 static int check_k_link(
MI_CHECK *param,
register MI_INFO *info, uint32_t nr)
250 uint32_t block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
252 char llbuff[21], llbuff2[21];
255 if (param->testflag & T_VERBOSE)
256 printf(
"block_size %4u:", block_size);
258 next_link=info->s->state.key_del[nr];
259 records= (ha_rows) (info->state->key_file_length / block_size);
260 while (next_link != HA_OFFSET_ERROR && records > 0)
262 if (*killed_ptr(param))
264 if (param->testflag & T_VERBOSE)
265 printf(
"%16s",llstr(next_link,llbuff));
268 if (next_link + block_size > info->state->key_file_length)
270 mi_check_print_error(param,
"Invalid key block position: %s "
271 "key block size: %u file_length: %s",
272 llstr(next_link, llbuff), block_size,
273 llstr(info->state->key_file_length, llbuff2));
278 if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
280 mi_check_print_error(param,
"Mis-aligned key block: %s "
281 "minimum key block length: %u",
282 llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
292 if (not pread(info->s->kfile, info->buff, MI_MIN_KEY_BLOCK_LENGTH, next_link))
294 mi_check_print_error(param,
"key cache read error for block: %s", llstr(next_link,llbuff));
298 next_link=mi_sizekorr(buff);
300 param->key_file_blocks+=block_size;
302 if (param->testflag & T_VERBOSE)
304 if (next_link != HA_OFFSET_ERROR)
305 printf(
"%16s\n",llstr(next_link,llbuff));
309 return (next_link != HA_OFFSET_ERROR);
318 register my_off_t skr,size;
319 char buff[22],buff2[22];
321 if (!(param->testflag & T_SILENT)) puts(
"- check file-size");
323 size= lseek(info->s->kfile, 0, SEEK_END);
324 if ((skr=(my_off_t) info->state->key_file_length) != size)
327 if (skr > size && mi_is_any_key_active(info->s->state.key_map))
330 mi_check_print_error(param,
331 "Size of indexfile is: %-8s Should be: %s",
332 llstr(size,buff), llstr(skr,buff2));
335 mi_check_print_warning(param,
336 "Size of indexfile is: %-8s Should be: %s",
337 llstr(size,buff), llstr(skr,buff2));
339 if (!(param->testflag & T_VERY_SILENT) &&
340 ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
341 uint64_t2double(info->state->key_file_length) >
342 uint64_t2double(info->s->base.margin_key_file_length)*0.9)
343 mi_check_print_warning(param,
"Keyfile is almost full, %10s of %10s used",
344 llstr(info->state->key_file_length,buff),
345 llstr(info->s->base.max_key_file_length-1,buff));
347 size=lseek(info->dfile,0L,SEEK_END);
348 skr=(my_off_t) info->state->data_file_length;
349 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
350 skr+= MEMMAP_EXTRA_MARGIN;
352 if (info->data_file_type == STATIC_RECORD &&
353 skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
354 skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
358 info->state->data_file_length=size;
359 if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
362 mi_check_print_error(param,
"Size of datafile is: %-9s Should be: %s",
363 llstr(size,buff), llstr(skr,buff2));
364 param->testflag|=T_RETRY_WITHOUT_QUICK;
368 mi_check_print_warning(param,
369 "Size of datafile is: %-9s Should be: %s",
370 llstr(size,buff), llstr(skr,buff2));
373 if (!(param->testflag & T_VERY_SILENT) &&
374 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
375 uint64_t2double(info->state->data_file_length) >
376 (uint64_t2double(info->s->base.max_data_file_length)*0.9))
377 mi_check_print_warning(param,
"Datafile is almost full, %10s of %10s used",
378 llstr(info->state->data_file_length,buff),
379 llstr(info->s->base.max_data_file_length-1,buff2));
388 uint32_t key,found_keys=0,full_text_keys=0,result=0;
390 ha_checksum old_record_checksum,init_checksum;
391 my_off_t all_keydata,all_totaldata,key_totlength,length;
392 ulong *rec_per_key_part;
395 char buff[22],buff2[22];
397 if (!(param->testflag & T_SILENT))
398 puts(
"- check key delete-chain");
400 param->key_file_blocks=info->s->base.keystart;
401 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
402 if (check_k_link(param,info,key))
404 if (param->testflag & T_VERBOSE) puts(
"");
405 mi_check_print_error(param,
"key delete-link-chain corrupted");
409 if (!(param->testflag & T_SILENT)) puts(
"- check index reference");
411 all_keydata=all_totaldata=key_totlength=0;
412 old_record_checksum=0;
413 init_checksum=param->record_checksum;
414 if (!(share->options &
415 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
416 old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
417 share->base.pack_reclength;
418 rec_per_key_part= param->rec_per_key_part;
419 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
420 rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
422 param->key_crc[key]=0;
423 if (! mi_is_key_active(share->state.key_map, key))
426 assert(rec_per_key_part >= param->rec_per_key_part);
427 memcpy(rec_per_key_part,
428 (share->state.rec_per_key_part +
429 (rec_per_key_part - param->rec_per_key_part)),
430 keyinfo->keysegs*
sizeof(*rec_per_key_part));
435 param->record_checksum=init_checksum;
437 memset(¶m->unique_count, 0,
sizeof(param->unique_count));
438 memset(¶m->notnull_count, 0,
sizeof(param->notnull_count));
440 if ((!(param->testflag & T_SILENT)))
441 printf (
"- check data record references index: %d\n",key+1);
442 if (share->state.key_root[key] == HA_OFFSET_ERROR && (info->state->records == 0))
444 if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
445 DFLT_INIT_HITS,info->buff,0))
447 mi_check_print_error(param,
"Can't read indexpage from filepos: %s",
448 llstr(share->state.key_root[key],buff));
449 if (!(param->testflag & T_INFO))
454 param->key_file_blocks+=keyinfo->block_length;
456 param->keydata=param->totaldata=0;
459 if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
460 &keys, param->key_crc+key,1))
464 if (keys != info->state->records)
466 mi_check_print_error(param,
"Found %s keys of %s",llstr(keys,buff),
467 llstr(info->state->records,buff2));
468 if (!(param->testflag & T_INFO))
473 if (found_keys - full_text_keys == 1 &&
475 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
476 (param->testflag & T_DONT_CHECK_CHECKSUM)))
477 old_record_checksum=param->record_checksum;
478 else if (old_record_checksum != param->record_checksum)
481 mi_check_print_error(param,
"Key %u doesn't point at same records that key 1",
484 mi_check_print_error(param,
"Key 1 doesn't point at all records");
485 if (!(param->testflag & T_INFO))
491 if ((uint) share->base.auto_key -1 == key)
494 uint64_t auto_increment;
496 _mi_read_key_record(info, 0L, info->rec_buff);
497 auto_increment= retrieve_auto_increment(info, info->rec_buff);
498 if (auto_increment > info->s->state.auto_increment)
500 mi_check_print_warning(param,
"Auto-increment value: %s is smaller "
501 "than max used value: %s",
502 llstr(info->s->state.auto_increment,buff2),
503 llstr(auto_increment, buff));
505 if (param->testflag & T_AUTO_INC)
507 set_if_bigger(info->s->state.auto_increment,
509 set_if_bigger(info->s->state.auto_increment,
510 param->auto_increment_value);
514 mi_extra(info,HA_EXTRA_KEYREAD,0);
515 memset(info->lastkey, 0, keyinfo->seg->length);
516 if (!mi_rkey(info, info->rec_buff, key, (
const unsigned char*) info->lastkey,
517 (key_part_map)1, HA_READ_KEY_EXACT))
520 uint32_t save=param->warning_printed;
521 mi_check_print_warning(param,
"Found row where the auto_increment "
522 "column has the value 0");
523 param->warning_printed=save;
525 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
528 length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
529 if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
530 printf(
"Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
532 (
int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
533 (
int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
534 my_off_t2double(length)),
536 all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
539 if (param->testflag & T_STATISTICS)
540 update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
541 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
542 param->notnull_count: NULL,
543 (uint64_t)info->state->records);
545 if (param->testflag & T_INFO)
547 if (all_totaldata != 0L && found_keys > 0)
548 printf(
"Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
549 (
int) (my_off_t2double(all_keydata)*100.0/
550 my_off_t2double(all_totaldata)),
551 (
int) ((my_off_t2double(key_totlength) -
552 my_off_t2double(all_keydata))*100.0/
553 my_off_t2double(key_totlength)));
554 else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
557 if (param->key_file_blocks != info->state->key_file_length &&
558 param->keys_in_use != ~(uint64_t) 0)
559 mi_check_print_warning(param,
"Some data are unreferenced in keyfile");
560 if (found_keys != full_text_keys)
561 param->record_checksum=old_record_checksum-init_checksum;
563 param->record_checksum=0;
569 my_off_t page,
unsigned char *buff, ha_rows *keys,
570 ha_checksum *key_checksum, uint32_t level)
572 char llbuff[22],llbuff2[22];
575 if (page + keyinfo->block_length > info->state->key_file_length)
578 my_off_t max_length= lseek(info->s->kfile, 0, SEEK_END);
579 mi_check_print_error(param,
"Invalid key block position: %s "
580 "key block size: %u file_length: %s",
581 llstr(page, llbuff), keyinfo->block_length,
582 llstr(info->state->key_file_length, llbuff2));
583 if (page + keyinfo->block_length > max_length)
586 info->state->key_file_length= (max_length &
587 ~ (my_off_t) (keyinfo->block_length - 1));
591 if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
593 mi_check_print_error(param,
"Mis-aligned key block: %s "
594 "minimum key block length: %u",
595 llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
599 if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
601 mi_check_print_error(param,
"Can't read key from filepos: %s",
605 param->key_file_blocks+=keyinfo->block_length;
606 if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
632 void mi_collect_stats_nonulls_first(
HA_KEYSEG *keyseg, uint64_t *notnull,
635 uint32_t first_null, kp;
636 first_null= ha_find_null(keyseg, key) - keyseg;
641 for (kp= 0; kp < first_null; kp++)
672 int mi_collect_stats_nonulls_next(
HA_KEYSEG *keyseg, uint64_t *notnull,
673 unsigned char *prev_key,
unsigned char *last_key)
676 uint32_t first_null_seg, kp;
687 ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
688 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
689 seg= keyseg + diffs[0] - 1;
692 first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
693 for (kp= 0; kp < first_null_seg; kp++)
708 my_off_t page,
unsigned char *buff, ha_rows *keys,
709 ha_checksum *key_checksum, uint32_t level)
712 uint32_t used_length,comp_flag,nod_flag,key_length=0;
713 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*endpos;
714 my_off_t next_page,record;
716 uint32_t diff_pos[2];
718 if (!(temp_buff=(
unsigned char*) malloc(keyinfo->block_length)))
720 mi_check_print_error(param,
"Not enough memory for keyblock");
724 if (keyinfo->flag & HA_NOSAME)
725 comp_flag=SEARCH_FIND | SEARCH_UPDATE;
727 comp_flag=SEARCH_SAME;
728 nod_flag=mi_test_if_nod(buff);
729 used_length=mi_getint(buff);
730 keypos=buff+2+nod_flag;
731 endpos=buff+used_length;
733 param->keydata+=used_length; param->totaldata+=keyinfo->block_length;
735 if (level > param->max_level)
736 param->max_level=level;
738 if (used_length > keyinfo->block_length)
740 mi_check_print_error(param,
"Wrong pageinfo at page: %s",
746 if (*killed_ptr(param))
748 memcpy(info->lastkey,key,key_length);
749 info->lastkey_length=key_length;
752 next_page=_mi_kpos(nod_flag,keypos);
753 if (chk_index_down(param,info,keyinfo,next_page,
754 temp_buff,keys,key_checksum,level+1))
757 if (keypos >= endpos ||
758 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
760 assert(key_length <=
sizeof(key));
763 mi_check_print_error(param,
"Wrong key block length at page: %s",llstr(page,llbuff));
767 (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
768 comp_flag, diff_pos)) >=0)
770 if (comp_flag & SEARCH_FIND && flag == 0)
771 mi_check_print_error(param,
"Found duplicated key at page %s",llstr(page,llbuff));
773 mi_check_print_error(param,
"Key in wrong position at page %s",llstr(page,llbuff));
776 if (param->testflag & T_STATISTICS)
780 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
781 ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
782 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
784 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
786 diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
787 param->notnull_count,
790 param->unique_count[diff_pos[0]-1]++;
794 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
795 mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
799 (*key_checksum)+= mi_byte_checksum((
unsigned char*) key,
800 key_length- info->s->rec_reflength);
801 record= _mi_dpos(info,0,key+key_length);
802 if (record >= info->state->data_file_length)
804 mi_check_print_error(param,
"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
807 param->record_checksum+=(ha_checksum) record;
809 if (keypos != endpos)
811 mi_check_print_error(param,
"Keyblock size at page %s is not correct. Block length: %d key length: %d",
812 llstr(page,llbuff), used_length, (keypos - buff));
825 static ha_checksum calc_checksum(ha_rows count)
841 return((ha_checksum) sum);
852 length= info->s->rec_reflength;
853 for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
854 length+= keyseg->length;
864 int error,got_error,flag;
865 uint key, left_length= 0, b_type;
866 ha_rows records, del_blocks;
867 my_off_t used, empty, pos, splits, start_recpos= 0,
868 del_length, link_used, start_block;
869 unsigned char *record= NULL, *to= NULL;
870 char llbuff[22],llbuff2[22],llbuff3[22];
871 ha_checksum intern_record_checksum;
872 ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
876 if (!(param->testflag & T_SILENT))
879 puts(
"- check records and index references");
881 puts(
"- check record links");
884 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
886 mi_check_print_error(param,
"Not enough memory for record");
889 records=del_blocks=0;
890 used=link_used=splits=del_length=0;
891 intern_record_checksum=param->glob_crc=0;
893 empty=info->s->pack.header_length;
895 pos= param->read_cache.tell();
896 memset(key_checksum, 0, info->s->base.keys *
sizeof(key_checksum[0]));
897 while (pos < info->state->data_file_length)
899 if (*killed_ptr(param))
901 switch (info->s->data_file_type) {
903 if (param->read_cache.read(record, info->s->base.pack_reclength))
906 pos+=info->s->base.pack_reclength;
911 del_length+=info->s->base.pack_reclength;
914 param->glob_crc+= mi_static_checksum(info,record);
915 used+=info->s->base.pack_reclength;
918 flag=block_info.second_read=0;
919 block_info.next_filepos=pos;
922 if (_mi_read_cache(¶m->read_cache,(
unsigned char*) block_info.header,
923 (start_block=block_info.next_filepos),
924 sizeof(block_info.header),
925 (flag ? 0 : READING_NEXT) | READING_HEADER))
927 if (start_block & (MI_DYN_ALIGN_SIZE-1))
929 mi_check_print_error(param,
"Wrong aligned block at %s",
930 llstr(start_block,llbuff));
933 b_type=_mi_get_block_info(&block_info,-1,start_block);
934 if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
937 if (b_type & BLOCK_SYNC_ERROR)
941 mi_check_print_error(param,
"Unexpected byte: %d at link: %s",
942 (
int) block_info.header[0],
943 llstr(start_block,llbuff));
946 pos=block_info.filepos+block_info.block_len;
949 if (b_type & BLOCK_DELETED)
951 if (block_info.block_len < info->s->base.min_block_length)
953 mi_check_print_error(param,
954 "Deleted block with impossible length %lu at %s",
955 block_info.block_len,llstr(pos,llbuff));
958 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
959 block_info.next_filepos >= info->state->data_file_length) ||
960 (block_info.prev_filepos != HA_OFFSET_ERROR &&
961 block_info.prev_filepos >= info->state->data_file_length))
963 mi_check_print_error(param,
"Delete link points outside datafile at %s",
968 del_length+=block_info.block_len;
969 pos=block_info.filepos+block_info.block_len;
973 mi_check_print_error(param,
"Wrong bytesec: %d-%d-%d at linkstart: %s",
974 block_info.header[0],block_info.header[1],
975 block_info.header[2],
976 llstr(start_block,llbuff));
979 if (info->state->data_file_length < block_info.filepos+
980 block_info.block_len)
982 mi_check_print_error(param,
983 "Recordlink that points outside datafile at %s",
992 pos=block_info.filepos+block_info.block_len;
993 if (block_info.rec_len > (uint) info->s->base.max_pack_length)
995 mi_check_print_error(param,
"Found too long record (%lu) at %s",
996 (ulong) block_info.rec_len,
997 llstr(start_recpos,llbuff));
1001 if (info->s->base.blobs)
1003 if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1006 mi_check_print_error(param,
1007 "Not enough memory (%lu) for blob at %s",
1008 (ulong) block_info.rec_len,
1009 llstr(start_recpos,llbuff));
1016 left_length=block_info.rec_len;
1018 if (left_length < block_info.data_len)
1020 mi_check_print_error(param,
"Found too long record (%lu) at %s",
1021 (ulong) block_info.data_len,
1022 llstr(start_recpos,llbuff));
1026 if (_mi_read_cache(¶m->read_cache,(
unsigned char*) to,block_info.filepos,
1027 (uint) block_info.data_len,
1028 flag == 1 ? READING_NEXT : 0))
1030 to+=block_info.data_len;
1031 link_used+= block_info.filepos-start_block;
1032 used+= block_info.filepos - start_block + block_info.data_len;
1033 empty+=block_info.block_len-block_info.data_len;
1034 left_length-=block_info.data_len;
1037 if (b_type & BLOCK_LAST)
1039 mi_check_print_error(param,
1040 "Wrong record length %s of %s at %s",
1041 llstr(block_info.rec_len-left_length,llbuff),
1042 llstr(block_info.rec_len, llbuff2),
1043 llstr(start_recpos,llbuff3));
1047 if (info->state->data_file_length < block_info.next_filepos)
1049 mi_check_print_error(param,
1050 "Found next-recordlink that points outside datafile at %s",
1051 llstr(block_info.filepos,llbuff));
1056 }
while (left_length);
1059 if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1062 mi_check_print_error(param,
"Found wrong record at %s",
1063 llstr(start_recpos,llbuff));
1068 info->checksum=mi_checksum(info,record);
1069 if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1071 if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1072 test(info->s->calc_checksum)))
1074 mi_check_print_error(param,
"Found wrong packed record at %s",
1075 llstr(start_recpos,llbuff));
1080 param->glob_crc+= info->checksum;
1084 pos=block_info.filepos+block_info.block_len;
1086 case COMPRESSED_RECORD:
1092 intern_record_checksum+=(ha_checksum) start_recpos;
1094 if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1096 printf(
"%s\r", llstr(records,llbuff)); fflush(stdout);
1101 for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1104 if (mi_is_key_active(info->s->state.key_map, key))
1107 uint32_t key_length=_mi_make_key(info,key,info->lastkey,record,
1115 _mi_search(info,keyinfo,info->lastkey,key_length,
1116 SEARCH_SAME, info->s->state.key_root[key]);
1119 mi_check_print_error(param,
"Record at: %10s "
1120 "Can't find key for index: %2d",
1121 llstr(start_recpos,llbuff),key+1);
1122 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1127 key_checksum[key]+=mi_byte_checksum((
unsigned char*) info->lastkey,
1136 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1141 if (param->testflag & T_WRITE_LOOP)
1143 fputs(
" \r",stdout); fflush(stdout);
1145 if (records != info->state->records)
1147 mi_check_print_error(param,
"Record-count is not ok; is %-10s Should be: %s",
1148 llstr(records,llbuff), llstr(info->state->records,llbuff2));
1151 else if (param->record_checksum &&
1152 param->record_checksum != intern_record_checksum)
1154 mi_check_print_error(param,
1155 "Keypointers and record positions doesn't match");
1158 else if (param->glob_crc != info->state->checksum &&
1160 (HA_OPTION_COMPRESS_RECORD)))
1162 mi_check_print_warning(param,
1163 "Record checksum is not the same as checksum stored in the index file\n");
1168 for (key=0 ; key < info->s->base.keys; key++)
1170 if (key_checksum[key] != param->key_crc[key])
1172 mi_check_print_error(param,
"Checksum for key: %2d doesn't match checksum for records",
1179 if (del_length != info->state->empty)
1181 mi_check_print_warning(param,
1182 "Found %s deleted space. Should be %s",
1183 llstr(del_length,llbuff2),
1184 llstr(info->state->empty,llbuff));
1186 if (used+empty+del_length != info->state->data_file_length)
1188 mi_check_print_warning(param,
1189 "Found %s record-data and %s unused data and %s deleted-data",
1190 llstr(used,llbuff),llstr(empty,llbuff2),
1191 llstr(del_length,llbuff3));
1192 mi_check_print_warning(param,
1193 "Total %s, Should be: %s",
1194 llstr((used+empty+del_length),llbuff),
1195 llstr(info->state->data_file_length,llbuff2));
1197 if (del_blocks != info->state->del)
1199 mi_check_print_warning(param,
1200 "Found %10s deleted blocks Should be: %s",
1201 llstr(del_blocks,llbuff),
1202 llstr(info->state->del,llbuff2));
1204 if (splits != info->s->state.split)
1206 mi_check_print_warning(param,
1207 "Found %10s parts Should be: %s parts",
1208 llstr(splits,llbuff),
1209 llstr(info->s->state.split,llbuff2));
1211 if (param->testflag & T_INFO)
1213 if (param->warning_printed || param->error_printed)
1215 if (used != 0 && ! param->error_printed)
1217 printf(
"Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1218 llstr(records,llbuff), (
long)((used-link_used)/records),
1219 (info->s->base.blobs ? 0.0 :
1220 (uint64_t2double((uint64_t) info->s->base.reclength*records)-
1221 my_off_t2double(used))/
1222 uint64_t2double((uint64_t) info->s->base.reclength*records)*100.0));
1223 printf(
"Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1224 (uint64_t2double(used-link_used)/uint64_t2double(used-link_used+empty)*100.0),
1225 (!records ? 100 : (
int) (uint64_t2double(del_length+empty)/
1226 my_off_t2double(used)*100.0)),
1227 uint64_t2double(splits - del_blocks) / records);
1229 printf(
"Record blocks:%12s Delete blocks:%10s\n",
1230 llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1231 printf(
"Record data: %12s Deleted data: %10s\n",
1232 llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1233 printf(
"Lost space: %12s Linkdata: %10s\n",
1234 llstr(empty,llbuff),llstr(link_used,llbuff2));
1236 free(mi_get_rec_buff_ptr(info, record));
1239 mi_check_print_error(param,
"got error: %d when reading datafile at record: %s",errno, llstr(records,llbuff));
1241 free(mi_get_rec_buff_ptr(info, record));
1242 param->testflag|=T_RETRY_WITHOUT_QUICK;
1300 static void mi_drop_all_indexes(
MI_CHECK *param,
MI_INFO *info,
bool force)
1313 if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1315 for (i= 0; i < share->base.keys; i++)
1317 if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1318 !mi_is_key_active(state->key_map, i))
1328 if (i >= share->base.keys)
1342 mi_clear_all_keys_active(state->key_map);
1346 for (i= 0; i < share->base.keys; i++)
1347 state->key_root[i]= HA_OFFSET_ERROR;
1350 for (i= 0; i < state->header.max_block_size_index; i++)
1351 state->key_del[i]= HA_OFFSET_ERROR;
1354 info->state->key_file_length= share->base.keystart;
1362 char * name,
int rep_quick)
1364 int error,got_error;
1365 ha_rows start_records,new_header_length;
1369 char llbuff[22],llbuff2[22];
1373 memset(&sort_info, 0,
sizeof(sort_info));
1374 memset(&sort_param, 0,
sizeof(sort_param));
1375 start_records=info->state->records;
1376 new_header_length=(param->testflag & T_UNPACK) ? 0L :
1377 share->pack.header_length;
1380 sort_param.sort_info=&sort_info;
1382 if (!(param->testflag & T_SILENT))
1384 printf(
"- recovering (with keycache) MyISAM-table '%s'\n",name);
1385 printf(
"Data records: %s\n", llstr(info->state->records,llbuff));
1387 param->testflag|=T_REP;
1389 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1390 param->testflag|=T_CALC_CHECKSUM;
1392 if (!param->using_global_keycache)
1395 if (param->read_cache.
init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1397 memset(&info->rec_cache, 0,
sizeof(info->rec_cache));
1402 if (info->rec_cache.
init_io_cache(-1, (uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL)))
1407 info->opt_flag|=WRITE_CACHE_USED;
1408 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1409 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1411 mi_check_print_error(param,
"Not enough memory for extra record");
1418 if ((new_file=my_create(internal::fn_format(param->temp_filename,
1419 share->data_file_name,
"",
1421 0,param->tmpfile_createflag,
1424 mi_check_print_error(param,
"Can't create new tempfile: '%s'",
1425 param->temp_filename);
1428 if (new_header_length &&
1429 filecopy(param,new_file,info->dfile,0L,new_header_length,
1432 info->s->state.dellink= HA_OFFSET_ERROR;
1433 info->rec_cache.file=new_file;
1434 if (param->testflag & T_UNPACK)
1436 share->options&= ~HA_OPTION_COMPRESS_RECORD;
1437 mi_int2store(share->state.header.options,share->options);
1440 sort_info.info=info;
1441 sort_info.param = param;
1442 sort_param.read_cache=param->read_cache;
1443 sort_param.pos=sort_param.max_pos=share->pack.header_length;
1444 sort_param.filepos=new_header_length;
1445 param->read_cache.end_of_file=sort_info.filelength=
1446 lseek(info->dfile,0L,SEEK_END);
1448 sort_param.fix_datafile= (bool) (! rep_quick);
1449 sort_param.master=1;
1450 sort_info.max_records= ~(ha_rows) 0;
1452 set_data_file_type(&sort_info, share);
1453 del=info->state->del;
1454 info->state->records=info->state->del=share->state.split=0;
1455 info->state->empty=0;
1457 if (param->testflag & T_CALC_CHECKSUM)
1458 sort_param.calc_checksum= 1;
1460 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1463 if (param->testflag & T_CREATE_MISSING_KEYS)
1464 mi_set_all_keys_active(share->state.key_map, share->base.keys);
1465 mi_drop_all_indexes(param, info,
true);
1470 while (!(error=sort_get_next_record(&sort_param)))
1472 if (writekeys(&sort_param))
1474 if (errno != HA_ERR_FOUND_DUPP_KEY)
1476 mi_check_print_info(param,
"Duplicate key %2d for record at %10s against new record at %10s",
1478 llstr(sort_param.start_recpos,llbuff),
1479 llstr(info->dupp_key_pos,llbuff2));
1480 if (param->testflag & T_VERBOSE)
1482 _mi_make_key(info,(uint) info->errkey,info->lastkey,
1483 sort_param.record,0L);
1486 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1488 param->testflag|=T_RETRY_WITHOUT_QUICK;
1489 param->error_printed=1;
1494 if (sort_write_record(&sort_param))
1497 if (error > 0 || write_data_suffix(&sort_info, (
bool)!rep_quick) ||
1498 info->rec_cache.flush() || param->read_cache.error < 0)
1501 if (param->testflag & T_WRITE_LOOP)
1503 fputs(
" \r",stdout); fflush(stdout);
1505 if (ftruncate(share->kfile, info->state->key_file_length))
1507 mi_check_print_warning(param,
1508 "Can't change size of indexfile, error: %d",
1513 if (rep_quick && del+sort_info.dupp != info->state->del)
1515 mi_check_print_error(param,
"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1516 mi_check_print_error(param,
"Run recovery again without -q");
1518 param->retry_repair=1;
1519 param->testflag|=T_RETRY_WITHOUT_QUICK;
1522 if (param->testflag & T_SAFE_REPAIR)
1525 if (info->state->records+1 < start_records)
1527 info->state->records=start_records;
1535 internal::my_close(info->dfile,MYF(0));
1536 info->dfile=new_file;
1537 info->state->data_file_length=sort_param.filepos;
1538 share->state.version=(ulong) time((time_t*) 0);
1542 info->state->data_file_length=sort_param.max_pos;
1544 if (param->testflag & T_CALC_CHECKSUM)
1545 info->state->checksum=param->glob_crc;
1547 if (!(param->testflag & T_SILENT))
1549 if (start_records != info->state->records)
1550 printf(
"Data records: %s\n", llstr(info->state->records,llbuff));
1552 mi_check_print_warning(param,
1553 "%s records have been removed",
1554 llstr(sort_info.dupp,llbuff));
1559 if (&share->state.state != info->state)
1560 memcpy( &share->state.state, info->state,
sizeof(*info->state));
1568 internal::my_close(new_file,MYF(0));
1569 info->dfile=new_file= -1;
1570 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1571 DATA_TMP_EXT, share->base.raid_chunks,
1573 mi_open_datafile(info,share,-1))
1579 if (! param->error_printed)
1580 mi_check_print_error(param,
"%d for record at pos %s",errno,
1581 llstr(sort_param.start_recpos,llbuff));
1584 internal::my_close(new_file,MYF(0));
1585 my_delete(param->temp_filename, MYF(MY_WME));
1586 info->rec_cache.file=-1;
1588 mi_mark_crashed_on_repair(info);
1591 void * rec_buff_ptr= NULL;
1592 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
1594 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
1598 free(sort_info.buff);
1600 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1602 if (not got_error && param->testflag & T_UNPACK)
1604 share->state.header.options[0]&= (
unsigned char) ~HA_OPTION_COMPRESS_RECORD;
1605 share->pack.header_length=0;
1606 share->data_file_type=sort_info.new_data_file_type;
1608 share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1609 STATE_NOT_ANALYZED);
1618 register uint32_t i;
1620 MI_INFO *info= sort_param->sort_info->info;
1621 unsigned char *buff= sort_param->record;
1622 my_off_t filepos= sort_param->filepos;
1624 key=info->lastkey+info->s->base.max_key_length;
1625 for (i=0 ; i < info->s->base.keys ; i++)
1627 if (mi_is_key_active(info->s->state.key_map, i))
1630 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1631 if (_mi_ck_write(info,i,key,key_length))
1639 if (errno == HA_ERR_FOUND_DUPP_KEY)
1641 info->errkey=(int) i;
1644 if (mi_is_key_active(info->s->state.key_map, i))
1647 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1648 if (_mi_ck_delete(info,i,key,key_length))
1655 if (sort_param->calc_checksum)
1656 sort_param->sort_info->param->glob_crc-= info->checksum;
1663 int movepoint(
register MI_INFO *info,
unsigned char *record, my_off_t oldpos,
1664 my_off_t newpos, uint32_t prot_key)
1666 register uint32_t i;
1668 uint32_t key_length;
1670 key=info->lastkey+info->s->base.max_key_length;
1671 for (i=0 ; i < info->s->base.keys; i++)
1673 if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1675 key_length=_mi_make_key(info,i,key,record,oldpos);
1676 if (info->s->keyinfo[i].flag & HA_NOSAME)
1680 keyinfo=info->s->keyinfo+i;
1681 if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1682 (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1683 info->s->state.key_root[i]))
1685 nod_flag=mi_test_if_nod(info->buff);
1686 _mi_dpointer(info,info->int_keypos-nod_flag-
1687 info->s->rec_reflength,newpos);
1688 if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1689 DFLT_INIT_HITS,info->buff))
1694 if (_mi_ck_delete(info,i,key,key_length))
1696 key_length=_mi_make_key(info,i,key,record,newpos);
1697 if (_mi_ck_write(info,i,key,key_length))
1715 int mi_sort_index(
MI_CHECK *param,
register MI_INFO *info,
char * name)
1717 register uint32_t key;
1720 my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1721 uint32_t r_locks,w_locks;
1727 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1730 if (!(param->testflag & T_SILENT))
1731 printf(
"- Sorting index for MyISAM-table '%s'\n",name);
1734 internal::fn_format(param->temp_filename,name,
"", MI_NAME_IEXT,2+4+32);
1735 if ((new_file=my_create(internal::fn_format(param->temp_filename,param->temp_filename,
1736 "", INDEX_TMP_EXT,2+4),
1737 0,param->tmpfile_createflag,MYF(0))) <= 0)
1739 mi_check_print_error(param,
"Can't create new tempfile: '%s'",
1740 param->temp_filename);
1743 if (filecopy(param, new_file,share->kfile,0L,
1744 (ulong) share->base.keystart,
"headerblock"))
1747 param->new_file_pos=share->base.keystart;
1748 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1751 if (! mi_is_key_active(info->s->state.key_map, key))
1754 if (share->state.key_root[key] != HA_OFFSET_ERROR)
1756 index_pos[key]=param->new_file_pos;
1757 if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1762 index_pos[key]= HA_OFFSET_ERROR;
1765 share->state.version=(ulong) time(NULL);
1766 old_state= share->state;
1767 r_locks= share->r_locks;
1768 w_locks= share->w_locks;
1769 old_lock= info->lock_type;
1772 share->r_locks= share->w_locks= share->tot_locks= 0;
1773 (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1774 internal::my_close(share->kfile,MYF(MY_WME));
1776 internal::my_close(new_file,MYF(MY_WME));
1777 if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1779 mi_open_keyfile(share))
1781 info->lock_type= F_UNLCK;
1782 _mi_readinfo(info,F_WRLCK,0);
1783 info->lock_type= old_lock;
1784 share->r_locks= r_locks;
1785 share->w_locks= w_locks;
1786 share->tot_locks= r_locks+w_locks;
1787 share->state= old_state;
1789 info->state->key_file_length=param->new_file_pos;
1790 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1791 for (key=0 ; key < info->s->base.keys ; key++)
1792 info->s->state.key_root[key]=index_pos[key];
1793 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
1794 info->s->state.key_del[key]= HA_OFFSET_ERROR;
1796 info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
1800 internal::my_close(new_file,MYF(MY_WME));
1802 my_delete(param->temp_filename,MYF(MY_WME));
1810 my_off_t pagepos,
int new_file)
1812 uint32_t length,nod_flag,used_length, key_length;
1813 unsigned char *buff,*keypos,*endpos;
1814 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF];
1815 my_off_t new_page_pos,next_page;
1818 new_page_pos=param->new_file_pos;
1819 param->new_file_pos+=keyinfo->block_length;
1821 if (!(buff=(
unsigned char*) malloc(keyinfo->block_length)))
1823 mi_check_print_error(param,
"Not enough memory for key block");
1826 if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
1828 mi_check_print_error(param,
"Can't read key block from filepos: %s",
1829 llstr(pagepos,llbuff));
1832 if ((nod_flag=mi_test_if_nod(buff)))
1834 used_length=mi_getint(buff);
1835 keypos=buff+2+nod_flag;
1836 endpos=buff+used_length;
1841 next_page=_mi_kpos(nod_flag,keypos);
1842 _mi_kpointer(info,keypos-nod_flag,param->new_file_pos);
1843 if (sort_one_index(param,info,keyinfo,next_page,new_file))
1848 if (keypos >= endpos ||
1849 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
1851 assert(keypos <= endpos);
1856 length=mi_getint(buff);
1857 memset(buff+length, 0, keyinfo->block_length-length);
1858 if (my_pwrite(new_file,(
unsigned char*) buff,(uint) keyinfo->block_length,
1859 new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
1861 mi_check_print_error(param,
"Can't write indexblock, error: %d",errno);
1881 int change_to_newfile(
const char * filename,
const char * old_ext,
1882 const char * new_ext,
1883 uint32_t raid_chunks,
1887 char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
1889 (void) internal::fn_format(old_filename,filename,
"",old_ext,2+4+32);
1890 return my_redel(old_filename,
1891 internal::fn_format(new_filename,old_filename,
"",new_ext,2+4),
1892 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
1899 int filecopy(
MI_CHECK *param,
int to,
int from,my_off_t start,
1900 my_off_t length,
const char *type)
1902 char tmp_buff[IO_SIZE],*buff;
1905 buff_length=(ulong) min(param->write_buffer_length, (
size_t)length);
1906 if (!(buff=(
char *)malloc(buff_length)))
1908 buff=tmp_buff; buff_length=IO_SIZE;
1911 lseek(from,start,SEEK_SET);
1912 while (length > buff_length)
1914 if (my_read(from,(
unsigned char*) buff,buff_length,MYF(MY_NABP)) ||
1915 my_write(to,(
unsigned char*) buff,buff_length,param->myf_rw))
1917 length-= buff_length;
1919 if (my_read(from,(
unsigned char*) buff,(uint) length,MYF(MY_NABP)) ||
1920 my_write(to,(
unsigned char*) buff,(uint) length,param->myf_rw))
1922 if (buff != tmp_buff)
1926 if (buff != tmp_buff)
1928 mi_check_print_error(param,
"Can't copy %s to tempfile, error %d",
1950 const char * name,
int rep_quick)
1955 ha_rows start_records;
1956 my_off_t new_header_length,del;
1961 ulong *rec_per_key_part;
1964 uint64_t key_map= 0;
1966 start_records=info->state->records;
1969 new_header_length=(param->testflag & T_UNPACK) ? 0 :
1970 share->pack.header_length;
1971 if (!(param->testflag & T_SILENT))
1973 printf(
"- recovering (with sort) MyISAM-table '%s'\n",name);
1974 printf(
"Data records: %s\n", llstr(start_records,llbuff));
1976 param->testflag|=T_REP;
1978 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1979 param->testflag|=T_CALC_CHECKSUM;
1981 memset(&sort_info, 0,
sizeof(sort_info));
1982 memset(&sort_param, 0,
sizeof(sort_param));
1983 if (!(sort_info.key_block=
1984 alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length))
1985 || param->read_cache.
init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))
1986 || (! rep_quick && info->rec_cache.
init_io_cache(info->dfile, (uint) param->write_buffer_length, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
1990 sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
1991 info->opt_flag|=WRITE_CACHE_USED;
1992 info->rec_cache.file=info->dfile;
1994 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1995 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1997 mi_check_print_error(param,
"Not enough memory for extra record");
2003 if ((new_file=my_create(internal::fn_format(param->temp_filename,
2004 share->data_file_name,
"",
2006 0,param->tmpfile_createflag,
2009 mi_check_print_error(param,
"Can't create new tempfile: '%s'",
2010 param->temp_filename);
2013 if (new_header_length &&
2014 filecopy(param, new_file,info->dfile,0L,new_header_length,
2017 if (param->testflag & T_UNPACK)
2019 share->options&= ~HA_OPTION_COMPRESS_RECORD;
2020 mi_int2store(share->state.header.options,share->options);
2022 share->state.dellink= HA_OFFSET_ERROR;
2023 info->rec_cache.file=new_file;
2026 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2029 mi_drop_all_indexes(param, info,
false);
2030 key_map= share->state.key_map;
2031 if (param->testflag & T_CREATE_MISSING_KEYS)
2037 sort_info.info=info;
2038 sort_info.param = param;
2040 set_data_file_type(&sort_info, share);
2041 sort_param.filepos=new_header_length;
2044 param->read_cache.end_of_file=sort_info.filelength=
2045 lseek(param->read_cache.file,0L,SEEK_END);
2047 sort_param.wordlist=NULL;
2049 if (share->data_file_type == DYNAMIC_RECORD)
2050 length=max(share->base.min_pack_length+1,share->base.min_block_length);
2051 else if (share->data_file_type == COMPRESSED_RECORD)
2052 length=share->base.min_block_length;
2054 length=share->base.pack_reclength;
2055 sort_info.max_records=
2056 ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2057 (ha_rows) (sort_info.filelength/length+1));
2058 sort_param.key_cmp=sort_key_cmp;
2059 sort_param.lock_in_memory=lock_memory;
2060 sort_param.sort_info=&sort_info;
2061 sort_param.fix_datafile= (bool) (! rep_quick);
2062 sort_param.master =1;
2064 del=info->state->del;
2066 if (param->testflag & T_CALC_CHECKSUM)
2067 sort_param.calc_checksum= 1;
2069 rec_per_key_part= param->rec_per_key_part;
2070 for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2071 rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2073 sort_param.read_cache=param->read_cache;
2074 sort_param.keyinfo=share->keyinfo+sort_param.key;
2075 sort_param.seg=sort_param.keyinfo->seg;
2080 if (! mi_is_key_active(key_map, sort_param.key))
2083 assert(rec_per_key_part >= param->rec_per_key_part);
2084 memcpy(rec_per_key_part,
2085 (share->state.rec_per_key_part +
2086 (rec_per_key_part - param->rec_per_key_part)),
2087 sort_param.keyinfo->keysegs*
sizeof(*rec_per_key_part));
2091 if ((!(param->testflag & T_SILENT)))
2092 printf (
"- Fixing index %d\n",sort_param.key+1);
2093 sort_param.max_pos=sort_param.pos=share->pack.header_length;
2094 keyseg=sort_param.seg;
2095 memset(sort_param.unique, 0,
sizeof(sort_param.unique));
2096 sort_param.key_length=share->rec_reflength;
2097 for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2099 sort_param.key_length+=keyseg[i].length;
2100 if (keyseg[i].flag & HA_SPACE_PACK)
2101 sort_param.key_length+=get_pack_length(keyseg[i].length);
2102 if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2103 sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2104 if (keyseg[i].flag & HA_NULL_PART)
2105 sort_param.key_length++;
2107 info->state->records=info->state->del=share->state.split=0;
2108 info->state->empty=0;
2111 sort_param.key_read=sort_key_read;
2112 sort_param.key_write=sort_key_write;
2115 if (_create_index_by_sort(&sort_param,
2116 (
bool) (!(param->testflag & T_VERBOSE)),
2117 (uint) param->sort_buffer_length))
2119 param->retry_repair=1;
2123 sort_param.calc_checksum= 0;
2127 sort_info.max_records= (ha_rows) info->state->records;
2129 if (param->testflag & T_STATISTICS)
2130 update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2131 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2132 sort_param.notnull: NULL,
2133 (uint64_t) info->state->records);
2135 mi_set_key_active(share->state.key_map, sort_param.key);
2137 if (sort_param.fix_datafile)
2139 param->read_cache.end_of_file=sort_param.filepos;
2140 if (write_data_suffix(&sort_info, 1) || info->rec_cache.
end_io_cache())
2144 if (param->testflag & T_SAFE_REPAIR)
2147 if (info->state->records+1 < start_records)
2149 info->state->records=start_records;
2153 share->state.state.data_file_length = info->state->data_file_length=
2156 share->state.version=(ulong) time((time_t*) 0);
2157 internal::my_close(info->dfile,MYF(0));
2158 info->dfile=new_file;
2159 share->data_file_type=sort_info.new_data_file_type;
2160 share->pack.header_length=(ulong) new_header_length;
2161 sort_param.fix_datafile=0;
2164 info->state->data_file_length=sort_param.max_pos;
2166 param->read_cache.file=info->dfile;
2167 param->read_cache.
reinit_io_cache(READ_CACHE,share->pack.header_length, 1,1);
2170 if (param->testflag & T_WRITE_LOOP)
2172 fputs(
" \r",stdout); fflush(stdout);
2175 if (rep_quick && del+sort_info.dupp != info->state->del)
2177 mi_check_print_error(param,
"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2178 mi_check_print_error(param,
"Run recovery again without -q");
2180 param->retry_repair=1;
2181 param->testflag|=T_RETRY_WITHOUT_QUICK;
2185 if (rep_quick & T_FORCE_UNIQUENESS)
2187 my_off_t skr=info->state->data_file_length+
2188 (share->options & HA_OPTION_COMPRESS_RECORD ?
2189 MEMMAP_EXTRA_MARGIN : 0);
2191 if (share->data_file_type == STATIC_RECORD &&
2192 skr < share->base.reloc*share->base.min_pack_length)
2193 skr=share->base.reloc*share->base.min_pack_length;
2195 if (skr != sort_info.filelength && !info->s->base.raid_type)
2196 if (ftruncate(info->dfile, skr))
2197 mi_check_print_warning(param,
2198 "Can't change size of datafile, error: %d",
2201 if (param->testflag & T_CALC_CHECKSUM)
2202 info->state->checksum=param->glob_crc;
2204 if (ftruncate(share->kfile, info->state->key_file_length))
2205 mi_check_print_warning(param,
2206 "Can't change size of indexfile, error: %d",
2209 if (!(param->testflag & T_SILENT))
2211 if (start_records != info->state->records)
2212 printf(
"Data records: %s\n", llstr(info->state->records,llbuff));
2214 mi_check_print_warning(param,
2215 "%s records have been removed",
2216 llstr(sort_info.dupp,llbuff));
2220 if (&share->state.state != info->state)
2221 memcpy( &share->state.state, info->state,
sizeof(*info->state));
2230 internal::my_close(new_file,MYF(0));
2231 info->dfile=new_file= -1;
2232 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2233 DATA_TMP_EXT, share->base.raid_chunks,
2235 mi_open_datafile(info,share,-1))
2241 if (! param->error_printed)
2242 mi_check_print_error(param,
"%d when fixing table",errno);
2245 internal::my_close(new_file,MYF(0));
2246 my_delete(param->temp_filename, MYF(MY_WME));
2247 if (info->dfile == new_file)
2250 mi_mark_crashed_on_repair(info);
2252 else if (key_map == share->state.key_map)
2253 share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2254 share->state.changed|=STATE_NOT_SORTED_PAGES;
2256 void * rec_buff_ptr= NULL;
2257 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
2259 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
2263 free((
unsigned char*) sort_info.key_block);
2264 free(sort_info.buff);
2266 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2267 if (!got_error && (param->testflag & T_UNPACK))
2269 share->state.header.options[0]&= (
unsigned char) ~HA_OPTION_COMPRESS_RECORD;
2270 share->pack.header_length=0;
2280 SORT_INFO *sort_info=sort_param->sort_info;
2281 MI_INFO *info=sort_info->info;
2283 if ((error=sort_get_next_record(sort_param)))
2285 if (info->state->records == sort_info->max_records)
2287 mi_check_print_error(sort_info->param,
2288 "Key %d - Found too many records; Can't continue",
2292 sort_param->real_key_length=
2293 (info->s->rec_reflength+
2294 _mi_make_key(info, sort_param->key, (
unsigned char*) key,
2295 sort_param->record, sort_param->filepos));
2296 #ifdef HAVE_VALGRIND
2297 memset((
unsigned char *)key+sort_param->real_key_length, 0,
2298 (sort_param->key_length-sort_param->real_key_length));
2300 return(sort_write_record(sort_param));
2339 uint32_t found_record,b_type,left_length;
2341 unsigned char *to= NULL;
2343 SORT_INFO *sort_info=sort_param->sort_info;
2345 MI_INFO *info=sort_info->info;
2347 char llbuff[22],llbuff2[22];
2349 if (*killed_ptr(param))
2352 switch (share->data_file_type) {
2356 if (sort_param->read_cache.read(sort_param->record, share->base.pack_reclength))
2358 if (sort_param->read_cache.error)
2359 param->out_flag |= O_DATA_LOST;
2360 param->retry_repair=1;
2361 param->testflag|=T_RETRY_WITHOUT_QUICK;
2364 sort_param->start_recpos=sort_param->pos;
2365 if (!sort_param->fix_datafile)
2367 sort_param->filepos=sort_param->pos;
2368 if (sort_param->master)
2369 share->state.split++;
2371 sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
2372 if (*sort_param->record)
2374 if (sort_param->calc_checksum)
2375 param->glob_crc+= (info->checksum=
2376 mi_static_checksum(info,sort_param->record));
2379 if (!sort_param->fix_datafile && sort_param->master)
2382 info->state->empty+=share->base.pack_reclength;
2385 case DYNAMIC_RECORD:
2386 pos= sort_param->pos;
2387 searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
2388 parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
2391 found_record=block_info.second_read= 0;
2395 pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
2396 param->testflag|=T_RETRY_WITHOUT_QUICK;
2397 sort_param->start_recpos=pos;
2401 if (pos > sort_param->max_pos)
2402 sort_param->max_pos=pos;
2403 if (pos & (MI_DYN_ALIGN_SIZE-1))
2405 if ((param->testflag & T_VERBOSE) || searching == 0)
2406 mi_check_print_info(param,
"Wrong aligned block at %s",
2411 if (found_record && pos == param->search_after_block)
2412 mi_check_print_info(param,
"Block: %s used by record at %s",
2413 llstr(param->search_after_block,llbuff),
2414 llstr(sort_param->start_recpos,llbuff2));
2415 if (_mi_read_cache(&sort_param->read_cache,
2416 (
unsigned char*) block_info.header,pos,
2417 MI_BLOCK_INFO_HEADER_LENGTH,
2418 (! found_record ? READING_NEXT : 0) |
2419 parallel_flag | READING_HEADER))
2423 mi_check_print_info(param,
2424 "Can't read whole record at %s (errno: %d)",
2425 llstr(sort_param->start_recpos,llbuff),errno);
2430 if (searching && ! sort_param->fix_datafile)
2432 param->error_printed=1;
2433 param->retry_repair=1;
2434 param->testflag|=T_RETRY_WITHOUT_QUICK;
2437 b_type=_mi_get_block_info(&block_info,-1,pos);
2438 if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
2439 ((b_type & BLOCK_FIRST) &&
2440 (block_info.rec_len < (uint) share->base.min_pack_length ||
2441 block_info.rec_len > (uint) share->base.max_pack_length)))
2444 if (param->testflag & T_VERBOSE || searching == 0)
2445 mi_check_print_info(param,
2446 "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
2447 block_info.header[0],block_info.header[1],
2448 block_info.header[2],llstr(pos,llbuff));
2451 block_info.second_read=0;
2454 for (i=MI_DYN_ALIGN_SIZE ;
2455 i < MI_BLOCK_INFO_HEADER_LENGTH ;
2456 i+= MI_DYN_ALIGN_SIZE)
2457 if (block_info.header[i] >= 1 &&
2458 block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
2461 sort_param->start_recpos=pos;
2464 if (b_type & BLOCK_DELETED)
2467 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2468 share->base.min_block_length)
2471 mi_check_print_info(param,
2472 "Deleted block with impossible length %u at %s",
2473 block_info.block_len,llstr(pos,llbuff));
2478 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
2479 block_info.next_filepos >=
2480 info->state->data_file_length) ||
2481 (block_info.prev_filepos != HA_OFFSET_ERROR &&
2482 block_info.prev_filepos >= info->state->data_file_length))
2485 mi_check_print_info(param,
2486 "Delete link points outside datafile at %s",
2496 pos+= MI_DYN_ALIGN_SIZE;
2497 sort_param->start_recpos=pos;
2498 block_info.second_read=0;
2504 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2505 share->base.min_block_length ||
2506 block_info.block_len > (uint) share->base.max_pack_length+
2510 mi_check_print_info(param,
2511 "Found block with impossible length %u at %s; Skipped",
2512 block_info.block_len+ (uint) (block_info.filepos-pos),
2517 pos+= MI_DYN_ALIGN_SIZE;
2518 sort_param->start_recpos=pos;
2519 block_info.second_read=0;
2523 if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
2525 if (!sort_param->fix_datafile && sort_param->master &&
2526 (b_type & BLOCK_DELETED))
2528 info->state->empty+=block_info.block_len;
2530 share->state.split++;
2536 pos+=MI_DYN_ALIGN_SIZE;
2537 sort_param->start_recpos=pos;
2540 pos=block_info.filepos+block_info.block_len;
2541 block_info.second_read=0;
2545 if (!sort_param->fix_datafile && sort_param->master)
2546 share->state.split++;
2547 if (! found_record++)
2549 sort_param->find_length=left_length=block_info.rec_len;
2550 sort_param->start_recpos=pos;
2551 if (!sort_param->fix_datafile)
2552 sort_param->filepos=sort_param->start_recpos;
2553 if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
2554 sort_param->pos=block_info.filepos+1;
2556 sort_param->pos=block_info.filepos+block_info.block_len;
2557 if (share->base.blobs)
2559 if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
2560 &(sort_param->rec_buff))))
2562 if (param->max_record_length >= block_info.rec_len)
2564 mi_check_print_error(param,
"Not enough memory for blob at %s (need %lu)",
2565 llstr(sort_param->start_recpos,llbuff),
2566 (ulong) block_info.rec_len);
2571 mi_check_print_info(param,
"Not enough memory for blob at %s (need %lu); Row skipped",
2572 llstr(sort_param->start_recpos,llbuff),
2573 (ulong) block_info.rec_len);
2579 to= sort_param->rec_buff;
2581 if (left_length < block_info.data_len || ! block_info.data_len)
2583 mi_check_print_info(param,
2584 "Found block with too small length at %s; Skipped",
2585 llstr(sort_param->start_recpos,llbuff));
2588 if (block_info.filepos + block_info.data_len >
2589 sort_param->read_cache.end_of_file)
2591 mi_check_print_info(param,
2592 "Found block that points outside data file at %s",
2593 llstr(sort_param->start_recpos,llbuff));
2602 uint32_t header_len= (uint) (block_info.filepos - pos);
2603 uint32_t prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
2605 if (prefetch_len > block_info.data_len)
2606 prefetch_len= block_info.data_len;
2609 memcpy(to, block_info.header + header_len, prefetch_len);
2610 block_info.filepos+= prefetch_len;
2611 block_info.data_len-= prefetch_len;
2612 left_length-= prefetch_len;
2616 if (block_info.data_len &&
2617 _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
2618 block_info.data_len,
2619 (found_record == 1 ? READING_NEXT : 0) |
2622 mi_check_print_info(param,
2623 "Read error for block at: %s (error: %d); Skipped",
2624 llstr(block_info.filepos,llbuff),errno);
2627 left_length-=block_info.data_len;
2628 to+=block_info.data_len;
2629 pos=block_info.next_filepos;
2630 if (pos == HA_OFFSET_ERROR && left_length)
2632 mi_check_print_info(param,
"Wrong block with wrong total length starting at %s",
2633 llstr(sort_param->start_recpos,llbuff));
2636 if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
2638 mi_check_print_info(param,
"Found link that points at %s (outside data file) at %s",
2640 llstr(sort_param->start_recpos,llbuff));
2643 }
while (left_length);
2645 if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
2646 sort_param->find_length) != MY_FILE_ERROR)
2648 if (sort_param->read_cache.error < 0)
2650 if (sort_param->calc_checksum)
2651 info->checksum= mi_checksum(info, sort_param->record);
2652 if ((param->testflag & (T_EXTEND | T_REP)) || searching)
2654 if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
2655 sort_param->find_length,
2656 (param->testflag & T_QUICK) &&
2657 sort_param->calc_checksum &&
2658 test(info->s->calc_checksum)))
2660 mi_check_print_info(param,
"Found wrong packed record at %s",
2661 llstr(sort_param->start_recpos,llbuff));
2665 if (sort_param->calc_checksum)
2666 param->glob_crc+= info->checksum;
2670 mi_check_print_info(param,
"Key %d - Found wrong stored record at %s",
2672 llstr(sort_param->start_recpos,llbuff));
2674 pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
2677 case COMPRESSED_RECORD:
2703 ulong block_length,reclength;
2704 unsigned char *from;
2705 SORT_INFO *sort_info=sort_param->sort_info;
2707 MI_INFO *info=sort_info->info;
2710 if (sort_param->fix_datafile)
2712 switch (sort_info->new_data_file_type) {
2714 if (info->rec_cache.write(sort_param->record, share->base.pack_reclength))
2716 mi_check_print_error(param,
"%d when writing to datafile",errno);
2719 sort_param->filepos+=share->base.pack_reclength;
2720 info->s->state.split++;
2723 case DYNAMIC_RECORD:
2725 from=sort_param->rec_buff;
2729 reclength=info->s->base.pack_reclength+
2730 _my_calc_total_blob_length(info,sort_param->record)+
2731 ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
2732 MI_DYN_DELETE_BLOCK_HEADER;
2733 if (sort_info->buff_length < reclength)
2736 tmpptr= realloc(sort_info->buff, reclength);
2739 sort_info->buff_length=reclength;
2740 sort_info->buff= (
unsigned char *)tmpptr;
2744 mi_check_print_error(param,
"Could not realloc() sort_info->buff "
2745 " to %ul bytes", reclength);
2749 from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
2752 info->checksum=mi_checksum(info,sort_param->record);
2753 reclength=_mi_rec_pack(info,from,sort_param->record);
2759 block_length=reclength+ 3 + test(reclength >= (65520-3));
2760 if (block_length < share->base.min_block_length)
2761 block_length=share->base.min_block_length;
2762 info->update|=HA_STATE_WRITE_AT_END;
2763 block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
2764 if (block_length > MI_MAX_BLOCK_LENGTH)
2765 block_length=MI_MAX_BLOCK_LENGTH;
2766 if (_mi_write_part_record(info,0L,block_length,
2767 sort_param->filepos+block_length,
2768 &from,&reclength,&flag))
2770 mi_check_print_error(param,
"%d when writing to datafile",errno);
2773 sort_param->filepos+=block_length;
2774 info->s->state.split++;
2775 }
while (reclength);
2778 case COMPRESSED_RECORD:
2783 if (sort_param->master)
2785 info->state->records++;
2786 if ((param->testflag & T_WRITE_LOOP) &&
2787 (info->state->records % WRITE_COUNT) == 0)
2790 printf(
"%s\r", llstr(info->state->records,llbuff));
2800 int sort_key_cmp(
MI_SORT_PARAM *sort_param,
const void *a,
const void *b)
2802 uint32_t not_used[2];
2803 return (ha_key_cmp(sort_param->seg, *((
unsigned char*
const *) a), *((
unsigned char*
const *) b),
2804 USE_WHOLE_KEY, SEARCH_SAME, not_used));
2808 int sort_key_write(
MI_SORT_PARAM *sort_param,
const void *a)
2810 uint32_t diff_pos[2];
2811 char llbuff[22],llbuff2[22];
2812 SORT_INFO *sort_info=sort_param->sort_info;
2816 if (sort_info->key_block->inited)
2818 cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2819 (
unsigned char*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
2821 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
2822 ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2823 (
unsigned char*) a, USE_WHOLE_KEY,
2824 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
2825 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2827 diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
2828 sort_param->notnull,
2829 sort_info->key_block->lastkey,
2832 sort_param->unique[diff_pos[0]-1]++;
2837 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2838 mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
2841 if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
2844 sort_info->info->lastpos=get_record_for_key(sort_info->info,
2845 sort_param->keyinfo,
2846 (
unsigned char*) a);
2847 mi_check_print_warning(param,
2848 "Duplicate key for record at %10s against record at %10s",
2849 llstr(sort_info->info->lastpos,llbuff),
2850 llstr(get_record_for_key(sort_info->info,
2851 sort_param->keyinfo,
2852 sort_info->key_block->
2855 param->testflag|=T_RETRY_WITHOUT_QUICK;
2856 return (sort_delete_record(sort_param));
2858 return (sort_insert_key(sort_param,sort_info->key_block,
2859 (
unsigned char*) a, HA_OFFSET_ERROR));
2866 unsigned char *key) {
2867 return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
2875 my_off_t prev_block)
2877 uint32_t a_length,t_length,nod_flag;
2878 my_off_t filepos,key_file_length;
2879 unsigned char *anc_buff,*lastkey;
2883 SORT_INFO *sort_info= sort_param->sort_info;
2886 anc_buff=key_block->buff;
2887 info=sort_info->info;
2888 lastkey=key_block->lastkey;
2889 nod_flag= (key_block == sort_info->key_block ? 0 :
2890 info->s->base.key_reflength);
2892 if (!key_block->inited)
2894 key_block->inited=1;
2895 if (key_block == sort_info->key_block_end)
2897 mi_check_print_error(param,
"To many key-block-levels; Try increasing sort_key_blocks");
2900 a_length=2+nod_flag;
2901 key_block->end_pos=anc_buff+2;
2905 a_length=mi_getint(anc_buff);
2909 _mi_kpointer(info,key_block->end_pos,prev_block);
2911 t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
2912 (
unsigned char*) 0,lastkey,lastkey,key,
2914 (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
2916 mi_putint(anc_buff,a_length,nod_flag);
2917 key_block->end_pos+=t_length;
2918 if (a_length <= keyinfo->block_length)
2920 _mi_move_key(keyinfo,key_block->lastkey,key);
2921 key_block->last_length=a_length-t_length;
2926 mi_putint(anc_buff,key_block->last_length,nod_flag);
2927 memset(anc_buff+key_block->last_length, 0,
2928 keyinfo->block_length - key_block->last_length);
2929 key_file_length=info->state->key_file_length;
2930 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
2934 if (key_file_length == info->state->key_file_length)
2936 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
2939 else if (my_pwrite(info->s->kfile,(
unsigned char*) anc_buff,
2940 (uint) keyinfo->block_length,filepos, param->myf_rw))
2944 if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
2948 key_block->inited=0;
2949 return(sort_insert_key(sort_param, key_block,key,prev_block));
2960 SORT_INFO *sort_info=sort_param->sort_info;
2962 MI_INFO *info=sort_info->info;
2964 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
2966 mi_check_print_error(param,
2967 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
2970 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
2972 mi_check_print_error(param,
2973 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
2977 old_file=info->dfile;
2978 info->dfile=info->rec_cache.file;
2979 if (sort_info->current_key)
2981 key=info->lastkey+info->s->base.max_key_length;
2982 if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
2983 error != HA_ERR_RECORD_DELETED)
2985 mi_check_print_error(param,
"Can't read record to be removed");
2986 info->dfile=old_file;
2990 for (i=0 ; i < sort_info->current_key ; i++)
2992 uint32_t key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
2993 if (_mi_ck_delete(info,i,key,key_length))
2995 mi_check_print_error(param,
"Can't delete key %d from record to be removed",i+1);
2996 info->dfile=old_file;
3000 if (sort_param->calc_checksum)
3001 param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
3003 error= info->rec_cache.flush() || (*info->s->delete_record)(info);
3004 info->dfile=old_file;
3005 info->state->records--;
3013 uint32_t nod_flag,length;
3014 my_off_t filepos,key_file_length;
3016 SORT_INFO *sort_info= sort_param->sort_info;
3017 myf myf_rw=sort_info->param->myf_rw;
3018 MI_INFO *info=sort_info->info;
3021 filepos= HA_OFFSET_ERROR;
3023 for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
3025 key_block->inited=0;
3026 length=mi_getint(key_block->buff);
3028 _mi_kpointer(info,key_block->end_pos,filepos);
3029 key_file_length=info->state->key_file_length;
3030 memset(key_block->buff+length, 0, keyinfo->block_length-length);
3031 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3035 if (key_file_length == info->state->key_file_length)
3037 if (_mi_write_keypage(info, keyinfo, filepos,
3038 DFLT_INIT_HITS, key_block->buff))
3041 else if (my_pwrite(info->s->kfile,(
unsigned char*) key_block->buff,
3042 (uint) keyinfo->block_length,filepos, myf_rw))
3046 info->s->state.key_root[sort_param->key]=filepos;
3053 uint32_t buffer_length)
3055 register uint32_t i;
3059 buffer_length+IO_SIZE)*blocks)))
3061 mi_check_print_error(param,
"Not enough memory for sort-key-blocks");
3064 for (i=0 ; i < blocks ; i++)
3067 block[i].buff=(
unsigned char*) (block+blocks)+(buffer_length+IO_SIZE)*i;
3075 int test_if_almost_full(
MI_INFO *info)
3077 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
3079 return (my_off_t)(lseek(info->s->kfile, 0L, SEEK_END) / 10 * 9) >
3080 (my_off_t) info->s->base.max_key_file_length ||
3081 (my_off_t)(lseek(info->dfile, 0L, SEEK_END) / 10 * 9) >
3082 (my_off_t) info->s->base.max_data_file_length;
3088 int write_data_suffix(
SORT_INFO *sort_info,
bool fix_datafile)
3090 MI_INFO *info=sort_info->info;
3092 if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
3094 unsigned char buff[MEMMAP_EXTRA_MARGIN];
3095 memset(buff, 0,
sizeof(buff));
3096 if (info->rec_cache.write(buff,
sizeof(buff)))
3098 mi_check_print_error(sort_info->param,
3099 "%d when writing to datafile",errno);
3102 sort_info->param->read_cache.end_of_file+=
sizeof(buff);
3113 if (update & UPDATE_OPEN_COUNT)
3115 share->state.open_count=0;
3116 share->global_changed=0;
3118 if (update & UPDATE_STAT)
3120 uint32_t i, key_parts= mi_uint2korr(share->state.header.key_parts);
3121 share->state.rec_per_key_rows=info->state->records;
3122 share->state.changed&= ~STATE_NOT_ANALYZED;
3123 if (info->state->records)
3125 for (i=0; i<key_parts; i++)
3127 if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
3128 share->state.changed|= STATE_NOT_ANALYZED;
3132 if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
3134 if (update & UPDATE_TIME)
3136 share->state.check_time= (long) time((time_t*) 0);
3137 if (!share->state.create_time)
3138 share->state.create_time=share->state.check_time;
3146 if (info->lock_type == F_WRLCK)
3147 share->state.state= *info->state;
3148 if (mi_state_info_write(share->kfile,&share->state,1+2))
3154 uint32_t r_locks=share->r_locks,w_locks=share->w_locks;
3155 share->r_locks= share->w_locks= share->tot_locks= 0;
3156 error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
3157 share->r_locks=r_locks;
3158 share->w_locks=w_locks;
3159 share->tot_locks=r_locks+w_locks;
3164 mi_check_print_error(param,
"%d when updating keyfile",errno);
3184 unsigned char *record= 0;
3186 if (!info->s->base.auto_key ||
3187 ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
3189 if (!(param->testflag & T_VERY_SILENT))
3190 mi_check_print_info(param,
3191 "Table: %s doesn't have an auto increment key\n",
3192 param->isam_file_name);
3195 if (!(param->testflag & T_SILENT) &&
3196 !(param->testflag & T_REP))
3197 printf(
"Updating MyISAM file: %s\n", param->isam_file_name);
3202 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
3204 mi_check_print_error(param,
"Not enough memory for extra record");
3208 mi_extra(info,HA_EXTRA_KEYREAD,0);
3209 if (mi_rlast(info, record, info->s->base.auto_key-1))
3211 if (errno != HA_ERR_END_OF_FILE)
3213 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3214 free(mi_get_rec_buff_ptr(info, record));
3215 mi_check_print_error(param,
"%d when reading last record",errno);
3219 info->s->state.auto_increment=param->auto_increment_value;
3223 uint64_t auto_increment= retrieve_auto_increment(info, record);
3224 set_if_bigger(info->s->state.auto_increment,auto_increment);
3226 set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
3228 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3229 free(mi_get_rec_buff_ptr(info, record));
3230 update_state_info(param, info, UPDATE_AUTO_INC);
3286 void update_key_parts(
MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
3287 uint64_t *unique, uint64_t *notnull,
3290 uint64_t count=0,tmp, unique_tuples;
3291 uint64_t tuples= records;
3293 for (parts=0 ; parts < keyinfo->keysegs ; parts++)
3295 count+=unique[parts];
3296 unique_tuples= count + 1;
3299 tuples= notnull[parts];
3305 unique_tuples -= (records - notnull[parts]);
3308 if (unique_tuples == 0)
3310 else if (count == 0)
3313 tmp= (tuples + unique_tuples/2) / unique_tuples;
3321 if (tmp >= (uint64_t) ~(ulong) 0)
3322 tmp=(uint64_t) ~(ulong) 0;
3324 *rec_per_key_part=(ulong) tmp;
3330 static ha_checksum mi_byte_checksum(
const unsigned char *buf, uint32_t length)
3333 const unsigned char *end=buf+length;
3334 for (crc=0; buf != end; buf++)
3335 crc=((crc << 1) + *((
unsigned char*) buf)) +
3336 test(crc & (((ha_checksum) 1) << (8*
sizeof(ha_checksum)-1)));
3340 static bool mi_too_big_key_for_sort(
MI_KEYDEF *key, ha_rows rows)
3342 uint32_t key_maxlength=key->maxlength;
3343 return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) &&
3344 ((uint64_t) rows * key_maxlength >
3345 (uint64_t) MAX_FILE_SIZE));
3357 void mi_disable_non_unique_index(
MI_INFO *info, ha_rows rows)
3363 assert(info->state->records == 0 &&
3364 (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
3365 for (i=0 ; i < share->base.keys ; i++,key++)
3367 if (!(key->flag & (HA_NOSAME | HA_AUTO_KEY)) &&
3368 ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
3370 mi_clear_key_active(share->state.key_map, i);
3371 info->update|= HA_STATE_CHANGED;
3383 bool mi_test_if_sort_rep(
MI_INFO *info, ha_rows rows,
3384 uint64_t key_map,
bool force)
3394 if (! mi_is_any_key_active(key_map))
3396 for (i=0 ; i < share->base.keys ; i++,key++)
3398 if (!force && mi_too_big_key_for_sort(key,rows))
3408 if ((sort_info->new_data_file_type=share->data_file_type) ==
3409 COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
3413 if (share->options & HA_OPTION_PACK_RECORD)
3414 sort_info->new_data_file_type = DYNAMIC_RECORD;
3416 sort_info->new_data_file_type = STATIC_RECORD;
3419 memcpy(&tmp, share,
sizeof(*share));
3420 tmp.options= ~HA_OPTION_COMPRESS_RECORD;
3421 mi_setup_functions(&tmp);
3422 share->delete_record=tmp.delete_record;
TODO: Rename this file - func.h is stupid.
int init_io_cache(int file, size_t cachesize, cache_type type, my_off_t seek_offset, bool use_async_io, myf cache_myflags)
Initialize an io_cache_st object.
int end_io_cache()
Free an io_cache_st object.
void free_root(myf MyFLAGS)
Deallocate everything used by alloc_root or just move used blocks to free list if called with MY_USED...
bool reinit_io_cache(cache_type type_arg, my_off_t seek_offset, bool use_async_io, bool clear_cache)
Reset the cache.