XZ Utils  5.2.2
Data Structures | Macros | Typedefs | Enumerations | Functions
stream_encoder_mt.c File Reference

Multithreaded .xz Stream encoder. More...

#include "filter_encoder.h"
#include "easy_preset.h"
#include "block_encoder.h"
#include "block_buffer_encoder.h"
#include "index_encoder.h"
#include "outqueue.h"

Data Structures

struct  worker_thread_s
 
struct  lzma_coder_s
 

Macros

#define BLOCK_SIZE_MAX   (UINT64_MAX / LZMA_THREADS_MAX)
 

Typedefs

typedef struct worker_thread_s worker_thread
 

Enumerations

enum  worker_state {
  THR_IDLE, THR_RUN, THR_FINISH, THR_STOP,
  THR_EXIT
}
 

Functions

static void worker_error (worker_thread *thr, lzma_ret ret)
 Tell the main thread that something has gone wrong. More...
 
static worker_state worker_encode (worker_thread *thr, worker_state state)
 
static MYTHREAD_RET_TYPE worker_start (void *thr_ptr)
 
static void threads_stop (lzma_coder *coder, bool wait_for_threads)
 Make the threads stop but not exit. Optionally wait for them to stop. More...
 
static void threads_end (lzma_coder *coder, const lzma_allocator *allocator)
 
static lzma_ret initialize_new_thread (lzma_coder *coder, const lzma_allocator *allocator)
 Initialize a new worker_thread structure and create a new thread. More...
 
static lzma_ret get_thread (lzma_coder *coder, const lzma_allocator *allocator)
 
static lzma_ret stream_encode_in (lzma_coder *coder, const lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, lzma_action action)
 
static bool wait_for_work (lzma_coder *coder, mythread_condtime *wait_abs, bool *has_blocked, bool has_input)
 
static lzma_ret stream_encode_mt (lzma_coder *coder, const lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action)
 
static void stream_encoder_mt_end (lzma_coder *coder, const lzma_allocator *allocator)
 
static lzma_ret get_options (const lzma_mt *options, lzma_options_easy *opt_easy, const lzma_filter **filters, uint64_t *block_size, uint64_t *outbuf_size_max)
 
static void get_progress (lzma_coder *coder, uint64_t *progress_in, uint64_t *progress_out)
 
static lzma_ret stream_encoder_mt_init (lzma_next_coder *next, const lzma_allocator *allocator, const lzma_mt *options)
 
lzma_ret lzma_stream_encoder_mt (lzma_stream *strm, const lzma_mt *options)
 Initialize multithreaded .xz Stream encoder. More...
 
uint64_t lzma_stream_encoder_mt_memusage (const lzma_mt *options)
 Calculate approximate memory usage of multithreaded .xz encoder. More...
 

Detailed Description

Multithreaded .xz Stream encoder.

Macro Definition Documentation

#define BLOCK_SIZE_MAX   (UINT64_MAX / LZMA_THREADS_MAX)

Maximum supported block size. This makes it simpler to prevent integer overflows if we are given unusually large block size.

Referenced by get_options().

Enumeration Type Documentation

Enumerator
THR_IDLE 

Waiting for work.

THR_RUN 

Encoding is in progress.

THR_FINISH 

Encoding is in progress but no more input data will be read.

THR_STOP 

The main thread wants the thread to stop whatever it was doing but not exit.

THR_EXIT 

The main thread wants the thread to exit. We could use cancellation but since there's stopped anyway, this is lazier.

Function Documentation

static void worker_error ( worker_thread thr,
lzma_ret  ret 
)
static
static void threads_stop ( lzma_coder coder,
bool  wait_for_threads 
)
static

Make the threads stop but not exit. Optionally wait for them to stop.

References THR_IDLE, THR_STOP, lzma_coder_s::threads, and lzma_coder_s::threads_initialized.

Referenced by get_options(), and wait_for_work().

static void threads_end ( lzma_coder coder,
const lzma_allocator allocator 
)
static

Stop the threads and free the resources associated with them. Wait until the threads have exited.

References lzma_free(), THR_EXIT, worker_thread_s::thread_id, lzma_coder_s::threads, and lzma_coder_s::threads_initialized.

Referenced by get_options(), and wait_for_work().

static lzma_ret initialize_new_thread ( lzma_coder coder,
const lzma_allocator allocator 
)
static
static bool wait_for_work ( lzma_coder coder,
mythread_condtime *  wait_abs,
bool *  has_blocked,
bool  has_input 
)
static
static lzma_ret get_options ( const lzma_mt options,
lzma_options_easy opt_easy,
const lzma_filter **  filters,
uint64_t *  block_size,
uint64_t *  outbuf_size_max 
)
static

Options handling for lzma_stream_encoder_mt_init() and lzma_stream_encoder_mt_memusage()

References lzma_mt::block_size, lzma_coder_s::block_size, BLOCK_SIZE_MAX, lzma_stream_flags::check, lzma_mt::check, lzma_next_coder_s::code, lzma_next_coder_s::coder, lzma_next_coder_s::end, lzma_options_easy::filters, filters, lzma_coder_s::filters, lzma_mt::filters, lzma_mt::flags, lzma_next_coder_s::get_progress, lzma_coder_s::header, lzma_coder_s::header_pos, lzma_filter::id, lzma_coder_s::index, lzma_coder_s::index_encoder, lzma_alloc(), lzma_block_buffer_bound64(), LZMA_CHECK_ID_MAX, lzma_check_is_supported(), lzma_easy_preset(), lzma_filters_copy(), lzma_free(), lzma_index_end(), lzma_index_init(), LZMA_MEM_ERROR, LZMA_NEXT_CODER_INIT, lzma_next_coder_init, LZMA_OK, LZMA_OPTIONS_ERROR, lzma_outq_init(), LZMA_PROG_ERROR, lzma_raw_encoder_memusage(), lzma_stream_header_encode(), LZMA_STREAM_HEADER_SIZE, LZMA_THREADS_MAX, LZMA_UNSUPPORTED_CHECK, LZMA_VLI_UNKNOWN, worker_thread_s::next, lzma_filter::options, lzma_coder_s::outq, lzma_mt::preset, worker_thread_s::progress_in, lzma_coder_s::progress_in, worker_thread_s::progress_out, lzma_coder_s::progress_out, return_if_error, lzma_coder_s::stream_flags, lzma_coder_s::thr, lzma_coder_s::thread_error, lzma_mt::threads, lzma_coder_s::threads, threads_end(), lzma_coder_s::threads_free, lzma_coder_s::threads_initialized, lzma_coder_s::threads_max, threads_stop(), lzma_mt::timeout, lzma_coder_s::timeout, and lzma_stream_flags::version.

Referenced by lzma_outq_init(), lzma_outq_memusage(), and lzma_stream_encoder_mt_memusage().

lzma_ret lzma_stream_encoder_mt ( lzma_stream strm,
const lzma_mt options 
)

Initialize multithreaded .xz Stream encoder.

This provides the functionality of lzma_easy_encoder() and lzma_stream_encoder() as a single function for multithreaded use.

The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be added in the future.

Parameters
strmPointer to properly prepared lzma_stream
optionsPointer to multithreaded compression options
Returns
- LZMA_OK
  • LZMA_MEM_ERROR
  • LZMA_UNSUPPORTED_CHECK
  • LZMA_OPTIONS_ERROR
  • LZMA_PROG_ERROR

References lzma_stream::internal, LZMA_FINISH, LZMA_FULL_BARRIER, LZMA_FULL_FLUSH, lzma_next_strm_init, LZMA_OK, LZMA_RUN, strm, and lzma_internal_s::supported_actions.

Referenced by coder_init().

uint64_t lzma_stream_encoder_mt_memusage ( const lzma_mt options)

Calculate approximate memory usage of multithreaded .xz encoder.

Since doing the encoding in threaded mode doesn't affect the memory requirements of single-threaded decompressor, you can use lzma_easy_decoder_memusage(options->preset) or lzma_raw_decoder_memusage(options->filters) to calculate the decompressor memory requirements.

Parameters
optionsCompression options
Returns
Number of bytes of memory required for encoding with the given options. If an error occurs, for example due to unsupported preset or filter chain, UINT64_MAX is returned.

References filters, get_options(), LZMA_MEMUSAGE_BASE, LZMA_OK, lzma_outq_memusage(), lzma_raw_encoder_memusage(), and lzma_mt::threads.

Referenced by coder_add_filter().