Base class containing the implementation details that are independent of the item type. More...

#include <tpie/compressed/stream.h>

Inherited by tpie::file_stream< element_type >, tpie::file_stream< item_type >, tpie::file_stream< tpie::blocks::block_handle >, tpie::file_stream< tpie::stream_position >, and tpie::file_stream< T >.

Classes
struct	seek_state

Public Types
typedef std::shared_ptr < compressor_buffer >	buffer_t

Public Member Functions
bool	is_readable () const throw ()

bool	is_writable () const throw ()

memory_size_type	block_items () const

memory_size_type	block_size () const

template<typename TT >
void	read_user_data (TT &data)

memory_size_type	read_user_data (void *data, memory_size_type count)

template<typename TT >
void	write_user_data (const TT &data)

void	write_user_data (const void *data, memory_size_type count)

memory_size_type	user_data_size () const

memory_size_type	max_user_data_size () const

const std::string &	path () const

void	open (const std::string &path, access_type accessType, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none)
	Deprecated interface for opening a named stream. More...

void	open (memory_size_type userDataSize, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none)
	Deprecated interface for opening an unnamed temporary stream. More...

void	open (temp_file &file, access_type accessType, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none)
	Deprecated interface for opening a temporary stream. More...

void	open (const std::string &path, compression_flags compressionFlags)
	Deprecated interface for opening a named stream. More...

void	open (compression_flags compressionFlags)
	Deprecated interface for opening an unnamed temporary stream. More...

void	open (temp_file &file, compression_flags compressionFlags)
	Deprecated interface for opening a temporary stream. More...

void	open (const std::string &path, open::type openFlags=open::defaults, memory_size_type userDataSize=0)
	Open and possibly create a stream. More...

void	open (open::type openFlags=open::defaults, memory_size_type userDataSize=0)
	Open and create an unnamed temporary stream. More...

void	open (temp_file &file, open::type openFlags=open::defaults, memory_size_type userDataSize=0)
	Open and possibly create a temporary stream. More...

void	close ()

bool	is_open () const

stream_size_type	size () const

stream_size_type	file_size () const

stream_size_type	offset () const

Static Public Member Functions
static memory_size_type	block_size (double blockFactor) throw ()

static double	calculate_block_factor (memory_size_type blockSize) throw ()

static memory_size_type	block_memory_usage (double blockFactor)

Protected Member Functions
	compressed_stream_base (memory_size_type itemSize, double blockFactor)

virtual void	flush_block (compressor_thread_lock &)=0

virtual void	post_open ()=0

void	open_inner (const std::string &path, open::type openFlags, memory_size_type userDataSize)

compressor_thread &	compressor ()

void	finish_requests (compressor_thread_lock &l)

stream_size_type	last_block_read_offset (compressor_thread_lock &l)
	Blocks to take the compressor lock. More...

stream_size_type	current_file_size (compressor_thread_lock &l)
	Blocks to take the compressor lock. More...

bool	use_compression ()

void	uncache_read_writes ()
	Reset cheap read/write counts to zero so that the next read/write operation will check stream state properly. More...

Protected Attributes
bool	m_bufferDirty
	Whether the current block must be written out to disk before being ejected. More...

memory_size_type	m_blockItems
	Number of items in a logical block. More...

memory_size_type	m_blockSize
	Size (in bytes) of a logical (uncompressed) block. More...

bool	m_canRead
	Whether we are open for reading. More...

bool	m_canWrite
	Whether we are open for writing. More...

bool	m_open
	Whether we are open. More...

memory_size_type	m_itemSize
	Size of a single item. More...

memory_size_type	m_cachedReads
	Number of cheap, unchecked reads we can do next. More...

memory_size_type	m_cachedWrites
	Number of cheap, unchecked writes we can do next. More...

tpie::unique_ptr< temp_file >	m_ownedTempFile
	The anonymous temporary file we have opened (when appropriate). More...

temp_file *	m_tempFile
	The temporary file we have opened (when appropriate). More...

file_accessor::byte_stream_accessor < default_raw_file_accessor >	m_byteStreamAccessor
	File accessor. More...

stream_size_type	m_size
	Number of logical items in the stream. More...

stream_buffers	m_buffers
	Buffer manager for this entire stream. More...

buffer_t	m_buffer
	Buffer holding the items of the block currently being read/written. More...

stream_size_type	m_streamBlocks
	The number of blocks written to the file. More...

stream_size_type	m_lastBlockReadOffset
	When use_compression() is true: Read offset of the last block in the stream. More...

stream_size_type	m_currentFileSize

compressor_response	m_response
	Response from compressor thread; protected by compressor thread mutex. More...

bool	m_updateReadOffsetFromWrite = false
	When use_compression() is true: Indicates whether m_response is the response to a write request. More...

stream_size_type	m_lastWriteBlockNumber

seek_state::type	m_seekState

stream_size_type	m_readOffset
	Position relating to the currently loaded buffer. More...

stream_size_type	m_offset
	Offset of next item to read/write, relative to beginning of stream. More...

stream_position	m_nextPosition
	If seekState is `position`, seek to this position before reading/writing. More...

stream_size_type	m_nextReadOffset

Detailed Description

Base class containing the implementation details that are independent of the item type.

Definition at line 125 of file stream.h.

Member Function Documentation

stream_size_type tpie::compressed_stream_base::current_file_size ( compressor_thread_lock & l )

protected

Blocks to take the compressor lock.

Precondition: use_compression()

TODO: Should probably investigate when this reports a useful value.

Referenced by tpie::file_stream< tpie::stream_position >::get_position().

stream_size_type tpie::compressed_stream_base::last_block_read_offset ( compressor_thread_lock & l )

protected

Blocks to take the compressor lock.

Precondition: use_compression()

TODO: Should probably investigate when this reports a useful value.

Referenced by tpie::file_stream< tpie::stream_position >::get_position().

void tpie::compressed_stream_base::open	(	const std::string &	path,
		access_type	accessType,
		memory_size_type	userDataSize = `0`,
		cache_hint	cacheHint = `access_sequential`,
		compression_flags	compressionFlags = `compression_none`
	)

inline

Deprecated interface for opening a named stream.

If compressionFlags is compression_none and the file does not already exist, no compression will be used when writing. If compressionFlags is compression_normal and the file does not already exist, compression will be used when writing. If the file already exists, the compression flags of the existing file are used instead.

Definition at line 204 of file stream.h.

Referenced by tpie::pipelining::bits::named_output_t< T >::begin(), open(), and tpie::pipelining::bits::named_pull_input_t< T >::propagate().

     {
         open(path, open::translate(accessType, cacheHint, compressionFlags), userDataSize);
     }

void tpie::compressed_stream_base::open	(	memory_size_type	userDataSize,
		cache_hint	cacheHint = `access_sequential`,
		compression_flags	compressionFlags = `compression_none`
	)

inline

Deprecated interface for opening an unnamed temporary stream.

Definition at line 216 of file stream.h.

References tpie::access_read_write, and open().

                                                                    {
         open(open::translate(access_read_write, cacheHint, compressionFlags), userDataSize);
     }

void tpie::compressed_stream_base::open	(	temp_file &	file,
		access_type	accessType,
		memory_size_type	userDataSize = `0`,
		cache_hint	cacheHint = `access_sequential`,
		compression_flags	compressionFlags = `compression_none`
	)

inline

Deprecated interface for opening a temporary stream.

Definition at line 225 of file stream.h.

References open().

                                                                    {
         open(file, open::translate(accessType, cacheHint, compressionFlags), userDataSize);
     }

void tpie::compressed_stream_base::open	(	const std::string &	path,
		compression_flags	compressionFlags
	)

inline

Deprecated interface for opening a named stream.

Definition at line 236 of file stream.h.

References tpie::access_read_write, tpie::access_sequential, and open().

                                                                           {
         const memory_size_type userDataSize = 0;
         open(path, open::translate(access_read_write, access_sequential, compressionFlags), userDataSize);
     }

void tpie::compressed_stream_base::open ( compression_flags compressionFlags )

inline

Deprecated interface for opening an unnamed temporary stream.

Definition at line 244 of file stream.h.

References tpie::access_read_write, tpie::access_sequential, and open().

                                                   {
         const memory_size_type userDataSize = 0;
         open(open::translate(access_read_write, access_sequential, compressionFlags), userDataSize);
     }

void tpie::compressed_stream_base::open	(	temp_file &	file,
		compression_flags	compressionFlags
	)

inline

Deprecated interface for opening a temporary stream.

Definition at line 252 of file stream.h.

References tpie::access_read_write, tpie::access_sequential, and open().

                                                                     {
         const memory_size_type userDataSize = 0;
         open(file, open::translate(access_read_write, access_sequential, compressionFlags), userDataSize);
     }

void tpie::compressed_stream_base::open	(	const std::string &	path,
		open::type	openFlags = `open::defaults`,
		memory_size_type	userDataSize = `0`
	)

Open and possibly create a stream.

The stream is created if it does not exist and opened for reading and writing, but this can be changed with open::read_only or open::write_only; see below.

The flags supplied to openFlags should be a combination of the following from open::type, OR'ed together:

open::read_only Open for reading only, and fail if the stream does not exist.

open::write_only Open for writing only, and truncate the stream if it exists.

open::access_normal By default, POSIX_FADV_SEQUENTIAL is passed to the open syscall to indicate that the OS should optimize for sequential access; this flag disables that flag.

open::access_random Pass POSIX_FADV_RANDOM to the open syscall to make the OS optimize for random access.

open::compression_normal Create the stream in compression mode if it does not already exist, and compress written blocks according to available resources (for instance CPU time and memory).

open::compression_all Create the stream in compression mode if it does not already exist, and compress all written blocks using the preferred compression scheme, which can be set using tpie::the_compressor_thread().set_preferred_compression().

Parameters

path	The path to the file to open
openFlags	A bit-wise combination of the flags; see above.
userDataSize	Required user data capacity in stream header.

void tpie::compressed_stream_base::open	(	open::type	openFlags = `open::defaults`,
		memory_size_type	userDataSize = `0`
	)

Open and create an unnamed temporary stream.

Parameters

openFlags	A bit-wise combination of the flags; see above.
userDataSize	Required user data capacity in stream header.

void tpie::compressed_stream_base::open	(	temp_file &	file,
		open::type	openFlags = `open::defaults`,
		memory_size_type	userDataSize = `0`
	)

Open and possibly create a temporary stream.

Parameters

file	The temporary file to open
openFlags	A bit-wise combination of the flags; see above.
userDataSize	Required user data capacity in stream header.

void tpie::compressed_stream_base::uncache_read_writes ( )

inlineprotected

Reset cheap read/write counts to zero so that the next read/write operation will check stream state properly.

Definition at line 345 of file stream.h.

References m_cachedReads, and m_cachedWrites.

Referenced by tpie::file_stream< tpie::stream_position >::seek(), tpie::file_stream< tpie::stream_position >::set_position(), and tpie::file_stream< tpie::stream_position >::truncate().

                                {
         m_cachedReads = m_cachedWrites = 0;
     }

Member Data Documentation

memory_size_type tpie::compressed_stream_base::m_blockItems

protected

Number of items in a logical block.

Definition at line 380 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::seek().

memory_size_type tpie::compressed_stream_base::m_blockSize

protected

Size (in bytes) of a logical (uncompressed) block.

Definition at line 382 of file stream.h.

buffer_t tpie::compressed_stream_base::m_buffer

protected

Buffer holding the items of the block currently being read/written.

Definition at line 407 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().

bool tpie::compressed_stream_base::m_bufferDirty

protected

Whether the current block must be written out to disk before being ejected.

Invariants: If m_bufferDirty is true and use_compression() is true, block_number() is either m_streamBlocks or m_streamBlocks - 1. If block_number() is m_streamBlocks, m_bufferDirty is true.

Definition at line 378 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::peek().

stream_buffers tpie::compressed_stream_base::m_buffers

protected

Buffer manager for this entire stream.

Definition at line 405 of file stream.h.

file_accessor::byte_stream_accessor<default_raw_file_accessor> tpie::compressed_stream_base::m_byteStreamAccessor

protected

File accessor.

Definition at line 401 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::describe().

memory_size_type tpie::compressed_stream_base::m_cachedReads

protected

Number of cheap, unchecked reads we can do next.

Definition at line 392 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::can_read(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::read(), and uncache_read_writes().

memory_size_type tpie::compressed_stream_base::m_cachedWrites

protected

Number of cheap, unchecked writes we can do next.

Definition at line 394 of file stream.h.

Referenced by uncache_read_writes().

bool tpie::compressed_stream_base::m_canRead

protected

Whether we are open for reading.

Definition at line 384 of file stream.h.

bool tpie::compressed_stream_base::m_canWrite

protected

Whether we are open for writing.

Definition at line 386 of file stream.h.

memory_size_type tpie::compressed_stream_base::m_itemSize

protected

Size of a single item.

itemSize * blockItems == blockSize.

Definition at line 390 of file stream.h.

stream_size_type tpie::compressed_stream_base::m_lastBlockReadOffset

protected

When use_compression() is true: Read offset of the last block in the stream.

Necessary to support seeking to the end.

Definition at line 416 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::describe().

stream_position tpie::compressed_stream_base::m_nextPosition

protected

If seekState is position, seek to this position before reading/writing.

Definition at line 456 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::set_position().

stream_size_type tpie::compressed_stream_base::m_offset

protected

Offset of next item to read/write, relative to beginning of stream.

Invariants:

block_number() in [0, m_streamBlocks] offset in [0, size] block_item_index() in [0, m_blockSize) offset == block_number() * m_blockItems + block_item_index()

block_item_index() <= offset.

If block_number() == m_streamBlocks, we are in a block that has not yet been written to disk.

Definition at line 453 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::read(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().

bool tpie::compressed_stream_base::m_open

protected

Whether we are open.

Definition at line 388 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::can_read(), and tpie::file_stream< tpie::stream_position >::can_read_back().

tpie::unique_ptr<temp_file> tpie::compressed_stream_base::m_ownedTempFile

protected

The anonymous temporary file we have opened (when appropriate).

Definition at line 396 of file stream.h.

stream_size_type tpie::compressed_stream_base::m_readOffset

protected

Position relating to the currently loaded buffer.

readOffset is only valid during reading. Invariants:

If use_compression() == false, readOffset == 0. If offset == 0, then readOffset == block_item_index() == block_number() == 0.

Definition at line 438 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().

compressor_response tpie::compressed_stream_base::m_response

protected

Response from compressor thread; protected by compressor thread mutex.

Definition at line 420 of file stream.h.

stream_size_type tpie::compressed_stream_base::m_size

protected

Number of logical items in the stream.

Definition at line 403 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::peek(), and tpie::file_stream< tpie::stream_position >::truncate().

stream_size_type tpie::compressed_stream_base::m_streamBlocks

protected

The number of blocks written to the file.

We must always have (m_streamBlocks+1) * m_blockItems <= m_size.

Definition at line 411 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::seek().

temp_file* tpie::compressed_stream_base::m_tempFile

protected

The temporary file we have opened (when appropriate).

When m_ownedTempFile.get() != 0, m_tempFile == m_ownedTempFile.get().

Definition at line 399 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::truncate().

bool tpie::compressed_stream_base::m_updateReadOffsetFromWrite = false

protected

When use_compression() is true: Indicates whether m_response is the response to a write request.

Used for knowing where to read next in read/read_back.

Definition at line 426 of file stream.h.

Referenced by tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().

The documentation for this class was generated from the following file:

compressed/stream.h

TPIE

Classes

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Member Function Documentation

Member Data Documentation