tpie::file_stream_base Class Reference

Inherits tpie::file_base_crtp< child_t >, and tpie::stream_crtp< file_stream_base >.

Inherited by tpie::uncompressed_stream< T >.

Classes
struct	block_t

Public Types
typedef file_base_crtp< file_stream_base >	p_t
typedef stream_crtp< file_stream_base >	s_t
enum	offset_type
	Type describing how we should interpret the offset supplied to seek. More...

Public Member Functions
void	close ()
	Close the file and release resources. More...
void	truncate (stream_size_type size)
	Truncate file to given size. More...
bool	is_readable () const throw ()
	Check if we can read from the file. More...
bool	is_writable () const throw ()
	Check if we can write to the file. More...
memory_size_type	block_size () const
	Get the size of a block in bytes. More...
memory_size_type	block_items () const
	Get the number of items per block. More...
template<typename TT >
void	read_user_data (TT &data)
	Read the user data associated with the file. More...
memory_size_type	read_user_data (void *data, memory_size_type count)
	Read variable length user data associated with the file. More...
template<typename TT >
void	write_user_data (const TT &data)
	Write user data to the stream. More...
void	write_user_data (const void *data, memory_size_type count)
	Write variable length user data associated with the file. More...
memory_size_type	user_data_size () const
	Get current user data size. More...
memory_size_type	max_user_data_size () const
	Get maximum user data size. More...
const std::string &	path () const throw ()
	The path of the file opened or the empty string. More...
void	open (const std::string &path, access_type accessType=access_read_write, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential)
	Open a file. More...
void	open (memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential)
	Open an anonymous temporary file. More...
void	open (temp_file &file, access_type accessType=access_read_write, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential)
	Open a temporary file. More...
bool	is_open () const
	Check if file is open. More...
stream_size_type	file_size () const throw ()
	Get the size of the file measured in items. More...
void	seek (stream_offset_type offset, offset_type whence=beginning)
	Moves the logical offset in the stream. More...
stream_size_type	offset () const throw ()
	Calculate the current offset in the stream. More...
bool	can_read () const throw ()
	Check if we can read an item with read(). More...
bool	can_read_back () const throw ()
	Check if we can read an item with read_back(). More...
stream_size_type	size () const throw ()
	Get the size of the file measured in items. More...

Static Public Member Functions
static memory_size_type	block_size (double blockFactor) throw ()
	Calculate the block size in bytes used by a stream. More...
static double	calculate_block_factor (memory_size_type blockSize) throw ()
	Find the block factor that would result in the given block size measured in bytes. More...
static memory_size_type	block_memory_usage (double blockFactor)
	Amount of memory used by a single block given the block factor. More...

Protected Member Functions
	file_stream_base (memory_size_type itemSize, double blockFactor, file_accessor::file_accessor *fileAccessor)
void	swap (file_stream_base &other)
void	open_inner (const std::string &path, access_type accessType, memory_size_type userDataSize, cache_hint cacheHint)
void	get_block (stream_size_type block)
	Use file_accessor to fetch indicated block number into m_block. More...
void	flush_block ()
	Write block to disk. More...
void	update_vars ()
void	initialize ()
void	write_update ()
template<typename BT >
void	read_block (BT &b, stream_size_type block)
void	get_block_check (stream_size_type block)
void	update_block ()
	Fetch block from disk as indicated by m_nextBlock, writing old block to disk if needed. More...

Static Protected Member Functions
static void	read_array (Stream &stream, const IT &start, const IT &end)
	Reads several items from the stream. More...
static void	write_array (Stream &stream, const IT &start, const IT &end)
	Write several items to the stream. More...

Protected Attributes
block_t	m_block
memory_size_type	m_blockItems
memory_size_type	m_blockSize
bool	m_canRead
bool	m_canWrite
bool	m_open
memory_size_type	m_itemSize
file_accessor::file_accessor *	m_fileAccessor
tpie::unique_ptr< temp_file >	m_ownedTempFile
temp_file *	m_tempFile
stream_size_type	m_size
memory_size_type	m_index
	Item index into the current block, or maxint if we don't have a block. More...
stream_size_type	m_nextBlock
	After a cross-block seek: Block index of next block, or maxint if the current block is good enough OR if we haven't read/written anything yet. More...
memory_size_type	m_nextIndex
	After a cross-block seek: Item index into next block. More...
stream_size_type	m_blockStartIndex
	The file-level item index of the first item in the current block. More...

Friends
class	file_base_crtp< file_stream_base >
class	stream_crtp< file_stream_base >

Detailed Description

Definition at line 33 of file file_stream_base.h.

Member Enumeration Documentation

offset_type

enum tpie::stream_crtp::offset_type

inherited

Type describing how we should interpret the offset supplied to seek.

Definition at line 39 of file stream_crtp.h.

                     {
        beginning,
        end,
        current
    };

Member Function Documentation

block_items()

template<typename child_t >

memory_size_type tpie::file_base_crtp< child_t >::block_items ( ) const

inlineinherited

Get the number of items per block.

Definition at line 106 of file file_base_crtp.h.

                                         {
        return m_blockItems;
    }

block_memory_usage()

template<typename child_t >

static memory_size_type tpie::file_base_crtp< child_t >::block_memory_usage ( double  blockFactor )

inlinestaticinherited

Amount of memory used by a single block given the block factor.

Definition at line 99 of file file_base_crtp.h.

                                                                          {
        return block_size(blockFactor);
    }

Referenced by tpie::uncompressed_stream< T >::memory_usage().

block_size() [1/2]

template<typename child_t >

memory_size_type tpie::file_base_crtp< child_t >::block_size ( ) const

inlineinherited

Get the size of a block in bytes.

Definition at line 113 of file file_base_crtp.h.

                                        {
        return m_blockSize;
    }

Referenced by tpie::file_base_crtp< file_stream_base >::block_memory_usage(), and tpie::file_base_crtp< file_stream_base >::calculate_block_factor().

block_size() [2/2]

template<typename child_t >

static memory_size_type tpie::file_base_crtp< child_t >::block_size ( double  blockFactor ) throw ( )

inlinestaticinherited

Calculate the block size in bytes used by a stream.

We have block_size(calculate_block_factor(b)) ~= b.

Parameters

blockFactor

Factor of the global block size to use.

Returns

Size in bytes.

Definition at line 79 of file file_base_crtp.h.

                                                                           {
        return static_cast<memory_size_type>(get_block_size() * blockFactor);
    }

calculate_block_factor()

template<typename child_t >

static double tpie::file_base_crtp< child_t >::calculate_block_factor ( memory_size_type  blockSize ) throw ( )

inlinestaticinherited

Find the block factor that would result in the given block size measured in bytes.

We have calculate_block_factor(block_size(f)) ~= f.

Parameters

blockSize

The sought block size.

Returns

The block factor needed to achieve this block size.

Definition at line 92 of file file_base_crtp.h.

                                                                                     {
        return (double)blockSize / (double)block_size(1.0);
    }

can_read()

bool tpie::stream_crtp< file_stream_base >::can_read throw ( )

inlineinherited

Check if we can read an item with read().

This is logically equivalent to:

return offset() < size();

but it might be faster.

Returns

Whether or not we can read more items from the stream.

Definition at line 109 of file stream_crtp.h.

                                         {
        assert(self().get_file().is_open());
        if (m_index < self().get_block().size ) return true;
        return offset() < self().size();
    }

can_read_back()

bool tpie::stream_crtp< file_stream_base >::can_read_back throw ( )

inlineinherited

Check if we can read an item with read_back().

Returns

Whether or not we can read an item with read_back().

Definition at line 120 of file stream_crtp.h.

                                              {
        assert(self().get_file().is_open());
        if (m_nextBlock == std::numeric_limits<stream_size_type>::max())
            return m_index > 0 || m_blockStartIndex > 0;
        else
            return m_nextIndex > 0 || m_nextBlock > 0;
    }

close()

void tpie::file_stream_base::close ( )

inline

Close the file and release resources.

This will close the file and resources used by buffers and such.

Definition at line 52 of file file_stream_base.h.

                        {
        if (m_open) flush_block();
        tpie_delete_array(m_block.data, m_itemSize * m_blockItems);
        m_block.data = 0;
        p_t::close();
    }

References tpie::tpie_delete_array().

file_size()

template<typename child_t >

stream_size_type tpie::file_base_crtp< child_t >::file_size ( ) const throw ( )

inlineinherited

Get the size of the file measured in items.

If there are streams of this file that have extended the stream length but have not yet flushed these writes, we might report an incorrect size.

Returns

The number of items in the file.

Definition at line 269 of file file_base_crtp.h.

                                                      {
        return m_size;
    }

flush_block()

void tpie::file_stream_base::flush_block ( )

inlineprotected

Write block to disk.

Definition at line 140 of file file_stream_base.h.

                              {
        if (m_block.dirty) {
            assert(m_canWrite);
            update_vars();
            m_fileAccessor->write_block(m_block.data, m_block.number, m_block.size);
            if (m_tempFile)
                m_tempFile->update_recorded_size(m_fileAccessor->byte_size());
        }
        m_block.dirty = false;
    }

get_block()

void tpie::file_stream_base::get_block ( stream_size_type  block )

protected

Use file_accessor to fetch indicated block number into m_block.

is_open()

template<typename child_t >

bool tpie::file_base_crtp< child_t >::is_open ( ) const

inlineinherited

Check if file is open.

Definition at line 257 of file file_base_crtp.h.

                                {
        return m_open;
    }

is_readable()

template<typename child_t >

bool tpie::file_base_crtp< child_t >::is_readable ( ) const throw ( )

inlineinherited

Check if we can read from the file.

Returns

True if we can read from the file.

Definition at line 57 of file file_base_crtp.h.

                                     {
        return m_canRead;
    }

is_writable()

template<typename child_t >

bool tpie::file_base_crtp< child_t >::is_writable ( ) const throw ( )

inlineinherited

Check if we can write to the file.

Returns

True if we can write to the file.

Definition at line 66 of file file_base_crtp.h.

                                     {
        return m_canWrite;
    }

Referenced by tpie::uncompressed_stream< T >::write().

max_user_data_size()

template<typename child_t >

memory_size_type tpie::file_base_crtp< child_t >::max_user_data_size ( ) const

inlineinherited

Get maximum user data size.

Definition at line 184 of file file_base_crtp.h.

                                                {
        assert(m_open);
        return m_fileAccessor->max_user_data_size();
    }

Referenced by tpie::file_base_crtp< file_stream_base >::write_user_data().

offset()

stream_size_type tpie::stream_crtp< file_stream_base >::offset throw ( )

inlineinherited

Calculate the current offset in the stream.

Returns

The current offset in the stream

Definition at line 91 of file stream_crtp.h.

                                                   {
        assert(self().get_file().is_open());
        if (m_nextBlock == std::numeric_limits<stream_size_type>::max())
            return m_index + m_blockStartIndex;
        return m_nextIndex + m_nextBlock * self().get_file().block_items();
    }

open() [1/3]

template<typename child_t >

void tpie::file_base_crtp< child_t >::open ( const std::string &  path, access_type  accessType = access_read_write, memory_size_type  userDataSize = 0, cache_hint  cacheHint = access_sequential  )

inlineinherited

Open a file.

Parameters

path	The path of the file to open.
accessType	The mode of operation.
userDataSize	The size of the user data we want to store in the file.

Definition at line 208 of file file_base_crtp.h.

                                                             {
        self().close();
        self().open_inner(path, accessType, userDataSize, cacheHint);
    }

open() [2/3]

template<typename child_t >

void tpie::file_base_crtp< child_t >::open ( memory_size_type  userDataSize = 0, cache_hint  cacheHint = access_sequential  )

inlineinherited

Open an anonymous temporary file.

The temporary file is deleted when this file is closed.

Definition at line 220 of file file_base_crtp.h.

                                                             {
        self().close();
        m_ownedTempFile.reset(tpie_new<temp_file>());
        m_tempFile=m_ownedTempFile.get();
        self().open_inner(m_tempFile->path(), access_read_write, userDataSize, cacheHint);
    }

open() [3/3]

template<typename child_t >

void tpie::file_base_crtp< child_t >::open ( temp_file &  file, access_type  accessType = access_read_write, memory_size_type  userDataSize = 0, cache_hint  cacheHint = access_sequential  )

inlineinherited

Open a temporary file.

The temporary file is not deleted when this file is closed, so several tpie::file objects may use the same temporary file consecutively.

Definition at line 233 of file file_base_crtp.h.

                                                             {
        self().close();
        m_tempFile=&file;
        self().open_inner(m_tempFile->path(), accessType, userDataSize, cacheHint);
    }

path()

template<typename child_t >

const std::string& tpie::file_base_crtp< child_t >::path ( ) const throw ( )

inlineinherited

The path of the file opened or the empty string.

Returns

The path of the currently opened file.

Definition at line 195 of file file_base_crtp.h.

                                                  {
        assert(m_open);
        return m_fileAccessor->path();
    }

Referenced by tpie::pipelining::serialization_bits::reverser_input_t< T >::begin(), tpie::pipelining::serialization_bits::buffer_input_t< T >::begin(), and tpie::file_base_crtp< file_stream_base >::open().

read_array()

static void tpie::stream_crtp< file_stream_base >::read_array ( Stream &  stream, const IT &  start, const IT &  end  )

inlinestaticprotectedinherited

Reads several items from the stream.

Implementation note: If your iterator type is efficiently copyable with std::copy, then this will also read efficiently from the internal TPIE buffer.

Template Parameters

IT	The type of Random Access Iterators used to supply the items.

Parameters

start	Iterator to the first spot to write to.
end	Iterator past the last spot to write to.

Exceptions

end_of_stream_exception

If there are not enough elements in the stream to fill all the spots between start and end.

Definition at line 166 of file stream_crtp.h.

                                                                                     {
        typedef typename Stream::item_type T;
        IT i = start;
        while (i != end) {
            if (stream.m_index >= stream.block_items()) {
                // check to make sure we have enough items in the stream
                stream_size_type offs = stream.offset();
                if (offs >= stream.size()
                    || offs + (end-i) > stream.size()) {
 
                    throw end_of_stream_exception();
                }
 
                // fetch next block from disk
                stream.update_block();
            }
 
            T * src = reinterpret_cast<T*>(stream.get_block().data) + stream.m_index;
 
            // either read the rest of the block or until `end'
            memory_size_type count = std::min(stream.block_items()-stream.m_index, static_cast<memory_size_type>(end-i));
 
            std::copy(src, src + count, i);
 
            // advance output iterator
            i += count;
 
            // advance input position
            stream.m_index += count;
        }
    }

read_user_data() [1/2]

template<typename child_t >

template<typename TT >

void tpie::file_base_crtp< child_t >::read_user_data ( TT &  data )

inlineinherited

Read the user data associated with the file.

Parameters

data	Where to store the user data.

Template Parameters

TT	The type of user data. sizeof(TT) must be less than or equal to the maximum user data size of the stream. TT must be trivially copyable.

Definition at line 126 of file file_base_crtp.h.

                                   {
        assert(m_open);
        if (sizeof(TT) != user_data_size()) throw io_exception("Wrong user data size");
        m_fileAccessor->read_user_data(reinterpret_cast<void*>(&data), sizeof(TT));
    }

read_user_data() [2/2]

template<typename child_t >

memory_size_type tpie::file_base_crtp< child_t >::read_user_data ( void *  data, memory_size_type  count  )

inlineinherited

Read variable length user data associated with the file.

Parameters

data	The buffer in which to write data.
count	The size of the buffer.

Returns

Number of bytes of user data actually read.

Definition at line 139 of file file_base_crtp.h.

                                                                         {
        assert(m_open);
        return m_fileAccessor->read_user_data(data, count);
    }

seek()

void tpie::stream_crtp< file_stream_base >::seek ( stream_offset_type  offset, offset_type  whence = beginning  )

inlineinherited

Moves the logical offset in the stream.

Parameters

offset	Where to move the logical offset to.
whence	Move the offset relative to what.

Definition at line 51 of file stream_crtp.h.

                                                                              {
        assert(self().get_file().is_open());
        if (whence == end)
            offset += self().size();
        else if (whence == current) {
            // are we seeking into the current block?
            if (offset >= 0 || static_cast<stream_size_type>(-offset) <= m_index) {
                stream_size_type new_index = static_cast<stream_offset_type>(offset+m_index);
 
                if (new_index < self().get_file().block_items()) {
                    self().update_vars();
                    m_index = static_cast<memory_size_type>(new_index);
                    return;
                }
            }
 
            offset += self().offset();
        }
        if (0 > offset || (stream_size_type)offset > self().size())
            throw io_exception("Tried to seek out of file");
        self().update_vars();
        stream_size_type b = static_cast<stream_size_type>(offset) / self().get_file().block_items();
        m_index = static_cast<memory_size_type>(offset - b* self().get_file().block_items());
        if (b == self().get_block().number) {
            m_nextBlock = std::numeric_limits<stream_size_type>::max();
            m_nextIndex = std::numeric_limits<memory_size_type>::max();
            assert(self().offset() == (stream_size_type)offset);
            return;
        }
        m_nextBlock = b;
        m_nextIndex = m_index;
        m_index = std::numeric_limits<memory_size_type>::max();
        assert(self().offset() == (stream_size_type)offset);
    }

size()

stream_size_type tpie::stream_crtp< file_stream_base >::size throw ( )

inlineinherited

Get the size of the file measured in items.

Returns

The number of items in the file.

Definition at line 133 of file stream_crtp.h.

                                                 {
        // XXX update_vars changes internal state in a way that is not visible
        // through the class interface.
        // therefore, a const_cast is warranted.
        const_cast<child_t&>(self()).update_vars();
        return self().get_file().file_size();
    }

truncate()

void tpie::file_stream_base::truncate ( stream_size_type  size )

inline

Truncate file to given size.

May only be used when no streams are opened to this file.

See also

file_base::truncate()

Note that when using a file_stream the stream will automatically be rewound if it is beyond the new end of the file.

Definition at line 67 of file file_stream_base.h.

                                                {
        stream_size_type o=offset();
        flush_block();
        m_block.number = std::numeric_limits<stream_size_type>::max();
        m_nextBlock = std::numeric_limits<stream_size_type>::max();
        m_nextIndex = std::numeric_limits<memory_size_type>::max();
        m_index = std::numeric_limits<memory_size_type>::max();
        m_size = size;
        m_fileAccessor->truncate(size);
        if (m_tempFile)
            m_tempFile->update_recorded_size(m_fileAccessor->byte_size());
        seek(std::min(o, size));
    }

update_block()

void tpie::stream_crtp< file_stream_base >::update_block

protectedinherited

Fetch block from disk as indicated by m_nextBlock, writing old block to disk if needed.

Update m_block, m_index, m_nextBlock and m_nextIndex. If m_nextBlock is maxint, use next block is the one numbered m_block->number+1. m_index is updated with the value of m_nextIndex.

user_data_size()

template<typename child_t >

memory_size_type tpie::file_base_crtp< child_t >::user_data_size ( ) const

inlineinherited

Get current user data size.

Definition at line 176 of file file_base_crtp.h.

                                            {
        assert(m_open);
        return m_fileAccessor->user_data_size();
    }

Referenced by tpie::file_base_crtp< file_stream_base >::read_user_data().

write_array()

static void tpie::stream_crtp< file_stream_base >::write_array ( Stream &  stream, const IT &  start, const IT &  end  )

inlinestaticprotectedinherited

Write several items to the stream.

Implementation note: If your iterator type is efficiently copyable with std::copy, then this will also write efficiently into the internal TPIE buffer.

Template Parameters

IT	The type of Random Access Iterators used to supply the items.

Parameters

start	Iterator to the first item to write.
end	Iterator past the last item to write.

Definition at line 212 of file stream_crtp.h.

                                                                                      {
        typedef typename Stream::item_type T;
        IT i = start;
        while (i != end) {
            if (stream.m_index >= stream.block_items()) stream.update_block();
 
            size_t streamRemaining = end - i;
            size_t blockRemaining = stream.block_items()-stream.m_index;
 
            IT till = (blockRemaining < streamRemaining) ? (i + blockRemaining) : end;
 
            T * dest = reinterpret_cast<T*>(stream.get_block().data) + stream.m_index;
 
            std::copy(i, till, dest);
 
            stream.m_index += till - i;
            stream.write_update();
            i = till;
        }
    }

write_user_data() [1/2]

template<typename child_t >

template<typename TT >

void tpie::file_base_crtp< child_t >::write_user_data ( const TT &  data )

inlineinherited

Write user data to the stream.

Parameters

data	The user data to store in the stream.

Template Parameters

TT	The type of user data. sizeof(TT) must be less than or equal to the maximum user data size of the stream. TT must be trivially copyable.

Definition at line 153 of file file_base_crtp.h.

                                          {
        assert(m_open);
        if (sizeof(TT) > max_user_data_size()) throw io_exception("Wrong user data size");
        m_fileAccessor->write_user_data(reinterpret_cast<const void*>(&data), sizeof(TT));
    }

write_user_data() [2/2]

template<typename child_t >

void tpie::file_base_crtp< child_t >::write_user_data ( const void *  data, memory_size_type  count  )

inlineinherited

Write variable length user data associated with the file.

Throws a stream_exception if the size of the user data exceeds the maximum user data size of the stream.

Parameters

data	The buffer from which to read data.
count	The size of the user data.

Definition at line 168 of file file_base_crtp.h.

                                                                    {
        assert(m_open);
        m_fileAccessor->write_user_data(data, count);
    }

Member Data Documentation

m_blockStartIndex

stream_size_type tpie::stream_crtp< file_stream_base >::m_blockStartIndex

protectedinherited

The file-level item index of the first item in the current block.

When m_block is not the null block, this should be equal to m_block->number * block_items().

Definition at line 256 of file stream_crtp.h.

m_index

memory_size_type tpie::stream_crtp< file_stream_base >::m_index

protectedinherited

Item index into the current block, or maxint if we don't have a block.

Definition at line 245 of file stream_crtp.h.

m_nextBlock

stream_size_type tpie::stream_crtp< file_stream_base >::m_nextBlock

protectedinherited

After a cross-block seek: Block index of next block, or maxint if the current block is good enough OR if we haven't read/written anything yet.

Definition at line 249 of file stream_crtp.h.

m_nextIndex

memory_size_type tpie::stream_crtp< file_stream_base >::m_nextIndex

protectedinherited

After a cross-block seek: Item index into next block.

Otherwise, maxint as with m_nextBlock.

Definition at line 252 of file stream_crtp.h.

---

The documentation for this class was generated from the following file:

• file_stream_base.h